Ignore:
Timestamp:
Apr 28, 2012, 12:36:51 AM (7 years ago)
Author:
bruno
Message:
  • Fix Plugin.pm for caspar structure management (don't use tables but hashes)
  • more code in cb still not ready.
File:
1 edited

Legend:

Unmodified
Added
Removed
  • projects/casparbuster/devel/bin/cb

    r1485 r1487  
    8585#use Cwd 'realpath';
    8686use File::Find;
    87 use File::Copy;
    88 use File::Basename;
    89 use File::Path;
    90 use File::Glob ':glob';
     87use Archive::Tar;
    9188use Getopt::Long;
    9289use Pod::Usage;
    9390use Data::Dumper;
    94 use List::Util qw(first);
    9591use ProjectBuilder::Base;
    9692use ProjectBuilder::Conf;
     
    164160pb_log(2,"%cb: ",Dumper($cb));
    165161
    166 # Check for mandatory params
     162die "No machine should be given when using machine structure" if (($cb->{'usemachines'}->{$appname} !~ /true/) && (defined $machine));
    167163
    168164if (defined $plugin) {
     
    174170eval { $basedir =~ s/(\$ENV.+\})/$1/eeg };
    175171
     172# Create basedir if it doesn't exist
     173die "Unable to find base directory at $basedir" if (not -d $basedir);
     174
    176175pb_log(1, "DEBUG MODE, not doing anything, just printing\nDEBUG: basedir = $basedir\n");
     176
     177# Create database if not existing and give a handler
     178my $db = "$basedir/$cb->{'database'}->{$appname}";
     179
     180my $precmd = "";
     181if (! -f $db) {
     182    $precmd = "CREATE TABLE dates (id INTEGER PRIMARY KEY AUTOINCREMENT, date INTEGER, file VARCHAR[65535], machine VARCHAR[65535])";
     183}
     184
     185my $dbh = DBI->connect("dbi:SQLite:dbname=$db","","",
     186            { RaiseError => 1, AutoCommit => 1 })
     187            || die "Unable to connect to $db";
     188
     189if ($precmd ne "") {
     190    my $sth = $dbh->prepare(qq{$precmd}) || die "Unable to create table into $db";
     191    if ($debug) {
     192        pb_log(1,"DEBUG: Creating DB $db\n");
     193        pb_log(1,"DEBUG: with command $precmd\n");
     194    } else {
     195        $sth->execute();
     196    }
     197    $sth->finish();
     198}
     199
     200# Define destination dir and populate with a VCS export
     201my $dest = "$ENV{'TMPDIR'}/vcs.$$";
     202my $scheme = $cb->{'cms'}->{$appname};
     203pb_vcs_export(pb_vcs_get_uri($scheme,$basedir),$basedir,$dest);
     204
     205# Now distribute to the right machines
     206if defined ($machine) {
     207    cb_distribute($machine);
     208} else {
     209    if ($cb->{'usemachines'}->{$appname} =~ /true/) {
     210        # First dir level is the machine, then the content
     211        opendir(DIR,$dest) || die "Unable to open $dest: $!";
     212        foreach my $m (readdir(DIR)) {
     213            next if ($m =~ /^\./);
     214            next if (! -d $m);
     215            # Machine name
     216            cb_distribute($m);
     217        }
     218        closedir(DIR);
     219    } else {
     220        cb_distribute(undef);
     221    }
     222}
     223
     224# Cleanup
     225pb_rm_rf($dest);
     226$dbh->disconnect;
     227
     228# End of Main
     229
     230# Distribute files to target machines
     231sub cb_distribute {
     232
     233my $machine = shift;
     234
     235pb_log(2,"Entering into cb_distribute with machine $machine\n");
    177236
    178237# Use potentially a remote account if defined
     
    183242pb_log(1, "DEBUG: remote = $remote\n") if (defined $remote);
    184243
    185 # Create basedir if it doesn't exist
    186 die "Unable to find base directory at $basedir" if (not -d $basedir);
    187 
    188244# Now handle plugins if any
    189245my $cbp = ();
    190246
     247# Handle plugins
    191248if (defined $plugin) {
    192249    foreach my $p (split(/,/,$plugin)) {   
    193250        pb_log(1,"Getting context for plugin $p\n");
    194251        $cbp = cb_plugin_get($p,$cbp);
    195         pb_log(2,"cbp: ".Dumper($cbp)."\n");
    196     }
    197 }
    198 
    199 my $db = "$cb->{'basedir'}/$cb->{'database'}";
    200 
    201 my $precmd = "";
    202 if (! -f $db) {
    203     $precmd = "CREATE TABLE dates (id INTEGER PRIMARY KEY AUTOINCREMENT, date DATE, file VARCHAR[65535])";
    204 }
    205 
    206 my $dbh = DBI->connect("dbi:SQLite:dbname=$db","","",
    207             { RaiseError => 1, AutoCommit => 1 })
    208             || die "Unable to connect to $db";
    209 
    210 if ($precmd ne "") {
    211     my $sth = $dbh->prepare(qq{$precmd}) || die "Unable to create table into $db";
    212     if ($debug) {
    213             pb_log(1,"DEBUG: Creatung DB $db\n");
    214             pb_log(1,"DEBUG: with command $precmd\n");
     252        # Adds mtime info to the plugin structure
     253        foreach my $type ('files','dirs') {
     254            foreach my $f (keys %{$cbp->{$p}->{$type}}) {
     255                my $tdir = $dest;
     256                if (defined $machine) {
     257                    $tdir = "$dest/$machine";
     258                }
     259                if (-r "$tdir/$f") {
     260                    my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime,$mtime,$ctime,$blksize,$blocks) = stat("$tdir/$f") || die "Unable to stat $tdir: $!";
     261                    $cbp->{$p}->{$type}->{$f}->{'mtime'} = $mtime;
     262                } else {
     263                    pb_log(0,"WARNING: Unable to read $tdir/$f from plugin $p\n");
     264                }   
     265            }
     266        }
     267    }
     268}
     269
     270# Handle this source
     271if (defined $source) {
     272    my $fullsource = "$source";
     273    $fullsource = "$machine/$source" if (defined $machine);
     274    pb_log(2,"fullsource is $fullsource\n");
     275    my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime,$mtime,$ctime,$blksize,$blocks) = stat("$dest/$fullsource") || die "Unable to stat $fullsource: $!";
     276    my $type = 'files';
     277    if (-d $fullsource) {
     278        $type = 'dirs';
     279    }
     280    die "ERROR: Only able to handle files or dirs with option --source\n" if ((! -d $fullsource) && (! -f $fullsource));
     281
     282    $cbp->{"cb.source"}->{$type}->{$source}->{'uid'} = $uid;
     283    $cbp->{"cb.source"}->{$type}->{$source}->{'gid'} = $gid;
     284    $cbp->{"cb.source"}->{$type}->{$source}->{'mode'} = $mode;
     285    $cbp->{"cb.source"}->{$type}->{$source}->{'mtime'} = $mtime;
     286    }
     287}
     288
     289if ((not defined $source) && (not defined $plugin)) {
     290    # Here we need to take all content under $dest considering that machine
     291    if (defined $machine) {
     292        find(\&cb_add_to_cbp,("$dest/$machine"));
    215293    } else {
    216         $sth->execute();
    217     }
    218 }
    219 
    220 my $dest = "$ENV{'TMPDIR'}/vcs.$$";
    221 pb_vcs_export(,$cb->{'basedir'},$dest);
    222 
    223 # Is the source a file or a dir ? Split the source parameter in 2
    224 my $srcdir = undef;
    225 my $srcfile = undef;
    226 my $cmd = undef;
    227 
    228 if (not defined $machine) {
    229     if (-d $source) {
    230         $srcdir = $source;
    231     } else {
    232         $srcdir = dirname($source);
    233         $srcfile = basename($source);
    234     }
     294        find(\&cb_add_to_cbp,($dest));
     295    }
     296}
     297pb_log(2,"INFO: RAW cbp: ".Dumper($cbp)."\n");
     298
     299# Clean up cbp structure by comparing with data stored in the DB
     300# Only keep the more recent modified content
     301my $mac = $machine;
     302$mac = "localhost" if (not defined $machine);
     303my $all = $dbh->selectall_arrayref("SELECT id,date,file,machine FROM dates WHERE machine=$mac");
     304# Check what in cbp is in the DB and deploy only if necessary
     305foreach my $k (keys %{$cbp}) {
     306    foreach my $type ('files','dirs') {
     307        foreach my $o (keys %{$cbp->{$k}->{$type}}) {
     308            # Compare with info from DB
     309            foreach my $row (@$all) {
     310                my ($id, $date, $file, $mac1) = @$row;
     311                # If less recent than in the DB remove it
     312                delete($cbp->{$k}->{$type}->{$o}) if (($file eq $o) && ($date > $cbp->{$k}->{$type}->{$o}->{'mtime'}));
     313                }   
     314            }
     315        }
     316    }
     317}
     318pb_log(2,"INFO: cleaned cbp: ".Dumper($cbp)."\n");
     319
     320# Now create a tar containing all the relevant content
     321my $tar = Archive::Tar->new;
     322$tar->add_files((keys %{$cbp->{$k}->{'files'}}),(keys %{$cbp->{$k}->{'dirs'}}));
     323$tar->write("$ENV{'TMPDIR'}/content$$.tar");
     324
     325# Copy the tar file and extract it + set up modes/uids/gids
     326if (defined $machine) {
     327    $cmd = "scp -p -q $remote:$source $target";
    235328} else {
    236     $cmd = "ssh -q $remote \'echo \"if [ -d $source ]; then exit 0; else exit -1; fi\" | sudo bash\'";
    237     my $res = pb_system($cmd,"","quiet");
    238     pb_log(2,"DEBUG: Found res = $res\n");
    239     if ($res == 0) {
    240         $srcdir = $source;
    241         pb_log(1,"DEBUG: Found remote dir = $source\n");
    242     } else {
    243         $srcdir = dirname($source);
    244         $srcfile = basename($source);
    245         pb_log(1,"DEBUG: Found remote file = $source\n");
    246     }
    247 }
    248 
    249 pb_log(1,"DEBUG: Found srcdir = $srcdir\n");
    250 if (defined $srcfile) {
    251     pb_log(1,"DEBUG: Found srcfile = $srcfile\n");
     329    $cmd = "cp -p $source $target";
     330}
     331
     332if ($debug) {
     333    pb_log(1,"DEBUG: launching $cmd\n");
    252334} else {
    253     pb_log(1,"DEBUG: Found no srcfile\n");
    254 }
    255 
    256 # Deduce the target directory from the local structure and the source
    257 my $target = $basedir;
    258 $target .= "/$machine" if defined ($machine);
    259 $target .= "$srcdir";
    260 
    261 my $scheme = $cb->{'cms'}->{$appname};
    262 
    263 # If both source and target are dirs, then copy into the parent of the target
    264 $target = basename($target) if ((not defined $srcfile) && (-d $target));
    265 
    266 # Create target if it doesn't exist when we have to copy a file
    267 if ((not -d $target) && (defined $srcfile)) {
    268     if ($debug) {
    269         pb_log(1,"DEBUG: Creating recursively directory $target\n");
    270     } else {
    271         pb_mkdir_p($target) || die "Unable to recursively create $target: $!";
    272         pb_vcs_add($scheme,$target);
    273         pb_log(0,"INFO: Created $target and added it to your $scheme system\n");
    274     }
    275 }
    276 
    277 # We need to know where to get the content from
    278 my $cmdopt = "";
    279 
    280 # Recursive if we copy dirs
    281 $cmdopt = "-r" if (not defined $srcfile);
    282 
    283 if (defined $machine) {
    284     $cmd = "scp -p -q $cmdopt $remote:$source $target";
    285 } else {
    286     $cmd = "cp -p $cmdopt $source $target";
    287 }
    288 
    289 # Now add content if not already there
    290 if (defined $srcfile) {
    291     # File case
    292     if ((! -f "$target/$srcfile") || (defined $force)){
    293         if ($debug) {
    294             pb_log(1,"DEBUG: launching $cmd\n");
    295         } else {
    296             pb_system($cmd);
    297             pb_vcs_add($scheme,"$target/$srcfile");
    298             pb_log(0,"INFO: Created $target/$srcfile and added it to your $scheme system\n");
    299         }
    300     } else {
    301         pb_log(0,"INFO: File $target/$srcfile already there\n");
    302     }
    303 } else {
    304     # Directory case
    305     if ($debug) {
    306         pb_log(1,"DEBUG: launching $cmd\n");
    307     } else {
    308         pb_system($cmd);
    309         pb_vcs_add($scheme,"$target");
    310         pb_log(0,"INFO: Created $target and added it to your $scheme system\n");
    311     }
    312 }
    313 pb_log(2,"Exiting cb_busterize\n");
    314 }
     335    pb_system($cmd);
     336    pb_log(0,"INFO: Created $target/$srcfile and added it to your $scheme system\n");
     337}
     338pb_log(2,"Exiting cb_distribute\n");
     339}
     340
     341sub cb_add_to_cbp {
     342
     343my $type = 'files';
     344if (-d $File::Find::name) {
     345    $type = 'dirs';
     346}
     347
     348my $destname = $File::Find::name;
     349# Target name is without the $dest part
     350$destname =~ s|^$dest||;
     351my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime,$mtime,$ctime,$blksize,$blocks) = stat("$File::Find::name") || die "Unable to stat $File::Find::name: $!";
     352$cbp->{"cb.full"}->{$type}->{$destname}->{'uid'} = $uid;
     353$cbp->{"cb.full"}->{$type}->{$destname}->{'gid'} = $gid;
     354$cbp->{"cb.full"}->{$type}->{$destname}->{'mode'} = $mode;
     355$cbp->{"cb.full"}->{$type}->{$destname}->{'mtime'} = $mtime;
     356pb_log(2,"Adding $destname ($uid,$gid,$mode) to cbp\n");
     357}
Note: See TracChangeset for help on using the changeset viewer.