Changeset 1662 in ProjectBuilder


Ignore:
Timestamp:
Oct 22, 2012, 3:42:09 AM (12 years ago)
Author:
Bruno Cornec
Message:
  • Lots of modificiation to have cb deliver common content first
File:
1 edited

Legend:

Unmodified
Added
Removed

  • projects/casparbuster/devel/bin/cb

    r1494 r1662  
    4040The files will be pushed to this machine, and a subdirectory named after the machine
    4141will be used under the basedir to look at the directory structure to deploy
     42When no machine is given, all machnes available are processed
    4243
    4344=item B<--source> I<path>
    … …  
    8485use CasparBuster::Plugin;
    8586#use Cwd 'realpath';
     87use Carp qw/confess cluck/;
    8688use File::Find;
    8789use Archive::Tar;
    … …  
    108110my $log = undef;
    109111my $LOG = undef;
     112my $findtarget = undef;
    110113
    111114my ($cbver,$cbrev) = cb_version_init();
    112115my $appname = "cb";
    113116$ENV{'PBPROJ'} = $appname;
    114 pb_temp_init();
    115117
    116118# Initialize the syntax string
    … …  
    145147}
    146148
    147 $pbdebug = $debug;
    148149pb_log_init($debug, $LOG);
    149 pb_log(0,"Starting cb\n");
     150pb_temp_init();
     151pb_log(1,"Starting cb\n");
    150152
    151153# Get conf file in context
    … …  
    160162my $cbp = ();
    161163my $cb = \%cb;
    162 ($cb->{'basedir'},$cb->{'usemachines'},$cb->{'cms'},$cb->{'database'}) = pb_conf_get("cbbasedir","cbusemachines","cbcms","cbdatabase");
     164($cb->{'basedir'},$cb->{'cms'},$cb->{'database'}) = pb_conf_get("cbbasedir","cbcms","cbdatabase");
    163165pb_log(2,"%cb: ",Dumper($cb));
    164 
    165 die "No machine should be given when using machine structure" if (($cb->{'usemachines'}->{$appname} !~ /true/) && (defined $machine));
    166166
    167167if (defined $plugin) {
    … …  
    203203
    204204# Define destination dir and populate with a VCS export
    205 my $dest = "$ENV{'TMPDIR'}/vcs.$$";
     205my $dest = "$ENV{'PBTMP'}/vcs.$$";
    206206my $scheme = $cb->{'cms'}->{$appname};
     207
     208# Avoids too many permission changes
     209umask(0022);
    207210pb_vcs_export(pb_vcs_get_uri($scheme,$basedir),$basedir,$dest);
    208211
    … …  
    211214    cb_distribute($machine);
    212215} else {
    213     if ($cb->{'usemachines'}->{$appname} =~ /true/) {
    214         # First dir level is the machine, then the content
    215         opendir(DIR,$dest) || die "Unable to open $dest: $!";
    216         foreach my $m (readdir(DIR)) {
    217             next if ($m =~ /^\./);
    218             next if (! -d $m);
    219             # Machine name
    220             cb_distribute($m);
    221         }
    222         closedir(DIR);
    223     } else {
    224         cb_distribute(undef);
     216    # Distribute to all
     217    # First dir level is the machine, then the content
     218    opendir(DIR,$dest) || die "Unable to open $dest: $!";
     219    foreach my $m (readdir(DIR)) {
     220        next if ($m =~ /^\./);
     221        next if (! -d $m);
     222        # Machine name
     223        cb_distribute($m);
     224    closedir(DIR);
    225225    }
    226226}
    227227
    228228# Cleanup
    229 if (not $debug) {
    230     pb_rm_rf($dest);
    231 } else {
    232     pb_log(0,"DEBUG: Please remove manually with rm -rf $dest\n")
    233 }
    234229$dbh->disconnect;
     230pb_exit();
    235231
    236232# End of Main
    … …  
    242238
    243239pb_log(2,"Entering into cb_distribute with machine $machine\n");
     240confess "No machine fiven to cb_distribute" if (not defined $machine);
    244241
    245242# Use potentially a remote account if defined
    246243my $remote = undef;
    247 my ($account) = pb_conf_get_if("cbaccount") if (defined $machine);
    248 $remote = $account->{$machine} if ((defined $account) && (defined $machine) && (defined $account->{$machine}));
     244my ($account) = pb_conf_get_if("cbaccount");
     245$remote = $account->{$machine} if ((defined $account) && (defined $account->{$machine}));
    249246pb_log(1, "DEBUG: remote account1 = $remote\n") if (defined $remote);
    250247$remote = getpwuid($<) if (not defined $remote);
    … …  
    259256        foreach my $type ('files','dirs','dirsandfiles') {
    260257            foreach my $f (keys %{$cbp->{$p}->{$type}}) {
    261                 my $tdir = $dest;
    262                 if (defined $machine) {
    263                     $tdir = "$dest/$machine";
    264                 }
     258                my $tdir = "$dest/$machine";
    265259                if (-r "$tdir/$f") {
    266260                    my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime,$mtime,$ctime,$blksize,$blocks) = stat("$tdir/$f") || die "Unable to stat $tdir: $!";
    … …  
    277271if (defined $source) {
    278272    my $fullsource = "$source";
    279     $fullsource = "$machine/$source" if (defined $machine);
     273    $fullsource = "$machine/$source";
    280274    pb_log(2,"fullsource is $fullsource\n");
    281275    my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime,$mtime,$ctime,$blksize,$blocks) = stat("$dest/$fullsource") || die "Unable to stat $fullsource: $!";
    … …  
    286280    die "ERROR: Only able to handle files or dirs with option --source\n" if ((! -d $fullsource) && (! -f $fullsource));
    287281
     282    # We should get uid/gid fro elsewhere as they're probably wrong locally
    288283    $cbp->{"cb.source"}->{$type}->{$source}->{'uid'} = $uid;
    289284    $cbp->{"cb.source"}->{$type}->{$source}->{'gid'} = $gid;
    … …  
    292287}
    293288
     289($cb->{'commondir'},$cb->{'websrv'},$cb->{'webdir'}) = pb_conf_get_if("cbcommondir","cbwebsrv","cbwebdir");
     290
    294291if ((not defined $source) && (not defined $plugin)) {
    295292    # Here we need to take all content under $dest considering that machine
    296     if (defined $machine) {
    297         find(\&cb_add_to_cbp,("$dest/$machine"));
    298     } else {
    299         find(\&cb_add_to_cbp,($dest));
    300     }
    301 }
    302 pb_log(2,"INFO: RAW cbp: ".Dumper(%$cbp)."\n");
     293    $findtarget = "$dest/$machine";
     294    find(\&cb_add_to_cbp,($findtarget));
     295}
     296pb_log(1,"INFO: RAW cbp: ".Dumper(%$cbp)."\n");
    303297
    304298# Clean up cbp structure by comparing with data stored in the DB
    305299# Only keep the more recent modified content
    306 my $mac = $machine;
    307 $mac = "localhost" if (not defined $machine);
    308300# Allow for errors to occur at DBI level
    309301$dbh->{RaiseError} = 0;
    310302my $checkdb = 1;
    311 my $dbcmd = "SELECT id,date,file,machine FROM dates WHERE machine=\"$mac\"";
     303my $dbcmd = "SELECT id,date,file,machine FROM dates WHERE machine=\"$machine\"";
    312304if (! ($sth = $dbh->prepare(qq{$dbcmd}))) {
    313305        pb_log(0,"Unable to prepare DB statement $dbcmd\n");
    … …  
    319311if ($checkdb == 1) {
    320312    $sth->execute();
    321     # Check what in cbp is in the DB and deploy only if necessary
     313    # Check what in cbp is in the DB and deploy only if necessary or forced
    322314    foreach my $k (keys %{$cbp}) {
    323315        foreach my $type ('files','dirs','dirsandfiles') {
    … …  
    341333# We need to loop separately to allow for DB to not exist in the previous loop !
    342334my $tdir = undef;
    343 if (defined $machine) {
    344     $tdir = "$dest/$machine";
    345 } else {
    346     $tdir = "$dest";
    347 }
     335$tdir = "$dest/$machine";
    348336chdir("$tdir") || die "ERROR: Unable to chdir to $tdir\n";
    349337pb_log(2,"Working now under $tdir\n");
    … …  
    356344        # TODO: for dirs we may remove the files below ?
    357345        foreach my $o ((keys %{$cbp->{$k}->{$type}})) {
    358             if (not defined $cbp->{$k}->{$type}->{$o}->{'deleted'}) {
    359                 $tar->add_files("./$o");
     346            if ((defined $force) || (not defined $cbp->{$k}->{$type}->{$o}->{'deleted'})) {
     347                $tar->add_files("$o");
    360348                # Add an entry to the DB
    361349                if (defined $dbid->{$o}) {
    362350                    # Modify an existing entry
    363                     $dbcmd = "UPDATE dates SET date=$curdate,file=$o WHERE id='?'";
     351                    $dbcmd = "UPDATE dates SET date=\"$curdate\",file=\"$o\" WHERE id=\"$dbid->{$o}\"";
    364352                    if (not $debug) {
    365353                        $sth = $dbh->prepare(qq{$dbcmd});
    366                         $sth = $dbh->execute($dbid->{$o});
    367                     } else {
    368                         pb_log(0,"Executing in DB: $dbcmd with curdate=$curdate,file=$o,id=$dbid->{$o}\n");
     354                        $sth->execute();
    369355                    }
     356                    pb_log(0,"Executing in DB: $dbcmd with curdate=$curdate,file=$o,id=$dbid->{$o}\n");
    370357                } else {
    371358                    # Add an new entry
    372                     $dbcmd = "INSERT INTO dates VALUES (NULL,?,?,$mac)";
     359                    $dbcmd = "INSERT INTO dates VALUES (NULL,?,?,\"$machine\")";
    373360                    if (not $debug) {
    374361                        $sth = $dbh->prepare(qq{$dbcmd});
    375                         $sth = $dbh->execute($curdate,$o);
    376                     } else {
    377                         pb_log(0,"Executing in DB: $dbcmd with curdate=$curdate,file=$o,machine=$mac\n");
     362                        $sth->execute($curdate,$o);
    378363                    }
     364                    pb_log(0,"Executing in DB: $dbcmd with curdate=$curdate,file=$o,machine=$machine\n");
    379365                }
    380366                if (not $debug) {
    … …  
    385371    }
    386372}
    387 my $tarfile = "$ENV{'TMPDIR'}/cbcontent$$.tar";
     373my $tarfile = "$ENV{'PBTMP'}/cbcontent$$.tar";
    388374$tar->write("$tarfile");
    389375
    … …  
    392378
    393379# deal with content first
    394 if (defined $machine) {
    395     # Create remote connection and copy tar file there
    396     $ssh2 = Net::SSH2->new();
    397     $ssh2->connect($machine);
    398     my $hdir = (getpwnam(getpwuid($<)))[7];
    399     if ($ssh2->auth_publickey($remote,"$hdir/.ssh/id_dsa.pub","$hdir/.ssh/id_dsa")) {
    400         $chan = $ssh2->channel();
    401         if (not $debug) {
    402             $chan->exec("if [ ! -d $ENV{'TMPDIR'} ]; then mkdir -p $ENV{'TMPDIR'} fi");
    403             $ssh2->scp_put($tarfile,$ENV{'TMPDIR'});
    404             #$chan->exec("cd / ; tar xf $tarfile");
    405         } else {
    406             pb_log(1,"INFO: creating $ENV{'TMPDIR'} if needed on $remote\@$machine and copying content\n");
    407         }
    408     } else {
    409         pb_log(0,"ERROR: Unable to authenticate to $remote\@$machine\n");
    410         return;
    411     }
    412 }
    413 
    414 # Pointer to function depending whether we're local or remote
    415 my $func;
    416 if (defined $machine) {
    417     $func = \&{ $chan->exec };
     380# Create remote connection and copy tar file there
     381$ssh2 = Net::SSH2->new();
     382$ssh2->debug(1) if ($debug > 2);
     383
     384my $mach = $machine;
     385if ((defined $cb->{'commondir'}) && (defined $cb->{'commondir'}->{$machine})) {
     386    confess "Please provide a cbwebsrv config parameter in order to use common delivery" if ((not defined $cb->{'websrv'}) && (not defined $cb->{'websrv'}->{$machine}));
     387    $mach = $cb->{'websrv'}->{$machine};
     388}
     389$ssh2->connect($mach);
     390my $hdir = (getpwnam(getpwuid($<)))[7];
     391if ($ssh2->auth_publickey($remote,"$hdir/.ssh/id_dsa.pub","$hdir/.ssh/id_dsa")) {
     392    $chan = $ssh2->channel();
     393    $chan->exec("mkdir -p $ENV{'PBTMP'}");
     394    $chan->close;
     395
     396    $ssh2->scp_put($tarfile,$tarfile);
     397    pb_log(0,"INFO: Created $ENV{'PBTMP'} on $remote\@$mach and copying content\n");
    418398} else {
    419     $func = \&pb_system;
    420 }
    421 
     399    pb_log(0,"ERROR: Unable to authenticate to $remote\@$mach\n");
     400    return;
     401}
     402
     403my $path = "/";
     404my $tbextract = "";
     405if ((defined $cb->{'commondir'}) && (defined $cb->{'commondir'}->{$machine})) {
     406    $path = $cb->{'webdir'}->{$machine};
     407    $tbextract = $cb->{'commondir'}->{$machine};
     408}
     409$chan = $ssh2->channel();
     410$chan->shell();
    422411if (not $debug) {
    423     &$func("cd / ; tar xf $tarfile");
     412    print $chan "sudo tar -C $path -x -f $tarfile $tbextract\n";
     413    pb_log(0,"WARNING: $_\n") while (<$chan>);
    424414} else {
    425     pb_log(1,"INFO: Extracting (on $mac) $tarfile under /\n");
    426 }
     415    print $chan "tar -C $path -t -f $tarfile $tbextract\n";
     416    pb_log(2,"INFO: tar content: $_") while (<$chan>);
     417}
     418
     419pb_log(0,"INFO: Extracting $tbextract (on $machine) $tarfile under $path\n");
    427420
    428421foreach my $k (keys %{$cbp}) {
    … …  
    430423        # TODO: do we act recursively for dirsandfiles at least for uid/gid ?
    431424        foreach my $o ((keys %{$cbp->{$k}->{$type}})) {
    432             if (not defined $cbp->{$k}->{$type}->{$o}->{'deleted'}) {
     425            # Note that $path/$o is remote only
     426            if ((defined $force) || (not defined $cbp->{$k}->{$type}->{$o}->{'deleted'})) {
    433427                if ($debug) {
    434                     pb_log(1,"INFO: Executing (on $mac) sudo chown $cbp->{$k}->{$type}->{$o}->{'uid'}:$cbp->{$k}->{$type}->{$o}->{'gid'} $o\n");
    435                     pb_log(1,"INFO: Executing (on $mac) sudo chmod $cbp->{$k}->{$type}->{$o}->{'mode'} $o\n");
     428                    pb_log(1,"INFO: Executing (on $machine) sudo chown $cbp->{$k}->{$type}->{$o}->{'uid'}:$cbp->{$k}->{$type}->{$o}->{'gid'} $path/$o\n");
     429                    pb_log(1,"INFO: Executing (on $machine) sudo chmod $cbp->{$k}->{$type}->{$o}->{'mode'} $path/$o\n");
    436430                } else {
    437                     &$func("sudo chown $cbp->{$k}->{$type}->{$o}->{'uid'}:$cbp->{$k}->{$type}->{$o}->{'gid'} $o");
    438                     &$func("sudo chmod $cbp->{$k}->{$type}->{$o}->{'mode'} $o");
     431                    # TODO: remove hardcoded commands
     432                    print $chan "sudo chown $cbp->{$k}->{$type}->{$o}->{'uid'}:$cbp->{$k}->{$type}->{$o}->{'gid'} $path/$o\n";
     433                    print $chan "sudo chmod $cbp->{$k}->{$type}->{$o}->{'mode'} $path/$o\n";
    439434                }
     435                pb_log(0,"INFO: Delivering $path/$o on $machine\n");
    440436            }
    441437        }
    442438    }
    443439    if (defined $cbp->{$k}->{'reloadscript'}) {
    444         if ($debug) {
    445             pb_log(1,"INFO: Executing (on $mac) $cbp->{$k}->{'reloadscript'}\n");
    446         } else {
    447             &$func("$cbp->{$k}->{'reloadscript'}");
     440        if (not $debug) {
     441            print $chan "$cbp->{$k}->{'reloadscript'}\n";
    448442        }
    449     }
    450 }
    451 
    452 if (defined $machine) {
    453     # Remote cleanup
    454     $chan->exec("rm -rf $ENV{'TMPDIR'}");
    455     $ssh2->disconnect();
    456 }
    457 
    458 # Cleanup
     443        pb_log(0,"INFO: Executing (on $machine) $cbp->{$k}->{'reloadscript'}\n");
     444    }
     445}
     446
     447# Remote cleanup
    459448if (not $debug) {
    460     unlink("$tarfile");
     449    print $chan "rm -rf $ENV{'PBTMP'}\n";
    461450} else {
    462     pb_log(0,"DEBUG: Please remove manually with rm -f $tarfile\n")
    463 }
     451    pb_log(1,"Remove remote directory $ENV{'PBTMP'} on $machine\n");
     452}
     453$chan->close();
     454$ssh2->disconnect();
     455
     456chdir("/");
    464457pb_log(2,"Exiting cb_distribute\n");
    465458}
    … …  
    467460sub cb_add_to_cbp {
    468461
    469 pb_log(3,"Entering into cb_addto_cbp\n");
     462pb_log(3,"Entering into cb_add_to_cbp\n");
    470463my $type = 'files';
    471464if (-d $File::Find::name) {
    … …  
    473466}
    474467
    475 my $destname = $File::Find::name;
    476 # Target name is without the $dest part
    477 $destname =~ s|^$dest||;
    478 my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime,$mtime,$ctime,$blksize,$blocks) = stat("$File::Find::name") || die "Unable to stat $File::Find::name: $!";
    479 $cbp->{"cb.full"}->{$type}->{$destname}->{'uid'} = $uid;
    480 $cbp->{"cb.full"}->{$type}->{$destname}->{'gid'} = $gid;
    481 $cbp->{"cb.full"}->{$type}->{$destname}->{'mode'} = $mode;
    482 $cbp->{"cb.full"}->{$type}->{$destname}->{'mtime'} = $mtime;
    483 pb_log(2,"Adding $destname ($uid,$gid,$mode) to cbp\n");
    484 }
     468# Target name is without the $findtarget part
     469my $targetname = $File::Find::name;
     470$targetname =~ s|^$findtarget[/]*||;
     471
     472return if ($targetname eq "");
     473
     474my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime,$mtime,$ctime,$blksize,$blocks) = stat($File::Find::name);
     475die "Unable to stat $File::Find::name" if (not defined $mode);
     476# We should get uid/gid fro elsewhere as they're probably wrong locally
     477$cbp->{"cb.full"}->{$type}->{$targetname}->{'uid'} = $uid;
     478$cbp->{"cb.full"}->{$type}->{$targetname}->{'gid'} = $gid;
     479$cbp->{"cb.full"}->{$type}->{$targetname}->{'mode'} = sprintf("%04o",$mode & 07777);
     480$cbp->{"cb.full"}->{$type}->{$targetname}->{'mtime'} = $mtime;
     481pb_log(2,"Adding $File::Find::name ($uid,$gid,$mode) to cbp\n");
     482}
Note: See TracChangeset for help on using the changeset viewer.