Changeset 1487 in ProjectBuilder

Timestamp:

Apr 28, 2012, 12:36:51 AM (13 years ago)

Author:

Bruno Cornec

Message:

• Fix Plugin.pm for caspar structure management (don't use tables but hashes)
• more code in cb still not ready.

Files:

• devel/pb/lib/ProjectBuilder/CMS.pm (modified) (1 diff)
• projects/casparbuster/devel/bin/cb (modified) (4 diffs)
• projects/casparbuster/devel/bin/cbusterize (modified) (1 diff)
• projects/casparbuster/devel/lib/CasparBuster/Plugin.pm (modified) (1 diff)
• projects/casparbuster/pbconf/devel/cb/pbcl (added)

devel/pb/lib/ProjectBuilder/CMS.pm

@@ -302 +302 @@
     $uri =~ s/^svk://;
     if (($scheme2 =~ /^git/) || ($scheme2 =~ /^hg/)) {
-        # These VCS manages branches internally not with different tree structures
+        # These VCS manage branches internally not with different tree structures
         # Assuming it's correct for now.
     } elsif ($cmsurl ne $uri) {

projects/casparbuster/devel/bin/cb

@@ -85 +85 @@
 #use Cwd 'realpath';
 use File::Find;
-use File::Copy;
-use File::Basename;
-use File::Path;
-use File::Glob ':glob';
+use Archive::Tar;
 use Getopt::Long;
 use Pod::Usage;
 use Data::Dumper;
-use List::Util qw(first);
 use ProjectBuilder::Base;
 use ProjectBuilder::Conf;
@@ -164 +160 @@
 pb_log(2,"%cb: ",Dumper($cb));
 
-# Check for mandatory params
+die "No machine should be given when using machine structure" if (($cb->{'usemachines'}->{$appname} !~ /true/) && (defined $machine));
 
 if (defined $plugin) {
@@ -174 +170 @@
 eval { $basedir =~ s/(\$ENV.+\})/$1/eeg };
 
+# Create basedir if it doesn't exist
+die "Unable to find base directory at $basedir" if (not -d $basedir);
+
 pb_log(1, "DEBUG MODE, not doing anything, just printing\nDEBUG: basedir = $basedir\n");
+
+# Create database if not existing and give a handler
+my $db = "$basedir/$cb->{'database'}->{$appname}";
+
+my $precmd = "";
+if (! -f $db) {
+    $precmd = "CREATE TABLE dates (id INTEGER PRIMARY KEY AUTOINCREMENT, date INTEGER, file VARCHAR[65535], machine VARCHAR[65535])";
+}
+
+my $dbh = DBI->connect("dbi:SQLite:dbname=$db","","",
+            { RaiseError => 1, AutoCommit => 1 })
+            || die "Unable to connect to $db";
+
+if ($precmd ne "") {
+    my $sth = $dbh->prepare(qq{$precmd}) || die "Unable to create table into $db";
+    if ($debug) {
+        pb_log(1,"DEBUG: Creating DB $db\n");
+        pb_log(1,"DEBUG: with command $precmd\n");
+    } else {
+        $sth->execute();
+    }
+    $sth->finish();
+}
+
+# Define destination dir and populate with a VCS export
+my $dest = "$ENV{'TMPDIR'}/vcs.$$";
+my $scheme = $cb->{'cms'}->{$appname};
+pb_vcs_export(pb_vcs_get_uri($scheme,$basedir),$basedir,$dest);
+
+# Now distribute to the right machines
+if defined ($machine) {
+    cb_distribute($machine);
+} else {
+    if ($cb->{'usemachines'}->{$appname} =~ /true/) {
+        # First dir level is the machine, then the content
+        opendir(DIR,$dest) || die "Unable to open $dest: $!";
+        foreach my $m (readdir(DIR)) {
+            next if ($m =~ /^\./);
+            next if (! -d $m);
+            # Machine name
+            cb_distribute($m);
+        }
+        closedir(DIR);
+    } else {
+        cb_distribute(undef);
+    }
+}
+
+# Cleanup
+pb_rm_rf($dest);
+$dbh->disconnect;
+
+# End of Main
+
+# Distribute files to target machines
+sub cb_distribute {
+
+my $machine = shift;
+
+pb_log(2,"Entering into cb_distribute with machine $machine\n");
 
 # Use potentially a remote account if defined
@@ -183 +242 @@
 pb_log(1, "DEBUG: remote = $remote\n") if (defined $remote);
 
-# Create basedir if it doesn't exist
-die "Unable to find base directory at $basedir" if (not -d $basedir);
-
 # Now handle plugins if any
 my $cbp = ();
 
+# Handle plugins
 if (defined $plugin) {
     foreach my $p (split(/,/,$plugin)) {    
         pb_log(1,"Getting context for plugin $p\n");
         $cbp = cb_plugin_get($p,$cbp);
-        pb_log(2,"cbp: ".Dumper($cbp)."\n");
-    }
-}
-
-my $db = "$cb->{'basedir'}/$cb->{'database'}";
-
-my $precmd = "";
-if (! -f $db) {
-    $precmd = "CREATE TABLE dates (id INTEGER PRIMARY KEY AUTOINCREMENT, date DATE, file VARCHAR[65535])";
-}
-
-my $dbh = DBI->connect("dbi:SQLite:dbname=$db","","",
-            { RaiseError => 1, AutoCommit => 1 })
-            || die "Unable to connect to $db";
-
-if ($precmd ne "") {
-    my $sth = $dbh->prepare(qq{$precmd}) || die "Unable to create table into $db";
-    if ($debug) {
-            pb_log(1,"DEBUG: Creatung DB $db\n");
-            pb_log(1,"DEBUG: with command $precmd\n");
+        # Adds mtime info to the plugin structure
+        foreach my $type ('files','dirs') {
+            foreach my $f (keys %{$cbp->{$p}->{$type}}) {
+                my $tdir = $dest;
+                if (defined $machine) {
+                    $tdir = "$dest/$machine";
+                }
+                if (-r "$tdir/$f") {
+                    my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime,$mtime,$ctime,$blksize,$blocks) = stat("$tdir/$f") || die "Unable to stat $tdir: $!";
+                    $cbp->{$p}->{$type}->{$f}->{'mtime'} = $mtime;
+                } else {
+                    pb_log(0,"WARNING: Unable to read $tdir/$f from plugin $p\n");
+                }   
+            }
+        }
+    }
+}
+
+# Handle this source
+if (defined $source) {
+    my $fullsource = "$source";
+    $fullsource = "$machine/$source" if (defined $machine);
+    pb_log(2,"fullsource is $fullsource\n");
+    my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime,$mtime,$ctime,$blksize,$blocks) = stat("$dest/$fullsource") || die "Unable to stat $fullsource: $!";
+    my $type = 'files';
+    if (-d $fullsource) {
+        $type = 'dirs';
+    }
+    die "ERROR: Only able to handle files or dirs with option --source\n" if ((! -d $fullsource) && (! -f $fullsource));
+
+    $cbp->{"cb.source"}->{$type}->{$source}->{'uid'} = $uid;
+    $cbp->{"cb.source"}->{$type}->{$source}->{'gid'} = $gid;
+    $cbp->{"cb.source"}->{$type}->{$source}->{'mode'} = $mode;
+    $cbp->{"cb.source"}->{$type}->{$source}->{'mtime'} = $mtime;
+    }
+}
+
+if ((not defined $source) && (not defined $plugin)) {
+    # Here we need to take all content under $dest considering that machine
+    if (defined $machine) {
+        find(\&cb_add_to_cbp,("$dest/$machine"));
     } else {
-        $sth->execute();
-    }
-}
-
-my $dest = "$ENV{'TMPDIR'}/vcs.$$";
-pb_vcs_export(,$cb->{'basedir'},$dest);
-
-# Is the source a file or a dir ? Split the source parameter in 2
-my $srcdir = undef;
-my $srcfile = undef;
-my $cmd = undef;
-
-if (not defined $machine) {
-    if (-d $source) {
-        $srcdir = $source;
-    } else {
-        $srcdir = dirname($source);
-        $srcfile = basename($source);
-    }
+        find(\&cb_add_to_cbp,($dest));
+    }
+}
+pb_log(2,"INFO: RAW cbp: ".Dumper($cbp)."\n");
+
+# Clean up cbp structure by comparing with data stored in the DB
+# Only keep the more recent modified content
+my $mac = $machine;
+$mac = "localhost" if (not defined $machine);
+my $all = $dbh->selectall_arrayref("SELECT id,date,file,machine FROM dates WHERE machine=$mac");
+# Check what in cbp is in the DB and deploy only if necessary
+foreach my $k (keys %{$cbp}) {
+    foreach my $type ('files','dirs') {
+        foreach my $o (keys %{$cbp->{$k}->{$type}}) {
+            # Compare with info from DB
+            foreach my $row (@$all) {
+                my ($id, $date, $file, $mac1) = @$row;
+                # If less recent than in the DB remove it
+                delete($cbp->{$k}->{$type}->{$o}) if (($file eq $o) && ($date > $cbp->{$k}->{$type}->{$o}->{'mtime'}));
+                }   
+            }
+        }
+    }
+}
+pb_log(2,"INFO: cleaned cbp: ".Dumper($cbp)."\n");
+
+# Now create a tar containing all the relevant content
+my $tar = Archive::Tar->new;
+$tar->add_files((keys %{$cbp->{$k}->{'files'}}),(keys %{$cbp->{$k}->{'dirs'}}));
+$tar->write("$ENV{'TMPDIR'}/content$$.tar");
+
+# Copy the tar file and extract it + set up modes/uids/gids
+if (defined $machine) {
+    $cmd = "scp -p -q $remote:$source $target";
 } else {
-    $cmd = "ssh -q $remote \'echo \"if [ -d $source ]; then exit 0; else exit -1; fi\" | sudo bash\'";
-    my $res = pb_system($cmd,"","quiet");
-    pb_log(2,"DEBUG: Found res = $res\n");
-    if ($res == 0) {
-        $srcdir = $source;
-        pb_log(1,"DEBUG: Found remote dir = $source\n");
-    } else {
-        $srcdir = dirname($source);
-        $srcfile = basename($source);
-        pb_log(1,"DEBUG: Found remote file = $source\n");
-    }
-}
-
-pb_log(1,"DEBUG: Found srcdir = $srcdir\n");
-if (defined $srcfile) {
-    pb_log(1,"DEBUG: Found srcfile = $srcfile\n");
+    $cmd = "cp -p $source $target";
+}
+
+if ($debug) {
+    pb_log(1,"DEBUG: launching $cmd\n");
 } else {
-    pb_log(1,"DEBUG: Found no srcfile\n");
-}
-
-# Deduce the target directory from the local structure and the source
-my $target = $basedir;
-$target .= "/$machine" if defined ($machine);
-$target .= "$srcdir";
-
-my $scheme = $cb->{'cms'}->{$appname};
-
-# If both source and target are dirs, then copy into the parent of the target
-$target = basename($target) if ((not defined $srcfile) && (-d $target));
-
-# Create target if it doesn't exist when we have to copy a file
-if ((not -d $target) && (defined $srcfile)) {
-    if ($debug) {
-        pb_log(1,"DEBUG: Creating recursively directory $target\n");
-    } else {
-        pb_mkdir_p($target) || die "Unable to recursively create $target: $!";
-        pb_vcs_add($scheme,$target);
-        pb_log(0,"INFO: Created $target and added it to your $scheme system\n");
-    }
-}
-
-# We need to know where to get the content from
-my $cmdopt = "";
-
-# Recursive if we copy dirs
-$cmdopt = "-r" if (not defined $srcfile);
-
-if (defined $machine) {
-    $cmd = "scp -p -q $cmdopt $remote:$source $target";
-} else {
-    $cmd = "cp -p $cmdopt $source $target";
-}
-
-# Now add content if not already there
-if (defined $srcfile) {
-    # File case
-    if ((! -f "$target/$srcfile") || (defined $force)){
-        if ($debug) {
-            pb_log(1,"DEBUG: launching $cmd\n");
-        } else {
-            pb_system($cmd);
-            pb_vcs_add($scheme,"$target/$srcfile");
-            pb_log(0,"INFO: Created $target/$srcfile and added it to your $scheme system\n");
-        }
-    } else {
-        pb_log(0,"INFO: File $target/$srcfile already there\n");
-    }
-} else {
-    # Directory case
-    if ($debug) {
-        pb_log(1,"DEBUG: launching $cmd\n");
-    } else {
-        pb_system($cmd);
-        pb_vcs_add($scheme,"$target");
-        pb_log(0,"INFO: Created $target and added it to your $scheme system\n");
-    }
-}
-pb_log(2,"Exiting cb_busterize\n");
-}
+    pb_system($cmd);
+    pb_log(0,"INFO: Created $target/$srcfile and added it to your $scheme system\n");
+}
+pb_log(2,"Exiting cb_distribute\n");
+}
+
+sub cb_add_to_cbp {
+
+my $type = 'files';
+if (-d $File::Find::name) {
+    $type = 'dirs';
+}
+
+my $destname = $File::Find::name;
+# Target name is without the $dest part
+$destname =~ s|^$dest||;
+my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime,$mtime,$ctime,$blksize,$blocks) = stat("$File::Find::name") || die "Unable to stat $File::Find::name: $!";
+$cbp->{"cb.full"}->{$type}->{$destname}->{'uid'} = $uid;
+$cbp->{"cb.full"}->{$type}->{$destname}->{'gid'} = $gid;
+$cbp->{"cb.full"}->{$type}->{$destname}->{'mode'} = $mode;
+$cbp->{"cb.full"}->{$type}->{$destname}->{'mtime'} = $mtime;
+pb_log(2,"Adding $destname ($uid,$gid,$mode) to cbp\n");
+}

projects/casparbuster/devel/bin/cbusterize

@@ -110 +110 @@
 use CasparBuster::Env;
 use CasparBuster::Plugin;
-#use Cwd 'realpath';
-use File::Find;
-use File::Copy;
 use File::Basename;
 use File::Path;
-use File::Glob ':glob';
 use Getopt::Long;
 use Pod::Usage;
 use Data::Dumper;
-use List::Util qw(first);
 use ProjectBuilder::Base;
 use ProjectBuilder::Conf;

projects/casparbuster/devel/lib/CasparBuster/Plugin.pm

@@ -86 +86 @@
 }
 if ((defined $dlist) && (defined $dlist->{$plugin}) && ($dlist->{$plugin} !~ /^\s*$/)) {
-    my $i = 0;
     foreach my $block (split(/;/,$dlist->{$plugin})) {
         pb_log(3,"block : $block\n");
-        my $tmp;
-        ($cbp->{$plugin}->{'dirs'}->{'name'}[$i],$tmp) = split(/\|/,$block);
-        ($cbp->{$plugin}->{'dirs'}->{'uid'}[$i],$cbp->{$plugin}->{'dirs'}->{'gid'}[$i],$cbp->{$plugin}->{'dirs'}->{'mode'}[$i]) = split(/\,/,$tmp);
-        $i++;
+        ($name,$tmp) = split(/\|/,$block);
+        ($cbp->{$plugin}->{'dirs'}->{$name}->{'uid'},$cbp->{$plugin}->{'dirs'}->{$name}->{'gid'},$cbp->{$plugin}->{'dirs'}->{$name}->{'mode'}) = split(/\,/,$tmp);
     }
 }