source: projects/casparbuster/devel/bin/cb

Last change on this file was 1765, checked in by bruno, 6 years ago
  • Adds support for Debian 7.0 to some projects
  • latest cb commits
  • mindi 2.1.5 announce
  • Property svn:executable set to *
File size: 15.4 KB
Line 
1#!/usr/bin/perl -w
2#
3=head1 NAME
4
5cb - CasparBuster looks at the structure in your CMS environment and deploy it to the target systems as needed
6
7=head1 SYNOPSIS
8
9cb [options]
10
11 Options:
12   --debug  |-d         debug mode
13   --help   |-h         brief help message
14   --man            full documentation
15   --force  |-f         force copy of files, even if they exist
16   --source |-s <file/dir>  directory or files to copy from the CasparBuster tree (',' separated if many) to the target
17   --plugin |-p <plugin name>   plugin defining what to copy from the CasparBuster tree (',' separated if many) to the target
18   --machine|-m <machine>   machine to deploy on.
19
20=head1 OPTIONS
21
22=over 4
23
24=item B<--debug>
25
26Enter debug mode. This will print what would be done. No commands are executed,
27so this is safe to use when testing.
28
29=item B<--help>
30
31Print a brief help message and exits.
32
33=item B<--man>
34
35Prints the manual page and exits.
36
37=item B<--machine> I<machine name>
38
39Specify the machine to consider when dealing with the CasparBuster structure.
40The files will be pushed to this machine, and a subdirectory named after the machine
41will be used under the basedir to look at the directory structure to deploy
42When no machine is given, all machnes available are processed
43
44=item B<--source> I<path>
45
46Specify the path of the source file or directory to deploy with CasparBuster. Multiple paths can be specified separated by ','.
47
48=item B<--plugin> I<name>
49
50Specify the name of the plugin to deploy with CasparBuster. Multiple plugins can be specified separated by ','.
51A plugin defines a set of files (with their mode and owner), a set of directories (with their mode and owner) and a set of scripts to launch once the files are copied remotely.
52
53=back
54
55=head1 DESCRIPTION
56
57Deploy the standard CasparBuster structure created by I<cbusterize>. It will reinstall all files and directory in the plugin, with correct owner, group and mode, and launch at the end the script to re-enable potentially the service using the updated files.
58
59=head1 EXAMPLES
60
61    # this will deploy the appropriate CasparBuster environment for DHCP
62    # from the base ~/prj/musique-ancienne.org directory (Cf cbbasedir in cb.conf)
63    # containing the directory victoria2 for this machine
64    # to which it will copy the required files
65
66    cb -m victoria2 -p dhcpd
67
68=head1 AUTHOR
69
70=over 4
71
72Bruno Cornec, http://brunocornec.wordpress.com
73
74=back
75
76=head1 LICENSE
77
78Copyright (C) 2012  Bruno Cornec <bruno@project-builder.org>
79Released under the GPLv2 or the Artistic license at your will.
80
81=cut
82use strict;
83use CasparBuster::Version;
84use CasparBuster::Env;
85use CasparBuster::Plugin;
86use CasparBuster::SSH;
87#use Cwd 'realpath';
88use Carp qw/confess cluck/;
89use File::Find;
90use Archive::Tar;
91use Getopt::Long;
92use Pod::Usage;
93use Data::Dumper;
94use Time::Local;
95use Net::SSH2;
96use ProjectBuilder::Base;
97use ProjectBuilder::Conf;
98use ProjectBuilder::VCS;
99use DBI;
100use DBD::SQLite;
101
102# settings
103my $debug = 0;
104my $help = undef;
105my $man = undef;
106my $source = undef;
107my $machine = undef;
108my $plugin = undef;
109my $quiet = undef;
110my $force = undef;
111my $log = undef;
112my $LOG = undef;
113my $findtarget = undef;
114
115my ($cbver,$cbrev) = cb_version_init();
116my $appname = "cb";
117$ENV{'PBPROJ'} = $appname;
118
119# Initialize the syntax string
120pb_syntax_init("$appname (aka CasparBuster) Version $cbver-$cbrev\n");
121
122# parse command-line options
123GetOptions(
124    'machine|m=s' => \$machine,
125    'debug|d+'    => \$debug,
126    'help|h'      => \$help,
127    'quiet|q'     => \$quiet,
128    'force|f'     => \$force,
129    'man'         => \$man,
130    'logfile|l=s' => \$log,
131    'source|s=s'  => \$source,
132    'plugin|p=s'  => \$plugin,
133) || pb_syntax(-1,0);
134
135if (defined $help) {
136    pb_syntax(0,1);
137}
138if (defined $man) {
139    pb_syntax(0,2);
140}
141if (defined $quiet) {
142    $debug=-1;
143}
144if (defined $log) {
145    open(LOG,"> $log") || die "Unable to log to $log: $!";
146    $LOG = \*LOG;
147    $debug = 0  if ($debug == -1);
148}
149
150pb_log_init($debug, $LOG);
151pb_temp_init();
152pb_log(1,"Starting cb\n");
153
154# Get conf file in context
155pb_conf_init($appname);
156# The personal one if there is such
157pb_conf_add("$ENV{'HOME'}/.cbrc") if (-f "$ENV{'HOME'}/.cbrc");
158# The system one
159pb_conf_add(cb_env_conffile());
160
161# Get configuration parameters
162my %cb;
163my $cbp = ();
164my $cb = \%cb;
165($cb->{'basedir'},$cb->{'cms'},$cb->{'database'}) = pb_conf_get("cbbasedir","cbcms","cbdatabase");
166pb_log(2,"%cb: ",Dumper($cb));
167
168my $basedir = $cb->{'basedir'}->{$appname};
169eval { $basedir =~ s/(\$ENV.+\})/$1/eeg };
170
171# Create basedir if it doesn't exist
172die "Unable to find base directory at $basedir" if (not -d $basedir);
173
174pb_log(1, "DEBUG MODE, not doing anything, just printing\nDEBUG: basedir = $basedir\n");
175
176# Create database if not existing and give a handler
177my $db = "$basedir/$cb->{'database'}->{$appname}";
178
179my $precmd = "";
180if (! -f $db) {
181    $precmd = "CREATE TABLE dates (id INTEGER PRIMARY KEY AUTOINCREMENT, date INTEGER, file VARCHAR[65535], machine VARCHAR[65535], mode VARCHAR[4], uid VARCHAR[5], gid VARCHAR[5])";
182}
183
184my $dbh = DBI->connect("dbi:SQLite:dbname=$db","","",
185            { RaiseError => 1, AutoCommit => 1 })
186            || die "Unable to connect to $db";
187my $sth;
188
189if ($precmd ne "") {
190    $sth = $dbh->prepare(qq{$precmd}) || die "Unable to create table into $db";
191    if ($debug) {
192        pb_log(1,"DEBUG: Creating DB $db\n");
193        pb_log(1,"DEBUG: with command $precmd\n");
194    } else {
195        $sth->execute();
196    }
197    $sth->finish();
198}
199
200# Define destination dir and populate with a VCS export
201my $dest = "$ENV{'PBTMP'}/vcs.$$";
202my $scheme = $cb->{'cms'}->{$appname};
203
204# Avoids too many permission changes
205umask(0022);
206pb_vcs_export(pb_vcs_get_uri($scheme,$basedir),$basedir,$dest);
207
208# Load all plugins plus the additional manually defined
209cb_plugin_load();
210if (defined $plugin) {
211    pb_conf_add("$basedir/plugins/$plugin") if (-f "$basedir/plugins/$plugin");
212}
213
214# Now distribute to the right machines
215if (defined $machine) {
216    cb_distribute($machine);
217} else {
218    # Distribute to all
219    # First dir level is the machine, then the content
220    opendir(DIR,$dest) || die "Unable to open $dest: $!";
221    foreach my $m (readdir(DIR)) {
222        next if ($m =~ /^\./);
223        next if (! -d $m);
224        # Machine name
225        cb_distribute($m);
226    closedir(DIR);
227    }
228}
229
230# Cleanup
231$dbh->disconnect;
232pb_exit();
233
234# End of Main
235
236# Distribute files to target machines
237sub cb_distribute {
238
239my $machine = shift;
240
241pb_log(2,"Entering into cb_distribute with machine $machine\n");
242confess "No machine given to cb_distribute" if (not defined $machine);
243
244# Use potentially a remote account if defined
245my $remote = undef;
246my ($account) = pb_conf_get_if("cbaccount");
247$remote = $account->{$machine} if ((defined $account) && (defined $account->{$machine}));
248pb_log(1, "DEBUG: remote account1 = $remote\n") if (defined $remote);
249$remote = getpwuid($<) if (not defined $remote);
250pb_log(1, "DEBUG: remote account2 = $remote\n");
251
252# Now handle plugins if any
253if (defined $plugin) {
254    foreach my $p (split(/,/,$plugin)) {   
255        pb_log(1,"Getting context for plugin $p\n");
256        $cbp = cb_plugin_get($p,$cbp);
257        # Adds mtime info to the plugin structure
258        foreach my $type ('files','dirs','dirsandfiles') {
259            foreach my $f (keys %{$cbp->{$p}->{$type}}) {
260                my $tdir = "$dest/$machine";
261                if (-r "$tdir/$f") {
262                    my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime,$mtime,$ctime,$blksize,$blocks) = stat("$tdir/$f") || die "Unable to stat $tdir: $!";
263                    $cbp->{$p}->{$type}->{$f}->{'mtime'} = $mtime;
264                } else {
265                    pb_log(0,"WARNING: Unable to read $tdir/$f from plugin $p\n");
266                }   
267            }
268        }
269    }
270}
271
272# Handle this source
273if (defined $source) {
274    my $fullsource = "$source";
275    $fullsource = "$machine/$source";
276    pb_log(2,"fullsource is $fullsource\n");
277    my $type = 'files';
278    if (-d $fullsource) {
279        $type = 'dirsandfiles';
280    }
281    die "ERROR: Only able to handle files or dirs with option --source\n" if ((! -d $fullsource) && (! -f $fullsource));
282
283    cb_fill_cbp("cb.source","$dest/$fullsource",$type,$source)
284}
285
286($cb->{'commondir'},$cb->{'websrv'},$cb->{'webdir'}) = pb_conf_get_if("cbcommondir","cbwebsrv","cbwebdir");
287
288if ((not defined $source) && (not defined $plugin)) {
289    # Here we need to take all content under $dest considering that machine
290    $findtarget = "$dest/$machine";
291    find(\&cb_add_to_cbp,($findtarget));
292    # And we also need all what is common, but not what is for the web side
293    foreach my $c (keys $cb->{'commondir'}) {
294        $findtarget = "$dest/$c";
295        opendir(DIR,"$findtarget") || die "Unable to open $dest/$c: $!";
296        foreach my $m (readdir(DIR)) {
297            next if ($m =~ /^\./);
298            next if ($m eq $cb->{'commondir'}->{$c});
299            find(\&cb_add_to_cbp,("$findtarget/$m"));
300        }
301        closedir(DIR);
302    }
303}
304pb_log(1,"INFO: RAW cbp: ".Dumper(%$cbp)."\n");
305
306# Clean up cbp structure by comparing with data stored in the DB
307# Only keep the more recent modified content
308# Allow for errors to occur at DBI level
309$dbh->{RaiseError} = 0;
310my $checkdb = 1;
311my $dbcmd = "SELECT id,date,file,machine FROM dates WHERE machine=\"$machine\"";
312if (! ($sth = $dbh->prepare(qq{$dbcmd}))) {
313        pb_log(0,"Unable to prepare DB statement $dbcmd\n");
314        $checkdb = 0;
315}
316# DisAllow for errors to occur at DBI level
317$dbh->{RaiseError} = 1;
318my $dbid = ();
319if ($checkdb == 1) {
320    $sth->execute();
321    # Check what in cbp is in the DB and deploy only if necessary or forced
322    foreach my $k (keys %{$cbp}) {
323        foreach my $type ('files','dirs','dirsandfiles') {
324            foreach my $o (keys %{$cbp->{$k}->{$type}}) {
325                # Compare with info from DB
326                foreach my $row ($sth->fetch) {
327                    next if (not defined $row);
328                    my ($id, $date, $file, $mac1) = @$row;
329                    # If less recent than in the DB remove it
330                    $cbp->{$k}->{$type}->{$o}->{'deleted'} = "true" if ((defined $file) && ($file eq $o) && ($date > $cbp->{$k}->{$type}->{$o}->{'mtime'}));
331                    $dbid->{$o} = $id;
332                }
333            }
334        }
335    }
336    $sth->finish();
337}
338pb_log(2,"INFO: cleaned cbp: ".Dumper($cbp)."\n");
339
340# Now create a tar containing all the relevant content
341# We need to loop separately to allow for DB to not exist in the previous loop !
342my $tdir = undef;
343$tdir = "$dest/$machine";
344chdir("$tdir") || die "ERROR: Unable to chdir to $tdir\n";
345pb_log(2,"Working now under $tdir\n");
346
347my $tar = Archive::Tar->new;
348my $curdate = time();
349foreach my $k (keys %{$cbp}) {
350    foreach my $type ('files','dirs','dirsandfiles') {
351        # TODO: for dirs we may remove the files below ?
352        foreach my $o ((keys %{$cbp->{$k}->{$type}})) {
353            if ((defined $force) || (not defined $cbp->{$k}->{$type}->{$o}->{'deleted'})) {
354                if ( -r "$tdir/$o" ) {
355                    pb_log(1,"INFO: Adding to the tar file $tdir/$o\n");
356                    chdir($tdir);
357                    $tar->add_files("$o");
358                } else {
359                    # It's in the common place instead
360                    foreach my $c (keys $cb->{'commondir'}) {
361                        if (-r "$dest/$c/$o") {
362                            pb_log(1,"INFO: Adding to the tar file $dest/$c/$o\n");
363                            chdir("$dest/$c");
364                            $tar->add_files("$o");
365                        }
366                    }
367                }
368                # Add an entry to the DB
369                if (defined $dbid->{$o}) {
370                    # Modify an existing entry
371                    $dbcmd = "UPDATE dates SET date=\"$curdate\",file=\"$o\" WHERE id=\"$dbid->{$o}\"";
372                    if (not $debug) {
373                        $sth = $dbh->prepare(qq{$dbcmd});
374                        $sth->execute();
375                    }
376                    pb_log(0,"Executing in DB: $dbcmd with curdate=$curdate,file=$o,id=$dbid->{$o}\n");
377                } else {
378                    # Add an new entry
379                    $dbcmd = "INSERT INTO dates VALUES (NULL,?,?,\"$machine\")";
380                    if (not $debug) {
381                        $sth = $dbh->prepare(qq{$dbcmd});
382                        $sth->execute($curdate,$o);
383                    }
384                    pb_log(0,"Executing in DB: $dbcmd with curdate=$curdate,file=$o,machine=$machine\n");
385                }
386                if (not $debug) {
387                    $sth->finish();
388                }
389            }
390        }
391    }
392}
393my $tarfile = "$ENV{'PBTMP'}/cbcontent$$.tar";
394$tar->write("$tarfile");
395
396my $ssh2;
397my $chan;
398
399my $mach = $machine;
400if ((defined $cb->{'commondir'}) && (defined $cb->{'commondir'}->{$machine})) {
401    confess "Please provide a cbwebsrv config parameter in order to use common delivery" if ((not defined $cb->{'websrv'}) && (not defined $cb->{'websrv'}->{$machine}));
402    $mach = $cb->{'websrv'}->{$machine};
403}
404
405$ssh2 = cb_ssh_init($remote,$mach,$debug);
406
407if (!($ssh2->scp_put($tarfile,$tarfile))) {
408    my @error = $ssh2->error();
409    print "@error\n";
410    confess "Unable to copy tar file $tarfile to $mach\n";
411}
412pb_log(0,"INFO: Copying content under $ENV{'PBTMP'} on $remote\@$mach\n");
413
414my $path = "/";
415my $tbextract = "";
416if ((defined $cb->{'commondir'}) && (defined $cb->{'commondir'}->{$machine})) {
417    $path = $cb->{'webdir'}->{$machine};
418    #$tbextract = $cb->{'commondir'}->{$machine};
419}
420$chan = $ssh2->channel();
421confess "Unable to launch remote shell through Net:SSH2 ($remote\@$mach)" if (not $chan->shell());
422
423if (not $debug) {
424    # Reminder: sudo should be configured for this account as Defaults !requiretty for this to work
425    print $chan "sudo tar -C $path --no-overwrite-dir -x -f $tarfile $tbextract\n";
426    pb_log(0,"WARNING: $_\n") while (<$chan>);
427} else {
428    print $chan "tar -C $path -t -f $tarfile $tbextract\n";
429    pb_log(2,"INFO: tar content: $_") while (<$chan>);
430}
431
432pb_log(0,"INFO: Extracting $tbextract (on $mach) $tarfile under $path\n");
433
434foreach my $k (keys %{$cbp}) {
435    foreach my $type ('files','dirs','dirsandfiles') {
436        # TODO: do we act recursively for dirsandfiles at least for uid/gid ?
437        foreach my $o ((keys %{$cbp->{$k}->{$type}})) {
438            # Note that $path/$o is remote only
439            if ((defined $force) || (not defined $cbp->{$k}->{$type}->{$o}->{'deleted'})) {
440                if ($debug) {
441                    #pb_log(1,"INFO: Executing (on $mach) sudo chown $cbp->{$k}->{$type}->{$o}->{'uid'}:$cbp->{$k}->{$type}->{$o}->{'gid'} $path/$o\n");
442                    #pb_log(1,"INFO: Executing (on $mach) sudo chmod $cbp->{$k}->{$type}->{$o}->{'mode'} $path/$o\n");
443                } else {
444                    # TODO: remove hardcoded commands
445                    #print $chan "sudo chown $cbp->{$k}->{$type}->{$o}->{'uid'}:$cbp->{$k}->{$type}->{$o}->{'gid'} $path/$o\n";
446                    # TODO: get a correct mode before setting it up
447                    #print $chan "sudo chmod $cbp->{$k}->{$type}->{$o}->{'mode'} $path/$o\n";
448                }
449                pb_log(0,"INFO: Delivering $path/$o on $mach\n");
450            }
451        }
452    }
453    if (defined $cbp->{$k}->{'reloadscript'}) {
454        if (not $debug) {
455            print $chan "sudo $cbp->{$k}->{'reloadscript'}\n";
456        }
457        pb_log(0,"INFO: Executing (on $mach) $cbp->{$k}->{'reloadscript'} as root\n");
458    }
459}
460
461pb_log(0,"INFO: Executing (on $mach) /usr/local/bin/mk$mach if present as root\n");
462if (not $debug) {
463        # Using Net::SSH2 here was not working (due to the shell ?)
464        pb_system("ssh $remote\@$mach \"sudo /usr/local/bin/mk$mach\"","WAIT: Executing (on $mach) /usr/local/bin/mk$mach if present as root","verbose");
465}
466
467# Remote cleanup
468if (not $debug) {
469    print $chan "rm -rf $ENV{'PBTMP'}\n";
470} else {
471    pb_log(1,"INFO: Please remove remote directory $ENV{'PBTMP'} on $mach\n");
472}
473$chan->close();
474
475cb_ssh_close($ssh2);
476
477chdir("/");
478pb_log(2,"Exiting cb_distribute\n");
479}
480
481sub cb_add_to_cbp {
482
483pb_log(3,"Entering into cb_add_to_cbp\n");
484my $type = 'files';
485if (-d $File::Find::name) {
486    $type = 'dirs';
487}
488
489# Target name is without the $findtarget part
490my $targetname = $File::Find::name;
491$targetname =~ s|^$findtarget[/]*||;
492
493return if ($targetname eq "");
494
495cb_fill_cbp("cb.full",$File::Find::name,$type,$targetname)
496}
497
498sub cb_fill_cbp {
499
500my $k = shift;
501my $f = shift;
502my $type = shift;
503my $targetname = shift;
504
505my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime,$mtime,$ctime,$blksize,$blocks) = stat($f);
506($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime,$mtime,$ctime,$blksize,$blocks) = lstat($f) if (not defined $mode);
507die "Unable to stat $f" if (not defined $mode);
508# We should get uid/gid from elsewhere as they're probably wrong locally
509$cbp->{$k}->{$type}->{$targetname}->{'uid'} = $uid;
510$cbp->{$k}->{$type}->{$targetname}->{'gid'} = $gid;
511$cbp->{$k}->{$type}->{$targetname}->{'mode'} = sprintf("%04o",$mode & 07777);
512$cbp->{$k}->{$type}->{$targetname}->{'mtime'} = $mtime;
513pb_log(2,"Adding $f ($uid,$gid,$mode) to cbp\n");
514}
Note: See TracBrowser for help on using the repository browser.