Changeset 1386 in ProjectBuilder

Timestamp:

Jan 2, 2012, 3:47:24 AM (12 years ago)

Author:

Bruno Cornec

Message:

• Adds option checkexclude and command checkssh with the -rebuild option: this allows to check after the build which packages have not been built correctly and to potentially relaunch the build (however why it didn't built is left as an exercise to the reader !)

Location:

devel

Files:

• pb-modules/etc/pb.conf.pod (modified) (1 diff)
• pb-modules/lib/ProjectBuilder/Env.pm (modified) (1 diff)
• pb/bin/pb (modified) (14 diffs)

devel/pb-modules/etc/pb.conf.pod

@@ -22 +22 @@
 
 =over 4
+
+=item B<checkexclude>
+
+ Nature: Optional
+ Key: package (as provided in defpkgdir or extpkgdir)
+ Value: coma separated list of OS (could be from the most generic up to the most specific from ostype, osfamily, os, os-ver, os-ver-arch) that are excluded from the checkssh command (no build made for them). The OS name is generaly used here.
+ Conffile: project
+ Example: checkexclude pkg1 = centos,lsb,solaris
 
 =item B<cpandir>

devel/pb-modules/lib/ProjectBuilder/Env.pm

@@ -308 +308 @@
 #
 
-if (($action =~ /^cms2/) || ($action =~ /^sbx2/) || ($action =~ /^newver$/) || ($action =~ /pbinit/) || ($action =~ /^newproj$/) || ($action =~ /^announce/)) {
+if (($action =~ /^cms2/) || ($action =~ /^sbx2/) || ($action =~ /^newver$/) || ($action =~ /pbinit/) || ($action =~ /^newproj$/) || ($action =~ /^announce/) || ($action =~ /^checkssh/)) {
 
     #

devel/pb/bin/pb

@@ -81 +81 @@
 pb [-vhSq][-r pbroot][-p project][[-s script -a account -P port][-t os-ver-arch][-m os-ver-arch[,...]]][-g][-i iso] <action> [<pkg1> ...]
 
-pb [--verbose][--help][--man][--quiet][--snapshot][--revision pbroot][--project project][[--script script --account account --port port][--target os-ver-arch][--machine os-ver-arch[,...]]][--nographic][--iso iso] <action> [<pkg1> ...]
+pb [--verbose][--help][--man][--quiet][--snapshot][--revision pbroot][--project project][[--script script --account account --port port][--target os-ver-arch][--machine os-ver-arch[,...]]][--nographic][--iso iso][--rebuild] <action> [<pkg1> ...]
 
 =head1 OPTIONS
@@ -149 +149 @@
 
 New version of the project to create based on the current one.
+
+=item B<--rebuild>
+
+Only valid with the checkssh action, it alllows to automatically relaunch the build of the failed packages
 
 =back 
@@ -335 +339 @@
 
 Test a package in a remote machine
+
+=item B<checkssh>
+
+Check the delivery of the packages on the repository
 
 =item B<newver>
@@ -473 +481 @@
         "port|P=i" => \$opts{'P'},
         "project|p=s" => \$opts{'p'},
+        "rebuild" => \$opts{'rebuild'},
         "iso|i=s" => \$opts{'i'},
         "version|V=s" => \$opts{'V'},
@@ -697 +706 @@
     require DBD::SQLite;
 
-    pb_announce();
+    pb_announce("Announce");
+} elsif ($action =~ /^checkssh$/) {
+    pb_announce("Check");
 } elsif ($action =~ /^sbx2webpkg$/) {
     require DBI;
@@ -1627 +1638 @@
         $pbos = pb_distro_get_context($v);
 
-        # Get list of packages to build
         # Get content saved in cms2build
         my ($pkg) = pb_conf_read("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.pb","pbpkg");
@@ -2044 +2054 @@
         $shcmd .= " \"echo \'cd $tdir ; if [ -x $pbscript ]; then ./$pbscript; fi ; rm -f ./$pbscript\' | bash\"";
     }
-    pb_system("$shcmd","Executing pbscript on $cptarget if needed","verbose");
+    my $cmdverb = "verbose";
+    if (($cmt ne "Announce") && ($cmt ne "CPAN")) {
+        $cmdverb = undef;
+    }
+    pb_system("$shcmd","Executing pbscript on $cptarget if needed",$cmdverb);
     if ($cmt =~ /^(V[EM]|RM)build/) {
         # Get back info on pkg produced, compute their name and get them from the VM/RM
@@ -3208 +3222 @@
 sub pb_announce {
 
+    my $antype = shift;
+
     # Get all required parameters
     my ($pbpackager,$pbrepo,$pbml,$pbsmtp) = pb_conf_get("pbpackager","pbrepo","pbml","pbsmtp");
@@ -3220 +3236 @@
     my $distrolist = pb_get_distros($pbos,undef);
     my %dl;
+    my %theorlist;
+    my %archlist;
     foreach my $d (split(/,/,$distrolist)) {
-        my ($d1,$d2, $d3) = split(/-/,$d);
+        my ($d1,$d2,$d3) = split(/-/,$d);
         $dl{$d1}++;
     }
@@ -3252 +3270 @@
         $findstr .= "-name \'$pbrealpkgrpm-$pbver-$pbtag\.*.rpm\' -o -name \'$pbrealpkgdeb"."_$pbver*\.deb\' -o -name \'$pbrealpkg-$pbver*\.ebuild\' -o -name \'$pbrealpkg-$pbver*\.pkg\' -o -name \'$pbrealpkg-$pbver*\.sd\' ";
 
-        my $chglog;
-
-        # Get project info on log file and generate tmp files used later on
-        pb_cms_init($pbinit);
-        $chglog = "$ENV{'PBROOTDIR'}/$pbpkg/pbcl";
-        $chglog = "$ENV{'PBROOTDIR'}/pbcl" if (! -f $chglog);
-        $chglog = undef if (! -f $chglog);
-
-        open(OUT,"> $ENV{'PBTMP'}/$pbpkg.ann") || die "Unable to create $ENV{'PBTMP'}/$pbpkg.ann: $!";
-        my $pb;
-        $pb->{'realpkg'} = $pbrealpkg;
-        $pb->{'ver'} = $pbver;
-        $pb->{'tag'} = $pbtag;
-        $pb->{'date'} = $pbdate;
-        $pb->{'chglog'} = $chglog;
-        $pb->{'packager'} = $pbpackager;
-        $pb->{'proj'} = $ENV{'PBPROJ'};
-        $pb->{'repo'} = $pbrepo;
-        $pb->{'pbos'}->{'type'} = "announce";
-        $pb->{'pbos'}->{'suffix'} = "none";
-        pb_changelog($pb,\*OUT,"yes");
-        close(OUT);
-        push(@files,"$ENV{'PBTMP'}/$pbpkg.ann");
-    }
-    $findstr .= " | grep -Ev \'src.rpm\'";
+        if ($antype eq "Announce") {
+            my $chglog;
+
+            pb_cms_init($pbinit);
+            # Get project info on log file and generate tmp files used later on
+            $chglog = "$ENV{'PBROOTDIR'}/$pbpkg/pbcl";
+            $chglog = "$ENV{'PBROOTDIR'}/pbcl" if (! -f $chglog);
+            $chglog = undef if (! -f $chglog);
+
+            open(OUT,"> $ENV{'PBTMP'}/$pbpkg.ann") || die "Unable to create $ENV{'PBTMP'}/$pbpkg.ann: $!";
+            my $pb;
+            $pb->{'realpkg'} = $pbrealpkg;
+            $pb->{'ver'} = $pbver;
+            $pb->{'tag'} = $pbtag;
+            $pb->{'date'} = $pbdate;
+            $pb->{'chglog'} = $chglog;
+            $pb->{'packager'} = $pbpackager;
+            $pb->{'proj'} = $ENV{'PBPROJ'};
+            $pb->{'repo'} = $pbrepo;
+            $pb->{'pbos'}->{'type'} = "announce";
+            $pb->{'pbos'}->{'suffix'} = "none";
+            pb_changelog($pb,\*OUT,"yes");
+            close(OUT);
+            push(@files,"$ENV{'PBTMP'}/$pbpkg.ann");
+        } else {
+            # For the check we also build the theoritical complete list we should get
+            foreach my $d (split(/,/,$distrolist)) {
+                $pbos = pb_distro_get_context($d);
+                if ($pbos->{'type'} eq "rpm") {
+                    $theorlist{"$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}/$pbrealpkgrpm-$pbver-$pbtag$pbos->{'suffix'}"} = 0;
+                } elsif ($pbos->{'type'} eq "deb") {
+                    $theorlist{"$pbos->{'name'}/$pbos->{'version'}/$pbrealpkgdeb"."_$pbver-$pbtag"} = 0;
+                    $archlist{"$pbos->{'name'}/$pbos->{'version'}/$pbrealpkgdeb"."_$pbver-$pbtag"} = "$pbos->{'arch'}";
+                } elsif ($pbos->{'type'} eq "ebuild") {
+                    $theorlist{"$pbos->{'name'}/$pbos->{'version'}/$pbrealpkg-$pbver-r$pbtag.ebuild"} = 0;
+                    $archlist{"$pbos->{'name'}/$pbos->{'version'}/$pbrealpkg-$pbver-r$pbtag.ebuild"} = "$pbos->{'arch'}";
+                } elsif ($pbos->{'type'} eq "pkg") {
+                    $theorlist{"$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}/$pbrealpkg-$pbver-$pbtag.pkg"} = 0;
+                } else {
+                    pb_log(1,"No theoritical list possible for type $pbos->{'type'}\n");
+                }
+            }
+        }
+        pb_log(2,"theorlist : ".Dumper(%theorlist)."\n");
+    }
+    $findstr .= " | grep -Ev \'src.rpm\'" if ($antype eq "Announce");
 
     # Prepare the command to run and execute it
@@ -3286 +3325 @@
     pb_send2target("Announce");
 
-    # Get subject line
     my $sl = "Project $ENV{'PBPROJ'} version $ENV{'PBPROJVER'} is now available";
-    pb_log(0,"Please enter the title of your announce\n");
-    pb_log(0,"(By default: $sl)\n");
-    my $sl2 = <STDIN>;
-    $sl = $sl2 if ($sl2 !~ /^$/);
-
-    # Prepare a template of announce
-    open(ANN,"> $ENV{'PBTMP'}/announce.html") || die "Unable to create $ENV{'PBTMP'}/announce.html: $!";
-    print ANN << "EOF";
+    if ($antype eq "Announce") {
+        # Get subject line
+        pb_log(0,"Please enter the title of your announce\n");
+        pb_log(0,"(By default: $sl)\n");
+        my $sl2 = <STDIN>;
+        $sl = $sl2 if ($sl2 !~ /^$/);
+
+        # Prepare a template of announce
+        open(ANN,"> $ENV{'PBTMP'}/announce.html") || die "Unable to create $ENV{'PBTMP'}/announce.html: $!";
+        print ANN << "EOF";
 $sl</p>
 
@@ -3304 +3344 @@
 <p>
 EOF
+    }
+
     open(LOG,"$ENV{'PBTMP'}/system.$$.log") || die "Unable to read $ENV{'PBTMP'}/system.$$.log: $!";
-    my $col = 2;
-    my $i = 1;
-    print ANN << 'EOF';
+    if ($antype eq "Announce") {
+        my $col = 2;
+        my $i = 1;
+        print ANN << 'EOF';
 <TABLE WIDTH="700" CELLPADDING="0" CELLSPACING="0" BORDER="0">
 <TR>
 EOF
-    while (<LOG>) {
-        print ANN "<TD><A HREF=\"$pbrepo->{$ENV{'PBPROJ'}}/$_\">$_</A></TD>";
-        $i++;
-        if ($i > $col) {
-            print ANN "</TR>\n<TR>";
-            $i = 1;
-        }
+        while (<LOG>) {
+            print ANN "<TD><A HREF=\"$pbrepo->{$ENV{'PBPROJ'}}/$_\">$_</A></TD>";
+            $i++;
+            if ($i > $col) {
+                print ANN "</TR>\n<TR>";
+                $i = 1;
+            }
+        }
+    } else {
+        # In Check mode we need to compare the 2 lists (real and theoritical)
+        while (<LOG>) {
+            # Get package name and remove what is in extra for the theoritical list (arch at the end)
+            chomp();
+            # skip find errors
+            next if (/^find:/);
+            my $p = $_;
+            $p =~ s/\.(i[3456]86|x86_64|noarch|src)\.rpm$//;
+            $p =~ s/_(i[3456]86|amd64|all).deb$//;
+            $theorlist{$p} = -2 if (not defined $theorlist{$p});
+            $theorlist{$p} = $theorlist{$p} + 1;
+        }
+        pb_log(2,"theorlist : ".Dumper(%theorlist)."\n");
     }
     close(LOG);
+
+    if ($antype eq "Check") {
+        my ($chkex) = pb_conf_get_if("checkexclude");
+        my $vmbuildlist = "";
+        my $vebuildlist = "";
+        my $rmbuildlist = "";
+        my @pt = pb_conf_get_if("vmlist","velist","rmlist");
+        foreach my $t (sort keys %theorlist) {
+            if (defined $theorlist{$t} and $theorlist{$t} >= 1) {
+                pb_log(1,"Packages found for $t\n");
+            } elsif (defined $theorlist{$t} and $theorlist{$t} < 0) {
+                pb_log(0,"Extra Package found for $t\n");
+            } else {
+                my ($os,$ver,$arch,$package) = split(/\//,$t);
+                # Some distro have no arch subdir
+                if (not defined $package) {
+                    $package = $arch;
+                    # TODO: If both arch have failed, we just make the last one
+                    $arch = $archlist{$t};
+                }
+                my $pbos = pb_distro_get_context("$os-$ver-$arch");
+                my $pkgn = $package;
+                if ($pbos->{'type'} ne "deb") {
+                    # package name is more easily found from the end for non deb 
+                    # as '-' is the separator, but it can also be used in names
+                    $pkgn = reverse($package);
+                    # search the second '-' and isolate the now last part which is the full name
+                    $pkgn =~ s/([^-]+)-([^-]+)-([\S])+$/$3/;
+                } else {
+                    $pkgn =~ s/([^_]+)_([\S])+$/$2/;
+                }
+                my $found = 0;
+                # Handle the exclusion of OSes
+                my $excl = "";
+                $excl .= $chkex->{$pkgn} if (defined $chkex->{$pkgn});
+                $excl .= $chkex->{"all"} if (defined $chkex->{"all"});
+                foreach my $ex (split(/,/,$excl)) {
+                    $found = 1 if ("$os-$ver-$arch" =~ /^$ex/);
+                }
+                # Skip as excluded
+                next if ($found == 1);
+                # Avoid duplicates
+                next if ($vmbuildlist =~ /$os-$ver-$arch/);
+                next if ($vebuildlist =~ /$os-$ver-$arch/);
+                next if ($rmbuildlist =~ /$os-$ver-$arch/);
+                pb_log(0,"Package NOT found for $t\n");
+                # check with which method we need to build
+                if ((defined $pt[0]->{$ENV{'PBPROJ'}}) and ($pt[0]->{$ENV{'PBPROJ'}} =~ /$os-$ver-$arch/)) {
+                    $vmbuildlist = "$os-$ver-$arch" if ($vmbuildlist eq "");
+                    $vmbuildlist .= ",$os-$ver-$arch";
+                }
+                if ((defined $pt[1]->{$ENV{'PBPROJ'}}) and ($pt[1]->{$ENV{'PBPROJ'}} =~ /$os-$ver-$arch/)) {
+                    $vebuildlist = "$os-$ver-$arch" if ($vebuildlist eq "");
+                    $vebuildlist .= ",$os-$ver-$arch";
+                }
+                if ((defined $pt[2]->{$ENV{'PBPROJ'}}) and ($pt[2]->{$ENV{'PBPROJ'}} =~ /$os-$ver-$arch/)) {
+                    $rmbuildlist = "$os-$ver-$arch" if ($rmbuildlist eq "");
+                    $rmbuildlist .= ",$os-$ver-$arch";
+                }
+            }
+        }
+        # If we want to rebuild automatically, let's do it
+        if (defined $opts{'rebuild'}) {
+            # SandBox or CMS
+            pb_log(0,"Rebuilding from SandBox\n");
+            pb_log(0,"for VMs: $vmbuildlist\n") if ($vmbuildlist ne "");
+            pb_log(0,"for VEs: $vebuildlist\n") if ($vebuildlist ne "");
+            pb_log(0,"for RMs: $rmbuildlist\n") if ($rmbuildlist ne "");
+            pb_cms2build("SandBox");
+            # Which mode
+            $ENV{'PBV'} = $vmbuildlist;
+            pb_build2v("vm","build") if ($vmbuildlist ne "");
+            $ENV{'PBV'} = $vebuildlist;
+            pb_build2v("ve","build") if ($vebuildlist ne "");
+            $ENV{'PBV'} = $rmbuildlist;
+            pb_build2v("rm","build") if ($rmbuildlist ne "");
+        }
+        # For the check part this is now finished
+        return;
+    }
+
     print ANN << "EOF";
 </TR>
@@ -3799 +3938 @@
         $changes .= " $ENV{'PBBUILDDIR'}/$c" if ($c =~ /\.changes$/);
     }
-    pb_system("debsign -m\'$ENV{'PBPACKAGER'}\' $changes","Signing DEB packages");
+    my $debsigncmd = pb_check_req("debsign",1);
+    pb_system("$debsigncmd -m\'$ENV{'PBPACKAGER'}\' $changes","Signing DEB packages");
 } else {
     pb_log(0,"I don't know yet how to sign packages for type $pbos->{'type'}.\nPlease give feedback to dev team\n");
@@ -3821 +3961 @@
     }
     if (defined $pt[1]->{$ENV{'PBPROJ'}}) {
-        # The 2 lists needs to be grouped with a ',' separating them
+        # The 2 lists need to be grouped with a ',' separating them
         if ($tmpl ne "") {
             $tmpl .= ",";