Changeset 1386


Ignore:
Timestamp:
Jan 2, 2012, 3:47:24 AM (8 years ago)
Author:
bruno
Message:
  • Adds option checkexclude and command checkssh with the -rebuild option: this allows to check after the build which packages have not been built correctly and to potentially relaunch the build (however why it didn't built is left as an exercise to the reader !)
Location:
devel
Files:
3 edited

Legend:

Unmodified
Added
Removed
  • devel/pb-modules/etc/pb.conf.pod

    r1374 r1386  
    2222
    2323=over 4
     24
     25=item B<checkexclude>
     26
     27 Nature: Optional
     28 Key: package (as provided in defpkgdir or extpkgdir)
     29 Value: coma separated list of OS (could be from the most generic up to the most specific from ostype, osfamily, os, os-ver, os-ver-arch) that are excluded from the checkssh command (no build made for them). The OS name is generaly used here.
     30 Conffile: project
     31 Example: checkexclude pkg1 = centos,lsb,solaris
    2432
    2533=item B<cpandir>
  • devel/pb-modules/lib/ProjectBuilder/Env.pm

    r1363 r1386  
    308308#
    309309
    310 if (($action =~ /^cms2/) || ($action =~ /^sbx2/) || ($action =~ /^newver$/) || ($action =~ /pbinit/) || ($action =~ /^newproj$/) || ($action =~ /^announce/)) {
     310if (($action =~ /^cms2/) || ($action =~ /^sbx2/) || ($action =~ /^newver$/) || ($action =~ /pbinit/) || ($action =~ /^newproj$/) || ($action =~ /^announce/) || ($action =~ /^checkssh/)) {
    311311
    312312    #
  • devel/pb/bin/pb

    r1384 r1386  
    8181pb [-vhSq][-r pbroot][-p project][[-s script -a account -P port][-t os-ver-arch][-m os-ver-arch[,...]]][-g][-i iso] <action> [<pkg1> ...]
    8282
    83 pb [--verbose][--help][--man][--quiet][--snapshot][--revision pbroot][--project project][[--script script --account account --port port][--target os-ver-arch][--machine os-ver-arch[,...]]][--nographic][--iso iso] <action> [<pkg1> ...]
     83pb [--verbose][--help][--man][--quiet][--snapshot][--revision pbroot][--project project][[--script script --account account --port port][--target os-ver-arch][--machine os-ver-arch[,...]]][--nographic][--iso iso][--rebuild] <action> [<pkg1> ...]
    8484
    8585=head1 OPTIONS
     
    149149
    150150New version of the project to create based on the current one.
     151
     152=item B<--rebuild>
     153
     154Only valid with the checkssh action, it alllows to automatically relaunch the build of the failed packages
    151155
    152156=back
     
    335339
    336340Test a package in a remote machine
     341
     342=item B<checkssh>
     343
     344Check the delivery of the packages on the repository
    337345
    338346=item B<newver>
     
    473481        "port|P=i" => \$opts{'P'},
    474482        "project|p=s" => \$opts{'p'},
     483        "rebuild" => \$opts{'rebuild'},
    475484        "iso|i=s" => \$opts{'i'},
    476485        "version|V=s" => \$opts{'V'},
     
    697706    require DBD::SQLite;
    698707
    699     pb_announce();
     708    pb_announce("Announce");
     709} elsif ($action =~ /^checkssh$/) {
     710    pb_announce("Check");
    700711} elsif ($action =~ /^sbx2webpkg$/) {
    701712    require DBI;
     
    16271638        $pbos = pb_distro_get_context($v);
    16281639
    1629         # Get list of packages to build
    16301640        # Get content saved in cms2build
    16311641        my ($pkg) = pb_conf_read("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.pb","pbpkg");
     
    20442054        $shcmd .= " \"echo \'cd $tdir ; if [ -x $pbscript ]; then ./$pbscript; fi ; rm -f ./$pbscript\' | bash\"";
    20452055    }
    2046     pb_system("$shcmd","Executing pbscript on $cptarget if needed","verbose");
     2056    my $cmdverb = "verbose";
     2057    if (($cmt ne "Announce") && ($cmt ne "CPAN")) {
     2058        $cmdverb = undef;
     2059    }
     2060    pb_system("$shcmd","Executing pbscript on $cptarget if needed",$cmdverb);
    20472061    if ($cmt =~ /^(V[EM]|RM)build/) {
    20482062        # Get back info on pkg produced, compute their name and get them from the VM/RM
     
    32083222sub pb_announce {
    32093223
     3224    my $antype = shift;
     3225
    32103226    # Get all required parameters
    32113227    my ($pbpackager,$pbrepo,$pbml,$pbsmtp) = pb_conf_get("pbpackager","pbrepo","pbml","pbsmtp");
     
    32203236    my $distrolist = pb_get_distros($pbos,undef);
    32213237    my %dl;
     3238    my %theorlist;
     3239    my %archlist;
    32223240    foreach my $d (split(/,/,$distrolist)) {
    3223         my ($d1,$d2, $d3) = split(/-/,$d);
     3241        my ($d1,$d2,$d3) = split(/-/,$d);
    32243242        $dl{$d1}++;
    32253243    }
     
    32523270        $findstr .= "-name \'$pbrealpkgrpm-$pbver-$pbtag\.*.rpm\' -o -name \'$pbrealpkgdeb"."_$pbver*\.deb\' -o -name \'$pbrealpkg-$pbver*\.ebuild\' -o -name \'$pbrealpkg-$pbver*\.pkg\' -o -name \'$pbrealpkg-$pbver*\.sd\' ";
    32533271
    3254         my $chglog;
    3255 
    3256         # Get project info on log file and generate tmp files used later on
    3257         pb_cms_init($pbinit);
    3258         $chglog = "$ENV{'PBROOTDIR'}/$pbpkg/pbcl";
    3259         $chglog = "$ENV{'PBROOTDIR'}/pbcl" if (! -f $chglog);
    3260         $chglog = undef if (! -f $chglog);
    3261 
    3262         open(OUT,"> $ENV{'PBTMP'}/$pbpkg.ann") || die "Unable to create $ENV{'PBTMP'}/$pbpkg.ann: $!";
    3263         my $pb;
    3264         $pb->{'realpkg'} = $pbrealpkg;
    3265         $pb->{'ver'} = $pbver;
    3266         $pb->{'tag'} = $pbtag;
    3267         $pb->{'date'} = $pbdate;
    3268         $pb->{'chglog'} = $chglog;
    3269         $pb->{'packager'} = $pbpackager;
    3270         $pb->{'proj'} = $ENV{'PBPROJ'};
    3271         $pb->{'repo'} = $pbrepo;
    3272         $pb->{'pbos'}->{'type'} = "announce";
    3273         $pb->{'pbos'}->{'suffix'} = "none";
    3274         pb_changelog($pb,\*OUT,"yes");
    3275         close(OUT);
    3276         push(@files,"$ENV{'PBTMP'}/$pbpkg.ann");
    3277     }
    3278     $findstr .= " | grep -Ev \'src.rpm\'";
     3272        if ($antype eq "Announce") {
     3273            my $chglog;
     3274
     3275            pb_cms_init($pbinit);
     3276            # Get project info on log file and generate tmp files used later on
     3277            $chglog = "$ENV{'PBROOTDIR'}/$pbpkg/pbcl";
     3278            $chglog = "$ENV{'PBROOTDIR'}/pbcl" if (! -f $chglog);
     3279            $chglog = undef if (! -f $chglog);
     3280
     3281            open(OUT,"> $ENV{'PBTMP'}/$pbpkg.ann") || die "Unable to create $ENV{'PBTMP'}/$pbpkg.ann: $!";
     3282            my $pb;
     3283            $pb->{'realpkg'} = $pbrealpkg;
     3284            $pb->{'ver'} = $pbver;
     3285            $pb->{'tag'} = $pbtag;
     3286            $pb->{'date'} = $pbdate;
     3287            $pb->{'chglog'} = $chglog;
     3288            $pb->{'packager'} = $pbpackager;
     3289            $pb->{'proj'} = $ENV{'PBPROJ'};
     3290            $pb->{'repo'} = $pbrepo;
     3291            $pb->{'pbos'}->{'type'} = "announce";
     3292            $pb->{'pbos'}->{'suffix'} = "none";
     3293            pb_changelog($pb,\*OUT,"yes");
     3294            close(OUT);
     3295            push(@files,"$ENV{'PBTMP'}/$pbpkg.ann");
     3296        } else {
     3297            # For the check we also build the theoritical complete list we should get
     3298            foreach my $d (split(/,/,$distrolist)) {
     3299                $pbos = pb_distro_get_context($d);
     3300                if ($pbos->{'type'} eq "rpm") {
     3301                    $theorlist{"$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}/$pbrealpkgrpm-$pbver-$pbtag$pbos->{'suffix'}"} = 0;
     3302                } elsif ($pbos->{'type'} eq "deb") {
     3303                    $theorlist{"$pbos->{'name'}/$pbos->{'version'}/$pbrealpkgdeb"."_$pbver-$pbtag"} = 0;
     3304                    $archlist{"$pbos->{'name'}/$pbos->{'version'}/$pbrealpkgdeb"."_$pbver-$pbtag"} = "$pbos->{'arch'}";
     3305                } elsif ($pbos->{'type'} eq "ebuild") {
     3306                    $theorlist{"$pbos->{'name'}/$pbos->{'version'}/$pbrealpkg-$pbver-r$pbtag.ebuild"} = 0;
     3307                    $archlist{"$pbos->{'name'}/$pbos->{'version'}/$pbrealpkg-$pbver-r$pbtag.ebuild"} = "$pbos->{'arch'}";
     3308                } elsif ($pbos->{'type'} eq "pkg") {
     3309                    $theorlist{"$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}/$pbrealpkg-$pbver-$pbtag.pkg"} = 0;
     3310                } else {
     3311                    pb_log(1,"No theoritical list possible for type $pbos->{'type'}\n");
     3312                }
     3313            }
     3314        }
     3315        pb_log(2,"theorlist : ".Dumper(%theorlist)."\n");
     3316    }
     3317    $findstr .= " | grep -Ev \'src.rpm\'" if ($antype eq "Announce");
    32793318
    32803319    # Prepare the command to run and execute it
     
    32863325    pb_send2target("Announce");
    32873326
    3288     # Get subject line
    32893327    my $sl = "Project $ENV{'PBPROJ'} version $ENV{'PBPROJVER'} is now available";
    3290     pb_log(0,"Please enter the title of your announce\n");
    3291     pb_log(0,"(By default: $sl)\n");
    3292     my $sl2 = <STDIN>;
    3293     $sl = $sl2 if ($sl2 !~ /^$/);
    3294 
    3295     # Prepare a template of announce
    3296     open(ANN,"> $ENV{'PBTMP'}/announce.html") || die "Unable to create $ENV{'PBTMP'}/announce.html: $!";
    3297     print ANN << "EOF";
     3328    if ($antype eq "Announce") {
     3329        # Get subject line
     3330        pb_log(0,"Please enter the title of your announce\n");
     3331        pb_log(0,"(By default: $sl)\n");
     3332        my $sl2 = <STDIN>;
     3333        $sl = $sl2 if ($sl2 !~ /^$/);
     3334
     3335        # Prepare a template of announce
     3336        open(ANN,"> $ENV{'PBTMP'}/announce.html") || die "Unable to create $ENV{'PBTMP'}/announce.html: $!";
     3337        print ANN << "EOF";
    32983338$sl</p>
    32993339
     
    33043344<p>
    33053345EOF
     3346    }
     3347
    33063348    open(LOG,"$ENV{'PBTMP'}/system.$$.log") || die "Unable to read $ENV{'PBTMP'}/system.$$.log: $!";
    3307     my $col = 2;
    3308     my $i = 1;
    3309     print ANN << 'EOF';
     3349    if ($antype eq "Announce") {
     3350        my $col = 2;
     3351        my $i = 1;
     3352        print ANN << 'EOF';
    33103353<TABLE WIDTH="700" CELLPADDING="0" CELLSPACING="0" BORDER="0">
    33113354<TR>
    33123355EOF
    3313     while (<LOG>) {
    3314         print ANN "<TD><A HREF=\"$pbrepo->{$ENV{'PBPROJ'}}/$_\">$_</A></TD>";
    3315         $i++;
    3316         if ($i > $col) {
    3317             print ANN "</TR>\n<TR>";
    3318             $i = 1;
    3319         }
     3356        while (<LOG>) {
     3357            print ANN "<TD><A HREF=\"$pbrepo->{$ENV{'PBPROJ'}}/$_\">$_</A></TD>";
     3358            $i++;
     3359            if ($i > $col) {
     3360                print ANN "</TR>\n<TR>";
     3361                $i = 1;
     3362            }
     3363        }
     3364    } else {
     3365        # In Check mode we need to compare the 2 lists (real and theoritical)
     3366        while (<LOG>) {
     3367            # Get package name and remove what is in extra for the theoritical list (arch at the end)
     3368            chomp();
     3369            # skip find errors
     3370            next if (/^find:/);
     3371            my $p = $_;
     3372            $p =~ s/\.(i[3456]86|x86_64|noarch|src)\.rpm$//;
     3373            $p =~ s/_(i[3456]86|amd64|all).deb$//;
     3374            $theorlist{$p} = -2 if (not defined $theorlist{$p});
     3375            $theorlist{$p} = $theorlist{$p} + 1;
     3376        }
     3377        pb_log(2,"theorlist : ".Dumper(%theorlist)."\n");
    33203378    }
    33213379    close(LOG);
     3380
     3381    if ($antype eq "Check") {
     3382        my ($chkex) = pb_conf_get_if("checkexclude");
     3383        my $vmbuildlist = "";
     3384        my $vebuildlist = "";
     3385        my $rmbuildlist = "";
     3386        my @pt = pb_conf_get_if("vmlist","velist","rmlist");
     3387        foreach my $t (sort keys %theorlist) {
     3388            if (defined $theorlist{$t} and $theorlist{$t} >= 1) {
     3389                pb_log(1,"Packages found for $t\n");
     3390            } elsif (defined $theorlist{$t} and $theorlist{$t} < 0) {
     3391                pb_log(0,"Extra Package found for $t\n");
     3392            } else {
     3393                my ($os,$ver,$arch,$package) = split(/\//,$t);
     3394                # Some distro have no arch subdir
     3395                if (not defined $package) {
     3396                    $package = $arch;
     3397                    # TODO: If both arch have failed, we just make the last one
     3398                    $arch = $archlist{$t};
     3399                }
     3400                my $pbos = pb_distro_get_context("$os-$ver-$arch");
     3401                my $pkgn = $package;
     3402                if ($pbos->{'type'} ne "deb") {
     3403                    # package name is more easily found from the end for non deb
     3404                    # as '-' is the separator, but it can also be used in names
     3405                    $pkgn = reverse($package);
     3406                    # search the second '-' and isolate the now last part which is the full name
     3407                    $pkgn =~ s/([^-]+)-([^-]+)-([\S])+$/$3/;
     3408                } else {
     3409                    $pkgn =~ s/([^_]+)_([\S])+$/$2/;
     3410                }
     3411                my $found = 0;
     3412                # Handle the exclusion of OSes
     3413                my $excl = "";
     3414                $excl .= $chkex->{$pkgn} if (defined $chkex->{$pkgn});
     3415                $excl .= $chkex->{"all"} if (defined $chkex->{"all"});
     3416                foreach my $ex (split(/,/,$excl)) {
     3417                    $found = 1 if ("$os-$ver-$arch" =~ /^$ex/);
     3418                }
     3419                # Skip as excluded
     3420                next if ($found == 1);
     3421                # Avoid duplicates
     3422                next if ($vmbuildlist =~ /$os-$ver-$arch/);
     3423                next if ($vebuildlist =~ /$os-$ver-$arch/);
     3424                next if ($rmbuildlist =~ /$os-$ver-$arch/);
     3425                pb_log(0,"Package NOT found for $t\n");
     3426                # check with which method we need to build
     3427                if ((defined $pt[0]->{$ENV{'PBPROJ'}}) and ($pt[0]->{$ENV{'PBPROJ'}} =~ /$os-$ver-$arch/)) {
     3428                    $vmbuildlist = "$os-$ver-$arch" if ($vmbuildlist eq "");
     3429                    $vmbuildlist .= ",$os-$ver-$arch";
     3430                }
     3431                if ((defined $pt[1]->{$ENV{'PBPROJ'}}) and ($pt[1]->{$ENV{'PBPROJ'}} =~ /$os-$ver-$arch/)) {
     3432                    $vebuildlist = "$os-$ver-$arch" if ($vebuildlist eq "");
     3433                    $vebuildlist .= ",$os-$ver-$arch";
     3434                }
     3435                if ((defined $pt[2]->{$ENV{'PBPROJ'}}) and ($pt[2]->{$ENV{'PBPROJ'}} =~ /$os-$ver-$arch/)) {
     3436                    $rmbuildlist = "$os-$ver-$arch" if ($rmbuildlist eq "");
     3437                    $rmbuildlist .= ",$os-$ver-$arch";
     3438                }
     3439            }
     3440        }
     3441        # If we want to rebuild automatically, let's do it
     3442        if (defined $opts{'rebuild'}) {
     3443            # SandBox or CMS
     3444            pb_log(0,"Rebuilding from SandBox\n");
     3445            pb_log(0,"for VMs: $vmbuildlist\n") if ($vmbuildlist ne "");
     3446            pb_log(0,"for VEs: $vebuildlist\n") if ($vebuildlist ne "");
     3447            pb_log(0,"for RMs: $rmbuildlist\n") if ($rmbuildlist ne "");
     3448            pb_cms2build("SandBox");
     3449            # Which mode
     3450            $ENV{'PBV'} = $vmbuildlist;
     3451            pb_build2v("vm","build") if ($vmbuildlist ne "");
     3452            $ENV{'PBV'} = $vebuildlist;
     3453            pb_build2v("ve","build") if ($vebuildlist ne "");
     3454            $ENV{'PBV'} = $rmbuildlist;
     3455            pb_build2v("rm","build") if ($rmbuildlist ne "");
     3456        }
     3457        # For the check part this is now finished
     3458        return;
     3459    }
     3460
    33223461    print ANN << "EOF";
    33233462</TR>
     
    37993938        $changes .= " $ENV{'PBBUILDDIR'}/$c" if ($c =~ /\.changes$/);
    38003939    }
    3801     pb_system("debsign -m\'$ENV{'PBPACKAGER'}\' $changes","Signing DEB packages");
     3940    my $debsigncmd = pb_check_req("debsign",1);
     3941    pb_system("$debsigncmd -m\'$ENV{'PBPACKAGER'}\' $changes","Signing DEB packages");
    38023942} else {
    38033943    pb_log(0,"I don't know yet how to sign packages for type $pbos->{'type'}.\nPlease give feedback to dev team\n");
     
    38213961    }
    38223962    if (defined $pt[1]->{$ENV{'PBPROJ'}}) {
    3823         # The 2 lists needs to be grouped with a ',' separating them
     3963        # The 2 lists need to be grouped with a ',' separating them
    38243964        if ($tmpl ne "") {
    38253965            $tmpl .= ",";
Note: See TracChangeset for help on using the changeset viewer.