Changeset 1906 in ProjectBuilder


Ignore:
Timestamp:
Oct 9, 2014, 12:14:44 PM (10 years ago)
Author:
Bruno Cornec
Message:
  • Add a notion of state for building and a preparation phase separated from the build phase (to allow docker reuse of images
File:
1 edited

Legend:

Unmodified
Added
Removed

  • devel/pb/bin/pb

    r1903 r1906  
    198198parameters are packages to build
    199199if not using default list
     200
     201=item B<build2prep>
     202
     203Prepare the environment for build by installing required dependencies. Done once on the build system.
    200204
    201205=item B<build2pkg>
    … …  
    825829} elsif ($action =~ /^sbx2build$/) {
    826830    pb_cms2build("SandBox");
     831} elsif ($action =~ /^build2prep/) {
     832    pb_build2prep();
    827833} elsif ($action =~ /^build2pkg/) {
    828834    pb_build2pkg($do_install);
    … …  
    14651471}
    14661472
     1473sub pb_build2prep {
     1474
     1475    pb_log(0,"INFO: ------ Starting to prepare build environement ------\n");
     1476
     1477    # Get the running distro to build on
     1478    my $pbos = pb_distro_get_context();
     1479
     1480    # If needed we may add repository to the build env
     1481    pb_distro_setuprepo($pbos);
     1482
     1483    # Get list of packages to build
     1484    my $ptr = pb_get_pkg();
     1485    @pkgs = @$ptr;
     1486   
     1487    # Get content saved in cms2build
     1488    my ($pkg) = pb_conf_read("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.pb","pbpkg");
     1489    $pkg = { } if (not defined $pkg);
     1490    my $pbextdir = pb_get_extdir();
     1491
     1492    foreach my $pbpkg (@pkgs) {
     1493        my $vertag = $pkg->{$pbpkg};
     1494        my @buildfiles;
     1495        pb_log(2,"Vertag: $vertag\n");
     1496        # get the version of the current package - maybe different
     1497        ($pbver,$pbtag) = split(/-/,$vertag);
     1498
     1499        my $src2="$ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.pbconf.tar.gz";
     1500        pb_log(2,"Pbconf file: $src2\n");
     1501        pb_log(2,"Working directory: $ENV{'PBBUILDDIR'}\n");
     1502
     1503        my $ftype;
     1504        if ($pbos->{'type'} eq "rpm") {
     1505            $ftype = "\.spec\$";
     1506        } elsif ($pbos->{'type'} eq "deb") {
     1507            $ftype = "\/control\$";
     1508        } elsif ($pbos->{'type'} eq "ebuild") {
     1509            $ftype = "\.ebuild\$";
     1510        } elsif ($pbos->{'type'} eq "tgz") {
     1511            $ftype = "\/pbslack\$";
     1512        } elsif ($pbos->{'type'} eq "pkg") {
     1513            # Solaris - no deps management up to Solaris 10
     1514        } elsif ($pbos->{'type'} eq "hpux") {
     1515            # TODO: HP-UX
     1516        } else {
     1517            die "Unknown OS type format $pbos->{'type'}";
     1518        }
     1519
     1520        @buildfiles = pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/","$ENV{'PBTMP'}","spec");
     1521        pb_log(2,"buildfiles ".Dumper(\@buildfiles)."\n");
     1522        foreach my $f (@buildfiles) {
     1523            if ($f =~ /$ftype/) {
     1524                pb_distro_installdeps($f,$pbos);
     1525                last;
     1526            }
     1527        }
     1528    }
     1529    pb_log(0,"INFO: ------ Finished preparing build environment ------\n");
     1530    open(PKG,">> $ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.pb") || die "Unable to append to $ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.pb";
     1531    print PKG "pbstep $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'} = 2\n";
     1532    close(PKG);
     1533}
     1534
    14671535sub pb_build2pkg {
    14681536
    … …  
    14731541    my $pbos = pb_distro_get_context();
    14741542
    1475     # If needed we may add repository to the build env
    1476     pb_distro_setuprepo($pbos);
    1477 
     1543    # Check whether the preparation phase has already been done or not
     1544    my ($pbstep) = pb_distro_get_param($pbos,pb_conf_read_if("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.pb","pbstep"));
     1545    pb_build2prep() if ((not defined $pbstep) || ($pbstep < 2));
     1546   
    14781547    # Get list of packages to build
    14791548    my $ptr = pb_get_pkg();
    … …  
    15521621            foreach my $f (@specfile) {
    15531622                if ($f =~ /\.spec$/) {
    1554                     # This could cause an issue in // mode
    1555                     pb_distro_installdeps($f,$pbos);
    15561623                    pb_system("$buildcmd $specialdef --define \"packager $ENV{'PBPACKAGER'}\" --define \"_topdir $ENV{'PBBUILDDIR'}\" -ba $f","Building package with $f under $ENV{'PBBUILDDIR'}","verbose");
    15571624                    last;
    … …  
    16241691            #pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbsrc/","$ENV{'PBBUILDDIR'}/debian","src");
    16251692
    1626             # This can create problems in // mode as well
    1627             pb_distro_installdeps("debian/control",$pbos);
    16281693            pb_system("dpkg-buildpackage -us -uc -rfakeroot","Building package","verbose");
    16291694
    … …  
    17041769            foreach my $f (@ebuildfile) {
    17051770                if ($f =~ /\.ebuild$/) {
    1706                     pb_distro_installdeps($f,$pbos);
    17071771                    move($f,"$tmpe/$pbpkg-$pbver.ebuild");
    17081772                    pb_system("cd $tmpe ; ebuild $pbpkg-$pbver.ebuild clean ; ebuild $pbpkg-$pbver.ebuild digest ; ebuild $pbpkg-$pbver.ebuild package","verbose");
    … …  
    17331797            symlink "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}","install" || die "Unable to symlink to pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
    17341798            if (-x "install/pbslack") {
    1735                 pb_distro_installdeps("./install/pbslack",$pbos);
    17361799                pb_system("./install/pbslack","Building software");
    17371800                pb_system("sudo /sbin/makepkg -p -l y -c y $pbpkg","Packaging $pbpkg","verbose");
    … …  
    18711934    print KEEP "$made\n";
    18721935    close(KEEP);
     1936    open(PKG,">> $ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.pb") || die "Unable to append to $ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.pb";
     1937    print PKG "pbstep $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'} = 3\n";
     1938    close(PKG);
    18731939    pb_distro_installdeps(undef,$pbos,$ret) if ($do_install);
    18741940}
    … …  
    23532419    my $context = "$ENV{'PBTMP'}";
    23542420    my %tag;
     2421    my $dkaccount = "";
     2422    my $cmd1 = "";
    23552423    if ($cmt =~ /^VE/) {
    23562424        $tp = pb_path_expand($vepath->{$ENV{'PBPROJ'}});
    2357         $tpdir = pb_path_expand("$tp/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}");
     2425        if ($vetype eq "docker") {
     2426            $tpdir = "/";
     2427        } else {
     2428            $tpdir = pb_path_expand("$tp/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}");
     2429        }
    23582430        $vetype = pb_ve_get_type($vetype);
    23592431        my $arch = pb_get_arch();
    … …  
    23682440            my ($dockerregistry) = pb_conf_get("dockerregistry");
    23692441            $docrepo = pb_ve_docker_repo($dockerregistry->{$ENV{'PBPROJ'}});
    2370             my $cmd1 = pb_check_req("docker",0);
     2442            $cmd1 = pb_check_req("docker",0);
    23712443            #my ($dockeropt) =  pb_conf_get_if("dockeropt");
    23722444            # pbimage is used for docker and is setup to the right name depending on the step:
    23732445            # TODO: This is not coded yet
    23742446            my $pbimage = undef;
    2375             # step 0 : nothing at creation -> tag n-v-a (made below)
     2447            # step 0 : nothing at creation -> tag n-v-a (made in VE.pm)
    23762448            # step 1 : n-v-a + setup -> tag n-v-a-pb
    23772449            # step 2 : n-v-a-pb + build -> tag n-v-a-pb-pbproj
    … …  
    23992471            if ($pbstep == 1) {
    24002472                # setup done as root
    2401                 print DOCKER "USER root\n";
     2473                $dkaccount = "root";
    24022474            } else {
    2403                 print DOCKER "USER pb\n";
    2404             }
    2405             $shcmd = "$cmd1 build -t $tag{$pbstep+1} $context";
     2475                $dkaccount = "pb";
     2476            }
     2477            print DOCKER "USER $dkaccount\n";
     2478            if ($pbstep <= 2) {
     2479                $shcmd = "$cmd1 build -t $tag{$pbstep+1} $context";
     2480            } else {
     2481                # As we are in run phase use docker run. cmd will be completed below
     2482                $shcmd = "$cmd1 run --cidfile=\"$ENV{'PBTMP'}/ctn.cid\" $tag{$pbstep}";
     2483            }
    24062484            #$shcmd = "$cmd1 build $dockeropt->{$ENV{'PBPROJ'}} -t $tag{$pbstep+1} $context";
    24072485        }
    … …  
    24192497            close(PASS);
    24202498        }
     2499        if (($cmt =~ /VE/) && ($vetype eq "docker")) {
     2500            if ($pbstep >= 2) {
     2501                # We need to get the home dir of the target account to deliver in the right place
     2502                $homedir = `$cmd1 run --cidfile="$ENV{'PBTMP'}/ctn.cid" $tag{$pbstep} grep -E '^$dkaccount:' /etc/passwd | cut -d: -f6`;
     2503                chomp($homedir);
     2504                open(CID,"$ENV{'PBTMP'}/ctn.cid") || confess "Unable to open $ENV{'PBTMP'}/ctn.cid";
     2505                my $cid = <CID>;
     2506                close(CID);
     2507                pb_system("$cmd1 rm $cid","","quiet");
     2508            } else {
     2509                $homedir = "/";
     2510            }
     2511
     2512        }
    24212513        $cptarget = "$tpdir/$homedir/$tdir";
    2422         if ($vetype eq "docker") {
    2423             $cptarget = "docker container $shcmd";
    2424         }
    24252514        if ($cmt eq "VEbuild") {
    24262515            $cp2target = "$tpdir/$homedir/$bdir";
    … …  
    24622551    # Do not touch when just announcing
    24632552    if (($cmt ne "Announce") && ($cmt ne "CPAN")) {
    2464         # For docker everything is done in the Dockerfile
    2465         if (($cmt =~ /^VE/) && ($vetype eq "docker")) {
    2466             print DOCKER "RUN mkdir -p $tdir\n";
    2467             print DOCKER "RUN cd $tdir ; for i in $basesrc; do if [ -f \$i ]; then rm -f \$i; fi; done\n";
    2468             print DOCKER "RUN $cmd\n" if ((defined $cmd) && ($cmd ne ""));
     2553        # For docker everything is done in the Dockerfile for steps < 3
     2554        if (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep < 3)) {
     2555            print DOCKER "RUN mkdir -p $cptarget\n";
     2556            print DOCKER "RUN cd $cptarget ; for i in $basesrc; do if [ -f \$i ]; then rm -f \$i; fi; done\n";
    24692557        } else {
    24702558            pb_system("$shcmd \"mkdir -p $tdir ; cd $tdir ; echo \'for i in $basesrc; do if [ -f \$i ]; then rm -f \$i; fi; done\ ; $cmd' | bash -e\"","Preparing $tdir on $cptarget");
    … …  
    24742562    }
    24752563    # For docker everything is done in the Dockerfile
    2476     if (($cmt =~ /^VE/) && ($vetype eq "docker")) {
     2564    if (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep < 3)) {
    24772565        foreach my $f (split(/ +/,$src)) {
    24782566            copy("$f","$context");
    2479             print DOCKER "ADD ".basename($f)." $tdir/\n";
    2480         }
     2567            print DOCKER "COPY ".basename($f)." $cptarget/\n";
     2568        }
     2569        print DOCKER "RUN cd $cptarget ; $cmd\n" if ((defined $cmd) && ($cmd ne ""));
    24812570    } else {
    24822571        pb_system("cd $ENV{'PBBUILDDIR'} ; $cpcmd $src $cptarget 2> /dev/null","$cmt delivery in $cptarget");
    … …  
    24902579            pb_system("$shcmdroot sed '/requiretty/d' /etc/sudoers > /tmp/sudoers.new ; mv /tmp/sudoers.new $tpdir/tmp/sudoers.new ; $shcmdroot mv /tmp/sudoers.new /etc/sudoers ; $shcmdroot chown root:root /etc/sudoers ; $shcmdroot chmod $sudomode /etc/sudoers","Removing again potential requiretty in sudoers as sed -i failed") if (($res ne 0) && (-f "$tpdir/etc/sudoers"));
    24912580            pb_system("$shcmd \"sudo chown -R $mac $tdir\"","Adapt owner in $tdir to $mac");
     2581        } else {
     2582            print DOCKER "RUN sed -i '/requiretty/d' /etc/sudoers" if ($pbstep == 1);
    24922583        }
    24932584    }
    … …  
    25042595    my $shcmdbase = $shcmd;
    25052596    if ($cmt ne "CPAN") {
    2506         if (($cmt =~ /^VE/) && ($vetype eq "docker")) {
     2597        if (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep < 3)) {
    25072598            copy("$pbscript1","$context");
    25082599            my $s = basename($pbscript1);
    2509             print DOCKER "ADD $s $tdir/\n";
    2510             # sleep to avoid text file busy errors
    2511             print DOCKER "RUN sleep 2\n";
    2512             print DOCKER "RUN cd $tdir ; chmod 755 $s ; ./$s ; if [ '$pbkeep' == '0' ]; then rm -f $s; fi\n";
     2600            print DOCKER "COPY $s $cptarget/\n";
     2601            # call directly with perl to avoid text file busy error messages
     2602            print DOCKER "RUN cd $cptarget ; perl ./$s ; if [ '$pbkeep' == '0' ]; then rm -f $s; fi\n";
     2603            print DOCKER "RUN rm -rf $cptarget/*\n";
     2604            print DOCKER "RUN chown -R pb $cptarget\n" if ($pbstep == 1);
    25132605            close(DOCKER);
    25142606        } else {
    … …  
    26212713my $pbscript=shift;
    26222714my $vtype=shift;
    2623 my $pbstep=shift;   # Which step are we in (0: create, 1: setup, 2: build, 3: use)
     2715my $pbstep=shift;   # Which step are we in (0: create, 1: setup pb, 2: prep pbproj, 3: use to build)
    26242716my $pbforce=shift || 0; # Force stop of VM. Default not.
    26252717my $snapme=shift || 0;  # Do we have to create a snapshot. Default not.
    … …  
    26852777    my $pbscript = $pbscript->{$v} if (defined $pbscript->{$v});
    26862778    $pbscript = $pbscript->{'default'} if (not defined $pbscript);
    2687     die "No script defined so unable to launch it in $v" if (not defined $pbscript);
     2779    confess "No script defined so unable to launch it in $v" if (not defined $pbscript);
    26882780    ($vexist,$vpid) = pb_launchv($pbscript,$vtype,$v,$action,$pbstep,$pbforce,$snapme,$pbsnap,$pbimage);
    26892781   
    … …  
    27362828# Launch the VMs/VEs
    27372829if ($vtype eq "vm") {
    2738     die "-i image parameter needed" if (((not defined $pbimage) || ($pbimage eq "")) && ($pbstep == 0));
     2830    confess "-i image parameter needed" if (((not defined $pbimage) || ($pbimage eq "")) && ($pbstep == 0));
    27392831
    27402832    my ($ptr,$ptr2,$vmpath,$vmport,$vms) = pb_conf_get("vmtype","vmcmd","vmpath","vmport","vmsize");
    … …  
    29423034
    29433035my $vtype = shift;
    2944 my $action = shift || "build";
    2945 my $pbstep = 2;
     3036my $action = shift || "build";   #build, test or prep
    29463037my $pbforce=shift || 0;         # Force stop of VM. Default not.
     3038
     3039my $pbstep;
     3040
     3041if ($action eq "prep") {
     3042    $pbstep = 2;
     3043} else {
     3044    $pbstep = 3;
     3045}
    29473046
    29483047# We prepare scripts for all V of this vtype to allow sharing that part
    … …  
    29743073foreach my $v (@$v) {
    29753074    $counter++;
     3075    $pbscript{$v} = "$ENV{'PBDESTDIR'}/pb$action-$v";
    29763076    # Modulo 2 * pbparallel (to avoid synchronization problems)
    29773077    $counter = 1 if ((defined $pbparallel) && ($counter > 2 * $pbparallel));
    29783078    $pm->start($counter) and next if (defined $pbparallel);
     3079    #
    29793080    # Prepare the script to be executed on the VM/VE/RM
    2980     # in $ENV{'PBDESTDIR'}/pbscript.$$
    2981     #
    2982     $pbscript{$v} = "$ENV{'PBDESTDIR'}/pbscript.$$";
    29833081    open(SCRIPT,"> $pbscript{$v}") || die "Unable to create $pbscript{$v}";
    29843082    print SCRIPT "#!/bin/bash\n";
    … …  
    29983096
    29993097    print SCRIPT "echo ... Execution needed\n";
    3000     print SCRIPT "echo ==== Start of script $$ for $vtype $v =====\n";
     3098    print SCRIPT "echo ==== Start of script for $vtype $v =====\n";
    30013099    print SCRIPT "# This is in directory delivery\n";
    30023100    print SCRIPT "# Setup the variables required for building\n";
    … …  
    30593157
    30603158    if (($action eq "test") && (! -x "$ENV{'PBDESTDIR'}/pbtest")) {
    3061             die "No test script ($ENV{'PBDESTDIR'}/pbtest) found when in test mode. Aborting ...";
     3159            confess "No test script ($ENV{'PBDESTDIR'}/pbtest) found when in test mode. Aborting ...";
    30623160    }
    30633161    print SCRIPT "pb --version\n";
    3064     print SCRIPT "pb $verbose -p $ENV{'PBPROJ'} $action"."2pkg $p\n";
     3162    my $act = "$action"."2pkg";
     3163    $act = "build2prep" if ($action eq "prep");
     3164    print SCRIPT "pb $verbose -p $ENV{'PBPROJ'} $act $p\n";
    30653165
    30663166    if (($vtype eq "ve") && ($vetype ne "docker")) {
Note: See TracChangeset for help on using the changeset viewer.