Changeset 2426


Ignore:
Timestamp:
Sep 1, 2019, 3:18:34 AM (7 weeks ago)
Author:
bruno
Message:

Start adding install and test target for pb - local install works already

Location:
devel
Files:
4 edited

Legend:

Unmodified
Added
Removed
  • devel/pb-modules/etc/pb.yml.pod

    r2410 r2426  
    2727=over 4
    2828
    29 =item B<addrepo>
    30 
    31  Nature: Optional
    32  Key: OS (could be from the most generic up to the most specific from ostype, osfamily, os, os-ver, os-ver-arch). The family name is generaly used here.
    33  Value: comma separated list of URLs that point to repository files, or packages to install. The values may not include substitutions.
    34  Conffile: project
    35  Example: addrepo:
     29=item B<addbuildrepo>
     30
     31 Nature: Optional
     32 Key: OS (could be from the most generic up to the most specific from ostype, osfamily, os, os-ver, os-ver-arch). The family name is generaly used here.
     33 Value: comma separated list of URLs that point to repository files, or packages to install at project build time. The values may not include substitutions.
     34 Conffile: project
     35 Example: addbuildrepo:
    3636            centos-5-x86_64: file:///prj/extras.repo,http://mirror.centos.org/centos/5.8/extras/x86_64/RPMS/chrpath-0.13-3.el5.centos.x86_64.rpm
     37
     38=item B<addinstallrepo>
     39
     40 Nature: Optional
     41 Key: OS (could be from the most generic up to the most specific from ostype, osfamily, os, os-ver, os-ver-arch). The family name is generaly used here.
     42 Value: comma separated list of URLs that point to repository files, or packages to install at project installation time. The values may not include substitutions.
     43 Conffile: project
     44 Example: addinstallrepo:
     45            centos-7-x86_64: ftp://ftp.project-builder.org/centos/7/x86_64/pb.repo
     46
     47=item B<addtestrepo>
     48
     49 Nature: Optional
     50 Key: OS (could be from the most generic up to the most specific from ostype, osfamily, os, os-ver, os-ver-arch). The family name is generaly used here.
     51 Value: comma separated list of URLs that point to repository files, or packages to install at project test time. The values may not include substitutions.
     52 Conffile: project
     53 Example: addtestrepo:
     54            centos-7-x86_64: ftp://ftp.project-builder.org/centos/7/x86_64/pb.repo
    3755
    3856=item B<cachedir>
  • devel/pb-modules/lib/ProjectBuilder/Conf.pm

    r2402 r2426  
    2929 
    3030our @ISA = qw(Exporter);
    31 our @EXPORT = qw(pb_conf_init pb_conf_add pb_conf_read pb_conf_read_if pb_conf_write pb_conf_get pb_conf_get_if pb_conf_get_all pb_conf_get_hash pb_conf_cache pb_conf_update_v0);
     31our @EXPORT = qw(pb_conf_init pb_conf_add pb_conf_read pb_conf_read_if pb_conf_write pb_conf_get pb_conf_get_if pb_conf_get_all pb_conf_get_hash pb_conf_cache pb_conf_update_v0 pb_conf_get_in_hash_if);
    3232($VERSION,$REVISION,$PBCONFVER) = pb_version_init();
    3333
  • devel/pb-modules/lib/ProjectBuilder/Distribution.pm

    r2410 r2426  
    6969=over 4
    7070
    71 =item B<pb_distro_conffile>
    72 
    73 This function returns the mandatory configuration file used for distribution/OS detection
     71=item B<pb_distro_api>
     72
     73This function returns the mandatory configuration file used for api
    7474
    7575=cut
     
    605605
    606606pb_log(3, "INFO: Adding osrepo from config file\n");
    607 return(pb_distro_setuprepo_gen_conf($pbos,pb_distro_conffile(),"osrepo"));
     607my %h;
     608my $h = \%h;
     609# Adds conf file for availability of conf elements either from the local build env or from a VE/VM/RM in which the conf file has been passed
     610$h = pb_conf_cache(pb_distro_conffile(),$h);
     611my ($osrepo) = pb_conf_get_in_hash_if($h,"osrepo");
     612return(pb_distro_setuprepo_gen($pbos,$osrepo->{$ENV{'PBPROJ'}}));
    608613}
    609614
    610615=item B<pb_distro_setuprepo>
    611616
    612 This function sets up potential additional repository to the build environment
     617This function sets up potential additional repository to the build/install/test environment
    613618
    614619=cut
     
    617622
    618623my $pbos = shift;
    619 
    620 pb_log(3, "INFO: Adding addrepo from config file\n");
    621 return(pb_distro_setuprepo_gen_conf($pbos,"$ENV{'PBDESTDIR'}/pbrc.yml","addrepo"));
    622 }
    623 =item B<pb_distro_setuprepo_gen_conf>
    624 
    625 This function sets up in a generic way potential additional repository using conf files
    626 
    627 =cut
    628 
    629 sub pb_distro_setuprepo_gen_conf {
    630 
    631 my $pbos = shift;
    632 my $pbconf = shift;
    633 my $pbkey = shift;
    634 
    635 return undef if (not defined $pbconf);
    636 return undef if (not defined $pbkey);
    637 my ($addrepo) = pb_conf_read($pbconf,$pbkey);
    638 return undef if (not defined $addrepo);
    639 
    640 my $param = pb_distro_get_param($pbos,$addrepo);
    641 return undef if ($param eq "");
    642 
    643 pb_log(1, "INFO: Adding repos $param\n");
    644 return(pb_distro_setuprepo_gen($pbos,$param));
    645 }
    646 
     624my $repotype = shift;
     625
     626my %h;
     627my $h = \%h;
     628# Adds conf file for availability of conf elements either from the local build env or from a VE/VM/RM in which the conf file has been passed
     629$h = pb_conf_cache("$ENV{'PBROOTDIR'}/$ENV{'PBPROJ'}.yml",$h) if ((defined $ENV{'PBROOTDIR'}) && (-f "$ENV{'PBROOTDIR'}/$ENV{'PBPROJ'}.yml"));
     630$h = pb_conf_cache("$ENV{'PBDESTDIR'}/pbrc.yml",$h) if ((defined $ENV{'PBDESTDIR'}) && (-f "$ENV{'PBDESTDIR'}/pbrc.yml"));
     631
     632pb_log(3, "INFO: Adding addrepo from config file for $repotype\n");
     633if ($repotype =~ /install/) {
     634    # Give a probable default if nothing is provided to avoid overloading conf files
     635    #
     636    my ($addrepo) = pb_conf_get_in_hash_if($h,"addinstallrepo");
     637    if ((not defined $addrepo) || (not defined $addrepo->{$ENV{'PBPROJ'}})) {
     638        my ($pbrepo) = pb_conf_get_in_hash_if($h,"pbrepo");
     639        return undef if (not defined $pbrepo);
     640        my $url = "$pbrepo->{$ENV{'PBPROJ'}}";
     641        my ($testver,$delivery) = pb_conf_get_in_hash_if($h,"testver","delivery");
     642        $delivery->{$ENV{'PBPROJ'}} = "" if (not defined $delivery->{$ENV{'PBPROJ'}});
     643        $url .= "/$delivery->{$ENV{'PBPROJ'}}/";
     644        my $repotag = "";
     645        $repotag = "-$delivery->{$ENV{'PBPROJ'}}" if ($delivery->{$ENV{'PBPROJ'}} ne "");
     646        $url .= "/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}/$ENV{PBPROJ}$repotag.";
     647        my $ext = "";
     648        if ($pbos->{'type'} eq "rpm") {
     649            $ext = "repo";
     650            if ($pbos->{'family'} eq "md") {
     651                $ext = "addmedia";
     652            }
     653        } elsif ($pbos->{'type'} eq "deb") {
     654            $ext = ".sources.list";
     655        }
     656        $url .= $ext;
     657        return(pb_distro_setuprepo_gen($pbos,$url));
     658    } else {
     659        my ($addrepo) = pb_conf_get_in_hash_if($h,"addinstallrepo");
     660        return(pb_distro_setuprepo_gen($pbos,$addrepo->{$ENV{'PBPROJ'}}));
     661    }
     662} else {
     663    my ($addrepo) = pb_conf_get_in_hash_if($h,"add".$repotype."repo");
     664    return(pb_distro_setuprepo_gen($pbos,$addrepo->{$ENV{'PBPROJ'}}));
     665}
     666}
    647667
    648668=item B<pb_distro_setuprepo_gen>
  • devel/pb/bin/pb

    r2396 r2426  
    391391Update the distribution in the remote machine
    392392
     393=item B<install2pkg>
     394
     395Install the project packages locally
     396
    393397=item B<test2pkg>
    394398
    395 Test a package locally
     399Test the project packages locally
     400
     401=item B<install2vm>
     402
     403Install the project packages in a virtual machine
     404
     405=item B<install2ve>
     406
     407Install the project packages in a virtual environment
     408
     409=item B<install2rm>
     410
     411Install the project packages in a remote machine
    396412
    397413=item B<test2vm>
    398414
    399 Test a package in a virtual machine
     415Test the project packages in a virtual machine, after installation
    400416
    401417=item B<test2ve>
    402418
    403 Test a package in a virtual environment
     419Test the project packages in a virtual environment, after installation
    404420
    405421=item B<test2rm>
    406422
    407 Test a package in a remote machine
     423Test the project packages in a remote machine, after installation
    408424
    409425=item B<checkssh>
     
    702718You may also need to specify additional repository files to use or rpms to install. Note the URL for repositories is not the URL of the repository, but the URL of a file that can be put in the yum.repos.d or apt.sources.d directory.
    703719
    704   addrepo:
     720  addbuildrepo:
    705721    centos-5-i386: http://localhost/pb/centos-extras.repo,http://mirror.centos.org/centos/5/extras/i386/RPMS/chrpath-0.13-3.el5.centos.i386.rpm
    706722
     
    911927    pb_cms2build("CMS");
    912928    pb_build2ssh();
    913 } elsif ($action =~ /^sbx2ssh$/) {
     929} elsif (($action =~ /^sbx2ssh$/) || ($action =~ /^sbx2pkg2ssh$/)) {
    914930    pb_cms2build("SandBox");
    915931    pb_build2ssh();
     
    955971    pb_cms2build("SandBox");
    956972    pb_build2v("vm","build");
     973} elsif ($action =~ /^install2vm$/) {
     974    pb_build2v("vm","install");
     975} elsif ($action =~ /^install2ve$/) {
     976    pb_build2v("ve","install");
     977} elsif ($action =~ /^install2rm$/) {
     978    pb_build2v("rm","install");
    957979} elsif ($action =~ /^launchvm$/) {
    958980    pb_parallel_launchv(undef,"vm",undef,3,$pbimage);
     
    10801102} elsif ($action =~ /^snapvm$/) {
    10811103    pb_snap2v("vm");
     1104} elsif ($action =~ /^install2pkg$/) {
     1105    pb_install2pkg();
    10821106} elsif ($action =~ /^test2pkg$/) {
    10831107    pb_test2pkg();
     
    15871611        # Do we have additional script to run to prepare the environement for the project ?
    15881612        # Then include it in the pbconf delivery
    1589         foreach my $pbvf (<$ENV{'PBROOTDIR'}/$pbpkg/pbv*.pre>,<$ENV{'PBROOTDIR'}/$pbpkg/pbv*.post>, <$ENV{'PBROOTDIR'}/$pbpkg/pbtest*>) {
     1613        foreach my $pbvf (<$ENV{'PBROOTDIR'}/$pbpkg/pbv*.pre>,<$ENV{'PBROOTDIR'}/$pbpkg/pbv*.post>) {
    15901614            pb_log(1,"Handling file $pbvf\n");
    15911615            if (-x "$pbvf") {
     
    16441668}
    16451669
    1646 sub pb_test2pkg {
     1670sub pb_install2pkg {
    16471671    # Get the running distro to test on
    16481672    my $pbos = pb_distro_get_context();
     
    16541678
    16551679    # Additional potential repo
    1656     my $forcerepo = pb_distro_setuprepo($pbos);
     1680    my $forcerepo = pb_distro_setuprepo($pbos,"install");
     1681    foreach my $pbpkg (@pkgs) {
     1682        # We need to install the package to test, and deps brought with it
     1683        pb_distro_installdeps(undef,$pbos,$forcerepo,$pbpkg,"local");
     1684    }
     1685}
     1686
     1687sub pb_test2pkg {
     1688    # Get the running distro to test on
     1689    my $pbos = pb_distro_get_context();
     1690
     1691    # Get list of packages to test
     1692    # Get content saved in cms2build
     1693    my $ptr = pb_get_pkg();
     1694    @pkgs = @$ptr;
     1695
     1696    # Additional potential repo
     1697    my $forcerepo = pb_distro_setuprepo($pbos,"test");
    16571698    foreach my $pbpkg (@pkgs) {
    16581699        # We need to install the package to test, and deps brought with it
     
    16701711
    16711712    # If needed we may add repository to the build env
    1672     my $forcerepo = pb_distro_setuprepo($pbos);
     1713    my $forcerepo = pb_distro_setuprepo($pbos,"build");
    16731714    pb_log(1,"Forcing full package install with forcerepo\n") if (defined $forcerepo);
    16741715
     
    17271768    my $do_install = shift;
    17281769
    1729         # Call build2prep in case we have new deps we want to take in account
    1730         # without redoing the prep phase (can be long and that helps debugging
    1731         # build dependencies issues
     1770    # Call build2prep in case we have new deps we want to take in account
     1771    # without redoing the prep phase (can be long and that helps debugging
     1772    # build dependencies issues
    17321773    pb_build2prep();
    17331774    pb_log(0,"INFO: ------ Starting to build package ------\n");
     
    21752216    my $vpid = shift;           # 0 is FALSE
    21762217    my $snapme = shift;         # 0 is FALSE
    2177     my $pbstep = shift;         # 3 is usage of container
     2218    my $pbstep = shift;         # == 3 is usage of container
    21782219
    21792220    $vexist = 0 if (not defined $vexist);
     
    22902331    } elsif ($cmt =~ /(V[EM]|RM)Script/) {
    22912332        $src="$src $pbscript{$v}";
    2292     } elsif ($cmt =~ /(V[EM]|RM)test/) {
    2293         $src="$src $ENV{'PBROOTDIR'}/$ENV{'PBPROJ'}.yml $ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml $ENV{'PBETC'} $ENV{'PBDESTDIR'}/pbrc.yml $pbscript{$v} $ENV{'PBDESTDIR'}/pbtest";
     2333    } elsif ($cmt =~ /(V[EM]|RM)(install|test)/) {
     2334        $src="$src $ENV{'PBROOTDIR'}/$ENV{'PBPROJ'}.yml $ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml $ENV{'PBETC'} $ENV{'PBDESTDIR'}/pbrc.yml $pbscript{$v}";
    22942335    } elsif (($cmt eq "Announce") || ($cmt eq "Web") || ($cmt eq "CPAN")) {
    22952336        $src="$src $ENV{'PBTMP'}/pbscript";
     
    23712412    } elsif ($cmt eq "CPAN") {
    23722413        $tdir = "$remdir";
    2373     } elsif ($cmt =~ /(V[EM]|RM)(build|test|prep)/) {
     2414    } elsif ($cmt =~ /(V[EM]|RM)(build|test|install|prep)/) {
    23742415        $tdir = $remdir."/$ENV{'PBPROJ'}/pbdelivery";
    23752416        $bdir = $remdir."/$ENV{'PBPROJ'}/pbbuild";
     
    26722713            # step 2 : n-v-a-pb + build -> tag n-v-a-pb-pbproj
    26732714            # step 3 : n-v-a-pb-pbproj at use
     2715            # step 4 : n-v-a-pb-pbproj-install based on #3
     2716            # step 5 : n-v-a-pb-pbproj-test based on #4
    26742717            if ((not defined $pbimage) || ($pbimage eq "")) {
    26752718                # If no image name given, create a naming convention
     
    26772720                $tag{2} = "$tag{1}-pb";
    26782721                $tag{3} = "$tag{2}-$ENV{'PBPROJ'}";
     2722                $tag{4} = "$tag{3}-install";
     2723                $tag{5} = "$tag{3}-test";
    26792724            } else {
    26802725                # If we were given an image name, just use it
     
    26822727                $tag{2} = $pbimage;
    26832728                $tag{3} = $pbimage;
     2729                $tag{4} = $pbimage;
     2730                $tag{5} = $pbimage;
    26842731            }
    26852732            # If we do not create the image, then use the one we should have
     
    27382785        }
    27392786        $cptarget = "$tpdir/$homedir/$tdir";
    2740         if ($cmt =~ /VE(build|prep)/) {
     2787        if ($cmt =~ /VE(build|prep|install|test)/) {
    27412788            $cp2target = "$tpdir/$homedir/$bdir";
    27422789        }
     
    29092956        print DOCKER "(cd $homedir/$tdir-2 ; ls -al)\n";
    29102957        print DOCKER pb_get_content($pbscript2);
    2911         print DOCKER "\n# Get content back from guest in a host dir\n";
    2912         # Host dir seen as TDIR in container
    2913         $cp2target = "$ENV{'PBBUILDDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
    2914         pb_mkdir_p("$cp2target");
    2915         chmod 0777,"$cp2target";
    2916         print DOCKER "TDIR=$homedir/$bdir-2/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}\n";
    2917         print DOCKER "for i in $homedir/$bdir/*; do\n";
    2918         print DOCKER "  # Cleanup first to avoid rights issues for root owned files e.g.\n";
    2919         print DOCKER "  j=\`basename \$i\`\n";
    2920         print DOCKER "  if [ -d \$TDIR/\$j ]; then\n";
    2921         print DOCKER "    rmdir --ignore-fail-on-non-empty \$TDIR/\$j\n";
    2922         print DOCKER "  else\n";
    2923         print DOCKER "    rm -f \$TDIR/\$j\n";
    2924         print DOCKER "  fi\n";
    2925         print DOCKER "  cp -r \$i \$TDIR\n";
    2926         print DOCKER "done\n";
    2927         print DOCKER "find \$TDIR -type f -print0 | xargs -0 chmod 644 && /bin/true\n";
    2928         print DOCKER "find \$TDIR -type d -print0 | xargs -0 chmod 777 2> /dev/null && /bin/true\n";
    2929         #print DOCKER "ls -al \$TDIR\n";
     2958        # Not useful for test or install
     2959        if ($pbstep == 3) {
     2960            print DOCKER "\n# Get content back from guest in a host dir\n";
     2961            # Host dir seen as TDIR in container
     2962            $cp2target = "$ENV{'PBBUILDDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
     2963            pb_mkdir_p("$cp2target");
     2964            chmod 0777,"$cp2target";
     2965            print DOCKER "TDIR=$homedir/$bdir-2/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}\n";
     2966            print DOCKER "for i in $homedir/$bdir/*; do\n";
     2967            print DOCKER "  # Cleanup first to avoid rights issues for root owned files e.g.\n";
     2968            print DOCKER "  j=\`basename \$i\`\n";
     2969            print DOCKER "  if [ -d \$TDIR/\$j ]; then\n";
     2970            print DOCKER "    rmdir --ignore-fail-on-non-empty \$TDIR/\$j\n";
     2971            print DOCKER "  else\n";
     2972            print DOCKER "    rm -f \$TDIR/\$j\n";
     2973            print DOCKER "  fi\n";
     2974            print DOCKER "  cp -r \$i \$TDIR\n";
     2975            print DOCKER "done\n";
     2976            print DOCKER "find \$TDIR -type f -print0 | xargs -0 chmod 644 && /bin/true\n";
     2977            print DOCKER "find \$TDIR -type d -print0 | xargs -0 chmod 777 2> /dev/null && /bin/true\n";
     2978            #print DOCKER "ls -al \$TDIR\n";
     2979        }
    29302980        print DOCKER "exit 0\n";
    29312981        close(DOCKER);
     
    30733123my $vtype = shift;
    30743124my $action = shift;         # It an action is defined then use send2target
    3075 my $pbstep=shift;           # Which step are we in (0: create, 1: setup, 2: build 3: use)
     3125my $pbstep=shift;           # Which step are we in (0: create, 1: setup, 2: build 3: use 4: install 5: test)
    30763126my $pbimage=shift;          # Which image to use to start the VM/VE
    30773127my $pbforce=shift;          # Force stop of VM. Default not.
     
    31543204my $v = shift;              # Only 1 VM/VE/RM treated here
    31553205my $action = shift;         # It an action is defined then use send2target
    3156 my $pbstep=shift;           # Which step are we in (0: create, 1: setup, 2: build 3: use)
     3206my $pbstep=shift;           # Which step are we in (0: create, 1: setup, 2: build 3: use 4: install 5: test)
    31573207my $pbforce=shift;          # Force stop of VM. Default not.
    31583208my $snapme = shift;         # By default do not snap a VM/VE/RM
     
    34203470if ($action eq "prep") {
    34213471    $pbstep = 2;
    3422 } else {
     3472} elsif ($action eq "install") {
     3473    $pbstep = 4;
     3474} elsif ($action eq "test") {
     3475    $pbstep = 5;
     3476} elsif ($action eq "build") {
    34233477    $pbstep = 3;
    34243478    # Need to be traversed by others if using containers
     
    34263480        chmod 0751,"$ENV{'PBBUILDDIR'}";
    34273481    }
     3482} else {
     3483    pb_log(0,"pb_build2v: unknown action $action\n");
     3484    confess "Unable to continue";
    34283485}
    34293486
     
    35473604    print SCRIPT "echo $action"."ing packages on $vtype...\n";
    35483605
    3549     if (($action eq "test") && (! -x "$ENV{'PBDESTDIR'}/pbtest")) {
    3550         cluck "No test script ($ENV{'PBDESTDIR'}/pbtest) found when in test mode. Aborting ...";
    3551         next;
    3552     }
    35533606    print SCRIPT "pb --version\n";
    35543607    my $act = "$action"."2pkg";
     
    40664119    }
    40674120
    4068     my ($addrepo) = pb_conf_read("$ENV{'PBDESTDIR'}/pbrc.yml","addrepo");
     4121    my ($addrepo) = pb_conf_read("$ENV{'PBDESTDIR'}/pbrc.yml","addbuildrepo");
    40694122    my $pbrepo = pb_distro_get_param($pbos,$addrepo);
    40704123    print SCRIPT << 'EOF';
Note: See TracChangeset for help on using the changeset viewer.