Changeset 2429 in ProjectBuilder for devel

Timestamp:

Sep 2, 2019, 12:39:58 PM (6 years ago)

Author:

Bruno Cornec

Message:

More improvements around CI/CD

• Working sbx|build2docker with pb_all_docker,
• ability to force installtype,
• fix for pkg and file base install of pb
• fix setuposrepo to work on a per distro base, not project

Location:

devel

Files:

• pb-modules/lib/ProjectBuilder/Distribution.pm (modified) (1 diff)
• pb/bin/pb (modified) (13 diffs)

devel/pb-modules/lib/ProjectBuilder/Distribution.pm

@@ -627 +627 @@
 $h = pb_conf_cache(pb_distro_conffile(),$h);
 my ($osrepo) = pb_conf_get_in_hash_if($h,"osrepo");
-return(pb_distro_setuprepo_gen($pbos,$osrepo->{$ENV{'PBPROJ'}}));
+my $repo = pb_distro_get_param($pbos,$osrepo);
+return(pb_distro_setuprepo_gen($pbos,$repo));
 }
 

devel/pb/bin/pb

@@ -993 +993 @@
     pb_newver();
 } elsif ($action =~ /^sbx2docker$/) {
-    my $savproj = $ENV{'PBPROJ'};
-    my $pkg;
-    my @pkg;
-    if ($#ARGV == -1) {
-        $pkg = pb_cms_get_pkg($defpkgdir,$extpkgdir);
-        @pkgs = @$pkg;
-    } else {
-        @pkgs = @ARGV;
-    }
-    #pb_log(1,"Packages to print:".Dumper(@pkgs)."\n");
-    
-    # Get the list of all VE we need to work on
-    my ($vm,$all);
-    if (not defined $ENV{'PBV'}) {
-        ($vm,$all) = pb_get2v("ve");
-    } else {
-        @$vm = split(/,/,$ENV{'PBV'});
-    }
-    
-    # TODO: parallelize
-    foreach my $v (@$vm) {
-        my $done = 0;
-        my $pbos = pb_distro_get_context($v);
-        # Docker images are (for now !) regular and use distro:ver
-        $pbimage = "$pbos->{'name'}".":$pbos->{'version'}";
-        # the first steps needs to be done with the pb project
-        $ENV{'PBPROJ'} = $appname;
-        my $found = pb_ve_docker_get_image("$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}");
-        if (($pbforce == 1) || (not defined $found)) {
-            # pb -p pb -m distro-ver-arch newve -i distro:ver
-            pb_log(0,"Docker container $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'} not found, creating it\n");
-            pb_parallel_launchv(undef,"ve",undef,0,$pbimage);
-        }
-        $found = pb_ve_docker_get_image("$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-pb");
-        if (($pbforce == 1) || (not defined $found)) {
-            pb_log(0,"Docker container $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-pb not found, creating it\n");
-            # pb -p pb -m distro-ver-arch setupve
-            if ($savproj eq "pb") {
-                pb_cms2build("SandBox");
-                $done = 1;
-                pb_setup2v("ve","SandBox");
-            } else {
-                pb_setup2v("ve");
-            }
-        }
-        # Now back to our original project
-        $ENV{'PBPROJ'} = $savproj;
-        $found = pb_ve_docker_get_image("$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-pb-$ENV{'PBPROJ'}");
-        if (($pbforce == 1) || (not defined $found)) {
-            pb_log(0,"Docker container $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-pb-$ENV{'PBPROJ'} not found, creating it\n");
-            # pb -p pb -m distro-ver-arch prepve
-            if ($savproj eq "pb") {
-                pb_cms2build("SandBox");
-                $done = 1;
-            }
-            pb_build2v("ve","prep");
-        }
-        pb_log(0,"Instantiating a Docker container from $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-pb-$ENV{'PBPROJ'}\n");
-        if (($savproj ne "pb") || ($done == 0)) {
-            pb_cms2build("SandBox");
-        }
-        pb_build2v("ve","build");
-    }
+    pb_cms2build("SandBox");
+    pb_all_docker($action);
+} elsif ($action =~ /^build2docker$/) {
+    pb_all_docker($action);
 } elsif ($action =~ /^newve$/) {
     pb_parallel_launchv(undef,"ve",undef,0,$pbimage);
@@ -1695 +1636 @@
 
     # Additional potential repo
-    my $forcerepo = pb_distro_setuprepo($pbos,"test");
+    pb_distro_setuprepo($pbos,"test");
     foreach my $pbpkg (@pkgs) {
     }
@@ -2718 +2659 @@
                 $tag{2} = "$tag{1}-pb";
                 $tag{3} = "$tag{2}-$ENV{'PBPROJ'}";
-                $tag{4} = "$tag{2}-install";
-                $tag{5} = "$tag{2}-test";
+                $tag{4} = "$tag{3}-install";
+                $tag{5} = "$tag{3}-test";
             } else {
                 # If we were given an image name, just use it
@@ -2747 +2688 @@
             }
             print DOCKER "USER $dkaccount\n";
-            if ($pbstep <= 2) {
+            if (($pbstep <= 2) || ($pbstep > 3)) {
                 $shcmd = "$cmd1 build -t $tag{$pbstep+1} $context";
+                #$shcmd = "$cmd1 build $dockeropt->{$ENV{'PBPROJ'}} -t $tag{$pbstep+1} $context";
             } else {
                 # As we are in run phase use docker run. cmd will be completed below
@@ -2754 +2696 @@
                 $shcmd = "$cmd1 run $rmatend --cidfile=\"$cidfile\" --user $dkaccount --name $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-$cmt";
             }
-            #$shcmd = "$cmd1 build $dockeropt->{$ENV{'PBPROJ'}} -t $tag{$pbstep+1} $context";
         }
         $shcmd = "setarch i386 $shcmd" if (($pbos->{'arch'} =~ /i[3456]86/) && ($arch eq 'x86_64'));
@@ -3802 +3743 @@
 my $vtype = shift;
 my $sbx = shift;
+my $installtype = shift;        # either "file" or "pkg" to overwrite conf file
 my $pbstep = 1;
 
@@ -4124 +4066 @@
 pb_apply_conf_proxy($pbos);
 
-# First install all required packages
+# Clean up packages DB
 pb_system("dnf clean all","Cleaning dnf env","mayfail") if (($pbos->{'name'} eq "fedora") && ($pbos->{'version'} >= 22));
 pb_system("yum clean all","Cleaning yum env","mayfail") if ((($pbos->{'name'} eq "fedora") && ($pbos->{'version'} < 22)) || ($pbos->{'name'} eq "asianux") || ($pbos->{'name'} eq "rhel"));
-my ($ospkgdep) = pb_conf_get_if("ospkgdep");
-
-my $pkgdep = pb_distro_get_param($pbos,$ospkgdep);
-my $forcerepo;
-EOF
-    print SCRIPT "\$forcerepo = pb_distro_setuprepo_gen(\$pbos,\"$pbrepo\");\n" if ((defined $pbrepo) && ($pbrepo ne ""));
-    print SCRIPT << 'EOF';
-pb_distro_installdeps(undef,$pbos,$forcerepo,pb_distro_only_deps_needed($pbos,join(' ',split(/,/,$pkgdep))));
 
 EOF
@@ -4140 +4074 @@
     # Install from sandbox mean a file base install
     $itype = "file" if (defined $sbx);
+    $itype = $installtype if (defined $installtype);
     if ($itype =~ /^file/) {
+        my ($addrepo) = pb_conf_read("$ENV{'PBDESTDIR'}/pbrc.yml","addbuildrepo");
+        my $pbrepo = pb_distro_get_param($pbos,$addrepo);
+        print SCRIPT << 'EOF';
+
+# First install all required packages
+my ($ospkgdep) = pb_conf_get_if("ospkgdep");
+my $pkgdep = pb_distro_get_param($pbos,$ospkgdep);
+my $forcerepo;
+EOF
+        print SCRIPT "\$forcerepo = pb_distro_setuprepo_gen(\$pbos,\"$pbrepo\");\n" if ((defined $pbrepo) && ($pbrepo ne ""));
+        print SCRIPT << 'EOF';
+pb_distro_installdeps(undef,$pbos,$forcerepo,pb_distro_only_deps_needed($pbos,join(' ',split(/,/,$pkgdep))));
+
+EOF
         my $cmdget;
-        my ($pbver1,$pbver2,$pbextdir);
-        my ($tmp1,$tmp2);
         if (defined $sbx) {
             # Install from sandbox mean using the result of the just passed sbx2build command
             # Get content saved in cms2build
             my ($pkg) = pb_conf_read("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml","pbpkg");
-            $pbextdir = pb_get_extdir();
             confess "Unable to get package list" if (not defined $pkg);
-
+    
             # We consider 2 specific packages
             my $vertag1 = $pkg->{"ProjectBuilder"};
@@ -4157 +4103 @@
             pb_log(2,"Vertag1: $vertag1\n");
             pb_log(2,"Vertag2: $vertag2\n");
-            ($pbver1,$tmp1) = split(/-/,$vertag1);
-            ($pbver2,$tmp2) = split(/-/,$vertag2);
+            my ($pbver1,$tmp1) = split(/-/,$vertag1);
+            my ($pbver2,$tmp2) = split(/-/,$vertag2);
+            my $pbextdir = pb_get_extdir();
             $cmdget = "mv /tmp/ProjectBuilder-$pbver1$pbextdir.tar.gz ProjectBuilder-latest.tar.gz ; mv /tmp/project-builder-$pbver2$pbextdir.tar.gz project-builder-latest.tar.gz";
         } else {
@@ -4189 +4136 @@
         print SCRIPT " $cmdget ; ";
         print SCRIPT << "EOF"
-gzip -cd ProjectBuilder-latest.tar.gz | tar xf - ; cd ProjectBuilder-$pbver1$pbextdir ; perl Makefile.PL ; make ; make install ; cd .. ; rm -rf ProjectBuilder-* ; gzip -cd project-builder-latest.tar.gz | tar xf - ; cd project-builder-$pbver2$pbextdir ; perl Makefile.PL ; make ; make install ; cd .. ; rm -rf project-builder-* ;","Building Project-Builder");
+gzip -cd ProjectBuilder-latest.tar.gz | tar xf - ; cd ProjectBuilder-`tar tfz ProjectBuilder-latest.tar.gz | head -1 | cut -d- -f2 | cut -d/ -f1` ; perl Makefile.PL ; make ; make install ; cd .. ; rm -rf ProjectBuilder-* ; gzip -cd project-builder-latest.tar.gz | tar xf - ; cd project-builder-`tar tfz project-builder-latest.tar.gz | head -1 | cut -d- -f3 | cut -d/ -f1` ; perl Makefile.PL ; make ; make install ; cd .. ; rm -rf project-builder-* ;","Building Project-Builder");
 EOF
     } elsif ($itype =~ /^pkg/) {
@@ -4196 +4143 @@
 my $pkgforpb = pb_distro_get_param($pbos,pb_conf_get_if("ospkg"));
 my $forcerepo = pb_distro_setuposrepo($pbos);
-pb_log(1,"Forcing full package install with forcerepo\n") if (defined $forcerepo);
-pb_distro_installdeps(undef,$pbos,$forcerepo,pb_distro_only_deps_needed($pbos,join(' ',split(/,/,$pkgforpb))));
+pb_distro_installpkgs($pbos,pb_distro_only_deps_needed($pbos,join(' ',split(/,/,$pkgforpb))));
 EOF
     } else {
@@ -4358 +4304 @@
 }
 return;
+}
+
+# Build all docker CTN required to build an app
+sub pb_all_docker {
+    my $action = shift;
+    my $savproj = $ENV{'PBPROJ'};
+    my $pkg;
+    my @pkg;
+    if ($#ARGV == -1) {
+        $pkg = pb_cms_get_pkg($defpkgdir,$extpkgdir);
+        @pkgs = @$pkg;
+    } else {
+        @pkgs = @ARGV;
+    }
+    #pb_log(1,"Packages to print:".Dumper(@pkgs)."\n");
+    
+    # Get the list of all VE we need to work on
+    my ($vm,$all);
+    if (not defined $ENV{'PBV'}) {
+        ($vm,$all) = pb_get2v("ve");
+    } else {
+        @$vm = split(/,/,$ENV{'PBV'});
+    }
+    
+    my $docrepo = pb_ve_docker_repo();
+    # TODO: parallelize
+    foreach my $v (@$vm) {
+        my $done = 0;
+        my $lmsg = "";
+        my $lctn = "";
+        my $pbos = pb_distro_get_context($v);
+        # Docker images are (for now !) regular and use distro:ver
+        $pbimage = "$pbos->{'name'}".":$pbos->{'version'}";
+        # The two first steps need to be done with the pb project and using a stable pb version
+        $ENV{'PBPROJ'} = $appname;
+        $lctn = "$docrepo$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
+        my $found = pb_ve_docker_get_image($lctn);
+        if (($pbforce == 1) || (not defined $found)) {
+            # pb -p pb -m distro-ver-arch newve -i distro:ver
+            ($pbforce == 1) ? $lmsg = "Forced" : $lmsg = "not found";
+            pb_log(0,"Docker container $lctn $lmsg, creating it\n");
+            pb_parallel_launchv(undef,"ve",undef,0,$pbimage);
+        }
+        $lctn = "$docrepo$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-pb";
+        $found = pb_ve_docker_get_image($lctn);
+        if (($pbforce == 1) || (not defined $found)) {
+            ($pbforce == 1) ? $lmsg = "Forced" : $lmsg = "not found";
+            pb_log(0,"Docker container $lctn $lmsg, creating it\n");
+            # pb -p pb -m distro-ver-arch setupve
+            if ($savproj eq "pb") {
+                pb_cms2build("SandBox") if ($action !~ /sbx2/);
+                $done = 1;
+                pb_setup2v("ve","SandBox");
+            } else {
+                # Here we install in the CTN pb from upstream packages to have a stable version
+                pb_setup2v("ve",undef,"pkg");
+            }
+        }
+        # Now back to our original project
+        $ENV{'PBPROJ'} = $savproj;
+        $lctn = "$docrepo$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-pb-$ENV{'PBPROJ'}";
+        $found = pb_ve_docker_get_image($lctn);
+        if (($pbforce == 1) || (not defined $found)) {
+            ($pbforce == 1) ? $lmsg = "Forced" : $lmsg = "not found";
+            pb_log(0,"Docker container $lctn $lmsg, creating it\n");
+            # pb -p pb -m distro-ver-arch prepve
+            if ($savproj eq "pb") {
+                pb_cms2build("SandBox") if ($action !~ /sbx2/);
+                $done = 1;
+            }
+            pb_build2v("ve","prep");
+        }
+        pb_log(0,"Instantiating a Docker container from $lctn\n");
+        if (($savproj ne "pb") || ($done == 0)) {
+            pb_cms2build("SandBox") if ($action !~ /sbx2/);
+        }
+        pb_build2v("ve","build");
+    }
 }
 
@@ -5323 +5347 @@
 }
 
-# Unused now
-sub pb_keep_step {
-
-my $pbos = shift;
-my $pbstep = shift;
-
-$pbstep = 3 if (not defined $pbstep);
-
-my %h;
-my $h = \%h;
-$h = pb_conf_cache("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml",$h);
-$h->{'pbstep'}->{"$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}"} = $pbstep;
-pb_conf_write("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml",$h);
-}
-
 sub create_solaris_prototype {