Changeset 2436 in ProjectBuilder

Timestamp:

Sep 6, 2019, 3:37:09 PM (5 years ago)

Author:

Bruno Cornec

Message:

force explicit analysis of boolean and when using cluck and more fixes to pb_all_docker

File:

• devel/pb/bin/pb (modified) (7 diffs)

devel/pb/bin/pb

@@ -1969 +1969 @@
             my $keyfile = pb_ssh_get(1);
             pb_mkdir_p("$ENV{'HOME'}/.abuild");
-            open(CONF,"> $ENV{'HOME'}/.abuild/abuild.conf") || cluck "Unable to create the abuild conf file" && next;
+            open(CONF,"> $ENV{'HOME'}/.abuild/abuild.conf") || (cluck "Unable to create the abuild conf file" && next);
             print CONF "PACKAGER_PRIVKEY=\"$keyfile\"\n" if (defined $keyfile);
             close(CONF);
@@ -2602 +2602 @@
     # Remove a potential $ENV{'HOME'} as tdir should be relative to pb's home
     if ($cmt =~ /^VE/o) {
-        $tdir =~ s|\$ENV.+\}/|| or cluck "for $cmt: $tdir must have \$ENV{'HOME'} in it or the remainder of the code won't work since some parts use relative pathing and others will try absolute.  Use a symlink if necessary to place the real files in a different place than under your home directory" && return;
+        $tdir =~ s|\$ENV.+\}/|| or (cluck "for $cmt: $tdir must have \$ENV{'HOME'} in it or the remainder of the code won't work since some parts use relative pathing and others will try absolute.  Use a symlink if necessary to place the real files in a different place than under your home directory" && return);
     } else {
         # process, ok if not present.
@@ -2692 +2692 @@
                 #$shcmd = "$cmd1 build $dockeropt->{$ENV{'PBPROJ'}} -t $tag{$pbstep+1} $context";
             } else {
+                my $rmatend = "";
                 # As we are in run phase use docker run. cmd will be completed below
-                my $rmatend = "--rm" if ($pbdebug lt 1);
+                $rmatend = "--rm" if ($pbdebug lt 1);
                 $shcmd = "$cmd1 run $rmatend --cidfile=\"$cidfile\" --user $dkaccount --name $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-$cmt";
             }
@@ -2884 +2885 @@
         }
         my $ts = "$ENV{'PBTMP'}/vebuild/".basename($pbscript2);
-        open(DOCKER," > $ts") || cluck "Unable to create $ts" && return;
+        open(DOCKER," > $ts") || (cluck "Unable to create $ts" && return);
         print DOCKER "#!/bin/bash\n";
         print DOCKER "# Put content from host to guest, copying to have the correct uid/gid and rights to do so\n";
@@ -4308 +4309 @@
 # Build all docker CTN required to build an app
 sub pb_all_docker {
-    my $action = shift;
-    my $savproj = $ENV{'PBPROJ'};
-    my $pkg;
-    my @pkg;
-    if ($#ARGV == -1) {
-        $pkg = pb_cms_get_pkg($defpkgdir,$extpkgdir);
-        @pkgs = @$pkg;
-    } else {
-        @pkgs = @ARGV;
-    }
-    #pb_log(1,"Packages to print:".Dumper(@pkgs)."\n");
-    
-    # Get the list of all VE we need to work on
-    my ($vm,$all);
-    if (not defined $ENV{'PBV'}) {
-        ($vm,$all) = pb_get2v("ve");
-    } else {
-        @$vm = split(/,/,$ENV{'PBV'});
-    }
-    
-    my $docrepo = pb_ve_docker_repo();
-    # TODO: parallelize
-    foreach my $v (@$vm) {
-        my $done = 0;
-        my $lmsg = "";
-        my $lctn = "";
-        my $pbos = pb_distro_get_context($v);
-        # Docker images are (for now !) regular and use distro:ver
-        $pbimage = "$pbos->{'name'}".":$pbos->{'version'}";
-        # The two first steps need to be done with the pb project and using a stable pb version
-        $ENV{'PBPROJ'} = $appname;
-        $lctn = "$docrepo$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
-        my $found = pb_ve_docker_get_image($lctn);
-        if (($pbforce == 1) || (not defined $found)) {
-            # pb -p pb -m distro-ver-arch newve -i distro:ver
-            ($pbforce == 1) ? $lmsg = "Forced" : $lmsg = "not found";
-            pb_log(0,"Docker container $lctn $lmsg, creating it\n");
-            pb_parallel_launchv(undef,"ve",undef,0,$pbimage);
-        }
-        $lctn = "$docrepo$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-pb";
-        $found = pb_ve_docker_get_image($lctn);
-        if (($pbforce == 1) || (not defined $found)) {
-            ($pbforce == 1) ? $lmsg = "Forced" : $lmsg = "not found";
-            pb_log(0,"Docker container $lctn $lmsg, creating it\n");
-            # pb -p pb -m distro-ver-arch setupve
-            if ($savproj eq "pb") {
-                pb_cms2build("SandBox") if ($action !~ /sbx2/);
-                $done = 1;
-                pb_setup2v("ve","SandBox");
-            } else {
-                # Here we install in the CTN pb from upstream packages to have a stable version
-                pb_setup2v("ve",undef,"pkg");
-            }
-        }
-        # Now back to our original project
-        $ENV{'PBPROJ'} = $savproj;
-        $lctn = "$docrepo$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-pb-$ENV{'PBPROJ'}";
-        $found = pb_ve_docker_get_image($lctn);
-        if (($pbforce == 1) || (not defined $found)) {
-            ($pbforce == 1) ? $lmsg = "Forced" : $lmsg = "not found";
-            pb_log(0,"Docker container $lctn $lmsg, creating it\n");
-            # pb -p pb -m distro-ver-arch prepve
-            if ($savproj eq "pb") {
-                pb_cms2build("SandBox") if ($action !~ /sbx2/);
-                $done = 1;
-            }
-            pb_build2v("ve","prep");
-        }
-        pb_log(0,"Instantiating a Docker container from $lctn\n");
-        if (($savproj ne "pb") || ($done == 0)) {
+
+my $action = shift;
+my $savproj = $ENV{'PBPROJ'};
+my $pkg;
+my @pkg;
+if ($#ARGV == -1) {
+    $pkg = pb_cms_get_pkg($defpkgdir,$extpkgdir);
+    @pkgs = @$pkg;
+} else {
+    @pkgs = @ARGV;
+}
+#pb_log(1,"Packages to print:".Dumper(@pkgs)."\n");
+
+# Get the list of all VE we need to work on
+my ($vm,$all);
+if (not defined $ENV{'PBV'}) {
+    ($vm,$all) = pb_get2v("ve");
+} else {
+    @$vm = split(/,/,$ENV{'PBV'});
+}
+
+# Adapt // mode to memory size
+$pbparallel = pb_set_parallel("ve");
+
+pb_log(2,"DEBUG: pb_all_docker(action: $action)\n");
+my $docrepo = pb_ve_docker_repo();
+
+my ($vexist,$vpid) = (undef,undef);
+my $pm;
+my $all_ok = 1;
+if (defined $pbparallel) {
+    $pm = new Parallel::ForkManager($pbparallel);
+
+    $pm->run_on_finish(sub { my ($pid, $code, $id, $signal, $dump) = @_;
+                            unless ($code == 0 && $signal == 0 && $dump == 0) {
+                                $all_ok = 0;
+                                pb_log(0,"ERROR: pid $pid failed\n");
+                                pb_log(0,"code: $code - signal: $signal - dump: $dump\n");
+                            }
+                        });
+}
+
+my $counter = 0;
+foreach my $v (@$vm) {
+    $counter++;
+    # Modulo 2 * pbparallel (to avoid synchronization problems)
+    $counter = 1 if ((defined $pbparallel) && ($counter > 2 * $pbparallel));
+    $pm->start($counter) and next if (defined $pbparallel);
+
+    my $done = 0;
+    my $lmsg = "";
+    my $lctn = "";
+    my $pbos = pb_distro_get_context($v);
+    # Docker images are (for now !) regular and use distro:ver
+    $pbimage = "$pbos->{'name'}".":$pbos->{'version'}";
+    # The two first steps need to be done with the pb project and using a stable pb version
+    $ENV{'PBPROJ'} = $appname;
+    $lctn = "$docrepo$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
+    my $found = pb_ve_docker_get_image($lctn);
+    if (($pbforce == 1) || (not defined $found)) {
+        # pb -p pb -m distro-ver-arch newve -i distro:ver
+        ($pbforce == 1) ? $lmsg = "Forced" : $lmsg = "not found";
+        pb_log(0,"Docker container $lctn $lmsg, creating it\n");
+        pb_launchv(undef,"ve",$v,$action,0,$pbforce,undef,undef,$pbimage);
+    }
+    $lctn = "$docrepo$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-pb";
+    $found = pb_ve_docker_get_image($lctn);
+    if (($pbforce == 1) || (not defined $found)) {
+        ($pbforce == 1) ? $lmsg = "Forced" : $lmsg = "not found";
+        pb_log(0,"Docker container $lctn $lmsg, creating it\n");
+        # pb -p pb -m distro-ver-arch setupve
+        if ($savproj eq "pb") {
             pb_cms2build("SandBox") if ($action !~ /sbx2/);
-        }
-        pb_build2v("ve","build");
-    }
+            $done = 1;
+            pb_setup2v("ve","SandBox");
+        } else {
+            # Here we install in the CTN pb from upstream packages to have a stable version
+            pb_setup2v("ve",undef,"pkg");
+        }
+    }
+    # Now back to our original project
+    $ENV{'PBPROJ'} = $savproj;
+    $lctn = "$docrepo$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-pb-$ENV{'PBPROJ'}";
+    $found = pb_ve_docker_get_image($lctn);
+    if (($pbforce == 1) || (not defined $found)) {
+        ($pbforce == 1) ? $lmsg = "Forced" : $lmsg = "not found";
+        pb_log(0,"Docker container $lctn $lmsg, creating it\n");
+        # pb -p pb -m distro-ver-arch prepve
+        if ($savproj eq "pb") {
+            pb_cms2build("SandBox") if ($action !~ /sbx2/);
+            $done = 1;
+        }
+        pb_build2v("ve","prep");
+    }
+    pb_log(0,"Instantiating a Docker container from $lctn\n");
+    if (($savproj ne "pb") || ($done == 0)) {
+        pb_cms2build("SandBox") if ($action !~ /sbx2/);
+    }
+    pb_build2v("ve","build");
+    $pm->finish if (defined $pbparallel);
+}
+$pm->wait_all_children if (defined $pbparallel);
+confess "Aborting, one or more of the children failed." if ((not $all_ok) && ($Global::pb_stop_on_error));
 }
 
@@ -4993 +5023 @@
 my $bdir;
 
-opendir($bdir,"$dir") || cluck "Unable to open directory $dir: $!" && return;
+opendir($bdir,"$dir") || (cluck "Unable to open directory $dir: $!" && return);
 foreach my $f (readdir($bdir)) {
     next if ($f =~ /^\./);
@@ -5266 +5296 @@
         cluck("No pbpassphrase nor pbpassfile defined, unable to handle GPG Release file\n");
     } else {
-        open(PASS,"$ENV{'PBPASSFILE'}") || cluck("Unable to open $ENV{'PBPASSFILE'}\n") && return;
+        open(PASS,"$ENV{'PBPASSFILE'}") || (cluck("Unable to open $ENV{'PBPASSFILE'}\n") && return);
         my $pass = <PASS>;
         chomp($pass);