Changeset 1137 in ProjectBuilder

Timestamp:

Jan 7, 2011, 12:37:04 AM (13 years ago)

Author:

Bruno Cornec

Message:

• build2pkg is now using Parallel::ForkManager to generate packages in parallel
• pb_system fixed to support parallel calls

Location:

devel

Files:

• pb-modules/lib/ProjectBuilder/Base.pm (modified) (3 diffs)
• pb-modules/lib/ProjectBuilder/Distribution.pm (modified) (1 diff)
• pb/bin/pb (modified) (7 diffs)

devel/pb-modules/lib/ProjectBuilder/Base.pm

@@ -146 +146 @@
 pb_log(0,"$cmt... ") if ((! defined $verbose) || ($verbose ne "quiet"));
 pb_log(1,"Executing $cmd\n");
-unlink("$ENV{'PBTMP'}/system.log") if (-f "$ENV{'PBTMP'}/system.log");
-$redir = "2>> $ENV{'PBTMP'}/system.log 1>> $ENV{'PBTMP'}/system.log" if ((! defined $verbose) || ($verbose ne "noredir"));
+unlink("$ENV{'PBTMP'}/system.$$.log") if (-f "$ENV{'PBTMP'}/system.$$.log");
+$redir = "2>> $ENV{'PBTMP'}/system.$$.log 1>> $ENV{'PBTMP'}/system.$$.log" if ((! defined $verbose) || ($verbose ne "noredir"));
 system("$cmd $redir");
 my $res = $?;
@@ -158 +158 @@
 if ($res == -1) {
     pb_log(0,"failed to execute ($cmd): $!\n") if ((! defined $verbose) || ($verbose ne "quiet"));
-    pb_display_file("$ENV{'PBTMP'}/system.log") if ((-f "$ENV{'PBTMP'}/system.log") and ((! defined $verbose) || ($verbose ne "quiet")));
+    pb_display_file("$ENV{'PBTMP'}/system.$$.log") if ((-f "$ENV{'PBTMP'}/system.$$.log") and ((! defined $verbose) || ($verbose ne "quiet")));
 } elsif ($res & 127) {
     pb_log(0, "child ($cmd) died with signal ".($? & 127).", ".($? & 128) ? 'with' : 'without'." coredump\n") if ((! defined $verbose) || ($verbose ne "quiet"));
-    pb_display_file("$ENV{'PBTMP'}/system.log") if ((-f "$ENV{'PBTMP'}/system.log") and ((! defined $verbose) || ($verbose ne "quiet")));
+    pb_display_file("$ENV{'PBTMP'}/system.$$.log") if ((-f "$ENV{'PBTMP'}/system.$$.log") and ((! defined $verbose) || ($verbose ne "quiet")));
 } elsif ($res == 0) {
     pb_log(0,"OK\n") if ((! defined $verbose) || ($verbose ne "quiet"));
-    pb_display_file("$ENV{'PBTMP'}/system.log") if ((defined $verbose) and (-f "$ENV{'PBTMP'}/system.log") and ($verbose ne "quiet"));
+    pb_display_file("$ENV{'PBTMP'}/system.$$.log") if ((defined $verbose) and (-f "$ENV{'PBTMP'}/system.$$.log") and ($verbose ne "quiet"));
 } else {
     pb_log(0, "child ($cmd) exited with value ".($? >> 8)."\n") if ((! defined $verbose) || ($verbose ne "quiet"));
-    pb_display_file("$ENV{'PBTMP'}/system.log") if ((-f "$ENV{'PBTMP'}/system.log") and ((! defined $verbose) || ($verbose ne "quiet")));
+    pb_display_file("$ENV{'PBTMP'}/system.$$.log") if ((-f "$ENV{'PBTMP'}/system.$$.log") and ((! defined $verbose) || ($verbose ne "quiet")));
 }
 return($res);
@@ -312 +312 @@
 =item B<pb_set_content>
 
-This function put the content of a file into the file passed in parameter.
+This function put the content of a variable passed as second parameter into the file passed as first parameter.
 
 =cut

devel/pb-modules/lib/ProjectBuilder/Distribution.pm

@@ -302 +302 @@
 return if ((not defined $deps) || ($deps =~ /^\s*$/));
 if ($deps !~ /^[    ]*$/) {
+    # This may not be // proof. We should test for availability of repo and sleep if not
     pb_system("$dupd $deps","Installing dependencies ($deps)");
     }

devel/pb/bin/pb

@@ -1039 +1039 @@
     pb_log(2,"DEBUG: distro tuple: ".join(',',($ddir, $dver, $dfam, $dtype, $pbsuf, $pbupd, $pbins, $arch))."\n");
 
+    # If needed we may add repository to the build env
+    pb_distro_setuprepo($ddir,$dver,$arch,$dtype,$dfam,$dos);
+
     # Get list of packages to build
     my $ptr = pb_get_pkg();
@@ -1049 +1052 @@
     chdir "$ENV{'PBBUILDDIR'}";
     my $made = ""; # pkgs made during build
+    my $pm = new Parallel::ForkManager($pbparallel) if (defined $pbparallel);
+
+    # We need to communicate info back from the children if parallel so prepare a dir for that
+    my $tmpd = "$ENV{'PBTMP'}/build.$$";
+    pb_mkdir_p($tmpd) if (defined $pbparallel);
+
     foreach my $pbpkg (@pkgs) {
+        $pm->start and next if (defined $pbparallel);
+
         my $vertag = $pkg->{$pbpkg};
         # get the version of the current package - maybe different
@@ -1085 +1096 @@
             }
 
-            # If needed we may add repository to the build env
-            pb_distro_setuprepo($ddir,$dver,$arch,$dtype,$dfam,$dos);
             foreach my $f (@specfile) {
                 if ($f =~ /\.spec$/) {
+                    # This could cause an issue in // mode
                     pb_distro_installdeps($f,$dtype,$pbins);
                     pb_system("rpmbuild $specialdef --define \"packager $ENV{'PBPACKAGER'}\" --define \"_topdir $ENV{'PBBUILDDIR'}\" -ba $f","Building package with $f under $ENV{'PBBUILDDIR'}","verbose");
@@ -1095 +1105 @@
             }
             # Get the name of the generated packages
-            open(LOG,"$ENV{'PBTMP'}/system.log") || die "Unable to open $ENV{'PBTMP'}/system.log";
+            open(LOG,"$ENV{'PBTMP'}/system.$$.log") || die "Unable to open $ENV{'PBTMP'}/system.$$.log";
             while (<LOG>) {
                 chomp($_);
@@ -1114 +1124 @@
             chmod 0755,"debian/rules";
 
-            pb_distro_setuprepo($ddir,$dver,$arch,$dtype,$dfam,$dos);
             pb_distro_installdeps("debian/control",$dtype,$pbins);
             pb_system("dpkg-buildpackage -us -uc -rfakeroot","Building package","verbose");
             # Get the name of the generated packages
-            open(LOG,"$ENV{'PBTMP'}/system.log") || die "Unable to open $ENV{'PBTMP'}/system.log";
+            open(LOG,"$ENV{'PBTMP'}/system.$$.log") || die "Unable to open $ENV{'PBTMP'}/system.$$.log";
             while (<LOG>) {
                 chomp();
@@ -1234 +1243 @@
             die "Unknown dtype format $dtype";
         }
-    }
+        if (defined $pbparallel) {
+            # Communicate results back to parent
+            pb_set_content("$tmpd/$$",$made);
+            $pm->finish;
+        }
+    }
+    if (defined $pbparallel) {
+        # In the parent, we need to get the result from the children
+        $pm->wait_all_children;
+        foreach my $f (<$tmpd/*>) {
+            $made .= " ".pb_get_content($f);
+        }
+        pb_rm_rf($tmpd);
+    }
+
     # Find the appropriate check cmd/opts
     my ($oschkcmd,$oschkopt) = pb_conf_get_if("oschkcmd","oschkopt");
@@ -2961 +2984 @@
 <p>
 EOF
-    open(LOG,"$ENV{'PBTMP'}/system.log") || die "Unable to read $ENV{'PBTMP'}/system.log: $!";
+    open(LOG,"$ENV{'PBTMP'}/system.$$.log") || die "Unable to read $ENV{'PBTMP'}/system.$$.log: $!";
     my $col = 2;
     my $i = 1;