Changeset 1143 in ProjectBuilder

Timestamp:

Jan 10, 2011, 2:26:33 AM (13 years ago)

Author:

Bruno Cornec

Message:

• Adds function pb_set_parallel which set $pbparallel depending on memory size for VMs
• Adds pb_set_port and pb_get_port functions to deal with SSH communication port with VMs using a range
• build2v and setup2v are now parallel

File:

• devel/pb/bin/pb (modified) (10 diffs)

devel/pb/bin/pb

@@ -1668 +1668 @@
     }
 
-    # Useless for VE
-    my $nport;
-    if ($cmt !~ /^VE/) {
-        $nport = $sshport->{$ENV{'PBPROJ'}};
-        $nport = "$pbport" if (defined $pbport);
-    }
+    # Useless for VE 
+    my $nport = pb_get_port($sshport->{$ENV{'PBPROJ'}},$cmt) if ($cmt !~ /^VE/);
 
     # Remove a potential $ENV{'HOME'} as tdir should be relative to pb's home
@@ -1915 +1911 @@
         # TODO: vmmonport should be optional
         my ($ptr,$ptr2,$vmpath,$vmport,$vmsize,$vmmonport) = pb_conf_get("vmtype","vmcmd","vmpath","vmport","vmsize","vmmonport");
-        my ($vmopt,$vmtmout,$vmsnap) = pb_conf_get_if("vmopt","vmtmout","vmsnap");
+        my ($vmopt,$vmmem,$vmtmout,$vmsnap) = pb_conf_get_if("vmopt","vmmem","vmtmout","vmsnap");
 
         my $vmtype = $ptr->{$ENV{'PBPROJ'}};
@@ -1938 +1934 @@
         } elsif (defined $vmopt->{$ENV{'PBPROJ'}}) {
             $ENV{'PBVMOPT'} .= " $vmopt->{$ENV{'PBPROJ'}}" if ($ENV{'PBVMOPT'} !~ / $vmopt->{$ENV{'PBPROJ'}}/);
+        }
+
+        # How much memory to allocate for VMs
+        if (defined $vmmem->{$v}) {
+            $ENV{'PBVMOPT'} .= " -m $vmmem->{$v}";
+        } elsif (defined $vmmem->{$ENV{'PBPROJ'}}) {
+            $ENV{'PBVMOPT'} .= " -m $vmmem->{$ENV{'PBPROJ'}}";
         }
 
@@ -1963 +1966 @@
             $ENV{'PBVMTMOUT'} = $vmtmout->{$ENV{'PBPROJ'}};
         }
-        my $nport = $vmport->{$ENV{'PBPROJ'}};
-        $nport = "$pbport" if (defined $pbport);
+        my $nport = pb_get_port($vmport->{$ENV{'PBPROJ'}});
     
         my $cmd;
@@ -2224 +2226 @@
 pb_build2ssh() if (($all == 1) && ($action eq "build"));
 
+# Adapt // mode to memory size
+$pbparallel = pb_set_parallel($vtype);
+
 my ($vmexist,$vmpid) = (undef,undef);
 my $pm = new Parallel::ForkManager($pbparallel) if (defined $pbparallel);
 
+# Set which port the VM will use to communicate
+$pm->run_on_start(pb_set_port();
+
+my $counter = 0;
 foreach my $v (@$v) {
-    $pm->start and next if (defined $pbparallel);
+    $counter++;
+    # Modulo pbparallel
+    $counter = 1 if ($counter > $pbparallel);
+    $pm->start($counter) and next if (defined $pbparallel);
     # Prepare the script to be executed on the VM/VE
     # in $ENV{'PBDESTDIR'}/pbscript.$$
@@ -2331 +2343 @@
 
     if ($vtype eq "vm") {
-        # Skip that VM if it something went wrong
+        # Skip that VM if something went wrong
         if (($vmpid == 0) && ($vmexist == 0)) {
             $pm->finish if (defined $pbparallel);
@@ -2500 +2512 @@
 my $pbscript = "$ENV{'PBDESTDIR'}/setupv";
 
+# Adapt // mode to memory size
+$pbparallel = pb_set_parallel($vtype);
+
+my $pm = new Parallel::ForkManager($pbparallel) if (defined $pbparallel);
+
+# Set which port the VM will use to communicate
+$pm->run_on_start(pb_set_port();
+
+my $counter = 0;
 foreach my $v (@$vm) {
+    $counter++;
+    # Modulo pbparallel
+    $counter = 1 if ($counter > $pbparallel);
+    $pm->start($counter) and next if (defined $pbparallel);
+
     # Deal with date sync.
     my ($ntpline,$dateline) = pb_date2v($vtype,$v);
@@ -2547 +2573 @@
         
         my ($vmhost,$vmport,$vmntp) = pb_conf_get("vmhost","vmport","vmntp");
-        my $nport = $vmport->{$ENV{'PBPROJ'}};
-        $nport = "$pbport" if (defined $pbport);
+        my $nport = pb_get_port($vmport->{$ENV{'PBPROJ'}});
     
         # Skip that VM if something went wrong
@@ -2880 +2905 @@
     
     pb_script2v($pbscript,$vtype,$pbforce,$v);
+    $pm->finish if (defined $pbparallel);
+}
+if (defined $pbparallel) {
+    $pm->wait_all_children;
 }
 return;
@@ -3429 +3458 @@
 }
 
+# Manages VM SSH port communication
+sub pb_get_port {
+
+my $port = shift;
+my $cmt = shift;
+
+die "No port passed in parameter. Report to dev team\n" if (not defined $port);
+my $nport = $port;
+# Maybe a port was given as parameter so overwrite
+$nport = "$pbport" if (defined $pbport);
+# Maybe in // mode so use the env var set up as an offset to the base port, except when called from send2target for Packages
+if ($cmt ne "Packages") {
+    $nport += $ENV{'PBVMPORT'} if ((defined $pbparallel) && (defined $ENV{'PBVMPORT'}));
+}
+return($nport);
+}
+
+sub pb_set_port { 
+        
+my ($pid,$ident) = @_;
+$ENV{'PBVMPORT'} = $ident;
+}
+
+sub pb_set_parallel {
+
+my $vtype = shift;
+
+my ($pbvmmem) = pb_conf_get_if("pbvmmem");
+
+# Take care of memory size if VM, parallel mode and more than 1 action
+if ((defined $pbparallel) && ($pbparallel ne 1) && ($vtype eq "vm")) {
+    eval
+    {
+        require Linux::SysInfo;
+        Linux::SysInfo->import();
+    };
+    if ($@) {
+        # Linux::SysInfo not found, defaulting to 4
+        $pbparallel = 4;
+    } else {
+        # Using the memory size
+        my $si = Linux::SysInfo::sysinfo();
+        if (not defined $si) {
+            $pbparallel = 4 
+        } else {
+            # Keep the number of VM whose memory can be allocated
+            my $ram = $si->{"totalram"}-$si->{"sharedram"}-$si->{"bufferram"};
+            my $ram2;
+            if ((defined $pbvmmem) and (defined $pbvmmem->{$ENV{'PBPROJ'}})) {
+                $ram2 = $pbvmmem->{$ENV{'PBPROJ'}};
+            } else {
+                # Default for KVM/QEMU
+                $ram2 = 128;
+            }
+            $pbparallel = sprintf("%d",$ram/$ram2);
+        }
+    }
+}
+return($pbparallel);
+}
+
+
 1;