Changeset 1153 in ProjectBuilder for devel/pb

Timestamp:

Jan 12, 2011, 2:42:12 AM (14 years ago)

Author:

Bruno Cornec

Message:

• Avoid File::MimeInfo hard requirement. Only abort if not found when needed.
• Improve report when a perl module is missing
• Kill an existing crashed VM using an SSH port needed for another VM (should avoid crashed VM to stay when building for all VMs)
• Use a new parameter vmbuildtm as a timeout before killing the VM (shoudl correspond to build + transfer time)
• use twice the number of VMs for ports in the range for SSH communication to allow for VMs to finish in an unordered way.
• Fix a bug in test modules when using Test simple only
• Mail::Sendmail is now optional for Log module as well, even if not used yet
• Update pb.conf doc with info for vmbuildtm and vmmem
• Ready for 0.10.1

Location:

devel/pb

Files:

• bin/pb (modified) (8 diffs)
• lib/ProjectBuilder/CMS.pm (modified) (1 diff)

devel/pb/bin/pb

@@ -523 +523 @@
     if ($@) {
         # Sys::CPU not found, defaulting to 1
+        pb_log(1,"ADVISE: Install Sys::CPU to benefit from automatic parallelism optimization.\nOr use pbparallel in your pb.conf file\nOnly 1 process at a time for the moment\n");
         $pbparallel = 1;
     } else {
         # Using the number of cores
         $pbparallel = Sys::CPU::cpu_count();
+        pb_log(1,"Using parallel mode with $pbparallel processes\n");
     }
 }
@@ -538 +540 @@
 if ($@) {
     $pbparallel = undef;
+    pb_log(1,"ADVISE: Install Parallel::ForkManager to benefit from automatic parallelism optimization.\nOnly 1 process at a time for the moment\n");
 }
 
@@ -1805 +1808 @@
             # For monitoring control
             if ((($vmtype->{$ENV{'PBPROJ'}}) eq "kvm") || (($vmtype->{$ENV{'PBPROJ'}}) eq "qemu")) {
+                eval
+                {
                 require Net::Telnet;
-                my $t = new Net::Telnet (Timeout => 120, Host => "localhost", Port => $vmmonport->{$ENV{'PBPROJ'}}) || die "Unable to dialog on the monitor";
-                # move to monitor mode
-                my @lines = $t->cmd("c");
-                # Create a snapshot named pb
-                @lines = $t->cmd("savevm pb");
-                # Write the new status in the VM
-                @lines = $t->cmd("commit all");
-                # End
-                @lines = $t->cmd("quit");
+                Net::Telnet->import();
+                };
+                if ($@) {
+                    # Net::Telnet not found
+                    pb_log(1,"ADVISE: Install Net::Telnet to benefit from monitoring control and snapshot feature.\nWARNING: No snapshot created");
+                } else {
+                    my $t = new Net::Telnet (Timeout => 120, Host => "localhost", Port => $vmmonport->{$ENV{'PBPROJ'}}) || die "Unable to dialog on the monitor";
+                    # move to monitor mode
+                    my @lines = $t->cmd("c");
+                    # Create a snapshot named pb
+                    @lines = $t->cmd("savevm pb");
+                    # Write the new status in the VM
+                    @lines = $t->cmd("commit all");
+                    # End
+                    @lines = $t->cmd("quit");
+                }
             }
         }
@@ -1911 +1923 @@
         # TODO: vmmonport should be optional
         my ($ptr,$ptr2,$vmpath,$vmport,$vmsize,$vmmonport) = pb_conf_get("vmtype","vmcmd","vmpath","vmport","vmsize","vmmonport");
-        my ($vmopt,$vmmem,$vmtmout,$vmsnap) = pb_conf_get_if("vmopt","vmmem","vmtmout","vmsnap");
+        my ($vmopt,$vmmem,$vmtmout,$vmsnap,$vmbuildtm) = pb_conf_get_if("vmopt","vmmem","vmtmout","vmsnap","vmbuildtm");
 
         my $vmtype = $ptr->{$ENV{'PBPROJ'}};
@@ -2001 +2013 @@
                 pb_log(0,"Unable to find VM $vmm\n");
             } else {
+                # Is the SSH port free? if not kill the existing process using it after a build timeout period
+                my $vmssh = pb_check_ps($tmpcmd,"tcp:$nport:10.0.2.15:22");
+                if ($vmssh) {
+                    my $buildtm = $ENV{'PBVMTMOUT'};
+                    if (defined $vmbuildtm->{$v}) {
+                        $buildtm = $vmbuildtm->{$v};
+                    } elsif (defined $vmbuildtm->{$ENV{'PBPROJ'}}) {
+                        $buildtm = $vmbuildtm->{$ENV{'PBPROJ'}};
+                    }
+
+                    sleep $buildtm;
+                    pb_log(0,"WARNING: Killing the process ($vmssh) using port $nport (previous failed VM ?)\n");
+                    kill 15,$vmssh;
+                }
                 pb_system("$cmd &","Launching the VM $vmm");
+                # Using system allows to kill it externaly if needed
                 pb_system("sleep $ENV{'PBVMTMOUT'}","Waiting $ENV{'PBVMTMOUT'} s for VM $v to come up");
                 $vmpid = pb_check_ps($tmpcmd,$vmm);
@@ -2238 +2265 @@
 foreach my $v (@$v) {
     $counter++;
-    # Modulo pbparallel
-    $counter = 1 if ($counter > $pbparallel);
+    # Modulo 2 * pbparallel (to avoid synchronization problems)
+    $counter = 1 if ($counter > 2 * $pbparallel);
     $pm->start($counter) and next if (defined $pbparallel);
     # Prepare the script to be executed on the VM/VE
@@ -3498 +3525 @@
     };
     if ($@) {
-        # Linux::SysInfo not found, defaulting to 4
-        $pbparallel = 4;
+        # Linux::SysInfo not found
+        pb_log(1,"ADVISE: Install Linux::SysInfo to benefit from automatic parallelism optimization.\nOr optimize manually pbparallel in your pb.conf file\nUsing $pbparallel processes max at a time for the moment\nWARNING: This may consume too much memory for your system");
     } else {
         # Using the memory size
         my $si = Linux::SysInfo::sysinfo();
         if (not defined $si) {
-            $pbparallel = 4 
+            pb_log(1,"ADVISE: Install Linux::SysInfo to benefit from automatic parallelism optimization.\nOr optimize manually pbparallel in your pb.conf file\nUsing $pbparallel processes max at a time for the moment\nWARNING: This may consume too much memory for your system");
         } else {
             # Keep the number of VM whose memory can be allocated
             my $ram = $si->{"totalram"}-$si->{"sharedram"}-$si->{"bufferram"};
             my $ram2;
-            my ($pbvmmem) = pb_conf_get_if("pbvmmem");
-
-            if ((defined $pbvmmem) and (defined $pbvmmem->{$ENV{'PBPROJ'}})) {
-                $ram2 = $pbvmmem->{$ENV{'PBPROJ'}};
+            my ($vmmem) = pb_conf_get_if("vmmem");
+
+            my $v = "default";
+            if ((defined $vmmem) and (defined $vmmem->{$v})) {
+                $ram2 = $vmmem->{$v};
             } else {
                 # Default for KVM/QEMU
@@ -3519 +3547 @@
             $pbparallel = sprintf("%d",$ram/$ram2);
         }
+        pb_log(1,"Using $pbparallel processes at a time\n");
     }
 }

devel/pb/lib/ProjectBuilder/CMS.pm

@@ -209 +209 @@
     return("$ENV{'PBTMP'}/$f");
 } elsif ($scheme eq "file") {
-    use File::MimeInfo; 
+    eval
+    {
+        require File::MimeInfo;
+        File::MimeInfo->import();
+    };
+    if ($@) {
+        # File::MimeInfo not found
+        die("ERROR: Install File::MimeInfo to handle scheme $scheme\n");
+    }
+
     my $mm = mimetype($path);
     pb_log(2,"mimetype: $mm\n");