Changeset 1153 in ProjectBuilder

Timestamp:

Jan 12, 2011, 2:42:12 AM (13 years ago)

Author:

Bruno Cornec

Message:

• Avoid File::MimeInfo hard requirement. Only abort if not found when needed.
• Improve report when a perl module is missing
• Kill an existing crashed VM using an SSH port needed for another VM (should avoid crashed VM to stay when building for all VMs)
• Use a new parameter vmbuildtm as a timeout before killing the VM (shoudl correspond to build + transfer time)
• use twice the number of VMs for ports in the range for SSH communication to allow for VMs to finish in an unordered way.
• Fix a bug in test modules when using Test simple only
• Mail::Sendmail is now optional for Log module as well, even if not used yet
• Update pb.conf doc with info for vmbuildtm and vmmem
• Ready for 0.10.1

Location:

devel

Files:

• pb-modules/etc/pb.conf.pod (modified) (3 diffs)
• pb-modules/lib/ProjectBuilder/Log.pm (modified) (1 diff)
• pb-modules/t/Base.t (modified) (1 diff)
• pb-modules/t/Log.t (modified) (1 diff)
• pb/bin/pb (modified) (8 diffs)
• pb/lib/ProjectBuilder/CMS.pm (modified) (1 diff)

devel/pb-modules/etc/pb.conf.pod

@@ -491 +491 @@
  Example: vertype default = chroot
 
+=item B<vmbuildtm>
+
+ Nature: Optional
+ Key: OS (could be from the most generic up to the most specific from ostype, osfamily, os, os-ver, os-ver-arch). Default value is set to vmtmout
+ Value: Time in seconds to wait before killing the VM if SSH port already used. This should corresponds to the time to send files, build the project and get packages in the VM.
+ Conffile: project
+ Example: vmbuildtm default = 600,vmbuildtm mandriva-2009.0-x86_64 = 1200
+
 =item B<vmcmd>
 
@@ -524 +532 @@
  Example: vmlogin default = pb
 
+=item B<vmmem>
+
+ Nature: Optional
+ Key: project (as defined in the -p option or PBPROJ environment variable)
+ Value: Memory size in MB to allocate to the VM.
+ Conffile: vm
+ Example: vmmem default = 512
+
 =item B<vmmonport>
 
@@ -592 +608 @@
  Nature: Optional
  Key: OS (could be from the most generic up to the most specific from ostype, osfamily, os, os-ver, os-ver-arch). NB: a default value is highly interesting here.
- Value: Time in seconds to way beofre interacting with the VM. This should corresponds to the time the VM takes to boot.
+ Value: Time in seconds to wait before interacting with the VM. This should correspond to the time the VM takes to boot.
  Conffile: vm
  Example: vmtmout default = 180,vmtmout mandriva-2009.0-x86_64 = 500

devel/pb-modules/lib/ProjectBuilder/Log.pm

@@ -227 +227 @@
 # mails the summary to $to
 sub mailSummary {
-    require Mail::Sendmail;
-    my $self = shift;
-    my $to = shift || "";
-
-    if ($to eq "") {
-        pb_log(0,"Please give a To: address\n");
-        return;
-    }
-    my %mail = (    
-        To => $to,
-        From => "pb\@localhost",
-        Message => $self->summary()
-    );
-    if (! sendmail(%mail)) { 
-        if (defined $Mail::Sendmail::error) {
-            return $Mail::Sendmail::error;
-        } else {
-            return "Unkown error";
-        }
-    }
-    pb_log(0,"Mail send to ". $to ."\n");
+    eval
+    {
+        require Mail::Sendmail;
+        Mail::Sendmail->import();
+    };
+    if ($@) {
+        # Mail::Sendmail not found not sending mail !
+        pb_log(0,"No Mail::Sendmail module found so not sending any mail !\n");
+    } else {
+        my $self = shift;
+        my $to = shift || "";
+
+        if ($to eq "") {
+            pb_log(0,"Please give a To: address\n");
+            return;
+        }
+        my %mail = (    
+            To => $to,
+            From => "pb\@localhost",
+            Message => $self->summary()
+        );
+        if (! sendmail(%mail)) { 
+            if (defined $Mail::Sendmail::error) {
+                return $Mail::Sendmail::error;
+            } else {
+                return "Unkown error";
+            }
+        }
+        pb_log(0,"Mail send to ". $to ."\n");
+    }
 }
 

devel/pb-modules/t/Base.t

@@ -19 +19 @@
 if ($@) {
     require Test;
+    Test->import();
     plan(tests => 1);
     print "# Faking tests as Test::More is not available in an appropriate version\n";

devel/pb-modules/t/Log.t

@@ -19 +19 @@
 if ($@) {
     require Test;
+    Test->import();
     plan(tests => 1);
     print "# Faking tests as test::More is not available in an appropriate version\n";

devel/pb/bin/pb

@@ -523 +523 @@
     if ($@) {
         # Sys::CPU not found, defaulting to 1
+        pb_log(1,"ADVISE: Install Sys::CPU to benefit from automatic parallelism optimization.\nOr use pbparallel in your pb.conf file\nOnly 1 process at a time for the moment\n");
         $pbparallel = 1;
     } else {
         # Using the number of cores
         $pbparallel = Sys::CPU::cpu_count();
+        pb_log(1,"Using parallel mode with $pbparallel processes\n");
     }
 }
@@ -538 +540 @@
 if ($@) {
     $pbparallel = undef;
+    pb_log(1,"ADVISE: Install Parallel::ForkManager to benefit from automatic parallelism optimization.\nOnly 1 process at a time for the moment\n");
 }
 
@@ -1805 +1808 @@
             # For monitoring control
             if ((($vmtype->{$ENV{'PBPROJ'}}) eq "kvm") || (($vmtype->{$ENV{'PBPROJ'}}) eq "qemu")) {
+                eval
+                {
                 require Net::Telnet;
-                my $t = new Net::Telnet (Timeout => 120, Host => "localhost", Port => $vmmonport->{$ENV{'PBPROJ'}}) || die "Unable to dialog on the monitor";
-                # move to monitor mode
-                my @lines = $t->cmd("c");
-                # Create a snapshot named pb
-                @lines = $t->cmd("savevm pb");
-                # Write the new status in the VM
-                @lines = $t->cmd("commit all");
-                # End
-                @lines = $t->cmd("quit");
+                Net::Telnet->import();
+                };
+                if ($@) {
+                    # Net::Telnet not found
+                    pb_log(1,"ADVISE: Install Net::Telnet to benefit from monitoring control and snapshot feature.\nWARNING: No snapshot created");
+                } else {
+                    my $t = new Net::Telnet (Timeout => 120, Host => "localhost", Port => $vmmonport->{$ENV{'PBPROJ'}}) || die "Unable to dialog on the monitor";
+                    # move to monitor mode
+                    my @lines = $t->cmd("c");
+                    # Create a snapshot named pb
+                    @lines = $t->cmd("savevm pb");
+                    # Write the new status in the VM
+                    @lines = $t->cmd("commit all");
+                    # End
+                    @lines = $t->cmd("quit");
+                }
             }
         }
@@ -1911 +1923 @@
         # TODO: vmmonport should be optional
         my ($ptr,$ptr2,$vmpath,$vmport,$vmsize,$vmmonport) = pb_conf_get("vmtype","vmcmd","vmpath","vmport","vmsize","vmmonport");
-        my ($vmopt,$vmmem,$vmtmout,$vmsnap) = pb_conf_get_if("vmopt","vmmem","vmtmout","vmsnap");
+        my ($vmopt,$vmmem,$vmtmout,$vmsnap,$vmbuildtm) = pb_conf_get_if("vmopt","vmmem","vmtmout","vmsnap","vmbuildtm");
 
         my $vmtype = $ptr->{$ENV{'PBPROJ'}};
@@ -2001 +2013 @@
                 pb_log(0,"Unable to find VM $vmm\n");
             } else {
+                # Is the SSH port free? if not kill the existing process using it after a build timeout period
+                my $vmssh = pb_check_ps($tmpcmd,"tcp:$nport:10.0.2.15:22");
+                if ($vmssh) {
+                    my $buildtm = $ENV{'PBVMTMOUT'};
+                    if (defined $vmbuildtm->{$v}) {
+                        $buildtm = $vmbuildtm->{$v};
+                    } elsif (defined $vmbuildtm->{$ENV{'PBPROJ'}}) {
+                        $buildtm = $vmbuildtm->{$ENV{'PBPROJ'}};
+                    }
+
+                    sleep $buildtm;
+                    pb_log(0,"WARNING: Killing the process ($vmssh) using port $nport (previous failed VM ?)\n");
+                    kill 15,$vmssh;
+                }
                 pb_system("$cmd &","Launching the VM $vmm");
+                # Using system allows to kill it externaly if needed
                 pb_system("sleep $ENV{'PBVMTMOUT'}","Waiting $ENV{'PBVMTMOUT'} s for VM $v to come up");
                 $vmpid = pb_check_ps($tmpcmd,$vmm);
@@ -2238 +2265 @@
 foreach my $v (@$v) {
     $counter++;
-    # Modulo pbparallel
-    $counter = 1 if ($counter > $pbparallel);
+    # Modulo 2 * pbparallel (to avoid synchronization problems)
+    $counter = 1 if ($counter > 2 * $pbparallel);
     $pm->start($counter) and next if (defined $pbparallel);
     # Prepare the script to be executed on the VM/VE
@@ -3498 +3525 @@
     };
     if ($@) {
-        # Linux::SysInfo not found, defaulting to 4
-        $pbparallel = 4;
+        # Linux::SysInfo not found
+        pb_log(1,"ADVISE: Install Linux::SysInfo to benefit from automatic parallelism optimization.\nOr optimize manually pbparallel in your pb.conf file\nUsing $pbparallel processes max at a time for the moment\nWARNING: This may consume too much memory for your system");
     } else {
         # Using the memory size
         my $si = Linux::SysInfo::sysinfo();
         if (not defined $si) {
-            $pbparallel = 4 
+            pb_log(1,"ADVISE: Install Linux::SysInfo to benefit from automatic parallelism optimization.\nOr optimize manually pbparallel in your pb.conf file\nUsing $pbparallel processes max at a time for the moment\nWARNING: This may consume too much memory for your system");
         } else {
             # Keep the number of VM whose memory can be allocated
             my $ram = $si->{"totalram"}-$si->{"sharedram"}-$si->{"bufferram"};
             my $ram2;
-            my ($pbvmmem) = pb_conf_get_if("pbvmmem");
-
-            if ((defined $pbvmmem) and (defined $pbvmmem->{$ENV{'PBPROJ'}})) {
-                $ram2 = $pbvmmem->{$ENV{'PBPROJ'}};
+            my ($vmmem) = pb_conf_get_if("vmmem");
+
+            my $v = "default";
+            if ((defined $vmmem) and (defined $vmmem->{$v})) {
+                $ram2 = $vmmem->{$v};
             } else {
                 # Default for KVM/QEMU
@@ -3519 +3547 @@
             $pbparallel = sprintf("%d",$ram/$ram2);
         }
+        pb_log(1,"Using $pbparallel processes at a time\n");
     }
 }

devel/pb/lib/ProjectBuilder/CMS.pm

@@ -209 +209 @@
     return("$ENV{'PBTMP'}/$f");
 } elsif ($scheme eq "file") {
-    use File::MimeInfo; 
+    eval
+    {
+        require File::MimeInfo;
+        File::MimeInfo->import();
+    };
+    if ($@) {
+        # File::MimeInfo not found
+        die("ERROR: Install File::MimeInfo to handle scheme $scheme\n");
+    }
+
     my $mm = mimetype($path);
     pb_log(2,"mimetype: $mm\n");