#!/usr/bin/perl -w # # Project Builder main application # # $Id$ # # Copyright B. Cornec 2007-2011 # Provided under the GPL v2 # Syntax: see at end use strict 'vars'; # The modules mentioned here are required by pb when used both # locally or inside a VE/VM/RM # Additional required modules only used locally are called with a require # in their respective section use Getopt::Long qw(:config auto_abbrev no_ignore_case); use Carp 'cluck'; use Data::Dumper; use English; use File::Basename; use File::Copy; use File::stat; use File::Temp qw(tempdir); use File::Find; use Time::localtime qw(localtime); use POSIX qw(strftime); use lib qw (lib); use ProjectBuilder::Version; use ProjectBuilder::Base; use ProjectBuilder::Display; use ProjectBuilder::Conf; use ProjectBuilder::Distribution; use ProjectBuilder::VCS; use ProjectBuilder::CMS; use ProjectBuilder::Env; use ProjectBuilder::Filter; use ProjectBuilder::Changelog; use ProjectBuilder::VE; # Global variables $Global::pb_stop_on_error = 1; my %opts; # CLI Options my $action; # action to realize my $test = "FALSE"; # Not used my $pbforce = 0; # Force VE/VM rebuild my $pbsnap = 0; # Do not use snapshot mode for VM/VE by default my $pbkeep = 0; # keep temporary directory at the end my $option = ""; # Not used my @pkgs; # list of packages my $pbtag; # Global Tag variable my $pbver; # Global Version variable my $pbscript; # Name of the script my %pbver; # per package my %pbtag; # per package my $pbrev; # Global REVISION variable my $pbaccount; # Login to use to connect to the VM/RM my $pbtarget = undef; # Target os-ver-arch you want to build for my $pbport; # Port to use to connect to the VM/RM my $newver; # New version to create my $iso = undef; # ISO image for the VM to create my @date = pb_get_date(); my $pbdate = strftime("%Y-%m-%d", @date); =pod =head1 NAME pb, aka project-builder.org - builds packages for your projects =head1 DESCRIPTION pb helps you build various packages directly from your project sources. Those sources could be handled by a CMS (Configuration Management System) such as Subversion, CVS, Git, Mercurial... or being a simple reference to a compressed tar file. It's based on a set of configuration files, a set of provided macros to help you keeping build files as generic as possible. For example, a single .spec file should be required to generate for all rpm based distributions, even if you could also have multiple .spec files if required. =head1 SYNOPSIS pb [-vhSq][-r pbroot][-p project][[-s script -a account -P port][-t [os-ver-arch]][-m os-ver-arch[,...]]][-g][-i iso] [ ...] pb [--verbose][--help][--man][--quiet][--snapshot][--revision pbroot][--project project][[--script script --account account --port port][--target [os-ver-arch]][--machine os-ver-arch[,...]]][--nographic][--iso iso][--rebuild] [ ...] =head1 OPTIONS =over 4 =item B<-v|--verbose> Print a brief help message and exits. =item B<-q|--quiet> Do not print any output. =item B<-h|--help> Print a brief help message and exits. =item B<-S|--snapshot> Use the snapshot mode of VMs or VEs =item B<--man> Prints the manual page and exits. =item B<-t|--target os-ver-arch> Name of the target system you want to build for. All if none precised. =item B<-m|--machine os-ver-arch[,os-ver-arch,...]> Name of the Virtual Machines (VM), Virtual Environments (VE) or Remote Machines (RM) you want to build on (comma separated). All if none precised (or use the env variable PBV). =item B<-s|--script script> Name of the script you want to execute on the related VMs/VEs/RMs. =item B<-g|--nographic> Do not launch VMs in graphical mode. =item B<-i|--iso iso_image> Name of the ISO image of the distribution you want to install on the related VMs. =item B<-a|--account account> Name of the account to use to connect on the related VMs/RMs. =item B<-P|--port port_number> Port number to use to connect on the related VMs/RMs."; =item B<-p|--project project_name> Name of the project you're working on (or use the env variable PBPROJ) =item B<-r|--revision revision> Path Name of the project revision under the CMS (or use the env variable PBROOT) =item B<-V|--version new_version> New version of the project to create based on the current one. =item B<-k|--keep> Keep the temporary dir where files have been created in or der to help debug =item B<--rebuild> Only valid with the checkssh action, it alllows to automatically relaunch the build of the failed packages =item B<--no-stop-on-error> Continue through errors with best effort. =back =head1 ARGUMENTS can be: =over 4 =item B Create tar files for the project under your CMS. Current state of the exported content is taken. CMS supported are SVN, SVK, CVS, Git and Mercurial parameters are packages to build if not using default list =item B Create tar files for the project under your CMS. Current state of the CMS is taken. CMS supported are SVN, SVK, CVS, Git and Mercurial parameters are packages to build if not using default list =item B Create packages for your running distribution =item B cms2build + build2pkg =item B sbx2build + build2pkg =item B Send the tar files to a SSH host =item B sbx2build + build2ssh =item B cms2build + build2ssh =item B Send the packages built to a SSH host =item B Create packages in VMs, launching them if needed and send those packages to a SSH host once built VM type supported are QEMU and KVM =item B Create packages in VEs, creating it if needed and send those packages to a SSH host once built =item B Create packages in RMs, which should pre-exist, and send those packages to a SSH host once built RM means Remote Machine, and could be a physical or Virtual one. This is one buildfarm integration for pb. =item B sbx2build + build2vm =item B sbx2build + build2ve =item B sbx2build + build2rm =item B cms2build + build2vm =item B cms2build + build2ve =item B cms2build + build2rm =item B Launch one virtual machine =item B Launch one virtual environment =item B Launch one virtual machine if needed and executes a script on it =item B Execute a script in a virtual environment =item B Execute a script on a remote machine =item B Create a new virtual machine =item B Create a new virtual environment =item B Setup a virtual machine for pb usage =item B Setup a virtual environment for pb usage =item B Setup a remote machine for pb usage =item B Setup a virtual machine for pb usage using the sandbox version of pb instead of the latest stable Reserved to dev team. =item B Setup a virtual environment for pb usage using the sandbox version of pb instead of the latest stable Reserved to dev team. =item B Setup a remote machine for pb usage using the sandbox version of pb instead of the latest stable Reserved to dev team. =item B Snapshot a virtual machine for pb usage =item B Snapshot a virtual environment for pb usage =item B Update the distribution in the virtual machine =item B Update the distribution in the virtual environment =item B Update the distribution in the remote machine =item B Test a package locally =item B Test a package in a virtual machine =item B Test a package in a virtual environment =item B Test a package in a remote machine =item B Check the delivery of the packages on the repository =item B Create a new version of the project derived from the current one =item B Create a new project and a template set of configuration files under pbconf =item B Announce the availability of the project through various means =item B Create tar files for the website under your CMS. Current state of the exported content is taken. Deliver the content to the target server using ssh from the exported dir. =item B Create tar files for the website from your CMS. Deliver the content to the target server using ssh from the DVCS. =item B Create tar files for the website under your CMS. Current state of the exported content is taken. =item B Create tar files for the website under your CMS. =item B Print the full configuration parameters as found in the various configuration files. help to debug conf issues. =item B Purge the build and delivery directories related to the current project =item B Purge the ssh server of its packages (only for testver and test packages) =back can be a list of packages, the keyword 'all' or nothing, in which case the default list of packages is taken (corresponding to the defpkgdir list of arguments in the configuration file). =head1 WEB SITES The main Web site of the project is available at L. Bug reports should be filled using the trac instance of the project at L. =head1 USER MAILING LIST None exists for the moment. =head1 CONFIGURATION FILES Each pb user may have a configuration in F<$HOME/.pbrc>. The values in this file may overwrite any other configuration file value. Here is an example of such a configuration file: # # Define for each project the URL of its pbconf repository # No default option allowed here as they need to be all different # # URL of the pbconf content # This is the format of a classical URL with the extension of additional schema such as # svn+ssh, cvs+ssh, ... # pbconfurl linuxcoe = cvs+ssh://:ext:bcornec@linuxcoe.cvs.sourceforge.net:/cvsroot/linuxcoe/pbconf # This is normaly defined in the project's configuration file # Url of the project # pburl linuxcoe = cvs+ssh://:ext:bcornec@linuxcoe.cvs.sourceforge.net:/cvsroot/linuxcoe # All these URLs needs to be defined here as the are the entry point # for how to build packages for the project # pbconfurl pb = svn+ssh://svn.project-builder.org/mondo/svn/pb/pbconf pbconfurl mondorescue = svn+ssh://svn.project-builder.org/mondo/svn/project-builder/mondorescue/pbconf pbconfurl collectl = svn+ssh://bruno@svn.mondorescue.org/mondo/svn/project-builder/collectl/pbconf pbconfurl netperf = svn+ssh://svn.mondorescue.org/mondo/svn/project-builder/netperf/pbconf # Under that dir will take place everything related to pb # If you want to use VMs/chroot/..., then use $ENV{'HOME'} to make it portable # to your VMs/chroot/... # if not defined then /var/cache pbdefdir default = $ENV{'HOME'}/project-builder pbdefdir pb = $ENV{'HOME'} pbdefdir linuxcoe = $ENV{'HOME'}/LinuxCOE/cvs pbdefdir mondorescue = $ENV{'HOME'}/mondo/svn # pbconfdir points to the directory where the CMS content of the pbconfurl is checked out # If not defined, pbconfdir is under pbdefdir/pbproj/pbconf pbconfdir linuxcoe = $ENV{'HOME'}/LinuxCOE/cvs/pbconf pbconfdir mondorescue = $ENV{'HOME'}/mondo/svn/pbconf # pbdir points to the directory where the CMS content of the pburl is checked out # If not defined, pbdir is under pbdefdir/pbproj # Only defined if we have access to the dev of the project pbdir linuxcoe = $ENV{'HOME'}/LinuxCOE/cvs pbdir mondorescue = $ENV{'HOME'}/mondo/svn # -daemonize doesn't work with qemu 0.8.2 vmopt default = -m 384 =head1 COMMAND DETAILS =head2 newproj The newproj command creates a new project-builder project. To run this command you first need to define two variables in your ~/.pbrc file: pbconfurl I = file:///home/anderse/.git/project-builder-config/I pbdefdir default = $ENV{HOME}/cache-project-builder The first line defines the version controlled configuration information and the second defines the root directory for project-builder to use. You can then run the command: % pb -p I<$project> -r I<$version> newproj I<$pkg> to create the new project. Running the newproj command will then generate the file $pbdefdir/$project/pbconf/$version/$project.pb, and the directory $pbdefdir/$project/pbconf/$version/$pkg. You will need to edit those files to make the later commands work. =head2 cms2build The cms2build command takes your files from the content management system and makes the two tar files that are necessary for building files. You need to have run the newproj command first. Then there are several steps for running this command: =over 4 =item Update your $project.pb configuration file. You need to set the pburl, pbrepo, pbwf, pbpackager, projver, projtag, testver, delivery, and defpkgdir lines as described in the configuration file. The pburl entry is used to find the source for your package. The pbrepo entry is used to build the .repo or .sources.list files for use by downloaders of the package. The pbwf entry indicates that the source tar file is named by I-I. The pbpackager entry will be stored in the packages and should be you or your team. The projver/projtag entries indicate the version of the software and the version of the packaging scripts. The testver entry when true indicates that the package is in a test version, so no log file is computed (can be long), and version is made up using a timstamp. The delivery entry gives the subdirectory under which the packages will be delivered on the repository, and the defpkgdir entry corresponds to the local subdirectory hosting the package content. For example: pburl Lintel = file:///home/anderse/projects/Lintel-0.2012.02.28.tar.gz pbrepo Lintel = http://tesla.hpl.hp.com/opensource pbwf Lintel = 1 pbpackager Lintel = Eric Anderson projver Lintel = 0.2012.02.28 projtag Lintel = 1 testver Lintel = false delivery Lintel = production defpkgdir Lintel = Lintel-0.2012.02.28 =item Create the build .tar.gz files: Then you need to take those files and create the initial tar files. Run a command like: % pb -p $project -r $version cms2build To create the $pbdefdir/$project/delivery/$project-$version.{,pbconf}.tar.gz files, the $version-$projtag.pb and pbrc files in the same directory. =back =head2 build2pkg The build2pkg command takes the tar files created in the cms2build step and attempts to build binary packages for your current operating system. There are two steps: =over 4 =item Update your filters and build files. You probably need to edit the files describing the build steps in one of the $pbdefdir/$project/pbconf/$version/$project/{deb,rpm,pkg} directories and the filters in $pbdefdir/$project/pbconf/$version/pbfilter. Note that you can define additional filters and transforms in the *.pbf files. The build files will be filtered by the filters defined in the *.pbf files to generate the inputs to the build step. Therefore, if you change those files, you need to re-run the cms2build step. =item Build the package. Then you can run a command like: % pb -p $project -r $version build2pkg To create the files in $project/build that comprise your binary package(s). =back =head2 newve The newve command creates a new virtual environment, i.e. a chrooted OS for building packages. Using a virtual environment is an efficient way to build packages on a related set of operating systems. The OS's have to be related because the kernel will be shared. Steps: =over 4 =item Update ~/.pbrc Update your ~/.pbrc file to specify the vepath, velist, velogin, and vetype variables, e.g.: vepath default = $ENV{HOME}/cache-project-builder/chroot velist default = debian-6.0-i386 velogin default = pb vetype default = chroot If you are building for rpm style OS's, update the verpmtype option, and install the appropriate tool. verpmtype default = rpmbootstrap You may also choose to specify a mirror for the OS packages, and optionally http/ftp proxies. You can specify the proxies either through environment variables ($http_proxy/$ftp_proxy) or in the configuration file. The configuration file will be used if no corresponding environment variable has been set. For example, for debian and with a local squid proxy: rbsmirrorsrv debian = http://mirrors1.kernel.org/debian/ http_proxy default = http://localhost:3128/ ftp_proxy default = http://localhost:3128/ =item Run the cms2build command If you have deleted your $package/delivery directory, re-run the cms2build command as in the earlier step. This step is necessary to generate the I/delivery/pbrc file. =item Create the new virtual environment Initialize the new operating system. This step will install the core OS packages for the virtual environment, e.g.: % pb -v -p $project -m debian-6.0-i386 newve =back =head2 setupve The setupve command prepares a virtual environment for use by project builder. In particular it installs project-builder from the packages into the virtual environment. Two sub-steps are necessary: =over 4 =item Update $project.pb You need to have a sshhost entry for setupve to work, so add one, even an invalid one, e.g.: sshhost $project = foo.example.org =item Setup the virtual environment % pb -v -p $project -m debian-6.0-i386 setupve If you prefer to install the current SVN version of project builder, you can substitute the setupve option by the sbx2setupv one. =back =head2 build2ve The build2ve command is similar to the build2pkg command in that it will take the sources created by cms2build and turn them into binary packages. The command has two differences. First, it creates the packages in a virtual environment, i.e. the one made by an earlier setupve setup. Second it copies the resulting packages to a repository and builds the repository meta-data needed. Three sub-steps are needed: =over 4 =item Update $project.pb You need to have a valid sshdir and sshhost entry for build2ve to work, so add them. Note that you need to be able to ssh from the host you run the command on to the repository host, preferably without needing to type in a password, so using ssh-agent or having a special passwordless project-builder ssh key will make this step easier. sshhost $project = localhost sshdir $project = $home/cache-project-builder/repos You may also need to specify additional repository files to use or rpms to install. Note the URL for repositories is not the URL of the repository, but the URL of a file that can be put in the yum.repos.d or apt.sources.d directory. addrepo centos-5-i386 = http://localhost/pb/centos-extras.repo,http://mirror.centos.org/centos/5/extras/i386/RPMS/chrpath-0.13-3.el5.centos.i386.rpm =item Update your filters and build files You may need to update your filter files (*.pbf) as in the build2pkg step if you are building for a new OS or architecture. =item Build the packages and copy them to the repository % pb -v -p $project -m debian-6.0-i386 build2ve =back *Debugging:* If the build fails (and you did not specify the --no-stop-on-error) option, then the virtual environment and scripts should still be present and configured to build the package. You can run a command like 'sudo setarch i386 chroot $path bash' in order to get into the environment. In your log you should see a command like that. From there you can go into the /home/pb directory as the pb user and run the same style of pb commands as you did when doing build2pkg. This will help you figure out what has gone wrong in the build in the virtual environment. =head1 AUTHORS The Project-Builder.org team L lead by Bruno Cornec L. =head1 COPYRIGHT Project-Builder.org is distributed under the GPL v2.0 license described in the file C included with the distribution. =cut # --------------------------------------------------------------------------- my ($projectbuilderver,$projectbuilderrev) = pb_version_init(); my $appname = "pb"; # Initialize the syntax string pb_syntax_init("$appname (aka project-builder.org) Version $projectbuilderver-$projectbuilderrev\n"); GetOptions("help|?|h" => \$opts{'h'}, "man" => \$opts{'man'}, "verbose|v+" => \$opts{'v'}, "snapshot|S" => \$opts{'S'}, "quiet|q" => \$opts{'q'}, "log-files|l=s" => \$opts{'l'}, "force|f" => \$opts{'f'}, "account|a=s" => \$opts{'a'}, "revision|r=s" => \$opts{'r'}, "script|s=s" => \$opts{'s'}, "machines|mock|m=s" => \$opts{'m'}, "target|t:s" => \$opts{'t'}, "nographic|g" => \$opts{'g'}, "port|P=i" => \$opts{'P'}, "project|p=s" => \$opts{'p'}, "rebuild" => \$opts{'rebuild'}, "iso|i=s" => \$opts{'i'}, "version|V=s" => \$opts{'V'}, "keep|k" => \$opts{'k'}, "stop-on-error!" => \$Global::pb_stop_on_error, ) || pb_syntax(-1,0); if (defined $opts{'h'}) { pb_syntax(0,1); } if (defined $opts{'man'}) { pb_syntax(0,2); } if (defined $opts{'v'}) { $pbdebug = $opts{'v'}; } if (defined $opts{'f'}) { $pbforce=1; } if (defined $opts{'q'}) { $pbdebug=-1; } if (defined $opts{'S'}) { $pbsnap=1; } if (defined $opts{'k'}) { $pbkeep=1; } if (defined $opts{'l'}) { open(pbLOG,"> $opts{'l'}") || die "Unable to log to $opts{'l'}: $!"; $pbLOG = \*pbLOG; $pbdebug = 0 if ($pbdebug == -1); } pb_log_init($pbdebug, $pbLOG); pb_display_init("text",""); # Handle root of the project if defined if (defined $opts{'r'}) { $ENV{'PBROOTDIR'} = $opts{'r'}; } # Handle virtual machines if any if (defined $opts{'m'}) { $ENV{'PBV'} = $opts{'m'}; } if (defined $opts{'s'}) { $pbscript = $opts{'s'}; } if (defined $opts{'a'}) { $pbaccount = $opts{'a'}; die "option -a requires a -s script option" if (not defined $pbscript); } if (defined $opts{'P'}) { $pbport = $opts{'P'}; } if (defined $opts{'V'}) { $newver = $opts{'V'}; } if (defined $opts{'i'}) { $iso = $opts{'i'}; } if (defined $opts{'t'}) { $pbtarget = $opts{'t'}; } # Get Action $action = shift @ARGV; die pb_syntax(-1,1) if (not defined $action); my ($filteredfiles, $supfiles, $defpkgdir, $extpkgdir); my $pbinit = undef; $pbinit = 1 if ($action =~ /^newproj$/); # Handles project name if any # And get global params ($filteredfiles, $supfiles, $defpkgdir, $extpkgdir) = pb_env_init($opts{'p'},$pbinit,$action,$pbkeep); # # Check for command requirements # my ($req,$opt,$pbpara) = pb_conf_get_if("oscmd","oscmdopt","pbparallel"); pb_check_requirements($req,$opt,$appname); # # Check if we can launch some actions in // with Parallel::ForkManager # my $pbparallel = $pbpara->{$appname} if (defined $pbpara); if (not defined $pbparallel) { eval { require Sys::CPU; Sys::CPU->import(); }; if ($@) { # Sys::CPU not found, defaulting to 1 pb_log(1,"ADVISE: Install Sys::CPU to benefit from automatic parallelism optimization.\nOr use pbparallel in your pb.conf file\nOnly 1 process at a time for the moment\n"); $pbparallel = 1; } else { # Using the number of cores $pbparallel = Sys::CPU::cpu_count(); pb_log(1,"Using parallel mode with $pbparallel processes\n"); } } eval { require Parallel::ForkManager; Parallel::ForkManager->import(); }; # Parallel::ForkManager not found so no // actions if ($@) { $pbparallel = undef; pb_log(1,"ADVISE: Install Parallel::ForkManager to benefit from automatic parallelism optimization.\nOnly 1 process at a time for the moment\n"); } pb_log(0,"Project: $ENV{'PBPROJ'}\n"); pb_log(0,"Action: $action\n"); # Act depending on action if ($action =~ /^cms2build$/) { pb_cms2build("CMS"); } elsif ($action =~ /^sbx2build$/) { pb_cms2build("SandBox"); } elsif ($action =~ /^build2pkg$/) { pb_build2pkg(); } elsif ($action =~ /^cms2pkg$/) { pb_cms2build("CMS"); pb_build2pkg(); } elsif ($action =~ /^sbx2pkg$/) { pb_cms2build("SandBox"); pb_build2pkg(); } elsif ($action =~ /^build2ssh$/) { pb_build2ssh(); } elsif ($action =~ /^cms2ssh$/) { pb_cms2build("CMS"); pb_build2ssh(); } elsif ($action =~ /^sbx2ssh$/) { pb_cms2build("SandBox"); pb_build2ssh(); } elsif ($action =~ /^pkg2ssh$/) { pb_pkg2ssh(); } elsif ($action =~ /^build2rm$/) { pb_build2v("rm","build"); } elsif ($action =~ /^build2ve$/) { pb_build2v("ve","build"); } elsif ($action =~ /^build2vm$/) { pb_build2v("vm","build"); } elsif ($action =~ /^cms2rm$/) { pb_cms2build("CMS"); pb_build2v("rm","build"); } elsif ($action =~ /^cms2ve$/) { pb_cms2build("CMS"); pb_build2v("ve","build"); } elsif ($action =~ /^sbx2rm$/) { pb_cms2build("SandBox"); pb_build2v("rm","build"); } elsif ($action =~ /^sbx2ve$/) { pb_cms2build("SandBox"); pb_build2v("ve","build"); } elsif ($action =~ /^cms2vm$/) { pb_cms2build("CMS"); pb_build2v("vm","build"); } elsif ($action =~ /^sbx2vm$/) { pb_cms2build("SandBox"); pb_build2v("vm","build"); } elsif ($action =~ /^launchvm$/) { pb_launchv("vm",$ENV{'PBV'},0); } elsif ($action =~ /^launchve$/) { pb_launchv("ve",$ENV{'PBV'},0); } elsif ($action =~ /^script2vm$/) { pb_script2v($pbscript,"vm"); } elsif ($action =~ /^script2ve$/) { pb_script2v($pbscript,"ve"); } elsif ($action =~ /^script2rm$/) { pb_script2v($pbscript,"rm"); } elsif ($action =~ /^newver$/) { pb_newver(); } elsif ($action =~ /^newve$/) { pb_launchv("ve",$ENV{'PBV'},1); } elsif ($action =~ /^newvm$/) { pb_launchv("vm",$ENV{'PBV'},1); pb_log(0, "Please ensure that sshd is running in your VM by default\n"); pb_log(0, "and that it allows remote root login (PermitRootLogin yes in /etc/ssh/sshd_config)\n"); pb_log(0, "Also ensure that network is up, firewalling correctly configured\n"); pb_log(0, "and perl, sudo, ntpdate and scp/ssh installed\n"); pb_log(0, "You should then be able to login with ssh -p VMPORT root\@localhost (if VM started with pb)\n"); } elsif ($action =~ /^setuprm$/) { pb_setup2v("rm"); } elsif ($action =~ /^setupve$/) { pb_setup2v("ve"); } elsif ($action =~ /^setupvm$/) { pb_setup2v("vm"); } elsif ($action =~ /^sbx2setuprm$/) { die "This feature is limited to the pb project" if ($ENV{'PBPROJ'} ne $appname); pb_cms2build("SandBox"); pb_setup2v("rm","SandBox"); } elsif ($action =~ /^sbx2setupve$/) { die "This feature is limited to the pb project" if ($ENV{'PBPROJ'} ne $appname); pb_cms2build("SandBox"); pb_setup2v("ve","SandBox"); } elsif ($action =~ /^sbx2setupvm$/) { die "This feature is limited to the pb project" if ($ENV{'PBPROJ'} ne $appname); pb_cms2build("SandBox"); pb_setup2v("vm","SandBox"); } elsif ($action =~ /^updaterm$/) { pb_update2v("rm"); } elsif ($action =~ /^updateve$/) { pb_update2v("ve"); } elsif ($action =~ /^updatevm$/) { pb_update2v("vm"); } elsif ($action =~ /^snapve$/) { pb_snap2v("ve"); } elsif ($action =~ /^snapvm$/) { pb_snap2v("vm"); } elsif ($action =~ /^test2pkg$/) { pb_test2pkg(); } elsif ($action =~ /^test2rm$/) { pb_build2v("rm","test"); } elsif ($action =~ /^test2ve$/) { pb_build2v("ve","test"); } elsif ($action =~ /^test2vm$/) { pb_build2v("vm","test"); } elsif ($action =~ /^newproj$/) { # Nothing to do - already done in pb_env_init } elsif ($action =~ /^getconf$/) { my $pbos = pb_distro_get_context(); pb_conf_print(); } elsif ($action =~ /^clean$/) { pb_clean(); } elsif ($action =~ /^announce$/) { # For announce only. Require avoids the systematic load of these modules require DBI; require DBD::SQLite; pb_announce("Announce"); } elsif ($action =~ /^cleanssh$/) { pb_announce("Clean"); } elsif ($action =~ /^checkssh$/) { pb_announce("Check"); } elsif ($action =~ /^sbx2webpkg$/) { require DBI; require DBD::SQLite; pb_cms2build("SandBox","Web"); } elsif ($action =~ /^sbx2webssh$/) { require DBI; require DBD::SQLite; pb_cms2build("SandBox","Web"); pb_send2target("Web"); } elsif ($action =~ /^cms2webpkg$/) { require DBI; require DBD::SQLite; pb_cms2build("CMS","Web"); } elsif ($action =~ /^cms2webssh$/) { require DBI; require DBD::SQLite; pb_cms2build("CMS","Web"); pb_send2target("Web"); } else { pb_log(0,"\'$action\' is not available\n"); pb_syntax(-2,1); } sub pb_cms2build { my $param = shift || undef; my $web = shift || undef; my $pkg; my @pkgs; my $webdir; my %pkgs; my $pb; # Structure to store conf info die "pb_cms2build requires a parameter: SandBox or CMS" if (not defined $param); # If Website, then pkg is only the website if (defined $web) { ($webdir) = pb_conf_get("webdir"); pb_log(2,"webdir: ".Dumper($webdir)."\n"); $pkgs[0] = $webdir->{$ENV{'PBPROJ'}}; $extpkgdir = $webdir; pb_log(0,"Package: $pkgs[0]\n"); } else { $pkg = pb_cms_get_pkg($defpkgdir,$extpkgdir); @pkgs = @$pkg; } my ($scheme, $uri) = pb_cms_init($pbinit,$param); # We need 2 lines here my ($pkgv, $pkgt, $testver) = pb_conf_get_if("pkgver","pkgtag","testver"); # declare packager and repo for filtering my ($tmp1, $tmp2) = pb_conf_get("pbpackager","pbrepo"); $ENV{'PBPACKAGER'} = $tmp1->{$ENV{'PBPROJ'}}; $ENV{'PBREPO'} = $tmp2->{$ENV{'PBPROJ'}}; my ($delivery) = pb_conf_get_if("delivery"); $delivery->{$ENV{'PBPROJ'}} = "" if (not defined $delivery->{$ENV{'PBPROJ'}}); # If we deal with a test dir, we want to keep the date in tar file and dir name my $pbextdir = ""; if ((defined $testver) && (defined $testver->{$ENV{'PBPROJ'}}) && ($testver->{$ENV{'PBPROJ'}} =~ /true/i)) { $pbextdir = strftime("%Y%m%d%H%M%S", @date); } foreach my $pbpkg (@pkgs) { $ENV{'PBPKG'} = $pbpkg; if ((defined $pkgv) && (defined $pkgv->{$pbpkg})) { $pbver = $pkgv->{$pbpkg}; } else { $pbver = $ENV{'PBPROJVER'}; } # If it's a test version, then tag == 0 if ((defined $testver) && (defined $testver->{$ENV{'PBPROJ'}}) && ($testver->{$ENV{'PBPROJ'}} =~ /true/i)) { $pbtag = "0"; $ENV{'PBPROJTAG'} = $pbtag; } elsif ((defined $pkgt) && (defined $pkgt->{$pbpkg})) { $pbtag = $pkgt->{$pbpkg}; } else { $pbtag = $ENV{'PBPROJTAG'}; } $pbrev = $ENV{'PBREVISION'}; pb_log(0,"\n"); pb_log(0,"Management of $pbpkg $pbver-$pbtag (rev $pbrev)\n"); die "Unable to get env var PBDESTDIR" if (not defined $ENV{'PBDESTDIR'}); my $dest = "$ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir"; # Create the structure if needed pb_mkdir_p($dest); # Clean up dest if necessary. The export will recreate it pb_rm_rf($dest) if (-d $dest); # Export CMS tree for the concerned package to dest # And generate some additional files $OUTPUT_AUTOFLUSH=1; # computes in which dir we have to work my $dir = $defpkgdir->{$pbpkg}; $dir = $extpkgdir->{$pbpkg} if (not defined $dir); $dir = $webdir->{$ENV{'PBPROJ'}} if (defined $web); die "Variable \$dir not defined. Please report to dev team with log of a verbose run and this info ".Dumper($webdir) if (not defined $dir); pb_log(2,"def:".Dumper($defpkgdir)."ext: ".Dumper($extpkgdir)."dir: $dir\n"); # Exporting content from CMS my $sourcedir = undef; my $sourceuri = $uri; if ($param eq "SandBox") { # Point to the local instance $sourcedir = "$ENV{'PBDIR'}/$dir"; } else { # Get it from a subdir of the URI with same version as localy but different root # Only if using a real CMS my ($scheme, $account, $host, $port, $path) = pb_get_uri($uri); if (($scheme !~ /^file/) && ($scheme !~ /^(ht|f)tp/)) { $sourceuri = "$ENV{'PBDIR'}/$dir"; $sourceuri =~ s|^$ENV{'PBPROJDIR'}/|$uri/|; } } my $preserve = pb_vcs_export($sourceuri,$sourcedir,$dest); # Generated fake content for test versions to speed up stuff my $chglog; # Get project info on authors and log file # TODO: Make it CMS aware $chglog = "$ENV{'PBROOTDIR'}/$pbpkg/pbcl"; $chglog = "$ENV{'PBROOTDIR'}/pbcl" if (! -f $chglog); $chglog = undef if (! -f $chglog); # TODO: Make it CMS aware my $authors = "$ENV{'PBROOTDIR'}/$pbpkg/pbauthors"; $authors = "$ENV{'PBROOTDIR'}/pbauthors" if (! -f $authors); $authors = "/dev/null" if (! -f $authors); # Extract cms log history and store it if ((defined $chglog) && (! -f "$dest/NEWS")) { pb_log(2,"Generating NEWS file from $chglog\n"); copy($chglog,"$dest/NEWS") || die "Unable to create $dest/NEWS"; } pb_cms_log($scheme,"$ENV{'PBDIR'}/$dir",$dest,$chglog,$authors,$testver); my %build; # We want to at least build for the underlying distro # except if a target was given, in which case we only build for it # if -t was passed without target then build for the native distro. my $pbos = pb_distro_get_context($pbtarget); my $tmpl = pb_get_distros($pbos,$pbtarget); # Setup $pb structure to allow filtering later on, on files using that structure $pb->{'tag'} = $pbtag; $pb->{'rev'} = $pbrev; $pb->{'ver'} = $pbver; $pb->{'pkg'} = $pbpkg; $pb->{'suf'} = $pbos->{'suffix'}; $pb->{'realpkg'} = $pbpkg; $pb->{'date'} = $pbdate; $pb->{'defpkgdir'} = $defpkgdir; $pb->{'extpkgdir'} = $extpkgdir; $pb->{'chglog'} = $chglog; $pb->{'extdir'} = $pbextdir; $pb->{'packager'} = $ENV{'PBPACKAGER'}; $pb->{'proj'} = $ENV{'PBPROJ'}; $pb->{'repo'} = "$ENV{'PBREPO'}/$delivery->{$ENV{'PBPROJ'}}"; $pb->{'patches'} = (); $pb->{'sources'} = (); my $tmpd = "$ENV{'PBTMP'}/cms.$$"; pb_mkdir_p($tmpd) if (defined $pbparallel); # Get only all.pbf filter at this stage for pbinit my $ptr = pb_get_filters($pbpkg); # Do not do that for website if (not defined $web) { my %virt; # De-duplicate similar VM/VE/RM foreach my $d (split(/,/,$tmpl)) { # skip ill-formatted vms (name-ver-arch) next if ($d !~ /-/); $virt{$d} = $d; } # Try to use // processing here my $all_ok = 1; my $pm = new Parallel::ForkManager($pbparallel) if (defined $pbparallel); $pm->run_on_finish(sub { my ($pid, $code, $id, $signal, $dump) = @_; $all_ok = 0 unless (($code == 0) && ($signal == 0) && ($dump == 0)); }); pb_log(0,"Preparing delivery ...\n"); foreach my $v (keys %virt) { $pm->start and next if (defined $pbparallel); # Distro context my $pbos = pb_distro_get_context($v); $pb->{'pbos'} = $pbos; $pb->{'suf'} = $pbos->{'suffix'}; pb_log(3,"DEBUG: pb: ".Dumper($pb)."\n"); # Get all filters to apply $ptr = pb_get_filters($pbpkg,$pbos); pb_log(2,"DEBUG Filtering PBDATE => $pbdate, PBTAG => $pbtag, PBVER => $pbver\n"); # We need to compute the real name of the package my $pbrealpkg = pb_cms_get_real_pkg($pbpkg,$pbos->{'type'}); $pb->{'realpkg'} = $pbrealpkg; pb_log(1,"Virtual package $pbpkg has a real package name of $pbrealpkg on $pbos->{'name'}-$pbos->{'version'}\n") if ($pbrealpkg ne $pbpkg); # Filter build files from the less precise up to the most with overloading # Filter all files found, keeping the name, and generating in dest # Find all build files first relatively to PBROOTDIR # Find also all specific files referenced in the .pb conf file my %bfiles = (); my %pkgfiles = (); # Used in Filter.pm by pb_filter_file $build{$v} = "no"; if (-d "$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'os'}") { pb_list_bfiles("$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'os'}",$pbpkg,\%bfiles,\%pkgfiles,$supfiles); $build{$v} = "yes"; } if (-d "$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'type'}") { pb_list_bfiles("$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'type'}",$pbpkg,\%bfiles,\%pkgfiles,$supfiles); $build{$v} = "yes"; } if (-d "$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'family'}") { pb_list_bfiles("$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'family'}",$pbpkg,\%bfiles,\%pkgfiles,$supfiles); $build{$v} = "yes"; } if (-d "$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'name'}") { pb_list_bfiles("$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'name'}",$pbpkg,\%bfiles,\%pkgfiles,$supfiles); $build{$v} = "yes"; } if (-d "$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'name'}-$pbos->{'version'}") { pb_list_bfiles("$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'name'}-$pbos->{'version'}",$pbpkg,\%bfiles,\%pkgfiles,$supfiles); $build{$v} = "yes"; } if (-d "$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}") { pb_list_bfiles("$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}",$pbpkg,\%bfiles,\%pkgfiles,$supfiles); $build{$v} = "yes"; } pb_log(2,"DEBUG($v) bfiles: ".Dumper(\%bfiles)."\n"); if ($build{$v} ne "no") { # Apply now all the filters on all the files concerned # destination dir depends on the type of file # For patch support # TODO: Make it CMS aware $pb->{'patches'} = pb_list_sfiles("$ENV{'PBROOTDIR'}/$pbpkg/pbpatch", $pb->{'patches'}, $pbos, "$ENV{'PBROOTDIR'}/$pbpkg/pbextpatch"); pb_log(2,"DEBUG($v) patches: ".Dumper($pb->{'patches'})."\n"); # TODO: Make it CMS aware $pb->{'sources'} = pb_list_sfiles("$ENV{'PBROOTDIR'}/$pbpkg/pbsrc", $pb->{'sources'}, $pbos, "$ENV{'PBROOTDIR'}/$pbpkg/pbextsrc"); pb_log(2,"DEBUG($v) sources: ".Dumper($pb->{'sources'})."\n"); if (defined $pb->{'patches'}->{$v}) { # Filter potential patches (local + remote) pb_mkdir_p("$dest/pbconf/$v/pbpatch"); # If Debian based distribution, then prepare what will be done at build time my ($patchcmd,$patchopt); if ($pbos->{'type'} eq "deb") { ($patchcmd,$patchopt) = pb_distro_get_param($pbos,pb_conf_get_if("ospatchcmd","ospatchopt")); open(SCRIPT,"> $dest/pbconf/$v/pbpatch/pbapplypatch") || die "Unable to create $dest/pbconf/$v/pbpatch/pbapplypatch"; print SCRIPT "#!/bin/bash\n"; print SCRIPT "set -x\n" if ($pbdebug gt 1); } foreach my $pf (split(/,/,$pb->{'patches'}->{$v})) { my $pp = basename($pf); pb_vcs_export($pf,undef,"$dest/pbconf/$v/pbpatch"); pb_filter_file_inplace($ptr,"$dest/pbconf/$v/pbpatch/$pp",$pb); pb_system("gzip -9f $dest/pbconf/$v/pbpatch/$pp","","quiet"); if ($pbos->{'type'} eq "deb") { # If Debian based distribution, then prepare what will be done at build time # by applying the patches that will be available under the debian/patches dir print SCRIPT "$patchcmd $patchopt \< debian/patches/$pp\n"; } } if ($pbos->{'type'} eq "deb") { close(SCRIPT); chmod 0755,"$dest/pbconf/$v/pbpatch/pbapplypatch"; } #pb_system("cat $dest/pbconf/$v/pbpatch/pbapplypatch","APPLY","verbose"); } if (defined $pb->{'sources'}->{$v}) { pb_mkdir_p("$dest/pbconf/$v/pbsrc"); foreach my $pf (split(/,/,$pb->{'sources'}->{$v})) { my $pp = basename($pf); pb_vcs_export($pf,undef,"$dest/pbconf/$v/pbsrc"); pb_filter_file_inplace($ptr,"$dest/pbconf/$v/pbsrc/$pp",$pb); } } # Filter build files at the end, as they depend on patches and sources foreach my $f (keys %bfiles) { pb_filter_file("$ENV{'PBROOTDIR'}/$bfiles{$f}",$ptr,"$dest/pbconf/$v/$f",$pb); } foreach my $f (keys %pkgfiles) { pb_filter_file("$ENV{'PBROOTDIR'}/$pkgfiles{$f}",$ptr,"$dest/pbconf/$v/$f",$pb); } } if (defined $pbparallel) { # Communicate results back to parent my $str = ""; $str .= "build $v = $build{$v}\n" if (defined $build{$v}); $str .= "patches $v = $pb->{'patches'}->{$v}\n" if (defined $pb->{'patches'}->{$v}); $str .= "sources $v = $pb->{'sources'}->{$v}\n" if (defined $pb->{'sources'}->{$v}); pb_set_content("$tmpd/$$","$str"); $pm->finish; } } # In the parent, we need to get the result from the children $pm->wait_all_children if (defined $pbparallel); die "Aborting, one or more of the children failed." if ((not $all_ok) && ($Global::pb_stop_on_error)); my $made = ""; my %h = (); my %tmp; my %tmp2; my $pt; my $k; foreach $k (<$tmpd/*>) { $made .= pb_get_content($k); } pb_rm_rf($tmpd); pb_log(3,"MADE:\n$made"); # Rebuild local hashes foreach $k (split(/\n/,$made)) { if ($k =~ /^\s*([A-z0-9-_]+)\s+([[A-z0-9-_.]+)\s*=\s*(.+)$/) { $h{$1}->{$2}=$3; } } pb_log(2,"HASH: ".Dumper(%h)); # Patches pb_log(0,"Delivered and compressed patches "); $pt = $h{'patches'}; foreach $k (keys %$pt) { foreach my $v1 (split(/,/,$pt->{$k})) { $tmp{$v1} = ""; } } if (keys %tmp) { foreach $k (keys %tmp) { pb_log(0,"$k "); } } else { pb_log(0,"N/A"); } pb_log(0,"\n"); # Sources pb_log(0,"Delivered additional sources "); $pt = $h{'sources'}; foreach $k (keys %$pt) { foreach my $v1 (split(/,/,$pt->{$k})) { $tmp2{$v1} = ""; } } if (keys %tmp2) { foreach $k (keys %tmp2) { pb_log(0,"$k "); } } else { pb_log(0,"N/A"); } pb_log(0,"\n"); # Build files my @found; my @notfound; $pt = $h{'build'}; foreach my $b (keys %$pt) { push @found,$b if ($pt->{$b} =~ /yes/); push @notfound,$b if ($pt->{$b} =~ /no/); } pb_log(0,"Build files have been generated for ... ".join(',',sort(@found))."\n") if (@found); pb_log(0,"No Build files found for ".join(',',sort(@notfound))."\n") if (@notfound); } else { # Instead call News generation pb_web_news2html($dest); # And create an empty pbconf pb_mkdir_p("$dest/pbconf"); # And prepare the pbscript to execute remotely open(SCRIPT,"> $ENV{'PBTMP'}/pbscript") || die "Unable to create $ENV{'PBTMP'}/pbscript"; print SCRIPT "#!/bin/bash\n"; print SCRIPT "#set -x\n"; print SCRIPT "echo ... Extracting Website content\n"; print SCRIPT "find . -type f | grep -Ev '^./$pbpkg-$pbver$pbextdir.tar.gz|^./pbscript' | xargs rm -f non-existent\n"; print SCRIPT "find * -type d -depth | xargs rmdir 2> /dev/null \n"; print SCRIPT "tar xfz $pbpkg-$pbver$pbextdir.tar.gz\n"; print SCRIPT "mv $pbpkg-$pbver$pbextdir/* .\n"; print SCRIPT "rm -f $pbpkg-$pbver$pbextdir.tar.gz\n"; print SCRIPT "rmdir $pbpkg-$pbver$pbextdir\n"; print SCRIPT "find . -type f -print0 | xargs -0 chmod 644\n"; print SCRIPT "find . -type d -print0 | xargs -0 chmod 755\n"; close(SCRIPT); chmod 0755,"$ENV{'PBTMP'}/pbscript"; } # Apply filters to the non-build files my $liste =""; if (defined $filteredfiles->{$pbpkg}) { foreach my $f (split(/,/,$filteredfiles->{$pbpkg})) { pb_filter_file_inplace($ptr,"$dest/$f",$pb); $liste = "$f $liste"; } } pb_log(2,"Files ".$liste."have been filtered\n"); # TODO: Make it CMS aware # Execute the pbinit script if any if (-x "$ENV{'PBROOTDIR'}/$pbpkg/pbinit") { pb_filter_file("$ENV{'PBROOTDIR'}/$pbpkg/pbinit",$ptr,"$ENV{'PBTMP'}/pbinit",$pb); chmod 0755,"$ENV{'PBTMP'}/pbinit"; pb_system("cd $dest ; $ENV{'PBTMP'}/pbinit","Executing init script from $ENV{'PBROOTDIR'}/$pbpkg/pbinit under $dest","verbose"); } # Do we have additional script to run to prepare the environement for the project ? # Then include it in the pbconf delivery foreach my $pbvf (<$ENV{'PBROOTDIR'}/pbv*.pre>,<$ENV{'PBROOTDIR'}/pbv*.post>, <$ENV{'PBROOTDIR'}/pbtest*>) { if (-x "$pbvf") { my $target = "$ENV{'PBDESTDIR'}/".basename($pbvf); pb_filter_file("$pbvf",$ptr,$target,$pb); chmod 0755,"$target"; } } # Prepare the dest directory for archive chdir "$ENV{'PBDESTDIR'}" || die "Unable to change dir to $ENV{'PBDESTDIR'}"; if (defined $preserve) { # In that case we want to preserve the original tar file for checksum purposes # The one created is btw equivalent in that case to this one # Maybe check basename of both to be sure they are the same ? pb_log(0,"Preserving original tar file "); move("$preserve","$pbpkg-$pbver$pbextdir.tar.gz"); } else { # Possibility to look at PBSRC to guess more the filename pb_system("tar cfz $pbpkg-$pbver$pbextdir.tar.gz --exclude=$pbpkg-$pbver$pbextdir/pbconf $pbpkg-$pbver$pbextdir","Creating $pbpkg tar files compressed"); } pb_log(0,"Under $ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.tar.gz\n"); pb_system("tar cfz $pbpkg-$pbver$pbextdir.pbconf.tar.gz $pbpkg-$pbver$pbextdir/pbconf","Creating pbconf tar files compressed"); pb_log(0,"Under $ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.pbconf.tar.gz\n"); # Keep track of version-tag per pkg $pkgs{$pbpkg} = "$pbver-$pbtag"; # Final cleanup pb_rm_rf($dest) if (-d $dest); } # Keep track of per package version pb_log(2,"DEBUG pkgs: ".Dumper(%pkgs)."\n"); open(PKG,"> $ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.pb") || die "Unable to create $ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.pb"; foreach my $pbpkg (keys %pkgs) { print PKG "pbpkg $pbpkg = $pkgs{$pbpkg}\n"; } close(PKG); # Keep track of what is generated by default # We need to store the dir and info on version-tag # Base our content on the existing .pb file copy("$ENV{'PBROOTDIR'}/$ENV{'PBPROJ'}.pb","$ENV{'PBDESTDIR'}/pbrc"); open(LAST,">> $ENV{'PBDESTDIR'}/pbrc") || die "Unable to create $ENV{'PBDESTDIR'}/pbrc"; print LAST "pbroot $ENV{'PBPROJ'} = $ENV{'PBROOTDIR'}\n"; print LAST "projver $ENV{'PBPROJ'} = $ENV{'PBPROJVER'}\n"; print LAST "projtag $ENV{'PBPROJ'} = $ENV{'PBPROJTAG'}\n"; print LAST "pbpackager $ENV{'PBPROJ'} = $ENV{'PBPACKAGER'}\n"; print LAST "pbextdir $ENV{'PBPROJ'} = $pbextdir\n"; close(LAST); } sub pb_test2pkg { # Get the running distro to test on my $pbos = pb_distro_get_context(); # Get list of packages to test # Get content saved in cms2build my $ptr = pb_get_pkg(); @pkgs = @$ptr; # Additional potential repo pb_distro_setuprepo($pbos); foreach my $pbpkg (@pkgs) { # We need to install the package to test, and deps brought with it pb_distro_installdeps(undef,$pbos,$pbpkg); pb_system("$ENV{'PBDESTDIR'}/pbtest","Launching test for $pbpkg","verbose"); } } sub pb_build2pkg { pb_log(0,"INFO: ------ Starting to build package ------\n"); # Get the running distro to build on my $pbos = pb_distro_get_context(); # If needed we may add repository to the build env pb_distro_setuprepo($pbos); # Get list of packages to build my $ptr = pb_get_pkg(); @pkgs = @$ptr; # Get content saved in cms2build my ($pkg) = pb_conf_read("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.pb","pbpkg"); $pkg = { } if (not defined $pkg); my $pbextdir = pb_get_extdir(); pb_mkdir_p("$ENV{'PBBUILDDIR'}") if (! -d "$ENV{'PBBUILDDIR'}"); chdir "$ENV{'PBBUILDDIR'}" || die "Unable to chdir to $ENV{'PBBUILDDIR'}"; my $made = ""; # pkgs made during build my $pm; my $all_ok = 1; if (defined $pbparallel) { $pm = new Parallel::ForkManager($pbparallel); $pm->run_on_finish(sub { my ($pid, $code, $id, $signal, $dump) = @_; $all_ok = 0 unless (($code == 0) && ($signal == 0) && ($dump == 0)); }); } # We need to communicate info back from the children if parallel so prepare a dir for that my $tmpd = "$ENV{'PBTMP'}/build.$$"; pb_mkdir_p($tmpd) if (defined $pbparallel); foreach my $pbpkg (@pkgs) { $pm->start and next if (defined $pbparallel); my $vertag = $pkg->{$pbpkg}; pb_log(2,"Vertag: $vertag\n"); # get the version of the current package - maybe different ($pbver,$pbtag) = split(/-/,$vertag); my $src="$ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.tar.gz"; my $src2="$ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.pbconf.tar.gz"; pb_log(2,"Source file: $src\n"); pb_log(2,"Pbconf file: $src2\n"); pb_log(2,"Working directory: $ENV{'PBBUILDDIR'}\n"); if ($pbos->{'type'} eq "rpm") { foreach my $d ('RPMS','SRPMS','SPECS','SOURCES','BUILD') { if (! -d "$ENV{'PBBUILDDIR'}/$d") { pb_mkdir_p("$ENV{'PBBUILDDIR'}/$d"); } } # Remove in case a previous link/file was there unlink "$ENV{'PBBUILDDIR'}/SOURCES/".basename($src); symlink "$src","$ENV{'PBBUILDDIR'}/SOURCES/".basename($src) || die "Unable to symlink $src in $ENV{'PBBUILDDIR'}/SOURCES"; # We need to first extract the spec file my @specfile = pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/","$ENV{'PBBUILDDIR'}/SPECS","spec"); # We need to handle potential patches to upstream sources pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbpatch/","$ENV{'PBBUILDDIR'}/SOURCES","patch"); # We need to handle potential additional sources to upstream sources pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbsrc/","$ENV{'PBBUILDDIR'}/SOURCES","src"); pb_log(2,"specfile: ".Dumper(\@specfile)."\n"); # set LANGUAGE to check for correct log messages $ENV{'LANGUAGE'}="C"; # Older Redhat use _target_platform in %configure incorrectly my $specialdef = ""; if (($pbos->{'name'} eq "redhat") || (($pbos->{'name'} eq "rhel") && ($pbos->{'version'} eq "2.1"))) { $specialdef = "--define \'_target_platform \"\"\'"; } foreach my $f (@specfile) { if ($f =~ /\.spec$/) { # This could cause an issue in // mode pb_distro_installdeps($f,$pbos); pb_system("rpmbuild $specialdef --define \"packager $ENV{'PBPACKAGER'}\" --define \"_topdir $ENV{'PBBUILDDIR'}\" -ba $f","Building package with $f under $ENV{'PBBUILDDIR'}","verbose"); last; } } # Get the name of the generated packages open(LOG,"$ENV{'PBTMP'}/system.$$.log") || die "Unable to open $ENV{'PBTMP'}/system.$$.log"; while () { chomp($_); next if ($_ !~ /^Wrote:/); s|.*/([S]*RPMS.*)|$1|; $made .=" $_"; } close(LOG); } elsif ($pbos->{'type'} eq "deb") { pb_system("tar xfz $src","Extracting sources"); pb_system("tar xfz $src2","Extracting pbconf"); chdir "$pbpkg-$pbver$pbextdir" || die "Unable to chdir to $pbpkg-$pbver$pbextdir"; pb_rm_rf("debian"); my $confdir = "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}"; die "Configuration directory $confdir is not a directory\nIs os description listed in your {ve,vm,rm}list values?" if (not -d $confdir); symlink "$confdir","debian" || die "Unable to symlink 'debian' to $confdir"; chmod 0755,"debian/rules"; # We need to handle potential patches to upstream sources pb_mkdir_p("debian/patches"); my @f = pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbpatch/","debian/patches","patch"); # By default we use format 1.0 - Cf man dpkg-source my $debsrcfmt = "1.0"; my $debsrcfile = "debian/source/format"; if (-f $debsrcfile) { $debsrcfmt = pb_get_content($debsrcfile); } if ($debsrcfmt =~ /^3.*quilt/) { # If we use quilt to manage patches, we then setup the env correctly # as per http://pkg-perl.alioth.debian.org/howto/quilt.html # Generate Debian patch series for quilt open(SERIE,"> debian/patches/series") || die "Unable to write in debian/patches/series"; $ENV{'QUILT_PATCHES'}="debian/patches"; } else { # If we use dpatch to manage patches, we then setup the 00list file as well open(SERIE,"> debian/patches/00list") || die "Unable to write in debian/patches/00list"; } foreach my $f (sort @f) { # Skip the script made to apply the patches to the Debian tree next if ($f =~ /pbapplypatch/); # We also need to uncompress them pb_system("gzip -d $f","","quiet"); $f =~ s/\.gz$//; print SERIE "$f\n"; } close(SERIE); if (@f) { # We have patches... my $patch_file = "debian/patches/pbapplypatch"; if (($debsrcfmt =~ /^1.*/) && (-x $patch_file)) { # In that case we need to apply the patches ourselves locally pb_system("cat $patch_file","APPLY","verbose"); pb_system("$patch_file","Applying patches to $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'} tree"); } # ...so modify the name of files to be Debian compliant move("../$src","../$pbpkg-$pbver$pbextdir.orig.tar.gz"); } # We need to handle potential additional sources to upstream sources #pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbsrc/","$ENV{'PBBUILDDIR'}/debian","src"); pb_distro_installdeps("debian/control",$pbos); pb_system("dpkg-buildpackage -us -uc -rfakeroot","Building package","verbose"); # Get the name of the generated packages open(LOG,"$ENV{'PBTMP'}/system.$$.log") || die "Unable to open $ENV{'PBTMP'}/system.$$.log"; my $tmp = ""; while () { chomp(); next unless (/^dpkg-deb.*:\s+building\s+package\s+.*\s+in\s+\`\.\.\/(\S+)\'\./o); $tmp = $1; die "Missing file $tmp" if (not -f $tmp); $made = "$made $tmp"; } close(LOG); open(CTRL,"debian/control") or die "Unable to open debian/control: $!"; #$made="$made $tmp.dsc $tmp.tar.gz $tmp"."_*.deb $tmp"."_*.changes"; while () { next unless (/^Source: (\S+)/o); foreach my $glob (("$1\_*.changes", "$1\_*.dsc", "$1\_*.tar.gz")) { my @file = glob($glob); die "Missing file for $glob" unless @file > 0; die "Too many files for $glob" if @file > 1; die "Missing file $file[0]" if (not -f $file[0]); $made .= " $file[0]"; } } close(CTRL); pb_display_file("$ENV{'PBTMP'}/system.$$.log"); chdir ".." || die "Unable to chdir to parent dir"; pb_rm_rf("$pbpkg-$pbver"); } elsif ($pbos->{'type'} eq "ebuild") { my @ebuildfile; # For gentoo we need to take pb as subsystem name # We put every apps here under sys-apps. hope it's correct # We use pb's home dir in order to have a single OVERLAY line my $tmpe = "$ENV{'HOME'}/portage/pb/sys-apps/$pbpkg"; pb_mkdir_p($tmpe) if (! -d "$tmpe"); pb_mkdir_p("$ENV{'HOME'}/portage/distfiles") if (! -d "$ENV{'HOME'}/portage/distfiles"); # We need to first extract the ebuild file @ebuildfile = pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/","$tmpe","ebuild"); # Prepare the build env for gentoo my $found = 0; my $pbbd = $ENV{'HOME'}; $pbbd =~ s|/|\\/|g; if (-r "/etc/make.conf") { open(MAKE,"/etc/make.conf"); while () { $found = 1 if (/$pbbd\/portage/); } close(MAKE); } if ($found == 0) { pb_system("sudo sh -c 'echo PORTDIR_OVERLAY=\"$ENV{'HOME'}/portage\" >> /etc/make.conf'"); } #$found = 0; #if (-r "/etc/portage/package.keywords") { #open(KEYW,"/etc/portage/package.keywords"); #while () { #$found = 1 if (/portage\/pb/); #} #close(KEYW); #} #if ($found == 0) { #pb_system("sudo sh -c \"echo portage/pb >> /etc/portage/package.keywords\""); #} # Build foreach my $f (@ebuildfile) { if ($f =~ /\.ebuild$/) { pb_distro_installdeps($f,$pbos); move($f,"$tmpe/$pbpkg-$pbver.ebuild"); pb_system("cd $tmpe ; ebuild $pbpkg-$pbver.ebuild clean ; ebuild $pbpkg-$pbver.ebuild digest ; ebuild $pbpkg-$pbver.ebuild package","verbose"); # Now move it where pb expects it pb_mkdir_p("$ENV{'PBBUILDDIR'}/portage/pb/sys-apps/$pbpkg"); if ($pbtag eq 0) { # This is assumed to be a test version my $nver = substr($pbver,0,-14); my $ntag = substr($pbver,-14); my $ebtg = "portage/pb/sys-apps/$pbpkg/$pbpkg-$nver"."_p$ntag.ebuild"; move("$tmpe/$pbpkg-$pbver.ebuild","$ENV{'PBBUILDDIR'}/$ebtg"); $made="$made $ebtg"; } else { my $ebtg = "portage/pb/sys-apps/$pbpkg/$pbpkg-$pbver-r$pbtag.ebuild"; move("$tmpe/$pbpkg-$pbver.ebuild","$ENV{'PBBUILDDIR'}/$ebtg"); $made="$made $ebtg"; } } } } elsif ($pbos->{'type'} eq "tgz") { # Slackware family $made="$made $pbpkg/$pbpkg-$pbver-*-$pbtag.tgz"; pb_system("tar xfz $src","Extracting sources"); pb_system("tar xfz $src2","Extracting pbconf"); chdir "$pbpkg-$pbver$pbextdir" || die "Unable to chdir to $pbpkg-$pbver$pbextdir"; symlink "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}","install" || die "Unable to symlink to pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}"; if (-x "install/pbslack") { pb_distro_installdeps("./install/pbslack",$pbos); pb_system("./install/pbslack","Building software"); pb_system("sudo /sbin/makepkg -p -l y -c y $pbpkg","Packaging $pbpkg","verbose"); } chdir ".." || die "Unable to chdir to parent dir"; pb_rm_rf("$pbpkg-$pbver$pbextdir"); } elsif ($pbos->{'type'} eq "pkg") { # Solaris $made="$made $pbpkg-$pbver-$pbtag.pkg.gz"; my $pkgdestdir="$ENV{'PBBUILDDIR'}/install"; # Will host resulting packages pb_mkdir_p("$pbos->{'type'}"); pb_mkdir_p("$pkgdestdir/delivery"); pb_system("tar xfz $src","Extracting sources under $ENV{'PBBUILDDIR'}"); pb_system("tar xfz $src2","Extracting pbconf under $ENV{'PBBUILDDIR'}"); # We need to handle potential patches to upstream sources pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbpatch/","$ENV{'PBBUILDDIR'}","patch"); # We need to handle potential additional sources to upstream sources pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbsrc/","$ENV{'PBBUILDDIR'}","src"); chdir "$pbpkg-$pbver$pbextdir" || die "Unable to chdir to $pbpkg-$pbver$pbextdir"; if (-f "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbbuild") { chmod 0755,"pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbbuild"; # pkginfo file is mandatory die "Unable to find pkginfo file in pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}" if (! -f "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pkginfo"); # Build pb_system("pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbbuild $pkgdestdir/delivery","Building software and installing under $pkgdestdir/delivery"); # Copy complementary files if (-f "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/prototype") { copy("pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/prototype", $pkgdestdir) } else { # No prototype provided, calculating it open(PROTO,"> $pkgdestdir/prototype") || die "Unable to create prototype file"; print PROTO "i pkginfo\n"; print PROTO "i depend\n" if (-f "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/depend"); $ENV{'PBSOLDESTDIR'} = "$pkgdestdir/delivery"; find(\&create_solaris_prototype, "$pkgdestdir/delivery"); } copy("pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/depend", $pkgdestdir) if (-f "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/depend"); copy("pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pkginfo", $pkgdestdir); pb_system("cd $pkgdestdir/delivery ; pkgmk -o -f ../prototype -r $pkgdestdir/delivery -d $ENV{'PBBUILDDIR'}/$pbos->{'type'}","Packaging $pbpkg","verbose"); pb_system("cd $ENV{'PBBUILDDIR'}/$pbos->{'type'} ; echo \"\" | pkgtrans -o -n -s $ENV{'PBBUILDDIR'}/$pbos->{'type'} $ENV{'PBBUILDDIR'}/$pbpkg-$pbver-$pbtag.pkg all","Transforming $pbpkg","verbose"); pb_system("cd $ENV{'PBBUILDDIR'} ; gzip -9f $pbpkg-$pbver-$pbtag.pkg","Compressing $pbpkg-$pbver-$pbtag.pkg","verbose"); } else { pb_log(0,"No pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbbuild file found for $pbpkg-$pbver in \n"); } chdir ".." || die "Unable to chdir to parent dir"; pb_rm_rf("$pbpkg-$pbver$pbextdir","$ENV{'PBBUILDDIR'}/$pbos->{'type'}","$pkgdestdir"); } elsif ($pbos->{'type'} eq "hpux") { # HP-UX pb_system("tar xfz $src","Extracting sources"); pb_system("tar xfz $src2","Extracting pbconf"); chdir "$pbpkg-$pbver$pbextdir" || die "Unable to chdir to $pbpkg-$pbver$pbextdir"; pb_system("buildpackage ","Building package","verbose"); # Get the name of the generated packages open(LOG,"$ENV{'PBTMP'}/system.$$.log") || die "Unable to open $ENV{'PBTMP'}/system.$$.log"; while () { chomp(); my $tmp = $_; next if ($tmp !~ /^SD BUILD.*:/); $tmp =~ s|.*../(.*)_(.*).sd.*|$1|; $made = "$made $tmp"."_*.sd"; } close(LOG); $made="$made $pbpkg-$pbver-$pbtag.sd"; chdir ".." || die "Unable to chdir to parent dir"; pb_rm_rf("$pbpkg-$pbver$pbextdir"); } else { die "Unknown OS type format $pbos->{'type'}"; } if (defined $pbparallel) { # Communicate results back to parent pb_set_content("$tmpd/$$",$made); $pm->finish; } } if (defined $pbparallel) { # In the parent, we need to get the result from the children $pm->wait_all_children; foreach my $f (<$tmpd/*>) { $made .= " ".pb_get_content($f); } die "Aborting, one or more of the children failed." if ((not $all_ok) && ($Global::pb_stop_on_error)); pb_rm_rf($tmpd); } # Sign packages pb_sign_pkgs($pbos,$made); # Find the appropriate check cmd/opts my ($chkcmd,$chkopt) = pb_distro_get_param($pbos,pb_conf_get_if("oschkcmd","oschkopt")); # Packages check if needed if ($pbos->{'type'} eq "rpm") { if ((defined $chkcmd) && (-x $chkcmd)) { my $cmd = "$chkcmd"; $cmd .= " $chkopt" if (defined $chkopt); $cmd .= " $made"; my $ret = pb_system("$cmd","Checking validity of rpms with $chkcmd","verbose",1); pb_log(0,"ERROR: when checking packages validity\n") if ($ret ne 0); } my $rpms =""; my $srpms =""; foreach my $f (split(/ /,$made)) { $rpms .= "$ENV{'PBBUILDDIR'}/$f " if ($f =~ /^RPMS\//); $srpms .= "$ENV{'PBBUILDDIR'}/$f " if ($f =~ /^SRPMS\//); } pb_log(0,"SRPM packages generated: $srpms\n"); pb_log(0,"RPM packages generated: $rpms\n"); } elsif ($pbos->{'type'} eq "deb") { my $made2 = ""; foreach my $f (split(/ /,$made)) { $made2 .= "$f " if ($f =~ /\.deb$/); } if (-x $chkcmd) { my $ret = pb_system("$chkcmd $chkopt $made2","Checking validity of debs with $chkcmd","verbose",1); pb_log(0,"ERROR: when checking packages validity\n") if ($ret ne 0); } pb_log(0,"deb packages generated: $made2\n"); } else { pb_log(0,"No check done for $pbos->{'type'} yet\n"); pb_log(0,"Packages generated: $made\n"); } # Keep track of what is generated so that we can get them back from VMs/RMs my $pbkeep = "$ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}"; open(KEEP,"> $pbkeep") || die "Unable to create $pbkeep: $!"; print KEEP "$made\n"; close(KEEP); pb_log(0,"INFO: ------ Finished building package ------\n"); } sub create_solaris_prototype { my $uidgid = "bin bin"; my $pkgdestdir = $ENV{'PBSOLDESTDIR'}; return if ($_ =~ /^$pkgdestdir$/); if (-d $_) { my $n = $File::Find::name; $n =~ s~$pkgdestdir/~~; print PROTO "d none $n 0755 $uidgid\n"; } elsif (-x $_) { my $n = $File::Find::name; $n =~ s~$pkgdestdir/~~; print PROTO "f none $n 0755 $uidgid\n"; } elsif (-f $_) { my $n = $File::Find::name; $n =~ s~$pkgdestdir/~~; print PROTO "f none $n 0644 $uidgid\n"; } } sub pb_build2ssh { pb_send2target("Sources"); pb_send2target("CPAN"); } sub pb_pkg2ssh { pb_send2target("Packages"); } # By default deliver to the the public site hosting the # ftp structure (or whatever) or a VM/VE/RM sub pb_send2target { my $cmt = shift; my $v = shift || undef; my $vmexist = shift || 0; # 0 is FALSE my $vmpid = shift || 0; # 0 is FALSE my $snapme = shift || 0; # 0 is FALSE pb_log(2,"DEBUG: pb_send2target($cmt,".Dumper($v).",$vmexist,$vmpid)\n"); my $host = "sshhost"; my $login = "sshlogin"; my $dir = "sshdir"; my $port = "sshport"; my $conf = "sshconf"; my $tmout = undef; my $path = undef; if ($cmt =~ /^VM/) { $login = "vmlogin"; $dir = "pbdefdir"; # Specific VM $tmout = "vmtmout"; $path = "vmpath"; $host = "vmhost"; $port = "vmport"; } elsif ($cmt =~ /^RM/) { $login = "rmlogin"; $dir = "pbdefdir"; # Specific RM $tmout = "rmtmout"; $path = "rmpath"; $host = "rmhost"; $port = "rmport"; } elsif ($cmt =~ /^VE/) { $login = "velogin"; $dir = "pbdefdir"; # Specific VE $path = "vepath"; $conf = "rbsconf"; } elsif ($cmt eq "Web") { $host = "websshhost"; $login = "websshlogin"; $dir = "websshdir"; $port = "websshport"; } elsif ($cmt eq "CPAN") { $host = "cpanpause"; $login = ""; $dir = "cpandir"; $port = ""; } my $cmd = ""; my $src = ""; my $cpanpkg = 0; my $pbos; my $pbextdir = pb_get_extdir(); if ($cmt ne "Announce") { # Get list of packages to build my $ptr = pb_get_pkg(); @pkgs = @$ptr; # Get the running distro to consider $pbos = pb_distro_get_context($v); # Get content saved in cms2build my ($pkg) = pb_conf_read("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.pb","pbpkg"); $pkg = { } if (not defined $pkg); pb_mkdir_p("$ENV{'PBBUILDDIR'}") if (! -d "$ENV{'PBBUILDDIR'}"); chdir "$ENV{'PBBUILDDIR'}" || die "Unable to chdir to $ENV{'PBBUILDDIR'}"; foreach my $pbpkg (@pkgs) { my $vertag = $pkg->{$pbpkg}; # get the version of the current package - maybe different pb_log(2,"Vertag: $vertag\n"); ($pbver,$pbtag) = split(/-/,$vertag); if (($cmt eq "Sources") || ($cmt =~ /(V[EM]|RM)build/)) { $src = "$src $ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.tar.gz $ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.pbconf.tar.gz"; if ($cmd eq "") { $cmd = "ln -sf $pbpkg-$pbver$pbextdir.tar.gz $pbpkg-latest.tar.gz"; } else { $cmd = "$cmd ; ln -sf $pbpkg-$pbver$pbextdir.tar.gz $pbpkg-latest.tar.gz"; } } elsif ($cmt eq "Web") { $src = "$src $ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.tar.gz" } # Do we have one perl package my @nametype = pb_conf_get_if("namingtype"); my $type = $nametype[0]->{$pbpkg}; if ((defined $type) && ($type eq "perl")) { $cpanpkg = 1; } } # Adds conf file for availability of conf elements pb_conf_add("$ENV{'PBROOTDIR'}/$ENV{'PBPROJ'}.pb"); } my ($rbsconf,$testver,$delivery) = pb_conf_get_if($conf,"testver","delivery"); if ($cmt =~ /CPAN/) { # Do not deliver on Pause if this is a test version return if (not defined $testver); return if ($testver =~ /true/); # Do not deliver on Pause if this is not a perl package return if ($cpanpkg == 0); } if ($cmt =~ /(V[EM]|RM)build/) { $src="$src $ENV{'PBROOTDIR'}/$ENV{'PBPROJ'}.pb $ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.pb $ENV{'PBETC'} $ENV{'PBDESTDIR'}/pbrc $ENV{'PBDESTDIR'}/pbscript.$$"; } elsif ($cmt =~ /(V[EM]|RM)Script/) { $src="$src $ENV{'PBDESTDIR'}/pbscript.$$"; } elsif ($cmt =~ /(V[EM]|RM)test/) { $src="$src $ENV{'PBROOTDIR'}/$ENV{'PBPROJ'}.pb $ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.pb $ENV{'PBETC'} $ENV{'PBDESTDIR'}/pbrc $ENV{'PBDESTDIR'}/pbscript.$$ $ENV{'PBDESTDIR'}/pbtest"; } elsif (($cmt eq "Announce") || ($cmt eq "Web") || ($cmt eq "CPAN")) { $src="$src $ENV{'PBTMP'}/pbscript"; } elsif ($cmt eq "Packages") { # Get package list from file made during build2pkg open(KEEP,"$ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}") || die "Unable to read $ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}"; $src = ; chomp($src); close(KEEP); $src = "$src $ENV{'PBBUILDDIR'}/pbscript.$$"; } if (($cmt eq "Sources") || ($cmt eq "Packages") || ($cmt eq "CPAN")) { my ($pbpackager) = pb_conf_get("pbpackager"); $ENV{'PBPACKAGER'} = $pbpackager->{$ENV{'PBPROJ'}}; pb_log(0,"Exporting public key for $ENV{'PBPACKAGER'}\n"); # Using pb_system is not working due to redirection below system("gpg --export -a \'$ENV{'PBPACKAGER'}\' > $ENV{'PBDESTDIR'}/$ENV{'PBPROJ'}.pubkey"); chmod 0644,"$ENV{'PBDESTDIR'}/$ENV{'PBPROJ'}.pubkey"; $src = "$src $ENV{'PBDESTDIR'}/$ENV{'PBPROJ'}.pubkey"; } # Remove potential leading spaces (cause problem with basename) $src =~ s/^ *//; my $basesrc = ""; foreach my $i (split(/ +/,$src)) { $basesrc .= " ".basename($i); } pb_log(0,"Sources handled ($cmt): $src\n"); pb_log(2,"values: ".Dumper(($host,$login,$dir,$port,$tmout,$path,$conf))."\n"); my ($sshhost,$sshdir) = pb_conf_get($host,$dir); # Not mandatory... $delivery->{$ENV{'PBPROJ'}} = "" if (not defined $delivery->{$ENV{'PBPROJ'}}); my ($sshlogin,$sshport) = pb_conf_get_if($login,$port); $sshport->{$ENV{PBPROJ}} = 22 unless (defined $sshport->{$ENV{PBPROJ}}); $sshlogin->{$ENV{PBPROJ}} = getpwuid($UID) unless (defined $sshlogin->{$ENV{PBPROJ}}); my ($vtmout,$vepath); # ...Except those in virtual context if ($cmt =~ /^VE/) { ($vepath) = pb_conf_get($path); } if ($cmt =~ /^(V|R)M/) { $vtmout = pb_distro_get_param($pbos,pb_conf_get_if($tmout)); } my $remhost = $sshhost->{$ENV{'PBPROJ'}}; my $remdir = $sshdir->{$ENV{'PBPROJ'}}; if ($cmt =~ /^V[EM]|RM/) { # In that case our real host is in the xxhost with the OS as key, not project as above $remhost = pb_distro_get_param($pbos,$sshhost); } pb_log(2,"ssh: ".Dumper(($remhost,$sshlogin,$remdir,$sshport,$vepath,$rbsconf))."\n"); pb_log(2,"ssh: ".Dumper($vtmout)."\n") if (defined $vtmout); my $mac; if ($cmt !~ /^VE/) { $mac = "$sshlogin->{$ENV{'PBPROJ'}}\@$remhost"; # Overwrite account value if passed as parameter $mac = "$pbaccount\@$remhost" if (defined $pbaccount); pb_log(2, "DEBUG: pbaccount: $pbaccount => mac: $mac\n") if (defined $pbaccount); } else { # VE # Overwrite account value if passed as parameter (typically for setup2v) $mac = $sshlogin->{$ENV{'PBPROJ'}}; $mac = $pbaccount if (defined $pbaccount); } my $tdir; my $bdir; if (($cmt eq "Sources") || ($cmt =~ /(V[EM]|RM)Script/)) { $tdir = "$remdir/$delivery->{$ENV{'PBPROJ'}}/src"; } elsif ($cmt eq "CPAN") { $tdir = "$remdir"; } elsif ($cmt =~ /(V[EM]|RM)(build|test)/) { $tdir = $remdir."/$ENV{'PBPROJ'}/delivery"; $bdir = $remdir."/$ENV{'PBPROJ'}/build"; # Remove a potential $ENV{'HOME'} as bdir should be relative to pb's home $bdir =~ s|\$ENV.+\}/||; } elsif ($cmt eq "Announce") { $tdir = "$remdir/$delivery->{$ENV{'PBPROJ'}}"; } elsif ($cmt eq "Web") { $tdir = "$remdir/$delivery->{$ENV{'PBPROJ'}}"; } elsif ($cmt eq "Packages") { if (($pbos->{'type'} eq "rpm") || ($pbos->{'type'} eq "pkg") || ($pbos->{'type'} eq "hpux") || ($pbos->{'type'} eq "tgz")) { # put packages under an arch subdir $tdir = "$remdir/$delivery->{$ENV{'PBPROJ'}}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}"; } elsif (($pbos->{'type'} eq "deb") || ($pbos->{'type'} eq "ebuild")) { # No need for an arch subdir $tdir = "$remdir/$delivery->{$ENV{'PBPROJ'}}/$pbos->{'name'}/$pbos->{'version'}"; } else { die "Please teach the dev team where to deliver ($pbos->{'type'} type of packages\n"; } my $repodir = $tdir; $repodir =~ s|^$remdir/||; my ($pbrepo) = pb_conf_get("pbrepo"); # Repository management open(PBS,"> $ENV{'PBBUILDDIR'}/pbscript.$$") || die "Unable to create $ENV{'PBBUILDDIR'}/pbscript.$$"; if ($pbos->{'type'} eq "rpm") { my $pbsha = pb_distro_get_param($pbos,pb_conf_get("ossha")); # Also make a pbscript to generate yum/urpmi bases print PBS << "EOF"; #!/bin/bash # Prepare a script to ease yum setup EOF print PBS "set -x\n" if ($pbdebug gt 1); print PBS << "EOF"; cat > $ENV{'PBPROJ'}.repo << EOT [$ENV{'PBPROJ'}] name=$pbos->{'name'} $pbos->{'version'} $pbos->{'arch'} - $ENV{'PBPROJ'} Vanilla Packages baseurl=$pbrepo->{$ENV{'PBPROJ'}}/$repodir enabled=1 gpgcheck=1 gpgkey=$pbrepo->{$ENV{'PBPROJ'}}/$repodir/$ENV{'PBPROJ'}.pubkey EOT chmod 644 $ENV{'PBPROJ'}.repo # Clean up old repo content rm -rf headers/ repodata/ # Create yum repo if [ -x /usr/bin/yum-arch ]; then yum-arch . fi # Create repodata createrepo -s $pbsha . # Link to the key (cd repodata ; ln -sf ../$ENV{'PBPROJ'}.pubkey repomd.xml.key) # sign the repomd (at least useful for SLES - which requires a local key) # gpg -a --detach-sign repodata/repomd.xml # SLES also looks for media.1/info.txt EOF if ($pbos->{'family'} eq "md") { # For Mandriva add urpmi management print PBS << "EOF"; # Prepare a script to ease urpmi setup cat > $ENV{'PBPROJ'}.addmedia << EOT urpmi.addmedia $ENV{'PBPROJ'} $pbrepo->{$ENV{'PBPROJ'}}/$repodir with media_info/hdlist.cz EOT chmod 755 $ENV{'PBPROJ'}.addmedia # Clean up old repo content rm -f hdlist.cz synthesis.hdlist.cz # Create urpmi repo genhdlist2 --clean . if [ \$\? -ne 0 ]; then genhdlist . fi EOF } if ($pbos->{'name'} eq "fedora") { # Extract the spec file to please Fedora maintainers :-( print PBS << "EOF"; for p in $basesrc; do echo \$p | grep -q 'src.rpm' if [ \$\? -eq 0 ]; then rpm2cpio \$p | cpio -ivdum --quiet '*.spec' fi done EOF } if ($pbos->{'family'} eq "novell") { # Add ymp scripts for one-click install on SuSE print PBS << "EOF"; # Prepare a script to ease SuSE one-click install # Cf: http://de.opensuse.org/1-Klick-Installation/ISV # cat > $ENV{'PBPROJ'}.ymp << EOT $ENV{'PBPROJ'} Bundle Software bundle for the $ENV{'PBPROJ'} project This is the summary of the $ENV{'PBPROJ'} Project Details are available on a per package basis below false $ENV{'PBPROJ'} Repository This repository contains the $ENV{'PBPROJ'} project packages. This repository contains the $ENV{'PBPROJ'} project packages. $pbrepo->{$ENV{'PBPROJ'}}/$repodir EOT for p in $basesrc; do sum=`rpm -q --qf '%{SUMMARY}' \$p` name=`rpm -q --qf '%{NAME}' \$p` desc=`rpm -q --qf '%{description}' \$p` cat >> $ENV{'PBPROJ'}.ymp << EOT \$name \$sum \$desc EOT done cat >> $ENV{'PBPROJ'}.ymp << EOT EOT chmod 644 $ENV{'PBPROJ'}.ymp EOF } } elsif ($pbos->{'type'} eq "deb") { # Also make a pbscript to generate apt bases # Cf: http://www.debian.org/doc/manuals/repository-howto/repository-howto.fr.html # This dirname removes ver my $debarch = $pbos->{'arch'}; $debarch = "amd64" if ($pbos->{'arch'} eq "x86_64"); my $rpd = dirname("$pbrepo->{$ENV{'PBPROJ'}}/$repodir"); # Remove extra . in path to fix #522 $rpd =~ s|/./|/|g; my ($projcomponent_map) = pb_conf_get_if("projcomponent"); pb_log(2,"projcomponent = ".Dumper($projcomponent_map); my $projcomponent = $projcomponent_map->{$ENV{PBPROJ}}; $projcomponent ||= 'contrib'; print PBS << "EOF"; #!/bin/bash # Prepare a script to ease apt setup cat > $ENV{'PBPROJ'}.sources.list << EOT deb $rpd $pbos->{'version'} $projcomponent deb-src $rpd $pbos->{'version'} $projcomponent EOT chmod 644 $ENV{'PBPROJ'}.sources.list # Up two levels to deal with the dist dir cross versions cd .. mkdir -p dists/$pbos->{'version'}/$projcomponent/binary-$debarch dists/$pbos->{'version'}/$projcomponent/source # Prepare a script to create apt info file # Reuse twice after TMPD=`mktemp -d /tmp/pb.XXXXXXXXXX` || exit 1 mkdir -p \$TMPD cat > \$TMPD/Release << EOT Archive: unstable Component: $projcomponent Origin: $ENV{'PBPROJ'} Label: $ENV{'PBPROJ'} dev repository $pbrepo->{$ENV{'PBPROJ'}} EOT echo "Creating Packages metadata ($pbos->{'arch'} aka $debarch)" dpkg-scanpackages -a$debarch $pbos->{'version'} /dev/null | gzip -c9 > dists/$pbos->{'version'}/$projcomponent/binary-$debarch/Packages.gz dpkg-scanpackages -a$debarch $pbos->{'version'} /dev/null | bzip2 -c9 > dists/$pbos->{'version'}/$projcomponent/binary-$debarch/Packages.bz2 echo "Creating Contents metadata" apt-ftparchive contents $pbos->{'version'} | gzip -c9 > dists/$pbos->{'version'}/Contents.gz echo "Creating Release metadata ($pbos->{'arch'} aka $debarch)" cat \$TMPD/Release > dists/$pbos->{'version'}/$projcomponent/binary-$debarch/Release echo "Architecture: $debarch" >> dists/$pbos->{'version'}/$projcomponent/binary-$debarch/Release echo "Creating Source metadata" dpkg-scansources $pbos->{'version'} /dev/null | gzip -c9 > dists/$pbos->{'version'}/$projcomponent/source/Sources.gz cat \$TMPD/Release > dists/$pbos->{'version'}/$projcomponent/source/Release echo "Architecture: Source" >> dists/$pbos->{'version'}/$projcomponent/source/Release echo "Creating Release metadata" # Signing that file would be useful but uneasy as gpg keys are not there # Cf: http://wiki.debian.org/SecureApt # Same as for repomd apt-ftparchive release dists/$pbos->{'version'} > dists/$pbos->{'version'}/Release rm -rf \$TMPD EOF } elsif ($pbos->{'type'} eq "ebuild") { # make a pbscript to generate links to latest version print PBS << "EOF"; #!/bin/bash # Prepare a script to create correct links for p in $src; do echo \$p | grep -q '.ebuild' if [ \$\? -eq 0 ]; then j=`basename \$p` pp=`echo \$j | cut -d'-' -f1` ln -sf \$j \$pp.ebuild fi done EOF } close(PBS); chmod 0755,"$ENV{'PBBUILDDIR'}/pbscript.$$"; } else { return; } # Useless for VE my $nport = pb_get_port($sshport,$pbos,$cmt) if ($cmt !~ /^VE/); # Remove a potential $ENV{'HOME'} as tdir should be relative to pb's home $tdir =~ s|\$ENV.+\}/||; my $tm = ""; if ($cmt =~ /^(V|R)M/) { $tm = "sleep $vtmout" if (defined $vtmout); } # ssh communication if not VE or CPAN # should use a hash instead... my ($shcmd,$shcmdroot,$cpcmd,$cptarget,$cp2target); if ($cmt =~ /^VE/) { my $tp = pb_path_expand($vepath->{$ENV{'PBPROJ'}}); my $tpdir = pb_path_expand("$tp/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}"); my ($ptr) = pb_conf_get("vetype"); my $vetype = $ptr->{$ENV{'PBPROJ'}}; my $arch = pb_get_arch(); if ($vetype eq "chroot") { $shcmdroot = "sudo /usr/sbin/chroot $tpdir "; $shcmd = "$shcmdroot /bin/su - $mac -c "; } elsif ($vetype eq "schroot") { $shcmd = "schroot $tp -u $mac -- "; } $shcmd = "setarch i386 $shcmd" if (($pbos->{'arch'} =~ /i?86/) && ($arch eq 'x86_64')); $cpcmd = "sudo /bin/cp -r "; # We need to get the home dir of the target account to deliver in the right place open(PASS,"$tpdir/etc/passwd") || die "Unable to open $tpdir/etc/passwd: $!"; my $homedir = ""; while () { my ($c1,$c2,$c3,$c4,$c5,$c6,$c7) = split(/:/); $homedir = $c6 if ($c1 =~ /^$mac$/); pb_log(3,"Homedir: $homedir - account: $c6\n"); } close(PASS); $cptarget = "$tpdir/$homedir/$tdir"; if ($cmt eq "VEbuild") { $cp2target = "$tpdir/$homedir/$bdir"; } pb_log(2,"On VE using $cptarget as target dir to copy to\n"); } elsif ($cmt =~ /^CPAN/) { my $ftpput = pb_check_req("ncftpput",1); my $ftpget = pb_check_req("wget",1); my ($cpanuser,$cpanpasswd) = pb_conf_get("cpanuser","cpanpasswd"); my ($cpansubdir) = pb_conf_get_if("cpansubdir"); $shcmd = "$ftpget --post-data \'HIDDENNAME=".$cpanuser; $shcmd .= "&user=".$cpanuser; $shcmd .= "&password=".$cpanpasswd; $shcmd .= "&SUBMIT_pause99_add_uri_upload=\"Upload the checked files\""; $shcmd .= "&pause99_add_uri_subdirtext=".$cpansubdir if (defined $cpansubdir); foreach my $s (split(/ /,$src)) { $shcmd .= "&pause99_add_uri_upload=".basename($s); } $shcmd .= "'"; $cpcmd = "$ftpput $host $dir"; $cptarget = "CPAN"; } else { my $keyfile = pb_ssh_get(0); my $keyopt = defined $keyfile ? "-i $keyfile" : ""; my $sshcmd = pb_check_req("ssh",1); my $scpcmd = pb_check_req("scp",1); $shcmd = "$sshcmd $keyopt -q -o NoHostAuthenticationForLocalhost=yes -p $nport $mac"; $cpcmd = "$scpcmd $keyopt -p -o NoHostAuthenticationForLocalhost=yes -P $nport"; $cptarget = "$mac:$tdir"; if ($cmt =~ /^(V|R)Mbuild/) { $cp2target = "$mac:$bdir"; } } my $logres = ""; # Do not touch when just announcing if (($cmt ne "Announce") && ($cmt ne "CPAN")) { pb_system("$shcmd \"mkdir -p $tdir ; cd $tdir ; echo \'for i in $basesrc; do if [ -f \$i ]; then rm -f \$i; fi; done\ ; $cmd' | bash-e\"","Preparing $tdir on $cptarget"); } else { $logres = "> "; } pb_system("cd $ENV{'PBBUILDDIR'} ; $cpcmd $src $cptarget 2> /dev/null","$cmt delivery in $cptarget"); # For VE we need to change the owner manually if ($cmt =~ /^VE/) { if (defined $shcmdroot) { # This should help overcome a CentOS 5.8 bug as well as having a simper sequence pb_system("$shcmdroot \"chown -R $mac $tdir\"","Adapt owner in $tdir to $mac"); } else { pb_system("$shcmd \"sudo chown -R $mac $tdir\"","Adapt owner in $tdir to $mac"); } } # Use the right script name depending on context my $pbscript; if (($cmt =~ /^(V[EM]|RM)/) || ($cmt =~ /Packages/)){ $pbscript = "pbscript.$$"; } else { $pbscript = "pbscript"; } # It's already ready for CPAN my $shcmdbase = $shcmd; if ($cmt !~ /^CPAN/) { $shcmd .= " \"echo \'cd $tdir ; if [ -x $pbscript ]; then ./$pbscript; fi ; rm -f ./$pbscript\' | bash\""; } my $cmdverb = "verbose"; if (($cmt eq "Announce") || ($cmt eq "CPAN")) { $cmdverb = undef; } pb_system("$shcmd","Executing pbscript on $cptarget if needed",$cmdverb); if ($cmt =~ /^(V[EM]|RM)build/) { # Get back info on pkg produced, compute their name and get them from the VM/RM pb_system("$cpcmd $cp2target/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'} $ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.$$ 2> /dev/null","Get package names in $cp2target"); # For VE we need to change the owner manually if ($cmt eq "VEbuild") { pb_system("sudo chown $UID $ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.$$","Adapt owner in $tdir to $UID"); } if (not -f "$ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.$$") { pb_log(0,"Problem with VM/RM $v on $ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.$$"); } else { open(KEEP,"$ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.$$") || die "Unable to read $ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.$$"; my $src = ; chomp($src); close(KEEP); unlink("$ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.$$"); $src =~ s/^ *//; pb_mkdir_p("$ENV{'PBBUILDDIR'}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}"); # Change pgben to make the next send2target happy my $made = ""; # For VM/RM we don't want shell expansion to hapen locally but remotely my $delim = '\''; if ($cmt =~ /^VEbuild/) { # For VE we need to support shell expansion locally $delim = ""; } open(KEEP,"> $ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}") || die "Unable to write $ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}"; foreach my $p (split(/ +/,$src)) { my $j = basename($p); pb_system("$cpcmd $cp2target/$delim$p$delim $ENV{'PBBUILDDIR'}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'} 2> /dev/null","Recovery of package $j in $ENV{'PBBUILDDIR'}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}"); $made="$made $pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}/$j"; # if (($pbos->{'type'} ne "rpm") || ($j !~ /.src.rpm$/)); } print KEEP "$made\n"; close(KEEP); pb_system("$shcmdbase \"rm -rf $tdir $bdir\"","$cmt cleanup"); # Sign packages locally pb_sign_pkgs($pbos,$made); # We want to send them to the ssh account so overwrite what has been done before undef $pbaccount; pb_log(2,"Before sending pkgs, vmexist: $vmexist, vmpid: $vmpid\n"); pb_send2target("Packages",$pbos->{'name'}."-".$pbos->{'version'}."-".$pbos->{'arch'},$vmexist,$vmpid); pb_rm_rf("$ENV{'PBBUILDDIR'}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}"); } } unlink("$ENV{'PBDESTDIR'}/pbscript.$$") if ((($cmt =~ /^(V[ME]|RM)/) || ($cmt =~ /Packages/)) && ($pbkeep eq 0)); pb_log(2,"Before halt, vmexist: $vmexist, vmpid: $vmpid\n"); if ((! $vmexist) && ($cmt =~ /^VM/)) { # If in setupvm then takes a snapshot just before halting if ($snapme != 0) { my ($vmmonport,$vmtype) = pb_conf_get("vmmonport","vmtype"); # For monitoring control if ((($vmtype->{$ENV{'PBPROJ'}}) eq "kvm") || (($vmtype->{$ENV{'PBPROJ'}}) eq "qemu")) { eval { require Net::Telnet; Net::Telnet->import(); }; if ($@) { # Net::Telnet not found pb_log(1,"ADVISE: Install Net::Telnet to benefit from monitoring control and snapshot feature.\nWARNING: No snapshot created"); } else { my $t = new Net::Telnet (Timeout => 120, Host => "localhost", Port => $vmmonport->{$ENV{'PBPROJ'}}) || die "Unable to dialog on the monitor"; # move to monitor mode my @lines = $t->cmd("c"); # Create a snapshot named pb @lines = $t->cmd("savevm pb"); # Write the new status in the VM @lines = $t->cmd("commit all"); # End @lines = $t->cmd("quit"); } } } my $hoption = "-p"; my $hpath = pb_distro_get_param($pbos,pb_conf_get("ospathcmd-halt")); # Solaris doesn't support -p of halt if ($pbos->{'type'} eq "pkg") { $hoption = "" ; } pb_system("$shcmdbase \"sudo $hpath $hoption \"; $tm ; echo \'if [ -d /proc/$vmpid ]; then kill -9 $vmpid; fi \' | bash ; sleep 10","VM $v halt (pid $vmpid)"); } if (($cmt =~ /^VE/) && ($snapme != 0)) { my $tpdir = "$vepath->{$ENV{'PBPROJ'}}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}"; pb_system("sudo tar cz -C $tpdir -f $vepath->{$ENV{'PBPROJ'}}/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}.tar.gz .","Creating a snapshot of $tpdir"); } } sub pb_script2v { my $pbscript=shift; my $vtype=shift; my $pbforce=shift || 0; # Force stop of VM. Default not. my $vm1=shift || undef; # Only that VM/VE/RM to treat. Default all. my $snapme=shift || 0; # Do we have to create a snapshot. Default not. my $vm; my $all; pb_log(2,"DEBUG: pb_script2v($pbscript,$vtype,$pbforce,".Dumper($vm1).",$snapme)\n"); # Prepare the script to be executed on the VM/VE/RM # in $ENV{'PBDESTDIR'}/pbscript.$$ if ((defined $pbscript ) && ($pbscript ne "$ENV{'PBDESTDIR'}/pbscript.$$")) { copy($pbscript,"$ENV{'PBDESTDIR'}/pbscript.$$") || die "Unable to create $ENV{'PBDESTDIR'}/pbscript.$$"; chmod 0755,"$ENV{'PBDESTDIR'}/pbscript.$$"; } if (not defined $vm1) { ($vm,$all) = pb_get2v($vtype); } else { @$vm = ($vm1); } my ($vmexist,$vmpid) = (undef,undef); foreach my $v (@$vm) { # Launch VM/VE ($vmexist,$vmpid) = pb_launchv($vtype,$v,0,$snapme,$pbsnap); if ($vtype eq "vm") { pb_log(2,"DEBUG: After pb_launchv, vmexist: $vmexist, vmpid: $vmpid\n"); # Skip that VM/RM if something went wrong next if (($vmpid == 0) && ($vmexist == 0)); # If force stopping the VM then reset vmexist if ($pbforce == 1) { $vmpid = $vmexist; $vmexist = 0; } } else { #VE $vmexist = 0; $vmpid = 0; } # Gather all required files to send them to the VM/VE/RM # and launch the build through pbscript pb_log(2,"DEBUG: Before send2target, vmexist: $vmexist, vmpid: $vmpid\n"); pb_send2target(uc($vtype)."Script","$v",$vmexist,$vmpid,$snapme); } } sub pb_launchv { my $vtype = shift; my $v = shift; my $create = shift || 0; # By default do not create a VM/VE/RM my $snapme = shift || 0; # By default do not snap a VM/VE/RM my $usesnap = shift || 1; # By default study the usage of the snapshot feature of VM/VE/RM # If creation or snapshot creation mode, no snapshot usable if (($create == 1) || ($snapme == 1)) { $usesnap = 0; } pb_log(2,"DEBUG: pb_launchv($vtype,$v,$create,$snapme,$usesnap)\n"); die "No VM/VE/RM defined, unable to launch" if (not defined $v); # Keep only the first VM in case many were given if ($v =~ /,/) { pb_log(0,"WARNING: pruning to just the first of several vms listed ($v)\n"); $v =~ s/,.*//; } my $pbos = pb_distro_get_context($v); my $ftp_proxy = pb_distro_get_param($pbos,pb_conf_get_if("ftp_proxy")); my $http_proxy = pb_distro_get_param($pbos,pb_conf_get_if("http_proxy")); $ENV{ftp_proxy} ||= $ftp_proxy if ((defined $ftp_proxy) && ($ftp_proxy ne "")); $ENV{http_proxy} ||= $http_proxy if ((defined $http_proxy) && ($http_proxy ne "")); # Launch the VMs/VEs if ($vtype eq "vm") { die "-i iso parameter needed" if (((not defined $iso) || ($iso eq "")) && ($create != 0)); my ($ptr,$ptr2,$vmpath,$vmport,$vms) = pb_conf_get("vmtype","vmcmd","vmpath","vmport","vmsize"); my ($vmopt,$vmmm,$vmtmout,$vmsnap,$vmbuildtm,$vmmonport) = pb_conf_get_if("vmopt","vmmem","vmtmout","vmsnap","vmbuildtm","vmmonport"); my $vmsize = pb_distro_get_param($pbos,$vms); my $vmtype = $ptr->{$ENV{'PBPROJ'}}; my $vmcmd = $ptr2->{$ENV{'PBPROJ'}}; if (defined $opts{'g'}) { if (($vmtype eq "kvm") || ($vmtype eq "qemu")) { $ENV{'PBVMOPT'} = "--nographic"; } } if (not defined $ENV{'PBVMOPT'}) { $ENV{'PBVMOPT'} = ""; } # Save the current status for later restoration $ENV{'PBOLDVMOPT'} = $ENV{'PBVMOPT'}; # Set a default timeout of 2 minutes if (not defined $ENV{'PBVMTMOUT'}) { $ENV{'PBVMTMOUT'} = "120"; } if (defined $vmopt->{$v}) { $ENV{'PBVMOPT'} .= " $vmopt->{$v}" if ($ENV{'PBVMOPT'} !~ / $vmopt->{$v}/); } elsif (defined $vmopt->{$ENV{'PBPROJ'}}) { $ENV{'PBVMOPT'} .= " $vmopt->{$ENV{'PBPROJ'}}" if ($ENV{'PBVMOPT'} !~ / $vmopt->{$ENV{'PBPROJ'}}/); } # How much memory to allocate for VMs if (defined $vmmm) { my $vmmem = pb_distro_get_param($pbos,$vmmm); if (defined $vmmem) { $ENV{'PBVMOPT'} .= " -m $vmmem"; } } # Are we allowed to use snapshot feature if ($usesnap == 1) { if ((defined $vmsnap->{$v}) && ($vmsnap->{$v} =~ /true/i)) { $ENV{'PBVMOPT'} .= " -snapshot"; } elsif ((defined $vmsnap->{$ENV{'PBPROJ'}}) && ($vmsnap->{$ENV{'PBPROJ'}} =~ /true/i)) { $ENV{'PBVMOPT'} .= " -snapshot"; } elsif ($pbsnap eq 1) { $ENV{'PBVMOPT'} .= " -snapshot"; } } if ($snapme != 0) { if (($vmtype eq "kvm") || ($vmtype eq "qemu")) { # Configure the monitoring to automate the creation of the 'pb' snapshot $ENV{'PBVMOPT'} .= " -serial mon:telnet::$vmmonport->{$ENV{'PBPROJ'}},server,nowait" if ((defined $vmmonport) && (defined $vmmonport->{$ENV{'PBPROJ'}})); # In that case no snapshot call needed $ENV{'PBVMOPT'} =~ s/ -snapshot//; } } if (defined $vmtmout->{$v}) { $ENV{'PBVMTMOUT'} = $vmtmout->{$v}; } elsif (defined $vmtmout->{$ENV{'PBPROJ'}}) { $ENV{'PBVMTMOUT'} = $vmtmout->{$ENV{'PBPROJ'}}; } my $nport = pb_get_port($vmport,$pbos,$vtype); my $cmd; my $vmm; # has to be used for pb_check_ps if (($vmtype eq "qemu") || ($vmtype eq "kvm")) { $vmm = "$vmpath->{$ENV{'PBPROJ'}}/$v.qemu"; if (($create != 0) || (defined $iso)) { $ENV{'PBVMOPT'} .= " -cdrom $iso -boot d"; } # Always redirect the network and always try to use a 'pb' snapshot #$cmd = "$vmcmd $ENV{'PBVMOPT'} -net user,hostfwd=tcp:$nport:10.0.2.15:22 -loadvm pb $vmm" $cmd = "$vmcmd $ENV{'PBVMOPT'} -redir tcp:$nport:10.0.2.15:22 $vmm" } elsif ($vmtype eq "xen") { } elsif ($vmtype eq "vmware") { } else { die "VM of type $vmtype not supported. Report to the dev team"; } # Restore the ENV VAR Value $ENV{'PBVMOPT'} = $ENV{'PBOLDVMOPT'}; my ($tmpcmd,$void) = split(/ +/,$cmd); my $vmexist = pb_check_ps($tmpcmd,$vmm); my $vmpid = 0; if (! $vmexist) { if ($create != 0) { die("Found an existing Virtual machine $vmm. Won't overwrite") if (-r $vmm); if (($vmtype eq "qemu") || ($vmtype eq "xen") || ($vmtype eq "kvm")) { my $command = pb_check_req("qemu-img",0); pb_system("$command create -f qcow2 $vmm $vmsize","Creating the QEMU VM"); } elsif ($vmtype eq "vmware") { } else { } } if (! -f "$vmm") { pb_log(0,"Unable to find VM $vmm\n"); } else { # Is the SSH port free? if not kill the existing process using it after a build timeout period my $vmssh = pb_check_ps($tmpcmd,"tcp:$nport:10.0.2.15:22"); if ($vmssh) { my $buildtm = $ENV{'PBVMTMOUT'}; if (defined $vmbuildtm->{$v}) { $buildtm = $vmbuildtm->{$v}; } elsif (defined $vmbuildtm->{$ENV{'PBPROJ'}}) { $buildtm = $vmbuildtm->{$ENV{'PBPROJ'}}; } sleep $buildtm; pb_log(0,"WARNING: Killing the process ($vmssh) using port $nport (previous failed VM ?)\n"); kill 15,$vmssh; # Let it time to exit sleep 5; } pb_system("$cmd &","Launching the VM $vmm"); # Using system allows to kill it externaly if needed,sosupport that in the call pb_system("sleep $ENV{'PBVMTMOUT'}","Waiting $ENV{'PBVMTMOUT'} s for VM $v to come up",undef,1); $vmpid = pb_check_ps($tmpcmd,$vmm); pb_log(0,"VM $vmm launched (pid $vmpid)\n"); } } else { pb_log(0,"Found an existing VM $vmm (pid $vmexist)\n"); } pb_log(2,"DEBUG: pb_launchv returns ($vmexist,$vmpid)\n"); return($vmexist,$vmpid); } elsif ($vtype eq "ve") { # Force the creation of the VE and no snapshot usable pb_ve_launch($v,$create,$usesnap); } else { # RM here # Get distro context my $pbos = pb_distro_get_context($v); # Get RM context my ($ptr,$rmpath) = pb_conf_get("rmtype","rmpath"); # Nothing more to do for RM. No real launch # For the moment we support the RM is already running # For ProLiant may be able to power them on if needed later on as an example. } } # Return string for date synchro sub pb_date2v { my $vtype = shift; my $pbos = shift; # VE gets time from parent OS. return "/bin/true" if ($vtype) =~ /^ve/o; my ($ntp) = pb_conf_get_if($vtype."ntp"); my $vntp = $ntp->{$ENV{'PBPROJ'}} if (defined $ntp); my $ntpline = undef; if (defined $vntp) { # ntp command depends on pbos my $vntpcmd = pb_distro_get_param($pbos,pb_conf_get($vtype."ntpcmd")); $ntpline = "sudo $vntpcmd $vntp"; } # Force new date to be in the future compared to the date # of the host by adding 1 minute my @date=pb_get_date(); $date[1]++; my $upddate = strftime("%m%d%H%M%Y", @date); my $dateline = "sudo /bin/date $upddate"; if (defined $ntpline) { return($ntpline); } else { return($dateline); } } sub pb_build2v { my $vtype = shift; my $action = shift || "build"; my ($v,$all) = pb_get2v($vtype); # Send tar files when we do a global generation pb_build2ssh() if (($all == 1) && ($action eq "build")); # Adapt // mode to memory size $pbparallel = pb_set_parallel($vtype); my ($vmexist,$vmpid) = (undef,undef); my $pm; my $all_ok = 1; if (defined $pbparallel) { $pm = new Parallel::ForkManager($pbparallel); # Set which port the VM/RM will use to communicate $pm->run_on_start(\&pb_set_port); $pm->run_on_finish(sub { my ($pid, $code, $id, $signal, $dump) = @_; unless ($code == 0 && $signal == 0 && $dump == 0) { $all_ok = 0; pb_log(0,"ERROR: pid $pid failed\n"); } }); } my $counter = 0; foreach my $v (@$v) { $counter++; # Modulo 2 * pbparallel (to avoid synchronization problems) $counter = 1 if ((defined $pbparallel) && ($counter > 2 * $pbparallel)); $pm->start($counter) and next if (defined $pbparallel); # Prepare the script to be executed on the VM/VE/RM # in $ENV{'PBDESTDIR'}/pbscript.$$ open(SCRIPT,"> $ENV{'PBDESTDIR'}/pbscript.$$") || die "Unable to create $ENV{'PBDESTDIR'}/pbscript.$$"; print SCRIPT "#!/bin/bash\n"; # Transmit the verbosity level to the virtual env/mach. my $verbose = ""; my $i = 0; # minimal debug level while ($i lt $pbdebug) { $verbose .= "-v "; $i++; } print SCRIPT "set -e\n" if $Global::pb_stop_on_error; # Activate script verbosity if at least 2 for pbdebug print SCRIPT "set -x\n" if ($i gt 1); # Quiet if asked to be so on the original system $verbose = "-q" if ($pbdebug eq -1); print SCRIPT "echo ... Execution needed\n"; print SCRIPT "# This is in directory delivery\n"; print SCRIPT "# Setup the variables required for building\n"; print SCRIPT "export PBPROJ=$ENV{'PBPROJ'}\n"; if ($action eq "build") { print SCRIPT "# Preparation for pb\n"; print SCRIPT "mv .pbrc \$HOME\n"; print SCRIPT "cd ..\n"; } # VE needs a good /proc, tolerate one being potentially left around after a failure if ($vtype eq "ve") { print SCRIPT "[ -d /proc/1 ] || sudo /bin/mount -t proc /proc /proc\n"; } # Get distro context my $pbos = pb_distro_get_context($v); my $ntpline = pb_date2v($vtype,$pbos); print SCRIPT "# Time sync\n"; print SCRIPT "echo setting up date with $ntpline\n"; print SCRIPT "$ntpline\n"; # Use potential local proxy declaration in case we need it to download repo, pkgs, ... if (defined $ENV{'http_proxy'}) { print SCRIPT "export http_proxy=\"$ENV{'http_proxy'}\"\n"; } if (defined $ENV{'ftp_proxy'}) { print SCRIPT "export ftp_proxy=\"$ENV{'ftp_proxy'}\"\n"; } # Get list of packages to build/test and get some ENV vars as well my $ptr = pb_get_pkg(); @pkgs = @$ptr; my $p = join(' ',@pkgs) if (@pkgs); print SCRIPT "export PBPROJVER=$ENV{'PBPROJVER'}\n"; print SCRIPT "export PBPROJTAG=$ENV{'PBPROJTAG'}\n"; print SCRIPT "export PBPACKAGER=\"$ENV{'PBPACKAGER'}\"\n"; # We may need to do some other tasks before building. Read a script here to finish setup if (-x "$ENV{'PBDESTDIR'}/pb$vtype".".pre") { print SCRIPT "# Special pre-instructions to be launched\n"; print SCRIPT pb_get_content("$ENV{'PBDESTDIR'}/pb$vtype".".pre"); } if (-x "$ENV{'PBDESTDIR'}/pb$vtype"."$action.pre") { print SCRIPT "# Special pre-$action instructions to be launched\n"; print SCRIPT pb_get_content("$ENV{'PBDESTDIR'}/pb$vtype"."$action.pre"); } print SCRIPT "# $action\n"; print SCRIPT "echo $action"."ing packages on $vtype...\n"; if (($action eq "test") && (! -x "$ENV{'PBDESTDIR'}/pbtest")) { die "No test script ($ENV{'PBDESTDIR'}/pbtest) found when in test mode. Aborting ..."; } print SCRIPT "pb $verbose -p $ENV{'PBPROJ'} $action"."2pkg $p\n"; if ($vtype eq "ve") { print SCRIPT "sudo /bin/umount /proc\n"; } # We may need to do some other tasks after building. Read a script here to exit properly if (-x "$ENV{'PBDESTDIR'}/pb$vtype"."$action.post") { print SCRIPT "# Special post-$action instructions to be launched\n"; print SCRIPT pb_get_content("$ENV{'PBDESTDIR'}/pb$vtype"."$action.post"); } if (-x "$ENV{'PBDESTDIR'}/pb$vtype".".post") { print SCRIPT "# Special post-instructions to be launched\n"; print SCRIPT pb_get_content("$ENV{'PBDESTDIR'}/pb$vtype".".post"); } print SCRIPT q{echo "********** Successful exit of script $$ *********************"}, "\n"; close(SCRIPT); chmod 0755,"$ENV{'PBDESTDIR'}/pbscript.$$"; # Launch the VM/VE/RM ($vmexist,$vmpid) = pb_launchv($vtype,$v,0); if ($vtype eq "vm") { # Skip that VM if something went wrong if (($vmpid == 0) && ($vmexist == 0)) { $pm->finish if (defined $pbparallel); next; } } else { # VE/RM $vmexist = 0; $vmpid = 0; } # Gather all required files to send them to the VM/VE # and launch the build through pbscript pb_log(2,"Calling send2target $vtype,$v,$vmexist,$vmpid\n"); pb_send2target(uc($vtype).$action,"$v",$vmexist,$vmpid); $pm->finish if (defined $pbparallel); } $pm->wait_all_children if (defined $pbparallel); die "Aborting, one or more of the children failed." if ((not $all_ok) && ($Global::pb_stop_on_error)); } sub pb_clean { my $sleep=10; die "Unable to get env var PBDESTDIR" if (not defined $ENV{'PBDESTDIR'}); die "Unable to get env var PBBUILDDIR" if (not defined $ENV{'PBBUILDDIR'}); pb_log(0,"We will now wait $sleep s before removing both directories\n$ENV{'PBDESTDIR'} and $ENV{'PBBUILDDIR'}.\nPlease break me if this is wrong\n"); sleep $sleep; pb_rm_rf($ENV{'PBDESTDIR'}); pb_rm_rf($ENV{'PBBUILDDIR'}); } sub pb_newver { die "-V Version parameter needed" if ((not defined $newver) || ($newver eq "")); # Need this call for PBDIR my ($scheme2,$uri) = pb_cms_init($pbinit); my ($pbconf,$pburl) = pb_conf_get("pbconfurl","pburl"); $uri = $pbconf->{$ENV{'PBPROJ'}}; my ($scheme, $account, $host, $port, $path) = pb_get_uri($uri); # Checking CMS repositories status ($scheme2, $account, $host, $port, $path) = pb_get_uri($pburl->{$ENV{'PBPROJ'}}); if ($scheme !~ /^svn/) { die "Only SVN is supported at the moment"; } my $res = pb_vcs_isdiff($scheme,$ENV{'PBROOTDIR'}); die "ERROR: No differences accepted in CMS for $ENV{'PBROOTDIR'} before creating a new version" if ($res != 0); $res = pb_vcs_isdiff($scheme2,$ENV{'PBDIR'}); die "ERROR: No differences accepted in CMS for $ENV{'PBDIR'} before creating a new version" if ($res != 0); # Tree identical between PBCONFDIR and PBROOTDIR. The delta is what # we want to get for the root of the new URL my $oldver = $ENV{'PBROOTDIR'}; $oldver =~ s|^$ENV{'PBCONFDIR'}||; pb_log(2, "PBCONFDIR: $ENV{'PBCONFDIR'}\nPBROOTDIR: $ENV{'PBROOTDIR'}\n"); my $newurl = "$uri/$newver"; # Should probably use projver in the old file my $oldvertxt= basename($oldver); my $newvertxt = basename($newver); # Duplicate and extract project-builder part pb_log(2,"Copying $uri/$oldver to $newurl\n"); pb_vcs_copy($scheme,"$uri/$oldver",$newurl); pb_log(2,"Checkout $newurl to $ENV{'PBCONFDIR'}/$newver\n"); pb_vcs_up($scheme,"$ENV{'PBCONFDIR'}"); # Duplicate and extract project my $newurl2 = "$pburl->{$ENV{'PBPROJ'}}/$newver"; pb_log(2,"Copying $pburl->{$ENV{'PBPROJ'}}/$oldver to $newurl2\n"); pb_vcs_copy($scheme2,"$pburl->{$ENV{'PBPROJ'}}/$oldver",$newurl2); my $tmp = $ENV{'PBDIR'}; $tmp =~ s|$oldver$||; pb_log(2,"Checkout $newurl2 to $tmp/$newver\n"); pb_vcs_up($scheme2,"$tmp"); # Update the .pb file open(FILE,"$ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.pb") || die "Unable to open $ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.pb"; open(OUT,"> $ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.pb.new") || die "Unable to write to $ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.pb.new"; while() { if (/^projver\s+$ENV{'PBPROJ'}\s*=\s*$oldvertxt$/) { s/^projver\s+$ENV{'PBPROJ'}\s*=\s*$oldvertxt$/projver $ENV{'PBPROJ'} = $newvertxt/; pb_log(0,"Changing projver from $oldvertxt to $newvertxt in $ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.pb\n"); } if (/^testver/) { s/^testver/#testver/; pb_log(0,"Commenting testver in $ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.pb\n") if (/^testver/); } if (/^delivery/) { my $txt = $_; chomp($txt); pb_log(0,"Please check delivery ($txt) in $ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.pb\n"); } print OUT $_; } close(FILE); close(OUT); rename("$ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.pb.new","$ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.pb"); # Checking pbcl files foreach my $f (<$ENV{'PBROOTDIR'}/*/pbcl>) { # Compute new pbcl file my $f2 = $f; $f2 =~ s|$ENV{'PBROOTDIR'}|$ENV{'PBCONFDIR'}/$newver/|; open(PBCL,$f) || die "Unable to open $f"; my $foundnew = 0; while () { $foundnew = 1 if (/^$newvertxt \(/); } close(PBCL); open(OUT,"> $f2") || die "Unable to write to $f2: $!"; open(PBCL,$f) || die "Unable to open $f"; while () { print OUT "$_" if (not /^$oldvertxt \(/); if ((/^$oldvertxt \(/) && ($foundnew == 0)) { print OUT "$newvertxt ($pbdate)\n"; print OUT "- TBD\n"; print OUT "\n"; pb_log(0,"WARNING: version $newvertxt not found in $f so added to $f2...\n") if ($foundnew == 0); } } close(OUT); close(PBCL); } pb_log(2,"Checkin $ENV{'PBCONFDIR'}/$newver\n"); pb_cms_checkin($scheme,"$ENV{'PBCONFDIR'}/$newver",undef); } # # Return the list of VMs/VEs/RMs we are working on # $all is a flag to know if we return all of them # or only some (if all we publish also tar files in addition to pkgs # sub pb_get2v { my $vtype = shift; my @v; my $all = 0; my $pbv = 'PBV'; my $vlist = $vtype."list"; # Get VM/VE list if ((not defined $ENV{$pbv}) || ($ENV{$pbv} =~ /^all$/)) { my ($ptr) = pb_conf_get($vlist); $ENV{$pbv} = $ptr->{$ENV{'PBPROJ'}}; $all = 1; } pb_log(2,"$vtype: $ENV{$pbv}\n"); @v = split(/,/,$ENV{$pbv}); return(\@v,$all); } # This function creates a giant script to configure a particular VM/VE/RM, it then copies the # script to the target. # Function to create a potentialy missing pb account on the VM/VE/RM, and adds it to sudo # Needs to use root account to connect to the VM/VE/RM # pb will take your local public SSH key to access # the pb account in the VM/VE/RM later on if needed sub pb_setup2v { my $vtype = shift; my $sbx = shift || undef; my ($vm,$all) = pb_get2v($vtype); # Script generated my $pbscript = "$ENV{'PBDESTDIR'}/setupv"; # Adapt // mode to memory size $pbparallel = pb_set_parallel($vtype); my $pm; my $all_ok = 1; if (defined $pbparallel) { $pm = new Parallel::ForkManager($pbparallel); # Set which port the VM/RM will use to communicate $pm->run_on_start(\&pb_set_port); $pm->run_on_finish(sub { my ($pid, $code, $id, $signal, $dump) = @_; $all_ok = 0 unless (($code == 0) && ($signal == 0) && ($dump == 0)); }); } my $counter = 0; foreach my $v (@$vm) { $counter++; # Modulo pbparallel $counter = 1 if ((defined $pbparallel) && ($counter > $pbparallel)); $pm->start($counter) and next if (defined $pbparallel); # Get distro context my $pbos = pb_distro_get_context($v); # Deal with date sync. my $ntpline = pb_date2v($vtype,$pbos); # Name of the account to deal with for VM/VE/RM # Do not use the one passed potentially with -a my ($pbac) = pb_conf_get($vtype."login"); my ($key,$zero0,$zero1,$zero2); my ($vmexist,$vmpid); # Prepare the script to be executed on the VM/VE/RM # in $ENV{'PBDESTDIR'}/setupv open(SCRIPT,"> $pbscript") || die "Unable to create $pbscript"; print SCRIPT << 'EOF'; #!/usr/bin/perl -w use strict; use File::Copy; # We should not need in this script more functions than what is provided # by Base, Conf and Distribution to avoid problems at exec time. # They are appended at the end. # Define mandatory global vars our $pbdebug; our $pbLOG; our $pbsynmsg = "pbscript"; our $pbdisplaytype = "text"; our $pblocale = ""; pb_log_init($pbdebug, $pbLOG); EOF print SCRIPT << "EOF"; \$Global::pb_stop_on_error = $Global::pb_stop_on_error; pb_temp_init($pbkeep); pb_conf_init("$ENV{'PBPROJ'}"); EOF # Launch the VM/VE/RM - Usage of snapshot disabled ($vmexist,$vmpid) = pb_launchv($vtype,$v,0,0,0); my $keyfile; my $nport; my $vmhost; # Prepare the key to be used and transfered remotely $keyfile = pb_ssh_get(1); if ($vtype =~ /(v|r)m/) { my ($vmport); ($vmhost,$vmport) = pb_conf_get($vtype."host",$vtype."port"); $nport = pb_get_port($vmport,$pbos,$vtype); # Skip that VM/RM if something went wrong next if (($vmpid == 0) && ($vmexist == 0)); # Store the pub key part in a variable open(FILE,"$keyfile.pub") || die "Unable to open $keyfile.pub"; ($zero0,$zero1,$zero2) = split(/ /,); close(FILE); $key = "\Q$zero1"; # We call true to avoid problems if SELinux is not activated, but chcon is present and returns in that case 1 pb_system("cat $keyfile.pub | ssh -q -o UserKnownHostsFile=/dev/null -p $nport -i $keyfile root\@$vmhost->{$ENV{'PBPROJ'}} \"mkdir -p .ssh ; chmod 700 .ssh ; cat >> .ssh/authorized_keys ; chmod 600 .ssh/authorized_keys ; if [ -x /usr/bin/chcon ]; then /usr/bin/chcon -Rt home_ssh_t .ssh 2> /dev/null; /bin/true; fi\"","Copying local keys to $vtype. This may require the root password"); # once this is done, we can do what we need on the VM/RM remotely } elsif ($vtype eq "ve") { print SCRIPT << "EOF"; # For VE we need a good null dev pb_system("rm -f /dev/null; mknod /dev/null c 1 3; chmod 777 /dev/null"); # For VE we first need to mount some FS pb_system("mount -t proc /proc /proc") unless (-d "/proc/$$"); EOF } else { die "Unknown virtual type $vtype"; } if ($vtype =~ /(v|r)m/) { print SCRIPT << 'EOF'; # Removes duplicate in .ssh/authorized_keys of our key if needed # my $file1="$ENV{'HOME'}/.ssh/authorized_keys"; open(PBFILE,$file1) || die "Unable to open $file1"; open(PBOUT,"> $file1.new") || die "Unable to open $file1.new"; my $count = 0; while () { EOF print SCRIPT << "EOF"; if (/ $key /) { \$count++; } print PBOUT \$_ if ((\$count <= 1) || (\$_ !~ / $key /)); } close(PBFILE); close(PBOUT); rename("\$file1.new",\$file1); chmod 0600,\$file1; EOF } print SCRIPT << 'EOF'; # Adds $pbac->{$ENV{'PBPROJ'}} as an account if needed # my $file="/etc/passwd"; open(PBFILE,$file) || die "Unable to open $file"; my $found = 0; while () { EOF print SCRIPT << "EOF"; \$found = 1 if (/^$pbac->{$ENV{'PBPROJ'}}:/); EOF # TODO: use an external parameter my $home = "/home"; # Solaris doesn't like that we use /home $home = "/export/home" if ($pbos->{'type'} eq "pkg"); print SCRIPT << "EOF"; } close(PBFILE); if ( \$found == 0 ) { if ( ! -d "$home" ) { pb_mkdir_p("$home"); } EOF # TODO: Level of portability of these cmds ? Critical now for RM # TODO: Check existence before adding to avoid errors print SCRIPT << "EOF"; pb_system("/usr/sbin/groupadd $pbac->{$ENV{'PBPROJ'}}","Adding group $pbac->{$ENV{'PBPROJ'}}"); pb_system("/usr/sbin/useradd -g $pbac->{$ENV{'PBPROJ'}} -m -d $home/$pbac->{$ENV{'PBPROJ'}} $pbac->{$ENV{'PBPROJ'}}","Adding user $pbac->{$ENV{'PBPROJ'}} (group $pbac->{$ENV{'PBPROJ'}} - home $home/$pbac->{$ENV{'PBPROJ'}})"); } EOF # Copy the content of our local conf file to the VM/VE/RM my $content = pb_get_content(pb_distro_conffile()); print SCRIPT << "EOF"; # # Create a temporary local conf file for distribution support # This is created here before its use later. Its place is hardcoded, so no choice for the path # my \$tempconf = pb_distro_conffile(); pb_mkdir_p(dirname(\$tempconf)); open(CONF,"> \$tempconf") || die "Unable to create \$tempconf"; print CONF q{$content}; close(CONF); EOF if ($vtype =~ /(v|r)m/) { print SCRIPT << "EOF"; # allow ssh entry to build # mkdir "$home/$pbac->{$ENV{'PBPROJ'}}/.ssh",0700; # Allow those accessing root to access the build account copy("\$ENV{'HOME'}/.ssh/authorized_keys","$home/$pbac->{$ENV{'PBPROJ'}}/.ssh/authorized_keys"); chmod 0600,".ssh/authorized_keys"; pb_system("chown -R $pbac->{$ENV{'PBPROJ'}}:$pbac->{$ENV{'PBPROJ'}} $home/$pbac->{$ENV{'PBPROJ'}}","Finish setting up the account env for $pbac->{$ENV{'PBPROJ'}}"); EOF } print SCRIPT << 'EOF'; # No passwd for build account only keys $file="/etc/shadow"; if (-f $file) { open(PBFILE,$file) || die "Unable to open $file"; open(PBOUT,"> $file.new") || die "Unable to open $file.new"; while () { EOF print SCRIPT << "EOF"; s/^$pbac->{$ENV{'PBPROJ'}}:\!\!:/$pbac->{$ENV{'PBPROJ'}}:*:/; s/^$pbac->{$ENV{'PBPROJ'}}:\!:/$pbac->{$ENV{'PBPROJ'}}:*:/; #SLES 9 e.g. s/^$pbac->{$ENV{'PBPROJ'}}:\\*LK\\*:/$pbac->{$ENV{'PBPROJ'}}:NP:/; #Solaris e.g. EOF print SCRIPT << 'EOF'; print PBOUT $_; } close(PBFILE); close(PBOUT); rename("$file.new",$file); chmod 0640,$file; } # Keep the VM in text mode $file="/etc/inittab"; if (-f $file) { open(PBFILE,$file) || die "Unable to open $file"; open(PBOUT,"> $file.new") || die "Unable to open $file.new"; while () { s/^(..):5:initdefault:$/$1:3:initdefault:/; print PBOUT $_; } close(PBFILE); close(PBOUT); rename("$file.new",$file); chmod 0640,$file; } # pb has to be added to portage group on gentoo # We need to have that pb_distro_get_context function # Get it from Project-Builder::Distribution # And we now need the conf file required for this to work created above my $pbos = pb_distro_get_context(); print "distro tuple: ".Dumper($pbos)."\n"; # Adapt sudoers # sudo is not default on Solaris and needs to be installed first # from http://www.sunfreeware.com/programlistsparc10.html#sudo if ($pbos->{'type'} eq "pkg") { $file="/usr/local/etc/sudoers"; } else { $file="/etc/sudoers"; } open(PBFILE,$file) || die "Unable to open $file"; open(PBOUT,"> $file.new") || die "Unable to open $file.new"; while () { EOF # Skip what will be generated print SCRIPT << "EOF"; next if (/^$pbac->{$ENV{'PBPROJ'}}\\s+/); next if (/^Defaults:$pbac->{$ENV{'PBPROJ'}}\\s+/); next if (/^Defaults:root \!requiretty/); next if (/^Defaults:root env_keep/); EOF print SCRIPT << 'EOF'; s/Defaults\s+requiretty/# disable Defaults: requiretty/; print PBOUT $_; } close(PBFILE); EOF print SCRIPT << "EOF"; # Some distro force requiretty at compile time, so disable here print PBOUT "Defaults:$pbac->{$ENV{'PBPROJ'}} !requiretty\n"; print PBOUT "Defaults:root !requiretty\n"; # Keep proxy configuration while using sudo print PBOUT "Defaults:$pbac->{$ENV{'PBPROJ'}} env_keep += \\\"http_proxy ftp_proxy\\\"\n"; print PBOUT "Defaults:root env_keep += \\\"http_proxy ftp_proxy\\\"\n"; EOF # Try to restrict security to what is really needed if ($vtype =~ /^vm/) { my $hpath = pb_distro_get_param($pbos,pb_conf_get("ospathcmd-halt")); my @sudocmds = pb_get_sudocmds($pbos,$ntpline,"sudo $hpath"); print SCRIPT << "EOF"; # This is needed in order to be able on VM to halt the machine from the $pbac->{$ENV{'PBPROJ'}} account at least # Build account $pbac->{$ENV{'PBPROJ'}} in VM also needs to setup date and install deps. # Nothing else should be needed EOF foreach my $c (@sudocmds) { print SCRIPT "print PBOUT \"$pbac->{$ENV{'PBPROJ'}} ALL = NOPASSWD: $c\\n\";\n"; } } elsif ($vtype =~ /^rm/) { my @sudocmds = pb_get_sudocmds($pbos,$ntpline); print SCRIPT << "EOF"; # Build account $pbac->{$ENV{'PBPROJ'}} in RM only needs to setup date and install deps if needed each time EOF foreach my $c (@sudocmds) { print SCRIPT "print PBOUT \"$pbac->{$ENV{'PBPROJ'}} ALL = NOPASSWD: $c\\n\";\n"; } } else { print SCRIPT << "EOF"; # Build account $pbac->{$ENV{'PBPROJ'}} for VE needs to do a lot in the host (and chroot), so allow without restriction for now print PBOUT "$pbac->{$ENV{'PBPROJ'}} ALL=(ALL) NOPASSWD:ALL\n"; EOF } print SCRIPT << 'EOF'; close(PBOUT); rename("$file.new",$file); chmod 0440,$file; EOF if ($vtype =~ /(v|r)m/) { # Sync date # do it after sudoers is setup print SCRIPT "pb_system(\"$ntpline\");\n"; } # We may need a proxy configuration. Get it from the local env my $ftp_proxy = pb_distro_get_param($pbos,pb_conf_get_if("ftp_proxy")); my $http_proxy = pb_distro_get_param($pbos,pb_conf_get_if("http_proxy")); $ENV{'ftp_proxy'} ||= $ftp_proxy if ((defined $ftp_proxy) && ($ftp_proxy ne "")); $ENV{'http_proxy'} ||= $http_proxy if ((defined $http_proxy) && ($http_proxy ne "")); if (defined $ENV{'http_proxy'}) { print SCRIPT "\$ENV\{'http_proxy'\}=\"$ENV{'http_proxy'}\";\n"; } if (defined $ENV{'ftp_proxy'}) { print SCRIPT "\$ENV\{'ftp_proxy'\}=\"$ENV{'ftp_proxy'}\";\n"; } print SCRIPT << 'EOF'; # Suse wants sudoers as 640 if ((($pbos->{'name'} eq "sles") && (($pbos->{'version'} =~ /10/) || ($pbos->{'version'} =~ /9/))) || (($pbos->{'name'} eq "opensuse") && ($pbos->{'version'} =~ /10.[012]/))) { chmod 0640,$file; } # First install all required packages pb_system("yum clean all","Cleaning yum env") if (($pbos->{'name'} eq "fedora") || ($pbos->{'name'} eq "asianux") || ($pbos->{'name'} eq "rhel")); my ($ospkgdep) = pb_conf_get_if("ospkgdep"); my $pkgdep = pb_distro_get_param($pbos,$ospkgdep); pb_distro_installdeps(undef,$pbos,pb_distro_only_deps_needed($pbos,join(' ',split(/,/,$pkgdep)))); EOF my $itype = pb_distro_get_param($pbos,pb_conf_get("pbinstalltype")); # Install from sandbox mean a file base install $itype = "file" if (defined $sbx); if ($itype =~ /^file/) { my $cmdget; if (defined $sbx) { # Install from sandbox mean using the result of the just passed sbx2build command # Get content saved in cms2build my ($pkg) = pb_conf_read("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.pb","pbpkg"); my $pbextdir = pb_get_extdir(); die "Unable to get package list" if (not defined $pkg); # We consider 2 specific packages my $vertag1 = $pkg->{"ProjectBuilder"}; my $vertag2 = $pkg->{"project-builder"}; # get the version of the current package - maybe different pb_log(2,"Vertag1: $vertag1\n"); pb_log(2,"Vertag2: $vertag2\n"); my ($pbver1,$tmp1) = split(/-/,$vertag1); my ($pbver2,$tmp2) = split(/-/,$vertag2); # Copy inside the VE if ($vtype eq "ve") { my ($vepath) = pb_conf_get("vepath"); copy("$ENV{'PBDESTDIR'}/ProjectBuilder-$pbver1$pbextdir.tar.gz","$vepath->{$ENV{'PBPROJ'}}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}/tmp"); copy("$ENV{'PBDESTDIR'}/project-builder-$pbver2$pbextdir.tar.gz","$vepath->{$ENV{'PBPROJ'}}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}/tmp"); } else { pb_system("scp -i $keyfile -p -o UserKnownHostsFile=/dev/null -P $nport $ENV{'PBDESTDIR'}/ProjectBuilder-$pbver1$pbextdir.tar.gz $ENV{'PBDESTDIR'}/project-builder-$pbver2$pbextdir.tar.gz root\@$vmhost->{$ENV{'PBPROJ'}}:/tmp","Copying local project files to $vtype."); } $cmdget = "mv /tmp/ProjectBuilder-$pbver1$pbextdir.tar.gz ProjectBuilder-latest.tar.gz ; mv /tmp/project-builder-$pbver2$pbextdir.tar.gz project-builder-latest.tar.gz"; } else { # TODO: shouldn't we have a variable to specify the source file? # And shouldn't we be getting the one that matches the current pb version? # For the moment, just take the latest stable one $cmdget = "wget --passive-ftp ftp://ftp.project-builder.org/src/ProjectBuilder-latest.tar.gz; wget --passive-ftp ftp://ftp.project-builder.org/src/project-builder-latest.tar.gz"; } print SCRIPT << 'EOF'; # Then install manually the missing perl modules my ($osperldep,$osperlver) = pb_conf_get_if("osperldep","osperlver"); my $perldep = pb_distro_get_param($pbos,$osperldep); foreach my $m (split(/,/,$perldep)) { # Skip empty deps next if ($m =~ /^\s*$/); my $dir = $m; $dir =~ s/-.*//; pb_system("echo \"rm -rf $m* ; wget http://search.cpan.org/CPAN/modules/by-module/$dir/$m-$osperlver->{$m}.tar.gz ; gzip -cd $m-$osperlver->{$m}.tar.gz | tar xf - ; cd $m* ; if [ -f Build.PL ]; then perl Build.PL; ./Build ; ./Build install ; else perl Makefile.PL; make ; make install ; fi; cd .. ; rm -rf $m*\" | bash -e" ,"Installing perl module $m-$osperlver->{$m}"); } EOF print SCRIPT << 'EOF'; pb_system("rm -rf ProjectBuilder-* ; rm -rf project-builder-* ; rm -rf `perl -V:installvendorlib | awk -F\"'\" '{print \$2}'`/ProjectBuilder ; EOF print SCRIPT " $cmdget ; "; print SCRIPT << 'EOF' gzip -cd ProjectBuilder-latest.tar.gz | tar xf - ; cd ProjectBuilder-* ; perl Makefile.PL ; make ; make install ; cd .. ; rm -rf ProjectBuilder-* ; gzip -cd project-builder-latest.tar.gz | tar xf - ; cd project-builder-* ; perl Makefile.PL ; make ; make install ; cd .. ; rm -rf project-builder-* ;","Building Project-Builder"); EOF } elsif ($itype =~ /^pkg/) { # pkg based install. We need to point to the project-builder.org repository print SCRIPT << 'EOF'; my $pkgforpb = pb_distro_get_param($pbos,pb_conf_get_if("ospkg")); pb_distro_setuposrepo($pbos); pb_distro_installdeps(undef,$pbos,pb_distro_only_deps_needed($pbos,join(' ',split(/,/,$pkgforpb)))); EOF } else { # Unknown install type die("Unknown install type $itype->{$ENV{'PBPROJ'}} for param pbinstalltype"); } print SCRIPT << 'EOF'; pb_system("pb 2>&1 | head -5",undef,"verbose"); pb_system("pbdistrocheck",undef,"verbose"); EOF if ($vtype eq "ve") { print SCRIPT << 'EOF'; # For VE we need to umount some FS at the end pb_system("umount /proc"); # Create a basic network file if not already there my $nf="/etc/sysconfig/network"; if ((! -f $nf) && ($pbos->{'type'} eq "rpm") && ($pbos->{'family'} ne "novell")) { open(NF,"> $nf") || die "Unable to create $nf"; print NF "NETWORKING=yes\n"; print NF "HOSTNAME=localhost\n"; close(NF); } chmod 0755,$nf; EOF } # Adds pb_distro_get_context and all functions needed from ProjectBuilder::Distribution, Conf and Base foreach my $m ("ProjectBuilder/Base.pm","ProjectBuilder/Distribution.pm","ProjectBuilder/Conf.pm") { foreach my $d (@INC) { my $f = "$d/$m"; if (-f "$f") { open(PBD,"$f") || die "Unable to open $f: $!"; while () { next if (/^package/); next if (/^use Exporter/); next if (/^use ProjectBuilder::/); next if (/^our /); print SCRIPT $_; } close(PBD); # We just need the first one of each file wrt @INC - hopefully that's the right one. last; } } } # Use a fake pb_version_init version here print SCRIPT << "EOF"; sub pb_version_init { return("$projectbuilderver","$projectbuilderrev"); } 1; EOF close(SCRIPT); chmod 0755,"$pbscript"; # That build script needs to be run as root and force stop of VM at end $pbaccount = "root"; # Force shutdown of VM except if it was already launched my $pbforce = 0; if ((! $vmexist) && ($vtype eq "vm")) { $pbforce = 1; } pb_script2v($pbscript,$vtype,$pbforce,$v); $pm->finish if (defined $pbparallel); } $pm->wait_all_children if (defined $pbparallel); die "Aborting, one or more of the children failed." if ((not $all_ok) && ($Global::pb_stop_on_error)); return; } # Function to create a snapshot named 'pb' for VMs and a compressed tar for VEs sub pb_snap2v { my $vtype = shift; my ($vm,$all) = pb_get2v($vtype); # Script generated my $pbscript = "$ENV{'PBDESTDIR'}/snapv"; my ($pbac) = pb_conf_get($vtype."login"); foreach my $v (@$vm) { if ($vtype eq "ve") { # Get distro context my $pbos = pb_distro_get_context($v); my ($vepath) = pb_conf_get("vepath"); # Test if an existing snapshot exists and remove it if there is a VE if ((-f "$vepath->{$ENV{'PBPROJ'}}/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}.tar.gz") && (! -d "$vepath->{$ENV{'PBPROJ'}}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}")) { pb_system("sudo rm -f $vepath->{$ENV{'PBPROJ'}}/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}.tar.gz","Removing previous snapshot $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}.tar.gz"); } } # Prepare the script to be executed on the VM/VE open(SCRIPT,"> $pbscript") || die "Unable to create $pbscript"; print SCRIPT << 'EOF'; #!/bin/bash sleep 2 EOF close(SCRIPT); chmod 0755,"$pbscript"; # Force shutdown of VM/VE # Force snapshot of VM/VE pb_script2v($pbscript,$vtype,1,$v,1); } return; } # Function to update VMs/VEs/RMs with the latest distribution content sub pb_update2v { my $vtype = shift; my ($vm,$all) = pb_get2v($vtype); # Script generated my $pbscript = "$ENV{'PBDESTDIR'}/updatev"; my ($pbac) = pb_conf_get($vtype."login"); foreach my $v (@$vm) { # Get distro context my $pbos = pb_distro_get_context($v); # Prepare the script to be executed on the VM/VE/RM # in $ENV{'PBDESTDIR'}/updatev open(SCRIPT,"> $pbscript") || die "Unable to create $pbscript"; print SCRIPT << 'EOF'; #!/bin/bash sleep 2 EOF # VE needs a good /proc if ($vtype eq "ve") { print SCRIPT "sudo /bin/mount -t proc /proc /proc\n"; } print SCRIPT "$pbos->{'update'}\n"; if ($vtype eq "ve") { print SCRIPT "sudo /bin/umount /proc\n"; } close(SCRIPT); chmod 0755,"$pbscript"; # Force shutdown of VM except pb_script2v($pbscript,$vtype,1,$v); } return; } sub pb_announce { my $antype = shift; # Get all required parameters my ($pbpackager,$pbrepo,$pbml,$pbsmtp) = pb_conf_get("pbpackager","pbrepo","pbml","pbsmtp"); my ($pkgv, $pkgt, $testver) = pb_conf_get_if("pkgver","pkgtag","testver"); if (((not defined $testver) || (not defined $testver->{$ENV{'PBPROJ'}}) || ($testver->{$ENV{'PBPROJ'}} !~ /true/i)) && ($antype eq "Clean")) { # We just clean for test versions pb_log(0,"Unable to clean SSH repository for non testver version\n"); return; } my $pkg = pb_cms_get_pkg($defpkgdir,$extpkgdir); my @pkgs = @$pkg; my %pkgs; my $first = 0; # Get all distros concerned my $pbos = pb_distro_get_context(); my $distrolist = pb_get_distros($pbos,undef); my %dl; my %theorlist; my %archlist; foreach my $d (split(/,/,$distrolist)) { my ($d1,$d2,$d3) = split(/-/,$d); $dl{$d1}++; } # Command to find packages on repo my $findstr = "find ".join(" ",keys %dl)." "; my $srcstr = ""; # Generated announce files my @files; foreach my $pbpkg (@pkgs) { if ($first != 0) { $findstr .= "-o "; } $first++; if ((defined $pkgv) && (defined $pkgv->{$pbpkg})) { $pbver = $pkgv->{$pbpkg}; } else { $pbver = $ENV{'PBPROJVER'}; } if ((defined $pkgt) && (defined $pkgt->{$pbpkg})) { $pbtag = $pkgt->{$pbpkg}; } else { $pbtag = $ENV{'PBPROJTAG'}; } # TODO: use virtual/real names here now my $pbrealpkg = $pbpkg; my $pbrealpkgrpm = pb_cms_get_real_pkg($pbpkg,"rpm"); my $pbrealpkgdeb = pb_cms_get_real_pkg($pbpkg,"deb"); if ($antype eq "Clean") { # We only clean test versions anyway $pbtag = "0"; my $nver = $pbver; my $ntag = "[2-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]"; $pbver .= $ntag; $findstr .= "-name \'$pbrealpkgrpm-$pbver-$pbtag\.*.rpm\' -o -name \'$pbrealpkgrpm-debug-$pbver-$pbtag\.*.rpm\' -o -name \'$pbrealpkgdeb"."_$pbver-$pbtag"."_*\.deb\' -o -name \'$pbrealpkgdeb"."_$pbver-$pbtag.dsc\' -o -name \'$pbrealpkgdeb"."_$pbver-$pbtag.tar.gz\' -o -name \'$pbrealpkg-$nver"."_p$ntag\.ebuild\' -o -name \'$pbrealpkg-$pbver-$pbtag*\.pkg\' -o -name \'$pbrealpkg-$pbver-$pbtag*\.sd\' "; $srcstr .= "src/$pbrealpkg-$pbver.tar.gz src/$pbrealpkg-$pbver.pbconf.tar.gz "; } else { my @date=pb_get_date(); # the matching is only done on packages made the same day for test version. Hopefully this is enough my $nver = $pbver; if ((defined $testver) && (defined $testver->{$ENV{'PBPROJ'}}) && ($testver->{$ENV{'PBPROJ'}} =~ /true/i) && ($antype eq "Check")) { $pbtag = "0"; my $ntag .= strftime("%Y%m%d*", @date); $nver = $pbver."_p$ntag"; $pbver .= $ntag; } $findstr .= "-name \'$pbrealpkgrpm-$pbver-$pbtag\.*.rpm\' -o -name \'$pbrealpkgdeb"."_$pbver*\.deb\' -o -name \'$pbrealpkg-$nver*\.ebuild\' -o -name \'$pbrealpkg-$pbver*\.pkg\' -o -name \'$pbrealpkg-$pbver*\.sd\' "; } if ($antype eq "Announce") { my $chglog; pb_cms_init($pbinit); # Get project info on log file and generate tmp files used later on $chglog = "$ENV{'PBROOTDIR'}/$pbpkg/pbcl"; $chglog = "$ENV{'PBROOTDIR'}/pbcl" if (! -f $chglog); $chglog = undef if (! -f $chglog); open(OUT,"> $ENV{'PBTMP'}/$pbpkg.ann") || die "Unable to create $ENV{'PBTMP'}/$pbpkg.ann: $!"; my $pb; $pb->{'realpkg'} = $pbrealpkg; $pb->{'ver'} = $pbver; $pb->{'tag'} = $pbtag; $pb->{'date'} = $pbdate; $pb->{'extdir'} = pb_get_extdir(); $pb->{'chglog'} = $chglog; $pb->{'packager'} = $pbpackager; $pb->{'proj'} = $ENV{'PBPROJ'}; $pb->{'repo'} = $pbrepo; $pb->{'pbos'}->{'type'} = "announce"; $pb->{'pbos'}->{'suffix'} = "none"; pb_changelog($pb,\*OUT,"yes"); close(OUT); push(@files,"$ENV{'PBTMP'}/$pbpkg.ann"); } elsif ($antype eq "Check") { # For the check we also build the theoritical complete list we should get foreach my $d (split(/,/,$distrolist)) { $pbos = pb_distro_get_context($d); if ($pbos->{'type'} eq "rpm") { $theorlist{"$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}/$pbrealpkgrpm-$pbver-$pbtag$pbos->{'suffix'}"} = 0; } elsif ($pbos->{'type'} eq "deb") { $theorlist{"$pbos->{'name'}/$pbos->{'version'}/$pbrealpkgdeb"."_$pbver-$pbtag"} = 0; # TODO are we always using the last arch ? $archlist{"$pbos->{'name'}/$pbos->{'version'}/$pbrealpkgdeb"."_$pbver-$pbtag"} = "$pbos->{'arch'}"; } elsif ($pbos->{'type'} eq "ebuild") { my $prefix = "-r"; $prefix = "_p" if ((defined $testver) && (defined $testver->{$ENV{'PBPROJ'}}) && ($testver->{$ENV{'PBPROJ'}} =~ /true/i)); $theorlist{"$pbos->{'name'}/$pbos->{'version'}/$pbrealpkg-$pbver$prefix$pbtag.ebuild"} = 0; $archlist{"$pbos->{'name'}/$pbos->{'version'}/$pbrealpkg-$pbver$prefix$pbtag.ebuild"} = "$pbos->{'arch'}"; } elsif ($pbos->{'type'} eq "pkg") { $theorlist{"$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}/$pbrealpkg-$pbver-$pbtag.pkg"} = 0; } else { pb_log(1,"No theoritical list possible for type $pbos->{'type'}\n"); } } } pb_log(2,"theorlist : ".Dumper(%theorlist)."\n"); } if ($antype eq "Announce") { $findstr .= " | grep -Ev \'src.rpm\'"; } elsif ($antype eq "Clean") { $findstr .= " | xargs rm -f -v $srcstr "; } # Prepare the command to run and execute it open(PBS,"> $ENV{'PBTMP'}/pbscript") || die "Unable to create $ENV{'PBTMP'}/pbscript"; print PBS "#!/bin/bash\n"; print PBS "set -x\n" if ($pbdebug gt 1); print PBS "$findstr | sort 2> /dev/null\n"; close(PBS); chmod 0755,"$ENV{'PBTMP'}/pbscript"; pb_send2target("Announce"); my $sl = "Project $ENV{'PBPROJ'} version $ENV{'PBPROJVER'} is now available"; if ($antype eq "Announce") { # Get subject line pb_log(0,"Please enter the title of your announce\n"); pb_log(0,"(By default: $sl)\n"); my $sl2 = ; $sl = $sl2 if ($sl2 !~ /^$/); # Prepare a template of announce open(ANN,"> $ENV{'PBTMP'}/announce.html") || die "Unable to create $ENV{'PBTMP'}/announce.html: $!"; print ANN << "EOF"; $sl

The project team is happy to announce the availability of a newest version of $ENV{'PBPROJ'} $ENV{'PBPROJVER'}. Enjoy it as usual!

Now available at $pbrepo->{$ENV{'PBPROJ'}}

EOF } open(LOG,"$ENV{'PBTMP'}/system.$$.log") || die "Unable to read $ENV{'PBTMP'}/system.$$.log: $!"; if ($antype eq "Announce") { my $col = 2; my $i = 1; print ANN << 'EOF'; EOF while () { print ANN ""; $i++; if ($i > $col) { print ANN "\n"; $i = 1; } } } elsif ($antype eq "Clean") { while () { # skip errors next if ($_ !~ /^removed /); pb_log(0,"$_"); } } else { # In Check mode we need to compare the 2 lists (real and theoritical) while () { # Get package name and remove what is in extra for the theoritical list (arch at the end) chomp(); # skip find errors next if (/^find:/); my $p = $_; $p =~ s/\.(i[3456]86|x86_64|noarch|src)\.rpm$//; $p =~ s/_(i[3456]86|amd64|all).deb$//; $p =~ s/(-0\.[0-9]{8})[0-9]{6}/$1*/ if ((defined $testver) && (defined $testver->{$ENV{'PBPROJ'}}) && ($testver->{$ENV{'PBPROJ'}} =~ /true/i)); $p =~ s/(-r|_p[0-9]+)\.ebuild/$1*/ if ((defined $testver) && (defined $testver->{$ENV{'PBPROJ'}}) && ($testver->{$ENV{'PBPROJ'}} =~ /true/i)); $theorlist{$p} = -2 if (not defined $theorlist{$p}); $theorlist{$p} = $theorlist{$p} + 1; } pb_log(2,"theorlist : ".Dumper(%theorlist)."\n"); } close(LOG); # Nothing more for the cleanssh case return if ($antype eq "Clean"); if ($antype eq "Check") { my ($chkex) = pb_conf_get_if("checkexclude"); my $vmbuildlist = ""; my $vebuildlist = ""; my $rmbuildlist = ""; my @pt = pb_conf_get_if("vmlist","velist","rmlist"); foreach my $t (sort keys %theorlist) { if (defined $theorlist{$t} and $theorlist{$t} >= 1) { pb_log(1,"Packages found for $t\n"); } elsif (defined $theorlist{$t} and $theorlist{$t} < 0) { pb_log(0,"Extra Package found for $t\n"); } else { pb_log(2,"Analyzing $t\n"); my ($os,$ver,$arch,$package) = split(/\//,$t); # Some distro have no arch subdir if (not defined $package) { $package = $arch; # TODO: If both arch have failed, we just make the last one $arch = $archlist{$t}; } my $pbos = pb_distro_get_context("$os-$ver-$arch"); my $pkgn = $package; if ($pbos->{'type'} ne "deb") { # package name is more easily found from the end for non deb # as '-' is the separator, but it can also be used in names $pkgn = reverse($package); # search the second '-' and isolate the now last part which is the full name $pkgn =~ s/([^-]+)-([^-]+)-([\S])+$/$3/; } else { $pkgn =~ s/([^_]+)_([\S])+$/$2/; } my $found = 0; # Handle the exclusion of OSes my $excl = ""; $excl .= $chkex->{$pkgn} if (defined $chkex->{$pkgn}); $excl .= $chkex->{"all"} if (defined $chkex->{"all"}); foreach my $ex (split(/,/,$excl)) { $found = 1 if ("$os-$ver-$arch" =~ /^$ex/); } # Skip as excluded next if ($found == 1); pb_log(0,"Package NOT found for $t\n"); # Avoid duplicates in list next if ($vmbuildlist =~ /$os-$ver-$arch/); next if ($vebuildlist =~ /$os-$ver-$arch/); next if ($rmbuildlist =~ /$os-$ver-$arch/); # check with which method we need to build if ((defined $pt[0]->{$ENV{'PBPROJ'}}) and ($pt[0]->{$ENV{'PBPROJ'}} =~ /$os-$ver-$arch/)) { $vmbuildlist = "$os-$ver-$arch" if ($vmbuildlist eq ""); $vmbuildlist .= ",$os-$ver-$arch" if ($vmbuildlist !~ /$os-$ver-$arch/); next; } if ((defined $pt[1]->{$ENV{'PBPROJ'}}) and ($pt[1]->{$ENV{'PBPROJ'}} =~ /$os-$ver-$arch/)) { $vebuildlist = "$os-$ver-$arch" if ($vebuildlist eq ""); $vebuildlist .= ",$os-$ver-$arch" if ($vebuildlist !~ /$os-$ver-$arch/); next; } if ((defined $pt[2]->{$ENV{'PBPROJ'}}) and ($pt[2]->{$ENV{'PBPROJ'}} =~ /$os-$ver-$arch/)) { $rmbuildlist = "$os-$ver-$arch" if ($rmbuildlist eq ""); $rmbuildlist .= ",$os-$ver-$arch" if ($rmbuildlist !~ /$os-$ver-$arch/); } } } # If we want to rebuild automatically, let's do it if (defined $opts{'rebuild'}) { # SandBox or CMS pb_log(0,"Rebuilding from SandBox\n"); pb_log(0,"for VMs: $vmbuildlist\n") if ($vmbuildlist ne ""); pb_log(0,"for VEs: $vebuildlist\n") if ($vebuildlist ne ""); pb_log(0,"for RMs: $rmbuildlist\n") if ($rmbuildlist ne ""); pb_cms2build("SandBox"); # Which mode $ENV{'PBV'} = $vmbuildlist; pb_build2v("vm","build") if ($vmbuildlist ne ""); $ENV{'PBV'} = $vebuildlist; pb_build2v("ve","build") if ($vebuildlist ne ""); $ENV{'PBV'} = $rmbuildlist; pb_build2v("rm","build") if ($rmbuildlist ne ""); } # For the check part this is now finished return; } print ANN << "EOF";
{$ENV{'PBPROJ'}}/$_\">$_

As usual source packages are also available in the same directory.

Changes are :

EOF # Get each package changelog content foreach my $f (sort(@files)) { open(IN,"$f") || die "Unable to read $f:$!"; while () { print ANN $_; } close(IN); print ANN "

\n"; } print ANN "

\n"; close(ANN); # Allow for modification my $editor = "vi"; $editor = $ENV{'EDITOR'} if (defined $ENV{'EDITOR'}); pb_system("$editor $ENV{'PBTMP'}/announce.html","Allowing modification of the announce","noredir"); # Store it in DB for external usage (Web pages generation) my $db = "$ENV{'PBCONFDIR'}/announces3.sql"; my $precmd = ""; if (! -f $db) { $precmd = "CREATE TABLE announces (id INTEGER PRIMARY KEY AUTOINCREMENT, date DATE, announce VARCHAR[65535])"; } my $dbh = DBI->connect("dbi:SQLite:dbname=$db","","", { RaiseError => 1, AutoCommit => 1 }) || die "Unable to connect to $db"; if ($precmd ne "") { my $sth = $dbh->prepare(qq{$precmd}) || die "Unable to create table into $db"; $sth->execute(); } # To read whole file local $/; open(ANN,"$ENV{'PBTMP'}/announce.html") || die "Unable to read $ENV{'PBTMP'}/announce.html: $!"; my $announce = ; close(ANN); pb_log(2,"INSERT INTO announces VALUES (NULL, $pbdate, $announce)"); my $sth = $dbh->prepare(qq{INSERT INTO announces VALUES (NULL,?,?)}) || die "Unable to insert into $db"; $sth->execute($pbdate, $announce); $sth->finish(); $dbh->disconnect; # Then deliver it on the Web # $TOOLHOME/livwww www # Mail it to project's ML open(ML,"| w3m -dump -T text/html > $ENV{'PBTMP'}/announce.txt") || die "Unable to create $ENV{'PBTMP'}/announce.txt: $!"; print ML << 'EOF';

EOF open(ANN,"$ENV{'PBTMP'}/announce.html") || die "Unable to read $ENV{'PBTMP'}/announce.html: $!"; while() { print ML $_; } print ML << 'EOF'; EOF close(ML); # To read whole file local $/; open(ANN,"$ENV{'PBTMP'}/announce.txt") || die "Unable to read $ENV{'PBTMP'}/announce.txt: $!"; my $msg = ; close(ANN); # Preparation of headers eval { require Mail::Sendmail; Mail::Sendmail->import(); }; if ($@) { # Mail::Sendmail not found not sending mail ! pb_log(0,"No Mail::Sendmail module found so not sending any mail !\n"); } else { my %mail = ( To => $pbml->{$ENV{'PBPROJ'}}, From => $pbpackager->{$ENV{'PBPROJ'}}, Smtp => $pbsmtp->{$ENV{'PBPROJ'}}, Body => $msg, Subject => "[ANNOUNCE] $sl", ); # Send mail if (! sendmail(%mail)) { if ((defined $Mail::Sendmail::error) and (defined $Mail::Sendmail::log)) { die "Unable to send mail ($Mail::Sendmail::error): $Mail::Sendmail::log"; } } } } # # Creates a set of HTML file containing the news for the project # based on what has been generated by the pb_announce function # sub pb_web_news2html { my $dest = shift || $ENV{'PBTMP'}; # Get all required parameters my ($pkgv, $pkgt) = pb_conf_get_if("pkgver","pkgtag"); # DB of announces for external usage (Web pages generation) my $db = "$ENV{'PBCONFDIR'}/announces3.sql"; my $dbh = DBI->connect("dbi:SQLite:dbname=$db","","", { RaiseError => 1, AutoCommit => 1 }) || die "Unable to connect to $db"; # For date handling $ENV{LANGUAGE}="C"; my $firstjan = strftime("%Y-%m-%d", 0, 0, 0, 1, 0, localtime->year(), 0, 0, -1); my $oldfirst = strftime("%Y-%m-%d", 0, 0, 0, 1, 0, localtime->year()-1, 0, 0, -1); pb_log(2,"firstjan: $firstjan, oldfirst: $oldfirst, pbdate:$pbdate\n"); my $all = $dbh->selectall_arrayref("SELECT id,date,announce FROM announces ORDER BY date DESC"); my %news; $news{"cy"} = ""; # current year's news $news{"ly"} = ""; # last year news $news{"py"} = ""; # previous years news $news{"fp"} = ""; # first page news my $cpt = 4; # how many news for first page # Extract info from DB foreach my $row (@$all) { my ($id, $date, $announce) = @$row; $news{"cy"} = $news{"cy"}."

$date $announce\n" if ((($date cmp $pbdate) le 0) && (($firstjan cmp $date) le 0)); $news{"ly"} = $news{"ly"}."

$date $announce\n" if ((($date cmp $firstjan) le 0) && (($oldfirst cmp $date) le 0)); $news{"py"} = $news{"py"}."

$date $announce\n" if (($date cmp $oldfirst) le 0); $news{"fp"} = $news{"fp"}."

$date $announce\n" if ($cpt > 0); $cpt--; } pb_log(1,"news{fp}: ".$news{"fp"}."\n"); $dbh->disconnect; # Generate the HTML content foreach my $pref (keys %news) { open(NEWS,"> $dest/pb_web_$pref"."news.html") || die "Unable to create $dest/pb_web_$pref"."news.html: $!"; print NEWS "$news{$pref}"; close(NEWS); } } # Return the SSH key file to use # Potentially create it if needed sub pb_ssh_get { my $create = shift || 0; # Do not create keys by default my ($pbagent) = pb_conf_get_if("pbusesshagent"); # use ssh-agent if asked so. return(undef) if (($create == 0) && (defined $pbagent->{$ENV{'PBPROJ'}}) && ($pbagent->{$ENV{'PBPROJ'}} =~ /true/io)); # Check the SSH environment my $keyfile = undef; # We have specific keys by default $keyfile = "$ENV{'HOME'}/.ssh/pb_dsa"; if (!(-e $keyfile) && ($create eq 1)) { pb_system("ssh-keygen -q -b 1024 -N '' -f $keyfile -t dsa","Generating SSH keys for pb"); } $keyfile = "$ENV{'HOME'}/.ssh/id_rsa" if (-s "$ENV{'HOME'}/.ssh/id_rsa"); $keyfile = "$ENV{'HOME'}/.ssh/id_dsa" if (-s "$ENV{'HOME'}/.ssh/id_dsa"); $keyfile = "$ENV{'HOME'}/.ssh/pb_dsa" if (-s "$ENV{'HOME'}/.ssh/pb_dsa"); die "Unable to find your public ssh key under $ENV{'HOME'}/.ssh" if (not defined $keyfile); return($keyfile); } # Returns the pid of a running VM command using a specific VM file sub pb_check_ps { my $vmcmd = shift; my $vmm = shift; my $vmexist = 0; # FALSE by default open(PS, "ps auxhww|") || die "Unable to call ps"; while () { next if (! /$vmcmd/); next if (! /$vmm/); my ($void1, $void2); ($void1, $vmexist, $void2) = split(/ +/); last; } return($vmexist); } sub pb_extract_build_files { my $src=shift; my $dir=shift; my $ddir=shift; my $mandatory=shift || "spec"; my @files; my $flag = "mayfail" if (($mandatory eq "patch") || ($mandatory eq "src")); my $res; if ($src =~ /tar\.gz$/) { $res = pb_system("tar xfpz $src $dir","Extracting $mandatory files from $src",$flag); } elsif ($src =~ /tar\.bz2$/) { $res = pb_system("tar xfpj $src $dir","Extracting $mandatory files from $src",$flag); } else { die "Unknown compression algorithm for $src"; } # If not mandatory return now return() if (($res != 0) and (($mandatory eq "patch") || ($mandatory eq "src"))); opendir(DIR,"$dir") || die "Unable to open directory $dir: $!"; foreach my $f (readdir(DIR)) { next if ($f =~ /^\./); # Skip potential patch dir next if ($f =~ /^pbpatch/); # Skip potential source dir next if ($f =~ /^pbsrc/); # Skip potential backup files next if ($f =~ /~$/); move("$dir/$f","$ddir") || die "Unable to move $dir/$f to $ddir"; pb_log(2,"mv $dir/$f $ddir\n"); push @files,"$ddir/$f"; } closedir(DIR); # Not enough but still a first cleanup pb_rm_rf("$dir"); return(@files); } sub pb_list_bfiles { my $dir = shift; my $pbpkg = shift; my $bfiles = shift; my $pkgfiles = shift; my $supfiles = shift; # subdir to keep if recursive mode, empty by default my $subdir = shift || ""; # In a recursive function , we need a local var as DIR handle my $bdir; pb_log(2,"DEBUG: entering pb_list_bfiles in $dir: ".Dumper($bfiles)."\n"); opendir($bdir,"$dir") || die "Unable to open dir $dir: $!"; foreach my $f (readdir($bdir)) { pb_log(3,"DEBUG: pb_list_bfiles found $f\n"); next if ($f =~ /^\./); if (-d "$dir/$f") { # Recurse for directories (Debian 3.0 format e.g.) pb_log(2,"DEBUG: pb_list_bfiles recurse in $dir/$f\n"); pb_list_bfiles("$dir/$f",$pbpkg,$bfiles,$pkgfiles,$supfiles,$f); next; } my $key = $f; # if recursive then store also the subdir $key = "$subdir/$f" if ($subdir ne ""); $bfiles->{$key} = "$dir/$f"; $bfiles->{$key} =~ s~$ENV{'PBROOTDIR'}~~; if (defined $supfiles->{$pbpkg}) { $pkgfiles->{$key} = "$dir/$f" if ($f =~ /$supfiles->{$pbpkg}/); } } closedir($bdir); pb_log(2,"DEBUG: exiting pb_list_bfiles: ".Dumper($bfiles)."\n"); } sub pb_add_comma { my $str = shift; my $addstr = shift; $str .= "," if (defined $str); $str .= $addstr; return($str); } sub pb_list_sfiles { my $sdir = shift; my $ptr = shift; my $pbos = shift; my $extdir = shift; pb_log(2,"DEBUG: entering pb_list_sfiles: ".Dumper($ptr)."\n"); my $key = "$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}"; # Prepare local sources for this distro - They are always applied first - May be a problem one day # This function works for both patches and additional sources foreach my $p (sort(<$sdir/*>)) { $ptr->{$key} = pb_add_comma($ptr->{$key},"file://$p") if (($p =~ /\.all$/) || ($p =~ /\.$pbos->{'os'}$/) || ($p =~ /\.$pbos->{'type'}$/) || ($p =~ /\.$pbos->{'family'}$/) || ($p =~ /\.$pbos->{'name'}$/) || ($p =~ /\.$pbos->{'name'}-$pbos->{'version'}$/) ||($p =~ /\.$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}$/)); } # Prepare also remote sources to be included - Applied after the local ones foreach my $p ("all","$pbos->{'os'}","$pbos->{'type'}","$pbos->{'family'}","$pbos->{'name'}","$pbos->{'name'}-$pbos->{'version'}","$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}") { my $f = "$extdir.".".$p"; next if (not -f $f); if (not open(PATCH,$f)) { pb_display("Unable to open existing external source file content $f\n"); next; } while () { chomp(); $ptr->{$key} = pb_add_comma($ptr->{$key},"$_"); } close(PATCH); } pb_log(2,"DEBUG: exiting pb_list_sfiles: ".Dumper($ptr)."\n"); return($ptr); } # # Return the list of packages we are working on in a non CMS action # sub pb_get_pkg { my @pkgs = (); my ($var) = pb_conf_read("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.pb","pbpkg"); @pkgs = keys %$var; pb_log(0,"Packages: ".join(',',@pkgs)."\n"); return(\@pkgs); } # Manages VM/RM SSH port communication sub pb_get_port { my $port = shift; my $pbos = shift; my $cmt = shift; my $nport; die "No port passed in parameter. Report to dev team\n" if (not defined $port); # key is project on VM, but machine tuple for RM if ($cmt =~ /^RM/i) { $nport = $port->{"$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}"}; } else { $nport = $port->{$ENV{'PBPROJ'}}; } pb_log(2,"pb_get_port with $nport\n"); # Maybe a port was given as parameter so overwrite $nport = "$pbport" if (defined $pbport); # Maybe in // mode so use the env var set up as an offset to the base port, except when called from send2target for Packages or for RM if (($cmt ne "Packages") && ($cmt !~ /^RM/i)) { $nport += $ENV{'PBVMPORT'} if ((defined $pbparallel) && (defined $ENV{'PBVMPORT'})); } pb_log(2,"pb_get_port returns $nport\n"); return($nport); } sub pb_set_port { my ($pid,$ident) = @_; pb_log(2,"pb_set_port for VM ($pid), id $ident\n"); $ENV{'PBVMPORT'} = $ident; pb_log(2,"pb_set_port sets PBVMPORT in env to $ENV{'PBVMPORT'}\n"); } sub pb_set_parallel { my $vtype = shift; pb_log(2,"pb_set_parallel vtype: $vtype\n"); # Take care of memory size if VM, parallel mode and more than 1 action if ((defined $pbparallel) && ($pbparallel ne 1) && ($vtype eq "vm")) { eval { require Linux::SysInfo; Linux::SysInfo->import(); }; if ($@) { # Linux::SysInfo not found pb_log(1,"ADVISE: Install Linux::SysInfo to benefit from automatic parallelism optimization.\nOr optimize manually pbparallel in your pb.conf file\nUsing $pbparallel processes max at a time for the moment\nWARNING: This may consume too much memory for your system\n"); } else { # Using the memory size my $si = Linux::SysInfo::sysinfo(); if (not defined $si) { pb_log(1,"ADVISE: Install Linux::SysInfo to benefit from automatic parallelism optimization.\nOr optimize manually pbparallel in your pb.conf file\nUsing $pbparallel processes max at a time for the moment\nWARNING: This may consume too much memory for your system\n"); } else { # Keep the number of VM whose memory can be allocated my $ram = $si->{"totalram"}-$si->{"sharedram"}-$si->{"bufferram"}; my $ram2; my ($vmmem) = pb_conf_get_if("vmmem"); my $v = "default"; if ((defined $vmmem) and (defined $vmmem->{$v})) { $ram2 = $vmmem->{$v}; } else { # Default for KVM/QEMU $ram2 = 128; } $pbparallel = sprintf("%d",$ram/$ram2); } pb_log(1,"Using $pbparallel processes at a time\n"); } } pb_log(2,"pb_set_parallel returns: $pbparallel\n") if (defined $pbparallel); return($pbparallel); } sub pb_get_sudocmds { my $pbos = shift; my %sudocmds; pb_log(2,"pb_get_sudocmds entering with lines:".Dumper(@_)."\n"); foreach my $c (split(/;/,$pbos->{'update'}),split(/;/,$pbos->{'install'}),@_) { pb_log(2,"pb_get_sudocmds analyses $c\n"); next if ($c !~ /^\s*sudo/); # remove sudo and leading spaces $c =~ s/^\s*sudo\s+//; # keep only the command, not the params $c =~ s/([^\s]+)\s.*$/$1/; $sudocmds{$c} = ""; } pb_log(2,"pb_get_sudocmds returns ".Dumper(keys %sudocmds)."\n"); return(keys %sudocmds); } sub pb_sign_pkgs { my $pbos = shift; my $made = shift; pb_log(2,"entering pb_sign_pkg: $made ".Dumper($pbos)."\n"); my ($passfile, $passphrase, $passpath) = pb_conf_get_if("pbpassfile","pbpassphrase","pbpasspath"); $ENV{'PBPASSPHRASE'} = $passphrase->{$ENV{'PBPROJ'}} if ((not defined $ENV{'PBPASSPHRASE'}) && (defined $passphrase->{$ENV{'PBPROJ'}})); $ENV{'PBPASSFILE'} = $passfile->{$ENV{'PBPROJ'}} if ((not defined $ENV{'PBPASSFILE'})&& (defined $passfile->{$ENV{'PBPROJ'}})) ; $ENV{'PBPASSPATH'} = $passpath->{$ENV{'PBPROJ'}} if ((not defined $ENV{'PBPASSPATH'})&& (defined $passpath->{$ENV{'PBPROJ'}})) ; # Remove extra spaces $made =~ s/\s+/ /g; $made =~ s/^\s//g; $made =~ s/\s$//g; if ($pbos->{'type'} eq "rpm") { eval { require RPM4::Sign; RPM4::Sign->import(); }; if ($@) { # RPM4::Sign not found pb_log(1,"WARNING: Install RPM4::Sign to benefit from automatic package signing.\n"); } else { return if ((not defined $ENV{'PBPASSPHRASE'}) and (not defined $ENV{'PBPASSFILE'})); my $sign = RPM4::Sign->new( passphrase => $ENV{'PBPASSPHRASE'}, name => $ENV{'PBPACKAGER'}, path => $ENV{'PBPASSPATH'}, password_file => $ENV{'PBPASSFILE'}, ); pb_log(0,"Signing RPM packages...\n"); pb_log(2,"pb_sign_pkg: pkgs:".Dumper(split(/ /,$made))."\n"); $sign->rpmssign(split(/ /,$made)); } } elsif ($pbos->{'type'} eq "deb") { my $changes = ""; foreach my $c (split(/ /,$made)) { $changes .= " $ENV{'PBBUILDDIR'}/$c" if (($c =~ /\.changes$/) && (-f "$ENV{PBBUILDDIR}/$c")); } my $debsigncmd = pb_check_req("debsign",1); pb_system("$debsigncmd -m\'$ENV{'PBPACKAGER'}\' $changes","Signing DEB packages",undef,1) if ($changes ne ""); } else { pb_log(0,"I don't know yet how to sign packages for type $pbos->{'type'}.\nPlease give feedback to dev team\n"); } pb_log(2,"exiting pb_sign_pkg\n"); } # return list of all distributins supported, comma separated sub pb_get_distros { my $pbos = shift; my $pbtarget = shift; my @dists = ("$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}"); # Get list of distributions for which we need to generate build files if no target if (not defined $pbtarget) { my @pt = pb_conf_get_if("vmlist","velist","rmlist"); foreach my $pt (@pt) { push(@dists, split(/,/, $pt->{$ENV{PBPROJ}})) if defined $pt->{$ENV{PBPROJ}}; } # remove any whitespace grep(s/\s+//go, @dists); } return(join(",",@dists)); } sub pb_get_extdir () { # the pbrc file should contain it and whatever the key, we take it my ($ed) = pb_conf_read("$ENV{'PBDESTDIR'}/pbrc","pbextdir"); pb_log(2,"ed: ".Dumper($ed)."\n"); my $pbextdir = ""; foreach my $k (keys %$ed) { $pbextdir = $ed->{$k}; # In case we have an empty field, empty it completely pb_log(2,"pbextdir: ***$pbextdir***\n"); $pbextdir =~ s/^\s*$//; } pb_log(2,"pbextdir: ***$pbextdir***\n"); return($pbextdir); } 1;