source: devel/pb/bin/pb @ 2333

Last change on this file since 2333 was 2333, checked in by bruno, 17 months ago

Initial support of apk for pb

  • Property svn:executable set to *
File size: 181.7 KB
Line 
1#!/usr/bin/perl -w
2#
3# Project Builder main application
4#
5# $Id$
6#
7# Copyright B. Cornec 2007-today
8# Eric Anderson's changes are (c) Copyright 2012 Hewlett Packard
9# Provided under the GPL v2
10
11# Syntax: see at end
12
13use strict 'vars';
14
15# The modules mentioned here are required by pb when used both
16# locally or inside a VE/VM/RM
17# Additional required modules only used locally are called with a require
18# in their respective section
19use Getopt::Long qw(:config auto_abbrev no_ignore_case);
20use Carp qw/confess cluck/;
21use Data::Dumper;
22use English;
23use File::Basename;
24use File::Copy;
25use File::stat;
26use File::Find;
27use Time::localtime qw(localtime);
28use POSIX qw(strftime);
29use lib qw (lib);
30use ProjectBuilder::Version;
31use ProjectBuilder::Base;
32use ProjectBuilder::Display;
33use ProjectBuilder::Conf;
34use ProjectBuilder::Distribution;
35use ProjectBuilder::VCS;
36use ProjectBuilder::CMS;
37use ProjectBuilder::Env;
38use ProjectBuilder::Filter;
39use ProjectBuilder::Changelog;
40use ProjectBuilder::VE;
41
42# Global variables
43$Global::pb_stop_on_error = 0;      # False by default
44$Global::pb_show_sudo = 0;          # False by default
45
46my %opts;                           # CLI Options
47my $action;                         # action to realize
48my $test = "FALSE";                 # Not used
49my $pbforce = 0;                    # Force VM stop
50my $pbsnap = 0;                     # Do not use snapshot mode for VM/VE by default
51my $pbkeep = 0;                     # keep temporary directory at the end
52my $option = "";                    # Not used
53my @pkgs;                           # list of packages
54my $pbtag;                          # Global Tag variable
55my $pbver;                          # Global Version variable
56my %pbscript;                       # Name of the scripts per V
57my $pbrev;                          # Global REVISION variable
58my $pbtarget = undef;               # Target os-ver-arch you want to build for
59my $pbport;                         # Port to use to connect to the VM/RM
60my $newver;                         # New version to create
61my $pbimage = undef;                # ISO image for the VM to create or docker image for the VE
62my $vetype = undef;                 # Type of Virtual environment to deal with
63
64my @date = pb_get_date();
65my $pbdate = strftime("%Y-%m-%d", @date);
66
67=pod
68
69=head1 NAME
70
71pb, aka project-builder.org - builds packages for your projects
72
73=head1 DESCRIPTION
74
75pb helps you build various packages directly from your project sources.
76Those sources could be handled by a CMS (Configuration Management System)
77such as Subversion, CVS, Git, Mercurial... or being a simple reference to a compressed tar file.
78It's based on a set of configuration files, a set of provided macros to help
79you keeping build files as generic as possible. For example, a single .spec
80file should be required to generate for all rpm based distributions, even
81if you could also have multiple .spec files if required.
82
83=head1 SYNOPSIS
84
85pb [-vhSq][-r pbroot][-p project][[-s script -a account -P port][-T VEtype][-t [os-ver-arch]][-m os-ver-arch[,...]]][-g][-i image] <action> [<pkg1> ...]
86
87pb [--verbose][--help][--man][--quiet][--snapshot][--revision pbroot][--project project][[--script script --account account --port port][-T VEtype][--target [os-ver-arch]][--machine os-ver-arch[,...]]][--nographic][--image image][--rebuild] <action> [<pkg1> ...]
88
89=head1 OPTIONS
90
91=over 4
92
93=item B<-v|--verbose>
94
95Increase verbosity
96
97=item B<-q|--quiet>
98
99Do not print any output.
100
101=item B<-h|--help>
102
103Print a brief help message and exits.
104
105=item B<-S|--snapshot>
106
107Use the snapshot mode of VMs or VEs
108
109=item B<--man>
110
111Prints the manual page and exits.
112
113=item B<-t|--target os-ver-arch>
114
115Name of the target system you want to build for.
116All if none precised.
117
118=item B<-m|--machine os-ver-arch[,os-ver-arch,...]>
119
120Name of the Virtual Machines (VM), Virtual Environments (VE) or Remote Machines (RM)
121you want to build on (comma separated).
122All if none precised (or use the env variable PBV).
123
124=item B<-T|--vetype VEtype]>
125
126Type of Virtual Environments (VE)
127Can be chroot or docker.
128
129=item B<-s|--script script>
130
131Name of the script you want to execute on the related VMs/VEs/RMs.
132
133=item B<-g|--nographic>
134
135Do not launch VMs in graphical mode.
136
137=item B<-i|--image image>
138
139It could be either:
140- The name of the ISO image of the distribution you want to install on the related VMs
141- The name of the docker image of the distribution you want to install on the related VEs
142
143
144=item B<-a|--account account>
145
146Name of the account to use to connect on the related VMs/RMs.
147
148=item B<-P|--port port_number>
149
150Port number to use to connect on the related VMs/RMs.";
151
152=item B<-p|--project project_name>
153
154Name of the project you're working on (or use the env variable PBPROJ)
155
156=item B<-r|--revision revision>
157
158Path Name of the project revision under the CMS (or use the env variable PBROOT)
159
160=item B<-V|--version new_version>
161
162New version of the project to create based on the current one.
163
164=item B<-k|--keep>
165
166Keep the temporary dir where files have been created in or der to help debug
167
168=item B<--rebuild>
169
170Only valid with the checkssh action, it alllows to automatically relaunch the build of the failed packages
171
172=item B<--no-stop-on-error>
173
174Continue through errors with best effort.
175
176=back
177
178=head1 ARGUMENTS
179
180<action> can be:
181
182=over 4
183
184=item B<sbx2build>
185
186Create tar files for the project under your CMS.
187Current state of the exported content is taken.
188CMS supported are SVN, SVK, CVS, Git and Mercurial
189parameters are packages to build
190if not using default list
191
192=item B<cms2build>
193
194Create tar files for the project under your CMS.
195Current state of the CMS is taken.
196CMS supported are SVN, SVK, CVS, Git and Mercurial
197parameters are packages to build
198if not using default list
199
200=item B<build2prep>
201
202Prepare the environment for build by installing required dependencies. Done once on the build system.
203
204=item B<build2pkg>
205
206Create packages for your running distribution
207
208=item B<cms2pkg>
209
210cms2build + build2pkg
211
212=item B<sbx2pkg>
213
214sbx2build + build2pkg
215
216=item B<sbx2pkg2ins>
217
218sbx2pkg + final install of packages
219
220=item B<sbx2prep>
221
222sbx2build + build2prep
223
224=item B<build2ssh>
225
226Send the tar files to a SSH host
227
228=item B<sbx2ssh>
229
230sbx2build + build2ssh
231
232=item B<cms2ssh>
233
234cms2build + build2ssh
235
236=item B<pkg2ssh>
237
238Send the packages built to a SSH host
239
240=item B<build2vm>
241
242Create packages in VMs, launching them if needed
243and send those packages to a SSH host once built
244VM type supported are QEMU and KVM
245
246=item B<build2ve>
247
248Create packages in VEs, creating it if needed
249and send those packages to a SSH host once built
250
251=item B<build2rm>
252
253Create packages in RMs, which should pre-exist,
254and send those packages to a SSH host once built
255RM means Remote Machine, and could be a physical or Virtual one.
256This is one buildfarm integration for pb.
257
258=item B<prepvm>
259
260Prepare the VMs to have all requirements to build the project
261
262=item B<prepve>
263
264Prepare the VEs to have all requirements to build the project
265
266=item B<preprm>
267
268Prepare the RMs to have all requirements to build the project
269
270=item B<sbx2vm>
271
272sbx2build + build2vm
273
274=item B<sbx2ve>
275
276sbx2build + build2ve
277
278=item B<sbx2docker>
279
280sbx2build + build2ve with a potential build of all necessary docker containers to perform it
281
282=item B<sbx2rm>
283
284sbx2build + build2rm
285
286=item B<cms2vm>
287
288cms2build + build2vm
289
290=item B<cms2ve>
291
292cms2build + build2ve
293
294=item B<cms2rm>
295
296cms2build + build2rm
297
298=item B<launchvm>
299
300Launch one virtual machine
301
302=item B<launchve>
303
304Launch one virtual environment
305
306=item B<script2vm>
307
308Launch one virtual machine if needed
309and executes a script on it
310
311=item B<script2ve>
312
313Execute a script in a virtual environment
314
315=item B<script2rm>
316
317Execute a script on a remote machine
318
319=item B<newvm>
320
321Create a new virtual machine
322
323=item B<newve>
324
325Create a new virtual environment
326
327=item B<setupvm>
328
329Setup a virtual machine for pb usage
330
331=item B<setupve>
332
333Setup a virtual environment for pb usage
334
335=item B<setuprm>
336
337Setup a remote machine for pb usage
338
339=item B<sbx2setupvm>
340
341Setup a virtual machine for pb usage using the sandbox version of pb instead of the latest stable
342Reserved to dev team.
343
344=item B<sbx2setupve>
345
346Setup a virtual environment for pb usage using the sandbox version of pb instead of the latest stable
347Reserved to dev team.
348
349=item B<sbx2setuprm>
350
351Setup a remote machine for pb usage using the sandbox version of pb instead of the latest stable
352Reserved to dev team.
353
354=item B<build2setupvm>
355
356Setup a virtual machine for pb usage using the build available
357Reserved to dev team.
358
359=item B<build2setupve>
360
361Setup a virtual environment for pb usage using the build available
362Reserved to dev team.
363
364=item B<build2setuprm>
365
366Setup a remote machine for pb usage using the build available
367Reserved to dev team.
368
369=item B<snapvm>
370
371Snapshot a virtual machine for pb usage
372
373=item B<snapve>
374
375Snapshot a virtual environment for pb usage
376
377=item B<updateconf>
378
379Update the configuration files to the latest format
380
381=item B<updatevm>
382
383Update the distribution in the virtual machine
384
385=item B<updateve>
386
387Update the distribution in the virtual environment
388
389=item B<updaterm>
390
391Update the distribution in the remote machine
392
393=item B<test2pkg>
394
395Test a package locally
396
397=item B<test2vm>
398
399Test a package in a virtual machine
400
401=item B<test2ve>
402
403Test a package in a virtual environment
404
405=item B<test2rm>
406
407Test a package in a remote machine
408
409=item B<checkssh>
410
411Check the delivery of the packages on the repository
412
413=item B<checkps>
414
415Check the process running the VM concerned
416
417=item B<newver>
418
419Create a new version of the project derived
420from the current one
421
422=item B<newproj>
423
424Create a new project and a template set of
425configuration files under pbconf
426
427=item B<announce>
428
429Announce the availability of the project through various means
430
431=item B<sbx2webssh>
432
433Create tar files for the website under your CMS.
434Current state of the exported content is taken.
435Deliver the content to the target server using ssh from the exported dir.
436
437=item B<cms2webssh>
438
439Create tar files for the website from your CMS.
440Deliver the content to the target server using ssh from the DVCS.
441
442=item B<sbx2webpkg>
443
444Create tar files for the website under your CMS.
445Current state of the exported content is taken.
446
447=item B<cms2webpkg>
448
449Create tar files for the website under your CMS.
450
451=item B<getconf>
452
453Print the full configuration parameters as found in the various configuration files. Help to debug conf issues.
454Also accepts a parameter to display only this value, and a VM/VE/RM
455
456=item B<getvar>
457
458Print the full variables expanded based on the distribution tuple. Help to debug conf issues.
459Also accepts a parameter to display only the values for this package, and a VM/VE/RM
460
461=item B<clean>
462
463Purge the build and delivery directories related to the current project
464
465=item B<cleanssh>
466
467Purge the ssh server of its packages (only for testver and test packages)
468
469=back
470
471<pkgs> can be a list of packages, the keyword 'all' or nothing, in which case the default list of packages is taken (corresponding to the defpkgdir list of arguments in the configuration file).
472
473=head1 WEB SITES
474
475The main Web site of the project is available at L<http://www.project-builder.org/>. Bug reports should be filled using the trac instance of the project at L<http://trac.project-builder.org/>.
476
477=head1 USER MAILING LIST
478
479None exists for the moment.
480
481=head1 CONFIGURATION FILES
482
483Each pb user may have a configuration in F<$HOME/.pbrc.yml>. The values in this file may overwrite any other configuration file value.
484
485Here is an example of such a configuration file:
486
487 #
488 # Define for each project the URL of its pbconf repository
489 # No default option allowed here as they need to be all different
490 #
491 # URL of the pbconf content
492 # This is the format of a classical URL with the extension of additional schema such as
493 # svn+ssh, cvs+ssh, ...
494 #
495  pbconfurl:
496    linuxcoe : cvs+ssh://:ext:bcornec@linuxcoe.cvs.sourceforge.net:/cvsroot/linuxcoe/pbconf
497
498 # This is normaly defined in the project's configuration file
499 # Url of the project
500 #
501  pbprojurl:
502    linuxcoe : cvs+ssh://:ext:bcornec@linuxcoe.cvs.sourceforge.net:/cvsroot/linuxcoe
503 
504 # All these URLs needs to be defined here as the are the entry point
505 # for how to build packages for the project
506 #
507  pbconfurl:
508    pb: svn+ssh://svn.project-builder.org/mondo/svn/pb/pbconf
509  pbconfurl:
510    mondorescue: svn+ssh://svn.project-builder.org/mondo/svn/project-builder/mondorescue/pbconf
511  pbconfurl:
512    collectl: svn+ssh://bruno@svn.mondorescue.org/mondo/svn/project-builder/collectl/pbconf
513  pbconfurl:
514    netperf: svn+ssh://svn.mondorescue.org/mondo/svn/project-builder/netperf/pbconf
515 
516 # Under that dir will take place everything related to pb
517 # If you want to use VMs/chroot/..., then use $ENV{'HOME'} to make it portable
518 # to your VMs/chroot/...
519 # if not defined then /var/cache
520  pbdefdir:
521    default: $ENV{'HOME'}/project-builder
522  pbdefdir:
523    pb: $ENV{'HOME'}
524  pbdefdir:
525    linuxcoe: $ENV{'HOME'}/LinuxCOE/cvs
526  pbdefdir:
527    mondorescue: $ENV{'HOME'}/mondo/svn
528 
529 # pbconfdir points to the directory where the CMS content of the pbconfurl is checked out
530 # If not defined, pbconfdir is under pbdefdir/pbproj/pbconf
531  pbconfdir:
532    linuxcoe: $ENV{'HOME'}/LinuxCOE/cvs/pbconf
533  pbconfdir:
534    mondorescue: $ENV{'HOME'}/mondo/svn/pbconf
535 
536 # pbdir points to the directory where the CMS content of the pbprojurl is checked out
537 # If not defined, pbdir is under pbdefdir/pbproj
538 # Only defined if we have access to the dev of the project
539  pbdir:
540    linuxcoe: $ENV{'HOME'}/LinuxCOE/cvs
541  pbdir:
542    mondorescue: $ENV{'HOME'}/mondo/svn
543 
544 # -daemonize doesn't work with qemu 0.8.2
545  vmopt:
546    default: -m 384
547
548=head1 COMMAND DETAILS
549
550=head2 newproj
551
552The newproj command creates a new project-builder project.  To run this command you first need to define two variables in your ~/.pbrc.yml file:
553
554 pbconfurl I<project> = file:///home/anderse/.git/project-builder-config/I<project>
555 pbdefdir default = $ENV{'HOME'}/cache-project-builder
556
557The first line defines the version controlled configuration information and the second defines the root directory for project-builder to use.
558
559You can then run the command:
560
561 % pb -p I<$project> -r I<$version> newproj I<$pkg>
562
563to create the new project. Running the newproj command will then generate the file $pbdefdir/$project/pbconf/$version/$project.yml, and the directory $pbdefdir/$project/pbconf/$version/$pkg. You will need to edit those files to make the later commands work.
564
565=head2 cms2build
566
567The cms2build command takes your files from the content management system and makes the two tar files that are necessary for building files. You need to have run the newproj command first.  Then there are several steps for running this command:
568
569=over 4
570
571=item Update your $project.yml configuration file.
572
573You need to set the pbprojurl, pbrepo, pbwf, pbpackager, projver, projtag, testver, delivery, and defpkgdir lines as described in the configuration file. The pbprojurl entry is used to find the source for your package. The pbrepo entry is used to build the .repo or .sources.list files for use by downloaders of the package. The pbwf entry indicates that the source tar file is named by I<package-name>-I<version>. The pbpackager entry will be stored in the packages and should be you or your team. The projver/projtag entries indicate the version of the software and the version of the packaging scripts. The testver entry when true indicates that the package is in a test version, so no log file is computed (can be long), and version is made up using a timstamp. The delivery entry gives the subdirectory under which the packages will be delivered on the repository, and the defpkgdir entry corresponds to the local subdirectory hosting the package content.
574
575For example:
576
577 pbprojurl Lintel = file:///home/anderse/projects/Lintel-0.2012.02.28.tar.gz
578 pbrepo Lintel = http://tesla.hpl.hp.com/opensource
579 pbwf Lintel = 1
580 pbpackager Lintel = Eric Anderson <eric.anderson4@hp.com>
581 projver Lintel = 0.2012.02.28
582 projtag Lintel = 1
583 testver Lintel = false
584 delivery Lintel = production
585 defpkgdir Lintel = Lintel-0.2012.02.28
586
587=item Create the build .tar.gz files:
588
589Then you need to take those files and create the initial tar files.  Run a command like:
590
591 % pb -p $project -r $version cms2build
592
593To create the $pbdefdir/$project/pbdelivery/$project-$version.{,pbconf}.tar.gz files, the $version-$projtag.yml and pbrc.yml files in the same directory.
594
595=back
596
597=head2 build2pkg
598
599The build2pkg command takes the tar files created in the cms2build step and attempts to build binary packages for your current operating system. There are two steps:
600
601=over 4
602
603=item Update your filters and build files.
604
605You probably need to edit the files describing the build steps in one of the $pbdefdir/$project/pbconf/$version/$project/{deb,rpm,pkg} directories and the filters in $pbdefdir/$project/pbconf/$version/pbfilter. Note that you can define additional filters and transformations in the filters files. The build files will be filtered by the filters defined in the filters files to generate the inputs to the build step. Therefore, if you change those files, you need to re-run the cms2build step.
606
607=item Build the package.
608
609Then you can run a command like:
610
611 % pb -p $project -r $version build2pkg
612
613To create the files in $project/pbbuild that comprise your binary package(s).
614
615=back
616
617=head2 newve
618 
619The newve command creates a new virtual environment, i.e. a chrooted OS for building packages. Using a virtual environment is an efficient way to build packages on a related set of operating systems. The OS's have to be related because the kernel will be shared.  Steps:
620
621=over 4
622
623=item Update ~/.pbrc.yml
624
625Update your ~/.pbrc.yml file to specify the vepath, velist, velogin, and vetype variables, e.g.:
626
627  vepath:
628    default: $ENV{'HOME'}/cache-project-builder/chroot
629  velist:
630    default: debian-6.0-i386
631  velogin:
632    default: pb
633  vetype:
634    default: chroot
635
636If you are building for rpm style OS's, update the verpmtype option, and install the appropriate tool.
637
638  verpmtype:
639    default: rpmbootstrap
640
641You may also choose to specify a mirror for the OS packages, and optionally http/ftp proxies.  You can specify the proxies either through environment variables ($http_proxy/$https_proxy/$ftp_proxy) or in the configuration file. The configuration file will be used if no corresponding environment variable has been set. For example, for debian and with a local squid proxy:
642
643  rbsmirrorsrv:
644    debian: http://mirrors1.kernel.org/debian/
645  http_proxy:
646    default: http://localhost:3128/
647  https_proxy:
648    default: http://localhost:3128/
649  ftp_proxy:
650    default: http://localhost:3128/
651
652=item Run the cms2build command
653
654If you have deleted your $package/pbdelivery directory, re-run the cms2build command as in the earlier step. This step is necessary to generate the I<package>/pbdelivery/pbrc.yml file.
655
656=item Create the new virtual environment
657
658Initialize the new operating system. This step will install the core OS packages for the virtual environment, e.g.:
659
660 % pb -v -p $project -m debian-6.0-i386 newve
661
662=back
663
664=head2 setupve
665
666The setupve command prepares a virtual environment for use by project builder. In particular it installs project-builder from the packages into the virtual environment. Two sub-steps are necessary:
667
668=over 4
669
670=item Update $project.yml
671
672You need to have a sshhost entry for setupve to work, so add one, even an invalid one, e.g.:
673
674  sshhost:
675    $project: foo.example.org
676
677=item Setup the virtual environment
678
679 % pb -v -p $project -m debian-6.0-i386 setupve
680
681If you prefer to install the current SVN version of project builder, you can substitute the setupve option by the sbx2setupv one.
682
683=back
684
685=head2 build2ve
686
687The build2ve command is similar to the build2pkg command in that it will take the sources created by cms2build and turn them into binary packages. The command has two differences. First, it creates the packages in a virtual environment, i.e. the one made by an earlier setupve setup. Second it copies the resulting packages to a repository and builds the repository meta-data needed.
688
689Three sub-steps are needed:
690
691=over 4
692
693=item Update $project.yml
694
695You need to have a valid sshdir and sshhost entry for build2ve to work, so add them. Note that you need to be able to ssh from the host you run the command on to the repository host, preferably without needing to type in a password, so using ssh-agent or having a special passwordless project-builder ssh key will make this step easier.
696
697  sshhost:
698    $project: localhost
699  sshdir:
700    $project: $home/cache-project-builder/repos
701
702You may also need to specify additional repository files to use or rpms to install. Note the URL for repositories is not the URL of the repository, but the URL of a file that can be put in the yum.repos.d or apt.sources.d directory.
703
704  addrepo:
705    centos-5-i386: http://localhost/pb/centos-extras.repo,http://mirror.centos.org/centos/5/extras/i386/RPMS/chrpath-0.13-3.el5.centos.i386.rpm
706
707=item Update your filters and build files
708
709You may need to update your filter files as in the build2pkg step if you are building for a new OS or architecture.
710
711=item Build the packages and copy them to the repository
712
713 % pb -v -p $project -m debian-6.0-i386 build2ve
714
715=back
716
717*Debugging:* If the build fails (and you did not specify the --no-stop-on-error) option, then the virtual environment and scripts should still be present and configured to build the package. You can run a command like 'sudo setarch i386 chroot $path bash' in order to get into the environment. In your log you should see a command like that. From there you can go into the /home/pb directory as the pb user and run the same style of pb commands as you did when doing build2pkg. This will help you figure out what has gone wrong in the build in the virtual environment.
718
719=head1 AUTHORS
720
721The Project-Builder.org team L<http://trac.project-builder.org/> lead by Bruno Cornec L<mailto:bruno@project-builder.org>.
722
723=head1 COPYRIGHT
724
725Project-Builder.org is distributed under the GPL v2.0 license
726described in the file C<COPYING> included with the distribution.
727
728=cut
729
730# ---------------------------------------------------------------------------
731
732my ($projectbuilderver,$projectbuilderrev,$projectbuilderconfver) = pb_version_init();
733my $appname = "pb";
734
735# Initialize the syntax string
736
737pb_syntax_init("$appname (aka project-builder.org) Version $projectbuilderver-$projectbuilderrev Conf $projectbuilderconfver\n");
738
739GetOptions("help|?|h+" => \$opts{'h'}, 
740        "account|a=s" => \$opts{'a'},
741        "force|f" => \$opts{'f'},
742        "nographic|g" => \$opts{'g'},
743        "image|i=s" => \$opts{'i'},
744        "log-files|l=s" => \$opts{'l'},
745        "machines|mock|m=s" => \$opts{'m'},
746        "man" => \$opts{'man'},
747        "port|P=i" => \$opts{'P'},
748        "project|p=s" => \$opts{'p'},
749        "quiet|q" => \$opts{'q'},
750        "revision|r=s" => \$opts{'r'},
751        "rebuild" => \$opts{'rebuild'},
752        "snapshot|S" => \$opts{'S'},
753        "script|s=s" => \$opts{'s'},
754        "type|T:s" => \$opts{'T'},
755        "target|t:s" => \$opts{'t'},
756        "version" => \$opts{'version'},
757        "V=s" => \$opts{'V'},
758        "verbose|v+" => \$opts{'v'},
759        "keep|k" => \$opts{'k'},
760        "stop-on-error!" => \$Global::pb_stop_on_error,
761) || pb_syntax(-1,0);
762
763if (defined $opts{'l'}) {
764    open(pbLOG,"> $opts{'l'}") || die "Unable to log to $opts{'l'}: $!";
765    $pbLOG = \*pbLOG;
766    $pbdebug = 0  if ($pbdebug == -1);
767    }
768pb_log_init($pbdebug, $pbLOG);
769
770if (defined $opts{'h'}) {
771    pb_syntax(0,1);
772}
773if (defined $opts{'version'}) {
774    pb_syntax(0,0);
775}
776if (defined $opts{'man'}) {
777    pb_syntax(0,2);
778}
779if (defined $opts{'v'}) {
780    $pbdebug = $opts{'v'};
781}
782if (defined $opts{'f'}) {
783    $pbforce=1;
784}
785if (defined $opts{'q'}) {
786    $pbdebug=-1;
787}
788if (defined $opts{'S'}) {
789    $pbsnap=1;
790}
791if (defined $opts{'k'}) {
792    $pbkeep=1;
793}
794pb_display_init("text","");
795
796# Handle root of the project if defined
797if (defined $opts{'r'}) {
798    $ENV{'PBROOTDIR'} = $opts{'r'};
799}
800# Handle virtual machines if any
801if (defined $opts{'m'}) {
802    $ENV{'PBV'} = $opts{'m'};
803}
804if (defined $opts{'s'}) {
805    $pbscript{'default'} = $opts{'s'};
806    die "option -s requires a script having a full path name" if ($pbscript{'default'} !~ /^\//);
807}
808if (defined $opts{'a'}) {
809    $ENV{'PBACCOUNT'} = $opts{'a'};
810    die "option -a requires a -s script option" if (not defined $pbscript{'default'});
811}
812if (defined $opts{'P'}) {
813    $pbport = $opts{'P'};
814}
815if (defined $opts{'V'}) {
816    $newver = $opts{'V'};
817}
818if (defined $opts{'i'}) {
819    $pbimage = $opts{'i'};
820}
821if (defined $opts{'t'}) {
822    $pbtarget = $opts{'t'};
823}
824if (defined $opts{'T'}) {
825    $vetype = $opts{'T'};
826}
827
828# Get Action
829$action = shift @ARGV;
830die pb_syntax(-1,1) if (not defined $action);
831
832my ($filteredfiles, $supfiles, $defpkgdir, $extpkgdir);
833my $pbinit = undef;
834$pbinit = 1 if ($action =~ /^newproj$/);
835
836# Handles project name if any
837# And get global params
838($filteredfiles, $supfiles, $defpkgdir, $extpkgdir) = pb_env_init($opts{'p'},$pbinit,$action,$pbkeep);
839
840#
841# Check for command requirements
842#
843my ($req,$opt,$pbpara,$pbshowsudo,$pbstoponerr) = pb_conf_get_if("oscmd","oscmdopt","pbparallel","pbshowsudo","pbstoponerr");
844pb_check_requirements($req,$opt,$appname);
845
846$Global::pb_show_sudo = 1 if ((defined $pbshowsudo) && (defined $pbshowsudo->{$ENV{'PBPROJ'}}) && ($pbshowsudo->{$ENV{'PBPROJ'}} =~ /true/oi));
847$Global::pb_stop_on_error = 1 if ((defined $pbstoponerr) && (defined $pbstoponerr->{$ENV{'PBPROJ'}}) && ($pbstoponerr->{$ENV{'PBPROJ'}} =~ /true/oi));
848
849#
850# Check if we can launch some actions in // with Parallel::ForkManager
851#
852my $pbparallel = $pbpara->{$appname} if (defined $pbpara);
853if (not defined $pbparallel) {
854    eval
855    {
856        require Sys::CPU;
857        Sys::CPU->import();
858    };
859    if ($@) {
860        # Sys::CPU not found, defaulting to 1
861        pb_log(1,"ADVISE: Install Sys::CPU to benefit from automatic parallelism optimization.\nOr use pbparallel in your pb.yml file\nOnly 1 process at a time for the moment\n");
862        $pbparallel = 1;
863    } else {
864        # Using the number of cores
865        $pbparallel = Sys::CPU::cpu_count();
866        pb_log(1,"Using parallel mode with $pbparallel processes\n");
867    }
868}
869
870eval
871{
872    require Parallel::ForkManager;
873    Parallel::ForkManager->import();
874};
875# Parallel::ForkManager not found so no // actions
876if ($@) {
877    $pbparallel = undef;
878    pb_log(1,"ADVISE: Install Parallel::ForkManager to benefit from automatic parallelism optimization.\nOnly 1 process at a time for the moment\n");
879}
880
881pb_log(0,"Project: $ENV{'PBPROJ'}\n");
882pb_log(0,"Action: $action\n");
883
884my $do_install = undef;
885# Possibility to add a 2ins to some commands to launch automatic installation
886$do_install = 1 if ($action =~ /2ins$/);
887
888# Act depending on action
889if ($action =~ /^cms2build$/) {
890    pb_cms2build("CMS");
891} elsif ($action =~ /^sbx2build$/) {
892    pb_cms2build("SandBox");
893} elsif ($action =~ /^build2prep/) {
894    pb_build2prep();
895} elsif ($action =~ /^sbx2prep/) {
896    pb_cms2build("SandBox");
897    pb_build2prep();
898} elsif ($action =~ /^build2pkg/) {
899    pb_build2pkg($do_install);
900} elsif ($action =~ /^cms2pkg/) {
901    pb_cms2build("CMS");
902    pb_build2pkg($do_install);
903} elsif ($action =~ /^sbx2pkg/) {
904    pb_cms2build("SandBox");
905    pb_build2pkg($do_install);
906} elsif ($action =~ /^build2ssh$/) {
907    pb_build2ssh();
908} elsif ($action =~ /^cms2ssh$/) {
909    pb_cms2build("CMS");
910    pb_build2ssh();
911} elsif ($action =~ /^sbx2ssh$/) {
912    pb_cms2build("SandBox");
913    pb_build2ssh();
914} elsif ($action =~ /^pkg2ssh$/) {
915    pb_pkg2ssh();
916} elsif ($action =~ /^build2rm$/) {
917    pb_build2v("rm","build");
918} elsif ($action =~ /^build2ve$/) {
919    pb_build2v("ve","build");
920} elsif ($action =~ /^build2vm$/) {
921    pb_build2v("vm","build");
922} elsif ($action =~ /^sbx2preprm$/) {
923    pb_cms2build("SandBox");
924    pb_build2v("rm","prep");
925} elsif ($action =~ /^sbx2prepve$/) {
926    pb_cms2build("SandBox");
927    pb_build2v("ve","prep");
928} elsif ($action =~ /^sbx2prepvm$/) {
929    pb_cms2build("SandBox");
930    pb_build2v("vm","prep");
931} elsif ($action =~ /^preprm$/) {
932    pb_build2v("rm","prep");
933} elsif ($action =~ /^prepve$/) {
934    pb_build2v("ve","prep");
935} elsif ($action =~ /^prepvm$/) {
936    pb_build2v("vm","prep");
937} elsif ($action =~ /^cms2rm$/) {
938    pb_cms2build("CMS");
939    pb_build2v("rm","build");
940} elsif ($action =~ /^cms2ve$/) {
941    pb_cms2build("CMS");
942    pb_build2v("ve","build");
943} elsif ($action =~ /^sbx2rm$/) {
944    pb_cms2build("SandBox");
945    pb_build2v("rm","build");
946} elsif ($action =~ /^sbx2ve$/) {
947    pb_cms2build("SandBox");
948    pb_build2v("ve","build");
949} elsif ($action =~ /^cms2vm$/) {
950    pb_cms2build("CMS");
951    pb_build2v("vm","build");
952} elsif ($action =~ /^sbx2vm$/) {
953    pb_cms2build("SandBox");
954    pb_build2v("vm","build");
955} elsif ($action =~ /^launchvm$/) {
956    pb_parallel_launchv(undef,"vm",undef,3,$pbimage);
957} elsif ($action =~ /^launchve$/) {
958    pb_parallel_launchv(undef,"ve",undef,3,$pbimage);
959} elsif ($action =~ /^script2vm$/) {
960    die "action script2vm requires a -s script option" if (not defined $pbscript{'default'});
961    pb_script2v(\%pbscript,"vm");
962} elsif ($action =~ /^script2ve$/) {
963    die "action script2ve requires a -s script option" if (not defined $pbscript{'default'});
964    pb_script2v(\%pbscript,"ve");
965} elsif ($action =~ /^script2rm$/) {
966    die "action script2rm requires a -s script option" if (not defined $pbscript{'default'});
967    pb_script2v(\%pbscript,"rm");
968} elsif ($action =~ /^newver$/) {
969    pb_newver();
970} elsif ($action =~ /^sbx2docker$/) {
971    my $savproj = $ENV{'PBPROJ'};
972    my $pkg;
973    my @pkg;
974    if ($#ARGV == -1) {
975        $pkg = pb_cms_get_pkg($defpkgdir,$extpkgdir);
976        @pkgs = @$pkg;
977    } else {
978        @pkgs = @ARGV;
979    }
980    #pb_log(1,"Packages to print:".Dumper(@pkgs)."\n");
981   
982    # Get the list of all VE we need to work on
983    my ($vm,$all);
984    if (not defined $ENV{'PBV'}) {
985        ($vm,$all) = pb_get2v("ve");
986    } else {
987        @$vm = split(/,/,$ENV{'PBV'});
988    }
989   
990    # TODO: parallelize
991    foreach my $v (@$vm) {
992        my $done = 0;
993        my $pbos = pb_distro_get_context($v);
994        # Docker images are (for now !) regular and use distro:ver
995        $pbimage = "$pbos->{'name'}".":$pbos->{'version'}";
996        # the first steps needs to be done with the pb project
997        $ENV{'PBPROJ'} = $appname;
998        my $found = pb_ve_docker_get_image("$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}");
999        if (($pbforce == 1) || (not defined $found)) {
1000            # pb -p pb -m distro-ver-arch newve -i distro:ver
1001            pb_log(0,"Docker container $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'} not found, creating it\n");
1002            pb_parallel_launchv(undef,"ve",undef,0,$pbimage);
1003        }
1004        $found = pb_ve_docker_get_image("$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-pb");
1005        if (($pbforce == 1) || (not defined $found)) {
1006            pb_log(0,"Docker container $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-pb not found, creating it\n");
1007            # pb -p pb -m distro-ver-arch setupve
1008            if ($savproj eq "pb") {
1009                pb_cms2build("SandBox");
1010                $done = 1;
1011                pb_setup2v("ve","SandBox");
1012            } else {
1013                pb_setup2v("ve");
1014            }
1015        }
1016        # Now back to our original project
1017        $ENV{'PBPROJ'} = $savproj;
1018        $found = pb_ve_docker_get_image("$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-pb-$ENV{'PBPROJ'}");
1019        if (($pbforce == 1) || (not defined $found)) {
1020            pb_log(0,"Docker container $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-pb-$ENV{'PBPROJ'} not found, creating it\n");
1021            # pb -p pb -m distro-ver-arch prepve
1022            if ($savproj eq "pb") {
1023                pb_cms2build("SandBox");
1024                $done = 1;
1025            }
1026            pb_build2v("ve","prep");
1027        }
1028        pb_log(0,"Instantiating a Docker container from $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-pb-$ENV{'PBPROJ'}\n");
1029        if (($savproj ne "pb") || ($done == 0)) {
1030            pb_cms2build("SandBox");
1031        }
1032        pb_build2v("ve","build");
1033    }
1034} elsif ($action =~ /^newve$/) {
1035    pb_parallel_launchv(undef,"ve",undef,0,$pbimage);
1036} elsif ($action =~ /^newvm$/) {
1037    pb_parallel_launchv(undef,"vm",undef,0,$pbimage);
1038    pb_log(0, "Please ensure that sshd is running in your VM by default\n");
1039    pb_log(0, "and that it allows remote root login (PermitRootLogin yes in /etc/ssh/sshd_config)\n");
1040    pb_log(0, "Also ensure that network is up, firewalling correctly configured\n");
1041    pb_log(0, "and perl, Data::Dumper, tar, sudo, ntpdate and scp/ssh installed\n");
1042    pb_log(0, "You should then be able to login with ssh -p VMPORT root\@localhost (if VM started with pb)\n");
1043} elsif ($action =~ /^setuprm$/) {
1044    pb_setup2v("rm");
1045} elsif ($action =~ /^setupve$/) {
1046    pb_setup2v("ve");
1047} elsif ($action =~ /^setupvm$/) {
1048    pb_setup2v("vm");
1049} elsif ($action =~ /^sbx2setuprm$/) {
1050    die "This feature is limited to the pb project" if ($ENV{'PBPROJ'} ne $appname);
1051    pb_cms2build("SandBox");
1052    pb_setup2v("rm","SandBox");
1053} elsif ($action =~ /^sbx2setupve$/) {
1054    die "This feature is limited to the pb project" if ($ENV{'PBPROJ'} ne $appname);
1055    pb_cms2build("SandBox");
1056    pb_setup2v("ve","SandBox");
1057} elsif ($action =~ /^sbx2setupvm$/) {
1058    die "This feature is limited to the pb project" if ($ENV{'PBPROJ'} ne $appname);
1059    pb_cms2build("SandBox");
1060    pb_setup2v("vm","SandBox");
1061} elsif ($action =~ /^build2setuprm$/) {
1062    die "This feature is limited to the pb project" if ($ENV{'PBPROJ'} ne $appname);
1063    pb_setup2v("rm","SandBox");
1064} elsif ($action =~ /^build2setupve$/) {
1065    die "This feature is limited to the pb project" if ($ENV{'PBPROJ'} ne $appname);
1066    pb_setup2v("ve","SandBox");
1067} elsif ($action =~ /^build2setupvm$/) {
1068    die "This feature is limited to the pb project" if ($ENV{'PBPROJ'} ne $appname);
1069    pb_setup2v("vm","SandBox");
1070} elsif ($action =~ /^updaterm$/) {
1071    pb_update2v("rm");
1072} elsif ($action =~ /^updateve$/) {
1073    pb_update2v("ve");
1074} elsif ($action =~ /^updatevm$/) {
1075    pb_update2v("vm");
1076} elsif ($action =~ /^snapve$/) {
1077    pb_snap2v("ve");
1078} elsif ($action =~ /^snapvm$/) {
1079    pb_snap2v("vm");
1080} elsif ($action =~ /^test2pkg$/) {
1081    pb_test2pkg();
1082} elsif ($action =~ /^test2rm$/) {
1083    pb_build2v("rm","test");
1084} elsif ($action =~ /^test2ve$/) {
1085    pb_build2v("ve","test");
1086} elsif ($action =~ /^test2vm$/) {
1087    pb_build2v("vm","test");
1088} elsif ($action =~ /^newproj$/) {
1089    # Nothing to do - already done in pb_env_init
1090} elsif ($action =~ /^updateconf$/) {
1091    foreach my $f (@ARGV) {
1092        my $fyml = $f;
1093        $fyml =~ s/\.pbf/.yml/;
1094        $fyml =~ s/\.pb/.yml/;
1095        $fyml =~ s/\.pbrc/.pbrc.yml/;
1096        pb_vcs_conf_update_v0($f,$fyml);
1097    }
1098} elsif ($action =~ /^getconf$/) {
1099    my $pbos = pb_distro_get_context($ENV{'PBV'});
1100    pb_log(1,"Arguments to print:".Dumper(@ARGV)."\n");
1101    pb_distro_conf_print($pbos,@ARGV);
1102} elsif ($action =~ /^getvar$/) {
1103    my $pbos = pb_distro_get_context($ENV{'PBV'});
1104    my $pkg;
1105    my @pkg;
1106    if ($#ARGV == -1) {
1107        $pkg = pb_cms_get_pkg($defpkgdir,$extpkgdir);
1108        @pkgs = @$pkg;
1109    } else {
1110        @pkgs = @ARGV;
1111    }
1112    pb_log(1,"Packages to print:".Dumper(@pkgs)."\n");
1113    pb_filter_var_print($pbos,@pkgs);
1114} elsif ($action =~ /^clean$/) {
1115    pb_clean();
1116} elsif ($action =~ /^announce$/) {
1117    # For announce only. Require avoids the systematic load of these modules
1118    require DBI;
1119    require DBD::SQLite;
1120
1121    pb_announce("Announce");
1122} elsif ($action =~ /^cleanssh$/) {
1123    pb_announce("Clean");
1124} elsif ($action =~ /^checkssh$/) {
1125    pb_announce("Check");
1126} elsif ($action =~ /^checkps$/) {
1127    my ($ptr2) = pb_conf_get("vmcmd");
1128    my $vmcmd = $ptr2->{$ENV{'PBPROJ'}};
1129    my ($vexist,$vmmport) = pb_check_ps($vmcmd,$ENV{'PBV'});
1130    if (! $vexist) {
1131        pb_log(0,"No VM found for $ENV{'PBV'}\n");
1132    } else {
1133        pb_log(0,"Found an existing VM for $ENV{'PBV'} PID: $vexist - SSH port: $vmmport\n");
1134    }
1135} elsif ($action =~ /^sbx2webpkg$/) {
1136    require DBI;
1137    require DBD::SQLite;
1138
1139    pb_cms2build("SandBox","Web");
1140} elsif ($action =~ /^sbx2webssh$/) {
1141    require DBI;
1142    require DBD::SQLite;
1143
1144    pb_cms2build("SandBox","Web");
1145    pb_send2target("Web");
1146} elsif ($action =~ /^cms2webpkg$/) {
1147    require DBI;
1148    require DBD::SQLite;
1149
1150    pb_cms2build("CMS","Web");
1151} elsif ($action =~ /^cms2webssh$/) {
1152    require DBI;
1153    require DBD::SQLite;
1154
1155    pb_cms2build("CMS","Web");
1156    pb_send2target("Web");
1157} else {
1158    pb_log(0,"\'$action\' is not available\n");
1159    pb_syntax(-2,1);
1160}
1161
1162pb_exit();
1163
1164sub pb_cms2build {
1165
1166    my $param = shift;
1167    my $web = shift;
1168
1169    my $pkg;
1170    my @pkgs;
1171    my $webdir;
1172
1173    my %pkgs;
1174    my $pb;             # Structure to store conf info
1175
1176    die "pb_cms2build requires a parameter: SandBox or CMS" if (not defined $param);
1177
1178    # If Website, then pkg is only the website
1179    if (defined $web) {
1180        ($webdir) = pb_conf_get("webdir");
1181        pb_log(2,"webdir: ".Dumper($webdir)."\n");
1182        $pkgs[0] = $webdir->{$ENV{'PBPROJ'}};
1183        $extpkgdir = $webdir;
1184        pb_log(0,"Package: $pkgs[0]\n");
1185    } else {
1186        $pkg = pb_cms_get_pkg($defpkgdir,$extpkgdir);
1187        @pkgs = @$pkg;
1188    }
1189
1190    my ($scheme, $uri) = pb_vcs_init($pbinit,$param);
1191
1192    # We need 2 lines here
1193    my ($pkgv, $pkgt, $testver) = pb_conf_get_if("pkgver","pkgtag","testver");
1194
1195    # declare packager and repo for filtering
1196    my ($tmp1, $tmp2) = pb_conf_get("pbpackager","pbrepo");
1197    $ENV{'PBPACKAGER'} = $tmp1->{$ENV{'PBPROJ'}};
1198    $ENV{'PBREPO'} = $tmp2->{$ENV{'PBPROJ'}};
1199    my ($delivery) = pb_conf_get_if("delivery");
1200    $delivery->{$ENV{'PBPROJ'}} = "" if (not defined $delivery->{$ENV{'PBPROJ'}});
1201
1202    # If we deal with a test dir, we want to keep the date in tar file and dir name
1203    my $pbextdirpkg = "";
1204    my $pbextdir = "";
1205    if ((defined $testver) && (defined $testver->{$ENV{'PBPROJ'}}) && ($testver->{$ENV{'PBPROJ'}} =~ /true/i)) {
1206        $pbextdirpkg = strftime("%Y%m%d%H%M%S", @date);
1207        $pbextdir = ".0.".$pbextdirpkg;
1208    }
1209
1210    foreach my $pbpkg (@pkgs) {
1211        $ENV{'PBPKG'} = $pbpkg;
1212        $pbrev = $ENV{'PBREVISION'};
1213
1214        if ((defined $pkgv) && (defined $pkgv->{$pbpkg})) {
1215            $pbver = $pkgv->{$pbpkg};
1216        } else {
1217            $pbver = $ENV{'PBPROJVER'};
1218        }
1219        # If it's a test version, then build a tag to fit distributions need
1220        if ((defined $testver) && (defined $testver->{$ENV{'PBPROJ'}}) && ($testver->{$ENV{'PBPROJ'}} =~ /true/i)) {
1221            my $letter = "";
1222            if ($scheme =~ /svn/) {
1223                $letter = "s";
1224            } elsif ($scheme =~ /cvs/) {
1225                $letter = "c";
1226            } elsif ($scheme =~ /svk/) {
1227                $letter = "k";
1228            } elsif ($scheme =~ /hg/) {
1229                $letter = "h";
1230            } elsif ($scheme =~ /git/) {
1231                $letter = "g";
1232            } else {
1233                $letter = "f";
1234            }
1235            $pbtag = "0.$pbextdirpkg.$letter$pbrev";
1236            $ENV{'PBPROJTAG'} = $pbtag;
1237        } elsif ((defined $pkgt) && (defined $pkgt->{$pbpkg})) {
1238            $pbtag = $pkgt->{$pbpkg};
1239        } else {
1240            $pbtag = $ENV{'PBPROJTAG'};
1241        }
1242
1243        pb_log(0,"\n");
1244        pb_log(0,"Management of $pbpkg $pbver-$pbtag\n");
1245        die "Unable to get env var PBDESTDIR" if (not defined $ENV{'PBDESTDIR'});
1246
1247        my $dest = "$ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir";
1248        # Create the structure if needed
1249        pb_mkdir_p($dest);
1250        # Clean up dest if necessary. The export will recreate it
1251        pb_rm_rf($dest) if (-d $dest);
1252
1253        # Export CMS tree for the concerned package to dest
1254        # And generate some additional files
1255        $OUTPUT_AUTOFLUSH=1;
1256
1257        # computes in which dir we have to work
1258        my $dir = $defpkgdir->{$pbpkg};
1259        $dir = $extpkgdir->{$pbpkg} if (not defined $dir);
1260        $dir = $webdir->{$ENV{'PBPROJ'}} if (defined $web);
1261        die "Variable \$dir not defined. Check the package name first.\nIf this is the one expected, please report to dev team with log of a verbose run and this info ".Dumper($defpkgdir,$extpkgdir,$webdir) if (not defined $dir);
1262        pb_log(2,"def:".Dumper($defpkgdir)."ext: ".Dumper($extpkgdir)."dir: $dir\n");
1263
1264        # Exporting content from CMS
1265        my $sourcedir = undef;
1266        my $sourceuri = $uri;
1267        if ($param eq "SandBox") {
1268            # Point to the local instance
1269            $sourcedir = "$ENV{'PBDIR'}/$dir";
1270        } else {
1271            # Get it from a subdir of the URI with same version as localy but different root
1272            # Only if using a real CMS
1273            my ($scheme, $account, $host, $port, $path) = pb_get_uri($uri);
1274            if (($scheme !~ /^file/) && ($scheme !~ /^(ht|f)tp/)) {
1275                $sourceuri = "$ENV{'PBDIR'}/$dir"; 
1276                $sourceuri =~ s|^$ENV{'PBPROJDIR'}/|$uri/|;
1277            }
1278        }
1279        my $preserve = pb_vcs_export($sourceuri,$sourcedir,$dest);
1280
1281        # Generated fake content for test versions to speed up stuff
1282        my $chglog;
1283
1284        # Get project info on authors and log file
1285        # TODO: Make it CMS aware
1286        $chglog = "$ENV{'PBROOTDIR'}/$pbpkg/pbcl";
1287        $chglog = "$ENV{'PBROOTDIR'}/pbcl" if (! -f $chglog);
1288        $chglog = undef if (! -f $chglog);
1289
1290        # TODO: Make it CMS aware
1291        my $authors = "$ENV{'PBROOTDIR'}/$pbpkg/pbauthors";
1292        $authors = "$ENV{'PBROOTDIR'}/pbauthors" if (! -f $authors);
1293        $authors = "/dev/null" if (! -f $authors);
1294
1295        # Extract cms log history and store it
1296        if ((defined $chglog) && (! -f "$dest/NEWS")) {
1297            pb_log(2,"Generating NEWS file from $chglog\n");
1298            copy($chglog,"$dest/NEWS") || die "Unable to create $dest/NEWS";
1299        }
1300        pb_cms_log($scheme,"$ENV{'PBDIR'}/$dir",$dest,$chglog,$authors,$testver);
1301
1302        my %build;
1303        # We want to at least build for the underlying distro
1304        # except if a target was given, in which case we only build for it
1305        # if -t was passed without target then build for the native distro.
1306        my $pbos = pb_distro_get_context($pbtarget);
1307        my $tmpl = pb_get_distros($pbos,$pbtarget);
1308
1309        # Setup $pb structure to allow filtering later on, on files using that structure
1310        $pb->{'tag'} = $pbtag;
1311        $pb->{'rev'} = $pbrev;
1312        $pb->{'ver'} = $pbver;
1313        $pb->{'pkg'} = $pbpkg;
1314        $pb->{'suf'} = $pbos->{'suffix'};
1315        $pb->{'realpkg'} = $pbpkg;
1316        $pb->{'date'} = $pbdate;
1317        $pb->{'defpkgdir'} = $defpkgdir;
1318        $pb->{'extpkgdir'} = $extpkgdir;
1319        $pb->{'chglog'} = $chglog;
1320        $pb->{'extdir'} = $pbextdir;
1321        $pb->{'packager'} = $ENV{'PBPACKAGER'};
1322        $pb->{'proj'} = $ENV{'PBPROJ'};
1323        $pb->{'repo'} = "$ENV{'PBREPO'}/$delivery->{$ENV{'PBPROJ'}}";
1324        $pb->{'patches'} = ();
1325        $pb->{'sources'} = ();
1326   
1327        my $tmpd = "$ENV{'PBTMP'}/cms.$$";
1328        pb_mkdir_p($tmpd) if (defined $pbparallel);
1329
1330        # Get only all.yml filter at this stage for pbinit
1331        my $ptr = pb_get_filters($pbpkg);
1332
1333        # Do not do that for website
1334        if (not defined $web) {
1335            my %virt;
1336            # De-duplicate similar VM/VE/RM
1337            foreach my $d (split(/,/,$tmpl)) {
1338                # skip ill-formatted vms (name-ver-arch)
1339                next if ($d !~ /-/);
1340                $virt{$d} = $d;
1341            }
1342
1343            # Try to use // processing here
1344            my $all_ok = 1;
1345            my $pm = new Parallel::ForkManager($pbparallel) if (defined $pbparallel);
1346            $pm->run_on_finish(sub { my ($pid, $code, $id, $signal, $dump) = @_;
1347                                    $all_ok = 0 unless (($code == 0) && ($signal == 0) && ($dump == 0)); }) if (defined $pbparallel);
1348
1349            pb_log(0,"Preparing delivery ...\n");
1350            foreach my $v (keys %virt) {
1351                $pm->start and next if (defined $pbparallel);
1352
1353                # Distro context
1354                my $pbos = pb_distro_get_context($v);
1355   
1356                $pb->{'pbos'} = $pbos;
1357                $pb->{'suf'} = $pbos->{'suffix'};
1358                pb_log(3,"DEBUG: pb: ".Dumper($pb)."\n");
1359
1360                # Get all filters to apply
1361                $ptr = pb_get_filters($pbpkg,$pbos);
1362   
1363                pb_log(2,"DEBUG Filtering PBDATE => $pbdate, PBTAG => $pbtag, PBVER => $pbver\n");
1364   
1365                # We need to compute the real name of the package
1366                my $pbrealpkg = pb_cms_get_real_pkg($pbpkg,$pbos->{'type'});
1367                $pb->{'realpkg'} = $pbrealpkg;
1368                pb_log(2,"Virtual package $pbpkg has a real package name of $pbrealpkg on $pbos->{'name'}-$pbos->{'version'}\n") if ($pbrealpkg ne $pbpkg);
1369   
1370                # Filter build files from the less precise up to the most with overloading
1371                # Filter all files found, keeping the name, and generating in dest
1372   
1373                # Find all build files first relatively to PBROOTDIR
1374                # Find also all specific files referenced in the .yml conf file
1375                my %bfiles = ();
1376                my %pkgfiles = ();
1377                # Used in Filter.pm by pb_filter_file
1378
1379                $build{$v} = "no";
1380                if (-d "$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'os'}") {
1381                    pb_list_bfiles("$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'os'}",$pbpkg,\%bfiles,\%pkgfiles,$supfiles);
1382                    $build{$v} = "yes";
1383                } 
1384                if (-d "$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'type'}") {
1385                    pb_list_bfiles("$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'type'}",$pbpkg,\%bfiles,\%pkgfiles,$supfiles);
1386                    $build{$v} = "yes";
1387                } 
1388                if (-d "$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'family'}") {
1389                    pb_list_bfiles("$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'family'}",$pbpkg,\%bfiles,\%pkgfiles,$supfiles);
1390                    $build{$v} = "yes";
1391                } 
1392                if (-d "$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'name'}") {
1393                    pb_list_bfiles("$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'name'}",$pbpkg,\%bfiles,\%pkgfiles,$supfiles);
1394                    $build{$v} = "yes";
1395                } 
1396                if (-d "$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'name'}-$pbos->{'version'}") {
1397                    pb_list_bfiles("$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'name'}-$pbos->{'version'}",$pbpkg,\%bfiles,\%pkgfiles,$supfiles);
1398                    $build{$v} = "yes";
1399                } 
1400                if (-d "$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}") {
1401                    pb_list_bfiles("$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}",$pbpkg,\%bfiles,\%pkgfiles,$supfiles);
1402                    $build{$v} = "yes";
1403                }
1404                pb_log(2,"DEBUG($v) bfiles: ".Dumper(\%bfiles)."\n");
1405   
1406                if ($build{$v} ne "no") {
1407                    # Apply now all the filters on all the files concerned
1408                    # destination dir depends on the type of file
1409                    # For patch support
1410                    # TODO: Make it CMS aware
1411                    $pb->{'patches'} = pb_list_sfiles("$ENV{'PBROOTDIR'}/$pbpkg/pbpatch", $pb->{'patches'}, $pbos, "$ENV{'PBROOTDIR'}/$pbpkg/pbextpatch");
1412                    pb_log(2,"DEBUG($v) patches: ".Dumper($pb->{'patches'})."\n");
1413                    # TODO: Make it CMS aware
1414                    $pb->{'sources'} = pb_list_sfiles("$ENV{'PBROOTDIR'}/$pbpkg/pbsrc", $pb->{'sources'}, $pbos, "$ENV{'PBROOTDIR'}/$pbpkg/pbextsrc");
1415                    pb_log(2,"DEBUG($v) sources: ".Dumper($pb->{'sources'})."\n");
1416   
1417                    if (defined $pb->{'patches'}->{$v}) {
1418                        # Filter potential patches (local + remote)
1419                        pb_mkdir_p("$dest/pbconf/$v/pbpatch");
1420                        # If Debian based distribution, then prepare what will be done at build time
1421                        my ($patchcmd,$patchopt);
1422                        if ($pbos->{'type'} eq "deb") {
1423                            ($patchcmd,$patchopt) = pb_distro_get_param($pbos,pb_conf_get_if("ospatchcmd","ospatchopt"));
1424                            open(SCRIPT,"> $dest/pbconf/$v/pbpatch/pbapplypatch") || die "Unable to create $dest/pbconf/$v/pbpatch/pbapplypatch";
1425                            print SCRIPT "#!/bin/bash\n";
1426                            print SCRIPT "set -x\n" if ($pbdebug gt 1);
1427                        }
1428                        foreach my $pf (split(/,/,$pb->{'patches'}->{$v})) {
1429                            my $pp = basename($pf);
1430                            pb_vcs_export($pf,undef,"$dest/pbconf/$v/pbpatch");
1431                            pb_filter_file_inplace($ptr,"$dest/pbconf/$v/pbpatch/$pp",$pb);
1432                            pb_system("gzip -9f $dest/pbconf/$v/pbpatch/$pp","","quiet");
1433                            if ($pbos->{'type'} eq "deb") {
1434                                # If Debian based distribution, then prepare what will be done at build time
1435                                # by applying the patches that will be available under the debian/patches dir
1436                                print SCRIPT "$patchcmd $patchopt \< debian/patches/$pp\n";
1437                            }
1438                        }
1439                        if ($pbos->{'type'} eq "deb") {
1440                            close(SCRIPT);
1441                            chmod 0755,"$dest/pbconf/$v/pbpatch/pbapplypatch";
1442                        }
1443                        #pb_system("cat $dest/pbconf/$v/pbpatch/pbapplypatch","APPLY","verbose");
1444                    }
1445                    if (defined $pb->{'sources'}->{$v}) {
1446                        pb_mkdir_p("$dest/pbconf/$v/pbsrc");
1447                        foreach my $pf (split(/,/,$pb->{'sources'}->{$v})) {
1448                            my $pp = basename($pf);
1449                            pb_vcs_export($pf,undef,"$dest/pbconf/$v/pbsrc");
1450                            pb_filter_file_inplace($ptr,"$dest/pbconf/$v/pbsrc/$pp",$pb);
1451                        }
1452                    }
1453                    # Filter build files at the end, as they depend on patches and sources
1454                    foreach my $f (keys %bfiles) {
1455                        pb_filter_file("$ENV{'PBROOTDIR'}/$bfiles{$f}",$ptr,"$dest/pbconf/$v/$f",$pb);
1456                    }
1457                    foreach my $f (keys %pkgfiles) {
1458                        pb_filter_file("$ENV{'PBROOTDIR'}/$pkgfiles{$f}",$ptr,"$dest/pbconf/$v/$f",$pb);
1459                    }
1460                }
1461
1462                if (defined $pbparallel) {
1463                    # Communicate results back to parent
1464                    my $str = "";
1465                    $str .= "build $v = $build{$v}\n" if (defined $build{$v});
1466                    $str .= "patches $v = $pb->{'patches'}->{$v}\n" if (defined $pb->{'patches'}->{$v});
1467                    $str .= "sources $v = $pb->{'sources'}->{$v}\n" if (defined $pb->{'sources'}->{$v});
1468                    pb_set_content("$tmpd/$$","$str");
1469                    $pm->finish;
1470                }
1471            }
1472            # In the parent, we need to get the result from the children
1473            $pm->wait_all_children if (defined $pbparallel);
1474            die "Aborting, one or more of the children failed." if ((not $all_ok) && ($Global::pb_stop_on_error));
1475            my $made = "";
1476            my %h = ();
1477            my %tmp;
1478            my %tmp2;
1479            my $pt;
1480            my $k;
1481
1482            foreach $k (<$tmpd/*>) {
1483                $made .= pb_get_content($k);
1484            }
1485            pb_rm_rf($tmpd);
1486            pb_log(3,"MADE:\n$made");
1487
1488            # Rebuild local hashes
1489            foreach $k (split(/\n/,$made)) {
1490                if ($k =~ /^\s*([A-z0-9-_]+)\s+([[A-z0-9-_.]+)\s*=\s*(.+)$/) {
1491                    $h{$1}->{$2}=$3;
1492                }
1493            }
1494            pb_log(2,"HASH: ".Dumper(%h));
1495
1496            # Patches
1497            pb_log(0,"Delivered and compressed patches ");
1498            $pt = $h{'patches'};
1499            foreach $k (keys %$pt) {
1500                foreach my $v1 (split(/,/,$pt->{$k})) {
1501                    $tmp{$v1} = "";
1502                }
1503            }
1504            if (keys %tmp) {
1505                foreach $k (keys %tmp) {
1506                    pb_log(0,"$k ");
1507                }
1508            } else {
1509                pb_log(0,"N/A");
1510            }
1511            pb_log(0,"\n");
1512
1513            # Sources
1514            pb_log(0,"Delivered additional sources ");
1515            $pt = $h{'sources'};
1516            foreach $k (keys %$pt) {
1517                foreach my $v1 (split(/,/,$pt->{$k})) {
1518                    $tmp2{$v1} = "";
1519                }
1520            }
1521            if (keys %tmp2) {
1522                foreach $k (keys %tmp2) {
1523                    pb_log(0,"$k ");
1524                }
1525            } else {
1526                pb_log(0,"N/A");
1527            }
1528            pb_log(0,"\n");
1529
1530            # Build files
1531            my @found;
1532            my @notfound;
1533            $pt = $h{'build'};
1534            foreach my $b (keys %$pt) {
1535                push @found,$b if ($pt->{$b} =~ /yes/);
1536                push @notfound,$b if ($pt->{$b} =~ /no/);
1537            }
1538            pb_log(0,"Build files have been generated for ... ".join(',',sort(@found))."\n") if (@found);
1539            pb_log(0,"No Build files found for ".join(',',sort(@notfound))."\n") if (@notfound);
1540
1541        } else {
1542            # Instead call News generation
1543            pb_web_news2html($dest);
1544            # And create an empty pbconf
1545            pb_mkdir_p("$dest/pbconf");
1546            # And prepare the pbscript to execute remotely
1547            open(SCRIPT,"> $ENV{'PBTMP'}/pbscript") || die "Unable to create $ENV{'PBTMP'}/pbscript";
1548            print SCRIPT "#!/bin/bash\n";
1549            print SCRIPT "#set -x\n";
1550            print SCRIPT "echo ... Extracting Website content\n";
1551            print SCRIPT "find . -type f | grep -Ev '^./$pbpkg-$pbver$pbextdir.tar.gz|^./pbscript' | xargs rm -f non-existent\n";
1552            print SCRIPT "find * -type d -depth | xargs rmdir 2> /dev/null \n";
1553            print SCRIPT "tar xfz $pbpkg-$pbver$pbextdir.tar.gz\n";
1554            print SCRIPT "mv $pbpkg-$pbver$pbextdir/* .\n";
1555            print SCRIPT "rm -f $pbpkg-$pbver$pbextdir.tar.gz\n";
1556            print SCRIPT "rmdir $pbpkg-$pbver$pbextdir\n";
1557            print SCRIPT "find . -type f -print0 | xargs -0 chmod 644\n";
1558            print SCRIPT "find . -type d -print0 | xargs -0 chmod 755\n";
1559            close(SCRIPT);
1560            chmod 0755,"$ENV{'PBTMP'}/pbscript";
1561        }
1562
1563        # Apply filters to the non-build files
1564        my $liste ="";
1565        if (defined $filteredfiles->{$pbpkg}) {
1566            foreach my $f (split(/,/,$filteredfiles->{$pbpkg})) {
1567                pb_filter_file_inplace($ptr,"$dest/$f",$pb);
1568                $liste = "$f $liste";
1569            }
1570        }
1571        pb_log(2,"Files ".$liste."have been filtered\n");
1572
1573        # TODO: Make it CMS aware
1574        # Execute the pbinit script if any
1575        if (-x "$ENV{'PBROOTDIR'}/$pbpkg/pbinit") {
1576            pb_filter_file("$ENV{'PBROOTDIR'}/$pbpkg/pbinit",$ptr,"$ENV{'PBTMP'}/pbinit",$pb);
1577            chmod 0755,"$ENV{'PBTMP'}/pbinit";
1578            pb_system("cd $dest ; $ENV{'PBTMP'}/pbinit","Executing init script from $ENV{'PBROOTDIR'}/$pbpkg/pbinit under $dest","verbose");
1579        }
1580
1581        # Do we have additional script to run to prepare the environement for the project ?
1582        # Then include it in the pbconf delivery
1583        foreach my $pbvf (<$ENV{'PBROOTDIR'}/$pbpkg/pbv*.pre>,<$ENV{'PBROOTDIR'}/$pbpkg/pbv*.post>, <$ENV{'PBROOTDIR'}/$pbpkg/pbtest*>) {
1584            pb_log(1,"Handling file $pbvf\n");
1585            if (-x "$pbvf") {
1586                my $target = "$ENV{'PBTMP'}/".basename($pbvf);
1587                pb_filter_file("$pbvf",$ptr,$target,$pb);
1588                chmod 0755,"$target";
1589                pb_log(1,"Capturing file $pbvf in $target\n");
1590            }
1591        }
1592
1593        # Prepare the dest directory for archive
1594        chdir "$ENV{'PBDESTDIR'}" || die "Unable to change dir to $ENV{'PBDESTDIR'}";
1595        if (defined $preserve) {
1596            # In that case we want to preserve the original tar file for checksum purposes
1597            # The one created is btw equivalent in that case to this one
1598            # Maybe check basename of both to be sure they are the same ?
1599            pb_log(0,"Preserving original tar file ");
1600            move("$preserve","$pbpkg-$pbver$pbextdir.tar.gz");
1601        } else {
1602            # Possibility to look at PBSRC to guess more the filename
1603            pb_system("tar cfz $pbpkg-$pbver$pbextdir.tar.gz --exclude=$pbpkg-$pbver$pbextdir/pbconf $pbpkg-$pbver$pbextdir","Creating $pbpkg tar files compressed");
1604        }
1605        pb_log(0,"Under $ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.tar.gz\n");
1606        pb_system("tar cfz $pbpkg-$pbver$pbextdir.pbconf.tar.gz $pbpkg-$pbver$pbextdir/pbconf","Creating pbconf tar files compressed");
1607        pb_log(0,"Under $ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.pbconf.tar.gz\n");
1608
1609        # Keep track of version-tag per pkg
1610        $pkgs{$pbpkg} = "$pbver-$pbtag";
1611
1612        # Final cleanup
1613        pb_rm_rf($dest) if (-d $dest);
1614    }
1615
1616    # Keep track of per package version
1617    pb_log(2,"DEBUG pkgs: ".Dumper(%pkgs)."\n");
1618    open(PKG,"> $ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml") || die "Unable to create $ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml";
1619    print PKG "---\n";
1620    print PKG "pbpkg:\n";
1621    foreach my $pbpkg (keys %pkgs) {
1622        print PKG "  $pbpkg: $pkgs{$pbpkg}\n";
1623    }
1624    close(PKG);
1625
1626    # Keep track of what is generated by default
1627    # We need to store the dir and info on version-tag
1628    # Base our content on the existing .yml file
1629    copy("$ENV{'PBROOTDIR'}/$ENV{'PBPROJ'}.yml","$ENV{'PBDESTDIR'}/pbrc.yml");
1630    my $lh;
1631    $lh = pb_conf_cache("$ENV{'PBDESTDIR'}/pbrc.yml",$lh);
1632    $lh->{'pbroot'}->{$ENV{'PBPROJ'}} = $ENV{'PBROOTDIR'};
1633    $lh->{'projver'}->{$ENV{'PBPROJ'}} = $ENV{'PBPROJVER'};
1634    $lh->{'projtag'}->{$ENV{'PBPROJ'}} = $ENV{'PBPROJTAG'};
1635    $lh->{'pbpackager'}->{$ENV{'PBPROJ'}} = $ENV{'PBPACKAGER'};
1636    $lh->{'pbextdir'}->{$ENV{'PBPROJ'}} = $pbextdir;
1637    pb_conf_write("$ENV{'PBDESTDIR'}/pbrc.yml",$lh);
1638}
1639
1640sub pb_test2pkg {
1641    # Get the running distro to test on
1642    my $pbos = pb_distro_get_context();
1643
1644    # Get list of packages to test
1645    # Get content saved in cms2build
1646    my $ptr = pb_get_pkg();
1647    @pkgs = @$ptr;
1648
1649    # Additional potential repo
1650    my $forcerepo = pb_distro_setuprepo($pbos);
1651    foreach my $pbpkg (@pkgs) {
1652        # We need to install the package to test, and deps brought with it
1653        pb_distro_installdeps(undef,$pbos,$forcerepo,$pbpkg);
1654        pb_system("$ENV{'PBDESTDIR'}/pbtest","Launching test for $pbpkg","verbose");
1655    }
1656}
1657
1658sub pb_build2prep {
1659
1660    pb_log(0,"INFO: ------ Starting to prepare build environement ------\n");
1661
1662    # Get the running distro to build on
1663    my $pbos = pb_distro_get_context();
1664
1665    # If needed we may add repository to the build env
1666    my $forcerepo = pb_distro_setuprepo($pbos);
1667    pb_log(1,"Forcing full package install with forcerepo\n") if (defined $forcerepo);
1668
1669    # Get list of packages to build
1670    my $ptr = pb_get_pkg();
1671    @pkgs = @$ptr;
1672   
1673    # Get content saved in cms2build
1674    my ($pkg) = pb_conf_read("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml","pbpkg");
1675    $pkg = { } if (not defined $pkg);
1676    my $pbextdir = pb_get_extdir();
1677
1678    foreach my $pbpkg (@pkgs) {
1679        my $vertag = $pkg->{$pbpkg};
1680        my @buildfiles;
1681        pb_log(2,"Vertag: $vertag\n");
1682        # get the version of the current package - maybe different
1683        ($pbver,$pbtag) = split(/-/,$vertag);
1684
1685        my $src2="$ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.pbconf.tar.gz";
1686        pb_log(2,"Pbconf file: $src2\n");
1687        pb_log(2,"Working directory: $ENV{'PBBUILDDIR'}\n");
1688
1689        my $ftype;
1690        if ($pbos->{'type'} eq "rpm") {
1691            $ftype = "\.spec\$";
1692        } elsif ($pbos->{'type'} eq "deb") {
1693            $ftype = "\/control\$";
1694        } elsif ($pbos->{'type'} eq "ebuild") {
1695            $ftype = "\.ebuild\$";
1696        } elsif ($pbos->{'type'} eq "tgz") {
1697            $ftype = "\/pbslack\$";
1698        } elsif ($pbos->{'type'} eq "pkg") {
1699            # Solaris - no deps management up to Solaris 10
1700        } elsif ($pbos->{'type'} eq "hpux") {
1701            # TODO: HP-UX
1702        } elsif ($pbos->{'type'} eq "apk") {
1703            # TODO: Alpine
1704        } else {
1705            die "Unknown OS type format $pbos->{'type'}";
1706        }
1707
1708        @buildfiles = pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/","$ENV{'PBTMP'}","build");
1709        pb_log(2,"buildfiles ".Dumper(\@buildfiles)."\n");
1710        foreach my $f (@buildfiles) {
1711            if ($f =~ /$ftype/) {
1712                pb_distro_installdeps($f,$pbos,$forcerepo);
1713            }
1714        }
1715    }
1716    pb_log(0,"INFO: ------ Finished preparing build environment ------\n");
1717}
1718
1719sub pb_build2pkg {
1720
1721    my $do_install = shift;
1722
1723        # Call build2prep in case we have new deps we want to take in account
1724        # without redoing the prep phase (can be long and that helps debugging
1725        # build dependencies issues
1726    pb_build2prep();
1727    pb_log(0,"INFO: ------ Starting to build package ------\n");
1728    # Get the running distro to build on
1729    my $pbos = pb_distro_get_context();
1730
1731    # Get list of packages to build
1732    my $ptr = pb_get_pkg();
1733    @pkgs = @$ptr;
1734
1735    # Get content saved in cms2build
1736    my ($pkg) = pb_conf_read("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml","pbpkg");
1737    $pkg = { } if (not defined $pkg);
1738    my $pbextdir = pb_get_extdir();
1739
1740    pb_mkdir_p("$ENV{'PBBUILDDIR'}") if (! -d "$ENV{'PBBUILDDIR'}");
1741    chdir "$ENV{'PBBUILDDIR'}" || die "Unable to chdir to $ENV{'PBBUILDDIR'}";
1742    my $made = ""; # pkgs made during build
1743    my $pm;
1744    my $all_ok = 1;
1745
1746    if (defined $pbparallel) {
1747        $pm = new Parallel::ForkManager($pbparallel);
1748        $pm->run_on_finish(sub { my ($pid, $code, $id, $signal, $dump) = @_;
1749                                $all_ok = 0 unless (($code == 0) && ($signal == 0) && ($dump == 0)); });
1750    }
1751
1752    # We need to communicate info back from the children if parallel so prepare a dir for that
1753    my $tmpd = "$ENV{'PBTMP'}/build.$$";
1754    pb_mkdir_p($tmpd) if (defined $pbparallel);
1755
1756    # Do it here as this doesn't work in // mode as it dies if it fails
1757    if ($pbos->{'type'} eq "rpm") {
1758        foreach my $d ('RPMS','SRPMS','SPECS','SOURCES','BUILD') {
1759                if (! -d "$ENV{'PBBUILDDIR'}/$d") {
1760                    pb_mkdir_p("$ENV{'PBBUILDDIR'}/$d");
1761                }
1762            }
1763    }
1764
1765    foreach my $pbpkg (@pkgs) {
1766        $pm->start and next if (defined $pbparallel);
1767
1768        my $vertag = $pkg->{$pbpkg};
1769        pb_log(2,"Vertag: $vertag\n");
1770        # get the version of the current package - maybe different
1771        ($pbver,$pbtag) = split(/-/,$vertag);
1772
1773        my $src="$ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.tar.gz";
1774        my $src2="$ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.pbconf.tar.gz";
1775        pb_log(2,"Source file: $src\n");
1776        pb_log(2,"Pbconf file: $src2\n");
1777
1778        pb_log(2,"Working directory: $ENV{'PBBUILDDIR'}\n");
1779        if ($pbos->{'type'} eq "rpm") {
1780            # Remove in case a previous link/file was there
1781            unlink "$ENV{'PBBUILDDIR'}/SOURCES/".basename($src);
1782            symlink "$src","$ENV{'PBBUILDDIR'}/SOURCES/".basename($src) || die "Unable to symlink $src in $ENV{'PBBUILDDIR'}/SOURCES";
1783            # We need to first extract the spec file
1784            my @buildfile = pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/","$ENV{'PBBUILDDIR'}/SPECS","build");
1785
1786            # We need to handle potential patches to upstream sources
1787            pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbpatch/","$ENV{'PBBUILDDIR'}/SOURCES","patch");
1788
1789            # We need to handle potential additional sources to upstream sources
1790            pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbsrc/","$ENV{'PBBUILDDIR'}/SOURCES","src");
1791
1792            pb_log(2,"buildfile: ".Dumper(\@buildfile)."\n");
1793            # set LANGUAGE to check for correct log messages
1794            $ENV{'LANGUAGE'}="C";
1795            # Older Redhat use _target_platform in %configure incorrectly
1796            my $specialdef = "";
1797            if (($pbos->{'name'} eq "rhel") && ($pbos->{'version'} eq "2.1")) {
1798                $specialdef = "--define \'_target_platform \"\"\'";
1799            }
1800            my $buildcmd = "rpmbuild";
1801            if (($pbos->{'name'} eq "redhat") && ($pbos->{'version'} =~ /^6/)){
1802                $buildcmd = "rpm" ;
1803            }
1804
1805            foreach my $f (@buildfile) {
1806                if ($f =~ /\.spec$/) {
1807                    pb_system("$buildcmd $specialdef --define \"packager $ENV{'PBPACKAGER'}\" --define \"_topdir $ENV{'PBBUILDDIR'}\" -ba $f","Building package with $f under $ENV{'PBBUILDDIR'}","verbose");
1808                    last;
1809                }
1810            }
1811            # Get the name of the generated packages
1812            open(LOG,"$ENV{'PBTMP'}/system.$$.log") || die "Unable to open $ENV{'PBTMP'}/system.$$.log";
1813            while (<LOG>) {
1814                chomp($_);
1815                next if ($_ !~ /^Wrote:/);
1816                s|.*/([S]*RPMS.*)|$1|;
1817                $made .=" $_";
1818            }
1819            close(LOG);
1820
1821        } elsif ($pbos->{'type'} eq "deb") {
1822            pb_system("tar xfz $src","Extracting sources");
1823            pb_system("tar xfz $src2","Extracting pbconf");
1824
1825            chdir "$pbpkg-$pbver$pbextdir" || die "Unable to chdir to $pbpkg-$pbver$pbextdir";
1826            pb_rm_rf("debian");
1827            my $confdir = "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
1828            die "Configuration directory $confdir is not a directory\nIs os description listed in your {ve,vm,rm}list values?" if (not -d $confdir);
1829            symlink "$confdir","debian" || die "Unable to symlink 'debian' to $confdir";
1830            chmod 0755,"debian/rules";
1831
1832            # We need to handle potential patches to upstream sources
1833            pb_mkdir_p("debian/patches");
1834            my @f = pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbpatch/","debian/patches","patch");
1835
1836            # By default we use format 1.0 - Cf man dpkg-source
1837            my $debsrcfmt = "1.0";
1838            my $debsrcfile = "debian/source/format";
1839            if (-f $debsrcfile) {
1840                $debsrcfmt = pb_get_content($debsrcfile);
1841            }
1842
1843            if ($debsrcfmt =~ /^3.*quilt/) {
1844                # If we use quilt to manage patches, we then setup the env correctly
1845                # as per http://pkg-perl.alioth.debian.org/howto/quilt.html
1846                # Generate Debian patch series for quilt
1847                open(SERIE,"> debian/patches/series") || die "Unable to write in debian/patches/series";
1848                $ENV{'QUILT_PATCHES'}="debian/patches";
1849            } else {
1850                # If we use dpatch to manage patches, we then setup the 00list file as well
1851                open(SERIE,"> debian/patches/00list") || die "Unable to write in debian/patches/00list";
1852            }
1853            foreach my $f (sort @f) {
1854                # Skip the script made to apply the patches to the Debian tree
1855                next if ($f =~ /pbapplypatch/);
1856                # We also need to uncompress them
1857                pb_system("gzip -d $f","","quiet");
1858                $f =~ s/\.gz$//;
1859                print SERIE "$f\n";
1860            }
1861            close(SERIE);
1862            if (@f) {
1863                # We have patches...
1864                my $patch_file = "debian/patches/pbapplypatch";
1865                if (($debsrcfmt =~ /^1.*/) && (-x $patch_file)) {
1866                    # In that case we need to apply the patches ourselves locally
1867                    pb_system("cat $patch_file","APPLY","verbose");
1868                    pb_system("$patch_file","Applying patches to $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'} tree");
1869                }
1870                # ...so modify the name of files to be Debian compliant
1871                move("../$src","../$pbpkg-$pbver$pbextdir.orig.tar.gz");
1872            }
1873
1874            # We need to handle potential additional sources to upstream sources
1875            #pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbsrc/","$ENV{'PBBUILDDIR'}/debian","src");
1876
1877            pb_system("dpkg-buildpackage -us -uc -rfakeroot","Building package","verbose");
1878
1879            # Get the name of the generated packages
1880            open(LOG,"$ENV{'PBTMP'}/system.$$.log") || die "Unable to open $ENV{'PBTMP'}/system.$$.log";
1881            while (<LOG>) {
1882                chomp();
1883                pb_log(3,"In loop: $_\n");
1884                next unless (/^dpkg-deb.*\.\.\/(.*_.*\.deb).*/);
1885                my $tmp = $1;
1886                #doesn't work in my case
1887                #die "Missing file $tmp" if (not -f "../$tmp");
1888                $made = "$made $tmp";
1889            }
1890            close(LOG);
1891            pb_log(2,"Now made is: $made\n");
1892
1893            open(CTRL,"debian/control") || die "Unable to open debian/control: $!";
1894            while (<CTRL>) {
1895                pb_log(3,"In loop: $_\n");
1896                next unless (/^Source: (\S+)/o);
1897                my $tmp = $1;
1898                $made = "$made $tmp"."_*.dsc $tmp"."_*.tar.gz $tmp"."_*.changes";
1899                #doesn't work in my case
1900                #foreach my $glob (("$1\_*.changes", "$1\_*.dsc", "$1\_*.tar.gz")) {
1901                    #my @file = glob($glob);
1902                    #die "Missing file for $glob" unless (@file > 0);
1903                    #die "Too many files for $glob" if (@file > 1);
1904                    #die "Missing file $file[0]" if (not -f $file[0]);
1905                    #$made .= " $file[0]";
1906                #}
1907            }
1908            close(CTRL);
1909            #pb_display_file("$ENV{'PBTMP'}/system.$$.log");
1910            pb_log(2,"Finally made is: $made\n");
1911
1912            chdir ".." || die "Unable to chdir to parent dir";
1913            pb_rm_rf("$pbpkg-$pbver$pbextdir");
1914        } elsif ($pbos->{'type'} eq "ebuild") {
1915            my @ebuildfile;
1916            # For gentoo we need to take pb as subsystem name
1917            # We put every apps here under sys-apps. hope it's correct
1918            # We use pb's home dir in order to have a single OVERLAY line
1919            my $tmpe = "$ENV{'HOME'}/portage/pb/sys-apps/$pbpkg";
1920            pb_mkdir_p($tmpe) if (! -d "$tmpe");
1921            pb_mkdir_p("$ENV{'HOME'}/portage/distfiles") if (! -d "$ENV{'HOME'}/portage/distfiles");
1922
1923            # We need to first extract the ebuild file
1924            @ebuildfile = pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/","$tmpe","build");
1925
1926            # Prepare the build env for gentoo
1927            my $found = 0;
1928            my $pbbd = $ENV{'HOME'};
1929            $pbbd =~ s|/|\\/|g;
1930            if (-r "/etc/make.conf") {
1931                open(MAKE,"/etc/make.conf");
1932                while (<MAKE>) {
1933                    $found = 1 if (/$pbbd\/portage/);
1934                }
1935                close(MAKE);
1936            }
1937            if ($found == 0) {
1938                pb_system("sudo sh -c 'echo PORTDIR_OVERLAY=\"$ENV{'HOME'}/portage\" >> /etc/make.conf'");
1939            }
1940            #$found = 0;
1941            #if (-r "/etc/portage/package.keywords") {
1942            #open(KEYW,"/etc/portage/package.keywords");
1943            #while (<KEYW>) {
1944            #$found = 1 if (/portage\/pb/);
1945            #}
1946            #close(KEYW);
1947            #}
1948            #if ($found == 0) {
1949            #pb_system("sudo sh -c \"echo portage/pb >> /etc/portage/package.keywords\"");
1950            #}
1951
1952            # Build
1953            foreach my $f (@ebuildfile) {
1954                if ($f =~ /\.ebuild$/) {
1955                    move($f,"$tmpe/$pbpkg-$pbver.ebuild");
1956                    pb_system("cd $tmpe ; ebuild $pbpkg-$pbver.ebuild clean ; ebuild $pbpkg-$pbver.ebuild digest ; ebuild $pbpkg-$pbver.ebuild package","verbose");
1957                    # Now move it where pb expects it
1958                    pb_mkdir_p("$ENV{'PBBUILDDIR'}/portage/pb/sys-apps/$pbpkg");
1959                    if ($pbtag eq 0) {
1960                        # This is assumed to be a test version
1961                        my $nver = substr($pbver,0,-14);
1962                        my $ntag = substr($pbver,-14);
1963                        my $ebtg = "portage/pb/sys-apps/$pbpkg/$pbpkg-$nver"."_p$ntag.ebuild";
1964                        move("$tmpe/$pbpkg-$pbver.ebuild","$ENV{'PBBUILDDIR'}/$ebtg");
1965                        $made="$made $ebtg";
1966                    } else {
1967                        my $ebtg = "portage/pb/sys-apps/$pbpkg/$pbpkg-$pbver-r$pbtag.ebuild";
1968                        move("$tmpe/$pbpkg-$pbver.ebuild","$ENV{'PBBUILDDIR'}/$ebtg");
1969                        $made="$made $ebtg";
1970                    }
1971                }
1972            }
1973
1974        } elsif ($pbos->{'type'} eq "tgz") {
1975            # Slackware family
1976            $made="$made $pbpkg/$pbpkg-$pbver-*-$pbtag.tgz";
1977
1978            pb_system("tar xfz $src","Extracting sources");
1979            pb_system("tar xfz $src2","Extracting pbconf");
1980            chdir "$pbpkg-$pbver$pbextdir" || die "Unable to chdir to $pbpkg-$pbver$pbextdir";
1981            symlink "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}","install" || die "Unable to symlink to pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
1982            if (-x "install/pbslack") {
1983                pb_system("./install/pbslack","Building software");
1984                pb_system("sudo /sbin/makepkg -p -l y -c y $pbpkg","Packaging $pbpkg","verbose");
1985            }
1986            chdir ".." || die "Unable to chdir to parent dir";
1987            pb_rm_rf("$pbpkg-$pbver$pbextdir");
1988        } elsif ($pbos->{'type'} eq "pkg") {
1989            # Solaris
1990            $made="$made $pbpkg-$pbver-$pbtag.pkg.gz";
1991            my $pkgdestdir="$ENV{'PBBUILDDIR'}/install";
1992
1993            # Will host resulting packages
1994            pb_mkdir_p("$pbos->{'type'}");
1995            pb_mkdir_p("$pkgdestdir/pbdelivery");
1996            pb_system("tar xfz $src","Extracting sources under $ENV{'PBBUILDDIR'}");
1997            pb_system("tar xfz $src2","Extracting pbconf under $ENV{'PBBUILDDIR'}");
1998            # We need to handle potential patches to upstream sources
1999            pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbpatch/","$ENV{'PBBUILDDIR'}","patch");
2000
2001            # We need to handle potential additional sources to upstream sources
2002            pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbsrc/","$ENV{'PBBUILDDIR'}","src");
2003
2004            chdir "$pbpkg-$pbver$pbextdir" || die "Unable to chdir to $pbpkg-$pbver$pbextdir";
2005            if (-f "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbbuild") {
2006                chmod 0755,"pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbbuild";
2007                # pkginfo file is mandatory
2008                die "Unable to find pkginfo file in pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}" if (! -f "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pkginfo");
2009                # Build
2010                pb_system("pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbbuild $pkgdestdir/pbdelivery","Building software and installing under $pkgdestdir/pbdelivery");
2011                # Copy complementary files
2012                if (-f "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/prototype") {
2013                    copy("pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/prototype", $pkgdestdir) 
2014                } else {
2015                    # No prototype provided, calculating it
2016                    open(PROTO,"> $pkgdestdir/prototype") || die "Unable to create prototype file";
2017                    print PROTO "i pkginfo\n";
2018                    print PROTO "i depend\n" if (-f "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/depend");
2019                    $ENV{'PBSOLDESTDIR'} = "$pkgdestdir/pbdelivery";
2020                    find(\&create_solaris_prototype, "$pkgdestdir/pbdelivery");
2021                }
2022                copy("pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/depend", $pkgdestdir) if (-f "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/depend");
2023                copy("pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pkginfo", $pkgdestdir);
2024                pb_system("cd $pkgdestdir/pbdelivery ; pkgmk -o -f ../prototype -r $pkgdestdir/pbdelivery -d $ENV{'PBBUILDDIR'}/$pbos->{'type'}","Packaging $pbpkg","verbose");
2025                pb_system("cd $ENV{'PBBUILDDIR'}/$pbos->{'type'} ;  echo \"\" | pkgtrans -o -n -s $ENV{'PBBUILDDIR'}/$pbos->{'type'} $ENV{'PBBUILDDIR'}/$pbpkg-$pbver-$pbtag.pkg all","Transforming $pbpkg","verbose");
2026                pb_system("cd $ENV{'PBBUILDDIR'} ;  gzip -9f $pbpkg-$pbver-$pbtag.pkg","Compressing $pbpkg-$pbver-$pbtag.pkg","verbose");
2027            } else {
2028                pb_log(0,"No pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbbuild file found for $pbpkg-$pbver\n");
2029            }
2030            chdir ".." || die "Unable to chdir to parent dir";
2031            pb_rm_rf("$pbpkg-$pbver$pbextdir","$ENV{'PBBUILDDIR'}/$pbos->{'type'}","$pkgdestdir");
2032        } elsif ($pbos->{'type'} eq "hpux") {
2033            # HP-UX
2034            pb_system("tar xfz $src","Extracting sources");
2035            pb_system("tar xfz $src2","Extracting pbconf");
2036
2037            chdir "$pbpkg-$pbver$pbextdir" || die "Unable to chdir to $pbpkg-$pbver$pbextdir";
2038            pb_system("buildpackage ","Building package","verbose");
2039            # Get the name of the generated packages
2040            open(LOG,"$ENV{'PBTMP'}/system.$$.log") || die "Unable to open $ENV{'PBTMP'}/system.$$.log";
2041            while (<LOG>) {
2042                chomp();
2043                my $tmp = $_;
2044                next if ($tmp !~ /^SD BUILD.*:/);
2045                $tmp =~ s|.*../(.*)_(.*).sd.*|$1|;
2046                $made = "$made $tmp"."_*.sd";
2047            }
2048            close(LOG);
2049            $made="$made $pbpkg-$pbver-$pbtag.sd";
2050
2051            chdir ".." || die "Unable to chdir to parent dir";
2052            pb_rm_rf("$pbpkg-$pbver$pbextdir");
2053        } else {
2054            die "Unknown OS type format $pbos->{'type'}";
2055        }
2056        if (defined $pbparallel) {
2057            # Communicate results back to parent
2058            pb_set_content("$tmpd/$$",$made);
2059            $pm->finish;
2060        }
2061    }
2062    if (defined $pbparallel) {
2063        # In the parent, we need to get the result from the children
2064        $pm->wait_all_children;
2065        foreach my $f (<$tmpd/*>) {
2066            $made .= " ".pb_get_content($f);
2067        }
2068        die "Aborting, one or more of the children failed." if ((not $all_ok) && ($Global::pb_stop_on_error));
2069        pb_rm_rf($tmpd);
2070    }
2071
2072    # Sign packages
2073    pb_sign_pkgs($pbos,$made);
2074    pb_log(0,"INFO: ------ Finished building package ------\n");
2075
2076    # Find the appropriate check cmd/opts
2077    my ($chkcmd,$chkopt) = pb_distro_get_param($pbos,pb_conf_get_if("oschkcmd","oschkopt"));
2078
2079    my $ret = "";
2080    # Packages check if needed
2081    if ($pbos->{'type'} eq "rpm") {
2082        if ((defined  $chkcmd) && (-x $chkcmd)) {
2083            my $cmd = "$chkcmd";
2084            $cmd .= " $chkopt" if (defined $chkopt);
2085            $cmd .= " $made";
2086            my $ret = pb_system("$cmd","Checking validity of rpms with $chkcmd","mayfailverbose");
2087            pb_log(0,"ERROR: when checking packages validity\n") if ($ret ne 0);
2088        }
2089        my $rpms ="";
2090        my $srpms ="";
2091        foreach my $f (split(/ /,$made)) {
2092            $rpms .= "$ENV{'PBBUILDDIR'}/$f " if ($f =~ /^RPMS\//);
2093            $srpms .= "$ENV{'PBBUILDDIR'}/$f " if ($f =~ /^SRPMS\//);
2094        }
2095        pb_log(0,"SRPM packages generated: $srpms\n");
2096        pb_log(0,"RPM packages generated: $rpms\n");
2097        $ret = $rpms;
2098    } elsif ($pbos->{'type'} eq "deb") {
2099        my $made2 = "";
2100        foreach my $f (split(/ /,$made)) {
2101            $made2 .= "$f " if ($f =~ /\.deb$/);
2102        }
2103        if ((defined  $chkcmd) && (-x $chkcmd)) {
2104            my $ret = pb_system("$chkcmd $chkopt $made2","Checking validity of debs with $chkcmd","mayfail");
2105            pb_log(0,"ERROR: when checking packages validity\n") if ($ret ne 0);
2106        }
2107        pb_log(0,"deb packages generated: $made2\n");
2108        $ret = $made2;
2109    } else {
2110        pb_log(0,"No check done for $pbos->{'type'} yet\n");
2111        pb_log(0,"Packages generated: $made\n");
2112        $ret = $made;
2113    }
2114
2115    # Keep track of what is generated so that we can get them back from VMs/RMs
2116    my $pbstore = "$ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
2117    open(KEEP,"> $pbstore") || die "Unable to create $pbstore $!";
2118    print KEEP "$made\n";
2119    close(KEEP);
2120    pb_distro_installdeps(undef,$pbos,undef,$ret,"local") if ($do_install);
2121}
2122
2123sub pb_build2ssh {
2124    pb_send2target("Sources");
2125    pb_send2target("CPAN");
2126}
2127
2128sub pb_pkg2ssh {
2129    pb_send2target("Packages");
2130}
2131
2132# By default deliver to the the public site hosting the
2133# ftp structure (or whatever) or a VM/VE/RM
2134sub pb_send2target {
2135
2136    my $cmt = shift;
2137    my $pbscript1 = shift;
2138    my $v = shift;
2139    my $vexist = shift;         # 0 is FALSE
2140    my $vpid = shift;           # 0 is FALSE
2141    my $snapme = shift;         # 0 is FALSE
2142    my $pbstep = shift;         # 3 is usage of container
2143
2144    $vexist = 0 if (not defined $vexist);
2145    $vpid = 0 if (not defined $vpid);
2146    $snapme = 0 if (not defined $snapme);
2147    $pbstep = 3 if (not defined $pbstep);
2148    pb_log(2,"DEBUG: pb_send2target($cmt,".Dumper($v).",$vexist,$vpid)\n");
2149    my $host = "sshhost";
2150    my $login = "sshlogin";
2151    my $dir = "sshdir";
2152    my $port = "sshport";
2153    my $conf = "sshconf";
2154    my $tmout = undef;
2155    my $path = undef;
2156
2157    if ($cmt =~ /^VM/) {
2158        $login = "vmlogin";
2159        $dir = "pbdefdir";
2160        # Specific VM
2161        $tmout = "vmtmout";
2162        $path = "vmpath";
2163        $host = "vmhost";
2164        $port = "vmport";
2165    } elsif ($cmt =~ /^RM/) {
2166        $login = "rmlogin";
2167        $dir = "pbdefdir";
2168        # Specific RM
2169        $tmout = "rmtmout";
2170        $path = "rmpath";
2171        $host = "rmhost";
2172        $port = "rmport";
2173    } elsif ($cmt =~ /^VE/) {
2174        $login = "velogin";
2175        $dir = "pbdefdir";
2176        # Specific VE
2177        $path = "vepath";
2178        $conf = "rbsconf";
2179    } elsif ($cmt eq "Web") {
2180        $host = "websshhost";
2181        $login = "websshlogin";
2182        $dir = "websshdir";
2183        $port = "websshport";
2184    } elsif ($cmt eq "CPAN") {
2185        $host = "cpanpause";
2186        $login = "";
2187        $dir = "cpandir";
2188        $port = "";
2189    }
2190    my $cmd = "";
2191    my $src = "";
2192    my $cpanpkg = 0;
2193    my $pbos;
2194
2195    my $pbextdir = pb_get_extdir();
2196
2197    if ($cmt ne "Announce") {
2198        # Get list of packages to build
2199        my $ptr = pb_get_pkg();
2200        my @pkgs = ();
2201        @pkgs = @$ptr if (defined $ptr);
2202
2203        # Get the running distro to consider
2204        $pbos = pb_distro_get_context($v);
2205
2206        my $pkg = { };
2207        # Get content saved in cms2build
2208        if ((defined $ENV{'PBDESTDIR'}) && (defined $ENV{'PBPROJVER'}) && (defined $ENV{'PBPROJTAG'}) && (-f "$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml")) {
2209            ($pkg) = pb_conf_read("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml","pbpkg");
2210            $pkg = { } if (not defined $pkg);
2211        }
2212
2213        pb_mkdir_p("$ENV{'PBBUILDDIR'}") if (! -d "$ENV{'PBBUILDDIR'}");
2214        chdir "$ENV{'PBBUILDDIR'}" || die "Unable to chdir to $ENV{'PBBUILDDIR'}";
2215        foreach my $pbpkg (@pkgs) {
2216            my $vertag = $pkg->{$pbpkg};
2217            # get the version of the current package - maybe different
2218            pb_log(2,"Vertag: $vertag\n");
2219            ($pbver,$pbtag) = split(/-/,$vertag);
2220
2221            if (($cmt eq "Sources") || ($cmt =~ /(V[EM]|RM)(build|prep)/)) {
2222                $src = "$src $ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.tar.gz $ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.pbconf.tar.gz";
2223                my $cmd2 = "ln -sf $pbpkg-$pbver$pbextdir.tar.gz $pbpkg-latest.tar.gz";
2224                #my $cmd2 = "ln $pbpkg-$pbver$pbextdir.tar.gz $pbpkg-latest.tar.gz 2> /dev/null ; if [ \$? -ne 0 ]; then ln -sf $pbpkg-$pbver$pbextdir.tar.gz $pbpkg-latest.tar.gz ; fi";
2225                if ($cmd eq "") {
2226                    $cmd = $cmd2;
2227                } else {
2228                    $cmd = "$cmd ; $cmd2";
2229                }
2230            } elsif ($cmt eq "Web") {
2231                $src = "$src $ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.tar.gz"
2232            }
2233
2234            # Do we have one perl package
2235            my @nametype = pb_conf_get_if("namingtype");
2236            my $type = $nametype[0]->{$pbpkg};
2237            if ((defined $type) && ($type eq "perl")) {
2238                $cpanpkg = 1;
2239            }
2240        }
2241        # Adds conf file for availability of conf elements
2242        pb_conf_add("$ENV{'PBROOTDIR'}/$ENV{'PBPROJ'}.yml") if ((defined $ENV{'PBROOTDIR'}) && (-f "$ENV{'PBROOTDIR'}/$ENV{'PBPROJ'}.yml"));
2243    }
2244    my ($rbsconf,$testver,$delivery) = pb_conf_get_if($conf,"testver","delivery");
2245    if ($cmt eq "CPAN") {
2246        # Do not deliver on Pause if this is a test version
2247        return if (not defined $testver);
2248        return if ($testver =~ /true/);
2249        # Do not deliver on Pause if this is not a perl package
2250        return if ($cpanpkg == 0);
2251    }
2252
2253    if ($cmt =~ /(V[EM]|RM)(build|prep)/) {
2254        $src="$src $ENV{'PBROOTDIR'}/$ENV{'PBPROJ'}.yml $ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml $ENV{'PBETC'} $ENV{'PBDESTDIR'}/pbrc.yml $pbscript{$v}";
2255    } elsif ($cmt =~ /(V[EM]|RM)Script/) {
2256        $src="$src $pbscript{$v}";
2257    } elsif ($cmt =~ /(V[EM]|RM)test/) {
2258        $src="$src $ENV{'PBROOTDIR'}/$ENV{'PBPROJ'}.yml $ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml $ENV{'PBETC'} $ENV{'PBDESTDIR'}/pbrc.yml $pbscript{$v} $ENV{'PBDESTDIR'}/pbtest";
2259    } elsif (($cmt eq "Announce") || ($cmt eq "Web") || ($cmt eq "CPAN")) {
2260        $src="$src $ENV{'PBTMP'}/pbscript";
2261    } elsif ($cmt eq "Packages") {
2262        # Get package list from file made during build2pkg
2263        open(KEEP,"$ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}") || die "Unable to read $ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
2264        $src = <KEEP>;
2265        chomp($src);
2266        close(KEEP);
2267    }
2268    if (($cmt eq "Sources") || ($cmt eq "Packages") || ($cmt eq "CPAN")) {
2269        my ($pbpackager) = pb_conf_get("pbpackager");
2270        $ENV{'PBPACKAGER'} = $pbpackager->{$ENV{'PBPROJ'}};
2271        pb_log(0,"Exporting public key for $ENV{'PBPACKAGER'}\n");
2272        # Using pb_system is not working due to redirection below
2273        system("gpg --export -a \'$ENV{'PBPACKAGER'}\' > $ENV{'PBDESTDIR'}/$ENV{'PBPROJ'}.pubkey");
2274        chmod 0644,"$ENV{'PBDESTDIR'}/$ENV{'PBPROJ'}.pubkey";
2275        $src = "$src $ENV{'PBDESTDIR'}/$ENV{'PBPROJ'}.pubkey";
2276
2277        my ($gpgaddkey) = pb_conf_get_if("pbadditionalgpg");
2278        my $pbadditionalgpg;
2279        $pbadditionalgpg = $gpgaddkey->{$ENV{'PBPROJ'}} if (defined $gpgaddkey);
2280        if (defined $pbadditionalgpg) {
2281            foreach my $k (split(/,/,$pbadditionalgpg)) {
2282                pb_log(0,"Exporting public key $k\n");
2283                system("gpg --export -a \'$k\' > $ENV{'PBDESTDIR'}/$ENV{'PBPROJ'}-$k.pubkey");
2284                $src = "$src $ENV{'PBDESTDIR'}/$ENV{'PBPROJ'}-$k.pubkey";
2285            }
2286        }
2287    }
2288    # Remove potential leading spaces (cause problem with basename)
2289    $src =~ s/^ *//;
2290    my $basesrc = "";
2291    foreach my $i (split(/ +/,$src)) {
2292        $basesrc .= " ".basename($i);
2293    }
2294
2295    pb_log(2,"values: ".Dumper(($host,$login,$dir,$port,$tmout,$path,$conf))."\n");
2296    my ($sshhost,$sshdir) = pb_conf_get($host,$dir);
2297    # Not mandatory...
2298    $delivery->{$ENV{'PBPROJ'}} = "" if (not defined $delivery->{$ENV{'PBPROJ'}});
2299    my ($sshlogin,$sshport) = pb_conf_get_if($login,$port);
2300    $sshport->{$ENV{'PBPROJ'}} = 22 if (not defined $sshport->{$ENV{'PBPROJ'}});
2301    $sshlogin->{$ENV{'PBPROJ'}} = getpwuid($UID) if (not defined $sshlogin->{$ENV{'PBPROJ'}});
2302    my ($vtmout,$vepath);
2303    # ...Except those in virtual context
2304    if ($cmt =~ /^VE/) {
2305        ($vepath) = pb_conf_get($path);
2306    }
2307    if ($cmt =~ /^(V|R)M/) {
2308        $vtmout = pb_distro_get_param($pbos,pb_conf_get_if($tmout));
2309    }
2310    my $remhost = $sshhost->{$ENV{'PBPROJ'}};
2311    my $remdir = $sshdir->{$ENV{'PBPROJ'}};
2312    if ($cmt =~ /^V[EM]|RM/) {
2313        # In that case our real host is in the xxhost with the OS as key, not project as above
2314        $remhost = pb_distro_get_param($pbos,$sshhost);
2315    }
2316    pb_log(2,"ssh: ".Dumper(($remhost,$sshlogin,$remdir,$sshport,$vepath,$rbsconf))."\n");
2317    pb_log(2,"ssh: ".Dumper($vtmout)."\n") if (defined $vtmout);
2318
2319    my $mac;
2320    if ($cmt !~ /^VE/) {
2321        $mac = "$sshlogin->{$ENV{'PBPROJ'}}\@$remhost";
2322        # Overwrite account value if passed as parameter or forced
2323        $mac = "$ENV{'PBACCOUNT'}\@$remhost" if (defined $ENV{'PBACCOUNT'});
2324    } else {
2325        # VE
2326        $mac = $sshlogin->{$ENV{'PBPROJ'}};
2327        # Overwrite account value if passed as parameter (typically for setup2v)
2328        $mac = $ENV{'PBACCOUNT'} if (defined $ENV{'PBACCOUNT'});
2329    }
2330    pb_log(2, "DEBUG: mac: $mac\n");
2331
2332    my $tdir;
2333    my $bdir;
2334    if (($cmt eq "Sources") || ($cmt =~ /(V[EM]|RM)Script/)) {
2335        $tdir = "$remdir/$delivery->{$ENV{'PBPROJ'}}/src";
2336    } elsif ($cmt eq "CPAN") {
2337        $tdir = "$remdir";
2338    } elsif ($cmt =~ /(V[EM]|RM)(build|test|prep)/) {
2339        $tdir = $remdir."/$ENV{'PBPROJ'}/pbdelivery";
2340        $bdir = $remdir."/$ENV{'PBPROJ'}/pbbuild";
2341        # Remove a potential $ENV{'HOME'} as bdir should be relative to pb's home
2342        $bdir =~ s|\$ENV.+\}/||;
2343    } elsif ($cmt eq "Announce") {
2344        $tdir = "$remdir/$delivery->{$ENV{'PBPROJ'}}";
2345    } elsif ($cmt eq "Web") {
2346        $tdir = "$remdir/$delivery->{$ENV{'PBPROJ'}}";
2347    } elsif ($cmt eq "Packages") {
2348        if (($pbos->{'type'} eq "rpm") || ($pbos->{'type'} eq "pkg") || ($pbos->{'type'} eq "hpux") || ($pbos->{'type'} eq "tgz")) {
2349            # put packages under an arch subdir
2350            $tdir = "$remdir/$delivery->{$ENV{'PBPROJ'}}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}";
2351        } elsif (($pbos->{'type'} eq "deb") || ($pbos->{'type'} eq "ebuild")) {
2352            # No need for an arch subdir
2353            $tdir = "$remdir/$delivery->{$ENV{'PBPROJ'}}/$pbos->{'name'}/$pbos->{'version'}";
2354        } else {
2355            die "Please teach the dev team where to deliver ($pbos->{'type'} type of packages\n";
2356        }
2357
2358        my $repodir = $tdir;
2359        $repodir =~ s|^$remdir/||;
2360        my $repotag = "";
2361        $repotag = "-$delivery->{$ENV{'PBPROJ'}}" if ($delivery->{$ENV{'PBPROJ'}} ne "");
2362
2363        my ($pbrepo) = pb_conf_get("pbrepo");
2364
2365        # Script for Repository management
2366        $v = "$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}" if (not defined $v);
2367        $pbscript{$v} = "$ENV{'PBBUILDDIR'}/pbscript.$$";
2368       
2369        # That script should be part of the delivery
2370        $src = "$src $pbscript{$v}";
2371        open(PBS,"> $pbscript{$v}") || die "Unable to create $pbscript{$v}";
2372        if ($pbos->{'type'} eq "rpm") {
2373            my $pbsha = pb_distro_get_param($pbos,pb_conf_get("ossha"));
2374            my ($gpgcheck) = pb_conf_get_if("pbgpgcheck");
2375            my $pbgpgcheck;
2376            $pbgpgcheck = $gpgcheck->{$ENV{'PBPROJ'}} if (defined $gpgcheck);
2377            my $pbgpgkey = "$pbrepo->{$ENV{'PBPROJ'}}/$repodir/$ENV{'PBPROJ'}.pubkey";
2378            my ($gpgaddpkey) = pb_conf_get_if("pbadditionalgpg");
2379            my $pbadditionalgpg;
2380            $pbadditionalgpg = $gpgaddpkey->{$ENV{'PBPROJ'}} if (defined $gpgaddpkey);
2381            if (defined $pbadditionalgpg) {
2382                foreach my $k (split(/,/,$pbadditionalgpg)) {
2383                    $pbgpgkey .= " $pbrepo->{$ENV{'PBPROJ'}}/$repodir/$ENV{'PBPROJ'}-$k.pubkey";
2384                }
2385            }
2386            # By default force GPG check in repo even if we support signature of packages to fail. This is a best practice
2387            $pbgpgcheck = 1 if (not defined $pbgpgcheck);
2388            # Also make a pbscript to generate dnf/yum/urpmi bases
2389            print PBS << "EOF";
2390#!/bin/bash
2391# Prepare a script to ease dnf/yum setup
2392EOF
2393            print PBS "set -x\n" if ($pbdebug gt 1);
2394            print PBS << "EOF";
2395cat > $ENV{'PBPROJ'}$repotag.repo << EOT
2396[$ENV{'PBPROJ'}$repotag]
2397name=$pbos->{'name'} $pbos->{'version'} $pbos->{'arch'} - $ENV{'PBPROJ'} Vanilla Packages
2398baseurl=$pbrepo->{$ENV{'PBPROJ'}}/$repodir
2399enabled=1
2400gpgcheck=$pbgpgcheck
2401gpgkey=$pbgpgkey
2402EOT
2403chmod 644 $ENV{'PBPROJ'}$repotag.repo
2404
2405# Clean up old repo content
2406rm -rf headers/ repodata/
2407# Create yum repo
2408if [ -x /usr/bin/yum-arch ]; then
2409    yum-arch .
2410fi
2411# Create repodata
2412createrepo -s $pbsha .
2413# Link to the key
2414if [ -s $ENV{'PBPROJ'}.pubkey ]; then
2415    # Avoiding creating empty repomd.xml.key; fails on opensuse 12.1
2416    (cd repodata ; ln -sf ../$ENV{'PBPROJ'}.pubkey repomd.xml.key)
2417fi
2418# sign the repomd (at least useful for SLES - which requires a local key)
2419# gpg -a --detach-sign repodata/repomd.xml
2420# SLES also looks for media.1/info.txt
2421EOF
2422            if ($pbos->{'family'} eq "md") {
2423                # For Mandriva add urpmi management
2424                print PBS << "EOF";
2425# Prepare a script to ease urpmi setup
2426cat > $ENV{'PBPROJ'}$repotag.addmedia << EOT
2427#rpm --import $pbrepo->{$ENV{'PBPROJ'}}/$repodir/$ENV{'PBPROJ'}.pubkey
2428urpmi.addmedia $ENV{'PBPROJ'}$repotag $pbrepo->{$ENV{'PBPROJ'}}/$repodir with media_info/hdlist.cz
2429EOT
2430chmod 755 $ENV{'PBPROJ'}$repotag.addmedia
2431
2432# Clean up old repo content
2433rm -f hdlist.cz synthesis.hdlist.cz
2434# Create urpmi repo
2435genhdlist2 --clean .
2436if [ \$\? -ne 0 ]; then
2437    genhdlist .
2438fi
2439(cd media_info ; ln -sf ../$ENV{'PBPROJ'}.pubkey pubkey)
2440EOF
2441            }
2442            if ($pbos->{'name'} eq "fedora") {
2443                # Extract the spec file to please Fedora maintainers :-(
2444                print PBS << "EOF";
2445for p in $basesrc; do
2446    echo \$p | grep -q 'src.rpm'
2447    if [ \$\? -eq 0 ]; then
2448        rpm2cpio \$p | cpio -ivdum --quiet '*.spec'
2449    fi
2450done
2451EOF
2452            }
2453            if ($pbos->{'family'} eq "novell") {
2454                # Add ymp scripts for one-click install on SuSE
2455                print PBS << "EOF";
2456# Prepare a script to ease SuSE one-click install
2457# Cf: http://de.opensuse.org/1-Klick-Installation/ISV
2458#
2459cat > $ENV{'PBPROJ'}$repotag.ymp << EOT
2460<?xml version="1.0" encoding="utf-8"?>
2461<!-- vim: set sw=2 ts=2 ai et: -->
2462<metapackage xmlns:os="http://opensuse.org/Standards/One_Click_Install" xmlns="http://opensuse.org/Standards/One_Click_Install">
2463    <group><!-- The group of software, typically one for project-builder.org -->
2464        <name>$ENV{'PBPROJ'} Bundle</name> <!-- Name of the software group -->
2465        <summary>Software bundle for the $ENV{'PBPROJ'}$repotag project</summary> <!--This message is shown to the user and should describe the whole bundle -->
2466        <description>This is the summary of the $ENV{'PBPROJ'}$repotag Project
2467             
2468            Details are available on a per package basis below
2469
2470        </description><!--This is also shown to the user -->
2471        <remainSubscribed>false</remainSubscribed> <!-- Don't know what it mean -->
2472        <repositories><!-- List of needed repositories -->
2473            <repository>
2474                <name>$ENV{'PBPROJ'}$repotag Repository</name> <!-- Name of the repository  -->
2475                <summary>This repository contains the $ENV{'PBPROJ'}$repotag project packages.</summary> <!-- Summary of the repository -->
2476                <description>This repository contains the $ENV{'PBPROJ'}$repotag project packages.</description><!-- This description is shown to the user -->
2477                <url>$pbrepo->{$ENV{'PBPROJ'}}/$repodir</url><!--URL of repository, which is added -->
2478            </repository>
2479        </repositories>
2480        <software><!-- A List of packages, which should be added through the one-click-installation -->
2481EOT
2482for p in $basesrc; do
2483    sum=`rpm -q --qf '%{SUMMARY}' \$p`
2484    name=`rpm -q --qf '%{NAME}' \$p`
2485    desc=`rpm -q --qf '%{description}' \$p`
2486    cat >> $ENV{'PBPROJ'}$repotag.ymp << EOT
2487            <item>
2488                <name>\$name</name><!-- Name of the package, is shown to the user and used to identify the package at the repository -->
2489                <summary>\$sum</summary> <!-- Summary of the package -->
2490                <description>\$desc</description> <!-- Description, is shown to the user -->
2491            </item>
2492EOT
2493done
2494cat >> $ENV{'PBPROJ'}$repotag.ymp << EOT
2495        </software>
2496    </group>
2497</metapackage>
2498EOT
2499chmod 644 $ENV{'PBPROJ'}$repotag.ymp
2500EOF
2501            }
2502        } elsif ($pbos->{'type'} eq "deb") {
2503            # Also make a pbscript to generate apt bases
2504            # Cf: http://www.debian.org/doc/manuals/repository-howto/repository-howto.fr.html
2505            # This dirname removes ver
2506            my $debarch = pb_get_debarch($pbos);
2507            my $projcomponent = pb_get_debpc($pbos);
2508            my $rpd = dirname("$pbrepo->{$ENV{'PBPROJ'}}/$repodir");
2509            # Remove extra . in path to fix #522
2510            $rpd =~ s|/./|/|g;
2511            print PBS << "EOF";
2512#!/bin/bash
2513# Prepare a script to ease apt setup
2514cat > $ENV{'PBPROJ'}$repotag.sources.list << EOT
2515deb $rpd $pbos->{'version'} $projcomponent
2516deb-src $rpd $pbos->{'version'} $projcomponent
2517EOT
2518chmod 644 $ENV{'PBPROJ'}$repotag.sources.list
2519
2520# Up two levels to deal with the dist dir cross versions
2521cd ..
2522mkdir -p dists/$pbos->{'version'}/$projcomponent/binary-$debarch dists/$pbos->{'version'}/$projcomponent/source dists/$pbos->{'version'}/$projcomponent/binary-all
2523
2524# Prepare a script to create apt info file
2525# Reuse twice after
2526TMPD=`mktemp -d /tmp/pb.XXXXXXXXXX` || exit 1
2527mkdir -p \$TMPD
2528cat > \$TMPD/Release << EOT
2529Archive: unstable
2530Component: $projcomponent
2531Origin: $ENV{'PBPROJ'}$repotag
2532Label: $ENV{'PBPROJ'}$repotag dev repository $pbrepo->{$ENV{'PBPROJ'}}
2533EOT
2534
2535echo "Creating Packages metadata"
2536for i in dists/$pbos->{version}/$projcomponent/binary-*; do
2537    arch=`basename \$i | sed 's/binary-//'`
2538    echo "Packages for \$arch:"
2539    dpkg-scanpackages -a\$arch $pbos->{'version'} /dev/null > dists/$pbos->{'version'}/$projcomponent/binary-\$arch/Packages
2540    gzip -9 < dists/$pbos->{'version'}/$projcomponent/binary-\$arch/Packages > dists/$pbos->{'version'}/$projcomponent/binary-\$arch/Packages.gz
2541    bzip2 -9 < dists/$pbos->{'version'}/$projcomponent/binary-\$arch/Packages > dists/$pbos->{'version'}/$projcomponent/binary-\$arch/Packages.bz2
2542done
2543echo "Creating Contents metadata"
2544apt-ftparchive contents $pbos->{'version'} | gzip -c9 > dists/$pbos->{'version'}/Contents.gz
2545echo "Creating Release metadata ($pbos->{'arch'} aka $debarch)"
2546cat \$TMPD/Release > dists/$pbos->{'version'}/$projcomponent/binary-all/Release
2547echo "Architecture: all" >> dists/$pbos->{'version'}/$projcomponent/binary-all/Release
2548cat \$TMPD/Release > dists/$pbos->{'version'}/$projcomponent/binary-$debarch/Release
2549echo "Architecture: $debarch" >> dists/$pbos->{'version'}/$projcomponent/binary-$debarch/Release
2550echo "Creating Source metadata"
2551dpkg-scansources $pbos->{'version'} /dev/null > dists/$pbos->{'version'}/$projcomponent/source/Sources
2552gzip -9 < dists/$pbos->{'version'}/$projcomponent/source/Sources > dists/$pbos->{'version'}/$projcomponent/source/Sources.gz
2553bzip2 -9 < dists/$pbos->{'version'}/$projcomponent/source/Sources > dists/$pbos->{'version'}/$projcomponent/source/Sources.bz2
2554cat \$TMPD/Release > dists/$pbos->{'version'}/$projcomponent/source/Release
2555echo "Architecture: Source" >> dists/$pbos->{'version'}/$projcomponent/source/Release
2556echo "Creating Release metadata"
2557apt-ftparchive -o APT::FTPArchive::Release::Suite=$pbos->{version} release dists/$pbos->{'version'} > dists/$pbos->{'version'}/Release
2558rm -rf \$TMPD
2559EOF
2560        } elsif ($pbos->{'type'} eq "ebuild") {
2561            # make a pbscript to generate links to latest version
2562            print PBS << "EOF";
2563#!/bin/bash
2564# Prepare a script to create correct links
2565cd $tdir
2566for p in $src; do
2567    echo \$p | grep -q '.ebuild'
2568    if [ \$\? -eq 0 ]; then
2569        j=`basename \$p`
2570        pp=`echo \$j | perl -pe 's/-[0-9\.]+-r[0-9].*\.ebuild/.ebuild/'`
2571        ln -sf \$j \$pp
2572    fi
2573done
2574EOF
2575        }
2576        close(PBS);
2577        chmod 0755,$pbscript{$v};
2578    } else {
2579        return;
2580    }
2581
2582    # Useless for VE
2583    my $nport = pb_get_port($sshport,$pbos,$cmt) if ($cmt !~ /^VE/);
2584
2585    # Remove a potential $ENV{'HOME'} as tdir should be relative to pb's home
2586    if ($cmt =~ /^VE/o) {
2587        $tdir =~ s|\$ENV.+\}/|| or confess "for $cmt: $tdir must have \$ENV{'HOME'} in it or the remainder of the code won't work since some parts use relative pathing and others will try absolute.  Use a symlink if necessary to place the real files in a different place than under your home directory";
2588    } else {
2589        # process, ok if not present.
2590        $tdir =~ s|\$ENV.+\}/||;
2591    }
2592
2593    my $tm = "";
2594    if ($cmt =~ /^(V|R)M/) {
2595        $tm = "sleep $vtmout" if (defined $vtmout);
2596    }
2597
2598    # ssh communication if not VE or CPAN
2599    # should use a hash instead...
2600    my ($shcmd,$shcmdroot,$cpcmd,$cptarget,$cp2target);
2601    my $tpdir;
2602    my $tp;
2603    my $context = "$ENV{'PBTMP'}";
2604    my %tag;
2605    my $dkaccount = "";
2606    my $cmd1 = "";
2607    my $cidfile;
2608    my $homedir = "";
2609    if ($cmt =~ /^VE/) {
2610        $tp = pb_path_expand($vepath->{$ENV{'PBPROJ'}});
2611        $vetype = pb_ve_get_type($vetype);
2612        if ($vetype eq "docker") {
2613            $tpdir = "/";
2614            $cidfile = "$context/ctn-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}.cid";
2615        } else {
2616            $tpdir = pb_path_expand("$tp/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}");
2617        }
2618        my $arch = pb_get_arch();
2619        if ($vetype eq "chroot") {
2620            $shcmdroot = "sudo /usr/sbin/chroot $tpdir ";
2621            $shcmd = "$shcmdroot /bin/su - $mac -c ";
2622        } elsif ($vetype eq "schroot") {
2623            $shcmd = "schroot $tp -u $mac -- ";
2624        } elsif ($vetype eq "docker") {
2625            # docker manages the storage so rely on it
2626            $shcmdroot = "";
2627            my $docrepo = pb_ve_docker_repo();
2628            $cmd1 = pb_check_req("docker",0);
2629            #my ($dockeropt) =  pb_conf_get_if("dockeropt");
2630            # pbimage is used for docker and is setup to the right name depending on the step:
2631            # TODO: This is not coded yet
2632            my $pbimage = undef;
2633            # step 0 : nothing at creation -> tag n-v-a (made in VE.pm)
2634            # step 1 : n-v-a + setup -> tag n-v-a-pb
2635            # step 2 : n-v-a-pb + build -> tag n-v-a-pb-pbproj
2636            # step 3 : n-v-a-pb-pbproj at use
2637            if ((not defined $pbimage) || ($pbimage eq "")) {
2638                # If no image name given, create a naming convention
2639                $tag{1} = "$docrepo$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
2640                $tag{2} = "$tag{1}-pb";
2641                $tag{3} = "$tag{2}-$ENV{'PBPROJ'}";
2642            } else {
2643                # If we were given an image name, just use it
2644                $tag{1} = $pbimage;
2645                $tag{2} = $pbimage;
2646                $tag{3} = $pbimage;
2647            }
2648            # If we do not create the image, then use the one we should have
2649            # TODO: we don't do anything with found !!
2650            my $found = pb_ve_docker_get_image($tag{$pbstep});
2651            confess "No image $tag{$pbstep} available in Docker, please create one first\n" if (not defined $found);
2652            # Now we use that image to do what is needed
2653            # use a dockerfile to ease the creation of next images
2654            # the context is in an empty temp dir
2655            my $tmpd = "$context/Dockerfile";
2656            open(DOCKER, "> $tmpd") || die "Unable to create the docker file $tmpd";
2657            print DOCKER "FROM $tag{$pbstep}\n";
2658            print DOCKER "MAINTAINER project-builder.org aka pb\n";
2659            if ($pbstep == 1) {
2660                # setup done as root
2661                $dkaccount = "root";
2662            } else {
2663                # Rest done as pb
2664                $dkaccount = "pb";
2665            }
2666            print DOCKER "USER $dkaccount\n";
2667            if ($pbstep <= 2) {
2668                $shcmd = "$cmd1 build -t $tag{$pbstep+1} $context";
2669            } else {
2670                # As we are in run phase use docker run. cmd will be completed below
2671                $shcmd = "$cmd1 run --cidfile=\"$cidfile\" --user $dkaccount --name $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-$cmt";
2672            }
2673            #$shcmd = "$cmd1 build $dockeropt->{$ENV{'PBPROJ'}} -t $tag{$pbstep+1} $context";
2674        }
2675        $shcmd = "setarch i386 $shcmd" if (($pbos->{'arch'} =~ /i[3456]86/) && ($arch eq 'x86_64'));
2676        if (($cmt =~ /VE/) && ($vetype ne "docker")) {
2677            # We need to get the home dir of the target account to deliver in the right place
2678            open(PASS,"$tpdir/etc/passwd") || die "Unable to open $tpdir/etc/passwd: $!";
2679            while (<PASS>) {
2680                my ($c1,$c2,$c3,$c4,$c5,$c6,$c7) = split(/:/);
2681                $homedir = $c6 if ($c1 =~ /^$mac$/);
2682                pb_log(3,"Homedir: $homedir - account: $c6\n");
2683            }
2684            close(PASS);
2685            $cpcmd = "sudo /bin/cp -r ";
2686        }
2687        if (($cmt =~ /VE/) && ($vetype eq "docker")) {
2688            if ($pbstep >= 2) {
2689                # We need to get the home dir of the target account to deliver in the right place
2690                $homedir = `$cmd1 run --cidfile="$cidfile" $tag{$pbstep} grep -E '^$dkaccount:' /etc/passwd | cut -d: -f6`;
2691                chomp($homedir);
2692                open(CID,"$cidfile") || confess "Unable to open $cidfile";
2693                my $cid = <CID>;
2694                close(CID);
2695                pb_system("$cmd1 rm $cid","","quiet");
2696                unlink("$cidfile");
2697            } else {
2698                $homedir = "/";
2699            }
2700            $cpcmd = "cp -r ";
2701        }
2702        $cptarget = "$tpdir/$homedir/$tdir";
2703        if ($cmt =~ /VE(build|prep)/) {
2704            $cp2target = "$tpdir/$homedir/$bdir";
2705        }
2706        pb_log(2,"On VE using $cptarget as target dir to copy to\n");
2707    } elsif ($cmt eq "CPAN") {
2708        my $ftpput = pb_check_req("ncftpput",1);
2709        my $ftpget = pb_check_req("wget",1);
2710        my ($cpanuser,$cpanpasswd) = pb_conf_get_if("cpanuser","cpanpasswd");
2711        return if ((not defined $cpanuser) || (not defined $cpanuser->{$ENV{'PBPROJ'}}));
2712        return if ((not defined $cpanpasswd) || (not defined $cpanpasswd->{$ENV{'PBPROJ'}}));
2713        my ($cpansubdir) = pb_conf_get_if("cpansubdir");
2714        $shcmd = "$ftpget --post-data \'HIDDENNAME=".$cpanuser->{$ENV{'PBPROJ'}};
2715        $shcmd .= "&user=".$cpanuser->{$ENV{'PBPROJ'}};
2716        $shcmd .= "&password=".$cpanpasswd->{$ENV{'PBPROJ'}};
2717        $shcmd .= "&SUBMIT_pause99_add_uri_upload=\"Upload the checked files\"";
2718        $shcmd .= "&pause99_add_uri_subdirtext=".$cpansubdir->{$ENV{'PBPROJ'}} if ((defined $cpansubdir) && (defined $cpansubdir->{$ENV{'PBPROJ'}}));
2719        foreach my $s (split(/ /,$src)) {
2720            $shcmd .= "&pause99_add_uri_upload=".basename($s);
2721        }
2722        $shcmd .= "'";
2723        $cpcmd = "$ftpput $host $dir";
2724        $cptarget = "CPAN";
2725    } else {
2726        my $keyopt = "";
2727        if ($cmt !~ /Packages/) {
2728            my $keyfile = pb_ssh_get(0,$remhost,$nport);
2729            $keyopt = "-i $keyfile" if (defined $keyfile);
2730        }
2731        my $sshcmd = pb_check_req("ssh",1);
2732        my $scpcmd = pb_check_req("scp",1);
2733        $shcmd = "$sshcmd $keyopt -q -o NoHostAuthenticationForLocalhost=yes -p $nport $mac";
2734        $cpcmd = "$scpcmd $keyopt -p -o NoHostAuthenticationForLocalhost=yes -P $nport";
2735        $cptarget = "$mac:$tdir";
2736        if ($cmt =~ /^(V|R)M(build|prep)/) {
2737            $cp2target = "$mac:$bdir";
2738        }
2739    }
2740   
2741    my $logres = "";
2742    # Do not touch when just announcing
2743    if (($cmt ne "Announce") && ($cmt ne "CPAN")) {
2744        # For docker everything is done in the Dockerfile for steps < 3
2745        if (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep < 3)) {
2746            print DOCKER "RUN mkdir -p $cptarget\n";
2747            print DOCKER "RUN cd $cptarget ; for i in $basesrc; do if [ -f \$i ]; then rm -f \$i; fi; done\n";
2748        } elsif (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep >= 3)) {
2749            # Nothing to do here
2750        } else {
2751            pb_system("$shcmd \"mkdir -p $tdir ; cd $tdir ; echo \'for i in $basesrc; do if [ -f \$i ]; then rm -f \$i; fi; done ; $cmd\' | bash -e\"","Preparing $tdir on $cptarget");
2752        }
2753    } else {
2754        $logres = "> ";
2755    }
2756
2757    if ($cmt =~ /SandBox/) {
2758        # Install from sandbox mean using the result of the just passed sbx2build command
2759        # Get content saved in cms2build
2760        my ($pkg) = pb_conf_read("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml","pbpkg");
2761        my $pbextdir = pb_get_extdir();
2762        confess "Unable to get package list" if (not defined $pkg);
2763
2764        # We consider 2 specific packages
2765        my $vertag1 = $pkg->{"ProjectBuilder"};
2766        my $vertag2 = $pkg->{"project-builder"};
2767        # get the version of the current package - maybe different
2768        pb_log(2,"Vertag1: $vertag1\n");
2769        pb_log(2,"Vertag2: $vertag2\n");
2770        my ($pbver1,$tmp1) = split(/-/,$vertag1);
2771        my ($pbver2,$tmp2) = split(/-/,$vertag2);
2772        if ($cmt =~ /VE/) {
2773            # Copy inside the VE
2774            if ($vetype ne "docker") {
2775                copy("$ENV{'PBDESTDIR'}/ProjectBuilder-$pbver1$pbextdir.tar.gz","$tpdir/tmp");
2776                copy("$ENV{'PBDESTDIR'}/project-builder-$pbver2$pbextdir.tar.gz","$tpdir/tmp");
2777            } else {
2778                copy("$ENV{'PBDESTDIR'}/ProjectBuilder-$pbver1$pbextdir.tar.gz","$context");
2779                copy("$ENV{'PBDESTDIR'}/project-builder-$pbver2$pbextdir.tar.gz","$context");
2780                print DOCKER "COPY ProjectBuilder-$pbver1$pbextdir.tar.gz /tmp/\n";
2781                print DOCKER "COPY project-builder-$pbver2$pbextdir.tar.gz /tmp/\n";
2782            }
2783        } else {
2784            pb_system("$cpcmd $ENV{'PBDESTDIR'}/ProjectBuilder-$pbver1$pbextdir.tar.gz $ENV{'PBDESTDIR'}/project-builder-$pbver2$pbextdir.tar.gz $mac:/tmp","Copying local project files to VM/RM");
2785        }
2786    }
2787           
2788    pb_log(0,"Sources handled ($cmt): $src\n");
2789    # For docker everything is done in the Dockerfile
2790    if (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep < 3)) {
2791        foreach my $f (split(/ +/,$src)) {
2792            copy("$f","$context");
2793            print DOCKER "COPY ".basename($f)." $cptarget/\n";
2794        }
2795        print DOCKER "RUN cd $cptarget ; $cmd\n" if ((defined $cmd) && ($cmd ne ""));
2796    } elsif (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep >= 3)) {
2797        # Nothing to do
2798    } else {
2799        pb_system("cd $ENV{'PBBUILDDIR'} ; $cpcmd $src $cptarget 2> /dev/null","$cmt delivery in $cptarget");
2800    }
2801
2802    # For VE we need to change the owner manually
2803    if ($cmt =~ /^VE/) {
2804        if ($vetype ne "docker") {
2805            my $sudomode = pb_distro_get_param($pbos,pb_conf_get("ossudoersmode"));
2806            my $res = pb_system("$shcmdroot sed -i '/requiretty/d' /etc/sudoers","Removing potential requiretty in sudoers","quiet");
2807            pb_system("$shcmdroot sed '/requiretty/d' /etc/sudoers > /tmp/sudoers.new ; mv /tmp/sudoers.new $tpdir/tmp/sudoers.new ; $shcmdroot mv /tmp/sudoers.new /etc/sudoers ; $shcmdroot chown root:root /etc/sudoers ; $shcmdroot chmod $sudomode /etc/sudoers","Removing again potential requiretty in sudoers as sed -i failed") if (($res ne 0) && (-f "$tpdir/etc/sudoers"));
2808            pb_system("$shcmd \"sudo chown -R $mac $tdir\"","Adapt owner in $tdir to $mac");
2809        } else {
2810            print DOCKER "RUN sed -i '/requiretty/d' /etc/sudoers\n" if ($pbstep == 1);
2811        }
2812    }
2813
2814    # Use the right script name depending on context
2815    my $pbscript2;
2816    if (($cmt =~ /^(V[EM]|RM)/) || ($cmt =~ /Packages/)){
2817        $pbscript2 = $pbscript{$v};
2818    } else {
2819        $pbscript2 = "pbscript";
2820    }
2821
2822    # It's already ready for CPAN
2823    my $shcmdbase = $shcmd;
2824    my $dkcmd;
2825    if ($cmt ne "CPAN") {
2826        if (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep < 3)) {
2827            copy("$pbscript2","$context");
2828            my $s = basename($pbscript2);
2829            print DOCKER "COPY $s $cptarget/\n";
2830            # call directly with perl to avoid text file busy error messages
2831            print DOCKER "RUN cd $cptarget ; perl ./$s ; if [ '$pbkeep' = '0' ]; then rm -f $s; fi\n";
2832            print DOCKER "RUN rm -rf $cptarget/*\n";
2833            print DOCKER "RUN chown -R $dkaccount $cptarget\n" if ($pbstep == 1);
2834            close(DOCKER);
2835        } elsif (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep >= 3)) {
2836            $dkcmd = "/bin/bash $homedir/$tdir-2/".basename($pbscript2);
2837        } else {
2838            my $s = basename($pbscript2);
2839            $shcmd .= " \"echo \'cd $tdir ; if [ -x $s ]; then ./$s; fi ;  if [ $pbkeep == 0 ]; then rm -f $s; fi\' | bash\"";
2840        }
2841    }
2842    my $cmdverb = "verbose";
2843    if (($cmt eq "Announce") || ($cmt eq "CPAN")) {
2844        $cmdverb = undef;
2845    } elsif (defined ($v)) {
2846        $cmdverb = "verbose_\[$v\] ";
2847    }
2848    # this is where we launch the execution
2849    if (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep >= 3)) {
2850        pb_mkdir_p("$ENV{'PBTMP'}/vebuild");
2851        my $tsrc = "";
2852        foreach my $f (split(/ +/,$src)) {
2853            copy("$f","$ENV{'PBTMP'}/vebuild");
2854            $tsrc .= " ".basename($f);
2855        }
2856        my $ts = "$ENV{'PBTMP'}/vebuild/".basename($pbscript2);
2857        open(DOCKER," > $ts") || confess "Unable to create $ts";
2858        print DOCKER "#!/bin/bash\n";
2859        print DOCKER "# Put content from host to guest, copying to have the correct uid/gid and rights to do so\n";
2860        print DOCKER "# set -x\n";
2861        print DOCKER "for i in $tsrc; do\n";
2862        print DOCKER "  # Cleanup first to avoid rights issues for root owned files e.g.\n";
2863        print DOCKER "  rm -f $homedir/$tdir/\$i\n";
2864        print DOCKER "  cp $homedir/$tdir-2/\$i $homedir/$tdir\n";
2865        print DOCKER "done\n";
2866        print DOCKER "(cd $homedir/$tdir-2 ; ls -al)\n";
2867        print DOCKER pb_get_content($pbscript2);
2868        print DOCKER "\n# Get content back from guest in a host dir\n";
2869        # Host dir seen as TDIR in container
2870        $cp2target = "$ENV{'PBBUILDDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
2871        pb_mkdir_p("$cp2target");
2872        chmod 0777,"$cp2target";
2873        print DOCKER "TDIR=$homedir/$bdir-2/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}\n";
2874        print DOCKER "for i in $homedir/$bdir/*; do\n";
2875        print DOCKER "  # Cleanup first to avoid rights issues for root owned files e.g.\n";
2876        print DOCKER "  j=\`basename \$i\`\n";
2877        print DOCKER "  rm -f \$TDIR/\$j\n";
2878        print DOCKER "  cp -r \$i \$TDIR\n";
2879        print DOCKER "done\n";
2880        print DOCKER "find \$TDIR -type f -print0 | xargs -0 chmod 644\n";
2881        print DOCKER "find \$TDIR -type d -print0 | xargs -0 chmod 777 2> /dev/null\n";
2882        #print DOCKER "ls -al \$TDIR\n";
2883        print DOCKER "exit 0\n";
2884        close(DOCKER);
2885        $shcmd .= " -v $ENV{'PBBUILDDIR'}:$homedir/$bdir-2 -v $ENV{'PBTMP'}/vebuild:$homedir/$tdir-2 -w $homedir/$tdir-2 $tag{$pbstep} $dkcmd";
2886    }
2887    my $ret = pb_system("$shcmd","Executing pbscript on $cptarget if needed",$cmdverb);
2888   
2889    # If target is deb family then sign Release file for Packages
2890    my $pbos2 = pb_distro_get_context($v);
2891    if (($cmt =~ /Packages/) && ($pbos2->{'type'} eq "deb")) {
2892        my $debarch = pb_get_debarch($pbos2);
2893        my $projcomponent = pb_get_debpc($pbos2);
2894        for my $f ("dists/$pbos2->{'version'}/$projcomponent/binary-$debarch/Release","dists/$pbos2->{'version'}/$projcomponent/binary-all/Release","dists/$pbos2->{'version'}/$projcomponent/source/Release","dists/$pbos2->{'version'}/Release") {
2895            my $debdir = dirname($cptarget);
2896            pb_system("cd $ENV{'PBBUILDDIR'} ; $cpcmd $debdir/$f . 2> /dev/null","Getting deb $f file from $debdir");
2897            pb_sign_file("$ENV{'PBBUILDDIR'}/Release");
2898            pb_system("$cpcmd $ENV{'PBBUILDDIR'}/Release.gpg $debdir/$f.gpg 2> /dev/null","Putting deb $f.gpg file to $debdir");
2899            my ($void,$debfile) = split(/:/,$debdir);
2900            pb_system("$shcmdbase \"chmod 644 $debfile/$f.gpg\"","Set mode to 644 for $debfile/$f.gpg");
2901        }
2902    } 
2903
2904    if ($cmt =~ /^(V[EM]|RM)build/) {
2905        # Get back info on pkg produced, compute their name and get them from the VM/RM
2906        pb_system("$cpcmd $cp2target/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'} $ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.$$ 2> /dev/null","Get package names in $cp2target");
2907        if (not -f "$ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.$$") {
2908            pb_log(0,"ERROR with VM/RM $v on getting $ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.$$\n");
2909        } else {
2910            open(KEEP,"$ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.$$") || die "Unable to read $ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.$$";
2911            my $src = <KEEP>;
2912            chomp($src);
2913            close(KEEP);
2914            unlink("$ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.$$");
2915
2916            $src =~ s/^ *//;
2917            pb_mkdir_p("$ENV{'PBBUILDDIR'}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}");
2918            # Change pgben to make the next send2target happy
2919            my $made = "";
2920   
2921            # For VM/RM we don't want shell expansion to hapen locally but remotely
2922            my $delim = '\'';
2923            if ($cmt =~ /^VEbuild/) {
2924                # For VE we need to support shell expansion locally
2925                $delim = "";
2926            }   
2927
2928            open(KEEP,"> $ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}") || die "Unable to write $ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
2929            foreach my $p (split(/ +/,$src)) {
2930                my $j = basename($p);
2931                pb_system("$cpcmd $cp2target/$delim$p$delim $ENV{'PBBUILDDIR'}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'} 2> /dev/null","Recovery of package $j in $ENV{'PBBUILDDIR'}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}");
2932                $made="$made $pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}/$j"; # if (($pbos->{'type'} ne "rpm") || ($j !~ /.src.rpm$/));
2933            }
2934       
2935            print KEEP "$made\n";
2936            close(KEEP);
2937            if (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep >= 3)) {
2938                pb_rm_rf("$cp2target");
2939                pb_system("$cmd1 rm $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-$cmt","","quiet");
2940            } else {
2941                pb_system("$shcmdbase \"rm -rf $tdir $bdir\"","$cmt cleanup");
2942            }
2943
2944            # Sign packages locally
2945            pb_sign_pkgs($pbos,$made);
2946
2947            # We want to send them to the ssh account so overwrite what has been done before
2948            pb_log(2,"Before sending pkgs, vexist: $vexist, vpid: $vpid\n");
2949            pb_send2target("Packages","$ENV{'PBDESTDIR'}/pbscript.$$",$pbos->{'name'}."-".$pbos->{'version'}."-".$pbos->{'arch'},$vexist,$vpid);
2950            pb_rm_rf("$ENV{'PBBUILDDIR'}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}");
2951        }
2952    }
2953    unlink("$pbscript{$v}") if ((($cmt =~ /^(V[ME]|RM)/) || ($cmt =~ /Packages/)) && ($pbkeep eq 0) && ($cmt !~ /Script/));
2954
2955    pb_log(2,"Before halt, vexist: $vexist, vpid: $vpid\n");
2956    if ((! $vexist) && ($cmt =~ /^VM/)) {
2957        # If in setupvm then takes a snapshot just before halting
2958        if ($snapme != 0) {
2959            my ($vmmonport,$vmtype) = pb_conf_get("vmmonport","vmtype");
2960            # For monitoring control
2961            if ((($vmtype->{$ENV{'PBPROJ'}}) eq "kvm") || (($vmtype->{$ENV{'PBPROJ'}}) eq "qemu")) {
2962                eval
2963                {
2964                require Net::Telnet;
2965                Net::Telnet->import();
2966                };
2967                if ($@) {
2968                    # Net::Telnet not found
2969                    pb_log(1,"ADVISE: Install Net::Telnet to benefit from monitoring control and snapshot feature.\nWARNING: No snapshot created");
2970                } else {
2971                    my $t = new Net::Telnet (Timeout => 120, Host => "localhost", Port => $vmmonport->{$ENV{'PBPROJ'}}) || die "Unable to dialog on the monitor";
2972                    # move to monitor mode
2973                    my @lines = $t->cmd("c");
2974                    # Create a snapshot named pb
2975                    @lines = $t->cmd("savevm pb");
2976                    # Write the new status in the VM
2977                    @lines = $t->cmd("commit all");
2978                    # End
2979                    @lines = $t->cmd("quit");
2980                }
2981            }
2982        }
2983        my $hoption = "-p";
2984        my $hpath = pb_distro_get_param($pbos,pb_conf_get("ospathcmd-halt"));
2985        # Solaris doesn't support -p of halt
2986        if ($pbos->{'type'} eq "pkg") {
2987            $hoption = "" ;
2988        }
2989        pb_system("$shcmdbase \"sudo $hpath $hoption \"; $tm ; echo \'if [ -d /proc/$vpid ]; then kill -9 $vpid; fi \' | bash ; sleep 10","VM $v halt (pid $vpid)");
2990    }
2991    if (($cmt =~ /^VE/) && ($snapme != 0)) {
2992        if ($vetype eq "docker") {
2993            foreach my $f (split(/ +/,$src)) {
2994                unlink("$context/$f");
2995            }
2996        } else {
2997            pb_ve_snap($pbos,$tp);
2998        }
2999    }
3000}
3001
3002sub pb_script2v {
3003
3004my $pbscript=shift;
3005my $vtype=shift;
3006my $pbstep=shift;   # Which step are we in (0: create, 1: setup pb, 2: prep pbproj, 3: use to build)
3007my $pbforce=shift;  # Force stop of VM. Default not.
3008my $snapme=shift;   # Do we have to create a snapshot. Default not.
3009my $pbimage=shift;  # Which image to use to start the VM/VE
3010
3011$pbforce = 0 if (not defined $pbforce);
3012$snapme = 0 if (not defined $snapme);
3013$pbstep = 3 if (not defined $pbstep);
3014
3015pb_log(2,"DEBUG: pb_script2v($vtype,$pbstep,$pbforce,$snapme)\n");
3016pb_log(2,"DEBUG: pb_script2v(pbscript: ".Dumper($pbscript)."\n") if (defined $pbscript);
3017pb_parallel_launchv($pbscript,$vtype,uc($vtype)."Script",$pbstep,$pbimage,$pbforce,$snapme,$pbsnap);
3018}
3019
3020sub pb_parallel_launchv {
3021
3022my $pbscript=shift;
3023my $vtype = shift;
3024my $action = shift;         # It an action is defined then use send2target
3025my $pbstep=shift;           # Which step are we in (0: create, 1: setup, 2: build 3: use)
3026my $pbimage=shift;          # Which image to use to start the VM/VE
3027my $pbforce=shift;          # Force stop of VM. Default not.
3028my $snapme = shift;         # By default do not snap a VM/VE/RM
3029my $usesnap = shift;        # By default study the usage of the snapshot feature of VM/VE/RM   
3030my $vm;
3031my $all;
3032
3033$pbforce = 0 if (not defined $pbforce);
3034$snapme = 0 if (not defined $snapme);
3035$usesnap = 1 if (not defined $usesnap);
3036
3037# Adapt // mode to memory size
3038$pbparallel = pb_set_parallel($vtype);
3039
3040pb_log(2,"DEBUG: pb_parallel_launch(vtype,step,force,snapme,usesnap: $vtype,$pbstep,$pbforce,$snapme,$usesnap)\n");
3041pb_log(2,"DEBUG: pb_parallel_launch(pbscript: ".Dumper($pbscript)."\n");
3042
3043my ($vexist,$vpid) = (undef,undef);
3044my $pm;
3045my $all_ok = 1;
3046if (defined $pbparallel) {
3047    $pm = new Parallel::ForkManager($pbparallel);
3048
3049    # Set which port the VM/RM will use to communicate
3050    $pm->run_on_start(\&pb_set_port);
3051    $pm->run_on_finish(sub { my ($pid, $code, $id, $signal, $dump) = @_;
3052                            unless ($code == 0 && $signal == 0 && $dump == 0) {
3053                                $all_ok = 0;
3054                                pb_log(0,"ERROR: pid $pid failed\n");
3055                            }
3056                        });
3057}
3058
3059# Determine on which V to work
3060if (defined $ENV{'PBV'}) {
3061    @$vm = split(/,/,$ENV{'PBV'});
3062} else {
3063    ($vm,$all) = pb_get2v($vtype);
3064}
3065
3066my $counter = 0;
3067foreach my $v (@$vm) {
3068    $counter++;
3069    # Modulo 2 * pbparallel (to avoid synchronization problems)
3070    $counter = 1 if ((defined $pbparallel) && ($counter > 2 * $pbparallel));
3071    $pm->start($counter) and next if (defined $pbparallel);
3072   
3073    # If launched via script2v then get the script passed as the one for the that VM
3074    if (not defined $pbscript->{$v}) {
3075        $pbscript->{$v} = $pbscript->{'default'};
3076    }
3077    #
3078    # Launch a single operation
3079    ($vexist,$vpid) = pb_launchv($pbscript->{$v},$vtype,$v,$action,$pbstep,$pbforce,$snapme,$pbsnap,$pbimage);
3080   
3081    # Skip that VM/RM if something went wrong
3082    if (($vtype =~ /(v|r)m/) && ($vpid == 0) && ($vexist == 0)) {
3083        $pm->finish if (defined $pbparallel);
3084        next;
3085    }
3086
3087   
3088    $pm->finish if (defined $pbparallel);
3089}
3090$pm->wait_all_children if (defined $pbparallel);
3091die "Aborting, one or more of the children failed." if ((not $all_ok) && ($Global::pb_stop_on_error));
3092}
3093
3094sub pb_launchv {
3095
3096my $pbscript1=shift;        # Just the script name to execute, not the hash
3097my $vtype = shift;
3098my $v = shift;              # Only 1 VM/VE/RM treated here
3099my $action = shift;         # It an action is defined then use send2target
3100my $pbstep=shift;           # Which step are we in (0: create, 1: setup, 2: build 3: use)
3101my $pbforce=shift;          # Force stop of VM. Default not.
3102my $snapme = shift;         # By default do not snap a VM/VE/RM
3103my $usesnap = shift;            # By default study the usage of the snapshot feature of VM/VE/RM   
3104my $pbimage=shift;          # Which image to use to start the VM/VE
3105my $vexist = undef;
3106my $vpid = undef;
3107
3108$pbforce = 0 if (not defined $pbforce);
3109$snapme = 0 if (not defined $snapme);
3110$usesnap = 1 if (not defined $usesnap);
3111
3112# Check that v exists
3113die "No VM/VE/RM defined, unable to launch" if (not defined $v);
3114
3115# If creation or snapshot creation mode, no snapshot usable
3116if (($pbstep == 0) || ($snapme == 1)) {
3117    $usesnap = 0;
3118}
3119
3120pb_log(2,"DEBUG: pb_launchv(vtype,v,step,force,snapme,usesnap: $vtype,$v,$pbstep,$pbforce,$snapme,$usesnap)\n");
3121pb_log(2,"DEBUG: pb_launchv(pbimage: $pbimage)\n") if (defined $pbimage);
3122pb_log(2,"DEBUG: pb_launchv(pbscript $pbscript1)\n") if (defined $pbscript1);
3123pb_log(2,"DEBUG: pb_launchv(action: $action)\n") if (defined $action);
3124# Keep only the first VM in case many were given
3125if ($v =~ /,/) {
3126    pb_log(0,"WARNING: pruning to just the first of several VM/VE/RMs listed ($v)\n");
3127    $v =~ s/,.*//;
3128}
3129
3130my $pbos = pb_distro_get_context($v);
3131
3132pb_apply_conf_proxy($pbos);
3133
3134# Launch the VMs/VEs
3135if ($vtype eq "vm") {
3136    confess "-i image parameter needed" if (((not defined $pbimage) || ($pbimage eq "")) && ($pbstep == 0));
3137
3138    my ($ptr,$ptr2,$vmpath,$vmport,$vms) = pb_conf_get("vmtype","vmcmd","vmpath","vmport","vmsize");
3139    my ($vmopt,$vmmm,$vmtmout,$vmsnap,$vmbuildtm,$vmmonport) = pb_conf_get_if("vmopt","vmmem","vmtmout","vmsnap","vmbuildtm","vmmonport");
3140    my $vmsize = pb_distro_get_param($pbos,$vms);
3141
3142    my $vmtype = $ptr->{$ENV{'PBPROJ'}};
3143    my $vmcmd = $ptr2->{$ENV{'PBPROJ'}};
3144
3145    if (defined $opts{'g'}) {
3146        if (($vmtype eq "kvm") || ($vmtype eq "qemu")) {
3147            $ENV{'PBVMOPT'} = "--nographic";
3148        }
3149    }
3150    if (not defined $ENV{'PBVMOPT'}) {
3151        $ENV{'PBVMOPT'} = "";
3152    }
3153    # Save the current status for later restoration
3154    $ENV{'PBOLDVMOPT'} = $ENV{'PBVMOPT'};
3155
3156    if (defined $vmopt->{$v}) {
3157        $ENV{'PBVMOPT'} .= " $vmopt->{$v}" if ($ENV{'PBVMOPT'} !~ / $vmopt->{$v}/);
3158    } elsif (defined $vmopt->{$ENV{'PBPROJ'}}) {
3159        $ENV{'PBVMOPT'} .= " $vmopt->{$ENV{'PBPROJ'}}" if ($ENV{'PBVMOPT'} !~ / $vmopt->{$ENV{'PBPROJ'}}/);
3160    }
3161
3162    # How much memory to allocate for VMs
3163    if (defined $vmmm) {
3164        my $vmmem = pb_distro_get_param($pbos,$vmmm);
3165        if (defined $vmmem) {
3166            $ENV{'PBVMOPT'} .= " -m $vmmem";
3167        }
3168    }
3169
3170    # Are we allowed to use snapshot feature
3171    if ($usesnap == 1) {
3172        if ((defined $vmsnap->{$v}) && ($vmsnap->{$v} =~ /true/i)) {
3173            $ENV{'PBVMOPT'} .= " -snapshot";
3174        } elsif ((defined $vmsnap->{$ENV{'PBPROJ'}}) && ($vmsnap->{$ENV{'PBPROJ'}} =~ /true/i)) {
3175            $ENV{'PBVMOPT'} .= " -snapshot";
3176        } elsif ($pbsnap eq 1) {
3177            $ENV{'PBVMOPT'} .= " -snapshot";
3178        }
3179    } 
3180    if ($snapme != 0) {
3181        if (($vmtype eq "kvm") || ($vmtype eq "qemu")) {
3182            # Configure the monitoring to automate the creation of the 'pb' snapshot
3183            $ENV{'PBVMOPT'} .= " -serial mon:telnet::$vmmonport->{$ENV{'PBPROJ'}},server,nowait" if ((defined $vmmonport) && (defined $vmmonport->{$ENV{'PBPROJ'}}));
3184            # In that case no snapshot call needed
3185            $ENV{'PBVMOPT'} =~ s/ -snapshot//;
3186        }
3187    }
3188   
3189    # Set a default timeout of 2 minutes if none defined externally or in conf files
3190    if (not defined $ENV{'PBVMTMOUT'}) {
3191        if (defined $vmtmout->{$v}) {
3192            $ENV{'PBVMTMOUT'} = $vmtmout->{$v};
3193        } elsif (defined $vmtmout->{$ENV{'PBPROJ'}}) {
3194            $ENV{'PBVMTMOUT'} = $vmtmout->{$ENV{'PBPROJ'}};
3195        }
3196    }
3197    if (not defined $ENV{'PBVMTMOUT'}) {
3198        $ENV{'PBVMTMOUT'} = "120";
3199    }
3200
3201    my $nport = pb_get_port($vmport,$pbos,$vtype);
3202
3203    my $cmd;
3204    my $vmm;        # has to be used for pb_check_ps
3205    if (($vmtype eq "qemu") || ($vmtype eq "kvm")) {
3206        $vmm = "$vmpath->{$ENV{'PBPROJ'}}/$v.qemu";
3207        if (($pbstep == 0) || (defined $pbimage)) {
3208            $ENV{'PBVMOPT'} .= " -cdrom $pbimage -boot d";
3209        }
3210        # Always redirect the network and always try to use a 'pb' snapshot
3211        #$cmd = "$vmcmd $ENV{'PBVMOPT'} -net nic,model=virtio -net user,hostfwd=tcp::$nport:22 -loadvm pb $vmm"
3212        #$cmd = "$vmcmd $ENV{'PBVMOPT'} -redir tcp:$nport:10.0.2.15:22 $vmm"
3213        $cmd = "$vmcmd $ENV{'PBVMOPT'} -net nic -net user,hostfwd=tcp:127.0.0.1:$nport-:22 $vmm"
3214    } elsif ($vmtype eq "xen") {
3215    } elsif ($vmtype eq "vmware") {
3216    } else {
3217        die "VM of type $vmtype not supported. Report to the dev team";
3218    }
3219    # Restore the ENV VAR Value
3220    $ENV{'PBVMOPT'} = $ENV{'PBOLDVMOPT'};
3221
3222    my ($tmpcmd,$void) = split(/ +/,$cmd);
3223    my $vmmport = undef;
3224    ($vexist,$vmmport) = pb_check_ps($tmpcmd,$vmm);
3225    $vpid = 0;
3226   
3227    if (! $vexist) {
3228        if ($pbstep == 0) {
3229            die("Found an existing Virtual machine $vmm. Won't overwrite") if (-r $vmm);
3230            if (($vmtype eq "qemu") || ($vmtype eq "xen") || ($vmtype eq "kvm")) {
3231                my $command = pb_check_req("qemu-img",0);
3232                pb_system("$command create -f qcow2 $vmm $vmsize","Creating the QEMU VM");
3233            } elsif ($vmtype eq "vmware") {
3234            } else {
3235            }
3236        }
3237        if (! -f "$vmm") {
3238            pb_log(0,"Unable to find VM $vmm\n");
3239        } else {
3240            # Is the SSH port free? if not kill the existing process using it after a build timeout period
3241            my ($vmssh,$void) = pb_check_ps($tmpcmd,$vmm);
3242            if ($vmssh) {
3243                my $buildtm = $ENV{'PBVMTMOUT'};
3244                if (defined $vmbuildtm->{$v}) {
3245                    $buildtm = $vmbuildtm->{$v};
3246                } elsif (defined $vmbuildtm->{$ENV{'PBPROJ'}}) {
3247                    $buildtm = $vmbuildtm->{$ENV{'PBPROJ'}};
3248                }
3249
3250                sleep $buildtm;
3251                pb_log(0,"WARNING: Killing the process ($vmssh) using port $void (previous failed VM ?)\n");
3252                kill 15,$vmssh;
3253                # Let it time to exit
3254                sleep 5;
3255            }
3256            pb_system("$cmd &","Launching the VM $vmm");
3257            # Using system allows to kill it externaly if needed,sosupport that in the call
3258            pb_system("sleep $ENV{'PBVMTMOUT'}","Waiting $ENV{'PBVMTMOUT'} s for VM $v to come up","mayfail");
3259            ($vpid,$void) = pb_check_ps($tmpcmd,$vmm);
3260            pb_log(0,"VM $vmm launched (pid $vpid)\n");
3261        }
3262    } else {
3263        pb_log(0,"Found an existing VM $vmm (pid $vexist - Port $vmmport)\n");
3264        # Set the correct port here based on what is done
3265        $pbport = $vmmport;
3266        $vpid = $vexist;
3267    }
3268    pb_log(2,"DEBUG: pb_launchv returns ($vexist,$vpid)\n");
3269
3270    # After the VM is launched, then setup the SSH access if not in creation phase
3271    if ($pbstep != 0) {
3272        if (defined $pbport) {
3273            pb_ssh_setup($vtype,$pbport);
3274        } else {
3275            pb_ssh_setup($vtype,$nport);
3276        }
3277    }
3278   
3279    # Skip that VM if something went wrong
3280    return($vexist,$vpid) if (($vpid == 0) && ($vexist == 0));
3281   
3282    # If force stopping the VM then reset vexist
3283    if ($pbforce == 1) {
3284        $vpid = $vexist;
3285        $vexist = 0;
3286    }
3287} elsif ($vtype eq "ve") {
3288    # Force the creation of the VE and no snapshot usable
3289    $vetype = pb_ve_get_type($vetype);
3290    pb_ve_launch($v,$pbstep,$usesnap,$vetype,$pbimage);
3291    $vexist = 0;
3292    $vpid = 0;
3293} else {
3294    # Get RM context
3295    my ($ptr,$rmpath,$rmport) = pb_conf_get("rmtype","rmpath","rmport");
3296
3297    # Nothing more to do for RM. No real launch
3298    # For the moment we support the RM is already running
3299    # For ProLiant may be able to power them on if needed later on as an example.
3300    $vexist = 0;
3301    $vpid = 0;
3302   
3303    # After the RM is launched, then setup the SSH access
3304    my $nport = pb_get_port($rmport,$pbos,$vtype);
3305    pb_ssh_setup($vtype,$nport);
3306}
3307
3308# Gather all required files to send them to the VM/VE/RM
3309# and launch the build through pbscript
3310if (defined $action) {
3311    pb_log(2,"DEBUG: Before send2target, vexist: $vexist, vpid: $vpid, vtype: $vtype, v: $v, action: $action\n");
3312    pb_send2target($action,$pbscript1,$v,$vexist,$vpid,$snapme,$pbstep);
3313}
3314return($vexist,$vpid);
3315}
3316
3317# Return string for date synchro
3318sub pb_date2v {
3319
3320my $vtype = shift;
3321my $pbos = shift;
3322my $nontp = shift || 0;
3323
3324pb_log(1,"Entering pb_date2v\n");
3325# VE gets time from parent OS.
3326return "/bin/true" if ($vtype) =~ /^ve/o; 
3327
3328my ($ntp) = pb_conf_get_if($vtype."ntp");
3329my $vntp = undef;
3330my $ntpline = undef;
3331$vntp = $ntp->{$ENV{'PBPROJ'}} if (defined $ntp);
3332
3333if ((defined $vntp) && ($vntp !~ /^\s*$/)) {
3334    pb_log(2,"ntp server is $vntp\n");
3335    # ntp command depends on pbos
3336    my $vntpcmd = pb_distro_get_param($pbos,pb_conf_get("oscmdntp"));
3337    $ntpline = "sudo $vntpcmd $vntp";
3338}
3339# Force new date to be in the future compared to the date
3340# of the host by adding 1 minute
3341my @date=pb_get_date();
3342$date[1]++;
3343my $upddate = strftime("%m%d%H%M%Y", @date);
3344my $dateline = "sudo /bin/date $upddate";
3345if ((defined $ntpline) && ($nontp == 0)) {
3346    pb_log(1,"pb_date2v returns $ntpline\n");
3347    return($ntpline);
3348} else {
3349    pb_log(1,"pb_date2v returns $dateline\n");
3350    return($dateline);
3351}
3352}
3353
3354sub pb_build2v {
3355
3356my $vtype = shift;
3357my $action = shift;     #build, test or prep
3358my $pbforce=shift;      # Force stop of VM. Default not.
3359
3360$action = "build" if (not defined $action);
3361$pbforce = 0 if (not defined $pbforce);
3362
3363my $pbstep;
3364
3365$vetype = pb_ve_get_type($vetype);
3366if ($action eq "prep") {
3367    $pbstep = 2;
3368} else {
3369    $pbstep = 3;
3370    # Need to be traversed by others if using containers
3371    if (($vtype eq "ve") && ($vetype eq "docker")) {
3372        chmod 0751,"$ENV{'PBBUILDDIR'}";
3373    }
3374}
3375
3376pb_log(1,"Entering in pb_build2v ($action in $vtype) \n");
3377
3378# We prepare scripts for all V of this vtype to allow sharing that part
3379my ($v,$all) = pb_get2v($vtype);
3380
3381# Send tar files when we do a global generation
3382pb_build2ssh() if (($all == 1) && ($action eq "build"));
3383
3384# Adapt // mode to memory size
3385$pbparallel = pb_set_parallel($vtype);
3386
3387my ($vexist,$vpid) = (undef,undef);
3388my $pm;
3389my $all_ok = 1;
3390if (defined $pbparallel) {
3391    $pm = new Parallel::ForkManager($pbparallel);
3392
3393    # Set which port the VM/RM will use to communicate
3394    $pm->run_on_start(\&pb_set_port);
3395    $pm->run_on_finish(sub { my ($pid, $code, $id, $signal, $dump) = @_;
3396                            unless ($code == 0 && $signal == 0 && $dump == 0) {
3397                                $all_ok = 0;
3398                                pb_log(0,"ERROR: pid $pid failed\n");
3399                            }
3400                        });
3401}
3402
3403my $counter = 0;
3404foreach my $v (@$v) {
3405    $counter++;
3406    $pbscript{$v} = "$ENV{'PBDESTDIR'}/pb$action-$v";
3407    # Modulo 2 * pbparallel (to avoid synchronization problems)
3408    $counter = 1 if ((defined $pbparallel) && ($counter > 2 * $pbparallel));
3409    $pm->start($counter) and next if (defined $pbparallel);
3410    #
3411    # Prepare the script to be executed on the VM/VE/RM
3412    open(SCRIPT,"> $pbscript{$v}") || die "Unable to create $pbscript{$v}";
3413    print SCRIPT "#!/bin/bash\n";
3414
3415    # Transmit the verbosity level to the virtual env/mach.
3416    my $verbose = "";
3417    my $i = 0;                          # minimal debug level
3418    while ($i lt $pbdebug) {
3419        $verbose .= "-v ";
3420        $i++;
3421    }
3422    print SCRIPT "set -e\n" if ($Global::pb_stop_on_error);
3423    # Activate script verbosity if at least 2 for pbdebug
3424    print SCRIPT "set -x\n" if ($i gt 1);
3425    # Quiet if asked to be so on the original system
3426    $verbose = "-q" if ($pbdebug eq -1);
3427
3428    print SCRIPT "echo ... Execution needed\n";
3429    print SCRIPT "echo ==== Start of script for $vtype $v =====\n";
3430    print SCRIPT "# This is in directory pbdelivery\n";
3431    print SCRIPT "# Setup the variables required for building\n";
3432    print SCRIPT "export PBPROJ=$ENV{'PBPROJ'}\n";
3433
3434    if (($action eq "build") || ($action eq "prep")) {
3435        print SCRIPT "# Preparation for pb\n";
3436        print SCRIPT "rm -f \$HOME/.pbrc.yml\n";
3437        print SCRIPT "rm -f \$HOME/.pbrc\n";
3438        print SCRIPT "cp .pbrc.yml \$HOME\n";
3439        # TODO: Why ?
3440        print SCRIPT "cd ..\n";
3441    }
3442
3443    # VE needs a good /proc, tolerate one being potentially left around after a failure
3444    if (($vtype eq "ve") && ($vetype ne "docker")) {
3445        print SCRIPT "[ -d /proc/1 ] || sudo /bin/mount -t proc /proc /proc\n";
3446    }
3447
3448    # Get distro context
3449    my $pbos = pb_distro_get_context($v);
3450   
3451    my $ntpline = pb_date2v($vtype,$pbos);
3452    print SCRIPT "# Time sync\n";
3453    print SCRIPT "echo setting up date with $ntpline\n";
3454    print SCRIPT "$ntpline\n";
3455    print SCRIPT "if [ \$\? -ne 0 ]; then\n";
3456    # Force if bad status a trick using date
3457    $ntpline = pb_date2v($vtype,$pbos, 1);
3458    print SCRIPT "  $ntpline\n";
3459    print SCRIPT "fi\n";
3460    # Use potential local proxy declaration in case we need it to download repo, pkgs, ...
3461    if (defined $ENV{'http_proxy'}) {
3462        print SCRIPT "export http_proxy=\"$ENV{'http_proxy'}\"\n";
3463    }
3464    if (defined $ENV{'https_proxy'}) {
3465        print SCRIPT "export https_proxy=\"$ENV{'https_proxy'}\"\n";
3466    }
3467    if (defined $ENV{'ftp_proxy'}) {
3468        print SCRIPT "export ftp_proxy=\"$ENV{'ftp_proxy'}\"\n";
3469    }
3470
3471    # Get list of packages to build/test and get some ENV vars as well
3472    my $ptr = pb_get_pkg();
3473    @pkgs = @$ptr;
3474    my $p = join(' ',@pkgs) if (@pkgs);
3475    print SCRIPT "export PBPROJVER=$ENV{'PBPROJVER'}\n";
3476    print SCRIPT "export PBPROJTAG=$ENV{'PBPROJTAG'}\n";
3477    print SCRIPT "export PBPACKAGER=\"$ENV{'PBPACKAGER'}\"\n";
3478
3479    # We may need to do some other tasks before building. Read a script here to finish setup
3480    if (-x "$ENV{'PBTMP'}/pb$vtype".".pre") {
3481        print SCRIPT "# Special pre-instructions to be launched\n";
3482        print SCRIPT pb_get_content("$ENV{'PBTMP'}/pb$vtype".".pre");
3483    }
3484
3485    if (-x "$ENV{'PBTMP'}/pb$vtype"."$action.pre") {
3486        print SCRIPT "# Special pre-$action instructions to be launched\n";
3487        print SCRIPT pb_get_content("$ENV{'PBTMP'}/pb$vtype"."$action.pre");
3488    }
3489
3490    print SCRIPT "# $action\n";
3491    print SCRIPT "echo $action"."ing packages on $vtype...\n";
3492
3493    if (($action eq "test") && (! -x "$ENV{'PBDESTDIR'}/pbtest")) {
3494        confess "No test script ($ENV{'PBDESTDIR'}/pbtest) found when in test mode. Aborting ...";
3495    }
3496    print SCRIPT "pb --version\n";
3497    my $act = "$action"."2pkg";
3498    $act = "build2prep" if ($action eq "prep");
3499    print SCRIPT "pb $verbose -p $ENV{'PBPROJ'} $act $p\n";
3500
3501    if (($vtype eq "ve") && ($vetype ne "docker")) {
3502        print SCRIPT "sudo /bin/umount /proc\n";
3503    }
3504
3505    # We may need to do some other tasks after building. Read a script here to exit properly
3506    if (-x "$ENV{'PBTMP'}/pb$vtype"."$action.post") {
3507        print SCRIPT "# Special post-$action instructions to be launched\n";
3508        print SCRIPT pb_get_content("$ENV{'PBTMP'}/pb$vtype"."$action.post");
3509    }
3510
3511    if (-x "$ENV{'PBTMP'}/pb$vtype".".post") {
3512        print SCRIPT "# Special post-instructions to be launched\n";
3513        print SCRIPT pb_get_content("$ENV{'PBTMP'}/pb$vtype".".post");
3514    }
3515
3516    print SCRIPT "echo ==== End of script $$ for $vtype $v =====\n";
3517    close(SCRIPT);
3518    chmod 0755,"$pbscript{$v}";
3519   
3520    $pm->finish if (defined $pbparallel);
3521}
3522$pm->wait_all_children if (defined $pbparallel);
3523die "Aborting, one or more of the children failed." if ((not $all_ok) && ($Global::pb_stop_on_error));
3524
3525# Launch the VM/VE/RM
3526# docker image to use if needed
3527# Gather all required files to send them to the VM/VE
3528# and launch the build through pbscript
3529pb_parallel_launchv(\%pbscript,$vtype,uc($vtype).$action,$pbstep,$pbimage,$pbforce);
3530}
3531
3532
3533sub pb_clean {
3534
3535    my $sleep=10;
3536    die "Unable to get env var PBDESTDIR" if (not defined $ENV{'PBDESTDIR'});
3537    die "Unable to get env var PBBUILDDIR" if (not defined $ENV{'PBBUILDDIR'});
3538    pb_log(0,"We will now wait $sleep s before removing both directories\n$ENV{'PBDESTDIR'} and $ENV{'PBBUILDDIR'}.\nPlease break me if this is wrong\n");
3539    sleep $sleep;
3540    pb_rm_rf($ENV{'PBDESTDIR'});
3541    pb_rm_rf($ENV{'PBBUILDDIR'});
3542}
3543
3544sub pb_newver {
3545
3546    die "-V Version parameter needed" if ((not defined $newver) || ($newver eq ""));
3547
3548    # Need this call for PBDIR
3549    my ($scheme2,$uri) = pb_vcs_init($pbinit);
3550
3551    my ($pbconf,$pbprojurl) = pb_conf_get("pbconfurl","pbprojurl");
3552    $uri = $pbconf->{$ENV{'PBPROJ'}};
3553    my ($scheme, $account, $host, $port, $path) = pb_get_uri($uri);
3554
3555    # Checking CMS repositories status
3556    ($scheme2, $account, $host, $port, $path) = pb_get_uri($pbprojurl->{$ENV{'PBPROJ'}});
3557
3558    if ($scheme !~ /^svn/) {
3559        die "Only SVN is supported at the moment";
3560    }
3561
3562    my $res = pb_vcs_isdiff($scheme,$ENV{'PBROOTDIR'});
3563    die "ERROR: No differences accepted in CMS for $ENV{'PBROOTDIR'} before creating a new version" if ($res != 0);
3564
3565    $res = pb_vcs_isdiff($scheme2,$ENV{'PBDIR'});
3566    die "ERROR: No differences accepted in CMS for $ENV{'PBDIR'} before creating a new version" if ($res != 0);
3567
3568    # Tree identical between PBCONFDIR and PBROOTDIR. The delta is what
3569    # we want to get for the root of the new URL
3570
3571    my $oldver = $ENV{'PBROOTDIR'};
3572    $oldver =~ s|^$ENV{'PBCONFDIR'}||;
3573
3574    pb_log(2, "PBCONFDIR: $ENV{'PBCONFDIR'}\nPBROOTDIR: $ENV{'PBROOTDIR'}\n");
3575
3576    my $newurl = "$uri/$newver";
3577    # Should probably use projver in the old file
3578    my $oldvertxt= basename($oldver);
3579    my $newvertxt = basename($newver);
3580
3581    # Duplicate and extract project-builder part
3582    pb_log(2,"Copying $uri/$oldver to $newurl\n");
3583    pb_vcs_copy($scheme,"$uri/$oldver",$newurl);
3584    pb_log(2,"Checkout $newurl to $ENV{'PBCONFDIR'}/$newver\n");
3585    pb_vcs_up($scheme,"$ENV{'PBCONFDIR'}");
3586
3587    # Duplicate and extract project
3588    my $newurl2 = "$pbprojurl->{$ENV{'PBPROJ'}}/$newver";
3589
3590    pb_log(2,"Copying $pbprojurl->{$ENV{'PBPROJ'}}/$oldver to $newurl2\n");
3591    pb_vcs_copy($scheme2,"$pbprojurl->{$ENV{'PBPROJ'}}/$oldver",$newurl2);
3592
3593    my $tmp = $ENV{'PBDIR'};
3594    $tmp =~ s|$oldver$||;
3595    pb_log(2,"Checkout $newurl2 to $tmp/$newver\n");
3596    pb_vcs_up($scheme2,"$tmp");
3597
3598    # Update the .yml file
3599    open(FILE,"$ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml") || die "Unable to open $ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml";
3600    open(OUT,"> $ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml.new") || die "Unable to write to $ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml.new";
3601    while(<FILE>) {
3602        if (/^projver\s+$ENV{'PBPROJ'}\s*=\s*$oldvertxt$/) {
3603            s/^projver\s+$ENV{'PBPROJ'}\s*=\s*$oldvertxt$/projver $ENV{'PBPROJ'} = $newvertxt/;
3604            pb_log(0,"Changing projver from $oldvertxt to $newvertxt in $ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml\n");
3605        }
3606        if (/^testver/) {
3607            s/^testver/#testver/;
3608            pb_log(0,"Commenting testver in $ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml\n") if (/^testver/);
3609        }
3610        if (/^delivery/) {
3611            my $txt = $_;
3612            chomp($txt);
3613            pb_log(0,"Please check delivery ($txt) in $ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml\n");
3614        }
3615        print OUT $_;
3616    }
3617    close(FILE);
3618    close(OUT);
3619    rename("$ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml.new","$ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml");
3620
3621    # Checking pbcl files
3622    foreach my $f (<$ENV{'PBROOTDIR'}/*/pbcl>) {
3623        # Compute new pbcl file
3624        my $f2 = $f;
3625        $f2 =~ s|$ENV{'PBROOTDIR'}|$ENV{'PBCONFDIR'}/$newver/|;
3626        open(PBCL,$f) || die "Unable to open $f";
3627        my $foundnew = 0;
3628        while (<PBCL>) {
3629            $foundnew = 1 if (/^$newvertxt \(/);
3630        }
3631        close(PBCL);
3632        open(OUT,"> $f2") || die "Unable to write to $f2: $!";
3633        open(PBCL,$f) || die "Unable to open $f";
3634        # We skip as long as we do not reach the new version in the log (case of devel)
3635        my $skip = 1;
3636        # We let the 4 first lines
3637        my $cnt = 0;
3638        while (<PBCL>) {
3639            $skip = 0 if (/^$newvertxt \(/);
3640            if ((/^$oldvertxt \(/) && ($foundnew == 0)) {
3641                print OUT "$newvertxt ($pbdate)\n";
3642                print OUT "- TBD\n";
3643                print OUT "\n";
3644                pb_log(0,"WARNING: version $newvertxt not found in $f so added to $f2...\n") if ($foundnew == 0);
3645            }
3646            if ($cnt <= 4) {
3647                print OUT "$_" if (not /^$oldvertxt \(/);
3648            } else {
3649                print OUT "$_" if ($skip == 0);
3650            }
3651            $cnt++;
3652        }
3653        close(OUT);
3654        close(PBCL);
3655    }
3656
3657    pb_log(2,"Checkin $ENV{'PBCONFDIR'}/$newver\n");
3658    pb_vcs_checkin($scheme,"$ENV{'PBCONFDIR'}/$newver","updated to $ENV{'PBCONFDIR'}");
3659}
3660
3661#
3662# Return the list of VMs/VEs/RMs we are working on
3663# $all is a flag to know if we return all of them
3664# or only some (if all we publish also tar files in addition to pkgs
3665#
3666sub pb_get2v {
3667
3668my $vtype = shift;
3669my @v;
3670my $all = 0;
3671my $pbv = 'PBV';
3672my $vlist = $vtype."list";
3673
3674# Get VM/VE list
3675if ((not defined $ENV{$pbv}) || ($ENV{$pbv} =~ /^all$/)) {
3676    my ($ptr) = pb_conf_get($vlist);
3677    $ENV{$pbv} = $ptr->{$ENV{'PBPROJ'}};
3678    $all = 1;
3679}
3680pb_log(2,"$vtype: $ENV{$pbv}\n");
3681@v = split(/,/,$ENV{$pbv});
3682return(\@v,$all);
3683}
3684
3685# This function creates a giant script to configure a particular VM/VE/RM, it then copies the
3686# script to the target.
3687
3688# Function to create a potentialy missing pb account on the VM/VE/RM, and adds it to sudo
3689# Needs to use root account to connect to the VM/VE/RM
3690# pb will take your local public SSH key to access
3691# the pb account in the VM/VE/RM later on if needed
3692sub pb_setup2v {
3693
3694my $vtype = shift;
3695my $sbx = shift;
3696my $pbstep = 1;
3697
3698my ($vm,$all) = pb_get2v($vtype);
3699
3700# Adapt // mode to memory size
3701$pbparallel = pb_set_parallel($vtype);
3702
3703my $pm;
3704my $all_ok = 1;
3705if (defined $pbparallel) {
3706    $pm = new Parallel::ForkManager($pbparallel);
3707
3708    # Set which port the VM/RM will use to communicate
3709    $pm->run_on_start(\&pb_set_port);
3710    $pm->run_on_finish(sub { my ($pid, $code, $id, $signal, $dump) = @_;
3711                            $all_ok = 0 unless (($code == 0) && ($signal == 0) && ($dump == 0)); 
3712                        });
3713}
3714
3715my $counter = 0;
3716foreach my $v (@$vm) {
3717    # Script generated - should be before the fork - after the %pbscript hash isn't updated correctly
3718    $pbscript{$v} = "$ENV{'PBDESTDIR'}/setupv-$v";
3719
3720    $counter++;
3721    if (defined $pbparallel) {
3722        # Modulo 2 * pbparallel (to avoid synchronization problems)
3723        $counter = 1 if ($counter > 2 * $pbparallel);
3724        $pm->start($counter) and next;
3725    }
3726
3727    # Get distro context
3728    my $pbos = pb_distro_get_context($v);
3729   
3730    # Deal with date sync.
3731    my $ntpline = pb_date2v($vtype,$pbos);
3732
3733    # Name of the account to deal with for VM/VE/RM
3734    # Do not use the one passed potentially with -a
3735    my ($pbac) = pb_conf_get($vtype."login");
3736    confess "No $vtype"."login defined in your project yml conf file" if ((not defined $pbac) || (not defined $pbac->{$ENV{'PBPROJ'}}));
3737
3738    my $key = "";
3739
3740    # Prepare the script to be executed on the VM/VE/RM
3741    # in $ENV{'PBDESTDIR'}/setupv
3742
3743    pb_log(2,"Preparing setup script for $v in $pbscript{$v}\n");
3744    open(SCRIPT,"> $pbscript{$v}") || die "Unable to create $pbscript{$v}";
3745   
3746    print SCRIPT << 'EOF';
3747#!/usr/bin/perl -w
3748
3749use strict;
3750use File::Copy;
3751
3752# We should not need in this script more functions than what is provided
3753# by Base, Conf and Distribution to avoid problems at exec time.
3754# They are appended at the end.
3755
3756# Define mandatory global vars
3757EOF
3758    my $ppref = "our";
3759    $ppref = "my" if (($pbos->{'name'} =~ /^redhat/) && ($pbos->{'version'} =~ /^6/));
3760    print SCRIPT << "EOF";
3761$ppref \$pbdebug = $pbdebug;
3762$ppref \$pbLOG;
3763$ppref \$pbsynmsg = "pbscript";
3764$ppref \$pbdisplaytype = "text";
3765$ppref \$pblocale = "";
3766$ppref \$PBCONFVER = $projectbuilderconfver;
3767EOF
3768    print SCRIPT << 'EOF';
3769pb_log_init($pbdebug, $pbLOG);
3770EOF
3771    print SCRIPT << "EOF";
3772\$Global::pb_stop_on_error = $Global::pb_stop_on_error;
3773pb_temp_init($pbkeep);
3774pb_conf_init("$ENV{'PBPROJ'}");
3775
3776EOF
3777
3778    if ($vtype eq "ve") {
3779        $vetype = pb_ve_get_type($vetype);
3780        if ($vetype ne "docker") {
3781            print SCRIPT << "EOF";
3782# For VE we first need to mount some FS
3783pb_system("mount -t proc /proc /proc") unless (-d "/proc/$$");
3784EOF
3785        }
3786        print SCRIPT << "EOF";
3787# For VE we need a good null dev
3788# Except for RedHat 6.2 where it's good and doesn't like being recreated
3789EOF
3790        print SCRIPT "pb_system('rm -f /dev/null; mknod /dev/null c 1 3; chmod 777 /dev/null');\n" unless (($pbos->{'name'} =~ /^redhat/) && ($pbos->{'version'} =~ /^6/));
3791    } elsif ($vtype =~ /(v|r)m/) {
3792        # Nothing to do now.
3793        my $useless = 0;
3794    } else {
3795        die "Unknown virtual type $vtype";
3796    }
3797
3798    print SCRIPT << 'EOF';
3799
3800# Adds $pbac->{$ENV{'PBPROJ'}} as an account if needed
3801#
3802my $file="/etc/passwd";
3803open(PBFILE,$file) || die "Unable to open $file";
3804my $found = 0;
3805while (<PBFILE>) {
3806EOF
3807    print SCRIPT << "EOF";
3808    \$found = 1 if (/^$pbac->{$ENV{'PBPROJ'}}:/);
3809EOF
3810
3811# TODO: use an external parameter
3812my $home = "/home";
3813# Solaris doesn't like that we use /home
3814$home = "/export/home" if ($pbos->{'type'} eq "pkg");
3815
3816    print SCRIPT << "EOF";
3817}
3818close(PBFILE);
3819
3820if ( \$found == 0 ) {
3821    if ( ! -d "$home" ) {
3822        pb_mkdir_p("$home");
3823    }
3824EOF
3825    # TODO: Level of portability of these cmds ? Critical now for RM
3826    # TODO: use an external parameter
3827    print SCRIPT << "EOF";
3828    # Most Linux distros
3829    if ( -x "/usr/sbin/groupadd" ) {
3830        pb_system("/usr/sbin/groupadd $pbac->{$ENV{'PBPROJ'}}","Adding group $pbac->{$ENV{'PBPROJ'}}");
3831    # Alpine with busybox
3832    } elsif ( -x "/usr/sbin/addgroup" ) {
3833        pb_system("/usr/sbin/addgroup $pbac->{$ENV{'PBPROJ'}}","Adding group $pbac->{$ENV{'PBPROJ'}}");
3834    } else {
3835        die "Unable to find a command to add a group on this distribution";
3836    }
3837    # Most Linux distros
3838    if ( -x "/usr/sbin/useradd" ) {
3839        pb_system("/usr/sbin/useradd -g $pbac->{$ENV{'PBPROJ'}} -m -d $home/$pbac->{$ENV{'PBPROJ'}} $pbac->{$ENV{'PBPROJ'}}","Adding user $pbac->{$ENV{'PBPROJ'}} (group $pbac->{$ENV{'PBPROJ'}} - home $home/$pbac->{$ENV{'PBPROJ'}})");
3840    # Alpine with busybox
3841    } elsif ( -x "/usr/sbin/adduser" ) {
3842        pb_system("/usr/sbin/adduser -G $pbac->{$ENV{'PBPROJ'}} -h $home/$pbac->{$ENV{'PBPROJ'}} -D $pbac->{$ENV{'PBPROJ'}}","Adding user $pbac->{$ENV{'PBPROJ'}} (group $pbac->{$ENV{'PBPROJ'}} - home $home/$pbac->{$ENV{'PBPROJ'}})");
3843    } else {
3844        die "Unable to find a command to add a user on this distribution";
3845    }
3846}
3847EOF
3848
3849    # Copy the content of our local conf file to the VM/VE/RM
3850    my $content = pb_get_content(pb_distro_conffile());
3851    print SCRIPT << "EOF";
3852    #
3853    # Create a temporary local conf file for distribution support
3854    # This is created here before its use later. Its place is hardcoded, so no choice for the path
3855    #
3856    my \$tempconf = pb_distro_conffile();
3857    pb_mkdir_p(dirname(\$tempconf));
3858    open(CONF,"> \$tempconf") || die "Unable to create \$tempconf";
3859    print CONF q{$content};
3860    close(CONF);
3861EOF
3862
3863    if ($vtype =~ /(v|r)m/) {
3864        print SCRIPT << "EOF";
3865# allow ssh entry to build
3866#
3867mkdir "$home/$pbac->{$ENV{'PBPROJ'}}/.ssh",0700;
3868# Allow those accessing root to access the build account
3869copy("\$ENV{'HOME'}/.ssh/authorized_keys","$home/$pbac->{$ENV{'PBPROJ'}}/.ssh/authorized_keys");
3870chmod 0600,".ssh/authorized_keys";
3871pb_system("chown -R $pbac->{$ENV{'PBPROJ'}}:$pbac->{$ENV{'PBPROJ'}} $home/$pbac->{$ENV{'PBPROJ'}}","Finish setting up the account env for $pbac->{$ENV{'PBPROJ'}}");
3872
3873EOF
3874}
3875    print SCRIPT << 'EOF';
3876# No passwd for build account only keys
3877$file="/etc/shadow";
3878if (-f $file) {
3879    open(PBFILE,$file) || die "Unable to open $file";
3880    open(PBOUT,"> $file.new") || die "Unable to open $file.new";
3881    while (<PBFILE>) {
3882EOF
3883    print SCRIPT << "EOF";
3884        s/^$pbac->{$ENV{'PBPROJ'}}:\!\!:/$pbac->{$ENV{'PBPROJ'}}:*:/;
3885        s/^$pbac->{$ENV{'PBPROJ'}}:\!:/$pbac->{$ENV{'PBPROJ'}}:*:/; #SLES 9 e.g.
3886        s/^$pbac->{$ENV{'PBPROJ'}}:\\*LK\\*:/$pbac->{$ENV{'PBPROJ'}}:NP:/;  #Solaris e.g.
3887EOF
3888        print SCRIPT << 'EOF';
3889        print PBOUT $_;
3890    }
3891    close(PBFILE);
3892    close(PBOUT);
3893    rename("$file.new",$file);
3894    chmod 0640,$file;
3895    }
3896
3897# TODO: systemd support
3898# Keep the VM in text mode
3899$file="/etc/inittab";
3900if (-f $file) {
3901    open(PBFILE,$file) || die "Unable to open $file";
3902    open(PBOUT,"> $file.new") || die "Unable to open $file.new";
3903    while (<PBFILE>) {
3904        s/^(..):5:initdefault:$/$1:3:initdefault:/;
3905        print PBOUT $_;
3906    }
3907    close(PBFILE);
3908    close(PBOUT);
3909    rename("$file.new",$file);
3910    chmod 0640,$file;
3911}
3912
3913# pb has to be added to portage group on gentoo
3914
3915# We need to have that pb_distro_get_context function
3916# Get it from ProjectBuilder::Distribution
3917# And we now need the conf file required for this to work created above
3918
3919my $pbos = pb_distro_get_context(); 
3920print "distro tuple: ".Dumper($pbos)."\n";
3921
3922# Adapt sudoers
3923# sudo is not default on Solaris and needs to be installed first
3924# from http://www.sunfreeware.com/programlistsparc10.html#sudo
3925if ($pbos->{'type'} eq "pkg") {
3926    $file="/usr/local/etc/sudoers";
3927} else {
3928    $file="/etc/sudoers";
3929}
3930open(PBFILE,$file) || die "Unable to open $file";
3931open(PBOUT,"> $file.new") || die "Unable to open $file.new";
3932while (<PBFILE>) {
3933EOF
3934    # Skip what will be generated
3935    print SCRIPT << "EOF";
3936    next if (/^$pbac->{$ENV{'PBPROJ'}}\\s+/);
3937    next if (/^Defaults:$pbac->{$ENV{'PBPROJ'}}\\s+/);
3938    next if (/^Defaults:root\\s+\!requiretty/);
3939    next if (/^Defaults:root\\s+env_keep/);
3940EOF
3941    print SCRIPT << 'EOF';
3942    s/Defaults\s+requiretty/# disable Defaults: requiretty/;
3943    print PBOUT $_;
3944}
3945close(PBFILE);
3946EOF
3947    print SCRIPT << "EOF";
3948# Some distro force requiretty at compile time, so disable here
3949print PBOUT "Defaults:$pbac->{$ENV{'PBPROJ'}} !requiretty\\n";
3950print PBOUT "Defaults:root !requiretty\\n";
3951EOF
3952    # RH 6.2 sudo doesn't support env_keep
3953    if (($pbos->{'name'} ne "redhat") && ($pbos->{'version'} ne "6.2")) {
3954        print SCRIPT << "EOF";
3955# Keep proxy configuration while using sudo
3956print PBOUT "Defaults:$pbac->{$ENV{'PBPROJ'}}    env_keep += \\\"http_proxy https_proxy ftp_proxy\\\"\\n";
3957print PBOUT "Defaults:root    env_keep += \\\"http_proxy https_proxy ftp_proxy\\\"\\n";
3958EOF
3959    }
3960    # Try to restrict security to what is really needed
3961    if ($vtype =~ /^vm/) {
3962        my $hpath = pb_distro_get_param($pbos,pb_conf_get("ospathcmd-halt"));
3963        my @sudocmds = pb_get_sudocmds($pbos,$ntpline,"sudo $hpath","sudo /bin/date");
3964        print SCRIPT << "EOF";
3965# This is needed in order to be able on VM to halt the machine from the $pbac->{$ENV{'PBPROJ'}} account at least
3966# Build account $pbac->{$ENV{'PBPROJ'}} in VM also needs to setup date and install deps.
3967# Nothing else should be needed
3968EOF
3969        foreach my $c (@sudocmds) {
3970            print SCRIPT "print PBOUT \"$pbac->{$ENV{'PBPROJ'}}   ALL = NOPASSWD: $c\\n\";\n";
3971        }
3972    } elsif ($vtype =~ /^rm/) {
3973        my @sudocmds = pb_get_sudocmds($pbos,$ntpline);
3974        print SCRIPT << "EOF";
3975# Build account $pbac->{$ENV{'PBPROJ'}} in RM only needs to setup date and install deps if needed each time
3976EOF
3977        foreach my $c (@sudocmds) {
3978            print SCRIPT "print PBOUT \"$pbac->{$ENV{'PBPROJ'}}   ALL = NOPASSWD: $c\\n\";\n";
3979        }
3980    } else {
3981        print SCRIPT << "EOF";
3982# Build account $pbac->{$ENV{'PBPROJ'}} for VE needs to do a lot in the host (and chroot), so allow without restriction for now
3983print PBOUT "$pbac->{$ENV{'PBPROJ'}}   ALL=(ALL) NOPASSWD:ALL\n";
3984EOF
3985    }
3986    my $sudomode = pb_distro_get_param($pbos,pb_conf_get("ossudoersmode"));
3987    print SCRIPT << 'EOF';
3988close(PBOUT);
3989rename("$file.new",$file);
3990EOF
3991    print SCRIPT << "EOF";
3992chmod 0$sudomode,\$file;
3993
3994EOF
3995
3996    if ($vtype =~ /(v|r)m/) {
3997        # Sync date
3998        # do it after sudoers is setup
3999        print SCRIPT "my \$res = pb_system(\"$ntpline\",\"Updating time with $ntpline\",\"mayfailverbose\");\n";
4000        print SCRIPT "if (\$res != 0) {\n";
4001        # try again with another method
4002        $ntpline = pb_date2v($vtype,$pbos,1);
4003        print SCRIPT "  pb_system(\"$ntpline\",\"Updating time with $ntpline\",\"mayfailverbose\");\n";
4004        print SCRIPT "}\n";
4005    }
4006
4007    my ($addrepo) = pb_conf_read("$ENV{'PBDESTDIR'}/pbrc.yml","addrepo");
4008    my $pbrepo = pb_distro_get_param($pbos,$addrepo);
4009    print SCRIPT << 'EOF';
4010   
4011# We may need a proxy configuration. Get it from the local env
4012pb_apply_conf_proxy($pbos);
4013
4014# First install all required packages
4015pb_system("dnf clean all","Cleaning dnf env","mayfail") if (($pbos->{'name'} eq "fedora") && ($pbos->{'version'} >= 22));
4016pb_system("yum clean all","Cleaning yum env","mayfail") if ((($pbos->{'name'} eq "fedora") && ($pbos->{'version'} < 22)) || ($pbos->{'name'} eq "asianux") || ($pbos->{'name'} eq "rhel"));
4017my ($ospkgdep) = pb_conf_get_if("ospkgdep");
4018
4019my $pkgdep = pb_distro_get_param($pbos,$ospkgdep);
4020my $forcerepo;
4021EOF
4022    print SCRIPT "\$forcerepo = pb_distro_setuprepo_gen(\$pbos,\"$pbrepo\");\n" if ((defined $pbrepo) && ($pbrepo ne ""));
4023    print SCRIPT << 'EOF';
4024pb_distro_installdeps(undef,$pbos,$forcerepo,pb_distro_only_deps_needed($pbos,join(' ',split(/,/,$pkgdep))));
4025
4026EOF
4027    my $itype = pb_distro_get_param($pbos,pb_conf_get("pbinstalltype"));
4028    # Install from sandbox mean a file base install
4029    $itype = "file" if (defined $sbx);
4030    if ($itype =~ /^file/) {
4031        my $cmdget;
4032        my ($pbver1,$pbver2,$pbextdir);
4033        my ($tmp1,$tmp2);
4034        if (defined $sbx) {
4035            # Install from sandbox mean using the result of the just passed sbx2build command
4036            # Get content saved in cms2build
4037            my ($pkg) = pb_conf_read("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml","pbpkg");
4038            $pbextdir = pb_get_extdir();
4039            die "Unable to get package list" if (not defined $pkg);
4040
4041            # We consider 2 specific packages
4042            my $vertag1 = $pkg->{"ProjectBuilder"};
4043            my $vertag2 = $pkg->{"project-builder"};
4044            # get the version of the current package - maybe different
4045            pb_log(2,"Vertag1: $vertag1\n");
4046            pb_log(2,"Vertag2: $vertag2\n");
4047            ($pbver1,$tmp1) = split(/-/,$vertag1);
4048            ($pbver2,$tmp2) = split(/-/,$vertag2);
4049            $cmdget = "mv /tmp/ProjectBuilder-$pbver1$pbextdir.tar.gz ProjectBuilder-latest.tar.gz ; mv /tmp/project-builder-$pbver2$pbextdir.tar.gz project-builder-latest.tar.gz";
4050        } else {
4051            # TODO: shouldn't we have a variable to specify the source file?
4052            # And shouldn't we be getting the one that matches the current pb version?
4053            # For the moment, just take the latest stable one
4054            my $wget = pb_check_req("wget",0);
4055            $cmdget = "$wget --passive-ftp ftp://ftp.project-builder.org/src/ProjectBuilder-latest.tar.gz; $wget --passive-ftp ftp://ftp.project-builder.org/src/project-builder-latest.tar.gz";
4056        }
4057        print SCRIPT << 'EOF';
4058# Then install manually the missing perl modules
4059my ($osperldep,$osperlver) = pb_conf_get_if("osperldep","osperlver");
4060   
4061my $perldep = pb_distro_get_param($pbos,$osperldep);
4062my $bashopt = "";
4063$bashopt = "-x" if ($pbdebug ge 1);
4064my $verbopt = "mayfail";
4065$verbopt = "mayfailverbose" if ($pbdebug ge 1);
4066foreach my $m (split(/,/,$perldep)) {
4067    # Skip empty deps
4068    next if ($m =~ /^\s*$/);
4069    my $dir = $m;
4070    $dir =~ s/-.*//;
4071    pb_system("echo \"rm -rf $m* ; wget http://search.cpan.org/CPAN/modules/by-module/$dir/$m-$osperlver->{$m}.tar.gz ; gzip -cd $m-$osperlver->{$m}.tar.gz | tar xf - ; cd $m* ; if [ -f Build.PL ]; then perl Build.PL; ./Build ; ./Build install ; else perl Makefile.PL; make ; make install ; fi; cd .. ; rm -rf $m*\" | bash -e $bashopt" ,"Installing perl module $m-$osperlver->{$m}",$verbopt);
4072}
4073EOF
4074        print SCRIPT << 'EOF';
4075pb_system("rm -rf ProjectBuilder-* ; rm -rf project-builder-* ; rm -rf `perl -V:installvendorlib  | awk -F\"'\" '{print \$2}'`/ProjectBuilder ;
4076EOF
4077        print SCRIPT " $cmdget ; ";
4078        print SCRIPT << "EOF"
4079gzip -cd ProjectBuilder-latest.tar.gz | tar xf - ; cd ProjectBuilder-$pbver1$pbextdir ; perl Makefile.PL ; make ; make install ; cd .. ; rm -rf ProjectBuilder-* ; gzip -cd project-builder-latest.tar.gz | tar xf - ; cd project-builder-$pbver2$pbextdir ; perl Makefile.PL ; make ; make install ; cd .. ; rm -rf project-builder-* ;","Building Project-Builder");
4080EOF
4081    } elsif ($itype =~ /^pkg/) {
4082        # pkg based install. We need to point to the project-builder.org repository
4083        print SCRIPT << 'EOF';
4084my $pkgforpb = pb_distro_get_param($pbos,pb_conf_get_if("ospkg"));
4085my $forcerepo = pb_distro_setuposrepo($pbos);
4086pb_log(1,"Forcing full package install with forcerepo\n") if (defined $forcerepo);
4087pb_distro_installdeps(undef,$pbos,$forcerepo,pb_distro_only_deps_needed($pbos,join(' ',split(/,/,$pkgforpb))));
4088EOF
4089    } else {
4090        # Unknown install type
4091        die("Unknown install type $itype->{$ENV{'PBPROJ'}} for param pbinstalltype");
4092    }
4093    print SCRIPT << 'EOF';
4094pb_system("pbdistrocheck",undef,"verbose");
4095pb_system("pb --version",undef,"verbose");
4096EOF
4097    if ($vtype eq "ve"){
4098        if ($vetype ne "docker") {
4099            print SCRIPT << 'EOF';
4100# For VE we need to umount some FS at the end
4101
4102pb_system("umount /proc","Unmounting /proc","mayfail");
4103EOF
4104        }
4105        print SCRIPT << 'EOF';
4106# Create a basic network file if not already there
4107
4108my $nf="/etc/sysconfig/network";
4109if ((! -f $nf) && ($pbos->{'type'} eq "rpm") && ($pbos->{'family'} ne "novell")) {
4110    open(NF,"> $nf") || die "Unable to create $nf";
4111    print NF "NETWORKING=yes\n";
4112    print NF "HOSTNAME=localhost\n";
4113    close(NF);
4114}
4115chmod 0755,$nf;
4116EOF
4117    }
4118
4119    # Adds pb_distro_get_context and all functions needed from ProjectBuilder::Distribution, Conf and Base
4120    foreach my $m ("ProjectBuilder/Base.pm","ProjectBuilder/Distribution.pm","ProjectBuilder/Conf.pm") {
4121        foreach my $d (@INC) {
4122            my $f = "$d/$m";
4123            if (-f "$f") {
4124                open(PBD,"$f") || die "Unable to open $f: $!";
4125                while (<PBD>) {
4126                    next if (/^package/);
4127                    next if (/^use Exporter/);
4128                    next if (/^use ProjectBuilder::/);
4129                    next if (/^our /);
4130                    print SCRIPT $_;
4131                }
4132                close(PBD);
4133                # We just need the first one of each file wrt @INC - hopefully that's the right one.
4134                last;
4135            }
4136        }
4137    }
4138    # Use a fake pb_version_init version here
4139    print SCRIPT << "EOF";
4140sub pb_version_init {
4141
4142return("$projectbuilderver","$projectbuilderrev","$projectbuilderconfver");
4143}
41441;
4145EOF
4146    close(SCRIPT);
4147    chmod 0755,"$pbscript{$v}";
4148
4149    $pm->finish if (defined $pbparallel);
4150}
4151$pm->wait_all_children if (defined $pbparallel);
4152die "Aborting, one or more of the children failed." if ((not $all_ok) && ($Global::pb_stop_on_error));
4153
4154# These build scripts need to be run as root and force stop of VM at end
4155$ENV{'PBACCOUNT'} = "root";
4156
4157$sbx = "" if (not defined $sbx);
4158# Launch the VM/VE/RM
4159pb_log(2,"DEBUG: before parallel launch, pbscript hash is:".Dumper(%pbscript)."\n");
4160pb_parallel_launchv(\%pbscript,$vtype,uc($vtype)."Script".$sbx,$pbstep,$pbimage,$pbforce,0,$pbsnap);
4161
4162return;
4163}
4164
4165# Function to create a snapshot named 'pb' for VMs and a compressed tar for VEs
4166sub pb_snap2v {
4167
4168my $vtype = shift;
4169my $pbstep = 2;
4170
4171my ($vm,$all) = pb_get2v($vtype);
4172
4173# Script generated
4174my $pbscript = "$ENV{'PBDESTDIR'}/snapv.$$";
4175
4176my ($pbac) = pb_conf_get($vtype."login");
4177
4178foreach my $v (@$vm) {
4179    if ($vtype eq "ve") {
4180        # Get distro context
4181        my $pbos = pb_distro_get_context($v);
4182        my ($vepath) = pb_conf_get("vepath");
4183
4184        # Test if an existing snapshot exists and remove it if there is a VE
4185        if ((-f "$vepath->{$ENV{'PBPROJ'}}/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}.tar.gz") &&
4186            (! -d "$vepath->{$ENV{'PBPROJ'}}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}")) {
4187                pb_system("sudo rm -f $vepath->{$ENV{'PBPROJ'}}/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}.tar.gz","Removing previous snapshot $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}.tar.gz");
4188        }
4189    }
4190
4191    # Prepare the script to be executed on the VM/VE
4192    open(SCRIPT,"> $pbscript") || die "Unable to create $pbscript";
4193    print SCRIPT << 'EOF';
4194    #!/bin/bash
4195    sleep 2
4196EOF
4197    close(SCRIPT);
4198    chmod 0755,"$pbscript";
4199
4200    # Force shutdown of VM/VE
4201    # Force snapshot of VM/VE
4202    pb_script2v($pbscript,$vtype,$pbstep,1,$v,1);
4203}
4204return;
4205}
4206
4207# Function to update VMs/VEs/RMs with the latest distribution content
4208sub pb_update2v {
4209
4210my $vtype = shift;
4211my $pbstep = 2;
4212
4213my ($vm,$all) = pb_get2v($vtype);
4214
4215# Script generated
4216my $pbscript = "$ENV{'PBDESTDIR'}/updatev.$$";
4217
4218my ($pbac) = pb_conf_get($vtype."login");
4219
4220foreach my $v (@$vm) {
4221    # Get distro context
4222    my $pbos = pb_distro_get_context($v);
4223
4224    # Prepare the script to be executed on the VM/VE/RM
4225    # in $ENV{'PBDESTDIR'}/updatev
4226    open(SCRIPT,"> $pbscript") || die "Unable to create $pbscript";
4227   
4228    print SCRIPT << 'EOF';
4229    #!/bin/bash
4230    sleep 2
4231EOF
4232    # VE needs a good /proc
4233    $vetype = pb_ve_get_type($vetype);
4234    if (($vtype eq "ve") && ($vetype ne "docker")) {
4235        print SCRIPT "sudo /bin/mount -t proc /proc /proc\n";
4236    }
4237    print SCRIPT "$pbos->{'update'}\n";
4238    if (($vtype eq "ve") && ($vetype ne "docker")) {
4239        print SCRIPT "sudo /bin/umount /proc\n";
4240    }
4241    close(SCRIPT);
4242    chmod 0755,"$pbscript";
4243
4244    # Force shutdown of VM except
4245    pb_script2v($pbscript,$vtype,$pbstep,1,$v);
4246}
4247return;
4248}
4249
4250sub pb_announce {
4251
4252    my $antype = shift;
4253
4254    # Get all required parameters
4255    my ($pbpackager,$pbrepo) = pb_conf_get("pbpackager","pbrepo");
4256    my ($pkgv, $pkgt, $testver) = pb_conf_get_if("pkgver","pkgtag","testver");
4257    if (((not defined $testver) || (not defined $testver->{$ENV{'PBPROJ'}}) || ($testver->{$ENV{'PBPROJ'}} !~ /true/i)) && ($antype eq "Clean")) {
4258        # We just clean for test versions
4259        pb_log(0,"Unable to clean SSH repository for non testver version\n");
4260        return;
4261    }
4262    my $pkg = pb_cms_get_pkg($defpkgdir,$extpkgdir);
4263    my @pkgs = @$pkg;
4264    my %pkgs;
4265    my $first = 0;
4266
4267    # Get all distros concerned
4268    my $pbos = pb_distro_get_context();
4269    my $distrolist = pb_get_distros($pbos,undef);
4270    my %dl;
4271    my %theorlist;
4272    my %archlist;
4273    foreach my $d (split(/,/,$distrolist)) {
4274        my ($d1,$d2,$d3) = split(/-/,$d);
4275        $dl{$d1}++;
4276    }
4277
4278    # Command to find packages on repo
4279    my $findstr = "find ".join(" ",keys %dl)." ";
4280    my $srcstr = "";
4281    # Generated announce files
4282    my @files;
4283
4284    foreach my $pbpkg (@pkgs) {
4285        if ($first != 0) {
4286            $findstr .= "-o ";
4287        }
4288        $first++;
4289        if ((defined $pkgv) && (defined $pkgv->{$pbpkg})) {
4290            $pbver = $pkgv->{$pbpkg};
4291        } else {
4292            $pbver = $ENV{'PBPROJVER'};
4293        }
4294        if ((defined $pkgt) && (defined $pkgt->{$pbpkg})) {
4295            $pbtag = $pkgt->{$pbpkg};
4296        } else {
4297            $pbtag = $ENV{'PBPROJTAG'};
4298        }
4299
4300        # TODO: use virtual/real names here now
4301        my $pbrealpkg = $pbpkg;
4302        my $pbrealpkgrpm = pb_cms_get_real_pkg($pbpkg,"rpm");
4303        my $pbrealpkgdeb = pb_cms_get_real_pkg($pbpkg,"deb");
4304        if ($antype eq "Clean") {
4305            # We only clean test versions anyway
4306            $pbtag = "0";
4307            my $nver = $pbver;
4308            my $ntag = "[2-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]";
4309            $pbver .= $ntag;
4310            $findstr .= "-name \'$pbrealpkgrpm-$pbver-$pbtag\.*.rpm\' -o -name \'$pbrealpkgrpm-debug-$pbver-$pbtag\.*.rpm\' -o -name \'$pbrealpkgdeb"."_$pbver-$pbtag"."_*\.deb\' -o -name \'$pbrealpkgdeb"."_$pbver-$pbtag.dsc\' -o -name \'$pbrealpkgdeb"."_$pbver-$pbtag.tar.gz\' -o -name \'$pbrealpkg-$nver"."_p$ntag\.ebuild\' -o -name \'$pbrealpkg-$pbver-$pbtag*\.pkg\.gz\' -o -name \'$pbrealpkg-$pbver-$pbtag*\.sd\' ";
4311            $srcstr .= "src/$pbrealpkg-$pbver.tar.gz src/$pbrealpkg-$pbver.pbconf.tar.gz ";
4312        } else {
4313            # "Announce" and "Check"
4314            my @date=pb_get_date();
4315            # the matching is only done on packages made the same day for test version. Hopefully this is enough
4316            my $nver;
4317            if ((defined $testver) && (defined $testver->{$ENV{'PBPROJ'}}) && ($testver->{$ENV{'PBPROJ'}} =~ /true/i)) {
4318                $pbtag = "0";
4319                my $ntag .= strftime("%Y%m%d*", @date);
4320                $nver = $pbver."_p$ntag*";
4321                $pbver .= "$ntag*";
4322            } else { 
4323                $nver = $pbver."-r$pbtag";
4324            }
4325            $findstr .= "-name \'$pbrealpkgrpm-$pbver-$pbtag\.*.rpm\' -o -name \'$pbrealpkgdeb"."_$pbver-$pbtag"."_*\.deb\' -o -name \'$pbrealpkg-$nver\.ebuild\' -o -name \'$pbrealpkg-$pbver\.pkg\.gz\' -o -name \'$pbrealpkg-$pbver\.sd\' ";
4326        }
4327
4328        if ($antype eq "Announce") {
4329            my $chglog;
4330
4331            pb_vcs_init($pbinit);
4332            # Get project info on log file and generate tmp files used later on
4333            $chglog = "$ENV{'PBROOTDIR'}/$pbpkg/pbcl";
4334            $chglog = "$ENV{'PBROOTDIR'}/pbcl" if (! -f $chglog);
4335            $chglog = undef if (! -f $chglog);
4336
4337            open(OUT,"> $ENV{'PBTMP'}/$pbpkg.ann") || die "Unable to create $ENV{'PBTMP'}/$pbpkg.ann: $!";
4338            my $pb;
4339            $pb->{'realpkg'} = $pbrealpkg;
4340            $pb->{'ver'} = $pbver;
4341            $pb->{'tag'} = $pbtag;
4342            $pb->{'date'} = $pbdate;
4343            $pb->{'extdir'} = pb_get_extdir();
4344            $pb->{'chglog'} = $chglog;
4345            $pb->{'packager'} = $pbpackager;
4346            $pb->{'proj'} = $ENV{'PBPROJ'};
4347            $pb->{'repo'} = $pbrepo;
4348            $pb->{'pbos'}->{'type'} = "announce";
4349            $pb->{'pbos'}->{'suffix'} = "none";
4350            pb_changelog($pb,\*OUT,"yes");
4351            close(OUT);
4352            push(@files,"$ENV{'PBTMP'}/$pbpkg.ann");
4353        } elsif ($antype eq "Check") {
4354            # For the check we also build the theoritical complete list we should get
4355            foreach my $d (split(/,/,$distrolist)) {
4356                $pbos = pb_distro_get_context($d);
4357                if ($pbos->{'type'} eq "rpm") {
4358                    $theorlist{"$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}/$pbrealpkgrpm-$pbver-$pbtag$pbos->{'suffix'}"} = 0;
4359                } elsif ($pbos->{'type'} eq "deb") {
4360                    $theorlist{"$pbos->{'name'}/$pbos->{'version'}/$pbrealpkgdeb"."_$pbver-$pbtag"} = 0;
4361                    # TODO are we always using the last arch ?
4362                    $archlist{"$pbos->{'name'}/$pbos->{'version'}/$pbrealpkgdeb"."_$pbver-$pbtag"} = "$pbos->{'arch'}";
4363                } elsif ($pbos->{'type'} eq "ebuild") {
4364                    my $prefix = "-r";
4365                    $prefix = "_p" if ((defined $testver) && (defined $testver->{$ENV{'PBPROJ'}}) && ($testver->{$ENV{'PBPROJ'}} =~ /true/i));
4366                    $theorlist{"$pbos->{'name'}/$pbos->{'version'}/$pbrealpkg-$pbver$prefix$pbtag.ebuild"} = 0;
4367                    $archlist{"$pbos->{'name'}/$pbos->{'version'}/$pbrealpkg-$pbver$prefix$pbtag.ebuild"} = "$pbos->{'arch'}";
4368                } elsif ($pbos->{'type'} eq "pkg") {
4369                    $theorlist{"$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}/$pbrealpkg-$pbver-$pbtag.pkg.gz"} = 0;
4370                } else {
4371                    pb_log(1,"No theoritical list possible for type $pbos->{'type'}\n");
4372                }
4373            }
4374        }
4375        pb_log(2,"theorlist initial: ".Dumper(%theorlist)."\n");
4376        pb_log(2,"archlist: ".Dumper(%archlist)."\n");
4377    }
4378    if ($antype eq "Announce") {
4379        $findstr .= " | grep -Ev \'src.rpm\'";
4380    } elsif ($antype eq "Clean") {
4381        $findstr .= " | xargs rm -f -v $srcstr ";
4382    }
4383
4384    # Prepare the command to run and execute it
4385    open(PBS,"> $ENV{'PBTMP'}/pbscript") || die "Unable to create $ENV{'PBTMP'}/pbscript";
4386    print PBS "#!/bin/bash\n";
4387    print PBS "set -x\n" if ($pbdebug gt 1);
4388    print PBS "$findstr | sort 2> /dev/null\n";
4389    close(PBS);
4390    chmod 0755,"$ENV{'PBTMP'}/pbscript";
4391    pb_send2target("Announce","$ENV{'PBTMP'}/pbscript");
4392
4393    my $sl = "Project $ENV{'PBPROJ'} version $ENV{'PBPROJVER'} is now available";
4394    if ($antype eq "Announce") {
4395        # Get subject line
4396        pb_log(0,"Please enter the title of your announce\n");
4397        pb_log(0,"(By default: $sl)\n");
4398        my $sl2 = <STDIN>;
4399        $sl = $sl2 if ($sl2 !~ /^$/);
4400
4401        # Prepare a template of announce
4402        open(ANN,"> $ENV{'PBTMP'}/announce.html") || die "Unable to create $ENV{'PBTMP'}/announce.html: $!";
4403        print ANN << "EOF";
4404$sl</p>
4405
4406<p>The project team is happy to announce the availability of a newest version of $ENV{'PBPROJ'} $ENV{'PBPROJVER'}. Enjoy it as usual!</p>
4407<p>
4408Now available at <a href="$pbrepo->{$ENV{'PBPROJ'}}">$pbrepo->{$ENV{'PBPROJ'}}</a>
4409</p>
4410<p>
4411EOF
4412    }
4413
4414    open(LOG,"$ENV{'PBTMP'}/system.$$.log") || die "Unable to read $ENV{'PBTMP'}/system.$$.log: $!";
4415    if ($antype eq "Announce") {
4416        my $col = 2;
4417        my $i = 1;
4418        print ANN << 'EOF';
4419<TABLE WIDTH="100%" CELLPADDING="0" CELLSPACING="0" BORDER="0">
4420<TR>
4421EOF
4422        while (<LOG>) {
4423            print ANN "<TD><A HREF=\"$pbrepo->{$ENV{'PBPROJ'}}/$_\">$_</A></TD>";
4424            $i++;
4425            if ($i > $col) {
4426                print ANN "</TR>\n<TR>";
4427                $i = 1;
4428            }
4429        }
4430    } elsif ($antype eq "Clean") {
4431        while (<LOG>) {
4432            # skip errors
4433            next if ($_ !~ /^removed /);
4434            pb_log(0,"$_");
4435        }
4436    } else {
4437        # In "Check" mode we need to compare the 2 lists (real and theoritical)
4438        while (<LOG>) {
4439            # Get package name and remove what is in extra for the theoritical list (arch at the end)
4440            chomp();
4441            # skip find errors
4442            next if (/^find:/);
4443            my $p = $_;
4444            $p =~ s/\.(i[3456]86|x86_64|noarch|src)\.rpm$//;
4445            $p =~ s/_(i[3456]86|amd64|all)\.deb$//;
4446            $p =~ s/(-0\.[0-9]{8})[0-9]{6}/$1*/ if ((defined $testver) && (defined $testver->{$ENV{'PBPROJ'}}) && ($testver->{$ENV{'PBPROJ'}} =~ /true/i));
4447            $p =~ s/(-r|_p[0-9]+)\.ebuild/$1*/ if ((defined $testver) && (defined $testver->{$ENV{'PBPROJ'}}) && ($testver->{$ENV{'PBPROJ'}} =~ /true/i));
4448            $theorlist{$p} = -2 if (not defined $theorlist{$p});
4449            $theorlist{$p} = $theorlist{$p} + 1;
4450        }
4451        pb_log(2,"theorlist after check: ".Dumper(%theorlist)."\n");
4452    }
4453    close(LOG);
4454
4455    # Nothing more for the cleanssh case
4456    return if ($antype eq "Clean");
4457
4458    if ($antype eq "Check") {
4459        my ($chkex) = pb_conf_get_if("checkexclude");
4460        my $vmbuildlist = "";
4461        my $vebuildlist = "";
4462        my $rmbuildlist = "";
4463        my @pt = pb_conf_get_if("vmlist","velist","rmlist");
4464        foreach my $t (sort keys %theorlist) {
4465            if (defined $theorlist{$t} and $theorlist{$t} >= 1) {
4466                pb_log(1,"Packages found for $t\n");
4467            } elsif (defined $theorlist{$t} and $theorlist{$t} < 0) {
4468                pb_log(0,"Extra Package found for $t\n");
4469            } else {
4470                pb_log(2,"Analyzing $t\n");
4471                my ($os,$ver,$arch,$package) = split(/\//,$t);
4472                # Some distro have no arch subdir
4473                if (not defined $package) {
4474                    $package = $arch;
4475                    # TODO: If both arch have failed, we just make the last one
4476                    $arch = $archlist{$t} if (defined $archlist{$t});
4477                }
4478                my $pbos = pb_distro_get_context("$os-$ver-$arch");
4479                my $pkgn = $package;
4480                if ($pbos->{'type'} ne "deb") {
4481                    # package name is more easily found from the end for non deb
4482                    # as '-' is the separator, but it can also be used in names
4483                    $pkgn = reverse($package);
4484                    # search the second '-' and isolate the now last part which is the full name
4485                    $pkgn =~ s/([^-]+)-([^-]+)-([\S])+$/$3/;
4486                } else {
4487                    $pkgn =~ s/([^_]+)_([\S])+$/$2/;
4488                }
4489                my $found = 0;
4490                # Handle the exclusion of OSes
4491                my $excl = "";
4492                $excl .= $chkex->{$pkgn} if (defined $chkex->{$pkgn});
4493                $excl .= $chkex->{"all"} if (defined $chkex->{"all"});
4494
4495                pb_log(2,"Searching in excl $excl\n");
4496                foreach my $ex (split(/,/,$excl)) {
4497                    $found = 1 if ("$os-$ver-$arch" =~ /^$ex/);
4498                }
4499                # Skip as excluded
4500                next if ($found == 1);
4501                pb_log(0,"Package NOT found for $t\n");
4502                # Avoid duplicates in list
4503                next if ($vmbuildlist =~ /$os-$ver-$arch/);
4504                next if ($vebuildlist =~ /$os-$ver-$arch/);
4505                next if ($rmbuildlist =~ /$os-$ver-$arch/);
4506                # check with which method we need to build
4507                if ((defined $pt[0]->{$ENV{'PBPROJ'}}) and ($pt[0]->{$ENV{'PBPROJ'}} =~ /$os-$ver-$arch/)) {
4508                    $vmbuildlist = "$os-$ver-$arch" if ($vmbuildlist eq "");
4509                    $vmbuildlist .= ",$os-$ver-$arch" if ($vmbuildlist !~ /$os-$ver-$arch/);
4510                    next;
4511                }
4512                if ((defined $pt[1]->{$ENV{'PBPROJ'}}) and ($pt[1]->{$ENV{'PBPROJ'}} =~ /$os-$ver-$arch/)) {
4513                    $vebuildlist = "$os-$ver-$arch" if ($vebuildlist eq "");
4514                    $vebuildlist .= ",$os-$ver-$arch" if ($vebuildlist !~ /$os-$ver-$arch/);
4515                    next;
4516                }
4517                if ((defined $pt[2]->{$ENV{'PBPROJ'}}) and ($pt[2]->{$ENV{'PBPROJ'}} =~ /$os-$ver-$arch/)) {
4518                    $rmbuildlist = "$os-$ver-$arch" if ($rmbuildlist eq "");
4519                    $rmbuildlist .= ",$os-$ver-$arch" if ($rmbuildlist !~ /$os-$ver-$arch/);
4520                }
4521            }
4522        }
4523        # If we want to rebuild automatically, let's do it
4524        if (defined $opts{'rebuild'}) {
4525            # SandBox or CMS
4526            pb_log(0,"Rebuilding from SandBox\n");
4527            pb_log(0,"for VMs: $vmbuildlist\n") if ($vmbuildlist ne "");
4528            pb_log(0,"for VEs: $vebuildlist\n") if ($vebuildlist ne "");
4529            pb_log(0,"for RMs: $rmbuildlist\n") if ($rmbuildlist ne "");
4530            pb_cms2build("SandBox");
4531            # Which mode
4532            $ENV{'PBV'} = $vmbuildlist;
4533            pb_build2v("vm","build") if ($vmbuildlist ne "");
4534            $ENV{'PBV'} = $vebuildlist;
4535            pb_build2v("ve","build") if ($vebuildlist ne "");
4536            $ENV{'PBV'} = $rmbuildlist;
4537            pb_build2v("rm","build") if ($rmbuildlist ne "");
4538        }
4539        # For the check part this is now finished
4540        return;
4541    }
4542
4543    print ANN << "EOF";
4544</TR>
4545</TABLE>
4546</p>
4547
4548<p>As usual source packages are also available in the same directory.</p>
4549
4550<p>
4551Changes are :
4552</p>
4553<p>
4554EOF
4555    # Get each package changelog content
4556    foreach my $f (sort(@files)) {
4557        open(IN,"$f") || die "Unable to read $f:$!";
4558        while (<IN>) {
4559            print ANN $_;
4560        }
4561        close(IN);
4562        print ANN "</p><p>\n";
4563    }
4564    print ANN "</p>\n";
4565    close(ANN);
4566
4567    # Allow for modification
4568    my $editor = "vi";
4569    $editor = $ENV{'EDITOR'} if (defined $ENV{'EDITOR'});
4570    pb_system("$editor $ENV{'PBTMP'}/announce.html","Allowing modification of the announce","noredir");
4571
4572    # Store it in DB for external usage (Web pages generation)
4573    my $dbh = pb_connect_db();
4574
4575    # To read whole file
4576    local $/;
4577    open(ANN,"$ENV{'PBTMP'}/announce.html") || die "Unable to read $ENV{'PBTMP'}/announce.html: $!";
4578    my $announce = <ANN>;
4579    close(ANN);
4580   
4581    pb_log(2,"INSERT INTO announces VALUES (NULL, $pbdate, $announce)");
4582    my $sth = $dbh->prepare(qq{INSERT INTO announces VALUES (NULL,?,?)})
4583                    || die "Unable to insert into db";
4584    $sth->execute($pbdate, $announce);
4585    $sth->finish();
4586    $dbh->disconnect;
4587
4588    # Then deliver it on the Web
4589    # $TOOLHOME/livwww www
4590
4591    # Mail it to project's ML
4592    open(ML,"| w3m -dump -T text/html > $ENV{'PBTMP'}/announce.txt") || die "Unable to create $ENV{'PBTMP'}/announce.txt: $!";
4593    print ML << 'EOF';
4594<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/x html1/DTD/xhtml1-strict.dtd">
4595
4596<html xmlns="http://www.w3.org/1999/xhtml" dir="ltr" xml:lang="en" lang="en">
4597  <head>
4598  </head>
4599  <body>
4600  <p>
4601EOF
4602    open(ANN,"$ENV{'PBTMP'}/announce.html") || die "Unable to read $ENV{'PBTMP'}/announce.html: $!";
4603    while(<ANN>) {
4604        print ML $_;
4605    }
4606    print ML << 'EOF';
4607</body>
4608</html>
4609EOF
4610    close(ML);
4611
4612    # To read whole file
4613    local $/;
4614    open(ANN,"$ENV{'PBTMP'}/announce.txt") || die "Unable to read $ENV{'PBTMP'}/announce.txt: $!";
4615    my $msg = <ANN>;
4616    close(ANN);
4617   
4618    # Preparation of headers
4619    eval
4620    {
4621        require Mail::Sendmail;
4622        Mail::Sendmail->import();
4623    };
4624    if ($@) {
4625        # Mail::Sendmail not found not sending mail !
4626        pb_log(0,"No Mail::Sendmail module found so not sending any mail !\n");
4627    } else {
4628        my ($pbml,$pbsmtp) = pb_conf_get("pbml","pbsmtp");
4629        my %mail = (   
4630            To          =>  $pbml->{$ENV{'PBPROJ'}},
4631            From        =>  $pbpackager->{$ENV{'PBPROJ'}},
4632            Smtp        =>  $pbsmtp->{$ENV{'PBPROJ'}},
4633            Body        =>  $msg,
4634            Subject     =>  "[ANNOUNCE] $sl",
4635        );
4636           
4637        # Send mail
4638        if (! sendmail(%mail)) {
4639            if ((defined $Mail::Sendmail::error) and (defined $Mail::Sendmail::log)) {
4640                die "Unable to send mail ($Mail::Sendmail::error): $Mail::Sendmail::log";
4641            }
4642        }
4643    }
4644}
4645
4646#
4647# Creates a set of HTML file containing the news for the project
4648# based on what has been generated by the pb_announce function
4649#
4650sub pb_web_news2html {
4651
4652    my $dest = shift || $ENV{'PBTMP'};
4653
4654    # Get all required parameters
4655    my ($pkgv, $pkgt) = pb_conf_get_if("pkgver","pkgtag");
4656
4657
4658    my $dbh = pb_connect_db();
4659    # For date handling
4660    $ENV{'LANGUAGE'}="C";
4661    my $firstjan = strftime("%Y-%m-%d", 0, 0, 0, 1, 0, localtime->year(), 0, 0, -1);
4662    my $oldfirst = strftime("%Y-%m-%d", 0, 0, 0, 1, 0, localtime->year()-1, 0, 0, -1);
4663    pb_log(2,"firstjan: $firstjan, oldfirst: $oldfirst, pbdate:$pbdate\n");
4664    my $all = $dbh->selectall_arrayref("SELECT id,date,announce FROM announces ORDER BY date DESC");
4665    my %news;
4666    $news{"cy"} = "";   # current year's news
4667    $news{"ly"} = "";   # last year news
4668    $news{"py"} = "";   # previous years news
4669    $news{"fp"} = "";   # first page news
4670    my $cpt = 4;        # how many news for first page
4671    # Extract info from DB
4672    foreach my $row (@$all) {
4673        my ($id, $date, $announce) = @$row;
4674        $news{"cy"} = $news{"cy"}."<p><B>$date</B> $announce\n" if ((($date cmp $pbdate) le 0) && (($firstjan cmp $date) le 0));
4675        $news{"ly"} = $news{"ly"}."<p><B>$date</B> $announce\n" if ((($date cmp $firstjan) le 0) && (($oldfirst cmp $date) le 0));
4676        $news{"py"} = $news{"py"}."<p><B>$date</B> $announce\n" if (($date cmp $oldfirst) le 0);
4677        $news{"fp"} = $news{"fp"}."<p><B>$date</B> $announce\n" if ($cpt > 0);
4678        $cpt--;
4679    }
4680    pb_log(1,"news{fp}: ".$news{"fp"}."\n");
4681    $dbh->disconnect;
4682
4683    # Generate the HTML content
4684    foreach my $pref (keys %news) {
4685        open(NEWS,"> $dest/pb_web_$pref"."news.html") || die "Unable to create $dest/pb_web_$pref"."news.html: $!";
4686        print NEWS "$news{$pref}";
4687        close(NEWS);
4688    }
4689}
4690
4691sub pb_connect_db {
4692
4693    # Store in DB for external usage (Web pages generation)
4694    my $db = "$ENV{'PBCONFDIR'}/announces3.sql";
4695
4696    my $precmd = "";
4697    if (! -f $db) {
4698        $precmd = "CREATE TABLE announces (id INTEGER PRIMARY KEY AUTOINCREMENT, date DATE, announce VARCHAR[65535])";
4699        # we should add the $db to the VCS as well here
4700    }
4701
4702    my $dbh = DBI->connect("dbi:SQLite:dbname=$db","","",
4703                        { RaiseError => 1, AutoCommit => 1 })
4704                        || die "Unable to connect to $db";
4705
4706    if ($precmd ne "") {
4707        my $sth = $dbh->prepare(qq{$precmd})
4708                    || die "Unable to create table into $db";
4709        $sth->execute();
4710    }
4711
4712    return($dbh);
4713}
4714
4715
4716# Return the SSH key file to use
4717# Potentially create it if needed
4718
4719sub pb_ssh_get {
4720
4721my $create = shift || 0;    # Do not create keys by default. In that case prefer native keys, not pb's ones
4722my $target = shift || "localhost";
4723my $nport = shift || "22";
4724
4725my ($pbagent) = pb_conf_get_if("pbusesshagent");
4726# use ssh-agent if asked so.
4727return(undef) if (($create == 0) && (defined $pbagent->{$ENV{'PBPROJ'}}) && ($pbagent->{$ENV{'PBPROJ'}} =~ /true/io));
4728
4729# Check the SSH environment
4730my $keyfile = undef;
4731
4732# Check whether ecdsa is accepted by the remote host
4733my $ecdsa = 0;
4734my $dsa = 0;
4735my $rsa = 0;
4736open(SCAN,"ssh-keyscan -p $nport -t ecdsa $target 2>/dev/null |") || pb_log(0,"Unable to check ECDSA support for $target");
4737while(<SCAN>) {
4738    $ecdsa = 1 if ($_ =~ /ecdsa-/);
4739    pb_log(2,"Found ECDSA support on $target ($nport)\n");
4740}
4741close(SCAN);
4742
4743open(SCAN,"ssh-keyscan -p $nport -t rsa $target 2>/dev/null |") || pb_log(0,"Unable to check DSA support for $target");
4744while(<SCAN>) {
4745    $rsa = 1 if ($_ =~ /ssh-rsa/);
4746    pb_log(2,"Found RSA support on $target ($nport)\n");
4747}
4748close(SCAN);
4749
4750open(SCAN,"ssh-keyscan -p $nport -t dsa $target 2>/dev/null |") || pb_log(0,"Unable to check DSA support for $target");
4751while(<SCAN>) {
4752    $dsa = 1 if ($_ =~ /ssh-dss/);
4753    pb_log(2,"Found DSA support on $target ($nport)\n");
4754}
4755close(SCAN);
4756
4757# We have specific keys by default - create them if needed
4758if (!(-e "$ENV{'HOME'}/.ssh/pb_ecdsa") && ($create eq 1)) {
4759    my $res = pb_system("ssh-keygen -q -b 521 -N '' -f $ENV{'HOME'}/.ssh/pb_ecdsa -t ecdsa","Generating ECDSA SSH keys for pb");
4760    if (($res != 0) && !(-e "$ENV{'HOME'}/.ssh/pb_dsa") && ($create eq 1)) {
4761        # ecdsa may not be supported if old ssh
4762        pb_system("ssh-keygen -q -b 1024 -N '' -f $ENV{'HOME'}/.ssh/pb_dsa -t dsa","Generating DSA SSH keys for pb");
4763    }
4764}
4765
4766# Some ECDSA implementation are not working correctly e.g. RHEL7, Fedora 20/21
4767if (!(-e "$ENV{'HOME'}/.ssh/pb_rsa") && ($create eq 1)) {
4768    pb_system("ssh-keygen -q -b 2048 -N '' -f $ENV{'HOME'}/.ssh/pb_rsa -t rsa","Generating RSA SSH keys for pb");
4769}
4770
4771# We still favour RSA and fall back to ECDSA if not available
4772$keyfile = "$ENV{'HOME'}/.ssh/id_rsa" if (-s "$ENV{'HOME'}/.ssh/id_rsa");
4773if ($rsa != 0) {
4774    # And we favour a specific key over a generic one except if in delivery to ftp server
4775    $keyfile = "$ENV{'HOME'}/.ssh/id_rsa" if ((-s "$ENV{'HOME'}/.ssh/id_rsa") && ($create eq 0));
4776    $keyfile = "$ENV{'HOME'}/.ssh/pb_rsa" if (-s "$ENV{'HOME'}/.ssh/pb_rsa");
4777} else {
4778    if ($ecdsa != 0) {
4779        # And we favour a specific key over a generic one except if in delivery to ftp server
4780        $keyfile = "$ENV{'HOME'}/.ssh/id_ecdsa" if ((-s "$ENV{'HOME'}/.ssh/id_ecdsa") && ($ecdsa != 0) && ($create eq 0));
4781        $keyfile = "$ENV{'HOME'}/.ssh/pb_ecdsa" if ((-s "$ENV{'HOME'}/.ssh/pb_ecdsa") && ($ecdsa != 0));
4782    } elsif ($dsa != 0) {
4783        # And we favour a specific key over a generic one except if in delivery to ftp server
4784        $keyfile = "$ENV{'HOME'}/.ssh/id_dsa" if ((-s "$ENV{'HOME'}/.ssh/id_dsa") && ($dsa != 0) && ($create eq 0));
4785        $keyfile = "$ENV{'HOME'}/.ssh/pb_dsa" if ((-s "$ENV{'HOME'}/.ssh/pb_dsa") && ($dsa != 0));
4786    }
4787}
4788if (defined $keyfile) {
4789    pb_log(2,"Using ssh key file $keyfile\n");
4790} else {
4791    pb_log(0, "Unable to find your public ssh key under $ENV{'HOME'}/.ssh\n");
4792}
4793return($keyfile);
4794}
4795
4796
4797# Returns the pid of a running VM command using a specific VM file, and the port used if the VM exists
4798sub pb_check_ps {
4799    my $vmcmd = shift;
4800    my $vmm = shift;
4801    my $vexist = 0;             # FALSE by default
4802    my $vmport = undef;         # NONE by default
4803
4804    if ($vmcmd =~ /qemu/) {
4805        # Remove the suffix as qemu changes it at launch
4806        $vmcmd =~ s/qemu-.*/qemu/;
4807    }
4808    open(PS, "ps auxhww|") || die "Unable to call ps";
4809    while (<PS>) {
4810        next if (! /$vmcmd/);
4811        next if (! /$vmm/);
4812        my $void1;
4813        ($void1, $vexist, $vmport) = split(/ +/,$_,3);
4814        pb_log(2,"pb_check_ps $vmport\n");
4815        $vmport =~ s/.*=tcp:127\.0\.0\.1:([0-9]+)-:22.*/$1/;
4816        chomp($vmport);
4817        pb_log(2,"pb_check_ps found a VM $vexist using port $vmport\n");
4818        last;
4819    }
4820    return($vexist,$vmport);
4821}
4822
4823
4824sub pb_extract_build_files {
4825
4826my $src=shift;
4827my $dir=shift;
4828my $ddir=shift;
4829my $mandatory=shift || "build";
4830
4831my $flag = "mayfail" if (($mandatory eq "patch") || ($mandatory eq "src"));
4832my $res;
4833
4834if ($src =~ /tar\.gz$/) {
4835    $res = pb_system("tar xfpz $src $dir","Extracting $mandatory files from $src",$flag);
4836} elsif ($src =~ /tar\.bz2$/) {
4837    $res = pb_system("tar xfpj $src $dir","Extracting $mandatory files from $src",$flag);
4838} else {
4839    die "Unknown compression algorithm for $src";
4840}
4841# If not mandatory return now
4842return() if (($res != 0) and (($mandatory eq "patch") || ($mandatory eq "src")));
4843return(pb_move_extracted_files($dir,$ddir));
4844}
4845
4846sub pb_move_extracted_files {
4847
4848my $dir = shift;
4849my $ddir = shift;
4850my @files;
4851
4852# In a recursive function , we need a local var as DIR handle
4853my $bdir;
4854
4855opendir($bdir,"$dir") || confess "Unable to open directory $dir: $!";
4856foreach my $f (readdir($bdir)) {
4857    next if ($f =~ /^\./);
4858    # Skip potential patch dir
4859    next if ($f =~ /^pbpatch/);
4860    # Skip potential source dir
4861    next if ($f =~ /^pbsrc/);
4862    # Skip potential backup files
4863    next if ($f =~ /~$/);
4864    if ((-d "$dir/$f") && (! -l "$dir/$f")) {
4865        # Case of the bug debian subdir
4866        pb_mkdir_p("$ddir/$f");
4867        push @files,pb_move_extracted_files("$dir/$f","$ddir/$f");
4868    } else {
4869        move("$dir/$f","$ddir") || die "Unable to move $dir/$f to $ddir";
4870        pb_log(2,"mv $dir/$f $ddir\n");
4871        push @files,"$ddir/$f";
4872    }
4873}
4874closedir($bdir);
4875# Not enough but still a first cleanup
4876pb_rm_rf("$dir");
4877return(@files);
4878}
4879
4880sub pb_list_bfiles {
4881
4882my $dir = shift;
4883my $pbpkg = shift;
4884my $bfiles = shift;
4885my $pkgfiles = shift;
4886my $supfiles = shift;
4887# subdir to keep if recursive mode, empty by default
4888my $subdir = shift || "";
4889# In a recursive function , we need a local var as DIR handle
4890my $bdir;
4891
4892pb_log(2,"DEBUG: entering pb_list_bfiles in $dir: ".Dumper($bfiles)."\n");
4893opendir($bdir,"$dir") || die "Unable to open dir $dir: $!";
4894foreach my $f (readdir($bdir)) {
4895    pb_log(3,"DEBUG: pb_list_bfiles found $f\n");
4896    next if ($f =~ /^\./);
4897    if (-d "$dir/$f") {
4898        # Recurse for directories (Debian 3.0 format e.g.)
4899        pb_log(2,"DEBUG: pb_list_bfiles recurse in $dir/$f\n");
4900        pb_list_bfiles("$dir/$f",$pbpkg,$bfiles,$pkgfiles,$supfiles,$f);
4901        next;
4902    }
4903
4904    my $key = $f;
4905    # if recursive then store also the subdir
4906    $key = "$subdir/$f" if ($subdir ne "");
4907    $bfiles->{$key} = "$dir/$f";
4908    $bfiles->{$key} =~ s~$ENV{'PBROOTDIR'}~~;
4909    if (defined $supfiles->{$pbpkg}) {
4910        $pkgfiles->{$key} = "$dir/$f" if ($f =~ /$supfiles->{$pbpkg}/);
4911    }
4912}
4913closedir($bdir);
4914pb_log(2,"DEBUG: exiting pb_list_bfiles: ".Dumper($bfiles)."\n");
4915}
4916
4917sub pb_add_comma {
4918
4919my $str = shift;
4920my $addstr = shift;
4921
4922$str .= "," if (defined $str);
4923$str .= $addstr;
4924return($str);
4925}
4926
4927sub pb_list_sfiles {
4928
4929my $sdir = shift;
4930my $ptr = shift;
4931my $pbos = shift;
4932my $extdir = shift;
4933
4934pb_log(2,"DEBUG: entering pb_list_sfiles: ".Dumper($ptr)."\n");
4935my $key = "$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
4936
4937# Prepare local sources for this distro - They are always applied first - May be a problem one day
4938# This function works for both patches and additional sources
4939foreach my $p (sort(<$sdir/*>)) {
4940    $ptr->{$key} = pb_add_comma($ptr->{$key},"file://$p") if (($p =~ /\.all$/) || ($p =~ /\.$pbos->{'os'}$/) || ($p =~ /\.$pbos->{'type'}$/) || ($p =~ /\.$pbos->{'family'}$/) || ($p =~ /\.$pbos->{'name'}$/) || ($p =~ /\.$pbos->{'name'}-$pbos->{'version'}$/) ||($p =~ /\.$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}$/));
4941}
4942
4943# Prepare also remote sources to be included - Applied after the local ones
4944foreach my $p ("all","$pbos->{'os'}","$pbos->{'type'}","$pbos->{'family'}","$pbos->{'name'}","$pbos->{'name'}-$pbos->{'version'}","$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}") {
4945    my $f = "$extdir.".".$p";
4946    next if (not -f $f);
4947    if (not open(PATCH,$f)) {
4948        pb_display("Unable to open existing external source file content $f\n");
4949        next;
4950    }
4951    while (<PATCH>) {
4952        chomp();
4953        $ptr->{$key} = pb_add_comma($ptr->{$key},"$_");
4954    }
4955    close(PATCH);
4956}
4957pb_log(2,"DEBUG: exiting pb_list_sfiles: ".Dumper($ptr)."\n");
4958return($ptr);
4959}
4960   
4961#
4962# Return the list of packages we are working on in a non CMS action
4963#
4964sub pb_get_pkg {
4965
4966my @pkgs = ();
4967
4968# In script actions that file may not exist.
4969return(\@pkgs) if ((not defined $ENV{'PBDESTDIR'}) || (not defined $ENV{'PBPROJVER'}) || (not defined $ENV{'PBPROJTAG'}) || (! -f "$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml"));
4970
4971my ($var) = pb_conf_read("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml","pbpkg");
4972@pkgs = keys %$var if (defined $var);
4973
4974pb_log(0,"Packages: ".join(',',@pkgs)."\n") if (@pkgs);
4975return(\@pkgs);
4976}
4977
4978# Manages VM/RM SSH port communication
4979sub pb_get_port {
4980
4981my $port = shift;
4982my $pbos = shift;
4983my $cmt = shift;
4984my $nport;
4985
4986die "No port passed in parameter. Report to dev team\n" if (not defined $port);
4987# key is project on VM, but machine tuple for RM
4988if ($cmt =~ /^RM/i) {
4989    $nport = $port->{"$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}"};
4990} else {
4991    $nport = $port->{$ENV{'PBPROJ'}};
4992}
4993pb_log(2,"pb_get_port with $nport\n");
4994if ($cmt =~ /^VM/i) {
4995    # Maybe a port was given as parameter so overwrite
4996    $nport = "$pbport" if (defined $pbport);
4997    # Maybe in // mode so use the env var set up as an offset to the base port
4998    $nport = $nport + $ENV{'PBVMPORT'} -1 if ((defined $pbparallel) && (defined $ENV{'PBVMPORT'}));
4999}
5000pb_log(2,"pb_get_port returns $nport\n");
5001return($nport);
5002}
5003
5004sub pb_set_port { 
5005       
5006my ($pid,$ident) = @_;
5007pb_log(2,"pb_set_port for VM ($pid), id $ident\n");
5008$ENV{'PBVMPORT'} = $ident;
5009pb_log(2,"pb_set_port sets PBVMPORT in env to $ENV{'PBVMPORT'}\n");
5010}
5011
5012sub pb_set_parallel {
5013
5014my $vtype = shift;
5015
5016pb_log(2,"pb_set_parallel vtype: $vtype\n");
5017# Take care of memory size if VM, parallel mode and more than 1 action
5018if ((defined $pbparallel) && ($pbparallel ne 1) && ($vtype eq "vm")) {
5019    eval
5020    {
5021        require Linux::SysInfo;
5022        Linux::SysInfo->import();
5023    };
5024    if ($@) {
5025        # Linux::SysInfo not found
5026        pb_log(1,"ADVISE: Install Linux::SysInfo to benefit from automatic parallelism optimization.\nOr optimize manually pbparallel in your pb.yml file\nUsing $pbparallel processes max at a time for the moment\nWARNING: This may consume too much memory for your system\n");
5027    } else {
5028        # Using the memory size
5029        my $si = Linux::SysInfo::sysinfo();
5030        if (not defined $si) {
5031            pb_log(1,"ADVISE: Install Linux::SysInfo to benefit from automatic parallelism optimization.\nOr optimize manually pbparallel in your pb.yml file\nUsing $pbparallel processes max at a time for the moment\nWARNING: This may consume too much memory for your system\n");
5032        } else {
5033            # Keep the number of VM whose memory can be allocated
5034            my $ram = $si->{"totalram"}-$si->{"sharedram"}-$si->{"bufferram"};
5035            my $ram2;
5036            my ($vmmem) = pb_conf_get_if("vmmem");
5037
5038            my $v = "default";
5039            if ((defined $vmmem) and (defined $vmmem->{$v})) {
5040                $ram2 = $vmmem->{$v};
5041            } else {
5042                # Default for KVM/QEMU
5043                $ram2 = 128;
5044            }
5045            $pbparallel = sprintf("%d",$ram/$ram2);
5046        }
5047        pb_log(1,"Using $pbparallel processes at a time\n");
5048    }
5049}
5050pb_log(2,"pb_set_parallel returns: $pbparallel\n") if (defined $pbparallel);
5051return($pbparallel);
5052}
5053
5054sub pb_get_sudocmds { 
5055       
5056my $pbos = shift;
5057my @cmds = @_;
5058my %sudocmds;
5059
5060pb_log(2,"pb_get_sudocmds entering with lines:".Dumper(@cmds)."\n");
5061foreach my $c (split(/;/,$pbos->{'update'}),split(/;/,$pbos->{'install'}),@cmds) {
5062    pb_log(2,"pb_get_sudocmds analyses $c\n");
5063    next if ($c !~ /^\s*sudo/);
5064    # remove sudo and leading spaces
5065    $c =~ s/^\s*sudo\s+//;
5066    # keep only the command, not the params
5067    $c =~ s/([^\s]+)\s.*$/$1/;
5068    $sudocmds{$c} = "";
5069}
5070pb_log(2,"pb_get_sudocmds returns ".Dumper(keys %sudocmds)."\n");
5071return(keys %sudocmds);
5072}
5073
5074sub pb_sign_setenv {
5075
5076my ($passfile, $passphrase, $passpath) = pb_conf_get_if("pbpassfile","pbpassphrase","pbpasspath");
5077$ENV{'PBPASSPHRASE'} = $passphrase->{$ENV{'PBPROJ'}} if ((not defined $ENV{'PBPASSPHRASE'}) && (defined $passphrase->{$ENV{'PBPROJ'}}));
5078$ENV{'PBPASSFILE'} = $passfile->{$ENV{'PBPROJ'}} if ((not defined $ENV{'PBPASSFILE'})&& (defined $passfile->{$ENV{'PBPROJ'}})) ;
5079$ENV{'PBPASSPATH'} = $passpath->{$ENV{'PBPROJ'}} if ((not defined $ENV{'PBPASSPATH'})&& (defined $passpath->{$ENV{'PBPROJ'}})) ;
5080}
5081
5082sub pb_sign_setenv_rpm {
5083
5084pb_sign_setenv();
5085
5086eval
5087{
5088    require RPM4::Sign;
5089    RPM4::Sign->import();
5090};
5091if ($@) {
5092    # RPM4::Sign not found
5093    pb_log(1,"WARNING: Install RPM4::Sign to benefit from automatic package signing.\n");
5094    return(undef);
5095} else {
5096    return(undef) if (((not defined $ENV{'PBPASSPHRASE'}) and (not defined $ENV{'PBPASSFILE'})) || (not defined $ENV{'PBPACKAGER'}) || (not defined $ENV{'PBPASSPATH'}));
5097    my $sign = RPM4::Sign->new(
5098        passphrase => $ENV{'PBPASSPHRASE'},
5099        name => $ENV{'PBPACKAGER'},
5100        path => $ENV{'PBPASSPATH'},
5101        password_file => $ENV{'PBPASSFILE'},
5102    );
5103    return(undef) if (not defined $sign);
5104    return(undef) if (not defined $sign->{name});
5105    return(undef) if (not defined $sign->{path});
5106    return(undef) if ((not defined $sign->{passphrase}) && (not defined $sign->{password_file}));
5107    return(undef) if (not defined $sign->{keyid});
5108
5109    return($sign);
5110}
5111}
5112
5113# Cf: http://wiki.debian.org/SecureApt
5114
5115sub pb_sign_file {
5116
5117my $file = shift;
5118