source: devel/pb/bin/pb @ 2259

Last change on this file since 2259 was 2259, checked in by bruno, 2 years ago

Return from Env.pm early enough if command is updateconf

  • Property svn:executable set to *
File size: 179.5 KB
Line 
1#!/usr/bin/perl -w
2#
3# Project Builder main application
4#
5# $Id$
6#
7# Copyright B. Cornec 2007-today
8# Eric Anderson's changes are (c) Copyright 2012 Hewlett Packard
9# Provided under the GPL v2
10
11# Syntax: see at end
12
13use strict 'vars';
14
15# The modules mentioned here are required by pb when used both
16# locally or inside a VE/VM/RM
17# Additional required modules only used locally are called with a require
18# in their respective section
19use Getopt::Long qw(:config auto_abbrev no_ignore_case);
20use Carp qw/confess cluck/;
21use Data::Dumper;
22use English;
23use File::Basename;
24use File::Copy;
25use File::stat;
26use File::Find;
27use Time::localtime qw(localtime);
28use POSIX qw(strftime);
29use lib qw (lib);
30use ProjectBuilder::Version;
31use ProjectBuilder::Base;
32use ProjectBuilder::Display;
33use ProjectBuilder::Conf;
34use ProjectBuilder::Distribution;
35use ProjectBuilder::VCS;
36use ProjectBuilder::CMS;
37use ProjectBuilder::Env;
38use ProjectBuilder::Filter;
39use ProjectBuilder::Changelog;
40use ProjectBuilder::VE;
41
42# Global variables
43$Global::pb_stop_on_error = 0;      # False by default
44$Global::pb_show_sudo = 0;          # False by default
45
46my %opts;                           # CLI Options
47my $action;                         # action to realize
48my $test = "FALSE";                 # Not used
49my $pbforce = 0;                    # Force VM stop
50my $pbsnap = 0;                     # Do not use snapshot mode for VM/VE by default
51my $pbkeep = 0;                     # keep temporary directory at the end
52my $option = "";                    # Not used
53my @pkgs;                           # list of packages
54my $pbtag;                          # Global Tag variable
55my $pbver;                          # Global Version variable
56my %pbscript;                       # Name of the scripts per V
57my $pbrev;                          # Global REVISION variable
58my $pbtarget = undef;               # Target os-ver-arch you want to build for
59my $pbport;                         # Port to use to connect to the VM/RM
60my $newver;                         # New version to create
61my $pbimage = undef;                # ISO image for the VM to create or docker image for the VE
62my $vetype = undef;                 # Type of Virtual environment to deal with
63
64my @date = pb_get_date();
65my $pbdate = strftime("%Y-%m-%d", @date);
66
67=pod
68
69=head1 NAME
70
71pb, aka project-builder.org - builds packages for your projects
72
73=head1 DESCRIPTION
74
75pb helps you build various packages directly from your project sources.
76Those sources could be handled by a CMS (Configuration Management System)
77such as Subversion, CVS, Git, Mercurial... or being a simple reference to a compressed tar file.
78It's based on a set of configuration files, a set of provided macros to help
79you keeping build files as generic as possible. For example, a single .spec
80file should be required to generate for all rpm based distributions, even
81if you could also have multiple .spec files if required.
82
83=head1 SYNOPSIS
84
85pb [-vhSq][-r pbroot][-p project][[-s script -a account -P port][-T VEtype][-t [os-ver-arch]][-m os-ver-arch[,...]]][-g][-i image] <action> [<pkg1> ...]
86
87pb [--verbose][--help][--man][--quiet][--snapshot][--revision pbroot][--project project][[--script script --account account --port port][-T VEtype][--target [os-ver-arch]][--machine os-ver-arch[,...]]][--nographic][--image image][--rebuild] <action> [<pkg1> ...]
88
89=head1 OPTIONS
90
91=over 4
92
93=item B<-v|--verbose>
94
95Increase verbosity
96
97=item B<-q|--quiet>
98
99Do not print any output.
100
101=item B<-h|--help>
102
103Print a brief help message and exits.
104
105=item B<-S|--snapshot>
106
107Use the snapshot mode of VMs or VEs
108
109=item B<--man>
110
111Prints the manual page and exits.
112
113=item B<-t|--target os-ver-arch>
114
115Name of the target system you want to build for.
116All if none precised.
117
118=item B<-m|--machine os-ver-arch[,os-ver-arch,...]>
119
120Name of the Virtual Machines (VM), Virtual Environments (VE) or Remote Machines (RM)
121you want to build on (comma separated).
122All if none precised (or use the env variable PBV).
123
124=item B<-T|--vetype VEtype]>
125
126Type of Virtual Environments (VE)
127Can be chroot or docker.
128
129=item B<-s|--script script>
130
131Name of the script you want to execute on the related VMs/VEs/RMs.
132
133=item B<-g|--nographic>
134
135Do not launch VMs in graphical mode.
136
137=item B<-i|--image image>
138
139It could be either:
140- The name of the ISO image of the distribution you want to install on the related VMs
141- The name of the docker image of the distribution you want to install on the related VEs
142
143
144=item B<-a|--account account>
145
146Name of the account to use to connect on the related VMs/RMs.
147
148=item B<-P|--port port_number>
149
150Port number to use to connect on the related VMs/RMs.";
151
152=item B<-p|--project project_name>
153
154Name of the project you're working on (or use the env variable PBPROJ)
155
156=item B<-r|--revision revision>
157
158Path Name of the project revision under the CMS (or use the env variable PBROOT)
159
160=item B<-V|--version new_version>
161
162New version of the project to create based on the current one.
163
164=item B<-k|--keep>
165
166Keep the temporary dir where files have been created in or der to help debug
167
168=item B<--rebuild>
169
170Only valid with the checkssh action, it alllows to automatically relaunch the build of the failed packages
171
172=item B<--no-stop-on-error>
173
174Continue through errors with best effort.
175
176=back
177
178=head1 ARGUMENTS
179
180<action> can be:
181
182=over 4
183
184=item B<sbx2build>
185
186Create tar files for the project under your CMS.
187Current state of the exported content is taken.
188CMS supported are SVN, SVK, CVS, Git and Mercurial
189parameters are packages to build
190if not using default list
191
192=item B<cms2build>
193
194Create tar files for the project under your CMS.
195Current state of the CMS is taken.
196CMS supported are SVN, SVK, CVS, Git and Mercurial
197parameters are packages to build
198if not using default list
199
200=item B<build2prep>
201
202Prepare the environment for build by installing required dependencies. Done once on the build system.
203
204=item B<build2pkg>
205
206Create packages for your running distribution
207
208=item B<cms2pkg>
209
210cms2build + build2pkg
211
212=item B<sbx2pkg>
213
214sbx2build + build2pkg
215
216=item B<sbx2pkg2ins>
217
218sbx2pkg + final install of packages
219
220=item B<build2ssh>
221
222Send the tar files to a SSH host
223
224=item B<sbx2ssh>
225
226sbx2build + build2ssh
227
228=item B<cms2ssh>
229
230cms2build + build2ssh
231
232=item B<pkg2ssh>
233
234Send the packages built to a SSH host
235
236=item B<build2vm>
237
238Create packages in VMs, launching them if needed
239and send those packages to a SSH host once built
240VM type supported are QEMU and KVM
241
242=item B<build2ve>
243
244Create packages in VEs, creating it if needed
245and send those packages to a SSH host once built
246
247=item B<build2rm>
248
249Create packages in RMs, which should pre-exist,
250and send those packages to a SSH host once built
251RM means Remote Machine, and could be a physical or Virtual one.
252This is one buildfarm integration for pb.
253
254=item B<prepvm>
255
256Prepare the VMs to have all requirements to build the project
257
258=item B<prepve>
259
260Prepare the VEs to have all requirements to build the project
261
262=item B<preprm>
263
264Prepare the RMs to have all requirements to build the project
265
266=item B<sbx2vm>
267
268sbx2build + build2vm
269
270=item B<sbx2ve>
271
272sbx2build + build2ve
273
274=item B<sbx2docker>
275
276sbx2build + build2ve with a potential build of all necessary docker containers to perform it
277
278=item B<sbx2rm>
279
280sbx2build + build2rm
281
282=item B<cms2vm>
283
284cms2build + build2vm
285
286=item B<cms2ve>
287
288cms2build + build2ve
289
290=item B<cms2rm>
291
292cms2build + build2rm
293
294=item B<launchvm>
295
296Launch one virtual machine
297
298=item B<launchve>
299
300Launch one virtual environment
301
302=item B<script2vm>
303
304Launch one virtual machine if needed
305and executes a script on it
306
307=item B<script2ve>
308
309Execute a script in a virtual environment
310
311=item B<script2rm>
312
313Execute a script on a remote machine
314
315=item B<newvm>
316
317Create a new virtual machine
318
319=item B<newve>
320
321Create a new virtual environment
322
323=item B<setupvm>
324
325Setup a virtual machine for pb usage
326
327=item B<setupve>
328
329Setup a virtual environment for pb usage
330
331=item B<setuprm>
332
333Setup a remote machine for pb usage
334
335=item B<sbx2setupvm>
336
337Setup a virtual machine for pb usage using the sandbox version of pb instead of the latest stable
338Reserved to dev team.
339
340=item B<sbx2setupve>
341
342Setup a virtual environment for pb usage using the sandbox version of pb instead of the latest stable
343Reserved to dev team.
344
345=item B<sbx2setuprm>
346
347Setup a remote machine for pb usage using the sandbox version of pb instead of the latest stable
348Reserved to dev team.
349
350=item B<build2setupvm>
351
352Setup a virtual machine for pb usage using the build available
353Reserved to dev team.
354
355=item B<build2setupve>
356
357Setup a virtual environment for pb usage using the build available
358Reserved to dev team.
359
360=item B<build2setuprm>
361
362Setup a remote machine for pb usage using the build available
363Reserved to dev team.
364
365=item B<snapvm>
366
367Snapshot a virtual machine for pb usage
368
369=item B<snapve>
370
371Snapshot a virtual environment for pb usage
372
373=item B<updateconf>
374
375Update the configuration files to the latest format
376
377=item B<updatevm>
378
379Update the distribution in the virtual machine
380
381=item B<updateve>
382
383Update the distribution in the virtual environment
384
385=item B<updaterm>
386
387Update the distribution in the remote machine
388
389=item B<test2pkg>
390
391Test a package locally
392
393=item B<test2vm>
394
395Test a package in a virtual machine
396
397=item B<test2ve>
398
399Test a package in a virtual environment
400
401=item B<test2rm>
402
403Test a package in a remote machine
404
405=item B<checkssh>
406
407Check the delivery of the packages on the repository
408
409=item B<checkps>
410
411Check the process running the VM concerned
412
413=item B<newver>
414
415Create a new version of the project derived
416from the current one
417
418=item B<newproj>
419
420Create a new project and a template set of
421configuration files under pbconf
422
423=item B<announce>
424
425Announce the availability of the project through various means
426
427=item B<sbx2webssh>
428
429Create tar files for the website under your CMS.
430Current state of the exported content is taken.
431Deliver the content to the target server using ssh from the exported dir.
432
433=item B<cms2webssh>
434
435Create tar files for the website from your CMS.
436Deliver the content to the target server using ssh from the DVCS.
437
438=item B<sbx2webpkg>
439
440Create tar files for the website under your CMS.
441Current state of the exported content is taken.
442
443=item B<cms2webpkg>
444
445Create tar files for the website under your CMS.
446
447=item B<getconf>
448
449Print the full configuration parameters as found in the various configuration files. Help to debug conf issues.
450Also accepts a parameter to display only this value, and a VM/VE/RM
451
452=item B<getvar>
453
454Print the full variables expanded based on the distribution tuple. Help to debug conf issues.
455Also accepts a parameter to display only the values for this package, and a VM/VE/RM
456
457=item B<clean>
458
459Purge the build and delivery directories related to the current project
460
461=item B<cleanssh>
462
463Purge the ssh server of its packages (only for testver and test packages)
464
465=back
466
467<pkgs> can be a list of packages, the keyword 'all' or nothing, in which case the default list of packages is taken (corresponding to the defpkgdir list of arguments in the configuration file).
468
469=head1 WEB SITES
470
471The main Web site of the project is available at L<http://www.project-builder.org/>. Bug reports should be filled using the trac instance of the project at L<http://trac.project-builder.org/>.
472
473=head1 USER MAILING LIST
474
475None exists for the moment.
476
477=head1 ENVIRONMENT VARIABLES
478
479=item B<PBACCOUNT>   
480
481Login to use to connect to the VM/RM, undef by default
482
483=back
484
485=head1 CONFIGURATION FILES
486
487Each pb user may have a configuration in F<$HOME/.pbrc.yml>. The values in this file may overwrite any other configuration file value.
488
489Here is an example of such a configuration file:
490
491 #
492 # Define for each project the URL of its pbconf repository
493 # No default option allowed here as they need to be all different
494 #
495 # URL of the pbconf content
496 # This is the format of a classical URL with the extension of additional schema such as
497 # svn+ssh, cvs+ssh, ...
498 #
499  pbconfurl:
500    linuxcoe : cvs+ssh://:ext:bcornec@linuxcoe.cvs.sourceforge.net:/cvsroot/linuxcoe/pbconf
501
502 # This is normaly defined in the project's configuration file
503 # Url of the project
504 #
505  pburl:
506    linuxcoe : cvs+ssh://:ext:bcornec@linuxcoe.cvs.sourceforge.net:/cvsroot/linuxcoe
507 
508 # All these URLs needs to be defined here as the are the entry point
509 # for how to build packages for the project
510 #
511  pbconfurl:
512    pb: svn+ssh://svn.project-builder.org/mondo/svn/pb/pbconf
513  pbconfurl:
514    mondorescue: svn+ssh://svn.project-builder.org/mondo/svn/project-builder/mondorescue/pbconf
515  pbconfurl:
516    collectl: svn+ssh://bruno@svn.mondorescue.org/mondo/svn/project-builder/collectl/pbconf
517  pbconfurl:
518    netperf: svn+ssh://svn.mondorescue.org/mondo/svn/project-builder/netperf/pbconf
519 
520 # Under that dir will take place everything related to pb
521 # If you want to use VMs/chroot/..., then use $ENV{'HOME'} to make it portable
522 # to your VMs/chroot/...
523 # if not defined then /var/cache
524  pbdefdir:
525    default: $ENV{'HOME'}/project-builder
526  pbdefdir:
527    pb: $ENV{'HOME'}
528  pbdefdir:
529    linuxcoe: $ENV{'HOME'}/LinuxCOE/cvs
530  pbdefdir:
531    mondorescue: $ENV{'HOME'}/mondo/svn
532 
533 # pbconfdir points to the directory where the CMS content of the pbconfurl is checked out
534 # If not defined, pbconfdir is under pbdefdir/pbproj/pbconf
535  pbconfdir:
536    linuxcoe: $ENV{'HOME'}/LinuxCOE/cvs/pbconf
537  pbconfdir:
538    mondorescue: $ENV{'HOME'}/mondo/svn/pbconf
539 
540 # pbdir points to the directory where the CMS content of the pburl is checked out
541 # If not defined, pbdir is under pbdefdir/pbproj
542 # Only defined if we have access to the dev of the project
543  pbdir:
544    linuxcoe: $ENV{'HOME'}/LinuxCOE/cvs
545  pbdir:
546    mondorescue: $ENV{'HOME'}/mondo/svn
547 
548 # -daemonize doesn't work with qemu 0.8.2
549  vmopt:
550    default: -m 384
551
552=head1 COMMAND DETAILS
553
554=head2 newproj
555
556The newproj command creates a new project-builder project.  To run this command you first need to define two variables in your ~/.pbrc.yml file:
557
558 pbconfurl I<project> = file:///home/anderse/.git/project-builder-config/I<project>
559 pbdefdir default = $ENV{HOME}/cache-project-builder
560
561The first line defines the version controlled configuration information and the second defines the root directory for project-builder to use.
562
563You can then run the command:
564
565 % pb -p I<$project> -r I<$version> newproj I<$pkg>
566
567to create the new project. Running the newproj command will then generate the file $pbdefdir/$project/pbconf/$version/$project.yml, and the directory $pbdefdir/$project/pbconf/$version/$pkg. You will need to edit those files to make the later commands work.
568
569=head2 cms2build
570
571The cms2build command takes your files from the content management system and makes the two tar files that are necessary for building files. You need to have run the newproj command first.  Then there are several steps for running this command:
572
573=over 4
574
575=item Update your $project.yml configuration file.
576
577You need to set the pburl, pbrepo, pbwf, pbpackager, projver, projtag, testver, delivery, and defpkgdir lines as described in the configuration file. The pburl entry is used to find the source for your package. The pbrepo entry is used to build the .repo or .sources.list files for use by downloaders of the package. The pbwf entry indicates that the source tar file is named by I<package-name>-I<version>. The pbpackager entry will be stored in the packages and should be you or your team. The projver/projtag entries indicate the version of the software and the version of the packaging scripts. The testver entry when true indicates that the package is in a test version, so no log file is computed (can be long), and version is made up using a timstamp. The delivery entry gives the subdirectory under which the packages will be delivered on the repository, and the defpkgdir entry corresponds to the local subdirectory hosting the package content.
578
579For example:
580
581 pburl Lintel = file:///home/anderse/projects/Lintel-0.2012.02.28.tar.gz
582 pbrepo Lintel = http://tesla.hpl.hp.com/opensource
583 pbwf Lintel = 1
584 pbpackager Lintel = Eric Anderson <eric.anderson4@hp.com>
585 projver Lintel = 0.2012.02.28
586 projtag Lintel = 1
587 testver Lintel = false
588 delivery Lintel = production
589 defpkgdir Lintel = Lintel-0.2012.02.28
590
591=item Create the build .tar.gz files:
592
593Then you need to take those files and create the initial tar files.  Run a command like:
594
595 % pb -p $project -r $version cms2build
596
597To create the $pbdefdir/$project/pbdelivery/$project-$version.{,pbconf}.tar.gz files, the $version-$projtag.yml and pbrc.yml files in the same directory.
598
599=back
600
601=head2 build2pkg
602
603The build2pkg command takes the tar files created in the cms2build step and attempts to build binary packages for your current operating system. There are two steps:
604
605=over 4
606
607=item Update your filters and build files.
608
609You probably need to edit the files describing the build steps in one of the $pbdefdir/$project/pbconf/$version/$project/{deb,rpm,pkg} directories and the filters in $pbdefdir/$project/pbconf/$version/pbfilter. Note that you can define additional filters and transformations in the filters files. The build files will be filtered by the filters defined in the filters files to generate the inputs to the build step. Therefore, if you change those files, you need to re-run the cms2build step.
610
611=item Build the package.
612
613Then you can run a command like:
614
615 % pb -p $project -r $version build2pkg
616
617To create the files in $project/pbbuild that comprise your binary package(s).
618
619=back
620
621=head2 newve
622 
623The newve command creates a new virtual environment, i.e. a chrooted OS for building packages. Using a virtual environment is an efficient way to build packages on a related set of operating systems. The OS's have to be related because the kernel will be shared.  Steps:
624
625=over 4
626
627=item Update ~/.pbrc.yml
628
629Update your ~/.pbrc.yml file to specify the vepath, velist, velogin, and vetype variables, e.g.:
630
631  vepath:
632    default: $ENV{HOME}/cache-project-builder/chroot
633  velist:
634    default: debian-6.0-i386
635  velogin:
636    default: pb
637  vetype:
638    default: chroot
639
640If you are building for rpm style OS's, update the verpmtype option, and install the appropriate tool.
641
642  verpmtype:
643    default: rpmbootstrap
644
645You may also choose to specify a mirror for the OS packages, and optionally http/ftp proxies.  You can specify the proxies either through environment variables ($http_proxy/$https_proxy/$ftp_proxy) or in the configuration file. The configuration file will be used if no corresponding environment variable has been set. For example, for debian and with a local squid proxy:
646
647  rbsmirrorsrv:
648    debian: http://mirrors1.kernel.org/debian/
649  http_proxy:
650    default: http://localhost:3128/
651  https_proxy:
652    default: http://localhost:3128/
653  ftp_proxy:
654    default: http://localhost:3128/
655
656=item Run the cms2build command
657
658If you have deleted your $package/pbdelivery directory, re-run the cms2build command as in the earlier step. This step is necessary to generate the I<package>/pbdelivery/pbrc.yml file.
659
660=item Create the new virtual environment
661
662Initialize the new operating system. This step will install the core OS packages for the virtual environment, e.g.:
663
664 % pb -v -p $project -m debian-6.0-i386 newve
665
666=back
667
668=head2 setupve
669
670The setupve command prepares a virtual environment for use by project builder. In particular it installs project-builder from the packages into the virtual environment. Two sub-steps are necessary:
671
672=over 4
673
674=item Update $project.yml
675
676You need to have a sshhost entry for setupve to work, so add one, even an invalid one, e.g.:
677
678  sshhost:
679    $project: foo.example.org
680
681=item Setup the virtual environment
682
683 % pb -v -p $project -m debian-6.0-i386 setupve
684
685If you prefer to install the current SVN version of project builder, you can substitute the setupve option by the sbx2setupv one.
686
687=back
688
689=head2 build2ve
690
691The build2ve command is similar to the build2pkg command in that it will take the sources created by cms2build and turn them into binary packages. The command has two differences. First, it creates the packages in a virtual environment, i.e. the one made by an earlier setupve setup. Second it copies the resulting packages to a repository and builds the repository meta-data needed.
692
693Three sub-steps are needed:
694
695=over 4
696
697=item Update $project.yml
698
699You need to have a valid sshdir and sshhost entry for build2ve to work, so add them. Note that you need to be able to ssh from the host you run the command on to the repository host, preferably without needing to type in a password, so using ssh-agent or having a special passwordless project-builder ssh key will make this step easier.
700
701  sshhost:
702    $project: localhost
703  sshdir:
704    $project: $home/cache-project-builder/repos
705
706You may also need to specify additional repository files to use or rpms to install. Note the URL for repositories is not the URL of the repository, but the URL of a file that can be put in the yum.repos.d or apt.sources.d directory.
707
708  addrepo:
709    centos-5-i386: http://localhost/pb/centos-extras.repo,http://mirror.centos.org/centos/5/extras/i386/RPMS/chrpath-0.13-3.el5.centos.i386.rpm
710
711=item Update your filters and build files
712
713You may need to update your filter files as in the build2pkg step if you are building for a new OS or architecture.
714
715=item Build the packages and copy them to the repository
716
717 % pb -v -p $project -m debian-6.0-i386 build2ve
718
719=back
720
721*Debugging:* If the build fails (and you did not specify the --no-stop-on-error) option, then the virtual environment and scripts should still be present and configured to build the package. You can run a command like 'sudo setarch i386 chroot $path bash' in order to get into the environment. In your log you should see a command like that. From there you can go into the /home/pb directory as the pb user and run the same style of pb commands as you did when doing build2pkg. This will help you figure out what has gone wrong in the build in the virtual environment.
722
723=head1 AUTHORS
724
725The Project-Builder.org team L<http://trac.project-builder.org/> lead by Bruno Cornec L<mailto:bruno@project-builder.org>.
726
727=head1 COPYRIGHT
728
729Project-Builder.org is distributed under the GPL v2.0 license
730described in the file C<COPYING> included with the distribution.
731
732=cut
733
734# ---------------------------------------------------------------------------
735
736my ($projectbuilderver,$projectbuilderrev) = pb_version_init();
737my $appname = "pb";
738
739# Initialize the syntax string
740
741pb_syntax_init("$appname (aka project-builder.org) Version $projectbuilderver-$projectbuilderrev\n");
742
743GetOptions("help|?|h+" => \$opts{'h'}, 
744        "account|a=s" => \$opts{'a'},
745        "force|f" => \$opts{'f'},
746        "nographic|g" => \$opts{'g'},
747        "image|i=s" => \$opts{'i'},
748        "log-files|l=s" => \$opts{'l'},
749        "machines|mock|m=s" => \$opts{'m'},
750        "man" => \$opts{'man'},
751        "port|P=i" => \$opts{'P'},
752        "project|p=s" => \$opts{'p'},
753        "quiet|q" => \$opts{'q'},
754        "revision|r=s" => \$opts{'r'},
755        "rebuild" => \$opts{'rebuild'},
756        "snapshot|S" => \$opts{'S'},
757        "script|s=s" => \$opts{'s'},
758        "type|T:s" => \$opts{'T'},
759        "target|t:s" => \$opts{'t'},
760        "version" => \$opts{'version'},
761        "V=s" => \$opts{'V'},
762        "verbose|v+" => \$opts{'v'},
763        "keep|k" => \$opts{'k'},
764        "stop-on-error!" => \$Global::pb_stop_on_error,
765) || pb_syntax(-1,0);
766
767if (defined $opts{'l'}) {
768    open(pbLOG,"> $opts{'l'}") || die "Unable to log to $opts{'l'}: $!";
769    $pbLOG = \*pbLOG;
770    $pbdebug = 0  if ($pbdebug == -1);
771    }
772pb_log_init($pbdebug, $pbLOG);
773
774if (defined $opts{'h'}) {
775    pb_syntax(0,1);
776}
777if (defined $opts{'version'}) {
778    pb_syntax(0,0);
779}
780if (defined $opts{'man'}) {
781    pb_syntax(0,2);
782}
783if (defined $opts{'v'}) {
784    $pbdebug = $opts{'v'};
785}
786if (defined $opts{'f'}) {
787    $pbforce=1;
788}
789if (defined $opts{'q'}) {
790    $pbdebug=-1;
791}
792if (defined $opts{'S'}) {
793    $pbsnap=1;
794}
795if (defined $opts{'k'}) {
796    $pbkeep=1;
797}
798pb_display_init("text","");
799
800# Handle root of the project if defined
801if (defined $opts{'r'}) {
802    $ENV{'PBROOTDIR'} = $opts{'r'};
803}
804# Handle virtual machines if any
805if (defined $opts{'m'}) {
806    $ENV{'PBV'} = $opts{'m'};
807}
808if (defined $opts{'s'}) {
809    $pbscript{'default'} = $opts{'s'};
810    die "option -s requires a script having a full path name" if ($pbscript{'default'} !~ /^\//);
811}
812if (defined $opts{'a'}) {
813    $ENV{'PBACCOUNT'} = $opts{'a'};
814    die "option -a requires a -s script option" if (not defined $pbscript{'default'});
815}
816if (defined $opts{'P'}) {
817    $pbport = $opts{'P'};
818}
819if (defined $opts{'V'}) {
820    $newver = $opts{'V'};
821}
822if (defined $opts{'i'}) {
823    $pbimage = $opts{'i'};
824}
825if (defined $opts{'t'}) {
826    $pbtarget = $opts{'t'};
827}
828if (defined $opts{'T'}) {
829    $vetype = $opts{'T'};
830}
831
832# Get Action
833$action = shift @ARGV;
834die pb_syntax(-1,1) if (not defined $action);
835
836my ($filteredfiles, $supfiles, $defpkgdir, $extpkgdir);
837my $pbinit = undef;
838$pbinit = 1 if ($action =~ /^newproj$/);
839
840# Handles project name if any
841# And get global params
842($filteredfiles, $supfiles, $defpkgdir, $extpkgdir) = pb_env_init($opts{'p'},$pbinit,$action,$pbkeep);
843
844#
845# Check for command requirements
846#
847my ($req,$opt,$pbpara,$pbshowsudo,$pbstoponerr) = pb_conf_get_if("oscmd","oscmdopt","pbparallel","pbshowsudo","pbstoponerr");
848pb_check_requirements($req,$opt,$appname);
849
850$Global::pb_show_sudo = 1 if ((defined $pbshowsudo) && (defined $pbshowsudo->{$ENV{'PBPROJ'}}) && ($pbshowsudo->{$ENV{'PBPROJ'}} =~ /true/oi));
851$Global::pb_stop_on_error = 1 if ((defined $pbstoponerr) && (defined $pbstoponerr->{$ENV{'PBPROJ'}}) && ($pbstoponerr->{$ENV{'PBPROJ'}} =~ /true/oi));
852
853#
854# Check if we can launch some actions in // with Parallel::ForkManager
855#
856my $pbparallel = $pbpara->{$appname} if (defined $pbpara);
857if (not defined $pbparallel) {
858    eval
859    {
860        require Sys::CPU;
861        Sys::CPU->import();
862    };
863    if ($@) {
864        # Sys::CPU not found, defaulting to 1
865        pb_log(1,"ADVISE: Install Sys::CPU to benefit from automatic parallelism optimization.\nOr use pbparallel in your pb.conf file\nOnly 1 process at a time for the moment\n");
866        $pbparallel = 1;
867    } else {
868        # Using the number of cores
869        $pbparallel = Sys::CPU::cpu_count();
870        pb_log(1,"Using parallel mode with $pbparallel processes\n");
871    }
872}
873
874eval
875{
876    require Parallel::ForkManager;
877    Parallel::ForkManager->import();
878};
879# Parallel::ForkManager not found so no // actions
880if ($@) {
881    $pbparallel = undef;
882    pb_log(1,"ADVISE: Install Parallel::ForkManager to benefit from automatic parallelism optimization.\nOnly 1 process at a time for the moment\n");
883}
884
885pb_log(0,"Project: $ENV{'PBPROJ'}\n");
886pb_log(0,"Action: $action\n");
887
888my $do_install = undef;
889# Possibility to add a 2ins to some commands to launch automatic installation
890$do_install = 1 if ($action =~ /2ins$/);
891
892# Act depending on action
893if ($action =~ /^cms2build$/) {
894    pb_cms2build("CMS");
895} elsif ($action =~ /^sbx2build$/) {
896    pb_cms2build("SandBox");
897} elsif ($action =~ /^build2prep/) {
898    pb_build2prep();
899} elsif ($action =~ /^build2pkg/) {
900    pb_build2pkg($do_install);
901} elsif ($action =~ /^cms2pkg/) {
902    pb_cms2build("CMS");
903    pb_build2pkg($do_install);
904} elsif ($action =~ /^sbx2pkg/) {
905    pb_cms2build("SandBox");
906    pb_build2pkg($do_install);
907} elsif ($action =~ /^build2ssh$/) {
908    pb_build2ssh();
909} elsif ($action =~ /^cms2ssh$/) {
910    pb_cms2build("CMS");
911    pb_build2ssh();
912} elsif ($action =~ /^sbx2ssh$/) {
913    pb_cms2build("SandBox");
914    pb_build2ssh();
915} elsif ($action =~ /^pkg2ssh$/) {
916    pb_pkg2ssh();
917} elsif ($action =~ /^build2rm$/) {
918    pb_build2v("rm","build");
919} elsif ($action =~ /^build2ve$/) {
920    pb_build2v("ve","build");
921} elsif ($action =~ /^build2vm$/) {
922    pb_build2v("vm","build");
923} elsif ($action =~ /^sbx2preprm$/) {
924    pb_cms2build("SandBox");
925    pb_build2v("rm","prep");
926} elsif ($action =~ /^sbx2prepve$/) {
927    pb_cms2build("SandBox");
928    pb_build2v("ve","prep");
929} elsif ($action =~ /^sbx2prepvm$/) {
930    pb_cms2build("SandBox");
931    pb_build2v("vm","prep");
932} elsif ($action =~ /^preprm$/) {
933    pb_build2v("rm","prep");
934} elsif ($action =~ /^prepve$/) {
935    pb_build2v("ve","prep");
936} elsif ($action =~ /^prepvm$/) {
937    pb_build2v("vm","prep");
938} elsif ($action =~ /^cms2rm$/) {
939    pb_cms2build("CMS");
940    pb_build2v("rm","build");
941} elsif ($action =~ /^cms2ve$/) {
942    pb_cms2build("CMS");
943    pb_build2v("ve","build");
944} elsif ($action =~ /^sbx2rm$/) {
945    pb_cms2build("SandBox");
946    pb_build2v("rm","build");
947} elsif ($action =~ /^sbx2ve$/) {
948    pb_cms2build("SandBox");
949    pb_build2v("ve","build");
950} elsif ($action =~ /^cms2vm$/) {
951    pb_cms2build("CMS");
952    pb_build2v("vm","build");
953} elsif ($action =~ /^sbx2vm$/) {
954    pb_cms2build("SandBox");
955    pb_build2v("vm","build");
956} elsif ($action =~ /^launchvm$/) {
957    pb_parallel_launchv(undef,"vm",undef,3,$pbimage);
958} elsif ($action =~ /^launchve$/) {
959    pb_parallel_launchv(undef,"ve",undef,3,$pbimage);
960} elsif ($action =~ /^script2vm$/) {
961    die "action script2vm requires a -s script option" if (not defined $pbscript{'default'});
962    pb_script2v(\%pbscript,"vm");
963} elsif ($action =~ /^script2ve$/) {
964    die "action script2ve requires a -s script option" if (not defined $pbscript{'default'});
965    pb_script2v(\%pbscript,"ve");
966} elsif ($action =~ /^script2rm$/) {
967    die "action script2rm requires a -s script option" if (not defined $pbscript{'default'});
968    pb_script2v(\%pbscript,"rm");
969} elsif ($action =~ /^newver$/) {
970    pb_newver();
971} elsif ($action =~ /^sbx2docker$/) {
972    my $savproj = $ENV{'PBPROJ'};
973    my $pkg;
974    my @pkg;
975    if ($#ARGV == -1) {
976        $pkg = pb_cms_get_pkg($defpkgdir,$extpkgdir);
977        @pkgs = @$pkg;
978    } else {
979        @pkgs = @ARGV;
980    }
981    #pb_log(1,"Packages to print:".Dumper(@pkgs)."\n");
982   
983    # Get the list of all VE we need to work on
984    my ($vm,$all);
985    if (not defined $ENV{'PBV'}) {
986        ($vm,$all) = pb_get2v("ve");
987    } else {
988        @$vm = split(/,/,$ENV{'PBV'});
989    }
990   
991    # TODO: parallelize
992    foreach my $v (@$vm) {
993        my $done = 0;
994        my $pbos = pb_distro_get_context($v);
995        # Docker images are (for now !) regular and use distro:ver
996        $pbimage = "$pbos->{'name'}".":$pbos->{'version'}";
997        # the first steps needs to be done with the pb project
998        $ENV{'PBPROJ'} = $appname;
999        my $found = pb_ve_docker_get_image("$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}");
1000        if (($pbforce == 1) || (not defined $found)) {
1001            # pb -p pb -m distro-ver-arch newve -i distro:ver
1002            pb_log(0,"Docker container $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'} not found, creating it\n");
1003            pb_parallel_launchv(undef,"ve",undef,0,$pbimage);
1004        }
1005        $found = pb_ve_docker_get_image("$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-pb");
1006        if (($pbforce == 1) || (not defined $found)) {
1007            pb_log(0,"Docker container $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-pb not found, creating it\n");
1008            # pb -p pb -m distro-ver-arch setupve
1009            if ($savproj eq "pb") {
1010                pb_cms2build("SandBox");
1011                $done = 1;
1012                pb_setup2v("ve","SandBox");
1013            } else {
1014                pb_setup2v("ve");
1015            }
1016        }
1017        # Now back to our original project
1018        $ENV{'PBPROJ'} = $savproj;
1019        $found = pb_ve_docker_get_image("$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-pb-$ENV{'PBPROJ'}");
1020        if (($pbforce == 1) || (not defined $found)) {
1021            pb_log(0,"Docker container $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-pb-$ENV{'PBPROJ'} not found, creating it\n");
1022            # pb -p pb -m distro-ver-arch prepve
1023            if ($savproj eq "pb") {
1024                pb_cms2build("SandBox");
1025                $done = 1;
1026            }
1027            pb_build2v("ve","prep");
1028        }
1029        pb_log(0,"Instantiating a Docker container from $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-pb-$ENV{'PBPROJ'}\n");
1030        if (($savproj ne "pb") || ($done == 0)) {
1031            pb_cms2build("SandBox");
1032        }
1033        pb_build2v("ve","build");
1034    }
1035} elsif ($action =~ /^newve$/) {
1036    pb_parallel_launchv(undef,"ve",undef,0,$pbimage);
1037} elsif ($action =~ /^newvm$/) {
1038    pb_parallel_launchv(undef,"vm",undef,0,$pbimage);
1039    pb_log(0, "Please ensure that sshd is running in your VM by default\n");
1040    pb_log(0, "and that it allows remote root login (PermitRootLogin yes in /etc/ssh/sshd_config)\n");
1041    pb_log(0, "Also ensure that network is up, firewalling correctly configured\n");
1042    pb_log(0, "and perl, Data::Dumper, tar, sudo, ntpdate and scp/ssh installed\n");
1043    pb_log(0, "You should then be able to login with ssh -p VMPORT root\@localhost (if VM started with pb)\n");
1044} elsif ($action =~ /^setuprm$/) {
1045    pb_setup2v("rm");
1046} elsif ($action =~ /^setupve$/) {
1047    pb_setup2v("ve");
1048} elsif ($action =~ /^setupvm$/) {
1049    pb_setup2v("vm");
1050} elsif ($action =~ /^sbx2setuprm$/) {
1051    die "This feature is limited to the pb project" if ($ENV{'PBPROJ'} ne $appname);
1052    pb_cms2build("SandBox");
1053    pb_setup2v("rm","SandBox");
1054} elsif ($action =~ /^sbx2setupve$/) {
1055    die "This feature is limited to the pb project" if ($ENV{'PBPROJ'} ne $appname);
1056    pb_cms2build("SandBox");
1057    pb_setup2v("ve","SandBox");
1058} elsif ($action =~ /^sbx2setupvm$/) {
1059    die "This feature is limited to the pb project" if ($ENV{'PBPROJ'} ne $appname);
1060    pb_cms2build("SandBox");
1061    pb_setup2v("vm","SandBox");
1062} elsif ($action =~ /^build2setuprm$/) {
1063    die "This feature is limited to the pb project" if ($ENV{'PBPROJ'} ne $appname);
1064    pb_setup2v("rm","SandBox");
1065} elsif ($action =~ /^build2setupve$/) {
1066    die "This feature is limited to the pb project" if ($ENV{'PBPROJ'} ne $appname);
1067    pb_setup2v("ve","SandBox");
1068} elsif ($action =~ /^build2setupvm$/) {
1069    die "This feature is limited to the pb project" if ($ENV{'PBPROJ'} ne $appname);
1070    pb_setup2v("vm","SandBox");
1071} elsif ($action =~ /^updaterm$/) {
1072    pb_update2v("rm");
1073} elsif ($action =~ /^updateve$/) {
1074    pb_update2v("ve");
1075} elsif ($action =~ /^updatevm$/) {
1076    pb_update2v("vm");
1077} elsif ($action =~ /^snapve$/) {
1078    pb_snap2v("ve");
1079} elsif ($action =~ /^snapvm$/) {
1080    pb_snap2v("vm");
1081} elsif ($action =~ /^test2pkg$/) {
1082    pb_test2pkg();
1083} elsif ($action =~ /^test2rm$/) {
1084    pb_build2v("rm","test");
1085} elsif ($action =~ /^test2ve$/) {
1086    pb_build2v("ve","test");
1087} elsif ($action =~ /^test2vm$/) {
1088    pb_build2v("vm","test");
1089} elsif ($action =~ /^newproj$/) {
1090    # Nothing to do - already done in pb_env_init
1091} elsif ($action =~ /^updateconf$/) {
1092    foreach my $f (@ARGV) {
1093        my $fyml = $f;
1094        $fyml =~ s/\.pbf/.yml/;
1095        $fyml =~ s/\.pb/.yml/;
1096        $fyml =~ s/\.pbrc/.pbrc.yml/;
1097        pb_conf_update_v0($f,$fyml);
1098    }
1099} elsif ($action =~ /^getconf$/) {
1100    my $pbos = pb_distro_get_context($ENV{'PBV'});
1101    pb_log(1,"Arguments to print:".Dumper(@ARGV)."\n");
1102    pb_distro_conf_print($pbos,@ARGV);
1103} elsif ($action =~ /^getvar$/) {
1104    my $pbos = pb_distro_get_context($ENV{'PBV'});
1105    my $pkg;
1106    my @pkg;
1107    if ($#ARGV == -1) {
1108        $pkg = pb_cms_get_pkg($defpkgdir,$extpkgdir);
1109        @pkgs = @$pkg;
1110    } else {
1111        @pkgs = @ARGV;
1112    }
1113    pb_log(1,"Packages to print:".Dumper(@pkgs)."\n");
1114    pb_filter_var_print($pbos,@pkgs);
1115} elsif ($action =~ /^clean$/) {
1116    pb_clean();
1117} elsif ($action =~ /^announce$/) {
1118    # For announce only. Require avoids the systematic load of these modules
1119    require DBI;
1120    require DBD::SQLite;
1121
1122    pb_announce("Announce");
1123} elsif ($action =~ /^cleanssh$/) {
1124    pb_announce("Clean");
1125} elsif ($action =~ /^checkssh$/) {
1126    pb_announce("Check");
1127} elsif ($action =~ /^checkps$/) {
1128    my ($ptr2) = pb_conf_get("vmcmd");
1129    my $vmcmd = $ptr2->{$ENV{'PBPROJ'}};
1130    my ($vexist,$vmmport) = pb_check_ps($vmcmd,$ENV{'PBV'});
1131    if (! $vexist) {
1132        pb_log(0,"No VM found for $ENV{'PBV'}\n");
1133    } else {
1134        pb_log(0,"Found an existing VM for $ENV{'PBV'} PID: $vexist - SSH port: $vmmport\n");
1135    }
1136} elsif ($action =~ /^sbx2webpkg$/) {
1137    require DBI;
1138    require DBD::SQLite;
1139
1140    pb_cms2build("SandBox","Web");
1141} elsif ($action =~ /^sbx2webssh$/) {
1142    require DBI;
1143    require DBD::SQLite;
1144
1145    pb_cms2build("SandBox","Web");
1146    pb_send2target("Web");
1147} elsif ($action =~ /^cms2webpkg$/) {
1148    require DBI;
1149    require DBD::SQLite;
1150
1151    pb_cms2build("CMS","Web");
1152} elsif ($action =~ /^cms2webssh$/) {
1153    require DBI;
1154    require DBD::SQLite;
1155
1156    pb_cms2build("CMS","Web");
1157    pb_send2target("Web");
1158} else {
1159    pb_log(0,"\'$action\' is not available\n");
1160    pb_syntax(-2,1);
1161}
1162
1163pb_exit();
1164
1165sub pb_cms2build {
1166
1167    my $param = shift;
1168    my $web = shift;
1169
1170    my $pkg;
1171    my @pkgs;
1172    my $webdir;
1173
1174    my %pkgs;
1175    my $pb;             # Structure to store conf info
1176
1177    die "pb_cms2build requires a parameter: SandBox or CMS" if (not defined $param);
1178
1179    # If Website, then pkg is only the website
1180    if (defined $web) {
1181        ($webdir) = pb_conf_get("webdir");
1182        pb_log(2,"webdir: ".Dumper($webdir)."\n");
1183        $pkgs[0] = $webdir->{$ENV{'PBPROJ'}};
1184        $extpkgdir = $webdir;
1185        pb_log(0,"Package: $pkgs[0]\n");
1186    } else {
1187        $pkg = pb_cms_get_pkg($defpkgdir,$extpkgdir);
1188        @pkgs = @$pkg;
1189    }
1190
1191    my ($scheme, $uri) = pb_cms_init($pbinit,$param);
1192
1193    # We need 2 lines here
1194    my ($pkgv, $pkgt, $testver) = pb_conf_get_if("pkgver","pkgtag","testver");
1195
1196    # declare packager and repo for filtering
1197    my ($tmp1, $tmp2) = pb_conf_get("pbpackager","pbrepo");
1198    $ENV{'PBPACKAGER'} = $tmp1->{$ENV{'PBPROJ'}};
1199    $ENV{'PBREPO'} = $tmp2->{$ENV{'PBPROJ'}};
1200    my ($delivery) = pb_conf_get_if("delivery");
1201    $delivery->{$ENV{'PBPROJ'}} = "" if (not defined $delivery->{$ENV{'PBPROJ'}});
1202
1203    # If we deal with a test dir, we want to keep the date in tar file and dir name
1204    my $pbextdirpkg = "";
1205    my $pbextdir = "";
1206    if ((defined $testver) && (defined $testver->{$ENV{'PBPROJ'}}) && ($testver->{$ENV{'PBPROJ'}} =~ /true/i)) {
1207        $pbextdirpkg = strftime("%Y%m%d%H%M%S", @date);
1208        $pbextdir = ".0.".$pbextdirpkg;
1209    }
1210
1211    foreach my $pbpkg (@pkgs) {
1212        $ENV{'PBPKG'} = $pbpkg;
1213        $pbrev = $ENV{'PBREVISION'};
1214
1215        if ((defined $pkgv) && (defined $pkgv->{$pbpkg})) {
1216            $pbver = $pkgv->{$pbpkg};
1217        } else {
1218            $pbver = $ENV{'PBPROJVER'};
1219        }
1220        # If it's a test version, then build a tag to fit distributions need
1221        if ((defined $testver) && (defined $testver->{$ENV{'PBPROJ'}}) && ($testver->{$ENV{'PBPROJ'}} =~ /true/i)) {
1222            my $letter = "";
1223            if ($scheme =~ /svn/) {
1224                $letter = "s";
1225            } elsif ($scheme =~ /cvs/) {
1226                $letter = "c";
1227            } elsif ($scheme =~ /svk/) {
1228                $letter = "k";
1229            } elsif ($scheme =~ /hg/) {
1230                $letter = "h";
1231            } elsif ($scheme =~ /git/) {
1232                $letter = "g";
1233            } else {
1234                $letter = "f";
1235            }
1236            $pbtag = "0.$pbextdirpkg.$letter$pbrev";
1237            $ENV{'PBPROJTAG'} = $pbtag;
1238        } elsif ((defined $pkgt) && (defined $pkgt->{$pbpkg})) {
1239            $pbtag = $pkgt->{$pbpkg};
1240        } else {
1241            $pbtag = $ENV{'PBPROJTAG'};
1242        }
1243
1244        pb_log(0,"\n");
1245        pb_log(0,"Management of $pbpkg $pbver-$pbtag\n");
1246        die "Unable to get env var PBDESTDIR" if (not defined $ENV{'PBDESTDIR'});
1247
1248        my $dest = "$ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir";
1249        # Create the structure if needed
1250        pb_mkdir_p($dest);
1251        # Clean up dest if necessary. The export will recreate it
1252        pb_rm_rf($dest) if (-d $dest);
1253
1254        # Export CMS tree for the concerned package to dest
1255        # And generate some additional files
1256        $OUTPUT_AUTOFLUSH=1;
1257
1258        # computes in which dir we have to work
1259        my $dir = $defpkgdir->{$pbpkg};
1260        $dir = $extpkgdir->{$pbpkg} if (not defined $dir);
1261        $dir = $webdir->{$ENV{'PBPROJ'}} if (defined $web);
1262        die "Variable \$dir not defined. Check the package name first.\nIf this is the one expected, please report to dev team with log of a verbose run and this info ".Dumper($defpkgdir,$extpkgdir,$webdir) if (not defined $dir);
1263        pb_log(2,"def:".Dumper($defpkgdir)."ext: ".Dumper($extpkgdir)."dir: $dir\n");
1264
1265        # Exporting content from CMS
1266        my $sourcedir = undef;
1267        my $sourceuri = $uri;
1268        if ($param eq "SandBox") {
1269            # Point to the local instance
1270            $sourcedir = "$ENV{'PBDIR'}/$dir";
1271        } else {
1272            # Get it from a subdir of the URI with same version as localy but different root
1273            # Only if using a real CMS
1274            my ($scheme, $account, $host, $port, $path) = pb_get_uri($uri);
1275            if (($scheme !~ /^file/) && ($scheme !~ /^(ht|f)tp/)) {
1276                $sourceuri = "$ENV{'PBDIR'}/$dir"; 
1277                $sourceuri =~ s|^$ENV{'PBPROJDIR'}/|$uri/|;
1278            }
1279        }
1280        my $preserve = pb_vcs_export($sourceuri,$sourcedir,$dest);
1281
1282        # Generated fake content for test versions to speed up stuff
1283        my $chglog;
1284
1285        # Get project info on authors and log file
1286        # TODO: Make it CMS aware
1287        $chglog = "$ENV{'PBROOTDIR'}/$pbpkg/pbcl";
1288        $chglog = "$ENV{'PBROOTDIR'}/pbcl" if (! -f $chglog);
1289        $chglog = undef if (! -f $chglog);
1290
1291        # TODO: Make it CMS aware
1292        my $authors = "$ENV{'PBROOTDIR'}/$pbpkg/pbauthors";
1293        $authors = "$ENV{'PBROOTDIR'}/pbauthors" if (! -f $authors);
1294        $authors = "/dev/null" if (! -f $authors);
1295
1296        # Extract cms log history and store it
1297        if ((defined $chglog) && (! -f "$dest/NEWS")) {
1298            pb_log(2,"Generating NEWS file from $chglog\n");
1299            copy($chglog,"$dest/NEWS") || die "Unable to create $dest/NEWS";
1300        }
1301        pb_cms_log($scheme,"$ENV{'PBDIR'}/$dir",$dest,$chglog,$authors,$testver);
1302
1303        my %build;
1304        # We want to at least build for the underlying distro
1305        # except if a target was given, in which case we only build for it
1306        # if -t was passed without target then build for the native distro.
1307        my $pbos = pb_distro_get_context($pbtarget);
1308        my $tmpl = pb_get_distros($pbos,$pbtarget);
1309
1310        # Setup $pb structure to allow filtering later on, on files using that structure
1311        $pb->{'tag'} = $pbtag;
1312        $pb->{'rev'} = $pbrev;
1313        $pb->{'ver'} = $pbver;
1314        $pb->{'pkg'} = $pbpkg;
1315        $pb->{'suf'} = $pbos->{'suffix'};
1316        $pb->{'realpkg'} = $pbpkg;
1317        $pb->{'date'} = $pbdate;
1318        $pb->{'defpkgdir'} = $defpkgdir;
1319        $pb->{'extpkgdir'} = $extpkgdir;
1320        $pb->{'chglog'} = $chglog;
1321        $pb->{'extdir'} = $pbextdir;
1322        $pb->{'packager'} = $ENV{'PBPACKAGER'};
1323        $pb->{'proj'} = $ENV{'PBPROJ'};
1324        $pb->{'repo'} = "$ENV{'PBREPO'}/$delivery->{$ENV{'PBPROJ'}}";
1325        $pb->{'patches'} = ();
1326        $pb->{'sources'} = ();
1327   
1328        my $tmpd = "$ENV{'PBTMP'}/cms.$$";
1329        pb_mkdir_p($tmpd) if (defined $pbparallel);
1330
1331        # Get only all.yml filter at this stage for pbinit
1332        my $ptr = pb_get_filters($pbpkg);
1333
1334        # Do not do that for website
1335        if (not defined $web) {
1336            my %virt;
1337            # De-duplicate similar VM/VE/RM
1338            foreach my $d (split(/,/,$tmpl)) {
1339                # skip ill-formatted vms (name-ver-arch)
1340                next if ($d !~ /-/);
1341                $virt{$d} = $d;
1342            }
1343
1344            # Try to use // processing here
1345            my $all_ok = 1;
1346            my $pm = new Parallel::ForkManager($pbparallel) if (defined $pbparallel);
1347            $pm->run_on_finish(sub { my ($pid, $code, $id, $signal, $dump) = @_;
1348                                    $all_ok = 0 unless (($code == 0) && ($signal == 0) && ($dump == 0)); }) if (defined $pbparallel);
1349
1350            pb_log(0,"Preparing delivery ...\n");
1351            foreach my $v (keys %virt) {
1352                $pm->start and next if (defined $pbparallel);
1353
1354                # Distro context
1355                my $pbos = pb_distro_get_context($v);
1356   
1357                $pb->{'pbos'} = $pbos;
1358                $pb->{'suf'} = $pbos->{'suffix'};
1359                pb_log(3,"DEBUG: pb: ".Dumper($pb)."\n");
1360
1361                # Get all filters to apply
1362                $ptr = pb_get_filters($pbpkg,$pbos);
1363   
1364                pb_log(2,"DEBUG Filtering PBDATE => $pbdate, PBTAG => $pbtag, PBVER => $pbver\n");
1365   
1366                # We need to compute the real name of the package
1367                my $pbrealpkg = pb_cms_get_real_pkg($pbpkg,$pbos->{'type'});
1368                $pb->{'realpkg'} = $pbrealpkg;
1369                pb_log(2,"Virtual package $pbpkg has a real package name of $pbrealpkg on $pbos->{'name'}-$pbos->{'version'}\n") if ($pbrealpkg ne $pbpkg);
1370   
1371                # Filter build files from the less precise up to the most with overloading
1372                # Filter all files found, keeping the name, and generating in dest
1373   
1374                # Find all build files first relatively to PBROOTDIR
1375                # Find also all specific files referenced in the .yml conf file
1376                my %bfiles = ();
1377                my %pkgfiles = ();
1378                # Used in Filter.pm by pb_filter_file
1379
1380                $build{$v} = "no";
1381                if (-d "$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'os'}") {
1382                    pb_list_bfiles("$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'os'}",$pbpkg,\%bfiles,\%pkgfiles,$supfiles);
1383                    $build{$v} = "yes";
1384                } 
1385                if (-d "$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'type'}") {
1386                    pb_list_bfiles("$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'type'}",$pbpkg,\%bfiles,\%pkgfiles,$supfiles);
1387                    $build{$v} = "yes";
1388                } 
1389                if (-d "$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'family'}") {
1390                    pb_list_bfiles("$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'family'}",$pbpkg,\%bfiles,\%pkgfiles,$supfiles);
1391                    $build{$v} = "yes";
1392                } 
1393                if (-d "$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'name'}") {
1394                    pb_list_bfiles("$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'name'}",$pbpkg,\%bfiles,\%pkgfiles,$supfiles);
1395                    $build{$v} = "yes";
1396                } 
1397                if (-d "$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'name'}-$pbos->{'version'}") {
1398                    pb_list_bfiles("$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'name'}-$pbos->{'version'}",$pbpkg,\%bfiles,\%pkgfiles,$supfiles);
1399                    $build{$v} = "yes";
1400                } 
1401                if (-d "$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}") {
1402                    pb_list_bfiles("$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}",$pbpkg,\%bfiles,\%pkgfiles,$supfiles);
1403                    $build{$v} = "yes";
1404                }
1405                pb_log(2,"DEBUG($v) bfiles: ".Dumper(\%bfiles)."\n");
1406   
1407                if ($build{$v} ne "no") {
1408                    # Apply now all the filters on all the files concerned
1409                    # destination dir depends on the type of file
1410                    # For patch support
1411                    # TODO: Make it CMS aware
1412                    $pb->{'patches'} = pb_list_sfiles("$ENV{'PBROOTDIR'}/$pbpkg/pbpatch", $pb->{'patches'}, $pbos, "$ENV{'PBROOTDIR'}/$pbpkg/pbextpatch");
1413                    pb_log(2,"DEBUG($v) patches: ".Dumper($pb->{'patches'})."\n");
1414                    # TODO: Make it CMS aware
1415                    $pb->{'sources'} = pb_list_sfiles("$ENV{'PBROOTDIR'}/$pbpkg/pbsrc", $pb->{'sources'}, $pbos, "$ENV{'PBROOTDIR'}/$pbpkg/pbextsrc");
1416                    pb_log(2,"DEBUG($v) sources: ".Dumper($pb->{'sources'})."\n");
1417   
1418                    if (defined $pb->{'patches'}->{$v}) {
1419                        # Filter potential patches (local + remote)
1420                        pb_mkdir_p("$dest/pbconf/$v/pbpatch");
1421                        # If Debian based distribution, then prepare what will be done at build time
1422                        my ($patchcmd,$patchopt);
1423                        if ($pbos->{'type'} eq "deb") {
1424                            ($patchcmd,$patchopt) = pb_distro_get_param($pbos,pb_conf_get_if("ospatchcmd","ospatchopt"));
1425                            open(SCRIPT,"> $dest/pbconf/$v/pbpatch/pbapplypatch") || die "Unable to create $dest/pbconf/$v/pbpatch/pbapplypatch";
1426                            print SCRIPT "#!/bin/bash\n";
1427                            print SCRIPT "set -x\n" if ($pbdebug gt 1);
1428                        }
1429                        foreach my $pf (split(/,/,$pb->{'patches'}->{$v})) {
1430                            my $pp = basename($pf);
1431                            pb_vcs_export($pf,undef,"$dest/pbconf/$v/pbpatch");
1432                            pb_filter_file_inplace($ptr,"$dest/pbconf/$v/pbpatch/$pp",$pb);
1433                            pb_system("gzip -9f $dest/pbconf/$v/pbpatch/$pp","","quiet");
1434                            if ($pbos->{'type'} eq "deb") {
1435                                # If Debian based distribution, then prepare what will be done at build time
1436                                # by applying the patches that will be available under the debian/patches dir
1437                                print SCRIPT "$patchcmd $patchopt \< debian/patches/$pp\n";
1438                            }
1439                        }
1440                        if ($pbos->{'type'} eq "deb") {
1441                            close(SCRIPT);
1442                            chmod 0755,"$dest/pbconf/$v/pbpatch/pbapplypatch";
1443                        }
1444                        #pb_system("cat $dest/pbconf/$v/pbpatch/pbapplypatch","APPLY","verbose");
1445                    }
1446                    if (defined $pb->{'sources'}->{$v}) {
1447                        pb_mkdir_p("$dest/pbconf/$v/pbsrc");
1448                        foreach my $pf (split(/,/,$pb->{'sources'}->{$v})) {
1449                            my $pp = basename($pf);
1450                            pb_vcs_export($pf,undef,"$dest/pbconf/$v/pbsrc");
1451                            pb_filter_file_inplace($ptr,"$dest/pbconf/$v/pbsrc/$pp",$pb);
1452                        }
1453                    }
1454                    # Filter build files at the end, as they depend on patches and sources
1455                    foreach my $f (keys %bfiles) {
1456                        pb_filter_file("$ENV{'PBROOTDIR'}/$bfiles{$f}",$ptr,"$dest/pbconf/$v/$f",$pb);
1457                    }
1458                    foreach my $f (keys %pkgfiles) {
1459                        pb_filter_file("$ENV{'PBROOTDIR'}/$pkgfiles{$f}",$ptr,"$dest/pbconf/$v/$f",$pb);
1460                    }
1461                }
1462
1463                if (defined $pbparallel) {
1464                    # Communicate results back to parent
1465                    my $str = "";
1466                    $str .= "build $v = $build{$v}\n" if (defined $build{$v});
1467                    $str .= "patches $v = $pb->{'patches'}->{$v}\n" if (defined $pb->{'patches'}->{$v});
1468                    $str .= "sources $v = $pb->{'sources'}->{$v}\n" if (defined $pb->{'sources'}->{$v});
1469                    pb_set_content("$tmpd/$$","$str");
1470                    $pm->finish;
1471                }
1472            }
1473            # In the parent, we need to get the result from the children
1474            $pm->wait_all_children if (defined $pbparallel);
1475            die "Aborting, one or more of the children failed." if ((not $all_ok) && ($Global::pb_stop_on_error));
1476            my $made = "";
1477            my %h = ();
1478            my %tmp;
1479            my %tmp2;
1480            my $pt;
1481            my $k;
1482
1483            foreach $k (<$tmpd/*>) {
1484                $made .= pb_get_content($k);
1485            }
1486            pb_rm_rf($tmpd);
1487            pb_log(3,"MADE:\n$made");
1488
1489            # Rebuild local hashes
1490            foreach $k (split(/\n/,$made)) {
1491                if ($k =~ /^\s*([A-z0-9-_]+)\s+([[A-z0-9-_.]+)\s*=\s*(.+)$/) {
1492                    $h{$1}->{$2}=$3;
1493                }
1494            }
1495            pb_log(2,"HASH: ".Dumper(%h));
1496
1497            # Patches
1498            pb_log(0,"Delivered and compressed patches ");
1499            $pt = $h{'patches'};
1500            foreach $k (keys %$pt) {
1501                foreach my $v1 (split(/,/,$pt->{$k})) {
1502                    $tmp{$v1} = "";
1503                }
1504            }
1505            if (keys %tmp) {
1506                foreach $k (keys %tmp) {
1507                    pb_log(0,"$k ");
1508                }
1509            } else {
1510                pb_log(0,"N/A");
1511            }
1512            pb_log(0,"\n");
1513
1514            # Sources
1515            pb_log(0,"Delivered additional sources ");
1516            $pt = $h{'sources'};
1517            foreach $k (keys %$pt) {
1518                foreach my $v1 (split(/,/,$pt->{$k})) {
1519                    $tmp2{$v1} = "";
1520                }
1521            }
1522            if (keys %tmp2) {
1523                foreach $k (keys %tmp2) {
1524                    pb_log(0,"$k ");
1525                }
1526            } else {
1527                pb_log(0,"N/A");
1528            }
1529            pb_log(0,"\n");
1530
1531            # Build files
1532            my @found;
1533            my @notfound;
1534            $pt = $h{'build'};
1535            foreach my $b (keys %$pt) {
1536                push @found,$b if ($pt->{$b} =~ /yes/);
1537                push @notfound,$b if ($pt->{$b} =~ /no/);
1538            }
1539            pb_log(0,"Build files have been generated for ... ".join(',',sort(@found))."\n") if (@found);
1540            pb_log(0,"No Build files found for ".join(',',sort(@notfound))."\n") if (@notfound);
1541
1542        } else {
1543            # Instead call News generation
1544            pb_web_news2html($dest);
1545            # And create an empty pbconf
1546            pb_mkdir_p("$dest/pbconf");
1547            # And prepare the pbscript to execute remotely
1548            open(SCRIPT,"> $ENV{'PBTMP'}/pbscript") || die "Unable to create $ENV{'PBTMP'}/pbscript";
1549            print SCRIPT "#!/bin/bash\n";
1550            print SCRIPT "#set -x\n";
1551            print SCRIPT "echo ... Extracting Website content\n";
1552            print SCRIPT "find . -type f | grep -Ev '^./$pbpkg-$pbver$pbextdir.tar.gz|^./pbscript' | xargs rm -f non-existent\n";
1553            print SCRIPT "find * -type d -depth | xargs rmdir 2> /dev/null \n";
1554            print SCRIPT "tar xfz $pbpkg-$pbver$pbextdir.tar.gz\n";
1555            print SCRIPT "mv $pbpkg-$pbver$pbextdir/* .\n";
1556            print SCRIPT "rm -f $pbpkg-$pbver$pbextdir.tar.gz\n";
1557            print SCRIPT "rmdir $pbpkg-$pbver$pbextdir\n";
1558            print SCRIPT "find . -type f -print0 | xargs -0 chmod 644\n";
1559            print SCRIPT "find . -type d -print0 | xargs -0 chmod 755\n";
1560            close(SCRIPT);
1561            chmod 0755,"$ENV{'PBTMP'}/pbscript";
1562        }
1563
1564        # Apply filters to the non-build files
1565        my $liste ="";
1566        if (defined $filteredfiles->{$pbpkg}) {
1567            foreach my $f (split(/,/,$filteredfiles->{$pbpkg})) {
1568                pb_filter_file_inplace($ptr,"$dest/$f",$pb);
1569                $liste = "$f $liste";
1570            }
1571        }
1572        pb_log(2,"Files ".$liste."have been filtered\n");
1573
1574        # TODO: Make it CMS aware
1575        # Execute the pbinit script if any
1576        if (-x "$ENV{'PBROOTDIR'}/$pbpkg/pbinit") {
1577            pb_filter_file("$ENV{'PBROOTDIR'}/$pbpkg/pbinit",$ptr,"$ENV{'PBTMP'}/pbinit",$pb);
1578            chmod 0755,"$ENV{'PBTMP'}/pbinit";
1579            pb_system("cd $dest ; $ENV{'PBTMP'}/pbinit","Executing init script from $ENV{'PBROOTDIR'}/$pbpkg/pbinit under $dest","verbose");
1580        }
1581
1582        # Do we have additional script to run to prepare the environement for the project ?
1583        # Then include it in the pbconf delivery
1584        foreach my $pbvf (<$ENV{'PBROOTDIR'}/$pbpkg/pbv*.pre>,<$ENV{'PBROOTDIR'}/$pbpkg/pbv*.post>, <$ENV{'PBROOTDIR'}/$pbpkg/pbtest*>) {
1585            pb_log(1,"Handling file $pbvf\n");
1586            if (-x "$pbvf") {
1587                my $target = "$ENV{'PBTMP'}/".basename($pbvf);
1588                pb_filter_file("$pbvf",$ptr,$target,$pb);
1589                chmod 0755,"$target";
1590                pb_log(1,"Capturing file $pbvf in $target\n");
1591            }
1592        }
1593
1594        # Prepare the dest directory for archive
1595        chdir "$ENV{'PBDESTDIR'}" || die "Unable to change dir to $ENV{'PBDESTDIR'}";
1596        if (defined $preserve) {
1597            # In that case we want to preserve the original tar file for checksum purposes
1598            # The one created is btw equivalent in that case to this one
1599            # Maybe check basename of both to be sure they are the same ?
1600            pb_log(0,"Preserving original tar file ");
1601            move("$preserve","$pbpkg-$pbver$pbextdir.tar.gz");
1602        } else {
1603            # Possibility to look at PBSRC to guess more the filename
1604            pb_system("tar cfz $pbpkg-$pbver$pbextdir.tar.gz --exclude=$pbpkg-$pbver$pbextdir/pbconf $pbpkg-$pbver$pbextdir","Creating $pbpkg tar files compressed");
1605        }
1606        pb_log(0,"Under $ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.tar.gz\n");
1607        pb_system("tar cfz $pbpkg-$pbver$pbextdir.pbconf.tar.gz $pbpkg-$pbver$pbextdir/pbconf","Creating pbconf tar files compressed");
1608        pb_log(0,"Under $ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.pbconf.tar.gz\n");
1609
1610        # Keep track of version-tag per pkg
1611        $pkgs{$pbpkg} = "$pbver-$pbtag";
1612
1613        # Final cleanup
1614        pb_rm_rf($dest) if (-d $dest);
1615    }
1616
1617    # Keep track of per package version
1618    pb_log(2,"DEBUG pkgs: ".Dumper(%pkgs)."\n");
1619    open(PKG,"> $ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml") || die "Unable to create $ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml";
1620    print PKG "---\n";
1621    foreach my $pbpkg (keys %pkgs) {
1622        print PKG "pbpkg:\n";
1623        print PKG "  $pbpkg: $pkgs{$pbpkg}\n";
1624    }
1625    close(PKG);
1626
1627    # Keep track of what is generated by default
1628    # We need to store the dir and info on version-tag
1629    # Base our content on the existing .yml file
1630    copy("$ENV{'PBROOTDIR'}/$ENV{'PBPROJ'}.yml","$ENV{'PBDESTDIR'}/pbrc.yml");
1631    open(LAST,">> $ENV{'PBDESTDIR'}/pbrc.yml") || die "Unable to append to $ENV{'PBDESTDIR'}/pbrc.yml";
1632    print LAST "pbroot:\n";
1633    print LAST " $ENV{'PBPROJ'}: $ENV{'PBROOTDIR'}\n";
1634    print LAST "projver:\n";
1635    print LAST "  $ENV{'PBPROJ'}: $ENV{'PBPROJVER'}\n";
1636    print LAST "projtag:\n";
1637    print LAST "  $ENV{'PBPROJ'}: $ENV{'PBPROJTAG'}\n";
1638    print LAST "pbpackager:\n";
1639    print LAST "  $ENV{'PBPROJ'}: $ENV{'PBPACKAGER'}\n";
1640    print LAST "pbextdir:\n";
1641    print LAST "  $ENV{'PBPROJ'}: $pbextdir\n";
1642    close(LAST);
1643}
1644
1645sub pb_test2pkg {
1646    # Get the running distro to test on
1647    my $pbos = pb_distro_get_context();
1648
1649    # Get list of packages to test
1650    # Get content saved in cms2build
1651    my $ptr = pb_get_pkg();
1652    @pkgs = @$ptr;
1653
1654    # Additional potential repo
1655    my $forcerepo = pb_distro_setuprepo($pbos);
1656    foreach my $pbpkg (@pkgs) {
1657        # We need to install the package to test, and deps brought with it
1658        pb_distro_installdeps(undef,$pbos,$forcerepo,$pbpkg);
1659        pb_system("$ENV{'PBDESTDIR'}/pbtest","Launching test for $pbpkg","verbose");
1660    }
1661}
1662
1663sub pb_build2prep {
1664
1665    pb_log(0,"INFO: ------ Starting to prepare build environement ------\n");
1666
1667    # Get the running distro to build on
1668    my $pbos = pb_distro_get_context();
1669
1670    # If needed we may add repository to the build env
1671    my $forcerepo = pb_distro_setuprepo($pbos);
1672    pb_log(1,"Forcing full package install with forcerepo\n") if (defined $forcerepo);
1673
1674    # Get list of packages to build
1675    my $ptr = pb_get_pkg();
1676    @pkgs = @$ptr;
1677   
1678    # Get content saved in cms2build
1679    my ($pkg) = pb_conf_read("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml","pbpkg");
1680    $pkg = { } if (not defined $pkg);
1681    my $pbextdir = pb_get_extdir();
1682
1683    foreach my $pbpkg (@pkgs) {
1684        my $vertag = $pkg->{$pbpkg};
1685        my @buildfiles;
1686        pb_log(2,"Vertag: $vertag\n");
1687        # get the version of the current package - maybe different
1688        ($pbver,$pbtag) = split(/-/,$vertag);
1689
1690        my $src2="$ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.pbconf.tar.gz";
1691        pb_log(2,"Pbconf file: $src2\n");
1692        pb_log(2,"Working directory: $ENV{'PBBUILDDIR'}\n");
1693
1694        my $ftype;
1695        if ($pbos->{'type'} eq "rpm") {
1696            $ftype = "\.spec\$";
1697        } elsif ($pbos->{'type'} eq "deb") {
1698            $ftype = "\/control\$";
1699        } elsif ($pbos->{'type'} eq "ebuild") {
1700            $ftype = "\.ebuild\$";
1701        } elsif ($pbos->{'type'} eq "tgz") {
1702            $ftype = "\/pbslack\$";
1703        } elsif ($pbos->{'type'} eq "pkg") {
1704            # Solaris - no deps management up to Solaris 10
1705        } elsif ($pbos->{'type'} eq "hpux") {
1706            # TODO: HP-UX
1707        } else {
1708            die "Unknown OS type format $pbos->{'type'}";
1709        }
1710
1711        @buildfiles = pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/","$ENV{'PBTMP'}","build");
1712        pb_log(2,"buildfiles ".Dumper(\@buildfiles)."\n");
1713        foreach my $f (@buildfiles) {
1714            if ($f =~ /$ftype/) {
1715                pb_distro_installdeps($f,$pbos,$forcerepo);
1716            }
1717        }
1718    }
1719    pb_log(0,"INFO: ------ Finished preparing build environment ------\n");
1720}
1721
1722sub pb_build2pkg {
1723
1724    my $do_install = shift;
1725
1726        # Call build2prep in case we have new deps we want to take in account
1727        # without redoing the prep phase (can be long and that helps debugging
1728        # build dependencies issues
1729    pb_build2prep();
1730    pb_log(0,"INFO: ------ Starting to build package ------\n");
1731    # Get the running distro to build on
1732    my $pbos = pb_distro_get_context();
1733
1734    # Get list of packages to build
1735    my $ptr = pb_get_pkg();
1736    @pkgs = @$ptr;
1737
1738    # Get content saved in cms2build
1739    my ($pkg) = pb_conf_read("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml","pbpkg");
1740    $pkg = { } if (not defined $pkg);
1741    my $pbextdir = pb_get_extdir();
1742
1743    pb_mkdir_p("$ENV{'PBBUILDDIR'}") if (! -d "$ENV{'PBBUILDDIR'}");
1744    chdir "$ENV{'PBBUILDDIR'}" || die "Unable to chdir to $ENV{'PBBUILDDIR'}";
1745    my $made = ""; # pkgs made during build
1746    my $pm;
1747    my $all_ok = 1;
1748
1749    if (defined $pbparallel) {
1750        $pm = new Parallel::ForkManager($pbparallel);
1751        $pm->run_on_finish(sub { my ($pid, $code, $id, $signal, $dump) = @_;
1752                                $all_ok = 0 unless (($code == 0) && ($signal == 0) && ($dump == 0)); });
1753    }
1754
1755    # We need to communicate info back from the children if parallel so prepare a dir for that
1756    my $tmpd = "$ENV{'PBTMP'}/build.$$";
1757    pb_mkdir_p($tmpd) if (defined $pbparallel);
1758
1759    # Do it here as this doesn't work in // mode as it dies if it fails
1760    if ($pbos->{'type'} eq "rpm") {
1761        foreach my $d ('RPMS','SRPMS','SPECS','SOURCES','BUILD') {
1762                if (! -d "$ENV{'PBBUILDDIR'}/$d") {
1763                    pb_mkdir_p("$ENV{'PBBUILDDIR'}/$d");
1764                }
1765            }
1766    }
1767
1768    foreach my $pbpkg (@pkgs) {
1769        $pm->start and next if (defined $pbparallel);
1770
1771        my $vertag = $pkg->{$pbpkg};
1772        pb_log(2,"Vertag: $vertag\n");
1773        # get the version of the current package - maybe different
1774        ($pbver,$pbtag) = split(/-/,$vertag);
1775
1776        my $src="$ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.tar.gz";
1777        my $src2="$ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.pbconf.tar.gz";
1778        pb_log(2,"Source file: $src\n");
1779        pb_log(2,"Pbconf file: $src2\n");
1780
1781        pb_log(2,"Working directory: $ENV{'PBBUILDDIR'}\n");
1782        if ($pbos->{'type'} eq "rpm") {
1783            # Remove in case a previous link/file was there
1784            unlink "$ENV{'PBBUILDDIR'}/SOURCES/".basename($src);
1785            symlink "$src","$ENV{'PBBUILDDIR'}/SOURCES/".basename($src) || die "Unable to symlink $src in $ENV{'PBBUILDDIR'}/SOURCES";
1786            # We need to first extract the spec file
1787            my @buildfile = pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/","$ENV{'PBBUILDDIR'}/SPECS","build");
1788
1789            # We need to handle potential patches to upstream sources
1790            pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbpatch/","$ENV{'PBBUILDDIR'}/SOURCES","patch");
1791
1792            # We need to handle potential additional sources to upstream sources
1793            pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbsrc/","$ENV{'PBBUILDDIR'}/SOURCES","src");
1794
1795            pb_log(2,"buildfile: ".Dumper(\@buildfile)."\n");
1796            # set LANGUAGE to check for correct log messages
1797            $ENV{'LANGUAGE'}="C";
1798            # Older Redhat use _target_platform in %configure incorrectly
1799            my $specialdef = "";
1800            if (($pbos->{'name'} eq "rhel") && ($pbos->{'version'} eq "2.1")) {
1801                $specialdef = "--define \'_target_platform \"\"\'";
1802            }
1803            my $buildcmd = "rpmbuild";
1804            if (($pbos->{'name'} eq "redhat") && ($pbos->{'version'} =~ /^6/)){
1805                $buildcmd = "rpm" ;
1806            }
1807
1808            foreach my $f (@buildfile) {
1809                if ($f =~ /\.spec$/) {
1810                    pb_system("$buildcmd $specialdef --define \"packager $ENV{'PBPACKAGER'}\" --define \"_topdir $ENV{'PBBUILDDIR'}\" -ba $f","Building package with $f under $ENV{'PBBUILDDIR'}","verbose");
1811                    last;
1812                }
1813            }
1814            # Get the name of the generated packages
1815            open(LOG,"$ENV{'PBTMP'}/system.$$.log") || die "Unable to open $ENV{'PBTMP'}/system.$$.log";
1816            while (<LOG>) {
1817                chomp($_);
1818                next if ($_ !~ /^Wrote:/);
1819                s|.*/([S]*RPMS.*)|$1|;
1820                $made .=" $_";
1821            }
1822            close(LOG);
1823
1824        } elsif ($pbos->{'type'} eq "deb") {
1825            pb_system("tar xfz $src","Extracting sources");
1826            pb_system("tar xfz $src2","Extracting pbconf");
1827
1828            chdir "$pbpkg-$pbver$pbextdir" || die "Unable to chdir to $pbpkg-$pbver$pbextdir";
1829            pb_rm_rf("debian");
1830            my $confdir = "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
1831            die "Configuration directory $confdir is not a directory\nIs os description listed in your {ve,vm,rm}list values?" if (not -d $confdir);
1832            symlink "$confdir","debian" || die "Unable to symlink 'debian' to $confdir";
1833            chmod 0755,"debian/rules";
1834
1835            # We need to handle potential patches to upstream sources
1836            pb_mkdir_p("debian/patches");
1837            my @f = pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbpatch/","debian/patches","patch");
1838
1839            # By default we use format 1.0 - Cf man dpkg-source
1840            my $debsrcfmt = "1.0";
1841            my $debsrcfile = "debian/source/format";
1842            if (-f $debsrcfile) {
1843                $debsrcfmt = pb_get_content($debsrcfile);
1844            }
1845
1846            if ($debsrcfmt =~ /^3.*quilt/) {
1847                # If we use quilt to manage patches, we then setup the env correctly
1848                # as per http://pkg-perl.alioth.debian.org/howto/quilt.html
1849                # Generate Debian patch series for quilt
1850                open(SERIE,"> debian/patches/series") || die "Unable to write in debian/patches/series";
1851                $ENV{'QUILT_PATCHES'}="debian/patches";
1852            } else {
1853                # If we use dpatch to manage patches, we then setup the 00list file as well
1854                open(SERIE,"> debian/patches/00list") || die "Unable to write in debian/patches/00list";
1855            }
1856            foreach my $f (sort @f) {
1857                # Skip the script made to apply the patches to the Debian tree
1858                next if ($f =~ /pbapplypatch/);
1859                # We also need to uncompress them
1860                pb_system("gzip -d $f","","quiet");
1861                $f =~ s/\.gz$//;
1862                print SERIE "$f\n";
1863            }
1864            close(SERIE);
1865            if (@f) {
1866                # We have patches...
1867                my $patch_file = "debian/patches/pbapplypatch";
1868                if (($debsrcfmt =~ /^1.*/) && (-x $patch_file)) {
1869                    # In that case we need to apply the patches ourselves locally
1870                    pb_system("cat $patch_file","APPLY","verbose");
1871                    pb_system("$patch_file","Applying patches to $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'} tree");
1872                }
1873                # ...so modify the name of files to be Debian compliant
1874                move("../$src","../$pbpkg-$pbver$pbextdir.orig.tar.gz");
1875            }
1876
1877            # We need to handle potential additional sources to upstream sources
1878            #pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbsrc/","$ENV{'PBBUILDDIR'}/debian","src");
1879
1880            pb_system("dpkg-buildpackage -us -uc -rfakeroot","Building package","verbose");
1881
1882            # Get the name of the generated packages
1883            open(LOG,"$ENV{'PBTMP'}/system.$$.log") || die "Unable to open $ENV{'PBTMP'}/system.$$.log";
1884            while (<LOG>) {
1885                chomp();
1886                pb_log(3,"In loop: $_\n");
1887                next unless (/^dpkg-deb.*\.\.\/(.*_.*\.deb).*/);
1888                my $tmp = $1;
1889                #doesn't work in my case
1890                #die "Missing file $tmp" if (not -f "../$tmp");
1891                $made = "$made $tmp";
1892            }
1893            close(LOG);
1894            pb_log(2,"Now made is: $made\n");
1895
1896            open(CTRL,"debian/control") || die "Unable to open debian/control: $!";
1897            while (<CTRL>) {
1898                pb_log(3,"In loop: $_\n");
1899                next unless (/^Source: (\S+)/o);
1900                my $tmp = $1;
1901                $made = "$made $tmp"."_*.dsc $tmp"."_*.tar.gz $tmp"."_*.changes";
1902                #doesn't work in my case
1903                #foreach my $glob (("$1\_*.changes", "$1\_*.dsc", "$1\_*.tar.gz")) {
1904                    #my @file = glob($glob);
1905                    #die "Missing file for $glob" unless (@file > 0);
1906                    #die "Too many files for $glob" if (@file > 1);
1907                    #die "Missing file $file[0]" if (not -f $file[0]);
1908                    #$made .= " $file[0]";
1909                #}
1910            }
1911            close(CTRL);
1912            #pb_display_file("$ENV{'PBTMP'}/system.$$.log");
1913            pb_log(2,"Finally made is: $made\n");
1914
1915            chdir ".." || die "Unable to chdir to parent dir";
1916            pb_rm_rf("$pbpkg-$pbver$pbextdir");
1917        } elsif ($pbos->{'type'} eq "ebuild") {
1918            my @ebuildfile;
1919            # For gentoo we need to take pb as subsystem name
1920            # We put every apps here under sys-apps. hope it's correct
1921            # We use pb's home dir in order to have a single OVERLAY line
1922            my $tmpe = "$ENV{'HOME'}/portage/pb/sys-apps/$pbpkg";
1923            pb_mkdir_p($tmpe) if (! -d "$tmpe");
1924            pb_mkdir_p("$ENV{'HOME'}/portage/distfiles") if (! -d "$ENV{'HOME'}/portage/distfiles");
1925
1926            # We need to first extract the ebuild file
1927            @ebuildfile = pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/","$tmpe","build");
1928
1929            # Prepare the build env for gentoo
1930            my $found = 0;
1931            my $pbbd = $ENV{'HOME'};
1932            $pbbd =~ s|/|\\/|g;
1933            if (-r "/etc/make.conf") {
1934                open(MAKE,"/etc/make.conf");
1935                while (<MAKE>) {
1936                    $found = 1 if (/$pbbd\/portage/);
1937                }
1938                close(MAKE);
1939            }
1940            if ($found == 0) {
1941                pb_system("sudo sh -c 'echo PORTDIR_OVERLAY=\"$ENV{'HOME'}/portage\" >> /etc/make.conf'");
1942            }
1943            #$found = 0;
1944            #if (-r "/etc/portage/package.keywords") {
1945            #open(KEYW,"/etc/portage/package.keywords");
1946            #while (<KEYW>) {
1947            #$found = 1 if (/portage\/pb/);
1948            #}
1949            #close(KEYW);
1950            #}
1951            #if ($found == 0) {
1952            #pb_system("sudo sh -c \"echo portage/pb >> /etc/portage/package.keywords\"");
1953            #}
1954
1955            # Build
1956            foreach my $f (@ebuildfile) {
1957                if ($f =~ /\.ebuild$/) {
1958                    move($f,"$tmpe/$pbpkg-$pbver.ebuild");
1959                    pb_system("cd $tmpe ; ebuild $pbpkg-$pbver.ebuild clean ; ebuild $pbpkg-$pbver.ebuild digest ; ebuild $pbpkg-$pbver.ebuild package","verbose");
1960                    # Now move it where pb expects it
1961                    pb_mkdir_p("$ENV{'PBBUILDDIR'}/portage/pb/sys-apps/$pbpkg");
1962                    if ($pbtag eq 0) {
1963                        # This is assumed to be a test version
1964                        my $nver = substr($pbver,0,-14);
1965                        my $ntag = substr($pbver,-14);
1966                        my $ebtg = "portage/pb/sys-apps/$pbpkg/$pbpkg-$nver"."_p$ntag.ebuild";
1967                        move("$tmpe/$pbpkg-$pbver.ebuild","$ENV{'PBBUILDDIR'}/$ebtg");
1968                        $made="$made $ebtg";
1969                    } else {
1970                        my $ebtg = "portage/pb/sys-apps/$pbpkg/$pbpkg-$pbver-r$pbtag.ebuild";
1971                        move("$tmpe/$pbpkg-$pbver.ebuild","$ENV{'PBBUILDDIR'}/$ebtg");
1972                        $made="$made $ebtg";
1973                    }
1974                }
1975            }
1976
1977        } elsif ($pbos->{'type'} eq "tgz") {
1978            # Slackware family
1979            $made="$made $pbpkg/$pbpkg-$pbver-*-$pbtag.tgz";
1980
1981            pb_system("tar xfz $src","Extracting sources");
1982            pb_system("tar xfz $src2","Extracting pbconf");
1983            chdir "$pbpkg-$pbver$pbextdir" || die "Unable to chdir to $pbpkg-$pbver$pbextdir";
1984            symlink "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}","install" || die "Unable to symlink to pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
1985            if (-x "install/pbslack") {
1986                pb_system("./install/pbslack","Building software");
1987                pb_system("sudo /sbin/makepkg -p -l y -c y $pbpkg","Packaging $pbpkg","verbose");
1988            }
1989            chdir ".." || die "Unable to chdir to parent dir";
1990            pb_rm_rf("$pbpkg-$pbver$pbextdir");
1991        } elsif ($pbos->{'type'} eq "pkg") {
1992            # Solaris
1993            $made="$made $pbpkg-$pbver-$pbtag.pkg.gz";
1994            my $pkgdestdir="$ENV{'PBBUILDDIR'}/install";
1995
1996            # Will host resulting packages
1997            pb_mkdir_p("$pbos->{'type'}");
1998            pb_mkdir_p("$pkgdestdir/pbdelivery");
1999            pb_system("tar xfz $src","Extracting sources under $ENV{'PBBUILDDIR'}");
2000            pb_system("tar xfz $src2","Extracting pbconf under $ENV{'PBBUILDDIR'}");
2001            # We need to handle potential patches to upstream sources
2002            pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbpatch/","$ENV{'PBBUILDDIR'}","patch");
2003
2004            # We need to handle potential additional sources to upstream sources
2005            pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbsrc/","$ENV{'PBBUILDDIR'}","src");
2006
2007            chdir "$pbpkg-$pbver$pbextdir" || die "Unable to chdir to $pbpkg-$pbver$pbextdir";
2008            if (-f "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbbuild") {
2009                chmod 0755,"pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbbuild";
2010                # pkginfo file is mandatory
2011                die "Unable to find pkginfo file in pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}" if (! -f "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pkginfo");
2012                # Build
2013                pb_system("pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbbuild $pkgdestdir/pbdelivery","Building software and installing under $pkgdestdir/pbdelivery");
2014                # Copy complementary files
2015                if (-f "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/prototype") {
2016                    copy("pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/prototype", $pkgdestdir) 
2017                } else {
2018                    # No prototype provided, calculating it
2019                    open(PROTO,"> $pkgdestdir/prototype") || die "Unable to create prototype file";
2020                    print PROTO "i pkginfo\n";
2021                    print PROTO "i depend\n" if (-f "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/depend");
2022                    $ENV{'PBSOLDESTDIR'} = "$pkgdestdir/pbdelivery";
2023                    find(\&create_solaris_prototype, "$pkgdestdir/pbdelivery");
2024                }
2025                copy("pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/depend", $pkgdestdir) if (-f "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/depend");
2026                copy("pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pkginfo", $pkgdestdir);
2027                pb_system("cd $pkgdestdir/pbdelivery ; pkgmk -o -f ../prototype -r $pkgdestdir/pbdelivery -d $ENV{'PBBUILDDIR'}/$pbos->{'type'}","Packaging $pbpkg","verbose");
2028                pb_system("cd $ENV{'PBBUILDDIR'}/$pbos->{'type'} ;  echo \"\" | pkgtrans -o -n -s $ENV{'PBBUILDDIR'}/$pbos->{'type'} $ENV{'PBBUILDDIR'}/$pbpkg-$pbver-$pbtag.pkg all","Transforming $pbpkg","verbose");
2029                pb_system("cd $ENV{'PBBUILDDIR'} ;  gzip -9f $pbpkg-$pbver-$pbtag.pkg","Compressing $pbpkg-$pbver-$pbtag.pkg","verbose");
2030            } else {
2031                pb_log(0,"No pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbbuild file found for $pbpkg-$pbver\n");
2032            }
2033            chdir ".." || die "Unable to chdir to parent dir";
2034            pb_rm_rf("$pbpkg-$pbver$pbextdir","$ENV{'PBBUILDDIR'}/$pbos->{'type'}","$pkgdestdir");
2035        } elsif ($pbos->{'type'} eq "hpux") {
2036            # HP-UX
2037            pb_system("tar xfz $src","Extracting sources");
2038            pb_system("tar xfz $src2","Extracting pbconf");
2039
2040            chdir "$pbpkg-$pbver$pbextdir" || die "Unable to chdir to $pbpkg-$pbver$pbextdir";
2041            pb_system("buildpackage ","Building package","verbose");
2042            # Get the name of the generated packages
2043            open(LOG,"$ENV{'PBTMP'}/system.$$.log") || die "Unable to open $ENV{'PBTMP'}/system.$$.log";
2044            while (<LOG>) {
2045                chomp();
2046                my $tmp = $_;
2047                next if ($tmp !~ /^SD BUILD.*:/);
2048                $tmp =~ s|.*../(.*)_(.*).sd.*|$1|;
2049                $made = "$made $tmp"."_*.sd";
2050            }
2051            close(LOG);
2052            $made="$made $pbpkg-$pbver-$pbtag.sd";
2053
2054            chdir ".." || die "Unable to chdir to parent dir";
2055            pb_rm_rf("$pbpkg-$pbver$pbextdir");
2056        } else {
2057            die "Unknown OS type format $pbos->{'type'}";
2058        }
2059        if (defined $pbparallel) {
2060            # Communicate results back to parent
2061            pb_set_content("$tmpd/$$",$made);
2062            $pm->finish;
2063        }
2064    }
2065    if (defined $pbparallel) {
2066        # In the parent, we need to get the result from the children
2067        $pm->wait_all_children;
2068        foreach my $f (<$tmpd/*>) {
2069            $made .= " ".pb_get_content($f);
2070        }
2071        die "Aborting, one or more of the children failed." if ((not $all_ok) && ($Global::pb_stop_on_error));
2072        pb_rm_rf($tmpd);
2073    }
2074
2075    # Sign packages
2076    pb_sign_pkgs($pbos,$made);
2077    pb_log(0,"INFO: ------ Finished building package ------\n");
2078
2079    # Find the appropriate check cmd/opts
2080    my ($chkcmd,$chkopt) = pb_distro_get_param($pbos,pb_conf_get_if("oschkcmd","oschkopt"));
2081
2082    my $ret = "";
2083    # Packages check if needed
2084    if ($pbos->{'type'} eq "rpm") {
2085        if ((defined  $chkcmd) && (-x $chkcmd)) {
2086            my $cmd = "$chkcmd";
2087            $cmd .= " $chkopt" if (defined $chkopt);
2088            $cmd .= " $made";
2089            my $ret = pb_system("$cmd","Checking validity of rpms with $chkcmd","mayfailverbose");
2090            pb_log(0,"ERROR: when checking packages validity\n") if ($ret ne 0);
2091        }
2092        my $rpms ="";
2093        my $srpms ="";
2094        foreach my $f (split(/ /,$made)) {
2095            $rpms .= "$ENV{'PBBUILDDIR'}/$f " if ($f =~ /^RPMS\//);
2096            $srpms .= "$ENV{'PBBUILDDIR'}/$f " if ($f =~ /^SRPMS\//);
2097        }
2098        pb_log(0,"SRPM packages generated: $srpms\n");
2099        pb_log(0,"RPM packages generated: $rpms\n");
2100        $ret = $rpms;
2101    } elsif ($pbos->{'type'} eq "deb") {
2102        my $made2 = "";
2103        foreach my $f (split(/ /,$made)) {
2104            $made2 .= "$f " if ($f =~ /\.deb$/);
2105        }
2106        if ((defined  $chkcmd) && (-x $chkcmd)) {
2107            my $ret = pb_system("$chkcmd $chkopt $made2","Checking validity of debs with $chkcmd","mayfail");
2108            pb_log(0,"ERROR: when checking packages validity\n") if ($ret ne 0);
2109        }
2110        pb_log(0,"deb packages generated: $made2\n");
2111        $ret = $made2;
2112    } else {
2113        pb_log(0,"No check done for $pbos->{'type'} yet\n");
2114        pb_log(0,"Packages generated: $made\n");
2115        $ret = $made;
2116    }
2117
2118    # Keep track of what is generated so that we can get them back from VMs/RMs
2119    my $pbstore = "$ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
2120    open(KEEP,"> $pbstore") || die "Unable to create $pbstore $!";
2121    print KEEP "$made\n";
2122    close(KEEP);
2123    pb_distro_installdeps(undef,$pbos,undef,$ret) if ($do_install);
2124}
2125
2126sub pb_build2ssh {
2127    pb_send2target("Sources");
2128    pb_send2target("CPAN");
2129}
2130
2131sub pb_pkg2ssh {
2132    pb_send2target("Packages");
2133}
2134
2135# By default deliver to the the public site hosting the
2136# ftp structure (or whatever) or a VM/VE/RM
2137sub pb_send2target {
2138
2139    my $cmt = shift;
2140    my $pbscript1 = shift;
2141    my $v = shift;
2142    my $vexist = shift;         # 0 is FALSE
2143    my $vpid = shift;           # 0 is FALSE
2144    my $snapme = shift;         # 0 is FALSE
2145    my $pbstep = shift;         # 3 is usage of container
2146
2147    $vexist = 0 if (not defined $vexist);
2148    $vpid = 0 if (not defined $vpid);
2149    $snapme = 0 if (not defined $snapme);
2150    $pbstep = 3 if (not defined $pbstep);
2151    pb_log(2,"DEBUG: pb_send2target($cmt,".Dumper($v).",$vexist,$vpid)\n");
2152    my $host = "sshhost";
2153    my $login = "sshlogin";
2154    my $dir = "sshdir";
2155    my $port = "sshport";
2156    my $conf = "sshconf";
2157    my $tmout = undef;
2158    my $path = undef;
2159
2160    if ($cmt =~ /^VM/) {
2161        $login = "vmlogin";
2162        $dir = "pbdefdir";
2163        # Specific VM
2164        $tmout = "vmtmout";
2165        $path = "vmpath";
2166        $host = "vmhost";
2167        $port = "vmport";
2168    } elsif ($cmt =~ /^RM/) {
2169        $login = "rmlogin";
2170        $dir = "pbdefdir";
2171        # Specific RM
2172        $tmout = "rmtmout";
2173        $path = "rmpath";
2174        $host = "rmhost";
2175        $port = "rmport";
2176    } elsif ($cmt =~ /^VE/) {
2177        $login = "velogin";
2178        $dir = "pbdefdir";
2179        # Specific VE
2180        $path = "vepath";
2181        $conf = "rbsconf";
2182    } elsif ($cmt eq "Web") {
2183        $host = "websshhost";
2184        $login = "websshlogin";
2185        $dir = "websshdir";
2186        $port = "websshport";
2187    } elsif ($cmt eq "CPAN") {
2188        $host = "cpanpause";
2189        $login = "";
2190        $dir = "cpandir";
2191        $port = "";
2192    }
2193    my $cmd = "";
2194    my $src = "";
2195    my $cpanpkg = 0;
2196    my $pbos;
2197
2198    my $pbextdir = pb_get_extdir();
2199
2200    if ($cmt ne "Announce") {
2201        # Get list of packages to build
2202        my $ptr = pb_get_pkg();
2203        my @pkgs = ();
2204        @pkgs = @$ptr if (defined $ptr);
2205
2206        # Get the running distro to consider
2207        $pbos = pb_distro_get_context($v);
2208
2209        my $pkg = { };
2210        # Get content saved in cms2build
2211        if ((defined $ENV{'PBDESTDIR'}) && (defined $ENV{'PBPROJVER'}) && (defined $ENV{'PBPROJTAG'}) && (-f "$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml")) {
2212            ($pkg) = pb_conf_read("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml","pbpkg");
2213            $pkg = { } if (not defined $pkg);
2214        }
2215
2216        pb_mkdir_p("$ENV{'PBBUILDDIR'}") if (! -d "$ENV{'PBBUILDDIR'}");
2217        chdir "$ENV{'PBBUILDDIR'}" || die "Unable to chdir to $ENV{'PBBUILDDIR'}";
2218        foreach my $pbpkg (@pkgs) {
2219            my $vertag = $pkg->{$pbpkg};
2220            # get the version of the current package - maybe different
2221            pb_log(2,"Vertag: $vertag\n");
2222            ($pbver,$pbtag) = split(/-/,$vertag);
2223
2224            if (($cmt eq "Sources") || ($cmt =~ /(V[EM]|RM)(build|prep)/)) {
2225                $src = "$src $ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.tar.gz $ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.pbconf.tar.gz";
2226                my $cmd2 = "ln -sf $pbpkg-$pbver$pbextdir.tar.gz $pbpkg-latest.tar.gz";
2227                #my $cmd2 = "ln $pbpkg-$pbver$pbextdir.tar.gz $pbpkg-latest.tar.gz 2> /dev/null ; if [ \$? -ne 0 ]; then ln -sf $pbpkg-$pbver$pbextdir.tar.gz $pbpkg-latest.tar.gz ; fi";
2228                if ($cmd eq "") {
2229                    $cmd = $cmd2;
2230                } else {
2231                    $cmd = "$cmd ; $cmd2";
2232                }
2233            } elsif ($cmt eq "Web") {
2234                $src = "$src $ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.tar.gz"
2235            }
2236
2237            # Do we have one perl package
2238            my @nametype = pb_conf_get_if("namingtype");
2239            my $type = $nametype[0]->{$pbpkg};
2240            if ((defined $type) && ($type eq "perl")) {
2241                $cpanpkg = 1;
2242            }
2243        }
2244        # Adds conf file for availability of conf elements
2245        pb_conf_add("$ENV{'PBROOTDIR'}/$ENV{'PBPROJ'}.yml") if ((defined $ENV{'PBROOTDIR'}) && (-f "$ENV{'PBROOTDIR'}/$ENV{'PBPROJ'}.yml"));
2246    }
2247    my ($rbsconf,$testver,$delivery) = pb_conf_get_if($conf,"testver","delivery");
2248    if ($cmt eq "CPAN") {
2249        # Do not deliver on Pause if this is a test version
2250        return if (not defined $testver);
2251        return if ($testver =~ /true/);
2252        # Do not deliver on Pause if this is not a perl package
2253        return if ($cpanpkg == 0);
2254    }
2255
2256    if ($cmt =~ /(V[EM]|RM)(build|prep)/) {
2257        $src="$src $ENV{'PBROOTDIR'}/$ENV{'PBPROJ'}.yml $ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml $ENV{'PBETC'} $ENV{'PBDESTDIR'}/pbrc.yml $pbscript{$v}";
2258    } elsif ($cmt =~ /(V[EM]|RM)Script/) {
2259        $src="$src $pbscript{$v}";
2260    } elsif ($cmt =~ /(V[EM]|RM)test/) {
2261        $src="$src $ENV{'PBROOTDIR'}/$ENV{'PBPROJ'}.yml $ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml $ENV{'PBETC'} $ENV{'PBDESTDIR'}/pbrc.yml $pbscript{$v} $ENV{'PBDESTDIR'}/pbtest";
2262    } elsif (($cmt eq "Announce") || ($cmt eq "Web") || ($cmt eq "CPAN")) {
2263        $src="$src $ENV{'PBTMP'}/pbscript";
2264    } elsif ($cmt eq "Packages") {
2265        # Get package list from file made during build2pkg
2266        open(KEEP,"$ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}") || die "Unable to read $ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
2267        $src = <KEEP>;
2268        chomp($src);
2269        close(KEEP);
2270    }
2271    if (($cmt eq "Sources") || ($cmt eq "Packages") || ($cmt eq "CPAN")) {
2272        my ($pbpackager) = pb_conf_get("pbpackager");
2273        $ENV{'PBPACKAGER'} = $pbpackager->{$ENV{'PBPROJ'}};
2274        pb_log(0,"Exporting public key for $ENV{'PBPACKAGER'}\n");
2275        # Using pb_system is not working due to redirection below
2276        system("gpg --export -a \'$ENV{'PBPACKAGER'}\' > $ENV{'PBDESTDIR'}/$ENV{'PBPROJ'}.pubkey");
2277        chmod 0644,"$ENV{'PBDESTDIR'}/$ENV{'PBPROJ'}.pubkey";
2278        $src = "$src $ENV{'PBDESTDIR'}/$ENV{'PBPROJ'}.pubkey";
2279
2280        my ($gpgaddkey) = pb_conf_get_if("pbadditionalgpg");
2281        my $pbadditionalgpg;
2282        $pbadditionalgpg = $gpgaddkey->{$ENV{PBPROJ}} if (defined $gpgaddkey);
2283        if (defined $pbadditionalgpg) {
2284            foreach my $k (split(/,/,$pbadditionalgpg)) {
2285                pb_log(0,"Exporting public key $k\n");
2286                system("gpg --export -a \'$k\' > $ENV{'PBDESTDIR'}/$ENV{'PBPROJ'}-$k.pubkey");
2287                $src = "$src $ENV{'PBDESTDIR'}/$ENV{'PBPROJ'}-$k.pubkey";
2288            }
2289        }
2290    }
2291    # Remove potential leading spaces (cause problem with basename)
2292    $src =~ s/^ *//;
2293    my $basesrc = "";
2294    foreach my $i (split(/ +/,$src)) {
2295        $basesrc .= " ".basename($i);
2296    }
2297
2298    pb_log(2,"values: ".Dumper(($host,$login,$dir,$port,$tmout,$path,$conf))."\n");
2299    my ($sshhost,$sshdir) = pb_conf_get($host,$dir);
2300    # Not mandatory...
2301    $delivery->{$ENV{'PBPROJ'}} = "" if (not defined $delivery->{$ENV{'PBPROJ'}});
2302    my ($sshlogin,$sshport) = pb_conf_get_if($login,$port);
2303    $sshport->{$ENV{PBPROJ}} = 22 if (not defined $sshport->{$ENV{PBPROJ}});
2304    $sshlogin->{$ENV{PBPROJ}} = getpwuid($UID) if (not defined $sshlogin->{$ENV{PBPROJ}});
2305    my ($vtmout,$vepath);
2306    # ...Except those in virtual context
2307    if ($cmt =~ /^VE/) {
2308        ($vepath) = pb_conf_get($path);
2309    }
2310    if ($cmt =~ /^(V|R)M/) {
2311        $vtmout = pb_distro_get_param($pbos,pb_conf_get_if($tmout));
2312    }
2313    my $remhost = $sshhost->{$ENV{'PBPROJ'}};
2314    my $remdir = $sshdir->{$ENV{'PBPROJ'}};
2315    if ($cmt =~ /^V[EM]|RM/) {
2316        # In that case our real host is in the xxhost with the OS as key, not project as above
2317        $remhost = pb_distro_get_param($pbos,$sshhost);
2318    }
2319    pb_log(2,"ssh: ".Dumper(($remhost,$sshlogin,$remdir,$sshport,$vepath,$rbsconf))."\n");
2320    pb_log(2,"ssh: ".Dumper($vtmout)."\n") if (defined $vtmout);
2321
2322    my $mac;
2323    if ($cmt !~ /^VE/) {
2324        $mac = "$sshlogin->{$ENV{'PBPROJ'}}\@$remhost";
2325        # Overwrite account value if passed as parameter or forced
2326        $mac = "$ENV{'PBACCOUNT'}\@$remhost" if (defined $ENV{'PBACCOUNT'});
2327    } else {
2328        # VE
2329        $mac = $sshlogin->{$ENV{'PBPROJ'}};
2330        # Overwrite account value if passed as parameter (typically for setup2v)
2331        $mac = $ENV{'PBACCOUNT'} if (defined $ENV{'PBACCOUNT'});
2332    }
2333    pb_log(2, "DEBUG: mac: $mac\n");
2334
2335    my $tdir;
2336    my $bdir;
2337    if (($cmt eq "Sources") || ($cmt =~ /(V[EM]|RM)Script/)) {
2338        $tdir = "$remdir/$delivery->{$ENV{'PBPROJ'}}/src";
2339    } elsif ($cmt eq "CPAN") {
2340        $tdir = "$remdir";
2341    } elsif ($cmt =~ /(V[EM]|RM)(build|test|prep)/) {
2342        $tdir = $remdir."/$ENV{'PBPROJ'}/pbdelivery";
2343        $bdir = $remdir."/$ENV{'PBPROJ'}/pbbuild";
2344        # Remove a potential $ENV{'HOME'} as bdir should be relative to pb's home
2345        $bdir =~ s|\$ENV.+\}/||;
2346    } elsif ($cmt eq "Announce") {
2347        $tdir = "$remdir/$delivery->{$ENV{'PBPROJ'}}";
2348    } elsif ($cmt eq "Web") {
2349        $tdir = "$remdir/$delivery->{$ENV{'PBPROJ'}}";
2350    } elsif ($cmt eq "Packages") {
2351        if (($pbos->{'type'} eq "rpm") || ($pbos->{'type'} eq "pkg") || ($pbos->{'type'} eq "hpux") || ($pbos->{'type'} eq "tgz")) {
2352            # put packages under an arch subdir
2353            $tdir = "$remdir/$delivery->{$ENV{'PBPROJ'}}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}";
2354        } elsif (($pbos->{'type'} eq "deb") || ($pbos->{'type'} eq "ebuild")) {
2355            # No need for an arch subdir
2356            $tdir = "$remdir/$delivery->{$ENV{'PBPROJ'}}/$pbos->{'name'}/$pbos->{'version'}";
2357        } else {
2358            die "Please teach the dev team where to deliver ($pbos->{'type'} type of packages\n";
2359        }
2360
2361        my $repodir = $tdir;
2362        $repodir =~ s|^$remdir/||;
2363        my $repotag = "";
2364        $repotag = "-$delivery->{$ENV{'PBPROJ'}}" if ($delivery->{$ENV{'PBPROJ'}} ne "");
2365
2366        my ($pbrepo) = pb_conf_get("pbrepo");
2367
2368        # Script for Repository management
2369        $v = "$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}" if (not defined $v);
2370        $pbscript{$v} = "$ENV{'PBBUILDDIR'}/pbscript.$$";
2371       
2372        # That script should be part of the delivery
2373        $src = "$src $pbscript{$v}";
2374        open(PBS,"> $pbscript{$v}") || die "Unable to create $pbscript{$v}";
2375        if ($pbos->{'type'} eq "rpm") {
2376            my $pbsha = pb_distro_get_param($pbos,pb_conf_get("ossha"));
2377            my ($gpgcheck) = pb_conf_get_if("pbgpgcheck");
2378            my $pbgpgcheck;
2379            $pbgpgcheck = $gpgcheck->{$ENV{PBPROJ}} if (defined $gpgcheck);
2380            my $pbgpgkey = "$pbrepo->{$ENV{'PBPROJ'}}/$repodir/$ENV{'PBPROJ'}.pubkey";
2381            my ($gpgaddpkey) = pb_conf_get_if("pbadditionalgpg");
2382            my $pbadditionalgpg;
2383            $pbadditionalgpg = $gpgaddpkey->{$ENV{PBPROJ}} if (defined $gpgaddpkey);
2384            if (defined $pbadditionalgpg) {
2385                foreach my $k (split(/,/,$pbadditionalgpg)) {
2386                    $pbgpgkey .= " $pbrepo->{$ENV{'PBPROJ'}}/$repodir/$ENV{'PBPROJ'}-$k.pubkey";
2387                }
2388            }
2389            # By default force GPG check in repo even if we support signature of packages to fail. This is a best practice
2390            $pbgpgcheck = 1 if (not defined $pbgpgcheck);
2391            # Also make a pbscript to generate dnf/yum/urpmi bases
2392            print PBS << "EOF";
2393#!/bin/bash
2394# Prepare a script to ease dnf/yum setup
2395EOF
2396            print PBS "set -x\n" if ($pbdebug gt 1);
2397            print PBS << "EOF";
2398cat > $ENV{'PBPROJ'}$repotag.repo << EOT
2399[$ENV{'PBPROJ'}$repotag]
2400name=$pbos->{'name'} $pbos->{'version'} $pbos->{'arch'} - $ENV{'PBPROJ'} Vanilla Packages
2401baseurl=$pbrepo->{$ENV{'PBPROJ'}}/$repodir
2402enabled=1
2403gpgcheck=$pbgpgcheck
2404gpgkey=$pbgpgkey
2405EOT
2406chmod 644 $ENV{'PBPROJ'}$repotag.repo
2407
2408# Clean up old repo content
2409rm -rf headers/ repodata/
2410# Create yum repo
2411if [ -x /usr/bin/yum-arch ]; then
2412    yum-arch .
2413fi
2414# Create repodata
2415createrepo -s $pbsha .
2416# Link to the key
2417if [ -s $ENV{'PBPROJ'}.pubkey ]; then
2418    # Avoiding creating empty repomd.xml.key; fails on opensuse 12.1
2419    (cd repodata ; ln -sf ../$ENV{'PBPROJ'}.pubkey repomd.xml.key)
2420fi
2421# sign the repomd (at least useful for SLES - which requires a local key)
2422# gpg -a --detach-sign repodata/repomd.xml
2423# SLES also looks for media.1/info.txt
2424EOF
2425            if ($pbos->{'family'} eq "md") {
2426                # For Mandriva add urpmi management
2427                print PBS << "EOF";
2428# Prepare a script to ease urpmi setup
2429cat > $ENV{'PBPROJ'}$repotag.addmedia << EOT
2430#rpm --import $pbrepo->{$ENV{'PBPROJ'}}/$repodir/$ENV{'PBPROJ'}.pubkey
2431urpmi.addmedia $ENV{'PBPROJ'}$repotag $pbrepo->{$ENV{'PBPROJ'}}/$repodir with media_info/hdlist.cz
2432EOT
2433chmod 755 $ENV{'PBPROJ'}$repotag.addmedia
2434
2435# Clean up old repo content
2436rm -f hdlist.cz synthesis.hdlist.cz
2437# Create urpmi repo
2438genhdlist2 --clean .
2439if [ \$\? -ne 0 ]; then
2440    genhdlist .
2441fi
2442(cd media_info ; ln -sf ../$ENV{'PBPROJ'}.pubkey pubkey)
2443EOF
2444            }
2445            if ($pbos->{'name'} eq "fedora") {
2446                # Extract the spec file to please Fedora maintainers :-(
2447                print PBS << "EOF";
2448for p in $basesrc; do
2449    echo \$p | grep -q 'src.rpm'
2450    if [ \$\? -eq 0 ]; then
2451        rpm2cpio \$p | cpio -ivdum --quiet '*.spec'
2452    fi
2453done
2454EOF
2455            }
2456            if ($pbos->{'family'} eq "novell") {
2457                # Add ymp scripts for one-click install on SuSE
2458                print PBS << "EOF";
2459# Prepare a script to ease SuSE one-click install
2460# Cf: http://de.opensuse.org/1-Klick-Installation/ISV
2461#
2462cat > $ENV{'PBPROJ'}$repotag.ymp << EOT
2463<?xml version="1.0" encoding="utf-8"?>
2464<!-- vim: set sw=2 ts=2 ai et: -->
2465<metapackage xmlns:os="http://opensuse.org/Standards/One_Click_Install" xmlns="http://opensuse.org/Standards/One_Click_Install">
2466    <group><!-- The group of software, typically one for project-builder.org -->
2467        <name>$ENV{'PBPROJ'} Bundle</name> <!-- Name of the software group -->
2468        <summary>Software bundle for the $ENV{'PBPROJ'}$repotag project</summary> <!--This message is shown to the user and should describe the whole bundle -->
2469        <description>This is the summary of the $ENV{'PBPROJ'}$repotag Project
2470             
2471            Details are available on a per package basis below
2472
2473        </description><!--This is also shown to the user -->
2474        <remainSubscribed>false</remainSubscribed> <!-- Don't know what it mean -->
2475        <repositories><!-- List of needed repositories -->
2476            <repository>
2477                <name>$ENV{'PBPROJ'}$repotag Repository</name> <!-- Name of the repository  -->
2478                <summary>This repository contains the $ENV{'PBPROJ'}$repotag project packages.</summary> <!-- Summary of the repository -->
2479                <description>This repository contains the $ENV{'PBPROJ'}$repotag project packages.</description><!-- This description is shown to the user -->
2480                <url>$pbrepo->{$ENV{'PBPROJ'}}/$repodir</url><!--URL of repository, which is added -->
2481            </repository>
2482        </repositories>
2483        <software><!-- A List of packages, which should be added through the one-click-installation -->
2484EOT
2485for p in $basesrc; do
2486    sum=`rpm -q --qf '%{SUMMARY}' \$p`
2487    name=`rpm -q --qf '%{NAME}' \$p`
2488    desc=`rpm -q --qf '%{description}' \$p`
2489    cat >> $ENV{'PBPROJ'}$repotag.ymp << EOT
2490            <item>
2491                <name>\$name</name><!-- Name of the package, is shown to the user and used to identify the package at the repository -->
2492                <summary>\$sum</summary> <!-- Summary of the package -->
2493                <description>\$desc</description> <!-- Description, is shown to the user -->
2494            </item>
2495EOT
2496done
2497cat >> $ENV{'PBPROJ'}$repotag.ymp << EOT
2498        </software>
2499    </group>
2500</metapackage>
2501EOT
2502chmod 644 $ENV{'PBPROJ'}$repotag.ymp
2503EOF
2504            }
2505        } elsif ($pbos->{'type'} eq "deb") {
2506            # Also make a pbscript to generate apt bases
2507            # Cf: http://www.debian.org/doc/manuals/repository-howto/repository-howto.fr.html
2508            # This dirname removes ver
2509            my $debarch = pb_get_debarch($pbos);
2510            my $projcomponent = pb_get_debpc($pbos);
2511            my $rpd = dirname("$pbrepo->{$ENV{'PBPROJ'}}/$repodir");
2512            # Remove extra . in path to fix #522
2513            $rpd =~ s|/./|/|g;
2514            print PBS << "EOF";
2515#!/bin/bash
2516# Prepare a script to ease apt setup
2517cat > $ENV{'PBPROJ'}$repotag.sources.list << EOT
2518deb $rpd $pbos->{'version'} $projcomponent
2519deb-src $rpd $pbos->{'version'} $projcomponent
2520EOT
2521chmod 644 $ENV{'PBPROJ'}$repotag.sources.list
2522
2523# Up two levels to deal with the dist dir cross versions
2524cd ..
2525mkdir -p dists/$pbos->{'version'}/$projcomponent/binary-$debarch dists/$pbos->{'version'}/$projcomponent/source dists/$pbos->{'version'}/$projcomponent/binary-all
2526
2527# Prepare a script to create apt info file
2528# Reuse twice after
2529TMPD=`mktemp -d /tmp/pb.XXXXXXXXXX` || exit 1
2530mkdir -p \$TMPD
2531cat > \$TMPD/Release << EOT
2532Archive: unstable
2533Component: $projcomponent
2534Origin: $ENV{'PBPROJ'}$repotag
2535Label: $ENV{'PBPROJ'}$repotag dev repository $pbrepo->{$ENV{'PBPROJ'}}
2536EOT
2537
2538echo "Creating Packages metadata"
2539for i in dists/$pbos->{version}/$projcomponent/binary-*; do
2540    arch=`basename \$i | sed 's/binary-//'`
2541    echo "Packages for \$arch:"
2542    dpkg-scanpackages -a\$arch $pbos->{'version'} /dev/null > dists/$pbos->{'version'}/$projcomponent/binary-\$arch/Packages
2543    gzip -9 < dists/$pbos->{'version'}/$projcomponent/binary-\$arch/Packages > dists/$pbos->{'version'}/$projcomponent/binary-\$arch/Packages.gz
2544    bzip2 -9 < dists/$pbos->{'version'}/$projcomponent/binary-\$arch/Packages > dists/$pbos->{'version'}/$projcomponent/binary-\$arch/Packages.bz2
2545done
2546echo "Creating Contents metadata"
2547apt-ftparchive contents $pbos->{'version'} | gzip -c9 > dists/$pbos->{'version'}/Contents.gz
2548echo "Creating Release metadata ($pbos->{'arch'} aka $debarch)"
2549cat \$TMPD/Release > dists/$pbos->{'version'}/$projcomponent/binary-all/Release
2550echo "Architecture: all" >> dists/$pbos->{'version'}/$projcomponent/binary-all/Release
2551cat \$TMPD/Release > dists/$pbos->{'version'}/$projcomponent/binary-$debarch/Release
2552echo "Architecture: $debarch" >> dists/$pbos->{'version'}/$projcomponent/binary-$debarch/Release
2553echo "Creating Source metadata"
2554dpkg-scansources $pbos->{'version'} /dev/null > dists/$pbos->{'version'}/$projcomponent/source/Sources
2555gzip -9 < dists/$pbos->{'version'}/$projcomponent/source/Sources > dists/$pbos->{'version'}/$projcomponent/source/Sources.gz
2556bzip2 -9 < dists/$pbos->{'version'}/$projcomponent/source/Sources > dists/$pbos->{'version'}/$projcomponent/source/Sources.bz2
2557cat \$TMPD/Release > dists/$pbos->{'version'}/$projcomponent/source/Release
2558echo "Architecture: Source" >> dists/$pbos->{'version'}/$projcomponent/source/Release
2559echo "Creating Release metadata"
2560apt-ftparchive -o APT::FTPArchive::Release::Suite=$pbos->{version} release dists/$pbos->{'version'} > dists/$pbos->{'version'}/Release
2561rm -rf \$TMPD
2562EOF
2563        } elsif ($pbos->{'type'} eq "ebuild") {
2564            # make a pbscript to generate links to latest version
2565            print PBS << "EOF";
2566#!/bin/bash
2567# Prepare a script to create correct links
2568cd $tdir
2569for p in $src; do
2570    echo \$p | grep -q '.ebuild'
2571    if [ \$\? -eq 0 ]; then
2572        j=`basename \$p`
2573        pp=`echo \$j | perl -pe 's/-[0-9\.]+-r[0-9].*\.ebuild/.ebuild/'`
2574        ln -sf \$j \$pp
2575    fi
2576done
2577EOF
2578        }
2579        close(PBS);
2580        chmod 0755,$pbscript{$v};
2581    } else {
2582        return;
2583    }
2584
2585    # Useless for VE
2586    my $nport = pb_get_port($sshport,$pbos,$cmt) if ($cmt !~ /^VE/);
2587
2588    # Remove a potential $ENV{'HOME'} as tdir should be relative to pb's home
2589    if ($cmt =~ /^VE/o) {
2590        $tdir =~ s|\$ENV.+\}/|| or confess "for $cmt: $tdir must have \$ENV{'HOME'} in it or the remainder of the code won't work since some parts use relative pathing and others will try absolute.  Use a symlink if necessary to place the real files in a different place than under your home directory";
2591    } else {
2592        # process, ok if not present.
2593        $tdir =~ s|\$ENV.+\}/||;
2594    }
2595
2596    my $tm = "";
2597    if ($cmt =~ /^(V|R)M/) {
2598        $tm = "sleep $vtmout" if (defined $vtmout);
2599    }
2600
2601    # ssh communication if not VE or CPAN
2602    # should use a hash instead...
2603    my ($shcmd,$shcmdroot,$cpcmd,$cptarget,$cp2target);
2604    my $tpdir;
2605    my $tp;
2606    my $context = "$ENV{'PBTMP'}";
2607    my %tag;
2608    my $dkaccount = "";
2609    my $cmd1 = "";
2610    my $cidfile;
2611    my $homedir = "";
2612    if ($cmt =~ /^VE/) {
2613        $tp = pb_path_expand($vepath->{$ENV{'PBPROJ'}});
2614        $vetype = pb_ve_get_type($vetype);
2615        if ($vetype eq "docker") {
2616            $tpdir = "/";
2617            $cidfile = "$context/ctn-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}.cid";
2618        } else {
2619            $tpdir = pb_path_expand("$tp/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}");
2620        }
2621        my $arch = pb_get_arch();
2622        if ($vetype eq "chroot") {
2623            $shcmdroot = "sudo /usr/sbin/chroot $tpdir ";
2624            $shcmd = "$shcmdroot /bin/su - $mac -c ";
2625        } elsif ($vetype eq "schroot") {
2626            $shcmd = "schroot $tp -u $mac -- ";
2627        } elsif ($vetype eq "docker") {
2628            # docker manages the storage so rely on it
2629            $shcmdroot = "";
2630            my $docrepo = pb_ve_docker_repo();
2631            $cmd1 = pb_check_req("docker",0);
2632            #my ($dockeropt) =  pb_conf_get_if("dockeropt");
2633            # pbimage is used for docker and is setup to the right name depending on the step:
2634            # TODO: This is not coded yet
2635            my $pbimage = undef;
2636            # step 0 : nothing at creation -> tag n-v-a (made in VE.pm)
2637            # step 1 : n-v-a + setup -> tag n-v-a-pb
2638            # step 2 : n-v-a-pb + build -> tag n-v-a-pb-pbproj
2639            # step 3 : n-v-a-pb-pbproj at use
2640            if ((not defined $pbimage) || ($pbimage eq "")) {
2641                # If no image name given, create a naming convention
2642                $tag{1} = "$docrepo$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
2643                $tag{2} = "$tag{1}-pb";
2644                $tag{3} = "$tag{2}-$ENV{'PBPROJ'}";
2645            } else {
2646                # If we were given an image name, just use it
2647                $tag{1} = $pbimage;
2648                $tag{2} = $pbimage;
2649                $tag{3} = $pbimage;
2650            }
2651            # If we do not create the image, then use the one we should have
2652            # TODO: we don't do anything with found !!
2653            my $found = pb_ve_docker_get_image($tag{$pbstep});
2654            confess "No image $tag{$pbstep} available in Docker, please create one first\n" if (not defined $found);
2655            # Now we use that image to do what is needed
2656            # use a dockerfile to ease the creation of next images
2657            # the context is in an empty temp dir
2658            my $tmpd = "$context/Dockerfile";
2659            open(DOCKER, "> $tmpd") || die "Unable to create the docker file $tmpd";
2660            print DOCKER "FROM $tag{$pbstep}\n";
2661            print DOCKER "MAINTAINER project-builder.org aka pb\n";
2662            if ($pbstep == 1) {
2663                # setup done as root
2664                $dkaccount = "root";
2665            } else {
2666                # Rest done as pb
2667                $dkaccount = "pb";
2668            }
2669            print DOCKER "USER $dkaccount\n";
2670            if ($pbstep <= 2) {
2671                $shcmd = "$cmd1 build -t $tag{$pbstep+1} $context";
2672            } else {
2673                # As we are in run phase use docker run. cmd will be completed below
2674                $shcmd = "$cmd1 run --cidfile=\"$cidfile\" --user $dkaccount --name $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-$cmt";
2675            }
2676            #$shcmd = "$cmd1 build $dockeropt->{$ENV{'PBPROJ'}} -t $tag{$pbstep+1} $context";
2677        }
2678        $shcmd = "setarch i386 $shcmd" if (($pbos->{'arch'} =~ /i[3456]86/) && ($arch eq 'x86_64'));
2679        if (($cmt =~ /VE/) && ($vetype ne "docker")) {
2680            # We need to get the home dir of the target account to deliver in the right place
2681            open(PASS,"$tpdir/etc/passwd") || die "Unable to open $tpdir/etc/passwd: $!";
2682            while (<PASS>) {
2683                my ($c1,$c2,$c3,$c4,$c5,$c6,$c7) = split(/:/);
2684                $homedir = $c6 if ($c1 =~ /^$mac$/);
2685                pb_log(3,"Homedir: $homedir - account: $c6\n");
2686            }
2687            close(PASS);
2688            $cpcmd = "sudo /bin/cp -r ";
2689        }
2690        if (($cmt =~ /VE/) && ($vetype eq "docker")) {
2691            if ($pbstep >= 2) {
2692                # We need to get the home dir of the target account to deliver in the right place
2693                $homedir = `$cmd1 run --cidfile="$cidfile" $tag{$pbstep} grep -E '^$dkaccount:' /etc/passwd | cut -d: -f6`;
2694                chomp($homedir);
2695                open(CID,"$cidfile") || confess "Unable to open $cidfile";
2696                my $cid = <CID>;
2697                close(CID);
2698                pb_system("$cmd1 rm $cid","","quiet");
2699                unlink("$cidfile");
2700            } else {
2701                $homedir = "/";
2702            }
2703            $cpcmd = "cp -r ";
2704        }
2705        $cptarget = "$tpdir/$homedir/$tdir";
2706        if ($cmt =~ /VE(build|prep)/) {
2707            $cp2target = "$tpdir/$homedir/$bdir";
2708        }
2709        pb_log(2,"On VE using $cptarget as target dir to copy to\n");
2710    } elsif ($cmt eq "CPAN") {
2711        my $ftpput = pb_check_req("ncftpput",1);
2712        my $ftpget = pb_check_req("wget",1);
2713        my ($cpanuser,$cpanpasswd) = pb_conf_get_if("cpanuser","cpanpasswd");
2714        return if ((not defined $cpanuser) || (not defined $cpanuser->{$ENV{'PBPROJ'}}));
2715        return if ((not defined $cpanpasswd) || (not defined $cpanpasswd->{$ENV{'PBPROJ'}}));
2716        my ($cpansubdir) = pb_conf_get_if("cpansubdir");
2717        $shcmd = "$ftpget --post-data \'HIDDENNAME=".$cpanuser->{$ENV{'PBPROJ'}};
2718        $shcmd .= "&user=".$cpanuser->{$ENV{'PBPROJ'}};
2719        $shcmd .= "&password=".$cpanpasswd->{$ENV{'PBPROJ'}};
2720        $shcmd .= "&SUBMIT_pause99_add_uri_upload=\"Upload the checked files\"";
2721        $shcmd .= "&pause99_add_uri_subdirtext=".$cpansubdir->{$ENV{'PBPROJ'}} if ((defined $cpansubdir) && (defined $cpansubdir->{$ENV{'PBPROJ'}}));
2722        foreach my $s (split(/ /,$src)) {
2723            $shcmd .= "&pause99_add_uri_upload=".basename($s);
2724        }
2725        $shcmd .= "'";
2726        $cpcmd = "$ftpput $host $dir";
2727        $cptarget = "CPAN";
2728    } else {
2729        my $keyopt = "";
2730        if ($cmt !~ /Packages/) {
2731            my $keyfile = pb_ssh_get(0,$remhost,$nport);
2732            $keyopt = "-i $keyfile" if (defined $keyfile);
2733        }
2734        my $sshcmd = pb_check_req("ssh",1);
2735        my $scpcmd = pb_check_req("scp",1);
2736        $shcmd = "$sshcmd $keyopt -q -o NoHostAuthenticationForLocalhost=yes -p $nport $mac";
2737        $cpcmd = "$scpcmd $keyopt -p -o NoHostAuthenticationForLocalhost=yes -P $nport";
2738        $cptarget = "$mac:$tdir";
2739        if ($cmt =~ /^(V|R)M(build|prep)/) {
2740            $cp2target = "$mac:$bdir";
2741        }
2742    }
2743   
2744    my $logres = "";
2745    # Do not touch when just announcing
2746    if (($cmt ne "Announce") && ($cmt ne "CPAN")) {
2747        # For docker everything is done in the Dockerfile for steps < 3
2748        if (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep < 3)) {
2749            print DOCKER "RUN mkdir -p $cptarget\n";
2750            print DOCKER "RUN cd $cptarget ; for i in $basesrc; do if [ -f \$i ]; then rm -f \$i; fi; done\n";
2751        } elsif (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep >= 3)) {
2752            # Nothing to do here
2753        } else {
2754            pb_system("$shcmd \"mkdir -p $tdir ; cd $tdir ; echo \'for i in $basesrc; do if [ -f \$i ]; then rm -f \$i; fi; done ; $cmd\' | bash -e\"","Preparing $tdir on $cptarget");
2755        }
2756    } else {
2757        $logres = "> ";
2758    }
2759
2760    if ($cmt =~ /SandBox/) {
2761        # Install from sandbox mean using the result of the just passed sbx2build command
2762        # Get content saved in cms2build
2763        my ($pkg) = pb_conf_read("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml","pbpkg");
2764        my $pbextdir = pb_get_extdir();
2765        confess "Unable to get package list" if (not defined $pkg);
2766
2767        # We consider 2 specific packages
2768        my $vertag1 = $pkg->{"ProjectBuilder"};
2769        my $vertag2 = $pkg->{"project-builder"};
2770        # get the version of the current package - maybe different
2771        pb_log(2,"Vertag1: $vertag1\n");
2772        pb_log(2,"Vertag2: $vertag2\n");
2773        my ($pbver1,$tmp1) = split(/-/,$vertag1);
2774        my ($pbver2,$tmp2) = split(/-/,$vertag2);
2775        if ($cmt =~ /VE/) {
2776            # Copy inside the VE
2777            if ($vetype ne "docker") {
2778                copy("$ENV{'PBDESTDIR'}/ProjectBuilder-$pbver1$pbextdir.tar.gz","$tpdir/tmp");
2779                copy("$ENV{'PBDESTDIR'}/project-builder-$pbver2$pbextdir.tar.gz","$tpdir/tmp");
2780            } else {
2781                copy("$ENV{'PBDESTDIR'}/ProjectBuilder-$pbver1$pbextdir.tar.gz","$context");
2782                copy("$ENV{'PBDESTDIR'}/project-builder-$pbver2$pbextdir.tar.gz","$context");
2783                print DOCKER "COPY ProjectBuilder-$pbver1$pbextdir.tar.gz /tmp/\n";
2784                print DOCKER "COPY project-builder-$pbver2$pbextdir.tar.gz /tmp/\n";
2785            }
2786        } else {
2787            pb_system("$cpcmd $ENV{'PBDESTDIR'}/ProjectBuilder-$pbver1$pbextdir.tar.gz $ENV{'PBDESTDIR'}/project-builder-$pbver2$pbextdir.tar.gz $mac:/tmp","Copying local project files to VM/RM");
2788        }
2789    }
2790           
2791    pb_log(0,"Sources handled ($cmt): $src\n");
2792    # For docker everything is done in the Dockerfile
2793    if (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep < 3)) {
2794        foreach my $f (split(/ +/,$src)) {
2795            copy("$f","$context");
2796            print DOCKER "COPY ".basename($f)." $cptarget/\n";
2797        }
2798        print DOCKER "RUN cd $cptarget ; $cmd\n" if ((defined $cmd) && ($cmd ne ""));
2799    } elsif (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep >= 3)) {
2800        # Nothing to do
2801    } else {
2802        pb_system("cd $ENV{'PBBUILDDIR'} ; $cpcmd $src $cptarget 2> /dev/null","$cmt delivery in $cptarget");
2803    }
2804
2805    # For VE we need to change the owner manually
2806    if ($cmt =~ /^VE/) {
2807        if ($vetype ne "docker") {
2808            my $sudomode = pb_distro_get_param($pbos,pb_conf_get("ossudoersmode"));
2809            my $res = pb_system("$shcmdroot sed -i '/requiretty/d' /etc/sudoers","Removing potential requiretty in sudoers","quiet");
2810            pb_system("$shcmdroot sed '/requiretty/d' /etc/sudoers > /tmp/sudoers.new ; mv /tmp/sudoers.new $tpdir/tmp/sudoers.new ; $shcmdroot mv /tmp/sudoers.new /etc/sudoers ; $shcmdroot chown root:root /etc/sudoers ; $shcmdroot chmod $sudomode /etc/sudoers","Removing again potential requiretty in sudoers as sed -i failed") if (($res ne 0) && (-f "$tpdir/etc/sudoers"));
2811            pb_system("$shcmd \"sudo chown -R $mac $tdir\"","Adapt owner in $tdir to $mac");
2812        } else {
2813            print DOCKER "RUN sed -i '/requiretty/d' /etc/sudoers\n" if ($pbstep == 1);
2814        }
2815    }
2816
2817    # Use the right script name depending on context
2818    my $pbscript2;
2819    if (($cmt =~ /^(V[EM]|RM)/) || ($cmt =~ /Packages/)){
2820        $pbscript2 = $pbscript{$v};
2821    } else {
2822        $pbscript2 = "pbscript";
2823    }
2824
2825    # It's already ready for CPAN
2826    my $shcmdbase = $shcmd;
2827    my $dkcmd;
2828    if ($cmt ne "CPAN") {
2829        if (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep < 3)) {
2830            copy("$pbscript2","$context");
2831            my $s = basename($pbscript2);
2832            print DOCKER "COPY $s $cptarget/\n";
2833            # call directly with perl to avoid text file busy error messages
2834            print DOCKER "RUN cd $cptarget ; perl ./$s ; if [ '$pbkeep' = '0' ]; then rm -f $s; fi\n";
2835            print DOCKER "RUN rm -rf $cptarget/*\n";
2836            print DOCKER "RUN chown -R $dkaccount $cptarget\n" if ($pbstep == 1);
2837            close(DOCKER);
2838        } elsif (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep >= 3)) {
2839            $dkcmd = "/bin/bash $homedir/$tdir-2/".basename($pbscript2);
2840        } else {
2841            my $s = basename($pbscript2);
2842            $shcmd .= " \"echo \'cd $tdir ; if [ -x $s ]; then ./$s; fi ;  if [ $pbkeep == 0 ]; then rm -f $s; fi\' | bash\"";
2843        }
2844    }
2845    my $cmdverb = "verbose";
2846    if (($cmt eq "Announce") || ($cmt eq "CPAN")) {
2847        $cmdverb = undef;
2848    } elsif (defined ($v)) {
2849        $cmdverb = "verbose_\[$v\] ";
2850    }
2851    # this is where we launch the execution
2852    if (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep >= 3)) {
2853        pb_mkdir_p("$ENV{'PBTMP'}/vebuild");
2854        my $tsrc = "";
2855        foreach my $f (split(/ +/,$src)) {
2856            copy("$f","$ENV{'PBTMP'}/vebuild");
2857            $tsrc .= " ".basename($f);
2858        }
2859        my $ts = "$ENV{'PBTMP'}/vebuild/".basename($pbscript2);
2860        open(DOCKER," > $ts") || confess "Unable to create $ts";
2861        print DOCKER "#!/bin/bash\n";
2862        print DOCKER "# Put content from host to guest, copying to have the correct uid/gid and rights to do so\n";
2863        print DOCKER "# set -x\n";
2864        print DOCKER "for i in $tsrc; do\n";
2865        print DOCKER "  # Cleanup first to avoid rights issues for root owned files e.g.\n";
2866        print DOCKER "  rm -f $homedir/$tdir/\$i\n";
2867        print DOCKER "  cp $homedir/$tdir-2/\$i $homedir/$tdir\n";
2868        print DOCKER "done\n";
2869        print DOCKER "(cd $homedir/$tdir-2 ; ls -al)\n";
2870        print DOCKER pb_get_content($pbscript2);
2871        print DOCKER "\n# Get content back from guest in a host dir\n";
2872        # Host dir seen as TDIR in container
2873        $cp2target = "$ENV{'PBBUILDDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
2874        pb_mkdir_p("$cp2target");
2875        chmod 0777,"$cp2target";
2876        print DOCKER "TDIR=$homedir/$bdir-2/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}\n";
2877        print DOCKER "for i in $homedir/$bdir/*; do\n";
2878        print DOCKER "  # Cleanup first to avoid rights issues for root owned files e.g.\n";
2879        print DOCKER "  j=\`basename \$i\`\n";
2880        print DOCKER "  rm -f \$TDIR/\$j\n";
2881        print DOCKER "  cp -r \$i \$TDIR\n";
2882        print DOCKER "done\n";
2883        print DOCKER "find \$TDIR -type f -print0 | xargs -0 chmod 644\n";
2884        print DOCKER "find \$TDIR -type d -print0 | xargs -0 chmod 777 2> /dev/null\n";
2885        #print DOCKER "ls -al \$TDIR\n";
2886        print DOCKER "exit 0\n";
2887        close(DOCKER);
2888        $shcmd .= " -v $ENV{'PBBUILDDIR'}:$homedir/$bdir-2 -v $ENV{'PBTMP'}/vebuild:$homedir/$tdir-2 -w $homedir/$tdir-2 $tag{$pbstep} $dkcmd";
2889    }
2890    my $ret = pb_system("$shcmd","Executing pbscript on $cptarget if needed",$cmdverb);
2891   
2892    # If target is deb family then sign Release file for Packages
2893    my $pbos2 = pb_distro_get_context($v);
2894    if (($cmt =~ /Packages/) && ($pbos2->{'type'} eq "deb")) {
2895        my $debarch = pb_get_debarch($pbos2);
2896        my $projcomponent = pb_get_debpc($pbos2);
2897        for my $f ("dists/$pbos2->{'version'}/$projcomponent/binary-$debarch/Release","dists/$pbos2->{'version'}/$projcomponent/binary-all/Release","dists/$pbos2->{'version'}/$projcomponent/source/Release","dists/$pbos2->{'version'}/Release") {
2898            my $debdir = dirname($cptarget);
2899            pb_system("cd $ENV{'PBBUILDDIR'} ; $cpcmd $debdir/$f . 2> /dev/null","Getting deb $f file from $debdir");
2900            pb_sign_file("$ENV{'PBBUILDDIR'}/Release");
2901            pb_system("$cpcmd $ENV{'PBBUILDDIR'}/Release.gpg $debdir/$f.gpg 2> /dev/null","Putting deb $f.gpg file to $debdir");
2902            pb_system("$shcmdbase \"chmod 644 $debdir/$f.gpg\"","Set mode to 644 for $debdir");
2903        }
2904    } 
2905
2906    if ($cmt =~ /^(V[EM]|RM)build/) {
2907        # Get back info on pkg produced, compute their name and get them from the VM/RM
2908        pb_system("$cpcmd $cp2target/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'} $ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.$$ 2> /dev/null","Get package names in $cp2target");
2909        if (not -f "$ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.$$") {
2910            pb_log(0,"ERROR with VM/RM $v on getting $ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.$$\n");
2911        } else {
2912            open(KEEP,"$ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.$$") || die "Unable to read $ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.$$";
2913            my $src = <KEEP>;
2914            chomp($src);
2915            close(KEEP);
2916            unlink("$ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.$$");
2917
2918            $src =~ s/^ *//;
2919            pb_mkdir_p("$ENV{'PBBUILDDIR'}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}");
2920            # Change pgben to make the next send2target happy
2921            my $made = "";
2922   
2923            # For VM/RM we don't want shell expansion to hapen locally but remotely
2924            my $delim = '\'';
2925            if ($cmt =~ /^VEbuild/) {
2926                # For VE we need to support shell expansion locally
2927                $delim = "";
2928            }   
2929
2930            open(KEEP,"> $ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}") || die "Unable to write $ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
2931            foreach my $p (split(/ +/,$src)) {
2932                my $j = basename($p);
2933                pb_system("$cpcmd $cp2target/$delim$p$delim $ENV{'PBBUILDDIR'}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'} 2> /dev/null","Recovery of package $j in $ENV{'PBBUILDDIR'}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}");
2934                $made="$made $pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}/$j"; # if (($pbos->{'type'} ne "rpm") || ($j !~ /.src.rpm$/));
2935            }
2936       
2937            print KEEP "$made\n";
2938            close(KEEP);
2939            if (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep >= 3)) {
2940                pb_rm_rf("$cp2target");
2941                pb_system("$cmd1 rm $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-$cmt","","quiet");
2942            } else {
2943                pb_system("$shcmdbase \"rm -rf $tdir $bdir\"","$cmt cleanup");
2944            }
2945
2946            # Sign packages locally
2947            pb_sign_pkgs($pbos,$made);
2948
2949            # We want to send them to the ssh account so overwrite what has been done before
2950            pb_log(2,"Before sending pkgs, vexist: $vexist, vpid: $vpid\n");
2951            pb_send2target("Packages","$ENV{'PBDESTDIR'}/pbscript.$$",$pbos->{'name'}."-".$pbos->{'version'}."-".$pbos->{'arch'},$vexist,$vpid);
2952            pb_rm_rf("$ENV{'PBBUILDDIR'}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}");
2953        }
2954    }
2955    unlink("$pbscript{$v}") if ((($cmt =~ /^(V[ME]|RM)/) || ($cmt =~ /Packages/)) && ($pbkeep eq 0) && ($cmt !~ /Script/));
2956
2957    pb_log(2,"Before halt, vexist: $vexist, vpid: $vpid\n");
2958    if ((! $vexist) && ($cmt =~ /^VM/)) {
2959        # If in setupvm then takes a snapshot just before halting
2960        if ($snapme != 0) {
2961            my ($vmmonport,$vmtype) = pb_conf_get("vmmonport","vmtype");
2962            # For monitoring control
2963            if ((($vmtype->{$ENV{'PBPROJ'}}) eq "kvm") || (($vmtype->{$ENV{'PBPROJ'}}) eq "qemu")) {
2964                eval
2965                {
2966                require Net::Telnet;
2967                Net::Telnet->import();
2968                };
2969                if ($@) {
2970                    # Net::Telnet not found
2971                    pb_log(1,"ADVISE: Install Net::Telnet to benefit from monitoring control and snapshot feature.\nWARNING: No snapshot created");
2972                } else {
2973                    my $t = new Net::Telnet (Timeout => 120, Host => "localhost", Port => $vmmonport->{$ENV{'PBPROJ'}}) || die "Unable to dialog on the monitor";
2974                    # move to monitor mode
2975                    my @lines = $t->cmd("c");
2976                    # Create a snapshot named pb
2977                    @lines = $t->cmd("savevm pb");
2978                    # Write the new status in the VM
2979                    @lines = $t->cmd("commit all");
2980                    # End
2981                    @lines = $t->cmd("quit");
2982                }
2983            }
2984        }
2985        my $hoption = "-p";
2986        my $hpath = pb_distro_get_param($pbos,pb_conf_get("ospathcmd-halt"));
2987        # Solaris doesn't support -p of halt
2988        if ($pbos->{'type'} eq "pkg") {
2989            $hoption = "" ;
2990        }
2991        pb_system("$shcmdbase \"sudo $hpath $hoption \"; $tm ; echo \'if [ -d /proc/$vpid ]; then kill -9 $vpid; fi \' | bash ; sleep 10","VM $v halt (pid $vpid)");
2992    }
2993    if (($cmt =~ /^VE/) && ($snapme != 0)) {
2994        if ($vetype eq "docker") {
2995            foreach my $f (split(/ +/,$src)) {
2996                unlink("$context/$f");
2997            }
2998        } else {
2999            pb_ve_snap($pbos,$tp);
3000        }
3001    }
3002}
3003
3004sub pb_script2v {
3005
3006my $pbscript=shift;
3007my $vtype=shift;
3008my $pbstep=shift;   # Which step are we in (0: create, 1: setup pb, 2: prep pbproj, 3: use to build)
3009my $pbforce=shift;  # Force stop of VM. Default not.
3010my $snapme=shift;   # Do we have to create a snapshot. Default not.
3011my $pbimage=shift;  # Which image to use to start the VM/VE
3012
3013$pbforce = 0 if (not defined $pbforce);
3014$snapme = 0 if (not defined $snapme);
3015$pbstep = 3 if (not defined $pbstep);
3016
3017pb_log(2,"DEBUG: pb_script2v($vtype,$pbstep,$pbforce,$snapme)\n");
3018pb_log(2,"DEBUG: pb_script2v(pbscript: ".Dumper($pbscript)."\n") if (defined $pbscript);
3019pb_parallel_launchv($pbscript,$vtype,uc($vtype)."Script",$pbstep,$pbimage,$pbforce,$snapme,$pbsnap);
3020}
3021
3022sub pb_parallel_launchv {
3023
3024my $pbscript=shift;
3025my $vtype = shift;
3026my $action = shift;         # It an action is defined then use send2target
3027my $pbstep=shift;           # Which step are we in (0: create, 1: setup, 2: build 3: use)
3028my $pbimage=shift;          # Which image to use to start the VM/VE
3029my $pbforce=shift;          # Force stop of VM. Default not.
3030my $snapme = shift;         # By default do not snap a VM/VE/RM
3031my $usesnap = shift;        # By default study the usage of the snapshot feature of VM/VE/RM   
3032my $vm;
3033my $all;
3034
3035$pbforce = 0 if (not defined $pbforce);
3036$snapme = 0 if (not defined $snapme);
3037$usesnap = 1 if (not defined $usesnap);
3038
3039# Adapt // mode to memory size
3040$pbparallel = pb_set_parallel($vtype);
3041
3042pb_log(2,"DEBUG: pb_parallel_launch(vtype,step,force,snapme,usesnap: $vtype,$pbstep,$pbforce,$snapme,$usesnap)\n");
3043pb_log(2,"DEBUG: pb_parallel_launch(pbscript: ".Dumper($pbscript)."\n");
3044
3045my ($vexist,$vpid) = (undef,undef);
3046my $pm;
3047my $all_ok = 1;
3048if (defined $pbparallel) {
3049    $pm = new Parallel::ForkManager($pbparallel);
3050
3051    # Set which port the VM/RM will use to communicate
3052    $pm->run_on_start(\&pb_set_port);
3053    $pm->run_on_finish(sub { my ($pid, $code, $id, $signal, $dump) = @_;
3054                            unless ($code == 0 && $signal == 0 && $dump == 0) {
3055                                $all_ok = 0;
3056                                pb_log(0,"ERROR: pid $pid failed\n");
3057                            }
3058                        });
3059}
3060
3061# Determine on which V to work
3062if (defined $ENV{'PBV'}) {
3063    @$vm = split(/,/,$ENV{'PBV'});
3064} else {
3065    ($vm,$all) = pb_get2v($vtype);
3066}
3067
3068my $counter = 0;
3069foreach my $v (@$vm) {
3070    $counter++;
3071    # Modulo 2 * pbparallel (to avoid synchronization problems)
3072    $counter = 1 if ((defined $pbparallel) && ($counter > 2 * $pbparallel));
3073    $pm->start($counter) and next if (defined $pbparallel);
3074   
3075    # If launched via script2v then get the script passed as the one for the that VM
3076    if (not defined $pbscript->{$v}) {
3077        $pbscript->{$v} = $pbscript->{'default'};
3078    }
3079    #
3080    # Launch a single operation
3081    ($vexist,$vpid) = pb_launchv($pbscript->{$v},$vtype,$v,$action,$pbstep,$pbforce,$snapme,$pbsnap,$pbimage);
3082   
3083    # Skip that VM/RM if something went wrong
3084    if (($vtype =~ /(v|r)m/) && ($vpid == 0) && ($vexist == 0)) {
3085        $pm->finish if (defined $pbparallel);
3086        next;
3087    }
3088
3089   
3090    $pm->finish if (defined $pbparallel);
3091}
3092$pm->wait_all_children if (defined $pbparallel);
3093die "Aborting, one or more of the children failed." if ((not $all_ok) && ($Global::pb_stop_on_error));
3094}
3095
3096sub pb_launchv {
3097
3098my $pbscript1=shift;        # Just the script name to execute, not the hash
3099my $vtype = shift;
3100my $v = shift;              # Only 1 VM/VE/RM treated here
3101my $action = shift;         # It an action is defined then use send2target
3102my $pbstep=shift;           # Which step are we in (0: create, 1: setup, 2: build 3: use)
3103my $pbforce=shift;          # Force stop of VM. Default not.
3104my $snapme = shift;         # By default do not snap a VM/VE/RM
3105my $usesnap = shift;            # By default study the usage of the snapshot feature of VM/VE/RM   
3106my $pbimage=shift;          # Which image to use to start the VM/VE
3107my $vexist = undef;
3108my $vpid = undef;
3109
3110$pbforce = 0 if (not defined $pbforce);
3111$snapme = 0 if (not defined $snapme);
3112$usesnap = 1 if (not defined $usesnap);
3113
3114# Check that v exists
3115die "No VM/VE/RM defined, unable to launch" if (not defined $v);
3116
3117# If creation or snapshot creation mode, no snapshot usable
3118if (($pbstep == 0) || ($snapme == 1)) {
3119    $usesnap = 0;
3120}
3121
3122pb_log(2,"DEBUG: pb_launchv(vtype,v,step,force,snapme,usesnap: $vtype,$v,$pbstep,$pbforce,$snapme,$usesnap)\n");
3123pb_log(2,"DEBUG: pb_launchv(pbimage: $pbimage)\n") if (defined $pbimage);
3124pb_log(2,"DEBUG: pb_launchv(pbscript $pbscript1)\n") if (defined $pbscript1);
3125pb_log(2,"DEBUG: pb_launchv(action: $action)\n") if (defined $action);
3126# Keep only the first VM in case many were given
3127if ($v =~ /,/) {
3128    pb_log(0,"WARNING: pruning to just the first of several VM/VE/RMs listed ($v)\n");
3129    $v =~ s/,.*//;
3130}
3131
3132my $pbos = pb_distro_get_context($v);
3133
3134pb_apply_conf_proxy($pbos);
3135
3136# Launch the VMs/VEs
3137if ($vtype eq "vm") {
3138    confess "-i image parameter needed" if (((not defined $pbimage) || ($pbimage eq "")) && ($pbstep == 0));
3139
3140    my ($ptr,$ptr2,$vmpath,$vmport,$vms) = pb_conf_get("vmtype","vmcmd","vmpath","vmport","vmsize");
3141    my ($vmopt,$vmmm,$vmtmout,$vmsnap,$vmbuildtm,$vmmonport) = pb_conf_get_if("vmopt","vmmem","vmtmout","vmsnap","vmbuildtm","vmmonport");
3142    my $vmsize = pb_distro_get_param($pbos,$vms);
3143
3144    my $vmtype = $ptr->{$ENV{'PBPROJ'}};
3145    my $vmcmd = $ptr2->{$ENV{'PBPROJ'}};
3146
3147    if (defined $opts{'g'}) {
3148        if (($vmtype eq "kvm") || ($vmtype eq "qemu")) {
3149            $ENV{'PBVMOPT'} = "--nographic";
3150        }
3151    }
3152    if (not defined $ENV{'PBVMOPT'}) {
3153        $ENV{'PBVMOPT'} = "";
3154    }
3155    # Save the current status for later restoration
3156    $ENV{'PBOLDVMOPT'} = $ENV{'PBVMOPT'};
3157    # Set a default timeout of 2 minutes
3158    if (not defined $ENV{'PBVMTMOUT'}) {
3159        $ENV{'PBVMTMOUT'} = "120";
3160    }
3161    if (defined $vmopt->{$v}) {
3162        $ENV{'PBVMOPT'} .= " $vmopt->{$v}" if ($ENV{'PBVMOPT'} !~ / $vmopt->{$v}/);
3163    } elsif (defined $vmopt->{$ENV{'PBPROJ'}}) {
3164        $ENV{'PBVMOPT'} .= " $vmopt->{$ENV{'PBPROJ'}}" if ($ENV{'PBVMOPT'} !~ / $vmopt->{$ENV{'PBPROJ'}}/);
3165    }
3166
3167    # How much memory to allocate for VMs
3168    if (defined $vmmm) {
3169        my $vmmem = pb_distro_get_param($pbos,$vmmm);
3170        if (defined $vmmem) {
3171            $ENV{'PBVMOPT'} .= " -m $vmmem";
3172        }
3173    }
3174
3175    # Are we allowed to use snapshot feature
3176    if ($usesnap == 1) {
3177        if ((defined $vmsnap->{$v}) && ($vmsnap->{$v} =~ /true/i)) {
3178            $ENV{'PBVMOPT'} .= " -snapshot";
3179        } elsif ((defined $vmsnap->{$ENV{'PBPROJ'}}) && ($vmsnap->{$ENV{'PBPROJ'}} =~ /true/i)) {
3180            $ENV{'PBVMOPT'} .= " -snapshot";
3181        } elsif ($pbsnap eq 1) {
3182            $ENV{'PBVMOPT'} .= " -snapshot";
3183        }
3184    } 
3185    if ($snapme != 0) {
3186        if (($vmtype eq "kvm") || ($vmtype eq "qemu")) {
3187            # Configure the monitoring to automate the creation of the 'pb' snapshot
3188            $ENV{'PBVMOPT'} .= " -serial mon:telnet::$vmmonport->{$ENV{'PBPROJ'}},server,nowait" if ((defined $vmmonport) && (defined $vmmonport->{$ENV{'PBPROJ'}}));
3189            # In that case no snapshot call needed
3190            $ENV{'PBVMOPT'} =~ s/ -snapshot//;
3191        }
3192    }
3193    if (defined $vmtmout->{$v}) {
3194        $ENV{'PBVMTMOUT'} = $vmtmout->{$v};
3195    } elsif (defined $vmtmout->{$ENV{'PBPROJ'}}) {
3196        $ENV{'PBVMTMOUT'} = $vmtmout->{$ENV{'PBPROJ'}};
3197    }
3198    my $nport = pb_get_port($vmport,$pbos,$vtype);
3199
3200    my $cmd;
3201    my $vmm;        # has to be used for pb_check_ps
3202    if (($vmtype eq "qemu") || ($vmtype eq "kvm")) {
3203        $vmm = "$vmpath->{$ENV{'PBPROJ'}}/$v.qemu";
3204        if (($pbstep == 0) || (defined $pbimage)) {
3205            $ENV{'PBVMOPT'} .= " -cdrom $pbimage -boot d";
3206        }
3207        # Always redirect the network and always try to use a 'pb' snapshot
3208        #$cmd = "$vmcmd $ENV{'PBVMOPT'} -net nic,model=virtio -net user,hostfwd=tcp::$nport:22 -loadvm pb $vmm"
3209        #$cmd = "$vmcmd $ENV{'PBVMOPT'} -redir tcp:$nport:10.0.2.15:22 $vmm"
3210        $cmd = "$vmcmd $ENV{'PBVMOPT'} -net nic -net user,hostfwd=tcp:127.0.0.1:$nport-:22 $vmm"
3211    } elsif ($vmtype eq "xen") {
3212    } elsif ($vmtype eq "vmware") {
3213    } else {
3214        die "VM of type $vmtype not supported. Report to the dev team";
3215    }
3216    # Restore the ENV VAR Value
3217    $ENV{'PBVMOPT'} = $ENV{'PBOLDVMOPT'};
3218
3219    my ($tmpcmd,$void) = split(/ +/,$cmd);
3220    my $vmmport = undef;
3221    ($vexist,$vmmport) = pb_check_ps($tmpcmd,$vmm);
3222    $vpid = 0;
3223   
3224    if (! $vexist) {
3225        if ($pbstep == 0) {
3226            die("Found an existing Virtual machine $vmm. Won't overwrite") if (-r $vmm);
3227            if (($vmtype eq "qemu") || ($vmtype eq "xen") || ($vmtype eq "kvm")) {
3228                my $command = pb_check_req("qemu-img",0);
3229                pb_system("$command create -f qcow2 $vmm $vmsize","Creating the QEMU VM");
3230            } elsif ($vmtype eq "vmware") {
3231            } else {
3232            }
3233        }
3234        if (! -f "$vmm") {
3235            pb_log(0,"Unable to find VM $vmm\n");
3236        } else {
3237            # Is the SSH port free? if not kill the existing process using it after a build timeout period
3238            my ($vmssh,$void) = pb_check_ps($tmpcmd,$vmm);
3239            if ($vmssh) {
3240                my $buildtm = $ENV{'PBVMTMOUT'};
3241                if (defined $vmbuildtm->{$v}) {
3242                    $buildtm = $vmbuildtm->{$v};
3243                } elsif (defined $vmbuildtm->{$ENV{'PBPROJ'}}) {
3244                    $buildtm = $vmbuildtm->{$ENV{'PBPROJ'}};
3245                }
3246
3247                sleep $buildtm;
3248                pb_log(0,"WARNING: Killing the process ($vmssh) using port $void (previous failed VM ?)\n");
3249                kill 15,$vmssh;
3250                # Let it time to exit
3251                sleep 5;
3252            }
3253            pb_system("$cmd &","Launching the VM $vmm");
3254            # Using system allows to kill it externaly if needed,sosupport that in the call
3255            pb_system("sleep $ENV{'PBVMTMOUT'}","Waiting $ENV{'PBVMTMOUT'} s for VM $v to come up","mayfail");
3256            ($vpid,$void) = pb_check_ps($tmpcmd,$vmm);
3257            pb_log(0,"VM $vmm launched (pid $vpid)\n");
3258        }
3259    } else {
3260        pb_log(0,"Found an existing VM $vmm (pid $vexist - Port $vmmport)\n");
3261        # Set the correct port here based on what is done
3262        $pbport = $vmmport;
3263        $vpid = $vexist;
3264    }
3265    pb_log(2,"DEBUG: pb_launchv returns ($vexist,$vpid)\n");
3266
3267    # After the VM is launched, then setup the SSH access if not in creation phase
3268    if ($pbstep != 0) {
3269        if (defined $pbport) {
3270            pb_ssh_setup($vtype,$pbport);
3271        } else {
3272            pb_ssh_setup($vtype,$nport);
3273        }
3274    }
3275   
3276    # Skip that VM if something went wrong
3277    return($vexist,$vpid) if (($vpid == 0) && ($vexist == 0));
3278   
3279    # If force stopping the VM then reset vexist
3280    if ($pbforce == 1) {
3281        $vpid = $vexist;
3282        $vexist = 0;
3283    }
3284} elsif ($vtype eq "ve") {
3285    # Force the creation of the VE and no snapshot usable
3286    $vetype = pb_ve_get_type($vetype);
3287    pb_ve_launch($v,$pbstep,$usesnap,$vetype,$pbimage);
3288    $vexist = 0;
3289    $vpid = 0;
3290} else {
3291    # Get RM context
3292    my ($ptr,$rmpath,$rmport) = pb_conf_get("rmtype","rmpath","rmport");
3293
3294    # Nothing more to do for RM. No real launch
3295    # For the moment we support the RM is already running
3296    # For ProLiant may be able to power them on if needed later on as an example.
3297    $vexist = 0;
3298    $vpid = 0;
3299   
3300    # After the RM is launched, then setup the SSH access
3301    my $nport = pb_get_port($rmport,$pbos,$vtype);
3302    pb_ssh_setup($vtype,$nport);
3303}
3304
3305# Gather all required files to send them to the VM/VE/RM
3306# and launch the build through pbscript
3307if (defined $action) {
3308    pb_log(2,"DEBUG: Before send2target, vexist: $vexist, vpid: $vpid, vtype: $vtype, v: $v, action: $action\n");
3309    pb_send2target($action,$pbscript1,$v,$vexist,$vpid,$snapme,$pbstep);
3310}
3311return($vexist,$vpid);
3312}
3313
3314# Return string for date synchro
3315sub pb_date2v {
3316
3317my $vtype = shift;
3318my $pbos = shift;
3319my $nontp = shift || 0;
3320
3321pb_log(1,"Entering pb_date2v\n");
3322# VE gets time from parent OS.
3323return "/bin/true" if ($vtype) =~ /^ve/o; 
3324
3325my ($ntp) = pb_conf_get_if($vtype."ntp");
3326my $vntp = undef;
3327my $ntpline = undef;
3328$vntp = $ntp->{$ENV{'PBPROJ'}} if (defined $ntp);
3329
3330if ((defined $vntp) && ($vntp !~ /^\s*$/)) {
3331    pb_log(2,"ntp server is $vntp\n");
3332    # ntp command depends on pbos
3333    my $vntpcmd = pb_distro_get_param($pbos,pb_conf_get("oscmdntp"));
3334    $ntpline = "sudo $vntpcmd $vntp";
3335}
3336# Force new date to be in the future compared to the date
3337# of the host by adding 1 minute
3338my @date=pb_get_date();
3339$date[1]++;
3340my $upddate = strftime("%m%d%H%M%Y", @date);
3341my $dateline = "sudo /bin/date $upddate";
3342if ((defined $ntpline) && ($nontp == 0)) {
3343    pb_log(1,"pb_date2v returns $ntpline\n");
3344    return($ntpline);
3345} else {
3346    pb_log(1,"pb_date2v returns $dateline\n");
3347    return($dateline);
3348}
3349}
3350
3351sub pb_build2v {
3352
3353my $vtype = shift;
3354my $action = shift;     #build, test or prep
3355my $pbforce=shift;      # Force stop of VM. Default not.
3356
3357$action = "build" if (not defined $action);
3358$pbforce = 0 if (not defined $pbforce);
3359
3360my $pbstep;
3361
3362$vetype = pb_ve_get_type($vetype);
3363if ($action eq "prep") {
3364    $pbstep = 2;
3365} else {
3366    $pbstep = 3;
3367    # Need to be traversed by others if using containers
3368    if (($vtype eq "ve") && ($vetype eq "docker")) {
3369        chmod 0751,"$ENV{'PBBUILDDIR'}";
3370    }
3371}
3372
3373pb_log(1,"Entering in pb_build2v ($action in $vtype) \n");
3374
3375# We prepare scripts for all V of this vtype to allow sharing that part
3376my ($v,$all) = pb_get2v($vtype);
3377
3378# Send tar files when we do a global generation
3379pb_build2ssh() if (($all == 1) && ($action eq "build"));
3380
3381# Adapt // mode to memory size
3382$pbparallel = pb_set_parallel($vtype);
3383
3384my ($vexist,$vpid) = (undef,undef);
3385my $pm;
3386my $all_ok = 1;
3387if (defined $pbparallel) {
3388    $pm = new Parallel::ForkManager($pbparallel);
3389
3390    # Set which port the VM/RM will use to communicate
3391    $pm->run_on_start(\&pb_set_port);
3392    $pm->run_on_finish(sub { my ($pid, $code, $id, $signal, $dump) = @_;
3393                            unless ($code == 0 && $signal == 0 && $dump == 0) {
3394                                $all_ok = 0;
3395                                pb_log(0,"ERROR: pid $pid failed\n");
3396                            }
3397                        });
3398}
3399
3400my $counter = 0;
3401foreach my $v (@$v) {
3402    $counter++;
3403    $pbscript{$v} = "$ENV{'PBDESTDIR'}/pb$action-$v";
3404    # Modulo 2 * pbparallel (to avoid synchronization problems)
3405    $counter = 1 if ((defined $pbparallel) && ($counter > 2 * $pbparallel));
3406    $pm->start($counter) and next if (defined $pbparallel);
3407    #
3408    # Prepare the script to be executed on the VM/VE/RM
3409    open(SCRIPT,"> $pbscript{$v}") || die "Unable to create $pbscript{$v}";
3410    print SCRIPT "#!/bin/bash\n";
3411
3412    # Transmit the verbosity level to the virtual env/mach.
3413    my $verbose = "";
3414    my $i = 0;                          # minimal debug level
3415    while ($i lt $pbdebug) {
3416        $verbose .= "-v ";
3417        $i++;
3418    }
3419    print SCRIPT "set -e\n" if ($Global::pb_stop_on_error);
3420    # Activate script verbosity if at least 2 for pbdebug
3421    print SCRIPT "set -x\n" if ($i gt 1);
3422    # Quiet if asked to be so on the original system
3423    $verbose = "-q" if ($pbdebug eq -1);
3424
3425    print SCRIPT "echo ... Execution needed\n";
3426    print SCRIPT "echo ==== Start of script for $vtype $v =====\n";
3427    print SCRIPT "# This is in directory pbdelivery\n";
3428    print SCRIPT "# Setup the variables required for building\n";
3429    print SCRIPT "export PBPROJ=$ENV{'PBPROJ'}\n";
3430
3431    if (($action eq "build") || ($action eq "prep")) {
3432        print SCRIPT "# Preparation for pb\n";
3433        print SCRIPT "rm -f \$HOME/.pbrc.yml\n";
3434        print SCRIPT "rm -f \$HOME/.pbrc\n";
3435        print SCRIPT "cp .pbrc.yml \$HOME\n";
3436        # TODO: Why ?
3437        print SCRIPT "cd ..\n";
3438    }
3439
3440    # VE needs a good /proc, tolerate one being potentially left around after a failure
3441    if (($vtype eq "ve") && ($vetype ne "docker")) {
3442        print SCRIPT "[ -d /proc/1 ] || sudo /bin/mount -t proc /proc /proc\n";
3443    }
3444
3445    # Get distro context
3446    my $pbos = pb_distro_get_context($v);
3447   
3448    my $ntpline = pb_date2v($vtype,$pbos);
3449    print SCRIPT "# Time sync\n";
3450    print SCRIPT "echo setting up date with $ntpline\n";
3451    print SCRIPT "$ntpline\n";
3452    print SCRIPT "if [ \$\? -ne 0 ]; then\n";
3453    # Force if bad status a trick using date
3454    $ntpline = pb_date2v($vtype,$pbos, 1);
3455    print SCRIPT "  $ntpline\n";
3456    print SCRIPT "fi\n";
3457    # Use potential local proxy declaration in case we need it to download repo, pkgs, ...
3458    if (defined $ENV{'http_proxy'}) {
3459        print SCRIPT "export http_proxy=\"$ENV{'http_proxy'}\"\n";
3460    }
3461    if (defined $ENV{'https_proxy'}) {
3462        print SCRIPT "export https_proxy=\"$ENV{'https_proxy'}\"\n";
3463    }
3464    if (defined $ENV{'ftp_proxy'}) {
3465        print SCRIPT "export ftp_proxy=\"$ENV{'ftp_proxy'}\"\n";
3466    }
3467
3468    # Get list of packages to build/test and get some ENV vars as well
3469    my $ptr = pb_get_pkg();
3470    @pkgs = @$ptr;
3471    my $p = join(' ',@pkgs) if (@pkgs);
3472    print SCRIPT "export PBPROJVER=$ENV{'PBPROJVER'}\n";
3473    print SCRIPT "export PBPROJTAG=$ENV{'PBPROJTAG'}\n";
3474    print SCRIPT "export PBPACKAGER=\"$ENV{'PBPACKAGER'}\"\n";
3475
3476    # We may need to do some other tasks before building. Read a script here to finish setup
3477    if (-x "$ENV{'PBTMP'}/pb$vtype".".pre") {
3478        print SCRIPT "# Special pre-instructions to be launched\n";
3479        print SCRIPT pb_get_content("$ENV{'PBTMP'}/pb$vtype".".pre");
3480    }
3481
3482    if (-x "$ENV{'PBTMP'}/pb$vtype"."$action.pre") {
3483        print SCRIPT "# Special pre-$action instructions to be launched\n";
3484        print SCRIPT pb_get_content("$ENV{'PBTMP'}/pb$vtype"."$action.pre");
3485    }
3486
3487    print SCRIPT "# $action\n";
3488    print SCRIPT "echo $action"."ing packages on $vtype...\n";
3489
3490    if (($action eq "test") && (! -x "$ENV{'PBDESTDIR'}/pbtest")) {
3491        confess "No test script ($ENV{'PBDESTDIR'}/pbtest) found when in test mode. Aborting ...";
3492    }
3493    print SCRIPT "pb --version\n";
3494    my $act = "$action"."2pkg";
3495    $act = "build2prep" if ($action eq "prep");
3496    print SCRIPT "pb $verbose -p $ENV{'PBPROJ'} $act $p\n";
3497
3498    if (($vtype eq "ve") && ($vetype ne "docker")) {
3499        print SCRIPT "sudo /bin/umount /proc\n";
3500    }
3501
3502    # We may need to do some other tasks after building. Read a script here to exit properly
3503    if (-x "$ENV{'PBTMP'}/pb$vtype"."$action.post") {
3504        print SCRIPT "# Special post-$action instructions to be launched\n";
3505        print SCRIPT pb_get_content("$ENV{'PBTMP'}/pb$vtype"."$action.post");
3506    }
3507
3508    if (-x "$ENV{'PBTMP'}/pb$vtype".".post") {
3509        print SCRIPT "# Special post-instructions to be launched\n";
3510        print SCRIPT pb_get_content("$ENV{'PBTMP'}/pb$vtype".".post");
3511    }
3512
3513    print SCRIPT "echo ==== End of script $$ for $vtype $v =====\n";
3514    close(SCRIPT);
3515    chmod 0755,"$pbscript{$v}";
3516   
3517    $pm->finish if (defined $pbparallel);
3518}
3519$pm->wait_all_children if (defined $pbparallel);
3520die "Aborting, one or more of the children failed." if ((not $all_ok) && ($Global::pb_stop_on_error));
3521
3522# Launch the VM/VE/RM
3523# docker image to use if needed
3524# Gather all required files to send them to the VM/VE
3525# and launch the build through pbscript
3526pb_parallel_launchv(\%pbscript,$vtype,uc($vtype).$action,$pbstep,$pbimage,$pbforce);
3527}
3528
3529
3530sub pb_clean {
3531
3532    my $sleep=10;
3533    die "Unable to get env var PBDESTDIR" if (not defined $ENV{'PBDESTDIR'});
3534    die "Unable to get env var PBBUILDDIR" if (not defined $ENV{'PBBUILDDIR'});
3535    pb_log(0,"We will now wait $sleep s before removing both directories\n$ENV{'PBDESTDIR'} and $ENV{'PBBUILDDIR'}.\nPlease break me if this is wrong\n");
3536    sleep $sleep;
3537    pb_rm_rf($ENV{'PBDESTDIR'});
3538    pb_rm_rf($ENV{'PBBUILDDIR'});
3539}
3540
3541sub pb_newver {
3542
3543    die "-V Version parameter needed" if ((not defined $newver) || ($newver eq ""));
3544
3545    # Need this call for PBDIR
3546    my ($scheme2,$uri) = pb_cms_init($pbinit);
3547
3548    my ($pbconf,$pburl) = pb_conf_get("pbconfurl","pburl");
3549    $uri = $pbconf->{$ENV{'PBPROJ'}};
3550    my ($scheme, $account, $host, $port, $path) = pb_get_uri($uri);
3551
3552    # Checking CMS repositories status
3553    ($scheme2, $account, $host, $port, $path) = pb_get_uri($pburl->{$ENV{'PBPROJ'}});
3554
3555    if ($scheme !~ /^svn/) {
3556        die "Only SVN is supported at the moment";
3557    }
3558
3559    my $res = pb_vcs_isdiff($scheme,$ENV{'PBROOTDIR'});
3560    die "ERROR: No differences accepted in CMS for $ENV{'PBROOTDIR'} before creating a new version" if ($res != 0);
3561
3562    $res = pb_vcs_isdiff($scheme2,$ENV{'PBDIR'});
3563    die "ERROR: No differences accepted in CMS for $ENV{'PBDIR'} before creating a new version" if ($res != 0);
3564
3565    # Tree identical between PBCONFDIR and PBROOTDIR. The delta is what
3566    # we want to get for the root of the new URL
3567
3568    my $oldver = $ENV{'PBROOTDIR'};
3569    $oldver =~ s|^$ENV{'PBCONFDIR'}||;
3570
3571    pb_log(2, "PBCONFDIR: $ENV{'PBCONFDIR'}\nPBROOTDIR: $ENV{'PBROOTDIR'}\n");
3572
3573    my $newurl = "$uri/$newver";
3574    # Should probably use projver in the old file
3575    my $oldvertxt= basename($oldver);
3576    my $newvertxt = basename($newver);
3577
3578    # Duplicate and extract project-builder part
3579    pb_log(2,"Copying $uri/$oldver to $newurl\n");
3580    pb_vcs_copy($scheme,"$uri/$oldver",$newurl);
3581    pb_log(2,"Checkout $newurl to $ENV{'PBCONFDIR'}/$newver\n");
3582    pb_vcs_up($scheme,"$ENV{'PBCONFDIR'}");
3583
3584    # Duplicate and extract project
3585    my $newurl2 = "$pburl->{$ENV{'PBPROJ'}}/$newver";
3586
3587    pb_log(2,"Copying $pburl->{$ENV{'PBPROJ'}}/$oldver to $newurl2\n");
3588    pb_vcs_copy($scheme2,"$pburl->{$ENV{'PBPROJ'}}/$oldver",$newurl2);
3589
3590    my $tmp = $ENV{'PBDIR'};
3591    $tmp =~ s|$oldver$||;
3592    pb_log(2,"Checkout $newurl2 to $tmp/$newver\n");
3593    pb_vcs_up($scheme2,"$tmp");
3594
3595    # Update the .yml file
3596    open(FILE,"$ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml") || die "Unable to open $ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml";
3597    open(OUT,"> $ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml.new") || die "Unable to write to $ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml.new";
3598    while(<FILE>) {
3599        if (/^projver\s+$ENV{'PBPROJ'}\s*=\s*$oldvertxt$/) {
3600            s/^projver\s+$ENV{'PBPROJ'}\s*=\s*$oldvertxt$/projver $ENV{'PBPROJ'} = $newvertxt/;
3601            pb_log(0,"Changing projver from $oldvertxt to $newvertxt in $ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml\n");
3602        }
3603        if (/^testver/) {
3604            s/^testver/#testver/;
3605            pb_log(0,"Commenting testver in $ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml\n") if (/^testver/);
3606        }
3607        if (/^delivery/) {
3608            my $txt = $_;
3609            chomp($txt);
3610            pb_log(0,"Please check delivery ($txt) in $ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml\n");
3611        }
3612        print OUT $_;
3613    }
3614    close(FILE);
3615    close(OUT);
3616    rename("$ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml.new","$ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml");
3617
3618    # Checking pbcl files
3619    foreach my $f (<$ENV{'PBROOTDIR'}/*/pbcl>) {
3620        # Compute new pbcl file
3621        my $f2 = $f;
3622        $f2 =~ s|$ENV{'PBROOTDIR'}|$ENV{'PBCONFDIR'}/$newver/|;
3623        open(PBCL,$f) || die "Unable to open $f";
3624        my $foundnew = 0;
3625        while (<PBCL>) {
3626            $foundnew = 1 if (/^$newvertxt \(/);
3627        }
3628        close(PBCL);
3629        open(OUT,"> $f2") || die "Unable to write to $f2: $!";
3630        open(PBCL,$f) || die "Unable to open $f";
3631        # We skip as long as we do not reach the new version in the log (case of devel)
3632        my $skip = 1;
3633        # We let the 4 first lines
3634        my $cnt = 0;
3635        while (<PBCL>) {
3636            $skip = 0 if (/^$newvertxt \(/);
3637            if ((/^$oldvertxt \(/) && ($foundnew == 0)) {
3638                print OUT "$newvertxt ($pbdate)\n";
3639                print OUT "- TBD\n";
3640                print OUT "\n";
3641                pb_log(0,"WARNING: version $newvertxt not found in $f so added to $f2...\n") if ($foundnew == 0);
3642            }
3643            if ($cnt <= 4) {
3644                print OUT "$_" if (not /^$oldvertxt \(/);
3645            } else {
3646                print OUT "$_" if ($skip == 0);
3647            }
3648            $cnt++;
3649        }
3650        close(OUT);
3651        close(PBCL);
3652    }
3653
3654    pb_log(2,"Checkin $ENV{'PBCONFDIR'}/$newver\n");
3655    pb_vcs_checkin($scheme,"$ENV{'PBCONFDIR'}/$newver","updated to $ENV{'PBCONFDIR'}");
3656}
3657
3658#
3659# Return the list of VMs/VEs/RMs we are working on
3660# $all is a flag to know if we return all of them
3661# or only some (if all we publish also tar files in addition to pkgs
3662#
3663sub pb_get2v {
3664
3665my $vtype = shift;
3666my @v;
3667my $all = 0;
3668my $pbv = 'PBV';
3669my $vlist = $vtype."list";
3670
3671# Get VM/VE list
3672if ((not defined $ENV{$pbv}) || ($ENV{$pbv} =~ /^all$/)) {
3673    my ($ptr) = pb_conf_get($vlist);
3674    $ENV{$pbv} = $ptr->{$ENV{'PBPROJ'}};
3675    $all = 1;
3676}
3677pb_log(2,"$vtype: $ENV{$pbv}\n");
3678@v = split(/,/,$ENV{$pbv});
3679return(\@v,$all);
3680}
3681
3682# This function creates a giant script to configure a particular VM/VE/RM, it then copies the
3683# script to the target.
3684
3685# Function to create a potentialy missing pb account on the VM/VE/RM, and adds it to sudo
3686# Needs to use root account to connect to the VM/VE/RM
3687# pb will take your local public SSH key to access
3688# the pb account in the VM/VE/RM later on if needed
3689sub pb_setup2v {
3690
3691my $vtype = shift;
3692my $sbx = shift;
3693my $pbstep = 1;
3694
3695my ($vm,$all) = pb_get2v($vtype);
3696
3697# Adapt // mode to memory size
3698$pbparallel = pb_set_parallel($vtype);
3699
3700my $pm;
3701my $all_ok = 1;
3702if (defined $pbparallel) {
3703    $pm = new Parallel::ForkManager($pbparallel);
3704
3705    # Set which port the VM/RM will use to communicate
3706    $pm->run_on_start(\&pb_set_port);
3707    $pm->run_on_finish(sub { my ($pid, $code, $id, $signal, $dump) = @_;
3708                            $all_ok = 0 unless (($code == 0) && ($signal == 0) && ($dump == 0)); 
3709                        });
3710}
3711
3712my $counter = 0;
3713foreach my $v (@$vm) {
3714    # Script generated - should be before the fork - after the %pbscript hash isn't updated correctly
3715    $pbscript{$v} = "$ENV{'PBDESTDIR'}/setupv-$v";
3716
3717    $counter++;
3718    # Modulo 2 * pbparallel (to avoid synchronization problems)
3719    $counter = 1 if ((defined $pbparallel) && ($counter > 2 * $pbparallel));
3720    $pm->start($counter) and next if (defined $pbparallel);
3721
3722    # Get distro context
3723    my $pbos = pb_distro_get_context($v);
3724   
3725    # Deal with date sync.
3726    my $ntpline = pb_date2v($vtype,$pbos);
3727
3728    # Name of the account to deal with for VM/VE/RM
3729    # Do not use the one passed potentially with -a
3730    my ($pbac) = pb_conf_get($vtype."login");
3731    my $key = "";
3732
3733    # Prepare the script to be executed on the VM/VE/RM
3734    # in $ENV{'PBDESTDIR'}/setupv
3735
3736    pb_log(2,"Preparing setup script for $v in $pbscript{$v}\n");
3737    open(SCRIPT,"> $pbscript{$v}") || die "Unable to create $pbscript{$v}";
3738   
3739    print SCRIPT << 'EOF';
3740#!/usr/bin/perl -w
3741
3742use strict;
3743use File::Copy;
3744
3745# We should not need in this script more functions than what is provided
3746# by Base, Conf and Distribution to avoid problems at exec time.
3747# They are appended at the end.
3748
3749# Define mandatory global vars
3750EOF
3751    my $ppref = "our";
3752    $ppref = "my" if (($pbos->{'name'} =~ /^redhat/) && ($pbos->{'version'} =~ /^6/));
3753    print SCRIPT << "EOF";
3754$ppref \$pbdebug = $pbdebug;
3755$ppref \$pbLOG;
3756$ppref \$pbsynmsg = "pbscript";
3757$ppref \$pbdisplaytype = "text";
3758$ppref \$pblocale = "";
3759EOF
3760    print SCRIPT << 'EOF';
3761pb_log_init($pbdebug, $pbLOG);
3762EOF
3763    print SCRIPT << "EOF";
3764\$Global::pb_stop_on_error = $Global::pb_stop_on_error;
3765pb_temp_init($pbkeep);
3766pb_conf_init("$ENV{'PBPROJ'}");
3767
3768EOF
3769
3770    if ($vtype eq "ve") {
3771        $vetype = pb_ve_get_type($vetype);
3772        if ($vetype ne "docker") {
3773            print SCRIPT << "EOF";
3774# For VE we first need to mount some FS
3775pb_system("mount -t proc /proc /proc") unless (-d "/proc/$$");
3776EOF
3777        }
3778        print SCRIPT << "EOF";
3779# For VE we need a good null dev
3780# Except for RedHat 6.2 where it's good and doesnt like being recreated
3781EOF
3782        print SCRIPT "pb_system('rm -f /dev/null; mknod /dev/null c 1 3; chmod 777 /dev/null');\n" unless (($pbos->{'name'} =~ /^redhat/) && ($pbos->{'version'} =~ /^6/));
3783    } elsif ($vtype =~ /(v|r)m/) {
3784        # Nothing to do now.
3785        my $useless = 0;
3786    } else {
3787        die "Unknown virtual type $vtype";
3788    }
3789
3790    print SCRIPT << 'EOF';
3791
3792# Adds $pbac->{$ENV{'PBPROJ'}} as an account if needed
3793#
3794my $file="/etc/passwd";
3795open(PBFILE,$file) || die "Unable to open $file";
3796my $found = 0;
3797while (<PBFILE>) {
3798EOF
3799    print SCRIPT << "EOF";
3800    \$found = 1 if (/^$pbac->{$ENV{'PBPROJ'}}:/);
3801EOF
3802
3803# TODO: use an external parameter
3804my $home = "/home";
3805# Solaris doesn't like that we use /home
3806$home = "/export/home" if ($pbos->{'type'} eq "pkg");
3807
3808    print SCRIPT << "EOF";
3809}
3810close(PBFILE);
3811
3812if ( \$found == 0 ) {
3813    if ( ! -d "$home" ) {
3814        pb_mkdir_p("$home");
3815    }
3816EOF
3817    # TODO: Level of portability of these cmds ? Critical now for RM
3818    # TODO: Check existence before adding to avoid errors
3819    print SCRIPT << "EOF";
3820pb_system("/usr/sbin/groupadd $pbac->{$ENV{'PBPROJ'}}","Adding group $pbac->{$ENV{'PBPROJ'}}");
3821pb_system("/usr/sbin/useradd -g $pbac->{$ENV{'PBPROJ'}} -m -d $home/$pbac->{$ENV{'PBPROJ'}} $pbac->{$ENV{'PBPROJ'}}","Adding user $pbac->{$ENV{'PBPROJ'}} (group $pbac->{$ENV{'PBPROJ'}} - home $home/$pbac->{$ENV{'PBPROJ'}})");
3822}
3823EOF
3824
3825    # Copy the content of our local conf file to the VM/VE/RM
3826    my $content = pb_get_content(pb_distro_conffile());
3827    print SCRIPT << "EOF";
3828    #
3829    # Create a temporary local conf file for distribution support
3830    # This is created here before its use later. Its place is hardcoded, so no choice for the path
3831    #
3832    my \$tempconf = pb_distro_conffile();
3833    pb_mkdir_p(dirname(\$tempconf));
3834    open(CONF,"> \$tempconf") || die "Unable to create \$tempconf";
3835    print CONF q{$content};
3836    close(CONF);
3837EOF
3838
3839    if ($vtype =~ /(v|r)m/) {
3840        print SCRIPT << "EOF";
3841# allow ssh entry to build
3842#
3843mkdir "$home/$pbac->{$ENV{'PBPROJ'}}/.ssh",0700;
3844# Allow those accessing root to access the build account
3845copy("\$ENV{'HOME'}/.ssh/authorized_keys","$home/$pbac->{$ENV{'PBPROJ'}}/.ssh/authorized_keys");
3846chmod 0600,".ssh/authorized_keys";
3847pb_system("chown -R $pbac->{$ENV{'PBPROJ'}}:$pbac->{$ENV{'PBPROJ'}} $home/$pbac->{$ENV{'PBPROJ'}}","Finish setting up the account env for $pbac->{$ENV{'PBPROJ'}}");
3848
3849EOF
3850}
3851    print SCRIPT << 'EOF';
3852# No passwd for build account only keys
3853$file="/etc/shadow";
3854if (-f $file) {
3855    open(PBFILE,$file) || die "Unable to open $file";
3856    open(PBOUT,"> $file.new") || die "Unable to open $file.new";
3857    while (<PBFILE>) {
3858EOF
3859    print SCRIPT << "EOF";
3860        s/^$pbac->{$ENV{'PBPROJ'}}:\!\!:/$pbac->{$ENV{'PBPROJ'}}:*:/;
3861        s/^$pbac->{$ENV{'PBPROJ'}}:\!:/$pbac->{$ENV{'PBPROJ'}}:*:/; #SLES 9 e.g.
3862        s/^$pbac->{$ENV{'PBPROJ'}}:\\*LK\\*:/$pbac->{$ENV{'PBPROJ'}}:NP:/;  #Solaris e.g.
3863EOF
3864        print SCRIPT << 'EOF';
3865        print PBOUT $_;
3866    }
3867    close(PBFILE);
3868    close(PBOUT);
3869    rename("$file.new",$file);
3870    chmod 0640,$file;
3871    }
3872
3873# TODO: systemd support
3874# Keep the VM in text mode
3875$file="/etc/inittab";
3876if (-f $file) {
3877    open(PBFILE,$file) || die "Unable to open $file";
3878    open(PBOUT,"> $file.new") || die "Unable to open $file.new";
3879    while (<PBFILE>) {
3880        s/^(..):5:initdefault:$/$1:3:initdefault:/;
3881        print PBOUT $_;
3882    }
3883    close(PBFILE);
3884    close(PBOUT);
3885    rename("$file.new",$file);
3886    chmod 0640,$file;
3887}
3888
3889# pb has to be added to portage group on gentoo
3890
3891# We need to have that pb_distro_get_context function
3892# Get it from Project-Builder::Distribution
3893# And we now need the conf file required for this to work created above
3894
3895my $pbos = pb_distro_get_context(); 
3896print "distro tuple: ".Dumper($pbos)."\n";
3897
3898# Adapt sudoers
3899# sudo is not default on Solaris and needs to be installed first
3900# from http://www.sunfreeware.com/programlistsparc10.html#sudo
3901if ($pbos->{'type'} eq "pkg") {
3902    $file="/usr/local/etc/sudoers";
3903} else {
3904    $file="/etc/sudoers";
3905}
3906open(PBFILE,$file) || die "Unable to open $file";
3907open(PBOUT,"> $file.new") || die "Unable to open $file.new";
3908while (<PBFILE>) {
3909EOF
3910    # Skip what will be generated
3911    print SCRIPT << "EOF";
3912    next if (/^$pbac->{$ENV{'PBPROJ'}}\\s+/);
3913    next if (/^Defaults:$pbac->{$ENV{'PBPROJ'}}\\s+/);
3914    next if (/^Defaults:root\\s+\!requiretty/);
3915    next if (/^Defaults:root\\s+env_keep/);
3916EOF
3917    print SCRIPT << 'EOF';
3918    s/Defaults\s+requiretty/# disable Defaults: requiretty/;
3919    print PBOUT $_;
3920}
3921close(PBFILE);
3922EOF
3923    print SCRIPT << "EOF";
3924# Some distro force requiretty at compile time, so disable here
3925print PBOUT "Defaults:$pbac->{$ENV{'PBPROJ'}} !requiretty\\n";
3926print PBOUT "Defaults:root !requiretty\\n";
3927EOF
3928    # RH 6.2 sudo doesn't support env_keep
3929    if (($pbos->{'name'} ne "redhat") && ($pbos->{'version'} ne "6.2")) {
3930        print SCRIPT << "EOF";
3931# Keep proxy configuration while using sudo
3932print PBOUT "Defaults:$pbac->{$ENV{'PBPROJ'}}    env_keep += \\\"http_proxy https_proxy ftp_proxy\\\"\\n";
3933print PBOUT "Defaults:root    env_keep += \\\"http_proxy https_proxy ftp_proxy\\\"\\n";
3934EOF
3935    }
3936    # Try to restrict security to what is really needed
3937    if ($vtype =~ /^vm/) {
3938        my $hpath = pb_distro_get_param($pbos,pb_conf_get("ospathcmd-halt"));
3939        my @sudocmds = pb_get_sudocmds($pbos,$ntpline,"sudo $hpath","sudo /bin/date");
3940        print SCRIPT << "EOF";
3941# This is needed in order to be able on VM to halt the machine from the $pbac->{$ENV{'PBPROJ'}} account at least
3942# Build account $pbac->{$ENV{'PBPROJ'}} in VM also needs to setup date and install deps.
3943# Nothing else should be needed
3944EOF
3945        foreach my $c (@sudocmds) {
3946            print SCRIPT "print PBOUT \"$pbac->{$ENV{'PBPROJ'}}   ALL = NOPASSWD: $c\\n\";\n";
3947        }
3948    } elsif ($vtype =~ /^rm/) {
3949        my @sudocmds = pb_get_sudocmds($pbos,$ntpline);
3950        print SCRIPT << "EOF";
3951# Build account $pbac->{$ENV{'PBPROJ'}} in RM only needs to setup date and install deps if needed each time
3952EOF
3953        foreach my $c (@sudocmds) {
3954            print SCRIPT "print PBOUT \"$pbac->{$ENV{'PBPROJ'}}   ALL = NOPASSWD: $c\\n\";\n";
3955        }
3956    } else {
3957        print SCRIPT << "EOF";
3958# Build account $pbac->{$ENV{'PBPROJ'}} for VE needs to do a lot in the host (and chroot), so allow without restriction for now
3959print PBOUT "$pbac->{$ENV{'PBPROJ'}}   ALL=(ALL) NOPASSWD:ALL\n";
3960EOF
3961    }
3962    my $sudomode = pb_distro_get_param($pbos,pb_conf_get("ossudoersmode"));
3963    print SCRIPT << 'EOF';
3964close(PBOUT);
3965rename("$file.new",$file);
3966EOF
3967    print SCRIPT << "EOF";
3968chmod 0$sudomode,\$file;
3969
3970EOF
3971
3972    if ($vtype =~ /(v|r)m/) {
3973        # Sync date
3974        # do it after sudoers is setup
3975        print SCRIPT "my \$res = pb_system(\"$ntpline\",\"Updating time with $ntpline\",\"mayfailverbose\");\n";
3976        print SCRIPT "if (\$res != 0) {\n";
3977        # try again with another method
3978        $ntpline = pb_date2v($vtype,$pbos,1);
3979        print SCRIPT "  pb_system(\"$ntpline\",\"Updating time with $ntpline\",\"mayfailverbose\");\n";
3980        print SCRIPT "}\n";
3981    }
3982
3983    my ($addrepo) = pb_conf_read("$ENV{'PBDESTDIR'}/pbrc.yml","addrepo");
3984    my $pbrepo = pb_distro_get_param($pbos,$addrepo);
3985    print SCRIPT << 'EOF';
3986   
3987# We may need a proxy configuration. Get it from the local env
3988pb_apply_conf_proxy($pbos);
3989
3990# First install all required packages
3991pb_system("dnf clean all","Cleaning dnf env","mayfail") if (($pbos->{'name'} eq "fedora") && ($pbos->{'version'} >= 22));
3992pb_system("yum clean all","Cleaning yum env","mayfail") if ((($pbos->{'name'} eq "fedora") && ($pbos->{'version'} < 22)) || ($pbos->{'name'} eq "asianux") || ($pbos->{'name'} eq "rhel"));
3993my ($ospkgdep) = pb_conf_get_if("ospkgdep");
3994
3995my $pkgdep = pb_distro_get_param($pbos,$ospkgdep);
3996my $forcerepo;
3997EOF
3998    print SCRIPT "\$forcerepo = pb_distro_setuprepo_gen(\$pbos,\"$pbrepo\");\n" if ((defined $pbrepo) && ($pbrepo ne ""));
3999    print SCRIPT << 'EOF';
4000pb_distro_installdeps(undef,$pbos,$forcerepo,pb_distro_only_deps_needed($pbos,join(' ',split(/,/,$pkgdep))));
4001
4002EOF
4003    my $itype = pb_distro_get_param($pbos,pb_conf_get("pbinstalltype"));
4004    # Install from sandbox mean a file base install
4005    $itype = "file" if (defined $sbx);
4006    if ($itype =~ /^file/) {
4007        my $cmdget;
4008        if (defined $sbx) {
4009            # Install from sandbox mean using the result of the just passed sbx2build command
4010            # Get content saved in cms2build
4011            my ($pkg) = pb_conf_read("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml","pbpkg");
4012            my $pbextdir = pb_get_extdir();
4013            die "Unable to get package list" if (not defined $pkg);
4014
4015            # We consider 2 specific packages
4016            my $vertag1 = $pkg->{"ProjectBuilder"};
4017            my $vertag2 = $pkg->{"project-builder"};
4018            # get the version of the current package - maybe different
4019            pb_log(2,"Vertag1: $vertag1\n");
4020            pb_log(2,"Vertag2: $vertag2\n");
4021            my ($pbver1,$tmp1) = split(/-/,$vertag1);
4022            my ($pbver2,$tmp2) = split(/-/,$vertag2);
4023            $cmdget = "mv /tmp/ProjectBuilder-$pbver1$pbextdir.tar.gz ProjectBuilder-latest.tar.gz ; mv /tmp/project-builder-$pbver2$pbextdir.tar.gz project-builder-latest.tar.gz";
4024        } else {
4025            # TODO: shouldn't we have a variable to specify the source file?
4026            # And shouldn't we be getting the one that matches the current pb version?
4027            # For the moment, just take the latest stable one
4028            my $wget = pb_check_req("wget",0);
4029            $cmdget = "$wget --passive-ftp ftp://ftp.project-builder.org/src/ProjectBuilder-latest.tar.gz; $wget --passive-ftp ftp://ftp.project-builder.org/src/project-builder-latest.tar.gz";
4030        }
4031        print SCRIPT << 'EOF';
4032# Then install manually the missing perl modules
4033my ($osperldep,$osperlver) = pb_conf_get_if("osperldep","osperlver");
4034   
4035my $perldep = pb_distro_get_param($pbos,$osperldep);
4036my $bashopt = "";
4037$bashopt = "-x" if ($pbdebug ge 1);
4038my $verbopt = "mayfail";
4039$verbopt = "mayfailverbose" if ($pbdebug ge 1);
4040foreach my $m (split(/,/,$perldep)) {
4041    # Skip empty deps
4042    next if ($m =~ /^\s*$/);
4043    my $dir = $m;
4044    $dir =~ s/-.*//;
4045    pb_system("echo \"rm -rf $m* ; wget http://search.cpan.org/CPAN/modules/by-module/$dir/$m-$osperlver->{$m}.tar.gz ; gzip -cd $m-$osperlver->{$m}.tar.gz | tar xf - ; cd $m* ; if [ -f Build.PL ]; then perl Build.PL; ./Build ; ./Build install ; else perl Makefile.PL; make ; make install ; fi; cd .. ; rm -rf $m*\" | bash -e $bashopt" ,"Installing perl module $m-$osperlver->{$m}",$verbopt);
4046}
4047EOF
4048        print SCRIPT << 'EOF';
4049pb_system("rm -rf ProjectBuilder-* ; rm -rf project-builder-* ; rm -rf `perl -V:installvendorlib  | awk -F\"'\" '{print \$2}'`/ProjectBuilder ;
4050EOF
4051        print SCRIPT " $cmdget ; ";
4052        print SCRIPT << 'EOF'
4053gzip -cd ProjectBuilder-latest.tar.gz | tar xf - ; cd ProjectBuilder-* ; perl Makefile.PL ; make ; make install ; cd .. ; rm -rf ProjectBuilder-* ; gzip -cd project-builder-latest.tar.gz | tar xf - ; cd project-builder-* ; perl Makefile.PL ; make ; make install ; cd .. ; rm -rf project-builder-* ;","Building Project-Builder");
4054EOF
4055    } elsif ($itype =~ /^pkg/) {
4056        # pkg based install. We need to point to the project-builder.org repository
4057        print SCRIPT << 'EOF';
4058my $pkgforpb = pb_distro_get_param($pbos,pb_conf_get_if("ospkg"));
4059my $forcerepo = pb_distro_setuposrepo($pbos);
4060pb_log(1,"Forcing full package install with forcerepo\n") if (defined $forcerepo);
4061pb_distro_installdeps(undef,$pbos,$forcerepo,pb_distro_only_deps_needed($pbos,join(' ',split(/,/,$pkgforpb))));
4062EOF
4063    } else {
4064        # Unknown install type
4065        die("Unknown install type $itype->{$ENV{'PBPROJ'}} for param pbinstalltype");
4066    }
4067    print SCRIPT << 'EOF';
4068pb_system("pbdistrocheck",undef,"verbose");
4069pb_system("pb --version",undef,"verbose");
4070EOF
4071    if ($vtype eq "ve"){
4072        if ($vetype ne "docker") {
4073            print SCRIPT << 'EOF';
4074# For VE we need to umount some FS at the end
4075
4076pb_system("umount /proc","Unmounting /proc","mayfail");
4077EOF
4078        }
4079        print SCRIPT << 'EOF';
4080# Create a basic network file if not already there
4081
4082my $nf="/etc/sysconfig/network";
4083if ((! -f $nf) && ($pbos->{'type'} eq "rpm") && ($pbos->{'family'} ne "novell")) {
4084    open(NF,"> $nf") || die "Unable to create $nf";
4085    print NF "NETWORKING=yes\n";
4086    print NF "HOSTNAME=localhost\n";
4087    close(NF);
4088}
4089chmod 0755,$nf;
4090EOF
4091    }
4092
4093    # Adds pb_distro_get_context and all functions needed from ProjectBuilder::Distribution, Conf and Base
4094    foreach my $m ("ProjectBuilder/Base.pm","ProjectBuilder/Distribution.pm","ProjectBuilder/Conf.pm") {
4095        foreach my $d (@INC) {
4096            my $f = "$d/$m";
4097            if (-f "$f") {
4098                open(PBD,"$f") || die "Unable to open $f: $!";
4099                while (<PBD>) {
4100                    next if (/^package/);
4101                    next if (/^use Exporter/);
4102                    next if (/^use ProjectBuilder::/);
4103                    next if (/^our /);
4104                    print SCRIPT $_;
4105                }
4106                close(PBD);
4107                # We just need the first one of each file wrt @INC - hopefully that's the right one.
4108                last;
4109            }
4110        }
4111    }
4112    # Use a fake pb_version_init version here
4113    print SCRIPT << "EOF";
4114sub pb_version_init {
4115
4116return("$projectbuilderver","$projectbuilderrev");
4117}
41181;
4119EOF
4120    close(SCRIPT);
4121    chmod 0755,"$pbscript{$v}";
4122
4123    $pm->finish if (defined $pbparallel);
4124}
4125$pm->wait_all_children if (defined $pbparallel);
4126die "Aborting, one or more of the children failed." if ((not $all_ok) && ($Global::pb_stop_on_error));
4127
4128# These build scripts need to be run as root and force stop of VM at end
4129$ENV{'PBACCOUNT'} = "root";
4130
4131$sbx = "" if (not defined $sbx);
4132# Launch the VM/VE/RM
4133pb_log(2,"DEBUG: before parallel launch, pbscript hash is:".Dumper(%pbscript)."\n");
4134pb_parallel_launchv(\%pbscript,$vtype,uc($vtype)."Script".$sbx,$pbstep,$pbimage,$pbforce,0,$pbsnap);
4135
4136return;
4137}
4138
4139# Function to create a snapshot named 'pb' for VMs and a compressed tar for VEs
4140sub pb_snap2v {
4141
4142my $vtype = shift;
4143my $pbstep = 2;
4144
4145my ($vm,$all) = pb_get2v($vtype);
4146
4147# Script generated
4148my $pbscript = "$ENV{'PBDESTDIR'}/snapv.$$";
4149
4150my ($pbac) = pb_conf_get($vtype."login");
4151
4152foreach my $v (@$vm) {
4153    if ($vtype eq "ve") {
4154        # Get distro context
4155        my $pbos = pb_distro_get_context($v);
4156        my ($vepath) = pb_conf_get("vepath");
4157
4158        # Test if an existing snapshot exists and remove it if there is a VE
4159        if ((-f "$vepath->{$ENV{'PBPROJ'}}/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}.tar.gz") &&
4160            (! -d "$vepath->{$ENV{'PBPROJ'}}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}")) {
4161                pb_system("sudo rm -f $vepath->{$ENV{'PBPROJ'}}/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}.tar.gz","Removing previous snapshot $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}.tar.gz");
4162        }
4163    }
4164
4165    # Prepare the script to be executed on the VM/VE
4166    open(SCRIPT,"> $pbscript") || die "Unable to create $pbscript";
4167    print SCRIPT << 'EOF';
4168    #!/bin/bash
4169    sleep 2
4170EOF
4171    close(SCRIPT);
4172    chmod 0755,"$pbscript";
4173
4174    # Force shutdown of VM/VE
4175    # Force snapshot of VM/VE
4176    pb_script2v($pbscript,$vtype,$pbstep,1,$v,1);
4177}
4178return;
4179}
4180
4181# Function to update VMs/VEs/RMs with the latest distribution content
4182sub pb_update2v {
4183
4184my $vtype = shift;
4185my $pbstep = 2;
4186
4187my ($vm,$all) = pb_get2v($vtype);
4188
4189# Script generated
4190my $pbscript = "$ENV{'PBDESTDIR'}/updatev.$$";
4191
4192my ($pbac) = pb_conf_get($vtype."login");
4193
4194foreach my $v (@$vm) {
4195    # Get distro context
4196    my $pbos = pb_distro_get_context($v);
4197
4198    # Prepare the script to be executed on the VM/VE/RM
4199    # in $ENV{'PBDESTDIR'}/updatev
4200    open(SCRIPT,"> $pbscript") || die "Unable to create $pbscript";
4201   
4202    print SCRIPT << 'EOF';
4203    #!/bin/bash
4204    sleep 2
4205EOF
4206    # VE needs a good /proc
4207    $vetype = pb_ve_get_type($vetype);
4208    if (($vtype eq "ve") && ($vetype ne "docker")) {
4209        print SCRIPT "sudo /bin/mount -t proc /proc /proc\n";
4210    }
4211    print SCRIPT "$pbos->{'update'}\n";
4212    if (($vtype eq "ve") && ($vetype ne "docker")) {
4213        print SCRIPT "sudo /bin/umount /proc\n";
4214    }
4215    close(SCRIPT);
4216    chmod 0755,"$pbscript";
4217
4218    # Force shutdown of VM except
4219    pb_script2v($pbscript,$vtype,$pbstep,1,$v);
4220}
4221return;
4222}
4223
4224sub pb_announce {
4225
4226    my $antype = shift;
4227
4228    # Get all required parameters
4229    my ($pbpackager,$pbrepo) = pb_conf_get("pbpackager","pbrepo");
4230    my ($pkgv, $pkgt, $testver) = pb_conf_get_if("pkgver","pkgtag","testver");
4231    if (((not defined $testver) || (not defined $testver->{$ENV{'PBPROJ'}}) || ($testver->{$ENV{'PBPROJ'}} !~ /true/i)) && ($antype eq "Clean")) {
4232        # We just clean for test versions
4233        pb_log(0,"Unable to clean SSH repository for non testver version\n");
4234        return;
4235    }
4236    my $pkg = pb_cms_get_pkg($defpkgdir,$extpkgdir);
4237    my @pkgs = @$pkg;
4238    my %pkgs;
4239    my $first = 0;
4240
4241    # Get all distros concerned
4242    my $pbos = pb_distro_get_context();
4243    my $distrolist = pb_get_distros($pbos,undef);
4244    my %dl;
4245    my %theorlist;
4246    my %archlist;
4247    foreach my $d (split(/,/,$distrolist)) {
4248        my ($d1,$d2,$d3) = split(/-/,$d);
4249        $dl{$d1}++;
4250    }
4251
4252    # Command to find packages on repo
4253    my $findstr = "find ".join(" ",keys %dl)." ";
4254    my $srcstr = "";
4255    # Generated announce files
4256    my @files;
4257
4258    foreach my $pbpkg (@pkgs) {
4259        if ($first != 0) {
4260            $findstr .= "-o ";
4261        }
4262        $first++;
4263        if ((defined $pkgv) && (defined $pkgv->{$pbpkg})) {
4264            $pbver = $pkgv->{$pbpkg};
4265        } else {
4266            $pbver = $ENV{'PBPROJVER'};
4267        }
4268        if ((defined $pkgt) && (defined $pkgt->{$pbpkg})) {
4269            $pbtag = $pkgt->{$pbpkg};
4270        } else {
4271            $pbtag = $ENV{'PBPROJTAG'};
4272        }
4273
4274        # TODO: use virtual/real names here now
4275        my $pbrealpkg = $pbpkg;
4276        my $pbrealpkgrpm = pb_cms_get_real_pkg($pbpkg,"rpm");
4277        my $pbrealpkgdeb = pb_cms_get_real_pkg($pbpkg,"deb");
4278        if ($antype eq "Clean") {
4279            # We only clean test versions anyway
4280            $pbtag = "0";
4281            my $nver = $pbver;
4282            my $ntag = "[2-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]";
4283            $pbver .= $ntag;
4284            $findstr .= "-name \'$pbrealpkgrpm-$pbver-$pbtag\.*.rpm\' -o -name \'$pbrealpkgrpm-debug-$pbver-$pbtag\.*.rpm\' -o -name \'$pbrealpkgdeb"."_$pbver-$pbtag"."_*\.deb\' -o -name \'$pbrealpkgdeb"."_$pbver-$pbtag.dsc\' -o -name \'$pbrealpkgdeb"."_$pbver-$pbtag.tar.gz\' -o -name \'$pbrealpkg-$nver"."_p$ntag\.ebuild\' -o -name \'$pbrealpkg-$pbver-$pbtag*\.pkg\.gz\' -o -name \'$pbrealpkg-$pbver-$pbtag*\.sd\' ";
4285            $srcstr .= "src/$pbrealpkg-$pbver.tar.gz src/$pbrealpkg-$pbver.pbconf.tar.gz ";
4286        } else {
4287            # "Announce" and "Check"
4288            my @date=pb_get_date();
4289            # the matching is only done on packages made the same day for test version. Hopefully this is enough
4290            my $nver;
4291            if ((defined $testver) && (defined $testver->{$ENV{'PBPROJ'}}) && ($testver->{$ENV{'PBPROJ'}} =~ /true/i)) {
4292                $pbtag = "0";
4293                my $ntag .= strftime("%Y%m%d*", @date);
4294                $nver = $pbver."_p$ntag*";
4295                $pbver .= "$ntag*";
4296            } else { 
4297                $nver = $pbver."-r$pbtag";
4298            }
4299            $findstr .= "-name \'$pbrealpkgrpm-$pbver-$pbtag\.*.rpm\' -o -name \'$pbrealpkgdeb"."_$pbver-$pbtag"."_*\.deb\' -o -name \'$pbrealpkg-$nver\.ebuild\' -o -name \'$pbrealpkg-$pbver\.pkg\.gz\' -o -name \'$pbrealpkg-$pbver\.sd\' ";
4300        }
4301
4302        if ($antype eq "Announce") {
4303            my $chglog;
4304
4305            pb_cms_init($pbinit);
4306            # Get project info on log file and generate tmp files used later on
4307            $chglog = "$ENV{'PBROOTDIR'}/$pbpkg/pbcl";
4308            $chglog = "$ENV{'PBROOTDIR'}/pbcl" if (! -f $chglog);
4309            $chglog = undef if (! -f $chglog);
4310
4311            open(OUT,"> $ENV{'PBTMP'}/$pbpkg.ann") || die "Unable to create $ENV{'PBTMP'}/$pbpkg.ann: $!";
4312            my $pb;
4313            $pb->{'realpkg'} = $pbrealpkg;
4314            $pb->{'ver'} = $pbver;
4315            $pb->{'tag'} = $pbtag;
4316            $pb->{'date'} = $pbdate;
4317            $pb->{'extdir'} = pb_get_extdir();
4318            $pb->{'chglog'} = $chglog;
4319            $pb->{'packager'} = $pbpackager;
4320            $pb->{'proj'} = $ENV{'PBPROJ'};
4321            $pb->{'repo'} = $pbrepo;
4322            $pb->{'pbos'}->{'type'} = "announce";
4323            $pb->{'pbos'}->{'suffix'} = "none";
4324            pb_changelog($pb,\*OUT,"yes");
4325            close(OUT);
4326            push(@files,"$ENV{'PBTMP'}/$pbpkg.ann");
4327        } elsif ($antype eq "Check") {
4328            # For the check we also build the theoritical complete list we should get
4329            foreach my $d (split(/,/,$distrolist)) {
4330                $pbos = pb_distro_get_context($d);
4331                if ($pbos->{'type'} eq "rpm") {
4332                    $theorlist{"$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}/$pbrealpkgrpm-$pbver-$pbtag$pbos->{'suffix'}"} = 0;
4333                } elsif ($pbos->{'type'} eq "deb") {
4334                    $theorlist{"$pbos->{'name'}/$pbos->{'version'}/$pbrealpkgdeb"."_$pbver-$pbtag"} = 0;
4335                    # TODO are we always using the last arch ?
4336                    $archlist{"$pbos->{'name'}/$pbos->{'version'}/$pbrealpkgdeb"."_$pbver-$pbtag"} = "$pbos->{'arch'}";
4337                } elsif ($pbos->{'type'} eq "ebuild") {
4338                    my $prefix = "-r";
4339                    $prefix = "_p" if ((defined $testver) && (defined $testver->{$ENV{'PBPROJ'}}) && ($testver->{$ENV{'PBPROJ'}} =~ /true/i));
4340                    $theorlist{"$pbos->{'name'}/$pbos->{'version'}/$pbrealpkg-$pbver$prefix$pbtag.ebuild"} = 0;
4341                    $archlist{"$pbos->{'name'}/$pbos->{'version'}/$pbrealpkg-$pbver$prefix$pbtag.ebuild"} = "$pbos->{'arch'}";
4342                } elsif ($pbos->{'type'} eq "pkg") {
4343                    $theorlist{"$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}/$pbrealpkg-$pbver-$pbtag.pkg.gz"} = 0;
4344                } else {
4345                    pb_log(1,"No theoritical list possible for type $pbos->{'type'}\n");
4346                }
4347            }
4348        }
4349        pb_log(2,"theorlist initial: ".Dumper(%theorlist)."\n");
4350        pb_log(2,"archlist: ".Dumper(%archlist)."\n");
4351    }
4352    if ($antype eq "Announce") {
4353        $findstr .= " | grep -Ev \'src.rpm\'";
4354    } elsif ($antype eq "Clean") {
4355        $findstr .= " | xargs rm -f -v $srcstr ";
4356    }
4357
4358    # Prepare the command to run and execute it
4359    open(PBS,"> $ENV{'PBTMP'}/pbscript") || die "Unable to create $ENV{'PBTMP'}/pbscript";
4360    print PBS "#!/bin/bash\n";
4361    print PBS "set -x\n" if ($pbdebug gt 1);
4362    print PBS "$findstr | sort 2> /dev/null\n";
4363    close(PBS);
4364    chmod 0755,"$ENV{'PBTMP'}/pbscript";
4365    pb_send2target("Announce","$ENV{'PBTMP'}/pbscript");
4366
4367    my $sl = "Project $ENV{'PBPROJ'} version $ENV{'PBPROJVER'} is now available";
4368    if ($antype eq "Announce") {
4369        # Get subject line
4370        pb_log(0,"Please enter the title of your announce\n");
4371        pb_log(0,"(By default: $sl)\n");
4372        my $sl2 = <STDIN>;
4373        $sl = $sl2 if ($sl2 !~ /^$/);
4374
4375        # Prepare a template of announce
4376        open(ANN,"> $ENV{'PBTMP'}/announce.html") || die "Unable to create $ENV{'PBTMP'}/announce.html: $!";
4377        print ANN << "EOF";
4378$sl</p>
4379
4380<p>The project team is happy to announce the availability of a newest version of $ENV{'PBPROJ'} $ENV{'PBPROJVER'}. Enjoy it as usual!</p>
4381<p>
4382Now available at <a href="$pbrepo->{$ENV{'PBPROJ'}}">$pbrepo->{$ENV{'PBPROJ'}}</a>
4383</p>
4384<p>
4385EOF
4386    }
4387
4388    open(LOG,"$ENV{'PBTMP'}/system.$$.log") || die "Unable to read $ENV{'PBTMP'}/system.$$.log: $!";
4389    if ($antype eq "Announce") {
4390        my $col = 2;
4391        my $i = 1;
4392        print ANN << 'EOF';
4393<TABLE WIDTH="100%" CELLPADDING="0" CELLSPACING="0" BORDER="0">
4394<TR>
4395EOF
4396        while (<LOG>) {
4397            print ANN "<TD><A HREF=\"$pbrepo->{$ENV{'PBPROJ'}}/$_\">$_</A></TD>";
4398            $i++;
4399            if ($i > $col) {
4400                print ANN "</TR>\n<TR>";
4401                $i = 1;
4402            }
4403        }
4404    } elsif ($antype eq "Clean") {
4405        while (<LOG>) {
4406            # skip errors
4407            next if ($_ !~ /^removed /);
4408            pb_log(0,"$_");
4409        }
4410    } else {
4411        # In "Check" mode we need to compare the 2 lists (real and theoritical)
4412        while (<LOG>) {
4413            # Get package name and remove what is in extra for the theoritical list (arch at the end)
4414            chomp();
4415            # skip find errors
4416            next if (/^find:/);
4417            my $p = $_;
4418            $p =~ s/\.(i[3456]86|x86_64|noarch|src)\.rpm$//;
4419            $p =~ s/_(i[3456]86|amd64|all)\.deb$//;
4420            $p =~ s/(-0\.[0-9]{8})[0-9]{6}/$1*/ if ((defined $testver) && (defined $testver->{$ENV{'PBPROJ'}}) && ($testver->{$ENV{'PBPROJ'}} =~ /true/i));
4421            $p =~ s/(-r|_p[0-9]+)\.ebuild/$1*/ if ((defined $testver) && (defined $testver->{$ENV{'PBPROJ'}}) && ($testver->{$ENV{'PBPROJ'}} =~ /true/i));
4422            $theorlist{$p} = -2 if (not defined $theorlist{$p});
4423            $theorlist{$p} = $theorlist{$p} + 1;
4424        }
4425        pb_log(2,"theorlist after check: ".Dumper(%theorlist)."\n");
4426    }
4427    close(LOG);
4428
4429    # Nothing more for the cleanssh case
4430    return if ($antype eq "Clean");
4431
4432    if ($antype eq "Check") {
4433        my ($chkex) = pb_conf_get_if("checkexclude");
4434        my $vmbuildlist = "";
4435        my $vebuildlist = "";
4436        my $rmbuildlist = "";
4437        my @pt = pb_conf_get_if("vmlist","velist","rmlist");
4438        foreach my $t (sort keys %theorlist) {
4439            if (defined $theorlist{$t} and $theorlist{$t} >= 1) {
4440                pb_log(1,"Packages found for $t\n");
4441            } elsif (defined $theorlist{$t} and $theorlist{$t} < 0) {
4442                pb_log(0,"Extra Package found for $t\n");
4443            } else {
4444                pb_log(2,"Analyzing $t\n");
4445                my ($os,$ver,$arch,$package) = split(/\//,$t);
4446                # Some distro have no arch subdir
4447                if (not defined $package) {
4448                    $package = $arch;
4449                    # TODO: If both arch have failed, we just make the last one
4450                    $arch = $archlist{$t} if (defined $archlist{$t});
4451                }
4452                my $pbos = pb_distro_get_context("$os-$ver-$arch");
4453                my $pkgn = $package;
4454                if ($pbos->{'type'} ne "deb") {
4455                    # package name is more easily found from the end for non deb
4456                    # as '-' is the separator, but it can also be used in names
4457                    $pkgn = reverse($package);
4458                    # search the second '-' and isolate the now last part which is the full name
4459                    $pkgn =~ s/([^-]+)-([^-]+)-([\S])+$/$3/;
4460                } else {
4461                    $pkgn =~ s/([^_]+)_([\S])+$/$2/;
4462                }
4463                my $found = 0;
4464                # Handle the exclusion of OSes
4465                my $excl = "";
4466                $excl .= $chkex->{$pkgn} if (defined $chkex->{$pkgn});
4467                $excl .= $chkex->{"all"} if (defined $chkex->{"all"});
4468
4469                pb_log(2,"Searching in excl $excl\n");
4470                foreach my $ex (split(/,/,$excl)) {
4471                    $found = 1 if ("$os-$ver-$arch" =~ /^$ex/);
4472                }
4473                # Skip as excluded
4474                next if ($found == 1);
4475                pb_log(0,"Package NOT found for $t\n");
4476                # Avoid duplicates in list
4477                next if ($vmbuildlist =~ /$os-$ver-$arch/);
4478                next if ($vebuildlist =~ /$os-$ver-$arch/);
4479                next if ($rmbuildlist =~ /$os-$ver-$arch/);
4480                # check with which method we need to build
4481                if ((defined $pt[0]->{$ENV{'PBPROJ'}}) and ($pt[0]->{$ENV{'PBPROJ'}} =~ /$os-$ver-$arch/)) {
4482                    $vmbuildlist = "$os-$ver-$arch" if ($vmbuildlist eq "");
4483                    $vmbuildlist .= ",$os-$ver-$arch" if ($vmbuildlist !~ /$os-$ver-$arch/);
4484                    next;
4485                }
4486                if ((defined $pt[1]->{$ENV{'PBPROJ'}}) and ($pt[1]->{$ENV{'PBPROJ'}} =~ /$os-$ver-$arch/)) {
4487                    $vebuildlist = "$os-$ver-$arch" if ($vebuildlist eq "");
4488                    $vebuildlist .= ",$os-$ver-$arch" if ($vebuildlist !~ /$os-$ver-$arch/);
4489                    next;
4490                }
4491                if ((defined $pt[2]->{$ENV{'PBPROJ'}}) and ($pt[2]->{$ENV{'PBPROJ'}} =~ /$os-$ver-$arch/)) {
4492                    $rmbuildlist = "$os-$ver-$arch" if ($rmbuildlist eq "");
4493                    $rmbuildlist .= ",$os-$ver-$arch" if ($rmbuildlist !~ /$os-$ver-$arch/);
4494                }
4495            }
4496        }
4497        # If we want to rebuild automatically, let's do it
4498        if (defined $opts{'rebuild'}) {
4499            # SandBox or CMS
4500            pb_log(0,"Rebuilding from SandBox\n");
4501            pb_log(0,"for VMs: $vmbuildlist\n") if ($vmbuildlist ne "");
4502            pb_log(0,"for VEs: $vebuildlist\n") if ($vebuildlist ne "");
4503            pb_log(0,"for RMs: $rmbuildlist\n") if ($rmbuildlist ne "");
4504            pb_cms2build("SandBox");
4505            # Which mode
4506            $ENV{'PBV'} = $vmbuildlist;
4507            pb_build2v("vm","build") if ($vmbuildlist ne "");
4508            $ENV{'PBV'} = $vebuildlist;
4509            pb_build2v("ve","build") if ($vebuildlist ne "");
4510            $ENV{'PBV'} = $rmbuildlist;
4511            pb_build2v("rm","build") if ($rmbuildlist ne "");
4512        }
4513        # For the check part this is now finished
4514        return;
4515    }
4516
4517    print ANN << "EOF";
4518</TR>
4519</TABLE>
4520</p>
4521
4522<p>As usual source packages are also available in the same directory.</p>
4523
4524<p>
4525Changes are :
4526</p>
4527<p>
4528EOF
4529    # Get each package changelog content
4530    foreach my $f (sort(@files)) {
4531        open(IN,"$f") || die "Unable to read $f:$!";
4532        while (<IN>) {
4533            print ANN $_;
4534        }
4535        close(IN);
4536        print ANN "</p><p>\n";
4537    }
4538    print ANN "</p>\n";
4539    close(ANN);
4540
4541    # Allow for modification
4542    my $editor = "vi";
4543    $editor = $ENV{'EDITOR'} if (defined $ENV{'EDITOR'});
4544    pb_system("$editor $ENV{'PBTMP'}/announce.html","Allowing modification of the announce","noredir");
4545
4546    # Store it in DB for external usage (Web pages generation)
4547    my $dbh = pb_connect_db();
4548
4549    # To read whole file
4550    local $/;
4551    open(ANN,"$ENV{'PBTMP'}/announce.html") || die "Unable to read $ENV{'PBTMP'}/announce.html: $!";
4552    my $announce = <ANN>;
4553    close(ANN);
4554   
4555    pb_log(2,"INSERT INTO announces VALUES (NULL, $pbdate, $announce)");
4556    my $sth = $dbh->prepare(qq{INSERT INTO announces VALUES (NULL,?,?)})
4557                    || die "Unable to insert into db";
4558    $sth->execute($pbdate, $announce);
4559    $sth->finish();
4560    $dbh->disconnect;
4561
4562    # Then deliver it on the Web
4563    # $TOOLHOME/livwww www
4564
4565    # Mail it to project's ML
4566    open(ML,"| w3m -dump -T text/html > $ENV{'PBTMP'}/announce.txt") || die "Unable to create $ENV{'PBTMP'}/announce.txt: $!";
4567    print ML << 'EOF';
4568<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/x html1/DTD/xhtml1-strict.dtd">
4569
4570<html xmlns="http://www.w3.org/1999/xhtml" dir="ltr" xml:lang="en" lang="en">
4571  <head>
4572  </head>
4573  <body>
4574  <p>
4575EOF
4576    open(ANN,"$ENV{'PBTMP'}/announce.html") || die "Unable to read $ENV{'PBTMP'}/announce.html: $!";
4577    while(<ANN>) {
4578        print ML $_;
4579    }
4580    print ML << 'EOF';
4581</body>
4582</html>
4583EOF
4584    close(ML);
4585
4586    # To read whole file
4587    local $/;
4588    open(ANN,"$ENV{'PBTMP'}/announce.txt") || die "Unable to read $ENV{'PBTMP'}/announce.txt: $!";
4589    my $msg = <ANN>;
4590    close(ANN);
4591   
4592    # Preparation of headers
4593    eval
4594    {
4595        require Mail::Sendmail;
4596        Mail::Sendmail->import();
4597    };
4598    if ($@) {
4599        # Mail::Sendmail not found not sending mail !
4600        pb_log(0,"No Mail::Sendmail module found so not sending any mail !\n");
4601    } else {
4602        my ($pbml,$pbsmtp) = pb_conf_get("pbml","pbsmtp");
4603        my %mail = (   
4604            To          =>  $pbml->{$ENV{'PBPROJ'}},
4605            From        =>  $pbpackager->{$ENV{'PBPROJ'}},
4606            Smtp        =>  $pbsmtp->{$ENV{'PBPROJ'}},
4607            Body        =>  $msg,
4608            Subject     =>  "[ANNOUNCE] $sl",
4609        );
4610           
4611        # Send mail
4612        if (! sendmail(%mail)) {
4613            if ((defined $Mail::Sendmail::error) and (defined $Mail::Sendmail::log)) {
4614                die "Unable to send mail ($Mail::Sendmail::error): $Mail::Sendmail::log";
4615            }
4616        }
4617    }
4618}
4619
4620#
4621# Creates a set of HTML file containing the news for the project
4622# based on what has been generated by the pb_announce function
4623#
4624sub pb_web_news2html {
4625
4626    my $dest = shift || $ENV{'PBTMP'};
4627
4628    # Get all required parameters
4629    my ($pkgv, $pkgt) = pb_conf_get_if("pkgver","pkgtag");
4630
4631
4632    my $dbh = pb_connect_db();
4633    # For date handling
4634    $ENV{LANGUAGE}="C";
4635    my $firstjan = strftime("%Y-%m-%d", 0, 0, 0, 1, 0, localtime->year(), 0, 0, -1);
4636    my $oldfirst = strftime("%Y-%m-%d", 0, 0, 0, 1, 0, localtime->year()-1, 0, 0, -1);
4637    pb_log(2,"firstjan: $firstjan, oldfirst: $oldfirst, pbdate:$pbdate\n");
4638    my $all = $dbh->selectall_arrayref("SELECT id,date,announce FROM announces ORDER BY date DESC");
4639    my %news;
4640    $news{"cy"} = "";   # current year's news
4641    $news{"ly"} = "";   # last year news
4642    $news{"py"} = "";   # previous years news
4643    $news{"fp"} = "";   # first page news
4644    my $cpt = 4;        # how many news for first page
4645    # Extract info from DB
4646    foreach my $row (@$all) {
4647        my ($id, $date, $announce) = @$row;
4648        $news{"cy"} = $news{"cy"}."<p><B>$date</B> $announce\n" if ((($date cmp $pbdate) le 0) && (($firstjan cmp $date) le 0));
4649        $news{"ly"} = $news{"ly"}."<p><B>$date</B> $announce\n" if ((($date cmp $firstjan) le 0) && (($oldfirst cmp $date) le 0));
4650        $news{"py"} = $news{"py"}."<p><B>$date</B> $announce\n" if (($date cmp $oldfirst) le 0);
4651        $news{"fp"} = $news{"fp"}."<p><B>$date</B> $announce\n" if ($cpt > 0);
4652        $cpt--;
4653    }
4654    pb_log(1,"news{fp}: ".$news{"fp"}."\n");
4655    $dbh->disconnect;
4656
4657    # Generate the HTML content
4658    foreach my $pref (keys %news) {
4659        open(NEWS,"> $dest/pb_web_$pref"."news.html") || die "Unable to create $dest/pb_web_$pref"."news.html: $!";
4660        print NEWS "$news{$pref}";
4661        close(NEWS);
4662    }
4663}
4664
4665sub pb_connect_db {
4666
4667    # Store in DB for external usage (Web pages generation)
4668    my $db = "$ENV{'PBCONFDIR'}/announces3.sql";
4669
4670    my $precmd = "";
4671    if (! -f $db) {
4672        $precmd = "CREATE TABLE announces (id INTEGER PRIMARY KEY AUTOINCREMENT, date DATE, announce VARCHAR[65535])";
4673        # we should add the $db to the VCS as well here
4674    }
4675
4676    my $dbh = DBI->connect("dbi:SQLite:dbname=$db","","",
4677                        { RaiseError => 1, AutoCommit => 1 })
4678                        || die "Unable to connect to $db";
4679
4680    if ($precmd ne "") {
4681        my $sth = $dbh->prepare(qq{$precmd})
4682                    || die "Unable to create table into $db";
4683        $sth->execute();
4684    }
4685
4686    return($dbh);
4687}
4688
4689
4690# Return the SSH key file to use
4691# Potentially create it if needed
4692
4693sub pb_ssh_get {
4694
4695my $create = shift || 0;    # Do not create keys by default. In that case prefer native keys, not pb's ones
4696my $target = shift || "localhost";
4697my $nport = shift || "22";
4698
4699my ($pbagent) = pb_conf_get_if("pbusesshagent");
4700# use ssh-agent if asked so.
4701return(undef) if (($create == 0) && (defined $pbagent->{$ENV{'PBPROJ'}}) && ($pbagent->{$ENV{'PBPROJ'}} =~ /true/io));
4702
4703# Check the SSH environment
4704my $keyfile = undef;
4705
4706# Check whether ecdsa is accepted by the remote host
4707my $ecdsa = 0;
4708my $dsa = 0;
4709open(SCAN,"ssh-keyscan -p $nport -t ecdsa $target 2>/dev/null |") || pb_log(0,"Unable to check ECDSA support for $target");
4710while(<SCAN>) {
4711    $ecdsa = 1 if ($_ =~ /ecdsa-/);
4712    pb_log(2,"Found ECDSA support on $target ($nport)\n");
4713}
4714close(SCAN);
4715
4716open(SCAN,"ssh-keyscan -p $nport -t dsa $target 2>/dev/null |") || pb_log(0,"Unable to check DSA support for $target");
4717while(<SCAN>) {
4718    $dsa = 1 if ($_ =~ /ssh-dss/);
4719    pb_log(2,"Found DSA support on $target ($nport)\n");
4720}
4721close(SCAN);
4722
4723# We have specific keys by default - create them if needed
4724if (!(-e "$ENV{'HOME'}/.ssh/pb_ecdsa") && ($create eq 1)) {
4725    my $res = pb_system("ssh-keygen -q -b 521 -N '' -f $ENV{'HOME'}/.ssh/pb_ecdsa -t ecdsa","Generating ECDSA SSH keys for pb");
4726    if (($res != 0) && !(-e "$ENV{'HOME'}/.ssh/pb_dsa") && ($create eq 1)) {
4727        # ecdsa may not be supported if old ssh
4728        pb_system("ssh-keygen -q -b 1024 -N '' -f $ENV{'HOME'}/.ssh/pb_dsa -t dsa","Generating DSA SSH keys for pb");
4729    }
4730}
4731
4732#$keyfile = "$ENV{'HOME'}/.ssh/id_rsa" if (-s "$ENV{'HOME'}/.ssh/id_rsa");
4733# We still favoud DSA and fall back to ECDSA if not available
4734if ($dsa != 0) {
4735    # And we favour a specific key over a generic one except if in delivery to ftp server
4736    $keyfile = "$ENV{'HOME'}/.ssh/id_dsa" if ((-s "$ENV{'HOME'}/.ssh/id_dsa") && ($create eq 0));
4737    $keyfile = "$ENV{'HOME'}/.ssh/pb_dsa" if (-s "$ENV{'HOME'}/.ssh/pb_dsa");
4738} else {
4739    # And we favour a specific key over a generic one except if in delivery to ftp server
4740    $keyfile = "$ENV{'HOME'}/.ssh/id_ecdsa" if ((-s "$ENV{'HOME'}/.ssh/id_ecdsa") && ($ecdsa != 0) && ($create eq 0));
4741    $keyfile = "$ENV{'HOME'}/.ssh/pb_ecdsa" if ((-s "$ENV{'HOME'}/.ssh/pb_ecdsa") && ($ecdsa != 0));
4742}
4743if (defined $keyfile) {
4744    pb_log(2,"Using ssh key file $keyfile\n");
4745} else {
4746    pb_log(0, "Unable to find your public ssh key under $ENV{'HOME'}/.ssh\n");
4747}
4748return($keyfile);
4749}
4750
4751
4752# Returns the pid of a running VM command using a specific VM file, and the port used if the VM exists
4753sub pb_check_ps {
4754    my $vmcmd = shift;
4755    my $vmm = shift;
4756    my $vexist = 0;     # FALSE by default
4757    my $vmport = undef;         # NONE by default
4758
4759    open(PS, "ps auxhww|") || die "Unable to call ps";
4760    while (<PS>) {
4761        next if (! /$vmcmd/);
4762        next if (! /$vmm/);
4763        my $void1;
4764        ($void1, $vexist, $vmport) = split(/ +/,$_,3);
4765        pb_log(2,"pb_check_ps $vmport\n");
4766        $vmport =~ s/.*=tcp:127\.0\.0\.1:([0-9]+)-:22.*/$1/;
4767        chomp($vmport);
4768        pb_log(2,"pb_check_ps found a VM $vexist using port $vmport\n");
4769        last;
4770    }
4771    return($vexist,$vmport);
4772}
4773
4774
4775sub pb_extract_build_files {
4776
4777my $src=shift;
4778my $dir=shift;
4779my $ddir=shift;
4780my $mandatory=shift || "build";
4781
4782my $flag = "mayfail" if (($mandatory eq "patch") || ($mandatory eq "src"));
4783my $res;
4784
4785if ($src =~ /tar\.gz$/) {
4786    $res = pb_system("tar xfpz $src $dir","Extracting $mandatory files from $src",$flag);
4787} elsif ($src =~ /tar\.bz2$/) {
4788    $res = pb_system("tar xfpj $src $dir","Extracting $mandatory files from $src",$flag);
4789} else {
4790    die "Unknown compression algorithm for $src";
4791}
4792# If not mandatory return now
4793return() if (($res != 0) and (($mandatory eq "patch") || ($mandatory eq "src")));
4794return(pb_move_extracted_files($dir,$ddir));
4795}
4796
4797sub pb_move_extracted_files {
4798
4799my $dir = shift;
4800my $ddir = shift;
4801my @files;
4802
4803# In a recursive function , we need a local var as DIR handle
4804my $bdir;
4805
4806opendir($bdir,"$dir") || confess "Unable to open directory $dir: $!";
4807foreach my $f (readdir($bdir)) {
4808    next if ($f =~ /^\./);
4809    # Skip potential patch dir
4810    next if ($f =~ /^pbpatch/);
4811    # Skip potential source dir
4812    next if ($f =~ /^pbsrc/);
4813    # Skip potential backup files
4814    next if ($f =~ /~$/);
4815    if ((-d "$dir/$f") && (! -l "$dir/$f")) {
4816        # Case of the bug debian subdir
4817        pb_mkdir_p("$ddir/$f");
4818        push @files,pb_move_extracted_files("$dir/$f","$ddir/$f");
4819    } else {
4820        move("$dir/$f","$ddir") || die "Unable to move $dir/$f to $ddir";
4821        pb_log(2,"mv $dir/$f $ddir\n");
4822        push @files,"$ddir/$f";
4823    }
4824}
4825closedir($bdir);
4826# Not enough but still a first cleanup
4827pb_rm_rf("$dir");
4828return(@files);
4829}
4830
4831sub pb_list_bfiles {
4832
4833my $dir = shift;
4834my $pbpkg = shift;
4835my $bfiles = shift;
4836my $pkgfiles = shift;
4837my $supfiles = shift;
4838# subdir to keep if recursive mode, empty by default
4839my $subdir = shift || "";
4840# In a recursive function , we need a local var as DIR handle
4841my $bdir;
4842
4843pb_log(2,"DEBUG: entering pb_list_bfiles in $dir: ".Dumper($bfiles)."\n");
4844opendir($bdir,"$dir") || die "Unable to open dir $dir: $!";
4845foreach my $f (readdir($bdir)) {
4846    pb_log(3,"DEBUG: pb_list_bfiles found $f\n");
4847    next if ($f =~ /^\./);
4848    if (-d "$dir/$f") {
4849        # Recurse for directories (Debian 3.0 format e.g.)
4850        pb_log(2,"DEBUG: pb_list_bfiles recurse in $dir/$f\n");
4851        pb_list_bfiles("$dir/$f",$pbpkg,$bfiles,$pkgfiles,$supfiles,$f);
4852        next;
4853    }
4854
4855    my $key = $f;
4856    # if recursive then store also the subdir
4857    $key = "$subdir/$f" if ($subdir ne "");
4858    $bfiles->{$key} = "$dir/$f";
4859    $bfiles->{$key} =~ s~$ENV{'PBROOTDIR'}~~;
4860    if (defined $supfiles->{$pbpkg}) {
4861        $pkgfiles->{$key} = "$dir/$f" if ($f =~ /$supfiles->{$pbpkg}/);
4862    }
4863}
4864closedir($bdir);
4865pb_log(2,"DEBUG: exiting pb_list_bfiles: ".Dumper($bfiles)."\n");
4866}
4867
4868sub pb_add_comma {
4869
4870my $str = shift;
4871my $addstr = shift;
4872
4873$str .= "," if (defined $str);
4874$str .= $addstr;
4875return($str);
4876}
4877
4878sub pb_list_sfiles {
4879
4880my $sdir = shift;
4881my $ptr = shift;
4882my $pbos = shift;
4883my $extdir = shift;
4884
4885pb_log(2,"DEBUG: entering pb_list_sfiles: ".Dumper($ptr)."\n");
4886my $key = "$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
4887
4888# Prepare local sources for this distro - They are always applied first - May be a problem one day
4889# This function works for both patches and additional sources
4890foreach my $p (sort(<$sdir/*>)) {
4891    $ptr->{$key} = pb_add_comma($ptr->{$key},"file://$p") if (($p =~ /\.all$/) || ($p =~ /\.$pbos->{'os'}$/) || ($p =~ /\.$pbos->{'type'}$/) || ($p =~ /\.$pbos->{'family'}$/) || ($p =~ /\.$pbos->{'name'}$/) || ($p =~ /\.$pbos->{'name'}-$pbos->{'version'}$/) ||($p =~ /\.$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}$/));
4892}
4893
4894# Prepare also remote sources to be included - Applied after the local ones
4895foreach my $p ("all","$pbos->{'os'}","$pbos->{'type'}","$pbos->{'family'}","$pbos->{'name'}","$pbos->{'name'}-$pbos->{'version'}","$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}") {
4896    my $f = "$extdir.".".$p";
4897    next if (not -f $f);
4898    if (not open(PATCH,$f)) {
4899        pb_display("Unable to open existing external source file content $f\n");
4900        next;
4901    }
4902    while (<PATCH>) {
4903        chomp();
4904        $ptr->{$key} = pb_add_comma($ptr->{$key},"$_");
4905    }
4906    close(PATCH);
4907}
4908pb_log(2,"DEBUG: exiting pb_list_sfiles: ".Dumper($ptr)."\n");
4909return($ptr);
4910}
4911   
4912#
4913# Return the list of packages we are working on in a non CMS action
4914#
4915sub pb_get_pkg {
4916
4917my @pkgs = ();
4918
4919# In script actions that file may not exist.
4920return(\@pkgs) if ((not defined $ENV{'PBDESTDIR'}) || (not defined $ENV{'PBPROJVER'}) || (not defined $ENV{'PBPROJTAG'}) || (! -f "$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml"));
4921
4922my ($var) = pb_conf_read("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml","pbpkg");
4923@pkgs = keys %$var if (defined $var);
4924
4925pb_log(0,"Packages: ".join(',',@pkgs)."\n") if (@pkgs);
4926return(\@pkgs);
4927}
4928
4929# Manages VM/RM SSH port communication
4930sub pb_get_port {
4931
4932my $port = shift;
4933my $pbos = shift;
4934my $cmt = shift;
4935my $nport;
4936
4937die "No port passed in parameter. Report to dev team\n" if (not defined $port);
4938# key is project on VM, but machine tuple for RM
4939if ($cmt =~ /^RM/i) {
4940    $nport = $port->{"$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}"};
4941} else {
4942    $nport = $port->{$ENV{'PBPROJ'}};
4943}
4944pb_log(2,"pb_get_port with $nport\n");
4945if ($cmt =~ /^VM/i) {
4946    # Maybe a port was given as parameter so overwrite
4947    $nport = "$pbport" if (defined $pbport);
4948    # Maybe in // mode so use the env var set up as an offset to the base port
4949    $nport = $nport + $ENV{'PBVMPORT'} -1 if ((defined $pbparallel) && (defined $ENV{'PBVMPORT'}));
4950}
4951pb_log(2,"pb_get_port returns $nport\n");
4952return($nport);
4953}
4954
4955sub pb_set_port { 
4956       
4957my ($pid,$ident) = @_;
4958pb_log(2,"pb_set_port for VM ($pid), id $ident\n");
4959$ENV{'PBVMPORT'} = $ident;
4960pb_log(2,"pb_set_port sets PBVMPORT in env to $ENV{'PBVMPORT'}\n");
4961}
4962
4963sub pb_set_parallel {
4964
4965my $vtype = shift;
4966
4967pb_log(2,"pb_set_parallel vtype: $vtype\n");
4968# Take care of memory size if VM, parallel mode and more than 1 action
4969if ((defined $pbparallel) && ($pbparallel ne 1) && ($vtype eq "vm")) {
4970    eval
4971    {
4972        require Linux::SysInfo;
4973        Linux::SysInfo->import();
4974    };
4975    if ($@) {
4976        # Linux::SysInfo not found
4977        pb_log(1,"ADVISE: Install Linux::SysInfo to benefit from automatic parallelism optimization.\nOr optimize manually pbparallel in your pb.conf file\nUsing $pbparallel processes max at a time for the moment\nWARNING: This may consume too much memory for your system\n");
4978    } else {
4979        # Using the memory size
4980        my $si = Linux::SysInfo::sysinfo();
4981        if (not defined $si) {
4982            pb_log(1,"ADVISE: Install Linux::SysInfo to benefit from automatic parallelism optimization.\nOr optimize manually pbparallel in your pb.conf file\nUsing $pbparallel processes max at a time for the moment\nWARNING: This may consume too much memory for your system\n");
4983        } else {
4984            # Keep the number of VM whose memory can be allocated
4985            my $ram = $si->{"totalram"}-$si->{"sharedram"}-$si->{"bufferram"};
4986            my $ram2;
4987            my ($vmmem) = pb_conf_get_if("vmmem");
4988
4989            my $v = "default";
4990            if ((defined $vmmem) and (defined $vmmem->{$v})) {
4991                $ram2 = $vmmem->{$v};
4992            } else {
4993                # Default for KVM/QEMU
4994                $ram2 = 128;
4995            }
4996            $pbparallel = sprintf("%d",$ram/$ram2);
4997        }
4998        pb_log(1,"Using $pbparallel processes at a time\n");
4999    }
5000}
5001pb_log(2,"pb_set_parallel returns: $pbparallel\n") if (defined $pbparallel);
5002return($pbparallel);
5003}
5004
5005sub pb_get_sudocmds { 
5006       
5007my $pbos = shift;
5008my @cmds = @_;
5009my %sudocmds;
5010
5011pb_log(2,"pb_get_sudocmds entering with lines:".Dumper(@cmds)."\n");
5012foreach my $c (split(/;/,$pbos->{'update'}),split(/;/,$pbos->{'install'}),@cmds) {
5013    pb_log(2,"pb_get_sudocmds analyses $c\n");
5014    next if ($c !~ /^\s*sudo/);
5015    # remove sudo and leading spaces
5016    $c =~ s/^\s*sudo\s+//;
5017    # keep only the command, not the params
5018    $c =~ s/([^\s]+)\s.*$/$1/;
5019    $sudocmds{$c} = "";
5020}
5021pb_log(2,"pb_get_sudocmds returns ".Dumper(keys %sudocmds)."\n");
5022return(keys %sudocmds);
5023}
5024
5025sub pb_sign_setenv {
5026
5027my ($passfile, $passphrase, $passpath) = pb_conf_get_if("pbpassfile","pbpassphrase","pbpasspath");
5028$ENV{'PBPASSPHRASE'} = $passphrase->{$ENV{'PBPROJ'}} if ((not defined $ENV{'PBPASSPHRASE'}) && (defined $passphrase->{$ENV{'PBPROJ'}}));
5029$ENV{'PBPASSFILE'} = $passfile->{$ENV{'PBPROJ'}} if ((not defined $ENV{'PBPASSFILE'})&& (defined $passfile->{$ENV{'PBPROJ'}})) ;
5030$ENV{'PBPASSPATH'} = $passpath->{$ENV{'PBPROJ'}} if ((not defined $ENV{'PBPASSPATH'})&& (defined $passpath->{$ENV{'PBPROJ'}})) ;
5031}
5032
5033sub pb_sign_setenv_rpm {
5034
5035pb_sign_setenv();
5036
5037eval
5038{
5039    require RPM4::Sign;
5040    RPM4::Sign->import();
5041};
5042if ($@) {
5043    # RPM4::Sign not found
5044    pb_log(1,"WARNING: Install RPM4::Sign to benefit from automatic package signing.\n");
5045    return(undef);
5046} else {
5047    return(undef) if (((not defined $ENV{'PBPASSPHRASE'}) and (not defined $ENV{'PBPASSFILE'})) || (not defined $ENV{'PBPACKAGER'}) || (not defined $ENV{'PBPASSPATH'}));
5048    my $sign = RPM4::Sign->new(
5049        passphrase => $ENV{'PBPASSPHRASE'},
5050        name => $ENV{'PBPACKAGER'},
5051        path => $ENV{'PBPASSPATH'},
5052        password_file => $ENV{'PBPASSFILE'},
5053    );
5054    return(undef) if (not defined $sign);
5055    return(undef) if (not defined $sign->{name});
5056    return(undef) if (not defined $sign->{path});
5057    return(undef) if ((not defined $sign->{passphrase}) && (not defined $sign->{password_file}));
5058    return(undef) if (not defined $sign->{keyid});
5059
5060    return($sign);
5061}
5062}
5063
5064# Cf: http://wiki.debian.org/SecureApt
5065
5066sub pb_sign_file {
5067
5068my $file = shift;
5069
5070pb_sign_setenv();
5071pb_log(0,"Signing Debian Release file as $ENV{'PBPACKAGER'}...\n");
5072
5073if (not defined $ENV{'PBPASSPHRASE'}) {
5074    if (not defined $ENV{'PBPASSFILE'}) {
5075        confess("No pbpassphrase nor pbpassfile defined, unable to handle GPG Release file\n");
5076    } else {
5077        open(PASS,"$ENV{'PBPASSFILE'}") || confess("Unable to open $ENV{'PBPASSFILE'}\n");
5078        my $pass = <PASS>;
5079        chomp($pass);
5080        $ENV{'PBPASSPHRASE'} = $pass;
5081        close(PASS);
5082    }
5083}
5084
5085unlink("$file.gpg");
5086pb_system("echo \"$ENV{'PBPASSPHRASE'}\" | gpg --sign --detach -a -o $file.gpg -r \"$ENV{'PBPACKAGER'}\" -b --use-agent --passphrase-fd 0 $file","Creating the signed version of $file with $ENV{'PBPACKAGER'} GPG key...\n");
5087}
5088
5089sub pb_sign_pkgs {
5090
5091my $pbos = shift;
5092my $made = shift;
5093
5094pb_log(2,"entering pb_sign_pkg: $made ".Dumper($pbos)."\n");
5095# Remove extra spaces
5096$made =~ s/\s+/ /g;
5097$made =~ s/^\s//g;
5098$made =~ s/\s$//g;
5099
5100if ($pbos->{'type'} eq "rpm") {
5101    my $sign = pb_sign_setenv_rpm();
5102    if (defined $sign) {
5103        pb_log(0,"Signing RPM packages as $ENV{'PBPACKAGER'}...\n");
5104        pb_log(2,"pb_sign_pkg: pkgs:".Dumper(split(/ /,$made))."\n");
5105        $sign->rpmssign(split(/ /,$made));
5106    }
5107} elsif ($pbos->{'type'} eq "deb") {
5108    my $changes = "";
5109    foreach my $c (split(/ /,$made)) {
5110        $changes .= " $ENV{'PBBUILDDIR'}/$c" if (($c =~ /\.changes$/) && (-f "$ENV{PBBUILDDIR}/$c"));
5111    }
5112    my $debsigncmd = pb_check_req("debsign",1);
5113    pb_system("$debsigncmd -m\'$ENV{'PBPACKAGER'}\' $changes","Signing DEB packages as $ENV{'PBPACKAGER'}...","mayfail") if (($changes ne "") && (defined $debsigncmd));
5114} else {
5115    pb_log(0,"I don't know yet how to sign packages for type $pbos->{'type'}.\nPlease give feedback to dev team\n");
5116}
5117pb_log(2,"exiting pb_sign_pkg\n");
5118}
5119
5120# return list of all distributions supported, comma separated
5121sub pb_get_distros {
5122
5123my $pbos = shift;
5124my $pbtarget = shift;
5125
5126my @dists = ("$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}");
5127
5128# Get list of distributions for which we need to generate build files if no target
5129if (not defined $pbtarget) {
5130    my @pt = pb_conf_get_if("vmlist","velist","rmlist");
5131    foreach my $pt (@pt) {
5132        push(@dists, split(/,/, $pt->{$ENV{PBPROJ}}))