source: devel/pb/bin/pb @ 2350

Last change on this file since 2350 was 2350, checked in by bruno, 9 months ago

Improve printing for getconf

  • Property svn:executable set to *
File size: 183.4 KB
Line 
1#!/usr/bin/perl -w
2#
3# Project Builder main application
4#
5# $Id$
6#
7# Copyright B. Cornec 2007-today
8# Eric Anderson's changes are (c) Copyright 2012 Hewlett Packard
9# Provided under the GPL v2
10
11# Syntax: see at end
12
13use strict 'vars';
14
15# The modules mentioned here are required by pb when used both
16# locally or inside a VE/VM/RM
17# Additional required modules only used locally are called with a require
18# in their respective section
19use Getopt::Long qw(:config auto_abbrev no_ignore_case);
20use Carp qw/confess cluck/;
21use Data::Dumper;
22use English;
23use File::Basename;
24use File::Copy;
25use File::stat;
26use File::Find;
27use Time::localtime qw(localtime);
28use POSIX qw(strftime);
29use lib qw (lib);
30use ProjectBuilder::Version;
31use ProjectBuilder::Base;
32use ProjectBuilder::Display;
33use ProjectBuilder::Conf;
34use ProjectBuilder::Distribution;
35use ProjectBuilder::VCS;
36use ProjectBuilder::CMS;
37use ProjectBuilder::Env;
38use ProjectBuilder::Filter;
39use ProjectBuilder::Changelog;
40use ProjectBuilder::VE;
41
42# Global variables
43$Global::pb_stop_on_error = 0;      # False by default
44$Global::pb_show_sudo = 0;          # False by default
45
46my %opts;                           # CLI Options
47my $action;                         # action to realize
48my $test = "FALSE";                 # Not used
49my $pbforce = 0;                    # Force VM stop
50my $pbsnap = 0;                     # Do not use snapshot mode for VM/VE by default
51my $pbkeep = 0;                     # keep temporary directory at the end
52my $option = "";                    # Not used
53my @pkgs;                           # list of packages
54my $pbtag;                          # Global Tag variable
55my $pbver;                          # Global Version variable
56my %pbscript;                       # Name of the scripts per V
57my $pbrev;                          # Global REVISION variable
58my $pbtarget = undef;               # Target os-ver-arch you want to build for
59my $pbport;                         # Port to use to connect to the VM/RM
60my $newver;                         # New version to create
61my $pbimage = undef;                # ISO image for the VM to create or docker image for the VE
62my $vetype = undef;                 # Type of Virtual environment to deal with
63
64my @date = pb_get_date();
65my $pbdate = strftime("%Y-%m-%d", @date);
66
67=pod
68
69=head1 NAME
70
71pb, aka project-builder.org - builds packages for your projects
72
73=head1 DESCRIPTION
74
75pb helps you build various packages directly from your project sources.
76Those sources could be handled by a CMS (Configuration Management System)
77such as Subversion, CVS, Git, Mercurial... or being a simple reference to a compressed tar file.
78It's based on a set of configuration files, a set of provided macros to help
79you keeping build files as generic as possible. For example, a single .spec
80file should be required to generate for all rpm based distributions, even
81if you could also have multiple .spec files if required.
82
83=head1 SYNOPSIS
84
85pb [-vhSq][-r pbroot][-p project][[-s script -a account -P port][-T VEtype][-t [os-ver-arch]][-m os-ver-arch[,...]]][-g][-i image] <action> [<pkg1> ...]
86
87pb [--verbose][--help][--man][--quiet][--snapshot][--revision pbroot][--project project][[--script script --account account --port port][-T VEtype][--target [os-ver-arch]][--machine os-ver-arch[,...]]][--nographic][--image image][--rebuild] <action> [<pkg1> ...]
88
89=head1 OPTIONS
90
91=over 4
92
93=item B<-v|--verbose>
94
95Increase verbosity
96
97=item B<-q|--quiet>
98
99Do not print any output.
100
101=item B<-h|--help>
102
103Print a brief help message and exits.
104
105=item B<-S|--snapshot>
106
107Use the snapshot mode of VMs or VEs
108
109=item B<--man>
110
111Prints the manual page and exits.
112
113=item B<-t|--target os-ver-arch>
114
115Name of the target system you want to build for.
116All if none precised.
117
118=item B<-m|--machine os-ver-arch[,os-ver-arch,...]>
119
120Name of the Virtual Machines (VM), Virtual Environments (VE) or Remote Machines (RM)
121you want to build on (comma separated).
122All if none precised (or use the env variable PBV).
123
124=item B<-T|--vetype VEtype]>
125
126Type of Virtual Environments (VE)
127Can be chroot or docker.
128
129=item B<-s|--script script>
130
131Name of the script you want to execute on the related VMs/VEs/RMs.
132
133=item B<-g|--nographic>
134
135Do not launch VMs in graphical mode.
136
137=item B<-i|--image image>
138
139It could be either:
140- The name of the ISO image of the distribution you want to install on the related VMs
141- The name of the docker image of the distribution you want to install on the related VEs
142
143
144=item B<-a|--account account>
145
146Name of the account to use to connect on the related VMs/RMs.
147
148=item B<-P|--port port_number>
149
150Port number to use to connect on the related VMs/RMs.";
151
152=item B<-p|--project project_name>
153
154Name of the project you're working on (or use the env variable PBPROJ)
155
156=item B<-r|--revision revision>
157
158Path Name of the project revision under the CMS (or use the env variable PBROOT)
159
160=item B<-V|--version new_version>
161
162New version of the project to create based on the current one.
163
164=item B<-k|--keep>
165
166Keep the temporary dir where files have been created in or der to help debug
167
168=item B<--rebuild>
169
170Only valid with the checkssh action, it alllows to automatically relaunch the build of the failed packages
171
172=item B<--no-stop-on-error>
173
174Continue through errors with best effort.
175
176=back
177
178=head1 ARGUMENTS
179
180<action> can be:
181
182=over 4
183
184=item B<sbx2build>
185
186Create tar files for the project under your CMS.
187Current state of the exported content is taken.
188CMS supported are SVN, SVK, CVS, Git and Mercurial
189parameters are packages to build
190if not using default list
191
192=item B<cms2build>
193
194Create tar files for the project under your CMS.
195Current state of the CMS is taken.
196CMS supported are SVN, SVK, CVS, Git and Mercurial
197parameters are packages to build
198if not using default list
199
200=item B<build2prep>
201
202Prepare the environment for build by installing required dependencies. Done once on the build system.
203
204=item B<build2pkg>
205
206Create packages for your running distribution
207
208=item B<cms2pkg>
209
210cms2build + build2pkg
211
212=item B<sbx2pkg>
213
214sbx2build + build2pkg
215
216=item B<sbx2pkg2ins>
217
218sbx2pkg + final install of packages
219
220=item B<sbx2prep>
221
222sbx2build + build2prep
223
224=item B<build2ssh>
225
226Send the tar files to a SSH host
227
228=item B<sbx2ssh>
229
230sbx2build + build2ssh
231
232=item B<cms2ssh>
233
234cms2build + build2ssh
235
236=item B<pkg2ssh>
237
238Send the packages built to a SSH host
239
240=item B<build2vm>
241
242Create packages in VMs, launching them if needed
243and send those packages to a SSH host once built
244VM type supported are QEMU and KVM
245
246=item B<build2ve>
247
248Create packages in VEs, creating it if needed
249and send those packages to a SSH host once built
250
251=item B<build2rm>
252
253Create packages in RMs, which should pre-exist,
254and send those packages to a SSH host once built
255RM means Remote Machine, and could be a physical or Virtual one.
256This is one buildfarm integration for pb.
257
258=item B<prepvm>
259
260Prepare the VMs to have all requirements to build the project
261
262=item B<prepve>
263
264Prepare the VEs to have all requirements to build the project
265
266=item B<preprm>
267
268Prepare the RMs to have all requirements to build the project
269
270=item B<sbx2vm>
271
272sbx2build + build2vm
273
274=item B<sbx2ve>
275
276sbx2build + build2ve
277
278=item B<sbx2docker>
279
280sbx2build + build2ve with a potential build of all necessary docker containers to perform it
281
282=item B<sbx2rm>
283
284sbx2build + build2rm
285
286=item B<cms2vm>
287
288cms2build + build2vm
289
290=item B<cms2ve>
291
292cms2build + build2ve
293
294=item B<cms2rm>
295
296cms2build + build2rm
297
298=item B<launchvm>
299
300Launch one virtual machine
301
302=item B<launchve>
303
304Launch one virtual environment
305
306=item B<script2vm>
307
308Launch one virtual machine if needed
309and executes a script on it
310
311=item B<script2ve>
312
313Execute a script in a virtual environment
314
315=item B<script2rm>
316
317Execute a script on a remote machine
318
319=item B<newvm>
320
321Create a new virtual machine
322
323=item B<newve>
324
325Create a new virtual environment
326
327=item B<setupvm>
328
329Setup a virtual machine for pb usage
330
331=item B<setupve>
332
333Setup a virtual environment for pb usage
334
335=item B<setuprm>
336
337Setup a remote machine for pb usage
338
339=item B<sbx2setupvm>
340
341Setup a virtual machine for pb usage using the sandbox version of pb instead of the latest stable
342Reserved to dev team.
343
344=item B<sbx2setupve>
345
346Setup a virtual environment for pb usage using the sandbox version of pb instead of the latest stable
347Reserved to dev team.
348
349=item B<sbx2setuprm>
350
351Setup a remote machine for pb usage using the sandbox version of pb instead of the latest stable
352Reserved to dev team.
353
354=item B<build2setupvm>
355
356Setup a virtual machine for pb usage using the build available
357Reserved to dev team.
358
359=item B<build2setupve>
360
361Setup a virtual environment for pb usage using the build available
362Reserved to dev team.
363
364=item B<build2setuprm>
365
366Setup a remote machine for pb usage using the build available
367Reserved to dev team.
368
369=item B<snapvm>
370
371Snapshot a virtual machine for pb usage
372
373=item B<snapve>
374
375Snapshot a virtual environment for pb usage
376
377=item B<updateconf>
378
379Update the configuration files to the latest format
380
381=item B<updatevm>
382
383Update the distribution in the virtual machine
384
385=item B<updateve>
386
387Update the distribution in the virtual environment
388
389=item B<updaterm>
390
391Update the distribution in the remote machine
392
393=item B<test2pkg>
394
395Test a package locally
396
397=item B<test2vm>
398
399Test a package in a virtual machine
400
401=item B<test2ve>
402
403Test a package in a virtual environment
404
405=item B<test2rm>
406
407Test a package in a remote machine
408
409=item B<checkssh>
410
411Check the delivery of the packages on the repository
412
413=item B<checkps>
414
415Check the process running the VM concerned
416
417=item B<newver>
418
419Create a new version of the project derived
420from the current one
421
422=item B<newproj>
423
424Create a new project and a template set of
425configuration files under pbconf
426
427=item B<announce>
428
429Announce the availability of the project through various means
430
431=item B<sbx2webssh>
432
433Create tar files for the website under your CMS.
434Current state of the exported content is taken.
435Deliver the content to the target server using ssh from the exported dir.
436
437=item B<cms2webssh>
438
439Create tar files for the website from your CMS.
440Deliver the content to the target server using ssh from the DVCS.
441
442=item B<sbx2webpkg>
443
444Create tar files for the website under your CMS.
445Current state of the exported content is taken.
446
447=item B<cms2webpkg>
448
449Create tar files for the website under your CMS.
450
451=item B<getconf>
452
453Print the full configuration parameters as found in the various configuration files. Help to debug conf issues.
454Also accepts a parameter to display only this value, and a VM/VE/RM
455
456=item B<getvar>
457
458Print the full variables expanded based on the distribution tuple. Help to debug conf issues.
459Also accepts a parameter to display only the values for this package, and a VM/VE/RM
460
461=item B<clean>
462
463Purge the build and delivery directories related to the current project
464
465=item B<cleanssh>
466
467Purge the ssh server of its packages (only for testver and test packages)
468
469=back
470
471<pkgs> can be a list of packages, the keyword 'all' or nothing, in which case the default list of packages is taken (corresponding to the defpkgdir list of arguments in the configuration file).
472
473=head1 WEB SITES
474
475The main Web site of the project is available at L<http://www.project-builder.org/>. Bug reports should be filled using the trac instance of the project at L<http://trac.project-builder.org/>.
476
477=head1 USER MAILING LIST
478
479None exists for the moment.
480
481=head1 CONFIGURATION FILES
482
483Each pb user may have a configuration in F<$HOME/.pbrc.yml>. The values in this file may overwrite any other configuration file value.
484
485Here is an example of such a configuration file:
486
487 #
488 # Define for each project the URL of its pbconf repository
489 # No default option allowed here as they need to be all different
490 #
491 # URL of the pbconf content
492 # This is the format of a classical URL with the extension of additional schema such as
493 # svn+ssh, cvs+ssh, ...
494 #
495  pbconfurl:
496    linuxcoe : cvs+ssh://:ext:bcornec@linuxcoe.cvs.sourceforge.net:/cvsroot/linuxcoe/pbconf
497
498 # This is normaly defined in the project's configuration file
499 # Url of the project
500 #
501  pbprojurl:
502    linuxcoe : cvs+ssh://:ext:bcornec@linuxcoe.cvs.sourceforge.net:/cvsroot/linuxcoe
503 
504 # All these URLs needs to be defined here as the are the entry point
505 # for how to build packages for the project
506 #
507  pbconfurl:
508    pb: svn+ssh://svn.project-builder.org/mondo/svn/pb/pbconf
509  pbconfurl:
510    mondorescue: svn+ssh://svn.project-builder.org/mondo/svn/project-builder/mondorescue/pbconf
511  pbconfurl:
512    collectl: svn+ssh://bruno@svn.mondorescue.org/mondo/svn/project-builder/collectl/pbconf
513  pbconfurl:
514    netperf: svn+ssh://svn.mondorescue.org/mondo/svn/project-builder/netperf/pbconf
515 
516 # Under that dir will take place everything related to pb
517 # If you want to use VMs/chroot/..., then use $ENV{'HOME'} to make it portable
518 # to your VMs/chroot/...
519 # if not defined then /var/cache
520  pbdefdir:
521    default: $ENV{'HOME'}/project-builder
522  pbdefdir:
523    pb: $ENV{'HOME'}
524  pbdefdir:
525    linuxcoe: $ENV{'HOME'}/LinuxCOE/cvs
526  pbdefdir:
527    mondorescue: $ENV{'HOME'}/mondo/svn
528 
529 # pbconfdir points to the directory where the CMS content of the pbconfurl is checked out
530 # If not defined, pbconfdir is under pbdefdir/pbproj/pbconf
531  pbconfdir:
532    linuxcoe: $ENV{'HOME'}/LinuxCOE/cvs/pbconf
533  pbconfdir:
534    mondorescue: $ENV{'HOME'}/mondo/svn/pbconf
535 
536 # pbdir points to the directory where the CMS content of the pbprojurl is checked out
537 # If not defined, pbdir is under pbdefdir/pbproj
538 # Only defined if we have access to the dev of the project
539  pbdir:
540    linuxcoe: $ENV{'HOME'}/LinuxCOE/cvs
541  pbdir:
542    mondorescue: $ENV{'HOME'}/mondo/svn
543 
544 # -daemonize doesn't work with qemu 0.8.2
545  vmopt:
546    default: -m 384
547
548=head1 COMMAND DETAILS
549
550=head2 newproj
551
552The newproj command creates a new project-builder project.  To run this command you first need to define two variables in your ~/.pbrc.yml file:
553
554 pbconfurl I<project> = file:///home/anderse/.git/project-builder-config/I<project>
555 pbdefdir default = $ENV{'HOME'}/cache-project-builder
556
557The first line defines the version controlled configuration information and the second defines the root directory for project-builder to use.
558
559You can then run the command:
560
561 % pb -p I<$project> -r I<$version> newproj I<$pkg>
562
563to create the new project. Running the newproj command will then generate the file $pbdefdir/$project/pbconf/$version/$project.yml, and the directory $pbdefdir/$project/pbconf/$version/$pkg. You will need to edit those files to make the later commands work.
564
565=head2 cms2build
566
567The cms2build command takes your files from the content management system and makes the two tar files that are necessary for building files. You need to have run the newproj command first.  Then there are several steps for running this command:
568
569=over 4
570
571=item Update your $project.yml configuration file.
572
573You need to set the pbprojurl, pbrepo, pbwf, pbpackager, projver, projtag, testver, delivery, and defpkgdir lines as described in the configuration file. The pbprojurl entry is used to find the source for your package. The pbrepo entry is used to build the .repo or .sources.list files for use by downloaders of the package. The pbwf entry indicates that the source tar file is named by I<package-name>-I<version>. The pbpackager entry will be stored in the packages and should be you or your team. The projver/projtag entries indicate the version of the software and the version of the packaging scripts. The testver entry when true indicates that the package is in a test version, so no log file is computed (can be long), and version is made up using a timstamp. The delivery entry gives the subdirectory under which the packages will be delivered on the repository, and the defpkgdir entry corresponds to the local subdirectory hosting the package content.
574
575For example:
576
577 pbprojurl Lintel = file:///home/anderse/projects/Lintel-0.2012.02.28.tar.gz
578 pbrepo Lintel = http://tesla.hpl.hp.com/opensource
579 pbwf Lintel = 1
580 pbpackager Lintel = Eric Anderson <eric.anderson4@hp.com>
581 projver Lintel = 0.2012.02.28
582 projtag Lintel = 1
583 testver Lintel = false
584 delivery Lintel = production
585 defpkgdir Lintel = Lintel-0.2012.02.28
586
587=item Create the build .tar.gz files:
588
589Then you need to take those files and create the initial tar files.  Run a command like:
590
591 % pb -p $project -r $version cms2build
592
593To create the $pbdefdir/$project/pbdelivery/$project-$version.{,pbconf}.tar.gz files, the $version-$projtag.yml and pbrc.yml files in the same directory.
594
595=back
596
597=head2 build2pkg
598
599The build2pkg command takes the tar files created in the cms2build step and attempts to build binary packages for your current operating system. There are two steps:
600
601=over 4
602
603=item Update your filters and build files.
604
605You probably need to edit the files describing the build steps in one of the $pbdefdir/$project/pbconf/$version/$project/{deb,rpm,pkg} directories and the filters in $pbdefdir/$project/pbconf/$version/pbfilter. Note that you can define additional filters and transformations in the filters files. The build files will be filtered by the filters defined in the filters files to generate the inputs to the build step. Therefore, if you change those files, you need to re-run the cms2build step.
606
607=item Build the package.
608
609Then you can run a command like:
610
611 % pb -p $project -r $version build2pkg
612
613To create the files in $project/pbbuild that comprise your binary package(s).
614
615=back
616
617=head2 newve
618 
619The newve command creates a new virtual environment, i.e. a chrooted OS for building packages. Using a virtual environment is an efficient way to build packages on a related set of operating systems. The OS's have to be related because the kernel will be shared.  Steps:
620
621=over 4
622
623=item Update ~/.pbrc.yml
624
625Update your ~/.pbrc.yml file to specify the vepath, velist, velogin, and vetype variables, e.g.:
626
627  vepath:
628    default: $ENV{'HOME'}/cache-project-builder/chroot
629  velist:
630    default: debian-6.0-i386
631  velogin:
632    default: pb
633  vetype:
634    default: chroot
635
636If you are building for rpm style OS's, update the verpmtype option, and install the appropriate tool.
637
638  verpmtype:
639    default: rpmbootstrap
640
641You may also choose to specify a mirror for the OS packages, and optionally http/ftp proxies.  You can specify the proxies either through environment variables ($http_proxy/$https_proxy/$ftp_proxy) or in the configuration file. The configuration file will be used if no corresponding environment variable has been set. For example, for debian and with a local squid proxy:
642
643  rbsmirrorsrv:
644    debian: http://mirrors1.kernel.org/debian/
645  http_proxy:
646    default: http://localhost:3128/
647  https_proxy:
648    default: http://localhost:3128/
649  ftp_proxy:
650    default: http://localhost:3128/
651
652=item Run the cms2build command
653
654If you have deleted your $package/pbdelivery directory, re-run the cms2build command as in the earlier step. This step is necessary to generate the I<package>/pbdelivery/pbrc.yml file.
655
656=item Create the new virtual environment
657
658Initialize the new operating system. This step will install the core OS packages for the virtual environment, e.g.:
659
660 % pb -v -p $project -m debian-6.0-i386 newve
661
662=back
663
664=head2 setupve
665
666The setupve command prepares a virtual environment for use by project builder. In particular it installs project-builder from the packages into the virtual environment. Two sub-steps are necessary:
667
668=over 4
669
670=item Update $project.yml
671
672You need to have a sshhost entry for setupve to work, so add one, even an invalid one, e.g.:
673
674  sshhost:
675    $project: foo.example.org
676
677=item Setup the virtual environment
678
679 % pb -v -p $project -m debian-6.0-i386 setupve
680
681If you prefer to install the current SVN version of project builder, you can substitute the setupve option by the sbx2setupv one.
682
683=back
684
685=head2 build2ve
686
687The build2ve command is similar to the build2pkg command in that it will take the sources created by cms2build and turn them into binary packages. The command has two differences. First, it creates the packages in a virtual environment, i.e. the one made by an earlier setupve setup. Second it copies the resulting packages to a repository and builds the repository meta-data needed.
688
689Three sub-steps are needed:
690
691=over 4
692
693=item Update $project.yml
694
695You need to have a valid sshdir and sshhost entry for build2ve to work, so add them. Note that you need to be able to ssh from the host you run the command on to the repository host, preferably without needing to type in a password, so using ssh-agent or having a special passwordless project-builder ssh key will make this step easier.
696
697  sshhost:
698    $project: localhost
699  sshdir:
700    $project: $home/cache-project-builder/repos
701
702You may also need to specify additional repository files to use or rpms to install. Note the URL for repositories is not the URL of the repository, but the URL of a file that can be put in the yum.repos.d or apt.sources.d directory.
703
704  addrepo:
705    centos-5-i386: http://localhost/pb/centos-extras.repo,http://mirror.centos.org/centos/5/extras/i386/RPMS/chrpath-0.13-3.el5.centos.i386.rpm
706
707=item Update your filters and build files
708
709You may need to update your filter files as in the build2pkg step if you are building for a new OS or architecture.
710
711=item Build the packages and copy them to the repository
712
713 % pb -v -p $project -m debian-6.0-i386 build2ve
714
715=back
716
717*Debugging:* If the build fails (and you did not specify the --no-stop-on-error) option, then the virtual environment and scripts should still be present and configured to build the package. You can run a command like 'sudo setarch i386 chroot $path bash' in order to get into the environment. In your log you should see a command like that. From there you can go into the /home/pb directory as the pb user and run the same style of pb commands as you did when doing build2pkg. This will help you figure out what has gone wrong in the build in the virtual environment.
718
719=head1 AUTHORS
720
721The Project-Builder.org team L<http://trac.project-builder.org/> lead by Bruno Cornec L<mailto:bruno@project-builder.org>.
722
723=head1 COPYRIGHT
724
725Project-Builder.org is distributed under the GPL v2.0 license
726described in the file C<COPYING> included with the distribution.
727
728=cut
729
730# ---------------------------------------------------------------------------
731
732my ($projectbuilderver,$projectbuilderrev,$projectbuilderconfver) = pb_version_init();
733my $appname = "pb";
734
735# Initialize the syntax string
736
737pb_syntax_init("$appname (aka project-builder.org) Version $projectbuilderver-$projectbuilderrev Conf $projectbuilderconfver\n");
738
739GetOptions("help|?|h+" => \$opts{'h'}, 
740        "account|a=s" => \$opts{'a'},
741        "force|f" => \$opts{'f'},
742        "nographic|g" => \$opts{'g'},
743        "image|i=s" => \$opts{'i'},
744        "log-files|l=s" => \$opts{'l'},
745        "machines|mock|m=s" => \$opts{'m'},
746        "man" => \$opts{'man'},
747        "port|P=i" => \$opts{'P'},
748        "project|p=s" => \$opts{'p'},
749        "quiet|q" => \$opts{'q'},
750        "revision|r=s" => \$opts{'r'},
751        "rebuild" => \$opts{'rebuild'},
752        "snapshot|S" => \$opts{'S'},
753        "script|s=s" => \$opts{'s'},
754        "type|T:s" => \$opts{'T'},
755        "target|t:s" => \$opts{'t'},
756        "version" => \$opts{'version'},
757        "V=s" => \$opts{'V'},
758        "verbose|v+" => \$opts{'v'},
759        "keep|k" => \$opts{'k'},
760        "stop-on-error!" => \$Global::pb_stop_on_error,
761) || pb_syntax(-1,0);
762
763if (defined $opts{'l'}) {
764    open(pbLOG,"> $opts{'l'}") || die "Unable to log to $opts{'l'}: $!";
765    $pbLOG = \*pbLOG;
766    $pbdebug = 0  if ($pbdebug == -1);
767    }
768pb_log_init($pbdebug, $pbLOG);
769
770if (defined $opts{'h'}) {
771    pb_syntax(0,1);
772}
773if (defined $opts{'version'}) {
774    pb_syntax(0,0);
775}
776if (defined $opts{'man'}) {
777    pb_syntax(0,2);
778}
779if (defined $opts{'v'}) {
780    $pbdebug = $opts{'v'};
781}
782if (defined $opts{'f'}) {
783    $pbforce=1;
784}
785if (defined $opts{'q'}) {
786    $pbdebug=-1;
787}
788if (defined $opts{'S'}) {
789    $pbsnap=1;
790}
791if (defined $opts{'k'}) {
792    $pbkeep=1;
793}
794pb_display_init("text","");
795
796# Handle root of the project if defined
797if (defined $opts{'r'}) {
798    $ENV{'PBROOTDIR'} = $opts{'r'};
799}
800# Handle virtual machines if any
801if (defined $opts{'m'}) {
802    $ENV{'PBV'} = $opts{'m'};
803}
804if (defined $opts{'s'}) {
805    $pbscript{'default'} = $opts{'s'};
806    die "option -s requires a script having a full path name" if ($pbscript{'default'} !~ /^\//);
807}
808if (defined $opts{'a'}) {
809    $ENV{'PBACCOUNT'} = $opts{'a'};
810    die "option -a requires a -s script option" if (not defined $pbscript{'default'});
811}
812if (defined $opts{'P'}) {
813    $pbport = $opts{'P'};
814}
815if (defined $opts{'V'}) {
816    $newver = $opts{'V'};
817}
818if (defined $opts{'i'}) {
819    $pbimage = $opts{'i'};
820}
821if (defined $opts{'t'}) {
822    $pbtarget = $opts{'t'};
823}
824if (defined $opts{'T'}) {
825    $vetype = $opts{'T'};
826}
827
828# Get Action
829$action = shift @ARGV;
830die pb_syntax(-1,1) if (not defined $action);
831
832my ($filteredfiles, $supfiles, $defpkgdir, $extpkgdir);
833my $pbinit = undef;
834$pbinit = 1 if ($action =~ /^newproj$/);
835
836# Handles project name if any
837# And get global params
838($filteredfiles, $supfiles, $defpkgdir, $extpkgdir) = pb_env_init($opts{'p'},$pbinit,$action,$pbkeep);
839
840#
841# Check for command requirements
842#
843my ($req,$opt,$pbpara,$pbshowsudo,$pbstoponerr) = pb_conf_get_if("oscmd","oscmdopt","pbparallel","pbshowsudo","pbstoponerr");
844pb_check_requirements($req,$opt,$appname);
845
846$Global::pb_show_sudo = 1 if ((defined $pbshowsudo) && (defined $pbshowsudo->{$ENV{'PBPROJ'}}) && ($pbshowsudo->{$ENV{'PBPROJ'}} =~ /true/oi));
847$Global::pb_stop_on_error = 1 if ((defined $pbstoponerr) && (defined $pbstoponerr->{$ENV{'PBPROJ'}}) && ($pbstoponerr->{$ENV{'PBPROJ'}} =~ /true/oi));
848
849#
850# Check if we can launch some actions in // with Parallel::ForkManager
851#
852my $pbparallel = $pbpara->{$appname} if (defined $pbpara);
853if (not defined $pbparallel) {
854    eval
855    {
856        require Sys::CPU;
857        Sys::CPU->import();
858    };
859    if ($@) {
860        # Sys::CPU not found, defaulting to 1
861        pb_log(1,"ADVISE: Install Sys::CPU to benefit from automatic parallelism optimization.\nOr use pbparallel in your pb.yml file\nOnly 1 process at a time for the moment\n");
862        $pbparallel = 1;
863    } else {
864        # Using the number of cores
865        $pbparallel = Sys::CPU::cpu_count();
866        pb_log(1,"Using parallel mode with $pbparallel processes\n");
867    }
868}
869
870eval
871{
872    require Parallel::ForkManager;
873    Parallel::ForkManager->import();
874};
875# Parallel::ForkManager not found so no // actions
876if ($@) {
877    $pbparallel = undef;
878    pb_log(1,"ADVISE: Install Parallel::ForkManager to benefit from automatic parallelism optimization.\nOnly 1 process at a time for the moment\n");
879}
880
881if ($action ne "getconf") {
882    pb_log(0,"Project: $ENV{'PBPROJ'}\n");
883    pb_log(0,"Action: $action\n");
884}
885
886my $do_install = undef;
887# Possibility to add a 2ins to some commands to launch automatic installation
888$do_install = 1 if ($action =~ /2ins$/);
889
890# Act depending on action
891if ($action =~ /^cms2build$/) {
892    pb_cms2build("CMS");
893} elsif ($action =~ /^sbx2build$/) {
894    pb_cms2build("SandBox");
895} elsif ($action =~ /^build2prep$/) {
896    pb_build2prep();
897} elsif ($action =~ /^sbx2prep$/) {
898    pb_cms2build("SandBox");
899    pb_build2prep();
900} elsif ($action =~ /^build2pkg/) {
901    pb_build2pkg($do_install);
902} elsif ($action =~ /^cms2pkg/) {
903    pb_cms2build("CMS");
904    pb_build2pkg($do_install);
905} elsif ($action =~ /^sbx2pkg/) {
906    pb_cms2build("SandBox");
907    pb_build2pkg($do_install);
908} elsif ($action =~ /^build2ssh$/) {
909    pb_build2ssh();
910} elsif ($action =~ /^cms2ssh$/) {
911    pb_cms2build("CMS");
912    pb_build2ssh();
913} elsif ($action =~ /^sbx2ssh$/) {
914    pb_cms2build("SandBox");
915    pb_build2ssh();
916} elsif ($action =~ /^pkg2ssh$/) {
917    pb_pkg2ssh();
918} elsif ($action =~ /^build2rm$/) {
919    pb_build2v("rm","build");
920} elsif ($action =~ /^build2ve$/) {
921    pb_build2v("ve","build");
922} elsif ($action =~ /^build2vm$/) {
923    pb_build2v("vm","build");
924} elsif ($action =~ /^sbx2preprm$/) {
925    pb_cms2build("SandBox");
926    pb_build2v("rm","prep");
927} elsif ($action =~ /^sbx2prepve$/) {
928    pb_cms2build("SandBox");
929    pb_build2v("ve","prep");
930} elsif ($action =~ /^sbx2prepvm$/) {
931    pb_cms2build("SandBox");
932    pb_build2v("vm","prep");
933} elsif ($action =~ /^preprm$/) {
934    pb_build2v("rm","prep");
935} elsif ($action =~ /^prepve$/) {
936    pb_build2v("ve","prep");
937} elsif ($action =~ /^prepvm$/) {
938    pb_build2v("vm","prep");
939} elsif ($action =~ /^cms2rm$/) {
940    pb_cms2build("CMS");
941    pb_build2v("rm","build");
942} elsif ($action =~ /^cms2ve$/) {
943    pb_cms2build("CMS");
944    pb_build2v("ve","build");
945} elsif ($action =~ /^sbx2rm$/) {
946    pb_cms2build("SandBox");
947    pb_build2v("rm","build");
948} elsif ($action =~ /^sbx2ve$/) {
949    pb_cms2build("SandBox");
950    pb_build2v("ve","build");
951} elsif ($action =~ /^cms2vm$/) {
952    pb_cms2build("CMS");
953    pb_build2v("vm","build");
954} elsif ($action =~ /^sbx2vm$/) {
955    pb_cms2build("SandBox");
956    pb_build2v("vm","build");
957} elsif ($action =~ /^launchvm$/) {
958    pb_parallel_launchv(undef,"vm",undef,3,$pbimage);
959} elsif ($action =~ /^launchve$/) {
960    pb_parallel_launchv(undef,"ve",undef,3,$pbimage);
961} elsif ($action =~ /^script2vm$/) {
962    die "action script2vm requires a -s script option" if (not defined $pbscript{'default'});
963    pb_script2v(\%pbscript,"vm");
964} elsif ($action =~ /^script2ve$/) {
965    die "action script2ve requires a -s script option" if (not defined $pbscript{'default'});
966    pb_script2v(\%pbscript,"ve");
967} elsif ($action =~ /^script2rm$/) {
968    die "action script2rm requires a -s script option" if (not defined $pbscript{'default'});
969    pb_script2v(\%pbscript,"rm");
970} elsif ($action =~ /^newver$/) {
971    pb_newver();
972} elsif ($action =~ /^sbx2docker$/) {
973    my $savproj = $ENV{'PBPROJ'};
974    my $pkg;
975    my @pkg;
976    if ($#ARGV == -1) {
977        $pkg = pb_cms_get_pkg($defpkgdir,$extpkgdir);
978        @pkgs = @$pkg;
979    } else {
980        @pkgs = @ARGV;
981    }
982    #pb_log(1,"Packages to print:".Dumper(@pkgs)."\n");
983   
984    # Get the list of all VE we need to work on
985    my ($vm,$all);
986    if (not defined $ENV{'PBV'}) {
987        ($vm,$all) = pb_get2v("ve");
988    } else {
989        @$vm = split(/,/,$ENV{'PBV'});
990    }
991   
992    # TODO: parallelize
993    foreach my $v (@$vm) {
994        my $done = 0;
995        my $pbos = pb_distro_get_context($v);
996        # Docker images are (for now !) regular and use distro:ver
997        $pbimage = "$pbos->{'name'}".":$pbos->{'version'}";
998        # the first steps needs to be done with the pb project
999        $ENV{'PBPROJ'} = $appname;
1000        my $found = pb_ve_docker_get_image("$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}");
1001        if (($pbforce == 1) || (not defined $found)) {
1002            # pb -p pb -m distro-ver-arch newve -i distro:ver
1003            pb_log(0,"Docker container $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'} not found, creating it\n");
1004            pb_parallel_launchv(undef,"ve",undef,0,$pbimage);
1005        }
1006        $found = pb_ve_docker_get_image("$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-pb");
1007        if (($pbforce == 1) || (not defined $found)) {
1008            pb_log(0,"Docker container $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-pb not found, creating it\n");
1009            # pb -p pb -m distro-ver-arch setupve
1010            if ($savproj eq "pb") {
1011                pb_cms2build("SandBox");
1012                $done = 1;
1013                pb_setup2v("ve","SandBox");
1014            } else {
1015                pb_setup2v("ve");
1016            }
1017        }
1018        # Now back to our original project
1019        $ENV{'PBPROJ'} = $savproj;
1020        $found = pb_ve_docker_get_image("$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-pb-$ENV{'PBPROJ'}");
1021        if (($pbforce == 1) || (not defined $found)) {
1022            pb_log(0,"Docker container $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-pb-$ENV{'PBPROJ'} not found, creating it\n");
1023            # pb -p pb -m distro-ver-arch prepve
1024            if ($savproj eq "pb") {
1025                pb_cms2build("SandBox");
1026                $done = 1;
1027            }
1028            pb_build2v("ve","prep");
1029        }
1030        pb_log(0,"Instantiating a Docker container from $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-pb-$ENV{'PBPROJ'}\n");
1031        if (($savproj ne "pb") || ($done == 0)) {
1032            pb_cms2build("SandBox");
1033        }
1034        pb_build2v("ve","build");
1035    }
1036} elsif ($action =~ /^newve$/) {
1037    pb_parallel_launchv(undef,"ve",undef,0,$pbimage);
1038} elsif ($action =~ /^newvm$/) {
1039    pb_parallel_launchv(undef,"vm",undef,0,$pbimage);
1040    pb_log(0, "Please ensure that sshd is running in your VM by default\n");
1041    pb_log(0, "and that it allows remote root login (PermitRootLogin yes in /etc/ssh/sshd_config)\n");
1042    pb_log(0, "Also ensure that network is up, firewalling correctly configured\n");
1043    pb_log(0, "and perl, Data::Dumper, tar, sudo, ntpdate and scp/ssh installed\n");
1044    pb_log(0, "You should then be able to login with ssh -p VMPORT root\@localhost (if VM started with pb)\n");
1045} elsif ($action =~ /^setuprm$/) {
1046    pb_setup2v("rm");
1047} elsif ($action =~ /^setupve$/) {
1048    pb_setup2v("ve");
1049} elsif ($action =~ /^setupvm$/) {
1050    pb_setup2v("vm");
1051} elsif ($action =~ /^sbx2setuprm$/) {
1052    die "This feature is limited to the pb project" if ($ENV{'PBPROJ'} ne $appname);
1053    pb_cms2build("SandBox");
1054    pb_setup2v("rm","SandBox");
1055} elsif ($action =~ /^sbx2setupve$/) {
1056    die "This feature is limited to the pb project" if ($ENV{'PBPROJ'} ne $appname);
1057    pb_cms2build("SandBox");
1058    pb_setup2v("ve","SandBox");
1059} elsif ($action =~ /^sbx2setupvm$/) {
1060    die "This feature is limited to the pb project" if ($ENV{'PBPROJ'} ne $appname);
1061    pb_cms2build("SandBox");
1062    pb_setup2v("vm","SandBox");
1063} elsif ($action =~ /^build2setuprm$/) {
1064    die "This feature is limited to the pb project" if ($ENV{'PBPROJ'} ne $appname);
1065    pb_setup2v("rm","SandBox");
1066} elsif ($action =~ /^build2setupve$/) {
1067    die "This feature is limited to the pb project" if ($ENV{'PBPROJ'} ne $appname);
1068    pb_setup2v("ve","SandBox");
1069} elsif ($action =~ /^build2setupvm$/) {
1070    die "This feature is limited to the pb project" if ($ENV{'PBPROJ'} ne $appname);
1071    pb_setup2v("vm","SandBox");
1072} elsif ($action =~ /^updaterm$/) {
1073    pb_update2v("rm");
1074} elsif ($action =~ /^updateve$/) {
1075    pb_update2v("ve");
1076} elsif ($action =~ /^updatevm$/) {
1077    pb_update2v("vm");
1078} elsif ($action =~ /^snapve$/) {
1079    pb_snap2v("ve");
1080} elsif ($action =~ /^snapvm$/) {
1081    pb_snap2v("vm");
1082} elsif ($action =~ /^test2pkg$/) {
1083    pb_test2pkg();
1084} elsif ($action =~ /^test2rm$/) {
1085    pb_build2v("rm","test");
1086} elsif ($action =~ /^test2ve$/) {
1087    pb_build2v("ve","test");
1088} elsif ($action =~ /^test2vm$/) {
1089    pb_build2v("vm","test");
1090} elsif ($action =~ /^newproj$/) {
1091    # Nothing to do - already done in pb_env_init
1092} elsif ($action =~ /^updateconf$/) {
1093    foreach my $f (@ARGV) {
1094        my $fyml = $f;
1095        $fyml =~ s/\.pbf/.yml/;
1096        $fyml =~ s/\.pb/.yml/;
1097        $fyml =~ s/\.pbrc/.pbrc.yml/;
1098        pb_vcs_conf_update_v0($f,$fyml);
1099    }
1100} elsif ($action =~ /^getconf$/) {
1101    my $pbos = pb_distro_get_context($ENV{'PBV'});
1102    pb_log(1,"Arguments to print:".Dumper(@ARGV)."\n");
1103    pb_distro_conf_print($pbos,@ARGV);
1104} elsif ($action =~ /^getvar$/) {
1105    my $pbos = pb_distro_get_context($ENV{'PBV'});
1106    my $pkg;
1107    my @pkg;
1108    if ($#ARGV == -1) {
1109        $pkg = pb_cms_get_pkg($defpkgdir,$extpkgdir);
1110        @pkgs = @$pkg;
1111    } else {
1112        @pkgs = @ARGV;
1113    }
1114    pb_log(1,"Packages to print:".Dumper(@pkgs)."\n");
1115    pb_filter_var_print($pbos,@pkgs);
1116} elsif ($action =~ /^clean$/) {
1117    pb_clean();
1118} elsif ($action =~ /^announce$/) {
1119    # For announce only. Require avoids the systematic load of these modules
1120    require DBI;
1121    require DBD::SQLite;
1122
1123    pb_announce("Announce");
1124} elsif ($action =~ /^cleanssh$/) {
1125    pb_announce("Clean");
1126} elsif ($action =~ /^checkssh$/) {
1127    pb_announce("Check");
1128} elsif ($action =~ /^checkps$/) {
1129    my ($ptr2) = pb_conf_get("vmcmd");
1130    my $vmcmd = $ptr2->{$ENV{'PBPROJ'}};
1131    my ($vexist,$vmmport) = pb_check_ps($vmcmd,$ENV{'PBV'});
1132    if (! $vexist) {
1133        pb_log(0,"No VM found for $ENV{'PBV'}\n");
1134    } else {
1135        pb_log(0,"Found an existing VM for $ENV{'PBV'} PID: $vexist - SSH port: $vmmport\n");
1136    }
1137} elsif ($action =~ /^sbx2webpkg$/) {
1138    require DBI;
1139    require DBD::SQLite;
1140
1141    pb_cms2build("SandBox","Web");
1142} elsif ($action =~ /^sbx2webssh$/) {
1143    require DBI;
1144    require DBD::SQLite;
1145
1146    pb_cms2build("SandBox","Web");
1147    pb_send2target("Web");
1148} elsif ($action =~ /^cms2webpkg$/) {
1149    require DBI;
1150    require DBD::SQLite;
1151
1152    pb_cms2build("CMS","Web");
1153} elsif ($action =~ /^cms2webssh$/) {
1154    require DBI;
1155    require DBD::SQLite;
1156
1157    pb_cms2build("CMS","Web");
1158    pb_send2target("Web");
1159} else {
1160    pb_log(0,"\'$action\' is not available\n");
1161    pb_syntax(-2,1);
1162}
1163
1164pb_exit();
1165
1166sub pb_cms2build {
1167
1168    my $param = shift;
1169    my $web = shift;
1170
1171    my $pkg;
1172    my @pkgs;
1173    my $webdir;
1174
1175    my %pkgs;
1176    my $pb;             # Structure to store conf info
1177
1178    die "pb_cms2build requires a parameter: SandBox or CMS" if (not defined $param);
1179
1180    # If Website, then pkg is only the website
1181    if (defined $web) {
1182        ($webdir) = pb_conf_get("webdir");
1183        pb_log(2,"webdir: ".Dumper($webdir)."\n");
1184        $pkgs[0] = $webdir->{$ENV{'PBPROJ'}};
1185        $extpkgdir = $webdir;
1186        pb_log(0,"Package: $pkgs[0]\n");
1187    } else {
1188        $pkg = pb_cms_get_pkg($defpkgdir,$extpkgdir);
1189        @pkgs = @$pkg;
1190    }
1191
1192    my ($scheme, $uri) = pb_vcs_init($pbinit,$param);
1193
1194    # We need 2 lines here
1195    my ($pkgv, $pkgt, $testver) = pb_conf_get_if("pkgver","pkgtag","testver");
1196
1197    # declare packager and repo for filtering
1198    my ($tmp1, $tmp2) = pb_conf_get("pbpackager","pbrepo");
1199    $ENV{'PBPACKAGER'} = $tmp1->{$ENV{'PBPROJ'}};
1200    $ENV{'PBREPO'} = $tmp2->{$ENV{'PBPROJ'}};
1201    my ($delivery) = pb_conf_get_if("delivery");
1202    $delivery->{$ENV{'PBPROJ'}} = "" if (not defined $delivery->{$ENV{'PBPROJ'}});
1203
1204    # If we deal with a test dir, we want to keep the date in tar file and dir name
1205    my $pbextdirpkg = "";
1206    my $pbextdir = "";
1207    if ((defined $testver) && (defined $testver->{$ENV{'PBPROJ'}}) && ($testver->{$ENV{'PBPROJ'}} =~ /true/i)) {
1208        $pbextdirpkg = strftime("%Y%m%d%H%M%S", @date);
1209        $pbextdir = ".0.".$pbextdirpkg;
1210    }
1211
1212    foreach my $pbpkg (@pkgs) {
1213        $ENV{'PBPKG'} = $pbpkg;
1214        $pbrev = $ENV{'PBREVISION'};
1215
1216        if ((defined $pkgv) && (defined $pkgv->{$pbpkg})) {
1217            $pbver = $pkgv->{$pbpkg};
1218        } else {
1219            $pbver = $ENV{'PBPROJVER'};
1220        }
1221        # If it's a test version, then build a tag to fit distributions need
1222        if ((defined $testver) && (defined $testver->{$ENV{'PBPROJ'}}) && ($testver->{$ENV{'PBPROJ'}} =~ /true/i)) {
1223            my $letter = "";
1224            if ($scheme =~ /svn/) {
1225                $letter = "s";
1226            } elsif ($scheme =~ /cvs/) {
1227                $letter = "c";
1228            } elsif ($scheme =~ /svk/) {
1229                $letter = "k";
1230            } elsif ($scheme =~ /hg/) {
1231                $letter = "h";
1232            } elsif ($scheme =~ /git/) {
1233                $letter = "g";
1234            } else {
1235                $letter = "f";
1236            }
1237            $pbtag = "0.$pbextdirpkg.$letter$pbrev";
1238            $ENV{'PBPROJTAG'} = $pbtag;
1239        } elsif ((defined $pkgt) && (defined $pkgt->{$pbpkg})) {
1240            $pbtag = $pkgt->{$pbpkg};
1241        } else {
1242            $pbtag = $ENV{'PBPROJTAG'};
1243        }
1244
1245        pb_log(0,"\n");
1246        pb_log(0,"Management of $pbpkg $pbver-$pbtag\n");
1247        die "Unable to get env var PBDESTDIR" if (not defined $ENV{'PBDESTDIR'});
1248
1249        my $dest = "$ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir";
1250        # Create the structure if needed
1251        pb_mkdir_p($dest);
1252        # Clean up dest if necessary. The export will recreate it
1253        pb_rm_rf($dest) if (-d $dest);
1254
1255        # Export CMS tree for the concerned package to dest
1256        # And generate some additional files
1257        $OUTPUT_AUTOFLUSH=1;
1258
1259        # computes in which dir we have to work
1260        my $dir = $defpkgdir->{$pbpkg};
1261        $dir = $extpkgdir->{$pbpkg} if (not defined $dir);
1262        $dir = $webdir->{$ENV{'PBPROJ'}} if (defined $web);
1263        die "Variable \$dir not defined. Check the package name first.\nIf this is the one expected, please report to dev team with log of a verbose run and this info ".Dumper($defpkgdir,$extpkgdir,$webdir) if (not defined $dir);
1264        pb_log(2,"def:".Dumper($defpkgdir)."ext: ".Dumper($extpkgdir)."dir: $dir\n");
1265
1266        # Exporting content from CMS
1267        my $sourcedir = undef;
1268        my $sourceuri = $uri;
1269        if ($param eq "SandBox") {
1270            # Point to the local instance
1271            $sourcedir = "$ENV{'PBDIR'}/$dir";
1272        } else {
1273            # Get it from a subdir of the URI with same version as localy but different root
1274            # Only if using a real CMS
1275            my ($scheme, $account, $host, $port, $path) = pb_get_uri($uri);
1276            if (($scheme !~ /^file/) && ($scheme !~ /^(ht|f)tp/)) {
1277                $sourceuri = "$ENV{'PBDIR'}/$dir"; 
1278                $sourceuri =~ s|^$ENV{'PBPROJDIR'}/|$uri/|;
1279            }
1280        }
1281        my $preserve = pb_vcs_export($sourceuri,$sourcedir,$dest);
1282
1283        # Generated fake content for test versions to speed up stuff
1284        my $chglog;
1285
1286        # Get project info on authors and log file
1287        # TODO: Make it CMS aware
1288        $chglog = "$ENV{'PBROOTDIR'}/$pbpkg/pbcl";
1289        $chglog = "$ENV{'PBROOTDIR'}/pbcl" if (! -f $chglog);
1290        $chglog = undef if (! -f $chglog);
1291
1292        # TODO: Make it CMS aware
1293        my $authors = "$ENV{'PBROOTDIR'}/$pbpkg/pbauthors";
1294        $authors = "$ENV{'PBROOTDIR'}/pbauthors" if (! -f $authors);
1295        $authors = "/dev/null" if (! -f $authors);
1296
1297        # Extract cms log history and store it
1298        if ((defined $chglog) && (! -f "$dest/NEWS")) {
1299            pb_log(2,"Generating NEWS file from $chglog\n");
1300            copy($chglog,"$dest/NEWS") || die "Unable to create $dest/NEWS";
1301        }
1302        pb_cms_log($scheme,"$ENV{'PBDIR'}/$dir",$dest,$chglog,$authors,$testver);
1303
1304        my %build;
1305        # We want to at least build for the underlying distro
1306        # except if a target was given, in which case we only build for it
1307        # if -t was passed without target then build for the native distro.
1308        my $pbos = pb_distro_get_context($pbtarget);
1309        my $tmpl = pb_get_distros($pbos,$pbtarget);
1310
1311        # Setup $pb structure to allow filtering later on, on files using that structure
1312        $pb->{'tag'} = $pbtag;
1313        $pb->{'rev'} = $pbrev;
1314        $pb->{'ver'} = $pbver;
1315        $pb->{'pkg'} = $pbpkg;
1316        $pb->{'suf'} = $pbos->{'suffix'};
1317        $pb->{'realpkg'} = $pbpkg;
1318        $pb->{'date'} = $pbdate;
1319        $pb->{'defpkgdir'} = $defpkgdir;
1320        $pb->{'extpkgdir'} = $extpkgdir;
1321        $pb->{'chglog'} = $chglog;
1322        $pb->{'extdir'} = $pbextdir;
1323        $pb->{'packager'} = $ENV{'PBPACKAGER'};
1324        $pb->{'proj'} = $ENV{'PBPROJ'};
1325        $pb->{'repo'} = "$ENV{'PBREPO'}/$delivery->{$ENV{'PBPROJ'}}";
1326        $pb->{'patches'} = ();
1327        $pb->{'sources'} = ();
1328   
1329        my $tmpd = "$ENV{'PBTMP'}/cms.$$";
1330        pb_mkdir_p($tmpd) if (defined $pbparallel);
1331
1332        # Get only all.yml filter at this stage for pbinit
1333        my $ptr = pb_get_filters($pbpkg);
1334
1335        # Do not do that for website
1336        if (not defined $web) {
1337            my %virt;
1338            # De-duplicate similar VM/VE/RM
1339            foreach my $d (split(/,/,$tmpl)) {
1340                # skip ill-formatted vms (name-ver-arch)
1341                next if ($d !~ /-/);
1342                $virt{$d} = $d;
1343            }
1344
1345            # Try to use // processing here
1346            my $all_ok = 1;
1347            my $pm = new Parallel::ForkManager($pbparallel) if (defined $pbparallel);
1348            $pm->run_on_finish(sub { my ($pid, $code, $id, $signal, $dump) = @_;
1349                                    $all_ok = 0 unless (($code == 0) && ($signal == 0) && ($dump == 0)); }) if (defined $pbparallel);
1350
1351            pb_log(0,"Preparing delivery ...\n");
1352            foreach my $v (keys %virt) {
1353                $pm->start and next if (defined $pbparallel);
1354
1355                # Distro context
1356                my $pbos = pb_distro_get_context($v);
1357   
1358                $pb->{'pbos'} = $pbos;
1359                $pb->{'suf'} = $pbos->{'suffix'};
1360                pb_log(3,"DEBUG: pb: ".Dumper($pb)."\n");
1361
1362                # Get all filters to apply
1363                $ptr = pb_get_filters($pbpkg,$pbos);
1364   
1365                pb_log(2,"DEBUG Filtering PBDATE => $pbdate, PBTAG => $pbtag, PBVER => $pbver\n");
1366   
1367                # We need to compute the real name of the package
1368                my $pbrealpkg = pb_cms_get_real_pkg($pbpkg,$pbos->{'type'});
1369                $pb->{'realpkg'} = $pbrealpkg;
1370                pb_log(2,"Virtual package $pbpkg has a real package name of $pbrealpkg on $pbos->{'name'}-$pbos->{'version'}\n") if ($pbrealpkg ne $pbpkg);
1371   
1372                # Filter build files from the less precise up to the most with overloading
1373                # Filter all files found, keeping the name, and generating in dest
1374   
1375                # Find all build files first relatively to PBROOTDIR
1376                # Find also all specific files referenced in the .yml conf file
1377                my %bfiles = ();
1378                my %pkgfiles = ();
1379                # Used in Filter.pm by pb_filter_file
1380
1381                $build{$v} = "no";
1382                if (-d "$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'os'}") {
1383                    pb_list_bfiles("$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'os'}",$pbpkg,\%bfiles,\%pkgfiles,$supfiles);
1384                    $build{$v} = "yes";
1385                } 
1386                if (-d "$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'type'}") {
1387                    pb_list_bfiles("$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'type'}",$pbpkg,\%bfiles,\%pkgfiles,$supfiles);
1388                    $build{$v} = "yes";
1389                } 
1390                if (-d "$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'family'}") {
1391                    pb_list_bfiles("$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'family'}",$pbpkg,\%bfiles,\%pkgfiles,$supfiles);
1392                    $build{$v} = "yes";
1393                } 
1394                if (-d "$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'name'}") {
1395                    pb_list_bfiles("$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'name'}",$pbpkg,\%bfiles,\%pkgfiles,$supfiles);
1396                    $build{$v} = "yes";
1397                } 
1398                if (-d "$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'name'}-$pbos->{'version'}") {
1399                    pb_list_bfiles("$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'name'}-$pbos->{'version'}",$pbpkg,\%bfiles,\%pkgfiles,$supfiles);
1400                    $build{$v} = "yes";
1401                } 
1402                if (-d "$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}") {
1403                    pb_list_bfiles("$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}",$pbpkg,\%bfiles,\%pkgfiles,$supfiles);
1404                    $build{$v} = "yes";
1405                }
1406                pb_log(2,"DEBUG($v) bfiles: ".Dumper(\%bfiles)."\n");
1407   
1408                if ($build{$v} ne "no") {
1409                    # Apply now all the filters on all the files concerned
1410                    # destination dir depends on the type of file
1411                    # For patch support
1412                    # TODO: Make it CMS aware
1413                    $pb->{'patches'} = pb_list_sfiles("$ENV{'PBROOTDIR'}/$pbpkg/pbpatch", $pb->{'patches'}, $pbos, "$ENV{'PBROOTDIR'}/$pbpkg/pbextpatch");
1414                    pb_log(2,"DEBUG($v) patches: ".Dumper($pb->{'patches'})."\n");
1415                    # TODO: Make it CMS aware
1416                    $pb->{'sources'} = pb_list_sfiles("$ENV{'PBROOTDIR'}/$pbpkg/pbsrc", $pb->{'sources'}, $pbos, "$ENV{'PBROOTDIR'}/$pbpkg/pbextsrc");
1417                    pb_log(2,"DEBUG($v) sources: ".Dumper($pb->{'sources'})."\n");
1418   
1419                    if (defined $pb->{'patches'}->{$v}) {
1420                        # Filter potential patches (local + remote)
1421                        pb_mkdir_p("$dest/pbconf/$v/pbpatch");
1422                        # If Debian based distribution, then prepare what will be done at build time
1423                        my ($patchcmd,$patchopt);
1424                        if ($pbos->{'type'} eq "deb") {
1425                            ($patchcmd,$patchopt) = pb_distro_get_param($pbos,pb_conf_get_if("ospatchcmd","ospatchopt"));
1426                            open(SCRIPT,"> $dest/pbconf/$v/pbpatch/pbapplypatch") || die "Unable to create $dest/pbconf/$v/pbpatch/pbapplypatch";
1427                            print SCRIPT "#!/bin/bash\n";
1428                            print SCRIPT "set -x\n" if ($pbdebug gt 1);
1429                        }
1430                        foreach my $pf (split(/,/,$pb->{'patches'}->{$v})) {
1431                            my $pp = basename($pf);
1432                            pb_vcs_export($pf,undef,"$dest/pbconf/$v/pbpatch");
1433                            pb_filter_file_inplace($ptr,"$dest/pbconf/$v/pbpatch/$pp",$pb);
1434                            pb_system("gzip -9f $dest/pbconf/$v/pbpatch/$pp","","quiet");
1435                            if ($pbos->{'type'} eq "deb") {
1436                                # If Debian based distribution, then prepare what will be done at build time
1437                                # by applying the patches that will be available under the debian/patches dir
1438                                print SCRIPT "$patchcmd $patchopt \< debian/patches/$pp\n";
1439                            }
1440                        }
1441                        if ($pbos->{'type'} eq "deb") {
1442                            close(SCRIPT);
1443                            chmod 0755,"$dest/pbconf/$v/pbpatch/pbapplypatch";
1444                        }
1445                        #pb_system("cat $dest/pbconf/$v/pbpatch/pbapplypatch","APPLY","verbose");
1446                    }
1447                    if (defined $pb->{'sources'}->{$v}) {
1448                        pb_mkdir_p("$dest/pbconf/$v/pbsrc");
1449                        foreach my $pf (split(/,/,$pb->{'sources'}->{$v})) {
1450                            my $pp = basename($pf);
1451                            pb_vcs_export($pf,undef,"$dest/pbconf/$v/pbsrc");
1452                            pb_filter_file_inplace($ptr,"$dest/pbconf/$v/pbsrc/$pp",$pb);
1453                        }
1454                    }
1455                    # Filter build files at the end, as they depend on patches and sources
1456                    foreach my $f (keys %bfiles) {
1457                        pb_filter_file("$ENV{'PBROOTDIR'}/$bfiles{$f}",$ptr,"$dest/pbconf/$v/$f",$pb);
1458                    }
1459                    foreach my $f (keys %pkgfiles) {
1460                        pb_filter_file("$ENV{'PBROOTDIR'}/$pkgfiles{$f}",$ptr,"$dest/pbconf/$v/$f",$pb);
1461                    }
1462                }
1463
1464                if (defined $pbparallel) {
1465                    # Communicate results back to parent
1466                    my $str = "";
1467                    $str .= "build $v = $build{$v}\n" if (defined $build{$v});
1468                    $str .= "patches $v = $pb->{'patches'}->{$v}\n" if (defined $pb->{'patches'}->{$v});
1469                    $str .= "sources $v = $pb->{'sources'}->{$v}\n" if (defined $pb->{'sources'}->{$v});
1470                    pb_set_content("$tmpd/$$","$str");
1471                    $pm->finish;
1472                }
1473            }
1474            # In the parent, we need to get the result from the children
1475            $pm->wait_all_children if (defined $pbparallel);
1476            die "Aborting, one or more of the children failed." if ((not $all_ok) && ($Global::pb_stop_on_error));
1477            my $made = "";
1478            my %h = ();
1479            my %tmp;
1480            my %tmp2;
1481            my $pt;
1482            my $k;
1483
1484            foreach $k (<$tmpd/*>) {
1485                $made .= pb_get_content($k);
1486            }
1487            pb_rm_rf($tmpd);
1488            pb_log(3,"MADE:\n$made");
1489
1490            # Rebuild local hashes
1491            foreach $k (split(/\n/,$made)) {
1492                if ($k =~ /^\s*([A-z0-9-_]+)\s+([[A-z0-9-_.]+)\s*=\s*(.+)$/) {
1493                    $h{$1}->{$2}=$3;
1494                }
1495            }
1496            pb_log(2,"HASH: ".Dumper(%h));
1497
1498            # Patches
1499            pb_log(0,"Delivered and compressed patches ");
1500            $pt = $h{'patches'};
1501            foreach $k (keys %$pt) {
1502                foreach my $v1 (split(/,/,$pt->{$k})) {
1503                    $tmp{$v1} = "";
1504                }
1505            }
1506            if (keys %tmp) {
1507                foreach $k (keys %tmp) {
1508                    pb_log(0,"$k ");
1509                }
1510            } else {
1511                pb_log(0,"N/A");
1512            }
1513            pb_log(0,"\n");
1514
1515            # Sources
1516            pb_log(0,"Delivered additional sources ");
1517            $pt = $h{'sources'};
1518            foreach $k (keys %$pt) {
1519                foreach my $v1 (split(/,/,$pt->{$k})) {
1520                    $tmp2{$v1} = "";
1521                }
1522            }
1523            if (keys %tmp2) {
1524                foreach $k (keys %tmp2) {
1525                    pb_log(0,"$k ");
1526                }
1527            } else {
1528                pb_log(0,"N/A");
1529            }
1530            pb_log(0,"\n");
1531
1532            # Build files
1533            my @found;
1534            my @notfound;
1535            $pt = $h{'build'};
1536            foreach my $b (keys %$pt) {
1537                push @found,$b if ($pt->{$b} =~ /yes/);
1538                push @notfound,$b if ($pt->{$b} =~ /no/);
1539            }
1540            pb_log(0,"Build files have been generated for ... ".join(',',sort(@found))."\n") if (@found);
1541            pb_log(0,"No Build files found for ".join(',',sort(@notfound))."\n") if (@notfound);
1542
1543        } else {
1544            # Instead call News generation
1545            pb_web_news2html($dest);
1546            # And create an empty pbconf
1547            pb_mkdir_p("$dest/pbconf");
1548            # And prepare the pbscript to execute remotely
1549            open(SCRIPT,"> $ENV{'PBTMP'}/pbscript") || die "Unable to create $ENV{'PBTMP'}/pbscript";
1550            print SCRIPT "#!/bin/bash\n";
1551            print SCRIPT "#set -x\n";
1552            print SCRIPT "echo ... Extracting Website content\n";
1553            print SCRIPT "find . -type f | grep -Ev '^./$pbpkg-$pbver$pbextdir.tar.gz|^./pbscript' | xargs rm -f non-existent\n";
1554            print SCRIPT "find * -type d -depth | xargs rmdir 2> /dev/null \n";
1555            print SCRIPT "tar xfz $pbpkg-$pbver$pbextdir.tar.gz\n";
1556            print SCRIPT "mv $pbpkg-$pbver$pbextdir/* .\n";
1557            print SCRIPT "rm -f $pbpkg-$pbver$pbextdir.tar.gz\n";
1558            print SCRIPT "rmdir $pbpkg-$pbver$pbextdir\n";
1559            print SCRIPT "find . -type f -print0 | xargs -0 chmod 644\n";
1560            print SCRIPT "find . -type d -print0 | xargs -0 chmod 755\n";
1561            close(SCRIPT);
1562            chmod 0755,"$ENV{'PBTMP'}/pbscript";
1563        }
1564
1565        # Apply filters to the non-build files
1566        my $liste ="";
1567        if (defined $filteredfiles->{$pbpkg}) {
1568            foreach my $f (split(/,/,$filteredfiles->{$pbpkg})) {
1569                pb_filter_file_inplace($ptr,"$dest/$f",$pb);
1570                $liste = "$f $liste";
1571            }
1572        }
1573        pb_log(2,"Files ".$liste."have been filtered\n");
1574
1575        # TODO: Make it CMS aware
1576        # Execute the pbinit script if any
1577        if (-x "$ENV{'PBROOTDIR'}/$pbpkg/pbinit") {
1578            pb_filter_file("$ENV{'PBROOTDIR'}/$pbpkg/pbinit",$ptr,"$ENV{'PBTMP'}/pbinit",$pb);
1579            chmod 0755,"$ENV{'PBTMP'}/pbinit";
1580            pb_system("cd $dest ; $ENV{'PBTMP'}/pbinit","Executing init script from $ENV{'PBROOTDIR'}/$pbpkg/pbinit under $dest","verbose");
1581        }
1582
1583        # Do we have additional script to run to prepare the environement for the project ?
1584        # Then include it in the pbconf delivery
1585        foreach my $pbvf (<$ENV{'PBROOTDIR'}/$pbpkg/pbv*.pre>,<$ENV{'PBROOTDIR'}/$pbpkg/pbv*.post>, <$ENV{'PBROOTDIR'}/$pbpkg/pbtest*>) {
1586            pb_log(1,"Handling file $pbvf\n");
1587            if (-x "$pbvf") {
1588                my $target = "$ENV{'PBTMP'}/".basename($pbvf);
1589                pb_filter_file("$pbvf",$ptr,$target,$pb);
1590                chmod 0755,"$target";
1591                pb_log(1,"Capturing file $pbvf in $target\n");
1592            }
1593        }
1594
1595        # Prepare the dest directory for archive
1596        chdir "$ENV{'PBDESTDIR'}" || die "Unable to change dir to $ENV{'PBDESTDIR'}";
1597        if (defined $preserve) {
1598            # In that case we want to preserve the original tar file for checksum purposes
1599            # The one created is btw equivalent in that case to this one
1600            # Maybe check basename of both to be sure they are the same ?
1601            pb_log(0,"Preserving original tar file ");
1602            move("$preserve","$pbpkg-$pbver$pbextdir.tar.gz");
1603        } else {
1604            # Possibility to look at PBSRC to guess more the filename
1605            pb_system("tar cfz $pbpkg-$pbver$pbextdir.tar.gz --exclude=$pbpkg-$pbver$pbextdir/pbconf $pbpkg-$pbver$pbextdir","Creating $pbpkg tar files compressed");
1606        }
1607        pb_log(0,"Under $ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.tar.gz\n");
1608        pb_system("tar cfz $pbpkg-$pbver$pbextdir.pbconf.tar.gz $pbpkg-$pbver$pbextdir/pbconf","Creating pbconf tar files compressed");
1609        pb_log(0,"Under $ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.pbconf.tar.gz\n");
1610
1611        # Keep track of version-tag per pkg
1612        $pkgs{$pbpkg} = "$pbver-$pbtag";
1613
1614        # Final cleanup
1615        pb_rm_rf($dest) if (-d $dest);
1616    }
1617
1618    # Keep track of per package version
1619    pb_log(2,"DEBUG pkgs: ".Dumper(%pkgs)."\n");
1620    open(PKG,"> $ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml") || die "Unable to create $ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml";
1621    print PKG "---\n";
1622    print PKG "pbpkg:\n";
1623    foreach my $pbpkg (keys %pkgs) {
1624        print PKG "  $pbpkg: $pkgs{$pbpkg}\n";
1625    }
1626    close(PKG);
1627
1628    # Keep track of what is generated by default
1629    # We need to store the dir and info on version-tag
1630    # Base our content on the existing .yml file
1631    copy("$ENV{'PBROOTDIR'}/$ENV{'PBPROJ'}.yml","$ENV{'PBDESTDIR'}/pbrc.yml");
1632    my $lh;
1633    $lh = pb_conf_cache("$ENV{'PBDESTDIR'}/pbrc.yml",$lh);
1634    $lh->{'pbroot'}->{$ENV{'PBPROJ'}} = $ENV{'PBROOTDIR'};
1635    $lh->{'projver'}->{$ENV{'PBPROJ'}} = $ENV{'PBPROJVER'};
1636    $lh->{'projtag'}->{$ENV{'PBPROJ'}} = $ENV{'PBPROJTAG'};
1637    $lh->{'pbpackager'}->{$ENV{'PBPROJ'}} = $ENV{'PBPACKAGER'};
1638    $lh->{'pbextdir'}->{$ENV{'PBPROJ'}} = $pbextdir;
1639    pb_conf_write("$ENV{'PBDESTDIR'}/pbrc.yml",$lh);
1640}
1641
1642sub pb_test2pkg {
1643    # Get the running distro to test on
1644    my $pbos = pb_distro_get_context();
1645
1646    # Get list of packages to test
1647    # Get content saved in cms2build
1648    my $ptr = pb_get_pkg();
1649    @pkgs = @$ptr;
1650
1651    # Additional potential repo
1652    my $forcerepo = pb_distro_setuprepo($pbos);
1653    foreach my $pbpkg (@pkgs) {
1654        # We need to install the package to test, and deps brought with it
1655        pb_distro_installdeps(undef,$pbos,$forcerepo,$pbpkg);
1656        pb_system("$ENV{'PBDESTDIR'}/pbtest","Launching test for $pbpkg","verbose");
1657    }
1658}
1659
1660sub pb_build2prep {
1661
1662    pb_log(0,"INFO: ------ Starting to prepare build environement ------\n");
1663
1664    # Get the running distro to build on
1665    my $pbos = pb_distro_get_context();
1666
1667    # If needed we may add repository to the build env
1668    my $forcerepo = pb_distro_setuprepo($pbos);
1669    pb_log(1,"Forcing full package install with forcerepo\n") if (defined $forcerepo);
1670
1671    # Get list of packages to build
1672    my $ptr = pb_get_pkg();
1673    @pkgs = @$ptr;
1674   
1675    # Get content saved in cms2build
1676    my ($pkg) = pb_conf_read("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml","pbpkg");
1677    $pkg = { } if (not defined $pkg);
1678    my $pbextdir = pb_get_extdir();
1679
1680    foreach my $pbpkg (@pkgs) {
1681        my $vertag = $pkg->{$pbpkg};
1682        my @buildfiles;
1683        pb_log(2,"Vertag: $vertag\n");
1684        # get the version of the current package - maybe different
1685        ($pbver,$pbtag) = split(/-/,$vertag);
1686
1687        my $src2="$ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.pbconf.tar.gz";
1688        pb_log(2,"Pbconf file: $src2\n");
1689        pb_log(2,"Working directory: $ENV{'PBBUILDDIR'}\n");
1690
1691        my $ftype;
1692        if ($pbos->{'type'} eq "rpm") {
1693            $ftype = "\.spec\$";
1694        } elsif ($pbos->{'type'} eq "deb") {
1695            $ftype = "\/control\$";
1696        } elsif ($pbos->{'type'} eq "ebuild") {
1697            $ftype = "\.ebuild\$";
1698        } elsif ($pbos->{'type'} eq "tgz") {
1699            $ftype = "\/pbslack\$";
1700        } elsif ($pbos->{'type'} eq "pkg") {
1701            # Solaris - no deps management up to Solaris 10
1702        } elsif ($pbos->{'type'} eq "hpux") {
1703            # TODO: HP-UX
1704        } elsif ($pbos->{'type'} eq "apk") {
1705            $ftype = "\/APKBUILD\$";
1706        } else {
1707            die "Unknown OS type format $pbos->{'type'}";
1708        }
1709
1710        @buildfiles = pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/","$ENV{'PBTMP'}","build");
1711        pb_log(2,"buildfiles ".Dumper(\@buildfiles)."\n");
1712        foreach my $f (@buildfiles) {
1713            if ($f =~ /$ftype/) {
1714                pb_distro_installdeps($f,$pbos,$forcerepo);
1715            }
1716        }
1717    }
1718    pb_log(0,"INFO: ------ Finished preparing build environment ------\n");
1719}
1720
1721sub pb_build2pkg {
1722
1723    my $do_install = shift;
1724
1725        # Call build2prep in case we have new deps we want to take in account
1726        # without redoing the prep phase (can be long and that helps debugging
1727        # build dependencies issues
1728    pb_build2prep();
1729    pb_log(0,"INFO: ------ Starting to build package ------\n");
1730    # Get the running distro to build on
1731    my $pbos = pb_distro_get_context();
1732
1733    # Get list of packages to build
1734    my $ptr = pb_get_pkg();
1735    @pkgs = @$ptr;
1736
1737    # Get content saved in cms2build
1738    my ($pkg) = pb_conf_read("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml","pbpkg");
1739    $pkg = { } if (not defined $pkg);
1740    my $pbextdir = pb_get_extdir();
1741
1742    pb_mkdir_p("$ENV{'PBBUILDDIR'}") if (! -d "$ENV{'PBBUILDDIR'}");
1743    chdir "$ENV{'PBBUILDDIR'}" || die "Unable to chdir to $ENV{'PBBUILDDIR'}";
1744    my $made = ""; # pkgs made during build
1745    my $pm;
1746    my $all_ok = 1;
1747
1748    if (defined $pbparallel) {
1749        $pm = new Parallel::ForkManager($pbparallel);
1750        $pm->run_on_finish(sub { my ($pid, $code, $id, $signal, $dump) = @_;
1751                                $all_ok = 0 unless (($code == 0) && ($signal == 0) && ($dump == 0)); });
1752    }
1753
1754    # We need to communicate info back from the children if parallel so prepare a dir for that
1755    my $tmpd = "$ENV{'PBTMP'}/build.$$";
1756    pb_mkdir_p($tmpd) if (defined $pbparallel);
1757
1758    # Do it here as this doesn't work in // mode as it dies if it fails
1759    if ($pbos->{'type'} eq "rpm") {
1760        foreach my $d ('RPMS','SRPMS','SPECS','SOURCES','BUILD') {
1761                if (! -d "$ENV{'PBBUILDDIR'}/$d") {
1762                    pb_mkdir_p("$ENV{'PBBUILDDIR'}/$d");
1763                }
1764            }
1765    }
1766
1767    foreach my $pbpkg (@pkgs) {
1768        $pm->start and next if (defined $pbparallel);
1769
1770        my $vertag = $pkg->{$pbpkg};
1771        pb_log(2,"Vertag: $vertag\n");
1772        # get the version of the current package - maybe different
1773        ($pbver,$pbtag) = split(/-/,$vertag);
1774
1775        my $src="$ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.tar.gz";
1776        my $src2="$ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.pbconf.tar.gz";
1777        pb_log(2,"Source file: $src\n");
1778        pb_log(2,"Pbconf file: $src2\n");
1779
1780        pb_log(2,"Working directory: $ENV{'PBBUILDDIR'}\n");
1781        if ($pbos->{'type'} eq "rpm") {
1782            # Remove in case a previous link/file was there
1783            unlink "$ENV{'PBBUILDDIR'}/SOURCES/".basename($src);
1784            symlink "$src","$ENV{'PBBUILDDIR'}/SOURCES/".basename($src) || die "Unable to symlink $src in $ENV{'PBBUILDDIR'}/SOURCES";
1785            # We need to first extract the spec file
1786            my @buildfile = pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/","$ENV{'PBBUILDDIR'}/SPECS","build");
1787
1788            # We need to handle potential patches to upstream sources
1789            pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbpatch/","$ENV{'PBBUILDDIR'}/SOURCES","patch");
1790
1791            # We need to handle potential additional sources to upstream sources
1792            pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbsrc/","$ENV{'PBBUILDDIR'}/SOURCES","src");
1793
1794            pb_log(2,"buildfile: ".Dumper(\@buildfile)."\n");
1795            # set LANGUAGE to check for correct log messages
1796            $ENV{'LANGUAGE'}="C";
1797            # Older Redhat use _target_platform in %configure incorrectly
1798            my $specialdef = "";
1799            if (($pbos->{'name'} eq "rhel") && ($pbos->{'version'} eq "2.1")) {
1800                $specialdef = "--define \'_target_platform \"\"\'";
1801            }
1802            my $buildcmd = "rpmbuild";
1803            if (($pbos->{'name'} eq "redhat") && ($pbos->{'version'} =~ /^6/)){
1804                $buildcmd = "rpm" ;
1805            }
1806
1807            foreach my $f (@buildfile) {
1808                if ($f =~ /\.spec$/) {
1809                    pb_system("$buildcmd $specialdef --define \"packager $ENV{'PBPACKAGER'}\" --define \"_topdir $ENV{'PBBUILDDIR'}\" -ba $f","Building package with $f under $ENV{'PBBUILDDIR'}","verbose");
1810                    last;
1811                }
1812            }
1813            # Get the name of the generated packages
1814            open(LOG,"$ENV{'PBTMP'}/system.$$.log") || die "Unable to open $ENV{'PBTMP'}/system.$$.log";
1815            while (<LOG>) {
1816                chomp($_);
1817                next if ($_ !~ /^Wrote:/);
1818                s|.*/([S]*RPMS.*)|$1|;
1819                $made .=" $_";
1820            }
1821            close(LOG);
1822
1823        } elsif ($pbos->{'type'} eq "deb") {
1824            pb_system("tar xfz $src","Extracting sources");
1825            pb_system("tar xfz $src2","Extracting pbconf");
1826
1827            chdir "$pbpkg-$pbver$pbextdir" || die "Unable to chdir to $pbpkg-$pbver$pbextdir";
1828            pb_rm_rf("debian");
1829            my $confdir = "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
1830            die "Configuration directory $confdir is not a directory\nIs os description listed in your {ve,vm,rm}list values?" if (not -d $confdir);
1831            symlink "$confdir","debian" || die "Unable to symlink 'debian' to $confdir";
1832            chmod 0755,"debian/rules";
1833
1834            # We need to handle potential patches to upstream sources
1835            pb_mkdir_p("debian/patches");
1836            my @f = pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbpatch/","debian/patches","patch");
1837
1838            # By default we use format 1.0 - Cf man dpkg-source
1839            my $debsrcfmt = "1.0";
1840            my $debsrcfile = "debian/source/format";
1841            if (-f $debsrcfile) {
1842                $debsrcfmt = pb_get_content($debsrcfile);
1843            }
1844
1845            if ($debsrcfmt =~ /^3.*quilt/) {
1846                # If we use quilt to manage patches, we then setup the env correctly
1847                # as per http://pkg-perl.alioth.debian.org/howto/quilt.html
1848                # Generate Debian patch series for quilt
1849                open(SERIE,"> debian/patches/series") || die "Unable to write in debian/patches/series";
1850                $ENV{'QUILT_PATCHES'}="debian/patches";
1851            } else {
1852                # If we use dpatch to manage patches, we then setup the 00list file as well
1853                open(SERIE,"> debian/patches/00list") || die "Unable to write in debian/patches/00list";
1854            }
1855            foreach my $f (sort @f) {
1856                # Skip the script made to apply the patches to the Debian tree
1857                next if ($f =~ /pbapplypatch/);
1858                # We also need to uncompress them
1859                pb_system("gzip -d $f","","quiet");
1860                $f =~ s/\.gz$//;
1861                print SERIE "$f\n";
1862            }
1863            close(SERIE);
1864            if (@f) {
1865                # We have patches...
1866                my $patch_file = "debian/patches/pbapplypatch";
1867                if (($debsrcfmt =~ /^1.*/) && (-x $patch_file)) {
1868                    # In that case we need to apply the patches ourselves locally
1869                    pb_system("cat $patch_file","APPLY","verbose");
1870                    pb_system("$patch_file","Applying patches to $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'} tree");
1871                }
1872                # ...so modify the name of files to be Debian compliant
1873                move("../$src","../$pbpkg-$pbver$pbextdir.orig.tar.gz");
1874            }
1875
1876            # We need to handle potential additional sources to upstream sources
1877            #pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbsrc/","$ENV{'PBBUILDDIR'}/debian","src");
1878
1879            pb_system("dpkg-buildpackage -us -uc -rfakeroot","Building package","verbose");
1880
1881            # Get the name of the generated packages
1882            open(LOG,"$ENV{'PBTMP'}/system.$$.log") || die "Unable to open $ENV{'PBTMP'}/system.$$.log";
1883            while (<LOG>) {
1884                chomp();
1885                pb_log(3,"In loop: $_\n");
1886                next unless (/^dpkg-deb.*\.\.\/(.*_.*\.deb).*/);
1887                my $tmp = $1;
1888                #doesn't work in my case
1889                #die "Missing file $tmp" if (not -f "../$tmp");
1890                $made = "$made $tmp";
1891            }
1892            close(LOG);
1893            pb_log(2,"Now made is: $made\n");
1894
1895            open(CTRL,"debian/control") || die "Unable to open debian/control: $!";
1896            while (<CTRL>) {
1897                pb_log(3,"In loop: $_\n");
1898                next unless (/^Source: (\S+)/o);
1899                my $tmp = $1;
1900                $made = "$made $tmp"."_*.dsc $tmp"."_*.tar.gz $tmp"."_*.changes";
1901                #doesn't work in my case
1902                #foreach my $glob (("$1\_*.changes", "$1\_*.dsc", "$1\_*.tar.gz")) {
1903                    #my @file = glob($glob);
1904                    #die "Missing file for $glob" unless (@file > 0);
1905                    #die "Too many files for $glob" if (@file > 1);
1906                    #die "Missing file $file[0]" if (not -f $file[0]);
1907                    #$made .= " $file[0]";
1908                #}
1909            }
1910            close(CTRL);
1911            #pb_display_file("$ENV{'PBTMP'}/system.$$.log");
1912            pb_log(2,"Finally made is: $made\n");
1913
1914            chdir ".." || die "Unable to chdir to parent dir";
1915            pb_rm_rf("$pbpkg-$pbver$pbextdir");
1916        } elsif ($pbos->{'type'} eq "ebuild") {
1917            my @ebuildfile;
1918            # For gentoo we need to take pb as subsystem name
1919            # We put every apps here under sys-apps. hope it's correct
1920            # We use pb's home dir in order to have a single OVERLAY line
1921            my $tmpe = "$ENV{'HOME'}/portage/pb/sys-apps/$pbpkg";
1922            pb_mkdir_p($tmpe) if (! -d "$tmpe");
1923            pb_mkdir_p("$ENV{'HOME'}/portage/distfiles") if (! -d "$ENV{'HOME'}/portage/distfiles");
1924
1925            # We need to first extract the ebuild file
1926            @ebuildfile = pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/","$tmpe","build");
1927
1928            # Prepare the build env for gentoo
1929            my $found = 0;
1930            my $pbbd = $ENV{'HOME'};
1931            $pbbd =~ s|/|\\/|g;
1932            if (-r "/etc/make.conf") {
1933                open(MAKE,"/etc/make.conf");
1934                while (<MAKE>) {
1935                    $found = 1 if (/$pbbd\/portage/);
1936                }
1937                close(MAKE);
1938            }
1939            if ($found == 0) {
1940                pb_system("sudo sh -c 'echo PORTDIR_OVERLAY=\"$ENV{'HOME'}/portage\" >> /etc/make.conf'");
1941            }
1942            #$found = 0;
1943            #if (-r "/etc/portage/package.keywords") {
1944            #open(KEYW,"/etc/portage/package.keywords");
1945            #while (<KEYW>) {
1946            #$found = 1 if (/portage\/pb/);
1947            #}
1948            #close(KEYW);
1949            #}
1950            #if ($found == 0) {
1951            #pb_system("sudo sh -c \"echo portage/pb >> /etc/portage/package.keywords\"");
1952            #}
1953
1954            # Build
1955            foreach my $f (@ebuildfile) {
1956                if ($f =~ /\.ebuild$/) {
1957                    move($f,"$tmpe/$pbpkg-$pbver.ebuild");
1958                    pb_system("cd $tmpe ; ebuild $pbpkg-$pbver.ebuild clean ; ebuild $pbpkg-$pbver.ebuild digest ; ebuild $pbpkg-$pbver.ebuild package","verbose");
1959                    # Now move it where pb expects it
1960                    pb_mkdir_p("$ENV{'PBBUILDDIR'}/portage/pb/sys-apps/$pbpkg");
1961                    if ($pbtag eq 0) {
1962                        # This is assumed to be a test version
1963                        my $nver = substr($pbver,0,-14);
1964                        my $ntag = substr($pbver,-14);
1965                        my $ebtg = "portage/pb/sys-apps/$pbpkg/$pbpkg-$nver"."_p$ntag.ebuild";
1966                        move("$tmpe/$pbpkg-$pbver.ebuild","$ENV{'PBBUILDDIR'}/$ebtg");
1967                        $made="$made $ebtg";
1968                    } else {
1969                        my $ebtg = "portage/pb/sys-apps/$pbpkg/$pbpkg-$pbver-r$pbtag.ebuild";
1970                        move("$tmpe/$pbpkg-$pbver.ebuild","$ENV{'PBBUILDDIR'}/$ebtg");
1971                        $made="$made $ebtg";
1972                    }
1973                }
1974            }
1975
1976        } elsif ($pbos->{'type'} eq "apk") {
1977            pb_mkdir_p("$ENV{'PBBUILDDIR'}/tmp");
1978
1979            # We need to first extract the APKBUILD file
1980            my @apkfile = pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/","$ENV{'PBBUILDDIR'}/tmp","build");
1981
1982            chdir "$ENV{'PBBUILDDIR'}/tmp" || die "Unable to chdir to $ENV{'PBBUILDDIR'}/tmp";
1983
1984            # Build
1985            my $keyfile = pb_ssh_get(1);
1986            pb_mkdir_p("$ENV{'HOME'}/.abuild");
1987            open(CONF,"> $ENV{'HOME'}/.abuild/abuild.conf") || confess "Unable to create the abuild conf file";
1988            print CONF "PACKAGER_PRIVKEY=\"$keyfile\"\n" if (defined $keyfile);
1989            close(CONF);
1990            foreach my $f (@apkfile) {
1991                if ($f =~ /APKBUILD$/) {
1992                    pb_system("ls -al $ENV{'HOME'}/.ssh ; pwd ; ls -alR ../.. ; sh -x /usr/bin/abuild checksum ; abuild unpack ; abuild -r -P .. build ; abuild -P .. rootpkg ; abuild -P pkg index","verbose");
1993                }
1994            }
1995            # Get the name of the generated packages
1996            open(LOG,"$ENV{'PBTMP'}/system.$$.log") || die "Unable to open $ENV{'PBTMP'}/system.$$.log";
1997            while (<LOG>) {
1998                chomp($_);
1999                next if ($_ !~ /: Create /);
2000                s|.*: Create (.*.apk)|$1|;
2001                $made .= " $_";
2002            }
2003            close(LOG);
2004
2005        } elsif ($pbos->{'type'} eq "tgz") {
2006            # Slackware family
2007            $made="$made $pbpkg/$pbpkg-$pbver-*-$pbtag.tgz";
2008
2009            pb_system("tar xfz $src","Extracting sources");
2010            pb_system("tar xfz $src2","Extracting pbconf");
2011            chdir "$pbpkg-$pbver$pbextdir" || die "Unable to chdir to $pbpkg-$pbver$pbextdir";
2012            symlink "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}","install" || die "Unable to symlink to pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
2013            if (-x "install/pbslack") {
2014                pb_system("./install/pbslack","Building software");
2015                pb_system("sudo /sbin/makepkg -p -l y -c y $pbpkg","Packaging $pbpkg","verbose");
2016            }
2017            chdir ".." || die "Unable to chdir to parent dir";
2018            pb_rm_rf("$pbpkg-$pbver$pbextdir");
2019        } elsif ($pbos->{'type'} eq "pkg") {
2020            # Solaris
2021            $made="$made $pbpkg-$pbver-$pbtag.pkg.gz";
2022            my $pkgdestdir="$ENV{'PBBUILDDIR'}/install";
2023
2024            # Will host resulting packages
2025            pb_mkdir_p("$pbos->{'type'}");
2026            pb_mkdir_p("$pkgdestdir/pbdelivery");
2027            pb_system("tar xfz $src","Extracting sources under $ENV{'PBBUILDDIR'}");
2028            pb_system("tar xfz $src2","Extracting pbconf under $ENV{'PBBUILDDIR'}");
2029            # We need to handle potential patches to upstream sources
2030            pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbpatch/","$ENV{'PBBUILDDIR'}","patch");
2031
2032            # We need to handle potential additional sources to upstream sources
2033            pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbsrc/","$ENV{'PBBUILDDIR'}","src");
2034
2035            chdir "$pbpkg-$pbver$pbextdir" || die "Unable to chdir to $pbpkg-$pbver$pbextdir";
2036            if (-f "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbbuild") {
2037                chmod 0755,"pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbbuild";
2038                # pkginfo file is mandatory
2039                die "Unable to find pkginfo file in pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}" if (! -f "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pkginfo");
2040                # Build
2041                pb_system("pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbbuild $pkgdestdir/pbdelivery","Building software and installing under $pkgdestdir/pbdelivery");
2042                # Copy complementary files
2043                if (-f "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/prototype") {
2044                    copy("pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/prototype", $pkgdestdir) 
2045                } else {
2046                    # No prototype provided, calculating it
2047                    open(PROTO,"> $pkgdestdir/prototype") || die "Unable to create prototype file";
2048                    print PROTO "i pkginfo\n";
2049                    print PROTO "i depend\n" if (-f "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/depend");
2050                    $ENV{'PBSOLDESTDIR'} = "$pkgdestdir/pbdelivery";
2051                    find(\&create_solaris_prototype, "$pkgdestdir/pbdelivery");
2052                }
2053                copy("pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/depend", $pkgdestdir) if (-f "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/depend");
2054                copy("pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pkginfo", $pkgdestdir);
2055                pb_system("cd $pkgdestdir/pbdelivery ; pkgmk -o -f ../prototype -r $pkgdestdir/pbdelivery -d $ENV{'PBBUILDDIR'}/$pbos->{'type'}","Packaging $pbpkg","verbose");
2056                pb_system("cd $ENV{'PBBUILDDIR'}/$pbos->{'type'} ;  echo \"\" | pkgtrans -o -n -s $ENV{'PBBUILDDIR'}/$pbos->{'type'} $ENV{'PBBUILDDIR'}/$pbpkg-$pbver-$pbtag.pkg all","Transforming $pbpkg","verbose");
2057                pb_system("cd $ENV{'PBBUILDDIR'} ;  gzip -9f $pbpkg-$pbver-$pbtag.pkg","Compressing $pbpkg-$pbver-$pbtag.pkg","verbose");
2058            } else {
2059                pb_log(0,"No pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbbuild file found for $pbpkg-$pbver\n");
2060            }
2061            chdir ".." || die "Unable to chdir to parent dir";
2062            pb_rm_rf("$pbpkg-$pbver$pbextdir","$ENV{'PBBUILDDIR'}/$pbos->{'type'}","$pkgdestdir");
2063        } elsif ($pbos->{'type'} eq "hpux") {
2064            # HP-UX
2065            pb_system("tar xfz $src","Extracting sources");
2066            pb_system("tar xfz $src2","Extracting pbconf");
2067
2068            chdir "$pbpkg-$pbver$pbextdir" || die "Unable to chdir to $pbpkg-$pbver$pbextdir";
2069            pb_system("buildpackage ","Building package","verbose");
2070            # Get the name of the generated packages
2071            open(LOG,"$ENV{'PBTMP'}/system.$$.log") || die "Unable to open $ENV{'PBTMP'}/system.$$.log";
2072            while (<LOG>) {
2073                chomp();
2074                my $tmp = $_;
2075                next if ($tmp !~ /^SD BUILD.*:/);
2076                $tmp =~ s|.*../(.*)_(.*).sd.*|$1|;
2077                $made = "$made $tmp"."_*.sd";
2078            }
2079            close(LOG);
2080            $made="$made $pbpkg-$pbver-$pbtag.sd";
2081
2082            chdir ".." || die "Unable to chdir to parent dir";
2083            pb_rm_rf("$pbpkg-$pbver$pbextdir");
2084        } else {
2085            die "Unknown OS type format $pbos->{'type'}";
2086        }
2087        if (defined $pbparallel) {
2088            # Communicate results back to parent
2089            pb_set_content("$tmpd/$$",$made);
2090            $pm->finish;
2091        }
2092    }
2093    if (defined $pbparallel) {
2094        # In the parent, we need to get the result from the children
2095        $pm->wait_all_children;
2096        foreach my $f (<$tmpd/*>) {
2097            $made .= " ".pb_get_content($f);
2098        }
2099        die "Aborting, one or more of the children failed." if ((not $all_ok) && ($Global::pb_stop_on_error));
2100        pb_rm_rf($tmpd);
2101    }
2102
2103    # Sign packages
2104    pb_sign_pkgs($pbos,$made);
2105    pb_log(0,"INFO: ------ Finished building package ------\n");
2106
2107    # Find the appropriate check cmd/opts
2108    my ($chkcmd,$chkopt) = pb_distro_get_param($pbos,pb_conf_get_if("oschkcmd","oschkopt"));
2109
2110    my $ret = "";
2111    # Packages check if needed
2112    if ($pbos->{'type'} eq "rpm") {
2113        if ((defined  $chkcmd) && (-x $chkcmd)) {
2114            my $cmd = "$chkcmd";
2115            $cmd .= " $chkopt" if (defined $chkopt);
2116            $cmd .= " $made";
2117            my $ret = pb_system("$cmd","Checking validity of rpms with $chkcmd","mayfailverbose");
2118            pb_log(0,"ERROR: when checking packages validity\n") if ($ret ne 0);
2119        }
2120        my $rpms ="";
2121        my $srpms ="";
2122        foreach my $f (split(/ /,$made)) {
2123            $rpms .= "$ENV{'PBBUILDDIR'}/$f " if ($f =~ /^RPMS\//);
2124            $srpms .= "$ENV{'PBBUILDDIR'}/$f " if ($f =~ /^SRPMS\//);
2125        }
2126        pb_log(0,"SRPM packages generated: $srpms\n");
2127        pb_log(0,"RPM packages generated: $rpms\n");
2128        $ret = $rpms;
2129    } elsif ($pbos->{'type'} eq "deb") {
2130        my $made2 = "";
2131        foreach my $f (split(/ /,$made)) {
2132            $made2 .= "$f " if ($f =~ /\.deb$/);
2133        }
2134        if ((defined  $chkcmd) && (-x $chkcmd)) {
2135            my $ret = pb_system("$chkcmd $chkopt $made2","Checking validity of debs with $chkcmd","mayfail");
2136            pb_log(0,"ERROR: when checking packages validity\n") if ($ret ne 0);
2137        }
2138        pb_log(0,"deb packages generated: $made2\n");
2139        $ret = $made2;
2140    } else {
2141        pb_log(0,"No check done for $pbos->{'type'} yet\n");
2142        pb_log(0,"Packages generated: $made\n");
2143        $ret = $made;
2144    }
2145
2146    # Keep track of what is generated so that we can get them back from VMs/RMs
2147    my $pbstore = "$ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
2148    open(KEEP,"> $pbstore") || die "Unable to create $pbstore $!";
2149    print KEEP "$made\n";
2150    close(KEEP);
2151    pb_distro_installdeps(undef,$pbos,undef,$ret,"local") if ($do_install);
2152}
2153
2154sub pb_build2ssh {
2155    pb_send2target("Sources");
2156    pb_send2target("CPAN");
2157}
2158
2159sub pb_pkg2ssh {
2160    pb_send2target("Packages");
2161}
2162
2163# By default deliver to the the public site hosting the
2164# ftp structure (or whatever) or a VM/VE/RM
2165sub pb_send2target {
2166
2167    my $cmt = shift;
2168    my $pbscript1 = shift;
2169    my $v = shift;
2170    my $vexist = shift;         # 0 is FALSE
2171    my $vpid = shift;           # 0 is FALSE
2172    my $snapme = shift;         # 0 is FALSE
2173    my $pbstep = shift;         # 3 is usage of container
2174
2175    $vexist = 0 if (not defined $vexist);
2176    $vpid = 0 if (not defined $vpid);
2177    $snapme = 0 if (not defined $snapme);
2178    $pbstep = 3 if (not defined $pbstep);
2179    pb_log(2,"DEBUG: pb_send2target($cmt,".Dumper($v).",$vexist,$vpid)\n");
2180    my $host = "sshhost";
2181    my $login = "sshlogin";
2182    my $dir = "sshdir";
2183    my $port = "sshport";
2184    my $conf = "sshconf";
2185    my $tmout = undef;
2186    my $path = undef;
2187
2188    if ($cmt =~ /^VM/) {
2189        $login = "vmlogin";
2190        $dir = "pbdefdir";
2191        # Specific VM
2192        $tmout = "vmtmout";
2193        $path = "vmpath";
2194        $host = "vmhost";
2195        $port = "vmport";
2196    } elsif ($cmt =~ /^RM/) {
2197        $login = "rmlogin";
2198        $dir = "pbdefdir";
2199        # Specific RM
2200        $tmout = "rmtmout";
2201        $path = "rmpath";
2202        $host = "rmhost";
2203        $port = "rmport";
2204    } elsif ($cmt =~ /^VE/) {
2205        $login = "velogin";
2206        $dir = "pbdefdir";
2207        # Specific VE
2208        $path = "vepath";
2209        $conf = "rbsconf";
2210    } elsif ($cmt eq "Web") {
2211        $host = "websshhost";
2212        $login = "websshlogin";
2213        $dir = "websshdir";
2214        $port = "websshport";
2215    } elsif ($cmt eq "CPAN") {
2216        $host = "cpanpause";
2217        $login = "";
2218        $dir = "cpandir";
2219        $port = "";
2220    }
2221    my $cmd = "";
2222    my $src = "";
2223    my $cpanpkg = 0;
2224    my $pbos;
2225
2226    my $pbextdir = pb_get_extdir();
2227
2228    if ($cmt ne "Announce") {
2229        # Get list of packages to build
2230        my $ptr = pb_get_pkg();
2231        my @pkgs = ();
2232        @pkgs = @$ptr if (defined $ptr);
2233
2234        # Get the running distro to consider
2235        $pbos = pb_distro_get_context($v);
2236
2237        my $pkg = { };
2238        # Get content saved in cms2build
2239        if ((defined $ENV{'PBDESTDIR'}) && (defined $ENV{'PBPROJVER'}) && (defined $ENV{'PBPROJTAG'}) && (-f "$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml")) {
2240            ($pkg) = pb_conf_read("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml","pbpkg");
2241            $pkg = { } if (not defined $pkg);
2242        }
2243
2244        pb_mkdir_p("$ENV{'PBBUILDDIR'}") if (! -d "$ENV{'PBBUILDDIR'}");
2245        chdir "$ENV{'PBBUILDDIR'}" || die "Unable to chdir to $ENV{'PBBUILDDIR'}";
2246        foreach my $pbpkg (@pkgs) {
2247            my $vertag = $pkg->{$pbpkg};
2248            # get the version of the current package - maybe different
2249            pb_log(2,"Vertag: $vertag\n");
2250            ($pbver,$pbtag) = split(/-/,$vertag);
2251
2252            if (($cmt eq "Sources") || ($cmt =~ /(V[EM]|RM)(build|prep)/)) {
2253                $src = "$src $ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.tar.gz $ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.pbconf.tar.gz";
2254                my $cmd2 = "ln -sf $pbpkg-$pbver$pbextdir.tar.gz $pbpkg-latest.tar.gz";
2255                #my $cmd2 = "ln $pbpkg-$pbver$pbextdir.tar.gz $pbpkg-latest.tar.gz 2> /dev/null ; if [ \$? -ne 0 ]; then ln -sf $pbpkg-$pbver$pbextdir.tar.gz $pbpkg-latest.tar.gz ; fi";
2256                if ($cmd eq "") {
2257                    $cmd = $cmd2;
2258                } else {
2259                    $cmd = "$cmd ; $cmd2";
2260                }
2261            } elsif ($cmt eq "Web") {
2262                $src = "$src $ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.tar.gz"
2263            }
2264
2265            # Do we have one perl package
2266            my @nametype = pb_conf_get_if("namingtype");
2267            my $type = $nametype[0]->{$pbpkg};
2268            if ((defined $type) && ($type eq "perl")) {
2269                $cpanpkg = 1;
2270            }
2271        }
2272        # Adds conf file for availability of conf elements
2273        pb_conf_add("$ENV{'PBROOTDIR'}/$ENV{'PBPROJ'}.yml") if ((defined $ENV{'PBROOTDIR'}) && (-f "$ENV{'PBROOTDIR'}/$ENV{'PBPROJ'}.yml"));
2274    }
2275    my ($rbsconf,$testver,$delivery) = pb_conf_get_if($conf,"testver","delivery");
2276    if ($cmt eq "CPAN") {
2277        # Do not deliver on Pause if this is a test version
2278        return if (not defined $testver);
2279        return if ($testver =~ /true/);
2280        # Do not deliver on Pause if this is not a perl package
2281        return if ($cpanpkg == 0);
2282    }
2283
2284    if ($cmt =~ /(V[EM]|RM)(build|prep)/) {
2285        $src="$src $ENV{'PBROOTDIR'}/$ENV{'PBPROJ'}.yml $ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml $ENV{'PBETC'} $ENV{'PBDESTDIR'}/pbrc.yml $pbscript{$v}";
2286    } elsif ($cmt =~ /(V[EM]|RM)Script/) {
2287        $src="$src $pbscript{$v}";
2288    } elsif ($cmt =~ /(V[EM]|RM)test/) {
2289        $src="$src $ENV{'PBROOTDIR'}/$ENV{'PBPROJ'}.yml $ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml $ENV{'PBETC'} $ENV{'PBDESTDIR'}/pbrc.yml $pbscript{$v} $ENV{'PBDESTDIR'}/pbtest";
2290    } elsif (($cmt eq "Announce") || ($cmt eq "Web") || ($cmt eq "CPAN")) {
2291        $src="$src $ENV{'PBTMP'}/pbscript";
2292    } elsif ($cmt eq "Packages") {
2293        # Get package list from file made during build2pkg
2294        open(KEEP,"$ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}") || die "Unable to read $ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
2295        $src = <KEEP>;
2296        chomp($src);
2297        close(KEEP);
2298    }
2299    if (($cmt eq "Sources") || ($cmt eq "Packages") || ($cmt eq "CPAN")) {
2300        my ($pbpackager) = pb_conf_get("pbpackager");
2301        $ENV{'PBPACKAGER'} = $pbpackager->{$ENV{'PBPROJ'}};
2302        pb_log(0,"Exporting public key for $ENV{'PBPACKAGER'}\n");
2303        # Using pb_system is not working due to redirection below
2304        system("gpg --export -a \'$ENV{'PBPACKAGER'}\' > $ENV{'PBDESTDIR'}/$ENV{'PBPROJ'}.pubkey");
2305        chmod 0644,"$ENV{'PBDESTDIR'}/$ENV{'PBPROJ'}.pubkey";
2306        $src = "$src $ENV{'PBDESTDIR'}/$ENV{'PBPROJ'}.pubkey";
2307
2308        my ($gpgaddkey) = pb_conf_get_if("pbadditionalgpg");
2309        my $pbadditionalgpg;
2310        $pbadditionalgpg = $gpgaddkey->{$ENV{'PBPROJ'}} if (defined $gpgaddkey);
2311        if (defined $pbadditionalgpg) {
2312            foreach my $k (split(/,/,$pbadditionalgpg)) {
2313                pb_log(0,"Exporting public key $k\n");
2314                system("gpg --export -a \'$k\' > $ENV{'PBDESTDIR'}/$ENV{'PBPROJ'}-$k.pubkey");
2315                $src = "$src $ENV{'PBDESTDIR'}/$ENV{'PBPROJ'}-$k.pubkey";
2316            }
2317        }
2318    }
2319    # Remove potential leading spaces (cause problem with basename)
2320    $src =~ s/^ *//;
2321    my $basesrc = "";
2322    foreach my $i (split(/ +/,$src)) {
2323        $basesrc .= " ".basename($i);
2324    }
2325
2326    pb_log(2,"values: ".Dumper(($host,$login,$dir,$port,$tmout,$path,$conf))."\n");
2327    my ($sshhost,$sshdir) = pb_conf_get($host,$dir);
2328    # Not mandatory...
2329    $delivery->{$ENV{'PBPROJ'}} = "" if (not defined $delivery->{$ENV{'PBPROJ'}});
2330    my ($sshlogin,$sshport) = pb_conf_get_if($login,$port);
2331    $sshport->{$ENV{'PBPROJ'}} = 22 if (not defined $sshport->{$ENV{'PBPROJ'}});
2332    $sshlogin->{$ENV{'PBPROJ'}} = getpwuid($UID) if (not defined $sshlogin->{$ENV{'PBPROJ'}});
2333    my ($vtmout,$vepath);
2334    # ...Except those in virtual context
2335    if ($cmt =~ /^VE/) {
2336        ($vepath) = pb_conf_get($path);
2337    }
2338    if ($cmt =~ /^(V|R)M/) {
2339        $vtmout = pb_distro_get_param($pbos,pb_conf_get_if($tmout));
2340    }
2341    my $remhost = $sshhost->{$ENV{'PBPROJ'}};
2342    my $remdir = $sshdir->{$ENV{'PBPROJ'}};
2343    if ($cmt =~ /^V[EM]|RM/) {
2344        # In that case our real host is in the xxhost with the OS as key, not project as above
2345        $remhost = pb_distro_get_param($pbos,$sshhost);
2346    }
2347    pb_log(2,"ssh: ".Dumper(($remhost,$sshlogin,$remdir,$sshport,$vepath,$rbsconf))."\n");
2348    pb_log(2,"ssh: ".Dumper($vtmout)."\n") if (defined $vtmout);
2349
2350    my $mac;
2351    if ($cmt !~ /^VE/) {
2352        $mac = "$sshlogin->{$ENV{'PBPROJ'}}\@$remhost";
2353        # Overwrite account value if passed as parameter or forced
2354        $mac = "$ENV{'PBACCOUNT'}\@$remhost" if (defined $ENV{'PBACCOUNT'});
2355    } else {
2356        # VE
2357        $mac = $sshlogin->{$ENV{'PBPROJ'}};
2358        # Overwrite account value if passed as parameter (typically for setup2v)
2359        $mac = $ENV{'PBACCOUNT'} if (defined $ENV{'PBACCOUNT'});
2360    }
2361    pb_log(2, "DEBUG: mac: $mac\n");
2362
2363    my $tdir;
2364    my $bdir;
2365    if (($cmt eq "Sources") || ($cmt =~ /(V[EM]|RM)Script/)) {
2366        $tdir = "$remdir/$delivery->{$ENV{'PBPROJ'}}/src";
2367    } elsif ($cmt eq "CPAN") {
2368        $tdir = "$remdir";
2369    } elsif ($cmt =~ /(V[EM]|RM)(build|test|prep)/) {
2370        $tdir = $remdir."/$ENV{'PBPROJ'}/pbdelivery";
2371        $bdir = $remdir."/$ENV{'PBPROJ'}/pbbuild";
2372        # Remove a potential $ENV{'HOME'} as bdir should be relative to pb's home
2373        $bdir =~ s|\$ENV.+\}/||;
2374    } elsif ($cmt eq "Announce") {
2375        $tdir = "$remdir/$delivery->{$ENV{'PBPROJ'}}";
2376    } elsif ($cmt eq "Web") {
2377        $tdir = "$remdir/$delivery->{$ENV{'PBPROJ'}}";
2378    } elsif ($cmt eq "Packages") {
2379        if (($pbos->{'type'} eq "rpm") || ($pbos->{'type'} eq "pkg") || ($pbos->{'type'} eq "hpux") || ($pbos->{'type'} eq "tgz") || ($pbos->{'type'} eq "apk")) {
2380            # put packages under an arch subdir
2381            $tdir = "$remdir/$delivery->{$ENV{'PBPROJ'}}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}";
2382        } elsif (($pbos->{'type'} eq "deb") || ($pbos->{'type'} eq "ebuild")) {
2383            # No need for an arch subdir
2384            $tdir = "$remdir/$delivery->{$ENV{'PBPROJ'}}/$pbos->{'name'}/$pbos->{'version'}";
2385        } else {
2386            die "Please teach the dev team where to deliver ($pbos->{'type'} type of packages\n";
2387        }
2388
2389        my $repodir = $tdir;
2390        $repodir =~ s|^$remdir/||;
2391        my $repotag = "";
2392        $repotag = "-$delivery->{$ENV{'PBPROJ'}}" if ($delivery->{$ENV{'PBPROJ'}} ne "");
2393
2394        my ($pbrepo) = pb_conf_get("pbrepo");
2395
2396        # Script for Repository management
2397        $v = "$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}" if (not defined $v);
2398        $pbscript{$v} = "$ENV{'PBBUILDDIR'}/pbscript.$$";
2399       
2400        # That script should be part of the delivery
2401        $src = "$src $pbscript{$v}";
2402        open(PBS,"> $pbscript{$v}") || die "Unable to create $pbscript{$v}";
2403        if ($pbos->{'type'} eq "rpm") {
2404            my $pbsha = pb_distro_get_param($pbos,pb_conf_get("ossha"));
2405            my ($gpgcheck) = pb_conf_get_if("pbgpgcheck");
2406            my $pbgpgcheck;
2407            $pbgpgcheck = $gpgcheck->{$ENV{'PBPROJ'}} if (defined $gpgcheck);
2408            my $pbgpgkey = "$pbrepo->{$ENV{'PBPROJ'}}/$repodir/$ENV{'PBPROJ'}.pubkey";
2409            my ($gpgaddpkey) = pb_conf_get_if("pbadditionalgpg");
2410            my $pbadditionalgpg;
2411            $pbadditionalgpg = $gpgaddpkey->{$ENV{'PBPROJ'}} if (defined $gpgaddpkey);
2412            if (defined $pbadditionalgpg) {
2413                foreach my $k (split(/,/,$pbadditionalgpg)) {
2414                    $pbgpgkey .= " $pbrepo->{$ENV{'PBPROJ'}}/$repodir/$ENV{'PBPROJ'}-$k.pubkey";
2415                }
2416            }
2417            # By default force GPG check in repo even if we support signature of packages to fail. This is a best practice
2418            $pbgpgcheck = 1 if (not defined $pbgpgcheck);
2419            # Also make a pbscript to generate dnf/yum/urpmi bases
2420            print PBS << "EOF";
2421#!/bin/bash
2422# Prepare a script to ease dnf/yum setup
2423EOF
2424            print PBS "set -x\n" if ($pbdebug gt 1);
2425            print PBS << "EOF";
2426cat > $ENV{'PBPROJ'}$repotag.repo << EOT
2427[$ENV{'PBPROJ'}$repotag]
2428name=$pbos->{'name'} $pbos->{'version'} $pbos->{'arch'} - $ENV{'PBPROJ'} Vanilla Packages
2429baseurl=$pbrepo->{$ENV{'PBPROJ'}}/$repodir
2430enabled=1
2431gpgcheck=$pbgpgcheck
2432gpgkey=$pbgpgkey
2433EOT
2434chmod 644 $ENV{'PBPROJ'}$repotag.repo
2435
2436# Clean up old repo content
2437rm -rf headers/ repodata/
2438# Create yum repo
2439if [ -x /usr/bin/yum-arch ]; then
2440    yum-arch .
2441fi
2442# Create repodata
2443createrepo -s $pbsha .
2444# Link to the key
2445if [ -s $ENV{'PBPROJ'}.pubkey ]; then
2446    # Avoiding creating empty repomd.xml.key; fails on opensuse 12.1
2447    (cd repodata ; ln -sf ../$ENV{'PBPROJ'}.pubkey repomd.xml.key)
2448fi
2449# sign the repomd (at least useful for SLES - which requires a local key)
2450# gpg -a --detach-sign repodata/repomd.xml
2451# SLES also looks for media.1/info.txt
2452EOF
2453            if ($pbos->{'family'} eq "md") {
2454                # For Mandriva add urpmi management
2455                print PBS << "EOF";
2456# Prepare a script to ease urpmi setup
2457cat > $ENV{'PBPROJ'}$repotag.addmedia << EOT
2458#rpm --import $pbrepo->{$ENV{'PBPROJ'}}/$repodir/$ENV{'PBPROJ'}.pubkey
2459urpmi.addmedia $ENV{'PBPROJ'}$repotag $pbrepo->{$ENV{'PBPROJ'}}/$repodir with media_info/hdlist.cz
2460EOT
2461chmod 755 $ENV{'PBPROJ'}$repotag.addmedia
2462
2463# Clean up old repo content
2464rm -f hdlist.cz synthesis.hdlist.cz
2465# Create urpmi repo
2466genhdlist2 --clean .
2467if [ \$\? -ne 0 ]; then
2468    genhdlist .
2469fi
2470(cd media_info ; ln -sf ../$ENV{'PBPROJ'}.pubkey pubkey)
2471EOF
2472            }
2473            if ($pbos->{'name'} eq "fedora") {
2474                # Extract the spec file to please Fedora maintainers :-(
2475                print PBS << "EOF";
2476for p in $basesrc; do
2477    echo \$p | grep -q 'src.rpm'
2478    if [ \$\? -eq 0 ]; then
2479        rpm2cpio \$p | cpio -ivdum --quiet '*.spec'
2480    fi
2481done
2482EOF
2483            }
2484            if ($pbos->{'family'} eq "novell") {
2485                # Add ymp scripts for one-click install on SuSE
2486                print PBS << "EOF";
2487# Prepare a script to ease SuSE one-click install
2488# Cf: http://de.opensuse.org/1-Klick-Installation/ISV
2489#
2490cat > $ENV{'PBPROJ'}$repotag.ymp << EOT
2491<?xml version="1.0" encoding="utf-8"?>
2492<!-- vim: set sw=2 ts=2 ai et: -->
2493<metapackage xmlns:os="http://opensuse.org/Standards/One_Click_Install" xmlns="http://opensuse.org/Standards/One_Click_Install">
2494    <group><!-- The group of software, typically one for project-builder.org -->
2495        <name>$ENV{'PBPROJ'} Bundle</name> <!-- Name of the software group -->
2496        <summary>Software bundle for the $ENV{'PBPROJ'}$repotag project</summary> <!--This message is shown to the user and should describe the whole bundle -->
2497        <description>This is the summary of the $ENV{'PBPROJ'}$repotag Project
2498             
2499            Details are available on a per package basis below
2500
2501        </description><!--This is also shown to the user -->
2502        <remainSubscribed>false</remainSubscribed> <!-- Don't know what it mean -->
2503        <repositories><!-- List of needed repositories -->
2504            <repository>
2505                <name>$ENV{'PBPROJ'}$repotag Repository</name> <!-- Name of the repository  -->
2506                <summary>This repository contains the $ENV{'PBPROJ'}$repotag project packages.</summary> <!-- Summary of the repository -->
2507                <description>This repository contains the $ENV{'PBPROJ'}$repotag project packages.</description><!-- This description is shown to the user -->
2508                <url>$pbrepo->{$ENV{'PBPROJ'}}/$repodir</url><!--URL of repository, which is added -->
2509            </repository>
2510        </repositories>
2511        <software><!-- A List of packages, which should be added through the one-click-installation -->
2512EOT
2513for p in $basesrc; do
2514    sum=`rpm -q --qf '%{SUMMARY}' \$p`
2515    name=`rpm -q --qf '%{NAME}' \$p`
2516    desc=`rpm -q --qf '%{description}' \$p`
2517    cat >> $ENV{'PBPROJ'}$repotag.ymp << EOT
2518            <item>
2519                <name>\$name</name><!-- Name of the package, is shown to the user and used to identify the package at the repository -->
2520                <summary>\$sum</summary> <!-- Summary of the package -->
2521                <description>\$desc</description> <!-- Description, is shown to the user -->
2522            </item>
2523EOT
2524done
2525cat >> $ENV{'PBPROJ'}$repotag.ymp << EOT
2526        </software>
2527    </group>
2528</metapackage>
2529EOT
2530chmod 644 $ENV{'PBPROJ'}$repotag.ymp
2531EOF
2532            }
2533        } elsif ($pbos->{'type'} eq "deb") {
2534            # Also make a pbscript to generate apt bases
2535            # Cf: http://www.debian.org/doc/manuals/repository-howto/repository-howto.fr.html
2536            # This dirname removes ver
2537            my $debarch = pb_get_debarch($pbos);
2538            my $projcomponent = pb_get_debpc($pbos);
2539            my $rpd = dirname("$pbrepo->{$ENV{'PBPROJ'}}/$repodir");
2540            # Remove extra . in path to fix #522
2541            $rpd =~ s|/./|/|g;
2542            print PBS << "EOF";
2543#!/bin/bash
2544# Prepare a script to ease apt setup
2545cat > $ENV{'PBPROJ'}$repotag.sources.list << EOT
2546deb $rpd $pbos->{'version'} $projcomponent
2547deb-src $rpd $pbos->{'version'} $projcomponent
2548EOT
2549chmod 644 $ENV{'PBPROJ'}$repotag.sources.list
2550
2551# Up two levels to deal with the dist dir cross versions
2552cd ..
2553mkdir -p dists/$pbos->{'version'}/$projcomponent/binary-$debarch dists/$pbos->{'version'}/$projcomponent/source dists/$pbos->{'version'}/$projcomponent/binary-all
2554
2555# Prepare a script to create apt info file
2556# Reuse twice after
2557TMPD=`mktemp -d /tmp/pb.XXXXXXXXXX` || exit 1
2558mkdir -p \$TMPD
2559cat > \$TMPD/Release << EOT
2560Archive: unstable
2561Component: $projcomponent
2562Origin: $ENV{'PBPROJ'}$repotag
2563Label: $ENV{'PBPROJ'}$repotag dev repository $pbrepo->{$ENV{'PBPROJ'}}
2564EOT
2565
2566echo "Creating Packages metadata"
2567for i in dists/$pbos->{version}/$projcomponent/binary-*; do
2568    arch=`basename \$i | sed 's/binary-//'`
2569    echo "Packages for \$arch:"
2570    dpkg-scanpackages -a\$arch $pbos->{'version'} /dev/null > dists/$pbos->{'version'}/$projcomponent/binary-\$arch/Packages
2571    gzip -9 < dists/$pbos->{'version'}/$projcomponent/binary-\$arch/Packages > dists/$pbos->{'version'}/$projcomponent/binary-\$arch/Packages.gz
2572    bzip2 -9 < dists/$pbos->{'version'}/$projcomponent/binary-\$arch/Packages > dists/$pbos->{'version'}/$projcomponent/binary-\$arch/Packages.bz2
2573done
2574echo "Creating Contents metadata"
2575apt-ftparchive contents $pbos->{'version'} | gzip -c9 > dists/$pbos->{'version'}/Contents.gz
2576echo "Creating Release metadata ($pbos->{'arch'} aka $debarch)"
2577cat \$TMPD/Release > dists/$pbos->{'version'}/$projcomponent/binary-all/Release
2578echo "Architecture: all" >> dists/$pbos->{'version'}/$projcomponent/binary-all/Release
2579cat \$TMPD/Release > dists/$pbos->{'version'}/$projcomponent/binary-$debarch/Release
2580echo "Architecture: $debarch" >> dists/$pbos->{'version'}/$projcomponent/binary-$debarch/Release
2581echo "Creating Source metadata"
2582dpkg-scansources $pbos->{'version'} /dev/null > dists/$pbos->{'version'}/$projcomponent/source/Sources
2583gzip -9 < dists/$pbos->{'version'}/$projcomponent/source/Sources > dists/$pbos->{'version'}/$projcomponent/source/Sources.gz
2584bzip2 -9 < dists/$pbos->{'version'}/$projcomponent/source/Sources > dists/$pbos->{'version'}/$projcomponent/source/Sources.bz2
2585cat \$TMPD/Release > dists/$pbos->{'version'}/$projcomponent/source/Release
2586echo "Architecture: Source" >> dists/$pbos->{'version'}/$projcomponent/source/Release
2587echo "Creating Release metadata"
2588apt-ftparchive -o APT::FTPArchive::Release::Suite=$pbos->{version} release dists/$pbos->{'version'} > dists/$pbos->{'version'}/Release
2589rm -rf \$TMPD
2590EOF
2591        } elsif ($pbos->{'type'} eq "ebuild") {
2592            # make a pbscript to generate links to latest version
2593            print PBS << "EOF";
2594#!/bin/bash
2595# Prepare a script to create correct links
2596cd $tdir
2597for p in $src; do
2598    echo \$p | grep -q '.ebuild'
2599    if [ \$\? -eq 0 ]; then
2600        j=`basename \$p`
2601        pp=`echo \$j | perl -pe 's/-[0-9\.]+-r[0-9].*\.ebuild/.ebuild/'`
2602        ln -sf \$j \$pp
2603    fi
2604done
2605EOF
2606        } elsif ($pbos->{'type'} eq "apk") {
2607            # TODO
2608        }
2609        close(PBS);
2610        chmod 0755,$pbscript{$v};
2611    } else {
2612        return;
2613    }
2614
2615    # Useless for VE
2616    my $nport = pb_get_port($sshport,$pbos,$cmt) if ($cmt !~ /^VE/);
2617
2618    # Remove a potential $ENV{'HOME'} as tdir should be relative to pb's home
2619    if ($cmt =~ /^VE/o) {
2620        $tdir =~ s|\$ENV.+\}/|| or confess "for $cmt: $tdir must have \$ENV{'HOME'} in it or the remainder of the code won't work since some parts use relative pathing and others will try absolute.  Use a symlink if necessary to place the real files in a different place than under your home directory";
2621    } else {
2622        # process, ok if not present.
2623        $tdir =~ s|\$ENV.+\}/||;
2624    }
2625
2626    my $tm = "";
2627    if ($cmt =~ /^(V|R)M/) {
2628        $tm = "sleep $vtmout" if (defined $vtmout);
2629    }
2630
2631    # ssh communication if not VE or CPAN
2632    # should use a hash instead...
2633    my ($shcmd,$shcmdroot,$cpcmd,$cptarget,$cp2target);
2634    my $tpdir;
2635    my $tp;
2636    my $context = "$ENV{'PBTMP'}";
2637    my %tag;
2638    my $dkaccount = "";
2639    my $cmd1 = "";
2640    my $cidfile;
2641    my $homedir = "";
2642    if ($cmt =~ /^VE/) {
2643        $tp = pb_path_expand($vepath->{$ENV{'PBPROJ'}});
2644        $vetype = pb_ve_get_type($vetype);
2645        if ($vetype eq "docker") {
2646            $tpdir = "/";
2647            $cidfile = "$context/ctn-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}.cid";
2648        } else {
2649            $tpdir = pb_path_expand("$tp/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}");
2650        }
2651        my $arch = pb_get_arch();
2652        if ($vetype eq "chroot") {
2653            $shcmdroot = "sudo /usr/sbin/chroot $tpdir ";
2654            $shcmd = "$shcmdroot /bin/su - $mac -c ";
2655        } elsif ($vetype eq "schroot") {
2656            $shcmd = "schroot $tp -u $mac -- ";
2657        } elsif ($vetype eq "docker") {
2658            # docker manages the storage so rely on it
2659            $shcmdroot = "";
2660            my $docrepo = pb_ve_docker_repo();
2661            $cmd1 = pb_check_req("docker",0);
2662            #my ($dockeropt) =  pb_conf_get_if("dockeropt");
2663            # pbimage is used for docker and is setup to the right name depending on the step:
2664            # TODO: This is not coded yet
2665            my $pbimage = undef;
2666            # step 0 : nothing at creation -> tag n-v-a (made in VE.pm)
2667            # step 1 : n-v-a + setup -> tag n-v-a-pb
2668            # step 2 : n-v-a-pb + build -> tag n-v-a-pb-pbproj
2669            # step 3 : n-v-a-pb-pbproj at use
2670            if ((not defined $pbimage) || ($pbimage eq "")) {
2671                # If no image name given, create a naming convention
2672                $tag{1} = "$docrepo$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
2673                $tag{2} = "$tag{1}-pb";
2674                $tag{3} = "$tag{2}-$ENV{'PBPROJ'}";
2675            } else {
2676                # If we were given an image name, just use it
2677                $tag{1} = $pbimage;
2678                $tag{2} = $pbimage;
2679                $tag{3} = $pbimage;
2680            }
2681            # If we do not create the image, then use the one we should have
2682            # TODO: we don't do anything with found !!
2683            my $found = pb_ve_docker_get_image($tag{$pbstep});
2684            confess "No image $tag{$pbstep} available in Docker, please create one first\n" if (not defined $found);
2685            # Now we use that image to do what is needed
2686            # use a dockerfile to ease the creation of next images
2687            # the context is in an empty temp dir
2688            my $tmpd = "$context/Dockerfile";
2689            open(DOCKER, "> $tmpd") || die "Unable to create the docker file $tmpd";
2690            print DOCKER "FROM $tag{$pbstep}\n";
2691            print DOCKER "MAINTAINER project-builder.org aka pb\n";
2692            if ($pbstep == 1) {
2693                # setup done as root
2694                $dkaccount = "root";
2695            } else {
2696                # Rest done as pb
2697                $dkaccount = "pb";
2698            }
2699            print DOCKER "USER $dkaccount\n";
2700            if ($pbstep <= 2) {
2701                $shcmd = "$cmd1 build -t $tag{$pbstep+1} $context";
2702            } else {
2703                # As we are in run phase use docker run. cmd will be completed below
2704                $shcmd = "$cmd1 run --cidfile=\"$cidfile\" --user $dkaccount --name $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-$cmt";
2705            }
2706            #$shcmd = "$cmd1 build $dockeropt->{$ENV{'PBPROJ'}} -t $tag{$pbstep+1} $context";
2707        }
2708        $shcmd = "setarch i386 $shcmd" if (($pbos->{'arch'} =~ /i[3456]86/) && ($arch eq 'x86_64'));
2709        if (($cmt =~ /VE/) && ($vetype ne "docker")) {
2710            # We need to get the home dir of the target account to deliver in the right place
2711            open(PASS,"$tpdir/etc/passwd") || die "Unable to open $tpdir/etc/passwd: $!";
2712            while (<PASS>) {
2713                my ($c1,$c2,$c3,$c4,$c5,$c6,$c7) = split(/:/);
2714                $homedir = $c6 if ($c1 =~ /^$mac$/);
2715                pb_log(3,"Homedir: $homedir - account: $c6\n");
2716            }
2717            close(PASS);
2718            $cpcmd = "sudo /bin/cp -r ";
2719        }
2720        if (($cmt =~ /VE/) && ($vetype eq "docker")) {
2721            if ($pbstep >= 2) {
2722                # We need to get the home dir of the target account to deliver in the right place
2723                $homedir = `$cmd1 run --cidfile="$cidfile" $tag{$pbstep} grep -E '^$dkaccount:' /etc/passwd | cut -d: -f6`;
2724                chomp($homedir);
2725                open(CID,"$cidfile") || confess "Unable to open $cidfile";
2726                my $cid = <CID>;
2727                close(CID);
2728                pb_system("$cmd1 rm $cid","","quiet");
2729                unlink("$cidfile");
2730            } else {
2731                $homedir = "/";
2732            }
2733            $cpcmd = "cp -r ";
2734        }
2735        $cptarget = "$tpdir/$homedir/$tdir";
2736        if ($cmt =~ /VE(build|prep)/) {
2737            $cp2target = "$tpdir/$homedir/$bdir";
2738        }
2739        pb_log(2,"On VE using $cptarget as target dir to copy to\n");
2740    } elsif ($cmt eq "CPAN") {
2741        my $ftpput = pb_check_req("ncftpput",1);
2742        my $ftpget = pb_check_req("wget",1);
2743        my ($cpanuser,$cpanpasswd) = pb_conf_get_if("cpanuser","cpanpasswd");
2744        return if ((not defined $cpanuser) || (not defined $cpanuser->{$ENV{'PBPROJ'}}));
2745        return if ((not defined $cpanpasswd) || (not defined $cpanpasswd->{$ENV{'PBPROJ'}}));
2746        my ($cpansubdir) = pb_conf_get_if("cpansubdir");
2747        $shcmd = "$ftpget --post-data \'HIDDENNAME=".$cpanuser->{$ENV{'PBPROJ'}};
2748        $shcmd .= "&user=".$cpanuser->{$ENV{'PBPROJ'}};
2749        $shcmd .= "&password=".$cpanpasswd->{$ENV{'PBPROJ'}};
2750        $shcmd .= "&SUBMIT_pause99_add_uri_upload=\"Upload the checked files\"";
2751        $shcmd .= "&pause99_add_uri_subdirtext=".$cpansubdir->{$ENV{'PBPROJ'}} if ((defined $cpansubdir) && (defined $cpansubdir->{$ENV{'PBPROJ'}}));
2752        foreach my $s (split(/ /,$src)) {
2753            $shcmd .= "&pause99_add_uri_upload=".basename($s);
2754        }
2755        $shcmd .= "'";
2756        $cpcmd = "$ftpput $host $dir";
2757        $cptarget = "CPAN";
2758    } else {
2759        my $keyopt = "";
2760        if ($cmt !~ /Packages/) {
2761            my $keyfile = pb_ssh_get(0,$remhost,$nport);
2762            $keyopt = "-i $keyfile" if (defined $keyfile);
2763        }
2764        my $sshcmd = pb_check_req("ssh",1);
2765        my $scpcmd = pb_check_req("scp",1);
2766        $shcmd = "$sshcmd $keyopt -q -o NoHostAuthenticationForLocalhost=yes -p $nport $mac";
2767        $cpcmd = "$scpcmd $keyopt -p -o NoHostAuthenticationForLocalhost=yes -P $nport";
2768        $cptarget = "$mac:$tdir";
2769        if ($cmt =~ /^(V|R)M(build|prep)/) {
2770            $cp2target = "$mac:$bdir";
2771        }
2772    }
2773   
2774    my $logres = "";
2775    # Do not touch when just announcing
2776    if (($cmt ne "Announce") && ($cmt ne "CPAN")) {
2777        # For docker everything is done in the Dockerfile for steps < 3
2778        if (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep < 3)) {
2779            print DOCKER "RUN mkdir -p $cptarget\n";
2780            print DOCKER "RUN cd $cptarget ; for i in $basesrc; do if [ -f \$i ]; then rm -f \$i; fi; done\n";
2781        } elsif (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep >= 3)) {
2782            # Nothing to do here
2783        } else {
2784            pb_system("$shcmd \"mkdir -p $tdir ; cd $tdir ; echo \'for i in $basesrc; do if [ -f \$i ]; then rm -f \$i; fi; done ; $cmd\' | bash -e\"","Preparing $tdir on $cptarget");
2785        }
2786    } else {
2787        $logres = "> ";
2788    }
2789
2790    if ($cmt =~ /SandBox/) {
2791        # Install from sandbox mean using the result of the just passed sbx2build command
2792        # Get content saved in cms2build
2793        my ($pkg) = pb_conf_read("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml","pbpkg");
2794        my $pbextdir = pb_get_extdir();
2795        confess "Unable to get package list" if (not defined $pkg);
2796
2797        # We consider 2 specific packages
2798        my $vertag1 = $pkg->{"ProjectBuilder"};
2799        my $vertag2 = $pkg->{"project-builder"};
2800        # get the version of the current package - maybe different
2801        pb_log(2,"Vertag1: $vertag1\n");
2802        pb_log(2,"Vertag2: $vertag2\n");
2803        my ($pbver1,$tmp1) = split(/-/,$vertag1);
2804        my ($pbver2,$tmp2) = split(/-/,$vertag2);
2805        if ($cmt =~ /VE/) {
2806            # Copy inside the VE
2807            if ($vetype ne "docker") {
2808                copy("$ENV{'PBDESTDIR'}/ProjectBuilder-$pbver1$pbextdir.tar.gz","$tpdir/tmp");
2809                copy("$ENV{'PBDESTDIR'}/project-builder-$pbver2$pbextdir.tar.gz","$tpdir/tmp");
2810            } else {
2811                copy("$ENV{'PBDESTDIR'}/ProjectBuilder-$pbver1$pbextdir.tar.gz","$context");
2812                copy("$ENV{'PBDESTDIR'}/project-builder-$pbver2$pbextdir.tar.gz","$context");
2813                print DOCKER "COPY ProjectBuilder-$pbver1$pbextdir.tar.gz /tmp/\n";
2814                print DOCKER "COPY project-builder-$pbver2$pbextdir.tar.gz /tmp/\n";
2815            }
2816        } else {
2817            pb_system("$cpcmd $ENV{'PBDESTDIR'}/ProjectBuilder-$pbver1$pbextdir.tar.gz $ENV{'PBDESTDIR'}/project-builder-$pbver2$pbextdir.tar.gz $mac:/tmp","Copying local project files to VM/RM");
2818        }
2819    }
2820           
2821    pb_log(0,"Sources handled ($cmt): $src\n");
2822    # For docker everything is done in the Dockerfile
2823    if (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep < 3)) {
2824        foreach my $f (split(/ +/,$src)) {
2825            copy("$f","$context");
2826            print DOCKER "COPY ".basename($f)." $cptarget/\n";
2827        }
2828        print DOCKER "RUN cd $cptarget ; $cmd\n" if ((defined $cmd) && ($cmd ne ""));
2829    } elsif (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep >= 3)) {
2830        # Nothing to do
2831    } else {
2832        pb_system("cd $ENV{'PBBUILDDIR'} ; $cpcmd $src $cptarget 2> /dev/null","$cmt delivery in $cptarget");
2833    }
2834
2835    # For VE we need to change the owner manually
2836    if ($cmt =~ /^VE/) {
2837        if ($vetype ne "docker") {
2838            my $sudomode = pb_distro_get_param($pbos,pb_conf_get("ossudoersmode"));
2839            my $res = pb_system("$shcmdroot sed -i '/requiretty/d' /etc/sudoers","Removing potential requiretty in sudoers","quiet");
2840            pb_system("$shcmdroot sed '/requiretty/d' /etc/sudoers > /tmp/sudoers.new ; mv /tmp/sudoers.new $tpdir/tmp/sudoers.new ; $shcmdroot mv /tmp/sudoers.new /etc/sudoers ; $shcmdroot chown root:root /etc/sudoers ; $shcmdroot chmod $sudomode /etc/sudoers","Removing again potential requiretty in sudoers as sed -i failed") if (($res ne 0) && (-f "$tpdir/etc/sudoers"));
2841            pb_system("$shcmd \"sudo chown -R $mac $tdir\"","Adapt owner in $tdir to $mac");
2842        } else {
2843            print DOCKER "RUN sed -i '/requiretty/d' /etc/sudoers\n" if ($pbstep == 1);
2844        }
2845    }
2846
2847    # Use the right script name depending on context
2848    my $pbscript2;
2849    if (($cmt =~ /^(V[EM]|RM)/) || ($cmt =~ /Packages/)){
2850        $pbscript2 = $pbscript{$v};
2851    } else {
2852        $pbscript2 = "pbscript";
2853    }
2854
2855    # It's already ready for CPAN
2856    my $shcmdbase = $shcmd;
2857    my $dkcmd;
2858    if ($cmt ne "CPAN") {
2859        if (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep < 3)) {
2860            copy("$pbscript2","$context");
2861            my $s = basename($pbscript2);
2862            print DOCKER "COPY $s $cptarget/\n";
2863            # call directly with perl to avoid text file busy error messages
2864            print DOCKER "RUN cd $cptarget ; perl ./$s ; if [ '$pbkeep' = '0' ]; then rm -f $s; fi\n";
2865            print DOCKER "RUN rm -rf $cptarget/*\n";
2866            print DOCKER "RUN chown -R $dkaccount $cptarget\n" if ($pbstep == 1);
2867            close(DOCKER);
2868        } elsif (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep >= 3)) {
2869            $dkcmd = "/bin/bash $homedir/$tdir-2/".basename($pbscript2);
2870        } else {
2871            my $s = basename($pbscript2);
2872            $shcmd .= " \"echo \'cd $tdir ; if [ -x $s ]; then ./$s; fi ;  if [ $pbkeep == 0 ]; then rm -f $s; fi\' | bash\"";
2873        }
2874    }
2875    my $cmdverb = "verbose";
2876    if (($cmt eq "Announce") || ($cmt eq "CPAN")) {
2877        $cmdverb = undef;
2878    } elsif (defined ($v)) {
2879        $cmdverb = "verbose_\[$v\] ";
2880    }
2881    # this is where we launch the execution
2882    if (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep >= 3)) {
2883        pb_mkdir_p("$ENV{'PBTMP'}/vebuild");
2884        my $tsrc = "";
2885        foreach my $f (split(/ +/,$src)) {
2886            copy("$f","$ENV{'PBTMP'}/vebuild");
2887            $tsrc .= " ".basename($f);
2888        }
2889        my $ts = "$ENV{'PBTMP'}/vebuild/".basename($pbscript2);
2890        open(DOCKER," > $ts") || confess "Unable to create $ts";
2891        print DOCKER "#!/bin/bash\n";
2892        print DOCKER "# Put content from host to guest, copying to have the correct uid/gid and rights to do so\n";
2893        print DOCKER "# set -x\n";
2894        print DOCKER "for i in $tsrc; do\n";
2895        print DOCKER "  # Cleanup first to avoid rights issues for root owned files e.g.\n";
2896        print DOCKER "  rm -f $homedir/$tdir/\$i\n";
2897        print DOCKER "  cp $homedir/$tdir-2/\$i $homedir/$tdir\n";
2898        print DOCKER "done\n";
2899        print DOCKER "(cd $homedir/$tdir-2 ; ls -al)\n";
2900        print DOCKER pb_get_content($pbscript2);
2901        print DOCKER "\n# Get content back from guest in a host dir\n";
2902        # Host dir seen as TDIR in container
2903        $cp2target = "$ENV{'PBBUILDDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
2904        pb_mkdir_p("$cp2target");
2905        chmod 0777,"$cp2target";
2906        print DOCKER "TDIR=$homedir/$bdir-2/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}\n";
2907        print DOCKER "for i in $homedir/$bdir/*; do\n";
2908        print DOCKER "  # Cleanup first to avoid rights issues for root owned files e.g.\n";
2909        print DOCKER "  j=\`basename \$i\`\n";
2910        print DOCKER "  rm -f \$TDIR/\$j\n";
2911        print DOCKER "  cp -r \$i \$TDIR\n";
2912        print DOCKER "done\n";
2913        print DOCKER "find \$TDIR -type f -print0 | xargs -0 chmod 644\n";
2914        print DOCKER "find \$TDIR -type d -print0 | xargs -0 chmod 777 2> /dev/null\n";
2915        #print DOCKER "ls -al \$TDIR\n";
2916        print DOCKER "exit 0\n";
2917        close(DOCKER);
2918        $shcmd .= " -v $ENV{'PBBUILDDIR'}:$homedir/$bdir-2 -v $ENV{'PBTMP'}/vebuild:$homedir/$tdir-2 -w $homedir/$tdir-2 $tag{$pbstep} $dkcmd";
2919    }
2920    my $ret = pb_system("$shcmd","Executing pbscript on $cptarget if needed",$cmdverb);
2921   
2922    # If target is deb family then sign Release file for Packages
2923    my $pbos2 = pb_distro_get_context($v);
2924    if (($cmt =~ /Packages/) && ($pbos2->{'type'} eq "deb")) {
2925        my $debarch = pb_get_debarch($pbos2);
2926        my $projcomponent = pb_get_debpc($pbos2);
2927        for my $f ("dists/$pbos2->{'version'}/$projcomponent/binary-$debarch/Release","dists/$pbos2->{'version'}/$projcomponent/binary-all/Release","dists/$pbos2->{'version'}/$projcomponent/source/Release","dists/$pbos2->{'version'}/Release") {
2928            my $debdir = dirname($cptarget);
2929            pb_system("cd $ENV{'PBBUILDDIR'} ; $cpcmd $debdir/$f . 2> /dev/null","Getting deb $f file from $debdir");
2930            pb_sign_file("$ENV{'PBBUILDDIR'}/Release");
2931            pb_system("$cpcmd $ENV{'PBBUILDDIR'}/Release.gpg $debdir/$f.gpg 2> /dev/null","Putting deb $f.gpg file to $debdir");
2932            my ($void,$debfile) = split(/:/,$debdir);
2933            pb_system("$shcmdbase \"chmod 644 $debfile/$f.gpg\"","Set mode to 644 for $debfile/$f.gpg");
2934        }
2935    } 
2936
2937    if ($cmt =~ /^(V[EM]|RM)build/) {
2938        # Get back info on pkg produced, compute their name and get them from the VM/RM
2939        pb_system("$cpcmd $cp2target/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'} $ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.$$ 2> /dev/null","Get package names in $cp2target");
2940        if (not -f "$ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.$$") {
2941            pb_log(0,"ERROR with VM/RM $v on getting $ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.$$\n");
2942        } else {
2943            open(KEEP,"$ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.$$") || die "Unable to read $ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.$$";
2944            my $src = <KEEP>;
2945            chomp($src);
2946            close(KEEP);
2947            unlink("$ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.$$");
2948
2949            $src =~ s/^ *//;
2950            pb_mkdir_p("$ENV{'PBBUILDDIR'}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}");
2951            # Change pgben to make the next send2target happy
2952            my $made = "";
2953   
2954            # For VM/RM we don't want shell expansion to hapen locally but remotely
2955            my $delim = '\'';
2956            if ($cmt =~ /^VEbuild/) {
2957                # For VE we need to support shell expansion locally
2958                $delim = "";
2959            }   
2960
2961            open(KEEP,"> $ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}") || die "Unable to write $ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
2962            foreach my $p (split(/ +/,$src)) {
2963                my $j = basename($p);
2964                pb_system("$cpcmd $cp2target/$delim$p$delim $ENV{'PBBUILDDIR'}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'} 2> /dev/null","Recovery of package $j in $ENV{'PBBUILDDIR'}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}");
2965                $made="$made $pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}/$j"; # if (($pbos->{'type'} ne "rpm") || ($j !~ /.src.rpm$/));
2966            }
2967       
2968            print KEEP "$made\n";
2969            close(KEEP);
2970            if (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep >= 3)) {
2971                pb_rm_rf("$cp2target");
2972                pb_system("$cmd1 rm $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-$cmt","","quiet");
2973            } else {
2974                pb_system("$shcmdbase \"rm -rf $tdir $bdir\"","$cmt cleanup");
2975            }
2976
2977            # Sign packages locally
2978            pb_sign_pkgs($pbos,$made);
2979
2980            # We want to send them to the ssh account so overwrite what has been done before
2981            pb_log(2,"Before sending pkgs, vexist: $vexist, vpid: $vpid\n");
2982            pb_send2target("Packages","$ENV{'PBDESTDIR'}/pbscript.$$",$pbos->{'name'}."-".$pbos->{'version'}."-".$pbos->{'arch'},$vexist,$vpid);
2983            pb_rm_rf("$ENV{'PBBUILDDIR'}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}");
2984        }
2985    }
2986    unlink("$pbscript{$v}") if ((($cmt =~ /^(V[ME]|RM)/) || ($cmt =~ /Packages/)) && ($pbkeep eq 0) && ($cmt !~ /Script/));
2987
2988    pb_log(2,"Before halt, vexist: $vexist, vpid: $vpid\n");
2989    if ((! $vexist) && ($cmt =~ /^VM/)) {
2990        # If in setupvm then takes a snapshot just before halting
2991        if ($snapme != 0) {
2992            my ($vmmonport,$vmtype) = pb_conf_get("vmmonport","vmtype");
2993            # For monitoring control
2994            if ((($vmtype->{$ENV{'PBPROJ'}}) eq "kvm") || (($vmtype->{$ENV{'PBPROJ'}}) eq "qemu")) {
2995                eval
2996                {
2997                require Net::Telnet;
2998                Net::Telnet->import();
2999                };
3000                if ($@) {
3001                    # Net::Telnet not found
3002                    pb_log(1,"ADVISE: Install Net::Telnet to benefit from monitoring control and snapshot feature.\nWARNING: No snapshot created");
3003                } else {
3004                    my $t = new Net::Telnet (Timeout => 120, Host => "localhost", Port => $vmmonport->{$ENV{'PBPROJ'}}) || die "Unable to dialog on the monitor";
3005                    # move to monitor mode
3006                    my @lines = $t->cmd("c");
3007                    # Create a snapshot named pb
3008                    @lines = $t->cmd("savevm pb");
3009                    # Write the new status in the VM
3010                    @lines = $t->cmd("commit all");
3011                    # End
3012                    @lines = $t->cmd("quit");
3013                }
3014            }
3015        }
3016        my $hoption = "-p";
3017        my $hpath = pb_distro_get_param($pbos,pb_conf_get("ospathcmd-halt"));
3018        # Solaris doesn't support -p of halt
3019        if ($pbos->{'type'} eq "pkg") {
3020            $hoption = "" ;
3021        }
3022        pb_system("$shcmdbase \"sudo $hpath $hoption \"; $tm ; echo \'if [ -d /proc/$vpid ]; then kill -9 $vpid; fi \' | bash ; sleep 10","VM $v halt (pid $vpid)");
3023    }
3024    if (($cmt =~ /^VE/) && ($snapme != 0)) {
3025        if ($vetype eq "docker") {
3026            foreach my $f (split(/ +/,$src)) {
3027                unlink("$context/$f");
3028            }
3029        } else {
3030            pb_ve_snap($pbos,$tp);
3031        }
3032    }
3033}
3034
3035sub pb_script2v {
3036
3037my $pbscript=shift;
3038my $vtype=shift;
3039my $pbstep=shift;   # Which step are we in (0: create, 1: setup pb, 2: prep pbproj, 3: use to build)
3040my $pbforce=shift;  # Force stop of VM. Default not.
3041my $snapme=shift;   # Do we have to create a snapshot. Default not.
3042my $pbimage=shift;  # Which image to use to start the VM/VE
3043
3044$pbforce = 0 if (not defined $pbforce);
3045$snapme = 0 if (not defined $snapme);
3046$pbstep = 3 if (not defined $pbstep);
3047
3048pb_log(2,"DEBUG: pb_script2v($vtype,$pbstep,$pbforce,$snapme)\n");
3049pb_log(2,"DEBUG: pb_script2v(pbscript: ".Dumper($pbscript)."\n") if (defined $pbscript);
3050pb_parallel_launchv($pbscript,$vtype,uc($vtype)."Script",$pbstep,$pbimage,$pbforce,$snapme,$pbsnap);
3051}
3052
3053sub pb_parallel_launchv {
3054
3055my $pbscript=shift;
3056my $vtype = shift;
3057my $action = shift;         # It an action is defined then use send2target
3058my $pbstep=shift;           # Which step are we in (0: create, 1: setup, 2: build 3: use)
3059my $pbimage=shift;          # Which image to use to start the VM/VE
3060my $pbforce=shift;          # Force stop of VM. Default not.
3061my $snapme = shift;         # By default do not snap a VM/VE/RM
3062my $usesnap = shift;        # By default study the usage of the snapshot feature of VM/VE/RM   
3063my $vm;
3064my $all;
3065
3066$pbforce = 0 if (not defined $pbforce);
3067$snapme = 0 if (not defined $snapme);
3068$usesnap = 1 if (not defined $usesnap);
3069
3070# Adapt // mode to memory size
3071$pbparallel = pb_set_parallel($vtype);
3072
3073pb_log(2,"DEBUG: pb_parallel_launch(vtype,step,force,snapme,usesnap: $vtype,$pbstep,$pbforce,$snapme,$usesnap)\n");
3074pb_log(2,"DEBUG: pb_parallel_launch(pbscript: ".Dumper($pbscript)."\n");
3075
3076my ($vexist,$vpid) = (undef,undef);
3077my $pm;
3078my $all_ok = 1;
3079if (defined $pbparallel) {
3080    $pm = new Parallel::ForkManager($pbparallel);
3081
3082    # Set which port the VM/RM will use to communicate
3083    $pm->run_on_start(\&pb_set_port);
3084    $pm->run_on_finish(sub { my ($pid, $code, $id, $signal, $dump) = @_;
3085                            unless ($code == 0 && $signal == 0 && $dump == 0) {
3086                                $all_ok = 0;
3087                                pb_log(0,"ERROR: pid $pid failed\n");
3088                            }
3089                        });
3090}
3091
3092# Determine on which V to work
3093if (defined $ENV{'PBV'}) {
3094    @$vm = split(/,/,$ENV{'PBV'});
3095} else {
3096    ($vm,$all) = pb_get2v($vtype);
3097}
3098
3099$vetype = pb_ve_get_type($vetype);
3100if ((($vtype =~ /vm/) || (($vtype =~ /ve/) && ($vetype eq "docker"))) && (((not defined $pbimage) || ($pbimage eq "")) && ($pbstep == 0))) {
3101    confess "-i image parameter needed";
3102}
3103
3104my $counter = 0;
3105foreach my $v (@$vm) {
3106    $counter++;
3107    # Modulo 2 * pbparallel (to avoid synchronization problems)
3108    $counter = 1 if ((defined $pbparallel) && ($counter > 2 * $pbparallel));
3109    $pm->start($counter) and next if (defined $pbparallel);
3110   
3111    # If launched via script2v then get the script passed as the one for the that VM
3112    if (not defined $pbscript->{$v}) {
3113        $pbscript->{$v} = $pbscript->{'default'};
3114    }
3115    #
3116    # Launch a single operation
3117    ($vexist,$vpid) = pb_launchv($pbscript->{$v},$vtype,$v,$action,$pbstep,$pbforce,$snapme,$pbsnap,$pbimage);
3118   
3119    # Skip that VM/RM if something went wrong
3120    if (($vtype =~ /(v|r)m/) && ($vpid == 0) && ($vexist == 0)) {
3121        $pm->finish if (defined $pbparallel);
3122        next;
3123    }
3124
3125   
3126    $pm->finish if (defined $pbparallel);
3127}
3128$pm->wait_all_children if (defined $pbparallel);
3129die "Aborting, one or more of the children failed." if ((not $all_ok) && ($Global::pb_stop_on_error));
3130}
3131
3132sub pb_launchv {
3133
3134my $pbscript1=shift;        # Just the script name to execute, not the hash
3135my $vtype = shift;
3136my $v = shift;              # Only 1 VM/VE/RM treated here
3137my $action = shift;         # It an action is defined then use send2target
3138my $pbstep=shift;           # Which step are we in (0: create, 1: setup, 2: build 3: use)
3139my $pbforce=shift;          # Force stop of VM. Default not.
3140my $snapme = shift;         # By default do not snap a VM/VE/RM
3141my $usesnap = shift;            # By default study the usage of the snapshot feature of VM/VE/RM   
3142my $pbimage=shift;          # Which image to use to start the VM/VE
3143my $vexist = undef;
3144my $vpid = undef;
3145
3146$pbforce = 0 if (not defined $pbforce);
3147$snapme = 0 if (not defined $snapme);
3148$usesnap = 1 if (not defined $usesnap);
3149
3150# Check that v exists
3151die "No VM/VE/RM defined, unable to launch" if (not defined $v);
3152
3153# If creation or snapshot creation mode, no snapshot usable
3154if (($pbstep == 0) || ($snapme == 1)) {
3155    $usesnap = 0;
3156}
3157
3158pb_log(2,"DEBUG: pb_launchv(vtype,v,step,force,snapme,usesnap: $vtype,$v,$pbstep,$pbforce,$snapme,$usesnap)\n");
3159pb_log(2,"DEBUG: pb_launchv(pbimage: $pbimage)\n") if (defined $pbimage);
3160pb_log(2,"DEBUG: pb_launchv(pbscript $pbscript1)\n") if (defined $pbscript1);
3161pb_log(2,"DEBUG: pb_launchv(action: $action)\n") if (defined $action);
3162# Keep only the first VM in case many were given
3163if ($v =~ /,/) {
3164    pb_log(0,"WARNING: pruning to just the first of several VM/VE/RMs listed ($v)\n");
3165    $v =~ s/,.*//;
3166}
3167
3168my $pbos = pb_distro_get_context($v);
3169
3170pb_apply_conf_proxy($pbos);
3171
3172# Launch the VMs/VEs
3173if ($vtype eq "vm") {
3174    my ($ptr,$ptr2,$vmpath,$vmport,$vms) = pb_conf_get("vmtype","vmcmd","vmpath","vmport","vmsize");
3175    my ($vmopt,$vmmm,$vmtmout,$vmsnap,$vmbuildtm,$vmmonport) = pb_conf_get_if("vmopt","vmmem","vmtmout","vmsnap","vmbuildtm","vmmonport");
3176    my $vmsize = pb_distro_get_param($pbos,$vms);
3177
3178    my $vmtype = $ptr->{$ENV{'PBPROJ'}};
3179    my $vmcmd = $ptr2->{$ENV{'PBPROJ'}};
3180
3181    if (defined $opts{'g'}) {
3182        if (($vmtype eq "kvm") || ($vmtype eq "qemu")) {
3183            $ENV{'PBVMOPT'} = "--nographic";
3184        }
3185    }
3186    if (not defined $ENV{'PBVMOPT'}) {
3187        $ENV{'PBVMOPT'} = "";
3188    }
3189    # Save the current status for later restoration
3190    $ENV{'PBOLDVMOPT'} = $ENV{'PBVMOPT'};
3191
3192    if (defined $vmopt->{$v}) {
3193        $ENV{'PBVMOPT'} .= " $vmopt->{$v}" if ($ENV{'PBVMOPT'} !~ / $vmopt->{$v}/);
3194    } elsif (defined $vmopt->{$ENV{'PBPROJ'}}) {
3195        $ENV{'PBVMOPT'} .= " $vmopt->{$ENV{'PBPROJ'}}" if ($ENV{'PBVMOPT'} !~ / $vmopt->{$ENV{'PBPROJ'}}/);
3196    }
3197
3198    # How much memory to allocate for VMs
3199    if (defined $vmmm) {
3200        my $vmmem = pb_distro_get_param($pbos,$vmmm);
3201        if (defined $vmmem) {
3202            $ENV{'PBVMOPT'} .= " -m $vmmem";
3203        }
3204    }
3205
3206    # Are we allowed to use snapshot feature
3207    if ($usesnap == 1) {
3208        if ((defined $vmsnap->{$v}) && ($vmsnap->{$v} =~ /true/i)) {
3209            $ENV{'PBVMOPT'} .= " -snapshot";
3210        } elsif ((defined $vmsnap->{$ENV{'PBPROJ'}}) && ($vmsnap->{$ENV{'PBPROJ'}} =~ /true/i)) {
3211            $ENV{'PBVMOPT'} .= " -snapshot";
3212        } elsif ($pbsnap eq 1) {
3213            $ENV{'PBVMOPT'} .= " -snapshot";
3214        }
3215    } 
3216    if ($snapme != 0) {
3217        if (($vmtype eq "kvm") || ($vmtype eq "qemu")) {
3218            # Configure the monitoring to automate the creation of the 'pb' snapshot
3219            $ENV{'PBVMOPT'} .= " -serial mon:telnet::$vmmonport->{$ENV{'PBPROJ'}},server,nowait" if ((defined $vmmonport) && (defined $vmmonport->{$ENV{'PBPROJ'}}));
3220            # In that case no snapshot call needed
3221            $ENV{'PBVMOPT'} =~ s/ -snapshot//;
3222        }
3223    }
3224   
3225    # Set a default timeout of 2 minutes if none defined externally or in conf files
3226    if (not defined $ENV{'PBVMTMOUT'}) {
3227        if (defined $vmtmout->{$v}) {
3228            $ENV{'PBVMTMOUT'} = $vmtmout->{$v};
3229        } elsif (defined $vmtmout->{$ENV{'PBPROJ'}}) {
3230            $ENV{'PBVMTMOUT'} = $vmtmout->{$ENV{'PBPROJ'}};
3231        }
3232    }
3233    if (not defined $ENV{'PBVMTMOUT'}) {
3234        $ENV{'PBVMTMOUT'} = "120";
3235    }
3236
3237    my $nport = pb_get_port($vmport,$pbos,$vtype);
3238
3239    my $cmd;
3240    my $vmm;        # has to be used for pb_check_ps
3241    if (($vmtype eq "qemu") || ($vmtype eq "kvm")) {
3242        $vmm = "$vmpath->{$ENV{'PBPROJ'}}/$v.qemu";
3243        if (($pbstep == 0) || (defined $pbimage)) {
3244            $ENV{'PBVMOPT'} .= " -cdrom $pbimage -boot d";
3245        }
3246        # Always redirect the network and always try to use a 'pb' snapshot
3247        #$cmd = "$vmcmd $ENV{'PBVMOPT'} -net nic,model=virtio -net user,hostfwd=tcp::$nport:22 -loadvm pb $vmm"
3248        #$cmd = "$vmcmd $ENV{'PBVMOPT'} -redir tcp:$nport:10.0.2.15:22 $vmm"
3249        $cmd = "$vmcmd $ENV{'PBVMOPT'} -net nic -net user,hostfwd=tcp:127.0.0.1:$nport-:22 $vmm"
3250    } elsif ($vmtype eq "xen") {
3251    } elsif ($vmtype eq "vmware") {
3252    } else {
3253        die "VM of type $vmtype not supported. Report to the dev team";
3254    }
3255    # Restore the ENV VAR Value
3256    $ENV{'PBVMOPT'} = $ENV{'PBOLDVMOPT'};
3257
3258    my ($tmpcmd,$void) = split(/ +/,$cmd);
3259    my $vmmport = undef;
3260    ($vexist,$vmmport) = pb_check_ps($tmpcmd,$vmm);
3261    $vpid = 0;
3262   
3263    if (! $vexist) {
3264        if ($pbstep == 0) {
3265            die("Found an existing Virtual machine $vmm. Won't overwrite") if (-r $vmm);
3266            if (($vmtype eq "qemu") || ($vmtype eq "xen") || ($vmtype eq "kvm")) {
3267                my $command = pb_check_req("qemu-img",0);
3268                pb_system("$command create -f qcow2 $vmm $vmsize","Creating the QEMU VM");
3269            } elsif ($vmtype eq "vmware") {
3270            } else {
3271            }
3272        }
3273        if (! -f "$vmm") {
3274            pb_log(0,"Unable to find VM $vmm\n");
3275        } else {
3276            # Is the SSH port free? if not kill the existing process using it after a build timeout period
3277            my ($vmssh,$void) = pb_check_ps($tmpcmd,$vmm);
3278            if ($vmssh) {
3279                my $buildtm = $ENV{'PBVMTMOUT'};
3280                if (defined $vmbuildtm->{$v}) {
3281                    $buildtm = $vmbuildtm->{$v};
3282                } elsif (defined $vmbuildtm->{$ENV{'PBPROJ'}}) {
3283                    $buildtm = $vmbuildtm->{$ENV{'PBPROJ'}};
3284                }
3285
3286                sleep $buildtm;
3287                pb_log(0,"WARNING: Killing the process ($vmssh) using port $void (previous failed VM ?)\n");
3288                kill 15,$vmssh;
3289                # Let it time to exit
3290                sleep 5;
3291            }
3292            pb_system("$cmd &","Launching the VM $vmm");
3293            # Using system allows to kill it externaly if needed,sosupport that in the call
3294            pb_system("sleep $ENV{'PBVMTMOUT'}","Waiting $ENV{'PBVMTMOUT'} s for VM $v to come up","mayfail");
3295            ($vpid,$void) = pb_check_ps($tmpcmd,$vmm);
3296            pb_log(0,"VM $vmm launched (pid $vpid)\n");
3297        }
3298    } else {
3299        pb_log(0,"Found an existing VM $vmm (pid $vexist - Port $vmmport)\n");
3300        # Set the correct port here based on what is done
3301        $pbport = $vmmport;
3302        $vpid = $vexist;
3303    }
3304    pb_log(2,"DEBUG: pb_launchv returns ($vexist,$vpid)\n");
3305
3306    # After the VM is launched, then setup the SSH access if not in creation phase
3307    if ($pbstep != 0) {
3308        if (defined $pbport) {
3309            pb_ssh_setup($vtype,$pbport);
3310        } else {
3311            pb_ssh_setup($vtype,$nport);
3312        }
3313    }
3314   
3315    # Skip that VM if something went wrong
3316    return($vexist,$vpid) if (($vpid == 0) && ($vexist == 0));
3317   
3318    # If force stopping the VM then reset vexist
3319    if ($pbforce == 1) {
3320        $vpid = $vexist;
3321        $vexist = 0;
3322    }
3323} elsif ($vtype eq "ve") {
3324    # Force the creation of the VE and no snapshot usable
3325    $vetype = pb_ve_get_type($vetype);
3326    pb_ve_launch($v,$pbstep,$usesnap,$vetype,$pbimage);
3327    $vexist = 0;
3328    $vpid = 0;
3329} else {
3330    # Get RM context
3331    my ($ptr,$rmpath,$rmport) = pb_conf_get("rmtype","rmpath","rmport");
3332
3333    # Nothing more to do for RM. No real launch
3334    # For the moment we support the RM is already running
3335    # For ProLiant may be able to power them on if needed later on as an example.
3336    $vexist = 0;
3337    $vpid = 0;
3338   
3339    # After the RM is launched, then setup the SSH access
3340    my $nport = pb_get_port($rmport,$pbos,$vtype);
3341    pb_ssh_setup($vtype,$nport);
3342}
3343
3344# Gather all required files to send them to the VM/VE/RM
3345# and launch the build through pbscript
3346if (defined $action) {
3347    pb_log(2,"DEBUG: Before send2target, vexist: $vexist, vpid: $vpid, vtype: $vtype, v: $v, action: $action\n");
3348    pb_send2target($action,$pbscript1,$v,$vexist,$vpid,$snapme,$pbstep);
3349}
3350return($vexist,$vpid);
3351}
3352
3353# Return string for date synchro
3354sub pb_date2v {
3355
3356my $vtype = shift;
3357my $pbos = shift;
3358my $nontp = shift || 0;
3359
3360pb_log(1,"Entering pb_date2v\n");
3361# VE gets time from parent OS.
3362return "/bin/true" if ($vtype) =~ /^ve/o; 
3363
3364my ($ntp) = pb_conf_get_if($vtype."ntp");
3365my $vntp = undef;
3366my $ntpline = undef;
3367$vntp = $ntp->{$ENV{'PBPROJ'}} if (defined $ntp);
3368
3369if ((defined $vntp) && ($vntp !~ /^\s*$/)) {
3370    pb_log(2,"ntp server is $vntp\n");
3371    # ntp command depends on pbos
3372    my $vntpcmd = pb_distro_get_param($pbos,pb_conf_get("oscmdntp"));
3373    $ntpline = "sudo $vntpcmd $vntp";
3374}
3375# Force new date to be in the future compared to the date
3376# of the host by adding 1 minute
3377my @date=pb_get_date();
3378$date[1]++;
3379my $upddate = strftime("%m%d%H%M%Y", @date);
3380my $dateline = "sudo /bin/date $upddate";
3381if ((defined $ntpline) && ($nontp == 0)) {
3382    pb_log(1,"pb_date2v returns $ntpline\n");
3383    return($ntpline);
3384} else {
3385    pb_log(1,"pb_date2v returns $dateline\n");
3386    return($dateline);
3387}
3388}
3389
3390sub pb_build2v {
3391
3392my $vtype = shift;
3393my $action = shift;     #build, test or prep
3394my $pbforce=shift;      # Force stop of VM. Default not.
3395
3396$action = "build" if (not defined $action);
3397$pbforce = 0 if (not defined $pbforce);
3398
3399my $pbstep;
3400
3401$vetype = pb_ve_get_type($vetype);
3402if ($action eq "prep") {
3403    $pbstep = 2;
3404} else {
3405    $pbstep = 3;
3406    # Need to be traversed by others if using containers
3407    if (($vtype eq "ve") && ($vetype eq "docker")) {
3408        chmod 0751,"$ENV{'PBBUILDDIR'}";
3409    }
3410}
3411
3412pb_log(1,"Entering in pb_build2v ($action in $vtype) \n");
3413
3414# We prepare scripts for all V of this vtype to allow sharing that part
3415my ($v,$all) = pb_get2v($vtype);
3416
3417# Send tar files when we do a global generation
3418pb_build2ssh() if (($all == 1) && ($action eq "build"));
3419
3420# Adapt // mode to memory size
3421$pbparallel = pb_set_parallel($vtype);
3422
3423my ($vexist,$vpid) = (undef,undef);
3424my $pm;
3425my $all_ok = 1;
3426if (defined $pbparallel) {
3427    $pm = new Parallel::ForkManager($pbparallel);
3428
3429    # Set which port the VM/RM will use to communicate
3430    $pm->run_on_start(\&pb_set_port);
3431    $pm->run_on_finish(sub { my ($pid, $code, $id, $signal, $dump) = @_;
3432                            unless ($code == 0 && $signal == 0 && $dump == 0) {
3433                                $all_ok = 0;
3434                                pb_log(0,"ERROR: pid $pid failed\n");
3435                            }
3436                        });
3437}
3438
3439my $counter = 0;
3440foreach my $v (@$v) {
3441    $counter++;
3442    $pbscript{$v} = "$ENV{'PBDESTDIR'}/pb$action-$v";
3443    # Modulo 2 * pbparallel (to avoid synchronization problems)
3444    $counter = 1 if ((defined $pbparallel) && ($counter > 2 * $pbparallel));
3445    $pm->start($counter) and next if (defined $pbparallel);
3446    #
3447    # Prepare the script to be executed on the VM/VE/RM
3448    open(SCRIPT,"> $pbscript{$v}") || die "Unable to create $pbscript{$v}";
3449    print SCRIPT "#!/bin/bash\n";
3450
3451    # Transmit the verbosity level to the virtual env/mach.
3452    my $verbose = "";
3453    my $i = 0;                          # minimal debug level
3454    while ($i lt $pbdebug) {
3455        $verbose .= "-v ";
3456        $i++;
3457    }
3458    print SCRIPT "set -e\n" if ($Global::pb_stop_on_error);
3459    # Activate script verbosity if at least 2 for pbdebug
3460    print SCRIPT "set -x\n" if ($i gt 1);
3461    # Quiet if asked to be so on the original system
3462    $verbose = "-q" if ($pbdebug eq -1);
3463
3464    print SCRIPT "echo ... Execution needed\n";
3465    print SCRIPT "echo ==== Start of script for $vtype $v =====\n";
3466    print SCRIPT "# This is in directory pbdelivery\n";
3467    print SCRIPT "# Setup the variables required for building\n";
3468    print SCRIPT "export PBPROJ=$ENV{'PBPROJ'}\n";
3469
3470    if (($action eq "build") || ($action eq "prep")) {
3471        print SCRIPT "# Preparation for pb\n";
3472        print SCRIPT "rm -f \$HOME/.pbrc.yml\n";
3473        print SCRIPT "rm -f \$HOME/.pbrc\n";
3474        print SCRIPT "cp .pbrc.yml \$HOME\n";
3475        # TODO: Why ?
3476        print SCRIPT "cd ..\n";
3477    }
3478
3479    # VE needs a good /proc, tolerate one being potentially left around after a failure
3480    if (($vtype eq "ve") && ($vetype ne "docker")) {
3481        print SCRIPT "[ -d /proc/1 ] || sudo /bin/mount -t proc /proc /proc\n";
3482    }
3483
3484    # Get distro context
3485    my $pbos = pb_distro_get_context($v);
3486   
3487    my $ntpline = pb_date2v($vtype,$pbos);
3488    print SCRIPT "# Time sync\n";
3489    print SCRIPT "echo setting up date with $ntpline\n";
3490    print SCRIPT "$ntpline\n";
3491    print SCRIPT "if [ \$\? -ne 0 ]; then\n";
3492    # Force if bad status a trick using date
3493    $ntpline = pb_date2v($vtype,$pbos, 1);
3494    print SCRIPT "  $ntpline\n";
3495    print SCRIPT "fi\n";
3496    # Use potential local proxy declaration in case we need it to download repo, pkgs, ...
3497    if (defined $ENV{'http_proxy'}) {
3498        print SCRIPT "export http_proxy=\"$ENV{'http_proxy'}\"\n";
3499    }
3500    if (defined $ENV{'https_proxy'}) {
3501        print SCRIPT "export https_proxy=\"$ENV{'https_proxy'}\"\n";
3502    }
3503    if (defined $ENV{'ftp_proxy'}) {
3504        print SCRIPT "export ftp_proxy=\"$ENV{'ftp_proxy'}\"\n";
3505    }
3506
3507    # Get list of packages to build/test and get some ENV vars as well
3508    my $ptr = pb_get_pkg();
3509    @pkgs = @$ptr;
3510    my $p = join(' ',@pkgs) if (@pkgs);
3511    print SCRIPT "export PBPROJVER=$ENV{'PBPROJVER'}\n";
3512    print SCRIPT "export PBPROJTAG=$ENV{'PBPROJTAG'}\n";
3513    print SCRIPT "export PBPACKAGER=\"$ENV{'PBPACKAGER'}\"\n";
3514    # TODO: setup apk pkg/keys here
3515
3516    # We may need to do some other tasks before building. Read a script here to finish setup
3517    if (-x "$ENV{'PBTMP'}/pb$vtype".".pre") {
3518        print SCRIPT "# Special pre-instructions to be launched\n";
3519        print SCRIPT pb_get_content("$ENV{'PBTMP'}/pb$vtype".".pre");
3520    }
3521
3522    if (-x "$ENV{'PBTMP'}/pb$vtype"."$action.pre") {
3523        print SCRIPT "# Special pre-$action instructions to be launched\n";
3524        print SCRIPT pb_get_content("$ENV{'PBTMP'}/pb$vtype"."$action.pre");
3525    }
3526
3527    print SCRIPT "# $action\n";
3528    print SCRIPT "echo $action"."ing packages on $vtype...\n";
3529
3530    if (($action eq "test") && (! -x "$ENV{'PBDESTDIR'}/pbtest")) {
3531        confess "No test script ($ENV{'PBDESTDIR'}/pbtest) found when in test mode. Aborting ...";
3532    }
3533    print SCRIPT "pb --version\n";
3534    my $act = "$action"."2pkg";
3535    $act = "build2prep" if ($action eq "prep");
3536    print SCRIPT "pb $verbose -p $ENV{'PBPROJ'} $act $p\n";
3537
3538    if (($vtype eq "ve") && ($vetype ne "docker")) {
3539        print SCRIPT "sudo /bin/umount /proc\n";
3540    }
3541
3542    # We may need to do some other tasks after building. Read a script here to exit properly
3543    if (-x "$ENV{'PBTMP'}/pb$vtype"."$action.post") {
3544        print SCRIPT "# Special post-$action instructions to be launched\n";
3545        print SCRIPT pb_get_content("$ENV{'PBTMP'}/pb$vtype"."$action.post");
3546    }
3547
3548    if (-x "$ENV{'PBTMP'}/pb$vtype".".post") {
3549        print SCRIPT "# Special post-instructions to be launched\n";
3550        print SCRIPT pb_get_content("$ENV{'PBTMP'}/pb$vtype".".post");
3551    }
3552
3553    print SCRIPT "echo ==== End of script $$ for $vtype $v =====\n";
3554    close(SCRIPT);
3555    chmod 0755,"$pbscript{$v}";
3556   
3557    $pm->finish if (defined $pbparallel);
3558}
3559$pm->wait_all_children if (defined $pbparallel);
3560die "Aborting, one or more of the children failed." if ((not $all_ok) && ($Global::pb_stop_on_error));
3561
3562# Launch the VM/VE/RM
3563# docker image to use if needed
3564# Gather all required files to send them to the VM/VE
3565# and launch the build through pbscript
3566pb_parallel_launchv(\%pbscript,$vtype,uc($vtype).$action,$pbstep,$pbimage,$pbforce);
3567}
3568
3569
3570sub pb_clean {
3571
3572    my $sleep=10;
3573    die "Unable to get env var PBDESTDIR" if (not defined $ENV{'PBDESTDIR'});
3574    die "Unable to get env var PBBUILDDIR" if (not defined $ENV{'PBBUILDDIR'});
3575    pb_log(0,"We will now wait $sleep s before removing both directories\n$ENV{'PBDESTDIR'} and $ENV{'PBBUILDDIR'}.\nPlease break me if this is wrong\n");
3576    sleep $sleep;
3577    pb_rm_rf($ENV{'PBDESTDIR'});
3578    pb_rm_rf($ENV{'PBBUILDDIR'});
3579}
3580
3581sub pb_newver {
3582
3583    die "-V Version parameter needed" if ((not defined $newver) || ($newver eq ""));
3584
3585    # Need this call for PBDIR
3586    my ($scheme2,$uri) = pb_vcs_init($pbinit);
3587
3588    my ($pbconf,$pbprojurl) = pb_conf_get("pbconfurl","pbprojurl");
3589    $uri = $pbconf->{$ENV{'PBPROJ'}};
3590    my ($scheme, $account, $host, $port, $path) = pb_get_uri($uri);
3591
3592    # Checking CMS repositories status
3593    ($scheme2, $account, $host, $port, $path) = pb_get_uri($pbprojurl->{$ENV{'PBPROJ'}});
3594
3595    if ($scheme !~ /^svn/) {
3596        die "Only SVN is supported at the moment";
3597    }
3598
3599    my $res = pb_vcs_isdiff($scheme,$ENV{'PBROOTDIR'});
3600    die "ERROR: No differences accepted in CMS for $ENV{'PBROOTDIR'} before creating a new version" if ($res != 0);
3601
3602    $res = pb_vcs_isdiff($scheme2,$ENV{'PBDIR'});
3603    die "ERROR: No differences accepted in CMS for $ENV{'PBDIR'} before creating a new version" if ($res != 0);
3604
3605    # Tree identical between PBCONFDIR and PBROOTDIR. The delta is what
3606    # we want to get for the root of the new URL
3607
3608    my $oldver = $ENV{'PBROOTDIR'};
3609    $oldver =~ s|^$ENV{'PBCONFDIR'}||;
3610
3611    pb_log(2, "PBCONFDIR: $ENV{'PBCONFDIR'}\nPBROOTDIR: $ENV{'PBROOTDIR'}\n");
3612
3613    my $newurl = "$uri/$newver";
3614    # Should probably use projver in the old file
3615    my $oldvertxt= basename($oldver);
3616    my $newvertxt = basename($newver);
3617
3618    # Duplicate and extract project-builder part
3619    pb_log(2,"Copying $uri/$oldver to $newurl\n");
3620    pb_vcs_copy($scheme,"$uri/$oldver",$newurl);
3621    pb_log(2,"Checkout $newurl to $ENV{'PBCONFDIR'}/$newver\n");
3622    pb_vcs_up($scheme,"$ENV{'PBCONFDIR'}");
3623
3624    # Duplicate and extract project
3625    my $newurl2 = "$pbprojurl->{$ENV{'PBPROJ'}}/$newver";
3626
3627    pb_log(2,"Copying $pbprojurl->{$ENV{'PBPROJ'}}/$oldver to $newurl2\n");
3628    pb_vcs_copy($scheme2,"$pbprojurl->{$ENV{'PBPROJ'}}/$oldver",$newurl2);
3629
3630    my $tmp = $ENV{'PBDIR'};
3631    $tmp =~ s|$oldver$||;
3632    pb_log(2,"Checkout $newurl2 to $tmp/$newver\n");
3633    pb_vcs_up($scheme2,"$tmp");
3634
3635    # Update the .yml file
3636    open(FILE,"$ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml") || die "Unable to open $ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml";
3637    open(OUT,"> $ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml.new") || die "Unable to write to $ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml.new";
3638    while(<FILE>) {
3639        if (/^projver\s+$ENV{'PBPROJ'}\s*=\s*$oldvertxt$/) {
3640            s/^projver\s+$ENV{'PBPROJ'}\s*=\s*$oldvertxt$/projver $ENV{'PBPROJ'} = $newvertxt/;
3641            pb_log(0,"Changing projver from $oldvertxt to $newvertxt in $ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml\n");
3642        }
3643        if (/^testver/) {
3644            s/^testver/#testver/;
3645            pb_log(0,"Commenting testver in $ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml\n") if (/^testver/);
3646        }
3647        if (/^delivery/) {
3648            my $txt = $_;
3649            chomp($txt);
3650            pb_log(0,"Please check delivery ($txt) in $ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml\n");
3651        }
3652        print OUT $_;
3653    }
3654    close(FILE);
3655    close(OUT);
3656    rename("$ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml.new","$ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml");
3657
3658    # Checking pbcl files
3659    foreach my $f (<$ENV{'PBROOTDIR'}/*/pbcl>) {
3660        # Compute new pbcl file
3661        my $f2 = $f;
3662        $f2 =~ s|$ENV{'PBROOTDIR'}|$ENV{'PBCONFDIR'}/$newver/|;
3663        open(PBCL,$f) || die "Unable to open $f";
3664        my $foundnew = 0;
3665        while (<PBCL>) {
3666            $foundnew = 1 if (/^$newvertxt \(/);
3667        }
3668        close(PBCL);
3669        open(OUT,"> $f2") || die "Unable to write to $f2: $!";
3670        open(PBCL,$f) || die "Unable to open $f";
3671        # We skip as long as we do not reach the new version in the log (case of devel)
3672        my $skip = 1;
3673        # We let the 4 first lines
3674        my $cnt = 0;
3675        while (<PBCL>) {
3676            $skip = 0 if (/^$newvertxt \(/);
3677            if ((/^$oldvertxt \(/) && ($foundnew == 0)) {
3678                print OUT "$newvertxt ($pbdate)\n";
3679                print OUT "- TBD\n";
3680                print OUT "\n";
3681                pb_log(0,"WARNING: version $newvertxt not found in $f so added to $f2...\n") if ($foundnew == 0);
3682            }
3683            if ($cnt <= 4) {
3684                print OUT "$_" if (not /^$oldvertxt \(/);
3685            } else {
3686                print OUT "$_" if ($skip == 0);
3687            }
3688            $cnt++;
3689        }
3690        close(OUT);
3691        close(PBCL);
3692    }
3693
3694    pb_log(2,"Checkin $ENV{'PBCONFDIR'}/$newver\n");
3695    pb_vcs_checkin($scheme,"$ENV{'PBCONFDIR'}/$newver","updated to $ENV{'PBCONFDIR'}");
3696}
3697
3698#
3699# Return the list of VMs/VEs/RMs we are working on
3700# $all is a flag to know if we return all of them
3701# or only some (if all we publish also tar files in addition to pkgs
3702#
3703sub pb_get2v {
3704
3705my $vtype = shift;
3706my @v;
3707my $all = 0;
3708my $pbv = 'PBV';
3709my $vlist = $vtype."list";
3710
3711# Get VM/VE list
3712if ((not defined $ENV{$pbv}) || ($ENV{$pbv} =~ /^all$/)) {
3713    my ($ptr) = pb_conf_get($vlist);
3714    $ENV{$pbv} = $ptr->{$ENV{'PBPROJ'}};
3715    $all = 1;
3716}
3717pb_log(2,"$vtype: $ENV{$pbv}\n");
3718@v = split(/,/,$ENV{$pbv});
3719return(\@v,$all);
3720}
3721
3722# This function creates a giant script to configure a particular VM/VE/RM, it then copies the
3723# script to the target.
3724
3725# Function to create a potentialy missing pb account on the VM/VE/RM, and adds it to sudo
3726# Needs to use root account to connect to the VM/VE/RM
3727# pb will take your local public SSH key to access
3728# the pb account in the VM/VE/RM later on if needed
3729sub pb_setup2v {
3730
3731my $vtype = shift;
3732my $sbx = shift;
3733my $pbstep = 1;
3734
3735my ($vm,$all) = pb_get2v($vtype);
3736
3737# Adapt // mode to memory size
3738$pbparallel = pb_set_parallel($vtype);
3739
3740my $pm;
3741my $all_ok = 1;
3742if (defined $pbparallel) {
3743    $pm = new Parallel::ForkManager($pbparallel);
3744
3745    # Set which port the VM/RM will use to communicate
3746    $pm->run_on_start(\&pb_set_port);
3747    $pm->run_on_finish(sub { my ($pid, $code, $id, $signal, $dump) = @_;
3748                            $all_ok = 0 unless (($code == 0) && ($signal == 0) && ($dump == 0)); 
3749                        });
3750}
3751
3752my $counter = 0;
3753foreach my $v (@$vm) {
3754    # Script generated - should be before the fork - after the %pbscript hash isn't updated correctly
3755    $pbscript{$v} = "$ENV{'PBDESTDIR'}/setupv-$v";
3756
3757    $counter++;
3758    if (defined $pbparallel) {
3759        # Modulo 2 * pbparallel (to avoid synchronization problems)
3760        $counter = 1 if ($counter > 2 * $pbparallel);
3761        $pm->start($counter) and next;
3762    }
3763
3764    # Get distro context
3765    my $pbos = pb_distro_get_context($v);
3766   
3767    # Deal with date sync.
3768    my $ntpline = pb_date2v($vtype,$pbos);
3769
3770    # Name of the account to deal with for VM/VE/RM
3771    # Do not use the one passed potentially with -a
3772    my ($pbac) = pb_conf_get($vtype."login");
3773    confess "No $vtype"."login defined in your project yml conf file" if ((not defined $pbac) || (not defined $pbac->{$ENV{'PBPROJ'}}));
3774
3775    my $key = "";
3776
3777    # Prepare the script to be executed on the VM/VE/RM
3778    # in $ENV{'PBDESTDIR'}/setupv
3779
3780    pb_log(2,"Preparing setup script for $v in $pbscript{$v}\n");
3781    open(SCRIPT,"> $pbscript{$v}") || die "Unable to create $pbscript{$v}";
3782   
3783    print SCRIPT << 'EOF';
3784#!/usr/bin/perl -w
3785
3786use strict;
3787use File::Copy;
3788
3789# We should not need in this script more functions than what is provided
3790# by Base, Conf and Distribution to avoid problems at exec time.
3791# They are appended at the end.
3792
3793# Define mandatory global vars
3794EOF
3795    my $ppref = "our";
3796    $ppref = "my" if (($pbos->{'name'} =~ /^redhat/) && ($pbos->{'version'} =~ /^6/));
3797    print SCRIPT << "EOF";
3798$ppref \$pbdebug = $pbdebug;
3799$ppref \$pbLOG;
3800$ppref \$pbsynmsg = "pbscript";
3801$ppref \$pbdisplaytype = "text";
3802$ppref \$pblocale = "";
3803$ppref \$PBCONFVER = $projectbuilderconfver;
3804EOF
3805    print SCRIPT << 'EOF';
3806pb_log_init($pbdebug, $pbLOG);
3807EOF
3808    print SCRIPT << "EOF";
3809\$Global::pb_stop_on_error = $Global::pb_stop_on_error;
3810pb_temp_init($pbkeep);
3811pb_conf_init("$ENV{'PBPROJ'}");
3812
3813EOF
3814
3815    if ($vtype eq "ve") {
3816        $vetype = pb_ve_get_type($vetype);
3817        if ($vetype ne "docker") {
3818            print SCRIPT << "EOF";
3819# For VE we first need to mount some FS
3820pb_system("mount -t proc /proc /proc") unless (-d "/proc/$$");
3821EOF
3822        }
3823        print SCRIPT << "EOF";
3824# For VE we need a good null dev
3825# Except for RedHat 6.2 where it's good and doesn't like being recreated
3826EOF
3827        print SCRIPT "pb_system('rm -f /dev/null; mknod /dev/null c 1 3; chmod 777 /dev/null');\n" unless (($pbos->{'name'} =~ /^redhat/) && ($pbos->{'version'} =~ /^6/));
3828    } elsif ($vtype =~ /(v|r)m/) {
3829        # Nothing to do now.
3830        my $useless = 0;
3831    } else {
3832        die "Unknown virtual type $vtype";
3833    }
3834
3835    print SCRIPT << 'EOF';
3836
3837# Adds $pbac->{$ENV{'PBPROJ'}} as an account if needed
3838#
3839my $file="/etc/passwd";
3840open(PBFILE,$file) || die "Unable to open $file";
3841my $found = 0;
3842while (<PBFILE>) {
3843EOF
3844    print SCRIPT << "EOF";
3845    \$found = 1 if (/^$pbac->{$ENV{'PBPROJ'}}:/);
3846EOF
3847
3848# TODO: use an external parameter
3849my $home = "/home";
3850# Solaris doesn't like that we use /home
3851$home = "/export/home" if ($pbos->{'type'} eq "pkg");
3852
3853    print SCRIPT << "EOF";
3854}
3855close(PBFILE);
3856
3857if ( \$found == 0 ) {
3858    if ( ! -d "$home" ) {
3859        pb_mkdir_p("$home");
3860    }
3861EOF
3862    # TODO: Level of portability of these cmds ? Critical now for RM
3863    # TODO: use an external parameter
3864    print SCRIPT << "EOF";
3865    # Most Linux distros
3866    if ( -x "/usr/sbin/groupadd" ) {
3867        pb_system("/usr/sbin/groupadd $pbac->{$ENV{'PBPROJ'}}","Adding group $pbac->{$ENV{'PBPROJ'}}");
3868    # Alpine with busybox
3869    } elsif ( -x "/usr/sbin/addgroup" ) {
3870        pb_system("/usr/sbin/addgroup $pbac->{$ENV{'PBPROJ'}}","Adding group $pbac->{$ENV{'PBPROJ'}}");
3871    } else {
3872        die "Unable to find a command to add a group on this distribution";
3873    }
3874    # Most Linux distros
3875    if ( -x "/usr/sbin/useradd" ) {
3876        pb_system("/usr/sbin/useradd -g $pbac->{$ENV{'PBPROJ'}} -m -d $home/$pbac->{$ENV{'PBPROJ'}} $pbac->{$ENV{'PBPROJ'}}","Adding user $pbac->{$ENV{'PBPROJ'}} (group $pbac->{$ENV{'PBPROJ'}} - home $home/$pbac->{$ENV{'PBPROJ'}})");
3877    # Alpine with busybox
3878    } elsif ( -x "/usr/sbin/adduser" ) {
3879        pb_system("/usr/sbin/adduser -G $pbac->{$ENV{'PBPROJ'}} -h $home/$pbac->{$ENV{'PBPROJ'}} -D $pbac->{$ENV{'PBPROJ'}}","Adding user $pbac->{$ENV{'PBPROJ'}} (group $pbac->{$ENV{'PBPROJ'}} - home $home/$pbac->{$ENV{'PBPROJ'}})");
3880        pb_system("/usr/sbin/addgroup $pbac->{$ENV{'PBPROJ'}} abuild","Adding $pbac->{$ENV{'PBPROJ'}} to group abuild");
3881    } else {
3882        die "Unable to find a command to add a user on this distribution";
3883    }
3884}
3885EOF
3886
3887    # Copy the content of our local conf file to the VM/VE/RM
3888    my $content = pb_get_content(pb_distro_conffile());
3889    print SCRIPT << "EOF";
3890    #
3891    # Create a temporary local conf file for distribution support
3892    # This is created here before its use later. Its place is hardcoded, so no choice for the path
3893    #
3894    my \$tempconf = pb_distro_conffile();
3895    pb_mkdir_p(dirname(\$tempconf));
3896    open(CONF,"> \$tempconf") || die "Unable to create \$tempconf";
3897    print CONF q{$content};
3898    close(CONF);
3899EOF
3900
3901    if ($vtype =~ /(v|r)m/) {
3902        print SCRIPT << "EOF";
3903# allow ssh entry to build
3904#
3905mkdir "$home/$pbac->{$ENV{'PBPROJ'}}/.ssh",0700;
3906# Allow those accessing root to access the build account
3907copy("\$ENV{'HOME'}/.ssh/authorized_keys","$home/$pbac->{$ENV{'PBPROJ'}}/.ssh/authorized_keys");
3908chmod 0600,".ssh/authorized_keys";
3909pb_system("chown -R $pbac->{$ENV{'PBPROJ'}}:$pbac->{$ENV{'PBPROJ'}} $home/$pbac->{$ENV{'PBPROJ'}}","Finish setting up the account env for $pbac->{$ENV{'PBPROJ'}}");
3910
3911EOF
3912}
3913    print SCRIPT << 'EOF';
3914# No passwd for build account only keys
3915$file="/etc/shadow";
3916if (-f $file) {
3917    open(PBFILE,$file) || die "Unable to open $file";
3918    open(PBOUT,"> $file.new") || die "Unable to open $file.new";
3919    while (<PBFILE>) {
3920EOF
3921    print SCRIPT << "EOF";
3922        s/^$pbac->{$ENV{'PBPROJ'}}:\!\!:/$pbac->{$ENV{'PBPROJ'}}:*:/;
3923        s/^$pbac->{$ENV{'PBPROJ'}}:\!:/$pbac->{$ENV{'PBPROJ'}}:*:/; #SLES 9 e.g.
3924        s/^$pbac->{$ENV{'PBPROJ'}}:\\*LK\\*:/$pbac->{$ENV{'PBPROJ'}}:NP:/;  #Solaris e.g.
3925EOF
3926        print SCRIPT << 'EOF';
3927        print PBOUT $_;
3928    }
3929    close(PBFILE);
3930    close(PBOUT);
3931    rename("$file.new",$file);
3932    chmod 0640,$file;
3933    }
3934
3935# TODO: systemd support
3936# Keep the VM in text mode
3937$file="/etc/inittab";
3938if (-f $file) {
3939    open(PBFILE,$file) || die "Unable to open $file";
3940    open(PBOUT,"> $file.new") || die "Unable to open $file.new";
3941    while (<PBFILE>) {
3942        s/^(..):5:initdefault:$/$1:3:initdefault:/;
3943        print PBOUT $_;
3944    }
3945    close(PBFILE);
3946    close(PBOUT);
3947    rename("$file.new",$file);
3948    chmod 0640,$file;
3949}
3950
3951# pb has to be added to portage group on gentoo
3952
3953# We need to have that pb_distro_get_context function
3954# Get it from ProjectBuilder::Distribution
3955# And we now need the conf file required for this to work created above
3956
3957my $pbos = pb_distro_get_context(); 
3958print "distro tuple: ".Dumper($pbos)."\n";
3959
3960# Adapt sudoers
3961# sudo is not default on Solaris and needs to be installed first
3962# from http://www.sunfreeware.com/programlistsparc10.html#sudo
3963if ($pbos->{'type'} eq "pkg") {
3964    $file="/usr/local/etc/sudoers";
3965} else {
3966    $file="/etc/sudoers";
3967}
3968open(PBFILE,$file) || die "Unable to open $file";
3969open(PBOUT,"> $file.new") || die "Unable to open $file.new";
3970while (<PBFILE>) {
3971EOF
3972    # Skip what will be generated
3973    print SCRIPT << "EOF";
3974    next if (/^$pbac->{$ENV{'PBPROJ'}}\\s+/);
3975    next if (/^Defaults:$pbac->{$ENV{'PBPROJ'}}\\s+/);
3976    next if (/^Defaults:root\\s+\!requiretty/);
3977    next if (/^Defaults:root\\s+env_keep/);
3978EOF
3979    print SCRIPT << 'EOF';
3980    s/Defaults\s+requiretty/# disable Defaults: requiretty/;
3981    print PBOUT $_;
3982}
3983close(PBFILE);
3984EOF
3985    print SCRIPT << "EOF";
3986# Some distro force requiretty at compile time, so disable here
3987print PBOUT "Defaults:$pbac->{$ENV{'PBPROJ'}} !requiretty\\n";
3988print PBOUT "Defaults:root !requiretty\\n";
3989EOF
3990    # RH 6.2 sudo doesn't support env_keep
3991    if (($pbos->{'name'} ne "redhat") && ($pbos->{'version'} ne "6.2")) {
3992        print SCRIPT << "EOF";
3993# Keep proxy configuration while using sudo
3994print PBOUT "Defaults:$pbac->{$ENV{'PBPROJ'}}    env_keep += \\\"http_proxy https_proxy ftp_proxy\\\"\\n";
3995print PBOUT "Defaults:root    env_keep += \\\"http_proxy https_proxy ftp_proxy\\\"\\n";
3996EOF
3997    }
3998    # Try to restrict security to what is really needed
3999    if ($vtype =~ /^vm/) {
4000        my $hpath = pb_distro_get_param($pbos,pb_conf_get("ospathcmd-halt"));
4001        my @sudocmds = pb_get_sudocmds($pbos,$ntpline,"sudo $hpath","sudo /bin/date");
4002        print SCRIPT << "EOF";
4003# This is needed in order to be able on VM to halt the machine from the $pbac->{$ENV{'PBPROJ'}} account at least
4004# Build account $pbac->{$ENV{'PBPROJ'}} in VM also needs to setup date and install deps.
4005# Nothing else should be needed
4006EOF
4007        foreach my $c (@sudocmds) {
4008            print SCRIPT "print PBOUT \"$pbac->{$ENV{'PBPROJ'}}   ALL = NOPASSWD: $c\\n\";\n";
4009        }
4010    } elsif ($vtype =~ /^rm/) {
4011        my @sudocmds = pb_get_sudocmds($pbos,$ntpline);
4012        print SCRIPT << "EOF";
4013# Build account $pbac->{$ENV{'PBPROJ'}} in RM only needs to setup date and install deps if needed each time
4014EOF
4015        foreach my $c (@sudocmds) {
4016            print SCRIPT "print PBOUT \"$pbac->{$ENV{'PBPROJ'}}   ALL = NOPASSWD: $c\\n\";\n";
4017        }
4018    } else {
4019        print SCRIPT << "EOF";
4020# Build account $pbac->{$ENV{'PBPROJ'}} for VE needs to do a lot in the host (and chroot), so allow without restriction for now
4021print PBOUT "$pbac->{$ENV{'PBPROJ'}}   ALL=(ALL) NOPASSWD:ALL\n";
4022EOF
4023    }
4024    my $sudomode = pb_distro_get_param($pbos,pb_conf_get("ossudoersmode"));
4025    print SCRIPT << 'EOF';
4026close(PBOUT);
4027rename("$file.new",$file);
4028EOF
4029    print SCRIPT << "EOF";
4030chmod 0$sudomode,\$file;
4031
4032EOF
4033
4034    if ($vtype =~ /(v|r)m/) {
4035        # Sync date
4036        # do it after sudoers is setup
4037        print SCRIPT "my \$res = pb_system(\"$ntpline\",\"Updating time with $ntpline\",\"mayfailverbose\");\n";
4038        print SCRIPT "if (\$res != 0) {\n";
4039        # try again with another method
4040        $ntpline = pb_date2v($vtype,$pbos,1);
4041        print SCRIPT "  pb_system(\"$ntpline\",\"Updating time with $ntpline\",\"mayfailverbose\");\n";
4042        print SCRIPT "}\n";
4043    }
4044
4045    my ($addrepo) = pb_conf_read("$ENV{'PBDESTDIR'}/pbrc.yml","addrepo");
4046    my $pbrepo = pb_distro_get_param($pbos,$addrepo);
4047    print SCRIPT << 'EOF';
4048   
4049# We may need a proxy configuration. Get it from the local env
4050pb_apply_conf_proxy($pbos);
4051
4052# First install all required packages
4053pb_system("dnf clean all","Cleaning dnf env","mayfail") if (($pbos->{'name'} eq "fedora") && ($pbos->{'version'} >= 22));
4054pb_system("yum clean all","Cleaning yum env","mayfail") if ((($pbos->{'name'} eq "fedora") && ($pbos->{'version'} < 22)) || ($pbos->{'name'} eq "asianux") || ($pbos->{'name'} eq "rhel"));
4055my ($ospkgdep) = pb_conf_get_if("ospkgdep");
4056
4057my $pkgdep = pb_distro_get_param($pbos,$ospkgdep);
4058my $forcerepo;
4059EOF
4060    print SCRIPT "\$forcerepo = pb_distro_setuprepo_gen(\$pbos,\"$pbrepo\");\n" if ((defined $pbrepo) && ($pbrepo ne ""));
4061    print SCRIPT << 'EOF';
4062pb_distro_installdeps(undef,$pbos,$forcerepo,pb_distro_only_deps_needed($pbos,join(' ',split(/,/,$pkgdep))));
4063
4064EOF
4065    my $itype = pb_distro_get_param($pbos,pb_conf_get("pbinstalltype"));
4066    # Install from sandbox mean a file base install
4067    $itype = "file" if (defined $sbx);
4068    if ($itype =~ /^file/) {
4069        my $cmdget;
4070        my ($pbver1,$pbver2,$pbextdir);
4071        my ($tmp1,$tmp2);
4072        if (defined $sbx) {
4073            # Install from sandbox mean using the result of the just passed sbx2build command
4074            # Get content saved in cms2build
4075            my ($pkg) = pb_conf_read("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml","pbpkg");
4076            $pbextdir = pb_get_extdir();
4077            die "Unable to get package list" if (not defined $pkg);
4078
4079            # We consider 2 specific packages
4080            my $vertag1 = $pkg->{"ProjectBuilder"};
4081            my $vertag2 = $pkg->{"project-builder"};
4082            # get the version of the current package - maybe different
4083            pb_log(2,"Vertag1: $vertag1\n");
4084            pb_log(2,"Vertag2: $vertag2\n");
4085            ($pbver1,$tmp1) = split(/-/,$vertag1);
4086            ($pbver2,$tmp2) = split(/-/,$vertag2);
4087            $cmdget = "mv /tmp/ProjectBuilder-$pbver1$pbextdir.tar.gz ProjectBuilder-latest.tar.gz ; mv /tmp/project-builder-$pbver2$pbextdir.tar.gz project-builder-latest.tar.gz";
4088        } else {
4089            # TODO: shouldn't we have a variable to specify the source file?
4090            # And shouldn't we be getting the one that matches the current pb version?
4091            # For the moment, just take the latest stable one
4092            my $wget = pb_check_req("wget",0);
4093            $cmdget = "$wget --passive-ftp ftp://ftp.project-builder.org/src/ProjectBuilder-latest.tar.gz; $wget --passive-ftp ftp://ftp.project-builder.org/src/project-builder-latest.tar.gz";
4094        }
4095        print SCRIPT << 'EOF';
4096# Then install manually the missing perl modules
4097my ($osperldep,$osperlver) = pb_conf_get_if("osperldep","osperlver");
4098   
4099my $perldep = pb_distro_get_param($pbos,$osperldep);
4100my $bashopt = "";
4101$bashopt = "-x" if ($pbdebug ge 1);
4102my $verbopt = "mayfail";
4103$verbopt = "mayfailverbose" if ($pbdebug ge 1);
4104foreach my $m (split(/,/,$perldep)) {
4105    # Skip empty deps
4106    next if ($m =~ /^\s*$/);
4107    my $dir = $m;
4108    $dir =~ s/-.*//;
4109    pb_system("echo \"rm -rf $m* ; wget http://search.cpan.org/CPAN/modules/by-module/$dir/$m-$osperlver->{$m}.tar.gz ; gzip -cd $m-$osperlver->{$m}.tar.gz | tar xf - ; cd $m* ; if [ -f Build.PL ]; then perl Build.PL; ./Build ; ./Build install ; else perl Makefile.PL; make ; make install ; fi; cd .. ; rm -rf $m*\" | bash -e $bashopt" ,"Installing perl module $m-$osperlver->{$m}",$verbopt);
4110}
4111EOF
4112        print SCRIPT << 'EOF';
4113pb_system("rm -rf ProjectBuilder-* ; rm -rf project-builder-* ; rm -rf `perl -V:installvendorlib  | awk -F\"'\" '{print \$2}'`/ProjectBuilder ;
4114EOF
4115        print SCRIPT " $cmdget ; ";
4116        print SCRIPT << "EOF"
4117gzip -cd ProjectBuilder-latest.tar.gz | tar xf - ; cd ProjectBuilder-$pbver1$pbextdir ; perl Makefile.PL ; make ; make install ; cd .. ; rm -rf ProjectBuilder-* ; gzip -cd project-builder-latest.tar.gz | tar xf - ; cd project-builder-$pbver2$pbextdir ; perl Makefile.PL ; make ; make install ; cd .. ; rm -rf project-builder-* ;","Building Project-Builder");
4118EOF
4119    } elsif ($itype =~ /^pkg/) {
4120        # pkg based install. We need to point to the project-builder.org repository
4121        print SCRIPT << 'EOF';
4122my $pkgforpb = pb_distro_get_param($pbos,pb_conf_get_if("ospkg"));
4123my $forcerepo = pb_distro_setuposrepo($pbos);
4124pb_log(1,"Forcing full package install with forcerepo\n") if (defined $forcerepo);
4125pb_distro_installdeps(undef,$pbos,$forcerepo,pb_distro_only_deps_needed($pbos,join(' ',split(/,/,$pkgforpb))));
4126EOF
4127    } else {
4128        # Unknown install type
4129        die("Unknown install type $itype->{$ENV{'PBPROJ'}} for param pbinstalltype");
4130    }
4131    print SCRIPT << 'EOF';
4132pb_system("pbdistrocheck",undef,"verbose");
4133pb_system("pb --version",undef,"verbose");
4134EOF
4135    if ($vtype eq "ve"){
4136        if ($vetype ne "docker") {
4137            print SCRIPT << 'EOF';
4138# For VE we need to umount some FS at the end
4139
4140pb_system("umount /proc","Unmounting /proc","mayfail");
4141EOF
4142        }
4143        print SCRIPT << 'EOF';
4144# Create a basic network file if not already there
4145
4146my $nf="/etc/sysconfig/network";
4147if ((! -f $nf) && ($pbos->{'type'} eq "rpm") && ($pbos->{'family'} ne "novell")) {
4148    open(NF,"> $nf") || die "Unable to create $nf";
4149    print NF "NETWORKING=yes\n";
4150    print NF "HOSTNAME=localhost\n";
4151    close(NF);
4152}
4153chmod 0755,$nf;
4154EOF
4155    }
4156
4157    # Adds pb_distro_get_context and all functions needed from ProjectBuilder::Distribution, Conf and Base
4158    foreach my $m ("ProjectBuilder/Base.pm","ProjectBuilder/Distribution.pm","ProjectBuilder/Conf.pm") {
4159        foreach my $d (@INC) {
4160            my $f = "$d/$m";
4161            if (-f "$f") {
4162                open(PBD,"$f") || die "Unable to open $f: $!";
4163                while (<PBD>) {
4164                    next if (/^package/);
4165                    next if (/^use Exporter/);
4166                    next if (/^use ProjectBuilder::/);
4167                    next if (/^our /);
4168                    print SCRIPT $_;
4169                }
4170                close(PBD);
4171                # We just need the first one of each file wrt @INC - hopefully that's the right one.
4172                last;
4173            }
4174        }
4175    }
4176    # Use a fake pb_version_init version here
4177    print SCRIPT << "EOF";
4178sub pb_version_init {
4179
4180return("$projectbuilderver","$projectbuilderrev","$projectbuilderconfver");
4181}
41821;
4183EOF
4184    close(SCRIPT);
4185    chmod 0755,"$pbscript{$v}";
4186
4187    $pm->finish if (defined $pbparallel);
4188}
4189$pm->wait_all_children if (defined $pbparallel);
4190die "Aborting, one or more of the children failed." if ((not $all_ok) && ($Global::pb_stop_on_error));
4191
4192# These build scripts need to be run as root and force stop of VM at end
4193$ENV{'PBACCOUNT'} = "root";
4194
4195$sbx = "" if (not defined $sbx);
4196# Launch the VM/VE/RM
4197pb_log(2,"DEBUG: before parallel launch, pbscript hash is:".Dumper(%pbscript)."\n");
4198pb_parallel_launchv(\%pbscript,$vtype,uc($vtype)."Script".$sbx,$pbstep,$pbimage,$pbforce,0,$pbsnap);
4199
4200return;
4201}
4202
4203# Function to create a snapshot named 'pb' for VMs and a compressed tar for VEs
4204sub pb_snap2v {
4205
4206my $vtype = shift;
4207my $pbstep = 2;
4208
4209my ($vm,$all) = pb_get2v($vtype);
4210
4211# Script generated
4212my $pbscript = "$ENV{'PBDESTDIR'}/snapv.$$";
4213
4214my ($pbac) = pb_conf_get($vtype."login");
4215
4216foreach my $v (@$vm) {
4217    if ($vtype eq "ve") {
4218        # Get distro context
4219        my $pbos = pb_distro_get_context($v);
4220        my ($vepath) = pb_conf_get("vepath");
4221
4222        # Test if an existing snapshot exists and remove it if there is a VE
4223        if ((-f "$vepath->{$ENV{'PBPROJ'}}/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}.tar.gz") &&
4224            (! -d "$vepath->{$ENV{'PBPROJ'}}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}")) {
4225                pb_system("sudo rm -f $vepath->{$ENV{'PBPROJ'}}/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}.tar.gz","Removing previous snapshot $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}.tar.gz");
4226        }
4227    }
4228
4229    # Prepare the script to be executed on the VM/VE
4230    open(SCRIPT,"> $pbscript") || die "Unable to create $pbscript";
4231    print SCRIPT << 'EOF';
4232    #!/bin/bash
4233    sleep 2
4234EOF
4235    close(SCRIPT);
4236    chmod 0755,"$pbscript";
4237
4238    # Force shutdown of VM/VE
4239    # Force snapshot of VM/VE
4240    pb_script2v($pbscript,$vtype,$pbstep,1,$v,1);
4241}
4242return;
4243}
4244
4245# Function to update VMs/VEs/RMs with the latest distribution content
4246sub pb_update2v {
4247
4248my $vtype = shift;
4249my $pbstep = 2;
4250
4251my ($vm,$all) = pb_get2v($vtype);
4252
4253# Script generated
4254my $pbscript = "$ENV{'PBDESTDIR'}/updatev.$$";
4255
4256my ($pbac) = pb_conf_get($vtype."login");
4257
4258foreach my $v (@$vm) {
4259    # Get distro context
4260    my $pbos = pb_distro_get_context($v);
4261
4262    # Prepare the script to be executed on the VM/VE/RM
4263    # in $ENV{'PBDESTDIR'}/updatev
4264    open(SCRIPT,"> $pbscript") || die "Unable to create $pbscript";
4265   
4266    print SCRIPT << 'EOF';
4267    #!/bin/bash
4268    sleep 2
4269EOF
4270    # VE needs a good /proc
4271    $vetype = pb_ve_get_type($vetype);
4272    if (($vtype eq "ve") && ($vetype ne "docker")) {
4273        print SCRIPT "sudo /bin/mount -t proc /proc /proc\n";
4274    }
4275    print SCRIPT "$pbos->{'update'}\n";
4276    if (($vtype eq "ve") && ($vetype ne "docker")) {
4277        print SCRIPT "sudo /bin/umount /proc\n";
4278    }
4279    close(SCRIPT);
4280    chmod 0755,"$pbscript";
4281
4282    # Force shutdown of VM except
4283    pb_script2v($pbscript,$vtype,$pbstep,1,$v);
4284}
4285return;
4286}
4287
4288sub pb_announce {
4289
4290    my $antype = shift;
4291
4292    # Get all required parameters
4293    my ($pbpackager,$pbrepo) = pb_conf_get("pbpackager","pbrepo");
4294    my ($pkgv, $pkgt, $testver) = pb_conf_get_if("pkgver","pkgtag","testver");
4295    if (((not defined $testver) || (not defined $testver->{$ENV{'PBPROJ'}}) || ($testver->{$ENV{'PBPROJ'}} !~ /true/i)) && ($antype eq "Clean")) {
4296        # We just clean for test versions
4297        pb_log(0,"Unable to clean SSH repository for non testver version\n");
4298        return;
4299    }
4300    my $pkg = pb_cms_get_pkg($defpkgdir,$extpkgdir);
4301    my @pkgs = @$pkg;
4302    my %pkgs;
4303    my $first = 0;
4304
4305    # Get all distros concerned
4306    my $pbos = pb_distro_get_context();
4307    my $distrolist = pb_get_distros($pbos,undef);
4308    my %dl;
4309    my %theorlist;
4310    my %archlist;
4311    foreach my $d (split(/,/,$distrolist)) {
4312        my ($d1,$d2,$d3) = split(/-/,$d);
4313        $dl{$d1}++;
4314    }
4315
4316    # Command to find packages on repo
4317    my $findstr = "find ".join(" ",keys %dl)." ";
4318    my $srcstr = "";
4319    # Generated announce files
4320    my @files;
4321
4322    foreach my $pbpkg (@pkgs) {
4323        if ($first != 0) {
4324            $findstr .= "-o ";
4325        }
4326        $first++;
4327        if ((defined $pkgv) && (defined $pkgv->{$pbpkg})) {
4328            $pbver = $pkgv->{$pbpkg};
4329        } else {
4330            $pbver = $ENV{'PBPROJVER'};
4331        }
4332        if ((defined $pkgt) && (defined $pkgt->{$pbpkg})) {
4333            $pbtag = $pkgt->{$pbpkg};
4334        } else {
4335            $pbtag = $ENV{'PBPROJTAG'};
4336        }
4337
4338        # TODO: use virtual/real names here now
4339        my $pbrealpkg = $pbpkg;
4340        my $pbrealpkgrpm = pb_cms_get_real_pkg($pbpkg,"rpm");
4341        my $pbrealpkgdeb = pb_cms_get_real_pkg($pbpkg,"deb");
4342        if ($antype eq "Clean") {
4343            # We only clean test versions anyway
4344            $pbtag = "0";
4345            my $nver = $pbver;
4346            my $ntag = "[2-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]";
4347            $pbver .= $ntag;
4348            # TODO add apk
4349            $findstr .= "-name \'$pbrealpkgrpm-$pbver-$pbtag\.*.rpm\' -o -name \'$pbrealpkgrpm-debug-$pbver-$pbtag\.*.rpm\' -o -name \'$pbrealpkgdeb"."_$pbver-$pbtag"."_*\.deb\' -o -name \'$pbrealpkgdeb"."_$pbver-$pbtag.dsc\' -o -name \'$pbrealpkgdeb"."_$pbver-$pbtag.tar.gz\' -o -name \'$pbrealpkg-$nver"."_p$ntag\.ebuild\' -o -name \'$pbrealpkg-$pbver-$pbtag*\.pkg\.gz\' -o -name \'$pbrealpkg-$pbver-$pbtag*\.sd\' ";
4350            $srcstr .= "src/$pbrealpkg-$pbver.tar.gz src/$pbrealpkg-$pbver.pbconf.tar.gz ";
4351        } else {
4352            # "Announce" and "Check"
4353            my @date=pb_get_date();
4354            # the matching is only done on packages made the same day for test version. Hopefully this is enough
4355            my $nver;
4356            if ((defined $testver) && (defined $testver->{$ENV{'PBPROJ'}}) && ($testver->{$ENV{'PBPROJ'}} =~ /true/i)) {
4357                $pbtag = "0";
4358                my $ntag .= strftime("%Y%m%d*", @date);
4359                $nver = $pbver."_p$ntag*";
4360                $pbver .= "$ntag*";
4361            } else { 
4362                $nver = $pbver."-r$pbtag";
4363            }
4364            $findstr .= "-name \'$pbrealpkgrpm-$pbver-$pbtag\.*.rpm\' -o -name \'$pbrealpkgdeb"."_$pbver-$pbtag"."_*\.deb\' -o -name \'$pbrealpkg-$nver\.ebuild\' -o -name \'$pbrealpkg-$pbver\.pkg\.gz\' -o -name \'$pbrealpkg-$pbver\.sd\' ";
4365        }
4366
4367        if ($antype eq "Announce") {
4368            my $chglog;
4369
4370            pb_vcs_init($pbinit);
4371            # Get project info on log file and generate tmp files used later on
4372            $chglog = "$ENV{'PBROOTDIR'}/$pbpkg/pbcl";
4373            $chglog = "$ENV{'PBROOTDIR'}/pbcl" if (! -f $chglog);
4374            $chglog = undef if (! -f $chglog);
4375
4376            open(OUT,"> $ENV{'PBTMP'}/$pbpkg.ann") || die "Unable to create $ENV{'PBTMP'}/$pbpkg.ann: $!";
4377            my $pb;
4378            $pb->{'realpkg'} = $pbrealpkg;
4379            $pb->{'ver'} = $pbver;
4380            $pb->{'tag'} = $pbtag;
4381            $pb->{'date'} = $pbdate;
4382            $pb->{'extdir'} = pb_get_extdir();
4383            $pb->{'chglog'} = $chglog;
4384            $pb->{'packager'} = $pbpackager;
4385            $pb->{'proj'} = $ENV{'PBPROJ'};
4386            $pb->{'repo'} = $pbrepo;
4387            $pb->{'pbos'}->{'type'} = "announce";
4388            $pb->{'pbos'}->{'suffix'} = "none";
4389            pb_changelog($pb,\*OUT,"yes");
4390            close(OUT);
4391            push(@files,"$ENV{'PBTMP'}/$pbpkg.ann");
4392        } elsif ($antype eq "Check") {
4393            # For the check we also build the theoritical complete list we should get
4394            foreach my $d (split(/,/,$distrolist)) {
4395                $pbos = pb_distro_get_context($d);
4396                if ($pbos->{'type'} eq "rpm") {
4397                    $theorlist{"$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}/$pbrealpkgrpm-$pbver-$pbtag$pbos->{'suffix'}"} = 0;
4398                } elsif ($pbos->{'type'} eq "deb") {
4399                    $theorlist{"$pbos->{'name'}/$pbos->{'version'}/$pbrealpkgdeb"."_$pbver-$pbtag"} = 0;
4400                    # TODO are we always using the last arch ?
4401                    $archlist{"$pbos->{'name'}/$pbos->{'version'}/$pbrealpkgdeb"."_$pbver-$pbtag"} = "$pbos->{'arch'}";
4402                } elsif ($pbos->{'type'} eq "ebuild") {
4403                    my $prefix = "-r";
4404                    $prefix = "_p" if ((defined $testver) && (defined $testver->{$ENV{'PBPROJ'}}) && ($testver->{$ENV{'PBPROJ'}} =~ /true/i));
4405                    $theorlist{"$pbos->{'name'}/$pbos->{'version'}/$pbrealpkg-$pbver$prefix$pbtag.ebuild"} = 0;
4406                    $archlist{"$pbos->{'name'}/$pbos->{'version'}/$pbrealpkg-$pbver$prefix$pbtag.ebuild"} = "$pbos->{'arch'}";
4407                } elsif ($pbos->{'type'} eq "apk") {
4408                    # TODO
4409                } elsif ($pbos->{'type'} eq "pkg") {
4410                    $theorlist{"$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}/$pbrealpkg-$pbver-$pbtag.pkg.gz"} = 0;
4411                } else {
4412                    pb_log(1,"No theoritical list possible for type $pbos->{'type'}\n");
4413                }
4414            }
4415        }
4416        pb_log(2,"theorlist initial: ".Dumper(%theorlist)."\n");
4417        pb_log(2,"archlist: ".Dumper(%archlist)."\n");
4418    }
4419    if ($antype eq "Announce") {
4420        $findstr .= " | grep -Ev \'src.rpm\'";
4421    } elsif ($antype eq "Clean") {
4422        $findstr .= " | xargs rm -f -v $srcstr ";
4423    }
4424
4425    # Prepare the command to run and execute it
4426    open(PBS,"> $ENV{'PBTMP'}/pbscript") || die "Unable to create $ENV{'PBTMP'}/pbscript";
4427    print PBS "#!/bin/bash\n";
4428    print PBS "set -x\n" if ($pbdebug gt 1);
4429    print PBS "$findstr | sort 2> /dev/null\n";
4430    close(PBS);
4431    chmod 0755,"$ENV{'PBTMP'}/pbscript";
4432    pb_send2target("Announce","$ENV{'PBTMP'}/pbscript");
4433
4434    my $sl = "Project $ENV{'PBPROJ'} version $ENV{'PBPROJVER'} is now available";
4435    if ($antype eq "Announce") {
4436        # Get subject line
4437        pb_log(0,"Please enter the title of your announce\n");
4438        pb_log(0,"(By default: $sl)\n");
4439        my $sl2 = <STDIN>;
4440        $sl = $sl2 if ($sl2 !~ /^$/);
4441
4442        # Prepare a template of announce
4443        open(ANN,"> $ENV{'PBTMP'}/announce.html") || die "Unable to create $ENV{'PBTMP'}/announce.html: $!";
4444        print ANN << "EOF";
4445$sl</p>
4446
4447<p>The project team is happy to announce the availability of a newest version of $ENV{'PBPROJ'} $ENV{'PBPROJVER'}. Enjoy it as usual!</p>
4448<p>
4449Now available at <a href="$pbrepo->{$ENV{'PBPROJ'}}">$pbrepo->{$ENV{'PBPROJ'}}</a>
4450</p>
4451<p>
4452EOF
4453    }
4454
4455    open(LOG,"$ENV{'PBTMP'}/system.$$.log") || die "Unable to read $ENV{'PBTMP'}/system.$$.log: $!";
4456    if ($antype eq "Announce") {
4457        my $col = 2;
4458        my $i = 1;
4459        print ANN << 'EOF';
4460<TABLE WIDTH="100%" CELLPADDING="0" CELLSPACING="0" BORDER="0">
4461<TR>
4462EOF
4463        while (<LOG>) {
4464            print ANN "<TD><A HREF=\"$pbrepo->{$ENV{'PBPROJ'}}/$_\">$_</A></TD>";
4465            $i++;
4466            if ($i > $col) {
4467                print ANN "</TR>\n<TR>";
4468                $i = 1;
4469            }
4470        }
4471    } elsif ($antype eq "Clean") {
4472        while (<LOG>) {
4473            # skip errors
4474            next if ($_ !~ /^removed /);
4475            pb_log(0,"$_");
4476        }
4477    } else {
4478        # In "Check" mode we need to compare the 2 lists (real and theoritical)
4479        while (<LOG>) {
4480            # Get package name and remove what is in extra for the theoritical list (arch at the end)
4481            chomp();
4482            # skip find errors
4483            next if (/^find:/);
4484            my $p = $_;
4485            $p =~ s/\.(i[3456]86|x86_64|noarch|src)\.rpm$//;
4486            $p =~ s/_(i[3456]86|amd64|all)\.deb$//;
4487            $p =~ s/(-0\.[0-9]{8})[0-9]{6}/$1*/ if ((defined $testver) && (defined $testver->{$ENV{'PBPROJ'}}) && ($testver->{$ENV{'PBPROJ'}} =~ /true/i));
4488            $p =~ s/(-r|_p[0-9]+)\.ebuild/$1*/ if ((defined $testver) && (defined $testver->{$ENV{'PBPROJ'}}) && ($testver->{$ENV{'PBPROJ'}} =~ /true/i));
4489            # TODO apk
4490            $theorlist{$p} = -2 if (not defined $theorlist{$p});
4491            $theorlist{$p} = $theorlist{$p} + 1;
4492        }
4493        pb_log(2,"theorlist after check: ".Dumper(%theorlist)."\n");
4494    }
4495    close(LOG);
4496
4497    # Nothing more for the cleanssh case
4498    return if ($antype eq "Clean");
4499
4500    if ($antype eq "Check") {
4501        my ($chkex) = pb_conf_get_if("checkexclude");
4502        my $vmbuildlist = "";
4503        my $vebuildlist = "";
4504        my $rmbuildlist = "";
4505        my @pt = pb_conf_get_if("vmlist","velist","rmlist");
4506        foreach my $t (sort keys %theorlist) {
4507            if (defined $theorlist{$t} and $theorlist{$t} >= 1) {
4508                pb_log(1,"Packages found for $t\n");
4509            } elsif (defined $theorlist{$t} and $theorlist{$t} < 0) {
4510                pb_log(0,"Extra Package found for $t\n");
4511            } else {
4512                pb_log(2,"Analyzing $t\n");
4513                my ($os,$ver,$arch,$package) = split(/\//,$t);
4514                # Some distro have no arch subdir
4515                if (not defined $package) {
4516                    $package = $arch;
4517                    # TODO: If both arch have failed, we just make the last one
4518                    $arch = $archlist{$t} if (defined $archlist{$t});
4519                }
4520                my $pbos = pb_distro_get_context("$os-$ver-$arch");
4521                my $pkgn = $package;
4522                if ($pbos->{'type'} ne "deb") {
4523                    # package name is more easily found from the end for non deb
4524                    # as '-' is the separator, but it can also be used in names
4525                    $pkgn = reverse($package);
4526                    # search the second '-' and isolate the now last part which is the full name
4527                    $pkgn =~ s/([^-]+)-([^-]+)-([\S])+$/$3/;
4528                } else {
4529                    $pkgn =~ s/([^_]+)_([\S])+$/$2/;
4530                }
4531                my $found = 0;
4532                # Handle the exclusion of OSes
4533                my $excl = "";
4534                $excl .= $chkex->{$pkgn} if (defined $chkex->{$pkgn});
4535                $excl .= $chkex->{"all"} if (defined $chkex->{"all"});
4536
4537                pb_log(2,"Searching in excl $excl\n");
4538                foreach my $ex (split(/,/,$excl)) {
4539                    $found = 1 if ("$os-$ver-$arch" =~ /^$ex/);
4540                }
4541                # Skip as excluded
4542                next if ($found == 1);
4543                pb_log(0,"Package NOT found for $t\n");
4544                # Avoid duplicates in list
4545                next if ($vmbuildlist =~ /$os-$ver-$arch/);
4546                next if ($vebuildlist =~ /$os-$ver-$arch/);
4547                next if ($rmbuildlist =~ /$os-$ver-$arch/);
4548                # check with which method we need to build
4549                if ((defined $pt[0]->{$ENV{'PBPROJ'}}) and ($pt[0]->{$ENV{'PBPROJ'}} =~ /$os-$ver-$arch/)) {
4550                    $vmbuildlist = "$os-$ver-$arch" if ($vmbuildlist eq "");
4551                    $vmbuildlist .= ",$os-$ver-$arch" if ($vmbuildlist !~ /$os-$ver-$arch/);
4552                    next;
4553                }
4554                if ((defined $pt[1]->{$ENV{'PBPROJ'}}) and ($pt[1]->{$ENV{'PBPROJ'}} =~ /$os-$ver-$arch/)) {
4555                    $vebuildlist = "$os-$ver-$arch" if ($vebuildlist eq "");
4556                    $vebuildlist .= ",$os-$ver-$arch" if ($vebuildlist !~ /$os-$ver-$arch/);
4557                    next;
4558                }
4559                if ((defined $pt[2]->{$ENV{'PBPROJ'}}) and ($pt[2]->{$ENV{'PBPROJ'}} =~ /$os-$ver-$arch/)) {
4560                    $rmbuildlist = "$os-$ver-$arch" if ($rmbuildlist eq "");
4561                    $rmbuildlist .= ",$os-$ver-$arch" if ($rmbuildlist !~ /$os-$ver-$arch/);
4562                }
4563            }
4564        }
4565        # If we want to rebuild automatically, let's do it
4566        if (defined $opts{'rebuild'}) {
4567            # SandBox or CMS
4568            pb_log(0,"Rebuilding from SandBox\n");
4569            pb_log(0,"for VMs: $vmbuildlist\n") if ($vmbuildlist ne "");
4570            pb_log(0,"for VEs: $vebuildlist\n") if ($vebuildlist ne "");
4571            pb_log(0,"for RMs: $rmbuildlist\n") if ($rmbuildlist ne "");
4572            pb_cms2build("SandBox");
4573            # Which mode
4574            $ENV{'PBV'} = $vmbuildlist;
4575            pb_build2v("vm","build") if ($vmbuildlist ne "");
4576            $ENV{'PBV'} = $vebuildlist;
4577            pb_build2v("ve","build") if ($vebuildlist ne "");
4578            $ENV{'PBV'} = $rmbuildlist;
4579            pb_build2v("rm","build") if ($rmbuildlist ne "");
4580        }
4581        # For the check part this is now finished
4582        return;
4583    }
4584
4585    print ANN << "EOF";
4586</TR>
4587</TABLE>
4588</p>
4589
4590<p>As usual source packages are also available in the same directory.</p>
4591
4592<p>
4593Changes are :
4594</p>
4595<p>
4596EOF
4597    # Get each package changelog content
4598    foreach my $f (sort(@files)) {
4599        open(IN,"$f") || die "Unable to read $f:$!";
4600        while (<IN>) {
4601            print ANN $_;
4602        }
4603        close(IN);
4604        print ANN "</p><p>\n";
4605    }
4606    print ANN "</p>\n";
4607    close(ANN);
4608
4609    # Allow for modification
4610    my $editor = "vi";
4611    $editor = $ENV{'EDITOR'} if (defined $ENV{'EDITOR'});
4612    pb_system("$editor $ENV{'PBTMP'}/announce.html","Allowing modification of the announce","noredir");
4613
4614    # Store it in DB for external usage (Web pages generation)
4615    my $dbh = pb_connect_db();
4616
4617    # To read whole file
4618    local $/;
4619    open(ANN,"$ENV{'PBTMP'}/announce.html") || die "Unable to read $ENV{'PBTMP'}/announce.html: $!";
4620    my $announce = <ANN>;
4621    close(ANN);
4622   
4623    pb_log(2,"INSERT INTO announces VALUES (NULL, $pbdate, $announce)");
4624    my $sth = $dbh->prepare(qq{INSERT INTO announces VALUES (NULL,?,?)})
4625                    || die "Unable to insert into db";
4626    $sth->execute($pbdate, $announce);
4627    $sth->finish();
4628    $dbh->disconnect;
4629
4630    # Then deliver it on the Web
4631    # $TOOLHOME/livwww www
4632
4633    # Mail it to project's ML
4634    open(ML,"| w3m -dump -T text/html > $ENV{'PBTMP'}/announce.txt") || die "Unable to create $ENV{'PBTMP'}/announce.txt: $!";
4635    print ML << 'EOF';
4636<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/x html1/DTD/xhtml1-strict.dtd">
4637
4638<html xmlns="http://www.w3.org/1999/xhtml" dir="ltr" xml:lang="en" lang="en">
4639  <head>
4640  </head>
4641  <body>
4642  <p>
4643EOF
4644    open(ANN,"$ENV{'PBTMP'}/announce.html") || die "Unable to read $ENV{'PBTMP'}/announce.html: $!";
4645    while(<ANN>) {
4646        print ML $_;
4647    }
4648    print ML << 'EOF';
4649</body>
4650</html>
4651EOF
4652    close(ML);
4653
4654    # To read whole file
4655    local $/;
4656    open(ANN,"$ENV{'PBTMP'}/announce.txt") || die "Unable to read $ENV{'PBTMP'}/announce.txt: $!";
4657    my $msg = <ANN>;
4658    close(ANN);
4659   
4660    # Preparation of headers
4661    eval
4662    {
4663        require Mail::Sendmail;
4664        Mail::Sendmail->import();
4665    };
4666    if ($@) {
4667        # Mail::Sendmail not found not sending mail !
4668        pb_log(0,"No Mail::Sendmail module found so not sending any mail !\n");
4669    } else {
4670        my ($pbml,$pbsmtp) = pb_conf_get("pbml","pbsmtp");
4671        my %mail = (   
4672            To          =>  $pbml->{$ENV{'PBPROJ'}},
4673            From        =>  $pbpackager->{$ENV{'PBPROJ'}},
4674            Smtp        =>  $pbsmtp->{$ENV{'PBPROJ'}},
4675            Body        =>  $msg,
4676            Subject     =>  "[ANNOUNCE] $sl",
4677        );
4678           
4679        # Send mail
4680        if (! sendmail(%mail)) {
4681            if ((defined $Mail::Sendmail::error) and (defined $Mail::Sendmail::log)) {
4682                die "Unable to send mail ($Mail::Sendmail::error): $Mail::Sendmail::log";
4683            }
4684        }
4685    }
4686}
4687
4688#
4689# Creates a set of HTML file containing the news for the project
4690# based on what has been generated by the pb_announce function
4691#
4692sub pb_web_news2html {
4693
4694    my $dest = shift || $ENV{'PBTMP'};
4695
4696    # Get all required parameters
4697    my ($pkgv, $pkgt) = pb_conf_get_if("pkgver","pkgtag");
4698
4699
4700    my $dbh = pb_connect_db();
4701    # For date handling
4702    $ENV{'LANGUAGE'}="C";
4703    my $firstjan = strftime("%Y-%m-%d", 0, 0, 0, 1, 0, localtime->year(), 0, 0, -1);
4704    my $oldfirst = strftime("%Y-%m-%d", 0, 0, 0, 1, 0, localtime->year()-1, 0, 0, -1);
4705    pb_log(2,"firstjan: $firstjan, oldfirst: $oldfirst, pbdate:$pbdate\n");
4706    my $all = $dbh->selectall_arrayref("SELECT id,date,announce FROM announces ORDER BY date DESC");
4707    my %news;
4708    $news{"cy"} = "";   # current year's news
4709    $news{"ly"} = "";   # last year news
4710    $news{"py"} = "";   # previous years news
4711    $news{"fp"} = "";   # first page news
4712    my $cpt = 4;        # how many news for first page
4713    # Extract info from DB
4714    foreach my $row (@$all) {
4715        my ($id, $date, $announce) = @$row;
4716        $news{"cy"} = $news{"cy"}."<p><B>$date</B> $announce\n" if ((($date cmp $pbdate) le 0) && (($firstjan cmp $date) le 0));
4717        $news{"ly"} = $news{"ly"}."<p><B>$date</B> $announce\n" if ((($date cmp $firstjan) le 0) && (($oldfirst cmp $date) le 0));
4718        $news{"py"} = $news{"py"}."<p><B>$date</B> $announce\n" if (($date cmp $oldfirst) le 0);
4719        $news{"fp"} = $news{"fp"}."<p><B>$date</B> $announce\n" if ($cpt > 0);
4720        $cpt--;
4721    }
4722    pb_log(1,"news{fp}: ".$news{"fp"}."\n");
4723    $dbh->disconnect;
4724
4725    # Generate the HTML content
4726    foreach my $pref (keys %news) {
4727        open(NEWS,"> $dest/pb_web_$pref"."news.html") || die "Unable to create $dest/pb_web_$pref"."news.html: $!";
4728        print NEWS "$news{$pref}";
4729        close(NEWS);
4730    }
4731}
4732
4733sub pb_connect_db {
4734
4735    # Store in DB for external usage (Web pages generation)
4736    my $db = "$ENV{'PBCONFDIR'}/announces3.sql";
4737
4738    my $precmd = "";
4739    if (! -f $db) {
4740        $precmd = "CREATE TABLE announces (id INTEGER PRIMARY KEY AUTOINCREMENT, date DATE, announce VARCHAR[65535])";
4741        # we should add the $db to the VCS as well here
4742    }
4743
4744    my $dbh = DBI->connect("dbi:SQLite:dbname=$db","","",
4745                        { RaiseError => 1, AutoCommit => 1 })
4746                        || die "Unable to connect to $db";
4747
4748    if ($precmd ne "") {
4749        my $sth = $dbh->prepare(qq{$precmd})
4750                    || die "Unable to create table into $db";
4751        $sth->execute();
4752    }
4753
4754    return($dbh);
4755}
4756
4757
4758# Return the SSH key file to use
4759# Potentially create it if needed
4760
4761sub pb_ssh_get {
4762
4763my $create = shift || 0;    # Do not create keys by default. In that case prefer native keys, not pb's ones
4764my $target = shift || "localhost";
4765my $nport = shift || "22";
4766
4767my ($pbagent) = pb_conf_get_if("pbusesshagent");
4768# use ssh-agent if asked so.
4769return(undef) if (($create == 0) && (defined $pbagent->{$ENV{'PBPROJ'}}) && ($pbagent->{$ENV{'PBPROJ'}} =~ /true/io));
4770
4771# Check the SSH environment
4772my $keyfile = undef;
4773
4774# Check whether ecdsa is accepted by the remote host
4775my $ecdsa = 0;
4776my $dsa = 0;
4777my $rsa = 0;
4778open(SCAN,"ssh-keyscan -p $nport -t ecdsa $target 2>/dev/null |") || pb_log(0,"Unable to check ECDSA support for $target");
4779while(<SCAN>) {
4780    $ecdsa = 1 if ($_ =~ /ecdsa-/);
4781    pb_log(2,"Found ECDSA support on $target ($nport)\n");
4782}
4783close(SCAN);
4784
4785open(SCAN,"ssh-keyscan -p $nport -t rsa $target 2>/dev/null |") || pb_log(0,"Unable to check DSA support for $target");
4786while(<SCAN>) {
4787    $rsa = 1 if ($_ =~ /ssh-rsa/);
4788    pb_log(2,"Found RSA support on $target ($nport)\n");
4789}
4790close(SCAN);
4791
4792open(SCAN,"ssh-keyscan -p $nport -t dsa $target 2>/dev/null |") || pb_log(0,"Unable to check DSA support for $target");
4793while(<SCAN>) {
4794    $dsa = 1 if ($_ =~ /ssh-dss/);
4795    pb_log(2,"Found DSA support on $target ($nport)\n");
4796}
4797close(SCAN);
4798
4799# We have specific keys by default - create them if needed
4800if (!(-e "$ENV{'HOME'}/.ssh/pb_ecdsa") && ($create eq 1)) {
4801    my $res = pb_system("ssh-keygen -q -b 521 -N '' -f $ENV{'HOME'}/.ssh/pb_ecdsa -t ecdsa","Generating ECDSA SSH keys for pb");
4802    if (($res != 0) && !(-e "$ENV{'HOME'}/.ssh/pb_dsa") && ($create eq 1)) {
4803        # ecdsa may not be supported if old ssh
4804        pb_system("ssh-keygen -q -b 1024 -N '' -f $ENV{'HOME'}/.ssh/pb_dsa -t dsa","Generating DSA SSH keys for pb");
4805    }
4806}
4807
4808# Some ECDSA implementation are not working correctly e.g. RHEL7, Fedora 20/21
4809if (!(-e "$ENV{'HOME'}/.ssh/pb_rsa") && ($create eq 1)) {
4810    pb_system("ssh-keygen -q -b 2048 -N '' -f $ENV{'HOME'}/.ssh/pb_rsa -t rsa","Generating RSA SSH keys for pb");
4811}
4812
4813# We still favour RSA and fall back to ECDSA if not available
4814$keyfile = "$ENV{'HOME'}/.ssh/id_rsa" if (-s "$ENV{'HOME'}/.ssh/id_rsa");
4815if ($rsa != 0) {
4816    # And we favour a specific key over a generic one except if in delivery to ftp server
4817    $keyfile = "$ENV{'HOME'}/.ssh/id_rsa" if ((-s "$ENV{'HOME'}/.ssh/id_rsa") && ($create eq 0));
4818    $keyfile = "$ENV{'HOME'}/.ssh/pb_rsa" if (-s "$ENV{'HOME'}/.ssh/pb_rsa");
4819} else {
4820    if ($ecdsa != 0) {
4821        # And we favour a specific key over a generic one except if in delivery to ftp server
4822        $keyfile = "$ENV{'HOME'}/.ssh/id_ecdsa" if ((-s "$ENV{'HOME'}/.ssh/id_ecdsa") && ($ecdsa != 0) && ($create eq 0));
4823        $keyfile = "$ENV{'HOME'}/.ssh/pb_ecdsa" if ((-s "$ENV{'HOME'}/.ssh/pb_ecdsa") && ($ecdsa != 0));
4824    } elsif ($dsa != 0) {
4825        # And we favour a specific key over a generic one except if in delivery to ftp server
4826        $keyfile = "$ENV{'HOME'}/.ssh/id_dsa" if ((-s "$ENV{'HOME'}/.ssh/id_dsa") && ($dsa != 0) && ($create eq 0));
4827        $keyfile = "$ENV{'HOME'}/.ssh/pb_dsa" if ((-s "$ENV{'HOME'}/.ssh/pb_dsa") && ($dsa != 0));
4828    }
4829}
4830if (defined $keyfile) {
4831    pb_log(2,"Using ssh key file $keyfile\n");
4832} else {
4833    pb_log(0, "Unable to find your public ssh key under $ENV{'HOME'}/.ssh\n");
4834}
4835return($keyfile);
4836}
4837
4838
4839# Returns the pid of a running VM command using a specific VM file, and the port used if the VM exists
4840sub pb_check_ps {
4841    my $vmcmd = shift;
4842    my $vmm = shift;
4843    my $vexist = 0;             # FALSE by default
4844    my $vmport = undef;         # NONE by default
4845
4846    if ($vmcmd =~ /qemu/) {
4847        # Remove the suffix as qemu changes it at launch
4848        $vmcmd =~ s/qemu-.*/qemu/;
4849    }
4850    open(PS, "ps auxhww|") || die "Unable to call ps";
4851    while (<PS>) {
4852        next if (! /$vmcmd/);
4853        next if (! /$vmm/);
4854        my $void1;
4855        ($void1, $vexist, $vmport) = split(/ +/,$_,3);
4856        pb_log(2,"pb_check_ps $vmport\n");
4857        $vmport =~ s/.*=tcp:127\.0\.0\.1:([0-9]+)-:22.*/$1/;
4858        chomp($vmport);
4859        pb_log(2,"pb_check_ps found a VM $vexist using port $vmport\n");
4860        last;
4861    }
4862    return($vexist,$vmport);
4863}
4864
4865
4866sub pb_extract_build_files {
4867
4868my $src=shift;
4869my $dir=shift;
4870my $ddir=shift;
4871my $mandatory=shift || "build";
4872
4873my $flag = "mayfail" if (($mandatory eq "patch") || ($mandatory eq "src"));
4874my $res;
4875
4876if ($src =~ /tar\.gz$/) {
4877    $res = pb_system("tar xfpz $src $dir","Extracting $mandatory files from $src",$flag);
4878} elsif ($src =~ /tar\.bz2$/) {
4879    $res = pb_system("tar xfpj $src $dir","Extracting $mandatory files from $src",$flag);
4880} else {
4881    die "Unknown compression algorithm for $src";
4882}
4883# If not mandatory return now
4884return() if (($res != 0) and (($mandatory eq "patch") || ($mandatory eq "src")));
4885return(pb_move_extracted_files($dir,$ddir));
4886}
4887
4888sub pb_move_extracted_files {
4889
4890my $dir = shift;
4891my $ddir = shift;
4892my @files;
4893
4894# In a recursive function , we need a local var as DIR handle
4895my $bdir;
4896
4897opendir($bdir,"$dir") || confess "Unable to open directory $dir: $!";
4898foreach my $f (readdir($bdir)) {
4899    next if ($f =~ /^\./);
4900    # Skip potential patch dir
4901    next if ($f =~ /^pbpatch/);
4902    # Skip potential source dir
4903    next if ($f =~ /^pbsrc/);
4904    # Skip potential backup files
4905    next if ($f =~ /~$/);
4906    if ((-d "$dir/$f") && (! -l "$dir/$f")) {
4907        # Case of the bug debian subdir
4908        pb_mkdir_p("$ddir/$f");
4909        push @files,pb_move_extracted_files("$dir/$f","$ddir/$f");
4910    } else {
4911        move("$dir/$f","$ddir") || die "Unable to move $dir/$f to $ddir";
4912        pb_log(2,"mv $dir/$f $ddir\n");
4913        push @files,"$ddir/$f";
4914    }
4915}
4916closedir($bdir);
4917# Not enough but still a first cleanup
4918pb_rm_rf("$dir");
4919return(@files);
4920}
4921
4922sub pb_list_bfiles {
4923
4924my $dir = shift;
4925my $pbpkg = shift;
4926my $bfiles = shift;
4927my $pkgfiles = shift;
4928my $supfiles = shift;
4929# subdir to keep if recursive mode, empty by default
4930my $subdir = shift || "";
4931# In a recursive function , we need a local var as DIR handle
4932my $bdir;
4933
4934pb_log(2,"DEBUG: entering pb_list_bfiles in $dir: ".Dumper($bfiles)."\n");
4935opendir($bdir,"$dir") || die "Unable to open dir $dir: $!";
4936foreach my $f (readdir($bdir)) {
4937    pb_log(3,"DEBUG: pb_list_bfiles found $f\n");
4938    next if ($f =~ /^\./);
4939    if (-d "$dir/$f") {
4940        # Recurse for directories (Debian 3.0 format e.g.)
4941        pb_log(2,"DEBUG: pb_list_bfiles recurse in $dir/$f\n");
4942        pb_list_bfiles("$dir/$f",$pbpkg,$bfiles,$pkgfiles,$supfiles,$f);
4943        next;
4944    }
4945
4946    my $key = $f;
4947    # if recursive then store also the subdir
4948    $key = "$subdir/$f" if ($subdir ne "");
4949    $bfiles->{$key} = "$dir/$f";
4950    $bfiles->{$key} =~ s~$ENV{'PBROOTDIR'}~~;
4951    if (defined $supfiles->{$pbpkg}) {
4952        $pkgfiles->{$key} = "$dir/$f" if ($f =~ /$supfiles->{$pbpkg}/);
4953    }
4954}
4955closedir($bdir);
4956pb_log(2,"DEBUG: exiting pb_list_bfiles: ".Dumper($bfiles)."\n");
4957}
4958
4959sub pb_add_comma {
4960
4961my $str = shift;
4962my $addstr = shift;
4963
4964$str .= "," if (defined $str);
4965$str .= $addstr;
4966return($str);
4967}
4968
4969sub pb_list_sfiles {
4970
4971my $sdir = shift;
4972my $ptr = shift;
4973my $pbos = shift;
4974my $extdir = shift;
4975
4976pb_log(2,"DEBUG: entering pb_list_sfiles: ".Dumper($ptr)."\n");
4977my $key = "$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
4978
4979# Prepare local sources for this distro - They are always applied first - May be a problem one day
4980# This function works for both patches and additional sources
4981foreach my $p (sort(<$sdir/*>)) {
4982    $ptr->{$key} = pb_add_comma($ptr->{$key},"file://$p") if (($p =~ /\.all$/) || ($p =~ /\.$pbos->{'os'}$/) || ($p =~ /\.$pbos->{'type'}$/) || ($p =~ /\.$pbos->{'family'}$/) || ($p =~ /\.$pbos->{'name'}$/) || ($p =~ /\.$pbos->{'name'}-$pbos->{'version'}$/) ||($p =~ /\.$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}$/));
4983}
4984
4985# Prepare also remote sources to be included - Applied after the local ones
4986foreach my $p ("all","$pbos->{'os'}","$pbos->{'type'}","$pbos->{'family'}","$pbos->{'name'}","$pbos->{'name'}-$pbos->{'version'}","$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}") {
4987    my $f = "$extdir.".".$p";
4988    next if (not -f $f);
4989    if (not open(PATCH,$f)) {
4990        pb_display("Unable to open existing external source file content $f\n");
4991        next;
4992    }
4993    while (<PATCH>) {
4994        chomp();
4995        $ptr->{$key} = pb_add_comma($ptr->{$key},"$_");
4996    }
4997    close(PATCH);
4998}
4999pb_log(2,"DEBUG: exiting pb_list_sfiles: ".Dumper($ptr)."\n");
5000return($ptr);
5001}
5002   
5003#
5004# Return the list of packages we are working on in a non CMS action
5005#
5006sub pb_get_pkg {
5007
5008my @pkgs = ();
5009
5010# In script actions that file may not exist.
5011return(\@pkgs) if ((not defined $ENV{'PBDESTDIR'}) || (not defined $ENV{'PBPROJVER'}) || (not defined $ENV{'PBPROJTAG'}) || (! -f "$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml"));
5012
5013my ($var) = pb_conf_read("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml","pbpkg");
5014@pkgs = keys %$var if (defined $var);
5015
5016pb_log(0,"Packages: ".join(',',@pkgs)."\n") if (@pkgs);
5017return(\@pkgs);
5018}
5019
5020# Manages VM/RM SSH port communication
5021sub pb_get_port {
5022
5023my $port = shift;
5024my $pbos = shift;
5025my $cmt = shift;
5026my $nport;
5027
5028die "No port passed in parameter. Report to dev team\n" if (not defined $port);
5029# key is project on VM, but machine tuple for RM
5030if ($cmt =~ /^RM/i) {
5031    $nport = $port->{"$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}"};
5032} else {
5033    $nport = $port->{$ENV{'PBPROJ'}};
5034}
5035pb_log(2,"pb_get_port with $nport\n");
5036if ($cmt =~ /^VM/i) {
5037    # Maybe a port was given as parameter so overwrite
5038    $nport = "$pbport" if (defined $pbport);
5039    # Maybe in // mode so use the env var set up as an offset to the base port
5040    $nport = $nport + $ENV{'PBVMPORT'} -1 if ((defined $pbparallel) && (defined $ENV{'PBVMPORT'}));
5041}
5042pb_log(2,"pb_get_port returns $nport\n");
5043return($nport);
5044}
5045
5046sub pb_set_port { 
5047       
5048my ($pid,$ident) = @_;
5049pb_log(2,"pb_set_port for VM ($pid), id $ident\n");
5050$ENV{'PBVMPORT'} = $ident;
5051pb_log(2,"pb_set_port sets PBVMPORT in env to $ENV{'PBVMPORT'}\n");
5052}
5053
5054sub pb_set_parallel {
5055
5056my $vtype = shift;
5057
5058pb_log(2,"pb_set_parallel vtype: $vtype\n");
5059# Take care of memory size if VM, parallel mode and more than 1 action
5060if ((defined $pbparallel) && ($pbparallel ne 1) && ($vtype eq "vm")) {
5061    eval
5062    {
5063        require Linux::SysInfo;
5064        Linux::SysInfo->import();
5065    };
5066    if ($@) {
5067        # Linux::SysInfo not found
5068        pb_log(1,"ADVISE: Install Linux::SysInfo to benefit from automatic parallelism optimization.\nOr optimize manually pbparallel in your pb.yml file\nUsing $pbparallel processes max at a time for the moment\nWARNING: This may consume too much memory for your system\n");
5069    } else {
5070        # Using the memory size
5071        my $si = Linux::SysInfo::sysinfo();
5072        if (not defined $si) {
5073            pb_log(1,"ADVISE: Install Linux::SysInfo to benefit from automatic parallelism optimization.\nOr optimize manually pbparallel in your pb.yml file\nUsing $pbparallel processes max at a time for the moment\nWARNING: This may consume too much memory for your system\n");
5074        } else {
5075            # Keep the number of VM whose memory can be allocated
5076            my $ram = $si->{"totalram"}-$si->{"sharedram"}-$si->{"bufferram"};
5077            my $ram2;
5078            my ($vmmem) = pb_conf_get_if("vmmem");
5079
5080            my $v = "default";
5081            if ((defined $vmmem) and (defined $vmmem->{$v})) {
5082                $ram2 = $vmmem->{$v};
5083            } else {
5084                # Default for KVM/QEMU
5085                $ram2 = 128;
5086            }
5087            $pbparallel = sprintf("%d",$ram/$ram2);
5088        }
5089        pb_log(1,"Using $pbparallel processes at a time\n");
5090    }
5091}
5092pb_log(2,"pb_set_parallel returns: $pbparallel\n") if (defined $pbparallel);
5093return($pbparallel);
5094}
5095
5096sub pb_get_sudocmds { 
5097       
5098my $pbos = shift;
5099my @cmds = @_;
5100my %sudocmds;
5101
5102pb_log(2,"pb_get_sudocmds entering with lines:".Dumper(@cmds)."\n");
5103foreach my $c (split(/;/,$pbos->{'update'}),split(/;/,$pbos->{'install'}),@cmds) {
5104    pb_log(2,"pb_get_sudocmds analyses $c\n");
5105    next if ($c !~ /^\s*sudo/);
5106    # remove sudo and leading spaces
5107    $c =~ s/^\s*sudo\s+//;
5108    # keep only the command, not the params
5109    $c =~ s/([^\s]+)\s.*$/$1/;
5110    $sudocmds{$c} = "";
5111}
5112pb_log(2,"pb_get_sudocmds returns ".Dumper(keys %sudocmds)."\n");
5113return(keys %sudocmds);
5114}
5115
5116sub pb_sign_setenv {
5117
5118my ($passfile, $passphrase, $passpath) = pb_conf_get_if("pbpassfile","pbpassphrase","pbpasspath");
5119$ENV{'PBPASSPHRASE'} = $passphrase->{$ENV{'PBPROJ'}} if ((not defined $ENV{'PBPASSPHRASE'}) && (defined $passphrase->{$ENV{'PBPROJ'}}));
5120$ENV{'