source: devel/pb/bin/pb @ 2357

Last change on this file since 2357 was 2357, checked in by bruno, 9 months ago

Fix fedora 23 ctn support

The way ctn was built, the /etc directory wasn't owned by root, not
allowing root to pass the sudo command.
We now force /etc owner and modes to avoid that type of issues.

  • Property svn:executable set to *
File size: 183.6 KB
Line 
1#!/usr/bin/perl -w
2#
3# Project Builder main application
4#
5# $Id$
6#
7# Copyright B. Cornec 2007-today
8# Eric Anderson's changes are (c) Copyright 2012 Hewlett Packard
9# Provided under the GPL v2
10
11# Syntax: see at end
12
13use strict 'vars';
14
15# The modules mentioned here are required by pb when used both
16# locally or inside a VE/VM/RM
17# Additional required modules only used locally are called with a require
18# in their respective section
19use Getopt::Long qw(:config auto_abbrev no_ignore_case);
20use Carp qw/confess cluck/;
21use Data::Dumper;
22use English;
23use File::Basename;
24use File::Copy;
25use File::stat;
26use File::Find;
27use Time::localtime qw(localtime);
28use POSIX qw(strftime);
29use lib qw (lib);
30use ProjectBuilder::Version;
31use ProjectBuilder::Base;
32use ProjectBuilder::Display;
33use ProjectBuilder::Conf;
34use ProjectBuilder::Distribution;
35use ProjectBuilder::VCS;
36use ProjectBuilder::CMS;
37use ProjectBuilder::Env;
38use ProjectBuilder::Filter;
39use ProjectBuilder::Changelog;
40use ProjectBuilder::VE;
41
42# Global variables
43$Global::pb_stop_on_error = 0;      # False by default
44$Global::pb_show_sudo = 0;          # False by default
45
46my %opts;                           # CLI Options
47my $action;                         # action to realize
48my $test = "FALSE";                 # Not used
49my $pbforce = 0;                    # Force VM stop
50my $pbsnap = 0;                     # Do not use snapshot mode for VM/VE by default
51my $pbkeep = 0;                     # keep temporary directory at the end
52my $option = "";                    # Not used
53my @pkgs;                           # list of packages
54my $pbtag;                          # Global Tag variable
55my $pbver;                          # Global Version variable
56my %pbscript;                       # Name of the scripts per V
57my $pbrev;                          # Global REVISION variable
58my $pbtarget = undef;               # Target os-ver-arch you want to build for
59my $pbport;                         # Port to use to connect to the VM/RM
60my $newver;                         # New version to create
61my $pbimage = undef;                # ISO image for the VM to create or docker image for the VE
62my $vetype = undef;                 # Type of Virtual environment to deal with
63
64my @date = pb_get_date();
65my $pbdate = strftime("%Y-%m-%d", @date);
66
67=pod
68
69=head1 NAME
70
71pb, aka project-builder.org - builds packages for your projects
72
73=head1 DESCRIPTION
74
75pb helps you build various packages directly from your project sources.
76Those sources could be handled by a CMS (Configuration Management System)
77such as Subversion, CVS, Git, Mercurial... or being a simple reference to a compressed tar file.
78It's based on a set of configuration files, a set of provided macros to help
79you keeping build files as generic as possible. For example, a single .spec
80file should be required to generate for all rpm based distributions, even
81if you could also have multiple .spec files if required.
82
83=head1 SYNOPSIS
84
85pb [-vhSq][-r pbroot][-p project][[-s script -a account -P port][-T VEtype][-t [os-ver-arch]][-m os-ver-arch[,...]]][-g][-i image] <action> [<pkg1> ...]
86
87pb [--verbose][--help][--man][--quiet][--snapshot][--revision pbroot][--project project][[--script script --account account --port port][-T VEtype][--target [os-ver-arch]][--machine os-ver-arch[,...]]][--nographic][--image image][--rebuild] <action> [<pkg1> ...]
88
89=head1 OPTIONS
90
91=over 4
92
93=item B<-v|--verbose>
94
95Increase verbosity
96
97=item B<-q|--quiet>
98
99Do not print any output.
100
101=item B<-h|--help>
102
103Print a brief help message and exits.
104
105=item B<-S|--snapshot>
106
107Use the snapshot mode of VMs or VEs
108
109=item B<--man>
110
111Prints the manual page and exits.
112
113=item B<-t|--target os-ver-arch>
114
115Name of the target system you want to build for.
116All if none precised.
117
118=item B<-m|--machine os-ver-arch[,os-ver-arch,...]>
119
120Name of the Virtual Machines (VM), Virtual Environments (VE) or Remote Machines (RM)
121you want to build on (comma separated).
122All if none precised (or use the env variable PBV).
123
124=item B<-T|--vetype VEtype]>
125
126Type of Virtual Environments (VE)
127Can be chroot or docker.
128
129=item B<-s|--script script>
130
131Name of the script you want to execute on the related VMs/VEs/RMs.
132
133=item B<-g|--nographic>
134
135Do not launch VMs in graphical mode.
136
137=item B<-i|--image image>
138
139It could be either:
140- The name of the ISO image of the distribution you want to install on the related VMs
141- The name of the docker image of the distribution you want to install on the related VEs
142
143
144=item B<-a|--account account>
145
146Name of the account to use to connect on the related VMs/RMs.
147
148=item B<-P|--port port_number>
149
150Port number to use to connect on the related VMs/RMs.";
151
152=item B<-p|--project project_name>
153
154Name of the project you're working on (or use the env variable PBPROJ)
155
156=item B<-r|--revision revision>
157
158Path Name of the project revision under the CMS (or use the env variable PBROOT)
159
160=item B<-V|--version new_version>
161
162New version of the project to create based on the current one.
163
164=item B<-k|--keep>
165
166Keep the temporary dir where files have been created in or der to help debug
167
168=item B<--rebuild>
169
170Only valid with the checkssh action, it alllows to automatically relaunch the build of the failed packages
171
172=item B<--no-stop-on-error>
173
174Continue through errors with best effort.
175
176=back
177
178=head1 ARGUMENTS
179
180<action> can be:
181
182=over 4
183
184=item B<sbx2build>
185
186Create tar files for the project under your CMS.
187Current state of the exported content is taken.
188CMS supported are SVN, SVK, CVS, Git and Mercurial
189parameters are packages to build
190if not using default list
191
192=item B<cms2build>
193
194Create tar files for the project under your CMS.
195Current state of the CMS is taken.
196CMS supported are SVN, SVK, CVS, Git and Mercurial
197parameters are packages to build
198if not using default list
199
200=item B<build2prep>
201
202Prepare the environment for build by installing required dependencies. Done once on the build system.
203
204=item B<build2pkg>
205
206Create packages for your running distribution
207
208=item B<cms2pkg>
209
210cms2build + build2pkg
211
212=item B<sbx2pkg>
213
214sbx2build + build2pkg
215
216=item B<sbx2pkg2ins>
217
218sbx2pkg + final install of packages
219
220=item B<sbx2prep>
221
222sbx2build + build2prep
223
224=item B<build2ssh>
225
226Send the tar files to a SSH host
227
228=item B<sbx2ssh>
229
230sbx2build + build2ssh
231
232=item B<cms2ssh>
233
234cms2build + build2ssh
235
236=item B<pkg2ssh>
237
238Send the packages built to a SSH host
239
240=item B<build2vm>
241
242Create packages in VMs, launching them if needed
243and send those packages to a SSH host once built
244VM type supported are QEMU and KVM
245
246=item B<build2ve>
247
248Create packages in VEs, creating it if needed
249and send those packages to a SSH host once built
250
251=item B<build2rm>
252
253Create packages in RMs, which should pre-exist,
254and send those packages to a SSH host once built
255RM means Remote Machine, and could be a physical or Virtual one.
256This is one buildfarm integration for pb.
257
258=item B<prepvm>
259
260Prepare the VMs to have all requirements to build the project
261
262=item B<prepve>
263
264Prepare the VEs to have all requirements to build the project
265
266=item B<preprm>
267
268Prepare the RMs to have all requirements to build the project
269
270=item B<sbx2vm>
271
272sbx2build + build2vm
273
274=item B<sbx2ve>
275
276sbx2build + build2ve
277
278=item B<sbx2docker>
279
280sbx2build + build2ve with a potential build of all necessary docker containers to perform it
281
282=item B<sbx2rm>
283
284sbx2build + build2rm
285
286=item B<cms2vm>
287
288cms2build + build2vm
289
290=item B<cms2ve>
291
292cms2build + build2ve
293
294=item B<cms2rm>
295
296cms2build + build2rm
297
298=item B<launchvm>
299
300Launch one virtual machine
301
302=item B<launchve>
303
304Launch one virtual environment
305
306=item B<script2vm>
307
308Launch one virtual machine if needed
309and executes a script on it
310
311=item B<script2ve>
312
313Execute a script in a virtual environment
314
315=item B<script2rm>
316
317Execute a script on a remote machine
318
319=item B<newvm>
320
321Create a new virtual machine
322
323=item B<newve>
324
325Create a new virtual environment
326
327=item B<setupvm>
328
329Setup a virtual machine for pb usage
330
331=item B<setupve>
332
333Setup a virtual environment for pb usage
334
335=item B<setuprm>
336
337Setup a remote machine for pb usage
338
339=item B<sbx2setupvm>
340
341Setup a virtual machine for pb usage using the sandbox version of pb instead of the latest stable
342Reserved to dev team.
343
344=item B<sbx2setupve>
345
346Setup a virtual environment for pb usage using the sandbox version of pb instead of the latest stable
347Reserved to dev team.
348
349=item B<sbx2setuprm>
350
351Setup a remote machine for pb usage using the sandbox version of pb instead of the latest stable
352Reserved to dev team.
353
354=item B<build2setupvm>
355
356Setup a virtual machine for pb usage using the build available
357Reserved to dev team.
358
359=item B<build2setupve>
360
361Setup a virtual environment for pb usage using the build available
362Reserved to dev team.
363
364=item B<build2setuprm>
365
366Setup a remote machine for pb usage using the build available
367Reserved to dev team.
368
369=item B<snapvm>
370
371Snapshot a virtual machine for pb usage
372
373=item B<snapve>
374
375Snapshot a virtual environment for pb usage
376
377=item B<updateconf>
378
379Update the configuration files to the latest format
380
381=item B<updatevm>
382
383Update the distribution in the virtual machine
384
385=item B<updateve>
386
387Update the distribution in the virtual environment
388
389=item B<updaterm>
390
391Update the distribution in the remote machine
392
393=item B<test2pkg>
394
395Test a package locally
396
397=item B<test2vm>
398
399Test a package in a virtual machine
400
401=item B<test2ve>
402
403Test a package in a virtual environment
404
405=item B<test2rm>
406
407Test a package in a remote machine
408
409=item B<checkssh>
410
411Check the delivery of the packages on the repository
412
413=item B<checkps>
414
415Check the process running the VM concerned
416
417=item B<newver>
418
419Create a new version of the project derived
420from the current one
421
422=item B<newproj>
423
424Create a new project and a template set of
425configuration files under pbconf
426
427=item B<announce>
428
429Announce the availability of the project through various means
430
431=item B<sbx2webssh>
432
433Create tar files for the website under your CMS.
434Current state of the exported content is taken.
435Deliver the content to the target server using ssh from the exported dir.
436
437=item B<cms2webssh>
438
439Create tar files for the website from your CMS.
440Deliver the content to the target server using ssh from the DVCS.
441
442=item B<sbx2webpkg>
443
444Create tar files for the website under your CMS.
445Current state of the exported content is taken.
446
447=item B<cms2webpkg>
448
449Create tar files for the website under your CMS.
450
451=item B<getconf>
452
453Print the full configuration parameters as found in the various configuration files. Help to debug conf issues.
454Also accepts a parameter to display only this value, and a VM/VE/RM
455
456=item B<getvar>
457
458Print the full variables expanded based on the distribution tuple. Help to debug conf issues.
459Also accepts a parameter to display only the values for this package, and a VM/VE/RM
460
461=item B<clean>
462
463Purge the build and delivery directories related to the current project
464
465=item B<cleanssh>
466
467Purge the ssh server of its packages (only for testver and test packages)
468
469=back
470
471<pkgs> can be a list of packages, the keyword 'all' or nothing, in which case the default list of packages is taken (corresponding to the defpkgdir list of arguments in the configuration file).
472
473=head1 WEB SITES
474
475The main Web site of the project is available at L<http://www.project-builder.org/>. Bug reports should be filled using the trac instance of the project at L<http://trac.project-builder.org/>.
476
477=head1 USER MAILING LIST
478
479None exists for the moment.
480
481=head1 CONFIGURATION FILES
482
483Each pb user may have a configuration in F<$HOME/.pbrc.yml>. The values in this file may overwrite any other configuration file value.
484
485Here is an example of such a configuration file:
486
487 #
488 # Define for each project the URL of its pbconf repository
489 # No default option allowed here as they need to be all different
490 #
491 # URL of the pbconf content
492 # This is the format of a classical URL with the extension of additional schema such as
493 # svn+ssh, cvs+ssh, ...
494 #
495  pbconfurl:
496    linuxcoe : cvs+ssh://:ext:bcornec@linuxcoe.cvs.sourceforge.net:/cvsroot/linuxcoe/pbconf
497
498 # This is normaly defined in the project's configuration file
499 # Url of the project
500 #
501  pbprojurl:
502    linuxcoe : cvs+ssh://:ext:bcornec@linuxcoe.cvs.sourceforge.net:/cvsroot/linuxcoe
503 
504 # All these URLs needs to be defined here as the are the entry point
505 # for how to build packages for the project
506 #
507  pbconfurl:
508    pb: svn+ssh://svn.project-builder.org/mondo/svn/pb/pbconf
509  pbconfurl:
510    mondorescue: svn+ssh://svn.project-builder.org/mondo/svn/project-builder/mondorescue/pbconf
511  pbconfurl:
512    collectl: svn+ssh://bruno@svn.mondorescue.org/mondo/svn/project-builder/collectl/pbconf
513  pbconfurl:
514    netperf: svn+ssh://svn.mondorescue.org/mondo/svn/project-builder/netperf/pbconf
515 
516 # Under that dir will take place everything related to pb
517 # If you want to use VMs/chroot/..., then use $ENV{'HOME'} to make it portable
518 # to your VMs/chroot/...
519 # if not defined then /var/cache
520  pbdefdir:
521    default: $ENV{'HOME'}/project-builder
522  pbdefdir:
523    pb: $ENV{'HOME'}
524  pbdefdir:
525    linuxcoe: $ENV{'HOME'}/LinuxCOE/cvs
526  pbdefdir:
527    mondorescue: $ENV{'HOME'}/mondo/svn
528 
529 # pbconfdir points to the directory where the CMS content of the pbconfurl is checked out
530 # If not defined, pbconfdir is under pbdefdir/pbproj/pbconf
531  pbconfdir:
532    linuxcoe: $ENV{'HOME'}/LinuxCOE/cvs/pbconf
533  pbconfdir:
534    mondorescue: $ENV{'HOME'}/mondo/svn/pbconf
535 
536 # pbdir points to the directory where the CMS content of the pbprojurl is checked out
537 # If not defined, pbdir is under pbdefdir/pbproj
538 # Only defined if we have access to the dev of the project
539  pbdir:
540    linuxcoe: $ENV{'HOME'}/LinuxCOE/cvs
541  pbdir:
542    mondorescue: $ENV{'HOME'}/mondo/svn
543 
544 # -daemonize doesn't work with qemu 0.8.2
545  vmopt:
546    default: -m 384
547
548=head1 COMMAND DETAILS
549
550=head2 newproj
551
552The newproj command creates a new project-builder project.  To run this command you first need to define two variables in your ~/.pbrc.yml file:
553
554 pbconfurl I<project> = file:///home/anderse/.git/project-builder-config/I<project>
555 pbdefdir default = $ENV{'HOME'}/cache-project-builder
556
557The first line defines the version controlled configuration information and the second defines the root directory for project-builder to use.
558
559You can then run the command:
560
561 % pb -p I<$project> -r I<$version> newproj I<$pkg>
562
563to create the new project. Running the newproj command will then generate the file $pbdefdir/$project/pbconf/$version/$project.yml, and the directory $pbdefdir/$project/pbconf/$version/$pkg. You will need to edit those files to make the later commands work.
564
565=head2 cms2build
566
567The cms2build command takes your files from the content management system and makes the two tar files that are necessary for building files. You need to have run the newproj command first.  Then there are several steps for running this command:
568
569=over 4
570
571=item Update your $project.yml configuration file.
572
573You need to set the pbprojurl, pbrepo, pbwf, pbpackager, projver, projtag, testver, delivery, and defpkgdir lines as described in the configuration file. The pbprojurl entry is used to find the source for your package. The pbrepo entry is used to build the .repo or .sources.list files for use by downloaders of the package. The pbwf entry indicates that the source tar file is named by I<package-name>-I<version>. The pbpackager entry will be stored in the packages and should be you or your team. The projver/projtag entries indicate the version of the software and the version of the packaging scripts. The testver entry when true indicates that the package is in a test version, so no log file is computed (can be long), and version is made up using a timstamp. The delivery entry gives the subdirectory under which the packages will be delivered on the repository, and the defpkgdir entry corresponds to the local subdirectory hosting the package content.
574
575For example:
576
577 pbprojurl Lintel = file:///home/anderse/projects/Lintel-0.2012.02.28.tar.gz
578 pbrepo Lintel = http://tesla.hpl.hp.com/opensource
579 pbwf Lintel = 1
580 pbpackager Lintel = Eric Anderson <eric.anderson4@hp.com>
581 projver Lintel = 0.2012.02.28
582 projtag Lintel = 1
583 testver Lintel = false
584 delivery Lintel = production
585 defpkgdir Lintel = Lintel-0.2012.02.28
586
587=item Create the build .tar.gz files:
588
589Then you need to take those files and create the initial tar files.  Run a command like:
590
591 % pb -p $project -r $version cms2build
592
593To create the $pbdefdir/$project/pbdelivery/$project-$version.{,pbconf}.tar.gz files, the $version-$projtag.yml and pbrc.yml files in the same directory.
594
595=back
596
597=head2 build2pkg
598
599The build2pkg command takes the tar files created in the cms2build step and attempts to build binary packages for your current operating system. There are two steps:
600
601=over 4
602
603=item Update your filters and build files.
604
605You probably need to edit the files describing the build steps in one of the $pbdefdir/$project/pbconf/$version/$project/{deb,rpm,pkg} directories and the filters in $pbdefdir/$project/pbconf/$version/pbfilter. Note that you can define additional filters and transformations in the filters files. The build files will be filtered by the filters defined in the filters files to generate the inputs to the build step. Therefore, if you change those files, you need to re-run the cms2build step.
606
607=item Build the package.
608
609Then you can run a command like:
610
611 % pb -p $project -r $version build2pkg
612
613To create the files in $project/pbbuild that comprise your binary package(s).
614
615=back
616
617=head2 newve
618 
619The newve command creates a new virtual environment, i.e. a chrooted OS for building packages. Using a virtual environment is an efficient way to build packages on a related set of operating systems. The OS's have to be related because the kernel will be shared.  Steps:
620
621=over 4
622
623=item Update ~/.pbrc.yml
624
625Update your ~/.pbrc.yml file to specify the vepath, velist, velogin, and vetype variables, e.g.:
626
627  vepath:
628    default: $ENV{'HOME'}/cache-project-builder/chroot
629  velist:
630    default: debian-6.0-i386
631  velogin:
632    default: pb
633  vetype:
634    default: chroot
635
636If you are building for rpm style OS's, update the verpmtype option, and install the appropriate tool.
637
638  verpmtype:
639    default: rpmbootstrap
640
641You may also choose to specify a mirror for the OS packages, and optionally http/ftp proxies.  You can specify the proxies either through environment variables ($http_proxy/$https_proxy/$ftp_proxy) or in the configuration file. The configuration file will be used if no corresponding environment variable has been set. For example, for debian and with a local squid proxy:
642
643  rbsmirrorsrv:
644    debian: http://mirrors1.kernel.org/debian/
645  http_proxy:
646    default: http://localhost:3128/
647  https_proxy:
648    default: http://localhost:3128/
649  ftp_proxy:
650    default: http://localhost:3128/
651
652=item Run the cms2build command
653
654If you have deleted your $package/pbdelivery directory, re-run the cms2build command as in the earlier step. This step is necessary to generate the I<package>/pbdelivery/pbrc.yml file.
655
656=item Create the new virtual environment
657
658Initialize the new operating system. This step will install the core OS packages for the virtual environment, e.g.:
659
660 % pb -v -p $project -m debian-6.0-i386 newve
661
662=back
663
664=head2 setupve
665
666The setupve command prepares a virtual environment for use by project builder. In particular it installs project-builder from the packages into the virtual environment. Two sub-steps are necessary:
667
668=over 4
669
670=item Update $project.yml
671
672You need to have a sshhost entry for setupve to work, so add one, even an invalid one, e.g.:
673
674  sshhost:
675    $project: foo.example.org
676
677=item Setup the virtual environment
678
679 % pb -v -p $project -m debian-6.0-i386 setupve
680
681If you prefer to install the current SVN version of project builder, you can substitute the setupve option by the sbx2setupv one.
682
683=back
684
685=head2 build2ve
686
687The build2ve command is similar to the build2pkg command in that it will take the sources created by cms2build and turn them into binary packages. The command has two differences. First, it creates the packages in a virtual environment, i.e. the one made by an earlier setupve setup. Second it copies the resulting packages to a repository and builds the repository meta-data needed.
688
689Three sub-steps are needed:
690
691=over 4
692
693=item Update $project.yml
694
695You need to have a valid sshdir and sshhost entry for build2ve to work, so add them. Note that you need to be able to ssh from the host you run the command on to the repository host, preferably without needing to type in a password, so using ssh-agent or having a special passwordless project-builder ssh key will make this step easier.
696
697  sshhost:
698    $project: localhost
699  sshdir:
700    $project: $home/cache-project-builder/repos
701
702You may also need to specify additional repository files to use or rpms to install. Note the URL for repositories is not the URL of the repository, but the URL of a file that can be put in the yum.repos.d or apt.sources.d directory.
703
704  addrepo:
705    centos-5-i386: http://localhost/pb/centos-extras.repo,http://mirror.centos.org/centos/5/extras/i386/RPMS/chrpath-0.13-3.el5.centos.i386.rpm
706
707=item Update your filters and build files
708
709You may need to update your filter files as in the build2pkg step if you are building for a new OS or architecture.
710
711=item Build the packages and copy them to the repository
712
713 % pb -v -p $project -m debian-6.0-i386 build2ve
714
715=back
716
717*Debugging:* If the build fails (and you did not specify the --no-stop-on-error) option, then the virtual environment and scripts should still be present and configured to build the package. You can run a command like 'sudo setarch i386 chroot $path bash' in order to get into the environment. In your log you should see a command like that. From there you can go into the /home/pb directory as the pb user and run the same style of pb commands as you did when doing build2pkg. This will help you figure out what has gone wrong in the build in the virtual environment.
718
719=head1 AUTHORS
720
721The Project-Builder.org team L<http://trac.project-builder.org/> lead by Bruno Cornec L<mailto:bruno@project-builder.org>.
722
723=head1 COPYRIGHT
724
725Project-Builder.org is distributed under the GPL v2.0 license
726described in the file C<COPYING> included with the distribution.
727
728=cut
729
730# ---------------------------------------------------------------------------
731
732my ($projectbuilderver,$projectbuilderrev,$projectbuilderconfver) = pb_version_init();
733my $appname = "pb";
734
735# Initialize the syntax string
736
737pb_syntax_init("$appname (aka project-builder.org) Version $projectbuilderver-$projectbuilderrev Conf $projectbuilderconfver\n");
738
739GetOptions("help|?|h+" => \$opts{'h'}, 
740        "account|a=s" => \$opts{'a'},
741        "force|f" => \$opts{'f'},
742        "nographic|g" => \$opts{'g'},
743        "image|i=s" => \$opts{'i'},
744        "log-files|l=s" => \$opts{'l'},
745        "machines|mock|m=s" => \$opts{'m'},
746        "man" => \$opts{'man'},
747        "port|P=i" => \$opts{'P'},
748        "project|p=s" => \$opts{'p'},
749        "quiet|q" => \$opts{'q'},
750        "revision|r=s" => \$opts{'r'},
751        "rebuild" => \$opts{'rebuild'},
752        "snapshot|S" => \$opts{'S'},
753        "script|s=s" => \$opts{'s'},
754        "type|T:s" => \$opts{'T'},
755        "target|t:s" => \$opts{'t'},
756        "version" => \$opts{'version'},
757        "V=s" => \$opts{'V'},
758        "verbose|v+" => \$opts{'v'},
759        "keep|k" => \$opts{'k'},
760        "stop-on-error!" => \$Global::pb_stop_on_error,
761) || pb_syntax(-1,0);
762
763if (defined $opts{'l'}) {
764    open(pbLOG,"> $opts{'l'}") || die "Unable to log to $opts{'l'}: $!";
765    $pbLOG = \*pbLOG;
766    $pbdebug = 0  if ($pbdebug == -1);
767    }
768pb_log_init($pbdebug, $pbLOG);
769
770if (defined $opts{'h'}) {
771    pb_syntax(0,1);
772}
773if (defined $opts{'version'}) {
774    pb_syntax(0,0);
775}
776if (defined $opts{'man'}) {
777    pb_syntax(0,2);
778}
779if (defined $opts{'v'}) {
780    $pbdebug = $opts{'v'};
781}
782if (defined $opts{'f'}) {
783    $pbforce=1;
784}
785if (defined $opts{'q'}) {
786    $pbdebug=-1;
787}
788if (defined $opts{'S'}) {
789    $pbsnap=1;
790}
791if (defined $opts{'k'}) {
792    $pbkeep=1;
793}
794pb_display_init("text","");
795
796# Handle root of the project if defined
797if (defined $opts{'r'}) {
798    $ENV{'PBROOTDIR'} = $opts{'r'};
799}
800# Handle virtual machines if any
801if (defined $opts{'m'}) {
802    $ENV{'PBV'} = $opts{'m'};
803}
804if (defined $opts{'s'}) {
805    $pbscript{'default'} = $opts{'s'};
806    die "option -s requires a script having a full path name" if ($pbscript{'default'} !~ /^\//);
807}
808if (defined $opts{'a'}) {
809    $ENV{'PBACCOUNT'} = $opts{'a'};
810    die "option -a requires a -s script option" if (not defined $pbscript{'default'});
811}
812if (defined $opts{'P'}) {
813    $pbport = $opts{'P'};
814}
815if (defined $opts{'V'}) {
816    $newver = $opts{'V'};
817}
818if (defined $opts{'i'}) {
819    $pbimage = $opts{'i'};
820}
821if (defined $opts{'t'}) {
822    $pbtarget = $opts{'t'};
823}
824if (defined $opts{'T'}) {
825    $vetype = $opts{'T'};
826}
827
828# Get Action
829$action = shift @ARGV;
830die pb_syntax(-1,1) if (not defined $action);
831
832my ($filteredfiles, $supfiles, $defpkgdir, $extpkgdir);
833my $pbinit = undef;
834$pbinit = 1 if ($action =~ /^newproj$/);
835
836# Handles project name if any
837# And get global params
838($filteredfiles, $supfiles, $defpkgdir, $extpkgdir) = pb_env_init($opts{'p'},$pbinit,$action,$pbkeep);
839
840#
841# Check for command requirements
842#
843my ($req,$opt,$pbpara,$pbshowsudo,$pbstoponerr) = pb_conf_get_if("oscmd","oscmdopt","pbparallel","pbshowsudo","pbstoponerr");
844pb_check_requirements($req,$opt,$appname);
845
846$Global::pb_show_sudo = 1 if ((defined $pbshowsudo) && (defined $pbshowsudo->{$ENV{'PBPROJ'}}) && ($pbshowsudo->{$ENV{'PBPROJ'}} =~ /true/oi));
847$Global::pb_stop_on_error = 1 if ((defined $pbstoponerr) && (defined $pbstoponerr->{$ENV{'PBPROJ'}}) && ($pbstoponerr->{$ENV{'PBPROJ'}} =~ /true/oi));
848
849#
850# Check if we can launch some actions in // with Parallel::ForkManager
851#
852my $pbparallel = $pbpara->{$appname} if (defined $pbpara);
853if (not defined $pbparallel) {
854    eval
855    {
856        require Sys::CPU;
857        Sys::CPU->import();
858    };
859    if ($@) {
860        # Sys::CPU not found, defaulting to 1
861        pb_log(1,"ADVISE: Install Sys::CPU to benefit from automatic parallelism optimization.\nOr use pbparallel in your pb.yml file\nOnly 1 process at a time for the moment\n");
862        $pbparallel = 1;
863    } else {
864        # Using the number of cores
865        $pbparallel = Sys::CPU::cpu_count();
866        pb_log(1,"Using parallel mode with $pbparallel processes\n");
867    }
868}
869
870eval
871{
872    require Parallel::ForkManager;
873    Parallel::ForkManager->import();
874};
875# Parallel::ForkManager not found so no // actions
876if ($@) {
877    $pbparallel = undef;
878    pb_log(1,"ADVISE: Install Parallel::ForkManager to benefit from automatic parallelism optimization.\nOnly 1 process at a time for the moment\n");
879}
880
881if ($action ne "getconf") {
882    pb_log(0,"Project: $ENV{'PBPROJ'}\n");
883    pb_log(0,"Action: $action\n");
884}
885
886my $do_install = undef;
887# Possibility to add a 2ins to some commands to launch automatic installation
888$do_install = 1 if ($action =~ /2ins$/);
889
890# Act depending on action
891if ($action =~ /^cms2build$/) {
892    pb_cms2build("CMS");
893} elsif ($action =~ /^sbx2build$/) {
894    pb_cms2build("SandBox");
895} elsif ($action =~ /^build2prep$/) {
896    pb_build2prep();
897} elsif ($action =~ /^sbx2prep$/) {
898    pb_cms2build("SandBox");
899    pb_build2prep();
900} elsif ($action =~ /^build2pkg/) {
901    pb_build2pkg($do_install);
902} elsif ($action =~ /^cms2pkg/) {
903    pb_cms2build("CMS");
904    pb_build2pkg($do_install);
905} elsif ($action =~ /^sbx2pkg/) {
906    pb_cms2build("SandBox");
907    pb_build2pkg($do_install);
908} elsif ($action =~ /^build2ssh$/) {
909    pb_build2ssh();
910} elsif ($action =~ /^cms2ssh$/) {
911    pb_cms2build("CMS");
912    pb_build2ssh();
913} elsif ($action =~ /^sbx2ssh$/) {
914    pb_cms2build("SandBox");
915    pb_build2ssh();
916} elsif ($action =~ /^pkg2ssh$/) {
917    pb_pkg2ssh();
918} elsif ($action =~ /^build2rm$/) {
919    pb_build2v("rm","build");
920} elsif ($action =~ /^build2ve$/) {
921    pb_build2v("ve","build");
922} elsif ($action =~ /^build2vm$/) {
923    pb_build2v("vm","build");
924} elsif ($action =~ /^sbx2preprm$/) {
925    pb_cms2build("SandBox");
926    pb_build2v("rm","prep");
927} elsif ($action =~ /^sbx2prepve$/) {
928    pb_cms2build("SandBox");
929    pb_build2v("ve","prep");
930} elsif ($action =~ /^sbx2prepvm$/) {
931    pb_cms2build("SandBox");
932    pb_build2v("vm","prep");
933} elsif ($action =~ /^preprm$/) {
934    pb_build2v("rm","prep");
935} elsif ($action =~ /^prepve$/) {
936    pb_build2v("ve","prep");
937} elsif ($action =~ /^prepvm$/) {
938    pb_build2v("vm","prep");
939} elsif ($action =~ /^cms2rm$/) {
940    pb_cms2build("CMS");
941    pb_build2v("rm","build");
942} elsif ($action =~ /^cms2ve$/) {
943    pb_cms2build("CMS");
944    pb_build2v("ve","build");
945} elsif ($action =~ /^sbx2rm$/) {
946    pb_cms2build("SandBox");
947    pb_build2v("rm","build");
948} elsif ($action =~ /^sbx2ve$/) {
949    pb_cms2build("SandBox");
950    pb_build2v("ve","build");
951} elsif ($action =~ /^cms2vm$/) {
952    pb_cms2build("CMS");
953    pb_build2v("vm","build");
954} elsif ($action =~ /^sbx2vm$/) {
955    pb_cms2build("SandBox");
956    pb_build2v("vm","build");
957} elsif ($action =~ /^launchvm$/) {
958    pb_parallel_launchv(undef,"vm",undef,3,$pbimage);
959} elsif ($action =~ /^launchve$/) {
960    pb_parallel_launchv(undef,"ve",undef,3,$pbimage);
961} elsif ($action =~ /^script2vm$/) {
962    die "action script2vm requires a -s script option" if (not defined $pbscript{'default'});
963    pb_script2v(\%pbscript,"vm");
964} elsif ($action =~ /^script2ve$/) {
965    die "action script2ve requires a -s script option" if (not defined $pbscript{'default'});
966    pb_script2v(\%pbscript,"ve");
967} elsif ($action =~ /^script2rm$/) {
968    die "action script2rm requires a -s script option" if (not defined $pbscript{'default'});
969    pb_script2v(\%pbscript,"rm");
970} elsif ($action =~ /^newver$/) {
971    pb_newver();
972} elsif ($action =~ /^sbx2docker$/) {
973    my $savproj = $ENV{'PBPROJ'};
974    my $pkg;
975    my @pkg;
976    if ($#ARGV == -1) {
977        $pkg = pb_cms_get_pkg($defpkgdir,$extpkgdir);
978        @pkgs = @$pkg;
979    } else {
980        @pkgs = @ARGV;
981    }
982    #pb_log(1,"Packages to print:".Dumper(@pkgs)."\n");
983   
984    # Get the list of all VE we need to work on
985    my ($vm,$all);
986    if (not defined $ENV{'PBV'}) {
987        ($vm,$all) = pb_get2v("ve");
988    } else {
989        @$vm = split(/,/,$ENV{'PBV'});
990    }
991   
992    # TODO: parallelize
993    foreach my $v (@$vm) {
994        my $done = 0;
995        my $pbos = pb_distro_get_context($v);
996        # Docker images are (for now !) regular and use distro:ver
997        $pbimage = "$pbos->{'name'}".":$pbos->{'version'}";
998        # the first steps needs to be done with the pb project
999        $ENV{'PBPROJ'} = $appname;
1000        my $found = pb_ve_docker_get_image("$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}");
1001        if (($pbforce == 1) || (not defined $found)) {
1002            # pb -p pb -m distro-ver-arch newve -i distro:ver
1003            pb_log(0,"Docker container $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'} not found, creating it\n");
1004            pb_parallel_launchv(undef,"ve",undef,0,$pbimage);
1005        }
1006        $found = pb_ve_docker_get_image("$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-pb");
1007        if (($pbforce == 1) || (not defined $found)) {
1008            pb_log(0,"Docker container $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-pb not found, creating it\n");
1009            # pb -p pb -m distro-ver-arch setupve
1010            if ($savproj eq "pb") {
1011                pb_cms2build("SandBox");
1012                $done = 1;
1013                pb_setup2v("ve","SandBox");
1014            } else {
1015                pb_setup2v("ve");
1016            }
1017        }
1018        # Now back to our original project
1019        $ENV{'PBPROJ'} = $savproj;
1020        $found = pb_ve_docker_get_image("$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-pb-$ENV{'PBPROJ'}");
1021        if (($pbforce == 1) || (not defined $found)) {
1022            pb_log(0,"Docker container $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-pb-$ENV{'PBPROJ'} not found, creating it\n");
1023            # pb -p pb -m distro-ver-arch prepve
1024            if ($savproj eq "pb") {
1025                pb_cms2build("SandBox");
1026                $done = 1;
1027            }
1028            pb_build2v("ve","prep");
1029        }
1030        pb_log(0,"Instantiating a Docker container from $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-pb-$ENV{'PBPROJ'}\n");
1031        if (($savproj ne "pb") || ($done == 0)) {
1032            pb_cms2build("SandBox");
1033        }
1034        pb_build2v("ve","build");
1035    }
1036} elsif ($action =~ /^newve$/) {
1037    pb_parallel_launchv(undef,"ve",undef,0,$pbimage);
1038} elsif ($action =~ /^newvm$/) {
1039    pb_parallel_launchv(undef,"vm",undef,0,$pbimage);
1040    pb_log(0, "Please ensure that sshd is running in your VM by default\n");
1041    pb_log(0, "and that it allows remote root login (PermitRootLogin yes in /etc/ssh/sshd_config)\n");
1042    pb_log(0, "Also ensure that network is up, firewalling correctly configured\n");
1043    pb_log(0, "and perl, Data::Dumper, tar, sudo, ntpdate and scp/ssh installed\n");
1044    pb_log(0, "You should then be able to login with ssh -p VMPORT root\@localhost (if VM started with pb)\n");
1045} elsif ($action =~ /^setuprm$/) {
1046    pb_setup2v("rm");
1047} elsif ($action =~ /^setupve$/) {
1048    pb_setup2v("ve");
1049} elsif ($action =~ /^setupvm$/) {
1050    pb_setup2v("vm");
1051} elsif ($action =~ /^sbx2setuprm$/) {
1052    die "This feature is limited to the pb project" if ($ENV{'PBPROJ'} ne $appname);
1053    pb_cms2build("SandBox");
1054    pb_setup2v("rm","SandBox");
1055} elsif ($action =~ /^sbx2setupve$/) {
1056    die "This feature is limited to the pb project" if ($ENV{'PBPROJ'} ne $appname);
1057    pb_cms2build("SandBox");
1058    pb_setup2v("ve","SandBox");
1059} elsif ($action =~ /^sbx2setupvm$/) {
1060    die "This feature is limited to the pb project" if ($ENV{'PBPROJ'} ne $appname);
1061    pb_cms2build("SandBox");
1062    pb_setup2v("vm","SandBox");
1063} elsif ($action =~ /^build2setuprm$/) {
1064    die "This feature is limited to the pb project" if ($ENV{'PBPROJ'} ne $appname);
1065    pb_setup2v("rm","SandBox");
1066} elsif ($action =~ /^build2setupve$/) {
1067    die "This feature is limited to the pb project" if ($ENV{'PBPROJ'} ne $appname);
1068    pb_setup2v("ve","SandBox");
1069} elsif ($action =~ /^build2setupvm$/) {
1070    die "This feature is limited to the pb project" if ($ENV{'PBPROJ'} ne $appname);
1071    pb_setup2v("vm","SandBox");
1072} elsif ($action =~ /^updaterm$/) {
1073    pb_update2v("rm");
1074} elsif ($action =~ /^updateve$/) {
1075    pb_update2v("ve");
1076} elsif ($action =~ /^updatevm$/) {
1077    pb_update2v("vm");
1078} elsif ($action =~ /^snapve$/) {
1079    pb_snap2v("ve");
1080} elsif ($action =~ /^snapvm$/) {
1081    pb_snap2v("vm");
1082} elsif ($action =~ /^test2pkg$/) {
1083    pb_test2pkg();
1084} elsif ($action =~ /^test2rm$/) {
1085    pb_build2v("rm","test");
1086} elsif ($action =~ /^test2ve$/) {
1087    pb_build2v("ve","test");
1088} elsif ($action =~ /^test2vm$/) {
1089    pb_build2v("vm","test");
1090} elsif ($action =~ /^newproj$/) {
1091    # Nothing to do - already done in pb_env_init
1092} elsif ($action =~ /^updateconf$/) {
1093    foreach my $f (@ARGV) {
1094        my $fyml = $f;
1095        $fyml =~ s/\.pbf/.yml/;
1096        $fyml =~ s/\.pb/.yml/;
1097        $fyml =~ s/\.pbrc/.pbrc.yml/;
1098        pb_vcs_conf_update_v0($f,$fyml);
1099    }
1100} elsif ($action =~ /^getconf$/) {
1101    my $pbos = pb_distro_get_context($ENV{'PBV'});
1102    pb_log(1,"Arguments to print:".Dumper(@ARGV)."\n");
1103    pb_distro_conf_print($pbos,@ARGV);
1104} elsif ($action =~ /^getvar$/) {
1105    my $pbos = pb_distro_get_context($ENV{'PBV'});
1106    my $pkg;
1107    my @pkg;
1108    if ($#ARGV == -1) {
1109        $pkg = pb_cms_get_pkg($defpkgdir,$extpkgdir);
1110        @pkgs = @$pkg;
1111    } else {
1112        @pkgs = @ARGV;
1113    }
1114    pb_log(1,"Packages to print:".Dumper(@pkgs)."\n");
1115    pb_filter_var_print($pbos,@pkgs);
1116} elsif ($action =~ /^clean$/) {
1117    pb_clean();
1118} elsif ($action =~ /^announce$/) {
1119    # For announce only. Require avoids the systematic load of these modules
1120    require DBI;
1121    require DBD::SQLite;
1122
1123    pb_announce("Announce");
1124} elsif ($action =~ /^cleanssh$/) {
1125    pb_announce("Clean");
1126} elsif ($action =~ /^checkssh$/) {
1127    pb_announce("Check");
1128} elsif ($action =~ /^checkps$/) {
1129    my ($ptr2) = pb_conf_get("vmcmd");
1130    my $vmcmd = $ptr2->{$ENV{'PBPROJ'}};
1131    my ($vexist,$vmmport) = pb_check_ps($vmcmd,$ENV{'PBV'});
1132    if (! $vexist) {
1133        pb_log(0,"No VM found for $ENV{'PBV'}\n");
1134    } else {
1135        pb_log(0,"Found an existing VM for $ENV{'PBV'} PID: $vexist - SSH port: $vmmport\n");
1136    }
1137} elsif ($action =~ /^sbx2webpkg$/) {
1138    require DBI;
1139    require DBD::SQLite;
1140
1141    pb_cms2build("SandBox","Web");
1142} elsif ($action =~ /^sbx2webssh$/) {
1143    require DBI;
1144    require DBD::SQLite;
1145
1146    pb_cms2build("SandBox","Web");
1147    pb_send2target("Web");
1148} elsif ($action =~ /^cms2webpkg$/) {
1149    require DBI;
1150    require DBD::SQLite;
1151
1152    pb_cms2build("CMS","Web");
1153} elsif ($action =~ /^cms2webssh$/) {
1154    require DBI;
1155    require DBD::SQLite;
1156
1157    pb_cms2build("CMS","Web");
1158    pb_send2target("Web");
1159} else {
1160    pb_log(0,"\'$action\' is not available\n");
1161    pb_syntax(-2,1);
1162}
1163
1164pb_exit();
1165
1166sub pb_cms2build {
1167
1168    my $param = shift;
1169    my $web = shift;
1170
1171    my $pkg;
1172    my @pkgs;
1173    my $webdir;
1174
1175    my %pkgs;
1176    my $pb;             # Structure to store conf info
1177
1178    die "pb_cms2build requires a parameter: SandBox or CMS" if (not defined $param);
1179
1180    # If Website, then pkg is only the website
1181    if (defined $web) {
1182        ($webdir) = pb_conf_get("webdir");
1183        pb_log(2,"webdir: ".Dumper($webdir)."\n");
1184        $pkgs[0] = $webdir->{$ENV{'PBPROJ'}};
1185        $extpkgdir = $webdir;
1186        pb_log(0,"Package: $pkgs[0]\n");
1187    } else {
1188        $pkg = pb_cms_get_pkg($defpkgdir,$extpkgdir);
1189        @pkgs = @$pkg;
1190    }
1191
1192    my ($scheme, $uri) = pb_vcs_init($pbinit,$param);
1193
1194    # We need 2 lines here
1195    my ($pkgv, $pkgt, $testver) = pb_conf_get_if("pkgver","pkgtag","testver");
1196
1197    # declare packager and repo for filtering
1198    my ($tmp1, $tmp2) = pb_conf_get("pbpackager","pbrepo");
1199    $ENV{'PBPACKAGER'} = $tmp1->{$ENV{'PBPROJ'}};
1200    $ENV{'PBREPO'} = $tmp2->{$ENV{'PBPROJ'}};
1201    my ($delivery) = pb_conf_get_if("delivery");
1202    $delivery->{$ENV{'PBPROJ'}} = "" if (not defined $delivery->{$ENV{'PBPROJ'}});
1203
1204    # If we deal with a test dir, we want to keep the date in tar file and dir name
1205    my $pbextdirpkg = "";
1206    my $pbextdir = "";
1207    if ((defined $testver) && (defined $testver->{$ENV{'PBPROJ'}}) && ($testver->{$ENV{'PBPROJ'}} =~ /true/i)) {
1208        $pbextdirpkg = strftime("%Y%m%d%H%M%S", @date);
1209        $pbextdir = ".0.".$pbextdirpkg;
1210    }
1211
1212    foreach my $pbpkg (@pkgs) {
1213        $ENV{'PBPKG'} = $pbpkg;
1214        $pbrev = $ENV{'PBREVISION'};
1215
1216        if ((defined $pkgv) && (defined $pkgv->{$pbpkg})) {
1217            $pbver = $pkgv->{$pbpkg};
1218        } else {
1219            $pbver = $ENV{'PBPROJVER'};
1220        }
1221        # If it's a test version, then build a tag to fit distributions need
1222        if ((defined $testver) && (defined $testver->{$ENV{'PBPROJ'}}) && ($testver->{$ENV{'PBPROJ'}} =~ /true/i)) {
1223            my $letter = "";
1224            if ($scheme =~ /svn/) {
1225                $letter = "s";
1226            } elsif ($scheme =~ /cvs/) {
1227                $letter = "c";
1228            } elsif ($scheme =~ /svk/) {
1229                $letter = "k";
1230            } elsif ($scheme =~ /hg/) {
1231                $letter = "h";
1232            } elsif ($scheme =~ /git/) {
1233                $letter = "g";
1234            } else {
1235                $letter = "f";
1236            }
1237            $pbtag = "0.$pbextdirpkg.$letter$pbrev";
1238            $ENV{'PBPROJTAG'} = $pbtag;
1239        } elsif ((defined $pkgt) && (defined $pkgt->{$pbpkg})) {
1240            $pbtag = $pkgt->{$pbpkg};
1241        } else {
1242            $pbtag = $ENV{'PBPROJTAG'};
1243        }
1244
1245        pb_log(0,"\n");
1246        pb_log(0,"Management of $pbpkg $pbver-$pbtag\n");
1247        die "Unable to get env var PBDESTDIR" if (not defined $ENV{'PBDESTDIR'});
1248
1249        my $dest = "$ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir";
1250        # Create the structure if needed
1251        pb_mkdir_p($dest);
1252        # Clean up dest if necessary. The export will recreate it
1253        pb_rm_rf($dest) if (-d $dest);
1254
1255        # Export CMS tree for the concerned package to dest
1256        # And generate some additional files
1257        $OUTPUT_AUTOFLUSH=1;
1258
1259        # computes in which dir we have to work
1260        my $dir = $defpkgdir->{$pbpkg};
1261        $dir = $extpkgdir->{$pbpkg} if (not defined $dir);
1262        $dir = $webdir->{$ENV{'PBPROJ'}} if (defined $web);
1263        die "Variable \$dir not defined. Check the package name first.\nIf this is the one expected, please report to dev team with log of a verbose run and this info ".Dumper($defpkgdir,$extpkgdir,$webdir) if (not defined $dir);
1264        pb_log(2,"def:".Dumper($defpkgdir)."ext: ".Dumper($extpkgdir)."dir: $dir\n");
1265
1266        # Exporting content from CMS
1267        my $sourcedir = undef;
1268        my $sourceuri = $uri;
1269        if ($param eq "SandBox") {
1270            # Point to the local instance
1271            $sourcedir = "$ENV{'PBDIR'}/$dir";
1272        } else {
1273            # Get it from a subdir of the URI with same version as localy but different root
1274            # Only if using a real CMS
1275            my ($scheme, $account, $host, $port, $path) = pb_get_uri($uri);
1276            if (($scheme !~ /^file/) && ($scheme !~ /^(ht|f)tp/)) {
1277                $sourceuri = "$ENV{'PBDIR'}/$dir"; 
1278                $sourceuri =~ s|^$ENV{'PBPROJDIR'}/|$uri/|;
1279            }
1280        }
1281        my $preserve = pb_vcs_export($sourceuri,$sourcedir,$dest);
1282
1283        # Generated fake content for test versions to speed up stuff
1284        my $chglog;
1285
1286        # Get project info on authors and log file
1287        # TODO: Make it CMS aware
1288        $chglog = "$ENV{'PBROOTDIR'}/$pbpkg/pbcl";
1289        $chglog = "$ENV{'PBROOTDIR'}/pbcl" if (! -f $chglog);
1290        $chglog = undef if (! -f $chglog);
1291
1292        # TODO: Make it CMS aware
1293        my $authors = "$ENV{'PBROOTDIR'}/$pbpkg/pbauthors";
1294        $authors = "$ENV{'PBROOTDIR'}/pbauthors" if (! -f $authors);
1295        $authors = "/dev/null" if (! -f $authors);
1296
1297        # Extract cms log history and store it
1298        if ((defined $chglog) && (! -f "$dest/NEWS")) {
1299            pb_log(2,"Generating NEWS file from $chglog\n");
1300            copy($chglog,"$dest/NEWS") || die "Unable to create $dest/NEWS";
1301        }
1302        pb_cms_log($scheme,"$ENV{'PBDIR'}/$dir",$dest,$chglog,$authors,$testver);
1303
1304        my %build;
1305        # We want to at least build for the underlying distro
1306        # except if a target was given, in which case we only build for it
1307        # if -t was passed without target then build for the native distro.
1308        my $pbos = pb_distro_get_context($pbtarget);
1309        my $tmpl = pb_get_distros($pbos,$pbtarget);
1310
1311        # Setup $pb structure to allow filtering later on, on files using that structure
1312        $pb->{'tag'} = $pbtag;
1313        $pb->{'rev'} = $pbrev;
1314        $pb->{'ver'} = $pbver;
1315        $pb->{'pkg'} = $pbpkg;
1316        $pb->{'suf'} = $pbos->{'suffix'};
1317        $pb->{'realpkg'} = $pbpkg;
1318        $pb->{'date'} = $pbdate;
1319        $pb->{'defpkgdir'} = $defpkgdir;
1320        $pb->{'extpkgdir'} = $extpkgdir;
1321        $pb->{'chglog'} = $chglog;
1322        $pb->{'extdir'} = $pbextdir;
1323        $pb->{'packager'} = $ENV{'PBPACKAGER'};
1324        $pb->{'proj'} = $ENV{'PBPROJ'};
1325        $pb->{'repo'} = "$ENV{'PBREPO'}/$delivery->{$ENV{'PBPROJ'}}";
1326        $pb->{'patches'} = ();
1327        $pb->{'sources'} = ();
1328   
1329        my $tmpd = "$ENV{'PBTMP'}/cms.$$";
1330        pb_mkdir_p($tmpd) if (defined $pbparallel);
1331
1332        # Get only all.yml filter at this stage for pbinit
1333        my $ptr = pb_get_filters($pbpkg);
1334
1335        # Do not do that for website
1336        if (not defined $web) {
1337            my %virt;
1338            # De-duplicate similar VM/VE/RM
1339            foreach my $d (split(/,/,$tmpl)) {
1340                # skip ill-formatted vms (name-ver-arch)
1341                next if ($d !~ /-/);
1342                $virt{$d} = $d;
1343            }
1344
1345            # Try to use // processing here
1346            my $all_ok = 1;
1347            my $pm = new Parallel::ForkManager($pbparallel) if (defined $pbparallel);
1348            $pm->run_on_finish(sub { my ($pid, $code, $id, $signal, $dump) = @_;
1349                                    $all_ok = 0 unless (($code == 0) && ($signal == 0) && ($dump == 0)); }) if (defined $pbparallel);
1350
1351            pb_log(0,"Preparing delivery ...\n");
1352            foreach my $v (keys %virt) {
1353                $pm->start and next if (defined $pbparallel);
1354
1355                # Distro context
1356                my $pbos = pb_distro_get_context($v);
1357   
1358                $pb->{'pbos'} = $pbos;
1359                $pb->{'suf'} = $pbos->{'suffix'};
1360                pb_log(3,"DEBUG: pb: ".Dumper($pb)."\n");
1361
1362                # Get all filters to apply
1363                $ptr = pb_get_filters($pbpkg,$pbos);
1364   
1365                pb_log(2,"DEBUG Filtering PBDATE => $pbdate, PBTAG => $pbtag, PBVER => $pbver\n");
1366   
1367                # We need to compute the real name of the package
1368                my $pbrealpkg = pb_cms_get_real_pkg($pbpkg,$pbos->{'type'});
1369                $pb->{'realpkg'} = $pbrealpkg;
1370                pb_log(2,"Virtual package $pbpkg has a real package name of $pbrealpkg on $pbos->{'name'}-$pbos->{'version'}\n") if ($pbrealpkg ne $pbpkg);
1371   
1372                # Filter build files from the less precise up to the most with overloading
1373                # Filter all files found, keeping the name, and generating in dest
1374   
1375                # Find all build files first relatively to PBROOTDIR
1376                # Find also all specific files referenced in the .yml conf file
1377                my %bfiles = ();
1378                my %pkgfiles = ();
1379                # Used in Filter.pm by pb_filter_file
1380
1381                $build{$v} = "no";
1382                if (-d "$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'os'}") {
1383                    pb_list_bfiles("$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'os'}",$pbpkg,\%bfiles,\%pkgfiles,$supfiles);
1384                    $build{$v} = "yes";
1385                } 
1386                if (-d "$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'type'}") {
1387                    pb_list_bfiles("$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'type'}",$pbpkg,\%bfiles,\%pkgfiles,$supfiles);
1388                    $build{$v} = "yes";
1389                } 
1390                if (-d "$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'family'}") {
1391                    pb_list_bfiles("$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'family'}",$pbpkg,\%bfiles,\%pkgfiles,$supfiles);
1392                    $build{$v} = "yes";
1393                } 
1394                if (-d "$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'name'}") {
1395                    pb_list_bfiles("$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'name'}",$pbpkg,\%bfiles,\%pkgfiles,$supfiles);
1396                    $build{$v} = "yes";
1397                } 
1398                if (-d "$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'name'}-$pbos->{'version'}") {
1399                    pb_list_bfiles("$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'name'}-$pbos->{'version'}",$pbpkg,\%bfiles,\%pkgfiles,$supfiles);
1400                    $build{$v} = "yes";
1401                } 
1402                if (-d "$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}") {
1403                    pb_list_bfiles("$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}",$pbpkg,\%bfiles,\%pkgfiles,$supfiles);
1404                    $build{$v} = "yes";
1405                }
1406                pb_log(2,"DEBUG($v) bfiles: ".Dumper(\%bfiles)."\n");
1407   
1408                if ($build{$v} ne "no") {
1409                    # Apply now all the filters on all the files concerned
1410                    # destination dir depends on the type of file
1411                    # For patch support
1412                    # TODO: Make it CMS aware
1413                    $pb->{'patches'} = pb_list_sfiles("$ENV{'PBROOTDIR'}/$pbpkg/pbpatch", $pb->{'patches'}, $pbos, "$ENV{'PBROOTDIR'}/$pbpkg/pbextpatch");
1414                    pb_log(2,"DEBUG($v) patches: ".Dumper($pb->{'patches'})."\n");
1415                    # TODO: Make it CMS aware
1416                    $pb->{'sources'} = pb_list_sfiles("$ENV{'PBROOTDIR'}/$pbpkg/pbsrc", $pb->{'sources'}, $pbos, "$ENV{'PBROOTDIR'}/$pbpkg/pbextsrc");
1417                    pb_log(2,"DEBUG($v) sources: ".Dumper($pb->{'sources'})."\n");
1418   
1419                    if (defined $pb->{'patches'}->{$v}) {
1420                        # Filter potential patches (local + remote)
1421                        pb_mkdir_p("$dest/pbconf/$v/pbpatch");
1422                        # If Debian based distribution, then prepare what will be done at build time
1423                        my ($patchcmd,$patchopt);
1424                        if ($pbos->{'type'} eq "deb") {
1425                            ($patchcmd,$patchopt) = pb_distro_get_param($pbos,pb_conf_get_if("ospatchcmd","ospatchopt"));
1426                            open(SCRIPT,"> $dest/pbconf/$v/pbpatch/pbapplypatch") || die "Unable to create $dest/pbconf/$v/pbpatch/pbapplypatch";
1427                            print SCRIPT "#!/bin/bash\n";
1428                            print SCRIPT "set -x\n" if ($pbdebug gt 1);
1429                        }
1430                        foreach my $pf (split(/,/,$pb->{'patches'}->{$v})) {
1431                            my $pp = basename($pf);
1432                            pb_vcs_export($pf,undef,"$dest/pbconf/$v/pbpatch");
1433                            pb_filter_file_inplace($ptr,"$dest/pbconf/$v/pbpatch/$pp",$pb);
1434                            pb_system("gzip -9f $dest/pbconf/$v/pbpatch/$pp","","quiet");
1435                            if ($pbos->{'type'} eq "deb") {
1436                                # If Debian based distribution, then prepare what will be done at build time
1437                                # by applying the patches that will be available under the debian/patches dir
1438                                print SCRIPT "$patchcmd $patchopt \< debian/patches/$pp\n";
1439                            }
1440                        }
1441                        if ($pbos->{'type'} eq "deb") {
1442                            close(SCRIPT);
1443                            chmod 0755,"$dest/pbconf/$v/pbpatch/pbapplypatch";
1444                        }
1445                        #pb_system("cat $dest/pbconf/$v/pbpatch/pbapplypatch","APPLY","verbose");
1446                    }
1447                    if (defined $pb->{'sources'}->{$v}) {
1448                        pb_mkdir_p("$dest/pbconf/$v/pbsrc");
1449                        foreach my $pf (split(/,/,$pb->{'sources'}->{$v})) {
1450                            my $pp = basename($pf);
1451                            pb_vcs_export($pf,undef,"$dest/pbconf/$v/pbsrc");
1452                            pb_filter_file_inplace($ptr,"$dest/pbconf/$v/pbsrc/$pp",$pb);
1453                        }
1454                    }
1455                    # Filter build files at the end, as they depend on patches and sources
1456                    foreach my $f (keys %bfiles) {
1457                        pb_filter_file("$ENV{'PBROOTDIR'}/$bfiles{$f}",$ptr,"$dest/pbconf/$v/$f",$pb);
1458                    }
1459                    foreach my $f (keys %pkgfiles) {
1460                        pb_filter_file("$ENV{'PBROOTDIR'}/$pkgfiles{$f}",$ptr,"$dest/pbconf/$v/$f",$pb);
1461                    }
1462                }
1463
1464                if (defined $pbparallel) {
1465                    # Communicate results back to parent
1466                    my $str = "";
1467                    $str .= "build $v = $build{$v}\n" if (defined $build{$v});
1468                    $str .= "patches $v = $pb->{'patches'}->{$v}\n" if (defined $pb->{'patches'}->{$v});
1469                    $str .= "sources $v = $pb->{'sources'}->{$v}\n" if (defined $pb->{'sources'}->{$v});
1470                    pb_set_content("$tmpd/$$","$str");
1471                    $pm->finish;
1472                }
1473            }
1474            # In the parent, we need to get the result from the children
1475            $pm->wait_all_children if (defined $pbparallel);
1476            die "Aborting, one or more of the children failed." if ((not $all_ok) && ($Global::pb_stop_on_error));
1477            my $made = "";
1478            my %h = ();
1479            my %tmp;
1480            my %tmp2;
1481            my $pt;
1482            my $k;
1483
1484            foreach $k (<$tmpd/*>) {
1485                $made .= pb_get_content($k);
1486            }
1487            pb_rm_rf($tmpd);
1488            pb_log(3,"MADE:\n$made");
1489
1490            # Rebuild local hashes
1491            foreach $k (split(/\n/,$made)) {
1492                if ($k =~ /^\s*([A-z0-9-_]+)\s+([[A-z0-9-_.]+)\s*=\s*(.+)$/) {
1493                    $h{$1}->{$2}=$3;
1494                }
1495            }
1496            pb_log(2,"HASH: ".Dumper(%h));
1497
1498            # Patches
1499            pb_log(0,"Delivered and compressed patches ");
1500            $pt = $h{'patches'};
1501            foreach $k (keys %$pt) {
1502                foreach my $v1 (split(/,/,$pt->{$k})) {
1503                    $tmp{$v1} = "";
1504                }
1505            }
1506            if (keys %tmp) {
1507                foreach $k (keys %tmp) {
1508                    pb_log(0,"$k ");
1509                }
1510            } else {
1511                pb_log(0,"N/A");
1512            }
1513            pb_log(0,"\n");
1514
1515            # Sources
1516            pb_log(0,"Delivered additional sources ");
1517            $pt = $h{'sources'};
1518            foreach $k (keys %$pt) {
1519                foreach my $v1 (split(/,/,$pt->{$k})) {
1520                    $tmp2{$v1} = "";
1521                }
1522            }
1523            if (keys %tmp2) {
1524                foreach $k (keys %tmp2) {
1525                    pb_log(0,"$k ");
1526                }
1527            } else {
1528                pb_log(0,"N/A");
1529            }
1530            pb_log(0,"\n");
1531
1532            # Build files
1533            my @found;
1534            my @notfound;
1535            $pt = $h{'build'};
1536            foreach my $b (keys %$pt) {
1537                push @found,$b if ($pt->{$b} =~ /yes/);
1538                push @notfound,$b if ($pt->{$b} =~ /no/);
1539            }
1540            pb_log(0,"Build files have been generated for ... ".join(',',sort(@found))."\n") if (@found);
1541            pb_log(0,"No Build files found for ".join(',',sort(@notfound))."\n") if (@notfound);
1542
1543        } else {
1544            # Instead call News generation
1545            pb_web_news2html($dest);
1546            # And create an empty pbconf
1547            pb_mkdir_p("$dest/pbconf");
1548            # And prepare the pbscript to execute remotely
1549            open(SCRIPT,"> $ENV{'PBTMP'}/pbscript") || die "Unable to create $ENV{'PBTMP'}/pbscript";
1550            print SCRIPT "#!/bin/bash\n";
1551            print SCRIPT "#set -x\n";
1552            print SCRIPT "echo ... Extracting Website content\n";
1553            print SCRIPT "find . -type f | grep -Ev '^./$pbpkg-$pbver$pbextdir.tar.gz|^./pbscript' | xargs rm -f non-existent\n";
1554            print SCRIPT "find * -type d -depth | xargs rmdir 2> /dev/null \n";
1555            print SCRIPT "tar xfz $pbpkg-$pbver$pbextdir.tar.gz\n";
1556            print SCRIPT "mv $pbpkg-$pbver$pbextdir/* .\n";
1557            print SCRIPT "rm -f $pbpkg-$pbver$pbextdir.tar.gz\n";
1558            print SCRIPT "rmdir $pbpkg-$pbver$pbextdir\n";
1559            print SCRIPT "find . -type f -print0 | xargs -0 chmod 644\n";
1560            print SCRIPT "find . -type d -print0 | xargs -0 chmod 755\n";
1561            close(SCRIPT);
1562            chmod 0755,"$ENV{'PBTMP'}/pbscript";
1563        }
1564
1565        # Apply filters to the non-build files
1566        my $liste ="";
1567        if (defined $filteredfiles->{$pbpkg}) {
1568            foreach my $f (split(/,/,$filteredfiles->{$pbpkg})) {
1569                pb_filter_file_inplace($ptr,"$dest/$f",$pb);
1570                $liste = "$f $liste";
1571            }
1572        }
1573        pb_log(2,"Files ".$liste."have been filtered\n");
1574
1575        # TODO: Make it CMS aware
1576        # Execute the pbinit script if any
1577        if (-x "$ENV{'PBROOTDIR'}/$pbpkg/pbinit") {
1578            pb_filter_file("$ENV{'PBROOTDIR'}/$pbpkg/pbinit",$ptr,"$ENV{'PBTMP'}/pbinit",$pb);
1579            chmod 0755,"$ENV{'PBTMP'}/pbinit";
1580            pb_system("cd $dest ; $ENV{'PBTMP'}/pbinit","Executing init script from $ENV{'PBROOTDIR'}/$pbpkg/pbinit under $dest","verbose");
1581        }
1582
1583        # Do we have additional script to run to prepare the environement for the project ?
1584        # Then include it in the pbconf delivery
1585        foreach my $pbvf (<$ENV{'PBROOTDIR'}/$pbpkg/pbv*.pre>,<$ENV{'PBROOTDIR'}/$pbpkg/pbv*.post>, <$ENV{'PBROOTDIR'}/$pbpkg/pbtest*>) {
1586            pb_log(1,"Handling file $pbvf\n");
1587            if (-x "$pbvf") {
1588                my $target = "$ENV{'PBTMP'}/".basename($pbvf);
1589                pb_filter_file("$pbvf",$ptr,$target,$pb);
1590                chmod 0755,"$target";
1591                pb_log(1,"Capturing file $pbvf in $target\n");
1592            }
1593        }
1594
1595        # Prepare the dest directory for archive
1596        chdir "$ENV{'PBDESTDIR'}" || die "Unable to change dir to $ENV{'PBDESTDIR'}";
1597        if (defined $preserve) {
1598            # In that case we want to preserve the original tar file for checksum purposes
1599            # The one created is btw equivalent in that case to this one
1600            # Maybe check basename of both to be sure they are the same ?
1601            pb_log(0,"Preserving original tar file ");
1602            move("$preserve","$pbpkg-$pbver$pbextdir.tar.gz");
1603        } else {
1604            # Possibility to look at PBSRC to guess more the filename
1605            pb_system("tar cfz $pbpkg-$pbver$pbextdir.tar.gz --exclude=$pbpkg-$pbver$pbextdir/pbconf $pbpkg-$pbver$pbextdir","Creating $pbpkg tar files compressed");
1606        }
1607        pb_log(0,"Under $ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.tar.gz\n");
1608        pb_system("tar cfz $pbpkg-$pbver$pbextdir.pbconf.tar.gz $pbpkg-$pbver$pbextdir/pbconf","Creating pbconf tar files compressed");
1609        pb_log(0,"Under $ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.pbconf.tar.gz\n");
1610
1611        # Keep track of version-tag per pkg
1612        $pkgs{$pbpkg} = "$pbver-$pbtag";
1613
1614        # Final cleanup
1615        pb_rm_rf($dest) if (-d $dest);
1616    }
1617
1618    # Keep track of per package version
1619    pb_log(2,"DEBUG pkgs: ".Dumper(%pkgs)."\n");
1620    open(PKG,"> $ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml") || die "Unable to create $ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml";
1621    print PKG "---\n";
1622    print PKG "pbpkg:\n";
1623    foreach my $pbpkg (keys %pkgs) {
1624        print PKG "  $pbpkg: $pkgs{$pbpkg}\n";
1625    }
1626    close(PKG);
1627
1628    # Keep track of what is generated by default
1629    # We need to store the dir and info on version-tag
1630    # Base our content on the existing .yml file
1631    copy("$ENV{'PBROOTDIR'}/$ENV{'PBPROJ'}.yml","$ENV{'PBDESTDIR'}/pbrc.yml");
1632    my $lh;
1633    $lh = pb_conf_cache("$ENV{'PBDESTDIR'}/pbrc.yml",$lh);
1634    $lh->{'pbroot'}->{$ENV{'PBPROJ'}} = $ENV{'PBROOTDIR'};
1635    $lh->{'projver'}->{$ENV{'PBPROJ'}} = $ENV{'PBPROJVER'};
1636    $lh->{'projtag'}->{$ENV{'PBPROJ'}} = $ENV{'PBPROJTAG'};
1637    $lh->{'pbpackager'}->{$ENV{'PBPROJ'}} = $ENV{'PBPACKAGER'};
1638    $lh->{'pbextdir'}->{$ENV{'PBPROJ'}} = $pbextdir;
1639    pb_conf_write("$ENV{'PBDESTDIR'}/pbrc.yml",$lh);
1640}
1641
1642sub pb_test2pkg {
1643    # Get the running distro to test on
1644    my $pbos = pb_distro_get_context();
1645
1646    # Get list of packages to test
1647    # Get content saved in cms2build
1648    my $ptr = pb_get_pkg();
1649    @pkgs = @$ptr;
1650
1651    # Additional potential repo
1652    my $forcerepo = pb_distro_setuprepo($pbos);
1653    foreach my $pbpkg (@pkgs) {
1654        # We need to install the package to test, and deps brought with it
1655        pb_distro_installdeps(undef,$pbos,$forcerepo,$pbpkg);
1656        pb_system("$ENV{'PBDESTDIR'}/pbtest","Launching test for $pbpkg","verbose");
1657    }
1658}
1659
1660sub pb_build2prep {
1661
1662    pb_log(0,"INFO: ------ Starting to prepare build environement ------\n");
1663
1664    # Get the running distro to build on
1665    my $pbos = pb_distro_get_context();
1666
1667    # If needed we may add repository to the build env
1668    my $forcerepo = pb_distro_setuprepo($pbos);
1669    pb_log(1,"Forcing full package install with forcerepo\n") if (defined $forcerepo);
1670
1671    # Get list of packages to build
1672    my $ptr = pb_get_pkg();
1673    @pkgs = @$ptr;
1674   
1675    # Get content saved in cms2build
1676    my ($pkg) = pb_conf_read("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml","pbpkg");
1677    $pkg = { } if (not defined $pkg);
1678    my $pbextdir = pb_get_extdir();
1679
1680    foreach my $pbpkg (@pkgs) {
1681        my $vertag = $pkg->{$pbpkg};
1682        my @buildfiles;
1683        pb_log(2,"Vertag: $vertag\n");
1684        # get the version of the current package - maybe different
1685        ($pbver,$pbtag) = split(/-/,$vertag);
1686
1687        my $src2="$ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.pbconf.tar.gz";
1688        pb_log(2,"Pbconf file: $src2\n");
1689        pb_log(2,"Working directory: $ENV{'PBBUILDDIR'}\n");
1690
1691        my $ftype;
1692        if ($pbos->{'type'} eq "rpm") {
1693            $ftype = "\.spec\$";
1694        } elsif ($pbos->{'type'} eq "deb") {
1695            $ftype = "\/control\$";
1696        } elsif ($pbos->{'type'} eq "ebuild") {
1697            $ftype = "\.ebuild\$";
1698        } elsif ($pbos->{'type'} eq "tgz") {
1699            $ftype = "\/pbslack\$";
1700        } elsif ($pbos->{'type'} eq "pkg") {
1701            # Solaris - no deps management up to Solaris 10
1702        } elsif ($pbos->{'type'} eq "hpux") {
1703            # TODO: HP-UX
1704        } elsif ($pbos->{'type'} eq "apk") {
1705            $ftype = "\/APKBUILD\$";
1706        } else {
1707            die "Unknown OS type format $pbos->{'type'}";
1708        }
1709
1710        @buildfiles = pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/","$ENV{'PBTMP'}","build");
1711        pb_log(2,"buildfiles ".Dumper(\@buildfiles)."\n");
1712        foreach my $f (@buildfiles) {
1713            if ($f =~ /$ftype/) {
1714                pb_distro_installdeps($f,$pbos,$forcerepo);
1715            }
1716        }
1717    }
1718    pb_log(0,"INFO: ------ Finished preparing build environment ------\n");
1719}
1720
1721sub pb_build2pkg {
1722
1723    my $do_install = shift;
1724
1725        # Call build2prep in case we have new deps we want to take in account
1726        # without redoing the prep phase (can be long and that helps debugging
1727        # build dependencies issues
1728    pb_build2prep();
1729    pb_log(0,"INFO: ------ Starting to build package ------\n");
1730    # Get the running distro to build on
1731    my $pbos = pb_distro_get_context();
1732
1733    # Get list of packages to build
1734    my $ptr = pb_get_pkg();
1735    @pkgs = @$ptr;
1736
1737    # Get content saved in cms2build
1738    my ($pkg) = pb_conf_read("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml","pbpkg");
1739    $pkg = { } if (not defined $pkg);
1740    my $pbextdir = pb_get_extdir();
1741
1742    pb_mkdir_p("$ENV{'PBBUILDDIR'}") if (! -d "$ENV{'PBBUILDDIR'}");
1743    chdir "$ENV{'PBBUILDDIR'}" || die "Unable to chdir to $ENV{'PBBUILDDIR'}";
1744    my $made = ""; # pkgs made during build
1745    my $pm;
1746    my $all_ok = 1;
1747
1748    if (defined $pbparallel) {
1749        $pm = new Parallel::ForkManager($pbparallel);
1750        $pm->run_on_finish(sub { my ($pid, $code, $id, $signal, $dump) = @_;
1751                                $all_ok = 0 unless (($code == 0) && ($signal == 0) && ($dump == 0)); });
1752    }
1753
1754    # We need to communicate info back from the children if parallel so prepare a dir for that
1755    my $tmpd = "$ENV{'PBTMP'}/build.$$";
1756    pb_mkdir_p($tmpd) if (defined $pbparallel);
1757
1758    # Do it here as this doesn't work in // mode as it dies if it fails
1759    if ($pbos->{'type'} eq "rpm") {
1760        foreach my $d ('RPMS','SRPMS','SPECS','SOURCES','BUILD') {
1761                if (! -d "$ENV{'PBBUILDDIR'}/$d") {
1762                    pb_mkdir_p("$ENV{'PBBUILDDIR'}/$d");
1763                }
1764            }
1765    }
1766
1767    foreach my $pbpkg (@pkgs) {
1768        $pm->start and next if (defined $pbparallel);
1769
1770        my $vertag = $pkg->{$pbpkg};
1771        pb_log(2,"Vertag: $vertag\n");
1772        # get the version of the current package - maybe different
1773        ($pbver,$pbtag) = split(/-/,$vertag);
1774
1775        my $src="$ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.tar.gz";
1776        my $src2="$ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.pbconf.tar.gz";
1777        pb_log(2,"Source file: $src\n");
1778        pb_log(2,"Pbconf file: $src2\n");
1779
1780        pb_log(2,"Working directory: $ENV{'PBBUILDDIR'}\n");
1781        if ($pbos->{'type'} eq "rpm") {
1782            # Remove in case a previous link/file was there
1783            unlink "$ENV{'PBBUILDDIR'}/SOURCES/".basename($src);
1784            symlink "$src","$ENV{'PBBUILDDIR'}/SOURCES/".basename($src) || die "Unable to symlink $src in $ENV{'PBBUILDDIR'}/SOURCES";
1785            # We need to first extract the spec file
1786            my @buildfile = pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/","$ENV{'PBBUILDDIR'}/SPECS","build");
1787
1788            # We need to handle potential patches to upstream sources
1789            pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbpatch/","$ENV{'PBBUILDDIR'}/SOURCES","patch");
1790
1791            # We need to handle potential additional sources to upstream sources
1792            pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbsrc/","$ENV{'PBBUILDDIR'}/SOURCES","src");
1793
1794            pb_log(2,"buildfile: ".Dumper(\@buildfile)."\n");
1795            # set LANGUAGE to check for correct log messages
1796            $ENV{'LANGUAGE'}="C";
1797            # Older Redhat use _target_platform in %configure incorrectly
1798            my $specialdef = "";
1799            if (($pbos->{'name'} eq "rhel") && ($pbos->{'version'} eq "2.1")) {
1800                $specialdef = "--define \'_target_platform \"\"\'";
1801            }
1802            my $buildcmd = "rpmbuild";
1803            if (($pbos->{'name'} eq "redhat") && ($pbos->{'version'} =~ /^6/)){
1804                $buildcmd = "rpm" ;
1805            }
1806
1807            foreach my $f (@buildfile) {
1808                if ($f =~ /\.spec$/) {
1809                    pb_system("$buildcmd $specialdef --define \"packager $ENV{'PBPACKAGER'}\" --define \"_topdir $ENV{'PBBUILDDIR'}\" -ba $f","Building package with $f under $ENV{'PBBUILDDIR'}","verbose");
1810                    last;
1811                }
1812            }
1813            # Get the name of the generated packages
1814            open(LOG,"$ENV{'PBTMP'}/system.$$.log") || die "Unable to open $ENV{'PBTMP'}/system.$$.log";
1815            while (<LOG>) {
1816                chomp($_);
1817                next if ($_ !~ /^Wrote:/);
1818                s|.*/([S]*RPMS.*)|$1|;
1819                $made .=" $_";
1820            }
1821            close(LOG);
1822
1823        } elsif ($pbos->{'type'} eq "deb") {
1824            pb_system("tar xfz $src","Extracting sources");
1825            pb_system("tar xfz $src2","Extracting pbconf");
1826
1827            chdir "$pbpkg-$pbver$pbextdir" || die "Unable to chdir to $pbpkg-$pbver$pbextdir";
1828            pb_rm_rf("debian");
1829            my $confdir = "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
1830            die "Configuration directory $confdir is not a directory\nIs os description listed in your {ve,vm,rm}list values?" if (not -d $confdir);
1831            symlink "$confdir","debian" || die "Unable to symlink 'debian' to $confdir";
1832            chmod 0755,"debian/rules";
1833
1834            # We need to handle potential patches to upstream sources
1835            pb_mkdir_p("debian/patches");
1836            my @f = pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbpatch/","debian/patches","patch");
1837
1838            # By default we use format 1.0 - Cf man dpkg-source
1839            my $debsrcfmt = "1.0";
1840            my $debsrcfile = "debian/source/format";
1841            if (-f $debsrcfile) {
1842                $debsrcfmt = pb_get_content($debsrcfile);
1843            }
1844
1845            if ($debsrcfmt =~ /^3.*quilt/) {
1846                # If we use quilt to manage patches, we then setup the env correctly
1847                # as per http://pkg-perl.alioth.debian.org/howto/quilt.html
1848                # Generate Debian patch series for quilt
1849                open(SERIE,"> debian/patches/series") || die "Unable to write in debian/patches/series";
1850                $ENV{'QUILT_PATCHES'}="debian/patches";
1851            } else {
1852                # If we use dpatch to manage patches, we then setup the 00list file as well
1853                open(SERIE,"> debian/patches/00list") || die "Unable to write in debian/patches/00list";
1854            }
1855            foreach my $f (sort @f) {
1856                # Skip the script made to apply the patches to the Debian tree
1857                next if ($f =~ /pbapplypatch/);
1858                # We also need to uncompress them
1859                pb_system("gzip -d $f","","quiet");
1860                $f =~ s/\.gz$//;
1861                print SERIE "$f\n";
1862            }
1863            close(SERIE);
1864            if (@f) {
1865                # We have patches...
1866                my $patch_file = "debian/patches/pbapplypatch";
1867                if (($debsrcfmt =~ /^1.*/) && (-x $patch_file)) {
1868                    # In that case we need to apply the patches ourselves locally
1869                    pb_system("cat $patch_file","APPLY","verbose");
1870                    pb_system("$patch_file","Applying patches to $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'} tree");
1871                }
1872                # ...so modify the name of files to be Debian compliant
1873                move("../$src","../$pbpkg-$pbver$pbextdir.orig.tar.gz");
1874            }
1875
1876            # We need to handle potential additional sources to upstream sources
1877            #pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbsrc/","$ENV{'PBBUILDDIR'}/debian","src");
1878
1879            pb_system("dpkg-buildpackage -us -uc -rfakeroot","Building package","verbose");
1880
1881            # Get the name of the generated packages
1882            open(LOG,"$ENV{'PBTMP'}/system.$$.log") || die "Unable to open $ENV{'PBTMP'}/system.$$.log";
1883            while (<LOG>) {
1884                chomp();
1885                pb_log(3,"In loop: $_\n");
1886                next unless (/^dpkg-deb.*\.\.\/(.*_.*\.deb).*/);
1887                my $tmp = $1;
1888                #doesn't work in my case
1889                #die "Missing file $tmp" if (not -f "../$tmp");
1890                $made = "$made $tmp";
1891            }
1892            close(LOG);
1893            pb_log(2,"Now made is: $made\n");
1894
1895            open(CTRL,"debian/control") || die "Unable to open debian/control: $!";
1896            while (<CTRL>) {
1897                pb_log(3,"In loop: $_\n");
1898                next unless (/^Source: (\S+)/o);
1899                my $tmp = $1;
1900                $made = "$made $tmp"."_*.dsc $tmp"."_*.tar.gz $tmp"."_*.changes";
1901                #doesn't work in my case
1902                #foreach my $glob (("$1\_*.changes", "$1\_*.dsc", "$1\_*.tar.gz")) {
1903                    #my @file = glob($glob);
1904                    #die "Missing file for $glob" unless (@file > 0);
1905                    #die "Too many files for $glob" if (@file > 1);
1906                    #die "Missing file $file[0]" if (not -f $file[0]);
1907                    #$made .= " $file[0]";
1908                #}
1909            }
1910            close(CTRL);
1911            #pb_display_file("$ENV{'PBTMP'}/system.$$.log");
1912            pb_log(2,"Finally made is: $made\n");
1913
1914            chdir ".." || die "Unable to chdir to parent dir";
1915            pb_rm_rf("$pbpkg-$pbver$pbextdir");
1916        } elsif ($pbos->{'type'} eq "ebuild") {
1917            my @ebuildfile;
1918            # For gentoo we need to take pb as subsystem name
1919            # We put every apps here under sys-apps. hope it's correct
1920            # We use pb's home dir in order to have a single OVERLAY line
1921            my $tmpe = "$ENV{'HOME'}/portage/pb/sys-apps/$pbpkg";
1922            pb_mkdir_p($tmpe) if (! -d "$tmpe");
1923            pb_mkdir_p("$ENV{'HOME'}/portage/distfiles") if (! -d "$ENV{'HOME'}/portage/distfiles");
1924
1925            # We need to first extract the ebuild file
1926            @ebuildfile = pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/","$tmpe","build");
1927
1928            # Prepare the build env for gentoo
1929            my $found = 0;
1930            my $pbbd = $ENV{'HOME'};
1931            $pbbd =~ s|/|\\/|g;
1932            if (-r "/etc/make.conf") {
1933                open(MAKE,"/etc/make.conf");
1934                while (<MAKE>) {
1935                    $found = 1 if (/$pbbd\/portage/);
1936                }
1937                close(MAKE);
1938            }
1939            if ($found == 0) {
1940                pb_system("sudo sh -c 'echo PORTDIR_OVERLAY=\"$ENV{'HOME'}/portage\" >> /etc/make.conf'");
1941            }
1942            #$found = 0;
1943            #if (-r "/etc/portage/package.keywords") {
1944            #open(KEYW,"/etc/portage/package.keywords");
1945            #while (<KEYW>) {
1946            #$found = 1 if (/portage\/pb/);
1947            #}
1948            #close(KEYW);
1949            #}
1950            #if ($found == 0) {
1951            #pb_system("sudo sh -c \"echo portage/pb >> /etc/portage/package.keywords\"");
1952            #}
1953
1954            # Build
1955            foreach my $f (@ebuildfile) {
1956                if ($f =~ /\.ebuild$/) {
1957                    move($f,"$tmpe/$pbpkg-$pbver.ebuild");
1958                    pb_system("cd $tmpe ; ebuild $pbpkg-$pbver.ebuild clean ; ebuild $pbpkg-$pbver.ebuild digest ; ebuild $pbpkg-$pbver.ebuild package","verbose");
1959                    # Now move it where pb expects it
1960                    pb_mkdir_p("$ENV{'PBBUILDDIR'}/portage/pb/sys-apps/$pbpkg");
1961                    if ($pbtag eq 0) {
1962                        # This is assumed to be a test version
1963                        my $nver = substr($pbver,0,-14);
1964                        my $ntag = substr($pbver,-14);
1965                        my $ebtg = "portage/pb/sys-apps/$pbpkg/$pbpkg-$nver"."_p$ntag.ebuild";
1966                        move("$tmpe/$pbpkg-$pbver.ebuild","$ENV{'PBBUILDDIR'}/$ebtg");
1967                        $made="$made $ebtg";
1968                    } else {
1969                        my $ebtg = "portage/pb/sys-apps/$pbpkg/$pbpkg-$pbver-r$pbtag.ebuild";
1970                        move("$tmpe/$pbpkg-$pbver.ebuild","$ENV{'PBBUILDDIR'}/$ebtg");
1971                        $made="$made $ebtg";
1972                    }
1973                }
1974            }
1975
1976        } elsif ($pbos->{'type'} eq "apk") {
1977            pb_mkdir_p("$ENV{'PBBUILDDIR'}/tmp");
1978
1979            # We need to first extract the APKBUILD file
1980            my @apkfile = pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/","$ENV{'PBBUILDDIR'}/tmp","build");
1981
1982            chdir "$ENV{'PBBUILDDIR'}/tmp" || die "Unable to chdir to $ENV{'PBBUILDDIR'}/tmp";
1983
1984            # Build
1985            my $keyfile = pb_ssh_get(1);
1986            pb_mkdir_p("$ENV{'HOME'}/.abuild");
1987            open(CONF,"> $ENV{'HOME'}/.abuild/abuild.conf") || confess "Unable to create the abuild conf file";
1988            print CONF "PACKAGER_PRIVKEY=\"$keyfile\"\n" if (defined $keyfile);
1989            close(CONF);
1990            foreach my $f (@apkfile) {
1991                if ($f =~ /APKBUILD$/) {
1992                    pb_system("ls -al $ENV{'HOME'}/.ssh ; pwd ; ls -alR ../.. ; sh -x /usr/bin/abuild checksum ; abuild unpack ; abuild -r -P .. build ; abuild -P .. rootpkg ; abuild -P pkg index","verbose");
1993                }
1994            }
1995            # Get the name of the generated packages
1996            open(LOG,"$ENV{'PBTMP'}/system.$$.log") || die "Unable to open $ENV{'PBTMP'}/system.$$.log";
1997            while (<LOG>) {
1998                chomp($_);
1999                next if ($_ !~ /: Create /);
2000                s|.*: Create (.*.apk)|$1|;
2001                $made .= " $_";
2002            }
2003            close(LOG);
2004
2005        } elsif ($pbos->{'type'} eq "tgz") {
2006            # Slackware family
2007            $made="$made $pbpkg/$pbpkg-$pbver-*-$pbtag.tgz";
2008
2009            pb_system("tar xfz $src","Extracting sources");
2010            pb_system("tar xfz $src2","Extracting pbconf");
2011            chdir "$pbpkg-$pbver$pbextdir" || die "Unable to chdir to $pbpkg-$pbver$pbextdir";
2012            symlink "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}","install" || die "Unable to symlink to pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
2013            if (-x "install/pbslack") {
2014                pb_system("./install/pbslack","Building software");
2015                pb_system("sudo /sbin/makepkg -p -l y -c y $pbpkg","Packaging $pbpkg","verbose");
2016            }
2017            chdir ".." || die "Unable to chdir to parent dir";
2018            pb_rm_rf("$pbpkg-$pbver$pbextdir");
2019        } elsif ($pbos->{'type'} eq "pkg") {
2020            # Solaris
2021            $made="$made $pbpkg-$pbver-$pbtag.pkg.gz";
2022            my $pkgdestdir="$ENV{'PBBUILDDIR'}/install";
2023
2024            # Will host resulting packages
2025            pb_mkdir_p("$pbos->{'type'}");
2026            pb_mkdir_p("$pkgdestdir/pbdelivery");
2027            pb_system("tar xfz $src","Extracting sources under $ENV{'PBBUILDDIR'}");
2028            pb_system("tar xfz $src2","Extracting pbconf under $ENV{'PBBUILDDIR'}");
2029            # We need to handle potential patches to upstream sources
2030            pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbpatch/","$ENV{'PBBUILDDIR'}","patch");
2031
2032            # We need to handle potential additional sources to upstream sources
2033            pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbsrc/","$ENV{'PBBUILDDIR'}","src");
2034
2035            chdir "$pbpkg-$pbver$pbextdir" || die "Unable to chdir to $pbpkg-$pbver$pbextdir";
2036            if (-f "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbbuild") {
2037                chmod 0755,"pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbbuild";
2038                # pkginfo file is mandatory
2039                die "Unable to find pkginfo file in pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}" if (! -f "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pkginfo");
2040                # Build
2041                pb_system("pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbbuild $pkgdestdir/pbdelivery","Building software and installing under $pkgdestdir/pbdelivery");
2042                # Copy complementary files
2043                if (-f "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/prototype") {
2044                    copy("pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/prototype", $pkgdestdir) 
2045                } else {
2046                    # No prototype provided, calculating it
2047                    open(PROTO,"> $pkgdestdir/prototype") || die "Unable to create prototype file";
2048                    print PROTO "i pkginfo\n";
2049                    print PROTO "i depend\n" if (-f "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/depend");
2050                    $ENV{'PBSOLDESTDIR'} = "$pkgdestdir/pbdelivery";
2051                    find(\&create_solaris_prototype, "$pkgdestdir/pbdelivery");
2052                }
2053                copy("pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/depend", $pkgdestdir) if (-f "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/depend");
2054                copy("pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pkginfo", $pkgdestdir);
2055                pb_system("cd $pkgdestdir/pbdelivery ; pkgmk -o -f ../prototype -r $pkgdestdir/pbdelivery -d $ENV{'PBBUILDDIR'}/$pbos->{'type'}","Packaging $pbpkg","verbose");
2056                pb_system("cd $ENV{'PBBUILDDIR'}/$pbos->{'type'} ;  echo \"\" | pkgtrans -o -n -s $ENV{'PBBUILDDIR'}/$pbos->{'type'} $ENV{'PBBUILDDIR'}/$pbpkg-$pbver-$pbtag.pkg all","Transforming $pbpkg","verbose");
2057                pb_system("cd $ENV{'PBBUILDDIR'} ;  gzip -9f $pbpkg-$pbver-$pbtag.pkg","Compressing $pbpkg-$pbver-$pbtag.pkg","verbose");
2058            } else {
2059                pb_log(0,"No pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbbuild file found for $pbpkg-$pbver\n");
2060            }
2061            chdir ".." || die "Unable to chdir to parent dir";
2062            pb_rm_rf("$pbpkg-$pbver$pbextdir","$ENV{'PBBUILDDIR'}/$pbos->{'type'}","$pkgdestdir");
2063        } elsif ($pbos->{'type'} eq "hpux") {
2064            # HP-UX
2065            pb_system("tar xfz $src","Extracting sources");
2066            pb_system("tar xfz $src2","Extracting pbconf");
2067
2068            chdir "$pbpkg-$pbver$pbextdir" || die "Unable to chdir to $pbpkg-$pbver$pbextdir";
2069            pb_system("buildpackage ","Building package","verbose");
2070            # Get the name of the generated packages
2071            open(LOG,"$ENV{'PBTMP'}/system.$$.log") || die "Unable to open $ENV{'PBTMP'}/system.$$.log";
2072            while (<LOG>) {
2073                chomp();
2074                my $tmp = $_;
2075                next if ($tmp !~ /^SD BUILD.*:/);
2076                $tmp =~ s|.*../(.*)_(.*).sd.*|$1|;
2077                $made = "$made $tmp"."_*.sd";
2078            }
2079            close(LOG);
2080            $made="$made $pbpkg-$pbver-$pbtag.sd";
2081
2082            chdir ".." || die "Unable to chdir to parent dir";
2083            pb_rm_rf("$pbpkg-$pbver$pbextdir");
2084        } else {
2085            die "Unknown OS type format $pbos->{'type'}";
2086        }
2087        if (defined $pbparallel) {
2088            # Communicate results back to parent
2089            pb_set_content("$tmpd/$$",$made);
2090            $pm->finish;
2091        }
2092    }
2093    if (defined $pbparallel) {
2094        # In the parent, we need to get the result from the children
2095        $pm->wait_all_children;
2096        foreach my $f (<$tmpd/*>) {
2097            $made .= " ".pb_get_content($f);
2098        }
2099        die "Aborting, one or more of the children failed." if ((not $all_ok) && ($Global::pb_stop_on_error));
2100        pb_rm_rf($tmpd);
2101    }
2102
2103    # Sign packages
2104    pb_sign_pkgs($pbos,$made);
2105    pb_log(0,"INFO: ------ Finished building package ------\n");
2106
2107    # Find the appropriate check cmd/opts
2108    my ($chkcmd,$chkopt) = pb_distro_get_param($pbos,pb_conf_get_if("oschkcmd","oschkopt"));
2109
2110    my $ret = "";
2111    # Packages check if needed
2112    if ($pbos->{'type'} eq "rpm") {
2113        if ((defined  $chkcmd) && (-x $chkcmd)) {
2114            my $cmd = "$chkcmd";
2115            $cmd .= " $chkopt" if (defined $chkopt);
2116            $cmd .= " $made";
2117            my $ret = pb_system("$cmd","Checking validity of rpms with $chkcmd","mayfailverbose");
2118            pb_log(0,"ERROR: when checking packages validity\n") if ($ret ne 0);
2119        }
2120        my $rpms ="";
2121        my $srpms ="";
2122        foreach my $f (split(/ /,$made)) {
2123            $rpms .= "$ENV{'PBBUILDDIR'}/$f " if ($f =~ /^RPMS\//);
2124            $srpms .= "$ENV{'PBBUILDDIR'}/$f " if ($f =~ /^SRPMS\//);
2125        }
2126        pb_log(0,"SRPM packages generated: $srpms\n");
2127        pb_log(0,"RPM packages generated: $rpms\n");
2128        $ret = $rpms;
2129    } elsif ($pbos->{'type'} eq "deb") {
2130        my $made2 = "";
2131        foreach my $f (split(/ /,$made)) {
2132            $made2 .= "$f " if ($f =~ /\.deb$/);
2133        }
2134        if ((defined  $chkcmd) && (-x $chkcmd)) {
2135            my $ret = pb_system("$chkcmd $chkopt $made2","Checking validity of debs with $chkcmd","mayfail");
2136            pb_log(0,"ERROR: when checking packages validity\n") if ($ret ne 0);
2137        }
2138        pb_log(0,"deb packages generated: $made2\n");
2139        $ret = $made2;
2140    } else {
2141        pb_log(0,"No check done for $pbos->{'type'} yet\n");
2142        pb_log(0,"Packages generated: $made\n");
2143        $ret = $made;
2144    }
2145
2146    # Keep track of what is generated so that we can get them back from VMs/RMs
2147    my $pbstore = "$ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
2148    open(KEEP,"> $pbstore") || die "Unable to create $pbstore $!";
2149    print KEEP "$made\n";
2150    close(KEEP);
2151    pb_distro_installdeps(undef,$pbos,undef,$ret,"local") if ($do_install);
2152}
2153
2154sub pb_build2ssh {
2155    pb_send2target("Sources");
2156    pb_send2target("CPAN");
2157}
2158
2159sub pb_pkg2ssh {
2160    pb_send2target("Packages");
2161}
2162
2163# By default deliver to the the public site hosting the
2164# ftp structure (or whatever) or a VM/VE/RM
2165sub pb_send2target {
2166
2167    my $cmt = shift;
2168    my $pbscript1 = shift;
2169    my $v = shift;
2170    my $vexist = shift;         # 0 is FALSE
2171    my $vpid = shift;           # 0 is FALSE
2172    my $snapme = shift;         # 0 is FALSE
2173    my $pbstep = shift;         # 3 is usage of container
2174
2175    $vexist = 0 if (not defined $vexist);
2176    $vpid = 0 if (not defined $vpid);
2177    $snapme = 0 if (not defined $snapme);
2178    $pbstep = 3 if (not defined $pbstep);
2179    pb_log(2,"DEBUG: pb_send2target($cmt,".Dumper($v).",$vexist,$vpid)\n");
2180    my $host = "sshhost";
2181    my $login = "sshlogin";
2182    my $dir = "sshdir";
2183    my $port = "sshport";
2184    my $conf = "sshconf";
2185    my $tmout = undef;
2186    my $path = undef;
2187
2188    if ($cmt =~ /^VM/) {
2189        $login = "vmlogin";
2190        $dir = "pbdefdir";
2191        # Specific VM
2192        $tmout = "vmtmout";
2193        $path = "vmpath";
2194        $host = "vmhost";
2195        $port = "vmport";
2196    } elsif ($cmt =~ /^RM/) {
2197        $login = "rmlogin";
2198        $dir = "pbdefdir";
2199        # Specific RM
2200        $tmout = "rmtmout";
2201        $path = "rmpath";
2202        $host = "rmhost";
2203        $port = "rmport";
2204    } elsif ($cmt =~ /^VE/) {
2205        $login = "velogin";
2206        $dir = "pbdefdir";
2207        # Specific VE
2208        $path = "vepath";
2209        $conf = "rbsconf";
2210    } elsif ($cmt eq "Web") {
2211        $host = "websshhost";
2212        $login = "websshlogin";
2213        $dir = "websshdir";
2214        $port = "websshport";
2215    } elsif ($cmt eq "CPAN") {
2216        $host = "cpanpause";
2217        $login = "";
2218        $dir = "cpandir";
2219        $port = "";
2220    }
2221    my $cmd = "";
2222    my $src = "";
2223    my $cpanpkg = 0;
2224    my $pbos;
2225
2226    my $pbextdir = pb_get_extdir();
2227
2228    if ($cmt ne "Announce") {
2229        # Get list of packages to build
2230        my $ptr = pb_get_pkg();
2231        my @pkgs = ();
2232        @pkgs = @$ptr if (defined $ptr);
2233
2234        # Get the running distro to consider
2235        $pbos = pb_distro_get_context($v);
2236
2237        my $pkg = { };
2238        # Get content saved in cms2build
2239        if ((defined $ENV{'PBDESTDIR'}) && (defined $ENV{'PBPROJVER'}) && (defined $ENV{'PBPROJTAG'}) && (-f "$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml")) {
2240            ($pkg) = pb_conf_read("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml","pbpkg");
2241            $pkg = { } if (not defined $pkg);
2242        }
2243
2244        pb_mkdir_p("$ENV{'PBBUILDDIR'}") if (! -d "$ENV{'PBBUILDDIR'}");
2245        chdir "$ENV{'PBBUILDDIR'}" || die "Unable to chdir to $ENV{'PBBUILDDIR'}";
2246        foreach my $pbpkg (@pkgs) {
2247            my $vertag = $pkg->{$pbpkg};
2248            # get the version of the current package - maybe different
2249            pb_log(2,"Vertag: $vertag\n");
2250            ($pbver,$pbtag) = split(/-/,$vertag);
2251
2252            if (($cmt eq "Sources") || ($cmt =~ /(V[EM]|RM)(build|prep)/)) {
2253                $src = "$src $ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.tar.gz $ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.pbconf.tar.gz";
2254                my $cmd2 = "ln -sf $pbpkg-$pbver$pbextdir.tar.gz $pbpkg-latest.tar.gz";
2255                #my $cmd2 = "ln $pbpkg-$pbver$pbextdir.tar.gz $pbpkg-latest.tar.gz 2> /dev/null ; if [ \$? -ne 0 ]; then ln -sf $pbpkg-$pbver$pbextdir.tar.gz $pbpkg-latest.tar.gz ; fi";
2256                if ($cmd eq "") {
2257                    $cmd = $cmd2;
2258                } else {
2259                    $cmd = "$cmd ; $cmd2";
2260                }
2261            } elsif ($cmt eq "Web") {
2262                $src = "$src $ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.tar.gz"
2263            }
2264
2265            # Do we have one perl package
2266            my @nametype = pb_conf_get_if("namingtype");
2267            my $type = $nametype[0]->{$pbpkg};
2268            if ((defined $type) && ($type eq "perl")) {
2269                $cpanpkg = 1;
2270            }
2271        }
2272        # Adds conf file for availability of conf elements
2273        pb_conf_add("$ENV{'PBROOTDIR'}/$ENV{'PBPROJ'}.yml") if ((defined $ENV{'PBROOTDIR'}) && (-f "$ENV{'PBROOTDIR'}/$ENV{'PBPROJ'}.yml"));
2274    }
2275    my ($rbsconf,$testver,$delivery) = pb_conf_get_if($conf,"testver","delivery");
2276    if ($cmt eq "CPAN") {
2277        # Do not deliver on Pause if this is a test version
2278        return if (not defined $testver);
2279        return if ($testver =~ /true/);
2280        # Do not deliver on Pause if this is not a perl package
2281        return if ($cpanpkg == 0);
2282    }
2283
2284    if ($cmt =~ /(V[EM]|RM)(build|prep)/) {
2285        $src="$src $ENV{'PBROOTDIR'}/$ENV{'PBPROJ'}.yml $ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml $ENV{'PBETC'} $ENV{'PBDESTDIR'}/pbrc.yml $pbscript{$v}";
2286    } elsif ($cmt =~ /(V[EM]|RM)Script/) {
2287        $src="$src $pbscript{$v}";
2288    } elsif ($cmt =~ /(V[EM]|RM)test/) {
2289        $src="$src $ENV{'PBROOTDIR'}/$ENV{'PBPROJ'}.yml $ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml $ENV{'PBETC'} $ENV{'PBDESTDIR'}/pbrc.yml $pbscript{$v} $ENV{'PBDESTDIR'}/pbtest";
2290    } elsif (($cmt eq "Announce") || ($cmt eq "Web") || ($cmt eq "CPAN")) {
2291        $src="$src $ENV{'PBTMP'}/pbscript";
2292    } elsif ($cmt eq "Packages") {
2293        # Get package list from file made during build2pkg
2294        open(KEEP,"$ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}") || die "Unable to read $ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
2295        $src = <KEEP>;
2296        chomp($src);
2297        close(KEEP);
2298    }
2299    if (($cmt eq "Sources") || ($cmt eq "Packages") || ($cmt eq "CPAN")) {
2300        my ($pbpackager) = pb_conf_get("pbpackager");
2301        $ENV{'PBPACKAGER'} = $pbpackager->{$ENV{'PBPROJ'}};
2302        pb_log(0,"Exporting public key for $ENV{'PBPACKAGER'}\n");
2303        # Using pb_system is not working due to redirection below
2304        system("gpg --export -a \'$ENV{'PBPACKAGER'}\' > $ENV{'PBDESTDIR'}/$ENV{'PBPROJ'}.pubkey");
2305        chmod 0644,"$ENV{'PBDESTDIR'}/$ENV{'PBPROJ'}.pubkey";
2306        $src = "$src $ENV{'PBDESTDIR'}/$ENV{'PBPROJ'}.pubkey";
2307
2308        my ($gpgaddkey) = pb_conf_get_if("pbadditionalgpg");
2309        my $pbadditionalgpg;
2310        $pbadditionalgpg = $gpgaddkey->{$ENV{'PBPROJ'}} if (defined $gpgaddkey);
2311        if (defined $pbadditionalgpg) {
2312            foreach my $k (split(/,/,$pbadditionalgpg)) {
2313                pb_log(0,"Exporting public key $k\n");
2314                system("gpg --export -a \'$k\' > $ENV{'PBDESTDIR'}/$ENV{'PBPROJ'}-$k.pubkey");
2315                $src = "$src $ENV{'PBDESTDIR'}/$ENV{'PBPROJ'}-$k.pubkey";
2316            }
2317        }
2318    }
2319    # Remove potential leading spaces (cause problem with basename)
2320    $src =~ s/^ *//;
2321    my $basesrc = "";
2322    foreach my $i (split(/ +/,$src)) {
2323        $basesrc .= " ".basename($i);
2324    }
2325
2326    pb_log(2,"values: ".Dumper(($host,$login,$dir,$port,$tmout,$path,$conf))."\n");
2327    my ($sshhost,$sshdir) = pb_conf_get($host,$dir);
2328    # Not mandatory...
2329    $delivery->{$ENV{'PBPROJ'}} = "" if (not defined $delivery->{$ENV{'PBPROJ'}});
2330    my ($sshlogin,$sshport) = pb_conf_get_if($login,$port);
2331    $sshport->{$ENV{'PBPROJ'}} = 22 if (not defined $sshport->{$ENV{'PBPROJ'}});
2332    $sshlogin->{$ENV{'PBPROJ'}} = getpwuid($UID) if (not defined $sshlogin->{$ENV{'PBPROJ'}});
2333    my ($vtmout,$vepath);
2334    # ...Except those in virtual context
2335    if ($cmt =~ /^VE/) {
2336        ($vepath) = pb_conf_get($path);
2337    }
2338    if ($cmt =~ /^(V|R)M/) {
2339        $vtmout = pb_distro_get_param($pbos,pb_conf_get_if($tmout));
2340    }
2341    my $remhost = $sshhost->{$ENV{'PBPROJ'}};
2342    my $remdir = $sshdir->{$ENV{'PBPROJ'}};
2343    if ($cmt =~ /^V[EM]|RM/) {
2344        # In that case our real host is in the xxhost with the OS as key, not project as above
2345        $remhost = pb_distro_get_param($pbos,$sshhost);
2346    }
2347    pb_log(2,"ssh: ".Dumper(($remhost,$sshlogin,$remdir,$sshport,$vepath,$rbsconf))."\n");
2348    pb_log(2,"ssh: ".Dumper($vtmout)."\n") if (defined $vtmout);
2349
2350    my $mac;
2351    if ($cmt !~ /^VE/) {
2352        $mac = "$sshlogin->{$ENV{'PBPROJ'}}\@$remhost";
2353        # Overwrite account value if passed as parameter or forced
2354        $mac = "$ENV{'PBACCOUNT'}\@$remhost" if (defined $ENV{'PBACCOUNT'});
2355    } else {
2356        # VE
2357        $mac = $sshlogin->{$ENV{'PBPROJ'}};
2358        # Overwrite account value if passed as parameter (typically for setup2v)
2359        $mac = $ENV{'PBACCOUNT'} if (defined $ENV{'PBACCOUNT'});
2360    }
2361    pb_log(2, "DEBUG: mac: $mac\n");
2362
2363    my $tdir;
2364    my $bdir;
2365    if (($cmt eq "Sources") || ($cmt =~ /(V[EM]|RM)Script/)) {
2366        $tdir = "$remdir/$delivery->{$ENV{'PBPROJ'}}/src";
2367    } elsif ($cmt eq "CPAN") {
2368        $tdir = "$remdir";
2369    } elsif ($cmt =~ /(V[EM]|RM)(build|test|prep)/) {
2370        $tdir = $remdir."/$ENV{'PBPROJ'}/pbdelivery";
2371        $bdir = $remdir."/$ENV{'PBPROJ'}/pbbuild";
2372        # Remove a potential $ENV{'HOME'} as bdir should be relative to pb's home
2373        $bdir =~ s|\$ENV.+\}/||;
2374    } elsif ($cmt eq "Announce") {
2375        $tdir = "$remdir/$delivery->{$ENV{'PBPROJ'}}";
2376    } elsif ($cmt eq "Web") {
2377        $tdir = "$remdir/$delivery->{$ENV{'PBPROJ'}}";
2378    } elsif ($cmt eq "Packages") {
2379        if (($pbos->{'type'} eq "rpm") || ($pbos->{'type'} eq "pkg") || ($pbos->{'type'} eq "hpux") || ($pbos->{'type'} eq "tgz") || ($pbos->{'type'} eq "apk")) {
2380            # put packages under an arch subdir
2381            $tdir = "$remdir/$delivery->{$ENV{'PBPROJ'}}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}";
2382        } elsif (($pbos->{'type'} eq "deb") || ($pbos->{'type'} eq "ebuild")) {
2383            # No need for an arch subdir
2384            $tdir = "$remdir/$delivery->{$ENV{'PBPROJ'}}/$pbos->{'name'}/$pbos->{'version'}";
2385        } else {
2386            die "Please teach the dev team where to deliver ($pbos->{'type'} type of packages\n";
2387        }
2388
2389        my $repodir = $tdir;
2390        $repodir =~ s|^$remdir/||;
2391        my $repotag = "";
2392        $repotag = "-$delivery->{$ENV{'PBPROJ'}}" if ($delivery->{$ENV{'PBPROJ'}} ne "");
2393
2394        my ($pbrepo) = pb_conf_get("pbrepo");
2395
2396        # Script for Repository management
2397        $v = "$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}" if (not defined $v);
2398        $pbscript{$v} = "$ENV{'PBBUILDDIR'}/pbscript.$$";
2399       
2400        # That script should be part of the delivery
2401        $src = "$src $pbscript{$v}";
2402        open(PBS,"> $pbscript{$v}") || die "Unable to create $pbscript{$v}";
2403        if ($pbos->{'type'} eq "rpm") {
2404            my $pbsha = pb_distro_get_param($pbos,pb_conf_get("ossha"));
2405            my ($gpgcheck) = pb_conf_get_if("pbgpgcheck");
2406            my $pbgpgcheck;
2407            $pbgpgcheck = $gpgcheck->{$ENV{'PBPROJ'}} if (defined $gpgcheck);
2408            my $pbgpgkey = "$pbrepo->{$ENV{'PBPROJ'}}/$repodir/$ENV{'PBPROJ'}.pubkey";
2409            my ($gpgaddpkey) = pb_conf_get_if("pbadditionalgpg");
2410            my $pbadditionalgpg;
2411            $pbadditionalgpg = $gpgaddpkey->{$ENV{'PBPROJ'}} if (defined $gpgaddpkey);
2412            if (defined $pbadditionalgpg) {
2413                foreach my $k (split(/,/,$pbadditionalgpg)) {
2414                    $pbgpgkey .= " $pbrepo->{$ENV{'PBPROJ'}}/$repodir/$ENV{'PBPROJ'}-$k.pubkey";
2415                }
2416            }
2417            # By default force GPG check in repo even if we support signature of packages to fail. This is a best practice
2418            $pbgpgcheck = 1 if (not defined $pbgpgcheck);
2419            # Also make a pbscript to generate dnf/yum/urpmi bases
2420            print PBS << "EOF";
2421#!/bin/bash
2422# Prepare a script to ease dnf/yum setup
2423EOF
2424            print PBS "set -x\n" if ($pbdebug gt 1);
2425            print PBS << "EOF";
2426cat > $ENV{'PBPROJ'}$repotag.repo << EOT
2427[$ENV{'PBPROJ'}$repotag]
2428name=$pbos->{'name'} $pbos->{'version'} $pbos->{'arch'} - $ENV{'PBPROJ'} Vanilla Packages
2429baseurl=$pbrepo->{$ENV{'PBPROJ'}}/$repodir
2430enabled=1
2431gpgcheck=$pbgpgcheck
2432gpgkey=$pbgpgkey
2433EOT
2434chmod 644 $ENV{'PBPROJ'}$repotag.repo
2435
2436# Clean up old repo content
2437rm -rf headers/ repodata/
2438# Create yum repo
2439if [ -x /usr/bin/yum-arch ]; then
2440    yum-arch .
2441fi
2442# Create repodata
2443createrepo -s $pbsha .
2444# Link to the key
2445if [ -s $ENV{'PBPROJ'}.pubkey ]; then
2446    # Avoiding creating empty repomd.xml.key; fails on opensuse 12.1
2447    (cd repodata ; ln -sf ../$ENV{'PBPROJ'}.pubkey repomd.xml.key)
2448fi
2449# sign the repomd (at least useful for SLES - which requires a local key)
2450# gpg -a --detach-sign repodata/repomd.xml
2451# SLES also looks for media.1/info.txt
2452EOF
2453            if ($pbos->{'family'} eq "md") {
2454                # For Mandriva add urpmi management
2455                print PBS << "EOF";
2456# Prepare a script to ease urpmi setup
2457cat > $ENV{'PBPROJ'}$repotag.addmedia << EOT
2458#rpm --import $pbrepo->{$ENV{'PBPROJ'}}/$repodir/$ENV{'PBPROJ'}.pubkey
2459urpmi.addmedia $ENV{'PBPROJ'}$repotag $pbrepo->{$ENV{'PBPROJ'}}/$repodir with media_info/hdlist.cz
2460EOT
2461chmod 755 $ENV{'PBPROJ'}$repotag.addmedia
2462
2463# Clean up old repo content
2464rm -f hdlist.cz synthesis.hdlist.cz
2465# Create urpmi repo
2466genhdlist2 --clean .
2467if [ \$\? -ne 0 ]; then
2468    genhdlist .
2469fi
2470(cd media_info ; ln -sf ../$ENV{'PBPROJ'}.pubkey pubkey)
2471EOF
2472            }
2473            if ($pbos->{'name'} eq "fedora") {
2474                # Extract the spec file to please Fedora maintainers :-(
2475                print PBS << "EOF";
2476for p in $basesrc; do
2477    echo \$p | grep -q 'src.rpm'
2478    if [ \$\? -eq 0 ]; then
2479        rpm2cpio \$p | cpio -ivdum --quiet '*.spec'
2480    fi
2481done
2482EOF
2483            }
2484            if ($pbos->{'family'} eq "novell") {
2485                # Add ymp scripts for one-click install on SuSE
2486                print PBS << "EOF";
2487# Prepare a script to ease SuSE one-click install
2488# Cf: http://de.opensuse.org/1-Klick-Installation/ISV
2489#
2490cat > $ENV{'PBPROJ'}$repotag.ymp << EOT
2491<?xml version="1.0" encoding="utf-8"?>
2492<!-- vim: set sw=2 ts=2 ai et: -->
2493<metapackage xmlns:os="http://opensuse.org/Standards/One_Click_Install" xmlns="http://opensuse.org/Standards/One_Click_Install">
2494    <group><!-- The group of software, typically one for project-builder.org -->
2495        <name>$ENV{'PBPROJ'} Bundle</name> <!-- Name of the software group -->
2496        <summary>Software bundle for the $ENV{'PBPROJ'}$repotag project</summary> <!--This message is shown to the user and should describe the whole bundle -->
2497        <description>This is the summary of the $ENV{'PBPROJ'}$repotag Project
2498             
2499            Details are available on a per package basis below
2500
2501        </description><!--This is also shown to the user -->
2502        <remainSubscribed>false</remainSubscribed> <!-- Don't know what it mean -->
2503        <repositories><!-- List of needed repositories -->
2504            <repository>
2505                <name>$ENV{'PBPROJ'}$repotag Repository</name> <!-- Name of the repository  -->
2506                <summary>This repository contains the $ENV{'PBPROJ'}$repotag project packages.</summary> <!-- Summary of the repository -->
2507                <description>This repository contains the $ENV{'PBPROJ'}$repotag project packages.</description><!-- This description is shown to the user -->
2508                <url>$pbrepo->{$ENV{'PBPROJ'}}/$repodir</url><!--URL of repository, which is added -->
2509            </repository>
2510        </repositories>
2511        <software><!-- A List of packages, which should be added through the one-click-installation -->
2512EOT
2513for p in $basesrc; do
2514    sum=`rpm -q --qf '%{SUMMARY}' \$p`
2515    name=`rpm -q --qf '%{NAME}' \$p`
2516    desc=`rpm -q --qf '%{description}' \$p`
2517    cat >> $ENV{'PBPROJ'}$repotag.ymp << EOT
2518            <item>
2519                <name>\$name</name><!-- Name of the package, is shown to the user and used to identify the package at the repository -->
2520                <summary>\$sum</summary> <!-- Summary of the package -->
2521                <description>\$desc</description> <!-- Description, is shown to the user -->
2522            </item>
2523EOT
2524done
2525cat >> $ENV{'PBPROJ'}$repotag.ymp << EOT
2526        </software>
2527    </group>
2528</metapackage>
2529EOT
2530chmod 644 $ENV{'PBPROJ'}$repotag.ymp
2531EOF
2532            }
2533        } elsif ($pbos->{'type'} eq "deb") {
2534            # Also make a pbscript to generate apt bases
2535            # Cf: http://www.debian.org/doc/manuals/repository-howto/repository-howto.fr.html
2536            # This dirname removes ver
2537            my $debarch = pb_get_debarch($pbos);
2538            my $projcomponent = pb_get_debpc($pbos);
2539            my $rpd = dirname("$pbrepo->{$ENV{'PBPROJ'}}/$repodir");
2540            # Remove extra . in path to fix #522
2541            $rpd =~ s|/./|/|g;
2542            print PBS << "EOF";
2543#!/bin/bash
2544# Prepare a script to ease apt setup
2545cat > $ENV{'PBPROJ'}$repotag.sources.list << EOT
2546deb $rpd $pbos->{'version'} $projcomponent
2547deb-src $rpd $pbos->{'version'} $projcomponent
2548EOT
2549chmod 644 $ENV{'PBPROJ'}$repotag.sources.list
2550
2551# Up two levels to deal with the dist dir cross versions
2552cd ..
2553mkdir -p dists/$pbos->{'version'}/$projcomponent/binary-$debarch dists/$pbos->{'version'}/$projcomponent/source dists/$pbos->{'version'}/$projcomponent/binary-all
2554
2555# Prepare a script to create apt info file
2556# Reuse twice after
2557TMPD=`mktemp -d /tmp/pb.XXXXXXXXXX` || exit 1
2558mkdir -p \$TMPD
2559cat > \$TMPD/Release << EOT
2560Archive: unstable
2561Component: $projcomponent
2562Origin: $ENV{'PBPROJ'}$repotag
2563Label: $ENV{'PBPROJ'}$repotag dev repository $pbrepo->{$ENV{'PBPROJ'}}
2564EOT
2565
2566echo "Creating Packages metadata"
2567for i in dists/$pbos->{version}/$projcomponent/binary-*; do
2568    arch=`basename \$i | sed 's/binary-//'`
2569    echo "Packages for \$arch:"
2570    dpkg-scanpackages -a\$arch $pbos->{'version'} /dev/null > dists/$pbos->{'version'}/$projcomponent/binary-\$arch/Packages
2571    gzip -9 < dists/$pbos->{'version'}/$projcomponent/binary-\$arch/Packages > dists/$pbos->{'version'}/$projcomponent/binary-\$arch/Packages.gz
2572    bzip2 -9 < dists/$pbos->{'version'}/$projcomponent/binary-\$arch/Packages > dists/$pbos->{'version'}/$projcomponent/binary-\$arch/Packages.bz2
2573done
2574echo "Creating Contents metadata"
2575apt-ftparchive contents $pbos->{'version'} | gzip -c9 > dists/$pbos->{'version'}/Contents.gz
2576echo "Creating Release metadata ($pbos->{'arch'} aka $debarch)"
2577cat \$TMPD/Release > dists/$pbos->{'version'}/$projcomponent/binary-all/Release
2578echo "Architecture: all" >> dists/$pbos->{'version'}/$projcomponent/binary-all/Release
2579cat \$TMPD/Release > dists/$pbos->{'version'}/$projcomponent/binary-$debarch/Release
2580echo "Architecture: $debarch" >> dists/$pbos->{'version'}/$projcomponent/binary-$debarch/Release
2581echo "Creating Source metadata"
2582dpkg-scansources $pbos->{'version'} /dev/null > dists/$pbos->{'version'}/$projcomponent/source/Sources
2583gzip -9 < dists/$pbos->{'version'}/$projcomponent/source/Sources > dists/$pbos->{'version'}/$projcomponent/source/Sources.gz
2584bzip2 -9 < dists/$pbos->{'version'}/$projcomponent/source/Sources > dists/$pbos->{'version'}/$projcomponent/source/Sources.bz2
2585cat \$TMPD/Release > dists/$pbos->{'version'}/$projcomponent/source/Release
2586echo "Architecture: Source" >> dists/$pbos->{'version'}/$projcomponent/source/Release
2587echo "Creating Release metadata"
2588apt-ftparchive -o APT::FTPArchive::Release::Suite=$pbos->{version} release dists/$pbos->{'version'} > dists/$pbos->{'version'}/Release
2589rm -rf \$TMPD
2590EOF
2591        } elsif ($pbos->{'type'} eq "ebuild") {
2592            # make a pbscript to generate links to latest version
2593            print PBS << "EOF";
2594#!/bin/bash
2595# Prepare a script to create correct links
2596cd $tdir
2597for p in $src; do
2598    echo \$p | grep -q '.ebuild'
2599    if [ \$\? -eq 0 ]; then
2600        j=`basename \$p`
2601        pp=`echo \$j | perl -pe 's/-[0-9\.]+-r[0-9].*\.ebuild/.ebuild/'`
2602        ln -sf \$j \$pp
2603    fi
2604done
2605EOF
2606        } elsif ($pbos->{'type'} eq "apk") {
2607            # TODO
2608        }
2609        close(PBS);
2610        chmod 0755,$pbscript{$v};
2611    } else {
2612        return;
2613    }
2614
2615    # Useless for VE
2616    my $nport = pb_get_port($sshport,$pbos,$cmt) if ($cmt !~ /^VE/);
2617
2618    # Remove a potential $ENV{'HOME'} as tdir should be relative to pb's home
2619    if ($cmt =~ /^VE/o) {
2620        $tdir =~ s|\$ENV.+\}/|| or confess "for $cmt: $tdir must have \$ENV{'HOME'} in it or the remainder of the code won't work since some parts use relative pathing and others will try absolute.  Use a symlink if necessary to place the real files in a different place than under your home directory";
2621    } else {
2622        # process, ok if not present.
2623        $tdir =~ s|\$ENV.+\}/||;
2624    }
2625
2626    my $tm = "";
2627    if ($cmt =~ /^(V|R)M/) {
2628        $tm = "sleep $vtmout" if (defined $vtmout);
2629    }
2630
2631    # ssh communication if not VE or CPAN
2632    # should use a hash instead...
2633    my ($shcmd,$shcmdroot,$cpcmd,$cptarget,$cp2target);
2634    my $tpdir;
2635    my $tp;
2636    my $context = "$ENV{'PBTMP'}";
2637    my %tag;
2638    my $dkaccount = "";
2639    my $cmd1 = "";
2640    my $cidfile;
2641    my $homedir = "";
2642    if ($cmt =~ /^VE/) {
2643        $tp = pb_path_expand($vepath->{$ENV{'PBPROJ'}});
2644        $vetype = pb_ve_get_type($vetype);
2645        if ($vetype eq "docker") {
2646            $tpdir = "/";
2647            $cidfile = "$context/ctn-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}.cid";
2648        } else {
2649            $tpdir = pb_path_expand("$tp/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}");
2650        }
2651        my $arch = pb_get_arch();
2652        if ($vetype eq "chroot") {
2653            $shcmdroot = "sudo /usr/sbin/chroot $tpdir ";
2654            $shcmd = "$shcmdroot /bin/su - $mac -c ";
2655        } elsif ($vetype eq "schroot") {
2656            $shcmd = "schroot $tp -u $mac -- ";
2657        } elsif ($vetype eq "docker") {
2658            # docker manages the storage so rely on it
2659            $shcmdroot = "";
2660            my $docrepo = pb_ve_docker_repo();
2661            $cmd1 = pb_check_req("docker",0);
2662            #my ($dockeropt) =  pb_conf_get_if("dockeropt");
2663            # pbimage is used for docker and is setup to the right name depending on the step:
2664            # TODO: This is not coded yet
2665            my $pbimage = undef;
2666            # step 0 : nothing at creation -> tag n-v-a (made in VE.pm)
2667            # step 1 : n-v-a + setup -> tag n-v-a-pb
2668            # step 2 : n-v-a-pb + build -> tag n-v-a-pb-pbproj
2669            # step 3 : n-v-a-pb-pbproj at use
2670            if ((not defined $pbimage) || ($pbimage eq "")) {
2671                # If no image name given, create a naming convention
2672                $tag{1} = "$docrepo$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
2673                $tag{2} = "$tag{1}-pb";
2674                $tag{3} = "$tag{2}-$ENV{'PBPROJ'}";
2675            } else {
2676                # If we were given an image name, just use it
2677                $tag{1} = $pbimage;
2678                $tag{2} = $pbimage;
2679                $tag{3} = $pbimage;
2680            }
2681            # If we do not create the image, then use the one we should have
2682            # TODO: we don't do anything with found !!
2683            my $found = pb_ve_docker_get_image($tag{$pbstep});
2684            confess "No image $tag{$pbstep} available in Docker, please create one first\n" if (not defined $found);
2685            # Now we use that image to do what is needed
2686            # use a dockerfile to ease the creation of next images
2687            # the context is in an empty temp dir
2688            my $tmpd = "$context/Dockerfile";
2689            open(DOCKER, "> $tmpd") || die "Unable to create the docker file $tmpd";
2690            print DOCKER "FROM $tag{$pbstep}\n";
2691            print DOCKER "MAINTAINER project-builder.org aka pb\n";
2692            if ($pbstep == 1) {
2693                # setup done as root
2694                $dkaccount = "root";
2695            } else {
2696                # Rest done as pb
2697                $dkaccount = "pb";
2698            }
2699            print DOCKER "USER $dkaccount\n";
2700            if ($pbstep <= 2) {
2701                $shcmd = "$cmd1 build -t $tag{$pbstep+1} $context";
2702            } else {
2703                # As we are in run phase use docker run. cmd will be completed below
2704                $shcmd = "$cmd1 run --cidfile=\"$cidfile\" --user $dkaccount --name $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-$cmt";
2705            }
2706            #$shcmd = "$cmd1 build $dockeropt->{$ENV{'PBPROJ'}} -t $tag{$pbstep+1} $context";
2707        }
2708        $shcmd = "setarch i386 $shcmd" if (($pbos->{'arch'} =~ /i[3456]86/) && ($arch eq 'x86_64'));
2709        if (($cmt =~ /VE/) && ($vetype ne "docker")) {
2710            # We need to get the home dir of the target account to deliver in the right place
2711            open(PASS,"$tpdir/etc/passwd") || die "Unable to open $tpdir/etc/passwd: $!";
2712            while (<PASS>) {
2713                my ($c1,$c2,$c3,$c4,$c5,$c6,$c7) = split(/:/);
2714                $homedir = $c6 if ($c1 =~ /^$mac$/);
2715                pb_log(3,"Homedir: $homedir - account: $c6\n");
2716            }
2717            close(PASS);
2718            $cpcmd = "sudo /bin/cp -r ";
2719        }
2720        if (($cmt =~ /VE/) && ($vetype eq "docker")) {
2721            if ($pbstep >= 2) {
2722                # We need to get the home dir of the target account to deliver in the right place
2723                $homedir = `$cmd1 run --cidfile="$cidfile" $tag{$pbstep} grep -E '^$dkaccount:' /etc/passwd | cut -d: -f6`;
2724                chomp($homedir);
2725                open(CID,"$cidfile") || confess "Unable to open $cidfile";
2726                my $cid = <CID>;
2727                close(CID);
2728                pb_system("$cmd1 rm $cid","","quiet");
2729                unlink("$cidfile");
2730            } else {
2731                $homedir = "/";
2732            }
2733            $cpcmd = "cp -r ";
2734        }
2735        $cptarget = "$tpdir/$homedir/$tdir";
2736        if ($cmt =~ /VE(build|prep)/) {
2737            $cp2target = "$tpdir/$homedir/$bdir";
2738        }
2739        pb_log(2,"On VE using $cptarget as target dir to copy to\n");
2740    } elsif ($cmt eq "CPAN") {
2741        my $ftpput = pb_check_req("ncftpput",1);
2742        my $ftpget = pb_check_req("wget",1);
2743        my ($cpanuser,$cpanpasswd) = pb_conf_get_if("cpanuser","cpanpasswd");
2744        return if ((not defined $cpanuser) || (not defined $cpanuser->{$ENV{'PBPROJ'}}));
2745        return if ((not defined $cpanpasswd) || (not defined $cpanpasswd->{$ENV{'PBPROJ'}}));
2746        my ($cpansubdir) = pb_conf_get_if("cpansubdir");
2747        $shcmd = "$ftpget --post-data \'HIDDENNAME=".$cpanuser->{$ENV{'PBPROJ'}};
2748        $shcmd .= "&user=".$cpanuser->{$ENV{'PBPROJ'}};
2749        $shcmd .= "&password=".$cpanpasswd->{$ENV{'PBPROJ'}};
2750        $shcmd .= "&SUBMIT_pause99_add_uri_upload=\"Upload the checked files\"";
2751        $shcmd .= "&pause99_add_uri_subdirtext=".$cpansubdir->{$ENV{'PBPROJ'}} if ((defined $cpansubdir) && (defined $cpansubdir->{$ENV{'PBPROJ'}}));
2752        foreach my $s (split(/ /,$src)) {
2753            $shcmd .= "&pause99_add_uri_upload=".basename($s);
2754        }
2755        $shcmd .= "'";
2756        $cpcmd = "$ftpput $host $dir";
2757        $cptarget = "CPAN";
2758    } else {
2759        my $keyopt = "";
2760        if ($cmt !~ /Packages/) {
2761            my $keyfile = pb_ssh_get(0,$remhost,$nport);
2762            $keyopt = "-i $keyfile" if (defined $keyfile);
2763        }
2764        my $sshcmd = pb_check_req("ssh",1);
2765        my $scpcmd = pb_check_req("scp",1);
2766        $shcmd = "$sshcmd $keyopt -q -o NoHostAuthenticationForLocalhost=yes -p $nport $mac";
2767        $cpcmd = "$scpcmd $keyopt -p -o NoHostAuthenticationForLocalhost=yes -P $nport";
2768        $cptarget = "$mac:$tdir";
2769        if ($cmt =~ /^(V|R)M(build|prep)/) {
2770            $cp2target = "$mac:$bdir";
2771        }
2772    }
2773   
2774    my $logres = "";
2775    # Do not touch when just announcing
2776    if (($cmt ne "Announce") && ($cmt ne "CPAN")) {
2777        # For docker everything is done in the Dockerfile for steps < 3
2778        if (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep < 3)) {
2779            print DOCKER "RUN mkdir -p $cptarget\n";
2780            print DOCKER "RUN cd $cptarget ; for i in $basesrc; do if [ -f \$i ]; then rm -f \$i; fi; done\n";
2781        } elsif (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep >= 3)) {
2782            # Nothing to do here
2783        } else {
2784            pb_system("$shcmd \"mkdir -p $tdir ; cd $tdir ; echo \'for i in $basesrc; do if [ -f \$i ]; then rm -f \$i; fi; done ; $cmd\' | bash -e\"","Preparing $tdir on $cptarget");
2785        }
2786    } else {
2787        $logres = "> ";
2788    }
2789
2790    if ($cmt =~ /SandBox/) {
2791        # Install from sandbox mean using the result of the just passed sbx2build command
2792        # Get content saved in cms2build
2793        my ($pkg) = pb_conf_read("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml","pbpkg");
2794        my $pbextdir = pb_get_extdir();
2795        confess "Unable to get package list" if (not defined $pkg);
2796
2797        # We consider 2 specific packages
2798        my $vertag1 = $pkg->{"ProjectBuilder"};
2799        my $vertag2 = $pkg->{"project-builder"};
2800        # get the version of the current package - maybe different
2801        pb_log(2,"Vertag1: $vertag1\n");
2802        pb_log(2,"Vertag2: $vertag2\n");
2803        my ($pbver1,$tmp1) = split(/-/,$vertag1);
2804        my ($pbver2,$tmp2) = split(/-/,$vertag2);
2805        if ($cmt =~ /VE/) {
2806            # Copy inside the VE
2807            if ($vetype ne "docker") {
2808                copy("$ENV{'PBDESTDIR'}/ProjectBuilder-$pbver1$pbextdir.tar.gz","$tpdir/tmp");
2809                copy("$ENV{'PBDESTDIR'}/project-builder-$pbver2$pbextdir.tar.gz","$tpdir/tmp");
2810            } else {
2811                copy("$ENV{'PBDESTDIR'}/ProjectBuilder-$pbver1$pbextdir.tar.gz","$context");
2812                copy("$ENV{'PBDESTDIR'}/project-builder-$pbver2$pbextdir.tar.gz","$context");
2813                print DOCKER "COPY ProjectBuilder-$pbver1$pbextdir.tar.gz /tmp/\n";
2814                print DOCKER "COPY project-builder-$pbver2$pbextdir.tar.gz /tmp/\n";
2815            }
2816        } else {
2817            pb_system("$cpcmd $ENV{'PBDESTDIR'}/ProjectBuilder-$pbver1$pbextdir.tar.gz $ENV{'PBDESTDIR'}/project-builder-$pbver2$pbextdir.tar.gz $mac:/tmp","Copying local project files to VM/RM");
2818        }
2819    }
2820           
2821    pb_log(0,"Sources handled ($cmt): $src\n");
2822    # For docker everything is done in the Dockerfile
2823    if (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep < 3)) {
2824        foreach my $f (split(/ +/,$src)) {
2825            copy("$f","$context");
2826            print DOCKER "COPY ".basename($f)." $cptarget/\n";
2827        }
2828        print DOCKER "RUN cd $cptarget ; $cmd\n" if ((defined $cmd) && ($cmd ne ""));
2829    } elsif (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep >= 3)) {
2830        # Nothing to do
2831    } else {
2832        pb_system("cd $ENV{'PBBUILDDIR'} ; $cpcmd $src $cptarget 2> /dev/null","$cmt delivery in $cptarget");
2833    }
2834
2835    # For VE we need to change the owner manually
2836    if ($cmt =~ /^VE/) {
2837        if ($vetype ne "docker") {
2838            my $sudomode = pb_distro_get_param($pbos,pb_conf_get("ossudoersmode"));
2839            my $res = pb_system("$shcmdroot sed -i '/requiretty/d' /etc/sudoers","Removing potential requiretty in sudoers","quiet");
2840            pb_system("$shcmdroot sed '/requiretty/d' /etc/sudoers > /tmp/sudoers.new ; mv /tmp/sudoers.new $tpdir/tmp/sudoers.new ; $shcmdroot mv /tmp/sudoers.new /etc/sudoers ; $shcmdroot chown root:root /etc/sudoers ; $shcmdroot chmod $sudomode /etc/sudoers","Removing again potential requiretty in sudoers as sed -i failed") if (($res ne 0) && (-f "$tpdir/etc/sudoers"));
2841            pb_system("$shcmd \"sudo chown -R $mac $tdir\"","Adapt owner in $tdir to $mac");
2842        } else {
2843            print DOCKER "RUN sed -i '/requiretty/d' /etc/sudoers\n" if ($pbstep == 1);
2844            # Bug in at least Fedora 23 where the dir wasn't owned correctly - doesn't hurt anyway
2845            print DOCKER "RUN chown root:root /etc\n" if ($pbstep == 1);
2846            print DOCKER "RUN chmod 755 /etc\n" if ($pbstep == 1);
2847        }
2848    }
2849
2850    # Use the right script name depending on context
2851    my $pbscript2;
2852    if (($cmt =~ /^(V[EM]|RM)/) || ($cmt =~ /Packages/)){
2853        $pbscript2 = $pbscript{$v};
2854    } else {
2855        $pbscript2 = "pbscript";
2856    }
2857
2858    # It's already ready for CPAN
2859    my $shcmdbase = $shcmd;
2860    my $dkcmd;
2861    if ($cmt ne "CPAN") {
2862        if (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep < 3)) {
2863            copy("$pbscript2","$context");
2864            my $s = basename($pbscript2);
2865            print DOCKER "COPY $s $cptarget/\n";
2866            # call directly with perl to avoid text file busy error messages
2867            print DOCKER "RUN cd $cptarget ; perl ./$s ; if [ '$pbkeep' = '0' ]; then rm -f $s; fi\n";
2868            print DOCKER "RUN rm -rf $cptarget/*\n";
2869            print DOCKER "RUN chown -R $dkaccount $cptarget\n" if ($pbstep == 1);
2870            close(DOCKER);
2871        } elsif (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep >= 3)) {
2872            $dkcmd = "/bin/bash $homedir/$tdir-2/".basename($pbscript2);
2873        } else {
2874            my $s = basename($pbscript2);
2875            $shcmd .= " \"echo \'cd $tdir ; if [ -x $s ]; then ./$s; fi ;  if [ $pbkeep == 0 ]; then rm -f $s; fi\' | bash\"";
2876        }
2877    }
2878    my $cmdverb = "verbose";
2879    if (($cmt eq "Announce") || ($cmt eq "CPAN")) {
2880        $cmdverb = undef;
2881    } elsif (defined ($v)) {
2882        $cmdverb = "verbose_\[$v\] ";
2883    }
2884    # this is where we launch the execution
2885    if (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep >= 3)) {
2886        pb_mkdir_p("$ENV{'PBTMP'}/vebuild");
2887        my $tsrc = "";
2888        foreach my $f (split(/ +/,$src)) {
2889            copy("$f","$ENV{'PBTMP'}/vebuild");
2890            $tsrc .= " ".basename($f);
2891        }
2892        my $ts = "$ENV{'PBTMP'}/vebuild/".basename($pbscript2);
2893        open(DOCKER," > $ts") || confess "Unable to create $ts";
2894        print DOCKER "#!/bin/bash\n";
2895        print DOCKER "# Put content from host to guest, copying to have the correct uid/gid and rights to do so\n";
2896        print DOCKER "# set -x\n";
2897        print DOCKER "for i in $tsrc; do\n";
2898        print DOCKER "  # Cleanup first to avoid rights issues for root owned files e.g.\n";
2899        print DOCKER "  rm -f $homedir/$tdir/\$i\n";
2900        print DOCKER "  cp $homedir/$tdir-2/\$i $homedir/$tdir\n";
2901        print DOCKER "done\n";
2902        print DOCKER "(cd $homedir/$tdir-2 ; ls -al)\n";
2903        print DOCKER pb_get_content($pbscript2);
2904        print DOCKER "\n# Get content back from guest in a host dir\n";
2905        # Host dir seen as TDIR in container
2906        $cp2target = "$ENV{'PBBUILDDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
2907        pb_mkdir_p("$cp2target");
2908        chmod 0777,"$cp2target";
2909        print DOCKER "TDIR=$homedir/$bdir-2/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}\n";
2910        print DOCKER "for i in $homedir/$bdir/*; do\n";
2911        print DOCKER "  # Cleanup first to avoid rights issues for root owned files e.g.\n";
2912        print DOCKER "  j=\`basename \$i\`\n";
2913        print DOCKER "  rm -f \$TDIR/\$j\n";
2914        print DOCKER "  cp -r \$i \$TDIR\n";
2915        print DOCKER "done\n";
2916        print DOCKER "find \$TDIR -type f -print0 | xargs -0 chmod 644\n";
2917        print DOCKER "find \$TDIR -type d -print0 | xargs -0 chmod 777 2> /dev/null\n";
2918        #print DOCKER "ls -al \$TDIR\n";
2919        print DOCKER "exit 0\n";
2920        close(DOCKER);
2921        $shcmd .= " -v $ENV{'PBBUILDDIR'}:$homedir/$bdir-2 -v $ENV{'PBTMP'}/vebuild:$homedir/$tdir-2 -w $homedir/$tdir-2 $tag{$pbstep} $dkcmd";
2922    }
2923    my $ret = pb_system("$shcmd","Executing pbscript on $cptarget if needed",$cmdverb);
2924   
2925    # If target is deb family then sign Release file for Packages
2926    my $pbos2 = pb_distro_get_context($v);
2927    if (($cmt =~ /Packages/) && ($pbos2->{'type'} eq "deb")) {
2928        my $debarch = pb_get_debarch($pbos2);
2929        my $projcomponent = pb_get_debpc($pbos2);
2930        for my $f ("dists/$pbos2->{'version'}/$projcomponent/binary-$debarch/Release","dists/$pbos2->{'version'}/$projcomponent/binary-all/Release","dists/$pbos2->{'version'}/$projcomponent/source/Release","dists/$pbos2->{'version'}/Release") {
2931            my $debdir = dirname($cptarget);
2932            pb_system("cd $ENV{'PBBUILDDIR'} ; $cpcmd $debdir/$f . 2> /dev/null","Getting deb $f file from $debdir");
2933            pb_sign_file("$ENV{'PBBUILDDIR'}/Release");
2934            pb_system("$cpcmd $ENV{'PBBUILDDIR'}/Release.gpg $debdir/$f.gpg 2> /dev/null","Putting deb $f.gpg file to $debdir");
2935            my ($void,$debfile) = split(/:/,$debdir);
2936            pb_system("$shcmdbase \"chmod 644 $debfile/$f.gpg\"","Set mode to 644 for $debfile/$f.gpg");
2937        }
2938    } 
2939
2940    if ($cmt =~ /^(V[EM]|RM)build/) {
2941        # Get back info on pkg produced, compute their name and get them from the VM/RM
2942        pb_system("$cpcmd $cp2target/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'} $ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.$$ 2> /dev/null","Get package names in $cp2target");
2943        if (not -f "$ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.$$") {
2944            pb_log(0,"ERROR with VM/RM $v on getting $ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.$$\n");
2945        } else {
2946            open(KEEP,"$ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.$$") || die "Unable to read $ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.$$";
2947            my $src = <KEEP>;
2948            chomp($src);
2949            close(KEEP);
2950            unlink("$ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.$$");
2951
2952            $src =~ s/^ *//;
2953            pb_mkdir_p("$ENV{'PBBUILDDIR'}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}");
2954            # Change pgben to make the next send2target happy
2955            my $made = "";
2956   
2957            # For VM/RM we don't want shell expansion to hapen locally but remotely
2958            my $delim = '\'';
2959            if ($cmt =~ /^VEbuild/) {
2960                # For VE we need to support shell expansion locally
2961                $delim = "";
2962            }   
2963
2964            open(KEEP,"> $ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}") || die "Unable to write $ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
2965            foreach my $p (split(/ +/,$src)) {
2966                my $j = basename($p);
2967                pb_system("$cpcmd $cp2target/$delim$p$delim $ENV{'PBBUILDDIR'}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'} 2> /dev/null","Recovery of package $j in $ENV{'PBBUILDDIR'}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}");
2968                $made="$made $pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}/$j"; # if (($pbos->{'type'} ne "rpm") || ($j !~ /.src.rpm$/));
2969            }
2970       
2971            print KEEP "$made\n";
2972            close(KEEP);
2973            if (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep >= 3)) {
2974                pb_rm_rf("$cp2target");
2975                pb_system("$cmd1 rm $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-$cmt","","quiet");
2976            } else {
2977                pb_system("$shcmdbase \"rm -rf $tdir $bdir\"","$cmt cleanup");
2978            }
2979
2980            # Sign packages locally
2981            pb_sign_pkgs($pbos,$made);
2982
2983            # We want to send them to the ssh account so overwrite what has been done before
2984            pb_log(2,"Before sending pkgs, vexist: $vexist, vpid: $vpid\n");
2985            pb_send2target("Packages","$ENV{'PBDESTDIR'}/pbscript.$$",$pbos->{'name'}."-".$pbos->{'version'}."-".$pbos->{'arch'},$vexist,$vpid);
2986            pb_rm_rf("$ENV{'PBBUILDDIR'}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}");
2987        }
2988    }
2989    unlink("$pbscript{$v}") if ((($cmt =~ /^(V[ME]|RM)/) || ($cmt =~ /Packages/)) && ($pbkeep eq 0) && ($cmt !~ /Script/));
2990
2991    pb_log(2,"Before halt, vexist: $vexist, vpid: $vpid\n");
2992    if ((! $vexist) && ($cmt =~ /^VM/)) {
2993        # If in setupvm then takes a snapshot just before halting
2994        if ($snapme != 0) {
2995            my ($vmmonport,$vmtype) = pb_conf_get("vmmonport","vmtype");
2996            # For monitoring control
2997            if ((($vmtype->{$ENV{'PBPROJ'}}) eq "kvm") || (($vmtype->{$ENV{'PBPROJ'}}) eq "qemu")) {
2998                eval
2999                {
3000                require Net::Telnet;
3001                Net::Telnet->import();
3002                };
3003                if ($@) {
3004                    # Net::Telnet not found
3005                    pb_log(1,"ADVISE: Install Net::Telnet to benefit from monitoring control and snapshot feature.\nWARNING: No snapshot created");
3006                } else {
3007                    my $t = new Net::Telnet (Timeout => 120, Host => "localhost", Port => $vmmonport->{$ENV{'PBPROJ'}}) || die "Unable to dialog on the monitor";
3008                    # move to monitor mode
3009                    my @lines = $t->cmd("c");
3010                    # Create a snapshot named pb
3011                    @lines = $t->cmd("savevm pb");
3012                    # Write the new status in the VM
3013                    @lines = $t->cmd("commit all");
3014                    # End
3015                    @lines = $t->cmd("quit");
3016                }
3017            }
3018        }
3019        my $hoption = "-p";
3020        my $hpath = pb_distro_get_param($pbos,pb_conf_get("ospathcmd-halt"));
3021        # Solaris doesn't support -p of halt
3022        if ($pbos->{'type'} eq "pkg") {
3023            $hoption = "" ;
3024        }
3025        pb_system("$shcmdbase \"sudo $hpath $hoption \"; $tm ; echo \'if [ -d /proc/$vpid ]; then kill -9 $vpid; fi \' | bash ; sleep 10","VM $v halt (pid $vpid)");
3026    }
3027    if (($cmt =~ /^VE/) && ($snapme != 0)) {
3028        if ($vetype eq "docker") {
3029            foreach my $f (split(/ +/,$src)) {
3030                unlink("$context/$f");
3031            }
3032        } else {
3033            pb_ve_snap($pbos,$tp);
3034        }
3035    }
3036}
3037
3038sub pb_script2v {
3039
3040my $pbscript=shift;
3041my $vtype=shift;
3042my $pbstep=shift;   # Which step are we in (0: create, 1: setup pb, 2: prep pbproj, 3: use to build)
3043my $pbforce=shift;  # Force stop of VM. Default not.
3044my $snapme=shift;   # Do we have to create a snapshot. Default not.
3045my $pbimage=shift;  # Which image to use to start the VM/VE
3046
3047$pbforce = 0 if (not defined $pbforce);
3048$snapme = 0 if (not defined $snapme);
3049$pbstep = 3 if (not defined $pbstep);
3050
3051pb_log(2,"DEBUG: pb_script2v($vtype,$pbstep,$pbforce,$snapme)\n");
3052pb_log(2,"DEBUG: pb_script2v(pbscript: ".Dumper($pbscript)."\n") if (defined $pbscript);
3053pb_parallel_launchv($pbscript,$vtype,uc($vtype)."Script",$pbstep,$pbimage,$pbforce,$snapme,$pbsnap);
3054}
3055
3056sub pb_parallel_launchv {
3057
3058my $pbscript=shift;
3059my $vtype = shift;
3060my $action = shift;         # It an action is defined then use send2target
3061my $pbstep=shift;           # Which step are we in (0: create, 1: setup, 2: build 3: use)
3062my $pbimage=shift;          # Which image to use to start the VM/VE
3063my $pbforce=shift;          # Force stop of VM. Default not.
3064my $snapme = shift;         # By default do not snap a VM/VE/RM
3065my $usesnap = shift;        # By default study the usage of the snapshot feature of VM/VE/RM   
3066my $vm;
3067my $all;
3068
3069$pbforce = 0 if (not defined $pbforce);
3070$snapme = 0 if (not defined $snapme);
3071$usesnap = 1 if (not defined $usesnap);
3072
3073# Adapt // mode to memory size
3074$pbparallel = pb_set_parallel($vtype);
3075
3076pb_log(2,"DEBUG: pb_parallel_launch(vtype,step,force,snapme,usesnap: $vtype,$pbstep,$pbforce,$snapme,$usesnap)\n");
3077pb_log(2,"DEBUG: pb_parallel_launch(pbscript: ".Dumper($pbscript)."\n");
3078
3079my ($vexist,$vpid) = (undef,undef);
3080my $pm;
3081my $all_ok = 1;
3082if (defined $pbparallel) {
3083    $pm = new Parallel::ForkManager($pbparallel);
3084
3085    # Set which port the VM/RM will use to communicate
3086    $pm->run_on_start(\&pb_set_port);
3087    $pm->run_on_finish(sub { my ($pid, $code, $id, $signal, $dump) = @_;
3088                            unless ($code == 0 && $signal == 0 && $dump == 0) {
3089                                $all_ok = 0;
3090                                pb_log(0,"ERROR: pid $pid failed\n");
3091                            }
3092                        });
3093}
3094
3095# Determine on which V to work
3096if (defined $ENV{'PBV'}) {
3097    @$vm = split(/,/,$ENV{'PBV'});
3098} else {
3099    ($vm,$all) = pb_get2v($vtype);
3100}
3101
3102$vetype = pb_ve_get_type($vetype);
3103if ((($vtype =~ /vm/) || (($vtype =~ /ve/) && ($vetype eq "docker"))) && (((not defined $pbimage) || ($pbimage eq "")) && ($pbstep == 0))) {
3104    confess "-i image parameter needed";
3105}
3106
3107my $counter = 0;
3108foreach my $v (@$vm) {
3109    $counter++;
3110    # Modulo 2 * pbparallel (to avoid synchronization problems)
3111    $counter = 1 if ((defined $pbparallel) && ($counter > 2 * $pbparallel));
3112    $pm->start($counter) and next if (defined $pbparallel);
3113   
3114    # If launched via script2v then get the script passed as the one for the that VM
3115    if (not defined $pbscript->{$v}) {
3116        $pbscript->{$v} = $pbscript->{'default'};
3117    }
3118    #
3119    # Launch a single operation
3120    ($vexist,$vpid) = pb_launchv($pbscript->{$v},$vtype,$v,$action,$pbstep,$pbforce,$snapme,$pbsnap,$pbimage);
3121   
3122    # Skip that VM/RM if something went wrong
3123    if (($vtype =~ /(v|r)m/) && ($vpid == 0) && ($vexist == 0)) {
3124        $pm->finish if (defined $pbparallel);
3125        next;
3126    }
3127
3128   
3129    $pm->finish if (defined $pbparallel);
3130}
3131$pm->wait_all_children if (defined $pbparallel);
3132die "Aborting, one or more of the children failed." if ((not $all_ok) && ($Global::pb_stop_on_error));
3133}
3134
3135sub pb_launchv {
3136
3137my $pbscript1=shift;        # Just the script name to execute, not the hash
3138my $vtype = shift;
3139my $v = shift;              # Only 1 VM/VE/RM treated here
3140my $action = shift;         # It an action is defined then use send2target
3141my $pbstep=shift;           # Which step are we in (0: create, 1: setup, 2: build 3: use)
3142my $pbforce=shift;          # Force stop of VM. Default not.
3143my $snapme = shift;         # By default do not snap a VM/VE/RM
3144my $usesnap = shift;            # By default study the usage of the snapshot feature of VM/VE/RM   
3145my $pbimage=shift;          # Which image to use to start the VM/VE
3146my $vexist = undef;
3147my $vpid = undef;
3148
3149$pbforce = 0 if (not defined $pbforce);
3150$snapme = 0 if (not defined $snapme);
3151$usesnap = 1 if (not defined $usesnap);
3152
3153# Check that v exists
3154die "No VM/VE/RM defined, unable to launch" if (not defined $v);
3155
3156# If creation or snapshot creation mode, no snapshot usable
3157if (($pbstep == 0) || ($snapme == 1)) {
3158    $usesnap = 0;
3159}
3160
3161pb_log(2,"DEBUG: pb_launchv(vtype,v,step,force,snapme,usesnap: $vtype,$v,$pbstep,$pbforce,$snapme,$usesnap)\n");
3162pb_log(2,"DEBUG: pb_launchv(pbimage: $pbimage)\n") if (defined $pbimage);
3163pb_log(2,"DEBUG: pb_launchv(pbscript $pbscript1)\n") if (defined $pbscript1);
3164pb_log(2,"DEBUG: pb_launchv(action: $action)\n") if (defined $action);
3165# Keep only the first VM in case many were given
3166if ($v =~ /,/) {
3167    pb_log(0,"WARNING: pruning to just the first of several VM/VE/RMs listed ($v)\n");
3168    $v =~ s/,.*//;
3169}
3170
3171my $pbos = pb_distro_get_context($v);
3172
3173pb_apply_conf_proxy($pbos);
3174
3175# Launch the VMs/VEs
3176if ($vtype eq "vm") {
3177    my ($ptr,$ptr2,$vmpath,$vmport,$vms) = pb_conf_get("vmtype","vmcmd","vmpath","vmport","vmsize");
3178    my ($vmopt,$vmmm,$vmtmout,$vmsnap,$vmbuildtm,$vmmonport) = pb_conf_get_if("vmopt","vmmem","vmtmout","vmsnap","vmbuildtm","vmmonport");
3179    my $vmsize = pb_distro_get_param($pbos,$vms);
3180
3181    my $vmtype = $ptr->{$ENV{'PBPROJ'}};
3182    my $vmcmd = $ptr2->{$ENV{'PBPROJ'}};
3183
3184    if (defined $opts{'g'}) {
3185        if (($vmtype eq "kvm") || ($vmtype eq "qemu")) {
3186            $ENV{'PBVMOPT'} = "--nographic";
3187        }
3188    }
3189    if (not defined $ENV{'PBVMOPT'}) {
3190        $ENV{'PBVMOPT'} = "";
3191    }
3192    # Save the current status for later restoration
3193    $ENV{'PBOLDVMOPT'} = $ENV{'PBVMOPT'};
3194
3195    if (defined $vmopt->{$v}) {
3196        $ENV{'PBVMOPT'} .= " $vmopt->{$v}" if ($ENV{'PBVMOPT'} !~ / $vmopt->{$v}/);
3197    } elsif (defined $vmopt->{$ENV{'PBPROJ'}}) {
3198        $ENV{'PBVMOPT'} .= " $vmopt->{$ENV{'PBPROJ'}}" if ($ENV{'PBVMOPT'} !~ / $vmopt->{$ENV{'PBPROJ'}}/);
3199    }
3200
3201    # How much memory to allocate for VMs
3202    if (defined $vmmm) {
3203        my $vmmem = pb_distro_get_param($pbos,$vmmm);
3204        if (defined $vmmem) {
3205            $ENV{'PBVMOPT'} .= " -m $vmmem";
3206        }
3207    }
3208
3209    # Are we allowed to use snapshot feature
3210    if ($usesnap == 1) {
3211        if ((defined $vmsnap->{$v}) && ($vmsnap->{$v} =~ /true/i)) {
3212            $ENV{'PBVMOPT'} .= " -snapshot";
3213        } elsif ((defined $vmsnap->{$ENV{'PBPROJ'}}) && ($vmsnap->{$ENV{'PBPROJ'}} =~ /true/i)) {
3214            $ENV{'PBVMOPT'} .= " -snapshot";
3215        } elsif ($pbsnap eq 1) {
3216            $ENV{'PBVMOPT'} .= " -snapshot";
3217        }
3218    } 
3219    if ($snapme != 0) {
3220        if (($vmtype eq "kvm") || ($vmtype eq "qemu")) {
3221            # Configure the monitoring to automate the creation of the 'pb' snapshot
3222            $ENV{'PBVMOPT'} .= " -serial mon:telnet::$vmmonport->{$ENV{'PBPROJ'}},server,nowait" if ((defined $vmmonport) && (defined $vmmonport->{$ENV{'PBPROJ'}}));
3223            # In that case no snapshot call needed
3224            $ENV{'PBVMOPT'} =~ s/ -snapshot//;
3225        }
3226    }
3227   
3228    # Set a default timeout of 2 minutes if none defined externally or in conf files
3229    if (not defined $ENV{'PBVMTMOUT'}) {
3230        if (defined $vmtmout->{$v}) {
3231            $ENV{'PBVMTMOUT'} = $vmtmout->{$v};
3232        } elsif (defined $vmtmout->{$ENV{'PBPROJ'}}) {
3233            $ENV{'PBVMTMOUT'} = $vmtmout->{$ENV{'PBPROJ'}};
3234        }
3235    }
3236    if (not defined $ENV{'PBVMTMOUT'}) {
3237        $ENV{'PBVMTMOUT'} = "120";
3238    }
3239
3240    my $nport = pb_get_port($vmport,$pbos,$vtype);
3241
3242    my $cmd;
3243    my $vmm;        # has to be used for pb_check_ps
3244    if (($vmtype eq "qemu") || ($vmtype eq "kvm")) {
3245        $vmm = "$vmpath->{$ENV{'PBPROJ'}}/$v.qemu";
3246        if (($pbstep == 0) || (defined $pbimage)) {
3247            $ENV{'PBVMOPT'} .= " -cdrom $pbimage -boot d";
3248        }
3249        # Always redirect the network and always try to use a 'pb' snapshot
3250        #$cmd = "$vmcmd $ENV{'PBVMOPT'} -net nic,model=virtio -net user,hostfwd=tcp::$nport:22 -loadvm pb $vmm"
3251        #$cmd = "$vmcmd $ENV{'PBVMOPT'} -redir tcp:$nport:10.0.2.15:22 $vmm"
3252        $cmd = "$vmcmd $ENV{'PBVMOPT'} -net nic -net user,hostfwd=tcp:127.0.0.1:$nport-:22 $vmm"
3253    } elsif ($vmtype eq "xen") {
3254    } elsif ($vmtype eq "vmware") {
3255    } else {
3256        die "VM of type $vmtype not supported. Report to the dev team";
3257    }
3258    # Restore the ENV VAR Value
3259    $ENV{'PBVMOPT'} = $ENV{'PBOLDVMOPT'};
3260
3261    my ($tmpcmd,$void) = split(/ +/,$cmd);
3262    my $vmmport = undef;
3263    ($vexist,$vmmport) = pb_check_ps($tmpcmd,$vmm);
3264    $vpid = 0;
3265   
3266    if (! $vexist) {
3267        if ($pbstep == 0) {
3268            die("Found an existing Virtual machine $vmm. Won't overwrite") if (-r $vmm);
3269            if (($vmtype eq "qemu") || ($vmtype eq "xen") || ($vmtype eq "kvm")) {
3270                my $command = pb_check_req("qemu-img",0);
3271                pb_system("$command create -f qcow2 $vmm $vmsize","Creating the QEMU VM");
3272            } elsif ($vmtype eq "vmware") {
3273            } else {
3274            }
3275        }
3276        if (! -f "$vmm") {
3277            pb_log(0,"Unable to find VM $vmm\n");
3278        } else {
3279            # Is the SSH port free? if not kill the existing process using it after a build timeout period
3280            my ($vmssh,$void) = pb_check_ps($tmpcmd,$vmm);
3281            if ($vmssh) {
3282                my $buildtm = $ENV{'PBVMTMOUT'};
3283                if (defined $vmbuildtm->{$v}) {
3284                    $buildtm = $vmbuildtm->{$v};
3285                } elsif (defined $vmbuildtm->{$ENV{'PBPROJ'}}) {
3286                    $buildtm = $vmbuildtm->{$ENV{'PBPROJ'}};
3287                }
3288
3289                sleep $buildtm;
3290                pb_log(0,"WARNING: Killing the process ($vmssh) using port $void (previous failed VM ?)\n");
3291                kill 15,$vmssh;
3292                # Let it time to exit
3293                sleep 5;
3294            }
3295            pb_system("$cmd &","Launching the VM $vmm");
3296            # Using system allows to kill it externaly if needed,sosupport that in the call
3297            pb_system("sleep $ENV{'PBVMTMOUT'}","Waiting $ENV{'PBVMTMOUT'} s for VM $v to come up","mayfail");
3298            ($vpid,$void) = pb_check_ps($tmpcmd,$vmm);
3299            pb_log(0,"VM $vmm launched (pid $vpid)\n");
3300        }
3301    } else {
3302        pb_log(0,"Found an existing VM $vmm (pid $vexist - Port $vmmport)\n");
3303        # Set the correct port here based on what is done
3304        $pbport = $vmmport;
3305        $vpid = $vexist;
3306    }
3307    pb_log(2,"DEBUG: pb_launchv returns ($vexist,$vpid)\n");
3308
3309    # After the VM is launched, then setup the SSH access if not in creation phase
3310    if ($pbstep != 0) {
3311        if (defined $pbport) {
3312            pb_ssh_setup($vtype,$pbport);
3313        } else {
3314            pb_ssh_setup($vtype,$nport);
3315        }
3316    }
3317   
3318    # Skip that VM if something went wrong
3319    return($vexist,$vpid) if (($vpid == 0) && ($vexist == 0));
3320   
3321    # If force stopping the VM then reset vexist
3322    if ($pbforce == 1) {
3323        $vpid = $vexist;
3324        $vexist = 0;
3325    }
3326} elsif ($vtype eq "ve") {
3327    # Force the creation of the VE and no snapshot usable
3328    $vetype = pb_ve_get_type($vetype);
3329    pb_ve_launch($v,$pbstep,$usesnap,$vetype,$pbimage);
3330    $vexist = 0;
3331    $vpid = 0;
3332} else {
3333    # Get RM context
3334    my ($ptr,$rmpath,$rmport) = pb_conf_get("rmtype","rmpath","rmport");
3335
3336    # Nothing more to do for RM. No real launch
3337    # For the moment we support the RM is already running
3338    # For ProLiant may be able to power them on if needed later on as an example.
3339    $vexist = 0;
3340    $vpid = 0;
3341   
3342    # After the RM is launched, then setup the SSH access
3343    my $nport = pb_get_port($rmport,$pbos,$vtype);
3344    pb_ssh_setup($vtype,$nport);
3345}
3346
3347# Gather all required files to send them to the VM/VE/RM
3348# and launch the build through pbscript
3349if (defined $action) {
3350    pb_log(2,"DEBUG: Before send2target, vexist: $vexist, vpid: $vpid, vtype: $vtype, v: $v, action: $action\n");
3351    pb_send2target($action,$pbscript1,$v,$vexist,$vpid,$snapme,$pbstep);
3352}
3353return($vexist,$vpid);
3354}
3355
3356# Return string for date synchro
3357sub pb_date2v {
3358
3359my $vtype = shift;
3360my $pbos = shift;
3361my $nontp = shift || 0;
3362
3363pb_log(1,"Entering pb_date2v\n");
3364# VE gets time from parent OS.
3365return "/bin/true" if ($vtype) =~ /^ve/o; 
3366
3367my ($ntp) = pb_conf_get_if($vtype."ntp");
3368my $vntp = undef;
3369my $ntpline = undef;
3370$vntp = $ntp->{$ENV{'PBPROJ'}} if (defined $ntp);
3371
3372if ((defined $vntp) && ($vntp !~ /^\s*$/)) {
3373    pb_log(2,"ntp server is $vntp\n");
3374    # ntp command depends on pbos
3375    my $vntpcmd = pb_distro_get_param($pbos,pb_conf_get("oscmdntp"));
3376    $ntpline = "sudo $vntpcmd $vntp";
3377}
3378# Force new date to be in the future compared to the date
3379# of the host by adding 1 minute
3380my @date=pb_get_date();
3381$date[1]++;
3382my $upddate = strftime("%m%d%H%M%Y", @date);
3383my $dateline = "sudo /bin/date $upddate";
3384if ((defined $ntpline) && ($nontp == 0)) {
3385    pb_log(1,"pb_date2v returns $ntpline\n");
3386    return($ntpline);
3387} else {
3388    pb_log(1,"pb_date2v returns $dateline\n");
3389    return($dateline);
3390}
3391}
3392
3393sub pb_build2v {
3394
3395my $vtype = shift;
3396my $action = shift;     #build, test or prep
3397my $pbforce=shift;      # Force stop of VM. Default not.
3398
3399$action = "build" if (not defined $action);
3400$pbforce = 0 if (not defined $pbforce);
3401
3402my $pbstep;
3403
3404$vetype = pb_ve_get_type($vetype);
3405if ($action eq "prep") {
3406    $pbstep = 2;
3407} else {
3408    $pbstep = 3;
3409    # Need to be traversed by others if using containers
3410    if (($vtype eq "ve") && ($vetype eq "docker")) {
3411        chmod 0751,"$ENV{'PBBUILDDIR'}";
3412    }
3413}
3414
3415pb_log(1,"Entering in pb_build2v ($action in $vtype) \n");
3416
3417# We prepare scripts for all V of this vtype to allow sharing that part
3418my ($v,$all) = pb_get2v($vtype);
3419
3420# Send tar files when we do a global generation
3421pb_build2ssh() if (($all == 1) && ($action eq "build"));
3422
3423# Adapt // mode to memory size
3424$pbparallel = pb_set_parallel($vtype);
3425
3426my ($vexist,$vpid) = (undef,undef);
3427my $pm;
3428my $all_ok = 1;
3429if (defined $pbparallel) {
3430    $pm = new Parallel::ForkManager($pbparallel);
3431
3432    # Set which port the VM/RM will use to communicate
3433    $pm->run_on_start(\&pb_set_port);
3434    $pm->run_on_finish(sub { my ($pid, $code, $id, $signal, $dump) = @_;
3435                            unless ($code == 0 && $signal == 0 && $dump == 0) {
3436                                $all_ok = 0;
3437                                pb_log(0,"ERROR: pid $pid failed\n");
3438                            }
3439                        });
3440}
3441
3442my $counter = 0;
3443foreach my $v (@$v) {
3444    $counter++;
3445    $pbscript{$v} = "$ENV{'PBDESTDIR'}/pb$action-$v";
3446    # Modulo 2 * pbparallel (to avoid synchronization problems)
3447    $counter = 1 if ((defined $pbparallel) && ($counter > 2 * $pbparallel));
3448    $pm->start($counter) and next if (defined $pbparallel);
3449    #
3450    # Prepare the script to be executed on the VM/VE/RM
3451    open(SCRIPT,"> $pbscript{$v}") || die "Unable to create $pbscript{$v}";
3452    print SCRIPT "#!/bin/bash\n";
3453
3454    # Transmit the verbosity level to the virtual env/mach.
3455    my $verbose = "";
3456    my $i = 0;                          # minimal debug level
3457    while ($i lt $pbdebug) {
3458        $verbose .= "-v ";
3459        $i++;
3460    }
3461    print SCRIPT "set -e\n" if ($Global::pb_stop_on_error);
3462    # Activate script verbosity if at least 2 for pbdebug
3463    print SCRIPT "set -x\n" if ($i gt 1);
3464    # Quiet if asked to be so on the original system
3465    $verbose = "-q" if ($pbdebug eq -1);
3466
3467    print SCRIPT "echo ... Execution needed\n";
3468    print SCRIPT "echo ==== Start of script for $vtype $v =====\n";
3469    print SCRIPT "# This is in directory pbdelivery\n";
3470    print SCRIPT "# Setup the variables required for building\n";
3471    print SCRIPT "export PBPROJ=$ENV{'PBPROJ'}\n";
3472
3473    if (($action eq "build") || ($action eq "prep")) {
3474        print SCRIPT "# Preparation for pb\n";
3475        print SCRIPT "rm -f \$HOME/.pbrc.yml\n";
3476        print SCRIPT "rm -f \$HOME/.pbrc\n";
3477        print SCRIPT "cp .pbrc.yml \$HOME\n";
3478        # TODO: Why ?
3479        print SCRIPT "cd ..\n";
3480    }
3481
3482    # VE needs a good /proc, tolerate one being potentially left around after a failure
3483    if (($vtype eq "ve") && ($vetype ne "docker")) {
3484        print SCRIPT "[ -d /proc/1 ] || sudo /bin/mount -t proc /proc /proc\n";
3485    }
3486
3487    # Get distro context
3488    my $pbos = pb_distro_get_context($v);
3489   
3490    my $ntpline = pb_date2v($vtype,$pbos);
3491    print SCRIPT "# Time sync\n";
3492    print SCRIPT "echo setting up date with $ntpline\n";
3493    print SCRIPT "$ntpline\n";
3494    print SCRIPT "if [ \$\? -ne 0 ]; then\n";
3495    # Force if bad status a trick using date
3496    $ntpline = pb_date2v($vtype,$pbos, 1);
3497    print SCRIPT "  $ntpline\n";
3498    print SCRIPT "fi\n";
3499    # Use potential local proxy declaration in case we need it to download repo, pkgs, ...
3500    if (defined $ENV{'http_proxy'}) {
3501        print SCRIPT "export http_proxy=\"$ENV{'http_proxy'}\"\n";
3502    }
3503    if (defined $ENV{'https_proxy'}) {
3504        print SCRIPT "export https_proxy=\"$ENV{'https_proxy'}\"\n";
3505    }
3506    if (defined $ENV{'ftp_proxy'}) {
3507        print SCRIPT "export ftp_proxy=\"$ENV{'ftp_proxy'}\"\n";
3508    }
3509
3510    # Get list of packages to build/test and get some ENV vars as well
3511    my $ptr = pb_get_pkg();
3512    @pkgs = @$ptr;
3513    my $p = join(' ',@pkgs) if (@pkgs);
3514    print SCRIPT "export PBPROJVER=$ENV{'PBPROJVER'}\n";
3515    print SCRIPT "export PBPROJTAG=$ENV{'PBPROJTAG'}\n";
3516    print SCRIPT "export PBPACKAGER=\"$ENV{'PBPACKAGER'}\"\n";
3517    # TODO: setup apk pkg/keys here
3518
3519    # We may need to do some other tasks before building. Read a script here to finish setup
3520    if (-x "$ENV{'PBTMP'}/pb$vtype".".pre") {
3521        print SCRIPT "# Special pre-instructions to be launched\n";
3522        print SCRIPT pb_get_content("$ENV{'PBTMP'}/pb$vtype".".pre");
3523    }
3524
3525    if (-x "$ENV{'PBTMP'}/pb$vtype"."$action.pre") {
3526        print SCRIPT "# Special pre-$action instructions to be launched\n";
3527        print SCRIPT pb_get_content("$ENV{'PBTMP'}/pb$vtype"."$action.pre");
3528    }
3529
3530    print SCRIPT "# $action\n";
3531    print SCRIPT "echo $action"."ing packages on $vtype...\n";
3532
3533    if (($action eq "test") && (! -x "$ENV{'PBDESTDIR'}/pbtest")) {
3534        confess "No test script ($ENV{'PBDESTDIR'}/pbtest) found when in test mode. Aborting ...";
3535    }
3536    print SCRIPT "pb --version\n";
3537    my $act = "$action"."2pkg";
3538    $act = "build2prep" if ($action eq "prep");
3539    print SCRIPT "pb $verbose -p $ENV{'PBPROJ'} $act $p\n";
3540
3541    if (($vtype eq "ve") && ($vetype ne "docker")) {
3542        print SCRIPT "sudo /bin/umount /proc\n";
3543    }
3544
3545    # We may need to do some other tasks after building. Read a script here to exit properly
3546    if (-x "$ENV{'PBTMP'}/pb$vtype"."$action.post") {
3547        print SCRIPT "# Special post-$action instructions to be launched\n";
3548        print SCRIPT pb_get_content("$ENV{'PBTMP'}/pb$vtype"."$action.post");
3549    }
3550
3551    if (-x "$ENV{'PBTMP'}/pb$vtype".".post") {
3552        print SCRIPT "# Special post-instructions to be launched\n";
3553        print SCRIPT pb_get_content("$ENV{'PBTMP'}/pb$vtype".".post");
3554    }
3555
3556    print SCRIPT "echo ==== End of script $$ for $vtype $v =====\n";
3557    close(SCRIPT);
3558    chmod 0755,"$pbscript{$v}";
3559   
3560    $pm->finish if (defined $pbparallel);
3561}
3562$pm->wait_all_children if (defined $pbparallel);
3563die "Aborting, one or more of the children failed." if ((not $all_ok) && ($Global::pb_stop_on_error));
3564
3565# Launch the VM/VE/RM
3566# docker image to use if needed
3567# Gather all required files to send them to the VM/VE
3568# and launch the build through pbscript
3569pb_parallel_launchv(\%pbscript,$vtype,uc($vtype).$action,$pbstep,$pbimage,$pbforce);
3570}
3571
3572
3573sub pb_clean {
3574
3575    my $sleep=10;
3576    die "Unable to get env var PBDESTDIR" if (not defined $ENV{'PBDESTDIR'});
3577    die "Unable to get env var PBBUILDDIR" if (not defined $ENV{'PBBUILDDIR'});
3578    pb_log(0,"We will now wait $sleep s before removing both directories\n$ENV{'PBDESTDIR'} and $ENV{'PBBUILDDIR'}.\nPlease break me if this is wrong\n");
3579    sleep $sleep;
3580    pb_rm_rf($ENV{'PBDESTDIR'});
3581    pb_rm_rf($ENV{'PBBUILDDIR'});
3582}
3583
3584sub pb_newver {
3585
3586    die "-V Version parameter needed" if ((not defined $newver) || ($newver eq ""));
3587
3588    # Need this call for PBDIR
3589    my ($scheme2,$uri) = pb_vcs_init($pbinit);
3590
3591    my ($pbconf,$pbprojurl) = pb_conf_get("pbconfurl","pbprojurl");
3592    $uri = $pbconf->{$ENV{'PBPROJ'}};
3593    my ($scheme, $account, $host, $port, $path) = pb_get_uri($uri);
3594
3595    # Checking CMS repositories status
3596    ($scheme2, $account, $host, $port, $path) = pb_get_uri($pbprojurl->{$ENV{'PBPROJ'}});
3597
3598    if ($scheme !~ /^svn/) {
3599        die "Only SVN is supported at the moment";
3600    }
3601
3602    my $res = pb_vcs_isdiff($scheme,$ENV{'PBROOTDIR'});
3603    die "ERROR: No differences accepted in CMS for $ENV{'PBROOTDIR'} before creating a new version" if ($res != 0);
3604
3605    $res = pb_vcs_isdiff($scheme2,$ENV{'PBDIR'});
3606    die "ERROR: No differences accepted in CMS for $ENV{'PBDIR'} before creating a new version" if ($res != 0);
3607
3608    # Tree identical between PBCONFDIR and PBROOTDIR. The delta is what
3609    # we want to get for the root of the new URL
3610
3611    my $oldver = $ENV{'PBROOTDIR'};
3612    $oldver =~ s|^$ENV{'PBCONFDIR'}||;
3613
3614    pb_log(2, "PBCONFDIR: $ENV{'PBCONFDIR'}\nPBROOTDIR: $ENV{'PBROOTDIR'}\n");
3615
3616    my $newurl = "$uri/$newver";
3617    # Should probably use projver in the old file
3618    my $oldvertxt= basename($oldver);
3619    my $newvertxt = basename($newver);
3620
3621    # Duplicate and extract project-builder part
3622    pb_log(2,"Copying $uri/$oldver to $newurl\n");
3623    pb_vcs_copy($scheme,"$uri/$oldver",$newurl);
3624    pb_log(2,"Checkout $newurl to $ENV{'PBCONFDIR'}/$newver\n");
3625    pb_vcs_up($scheme,"$ENV{'PBCONFDIR'}");
3626
3627    # Duplicate and extract project
3628    my $newurl2 = "$pbprojurl->{$ENV{'PBPROJ'}}/$newver";
3629
3630    pb_log(2,"Copying $pbprojurl->{$ENV{'PBPROJ'}}/$oldver to $newurl2\n");
3631    pb_vcs_copy($scheme2,"$pbprojurl->{$ENV{'PBPROJ'}}/$oldver",$newurl2);
3632
3633    my $tmp = $ENV{'PBDIR'};
3634    $tmp =~ s|$oldver$||;
3635    pb_log(2,"Checkout $newurl2 to $tmp/$newver\n");
3636    pb_vcs_up($scheme2,"$tmp");
3637
3638    # Update the .yml file
3639    open(FILE,"$ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml") || die "Unable to open $ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml";
3640    open(OUT,"> $ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml.new") || die "Unable to write to $ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml.new";
3641    while(<FILE>) {
3642        if (/^projver\s+$ENV{'PBPROJ'}\s*=\s*$oldvertxt$/) {
3643            s/^projver\s+$ENV{'PBPROJ'}\s*=\s*$oldvertxt$/projver $ENV{'PBPROJ'} = $newvertxt/;
3644            pb_log(0,"Changing projver from $oldvertxt to $newvertxt in $ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml\n");
3645        }
3646        if (/^testver/) {
3647            s/^testver/#testver/;
3648            pb_log(0,"Commenting testver in $ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml\n") if (/^testver/);
3649        }
3650        if (/^delivery/) {
3651            my $txt = $_;
3652            chomp($txt);
3653            pb_log(0,"Please check delivery ($txt) in $ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml\n");
3654        }
3655        print OUT $_;
3656    }
3657    close(FILE);
3658    close(OUT);
3659    rename("$ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml.new","$ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml");
3660
3661    # Checking pbcl files
3662    foreach my $f (<$ENV{'PBROOTDIR'}/*/pbcl>) {
3663        # Compute new pbcl file
3664        my $f2 = $f;
3665        $f2 =~ s|$ENV{'PBROOTDIR'}|$ENV{'PBCONFDIR'}/$newver/|;
3666        open(PBCL,$f) || die "Unable to open $f";
3667        my $foundnew = 0;
3668        while (<PBCL>) {
3669            $foundnew = 1 if (/^$newvertxt \(/);
3670        }
3671        close(PBCL);
3672        open(OUT,"> $f2") || die "Unable to write to $f2: $!";
3673        open(PBCL,$f) || die "Unable to open $f";
3674        # We skip as long as we do not reach the new version in the log (case of devel)
3675        my $skip = 1;
3676        # We let the 4 first lines
3677        my $cnt = 0;
3678        while (<PBCL>) {
3679            $skip = 0 if (/^$newvertxt \(/);
3680            if ((/^$oldvertxt \(/) && ($foundnew == 0)) {
3681                print OUT "$newvertxt ($pbdate)\n";
3682                print OUT "- TBD\n";
3683                print OUT "\n";
3684                pb_log(0,"WARNING: version $newvertxt not found in $f so added to $f2...\n") if ($foundnew == 0);
3685            }
3686            if ($cnt <= 4) {
3687                print OUT "$_" if (not /^$oldvertxt \(/);
3688            } else {
3689                print OUT "$_" if ($skip == 0);
3690            }
3691            $cnt++;
3692        }
3693        close(OUT);
3694        close(PBCL);
3695    }
3696
3697    pb_log(2,"Checkin $ENV{'PBCONFDIR'}/$newver\n");
3698    pb_vcs_checkin($scheme,"$ENV{'PBCONFDIR'}/$newver","updated to $ENV{'PBCONFDIR'}");
3699}
3700
3701#
3702# Return the list of VMs/VEs/RMs we are working on
3703# $all is a flag to know if we return all of them
3704# or only some (if all we publish also tar files in addition to pkgs
3705#
3706sub pb_get2v {
3707
3708my $vtype = shift;
3709my @v;
3710my $all = 0;
3711my $pbv = 'PBV';
3712my $vlist = $vtype."list";
3713
3714# Get VM/VE list
3715if ((not defined $ENV{$pbv}) || ($ENV{$pbv} =~ /^all$/)) {
3716    my ($ptr) = pb_conf_get($vlist);
3717    $ENV{$pbv} = $ptr->{$ENV{'PBPROJ'}};
3718    $all = 1;
3719}
3720pb_log(2,"$vtype: $ENV{$pbv}\n");
3721@v = split(/,/,$ENV{$pbv});
3722return(\@v,$all);
3723}
3724
3725# This function creates a giant script to configure a particular VM/VE/RM, it then copies the
3726# script to the target.
3727
3728# Function to create a potentialy missing pb account on the VM/VE/RM, and adds it to sudo
3729# Needs to use root account to connect to the VM/VE/RM
3730# pb will take your local public SSH key to access
3731# the pb account in the VM/VE/RM later on if needed
3732sub pb_setup2v {
3733
3734my $vtype = shift;
3735my $sbx = shift;
3736my $pbstep = 1;
3737
3738my ($vm,$all) = pb_get2v($vtype);
3739
3740# Adapt // mode to memory size
3741$pbparallel = pb_set_parallel($vtype);
3742
3743my $pm;
3744my $all_ok = 1;
3745if (defined $pbparallel) {
3746    $pm = new Parallel::ForkManager($pbparallel);
3747
3748    # Set which port the VM/RM will use to communicate
3749    $pm->run_on_start(\&pb_set_port);
3750    $pm->run_on_finish(sub { my ($pid, $code, $id, $signal, $dump) = @_;
3751                            $all_ok = 0 unless (($code == 0) && ($signal == 0) && ($dump == 0)); 
3752                        });
3753}
3754
3755my $counter = 0;
3756foreach my $v (@$vm) {
3757    # Script generated - should be before the fork - after the %pbscript hash isn't updated correctly
3758    $pbscript{$v} = "$ENV{'PBDESTDIR'}/setupv-$v";
3759
3760    $counter++;
3761    if (defined $pbparallel) {
3762        # Modulo 2 * pbparallel (to avoid synchronization problems)
3763        $counter = 1 if ($counter > 2 * $pbparallel);
3764        $pm->start($counter) and next;
3765    }
3766
3767    # Get distro context
3768    my $pbos = pb_distro_get_context($v);
3769   
3770    # Deal with date sync.
3771    my $ntpline = pb_date2v($vtype,$pbos);
3772
3773    # Name of the account to deal with for VM/VE/RM
3774    # Do not use the one passed potentially with -a
3775    my ($pbac) = pb_conf_get($vtype."login");
3776    confess "No $vtype"."login defined in your project yml conf file" if ((not defined $pbac) || (not defined $pbac->{$ENV{'PBPROJ'}}));
3777
3778    my $key = "";
3779
3780    # Prepare the script to be executed on the VM/VE/RM
3781    # in $ENV{'PBDESTDIR'}/setupv
3782
3783    pb_log(2,"Preparing setup script for $v in $pbscript{$v}\n");
3784    open(SCRIPT,"> $pbscript{$v}") || die "Unable to create $pbscript{$v}";
3785   
3786    print SCRIPT << 'EOF';
3787#!/usr/bin/perl -w
3788
3789use strict;
3790use File::Copy;
3791
3792# We should not need in this script more functions than what is provided
3793# by Base, Conf and Distribution to avoid problems at exec time.
3794# They are appended at the end.
3795
3796# Define mandatory global vars
3797EOF
3798    my $ppref = "our";
3799    $ppref = "my" if (($pbos->{'name'} =~ /^redhat/) && ($pbos->{'version'} =~ /^6/));
3800    print SCRIPT << "EOF";
3801$ppref \$pbdebug = $pbdebug;
3802$ppref \$pbLOG;
3803$ppref \$pbsynmsg = "pbscript";
3804$ppref \$pbdisplaytype = "text";
3805$ppref \$pblocale = "";
3806$ppref \$PBCONFVER = $projectbuilderconfver;
3807EOF
3808    print SCRIPT << 'EOF';
3809pb_log_init($pbdebug, $pbLOG);
3810EOF
3811    print SCRIPT << "EOF";
3812\$Global::pb_stop_on_error = $Global::pb_stop_on_error;
3813pb_temp_init($pbkeep);
3814pb_conf_init("$ENV{'PBPROJ'}");
3815
3816EOF
3817
3818    if ($vtype eq "ve") {
3819        $vetype = pb_ve_get_type($vetype);
3820        if ($vetype ne "docker") {
3821            print SCRIPT << "EOF";
3822# For VE we first need to mount some FS
3823pb_system("mount -t proc /proc /proc") unless (-d "/proc/$$");
3824EOF
3825        }
3826        print SCRIPT << "EOF";
3827# For VE we need a good null dev
3828# Except for RedHat 6.2 where it's good and doesn't like being recreated
3829EOF
3830        print SCRIPT "pb_system('rm -f /dev/null; mknod /dev/null c 1 3; chmod 777 /dev/null');\n" unless (($pbos->{'name'} =~ /^redhat/) && ($pbos->{'version'} =~ /^6/));
3831    } elsif ($vtype =~ /(v|r)m/) {
3832        # Nothing to do now.
3833        my $useless = 0;
3834    } else {
3835        die "Unknown virtual type $vtype";
3836    }
3837
3838    print SCRIPT << 'EOF';
3839
3840# Adds $pbac->{$ENV{'PBPROJ'}} as an account if needed
3841#
3842my $file="/etc/passwd";
3843open(PBFILE,$file) || die "Unable to open $file";
3844my $found = 0;
3845while (<PBFILE>) {
3846EOF
3847    print SCRIPT << "EOF";
3848    \$found = 1 if (/^$pbac->{$ENV{'PBPROJ'}}:/);
3849EOF
3850
3851# TODO: use an external parameter
3852my $home = "/home";
3853# Solaris doesn't like that we use /home
3854$home = "/export/home" if ($pbos->{'type'} eq "pkg");
3855
3856    print SCRIPT << "EOF";
3857}
3858close(PBFILE);
3859
3860if ( \$found == 0 ) {
3861    if ( ! -d "$home" ) {
3862        pb_mkdir_p("$home");
3863    }
3864EOF
3865    # TODO: Level of portability of these cmds ? Critical now for RM
3866    # TODO: use an external parameter
3867    print SCRIPT << "EOF";
3868    # Most Linux distros
3869    if ( -x "/usr/sbin/groupadd" ) {
3870        pb_system("/usr/sbin/groupadd $pbac->{$ENV{'PBPROJ'}}","Adding group $pbac->{$ENV{'PBPROJ'}}");
3871    # Alpine with busybox
3872    } elsif ( -x "/usr/sbin/addgroup" ) {
3873        pb_system("/usr/sbin/addgroup $pbac->{$ENV{'PBPROJ'}}","Adding group $pbac->{$ENV{'PBPROJ'}}");
3874    } else {
3875        die "Unable to find a command to add a group on this distribution";
3876    }
3877    # Most Linux distros
3878    if ( -x "/usr/sbin/useradd" ) {
3879        pb_system("/usr/sbin/useradd -g $pbac->{$ENV{'PBPROJ'}} -m -d $home/$pbac->{$ENV{'PBPROJ'}} $pbac->{$ENV{'PBPROJ'}}","Adding user $pbac->{$ENV{'PBPROJ'}} (group $pbac->{$ENV{'PBPROJ'}} - home $home/$pbac->{$ENV{'PBPROJ'}})");
3880    # Alpine with busybox
3881    } elsif ( -x "/usr/sbin/adduser" ) {
3882        pb_system("/usr/sbin/adduser -G $pbac->{$ENV{'PBPROJ'}} -h $home/$pbac->{$ENV{'PBPROJ'}} -D $pbac->{$ENV{'PBPROJ'}}","Adding user $pbac->{$ENV{'PBPROJ'}} (group $pbac->{$ENV{'PBPROJ'}} - home $home/$pbac->{$ENV{'PBPROJ'}})");
3883        pb_system("/usr/sbin/addgroup $pbac->{$ENV{'PBPROJ'}} abuild","Adding $pbac->{$ENV{'PBPROJ'}} to group abuild");
3884    } else {
3885        die "Unable to find a command to add a user on this distribution";
3886    }
3887}
3888EOF
3889
3890    # Copy the content of our local conf file to the VM/VE/RM
3891    my $content = pb_get_content(pb_distro_conffile());
3892    print SCRIPT << "EOF";
3893    #
3894    # Create a temporary local conf file for distribution support
3895    # This is created here before its use later. Its place is hardcoded, so no choice for the path
3896    #
3897    my \$tempconf = pb_distro_conffile();
3898    pb_mkdir_p(dirname(\$tempconf));
3899    open(CONF,"> \$tempconf") || die "Unable to create \$tempconf";
3900    print CONF q{$content};
3901    close(CONF);
3902EOF
3903
3904    if ($vtype =~ /(v|r)m/) {
3905        print SCRIPT << "EOF";
3906# allow ssh entry to build
3907#
3908mkdir "$home/$pbac->{$ENV{'PBPROJ'}}/.ssh",0700;
3909# Allow those accessing root to access the build account
3910copy("\$ENV{'HOME'}/.ssh/authorized_keys","$home/$pbac->{$ENV{'PBPROJ'}}/.ssh/authorized_keys");
3911chmod 0600,".ssh/authorized_keys";
3912pb_system("chown -R $pbac->{$ENV{'PBPROJ'}}:$pbac->{$ENV{'PBPROJ'}} $home/$pbac->{$ENV{'PBPROJ'}}","Finish setting up the account env for $pbac->{$ENV{'PBPROJ'}}");
3913
3914EOF
3915}
3916    print SCRIPT << 'EOF';
3917# No passwd for build account only keys
3918$file="/etc/shadow";
3919if (-f $file) {
3920    open(PBFILE,$file) || die "Unable to open $file";
3921    open(PBOUT,"> $file.new") || die "Unable to open $file.new";
3922    while (<PBFILE>) {
3923EOF
3924    print SCRIPT << "EOF";
3925        s/^$pbac->{$ENV{'PBPROJ'}}:\!\!:/$pbac->{$ENV{'PBPROJ'}}:*:/;
3926        s/^$pbac->{$ENV{'PBPROJ'}}:\!:/$pbac->{$ENV{'PBPROJ'}}:*:/; #SLES 9 e.g.
3927        s/^$pbac->{$ENV{'PBPROJ'}}:\\*LK\\*:/$pbac->{$ENV{'PBPROJ'}}:NP:/;  #Solaris e.g.
3928EOF
3929        print SCRIPT << 'EOF';
3930        print PBOUT $_;
3931    }
3932    close(PBFILE);
3933    close(PBOUT);
3934    rename("$file.new",$file);
3935    chmod 0640,$file;
3936    }
3937
3938# TODO: systemd support
3939# Keep the VM in text mode
3940$file="/etc/inittab";
3941if (-f $file) {
3942    open(PBFILE,$file) || die "Unable to open $file";
3943    open(PBOUT,"> $file.new") || die "Unable to open $file.new";
3944    while (<PBFILE>) {
3945        s/^(..):5:initdefault:$/$1:3:initdefault:/;
3946        print PBOUT $_;
3947    }
3948    close(PBFILE);
3949    close(PBOUT);
3950    rename("$file.new",$file);
3951    chmod 0640,$file;
3952}
3953
3954# pb has to be added to portage group on gentoo
3955
3956# We need to have that pb_distro_get_context function
3957# Get it from ProjectBuilder::Distribution
3958# And we now need the conf file required for this to work created above
3959
3960my $pbos = pb_distro_get_context(); 
3961print "distro tuple: ".Dumper($pbos)."\n";
3962
3963# Adapt sudoers
3964# sudo is not default on Solaris and needs to be installed first
3965# from http://www.sunfreeware.com/programlistsparc10.html#sudo
3966if ($pbos->{'type'} eq "pkg") {
3967    $file="/usr/local/etc/sudoers";
3968} else {
3969    $file="/etc/sudoers";
3970}
3971open(PBFILE,$file) || die "Unable to open $file";
3972open(PBOUT,"> $file.new") || die "Unable to open $file.new";
3973while (<PBFILE>) {
3974EOF
3975    # Skip what will be generated
3976    print SCRIPT << "EOF";
3977    next if (/^$pbac->{$ENV{'PBPROJ'}}\\s+/);
3978    next if (/^Defaults:$pbac->{$ENV{'PBPROJ'}}\\s+/);
3979    next if (/^Defaults:root\\s+\!requiretty/);
3980    next if (/^Defaults:root\\s+env_keep/);
3981EOF
3982    print SCRIPT << 'EOF';
3983    s/Defaults\s+requiretty/# disable Defaults: requiretty/;
3984    print PBOUT $_;
3985}
3986close(PBFILE);
3987EOF
3988    print SCRIPT << "EOF";
3989# Some distro force requiretty at compile time, so disable here
3990print PBOUT "Defaults:$pbac->{$ENV{'PBPROJ'}} !requiretty\\n";
3991print PBOUT "Defaults:root !requiretty\\n";
3992EOF
3993    # RH 6.2 sudo doesn't support env_keep
3994    if (($pbos->{'name'} ne "redhat") && ($pbos->{'version'} ne "6.2")) {
3995        print SCRIPT << "EOF";
3996# Keep proxy configuration while using sudo
3997print PBOUT "Defaults:$pbac->{$ENV{'PBPROJ'}}    env_keep += \\\"http_proxy https_proxy ftp_proxy\\\"\\n";
3998print PBOUT "Defaults:root    env_keep += \\\"http_proxy https_proxy ftp_proxy\\\"\\n";
3999EOF
4000    }
4001    # Try to restrict security to what is really needed
4002    if ($vtype =~ /^vm/) {
4003        my $hpath = pb_distro_get_param($pbos,pb_conf_get("ospathcmd-halt"));
4004        my @sudocmds = pb_get_sudocmds($pbos,$ntpline,"sudo $hpath","sudo /bin/date");
4005        print SCRIPT << "EOF";
4006# This is needed in order to be able on VM to halt the machine from the $pbac->{$ENV{'PBPROJ'}} account at least
4007# Build account $pbac->{$ENV{'PBPROJ'}} in VM also needs to setup date and install deps.
4008# Nothing else should be needed
4009EOF
4010        foreach my $c (@sudocmds) {
4011            print SCRIPT "print PBOUT \"$pbac->{$ENV{'PBPROJ'}}   ALL = NOPASSWD: $c\\n\";\n";
4012        }
4013    } elsif ($vtype =~ /^rm/) {
4014        my @sudocmds = pb_get_sudocmds($pbos,$ntpline);
4015        print SCRIPT << "EOF";
4016# Build account $pbac->{$ENV{'PBPROJ'}} in RM only needs to setup date and install deps if needed each time
4017EOF
4018        foreach my $c (@sudocmds) {
4019            print SCRIPT "print PBOUT \"$pbac->{$ENV{'PBPROJ'}}   ALL = NOPASSWD: $c\\n\";\n";
4020        }
4021    } else {
4022        print SCRIPT << "EOF";
4023# Build account $pbac->{$ENV{'PBPROJ'}} for VE needs to do a lot in the host (and chroot), so allow without restriction for now
4024print PBOUT "$pbac->{$ENV{'PBPROJ'}}   ALL=(ALL) NOPASSWD:ALL\n";
4025EOF
4026    }
4027    my $sudomode = pb_distro_get_param($pbos,pb_conf_get("ossudoersmode"));
4028    print SCRIPT << 'EOF';
4029close(PBOUT);
4030rename("$file.new",$file);
4031EOF
4032    print SCRIPT << "EOF";
4033chmod 0$sudomode,\$file;
4034
4035EOF
4036
4037    if ($vtype =~ /(v|r)m/) {
4038        # Sync date
4039        # do it after sudoers is setup
4040        print SCRIPT "my \$res = pb_system(\"$ntpline\",\"Updating time with $ntpline\",\"mayfailverbose\");\n";
4041        print SCRIPT "if (\$res != 0) {\n";
4042        # try again with another method
4043        $ntpline = pb_date2v($vtype,$pbos,1);
4044        print SCRIPT "  pb_system(\"$ntpline\",\"Updating time with $ntpline\",\"mayfailverbose\");\n";
4045        print SCRIPT "}\n";
4046    }
4047
4048    my ($addrepo) = pb_conf_read("$ENV{'PBDESTDIR'}/pbrc.yml","addrepo");
4049    my $pbrepo = pb_distro_get_param($pbos,$addrepo);
4050    print SCRIPT << 'EOF';
4051   
4052# We may need a proxy configuration. Get it from the local env
4053pb_apply_conf_proxy($pbos);
4054
4055# First install all required packages
4056pb_system("dnf clean all","Cleaning dnf env","mayfail") if (($pbos->{'name'} eq "fedora") && ($pbos->{'version'} >= 22));
4057pb_system("yum clean all","Cleaning yum env","mayfail") if ((($pbos->{'name'} eq "fedora") && ($pbos->{'version'} < 22)) || ($pbos->{'name'} eq "asianux") || ($pbos->{'name'} eq "rhel"));
4058my ($ospkgdep) = pb_conf_get_if("ospkgdep");
4059
4060my $pkgdep = pb_distro_get_param($pbos,$ospkgdep);
4061my $forcerepo;
4062EOF
4063    print SCRIPT "\$forcerepo = pb_distro_setuprepo_gen(\$pbos,\"$pbrepo\");\n" if ((defined $pbrepo) && ($pbrepo ne ""));
4064    print SCRIPT << 'EOF';
4065pb_distro_installdeps(undef,$pbos,$forcerepo,pb_distro_only_deps_needed($pbos,join(' ',split(/,/,$pkgdep))));
4066
4067EOF
4068    my $itype = pb_distro_get_param($pbos,pb_conf_get("pbinstalltype"));
4069    # Install from sandbox mean a file base install
4070    $itype = "file" if (defined $sbx);
4071    if ($itype =~ /^file/) {
4072        my $cmdget;
4073        my ($pbver1,$pbver2,$pbextdir);
4074        my ($tmp1,$tmp2);
4075        if (defined $sbx) {
4076            # Install from sandbox mean using the result of the just passed sbx2build command
4077            # Get content saved in cms2build
4078            my ($pkg) = pb_conf_read("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml","pbpkg");
4079            $pbextdir = pb_get_extdir();
4080            die "Unable to get package list" if (not defined $pkg);
4081
4082            # We consider 2 specific packages
4083            my $vertag1 = $pkg->{"ProjectBuilder"};
4084            my $vertag2 = $pkg->{"project-builder"};
4085            # get the version of the current package - maybe different
4086            pb_log(2,"Vertag1: $vertag1\n");
4087            pb_log(2,"Vertag2: $vertag2\n");
4088            ($pbver1,$tmp1) = split(/-/,$vertag1);
4089            ($pbver2,$tmp2) = split(/-/,$vertag2);
4090            $cmdget = "mv /tmp/ProjectBuilder-$pbver1$pbextdir.tar.gz ProjectBuilder-latest.tar.gz ; mv /tmp/project-builder-$pbver2$pbextdir.tar.gz project-builder-latest.tar.gz";
4091        } else {
4092            # TODO: shouldn't we have a variable to specify the source file?
4093            # And shouldn't we be getting the one that matches the current pb version?
4094            # For the moment, just take the latest stable one
4095            my $wget = pb_check_req("wget",0);
4096            $cmdget = "$wget --passive-ftp ftp://ftp.project-builder.org/src/ProjectBuilder-latest.tar.gz; $wget --passive-ftp ftp://ftp.project-builder.org/src/project-builder-latest.tar.gz";
4097        }
4098        print SCRIPT << 'EOF';
4099# Then install manually the missing perl modules
4100my ($osperldep,$osperlver) = pb_conf_get_if("osperldep","osperlver");
4101   
4102my $perldep = pb_distro_get_param($pbos,$osperldep);
4103my $bashopt = "";
4104$bashopt = "-x" if ($pbdebug ge 1);
4105my $verbopt = "mayfail";
4106$verbopt = "mayfailverbose" if ($pbdebug ge 1);
4107foreach my $m (split(/,/,$perldep)) {
4108    # Skip empty deps
4109    next if ($m =~ /^\s*$/);
4110    my $dir = $m;
4111    $dir =~ s/-.*//;
4112    pb_system("echo \"rm -rf $m* ; wget http://search.cpan.org/CPAN/modules/by-module/$dir/$m-$osperlver->{$m}.tar.gz ; gzip -cd $m-$osperlver->{$m}.tar.gz | tar xf - ; cd $m* ; if [ -f Build.PL ]; then perl Build.PL; ./Build ; ./Build install ; else perl Makefile.PL; make ; make install ; fi; cd .. ; rm -rf $m*\" | bash -e $bashopt" ,"Installing perl module $m-$osperlver->{$m}",$verbopt);
4113}
4114EOF
4115        print SCRIPT << 'EOF';
4116pb_system("rm -rf ProjectBuilder-* ; rm -rf project-builder-* ; rm -rf `perl -V:installvendorlib  | awk -F\"'\" '{print \$2}'`/ProjectBuilder ;
4117EOF
4118        print SCRIPT " $cmdget ; ";
4119        print SCRIPT << "EOF"
4120gzip -cd ProjectBuilder-latest.tar.gz | tar xf - ; cd ProjectBuilder-$pbver1$pbextdir ; perl Makefile.PL ; make ; make install ; cd .. ; rm -rf ProjectBuilder-* ; gzip -cd project-builder-latest.tar.gz | tar xf - ; cd project-builder-$pbver2$pbextdir ; perl Makefile.PL ; make ; make install ; cd .. ; rm -rf project-builder-* ;","Building Project-Builder");
4121EOF
4122    } elsif ($itype =~ /^pkg/) {
4123        # pkg based install. We need to point to the project-builder.org repository
4124        print SCRIPT << 'EOF';
4125my $pkgforpb = pb_distro_get_param($pbos,pb_conf_get_if("ospkg"));
4126my $forcerepo = pb_distro_setuposrepo($pbos);
4127pb_log(1,"Forcing full package install with forcerepo\n") if (defined $forcerepo);
4128pb_distro_installdeps(undef,$pbos,$forcerepo,pb_distro_only_deps_needed($pbos,join(' ',split(/,/,$pkgforpb))));
4129EOF
4130    } else {
4131        # Unknown install type
4132        die("Unknown install type $itype->{$ENV{'PBPROJ'}} for param pbinstalltype");
4133    }
4134    print SCRIPT << 'EOF';
4135pb_system("pbdistrocheck",undef,"verbose");
4136pb_system("pb --version",undef,"verbose");
4137EOF
4138    if ($vtype eq "ve"){
4139        if ($vetype ne "docker") {
4140            print SCRIPT << 'EOF';
4141# For VE we need to umount some FS at the end
4142
4143pb_system("umount /proc","Unmounting /proc","mayfail");
4144EOF
4145        }
4146        print SCRIPT << 'EOF';
4147# Create a basic network file if not already there
4148
4149my $nf="/etc/sysconfig/network";
4150if ((! -f $nf) && ($pbos->{'type'} eq "rpm") && ($pbos->{'family'} ne "novell")) {
4151    open(NF,"> $nf") || die "Unable to create $nf";
4152    print NF "NETWORKING=yes\n";
4153    print NF "HOSTNAME=localhost\n";
4154    close(NF);
4155}
4156chmod 0755,$nf;
4157EOF
4158    }
4159
4160    # Adds pb_distro_get_context and all functions needed from ProjectBuilder::Distribution, Conf and Base
4161    foreach my $m ("ProjectBuilder/Base.pm","ProjectBuilder/Distribution.pm","ProjectBuilder/Conf.pm") {
4162        foreach my $d (@INC) {
4163            my $f = "$d/$m";
4164            if (-f "$f") {
4165                open(PBD,"$f") || die "Unable to open $f: $!";
4166                while (<PBD>) {
4167                    next if (/^package/);
4168                    next if (/^use Exporter/);
4169                    next if (/^use ProjectBuilder::/);
4170                    next if (/^our /);
4171                    print SCRIPT $_;
4172                }
4173                close(PBD);
4174                # We just need the first one of each file wrt @INC - hopefully that's the right one.
4175                last;
4176            }
4177        }
4178    }
4179    # Use a fake pb_version_init version here
4180    print SCRIPT << "EOF";
4181sub pb_version_init {
4182
4183return("$projectbuilderver","$projectbuilderrev","$projectbuilderconfver");
4184}
41851;
4186EOF
4187    close(SCRIPT);
4188    chmod 0755,"$pbscript{$v}";
4189
4190    $pm->finish if (defined $pbparallel);
4191}
4192$pm->wait_all_children if (defined $pbparallel);
4193die "Aborting, one or more of the children failed." if ((not $all_ok) && ($Global::pb_stop_on_error));
4194
4195# These build scripts need to be run as root and force stop of VM at end
4196$ENV{'PBACCOUNT'} = "root";
4197
4198$sbx = "" if (not defined $sbx);
4199# Launch the VM/VE/RM
4200pb_log(2,"DEBUG: before parallel launch, pbscript hash is:".Dumper(%pbscript)."\n");
4201pb_parallel_launchv(\%pbscript,$vtype,uc($vtype)."Script".$sbx,$pbstep,$pbimage,$pbforce,0,$pbsnap);
4202
4203return;
4204}
4205
4206# Function to create a snapshot named 'pb' for VMs and a compressed tar for VEs
4207sub pb_snap2v {
4208
4209my $vtype = shift;
4210my $pbstep = 2;
4211
4212my ($vm,$all) = pb_get2v($vtype);
4213
4214# Script generated
4215my $pbscript = "$ENV{'PBDESTDIR'}/snapv.$$";
4216
4217my ($pbac) = pb_conf_get($vtype."login");
4218
4219foreach my $v (@$vm) {
4220    if ($vtype eq "ve") {
4221        # Get distro context
4222        my $pbos = pb_distro_get_context($v);
4223        my ($vepath) = pb_conf_get("vepath");
4224
4225        # Test if an existing snapshot exists and remove it if there is a VE
4226        if ((-f "$vepath->{$ENV{'PBPROJ'}}/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}.tar.gz") &&
4227            (! -d "$vepath->{$ENV{'PBPROJ'}}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}")) {
4228                pb_system("sudo rm -f $vepath->{$ENV{'PBPROJ'}}/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}.tar.gz","Removing previous snapshot $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}.tar.gz");
4229        }
4230    }
4231
4232    # Prepare the script to be executed on the VM/VE
4233    open(SCRIPT,"> $pbscript") || die "Unable to create $pbscript";
4234    print SCRIPT << 'EOF';
4235    #!/bin/bash
4236    sleep 2
4237EOF
4238    close(SCRIPT);
4239    chmod 0755,"$pbscript";
4240
4241    # Force shutdown of VM/VE
4242    # Force snapshot of VM/VE
4243    pb_script2v($pbscript,$vtype,$pbstep,1,$v,1);
4244}
4245return;
4246}
4247
4248# Function to update VMs/VEs/RMs with the latest distribution content
4249sub pb_update2v {
4250
4251my $vtype = shift;
4252my $pbstep = 2;
4253
4254my ($vm,$all) = pb_get2v($vtype);
4255
4256# Script generated
4257my $pbscript = "$ENV{'PBDESTDIR'}/updatev.$$";
4258
4259my ($pbac) = pb_conf_get($vtype."login");
4260
4261foreach my $v (@$vm) {
4262    # Get distro context
4263    my $pbos = pb_distro_get_context($v);
4264
4265    # Prepare the script to be executed on the VM/VE/RM
4266    # in $ENV{'PBDESTDIR'}/updatev
4267    open(SCRIPT,"> $pbscript") || die "Unable to create $pbscript";
4268   
4269    print SCRIPT << 'EOF';
4270    #!/bin/bash
4271    sleep 2
4272EOF
4273    # VE needs a good /proc
4274    $vetype = pb_ve_get_type($vetype);
4275    if (($vtype eq "ve") && ($vetype ne "docker")) {
4276        print SCRIPT "sudo /bin/mount -t proc /proc /proc\n";
4277    }
4278    print SCRIPT "$pbos->{'update'}\n";
4279    if (($vtype eq "ve") && ($vetype ne "docker")) {
4280        print SCRIPT "sudo /bin/umount /proc\n";
4281    }
4282    close(SCRIPT);
4283    chmod 0755,"$pbscript";
4284
4285    # Force shutdown of VM except
4286    pb_script2v($pbscript,$vtype,$pbstep,1,$v);
4287}
4288return;
4289}
4290
4291sub pb_announce {
4292
4293    my $antype = shift;
4294
4295    # Get all required parameters
4296    my ($pbpackager,$pbrepo) = pb_conf_get("pbpackager","pbrepo");
4297    my ($pkgv, $pkgt, $testver) = pb_conf_get_if("pkgver","pkgtag","testver");
4298    if (((not defined $testver) || (not defined $testver->{$ENV{'PBPROJ'}}) || ($testver->{$ENV{'PBPROJ'}} !~ /true/i)) && ($antype eq "Clean")) {
4299        # We just clean for test versions
4300        pb_log(0,"Unable to clean SSH repository for non testver version\n");
4301        return;
4302    }
4303    my $pkg = pb_cms_get_pkg($defpkgdir,$extpkgdir);
4304    my @pkgs = @$pkg;
4305    my %pkgs;
4306    my $first = 0;
4307
4308    # Get all distros concerned
4309    my $pbos = pb_distro_get_context();
4310    my $distrolist = pb_get_distros($pbos,undef);
4311    my %dl;
4312    my %theorlist;
4313    my %archlist;
4314    foreach my $d (split(/,/,$distrolist)) {
4315        my ($d1,$d2,$d3) = split(/-/,$d);
4316        $dl{$d1}++;
4317    }
4318
4319    # Command to find packages on repo
4320    my $findstr = "find ".join(" ",keys %dl)." ";
4321    my $srcstr = "";
4322    # Generated announce files
4323    my @files;
4324
4325    foreach my $pbpkg (@pkgs) {
4326        if ($first != 0) {
4327            $findstr .= "-o ";
4328        }
4329        $first++;
4330        if ((defined $pkgv) && (defined $pkgv->{$pbpkg})) {
4331            $pbver = $pkgv->{$pbpkg};
4332        } else {
4333            $pbver = $ENV{'PBPROJVER'};
4334        }
4335        if ((defined $pkgt) && (defined $pkgt->{$pbpkg})) {
4336            $pbtag = $pkgt->{$pbpkg};
4337        } else {
4338            $pbtag = $ENV{'PBPROJTAG'};
4339        }
4340
4341        # TODO: use virtual/real names here now
4342        my $pbrealpkg = $pbpkg;
4343        my $pbrealpkgrpm = pb_cms_get_real_pkg($pbpkg,"rpm");
4344        my $pbrealpkgdeb = pb_cms_get_real_pkg($pbpkg,"deb");
4345        if ($antype eq "Clean") {
4346            # We only clean test versions anyway
4347            $pbtag = "0";
4348            my $nver = $pbver;
4349            my $ntag = "[2-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]";
4350            $pbver .= $ntag;
4351            # TODO add apk
4352            $findstr .= "-name \'$pbrealpkgrpm-$pbver-$pbtag\.*.rpm\' -o -name \'$pbrealpkgrpm-debug-$pbver-$pbtag\.*.rpm\' -o -name \'$pbrealpkgdeb"."_$pbver-$pbtag"."_*\.deb\' -o -name \'$pbrealpkgdeb"."_$pbver-$pbtag.dsc\' -o -name \'$pbrealpkgdeb"."_$pbver-$pbtag.tar.gz\' -o -name \'$pbrealpkg-$nver"."_p$ntag\.ebuild\' -o -name \'$pbrealpkg-$pbver-$pbtag*\.pkg\.gz\' -o -name \'$pbrealpkg-$pbver-$pbtag*\.sd\' ";
4353            $srcstr .= "src/$pbrealpkg-$pbver.tar.gz src/$pbrealpkg-$pbver.pbconf.tar.gz ";
4354        } else {
4355            # "Announce" and "Check"
4356            my @date=pb_get_date();
4357            # the matching is only done on packages made the same day for test version. Hopefully this is enough
4358            my $nver;
4359            if ((defined $testver) && (defined $testver->{$ENV{'PBPROJ'}}) && ($testver->{$ENV{'PBPROJ'}} =~ /true/i)) {
4360                $pbtag = "0";
4361                my $ntag .= strftime("%Y%m%d*", @date);
4362                $nver = $pbver."_p$ntag*";
4363                $pbver .= "$ntag*";
4364            } else { 
4365                $nver = $pbver."-r$pbtag";
4366            }
4367            $findstr .= "-name \'$pbrealpkgrpm-$pbver-$pbtag\.*.rpm\' -o -name \'$pbrealpkgdeb"."_$pbver-$pbtag"."_*\.deb\' -o -name \'$pbrealpkg-$nver\.ebuild\' -o -name \'$pbrealpkg-$pbver\.pkg\.gz\' -o -name \'$pbrealpkg-$pbver\.sd\' ";
4368        }
4369
4370        if ($antype eq "Announce") {
4371            my $chglog;
4372
4373            pb_vcs_init($pbinit);
4374            # Get project info on log file and generate tmp files used later on
4375            $chglog = "$ENV{'PBROOTDIR'}/$pbpkg/pbcl";
4376            $chglog = "$ENV{'PBROOTDIR'}/pbcl" if (! -f $chglog);
4377            $chglog = undef if (! -f $chglog);
4378
4379            open(OUT,"> $ENV{'PBTMP'}/$pbpkg.ann") || die "Unable to create $ENV{'PBTMP'}/$pbpkg.ann: $!";
4380            my $pb;
4381            $pb->{'realpkg'} = $pbrealpkg;
4382            $pb->{'ver'} = $pbver;
4383            $pb->{'tag'} = $pbtag;
4384            $pb->{'date'} = $pbdate;
4385            $pb->{'extdir'} = pb_get_extdir();
4386            $pb->{'chglog'} = $chglog;
4387            $pb->{'packager'} = $pbpackager;
4388            $pb->{'proj'} = $ENV{'PBPROJ'};
4389            $pb->{'repo'} = $pbrepo;
4390            $pb->{'pbos'}->{'type'} = "announce";
4391            $pb->{'pbos'}->{'suffix'} = "none";
4392            pb_changelog($pb,\*OUT,"yes");
4393            close(OUT);
4394            push(@files,"$ENV{'PBTMP'}/$pbpkg.ann");
4395        } elsif ($antype eq "Check") {
4396            # For the check we also build the theoritical complete list we should get
4397            foreach my $d (split(/,/,$distrolist)) {
4398                $pbos = pb_distro_get_context($d);
4399                if ($pbos->{'type'} eq "rpm") {
4400                    $theorlist{"$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}/$pbrealpkgrpm-$pbver-$pbtag$pbos->{'suffix'}"} = 0;
4401                } elsif ($pbos->{'type'} eq "deb") {
4402                    $theorlist{"$pbos->{'name'}/$pbos->{'version'}/$pbrealpkgdeb"."_$pbver-$pbtag"} = 0;
4403                    # TODO are we always using the last arch ?
4404                    $archlist{"$pbos->{'name'}/$pbos->{'version'}/$pbrealpkgdeb"."_$pbver-$pbtag"} = "$pbos->{'arch'}";
4405                } elsif ($pbos->{'type'} eq "ebuild") {
4406                    my $prefix = "-r";
4407                    $prefix = "_p" if ((defined $testver) && (defined $testver->{$ENV{'PBPROJ'}}) && ($testver->{$ENV{'PBPROJ'}} =~ /true/i));
4408                    $theorlist{"$pbos->{'name'}/$pbos->{'version'}/$pbrealpkg-$pbver$prefix$pbtag.ebuild"} = 0;
4409                    $archlist{"$pbos->{'name'}/$pbos->{'version'}/$pbrealpkg-$pbver$prefix$pbtag.ebuild"} = "$pbos->{'arch'}";
4410                } elsif ($pbos->{'type'} eq "apk") {
4411                    # TODO
4412                } elsif ($pbos->{'type'} eq "pkg") {
4413                    $theorlist{"$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}/$pbrealpkg-$pbver-$pbtag.pkg.gz"} = 0;
4414                } else {
4415                    pb_log(1,"No theoritical list possible for type $pbos->{'type'}\n");
4416                }
4417            }
4418        }
4419        pb_log(2,"theorlist initial: ".Dumper(%theorlist)."\n");
4420        pb_log(2,"archlist: ".Dumper(%archlist)."\n");
4421    }
4422    if ($antype eq "Announce") {
4423        $findstr .= " | grep -Ev \'src.rpm\'";
4424    } elsif ($antype eq "Clean") {
4425        $findstr .= " | xargs rm -f -v $srcstr ";
4426    }
4427
4428    # Prepare the command to run and execute it
4429    open(PBS,"> $ENV{'PBTMP'}/pbscript") || die "Unable to create $ENV{'PBTMP'}/pbscript";
4430    print PBS "#!/bin/bash\n";
4431    print PBS "set -x\n" if ($pbdebug gt 1);
4432    print PBS "$findstr | sort 2> /dev/null\n";
4433    close(PBS);
4434    chmod 0755,"$ENV{'PBTMP'}/pbscript";
4435    pb_send2target("Announce","$ENV{'PBTMP'}/pbscript");
4436
4437    my $sl = "Project $ENV{'PBPROJ'} version $ENV{'PBPROJVER'} is now available";
4438    if ($antype eq "Announce") {
4439        # Get subject line
4440        pb_log(0,"Please enter the title of your announce\n");
4441        pb_log(0,"(By default: $sl)\n");
4442        my $sl2 = <STDIN>;
4443        $sl = $sl2 if ($sl2 !~ /^$/);
4444
4445        # Prepare a template of announce
4446        open(ANN,"> $ENV{'PBTMP'}/announce.html") || die "Unable to create $ENV{'PBTMP'}/announce.html: $!";
4447        print ANN << "EOF";
4448$sl</p>
4449
4450<p>The project team is happy to announce the availability of a newest version of $ENV{'PBPROJ'} $ENV{'PBPROJVER'}. Enjoy it as usual!</p>
4451<p>
4452Now available at <a href="$pbrepo->{$ENV{'PBPROJ'}}">$pbrepo->{$ENV{'PBPROJ'}}</a>
4453</p>
4454<p>
4455EOF
4456    }
4457
4458    open(LOG,"$ENV{'PBTMP'}/system.$$.log") || die "Unable to read $ENV{'PBTMP'}/system.$$.log: $!";
4459    if ($antype eq "Announce") {
4460        my $col = 2;
4461        my $i = 1;
4462        print ANN << 'EOF';
4463<TABLE WIDTH="100%" CELLPADDING="0" CELLSPACING="0" BORDER="0">
4464<TR>
4465EOF
4466        while (<LOG>) {
4467            print ANN "<TD><A HREF=\"$pbrepo->{$ENV{'PBPROJ'}}/$_\">$_</A></TD>";
4468            $i++;
4469            if ($i > $col) {
4470                print ANN "</TR>\n<TR>";
4471                $i = 1;
4472            }
4473        }
4474    } elsif ($antype eq "Clean") {
4475        while (<LOG>) {
4476            # skip errors
4477            next if ($_ !~ /^removed /);
4478            pb_log(0,"$_");
4479        }
4480    } else {
4481        # In "Check" mode we need to compare the 2 lists (real and theoritical)
4482        while (<LOG>) {
4483            # Get package name and remove what is in extra for the theoritical list (arch at the end)
4484            chomp();
4485            # skip find errors
4486            next if (/^find:/);
4487            my $p = $_;
4488            $p =~ s/\.(i[3456]86|x86_64|noarch|src)\.rpm$//;
4489            $p =~ s/_(i[3456]86|amd64|all)\.deb$//;
4490            $p =~ s/(-0\.[0-9]{8})[0-9]{6}/$1*/ if ((defined $testver) && (defined $testver->{$ENV{'PBPROJ'}}) && ($testver->{$ENV{'PBPROJ'}} =~ /true/i));
4491            $p =~ s/(-r|_p[0-9]+)\.ebuild/$1*/ if ((defined $testver) && (defined $testver->{$ENV{'PBPROJ'}}) && ($testver->{$ENV{'PBPROJ'}} =~ /true/i));
4492            # TODO apk
4493            $theorlist{$p} = -2 if (not defined $theorlist{$p});
4494            $theorlist{$p} = $theorlist{$p} + 1;
4495        }
4496        pb_log(2,"theorlist after check: ".Dumper(%theorlist)."\n");
4497    }
4498    close(LOG);
4499
4500    # Nothing more for the cleanssh case
4501    return if ($antype eq "Clean");
4502
4503    if ($antype eq "Check") {
4504        my ($chkex) = pb_conf_get_if("checkexclude");
4505        my $vmbuildlist = "";
4506        my $vebuildlist = "";
4507        my $rmbuildlist = "";
4508        my @pt = pb_conf_get_if("vmlist","velist","rmlist");
4509        foreach my $t (sort keys %theorlist) {
4510            if (defined $theorlist{$t} and $theorlist{$t} >= 1) {
4511                pb_log(1,"Packages found for $t\n");
4512            } elsif (defined $theorlist{$t} and $theorlist{$t} < 0) {
4513                pb_log(0,"Extra Package found for $t\n");
4514            } else {
4515                pb_log(2,"Analyzing $t\n");
4516                my ($os,$ver,$arch,$package) = split(/\//,$t);
4517                # Some distro have no arch subdir
4518                if (not defined $package) {
4519                    $package = $arch;
4520                    # TODO: If both arch have failed, we just make the last one
4521                    $arch = $archlist{$t} if (defined $archlist{$t});
4522                }
4523                my $pbos = pb_distro_get_context("$os-$ver-$arch");
4524                my $pkgn = $package;
4525                if ($pbos->{'type'} ne "deb") {
4526                    # package name is more easily found from the end for non deb
4527                    # as '-' is the separator, but it can also be used in names
4528                    $pkgn = reverse($package);
4529                    # search the second '-' and isolate the now last part which is the full name
4530                    $pkgn =~ s/([^-]+)-([^-]+)-([\S])+$/$3/;
4531                } else {
4532                    $pkgn =~ s/([^_]+)_([\S])+$/$2/;
4533                }
4534                my $found = 0;
4535                # Handle the exclusion of OSes
4536                my $excl = "";
4537                $excl .= $chkex->{$pkgn} if (defined $chkex->{$pkgn});
4538                $excl .= $chkex->{"all"} if (defined $chkex->{"all"});
4539
4540                pb_log(2,"Searching in excl $excl\n");
4541                foreach my $ex (split(/,/,$excl)) {
4542                    $found = 1 if ("$os-$ver-$arch" =~ /^$ex/);
4543                }
4544                # Skip as excluded
4545                next if ($found == 1);
4546                pb_log(0,"Package NOT found for $t\n");
4547                # Avoid duplicates in list
4548                next if ($vmbuildlist =~ /$os-$ver-$arch/);
4549                next if ($vebuildlist =~ /$os-$ver-$arch/);
4550                next if ($rmbuildlist =~ /$os-$ver-$arch/);
4551                # check with which method we need to build
4552                if ((defined $pt[0]->{$ENV{'PBPROJ'}}) and ($pt[0]->{$ENV{'PBPROJ'}} =~ /$os-$ver-$arch/)) {
4553                    $vmbuildlist = "$os-$ver-$arch" if ($vmbuildlist eq "");
4554                    $vmbuildlist .= ",$os-$ver-$arch" if ($vmbuildlist !~ /$os-$ver-$arch/);
4555                    next;
4556                }
4557                if ((defined $pt[1]->{$ENV{'PBPROJ'}}) and ($pt[1]->{$ENV{'PBPROJ'}} =~ /$os-$ver-$arch/)) {
4558                    $vebuildlist = "$os-$ver-$arch" if ($vebuildlist eq "");
4559                    $vebuildlist .= ",$os-$ver-$arch" if ($vebuildlist !~ /$os-$ver-$arch/);
4560                    next;
4561                }
4562                if ((defined $pt[2]->{$ENV{'PBPROJ'}}) and ($pt[2]->{$ENV{'PBPROJ'}} =~ /$os-$ver-$arch/)) {
4563                    $rmbuildlist = "$os-$ver-$arch" if ($rmbuildlist eq "");
4564                    $rmbuildlist .= ",$os-$ver-$arch" if ($rmbuildlist !~ /$os-$ver-$arch/);
4565                }
4566            }
4567        }
4568        # If we want to rebuild automatically, let's do it
4569        if (defined $opts{'rebuild'}) {
4570            # SandBox or CMS
4571            pb_log(0,"Rebuilding from SandBox\n");
4572            pb_log(0,"for VMs: $vmbuildlist\n") if ($vmbuildlist ne "");
4573            pb_log(0,"for VEs: $vebuildlist\n") if ($vebuildlist ne "");
4574            pb_log(0,"for RMs: $rmbuildlist\n") if ($rmbuildlist ne "");
4575            pb_cms2build("SandBox");
4576            # Which mode
4577            $ENV{'PBV'} = $vmbuildlist;
4578            pb_build2v("vm","build") if ($vmbuildlist ne "");
4579            $ENV{'PBV'} = $vebuildlist;
4580            pb_build2v("ve","build") if ($vebuildlist ne "");
4581            $ENV{'PBV'} = $rmbuildlist;
4582            pb_build2v("rm","build") if ($rmbuildlist ne "");
4583        }
4584        # For the check part this is now finished
4585        return;
4586    }
4587
4588    print ANN << "EOF";
4589</TR>
4590</TABLE>
4591</p>
4592
4593<p>As usual source packages are also available in the same directory.</p>
4594
4595<p>
4596Changes are :
4597</p>
4598<p>
4599EOF
4600    # Get each package changelog content
4601    foreach my $f (sort(@files)) {
4602        open(IN,"$f") || die "Unable to read $f:$!";
4603        while (<IN>) {
4604            print ANN $_;
4605        }
4606        close(IN);
4607        print ANN "</p><p>\n";
4608    }
4609    print ANN "</p>\n";
4610    close(ANN);
4611
4612    # Allow for modification
4613    my $editor = "vi";
4614    $editor = $ENV{'EDITOR'} if (defined $ENV{'EDITOR'});
4615    pb_system("$editor $ENV{'PBTMP'}/announce.html","Allowing modification of the announce","noredir");
4616
4617    # Store it in DB for external usage (Web pages generation)
4618    my $dbh = pb_connect_db();
4619
4620    # To read whole file
4621    local $/;
4622    open(ANN,"$ENV{'PBTMP'}/announce.html") || die "Unable to read $ENV{'PBTMP'}/announce.html: $!";
4623    my $announce = <ANN>;
4624    close(ANN);
4625   
4626    pb_log(2,"INSERT INTO announces VALUES (NULL, $pbdate, $announce)");
4627    my $sth = $dbh->prepare(qq{INSERT INTO announces VALUES (NULL,?,?)})
4628                    || die "Unable to insert into db";
4629    $sth->execute($pbdate, $announce);
4630    $sth->finish();
4631    $dbh->disconnect;
4632
4633    # Then deliver it on the Web
4634    # $TOOLHOME/livwww www
4635
4636    # Mail it to project's ML
4637    open(ML,"| w3m -dump -T text/html > $ENV{'PBTMP'}/announce.txt") || die "Unable to create $ENV{'PBTMP'}/announce.txt: $!";
4638    print ML << 'EOF';
4639<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/x html1/DTD/xhtml1-strict.dtd">
4640
4641<html xmlns="http://www.w3.org/1999/xhtml" dir="ltr" xml:lang="en" lang="en">
4642  <head>
4643  </head>
4644  <body>
4645  <p>
4646EOF
4647    open(ANN,"$ENV{'PBTMP'}/announce.html") || die "Unable to read $ENV{'PBTMP'}/announce.html: $!";
4648    while(<ANN>) {
4649        print ML $_;
4650    }
4651    print ML << 'EOF';
4652</body>
4653</html>
4654EOF
4655    close(ML);
4656
4657    # To read whole file
4658    local $/;
4659    open(ANN,"$ENV{'PBTMP'}/announce.txt") || die "Unable to read $ENV{'PBTMP'}/announce.txt: $!";
4660    my $msg = <ANN>;
4661    close(ANN);
4662   
4663    # Preparation of headers
4664    eval
4665    {
4666        require Mail::Sendmail;
4667        Mail::Sendmail->import();
4668    };
4669    if ($@) {
4670        # Mail::Sendmail not found not sending mail !
4671        pb_log(0,"No Mail::Sendmail module found so not sending any mail !\n");
4672    } else {
4673        my ($pbml,$pbsmtp) = pb_conf_get("pbml","pbsmtp");
4674        my %mail = (   
4675            To          =>  $pbml->{$ENV{'PBPROJ'}},
4676            From        =>  $pbpackager->{$ENV{'PBPROJ'}},
4677            Smtp        =>  $pbsmtp->{$ENV{'PBPROJ'}},
4678            Body        =>  $msg,
4679            Subject     =>  "[ANNOUNCE] $sl",
4680        );
4681           
4682        # Send mail
4683        if (! sendmail(%mail)) {
4684            if ((defined $Mail::Sendmail::error) and (defined $Mail::Sendmail::log)) {
4685                die "Unable to send mail ($Mail::Sendmail::error): $Mail::Sendmail::log";
4686            }
4687        }
4688    }
4689}
4690
4691#
4692# Creates a set of HTML file containing the news for the project
4693# based on what has been generated by the pb_announce function
4694#
4695sub pb_web_news2html {
4696
4697    my $dest = shift || $ENV{'PBTMP'};
4698
4699    # Get all required parameters
4700    my ($pkgv, $pkgt) = pb_conf_get_if("pkgver","pkgtag");
4701
4702
4703    my $dbh = pb_connect_db();
4704    # For date handling
4705    $ENV{'LANGUAGE'}="C";
4706    my $firstjan = strftime("%Y-%m-%d", 0, 0, 0, 1, 0, localtime->year(), 0, 0, -1);
4707    my $oldfirst = strftime("%Y-%m-%d", 0, 0, 0, 1, 0, localtime->year()-1, 0, 0, -1);
4708    pb_log(2,"firstjan: $firstjan, oldfirst: $oldfirst, pbdate:$pbdate\n");
4709    my $all = $dbh->selectall_arrayref("SELECT id,date,announce FROM announces ORDER BY date DESC");
4710    my %news;
4711    $news{"cy"} = "";   # current year's news
4712    $news{"ly"} = "";   # last year news
4713    $news{"py"} = "";   # previous years news
4714    $news{"fp"} = "";   # first page news
4715    my $cpt = 4;        # how many news for first page
4716    # Extract info from DB
4717    foreach my $row (@$all) {
4718        my ($id, $date, $announce) = @$row;
4719        $news{"cy"} = $news{"cy"}."<p><B>$date</B> $announce\n" if ((($date cmp $pbdate) le 0) && (($firstjan cmp $date) le 0));
4720        $news{"ly"} = $news{"ly"}."<p><B>$date</B> $announce\n" if ((($date cmp $firstjan) le 0) && (($oldfirst cmp $date) le 0));
4721        $news{"py"} = $news{"py"}."<p><B>$date</B> $announce\n" if (($date cmp $oldfirst) le 0);
4722        $news{"fp"} = $news{"fp"}."<p><B>$date</B> $announce\n" if ($cpt > 0);
4723        $cpt--;
4724    }
4725    pb_log(1,"news{fp}: ".$news{"fp"}."\n");
4726    $dbh->disconnect;
4727
4728    # Generate the HTML content
4729    foreach my $pref (keys %news) {
4730        open(NEWS,"> $dest/pb_web_$pref"."news.html") || die "Unable to create $dest/pb_web_$pref"."news.html: $!";
4731        print NEWS "$news{$pref}";
4732        close(NEWS);
4733    }
4734}
4735
4736sub pb_connect_db {
4737
4738    # Store in DB for external usage (Web pages generation)
4739    my $db = "$ENV{'PBCONFDIR'}/announces3.sql";
4740
4741    my $precmd = "";
4742    if (! -f $db) {
4743        $precmd = "CREATE TABLE announces (id INTEGER PRIMARY KEY AUTOINCREMENT, date DATE, announce VARCHAR[65535])";
4744        # we should add the $db to the VCS as well here
4745    }
4746
4747    my $dbh = DBI->connect("dbi:SQLite:dbname=$db","","",
4748                        { RaiseError => 1, AutoCommit => 1 })
4749                        || die "Unable to connect to $db";
4750
4751    if ($precmd ne "") {
4752        my $sth = $dbh->prepare(qq{$precmd})
4753                    || die "Unable to create table into $db";
4754        $sth->execute();
4755    }
4756
4757    return($dbh);
4758}
4759
4760
4761# Return the SSH key file to use
4762# Potentially create it if needed
4763
4764sub pb_ssh_get {
4765
4766my $create = shift || 0;    # Do not create keys by default. In that case prefer native keys, not pb's ones
4767my $target = shift || "localhost";
4768my $nport = shift || "22";
4769
4770my ($pbagent) = pb_conf_get_if("pbusesshagent");
4771# use ssh-agent if asked so.
4772return(undef) if (($create == 0) && (defined $pbagent->{$ENV{'PBPROJ'}}) && ($pbagent->{$ENV{'PBPROJ'}} =~ /true/io));
4773
4774# Check the SSH environment
4775my $keyfile = undef;
4776
4777# Check whether ecdsa is accepted by the remote host
4778my $ecdsa = 0;
4779my $dsa = 0;
4780my $rsa = 0;
4781open(SCAN,"ssh-keyscan -p $nport -t ecdsa $target 2>/dev/null |") || pb_log(0,"Unable to check ECDSA support for $target");
4782while(<SCAN>) {
4783    $ecdsa = 1 if ($_ =~ /ecdsa-/);
4784    pb_log(2,"Found ECDSA support on $target ($nport)\n");
4785}
4786close(SCAN);
4787
4788open(SCAN,"ssh-keyscan -p $nport -t rsa $target 2>/dev/null |") || pb_log(0,"Unable to check DSA support for $target");
4789while(<SCAN>) {
4790    $rsa = 1 if ($_ =~ /ssh-rsa/);
4791    pb_log(2,"Found RSA support on $target ($nport)\n");
4792}
4793close(SCAN);
4794
4795open(SCAN,"ssh-keyscan -p $nport -t dsa $target 2>/dev/null |") || pb_log(0,"Unable to check DSA support for $target");
4796while(<SCAN>) {
4797    $dsa = 1 if ($_ =~ /ssh-dss/);
4798    pb_log(2,"Found DSA support on $target ($nport)\n");
4799}
4800close(SCAN);
4801
4802# We have specific keys by default - create them if needed
4803if (!(-e "$ENV{'HOME'}/.ssh/pb_ecdsa") && ($create eq 1)) {
4804    my $res = pb_system("ssh-keygen -q -b 521 -N '' -f $ENV{'HOME'}/.ssh/pb_ecdsa -t ecdsa","Generating ECDSA SSH keys for pb");
4805    if (($res != 0) && !(-e "$ENV{'HOME'}/.ssh/pb_dsa") && ($create eq 1)) {
4806        # ecdsa may not be supported if old ssh
4807        pb_system("ssh-keygen -q -b 1024 -N '' -f $ENV{'HOME'}/.ssh/pb_dsa -t dsa","Generating DSA SSH keys for pb");
4808    }
4809}
4810
4811# Some ECDSA implementation are not working correctly e.g. RHEL7, Fedora 20/21
4812if (!(-e "$ENV{'HOME'}/.ssh/pb_rsa") && ($create eq 1)) {
4813    pb_system("ssh-keygen -q -b 2048 -N '' -f $ENV{'HOME'}/.ssh/pb_rsa -t rsa","Generating RSA SSH keys for pb");
4814}
4815
4816# We still favour RSA and fall back to ECDSA if not available
4817$keyfile = "$ENV{'HOME'}/.ssh/id_rsa" if (-s "$ENV{'HOME'}/.ssh/id_rsa");
4818if ($rsa != 0) {
4819    # And we favour a specific key over a generic one except if in delivery to ftp server
4820    $keyfile = "$ENV{'HOME'}/.ssh/id_rsa" if ((-s "$ENV{'HOME'}/.ssh/id_rsa") && ($create eq 0));
4821    $keyfile = "$ENV{'HOME'}/.ssh/pb_rsa" if (-s "$ENV{'HOME'}/.ssh/pb_rsa");
4822} else {
4823    if ($ecdsa != 0) {
4824        # And we favour a specific key over a generic one except if in delivery to ftp server
4825        $keyfile = "$ENV{'HOME'}/.ssh/id_ecdsa" if ((-s "$ENV{'HOME'}/.ssh/id_ecdsa") && ($ecdsa != 0) && ($create eq 0));
4826        $keyfile = "$ENV{'HOME'}/.ssh/pb_ecdsa" if ((-s "$ENV{'HOME'}/.ssh/pb_ecdsa") && ($ecdsa != 0));
4827    } elsif ($dsa != 0) {
4828        # And we favour a specific key over a generic one except if in delivery to ftp server
4829        $keyfile = "$ENV{'HOME'}/.ssh/id_dsa" if ((-s "$ENV{'HOME'}/.ssh/id_dsa") && ($dsa != 0) && ($create eq 0));
4830        $keyfile = "$ENV{'HOME'}/.ssh/pb_dsa" if ((-s "$ENV{'HOME'}/.ssh/pb_dsa") && ($dsa != 0));
4831    }
4832}
4833if (defined $keyfile) {
4834    pb_log(2,"Using ssh key file $keyfile\n");
4835} else {
4836    pb_log(0, "Unable to find your public ssh key under $ENV{'HOME'}/.ssh\n");
4837}
4838return($keyfile);
4839}
4840
4841
4842# Returns the pid of a running VM command using a specific VM file, and the port used if the VM exists
4843sub pb_check_ps {
4844    my $vmcmd = shift;
4845    my $vmm = shift;
4846    my $vexist = 0;             # FALSE by default
4847    my $vmport = undef;         # NONE by default
4848
4849    if ($vmcmd =~ /qemu/) {
4850        # Remove the suffix as qemu changes it at launch
4851        $vmcmd =~ s/qemu-.*/qemu/;
4852    }
4853    open(PS, "ps auxhww|") || die "Unable to call ps";
4854    while (<PS>) {
4855        next if (! /$vmcmd/);
4856        next if (! /$vmm/);
4857        my $void1;
4858        ($void1, $vexist, $vmport) = split(/ +/,$_,3);
4859        pb_log(2,"pb_check_ps $vmport\n");
4860        $vmport =~ s/.*=tcp:127\.0\.0\.1:([0-9]+)-:22.*/$1/;
4861        chomp($vmport);
4862        pb_log(2,"pb_check_ps found a VM $vexist using port $vmport\n");
4863        last;
4864    }
4865    return($vexist,$vmport);
4866}
4867
4868
4869sub pb_extract_build_files {
4870
4871my $src=shift;
4872my $dir=shift;
4873my $ddir=shift;
4874my $mandatory=shift || "build";
4875
4876my $flag = "mayfail" if (($mandatory eq "patch") || ($mandatory eq "src"));
4877my $res;
4878
4879if ($src =~ /tar\.gz$/) {
4880    $res = pb_system("tar xfpz $src $dir","Extracting $mandatory files from $src",$flag);
4881} elsif ($src =~ /tar\.bz2$/) {
4882    $res = pb_system("tar xfpj $src $dir","Extracting $mandatory files from $src",$flag);
4883} else {
4884    die "Unknown compression algorithm for $src";
4885}
4886# If not mandatory return now
4887return() if (($res != 0) and (($mandatory eq "patch") || ($mandatory eq "src")));
4888return(pb_move_extracted_files($dir,$ddir));
4889}
4890
4891sub pb_move_extracted_files {
4892
4893my $dir = shift;
4894my $ddir = shift;
4895my @files;
4896
4897# In a recursive function , we need a local var as DIR handle
4898my $bdir;
4899
4900opendir($bdir,"$dir") || confess "Unable to open directory $dir: $!";
4901foreach my $f (readdir($bdir)) {
4902    next if ($f =~ /^\./);
4903    # Skip potential patch dir
4904    next if ($f =~ /^pbpatch/);
4905    # Skip potential source dir
4906    next if ($f =~ /^pbsrc/);
4907    # Skip potential backup files
4908    next if ($f =~ /~$/);
4909    if ((-d "$dir/$f") && (! -l "$dir/$f")) {
4910        # Case of the bug debian subdir
4911        pb_mkdir_p("$ddir/$f");
4912        push @files,pb_move_extracted_files("$dir/$f","$ddir/$f");
4913    } else {
4914        move("$dir/$f","$ddir") || die "Unable to move $dir/$f to $ddir";
4915        pb_log(2,"mv $dir/$f $ddir\n");
4916        push @files,"$ddir/$f";
4917    }
4918}
4919closedir($bdir);
4920# Not enough but still a first cleanup
4921pb_rm_rf("$dir");
4922return(@files);
4923}
4924
4925sub pb_list_bfiles {
4926
4927my $dir = shift;
4928my $pbpkg = shift;
4929my $bfiles = shift;
4930my $pkgfiles = shift;
4931my $supfiles = shift;
4932# subdir to keep if recursive mode, empty by default
4933my $subdir = shift || "";
4934# In a recursive function , we need a local var as DIR handle
4935my $bdir;
4936
4937pb_log(2,"DEBUG: entering pb_list_bfiles in $dir: ".Dumper($bfiles)."\n");
4938opendir($bdir,"$dir") || die "Unable to open dir $dir: $!";
4939foreach my $f (readdir($bdir)) {
4940    pb_log(3,"DEBUG: pb_list_bfiles found $f\n");
4941    next if ($f =~ /^\./);
4942    if (-d "$dir/$f") {
4943        # Recurse for directories (Debian 3.0 format e.g.)
4944        pb_log(2,"DEBUG: pb_list_bfiles recurse in $dir/$f\n");
4945        pb_list_bfiles("$dir/$f",$pbpkg,$bfiles,$pkgfiles,$supfiles,$f);
4946        next;
4947    }
4948
4949    my $key = $f;
4950    # if recursive then store also the subdir
4951    $key = "$subdir/$f" if ($subdir ne "");
4952    $bfiles->{$key} = "$dir/$f";
4953    $bfiles->{$key} =~ s~$ENV{'PBROOTDIR'}~~;
4954    if (defined $supfiles->{$pbpkg}) {
4955        $pkgfiles->{$key} = "$dir/$f" if ($f =~ /$supfiles->{$pbpkg}/);
4956    }
4957}
4958closedir($bdir);
4959pb_log(2,"DEBUG: exiting pb_list_bfiles: ".Dumper($bfiles)."\n");
4960}
4961
4962sub pb_add_comma {
4963
4964my $str = shift;
4965my $addstr = shift;
4966
4967$str .= "," if (defined $str);
4968$str .= $addstr;
4969return($str);
4970}
4971
4972sub pb_list_sfiles {
4973
4974my $sdir = shift;
4975my $ptr = shift;
4976my $pbos = shift;
4977my $extdir = shift;
4978
4979pb_log(2,"DEBUG: entering pb_list_sfiles: ".Dumper($ptr)."\n");
4980my $key = "$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
4981
4982# Prepare local sources for this distro - They are always applied first - May be a problem one day
4983# This function works for both patches and additional sources
4984foreach my $p (sort(<$sdir/*>)) {
4985    $ptr->{$key} = pb_add_comma($ptr->{$key},"file://$p") if (($p =~ /\.all$/) || ($p =~ /\.$pbos->{'os'}$/) || ($p =~ /\.$pbos->{'type'}$/) || ($p =~ /\.$pbos->{'family'}$/) || ($p =~ /\.$pbos->{'name'}$/) || ($p =~ /\.$pbos->{'name'}-$pbos->{'version'}$/) ||($p =~ /\.$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}$/));
4986}
4987
4988# Prepare also remote sources to be included - Applied after the local ones
4989foreach my $p ("all","$pbos->{'os'}","$pbos->{'type'}","$pbos->{'family'}","$pbos->{'name'}","$pbos->{'name'}-$pbos->{'version'}","$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}") {
4990    my $f = "$extdir.".".$p";
4991    next if (not -f $f);
4992    if (not open(PATCH,$f)) {
4993        pb_display("Unable to open existing external source file content $f\n");
4994        next;
4995    }
4996    while (<PATCH>) {
4997        chomp();
4998        $ptr->{$key} = pb_add_comma($ptr->{$key},"$_");
4999    }
5000    close(PATCH);
5001}
5002pb_log(2,"DEBUG: exiting pb_list_sfiles: ".Dumper($ptr)."\n");
5003return($ptr);
5004}
5005   
5006#
5007# Return the list of packages we are working on in a non CMS action
5008#
5009sub pb_get_pkg {
5010
5011my @pkgs = ();
5012
5013# In script actions that file may not exist.
5014return(\@pkgs) if ((not defined $ENV{'PBDESTDIR'}) || (not defined $ENV{'PBPROJVER'}) || (not defined $ENV{'PBPROJTAG'}) || (! -f "$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml"));
5015
5016my ($var) = pb_conf_read("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml","pbpkg");
5017@pkgs = keys %$var if (defined $var);
5018
5019pb_log(0,"Packages: ".join(',',@pkgs)."\n") if (@pkgs);
5020return(\@pkgs);
5021}
5022
5023# Manages VM/RM SSH port communication
5024sub pb_get_port {
5025
5026my $port = shift;
5027my $pbos = shift;
5028my $cmt = shift;
5029my $nport;
5030
5031die "No port passed in parameter. Report to dev team\n" if (not defined $port);
5032# key is project on VM, but machine tuple for RM
5033if ($cmt =~ /^RM/i) {
5034    $nport = $port->{"$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}"};
5035} else {
5036    $nport = $port->{$ENV{'PBPROJ'}};
5037}
5038pb_log(2,"pb_get_port with $nport\n");
5039if ($cmt =~ /^VM/i) {
5040    # Maybe a port was given as parameter so overwrite
5041    $nport = "$pbport" if (defined $pbport);
5042    # Maybe in // mode so use the env var set up as an offset to the base port
5043    $nport = $nport + $ENV{'PBVMPORT'} -1 if ((defined $pbparallel) && (defined $ENV{'PBVMPORT'}));
5044}
5045pb_log(2,"pb_get_port returns $nport\n");
5046return($nport);
5047}
5048
5049sub pb_set_port { 
5050       
5051my ($pid,$ident) = @_;
5052pb_log(2,"pb_set_port for VM ($pid), id $ident\n");
5053$ENV{'PBVMPORT'} = $ident;
5054pb_log(2,"pb_set_port sets PBVMPORT in env to $ENV{'PBVMPORT'}\n");
5055}
5056
5057sub pb_set_parallel {
5058
5059my $vtype = shift;
5060
5061pb_log(2,"pb_set_parallel vtype: $vtype\n");
5062# Take care of memory size if VM, parallel mode and more than 1 action
5063if ((defined $pbparallel) && ($pbparallel ne 1) && ($vtype eq "vm")) {
5064    eval
5065    {
5066        require Linux::SysInfo;
5067        Linux::SysInfo->import();
5068    };
5069    if ($@) {
5070        # Linux::SysInfo not found
5071        pb_log(1,"ADVISE: Install Linux::SysInfo to benefit from automatic parallelism optimization.\nOr optimize manually pbparallel in your pb.yml file\nUsing $pbparallel processes max at a time for the moment\nWARNING: This may consume too much memory for your system\n");
5072    } else {
5073        # Using the memory size
5074        my $si = Linux::SysInfo::sysinfo();
5075        if (not defined $si) {
5076            pb_log(1,"ADVISE: Install Linux::SysInfo to benefit from automatic parallelism optimization.\nOr optimize manually pbparallel in your pb.yml file\nUsing $pbparallel processes max at a time for the moment\nWARNING: This may consume too much memory for your system\n");
5077        } else {
5078            # Keep the number of VM whose memory can be allocated
5079            my $ram = $si->{"totalram"}-$si->{"sharedram"}-$si->{"bufferram"};
5080            my $ram2;
5081            my ($vmmem) = pb_conf_get_if("vmmem");
5082
5083            my $v = "default";
5084            if ((defined $vmmem) and (defined $vmmem->{$v})) {
5085                $ram2 = $vmmem->{$v};
5086            } else {
5087                # Default for KVM/QEMU
5088                $ram2 = 128;
5089            }
5090            $pbparallel = sprintf("%d",$ram/$ram2);
5091        }
5092        pb_log(1,"Using $pbparallel processes at a time\n");
5093    }
5094}
5095pb_log(2,"pb_set_parallel returns: $pbparallel\n") if (defined $pbparallel);
5096return($pbparallel);
5097}
5098
5099sub pb_get_sudocmds { 
5100       
5101my $pbos = shift;
5102my @cmds = @_;
5103my %sudocmds;
5104
5105pb_log(2,"pb_get_sudocmds entering with lines:".Dumper(@cmds)."\n");
5106foreach my $c (split(/;/,$pbos->{'update'}),split(/;/,$pbos->{'install'}),@cmds) {
5107    pb_log(2,"pb_get_sudocmds analyses $c\n");
5108    next if ($c !~ /^\s*sudo/);
5109    # remove sudo and leading spaces
5110    $c =~ s/^\s*sudo\s+//;
5111    # keep only the command, not the params
5112    $c =~ s/([^\s]+)\s.*$/$1/;
5113    $sudocmds{$c} = "";
5114}
5115pb_log(2,"pb_get_sudocmds returns ".Dumper(keys %sudocmds)."\n");
5116return(keys %sudocmds);
5117}
5118
5119sub pb_sign_setenv {
5120
5121my ($passfile, $passphrase, $passpath) = pb_conf_get_if("pbpassfile","pbpassphrase","pbpasspath");