source: devel/pb/bin/pb @ 2337

Last change on this file since 2337 was 2337, checked in by bruno, 23 months ago

More alpine improvements (but still not done yet)

  • Property svn:executable set to *
File size: 183.3 KB
Line 
1#!/usr/bin/perl -w
2#
3# Project Builder main application
4#
5# $Id$
6#
7# Copyright B. Cornec 2007-today
8# Eric Anderson's changes are (c) Copyright 2012 Hewlett Packard
9# Provided under the GPL v2
10
11# Syntax: see at end
12
13use strict 'vars';
14
15# The modules mentioned here are required by pb when used both
16# locally or inside a VE/VM/RM
17# Additional required modules only used locally are called with a require
18# in their respective section
19use Getopt::Long qw(:config auto_abbrev no_ignore_case);
20use Carp qw/confess cluck/;
21use Data::Dumper;
22use English;
23use File::Basename;
24use File::Copy;
25use File::stat;
26use File::Find;
27use Time::localtime qw(localtime);
28use POSIX qw(strftime);
29use lib qw (lib);
30use ProjectBuilder::Version;
31use ProjectBuilder::Base;
32use ProjectBuilder::Display;
33use ProjectBuilder::Conf;
34use ProjectBuilder::Distribution;
35use ProjectBuilder::VCS;
36use ProjectBuilder::CMS;
37use ProjectBuilder::Env;
38use ProjectBuilder::Filter;
39use ProjectBuilder::Changelog;
40use ProjectBuilder::VE;
41
42# Global variables
43$Global::pb_stop_on_error = 0;      # False by default
44$Global::pb_show_sudo = 0;          # False by default
45
46my %opts;                           # CLI Options
47my $action;                         # action to realize
48my $test = "FALSE";                 # Not used
49my $pbforce = 0;                    # Force VM stop
50my $pbsnap = 0;                     # Do not use snapshot mode for VM/VE by default
51my $pbkeep = 0;                     # keep temporary directory at the end
52my $option = "";                    # Not used
53my @pkgs;                           # list of packages
54my $pbtag;                          # Global Tag variable
55my $pbver;                          # Global Version variable
56my %pbscript;                       # Name of the scripts per V
57my $pbrev;                          # Global REVISION variable
58my $pbtarget = undef;               # Target os-ver-arch you want to build for
59my $pbport;                         # Port to use to connect to the VM/RM
60my $newver;                         # New version to create
61my $pbimage = undef;                # ISO image for the VM to create or docker image for the VE
62my $vetype = undef;                 # Type of Virtual environment to deal with
63
64my @date = pb_get_date();
65my $pbdate = strftime("%Y-%m-%d", @date);
66
67=pod
68
69=head1 NAME
70
71pb, aka project-builder.org - builds packages for your projects
72
73=head1 DESCRIPTION
74
75pb helps you build various packages directly from your project sources.
76Those sources could be handled by a CMS (Configuration Management System)
77such as Subversion, CVS, Git, Mercurial... or being a simple reference to a compressed tar file.
78It's based on a set of configuration files, a set of provided macros to help
79you keeping build files as generic as possible. For example, a single .spec
80file should be required to generate for all rpm based distributions, even
81if you could also have multiple .spec files if required.
82
83=head1 SYNOPSIS
84
85pb [-vhSq][-r pbroot][-p project][[-s script -a account -P port][-T VEtype][-t [os-ver-arch]][-m os-ver-arch[,...]]][-g][-i image] <action> [<pkg1> ...]
86
87pb [--verbose][--help][--man][--quiet][--snapshot][--revision pbroot][--project project][[--script script --account account --port port][-T VEtype][--target [os-ver-arch]][--machine os-ver-arch[,...]]][--nographic][--image image][--rebuild] <action> [<pkg1> ...]
88
89=head1 OPTIONS
90
91=over 4
92
93=item B<-v|--verbose>
94
95Increase verbosity
96
97=item B<-q|--quiet>
98
99Do not print any output.
100
101=item B<-h|--help>
102
103Print a brief help message and exits.
104
105=item B<-S|--snapshot>
106
107Use the snapshot mode of VMs or VEs
108
109=item B<--man>
110
111Prints the manual page and exits.
112
113=item B<-t|--target os-ver-arch>
114
115Name of the target system you want to build for.
116All if none precised.
117
118=item B<-m|--machine os-ver-arch[,os-ver-arch,...]>
119
120Name of the Virtual Machines (VM), Virtual Environments (VE) or Remote Machines (RM)
121you want to build on (comma separated).
122All if none precised (or use the env variable PBV).
123
124=item B<-T|--vetype VEtype]>
125
126Type of Virtual Environments (VE)
127Can be chroot or docker.
128
129=item B<-s|--script script>
130
131Name of the script you want to execute on the related VMs/VEs/RMs.
132
133=item B<-g|--nographic>
134
135Do not launch VMs in graphical mode.
136
137=item B<-i|--image image>
138
139It could be either:
140- The name of the ISO image of the distribution you want to install on the related VMs
141- The name of the docker image of the distribution you want to install on the related VEs
142
143
144=item B<-a|--account account>
145
146Name of the account to use to connect on the related VMs/RMs.
147
148=item B<-P|--port port_number>
149
150Port number to use to connect on the related VMs/RMs.";
151
152=item B<-p|--project project_name>
153
154Name of the project you're working on (or use the env variable PBPROJ)
155
156=item B<-r|--revision revision>
157
158Path Name of the project revision under the CMS (or use the env variable PBROOT)
159
160=item B<-V|--version new_version>
161
162New version of the project to create based on the current one.
163
164=item B<-k|--keep>
165
166Keep the temporary dir where files have been created in or der to help debug
167
168=item B<--rebuild>
169
170Only valid with the checkssh action, it alllows to automatically relaunch the build of the failed packages
171
172=item B<--no-stop-on-error>
173
174Continue through errors with best effort.
175
176=back
177
178=head1 ARGUMENTS
179
180<action> can be:
181
182=over 4
183
184=item B<sbx2build>
185
186Create tar files for the project under your CMS.
187Current state of the exported content is taken.
188CMS supported are SVN, SVK, CVS, Git and Mercurial
189parameters are packages to build
190if not using default list
191
192=item B<cms2build>
193
194Create tar files for the project under your CMS.
195Current state of the CMS is taken.
196CMS supported are SVN, SVK, CVS, Git and Mercurial
197parameters are packages to build
198if not using default list
199
200=item B<build2prep>
201
202Prepare the environment for build by installing required dependencies. Done once on the build system.
203
204=item B<build2pkg>
205
206Create packages for your running distribution
207
208=item B<cms2pkg>
209
210cms2build + build2pkg
211
212=item B<sbx2pkg>
213
214sbx2build + build2pkg
215
216=item B<sbx2pkg2ins>
217
218sbx2pkg + final install of packages
219
220=item B<sbx2prep>
221
222sbx2build + build2prep
223
224=item B<build2ssh>
225
226Send the tar files to a SSH host
227
228=item B<sbx2ssh>
229
230sbx2build + build2ssh
231
232=item B<cms2ssh>
233
234cms2build + build2ssh
235
236=item B<pkg2ssh>
237
238Send the packages built to a SSH host
239
240=item B<build2vm>
241
242Create packages in VMs, launching them if needed
243and send those packages to a SSH host once built
244VM type supported are QEMU and KVM
245
246=item B<build2ve>
247
248Create packages in VEs, creating it if needed
249and send those packages to a SSH host once built
250
251=item B<build2rm>
252
253Create packages in RMs, which should pre-exist,
254and send those packages to a SSH host once built
255RM means Remote Machine, and could be a physical or Virtual one.
256This is one buildfarm integration for pb.
257
258=item B<prepvm>
259
260Prepare the VMs to have all requirements to build the project
261
262=item B<prepve>
263
264Prepare the VEs to have all requirements to build the project
265
266=item B<preprm>
267
268Prepare the RMs to have all requirements to build the project
269
270=item B<sbx2vm>
271
272sbx2build + build2vm
273
274=item B<sbx2ve>
275
276sbx2build + build2ve
277
278=item B<sbx2docker>
279
280sbx2build + build2ve with a potential build of all necessary docker containers to perform it
281
282=item B<sbx2rm>
283
284sbx2build + build2rm
285
286=item B<cms2vm>
287
288cms2build + build2vm
289
290=item B<cms2ve>
291
292cms2build + build2ve
293
294=item B<cms2rm>
295
296cms2build + build2rm
297
298=item B<launchvm>
299
300Launch one virtual machine
301
302=item B<launchve>
303
304Launch one virtual environment
305
306=item B<script2vm>
307
308Launch one virtual machine if needed
309and executes a script on it
310
311=item B<script2ve>
312
313Execute a script in a virtual environment
314
315=item B<script2rm>
316
317Execute a script on a remote machine
318
319=item B<newvm>
320
321Create a new virtual machine
322
323=item B<newve>
324
325Create a new virtual environment
326
327=item B<setupvm>
328
329Setup a virtual machine for pb usage
330
331=item B<setupve>
332
333Setup a virtual environment for pb usage
334
335=item B<setuprm>
336
337Setup a remote machine for pb usage
338
339=item B<sbx2setupvm>
340
341Setup a virtual machine for pb usage using the sandbox version of pb instead of the latest stable
342Reserved to dev team.
343
344=item B<sbx2setupve>
345
346Setup a virtual environment for pb usage using the sandbox version of pb instead of the latest stable
347Reserved to dev team.
348
349=item B<sbx2setuprm>
350
351Setup a remote machine for pb usage using the sandbox version of pb instead of the latest stable
352Reserved to dev team.
353
354=item B<build2setupvm>
355
356Setup a virtual machine for pb usage using the build available
357Reserved to dev team.
358
359=item B<build2setupve>
360
361Setup a virtual environment for pb usage using the build available
362Reserved to dev team.
363
364=item B<build2setuprm>
365
366Setup a remote machine for pb usage using the build available
367Reserved to dev team.
368
369=item B<snapvm>
370
371Snapshot a virtual machine for pb usage
372
373=item B<snapve>
374
375Snapshot a virtual environment for pb usage
376
377=item B<updateconf>
378
379Update the configuration files to the latest format
380
381=item B<updatevm>
382
383Update the distribution in the virtual machine
384
385=item B<updateve>
386
387Update the distribution in the virtual environment
388
389=item B<updaterm>
390
391Update the distribution in the remote machine
392
393=item B<test2pkg>
394
395Test a package locally
396
397=item B<test2vm>
398
399Test a package in a virtual machine
400
401=item B<test2ve>
402
403Test a package in a virtual environment
404
405=item B<test2rm>
406
407Test a package in a remote machine
408
409=item B<checkssh>
410
411Check the delivery of the packages on the repository
412
413=item B<checkps>
414
415Check the process running the VM concerned
416
417=item B<newver>
418
419Create a new version of the project derived
420from the current one
421
422=item B<newproj>
423
424Create a new project and a template set of
425configuration files under pbconf
426
427=item B<announce>
428
429Announce the availability of the project through various means
430
431=item B<sbx2webssh>
432
433Create tar files for the website under your CMS.
434Current state of the exported content is taken.
435Deliver the content to the target server using ssh from the exported dir.
436
437=item B<cms2webssh>
438
439Create tar files for the website from your CMS.
440Deliver the content to the target server using ssh from the DVCS.
441
442=item B<sbx2webpkg>
443
444Create tar files for the website under your CMS.
445Current state of the exported content is taken.
446
447=item B<cms2webpkg>
448
449Create tar files for the website under your CMS.
450
451=item B<getconf>
452
453Print the full configuration parameters as found in the various configuration files. Help to debug conf issues.
454Also accepts a parameter to display only this value, and a VM/VE/RM
455
456=item B<getvar>
457
458Print the full variables expanded based on the distribution tuple. Help to debug conf issues.
459Also accepts a parameter to display only the values for this package, and a VM/VE/RM
460
461=item B<clean>
462
463Purge the build and delivery directories related to the current project
464
465=item B<cleanssh>
466
467Purge the ssh server of its packages (only for testver and test packages)
468
469=back
470
471<pkgs> can be a list of packages, the keyword 'all' or nothing, in which case the default list of packages is taken (corresponding to the defpkgdir list of arguments in the configuration file).
472
473=head1 WEB SITES
474
475The main Web site of the project is available at L<http://www.project-builder.org/>. Bug reports should be filled using the trac instance of the project at L<http://trac.project-builder.org/>.
476
477=head1 USER MAILING LIST
478
479None exists for the moment.
480
481=head1 CONFIGURATION FILES
482
483Each pb user may have a configuration in F<$HOME/.pbrc.yml>. The values in this file may overwrite any other configuration file value.
484
485Here is an example of such a configuration file:
486
487 #
488 # Define for each project the URL of its pbconf repository
489 # No default option allowed here as they need to be all different
490 #
491 # URL of the pbconf content
492 # This is the format of a classical URL with the extension of additional schema such as
493 # svn+ssh, cvs+ssh, ...
494 #
495  pbconfurl:
496    linuxcoe : cvs+ssh://:ext:bcornec@linuxcoe.cvs.sourceforge.net:/cvsroot/linuxcoe/pbconf
497
498 # This is normaly defined in the project's configuration file
499 # Url of the project
500 #
501  pbprojurl:
502    linuxcoe : cvs+ssh://:ext:bcornec@linuxcoe.cvs.sourceforge.net:/cvsroot/linuxcoe
503 
504 # All these URLs needs to be defined here as the are the entry point
505 # for how to build packages for the project
506 #
507  pbconfurl:
508    pb: svn+ssh://svn.project-builder.org/mondo/svn/pb/pbconf
509  pbconfurl:
510    mondorescue: svn+ssh://svn.project-builder.org/mondo/svn/project-builder/mondorescue/pbconf
511  pbconfurl:
512    collectl: svn+ssh://bruno@svn.mondorescue.org/mondo/svn/project-builder/collectl/pbconf
513  pbconfurl:
514    netperf: svn+ssh://svn.mondorescue.org/mondo/svn/project-builder/netperf/pbconf
515 
516 # Under that dir will take place everything related to pb
517 # If you want to use VMs/chroot/..., then use $ENV{'HOME'} to make it portable
518 # to your VMs/chroot/...
519 # if not defined then /var/cache
520  pbdefdir:
521    default: $ENV{'HOME'}/project-builder
522  pbdefdir:
523    pb: $ENV{'HOME'}
524  pbdefdir:
525    linuxcoe: $ENV{'HOME'}/LinuxCOE/cvs
526  pbdefdir:
527    mondorescue: $ENV{'HOME'}/mondo/svn
528 
529 # pbconfdir points to the directory where the CMS content of the pbconfurl is checked out
530 # If not defined, pbconfdir is under pbdefdir/pbproj/pbconf
531  pbconfdir:
532    linuxcoe: $ENV{'HOME'}/LinuxCOE/cvs/pbconf
533  pbconfdir:
534    mondorescue: $ENV{'HOME'}/mondo/svn/pbconf
535 
536 # pbdir points to the directory where the CMS content of the pbprojurl is checked out
537 # If not defined, pbdir is under pbdefdir/pbproj
538 # Only defined if we have access to the dev of the project
539  pbdir:
540    linuxcoe: $ENV{'HOME'}/LinuxCOE/cvs
541  pbdir:
542    mondorescue: $ENV{'HOME'}/mondo/svn
543 
544 # -daemonize doesn't work with qemu 0.8.2
545  vmopt:
546    default: -m 384
547
548=head1 COMMAND DETAILS
549
550=head2 newproj
551
552The newproj command creates a new project-builder project.  To run this command you first need to define two variables in your ~/.pbrc.yml file:
553
554 pbconfurl I<project> = file:///home/anderse/.git/project-builder-config/I<project>
555 pbdefdir default = $ENV{'HOME'}/cache-project-builder
556
557The first line defines the version controlled configuration information and the second defines the root directory for project-builder to use.
558
559You can then run the command:
560
561 % pb -p I<$project> -r I<$version> newproj I<$pkg>
562
563to create the new project. Running the newproj command will then generate the file $pbdefdir/$project/pbconf/$version/$project.yml, and the directory $pbdefdir/$project/pbconf/$version/$pkg. You will need to edit those files to make the later commands work.
564
565=head2 cms2build
566
567The cms2build command takes your files from the content management system and makes the two tar files that are necessary for building files. You need to have run the newproj command first.  Then there are several steps for running this command:
568
569=over 4
570
571=item Update your $project.yml configuration file.
572
573You need to set the pbprojurl, pbrepo, pbwf, pbpackager, projver, projtag, testver, delivery, and defpkgdir lines as described in the configuration file. The pbprojurl entry is used to find the source for your package. The pbrepo entry is used to build the .repo or .sources.list files for use by downloaders of the package. The pbwf entry indicates that the source tar file is named by I<package-name>-I<version>. The pbpackager entry will be stored in the packages and should be you or your team. The projver/projtag entries indicate the version of the software and the version of the packaging scripts. The testver entry when true indicates that the package is in a test version, so no log file is computed (can be long), and version is made up using a timstamp. The delivery entry gives the subdirectory under which the packages will be delivered on the repository, and the defpkgdir entry corresponds to the local subdirectory hosting the package content.
574
575For example:
576
577 pbprojurl Lintel = file:///home/anderse/projects/Lintel-0.2012.02.28.tar.gz
578 pbrepo Lintel = http://tesla.hpl.hp.com/opensource
579 pbwf Lintel = 1
580 pbpackager Lintel = Eric Anderson <eric.anderson4@hp.com>
581 projver Lintel = 0.2012.02.28
582 projtag Lintel = 1
583 testver Lintel = false
584 delivery Lintel = production
585 defpkgdir Lintel = Lintel-0.2012.02.28
586
587=item Create the build .tar.gz files:
588
589Then you need to take those files and create the initial tar files.  Run a command like:
590
591 % pb -p $project -r $version cms2build
592
593To create the $pbdefdir/$project/pbdelivery/$project-$version.{,pbconf}.tar.gz files, the $version-$projtag.yml and pbrc.yml files in the same directory.
594
595=back
596
597=head2 build2pkg
598
599The build2pkg command takes the tar files created in the cms2build step and attempts to build binary packages for your current operating system. There are two steps:
600
601=over 4
602
603=item Update your filters and build files.
604
605You probably need to edit the files describing the build steps in one of the $pbdefdir/$project/pbconf/$version/$project/{deb,rpm,pkg} directories and the filters in $pbdefdir/$project/pbconf/$version/pbfilter. Note that you can define additional filters and transformations in the filters files. The build files will be filtered by the filters defined in the filters files to generate the inputs to the build step. Therefore, if you change those files, you need to re-run the cms2build step.
606
607=item Build the package.
608
609Then you can run a command like:
610
611 % pb -p $project -r $version build2pkg
612
613To create the files in $project/pbbuild that comprise your binary package(s).
614
615=back
616
617=head2 newve
618 
619The newve command creates a new virtual environment, i.e. a chrooted OS for building packages. Using a virtual environment is an efficient way to build packages on a related set of operating systems. The OS's have to be related because the kernel will be shared.  Steps:
620
621=over 4
622
623=item Update ~/.pbrc.yml
624
625Update your ~/.pbrc.yml file to specify the vepath, velist, velogin, and vetype variables, e.g.:
626
627  vepath:
628    default: $ENV{'HOME'}/cache-project-builder/chroot
629  velist:
630    default: debian-6.0-i386
631  velogin:
632    default: pb
633  vetype:
634    default: chroot
635
636If you are building for rpm style OS's, update the verpmtype option, and install the appropriate tool.
637
638  verpmtype:
639    default: rpmbootstrap
640
641You may also choose to specify a mirror for the OS packages, and optionally http/ftp proxies.  You can specify the proxies either through environment variables ($http_proxy/$https_proxy/$ftp_proxy) or in the configuration file. The configuration file will be used if no corresponding environment variable has been set. For example, for debian and with a local squid proxy:
642
643  rbsmirrorsrv:
644    debian: http://mirrors1.kernel.org/debian/
645  http_proxy:
646    default: http://localhost:3128/
647  https_proxy:
648    default: http://localhost:3128/
649  ftp_proxy:
650    default: http://localhost:3128/
651
652=item Run the cms2build command
653
654If you have deleted your $package/pbdelivery directory, re-run the cms2build command as in the earlier step. This step is necessary to generate the I<package>/pbdelivery/pbrc.yml file.
655
656=item Create the new virtual environment
657
658Initialize the new operating system. This step will install the core OS packages for the virtual environment, e.g.:
659
660 % pb -v -p $project -m debian-6.0-i386 newve
661
662=back
663
664=head2 setupve
665
666The setupve command prepares a virtual environment for use by project builder. In particular it installs project-builder from the packages into the virtual environment. Two sub-steps are necessary:
667
668=over 4
669
670=item Update $project.yml
671
672You need to have a sshhost entry for setupve to work, so add one, even an invalid one, e.g.:
673
674  sshhost:
675    $project: foo.example.org
676
677=item Setup the virtual environment
678
679 % pb -v -p $project -m debian-6.0-i386 setupve
680
681If you prefer to install the current SVN version of project builder, you can substitute the setupve option by the sbx2setupv one.
682
683=back
684
685=head2 build2ve
686
687The build2ve command is similar to the build2pkg command in that it will take the sources created by cms2build and turn them into binary packages. The command has two differences. First, it creates the packages in a virtual environment, i.e. the one made by an earlier setupve setup. Second it copies the resulting packages to a repository and builds the repository meta-data needed.
688
689Three sub-steps are needed:
690
691=over 4
692
693=item Update $project.yml
694
695You need to have a valid sshdir and sshhost entry for build2ve to work, so add them. Note that you need to be able to ssh from the host you run the command on to the repository host, preferably without needing to type in a password, so using ssh-agent or having a special passwordless project-builder ssh key will make this step easier.
696
697  sshhost:
698    $project: localhost
699  sshdir:
700    $project: $home/cache-project-builder/repos
701
702You may also need to specify additional repository files to use or rpms to install. Note the URL for repositories is not the URL of the repository, but the URL of a file that can be put in the yum.repos.d or apt.sources.d directory.
703
704  addrepo:
705    centos-5-i386: http://localhost/pb/centos-extras.repo,http://mirror.centos.org/centos/5/extras/i386/RPMS/chrpath-0.13-3.el5.centos.i386.rpm
706
707=item Update your filters and build files
708
709You may need to update your filter files as in the build2pkg step if you are building for a new OS or architecture.
710
711=item Build the packages and copy them to the repository
712
713 % pb -v -p $project -m debian-6.0-i386 build2ve
714
715=back
716
717*Debugging:* If the build fails (and you did not specify the --no-stop-on-error) option, then the virtual environment and scripts should still be present and configured to build the package. You can run a command like 'sudo setarch i386 chroot $path bash' in order to get into the environment. In your log you should see a command like that. From there you can go into the /home/pb directory as the pb user and run the same style of pb commands as you did when doing build2pkg. This will help you figure out what has gone wrong in the build in the virtual environment.
718
719=head1 AUTHORS
720
721The Project-Builder.org team L<http://trac.project-builder.org/> lead by Bruno Cornec L<mailto:bruno@project-builder.org>.
722
723=head1 COPYRIGHT
724
725Project-Builder.org is distributed under the GPL v2.0 license
726described in the file C<COPYING> included with the distribution.
727
728=cut
729
730# ---------------------------------------------------------------------------
731
732my ($projectbuilderver,$projectbuilderrev,$projectbuilderconfver) = pb_version_init();
733my $appname = "pb";
734
735# Initialize the syntax string
736
737pb_syntax_init("$appname (aka project-builder.org) Version $projectbuilderver-$projectbuilderrev Conf $projectbuilderconfver\n");
738
739GetOptions("help|?|h+" => \$opts{'h'}, 
740        "account|a=s" => \$opts{'a'},
741        "force|f" => \$opts{'f'},
742        "nographic|g" => \$opts{'g'},
743        "image|i=s" => \$opts{'i'},
744        "log-files|l=s" => \$opts{'l'},
745        "machines|mock|m=s" => \$opts{'m'},
746        "man" => \$opts{'man'},
747        "port|P=i" => \$opts{'P'},
748        "project|p=s" => \$opts{'p'},
749        "quiet|q" => \$opts{'q'},
750        "revision|r=s" => \$opts{'r'},
751        "rebuild" => \$opts{'rebuild'},
752        "snapshot|S" => \$opts{'S'},
753        "script|s=s" => \$opts{'s'},
754        "type|T:s" => \$opts{'T'},
755        "target|t:s" => \$opts{'t'},
756        "version" => \$opts{'version'},
757        "V=s" => \$opts{'V'},
758        "verbose|v+" => \$opts{'v'},
759        "keep|k" => \$opts{'k'},
760        "stop-on-error!" => \$Global::pb_stop_on_error,
761) || pb_syntax(-1,0);
762
763if (defined $opts{'l'}) {
764    open(pbLOG,"> $opts{'l'}") || die "Unable to log to $opts{'l'}: $!";
765    $pbLOG = \*pbLOG;
766    $pbdebug = 0  if ($pbdebug == -1);
767    }
768pb_log_init($pbdebug, $pbLOG);
769
770if (defined $opts{'h'}) {
771    pb_syntax(0,1);
772}
773if (defined $opts{'version'}) {
774    pb_syntax(0,0);
775}
776if (defined $opts{'man'}) {
777    pb_syntax(0,2);
778}
779if (defined $opts{'v'}) {
780    $pbdebug = $opts{'v'};
781}
782if (defined $opts{'f'}) {
783    $pbforce=1;
784}
785if (defined $opts{'q'}) {
786    $pbdebug=-1;
787}
788if (defined $opts{'S'}) {
789    $pbsnap=1;
790}
791if (defined $opts{'k'}) {
792    $pbkeep=1;
793}
794pb_display_init("text","");
795
796# Handle root of the project if defined
797if (defined $opts{'r'}) {
798    $ENV{'PBROOTDIR'} = $opts{'r'};
799}
800# Handle virtual machines if any
801if (defined $opts{'m'}) {
802    $ENV{'PBV'} = $opts{'m'};
803}
804if (defined $opts{'s'}) {
805    $pbscript{'default'} = $opts{'s'};
806    die "option -s requires a script having a full path name" if ($pbscript{'default'} !~ /^\//);
807}
808if (defined $opts{'a'}) {
809    $ENV{'PBACCOUNT'} = $opts{'a'};
810    die "option -a requires a -s script option" if (not defined $pbscript{'default'});
811}
812if (defined $opts{'P'}) {
813    $pbport = $opts{'P'};
814}
815if (defined $opts{'V'}) {
816    $newver = $opts{'V'};
817}
818if (defined $opts{'i'}) {
819    $pbimage = $opts{'i'};
820}
821if (defined $opts{'t'}) {
822    $pbtarget = $opts{'t'};
823}
824if (defined $opts{'T'}) {
825    $vetype = $opts{'T'};
826}
827
828# Get Action
829$action = shift @ARGV;
830die pb_syntax(-1,1) if (not defined $action);
831
832my ($filteredfiles, $supfiles, $defpkgdir, $extpkgdir);
833my $pbinit = undef;
834$pbinit = 1 if ($action =~ /^newproj$/);
835
836# Handles project name if any
837# And get global params
838($filteredfiles, $supfiles, $defpkgdir, $extpkgdir) = pb_env_init($opts{'p'},$pbinit,$action,$pbkeep);
839
840#
841# Check for command requirements
842#
843my ($req,$opt,$pbpara,$pbshowsudo,$pbstoponerr) = pb_conf_get_if("oscmd","oscmdopt","pbparallel","pbshowsudo","pbstoponerr");
844pb_check_requirements($req,$opt,$appname);
845
846$Global::pb_show_sudo = 1 if ((defined $pbshowsudo) && (defined $pbshowsudo->{$ENV{'PBPROJ'}}) && ($pbshowsudo->{$ENV{'PBPROJ'}} =~ /true/oi));
847$Global::pb_stop_on_error = 1 if ((defined $pbstoponerr) && (defined $pbstoponerr->{$ENV{'PBPROJ'}}) && ($pbstoponerr->{$ENV{'PBPROJ'}} =~ /true/oi));
848
849#
850# Check if we can launch some actions in // with Parallel::ForkManager
851#
852my $pbparallel = $pbpara->{$appname} if (defined $pbpara);
853if (not defined $pbparallel) {
854    eval
855    {
856        require Sys::CPU;
857        Sys::CPU->import();
858    };
859    if ($@) {
860        # Sys::CPU not found, defaulting to 1
861        pb_log(1,"ADVISE: Install Sys::CPU to benefit from automatic parallelism optimization.\nOr use pbparallel in your pb.yml file\nOnly 1 process at a time for the moment\n");
862        $pbparallel = 1;
863    } else {
864        # Using the number of cores
865        $pbparallel = Sys::CPU::cpu_count();
866        pb_log(1,"Using parallel mode with $pbparallel processes\n");
867    }
868}
869
870eval
871{
872    require Parallel::ForkManager;
873    Parallel::ForkManager->import();
874};
875# Parallel::ForkManager not found so no // actions
876if ($@) {
877    $pbparallel = undef;
878    pb_log(1,"ADVISE: Install Parallel::ForkManager to benefit from automatic parallelism optimization.\nOnly 1 process at a time for the moment\n");
879}
880
881pb_log(0,"Project: $ENV{'PBPROJ'}\n");
882pb_log(0,"Action: $action\n");
883
884my $do_install = undef;
885# Possibility to add a 2ins to some commands to launch automatic installation
886$do_install = 1 if ($action =~ /2ins$/);
887
888# Act depending on action
889if ($action =~ /^cms2build$/) {
890    pb_cms2build("CMS");
891} elsif ($action =~ /^sbx2build$/) {
892    pb_cms2build("SandBox");
893} elsif ($action =~ /^build2prep$/) {
894    pb_build2prep();
895} elsif ($action =~ /^sbx2prep$/) {
896    pb_cms2build("SandBox");
897    pb_build2prep();
898} elsif ($action =~ /^build2pkg/) {
899    pb_build2pkg($do_install);
900} elsif ($action =~ /^cms2pkg/) {
901    pb_cms2build("CMS");
902    pb_build2pkg($do_install);
903} elsif ($action =~ /^sbx2pkg/) {
904    pb_cms2build("SandBox");
905    pb_build2pkg($do_install);
906} elsif ($action =~ /^build2ssh$/) {
907    pb_build2ssh();
908} elsif ($action =~ /^cms2ssh$/) {
909    pb_cms2build("CMS");
910    pb_build2ssh();
911} elsif ($action =~ /^sbx2ssh$/) {
912    pb_cms2build("SandBox");
913    pb_build2ssh();
914} elsif ($action =~ /^pkg2ssh$/) {
915    pb_pkg2ssh();
916} elsif ($action =~ /^build2rm$/) {
917    pb_build2v("rm","build");
918} elsif ($action =~ /^build2ve$/) {
919    pb_build2v("ve","build");
920} elsif ($action =~ /^build2vm$/) {
921    pb_build2v("vm","build");
922} elsif ($action =~ /^sbx2preprm$/) {
923    pb_cms2build("SandBox");
924    pb_build2v("rm","prep");
925} elsif ($action =~ /^sbx2prepve$/) {
926    pb_cms2build("SandBox");
927    pb_build2v("ve","prep");
928} elsif ($action =~ /^sbx2prepvm$/) {
929    pb_cms2build("SandBox");
930    pb_build2v("vm","prep");
931} elsif ($action =~ /^preprm$/) {
932    pb_build2v("rm","prep");
933} elsif ($action =~ /^prepve$/) {
934    pb_build2v("ve","prep");
935} elsif ($action =~ /^prepvm$/) {
936    pb_build2v("vm","prep");
937} elsif ($action =~ /^cms2rm$/) {
938    pb_cms2build("CMS");
939    pb_build2v("rm","build");
940} elsif ($action =~ /^cms2ve$/) {
941    pb_cms2build("CMS");
942    pb_build2v("ve","build");
943} elsif ($action =~ /^sbx2rm$/) {
944    pb_cms2build("SandBox");
945    pb_build2v("rm","build");
946} elsif ($action =~ /^sbx2ve$/) {
947    pb_cms2build("SandBox");
948    pb_build2v("ve","build");
949} elsif ($action =~ /^cms2vm$/) {
950    pb_cms2build("CMS");
951    pb_build2v("vm","build");
952} elsif ($action =~ /^sbx2vm$/) {
953    pb_cms2build("SandBox");
954    pb_build2v("vm","build");
955} elsif ($action =~ /^launchvm$/) {
956    pb_parallel_launchv(undef,"vm",undef,3,$pbimage);
957} elsif ($action =~ /^launchve$/) {
958    pb_parallel_launchv(undef,"ve",undef,3,$pbimage);
959} elsif ($action =~ /^script2vm$/) {
960    die "action script2vm requires a -s script option" if (not defined $pbscript{'default'});
961    pb_script2v(\%pbscript,"vm");
962} elsif ($action =~ /^script2ve$/) {
963    die "action script2ve requires a -s script option" if (not defined $pbscript{'default'});
964    pb_script2v(\%pbscript,"ve");
965} elsif ($action =~ /^script2rm$/) {
966    die "action script2rm requires a -s script option" if (not defined $pbscript{'default'});
967    pb_script2v(\%pbscript,"rm");
968} elsif ($action =~ /^newver$/) {
969    pb_newver();
970} elsif ($action =~ /^sbx2docker$/) {
971    my $savproj = $ENV{'PBPROJ'};
972    my $pkg;
973    my @pkg;
974    if ($#ARGV == -1) {
975        $pkg = pb_cms_get_pkg($defpkgdir,$extpkgdir);
976        @pkgs = @$pkg;
977    } else {
978        @pkgs = @ARGV;
979    }
980    #pb_log(1,"Packages to print:".Dumper(@pkgs)."\n");
981   
982    # Get the list of all VE we need to work on
983    my ($vm,$all);
984    if (not defined $ENV{'PBV'}) {
985        ($vm,$all) = pb_get2v("ve");
986    } else {
987        @$vm = split(/,/,$ENV{'PBV'});
988    }
989   
990    # TODO: parallelize
991    foreach my $v (@$vm) {
992        my $done = 0;
993        my $pbos = pb_distro_get_context($v);
994        # Docker images are (for now !) regular and use distro:ver
995        $pbimage = "$pbos->{'name'}".":$pbos->{'version'}";
996        # the first steps needs to be done with the pb project
997        $ENV{'PBPROJ'} = $appname;
998        my $found = pb_ve_docker_get_image("$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}");
999        if (($pbforce == 1) || (not defined $found)) {
1000            # pb -p pb -m distro-ver-arch newve -i distro:ver
1001            pb_log(0,"Docker container $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'} not found, creating it\n");
1002            pb_parallel_launchv(undef,"ve",undef,0,$pbimage);
1003        }
1004        $found = pb_ve_docker_get_image("$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-pb");
1005        if (($pbforce == 1) || (not defined $found)) {
1006            pb_log(0,"Docker container $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-pb not found, creating it\n");
1007            # pb -p pb -m distro-ver-arch setupve
1008            if ($savproj eq "pb") {
1009                pb_cms2build("SandBox");
1010                $done = 1;
1011                pb_setup2v("ve","SandBox");
1012            } else {
1013                pb_setup2v("ve");
1014            }
1015        }
1016        # Now back to our original project
1017        $ENV{'PBPROJ'} = $savproj;
1018        $found = pb_ve_docker_get_image("$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-pb-$ENV{'PBPROJ'}");
1019        if (($pbforce == 1) || (not defined $found)) {
1020            pb_log(0,"Docker container $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-pb-$ENV{'PBPROJ'} not found, creating it\n");
1021            # pb -p pb -m distro-ver-arch prepve
1022            if ($savproj eq "pb") {
1023                pb_cms2build("SandBox");
1024                $done = 1;
1025            }
1026            pb_build2v("ve","prep");
1027        }
1028        pb_log(0,"Instantiating a Docker container from $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-pb-$ENV{'PBPROJ'}\n");
1029        if (($savproj ne "pb") || ($done == 0)) {
1030            pb_cms2build("SandBox");
1031        }
1032        pb_build2v("ve","build");
1033    }
1034} elsif ($action =~ /^newve$/) {
1035    pb_parallel_launchv(undef,"ve",undef,0,$pbimage);
1036} elsif ($action =~ /^newvm$/) {
1037    pb_parallel_launchv(undef,"vm",undef,0,$pbimage);
1038    pb_log(0, "Please ensure that sshd is running in your VM by default\n");
1039    pb_log(0, "and that it allows remote root login (PermitRootLogin yes in /etc/ssh/sshd_config)\n");
1040    pb_log(0, "Also ensure that network is up, firewalling correctly configured\n");
1041    pb_log(0, "and perl, Data::Dumper, tar, sudo, ntpdate and scp/ssh installed\n");
1042    pb_log(0, "You should then be able to login with ssh -p VMPORT root\@localhost (if VM started with pb)\n");
1043} elsif ($action =~ /^setuprm$/) {
1044    pb_setup2v("rm");
1045} elsif ($action =~ /^setupve$/) {
1046    pb_setup2v("ve");
1047} elsif ($action =~ /^setupvm$/) {
1048    pb_setup2v("vm");
1049} elsif ($action =~ /^sbx2setuprm$/) {
1050    die "This feature is limited to the pb project" if ($ENV{'PBPROJ'} ne $appname);
1051    pb_cms2build("SandBox");
1052    pb_setup2v("rm","SandBox");
1053} elsif ($action =~ /^sbx2setupve$/) {
1054    die "This feature is limited to the pb project" if ($ENV{'PBPROJ'} ne $appname);
1055    pb_cms2build("SandBox");
1056    pb_setup2v("ve","SandBox");
1057} elsif ($action =~ /^sbx2setupvm$/) {
1058    die "This feature is limited to the pb project" if ($ENV{'PBPROJ'} ne $appname);
1059    pb_cms2build("SandBox");
1060    pb_setup2v("vm","SandBox");
1061} elsif ($action =~ /^build2setuprm$/) {
1062    die "This feature is limited to the pb project" if ($ENV{'PBPROJ'} ne $appname);
1063    pb_setup2v("rm","SandBox");
1064} elsif ($action =~ /^build2setupve$/) {
1065    die "This feature is limited to the pb project" if ($ENV{'PBPROJ'} ne $appname);
1066    pb_setup2v("ve","SandBox");
1067} elsif ($action =~ /^build2setupvm$/) {
1068    die "This feature is limited to the pb project" if ($ENV{'PBPROJ'} ne $appname);
1069    pb_setup2v("vm","SandBox");
1070} elsif ($action =~ /^updaterm$/) {
1071    pb_update2v("rm");
1072} elsif ($action =~ /^updateve$/) {
1073    pb_update2v("ve");
1074} elsif ($action =~ /^updatevm$/) {
1075    pb_update2v("vm");
1076} elsif ($action =~ /^snapve$/) {
1077    pb_snap2v("ve");
1078} elsif ($action =~ /^snapvm$/) {
1079    pb_snap2v("vm");
1080} elsif ($action =~ /^test2pkg$/) {
1081    pb_test2pkg();
1082} elsif ($action =~ /^test2rm$/) {
1083    pb_build2v("rm","test");
1084} elsif ($action =~ /^test2ve$/) {
1085    pb_build2v("ve","test");
1086} elsif ($action =~ /^test2vm$/) {
1087    pb_build2v("vm","test");
1088} elsif ($action =~ /^newproj$/) {
1089    # Nothing to do - already done in pb_env_init
1090} elsif ($action =~ /^updateconf$/) {
1091    foreach my $f (@ARGV) {
1092        my $fyml = $f;
1093        $fyml =~ s/\.pbf/.yml/;
1094        $fyml =~ s/\.pb/.yml/;
1095        $fyml =~ s/\.pbrc/.pbrc.yml/;
1096        pb_vcs_conf_update_v0($f,$fyml);
1097    }
1098} elsif ($action =~ /^getconf$/) {
1099    my $pbos = pb_distro_get_context($ENV{'PBV'});
1100    pb_log(1,"Arguments to print:".Dumper(@ARGV)."\n");
1101    pb_distro_conf_print($pbos,@ARGV);
1102} elsif ($action =~ /^getvar$/) {
1103    my $pbos = pb_distro_get_context($ENV{'PBV'});
1104    my $pkg;
1105    my @pkg;
1106    if ($#ARGV == -1) {
1107        $pkg = pb_cms_get_pkg($defpkgdir,$extpkgdir);
1108        @pkgs = @$pkg;
1109    } else {
1110        @pkgs = @ARGV;
1111    }
1112    pb_log(1,"Packages to print:".Dumper(@pkgs)."\n");
1113    pb_filter_var_print($pbos,@pkgs);
1114} elsif ($action =~ /^clean$/) {
1115    pb_clean();
1116} elsif ($action =~ /^announce$/) {
1117    # For announce only. Require avoids the systematic load of these modules
1118    require DBI;
1119    require DBD::SQLite;
1120
1121    pb_announce("Announce");
1122} elsif ($action =~ /^cleanssh$/) {
1123    pb_announce("Clean");
1124} elsif ($action =~ /^checkssh$/) {
1125    pb_announce("Check");
1126} elsif ($action =~ /^checkps$/) {
1127    my ($ptr2) = pb_conf_get("vmcmd");
1128    my $vmcmd = $ptr2->{$ENV{'PBPROJ'}};
1129    my ($vexist,$vmmport) = pb_check_ps($vmcmd,$ENV{'PBV'});
1130    if (! $vexist) {
1131        pb_log(0,"No VM found for $ENV{'PBV'}\n");
1132    } else {
1133        pb_log(0,"Found an existing VM for $ENV{'PBV'} PID: $vexist - SSH port: $vmmport\n");
1134    }
1135} elsif ($action =~ /^sbx2webpkg$/) {
1136    require DBI;
1137    require DBD::SQLite;
1138
1139    pb_cms2build("SandBox","Web");
1140} elsif ($action =~ /^sbx2webssh$/) {
1141    require DBI;
1142    require DBD::SQLite;
1143
1144    pb_cms2build("SandBox","Web");
1145    pb_send2target("Web");
1146} elsif ($action =~ /^cms2webpkg$/) {
1147    require DBI;
1148    require DBD::SQLite;
1149
1150    pb_cms2build("CMS","Web");
1151} elsif ($action =~ /^cms2webssh$/) {
1152    require DBI;
1153    require DBD::SQLite;
1154
1155    pb_cms2build("CMS","Web");
1156    pb_send2target("Web");
1157} else {
1158    pb_log(0,"\'$action\' is not available\n");
1159    pb_syntax(-2,1);
1160}
1161
1162pb_exit();
1163
1164sub pb_cms2build {
1165
1166    my $param = shift;
1167    my $web = shift;
1168
1169    my $pkg;
1170    my @pkgs;
1171    my $webdir;
1172
1173    my %pkgs;
1174    my $pb;             # Structure to store conf info
1175
1176    die "pb_cms2build requires a parameter: SandBox or CMS" if (not defined $param);
1177
1178    # If Website, then pkg is only the website
1179    if (defined $web) {
1180        ($webdir) = pb_conf_get("webdir");
1181        pb_log(2,"webdir: ".Dumper($webdir)."\n");
1182        $pkgs[0] = $webdir->{$ENV{'PBPROJ'}};
1183        $extpkgdir = $webdir;
1184        pb_log(0,"Package: $pkgs[0]\n");
1185    } else {
1186        $pkg = pb_cms_get_pkg($defpkgdir,$extpkgdir);
1187        @pkgs = @$pkg;
1188    }
1189
1190    my ($scheme, $uri) = pb_vcs_init($pbinit,$param);
1191
1192    # We need 2 lines here
1193    my ($pkgv, $pkgt, $testver) = pb_conf_get_if("pkgver","pkgtag","testver");
1194
1195    # declare packager and repo for filtering
1196    my ($tmp1, $tmp2) = pb_conf_get("pbpackager","pbrepo");
1197    $ENV{'PBPACKAGER'} = $tmp1->{$ENV{'PBPROJ'}};
1198    $ENV{'PBREPO'} = $tmp2->{$ENV{'PBPROJ'}};
1199    my ($delivery) = pb_conf_get_if("delivery");
1200    $delivery->{$ENV{'PBPROJ'}} = "" if (not defined $delivery->{$ENV{'PBPROJ'}});
1201
1202    # If we deal with a test dir, we want to keep the date in tar file and dir name
1203    my $pbextdirpkg = "";
1204    my $pbextdir = "";
1205    if ((defined $testver) && (defined $testver->{$ENV{'PBPROJ'}}) && ($testver->{$ENV{'PBPROJ'}} =~ /true/i)) {
1206        $pbextdirpkg = strftime("%Y%m%d%H%M%S", @date);
1207        $pbextdir = ".0.".$pbextdirpkg;
1208    }
1209
1210    foreach my $pbpkg (@pkgs) {
1211        $ENV{'PBPKG'} = $pbpkg;
1212        $pbrev = $ENV{'PBREVISION'};
1213
1214        if ((defined $pkgv) && (defined $pkgv->{$pbpkg})) {
1215            $pbver = $pkgv->{$pbpkg};
1216        } else {
1217            $pbver = $ENV{'PBPROJVER'};
1218        }
1219        # If it's a test version, then build a tag to fit distributions need
1220        if ((defined $testver) && (defined $testver->{$ENV{'PBPROJ'}}) && ($testver->{$ENV{'PBPROJ'}} =~ /true/i)) {
1221            my $letter = "";
1222            if ($scheme =~ /svn/) {
1223                $letter = "s";
1224            } elsif ($scheme =~ /cvs/) {
1225                $letter = "c";
1226            } elsif ($scheme =~ /svk/) {
1227                $letter = "k";
1228            } elsif ($scheme =~ /hg/) {
1229                $letter = "h";
1230            } elsif ($scheme =~ /git/) {
1231                $letter = "g";
1232            } else {
1233                $letter = "f";
1234            }
1235            $pbtag = "0.$pbextdirpkg.$letter$pbrev";
1236            $ENV{'PBPROJTAG'} = $pbtag;
1237        } elsif ((defined $pkgt) && (defined $pkgt->{$pbpkg})) {
1238            $pbtag = $pkgt->{$pbpkg};
1239        } else {
1240            $pbtag = $ENV{'PBPROJTAG'};
1241        }
1242
1243        pb_log(0,"\n");
1244        pb_log(0,"Management of $pbpkg $pbver-$pbtag\n");
1245        die "Unable to get env var PBDESTDIR" if (not defined $ENV{'PBDESTDIR'});
1246
1247        my $dest = "$ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir";
1248        # Create the structure if needed
1249        pb_mkdir_p($dest);
1250        # Clean up dest if necessary. The export will recreate it
1251        pb_rm_rf($dest) if (-d $dest);
1252
1253        # Export CMS tree for the concerned package to dest
1254        # And generate some additional files
1255        $OUTPUT_AUTOFLUSH=1;
1256
1257        # computes in which dir we have to work
1258        my $dir = $defpkgdir->{$pbpkg};
1259        $dir = $extpkgdir->{$pbpkg} if (not defined $dir);
1260        $dir = $webdir->{$ENV{'PBPROJ'}} if (defined $web);
1261        die "Variable \$dir not defined. Check the package name first.\nIf this is the one expected, please report to dev team with log of a verbose run and this info ".Dumper($defpkgdir,$extpkgdir,$webdir) if (not defined $dir);
1262        pb_log(2,"def:".Dumper($defpkgdir)."ext: ".Dumper($extpkgdir)."dir: $dir\n");
1263
1264        # Exporting content from CMS
1265        my $sourcedir = undef;
1266        my $sourceuri = $uri;
1267        if ($param eq "SandBox") {
1268            # Point to the local instance
1269            $sourcedir = "$ENV{'PBDIR'}/$dir";
1270        } else {
1271            # Get it from a subdir of the URI with same version as localy but different root
1272            # Only if using a real CMS
1273            my ($scheme, $account, $host, $port, $path) = pb_get_uri($uri);
1274            if (($scheme !~ /^file/) && ($scheme !~ /^(ht|f)tp/)) {
1275                $sourceuri = "$ENV{'PBDIR'}/$dir"; 
1276                $sourceuri =~ s|^$ENV{'PBPROJDIR'}/|$uri/|;
1277            }
1278        }
1279        my $preserve = pb_vcs_export($sourceuri,$sourcedir,$dest);
1280
1281        # Generated fake content for test versions to speed up stuff
1282        my $chglog;
1283
1284        # Get project info on authors and log file
1285        # TODO: Make it CMS aware
1286        $chglog = "$ENV{'PBROOTDIR'}/$pbpkg/pbcl";
1287        $chglog = "$ENV{'PBROOTDIR'}/pbcl" if (! -f $chglog);
1288        $chglog = undef if (! -f $chglog);
1289
1290        # TODO: Make it CMS aware
1291        my $authors = "$ENV{'PBROOTDIR'}/$pbpkg/pbauthors";
1292        $authors = "$ENV{'PBROOTDIR'}/pbauthors" if (! -f $authors);
1293        $authors = "/dev/null" if (! -f $authors);
1294
1295        # Extract cms log history and store it
1296        if ((defined $chglog) && (! -f "$dest/NEWS")) {
1297            pb_log(2,"Generating NEWS file from $chglog\n");
1298            copy($chglog,"$dest/NEWS") || die "Unable to create $dest/NEWS";
1299        }
1300        pb_cms_log($scheme,"$ENV{'PBDIR'}/$dir",$dest,$chglog,$authors,$testver);
1301
1302        my %build;
1303        # We want to at least build for the underlying distro
1304        # except if a target was given, in which case we only build for it
1305        # if -t was passed without target then build for the native distro.
1306        my $pbos = pb_distro_get_context($pbtarget);
1307        my $tmpl = pb_get_distros($pbos,$pbtarget);
1308
1309        # Setup $pb structure to allow filtering later on, on files using that structure
1310        $pb->{'tag'} = $pbtag;
1311        $pb->{'rev'} = $pbrev;
1312        $pb->{'ver'} = $pbver;
1313        $pb->{'pkg'} = $pbpkg;
1314        $pb->{'suf'} = $pbos->{'suffix'};
1315        $pb->{'realpkg'} = $pbpkg;
1316        $pb->{'date'} = $pbdate;
1317        $pb->{'defpkgdir'} = $defpkgdir;
1318        $pb->{'extpkgdir'} = $extpkgdir;
1319        $pb->{'chglog'} = $chglog;
1320        $pb->{'extdir'} = $pbextdir;
1321        $pb->{'packager'} = $ENV{'PBPACKAGER'};
1322        $pb->{'proj'} = $ENV{'PBPROJ'};
1323        $pb->{'repo'} = "$ENV{'PBREPO'}/$delivery->{$ENV{'PBPROJ'}}";
1324        $pb->{'patches'} = ();
1325        $pb->{'sources'} = ();
1326   
1327        my $tmpd = "$ENV{'PBTMP'}/cms.$$";
1328        pb_mkdir_p($tmpd) if (defined $pbparallel);
1329
1330        # Get only all.yml filter at this stage for pbinit
1331        my $ptr = pb_get_filters($pbpkg);
1332
1333        # Do not do that for website
1334        if (not defined $web) {
1335            my %virt;
1336            # De-duplicate similar VM/VE/RM
1337            foreach my $d (split(/,/,$tmpl)) {
1338                # skip ill-formatted vms (name-ver-arch)
1339                next if ($d !~ /-/);
1340                $virt{$d} = $d;
1341            }
1342
1343            # Try to use // processing here
1344            my $all_ok = 1;
1345            my $pm = new Parallel::ForkManager($pbparallel) if (defined $pbparallel);
1346            $pm->run_on_finish(sub { my ($pid, $code, $id, $signal, $dump) = @_;
1347                                    $all_ok = 0 unless (($code == 0) && ($signal == 0) && ($dump == 0)); }) if (defined $pbparallel);
1348
1349            pb_log(0,"Preparing delivery ...\n");
1350            foreach my $v (keys %virt) {
1351                $pm->start and next if (defined $pbparallel);
1352
1353                # Distro context
1354                my $pbos = pb_distro_get_context($v);
1355   
1356                $pb->{'pbos'} = $pbos;
1357                $pb->{'suf'} = $pbos->{'suffix'};
1358                pb_log(3,"DEBUG: pb: ".Dumper($pb)."\n");
1359
1360                # Get all filters to apply
1361                $ptr = pb_get_filters($pbpkg,$pbos);
1362   
1363                pb_log(2,"DEBUG Filtering PBDATE => $pbdate, PBTAG => $pbtag, PBVER => $pbver\n");
1364   
1365                # We need to compute the real name of the package
1366                my $pbrealpkg = pb_cms_get_real_pkg($pbpkg,$pbos->{'type'});
1367                $pb->{'realpkg'} = $pbrealpkg;
1368                pb_log(2,"Virtual package $pbpkg has a real package name of $pbrealpkg on $pbos->{'name'}-$pbos->{'version'}\n") if ($pbrealpkg ne $pbpkg);
1369   
1370                # Filter build files from the less precise up to the most with overloading
1371                # Filter all files found, keeping the name, and generating in dest
1372   
1373                # Find all build files first relatively to PBROOTDIR
1374                # Find also all specific files referenced in the .yml conf file
1375                my %bfiles = ();
1376                my %pkgfiles = ();
1377                # Used in Filter.pm by pb_filter_file
1378
1379                $build{$v} = "no";
1380                if (-d "$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'os'}") {
1381                    pb_list_bfiles("$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'os'}",$pbpkg,\%bfiles,\%pkgfiles,$supfiles);
1382                    $build{$v} = "yes";
1383                } 
1384                if (-d "$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'type'}") {
1385                    pb_list_bfiles("$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'type'}",$pbpkg,\%bfiles,\%pkgfiles,$supfiles);
1386                    $build{$v} = "yes";
1387                } 
1388                if (-d "$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'family'}") {
1389                    pb_list_bfiles("$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'family'}",$pbpkg,\%bfiles,\%pkgfiles,$supfiles);
1390                    $build{$v} = "yes";
1391                } 
1392                if (-d "$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'name'}") {
1393                    pb_list_bfiles("$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'name'}",$pbpkg,\%bfiles,\%pkgfiles,$supfiles);
1394                    $build{$v} = "yes";
1395                } 
1396                if (-d "$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'name'}-$pbos->{'version'}") {
1397                    pb_list_bfiles("$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'name'}-$pbos->{'version'}",$pbpkg,\%bfiles,\%pkgfiles,$supfiles);
1398                    $build{$v} = "yes";
1399                } 
1400                if (-d "$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}") {
1401                    pb_list_bfiles("$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}",$pbpkg,\%bfiles,\%pkgfiles,$supfiles);
1402                    $build{$v} = "yes";
1403                }
1404                pb_log(2,"DEBUG($v) bfiles: ".Dumper(\%bfiles)."\n");
1405   
1406                if ($build{$v} ne "no") {
1407                    # Apply now all the filters on all the files concerned
1408                    # destination dir depends on the type of file
1409                    # For patch support
1410                    # TODO: Make it CMS aware
1411                    $pb->{'patches'} = pb_list_sfiles("$ENV{'PBROOTDIR'}/$pbpkg/pbpatch", $pb->{'patches'}, $pbos, "$ENV{'PBROOTDIR'}/$pbpkg/pbextpatch");
1412                    pb_log(2,"DEBUG($v) patches: ".Dumper($pb->{'patches'})."\n");
1413                    # TODO: Make it CMS aware
1414                    $pb->{'sources'} = pb_list_sfiles("$ENV{'PBROOTDIR'}/$pbpkg/pbsrc", $pb->{'sources'}, $pbos, "$ENV{'PBROOTDIR'}/$pbpkg/pbextsrc");
1415                    pb_log(2,"DEBUG($v) sources: ".Dumper($pb->{'sources'})."\n");
1416   
1417                    if (defined $pb->{'patches'}->{$v}) {
1418                        # Filter potential patches (local + remote)
1419                        pb_mkdir_p("$dest/pbconf/$v/pbpatch");
1420                        # If Debian based distribution, then prepare what will be done at build time
1421                        my ($patchcmd,$patchopt);
1422                        if ($pbos->{'type'} eq "deb") {
1423                            ($patchcmd,$patchopt) = pb_distro_get_param($pbos,pb_conf_get_if("ospatchcmd","ospatchopt"));
1424                            open(SCRIPT,"> $dest/pbconf/$v/pbpatch/pbapplypatch") || die "Unable to create $dest/pbconf/$v/pbpatch/pbapplypatch";
1425                            print SCRIPT "#!/bin/bash\n";
1426                            print SCRIPT "set -x\n" if ($pbdebug gt 1);
1427                        }
1428                        foreach my $pf (split(/,/,$pb->{'patches'}->{$v})) {
1429                            my $pp = basename($pf);
1430                            pb_vcs_export($pf,undef,"$dest/pbconf/$v/pbpatch");
1431                            pb_filter_file_inplace($ptr,"$dest/pbconf/$v/pbpatch/$pp",$pb);
1432                            pb_system("gzip -9f $dest/pbconf/$v/pbpatch/$pp","","quiet");
1433                            if ($pbos->{'type'} eq "deb") {
1434                                # If Debian based distribution, then prepare what will be done at build time
1435                                # by applying the patches that will be available under the debian/patches dir
1436                                print SCRIPT "$patchcmd $patchopt \< debian/patches/$pp\n";
1437                            }
1438                        }
1439                        if ($pbos->{'type'} eq "deb") {
1440                            close(SCRIPT);
1441                            chmod 0755,"$dest/pbconf/$v/pbpatch/pbapplypatch";
1442                        }
1443                        #pb_system("cat $dest/pbconf/$v/pbpatch/pbapplypatch","APPLY","verbose");
1444                    }
1445                    if (defined $pb->{'sources'}->{$v}) {
1446                        pb_mkdir_p("$dest/pbconf/$v/pbsrc");
1447                        foreach my $pf (split(/,/,$pb->{'sources'}->{$v})) {
1448                            my $pp = basename($pf);
1449                            pb_vcs_export($pf,undef,"$dest/pbconf/$v/pbsrc");
1450                            pb_filter_file_inplace($ptr,"$dest/pbconf/$v/pbsrc/$pp",$pb);
1451                        }
1452                    }
1453                    # Filter build files at the end, as they depend on patches and sources
1454                    foreach my $f (keys %bfiles) {
1455                        pb_filter_file("$ENV{'PBROOTDIR'}/$bfiles{$f}",$ptr,"$dest/pbconf/$v/$f",$pb);
1456                    }
1457                    foreach my $f (keys %pkgfiles) {
1458                        pb_filter_file("$ENV{'PBROOTDIR'}/$pkgfiles{$f}",$ptr,"$dest/pbconf/$v/$f",$pb);
1459                    }
1460                }
1461
1462                if (defined $pbparallel) {
1463                    # Communicate results back to parent
1464                    my $str = "";
1465                    $str .= "build $v = $build{$v}\n" if (defined $build{$v});
1466                    $str .= "patches $v = $pb->{'patches'}->{$v}\n" if (defined $pb->{'patches'}->{$v});
1467                    $str .= "sources $v = $pb->{'sources'}->{$v}\n" if (defined $pb->{'sources'}->{$v});
1468                    pb_set_content("$tmpd/$$","$str");
1469                    $pm->finish;
1470                }
1471            }
1472            # In the parent, we need to get the result from the children
1473            $pm->wait_all_children if (defined $pbparallel);
1474            die "Aborting, one or more of the children failed." if ((not $all_ok) && ($Global::pb_stop_on_error));
1475            my $made = "";
1476            my %h = ();
1477            my %tmp;
1478            my %tmp2;
1479            my $pt;
1480            my $k;
1481
1482            foreach $k (<$tmpd/*>) {
1483                $made .= pb_get_content($k);
1484            }
1485            pb_rm_rf($tmpd);
1486            pb_log(3,"MADE:\n$made");
1487
1488            # Rebuild local hashes
1489            foreach $k (split(/\n/,$made)) {
1490                if ($k =~ /^\s*([A-z0-9-_]+)\s+([[A-z0-9-_.]+)\s*=\s*(.+)$/) {
1491                    $h{$1}->{$2}=$3;
1492                }
1493            }
1494            pb_log(2,"HASH: ".Dumper(%h));
1495
1496            # Patches
1497            pb_log(0,"Delivered and compressed patches ");
1498            $pt = $h{'patches'};
1499            foreach $k (keys %$pt) {
1500                foreach my $v1 (split(/,/,$pt->{$k})) {
1501                    $tmp{$v1} = "";
1502                }
1503            }
1504            if (keys %tmp) {
1505                foreach $k (keys %tmp) {
1506                    pb_log(0,"$k ");
1507                }
1508            } else {
1509                pb_log(0,"N/A");
1510            }
1511            pb_log(0,"\n");
1512
1513            # Sources
1514            pb_log(0,"Delivered additional sources ");
1515            $pt = $h{'sources'};
1516            foreach $k (keys %$pt) {
1517                foreach my $v1 (split(/,/,$pt->{$k})) {
1518                    $tmp2{$v1} = "";
1519                }
1520            }
1521            if (keys %tmp2) {
1522                foreach $k (keys %tmp2) {
1523                    pb_log(0,"$k ");
1524                }
1525            } else {
1526                pb_log(0,"N/A");
1527            }
1528            pb_log(0,"\n");
1529
1530            # Build files
1531            my @found;
1532            my @notfound;
1533            $pt = $h{'build'};
1534            foreach my $b (keys %$pt) {
1535                push @found,$b if ($pt->{$b} =~ /yes/);
1536                push @notfound,$b if ($pt->{$b} =~ /no/);
1537            }
1538            pb_log(0,"Build files have been generated for ... ".join(',',sort(@found))."\n") if (@found);
1539            pb_log(0,"No Build files found for ".join(',',sort(@notfound))."\n") if (@notfound);
1540
1541        } else {
1542            # Instead call News generation
1543            pb_web_news2html($dest);
1544            # And create an empty pbconf
1545            pb_mkdir_p("$dest/pbconf");
1546            # And prepare the pbscript to execute remotely
1547            open(SCRIPT,"> $ENV{'PBTMP'}/pbscript") || die "Unable to create $ENV{'PBTMP'}/pbscript";
1548            print SCRIPT "#!/bin/bash\n";
1549            print SCRIPT "#set -x\n";
1550            print SCRIPT "echo ... Extracting Website content\n";
1551            print SCRIPT "find . -type f | grep -Ev '^./$pbpkg-$pbver$pbextdir.tar.gz|^./pbscript' | xargs rm -f non-existent\n";
1552            print SCRIPT "find * -type d -depth | xargs rmdir 2> /dev/null \n";
1553            print SCRIPT "tar xfz $pbpkg-$pbver$pbextdir.tar.gz\n";
1554            print SCRIPT "mv $pbpkg-$pbver$pbextdir/* .\n";
1555            print SCRIPT "rm -f $pbpkg-$pbver$pbextdir.tar.gz\n";
1556            print SCRIPT "rmdir $pbpkg-$pbver$pbextdir\n";
1557            print SCRIPT "find . -type f -print0 | xargs -0 chmod 644\n";
1558            print SCRIPT "find . -type d -print0 | xargs -0 chmod 755\n";
1559            close(SCRIPT);
1560            chmod 0755,"$ENV{'PBTMP'}/pbscript";
1561        }
1562
1563        # Apply filters to the non-build files
1564        my $liste ="";
1565        if (defined $filteredfiles->{$pbpkg}) {
1566            foreach my $f (split(/,/,$filteredfiles->{$pbpkg})) {
1567                pb_filter_file_inplace($ptr,"$dest/$f",$pb);
1568                $liste = "$f $liste";
1569            }
1570        }
1571        pb_log(2,"Files ".$liste."have been filtered\n");
1572
1573        # TODO: Make it CMS aware
1574        # Execute the pbinit script if any
1575        if (-x "$ENV{'PBROOTDIR'}/$pbpkg/pbinit") {
1576            pb_filter_file("$ENV{'PBROOTDIR'}/$pbpkg/pbinit",$ptr,"$ENV{'PBTMP'}/pbinit",$pb);
1577            chmod 0755,"$ENV{'PBTMP'}/pbinit";
1578            pb_system("cd $dest ; $ENV{'PBTMP'}/pbinit","Executing init script from $ENV{'PBROOTDIR'}/$pbpkg/pbinit under $dest","verbose");
1579        }
1580
1581        # Do we have additional script to run to prepare the environement for the project ?
1582        # Then include it in the pbconf delivery
1583        foreach my $pbvf (<$ENV{'PBROOTDIR'}/$pbpkg/pbv*.pre>,<$ENV{'PBROOTDIR'}/$pbpkg/pbv*.post>, <$ENV{'PBROOTDIR'}/$pbpkg/pbtest*>) {
1584            pb_log(1,"Handling file $pbvf\n");
1585            if (-x "$pbvf") {
1586                my $target = "$ENV{'PBTMP'}/".basename($pbvf);
1587                pb_filter_file("$pbvf",$ptr,$target,$pb);
1588                chmod 0755,"$target";
1589                pb_log(1,"Capturing file $pbvf in $target\n");
1590            }
1591        }
1592
1593        # Prepare the dest directory for archive
1594        chdir "$ENV{'PBDESTDIR'}" || die "Unable to change dir to $ENV{'PBDESTDIR'}";
1595        if (defined $preserve) {
1596            # In that case we want to preserve the original tar file for checksum purposes
1597            # The one created is btw equivalent in that case to this one
1598            # Maybe check basename of both to be sure they are the same ?
1599            pb_log(0,"Preserving original tar file ");
1600            move("$preserve","$pbpkg-$pbver$pbextdir.tar.gz");
1601        } else {
1602            # Possibility to look at PBSRC to guess more the filename
1603            pb_system("tar cfz $pbpkg-$pbver$pbextdir.tar.gz --exclude=$pbpkg-$pbver$pbextdir/pbconf $pbpkg-$pbver$pbextdir","Creating $pbpkg tar files compressed");
1604        }
1605        pb_log(0,"Under $ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.tar.gz\n");
1606        pb_system("tar cfz $pbpkg-$pbver$pbextdir.pbconf.tar.gz $pbpkg-$pbver$pbextdir/pbconf","Creating pbconf tar files compressed");
1607        pb_log(0,"Under $ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.pbconf.tar.gz\n");
1608
1609        # Keep track of version-tag per pkg
1610        $pkgs{$pbpkg} = "$pbver-$pbtag";
1611
1612        # Final cleanup
1613        pb_rm_rf($dest) if (-d $dest);
1614    }
1615
1616    # Keep track of per package version
1617    pb_log(2,"DEBUG pkgs: ".Dumper(%pkgs)."\n");
1618    open(PKG,"> $ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml") || die "Unable to create $ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml";
1619    print PKG "---\n";
1620    print PKG "pbpkg:\n";
1621    foreach my $pbpkg (keys %pkgs) {
1622        print PKG "  $pbpkg: $pkgs{$pbpkg}\n";
1623    }
1624    close(PKG);
1625
1626    # Keep track of what is generated by default
1627    # We need to store the dir and info on version-tag
1628    # Base our content on the existing .yml file
1629    copy("$ENV{'PBROOTDIR'}/$ENV{'PBPROJ'}.yml","$ENV{'PBDESTDIR'}/pbrc.yml");
1630    my $lh;
1631    $lh = pb_conf_cache("$ENV{'PBDESTDIR'}/pbrc.yml",$lh);
1632    $lh->{'pbroot'}->{$ENV{'PBPROJ'}} = $ENV{'PBROOTDIR'};
1633    $lh->{'projver'}->{$ENV{'PBPROJ'}} = $ENV{'PBPROJVER'};
1634    $lh->{'projtag'}->{$ENV{'PBPROJ'}} = $ENV{'PBPROJTAG'};
1635    $lh->{'pbpackager'}->{$ENV{'PBPROJ'}} = $ENV{'PBPACKAGER'};
1636    $lh->{'pbextdir'}->{$ENV{'PBPROJ'}} = $pbextdir;
1637    pb_conf_write("$ENV{'PBDESTDIR'}/pbrc.yml",$lh);
1638}
1639
1640sub pb_test2pkg {
1641    # Get the running distro to test on
1642    my $pbos = pb_distro_get_context();
1643
1644    # Get list of packages to test
1645    # Get content saved in cms2build
1646    my $ptr = pb_get_pkg();
1647    @pkgs = @$ptr;
1648
1649    # Additional potential repo
1650    my $forcerepo = pb_distro_setuprepo($pbos);
1651    foreach my $pbpkg (@pkgs) {
1652        # We need to install the package to test, and deps brought with it
1653        pb_distro_installdeps(undef,$pbos,$forcerepo,$pbpkg);
1654        pb_system("$ENV{'PBDESTDIR'}/pbtest","Launching test for $pbpkg","verbose");
1655    }
1656}
1657
1658sub pb_build2prep {
1659
1660    pb_log(0,"INFO: ------ Starting to prepare build environement ------\n");
1661
1662    # Get the running distro to build on
1663    my $pbos = pb_distro_get_context();
1664
1665    # If needed we may add repository to the build env
1666    my $forcerepo = pb_distro_setuprepo($pbos);
1667    pb_log(1,"Forcing full package install with forcerepo\n") if (defined $forcerepo);
1668
1669    # Get list of packages to build
1670    my $ptr = pb_get_pkg();
1671    @pkgs = @$ptr;
1672   
1673    # Get content saved in cms2build
1674    my ($pkg) = pb_conf_read("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml","pbpkg");
1675    $pkg = { } if (not defined $pkg);
1676    my $pbextdir = pb_get_extdir();
1677
1678    foreach my $pbpkg (@pkgs) {
1679        my $vertag = $pkg->{$pbpkg};
1680        my @buildfiles;
1681        pb_log(2,"Vertag: $vertag\n");
1682        # get the version of the current package - maybe different
1683        ($pbver,$pbtag) = split(/-/,$vertag);
1684
1685        my $src2="$ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.pbconf.tar.gz";
1686        pb_log(2,"Pbconf file: $src2\n");
1687        pb_log(2,"Working directory: $ENV{'PBBUILDDIR'}\n");
1688
1689        my $ftype;
1690        if ($pbos->{'type'} eq "rpm") {
1691            $ftype = "\.spec\$";
1692        } elsif ($pbos->{'type'} eq "deb") {
1693            $ftype = "\/control\$";
1694        } elsif ($pbos->{'type'} eq "ebuild") {
1695            $ftype = "\.ebuild\$";
1696        } elsif ($pbos->{'type'} eq "tgz") {
1697            $ftype = "\/pbslack\$";
1698        } elsif ($pbos->{'type'} eq "pkg") {
1699            # Solaris - no deps management up to Solaris 10
1700        } elsif ($pbos->{'type'} eq "hpux") {
1701            # TODO: HP-UX
1702        } elsif ($pbos->{'type'} eq "apk") {
1703            $ftype = "\/APKBUILD\$";
1704        } else {
1705            die "Unknown OS type format $pbos->{'type'}";
1706        }
1707
1708        @buildfiles = pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/","$ENV{'PBTMP'}","build");
1709        pb_log(2,"buildfiles ".Dumper(\@buildfiles)."\n");
1710        foreach my $f (@buildfiles) {
1711            if ($f =~ /$ftype/) {
1712                pb_distro_installdeps($f,$pbos,$forcerepo);
1713            }
1714        }
1715    }
1716    pb_log(0,"INFO: ------ Finished preparing build environment ------\n");
1717}
1718
1719sub pb_build2pkg {
1720
1721    my $do_install = shift;
1722
1723        # Call build2prep in case we have new deps we want to take in account
1724        # without redoing the prep phase (can be long and that helps debugging
1725        # build dependencies issues
1726    pb_build2prep();
1727    pb_log(0,"INFO: ------ Starting to build package ------\n");
1728    # Get the running distro to build on
1729    my $pbos = pb_distro_get_context();
1730
1731    # Get list of packages to build
1732    my $ptr = pb_get_pkg();
1733    @pkgs = @$ptr;
1734
1735    # Get content saved in cms2build
1736    my ($pkg) = pb_conf_read("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml","pbpkg");
1737    $pkg = { } if (not defined $pkg);
1738    my $pbextdir = pb_get_extdir();
1739
1740    pb_mkdir_p("$ENV{'PBBUILDDIR'}") if (! -d "$ENV{'PBBUILDDIR'}");
1741    chdir "$ENV{'PBBUILDDIR'}" || die "Unable to chdir to $ENV{'PBBUILDDIR'}";
1742    my $made = ""; # pkgs made during build
1743    my $pm;
1744    my $all_ok = 1;
1745
1746    if (defined $pbparallel) {
1747        $pm = new Parallel::ForkManager($pbparallel);
1748        $pm->run_on_finish(sub { my ($pid, $code, $id, $signal, $dump) = @_;
1749                                $all_ok = 0 unless (($code == 0) && ($signal == 0) && ($dump == 0)); });
1750    }
1751
1752    # We need to communicate info back from the children if parallel so prepare a dir for that
1753    my $tmpd = "$ENV{'PBTMP'}/build.$$";
1754    pb_mkdir_p($tmpd) if (defined $pbparallel);
1755
1756    # Do it here as this doesn't work in // mode as it dies if it fails
1757    if ($pbos->{'type'} eq "rpm") {
1758        foreach my $d ('RPMS','SRPMS','SPECS','SOURCES','BUILD') {
1759                if (! -d "$ENV{'PBBUILDDIR'}/$d") {
1760                    pb_mkdir_p("$ENV{'PBBUILDDIR'}/$d");
1761                }
1762            }
1763    }
1764
1765    foreach my $pbpkg (@pkgs) {
1766        $pm->start and next if (defined $pbparallel);
1767
1768        my $vertag = $pkg->{$pbpkg};
1769        pb_log(2,"Vertag: $vertag\n");
1770        # get the version of the current package - maybe different
1771        ($pbver,$pbtag) = split(/-/,$vertag);
1772
1773        my $src="$ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.tar.gz";
1774        my $src2="$ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.pbconf.tar.gz";
1775        pb_log(2,"Source file: $src\n");
1776        pb_log(2,"Pbconf file: $src2\n");
1777
1778        pb_log(2,"Working directory: $ENV{'PBBUILDDIR'}\n");
1779        if ($pbos->{'type'} eq "rpm") {
1780            # Remove in case a previous link/file was there
1781            unlink "$ENV{'PBBUILDDIR'}/SOURCES/".basename($src);
1782            symlink "$src","$ENV{'PBBUILDDIR'}/SOURCES/".basename($src) || die "Unable to symlink $src in $ENV{'PBBUILDDIR'}/SOURCES";
1783            # We need to first extract the spec file
1784            my @buildfile = pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/","$ENV{'PBBUILDDIR'}/SPECS","build");
1785
1786            # We need to handle potential patches to upstream sources
1787            pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbpatch/","$ENV{'PBBUILDDIR'}/SOURCES","patch");
1788
1789            # We need to handle potential additional sources to upstream sources
1790            pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbsrc/","$ENV{'PBBUILDDIR'}/SOURCES","src");
1791
1792            pb_log(2,"buildfile: ".Dumper(\@buildfile)."\n");
1793            # set LANGUAGE to check for correct log messages
1794            $ENV{'LANGUAGE'}="C";
1795            # Older Redhat use _target_platform in %configure incorrectly
1796            my $specialdef = "";
1797            if (($pbos->{'name'} eq "rhel") && ($pbos->{'version'} eq "2.1")) {
1798                $specialdef = "--define \'_target_platform \"\"\'";
1799            }
1800            my $buildcmd = "rpmbuild";
1801            if (($pbos->{'name'} eq "redhat") && ($pbos->{'version'} =~ /^6/)){
1802                $buildcmd = "rpm" ;
1803            }
1804
1805            foreach my $f (@buildfile) {
1806                if ($f =~ /\.spec$/) {
1807                    pb_system("$buildcmd $specialdef --define \"packager $ENV{'PBPACKAGER'}\" --define \"_topdir $ENV{'PBBUILDDIR'}\" -ba $f","Building package with $f under $ENV{'PBBUILDDIR'}","verbose");
1808                    last;
1809                }
1810            }
1811            # Get the name of the generated packages
1812            open(LOG,"$ENV{'PBTMP'}/system.$$.log") || die "Unable to open $ENV{'PBTMP'}/system.$$.log";
1813            while (<LOG>) {
1814                chomp($_);
1815                next if ($_ !~ /^Wrote:/);
1816                s|.*/([S]*RPMS.*)|$1|;
1817                $made .=" $_";
1818            }
1819            close(LOG);
1820
1821        } elsif ($pbos->{'type'} eq "deb") {
1822            pb_system("tar xfz $src","Extracting sources");
1823            pb_system("tar xfz $src2","Extracting pbconf");
1824
1825            chdir "$pbpkg-$pbver$pbextdir" || die "Unable to chdir to $pbpkg-$pbver$pbextdir";
1826            pb_rm_rf("debian");
1827            my $confdir = "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
1828            die "Configuration directory $confdir is not a directory\nIs os description listed in your {ve,vm,rm}list values?" if (not -d $confdir);
1829            symlink "$confdir","debian" || die "Unable to symlink 'debian' to $confdir";
1830            chmod 0755,"debian/rules";
1831
1832            # We need to handle potential patches to upstream sources
1833            pb_mkdir_p("debian/patches");
1834            my @f = pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbpatch/","debian/patches","patch");
1835
1836            # By default we use format 1.0 - Cf man dpkg-source
1837            my $debsrcfmt = "1.0";
1838            my $debsrcfile = "debian/source/format";
1839            if (-f $debsrcfile) {
1840                $debsrcfmt = pb_get_content($debsrcfile);
1841            }
1842
1843            if ($debsrcfmt =~ /^3.*quilt/) {
1844                # If we use quilt to manage patches, we then setup the env correctly
1845                # as per http://pkg-perl.alioth.debian.org/howto/quilt.html
1846                # Generate Debian patch series for quilt
1847                open(SERIE,"> debian/patches/series") || die "Unable to write in debian/patches/series";
1848                $ENV{'QUILT_PATCHES'}="debian/patches";
1849            } else {
1850                # If we use dpatch to manage patches, we then setup the 00list file as well
1851                open(SERIE,"> debian/patches/00list") || die "Unable to write in debian/patches/00list";
1852            }
1853            foreach my $f (sort @f) {
1854                # Skip the script made to apply the patches to the Debian tree
1855                next if ($f =~ /pbapplypatch/);
1856                # We also need to uncompress them
1857                pb_system("gzip -d $f","","quiet");
1858                $f =~ s/\.gz$//;
1859                print SERIE "$f\n";
1860            }
1861            close(SERIE);
1862            if (@f) {
1863                # We have patches...
1864                my $patch_file = "debian/patches/pbapplypatch";
1865                if (($debsrcfmt =~ /^1.*/) && (-x $patch_file)) {
1866                    # In that case we need to apply the patches ourselves locally
1867                    pb_system("cat $patch_file","APPLY","verbose");
1868                    pb_system("$patch_file","Applying patches to $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'} tree");
1869                }
1870                # ...so modify the name of files to be Debian compliant
1871                move("../$src","../$pbpkg-$pbver$pbextdir.orig.tar.gz");
1872            }
1873
1874            # We need to handle potential additional sources to upstream sources
1875            #pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbsrc/","$ENV{'PBBUILDDIR'}/debian","src");
1876
1877            pb_system("dpkg-buildpackage -us -uc -rfakeroot","Building package","verbose");
1878
1879            # Get the name of the generated packages
1880            open(LOG,"$ENV{'PBTMP'}/system.$$.log") || die "Unable to open $ENV{'PBTMP'}/system.$$.log";
1881            while (<LOG>) {
1882                chomp();
1883                pb_log(3,"In loop: $_\n");
1884                next unless (/^dpkg-deb.*\.\.\/(.*_.*\.deb).*/);
1885                my $tmp = $1;
1886                #doesn't work in my case
1887                #die "Missing file $tmp" if (not -f "../$tmp");
1888                $made = "$made $tmp";
1889            }
1890            close(LOG);
1891            pb_log(2,"Now made is: $made\n");
1892
1893            open(CTRL,"debian/control") || die "Unable to open debian/control: $!";
1894            while (<CTRL>) {
1895                pb_log(3,"In loop: $_\n");
1896                next unless (/^Source: (\S+)/o);
1897                my $tmp = $1;
1898                $made = "$made $tmp"."_*.dsc $tmp"."_*.tar.gz $tmp"."_*.changes";
1899                #doesn't work in my case
1900                #foreach my $glob (("$1\_*.changes", "$1\_*.dsc", "$1\_*.tar.gz")) {
1901                    #my @file = glob($glob);
1902                    #die "Missing file for $glob" unless (@file > 0);
1903                    #die "Too many files for $glob" if (@file > 1);
1904                    #die "Missing file $file[0]" if (not -f $file[0]);
1905                    #$made .= " $file[0]";
1906                #}
1907            }
1908            close(CTRL);
1909            #pb_display_file("$ENV{'PBTMP'}/system.$$.log");
1910            pb_log(2,"Finally made is: $made\n");
1911
1912            chdir ".." || die "Unable to chdir to parent dir";
1913            pb_rm_rf("$pbpkg-$pbver$pbextdir");
1914        } elsif ($pbos->{'type'} eq "ebuild") {
1915            my @ebuildfile;
1916            # For gentoo we need to take pb as subsystem name
1917            # We put every apps here under sys-apps. hope it's correct
1918            # We use pb's home dir in order to have a single OVERLAY line
1919            my $tmpe = "$ENV{'HOME'}/portage/pb/sys-apps/$pbpkg";
1920            pb_mkdir_p($tmpe) if (! -d "$tmpe");
1921            pb_mkdir_p("$ENV{'HOME'}/portage/distfiles") if (! -d "$ENV{'HOME'}/portage/distfiles");
1922
1923            # We need to first extract the ebuild file
1924            @ebuildfile = pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/","$tmpe","build");
1925
1926            # Prepare the build env for gentoo
1927            my $found = 0;
1928            my $pbbd = $ENV{'HOME'};
1929            $pbbd =~ s|/|\\/|g;
1930            if (-r "/etc/make.conf") {
1931                open(MAKE,"/etc/make.conf");
1932                while (<MAKE>) {
1933                    $found = 1 if (/$pbbd\/portage/);
1934                }
1935                close(MAKE);
1936            }
1937            if ($found == 0) {
1938                pb_system("sudo sh -c 'echo PORTDIR_OVERLAY=\"$ENV{'HOME'}/portage\" >> /etc/make.conf'");
1939            }
1940            #$found = 0;
1941            #if (-r "/etc/portage/package.keywords") {
1942            #open(KEYW,"/etc/portage/package.keywords");
1943            #while (<KEYW>) {
1944            #$found = 1 if (/portage\/pb/);
1945            #}
1946            #close(KEYW);
1947            #}
1948            #if ($found == 0) {
1949            #pb_system("sudo sh -c \"echo portage/pb >> /etc/portage/package.keywords\"");
1950            #}
1951
1952            # Build
1953            foreach my $f (@ebuildfile) {
1954                if ($f =~ /\.ebuild$/) {
1955                    move($f,"$tmpe/$pbpkg-$pbver.ebuild");
1956                    pb_system("cd $tmpe ; ebuild $pbpkg-$pbver.ebuild clean ; ebuild $pbpkg-$pbver.ebuild digest ; ebuild $pbpkg-$pbver.ebuild package","verbose");
1957                    # Now move it where pb expects it
1958                    pb_mkdir_p("$ENV{'PBBUILDDIR'}/portage/pb/sys-apps/$pbpkg");
1959                    if ($pbtag eq 0) {
1960                        # This is assumed to be a test version
1961                        my $nver = substr($pbver,0,-14);
1962                        my $ntag = substr($pbver,-14);
1963                        my $ebtg = "portage/pb/sys-apps/$pbpkg/$pbpkg-$nver"."_p$ntag.ebuild";
1964                        move("$tmpe/$pbpkg-$pbver.ebuild","$ENV{'PBBUILDDIR'}/$ebtg");
1965                        $made="$made $ebtg";
1966                    } else {
1967                        my $ebtg = "portage/pb/sys-apps/$pbpkg/$pbpkg-$pbver-r$pbtag.ebuild";
1968                        move("$tmpe/$pbpkg-$pbver.ebuild","$ENV{'PBBUILDDIR'}/$ebtg");
1969                        $made="$made $ebtg";
1970                    }
1971                }
1972            }
1973
1974        } elsif ($pbos->{'type'} eq "apk") {
1975            pb_mkdir_p("$ENV{'PBBUILDDIR'}/tmp");
1976
1977            # We need to first extract the APKBUILD file
1978            my @apkfile = pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/","$ENV{'PBBUILDDIR'}/tmp","build");
1979
1980            chdir "$ENV{'PBBUILDDIR'}/tmp" || die "Unable to chdir to $ENV{'PBBUILDDIR'}/tmp";
1981
1982            # Build
1983            my $keyfile = pb_ssh_get(1);
1984            pb_mkdir_p("$ENV{'HOME'}/.abuild");
1985            open(CONF,"> $ENV{'HOME'}/.abuild/abuild.conf") || confess "Unable to create the abuild conf file";
1986            print CONF "PACKAGER_PRIVKEY=\"$keyfile\"\n" if (defined $keyfile);
1987            close(CONF);
1988            foreach my $f (@apkfile) {
1989                if ($f =~ /APKBUILD$/) {
1990                    pb_system("ls -al $ENV{'HOME'}/.ssh ; pwd ; ls -alR ../.. ; sh -x /usr/bin/abuild checksum ; abuild unpack ; abuild -r -P .. build ; abuild -P .. rootpkg ; abuild -P pkg index","verbose");
1991                }
1992            }
1993            # Get the name of the generated packages
1994            open(LOG,"$ENV{'PBTMP'}/system.$$.log") || die "Unable to open $ENV{'PBTMP'}/system.$$.log";
1995            while (<LOG>) {
1996                chomp($_);
1997                next if ($_ !~ /: Create /);
1998                s|.*: Create (.*.apk)|$1|;
1999                $made .= " $_";
2000            }
2001            close(LOG);
2002
2003        } elsif ($pbos->{'type'} eq "tgz") {
2004            # Slackware family
2005            $made="$made $pbpkg/$pbpkg-$pbver-*-$pbtag.tgz";
2006
2007            pb_system("tar xfz $src","Extracting sources");
2008            pb_system("tar xfz $src2","Extracting pbconf");
2009            chdir "$pbpkg-$pbver$pbextdir" || die "Unable to chdir to $pbpkg-$pbver$pbextdir";
2010            symlink "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}","install" || die "Unable to symlink to pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
2011            if (-x "install/pbslack") {
2012                pb_system("./install/pbslack","Building software");
2013                pb_system("sudo /sbin/makepkg -p -l y -c y $pbpkg","Packaging $pbpkg","verbose");
2014            }
2015            chdir ".." || die "Unable to chdir to parent dir";
2016            pb_rm_rf("$pbpkg-$pbver$pbextdir");
2017        } elsif ($pbos->{'type'} eq "pkg") {
2018            # Solaris
2019            $made="$made $pbpkg-$pbver-$pbtag.pkg.gz";
2020            my $pkgdestdir="$ENV{'PBBUILDDIR'}/install";
2021
2022            # Will host resulting packages
2023            pb_mkdir_p("$pbos->{'type'}");
2024            pb_mkdir_p("$pkgdestdir/pbdelivery");
2025            pb_system("tar xfz $src","Extracting sources under $ENV{'PBBUILDDIR'}");
2026            pb_system("tar xfz $src2","Extracting pbconf under $ENV{'PBBUILDDIR'}");
2027            # We need to handle potential patches to upstream sources
2028            pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbpatch/","$ENV{'PBBUILDDIR'}","patch");
2029
2030            # We need to handle potential additional sources to upstream sources
2031            pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbsrc/","$ENV{'PBBUILDDIR'}","src");
2032
2033            chdir "$pbpkg-$pbver$pbextdir" || die "Unable to chdir to $pbpkg-$pbver$pbextdir";
2034            if (-f "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbbuild") {
2035                chmod 0755,"pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbbuild";
2036                # pkginfo file is mandatory
2037                die "Unable to find pkginfo file in pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}" if (! -f "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pkginfo");
2038                # Build
2039                pb_system("pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbbuild $pkgdestdir/pbdelivery","Building software and installing under $pkgdestdir/pbdelivery");
2040                # Copy complementary files
2041                if (-f "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/prototype") {
2042                    copy("pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/prototype", $pkgdestdir) 
2043                } else {
2044                    # No prototype provided, calculating it
2045                    open(PROTO,"> $pkgdestdir/prototype") || die "Unable to create prototype file";
2046                    print PROTO "i pkginfo\n";
2047                    print PROTO "i depend\n" if (-f "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/depend");
2048                    $ENV{'PBSOLDESTDIR'} = "$pkgdestdir/pbdelivery";
2049                    find(\&create_solaris_prototype, "$pkgdestdir/pbdelivery");
2050                }
2051                copy("pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/depend", $pkgdestdir) if (-f "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/depend");
2052                copy("pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pkginfo", $pkgdestdir);
2053                pb_system("cd $pkgdestdir/pbdelivery ; pkgmk -o -f ../prototype -r $pkgdestdir/pbdelivery -d $ENV{'PBBUILDDIR'}/$pbos->{'type'}","Packaging $pbpkg","verbose");
2054                pb_system("cd $ENV{'PBBUILDDIR'}/$pbos->{'type'} ;  echo \"\" | pkgtrans -o -n -s $ENV{'PBBUILDDIR'}/$pbos->{'type'} $ENV{'PBBUILDDIR'}/$pbpkg-$pbver-$pbtag.pkg all","Transforming $pbpkg","verbose");
2055                pb_system("cd $ENV{'PBBUILDDIR'} ;  gzip -9f $pbpkg-$pbver-$pbtag.pkg","Compressing $pbpkg-$pbver-$pbtag.pkg","verbose");
2056            } else {
2057                pb_log(0,"No pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbbuild file found for $pbpkg-$pbver\n");
2058            }
2059            chdir ".." || die "Unable to chdir to parent dir";
2060            pb_rm_rf("$pbpkg-$pbver$pbextdir","$ENV{'PBBUILDDIR'}/$pbos->{'type'}","$pkgdestdir");
2061        } elsif ($pbos->{'type'} eq "hpux") {
2062            # HP-UX
2063            pb_system("tar xfz $src","Extracting sources");
2064            pb_system("tar xfz $src2","Extracting pbconf");
2065
2066            chdir "$pbpkg-$pbver$pbextdir" || die "Unable to chdir to $pbpkg-$pbver$pbextdir";
2067            pb_system("buildpackage ","Building package","verbose");
2068            # Get the name of the generated packages
2069            open(LOG,"$ENV{'PBTMP'}/system.$$.log") || die "Unable to open $ENV{'PBTMP'}/system.$$.log";
2070            while (<LOG>) {
2071                chomp();
2072                my $tmp = $_;
2073                next if ($tmp !~ /^SD BUILD.*:/);
2074                $tmp =~ s|.*../(.*)_(.*).sd.*|$1|;
2075                $made = "$made $tmp"."_*.sd";
2076            }
2077            close(LOG);
2078            $made="$made $pbpkg-$pbver-$pbtag.sd";
2079
2080            chdir ".." || die "Unable to chdir to parent dir";
2081            pb_rm_rf("$pbpkg-$pbver$pbextdir");
2082        } else {
2083            die "Unknown OS type format $pbos->{'type'}";
2084        }
2085        if (defined $pbparallel) {
2086            # Communicate results back to parent
2087            pb_set_content("$tmpd/$$",$made);
2088            $pm->finish;
2089        }
2090    }
2091    if (defined $pbparallel) {
2092        # In the parent, we need to get the result from the children
2093        $pm->wait_all_children;
2094        foreach my $f (<$tmpd/*>) {
2095            $made .= " ".pb_get_content($f);
2096        }
2097        die "Aborting, one or more of the children failed." if ((not $all_ok) && ($Global::pb_stop_on_error));
2098        pb_rm_rf($tmpd);
2099    }
2100
2101    # Sign packages
2102    pb_sign_pkgs($pbos,$made);
2103    pb_log(0,"INFO: ------ Finished building package ------\n");
2104
2105    # Find the appropriate check cmd/opts
2106    my ($chkcmd,$chkopt) = pb_distro_get_param($pbos,pb_conf_get_if("oschkcmd","oschkopt"));
2107
2108    my $ret = "";
2109    # Packages check if needed
2110    if ($pbos->{'type'} eq "rpm") {
2111        if ((defined  $chkcmd) && (-x $chkcmd)) {
2112            my $cmd = "$chkcmd";
2113            $cmd .= " $chkopt" if (defined $chkopt);
2114            $cmd .= " $made";
2115            my $ret = pb_system("$cmd","Checking validity of rpms with $chkcmd","mayfailverbose");
2116            pb_log(0,"ERROR: when checking packages validity\n") if ($ret ne 0);
2117        }
2118        my $rpms ="";
2119        my $srpms ="";
2120        foreach my $f (split(/ /,$made)) {
2121            $rpms .= "$ENV{'PBBUILDDIR'}/$f " if ($f =~ /^RPMS\//);
2122            $srpms .= "$ENV{'PBBUILDDIR'}/$f " if ($f =~ /^SRPMS\//);
2123        }
2124        pb_log(0,"SRPM packages generated: $srpms\n");
2125        pb_log(0,"RPM packages generated: $rpms\n");
2126        $ret = $rpms;
2127    } elsif ($pbos->{'type'} eq "deb") {
2128        my $made2 = "";
2129        foreach my $f (split(/ /,$made)) {
2130            $made2 .= "$f " if ($f =~ /\.deb$/);
2131        }
2132        if ((defined  $chkcmd) && (-x $chkcmd)) {
2133            my $ret = pb_system("$chkcmd $chkopt $made2","Checking validity of debs with $chkcmd","mayfail");
2134            pb_log(0,"ERROR: when checking packages validity\n") if ($ret ne 0);
2135        }
2136        pb_log(0,"deb packages generated: $made2\n");
2137        $ret = $made2;
2138    } else {
2139        pb_log(0,"No check done for $pbos->{'type'} yet\n");
2140        pb_log(0,"Packages generated: $made\n");
2141        $ret = $made;
2142    }
2143
2144    # Keep track of what is generated so that we can get them back from VMs/RMs
2145    my $pbstore = "$ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
2146    open(KEEP,"> $pbstore") || die "Unable to create $pbstore $!";
2147    print KEEP "$made\n";
2148    close(KEEP);
2149    pb_distro_installdeps(undef,$pbos,undef,$ret,"local") if ($do_install);
2150}
2151
2152sub pb_build2ssh {
2153    pb_send2target("Sources");
2154    pb_send2target("CPAN");
2155}
2156
2157sub pb_pkg2ssh {
2158    pb_send2target("Packages");
2159}
2160
2161# By default deliver to the the public site hosting the
2162# ftp structure (or whatever) or a VM/VE/RM
2163sub pb_send2target {
2164
2165    my $cmt = shift;
2166    my $pbscript1 = shift;
2167    my $v = shift;
2168    my $vexist = shift;         # 0 is FALSE
2169    my $vpid = shift;           # 0 is FALSE
2170    my $snapme = shift;         # 0 is FALSE
2171    my $pbstep = shift;         # 3 is usage of container
2172
2173    $vexist = 0 if (not defined $vexist);
2174    $vpid = 0 if (not defined $vpid);
2175    $snapme = 0 if (not defined $snapme);
2176    $pbstep = 3 if (not defined $pbstep);
2177    pb_log(2,"DEBUG: pb_send2target($cmt,".Dumper($v).",$vexist,$vpid)\n");
2178    my $host = "sshhost";
2179    my $login = "sshlogin";
2180    my $dir = "sshdir";
2181    my $port = "sshport";
2182    my $conf = "sshconf";
2183    my $tmout = undef;
2184    my $path = undef;
2185
2186    if ($cmt =~ /^VM/) {
2187        $login = "vmlogin";
2188        $dir = "pbdefdir";
2189        # Specific VM
2190        $tmout = "vmtmout";
2191        $path = "vmpath";
2192        $host = "vmhost";
2193        $port = "vmport";
2194    } elsif ($cmt =~ /^RM/) {
2195        $login = "rmlogin";
2196        $dir = "pbdefdir";
2197        # Specific RM
2198        $tmout = "rmtmout";
2199        $path = "rmpath";
2200        $host = "rmhost";
2201        $port = "rmport";
2202    } elsif ($cmt =~ /^VE/) {
2203        $login = "velogin";
2204        $dir = "pbdefdir";
2205        # Specific VE
2206        $path = "vepath";
2207        $conf = "rbsconf";
2208    } elsif ($cmt eq "Web") {
2209        $host = "websshhost";
2210        $login = "websshlogin";
2211        $dir = "websshdir";
2212        $port = "websshport";
2213    } elsif ($cmt eq "CPAN") {
2214        $host = "cpanpause";
2215        $login = "";
2216        $dir = "cpandir";
2217        $port = "";
2218    }
2219    my $cmd = "";
2220    my $src = "";
2221    my $cpanpkg = 0;
2222    my $pbos;
2223
2224    my $pbextdir = pb_get_extdir();
2225
2226    if ($cmt ne "Announce") {
2227        # Get list of packages to build
2228        my $ptr = pb_get_pkg();
2229        my @pkgs = ();
2230        @pkgs = @$ptr if (defined $ptr);
2231
2232        # Get the running distro to consider
2233        $pbos = pb_distro_get_context($v);
2234
2235        my $pkg = { };
2236        # Get content saved in cms2build
2237        if ((defined $ENV{'PBDESTDIR'}) && (defined $ENV{'PBPROJVER'}) && (defined $ENV{'PBPROJTAG'}) && (-f "$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml")) {
2238            ($pkg) = pb_conf_read("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml","pbpkg");
2239            $pkg = { } if (not defined $pkg);
2240        }
2241
2242        pb_mkdir_p("$ENV{'PBBUILDDIR'}") if (! -d "$ENV{'PBBUILDDIR'}");
2243        chdir "$ENV{'PBBUILDDIR'}" || die "Unable to chdir to $ENV{'PBBUILDDIR'}";
2244        foreach my $pbpkg (@pkgs) {
2245            my $vertag = $pkg->{$pbpkg};
2246            # get the version of the current package - maybe different
2247            pb_log(2,"Vertag: $vertag\n");
2248            ($pbver,$pbtag) = split(/-/,$vertag);
2249
2250            if (($cmt eq "Sources") || ($cmt =~ /(V[EM]|RM)(build|prep)/)) {
2251                $src = "$src $ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.tar.gz $ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.pbconf.tar.gz";
2252                my $cmd2 = "ln -sf $pbpkg-$pbver$pbextdir.tar.gz $pbpkg-latest.tar.gz";
2253                #my $cmd2 = "ln $pbpkg-$pbver$pbextdir.tar.gz $pbpkg-latest.tar.gz 2> /dev/null ; if [ \$? -ne 0 ]; then ln -sf $pbpkg-$pbver$pbextdir.tar.gz $pbpkg-latest.tar.gz ; fi";
2254                if ($cmd eq "") {
2255                    $cmd = $cmd2;
2256                } else {
2257                    $cmd = "$cmd ; $cmd2";
2258                }
2259            } elsif ($cmt eq "Web") {
2260                $src = "$src $ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.tar.gz"
2261            }
2262
2263            # Do we have one perl package
2264            my @nametype = pb_conf_get_if("namingtype");
2265            my $type = $nametype[0]->{$pbpkg};
2266            if ((defined $type) && ($type eq "perl")) {
2267                $cpanpkg = 1;
2268            }
2269        }
2270        # Adds conf file for availability of conf elements
2271        pb_conf_add("$ENV{'PBROOTDIR'}/$ENV{'PBPROJ'}.yml") if ((defined $ENV{'PBROOTDIR'}) && (-f "$ENV{'PBROOTDIR'}/$ENV{'PBPROJ'}.yml"));
2272    }
2273    my ($rbsconf,$testver,$delivery) = pb_conf_get_if($conf,"testver","delivery");
2274    if ($cmt eq "CPAN") {
2275        # Do not deliver on Pause if this is a test version
2276        return if (not defined $testver);
2277        return if ($testver =~ /true/);
2278        # Do not deliver on Pause if this is not a perl package
2279        return if ($cpanpkg == 0);
2280    }
2281
2282    if ($cmt =~ /(V[EM]|RM)(build|prep)/) {
2283        $src="$src $ENV{'PBROOTDIR'}/$ENV{'PBPROJ'}.yml $ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml $ENV{'PBETC'} $ENV{'PBDESTDIR'}/pbrc.yml $pbscript{$v}";
2284    } elsif ($cmt =~ /(V[EM]|RM)Script/) {
2285        $src="$src $pbscript{$v}";
2286    } elsif ($cmt =~ /(V[EM]|RM)test/) {
2287        $src="$src $ENV{'PBROOTDIR'}/$ENV{'PBPROJ'}.yml $ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml $ENV{'PBETC'} $ENV{'PBDESTDIR'}/pbrc.yml $pbscript{$v} $ENV{'PBDESTDIR'}/pbtest";
2288    } elsif (($cmt eq "Announce") || ($cmt eq "Web") || ($cmt eq "CPAN")) {
2289        $src="$src $ENV{'PBTMP'}/pbscript";
2290    } elsif ($cmt eq "Packages") {
2291        # Get package list from file made during build2pkg
2292        open(KEEP,"$ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}") || die "Unable to read $ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
2293        $src = <KEEP>;
2294        chomp($src);
2295        close(KEEP);
2296    }
2297    if (($cmt eq "Sources") || ($cmt eq "Packages") || ($cmt eq "CPAN")) {
2298        my ($pbpackager) = pb_conf_get("pbpackager");
2299        $ENV{'PBPACKAGER'} = $pbpackager->{$ENV{'PBPROJ'}};
2300        pb_log(0,"Exporting public key for $ENV{'PBPACKAGER'}\n");
2301        # Using pb_system is not working due to redirection below
2302        system("gpg --export -a \'$ENV{'PBPACKAGER'}\' > $ENV{'PBDESTDIR'}/$ENV{'PBPROJ'}.pubkey");
2303        chmod 0644,"$ENV{'PBDESTDIR'}/$ENV{'PBPROJ'}.pubkey";
2304        $src = "$src $ENV{'PBDESTDIR'}/$ENV{'PBPROJ'}.pubkey";
2305
2306        my ($gpgaddkey) = pb_conf_get_if("pbadditionalgpg");
2307        my $pbadditionalgpg;
2308        $pbadditionalgpg = $gpgaddkey->{$ENV{'PBPROJ'}} if (defined $gpgaddkey);
2309        if (defined $pbadditionalgpg) {
2310            foreach my $k (split(/,/,$pbadditionalgpg)) {
2311                pb_log(0,"Exporting public key $k\n");
2312                system("gpg --export -a \'$k\' > $ENV{'PBDESTDIR'}/$ENV{'PBPROJ'}-$k.pubkey");
2313                $src = "$src $ENV{'PBDESTDIR'}/$ENV{'PBPROJ'}-$k.pubkey";
2314            }
2315        }
2316    }
2317    # Remove potential leading spaces (cause problem with basename)
2318    $src =~ s/^ *//;
2319    my $basesrc = "";
2320    foreach my $i (split(/ +/,$src)) {
2321        $basesrc .= " ".basename($i);
2322    }
2323
2324    pb_log(2,"values: ".Dumper(($host,$login,$dir,$port,$tmout,$path,$conf))."\n");
2325    my ($sshhost,$sshdir) = pb_conf_get($host,$dir);
2326    # Not mandatory...
2327    $delivery->{$ENV{'PBPROJ'}} = "" if (not defined $delivery->{$ENV{'PBPROJ'}});
2328    my ($sshlogin,$sshport) = pb_conf_get_if($login,$port);
2329    $sshport->{$ENV{'PBPROJ'}} = 22 if (not defined $sshport->{$ENV{'PBPROJ'}});
2330    $sshlogin->{$ENV{'PBPROJ'}} = getpwuid($UID) if (not defined $sshlogin->{$ENV{'PBPROJ'}});
2331    my ($vtmout,$vepath);
2332    # ...Except those in virtual context
2333    if ($cmt =~ /^VE/) {
2334        ($vepath) = pb_conf_get($path);
2335    }
2336    if ($cmt =~ /^(V|R)M/) {
2337        $vtmout = pb_distro_get_param($pbos,pb_conf_get_if($tmout));
2338    }
2339    my $remhost = $sshhost->{$ENV{'PBPROJ'}};
2340    my $remdir = $sshdir->{$ENV{'PBPROJ'}};
2341    if ($cmt =~ /^V[EM]|RM/) {
2342        # In that case our real host is in the xxhost with the OS as key, not project as above
2343        $remhost = pb_distro_get_param($pbos,$sshhost);
2344    }
2345    pb_log(2,"ssh: ".Dumper(($remhost,$sshlogin,$remdir,$sshport,$vepath,$rbsconf))."\n");
2346    pb_log(2,"ssh: ".Dumper($vtmout)."\n") if (defined $vtmout);
2347
2348    my $mac;
2349    if ($cmt !~ /^VE/) {
2350        $mac = "$sshlogin->{$ENV{'PBPROJ'}}\@$remhost";
2351        # Overwrite account value if passed as parameter or forced
2352        $mac = "$ENV{'PBACCOUNT'}\@$remhost" if (defined $ENV{'PBACCOUNT'});
2353    } else {
2354        # VE
2355        $mac = $sshlogin->{$ENV{'PBPROJ'}};
2356        # Overwrite account value if passed as parameter (typically for setup2v)
2357        $mac = $ENV{'PBACCOUNT'} if (defined $ENV{'PBACCOUNT'});
2358    }
2359    pb_log(2, "DEBUG: mac: $mac\n");
2360
2361    my $tdir;
2362    my $bdir;
2363    if (($cmt eq "Sources") || ($cmt =~ /(V[EM]|RM)Script/)) {
2364        $tdir = "$remdir/$delivery->{$ENV{'PBPROJ'}}/src";
2365    } elsif ($cmt eq "CPAN") {
2366        $tdir = "$remdir";
2367    } elsif ($cmt =~ /(V[EM]|RM)(build|test|prep)/) {
2368        $tdir = $remdir."/$ENV{'PBPROJ'}/pbdelivery";
2369        $bdir = $remdir."/$ENV{'PBPROJ'}/pbbuild";
2370        # Remove a potential $ENV{'HOME'} as bdir should be relative to pb's home
2371        $bdir =~ s|\$ENV.+\}/||;
2372    } elsif ($cmt eq "Announce") {
2373        $tdir = "$remdir/$delivery->{$ENV{'PBPROJ'}}";
2374    } elsif ($cmt eq "Web") {
2375        $tdir = "$remdir/$delivery->{$ENV{'PBPROJ'}}";
2376    } elsif ($cmt eq "Packages") {
2377        if (($pbos->{'type'} eq "rpm") || ($pbos->{'type'} eq "pkg") || ($pbos->{'type'} eq "hpux") || ($pbos->{'type'} eq "tgz") || ($pbos->{'type'} eq "apk")) {
2378            # put packages under an arch subdir
2379            $tdir = "$remdir/$delivery->{$ENV{'PBPROJ'}}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}";
2380        } elsif (($pbos->{'type'} eq "deb") || ($pbos->{'type'} eq "ebuild")) {
2381            # No need for an arch subdir
2382            $tdir = "$remdir/$delivery->{$ENV{'PBPROJ'}}/$pbos->{'name'}/$pbos->{'version'}";
2383        } else {
2384            die "Please teach the dev team where to deliver ($pbos->{'type'} type of packages\n";
2385        }
2386
2387        my $repodir = $tdir;
2388        $repodir =~ s|^$remdir/||;
2389        my $repotag = "";
2390        $repotag = "-$delivery->{$ENV{'PBPROJ'}}" if ($delivery->{$ENV{'PBPROJ'}} ne "");
2391
2392        my ($pbrepo) = pb_conf_get("pbrepo");
2393
2394        # Script for Repository management
2395        $v = "$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}" if (not defined $v);
2396        $pbscript{$v} = "$ENV{'PBBUILDDIR'}/pbscript.$$";
2397       
2398        # That script should be part of the delivery
2399        $src = "$src $pbscript{$v}";
2400        open(PBS,"> $pbscript{$v}") || die "Unable to create $pbscript{$v}";
2401        if ($pbos->{'type'} eq "rpm") {
2402            my $pbsha = pb_distro_get_param($pbos,pb_conf_get("ossha"));
2403            my ($gpgcheck) = pb_conf_get_if("pbgpgcheck");
2404            my $pbgpgcheck;
2405            $pbgpgcheck = $gpgcheck->{$ENV{'PBPROJ'}} if (defined $gpgcheck);
2406            my $pbgpgkey = "$pbrepo->{$ENV{'PBPROJ'}}/$repodir/$ENV{'PBPROJ'}.pubkey";
2407            my ($gpgaddpkey) = pb_conf_get_if("pbadditionalgpg");
2408            my $pbadditionalgpg;
2409            $pbadditionalgpg = $gpgaddpkey->{$ENV{'PBPROJ'}} if (defined $gpgaddpkey);
2410            if (defined $pbadditionalgpg) {
2411                foreach my $k (split(/,/,$pbadditionalgpg)) {
2412                    $pbgpgkey .= " $pbrepo->{$ENV{'PBPROJ'}}/$repodir/$ENV{'PBPROJ'}-$k.pubkey";
2413                }
2414            }
2415            # By default force GPG check in repo even if we support signature of packages to fail. This is a best practice
2416            $pbgpgcheck = 1 if (not defined $pbgpgcheck);
2417            # Also make a pbscript to generate dnf/yum/urpmi bases
2418            print PBS << "EOF";
2419#!/bin/bash
2420# Prepare a script to ease dnf/yum setup
2421EOF
2422            print PBS "set -x\n" if ($pbdebug gt 1);
2423            print PBS << "EOF";
2424cat > $ENV{'PBPROJ'}$repotag.repo << EOT
2425[$ENV{'PBPROJ'}$repotag]
2426name=$pbos->{'name'} $pbos->{'version'} $pbos->{'arch'} - $ENV{'PBPROJ'} Vanilla Packages
2427baseurl=$pbrepo->{$ENV{'PBPROJ'}}/$repodir
2428enabled=1
2429gpgcheck=$pbgpgcheck
2430gpgkey=$pbgpgkey
2431EOT
2432chmod 644 $ENV{'PBPROJ'}$repotag.repo
2433
2434# Clean up old repo content
2435rm -rf headers/ repodata/
2436# Create yum repo
2437if [ -x /usr/bin/yum-arch ]; then
2438    yum-arch .
2439fi
2440# Create repodata
2441createrepo -s $pbsha .
2442# Link to the key
2443if [ -s $ENV{'PBPROJ'}.pubkey ]; then
2444    # Avoiding creating empty repomd.xml.key; fails on opensuse 12.1
2445    (cd repodata ; ln -sf ../$ENV{'PBPROJ'}.pubkey repomd.xml.key)
2446fi
2447# sign the repomd (at least useful for SLES - which requires a local key)
2448# gpg -a --detach-sign repodata/repomd.xml
2449# SLES also looks for media.1/info.txt
2450EOF
2451            if ($pbos->{'family'} eq "md") {
2452                # For Mandriva add urpmi management
2453                print PBS << "EOF";
2454# Prepare a script to ease urpmi setup
2455cat > $ENV{'PBPROJ'}$repotag.addmedia << EOT
2456#rpm --import $pbrepo->{$ENV{'PBPROJ'}}/$repodir/$ENV{'PBPROJ'}.pubkey
2457urpmi.addmedia $ENV{'PBPROJ'}$repotag $pbrepo->{$ENV{'PBPROJ'}}/$repodir with media_info/hdlist.cz
2458EOT
2459chmod 755 $ENV{'PBPROJ'}$repotag.addmedia
2460
2461# Clean up old repo content
2462rm -f hdlist.cz synthesis.hdlist.cz
2463# Create urpmi repo
2464genhdlist2 --clean .
2465if [ \$\? -ne 0 ]; then
2466    genhdlist .
2467fi
2468(cd media_info ; ln -sf ../$ENV{'PBPROJ'}.pubkey pubkey)
2469EOF
2470            }
2471            if ($pbos->{'name'} eq "fedora") {
2472                # Extract the spec file to please Fedora maintainers :-(
2473                print PBS << "EOF";
2474for p in $basesrc; do
2475    echo \$p | grep -q 'src.rpm'
2476    if [ \$\? -eq 0 ]; then
2477        rpm2cpio \$p | cpio -ivdum --quiet '*.spec'
2478    fi
2479done
2480EOF
2481            }
2482            if ($pbos->{'family'} eq "novell") {
2483                # Add ymp scripts for one-click install on SuSE
2484                print PBS << "EOF";
2485# Prepare a script to ease SuSE one-click install
2486# Cf: http://de.opensuse.org/1-Klick-Installation/ISV
2487#
2488cat > $ENV{'PBPROJ'}$repotag.ymp << EOT
2489<?xml version="1.0" encoding="utf-8"?>
2490<!-- vim: set sw=2 ts=2 ai et: -->
2491<metapackage xmlns:os="http://opensuse.org/Standards/One_Click_Install" xmlns="http://opensuse.org/Standards/One_Click_Install">
2492    <group><!-- The group of software, typically one for project-builder.org -->
2493        <name>$ENV{'PBPROJ'} Bundle</name> <!-- Name of the software group -->
2494        <summary>Software bundle for the $ENV{'PBPROJ'}$repotag project</summary> <!--This message is shown to the user and should describe the whole bundle -->
2495        <description>This is the summary of the $ENV{'PBPROJ'}$repotag Project
2496             
2497            Details are available on a per package basis below
2498
2499        </description><!--This is also shown to the user -->
2500        <remainSubscribed>false</remainSubscribed> <!-- Don't know what it mean -->
2501        <repositories><!-- List of needed repositories -->
2502            <repository>
2503                <name>$ENV{'PBPROJ'}$repotag Repository</name> <!-- Name of the repository  -->
2504                <summary>This repository contains the $ENV{'PBPROJ'}$repotag project packages.</summary> <!-- Summary of the repository -->
2505                <description>This repository contains the $ENV{'PBPROJ'}$repotag project packages.</description><!-- This description is shown to the user -->
2506                <url>$pbrepo->{$ENV{'PBPROJ'}}/$repodir</url><!--URL of repository, which is added -->
2507            </repository>
2508        </repositories>
2509        <software><!-- A List of packages, which should be added through the one-click-installation -->
2510EOT
2511for p in $basesrc; do
2512    sum=`rpm -q --qf '%{SUMMARY}' \$p`
2513    name=`rpm -q --qf '%{NAME}' \$p`
2514    desc=`rpm -q --qf '%{description}' \$p`
2515    cat >> $ENV{'PBPROJ'}$repotag.ymp << EOT
2516            <item>
2517                <name>\$name</name><!-- Name of the package, is shown to the user and used to identify the package at the repository -->
2518                <summary>\$sum</summary> <!-- Summary of the package -->
2519                <description>\$desc</description> <!-- Description, is shown to the user -->
2520            </item>
2521EOT
2522done
2523cat >> $ENV{'PBPROJ'}$repotag.ymp << EOT
2524        </software>
2525    </group>
2526</metapackage>
2527EOT
2528chmod 644 $ENV{'PBPROJ'}$repotag.ymp
2529EOF
2530            }
2531        } elsif ($pbos->{'type'} eq "deb") {
2532            # Also make a pbscript to generate apt bases
2533            # Cf: http://www.debian.org/doc/manuals/repository-howto/repository-howto.fr.html
2534            # This dirname removes ver
2535            my $debarch = pb_get_debarch($pbos);
2536            my $projcomponent = pb_get_debpc($pbos);
2537            my $rpd = dirname("$pbrepo->{$ENV{'PBPROJ'}}/$repodir");
2538            # Remove extra . in path to fix #522
2539            $rpd =~ s|/./|/|g;
2540            print PBS << "EOF";
2541#!/bin/bash
2542# Prepare a script to ease apt setup
2543cat > $ENV{'PBPROJ'}$repotag.sources.list << EOT
2544deb $rpd $pbos->{'version'} $projcomponent
2545deb-src $rpd $pbos->{'version'} $projcomponent
2546EOT
2547chmod 644 $ENV{'PBPROJ'}$repotag.sources.list
2548
2549# Up two levels to deal with the dist dir cross versions
2550cd ..
2551mkdir -p dists/$pbos->{'version'}/$projcomponent/binary-$debarch dists/$pbos->{'version'}/$projcomponent/source dists/$pbos->{'version'}/$projcomponent/binary-all
2552
2553# Prepare a script to create apt info file
2554# Reuse twice after
2555TMPD=`mktemp -d /tmp/pb.XXXXXXXXXX` || exit 1
2556mkdir -p \$TMPD
2557cat > \$TMPD/Release << EOT
2558Archive: unstable
2559Component: $projcomponent
2560Origin: $ENV{'PBPROJ'}$repotag
2561Label: $ENV{'PBPROJ'}$repotag dev repository $pbrepo->{$ENV{'PBPROJ'}}
2562EOT
2563
2564echo "Creating Packages metadata"
2565for i in dists/$pbos->{version}/$projcomponent/binary-*; do
2566    arch=`basename \$i | sed 's/binary-//'`
2567    echo "Packages for \$arch:"
2568    dpkg-scanpackages -a\$arch $pbos->{'version'} /dev/null > dists/$pbos->{'version'}/$projcomponent/binary-\$arch/Packages
2569    gzip -9 < dists/$pbos->{'version'}/$projcomponent/binary-\$arch/Packages > dists/$pbos->{'version'}/$projcomponent/binary-\$arch/Packages.gz
2570    bzip2 -9 < dists/$pbos->{'version'}/$projcomponent/binary-\$arch/Packages > dists/$pbos->{'version'}/$projcomponent/binary-\$arch/Packages.bz2
2571done
2572echo "Creating Contents metadata"
2573apt-ftparchive contents $pbos->{'version'} | gzip -c9 > dists/$pbos->{'version'}/Contents.gz
2574echo "Creating Release metadata ($pbos->{'arch'} aka $debarch)"
2575cat \$TMPD/Release > dists/$pbos->{'version'}/$projcomponent/binary-all/Release
2576echo "Architecture: all" >> dists/$pbos->{'version'}/$projcomponent/binary-all/Release
2577cat \$TMPD/Release > dists/$pbos->{'version'}/$projcomponent/binary-$debarch/Release
2578echo "Architecture: $debarch" >> dists/$pbos->{'version'}/$projcomponent/binary-$debarch/Release
2579echo "Creating Source metadata"
2580dpkg-scansources $pbos->{'version'} /dev/null > dists/$pbos->{'version'}/$projcomponent/source/Sources
2581gzip -9 < dists/$pbos->{'version'}/$projcomponent/source/Sources > dists/$pbos->{'version'}/$projcomponent/source/Sources.gz
2582bzip2 -9 < dists/$pbos->{'version'}/$projcomponent/source/Sources > dists/$pbos->{'version'}/$projcomponent/source/Sources.bz2
2583cat \$TMPD/Release > dists/$pbos->{'version'}/$projcomponent/source/Release
2584echo "Architecture: Source" >> dists/$pbos->{'version'}/$projcomponent/source/Release
2585echo "Creating Release metadata"
2586apt-ftparchive -o APT::FTPArchive::Release::Suite=$pbos->{version} release dists/$pbos->{'version'} > dists/$pbos->{'version'}/Release
2587rm -rf \$TMPD
2588EOF
2589        } elsif ($pbos->{'type'} eq "ebuild") {
2590            # make a pbscript to generate links to latest version
2591            print PBS << "EOF";
2592#!/bin/bash
2593# Prepare a script to create correct links
2594cd $tdir
2595for p in $src; do
2596    echo \$p | grep -q '.ebuild'
2597    if [ \$\? -eq 0 ]; then
2598        j=`basename \$p`
2599        pp=`echo \$j | perl -pe 's/-[0-9\.]+-r[0-9].*\.ebuild/.ebuild/'`
2600        ln -sf \$j \$pp
2601    fi
2602done
2603EOF
2604        } elsif ($pbos->{'type'} eq "apk") {
2605            # TODO
2606        }
2607        close(PBS);
2608        chmod 0755,$pbscript{$v};
2609    } else {
2610        return;
2611    }
2612
2613    # Useless for VE
2614    my $nport = pb_get_port($sshport,$pbos,$cmt) if ($cmt !~ /^VE/);
2615
2616    # Remove a potential $ENV{'HOME'} as tdir should be relative to pb's home
2617    if ($cmt =~ /^VE/o) {
2618        $tdir =~ s|\$ENV.+\}/|| or confess "for $cmt: $tdir must have \$ENV{'HOME'} in it or the remainder of the code won't work since some parts use relative pathing and others will try absolute.  Use a symlink if necessary to place the real files in a different place than under your home directory";
2619    } else {
2620        # process, ok if not present.
2621        $tdir =~ s|\$ENV.+\}/||;
2622    }
2623
2624    my $tm = "";
2625    if ($cmt =~ /^(V|R)M/) {
2626        $tm = "sleep $vtmout" if (defined $vtmout);
2627    }
2628
2629    # ssh communication if not VE or CPAN
2630    # should use a hash instead...
2631    my ($shcmd,$shcmdroot,$cpcmd,$cptarget,$cp2target);
2632    my $tpdir;
2633    my $tp;
2634    my $context = "$ENV{'PBTMP'}";
2635    my %tag;
2636    my $dkaccount = "";
2637    my $cmd1 = "";
2638    my $cidfile;
2639    my $homedir = "";
2640    if ($cmt =~ /^VE/) {
2641        $tp = pb_path_expand($vepath->{$ENV{'PBPROJ'}});
2642        $vetype = pb_ve_get_type($vetype);
2643        if ($vetype eq "docker") {
2644            $tpdir = "/";
2645            $cidfile = "$context/ctn-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}.cid";
2646        } else {
2647            $tpdir = pb_path_expand("$tp/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}");
2648        }
2649        my $arch = pb_get_arch();
2650        if ($vetype eq "chroot") {
2651            $shcmdroot = "sudo /usr/sbin/chroot $tpdir ";
2652            $shcmd = "$shcmdroot /bin/su - $mac -c ";
2653        } elsif ($vetype eq "schroot") {
2654            $shcmd = "schroot $tp -u $mac -- ";
2655        } elsif ($vetype eq "docker") {
2656            # docker manages the storage so rely on it
2657            $shcmdroot = "";
2658            my $docrepo = pb_ve_docker_repo();
2659            $cmd1 = pb_check_req("docker",0);
2660            #my ($dockeropt) =  pb_conf_get_if("dockeropt");
2661            # pbimage is used for docker and is setup to the right name depending on the step:
2662            # TODO: This is not coded yet
2663            my $pbimage = undef;
2664            # step 0 : nothing at creation -> tag n-v-a (made in VE.pm)
2665            # step 1 : n-v-a + setup -> tag n-v-a-pb
2666            # step 2 : n-v-a-pb + build -> tag n-v-a-pb-pbproj
2667            # step 3 : n-v-a-pb-pbproj at use
2668            if ((not defined $pbimage) || ($pbimage eq "")) {
2669                # If no image name given, create a naming convention
2670                $tag{1} = "$docrepo$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
2671                $tag{2} = "$tag{1}-pb";
2672                $tag{3} = "$tag{2}-$ENV{'PBPROJ'}";
2673            } else {
2674                # If we were given an image name, just use it
2675                $tag{1} = $pbimage;
2676                $tag{2} = $pbimage;
2677                $tag{3} = $pbimage;
2678            }
2679            # If we do not create the image, then use the one we should have
2680            # TODO: we don't do anything with found !!
2681            my $found = pb_ve_docker_get_image($tag{$pbstep});
2682            confess "No image $tag{$pbstep} available in Docker, please create one first\n" if (not defined $found);
2683            # Now we use that image to do what is needed
2684            # use a dockerfile to ease the creation of next images
2685            # the context is in an empty temp dir
2686            my $tmpd = "$context/Dockerfile";
2687            open(DOCKER, "> $tmpd") || die "Unable to create the docker file $tmpd";
2688            print DOCKER "FROM $tag{$pbstep}\n";
2689            print DOCKER "MAINTAINER project-builder.org aka pb\n";
2690            if ($pbstep == 1) {
2691                # setup done as root
2692                $dkaccount = "root";
2693            } else {
2694                # Rest done as pb
2695                $dkaccount = "pb";
2696            }
2697            print DOCKER "USER $dkaccount\n";
2698            if ($pbstep <= 2) {
2699                $shcmd = "$cmd1 build -t $tag{$pbstep+1} $context";
2700            } else {
2701                # As we are in run phase use docker run. cmd will be completed below
2702                $shcmd = "$cmd1 run --cidfile=\"$cidfile\" --user $dkaccount --name $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-$cmt";
2703            }
2704            #$shcmd = "$cmd1 build $dockeropt->{$ENV{'PBPROJ'}} -t $tag{$pbstep+1} $context";
2705        }
2706        $shcmd = "setarch i386 $shcmd" if (($pbos->{'arch'} =~ /i[3456]86/) && ($arch eq 'x86_64'));
2707        if (($cmt =~ /VE/) && ($vetype ne "docker")) {
2708            # We need to get the home dir of the target account to deliver in the right place
2709            open(PASS,"$tpdir/etc/passwd") || die "Unable to open $tpdir/etc/passwd: $!";
2710            while (<PASS>) {
2711                my ($c1,$c2,$c3,$c4,$c5,$c6,$c7) = split(/:/);
2712                $homedir = $c6 if ($c1 =~ /^$mac$/);
2713                pb_log(3,"Homedir: $homedir - account: $c6\n");
2714            }
2715            close(PASS);
2716            $cpcmd = "sudo /bin/cp -r ";
2717        }
2718        if (($cmt =~ /VE/) && ($vetype eq "docker")) {
2719            if ($pbstep >= 2) {
2720                # We need to get the home dir of the target account to deliver in the right place
2721                $homedir = `$cmd1 run --cidfile="$cidfile" $tag{$pbstep} grep -E '^$dkaccount:' /etc/passwd | cut -d: -f6`;
2722                chomp($homedir);
2723                open(CID,"$cidfile") || confess "Unable to open $cidfile";
2724                my $cid = <CID>;
2725                close(CID);
2726                pb_system("$cmd1 rm $cid","","quiet");
2727                unlink("$cidfile");
2728            } else {
2729                $homedir = "/";
2730            }
2731            $cpcmd = "cp -r ";
2732        }
2733        $cptarget = "$tpdir/$homedir/$tdir";
2734        if ($cmt =~ /VE(build|prep)/) {
2735            $cp2target = "$tpdir/$homedir/$bdir";
2736        }
2737        pb_log(2,"On VE using $cptarget as target dir to copy to\n");
2738    } elsif ($cmt eq "CPAN") {
2739        my $ftpput = pb_check_req("ncftpput",1);
2740        my $ftpget = pb_check_req("wget",1);
2741        my ($cpanuser,$cpanpasswd) = pb_conf_get_if("cpanuser","cpanpasswd");
2742        return if ((not defined $cpanuser) || (not defined $cpanuser->{$ENV{'PBPROJ'}}));
2743        return if ((not defined $cpanpasswd) || (not defined $cpanpasswd->{$ENV{'PBPROJ'}}));
2744        my ($cpansubdir) = pb_conf_get_if("cpansubdir");
2745        $shcmd = "$ftpget --post-data \'HIDDENNAME=".$cpanuser->{$ENV{'PBPROJ'}};
2746        $shcmd .= "&user=".$cpanuser->{$ENV{'PBPROJ'}};
2747        $shcmd .= "&password=".$cpanpasswd->{$ENV{'PBPROJ'}};
2748        $shcmd .= "&SUBMIT_pause99_add_uri_upload=\"Upload the checked files\"";
2749        $shcmd .= "&pause99_add_uri_subdirtext=".$cpansubdir->{$ENV{'PBPROJ'}} if ((defined $cpansubdir) && (defined $cpansubdir->{$ENV{'PBPROJ'}}));
2750        foreach my $s (split(/ /,$src)) {
2751            $shcmd .= "&pause99_add_uri_upload=".basename($s);
2752        }
2753        $shcmd .= "'";
2754        $cpcmd = "$ftpput $host $dir";
2755        $cptarget = "CPAN";
2756    } else {
2757        my $keyopt = "";
2758        if ($cmt !~ /Packages/) {
2759            my $keyfile = pb_ssh_get(0,$remhost,$nport);
2760            $keyopt = "-i $keyfile" if (defined $keyfile);
2761        }
2762        my $sshcmd = pb_check_req("ssh",1);
2763        my $scpcmd = pb_check_req("scp",1);
2764        $shcmd = "$sshcmd $keyopt -q -o NoHostAuthenticationForLocalhost=yes -p $nport $mac";
2765        $cpcmd = "$scpcmd $keyopt -p -o NoHostAuthenticationForLocalhost=yes -P $nport";
2766        $cptarget = "$mac:$tdir";
2767        if ($cmt =~ /^(V|R)M(build|prep)/) {
2768            $cp2target = "$mac:$bdir";
2769        }
2770    }
2771   
2772    my $logres = "";
2773    # Do not touch when just announcing
2774    if (($cmt ne "Announce") && ($cmt ne "CPAN")) {
2775        # For docker everything is done in the Dockerfile for steps < 3
2776        if (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep < 3)) {
2777            print DOCKER "RUN mkdir -p $cptarget\n";
2778            print DOCKER "RUN cd $cptarget ; for i in $basesrc; do if [ -f \$i ]; then rm -f \$i; fi; done\n";
2779        } elsif (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep >= 3)) {
2780            # Nothing to do here
2781        } else {
2782            pb_system("$shcmd \"mkdir -p $tdir ; cd $tdir ; echo \'for i in $basesrc; do if [ -f \$i ]; then rm -f \$i; fi; done ; $cmd\' | bash -e\"","Preparing $tdir on $cptarget");
2783        }
2784    } else {
2785        $logres = "> ";
2786    }
2787
2788    if ($cmt =~ /SandBox/) {
2789        # Install from sandbox mean using the result of the just passed sbx2build command
2790        # Get content saved in cms2build
2791        my ($pkg) = pb_conf_read("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml","pbpkg");
2792        my $pbextdir = pb_get_extdir();
2793        confess "Unable to get package list" if (not defined $pkg);
2794
2795        # We consider 2 specific packages
2796        my $vertag1 = $pkg->{"ProjectBuilder"};
2797        my $vertag2 = $pkg->{"project-builder"};
2798        # get the version of the current package - maybe different
2799        pb_log(2,"Vertag1: $vertag1\n");
2800        pb_log(2,"Vertag2: $vertag2\n");
2801        my ($pbver1,$tmp1) = split(/-/,$vertag1);
2802        my ($pbver2,$tmp2) = split(/-/,$vertag2);
2803        if ($cmt =~ /VE/) {
2804            # Copy inside the VE
2805            if ($vetype ne "docker") {
2806                copy("$ENV{'PBDESTDIR'}/ProjectBuilder-$pbver1$pbextdir.tar.gz","$tpdir/tmp");
2807                copy("$ENV{'PBDESTDIR'}/project-builder-$pbver2$pbextdir.tar.gz","$tpdir/tmp");
2808            } else {
2809                copy("$ENV{'PBDESTDIR'}/ProjectBuilder-$pbver1$pbextdir.tar.gz","$context");
2810                copy("$ENV{'PBDESTDIR'}/project-builder-$pbver2$pbextdir.tar.gz","$context");
2811                print DOCKER "COPY ProjectBuilder-$pbver1$pbextdir.tar.gz /tmp/\n";
2812                print DOCKER "COPY project-builder-$pbver2$pbextdir.tar.gz /tmp/\n";
2813            }
2814        } else {
2815            pb_system("$cpcmd $ENV{'PBDESTDIR'}/ProjectBuilder-$pbver1$pbextdir.tar.gz $ENV{'PBDESTDIR'}/project-builder-$pbver2$pbextdir.tar.gz $mac:/tmp","Copying local project files to VM/RM");
2816        }
2817    }
2818           
2819    pb_log(0,"Sources handled ($cmt): $src\n");
2820    # For docker everything is done in the Dockerfile
2821    if (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep < 3)) {
2822        foreach my $f (split(/ +/,$src)) {
2823            copy("$f","$context");
2824            print DOCKER "COPY ".basename($f)." $cptarget/\n";
2825        }
2826        print DOCKER "RUN cd $cptarget ; $cmd\n" if ((defined $cmd) && ($cmd ne ""));
2827    } elsif (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep >= 3)) {
2828        # Nothing to do
2829    } else {
2830        pb_system("cd $ENV{'PBBUILDDIR'} ; $cpcmd $src $cptarget 2> /dev/null","$cmt delivery in $cptarget");
2831    }
2832
2833    # For VE we need to change the owner manually
2834    if ($cmt =~ /^VE/) {
2835        if ($vetype ne "docker") {
2836            my $sudomode = pb_distro_get_param($pbos,pb_conf_get("ossudoersmode"));
2837            my $res = pb_system("$shcmdroot sed -i '/requiretty/d' /etc/sudoers","Removing potential requiretty in sudoers","quiet");
2838            pb_system("$shcmdroot sed '/requiretty/d' /etc/sudoers > /tmp/sudoers.new ; mv /tmp/sudoers.new $tpdir/tmp/sudoers.new ; $shcmdroot mv /tmp/sudoers.new /etc/sudoers ; $shcmdroot chown root:root /etc/sudoers ; $shcmdroot chmod $sudomode /etc/sudoers","Removing again potential requiretty in sudoers as sed -i failed") if (($res ne 0) && (-f "$tpdir/etc/sudoers"));
2839            pb_system("$shcmd \"sudo chown -R $mac $tdir\"","Adapt owner in $tdir to $mac");
2840        } else {
2841            print DOCKER "RUN sed -i '/requiretty/d' /etc/sudoers\n" if ($pbstep == 1);
2842        }
2843    }
2844
2845    # Use the right script name depending on context
2846    my $pbscript2;
2847    if (($cmt =~ /^(V[EM]|RM)/) || ($cmt =~ /Packages/)){
2848        $pbscript2 = $pbscript{$v};
2849    } else {
2850        $pbscript2 = "pbscript";
2851    }
2852
2853    # It's already ready for CPAN
2854    my $shcmdbase = $shcmd;
2855    my $dkcmd;
2856    if ($cmt ne "CPAN") {
2857        if (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep < 3)) {
2858            copy("$pbscript2","$context");
2859            my $s = basename($pbscript2);
2860            print DOCKER "COPY $s $cptarget/\n";
2861            # call directly with perl to avoid text file busy error messages
2862            print DOCKER "RUN cd $cptarget ; perl ./$s ; if [ '$pbkeep' = '0' ]; then rm -f $s; fi\n";
2863            print DOCKER "RUN rm -rf $cptarget/*\n";
2864            print DOCKER "RUN chown -R $dkaccount $cptarget\n" if ($pbstep == 1);
2865            close(DOCKER);
2866        } elsif (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep >= 3)) {
2867            $dkcmd = "/bin/bash $homedir/$tdir-2/".basename($pbscript2);
2868        } else {
2869            my $s = basename($pbscript2);
2870            $shcmd .= " \"echo \'cd $tdir ; if [ -x $s ]; then ./$s; fi ;  if [ $pbkeep == 0 ]; then rm -f $s; fi\' | bash\"";
2871        }
2872    }
2873    my $cmdverb = "verbose";
2874    if (($cmt eq "Announce") || ($cmt eq "CPAN")) {
2875        $cmdverb = undef;
2876    } elsif (defined ($v)) {
2877        $cmdverb = "verbose_\[$v\] ";
2878    }
2879    # this is where we launch the execution
2880    if (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep >= 3)) {
2881        pb_mkdir_p("$ENV{'PBTMP'}/vebuild");
2882        my $tsrc = "";
2883        foreach my $f (split(/ +/,$src)) {
2884            copy("$f","$ENV{'PBTMP'}/vebuild");
2885            $tsrc .= " ".basename($f);
2886        }
2887        my $ts = "$ENV{'PBTMP'}/vebuild/".basename($pbscript2);
2888        open(DOCKER," > $ts") || confess "Unable to create $ts";
2889        print DOCKER "#!/bin/bash\n";
2890        print DOCKER "# Put content from host to guest, copying to have the correct uid/gid and rights to do so\n";
2891        print DOCKER "# set -x\n";
2892        print DOCKER "for i in $tsrc; do\n";
2893        print DOCKER "  # Cleanup first to avoid rights issues for root owned files e.g.\n";
2894        print DOCKER "  rm -f $homedir/$tdir/\$i\n";
2895        print DOCKER "  cp $homedir/$tdir-2/\$i $homedir/$tdir\n";
2896        print DOCKER "done\n";
2897        print DOCKER "(cd $homedir/$tdir-2 ; ls -al)\n";
2898        print DOCKER pb_get_content($pbscript2);
2899        print DOCKER "\n# Get content back from guest in a host dir\n";
2900        # Host dir seen as TDIR in container
2901        $cp2target = "$ENV{'PBBUILDDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
2902        pb_mkdir_p("$cp2target");
2903        chmod 0777,"$cp2target";
2904        print DOCKER "TDIR=$homedir/$bdir-2/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}\n";
2905        print DOCKER "for i in $homedir/$bdir/*; do\n";
2906        print DOCKER "  # Cleanup first to avoid rights issues for root owned files e.g.\n";
2907        print DOCKER "  j=\`basename \$i\`\n";
2908        print DOCKER "  rm -f \$TDIR/\$j\n";
2909        print DOCKER "  cp -r \$i \$TDIR\n";
2910        print DOCKER "done\n";
2911        print DOCKER "find \$TDIR -type f -print0 | xargs -0 chmod 644\n";
2912        print DOCKER "find \$TDIR -type d -print0 | xargs -0 chmod 777 2> /dev/null\n";
2913        #print DOCKER "ls -al \$TDIR\n";
2914        print DOCKER "exit 0\n";
2915        close(DOCKER);
2916        $shcmd .= " -v $ENV{'PBBUILDDIR'}:$homedir/$bdir-2 -v $ENV{'PBTMP'}/vebuild:$homedir/$tdir-2 -w $homedir/$tdir-2 $tag{$pbstep} $dkcmd";
2917    }
2918    my $ret = pb_system("$shcmd","Executing pbscript on $cptarget if needed",$cmdverb);
2919   
2920    # If target is deb family then sign Release file for Packages
2921    my $pbos2 = pb_distro_get_context($v);
2922    if (($cmt =~ /Packages/) && ($pbos2->{'type'} eq "deb")) {
2923        my $debarch = pb_get_debarch($pbos2);
2924        my $projcomponent = pb_get_debpc($pbos2);
2925        for my $f ("dists/$pbos2->{'version'}/$projcomponent/binary-$debarch/Release","dists/$pbos2->{'version'}/$projcomponent/binary-all/Release","dists/$pbos2->{'version'}/$projcomponent/source/Release","dists/$pbos2->{'version'}/Release") {
2926            my $debdir = dirname($cptarget);
2927            pb_system("cd $ENV{'PBBUILDDIR'} ; $cpcmd $debdir/$f . 2> /dev/null","Getting deb $f file from $debdir");
2928            pb_sign_file("$ENV{'PBBUILDDIR'}/Release");
2929            pb_system("$cpcmd $ENV{'PBBUILDDIR'}/Release.gpg $debdir/$f.gpg 2> /dev/null","Putting deb $f.gpg file to $debdir");
2930            my ($void,$debfile) = split(/:/,$debdir);
2931            pb_system("$shcmdbase \"chmod 644 $debfile/$f.gpg\"","Set mode to 644 for $debfile/$f.gpg");
2932        }
2933    } 
2934
2935    if ($cmt =~ /^(V[EM]|RM)build/) {
2936        # Get back info on pkg produced, compute their name and get them from the VM/RM
2937        pb_system("$cpcmd $cp2target/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'} $ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.$$ 2> /dev/null","Get package names in $cp2target");
2938        if (not -f "$ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.$$") {
2939            pb_log(0,"ERROR with VM/RM $v on getting $ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.$$\n");
2940        } else {
2941            open(KEEP,"$ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.$$") || die "Unable to read $ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.$$";
2942            my $src = <KEEP>;
2943            chomp($src);
2944            close(KEEP);
2945            unlink("$ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.$$");
2946
2947            $src =~ s/^ *//;
2948            pb_mkdir_p("$ENV{'PBBUILDDIR'}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}");
2949            # Change pgben to make the next send2target happy
2950            my $made = "";
2951   
2952            # For VM/RM we don't want shell expansion to hapen locally but remotely
2953            my $delim = '\'';
2954            if ($cmt =~ /^VEbuild/) {
2955                # For VE we need to support shell expansion locally
2956                $delim = "";
2957            }   
2958
2959            open(KEEP,"> $ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}") || die "Unable to write $ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
2960            foreach my $p (split(/ +/,$src)) {
2961                my $j = basename($p);
2962                pb_system("$cpcmd $cp2target/$delim$p$delim $ENV{'PBBUILDDIR'}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'} 2> /dev/null","Recovery of package $j in $ENV{'PBBUILDDIR'}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}");
2963                $made="$made $pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}/$j"; # if (($pbos->{'type'} ne "rpm") || ($j !~ /.src.rpm$/));
2964            }
2965       
2966            print KEEP "$made\n";
2967            close(KEEP);
2968            if (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep >= 3)) {
2969                pb_rm_rf("$cp2target");
2970                pb_system("$cmd1 rm $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-$cmt","","quiet");
2971            } else {
2972                pb_system("$shcmdbase \"rm -rf $tdir $bdir\"","$cmt cleanup");
2973            }
2974
2975            # Sign packages locally
2976            pb_sign_pkgs($pbos,$made);
2977
2978            # We want to send them to the ssh account so overwrite what has been done before
2979            pb_log(2,"Before sending pkgs, vexist: $vexist, vpid: $vpid\n");
2980            pb_send2target("Packages","$ENV{'PBDESTDIR'}/pbscript.$$",$pbos->{'name'}."-".$pbos->{'version'}."-".$pbos->{'arch'},$vexist,$vpid);
2981            pb_rm_rf("$ENV{'PBBUILDDIR'}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}");
2982        }
2983    }
2984    unlink("$pbscript{$v}") if ((($cmt =~ /^(V[ME]|RM)/) || ($cmt =~ /Packages/)) && ($pbkeep eq 0) && ($cmt !~ /Script/));
2985
2986    pb_log(2,"Before halt, vexist: $vexist, vpid: $vpid\n");
2987    if ((! $vexist) && ($cmt =~ /^VM/)) {
2988        # If in setupvm then takes a snapshot just before halting
2989        if ($snapme != 0) {
2990            my ($vmmonport,$vmtype) = pb_conf_get("vmmonport","vmtype");
2991            # For monitoring control
2992            if ((($vmtype->{$ENV{'PBPROJ'}}) eq "kvm") || (($vmtype->{$ENV{'PBPROJ'}}) eq "qemu")) {
2993                eval
2994                {
2995                require Net::Telnet;
2996                Net::Telnet->import();
2997                };
2998                if ($@) {
2999                    # Net::Telnet not found
3000                    pb_log(1,"ADVISE: Install Net::Telnet to benefit from monitoring control and snapshot feature.\nWARNING: No snapshot created");
3001                } else {
3002                    my $t = new Net::Telnet (Timeout => 120, Host => "localhost", Port => $vmmonport->{$ENV{'PBPROJ'}}) || die "Unable to dialog on the monitor";
3003                    # move to monitor mode
3004                    my @lines = $t->cmd("c");
3005                    # Create a snapshot named pb
3006                    @lines = $t->cmd("savevm pb");
3007                    # Write the new status in the VM
3008                    @lines = $t->cmd("commit all");
3009                    # End
3010                    @lines = $t->cmd("quit");
3011                }
3012            }
3013        }
3014        my $hoption = "-p";
3015        my $hpath = pb_distro_get_param($pbos,pb_conf_get("ospathcmd-halt"));
3016        # Solaris doesn't support -p of halt
3017        if ($pbos->{'type'} eq "pkg") {
3018            $hoption = "" ;
3019        }
3020        pb_system("$shcmdbase \"sudo $hpath $hoption \"; $tm ; echo \'if [ -d /proc/$vpid ]; then kill -9 $vpid; fi \' | bash ; sleep 10","VM $v halt (pid $vpid)");
3021    }
3022    if (($cmt =~ /^VE/) && ($snapme != 0)) {
3023        if ($vetype eq "docker") {
3024            foreach my $f (split(/ +/,$src)) {
3025                unlink("$context/$f");
3026            }
3027        } else {
3028            pb_ve_snap($pbos,$tp);
3029        }
3030    }
3031}
3032
3033sub pb_script2v {
3034
3035my $pbscript=shift;
3036my $vtype=shift;
3037my $pbstep=shift;   # Which step are we in (0: create, 1: setup pb, 2: prep pbproj, 3: use to build)
3038my $pbforce=shift;  # Force stop of VM. Default not.
3039my $snapme=shift;   # Do we have to create a snapshot. Default not.
3040my $pbimage=shift;  # Which image to use to start the VM/VE
3041
3042$pbforce = 0 if (not defined $pbforce);
3043$snapme = 0 if (not defined $snapme);
3044$pbstep = 3 if (not defined $pbstep);
3045
3046pb_log(2,"DEBUG: pb_script2v($vtype,$pbstep,$pbforce,$snapme)\n");
3047pb_log(2,"DEBUG: pb_script2v(pbscript: ".Dumper($pbscript)."\n") if (defined $pbscript);
3048pb_parallel_launchv($pbscript,$vtype,uc($vtype)."Script",$pbstep,$pbimage,$pbforce,$snapme,$pbsnap);
3049}
3050
3051sub pb_parallel_launchv {
3052
3053my $pbscript=shift;
3054my $vtype = shift;
3055my $action = shift;         # It an action is defined then use send2target
3056my $pbstep=shift;           # Which step are we in (0: create, 1: setup, 2: build 3: use)
3057my $pbimage=shift;          # Which image to use to start the VM/VE
3058my $pbforce=shift;          # Force stop of VM. Default not.
3059my $snapme = shift;         # By default do not snap a VM/VE/RM
3060my $usesnap = shift;        # By default study the usage of the snapshot feature of VM/VE/RM   
3061my $vm;
3062my $all;
3063
3064$pbforce = 0 if (not defined $pbforce);
3065$snapme = 0 if (not defined $snapme);
3066$usesnap = 1 if (not defined $usesnap);
3067
3068# Adapt // mode to memory size
3069$pbparallel = pb_set_parallel($vtype);
3070
3071pb_log(2,"DEBUG: pb_parallel_launch(vtype,step,force,snapme,usesnap: $vtype,$pbstep,$pbforce,$snapme,$usesnap)\n");
3072pb_log(2,"DEBUG: pb_parallel_launch(pbscript: ".Dumper($pbscript)."\n");
3073
3074my ($vexist,$vpid) = (undef,undef);
3075my $pm;
3076my $all_ok = 1;
3077if (defined $pbparallel) {
3078    $pm = new Parallel::ForkManager($pbparallel);
3079
3080    # Set which port the VM/RM will use to communicate
3081    $pm->run_on_start(\&pb_set_port);
3082    $pm->run_on_finish(sub { my ($pid, $code, $id, $signal, $dump) = @_;
3083                            unless ($code == 0 && $signal == 0 && $dump == 0) {
3084                                $all_ok = 0;
3085                                pb_log(0,"ERROR: pid $pid failed\n");
3086                            }
3087                        });
3088}
3089
3090# Determine on which V to work
3091if (defined $ENV{'PBV'}) {
3092    @$vm = split(/,/,$ENV{'PBV'});
3093} else {
3094    ($vm,$all) = pb_get2v($vtype);
3095}
3096
3097$vetype = pb_ve_get_type($vetype);
3098if ((($vtype =~ /vm/) || (($vtype =~ /ve/) && ($vetype eq "docker"))) && (((not defined $pbimage) || ($pbimage eq "")) && ($pbstep == 0))) {
3099    confess "-i image parameter needed";
3100}
3101
3102my $counter = 0;
3103foreach my $v (@$vm) {
3104    $counter++;
3105    # Modulo 2 * pbparallel (to avoid synchronization problems)
3106    $counter = 1 if ((defined $pbparallel) && ($counter > 2 * $pbparallel));
3107    $pm->start($counter) and next if (defined $pbparallel);
3108   
3109    # If launched via script2v then get the script passed as the one for the that VM
3110    if (not defined $pbscript->{$v}) {
3111        $pbscript->{$v} = $pbscript->{'default'};
3112    }
3113    #
3114    # Launch a single operation
3115    ($vexist,$vpid) = pb_launchv($pbscript->{$v},$vtype,$v,$action,$pbstep,$pbforce,$snapme,$pbsnap,$pbimage);
3116   
3117    # Skip that VM/RM if something went wrong
3118    if (($vtype =~ /(v|r)m/) && ($vpid == 0) && ($vexist == 0)) {
3119        $pm->finish if (defined $pbparallel);
3120        next;
3121    }
3122
3123   
3124    $pm->finish if (defined $pbparallel);
3125}
3126$pm->wait_all_children if (defined $pbparallel);
3127die "Aborting, one or more of the children failed." if ((not $all_ok) && ($Global::pb_stop_on_error));
3128}
3129
3130sub pb_launchv {
3131
3132my $pbscript1=shift;        # Just the script name to execute, not the hash
3133my $vtype = shift;
3134my $v = shift;              # Only 1 VM/VE/RM treated here
3135my $action = shift;         # It an action is defined then use send2target
3136my $pbstep=shift;           # Which step are we in (0: create, 1: setup, 2: build 3: use)
3137my $pbforce=shift;          # Force stop of VM. Default not.
3138my $snapme = shift;         # By default do not snap a VM/VE/RM
3139my $usesnap = shift;            # By default study the usage of the snapshot feature of VM/VE/RM   
3140my $pbimage=shift;          # Which image to use to start the VM/VE
3141my $vexist = undef;
3142my $vpid = undef;
3143
3144$pbforce = 0 if (not defined $pbforce);
3145$snapme = 0 if (not defined $snapme);
3146$usesnap = 1 if (not defined $usesnap);
3147
3148# Check that v exists
3149die "No VM/VE/RM defined, unable to launch" if (not defined $v);
3150
3151# If creation or snapshot creation mode, no snapshot usable
3152if (($pbstep == 0) || ($snapme == 1)) {
3153    $usesnap = 0;
3154}
3155
3156pb_log(2,"DEBUG: pb_launchv(vtype,v,step,force,snapme,usesnap: $vtype,$v,$pbstep,$pbforce,$snapme,$usesnap)\n");
3157pb_log(2,"DEBUG: pb_launchv(pbimage: $pbimage)\n") if (defined $pbimage);
3158pb_log(2,"DEBUG: pb_launchv(pbscript $pbscript1)\n") if (defined $pbscript1);
3159pb_log(2,"DEBUG: pb_launchv(action: $action)\n") if (defined $action);
3160# Keep only the first VM in case many were given
3161if ($v =~ /,/) {
3162    pb_log(0,"WARNING: pruning to just the first of several VM/VE/RMs listed ($v)\n");
3163    $v =~ s/,.*//;
3164}
3165
3166my $pbos = pb_distro_get_context($v);
3167
3168pb_apply_conf_proxy($pbos);
3169
3170# Launch the VMs/VEs
3171if ($vtype eq "vm") {
3172    my ($ptr,$ptr2,$vmpath,$vmport,$vms) = pb_conf_get("vmtype","vmcmd","vmpath","vmport","vmsize");
3173    my ($vmopt,$vmmm,$vmtmout,$vmsnap,$vmbuildtm,$vmmonport) = pb_conf_get_if("vmopt","vmmem","vmtmout","vmsnap","vmbuildtm","vmmonport");
3174    my $vmsize = pb_distro_get_param($pbos,$vms);
3175
3176    my $vmtype = $ptr->{$ENV{'PBPROJ'}};
3177    my $vmcmd = $ptr2->{$ENV{'PBPROJ'}};
3178
3179    if (defined $opts{'g'}) {
3180        if (($vmtype eq "kvm") || ($vmtype eq "qemu")) {
3181            $ENV{'PBVMOPT'} = "--nographic";
3182        }
3183    }
3184    if (not defined $ENV{'PBVMOPT'}) {
3185        $ENV{'PBVMOPT'} = "";
3186    }
3187    # Save the current status for later restoration
3188    $ENV{'PBOLDVMOPT'} = $ENV{'PBVMOPT'};
3189
3190    if (defined $vmopt->{$v}) {
3191        $ENV{'PBVMOPT'} .= " $vmopt->{$v}" if ($ENV{'PBVMOPT'} !~ / $vmopt->{$v}/);
3192    } elsif (defined $vmopt->{$ENV{'PBPROJ'}}) {
3193        $ENV{'PBVMOPT'} .= " $vmopt->{$ENV{'PBPROJ'}}" if ($ENV{'PBVMOPT'} !~ / $vmopt->{$ENV{'PBPROJ'}}/);
3194    }
3195
3196    # How much memory to allocate for VMs
3197    if (defined $vmmm) {
3198        my $vmmem = pb_distro_get_param($pbos,$vmmm);
3199        if (defined $vmmem) {
3200            $ENV{'PBVMOPT'} .= " -m $vmmem";
3201        }
3202    }
3203
3204    # Are we allowed to use snapshot feature
3205    if ($usesnap == 1) {
3206        if ((defined $vmsnap->{$v}) && ($vmsnap->{$v} =~ /true/i)) {
3207            $ENV{'PBVMOPT'} .= " -snapshot";
3208        } elsif ((defined $vmsnap->{$ENV{'PBPROJ'}}) && ($vmsnap->{$ENV{'PBPROJ'}} =~ /true/i)) {
3209            $ENV{'PBVMOPT'} .= " -snapshot";
3210        } elsif ($pbsnap eq 1) {
3211            $ENV{'PBVMOPT'} .= " -snapshot";
3212        }
3213    } 
3214    if ($snapme != 0) {
3215        if (($vmtype eq "kvm") || ($vmtype eq "qemu")) {
3216            # Configure the monitoring to automate the creation of the 'pb' snapshot
3217            $ENV{'PBVMOPT'} .= " -serial mon:telnet::$vmmonport->{$ENV{'PBPROJ'}},server,nowait" if ((defined $vmmonport) && (defined $vmmonport->{$ENV{'PBPROJ'}}));
3218            # In that case no snapshot call needed
3219            $ENV{'PBVMOPT'} =~ s/ -snapshot//;
3220        }
3221    }
3222   
3223    # Set a default timeout of 2 minutes if none defined externally or in conf files
3224    if (not defined $ENV{'PBVMTMOUT'}) {
3225        if (defined $vmtmout->{$v}) {
3226            $ENV{'PBVMTMOUT'} = $vmtmout->{$v};
3227        } elsif (defined $vmtmout->{$ENV{'PBPROJ'}}) {
3228            $ENV{'PBVMTMOUT'} = $vmtmout->{$ENV{'PBPROJ'}};
3229        }
3230    }
3231    if (not defined $ENV{'PBVMTMOUT'}) {
3232        $ENV{'PBVMTMOUT'} = "120";
3233    }
3234
3235    my $nport = pb_get_port($vmport,$pbos,$vtype);
3236
3237    my $cmd;
3238    my $vmm;        # has to be used for pb_check_ps
3239    if (($vmtype eq "qemu") || ($vmtype eq "kvm")) {
3240        $vmm = "$vmpath->{$ENV{'PBPROJ'}}/$v.qemu";
3241        if (($pbstep == 0) || (defined $pbimage)) {
3242            $ENV{'PBVMOPT'} .= " -cdrom $pbimage -boot d";
3243        }
3244        # Always redirect the network and always try to use a 'pb' snapshot
3245        #$cmd = "$vmcmd $ENV{'PBVMOPT'} -net nic,model=virtio -net user,hostfwd=tcp::$nport:22 -loadvm pb $vmm"
3246        #$cmd = "$vmcmd $ENV{'PBVMOPT'} -redir tcp:$nport:10.0.2.15:22 $vmm"
3247        $cmd = "$vmcmd $ENV{'PBVMOPT'} -net nic -net user,hostfwd=tcp:127.0.0.1:$nport-:22 $vmm"
3248    } elsif ($vmtype eq "xen") {
3249    } elsif ($vmtype eq "vmware") {
3250    } else {
3251        die "VM of type $vmtype not supported. Report to the dev team";
3252    }
3253    # Restore the ENV VAR Value
3254    $ENV{'PBVMOPT'} = $ENV{'PBOLDVMOPT'};
3255
3256    my ($tmpcmd,$void) = split(/ +/,$cmd);
3257    my $vmmport = undef;
3258    ($vexist,$vmmport) = pb_check_ps($tmpcmd,$vmm);
3259    $vpid = 0;
3260   
3261    if (! $vexist) {
3262        if ($pbstep == 0) {
3263            die("Found an existing Virtual machine $vmm. Won't overwrite") if (-r $vmm);
3264            if (($vmtype eq "qemu") || ($vmtype eq "xen") || ($vmtype eq "kvm")) {
3265                my $command = pb_check_req("qemu-img",0);
3266                pb_system("$command create -f qcow2 $vmm $vmsize","Creating the QEMU VM");
3267            } elsif ($vmtype eq "vmware") {
3268            } else {
3269            }
3270        }
3271        if (! -f "$vmm") {
3272            pb_log(0,"Unable to find VM $vmm\n");
3273        } else {
3274            # Is the SSH port free? if not kill the existing process using it after a build timeout period
3275            my ($vmssh,$void) = pb_check_ps($tmpcmd,$vmm);
3276            if ($vmssh) {
3277                my $buildtm = $ENV{'PBVMTMOUT'};
3278                if (defined $vmbuildtm->{$v}) {
3279                    $buildtm = $vmbuildtm->{$v};
3280                } elsif (defined $vmbuildtm->{$ENV{'PBPROJ'}}) {
3281                    $buildtm = $vmbuildtm->{$ENV{'PBPROJ'}};
3282                }
3283
3284                sleep $buildtm;
3285                pb_log(0,"WARNING: Killing the process ($vmssh) using port $void (previous failed VM ?)\n");
3286                kill 15,$vmssh;
3287                # Let it time to exit
3288                sleep 5;
3289            }
3290            pb_system("$cmd &","Launching the VM $vmm");
3291            # Using system allows to kill it externaly if needed,sosupport that in the call
3292            pb_system("sleep $ENV{'PBVMTMOUT'}","Waiting $ENV{'PBVMTMOUT'} s for VM $v to come up","mayfail");
3293            ($vpid,$void) = pb_check_ps($tmpcmd,$vmm);
3294            pb_log(0,"VM $vmm launched (pid $vpid)\n");
3295        }
3296    } else {
3297        pb_log(0,"Found an existing VM $vmm (pid $vexist - Port $vmmport)\n");
3298        # Set the correct port here based on what is done
3299        $pbport = $vmmport;
3300        $vpid = $vexist;
3301    }
3302    pb_log(2,"DEBUG: pb_launchv returns ($vexist,$vpid)\n");
3303
3304    # After the VM is launched, then setup the SSH access if not in creation phase
3305    if ($pbstep != 0) {
3306        if (defined $pbport) {
3307            pb_ssh_setup($vtype,$pbport);
3308        } else {
3309            pb_ssh_setup($vtype,$nport);
3310        }
3311    }
3312   
3313    # Skip that VM if something went wrong
3314    return($vexist,$vpid) if (($vpid == 0) && ($vexist == 0));
3315   
3316    # If force stopping the VM then reset vexist
3317    if ($pbforce == 1) {
3318        $vpid = $vexist;
3319        $vexist = 0;
3320    }
3321} elsif ($vtype eq "ve") {
3322    # Force the creation of the VE and no snapshot usable
3323    $vetype = pb_ve_get_type($vetype);
3324    pb_ve_launch($v,$pbstep,$usesnap,$vetype,$pbimage);
3325    $vexist = 0;
3326    $vpid = 0;
3327} else {
3328    # Get RM context
3329    my ($ptr,$rmpath,$rmport) = pb_conf_get("rmtype","rmpath","rmport");
3330
3331    # Nothing more to do for RM. No real launch
3332    # For the moment we support the RM is already running
3333    # For ProLiant may be able to power them on if needed later on as an example.
3334    $vexist = 0;
3335    $vpid = 0;
3336   
3337    # After the RM is launched, then setup the SSH access
3338    my $nport = pb_get_port($rmport,$pbos,$vtype);
3339    pb_ssh_setup($vtype,$nport);
3340}
3341
3342# Gather all required files to send them to the VM/VE/RM
3343# and launch the build through pbscript
3344if (defined $action) {
3345    pb_log(2,"DEBUG: Before send2target, vexist: $vexist, vpid: $vpid, vtype: $vtype, v: $v, action: $action\n");
3346    pb_send2target($action,$pbscript1,$v,$vexist,$vpid,$snapme,$pbstep);
3347}
3348return($vexist,$vpid);
3349}
3350
3351# Return string for date synchro
3352sub pb_date2v {
3353
3354my $vtype = shift;
3355my $pbos = shift;
3356my $nontp = shift || 0;
3357
3358pb_log(1,"Entering pb_date2v\n");
3359# VE gets time from parent OS.
3360return "/bin/true" if ($vtype) =~ /^ve/o; 
3361
3362my ($ntp) = pb_conf_get_if($vtype."ntp");
3363my $vntp = undef;
3364my $ntpline = undef;
3365$vntp = $ntp->{$ENV{'PBPROJ'}} if (defined $ntp);
3366
3367if ((defined $vntp) && ($vntp !~ /^\s*$/)) {
3368    pb_log(2,"ntp server is $vntp\n");
3369    # ntp command depends on pbos
3370    my $vntpcmd = pb_distro_get_param($pbos,pb_conf_get("oscmdntp"));
3371    $ntpline = "sudo $vntpcmd $vntp";
3372}
3373# Force new date to be in the future compared to the date
3374# of the host by adding 1 minute
3375my @date=pb_get_date();
3376$date[1]++;
3377my $upddate = strftime("%m%d%H%M%Y", @date);
3378my $dateline = "sudo /bin/date $upddate";
3379if ((defined $ntpline) && ($nontp == 0)) {
3380    pb_log(1,"pb_date2v returns $ntpline\n");
3381    return($ntpline);
3382} else {
3383    pb_log(1,"pb_date2v returns $dateline\n");
3384    return($dateline);
3385}
3386}
3387
3388sub pb_build2v {
3389
3390my $vtype = shift;
3391my $action = shift;     #build, test or prep
3392my $pbforce=shift;      # Force stop of VM. Default not.
3393
3394$action = "build" if (not defined $action);
3395$pbforce = 0 if (not defined $pbforce);
3396
3397my $pbstep;
3398
3399$vetype = pb_ve_get_type($vetype);
3400if ($action eq "prep") {
3401    $pbstep = 2;
3402} else {
3403    $pbstep = 3;
3404    # Need to be traversed by others if using containers
3405    if (($vtype eq "ve") && ($vetype eq "docker")) {
3406        chmod 0751,"$ENV{'PBBUILDDIR'}";
3407    }
3408}
3409
3410pb_log(1,"Entering in pb_build2v ($action in $vtype) \n");
3411
3412# We prepare scripts for all V of this vtype to allow sharing that part
3413my ($v,$all) = pb_get2v($vtype);
3414
3415# Send tar files when we do a global generation
3416pb_build2ssh() if (($all == 1) && ($action eq "build"));
3417
3418# Adapt // mode to memory size
3419$pbparallel = pb_set_parallel($vtype);
3420
3421my ($vexist,$vpid) = (undef,undef);
3422my $pm;
3423my $all_ok = 1;
3424if (defined $pbparallel) {
3425    $pm = new Parallel::ForkManager($pbparallel);
3426
3427    # Set which port the VM/RM will use to communicate
3428    $pm->run_on_start(\&pb_set_port);
3429    $pm->run_on_finish(sub { my ($pid, $code, $id, $signal, $dump) = @_;
3430                            unless ($code == 0 && $signal == 0 && $dump == 0) {
3431                                $all_ok = 0;
3432                                pb_log(0,"ERROR: pid $pid failed\n");
3433                            }
3434                        });
3435}
3436
3437my $counter = 0;
3438foreach my $v (@$v) {
3439    $counter++;
3440    $pbscript{$v} = "$ENV{'PBDESTDIR'}/pb$action-$v";
3441    # Modulo 2 * pbparallel (to avoid synchronization problems)
3442    $counter = 1 if ((defined $pbparallel) && ($counter > 2 * $pbparallel));
3443    $pm->start($counter) and next if (defined $pbparallel);
3444    #
3445    # Prepare the script to be executed on the VM/VE/RM
3446    open(SCRIPT,"> $pbscript{$v}") || die "Unable to create $pbscript{$v}";
3447    print SCRIPT "#!/bin/bash\n";
3448
3449    # Transmit the verbosity level to the virtual env/mach.
3450    my $verbose = "";
3451    my $i = 0;                          # minimal debug level
3452    while ($i lt $pbdebug) {
3453        $verbose .= "-v ";
3454        $i++;
3455    }
3456    print SCRIPT "set -e\n" if ($Global::pb_stop_on_error);
3457    # Activate script verbosity if at least 2 for pbdebug
3458    print SCRIPT "set -x\n" if ($i gt 1);
3459    # Quiet if asked to be so on the original system
3460    $verbose = "-q" if ($pbdebug eq -1);
3461
3462    print SCRIPT "echo ... Execution needed\n";
3463    print SCRIPT "echo ==== Start of script for $vtype $v =====\n";
3464    print SCRIPT "# This is in directory pbdelivery\n";
3465    print SCRIPT "# Setup the variables required for building\n";
3466    print SCRIPT "export PBPROJ=$ENV{'PBPROJ'}\n";
3467
3468    if (($action eq "build") || ($action eq "prep")) {
3469        print SCRIPT "# Preparation for pb\n";
3470        print SCRIPT "rm -f \$HOME/.pbrc.yml\n";
3471        print SCRIPT "rm -f \$HOME/.pbrc\n";
3472        print SCRIPT "cp .pbrc.yml \$HOME\n";
3473        # TODO: Why ?
3474        print SCRIPT "cd ..\n";
3475    }
3476
3477    # VE needs a good /proc, tolerate one being potentially left around after a failure
3478    if (($vtype eq "ve") && ($vetype ne "docker")) {
3479        print SCRIPT "[ -d /proc/1 ] || sudo /bin/mount -t proc /proc /proc\n";
3480    }
3481
3482    # Get distro context
3483    my $pbos = pb_distro_get_context($v);
3484   
3485    my $ntpline = pb_date2v($vtype,$pbos);
3486    print SCRIPT "# Time sync\n";
3487    print SCRIPT "echo setting up date with $ntpline\n";
3488    print SCRIPT "$ntpline\n";
3489    print SCRIPT "if [ \$\? -ne 0 ]; then\n";
3490    # Force if bad status a trick using date
3491    $ntpline = pb_date2v($vtype,$pbos, 1);
3492    print SCRIPT "  $ntpline\n";
3493    print SCRIPT "fi\n";
3494    # Use potential local proxy declaration in case we need it to download repo, pkgs, ...
3495    if (defined $ENV{'http_proxy'}) {
3496        print SCRIPT "export http_proxy=\"$ENV{'http_proxy'}\"\n";
3497    }
3498    if (defined $ENV{'https_proxy'}) {
3499        print SCRIPT "export https_proxy=\"$ENV{'https_proxy'}\"\n";
3500    }
3501    if (defined $ENV{'ftp_proxy'}) {
3502        print SCRIPT "export ftp_proxy=\"$ENV{'ftp_proxy'}\"\n";
3503    }
3504
3505    # Get list of packages to build/test and get some ENV vars as well
3506    my $ptr = pb_get_pkg();
3507    @pkgs = @$ptr;
3508    my $p = join(' ',@pkgs) if (@pkgs);
3509    print SCRIPT "export PBPROJVER=$ENV{'PBPROJVER'}\n";
3510    print SCRIPT "export PBPROJTAG=$ENV{'PBPROJTAG'}\n";
3511    print SCRIPT "export PBPACKAGER=\"$ENV{'PBPACKAGER'}\"\n";
3512    # TODO: setup apk pkg/keys here
3513
3514    # We may need to do some other tasks before building. Read a script here to finish setup
3515    if (-x "$ENV{'PBTMP'}/pb$vtype".".pre") {
3516        print SCRIPT "# Special pre-instructions to be launched\n";
3517        print SCRIPT pb_get_content("$ENV{'PBTMP'}/pb$vtype".".pre");
3518    }
3519
3520    if (-x "$ENV{'PBTMP'}/pb$vtype"."$action.pre") {
3521        print SCRIPT "# Special pre-$action instructions to be launched\n";
3522        print SCRIPT pb_get_content("$ENV{'PBTMP'}/pb$vtype"."$action.pre");
3523    }
3524
3525    print SCRIPT "# $action\n";
3526    print SCRIPT "echo $action"."ing packages on $vtype...\n";
3527
3528    if (($action eq "test") && (! -x "$ENV{'PBDESTDIR'}/pbtest")) {
3529        confess "No test script ($ENV{'PBDESTDIR'}/pbtest) found when in test mode. Aborting ...";
3530    }
3531    print SCRIPT "pb --version\n";
3532    my $act = "$action"."2pkg";
3533    $act = "build2prep" if ($action eq "prep");
3534    print SCRIPT "pb $verbose -p $ENV{'PBPROJ'} $act $p\n";
3535
3536    if (($vtype eq "ve") && ($vetype ne "docker")) {
3537        print SCRIPT "sudo /bin/umount /proc\n";
3538    }
3539
3540    # We may need to do some other tasks after building. Read a script here to exit properly
3541    if (-x "$ENV{'PBTMP'}/pb$vtype"."$action.post") {
3542        print SCRIPT "# Special post-$action instructions to be launched\n";
3543        print SCRIPT pb_get_content("$ENV{'PBTMP'}/pb$vtype"."$action.post");
3544    }
3545
3546    if (-x "$ENV{'PBTMP'}/pb$vtype".".post") {
3547        print SCRIPT "# Special post-instructions to be launched\n";
3548        print SCRIPT pb_get_content("$ENV{'PBTMP'}/pb$vtype".".post");
3549    }
3550
3551    print SCRIPT "echo ==== End of script $$ for $vtype $v =====\n";
3552    close(SCRIPT);
3553    chmod 0755,"$pbscript{$v}";
3554   
3555    $pm->finish if (defined $pbparallel);
3556}
3557$pm->wait_all_children if (defined $pbparallel);
3558die "Aborting, one or more of the children failed." if ((not $all_ok) && ($Global::pb_stop_on_error));
3559
3560# Launch the VM/VE/RM
3561# docker image to use if needed
3562# Gather all required files to send them to the VM/VE
3563# and launch the build through pbscript
3564pb_parallel_launchv(\%pbscript,$vtype,uc($vtype).$action,$pbstep,$pbimage,$pbforce);
3565}
3566
3567
3568sub pb_clean {
3569
3570    my $sleep=10;
3571    die "Unable to get env var PBDESTDIR" if (not defined $ENV{'PBDESTDIR'});
3572    die "Unable to get env var PBBUILDDIR" if (not defined $ENV{'PBBUILDDIR'});
3573    pb_log(0,"We will now wait $sleep s before removing both directories\n$ENV{'PBDESTDIR'} and $ENV{'PBBUILDDIR'}.\nPlease break me if this is wrong\n");
3574    sleep $sleep;
3575    pb_rm_rf($ENV{'PBDESTDIR'});
3576    pb_rm_rf($ENV{'PBBUILDDIR'});
3577}
3578
3579sub pb_newver {
3580
3581    die "-V Version parameter needed" if ((not defined $newver) || ($newver eq ""));
3582
3583    # Need this call for PBDIR
3584    my ($scheme2,$uri) = pb_vcs_init($pbinit);
3585
3586    my ($pbconf,$pbprojurl) = pb_conf_get("pbconfurl","pbprojurl");
3587    $uri = $pbconf->{$ENV{'PBPROJ'}};
3588    my ($scheme, $account, $host, $port, $path) = pb_get_uri($uri);
3589
3590    # Checking CMS repositories status
3591    ($scheme2, $account, $host, $port, $path) = pb_get_uri($pbprojurl->{$ENV{'PBPROJ'}});
3592
3593    if ($scheme !~ /^svn/) {
3594        die "Only SVN is supported at the moment";
3595    }
3596
3597    my $res = pb_vcs_isdiff($scheme,$ENV{'PBROOTDIR'});
3598    die "ERROR: No differences accepted in CMS for $ENV{'PBROOTDIR'} before creating a new version" if ($res != 0);
3599
3600    $res = pb_vcs_isdiff($scheme2,$ENV{'PBDIR'});
3601    die "ERROR: No differences accepted in CMS for $ENV{'PBDIR'} before creating a new version" if ($res != 0);
3602
3603    # Tree identical between PBCONFDIR and PBROOTDIR. The delta is what
3604    # we want to get for the root of the new URL
3605
3606    my $oldver = $ENV{'PBROOTDIR'};
3607    $oldver =~ s|^$ENV{'PBCONFDIR'}||;
3608
3609    pb_log(2, "PBCONFDIR: $ENV{'PBCONFDIR'}\nPBROOTDIR: $ENV{'PBROOTDIR'}\n");
3610
3611    my $newurl = "$uri/$newver";
3612    # Should probably use projver in the old file
3613    my $oldvertxt= basename($oldver);
3614    my $newvertxt = basename($newver);
3615
3616    # Duplicate and extract project-builder part
3617    pb_log(2,"Copying $uri/$oldver to $newurl\n");
3618    pb_vcs_copy($scheme,"$uri/$oldver",$newurl);
3619    pb_log(2,"Checkout $newurl to $ENV{'PBCONFDIR'}/$newver\n");
3620    pb_vcs_up($scheme,"$ENV{'PBCONFDIR'}");
3621
3622    # Duplicate and extract project
3623    my $newurl2 = "$pbprojurl->{$ENV{'PBPROJ'}}/$newver";
3624
3625    pb_log(2,"Copying $pbprojurl->{$ENV{'PBPROJ'}}/$oldver to $newurl2\n");
3626    pb_vcs_copy($scheme2,"$pbprojurl->{$ENV{'PBPROJ'}}/$oldver",$newurl2);
3627
3628    my $tmp = $ENV{'PBDIR'};
3629    $tmp =~ s|$oldver$||;
3630    pb_log(2,"Checkout $newurl2 to $tmp/$newver\n");
3631    pb_vcs_up($scheme2,"$tmp");
3632
3633    # Update the .yml file
3634    open(FILE,"$ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml") || die "Unable to open $ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml";
3635    open(OUT,"> $ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml.new") || die "Unable to write to $ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml.new";
3636    while(<FILE>) {
3637        if (/^projver\s+$ENV{'PBPROJ'}\s*=\s*$oldvertxt$/) {
3638            s/^projver\s+$ENV{'PBPROJ'}\s*=\s*$oldvertxt$/projver $ENV{'PBPROJ'} = $newvertxt/;
3639            pb_log(0,"Changing projver from $oldvertxt to $newvertxt in $ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml\n");
3640        }
3641        if (/^testver/) {
3642            s/^testver/#testver/;
3643            pb_log(0,"Commenting testver in $ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml\n") if (/^testver/);
3644        }
3645        if (/^delivery/) {
3646            my $txt = $_;
3647            chomp($txt);
3648            pb_log(0,"Please check delivery ($txt) in $ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml\n");
3649        }
3650        print OUT $_;
3651    }
3652    close(FILE);
3653    close(OUT);
3654    rename("$ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml.new","$ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml");
3655
3656    # Checking pbcl files
3657    foreach my $f (<$ENV{'PBROOTDIR'}/*/pbcl>) {
3658        # Compute new pbcl file
3659        my $f2 = $f;
3660        $f2 =~ s|$ENV{'PBROOTDIR'}|$ENV{'PBCONFDIR'}/$newver/|;
3661        open(PBCL,$f) || die "Unable to open $f";
3662        my $foundnew = 0;
3663        while (<PBCL>) {
3664            $foundnew = 1 if (/^$newvertxt \(/);
3665        }
3666        close(PBCL);
3667        open(OUT,"> $f2") || die "Unable to write to $f2: $!";
3668        open(PBCL,$f) || die "Unable to open $f";
3669        # We skip as long as we do not reach the new version in the log (case of devel)
3670        my $skip = 1;
3671        # We let the 4 first lines
3672        my $cnt = 0;
3673        while (<PBCL>) {
3674            $skip = 0 if (/^$newvertxt \(/);
3675            if ((/^$oldvertxt \(/) && ($foundnew == 0)) {
3676                print OUT "$newvertxt ($pbdate)\n";
3677                print OUT "- TBD\n";
3678                print OUT "\n";
3679                pb_log(0,"WARNING: version $newvertxt not found in $f so added to $f2...\n") if ($foundnew == 0);
3680            }
3681            if ($cnt <= 4) {
3682                print OUT "$_" if (not /^$oldvertxt \(/);
3683            } else {
3684                print OUT "$_" if ($skip == 0);
3685            }
3686            $cnt++;
3687        }
3688        close(OUT);
3689        close(PBCL);
3690    }
3691
3692    pb_log(2,"Checkin $ENV{'PBCONFDIR'}/$newver\n");
3693    pb_vcs_checkin($scheme,"$ENV{'PBCONFDIR'}/$newver","updated to $ENV{'PBCONFDIR'}");
3694}
3695
3696#
3697# Return the list of VMs/VEs/RMs we are working on
3698# $all is a flag to know if we return all of them
3699# or only some (if all we publish also tar files in addition to pkgs
3700#
3701sub pb_get2v {
3702
3703my $vtype = shift;
3704my @v;
3705my $all = 0;
3706my $pbv = 'PBV';
3707my $vlist = $vtype."list";
3708
3709# Get VM/VE list
3710if ((not defined $ENV{$pbv}) || ($ENV{$pbv} =~ /^all$/)) {
3711    my ($ptr) = pb_conf_get($vlist);
3712    $ENV{$pbv} = $ptr->{$ENV{'PBPROJ'}};
3713    $all = 1;
3714}
3715pb_log(2,"$vtype: $ENV{$pbv}\n");
3716@v = split(/,/,$ENV{$pbv});
3717return(\@v,$all);
3718}
3719
3720# This function creates a giant script to configure a particular VM/VE/RM, it then copies the
3721# script to the target.
3722
3723# Function to create a potentialy missing pb account on the VM/VE/RM, and adds it to sudo
3724# Needs to use root account to connect to the VM/VE/RM
3725# pb will take your local public SSH key to access
3726# the pb account in the VM/VE/RM later on if needed
3727sub pb_setup2v {
3728
3729my $vtype = shift;
3730my $sbx = shift;
3731my $pbstep = 1;
3732
3733my ($vm,$all) = pb_get2v($vtype);
3734
3735# Adapt // mode to memory size
3736$pbparallel = pb_set_parallel($vtype);
3737
3738my $pm;
3739my $all_ok = 1;
3740if (defined $pbparallel) {
3741    $pm = new Parallel::ForkManager($pbparallel);
3742
3743    # Set which port the VM/RM will use to communicate
3744    $pm->run_on_start(\&pb_set_port);
3745    $pm->run_on_finish(sub { my ($pid, $code, $id, $signal, $dump) = @_;
3746                            $all_ok = 0 unless (($code == 0) && ($signal == 0) && ($dump == 0)); 
3747                        });
3748}
3749
3750my $counter = 0;
3751foreach my $v (@$vm) {
3752    # Script generated - should be before the fork - after the %pbscript hash isn't updated correctly
3753    $pbscript{$v} = "$ENV{'PBDESTDIR'}/setupv-$v";
3754
3755    $counter++;
3756    if (defined $pbparallel) {
3757        # Modulo 2 * pbparallel (to avoid synchronization problems)
3758        $counter = 1 if ($counter > 2 * $pbparallel);
3759        $pm->start($counter) and next;
3760    }
3761
3762    # Get distro context
3763    my $pbos = pb_distro_get_context($v);
3764   
3765    # Deal with date sync.
3766    my $ntpline = pb_date2v($vtype,$pbos);
3767
3768    # Name of the account to deal with for VM/VE/RM
3769    # Do not use the one passed potentially with -a
3770    my ($pbac) = pb_conf_get($vtype."login");
3771    confess "No $vtype"."login defined in your project yml conf file" if ((not defined $pbac) || (not defined $pbac->{$ENV{'PBPROJ'}}));
3772
3773    my $key = "";
3774
3775    # Prepare the script to be executed on the VM/VE/RM
3776    # in $ENV{'PBDESTDIR'}/setupv
3777
3778    pb_log(2,"Preparing setup script for $v in $pbscript{$v}\n");
3779    open(SCRIPT,"> $pbscript{$v}") || die "Unable to create $pbscript{$v}";
3780   
3781    print SCRIPT << 'EOF';
3782#!/usr/bin/perl -w
3783
3784use strict;
3785use File::Copy;
3786
3787# We should not need in this script more functions than what is provided
3788# by Base, Conf and Distribution to avoid problems at exec time.
3789# They are appended at the end.
3790
3791# Define mandatory global vars
3792EOF
3793    my $ppref = "our";
3794    $ppref = "my" if (($pbos->{'name'} =~ /^redhat/) && ($pbos->{'version'} =~ /^6/));
3795    print SCRIPT << "EOF";
3796$ppref \$pbdebug = $pbdebug;
3797$ppref \$pbLOG;
3798$ppref \$pbsynmsg = "pbscript";
3799$ppref \$pbdisplaytype = "text";
3800$ppref \$pblocale = "";
3801$ppref \$PBCONFVER = $projectbuilderconfver;
3802EOF
3803    print SCRIPT << 'EOF';
3804pb_log_init($pbdebug, $pbLOG);
3805EOF
3806    print SCRIPT << "EOF";
3807\$Global::pb_stop_on_error = $Global::pb_stop_on_error;
3808pb_temp_init($pbkeep);
3809pb_conf_init("$ENV{'PBPROJ'}");
3810
3811EOF
3812
3813    if ($vtype eq "ve") {
3814        $vetype = pb_ve_get_type($vetype);
3815        if ($vetype ne "docker") {
3816            print SCRIPT << "EOF";
3817# For VE we first need to mount some FS
3818pb_system("mount -t proc /proc /proc") unless (-d "/proc/$$");
3819EOF
3820        }
3821        print SCRIPT << "EOF";
3822# For VE we need a good null dev
3823# Except for RedHat 6.2 where it's good and doesn't like being recreated
3824EOF
3825        print SCRIPT "pb_system('rm -f /dev/null; mknod /dev/null c 1 3; chmod 777 /dev/null');\n" unless (($pbos->{'name'} =~ /^redhat/) && ($pbos->{'version'} =~ /^6/));
3826    } elsif ($vtype =~ /(v|r)m/) {
3827        # Nothing to do now.
3828        my $useless = 0;
3829    } else {
3830        die "Unknown virtual type $vtype";
3831    }
3832
3833    print SCRIPT << 'EOF';
3834
3835# Adds $pbac->{$ENV{'PBPROJ'}} as an account if needed
3836#
3837my $file="/etc/passwd";
3838open(PBFILE,$file) || die "Unable to open $file";
3839my $found = 0;
3840while (<PBFILE>) {
3841EOF
3842    print SCRIPT << "EOF";
3843    \$found = 1 if (/^$pbac->{$ENV{'PBPROJ'}}:/);
3844EOF
3845
3846# TODO: use an external parameter
3847my $home = "/home";
3848# Solaris doesn't like that we use /home
3849$home = "/export/home" if ($pbos->{'type'} eq "pkg");
3850
3851    print SCRIPT << "EOF";
3852}
3853close(PBFILE);
3854
3855if ( \$found == 0 ) {
3856    if ( ! -d "$home" ) {
3857        pb_mkdir_p("$home");
3858    }
3859EOF
3860    # TODO: Level of portability of these cmds ? Critical now for RM
3861    # TODO: use an external parameter
3862    print SCRIPT << "EOF";
3863    # Most Linux distros
3864    if ( -x "/usr/sbin/groupadd" ) {
3865        pb_system("/usr/sbin/groupadd $pbac->{$ENV{'PBPROJ'}}","Adding group $pbac->{$ENV{'PBPROJ'}}");
3866    # Alpine with busybox
3867    } elsif ( -x "/usr/sbin/addgroup" ) {
3868        pb_system("/usr/sbin/addgroup $pbac->{$ENV{'PBPROJ'}}","Adding group $pbac->{$ENV{'PBPROJ'}}");
3869    } else {
3870        die "Unable to find a command to add a group on this distribution";
3871    }
3872    # Most Linux distros
3873    if ( -x "/usr/sbin/useradd" ) {
3874        pb_system("/usr/sbin/useradd -g $pbac->{$ENV{'PBPROJ'}} -m -d $home/$pbac->{$ENV{'PBPROJ'}} $pbac->{$ENV{'PBPROJ'}}","Adding user $pbac->{$ENV{'PBPROJ'}} (group $pbac->{$ENV{'PBPROJ'}} - home $home/$pbac->{$ENV{'PBPROJ'}})");
3875    # Alpine with busybox
3876    } elsif ( -x "/usr/sbin/adduser" ) {
3877        pb_system("/usr/sbin/adduser -G $pbac->{$ENV{'PBPROJ'}} -h $home/$pbac->{$ENV{'PBPROJ'}} -D $pbac->{$ENV{'PBPROJ'}}","Adding user $pbac->{$ENV{'PBPROJ'}} (group $pbac->{$ENV{'PBPROJ'}} - home $home/$pbac->{$ENV{'PBPROJ'}})");
3878        pb_system("/usr/sbin/addgroup $pbac->{$ENV{'PBPROJ'}} abuild","Adding $pbac->{$ENV{'PBPROJ'}} to group abuild");
3879    } else {
3880        die "Unable to find a command to add a user on this distribution";
3881    }
3882}
3883EOF
3884
3885    # Copy the content of our local conf file to the VM/VE/RM
3886    my $content = pb_get_content(pb_distro_conffile());
3887    print SCRIPT << "EOF";
3888    #
3889    # Create a temporary local conf file for distribution support
3890    # This is created here before its use later. Its place is hardcoded, so no choice for the path
3891    #
3892    my \$tempconf = pb_distro_conffile();
3893    pb_mkdir_p(dirname(\$tempconf));
3894    open(CONF,"> \$tempconf") || die "Unable to create \$tempconf";
3895    print CONF q{$content};
3896    close(CONF);
3897EOF
3898
3899    if ($vtype =~ /(v|r)m/) {
3900        print SCRIPT << "EOF";
3901# allow ssh entry to build
3902#
3903mkdir "$home/$pbac->{$ENV{'PBPROJ'}}/.ssh",0700;
3904# Allow those accessing root to access the build account
3905copy("\$ENV{'HOME'}/.ssh/authorized_keys","$home/$pbac->{$ENV{'PBPROJ'}}/.ssh/authorized_keys");
3906chmod 0600,".ssh/authorized_keys";
3907pb_system("chown -R $pbac->{$ENV{'PBPROJ'}}:$pbac->{$ENV{'PBPROJ'}} $home/$pbac->{$ENV{'PBPROJ'}}","Finish setting up the account env for $pbac->{$ENV{'PBPROJ'}}");
3908
3909EOF
3910}
3911    print SCRIPT << 'EOF';
3912# No passwd for build account only keys
3913$file="/etc/shadow";
3914if (-f $file) {
3915    open(PBFILE,$file) || die "Unable to open $file";
3916    open(PBOUT,"> $file.new") || die "Unable to open $file.new";
3917    while (<PBFILE>) {
3918EOF
3919    print SCRIPT << "EOF";
3920        s/^$pbac->{$ENV{'PBPROJ'}}:\!\!:/$pbac->{$ENV{'PBPROJ'}}:*:/;
3921        s/^$pbac->{$ENV{'PBPROJ'}}:\!:/$pbac->{$ENV{'PBPROJ'}}:*:/; #SLES 9 e.g.
3922        s/^$pbac->{$ENV{'PBPROJ'}}:\\*LK\\*:/$pbac->{$ENV{'PBPROJ'}}:NP:/;  #Solaris e.g.
3923EOF
3924        print SCRIPT << 'EOF';
3925        print PBOUT $_;
3926    }
3927    close(PBFILE);
3928    close(PBOUT);
3929    rename("$file.new",$file);
3930    chmod 0640,$file;
3931    }
3932
3933# TODO: systemd support
3934# Keep the VM in text mode
3935$file="/etc/inittab";
3936if (-f $file) {
3937    open(PBFILE,$file) || die "Unable to open $file";
3938    open(PBOUT,"> $file.new") || die "Unable to open $file.new";
3939    while (<PBFILE>) {
3940        s/^(..):5:initdefault:$/$1:3:initdefault:/;
3941        print PBOUT $_;
3942    }
3943    close(PBFILE);
3944    close(PBOUT);
3945    rename("$file.new",$file);
3946    chmod 0640,$file;
3947}
3948
3949# pb has to be added to portage group on gentoo
3950
3951# We need to have that pb_distro_get_context function
3952# Get it from ProjectBuilder::Distribution
3953# And we now need the conf file required for this to work created above
3954
3955my $pbos = pb_distro_get_context(); 
3956print "distro tuple: ".Dumper($pbos)."\n";
3957
3958# Adapt sudoers
3959# sudo is not default on Solaris and needs to be installed first
3960# from http://www.sunfreeware.com/programlistsparc10.html#sudo
3961if ($pbos->{'type'} eq "pkg") {
3962    $file="/usr/local/etc/sudoers";
3963} else {
3964    $file="/etc/sudoers";
3965}
3966open(PBFILE,$file) || die "Unable to open $file";
3967open(PBOUT,"> $file.new") || die "Unable to open $file.new";
3968while (<PBFILE>) {
3969EOF
3970    # Skip what will be generated
3971    print SCRIPT << "EOF";
3972    next if (/^$pbac->{$ENV{'PBPROJ'}}\\s+/);
3973    next if (/^Defaults:$pbac->{$ENV{'PBPROJ'}}\\s+/);
3974    next if (/^Defaults:root\\s+\!requiretty/);
3975    next if (/^Defaults:root\\s+env_keep/);
3976EOF
3977    print SCRIPT << 'EOF';
3978    s/Defaults\s+requiretty/# disable Defaults: requiretty/;
3979    print PBOUT $_;
3980}
3981close(PBFILE);
3982EOF
3983    print SCRIPT << "EOF";
3984# Some distro force requiretty at compile time, so disable here
3985print PBOUT "Defaults:$pbac->{$ENV{'PBPROJ'}} !requiretty\\n";
3986print PBOUT "Defaults:root !requiretty\\n";
3987EOF
3988    # RH 6.2 sudo doesn't support env_keep
3989    if (($pbos->{'name'} ne "redhat") && ($pbos->{'version'} ne "6.2")) {
3990        print SCRIPT << "EOF";
3991# Keep proxy configuration while using sudo
3992print PBOUT "Defaults:$pbac->{$ENV{'PBPROJ'}}    env_keep += \\\"http_proxy https_proxy ftp_proxy\\\"\\n";
3993print PBOUT "Defaults:root    env_keep += \\\"http_proxy https_proxy ftp_proxy\\\"\\n";
3994EOF
3995    }
3996    # Try to restrict security to what is really needed
3997    if ($vtype =~ /^vm/) {
3998        my $hpath = pb_distro_get_param($pbos,pb_conf_get("ospathcmd-halt"));
3999        my @sudocmds = pb_get_sudocmds($pbos,$ntpline,"sudo $hpath","sudo /bin/date");
4000        print SCRIPT << "EOF";
4001# This is needed in order to be able on VM to halt the machine from the $pbac->{$ENV{'PBPROJ'}} account at least
4002# Build account $pbac->{$ENV{'PBPROJ'}} in VM also needs to setup date and install deps.
4003# Nothing else should be needed
4004EOF
4005        foreach my $c (@sudocmds) {
4006            print SCRIPT "print PBOUT \"$pbac->{$ENV{'PBPROJ'}}   ALL = NOPASSWD: $c\\n\";\n";
4007        }
4008    } elsif ($vtype =~ /^rm/) {
4009        my @sudocmds = pb_get_sudocmds($pbos,$ntpline);
4010        print SCRIPT << "EOF";
4011# Build account $pbac->{$ENV{'PBPROJ'}} in RM only needs to setup date and install deps if needed each time
4012EOF
4013        foreach my $c (@sudocmds) {
4014            print SCRIPT "print PBOUT \"$pbac->{$ENV{'PBPROJ'}}   ALL = NOPASSWD: $c\\n\";\n";
4015        }
4016    } else {
4017        print SCRIPT << "EOF";
4018# Build account $pbac->{$ENV{'PBPROJ'}} for VE needs to do a lot in the host (and chroot), so allow without restriction for now
4019print PBOUT "$pbac->{$ENV{'PBPROJ'}}   ALL=(ALL) NOPASSWD:ALL\n";
4020EOF
4021    }
4022    my $sudomode = pb_distro_get_param($pbos,pb_conf_get("ossudoersmode"));
4023    print SCRIPT << 'EOF';
4024close(PBOUT);
4025rename("$file.new",$file);
4026EOF
4027    print SCRIPT << "EOF";
4028chmod 0$sudomode,\$file;
4029
4030EOF
4031
4032    if ($vtype =~ /(v|r)m/) {
4033        # Sync date
4034        # do it after sudoers is setup
4035        print SCRIPT "my \$res = pb_system(\"$ntpline\",\"Updating time with $ntpline\",\"mayfailverbose\");\n";
4036        print SCRIPT "if (\$res != 0) {\n";
4037        # try again with another method
4038        $ntpline = pb_date2v($vtype,$pbos,1);
4039        print SCRIPT "  pb_system(\"$ntpline\",\"Updating time with $ntpline\",\"mayfailverbose\");\n";
4040        print SCRIPT "}\n";
4041    }
4042
4043    my ($addrepo) = pb_conf_read("$ENV{'PBDESTDIR'}/pbrc.yml","addrepo");
4044    my $pbrepo = pb_distro_get_param($pbos,$addrepo);
4045    print SCRIPT << 'EOF';
4046   
4047# We may need a proxy configuration. Get it from the local env
4048pb_apply_conf_proxy($pbos);
4049
4050# First install all required packages
4051pb_system("dnf clean all","Cleaning dnf env","mayfail") if (($pbos->{'name'} eq "fedora") && ($pbos->{'version'} >= 22));
4052pb_system("yum clean all","Cleaning yum env","mayfail") if ((($pbos->{'name'} eq "fedora") && ($pbos->{'version'} < 22)) || ($pbos->{'name'} eq "asianux") || ($pbos->{'name'} eq "rhel"));
4053my ($ospkgdep) = pb_conf_get_if("ospkgdep");
4054
4055my $pkgdep = pb_distro_get_param($pbos,$ospkgdep);
4056my $forcerepo;
4057EOF
4058    print SCRIPT "\$forcerepo = pb_distro_setuprepo_gen(\$pbos,\"$pbrepo\");\n" if ((defined $pbrepo) && ($pbrepo ne ""));
4059    print SCRIPT << 'EOF';
4060pb_distro_installdeps(undef,$pbos,$forcerepo,pb_distro_only_deps_needed($pbos,join(' ',split(/,/,$pkgdep))));
4061
4062EOF
4063    my $itype = pb_distro_get_param($pbos,pb_conf_get("pbinstalltype"));
4064    # Install from sandbox mean a file base install
4065    $itype = "file" if (defined $sbx);
4066    if ($itype =~ /^file/) {
4067        my $cmdget;
4068        my ($pbver1,$pbver2,$pbextdir);
4069        my ($tmp1,$tmp2);
4070        if (defined $sbx) {
4071            # Install from sandbox mean using the result of the just passed sbx2build command
4072            # Get content saved in cms2build
4073            my ($pkg) = pb_conf_read("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml","pbpkg");
4074            $pbextdir = pb_get_extdir();
4075            die "Unable to get package list" if (not defined $pkg);
4076
4077            # We consider 2 specific packages
4078            my $vertag1 = $pkg->{"ProjectBuilder"};
4079            my $vertag2 = $pkg->{"project-builder"};
4080            # get the version of the current package - maybe different
4081            pb_log(2,"Vertag1: $vertag1\n");
4082            pb_log(2,"Vertag2: $vertag2\n");
4083            ($pbver1,$tmp1) = split(/-/,$vertag1);
4084            ($pbver2,$tmp2) = split(/-/,$vertag2);
4085            $cmdget = "mv /tmp/ProjectBuilder-$pbver1$pbextdir.tar.gz ProjectBuilder-latest.tar.gz ; mv /tmp/project-builder-$pbver2$pbextdir.tar.gz project-builder-latest.tar.gz";
4086        } else {
4087            # TODO: shouldn't we have a variable to specify the source file?
4088            # And shouldn't we be getting the one that matches the current pb version?
4089            # For the moment, just take the latest stable one
4090            my $wget = pb_check_req("wget",0);
4091            $cmdget = "$wget --passive-ftp ftp://ftp.project-builder.org/src/ProjectBuilder-latest.tar.gz; $wget --passive-ftp ftp://ftp.project-builder.org/src/project-builder-latest.tar.gz";
4092        }
4093        print SCRIPT << 'EOF';
4094# Then install manually the missing perl modules
4095my ($osperldep,$osperlver) = pb_conf_get_if("osperldep","osperlver");
4096   
4097my $perldep = pb_distro_get_param($pbos,$osperldep);
4098my $bashopt = "";
4099$bashopt = "-x" if ($pbdebug ge 1);
4100my $verbopt = "mayfail";
4101$verbopt = "mayfailverbose" if ($pbdebug ge 1);
4102foreach my $m (split(/,/,$perldep)) {
4103    # Skip empty deps
4104    next if ($m =~ /^\s*$/);
4105    my $dir = $m;
4106    $dir =~ s/-.*//;
4107    pb_system("echo \"rm -rf $m* ; wget http://search.cpan.org/CPAN/modules/by-module/$dir/$m-$osperlver->{$m}.tar.gz ; gzip -cd $m-$osperlver->{$m}.tar.gz | tar xf - ; cd $m* ; if [ -f Build.PL ]; then perl Build.PL; ./Build ; ./Build install ; else perl Makefile.PL; make ; make install ; fi; cd .. ; rm -rf $m*\" | bash -e $bashopt" ,"Installing perl module $m-$osperlver->{$m}",$verbopt);
4108}
4109EOF
4110        print SCRIPT << 'EOF';
4111pb_system("rm -rf ProjectBuilder-* ; rm -rf project-builder-* ; rm -rf `perl -V:installvendorlib  | awk -F\"'\" '{print \$2}'`/ProjectBuilder ;
4112EOF
4113        print SCRIPT " $cmdget ; ";
4114        print SCRIPT << "EOF"
4115gzip -cd ProjectBuilder-latest.tar.gz | tar xf - ; cd ProjectBuilder-$pbver1$pbextdir ; perl Makefile.PL ; make ; make install ; cd .. ; rm -rf ProjectBuilder-* ; gzip -cd project-builder-latest.tar.gz | tar xf - ; cd project-builder-$pbver2$pbextdir ; perl Makefile.PL ; make ; make install ; cd .. ; rm -rf project-builder-* ;","Building Project-Builder");
4116EOF
4117    } elsif ($itype =~ /^pkg/) {
4118        # pkg based install. We need to point to the project-builder.org repository
4119        print SCRIPT << 'EOF';
4120my $pkgforpb = pb_distro_get_param($pbos,pb_conf_get_if("ospkg"));
4121my $forcerepo = pb_distro_setuposrepo($pbos);
4122pb_log(1,"Forcing full package install with forcerepo\n") if (defined $forcerepo);
4123pb_distro_installdeps(undef,$pbos,$forcerepo,pb_distro_only_deps_needed($pbos,join(' ',split(/,/,$pkgforpb))));
4124EOF
4125    } else {
4126        # Unknown install type
4127        die("Unknown install type $itype->{$ENV{'PBPROJ'}} for param pbinstalltype");
4128    }
4129    print SCRIPT << 'EOF';
4130pb_system("pbdistrocheck",undef,"verbose");
4131pb_system("pb --version",undef,"verbose");
4132EOF
4133    if ($vtype eq "ve"){
4134        if ($vetype ne "docker") {
4135            print SCRIPT << 'EOF';
4136# For VE we need to umount some FS at the end
4137
4138pb_system("umount /proc","Unmounting /proc","mayfail");
4139EOF
4140        }
4141        print SCRIPT << 'EOF';
4142# Create a basic network file if not already there
4143
4144my $nf="/etc/sysconfig/network";
4145if ((! -f $nf) && ($pbos->{'type'} eq "rpm") && ($pbos->{'family'} ne "novell")) {
4146    open(NF,"> $nf") || die "Unable to create $nf";
4147    print NF "NETWORKING=yes\n";
4148    print NF "HOSTNAME=localhost\n";
4149    close(NF);
4150}
4151chmod 0755,$nf;
4152EOF
4153    }
4154
4155    # Adds pb_distro_get_context and all functions needed from ProjectBuilder::Distribution, Conf and Base
4156    foreach my $m ("ProjectBuilder/Base.pm","ProjectBuilder/Distribution.pm","ProjectBuilder/Conf.pm") {
4157        foreach my $d (@INC) {
4158            my $f = "$d/$m";
4159            if (-f "$f") {
4160                open(PBD,"$f") || die "Unable to open $f: $!";
4161                while (<PBD>) {
4162                    next if (/^package/);
4163                    next if (/^use Exporter/);
4164                    next if (/^use ProjectBuilder::/);
4165                    next if (/^our /);
4166                    print SCRIPT $_;
4167                }
4168                close(PBD);
4169                # We just need the first one of each file wrt @INC - hopefully that's the right one.
4170                last;
4171            }
4172        }
4173    }
4174    # Use a fake pb_version_init version here
4175    print SCRIPT << "EOF";
4176sub pb_version_init {
4177
4178return("$projectbuilderver","$projectbuilderrev","$projectbuilderconfver");
4179}
41801;
4181EOF
4182    close(SCRIPT);
4183    chmod 0755,"$pbscript{$v}";
4184
4185    $pm->finish if (defined $pbparallel);
4186}
4187$pm->wait_all_children if (defined $pbparallel);
4188die "Aborting, one or more of the children failed." if ((not $all_ok) && ($Global::pb_stop_on_error));
4189
4190# These build scripts need to be run as root and force stop of VM at end
4191$ENV{'PBACCOUNT'} = "root";
4192
4193$sbx = "" if (not defined $sbx);
4194# Launch the VM/VE/RM
4195pb_log(2,"DEBUG: before parallel launch, pbscript hash is:".Dumper(%pbscript)."\n");
4196pb_parallel_launchv(\%pbscript,$vtype,uc($vtype)."Script".$sbx,$pbstep,$pbimage,$pbforce,0,$pbsnap);
4197
4198return;
4199}
4200
4201# Function to create a snapshot named 'pb' for VMs and a compressed tar for VEs
4202sub pb_snap2v {
4203
4204my $vtype = shift;
4205my $pbstep = 2;
4206
4207my ($vm,$all) = pb_get2v($vtype);
4208
4209# Script generated
4210my $pbscript = "$ENV{'PBDESTDIR'}/snapv.$$";
4211
4212my ($pbac) = pb_conf_get($vtype."login");
4213
4214foreach my $v (@$vm) {
4215    if ($vtype eq "ve") {
4216        # Get distro context
4217        my $pbos = pb_distro_get_context($v);
4218        my ($vepath) = pb_conf_get("vepath");
4219
4220        # Test if an existing snapshot exists and remove it if there is a VE
4221        if ((-f "$vepath->{$ENV{'PBPROJ'}}/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}.tar.gz") &&
4222            (! -d "$vepath->{$ENV{'PBPROJ'}}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}")) {
4223                pb_system("sudo rm -f $vepath->{$ENV{'PBPROJ'}}/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}.tar.gz","Removing previous snapshot $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}.tar.gz");
4224        }
4225    }
4226
4227    # Prepare the script to be executed on the VM/VE
4228    open(SCRIPT,"> $pbscript") || die "Unable to create $pbscript";
4229    print SCRIPT << 'EOF';
4230    #!/bin/bash
4231    sleep 2
4232EOF
4233    close(SCRIPT);
4234    chmod 0755,"$pbscript";
4235
4236    # Force shutdown of VM/VE
4237    # Force snapshot of VM/VE
4238    pb_script2v($pbscript,$vtype,$pbstep,1,$v,1);
4239}
4240return;
4241}
4242
4243# Function to update VMs/VEs/RMs with the latest distribution content
4244sub pb_update2v {
4245
4246my $vtype = shift;
4247my $pbstep = 2;
4248
4249my ($vm,$all) = pb_get2v($vtype);
4250
4251# Script generated
4252my $pbscript = "$ENV{'PBDESTDIR'}/updatev.$$";
4253
4254my ($pbac) = pb_conf_get($vtype."login");
4255
4256foreach my $v (@$vm) {
4257    # Get distro context
4258    my $pbos = pb_distro_get_context($v);
4259
4260    # Prepare the script to be executed on the VM/VE/RM
4261    # in $ENV{'PBDESTDIR'}/updatev
4262    open(SCRIPT,"> $pbscript") || die "Unable to create $pbscript";
4263   
4264    print SCRIPT << 'EOF';
4265    #!/bin/bash
4266    sleep 2
4267EOF
4268    # VE needs a good /proc
4269    $vetype = pb_ve_get_type($vetype);
4270    if (($vtype eq "ve") && ($vetype ne "docker")) {
4271        print SCRIPT "sudo /bin/mount -t proc /proc /proc\n";
4272    }
4273    print SCRIPT "$pbos->{'update'}\n";
4274    if (($vtype eq "ve") && ($vetype ne "docker")) {
4275        print SCRIPT "sudo /bin/umount /proc\n";
4276    }
4277    close(SCRIPT);
4278    chmod 0755,"$pbscript";
4279
4280    # Force shutdown of VM except
4281    pb_script2v($pbscript,$vtype,$pbstep,1,$v);
4282}
4283return;
4284}
4285
4286sub pb_announce {
4287
4288    my $antype = shift;
4289
4290    # Get all required parameters
4291    my ($pbpackager,$pbrepo) = pb_conf_get("pbpackager","pbrepo");
4292    my ($pkgv, $pkgt, $testver) = pb_conf_get_if("pkgver","pkgtag","testver");
4293    if (((not defined $testver) || (not defined $testver->{$ENV{'PBPROJ'}}) || ($testver->{$ENV{'PBPROJ'}} !~ /true/i)) && ($antype eq "Clean")) {
4294        # We just clean for test versions
4295        pb_log(0,"Unable to clean SSH repository for non testver version\n");
4296        return;
4297    }
4298    my $pkg = pb_cms_get_pkg($defpkgdir,$extpkgdir);
4299    my @pkgs = @$pkg;
4300    my %pkgs;
4301    my $first = 0;
4302
4303    # Get all distros concerned
4304    my $pbos = pb_distro_get_context();
4305    my $distrolist = pb_get_distros($pbos,undef);
4306    my %dl;
4307    my %theorlist;
4308    my %archlist;
4309    foreach my $d (split(/,/,$distrolist)) {
4310        my ($d1,$d2,$d3) = split(/-/,$d);
4311        $dl{$d1}++;
4312    }
4313
4314    # Command to find packages on repo
4315    my $findstr = "find ".join(" ",keys %dl)." ";
4316    my $srcstr = "";
4317    # Generated announce files
4318    my @files;
4319
4320    foreach my $pbpkg (@pkgs) {
4321        if ($first != 0) {
4322            $findstr .= "-o ";
4323        }
4324        $first++;
4325        if ((defined $pkgv) && (defined $pkgv->{$pbpkg})) {
4326            $pbver = $pkgv->{$pbpkg};
4327        } else {
4328            $pbver = $ENV{'PBPROJVER'};
4329        }
4330        if ((defined $pkgt) && (defined $pkgt->{$pbpkg})) {
4331            $pbtag = $pkgt->{$pbpkg};
4332        } else {
4333            $pbtag = $ENV{'PBPROJTAG'};
4334        }
4335
4336        # TODO: use virtual/real names here now
4337        my $pbrealpkg = $pbpkg;
4338        my $pbrealpkgrpm = pb_cms_get_real_pkg($pbpkg,"rpm");
4339        my $pbrealpkgdeb = pb_cms_get_real_pkg($pbpkg,"deb");
4340        if ($antype eq "Clean") {
4341            # We only clean test versions anyway
4342            $pbtag = "0";
4343            my $nver = $pbver;
4344            my $ntag = "[2-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]";
4345            $pbver .= $ntag;
4346            # TODO add apk
4347            $findstr .= "-name \'$pbrealpkgrpm-$pbver-$pbtag\.*.rpm\' -o -name \'$pbrealpkgrpm-debug-$pbver-$pbtag\.*.rpm\' -o -name \'$pbrealpkgdeb"."_$pbver-$pbtag"."_*\.deb\' -o -name \'$pbrealpkgdeb"."_$pbver-$pbtag.dsc\' -o -name \'$pbrealpkgdeb"."_$pbver-$pbtag.tar.gz\' -o -name \'$pbrealpkg-$nver"."_p$ntag\.ebuild\' -o -name \'$pbrealpkg-$pbver-$pbtag*\.pkg\.gz\' -o -name \'$pbrealpkg-$pbver-$pbtag*\.sd\' ";
4348            $srcstr .= "src/$pbrealpkg-$pbver.tar.gz src/$pbrealpkg-$pbver.pbconf.tar.gz ";
4349        } else {
4350            # "Announce" and "Check"
4351            my @date=pb_get_date();
4352            # the matching is only done on packages made the same day for test version. Hopefully this is enough
4353            my $nver;
4354            if ((defined $testver) && (defined $testver->{$ENV{'PBPROJ'}}) && ($testver->{$ENV{'PBPROJ'}} =~ /true/i)) {
4355                $pbtag = "0";
4356                my $ntag .= strftime("%Y%m%d*", @date);
4357                $nver = $pbver."_p$ntag*";
4358                $pbver .= "$ntag*";
4359            } else { 
4360                $nver = $pbver."-r$pbtag";
4361            }
4362            $findstr .= "-name \'$pbrealpkgrpm-$pbver-$pbtag\.*.rpm\' -o -name \'$pbrealpkgdeb"."_$pbver-$pbtag"."_*\.deb\' -o -name \'$pbrealpkg-$nver\.ebuild\' -o -name \'$pbrealpkg-$pbver\.pkg\.gz\' -o -name \'$pbrealpkg-$pbver\.sd\' ";
4363        }
4364
4365        if ($antype eq "Announce") {
4366            my $chglog;
4367
4368            pb_vcs_init($pbinit);
4369            # Get project info on log file and generate tmp files used later on
4370            $chglog = "$ENV{'PBROOTDIR'}/$pbpkg/pbcl";
4371            $chglog = "$ENV{'PBROOTDIR'}/pbcl" if (! -f $chglog);
4372            $chglog = undef if (! -f $chglog);
4373
4374            open(OUT,"> $ENV{'PBTMP'}/$pbpkg.ann") || die "Unable to create $ENV{'PBTMP'}/$pbpkg.ann: $!";
4375            my $pb;
4376            $pb->{'realpkg'} = $pbrealpkg;
4377            $pb->{'ver'} = $pbver;
4378            $pb->{'tag'} = $pbtag;
4379            $pb->{'date'} = $pbdate;
4380            $pb->{'extdir'} = pb_get_extdir();
4381            $pb->{'chglog'} = $chglog;
4382            $pb->{'packager'} = $pbpackager;
4383            $pb->{'proj'} = $ENV{'PBPROJ'};
4384            $pb->{'repo'} = $pbrepo;
4385            $pb->{'pbos'}->{'type'} = "announce";
4386            $pb->{'pbos'}->{'suffix'} = "none";
4387            pb_changelog($pb,\*OUT,"yes");
4388            close(OUT);
4389            push(@files,"$ENV{'PBTMP'}/$pbpkg.ann");
4390        } elsif ($antype eq "Check") {
4391            # For the check we also build the theoritical complete list we should get
4392            foreach my $d (split(/,/,$distrolist)) {
4393                $pbos = pb_distro_get_context($d);
4394                if ($pbos->{'type'} eq "rpm") {
4395                    $theorlist{"$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}/$pbrealpkgrpm-$pbver-$pbtag$pbos->{'suffix'}"} = 0;
4396                } elsif ($pbos->{'type'} eq "deb") {
4397                    $theorlist{"$pbos->{'name'}/$pbos->{'version'}/$pbrealpkgdeb"."_$pbver-$pbtag"} = 0;
4398                    # TODO are we always using the last arch ?
4399                    $archlist{"$pbos->{'name'}/$pbos->{'version'}/$pbrealpkgdeb"."_$pbver-$pbtag"} = "$pbos->{'arch'}";
4400                } elsif ($pbos->{'type'} eq "ebuild") {
4401                    my $prefix = "-r";
4402                    $prefix = "_p" if ((defined $testver) && (defined $testver->{$ENV{'PBPROJ'}}) && ($testver->{$ENV{'PBPROJ'}} =~ /true/i));
4403                    $theorlist{"$pbos->{'name'}/$pbos->{'version'}/$pbrealpkg-$pbver$prefix$pbtag.ebuild"} = 0;
4404                    $archlist{"$pbos->{'name'}/$pbos->{'version'}/$pbrealpkg-$pbver$prefix$pbtag.ebuild"} = "$pbos->{'arch'}";
4405                } elsif ($pbos->{'type'} eq "apk") {
4406                    # TODO
4407                } elsif ($pbos->{'type'} eq "pkg") {
4408                    $theorlist{"$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}/$pbrealpkg-$pbver-$pbtag.pkg.gz"} = 0;
4409                } else {
4410                    pb_log(1,"No theoritical list possible for type $pbos->{'type'}\n");
4411                }
4412            }
4413        }
4414        pb_log(2,"theorlist initial: ".Dumper(%theorlist)."\n");
4415        pb_log(2,"archlist: ".Dumper(%archlist)."\n");
4416    }
4417    if ($antype eq "Announce") {
4418        $findstr .= " | grep -Ev \'src.rpm\'";
4419    } elsif ($antype eq "Clean") {
4420        $findstr .= " | xargs rm -f -v $srcstr ";
4421    }
4422
4423    # Prepare the command to run and execute it
4424    open(PBS,"> $ENV{'PBTMP'}/pbscript") || die "Unable to create $ENV{'PBTMP'}/pbscript";
4425    print PBS "#!/bin/bash\n";
4426    print PBS "set -x\n" if ($pbdebug gt 1);
4427    print PBS "$findstr | sort 2> /dev/null\n";
4428    close(PBS);
4429    chmod 0755,"$ENV{'PBTMP'}/pbscript";
4430    pb_send2target("Announce","$ENV{'PBTMP'}/pbscript");
4431
4432    my $sl = "Project $ENV{'PBPROJ'} version $ENV{'PBPROJVER'} is now available";
4433    if ($antype eq "Announce") {
4434        # Get subject line
4435        pb_log(0,"Please enter the title of your announce\n");
4436        pb_log(0,"(By default: $sl)\n");
4437        my $sl2 = <STDIN>;
4438        $sl = $sl2 if ($sl2 !~ /^$/);
4439
4440        # Prepare a template of announce
4441        open(ANN,"> $ENV{'PBTMP'}/announce.html") || die "Unable to create $ENV{'PBTMP'}/announce.html: $!";
4442        print ANN << "EOF";
4443$sl</p>
4444
4445<p>The project team is happy to announce the availability of a newest version of $ENV{'PBPROJ'} $ENV{'PBPROJVER'}. Enjoy it as usual!</p>
4446<p>
4447Now available at <a href="$pbrepo->{$ENV{'PBPROJ'}}">$pbrepo->{$ENV{'PBPROJ'}}</a>
4448</p>
4449<p>
4450EOF
4451    }
4452
4453    open(LOG,"$ENV{'PBTMP'}/system.$$.log") || die "Unable to read $ENV{'PBTMP'}/system.$$.log: $!";
4454    if ($antype eq "Announce") {
4455        my $col = 2;
4456        my $i = 1;
4457        print ANN << 'EOF';
4458<TABLE WIDTH="100%" CELLPADDING="0" CELLSPACING="0" BORDER="0">
4459<TR>
4460EOF
4461        while (<LOG>) {
4462            print ANN "<TD><A HREF=\"$pbrepo->{$ENV{'PBPROJ'}}/$_\">$_</A></TD>";
4463            $i++;
4464            if ($i > $col) {
4465                print ANN "</TR>\n<TR>";
4466                $i = 1;
4467            }
4468        }
4469    } elsif ($antype eq "Clean") {
4470        while (<LOG>) {
4471            # skip errors
4472            next if ($_ !~ /^removed /);
4473            pb_log(0,"$_");
4474        }
4475    } else {
4476        # In "Check" mode we need to compare the 2 lists (real and theoritical)
4477        while (<LOG>) {
4478            # Get package name and remove what is in extra for the theoritical list (arch at the end)
4479            chomp();
4480            # skip find errors
4481            next if (/^find:/);
4482            my $p = $_;
4483            $p =~ s/\.(i[3456]86|x86_64|noarch|src)\.rpm$//;
4484            $p =~ s/_(i[3456]86|amd64|all)\.deb$//;
4485            $p =~ s/(-0\.[0-9]{8})[0-9]{6}/$1*/ if ((defined $testver) && (defined $testver->{$ENV{'PBPROJ'}}) && ($testver->{$ENV{'PBPROJ'}} =~ /true/i));
4486            $p =~ s/(-r|_p[0-9]+)\.ebuild/$1*/ if ((defined $testver) && (defined $testver->{$ENV{'PBPROJ'}}) && ($testver->{$ENV{'PBPROJ'}} =~ /true/i));
4487            # TODO apk
4488            $theorlist{$p} = -2 if (not defined $theorlist{$p});
4489            $theorlist{$p} = $theorlist{$p} + 1;
4490        }
4491        pb_log(2,"theorlist after check: ".Dumper(%theorlist)."\n");
4492    }
4493    close(LOG);
4494
4495    # Nothing more for the cleanssh case
4496    return if ($antype eq "Clean");
4497
4498    if ($antype eq "Check") {
4499        my ($chkex) = pb_conf_get_if("checkexclude");
4500        my $vmbuildlist = "";
4501        my $vebuildlist = "";
4502        my $rmbuildlist = "";
4503        my @pt = pb_conf_get_if("vmlist","velist","rmlist");
4504        foreach my $t (sort keys %theorlist) {
4505            if (defined $theorlist{$t} and $theorlist{$t} >= 1) {
4506                pb_log(1,"Packages found for $t\n");
4507            } elsif (defined $theorlist{$t} and $theorlist{$t} < 0) {
4508                pb_log(0,"Extra Package found for $t\n");
4509            } else {
4510                pb_log(2,"Analyzing $t\n");
4511                my ($os,$ver,$arch,$package) = split(/\//,$t);
4512                # Some distro have no arch subdir
4513                if (not defined $package) {
4514                    $package = $arch;
4515                    # TODO: If both arch have failed, we just make the last one
4516                    $arch = $archlist{$t} if (defined $archlist{$t});
4517                }
4518                my $pbos = pb_distro_get_context("$os-$ver-$arch");
4519                my $pkgn = $package;
4520                if ($pbos->{'type'} ne "deb") {
4521                    # package name is more easily found from the end for non deb
4522                    # as '-' is the separator, but it can also be used in names
4523                    $pkgn = reverse($package);
4524                    # search the second '-' and isolate the now last part which is the full name
4525                    $pkgn =~ s/([^-]+)-([^-]+)-([\S])+$/$3/;
4526                } else {
4527                    $pkgn =~ s/([^_]+)_([\S])+$/$2/;
4528                }
4529                my $found = 0;
4530                # Handle the exclusion of OSes
4531                my $excl = "";
4532                $excl .= $chkex->{$pkgn} if (defined $chkex->{$pkgn});
4533                $excl .= $chkex->{"all"} if (defined $chkex->{"all"});
4534
4535                pb_log(2,"Searching in excl $excl\n");
4536                foreach my $ex (split(/,/,$excl)) {
4537                    $found = 1 if ("$os-$ver-$arch" =~ /^$ex/);
4538                }
4539                # Skip as excluded
4540                next if ($found == 1);
4541                pb_log(0,"Package NOT found for $t\n");
4542                # Avoid duplicates in list
4543                next if ($vmbuildlist =~ /$os-$ver-$arch/);
4544                next if ($vebuildlist =~ /$os-$ver-$arch/);
4545                next if ($rmbuildlist =~ /$os-$ver-$arch/);
4546                # check with which method we need to build
4547                if ((defined $pt[0]->{$ENV{'PBPROJ'}}) and ($pt[0]->{$ENV{'PBPROJ'}} =~ /$os-$ver-$arch/)) {
4548                    $vmbuildlist = "$os-$ver-$arch" if ($vmbuildlist eq "");
4549                    $vmbuildlist .= ",$os-$ver-$arch" if ($vmbuildlist !~ /$os-$ver-$arch/);
4550                    next;
4551                }
4552                if ((defined $pt[1]->{$ENV{'PBPROJ'}}) and ($pt[1]->{$ENV{'PBPROJ'}} =~ /$os-$ver-$arch/)) {
4553                    $vebuildlist = "$os-$ver-$arch" if ($vebuildlist eq "");
4554                    $vebuildlist .= ",$os-$ver-$arch" if ($vebuildlist !~ /$os-$ver-$arch/);
4555                    next;
4556                }
4557                if ((defined $pt[2]->{$ENV{'PBPROJ'}}) and ($pt[2]->{$ENV{'PBPROJ'}} =~ /$os-$ver-$arch/)) {
4558                    $rmbuildlist = "$os-$ver-$arch" if ($rmbuildlist eq "");
4559                    $rmbuildlist .= ",$os-$ver-$arch" if ($rmbuildlist !~ /$os-$ver-$arch/);
4560                }
4561            }
4562        }
4563        # If we want to rebuild automatically, let's do it
4564        if (defined $opts{'rebuild'}) {
4565            # SandBox or CMS
4566            pb_log(0,"Rebuilding from SandBox\n");
4567            pb_log(0,"for VMs: $vmbuildlist\n") if ($vmbuildlist ne "");
4568            pb_log(0,"for VEs: $vebuildlist\n") if ($vebuildlist ne "");
4569            pb_log(0,"for RMs: $rmbuildlist\n") if ($rmbuildlist ne "");
4570            pb_cms2build("SandBox");
4571            # Which mode
4572            $ENV{'PBV'} = $vmbuildlist;
4573            pb_build2v("vm","build") if ($vmbuildlist ne "");
4574            $ENV{'PBV'} = $vebuildlist;
4575            pb_build2v("ve","build") if ($vebuildlist ne "");
4576            $ENV{'PBV'} = $rmbuildlist;
4577            pb_build2v("rm","build") if ($rmbuildlist ne "");
4578        }
4579        # For the check part this is now finished
4580        return;
4581    }
4582
4583    print ANN << "EOF";
4584</TR>
4585</TABLE>
4586</p>
4587
4588<p>As usual source packages are also available in the same directory.</p>
4589
4590<p>
4591Changes are :
4592</p>
4593<p>
4594EOF
4595    # Get each package changelog content
4596    foreach my $f (sort(@files)) {
4597        open(IN,"$f") || die "Unable to read $f:$!";
4598        while (<IN>) {
4599            print ANN $_;
4600        }
4601        close(IN);
4602        print ANN "</p><p>\n";
4603    }
4604    print ANN "</p>\n";
4605    close(ANN);
4606
4607    # Allow for modification
4608    my $editor = "vi";
4609    $editor = $ENV{'EDITOR'} if (defined $ENV{'EDITOR'});
4610    pb_system("$editor $ENV{'PBTMP'}/announce.html","Allowing modification of the announce","noredir");
4611
4612    # Store it in DB for external usage (Web pages generation)
4613    my $dbh = pb_connect_db();
4614
4615    # To read whole file
4616    local $/;
4617    open(ANN,"$ENV{'PBTMP'}/announce.html") || die "Unable to read $ENV{'PBTMP'}/announce.html: $!";
4618    my $announce = <ANN>;
4619    close(ANN);
4620   
4621    pb_log(2,"INSERT INTO announces VALUES (NULL, $pbdate, $announce)");
4622    my $sth = $dbh->prepare(qq{INSERT INTO announces VALUES (NULL,?,?)})
4623                    || die "Unable to insert into db";
4624    $sth->execute($pbdate, $announce);
4625    $sth->finish();
4626    $dbh->disconnect;
4627
4628    # Then deliver it on the Web
4629    # $TOOLHOME/livwww www
4630
4631    # Mail it to project's ML
4632    open(ML,"| w3m -dump -T text/html > $ENV{'PBTMP'}/announce.txt") || die "Unable to create $ENV{'PBTMP'}/announce.txt: $!";
4633    print ML << 'EOF';
4634<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/x html1/DTD/xhtml1-strict.dtd">
4635
4636<html xmlns="http://www.w3.org/1999/xhtml" dir="ltr" xml:lang="en" lang="en">
4637  <head>
4638  </head>
4639  <body>
4640  <p>
4641EOF
4642    open(ANN,"$ENV{'PBTMP'}/announce.html") || die "Unable to read $ENV{'PBTMP'}/announce.html: $!";
4643    while(<ANN>) {
4644        print ML $_;
4645    }
4646    print ML << 'EOF';
4647</body>
4648</html>
4649EOF
4650    close(ML);
4651
4652    # To read whole file
4653    local $/;
4654    open(ANN,"$ENV{'PBTMP'}/announce.txt") || die "Unable to read $ENV{'PBTMP'}/announce.txt: $!";
4655    my $msg = <ANN>;
4656    close(ANN);
4657   
4658    # Preparation of headers
4659    eval
4660    {
4661        require Mail::Sendmail;
4662        Mail::Sendmail->import();
4663    };
4664    if ($@) {
4665        # Mail::Sendmail not found not sending mail !
4666        pb_log(0,"No Mail::Sendmail module found so not sending any mail !\n");
4667    } else {
4668        my ($pbml,$pbsmtp) = pb_conf_get("pbml","pbsmtp");
4669        my %mail = (   
4670            To          =>  $pbml->{$ENV{'PBPROJ'}},
4671            From        =>  $pbpackager->{$ENV{'PBPROJ'}},
4672            Smtp        =>  $pbsmtp->{$ENV{'PBPROJ'}},
4673            Body        =>  $msg,
4674            Subject     =>  "[ANNOUNCE] $sl",
4675        );
4676           
4677        # Send mail
4678        if (! sendmail(%mail)) {
4679            if ((defined $Mail::Sendmail::error) and (defined $Mail::Sendmail::log)) {
4680                die "Unable to send mail ($Mail::Sendmail::error): $Mail::Sendmail::log";
4681            }
4682        }
4683    }
4684}
4685
4686#
4687# Creates a set of HTML file containing the news for the project
4688# based on what has been generated by the pb_announce function
4689#
4690sub pb_web_news2html {
4691
4692    my $dest = shift || $ENV{'PBTMP'};
4693
4694    # Get all required parameters
4695    my ($pkgv, $pkgt) = pb_conf_get_if("pkgver","pkgtag");
4696
4697
4698    my $dbh = pb_connect_db();
4699    # For date handling
4700    $ENV{'LANGUAGE'}="C";
4701    my $firstjan = strftime("%Y-%m-%d", 0, 0, 0, 1, 0, localtime->year(), 0, 0, -1);
4702    my $oldfirst = strftime("%Y-%m-%d", 0, 0, 0, 1, 0, localtime->year()-1, 0, 0, -1);
4703    pb_log(2,"firstjan: $firstjan, oldfirst: $oldfirst, pbdate:$pbdate\n");
4704    my $all = $dbh->selectall_arrayref("SELECT id,date,announce FROM announces ORDER BY date DESC");
4705    my %news;
4706    $news{"cy"} = "";   # current year's news
4707    $news{"ly"} = "";   # last year news
4708    $news{"py"} = "";   # previous years news
4709    $news{"fp"} = "";   # first page news
4710    my $cpt = 4;        # how many news for first page
4711    # Extract info from DB
4712    foreach my $row (@$all) {
4713        my ($id, $date, $announce) = @$row;
4714        $news{"cy"} = $news{"cy"}."<p><B>$date</B> $announce\n" if ((($date cmp $pbdate) le 0) && (($firstjan cmp $date) le 0));
4715        $news{"ly"} = $news{"ly"}."<p><B>$date</B> $announce\n" if ((($date cmp $firstjan) le 0) && (($oldfirst cmp $date) le 0));
4716        $news{"py"} = $news{"py"}."<p><B>$date</B> $announce\n" if (($date cmp $oldfirst) le 0);
4717        $news{"fp"} = $news{"fp"}."<p><B>$date</B> $announce\n" if ($cpt > 0);
4718        $cpt--;
4719    }
4720    pb_log(1,"news{fp}: ".$news{"fp"}."\n");
4721    $dbh->disconnect;
4722
4723    # Generate the HTML content
4724    foreach my $pref (keys %news) {
4725        open(NEWS,"> $dest/pb_web_$pref"."news.html") || die "Unable to create $dest/pb_web_$pref"."news.html: $!";
4726        print NEWS "$news{$pref}";
4727        close(NEWS);
4728    }
4729}
4730
4731sub pb_connect_db {
4732
4733    # Store in DB for external usage (Web pages generation)
4734    my $db = "$ENV{'PBCONFDIR'}/announces3.sql";
4735
4736    my $precmd = "";
4737    if (! -f $db) {
4738        $precmd = "CREATE TABLE announces (id INTEGER PRIMARY KEY AUTOINCREMENT, date DATE, announce VARCHAR[65535])";
4739        # we should add the $db to the VCS as well here
4740    }
4741
4742    my $dbh = DBI->connect("dbi:SQLite:dbname=$db","","",
4743                        { RaiseError => 1, AutoCommit => 1 })
4744                        || die "Unable to connect to $db";
4745
4746    if ($precmd ne "") {
4747        my $sth = $dbh->prepare(qq{$precmd})
4748                    || die "Unable to create table into $db";
4749        $sth->execute();
4750    }
4751
4752    return($dbh);
4753}
4754
4755
4756# Return the SSH key file to use
4757# Potentially create it if needed
4758
4759sub pb_ssh_get {
4760
4761my $create = shift || 0;    # Do not create keys by default. In that case prefer native keys, not pb's ones
4762my $target = shift || "localhost";
4763my $nport = shift || "22";
4764
4765my ($pbagent) = pb_conf_get_if("pbusesshagent");
4766# use ssh-agent if asked so.
4767return(undef) if (($create == 0) && (defined $pbagent->{$ENV{'PBPROJ'}}) && ($pbagent->{$ENV{'PBPROJ'}} =~ /true/io));
4768
4769# Check the SSH environment
4770my $keyfile = undef;
4771
4772# Check whether ecdsa is accepted by the remote host
4773my $ecdsa = 0;
4774my $dsa = 0;
4775my $rsa = 0;
4776open(SCAN,"ssh-keyscan -p $nport -t ecdsa $target 2>/dev/null |") || pb_log(0,"Unable to check ECDSA support for $target");
4777while(<SCAN>) {
4778    $ecdsa = 1 if ($_ =~ /ecdsa-/);
4779    pb_log(2,"Found ECDSA support on $target ($nport)\n");
4780}
4781close(SCAN);
4782
4783open(SCAN,"ssh-keyscan -p $nport -t rsa $target 2>/dev/null |") || pb_log(0,"Unable to check DSA support for $target");
4784while(<SCAN>) {
4785    $rsa = 1 if ($_ =~ /ssh-rsa/);
4786    pb_log(2,"Found RSA support on $target ($nport)\n");
4787}
4788close(SCAN);
4789
4790open(SCAN,"ssh-keyscan -p $nport -t dsa $target 2>/dev/null |") || pb_log(0,"Unable to check DSA support for $target");
4791while(<SCAN>) {
4792    $dsa = 1 if ($_ =~ /ssh-dss/);
4793    pb_log(2,"Found DSA support on $target ($nport)\n");
4794}
4795close(SCAN);
4796
4797# We have specific keys by default - create them if needed
4798if (!(-e "$ENV{'HOME'}/.ssh/pb_ecdsa") && ($create eq 1)) {
4799    my $res = pb_system("ssh-keygen -q -b 521 -N '' -f $ENV{'HOME'}/.ssh/pb_ecdsa -t ecdsa","Generating ECDSA SSH keys for pb");
4800    if (($res != 0) && !(-e "$ENV{'HOME'}/.ssh/pb_dsa") && ($create eq 1)) {
4801        # ecdsa may not be supported if old ssh
4802        pb_system("ssh-keygen -q -b 1024 -N '' -f $ENV{'HOME'}/.ssh/pb_dsa -t dsa","Generating DSA SSH keys for pb");
4803    }
4804}
4805
4806# Some ECDSA implementation are not working correctly e.g. RHEL7, Fedora 20/21
4807if (!(-e "$ENV{'HOME'}/.ssh/pb_rsa") && ($create eq 1)) {
4808    pb_system("ssh-keygen -q -b 2048 -N '' -f $ENV{'HOME'}/.ssh/pb_rsa -t rsa","Generating RSA SSH keys for pb");
4809}
4810
4811# We still favour RSA and fall back to ECDSA if not available
4812$keyfile = "$ENV{'HOME'}/.ssh/id_rsa" if (-s "$ENV{'HOME'}/.ssh/id_rsa");
4813if ($rsa != 0) {
4814    # And we favour a specific key over a generic one except if in delivery to ftp server
4815    $keyfile = "$ENV{'HOME'}/.ssh/id_rsa" if ((-s "$ENV{'HOME'}/.ssh/id_rsa") && ($create eq 0));
4816    $keyfile = "$ENV{'HOME'}/.ssh/pb_rsa" if (-s "$ENV{'HOME'}/.ssh/pb_rsa");
4817} else {
4818    if ($ecdsa != 0) {
4819        # And we favour a specific key over a generic one except if in delivery to ftp server
4820        $keyfile = "$ENV{'HOME'}/.ssh/id_ecdsa" if ((-s "$ENV{'HOME'}/.ssh/id_ecdsa") && ($ecdsa != 0) && ($create eq 0));
4821        $keyfile = "$ENV{'HOME'}/.ssh/pb_ecdsa" if ((-s "$ENV{'HOME'}/.ssh/pb_ecdsa") && ($ecdsa != 0));
4822    } elsif ($dsa != 0) {
4823        # And we favour a specific key over a generic one except if in delivery to ftp server
4824        $keyfile = "$ENV{'HOME'}/.ssh/id_dsa" if ((-s "$ENV{'HOME'}/.ssh/id_dsa") && ($dsa != 0) && ($create eq 0));
4825        $keyfile = "$ENV{'HOME'}/.ssh/pb_dsa" if ((-s "$ENV{'HOME'}/.ssh/pb_dsa") && ($dsa != 0));
4826    }
4827}
4828if (defined $keyfile) {
4829    pb_log(2,"Using ssh key file $keyfile\n");
4830} else {
4831    pb_log(0, "Unable to find your public ssh key under $ENV{'HOME'}/.ssh\n");
4832}
4833return($keyfile);
4834}
4835
4836
4837# Returns the pid of a running VM command using a specific VM file, and the port used if the VM exists
4838sub pb_check_ps {
4839    my $vmcmd = shift;
4840    my $vmm = shift;
4841    my $vexist = 0;             # FALSE by default
4842    my $vmport = undef;         # NONE by default
4843
4844    if ($vmcmd =~ /qemu/) {
4845        # Remove the suffix as qemu changes it at launch
4846        $vmcmd =~ s/qemu-.*/qemu/;
4847    }
4848    open(PS, "ps auxhww|") || die "Unable to call ps";
4849    while (<PS>) {
4850        next if (! /$vmcmd/);
4851        next if (! /$vmm/);
4852        my $void1;
4853        ($void1, $vexist, $vmport) = split(/ +/,$_,3);
4854        pb_log(2,"pb_check_ps $vmport\n");
4855        $vmport =~ s/.*=tcp:127\.0\.0\.1:([0-9]+)-:22.*/$1/;
4856        chomp($vmport);
4857        pb_log(2,"pb_check_ps found a VM $vexist using port $vmport\n");
4858        last;
4859    }
4860    return($vexist,$vmport);
4861}
4862
4863
4864sub pb_extract_build_files {
4865
4866my $src=shift;
4867my $dir=shift;
4868my $ddir=shift;
4869my $mandatory=shift || "build";
4870
4871my $flag = "mayfail" if (($mandatory eq "patch") || ($mandatory eq "src"));
4872my $res;
4873
4874if ($src =~ /tar\.gz$/) {
4875    $res = pb_system("tar xfpz $src $dir","Extracting $mandatory files from $src",$flag);
4876} elsif ($src =~ /tar\.bz2$/) {
4877    $res = pb_system("tar xfpj $src $dir","Extracting $mandatory files from $src",$flag);
4878} else {
4879    die "Unknown compression algorithm for $src";
4880}
4881# If not mandatory return now
4882return() if (($res != 0) and (($mandatory eq "patch") || ($mandatory eq "src")));
4883return(pb_move_extracted_files($dir,$ddir));
4884}
4885
4886sub pb_move_extracted_files {
4887
4888my $dir = shift;
4889my $ddir = shift;
4890my @files;
4891
4892# In a recursive function , we need a local var as DIR handle
4893my $bdir;
4894
4895opendir($bdir,"$dir") || confess "Unable to open directory $dir: $!";
4896foreach my $f (readdir($bdir)) {
4897    next if ($f =~ /^\./);
4898    # Skip potential patch dir
4899    next if ($f =~ /^pbpatch/);
4900    # Skip potential source dir
4901    next if ($f =~ /^pbsrc/);
4902    # Skip potential backup files
4903    next if ($f =~ /~$/);
4904    if ((-d "$dir/$f") && (! -l "$dir/$f")) {
4905        # Case of the bug debian subdir
4906        pb_mkdir_p("$ddir/$f");
4907        push @files,pb_move_extracted_files("$dir/$f","$ddir/$f");
4908    } else {
4909        move("$dir/$f","$ddir") || die "Unable to move $dir/$f to $ddir";
4910        pb_log(2,"mv $dir/$f $ddir\n");
4911        push @files,"$ddir/$f";
4912    }
4913}
4914closedir($bdir);
4915# Not enough but still a first cleanup
4916pb_rm_rf("$dir");
4917return(@files);
4918}
4919
4920sub pb_list_bfiles {
4921
4922my $dir = shift;
4923my $pbpkg = shift;
4924my $bfiles = shift;
4925my $pkgfiles = shift;
4926my $supfiles = shift;
4927# subdir to keep if recursive mode, empty by default
4928my $subdir = shift || "";
4929# In a recursive function , we need a local var as DIR handle
4930my $bdir;
4931
4932pb_log(2,"DEBUG: entering pb_list_bfiles in $dir: ".Dumper($bfiles)."\n");
4933opendir($bdir,"$dir") || die "Unable to open dir $dir: $!";
4934foreach my $f (readdir($bdir)) {
4935    pb_log(3,"DEBUG: pb_list_bfiles found $f\n");
4936    next if ($f =~ /^\./);
4937    if (-d "$dir/$f") {
4938        # Recurse for directories (Debian 3.0 format e.g.)
4939        pb_log(2,"DEBUG: pb_list_bfiles recurse in $dir/$f\n");
4940        pb_list_bfiles("$dir/$f",$pbpkg,$bfiles,$pkgfiles,$supfiles,$f);
4941        next;
4942    }
4943
4944    my $key = $f;
4945    # if recursive then store also the subdir
4946    $key = "$subdir/$f" if ($subdir ne "");
4947    $bfiles->{$key} = "$dir/$f";
4948    $bfiles->{$key} =~ s~$ENV{'PBROOTDIR'}~~;
4949    if (defined $supfiles->{$pbpkg}) {
4950        $pkgfiles->{$key} = "$dir/$f" if ($f =~ /$supfiles->{$pbpkg}/);
4951    }
4952}
4953closedir($bdir);
4954pb_log(2,"DEBUG: exiting pb_list_bfiles: ".Dumper($bfiles)."\n");
4955}
4956
4957sub pb_add_comma {
4958
4959my $str = shift;
4960my $addstr = shift;
4961
4962$str .= "," if (defined $str);
4963$str .= $addstr;
4964return($str);
4965}
4966
4967sub pb_list_sfiles {
4968
4969my $sdir = shift;
4970my $ptr = shift;
4971my $pbos = shift;
4972my $extdir = shift;
4973
4974pb_log(2,"DEBUG: entering pb_list_sfiles: ".Dumper($ptr)."\n");
4975my $key = "$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
4976
4977# Prepare local sources for this distro - They are always applied first - May be a problem one day
4978# This function works for both patches and additional sources
4979foreach my $p (sort(<$sdir/*>)) {
4980    $ptr->{$key} = pb_add_comma($ptr->{$key},"file://$p") if (($p =~ /\.all$/) || ($p =~ /\.$pbos->{'os'}$/) || ($p =~ /\.$pbos->{'type'}$/) || ($p =~ /\.$pbos->{'family'}$/) || ($p =~ /\.$pbos->{'name'}$/) || ($p =~ /\.$pbos->{'name'}-$pbos->{'version'}$/) ||($p =~ /\.$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}$/));
4981}
4982
4983# Prepare also remote sources to be included - Applied after the local ones
4984foreach my $p ("all","$pbos->{'os'}","$pbos->{'type'}","$pbos->{'family'}","$pbos->{'name'}","$pbos->{'name'}-$pbos->{'version'}","$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}") {
4985    my $f = "$extdir.".".$p";
4986    next if (not -f $f);
4987    if (not open(PATCH,$f)) {
4988        pb_display("Unable to open existing external source file content $f\n");
4989        next;
4990    }
4991    while (<PATCH>) {
4992        chomp();
4993        $ptr->{$key} = pb_add_comma($ptr->{$key},"$_");
4994    }
4995    close(PATCH);
4996}
4997pb_log(2,"DEBUG: exiting pb_list_sfiles: ".Dumper($ptr)."\n");
4998return($ptr);
4999}
5000   
5001#
5002# Return the list of packages we are working on in a non CMS action
5003#
5004sub pb_get_pkg {
5005
5006my @pkgs = ();
5007
5008# In script actions that file may not exist.
5009return(\@pkgs) if ((not defined $ENV{'PBDESTDIR'}) || (not defined $ENV{'PBPROJVER'}) || (not defined $ENV{'PBPROJTAG'}) || (! -f "$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml"));
5010
5011my ($var) = pb_conf_read("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml","pbpkg");
5012@pkgs = keys %$var if (defined $var);
5013
5014pb_log(0,"Packages: ".join(',',@pkgs)."\n") if (@pkgs);
5015return(\@pkgs);
5016}
5017
5018# Manages VM/RM SSH port communication
5019sub pb_get_port {
5020
5021my $port = shift;
5022my $pbos = shift;
5023my $cmt = shift;
5024my $nport;
5025
5026die "No port passed in parameter. Report to dev team\n" if (not defined $port);
5027# key is project on VM, but machine tuple for RM
5028if ($cmt =~ /^RM/i) {
5029    $nport = $port->{"$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}"};
5030} else {
5031    $nport = $port->{$ENV{'PBPROJ'}};
5032}
5033pb_log(2,"pb_get_port with $nport\n");
5034if ($cmt =~ /^VM/i) {
5035    # Maybe a port was given as parameter so overwrite
5036    $nport = "$pbport" if (defined $pbport);
5037    # Maybe in // mode so use the env var set up as an offset to the base port
5038    $nport = $nport + $ENV{'PBVMPORT'} -1 if ((defined $pbparallel) && (defined $ENV{'PBVMPORT'}));
5039}
5040pb_log(2,"pb_get_port returns $nport\n");
5041return($nport);
5042}
5043
5044sub pb_set_port { 
5045       
5046my ($pid,$ident) = @_;
5047pb_log(2,"pb_set_port for VM ($pid), id $ident\n");
5048$ENV{'PBVMPORT'} = $ident;
5049pb_log(2,"pb_set_port sets PBVMPORT in env to $ENV{'PBVMPORT'}\n");
5050}
5051
5052sub pb_set_parallel {
5053
5054my $vtype = shift;
5055
5056pb_log(2,"pb_set_parallel vtype: $vtype\n");
5057# Take care of memory size if VM, parallel mode and more than 1 action
5058if ((defined $pbparallel) && ($pbparallel ne 1) && ($vtype eq "vm")) {
5059    eval
5060    {
5061        require Linux::SysInfo;
5062        Linux::SysInfo->import();
5063    };
5064    if ($@) {
5065        # Linux::SysInfo not found
5066        pb_log(1,"ADVISE: Install Linux::SysInfo to benefit from automatic parallelism optimization.\nOr optimize manually pbparallel in your pb.yml file\nUsing $pbparallel processes max at a time for the moment\nWARNING: This may consume too much memory for your system\n");
5067    } else {
5068        # Using the memory size
5069        my $si = Linux::SysInfo::sysinfo();
5070        if (not defined $si) {
5071            pb_log(1,"ADVISE: Install Linux::SysInfo to benefit from automatic parallelism optimization.\nOr optimize manually pbparallel in your pb.yml file\nUsing $pbparallel processes max at a time for the moment\nWARNING: This may consume too much memory for your system\n");
5072        } else {
5073            # Keep the number of VM whose memory can be allocated
5074            my $ram = $si->{"totalram"}-$si->{"sharedram"}-$si->{"bufferram"};
5075            my $ram2;
5076            my ($vmmem) = pb_conf_get_if("vmmem");
5077
5078            my $v = "default";
5079            if ((defined $vmmem) and (defined $vmmem->{$v})) {
5080                $ram2 = $vmmem->{$v};
5081            } else {
5082                # Default for KVM/QEMU
5083                $ram2 = 128;
5084            }
5085            $pbparallel = sprintf("%d",$ram/$ram2);
5086        }
5087        pb_log(1,"Using $pbparallel processes at a time\n");
5088    }
5089}
5090pb_log(2,"pb_set_parallel returns: $pbparallel\n") if (defined $pbparallel);
5091return($pbparallel);
5092}
5093
5094sub pb_get_sudocmds { 
5095       
5096my $pbos = shift;
5097my @cmds = @_;
5098my %sudocmds;
5099
5100pb_log(2,"pb_get_sudocmds entering with lines:".Dumper(@cmds)."\n");
5101foreach my $c (split(/;/,$pbos->{'update'}),split(/;/,$pbos->{'install'}),@cmds) {
5102    pb_log(2,"pb_get_sudocmds analyses $c\n");
5103    next if ($c !~ /^\s*sudo/);
5104    # remove sudo and leading spaces
5105    $c =~ s/^\s*sudo\s+//;
5106    # keep only the command, not the params
5107    $c =~ s/([^\s]+)\s.*$/$1/;
5108    $sudocmds{$c} = "";
5109}
5110pb_log(2,"pb_get_sudocmds returns ".Dumper(keys %sudocmds)."\n");
5111return(keys %sudocmds);
5112}
5113
5114sub pb_sign_setenv {
5115
5116my ($passfile, $passphrase, $passpath) = pb_conf_get_if("pbpassfile","pbpassphrase","pbpasspath");
5117$ENV{'PBPASSPHRASE'} = $passphrase->{$ENV{'PBPROJ'}} if ((not defined $ENV{'PBPASSPHRASE'}) && (defined $passphrase->{$ENV{'PBPROJ'}}));
5118$ENV{'PBPASSFILE'} = $passfile->{$ENV{'PBPROJ'}} if ((n