source: devel/pb/bin/pb @ 2601

Last change on this file since 2601 was 2601, checked in by Bruno Cornec, 3 months ago

Fix the PBSETUPV env var declaration

  • Property svn:executable set to *
File size: 190.4 KB
Line 
1#!/usr/bin/perl -w
2#
3# Project Builder main application
4#
5# $Id$
6#
7# Copyright B. Cornec 2007-today
8# Eric Anderson's changes are (c) Copyright 2012 Hewlett Packard
9# Provided under the GPL v2
10
11# Syntax: see at end
12
13use strict 'vars';
14
15# The modules mentioned here are required by pb when used both
16# locally or inside a VE/VM/RM
17# Additional required modules only used locally are called with a require
18# in their respective section
19use Getopt::Long qw(:config auto_abbrev no_ignore_case);
20use Carp qw/confess cluck/;
21use Data::Dumper;
22use English;
23use File::Basename;
24use File::Copy;
25use File::stat;
26use File::Find;
27use Time::localtime qw(localtime);
28use POSIX qw(strftime);
29use lib qw (lib);
30use ProjectBuilder::Version;
31use ProjectBuilder::Base;
32use ProjectBuilder::Display;
33use ProjectBuilder::Conf;
34use ProjectBuilder::Distribution;
35use ProjectBuilder::VCS;
36use ProjectBuilder::CMS;
37use ProjectBuilder::Env;
38use ProjectBuilder::Filter;
39use ProjectBuilder::Changelog;
40use ProjectBuilder::VE;
41
42# Global variables
43$Global::pb_stop_on_error = 0;      # False by default
44$Global::pb_show_sudo = 0;          # False by default
45
46my %opts;                           # CLI Options
47my $action;                         # action to realize
48my $test = "FALSE";                 # Not used
49my $pbforce = 0;                    # Force VM stop
50my $pbsnap = 0;                     # Do not use snapshot mode for VM/VE by default
51my $pbkeep = 0;                     # keep temporary directory at the end
52my $option = "";                    # Not used
53my @pkgs;                           # list of packages
54my $pbtag;                          # Global Tag variable
55my $pbver;                          # Global Version variable
56my %pbscript;                       # Name of the scripts per V
57my $pbrev;                          # Global REVISION variable
58my $pbtarget = undef;               # Target os-ver-arch you want to build for
59my $pbport;                         # Port to use to connect to the VM/RM
60my $newver;                         # New version to create
61my $pbimage = undef;                # ISO image for the VM to create or docker image for the VE
62my $vetype = undef;                 # Type of Virtual environment to deal with
63
64my @date = pb_get_date();
65my $pbdate = strftime("%Y-%m-%d", @date);
66
67=pod
68
69=head1 NAME
70
71pb, aka project-builder.org - builds packages for your projects
72
73=head1 DESCRIPTION
74
75pb helps you build various packages directly from your project sources.
76Those sources could be handled by a CMS (Configuration Management System)
77such as Subversion, CVS, Git, Mercurial... or being a simple reference to a compressed tar file.
78It's based on a set of configuration files, a set of provided macros to help
79you keeping build files as generic as possible. For example, a single .spec
80file should be required to generate for all rpm based distributions, even
81if you could also have multiple .spec files if required.
82
83=head1 SYNOPSIS
84
85pb [-vhSq][-r pbroot][-p project][[-s script -a account -P port][-T VEtype][-t [os-ver-arch]][-m os-ver-arch[,...]]][-g][-i image] <action> [<pkg1> ...]
86
87pb [--verbose][--help][--man][--quiet][--snapshot][--revision pbroot][--project project][[--script script --account account --port port][-T VEtype][--target [os-ver-arch]][--machine os-ver-arch[,...]]][--nographic][--image image][--rebuild] <action> [<pkg1> ...]
88
89=head1 OPTIONS
90
91=over 4
92
93=item B<-v|--verbose>
94
95Increase verbosity
96
97=item B<-q|--quiet>
98
99Do not print any output.
100
101=item B<-h|--help>
102
103Print a brief help message and exits.
104
105=item B<-S|--snapshot>
106
107Use the snapshot mode of VMs or VEs
108
109=item B<--man>
110
111Prints the manual page and exits.
112
113=item B<-t|--target os-ver-arch>
114
115Name of the target system you want to build for.
116All if none precised.
117
118=item B<-m|--machine os-ver-arch[,os-ver-arch,...]>
119
120Name of the Virtual Machines (VM), Virtual Environments (VE) or Remote Machines (RM)
121you want to build on (comma separated).
122All if none precised (or use the env variable PBV).
123
124=item B<-T|--vetype VEtype]>
125
126Type of Virtual Environments (VE)
127Can be chroot or docker.
128
129=item B<-s|--script script>
130
131Name of the script you want to execute on the related VMs/VEs/RMs.
132
133=item B<-g|--nographic>
134
135Do not launch VMs in graphical mode.
136
137=item B<-i|--image image>
138
139It could be either:
140- The name of the ISO image of the distribution you want to install on the related VMs
141- The name of the docker image of the distribution you want to install on the related VEs
142
143
144=item B<-a|--account account>
145
146Name of the account to use to connect on the related VMs/RMs.
147
148=item B<-P|--port port_number>
149
150Port number to use to connect on the related VMs/RMs.";
151
152=item B<-p|--project project_name>
153
154Name of the project you're working on (or use the env variable PBPROJ)
155
156=item B<-r|--revision revision>
157
158Path Name of the project revision under the CMS (or use the env variable PBROOT)
159
160=item B<-V|--version new_version>
161
162New version of the project to create based on the current one.
163
164=item B<-k|--keep>
165
166Keep the temporary dir where files have been created in or der to help debug
167
168=item B<--rebuild>
169
170Only valid with the checkssh action, it alllows to automatically relaunch the build of the failed packages
171
172=item B<--no-stop-on-error>
173
174Continue through errors with best effort.
175
176=back
177
178=head1 ARGUMENTS
179
180<action> can be:
181
182=over 4
183
184=item B<sbx2build>
185
186Create tar files for the project under your CMS.
187Current state of the exported content is taken.
188CMS supported are SVN, SVK, CVS, Git and Mercurial
189parameters are packages to build
190if not using default list
191
192=item B<cms2build>
193
194Create tar files for the project under your CMS.
195Current state of the CMS is taken.
196CMS supported are SVN, SVK, CVS, Git and Mercurial
197parameters are packages to build
198if not using default list
199
200=item B<build2prep>
201
202Prepare the environment for build by installing required dependencies. Done once on the build system.
203
204=item B<build2pkg>
205
206Create packages for your running distribution
207
208=item B<cms2pkg>
209
210cms2build + build2pkg
211
212=item B<sbx2pkg>
213
214sbx2build + build2pkg
215
216=item B<sbx2pkg2ins>
217
218sbx2pkg + final install of packages
219
220=item B<sbx2prep>
221
222sbx2build + build2prep
223
224=item B<build2ssh>
225
226Send the tar files to a SSH host
227
228=item B<sbx2ssh>
229
230sbx2build + build2ssh
231
232=item B<cms2ssh>
233
234cms2build + build2ssh
235
236=item B<pkg2ssh>
237
238Send the packages built to a SSH host
239
240=item B<build2vm>
241
242Create packages in VMs, launching them if needed
243and send those packages to a SSH host once built
244VM type supported are QEMU and KVM
245
246=item B<build2ve>
247
248Create packages in VEs, creating it if needed
249and send those packages to a SSH host once built
250
251=item B<build2rm>
252
253Create packages in RMs, which should pre-exist,
254and send those packages to a SSH host once built
255RM means Remote Machine, and could be a physical or Virtual one.
256This is one buildfarm integration for pb.
257
258=item B<prepvm>
259
260Prepare the VMs to have all requirements to build the project
261
262=item B<prepve>
263
264Prepare the VEs to have all requirements to build the project
265
266=item B<preprm>
267
268Prepare the RMs to have all requirements to build the project
269
270=item B<sbx2vm>
271
272sbx2build + build2vm
273
274=item B<sbx2ve>
275
276sbx2build + build2ve
277
278=item B<sbx2docker>
279
280sbx2build + build2ve with a potential build of all necessary docker containers to perform it
281
282=item B<build2docker>
283
284build2ve with a potential build of all necessary docker containers to perform it
285
286=item B<sbx2rm>
287
288sbx2build + build2rm
289
290=item B<cms2vm>
291
292cms2build + build2vm
293
294=item B<cms2ve>
295
296cms2build + build2ve
297
298=item B<cms2rm>
299
300cms2build + build2rm
301
302=item B<launchvm>
303
304Launch one virtual machine
305
306=item B<launchve>
307
308Launch one virtual environment
309
310=item B<script2vm>
311
312Launch one virtual machine if needed
313and executes a script on it
314
315=item B<script2ve>
316
317Execute a script in a virtual environment
318
319=item B<script2rm>
320
321Execute a script on a remote machine
322
323=item B<newvm>
324
325Create a new virtual machine
326
327=item B<newve>
328
329Create a new virtual environment
330
331=item B<setupvm>
332
333Setup a virtual machine for pb usage
334
335=item B<setupve>
336
337Setup a virtual environment for pb usage
338
339=item B<setuprm>
340
341Setup a remote machine for pb usage
342
343=item B<sbx2setupvm>
344
345Setup a virtual machine for pb usage using the sandbox version of pb instead of the latest stable
346Reserved to dev team.
347
348=item B<sbx2setupve>
349
350Setup a virtual environment for pb usage using the sandbox version of pb instead of the latest stable
351Reserved to dev team.
352
353=item B<sbx2setuprm>
354
355Setup a remote machine for pb usage using the sandbox version of pb instead of the latest stable
356Reserved to dev team.
357
358=item B<build2setupvm>
359
360Setup a virtual machine for pb usage using the build available
361Reserved to dev team.
362
363=item B<build2setupve>
364
365Setup a virtual environment for pb usage using the build available
366Reserved to dev team.
367
368=item B<build2setuprm>
369
370Setup a remote machine for pb usage using the build available
371Reserved to dev team.
372
373=item B<snapvm>
374
375Snapshot a virtual machine for pb usage
376
377=item B<snapve>
378
379Snapshot a virtual environment for pb usage
380
381=item B<updateconf>
382
383Update the configuration files to the latest format
384
385=item B<updatevm>
386
387Update the distribution in the virtual machine
388
389=item B<updateve>
390
391Update the distribution in the virtual environment
392
393=item B<updaterm>
394
395Update the distribution in the remote machine
396
397=item B<install2pkg>
398
399Install the project packages locally
400
401=item B<test2pkg>
402
403Test the project packages locally
404
405=item B<install2vm>
406
407Install the project packages in a virtual machine
408
409=item B<install2ve>
410
411Install the project packages in a virtual environment
412
413=item B<install2rm>
414
415Install the project packages in a remote machine
416
417=item B<test2vm>
418
419Test the project packages in a virtual machine, after installation
420
421=item B<test2ve>
422
423Test the project packages in a virtual environment, after installation
424
425=item B<test2rm>
426
427Test the project packages in a remote machine, after installation
428
429=item B<checkssh>
430
431Check the delivery of the packages on the repository
432
433=item B<checkps>
434
435Check the process running the VM concerned
436
437=item B<newver>
438
439Create a new version of the project derived
440from the current one
441
442=item B<newproj>
443
444Create a new project and a template set of
445configuration files under pbconf
446
447=item B<announce>
448
449Announce the availability of the project through various means
450
451=item B<sbx2webssh>
452
453Create tar files for the website under your CMS.
454Current state of the exported content is taken.
455Deliver the content to the target server using ssh from the exported dir.
456
457=item B<cms2webssh>
458
459Create tar files for the website from your CMS.
460Deliver the content to the target server using ssh from the DVCS.
461
462=item B<sbx2webpkg>
463
464Create tar files for the website under your CMS.
465Current state of the exported content is taken.
466
467=item B<cms2webpkg>
468
469Create tar files for the website under your CMS.
470
471=item B<getconf>
472
473Print the full configuration parameters as found in the various configuration files. Help to debug conf issues.
474Also accepts a parameter to display only this value, and a VM/VE/RM
475
476=item B<getvar>
477
478Print the full variables expanded based on the distribution tuple. Help to debug conf issues.
479Also accepts a parameter to display only the values for this package, and a VM/VE/RM
480
481=item B<clean>
482
483Purge the build and delivery directories related to the current project
484
485=item B<cleanssh>
486
487Purge the ssh server of its packages (only for testver and test packages)
488
489=back
490
491<pkgs> can be a list of packages, the keyword 'all' or nothing, in which case the default list of packages is taken (corresponding to the defpkgdir list of arguments in the configuration file).
492
493=head1 WEB SITES
494
495The main Web site of the project is available at L<http://www.project-builder.org/>. Bug reports should be filled using the trac instance of the project at L<http://trac.project-builder.org/>.
496
497=head1 USER MAILING LIST
498
499None exists for the moment.
500
501=head1 CONFIGURATION FILES
502
503Each pb user may have a configuration in F<$HOME/.pbrc.yml>. The values in this file may overwrite any other configuration file value.
504
505Here is an example of such a configuration file:
506
507 #
508 # Define for each project the URL of its pbconf repository
509 # No default option allowed here as they need to be all different
510 #
511 # URL of the pbconf content
512 # This is the format of a classical URL with the extension of additional schema such as
513 # svn+ssh, cvs+ssh, ...
514 #
515  pbconfurl:
516    linuxcoe : cvs+ssh://:ext:bcornec@linuxcoe.cvs.sourceforge.net:/cvsroot/linuxcoe/pbconf
517
518 # This is normaly defined in the project's configuration file
519 # Url of the project
520 #
521  pbprojurl:
522    linuxcoe : cvs+ssh://:ext:bcornec@linuxcoe.cvs.sourceforge.net:/cvsroot/linuxcoe
523 
524 # All these URLs needs to be defined here as the are the entry point
525 # for how to build packages for the project
526 #
527  pbconfurl:
528    pb: svn+ssh://svn.project-builder.org/mondo/svn/pb/pbconf
529  pbconfurl:
530    mondorescue: svn+ssh://svn.project-builder.org/mondo/svn/project-builder/mondorescue/pbconf
531  pbconfurl:
532    collectl: svn+ssh://bruno@svn.mondorescue.org/mondo/svn/project-builder/collectl/pbconf
533  pbconfurl:
534    netperf: svn+ssh://svn.mondorescue.org/mondo/svn/project-builder/netperf/pbconf
535 
536 # Under that dir will take place everything related to pb
537 # If you want to use VMs/chroot/..., then use $ENV{'HOME'} to make it portable
538 # to your VMs/chroot/...
539 # if not defined then /var/cache
540  pbdefdir:
541    default: $ENV{'HOME'}/project-builder
542  pbdefdir:
543    pb: $ENV{'HOME'}
544  pbdefdir:
545    linuxcoe: $ENV{'HOME'}/LinuxCOE/cvs
546  pbdefdir:
547    mondorescue: $ENV{'HOME'}/mondo/svn
548 
549 # pbconfdir points to the directory where the CMS content of the pbconfurl is checked out
550 # If not defined, pbconfdir is under pbdefdir/pbproj/pbconf
551  pbconfdir:
552    linuxcoe: $ENV{'HOME'}/LinuxCOE/cvs/pbconf
553  pbconfdir:
554    mondorescue: $ENV{'HOME'}/mondo/svn/pbconf
555 
556 # pbdir points to the directory where the CMS content of the pbprojurl is checked out
557 # If not defined, pbdir is under pbdefdir/pbproj
558 # Only defined if we have access to the dev of the project
559  pbdir:
560    linuxcoe: $ENV{'HOME'}/LinuxCOE/cvs
561  pbdir:
562    mondorescue: $ENV{'HOME'}/mondo/svn
563 
564 # -daemonize doesn't work with qemu 0.8.2
565  vmopt:
566    default: -m 384
567
568=head1 COMMAND DETAILS
569
570=head2 newproj
571
572The newproj command creates a new project-builder project.  To run this command you first need to define two variables in your ~/.pbrc.yml file:
573
574 pbconfurl I<project> = file:///home/anderse/.git/project-builder-config/I<project>
575 pbdefdir default = $ENV{'HOME'}/cache-project-builder
576
577The first line defines the version controlled configuration information and the second defines the root directory for project-builder to use.
578
579You can then run the command:
580
581 % pb -p I<$project> -r I<$version> newproj I<$pkg>
582
583to create the new project. Running the newproj command will then generate the file $pbdefdir/$project/pbconf/$version/$project.yml, and the directory $pbdefdir/$project/pbconf/$version/$pkg. You will need to edit those files to make the later commands work.
584
585=head2 cms2build
586
587The cms2build command takes your files from the content management system and makes the two tar files that are necessary for building files. You need to have run the newproj command first.  Then there are several steps for running this command:
588
589=over 4
590
591=item Update your $project.yml configuration file.
592
593You need to set the pbprojurl, pbrepo, pbwf, pbpackager, projver, projtag, testver, delivery, and defpkgdir lines as described in the configuration file. The pbprojurl entry is used to find the source for your package. The pbrepo entry is used to build the .repo or .sources.list files for use by downloaders of the package. The pbwf entry indicates that the source tar file is named by I<package-name>-I<version>. The pbpackager entry will be stored in the packages and should be you or your team. The projver/projtag entries indicate the version of the software and the version of the packaging scripts. The testver entry when true indicates that the package is in a test version, so no log file is computed (can be long), and version is made up using a timstamp. The delivery entry gives the subdirectory under which the packages will be delivered on the repository, and the defpkgdir entry corresponds to the local subdirectory hosting the package content.
594
595For example:
596
597 pbprojurl Lintel = file:///home/anderse/projects/Lintel-0.2012.02.28.tar.gz
598 pbrepo Lintel = http://tesla.hpl.hp.com/opensource
599 pbwf Lintel = 1
600 pbpackager Lintel = Eric Anderson <eric.anderson4@hp.com>
601 projver Lintel = 0.2012.02.28
602 projtag Lintel = 1
603 testver Lintel = false
604 delivery Lintel = production
605 defpkgdir Lintel = Lintel-0.2012.02.28
606
607=item Create the build .tar.gz files:
608
609Then you need to take those files and create the initial tar files.  Run a command like:
610
611 % pb -p $project -r $version cms2build
612
613To create the $pbdefdir/$project/pbdelivery/$project-$version.{,pbconf}.tar.gz files, the $version-$projtag.yml and pbrc.yml files in the same directory.
614
615=back
616
617=head2 build2pkg
618
619The build2pkg command takes the tar files created in the cms2build step and attempts to build binary packages for your current operating system. There are two steps:
620
621=over 4
622
623=item Update your filters and build files.
624
625You probably need to edit the files describing the build steps in one of the $pbdefdir/$project/pbconf/$version/$project/{deb,rpm,pkg} directories and the filters in $pbdefdir/$project/pbconf/$version/pbfilter. Note that you can define additional filters and transformations in the filters files. The build files will be filtered by the filters defined in the filters files to generate the inputs to the build step. Therefore, if you change those files, you need to re-run the cms2build step.
626
627=item Build the package.
628
629Then you can run a command like:
630
631 % pb -p $project -r $version build2pkg
632
633To create the files in $project/pbbuild that comprise your binary package(s).
634
635=back
636
637=head2 newve
638 
639The newve command creates a new virtual environment, i.e. a chrooted OS for building packages. Using a virtual environment is an efficient way to build packages on a related set of operating systems. The OS's have to be related because the kernel will be shared.  Steps:
640
641=over 4
642
643=item Update ~/.pbrc.yml
644
645Update your ~/.pbrc.yml file to specify the vepath, velist, velogin, and vetype variables, e.g.:
646
647  vepath:
648    default: $ENV{'HOME'}/cache-project-builder/chroot
649  velist:
650    default: debian-6.0-i386
651  velogin:
652    default: pb
653  vetype:
654    default: chroot
655
656If you are building for rpm style OS's, update the verpmtype option, and install the appropriate tool.
657
658  verpmtype:
659    default: rpmbootstrap
660
661You may also choose to specify a mirror for the OS packages, and optionally http/ftp proxies.  You can specify the proxies either through environment variables ($http_proxy/$https_proxy/$ftp_proxy) or in the configuration file. The configuration file will be used if no corresponding environment variable has been set. For example, for debian and with a local squid proxy:
662
663  rbsmirrorsrv:
664    debian: http://mirrors1.kernel.org/debian/
665  http_proxy:
666    default: http://localhost:3128/
667  https_proxy:
668    default: http://localhost:3128/
669  ftp_proxy:
670    default: http://localhost:3128/
671
672=item Run the cms2build command
673
674If you have deleted your $package/pbdelivery directory, re-run the cms2build command as in the earlier step. This step is necessary to generate the I<package>/pbdelivery/pbrc.yml file.
675
676=item Create the new virtual environment
677
678Initialize the new operating system. This step will install the core OS packages for the virtual environment, e.g.:
679
680 % pb -v -p $project -m debian-6.0-i386 newve
681
682=back
683
684=head2 setupve
685
686The setupve command prepares a virtual environment for use by project builder. In particular it installs project-builder from the packages into the virtual environment. Two sub-steps are necessary:
687
688=over 4
689
690=item Update $project.yml
691
692You need to have a sshhost entry for setupve to work, so add one, even an invalid one, e.g.:
693
694  sshhost:
695    $project: foo.example.org
696
697=item Setup the virtual environment
698
699 % pb -v -p $project -m debian-6.0-i386 setupve
700
701If you prefer to install the current SVN version of project builder, you can substitute the setupve option by the sbx2setupv one.
702
703=back
704
705=head2 build2ve
706
707The build2ve command is similar to the build2pkg command in that it will take the sources created by cms2build and turn them into binary packages. The command has two differences. First, it creates the packages in a virtual environment, i.e. the one made by an earlier setupve setup. Second it copies the resulting packages to a repository and builds the repository meta-data needed.
708
709Three sub-steps are needed:
710
711=over 4
712
713=item Update $project.yml
714
715You need to have a valid sshdir and sshhost entry for build2ve to work, so add them. Note that you need to be able to ssh from the host you run the command on to the repository host, preferably without needing to type in a password, so using ssh-agent or having a special passwordless project-builder ssh key will make this step easier.
716
717  sshhost:
718    $project: localhost
719  sshdir:
720    $project: $home/cache-project-builder/repos
721
722You may also need to specify additional repository files to use or rpms to install. Note the URL for repositories is not the URL of the repository, but the URL of a file that can be put in the yum.repos.d or apt.sources.d directory.
723
724  addbuildrepo:
725    centos-5-i386: http://localhost/pb/centos-extras.repo,http://mirror.centos.org/centos/5/extras/i386/RPMS/chrpath-0.13-3.el5.centos.i386.rpm
726
727=item Update your filters and build files
728
729You may need to update your filter files as in the build2pkg step if you are building for a new OS or architecture.
730
731=item Build the packages and copy them to the repository
732
733 % pb -v -p $project -m debian-6.0-i386 build2ve
734
735=back
736
737*Debugging:* If the build fails (and you did not specify the --no-stop-on-error) option, then the virtual environment and scripts should still be present and configured to build the package. You can run a command like 'sudo setarch i386 chroot $path bash' in order to get into the environment. In your log you should see a command like that. From there you can go into the /home/pb directory as the pb user and run the same style of pb commands as you did when doing build2pkg. This will help you figure out what has gone wrong in the build in the virtual environment.
738
739=head1 AUTHORS
740
741The Project-Builder.org team L<http://trac.project-builder.org/> lead by Bruno Cornec L<mailto:bruno@project-builder.org>.
742
743=head1 COPYRIGHT
744
745Project-Builder.org is distributed under the GPL v2.0 license
746described in the file C<COPYING> included with the distribution.
747
748=cut
749
750# ---------------------------------------------------------------------------
751
752my ($projectbuilderver,$projectbuilderrev,$projectbuilderconfver) = pb_version_init();
753my $appname = "pb";
754
755# Initialize the syntax string
756
757pb_syntax_init("$appname (aka project-builder.org) Version $projectbuilderver-$projectbuilderrev Conf $projectbuilderconfver\n");
758
759GetOptions("help|?|h+" => \$opts{'h'}, 
760        "account|a=s" => \$opts{'a'},
761        "force|f" => \$opts{'f'},
762        "nographic|g" => \$opts{'g'},
763        "image|i=s" => \$opts{'i'},
764        "log-files|l=s" => \$opts{'l'},
765        "machines|mock|m=s" => \$opts{'m'},
766        "man" => \$opts{'man'},
767        "port|P=i" => \$opts{'P'},
768        "project|p=s" => \$opts{'p'},
769        "quiet|q" => \$opts{'q'},
770        "revision|r=s" => \$opts{'r'},
771        "rebuild" => \$opts{'rebuild'},
772        "snapshot|S" => \$opts{'S'},
773        "script|s=s" => \$opts{'s'},
774        "type|T:s" => \$opts{'T'},
775        "target|t:s" => \$opts{'t'},
776        "version" => \$opts{'version'},
777        "V=s" => \$opts{'V'},
778        "verbose|v+" => \$opts{'v'},
779        "keep|k" => \$opts{'k'},
780        "stop-on-error!" => \$Global::pb_stop_on_error,
781) || pb_syntax(-1,0);
782
783if (defined $opts{'l'}) {
784    open(pbLOG,"> $opts{'l'}") || confess "Unable to log to $opts{'l'}: $!";
785    $pbLOG = \*pbLOG;
786    $pbdebug = 0  if ($pbdebug == -1);
787    }
788pb_log_init($pbdebug, $pbLOG);
789
790if (defined $opts{'h'}) {
791    pb_syntax(0,1);
792}
793if (defined $opts{'version'}) {
794    pb_syntax(0,0);
795}
796if (defined $opts{'man'}) {
797    pb_syntax(0,2);
798}
799if (defined $opts{'v'}) {
800    $pbdebug = $opts{'v'};
801}
802if (defined $opts{'f'}) {
803    $pbforce=1;
804}
805if (defined $opts{'q'}) {
806    $pbdebug=-1;
807}
808if (defined $opts{'S'}) {
809    $pbsnap=1;
810}
811if (defined $opts{'k'}) {
812    $pbkeep=1;
813}
814pb_display_init("text","");
815
816# Handle root of the project if defined
817if (defined $opts{'r'}) {
818    $ENV{'PBROOTDIR'} = $opts{'r'};
819}
820# Handle virtual machines if any
821if (defined $opts{'m'}) {
822    $ENV{'PBV'} = $opts{'m'};
823}
824if (defined $opts{'s'}) {
825    $pbscript{'default'} = $opts{'s'};
826    confess "option -s requires a script having a full path name" if ($pbscript{'default'} !~ /^\//);
827}
828if (defined $opts{'a'}) {
829    $ENV{'PBACCOUNT'} = $opts{'a'};
830    confess "option -a requires a -s script option" if (not defined $pbscript{'default'});
831}
832if (defined $opts{'P'}) {
833    $pbport = $opts{'P'};
834}
835if (defined $opts{'V'}) {
836    $newver = $opts{'V'};
837}
838if (defined $opts{'i'}) {
839    $pbimage = $opts{'i'};
840}
841if (defined $opts{'t'}) {
842    $pbtarget = $opts{'t'};
843}
844if (defined $opts{'T'}) {
845    $vetype = $opts{'T'};
846}
847
848# Get Action
849$action = shift @ARGV;
850confess pb_syntax(-1,1) if (not defined $action);
851
852my ($filteredfiles, $supfiles, $defpkgdir, $extpkgdir);
853my $pbinit = undef;
854$pbinit = 1 if ($action =~ /^newproj$/);
855
856# Handles project name if any
857# And get global params
858($filteredfiles, $supfiles, $defpkgdir, $extpkgdir) = pb_env_init($opts{'p'},$pbinit,$action,$pbkeep);
859
860#
861# Check for command requirements
862#
863my ($req,$opt,$pbpara,$pbshowsudo,$pbstoponerr) = pb_conf_get_if("oscmd","oscmdopt","pbparallel","pbshowsudo","pbstoponerr");
864pb_check_requirements($req,$opt,$appname);
865
866$Global::pb_show_sudo = 1 if ((defined $pbshowsudo) && (defined $pbshowsudo->{$ENV{'PBPROJ'}}) && ($pbshowsudo->{$ENV{'PBPROJ'}} =~ /true/oi));
867$Global::pb_stop_on_error = 1 if ((defined $pbstoponerr) && (defined $pbstoponerr->{$ENV{'PBPROJ'}}) && ($pbstoponerr->{$ENV{'PBPROJ'}} =~ /true/oi));
868
869#
870# Check if we can launch some actions in // with Parallel::ForkManager
871#
872my $pbparallel = $pbpara->{$appname} if (defined $pbpara);
873if (not defined $pbparallel) {
874    eval
875    {
876        require Sys::CPU;
877        Sys::CPU->import();
878    };
879    if ($@) {
880        # Sys::CPU not found, defaulting to 1
881        pb_log(1,"ADVISE: Install Sys::CPU to benefit from automatic parallelism optimization.\nOr use pbparallel in your pb.yml file\nOnly 1 process at a time for the moment\n");
882        $pbparallel = 1;
883    } else {
884        # Using the number of cores
885        $pbparallel = Sys::CPU::cpu_count();
886        pb_log(1,"Using parallel mode with $pbparallel processes\n");
887    }
888}
889
890eval
891{
892    require Parallel::ForkManager;
893    Parallel::ForkManager->import();
894};
895# Parallel::ForkManager not found so no // actions
896if ($@) {
897    $pbparallel = undef;
898    pb_log(1,"ADVISE: Install Parallel::ForkManager to benefit from automatic parallelism optimization.\nOnly 1 process at a time for the moment\n");
899}
900
901if ($action ne "getconf") {
902    pb_log(0,"Project: $ENV{'PBPROJ'}\n");
903    pb_log(0,"Action: $action\n");
904}
905
906my $do_install = undef;
907# Possibility to add a 2ins to some commands to launch automatic installation
908$do_install = 1 if ($action =~ /2ins$/);
909
910# Act depending on action
911if ($action =~ /^cms2build$/) {
912    pb_cms2build("CMS");
913} elsif ($action =~ /^sbx2build$/) {
914    pb_cms2build("SandBox");
915} elsif ($action =~ /^build2prep$/) {
916    pb_build2prep();
917} elsif ($action =~ /^sbx2prep$/) {
918    pb_cms2build("SandBox");
919    pb_build2prep();
920} elsif ($action =~ /^build2pkg/) {
921    pb_build2pkg($do_install);
922} elsif ($action =~ /^cms2pkg/) {
923    pb_cms2build("CMS");
924    pb_build2pkg($do_install);
925} elsif ($action =~ /^sbx2pkg/) {
926    pb_cms2build("SandBox");
927    pb_build2pkg($do_install);
928} elsif ($action =~ /^build2ssh$/) {
929    pb_build2ssh();
930} elsif ($action =~ /^cms2ssh$/) {
931    pb_cms2build("CMS");
932    pb_build2ssh();
933} elsif (($action =~ /^sbx2ssh$/) || ($action =~ /^sbx2pkg2ssh$/)) {
934    pb_cms2build("SandBox");
935    pb_build2ssh();
936} elsif ($action =~ /^pkg2ssh$/) {
937    pb_pkg2ssh();
938} elsif ($action =~ /^build2rm$/) {
939    pb_build2v("rm","build");
940} elsif ($action =~ /^build2ve$/) {
941    pb_build2v("ve","build");
942} elsif ($action =~ /^build2vm$/) {
943    pb_build2v("vm","build");
944} elsif ($action =~ /^sbx2preprm$/) {
945    pb_cms2build("SandBox");
946    pb_build2v("rm","prep");
947} elsif ($action =~ /^sbx2prepve$/) {
948    pb_cms2build("SandBox");
949    pb_build2v("ve","prep");
950} elsif ($action =~ /^sbx2prepvm$/) {
951    pb_cms2build("SandBox");
952    pb_build2v("vm","prep");
953} elsif ($action =~ /^preprm$/) {
954    pb_build2v("rm","prep");
955} elsif ($action =~ /^prepve$/) {
956    pb_build2v("ve","prep");
957} elsif ($action =~ /^prepvm$/) {
958    pb_build2v("vm","prep");
959} elsif ($action =~ /^cms2rm$/) {
960    pb_cms2build("CMS");
961    pb_build2v("rm","build");
962} elsif ($action =~ /^cms2ve$/) {
963    pb_cms2build("CMS");
964    pb_build2v("ve","build");
965} elsif ($action =~ /^sbx2rm$/) {
966    pb_cms2build("SandBox");
967    pb_build2v("rm","build");
968} elsif ($action =~ /^sbx2ve$/) {
969    pb_cms2build("SandBox");
970    pb_build2v("ve","build");
971} elsif ($action =~ /^cms2vm$/) {
972    pb_cms2build("CMS");
973    pb_build2v("vm","build");
974} elsif ($action =~ /^sbx2vm$/) {
975    pb_cms2build("SandBox");
976    pb_build2v("vm","build");
977} elsif ($action =~ /^install2vm$/) {
978    pb_build2v("vm","install");
979} elsif ($action =~ /^install2ve$/) {
980    pb_build2v("ve","install");
981} elsif ($action =~ /^install2rm$/) {
982    pb_build2v("rm","install");
983} elsif ($action =~ /^launchvm$/) {
984    pb_parallel_launchv(undef,"vm",undef,3,$pbimage);
985} elsif ($action =~ /^launchve$/) {
986    pb_parallel_launchv(undef,"ve",undef,3,$pbimage);
987} elsif ($action =~ /^script2vm$/) {
988    confess "action script2vm requires a -s script option" if (not defined $pbscript{'default'});
989    pb_script2v(\%pbscript,"vm");
990} elsif ($action =~ /^script2ve$/) {
991    confess "action script2ve requires a -s script option" if (not defined $pbscript{'default'});
992    pb_script2v(\%pbscript,"ve");
993} elsif ($action =~ /^script2rm$/) {
994    confess "action script2rm requires a -s script option" if (not defined $pbscript{'default'});
995    pb_script2v(\%pbscript,"rm");
996} elsif ($action =~ /^newver$/) {
997    pb_newver();
998} elsif ($action =~ /^sbx2docker$/) {
999    pb_cms2build("SandBox");
1000    pb_all_docker($action);
1001} elsif ($action =~ /^build2docker$/) {
1002    pb_all_docker($action);
1003} elsif ($action =~ /^newve$/) {
1004    pb_parallel_launchv(undef,"ve",undef,0,$pbimage);
1005} elsif ($action =~ /^newvm$/) {
1006    pb_parallel_launchv(undef,"vm",undef,0,$pbimage);
1007    pb_log(0, "Please ensure that sshd is running in your VM by default\n");
1008    pb_log(0, "and that it allows remote root login (PermitRootLogin yes in /etc/ssh/sshd_config)\n");
1009    pb_log(0, "Also ensure that network is up, firewalling correctly configured\n");
1010    pb_log(0, "and perl-YAML, perl-Date-Manip, tar, sudo, ntpdate, make and scp/ssh installed\n");
1011    pb_log(0, "You should then be able to login with ssh -p VMPORT root\@localhost (if VM started with pb)\n");
1012} elsif ($action =~ /^setuprm$/) {
1013    pb_setup2v("rm");
1014} elsif ($action =~ /^setupve$/) {
1015    pb_setup2v("ve");
1016} elsif ($action =~ /^setupvm$/) {
1017    pb_setup2v("vm");
1018} elsif ($action =~ /^sbx2setuprm$/) {
1019    confess "This feature is limited to the pb project" if ($ENV{'PBPROJ'} ne $appname);
1020    pb_cms2build("SandBox");
1021    pb_setup2v("rm","SandBox");
1022} elsif ($action =~ /^sbx2setupve$/) {
1023    confess "This feature is limited to the pb project" if ($ENV{'PBPROJ'} ne $appname);
1024    pb_cms2build("SandBox");
1025    pb_setup2v("ve","SandBox");
1026} elsif ($action =~ /^sbx2setupvm$/) {
1027    confess "This feature is limited to the pb project" if ($ENV{'PBPROJ'} ne $appname);
1028    pb_cms2build("SandBox");
1029    pb_setup2v("vm","SandBox");
1030} elsif ($action =~ /^build2setuprm$/) {
1031    confess "This feature is limited to the pb project" if ($ENV{'PBPROJ'} ne $appname);
1032    pb_setup2v("rm","SandBox");
1033} elsif ($action =~ /^build2setupve$/) {
1034    confess "This feature is limited to the pb project" if ($ENV{'PBPROJ'} ne $appname);
1035    pb_setup2v("ve","SandBox");
1036} elsif ($action =~ /^build2setupvm$/) {
1037    confess "This feature is limited to the pb project" if ($ENV{'PBPROJ'} ne $appname);
1038    pb_setup2v("vm","SandBox");
1039} elsif ($action =~ /^updaterm$/) {
1040    pb_update2v("rm");
1041} elsif ($action =~ /^updateve$/) {
1042    pb_update2v("ve");
1043} elsif ($action =~ /^updatevm$/) {
1044    pb_update2v("vm");
1045} elsif ($action =~ /^snapve$/) {
1046    pb_snap2v("ve");
1047} elsif ($action =~ /^snapvm$/) {
1048    pb_snap2v("vm");
1049} elsif ($action =~ /^install2pkg$/) {
1050    pb_install2pkg();
1051} elsif ($action =~ /^test2pkg$/) {
1052    pb_test2pkg();
1053} elsif ($action =~ /^test2rm$/) {
1054    pb_build2v("rm","test");
1055} elsif ($action =~ /^test2ve$/) {
1056    pb_build2v("ve","test");
1057} elsif ($action =~ /^test2vm$/) {
1058    pb_build2v("vm","test");
1059} elsif ($action =~ /^newproj$/) {
1060    # Nothing to do - already done in pb_env_init
1061} elsif ($action =~ /^updateconf$/) {
1062    foreach my $f (@ARGV) {
1063        my $fyml = $f;
1064        $fyml =~ s/\.pbf/.yml/;
1065        $fyml =~ s/\.pb/.yml/;
1066        $fyml =~ s/\.pbrc/.pbrc.yml/;
1067        pb_vcs_conf_update_v0($f,$fyml);
1068    }
1069} elsif ($action =~ /^getconf$/) {
1070    my $pbos = pb_distro_get_context($ENV{'PBV'});
1071    pb_log(1,"Arguments to print:".Dumper(@ARGV)."\n");
1072    pb_distro_conf_print($pbos,@ARGV);
1073} elsif ($action =~ /^getvar$/) {
1074    my $pbos = pb_distro_get_context($ENV{'PBV'});
1075    my $pkg;
1076    my @pkg;
1077    if ($#ARGV == -1) {
1078        $pkg = pb_cms_get_pkg($defpkgdir,$extpkgdir);
1079        @pkgs = @$pkg;
1080    } else {
1081        @pkgs = @ARGV;
1082    }
1083    pb_log(1,"Packages to print:".Dumper(@pkgs)."\n");
1084    pb_filter_var_print($pbos,@pkgs);
1085} elsif ($action =~ /^clean$/) {
1086    pb_clean();
1087} elsif ($action =~ /^announce$/) {
1088    # For announce only. Require avoids the systematic load of these modules
1089    require DBI;
1090    require DBD::SQLite;
1091
1092    pb_announce("Announce");
1093} elsif ($action =~ /^cleanssh$/) {
1094    pb_announce("Clean");
1095} elsif ($action =~ /^checkssh$/) {
1096    pb_announce("Check");
1097} elsif ($action =~ /^checkps$/) {
1098    my ($ptr2) = pb_conf_get("vmcmd");
1099    my $vmcmd = $ptr2->{$ENV{'PBPROJ'}};
1100    my ($vexist,$vmmport) = pb_check_ps($vmcmd,$ENV{'PBV'});
1101    if (! $vexist) {
1102        pb_log(0,"No VM found for $ENV{'PBV'}\n");
1103    } else {
1104        pb_log(0,"Found an existing VM for $ENV{'PBV'} PID: $vexist - SSH port: $vmmport\n");
1105    }
1106} elsif ($action =~ /^sbx2webpkg$/) {
1107    require DBI;
1108    require DBD::SQLite;
1109
1110    pb_cms2build("SandBox","Web");
1111} elsif ($action =~ /^sbx2webssh$/) {
1112    require DBI;
1113    require DBD::SQLite;
1114
1115    pb_cms2build("SandBox","Web");
1116    pb_send2target("Web");
1117} elsif ($action =~ /^cms2webpkg$/) {
1118    require DBI;
1119    require DBD::SQLite;
1120
1121    pb_cms2build("CMS","Web");
1122} elsif ($action =~ /^cms2webssh$/) {
1123    require DBI;
1124    require DBD::SQLite;
1125
1126    pb_cms2build("CMS","Web");
1127    pb_send2target("Web");
1128} else {
1129    pb_log(0,"\'$action\' is not available\n");
1130    pb_syntax(-2,1);
1131}
1132
1133pb_exit();
1134
1135sub pb_cms2build {
1136
1137    my $param = shift;
1138    my $web = shift;
1139
1140    my $pkg;
1141    my @pkgs;
1142    my $webdir;
1143
1144    my %pkgs;
1145    my $pb;             # Structure to store conf info
1146
1147    confess "pb_cms2build requires a parameter: SandBox or CMS" if (not defined $param);
1148
1149    # If Website, then pkg is only the website
1150    if (defined $web) {
1151        ($webdir) = pb_conf_get("webdir");
1152        pb_log(2,"webdir: ".Dumper($webdir)."\n");
1153        $pkgs[0] = $webdir->{$ENV{'PBPROJ'}};
1154        $extpkgdir = $webdir;
1155        pb_log(0,"Package: $pkgs[0]\n");
1156    } else {
1157        $pkg = pb_cms_get_pkg($defpkgdir,$extpkgdir);
1158        @pkgs = @$pkg;
1159    }
1160
1161    my ($scheme, $uri) = pb_vcs_init($pbinit,$param);
1162
1163    # We need 2 lines here
1164    my ($pkgv, $pkgt, $testver) = pb_conf_get_if("pkgver","pkgtag","testver");
1165
1166    # declare packager and repo for filtering
1167    my ($tmp1, $tmp2) = pb_conf_get("pbpackager","pbrepo");
1168    $ENV{'PBPACKAGER'} = $tmp1->{$ENV{'PBPROJ'}};
1169    $ENV{'PBREPO'} = $tmp2->{$ENV{'PBPROJ'}};
1170    my ($delivery) = pb_conf_get_if("delivery");
1171    $delivery->{$ENV{'PBPROJ'}} = "" if (not defined $delivery->{$ENV{'PBPROJ'}});
1172
1173    # If we deal with a test dir, we want to keep the date in tar file and dir name
1174    my $pbextdirpkg = "";
1175    my $pbextdir = "";
1176    if ((defined $testver) && (defined $testver->{$ENV{'PBPROJ'}}) && ($testver->{$ENV{'PBPROJ'}} =~ /true/i)) {
1177        $pbextdirpkg = strftime("%Y%m%d%H%M%S", @date);
1178        $pbextdir = ".0.".$pbextdirpkg;
1179    }
1180
1181    foreach my $pbpkg (@pkgs) {
1182        $ENV{'PBPKG'} = $pbpkg;
1183        $pbrev = $ENV{'PBREVISION'};
1184
1185        if ((defined $pkgv) && (defined $pkgv->{$pbpkg})) {
1186            $pbver = $pkgv->{$pbpkg};
1187        } else {
1188            $pbver = $ENV{'PBPROJVER'};
1189        }
1190        # If it's a test version, then build a tag to fit distributions need
1191        if ((defined $testver) && (defined $testver->{$ENV{'PBPROJ'}}) && ($testver->{$ENV{'PBPROJ'}} =~ /true/i)) {
1192            my $letter = "";
1193            if ($scheme =~ /svn/) {
1194                $letter = "s";
1195            } elsif ($scheme =~ /cvs/) {
1196                $letter = "c";
1197            } elsif ($scheme =~ /svk/) {
1198                $letter = "k";
1199            } elsif ($scheme =~ /hg/) {
1200                $letter = "h";
1201            } elsif ($scheme =~ /git/) {
1202                $letter = "g";
1203            } else {
1204                $letter = "f";
1205            }
1206            $pbtag = "0.$pbextdirpkg.$letter$pbrev";
1207            $ENV{'PBPROJTAG'} = $pbtag;
1208        } elsif ((defined $pkgt) && (defined $pkgt->{$pbpkg})) {
1209            $pbtag = $pkgt->{$pbpkg};
1210        } else {
1211            $pbtag = $ENV{'PBPROJTAG'};
1212        }
1213
1214        if (($pbtag =~ /:/) && ($scheme =~ /svn/)) {
1215            confess "RPM doesn't support release tag with ':'. Try to svn up before running pb";
1216        }
1217
1218        pb_log(0,"\n");
1219        pb_log(0,"Management of $pbpkg $pbver-$pbtag\n");
1220        confess "Unable to get env var PBDESTDIR" if (not defined $ENV{'PBDESTDIR'});
1221
1222        my $dest = "$ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir";
1223        # Create the structure if needed
1224        pb_mkdir_p($dest);
1225        # Clean up dest if necessary. The export will recreate it
1226        pb_rm_rf($dest) if (-d $dest);
1227
1228        # Export CMS tree for the concerned package to dest
1229        # And generate some additional files
1230        $OUTPUT_AUTOFLUSH=1;
1231
1232        # computes in which dir we have to work
1233        my $dir = $defpkgdir->{$pbpkg};
1234        $dir = $extpkgdir->{$pbpkg} if (not defined $dir);
1235        $dir = $webdir->{$ENV{'PBPROJ'}} if (defined $web);
1236        confess "Variable \$dir not defined. Check the package name first.\nIf this is the one expected, please report to dev team with log of a verbose run and this info ".Dumper($defpkgdir,$extpkgdir,$webdir) if (not defined $dir);
1237        pb_log(2,"def:".Dumper($defpkgdir)."ext: ".Dumper($extpkgdir)."dir: $dir\n");
1238
1239        # Exporting content from CMS
1240        my $sourcedir = undef;
1241        my $sourceuri = $uri;
1242        if ($param eq "SandBox") {
1243            # Point to the local instance
1244            $sourcedir = "$ENV{'PBDIR'}/$dir";
1245        } else {
1246            # Get it from a subdir of the URI with same version as localy but different root
1247            # Only if using a real CMS
1248            my ($scheme, $account, $host, $port, $path) = pb_get_uri($uri);
1249            if (($scheme !~ /^file/) && ($scheme !~ /^(ht|f)tp/)) {
1250                $sourceuri = "$ENV{'PBDIR'}/$dir"; 
1251                $sourceuri =~ s|^$ENV{'PBPROJDIR'}/|$uri/|;
1252            }
1253        }
1254        my $preserve = pb_vcs_export($sourceuri,$sourcedir,$dest);
1255
1256        # Generated fake content for test versions to speed up stuff
1257        my $chglog;
1258
1259        # Get project info on authors and log file
1260        # TODO: Make it CMS aware
1261        $chglog = "$ENV{'PBROOTDIR'}/$pbpkg/pbcl";
1262        $chglog = "$ENV{'PBROOTDIR'}/pbcl" if (! -f $chglog);
1263        $chglog = undef if (! -f $chglog);
1264
1265        # TODO: Make it CMS aware
1266        my $authors = "$ENV{'PBROOTDIR'}/$pbpkg/pbauthors";
1267        $authors = "$ENV{'PBROOTDIR'}/pbauthors" if (! -f $authors);
1268        $authors = "/dev/null" if (! -f $authors);
1269
1270        # Extract cms log history and store it
1271        if ((defined $chglog) && (! -f "$dest/NEWS")) {
1272            pb_log(2,"Generating NEWS file from $chglog\n");
1273            copy($chglog,"$dest/NEWS") || confess "Unable to create $dest/NEWS";
1274        }
1275        pb_cms_log($scheme,"$ENV{'PBDIR'}/$dir",$dest,$chglog,$authors,$testver);
1276
1277        my %build;
1278        # We want to at least build for the underlying distro
1279        # except if a target was given, in which case we only build for it
1280        # if -t was passed without target then build for the native distro.
1281        my $pbos = pb_distro_get_context($pbtarget);
1282        my $tmpl = pb_get_distros($pbos,$pbtarget);
1283
1284        # Setup $pb structure to allow filtering later on, on files using that structure
1285        $pb->{'tag'} = $pbtag;
1286        $pb->{'rev'} = $pbrev;
1287        $pb->{'ver'} = $pbver;
1288        $pb->{'pkg'} = $pbpkg;
1289        $pb->{'suf'} = $pbos->{'suffix'};
1290        $pb->{'realpkg'} = $pbpkg;
1291        $pb->{'date'} = $pbdate;
1292        $pb->{'defpkgdir'} = $defpkgdir;
1293        $pb->{'extpkgdir'} = $extpkgdir;
1294        $pb->{'chglog'} = $chglog;
1295        $pb->{'extdir'} = $pbextdir;
1296        $pb->{'packager'} = $ENV{'PBPACKAGER'};
1297        $pb->{'proj'} = $ENV{'PBPROJ'};
1298        $pb->{'repo'} = "$ENV{'PBREPO'}/$delivery->{$ENV{'PBPROJ'}}";
1299        $pb->{'patches'} = ();
1300        $pb->{'sources'} = ();
1301   
1302        my $tmpd = "$ENV{'PBTMP'}/cms.$$";
1303        pb_mkdir_p($tmpd) if (defined $pbparallel);
1304
1305        # Get only all.yml filter at this stage for pbinit
1306        my $ptr = pb_get_filters($pbpkg);
1307
1308        # Do not do that for website
1309        if (not defined $web) {
1310            my %virt;
1311            # De-duplicate similar VM/VE/RM
1312            foreach my $d (split(/,/,$tmpl)) {
1313                # skip ill-formatted vms (name-ver-arch)
1314                next if ($d !~ /-/);
1315                $virt{$d} = $d;
1316            }
1317
1318            # Try to use // processing here
1319            my $all_ok = 1;
1320            my $pm;
1321            if (defined $pbparallel) {
1322                $pm = new Parallel::ForkManager($pbparallel);
1323                $pm->set_waitpid_blocking_sleep(0);
1324                $pm->run_on_finish(sub { my ($pid, $code, $id, $signal, $dump) = @_;
1325                                    $all_ok = 0 unless (($code == 0) && ($signal == 0) && ($dump == 0)); });
1326            }
1327
1328            pb_log(0,"Preparing delivery ...\n");
1329            foreach my $v (keys %virt) {
1330                $pm->start and next if (defined $pbparallel);
1331
1332                # Distro context
1333                my $pbos = pb_distro_get_context($v);
1334   
1335                $pb->{'pbos'} = $pbos;
1336                $pb->{'suf'} = $pbos->{'suffix'};
1337                pb_log(3,"DEBUG: pb: ".Dumper($pb)."\n");
1338
1339                # Get all filters to apply
1340                $ptr = pb_get_filters($pbpkg,$pbos);
1341   
1342                pb_log(2,"DEBUG Filtering PBDATE => $pbdate, PBTAG => $pbtag, PBVER => $pbver\n");
1343   
1344                # We need to compute the real name of the package
1345                my $pbrealpkg = pb_cms_get_real_pkg($pbpkg,$pbos->{'type'});
1346                $pb->{'realpkg'} = $pbrealpkg;
1347                pb_log(2,"Virtual package $pbpkg has a real package name of $pbrealpkg on $pbos->{'name'}-$pbos->{'version'}\n") if ($pbrealpkg ne $pbpkg);
1348   
1349                # Filter build files from the less precise up to the most with overloading
1350                # Filter all files found, keeping the name, and generating in dest
1351   
1352                # Find all build files first relatively to PBROOTDIR
1353                # Find also all specific files referenced in the .yml conf file
1354                my %bfiles = ();
1355                my %pkgfiles = ();
1356                # Used in Filter.pm by pb_filter_file
1357
1358                $build{$v} = "no";
1359                if (-d "$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'os'}") {
1360                    pb_list_bfiles("$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'os'}",$pbpkg,\%bfiles,\%pkgfiles,$supfiles);
1361                    $build{$v} = "yes";
1362                } 
1363                if (-d "$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'type'}") {
1364                    pb_list_bfiles("$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'type'}",$pbpkg,\%bfiles,\%pkgfiles,$supfiles);
1365                    $build{$v} = "yes";
1366                } 
1367                if (-d "$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'family'}") {
1368                    pb_list_bfiles("$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'family'}",$pbpkg,\%bfiles,\%pkgfiles,$supfiles);
1369                    $build{$v} = "yes";
1370                } 
1371                if (-d "$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'name'}") {
1372                    pb_list_bfiles("$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'name'}",$pbpkg,\%bfiles,\%pkgfiles,$supfiles);
1373                    $build{$v} = "yes";
1374                } 
1375                if (-d "$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'name'}-$pbos->{'version'}") {
1376                    pb_list_bfiles("$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'name'}-$pbos->{'version'}",$pbpkg,\%bfiles,\%pkgfiles,$supfiles);
1377                    $build{$v} = "yes";
1378                } 
1379                if (-d "$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}") {
1380                    pb_list_bfiles("$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}",$pbpkg,\%bfiles,\%pkgfiles,$supfiles);
1381                    $build{$v} = "yes";
1382                }
1383                pb_log(2,"DEBUG($v) bfiles: ".Dumper(\%bfiles)."\n");
1384   
1385                if ($build{$v} ne "no") {
1386                    # Apply now all the filters on all the files concerned
1387                    # destination dir depends on the type of file
1388                    # For patch support
1389                    # TODO: Make it CMS aware
1390                    $pb->{'patches'} = pb_list_sfiles("$ENV{'PBROOTDIR'}/$pbpkg/pbpatch", $pb->{'patches'}, $pbos, "$ENV{'PBROOTDIR'}/$pbpkg/pbextpatch");
1391                    pb_log(2,"DEBUG($v) patches: ".Dumper($pb->{'patches'})."\n");
1392                    # TODO: Make it CMS aware
1393                    $pb->{'sources'} = pb_list_sfiles("$ENV{'PBROOTDIR'}/$pbpkg/pbsrc", $pb->{'sources'}, $pbos, "$ENV{'PBROOTDIR'}/$pbpkg/pbextsrc");
1394                    pb_log(2,"DEBUG($v) sources: ".Dumper($pb->{'sources'})."\n");
1395   
1396                    if (defined $pb->{'patches'}->{$v}) {
1397                        # Filter potential patches (local + remote)
1398                        pb_mkdir_p("$dest/pbconf/$v/pbpatch");
1399                        # If Debian based distribution, then prepare what will be done at build time
1400                        my ($patchcmd,$patchopt);
1401                        if ($pbos->{'type'} eq "deb") {
1402                            ($patchcmd,$patchopt) = pb_distro_get_if($pbos,("ospatchcmd","ospatchopt"));
1403                            open(SCRIPT,"> $dest/pbconf/$v/pbpatch/pbapplypatch") || confess "Unable to create $dest/pbconf/$v/pbpatch/pbapplypatch";
1404                            print SCRIPT "#!/bin/bash\n";
1405                            print SCRIPT "set -x\n" if ($pbdebug gt 1);
1406                        }
1407                        foreach my $pf (split(/,/,$pb->{'patches'}->{$v})) {
1408                            my $pp = basename($pf);
1409                            pb_vcs_export($pf,undef,"$dest/pbconf/$v/pbpatch");
1410                            pb_filter_file_inplace($ptr,"$dest/pbconf/$v/pbpatch/$pp",$pb);
1411                            pb_system("gzip -9f $dest/pbconf/$v/pbpatch/$pp","","quiet");
1412                            if ($pbos->{'type'} eq "deb") {
1413                                # If Debian based distribution, then prepare what will be done at build time
1414                                # by applying the patches that will be available under the debian/patches dir
1415                                print SCRIPT "$patchcmd $patchopt \< debian/patches/$pp\n";
1416                            }
1417                        }
1418                        if ($pbos->{'type'} eq "deb") {
1419                            close(SCRIPT);
1420                            chmod 0755,"$dest/pbconf/$v/pbpatch/pbapplypatch";
1421                        }
1422                        #pb_system("cat $dest/pbconf/$v/pbpatch/pbapplypatch","APPLY","verbose");
1423                    }
1424                    if (defined $pb->{'sources'}->{$v}) {
1425                        pb_mkdir_p("$dest/pbconf/$v/pbsrc");
1426                        foreach my $pf (split(/,/,$pb->{'sources'}->{$v})) {
1427                            my $pp = basename($pf);
1428                            pb_vcs_export($pf,undef,"$dest/pbconf/$v/pbsrc");
1429                            pb_filter_file_inplace($ptr,"$dest/pbconf/$v/pbsrc/$pp",$pb);
1430                        }
1431                    }
1432                    # Filter build files at the end, as they depend on patches and sources
1433                    foreach my $f (keys %bfiles) {
1434                        pb_filter_file("$ENV{'PBROOTDIR'}/$bfiles{$f}",$ptr,"$dest/pbconf/$v/$f",$pb);
1435                    }
1436                    foreach my $f (keys %pkgfiles) {
1437                        pb_filter_file("$ENV{'PBROOTDIR'}/$pkgfiles{$f}",$ptr,"$dest/pbconf/$v/$f",$pb);
1438                    }
1439                }
1440
1441                if (defined $pbparallel) {
1442                    # Communicate results back to parent
1443                    my $str = "";
1444                    $str .= "build $v = $build{$v}\n" if (defined $build{$v});
1445                    $str .= "patches $v = $pb->{'patches'}->{$v}\n" if (defined $pb->{'patches'}->{$v});
1446                    $str .= "sources $v = $pb->{'sources'}->{$v}\n" if (defined $pb->{'sources'}->{$v});
1447                    pb_set_content("$tmpd/$$","$str");
1448                    $pm->finish;
1449                }
1450            }
1451            # In the parent, we need to get the result from the children
1452            $pm->wait_all_children if (defined $pbparallel);
1453            confess "Aborting, one or more of the children failed." if ((not $all_ok) && ($Global::pb_stop_on_error));
1454            my $made = "";
1455            my %h = ();
1456            my %tmp;
1457            my %tmp2;
1458            my $pt;
1459            my $k;
1460
1461            foreach $k (<$tmpd/*>) {
1462                $made .= pb_get_content($k);
1463            }
1464            pb_rm_rf($tmpd);
1465            pb_log(3,"MADE:\n$made");
1466
1467            # Rebuild local hashes
1468            foreach $k (split(/\n/,$made)) {
1469                if ($k =~ /^\s*([A-z0-9-_]+)\s+([[A-z0-9-_.]+)\s*=\s*(.+)$/) {
1470                    $h{$1}->{$2}=$3;
1471                }
1472            }
1473            pb_log(2,"HASH: ".Dumper(%h));
1474
1475            # Patches
1476            pb_log(0,"Delivered and compressed patches ");
1477            $pt = $h{'patches'};
1478            foreach $k (keys %$pt) {
1479                foreach my $v1 (split(/,/,$pt->{$k})) {
1480                    $tmp{$v1} = "";
1481                }
1482            }
1483            if (keys %tmp) {
1484                foreach $k (keys %tmp) {
1485                    pb_log(0,"$k ");
1486                }
1487            } else {
1488                pb_log(0,"N/A");
1489            }
1490            pb_log(0,"\n");
1491
1492            # Sources
1493            pb_log(0,"Delivered additional sources ");
1494            $pt = $h{'sources'};
1495            foreach $k (keys %$pt) {
1496                foreach my $v1 (split(/,/,$pt->{$k})) {
1497                    $tmp2{$v1} = "";
1498                }
1499            }
1500            if (keys %tmp2) {
1501                foreach $k (keys %tmp2) {
1502                    pb_log(0,"$k ");
1503                }
1504            } else {
1505                pb_log(0,"N/A");
1506            }
1507            pb_log(0,"\n");
1508
1509            # Build files
1510            my @found;
1511            my @notfound;
1512            $pt = $h{'build'};
1513            foreach my $b (keys %$pt) {
1514                push @found,$b if ($pt->{$b} =~ /yes/);
1515                push @notfound,$b if ($pt->{$b} =~ /no/);
1516            }
1517            pb_log(0,"Build files have been generated for ... ".join(',',sort(@found))."\n") if (@found);
1518            pb_log(0,"No Build files found for ".join(',',sort(@notfound))."\n") if (@notfound);
1519
1520        } else {
1521            # Instead call News generation
1522            pb_web_news2html($dest);
1523            # And create an empty pbconf
1524            pb_mkdir_p("$dest/pbconf");
1525            # And prepare the pbscript to execute remotely
1526            open(SCRIPT,"> $ENV{'PBTMP'}/pbscript") || confess "Unable to create $ENV{'PBTMP'}/pbscript";
1527            print SCRIPT "#!/bin/bash\n";
1528            print SCRIPT "#set -x\n";
1529            print SCRIPT "echo ... Extracting Website content\n";
1530            print SCRIPT "find . -type f | grep -Ev '^./$pbpkg-$pbver$pbextdir.tar.gz|^./pbscript' | xargs rm -f non-existent\n";
1531            print SCRIPT "find * -type d -depth | xargs rmdir 2> /dev/null \n";
1532            print SCRIPT "tar xfz $pbpkg-$pbver$pbextdir.tar.gz\n";
1533            print SCRIPT "mv $pbpkg-$pbver$pbextdir/* .\n";
1534            print SCRIPT "rm -f $pbpkg-$pbver$pbextdir.tar.gz\n";
1535            print SCRIPT "rmdir $pbpkg-$pbver$pbextdir\n";
1536            print SCRIPT "find . -type f -print0 | xargs -0 chmod 644\n";
1537            print SCRIPT "find . -type d -print0 | xargs -0 chmod 755\n";
1538            close(SCRIPT);
1539            chmod 0755,"$ENV{'PBTMP'}/pbscript";
1540        }
1541
1542        # Apply filters to the non-build files
1543        my $liste ="";
1544        if (defined $filteredfiles->{$pbpkg}) {
1545            foreach my $f (split(/,/,$filteredfiles->{$pbpkg})) {
1546                pb_filter_file_inplace($ptr,"$dest/$f",$pb);
1547                $liste = "$f $liste";
1548            }
1549        }
1550        pb_log(2,"Files ".$liste."have been filtered\n");
1551
1552        # TODO: Make it CMS aware
1553        # Execute the pbinit script if any
1554        if (-x "$ENV{'PBROOTDIR'}/$pbpkg/pbinit") {
1555            pb_filter_file("$ENV{'PBROOTDIR'}/$pbpkg/pbinit",$ptr,"$ENV{'PBTMP'}/pbinit",$pb);
1556            chmod 0755,"$ENV{'PBTMP'}/pbinit";
1557            pb_system("cd $dest ; $ENV{'PBTMP'}/pbinit","Executing init script from $ENV{'PBROOTDIR'}/$pbpkg/pbinit under $dest","verbose");
1558        }
1559
1560        # Do we have additional script to run to prepare the environement for the project ?
1561        # Then include it in the pbconf delivery
1562        foreach my $pbvf (<$ENV{'PBROOTDIR'}/$pbpkg/pbv*.pre>,<$ENV{'PBROOTDIR'}/$pbpkg/pbv*.post>) {
1563            pb_log(1,"Handling file $pbvf\n");
1564            if (-x "$pbvf") {
1565                my $target = "$ENV{'PBTMP'}/".basename($pbvf);
1566                pb_filter_file("$pbvf",$ptr,$target,$pb);
1567                chmod 0755,"$target";
1568                pb_log(1,"Capturing file $pbvf in $target\n");
1569            }
1570        }
1571
1572        # Prepare the dest directory for archive
1573        chdir "$ENV{'PBDESTDIR'}" || confess "Unable to change dir to $ENV{'PBDESTDIR'}";
1574        if (defined $preserve) {
1575            # In that case we want to preserve the original tar file for checksum purposes
1576            # The one created is btw equivalent in that case to this one
1577            # Maybe check basename of both to be sure they are the same ?
1578            pb_log(0,"Preserving original tar file ");
1579            move("$preserve","$pbpkg-$pbver$pbextdir.tar.gz");
1580        } else {
1581            # Possibility to look at PBSRC to guess more the filename
1582            pb_system("tar cfz $pbpkg-$pbver$pbextdir.tar.gz --exclude=$pbpkg-$pbver$pbextdir/pbconf $pbpkg-$pbver$pbextdir","Creating $pbpkg tar files compressed");
1583        }
1584        pb_log(0,"Under $ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.tar.gz\n");
1585        pb_system("tar cfz $pbpkg-$pbver$pbextdir.pbconf.tar.gz $pbpkg-$pbver$pbextdir/pbconf","Creating pbconf tar files compressed");
1586        pb_log(0,"Under $ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.pbconf.tar.gz\n");
1587
1588        # Keep track of version-tag per pkg
1589        $pkgs{$pbpkg} = "$pbver-$pbtag";
1590
1591        # Final cleanup
1592        pb_rm_rf($dest) if (-d $dest);
1593    }
1594
1595    # Keep track of per package version
1596    pb_log(2,"DEBUG pkgs: ".Dumper(%pkgs)."\n");
1597    open(PKG,"> $ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml") || confess "Unable to create $ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml";
1598    print PKG "---\n";
1599    print PKG "pbpkg:\n";
1600    foreach my $pbpkg (keys %pkgs) {
1601        print PKG "  $pbpkg: $pkgs{$pbpkg}\n";
1602    }
1603    close(PKG);
1604
1605    # Keep track of what is generated by default
1606    # We need to store the dir and info on version-tag
1607    # Base our content on the existing .yml file
1608    copy("$ENV{'PBROOTDIR'}/$ENV{'PBPROJ'}.yml","$ENV{'PBDESTDIR'}/pbrc.yml");
1609    my $lh;
1610    $lh = pb_conf_cache("$ENV{'PBDESTDIR'}/pbrc.yml",$lh);
1611    $lh->{'pbroot'}->{$ENV{'PBPROJ'}} = $ENV{'PBROOTDIR'};
1612    $lh->{'projver'}->{$ENV{'PBPROJ'}} = $ENV{'PBPROJVER'};
1613    $lh->{'projtag'}->{$ENV{'PBPROJ'}} = $ENV{'PBPROJTAG'};
1614    $lh->{'pbpackager'}->{$ENV{'PBPROJ'}} = $ENV{'PBPACKAGER'};
1615    $lh->{'pbextdir'}->{$ENV{'PBPROJ'}} = $pbextdir;
1616    pb_conf_write("$ENV{'PBDESTDIR'}/pbrc.yml",$lh);
1617}
1618
1619sub pb_install2pkg {
1620    # Get the running distro to test on
1621    my $pbos = pb_distro_get_context();
1622
1623    # Get list of packages to test
1624    my $pkg = pb_cms_get_pkg($defpkgdir);
1625    my @pkgs = @$pkg;
1626    my @ipkgs;
1627    foreach my $pbpkg (@pkgs) {
1628        push(@ipkgs,pb_cms_get_real_pkg($pbpkg,$pbos->{'type'}));
1629    }
1630    # Additional potential repo
1631    pb_distro_setuprepo($pbos,"install");
1632    # We need to install the package to test, and deps brought with it
1633    pb_distro_installpkgs($pbos,join(' ',@ipkgs));
1634}
1635
1636sub pb_test2pkg {
1637    # Get the running distro to test on
1638    my $pbos = pb_distro_get_context();
1639
1640    # Get list of packages to test
1641    # Get content saved in cms2build
1642    my $ptr = pb_get_pkg();
1643    @pkgs = @$ptr;
1644
1645    # Additional potential repo
1646    pb_distro_setuprepo($pbos,"test");
1647    foreach my $pbpkg (@pkgs) {
1648    }
1649    pb_system("$ENV{'PBDESTDIR'}/pbtest","Launching test","verbose");
1650}
1651
1652sub pb_build2prep {
1653
1654    pb_log(0,"INFO: ------ Starting to prepare build environement ------\n");
1655
1656    # Get the running distro to build on
1657    my $pbos = pb_distro_get_context();
1658
1659    # If needed we may add repository to the build env
1660    my $forcerepo = pb_distro_setuprepo($pbos,"build");
1661    pb_log(1,"Forcing full package install with forcerepo\n") if (defined $forcerepo);
1662
1663    # Get list of packages to build
1664    my $ptr = pb_get_pkg();
1665    @pkgs = @$ptr;
1666   
1667    # Get content saved in cms2build
1668    my ($pkg) = pb_conf_read("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml","pbpkg");
1669    $pkg = { } if (not defined $pkg);
1670    my $pbextdir = pb_get_extdir();
1671
1672    foreach my $pbpkg (@pkgs) {
1673        my $vertag = $pkg->{$pbpkg};
1674        my @buildfiles;
1675        pb_log(2,"Vertag: $vertag\n");
1676        # get the version of the current package - maybe different
1677        ($pbver,$pbtag) = split(/-/,$vertag);
1678
1679        my $src2="$ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.pbconf.tar.gz";
1680        pb_log(2,"Pbconf file: $src2\n");
1681        pb_log(2,"Working directory: $ENV{'PBBUILDDIR'}\n");
1682
1683        my $ftype;
1684        if ($pbos->{'type'} eq "rpm") {
1685            $ftype = "\.spec\$";
1686        } elsif ($pbos->{'type'} eq "deb") {
1687            $ftype = "\/control\$";
1688        } elsif ($pbos->{'type'} eq "ebuild") {
1689            $ftype = "\.ebuild\$";
1690        } elsif ($pbos->{'type'} eq "tgz") {
1691            $ftype = "\/pbslack\$";
1692        } elsif ($pbos->{'type'} eq "pkg") {
1693            # Solaris - no deps management up to Solaris 10
1694        } elsif ($pbos->{'type'} eq "hpux") {
1695            # TODO: HP-UX
1696        } elsif ($pbos->{'type'} eq "apk") {
1697            $ftype = "\/APKBUILD\$";
1698        } else {
1699            confess "Unknown OS type format $pbos->{'type'}";
1700        }
1701
1702        @buildfiles = pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/","$ENV{'PBTMP'}","build");
1703        pb_log(2,"buildfiles ".Dumper(\@buildfiles)."\n");
1704        foreach my $f (@buildfiles) {
1705            if ($f =~ /$ftype/) {
1706                pb_distro_installdeps($f,$pbos,$forcerepo);
1707            }
1708        }
1709    }
1710    pb_log(0,"INFO: ------ Finished preparing build environment ------\n");
1711}
1712
1713sub pb_build2pkg {
1714
1715    my $do_install = shift;
1716
1717    # Call build2prep in case we have new deps we want to take in account
1718    # without redoing the prep phase (can be long and that helps debugging
1719    # build dependencies issues
1720    pb_build2prep();
1721    pb_log(0,"INFO: ------ Starting to build package ------\n");
1722    # Get the running distro to build on
1723    my $pbos = pb_distro_get_context();
1724
1725    # Get list of packages to build
1726    my $ptr = pb_get_pkg();
1727    @pkgs = @$ptr;
1728
1729    # Get content saved in cms2build
1730    my ($pkg) = pb_conf_read("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml","pbpkg");
1731    $pkg = { } if (not defined $pkg);
1732    my $pbextdir = pb_get_extdir();
1733
1734    pb_mkdir_p("$ENV{'PBBUILDDIR'}") if (! -d "$ENV{'PBBUILDDIR'}");
1735    chdir "$ENV{'PBBUILDDIR'}" || confess "Unable to chdir to $ENV{'PBBUILDDIR'}";
1736    my $made = ""; # pkgs made during build
1737    my $pm;
1738    my $all_ok = 1;
1739
1740    if (defined $pbparallel) {
1741        $pm = new Parallel::ForkManager($pbparallel);
1742        $pm->set_waitpid_blocking_sleep(0);
1743        $pm->run_on_finish(sub { my ($pid, $code, $id, $signal, $dump) = @_;
1744                                $all_ok = 0 unless (($code == 0) && ($signal == 0) && ($dump == 0)); });
1745    }
1746
1747    # We need to communicate info back from the children if parallel so prepare a dir for that
1748    my $tmpd = "$ENV{'PBTMP'}/build.$$";
1749    pb_mkdir_p($tmpd) if (defined $pbparallel);
1750
1751    # Do it here as this doesn't work in // mode as it dies if it fails
1752    if ($pbos->{'type'} eq "rpm") {
1753        foreach my $d ('RPMS','SRPMS','SPECS','SOURCES','BUILD') {
1754                if (! -d "$ENV{'PBBUILDDIR'}/$d") {
1755                    pb_mkdir_p("$ENV{'PBBUILDDIR'}/$d");
1756                }
1757            }
1758    }
1759
1760    foreach my $pbpkg (@pkgs) {
1761        $pm->start and next if (defined $pbparallel);
1762
1763        my $vertag = $pkg->{$pbpkg};
1764        pb_log(2,"Vertag: $vertag\n");
1765        # get the version of the current package - maybe different
1766        ($pbver,$pbtag) = split(/-/,$vertag);
1767
1768        my $src="$ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.tar.gz";
1769        my $src2="$ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.pbconf.tar.gz";
1770        pb_log(2,"Source file: $src\n");
1771        pb_log(2,"Pbconf file: $src2\n");
1772
1773        pb_log(2,"Working directory: $ENV{'PBBUILDDIR'}\n");
1774        if ($pbos->{'type'} eq "rpm") {
1775            # Remove in case a previous link/file was there
1776            unlink "$ENV{'PBBUILDDIR'}/SOURCES/".basename($src);
1777            symlink "$src","$ENV{'PBBUILDDIR'}/SOURCES/".basename($src) || confess "Unable to symlink $src in $ENV{'PBBUILDDIR'}/SOURCES";
1778            # We need to first extract the spec file
1779            my @buildfile = pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/","$ENV{'PBBUILDDIR'}/SPECS","build");
1780
1781            # We need to handle potential patches to upstream sources
1782            pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbpatch/","$ENV{'PBBUILDDIR'}/SOURCES","patch");
1783
1784            # We need to handle potential additional sources to upstream sources
1785            pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbsrc/","$ENV{'PBBUILDDIR'}/SOURCES","src");
1786
1787            pb_log(2,"buildfile: ".Dumper(\@buildfile)."\n");
1788            # set LANGUAGE to check for correct log messages
1789            $ENV{'LANGUAGE'}="C";
1790            # Older Redhat use _target_platform in %configure incorrectly
1791            my $specialdef = "";
1792            if (($pbos->{'name'} eq "rhel") && ($pbos->{'version'} eq "2.1")) {
1793                $specialdef = "--define \'_target_platform \"\"\'";
1794            }
1795            my $buildcmd = "rpmbuild";
1796            if (($pbos->{'name'} eq "redhat") && ($pbos->{'version'} =~ /^6/)){
1797                $buildcmd = "rpm" ;
1798            }
1799
1800            foreach my $f (@buildfile) {
1801                if ($f =~ /\.spec$/) {
1802                    pb_system("$buildcmd $specialdef --define \"packager $ENV{'PBPACKAGER'}\" --define \"_topdir $ENV{'PBBUILDDIR'}\" -ba $f","Building package with $f under $ENV{'PBBUILDDIR'}","verbose");
1803                    last;
1804                }
1805            }
1806            # Get the name of the generated packages
1807            open(LOG,"$ENV{'PBTMP'}/system.$$.log") || confess "Unable to open $ENV{'PBTMP'}/system.$$.log";
1808            while (<LOG>) {
1809                chomp($_);
1810                next if ($_ !~ /^Wrote:/);
1811                s|.*/([S]*RPMS.*)|$1|;
1812                $made .=" $_";
1813            }
1814            close(LOG);
1815
1816        } elsif ($pbos->{'type'} eq "deb") {
1817            pb_system("tar xfz $src","Extracting sources");
1818            pb_system("tar xfz $src2","Extracting pbconf");
1819
1820            chdir "$pbpkg-$pbver$pbextdir" || confess "Unable to chdir to $pbpkg-$pbver$pbextdir";
1821            pb_rm_rf("debian");
1822            my $confdir = "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
1823            confess "Configuration directory $confdir is not a directory\nIs os description listed in your {ve,vm,rm}list values?" if (not -d $confdir);
1824            symlink "$confdir","debian" || confess "Unable to symlink 'debian' to $confdir";
1825            chmod 0755,"debian/rules";
1826
1827            # We need to handle potential patches to upstream sources
1828            pb_mkdir_p("debian/patches");
1829            my @f = pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbpatch/","debian/patches","patch");
1830
1831            # By default we use format 1.0 - Cf man dpkg-source
1832            my $debsrcfmt = "1.0";
1833            my $debsrcfile = "debian/source/format";
1834            if (-f $debsrcfile) {
1835                $debsrcfmt = pb_get_content($debsrcfile);
1836            }
1837
1838            if ($debsrcfmt =~ /^3.*quilt/) {
1839                # If we use quilt to manage patches, we then setup the env correctly
1840                # as per http://pkg-perl.alioth.debian.org/howto/quilt.html
1841                # Generate Debian patch series for quilt
1842                open(SERIE,"> debian/patches/series") || confess "Unable to write in debian/patches/series";
1843                $ENV{'QUILT_PATCHES'}="debian/patches";
1844            } else {
1845                # If we use dpatch to manage patches, we then setup the 00list file as well
1846                open(SERIE,"> debian/patches/00list") || confess "Unable to write in debian/patches/00list";
1847            }
1848            foreach my $f (sort @f) {
1849                # Skip the script made to apply the patches to the Debian tree
1850                next if ($f =~ /pbapplypatch/);
1851                # We also need to uncompress them
1852                pb_system("gzip -d $f","","quiet");
1853                $f =~ s/\.gz$//;
1854                print SERIE "$f\n";
1855            }
1856            close(SERIE);
1857            if (@f) {
1858                # We have patches...
1859                my $patch_file = "debian/patches/pbapplypatch";
1860                if (($debsrcfmt =~ /^1.*/) && (-x $patch_file)) {
1861                    # In that case we need to apply the patches ourselves locally
1862                    pb_system("cat $patch_file","APPLY","verbose");
1863                    pb_system("$patch_file","Applying patches to $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'} tree");
1864                }
1865                # ...so modify the name of files to be Debian compliant
1866                move("../$src","../$pbpkg-$pbver$pbextdir.orig.tar.gz");
1867            }
1868
1869            # We need to handle potential additional sources to upstream sources
1870            #pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbsrc/","$ENV{'PBBUILDDIR'}/debian","src");
1871
1872            pb_system("dpkg-buildpackage -us -uc -rfakeroot","Building package","verbose");
1873
1874            # Get the name of the generated packages
1875            open(LOG,"$ENV{'PBTMP'}/system.$$.log") || confess "Unable to open $ENV{'PBTMP'}/system.$$.log";
1876            while (<LOG>) {
1877                chomp();
1878                pb_log(3,"In loop: $_\n");
1879                next unless (/^dpkg-deb.*\.\.\/(.*_.*\.deb).*/);
1880                my $tmp = $1;
1881                #doesn't work in my case
1882                #confess "Missing file $tmp" if (not -f "../$tmp");
1883                $made = "$made $tmp";
1884            }
1885            close(LOG);
1886            pb_log(2,"Now made is: $made\n");
1887
1888            open(CTRL,"debian/control") || confess "Unable to open debian/control: $!";
1889            while (<CTRL>) {
1890                pb_log(3,"In loop: $_\n");
1891                next unless (/^Source: (\S+)/o);
1892                my $tmp = $1;
1893                $made = "$made $tmp"."_*.dsc $tmp"."_*.tar.gz $tmp"."_*.changes";
1894                #doesn't work in my case
1895                #foreach my $glob (("$1\_*.changes", "$1\_*.dsc", "$1\_*.tar.gz")) {
1896                    #my @file = glob($glob);
1897                    #confess "Missing file for $glob" unless (@file > 0);
1898                    #confess "Too many files for $glob" if (@file > 1);
1899                    #confess "Missing file $file[0]" if (not -f $file[0]);
1900                    #$made .= " $file[0]";
1901                #}
1902            }
1903            close(CTRL);
1904            #pb_display_file("$ENV{'PBTMP'}/system.$$.log");
1905            pb_log(2,"Finally made is: $made\n");
1906
1907            chdir ".." || confess "Unable to chdir to parent dir";
1908            pb_rm_rf("$pbpkg-$pbver$pbextdir");
1909        } elsif ($pbos->{'type'} eq "ebuild") {
1910            my @ebuildfile;
1911            # For gentoo we need to take pb as subsystem name
1912            # We put every apps here under sys-apps. hope it's correct
1913            # We use pb's home dir in order to have a single OVERLAY line
1914            my $tmpe = "$ENV{'HOME'}/portage/pb/sys-apps/$pbpkg";
1915            pb_mkdir_p($tmpe) if (! -d "$tmpe");
1916            pb_mkdir_p("$ENV{'HOME'}/portage/distfiles") if (! -d "$ENV{'HOME'}/portage/distfiles");
1917
1918            # We need to first extract the ebuild file
1919            @ebuildfile = pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/","$tmpe","build");
1920
1921            # Prepare the build env for gentoo
1922            my $found = 0;
1923            my $pbbd = $ENV{'HOME'};
1924            $pbbd =~ s|/|\\/|g;
1925            if (-r "/etc/make.conf") {
1926                open(MAKE,"/etc/make.conf");
1927                while (<MAKE>) {
1928                    $found = 1 if (/$pbbd\/portage/);
1929                }
1930                close(MAKE);
1931            }
1932            if ($found == 0) {
1933                pb_system("sudo sh -c 'echo PORTDIR_OVERLAY=\"$ENV{'HOME'}/portage\" >> /etc/make.conf'");
1934            }
1935            #$found = 0;
1936            #if (-r "/etc/portage/package.keywords") {
1937            #open(KEYW,"/etc/portage/package.keywords");
1938            #while (<KEYW>) {
1939            #$found = 1 if (/portage\/pb/);
1940            #}
1941            #close(KEYW);
1942            #}
1943            #if ($found == 0) {
1944            #pb_system("sudo sh -c \"echo portage/pb >> /etc/portage/package.keywords\"");
1945            #}
1946
1947            # Build
1948            foreach my $f (@ebuildfile) {
1949                if ($f =~ /\.ebuild$/) {
1950                    move($f,"$tmpe/$pbpkg-$pbver.ebuild");
1951                    pb_system("cd $tmpe ; ebuild $pbpkg-$pbver.ebuild clean ; ebuild $pbpkg-$pbver.ebuild digest ; ebuild $pbpkg-$pbver.ebuild package","verbose");
1952                    # Now move it where pb expects it
1953                    pb_mkdir_p("$ENV{'PBBUILDDIR'}/portage/pb/sys-apps/$pbpkg");
1954                    if ($pbtag eq 0) {
1955                        # This is assumed to be a test version
1956                        my $nver = substr($pbver,0,-14);
1957                        my $ntag = substr($pbver,-14);
1958                        my $ebtg = "portage/pb/sys-apps/$pbpkg/$pbpkg-$nver"."_p$ntag.ebuild";
1959                        move("$tmpe/$pbpkg-$pbver.ebuild","$ENV{'PBBUILDDIR'}/$ebtg");
1960                        $made="$made $ebtg";
1961                    } else {
1962                        my $ebtg = "portage/pb/sys-apps/$pbpkg/$pbpkg-$pbver-r$pbtag.ebuild";
1963                        move("$tmpe/$pbpkg-$pbver.ebuild","$ENV{'PBBUILDDIR'}/$ebtg");
1964                        $made="$made $ebtg";
1965                    }
1966                }
1967            }
1968
1969        } elsif ($pbos->{'type'} eq "apk") {
1970            pb_mkdir_p("$ENV{'PBBUILDDIR'}/tmp");
1971
1972            # We need to first extract the APKBUILD file
1973            my @apkfile = pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/","$ENV{'PBBUILDDIR'}/tmp","build");
1974
1975            chdir "$ENV{'PBBUILDDIR'}/tmp" || confess "Unable to chdir to $ENV{'PBBUILDDIR'}/tmp";
1976
1977            # Build
1978            my $keyfile = pb_ssh_get(1);
1979            pb_mkdir_p("$ENV{'HOME'}/.abuild");
1980            open(CONF,"> $ENV{'HOME'}/.abuild/abuild.conf") || (cluck "Unable to create the abuild conf file" && next);
1981            print CONF "PACKAGER_PRIVKEY=\"$keyfile\"\n" if (defined $keyfile);
1982            close(CONF);
1983            foreach my $f (@apkfile) {
1984                if ($f =~ /APKBUILD$/) {
1985                    pb_system("ls -al $ENV{'HOME'}/.ssh ; pwd ; ls -alR ../.. ; sh -x /usr/bin/abuild checksum ; abuild unpack ; abuild -r -P .. build ; abuild -P .. rootpkg ; abuild -P pkg index","verbose");
1986                }
1987            }
1988            # Get the name of the generated packages
1989            open(LOG,"$ENV{'PBTMP'}/system.$$.log") || confess "Unable to open $ENV{'PBTMP'}/system.$$.log";
1990            while (<LOG>) {
1991                chomp($_);
1992                next if ($_ !~ /: Create /);
1993                s|.*: Create (.*.apk)|$1|;
1994                $made .= " $_";
1995            }
1996            close(LOG);
1997
1998        } elsif ($pbos->{'type'} eq "tgz") {
1999            # Slackware family
2000            $made="$made $pbpkg/$pbpkg-$pbver-*-$pbtag.tgz";
2001
2002            pb_system("tar xfz $src","Extracting sources");
2003            pb_system("tar xfz $src2","Extracting pbconf");
2004            chdir "$pbpkg-$pbver$pbextdir" || confess "Unable to chdir to $pbpkg-$pbver$pbextdir";
2005            symlink "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}","install" || confess "Unable to symlink to pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
2006            if (-x "install/pbslack") {
2007                pb_system("./install/pbslack","Building software");
2008                pb_system("sudo /sbin/makepkg -p -l y -c y $pbpkg","Packaging $pbpkg","verbose");
2009            }
2010            chdir ".." || confess "Unable to chdir to parent dir";
2011            pb_rm_rf("$pbpkg-$pbver$pbextdir");
2012        } elsif ($pbos->{'type'} eq "pkg") {
2013            # Solaris
2014            $made="$made $pbpkg-$pbver-$pbtag.pkg.gz";
2015            my $pkgdestdir="$ENV{'PBBUILDDIR'}/install";
2016
2017            # Will host resulting packages
2018            pb_mkdir_p("$pbos->{'type'}");
2019            pb_mkdir_p("$pkgdestdir/pbdelivery");
2020            pb_system("tar xfz $src","Extracting sources under $ENV{'PBBUILDDIR'}");
2021            pb_system("tar xfz $src2","Extracting pbconf under $ENV{'PBBUILDDIR'}");
2022            # We need to handle potential patches to upstream sources
2023            pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbpatch/","$ENV{'PBBUILDDIR'}","patch");
2024
2025            # We need to handle potential additional sources to upstream sources
2026            pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbsrc/","$ENV{'PBBUILDDIR'}","src");
2027
2028            chdir "$pbpkg-$pbver$pbextdir" || confess "Unable to chdir to $pbpkg-$pbver$pbextdir";
2029            if (-f "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbbuild") {
2030                chmod 0755,"pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbbuild";
2031                # pkginfo file is mandatory
2032                confess "Unable to find pkginfo file in pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}" if (! -f "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pkginfo");
2033                # Build
2034                pb_system("pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbbuild $pkgdestdir/pbdelivery","Building software and installing under $pkgdestdir/pbdelivery");
2035                # Copy complementary files
2036                if (-f "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/prototype") {
2037                    copy("pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/prototype", $pkgdestdir) 
2038                } else {
2039                    # No prototype provided, calculating it
2040                    open(PROTO,"> $pkgdestdir/prototype") || confess "Unable to create prototype file";
2041                    print PROTO "i pkginfo\n";
2042                    print PROTO "i depend\n" if (-f "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/depend");
2043                    $ENV{'PBSOLDESTDIR'} = "$pkgdestdir/pbdelivery";
2044                    find(\&create_solaris_prototype, "$pkgdestdir/pbdelivery");
2045                }
2046                copy("pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/depend", $pkgdestdir) if (-f "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/depend");
2047                copy("pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pkginfo", $pkgdestdir);
2048                pb_system("cd $pkgdestdir/pbdelivery ; pkgmk -o -f ../prototype -r $pkgdestdir/pbdelivery -d $ENV{'PBBUILDDIR'}/$pbos->{'type'}","Packaging $pbpkg","verbose");
2049                pb_system("cd $ENV{'PBBUILDDIR'}/$pbos->{'type'} ;  echo \"\" | pkgtrans -o -n -s $ENV{'PBBUILDDIR'}/$pbos->{'type'} $ENV{'PBBUILDDIR'}/$pbpkg-$pbver-$pbtag.pkg all","Transforming $pbpkg","verbose");
2050                pb_system("cd $ENV{'PBBUILDDIR'} ;  gzip -9f $pbpkg-$pbver-$pbtag.pkg","Compressing $pbpkg-$pbver-$pbtag.pkg","verbose");
2051            } else {
2052                pb_log(0,"No pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbbuild file found for $pbpkg-$pbver\n");
2053            }
2054            chdir ".." || confess "Unable to chdir to parent dir";
2055            pb_rm_rf("$pbpkg-$pbver$pbextdir","$ENV{'PBBUILDDIR'}/$pbos->{'type'}","$pkgdestdir");
2056        } elsif ($pbos->{'type'} eq "hpux") {
2057            # HP-UX
2058            pb_system("tar xfz $src","Extracting sources");
2059            pb_system("tar xfz $src2","Extracting pbconf");
2060
2061            chdir "$pbpkg-$pbver$pbextdir" || confess "Unable to chdir to $pbpkg-$pbver$pbextdir";
2062            pb_system("buildpackage ","Building package","verbose");
2063            # Get the name of the generated packages
2064            open(LOG,"$ENV{'PBTMP'}/system.$$.log") || confess "Unable to open $ENV{'PBTMP'}/system.$$.log";
2065            while (<LOG>) {
2066                chomp();
2067                my $tmp = $_;
2068                next if ($tmp !~ /^SD BUILD.*:/);
2069                $tmp =~ s|.*../(.*)_(.*).sd.*|$1|;
2070                $made = "$made $tmp"."_*.sd";
2071            }
2072            close(LOG);
2073            $made="$made $pbpkg-$pbver-$pbtag.sd";
2074
2075            chdir ".." || confess "Unable to chdir to parent dir";
2076            pb_rm_rf("$pbpkg-$pbver$pbextdir");
2077        } else {
2078            confess "Unknown OS type format $pbos->{'type'}";
2079        }
2080        if (defined $pbparallel) {
2081            # Communicate results back to parent
2082            pb_set_content("$tmpd/$$",$made);
2083            $pm->finish;
2084        }
2085    }
2086    if (defined $pbparallel) {
2087        # In the parent, we need to get the result from the children
2088        $pm->wait_all_children;
2089        foreach my $f (<$tmpd/*>) {
2090            $made .= " ".pb_get_content($f);
2091        }
2092        pb_rm_rf($tmpd);
2093        confess "Aborting, one or more of the children failed." if ((not $all_ok) && ($Global::pb_stop_on_error));
2094    }
2095
2096    # Sign packages
2097    pb_sign_pkgs($pbos,$made);
2098    pb_log(0,"INFO: ------ Finished building package ------\n");
2099
2100    # Find the appropriate check cmd/opts
2101    my ($chkcmd,$chkopt) = pb_distro_get_if($pbos,("oschkcmd","oschkopt"));
2102
2103    my $ret = "";
2104    # Packages check if needed
2105    if ($pbos->{'type'} eq "rpm") {
2106        if ((defined  $chkcmd) && (-x $chkcmd)) {
2107            my $cmd = "$chkcmd";
2108            $cmd .= " $chkopt" if (defined $chkopt);
2109            $cmd .= " $made";
2110            my $ret = pb_system("$cmd","Checking validity of rpms with $chkcmd","mayfailverbose");
2111            pb_log(0,"ERROR: when checking packages validity\n") if ($ret ne 0);
2112        }
2113        my $rpms ="";
2114        my $srpms ="";
2115        foreach my $f (split(/ /,$made)) {
2116            $rpms .= "$ENV{'PBBUILDDIR'}/$f " if ($f =~ /^RPMS\//);
2117            $srpms .= "$ENV{'PBBUILDDIR'}/$f " if ($f =~ /^SRPMS\//);
2118        }
2119        pb_log(0,"SRPM packages generated: $srpms\n");
2120        pb_log(0,"RPM packages generated: $rpms\n");
2121        $ret = $rpms;
2122    } elsif ($pbos->{'type'} eq "deb") {
2123        my $made2 = "";
2124        foreach my $f (split(/ /,$made)) {
2125            $made2 .= "$f " if ($f =~ /\.deb$/);
2126        }
2127        if ((defined  $chkcmd) && (-x $chkcmd)) {
2128            my $ret = pb_system("$chkcmd $chkopt $made2","Checking validity of debs with $chkcmd","mayfail");
2129            pb_log(0,"ERROR: when checking packages validity\n") if ($ret ne 0);
2130        }
2131        pb_log(0,"deb packages generated: $made2\n");
2132        $ret = $made2;
2133    } else {
2134        pb_log(0,"No check done for $pbos->{'type'} yet\n");
2135        pb_log(0,"Packages generated: $made\n");
2136        $ret = $made;
2137    }
2138
2139    # Keep track of what is generated so that we can get them back from VMs/RMs
2140    my $pbstore = "$ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
2141    open(KEEP,"> $pbstore") || confess "Unable to create $pbstore $!";
2142    print KEEP "$made\n";
2143    close(KEEP);
2144    pb_distro_installdeps(undef,$pbos,undef,$ret,"local") if ($do_install);
2145}
2146
2147sub pb_build2ssh {
2148    pb_send2target("Sources");
2149    pb_send2target("CPAN");
2150}
2151
2152sub pb_pkg2ssh {
2153    pb_send2target("Packages");
2154}
2155
2156# By default deliver to the the public site hosting the
2157# ftp structure (or whatever) or a VM/VE/RM
2158sub pb_send2target {
2159
2160    my $cmt = shift;
2161    my $pbscript1 = shift;
2162    my $v = shift;
2163    my $vexist = shift;         # 0 is FALSE
2164    my $vpid = shift;           # 0 is FALSE
2165    my $snapme = shift;         # 0 is FALSE
2166    my $pbstep = shift;         # == 3 is usage of container
2167
2168    $vexist = 0 if (not defined $vexist);
2169    $vpid = 0 if (not defined $vpid);
2170    $snapme = 0 if (not defined $snapme);
2171    $pbstep = 3 if (not defined $pbstep);
2172    pb_log(2,"DEBUG: pb_send2target($cmt,".Dumper($v).",$vexist,$vpid)\n");
2173    my $host = "sshhost";
2174    my $login = "sshlogin";
2175    my $dir = "sshdir";
2176    my $port = "sshport";
2177    my $conf = "sshconf";
2178    my $tmout = undef;
2179    my $path = undef;
2180
2181    if ($cmt =~ /^VM/) {
2182        $login = "vmlogin";
2183        $dir = "pbdefdir";
2184        # Specific VM
2185        $tmout = "vmtmout";
2186        $path = "vmpath";
2187        $host = "vmhost";
2188        $port = "vmport";
2189    } elsif ($cmt =~ /^RM/) {
2190        $login = "rmlogin";
2191        $dir = "pbdefdir";
2192        # Specific RM
2193        $tmout = "rmtmout";
2194        $path = "rmpath";
2195        $host = "rmhost";
2196        $port = "rmport";
2197    } elsif ($cmt =~ /^VE/) {
2198        $login = "velogin";
2199        $dir = "pbdefdir";
2200        # Specific VE
2201        $path = "vepath";
2202        $conf = "rbsconf";
2203    } elsif ($cmt eq "Web") {
2204        $host = "websshhost";
2205        $login = "websshlogin";
2206        $dir = "websshdir";
2207        $port = "websshport";
2208    } elsif ($cmt eq "CPAN") {
2209        $host = "cpanpause";
2210        $login = "";
2211        $dir = "cpandir";
2212        $port = "";
2213    }
2214    my $cmd = "";
2215    my $src = "";
2216    my $cpanpkg = 0;
2217    my $pbos;
2218
2219    my $pbextdir = pb_get_extdir();
2220
2221    if ($cmt ne "Announce") {
2222        # Get list of packages to build
2223        my $ptr = pb_get_pkg();
2224        my @pkgs = ();
2225        @pkgs = @$ptr if (defined $ptr);
2226
2227        # Get the running distro to consider
2228        $pbos = pb_distro_get_context($v);
2229
2230        my $pkg = { };
2231        # Get content saved in cms2build
2232        if ((defined $ENV{'PBDESTDIR'}) && (defined $ENV{'PBPROJVER'}) && (defined $ENV{'PBPROJTAG'}) && (-f "$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml")) {
2233            ($pkg) = pb_conf_read("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml","pbpkg");
2234            $pkg = { } if (not defined $pkg);
2235        }
2236
2237        pb_mkdir_p("$ENV{'PBBUILDDIR'}") if (! -d "$ENV{'PBBUILDDIR'}");
2238        chdir "$ENV{'PBBUILDDIR'}" || confess "Unable to chdir to $ENV{'PBBUILDDIR'}";
2239        foreach my $pbpkg (@pkgs) {
2240            my $vertag = $pkg->{$pbpkg};
2241            # get the version of the current package - maybe different
2242            pb_log(2,"Vertag: $vertag\n");
2243            ($pbver,$pbtag) = split(/-/,$vertag);
2244
2245            if (($cmt eq "Sources") || ($cmt =~ /(V[EM]|RM)(build|prep)/)) {
2246                $src = "$src $ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.tar.gz $ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.pbconf.tar.gz";
2247                my $cmd2 = "ln -sf $pbpkg-$pbver$pbextdir.tar.gz $pbpkg-latest.tar.gz";
2248                #my $cmd2 = "ln $pbpkg-$pbver$pbextdir.tar.gz $pbpkg-latest.tar.gz 2> /dev/null ; if [ \$? -ne 0 ]; then ln -sf $pbpkg-$pbver$pbextdir.tar.gz $pbpkg-latest.tar.gz ; fi";
2249                if ($cmd eq "") {
2250                    $cmd = $cmd2;
2251                } else {
2252                    $cmd = "$cmd ; $cmd2";
2253                }
2254            } elsif ($cmt eq "Web") {
2255                $src = "$src $ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.tar.gz"
2256            }
2257
2258            # Do we have one perl package
2259            my @nametype = pb_conf_get_if("namingtype");
2260            my $type = $nametype[0]->{$pbpkg};
2261            if ((defined $type) && ($type eq "perl")) {
2262                $cpanpkg = 1;
2263            }
2264        }
2265        # Adds conf file for availability of conf elements
2266        pb_conf_add("$ENV{'PBROOTDIR'}/$ENV{'PBPROJ'}.yml") if ((defined $ENV{'PBROOTDIR'}) && (-f "$ENV{'PBROOTDIR'}/$ENV{'PBPROJ'}.yml"));
2267    }
2268    my ($rbsconf,$testver,$delivery) = pb_conf_get_if($conf,"testver","delivery");
2269    if ($cmt eq "CPAN") {
2270        # Do not deliver on Pause if this is a test version
2271        return if (not defined $testver);
2272        return if ($testver =~ /true/);
2273        # Do not deliver on Pause if this is not a perl package
2274        return if ($cpanpkg == 0);
2275    }
2276
2277    if ($cmt =~ /(V[EM]|RM)(build|prep)/) {
2278        $src="$src $ENV{'PBROOTDIR'}/$ENV{'PBPROJ'}.yml $ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml $ENV{'PBETC'} $ENV{'PBDESTDIR'}/pbrc.yml $pbscript{$v}";
2279    } elsif ($cmt =~ /(V[EM]|RM)Script/) {
2280        $src="$src $pbscript{$v}";
2281    } elsif ($cmt =~ /(V[EM]|RM)(install|test)/) {
2282        $src="$src $ENV{'PBROOTDIR'}/$ENV{'PBPROJ'}.yml $ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml $ENV{'PBETC'} $ENV{'PBDESTDIR'}/pbrc.yml $pbscript{$v}";
2283    } elsif (($cmt eq "Announce") || ($cmt eq "Web") || ($cmt eq "CPAN")) {
2284        $src="$src $ENV{'PBTMP'}/pbscript";
2285    } elsif ($cmt eq "Packages") {
2286        # Get package list from file made during build2pkg
2287        open(KEEP,"$ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}") || confess "Unable to read $ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
2288        $src = <KEEP>;
2289        chomp($src);
2290        close(KEEP);
2291    }
2292
2293    $delivery->{$ENV{'PBPROJ'}} = "" if (not defined $delivery->{$ENV{'PBPROJ'}});
2294    my $repotag = "";
2295    $repotag = "-$delivery->{$ENV{'PBPROJ'}}" if ($delivery->{$ENV{'PBPROJ'}} ne "");
2296
2297    if (($cmt eq "Sources") || ($cmt eq "Packages") || ($cmt eq "CPAN")) {
2298        my ($pbpackager) = pb_conf_get("pbpackager");
2299        $ENV{'PBPACKAGER'} = $pbpackager->{$ENV{'PBPROJ'}};
2300        pb_log(0,"Exporting public key for $ENV{'PBPACKAGER'}\n");
2301        # Using pb_system is not working due to redirection below
2302        system("gpg --export -a \'$ENV{'PBPACKAGER'}\' > $ENV{'PBDESTDIR'}/$ENV{'PBPROJ'}$repotag.pubkey");
2303        chmod 0644,"$ENV{'PBDESTDIR'}/$ENV{'PBPROJ'}$repotag.pubkey";
2304        $src = "$src $ENV{'PBDESTDIR'}/$ENV{'PBPROJ'}$repotag.pubkey";
2305
2306        my ($gpgaddkey) = pb_conf_get_if("pbadditionalgpg");
2307        my $pbadditionalgpg;
2308        $pbadditionalgpg = $gpgaddkey->{$ENV{'PBPROJ'}} if (defined $gpgaddkey);
2309        if (defined $pbadditionalgpg) {
2310            foreach my $k (split(/,/,$pbadditionalgpg)) {
2311                pb_log(0,"Exporting public key $k\n");
2312                system("gpg --export -a \'$k\' > $ENV{'PBDESTDIR'}/$ENV{'PBPROJ'}$repotag-$k.pubkey");
2313                $src = "$src $ENV{'PBDESTDIR'}/$ENV{'PBPROJ'}$repotag-$k.pubkey";
2314            }
2315        }
2316    }
2317    # Remove potential leading spaces (cause problem with basename)
2318    $src =~ s/^ *//;
2319    my $basesrc = "";
2320    foreach my $i (split(/ +/,$src)) {
2321        $basesrc .= " ".basename($i);
2322    }
2323
2324    pb_log(2,"values: ".Dumper(($host,$login,$dir,$port,$tmout,$path,$conf))."\n");
2325    my ($sshhost,$sshdir) = pb_conf_get($host,$dir);
2326    # Not mandatory...
2327    my ($sshlogin,$sshport) = pb_conf_get_if($login,$port);
2328    $sshport->{$ENV{'PBPROJ'}} = 22 if (not defined $sshport->{$ENV{'PBPROJ'}});
2329    $sshlogin->{$ENV{'PBPROJ'}} = getpwuid($UID) if (not defined $sshlogin->{$ENV{'PBPROJ'}});
2330    my ($vtmout,$vepath);
2331    # ...Except those in virtual context
2332    if ($cmt =~ /^VE/) {
2333        ($vepath) = pb_conf_get($path);
2334    }
2335    if ($cmt =~ /^(V|R)M/) {
2336        $vtmout = pb_distro_get_if($pbos,$tmout);
2337    }
2338    my $remhost = $sshhost->{$ENV{'PBPROJ'}};
2339    my $remdir = $sshdir->{$ENV{'PBPROJ'}};
2340    if ($cmt =~ /^V[EM]|RM/) {
2341        # In that case our real host is in the xxhost with the OS as key, not project as above
2342        $remhost = pb_distro_get($pbos,$host);
2343    }
2344    pb_log(2,"ssh: ".Dumper(($remhost,$sshlogin,$remdir,$sshport,$vepath,$rbsconf))."\n");
2345    pb_log(2,"ssh: ".Dumper($vtmout)."\n") if (defined $vtmout);
2346
2347    my $mac;
2348    if ($cmt !~ /^VE/) {
2349        $mac = "$sshlogin->{$ENV{'PBPROJ'}}\@$remhost";
2350        # Overwrite account value if passed as parameter or forced
2351        $mac = "$ENV{'PBACCOUNT'}\@$remhost" if (defined $ENV{'PBACCOUNT'});
2352    } else {
2353        # VE
2354        $mac = $sshlogin->{$ENV{'PBPROJ'}};
2355        # Overwrite account value if passed as parameter (typically for setup2v)
2356        $mac = $ENV{'PBACCOUNT'} if (defined $ENV{'PBACCOUNT'});
2357    }
2358    pb_log(2, "DEBUG: mac: $mac\n");
2359
2360    my $tdir;
2361    my $bdir;
2362    if (($cmt eq "Sources") || ($cmt =~ /(V[EM]|RM)Script/)) {
2363        $tdir = "$remdir/$delivery->{$ENV{'PBPROJ'}}/src";
2364    } elsif ($cmt eq "CPAN") {
2365        $tdir = "$remdir";
2366    } elsif ($cmt =~ /(V[EM]|RM)(build|test|install|prep)/) {
2367        $tdir = $remdir."/$ENV{'PBPROJ'}/pbdelivery";
2368        $bdir = $remdir."/$ENV{'PBPROJ'}/pbbuild";
2369        # Remove a potential $ENV{'HOME'} as bdir should be relative to pb's home
2370        $bdir =~ s|\$ENV.+\}/||;
2371    } elsif ($cmt eq "Announce") {
2372        $tdir = "$remdir/$delivery->{$ENV{'PBPROJ'}}";
2373    } elsif ($cmt eq "Web") {
2374        $tdir = "$remdir/$delivery->{$ENV{'PBPROJ'}}";
2375    } elsif ($cmt eq "Packages") {
2376        if (($pbos->{'type'} eq "rpm") || ($pbos->{'type'} eq "pkg") || ($pbos->{'type'} eq "hpux") || ($pbos->{'type'} eq "tgz") || ($pbos->{'type'} eq "apk")) {
2377            # put packages under an arch subdir
2378            $tdir = "$remdir/$delivery->{$ENV{'PBPROJ'}}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}";
2379        } elsif (($pbos->{'type'} eq "deb") || ($pbos->{'type'} eq "ebuild")) {
2380            # No need for an arch subdir
2381            $tdir = "$remdir/$delivery->{$ENV{'PBPROJ'}}/$pbos->{'name'}/$pbos->{'version'}";
2382        } else {
2383            confess "Please teach the dev team where to deliver ($pbos->{'type'} type of packages\n";
2384        }
2385
2386        my $repodir = $tdir;
2387        $repodir =~ s|^$remdir/||;
2388        $repodir =~ s|//|/|g;
2389
2390        my ($pbrepo) = pb_conf_get("pbrepo");
2391
2392        # Script for Repository management
2393        $v = "$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}" if (not defined $v);
2394        $pbscript{$v} = "$ENV{'PBBUILDDIR'}/pbscript.$$";
2395       
2396        # That script should be part of the delivery
2397        $src = "$src $pbscript{$v}";
2398        open(PBS,"> $pbscript{$v}") || confess "Unable to create $pbscript{$v}";
2399        if ($pbos->{'type'} eq "rpm") {
2400            my $pbsha = pb_distro_get($pbos,"ossha");
2401            my ($gpgcheck) = pb_conf_get_if("pbgpgcheck");
2402            my $pbgpgcheck;
2403            $pbgpgcheck = $gpgcheck->{$ENV{'PBPROJ'}} if (defined $gpgcheck);
2404            my $pbgpgkey = "$pbrepo->{$ENV{'PBPROJ'}}/$repodir/$ENV{'PBPROJ'}$repotag.pubkey";
2405            my ($gpgaddpkey) = pb_conf_get_if("pbadditionalgpg");
2406            my $pbadditionalgpg;
2407            $pbadditionalgpg = $gpgaddpkey->{$ENV{'PBPROJ'}} if (defined $gpgaddpkey);
2408            if (defined $pbadditionalgpg) {
2409                foreach my $k (split(/,/,$pbadditionalgpg)) {
2410                    $pbgpgkey .= " $pbrepo->{$ENV{'PBPROJ'}}/$repodir/$ENV{'PBPROJ'}$repotag-$k.pubkey";
2411                }
2412            }
2413            # By default force GPG check in repo even if we support signature of packages to fail. This is a best practice
2414            $pbgpgcheck = 1 if (not defined $pbgpgcheck);
2415            # Also make a pbscript to generate dnf/yum/urpmi bases
2416            print PBS << "EOF";
2417#!/bin/bash
2418# Prepare a script to ease dnf/yum setup
2419EOF
2420            print PBS "set -x\n" if ($pbdebug gt 1);
2421            print PBS << "EOF";
2422cat > $ENV{'PBPROJ'}$repotag.repo << EOT
2423[$ENV{'PBPROJ'}$repotag]
2424name=$pbos->{'name'} $pbos->{'version'} $pbos->{'arch'} - $ENV{'PBPROJ'}$repotag Vanilla Packages
2425baseurl=$pbrepo->{$ENV{'PBPROJ'}}/$repodir
2426enabled=1
2427gpgcheck=$pbgpgcheck
2428gpgkey=$pbgpgkey
2429EOT
2430chmod 644 $ENV{'PBPROJ'}$repotag.repo
2431
2432# Clean up old repo content
2433rm -rf headers/ repodata/
2434# Create yum repo
2435if [ -x /usr/bin/yum-arch ]; then
2436    yum-arch .
2437fi
2438# Create repodata
2439createrepo -s $pbsha .
2440# Link to the key
2441if [ -s $ENV{'PBPROJ'}$repotag.pubkey ]; then
2442    # Avoiding creating empty repomd.xml.key; fails on opensuse 12.1
2443    # Signature done later
2444    (cd repodata ; ln -sf ../$ENV{'PBPROJ'}$repotag.pubkey repomd.xml.key)
2445fi
2446# SLES also looks for media.1/info.txt
2447EOF
2448            if ($pbos->{'family'} eq "md") {
2449                # For Mandriva add urpmi management
2450                print PBS << "EOF";
2451# Prepare a script to ease urpmi setup
2452cat > $ENV{'PBPROJ'}$repotag.addmedia << EOT
2453#rpm --import $pbrepo->{$ENV{'PBPROJ'}}/$repodir/$ENV{'PBPROJ'}$repotag.pubkey
2454urpmi.addmedia $ENV{'PBPROJ'}$repotag $pbrepo->{$ENV{'PBPROJ'}}/$repodir with media_info/hdlist.cz
2455EOT
2456chmod 755 $ENV{'PBPROJ'}$repotag.addmedia
2457
2458# Clean up old repo content
2459rm -f hdlist.cz synthesis.hdlist.cz
2460# Create urpmi repo
2461genhdlist2 --clean .
2462if [ \$\? -ne 0 ]; then
2463    genhdlist .
2464fi
2465(cd media_info ; ln -sf ../$ENV{'PBPROJ'}$repotag.pubkey pubkey)
2466EOF
2467            }
2468            if ($pbos->{'name'} eq "fedora") {
2469                # Extract the spec file to please Fedora maintainers :-(
2470                print PBS << "EOF";
2471for p in $basesrc; do
2472    echo \$p | grep -q 'src.rpm'
2473    if [ \$\? -eq 0 ]; then
2474        rpm2cpio \$p | cpio -ivdum --quiet '*.spec'
2475    fi
2476done
2477EOF
2478            }
2479            if ($pbos->{'family'} eq "novell") {
2480                # Add ymp scripts for one-click install on SuSE
2481                print PBS << "EOF";
2482# Prepare a script to ease SuSE one-click install
2483# Cf: http://de.opensuse.org/1-Klick-Installation/ISV
2484#
2485cat > $ENV{'PBPROJ'}$repotag.ymp << EOT
2486<?xml version="1.0" encoding="utf-8"?>
2487<!-- vim: set sw=2 ts=2 ai et: -->
2488<metapackage xmlns:os="http://opensuse.org/Standards/One_Click_Install" xmlns="http://opensuse.org/Standards/One_Click_Install">
2489    <group><!-- The group of software, typically one for project-builder.org -->
2490        <name>$ENV{'PBPROJ'} Bundle</name> <!-- Name of the software group -->
2491        <summary>Software bundle for the $ENV{'PBPROJ'}$repotag project</summary> <!--This message is shown to the user and should describe the whole bundle -->
2492        <description>This is the summary of the $ENV{'PBPROJ'}$repotag Project
2493             
2494            Details are available on a per package basis below
2495
2496        </description><!--This is also shown to the user -->
2497        <remainSubscribed>false</remainSubscribed> <!-- Don't know what it mean -->
2498        <repositories><!-- List of needed repositories -->
2499            <repository>
2500                <name>$ENV{'PBPROJ'}$repotag Repository</name> <!-- Name of the repository  -->
2501                <summary>This repository contains the $ENV{'PBPROJ'}$repotag project packages.</summary> <!-- Summary of the repository -->
2502                <description>This repository contains the $ENV{'PBPROJ'}$repotag project packages.</description><!-- This description is shown to the user -->
2503                <url>$pbrepo->{$ENV{'PBPROJ'}}/$repodir</url><!--URL of repository, which is added -->
2504            </repository>
2505        </repositories>
2506        <software><!-- A List of packages, which should be added through the one-click-installation -->
2507EOT
2508for p in $basesrc; do
2509    sum=`rpm -q --qf '%{SUMMARY}' \$p`
2510    name=`rpm -q --qf '%{NAME}' \$p`
2511    desc=`rpm -q --qf '%{description}' \$p`
2512    cat >> $ENV{'PBPROJ'}$repotag.ymp << EOT
2513            <item>
2514                <name>\$name</name><!-- Name of the package, is shown to the user and used to identify the package at the repository -->
2515                <summary>\$sum</summary> <!-- Summary of the package -->
2516                <description>\$desc</description> <!-- Description, is shown to the user -->
2517            </item>
2518EOT
2519done
2520cat >> $ENV{'PBPROJ'}$repotag.ymp << EOT
2521        </software>
2522    </group>
2523</metapackage>
2524EOT
2525chmod 644 $ENV{'PBPROJ'}$repotag.ymp
2526EOF
2527            }
2528        } elsif ($pbos->{'type'} eq "deb") {
2529            # Also make a pbscript to generate apt bases
2530            # Cf: http://www.debian.org/doc/manuals/repository-howto/repository-howto.fr.html
2531            # This dirname removes ver
2532            my $debarch = pb_get_debarch($pbos);
2533            my $projcomponent = pb_get_debpc($pbos);
2534            my $rpd = dirname("$pbrepo->{$ENV{'PBPROJ'}}/$repodir");
2535            # Remove extra . in path to fix #522
2536            $rpd =~ s|/./|/|g;
2537            print PBS << "EOF";
2538#!/bin/bash
2539# Prepare a script to ease apt setup
2540cat > $ENV{'PBPROJ'}$repotag.sources.list << EOT
2541deb $rpd $pbos->{'version'} $projcomponent
2542deb-src $rpd $pbos->{'version'} $projcomponent
2543EOT
2544chmod 644 $ENV{'PBPROJ'}$repotag.sources.list
2545
2546# Up two levels to deal with the dist dir cross versions
2547cd ..
2548mkdir -p dists/$pbos->{'version'}/$projcomponent/binary-$debarch dists/$pbos->{'version'}/$projcomponent/source dists/$pbos->{'version'}/$projcomponent/binary-all
2549
2550# Prepare a script to create apt info file
2551# Reuse twice after
2552TMPD=`mktemp -d /tmp/pb.XXXXXXXXXX` || exit 1
2553mkdir -p \$TMPD
2554cat > \$TMPD/Release << EOT
2555Archive: unstable
2556Component: $projcomponent
2557Origin: $ENV{'PBPROJ'}$repotag
2558Label: $ENV{'PBPROJ'}$repotag dev repository $pbrepo->{$ENV{'PBPROJ'}}
2559EOT
2560
2561echo "Creating Packages metadata"
2562for i in dists/$pbos->{version}/$projcomponent/binary-*; do
2563    arch=`basename \$i | sed 's/binary-//'`
2564    echo "Packages for \$arch:"
2565    dpkg-scanpackages -a\$arch $pbos->{'version'} /dev/null > dists/$pbos->{'version'}/$projcomponent/binary-\$arch/Packages
2566    gzip -9 < dists/$pbos->{'version'}/$projcomponent/binary-\$arch/Packages > dists/$pbos->{'version'}/$projcomponent/binary-\$arch/Packages.gz
2567    bzip2 -9 < dists/$pbos->{'version'}/$projcomponent/binary-\$arch/Packages > dists/$pbos->{'version'}/$projcomponent/binary-\$arch/Packages.bz2
2568done
2569echo "Creating Contents metadata"
2570apt-ftparchive contents $pbos->{'version'} | gzip -c9 > dists/$pbos->{'version'}/Contents.gz
2571echo "Creating Release metadata ($pbos->{'arch'} aka $debarch)"
2572cat \$TMPD/Release > dists/$pbos->{'version'}/$projcomponent/binary-all/Release
2573echo "Architecture: all" >> dists/$pbos->{'version'}/$projcomponent/binary-all/Release
2574cat \$TMPD/Release > dists/$pbos->{'version'}/$projcomponent/binary-$debarch/Release
2575echo "Architecture: $debarch" >> dists/$pbos->{'version'}/$projcomponent/binary-$debarch/Release
2576echo "Creating Source metadata"
2577dpkg-scansources $pbos->{'version'} /dev/null > dists/$pbos->{'version'}/$projcomponent/source/Sources
2578gzip -9 < dists/$pbos->{'version'}/$projcomponent/source/Sources > dists/$pbos->{'version'}/$projcomponent/source/Sources.gz
2579bzip2 -9 < dists/$pbos->{'version'}/$projcomponent/source/Sources > dists/$pbos->{'version'}/$projcomponent/source/Sources.bz2
2580cat \$TMPD/Release > dists/$pbos->{'version'}/$projcomponent/source/Release
2581echo "Architecture: Source" >> dists/$pbos->{'version'}/$projcomponent/source/Release
2582echo "Creating Release metadata"
2583apt-ftparchive -o APT::FTPArchive::Release::Suite=$pbos->{version} release dists/$pbos->{'version'} > dists/$pbos->{'version'}/Release
2584rm -rf \$TMPD
2585EOF
2586        } elsif ($pbos->{'type'} eq "ebuild") {
2587            # make a pbscript to generate links to latest version
2588            print PBS << "EOF";
2589#!/bin/bash
2590# Prepare a script to create correct links
2591cd $tdir
2592for p in $src; do
2593    echo \$p | grep -q '.ebuild'
2594    if [ \$\? -eq 0 ]; then
2595        j=`basename \$p`
2596        pp=`echo \$j | perl -pe 's/-[0-9\.]+-r[0-9].*\.ebuild/.ebuild/'`
2597        ln -sf \$j \$pp
2598    fi
2599done
2600EOF
2601        } elsif ($pbos->{'type'} eq "apk") {
2602            # TODO
2603        }
2604        close(PBS);
2605        chmod 0755,$pbscript{$v};
2606    } else {
2607        return;
2608    }
2609
2610    # Useless for VE
2611    my $nport = pb_get_port($sshport,$pbos,$cmt) if ($cmt !~ /^VE/);
2612
2613    # Remove a potential $ENV{'HOME'} as tdir should be relative to pb's home
2614    if ($cmt =~ /^VE/o) {
2615        $tdir =~ s|\$ENV.+\}/|| or (cluck "for $cmt: $tdir must have \$ENV{'HOME'} in it or the remainder of the code won't work since some parts use relative pathing and others will try absolute.  Use a symlink if necessary to place the real files in a different place than under your home directory" && return);
2616    } else {
2617        # process, ok if not present.
2618        $tdir =~ s|\$ENV.+\}/||;
2619    }
2620
2621    my $tm = "";
2622    if ($cmt =~ /^(V|R)M/) {
2623        $tm = "sleep $vtmout" if (defined $vtmout);
2624    }
2625
2626    # ssh communication if not VE or CPAN
2627    # should use a hash instead...
2628    my ($shcmd,$shcmdroot,$cpcmd,$cptarget,$cp2target);
2629    my $tpdir;
2630    my $tp;
2631    my $context = "$ENV{'PBTMP'}";
2632    my %tag;
2633    my $dkaccount = "";
2634    my $cmd1 = "";
2635    my $cidfile;
2636    my $homedir = "";
2637    my $origimage = undef;
2638    my $cmdverb = "verbose";
2639    if (($cmt eq "Announce") || ($cmt eq "CPAN")) {
2640        $cmdverb = undef;
2641    } elsif (defined ($v)) {
2642        $cmdverb = "verbose_\[$v\] ";
2643    }
2644    if ($cmt =~ /^VE/) {
2645        $tp = pb_path_expand($vepath->{$ENV{'PBPROJ'}});
2646        $vetype = pb_ve_get_type($vetype);
2647        if ($vetype eq "docker") {
2648            $tpdir = "/";
2649            $cidfile = "$context/ctn-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}.cid";
2650        } else {
2651            $tpdir = pb_path_expand("$tp/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}");
2652        }
2653        my $arch = pb_get_arch();
2654        if ($vetype eq "chroot") {
2655            $shcmdroot = "sudo /usr/sbin/chroot $tpdir ";
2656            $shcmd = "$shcmdroot /bin/su - $mac -c ";
2657        } elsif ($vetype eq "schroot") {
2658            $shcmd = "schroot $tp -u $mac -- ";
2659        } elsif ($vetype eq "docker") {
2660            # docker manages the storage so rely on it
2661            $shcmdroot = "";
2662            my $docrepo = pb_ve_docker_repo();
2663            $cmd1 = pb_check_req("docker",0);
2664            #my ($dockeropt) =  pb_conf_get_if("dockeropt");
2665            # pbimage is used for docker and is setup to the right name depending on the step:
2666            # TODO: This is not coded yet
2667            # step 0 : nothing at creation -> tag n-v-a (made in VE.pm)
2668            # step 1 : n-v-a + setup -> tag n-v-a-pb
2669            # step 2 : n-v-a-pb + build -> tag n-v-a-pb-pbproj
2670            # step 3 : n-v-a-pb-pbproj at use
2671            # step 4 : n-v-a-pb-pbproj-install based on #3
2672            # step 5 : n-v-a-pb-pbproj-test based on #4
2673            if ((not defined $pbimage) || ($pbimage eq "")) {
2674                # If no image name given, create a naming convention
2675                $tag{1} = "$docrepo$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
2676                $tag{2} = "$tag{1}-pb";
2677                $tag{3} = "$tag{2}-$ENV{'PBPROJ'}";
2678                $tag{4} = "$tag{2}-install";
2679                $tag{5} = "$tag{2}-test";
2680            } else {
2681                # If we were given an image name, just use it
2682                $tag{1} = $pbimage;
2683                $tag{2} = $pbimage;
2684                $tag{3} = $pbimage;
2685                $tag{4} = $pbimage;
2686                $tag{5} = $pbimage;
2687            }
2688            # If we do not create the image, then use the one we should have
2689            # TODO: we don't do anything with found !!
2690            my $found = pb_ve_docker_get_image($tag{$pbstep});
2691            if ($pbstep > 3) {
2692                # initiate these images from the last n-v-a-pb-pbproj below, removing existing ones
2693                if (defined $found) {
2694                    pb_system("$cmd1 rmi $tag{$pbstep}","Removing Docker image $tag{$pbstep} first",$cmdverb);
2695                }
2696                $origimage = "$tag{3}";
2697                $found = 0;
2698            } else {
2699                $origimage = "$tag{$pbstep}";
2700            }
2701            confess "No image $tag{$pbstep} available in Docker, please create one first\n" if ((not defined $found) && ($pbstep <= 3));
2702            # Now we use that image to do what is needed
2703            # use a dockerfile to ease the creation of next images
2704            # the context is in an empty temp dir
2705            my $tmpd = "$context/Dockerfile";
2706            open(DOCKER, "> $tmpd") || confess "Unable to create the docker file $tmpd";
2707            print DOCKER "FROM $origimage\n";
2708            print DOCKER "MAINTAINER project-builder.org aka pb\n";
2709            if ($pbstep == 1) {
2710                # setup done as root
2711                $dkaccount = "root";
2712            } else {
2713                # Rest done as pb
2714                $dkaccount = "pb";
2715            }
2716            print DOCKER "USER $dkaccount\n";
2717            my $rmatend = "";
2718            # As we are in run phase use docker run. cmd will be completed below
2719            $rmatend = "--rm" if ($pbdebug lt 1);
2720            if ($pbstep <= 2) {
2721                $shcmd = "$cmd1 build -t $tag{$pbstep+1} $context";
2722                #$shcmd = "$cmd1 build $dockeropt->{$ENV{'PBPROJ'}} -t $tag{$pbstep+1} $context";
2723            } elsif ($pbstep > 3) {
2724                $shcmd = "$cmd1 build -t $tag{$pbstep} $context ; ";
2725                $shcmd .= "$cmd1 run $rmatend --cidfile=\"$cidfile\" --user $dkaccount --name $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-$ENV{'PBPROJ'}";
2726            } else {
2727                # pbstep 3
2728                $shcmd = "$cmd1 run $rmatend --cidfile=\"$cidfile\" --user $dkaccount --name $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-$cmt";
2729            }
2730        }
2731        $shcmd = "setarch i386 $shcmd" if (($pbos->{'arch'} =~ /i[3456]86/) && ($arch eq 'x86_64'));
2732        if (($cmt =~ /VE/) && ($vetype ne "docker")) {
2733            # We need to get the home dir of the target account to deliver in the right place
2734            open(PASS,"$tpdir/etc/passwd") || confess "Unable to open $tpdir/etc/passwd: $!";
2735            while (<PASS>) {
2736                my ($c1,$c2,$c3,$c4,$c5,$c6,$c7) = split(/:/);
2737                $homedir = $c6 if ($c1 =~ /^$mac$/);
2738                pb_log(3,"Homedir: $homedir - account: $c6\n");
2739            }
2740            close(PASS);
2741            $cpcmd = "sudo /bin/cp -r ";
2742        }
2743        if (($cmt =~ /VE/) && ($vetype eq "docker")) {
2744            if ($pbstep >= 2) {
2745                # We need to get the home dir of the target account to deliver in the right place
2746                $homedir = `$cmd1 run --rm --cidfile="$cidfile" $origimage grep -E '^$dkaccount:' /etc/passwd | cut -d: -f6`;
2747                chomp($homedir);
2748                open(CID,"$cidfile") || cluck "Unable to open $cidfile";
2749                my $cid = <CID>;
2750                close(CID);
2751                unlink("$cidfile");
2752            } else {
2753                $homedir = "/";
2754            }
2755            $cpcmd = "cp -r ";
2756        }
2757        $cptarget = "$tpdir/$homedir/$tdir";
2758        if ($cmt =~ /VE(build|prep|install|test)/) {
2759            $cp2target = "$tpdir/$homedir/$bdir";
2760        }
2761        pb_log(2,"On VE using $cptarget as target dir to copy to\n");
2762    } elsif ($cmt eq "CPAN") {
2763        my $ftpput = pb_check_req("ncftpput",1);
2764        my $ftpget = pb_check_req("wget",1);
2765        my ($cpanuser,$cpanpasswd) = pb_conf_get_if("cpanuser","cpanpasswd");
2766        return if ((not defined $cpanuser) || (not defined $cpanuser->{$ENV{'PBPROJ'}}));
2767        return if ((not defined $cpanpasswd) || (not defined $cpanpasswd->{$ENV{'PBPROJ'}}));
2768        my ($cpansubdir) = pb_conf_get_if("cpansubdir");
2769        $shcmd = "$ftpget --post-data \'HIDDENNAME=".$cpanuser->{$ENV{'PBPROJ'}};
2770        $shcmd .= "&user=".$cpanuser->{$ENV{'PBPROJ'}};
2771        $shcmd .= "&password=".$cpanpasswd->{$ENV{'PBPROJ'}};
2772        $shcmd .= "&SUBMIT_pause99_add_uri_upload=\"Upload the checked files\"";
2773        $shcmd .= "&pause99_add_uri_subdirtext=".$cpansubdir->{$ENV{'PBPROJ'}} if ((defined $cpansubdir) && (defined $cpansubdir->{$ENV{'PBPROJ'}}));
2774        foreach my $s (split(/ /,$src)) {
2775            $shcmd .= "&pause99_add_uri_upload=".basename($s);
2776        }
2777        $shcmd .= "'";
2778        $cpcmd = "$ftpput $host $dir";
2779        $cptarget = "CPAN";
2780    } else {
2781        my $keyopt = "";
2782        if ($cmt !~ /Packages/) {
2783            my $keyfile = pb_ssh_get(0,$remhost,$nport);
2784            $keyopt = "-i $keyfile" if (defined $keyfile);
2785        }
2786        my $sshcmd = pb_check_req("ssh",1);
2787        my $scpcmd = pb_check_req("scp",1);
2788        $shcmd = "$sshcmd $keyopt -q -o NoHostAuthenticationForLocalhost=yes -p $nport $mac";
2789        $cpcmd = "$scpcmd $keyopt -p -o NoHostAuthenticationForLocalhost=yes -P $nport";
2790        $cptarget = "$mac:$tdir";
2791        if ($cmt =~ /^(V|R)M(build|prep)/) {
2792            $cp2target = "$mac:$bdir";
2793        }
2794    }
2795   
2796    my $logres = "";
2797    # Do not touch when just announcing
2798    if (($cmt ne "Announce") && ($cmt ne "CPAN")) {
2799        # For docker everything is done in the Dockerfile for steps < 3
2800        if (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep < 3)) {
2801            print DOCKER "RUN mkdir -p $cptarget\n";
2802            print DOCKER "RUN cd $cptarget ; for i in $basesrc; do if [ -f \$i ]; then rm -f \$i; fi; done\n";
2803        } elsif (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep >= 3)) {
2804            # Nothing to do here
2805        } else {
2806            pb_system("$shcmd \"mkdir -p $tdir ; cd $tdir ; echo \'for i in $basesrc; do if [ -f \$i ]; then rm -f \$i; fi; done ; $cmd\' | bash -e\"","Preparing $tdir on $cptarget");
2807        }
2808    } else {
2809        $logres = "> ";
2810    }
2811
2812    if ($cmt =~ /SandBox/) {
2813        # Install from sandbox mean using the result of the just passed sbx2build command
2814        # Get content saved in cms2build
2815        my ($pkg) = pb_conf_read("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml","pbpkg");
2816        my $pbextdir = pb_get_extdir();
2817        if (not defined $pkg) {
2818            cluck "Unable to get package list from $ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml";
2819            return;
2820        }
2821
2822        # We consider 2 specific packages
2823        my $vertag1 = $pkg->{"ProjectBuilder"};
2824        my $vertag2 = $pkg->{"project-builder"};
2825        # get the version of the current package - maybe different
2826        pb_log(2,"Vertag1: $vertag1\n");
2827        pb_log(2,"Vertag2: $vertag2\n");
2828        my ($pbver1,$tmp1) = split(/-/,$vertag1);
2829        my ($pbver2,$tmp2) = split(/-/,$vertag2);
2830        if ($cmt =~ /VE/) {
2831            # Copy inside the VE
2832            if ($vetype ne "docker") {
2833                copy("$ENV{'PBDESTDIR'}/ProjectBuilder-$pbver1$pbextdir.tar.gz","$tpdir/tmp");
2834                copy("$ENV{'PBDESTDIR'}/project-builder-$pbver2$pbextdir.tar.gz","$tpdir/tmp");
2835            } else {
2836                if ($pbstep <= 3) {
2837                    copy("$ENV{'PBDESTDIR'}/ProjectBuilder-$pbver1$pbextdir.tar.gz","$context");
2838                    copy("$ENV{'PBDESTDIR'}/project-builder-$pbver2$pbextdir.tar.gz","$context");
2839                    print DOCKER "COPY ProjectBuilder-$pbver1$pbextdir.tar.gz /tmp/\n";
2840                    print DOCKER "COPY project-builder-$pbver2$pbextdir.tar.gz /tmp/\n";
2841                }
2842            }
2843        } else {
2844            pb_system("$cpcmd $ENV{'PBDESTDIR'}/ProjectBuilder-$pbver1$pbextdir.tar.gz $ENV{'PBDESTDIR'}/project-builder-$pbver2$pbextdir.tar.gz $mac:/tmp","Copying local project files to VM/RM");
2845        }
2846    }
2847           
2848    pb_log(0,"Sources handled ($cmt): $src\n");
2849    # For docker everything is done in the Dockerfile
2850    if (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep < 3)) {
2851        foreach my $f (split(/ +/,$src)) {
2852            copy("$f","$context");
2853            print DOCKER "COPY ".basename($f)." $cptarget/\n";
2854        }
2855        print DOCKER "RUN cd $cptarget ; $cmd\n" if ((defined $cmd) && ($cmd ne ""));
2856    } elsif (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep >= 3)) {
2857        # Nothing to do
2858    } else {
2859        pb_system("cd $ENV{'PBBUILDDIR'} ; $cpcmd $src $cptarget 2> /dev/null","$cmt delivery in $cptarget");
2860    }
2861
2862    # For VE we need to change the owner manually
2863    if ($cmt =~ /^VE/) {
2864        if ($vetype ne "docker") {
2865            my $sudomode = pb_distro_get($pbos,"ossudoersmode");
2866            my $res = pb_system("$shcmdroot sed -i '/requiretty/d' /etc/sudoers","Removing potential requiretty in sudoers","quiet");
2867            pb_system("$shcmdroot sed '/requiretty/d' /etc/sudoers > /tmp/sudoers.new ; mv /tmp/sudoers.new $tpdir/tmp/sudoers.new ; $shcmdroot mv /tmp/sudoers.new /etc/sudoers ; $shcmdroot chown root:root /etc/sudoers ; $shcmdroot chmod $sudomode /etc/sudoers","Removing again potential requiretty in sudoers as sed -i failed") if (($res ne 0) && (-f "$tpdir/etc/sudoers"));
2868            pb_system("$shcmd \"sudo chown -R $mac $tdir\"","Adapt owner in $tdir to $mac");
2869        } else {
2870            print DOCKER "RUN sed -i '/requiretty/d' /etc/sudoers\n" if ($pbstep == 1);
2871            # Bug in at least Fedora 23 where the dir wasn't owned correctly - doesn't hurt anyway
2872            print DOCKER "RUN chown root:root /etc\n" if ($pbstep == 1);
2873            print DOCKER "RUN chmod 755 /etc\n" if ($pbstep == 1);
2874        }
2875    }
2876
2877    # Use the right script name depending on context
2878    my $pbscript2;
2879    if (($cmt =~ /^(V[EM]|RM)/) || ($cmt =~ /Packages/)){
2880        $pbscript2 = $pbscript{$v};
2881    } else {
2882        $pbscript2 = "pbscript";
2883    }
2884
2885    # It's already ready for CPAN
2886    my $shcmdbase = $shcmd;
2887    my $dkcmd;
2888    if ($cmt ne "CPAN") {
2889        if (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep < 3)) {
2890            copy("$pbscript2","$context");
2891            my $s = basename($pbscript2);
2892            print DOCKER "COPY $s $cptarget/\n";
2893            # call directly with perl to avoid text file busy error messages
2894            print DOCKER "RUN cd $cptarget ; perl ./$s ; if [ '$pbkeep' = '0' ]; then rm -f $s; fi\n";
2895            print DOCKER "RUN rm -rf $cptarget/*\n";
2896            print DOCKER "RUN chown -R $dkaccount $cptarget\n" if ($pbstep == 1);
2897            close(DOCKER);
2898        } elsif (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep >= 3)) {
2899            $dkcmd = "/bin/bash $homedir/$tdir-2/".basename($pbscript2);
2900        } else {
2901            my $s = basename($pbscript2);
2902            $shcmd .= " \"echo \'cd $tdir ; if [ -x $s ]; then ./$s; fi ;  if [ $pbkeep == 0 ]; then rm -f $s; fi\' | bash\"";
2903        }
2904    }
2905    # this is where we launch the execution
2906    if (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep >= 3)) {
2907        pb_mkdir_p("$ENV{'PBTMP'}/vebuild");
2908        my $tsrc = "";
2909        foreach my $f (split(/ +/,$src)) {
2910            copy("$f","$ENV{'PBTMP'}/vebuild");
2911            $tsrc .= " ".basename($f);
2912        }
2913        my $ts = "$ENV{'PBTMP'}/vebuild/".basename($pbscript2);
2914        open(DOCKER," > $ts") || (cluck "Unable to create $ts" && return);
2915        print DOCKER "#!/bin/bash\n";
2916        print DOCKER "# Put content from host to guest, copying to have the correct uid/gid and rights to do so\n";
2917        print DOCKER "# set -x\n";
2918        print DOCKER "for i in $tsrc; do\n";
2919        print DOCKER "  # Cleanup first to avoid rights issues for root owned files e.g.\n";
2920        print DOCKER "  rm -f $homedir/$tdir/\$i\n";
2921        print DOCKER "  cp $homedir/$tdir-2/\$i $homedir/$tdir\n";
2922        print DOCKER "done\n";
2923        print DOCKER "(cd $homedir/$tdir-2 ; ls -al)\n";
2924        print DOCKER pb_get_content($pbscript2);
2925        # Not useful for test or install
2926        if ($pbstep == 3) {
2927            print DOCKER "\n# Get content back from guest in a host dir\n";
2928            # Host dir seen as TDIR in container
2929            $cp2target = "$ENV{'PBBUILDDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
2930            pb_mkdir_p("$cp2target");
2931            chmod 0777,"$cp2target";
2932            print DOCKER "TDIR=$homedir/$bdir-2/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}\n";
2933            print DOCKER "for i in $homedir/$bdir/*; do\n";
2934            print DOCKER "  # Cleanup first to avoid rights issues for root owned files e.g.\n";
2935            print DOCKER "  j=\`basename \$i\`\n";
2936            print DOCKER "  if [ -d \$TDIR/\$j ]; then\n";
2937            print DOCKER "    rmdir --ignore-fail-on-non-empty \$TDIR/\$j\n";
2938            print DOCKER "  else\n";
2939            print DOCKER "    rm -f \$TDIR/\$j\n";
2940            print DOCKER "  fi\n";
2941            print DOCKER "  cp -r \$i \$TDIR\n";
2942            print DOCKER "done\n";
2943            print DOCKER "find \$TDIR -type f -print0 | xargs -0 chmod 644 && /bin/true\n";
2944            print DOCKER "find \$TDIR -type d -print0 | xargs -0 chmod 777 2> /dev/null && /bin/true\n";
2945            #print DOCKER "ls -al \$TDIR\n";
2946        }
2947        print DOCKER "exit 0\n";
2948        close(DOCKER);
2949        $shcmd .= " -v $ENV{'PBBUILDDIR'}:$homedir/$bdir-2 -v $ENV{'PBTMP'}/vebuild:$homedir/$tdir-2 -w $homedir/$tdir-2 $tag{$pbstep} $dkcmd";
2950    }
2951    my $ret = pb_system("$shcmd","Executing pbscript on $cptarget if needed",$cmdverb);
2952   
2953    # If target is deb family then sign Release file for Packages
2954    my $pbos2 = pb_distro_get_context($v);
2955
2956    if (($cmt =~ /Packages/) && ($pbos2->{'type'} eq "deb")) {
2957        my $debarch = pb_get_debarch($pbos2);
2958        my $projcomponent = pb_get_debpc($pbos2);
2959        for my $f ("dists/$pbos2->{'version'}/$projcomponent/binary-$debarch/Release","dists/$pbos2->{'version'}/$projcomponent/binary-all/Release","dists/$pbos2->{'version'}/$projcomponent/source/Release","dists/$pbos2->{'version'}/Release") {
2960            my $debdir = dirname($cptarget);
2961            pb_system("cd $ENV{'PBBUILDDIR'} ; $cpcmd $debdir/$f . 2> /dev/null","Getting deb $f file from $debdir");
2962            pb_sign_file("$ENV{'PBBUILDDIR'}/Release");
2963            pb_system("$cpcmd $ENV{'PBBUILDDIR'}/Release.gpg $debdir/$f.gpg 2> /dev/null","Putting deb $f.gpg file to $debdir");
2964            my ($void,$debfile) = split(/:/,$debdir);
2965            pb_system("$shcmdbase \"chmod 644 $debfile/$f.gpg\"","Set mode to 644 for $debfile/$f.gpg");
2966        }
2967    } 
2968
2969    if (($cmt =~ /Packages/) && ($pbos2->{'type'} eq "rpm")) {
2970        pb_system("cd $ENV{'PBBUILDDIR'} ; $cpcmd $cptarget/repodata/repomd.xml . 2> /dev/null","Getting repomd.xml file from $cptarget/repodata");
2971        pb_sign_file("$ENV{'PBBUILDDIR'}/repomd.xml");
2972        pb_system("$cpcmd $ENV{'PBBUILDDIR'}/repomd.xml.gpg $cptarget/repodata/repomd.xml.asc 2> /dev/null","Putting back repomd.xml.asc file to $cptarget/repodata");
2973        my ($void,$rpdir) = split(/:/,$cptarget);
2974        pb_system("$shcmdbase \"chmod 644 $rpdir/repodata/repomd.xml.asc\"","Set mode to 644 for $rpdir/repodata/repomd.xml.asc");
2975    } 
2976
2977    if ($cmt =~ /^(V[EM]|RM)build/) {
2978        # Get back info on pkg produced, compute their name and get them from the VM/RM
2979        pb_system("$cpcmd $cp2target/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'} $ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.$$ 2> /dev/null","Get package names in $cp2target");
2980        if (not -f "$ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.$$") {
2981            pb_log(0,"ERROR with $v on getting $ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.$$\n");
2982            pb_exit(-1);
2983        } else {
2984            open(KEEP,"$ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.$$") || confess "Unable to read $ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.$$";
2985            my $src = <KEEP>;
2986            chomp($src);
2987            close(KEEP);
2988            unlink("$ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.$$");
2989
2990            $src =~ s/^ *//;
2991            pb_mkdir_p("$ENV{'PBBUILDDIR'}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}");
2992            # Change pgben to make the next send2target happy
2993            my $made = "";
2994   
2995            # For VM/RM we don't want shell expansion to hapen locally but remotely
2996            my $delim = '\'';
2997            if ($cmt =~ /^VEbuild/) {
2998                # For VE we need to support shell expansion locally
2999                $delim = "";
3000            }   
3001
3002            open(KEEP,"> $ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}") || confess "Unable to write $ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
3003            foreach my $p (split(/ +/,$src)) {
3004                my $j = basename($p);
3005                pb_system("$cpcmd $cp2target/$delim$p$delim $ENV{'PBBUILDDIR'}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'} 2> /dev/null","Recovery of package $j in $ENV{'PBBUILDDIR'}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}");
3006                $made="$made $pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}/$j"; # if (($pbos->{'type'} ne "rpm") || ($j !~ /.src.rpm$/));
3007            }
3008       
3009            print KEEP "$made\n";
3010            close(KEEP);
3011            if (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep >= 3)) {
3012                pb_rm_rf("$cp2target");
3013                if ($pbdebug ge 1) {
3014                    pb_log(2,"WARNING: Container  $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-$cmt NOT removed\n");
3015                }
3016            } else {
3017                pb_system("$shcmdbase \"rm -rf $tdir $bdir\"","$cmt cleanup");
3018            }
3019
3020            # Sign packages locally
3021            pb_sign_pkgs($pbos,$made);
3022
3023            # We want to send them to the ssh account so overwrite what has been done before
3024            pb_log(2,"Before sending pkgs, vexist: $vexist, vpid: $vpid\n");
3025            pb_send2target("Packages","$ENV{'PBDESTDIR'}/pbscript.$$",$pbos->{'name'}."-".$pbos->{'version'}."-".$pbos->{'arch'},$vexist,$vpid);
3026            pb_rm_rf("$ENV{'PBBUILDDIR'}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}");
3027        }
3028    }
3029    unlink("$pbscript{$v}") if ((($cmt =~ /^(V[ME]|RM)/) || ($cmt =~ /Packages/)) && ($pbkeep eq 0) && ($cmt !~ /Script/));
3030
3031    pb_log(2,"Before halt, vexist: $vexist, vpid: $vpid\n");
3032    if ((! $vexist) && ($cmt =~ /^VM/)) {
3033        # If in setupvm then takes a snapshot just before halting
3034        if ($snapme != 0) {
3035            my ($vmmonport,$vmtype) = pb_conf_get("vmmonport","vmtype");
3036            # For monitoring control
3037            if ((($vmtype->{$ENV{'PBPROJ'}}) eq "kvm") || (($vmtype->{$ENV{'PBPROJ'}}) eq "qemu")) {
3038                eval
3039                {
3040                require Net::Telnet;
3041                Net::Telnet->import();
3042                };
3043                if ($@) {
3044                    # Net::Telnet not found
3045                    pb_log(1,"ADVISE: Install Net::Telnet to benefit from monitoring control and snapshot feature.\nWARNING: No snapshot created");
3046                } else {
3047                    my $t = new Net::Telnet (Timeout => 120, Host => "localhost", Port => $vmmonport->{$ENV{'PBPROJ'}}) || confess "Unable to dialog on the monitor";
3048                    # move to monitor mode
3049                    my @lines = $t->cmd("c");
3050                    # Create a snapshot named pb
3051                    @lines = $t->cmd("savevm pb");
3052                    # Write the new status in the VM
3053                    @lines = $t->cmd("commit all");
3054                    # End
3055                    @lines = $t->cmd("quit");
3056                }
3057            }
3058        }
3059        my $hoption = "-p";
3060        my $hpath = pb_distro_get($pbos,"ospathcmd-halt");
3061        # Solaris doesn't support -p of halt
3062        if ($pbos->{'type'} eq "pkg") {
3063            $hoption = "" ;
3064        }
3065        pb_system("$shcmdbase \"sudo $hpath $hoption \"; $tm ; echo \'if [ -d /proc/$vpid ]; then kill -9 $vpid; fi \' | bash ; sleep 10","VM $v halt (pid $vpid)");
3066    }
3067    if (($cmt =~ /^VE/) && ($snapme != 0)) {
3068        if ($vetype eq "docker") {
3069            foreach my $f (split(/ +/,$src)) {
3070                unlink("$context/$f");
3071            }
3072        } else {
3073            pb_ve_snap($pbos,$tp);
3074        }
3075    }
3076}
3077
3078sub pb_script2v {
3079
3080my $pbscript=shift;
3081my $vtype=shift;
3082my $pbstep=shift;   # Which step are we in (0: create, 1: setup pb, 2: prep pbproj, 3: use to build)
3083my $pbforce=shift;  # Force stop of VM. Default not.
3084my $snapme=shift;   # Do we have to create a snapshot. Default not.
3085my $pbimage=shift;  # Which image to use to start the VM/VE
3086
3087$pbforce = 0 if (not defined $pbforce);
3088$snapme = 0 if (not defined $snapme);
3089$pbstep = 3 if (not defined $pbstep);
3090
3091pb_log(2,"DEBUG: pb_script2v($vtype,$pbstep,$pbforce,$snapme)\n");
3092pb_log(2,"DEBUG: pb_script2v(pbscript: ".Dumper($pbscript)."\n") if (defined $pbscript);
3093pb_parallel_launchv($pbscript,$vtype,uc($vtype)."Script",$pbstep,$pbimage,$pbforce,$snapme,$pbsnap);
3094}
3095
3096sub pb_parallel_launchv {
3097
3098my $pbscript=shift;
3099my $vtype = shift;
3100my $action = shift;         # It an action is defined then use send2target
3101my $pbstep=shift;           # Which step are we in (0: create, 1: setup, 2: build 3: use 4: install 5: test)
3102my $pbimage=shift;          # Which image to use to start the VM/VE
3103my $pbforce=shift;          # Force stop of VM. Default not.
3104my $snapme = shift;         # By default do not snap a VM/VE/RM
3105my $usesnap = shift;        # By default study the usage of the snapshot feature of VM/VE/RM   
3106my $vm;
3107my $all;
3108
3109$pbforce = 0 if (not defined $pbforce);
3110$snapme = 0 if (not defined $snapme);
3111$usesnap = 1 if (not defined $usesnap);
3112
3113# Adapt // mode to memory size
3114$pbparallel = pb_set_parallel($vtype);
3115
3116pb_log(2,"DEBUG: pb_parallel_launch(vtype,step,force,snapme,usesnap: $vtype,$pbstep,$pbforce,$snapme,$usesnap)\n");
3117pb_log(2,"DEBUG: pb_parallel_launch(pbscript: ".Dumper($pbscript)."\n");
3118
3119my ($vexist,$vpid) = (undef,undef);
3120my $pm;
3121my $all_ok = 1;
3122if (defined $pbparallel) {
3123    $pm = new Parallel::ForkManager($pbparallel);
3124    $pm->set_waitpid_blocking_sleep(0);
3125
3126    # Set which port the VM/RM will use to communicate
3127    $pm->run_on_start(\&pb_set_port);
3128    $pm->run_on_finish(sub { my ($pid, $code, $id, $signal, $dump) = @_;
3129                            unless ($code == 0 && $signal == 0 && $dump == 0) {
3130                                $all_ok = 0;
3131                                pb_log(0,"ERROR: pid $pid failed\n");
3132                                pb_log(0,"code: $code - signal: $signal - dump: $dump\n");
3133                            }
3134                        });
3135}
3136
3137# Determine on which V to work
3138if (defined $ENV{'PBV'}) {
3139    @$vm = split(/,/,$ENV{'PBV'});
3140} else {
3141    ($vm,$all) = pb_get2v($vtype);
3142}
3143
3144$vetype = pb_ve_get_type($vetype);
3145if ((($vtype =~ /vm/) || (($vtype =~ /ve/) && ($vetype eq "docker"))) && (((not defined $pbimage) || ($pbimage eq "")) && ($pbstep == 0))) {
3146    confess "-i image parameter needed";
3147}
3148
3149my $counter = 0;
3150foreach my $v (@$vm) {
3151    $counter++;
3152    # Modulo 2 * pbparallel (to avoid synchronization problems)
3153    $counter = 1 if ((defined $pbparallel) && ($counter > 2 * $pbparallel));
3154    $pm->start($counter) and next if (defined $pbparallel);
3155   
3156    # If launched via script2v then get the script passed as the one for the that VM
3157    if (not defined $pbscript->{$v}) {
3158        $pbscript->{$v} = $pbscript->{'default'};
3159    }
3160    #
3161    # Launch a single operation
3162    ($vexist,$vpid) = pb_launchv($pbscript->{$v},$vtype,$v,$action,$pbstep,$pbforce,$snapme,$pbsnap,$pbimage);
3163   
3164    # Skip that VM/RM if something went wrong
3165    if (($vtype =~ /(v|r)m/) && ($vpid == 0) && ($vexist == 0)) {
3166        $pm->finish if (defined $pbparallel);
3167        next;
3168    }
3169
3170   
3171    $pm->finish if (defined $pbparallel);
3172}
3173$pm->wait_all_children if (defined $pbparallel);
3174confess "Aborting, one or more of the children failed." if ((not $all_ok) && ($Global::pb_stop_on_error));
3175}
3176
3177sub pb_launchv {
3178
3179my $pbscript1=shift;        # Just the script name to execute, not the hash
3180my $vtype = shift;
3181my $v = shift;              # Only 1 VM/VE/RM treated here
3182my $action = shift;         # It an action is defined then use send2target
3183my $pbstep=shift;           # Which step are we in (0: create, 1: setup, 2: build 3: use 4: install 5: test)
3184my $pbforce=shift;          # Force stop of VM. Default not.
3185my $snapme = shift;         # By default do not snap a VM/VE/RM
3186my $usesnap = shift;        # By default study the usage of the snapshot feature of VM/VE/RM   
3187my $pbimage=shift;          # Which image to use to start the VM/VE
3188my $vexist = undef;
3189my $vpid = undef;
3190
3191$pbforce = 0 if (not defined $pbforce);
3192$snapme = 0 if (not defined $snapme);
3193$usesnap = 1 if (not defined $usesnap);
3194
3195# Check that v exists
3196confess "No VM/VE/RM defined, unable to launch" if (not defined $v);
3197
3198# If creation or snapshot creation mode, no snapshot usable
3199if (($pbstep == 0) || ($snapme == 1)) {
3200    $usesnap = 0;
3201}
3202
3203pb_log(2,"DEBUG: pb_launchv(vtype,v,step,force,snapme,usesnap: $vtype,$v,$pbstep,$pbforce,$snapme,$usesnap)\n");
3204pb_log(2,"DEBUG: pb_launchv(pbimage: $pbimage)\n") if (defined $pbimage);
3205pb_log(2,"DEBUG: pb_launchv(pbscript $pbscript1)\n") if (defined $pbscript1);
3206pb_log(2,"DEBUG: pb_launchv(action: $action)\n") if (defined $action);
3207# Keep only the first VM in case many were given
3208if ($v =~ /,/) {
3209    pb_log(0,"WARNING: pruning to just the first of several VM/VE/RMs listed ($v)\n");
3210    $v =~ s/,.*//;
3211}
3212
3213my $pbos = pb_distro_get_context($v);
3214
3215pb_apply_conf_proxy($pbos);
3216
3217# Launch the VMs/VEs
3218if ($vtype eq "vm") {
3219    my ($ptr,$ptr2,$vmpath,$vmport) = pb_conf_get("vmtype","vmcmd","vmpath","vmport");
3220    my ($vmopt,$vmtmout,$vmsnap,$vmbuildtm,$vmmonport) = pb_conf_get_if("vmopt","vmtmout","vmsnap","vmbuildtm","vmmonport");
3221    my $vmsize = pb_distro_get($pbos,"vmsize");
3222
3223    my $vmtype = $ptr->{$ENV{'PBPROJ'}};
3224    my $vmcmd = $ptr2->{$ENV{'PBPROJ'}};
3225
3226    if (defined $opts{'g'}) {
3227        if (($vmtype eq "kvm") || ($vmtype eq "qemu")) {
3228            $ENV{'PBVMOPT'} = "--nographic";
3229        }
3230    }
3231    if (not defined $ENV{'PBVMOPT'}) {
3232        $ENV{'PBVMOPT'} = "";
3233    }
3234    # Save the current status for later restoration
3235    $ENV{'PBOLDVMOPT'} = $ENV{'PBVMOPT'};
3236
3237    if (defined $vmopt->{$v}) {
3238        $ENV{'PBVMOPT'} .= " $vmopt->{$v}" if ($ENV{'PBVMOPT'} !~ / $vmopt->{$v}/);
3239    } elsif (defined $vmopt->{$ENV{'PBPROJ'}}) {
3240        $ENV{'PBVMOPT'} .= " $vmopt->{$ENV{'PBPROJ'}}" if ($ENV{'PBVMOPT'} !~ / $vmopt->{$ENV{'PBPROJ'}}/);
3241    }
3242
3243    # How much memory to allocate for VMs
3244    my $vmmem = pb_distro_get_if($pbos,"vmmem");
3245    if (defined $vmmem) {
3246        $ENV{'PBVMOPT'} .= " -m $vmmem";
3247    }
3248
3249    # Are we allowed to use snapshot feature
3250    if ($usesnap == 1) {
3251        if ((defined $vmsnap->{$v}) && ($vmsnap->{$v} =~ /true/i)) {
3252            $ENV{'PBVMOPT'} .= " -snapshot";
3253        } elsif ((defined $vmsnap->{$ENV{'PBPROJ'}}) && ($vmsnap->{$ENV{'PBPROJ'}} =~ /true/i)) {
3254            $ENV{'PBVMOPT'} .= " -snapshot";
3255        } elsif ($pbsnap eq 1) {
3256            $ENV{'PBVMOPT'} .= " -snapshot";
3257        }
3258    } 
3259    if ($snapme != 0) {
3260        if (($vmtype eq "kvm") || ($vmtype eq "qemu")) {
3261            # Configure the monitoring to automate the creation of the 'pb' snapshot
3262            $ENV{'PBVMOPT'} .= " -serial mon:telnet::$vmmonport->{$ENV{'PBPROJ'}},server,nowait" if ((defined $vmmonport) && (defined $vmmonport->{$ENV{'PBPROJ'}}));
3263            # In that case no snapshot call needed
3264            $ENV{'PBVMOPT'} =~ s/ -snapshot//;
3265        }
3266    }
3267   
3268    # Set a default timeout of 2 minutes if none defined externally or in conf files
3269    if (not defined $ENV{'PBVMTMOUT'}) {
3270        if (defined $vmtmout->{$v}) {
3271            $ENV{'PBVMTMOUT'} = $vmtmout->{$v};
3272        } elsif (defined $vmtmout->{$ENV{'PBPROJ'}}) {
3273            $ENV{'PBVMTMOUT'} = $vmtmout->{$ENV{'PBPROJ'}};
3274        }
3275    }
3276    if (not defined $ENV{'PBVMTMOUT'}) {
3277        $ENV{'PBVMTMOUT'} = "120";
3278    }
3279
3280    my $nport = pb_get_port($vmport,$pbos,$vtype);
3281
3282    my $cmd;
3283    my $vmm;        # has to be used for pb_check_ps
3284    if (($vmtype eq "qemu") || ($vmtype eq "kvm")) {
3285        $vmm = "$vmpath->{$ENV{'PBPROJ'}}/$v.qemu";
3286        if (($pbstep == 0) || (defined $pbimage)) {
3287            $ENV{'PBVMOPT'} .= " -cdrom $pbimage -boot d";
3288        }
3289        # Always redirect the network and always try to use a 'pb' snapshot
3290        #$cmd = "$vmcmd $ENV{'PBVMOPT'} -net nic,model=virtio -net user,hostfwd=tcp::$nport:22 -loadvm pb $vmm"
3291        #$cmd = "$vmcmd $ENV{'PBVMOPT'} -redir tcp:$nport:10.0.2.15:22 $vmm"
3292        $cmd = "$vmcmd $ENV{'PBVMOPT'} -net nic -net user,hostfwd=tcp:127.0.0.1:$nport-:22 $vmm"
3293    } elsif ($vmtype eq "xen") {
3294    } elsif ($vmtype eq "vmware") {
3295    } else {
3296        confess "VM of type $vmtype not supported. Report to the dev team";
3297    }
3298    # Restore the ENV VAR Value
3299    $ENV{'PBVMOPT'} = $ENV{'PBOLDVMOPT'};
3300
3301    my ($tmpcmd,$void) = split(/ +/,$cmd);
3302    my $vmmport = undef;
3303    ($vexist,$vmmport) = pb_check_ps($tmpcmd,$vmm);
3304    $vpid = 0;
3305   
3306    if (! $vexist) {
3307        if ($pbstep == 0) {
3308            confess("Found an existing Virtual machine $vmm. Won't overwrite") if (-r $vmm);
3309            if (($vmtype eq "qemu") || ($vmtype eq "xen") || ($vmtype eq "kvm")) {
3310                my $command = pb_check_req("qemu-img",0);
3311                pb_system("$command create -f qcow2 $vmm $vmsize","Creating the QEMU VM");
3312            } elsif ($vmtype eq "vmware") {
3313            } else {
3314            }
3315        }
3316        if (! -f "$vmm") {
3317            pb_log(0,"Unable to find VM $vmm\n");
3318        } else {
3319            # Is the SSH port free? if not kill the existing process using it after a build timeout period
3320            my ($vmssh,$void) = pb_check_ps($tmpcmd,$vmm);
3321            if ($vmssh) {
3322                my $buildtm = $ENV{'PBVMTMOUT'};
3323                if (defined $vmbuildtm->{$v}) {
3324                    $buildtm = $vmbuildtm->{$v};
3325                } elsif (defined $vmbuildtm->{$ENV{'PBPROJ'}}) {
3326                    $buildtm = $vmbuildtm->{$ENV{'PBPROJ'}};
3327                }
3328
3329                sleep $buildtm;
3330                pb_log(0,"WARNING: Killing the process ($vmssh) using port $void (previous failed VM ?)\n");
3331                kill 15,$vmssh;
3332                # Let it time to exit
3333                sleep 5;
3334            }
3335            pb_system("$cmd &","Launching the VM $vmm");
3336            # Using system allows to kill it externaly if needed,sosupport that in the call
3337            pb_system("sleep $ENV{'PBVMTMOUT'}","Waiting $ENV{'PBVMTMOUT'} s for VM $v to come up","mayfail");
3338            ($vpid,$void) = pb_check_ps($tmpcmd,$vmm);
3339            pb_log(0,"VM $vmm launched (pid $vpid)\n");
3340        }
3341    } else {
3342        pb_log(0,"Found an existing VM $vmm (pid $vexist - Port $vmmport)\n");
3343        # Set the correct port here based on what is done
3344        $pbport = $vmmport;
3345        $vpid = $vexist;
3346    }
3347    pb_log(2,"DEBUG: pb_launchv returns ($vexist,$vpid)\n");
3348
3349    # After the VM is launched, then setup the SSH access if not in creation phase
3350    if ($pbstep != 0) {
3351        if (defined $pbport) {
3352            pb_ssh_setup($vtype,$pbport);
3353        } else {
3354            pb_ssh_setup($vtype,$nport);
3355        }
3356    }
3357   
3358    # Skip that VM if something went wrong
3359    return($vexist,$vpid) if (($vpid == 0) && ($vexist == 0));
3360   
3361    # If force stopping the VM then reset vexist
3362    if ($pbforce == 1) {
3363        $vpid = $vexist;
3364        $vexist = 0;
3365    }
3366} elsif ($vtype eq "ve") {
3367    # Force the creation of the VE and no snapshot usable
3368    $vetype = pb_ve_get_type($vetype);
3369    pb_ve_launch($v,$pbstep,$usesnap,$vetype,$pbimage);
3370    $vexist = 0;
3371    $vpid = 0;
3372} else {
3373    # Get RM context
3374    my ($ptr,$rmpath,$rmport) = pb_conf_get("rmtype","rmpath","rmport");
3375
3376    # Nothing more to do for RM. No real launch
3377    # For the moment we support the RM is already running
3378    # For ProLiant may be able to power them on if needed later on as an example.
3379    $vexist = 0;
3380    $vpid = 0;
3381   
3382    # After the RM is launched, then setup the SSH access
3383    my $nport = pb_get_port($rmport,$pbos,$vtype);
3384    pb_ssh_setup($vtype,$nport);
3385}
3386
3387# Gather all required files to send them to the VM/VE/RM
3388# and launch the build through pbscript
3389if (defined $action) {
3390    pb_log(2,"DEBUG: Before send2target, vexist: $vexist, vpid: $vpid, vtype: $vtype, v: $v, action: $action\n");
3391    pb_send2target($action,$pbscript1,$v,$vexist,$vpid,$snapme,$pbstep);
3392}
3393return($vexist,$vpid);
3394}
3395
3396# Return string for date synchro
3397sub pb_date2v {
3398
3399my $vtype = shift;
3400my $pbos = shift;
3401my $nontp = shift || 0;
3402
3403pb_log(1,"Entering pb_date2v\n");
3404# VE gets time from parent OS.
3405return "/bin/true" if ($vtype) =~ /^ve/o; 
3406
3407my ($ntp) = pb_conf_get_if($vtype."ntp");
3408my $vntp = undef;
3409my $ntpline = undef;
3410$vntp = $ntp->{$ENV{'PBPROJ'}} if (defined $ntp);
3411
3412if ((defined $vntp) && ($vntp !~ /^\s*$/)) {
3413    pb_log(2,"ntp server is $vntp\n");
3414    # ntp command depends on pbos
3415    my $vntpcmd = pb_distro_get($pbos,"oscmdntp");
3416    $ntpline = "sudo $vntpcmd $vntp";
3417}
3418# Force new date to be in the future compared to the date
3419# of the host by adding 1 minute
3420my @date=pb_get_date();
3421$date[1]++;
3422my $upddate = strftime("%m%d%H%M%Y", @date);
3423my $dateline = "sudo /bin/date $upddate";
3424if ((defined $ntpline) && ($nontp == 0)) {
3425    pb_log(1,"pb_date2v returns $ntpline\n");
3426    return($ntpline);
3427} else {
3428    pb_log(1,"pb_date2v returns $dateline\n");
3429    return($dateline);
3430}
3431}
3432
3433sub pb_build2v {
3434
3435my $vtype = shift;
3436my $action = shift;     #build, test or prep
3437my $pbforce=shift;      # Force stop of VM. Default not.
3438
3439$action = "build" if (not defined $action);
3440$pbforce = 0 if (not defined $pbforce);
3441
3442my $pbstep;
3443
3444$vetype = pb_ve_get_type($vetype);
3445if ($action eq "prep") {
3446    $pbstep = 2;
3447} elsif ($action eq "install") {
3448    $pbstep = 4;
3449} elsif ($action eq "test") {
3450    $pbstep = 5;
3451} elsif ($action eq "build") {
3452    $pbstep = 3;
3453    # Need to be traversed by others if using containers
3454    if (($vtype eq "ve") && ($vetype eq "docker")) {
3455        chmod 0751,"$ENV{'PBBUILDDIR'}";
3456    }
3457} else {
3458    pb_log(0,"pb_build2v: unknown action $action\n");
3459    confess "Unable to continue";
3460}
3461
3462pb_log(1,"Entering in pb_build2v ($action in $vtype) \n");
3463
3464my ($v,$all) = ("", 0);
3465# Determine on which V to work
3466if (defined $ENV{'PBV'}) {
3467    @$v = split(/,/,$ENV{'PBV'});
3468} else {
3469    ($v,$all) = pb_get2v($vtype);
3470}
3471
3472# Send tar files when we do a global generation
3473pb_build2ssh() if (($all == 1) && ($action eq "build"));
3474
3475# Adapt // mode to memory size
3476$pbparallel = pb_set_parallel($vtype);
3477
3478my ($vexist,$vpid) = (undef,undef);
3479my $pm;
3480my $all_ok = 1;
3481if (defined $pbparallel) {
3482    $pm = new Parallel::ForkManager($pbparallel);
3483    $pm->set_waitpid_blocking_sleep(0);
3484
3485    # Set which port the VM/RM will use to communicate
3486    $pm->run_on_start(\&pb_set_port);
3487    $pm->run_on_finish(sub { my ($pid, $code, $id, $signal, $dump) = @_;
3488                            unless ($code == 0 && $signal == 0 && $dump == 0) {
3489                                $all_ok = 0;
3490                                pb_log(0,"ERROR: pid $pid failed\n");
3491                                pb_log(0,"code: $code - signal: $signal - dump: $dump\n");
3492                            }
3493                        });
3494}
3495
3496my $counter = 0;
3497foreach my $v (@$v) {
3498    # Check that this VE/VM is in the list of systems for which we build
3499    if (($action eq "build") || ($action eq "prep")) {
3500        pb_check_v($v,$vtype);
3501    }
3502    $counter++;
3503    $pbscript{$v} = "$ENV{'PBDESTDIR'}/pb$action-$v";
3504    # Modulo 2 * pbparallel (to avoid synchronization problems)
3505    $counter = 1 if ((defined $pbparallel) && ($counter > 2 * $pbparallel));
3506    $pm->start($counter) and next if (defined $pbparallel);
3507    #
3508    # Prepare the script to be executed on the VM/VE/RM
3509    open(SCRIPT,"> $pbscript{$v}") || confess "Unable to create $pbscript{$v}";
3510    print SCRIPT "#!/bin/bash\n";
3511
3512    # Transmit the verbosity level to the virtual env/mach.
3513    my $verbose = "";
3514    my $i = 0;                          # minimal debug level
3515    while ($i lt $pbdebug) {
3516        $verbose .= "-v ";
3517        $i++;
3518    }
3519    print SCRIPT "set -e\n" if ($Global::pb_stop_on_error);
3520    # Activate script verbosity if at least 2 for pbdebug
3521    print SCRIPT "set -x\n" if ($i gt 1);
3522    # Quiet if asked to be so on the original system
3523    $verbose = "-q" if ($pbdebug eq -1);
3524
3525    print SCRIPT "echo ... Execution needed\n";
3526    print SCRIPT "echo ==== Start of script for $vtype $v =====\n";
3527    print SCRIPT "# This is in directory pbdelivery\n";
3528    print SCRIPT "# Setup the variables required for building\n";
3529    print SCRIPT "export PBPROJ=$ENV{'PBPROJ'}\n";
3530
3531    if (($action eq "build") || ($action eq "prep")) {
3532        print SCRIPT "# Preparation for pb\n";
3533        print SCRIPT "rm -f \$HOME/.pbrc.yml\n";
3534        print SCRIPT "rm -f \$HOME/.pbrc\n";
3535        print SCRIPT "cp .pbrc.yml \$HOME\n";
3536        # TODO: Why ?
3537        print SCRIPT "cd ..\n";
3538    }
3539
3540    # VE needs a good /proc, tolerate one being potentially left around after a failure
3541    if (($vtype eq "ve") && ($vetype ne "docker")) {
3542        print SCRIPT "[ -d /proc/1 ] || sudo /bin/mount -t proc /proc /proc\n";
3543    }
3544
3545    # Get distro context
3546    my $pbos = pb_distro_get_context($v);
3547   
3548    my $ntpline = pb_date2v($vtype,$pbos);
3549    print SCRIPT "# Time sync\n";
3550    print SCRIPT "echo setting up date with $ntpline\n";
3551    print SCRIPT "$ntpline\n";
3552    print SCRIPT "if [ \$\? -ne 0 ]; then\n";
3553    # Force if bad status a trick using date
3554    $ntpline = pb_date2v($vtype,$pbos, 1);
3555    print SCRIPT "  $ntpline\n";
3556    print SCRIPT "fi\n";
3557    # Use potential local proxy declaration in case we need it to download repo, pkgs, ...
3558    if (defined $ENV{'http_proxy'}) {
3559        print SCRIPT "export http_proxy=\"$ENV{'http_proxy'}\"\n";
3560    }
3561    if (defined $ENV{'https_proxy'}) {
3562        print SCRIPT "export https_proxy=\"$ENV{'https_proxy'}\"\n";
3563    }
3564    if (defined $ENV{'ftp_proxy'}) {
3565        print SCRIPT "export ftp_proxy=\"$ENV{'ftp_proxy'}\"\n";
3566    }
3567
3568    # Get list of packages to build/test and get some ENV vars as well
3569    my $ptr = pb_get_pkg();
3570    @pkgs = @$ptr;
3571    my $p = join(' ',@pkgs) if (@pkgs);
3572    print SCRIPT "export PBPROJVER=$ENV{'PBPROJVER'}\n";
3573    print SCRIPT "export PBPROJTAG=$ENV{'PBPROJTAG'}\n";
3574    print SCRIPT "export PBPACKAGER=\"$ENV{'PBPACKAGER'}\"\n";
3575    # TODO: setup apk pkg/keys here
3576
3577    # We may need to do some other tasks before building. Read a script here to finish setup
3578    if (-x "$ENV{'PBTMP'}/pb$vtype".".pre") {
3579        print SCRIPT "# Special pre-instructions to be launched\n";
3580        print SCRIPT pb_get_content("$ENV{'PBTMP'}/pb$vtype".".pre");
3581    }
3582
3583    if (-x "$ENV{'PBTMP'}/pb$vtype"."$action.pre") {
3584        print SCRIPT "# Special pre-$action instructions to be launched\n";
3585        print SCRIPT pb_get_content("$ENV{'PBTMP'}/pb$vtype"."$action.pre");
3586    }
3587
3588    print SCRIPT "# $action\n";
3589    print SCRIPT "echo $action"."ing packages on $vtype...\n";
3590
3591    print SCRIPT "pb --version\n";
3592    my $act = "$action"."2pkg";
3593    $act = "build2prep" if ($action eq "prep");
3594    print SCRIPT "pb $verbose -p $ENV{'PBPROJ'} $act $p\n";
3595
3596    if (($vtype eq "ve") && ($vetype ne "docker")) {
3597        print SCRIPT "sudo /bin/umount /proc\n";
3598    }
3599
3600    # We may need to do some other tasks after building. Read a script here to exit properly
3601    if (-x "$ENV{'PBTMP'}/pb$vtype"."$action.post") {
3602        print SCRIPT "# Special post-$action instructions to be launched\n";
3603        print SCRIPT pb_get_content("$ENV{'PBTMP'}/pb$vtype"."$action.post");
3604    }
3605
3606    if (-x "$ENV{'PBTMP'}/pb$vtype".".post") {
3607        print SCRIPT "# Special post-instructions to be launched\n";
3608        print SCRIPT pb_get_content("$ENV{'PBTMP'}/pb$vtype".".post");
3609    }
3610
3611    print SCRIPT "echo ==== End of script $$ for $vtype $v =====\n";
3612    close(SCRIPT);
3613    chmod 0755,"$pbscript{$v}";
3614   
3615    $pm->finish if (defined $pbparallel);
3616}
3617$pm->wait_all_children if (defined $pbparallel);
3618confess "Aborting, one or more of the children failed." if ((not $all_ok) && ($Global::pb_stop_on_error));
3619
3620# Launch the VM/VE/RM
3621# docker image to use if needed
3622# Gather all required files to send them to the VM/VE
3623# and launch the build through pbscript
3624pb_parallel_launchv(\%pbscript,$vtype,uc($vtype).$action,$pbstep,$pbimage,$pbforce);
3625}
3626
3627
3628sub pb_clean {
3629
3630    my $sleep=10;
3631    confess "Unable to get env var PBDESTDIR" if (not defined $ENV{'PBDESTDIR'});
3632    confess "Unable to get env var PBBUILDDIR" if (not defined $ENV{'PBBUILDDIR'});
3633    pb_log(0,"We will now wait $sleep s before removing both directories\n$ENV{'PBDESTDIR'} and $ENV{'PBBUILDDIR'}.\nPlease break me if this is wrong\n");
3634    sleep $sleep;
3635    pb_rm_rf($ENV{'PBDESTDIR'});
3636    pb_rm_rf($ENV{'PBBUILDDIR'});
3637}
3638
3639sub pb_newver {
3640
3641    confess "-V Version parameter needed" if ((not defined $newver) || ($newver eq ""));
3642
3643    # Need this call for PBDIR
3644    my ($scheme2,$uri) = pb_vcs_init($pbinit);
3645
3646    my ($pbconf,$pbprojurl) = pb_conf_get("pbconfurl","pbprojurl");
3647    $uri = $pbconf->{$ENV{'PBPROJ'}};
3648    my ($scheme, $account, $host, $port, $path) = pb_get_uri($uri);
3649
3650    # Checking CMS repositories status
3651    ($scheme2, $account, $host, $port, $path) = pb_get_uri($pbprojurl->{$ENV{'PBPROJ'}});
3652
3653    if ($scheme !~ /^svn/) {
3654        confess "Only SVN is supported at the moment";
3655    }
3656
3657    my $res = pb_vcs_isdiff($scheme,$ENV{'PBROOTDIR'});
3658    confess "ERROR: No differences accepted in CMS for $ENV{'PBROOTDIR'} before creating a new version" if ($res != 0);
3659
3660    $res = pb_vcs_isdiff($scheme2,$ENV{'PBDIR'});
3661    confess "ERROR: No differences accepted in CMS for $ENV{'PBDIR'} before creating a new version" if ($res != 0);
3662
3663    # Tree identical between PBCONFDIR and PBROOTDIR. The delta is what
3664    # we want to get for the root of the new URL
3665
3666    my $oldver = $ENV{'PBROOTDIR'};
3667    $oldver =~ s|^$ENV{'PBCONFDIR'}||;
3668
3669    pb_log(2, "PBCONFDIR: $ENV{'PBCONFDIR'}\nPBROOTDIR: $ENV{'PBROOTDIR'}\n");
3670
3671    my $newurl = "$uri/$newver";
3672    # Should probably use projver in the old file
3673    my $oldvertxt= basename($oldver);
3674    my $newvertxt = basename($newver);
3675
3676    # Duplicate and extract project-builder part
3677    pb_log(2,"Copying $uri/$oldver to $newurl\n");
3678    pb_vcs_copy($scheme,"$uri/$oldver",$newurl);
3679    pb_log(2,"Checkout $newurl to $ENV{'PBCONFDIR'}/$newver\n");
3680    pb_vcs_up($scheme,"$ENV{'PBCONFDIR'}");
3681
3682    # Duplicate and extract project
3683    my $newurl2 = "$pbprojurl->{$ENV{'PBPROJ'}}/$newver";
3684
3685    pb_log(2,"Copying $pbprojurl->{$ENV{'PBPROJ'}}/$oldver to $newurl2\n");
3686    pb_vcs_copy($scheme2,"$pbprojurl->{$ENV{'PBPROJ'}}/$oldver",$newurl2);
3687
3688    my $tmp = $ENV{'PBDIR'};
3689    $tmp =~ s|$oldver$||;
3690    pb_log(2,"Checkout $newurl2 to $tmp/$newver\n");
3691    pb_vcs_up($scheme2,"$tmp");
3692
3693    # Update the .yml file
3694    open(FILE,"$ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml") || confess "Unable to open $ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml";
3695    open(OUT,"> $ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml.new") || confess "Unable to write to $ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml.new";
3696    while(<FILE>) {
3697        if (/^projver\s+$ENV{'PBPROJ'}\s*=\s*$oldvertxt$/) {
3698            s/^projver\s+$ENV{'PBPROJ'}\s*=\s*$oldvertxt$/projver $ENV{'PBPROJ'} = $newvertxt/;
3699            pb_log(0,"Changing projver from $oldvertxt to $newvertxt in $ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.pb\n");
3700        }
3701        if (/^projver:\n\s+$ENV{'PBPROJ'}:\s*$oldvertxt$/) {
3702            s/^projver:\n\s+$ENV{'PBPROJ'}:\s*$oldvertxt$/projver:\n  $ENV{'PBPROJ'}: $newvertxt/;
3703            pb_log(0,"Changing projver from $oldvertxt to $newvertxt in $ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml\n");
3704        }
3705        if (/^testver/) {
3706            s/^testver:\n\s+$ENV{'PBPROJ'}:(.*)$/#testver:\n#   $ENV{'PBPROJ'}: $1/;
3707            pb_log(0,"Commenting testver in $ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml\n");
3708            my $txt = $_;
3709            chomp($txt);
3710        }
3711        if (/^delivery/) {
3712            my $txt = $_;
3713            chomp($txt);
3714            pb_log(0,"Please check delivery ($txt) in $ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml\n");
3715        }
3716        print OUT $_;
3717    }
3718    close(FILE);
3719    close(OUT);
3720    rename("$ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml.new","$ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml");
3721
3722    # Checking pbcl files
3723    foreach my $f (<$ENV{'PBROOTDIR'}/*/pbcl>) {
3724        # Compute new pbcl file
3725        my $f2 = $f;
3726        $f2 =~ s|$ENV{'PBROOTDIR'}|$ENV{'PBCONFDIR'}/$newver/|;
3727        open(PBCL,$f) || confess "Unable to open $f";
3728        my $foundnew = 0;
3729        while (<PBCL>) {
3730            $foundnew = 1 if (/^$newvertxt \(/);
3731        }
3732        close(PBCL);
3733        open(OUT,"> $f2") || confess "Unable to write to $f2: $!";
3734        open(PBCL,$f) || confess "Unable to open $f";
3735        # We skip as long as we do not reach the new version in the log (case of devel)
3736        my $skip = 1;
3737        # We let the 4 first lines
3738        my $cnt = 0;
3739        while (<PBCL>) {
3740            $skip = 0 if (/^$newvertxt \(/);
3741            if ((/^$oldvertxt \(/) && ($foundnew == 0)) {
3742                print OUT "$newvertxt ($pbdate)\n";
3743                print OUT "- TBD\n";
3744                print OUT "\n";
3745                pb_log(0,"WARNING: version $newvertxt not found in $f so added to $f2...\n") if ($foundnew == 0);
3746            }
3747            if ($cnt <= 3) {
3748                print OUT "$_" if (not /^$oldvertxt \(/);
3749            } else {
3750                print OUT "$_" if ($skip == 0);
3751            }
3752            $cnt++;
3753        }
3754        close(OUT);
3755        close(PBCL);
3756    }
3757
3758    pb_log(2,"Checkin $ENV{'PBCONFDIR'}/$newver\n");
3759    pb_vcs_checkin($scheme,"$ENV{'PBCONFDIR'}/$newver","updated to $ENV{'PBCONFDIR'}");
3760}
3761
3762# Return the list of all VMs/VEs/RMs we are working on through the content of v(e|m)list
3763# in conf file
3764
3765sub pb_get_allv {
3766
3767my $vtype = shift;
3768my $vlist = $vtype."list";
3769
3770my ($ptr) = pb_conf_get($vlist);
3771my @v = split(/,/,$ptr->{$ENV{'PBPROJ'}});
3772return(\@v);
3773}
3774
3775#
3776# Return the list of VMs/VEs/RMs we are working on
3777# $all is a flag to know if we return all of them
3778# or only some (if all we publish also tar files in addition to pkgs)
3779#
3780sub pb_get2v {
3781
3782my $vtype = shift;
3783my @v;
3784my $all = 0;
3785my $pbv = 'PBV';
3786my $vlist = $vtype."list";
3787
3788# Get VM/VE list
3789if ((not defined $ENV{$pbv}) || ($ENV{$pbv} =~ /^all$/)) {
3790    my ($ptr) = pb_conf_get($vlist);
3791    $ENV{$pbv} = $ptr->{$ENV{'PBPROJ'}};
3792    $all = 1;
3793}
3794pb_log(2,"$vtype: $ENV{$pbv}\n");
3795@v = split(/,/,$ENV{$pbv});
3796return(\@v,$all);
3797}
3798# This function creates a giant script to configure a particular VM/VE/RM, it then copies the
3799# script to the target.
3800
3801# Function to create a potentialy missing pb account on the VM/VE/RM, and adds it to sudo
3802# Needs to use root account to connect to the VM/VE/RM
3803# pb will take your local public SSH key to access
3804# the pb account in the VM/VE/RM later on if needed
3805sub pb_setup2v {
3806
3807my $vtype = shift;
3808my $sbx = shift;
3809my $installtype = shift;        # either "file" or "pkg" to overwrite conf file
3810my $pbstep = 1;
3811
3812my ($vm,$all) = pb_get2v($vtype);
3813
3814# Adapt // mode to memory size
3815$pbparallel = pb_set_parallel($vtype);
3816
3817my $pm;
3818my $all_ok = 1;
3819if (defined $pbparallel) {
3820    $pm = new Parallel::ForkManager($pbparallel);
3821    $pm->set_waitpid_blocking_sleep(0);
3822
3823    # Set which port the VM/RM will use to communicate
3824    $pm->run_on_start(\&pb_set_port);
3825    $pm->run_on_finish(sub { my ($pid, $code, $id, $signal, $dump) = @_;
3826                            $all_ok = 0 unless (($code == 0) && ($signal == 0) && ($dump == 0)); 
3827                        });
3828}
3829
3830my $counter = 0;
3831foreach my $v (@$vm) {
3832    # Verify that this V is in our list
3833    pb_check_v($v,$vtype);
3834   
3835    # Script generated - should be before the fork - after the %pbscript hash isn't updated correctly
3836    $pbscript{$v} = "$ENV{'PBDESTDIR'}/setupv-$v";
3837
3838    $counter++;
3839    if (defined $pbparallel) {
3840        # Modulo 2 * pbparallel (to avoid synchronization problems)
3841        $counter = 1 if ($counter > 2 * $pbparallel);
3842        $pm->start($counter) and next;
3843    }
3844
3845    # Get distro context
3846    my $pbos = pb_distro_get_context($v);
3847   
3848    # Deal with date sync.
3849    my $ntpline = pb_date2v($vtype,$pbos);
3850
3851    # Name of the account to deal with for VM/VE/RM
3852    # Do not use the one passed potentially with -a
3853    my ($pbac) = pb_conf_get($vtype."login");
3854    if ((not defined $pbac) || (not defined $pbac->{$ENV{'PBPROJ'}})) {
3855        cluck "No $vtype"."login defined in your project yml conf file";
3856        next;
3857    }
3858
3859    my $key = "";
3860
3861    # Prepare the script to be executed on the VM/VE/RM
3862    # in $ENV{'PBDESTDIR'}/setupv
3863
3864    pb_log(2,"Preparing setup script for $v in $pbscript{$v}\n");
3865    open(SCRIPT,"> $pbscript{$v}") || confess "Unable to create $pbscript{$v}";
3866   
3867    print SCRIPT << 'EOF';
3868#!/usr/bin/perl -w
3869
3870use strict;
3871use File::Copy;
3872
3873# We should not need in this script more functions than what is provided
3874# by Base, Conf and Distribution to avoid problems at exec time.
3875# # We also need some YAML interpreter
3876# They are appended at the end.
3877
3878# Define mandatory global vars
3879EOF
3880    my $ppref = "our";
3881    $ppref = "my" if (($pbos->{'name'} =~ /^redhat/) && ($pbos->{'version'} =~ /^6/));
3882    print SCRIPT << "EOF";
3883$ppref \$pbdebug = $pbdebug;
3884$ppref \$pbLOG;
3885$ppref \$pbsynmsg = "pbscript";
3886$ppref \$pbdisplaytype = "text";
3887$ppref \$pblocale = "";
3888$ppref \$PBCONFVER = $projectbuilderconfver;
3889\$ENV{'PBSETUPV'} = 1;
3890EOF
3891    print SCRIPT << 'EOF';
3892pb_log_init($pbdebug, $pbLOG);
3893EOF
3894    print SCRIPT << "EOF";
3895\$Global::pb_stop_on_error = $Global::pb_stop_on_error;
3896pb_temp_init($pbkeep);
3897pb_conf_init("$ENV{'PBPROJ'}");
3898
3899EOF
3900
3901    if ($vtype eq "ve") {
3902        $vetype = pb_ve_get_type($vetype);
3903        if ($vetype ne "docker") {
3904            print SCRIPT << "EOF";
3905# For VE we first need to mount some FS
3906pb_system("mount -t proc /proc /proc") unless (-d "/proc/$$");
3907EOF
3908        }
3909        print SCRIPT << "EOF";
3910# For VE we need a good null dev
3911# Except for RedHat 6.2 where it's good and doesn't like being recreated
3912EOF
3913        print SCRIPT "pb_system('rm -f /dev/null; mknod /dev/null c 1 3; chmod 777 /dev/null');\n" unless (($pbos->{'name'} =~ /^redhat/) && ($pbos->{'version'} =~ /^6/));
3914    } elsif ($vtype =~ /(v|r)m/) {
3915        # Nothing to do now.
3916        my $useless = 0;
3917    } else {
3918        confess "Unknown virtual type $vtype";
3919    }
3920
3921    print SCRIPT << 'EOF';
3922
3923# Adds $pbac->{$ENV{'PBPROJ'}} as an account if needed
3924#
3925my $file="/etc/passwd";
3926open(PBFILE,$file) || die "Unable to open $file";
3927my $found = 0;
3928while (<PBFILE>) {
3929EOF
3930    print SCRIPT << "EOF";
3931    \$found = 1 if (/^$pbac->{$ENV{'PBPROJ'}}:/);
3932EOF
3933
3934# TODO: use an external parameter
3935my $home = "/home";
3936# Solaris doesn't like that we use /home
3937$home = "/export/home" if ($pbos->{'type'} eq "pkg");
3938
3939    print SCRIPT << "EOF";
3940}
3941close(PBFILE);
3942
3943if ( \$found == 0 ) {
3944    if ( ! -d "$home" ) {
3945        pb_mkdir_p("$home");
3946    }
3947EOF
3948    # TODO: Level of portability of these cmds ? Critical now for RM
3949    # TODO: use an external parameter
3950    print SCRIPT << "EOF";
3951    # Most Linux distros
3952    if ( -x "/usr/sbin/groupadd" ) {
3953        pb_system("/usr/sbin/groupadd $pbac->{$ENV{'PBPROJ'}}","Adding group $pbac->{$ENV{'PBPROJ'}}");
3954    # Alpine with busybox
3955    } elsif ( -x "/usr/sbin/addgroup" ) {
3956        pb_system("/usr/sbin/addgroup $pbac->{$ENV{'PBPROJ'}}","Adding group $pbac->{$ENV{'PBPROJ'}}");
3957    } else {
3958        die "Unable to find a command to add a group on this distribution";
3959    }
3960    # Most Linux distros
3961    if ( -x "/usr/sbin/useradd" ) {
3962        pb_system("/usr/sbin/useradd -g $pbac->{$ENV{'PBPROJ'}} -m -d $home/$pbac->{$ENV{'PBPROJ'}} $pbac->{$ENV{'PBPROJ'}}","Adding user $pbac->{$ENV{'PBPROJ'}} (group $pbac->{$ENV{'PBPROJ'}} - home $home/$pbac->{$ENV{'PBPROJ'}})");
3963    # Alpine with busybox
3964    } elsif ( -x "/usr/sbin/adduser" ) {
3965        pb_system("/usr/sbin/adduser -G $pbac->{$ENV{'PBPROJ'}} -h $home/$pbac->{$ENV{'PBPROJ'}} -D $pbac->{$ENV{'PBPROJ'}}","Adding user $pbac->{$ENV{'PBPROJ'}} (group $pbac->{$ENV{'PBPROJ'}} - home $home/$pbac->{$ENV{'PBPROJ'}})");
3966        pb_system("/usr/sbin/addgroup $pbac->{$ENV{'PBPROJ'}} abuild","Adding $pbac->{$ENV{'PBPROJ'}} to group abuild");
3967    } else {
3968        die "Unable to find a command to add a user on this distribution";
3969    }
3970}
3971EOF
3972
3973    # Copy the content of our local conf file to the VM/VE/RM
3974    my $content = pb_get_content(pb_distro_conffile());
3975    print SCRIPT << "EOF";
3976    #
3977    # Create a temporary local conf file for distribution support
3978    # This is created here before its use later. Its place is hardcoded, so no choice for the path
3979    #
3980    my \$tempconf = pb_distro_conffile();
3981    pb_mkdir_p(dirname(\$tempconf));
3982    open(CONF,"> \$tempconf") || die "Unable to create \$tempconf";
3983    print CONF q{$content};
3984    close(CONF);
3985EOF
3986
3987    if ($vtype =~ /(v|r)m/) {
3988        print SCRIPT << "EOF";
3989# allow ssh entry to build
3990#
3991mkdir "$home/$pbac->{$ENV{'PBPROJ'}}/.ssh",0700;
3992# Allow those accessing root to access the build account
3993copy("\$ENV{'HOME'}/.ssh/authorized_keys","$home/$pbac->{$ENV{'PBPROJ'}}/.ssh/authorized_keys");
3994chmod 0600,".ssh/authorized_keys";
3995pb_system("chown -R $pbac->{$ENV{'PBPROJ'}}:$pbac->{$ENV{'PBPROJ'}} $home/$pbac->{$ENV{'PBPROJ'}}","Finish setting up the account env for $pbac->{$ENV{'PBPROJ'}}");
3996
3997EOF
3998}
3999    print SCRIPT << 'EOF';
4000# No passwd for build account only keys
4001$file="/etc/shadow";
4002if (-f $file) {
4003    open(PBFILE,$file) || die "Unable to open $file";
4004    open(PBOUT,"> $file.new") || die "Unable to open $file.new";
4005    while (<PBFILE>) {
4006EOF
4007    print SCRIPT << "EOF";
4008        s/^$pbac->{$ENV{'PBPROJ'}}:\!\!:/$pbac->{$ENV{'PBPROJ'}}:*:/;
4009        s/^$pbac->{$ENV{'PBPROJ'}}:\!:/$pbac->{$ENV{'PBPROJ'}}:*:/; #SLES 9 e.g.
4010        s/^$pbac->{$ENV{'PBPROJ'}}:\\*LK\\*:/$pbac->{$ENV{'PBPROJ'}}:NP:/;  #Solaris e.g.
4011EOF
4012        print SCRIPT << 'EOF';
4013        print PBOUT $_;
4014    }
4015    close(PBFILE);
4016    close(PBOUT);
4017    rename("$file.new",$file);
4018    chmod 0640,$file;
4019    }
4020
4021# TODO: systemd support
4022# Keep the VM in text mode
4023$file="/etc/inittab";
4024if (-f $file) {
4025    open(PBFILE,$file) || die "Unable to open $file";
4026    open(PBOUT,"> $file.new") || die "Unable to open $file.new";
4027    while (<PBFILE>) {
4028        s/^(..):5:initdefault:$/$1:3:initdefault:/;
4029        print PBOUT $_;
4030    }
4031    close(PBFILE);
4032    close(PBOUT);
4033    rename("$file.new",$file);
4034    chmod 0640,$file;
4035}
4036
4037# pb has to be added to portage group on gentoo
4038
4039# We need to have that pb_distro_get_context function
4040# Get it from ProjectBuilder::Distribution
4041# And we now need the conf file required for this to work created above
4042
4043my $pbos = pb_distro_get_context(); 
4044print "distro tuple: ".Dumper($pbos)."\n";
4045
4046# Adapt sudoers
4047# sudo is not default on Solaris and needs to be installed first
4048# from http://www.sunfreeware.com/programlistsparc10.html#sudo
4049if ($pbos->{'type'} eq "pkg") {
4050    $file="/usr/local/etc/sudoers";
4051} else {
4052    $file="/etc/sudoers";
4053}
4054open(PBFILE,$file) || die "Unable to open $file";
4055open(PBOUT,"> $file.new") || die "Unable to open $file.new";
4056while (<PBFILE>) {
4057EOF
4058    # Skip what will be generated
4059    print SCRIPT << "EOF";
4060    next if (/^$pbac->{$ENV{'PBPROJ'}}\\s+/);
4061    next if (/^Defaults:$pbac->{$ENV{'PBPROJ'}}\\s+/);
4062    next if (/^Defaults:root\\s+\!requiretty/);
4063    next if (/^Defaults:root\\s+env_keep/);
4064EOF
4065    print SCRIPT << 'EOF';
4066    s/Defaults\s+requiretty/# disable Defaults: requiretty/;
4067    print PBOUT $_;
4068}
4069close(PBFILE);
4070EOF
4071    print SCRIPT << "EOF";
4072# Some distro force requiretty at compile time, so disable here
4073print PBOUT "Defaults:$pbac->{$ENV{'PBPROJ'}} !requiretty\\n";
4074print PBOUT "Defaults:root !requiretty\\n";
4075EOF
4076    # RH 6.2 sudo doesn't support env_keep
4077    if (($pbos->{'name'} ne "redhat") && ($pbos->{'version'} ne "6.2")) {
4078        print SCRIPT << "EOF";
4079# Keep proxy configuration while using sudo
4080print PBOUT "Defaults:$pbac->{$ENV{'PBPROJ'}}    env_keep += \\\"http_proxy https_proxy ftp_proxy\\\"\\n";
4081print PBOUT "Defaults:root    env_keep += \\\"http_proxy https_proxy ftp_proxy\\\"\\n";
4082EOF
4083    }
4084    # Try to restrict security to what is really needed
4085    if ($vtype =~ /^vm/) {
4086        my $hpath = pb_distro_get($pbos,"ospathcmd-halt");
4087        my @sudocmds = pb_get_sudocmds($pbos,$ntpline,"sudo $hpath","sudo /bin/date");
4088        print SCRIPT << "EOF";
4089# This is needed in order to be able on VM to halt the machine from the $pbac->{$ENV{'PBPROJ'}} account at least
4090# Build account $pbac->{$ENV{'PBPROJ'}} in VM also needs to setup date and install deps.
4091# Nothing else should be needed
4092EOF
4093        foreach my $c (@sudocmds) {
4094            print SCRIPT "print PBOUT \"$pbac->{$ENV{'PBPROJ'}}   ALL = NOPASSWD: $c\\n\";\n";
4095        }
4096    } elsif ($vtype =~ /^rm/) {
4097        my @sudocmds = pb_get_sudocmds($pbos,$ntpline);
4098        print SCRIPT << "EOF";
4099# Build account $pbac->{$ENV{'PBPROJ'}} in RM only needs to setup date and install deps if needed each time
4100EOF
4101        foreach my $c (@sudocmds) {
4102            print SCRIPT "print PBOUT \"$pbac->{$ENV{'PBPROJ'}}   ALL = NOPASSWD: $c\\n\";\n";
4103        }
4104    } else {
4105        print SCRIPT << "EOF";
4106# Build account $pbac->{$ENV{'PBPROJ'}} for VE needs to do a lot in the host (and chroot), so allow without restriction for now
4107print PBOUT "$pbac->{$ENV{'PBPROJ'}}   ALL=(ALL) NOPASSWD:ALL\n";
4108EOF
4109    }
4110    my $sudomode = pb_distro_get($pbos,"ossudoersmode");
4111    print SCRIPT << 'EOF';
4112close(PBOUT);
4113rename("$file.new",$file);
4114EOF
4115    print SCRIPT << "EOF";
4116chmod 0$sudomode,\$file;
4117
4118EOF
4119
4120    if ($vtype =~ /(v|r)m/) {
4121        # Sync date
4122        # do it after sudoers is setup
4123        print SCRIPT "my \$res = pb_system(\"$ntpline\",\"Updating time with $ntpline\",\"mayfailverbose\");\n";
4124        print SCRIPT "if (\$res != 0) {\n";
4125        # try again with another method
4126        $ntpline = pb_date2v($vtype,$pbos,1);
4127        print SCRIPT "  pb_system(\"$ntpline\",\"Updating time with $ntpline\",\"mayfailverbose\");\n";
4128        print SCRIPT "}\n";
4129    }
4130
4131    print SCRIPT << 'EOF';
4132   
4133# We may need a proxy configuration. Get it from the local env
4134pb_apply_conf_proxy($pbos);
4135
4136# Clean up packages DB
4137pb_system("dnf clean all","Cleaning dnf env","mayfail") if (($pbos->{'name'} eq "fedora") && ($pbos->{'version'} >= 22));
4138pb_system("yum clean all","Cleaning yum env","mayfail") if ((($pbos->{'name'} eq "fedora") && ($pbos->{'version'} < 22)) || ($pbos->{'name'} eq "asianux") || ($pbos->{'name'} eq "rhel"));
4139
4140EOF
4141    my ($itype) = pb_conf_get("pbinstalltype");
4142    # Install from sandbox mean a file base install
4143    $itype = "file" if (defined $sbx);
4144    $itype = $installtype if (defined $installtype);
4145    pb_log(2,"DEBUG: Found installtype: $itype\n");
4146    if ($itype =~ /^file/) {
4147        my $pbrepo = pb_distro_get_if($pbos,"addbuildrepo");
4148        #my $pbrepo = pb_distro_get_in_conf_if($pbos,"$ENV{'PBDESTDIR'}/pbrc.yml","addbuildrepo");
4149        print SCRIPT << 'EOF';
4150
4151# First install all required packages
4152my $pkgdep = pb_distro_get_if($pbos,"ospkgdep");
4153my $forcerepo;
4154EOF
4155        print SCRIPT "\$forcerepo = pb_distro_setuprepo_gen(\$pbos,\"$pbrepo\");\n" if ((defined $pbrepo) && ($pbrepo ne ""));
4156        print SCRIPT << 'EOF';
4157pb_distro_installdeps(undef,$pbos,$forcerepo,pb_distro_only_deps_needed($pbos,join(' ',split(/,/,$pkgdep))));
4158
4159EOF
4160        my $cmdget;
4161        if (defined $sbx) {
4162            # Install from sandbox mean using the result of the just passed sbx2build command
4163            # Get content saved in cms2build
4164            my ($pkg) = pb_conf_read("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml","pbpkg");
4165            confess "Unable to get package list" if (not defined $pkg);
4166   
4167            # We consider 2 specific packages
4168            my $vertag1 = $pkg->{"ProjectBuilder"};
4169            my $vertag2 = $pkg->{"project-builder"};
4170            # get the version of the current package - maybe different
4171            pb_log(2,"Vertag1: $vertag1\n");
4172            pb_log(2,"Vertag2: $vertag2\n");
4173            my ($pbver1,$tmp1) = split(/-/,$vertag1);
4174            my ($pbver2,$tmp2) = split(/-/,$vertag2);
4175            my $pbextdir = pb_get_extdir();
4176            $cmdget = "mv /tmp/ProjectBuilder-$pbver1$pbextdir.tar.gz ProjectBuilder-latest.tar.gz ; mv /tmp/project-builder-$pbver2$pbextdir.tar.gz project-builder-latest.tar.gz";
4177        } else {
4178            # TODO: shouldn't we have a variable to specify the source file?
4179            # And shouldn't we be getting the one that matches the current pb version?
4180            # For the moment, just take the latest stable one
4181            my $wget = pb_check_req("wget",0);
4182            $cmdget = "$wget --passive-ftp ftp://ftp.project-builder.org/src/ProjectBuilder-latest.tar.gz; $wget --passive-ftp ftp://ftp.project-builder.org/src/project-builder-latest.tar.gz";
4183        }
4184        print SCRIPT << 'EOF';
4185# Then install manually the missing perl modules
4186my ($osperlver) = pb_conf_get("osperlver");
4187   
4188my $osperldep = pb_distro_get_if($pbos,"osperldep");
4189my $bashopt = "";
4190$bashopt = "-x" if ($pbdebug ge 1);
4191my $verbopt = "mayfail";
4192$verbopt = "mayfailverbose" if ($pbdebug ge 1);
4193if (defined $osperldep) {
4194    foreach my $m (split(/,/,$osperldep)) {
4195        # Skip empty deps
4196        next if ($m =~ /^\s*$/);
4197        my $dir = $m;
4198        $dir =~ s/-.*//;
4199        pb_system("echo \"rm -rf $m* ; wget http://search.cpan.org/CPAN/modules/by-module/$dir/$m-$osperlver->{$m}.tar.gz ; gzip -cd $m-$osperlver->{$m}.tar.gz | tar xf - ; cd $m* ; if [ -f Build.PL ]; then perl Build.PL; ./Build ; ./Build install ; else perl Makefile.PL; make ; make install ; fi; cd .. ; rm -rf $m*\" | bash -e $bashopt" ,"Installing perl module $m-$osperlver->{$m}",$verbopt);
4200    }
4201}
4202EOF
4203        print SCRIPT << 'EOF';
4204pb_system("rm -rf ProjectBuilder-* ; rm -rf project-builder-* ; rm -rf `perl -V:installvendorlib  | awk -F\"'\" '{print \$2}'`/ProjectBuilder ;
4205EOF
4206        print SCRIPT " $cmdget ; ";
4207        print SCRIPT << "EOF"
4208gzip -cd ProjectBuilder-latest.tar.gz | tar xf - ; cd ProjectBuilder-`tar tfz ProjectBuilder-latest.tar.gz | head -1 | cut -d- -f2 | cut -d/ -f1` ; perl Makefile.PL ; make ; make install ; cd .. ; rm -rf ProjectBuilder-* ; gzip -cd project-builder-latest.tar.gz | tar xf - ; cd project-builder-`tar tfz project-builder-latest.tar.gz | head -1 | cut -d- -f3 | cut -d/ -f1` ; perl Makefile.PL ; make ; make install ; cd .. ; rm -rf project-builder-* ;","Building Project-Builder");
4209EOF
4210    } elsif ($itype =~ /^pkg/) {
4211        # pkg based install. We need to point to the project-builder.org repository
4212        print SCRIPT << 'EOF';
4213my $pkgforpb = pb_distro_get_if($pbos,"ospkg");
4214my $forcerepo = pb_distro_setuposrepo($pbos);
4215pb_distro_installpkgs($pbos,pb_distro_only_deps_needed($pbos,join(' ',split(/,/,$pkgforpb))));
4216EOF
4217    } else {
4218        # Unknown install type
4219        confess("Unknown install type $itype->{$ENV{'PBPROJ'}} for param pbinstalltype");
4220    }
4221    print SCRIPT << 'EOF';
4222pb_system("pbdistrocheck",undef,"verbose");
4223pb_system("pb --version",undef,"verbose");
4224EOF
4225    if ($vtype eq "ve") {
4226        if ($vetype ne "docker") {
4227            print SCRIPT << 'EOF';
4228# For VE we need to umount some FS at the end
4229
4230pb_system("umount /proc","Unmounting /proc","mayfail");
4231EOF
4232        }
4233        print SCRIPT << 'EOF';
4234# Create a basic network file if not already there
4235
4236my $nf="/etc/sysconfig/network";
4237if ((! -f $nf) && ($pbos->{'type'} eq "rpm") && ($pbos->{'family'} ne "novell")) {
4238    open(NF,"> $nf") || die "Unable to create $nf";
4239    print NF "NETWORKING=yes\n";
4240    print NF "HOSTNAME=localhost\n";
4241    close(NF);
4242}
4243chmod 0755,$nf;
4244
4245EOF
4246    }
4247
4248    # Use a fake pb_version_init version here
4249    print SCRIPT << "EOF";
4250sub pb_version_init {
4251    return("$projectbuilderver","$projectbuilderrev","$projectbuilderconfver");
4252}
4253EOF
4254
4255    # Adds pb_distro_get_context and all functions needed from ProjectBuilder::Distribution, Conf and Base
4256    # Also adds the minimal independant YAML code from Tiny::YAML in case no other is around
4257    foreach my $m ("ProjectBuilder/Base.pm","ProjectBuilder/Distribution.pm","ProjectBuilder/Conf.pm","ProjectBuilder/YAML.pm") {
4258        foreach my $d (@INC) {
4259            my $f = "$d/$m";
4260            if (-f "$f") {
4261                open(PBD,"$f") || confess "Unable to open $f: $!";
4262                while (<PBD>) {
4263                    next if (/^package/);
4264                    next if (/^use Exporter/);
4265                    next if (/^use ProjectBuilder::/);
4266                    next if (/^1;$/);
4267                    # Remove lines with our except for the errstr var in YAML
4268                    next if ((/^our /) && ($_ !~ /errstr/));
4269                    print SCRIPT $_;
4270                }
4271                close(PBD);
4272                # We just need the first one of each file wrt @INC - hopefully that's the right one.
4273                last;
4274            }
4275        }
4276    }
4277    print SCRIPT << "EOF";
42781;
4279EOF
4280    close(SCRIPT);
4281    chmod 0755,"$pbscript{$v}";
4282
4283    $pm->finish if (defined $pbparallel);
4284}
4285$pm->wait_all_children if (defined $pbparallel);
4286confess "Aborting, one or more of the children failed." if ((not $all_ok) && ($Global::pb_stop_on_error));
4287
4288# These build scripts need to be run as root and force stop of VM at end
4289my $svgac = $ENV{'PBACCOUNT'};
4290$ENV{'PBACCOUNT'} = "root";
4291
4292$sbx = "" if (not defined $sbx);
4293# Launch the VM/VE/RM
4294pb_log(2,"DEBUG: before parallel launch, pbscript hash is:".Dumper(%pbscript)."\n");
4295pb_parallel_launchv(\%pbscript,$vtype,uc($vtype)."Script".$sbx,$pbstep,$pbimage,$pbforce,0,$pbsnap);
4296
4297$ENV{'PBACCOUNT'} = $svgac;
4298return;
4299}
4300
4301# Function to create a snapshot named 'pb' for VMs and a compressed tar for VEs
4302sub pb_snap2v {
4303
4304my $vtype = shift;
4305my $pbstep = 2;
4306
4307my ($vm,$all) = pb_get2v($vtype);
4308
4309# Script generated
4310my $pbscript = "$ENV{'PBDESTDIR'}/snapv.$$";
4311
4312my ($pbac) = pb_conf_get($vtype."login");
4313
4314foreach my $v (@$vm) {
4315    if ($vtype eq "ve") {
4316        # Get distro context
4317        my $pbos = pb_distro_get_context($v);
4318        my ($vepath) = pb_conf_get("vepath");
4319
4320        # Test if an existing snapshot exists and remove it if there is a VE
4321        if ((-f "$vepath->{$ENV{'PBPROJ'}}/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}.tar.gz") &&
4322            (! -d "$vepath->{$ENV{'PBPROJ'}}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}")) {
4323                pb_system("sudo rm -f $vepath->{$ENV{'PBPROJ'}}/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}.tar.gz","Removing previous snapshot $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}.tar.gz");
4324        }
4325    }
4326
4327    # Prepare the script to be executed on the VM/VE
4328    open(SCRIPT,"> $pbscript") || confess "Unable to create $pbscript";
4329    print SCRIPT << 'EOF';
4330    #!/bin/bash
4331    sleep 2
4332EOF
4333    close(SCRIPT);
4334    chmod 0755,"$pbscript";
4335
4336    # Force shutdown of VM/VE
4337    # Force snapshot of VM/VE
4338    pb_script2v($pbscript,$vtype,$pbstep,1,$v,1);
4339}
4340return;
4341}
4342
4343# Function to update VMs/VEs/RMs with the latest distribution content
4344sub pb_update2v {
4345
4346my $vtype = shift;
4347my $pbstep = 2;
4348
4349my ($vm,$all) = pb_get2v($vtype);
4350
4351# Script generated
4352my $pbscript = "$ENV{'PBDESTDIR'}/updatev.$$";
4353
4354my ($pbac) = pb_conf_get($vtype."login");
4355
4356foreach my $v (@$vm) {
4357    # Get distro context
4358    my $pbos = pb_distro_get_context($v);
4359
4360    # Prepare the script to be executed on the VM/VE/RM
4361    # in $ENV{'PBDESTDIR'}/updatev
4362    open(SCRIPT,"> $pbscript") || confess "Unable to create $pbscript";
4363   
4364    print SCRIPT << 'EOF';
4365    #!/bin/bash
4366    sleep 2
4367EOF
4368    # VE needs a good /proc
4369    $vetype = pb_ve_get_type($vetype);
4370    if (($vtype eq "ve") && ($vetype ne "docker")) {
4371        print SCRIPT "sudo /bin/mount -t proc /proc /proc\n";
4372    }
4373    print SCRIPT "$pbos->{'update'}\n";
4374    if (($vtype eq "ve") && ($vetype ne "docker")) {
4375        print SCRIPT "sudo /bin/umount /proc\n";
4376    }
4377    close(SCRIPT);
4378    chmod 0755,"$pbscript";
4379
4380    # Force shutdown of VM except
4381    pb_script2v($pbscript,$vtype,$pbstep,1,$v);
4382}
4383return;
4384}
4385
4386# Build all docker CTN required to build an app
4387sub pb_all_docker {
4388
4389my $action = shift;
4390my $savproj = $ENV{'PBPROJ'};
4391my $pkg;
4392my @pkg;
4393if ($#ARGV == -1) {
4394    $pkg = pb_cms_get_pkg($defpkgdir,$extpkgdir);
4395    @pkgs = @$pkg;
4396} else {
4397    @pkgs = @ARGV;
4398}
4399#pb_log(1,"Packages to print:".Dumper(@pkgs)."\n");
4400
4401# Get the list of all VE we need to work on
4402my ($vm,$all);
4403if (not defined $ENV{'PBV'}) {
4404    ($vm,$all) = pb_get2v("ve");
4405} else {
4406    @$vm = split(/,/,$ENV{'PBV'});
4407}
4408
4409# Adapt // mode to memory size
4410$pbparallel = pb_set_parallel("ve");
4411
4412pb_log(2,"DEBUG: pb_all_docker(action: $action)\n");
4413my $docrepo = pb_ve_docker_repo();
4414
4415my ($vexist,$vpid) = (undef,undef);
4416my $pm;
4417my $all_ok = 1;
4418if (defined $pbparallel) {
4419    $pm = new Parallel::ForkManager($pbparallel);
4420    $pm->set_waitpid_blocking_sleep(0);
4421
4422    $pm->run_on_finish(sub { my ($pid, $code, $id, $signal, $dump) = @_;
4423                            unless ($code == 0 && $signal == 0 && $dump == 0) {
4424                                $all_ok = 0;
4425                                pb_log(0,"ERROR: pid $pid failed\n");
4426                                pb_log(0,"code: $code - signal: $signal - dump: $dump\n");
4427                            }
4428                        });
4429}
4430
4431my $counter = 0;
4432foreach my $v (@$vm) {
4433    $counter++;
4434    # Modulo 2 * pbparallel (to avoid synchronization problems)
4435    $counter = 1 if ((defined $pbparallel) && ($counter > 2 * $pbparallel));
4436    $pm->start($counter) and next if (defined $pbparallel);
4437
4438    my $lmsg = "";
4439    my $lctn = "";
4440    my $pbos = pb_distro_get_context($v);
4441    # Docker images are (for now !) regular and use distro:ver
4442    $pbimage = "$pbos->{'name'}".":$pbos->{'version'}";
4443    # The two first steps need to be done with the pb project and using a stable pb version
4444    $ENV{'PBPROJ'} = $appname;
4445    $lctn = "$docrepo$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
4446    my $found = pb_ve_docker_get_image($lctn);
4447    if (($pbforce == 1) || (not defined $found)) {
4448        # pb -p pb -m distro-ver-arch newve -i distro:ver
4449        ($pbforce == 1) ? $lmsg = "Forced" : $lmsg = "not found";
4450        pb_log(0,"INFO: Docker image $lctn $lmsg, creating it\n");
4451        pb_launchv(undef,"ve",$v,$action,0,$pbforce,undef,undef,$pbimage);
4452    }
4453    # Remove ref to pbimage so the rest uses the previous one built
4454    $pbimage = undef;
4455    $lctn = "$docrepo$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-pb";
4456    $found = pb_ve_docker_get_image($lctn);
4457    if (($pbforce == 1) || (not defined $found)) {
4458        ($pbforce == 1) ? $lmsg = "Forced" : $lmsg = "not found";
4459        pb_log(0,"INFO: Docker image $lctn $lmsg, creating it\n");
4460        # pb -p pb -m distro-ver-arch setupve
4461        pb_log(2,"Using savproj: $savproj\n");
4462        if ($savproj ne "pb") {
4463            pb_cms2build("SandBox") if ($action =~ /sbx2/);
4464            # Here we install in the CTN pb from upstream packages to have a stable version
4465            pb_setup2v("ve",undef,"pkg");
4466        } else {
4467            pb_setup2v("ve","SandBox");
4468        }
4469    }
4470    # Now back to our original project
4471    $ENV{'PBPROJ'} = $savproj;
4472    $lctn = "$docrepo$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-pb-$ENV{'PBPROJ'}";
4473    $found = pb_ve_docker_get_image($lctn);
4474    if (($pbforce == 1) || (not defined $found)) {
4475        ($pbforce == 1) ? $lmsg = "Forced" : $lmsg = "not found";
4476        pb_log(0,"INFO: Docker image $lctn $lmsg, creating it\n");
4477        # pb -p proj -m distro-ver-arch prepve
4478        pb_build2v("ve","prep");
4479    }
4480    pb_log(0,"INFO: Instantiating a Docker container from $lctn\n");
4481    pb_build2v("ve","build");
4482    $pm->finish if (defined $pbparallel);
4483}
4484$pm->wait_all_children if (defined $pbparallel);
4485confess "Aborting, one or more of the children failed." if ((not $all_ok) && ($Global::pb_stop_on_error));
4486}
4487
4488sub pb_announce {
4489
4490    my $antype = shift;
4491
4492    # Get all required parameters
4493    my ($pbpackager,$pbrepo) = pb_conf_get("pbpackager","pbrepo");
4494    my ($pkgv, $pkgt, $testver) = pb_conf_get_if("pkgver","pkgtag","testver");
4495    if (((not defined $testver) || (not defined $testver->{$ENV{'PBPROJ'}}) || ($testver->{$ENV{'PBPROJ'}} !~ /true/i)) && ($antype eq "Clean")) {
4496        # We just clean for test versions
4497        pb_log(0,"Unable to clean SSH repository for non testver version\n");
4498        return;
4499    }
4500    my $pkg = pb_cms_get_pkg($defpkgdir,$extpkgdir);
4501    my @pkgs = @$pkg;
4502    my %pkgs;
4503    my $first = 0;
4504
4505    # Get all distros concerned
4506    my $pbos = pb_distro_get_context();
4507    my $distrolist = pb_get_distros($pbos,undef);
4508    my %dl;
4509    my %theorlist;
4510    my %archlist;
4511    foreach my $d (split(/,/,$distrolist)) {
4512        my ($d1,$d2,$d3) = split(/-/,$d);
4513        $dl{$d1}++;
4514    }
4515
4516    # Command to find packages on repo
4517    my $findstr = "find ".join(" ",keys %dl)." ";
4518    my $srcstr = "";
4519    # Generated announce files
4520    my @files;
4521
4522    foreach my $pbpkg (@pkgs) {
4523        if ($first != 0) {
4524            $findstr .= "-o ";
4525        }
4526        $first++;
4527        if ((defined $pkgv) && (defined $pkgv->{$pbpkg})) {
4528            $pbver = $pkgv->{$pbpkg};
4529        } else {
4530            $pbver = $ENV{'PBPROJVER'};
4531        }
4532        if ((defined $pkgt) && (defined $pkgt->{$pbpkg})) {
4533            $pbtag = $pkgt->{$pbpkg};
4534        } else {
4535            $pbtag = $ENV{'PBPROJTAG'};
4536        }
4537
4538        # TODO: use virtual/real names here now
4539        my $pbrealpkg = $pbpkg;
4540        my $pbrealpkgrpm = pb_cms_get_real_pkg($pbpkg,"rpm");
4541        my $pbrealpkgdeb = pb_cms_get_real_pkg($pbpkg,"deb");
4542        if ($antype eq "Clean") {
4543            # We only clean test versions anyway
4544            $pbtag = "0";
4545            my $nver = $pbver;
4546            my $ntag = "[2-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]";
4547            $pbver .= $ntag;
4548            # TODO add apk
4549            $findstr .= "-name \'$pbrealpkgrpm-$pbver-$pbtag\.*.rpm\' -o -name \'$pbrealpkgrpm-debug-$pbver-$pbtag\.*.rpm\' -o -name \'$pbrealpkgdeb"."_$pbver-$pbtag"."_*\.deb\' -o -name \'$pbrealpkgdeb"."_$pbver-$pbtag.dsc\' -o -name \'$pbrealpkgdeb"."_$pbver-$pbtag.tar.gz\' -o -name \'$pbrealpkg-$nver"."_p$ntag\.ebuild\' -o -name \'$pbrealpkg-$pbver-$pbtag*\.pkg\.gz\' -o -name \'$pbrealpkg-$pbver-$pbtag*\.sd\' ";
4550            $srcstr .= "src/$pbrealpkg-$pbver.tar.gz src/$pbrealpkg-$pbver.pbconf.tar.gz ";
4551        } else {
4552            # "Announce" and "Check"
4553            my @date=pb_get_date();
4554            # the matching is only done on packages made the same day for test version. Hopefully this is enough
4555            my $nver;
4556            if ((defined $testver) && (defined $testver->{$ENV{'PBPROJ'}}) && ($testver->{$ENV{'PBPROJ'}} =~ /true/i)) {
4557                $pbtag = "0";
4558                my $ntag .= strftime("%Y%m%d*", @date);
4559                $nver = $pbver."_p$ntag*";
4560                $pbver .= "$ntag*";
4561            } else { 
4562                $nver = $pbver."-r$pbtag";
4563            }
4564            $findstr .= "-name \'$pbrealpkgrpm-$pbver-$pbtag\.*.rpm\' -o -name \'$pbrealpkgdeb"."_$pbver-$pbtag"."_*\.deb\' -o -name \'$pbrealpkg-$nver\.ebuild\' -o -name \'$pbrealpkg-$pbver\.pkg\.gz\' -o -name \'$pbrealpkg-$pbver\.sd\' ";
4565        }
4566
4567        if ($antype eq "Announce") {
4568            my $chglog;
4569
4570            pb_vcs_init($pbinit);
4571            # Get project info on log file and generate tmp files used later on
4572            $chglog = "$ENV{'PBROOTDIR'}/$pbpkg/pbcl";
4573            $chglog = "$ENV{'PBROOTDIR'}/pbcl" if (! -f $chglog);
4574            $chglog = undef if (! -f $chglog);
4575
4576            open(OUT,"> $ENV{'PBTMP'}/$pbpkg.ann") || confess "Unable to create $ENV{'PBTMP'}/$pbpkg.ann: $!";
4577            my $pb;
4578            $pb->{'realpkg'} = $pbrealpkg;
4579            $pb->{'ver'} = $pbver;
4580            $pb->{'tag'} = $pbtag;
4581            $pb->{'date'} = $pbdate;
4582            $pb->{'extdir'} = pb_get_extdir();
4583            $pb->{'chglog'} = $chglog;
4584            $pb->{'packager'} = $pbpackager;
4585            $pb->{'proj'} = $ENV{'PBPROJ'};
4586            $pb->{'repo'} = $pbrepo;
4587            $pb->{'pbos'}->{'type'} = "announce";
4588            $pb->{'pbos'}->{'suffix'} = "none";
4589            pb_changelog($pb,\*OUT,"yes");
4590            close(OUT);
4591            push(@files,"$ENV{'PBTMP'}/$pbpkg.ann");
4592        } elsif ($antype eq "Check") {
4593            # For the check we also build the theoritical complete list we should get
4594            foreach my $d (split(/,/,$distrolist)) {
4595                $pbos = pb_distro_get_context($d);
4596                if ($pbos->{'type'} eq "rpm") {
4597                    $theorlist{"$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}/$pbrealpkgrpm-$pbver-$pbtag$pbos->{'suffix'}"} = 0;
4598                } elsif ($pbos->{'type'} eq "deb") {
4599                    $theorlist{"$pbos->{'name'}/$pbos->{'version'}/$pbrealpkgdeb"."_$pbver-$pbtag"} = 0;
4600                    # TODO are we always using the last arch ?
4601                    $archlist{"$pbos->{'name'}/$pbos->{'version'}/$pbrealpkgdeb"."_$pbver-$pbtag"} = "$pbos->{'arch'}";
4602                } elsif ($pbos->{'type'} eq "ebuild") {
4603                    my $prefix = "-r";
4604                    $prefix = "_p" if ((defined $testver) && (defined $testver->{$ENV{'PBPROJ'}}) && ($testver->{$ENV{'PBPROJ'}} =~ /true/i));
4605                    $theorlist{"$pbos->{'name'}/$pbos->{'version'}/$pbrealpkg-$pbver$prefix$pbtag.ebuild"} = 0;
4606                    $archlist{"$pbos->{'name'}/$pbos->{'version'}/$pbrealpkg-$pbver$prefix$pbtag.ebuild"} = "$pbos->{'arch'}";
4607                } elsif ($pbos->{'type'} eq "apk") {
4608                    # TODO
4609                } elsif ($pbos->{'type'} eq "pkg") {
4610                    $theorlist{"$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}/$pbrealpkg-$pbver-$pbtag.pkg.gz"} = 0;
4611                } else {
4612                    pb_log(1,"No theoritical list possible for type $pbos->{'type'}\n");
4613                }
4614            }
4615        }
4616        pb_log(2,"theorlist initial: ".Dumper(%theorlist)."\n");
4617        pb_log(2,"archlist: ".Dumper(%archlist)."\n");
4618    }
4619    if ($antype eq "Announce") {
4620        $findstr .= " | grep -Ev \'src.rpm\'";
4621    } elsif ($antype eq "Clean") {
4622        $findstr .= " | xargs rm -f -v $srcstr ";
4623    }
4624
4625    # Prepare the command to run and execute it
4626    open(PBS,"> $ENV{'PBTMP'}/pbscript") || confess "Unable to create $ENV{'PBTMP'}/pbscript";
4627    print PBS "#!/bin/bash\n";
4628    print PBS "set -x\n" if ($pbdebug gt 1);
4629    print PBS "$findstr | sort 2> /dev/null\n";
4630    close(PBS);
4631    chmod 0755,"$ENV{'PBTMP'}/pbscript";
4632    pb_send2target("Announce","$ENV{'PBTMP'}/pbscript");
4633
4634    my $sl = "Project $ENV{'PBPROJ'} version $ENV{'PBPROJVER'} is now available";
4635    if ($antype eq "Announce") {
4636        # Get subject line
4637        pb_log(0,"Please enter the title of your announce\n");
4638        pb_log(0,"(By default: $sl)\n");
4639        my $sl2 = <STDIN>;
4640        $sl = $sl2 if ($sl2 !~ /^$/);
4641
4642        # Prepare a template of announce
4643        open(ANN,"> $ENV{'PBTMP'}/announce.html") || confess "Unable to create $ENV{'PBTMP'}/announce.html: $!";
4644        print ANN << "EOF";
4645$sl</p>
4646
4647<p>The project team is happy to announce the availability of a newest version of $ENV{'PBPROJ'} $ENV{'PBPROJVER'}. Enjoy it as usual!</p>
4648<p>
4649Now available at <a href="$pbrepo->{$ENV{'PBPROJ'}}">$pbrepo->{$ENV{'PBPROJ'}}</a>
4650</p>
4651<p>
4652EOF
4653    }
4654
4655    open(LOG,"$ENV{'PBTMP'}/system.$$.log") || confess "Unable to read $ENV{'PBTMP'}/system.$$.log: $!";
4656    if ($antype eq "Announce") {
4657        my $col = 2;
4658        my $i = 1;
4659        print ANN << 'EOF';
4660<TABLE WIDTH="100%" CELLPADDING="0" CELLSPACING="0" BORDER="0">
4661<TR>
4662EOF
4663        while (<LOG>) {
4664            print ANN "<TD><A HREF=\"$pbrepo->{$ENV{'PBPROJ'}}/$_\">$_</A></TD>";
4665            $i++;
4666            if ($i > $col) {
4667                print ANN "</TR>\n<TR>";
4668                $i = 1;
4669            }
4670        }
4671    } elsif ($antype eq "Clean") {
4672        while (<LOG>) {
4673            # skip errors
4674            next if ($_ !~ /^removed /);
4675            pb_log(0,"$_");
4676        }
4677    } else {
4678        # In "Check" mode we need to compare the 2 lists (real and theoritical)
4679        while (<LOG>) {
4680            # Get package name and remove what is in extra for the theoritical list (arch at the end)
4681            chomp();
4682            # skip find errors
4683            next if (/^find:/);
4684            my $p = $_;
4685            $p =~ s/\.(i[3456]86|x86_64|noarch|src)\.rpm$//;
4686            $p =~ s/_(i[3456]86|amd64|all)\.deb$//;
4687            $p =~ s/(-0\.[0-9]{8})[0-9]{6}/$1*/ if ((defined $testver) && (defined $testver->{$ENV{'PBPROJ'}}) && ($testver->{$ENV{'PBPROJ'}} =~ /true/i));
4688            $p =~ s/(-r|_p[0-9]+)\.ebuild/$1*/ if ((defined $testver) && (defined $testver->{$ENV{'PBPROJ'}}) && ($testver->{$ENV{'PBPROJ'}} =~ /true/i));
4689            # TODO apk
4690            $theorlist{$p} = -2 if (not defined $theorlist{$p});
4691            $theorlist{$p} = $theorlist{$p} + 1;
4692        }
4693        pb_log(2,"theorlist after check: ".Dumper(%theorlist)."\n");
4694    }
4695    close(LOG);
4696
4697    # Nothing more for the cleanssh case
4698    return if ($antype eq "Clean");
4699
4700    if ($antype eq "Check") {
4701        my ($chkex) = pb_conf_get_if("checkexclude");
4702        my $vmbuildlist = "";
4703        my $vebuildlist = "";
4704        my $rmbuildlist = "";
4705        my @pt = pb_conf_get_if("vmlist","velist","rmlist");
4706        foreach my $t (sort keys %theorlist) {
4707            if (defined $theorlist{$t} and $theorlist{$t} >= 1) {
4708                pb_log(1,"Packages found for $t\n");
4709            } elsif (defined $theorlist{$t} and $theorlist{$t} < 0) {
4710                pb_log(0,"Extra Package found for $t\n");
4711            } else {
4712                pb_log(2,"Analyzing $t\n");
4713                my ($os,$ver,$arch,$package) = split(/\//,$t);
4714                # Some distro have no arch subdir
4715                if (not defined $package) {
4716                    $package = $arch;
4717                    # TODO: If both arch have failed, we just make the last one
4718                    $arch = $archlist{$t} if (defined $archlist{$t});
4719                }
4720                my $pbos = pb_distro_get_context("$os-$ver-$arch");
4721                my $pkgn = $package;
4722                if ($pbos->{'type'} ne "deb") {
4723                    # package name is more easily found from the end for non deb
4724                    # as '-' is the separator, but it can also be used in names
4725                    $pkgn = reverse($package);
4726                    # search the second '-' and isolate the now last part which is the full name
4727                    $pkgn =~ s/([^-]+)-([^-]+)-([\S])+$/$3/;
4728                } else {
4729                    $pkgn =~ s/([^_]+)_([\S])+$/$2/;
4730                }
4731                my $found = 0;
4732                # Handle the exclusion of OSes
4733                my $excl = "";
4734                $excl .= $chkex->{$pkgn} if (defined $chkex->{$pkgn});
4735                $excl .= $chkex->{"all"} if (defined $chkex->{"all"});
4736
4737                pb_log(2,"Searching in excl $excl\n");
4738                foreach my $ex (split(/,/,$excl)) {
4739                    $found = 1 if ("$os-$ver-$arch" =~ /^$ex/);
4740                }
4741                # Skip as excluded
4742                next if ($found == 1);
4743                pb_log(0,"Package NOT found for $t\n");
4744                # Avoid duplicates in list
4745                next if ($vmbuildlist =~ /$os-$ver-$arch/);
4746                next if ($vebuildlist =~ /$os-$ver-$arch/);
4747                next if ($rmbuildlist =~ /$os-$ver-$arch/);
4748                # check with which method we need to build
4749                if ((defined $pt[0]->{$ENV{'PBPROJ'}}) and ($pt[0]->{$ENV{'PBPROJ'}} =~ /$os-$ver-$arch/)) {
4750                    $vmbuildlist = "$os-$ver-$arch" if ($vmbuildlist eq "");
4751                    $vmbuildlist .= ",$os-$ver-$arch" if ($vmbuildlist !~ /$os-$ver-$arch/);
4752                    next;
4753                }
4754                if ((defined $pt[1]->{$ENV{'PBPROJ'}}) and ($pt[1]->{$ENV{'PBPROJ'}} =~ /$os-$ver-$arch/)) {
4755                    $vebuildlist = "$os-$ver-$arch" if ($vebuildlist eq "");
4756                    $vebuildlist .= ",$os-$ver-$arch" if ($vebuildlist !~ /$os-$ver-$arch/);
4757                    next;
4758                }
4759                if ((defined $pt[2]->{$ENV{'PBPROJ'}}) and ($pt[2]->{$ENV{'PBPROJ'}} =~ /$os-$ver-$arch/)) {
4760                    $rmbuildlist = "$os-$ver-$arch" if ($rmbuildlist eq "");
4761                    $rmbuildlist .= ",$os-$ver-$arch" if ($rmbuildlist !~ /$os-$ver-$arch/);
4762                }
4763            }
4764        }
4765        # If we want to rebuild automatically, let's do it
4766        if (defined $opts{'rebuild'}) {
4767            # SandBox or CMS
4768            pb_log(0,"Rebuilding from SandBox\n");
4769            pb_log(0,"for VMs: $vmbuildlist\n") if ($vmbuildlist ne "");
4770            pb_log(0,"for VEs: $vebuildlist\n") if ($vebuildlist ne "");
4771            pb_log(0,"for RMs: $rmbuildlist\n") if ($rmbuildlist ne "");
4772            pb_cms2build("SandBox");
4773            # Which mode
4774            $ENV{'PBV'} = $vmbuildlist;
4775            pb_build2v("vm","build") if ($vmbuildlist ne "");
4776            $ENV{'PBV'} = $vebuildlist;
4777            pb_build2v("ve","build") if ($vebuildlist ne "");
4778            $ENV{'PBV'} = $rmbuildlist;
4779            pb_build2v("rm","build") if ($rmbuildlist ne "");
4780        }
4781        # For the check part this is now finished
4782        return;
4783    }
4784
4785    print ANN << "EOF";
4786</TR>
4787</TABLE>
4788</p>
4789
4790<p>As usual source packages are also available in the same directory.</p>
4791
4792<p>
4793Changes are :
4794</p>
4795<p>
4796EOF
4797    # Get each package changelog content
4798    foreach my $f (sort(@files)) {
4799        open(IN,"$f") || confess "Unable to read $f:$!";
4800        while (<IN>) {
4801            print ANN $_;
4802        }
4803        close(IN);
4804        print ANN "</p><p>\n";
4805    }
4806    print ANN "</p>\n";
4807    close(ANN);
4808
4809    # Allow for modification
4810    my $editor = "vi";
4811    $editor = $ENV{'EDITOR'} if (defined $ENV{'EDITOR'});
4812    pb_system("$editor $ENV{'PBTMP'}/announce.html","Allowing modification of the announce","noredir");
4813
4814    # Store it in DB for external usage (Web pages generation)
4815    my $dbh = pb_connect_db();
4816
4817    # To read whole file
4818    local $/;
4819    open(ANN,"$ENV{'PBTMP'}/announce.html") || confess "Unable to read $ENV{'PBTMP'}/announce.html: $!";
4820    my $announce = <ANN>;
4821    close(ANN);
4822   
4823    pb_log(2,"INSERT INTO announces VALUES (NULL, $pbdate, $announce)");
4824    my $sth = $dbh->prepare(qq{INSERT INTO announces VALUES (NULL,?,?)})
4825                    || confess "Unable to insert into db";
4826    $sth->execute($pbdate, $announce);
4827    $sth->finish();
4828    $dbh->disconnect;
4829
4830    # Then deliver it on the Web
4831    # $TOOLHOME/livwww www
4832
4833    # Mail it to project's ML
4834    open(ML,"| w3m -dump -T text/html > $ENV{'PBTMP'}/announce.txt") || confess "Unable to create $ENV{'PBTMP'}/announce.txt: $!";
4835    print ML << 'EOF';
4836<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/x html1/DTD/xhtml1-strict.dtd">
4837
4838<html xmlns="http://www.w3.org/1999/xhtml" dir="ltr" xml:lang="en" lang="en">
4839  <head>
4840  </head>
4841  <body>
4842  <p>
4843EOF
4844    open(ANN,"$ENV{'PBTMP'}/announce.html") || confess "Unable to read $ENV{'PBTMP'}/announce.html: $!";
4845    while(<ANN>) {
4846        print ML $_;
4847    }
4848    print ML << 'EOF';
4849</body>
4850</html>
4851EOF
4852    close(ML);
4853
4854    # To read whole file
4855    local $/;
4856    open(ANN,"$ENV{'PBTMP'}/announce.txt") || confess "Unable to read $ENV{'PBTMP'}/announce.txt: $!";
4857    my $msg = <ANN>;
4858    close(ANN);
4859   
4860    # Preparation of headers
4861    eval
4862    {
4863        require Mail::Sendmail;
4864        Mail::Sendmail->import();
4865    };
4866    if ($@) {
4867        # Mail::Sendmail not found not sending mail !
4868        pb_log(0,"No Mail::Sendmail module found so not sending any mail !\n");
4869    } else {
4870        my ($pbml,$pbsmtp) = pb_conf_get("pbml","pbsmtp");
4871        my %mail = (   
4872            To          =>  $pbml->{$ENV{'PBPROJ'}},
4873            From        =>  $pbpackager->{$ENV{'PBPROJ'}},
4874            Smtp        =>  $pbsmtp->{$ENV{'PBPROJ'}},
4875            Body        =>  $msg,
4876            Subject     =>  "[ANNOUNCE] $sl",
4877        );
4878           
4879        # Send mail
4880        if (! sendmail(%mail)) {
4881            if ((defined $Mail::Sendmail::error) and (defined $Mail::Sendmail::log)) {
4882                confess "Unable to send mail ($Mail::Sendmail::error): $Mail::Sendmail::log";
4883            }
4884        }
4885    }
4886}
4887
4888#
4889# Creates a set of HTML file containing the news for the project
4890# based on what has been generated by the pb_announce function
4891#
4892sub pb_web_news2html {
4893
4894    my $dest = shift || $ENV{'PBTMP'};
4895
4896    # Get all required parameters
4897    my ($pkgv, $pkgt) = pb_conf_get_if("pkgver","pkgtag");
4898
4899
4900    my $dbh = pb_connect_db();
4901    # For date handling
4902    $ENV{'LANGUAGE'}="C";
4903    my $firstjan = strftime("%Y-%m-%d", 0, 0, 0, 1, 0, localtime->year(), 0, 0, -1);
4904    my $oldfirst = strftime("%Y-%m-%d", 0, 0, 0, 1, 0, localtime->year()-1, 0, 0, -1);
4905    pb_log(2,"firstjan: $firstjan, oldfirst: $oldfirst, pbdate:$pbdate\n");
4906    my $all = $dbh->selectall_arrayref("SELECT id,date,announce FROM announces ORDER BY date DESC");
4907    my %news;
4908    $news{"cy"} = "";   # current year's news
4909    $news{"ly"} = "";   # last year news
4910    $news{"py"} = "";   # previous years news
4911    $news{"fp"} = "";   # first page news
4912    my $cpt = 4;        # how many news for first page
4913    # Extract info from DB
4914    foreach my $row (@$all) {
4915        my ($id, $date, $announce) = @$row;
4916        $news{"cy"} = $news{"cy"}."<p><B>$date</B> $announce\n" if ((($date cmp $pbdate) le 0) && (($firstjan cmp $date) le 0));
4917        $news{"ly"} = $news{"ly"}."<p><B>$date</B> $announce\n" if ((($date cmp $firstjan) le 0) && (($oldfirst cmp $date) le 0));
4918        $news{"py"} = $news{"py"}."<p><B>$date</B> $announce\n" if (($date cmp $oldfirst) le 0);
4919        $news{"fp"} = $news{"fp"}."<p><B>$date</B> $announce\n" if ($cpt > 0);
4920        $cpt--;
4921    }
4922    pb_log(1,"news{fp}: ".$news{"fp"}."\n");
4923    $dbh->disconnect;
4924
4925    # Generate the HTML content
4926    foreach my $pref (keys %news) {
4927        open(NEWS,"> $dest/pb_web_$pref"."news.html") || confess "Unable to create $dest/pb_web_$pref"."news.html: $!";
4928        print NEWS "$news{$pref}";
4929        close(NEWS);
4930    }
4931}
4932
4933sub pb_connect_db {
4934
4935    # Store in DB for external usage (Web pages generation)
4936    my $db = "$ENV{'PBCONFDIR'}/announces3.sql";
4937
4938    my $precmd = "";
4939    if (! -f $db) {
4940        $precmd = "CREATE TABLE announces (id INTEGER PRIMARY KEY AUTOINCREMENT, date DATE, announce VARCHAR[65535])";
4941        # we should add the $db to the VCS as well here
4942    }
4943
4944    my $dbh = DBI->connect("dbi:SQLite:dbname=$db","","",
4945                        { RaiseError => 1, AutoCommit => 1 })
4946                        || confess "Unable to connect to $db";
4947
4948    if ($precmd ne "") {
4949        my $sth = $dbh->prepare(qq{$precmd})
4950                    || confess "Unable to create table into $db";
4951        $sth->execute();
4952    }
4953
4954    return($dbh);
4955}
4956
4957
4958# Return the SSH key file to use
4959# Potentially create it if needed
4960
4961sub pb_ssh_get {
4962
4963my $create = shift || 0;    # Do not create keys by default. In that case prefer native keys, not pb's ones
4964my $target = shift || "localhost";
4965my $nport = shift || "22";
4966
4967my ($pbagent) = pb_conf_get_if("pbusesshagent");
4968# use ssh-agent if asked so.
4969return(undef) if (($create == 0) && (defined $pbagent->{$ENV{'PBPROJ'}}) && ($pbagent->{$ENV{'PBPROJ'}} =~ /true/io));
4970
4971# Check the SSH environment
4972my $keyfile = undef;
4973
4974# Check whether ecdsa is accepted by the remote host
4975my $ecdsa = 0;
4976my $dsa = 0;
4977my $rsa = 0;
4978open(SCAN,"ssh-keyscan -p $nport -t ecdsa $target 2>/dev/null |") || pb_log(0,"Unable to check ECDSA support for $target");
4979while(<SCAN>) {
4980    $ecdsa = 1 if ($_ =~ /ecdsa-/);
4981    pb_log(2,"Found ECDSA support on $target ($nport)\n");
4982}
4983close(SCAN);
4984
4985open(SCAN,"ssh-keyscan -p $nport -t rsa $target 2>/dev/null |") || pb_log(0,"Unable to check DSA support for $target");
4986while(<SCAN>) {
4987    $rsa = 1 if ($_ =~ /ssh-rsa/);
4988    pb_log(2,"Found RSA support on $target ($nport)\n");
4989}
4990close(SCAN);
4991
4992open(SCAN,"ssh-keyscan -p $nport -t dsa $target 2>/dev/null |") || pb_log(0,"Unable to check DSA support for $target");
4993while(<SCAN>) {
4994    $dsa = 1 if ($_ =~ /ssh-dss/);
4995    pb_log(2,"Found DSA support on $target ($nport)\n");
4996}
4997close(SCAN);
4998
4999# We have specific keys by default - create them if needed
5000if (!(-e "$ENV{'HOME'}/.ssh/pb_ecdsa") && ($create eq 1)) {
5001    my $res = pb_system("ssh-keygen -q -b 521 -N '' -f $ENV{'HOME'}/.ssh/pb_ecdsa -t ecdsa","Generating ECDSA SSH keys for pb");
5002    if (($res != 0) && !(-e "$ENV{'HOME'}/.ssh/pb_dsa") && ($create eq 1)) {
5003        # ecdsa may not be supported if old ssh
5004        pb_system("ssh-keygen -q -b 1024 -N '' -f $ENV{'HOME'}/.ssh/pb_dsa -t dsa","Generating DSA SSH keys for pb");
5005    }
5006}
5007
5008if (!(-e "$ENV{'HOME'}/.ssh/pb_rsa") && ($create eq 1)) {
5009    pb_system("ssh-keygen -q -b 2048 -N '' -f $ENV{'HOME'}/.ssh/pb_rsa -t rsa","Generating RSA SSH keys for pb");
5010}
5011
5012# We still favour RSA and fall back to ECDSA if not available
5013# Some ECDSA implementation are not working correctly e.g. RHEL7, Fedora 20/21
5014#$keyfile = "$ENV{'HOME'}/.ssh/pb_rsa" if (-s "$ENV{'HOME'}/.ssh/pb_rsa");
5015if ($rsa != 0) {
5016    # And we favour a specific key over a generic one except if in delivery to ftp server
5017    $keyfile = "$ENV{'HOME'}/.ssh/id_rsa" if ((-s "$ENV{'HOME'}/.ssh/id_rsa") && ($create eq 0));
5018    $keyfile = "$ENV{'HOME'}/.ssh/pb_rsa" if (-s "$ENV{'HOME'}/.ssh/pb_rsa");
5019} else {
5020    if ($ecdsa != 0) {
5021        # And we favour a specific key over a generic one except if in delivery to ftp server
5022        $keyfile = "$ENV{'HOME'}/.ssh/id_ecdsa" if ((-s "$ENV{'HOME'}/.ssh/id_ecdsa") && ($ecdsa != 0) && ($create eq 0));
5023        $keyfile = "$ENV{'HOME'}/.ssh/pb_ecdsa" if ((-s "$ENV{'HOME'}/.ssh/pb_ecdsa") && ($ecdsa != 0));
5024    } elsif ($dsa != 0) {
5025        # And we favour a specific key over a generic one except if in delivery to ftp server
5026        $keyfile = "$ENV{'HOME'}/.ssh/id_dsa" if ((-s "$ENV{'HOME'}/.ssh/id_dsa") && ($dsa != 0) && ($create eq 0));
5027        $keyfile = "$ENV{'HOME'}/.ssh/pb_dsa" if ((-s "$ENV{'HOME'}/.ssh/pb_dsa") && ($dsa != 0));
5028    }
5029}
5030if (defined $keyfile) {
5031    pb_log(2,"Using ssh key file $keyfile\n");
5032} else {
5033    pb_log(0, "Unable to find your public ssh key under $ENV{'HOME'}/.ssh\n");
5034}
5035return($keyfile);
5036}
5037
5038
5039# Returns the pid of a running VM command using a specific VM file, and the port used if the VM exists
5040sub pb_check_ps {
5041    my $vmcmd = shift;
5042    my $vmm = shift;
5043    my $vexist = 0;             # FALSE by default
5044    my $vmport = undef;         # NONE by default
5045
5046    if ($vmcmd =~ /qemu/) {
5047        # Remove the suffix as qemu changes it at launch
5048        $vmcmd =~ s/qemu-.*/qemu/;
5049    }
5050    open(PS, "ps auxhww|") || confess "Unable to call ps";
5051    while (<PS>) {
5052        next if (! /$vmcmd/);
5053        next if (! /$vmm/);
5054        my $void1;
5055        ($void1, $vexist, $vmport) = split(/ +/,$_,3);
5056        pb_log(2,"pb_check_ps $vmport\n");
5057        $vmport =~ s/.*=tcp:127\.0\.0\.1:([0-9]+)-:22.*/$1/;
5058        chomp($vmport);
5059        pb_log(2,"pb_check_ps found a VM $vexist using port $vmport\n");
5060        last;
5061    }
5062    return($vexist,$vmport);
5063}
5064
5065
5066sub pb_extract_build_files {
5067
5068my $src=shift;
5069my $dir=shift;
5070my $ddir=shift;
5071my $mandatory=shift || "build";
5072
5073my $flag = "mayfail" if (($mandatory eq "patch") || ($mandatory eq "src"));
5074my $res;
5075
5076if ($src =~ /tar\.gz$/) {
5077    $res = pb_system("tar xfpz $src $dir","Extracting $mandatory files from $src",$flag);
5078} elsif ($src =~ /tar\.bz2$/) {
5079    $res = pb_system("tar xfpj $src $dir","Extracting $mandatory files from $src",$flag);
5080} else {
5081    confess "Unknown compression algorithm for $src";
5082}
5083# If not mandatory return now
5084return() if (($res != 0) and (($mandatory eq "patch") || ($mandatory eq "src")));
5085return(pb_move_extracted_files($dir,$ddir));
5086}
5087
5088sub pb_move_extracted_files {
5089
5090my $dir = shift;
5091my $ddir = shift;
5092my @files;
5093
5094# In a recursive function , we need a local var as DIR handle
5095my $bdir;
5096
5097opendir($bdir,"$dir") || (cluck "Unable to open directory $dir: $!" && return);
5098foreach my $f (readdir($bdir)) {
5099    next if ($f =~ /^\./);
5100    # Skip potential patch dir
5101    next if ($f =~ /^pbpatch/);
5102    # Skip potential source dir
5103    next if ($f =~ /^pbsrc/);
5104    # Skip potential backup files
5105    next if ($f =~ /~$/);
5106    if ((-d "$dir/$f") && (! -l "$dir/$f")) {
5107        # Case of the bug debian subdir
5108        pb_mkdir_p("$ddir/$f");
5109        push @files,pb_move_extracted_files("$dir/$f","$ddir/$f");
5110    } else {
5111        move("$dir/$f","$ddir") || confess "Unable to move $dir/$f to $ddir";
5112        pb_log(2,"mv $dir/$f $ddir\n");
5113        push @files,"$ddir/$f";
5114    }
5115}
5116closedir($bdir);
5117# Not enough but still a first cleanup
5118pb_rm_rf("$dir");
5119return(@files);
5120}
5121
5122sub pb_list_bfiles {
5123
5124my $dir = shift;
5125my $pbpkg = shift;
5126my $bfiles = shift;
5127my $pkgfiles = shift;
5128my $supfiles = shift;
5129# subdir to keep if recursive mode, empty by default
5130my $subdir = shift || "";
5131# In a recursive function , we need a local var as DIR handle
5132my $bdir;
5133
5134pb_log(2,"DEBUG: entering pb_list_bfiles in $dir: ".Dumper($bfiles)."\n");
5135opendir($bdir,"$dir") || confess "Unable to open dir $dir: $!";
5136foreach my $f (readdir($bdir)) {
5137    pb_log(3,"DEBUG: pb_list_bfiles found $f\n");
5138    next if ($f =~ /^\./);
5139    if (-d "$dir/$f") {
5140        # Recurse for directories (Debian 3.0 format e.g.)
5141        pb_log(2,"DEBUG: pb_list_bfiles recurse in $dir/$f\n");
5142        pb_list_bfiles("$dir/$f",$pbpkg,$bfiles,$pkgfiles,$supfiles,$f);
5143        next;
5144    }
5145
5146    my $key = $f;
5147