source: devel/pb/bin/pb

Last change on this file was 2450, checked in by bruno, 2 weeks ago

Improve newver

  • Fix #140 where one extra line was generated in the pbcl
  • Improve testver hanfling in YAML
  • Property svn:executable set to *
File size: 187.8 KB
Line 
1#!/usr/bin/perl -w
2#
3# Project Builder main application
4#
5# $Id$
6#
7# Copyright B. Cornec 2007-today
8# Eric Anderson's changes are (c) Copyright 2012 Hewlett Packard
9# Provided under the GPL v2
10
11# Syntax: see at end
12
13use strict 'vars';
14
15# The modules mentioned here are required by pb when used both
16# locally or inside a VE/VM/RM
17# Additional required modules only used locally are called with a require
18# in their respective section
19use Getopt::Long qw(:config auto_abbrev no_ignore_case);
20use Carp qw/confess cluck/;
21use Data::Dumper;
22use English;
23use File::Basename;
24use File::Copy;
25use File::stat;
26use File::Find;
27use Time::localtime qw(localtime);
28use POSIX qw(strftime);
29use lib qw (lib);
30use ProjectBuilder::Version;
31use ProjectBuilder::Base;
32use ProjectBuilder::Display;
33use ProjectBuilder::Conf;
34use ProjectBuilder::Distribution;
35use ProjectBuilder::VCS;
36use ProjectBuilder::CMS;
37use ProjectBuilder::Env;
38use ProjectBuilder::Filter;
39use ProjectBuilder::Changelog;
40use ProjectBuilder::VE;
41
42# Global variables
43$Global::pb_stop_on_error = 0;      # False by default
44$Global::pb_show_sudo = 0;          # False by default
45
46my %opts;                           # CLI Options
47my $action;                         # action to realize
48my $test = "FALSE";                 # Not used
49my $pbforce = 0;                    # Force VM stop
50my $pbsnap = 0;                     # Do not use snapshot mode for VM/VE by default
51my $pbkeep = 0;                     # keep temporary directory at the end
52my $option = "";                    # Not used
53my @pkgs;                           # list of packages
54my $pbtag;                          # Global Tag variable
55my $pbver;                          # Global Version variable
56my %pbscript;                       # Name of the scripts per V
57my $pbrev;                          # Global REVISION variable
58my $pbtarget = undef;               # Target os-ver-arch you want to build for
59my $pbport;                         # Port to use to connect to the VM/RM
60my $newver;                         # New version to create
61my $pbimage = undef;                # ISO image for the VM to create or docker image for the VE
62my $vetype = undef;                 # Type of Virtual environment to deal with
63
64my @date = pb_get_date();
65my $pbdate = strftime("%Y-%m-%d", @date);
66
67=pod
68
69=head1 NAME
70
71pb, aka project-builder.org - builds packages for your projects
72
73=head1 DESCRIPTION
74
75pb helps you build various packages directly from your project sources.
76Those sources could be handled by a CMS (Configuration Management System)
77such as Subversion, CVS, Git, Mercurial... or being a simple reference to a compressed tar file.
78It's based on a set of configuration files, a set of provided macros to help
79you keeping build files as generic as possible. For example, a single .spec
80file should be required to generate for all rpm based distributions, even
81if you could also have multiple .spec files if required.
82
83=head1 SYNOPSIS
84
85pb [-vhSq][-r pbroot][-p project][[-s script -a account -P port][-T VEtype][-t [os-ver-arch]][-m os-ver-arch[,...]]][-g][-i image] <action> [<pkg1> ...]
86
87pb [--verbose][--help][--man][--quiet][--snapshot][--revision pbroot][--project project][[--script script --account account --port port][-T VEtype][--target [os-ver-arch]][--machine os-ver-arch[,...]]][--nographic][--image image][--rebuild] <action> [<pkg1> ...]
88
89=head1 OPTIONS
90
91=over 4
92
93=item B<-v|--verbose>
94
95Increase verbosity
96
97=item B<-q|--quiet>
98
99Do not print any output.
100
101=item B<-h|--help>
102
103Print a brief help message and exits.
104
105=item B<-S|--snapshot>
106
107Use the snapshot mode of VMs or VEs
108
109=item B<--man>
110
111Prints the manual page and exits.
112
113=item B<-t|--target os-ver-arch>
114
115Name of the target system you want to build for.
116All if none precised.
117
118=item B<-m|--machine os-ver-arch[,os-ver-arch,...]>
119
120Name of the Virtual Machines (VM), Virtual Environments (VE) or Remote Machines (RM)
121you want to build on (comma separated).
122All if none precised (or use the env variable PBV).
123
124=item B<-T|--vetype VEtype]>
125
126Type of Virtual Environments (VE)
127Can be chroot or docker.
128
129=item B<-s|--script script>
130
131Name of the script you want to execute on the related VMs/VEs/RMs.
132
133=item B<-g|--nographic>
134
135Do not launch VMs in graphical mode.
136
137=item B<-i|--image image>
138
139It could be either:
140- The name of the ISO image of the distribution you want to install on the related VMs
141- The name of the docker image of the distribution you want to install on the related VEs
142
143
144=item B<-a|--account account>
145
146Name of the account to use to connect on the related VMs/RMs.
147
148=item B<-P|--port port_number>
149
150Port number to use to connect on the related VMs/RMs.";
151
152=item B<-p|--project project_name>
153
154Name of the project you're working on (or use the env variable PBPROJ)
155
156=item B<-r|--revision revision>
157
158Path Name of the project revision under the CMS (or use the env variable PBROOT)
159
160=item B<-V|--version new_version>
161
162New version of the project to create based on the current one.
163
164=item B<-k|--keep>
165
166Keep the temporary dir where files have been created in or der to help debug
167
168=item B<--rebuild>
169
170Only valid with the checkssh action, it alllows to automatically relaunch the build of the failed packages
171
172=item B<--no-stop-on-error>
173
174Continue through errors with best effort.
175
176=back
177
178=head1 ARGUMENTS
179
180<action> can be:
181
182=over 4
183
184=item B<sbx2build>
185
186Create tar files for the project under your CMS.
187Current state of the exported content is taken.
188CMS supported are SVN, SVK, CVS, Git and Mercurial
189parameters are packages to build
190if not using default list
191
192=item B<cms2build>
193
194Create tar files for the project under your CMS.
195Current state of the CMS is taken.
196CMS supported are SVN, SVK, CVS, Git and Mercurial
197parameters are packages to build
198if not using default list
199
200=item B<build2prep>
201
202Prepare the environment for build by installing required dependencies. Done once on the build system.
203
204=item B<build2pkg>
205
206Create packages for your running distribution
207
208=item B<cms2pkg>
209
210cms2build + build2pkg
211
212=item B<sbx2pkg>
213
214sbx2build + build2pkg
215
216=item B<sbx2pkg2ins>
217
218sbx2pkg + final install of packages
219
220=item B<sbx2prep>
221
222sbx2build + build2prep
223
224=item B<build2ssh>
225
226Send the tar files to a SSH host
227
228=item B<sbx2ssh>
229
230sbx2build + build2ssh
231
232=item B<cms2ssh>
233
234cms2build + build2ssh
235
236=item B<pkg2ssh>
237
238Send the packages built to a SSH host
239
240=item B<build2vm>
241
242Create packages in VMs, launching them if needed
243and send those packages to a SSH host once built
244VM type supported are QEMU and KVM
245
246=item B<build2ve>
247
248Create packages in VEs, creating it if needed
249and send those packages to a SSH host once built
250
251=item B<build2rm>
252
253Create packages in RMs, which should pre-exist,
254and send those packages to a SSH host once built
255RM means Remote Machine, and could be a physical or Virtual one.
256This is one buildfarm integration for pb.
257
258=item B<prepvm>
259
260Prepare the VMs to have all requirements to build the project
261
262=item B<prepve>
263
264Prepare the VEs to have all requirements to build the project
265
266=item B<preprm>
267
268Prepare the RMs to have all requirements to build the project
269
270=item B<sbx2vm>
271
272sbx2build + build2vm
273
274=item B<sbx2ve>
275
276sbx2build + build2ve
277
278=item B<sbx2docker>
279
280sbx2build + build2ve with a potential build of all necessary docker containers to perform it
281
282=item B<sbx2rm>
283
284sbx2build + build2rm
285
286=item B<cms2vm>
287
288cms2build + build2vm
289
290=item B<cms2ve>
291
292cms2build + build2ve
293
294=item B<cms2rm>
295
296cms2build + build2rm
297
298=item B<launchvm>
299
300Launch one virtual machine
301
302=item B<launchve>
303
304Launch one virtual environment
305
306=item B<script2vm>
307
308Launch one virtual machine if needed
309and executes a script on it
310
311=item B<script2ve>
312
313Execute a script in a virtual environment
314
315=item B<script2rm>
316
317Execute a script on a remote machine
318
319=item B<newvm>
320
321Create a new virtual machine
322
323=item B<newve>
324
325Create a new virtual environment
326
327=item B<setupvm>
328
329Setup a virtual machine for pb usage
330
331=item B<setupve>
332
333Setup a virtual environment for pb usage
334
335=item B<setuprm>
336
337Setup a remote machine for pb usage
338
339=item B<sbx2setupvm>
340
341Setup a virtual machine for pb usage using the sandbox version of pb instead of the latest stable
342Reserved to dev team.
343
344=item B<sbx2setupve>
345
346Setup a virtual environment for pb usage using the sandbox version of pb instead of the latest stable
347Reserved to dev team.
348
349=item B<sbx2setuprm>
350
351Setup a remote machine for pb usage using the sandbox version of pb instead of the latest stable
352Reserved to dev team.
353
354=item B<build2setupvm>
355
356Setup a virtual machine for pb usage using the build available
357Reserved to dev team.
358
359=item B<build2setupve>
360
361Setup a virtual environment for pb usage using the build available
362Reserved to dev team.
363
364=item B<build2setuprm>
365
366Setup a remote machine for pb usage using the build available
367Reserved to dev team.
368
369=item B<snapvm>
370
371Snapshot a virtual machine for pb usage
372
373=item B<snapve>
374
375Snapshot a virtual environment for pb usage
376
377=item B<updateconf>
378
379Update the configuration files to the latest format
380
381=item B<updatevm>
382
383Update the distribution in the virtual machine
384
385=item B<updateve>
386
387Update the distribution in the virtual environment
388
389=item B<updaterm>
390
391Update the distribution in the remote machine
392
393=item B<install2pkg>
394
395Install the project packages locally
396
397=item B<test2pkg>
398
399Test the project packages locally
400
401=item B<install2vm>
402
403Install the project packages in a virtual machine
404
405=item B<install2ve>
406
407Install the project packages in a virtual environment
408
409=item B<install2rm>
410
411Install the project packages in a remote machine
412
413=item B<test2vm>
414
415Test the project packages in a virtual machine, after installation
416
417=item B<test2ve>
418
419Test the project packages in a virtual environment, after installation
420
421=item B<test2rm>
422
423Test the project packages in a remote machine, after installation
424
425=item B<checkssh>
426
427Check the delivery of the packages on the repository
428
429=item B<checkps>
430
431Check the process running the VM concerned
432
433=item B<newver>
434
435Create a new version of the project derived
436from the current one
437
438=item B<newproj>
439
440Create a new project and a template set of
441configuration files under pbconf
442
443=item B<announce>
444
445Announce the availability of the project through various means
446
447=item B<sbx2webssh>
448
449Create tar files for the website under your CMS.
450Current state of the exported content is taken.
451Deliver the content to the target server using ssh from the exported dir.
452
453=item B<cms2webssh>
454
455Create tar files for the website from your CMS.
456Deliver the content to the target server using ssh from the DVCS.
457
458=item B<sbx2webpkg>
459
460Create tar files for the website under your CMS.
461Current state of the exported content is taken.
462
463=item B<cms2webpkg>
464
465Create tar files for the website under your CMS.
466
467=item B<getconf>
468
469Print the full configuration parameters as found in the various configuration files. Help to debug conf issues.
470Also accepts a parameter to display only this value, and a VM/VE/RM
471
472=item B<getvar>
473
474Print the full variables expanded based on the distribution tuple. Help to debug conf issues.
475Also accepts a parameter to display only the values for this package, and a VM/VE/RM
476
477=item B<clean>
478
479Purge the build and delivery directories related to the current project
480
481=item B<cleanssh>
482
483Purge the ssh server of its packages (only for testver and test packages)
484
485=back
486
487<pkgs> can be a list of packages, the keyword 'all' or nothing, in which case the default list of packages is taken (corresponding to the defpkgdir list of arguments in the configuration file).
488
489=head1 WEB SITES
490
491The main Web site of the project is available at L<http://www.project-builder.org/>. Bug reports should be filled using the trac instance of the project at L<http://trac.project-builder.org/>.
492
493=head1 USER MAILING LIST
494
495None exists for the moment.
496
497=head1 CONFIGURATION FILES
498
499Each pb user may have a configuration in F<$HOME/.pbrc.yml>. The values in this file may overwrite any other configuration file value.
500
501Here is an example of such a configuration file:
502
503 #
504 # Define for each project the URL of its pbconf repository
505 # No default option allowed here as they need to be all different
506 #
507 # URL of the pbconf content
508 # This is the format of a classical URL with the extension of additional schema such as
509 # svn+ssh, cvs+ssh, ...
510 #
511  pbconfurl:
512    linuxcoe : cvs+ssh://:ext:bcornec@linuxcoe.cvs.sourceforge.net:/cvsroot/linuxcoe/pbconf
513
514 # This is normaly defined in the project's configuration file
515 # Url of the project
516 #
517  pbprojurl:
518    linuxcoe : cvs+ssh://:ext:bcornec@linuxcoe.cvs.sourceforge.net:/cvsroot/linuxcoe
519 
520 # All these URLs needs to be defined here as the are the entry point
521 # for how to build packages for the project
522 #
523  pbconfurl:
524    pb: svn+ssh://svn.project-builder.org/mondo/svn/pb/pbconf
525  pbconfurl:
526    mondorescue: svn+ssh://svn.project-builder.org/mondo/svn/project-builder/mondorescue/pbconf
527  pbconfurl:
528    collectl: svn+ssh://bruno@svn.mondorescue.org/mondo/svn/project-builder/collectl/pbconf
529  pbconfurl:
530    netperf: svn+ssh://svn.mondorescue.org/mondo/svn/project-builder/netperf/pbconf
531 
532 # Under that dir will take place everything related to pb
533 # If you want to use VMs/chroot/..., then use $ENV{'HOME'} to make it portable
534 # to your VMs/chroot/...
535 # if not defined then /var/cache
536  pbdefdir:
537    default: $ENV{'HOME'}/project-builder
538  pbdefdir:
539    pb: $ENV{'HOME'}
540  pbdefdir:
541    linuxcoe: $ENV{'HOME'}/LinuxCOE/cvs
542  pbdefdir:
543    mondorescue: $ENV{'HOME'}/mondo/svn
544 
545 # pbconfdir points to the directory where the CMS content of the pbconfurl is checked out
546 # If not defined, pbconfdir is under pbdefdir/pbproj/pbconf
547  pbconfdir:
548    linuxcoe: $ENV{'HOME'}/LinuxCOE/cvs/pbconf
549  pbconfdir:
550    mondorescue: $ENV{'HOME'}/mondo/svn/pbconf
551 
552 # pbdir points to the directory where the CMS content of the pbprojurl is checked out
553 # If not defined, pbdir is under pbdefdir/pbproj
554 # Only defined if we have access to the dev of the project
555  pbdir:
556    linuxcoe: $ENV{'HOME'}/LinuxCOE/cvs
557  pbdir:
558    mondorescue: $ENV{'HOME'}/mondo/svn
559 
560 # -daemonize doesn't work with qemu 0.8.2
561  vmopt:
562    default: -m 384
563
564=head1 COMMAND DETAILS
565
566=head2 newproj
567
568The newproj command creates a new project-builder project.  To run this command you first need to define two variables in your ~/.pbrc.yml file:
569
570 pbconfurl I<project> = file:///home/anderse/.git/project-builder-config/I<project>
571 pbdefdir default = $ENV{'HOME'}/cache-project-builder
572
573The first line defines the version controlled configuration information and the second defines the root directory for project-builder to use.
574
575You can then run the command:
576
577 % pb -p I<$project> -r I<$version> newproj I<$pkg>
578
579to create the new project. Running the newproj command will then generate the file $pbdefdir/$project/pbconf/$version/$project.yml, and the directory $pbdefdir/$project/pbconf/$version/$pkg. You will need to edit those files to make the later commands work.
580
581=head2 cms2build
582
583The cms2build command takes your files from the content management system and makes the two tar files that are necessary for building files. You need to have run the newproj command first.  Then there are several steps for running this command:
584
585=over 4
586
587=item Update your $project.yml configuration file.
588
589You need to set the pbprojurl, pbrepo, pbwf, pbpackager, projver, projtag, testver, delivery, and defpkgdir lines as described in the configuration file. The pbprojurl entry is used to find the source for your package. The pbrepo entry is used to build the .repo or .sources.list files for use by downloaders of the package. The pbwf entry indicates that the source tar file is named by I<package-name>-I<version>. The pbpackager entry will be stored in the packages and should be you or your team. The projver/projtag entries indicate the version of the software and the version of the packaging scripts. The testver entry when true indicates that the package is in a test version, so no log file is computed (can be long), and version is made up using a timstamp. The delivery entry gives the subdirectory under which the packages will be delivered on the repository, and the defpkgdir entry corresponds to the local subdirectory hosting the package content.
590
591For example:
592
593 pbprojurl Lintel = file:///home/anderse/projects/Lintel-0.2012.02.28.tar.gz
594 pbrepo Lintel = http://tesla.hpl.hp.com/opensource
595 pbwf Lintel = 1
596 pbpackager Lintel = Eric Anderson <eric.anderson4@hp.com>
597 projver Lintel = 0.2012.02.28
598 projtag Lintel = 1
599 testver Lintel = false
600 delivery Lintel = production
601 defpkgdir Lintel = Lintel-0.2012.02.28
602
603=item Create the build .tar.gz files:
604
605Then you need to take those files and create the initial tar files.  Run a command like:
606
607 % pb -p $project -r $version cms2build
608
609To create the $pbdefdir/$project/pbdelivery/$project-$version.{,pbconf}.tar.gz files, the $version-$projtag.yml and pbrc.yml files in the same directory.
610
611=back
612
613=head2 build2pkg
614
615The build2pkg command takes the tar files created in the cms2build step and attempts to build binary packages for your current operating system. There are two steps:
616
617=over 4
618
619=item Update your filters and build files.
620
621You probably need to edit the files describing the build steps in one of the $pbdefdir/$project/pbconf/$version/$project/{deb,rpm,pkg} directories and the filters in $pbdefdir/$project/pbconf/$version/pbfilter. Note that you can define additional filters and transformations in the filters files. The build files will be filtered by the filters defined in the filters files to generate the inputs to the build step. Therefore, if you change those files, you need to re-run the cms2build step.
622
623=item Build the package.
624
625Then you can run a command like:
626
627 % pb -p $project -r $version build2pkg
628
629To create the files in $project/pbbuild that comprise your binary package(s).
630
631=back
632
633=head2 newve
634 
635The newve command creates a new virtual environment, i.e. a chrooted OS for building packages. Using a virtual environment is an efficient way to build packages on a related set of operating systems. The OS's have to be related because the kernel will be shared.  Steps:
636
637=over 4
638
639=item Update ~/.pbrc.yml
640
641Update your ~/.pbrc.yml file to specify the vepath, velist, velogin, and vetype variables, e.g.:
642
643  vepath:
644    default: $ENV{'HOME'}/cache-project-builder/chroot
645  velist:
646    default: debian-6.0-i386
647  velogin:
648    default: pb
649  vetype:
650    default: chroot
651
652If you are building for rpm style OS's, update the verpmtype option, and install the appropriate tool.
653
654  verpmtype:
655    default: rpmbootstrap
656
657You may also choose to specify a mirror for the OS packages, and optionally http/ftp proxies.  You can specify the proxies either through environment variables ($http_proxy/$https_proxy/$ftp_proxy) or in the configuration file. The configuration file will be used if no corresponding environment variable has been set. For example, for debian and with a local squid proxy:
658
659  rbsmirrorsrv:
660    debian: http://mirrors1.kernel.org/debian/
661  http_proxy:
662    default: http://localhost:3128/
663  https_proxy:
664    default: http://localhost:3128/
665  ftp_proxy:
666    default: http://localhost:3128/
667
668=item Run the cms2build command
669
670If you have deleted your $package/pbdelivery directory, re-run the cms2build command as in the earlier step. This step is necessary to generate the I<package>/pbdelivery/pbrc.yml file.
671
672=item Create the new virtual environment
673
674Initialize the new operating system. This step will install the core OS packages for the virtual environment, e.g.:
675
676 % pb -v -p $project -m debian-6.0-i386 newve
677
678=back
679
680=head2 setupve
681
682The setupve command prepares a virtual environment for use by project builder. In particular it installs project-builder from the packages into the virtual environment. Two sub-steps are necessary:
683
684=over 4
685
686=item Update $project.yml
687
688You need to have a sshhost entry for setupve to work, so add one, even an invalid one, e.g.:
689
690  sshhost:
691    $project: foo.example.org
692
693=item Setup the virtual environment
694
695 % pb -v -p $project -m debian-6.0-i386 setupve
696
697If you prefer to install the current SVN version of project builder, you can substitute the setupve option by the sbx2setupv one.
698
699=back
700
701=head2 build2ve
702
703The build2ve command is similar to the build2pkg command in that it will take the sources created by cms2build and turn them into binary packages. The command has two differences. First, it creates the packages in a virtual environment, i.e. the one made by an earlier setupve setup. Second it copies the resulting packages to a repository and builds the repository meta-data needed.
704
705Three sub-steps are needed:
706
707=over 4
708
709=item Update $project.yml
710
711You need to have a valid sshdir and sshhost entry for build2ve to work, so add them. Note that you need to be able to ssh from the host you run the command on to the repository host, preferably without needing to type in a password, so using ssh-agent or having a special passwordless project-builder ssh key will make this step easier.
712
713  sshhost:
714    $project: localhost
715  sshdir:
716    $project: $home/cache-project-builder/repos
717
718You may also need to specify additional repository files to use or rpms to install. Note the URL for repositories is not the URL of the repository, but the URL of a file that can be put in the yum.repos.d or apt.sources.d directory.
719
720  addbuildrepo:
721    centos-5-i386: http://localhost/pb/centos-extras.repo,http://mirror.centos.org/centos/5/extras/i386/RPMS/chrpath-0.13-3.el5.centos.i386.rpm
722
723=item Update your filters and build files
724
725You may need to update your filter files as in the build2pkg step if you are building for a new OS or architecture.
726
727=item Build the packages and copy them to the repository
728
729 % pb -v -p $project -m debian-6.0-i386 build2ve
730
731=back
732
733*Debugging:* If the build fails (and you did not specify the --no-stop-on-error) option, then the virtual environment and scripts should still be present and configured to build the package. You can run a command like 'sudo setarch i386 chroot $path bash' in order to get into the environment. In your log you should see a command like that. From there you can go into the /home/pb directory as the pb user and run the same style of pb commands as you did when doing build2pkg. This will help you figure out what has gone wrong in the build in the virtual environment.
734
735=head1 AUTHORS
736
737The Project-Builder.org team L<http://trac.project-builder.org/> lead by Bruno Cornec L<mailto:bruno@project-builder.org>.
738
739=head1 COPYRIGHT
740
741Project-Builder.org is distributed under the GPL v2.0 license
742described in the file C<COPYING> included with the distribution.
743
744=cut
745
746# ---------------------------------------------------------------------------
747
748my ($projectbuilderver,$projectbuilderrev,$projectbuilderconfver) = pb_version_init();
749my $appname = "pb";
750
751# Initialize the syntax string
752
753pb_syntax_init("$appname (aka project-builder.org) Version $projectbuilderver-$projectbuilderrev Conf $projectbuilderconfver\n");
754
755GetOptions("help|?|h+" => \$opts{'h'}, 
756        "account|a=s" => \$opts{'a'},
757        "force|f" => \$opts{'f'},
758        "nographic|g" => \$opts{'g'},
759        "image|i=s" => \$opts{'i'},
760        "log-files|l=s" => \$opts{'l'},
761        "machines|mock|m=s" => \$opts{'m'},
762        "man" => \$opts{'man'},
763        "port|P=i" => \$opts{'P'},
764        "project|p=s" => \$opts{'p'},
765        "quiet|q" => \$opts{'q'},
766        "revision|r=s" => \$opts{'r'},
767        "rebuild" => \$opts{'rebuild'},
768        "snapshot|S" => \$opts{'S'},
769        "script|s=s" => \$opts{'s'},
770        "type|T:s" => \$opts{'T'},
771        "target|t:s" => \$opts{'t'},
772        "version" => \$opts{'version'},
773        "V=s" => \$opts{'V'},
774        "verbose|v+" => \$opts{'v'},
775        "keep|k" => \$opts{'k'},
776        "stop-on-error!" => \$Global::pb_stop_on_error,
777) || pb_syntax(-1,0);
778
779if (defined $opts{'l'}) {
780    open(pbLOG,"> $opts{'l'}") || confess "Unable to log to $opts{'l'}: $!";
781    $pbLOG = \*pbLOG;
782    $pbdebug = 0  if ($pbdebug == -1);
783    }
784pb_log_init($pbdebug, $pbLOG);
785
786if (defined $opts{'h'}) {
787    pb_syntax(0,1);
788}
789if (defined $opts{'version'}) {
790    pb_syntax(0,0);
791}
792if (defined $opts{'man'}) {
793    pb_syntax(0,2);
794}
795if (defined $opts{'v'}) {
796    $pbdebug = $opts{'v'};
797}
798if (defined $opts{'f'}) {
799    $pbforce=1;
800}
801if (defined $opts{'q'}) {
802    $pbdebug=-1;
803}
804if (defined $opts{'S'}) {
805    $pbsnap=1;
806}
807if (defined $opts{'k'}) {
808    $pbkeep=1;
809}
810pb_display_init("text","");
811
812# Handle root of the project if defined
813if (defined $opts{'r'}) {
814    $ENV{'PBROOTDIR'} = $opts{'r'};
815}
816# Handle virtual machines if any
817if (defined $opts{'m'}) {
818    $ENV{'PBV'} = $opts{'m'};
819}
820if (defined $opts{'s'}) {
821    $pbscript{'default'} = $opts{'s'};
822    confess "option -s requires a script having a full path name" if ($pbscript{'default'} !~ /^\//);
823}
824if (defined $opts{'a'}) {
825    $ENV{'PBACCOUNT'} = $opts{'a'};
826    confess "option -a requires a -s script option" if (not defined $pbscript{'default'});
827}
828if (defined $opts{'P'}) {
829    $pbport = $opts{'P'};
830}
831if (defined $opts{'V'}) {
832    $newver = $opts{'V'};
833}
834if (defined $opts{'i'}) {
835    $pbimage = $opts{'i'};
836}
837if (defined $opts{'t'}) {
838    $pbtarget = $opts{'t'};
839}
840if (defined $opts{'T'}) {
841    $vetype = $opts{'T'};
842}
843
844# Get Action
845$action = shift @ARGV;
846confess pb_syntax(-1,1) if (not defined $action);
847
848my ($filteredfiles, $supfiles, $defpkgdir, $extpkgdir);
849my $pbinit = undef;
850$pbinit = 1 if ($action =~ /^newproj$/);
851
852# Handles project name if any
853# And get global params
854($filteredfiles, $supfiles, $defpkgdir, $extpkgdir) = pb_env_init($opts{'p'},$pbinit,$action,$pbkeep);
855
856#
857# Check for command requirements
858#
859my ($req,$opt,$pbpara,$pbshowsudo,$pbstoponerr) = pb_conf_get_if("oscmd","oscmdopt","pbparallel","pbshowsudo","pbstoponerr");
860pb_check_requirements($req,$opt,$appname);
861
862$Global::pb_show_sudo = 1 if ((defined $pbshowsudo) && (defined $pbshowsudo->{$ENV{'PBPROJ'}}) && ($pbshowsudo->{$ENV{'PBPROJ'}} =~ /true/oi));
863$Global::pb_stop_on_error = 1 if ((defined $pbstoponerr) && (defined $pbstoponerr->{$ENV{'PBPROJ'}}) && ($pbstoponerr->{$ENV{'PBPROJ'}} =~ /true/oi));
864
865#
866# Check if we can launch some actions in // with Parallel::ForkManager
867#
868my $pbparallel = $pbpara->{$appname} if (defined $pbpara);
869if (not defined $pbparallel) {
870    eval
871    {
872        require Sys::CPU;
873        Sys::CPU->import();
874    };
875    if ($@) {
876        # Sys::CPU not found, defaulting to 1
877        pb_log(1,"ADVISE: Install Sys::CPU to benefit from automatic parallelism optimization.\nOr use pbparallel in your pb.yml file\nOnly 1 process at a time for the moment\n");
878        $pbparallel = 1;
879    } else {
880        # Using the number of cores
881        $pbparallel = Sys::CPU::cpu_count();
882        pb_log(1,"Using parallel mode with $pbparallel processes\n");
883    }
884}
885
886eval
887{
888    require Parallel::ForkManager;
889    Parallel::ForkManager->import();
890};
891# Parallel::ForkManager not found so no // actions
892if ($@) {
893    $pbparallel = undef;
894    pb_log(1,"ADVISE: Install Parallel::ForkManager to benefit from automatic parallelism optimization.\nOnly 1 process at a time for the moment\n");
895}
896
897if ($action ne "getconf") {
898    pb_log(0,"Project: $ENV{'PBPROJ'}\n");
899    pb_log(0,"Action: $action\n");
900}
901
902my $do_install = undef;
903# Possibility to add a 2ins to some commands to launch automatic installation
904$do_install = 1 if ($action =~ /2ins$/);
905
906# Act depending on action
907if ($action =~ /^cms2build$/) {
908    pb_cms2build("CMS");
909} elsif ($action =~ /^sbx2build$/) {
910    pb_cms2build("SandBox");
911} elsif ($action =~ /^build2prep$/) {
912    pb_build2prep();
913} elsif ($action =~ /^sbx2prep$/) {
914    pb_cms2build("SandBox");
915    pb_build2prep();
916} elsif ($action =~ /^build2pkg/) {
917    pb_build2pkg($do_install);
918} elsif ($action =~ /^cms2pkg/) {
919    pb_cms2build("CMS");
920    pb_build2pkg($do_install);
921} elsif ($action =~ /^sbx2pkg/) {
922    pb_cms2build("SandBox");
923    pb_build2pkg($do_install);
924} elsif ($action =~ /^build2ssh$/) {
925    pb_build2ssh();
926} elsif ($action =~ /^cms2ssh$/) {
927    pb_cms2build("CMS");
928    pb_build2ssh();
929} elsif (($action =~ /^sbx2ssh$/) || ($action =~ /^sbx2pkg2ssh$/)) {
930    pb_cms2build("SandBox");
931    pb_build2ssh();
932} elsif ($action =~ /^pkg2ssh$/) {
933    pb_pkg2ssh();
934} elsif ($action =~ /^build2rm$/) {
935    pb_build2v("rm","build");
936} elsif ($action =~ /^build2ve$/) {
937    pb_build2v("ve","build");
938} elsif ($action =~ /^build2vm$/) {
939    pb_build2v("vm","build");
940} elsif ($action =~ /^sbx2preprm$/) {
941    pb_cms2build("SandBox");
942    pb_build2v("rm","prep");
943} elsif ($action =~ /^sbx2prepve$/) {
944    pb_cms2build("SandBox");
945    pb_build2v("ve","prep");
946} elsif ($action =~ /^sbx2prepvm$/) {
947    pb_cms2build("SandBox");
948    pb_build2v("vm","prep");
949} elsif ($action =~ /^preprm$/) {
950    pb_build2v("rm","prep");
951} elsif ($action =~ /^prepve$/) {
952    pb_build2v("ve","prep");
953} elsif ($action =~ /^prepvm$/) {
954    pb_build2v("vm","prep");
955} elsif ($action =~ /^cms2rm$/) {
956    pb_cms2build("CMS");
957    pb_build2v("rm","build");
958} elsif ($action =~ /^cms2ve$/) {
959    pb_cms2build("CMS");
960    pb_build2v("ve","build");
961} elsif ($action =~ /^sbx2rm$/) {
962    pb_cms2build("SandBox");
963    pb_build2v("rm","build");
964} elsif ($action =~ /^sbx2ve$/) {
965    pb_cms2build("SandBox");
966    pb_build2v("ve","build");
967} elsif ($action =~ /^cms2vm$/) {
968    pb_cms2build("CMS");
969    pb_build2v("vm","build");
970} elsif ($action =~ /^sbx2vm$/) {
971    pb_cms2build("SandBox");
972    pb_build2v("vm","build");
973} elsif ($action =~ /^install2vm$/) {
974    pb_build2v("vm","install");
975} elsif ($action =~ /^install2ve$/) {
976    pb_build2v("ve","install");
977} elsif ($action =~ /^install2rm$/) {
978    pb_build2v("rm","install");
979} elsif ($action =~ /^launchvm$/) {
980    pb_parallel_launchv(undef,"vm",undef,3,$pbimage);
981} elsif ($action =~ /^launchve$/) {
982    pb_parallel_launchv(undef,"ve",undef,3,$pbimage);
983} elsif ($action =~ /^script2vm$/) {
984    confess "action script2vm requires a -s script option" if (not defined $pbscript{'default'});
985    pb_script2v(\%pbscript,"vm");
986} elsif ($action =~ /^script2ve$/) {
987    confess "action script2ve requires a -s script option" if (not defined $pbscript{'default'});
988    pb_script2v(\%pbscript,"ve");
989} elsif ($action =~ /^script2rm$/) {
990    confess "action script2rm requires a -s script option" if (not defined $pbscript{'default'});
991    pb_script2v(\%pbscript,"rm");
992} elsif ($action =~ /^newver$/) {
993    pb_newver();
994} elsif ($action =~ /^sbx2docker$/) {
995    pb_cms2build("SandBox");
996    pb_all_docker($action);
997} elsif ($action =~ /^build2docker$/) {
998    pb_all_docker($action);
999} elsif ($action =~ /^newve$/) {
1000    pb_parallel_launchv(undef,"ve",undef,0,$pbimage);
1001} elsif ($action =~ /^newvm$/) {
1002    pb_parallel_launchv(undef,"vm",undef,0,$pbimage);
1003    pb_log(0, "Please ensure that sshd is running in your VM by default\n");
1004    pb_log(0, "and that it allows remote root login (PermitRootLogin yes in /etc/ssh/sshd_config)\n");
1005    pb_log(0, "Also ensure that network is up, firewalling correctly configured\n");
1006    pb_log(0, "and perl-YAML, perl-Date-Manip, tar, sudo, ntpdate, make and scp/ssh installed\n");
1007    pb_log(0, "You should then be able to login with ssh -p VMPORT root\@localhost (if VM started with pb)\n");
1008} elsif ($action =~ /^setuprm$/) {
1009    pb_setup2v("rm");
1010} elsif ($action =~ /^setupve$/) {
1011    pb_setup2v("ve");
1012} elsif ($action =~ /^setupvm$/) {
1013    pb_setup2v("vm");
1014} elsif ($action =~ /^sbx2setuprm$/) {
1015    confess "This feature is limited to the pb project" if ($ENV{'PBPROJ'} ne $appname);
1016    pb_cms2build("SandBox");
1017    pb_setup2v("rm","SandBox");
1018} elsif ($action =~ /^sbx2setupve$/) {
1019    confess "This feature is limited to the pb project" if ($ENV{'PBPROJ'} ne $appname);
1020    pb_cms2build("SandBox");
1021    pb_setup2v("ve","SandBox");
1022} elsif ($action =~ /^sbx2setupvm$/) {
1023    confess "This feature is limited to the pb project" if ($ENV{'PBPROJ'} ne $appname);
1024    pb_cms2build("SandBox");
1025    pb_setup2v("vm","SandBox");
1026} elsif ($action =~ /^build2setuprm$/) {
1027    confess "This feature is limited to the pb project" if ($ENV{'PBPROJ'} ne $appname);
1028    pb_setup2v("rm","SandBox");
1029} elsif ($action =~ /^build2setupve$/) {
1030    confess "This feature is limited to the pb project" if ($ENV{'PBPROJ'} ne $appname);
1031    pb_setup2v("ve","SandBox");
1032} elsif ($action =~ /^build2setupvm$/) {
1033    confess "This feature is limited to the pb project" if ($ENV{'PBPROJ'} ne $appname);
1034    pb_setup2v("vm","SandBox");
1035} elsif ($action =~ /^updaterm$/) {
1036    pb_update2v("rm");
1037} elsif ($action =~ /^updateve$/) {
1038    pb_update2v("ve");
1039} elsif ($action =~ /^updatevm$/) {
1040    pb_update2v("vm");
1041} elsif ($action =~ /^snapve$/) {
1042    pb_snap2v("ve");
1043} elsif ($action =~ /^snapvm$/) {
1044    pb_snap2v("vm");
1045} elsif ($action =~ /^install2pkg$/) {
1046    pb_install2pkg();
1047} elsif ($action =~ /^test2pkg$/) {
1048    pb_test2pkg();
1049} elsif ($action =~ /^test2rm$/) {
1050    pb_build2v("rm","test");
1051} elsif ($action =~ /^test2ve$/) {
1052    pb_build2v("ve","test");
1053} elsif ($action =~ /^test2vm$/) {
1054    pb_build2v("vm","test");
1055} elsif ($action =~ /^newproj$/) {
1056    # Nothing to do - already done in pb_env_init
1057} elsif ($action =~ /^updateconf$/) {
1058    foreach my $f (@ARGV) {
1059        my $fyml = $f;
1060        $fyml =~ s/\.pbf/.yml/;
1061        $fyml =~ s/\.pb/.yml/;
1062        $fyml =~ s/\.pbrc/.pbrc.yml/;
1063        pb_vcs_conf_update_v0($f,$fyml);
1064    }
1065} elsif ($action =~ /^getconf$/) {
1066    my $pbos = pb_distro_get_context($ENV{'PBV'});
1067    pb_log(1,"Arguments to print:".Dumper(@ARGV)."\n");
1068    pb_distro_conf_print($pbos,@ARGV);
1069} elsif ($action =~ /^getvar$/) {
1070    my $pbos = pb_distro_get_context($ENV{'PBV'});
1071    my $pkg;
1072    my @pkg;
1073    if ($#ARGV == -1) {
1074        $pkg = pb_cms_get_pkg($defpkgdir,$extpkgdir);
1075        @pkgs = @$pkg;
1076    } else {
1077        @pkgs = @ARGV;
1078    }
1079    pb_log(1,"Packages to print:".Dumper(@pkgs)."\n");
1080    pb_filter_var_print($pbos,@pkgs);
1081} elsif ($action =~ /^clean$/) {
1082    pb_clean();
1083} elsif ($action =~ /^announce$/) {
1084    # For announce only. Require avoids the systematic load of these modules
1085    require DBI;
1086    require DBD::SQLite;
1087
1088    pb_announce("Announce");
1089} elsif ($action =~ /^cleanssh$/) {
1090    pb_announce("Clean");
1091} elsif ($action =~ /^checkssh$/) {
1092    pb_announce("Check");
1093} elsif ($action =~ /^checkps$/) {
1094    my ($ptr2) = pb_conf_get("vmcmd");
1095    my $vmcmd = $ptr2->{$ENV{'PBPROJ'}};
1096    my ($vexist,$vmmport) = pb_check_ps($vmcmd,$ENV{'PBV'});
1097    if (! $vexist) {
1098        pb_log(0,"No VM found for $ENV{'PBV'}\n");
1099    } else {
1100        pb_log(0,"Found an existing VM for $ENV{'PBV'} PID: $vexist - SSH port: $vmmport\n");
1101    }
1102} elsif ($action =~ /^sbx2webpkg$/) {
1103    require DBI;
1104    require DBD::SQLite;
1105
1106    pb_cms2build("SandBox","Web");
1107} elsif ($action =~ /^sbx2webssh$/) {
1108    require DBI;
1109    require DBD::SQLite;
1110
1111    pb_cms2build("SandBox","Web");
1112    pb_send2target("Web");
1113} elsif ($action =~ /^cms2webpkg$/) {
1114    require DBI;
1115    require DBD::SQLite;
1116
1117    pb_cms2build("CMS","Web");
1118} elsif ($action =~ /^cms2webssh$/) {
1119    require DBI;
1120    require DBD::SQLite;
1121
1122    pb_cms2build("CMS","Web");
1123    pb_send2target("Web");
1124} else {
1125    pb_log(0,"\'$action\' is not available\n");
1126    pb_syntax(-2,1);
1127}
1128
1129pb_exit();
1130
1131sub pb_cms2build {
1132
1133    my $param = shift;
1134    my $web = shift;
1135
1136    my $pkg;
1137    my @pkgs;
1138    my $webdir;
1139
1140    my %pkgs;
1141    my $pb;             # Structure to store conf info
1142
1143    confess "pb_cms2build requires a parameter: SandBox or CMS" if (not defined $param);
1144
1145    # If Website, then pkg is only the website
1146    if (defined $web) {
1147        ($webdir) = pb_conf_get("webdir");
1148        pb_log(2,"webdir: ".Dumper($webdir)."\n");
1149        $pkgs[0] = $webdir->{$ENV{'PBPROJ'}};
1150        $extpkgdir = $webdir;
1151        pb_log(0,"Package: $pkgs[0]\n");
1152    } else {
1153        $pkg = pb_cms_get_pkg($defpkgdir,$extpkgdir);
1154        @pkgs = @$pkg;
1155    }
1156
1157    my ($scheme, $uri) = pb_vcs_init($pbinit,$param);
1158
1159    # We need 2 lines here
1160    my ($pkgv, $pkgt, $testver) = pb_conf_get_if("pkgver","pkgtag","testver");
1161
1162    # declare packager and repo for filtering
1163    my ($tmp1, $tmp2) = pb_conf_get("pbpackager","pbrepo");
1164    $ENV{'PBPACKAGER'} = $tmp1->{$ENV{'PBPROJ'}};
1165    $ENV{'PBREPO'} = $tmp2->{$ENV{'PBPROJ'}};
1166    my ($delivery) = pb_conf_get_if("delivery");
1167    $delivery->{$ENV{'PBPROJ'}} = "" if (not defined $delivery->{$ENV{'PBPROJ'}});
1168
1169    # If we deal with a test dir, we want to keep the date in tar file and dir name
1170    my $pbextdirpkg = "";
1171    my $pbextdir = "";
1172    if ((defined $testver) && (defined $testver->{$ENV{'PBPROJ'}}) && ($testver->{$ENV{'PBPROJ'}} =~ /true/i)) {
1173        $pbextdirpkg = strftime("%Y%m%d%H%M%S", @date);
1174        $pbextdir = ".0.".$pbextdirpkg;
1175    }
1176
1177    foreach my $pbpkg (@pkgs) {
1178        $ENV{'PBPKG'} = $pbpkg;
1179        $pbrev = $ENV{'PBREVISION'};
1180
1181        if ((defined $pkgv) && (defined $pkgv->{$pbpkg})) {
1182            $pbver = $pkgv->{$pbpkg};
1183        } else {
1184            $pbver = $ENV{'PBPROJVER'};
1185        }
1186        # If it's a test version, then build a tag to fit distributions need
1187        if ((defined $testver) && (defined $testver->{$ENV{'PBPROJ'}}) && ($testver->{$ENV{'PBPROJ'}} =~ /true/i)) {
1188            my $letter = "";
1189            if ($scheme =~ /svn/) {
1190                $letter = "s";
1191            } elsif ($scheme =~ /cvs/) {
1192                $letter = "c";
1193            } elsif ($scheme =~ /svk/) {
1194                $letter = "k";
1195            } elsif ($scheme =~ /hg/) {
1196                $letter = "h";
1197            } elsif ($scheme =~ /git/) {
1198                $letter = "g";
1199            } else {
1200                $letter = "f";
1201            }
1202            $pbtag = "0.$pbextdirpkg.$letter$pbrev";
1203            $ENV{'PBPROJTAG'} = $pbtag;
1204        } elsif ((defined $pkgt) && (defined $pkgt->{$pbpkg})) {
1205            $pbtag = $pkgt->{$pbpkg};
1206        } else {
1207            $pbtag = $ENV{'PBPROJTAG'};
1208        }
1209
1210        if (($pbtag =~ /:/) && ($scheme =~ /svn/)) {
1211            confess "RPM doesn't support release tag with ':'. Try to svn up before running pb";
1212        }
1213
1214        pb_log(0,"\n");
1215        pb_log(0,"Management of $pbpkg $pbver-$pbtag\n");
1216        confess "Unable to get env var PBDESTDIR" if (not defined $ENV{'PBDESTDIR'});
1217
1218        my $dest = "$ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir";
1219        # Create the structure if needed
1220        pb_mkdir_p($dest);
1221        # Clean up dest if necessary. The export will recreate it
1222        pb_rm_rf($dest) if (-d $dest);
1223
1224        # Export CMS tree for the concerned package to dest
1225        # And generate some additional files
1226        $OUTPUT_AUTOFLUSH=1;
1227
1228        # computes in which dir we have to work
1229        my $dir = $defpkgdir->{$pbpkg};
1230        $dir = $extpkgdir->{$pbpkg} if (not defined $dir);
1231        $dir = $webdir->{$ENV{'PBPROJ'}} if (defined $web);
1232        confess "Variable \$dir not defined. Check the package name first.\nIf this is the one expected, please report to dev team with log of a verbose run and this info ".Dumper($defpkgdir,$extpkgdir,$webdir) if (not defined $dir);
1233        pb_log(2,"def:".Dumper($defpkgdir)."ext: ".Dumper($extpkgdir)."dir: $dir\n");
1234
1235        # Exporting content from CMS
1236        my $sourcedir = undef;
1237        my $sourceuri = $uri;
1238        if ($param eq "SandBox") {
1239            # Point to the local instance
1240            $sourcedir = "$ENV{'PBDIR'}/$dir";
1241        } else {
1242            # Get it from a subdir of the URI with same version as localy but different root
1243            # Only if using a real CMS
1244            my ($scheme, $account, $host, $port, $path) = pb_get_uri($uri);
1245            if (($scheme !~ /^file/) && ($scheme !~ /^(ht|f)tp/)) {
1246                $sourceuri = "$ENV{'PBDIR'}/$dir"; 
1247                $sourceuri =~ s|^$ENV{'PBPROJDIR'}/|$uri/|;
1248            }
1249        }
1250        my $preserve = pb_vcs_export($sourceuri,$sourcedir,$dest);
1251
1252        # Generated fake content for test versions to speed up stuff
1253        my $chglog;
1254
1255        # Get project info on authors and log file
1256        # TODO: Make it CMS aware
1257        $chglog = "$ENV{'PBROOTDIR'}/$pbpkg/pbcl";
1258        $chglog = "$ENV{'PBROOTDIR'}/pbcl" if (! -f $chglog);
1259        $chglog = undef if (! -f $chglog);
1260
1261        # TODO: Make it CMS aware
1262        my $authors = "$ENV{'PBROOTDIR'}/$pbpkg/pbauthors";
1263        $authors = "$ENV{'PBROOTDIR'}/pbauthors" if (! -f $authors);
1264        $authors = "/dev/null" if (! -f $authors);
1265
1266        # Extract cms log history and store it
1267        if ((defined $chglog) && (! -f "$dest/NEWS")) {
1268            pb_log(2,"Generating NEWS file from $chglog\n");
1269            copy($chglog,"$dest/NEWS") || confess "Unable to create $dest/NEWS";
1270        }
1271        pb_cms_log($scheme,"$ENV{'PBDIR'}/$dir",$dest,$chglog,$authors,$testver);
1272
1273        my %build;
1274        # We want to at least build for the underlying distro
1275        # except if a target was given, in which case we only build for it
1276        # if -t was passed without target then build for the native distro.
1277        my $pbos = pb_distro_get_context($pbtarget);
1278        my $tmpl = pb_get_distros($pbos,$pbtarget);
1279
1280        # Setup $pb structure to allow filtering later on, on files using that structure
1281        $pb->{'tag'} = $pbtag;
1282        $pb->{'rev'} = $pbrev;
1283        $pb->{'ver'} = $pbver;
1284        $pb->{'pkg'} = $pbpkg;
1285        $pb->{'suf'} = $pbos->{'suffix'};
1286        $pb->{'realpkg'} = $pbpkg;
1287        $pb->{'date'} = $pbdate;
1288        $pb->{'defpkgdir'} = $defpkgdir;
1289        $pb->{'extpkgdir'} = $extpkgdir;
1290        $pb->{'chglog'} = $chglog;
1291        $pb->{'extdir'} = $pbextdir;
1292        $pb->{'packager'} = $ENV{'PBPACKAGER'};
1293        $pb->{'proj'} = $ENV{'PBPROJ'};
1294        $pb->{'repo'} = "$ENV{'PBREPO'}/$delivery->{$ENV{'PBPROJ'}}";
1295        $pb->{'patches'} = ();
1296        $pb->{'sources'} = ();
1297   
1298        my $tmpd = "$ENV{'PBTMP'}/cms.$$";
1299        pb_mkdir_p($tmpd) if (defined $pbparallel);
1300
1301        # Get only all.yml filter at this stage for pbinit
1302        my $ptr = pb_get_filters($pbpkg);
1303
1304        # Do not do that for website
1305        if (not defined $web) {
1306            my %virt;
1307            # De-duplicate similar VM/VE/RM
1308            foreach my $d (split(/,/,$tmpl)) {
1309                # skip ill-formatted vms (name-ver-arch)
1310                next if ($d !~ /-/);
1311                $virt{$d} = $d;
1312            }
1313
1314            # Try to use // processing here
1315            my $all_ok = 1;
1316            my $pm = new Parallel::ForkManager($pbparallel) if (defined $pbparallel);
1317            $pm->run_on_finish(sub { my ($pid, $code, $id, $signal, $dump) = @_;
1318                                    $all_ok = 0 unless (($code == 0) && ($signal == 0) && ($dump == 0)); }) if (defined $pbparallel);
1319
1320            pb_log(0,"Preparing delivery ...\n");
1321            foreach my $v (keys %virt) {
1322                $pm->start and next if (defined $pbparallel);
1323
1324                # Distro context
1325                my $pbos = pb_distro_get_context($v);
1326   
1327                $pb->{'pbos'} = $pbos;
1328                $pb->{'suf'} = $pbos->{'suffix'};
1329                pb_log(3,"DEBUG: pb: ".Dumper($pb)."\n");
1330
1331                # Get all filters to apply
1332                $ptr = pb_get_filters($pbpkg,$pbos);
1333   
1334                pb_log(2,"DEBUG Filtering PBDATE => $pbdate, PBTAG => $pbtag, PBVER => $pbver\n");
1335   
1336                # We need to compute the real name of the package
1337                my $pbrealpkg = pb_cms_get_real_pkg($pbpkg,$pbos->{'type'});
1338                $pb->{'realpkg'} = $pbrealpkg;
1339                pb_log(2,"Virtual package $pbpkg has a real package name of $pbrealpkg on $pbos->{'name'}-$pbos->{'version'}\n") if ($pbrealpkg ne $pbpkg);
1340   
1341                # Filter build files from the less precise up to the most with overloading
1342                # Filter all files found, keeping the name, and generating in dest
1343   
1344                # Find all build files first relatively to PBROOTDIR
1345                # Find also all specific files referenced in the .yml conf file
1346                my %bfiles = ();
1347                my %pkgfiles = ();
1348                # Used in Filter.pm by pb_filter_file
1349
1350                $build{$v} = "no";
1351                if (-d "$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'os'}") {
1352                    pb_list_bfiles("$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'os'}",$pbpkg,\%bfiles,\%pkgfiles,$supfiles);
1353                    $build{$v} = "yes";
1354                } 
1355                if (-d "$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'type'}") {
1356                    pb_list_bfiles("$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'type'}",$pbpkg,\%bfiles,\%pkgfiles,$supfiles);
1357                    $build{$v} = "yes";
1358                } 
1359                if (-d "$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'family'}") {
1360                    pb_list_bfiles("$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'family'}",$pbpkg,\%bfiles,\%pkgfiles,$supfiles);
1361                    $build{$v} = "yes";
1362                } 
1363                if (-d "$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'name'}") {
1364                    pb_list_bfiles("$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'name'}",$pbpkg,\%bfiles,\%pkgfiles,$supfiles);
1365                    $build{$v} = "yes";
1366                } 
1367                if (-d "$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'name'}-$pbos->{'version'}") {
1368                    pb_list_bfiles("$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'name'}-$pbos->{'version'}",$pbpkg,\%bfiles,\%pkgfiles,$supfiles);
1369                    $build{$v} = "yes";
1370                } 
1371                if (-d "$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}") {
1372                    pb_list_bfiles("$ENV{'PBROOTDIR'}/$pbpkg/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}",$pbpkg,\%bfiles,\%pkgfiles,$supfiles);
1373                    $build{$v} = "yes";
1374                }
1375                pb_log(2,"DEBUG($v) bfiles: ".Dumper(\%bfiles)."\n");
1376   
1377                if ($build{$v} ne "no") {
1378                    # Apply now all the filters on all the files concerned
1379                    # destination dir depends on the type of file
1380                    # For patch support
1381                    # TODO: Make it CMS aware
1382                    $pb->{'patches'} = pb_list_sfiles("$ENV{'PBROOTDIR'}/$pbpkg/pbpatch", $pb->{'patches'}, $pbos, "$ENV{'PBROOTDIR'}/$pbpkg/pbextpatch");
1383                    pb_log(2,"DEBUG($v) patches: ".Dumper($pb->{'patches'})."\n");
1384                    # TODO: Make it CMS aware
1385                    $pb->{'sources'} = pb_list_sfiles("$ENV{'PBROOTDIR'}/$pbpkg/pbsrc", $pb->{'sources'}, $pbos, "$ENV{'PBROOTDIR'}/$pbpkg/pbextsrc");
1386                    pb_log(2,"DEBUG($v) sources: ".Dumper($pb->{'sources'})."\n");
1387   
1388                    if (defined $pb->{'patches'}->{$v}) {
1389                        # Filter potential patches (local + remote)
1390                        pb_mkdir_p("$dest/pbconf/$v/pbpatch");
1391                        # If Debian based distribution, then prepare what will be done at build time
1392                        my ($patchcmd,$patchopt);
1393                        if ($pbos->{'type'} eq "deb") {
1394                            ($patchcmd,$patchopt) = pb_distro_get_param($pbos,pb_conf_get_if("ospatchcmd","ospatchopt"));
1395                            open(SCRIPT,"> $dest/pbconf/$v/pbpatch/pbapplypatch") || confess "Unable to create $dest/pbconf/$v/pbpatch/pbapplypatch";
1396                            print SCRIPT "#!/bin/bash\n";
1397                            print SCRIPT "set -x\n" if ($pbdebug gt 1);
1398                        }
1399                        foreach my $pf (split(/,/,$pb->{'patches'}->{$v})) {
1400                            my $pp = basename($pf);
1401                            pb_vcs_export($pf,undef,"$dest/pbconf/$v/pbpatch");
1402                            pb_filter_file_inplace($ptr,"$dest/pbconf/$v/pbpatch/$pp",$pb);
1403                            pb_system("gzip -9f $dest/pbconf/$v/pbpatch/$pp","","quiet");
1404                            if ($pbos->{'type'} eq "deb") {
1405                                # If Debian based distribution, then prepare what will be done at build time
1406                                # by applying the patches that will be available under the debian/patches dir
1407                                print SCRIPT "$patchcmd $patchopt \< debian/patches/$pp\n";
1408                            }
1409                        }
1410                        if ($pbos->{'type'} eq "deb") {
1411                            close(SCRIPT);
1412                            chmod 0755,"$dest/pbconf/$v/pbpatch/pbapplypatch";
1413                        }
1414                        #pb_system("cat $dest/pbconf/$v/pbpatch/pbapplypatch","APPLY","verbose");
1415                    }
1416                    if (defined $pb->{'sources'}->{$v}) {
1417                        pb_mkdir_p("$dest/pbconf/$v/pbsrc");
1418                        foreach my $pf (split(/,/,$pb->{'sources'}->{$v})) {
1419                            my $pp = basename($pf);
1420                            pb_vcs_export($pf,undef,"$dest/pbconf/$v/pbsrc");
1421                            pb_filter_file_inplace($ptr,"$dest/pbconf/$v/pbsrc/$pp",$pb);
1422                        }
1423                    }
1424                    # Filter build files at the end, as they depend on patches and sources
1425                    foreach my $f (keys %bfiles) {
1426                        pb_filter_file("$ENV{'PBROOTDIR'}/$bfiles{$f}",$ptr,"$dest/pbconf/$v/$f",$pb);
1427                    }
1428                    foreach my $f (keys %pkgfiles) {
1429                        pb_filter_file("$ENV{'PBROOTDIR'}/$pkgfiles{$f}",$ptr,"$dest/pbconf/$v/$f",$pb);
1430                    }
1431                }
1432
1433                if (defined $pbparallel) {
1434                    # Communicate results back to parent
1435                    my $str = "";
1436                    $str .= "build $v = $build{$v}\n" if (defined $build{$v});
1437                    $str .= "patches $v = $pb->{'patches'}->{$v}\n" if (defined $pb->{'patches'}->{$v});
1438                    $str .= "sources $v = $pb->{'sources'}->{$v}\n" if (defined $pb->{'sources'}->{$v});
1439                    pb_set_content("$tmpd/$$","$str");
1440                    $pm->finish;
1441                }
1442            }
1443            # In the parent, we need to get the result from the children
1444            $pm->wait_all_children if (defined $pbparallel);
1445            confess "Aborting, one or more of the children failed." if ((not $all_ok) && ($Global::pb_stop_on_error));
1446            my $made = "";
1447            my %h = ();
1448            my %tmp;
1449            my %tmp2;
1450            my $pt;
1451            my $k;
1452
1453            foreach $k (<$tmpd/*>) {
1454                $made .= pb_get_content($k);
1455            }
1456            pb_rm_rf($tmpd);
1457            pb_log(3,"MADE:\n$made");
1458
1459            # Rebuild local hashes
1460            foreach $k (split(/\n/,$made)) {
1461                if ($k =~ /^\s*([A-z0-9-_]+)\s+([[A-z0-9-_.]+)\s*=\s*(.+)$/) {
1462                    $h{$1}->{$2}=$3;
1463                }
1464            }
1465            pb_log(2,"HASH: ".Dumper(%h));
1466
1467            # Patches
1468            pb_log(0,"Delivered and compressed patches ");
1469            $pt = $h{'patches'};
1470            foreach $k (keys %$pt) {
1471                foreach my $v1 (split(/,/,$pt->{$k})) {
1472                    $tmp{$v1} = "";
1473                }
1474            }
1475            if (keys %tmp) {
1476                foreach $k (keys %tmp) {
1477                    pb_log(0,"$k ");
1478                }
1479            } else {
1480                pb_log(0,"N/A");
1481            }
1482            pb_log(0,"\n");
1483
1484            # Sources
1485            pb_log(0,"Delivered additional sources ");
1486            $pt = $h{'sources'};
1487            foreach $k (keys %$pt) {
1488                foreach my $v1 (split(/,/,$pt->{$k})) {
1489                    $tmp2{$v1} = "";
1490                }
1491            }
1492            if (keys %tmp2) {
1493                foreach $k (keys %tmp2) {
1494                    pb_log(0,"$k ");
1495                }
1496            } else {
1497                pb_log(0,"N/A");
1498            }
1499            pb_log(0,"\n");
1500
1501            # Build files
1502            my @found;
1503            my @notfound;
1504            $pt = $h{'build'};
1505            foreach my $b (keys %$pt) {
1506                push @found,$b if ($pt->{$b} =~ /yes/);
1507                push @notfound,$b if ($pt->{$b} =~ /no/);
1508            }
1509            pb_log(0,"Build files have been generated for ... ".join(',',sort(@found))."\n") if (@found);
1510            pb_log(0,"No Build files found for ".join(',',sort(@notfound))."\n") if (@notfound);
1511
1512        } else {
1513            # Instead call News generation
1514            pb_web_news2html($dest);
1515            # And create an empty pbconf
1516            pb_mkdir_p("$dest/pbconf");
1517            # And prepare the pbscript to execute remotely
1518            open(SCRIPT,"> $ENV{'PBTMP'}/pbscript") || confess "Unable to create $ENV{'PBTMP'}/pbscript";
1519            print SCRIPT "#!/bin/bash\n";
1520            print SCRIPT "#set -x\n";
1521            print SCRIPT "echo ... Extracting Website content\n";
1522            print SCRIPT "find . -type f | grep -Ev '^./$pbpkg-$pbver$pbextdir.tar.gz|^./pbscript' | xargs rm -f non-existent\n";
1523            print SCRIPT "find * -type d -depth | xargs rmdir 2> /dev/null \n";
1524            print SCRIPT "tar xfz $pbpkg-$pbver$pbextdir.tar.gz\n";
1525            print SCRIPT "mv $pbpkg-$pbver$pbextdir/* .\n";
1526            print SCRIPT "rm -f $pbpkg-$pbver$pbextdir.tar.gz\n";
1527            print SCRIPT "rmdir $pbpkg-$pbver$pbextdir\n";
1528            print SCRIPT "find . -type f -print0 | xargs -0 chmod 644\n";
1529            print SCRIPT "find . -type d -print0 | xargs -0 chmod 755\n";
1530            close(SCRIPT);
1531            chmod 0755,"$ENV{'PBTMP'}/pbscript";
1532        }
1533
1534        # Apply filters to the non-build files
1535        my $liste ="";
1536        if (defined $filteredfiles->{$pbpkg}) {
1537            foreach my $f (split(/,/,$filteredfiles->{$pbpkg})) {
1538                pb_filter_file_inplace($ptr,"$dest/$f",$pb);
1539                $liste = "$f $liste";
1540            }
1541        }
1542        pb_log(2,"Files ".$liste."have been filtered\n");
1543
1544        # TODO: Make it CMS aware
1545        # Execute the pbinit script if any
1546        if (-x "$ENV{'PBROOTDIR'}/$pbpkg/pbinit") {
1547            pb_filter_file("$ENV{'PBROOTDIR'}/$pbpkg/pbinit",$ptr,"$ENV{'PBTMP'}/pbinit",$pb);
1548            chmod 0755,"$ENV{'PBTMP'}/pbinit";
1549            pb_system("cd $dest ; $ENV{'PBTMP'}/pbinit","Executing init script from $ENV{'PBROOTDIR'}/$pbpkg/pbinit under $dest","verbose");
1550        }
1551
1552        # Do we have additional script to run to prepare the environement for the project ?
1553        # Then include it in the pbconf delivery
1554        foreach my $pbvf (<$ENV{'PBROOTDIR'}/$pbpkg/pbv*.pre>,<$ENV{'PBROOTDIR'}/$pbpkg/pbv*.post>) {
1555            pb_log(1,"Handling file $pbvf\n");
1556            if (-x "$pbvf") {
1557                my $target = "$ENV{'PBTMP'}/".basename($pbvf);
1558                pb_filter_file("$pbvf",$ptr,$target,$pb);
1559                chmod 0755,"$target";
1560                pb_log(1,"Capturing file $pbvf in $target\n");
1561            }
1562        }
1563
1564        # Prepare the dest directory for archive
1565        chdir "$ENV{'PBDESTDIR'}" || confess "Unable to change dir to $ENV{'PBDESTDIR'}";
1566        if (defined $preserve) {
1567            # In that case we want to preserve the original tar file for checksum purposes
1568            # The one created is btw equivalent in that case to this one
1569            # Maybe check basename of both to be sure they are the same ?
1570            pb_log(0,"Preserving original tar file ");
1571            move("$preserve","$pbpkg-$pbver$pbextdir.tar.gz");
1572        } else {
1573            # Possibility to look at PBSRC to guess more the filename
1574            pb_system("tar cfz $pbpkg-$pbver$pbextdir.tar.gz --exclude=$pbpkg-$pbver$pbextdir/pbconf $pbpkg-$pbver$pbextdir","Creating $pbpkg tar files compressed");
1575        }
1576        pb_log(0,"Under $ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.tar.gz\n");
1577        pb_system("tar cfz $pbpkg-$pbver$pbextdir.pbconf.tar.gz $pbpkg-$pbver$pbextdir/pbconf","Creating pbconf tar files compressed");
1578        pb_log(0,"Under $ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.pbconf.tar.gz\n");
1579
1580        # Keep track of version-tag per pkg
1581        $pkgs{$pbpkg} = "$pbver-$pbtag";
1582
1583        # Final cleanup
1584        pb_rm_rf($dest) if (-d $dest);
1585    }
1586
1587    # Keep track of per package version
1588    pb_log(2,"DEBUG pkgs: ".Dumper(%pkgs)."\n");
1589    open(PKG,"> $ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml") || confess "Unable to create $ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml";
1590    print PKG "---\n";
1591    print PKG "pbpkg:\n";
1592    foreach my $pbpkg (keys %pkgs) {
1593        print PKG "  $pbpkg: $pkgs{$pbpkg}\n";
1594    }
1595    close(PKG);
1596
1597    # Keep track of what is generated by default
1598    # We need to store the dir and info on version-tag
1599    # Base our content on the existing .yml file
1600    copy("$ENV{'PBROOTDIR'}/$ENV{'PBPROJ'}.yml","$ENV{'PBDESTDIR'}/pbrc.yml");
1601    my $lh;
1602    $lh = pb_conf_cache("$ENV{'PBDESTDIR'}/pbrc.yml",$lh);
1603    $lh->{'pbroot'}->{$ENV{'PBPROJ'}} = $ENV{'PBROOTDIR'};
1604    $lh->{'projver'}->{$ENV{'PBPROJ'}} = $ENV{'PBPROJVER'};
1605    $lh->{'projtag'}->{$ENV{'PBPROJ'}} = $ENV{'PBPROJTAG'};
1606    $lh->{'pbpackager'}->{$ENV{'PBPROJ'}} = $ENV{'PBPACKAGER'};
1607    $lh->{'pbextdir'}->{$ENV{'PBPROJ'}} = $pbextdir;
1608    pb_conf_write("$ENV{'PBDESTDIR'}/pbrc.yml",$lh);
1609}
1610
1611sub pb_install2pkg {
1612    # Get the running distro to test on
1613    my $pbos = pb_distro_get_context();
1614
1615    # Get list of packages to test
1616    # Get content saved in cms2build
1617    my $ptr = pb_get_pkg();
1618    @pkgs = @$ptr;
1619
1620    # Additional potential repo
1621    pb_distro_setuprepo($pbos,"install");
1622    foreach my $pbpkg (@pkgs) {
1623        # We need to install the package to test, and deps brought with it
1624        pb_distro_installpkgs($pbos,$pbpkg,"local");
1625    }
1626}
1627
1628sub pb_test2pkg {
1629    # Get the running distro to test on
1630    my $pbos = pb_distro_get_context();
1631
1632    # Get list of packages to test
1633    # Get content saved in cms2build
1634    my $ptr = pb_get_pkg();
1635    @pkgs = @$ptr;
1636
1637    # Additional potential repo
1638    pb_distro_setuprepo($pbos,"test");
1639    foreach my $pbpkg (@pkgs) {
1640    }
1641    pb_system("$ENV{'PBDESTDIR'}/pbtest","Launching test","verbose");
1642}
1643
1644sub pb_build2prep {
1645
1646    pb_log(0,"INFO: ------ Starting to prepare build environement ------\n");
1647
1648    # Get the running distro to build on
1649    my $pbos = pb_distro_get_context();
1650
1651    # If needed we may add repository to the build env
1652    my $forcerepo = pb_distro_setuprepo($pbos,"build");
1653    pb_log(1,"Forcing full package install with forcerepo\n") if (defined $forcerepo);
1654
1655    # Get list of packages to build
1656    my $ptr = pb_get_pkg();
1657    @pkgs = @$ptr;
1658   
1659    # Get content saved in cms2build
1660    my ($pkg) = pb_conf_read("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml","pbpkg");
1661    $pkg = { } if (not defined $pkg);
1662    my $pbextdir = pb_get_extdir();
1663
1664    foreach my $pbpkg (@pkgs) {
1665        my $vertag = $pkg->{$pbpkg};
1666        my @buildfiles;
1667        pb_log(2,"Vertag: $vertag\n");
1668        # get the version of the current package - maybe different
1669        ($pbver,$pbtag) = split(/-/,$vertag);
1670
1671        my $src2="$ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.pbconf.tar.gz";
1672        pb_log(2,"Pbconf file: $src2\n");
1673        pb_log(2,"Working directory: $ENV{'PBBUILDDIR'}\n");
1674
1675        my $ftype;
1676        if ($pbos->{'type'} eq "rpm") {
1677            $ftype = "\.spec\$";
1678        } elsif ($pbos->{'type'} eq "deb") {
1679            $ftype = "\/control\$";
1680        } elsif ($pbos->{'type'} eq "ebuild") {
1681            $ftype = "\.ebuild\$";
1682        } elsif ($pbos->{'type'} eq "tgz") {
1683            $ftype = "\/pbslack\$";
1684        } elsif ($pbos->{'type'} eq "pkg") {
1685            # Solaris - no deps management up to Solaris 10
1686        } elsif ($pbos->{'type'} eq "hpux") {
1687            # TODO: HP-UX
1688        } elsif ($pbos->{'type'} eq "apk") {
1689            $ftype = "\/APKBUILD\$";
1690        } else {
1691            confess "Unknown OS type format $pbos->{'type'}";
1692        }
1693
1694        @buildfiles = pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/","$ENV{'PBTMP'}","build");
1695        pb_log(2,"buildfiles ".Dumper(\@buildfiles)."\n");
1696        foreach my $f (@buildfiles) {
1697            if ($f =~ /$ftype/) {
1698                pb_distro_installdeps($f,$pbos,$forcerepo);
1699            }
1700        }
1701    }
1702    pb_log(0,"INFO: ------ Finished preparing build environment ------\n");
1703}
1704
1705sub pb_build2pkg {
1706
1707    my $do_install = shift;
1708
1709    # Call build2prep in case we have new deps we want to take in account
1710    # without redoing the prep phase (can be long and that helps debugging
1711    # build dependencies issues
1712    pb_build2prep();
1713    pb_log(0,"INFO: ------ Starting to build package ------\n");
1714    # Get the running distro to build on
1715    my $pbos = pb_distro_get_context();
1716
1717    # Get list of packages to build
1718    my $ptr = pb_get_pkg();
1719    @pkgs = @$ptr;
1720
1721    # Get content saved in cms2build
1722    my ($pkg) = pb_conf_read("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml","pbpkg");
1723    $pkg = { } if (not defined $pkg);
1724    my $pbextdir = pb_get_extdir();
1725
1726    pb_mkdir_p("$ENV{'PBBUILDDIR'}") if (! -d "$ENV{'PBBUILDDIR'}");
1727    chdir "$ENV{'PBBUILDDIR'}" || confess "Unable to chdir to $ENV{'PBBUILDDIR'}";
1728    my $made = ""; # pkgs made during build
1729    my $pm;
1730    my $all_ok = 1;
1731
1732    if (defined $pbparallel) {
1733        $pm = new Parallel::ForkManager($pbparallel);
1734        $pm->run_on_finish(sub { my ($pid, $code, $id, $signal, $dump) = @_;
1735                                $all_ok = 0 unless (($code == 0) && ($signal == 0) && ($dump == 0)); });
1736    }
1737
1738    # We need to communicate info back from the children if parallel so prepare a dir for that
1739    my $tmpd = "$ENV{'PBTMP'}/build.$$";
1740    pb_mkdir_p($tmpd) if (defined $pbparallel);
1741
1742    # Do it here as this doesn't work in // mode as it dies if it fails
1743    if ($pbos->{'type'} eq "rpm") {
1744        foreach my $d ('RPMS','SRPMS','SPECS','SOURCES','BUILD') {
1745                if (! -d "$ENV{'PBBUILDDIR'}/$d") {
1746                    pb_mkdir_p("$ENV{'PBBUILDDIR'}/$d");
1747                }
1748            }
1749    }
1750
1751    foreach my $pbpkg (@pkgs) {
1752        $pm->start and next if (defined $pbparallel);
1753
1754        my $vertag = $pkg->{$pbpkg};
1755        pb_log(2,"Vertag: $vertag\n");
1756        # get the version of the current package - maybe different
1757        ($pbver,$pbtag) = split(/-/,$vertag);
1758
1759        my $src="$ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.tar.gz";
1760        my $src2="$ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.pbconf.tar.gz";
1761        pb_log(2,"Source file: $src\n");
1762        pb_log(2,"Pbconf file: $src2\n");
1763
1764        pb_log(2,"Working directory: $ENV{'PBBUILDDIR'}\n");
1765        if ($pbos->{'type'} eq "rpm") {
1766            # Remove in case a previous link/file was there
1767            unlink "$ENV{'PBBUILDDIR'}/SOURCES/".basename($src);
1768            symlink "$src","$ENV{'PBBUILDDIR'}/SOURCES/".basename($src) || confess "Unable to symlink $src in $ENV{'PBBUILDDIR'}/SOURCES";
1769            # We need to first extract the spec file
1770            my @buildfile = pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/","$ENV{'PBBUILDDIR'}/SPECS","build");
1771
1772            # We need to handle potential patches to upstream sources
1773            pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbpatch/","$ENV{'PBBUILDDIR'}/SOURCES","patch");
1774
1775            # We need to handle potential additional sources to upstream sources
1776            pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbsrc/","$ENV{'PBBUILDDIR'}/SOURCES","src");
1777
1778            pb_log(2,"buildfile: ".Dumper(\@buildfile)."\n");
1779            # set LANGUAGE to check for correct log messages
1780            $ENV{'LANGUAGE'}="C";
1781            # Older Redhat use _target_platform in %configure incorrectly
1782            my $specialdef = "";
1783            if (($pbos->{'name'} eq "rhel") && ($pbos->{'version'} eq "2.1")) {
1784                $specialdef = "--define \'_target_platform \"\"\'";
1785            }
1786            my $buildcmd = "rpmbuild";
1787            if (($pbos->{'name'} eq "redhat") && ($pbos->{'version'} =~ /^6/)){
1788                $buildcmd = "rpm" ;
1789            }
1790
1791            foreach my $f (@buildfile) {
1792                if ($f =~ /\.spec$/) {
1793                    pb_system("$buildcmd $specialdef --define \"packager $ENV{'PBPACKAGER'}\" --define \"_topdir $ENV{'PBBUILDDIR'}\" -ba $f","Building package with $f under $ENV{'PBBUILDDIR'}","verbose");
1794                    last;
1795                }
1796            }
1797            # Get the name of the generated packages
1798            open(LOG,"$ENV{'PBTMP'}/system.$$.log") || confess "Unable to open $ENV{'PBTMP'}/system.$$.log";
1799            while (<LOG>) {
1800                chomp($_);
1801                next if ($_ !~ /^Wrote:/);
1802                s|.*/([S]*RPMS.*)|$1|;
1803                $made .=" $_";
1804            }
1805            close(LOG);
1806
1807        } elsif ($pbos->{'type'} eq "deb") {
1808            pb_system("tar xfz $src","Extracting sources");
1809            pb_system("tar xfz $src2","Extracting pbconf");
1810
1811            chdir "$pbpkg-$pbver$pbextdir" || confess "Unable to chdir to $pbpkg-$pbver$pbextdir";
1812            pb_rm_rf("debian");
1813            my $confdir = "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
1814            confess "Configuration directory $confdir is not a directory\nIs os description listed in your {ve,vm,rm}list values?" if (not -d $confdir);
1815            symlink "$confdir","debian" || confess "Unable to symlink 'debian' to $confdir";
1816            chmod 0755,"debian/rules";
1817
1818            # We need to handle potential patches to upstream sources
1819            pb_mkdir_p("debian/patches");
1820            my @f = pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbpatch/","debian/patches","patch");
1821
1822            # By default we use format 1.0 - Cf man dpkg-source
1823            my $debsrcfmt = "1.0";
1824            my $debsrcfile = "debian/source/format";
1825            if (-f $debsrcfile) {
1826                $debsrcfmt = pb_get_content($debsrcfile);
1827            }
1828
1829            if ($debsrcfmt =~ /^3.*quilt/) {
1830                # If we use quilt to manage patches, we then setup the env correctly
1831                # as per http://pkg-perl.alioth.debian.org/howto/quilt.html
1832                # Generate Debian patch series for quilt
1833                open(SERIE,"> debian/patches/series") || confess "Unable to write in debian/patches/series";
1834                $ENV{'QUILT_PATCHES'}="debian/patches";
1835            } else {
1836                # If we use dpatch to manage patches, we then setup the 00list file as well
1837                open(SERIE,"> debian/patches/00list") || confess "Unable to write in debian/patches/00list";
1838            }
1839            foreach my $f (sort @f) {
1840                # Skip the script made to apply the patches to the Debian tree
1841                next if ($f =~ /pbapplypatch/);
1842                # We also need to uncompress them
1843                pb_system("gzip -d $f","","quiet");
1844                $f =~ s/\.gz$//;
1845                print SERIE "$f\n";
1846            }
1847            close(SERIE);
1848            if (@f) {
1849                # We have patches...
1850                my $patch_file = "debian/patches/pbapplypatch";
1851                if (($debsrcfmt =~ /^1.*/) && (-x $patch_file)) {
1852                    # In that case we need to apply the patches ourselves locally
1853                    pb_system("cat $patch_file","APPLY","verbose");
1854                    pb_system("$patch_file","Applying patches to $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'} tree");
1855                }
1856                # ...so modify the name of files to be Debian compliant
1857                move("../$src","../$pbpkg-$pbver$pbextdir.orig.tar.gz");
1858            }
1859
1860            # We need to handle potential additional sources to upstream sources
1861            #pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbsrc/","$ENV{'PBBUILDDIR'}/debian","src");
1862
1863            pb_system("dpkg-buildpackage -us -uc -rfakeroot","Building package","verbose");
1864
1865            # Get the name of the generated packages
1866            open(LOG,"$ENV{'PBTMP'}/system.$$.log") || confess "Unable to open $ENV{'PBTMP'}/system.$$.log";
1867            while (<LOG>) {
1868                chomp();
1869                pb_log(3,"In loop: $_\n");
1870                next unless (/^dpkg-deb.*\.\.\/(.*_.*\.deb).*/);
1871                my $tmp = $1;
1872                #doesn't work in my case
1873                #confess "Missing file $tmp" if (not -f "../$tmp");
1874                $made = "$made $tmp";
1875            }
1876            close(LOG);
1877            pb_log(2,"Now made is: $made\n");
1878
1879            open(CTRL,"debian/control") || confess "Unable to open debian/control: $!";
1880            while (<CTRL>) {
1881                pb_log(3,"In loop: $_\n");
1882                next unless (/^Source: (\S+)/o);
1883                my $tmp = $1;
1884                $made = "$made $tmp"."_*.dsc $tmp"."_*.tar.gz $tmp"."_*.changes";
1885                #doesn't work in my case
1886                #foreach my $glob (("$1\_*.changes", "$1\_*.dsc", "$1\_*.tar.gz")) {
1887                    #my @file = glob($glob);
1888                    #confess "Missing file for $glob" unless (@file > 0);
1889                    #confess "Too many files for $glob" if (@file > 1);
1890                    #confess "Missing file $file[0]" if (not -f $file[0]);
1891                    #$made .= " $file[0]";
1892                #}
1893            }
1894            close(CTRL);
1895            #pb_display_file("$ENV{'PBTMP'}/system.$$.log");
1896            pb_log(2,"Finally made is: $made\n");
1897
1898            chdir ".." || confess "Unable to chdir to parent dir";
1899            pb_rm_rf("$pbpkg-$pbver$pbextdir");
1900        } elsif ($pbos->{'type'} eq "ebuild") {
1901            my @ebuildfile;
1902            # For gentoo we need to take pb as subsystem name
1903            # We put every apps here under sys-apps. hope it's correct
1904            # We use pb's home dir in order to have a single OVERLAY line
1905            my $tmpe = "$ENV{'HOME'}/portage/pb/sys-apps/$pbpkg";
1906            pb_mkdir_p($tmpe) if (! -d "$tmpe");
1907            pb_mkdir_p("$ENV{'HOME'}/portage/distfiles") if (! -d "$ENV{'HOME'}/portage/distfiles");
1908
1909            # We need to first extract the ebuild file
1910            @ebuildfile = pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/","$tmpe","build");
1911
1912            # Prepare the build env for gentoo
1913            my $found = 0;
1914            my $pbbd = $ENV{'HOME'};
1915            $pbbd =~ s|/|\\/|g;
1916            if (-r "/etc/make.conf") {
1917                open(MAKE,"/etc/make.conf");
1918                while (<MAKE>) {
1919                    $found = 1 if (/$pbbd\/portage/);
1920                }
1921                close(MAKE);
1922            }
1923            if ($found == 0) {
1924                pb_system("sudo sh -c 'echo PORTDIR_OVERLAY=\"$ENV{'HOME'}/portage\" >> /etc/make.conf'");
1925            }
1926            #$found = 0;
1927            #if (-r "/etc/portage/package.keywords") {
1928            #open(KEYW,"/etc/portage/package.keywords");
1929            #while (<KEYW>) {
1930            #$found = 1 if (/portage\/pb/);
1931            #}
1932            #close(KEYW);
1933            #}
1934            #if ($found == 0) {
1935            #pb_system("sudo sh -c \"echo portage/pb >> /etc/portage/package.keywords\"");
1936            #}
1937
1938            # Build
1939            foreach my $f (@ebuildfile) {
1940                if ($f =~ /\.ebuild$/) {
1941                    move($f,"$tmpe/$pbpkg-$pbver.ebuild");
1942                    pb_system("cd $tmpe ; ebuild $pbpkg-$pbver.ebuild clean ; ebuild $pbpkg-$pbver.ebuild digest ; ebuild $pbpkg-$pbver.ebuild package","verbose");
1943                    # Now move it where pb expects it
1944                    pb_mkdir_p("$ENV{'PBBUILDDIR'}/portage/pb/sys-apps/$pbpkg");
1945                    if ($pbtag eq 0) {
1946                        # This is assumed to be a test version
1947                        my $nver = substr($pbver,0,-14);
1948                        my $ntag = substr($pbver,-14);
1949                        my $ebtg = "portage/pb/sys-apps/$pbpkg/$pbpkg-$nver"."_p$ntag.ebuild";
1950                        move("$tmpe/$pbpkg-$pbver.ebuild","$ENV{'PBBUILDDIR'}/$ebtg");
1951                        $made="$made $ebtg";
1952                    } else {
1953                        my $ebtg = "portage/pb/sys-apps/$pbpkg/$pbpkg-$pbver-r$pbtag.ebuild";
1954                        move("$tmpe/$pbpkg-$pbver.ebuild","$ENV{'PBBUILDDIR'}/$ebtg");
1955                        $made="$made $ebtg";
1956                    }
1957                }
1958            }
1959
1960        } elsif ($pbos->{'type'} eq "apk") {
1961            pb_mkdir_p("$ENV{'PBBUILDDIR'}/tmp");
1962
1963            # We need to first extract the APKBUILD file
1964            my @apkfile = pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/","$ENV{'PBBUILDDIR'}/tmp","build");
1965
1966            chdir "$ENV{'PBBUILDDIR'}/tmp" || confess "Unable to chdir to $ENV{'PBBUILDDIR'}/tmp";
1967
1968            # Build
1969            my $keyfile = pb_ssh_get(1);
1970            pb_mkdir_p("$ENV{'HOME'}/.abuild");
1971            open(CONF,"> $ENV{'HOME'}/.abuild/abuild.conf") || (cluck "Unable to create the abuild conf file" && next);
1972            print CONF "PACKAGER_PRIVKEY=\"$keyfile\"\n" if (defined $keyfile);
1973            close(CONF);
1974            foreach my $f (@apkfile) {
1975                if ($f =~ /APKBUILD$/) {
1976                    pb_system("ls -al $ENV{'HOME'}/.ssh ; pwd ; ls -alR ../.. ; sh -x /usr/bin/abuild checksum ; abuild unpack ; abuild -r -P .. build ; abuild -P .. rootpkg ; abuild -P pkg index","verbose");
1977                }
1978            }
1979            # Get the name of the generated packages
1980            open(LOG,"$ENV{'PBTMP'}/system.$$.log") || confess "Unable to open $ENV{'PBTMP'}/system.$$.log";
1981            while (<LOG>) {
1982                chomp($_);
1983                next if ($_ !~ /: Create /);
1984                s|.*: Create (.*.apk)|$1|;
1985                $made .= " $_";
1986            }
1987            close(LOG);
1988
1989        } elsif ($pbos->{'type'} eq "tgz") {
1990            # Slackware family
1991            $made="$made $pbpkg/$pbpkg-$pbver-*-$pbtag.tgz";
1992
1993            pb_system("tar xfz $src","Extracting sources");
1994            pb_system("tar xfz $src2","Extracting pbconf");
1995            chdir "$pbpkg-$pbver$pbextdir" || confess "Unable to chdir to $pbpkg-$pbver$pbextdir";
1996            symlink "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}","install" || confess "Unable to symlink to pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
1997            if (-x "install/pbslack") {
1998                pb_system("./install/pbslack","Building software");
1999                pb_system("sudo /sbin/makepkg -p -l y -c y $pbpkg","Packaging $pbpkg","verbose");
2000            }
2001            chdir ".." || confess "Unable to chdir to parent dir";
2002            pb_rm_rf("$pbpkg-$pbver$pbextdir");
2003        } elsif ($pbos->{'type'} eq "pkg") {
2004            # Solaris
2005            $made="$made $pbpkg-$pbver-$pbtag.pkg.gz";
2006            my $pkgdestdir="$ENV{'PBBUILDDIR'}/install";
2007
2008            # Will host resulting packages
2009            pb_mkdir_p("$pbos->{'type'}");
2010            pb_mkdir_p("$pkgdestdir/pbdelivery");
2011            pb_system("tar xfz $src","Extracting sources under $ENV{'PBBUILDDIR'}");
2012            pb_system("tar xfz $src2","Extracting pbconf under $ENV{'PBBUILDDIR'}");
2013            # We need to handle potential patches to upstream sources
2014            pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbpatch/","$ENV{'PBBUILDDIR'}","patch");
2015
2016            # We need to handle potential additional sources to upstream sources
2017            pb_extract_build_files($src2,"$pbpkg-$pbver$pbextdir/pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbsrc/","$ENV{'PBBUILDDIR'}","src");
2018
2019            chdir "$pbpkg-$pbver$pbextdir" || confess "Unable to chdir to $pbpkg-$pbver$pbextdir";
2020            if (-f "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbbuild") {
2021                chmod 0755,"pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbbuild";
2022                # pkginfo file is mandatory
2023                confess "Unable to find pkginfo file in pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}" if (! -f "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pkginfo");
2024                # Build
2025                pb_system("pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbbuild $pkgdestdir/pbdelivery","Building software and installing under $pkgdestdir/pbdelivery");
2026                # Copy complementary files
2027                if (-f "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/prototype") {
2028                    copy("pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/prototype", $pkgdestdir) 
2029                } else {
2030                    # No prototype provided, calculating it
2031                    open(PROTO,"> $pkgdestdir/prototype") || confess "Unable to create prototype file";
2032                    print PROTO "i pkginfo\n";
2033                    print PROTO "i depend\n" if (-f "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/depend");
2034                    $ENV{'PBSOLDESTDIR'} = "$pkgdestdir/pbdelivery";
2035                    find(\&create_solaris_prototype, "$pkgdestdir/pbdelivery");
2036                }
2037                copy("pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/depend", $pkgdestdir) if (-f "pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/depend");
2038                copy("pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pkginfo", $pkgdestdir);
2039                pb_system("cd $pkgdestdir/pbdelivery ; pkgmk -o -f ../prototype -r $pkgdestdir/pbdelivery -d $ENV{'PBBUILDDIR'}/$pbos->{'type'}","Packaging $pbpkg","verbose");
2040                pb_system("cd $ENV{'PBBUILDDIR'}/$pbos->{'type'} ;  echo \"\" | pkgtrans -o -n -s $ENV{'PBBUILDDIR'}/$pbos->{'type'} $ENV{'PBBUILDDIR'}/$pbpkg-$pbver-$pbtag.pkg all","Transforming $pbpkg","verbose");
2041                pb_system("cd $ENV{'PBBUILDDIR'} ;  gzip -9f $pbpkg-$pbver-$pbtag.pkg","Compressing $pbpkg-$pbver-$pbtag.pkg","verbose");
2042            } else {
2043                pb_log(0,"No pbconf/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}/pbbuild file found for $pbpkg-$pbver\n");
2044            }
2045            chdir ".." || confess "Unable to chdir to parent dir";
2046            pb_rm_rf("$pbpkg-$pbver$pbextdir","$ENV{'PBBUILDDIR'}/$pbos->{'type'}","$pkgdestdir");
2047        } elsif ($pbos->{'type'} eq "hpux") {
2048            # HP-UX
2049            pb_system("tar xfz $src","Extracting sources");
2050            pb_system("tar xfz $src2","Extracting pbconf");
2051
2052            chdir "$pbpkg-$pbver$pbextdir" || confess "Unable to chdir to $pbpkg-$pbver$pbextdir";
2053            pb_system("buildpackage ","Building package","verbose");
2054            # Get the name of the generated packages
2055            open(LOG,"$ENV{'PBTMP'}/system.$$.log") || confess "Unable to open $ENV{'PBTMP'}/system.$$.log";
2056            while (<LOG>) {
2057                chomp();
2058                my $tmp = $_;
2059                next if ($tmp !~ /^SD BUILD.*:/);
2060                $tmp =~ s|.*../(.*)_(.*).sd.*|$1|;
2061                $made = "$made $tmp"."_*.sd";
2062            }
2063            close(LOG);
2064            $made="$made $pbpkg-$pbver-$pbtag.sd";
2065
2066            chdir ".." || confess "Unable to chdir to parent dir";
2067            pb_rm_rf("$pbpkg-$pbver$pbextdir");
2068        } else {
2069            confess "Unknown OS type format $pbos->{'type'}";
2070        }
2071        if (defined $pbparallel) {
2072            # Communicate results back to parent
2073            pb_set_content("$tmpd/$$",$made);
2074            $pm->finish;
2075        }
2076    }
2077    if (defined $pbparallel) {
2078        # In the parent, we need to get the result from the children
2079        $pm->wait_all_children;
2080        foreach my $f (<$tmpd/*>) {
2081            $made .= " ".pb_get_content($f);
2082        }
2083        confess "Aborting, one or more of the children failed." if ((not $all_ok) && ($Global::pb_stop_on_error));
2084        pb_rm_rf($tmpd);
2085    }
2086
2087    # Sign packages
2088    pb_sign_pkgs($pbos,$made);
2089    pb_log(0,"INFO: ------ Finished building package ------\n");
2090
2091    # Find the appropriate check cmd/opts
2092    my ($chkcmd,$chkopt) = pb_distro_get_param($pbos,pb_conf_get_if("oschkcmd","oschkopt"));
2093
2094    my $ret = "";
2095    # Packages check if needed
2096    if ($pbos->{'type'} eq "rpm") {
2097        if ((defined  $chkcmd) && (-x $chkcmd)) {
2098            my $cmd = "$chkcmd";
2099            $cmd .= " $chkopt" if (defined $chkopt);
2100            $cmd .= " $made";
2101            my $ret = pb_system("$cmd","Checking validity of rpms with $chkcmd","mayfailverbose");
2102            pb_log(0,"ERROR: when checking packages validity\n") if ($ret ne 0);
2103        }
2104        my $rpms ="";
2105        my $srpms ="";
2106        foreach my $f (split(/ /,$made)) {
2107            $rpms .= "$ENV{'PBBUILDDIR'}/$f " if ($f =~ /^RPMS\//);
2108            $srpms .= "$ENV{'PBBUILDDIR'}/$f " if ($f =~ /^SRPMS\//);
2109        }
2110        pb_log(0,"SRPM packages generated: $srpms\n");
2111        pb_log(0,"RPM packages generated: $rpms\n");
2112        $ret = $rpms;
2113    } elsif ($pbos->{'type'} eq "deb") {
2114        my $made2 = "";
2115        foreach my $f (split(/ /,$made)) {
2116            $made2 .= "$f " if ($f =~ /\.deb$/);
2117        }
2118        if ((defined  $chkcmd) && (-x $chkcmd)) {
2119            my $ret = pb_system("$chkcmd $chkopt $made2","Checking validity of debs with $chkcmd","mayfail");
2120            pb_log(0,"ERROR: when checking packages validity\n") if ($ret ne 0);
2121        }
2122        pb_log(0,"deb packages generated: $made2\n");
2123        $ret = $made2;
2124    } else {
2125        pb_log(0,"No check done for $pbos->{'type'} yet\n");
2126        pb_log(0,"Packages generated: $made\n");
2127        $ret = $made;
2128    }
2129
2130    # Keep track of what is generated so that we can get them back from VMs/RMs
2131    my $pbstore = "$ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
2132    open(KEEP,"> $pbstore") || confess "Unable to create $pbstore $!";
2133    print KEEP "$made\n";
2134    close(KEEP);
2135    pb_distro_installdeps(undef,$pbos,undef,$ret,"local") if ($do_install);
2136}
2137
2138sub pb_build2ssh {
2139    pb_send2target("Sources");
2140    pb_send2target("CPAN");
2141}
2142
2143sub pb_pkg2ssh {
2144    pb_send2target("Packages");
2145}
2146
2147# By default deliver to the the public site hosting the
2148# ftp structure (or whatever) or a VM/VE/RM
2149sub pb_send2target {
2150
2151    my $cmt = shift;
2152    my $pbscript1 = shift;
2153    my $v = shift;
2154    my $vexist = shift;         # 0 is FALSE
2155    my $vpid = shift;           # 0 is FALSE
2156    my $snapme = shift;         # 0 is FALSE
2157    my $pbstep = shift;         # == 3 is usage of container
2158
2159    $vexist = 0 if (not defined $vexist);
2160    $vpid = 0 if (not defined $vpid);
2161    $snapme = 0 if (not defined $snapme);
2162    $pbstep = 3 if (not defined $pbstep);
2163    pb_log(2,"DEBUG: pb_send2target($cmt,".Dumper($v).",$vexist,$vpid)\n");
2164    my $host = "sshhost";
2165    my $login = "sshlogin";
2166    my $dir = "sshdir";
2167    my $port = "sshport";
2168    my $conf = "sshconf";
2169    my $tmout = undef;
2170    my $path = undef;
2171
2172    if ($cmt =~ /^VM/) {
2173        $login = "vmlogin";
2174        $dir = "pbdefdir";
2175        # Specific VM
2176        $tmout = "vmtmout";
2177        $path = "vmpath";
2178        $host = "vmhost";
2179        $port = "vmport";
2180    } elsif ($cmt =~ /^RM/) {
2181        $login = "rmlogin";
2182        $dir = "pbdefdir";
2183        # Specific RM
2184        $tmout = "rmtmout";
2185        $path = "rmpath";
2186        $host = "rmhost";
2187        $port = "rmport";
2188    } elsif ($cmt =~ /^VE/) {
2189        $login = "velogin";
2190        $dir = "pbdefdir";
2191        # Specific VE
2192        $path = "vepath";
2193        $conf = "rbsconf";
2194    } elsif ($cmt eq "Web") {
2195        $host = "websshhost";
2196        $login = "websshlogin";
2197        $dir = "websshdir";
2198        $port = "websshport";
2199    } elsif ($cmt eq "CPAN") {
2200        $host = "cpanpause";
2201        $login = "";
2202        $dir = "cpandir";
2203        $port = "";
2204    }
2205    my $cmd = "";
2206    my $src = "";
2207    my $cpanpkg = 0;
2208    my $pbos;
2209
2210    my $pbextdir = pb_get_extdir();
2211
2212    if ($cmt ne "Announce") {
2213        # Get list of packages to build
2214        my $ptr = pb_get_pkg();
2215        my @pkgs = ();
2216        @pkgs = @$ptr if (defined $ptr);
2217
2218        # Get the running distro to consider
2219        $pbos = pb_distro_get_context($v);
2220
2221        my $pkg = { };
2222        # Get content saved in cms2build
2223        if ((defined $ENV{'PBDESTDIR'}) && (defined $ENV{'PBPROJVER'}) && (defined $ENV{'PBPROJTAG'}) && (-f "$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml")) {
2224            ($pkg) = pb_conf_read("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml","pbpkg");
2225            $pkg = { } if (not defined $pkg);
2226        }
2227
2228        pb_mkdir_p("$ENV{'PBBUILDDIR'}") if (! -d "$ENV{'PBBUILDDIR'}");
2229        chdir "$ENV{'PBBUILDDIR'}" || confess "Unable to chdir to $ENV{'PBBUILDDIR'}";
2230        foreach my $pbpkg (@pkgs) {
2231            my $vertag = $pkg->{$pbpkg};
2232            # get the version of the current package - maybe different
2233            pb_log(2,"Vertag: $vertag\n");
2234            ($pbver,$pbtag) = split(/-/,$vertag);
2235
2236            if (($cmt eq "Sources") || ($cmt =~ /(V[EM]|RM)(build|prep)/)) {
2237                $src = "$src $ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.tar.gz $ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.pbconf.tar.gz";
2238                my $cmd2 = "ln -sf $pbpkg-$pbver$pbextdir.tar.gz $pbpkg-latest.tar.gz";
2239                #my $cmd2 = "ln $pbpkg-$pbver$pbextdir.tar.gz $pbpkg-latest.tar.gz 2> /dev/null ; if [ \$? -ne 0 ]; then ln -sf $pbpkg-$pbver$pbextdir.tar.gz $pbpkg-latest.tar.gz ; fi";
2240                if ($cmd eq "") {
2241                    $cmd = $cmd2;
2242                } else {
2243                    $cmd = "$cmd ; $cmd2";
2244                }
2245            } elsif ($cmt eq "Web") {
2246                $src = "$src $ENV{'PBDESTDIR'}/$pbpkg-$pbver$pbextdir.tar.gz"
2247            }
2248
2249            # Do we have one perl package
2250            my @nametype = pb_conf_get_if("namingtype");
2251            my $type = $nametype[0]->{$pbpkg};
2252            if ((defined $type) && ($type eq "perl")) {
2253                $cpanpkg = 1;
2254            }
2255        }
2256        # Adds conf file for availability of conf elements
2257        pb_conf_add("$ENV{'PBROOTDIR'}/$ENV{'PBPROJ'}.yml") if ((defined $ENV{'PBROOTDIR'}) && (-f "$ENV{'PBROOTDIR'}/$ENV{'PBPROJ'}.yml"));
2258    }
2259    my ($rbsconf,$testver,$delivery) = pb_conf_get_if($conf,"testver","delivery");
2260    if ($cmt eq "CPAN") {
2261        # Do not deliver on Pause if this is a test version
2262        return if (not defined $testver);
2263        return if ($testver =~ /true/);
2264        # Do not deliver on Pause if this is not a perl package
2265        return if ($cpanpkg == 0);
2266    }
2267
2268    if ($cmt =~ /(V[EM]|RM)(build|prep)/) {
2269        $src="$src $ENV{'PBROOTDIR'}/$ENV{'PBPROJ'}.yml $ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml $ENV{'PBETC'} $ENV{'PBDESTDIR'}/pbrc.yml $pbscript{$v}";
2270    } elsif ($cmt =~ /(V[EM]|RM)Script/) {
2271        $src="$src $pbscript{$v}";
2272    } elsif ($cmt =~ /(V[EM]|RM)(install|test)/) {
2273        $src="$src $ENV{'PBROOTDIR'}/$ENV{'PBPROJ'}.yml $ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml $ENV{'PBETC'} $ENV{'PBDESTDIR'}/pbrc.yml $pbscript{$v}";
2274    } elsif (($cmt eq "Announce") || ($cmt eq "Web") || ($cmt eq "CPAN")) {
2275        $src="$src $ENV{'PBTMP'}/pbscript";
2276    } elsif ($cmt eq "Packages") {
2277        # Get package list from file made during build2pkg
2278        open(KEEP,"$ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}") || confess "Unable to read $ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
2279        $src = <KEEP>;
2280        chomp($src);
2281        close(KEEP);
2282    }
2283    if (($cmt eq "Sources") || ($cmt eq "Packages") || ($cmt eq "CPAN")) {
2284        my ($pbpackager) = pb_conf_get("pbpackager");
2285        $ENV{'PBPACKAGER'} = $pbpackager->{$ENV{'PBPROJ'}};
2286        pb_log(0,"Exporting public key for $ENV{'PBPACKAGER'}\n");
2287        # Using pb_system is not working due to redirection below
2288        system("gpg --export -a \'$ENV{'PBPACKAGER'}\' > $ENV{'PBDESTDIR'}/$ENV{'PBPROJ'}.pubkey");
2289        chmod 0644,"$ENV{'PBDESTDIR'}/$ENV{'PBPROJ'}.pubkey";
2290        $src = "$src $ENV{'PBDESTDIR'}/$ENV{'PBPROJ'}.pubkey";
2291
2292        my ($gpgaddkey) = pb_conf_get_if("pbadditionalgpg");
2293        my $pbadditionalgpg;
2294        $pbadditionalgpg = $gpgaddkey->{$ENV{'PBPROJ'}} if (defined $gpgaddkey);
2295        if (defined $pbadditionalgpg) {
2296            foreach my $k (split(/,/,$pbadditionalgpg)) {
2297                pb_log(0,"Exporting public key $k\n");
2298                system("gpg --export -a \'$k\' > $ENV{'PBDESTDIR'}/$ENV{'PBPROJ'}-$k.pubkey");
2299                $src = "$src $ENV{'PBDESTDIR'}/$ENV{'PBPROJ'}-$k.pubkey";
2300            }
2301        }
2302    }
2303    # Remove potential leading spaces (cause problem with basename)
2304    $src =~ s/^ *//;
2305    my $basesrc = "";
2306    foreach my $i (split(/ +/,$src)) {
2307        $basesrc .= " ".basename($i);
2308    }
2309
2310    pb_log(2,"values: ".Dumper(($host,$login,$dir,$port,$tmout,$path,$conf))."\n");
2311    my ($sshhost,$sshdir) = pb_conf_get($host,$dir);
2312    # Not mandatory...
2313    $delivery->{$ENV{'PBPROJ'}} = "" if (not defined $delivery->{$ENV{'PBPROJ'}});
2314    my ($sshlogin,$sshport) = pb_conf_get_if($login,$port);
2315    $sshport->{$ENV{'PBPROJ'}} = 22 if (not defined $sshport->{$ENV{'PBPROJ'}});
2316    $sshlogin->{$ENV{'PBPROJ'}} = getpwuid($UID) if (not defined $sshlogin->{$ENV{'PBPROJ'}});
2317    my ($vtmout,$vepath);
2318    # ...Except those in virtual context
2319    if ($cmt =~ /^VE/) {
2320        ($vepath) = pb_conf_get($path);
2321    }
2322    if ($cmt =~ /^(V|R)M/) {
2323        $vtmout = pb_distro_get_param($pbos,pb_conf_get_if($tmout));
2324    }
2325    my $remhost = $sshhost->{$ENV{'PBPROJ'}};
2326    my $remdir = $sshdir->{$ENV{'PBPROJ'}};
2327    if ($cmt =~ /^V[EM]|RM/) {
2328        # In that case our real host is in the xxhost with the OS as key, not project as above
2329        $remhost = pb_distro_get_param($pbos,$sshhost);
2330    }
2331    pb_log(2,"ssh: ".Dumper(($remhost,$sshlogin,$remdir,$sshport,$vepath,$rbsconf))."\n");
2332    pb_log(2,"ssh: ".Dumper($vtmout)."\n") if (defined $vtmout);
2333
2334    my $mac;
2335    if ($cmt !~ /^VE/) {
2336        $mac = "$sshlogin->{$ENV{'PBPROJ'}}\@$remhost";
2337        # Overwrite account value if passed as parameter or forced
2338        $mac = "$ENV{'PBACCOUNT'}\@$remhost" if (defined $ENV{'PBACCOUNT'});
2339    } else {
2340        # VE
2341        $mac = $sshlogin->{$ENV{'PBPROJ'}};
2342        # Overwrite account value if passed as parameter (typically for setup2v)
2343        $mac = $ENV{'PBACCOUNT'} if (defined $ENV{'PBACCOUNT'});
2344    }
2345    pb_log(2, "DEBUG: mac: $mac\n");
2346
2347    my $tdir;
2348    my $bdir;
2349    if (($cmt eq "Sources") || ($cmt =~ /(V[EM]|RM)Script/)) {
2350        $tdir = "$remdir/$delivery->{$ENV{'PBPROJ'}}/src";
2351    } elsif ($cmt eq "CPAN") {
2352        $tdir = "$remdir";
2353    } elsif ($cmt =~ /(V[EM]|RM)(build|test|install|prep)/) {
2354        $tdir = $remdir."/$ENV{'PBPROJ'}/pbdelivery";
2355        $bdir = $remdir."/$ENV{'PBPROJ'}/pbbuild";
2356        # Remove a potential $ENV{'HOME'} as bdir should be relative to pb's home
2357        $bdir =~ s|\$ENV.+\}/||;
2358    } elsif ($cmt eq "Announce") {
2359        $tdir = "$remdir/$delivery->{$ENV{'PBPROJ'}}";
2360    } elsif ($cmt eq "Web") {
2361        $tdir = "$remdir/$delivery->{$ENV{'PBPROJ'}}";
2362    } elsif ($cmt eq "Packages") {
2363        if (($pbos->{'type'} eq "rpm") || ($pbos->{'type'} eq "pkg") || ($pbos->{'type'} eq "hpux") || ($pbos->{'type'} eq "tgz") || ($pbos->{'type'} eq "apk")) {
2364            # put packages under an arch subdir
2365            $tdir = "$remdir/$delivery->{$ENV{'PBPROJ'}}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}";
2366        } elsif (($pbos->{'type'} eq "deb") || ($pbos->{'type'} eq "ebuild")) {
2367            # No need for an arch subdir
2368            $tdir = "$remdir/$delivery->{$ENV{'PBPROJ'}}/$pbos->{'name'}/$pbos->{'version'}";
2369        } else {
2370            confess "Please teach the dev team where to deliver ($pbos->{'type'} type of packages\n";
2371        }
2372
2373        my $repodir = $tdir;
2374        $repodir =~ s|^$remdir/||;
2375        my $repotag = "";
2376        $repotag = "-$delivery->{$ENV{'PBPROJ'}}" if ($delivery->{$ENV{'PBPROJ'}} ne "");
2377
2378        my ($pbrepo) = pb_conf_get("pbrepo");
2379
2380        # Script for Repository management
2381        $v = "$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}" if (not defined $v);
2382        $pbscript{$v} = "$ENV{'PBBUILDDIR'}/pbscript.$$";
2383       
2384        # That script should be part of the delivery
2385        $src = "$src $pbscript{$v}";
2386        open(PBS,"> $pbscript{$v}") || confess "Unable to create $pbscript{$v}";
2387        if ($pbos->{'type'} eq "rpm") {
2388            my $pbsha = pb_distro_get_param($pbos,pb_conf_get("ossha"));
2389            my ($gpgcheck) = pb_conf_get_if("pbgpgcheck");
2390            my $pbgpgcheck;
2391            $pbgpgcheck = $gpgcheck->{$ENV{'PBPROJ'}} if (defined $gpgcheck);
2392            my $pbgpgkey = "$pbrepo->{$ENV{'PBPROJ'}}/$repodir/$ENV{'PBPROJ'}.pubkey";
2393            my ($gpgaddpkey) = pb_conf_get_if("pbadditionalgpg");
2394            my $pbadditionalgpg;
2395            $pbadditionalgpg = $gpgaddpkey->{$ENV{'PBPROJ'}} if (defined $gpgaddpkey);
2396            if (defined $pbadditionalgpg) {
2397                foreach my $k (split(/,/,$pbadditionalgpg)) {
2398                    $pbgpgkey .= " $pbrepo->{$ENV{'PBPROJ'}}/$repodir/$ENV{'PBPROJ'}-$k.pubkey";
2399                }
2400            }
2401            # By default force GPG check in repo even if we support signature of packages to fail. This is a best practice
2402            $pbgpgcheck = 1 if (not defined $pbgpgcheck);
2403            # Also make a pbscript to generate dnf/yum/urpmi bases
2404            print PBS << "EOF";
2405#!/bin/bash
2406# Prepare a script to ease dnf/yum setup
2407EOF
2408            print PBS "set -x\n" if ($pbdebug gt 1);
2409            print PBS << "EOF";
2410cat > $ENV{'PBPROJ'}$repotag.repo << EOT
2411[$ENV{'PBPROJ'}$repotag]
2412name=$pbos->{'name'} $pbos->{'version'} $pbos->{'arch'} - $ENV{'PBPROJ'} Vanilla Packages
2413baseurl=$pbrepo->{$ENV{'PBPROJ'}}/$repodir
2414enabled=1
2415gpgcheck=$pbgpgcheck
2416gpgkey=$pbgpgkey
2417EOT
2418chmod 644 $ENV{'PBPROJ'}$repotag.repo
2419
2420# Clean up old repo content
2421rm -rf headers/ repodata/
2422# Create yum repo
2423if [ -x /usr/bin/yum-arch ]; then
2424    yum-arch .
2425fi
2426# Create repodata
2427createrepo -s $pbsha .
2428# Link to the key
2429if [ -s $ENV{'PBPROJ'}.pubkey ]; then
2430    # Avoiding creating empty repomd.xml.key; fails on opensuse 12.1
2431    (cd repodata ; ln -sf ../$ENV{'PBPROJ'}.pubkey repomd.xml.key)
2432fi
2433# sign the repomd (at least useful for SLES - which requires a local key)
2434# gpg -a --detach-sign repodata/repomd.xml
2435# SLES also looks for media.1/info.txt
2436EOF
2437            if ($pbos->{'family'} eq "md") {
2438                # For Mandriva add urpmi management
2439                print PBS << "EOF";
2440# Prepare a script to ease urpmi setup
2441cat > $ENV{'PBPROJ'}$repotag.addmedia << EOT
2442#rpm --import $pbrepo->{$ENV{'PBPROJ'}}/$repodir/$ENV{'PBPROJ'}.pubkey
2443urpmi.addmedia $ENV{'PBPROJ'}$repotag $pbrepo->{$ENV{'PBPROJ'}}/$repodir with media_info/hdlist.cz
2444EOT
2445chmod 755 $ENV{'PBPROJ'}$repotag.addmedia
2446
2447# Clean up old repo content
2448rm -f hdlist.cz synthesis.hdlist.cz
2449# Create urpmi repo
2450genhdlist2 --clean .
2451if [ \$\? -ne 0 ]; then
2452    genhdlist .
2453fi
2454(cd media_info ; ln -sf ../$ENV{'PBPROJ'}.pubkey pubkey)
2455EOF
2456            }
2457            if ($pbos->{'name'} eq "fedora") {
2458                # Extract the spec file to please Fedora maintainers :-(
2459                print PBS << "EOF";
2460for p in $basesrc; do
2461    echo \$p | grep -q 'src.rpm'
2462    if [ \$\? -eq 0 ]; then
2463        rpm2cpio \$p | cpio -ivdum --quiet '*.spec'
2464    fi
2465done
2466EOF
2467            }
2468            if ($pbos->{'family'} eq "novell") {
2469                # Add ymp scripts for one-click install on SuSE
2470                print PBS << "EOF";
2471# Prepare a script to ease SuSE one-click install
2472# Cf: http://de.opensuse.org/1-Klick-Installation/ISV
2473#
2474cat > $ENV{'PBPROJ'}$repotag.ymp << EOT
2475<?xml version="1.0" encoding="utf-8"?>
2476<!-- vim: set sw=2 ts=2 ai et: -->
2477<metapackage xmlns:os="http://opensuse.org/Standards/One_Click_Install" xmlns="http://opensuse.org/Standards/One_Click_Install">
2478    <group><!-- The group of software, typically one for project-builder.org -->
2479        <name>$ENV{'PBPROJ'} Bundle</name> <!-- Name of the software group -->
2480        <summary>Software bundle for the $ENV{'PBPROJ'}$repotag project</summary> <!--This message is shown to the user and should describe the whole bundle -->
2481        <description>This is the summary of the $ENV{'PBPROJ'}$repotag Project
2482             
2483            Details are available on a per package basis below
2484
2485        </description><!--This is also shown to the user -->
2486        <remainSubscribed>false</remainSubscribed> <!-- Don't know what it mean -->
2487        <repositories><!-- List of needed repositories -->
2488            <repository>
2489                <name>$ENV{'PBPROJ'}$repotag Repository</name> <!-- Name of the repository  -->
2490                <summary>This repository contains the $ENV{'PBPROJ'}$repotag project packages.</summary> <!-- Summary of the repository -->
2491                <description>This repository contains the $ENV{'PBPROJ'}$repotag project packages.</description><!-- This description is shown to the user -->
2492                <url>$pbrepo->{$ENV{'PBPROJ'}}/$repodir</url><!--URL of repository, which is added -->
2493            </repository>
2494        </repositories>
2495        <software><!-- A List of packages, which should be added through the one-click-installation -->
2496EOT
2497for p in $basesrc; do
2498    sum=`rpm -q --qf '%{SUMMARY}' \$p`
2499    name=`rpm -q --qf '%{NAME}' \$p`
2500    desc=`rpm -q --qf '%{description}' \$p`
2501    cat >> $ENV{'PBPROJ'}$repotag.ymp << EOT
2502            <item>
2503                <name>\$name</name><!-- Name of the package, is shown to the user and used to identify the package at the repository -->
2504                <summary>\$sum</summary> <!-- Summary of the package -->
2505                <description>\$desc</description> <!-- Description, is shown to the user -->
2506            </item>
2507EOT
2508done
2509cat >> $ENV{'PBPROJ'}$repotag.ymp << EOT
2510        </software>
2511    </group>
2512</metapackage>
2513EOT
2514chmod 644 $ENV{'PBPROJ'}$repotag.ymp
2515EOF
2516            }
2517        } elsif ($pbos->{'type'} eq "deb") {
2518            # Also make a pbscript to generate apt bases
2519            # Cf: http://www.debian.org/doc/manuals/repository-howto/repository-howto.fr.html
2520            # This dirname removes ver
2521            my $debarch = pb_get_debarch($pbos);
2522            my $projcomponent = pb_get_debpc($pbos);
2523            my $rpd = dirname("$pbrepo->{$ENV{'PBPROJ'}}/$repodir");
2524            # Remove extra . in path to fix #522
2525            $rpd =~ s|/./|/|g;
2526            print PBS << "EOF";
2527#!/bin/bash
2528# Prepare a script to ease apt setup
2529cat > $ENV{'PBPROJ'}$repotag.sources.list << EOT
2530deb $rpd $pbos->{'version'} $projcomponent
2531deb-src $rpd $pbos->{'version'} $projcomponent
2532EOT
2533chmod 644 $ENV{'PBPROJ'}$repotag.sources.list
2534
2535# Up two levels to deal with the dist dir cross versions
2536cd ..
2537mkdir -p dists/$pbos->{'version'}/$projcomponent/binary-$debarch dists/$pbos->{'version'}/$projcomponent/source dists/$pbos->{'version'}/$projcomponent/binary-all
2538
2539# Prepare a script to create apt info file
2540# Reuse twice after
2541TMPD=`mktemp -d /tmp/pb.XXXXXXXXXX` || exit 1
2542mkdir -p \$TMPD
2543cat > \$TMPD/Release << EOT
2544Archive: unstable
2545Component: $projcomponent
2546Origin: $ENV{'PBPROJ'}$repotag
2547Label: $ENV{'PBPROJ'}$repotag dev repository $pbrepo->{$ENV{'PBPROJ'}}
2548EOT
2549
2550echo "Creating Packages metadata"
2551for i in dists/$pbos->{version}/$projcomponent/binary-*; do
2552    arch=`basename \$i | sed 's/binary-//'`
2553    echo "Packages for \$arch:"
2554    dpkg-scanpackages -a\$arch $pbos->{'version'} /dev/null > dists/$pbos->{'version'}/$projcomponent/binary-\$arch/Packages
2555    gzip -9 < dists/$pbos->{'version'}/$projcomponent/binary-\$arch/Packages > dists/$pbos->{'version'}/$projcomponent/binary-\$arch/Packages.gz
2556    bzip2 -9 < dists/$pbos->{'version'}/$projcomponent/binary-\$arch/Packages > dists/$pbos->{'version'}/$projcomponent/binary-\$arch/Packages.bz2
2557done
2558echo "Creating Contents metadata"
2559apt-ftparchive contents $pbos->{'version'} | gzip -c9 > dists/$pbos->{'version'}/Contents.gz
2560echo "Creating Release metadata ($pbos->{'arch'} aka $debarch)"
2561cat \$TMPD/Release > dists/$pbos->{'version'}/$projcomponent/binary-all/Release
2562echo "Architecture: all" >> dists/$pbos->{'version'}/$projcomponent/binary-all/Release
2563cat \$TMPD/Release > dists/$pbos->{'version'}/$projcomponent/binary-$debarch/Release
2564echo "Architecture: $debarch" >> dists/$pbos->{'version'}/$projcomponent/binary-$debarch/Release
2565echo "Creating Source metadata"
2566dpkg-scansources $pbos->{'version'} /dev/null > dists/$pbos->{'version'}/$projcomponent/source/Sources
2567gzip -9 < dists/$pbos->{'version'}/$projcomponent/source/Sources > dists/$pbos->{'version'}/$projcomponent/source/Sources.gz
2568bzip2 -9 < dists/$pbos->{'version'}/$projcomponent/source/Sources > dists/$pbos->{'version'}/$projcomponent/source/Sources.bz2
2569cat \$TMPD/Release > dists/$pbos->{'version'}/$projcomponent/source/Release
2570echo "Architecture: Source" >> dists/$pbos->{'version'}/$projcomponent/source/Release
2571echo "Creating Release metadata"
2572apt-ftparchive -o APT::FTPArchive::Release::Suite=$pbos->{version} release dists/$pbos->{'version'} > dists/$pbos->{'version'}/Release
2573rm -rf \$TMPD
2574EOF
2575        } elsif ($pbos->{'type'} eq "ebuild") {
2576            # make a pbscript to generate links to latest version
2577            print PBS << "EOF";
2578#!/bin/bash
2579# Prepare a script to create correct links
2580cd $tdir
2581for p in $src; do
2582    echo \$p | grep -q '.ebuild'
2583    if [ \$\? -eq 0 ]; then
2584        j=`basename \$p`
2585        pp=`echo \$j | perl -pe 's/-[0-9\.]+-r[0-9].*\.ebuild/.ebuild/'`
2586        ln -sf \$j \$pp
2587    fi
2588done
2589EOF
2590        } elsif ($pbos->{'type'} eq "apk") {
2591            # TODO
2592        }
2593        close(PBS);
2594        chmod 0755,$pbscript{$v};
2595    } else {
2596        return;
2597    }
2598
2599    # Useless for VE
2600    my $nport = pb_get_port($sshport,$pbos,$cmt) if ($cmt !~ /^VE/);
2601
2602    # Remove a potential $ENV{'HOME'} as tdir should be relative to pb's home
2603    if ($cmt =~ /^VE/o) {
2604        $tdir =~ s|\$ENV.+\}/|| or (cluck "for $cmt: $tdir must have \$ENV{'HOME'} in it or the remainder of the code won't work since some parts use relative pathing and others will try absolute.  Use a symlink if necessary to place the real files in a different place than under your home directory" && return);
2605    } else {
2606        # process, ok if not present.
2607        $tdir =~ s|\$ENV.+\}/||;
2608    }
2609
2610    my $tm = "";
2611    if ($cmt =~ /^(V|R)M/) {
2612        $tm = "sleep $vtmout" if (defined $vtmout);
2613    }
2614
2615    # ssh communication if not VE or CPAN
2616    # should use a hash instead...
2617    my ($shcmd,$shcmdroot,$cpcmd,$cptarget,$cp2target);
2618    my $tpdir;
2619    my $tp;
2620    my $context = "$ENV{'PBTMP'}";
2621    my %tag;
2622    my $dkaccount = "";
2623    my $cmd1 = "";
2624    my $cidfile;
2625    my $homedir = "";
2626    if ($cmt =~ /^VE/) {
2627        $tp = pb_path_expand($vepath->{$ENV{'PBPROJ'}});
2628        $vetype = pb_ve_get_type($vetype);
2629        if ($vetype eq "docker") {
2630            $tpdir = "/";
2631            $cidfile = "$context/ctn-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}.cid";
2632        } else {
2633            $tpdir = pb_path_expand("$tp/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}");
2634        }
2635        my $arch = pb_get_arch();
2636        if ($vetype eq "chroot") {
2637            $shcmdroot = "sudo /usr/sbin/chroot $tpdir ";
2638            $shcmd = "$shcmdroot /bin/su - $mac -c ";
2639        } elsif ($vetype eq "schroot") {
2640            $shcmd = "schroot $tp -u $mac -- ";
2641        } elsif ($vetype eq "docker") {
2642            # docker manages the storage so rely on it
2643            $shcmdroot = "";
2644            my $docrepo = pb_ve_docker_repo();
2645            $cmd1 = pb_check_req("docker",0);
2646            #my ($dockeropt) =  pb_conf_get_if("dockeropt");
2647            # pbimage is used for docker and is setup to the right name depending on the step:
2648            # TODO: This is not coded yet
2649            my $pbimage = undef;
2650            # step 0 : nothing at creation -> tag n-v-a (made in VE.pm)
2651            # step 1 : n-v-a + setup -> tag n-v-a-pb
2652            # step 2 : n-v-a-pb + build -> tag n-v-a-pb-pbproj
2653            # step 3 : n-v-a-pb-pbproj at use
2654            # step 4 : n-v-a-pb-pbproj-install based on #3
2655            # step 5 : n-v-a-pb-pbproj-test based on #4
2656            if ((not defined $pbimage) || ($pbimage eq "")) {
2657                # If no image name given, create a naming convention
2658                $tag{1} = "$docrepo$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
2659                $tag{2} = "$tag{1}-pb";
2660                $tag{3} = "$tag{2}-$ENV{'PBPROJ'}";
2661                $tag{4} = "$tag{2}-install-$ENV{'PBPROJ'}";
2662                $tag{5} = "$tag{2}-test-$ENV{'PBPROJ'}";
2663            } else {
2664                # If we were given an image name, just use it
2665                $tag{1} = $pbimage;
2666                $tag{2} = $pbimage;
2667                $tag{3} = $pbimage;
2668                $tag{4} = $pbimage;
2669                $tag{5} = $pbimage;
2670            }
2671            # If we do not create the image, then use the one we should have
2672            # TODO: we don't do anything with found !!
2673            my $found = pb_ve_docker_get_image($tag{$pbstep});
2674            cluck "No image $tag{$pbstep} available in Docker, please create one first\n" if (not defined $found);
2675            # Now we use that image to do what is needed
2676            # use a dockerfile to ease the creation of next images
2677            # the context is in an empty temp dir
2678            my $tmpd = "$context/Dockerfile";
2679            open(DOCKER, "> $tmpd") || confess "Unable to create the docker file $tmpd";
2680            print DOCKER "FROM $tag{$pbstep}\n";
2681            print DOCKER "MAINTAINER project-builder.org aka pb\n";
2682            if ($pbstep == 1) {
2683                # setup done as root
2684                $dkaccount = "root";
2685            } else {
2686                # Rest done as pb
2687                $dkaccount = "pb";
2688            }
2689            print DOCKER "USER $dkaccount\n";
2690            if (($pbstep <= 2) || ($pbstep > 3)) {
2691                $shcmd = "$cmd1 build -t $tag{$pbstep+1} $context";
2692                #$shcmd = "$cmd1 build $dockeropt->{$ENV{'PBPROJ'}} -t $tag{$pbstep+1} $context";
2693            } else {
2694                my $rmatend = "";
2695                # As we are in run phase use docker run. cmd will be completed below
2696                $rmatend = "--rm" if ($pbdebug lt 1);
2697                $shcmd = "$cmd1 run $rmatend --cidfile=\"$cidfile\" --user $dkaccount --name $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-$cmt";
2698            }
2699        }
2700        $shcmd = "setarch i386 $shcmd" if (($pbos->{'arch'} =~ /i[3456]86/) && ($arch eq 'x86_64'));
2701        if (($cmt =~ /VE/) && ($vetype ne "docker")) {
2702            # We need to get the home dir of the target account to deliver in the right place
2703            open(PASS,"$tpdir/etc/passwd") || confess "Unable to open $tpdir/etc/passwd: $!";
2704            while (<PASS>) {
2705                my ($c1,$c2,$c3,$c4,$c5,$c6,$c7) = split(/:/);
2706                $homedir = $c6 if ($c1 =~ /^$mac$/);
2707                pb_log(3,"Homedir: $homedir - account: $c6\n");
2708            }
2709            close(PASS);
2710            $cpcmd = "sudo /bin/cp -r ";
2711        }
2712        if (($cmt =~ /VE/) && ($vetype eq "docker")) {
2713            if ($pbstep >= 2) {
2714                # We need to get the home dir of the target account to deliver in the right place
2715                $homedir = `$cmd1 run --rm --cidfile="$cidfile" $tag{$pbstep} grep -E '^$dkaccount:' /etc/passwd | cut -d: -f6`;
2716                chomp($homedir);
2717                open(CID,"$cidfile") || cluck "Unable to open $cidfile";
2718                my $cid = <CID>;
2719                close(CID);
2720                unlink("$cidfile");
2721            } else {
2722                $homedir = "/";
2723            }
2724            $cpcmd = "cp -r ";
2725        }
2726        $cptarget = "$tpdir/$homedir/$tdir";
2727        if ($cmt =~ /VE(build|prep|install|test)/) {
2728            $cp2target = "$tpdir/$homedir/$bdir";
2729        }
2730        pb_log(2,"On VE using $cptarget as target dir to copy to\n");
2731    } elsif ($cmt eq "CPAN") {
2732        my $ftpput = pb_check_req("ncftpput",1);
2733        my $ftpget = pb_check_req("wget",1);
2734        my ($cpanuser,$cpanpasswd) = pb_conf_get_if("cpanuser","cpanpasswd");
2735        return if ((not defined $cpanuser) || (not defined $cpanuser->{$ENV{'PBPROJ'}}));
2736        return if ((not defined $cpanpasswd) || (not defined $cpanpasswd->{$ENV{'PBPROJ'}}));
2737        my ($cpansubdir) = pb_conf_get_if("cpansubdir");
2738        $shcmd = "$ftpget --post-data \'HIDDENNAME=".$cpanuser->{$ENV{'PBPROJ'}};
2739        $shcmd .= "&user=".$cpanuser->{$ENV{'PBPROJ'}};
2740        $shcmd .= "&password=".$cpanpasswd->{$ENV{'PBPROJ'}};
2741        $shcmd .= "&SUBMIT_pause99_add_uri_upload=\"Upload the checked files\"";
2742        $shcmd .= "&pause99_add_uri_subdirtext=".$cpansubdir->{$ENV{'PBPROJ'}} if ((defined $cpansubdir) && (defined $cpansubdir->{$ENV{'PBPROJ'}}));
2743        foreach my $s (split(/ /,$src)) {
2744            $shcmd .= "&pause99_add_uri_upload=".basename($s);
2745        }
2746        $shcmd .= "'";
2747        $cpcmd = "$ftpput $host $dir";
2748        $cptarget = "CPAN";
2749    } else {
2750        my $keyopt = "";
2751        if ($cmt !~ /Packages/) {
2752            my $keyfile = pb_ssh_get(0,$remhost,$nport);
2753            $keyopt = "-i $keyfile" if (defined $keyfile);
2754        }
2755        my $sshcmd = pb_check_req("ssh",1);
2756        my $scpcmd = pb_check_req("scp",1);
2757        $shcmd = "$sshcmd $keyopt -q -o NoHostAuthenticationForLocalhost=yes -p $nport $mac";
2758        $cpcmd = "$scpcmd $keyopt -p -o NoHostAuthenticationForLocalhost=yes -P $nport";
2759        $cptarget = "$mac:$tdir";
2760        if ($cmt =~ /^(V|R)M(build|prep)/) {
2761            $cp2target = "$mac:$bdir";
2762        }
2763    }
2764   
2765    my $logres = "";
2766    # Do not touch when just announcing
2767    if (($cmt ne "Announce") && ($cmt ne "CPAN")) {
2768        # For docker everything is done in the Dockerfile for steps < 3
2769        if (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep < 3)) {
2770            print DOCKER "RUN mkdir -p $cptarget\n";
2771            print DOCKER "RUN cd $cptarget ; for i in $basesrc; do if [ -f \$i ]; then rm -f \$i; fi; done\n";
2772        } elsif (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep >= 3)) {
2773            # Nothing to do here
2774        } else {
2775            pb_system("$shcmd \"mkdir -p $tdir ; cd $tdir ; echo \'for i in $basesrc; do if [ -f \$i ]; then rm -f \$i; fi; done ; $cmd\' | bash -e\"","Preparing $tdir on $cptarget");
2776        }
2777    } else {
2778        $logres = "> ";
2779    }
2780
2781    if ($cmt =~ /SandBox/) {
2782        # Install from sandbox mean using the result of the just passed sbx2build command
2783        # Get content saved in cms2build
2784        my ($pkg) = pb_conf_read("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml","pbpkg");
2785        my $pbextdir = pb_get_extdir();
2786        if (not defined $pkg) {
2787            cluck "Unable to get package list from $ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml";
2788            return;
2789        }
2790
2791        # We consider 2 specific packages
2792        my $vertag1 = $pkg->{"ProjectBuilder"};
2793        my $vertag2 = $pkg->{"project-builder"};
2794        # get the version of the current package - maybe different
2795        pb_log(2,"Vertag1: $vertag1\n");
2796        pb_log(2,"Vertag2: $vertag2\n");
2797        my ($pbver1,$tmp1) = split(/-/,$vertag1);
2798        my ($pbver2,$tmp2) = split(/-/,$vertag2);
2799        if ($cmt =~ /VE/) {
2800            # Copy inside the VE
2801            if ($vetype ne "docker") {
2802                copy("$ENV{'PBDESTDIR'}/ProjectBuilder-$pbver1$pbextdir.tar.gz","$tpdir/tmp");
2803                copy("$ENV{'PBDESTDIR'}/project-builder-$pbver2$pbextdir.tar.gz","$tpdir/tmp");
2804            } else {
2805                copy("$ENV{'PBDESTDIR'}/ProjectBuilder-$pbver1$pbextdir.tar.gz","$context");
2806                copy("$ENV{'PBDESTDIR'}/project-builder-$pbver2$pbextdir.tar.gz","$context");
2807                print DOCKER "COPY ProjectBuilder-$pbver1$pbextdir.tar.gz /tmp/\n";
2808                print DOCKER "COPY project-builder-$pbver2$pbextdir.tar.gz /tmp/\n";
2809            }
2810        } else {
2811            pb_system("$cpcmd $ENV{'PBDESTDIR'}/ProjectBuilder-$pbver1$pbextdir.tar.gz $ENV{'PBDESTDIR'}/project-builder-$pbver2$pbextdir.tar.gz $mac:/tmp","Copying local project files to VM/RM");
2812        }
2813    }
2814           
2815    pb_log(0,"Sources handled ($cmt): $src\n");
2816    # For docker everything is done in the Dockerfile
2817    if (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep < 3)) {
2818        foreach my $f (split(/ +/,$src)) {
2819            copy("$f","$context");
2820            print DOCKER "COPY ".basename($f)." $cptarget/\n";
2821        }
2822        print DOCKER "RUN cd $cptarget ; $cmd\n" if ((defined $cmd) && ($cmd ne ""));
2823    } elsif (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep >= 3)) {
2824        # Nothing to do
2825    } else {
2826        pb_system("cd $ENV{'PBBUILDDIR'} ; $cpcmd $src $cptarget 2> /dev/null","$cmt delivery in $cptarget");
2827    }
2828
2829    # For VE we need to change the owner manually
2830    if ($cmt =~ /^VE/) {
2831        if ($vetype ne "docker") {
2832            my $sudomode = pb_distro_get_param($pbos,pb_conf_get("ossudoersmode"));
2833            my $res = pb_system("$shcmdroot sed -i '/requiretty/d' /etc/sudoers","Removing potential requiretty in sudoers","quiet");
2834            pb_system("$shcmdroot sed '/requiretty/d' /etc/sudoers > /tmp/sudoers.new ; mv /tmp/sudoers.new $tpdir/tmp/sudoers.new ; $shcmdroot mv /tmp/sudoers.new /etc/sudoers ; $shcmdroot chown root:root /etc/sudoers ; $shcmdroot chmod $sudomode /etc/sudoers","Removing again potential requiretty in sudoers as sed -i failed") if (($res ne 0) && (-f "$tpdir/etc/sudoers"));
2835            pb_system("$shcmd \"sudo chown -R $mac $tdir\"","Adapt owner in $tdir to $mac");
2836        } else {
2837            print DOCKER "RUN sed -i '/requiretty/d' /etc/sudoers\n" if ($pbstep == 1);
2838            # Bug in at least Fedora 23 where the dir wasn't owned correctly - doesn't hurt anyway
2839            print DOCKER "RUN chown root:root /etc\n" if ($pbstep == 1);
2840            print DOCKER "RUN chmod 755 /etc\n" if ($pbstep == 1);
2841        }
2842    }
2843
2844    # Use the right script name depending on context
2845    my $pbscript2;
2846    if (($cmt =~ /^(V[EM]|RM)/) || ($cmt =~ /Packages/)){
2847        $pbscript2 = $pbscript{$v};
2848    } else {
2849        $pbscript2 = "pbscript";
2850    }
2851
2852    # It's already ready for CPAN
2853    my $shcmdbase = $shcmd;
2854    my $dkcmd;
2855    if ($cmt ne "CPAN") {
2856        if (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep < 3)) {
2857            copy("$pbscript2","$context");
2858            my $s = basename($pbscript2);
2859            print DOCKER "COPY $s $cptarget/\n";
2860            # call directly with perl to avoid text file busy error messages
2861            print DOCKER "RUN cd $cptarget ; perl ./$s ; if [ '$pbkeep' = '0' ]; then rm -f $s; fi\n";
2862            print DOCKER "RUN rm -rf $cptarget/*\n";
2863            print DOCKER "RUN chown -R $dkaccount $cptarget\n" if ($pbstep == 1);
2864            close(DOCKER);
2865        } elsif (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep >= 3)) {
2866            $dkcmd = "/bin/bash $homedir/$tdir-2/".basename($pbscript2);
2867        } else {
2868            my $s = basename($pbscript2);
2869            $shcmd .= " \"echo \'cd $tdir ; if [ -x $s ]; then ./$s; fi ;  if [ $pbkeep == 0 ]; then rm -f $s; fi\' | bash\"";
2870        }
2871    }
2872    my $cmdverb = "verbose";
2873    if (($cmt eq "Announce") || ($cmt eq "CPAN")) {
2874        $cmdverb = undef;
2875    } elsif (defined ($v)) {
2876        $cmdverb = "verbose_\[$v\] ";
2877    }
2878    # this is where we launch the execution
2879    if (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep >= 3)) {
2880        pb_mkdir_p("$ENV{'PBTMP'}/vebuild");
2881        my $tsrc = "";
2882        foreach my $f (split(/ +/,$src)) {
2883            copy("$f","$ENV{'PBTMP'}/vebuild");
2884            $tsrc .= " ".basename($f);
2885        }
2886        my $ts = "$ENV{'PBTMP'}/vebuild/".basename($pbscript2);
2887        open(DOCKER," > $ts") || (cluck "Unable to create $ts" && return);
2888        print DOCKER "#!/bin/bash\n";
2889        print DOCKER "# Put content from host to guest, copying to have the correct uid/gid and rights to do so\n";
2890        print DOCKER "# set -x\n";
2891        print DOCKER "for i in $tsrc; do\n";
2892        print DOCKER "  # Cleanup first to avoid rights issues for root owned files e.g.\n";
2893        print DOCKER "  rm -f $homedir/$tdir/\$i\n";
2894        print DOCKER "  cp $homedir/$tdir-2/\$i $homedir/$tdir\n";
2895        print DOCKER "done\n";
2896        print DOCKER "(cd $homedir/$tdir-2 ; ls -al)\n";
2897        print DOCKER pb_get_content($pbscript2);
2898        # Not useful for test or install
2899        if ($pbstep == 3) {
2900            print DOCKER "\n# Get content back from guest in a host dir\n";
2901            # Host dir seen as TDIR in container
2902            $cp2target = "$ENV{'PBBUILDDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
2903            pb_mkdir_p("$cp2target");
2904            chmod 0777,"$cp2target";
2905            print DOCKER "TDIR=$homedir/$bdir-2/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}\n";
2906            print DOCKER "for i in $homedir/$bdir/*; do\n";
2907            print DOCKER "  # Cleanup first to avoid rights issues for root owned files e.g.\n";
2908            print DOCKER "  j=\`basename \$i\`\n";
2909            print DOCKER "  if [ -d \$TDIR/\$j ]; then\n";
2910            print DOCKER "    rmdir --ignore-fail-on-non-empty \$TDIR/\$j\n";
2911            print DOCKER "  else\n";
2912            print DOCKER "    rm -f \$TDIR/\$j\n";
2913            print DOCKER "  fi\n";
2914            print DOCKER "  cp -r \$i \$TDIR\n";
2915            print DOCKER "done\n";
2916            print DOCKER "find \$TDIR -type f -print0 | xargs -0 chmod 644 && /bin/true\n";
2917            print DOCKER "find \$TDIR -type d -print0 | xargs -0 chmod 777 2> /dev/null && /bin/true\n";
2918            #print DOCKER "ls -al \$TDIR\n";
2919        }
2920        print DOCKER "exit 0\n";
2921        close(DOCKER);
2922        $shcmd .= " -v $ENV{'PBBUILDDIR'}:$homedir/$bdir-2 -v $ENV{'PBTMP'}/vebuild:$homedir/$tdir-2 -w $homedir/$tdir-2 $tag{$pbstep} $dkcmd";
2923    }
2924    my $ret = pb_system("$shcmd","Executing pbscript on $cptarget if needed",$cmdverb);
2925   
2926    # If target is deb family then sign Release file for Packages
2927    my $pbos2 = pb_distro_get_context($v);
2928    if (($cmt =~ /Packages/) && ($pbos2->{'type'} eq "deb")) {
2929        my $debarch = pb_get_debarch($pbos2);
2930        my $projcomponent = pb_get_debpc($pbos2);
2931        for my $f ("dists/$pbos2->{'version'}/$projcomponent/binary-$debarch/Release","dists/$pbos2->{'version'}/$projcomponent/binary-all/Release","dists/$pbos2->{'version'}/$projcomponent/source/Release","dists/$pbos2->{'version'}/Release") {
2932            my $debdir = dirname($cptarget);
2933            pb_system("cd $ENV{'PBBUILDDIR'} ; $cpcmd $debdir/$f . 2> /dev/null","Getting deb $f file from $debdir");
2934            pb_sign_file("$ENV{'PBBUILDDIR'}/Release");
2935            pb_system("$cpcmd $ENV{'PBBUILDDIR'}/Release.gpg $debdir/$f.gpg 2> /dev/null","Putting deb $f.gpg file to $debdir");
2936            my ($void,$debfile) = split(/:/,$debdir);
2937            pb_system("$shcmdbase \"chmod 644 $debfile/$f.gpg\"","Set mode to 644 for $debfile/$f.gpg");
2938        }
2939    } 
2940
2941    if ($cmt =~ /^(V[EM]|RM)build/) {
2942        # Get back info on pkg produced, compute their name and get them from the VM/RM
2943        pb_system("$cpcmd $cp2target/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'} $ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.$$ 2> /dev/null","Get package names in $cp2target");
2944        if (not -f "$ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.$$") {
2945            pb_log(0,"ERROR with $v on getting $ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.$$\n");
2946            pb_exit(-1);
2947        } else {
2948            open(KEEP,"$ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.$$") || confess "Unable to read $ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.$$";
2949            my $src = <KEEP>;
2950            chomp($src);
2951            close(KEEP);
2952            unlink("$ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.$$");
2953
2954            $src =~ s/^ *//;
2955            pb_mkdir_p("$ENV{'PBBUILDDIR'}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}");
2956            # Change pgben to make the next send2target happy
2957            my $made = "";
2958   
2959            # For VM/RM we don't want shell expansion to hapen locally but remotely
2960            my $delim = '\'';
2961            if ($cmt =~ /^VEbuild/) {
2962                # For VE we need to support shell expansion locally
2963                $delim = "";
2964            }   
2965
2966            open(KEEP,"> $ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}") || confess "Unable to write $ENV{'PBBUILDDIR'}/pbgen-$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}-$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
2967            foreach my $p (split(/ +/,$src)) {
2968                my $j = basename($p);
2969                pb_system("$cpcmd $cp2target/$delim$p$delim $ENV{'PBBUILDDIR'}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'} 2> /dev/null","Recovery of package $j in $ENV{'PBBUILDDIR'}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}");
2970                $made="$made $pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}/$j"; # if (($pbos->{'type'} ne "rpm") || ($j !~ /.src.rpm$/));
2971            }
2972       
2973            print KEEP "$made\n";
2974            close(KEEP);
2975            if (($cmt =~ /^VE/) && ($vetype eq "docker") && ($pbstep >= 3)) {
2976                pb_rm_rf("$cp2target");
2977                if ($pbdebug ge 1) {
2978                    pb_log(2,"WARNING: Container  $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-$cmt NOT removed\n");
2979                }
2980            } else {
2981                pb_system("$shcmdbase \"rm -rf $tdir $bdir\"","$cmt cleanup");
2982            }
2983
2984            # Sign packages locally
2985            pb_sign_pkgs($pbos,$made);
2986
2987            # We want to send them to the ssh account so overwrite what has been done before
2988            pb_log(2,"Before sending pkgs, vexist: $vexist, vpid: $vpid\n");
2989            pb_send2target("Packages","$ENV{'PBDESTDIR'}/pbscript.$$",$pbos->{'name'}."-".$pbos->{'version'}."-".$pbos->{'arch'},$vexist,$vpid);
2990            pb_rm_rf("$ENV{'PBBUILDDIR'}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}");
2991        }
2992    }
2993    unlink("$pbscript{$v}") if ((($cmt =~ /^(V[ME]|RM)/) || ($cmt =~ /Packages/)) && ($pbkeep eq 0) && ($cmt !~ /Script/));
2994
2995    pb_log(2,"Before halt, vexist: $vexist, vpid: $vpid\n");
2996    if ((! $vexist) && ($cmt =~ /^VM/)) {
2997        # If in setupvm then takes a snapshot just before halting
2998        if ($snapme != 0) {
2999            my ($vmmonport,$vmtype) = pb_conf_get("vmmonport","vmtype");
3000            # For monitoring control
3001            if ((($vmtype->{$ENV{'PBPROJ'}}) eq "kvm") || (($vmtype->{$ENV{'PBPROJ'}}) eq "qemu")) {
3002                eval
3003                {
3004                require Net::Telnet;
3005                Net::Telnet->import();
3006                };
3007                if ($@) {
3008                    # Net::Telnet not found
3009                    pb_log(1,"ADVISE: Install Net::Telnet to benefit from monitoring control and snapshot feature.\nWARNING: No snapshot created");
3010                } else {
3011                    my $t = new Net::Telnet (Timeout => 120, Host => "localhost", Port => $vmmonport->{$ENV{'PBPROJ'}}) || confess "Unable to dialog on the monitor";
3012                    # move to monitor mode
3013                    my @lines = $t->cmd("c");
3014                    # Create a snapshot named pb
3015                    @lines = $t->cmd("savevm pb");
3016                    # Write the new status in the VM
3017                    @lines = $t->cmd("commit all");
3018                    # End
3019                    @lines = $t->cmd("quit");
3020                }
3021            }
3022        }
3023        my $hoption = "-p";
3024        my $hpath = pb_distro_get_param($pbos,pb_conf_get("ospathcmd-halt"));
3025        # Solaris doesn't support -p of halt
3026        if ($pbos->{'type'} eq "pkg") {
3027            $hoption = "" ;
3028        }
3029        pb_system("$shcmdbase \"sudo $hpath $hoption \"; $tm ; echo \'if [ -d /proc/$vpid ]; then kill -9 $vpid; fi \' | bash ; sleep 10","VM $v halt (pid $vpid)");
3030    }
3031    if (($cmt =~ /^VE/) && ($snapme != 0)) {
3032        if ($vetype eq "docker") {
3033            foreach my $f (split(/ +/,$src)) {
3034                unlink("$context/$f");
3035            }
3036        } else {
3037            pb_ve_snap($pbos,$tp);
3038        }
3039    }
3040}
3041
3042sub pb_script2v {
3043
3044my $pbscript=shift;
3045my $vtype=shift;
3046my $pbstep=shift;   # Which step are we in (0: create, 1: setup pb, 2: prep pbproj, 3: use to build)
3047my $pbforce=shift;  # Force stop of VM. Default not.
3048my $snapme=shift;   # Do we have to create a snapshot. Default not.
3049my $pbimage=shift;  # Which image to use to start the VM/VE
3050
3051$pbforce = 0 if (not defined $pbforce);
3052$snapme = 0 if (not defined $snapme);
3053$pbstep = 3 if (not defined $pbstep);
3054
3055pb_log(2,"DEBUG: pb_script2v($vtype,$pbstep,$pbforce,$snapme)\n");
3056pb_log(2,"DEBUG: pb_script2v(pbscript: ".Dumper($pbscript)."\n") if (defined $pbscript);
3057pb_parallel_launchv($pbscript,$vtype,uc($vtype)."Script",$pbstep,$pbimage,$pbforce,$snapme,$pbsnap);
3058}
3059
3060sub pb_parallel_launchv {
3061
3062my $pbscript=shift;
3063my $vtype = shift;
3064my $action = shift;         # It an action is defined then use send2target
3065my $pbstep=shift;           # Which step are we in (0: create, 1: setup, 2: build 3: use 4: install 5: test)
3066my $pbimage=shift;          # Which image to use to start the VM/VE
3067my $pbforce=shift;          # Force stop of VM. Default not.
3068my $snapme = shift;         # By default do not snap a VM/VE/RM
3069my $usesnap = shift;        # By default study the usage of the snapshot feature of VM/VE/RM   
3070my $vm;
3071my $all;
3072
3073$pbforce = 0 if (not defined $pbforce);
3074$snapme = 0 if (not defined $snapme);
3075$usesnap = 1 if (not defined $usesnap);
3076
3077# Adapt // mode to memory size
3078$pbparallel = pb_set_parallel($vtype);
3079
3080pb_log(2,"DEBUG: pb_parallel_launch(vtype,step,force,snapme,usesnap: $vtype,$pbstep,$pbforce,$snapme,$usesnap)\n");
3081pb_log(2,"DEBUG: pb_parallel_launch(pbscript: ".Dumper($pbscript)."\n");
3082
3083my ($vexist,$vpid) = (undef,undef);
3084my $pm;
3085my $all_ok = 1;
3086if (defined $pbparallel) {
3087    $pm = new Parallel::ForkManager($pbparallel);
3088
3089    # Set which port the VM/RM will use to communicate
3090    $pm->run_on_start(\&pb_set_port);
3091    $pm->run_on_finish(sub { my ($pid, $code, $id, $signal, $dump) = @_;
3092                            unless ($code == 0 && $signal == 0 && $dump == 0) {
3093                                $all_ok = 0;
3094                                pb_log(0,"ERROR: pid $pid failed\n");
3095                                pb_log(0,"code: $code - signal: $signal - dump: $dump\n");
3096                            }
3097                        });
3098}
3099
3100# Determine on which V to work
3101if (defined $ENV{'PBV'}) {
3102    @$vm = split(/,/,$ENV{'PBV'});
3103} else {
3104    ($vm,$all) = pb_get2v($vtype);
3105}
3106
3107$vetype = pb_ve_get_type($vetype);
3108if ((($vtype =~ /vm/) || (($vtype =~ /ve/) && ($vetype eq "docker"))) && (((not defined $pbimage) || ($pbimage eq "")) && ($pbstep == 0))) {
3109    confess "-i image parameter needed";
3110}
3111
3112my $counter = 0;
3113foreach my $v (@$vm) {
3114    $counter++;
3115    # Modulo 2 * pbparallel (to avoid synchronization problems)
3116    $counter = 1 if ((defined $pbparallel) && ($counter > 2 * $pbparallel));
3117    $pm->start($counter) and next if (defined $pbparallel);
3118   
3119    # If launched via script2v then get the script passed as the one for the that VM
3120    if (not defined $pbscript->{$v}) {
3121        $pbscript->{$v} = $pbscript->{'default'};
3122    }
3123    #
3124    # Launch a single operation
3125    ($vexist,$vpid) = pb_launchv($pbscript->{$v},$vtype,$v,$action,$pbstep,$pbforce,$snapme,$pbsnap,$pbimage);
3126   
3127    # Skip that VM/RM if something went wrong
3128    if (($vtype =~ /(v|r)m/) && ($vpid == 0) && ($vexist == 0)) {
3129        $pm->finish if (defined $pbparallel);
3130        next;
3131    }
3132
3133   
3134    $pm->finish if (defined $pbparallel);
3135}
3136$pm->wait_all_children if (defined $pbparallel);
3137confess "Aborting, one or more of the children failed." if ((not $all_ok) && ($Global::pb_stop_on_error));
3138}
3139
3140sub pb_launchv {
3141
3142my $pbscript1=shift;        # Just the script name to execute, not the hash
3143my $vtype = shift;
3144my $v = shift;              # Only 1 VM/VE/RM treated here
3145my $action = shift;         # It an action is defined then use send2target
3146my $pbstep=shift;           # Which step are we in (0: create, 1: setup, 2: build 3: use 4: install 5: test)
3147my $pbforce=shift;          # Force stop of VM. Default not.
3148my $snapme = shift;         # By default do not snap a VM/VE/RM
3149my $usesnap = shift;            # By default study the usage of the snapshot feature of VM/VE/RM   
3150my $pbimage=shift;          # Which image to use to start the VM/VE
3151my $vexist = undef;
3152my $vpid = undef;
3153
3154$pbforce = 0 if (not defined $pbforce);
3155$snapme = 0 if (not defined $snapme);
3156$usesnap = 1 if (not defined $usesnap);
3157
3158# Check that v exists
3159confess "No VM/VE/RM defined, unable to launch" if (not defined $v);
3160
3161# If creation or snapshot creation mode, no snapshot usable
3162if (($pbstep == 0) || ($snapme == 1)) {
3163    $usesnap = 0;
3164}
3165
3166pb_log(2,"DEBUG: pb_launchv(vtype,v,step,force,snapme,usesnap: $vtype,$v,$pbstep,$pbforce,$snapme,$usesnap)\n");
3167pb_log(2,"DEBUG: pb_launchv(pbimage: $pbimage)\n") if (defined $pbimage);
3168pb_log(2,"DEBUG: pb_launchv(pbscript $pbscript1)\n") if (defined $pbscript1);
3169pb_log(2,"DEBUG: pb_launchv(action: $action)\n") if (defined $action);
3170# Keep only the first VM in case many were given
3171if ($v =~ /,/) {
3172    pb_log(0,"WARNING: pruning to just the first of several VM/VE/RMs listed ($v)\n");
3173    $v =~ s/,.*//;
3174}
3175
3176my $pbos = pb_distro_get_context($v);
3177
3178pb_apply_conf_proxy($pbos);
3179
3180# Launch the VMs/VEs
3181if ($vtype eq "vm") {
3182    my ($ptr,$ptr2,$vmpath,$vmport,$vms) = pb_conf_get("vmtype","vmcmd","vmpath","vmport","vmsize");
3183    my ($vmopt,$vmmm,$vmtmout,$vmsnap,$vmbuildtm,$vmmonport) = pb_conf_get_if("vmopt","vmmem","vmtmout","vmsnap","vmbuildtm","vmmonport");
3184    my $vmsize = pb_distro_get_param($pbos,$vms);
3185
3186    my $vmtype = $ptr->{$ENV{'PBPROJ'}};
3187    my $vmcmd = $ptr2->{$ENV{'PBPROJ'}};
3188
3189    if (defined $opts{'g'}) {
3190        if (($vmtype eq "kvm") || ($vmtype eq "qemu")) {
3191            $ENV{'PBVMOPT'} = "--nographic";
3192        }
3193    }
3194    if (not defined $ENV{'PBVMOPT'}) {
3195        $ENV{'PBVMOPT'} = "";
3196    }
3197    # Save the current status for later restoration
3198    $ENV{'PBOLDVMOPT'} = $ENV{'PBVMOPT'};
3199
3200    if (defined $vmopt->{$v}) {
3201        $ENV{'PBVMOPT'} .= " $vmopt->{$v}" if ($ENV{'PBVMOPT'} !~ / $vmopt->{$v}/);
3202    } elsif (defined $vmopt->{$ENV{'PBPROJ'}}) {
3203        $ENV{'PBVMOPT'} .= " $vmopt->{$ENV{'PBPROJ'}}" if ($ENV{'PBVMOPT'} !~ / $vmopt->{$ENV{'PBPROJ'}}/);
3204    }
3205
3206    # How much memory to allocate for VMs
3207    if (defined $vmmm) {
3208        my $vmmem = pb_distro_get_param($pbos,$vmmm);
3209        if (defined $vmmem) {
3210            $ENV{'PBVMOPT'} .= " -m $vmmem";
3211        }
3212    }
3213
3214    # Are we allowed to use snapshot feature
3215    if ($usesnap == 1) {
3216        if ((defined $vmsnap->{$v}) && ($vmsnap->{$v} =~ /true/i)) {
3217            $ENV{'PBVMOPT'} .= " -snapshot";
3218        } elsif ((defined $vmsnap->{$ENV{'PBPROJ'}}) && ($vmsnap->{$ENV{'PBPROJ'}} =~ /true/i)) {
3219            $ENV{'PBVMOPT'} .= " -snapshot";
3220        } elsif ($pbsnap eq 1) {
3221            $ENV{'PBVMOPT'} .= " -snapshot";
3222        }
3223    } 
3224    if ($snapme != 0) {
3225        if (($vmtype eq "kvm") || ($vmtype eq "qemu")) {
3226            # Configure the monitoring to automate the creation of the 'pb' snapshot
3227            $ENV{'PBVMOPT'} .= " -serial mon:telnet::$vmmonport->{$ENV{'PBPROJ'}},server,nowait" if ((defined $vmmonport) && (defined $vmmonport->{$ENV{'PBPROJ'}}));
3228            # In that case no snapshot call needed
3229            $ENV{'PBVMOPT'} =~ s/ -snapshot//;
3230        }
3231    }
3232   
3233    # Set a default timeout of 2 minutes if none defined externally or in conf files
3234    if (not defined $ENV{'PBVMTMOUT'}) {
3235        if (defined $vmtmout->{$v}) {
3236            $ENV{'PBVMTMOUT'} = $vmtmout->{$v};
3237        } elsif (defined $vmtmout->{$ENV{'PBPROJ'}}) {
3238            $ENV{'PBVMTMOUT'} = $vmtmout->{$ENV{'PBPROJ'}};
3239        }
3240    }
3241    if (not defined $ENV{'PBVMTMOUT'}) {
3242        $ENV{'PBVMTMOUT'} = "120";
3243    }
3244
3245    my $nport = pb_get_port($vmport,$pbos,$vtype);
3246
3247    my $cmd;
3248    my $vmm;        # has to be used for pb_check_ps
3249    if (($vmtype eq "qemu") || ($vmtype eq "kvm")) {
3250        $vmm = "$vmpath->{$ENV{'PBPROJ'}}/$v.qemu";
3251        if (($pbstep == 0) || (defined $pbimage)) {
3252            $ENV{'PBVMOPT'} .= " -cdrom $pbimage -boot d";
3253        }
3254        # Always redirect the network and always try to use a 'pb' snapshot
3255        #$cmd = "$vmcmd $ENV{'PBVMOPT'} -net nic,model=virtio -net user,hostfwd=tcp::$nport:22 -loadvm pb $vmm"
3256        #$cmd = "$vmcmd $ENV{'PBVMOPT'} -redir tcp:$nport:10.0.2.15:22 $vmm"
3257        $cmd = "$vmcmd $ENV{'PBVMOPT'} -net nic -net user,hostfwd=tcp:127.0.0.1:$nport-:22 $vmm"
3258    } elsif ($vmtype eq "xen") {
3259    } elsif ($vmtype eq "vmware") {
3260    } else {
3261        confess "VM of type $vmtype not supported. Report to the dev team";
3262    }
3263    # Restore the ENV VAR Value
3264    $ENV{'PBVMOPT'} = $ENV{'PBOLDVMOPT'};
3265
3266    my ($tmpcmd,$void) = split(/ +/,$cmd);
3267    my $vmmport = undef;
3268    ($vexist,$vmmport) = pb_check_ps($tmpcmd,$vmm);
3269    $vpid = 0;
3270   
3271    if (! $vexist) {
3272        if ($pbstep == 0) {
3273            confess("Found an existing Virtual machine $vmm. Won't overwrite") if (-r $vmm);
3274            if (($vmtype eq "qemu") || ($vmtype eq "xen") || ($vmtype eq "kvm")) {
3275                my $command = pb_check_req("qemu-img",0);
3276                pb_system("$command create -f qcow2 $vmm $vmsize","Creating the QEMU VM");
3277            } elsif ($vmtype eq "vmware") {
3278            } else {
3279            }
3280        }
3281        if (! -f "$vmm") {
3282            pb_log(0,"Unable to find VM $vmm\n");
3283        } else {
3284            # Is the SSH port free? if not kill the existing process using it after a build timeout period
3285            my ($vmssh,$void) = pb_check_ps($tmpcmd,$vmm);
3286            if ($vmssh) {
3287                my $buildtm = $ENV{'PBVMTMOUT'};
3288                if (defined $vmbuildtm->{$v}) {
3289                    $buildtm = $vmbuildtm->{$v};
3290                } elsif (defined $vmbuildtm->{$ENV{'PBPROJ'}}) {
3291                    $buildtm = $vmbuildtm->{$ENV{'PBPROJ'}};
3292                }
3293
3294                sleep $buildtm;
3295                pb_log(0,"WARNING: Killing the process ($vmssh) using port $void (previous failed VM ?)\n");
3296                kill 15,$vmssh;
3297                # Let it time to exit
3298                sleep 5;
3299            }
3300            pb_system("$cmd &","Launching the VM $vmm");
3301            # Using system allows to kill it externaly if needed,sosupport that in the call
3302            pb_system("sleep $ENV{'PBVMTMOUT'}","Waiting $ENV{'PBVMTMOUT'} s for VM $v to come up","mayfail");
3303            ($vpid,$void) = pb_check_ps($tmpcmd,$vmm);
3304            pb_log(0,"VM $vmm launched (pid $vpid)\n");
3305        }
3306    } else {
3307        pb_log(0,"Found an existing VM $vmm (pid $vexist - Port $vmmport)\n");
3308        # Set the correct port here based on what is done
3309        $pbport = $vmmport;
3310        $vpid = $vexist;
3311    }
3312    pb_log(2,"DEBUG: pb_launchv returns ($vexist,$vpid)\n");
3313
3314    # After the VM is launched, then setup the SSH access if not in creation phase
3315    if ($pbstep != 0) {
3316        if (defined $pbport) {
3317            pb_ssh_setup($vtype,$pbport);
3318        } else {
3319            pb_ssh_setup($vtype,$nport);
3320        }
3321    }
3322   
3323    # Skip that VM if something went wrong
3324    return($vexist,$vpid) if (($vpid == 0) && ($vexist == 0));
3325   
3326    # If force stopping the VM then reset vexist
3327    if ($pbforce == 1) {
3328        $vpid = $vexist;
3329        $vexist = 0;
3330    }
3331} elsif ($vtype eq "ve") {
3332    # Force the creation of the VE and no snapshot usable
3333    $vetype = pb_ve_get_type($vetype);
3334    pb_ve_launch($v,$pbstep,$usesnap,$vetype,$pbimage);
3335    $vexist = 0;
3336    $vpid = 0;
3337} else {
3338    # Get RM context
3339    my ($ptr,$rmpath,$rmport) = pb_conf_get("rmtype","rmpath","rmport");
3340
3341    # Nothing more to do for RM. No real launch
3342    # For the moment we support the RM is already running
3343    # For ProLiant may be able to power them on if needed later on as an example.
3344    $vexist = 0;
3345    $vpid = 0;
3346   
3347    # After the RM is launched, then setup the SSH access
3348    my $nport = pb_get_port($rmport,$pbos,$vtype);
3349    pb_ssh_setup($vtype,$nport);
3350}
3351
3352# Gather all required files to send them to the VM/VE/RM
3353# and launch the build through pbscript
3354if (defined $action) {
3355    pb_log(2,"DEBUG: Before send2target, vexist: $vexist, vpid: $vpid, vtype: $vtype, v: $v, action: $action\n");
3356    pb_send2target($action,$pbscript1,$v,$vexist,$vpid,$snapme,$pbstep);
3357}
3358return($vexist,$vpid);
3359}
3360
3361# Return string for date synchro
3362sub pb_date2v {
3363
3364my $vtype = shift;
3365my $pbos = shift;
3366my $nontp = shift || 0;
3367
3368pb_log(1,"Entering pb_date2v\n");
3369# VE gets time from parent OS.
3370return "/bin/true" if ($vtype) =~ /^ve/o; 
3371
3372my ($ntp) = pb_conf_get_if($vtype."ntp");
3373my $vntp = undef;
3374my $ntpline = undef;
3375$vntp = $ntp->{$ENV{'PBPROJ'}} if (defined $ntp);
3376
3377if ((defined $vntp) && ($vntp !~ /^\s*$/)) {
3378    pb_log(2,"ntp server is $vntp\n");
3379    # ntp command depends on pbos
3380    my $vntpcmd = pb_distro_get_param($pbos,pb_conf_get("oscmdntp"));
3381    $ntpline = "sudo $vntpcmd $vntp";
3382}
3383# Force new date to be in the future compared to the date
3384# of the host by adding 1 minute
3385my @date=pb_get_date();
3386$date[1]++;
3387my $upddate = strftime("%m%d%H%M%Y", @date);
3388my $dateline = "sudo /bin/date $upddate";
3389if ((defined $ntpline) && ($nontp == 0)) {
3390    pb_log(1,"pb_date2v returns $ntpline\n");
3391    return($ntpline);
3392} else {
3393    pb_log(1,"pb_date2v returns $dateline\n");
3394    return($dateline);
3395}
3396}
3397
3398sub pb_build2v {
3399
3400my $vtype = shift;
3401my $action = shift;     #build, test or prep
3402my $pbforce=shift;      # Force stop of VM. Default not.
3403
3404$action = "build" if (not defined $action);
3405$pbforce = 0 if (not defined $pbforce);
3406
3407my $pbstep;
3408
3409$vetype = pb_ve_get_type($vetype);
3410if ($action eq "prep") {
3411    $pbstep = 2;
3412} elsif ($action eq "install") {
3413    $pbstep = 4;
3414} elsif ($action eq "test") {
3415    $pbstep = 5;
3416} elsif ($action eq "build") {
3417    $pbstep = 3;
3418    # Need to be traversed by others if using containers
3419    if (($vtype eq "ve") && ($vetype eq "docker")) {
3420        chmod 0751,"$ENV{'PBBUILDDIR'}";
3421    }
3422} else {
3423    pb_log(0,"pb_build2v: unknown action $action\n");
3424    confess "Unable to continue";
3425}
3426
3427pb_log(1,"Entering in pb_build2v ($action in $vtype) \n");
3428
3429# We prepare scripts for all V of this vtype to allow sharing that part
3430my ($v,$all) = pb_get2v($vtype);
3431
3432# Send tar files when we do a global generation
3433pb_build2ssh() if (($all == 1) && ($action eq "build"));
3434
3435# Adapt // mode to memory size
3436$pbparallel = pb_set_parallel($vtype);
3437
3438my ($vexist,$vpid) = (undef,undef);
3439my $pm;
3440my $all_ok = 1;
3441if (defined $pbparallel) {
3442    $pm = new Parallel::ForkManager($pbparallel);
3443
3444    # Set which port the VM/RM will use to communicate
3445    $pm->run_on_start(\&pb_set_port);
3446    $pm->run_on_finish(sub { my ($pid, $code, $id, $signal, $dump) = @_;
3447                            unless ($code == 0 && $signal == 0 && $dump == 0) {
3448                                $all_ok = 0;
3449                                pb_log(0,"ERROR: pid $pid failed\n");
3450                                pb_log(0,"code: $code - signal: $signal - dump: $dump\n");
3451                            }
3452                        });
3453}
3454
3455my $counter = 0;
3456foreach my $v (@$v) {
3457    $counter++;
3458    $pbscript{$v} = "$ENV{'PBDESTDIR'}/pb$action-$v";
3459    # Modulo 2 * pbparallel (to avoid synchronization problems)
3460    $counter = 1 if ((defined $pbparallel) && ($counter > 2 * $pbparallel));
3461    $pm->start($counter) and next if (defined $pbparallel);
3462    #
3463    # Prepare the script to be executed on the VM/VE/RM
3464    open(SCRIPT,"> $pbscript{$v}") || confess "Unable to create $pbscript{$v}";
3465    print SCRIPT "#!/bin/bash\n";
3466
3467    # Transmit the verbosity level to the virtual env/mach.
3468    my $verbose = "";
3469    my $i = 0;                          # minimal debug level
3470    while ($i lt $pbdebug) {
3471        $verbose .= "-v ";
3472        $i++;
3473    }
3474    print SCRIPT "set -e\n" if ($Global::pb_stop_on_error);
3475    # Activate script verbosity if at least 2 for pbdebug
3476    print SCRIPT "set -x\n" if ($i gt 1);
3477    # Quiet if asked to be so on the original system
3478    $verbose = "-q" if ($pbdebug eq -1);
3479
3480    print SCRIPT "echo ... Execution needed\n";
3481    print SCRIPT "echo ==== Start of script for $vtype $v =====\n";
3482    print SCRIPT "# This is in directory pbdelivery\n";
3483    print SCRIPT "# Setup the variables required for building\n";
3484    print SCRIPT "export PBPROJ=$ENV{'PBPROJ'}\n";
3485
3486    if (($action eq "build") || ($action eq "prep")) {
3487        print SCRIPT "# Preparation for pb\n";
3488        print SCRIPT "rm -f \$HOME/.pbrc.yml\n";
3489        print SCRIPT "rm -f \$HOME/.pbrc\n";
3490        print SCRIPT "cp .pbrc.yml \$HOME\n";
3491        # TODO: Why ?
3492        print SCRIPT "cd ..\n";
3493    }
3494
3495    # VE needs a good /proc, tolerate one being potentially left around after a failure
3496    if (($vtype eq "ve") && ($vetype ne "docker")) {
3497        print SCRIPT "[ -d /proc/1 ] || sudo /bin/mount -t proc /proc /proc\n";
3498    }
3499
3500    # Get distro context
3501    my $pbos = pb_distro_get_context($v);
3502   
3503    my $ntpline = pb_date2v($vtype,$pbos);
3504    print SCRIPT "# Time sync\n";
3505    print SCRIPT "echo setting up date with $ntpline\n";
3506    print SCRIPT "$ntpline\n";
3507    print SCRIPT "if [ \$\? -ne 0 ]; then\n";
3508    # Force if bad status a trick using date
3509    $ntpline = pb_date2v($vtype,$pbos, 1);
3510    print SCRIPT "  $ntpline\n";
3511    print SCRIPT "fi\n";
3512    # Use potential local proxy declaration in case we need it to download repo, pkgs, ...
3513    if (defined $ENV{'http_proxy'}) {
3514        print SCRIPT "export http_proxy=\"$ENV{'http_proxy'}\"\n";
3515    }
3516    if (defined $ENV{'https_proxy'}) {
3517        print SCRIPT "export https_proxy=\"$ENV{'https_proxy'}\"\n";
3518    }
3519    if (defined $ENV{'ftp_proxy'}) {
3520        print SCRIPT "export ftp_proxy=\"$ENV{'ftp_proxy'}\"\n";
3521    }
3522
3523    # Get list of packages to build/test and get some ENV vars as well
3524    my $ptr = pb_get_pkg();
3525    @pkgs = @$ptr;
3526    my $p = join(' ',@pkgs) if (@pkgs);
3527    print SCRIPT "export PBPROJVER=$ENV{'PBPROJVER'}\n";
3528    print SCRIPT "export PBPROJTAG=$ENV{'PBPROJTAG'}\n";
3529    print SCRIPT "export PBPACKAGER=\"$ENV{'PBPACKAGER'}\"\n";
3530    # TODO: setup apk pkg/keys here
3531
3532    # We may need to do some other tasks before building. Read a script here to finish setup
3533    if (-x "$ENV{'PBTMP'}/pb$vtype".".pre") {
3534        print SCRIPT "# Special pre-instructions to be launched\n";
3535        print SCRIPT pb_get_content("$ENV{'PBTMP'}/pb$vtype".".pre");
3536    }
3537
3538    if (-x "$ENV{'PBTMP'}/pb$vtype"."$action.pre") {
3539        print SCRIPT "# Special pre-$action instructions to be launched\n";
3540        print SCRIPT pb_get_content("$ENV{'PBTMP'}/pb$vtype"."$action.pre");
3541    }
3542
3543    print SCRIPT "# $action\n";
3544    print SCRIPT "echo $action"."ing packages on $vtype...\n";
3545
3546    print SCRIPT "pb --version\n";
3547    my $act = "$action"."2pkg";
3548    $act = "build2prep" if ($action eq "prep");
3549    print SCRIPT "pb $verbose -p $ENV{'PBPROJ'} $act $p\n";
3550
3551    if (($vtype eq "ve") && ($vetype ne "docker")) {
3552        print SCRIPT "sudo /bin/umount /proc\n";
3553    }
3554
3555    # We may need to do some other tasks after building. Read a script here to exit properly
3556    if (-x "$ENV{'PBTMP'}/pb$vtype"."$action.post") {
3557        print SCRIPT "# Special post-$action instructions to be launched\n";
3558        print SCRIPT pb_get_content("$ENV{'PBTMP'}/pb$vtype"."$action.post");
3559    }
3560
3561    if (-x "$ENV{'PBTMP'}/pb$vtype".".post") {
3562        print SCRIPT "# Special post-instructions to be launched\n";
3563        print SCRIPT pb_get_content("$ENV{'PBTMP'}/pb$vtype".".post");
3564    }
3565
3566    print SCRIPT "echo ==== End of script $$ for $vtype $v =====\n";
3567    close(SCRIPT);
3568    chmod 0755,"$pbscript{$v}";
3569   
3570    $pm->finish if (defined $pbparallel);
3571}
3572$pm->wait_all_children if (defined $pbparallel);
3573confess "Aborting, one or more of the children failed." if ((not $all_ok) && ($Global::pb_stop_on_error));
3574
3575# Launch the VM/VE/RM
3576# docker image to use if needed
3577# Gather all required files to send them to the VM/VE
3578# and launch the build through pbscript
3579pb_parallel_launchv(\%pbscript,$vtype,uc($vtype).$action,$pbstep,$pbimage,$pbforce);
3580}
3581
3582
3583sub pb_clean {
3584
3585    my $sleep=10;
3586    confess "Unable to get env var PBDESTDIR" if (not defined $ENV{'PBDESTDIR'});
3587    confess "Unable to get env var PBBUILDDIR" if (not defined $ENV{'PBBUILDDIR'});
3588    pb_log(0,"We will now wait $sleep s before removing both directories\n$ENV{'PBDESTDIR'} and $ENV{'PBBUILDDIR'}.\nPlease break me if this is wrong\n");
3589    sleep $sleep;
3590    pb_rm_rf($ENV{'PBDESTDIR'});
3591    pb_rm_rf($ENV{'PBBUILDDIR'});
3592}
3593
3594sub pb_newver {
3595
3596    confess "-V Version parameter needed" if ((not defined $newver) || ($newver eq ""));
3597
3598    # Need this call for PBDIR
3599    my ($scheme2,$uri) = pb_vcs_init($pbinit);
3600
3601    my ($pbconf,$pbprojurl) = pb_conf_get("pbconfurl","pbprojurl");
3602    $uri = $pbconf->{$ENV{'PBPROJ'}};
3603    my ($scheme, $account, $host, $port, $path) = pb_get_uri($uri);
3604
3605    # Checking CMS repositories status
3606    ($scheme2, $account, $host, $port, $path) = pb_get_uri($pbprojurl->{$ENV{'PBPROJ'}});
3607
3608    if ($scheme !~ /^svn/) {
3609        confess "Only SVN is supported at the moment";
3610    }
3611
3612    my $res = pb_vcs_isdiff($scheme,$ENV{'PBROOTDIR'});
3613    confess "ERROR: No differences accepted in CMS for $ENV{'PBROOTDIR'} before creating a new version" if ($res != 0);
3614
3615    $res = pb_vcs_isdiff($scheme2,$ENV{'PBDIR'});
3616    confess "ERROR: No differences accepted in CMS for $ENV{'PBDIR'} before creating a new version" if ($res != 0);
3617
3618    # Tree identical between PBCONFDIR and PBROOTDIR. The delta is what
3619    # we want to get for the root of the new URL
3620
3621    my $oldver = $ENV{'PBROOTDIR'};
3622    $oldver =~ s|^$ENV{'PBCONFDIR'}||;
3623
3624    pb_log(2, "PBCONFDIR: $ENV{'PBCONFDIR'}\nPBROOTDIR: $ENV{'PBROOTDIR'}\n");
3625
3626    my $newurl = "$uri/$newver";
3627    # Should probably use projver in the old file
3628    my $oldvertxt= basename($oldver);
3629    my $newvertxt = basename($newver);
3630
3631    # Duplicate and extract project-builder part
3632    pb_log(2,"Copying $uri/$oldver to $newurl\n");
3633    pb_vcs_copy($scheme,"$uri/$oldver",$newurl);
3634    pb_log(2,"Checkout $newurl to $ENV{'PBCONFDIR'}/$newver\n");
3635    pb_vcs_up($scheme,"$ENV{'PBCONFDIR'}");
3636
3637    # Duplicate and extract project
3638    my $newurl2 = "$pbprojurl->{$ENV{'PBPROJ'}}/$newver";
3639
3640    pb_log(2,"Copying $pbprojurl->{$ENV{'PBPROJ'}}/$oldver to $newurl2\n");
3641    pb_vcs_copy($scheme2,"$pbprojurl->{$ENV{'PBPROJ'}}/$oldver",$newurl2);
3642
3643    my $tmp = $ENV{'PBDIR'};
3644    $tmp =~ s|$oldver$||;
3645    pb_log(2,"Checkout $newurl2 to $tmp/$newver\n");
3646    pb_vcs_up($scheme2,"$tmp");
3647
3648    # Update the .yml file
3649    open(FILE,"$ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml") || confess "Unable to open $ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml";
3650    open(OUT,"> $ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml.new") || confess "Unable to write to $ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml.new";
3651    while(<FILE>) {
3652        if (/^projver\s+$ENV{'PBPROJ'}\s*=\s*$oldvertxt$/) {
3653            s/^projver\s+$ENV{'PBPROJ'}\s*=\s*$oldvertxt$/projver $ENV{'PBPROJ'} = $newvertxt/;
3654            pb_log(0,"Changing projver from $oldvertxt to $newvertxt in $ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.pb\n");
3655        }
3656        if (/^projver:\n\s+$ENV{'PBPROJ'}:\s*$oldvertxt$/) {
3657            s/^projver:\n\s+$ENV{'PBPROJ'}:\s*$oldvertxt$/projver:\n  $ENV{'PBPROJ'}: $newvertxt/;
3658            pb_log(0,"Changing projver from $oldvertxt to $newvertxt in $ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml\n");
3659        }
3660        if (/^testver/) {
3661            s/^testver:\n\s+$ENV{'PBPROJ'}:(.*)$/#testver:\n#\s+$ENV{'PBPROJ'}: $1/;
3662            pb_log(0,"Commenting testver in $ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml\n");
3663            my $txt = $_;
3664            chomp($txt);
3665        }
3666        if (/^delivery/) {
3667            my $txt = $_;
3668            chomp($txt);
3669            pb_log(0,"Please check delivery ($txt) in $ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml\n");
3670        }
3671        print OUT $_;
3672    }
3673    close(FILE);
3674    close(OUT);
3675    rename("$ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml.new","$ENV{'PBCONFDIR'}/$newver/$ENV{'PBPROJ'}.yml");
3676
3677    # Checking pbcl files
3678    foreach my $f (<$ENV{'PBROOTDIR'}/*/pbcl>) {
3679        # Compute new pbcl file
3680        my $f2 = $f;
3681        $f2 =~ s|$ENV{'PBROOTDIR'}|$ENV{'PBCONFDIR'}/$newver/|;
3682        open(PBCL,$f) || confess "Unable to open $f";
3683        my $foundnew = 0;
3684        while (<PBCL>) {
3685            $foundnew = 1 if (/^$newvertxt \(/);
3686        }
3687        close(PBCL);
3688        open(OUT,"> $f2") || confess "Unable to write to $f2: $!";
3689        open(PBCL,$f) || confess "Unable to open $f";
3690        # We skip as long as we do not reach the new version in the log (case of devel)
3691        my $skip = 1;
3692        # We let the 4 first lines
3693        my $cnt = 0;
3694        while (<PBCL>) {
3695            $skip = 0 if (/^$newvertxt \(/);
3696            if ((/^$oldvertxt \(/) && ($foundnew == 0)) {
3697                print OUT "$newvertxt ($pbdate)\n";
3698                print OUT "- TBD\n";
3699                print OUT "\n";
3700                pb_log(0,"WARNING: version $newvertxt not found in $f so added to $f2...\n") if ($foundnew == 0);
3701            }
3702            if ($cnt <= 3) {
3703                print OUT "$_" if (not /^$oldvertxt \(/);
3704            } else {
3705                print OUT "$_" if ($skip == 0);
3706            }
3707            $cnt++;
3708        }
3709        close(OUT);
3710        close(PBCL);
3711    }
3712
3713    pb_log(2,"Checkin $ENV{'PBCONFDIR'}/$newver\n");
3714    pb_vcs_checkin($scheme,"$ENV{'PBCONFDIR'}/$newver","updated to $ENV{'PBCONFDIR'}");
3715}
3716
3717#
3718# Return the list of VMs/VEs/RMs we are working on
3719# $all is a flag to know if we return all of them
3720# or only some (if all we publish also tar files in addition to pkgs
3721#
3722sub pb_get2v {
3723
3724my $vtype = shift;
3725my @v;
3726my $all = 0;
3727my $pbv = 'PBV';
3728my $vlist = $vtype."list";
3729
3730# Get VM/VE list
3731if ((not defined $ENV{$pbv}) || ($ENV{$pbv} =~ /^all$/)) {
3732    my ($ptr) = pb_conf_get($vlist);
3733    $ENV{$pbv} = $ptr->{$ENV{'PBPROJ'}};
3734    $all = 1;
3735}
3736pb_log(2,"$vtype: $ENV{$pbv}\n");
3737@v = split(/,/,$ENV{$pbv});
3738return(\@v,$all);
3739}
3740
3741# This function creates a giant script to configure a particular VM/VE/RM, it then copies the
3742# script to the target.
3743
3744# Function to create a potentialy missing pb account on the VM/VE/RM, and adds it to sudo
3745# Needs to use root account to connect to the VM/VE/RM
3746# pb will take your local public SSH key to access
3747# the pb account in the VM/VE/RM later on if needed
3748sub pb_setup2v {
3749
3750my $vtype = shift;
3751my $sbx = shift;
3752my $installtype = shift;        # either "file" or "pkg" to overwrite conf file
3753my $pbstep = 1;
3754
3755my ($vm,$all) = pb_get2v($vtype);
3756
3757# Adapt // mode to memory size
3758$pbparallel = pb_set_parallel($vtype);
3759
3760my $pm;
3761my $all_ok = 1;
3762if (defined $pbparallel) {
3763    $pm = new Parallel::ForkManager($pbparallel);
3764
3765    # Set which port the VM/RM will use to communicate
3766    $pm->run_on_start(\&pb_set_port);
3767    $pm->run_on_finish(sub { my ($pid, $code, $id, $signal, $dump) = @_;
3768                            $all_ok = 0 unless (($code == 0) && ($signal == 0) && ($dump == 0)); 
3769                        });
3770}
3771
3772my $counter = 0;
3773foreach my $v (@$vm) {
3774    # Script generated - should be before the fork - after the %pbscript hash isn't updated correctly
3775    $pbscript{$v} = "$ENV{'PBDESTDIR'}/setupv-$v";
3776
3777    $counter++;
3778    if (defined $pbparallel) {
3779        # Modulo 2 * pbparallel (to avoid synchronization problems)
3780        $counter = 1 if ($counter > 2 * $pbparallel);
3781        $pm->start($counter) and next;
3782    }
3783
3784    # Get distro context
3785    my $pbos = pb_distro_get_context($v);
3786   
3787    # Deal with date sync.
3788    my $ntpline = pb_date2v($vtype,$pbos);
3789
3790    # Name of the account to deal with for VM/VE/RM
3791    # Do not use the one passed potentially with -a
3792    my ($pbac) = pb_conf_get($vtype."login");
3793    if ((not defined $pbac) || (not defined $pbac->{$ENV{'PBPROJ'}})) {
3794        cluck "No $vtype"."login defined in your project yml conf file";
3795        next;
3796    }
3797
3798    my $key = "";
3799
3800    # Prepare the script to be executed on the VM/VE/RM
3801    # in $ENV{'PBDESTDIR'}/setupv
3802
3803    pb_log(2,"Preparing setup script for $v in $pbscript{$v}\n");
3804    open(SCRIPT,"> $pbscript{$v}") || confess "Unable to create $pbscript{$v}";
3805   
3806    print SCRIPT << 'EOF';
3807#!/usr/bin/perl -w
3808
3809use strict;
3810use File::Copy;
3811
3812# We should not need in this script more functions than what is provided
3813# by Base, Conf and Distribution to avoid problems at exec time.
3814# They are appended at the end.
3815
3816# Define mandatory global vars
3817EOF
3818    my $ppref = "our";
3819    $ppref = "my" if (($pbos->{'name'} =~ /^redhat/) && ($pbos->{'version'} =~ /^6/));
3820    print SCRIPT << "EOF";
3821$ppref \$pbdebug = $pbdebug;
3822$ppref \$pbLOG;
3823$ppref \$pbsynmsg = "pbscript";
3824$ppref \$pbdisplaytype = "text";
3825$ppref \$pblocale = "";
3826$ppref \$PBCONFVER = $projectbuilderconfver;
3827EOF
3828    print SCRIPT << 'EOF';
3829pb_log_init($pbdebug, $pbLOG);
3830EOF
3831    print SCRIPT << "EOF";
3832\$Global::pb_stop_on_error = $Global::pb_stop_on_error;
3833pb_temp_init($pbkeep);
3834pb_conf_init("$ENV{'PBPROJ'}");
3835
3836EOF
3837
3838    if ($vtype eq "ve") {
3839        $vetype = pb_ve_get_type($vetype);
3840        if ($vetype ne "docker") {
3841            print SCRIPT << "EOF";
3842# For VE we first need to mount some FS
3843pb_system("mount -t proc /proc /proc") unless (-d "/proc/$$");
3844EOF
3845        }
3846        print SCRIPT << "EOF";
3847# For VE we need a good null dev
3848# Except for RedHat 6.2 where it's good and doesn't like being recreated
3849EOF
3850        print SCRIPT "pb_system('rm -f /dev/null; mknod /dev/null c 1 3; chmod 777 /dev/null');\n" unless (($pbos->{'name'} =~ /^redhat/) && ($pbos->{'version'} =~ /^6/));
3851    } elsif ($vtype =~ /(v|r)m/) {
3852        # Nothing to do now.
3853        my $useless = 0;
3854    } else {
3855        confess "Unknown virtual type $vtype";
3856    }
3857
3858    print SCRIPT << 'EOF';
3859
3860# Adds $pbac->{$ENV{'PBPROJ'}} as an account if needed
3861#
3862my $file="/etc/passwd";
3863open(PBFILE,$file) || die "Unable to open $file";
3864my $found = 0;
3865while (<PBFILE>) {
3866EOF
3867    print SCRIPT << "EOF";
3868    \$found = 1 if (/^$pbac->{$ENV{'PBPROJ'}}:/);
3869EOF
3870
3871# TODO: use an external parameter
3872my $home = "/home";
3873# Solaris doesn't like that we use /home
3874$home = "/export/home" if ($pbos->{'type'} eq "pkg");
3875
3876    print SCRIPT << "EOF";
3877}
3878close(PBFILE);
3879
3880if ( \$found == 0 ) {
3881    if ( ! -d "$home" ) {
3882        pb_mkdir_p("$home");
3883    }
3884EOF
3885    # TODO: Level of portability of these cmds ? Critical now for RM
3886    # TODO: use an external parameter
3887    print SCRIPT << "EOF";
3888    # Most Linux distros
3889    if ( -x "/usr/sbin/groupadd" ) {
3890        pb_system("/usr/sbin/groupadd $pbac->{$ENV{'PBPROJ'}}","Adding group $pbac->{$ENV{'PBPROJ'}}");
3891    # Alpine with busybox
3892    } elsif ( -x "/usr/sbin/addgroup" ) {
3893        pb_system("/usr/sbin/addgroup $pbac->{$ENV{'PBPROJ'}}","Adding group $pbac->{$ENV{'PBPROJ'}}");
3894    } else {
3895        die "Unable to find a command to add a group on this distribution";
3896    }
3897    # Most Linux distros
3898    if ( -x "/usr/sbin/useradd" ) {
3899        pb_system("/usr/sbin/useradd -g $pbac->{$ENV{'PBPROJ'}} -m -d $home/$pbac->{$ENV{'PBPROJ'}} $pbac->{$ENV{'PBPROJ'}}","Adding user $pbac->{$ENV{'PBPROJ'}} (group $pbac->{$ENV{'PBPROJ'}} - home $home/$pbac->{$ENV{'PBPROJ'}})");
3900    # Alpine with busybox
3901    } elsif ( -x "/usr/sbin/adduser" ) {
3902        pb_system("/usr/sbin/adduser -G $pbac->{$ENV{'PBPROJ'}} -h $home/$pbac->{$ENV{'PBPROJ'}} -D $pbac->{$ENV{'PBPROJ'}}","Adding user $pbac->{$ENV{'PBPROJ'}} (group $pbac->{$ENV{'PBPROJ'}} - home $home/$pbac->{$ENV{'PBPROJ'}})");
3903        pb_system("/usr/sbin/addgroup $pbac->{$ENV{'PBPROJ'}} abuild","Adding $pbac->{$ENV{'PBPROJ'}} to group abuild");
3904    } else {
3905        die "Unable to find a command to add a user on this distribution";
3906    }
3907}
3908EOF
3909
3910    # Copy the content of our local conf file to the VM/VE/RM
3911    my $content = pb_get_content(pb_distro_conffile());
3912    print SCRIPT << "EOF";
3913    #
3914    # Create a temporary local conf file for distribution support
3915    # This is created here before its use later. Its place is hardcoded, so no choice for the path
3916    #
3917    my \$tempconf = pb_distro_conffile();
3918    pb_mkdir_p(dirname(\$tempconf));
3919    open(CONF,"> \$tempconf") || die "Unable to create \$tempconf";
3920    print CONF q{$content};
3921    close(CONF);
3922EOF
3923
3924    if ($vtype =~ /(v|r)m/) {
3925        print SCRIPT << "EOF";
3926# allow ssh entry to build
3927#
3928mkdir "$home/$pbac->{$ENV{'PBPROJ'}}/.ssh",0700;
3929# Allow those accessing root to access the build account
3930copy("\$ENV{'HOME'}/.ssh/authorized_keys","$home/$pbac->{$ENV{'PBPROJ'}}/.ssh/authorized_keys");
3931chmod 0600,".ssh/authorized_keys";
3932pb_system("chown -R $pbac->{$ENV{'PBPROJ'}}:$pbac->{$ENV{'PBPROJ'}} $home/$pbac->{$ENV{'PBPROJ'}}","Finish setting up the account env for $pbac->{$ENV{'PBPROJ'}}");
3933
3934EOF
3935}
3936    print SCRIPT << 'EOF';
3937# No passwd for build account only keys
3938$file="/etc/shadow";
3939if (-f $file) {
3940    open(PBFILE,$file) || die "Unable to open $file";
3941    open(PBOUT,"> $file.new") || die "Unable to open $file.new";
3942    while (<PBFILE>) {
3943EOF
3944    print SCRIPT << "EOF";
3945        s/^$pbac->{$ENV{'PBPROJ'}}:\!\!:/$pbac->{$ENV{'PBPROJ'}}:*:/;
3946        s/^$pbac->{$ENV{'PBPROJ'}}:\!:/$pbac->{$ENV{'PBPROJ'}}:*:/; #SLES 9 e.g.
3947        s/^$pbac->{$ENV{'PBPROJ'}}:\\*LK\\*:/$pbac->{$ENV{'PBPROJ'}}:NP:/;  #Solaris e.g.
3948EOF
3949        print SCRIPT << 'EOF';
3950        print PBOUT $_;
3951    }
3952    close(PBFILE);
3953    close(PBOUT);
3954    rename("$file.new",$file);
3955    chmod 0640,$file;
3956    }
3957
3958# TODO: systemd support
3959# Keep the VM in text mode
3960$file="/etc/inittab";
3961if (-f $file) {
3962    open(PBFILE,$file) || die "Unable to open $file";
3963    open(PBOUT,"> $file.new") || die "Unable to open $file.new";
3964    while (<PBFILE>) {
3965        s/^(..):5:initdefault:$/$1:3:initdefault:/;
3966        print PBOUT $_;
3967    }
3968    close(PBFILE);
3969    close(PBOUT);
3970    rename("$file.new",$file);
3971    chmod 0640,$file;
3972}
3973
3974# pb has to be added to portage group on gentoo
3975
3976# We need to have that pb_distro_get_context function
3977# Get it from ProjectBuilder::Distribution
3978# And we now need the conf file required for this to work created above
3979
3980my $pbos = pb_distro_get_context(); 
3981print "distro tuple: ".Dumper($pbos)."\n";
3982
3983# Adapt sudoers
3984# sudo is not default on Solaris and needs to be installed first
3985# from http://www.sunfreeware.com/programlistsparc10.html#sudo
3986if ($pbos->{'type'} eq "pkg") {
3987    $file="/usr/local/etc/sudoers";
3988} else {
3989    $file="/etc/sudoers";
3990}
3991open(PBFILE,$file) || die "Unable to open $file";
3992open(PBOUT,"> $file.new") || die "Unable to open $file.new";
3993while (<PBFILE>) {
3994EOF
3995    # Skip what will be generated
3996    print SCRIPT << "EOF";
3997    next if (/^$pbac->{$ENV{'PBPROJ'}}\\s+/);
3998    next if (/^Defaults:$pbac->{$ENV{'PBPROJ'}}\\s+/);
3999    next if (/^Defaults:root\\s+\!requiretty/);
4000    next if (/^Defaults:root\\s+env_keep/);
4001EOF
4002    print SCRIPT << 'EOF';
4003    s/Defaults\s+requiretty/# disable Defaults: requiretty/;
4004    print PBOUT $_;
4005}
4006close(PBFILE);
4007EOF
4008    print SCRIPT << "EOF";
4009# Some distro force requiretty at compile time, so disable here
4010print PBOUT "Defaults:$pbac->{$ENV{'PBPROJ'}} !requiretty\\n";
4011print PBOUT "Defaults:root !requiretty\\n";
4012EOF
4013    # RH 6.2 sudo doesn't support env_keep
4014    if (($pbos->{'name'} ne "redhat") && ($pbos->{'version'} ne "6.2")) {
4015        print SCRIPT << "EOF";
4016# Keep proxy configuration while using sudo
4017print PBOUT "Defaults:$pbac->{$ENV{'PBPROJ'}}    env_keep += \\\"http_proxy https_proxy ftp_proxy\\\"\\n";
4018print PBOUT "Defaults:root    env_keep += \\\"http_proxy https_proxy ftp_proxy\\\"\\n";
4019EOF
4020    }
4021    # Try to restrict security to what is really needed
4022    if ($vtype =~ /^vm/) {
4023        my $hpath = pb_distro_get_param($pbos,pb_conf_get("ospathcmd-halt"));
4024        my @sudocmds = pb_get_sudocmds($pbos,$ntpline,"sudo $hpath","sudo /bin/date");
4025        print SCRIPT << "EOF";
4026# This is needed in order to be able on VM to halt the machine from the $pbac->{$ENV{'PBPROJ'}} account at least
4027# Build account $pbac->{$ENV{'PBPROJ'}} in VM also needs to setup date and install deps.
4028# Nothing else should be needed
4029EOF
4030        foreach my $c (@sudocmds) {
4031            print SCRIPT "print PBOUT \"$pbac->{$ENV{'PBPROJ'}}   ALL = NOPASSWD: $c\\n\";\n";
4032        }
4033    } elsif ($vtype =~ /^rm/) {
4034        my @sudocmds = pb_get_sudocmds($pbos,$ntpline);
4035        print SCRIPT << "EOF";
4036# Build account $pbac->{$ENV{'PBPROJ'}} in RM only needs to setup date and install deps if needed each time
4037EOF
4038        foreach my $c (@sudocmds) {
4039            print SCRIPT "print PBOUT \"$pbac->{$ENV{'PBPROJ'}}   ALL = NOPASSWD: $c\\n\";\n";
4040        }
4041    } else {
4042        print SCRIPT << "EOF";
4043# Build account $pbac->{$ENV{'PBPROJ'}} for VE needs to do a lot in the host (and chroot), so allow without restriction for now
4044print PBOUT "$pbac->{$ENV{'PBPROJ'}}   ALL=(ALL) NOPASSWD:ALL\n";
4045EOF
4046    }
4047    my $sudomode = pb_distro_get_param($pbos,pb_conf_get("ossudoersmode"));
4048    print SCRIPT << 'EOF';
4049close(PBOUT);
4050rename("$file.new",$file);
4051EOF
4052    print SCRIPT << "EOF";
4053chmod 0$sudomode,\$file;
4054
4055EOF
4056
4057    if ($vtype =~ /(v|r)m/) {
4058        # Sync date
4059        # do it after sudoers is setup
4060        print SCRIPT "my \$res = pb_system(\"$ntpline\",\"Updating time with $ntpline\",\"mayfailverbose\");\n";
4061        print SCRIPT "if (\$res != 0) {\n";
4062        # try again with another method
4063        $ntpline = pb_date2v($vtype,$pbos,1);
4064        print SCRIPT "  pb_system(\"$ntpline\",\"Updating time with $ntpline\",\"mayfailverbose\");\n";
4065        print SCRIPT "}\n";
4066    }
4067
4068    print SCRIPT << 'EOF';
4069   
4070# We may need a proxy configuration. Get it from the local env
4071pb_apply_conf_proxy($pbos);
4072
4073# Clean up packages DB
4074pb_system("dnf clean all","Cleaning dnf env","mayfail") if (($pbos->{'name'} eq "fedora") && ($pbos->{'version'} >= 22));
4075pb_system("yum clean all","Cleaning yum env","mayfail") if ((($pbos->{'name'} eq "fedora") && ($pbos->{'version'} < 22)) || ($pbos->{'name'} eq "asianux") || ($pbos->{'name'} eq "rhel"));
4076
4077EOF
4078    my $itype = pb_distro_get_param($pbos,pb_conf_get("pbinstalltype"));
4079    # Install from sandbox mean a file base install
4080    $itype = "file" if (defined $sbx);
4081    $itype = $installtype if (defined $installtype);
4082    if ($itype =~ /^file/) {
4083        my ($addrepo) = pb_conf_read("$ENV{'PBDESTDIR'}/pbrc.yml","addbuildrepo");
4084        my $pbrepo = pb_distro_get_param($pbos,$addrepo);
4085        print SCRIPT << 'EOF';
4086
4087# First install all required packages
4088my ($ospkgdep) = pb_conf_get_if("ospkgdep");
4089my $pkgdep = pb_distro_get_param($pbos,$ospkgdep);
4090my $forcerepo;
4091EOF
4092        print SCRIPT "\$forcerepo = pb_distro_setuprepo_gen(\$pbos,\"$pbrepo\");\n" if ((defined $pbrepo) && ($pbrepo ne ""));
4093        print SCRIPT << 'EOF';
4094pb_distro_installdeps(undef,$pbos,$forcerepo,pb_distro_only_deps_needed($pbos,join(' ',split(/,/,$pkgdep))));
4095
4096EOF
4097        my $cmdget;
4098        if (defined $sbx) {
4099            # Install from sandbox mean using the result of the just passed sbx2build command
4100            # Get content saved in cms2build
4101            my ($pkg) = pb_conf_read("$ENV{'PBDESTDIR'}/$ENV{'PBPROJVER'}-$ENV{'PBPROJTAG'}.yml","pbpkg");
4102            confess "Unable to get package list" if (not defined $pkg);
4103   
4104            # We consider 2 specific packages
4105            my $vertag1 = $pkg->{"ProjectBuilder"};
4106            my $vertag2 = $pkg->{"project-builder"};
4107            # get the version of the current package - maybe different
4108            pb_log(2,"Vertag1: $vertag1\n");
4109            pb_log(2,"Vertag2: $vertag2\n");
4110            my ($pbver1,$tmp1) = split(/-/,$vertag1);
4111            my ($pbver2,$tmp2) = split(/-/,$vertag2);
4112            my $pbextdir = pb_get_extdir();
4113            $cmdget = "mv /tmp/ProjectBuilder-$pbver1$pbextdir.tar.gz ProjectBuilder-latest.tar.gz ; mv /tmp/project-builder-$pbver2$pbextdir.tar.gz project-builder-latest.tar.gz";
4114        } else {
4115            # TODO: shouldn't we have a variable to specify the source file?
4116            # And shouldn't we be getting the one that matches the current pb version?
4117            # For the moment, just take the latest stable one
4118            my $wget = pb_check_req("wget",0);
4119            $cmdget = "$wget --passive-ftp ftp://ftp.project-builder.org/src/ProjectBuilder-latest.tar.gz; $wget --passive-ftp ftp://ftp.project-builder.org/src/project-builder-latest.tar.gz";
4120        }
4121        print SCRIPT << 'EOF';
4122# Then install manually the missing perl modules
4123my ($osperldep,$osperlver) = pb_conf_get_if("osperldep","osperlver");
4124   
4125my $perldep = pb_distro_get_param($pbos,$osperldep);
4126my $bashopt = "";
4127$bashopt = "-x" if ($pbdebug ge 1);
4128my $verbopt = "mayfail";
4129$verbopt = "mayfailverbose" if ($pbdebug ge 1);
4130foreach my $m (split(/,/,$perldep)) {
4131    # Skip empty deps
4132    next if ($m =~ /^\s*$/);
4133    my $dir = $m;
4134    $dir =~ s/-.*//;
4135    pb_system("echo \"rm -rf $m* ; wget http://search.cpan.org/CPAN/modules/by-module/$dir/$m-$osperlver->{$m}.tar.gz ; gzip -cd $m-$osperlver->{$m}.tar.gz | tar xf - ; cd $m* ; if [ -f Build.PL ]; then perl Build.PL; ./Build ; ./Build install ; else perl Makefile.PL; make ; make install ; fi; cd .. ; rm -rf $m*\" | bash -e $bashopt" ,"Installing perl module $m-$osperlver->{$m}",$verbopt);
4136}
4137EOF
4138        print SCRIPT << 'EOF';
4139pb_system("rm -rf ProjectBuilder-* ; rm -rf project-builder-* ; rm -rf `perl -V:installvendorlib  | awk -F\"'\" '{print \$2}'`/ProjectBuilder ;
4140EOF
4141        print SCRIPT " $cmdget ; ";
4142        print SCRIPT << "EOF"
4143gzip -cd ProjectBuilder-latest.tar.gz | tar xf - ; cd ProjectBuilder-`tar tfz ProjectBuilder-latest.tar.gz | head -1 | cut -d- -f2 | cut -d/ -f1` ; perl Makefile.PL ; make ; make install ; cd .. ; rm -rf ProjectBuilder-* ; gzip -cd project-builder-latest.tar.gz | tar xf - ; cd project-builder-`tar tfz project-builder-latest.tar.gz | head -1 | cut -d- -f3 | cut -d/ -f1` ; perl Makefile.PL ; make ; make install ; cd .. ; rm -rf project-builder-* ;","Building Project-Builder");
4144EOF
4145    } elsif ($itype =~ /^pkg/) {
4146        # pkg based install. We need to point to the project-builder.org repository
4147        print SCRIPT << 'EOF';
4148my $pkgforpb = pb_distro_get_param($pbos,pb_conf_get_if("ospkg"));
4149my $forcerepo = pb_distro_setuposrepo($pbos);
4150pb_distro_installpkgs($pbos,pb_distro_only_deps_needed($pbos,join(' ',split(/,/,$pkgforpb))));
4151EOF
4152    } else {
4153        # Unknown install type
4154        confess("Unknown install type $itype->{$ENV{'PBPROJ'}} for param pbinstalltype");
4155    }
4156    print SCRIPT << 'EOF';
4157pb_system("pbdistrocheck",undef,"verbose");
4158pb_system("pb --version",undef,"verbose");
4159EOF
4160    if ($vtype eq "ve"){
4161        if ($vetype ne "docker") {
4162            print SCRIPT << 'EOF';
4163# For VE we need to umount some FS at the end
4164
4165pb_system("umount /proc","Unmounting /proc","mayfail");
4166EOF
4167        }
4168        print SCRIPT << 'EOF';
4169# Create a basic network file if not already there
4170
4171my $nf="/etc/sysconfig/network";
4172if ((! -f $nf) && ($pbos->{'type'} eq "rpm") && ($pbos->{'family'} ne "novell")) {
4173    open(NF,"> $nf") || die "Unable to create $nf";
4174    print NF "NETWORKING=yes\n";
4175    print NF "HOSTNAME=localhost\n";
4176    close(NF);
4177}
4178chmod 0755,$nf;
4179EOF
4180    }
4181
4182    # Adds pb_distro_get_context and all functions needed from ProjectBuilder::Distribution, Conf and Base
4183    foreach my $m ("ProjectBuilder/Base.pm","ProjectBuilder/Distribution.pm","ProjectBuilder/Conf.pm") {
4184        foreach my $d (@INC) {
4185            my $f = "$d/$m";
4186            if (-f "$f") {
4187                open(PBD,"$f") || confess "Unable to open $f: $!";
4188                while (<PBD>) {
4189                    next if (/^package/);
4190                    next if (/^use Exporter/);
4191                    next if (/^use ProjectBuilder::/);
4192                    next if (/^our /);
4193                    print SCRIPT $_;
4194                }
4195                close(PBD);
4196                # We just need the first one of each file wrt @INC - hopefully that's the right one.
4197                last;
4198            }
4199        }
4200    }
4201    # Use a fake pb_version_init version here
4202    print SCRIPT << "EOF";
4203sub pb_version_init {
4204
4205return("$projectbuilderver","$projectbuilderrev","$projectbuilderconfver");
4206}
42071;
4208EOF
4209    close(SCRIPT);
4210    chmod 0755,"$pbscript{$v}";
4211
4212    $pm->finish if (defined $pbparallel);
4213}
4214$pm->wait_all_children if (defined $pbparallel);
4215confess "Aborting, one or more of the children failed." if ((not $all_ok) && ($Global::pb_stop_on_error));
4216
4217# These build scripts need to be run as root and force stop of VM at end
4218$ENV{'PBACCOUNT'} = "root";
4219
4220$sbx = "" if (not defined $sbx);
4221# Launch the VM/VE/RM
4222pb_log(2,"DEBUG: before parallel launch, pbscript hash is:".Dumper(%pbscript)."\n");
4223pb_parallel_launchv(\%pbscript,$vtype,uc($vtype)."Script".$sbx,$pbstep,$pbimage,$pbforce,0,$pbsnap);
4224
4225return;
4226}
4227
4228# Function to create a snapshot named 'pb' for VMs and a compressed tar for VEs
4229sub pb_snap2v {
4230
4231my $vtype = shift;
4232my $pbstep = 2;
4233
4234my ($vm,$all) = pb_get2v($vtype);
4235
4236# Script generated
4237my $pbscript = "$ENV{'PBDESTDIR'}/snapv.$$";
4238
4239my ($pbac) = pb_conf_get($vtype."login");
4240
4241foreach my $v (@$vm) {
4242    if ($vtype eq "ve") {
4243        # Get distro context
4244        my $pbos = pb_distro_get_context($v);
4245        my ($vepath) = pb_conf_get("vepath");
4246
4247        # Test if an existing snapshot exists and remove it if there is a VE
4248        if ((-f "$vepath->{$ENV{'PBPROJ'}}/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}.tar.gz") &&
4249            (! -d "$vepath->{$ENV{'PBPROJ'}}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}")) {
4250                pb_system("sudo rm -f $vepath->{$ENV{'PBPROJ'}}/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}.tar.gz","Removing previous snapshot $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}.tar.gz");
4251        }
4252    }
4253
4254    # Prepare the script to be executed on the VM/VE
4255    open(SCRIPT,"> $pbscript") || confess "Unable to create $pbscript";
4256    print SCRIPT << 'EOF';
4257    #!/bin/bash
4258    sleep 2
4259EOF
4260    close(SCRIPT);
4261    chmod 0755,"$pbscript";
4262
4263    # Force shutdown of VM/VE
4264    # Force snapshot of VM/VE
4265    pb_script2v($pbscript,$vtype,$pbstep,1,$v,1);
4266}
4267return;
4268}
4269
4270# Function to update VMs/VEs/RMs with the latest distribution content
4271sub pb_update2v {
4272
4273my $vtype = shift;
4274my $pbstep = 2;
4275
4276my ($vm,$all) = pb_get2v($vtype);
4277
4278# Script generated
4279my $pbscript = "$ENV{'PBDESTDIR'}/updatev.$$";
4280
4281my ($pbac) = pb_conf_get($vtype."login");
4282
4283foreach my $v (@$vm) {
4284    # Get distro context
4285    my $pbos = pb_distro_get_context($v);
4286
4287    # Prepare the script to be executed on the VM/VE/RM
4288    # in $ENV{'PBDESTDIR'}/updatev
4289    open(SCRIPT,"> $pbscript") || confess "Unable to create $pbscript";
4290   
4291    print SCRIPT << 'EOF';
4292    #!/bin/bash
4293    sleep 2
4294EOF
4295    # VE needs a good /proc
4296    $vetype = pb_ve_get_type($vetype);
4297    if (($vtype eq "ve") && ($vetype ne "docker")) {
4298        print SCRIPT "sudo /bin/mount -t proc /proc /proc\n";
4299    }
4300    print SCRIPT "$pbos->{'update'}\n";
4301    if (($vtype eq "ve") && ($vetype ne "docker")) {
4302        print SCRIPT "sudo /bin/umount /proc\n";
4303    }
4304    close(SCRIPT);
4305    chmod 0755,"$pbscript";
4306
4307    # Force shutdown of VM except
4308    pb_script2v($pbscript,$vtype,$pbstep,1,$v);
4309}
4310return;
4311}
4312
4313# Build all docker CTN required to build an app
4314sub pb_all_docker {
4315
4316my $action = shift;
4317my $savproj = $ENV{'PBPROJ'};
4318my $pkg;
4319my @pkg;
4320if ($#ARGV == -1) {
4321    $pkg = pb_cms_get_pkg($defpkgdir,$extpkgdir);
4322    @pkgs = @$pkg;
4323} else {
4324    @pkgs = @ARGV;
4325}
4326#pb_log(1,"Packages to print:".Dumper(@pkgs)."\n");
4327
4328# Get the list of all VE we need to work on
4329my ($vm,$all);
4330if (not defined $ENV{'PBV'}) {
4331    ($vm,$all) = pb_get2v("ve");
4332} else {
4333    @$vm = split(/,/,$ENV{'PBV'});
4334}
4335
4336# Adapt // mode to memory size
4337$pbparallel = pb_set_parallel("ve");
4338
4339pb_log(2,"DEBUG: pb_all_docker(action: $action)\n");
4340my $docrepo = pb_ve_docker_repo();
4341
4342my ($vexist,$vpid) = (undef,undef);
4343my $pm;
4344my $all_ok = 1;
4345if (defined $pbparallel) {
4346    $pm = new Parallel::ForkManager($pbparallel);
4347
4348    $pm->run_on_finish(sub { my ($pid, $code, $id, $signal, $dump) = @_;
4349                            unless ($code == 0 && $signal == 0 && $dump == 0) {
4350                                $all_ok = 0;
4351                                pb_log(0,"ERROR: pid $pid failed\n");
4352                                pb_log(0,"code: $code - signal: $signal - dump: $dump\n");
4353                            }
4354                        });
4355}
4356
4357my $counter = 0;
4358foreach my $v (@$vm) {
4359    $counter++;
4360    # Modulo 2 * pbparallel (to avoid synchronization problems)
4361    $counter = 1 if ((defined $pbparallel) && ($counter > 2 * $pbparallel));
4362    $pm->start($counter) and next if (defined $pbparallel);
4363
4364    my $done = 0;
4365    my $lmsg = "";
4366    my $lctn = "";
4367    my $pbos = pb_distro_get_context($v);
4368    # Docker images are (for now !) regular and use distro:ver
4369    $pbimage = "$pbos->{'name'}".":$pbos->{'version'}";
4370    # The two first steps need to be done with the pb project and using a stable pb version
4371    $ENV{'PBPROJ'} = $appname;
4372    $lctn = "$docrepo$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
4373    my $found = pb_ve_docker_get_image($lctn);
4374    if (($pbforce == 1) || (not defined $found)) {
4375        # pb -p pb -m distro-ver-arch newve -i distro:ver
4376        ($pbforce == 1) ? $lmsg = "Forced" : $lmsg = "not found";
4377        pb_log(0,"Docker container $lctn $lmsg, creating it\n");
4378        pb_launchv(undef,"ve",$v,$action,0,$pbforce,undef,undef,$pbimage);
4379    }
4380    $lctn = "$docrepo$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-pb";
4381    $found = pb_ve_docker_get_image($lctn);
4382    if (($pbforce == 1) || (not defined $found)) {
4383        ($pbforce == 1) ? $lmsg = "Forced" : $lmsg = "not found";
4384        pb_log(0,"Docker container $lctn $lmsg, creating it\n");
4385        # pb -p pb -m distro-ver-arch setupve
4386        if ($savproj eq "pb") {
4387            pb_cms2build("SandBox") if ($action !~ /sbx2/);
4388            $done = 1;
4389            pb_setup2v("ve","SandBox");
4390        } else {
4391            # Here we install in the CTN pb from upstream packages to have a stable version
4392            pb_setup2v("ve",undef,"pkg");
4393        }
4394    }
4395    # Now back to our original project
4396    $ENV{'PBPROJ'} = $savproj;
4397    $lctn = "$docrepo$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}-pb-$ENV{'PBPROJ'}";
4398    $found = pb_ve_docker_get_image($lctn);
4399    if (($pbforce == 1) || (not defined $found)) {
4400        ($pbforce == 1) ? $lmsg = "Forced" : $lmsg = "not found";
4401        pb_log(0,"Docker container $lctn $lmsg, creating it\n");
4402        # pb -p pb -m distro-ver-arch prepve
4403        if ($savproj eq "pb") {
4404            pb_cms2build("SandBox") if ($action !~ /sbx2/);
4405            $done = 1;
4406        }
4407        pb_build2v("ve","prep");
4408    }
4409    pb_log(0,"Instantiating a Docker container from $lctn\n");
4410    if (($savproj ne "pb") || ($done == 0)) {
4411        pb_cms2build("SandBox") if ($action !~ /sbx2/);
4412    }
4413    pb_build2v("ve","build");
4414    $pm->finish if (defined $pbparallel);
4415}
4416$pm->wait_all_children if (defined $pbparallel);
4417confess "Aborting, one or more of the children failed." if ((not $all_ok) && ($Global::pb_stop_on_error));
4418}
4419
4420sub pb_announce {
4421
4422    my $antype = shift;
4423
4424    # Get all required parameters
4425    my ($pbpackager,$pbrepo) = pb_conf_get("pbpackager","pbrepo");
4426    my ($pkgv, $pkgt, $testver) = pb_conf_get_if("pkgver","pkgtag","testver");
4427    if (((not defined $testver) || (not defined $testver->{$ENV{'PBPROJ'}}) || ($testver->{$ENV{'PBPROJ'}} !~ /true/i)) && ($antype eq "Clean")) {
4428        # We just clean for test versions
4429        pb_log(0,"Unable to clean SSH repository for non testver version\n");
4430        return;
4431    }
4432    my $pkg = pb_cms_get_pkg($defpkgdir,$extpkgdir);
4433    my @pkgs = @$pkg;
4434    my %pkgs;
4435    my $first = 0;
4436
4437    # Get all distros concerned
4438    my $pbos = pb_distro_get_context();
4439    my $distrolist = pb_get_distros($pbos,undef);
4440    my %dl;
4441    my %theorlist;
4442    my %archlist;
4443    foreach my $d (split(/,/,$distrolist)) {
4444        my ($d1,$d2,$d3) = split(/-/,$d);
4445        $dl{$d1}++;
4446    }
4447
4448    # Command to find packages on repo
4449    my $findstr = "find ".join(" ",keys %dl)." ";
4450    my $srcstr = "";
4451    # Generated announce files
4452    my @files;
4453
4454    foreach my $pbpkg (@pkgs) {
4455        if ($first != 0) {
4456            $findstr .= "-o ";
4457        }
4458        $first++;
4459        if ((defined $pkgv) && (defined $pkgv->{$pbpkg})) {
4460            $pbver = $pkgv->{$pbpkg};
4461        } else {
4462            $pbver = $ENV{'PBPROJVER'};
4463        }
4464        if ((defined $pkgt) && (defined $pkgt->{$pbpkg})) {
4465            $pbtag = $pkgt->{$pbpkg};
4466        } else {
4467            $pbtag = $ENV{'PBPROJTAG'};
4468        }
4469
4470        # TODO: use virtual/real names here now
4471        my $pbrealpkg = $pbpkg;
4472        my $pbrealpkgrpm = pb_cms_get_real_pkg($pbpkg,"rpm");
4473        my $pbrealpkgdeb = pb_cms_get_real_pkg($pbpkg,"deb");
4474        if ($antype eq "Clean") {
4475            # We only clean test versions anyway
4476            $pbtag = "0";
4477            my $nver = $pbver;
4478            my $ntag = "[2-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]";
4479            $pbver .= $ntag;
4480            # TODO add apk
4481            $findstr .= "-name \'$pbrealpkgrpm-$pbver-$pbtag\.*.rpm\' -o -name \'$pbrealpkgrpm-debug-$pbver-$pbtag\.*.rpm\' -o -name \'$pbrealpkgdeb"."_$pbver-$pbtag"."_*\.deb\' -o -name \'$pbrealpkgdeb"."_$pbver-$pbtag.dsc\' -o -name \'$pbrealpkgdeb"."_$pbver-$pbtag.tar.gz\' -o -name \'$pbrealpkg-$nver"."_p$ntag\.ebuild\' -o -name \'$pbrealpkg-$pbver-$pbtag*\.pkg\.gz\' -o -name \'$pbrealpkg-$pbver-$pbtag*\.sd\' ";
4482            $srcstr .= "src/$pbrealpkg-$pbver.tar.gz src/$pbrealpkg-$pbver.pbconf.tar.gz ";
4483        } else {
4484            # "Announce" and "Check"
4485            my @date=pb_get_date();
4486            # the matching is only done on packages made the same day for test version. Hopefully this is enough
4487            my $nver;
4488            if ((defined $testver) && (defined $testver->{$ENV{'PBPROJ'}}) && ($testver->{$ENV{'PBPROJ'}} =~ /true/i)) {
4489                $pbtag = "0";
4490                my $ntag .= strftime("%Y%m%d*", @date);
4491                $nver = $pbver."_p$ntag*";
4492                $pbver .= "$ntag*";
4493            } else { 
4494                $nver = $pbver."-r$pbtag";
4495            }
4496            $findstr .= "-name \'$pbrealpkgrpm-$pbver-$pbtag\.*.rpm\' -o -name \'$pbrealpkgdeb"."_$pbver-$pbtag"."_*\.deb\' -o -name \'$pbrealpkg-$nver\.ebuild\' -o -name \'$pbrealpkg-$pbver\.pkg\.gz\' -o -name \'$pbrealpkg-$pbver\.sd\' ";
4497        }
4498
4499        if ($antype eq "Announce") {
4500            my $chglog;
4501
4502            pb_vcs_init($pbinit);
4503            # Get project info on log file and generate tmp files used later on
4504            $chglog = "$ENV{'PBROOTDIR'}/$pbpkg/pbcl";
4505            $chglog = "$ENV{'PBROOTDIR'}/pbcl" if (! -f $chglog);
4506            $chglog = undef if (! -f $chglog);
4507
4508            open(OUT,"> $ENV{'PBTMP'}/$pbpkg.ann") || confess "Unable to create $ENV{'PBTMP'}/$pbpkg.ann: $!";
4509            my $pb;
4510            $pb->{'realpkg'} = $pbrealpkg;
4511            $pb->{'ver'} = $pbver;
4512            $pb->{'tag'} = $pbtag;
4513            $pb->{'date'} = $pbdate;
4514            $pb->{'extdir'} = pb_get_extdir();
4515            $pb->{'chglog'} = $chglog;
4516            $pb->{'packager'} = $pbpackager;
4517            $pb->{'proj'} = $ENV{'PBPROJ'};
4518            $pb->{'repo'} = $pbrepo;
4519            $pb->{'pbos'}->{'type'} = "announce";
4520            $pb->{'pbos'}->{'suffix'} = "none";
4521            pb_changelog($pb,\*OUT,"yes");
4522            close(OUT);
4523            push(@files,"$ENV{'PBTMP'}/$pbpkg.ann");
4524        } elsif ($antype eq "Check") {
4525            # For the check we also build the theoritical complete list we should get
4526            foreach my $d (split(/,/,$distrolist)) {
4527                $pbos = pb_distro_get_context($d);
4528                if ($pbos->{'type'} eq "rpm") {
4529                    $theorlist{"$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}/$pbrealpkgrpm-$pbver-$pbtag$pbos->{'suffix'}"} = 0;
4530                } elsif ($pbos->{'type'} eq "deb") {
4531                    $theorlist{"$pbos->{'name'}/$pbos->{'version'}/$pbrealpkgdeb"."_$pbver-$pbtag"} = 0;
4532                    # TODO are we always using the last arch ?
4533                    $archlist{"$pbos->{'name'}/$pbos->{'version'}/$pbrealpkgdeb"."_$pbver-$pbtag"} = "$pbos->{'arch'}";
4534                } elsif ($pbos->{'type'} eq "ebuild") {
4535                    my $prefix = "-r";
4536                    $prefix = "_p" if ((defined $testver) && (defined $testver->{$ENV{'PBPROJ'}}) && ($testver->{$ENV{'PBPROJ'}} =~ /true/i));
4537                    $theorlist{"$pbos->{'name'}/$pbos->{'version'}/$pbrealpkg-$pbver$prefix$pbtag.ebuild"} = 0;
4538                    $archlist{"$pbos->{'name'}/$pbos->{'version'}/$pbrealpkg-$pbver$prefix$pbtag.ebuild"} = "$pbos->{'arch'}";
4539                } elsif ($pbos->{'type'} eq "apk") {
4540                    # TODO
4541                } elsif ($pbos->{'type'} eq "pkg") {
4542                    $theorlist{"$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}/$pbrealpkg-$pbver-$pbtag.pkg.gz"} = 0;
4543                } else {
4544                    pb_log(1,"No theoritical list possible for type $pbos->{'type'}\n");
4545                }
4546            }
4547        }
4548        pb_log(2,"theorlist initial: ".Dumper(%theorlist)."\n");
4549        pb_log(2,"archlist: ".Dumper(%archlist)."\n");
4550    }
4551    if ($antype eq "Announce") {
4552        $findstr .= " | grep -Ev \'src.rpm\'";
4553    } elsif ($antype eq "Clean") {
4554        $findstr .= " | xargs rm -f -v $srcstr ";
4555    }
4556
4557    # Prepare the command to run and execute it
4558    open(PBS,"> $ENV{'PBTMP'}/pbscript") || confess "Unable to create $ENV{'PBTMP'}/pbscript";
4559    print PBS "#!/bin/bash\n";
4560    print PBS "set -x\n" if ($pbdebug gt 1);
4561    print PBS "$findstr | sort 2> /dev/null\n";
4562    close(PBS);
4563    chmod 0755,"$ENV{'PBTMP'}/pbscript";
4564    pb_send2target("Announce","$ENV{'PBTMP'}/pbscript");
4565
4566    my $sl = "Project $ENV{'PBPROJ'} version $ENV{'PBPROJVER'} is now available";
4567    if ($antype eq "Announce") {
4568        # Get subject line
4569        pb_log(0,"Please enter the title of your announce\n");
4570        pb_log(0,"(By default: $sl)\n");
4571        my $sl2 = <STDIN>;
4572        $sl = $sl2 if ($sl2 !~ /^$/);
4573
4574        # Prepare a template of announce
4575        open(ANN,"> $ENV{'PBTMP'}/announce.html") || confess "Unable to create $ENV{'PBTMP'}/announce.html: $!";
4576        print ANN << "EOF";
4577$sl</p>
4578
4579<p>The project team is happy to announce the availability of a newest version of $ENV{'PBPROJ'} $ENV{'PBPROJVER'}. Enjoy it as usual!</p>
4580<p>
4581Now available at <a href="$pbrepo->{$ENV{'PBPROJ'}}">$pbrepo->{$ENV{'PBPROJ'}}</a>
4582</p>
4583<p>
4584EOF
4585    }
4586
4587    open(LOG,"$ENV{'PBTMP'}/system.$$.log") || confess "Unable to read $ENV{'PBTMP'}/system.$$.log: $!";
4588    if ($antype eq "Announce") {
4589        my $col = 2;
4590        my $i = 1;
4591        print ANN << 'EOF';
4592<TABLE WIDTH="100%" CELLPADDING="0" CELLSPACING="0" BORDER="0">
4593<TR>
4594EOF
4595        while (<LOG>) {
4596            print ANN "<TD><A HREF=\"$pbrepo->{$ENV{'PBPROJ'}}/$_\">$_</A></TD>";
4597            $i++;
4598            if ($i > $col) {
4599                print ANN "</TR>\n<TR>";
4600                $i = 1;
4601            }
4602        }
4603    } elsif ($antype eq "Clean") {
4604        while (<LOG>) {
4605            # skip errors
4606            next if ($_ !~ /^removed /);
4607            pb_log(0,"$_");
4608        }
4609    } else {
4610        # In "Check" mode we need to compare the 2 lists (real and theoritical)
4611        while (<LOG>) {
4612            # Get package name and remove what is in extra for the theoritical list (arch at the end)
4613            chomp();
4614            # skip find errors
4615            next if (/^find:/);
4616            my $p = $_;
4617            $p =~ s/\.(i[3456]86|x86_64|noarch|src)\.rpm$//;
4618            $p =~ s/_(i[3456]86|amd64|all)\.deb$//;
4619            $p =~ s/(-0\.[0-9]{8})[0-9]{6}/$1*/ if ((defined $testver) && (defined $testver->{$ENV{'PBPROJ'}}) && ($testver->{$ENV{'PBPROJ'}} =~ /true/i));
4620            $p =~ s/(-r|_p[0-9]+)\.ebuild/$1*/ if ((defined $testver) && (defined $testver->{$ENV{'PBPROJ'}}) && ($testver->{$ENV{'PBPROJ'}} =~ /true/i));
4621            # TODO apk
4622            $theorlist{$p} = -2 if (not defined $theorlist{$p});
4623            $theorlist{$p} = $theorlist{$p} + 1;
4624        }
4625        pb_log(2,"theorlist after check: ".Dumper(%theorlist)."\n");
4626    }
4627    close(LOG);
4628
4629    # Nothing more for the cleanssh case
4630    return if ($antype eq "Clean");
4631
4632    if ($antype eq "Check") {
4633        my ($chkex) = pb_conf_get_if("checkexclude");
4634        my $vmbuildlist = "";
4635        my $vebuildlist = "";
4636        my $rmbuildlist = "";
4637        my @pt = pb_conf_get_if("vmlist","velist","rmlist");
4638        foreach my $t (sort keys %theorlist) {
4639            if (defined $theorlist{$t} and $theorlist{$t} >= 1) {
4640                pb_log(1,"Packages found for $t\n");
4641            } elsif (defined $theorlist{$t} and $theorlist{$t} < 0) {
4642                pb_log(0,"Extra Package found for $t\n");
4643            } else {
4644                pb_log(2,"Analyzing $t\n");
4645                my ($os,$ver,$arch,$package) = split(/\//,$t);
4646                # Some distro have no arch subdir
4647                if (not defined $package) {
4648                    $package = $arch;
4649                    # TODO: If both arch have failed, we just make the last one
4650                    $arch = $archlist{$t} if (defined $archlist{$t});
4651                }
4652                my $pbos = pb_distro_get_context("$os-$ver-$arch");
4653                my $pkgn = $package;
4654                if ($pbos->{'type'} ne "deb") {
4655                    # package name is more easily found from the end for non deb
4656                    # as '-' is the separator, but it can also be used in names
4657                    $pkgn = reverse($package);
4658                    # search the second '-' and isolate the now last part which is the full name
4659                    $pkgn =~ s/([^-]+)-([^-]+)-([\S])+$/$3/;
4660                } else {
4661                    $pkgn =~ s/([^_]+)_([\S])+$/$2/;
4662                }
4663                my $found = 0;
4664                # Handle the exclusion of OSes
4665                my $excl = "";
4666                $excl .= $chkex->{$pkgn} if (defined $chkex->{$pkgn});
4667                $excl .= $chkex->{"all"} if (defined $chkex->{"all"});
4668
4669                pb_log(2,"Searching in excl $excl\n");
4670                foreach my $ex (split(/,/,$excl)) {
4671                    $found = 1 if ("$os-$ver-$arch" =~ /^$ex/);
4672                }
4673                # Skip as excluded
4674                next if ($found == 1);
4675                pb_log(0,"Package NOT found for $t\n");
4676                # Avoid duplicates in list
4677                next if ($vmbuildlist =~ /$os-$ver-$arch/);
4678                next if ($vebuildlist =~ /$os-$ver-$arch/);
4679                next if ($rmbuildlist =~ /$os-$ver-$arch/);
4680                # check with which method we need to build
4681                if ((defined $pt[0]->{$ENV{'PBPROJ'}}) and ($pt[0]->{$ENV{'PBPROJ'}} =~ /$os-$ver-$arch/)) {
4682                    $vmbuildlist = "$os-$ver-$arch" if ($vmbuildlist eq "");
4683                    $vmbuildlist .= ",$os-$ver-$arch" if ($vmbuildlist !~ /$os-$ver-$arch/);
4684                    next;
4685                }
4686                if ((defined $pt[1]->{$ENV{'PBPROJ'}}) and ($pt[1]->{$ENV{'PBPROJ'}} =~ /$os-$ver-$arch/)) {
4687                    $vebuildlist = "$os-$ver-$arch" if ($vebuildlist eq "");
4688                    $vebuildlist .= ",$os-$ver-$arch" if ($vebuildlist !~ /$os-$ver-$arch/);
4689                    next;
4690                }
4691                if ((defined $pt[2]->{$ENV{'PBPROJ'}}) and ($pt[2]->{$ENV{'PBPROJ'}} =~ /$os-$ver-$arch/)) {
4692                    $rmbuildlist = "$os-$ver-$arch" if ($rmbuildlist eq "");
4693                    $rmbuildlist .= ",$os-$ver-$arch" if ($rmbuildlist !~ /$os-$ver-$arch/);
4694                }
4695            }
4696        }
4697        # If we want to rebuild automatically, let's do it
4698        if (defined $opts{'rebuild'}) {
4699            # SandBox or CMS
4700            pb_log(0,"Rebuilding from SandBox\n");
4701            pb_log(0,"for VMs: $vmbuildlist\n") if ($vmbuildlist ne "");
4702            pb_log(0,"for VEs: $vebuildlist\n") if ($vebuildlist ne "");
4703            pb_log(0,"for RMs: $rmbuildlist\n") if ($rmbuildlist ne "");
4704            pb_cms2build("SandBox");
4705            # Which mode
4706            $ENV{'PBV'} = $vmbuildlist;
4707            pb_build2v("vm","build") if ($vmbuildlist ne "");
4708            $ENV{'PBV'} = $vebuildlist;
4709            pb_build2v("ve","build") if ($vebuildlist ne "");
4710            $ENV{'PBV'} = $rmbuildlist;
4711            pb_build2v("rm","build") if ($rmbuildlist ne "");
4712        }
4713        # For the check part this is now finished
4714        return;
4715    }
4716
4717    print ANN << "EOF";
4718</TR>
4719</TABLE>
4720</p>
4721
4722<p>As usual source packages are also available in the same directory.</p>
4723
4724<p>
4725Changes are :
4726</p>
4727<p>
4728EOF
4729    # Get each package changelog content
4730    foreach my $f (sort(@files)) {
4731        open(IN,"$f") || confess "Unable to read $f:$!";
4732        while (<IN>) {
4733            print ANN $_;
4734        }
4735        close(IN);
4736        print ANN "</p><p>\n";
4737    }
4738    print ANN "</p>\n";
4739    close(ANN);
4740
4741    # Allow for modification
4742    my $editor = "vi";
4743    $editor = $ENV{'EDITOR'} if (defined $ENV{'EDITOR'});
4744    pb_system("$editor $ENV{'PBTMP'}/announce.html","Allowing modification of the announce","noredir");
4745
4746    # Store it in DB for external usage (Web pages generation)
4747    my $dbh = pb_connect_db();
4748
4749    # To read whole file
4750    local $/;
4751    open(ANN,"$ENV{'PBTMP'}/announce.html") || confess "Unable to read $ENV{'PBTMP'}/announce.html: $!";
4752    my $announce = <ANN>;
4753    close(ANN);
4754   
4755    pb_log(2,"INSERT INTO announces VALUES (NULL, $pbdate, $announce)");
4756    my $sth = $dbh->prepare(qq{INSERT INTO announces VALUES (NULL,?,?)})
4757                    || confess "Unable to insert into db";
4758    $sth->execute($pbdate, $announce);
4759    $sth->finish();
4760    $dbh->disconnect;
4761
4762    # Then deliver it on the Web
4763    # $TOOLHOME/livwww www
4764
4765    # Mail it to project's ML
4766    open(ML,"| w3m -dump -T text/html > $ENV{'PBTMP'}/announce.txt") || confess "Unable to create $ENV{'PBTMP'}/announce.txt: $!";
4767    print ML << 'EOF';
4768<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/x html1/DTD/xhtml1-strict.dtd">
4769
4770<html xmlns="http://www.w3.org/1999/xhtml" dir="ltr" xml:lang="en" lang="en">
4771  <head>
4772  </head>
4773  <body>
4774  <p>
4775EOF
4776    open(ANN,"$ENV{'PBTMP'}/announce.html") || confess "Unable to read $ENV{'PBTMP'}/announce.html: $!";
4777    while(<ANN>) {
4778        print ML $_;
4779    }
4780    print ML << 'EOF';
4781</body>
4782</html>
4783EOF
4784    close(ML);
4785
4786    # To read whole file
4787    local $/;
4788    open(ANN,"$ENV{'PBTMP'}/announce.txt") || confess "Unable to read $ENV{'PBTMP'}/announce.txt: $!";
4789    my $msg = <ANN>;
4790    close(ANN);
4791   
4792    # Preparation of headers
4793    eval
4794    {
4795        require Mail::Sendmail;
4796        Mail::Sendmail->import();
4797    };
4798    if ($@) {
4799        # Mail::Sendmail not found not sending mail !
4800        pb_log(0,"No Mail::Sendmail module found so not sending any mail !\n");
4801    } else {
4802        my ($pbml,$pbsmtp) = pb_conf_get("pbml","pbsmtp");
4803        my %mail = (   
4804            To          =>  $pbml->{$ENV{'PBPROJ'}},
4805            From        =>  $pbpackager->{$ENV{'PBPROJ'}},
4806            Smtp        =>  $pbsmtp->{$ENV{'PBPROJ'}},
4807            Body        =>  $msg,
4808            Subject     =>  "[ANNOUNCE] $sl",
4809        );
4810           
4811        # Send mail
4812        if (! sendmail(%mail)) {
4813            if ((defined $Mail::Sendmail::error) and (defined $Mail::Sendmail::log)) {
4814                confess "Unable to send mail ($Mail::Sendmail::error): $Mail::Sendmail::log";
4815            }
4816        }
4817    }
4818}
4819
4820#
4821# Creates a set of HTML file containing the news for the project
4822# based on what has been generated by the pb_announce function
4823#
4824sub pb_web_news2html {
4825
4826    my $dest = shift || $ENV{'PBTMP'};
4827
4828    # Get all required parameters
4829    my ($pkgv, $pkgt) = pb_conf_get_if("pkgver","pkgtag");
4830
4831
4832    my $dbh = pb_connect_db();
4833    # For date handling
4834    $ENV{'LANGUAGE'}="C";
4835    my $firstjan = strftime("%Y-%m-%d", 0, 0, 0, 1, 0, localtime->year(), 0, 0, -1);
4836    my $oldfirst = strftime("%Y-%m-%d", 0, 0, 0, 1, 0, localtime->year()-1, 0, 0, -1);
4837    pb_log(2,"firstjan: $firstjan, oldfirst: $oldfirst, pbdate:$pbdate\n");
4838    my $all = $dbh->selectall_arrayref("SELECT id,date,announce FROM announces ORDER BY date DESC");
4839    my %news;
4840    $news{"cy"} = "";   # current year's news
4841    $news{"ly"} = "";   # last year news
4842    $news{"py"} = "";   # previous years news
4843    $news{"fp"} = "";   # first page news
4844    my $cpt = 4;        # how many news for first page
4845    # Extract info from DB
4846    foreach my $row (@$all) {
4847        my ($id, $date, $announce) = @$row;
4848        $news{"cy"} = $news{"cy"}."<p><B>$date</B> $announce\n" if ((($date cmp $pbdate) le 0) && (($firstjan cmp $date) le 0));
4849        $news{"ly"} = $news{"ly"}."<p><B>$date</B> $announce\n" if ((($date cmp $firstjan) le 0) && (($oldfirst cmp $date) le 0));
4850        $news{"py"} = $news{"py"}."<p><B>$date</B> $announce\n" if (($date cmp $oldfirst) le 0);
4851        $news{"fp"} = $news{"fp"}."<p><B>$date</B> $announce\n" if ($cpt > 0);
4852        $cpt--;
4853    }
4854    pb_log(1,"news{fp}: ".$news{"fp"}."\n");
4855    $dbh->disconnect;
4856
4857    # Generate the HTML content
4858    foreach my $pref (keys %news) {
4859        open(NEWS,"> $dest/pb_web_$pref"."news.html") || confess "Unable to create $dest/pb_web_$pref"."news.html: $!";
4860        print NEWS "$news{$pref}";
4861        close(NEWS);
4862    }
4863}
4864
4865sub pb_connect_db {
4866
4867    # Store in DB for external usage (Web pages generation)
4868    my $db = "$ENV{'PBCONFDIR'}/announces3.sql";
4869
4870    my $precmd = "";
4871    if (! -f $db) {
4872        $precmd = "CREATE TABLE announces (id INTEGER PRIMARY KEY AUTOINCREMENT, date DATE, announce VARCHAR[65535])";
4873        # we should add the $db to the VCS as well here
4874    }
4875
4876    my $dbh = DBI->connect("dbi:SQLite:dbname=$db","","",
4877                        { RaiseError => 1, AutoCommit => 1 })
4878                        || confess "Unable to connect to $db";
4879
4880    if ($precmd ne "") {
4881        my $sth = $dbh->prepare(qq{$precmd})
4882                    || confess "Unable to create table into $db";
4883        $sth->execute();
4884    }
4885
4886    return($dbh);
4887}
4888
4889
4890# Return the SSH key file to use
4891# Potentially create it if needed
4892
4893sub pb_ssh_get {
4894
4895my $create = shift || 0;    # Do not create keys by default. In that case prefer native keys, not pb's ones
4896my $target = shift || "localhost";
4897my $nport = shift || "22";
4898
4899my ($pbagent) = pb_conf_get_if("pbusesshagent");
4900# use ssh-agent if asked so.
4901return(undef) if (($create == 0) && (defined $pbagent->{$ENV{'PBPROJ'}}) && ($pbagent->{$ENV{'PBPROJ'}} =~ /true/io));
4902
4903# Check the SSH environment
4904my $keyfile = undef;
4905
4906# Check whether ecdsa is accepted by the remote host
4907my $ecdsa = 0;
4908my $dsa = 0;
4909my $rsa = 0;
4910open(SCAN,"ssh-keyscan -p $nport -t ecdsa $target 2>/dev/null |") || pb_log(0,"Unable to check ECDSA support for $target");
4911while(<SCAN>) {
4912    $ecdsa = 1 if ($_ =~ /ecdsa-/);
4913    pb_log(2,"Found ECDSA support on $target ($nport)\n");
4914}
4915close(SCAN);
4916
4917open(SCAN,"ssh-keyscan -p $nport -t rsa $target 2>/dev/null |") || pb_log(0,"Unable to check DSA support for $target");
4918while(<SCAN>) {
4919    $rsa = 1 if ($_ =~ /ssh-rsa/);
4920    pb_log(2,"Found RSA support on $target ($nport)\n");
4921}
4922close(SCAN);
4923
4924open(SCAN,"ssh-keyscan -p $nport -t dsa $target 2>/dev/null |") || pb_log(0,"Unable to check DSA support for $target");
4925while(<SCAN>) {
4926    $dsa = 1 if ($_ =~ /ssh-dss/);
4927    pb_log(2,"Found DSA support on $target ($nport)\n");
4928}
4929close(SCAN);
4930
4931# We have specific keys by default - create them if needed
4932if (!(-e "$ENV{'HOME'}/.ssh/pb_ecdsa") && ($create eq 1)) {
4933    my $res = pb_system("ssh-keygen -q -b 521 -N '' -f $ENV{'HOME'}/.ssh/pb_ecdsa -t ecdsa","Generating ECDSA SSH keys for pb");
4934    if (($res != 0) && !(-e "$ENV{'HOME'}/.ssh/pb_dsa") && ($create eq 1)) {
4935        # ecdsa may not be supported if old ssh
4936        pb_system("ssh-keygen -q -b 1024 -N '' -f $ENV{'HOME'}/.ssh/pb_dsa -t dsa","Generating DSA SSH keys for pb");
4937    }
4938}
4939
4940if (!(-e "$ENV{'HOME'}/.ssh/pb_rsa") && ($create eq 1)) {
4941    pb_system("ssh-keygen -q -b 2048 -N '' -f $ENV{'HOME'}/.ssh/pb_rsa -t rsa","Generating RSA SSH keys for pb");
4942}
4943
4944# We still favour RSA and fall back to ECDSA if not available
4945# Some ECDSA implementation are not working correctly e.g. RHEL7, Fedora 20/21
4946#$keyfile = "$ENV{'HOME'}/.ssh/pb_rsa" if (-s "$ENV{'HOME'}/.ssh/pb_rsa");
4947if ($rsa != 0) {
4948    # And we favour a specific key over a generic one except if in delivery to ftp server
4949    $keyfile = "$ENV{'HOME'}/.ssh/id_rsa" if ((-s "$ENV{'HOME'}/.ssh/id_rsa") && ($create eq 0));
4950    $keyfile = "$ENV{'HOME'}/.ssh/pb_rsa" if (-s "$ENV{'HOME'}/.ssh/pb_rsa");
4951} else {
4952    if ($ecdsa != 0) {
4953        # And we favour a specific key over a generic one except if in delivery to ftp server
4954        $keyfile = "$ENV{'HOME'}/.ssh/id_ecdsa" if ((-s "$ENV{'HOME'}/.ssh/id_ecdsa") && ($ecdsa != 0) && ($create eq 0));
4955        $keyfile = "$ENV{'HOME'}/.ssh/pb_ecdsa" if ((-s "$ENV{'HOME'}/.ssh/pb_ecdsa") && ($ecdsa != 0));
4956    } elsif ($dsa != 0) {
4957        # And we favour a specific key over a generic one except if in delivery to ftp server
4958        $keyfile = "$ENV{'HOME'}/.ssh/id_dsa" if ((-s "$ENV{'HOME'}/.ssh/id_dsa") && ($dsa != 0) && ($create eq 0));
4959        $keyfile = "$ENV{'HOME'}/.ssh/pb_dsa" if ((-s "$ENV{'HOME'}/.ssh/pb_dsa") && ($dsa != 0));
4960    }
4961}
4962if (defined $keyfile) {
4963    pb_log(2,"Using ssh key file $keyfile\n");
4964} else {
4965    pb_log(0, "Unable to find your public ssh key under $ENV{'HOME'}/.ssh\n");
4966}
4967return($keyfile);
4968}
4969
4970
4971# Returns the pid of a running VM command using a specific VM file, and the port used if the VM exists
4972sub pb_check_ps {
4973    my $vmcmd = shift;
4974    my $vmm = shift;
4975    my $vexist = 0;             # FALSE by default
4976    my $vmport = undef;         # NONE by default
4977
4978    if ($vmcmd =~ /qemu/) {
4979        # Remove the suffix as qemu changes it at launch
4980        $vmcmd =~ s/qemu-.*/qemu/;
4981    }
4982    open(PS, "ps auxhww|") || confess "Unable to call ps";
4983    while (<PS>) {
4984        next if (! /$vmcmd/);
4985        next if (! /$vmm/);
4986        my $void1;
4987        ($void1, $vexist, $vmport) = split(/ +/,$_,3);
4988        pb_log(2,"pb_check_ps $vmport\n");
4989        $vmport =~ s/.*=tcp:127\.0\.0\.1:([0-9]+)-:22.*/$1/;
4990        chomp($vmport);
4991        pb_log(2,"pb_check_ps found a VM $vexist using port $vmport\n");
4992        last;
4993    }
4994    return($vexist,$vmport);
4995}
4996
4997
4998sub pb_extract_build_files {
4999
5000my $src=shift;
5001my $dir=shift;
5002my $ddir=shift;
5003my $mandatory=shift || "build";
5004
5005my $flag = "mayfail" if (($mandatory eq "patch") || ($mandatory eq "src"));
5006my $res;
5007
5008if ($src =~ /tar\.gz$/) {
5009    $res = pb_system("tar xfpz $src $dir","Extracting $mandatory files from $src",$flag);
5010} elsif ($src =~ /tar\.bz2$/) {
5011    $res = pb_system("tar xfpj $src $dir","Extracting $mandatory files from $src",$flag);
5012} else {
5013    confess "Unknown compression algorithm for $src";
5014}
5015# If not mandatory return now
5016return() if (($res != 0) and (($mandatory eq "patch") || ($mandatory eq "src")));
5017return(pb_move_extracted_files($dir,$ddir));
5018}
5019
5020sub pb_move_extracted_files {
5021
5022my $dir = shift;
5023my $ddir = shift;
5024my @files;
5025
5026# In a recursive function , we need a local var as DIR handle
5027my $bdir;
5028
5029opendir($bdir,"$dir") || (cluck "Unable to open directory $dir: $!" && return);
5030foreach my $f (readdir($bdir)) {
5031    next if ($f =~ /^\./);
5032    # Skip potential patch dir
5033    next if ($f =~ /^pbpatch/);
5034    # Skip potential source dir
5035    next if ($f =~ /^pbsrc/);
5036    # Skip potential backup files
5037    next if ($f =~ /~$/);
5038    if ((-d "$dir/$f") && (! -l "$dir/$f")) {
5039        # Case of the bug debian subdir
5040        pb_mkdir_p("$ddir/$f");
5041        push @files,pb_move_extracted_files("$dir/$f","$ddir/$f");
5042    } else {
5043        move("$dir/$f","$ddir") || confess "Unable to move $dir/$f to $ddir";
5044        pb_log(2,"mv $dir/$f $ddir\n");
5045        push @files,"$ddir/$f";
5046    }
5047}
5048closedir($bdir);
5049# Not enough but still a first cleanup
5050pb_rm_rf("$dir");
5051return(@files);
5052}
5053
5054sub pb_list_bfiles {
5055
5056my $dir = shift;
5057my $pbpkg = shift;
5058my $bfiles = shift;
5059my $pkgfiles = shift;
5060my $supfiles = shift;
5061# subdir to keep if recursive mode, empty by default
5062my $subdir = shift || "";
5063# In a recursive function , we need a local var as DIR handle
5064my $bdir;
5065
5066pb_log(2,"DEBUG: entering pb_list_bfiles in $dir: ".Dumper($bfiles)."\n");
5067opendir($bdir,"$dir") || confess "Unable to open dir $dir: $!";
5068foreach my $f (readdir($bdir)) {
5069    pb_log(3,"DEBUG: pb_list_bfiles found $f\n");
5070    next if ($f =~ /^\./);
5071    if (-d "$dir/$f") {
5072        # Recurse for directories (Debian 3.0 format e.g.)
5073        pb_log(2,"DEBUG: pb_list_bfiles recurse in $dir/$f\n");
5074        pb_list_bfiles("$dir/$f",$pbpkg,$bfiles,$pkgfiles,$supfiles,$f);
5075        next;
5076    }
5077
5078    my $key = $f;
5079    # if recursive then store also the subdir
5080    $key = "$subdir/$f" if ($subdir ne "");
5081    $bfiles->{$key} = "$dir/$f";
5082    $bfiles->{$key} =~ s~$ENV{'PBROOTDIR'}~~;
5083    if (defined $supfiles->{$pbpkg}) {
5084        $pkgfiles->{$key} = "$dir/$f" if ($f =~ /$supfiles->{$pbpkg}/);
5085    }
5086}
5087closedir($bdir);
5088pb_log(2,"DEBUG: exiting pb_list_bfiles: ".Dumper($bfiles)."\n");
5089}
5090
5091sub pb_add_comma {
5092
5093my $str = shift;
5094my $addstr = shift;
5095
5096$str .= "," if (defined $str);
5097$str .= $addstr;
5098return($str);
5099}
5100
5101sub pb_list_sfiles {
5102
5103my $sdir = shift;
5104my $ptr = shift;
5105my $pbos = shift;
5106my $extdir = shift;
5107
5108pb_log(2,"DEBUG: entering pb_list_sfiles: ".Dumper($ptr)."\n");
5109my $key = "$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
5110
5111# Prepare local sources for this distro - They are always applied first - May be a problem one day
5112# This function works for both patches and additional sources
5113foreach my $p (sort(<$sdir/*>)) {
5114    $ptr->{$key} = pb_add_comma($ptr->{$key},"file://$p") if (($p =~ /\.all$/) || ($p =~ /\.$pbos->{'os'}$/) || ($p =~ /\.$pbos->{'type'}$/) || ($p =~ /\.$pbos->{'family'}$/) || ($p =~ /\.$pbos->{'name'}$/) || ($p =~ /\.$pbos->{'name'}-$pbos->{'version'}$/) ||($p =~ /\.$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}$/));
5115}
5116
5117# Prepare also remote sources to be included - Applied after the local ones
5118foreach my $p ("all","$pbos->{'os'}","$pbos->{'type'}","$pbos->{'family'}","$pbos->{'name'}","$pbos->{'name'}-$pbos->{'version'}","$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}") {
5119    my $f = "$extdir.".".$p";
5120    next if (not -f $f);
5121    if (not open(PATCH,$f)) {
5122        pb_display("Unable to open existing external source file content $f\n");
5123        next;
5124    }
5125    while (<PATCH>) {
5126        chomp();
5127        $ptr->{$key} = pb_add_comma($ptr->{$key},"$_");
5128    }
5129    close(PATCH);
5130}
5131pb_log(2,"DEBUG: exiting pb_list_sfiles: ".Dumper($ptr)."\n");
5132return($ptr);
5133}
5134   
5135#
5136# Return the list of packages we are working on in a non CMS action
5137#
5138sub pb_get_pkg {
5139
5140my @pkgs = ();
5141
5142# In script actions that file may not exist.
5143return(\@pkgs) if (