source: ProjectBuilder/devel/pb-modules/lib/ProjectBuilder/VE.pm@ 1903

Last change on this file since 1903 was 1903, checked in by Bruno Cornec, 10 years ago
  • Add entries for osrepo for mageia/mandriva distros to allow setupve to work for these distributions without resorting to sbx2setupve
  • pbkeep now managed in pb_temp_init to avoid removing the temps dir when debugging
  • Add functions pb_ve_docker_get_image and pb_ve_get_type
  • lots of interfaces changes to allow support of docker, including script names. May break other non docker features
  • both newve and setupve work with docker now.
  • manages public keys for urpmi distriutions
  • Adds function pb_parallel_launchv. Now pb_launchv is single V
File size: 14.4 KB
Line 
1#!/usr/bin/perl -w
2#
3# Common functions for virtual environment
4#
5# Copyright B. Cornec 2007-2012
6# Eric Anderson's changes are (c) Copyright 2012 Hewlett Packard
7# Provided under the GPL v2
8#
9# $Id$
10#
11
12package ProjectBuilder::VE;
13
14use strict;
15use Data::Dumper;
16use Carp 'confess';
17use English;
18use File::Basename;
19use ProjectBuilder::Version;
20use ProjectBuilder::Base;
21use ProjectBuilder::Conf;
22use ProjectBuilder::Distribution;
23
24# Global vars
25# Inherit from the "Exporter" module which handles exporting functions.
26
27use vars qw($VERSION $REVISION @ISA @EXPORT);
28use Exporter;
29
30# Export, by default, all the functions into the namespace of
31# any code which uses this module.
32
33our @ISA = qw(Exporter);
34our @EXPORT = qw(pb_ve_launch pb_ve_snap pb_ve_get_type pb_ve_docker_repo pb_ve_docker_get_image);
35
36($VERSION,$REVISION) = pb_version_init();
37
38=pod
39
40=head1 NAME
41
42ProjectBuilder::VE, part of the project-builder.org - module dealing with Virtual Environment
43
44=head1 DESCRIPTION
45
46This modules provides functions to deal with Virtual Environements (VE), aka chroot/containers.
47
48=head1 SYNOPSIS
49
50 use ProjectBuilder::VE;
51
52 #
53 # Return information on the running distro
54 #
55 my $pbos = pb_ve_launch();
56
57=head1 USAGE
58
59=over 4
60
61=item B<pb_ve_launch>
62
63This function launches a VE, creating it if necessary using multiple external potential tools.
64
65=cut
66
67sub pb_ve_launch {
68
69my $v = shift;
70my $pbscript = shift;
71my $pbforce = shift;
72my $pbstep= shift; # Which step are we in (0: create, 1: setup, 2: build, 3: use)
73my $locsnap = shift;
74my $vetype = shift;
75my $pbimage = shift;
76
77my $dockerregistry = undef;
78my $docrepo = undef; # By default no repository for docker available
79
80pb_log(2,"Entering pb_ve_launch at step $pbstep\n");
81# Get distro context
82my $pbos = pb_distro_get_context($v);
83
84$vetype = pb_ve_get_type($vetype);
85my ($vepath) = pb_conf_get("vepath");
86
87if (($vetype eq "chroot") || ($vetype eq "schroot") || ($vetype eq "docker")) {
88
89 # We need to avoid umask propagation to the VE
90 umask 0022;
91
92 # We can probably only get those params now we have the distro context
93 my ($rbsb4pi,$rbspi,$vesnap,$oscodename,$osmindep,$verebuild,$rbsmirrorsrv) = pb_conf_get_if("rbsb4pi","rbspi","vesnap","oscodename","osmindep","verebuild","rbsmirrorsrv");
94 if ($vetype eq "docker") {
95 # Check acces to registry
96 ($dockerregistry) = pb_conf_get("dockerregistry");
97 if ((defined $dockerregistry) && (defined $dockerregistry->{$ENV{'PBPROJ'}})) {
98 pb_ve_docker_registry($dockerregistry->{$ENV{'PBPROJ'}});
99 } else {
100 die "When using docker you need to declare a dockerregistry parameter. Read the man page"
101 }
102 }
103
104 # Architecture consistency
105 my $arch = pb_get_arch();
106 if ($arch ne $pbos->{'arch'}) {
107 die "Unable to launch a VE of architecture $pbos->{'arch'} on a $arch platform" unless (($pbos->{'arch'} =~ /i?86/o) && ($arch eq "x86_64"));
108 }
109
110 # If we are already root (from pbmkbm e.g.) don't use sudo, just call the command
111 my $sudocmd="";
112 if ($EFFECTIVE_USER_ID != 0) {
113 $sudocmd ="sudo ";
114 foreach my $proxy (qw/http_proxy ftp_proxy/) {
115 if (defined $ENV{$proxy}) {
116 open(CMD,"sudo sh -c 'echo \$$proxy' |") or die "can't run sudo sh?: $!";
117 $_ = <CMD>;
118 chomp();
119 die "sudo not passing through env var $proxy; '$ENV{$proxy}' != '$_'\nAdd line Defaults:`whoami` env_keep += \"$proxy\" to sudoers file?" unless $_ eq $ENV{$proxy};
120 close(CMD);
121 }
122 }
123 }
124
125 # Handle cross arch on Intel based platforms
126 $sudocmd = "setarch i386 $sudocmd" if (($pbos->{'arch'} =~ /i[3456]86/) && ($arch eq 'x86_64'));
127
128 my $root = pb_path_expand($vepath->{$ENV{PBPROJ}});
129
130 if (((((defined $verebuild) && ($verebuild->{$ENV{'PBPROJ'}} =~ /true/i)) || ($pbforce == 1)) && ($vetype ne "docker"))
131 # For docker we may have a reference image that we'll use
132 || (($vetype eq "docker") && ($pbstep == 0))) {
133
134 my ($verpmtype,$vedebtype) = pb_conf_get("verpmtype","vedebtype");
135 my ($rbsopt1) = pb_conf_get_if("rbsopt");
136
137 # We have to rebuild the chroot
138 if ($pbos->{'type'} eq "rpm") {
139
140 # Which tool is used
141 my $verpmstyle = $verpmtype->{$ENV{'PBPROJ'}};
142 die "No verpmtype defined for $ENV{PBPROJ}" unless (defined $verpmstyle);
143
144 # Get potential rbs option
145 my $rbsopt = "";
146 if (defined $rbsopt1) {
147 if (defined $rbsopt1->{$verpmstyle}) {
148 $rbsopt = $rbsopt1->{$verpmstyle};
149 } elsif (defined $rbsopt1->{$ENV{'PBPROJ'}}) {
150 $rbsopt = $rbsopt1->{$ENV{'PBPROJ'}};
151 } else {
152 $rbsopt = "";
153 }
154 }
155
156 my $postinstall = pb_ve_get_postinstall($pbos,$rbspi,$verpmstyle);
157 if ($verpmstyle eq "rinse") {
158 # Need to reshape the mirrors generated with local before-post-install script
159 my $b4post = "--before-post-install ";
160 my $postparam = pb_distro_get_param($pbos,$rbsb4pi);
161 if ($postparam eq "") {
162 $b4post = "";
163 } else {
164 $b4post .= $postparam;
165 }
166
167 # Need to reshape the package list for pb
168 my $addpkgs;
169 $postparam = "";
170 $postparam .= pb_distro_get_param($pbos,$osmindep);
171 if ($postparam eq "") {
172 $addpkgs = "";
173 } else {
174 my $pkgfile = "$ENV{'PBTMP'}/addpkgs.lis";
175 open(PKG,"> $pkgfile") || die "Unable to create $pkgfile";
176 foreach my $p (split(/,/,$postparam)) {
177 print PKG "$p\n";
178 }
179 close(PKG);
180 $addpkgs = "--add-pkg-list $pkgfile";
181 }
182
183 my $rinseverb = "";
184 $rinseverb = "--verbose" if ($pbdebug gt 0);
185 my ($rbsconf) = pb_conf_get("rbsconf");
186
187 my $command = pb_check_req("rinse",0);
188 pb_system("$sudocmd $command --directory \"$root/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}\" --arch \"$pbos->{'arch'}\" --distribution \"$pbos->{'name'}-$pbos->{'version'}\" --config \"$rbsconf->{$ENV{'PBPROJ'}}\" $b4post $postinstall $rbsopt $addpkgs $rinseverb","Creating the rinse VE for $pbos->{'name'}-$pbos->{'version'} ($pbos->{'arch'})", "verbose");
189 } elsif ($verpmstyle eq "rpmbootstrap") {
190 my $rbsverb = "";
191 foreach my $i (1..$pbdebug) {
192 $rbsverb .= " -v";
193 }
194 my $addpkgs = "";
195 my $postparam = "";
196 $postparam .= pb_distro_get_param($pbos,$osmindep);
197 if ($postparam eq "") {
198 $addpkgs = "";
199 } else {
200 $addpkgs = "-a $postparam";
201 }
202 my $command = pb_check_req("rpmbootstrap",0);
203 pb_system("$sudocmd $command $rbsopt $postinstall $addpkgs $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'} $rbsverb","Creating the rpmbootstrap VE for $pbos->{'name'}-$pbos->{'version'} ($pbos->{'arch'})", "verbose");
204 pb_system("$sudocmd /bin/umount $root/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}/proc","Umounting stale /proc","mayfail") if (-f "$root/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}/proc/cpuinfo");
205 } elsif ($verpmstyle eq "mock") {
206 my ($rbsconf) = pb_conf_get("rbsconf");
207 my $command = pb_check_req("mock",0);
208 pb_system("$sudocmd $command --init --resultdir=\"/tmp\" --configdir=\"$rbsconf->{$ENV{'PBPROJ'}}\" -r $v $rbsopt","Creating the mock VE for $pbos->{'name'}-$pbos->{'version'} ($pbos->{'arch'})");
209 # Once setup we need to install some packages, the pb account, ...
210 pb_system("$sudocmd $command --install --configdir=\"$rbsconf->{$ENV{'PBPROJ'}}\" -r $v su","Configuring the mock VE");
211 } else {
212 die "Unknown verpmtype type $verpmstyle. Report to dev team";
213 }
214 } elsif ($pbos->{'type'} eq "deb") {
215 my $vedebstyle = $vedebtype->{$ENV{'PBPROJ'}};
216
217 my $codename = pb_distro_get_param($pbos,$oscodename);
218 my $postparam = "";
219 my $addpkgs;
220 $postparam .= pb_distro_get_param($pbos,$osmindep);
221 if ($postparam eq "") {
222 $addpkgs = "";
223 } else {
224 $addpkgs = "--include $postparam";
225 }
226 my $debmir = "";
227 $debmir .= pb_distro_get_param($pbos,$rbsmirrorsrv);
228
229 # Get potential rbs option
230 my $rbsopt = "";
231 if (defined $rbsopt1) {
232 if (defined $rbsopt1->{$vedebstyle}) {
233 $rbsopt = $rbsopt1->{$vedebstyle};
234 } elsif (defined $rbsopt1->{$ENV{'PBPROJ'}}) {
235 $rbsopt = $rbsopt1->{$ENV{'PBPROJ'}};
236 } else {
237 $rbsopt = "";
238 }
239 }
240
241 # debootstrap works with amd64 not x86_64
242 my $debarch = $pbos->{'arch'};
243 $debarch = "amd64" if ($pbos->{'arch'} eq "x86_64");
244 if ($vedebstyle eq "debootstrap") {
245 my $dbsverb = "";
246 $dbsverb = "--verbose" if ($pbdebug gt 0);
247
248 # Some perl modules are in Universe on Ubuntu
249 $rbsopt .= " --components=main,universe" if ($pbos->{'name'} eq "ubuntu");
250
251 my $cmd1 = pb_check_req("mkdir",0);
252 my $cmd2 = pb_check_req("debootstrap",0);
253 pb_system("$sudocmd $cmd1 -p $root/$pbos->{name}/$pbos->{version}/$pbos->{arch} ; $sudocmd $cmd2 $dbsverb $rbsopt --arch=$debarch $addpkgs $codename \"$root/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}\" $debmir","Creating the debootstrap VE for $pbos->{'name'}-$pbos->{'version'} ($pbos->{'arch'})", "verbose");
254 # debootstrap doesn't create an /etc/hosts file
255 if (! -f "$root/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}/etc/hosts" ) {
256 my $cmd = pb_check_req("cp",0);
257 pb_system("$sudocmd $cmd /etc/hosts $root/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}/etc/hosts");
258 }
259 } else {
260 die "Unknown vedebtype type $vedebstyle. Report to dev team";
261 }
262 } elsif ($pbos->{'type'} eq "ebuild") {
263 die "Please teach the dev team how to build gentoo chroot";
264 } else {
265 die "Unknown distribution type $pbos->{'type'}. Report to dev team";
266 }
267 }
268
269 # Test if an existing snapshot exists and use it if appropriate
270 # And also use it if no local extracted VE is present
271 if ((-f "$root/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}.tar.gz") &&
272 (((defined $vesnap->{$v}) && ($vesnap->{$v} =~ /true/i)) ||
273 ((defined $vesnap->{$ENV{'PBPROJ'}}) && ($vesnap->{$ENV{'PBPROJ'}} =~ /true/i))) &&
274 ($locsnap eq 1) &&
275 ($vetype ne "docker") &&
276 (! -d "$root/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}")) {
277 my $cmd1 = pb_check_req("rm",0);
278 my $cmd2 = pb_check_req("mkdir",0);
279 my $cmd3 = pb_check_req("tar",0);
280 pb_system("$sudocmd $cmd1 -rf $root/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'} ; $sudocmd $cmd2 -p $root/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'} ; $sudocmd $cmd3 xz -C $root/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'} -f $root/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}.tar.gz","Extracting snapshot of $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}.tar.gz under $root/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}");
281 }
282
283 if ($vetype ne "docker") {
284 # Fix modes to allow access to the VE for pb user
285 my $command = pb_check_req("chmod",0);
286 pb_system("$sudocmd $command 755 $root/$pbos->{'name'} $root/$pbos->{'name'}/$pbos->{'version'} $root/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}","Fixing permissions");
287 }
288
289 # If docker, create the image and remove the now temp dir except if we had one already
290 if (($vetype eq "docker") && ($pbstep == 0)) {
291 $docrepo = pb_ve_docker_repo($dockerregistry->{$ENV{'PBPROJ'}});
292 my $cmd1 = pb_check_req("docker",0);
293 # step 0 : nothing at creation -> tag n-v-a (made below)
294
295 # Snaphot the VE to serve as an input for docker
296 pb_ve_snap($pbos,$root);
297 # Create the docker image from the previous bootstrap
298 # Need sudo to be able to create all files correctly
299 # TODO: check before that the image doesn't already exist in the docker registry
300 my $pbimage = "$docrepo:$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
301 pb_system("$sudocmd $cmd1 import - $pbimage < $root/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}.tar.gz");
302 pb_system("$cmd1 push $pbimage");
303 }
304
305 # Nothing more to do for VE. No real launch
306} else {
307 die "VE of type $vetype not supported. Report to the dev team";
308}
309}
310
311#
312# Return the postinstall line if needed
313#
314
315sub pb_ve_get_postinstall {
316
317my $pbos = shift;
318my $rbspi = shift;
319my $vestyle = shift;
320my $post = "";
321
322# Do we have a local post-install script
323if ($vestyle eq "rinse") {
324 $post = "--post-install ";
325} elsif ($vestyle eq "rpmbootstrap") {
326 $post = "-s ";
327}
328
329my $postparam = pb_distro_get_param($pbos,$rbspi);
330if ($postparam eq "") {
331 $post = "";
332} else {
333 $post .= $postparam;
334}
335return($post);
336}
337
338# Snapshot the VE
339sub pb_ve_snap {
340
341my $pbos = shift;
342my $root = shift;
343my $tpdir = "$root/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}";
344pb_system("sudo tar cz -C $tpdir -f $root/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}.tar.gz .","Creating a snapshot of $tpdir");
345}
346
347# Returns the docker registry to interact with
348sub pb_ve_docker_registry {
349
350my $dockerreg = shift || undef;
351my $wget = pb_check_req("wget",0);
352my ($scheme, $account, $host, $port, $path) = pb_get_uri($dockerreg);
353my $docreg = $scheme."://";
354$docreg .= $account."@" if ((defined $account) && ($account ne ""));
355$docreg .= $host;
356$docreg .= ":$port" if ((defined $port) && ($port ne ""));
357open(FD,"$wget $docreg -q -O -|") || die "Unable to talk to the docker registry $docreg";
358my $found = undef;
359while (<FD>) {
360 $found = 1 if (/docker-registry/);
361}
362close(FD);
363die "No correct docker-registry answering at $docreg. Please check your configuration" if (not defined $found);
364#
365return($docreg);
366}
367
368# Returns the docker repository to interact with
369sub pb_ve_docker_repo {
370
371my $dockerreg = shift || undef;
372my $docrepo = "";
373my ($scheme, $account, $host, $port, $path) = pb_get_uri($dockerreg);
374$docrepo .= $host;
375$docrepo .= ":$port" if ((defined $port) && ($port ne ""));
376$docrepo .= "$path";
377return($docrepo);
378}
379
380sub pb_ve_docker_get_image {
381
382my $pbimage = shift || undef;
383my $found = 0;
384
385die "Unable to handle an undef docker image" if (not defined $pbimage);
386
387# Check that this docker image exists
388my $cmd1 = pb_check_req("docker",0);
389open(CMD, "$cmd1 images |") || die "Unable to get docker image list";
390my ($repo, $tag, $id, $dummy);
391while (<CMD>) {
392 ($repo, $tag, $id, $dummy) = split(/\s+/,$_,4);
393 $found = $id if ("$repo:$tag" eq $pbimage);
394}
395close(CMD);
396return($found);
397}
398sub pb_ve_get_type {
399
400my $vetype = shift || undef;
401
402# Get VE context
403if (not defined $vetype) {
404 my ($ptr) = pb_conf_get("vetype");
405 $vetype = $ptr->{$ENV{'PBPROJ'}};
406}
407confess "No vetype defined for $ENV{PBPROJ}" unless (defined $vetype);
408pb_log(1, "Using vetype $vetype for $ENV{PBPROJ}\n");
409return($vetype);
410}
411
412
413=head1 WEB SITES
414
415The main Web site of the project is available at L<http://www.project-builder.org/>. Bug reports should be filled using the trac instance of the project at L<http://trac.project-builder.org/>.
416
417=head1 USER MAILING LIST
418
419None exists for the moment.
420
421=head1 AUTHORS
422
423The Project-Builder.org team L<http://trac.project-builder.org/> lead by Bruno Cornec L<mailto:bruno@project-builder.org>.
424
425=head1 COPYRIGHT
426
427Project-Builder.org is distributed under the GPL v2.0 license
428described in the file C<COPYING> included with the distribution.
429
430=cut
431
432
4331;
Note: See TracBrowser for help on using the repository browser.