Context Navigation

source: ProjectBuilder/devel/pb-modules/lib/ProjectBuilder/VE.pm@ 2032

Last change on this file since 2032 was 2032, checked in by Bruno Cornec, 8 years ago
Copyright update for 2016
File size: 15.5 KB

Rev	Line
[1348]	1	#!/usr/bin/perl -w
	2	#
	3	# Common functions for virtual environment
[1528]	4	#
[2032]	5	# Copyright B. Cornec 2007-2016
[1528]	6	# Eric Anderson's changes are (c) Copyright 2012 Hewlett Packard
	7	# Provided under the GPL v2
[1348]	8	#
	9	# $Id$
	10	#
	11
	12	package ProjectBuilder::VE;
	13
	14	use strict;
	15	use Data::Dumper;
[1506]	16	use Carp 'confess';
[1348]	17	use English;
[1903]	18	use File::Basename;
[1348]	19	use ProjectBuilder::Version;
	20	use ProjectBuilder::Base;
	21	use ProjectBuilder::Conf;
[1350]	22	use ProjectBuilder::Distribution;
[1348]	23
	24	# Global vars
	25	# Inherit from the "Exporter" module which handles exporting functions.
	26
	27	use vars qw($VERSION $REVISION @ISA @EXPORT);
	28	use Exporter;
	29
	30	# Export, by default, all the functions into the namespace of
	31	# any code which uses this module.
	32
	33	our @ISA = qw(Exporter);
[1903]	34	our @EXPORT = qw(pb_ve_launch pb_ve_snap pb_ve_get_type pb_ve_docker_repo pb_ve_docker_get_image);
[1348]	35
	36	($VERSION,$REVISION) = pb_version_init();
	37
	38	=pod
	39
	40	=head1 NAME
	41
	42	ProjectBuilder::VE, part of the project-builder.org - module dealing with Virtual Environment
	43
	44	=head1 DESCRIPTION
	45
[1900]	46	This modules provides functions to deal with Virtual Environements (VE), aka chroot/containers.
[1348]	47
	48	=head1 SYNOPSIS
	49
	50	use ProjectBuilder::VE;
	51
	52	#
	53	# Return information on the running distro
	54	#
	55	my $pbos = pb_ve_launch();
	56
	57	=head1 USAGE
	58
	59	=over 4
	60
	61	=item B<pb_ve_launch>
	62
[1900]	63	This function launches a VE, creating it if necessary using multiple external potential tools.
[1348]	64
	65	=cut
	66
	67	sub pb_ve_launch {
	68
[1903]	69	my $v = shift;
[1907]	70	my $pbforce = shift; # Which step are we in (0: create, 1: setup, 2: build, 3: use)
[1903]	71	my $locsnap = shift;
	72	my $vetype = shift;
	73	my $pbimage = shift;
[1348]	74
[1900]	75	my $dockerregistry = undef;
[2003]	76	my $docrepo = ""; # By default no repository for docker available
[1900]	77
[1907]	78	pb_log(2,"Entering pb_ve_launch at step $pbforce for type $vetype\n");
[1348]	79	# Get distro context
	80	my $pbos = pb_distro_get_context($v);
	81
[1903]	82	$vetype = pb_ve_get_type($vetype);
[1900]	83	my ($vepath) = pb_conf_get("vepath");
[1506]	84
[1966]	85	if ($vetype eq "docker") {
	86	# Check acces to registry
	87	($dockerregistry) = pb_conf_get("dockerregistry");
	88	if ((defined $dockerregistry) && (defined $dockerregistry->{$ENV{'PBPROJ'}})) {
	89	pb_ve_docker_registry($dockerregistry->{$ENV{'PBPROJ'}});
[2003]	90	$docrepo = pb_ve_docker_repo($dockerregistry->{$ENV{'PBPROJ'}}).":";
[1966]	91	}
	92	}
	93
[1967]	94	# Architecture consistency
	95	my $arch = pb_get_arch();
	96	if ($arch ne $pbos->{'arch'}) {
	97	die "Unable to launch a VE of architecture $pbos->{'arch'} on a $arch platform" unless (($pbos->{'arch'} =~ /i?86/o) && ($arch eq "x86_64"));
	98	}
	99
	100	# If we are already root (from pbmkbm e.g.) don't use sudo, just call the command
	101	my $sudocmd="";
	102	if ($EFFECTIVE_USER_ID != 0) {
	103	$sudocmd ="sudo ";
	104	foreach my $proxy (qw/http_proxy ftp_proxy/) {
	105	if (defined $ENV{$proxy}) {
	106	open(CMD,"sudo sh -c 'echo \$$proxy' \|") or die "can't run sudo sh?: $!";
	107	$_ = <CMD>;
	108	chomp();
	109	die "sudo not passing through env var $proxy; '$ENV{$proxy}' != '$_'\nAdd line Defaults:`whoami` env_keep += \"$proxy\" to sudoers file?" unless $_ eq $ENV{$proxy};
	110	close(CMD);
	111	}
	112	}
	113	}
	114
	115	# Handle cross arch on Intel based platforms
	116	$sudocmd = "setarch i386 $sudocmd" if (($pbos->{'arch'} =~ /i[3456]86/) && ($arch eq 'x86_64'));
	117
	118	my $root = pb_path_expand($vepath->{$ENV{PBPROJ}});
	119
[1900]	120	if (($vetype eq "chroot") \|\| ($vetype eq "schroot") \|\| ($vetype eq "docker")) {
[1348]	121
[1352]	122	# We need to avoid umask propagation to the VE
	123	umask 0022;
[1348]	124
[1352]	125	# We can probably only get those params now we have the distro context
	126	my ($rbsb4pi,$rbspi,$vesnap,$oscodename,$osmindep,$verebuild,$rbsmirrorsrv) = pb_conf_get_if("rbsb4pi","rbspi","vesnap","oscodename","osmindep","verebuild","rbsmirrorsrv");
[1348]	127
[1907]	128	if (((((defined $verebuild) && ($verebuild->{$ENV{'PBPROJ'}} =~ /true/i)) \|\| ($pbforce == 0)) && ($vetype ne "docker"))
[1900]	129	# For docker we may have a reference image that we'll use
[1966]	130	\|\| (($vetype eq "docker") && ($pbforce == 0) && ((not defined $pbimage) \|\| ($pbimage eq "")))) {
[1900]	131
[1352]	132	my ($verpmtype,$vedebtype) = pb_conf_get("verpmtype","vedebtype");
	133	my ($rbsopt1) = pb_conf_get_if("rbsopt");
	134
[1348]	135	# We have to rebuild the chroot
	136	if ($pbos->{'type'} eq "rpm") {
	137
	138	# Which tool is used
	139	my $verpmstyle = $verpmtype->{$ENV{'PBPROJ'}};
[1517]	140	die "No verpmtype defined for $ENV{PBPROJ}" unless (defined $verpmstyle);
[1348]	141
	142	# Get potential rbs option
	143	my $rbsopt = "";
	144	if (defined $rbsopt1) {
	145	if (defined $rbsopt1->{$verpmstyle}) {
	146	$rbsopt = $rbsopt1->{$verpmstyle};
	147	} elsif (defined $rbsopt1->{$ENV{'PBPROJ'}}) {
	148	$rbsopt = $rbsopt1->{$ENV{'PBPROJ'}};
	149	} else {
	150	$rbsopt = "";
	151	}
	152	}
	153
	154	my $postinstall = pb_ve_get_postinstall($pbos,$rbspi,$verpmstyle);
	155	if ($verpmstyle eq "rinse") {
	156	# Need to reshape the mirrors generated with local before-post-install script
	157	my $b4post = "--before-post-install ";
	158	my $postparam = pb_distro_get_param($pbos,$rbsb4pi);
	159	if ($postparam eq "") {
	160	$b4post = "";
	161	} else {
	162	$b4post .= $postparam;
	163	}
	164
	165	# Need to reshape the package list for pb
	166	my $addpkgs;
	167	$postparam = "";
	168	$postparam .= pb_distro_get_param($pbos,$osmindep);
	169	if ($postparam eq "") {
	170	$addpkgs = "";
	171	} else {
	172	my $pkgfile = "$ENV{'PBTMP'}/addpkgs.lis";
	173	open(PKG,"> $pkgfile") \|\| die "Unable to create $pkgfile";
	174	foreach my $p (split(/,/,$postparam)) {
	175	print PKG "$p\n";
	176	}
	177	close(PKG);
	178	$addpkgs = "--add-pkg-list $pkgfile";
	179	}
	180
	181	my $rinseverb = "";
	182	$rinseverb = "--verbose" if ($pbdebug gt 0);
	183	my ($rbsconf) = pb_conf_get("rbsconf");
	184
	185	my $command = pb_check_req("rinse",0);
[1506]	186	pb_system("$sudocmd $command --directory \"$root/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}\" --arch \"$pbos->{'arch'}\" --distribution \"$pbos->{'name'}-$pbos->{'version'}\" --config \"$rbsconf->{$ENV{'PBPROJ'}}\" $b4post $postinstall $rbsopt $addpkgs $rinseverb","Creating the rinse VE for $pbos->{'name'}-$pbos->{'version'} ($pbos->{'arch'})", "verbose");
[1348]	187	} elsif ($verpmstyle eq "rpmbootstrap") {
	188	my $rbsverb = "";
	189	foreach my $i (1..$pbdebug) {
	190	$rbsverb .= " -v";
	191	}
	192	my $addpkgs = "";
	193	my $postparam = "";
	194	$postparam .= pb_distro_get_param($pbos,$osmindep);
	195	if ($postparam eq "") {
	196	$addpkgs = "";
	197	} else {
	198	$addpkgs = "-a $postparam";
	199	}
	200	my $command = pb_check_req("rpmbootstrap",0);
	201	pb_system("$sudocmd $command $rbsopt $postinstall $addpkgs $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'} $rbsverb","Creating the rpmbootstrap VE for $pbos->{'name'}-$pbos->{'version'} ($pbos->{'arch'})", "verbose");
[1596]	202	pb_system("$sudocmd /bin/umount $root/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}/proc","Umounting stale /proc","mayfail") if (-f "$root/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}/proc/cpuinfo");
[1348]	203	} elsif ($verpmstyle eq "mock") {
	204	my ($rbsconf) = pb_conf_get("rbsconf");
	205	my $command = pb_check_req("mock",0);
	206	pb_system("$sudocmd $command --init --resultdir=\"/tmp\" --configdir=\"$rbsconf->{$ENV{'PBPROJ'}}\" -r $v $rbsopt","Creating the mock VE for $pbos->{'name'}-$pbos->{'version'} ($pbos->{'arch'})");
	207	# Once setup we need to install some packages, the pb account, ...
	208	pb_system("$sudocmd $command --install --configdir=\"$rbsconf->{$ENV{'PBPROJ'}}\" -r $v su","Configuring the mock VE");
	209	} else {
	210	die "Unknown verpmtype type $verpmstyle. Report to dev team";
	211	}
	212	} elsif ($pbos->{'type'} eq "deb") {
	213	my $vedebstyle = $vedebtype->{$ENV{'PBPROJ'}};
	214
	215	my $codename = pb_distro_get_param($pbos,$oscodename);
	216	my $postparam = "";
	217	my $addpkgs;
	218	$postparam .= pb_distro_get_param($pbos,$osmindep);
	219	if ($postparam eq "") {
	220	$addpkgs = "";
	221	} else {
	222	$addpkgs = "--include $postparam";
	223	}
	224	my $debmir = "";
	225	$debmir .= pb_distro_get_param($pbos,$rbsmirrorsrv);
	226
	227	# Get potential rbs option
	228	my $rbsopt = "";
	229	if (defined $rbsopt1) {
	230	if (defined $rbsopt1->{$vedebstyle}) {
	231	$rbsopt = $rbsopt1->{$vedebstyle};
	232	} elsif (defined $rbsopt1->{$ENV{'PBPROJ'}}) {
	233	$rbsopt = $rbsopt1->{$ENV{'PBPROJ'}};
	234	} else {
	235	$rbsopt = "";
	236	}
	237	}
	238
	239	# debootstrap works with amd64 not x86_64
	240	my $debarch = $pbos->{'arch'};
	241	$debarch = "amd64" if ($pbos->{'arch'} eq "x86_64");
	242	if ($vedebstyle eq "debootstrap") {
	243	my $dbsverb = "";
	244	$dbsverb = "--verbose" if ($pbdebug gt 0);
	245
	246	# Some perl modules are in Universe on Ubuntu
	247	$rbsopt .= " --components=main,universe" if ($pbos->{'name'} eq "ubuntu");
	248
[1588]	249	my $cmd1 = pb_check_req("mkdir",0);
	250	my $cmd2 = pb_check_req("debootstrap",0);
	251	pb_system("$sudocmd $cmd1 -p $root/$pbos->{name}/$pbos->{version}/$pbos->{arch} ; $sudocmd $cmd2 $dbsverb $rbsopt --arch=$debarch $addpkgs $codename \"$root/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}\" $debmir","Creating the debootstrap VE for $pbos->{'name'}-$pbos->{'version'} ($pbos->{'arch'})", "verbose");
[1348]	252	# debootstrap doesn't create an /etc/hosts file
[1506]	253	if (! -f "$root/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}/etc/hosts" ) {
[1588]	254	my $cmd = pb_check_req("cp",0);
	255	pb_system("$sudocmd $cmd /etc/hosts $root/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}/etc/hosts");
[1348]	256	}
	257	} else {
	258	die "Unknown vedebtype type $vedebstyle. Report to dev team";
	259	}
	260	} elsif ($pbos->{'type'} eq "ebuild") {
	261	die "Please teach the dev team how to build gentoo chroot";
	262	} else {
	263	die "Unknown distribution type $pbos->{'type'}. Report to dev team";
	264	}
	265	}
	266
	267	# Test if an existing snapshot exists and use it if appropriate
[1900]	268	# And also use it if no local extracted VE is present
[1506]	269	if ((-f "$root/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}.tar.gz") &&
[1348]	270	(((defined $vesnap->{$v}) && ($vesnap->{$v} =~ /true/i)) \|\|
[1646]	271	((defined $vesnap->{$ENV{'PBPROJ'}}) && ($vesnap->{$ENV{'PBPROJ'}} =~ /true/i))) &&
	272	($locsnap eq 1) &&
[1903]	273	($vetype ne "docker") &&
[1646]	274	(! -d "$root/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}")) {
[1588]	275	my $cmd1 = pb_check_req("rm",0);
	276	my $cmd2 = pb_check_req("mkdir",0);
	277	my $cmd3 = pb_check_req("tar",0);
	278	pb_system("$sudocmd $cmd1 -rf $root/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'} ; $sudocmd $cmd2 -p $root/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'} ; $sudocmd $cmd3 xz -C $root/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'} -f $root/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}.tar.gz","Extracting snapshot of $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}.tar.gz under $root/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}");
[1348]	279	}
[1597]	280
[1903]	281	if ($vetype ne "docker") {
	282	# Fix modes to allow access to the VE for pb user
	283	my $command = pb_check_req("chmod",0);
	284	pb_system("$sudocmd $command 755 $root/$pbos->{'name'} $root/$pbos->{'name'}/$pbos->{'version'} $root/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}","Fixing permissions");
	285	}
	286
[1900]	287	# If docker, create the image and remove the now temp dir except if we had one already
[1907]	288	if (($vetype eq "docker") && ($pbforce == 0)) {
[1903]	289	my $cmd1 = pb_check_req("docker",0);
	290	# step 0 : nothing at creation -> tag n-v-a (made below)
	291
[1966]	292	if ((not defined $pbimage) \|\| ($pbimage eq "")) {
	293	# Snaphot the VE to serve as an input for docker
	294	pb_ve_snap($pbos,$root);
	295	# Create the docker image from the previous bootstrap
	296	# Need sudo to be able to create all files correctly
	297	# TODO: check before that the image doesn't already exist in the docker registry
	298
[2003]	299	my $pbimage = "$docrepo$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}";
[1966]	300	pb_system("$sudocmd $cmd1 import - $pbimage < $root/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}.tar.gz");
	301	pb_system("$cmd1 push $pbimage");
	302	} else {
	303	# If we pass a parameter to -i, this is the name of an existing upstream image for that distro-ver-arch
[1967]	304	# That container should then be setup correctly which may not be the case yet
	305	#
	306	# We can probably only get those params now we have the distro context
	307	my ($osmindep) = pb_conf_get_if("osmindep");
	308	my $pkgs = pb_distro_get_param($pbos,$osmindep);
	309	$pkgs =~ s/,/ /g;
	310	my $tmpd = "$ENV{'PBTMP'}/Dockerfile";
	311	open(DOCKER, "> $tmpd") \|\| die "Unable to create the docker file $tmpd";
	312	print DOCKER "FROM $pbimage\n";
	313	print DOCKER "MAINTAINER project-builder.org aka pb\n";
	314	print DIOCKER "ENV ftp_proxy $ENV{ftp_proxy}\n" if (defined $ENV{ftp_proxy});
	315	print DIOCKER "ENV http_proxy $ENV{http_proxy}\n" if (defined $ENV{http_proxy});
	316	# We are root in that container so no need to sudo, which is present potentially
	317	my $cmd2 = $pbos->{'install'};
	318	$cmd2 =~ s/sudo //g;
	319	print DOCKER "RUN $cmd2 $pkgs\n";
	320	close(DOCKER);
[2003]	321	pb_system("cd $ENV{'PBTMP'} ; $sudocmd $cmd1 build -t $docrepo$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'} .","Installing dependencies $pkgs in Docker container $docrepo$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}");
[1967]	322	unlink($tmpd);
[1966]	323	}
[1900]	324	}
	325
[1348]	326	# Nothing more to do for VE. No real launch
	327	} else {
	328	die "VE of type $vetype not supported. Report to the dev team";
	329	}
	330	}
	331
	332	#
	333	# Return the postinstall line if needed
	334	#
	335
	336	sub pb_ve_get_postinstall {
	337
	338	my $pbos = shift;
	339	my $rbspi = shift;
	340	my $vestyle = shift;
	341	my $post = "";
	342
	343	# Do we have a local post-install script
	344	if ($vestyle eq "rinse") {
	345	$post = "--post-install ";
	346	} elsif ($vestyle eq "rpmbootstrap") {
	347	$post = "-s ";
	348	}
	349
	350	my $postparam = pb_distro_get_param($pbos,$rbspi);
	351	if ($postparam eq "") {
	352	$post = "";
	353	} else {
	354	$post .= $postparam;
	355	}
	356	return($post);
	357	}
	358
[1900]	359	# Snapshot the VE
	360	sub pb_ve_snap {
[1348]	361
[1900]	362	my $pbos = shift;
	363	my $root = shift;
	364	my $tpdir = "$root/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}";
	365	pb_system("sudo tar cz -C $tpdir -f $root/$pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}.tar.gz .","Creating a snapshot of $tpdir");
	366	}
	367
	368	# Returns the docker registry to interact with
	369	sub pb_ve_docker_registry {
	370
[1907]	371	my $dockerreg = shift;
[1900]	372	my $wget = pb_check_req("wget",0);
	373	my ($scheme, $account, $host, $port, $path) = pb_get_uri($dockerreg);
	374	my $docreg = $scheme."://";
	375	$docreg .= $account."@" if ((defined $account) && ($account ne ""));
	376	$docreg .= $host;
	377	$docreg .= ":$port" if ((defined $port) && ($port ne ""));
	378	open(FD,"$wget $docreg -q -O -\|") \|\| die "Unable to talk to the docker registry $docreg";
	379	my $found = undef;
	380	while (<FD>) {
	381	$found = 1 if (/docker-registry/);
	382	}
	383	close(FD);
	384	die "No correct docker-registry answering at $docreg. Please check your configuration" if (not defined $found);
	385	#
	386	return($docreg);
	387	}
	388
	389	# Returns the docker repository to interact with
	390	sub pb_ve_docker_repo {
	391
[1907]	392	my $dockerreg = shift;
[1900]	393	my $docrepo = "";
	394	my ($scheme, $account, $host, $port, $path) = pb_get_uri($dockerreg);
	395	$docrepo .= $host;
	396	$docrepo .= ":$port" if ((defined $port) && ($port ne ""));
	397	$docrepo .= "$path";
	398	return($docrepo);
	399	}
	400
[1903]	401	sub pb_ve_docker_get_image {
	402
[1907]	403	my $pbimage = shift;
[1903]	404	my $found = 0;
	405
	406	die "Unable to handle an undef docker image" if (not defined $pbimage);
	407
	408	# Check that this docker image exists
	409	my $cmd1 = pb_check_req("docker",0);
	410	open(CMD, "$cmd1 images \|") \|\| die "Unable to get docker image list";
	411	my ($repo, $tag, $id, $dummy);
	412	while (<CMD>) {
	413	($repo, $tag, $id, $dummy) = split(/\s+/,$_,4);
	414	$found = $id if ("$repo:$tag" eq $pbimage);
	415	}
	416	close(CMD);
	417	return($found);
	418	}
[1966]	419
[1903]	420	sub pb_ve_get_type {
	421
[1907]	422	my $vetype = shift;
[1903]	423
	424	# Get VE context
	425	if (not defined $vetype) {
	426	my ($ptr) = pb_conf_get("vetype");
	427	$vetype = $ptr->{$ENV{'PBPROJ'}};
	428	}
	429	confess "No vetype defined for $ENV{PBPROJ}" unless (defined $vetype);
	430	pb_log(1, "Using vetype $vetype for $ENV{PBPROJ}\n");
	431	return($vetype);
	432	}
	433
	434
[1348]	435	=head1 WEB SITES
	436
	437	The main Web site of the project is available at L<http://www.project-builder.org/>. Bug reports should be filled using the trac instance of the project at L<http://trac.project-builder.org/>.
	438
	439	=head1 USER MAILING LIST
	440
	441	None exists for the moment.
	442
	443	=head1 AUTHORS
	444
	445	The Project-Builder.org team L<http://trac.project-builder.org/> lead by Bruno Cornec L<mailto:bruno@project-builder.org>.
	446
	447	=head1 COPYRIGHT
	448
	449	Project-Builder.org is distributed under the GPL v2.0 license
	450	described in the file C<COPYING> included with the distribution.
	451
	452	=cut
	453
	454
	455	1;

Note: See TracBrowser for help on using the repository browser.

Download in other formats: