source: ProjectBuilder/devel/pbmkbm/bin/pbmkbm@ 2252

#!/usr/bin/perl -w
#
# pbmkbm, a project-builder.org utility to make boot media
#
# $Id$
#
# Copyright B. Cornec 2011-2016
# Provided under the GPL v2

# Syntax: see at end

use strict 'vars';
use Getopt::Long qw(:config auto_abbrev no_ignore_case);
use Carp qw/confess cluck/;
use Data::Dumper;
use English;
use File::Basename;
use File::Copy;
use File::Find;
use POSIX qw(strftime);

use ProjectBuilder::Version;
use ProjectBuilder::Base;
use ProjectBuilder::Env;
use ProjectBuilder::Conf;
use ProjectBuilder::Distribution;
use ProjectBuilder::VE;

# Global variables
my %opts;                   # CLI Options

=pod

=head1 NAME

pbmkbm -  a project-builder.org utility to make boot media

=head1 DESCRIPTION

pbmkbm creates a bootable media (CD/DVD, USB device, Network, tape, ...)
with a minimal distribution in it, suited for building packages for example. 
It aims at supporting all distributions supported by project-builder.org 
(RHEL, RH, Fedora, OpeSUSE, SLES, Mandriva, ...)

It is inspired by work done by Jean-Marc André around the HP SSSTK and 
aim at replacing the mindi project (http://www.mondorescue.org), but 
fully integrated with project-builder.org 

pbmkbm works in different phases. The first one is to check all

pbmkbm needs to gather a certain number of components that could come 
from various sources and could be put on a different target media.
We need a kernel, an initrd/initramfs for additional modules and init script,
a root filesystem and a boot configuration file.
Kernel, modules could come either from the local installed system 
(typically for disaster recovery context) or from a kernel package of a 
given configuration or a referenced content.
Utilities could come from busybox, local utilities or set of packages.
The root filesystem is made with them. 
The initrd/initramfs could be made internaly or by calling dracut.
The boot config file is generated from analysis content or provided externally.

=head1 SYNOPSIS

pbmkbm [-vhq][-t boot-type [-d device]][-b boot-method][-m os-ver-arch]
[-s script][-a pkg1[,pkg2,...]] [target-dir]

pbmkbm [--verbose][--help][--man][--quiet][--type boot-type [--device device]]
[--machine os-ver-arch][--boot boot-method]
[--script script][--add pkg1,[pkg2,...]] [target-dir]

=head1 OPTIONS

=over 4

=item B<-v|--verbose>

Print a brief help message and exits.

=item B<-h|--help>

Print a brief help message and exits.

=item B<--man>

Prints the manual page and exits.

=item B<-q|--quiet>

Do not print any output.

=item B<-t|--type boot-type>

Type of the boot device to generate. A boot-type can be:

=over 4 

=item B<iso>

Generate an ISO9660 image format (suitable to be burned later on or loopback mounted. Uses isolinux.

=item B<usb>

Generate a USB image format (typically a key of external hard drive). Uses syslinux.

=item B<pxe>

Generate a PXE environement (suitable to be integrated in a PXElinux configuration). Uses pxelinux.

=back

=item B<-d|--device device-file>

Name of the device or file on which you want to create the boot media.

=item B<-b|--boot boot-method>

This is the boot method to use to create the boot media. A boot-method can be:

=over 4 

=item B<native>

Use the tools of the native distribution to create the boot media. No other dependency.

=item B<ve>

Use the project-builder.org virtual environment notion to create the boot media. No other dependency outside of the project.

=item B<busybox>

Use the busybox tool to create the boot media. Cf: L<http://www.busybox.net>

=item B<dracut>

Use the dracut tool to create the boot media. Cf: L<http://www.dracut.net>

=back

=item B<-s|--script script>

Name of the script you want to execute on the related boot media at the end of the build.

=item B<-a|--add pkg1[,pkg2,...]>

Additional packages to add from the distribution you want to install on the related boot media 
at the end of the build.

=item B<-m|--machine os-ver-arch>

This is the target tuple operating system-version-architecture for which you want to create the boot media.

=back 

=head1 ARGUMENTS

target-dir is the directory under which the boot media will be build.

=over 4 

=back 

=head1 EXAMPLE

To setup a USB busybox based boot media on the /dev/sdb device for a Fedora 12 distribution with an i386 architecture issue:

pbmkbm -t usb -d /dev/sdb -m fedora-12-i386 -b busybox

To setup an ISO image under /tmp for a RHEL 6 x86_64 distribution issue using the native environment:

pbmkbm -t iso -d /tmp -m rhel-6-x86_64 -b ve

=head1 WEB SITES

The main Web site of the project is available at L<http://www.project-builder.org/>. 
Bug reports should be filled using the trac instance of the project at L<http://trac.project-builder.org/>.

=head1 USER MAILING LIST

Cf: L<http://www.mondorescue.org/sympa/info/pb-announce> for announces and 
L<http://www.mondorescue.org/sympa/info/pb-devel> for the development of the pb project.

=head1 CONFIGURATION FILE

Uses Project-Builder.org configuration file (/etc/pb/pb.conf or /usr/local/etc/pb/pb.conf)

=head1 AUTHORS

The Project-Builder.org team L<http://trac.project-builder.org/> lead by Bruno Cornec L<mailto:bruno@project-builder.org>.

=head1 COPYRIGHT

Project-Builder.org is distributed under the GPL v2.0 license
described in the file C<COPYING> included with the distribution.

=cut

# ---------------------------------------------------------------------------

my ($projectbuilderver,$projectbuilderrev) = pb_version_init();
my $appname = "pbmkbm";
$ENV{'PBPROJ'} = $appname;

$Global::pb_stop_on_error = 0;  # False by default

# Initialize the syntax string

pb_syntax_init("$appname Version $projectbuilderver-$projectbuilderrev\n");
pb_temp_init();

GetOptions("help|?|h" => \$opts{'h'}, 
    "man" => \$opts{'man'},
    "verbose|v+" => \$opts{'v'},
    "quiet|q" => \$opts{'q'},
    "log-files|l=s" => \$opts{'l'},
    "script|s=s" => \$opts{'s'},
    "machine|m=s" => \$opts{'m'},
    "add|a=s" => \$opts{'a'},
    "device|d=s" => \$opts{'d'},
    "type|t=s" => \$opts{'t'},
    "boot|b=s" => \$opts{'b'},
    "version|V=s" => \$opts{'V'},
    "stop-on-error!" => \$Global::pb_stop_on_error,
) || pb_syntax(-1,0);

if (defined $opts{'h'}) {
    pb_syntax(0,1);
}
if (defined $opts{'man'}) {
    pb_syntax(0,2);
}
if (defined $opts{'v'}) {
    $pbdebug = $opts{'v'};
}
if (defined $opts{'q'}) {
    $pbdebug=-1;
}
if (defined $opts{'l'}) {
    open(pbLOG,"> $opts{'l'}") || confess "ERROR: Unable to log to $opts{'l'}: $!";
    $pbLOG = \*pbLOG;
    $pbdebug = 0  if ($pbdebug == -1);
}
pb_log_init($pbdebug, $pbLOG);
pb_log(1,"$appname Version $projectbuilderver-$projectbuilderrev\n");
my @date = pb_get_date();
my $pbdate = strftime("%Y-%m-%d %H:%M:%S", @date);

pb_log(1,"Start: $pbdate\n");

# Get VE name
$ENV{'PBV'} = $opts{'m'};

#
# Initialize distribution info from pb conf file
#
my $pbos = pb_distro_get_context($ENV{'PBV'});
pb_log(0,"Starting boot media build for $pbos->{'name'}-$pbos->{'version'}-$pbos->{'arch'}\n");

pb_env_init_pbrc(); # to get content of HOME/.pbrc.yml

# Global hash containing all the configuration information
my %mkbm;

#
# Check target dir
# Create if not existent and use default if none given
#
$mkbm{'targetdir'} = shift @ARGV;

#
# Check for command requirements
#
my ($req,$opt,$pbstoponerr) = pb_conf_get_if("oscmd","oscmdopt","pbstoponerr");
$Global::pb_stop_on_error = 1 if ((defined $pbstoponerr) && (defined $pbstoponerr->{$ENV{'PBPROJ'}}) && ($pbstoponerr->{$ENV{'PBPROJ'}} =~ /true/oi));
#pb_check_requirements($req,$opt,$appname);

# After that we will need root access
confess "ERROR: $appname needs to be run as root" if ($EFFECTIVE_USER_ID != 0);

#
# Where is our build target directory
#
if (not defined $mkbm{'targetdir'}) {
    $mkbm{'targetdir'} = "/var/cache/pbmkbm";
    my ($vestdpath) = pb_conf_get_if("cachedir");
    $mkbm{'targetdir'} = "$vestdpath->{'default'}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}" if ((defined $vestdpath) && (defined $vestdpath->{'default'}));
    $mkbm{'targetdir'} = "$vestdpath->{'pbmkbm'}/$pbos->{'name'}/$pbos->{'version'}/$pbos->{'arch'}" if ((defined $vestdpath) && (defined $vestdpath->{'pbmkbm'}));
    pb_log(1,"No target-dir specified, using $mkbm{'targetdir'}\n");
}

# Point to the right subdir and create it if needed
pb_mkdir_p($mkbm{'targetdir'}) if (! -d $mkbm{'targetdir'});

# Log information on our system
# TODO: this should be put in a subfunction
my ($logcmd) = pb_conf_get("logcmd");
my ($logopt) = pb_conf_get_if("logopt");
# check that the command is there first and then use it.
if ($logcmd->{$appname} ne "internal") {
    $logcmd = pb_check_req($logcmd->{$appname},1);
    if (not defined $logcmd) {
        pb_log(1,"INFO: command $logcmd->{$appname} doesn't exist. No log report is available\n");
    } else {
        my $c = $logcmd->{$appname};
        $c .= " $logopt->{$appname}" if (defined $logopt->{$appname});
        pb_system("$c","Creating a log report of your system");
    }
} else {
    # We provide our own internal log reporter as a std one isn't found
    my ($lcmds,$lfiles) = pb_conf_get_if("logcommands","logfiles");
    pb_log(1,"------------------\n");
    if (defined $lcmds->{$appname}) {
        foreach my $c (split(/,/,$lcmds->{$appname})) {
            my $lcmd = $c;
            $lcmd =~ s/ .*$//;
            $lcmd = pb_check_req($lcmd,1);
            if (not defined $lcmd) {
                pb_log(1,"INFO: command $c doesn't exist\n");
                pb_log(1,"------------------\n");
                next;
            }
            pb_log(1,"Execution of $c\n");
            pb_log(1,"------------------\n");
            pb_system($c,"",1);
            pb_log(1,"------------------\n");
        }
    }
    if (defined $lfiles->{$appname}) {
        foreach my $f (split(/,/,$lfiles->{$appname})) {
            if (! -e $f) {
                pb_log(1,"INFO: $f doesn't exist\n");
                pb_log(1,"------------------\n");
                next;
            }
            if (! -r $f) {
                pb_log(1,"INFO: $f isn't readable\n");
                pb_log(1,"------------------\n");
                next;
            }
            if ((-f $f) || (-l $f)) {
                if (! open(FILE,$f)) {
                    pb_log(1,"INFO: Unable to open $f\n");
                    pb_log(1,"------------------\n");
                    next;
                }
                pb_log(1,"Content of $f\n");
                pb_log(1,"------------------\n");
                while (<FILE>) {
                    pb_log(1,$_);
                }
                close(FILE);
                pb_log(1,"------------------\n");
            } elsif (-d $f) {
                my $dh;
                if (! opendir($dh,$f)) {
                    pb_log(1,"INFO: Unable to opendir $f\n");
                    pb_log(1,"------------------\n");
                    next;
                }
                pb_log(1,"Content of directory $f\n");
                pb_log(1,"----------------------------\n");
                while (readdir $dh) {
                    my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime,$mtime,$ctime,$blksize,$blocks) = stat("$f/$_");
                    my $str = sprintf("%10s %2s %5s %5s %10s %14s %s",$mode,$ino,$uid,$gid,$size,$mtime,$_);
                    pb_log(1,"$str\n");
                }
                closedir($dh);
                pb_log(1,"------------------\n");
            } else {
                pb_log(1,"INFO: $f is special\n");
                pb_log(1,"------------------\n");
            }
        }
    }
}

# Now the preparation is over, we need to do something useful :-)
# But it all depends on how we're asked to do it.
#
# First we need to copy into the target dir all the relevant content
pb_mkbm_create_content();

# Then we need to package this content in the destination format
pb_mkbm_create_media();

@date = pb_get_date();
$pbdate = strftime("%Y-%m-%d %H:%M:%S", @date);

pb_log(1,"End: $pbdate\n");

pb_exit();

sub pb_mkbm_create_content {

pb_log(1,"Creating boot media content\n");

# If not defined use VE mode by default
$opts{'b'} = "ve" if (not defined $opts{'b'});

# Hash of target content
# atribute could be, copy, remove, link, dir
my %targettree;

if ($opts{'b'} eq "ve") {
    # Use project-builder VE mecanism to create a good VE !
    # TODO: This call is now assuming default vetype and image
    pb_ve_launch($ENV{'PBV'},1);
} elsif ($opts{'b'} eq "native") {
    # Use native tools to create a good VE !
} elsif ($opts{'b'} eq "drakut") {
    # Use drakut to create a good VE !
} elsif ($opts{'b'} eq "busybox") {
    # Use busybox to create a good VE !
    pb_mkbm_create_busybox_ve(\%targettree);
} else {
    confess "ERROR: Unknown method $opts{'b'} used to create the media content";
}

# Create the directory structure needed on the target dir
my ($tdirs,$bdirs,$bfiles,$bcmds) = pb_distro_get_param($pbos,pb_conf_get("mkbmtargetdirs","mkbmbootdirs","mkbmbootfiles","mkbmbootcmds"));
# Create empty dirs for these
foreach my $d (split(/,/,$tdirs)) {
    $targettree{$d} = "emptydir";
}
# And copy dirs for those
foreach my $d (split(/,/,$bfiles),split(/,/,$bdirs)) {
    pb_mkbm_store_in_tree($d,\%targettree);
}
pb_log(2,"INFO: Target Tree is now: ".Dumper(%targettree)."\n");
# Once the environment is made, add what is needed for this boot media to it.
# Keyboard
pb_mkbm_find_keyboard(\%targettree);
# Terminfo
pb_mkbm_find_terminfo(\%targettree);
# List of commands
# List of dependencies
# Kernel - We use 2 objects, the running kernel and the target kernel which could be different
my %rkernel;
my %tkernel;
pb_mkbm_find_kernel(\%rkernel);
pb_mkbm_find_modules($rkernel{"release"},\%targettree,\%rkernel);
# Initrd
# init 
# BootLoader and its configuration
# Additional data files coming from a potential caller (MondoRescue/Mindi e.g. with fstab, LVM, mountlist, ...)
pb_log(1,"INFO: Target Tree is now: ".Dumper(%targettree)."\n");
pb_log(1,"End of boot media creation\n");
}

sub pb_mkbm_create_busybox_ve {

my $tgtree = shift;

pb_log(1,"Analyzing your busybox's configuration\n");
# First, check which are the supported command in that version of busybox
# and create the links for it in the target VE

my $busycmd = pb_distro_get_param($pbos,pb_conf_get("ospathcmd-busybox"));
open(BUSY,"$busycmd |") || confess "ERROR: Unable to execute $busycmd";
my $cmdlist = 0;
while (<BUSY>) {
    pb_log(3,"busybox line : $_");
    chomp($_);
    # After these words, we have the list of functions, so trigger their analysis
    if ($_ =~ /defined functions:/) {
        $cmdlist = 1;
        next;
    }
    # Analyse the list of commands provided by that busybox (, separated)
    if ($cmdlist == 1) {
        pb_log(3,"Analyzing that busybox line\n");
        foreach my $c (split(/,/,$_)) {
            $c =~ s/\s*//g;
            # skip empty strings
            next if ($c =~ /^$/);
            my $c1 = pb_check_req($c,0);
            if (defined $c1) {
                $tgtree->{$c1} = "link:$busycmd";
            } else {
                # When not found on the system, create the link under /usr/bin by default
                $tgtree->{"/usr/bin/$c"} = "link:$busycmd";
            }
        }
    }
}
pb_log(1,"Target Tree is now: ".Dumper($tgtree)."\n");
close(BUSY);
pb_log(1,"End of busybox analysis\n");
}

sub pb_mkbm_find_keyboard {

my $tgtree = shift;

pb_log(1,"Analyzing your keyboard's configuration\n");
my $keyfile = pb_distro_get_param($pbos,pb_conf_get("ospathcmd-keyfile"));
confess "ERROR: Unable to read the keyfile $keyfile" if ((not defined $keyfile) || (! -r $keyfile));
my $keymapdir = pb_distro_get_param($pbos,pb_conf_get("ospathcmd-keymapdir"));
confess "ERROR: Unable to read the keymapdir $keymapdir" if ((not defined $keymapdir) || (! -d $keymapdir));
my $keymapre = pb_distro_get_param($pbos,pb_conf_get("ospathcmd-keymapre"));
confess "ERROR: Unable to read the keymapre $keymapre" if (not defined $keymapre);

# if a direct keymap file is given as keyfile, use only the first existing one and return
my $foundkmap = 0;
foreach my $f (split(/,/,$keyfile)) {
    next if ($f !~ /\.gz$/);
    $foundkmap = 1;
    if ((-l $f) || (-r $f)) {
        pb_mkbm_store_in_tree($f,$tgtree);
        pb_log(1,"Using Keymap file $f\n");
        last;
    } else {
        next;
    }
}
return() if ($foundkmap eq 1);

pb_log(1,"Using Keyfile $keyfile and Keymap directory $keymapdir\n");
my $locale="";
open(KEYMAP,"$keyfile") || confess "ERROR: Unable to read $keyfile";
# Depending on the format of the keymap we look for various strings
while (<KEYMAP>) {
    $locale =~ $keymapre;
}
close(KEYMAP);
pb_log(1,"Found locale $locale\n");

pb_log(1,"End of keyboard analysis\n");
}

sub pb_mkbm_find_terminfo {

my $tgtree = shift;

pb_log(1,"Analyzing your terminfo's configuration\n");
my $termdir = pb_distro_get_param($pbos,pb_conf_get("ospathcmd-termdir"));
confess "ERROR: Unable to read the keymapdir $termdir" if ((not defined $termdir) || (! -d $termdir));
$tgtree->{$termdir} = "recurdir";

pb_log(1,"End of terminfo analysis\n");
}

sub pb_mkbm_find_kernel {

my $kernel = shift;

pb_log(1,"Analyzing your kernel's configuration\n");
$kernel->{"is_xen"} = undef;
# See if we're booted from a Xen kernel
# From http://wiki.xensource.com/xenwiki/XenCommonProblems#head-26434581604cc8357d9762aaaf040e8d87b37752
if ( -f "/proc/xen/capabilities") {
    # It's a Xen kernel
    pb_log(2,"INFO: We found a Xen Kernel running\n");
    $kernel->{"is_xen"} = 1;
}
$kernel->{"release"} = pb_get_osrelease();

my $kfile = pb_distro_get_param($pbos,pb_conf_get_if("mkbmkernelfile"));
if ((defined $kfile) && ($kfile ne "")) {
    pb_log(1,"INFO: You specified your kernel as $kfile, so using it\n");
    $kernel->{"file"} = $kfile;
} else {
    $kernel->{"dir"} = pb_distro_get_param($pbos,pb_conf_get("mkbmkerneldir"));
    confess "ERROR: The mkbmkerneldir content ($kernel->{'dir'}) doesn't refer to a directory\n" if (! -d $kernel->{"dir"});
    pb_log(1,"INFO: Analyzing directory $kernel->{'dir'} to find your kernel\n");
    $kernel->{"namere"} = pb_distro_get_param($pbos,pb_conf_get("mkbmkernelnamere"));

    # TODO: Look at a better way to find the name of the kernel we run
    # look at /proc/sys/kernel/bootloader_type /proc/sys/kernel/bootloader_version
    # to have a better guess
    my $dh;
    confess "ERROR: Unable to open the mkbmkerneldir content ($kernel->{'dir'})\n" if (! opendir($dh,$kernel->{"dir"}));
    while (readdir $dh) {
        pb_log(3,"Potential kernel file: $_\n");
        # Skip non-files
        next if (! -f "$kernel->{'dir'}/$_");
        # Skip files not correpsonding to the RE planned
        next if ($_ !~ /$kernel->{"namere"}/);
        # We now have a candidate. Analyze further
        pb_log(3,"Potential kernel file 2: $_\n");
        eval
        {
            require File::MimeInfo;
            File::MimeInfo->import();
        };
        if ($@) {
            # File::MimeInfo not found
            confess "ERROR: Install File::MimeInfo to handle kernel file detection\n";
        }
        my $mm = mimetype("$kernel->{'dir'}/$_");
        # Skip symlinks
        next if ($mm =~ /inode\/symlink/);
        pb_log(2,"file $_ mimetype: $mm\n");
        if ($mm =~ /\/x-gzip/) {
            # on ia64 kernel are gzip compressed
        }
        next if (pb_get_content("$kernel->{'dir'}/$_") !~ /$kernel->{"release"}/);
        pb_log(3,"Potential kernel file 3: $_\n");
        $kernel->{"file"} = "$kernel->{'dir'}/$_";
        #my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime,$mtime,$ctime,$blksize,$blocks) = stat("$f/$_");
    }
    closedir($dh);
}
pb_log(1,"INFO: kernel is ".Dumper($kernel)."\n");
pb_log(1,"End of kernel analysis\n");
}

sub pb_mkbm_find_modules {

my $krel = shift;
my $tgtree = shift;
my $kernel = shift;

my $lsmod = pb_distro_get_param($pbos,pb_conf_get("ospathcmd-lsmod"));
my $depmod = pb_distro_get_param($pbos,pb_conf_get("ospathcmd-depmod"));
open(LSMOD,"$lsmod") || confess "ERROR: Unable to read $lsmod: $!";
while (<LSMOD>) {
    my ($mod,$void) = split(/\s+/);
    open(DEPMOD,"$depmod -S $kernel->{'release'} $mod |") || confess "ERROR: Unable to read $depmod -S $kernel->{'release'} $mod: $!";
    while (<DEPMOD>) {
        chomp();
        s/^insmod //;
        s/\s+$//;
        pb_mkbm_store_in_tree($_,$tgtree);
    }
    close(DEPMOD);
}
close(LSMOD);
}

sub pb_mkbm_create_media {

}


sub pb_mkbm_store_in_tree {

my $f = shift;
my $tgtree = shift;

if (-d $f) {
    $tgtree->{$f} = "recurdir";
} elsif (-l $f) {
    $tgtree->{$f} = "link:$f";
    pb_log(3,"INFO: Storing link $f in tree\n");
} elsif (-r $f) {
    $tgtree->{$f} = "file";
    pb_log(3,"INFO: Storing file $f in tree\n");
} else {
    pb_log(3,"INFO: file $f doesn't exist\n");
}
}

# Get the package list to download, store them in a cache directory
#
#my ($mkbmcachedir) = pb_conf_get_if("mkbmcachedir");
#my ($pkgs) = pb_distro_get_param($pbos,pb_conf_get("mkbmmindep"));

#
# /proc needed
#
#pb_system("mount -o bind /proc $targetdir/proc","Mounting /proc");

# Installed additional packages we were asked to
#if (defined $opts{'a'}) {
#$opts{'a'} =~ s/,/ /g;
#pb_system("chroot $targetdir /bin/bash -c \"$pbos->{'install'} $opts{'a'} \"","Adding packages to OS by running $pbos->{'install'} $opts{'a'}");
#}

#
# Clean up
#
#pb_log(1,"Cleaning up\n");
#pb_system("umount $targetdir/proc","Unmounting /proc");

# Executes post-install step if asked for
#if ($opts{'s'}) {
#pb_system("$opts{'s'} $targetdir","Executing the post-install script: $opts{'s'} $targetdir");
#}