#!/bin/bash # $Id$ set -e no_empty_builddir() { 'clear' cat <<- -EOF- ${DD_BORDER} ${tab_}${tab_}${BOLD}${RED}W A R N I N G${OFF} Looks like the \$BUILDDIR directory contains subdirectories from a previous HLFS build. Please format the partition mounted on \$BUILDDIR or set a different build directory before running jhalfs. ${OFF} ${DD_BORDER} -EOF- exit } HEADER="# This file is automatically generated by jhalfs # DO NOT EDIT THIS FILE MANUALLY # # Generated on `date \"+%F %X %Z\"`" #----------------------------------# wrt_target() { # Create target and initialize log file #----------------------------------# local i=$1 local PREV=$2 case $i in iteration* ) local LOGFILE=$this_script.log ;; * ) local LOGFILE=$this_script ;; esac ( cat << EOF $i: $PREV @\$(call echo_message, Building) @./progress_bar.sh \$@ & @echo -e "\n\`date\`\n\nKB: \`du -skx --exclude=${SCRIPT_ROOT} \$(MOUNT_PT)\`\n" >logs/$LOGFILE EOF ) >> $MKFILE.tmp } #----------------------------------# wrt_target_boot() { # Create target and initialize log file #----------------------------------# local i=$1 local PREV=$2 case $i in iteration* ) local LOGFILE=$this_script.log ;; * ) local LOGFILE=$this_script ;; esac ( cat << EOF $i: $PREV @\$(call echo_message, Building) @./progress_bar.sh \$@ & @echo -e "\n\`date\`\n\nKB: \`du -skx --exclude=${SCRIPT_ROOT}\`\n" >logs/$LOGFILE EOF ) >> $MKFILE.tmp } #----------------------------# get_package_tarball_name() { # #----------------------------# local script_name=$1 # The use of 'head' is necessary to limit the return value to the FIRST match.. # hopefully this will not cause problems. # case $script_name in tcl) echo $(grep "^tcl" $JHALFSDIR/pkg_tarball_list | head -n1 ) ;; linux-headers) echo $(grep "^linux-headers.*.bz2" $JHALFSDIR/pkg_tarball_list | head -n1 ) ;; *) echo $(grep "^$script_name-[[:digit:]]" $JHALFSDIR/pkg_tarball_list | head -n1 ) ;; esac } #----------------------------------# wrt_test_log() { # Initialize testsuite log file #----------------------------------# local TESTLOGFILE=$1 ( cat << EOF @echo "export TEST_LOG=$TESTLOGDIR/$TESTLOGFILE" >> envars && \\ su - \$(LUSER) -c "echo -e '\n\`date\`\n' >$TESTLOGDIR/$TESTLOGFILE" EOF ) >> $MKFILE.tmp } #----------------------------------# wrt_test_log2() { # #----------------------------------# local TESTLOGFILE=$1 ( cat << EOF @echo "export TEST_LOG=/$SCRIPT_ROOT/test-logs/$TESTLOGFILE" >> envars && \\ echo -e "\n\`date\`\n" >test-logs/$TESTLOGFILE EOF ) >> $MKFILE.tmp } #----------------------------------# wrt_target_vars() { # Target vars for hlfs (cross-build method) #----------------------------------# ( cat << EOF @echo "export target=$(uname -m)-${TARGET}" >> envars && \\ echo "export ldso=/lib/${LOADER}" >> envars EOF ) >> $MKFILE.tmp } #----------------------------------# wrt_copy_fstab() { # #----------------------------------# local i=$1 ( cat << EOF @cp -v $FSTAB \$(MOUNT_PT)/etc/fstab >>logs/$i 2>&1 EOF ) >> $MKFILE.tmp } #----------------------------------# wrt_copy_fstab2() { # #----------------------------------# local i=$1 ( cat << EOF @cp -v /sources/fstab /etc/fstab >>logs/$i 2>&1 EOF ) >> $MKFILE.tmp } #----------------------------------# wrt_report() { # #----------------------------------# ( cat << EOF create-sbu_du-report: $PREV @\$(call echo_message, Building) @./create-sbu_du-report.sh logs $VERSION @\$(call echo_report,$VERSION-SBU_DU-$(date --iso-8601).report) @touch \$@ EOF ) >> $MKFILE.tmp chapter789="$chapter789 create-sbu_du-report" } unset wrt_unpack #----------------------------------# wrt_unpack() { # Unpack and set 'ROOT' var #----------------------------------# local FILE=$1 local optSAVE_PREVIOUS=$2 if [[ "${optSAVE_PREVIOUS}" != "1" ]]; then ( cat << EOF @\$(call remove_existing_dirs,$FILE) EOF ) >> $MKFILE.tmp fi ( cat << EOF @\$(call unpack,$FILE) @\$(call get_pkg_root) EOF ) >> $MKFILE.tmp } unset wrt_unpack2 #----------------------------------# wrt_unpack2() { # #----------------------------------# local FILE=$1 local optSAVE_PREVIOUS=$2 if [ "${optSAVE_PREVIOUS}" != "1" ]; then ( cat << EOF @\$(call remove_existing_dirs,$FILE) EOF ) >> $MKFILE.tmp fi ( cat << EOF @\$(call unpack2,$FILE) @\$(call get_pkg_root,nouser) EOF ) >> $MKFILE.tmp } #----------------------------------# wrt_unpack3() { # #----------------------------------# local FILE=$1 local optSAVE_PREVIOUS=$2 if [ "${optSAVE_PREVIOUS}" != "1" ]; then ( cat << EOF @\$(call remove_existing_dirs2,$FILE) EOF ) >> $MKFILE.tmp fi ( cat << EOF @\$(call unpack3,$FILE) @\$(call get_pkg_root2) EOF ) >> $MKFILE.tmp } unset wrt_remove_build_dirs #----------------------------------# wrt_remove_build_dirs() { # #----------------------------------# local name=$1 ( cat << EOF @\$(call remove_build_dirs,$name) EOF ) >> $MKFILE.tmp } #----------------------------------# wrt_remove_build_dirs2() { # #----------------------------------# local name=$1 ( cat << EOF @\$(call remove_build_dirs2,$name) EOF ) >> $MKFILE.tmp } unset wrt_touch #----------------------------------# wrt_touch() { # #----------------------------------# ( cat << EOF @\$(call housekeeping) EOF ) >> $MKFILE.tmp } unset wrt_RunAsUser #----------------------------------# wrt_RunAsUser() { # Execute script inside time { }, footer to log file #----------------------------------# local this_script=$1 local file=$2 ( cat << EOF @( time { \$(SU_LUSER) "source \$(LUSER_HOME)/.bashrc && \$(CMDSDIR)/`dirname $file`/\$@" >> logs/\$@ 2>&1; } ) 2>> logs/\$@ && \\ \$(PRT_DU) >> logs/\$@ EOF ) >> $MKFILE.tmp } #----------------------------------# wrt_RunAsRoot() { # Some scripts must be run as root.. #----------------------------------# local ENV_MOUNT local this_script=$1 local file=$2 case ${PROGNAME} in lfs ) MOUNT_ENV="LFS" ;; blfs ) MOUNT_ENV="BLFS" ;; clfs ) MOUNT_ENV="CLFS" ;; clfs2 ) MOUNT_ENV="CLFS" ;; hlfs ) MOUNT_ENV="HLFS" ;; *) echo "undefined progname $PROGNAME"; exit 1 esac ( cat << EOF @( time { export ${MOUNT_ENV}=\$(MOUNT_PT) && ${PROGNAME}-commands/`dirname $file`/\$@ >>logs/\$@ 2>&1 ; } ) 2>>logs/\$@ && \\ \$(PRT_DU_CR) >>logs/\$@ EOF ) >> $MKFILE.tmp } #----------------------------------# wrt_run_as_root2() { # #----------------------------------# local this_script=$1 local file=$2 ( cat << EOF @( time { source envars && ${PROGNAME}-commands/`dirname $file`/\$@ >>logs/\$@ 2>&1 ; } ) 2>>logs/\$@ && \\ echo -e "\nKB: \`du -skx --exclude=${SCRIPT_ROOT} \`\n" >>logs/\$@ EOF ) >> $MKFILE.tmp } unset wrt_run_as_chroot1 #----------------------------------# wrt_run_as_chroot1() { # #----------------------------------# local this_script=$1 local file=$2 ( cat << EOF @( time { \$(CHROOT1) 'cd \$(SCRIPT_ROOT) && source envars && \$(crCMDSDIR)/$file >>logs/${this_script} 2>&1' ; } ) 2>>logs/${this_script} && \\ \$(PRT_DU_CR) >> logs/${this_script} EOF ) >> $MKFILE.tmp } unset wrt_run_as_chroot2 #----------------------------------# wrt_run_as_chroot2() { # #----------------------------------# local this_script=$1 local file=$2 ( cat << EOF @( time { \$(CHROOT2) 'cd \$(SCRIPT_ROOT) && source envars && \$(crCMDSDIR)/`dirname $file`/\$@ >>logs/\$@ 2>&1' ; } ) 2>>logs/\$@ && \\ \$(PRT_DU_CR) >> logs/\$@ EOF ) >> $MKFILE.tmp } unset wrt_target #----------------------------------# wrt_target() { # Create target and initialize log file #----------------------------------# local i=$1 local PREV=$2 case $i in iteration* ) local LOGFILE="\$@.log" ;; * ) local LOGFILE="\$@" ;; esac ( cat << EOF $i: $PREV @\$(call echo_message, Building) @./progress_bar.sh \$@ & @echo -e "\n\`date\`\n\nKB: \`du -skx --exclude=${SCRIPT_ROOT} \$(MOUNT_PT)\`\n" >logs/$LOGFILE EOF ) >> $MKFILE.tmp } #----------------------------------# wrt_target_boot() { # Create target and initialize log file #----------------------------------# local i=$1 local PREV=$2 case $i in iteration* ) local LOGFILE="\$@.log" ;; * ) local LOGFILE="\$@" ;; esac ( cat << EOF $i: $PREV @\$(call echo_message, Building) @./progress_bar.sh \$@ & @echo -e "\n\`date\`\n\nKB: \`du -skx --exclude=${SCRIPT_ROOT}\`\n" >logs/$LOGFILE EOF ) >> $MKFILE.tmp } #----------------------------# run_make() { # #----------------------------# # Test if make must be run. if [ "$RUNMAKE" = "y" ] ; then # Test to make sure we're running the build as root if [ "$UID" != "0" ] ; then echo "You must be logged in as root to successfully build the system." exit 1 fi # Build the system if [ -e $MKFILE ] ; then echo -ne "Building the system...\n" cd $JHALFSDIR && make echo -ne "done\n" fi fi } #----------------------------# clean_builddir() { # #----------------------------# # Test if the clean must be done. if [ "${CLEAN}" = "y" ]; then # Test to make sure we're running the clean as root if [ "$UID" != "0" ] ; then echo "You must be logged in as root to clean the build directory." exit 1 fi # Test to make sure that the build directory was populated by jhalfs if [ ! -d $JHALFSDIR ] || [ ! -d $BUILDDIR/sources ] ; then echo "Looks like $BUILDDIR was not populated by a previous jhalfs run." exit 1 else # Clean the build directory echo -ne "Cleaning $BUILDDIR...\n" rm -rf $BUILDDIR/{bin,boot,dev,etc,home,lib,media,mnt,opt,proc,root,sbin,srv,sys,tmp,tools,cross-tools,usr,var} echo -ne "Cleaning $JHALFSDIR...\n" rm -rf $JHALFSDIR/{0*,1*,envars,sources-dir,*commands,*logs,Makefile,*.xsl,makefile-functions,pkg_tarball_list,*.config,*.sh} echo -ne "Cleaning remainig extracted sources in $BUILDDIR/sources...\n" rm -rf `find $BUILDDIR/sources/* -maxdepth 0 -type d` echo -ne "done\n" fi fi } #----------------------------# get_book() { # #----------------------------# cd $JHALFSDIR if [ -z $WORKING_COPY ] ; then # Check for Subversion instead of just letting the script hit 'svn' and fail. test `type -p svn` || eval "echo \"This feature requires Subversion.\" exit 1" echo -n "Downloading the $PROGNAME document, $LFSVRS version... " case $PROGNAME in lfs) svn_root="LFS" ;; hlfs) svn_root="HLFS" ;; clfs) svn_root="cross-lfs" ;; clfs2) svn_root="cross-lfs" ;; *) echo "BOOK not defined in function " exit 1 ;; esac # Grab a fresh book if it's missing, otherwise, update it from the # repo. If we've already extracted the commands, move on to getting the # sources. if [ -d ${PROGNAME}-$LFSVRS ] ; then cd ${PROGNAME}-$LFSVRS if LC_ALL=C svn up | grep -q At && \ test -d $JHALFSDIR/${PROGNAME}-commands && \ test -f $JHALFSDIR/pkg_tarball_list ; then # Set the canonical book version echo -ne "done\n" cd $JHALFSDIR case $PROGNAME in clfs | clfs2) VERSION=$(xmllint --noent $BOOK/prologue/$ARCH/bookinfo.xml 2>/dev/null | grep subtitle | sed -e 's/^.*ion //' -e 's/<\/.*//') ;; *) VERSION=$(xmllint --noent $BOOK/prologue/bookinfo.xml 2>/dev/null | grep subtitle | sed -e 's/^.*ion //' -e 's/<\/.*//') ;; esac get_sources else echo -ne "done\n" extract_commands fi else svn co $SVN/${svn_root}/${TREE} ${PROGNAME}-$LFSVRS >>$LOGDIR/$LOG 2>&1 echo -ne "done\n" extract_commands fi else echo -ne "Using $BOOK as book's sources ...\n" extract_commands fi echo -ne " Document version ${L_arrow}${BOLD}${VERSION}${R_arrow}\n" } #----------------------------# extract_commands() { # #----------------------------# # Check for libxslt instead of just letting the script hit 'xsltproc' and fail. test `type -p xsltproc` || eval "echo \"This feature requires libxslt.\" exit 1" cd $JHALFSDIR case $PROGNAME in clfs | clfs2 ) VERSION=$(xmllint --noent $BOOK/prologue/$ARCH/bookinfo.xml 2>/dev/null | grep subtitle | sed -e 's/^.*ion //' -e 's/<\/.*//') ;; *) VERSION=$(xmllint --noent $BOOK/prologue/bookinfo.xml 2>/dev/null | grep subtitle | sed -e 's/^.*ion //' -e 's/<\/.*//') ;; esac # Start clean if [ -d ${PROGNAME}-commands ]; then rm -rf ${PROGNAME}-commands mkdir -v ${PROGNAME}-commands fi echo -n "Extracting commands for" # Dump the commands in shell script form from the HLFS book. case ${PROGNAME} in clfs) echo -n " ${L_arrow}${BOLD}$ARCH${R_arrow} target architecture" xsltproc --nonet \ --xinclude \ --stringparam method $METHOD \ --stringparam testsuite $TEST \ --stringparam bomb-testsuite $BOMB_TEST \ --stringparam vim-lang $VIMLANG \ --stringparam timezone $TIMEZONE \ --stringparam page $PAGE \ --stringparam lang $LANG \ --stringparam keymap $KEYMAP \ -o ./${PROGNAME}-commands/ $XSL $BOOK/$ARCH-index.xml >>$LOGDIR/$LOG 2>&1 ;; clfs2) echo -n " ${L_arrow}${BOLD}$ARCH${R_arrow} target architecture" xsltproc --nonet \ --xinclude \ --stringparam vim-lang $VIMLANG \ --stringparam timezone $TIMEZONE \ --stringparam page $PAGE \ --stringparam lang $LANG \ --stringparam keymap $KEYMAP \ -o ./${PROGNAME}-commands/ $XSL $BOOK/$ARCH-index.xml >>$LOGDIR/$LOG 2>&1 ;; hlfs) echo -n " ${L_arrow}${BOLD}$MODEL${R_arrow} HLFS libc implementation" xsltproc --nonet \ --xinclude \ --stringparam model $MODEL \ --stringparam testsuite $TEST \ --stringparam bomb-testsuite $BOMB_TEST \ --stringparam timezone $TIMEZONE \ --stringparam page $PAGE \ --stringparam lang $LANG \ --stringparam lc_all $LC_ALL \ --stringparam keymap $KEYMAP \ --stringparam grsecurity_host $GRSECURITY_HOST \ -o ./${PROGNAME}-commands/ $XSL $BOOK/index.xml >>$LOGDIR/$LOG 2>&1 ;; lfs) echo -n " ${L_arrow}${BOLD}LFS${R_arrow} build" xsltproc --nonet \ --xinclude \ --stringparam testsuite $TEST \ --stringparam bomb-testsuite $BOMB_TEST \ --stringparam vim-lang $VIMLANG \ --stringparam timezone $TIMEZONE \ --stringparam page $PAGE \ --stringparam lang $LANG \ -o ./${PROGNAME}-commands/ $XSL $BOOK/index.xml >>$LOGDIR/$LOG 2>&1 ;; *) exit 1 ;; esac echo " ...OK" # Make the scripts executable. chmod -R +x $JHALFSDIR/${PROGNAME}-commands # Create the packages file. We need it for proper Makefile creation create_package_list # Done. Moving on... get_sources } #----------------------------# create_package_list() { # #----------------------------# # Create the packages file. We need it for proper Makefile creation rm -f pkg_tarball_list echo -n "Creating <${PROGNAME}> list of tarball names for $BOOK $ARCH" case ${PROGNAME} in clfs | clfs2) xsltproc --nonet --xinclude -o pkg_tarball_list packages.xsl \ $BOOK/materials/${ARCH}-chapter.xml >>$LOGDIR/$LOG 2>&1 ;; hlfs) xsltproc --nonet --xinclude -o pkg_tarball_list packages.xsl \ $BOOK/chapter04/chapter04.xml >>$LOGDIR/$LOG 2>&1 ;; lfs) xsltproc --nonet --xinclude -o pkg_tarball_list packages.xsl \ $BOOK/chapter03/chapter03.xml >>$LOGDIR/$LOG 2>&1 ;; esac echo " ...OK" } #----------------------------# get_sources() { # Download file, write name to MISSING_FILES.DMP if an error #----------------------------# local saveIFS=$IFS local IFS line URL1 URL2 FILE BOOKMD5 MD5 HAVEMD5 fromARCHIVE # Test if the packages must be downloaded [ ! "$GETPKG" = "y" ] && return gs_wrt_message(){ echo "${RED}$1${OFF}" echo "$1" >> MISSING_FILES.DMP } # Housekeeping [[ ! -d $BUILDDIR/sources ]] && mkdir $BUILDDIR/sources cd $BUILDDIR/sources [[ -f MD5SUMS ]] && rm MD5SUMS [[ -f MISSING_FILES.DMP ]] && rm MISSING_FILES.DMP [[ -f urls.lst ]] && rm urls.lst # Generate URLs file create_urls IFS=$'\x0A' # Modify the 'internal field separator' to break on 'LF' only for line in `cat urls.lst`; do IFS=$saveIFS # Restore the system defaults # Skip some packages if they aren't needed case $line in */tcl* | */expect* | */dejagnu* | */tree* | */gcc-testsuite* ) [[ "$TEST" = "0" ]] && continue ;; */vim-*-lang* ) [[ "$VIMLANG" = "0" ]] && continue ;; *linux/linux-* ) [[ -z "$CONFIG" ]] && [[ -z "$BOOT_CONFIG" ]] && \ [[ "$GETKERNEL" = "n" ]] && continue ;; esac # Locations URL1=`echo $line | cut -d" " -f2` # Preferred URL URL2=`echo $line | cut -d" " -f1` # Fallback Upstream URL FILE=`basename $URL1` # File name BOOKMD5=`echo $line | cut -d" " -f3` # MD5 book value # Validation pair MD5="$BOOKMD5 $FILE" HAVEMD5=1 set -e # If the file exists in the archive copy it to the # $BUILDDIR/sources dir. MD5SUM will be validated later. if [ ! -z ${SRC_ARCHIVE} ] && [ -d ${SRC_ARCHIVE} ] && [ -f ${SRC_ARCHIVE}/$FILE ]; then cp ${SRC_ARCHIVE}/$FILE . echo "$FILE: -- copied from $SRC_ARCHIVE" fromARCHIVE=1 else echo "${BOLD}${YELLOW}$FILE: not found in ${SRC_ARCHIVE}${OFF}" fromARCHIVE=0 # If the file does not exist yet in /sources download a fresh one if [ ! -f $FILE ] ; then if ! wget $URL1 && ! wget $URL2 ; then gs_wrt_message "$FILE not found in the SRC_ARCHIVE or on any server..SKIPPING" continue fi fi fi # IF the md5sum does not match the existing files if ! echo "$MD5" | md5sum -c - >/dev/null ; then [[ $fromARCHIVE = "1" ]] && echo "${BOLD}${YELLOW}MD5SUM did not match SRC_ARCHIVE copy${OFF}" [[ $fromARCHIVE = "0" ]] && echo "${BOLD}${YELLOW}MD5SUM did not match REMOTE copy${OFF}" # Remove the old file and download a new one rm -fv $FILE # Force storage in SRC_ARCHIVE fromARCHIVE=0; # Try to retrieve again the file. Servers in reverse order. if ! wget $URL2 && ! wget $URL1 ; then gs_wrt_message "$FILE not found on the servers.. SKIPPING" continue fi fi # Validate the MD5SUM one last time if ! echo "$MD5" | md5sum -c - >/dev/null ; then gs_wrt_message "$FILE does not match MD5SUMS value" # Force generation of MD5SUM HAVEMD5=0 fi # Generate a fresh MD5SUM for this file if [[ "$HAVEMD5" = "0" ]] ; then echo "${BOLD}${YELLOW}Generating a new MD5SUM for ${OFF}$FILE" echo "NEW MD5SUM: $(md5sum $FILE)" >> MISSING_FILES.DMP fi # Good or bad we write the original md5sum to a file echo "$MD5" >> MD5SUMS # Copy the freshly downloaded file # to the source archive. if [ ! -z ${SRC_ARCHIVE} ] && [ -d ${SRC_ARCHIVE} ] && [ -w ${SRC_ARCHIVE} ] && [ "$fromARCHIVE" = "0" ] ; then echo "Storing file:<$FILE> in the package archive" cp -f $FILE ${SRC_ARCHIVE} fi done if [[ -s MISSING_FILES.DMP ]]; then echo -e "\n\n${tab_}${RED} One or more files were not retrieved or have bad MD5SUMS.\n${tab_} Check ${L_arrow}$BUILDDIR/sources/MISSING_FILES.DMP${R_arrow} for names ${OFF}\n" # Do not allow the automatic execution of the Makefile. echo "${tab_}${BOLD}${RED}*** ${YELLOW}Automatic execution of the generated makefile has been inhibited. ${RED}***${OFF}${nl_}" RUNMAKE="n" fi } #----------------------------# create_urls() { # #----------------------------# cd $JHALFSDIR case ${PROGNAME} in clfs) echo -n "Creating CLFS <${ARCH}> specific URLs file" xsltproc --nonet --xinclude \ --stringparam server $SERVER \ -o $BUILDDIR/sources/urls.lst urls.xsl \ $BOOK/materials/$ARCH-chapter.xml >>$LOGDIR/$LOG 2>&1 echo " ...OK" ;; clfs2) echo -n "Creating CLFS2 <${ARCH}> specific URLs file" xsltproc --nonet --xinclude \ --stringparam server $SERVER \ -o $BUILDDIR/sources/urls.lst urls.xsl \ $BOOK/materials/$ARCH-chapter.xml >>$LOGDIR/$LOG 2>&1 echo " ...OK" ;; hlfs) echo -n "Creating HLFS <${MODEL}> specific URLs file" xsltproc --nonet --xinclude \ --stringparam server $SERVER \ --stringparam model $MODEL \ -o $BUILDDIR/sources/urls.lst urls.xsl \ $BOOK/chapter04/chapter04.xml >>$LOGDIR/$LOG 2>&1 echo " ...OK" ;; lfs) echo -n "Creating LFS specific URLs file" xsltproc --nonet --xinclude \ --stringparam server $SERVER \ -o ../sources/urls.lst urls.xsl \ $BOOK/chapter03/chapter03.xml >>$LOGDIR/$LOG 2>&1 echo " ...OK" ;; esac cd $BUILDDIR/sources }