From 65d83a612a8ede044d17f1ffa3b5bf9593cfbb11 Mon Sep 17 00:00:00 2001 From: Manuel Canales Esparcia Date: Wed, 3 May 2006 19:35:01 +0000 Subject: [PATCH] Synchrnized experimental branch with trunk. --- LFS/master.sh | 4 - README | 26 ++++- common/common-functions | 196 +++++++++++++++++++------------- common/config | 2 +- common/create-sbu_du-report.sh | 10 +- common/func_compare.sh | 1 + common/func_validate_configs.sh | 2 +- common/urls.xsl | 2 +- master.sh | 14 +-- 9 files changed, 158 insertions(+), 99 deletions(-) diff --git a/LFS/master.sh b/LFS/master.sh index 9ab8a92..7fb6966 100755 --- a/LFS/master.sh +++ b/LFS/master.sh @@ -161,10 +161,6 @@ chapter6_Makefiles() { done # Remove Bzip2 binaries before make install sed -e 's@make install@rm -vf /usr/bin/bz*\n&@' -i chapter06$N/*-bzip2 - # Fix how Module-Init-Tools do the install target - sed -e 's@make install@make INSTALL=install install@' -i chapter06$N/*-module-init-tools - # Delete *old Readline libraries just after make install - sed -e 's@make install@&\nrm -v /lib/lib{history,readline}*old@' -i chapter06$N/*-readline # Let some Udev pre-installation commands to fail sed -e 's@/lib/udev/devices/fd@& || true@' \ -e 's/mknod -m.*/& || true/' -i chapter06$N/*-udev diff --git a/README b/README index 35f85f1..5a702bf 100644 --- a/README +++ b/README @@ -75,7 +75,23 @@ $Id$ 6. FAQ:: Q. "It doesn't work" A. Yes it does.. - + + Q. "How do I specify the build location" + A. The original LFS document worked against the well know location /mnt/lfs. + This script automates the build of all of the LFS series of books and uses + a generic location $BUILDDIR with a default value a /mnt/build_dir. + You may change this value to suit your needs. + + The layout below $BUILDDIR is as follows. + $BUILDDIR/ + jhalfs (makefile,cmd scripts,logs..etc) + sources (where packages reside + temptools (temporary bootstrap system + cross-tools (CLFS only) + ... + FHS dir structure + ... + Q. "What is the function of the SRC_ARCHIVE variable A. When then symlinked master.sh runs it creates a local copy of the necessary packages in BUILDDIR/sources by downloading the files. If @@ -83,7 +99,13 @@ $Id$ this location for the file and, if found, will copy it to BUILDDIR/sources. If the files are not found in SRC_ARCHIVE _and_ you have write priv to the directory any downloaded files will be mirrored there. - + + Q. "How do I set the SRC_ARCHIVE location" + A. The best way to set the value of SRC_ARCHIVE is + export SRC_ARCHIVE=/wherever/you/store/downloaded/packages + OR + you can change the setting in .common/config. + Q. "Why have 2 copies of the files." A. The package files must be visible during the chroot phase and this is a simple and reliable method of doing so. This method also handles the CLFS diff --git a/common/common-functions b/common/common-functions index 2ef41b0..3591c1b 100644 --- a/common/common-functions +++ b/common/common-functions @@ -24,9 +24,9 @@ declare -r HIDDEN=$'\e[8m' declare -r tab_=$'\t' declare -r nl_=$'\n' -declare -r DD_BORDER="${BOLD}${WHITE}==============================================================================${OFF}" -declare -r SD_BORDER="${BOLD}${WHITE}------------------------------------------------------------------------------${OFF}" -declare -r STAR_BORDER="${BOLD}${WHITE}******************************************************************************${OFF}" +declare -r DD_BORDER="${BOLD}==============================================================================${OFF}" +declare -r SD_BORDER="${BOLD}------------------------------------------------------------------------------${OFF}" +declare -r STAR_BORDER="${BOLD}******************************************************************************${OFF}" # bold yellow > < pair declare -r R_arrow=$'\e[1;33m>\e[0m' @@ -53,6 +53,7 @@ ${BOLD} -B, --book VER${OFF} ${BOLD} -D --directory DIR${OFF} use DIR directory for building ${BOLD}$(echo $PROGNAME | tr [a-z] [A-Z])${OFF}; all files jhalfs-X produces will be in the directory DIR/jhalfs. + The current setting for BUILDDIR is "$BUILDDIR" ${BOLD} -G, --get-packages${OFF} download the packages and patches. This assumes that the server declared @@ -244,9 +245,9 @@ EOF } -#=============================# -wrt_unpack3() { # -#=============================# +#----------------------------------# +wrt_unpack3() { # +#----------------------------------# local FILE=$1 ( cat << EOF @@ -300,9 +301,9 @@ EOF } -#=============================# -wrt_run_as_root2() { # -#=============================# +#----------------------------------# +wrt_run_as_root2() { # +#----------------------------------# local this_script=$1 local file=$2 ( @@ -399,7 +400,7 @@ EOF #----------------------------------# -wrt_report() { # +wrt_report() { # #----------------------------------# ( cat << EOF @@ -417,7 +418,7 @@ chapter789="$chapter789 create-sbu_du-report" #----------------------------# -run_make() { +run_make() { # #----------------------------# # Test if make must be run. if [ "$RUNMAKE" = "1" ] ; then @@ -437,7 +438,7 @@ run_make() { #----------------------------# -clean_builddir() { +clean_builddir() { # #----------------------------# # Test if the clean must be done. if [ "$CLEAN" = "1" ] ; then @@ -464,7 +465,7 @@ clean_builddir() { } #----------------------------# -get_book() { +get_book() { # #----------------------------# cd $JHALFSDIR @@ -487,8 +488,8 @@ get_book() { # sources. if [ -d ${PROGNAME}-$LFSVRS ] ; then cd ${PROGNAME}-$LFSVRS - if LC_ALL=C svn up | grep -q At && test -d $JHALFSDIR/commands && \ - test -f $JHALFSDIR/packages && test -f $JHALFSDIR/patches ; then + if LC_ALL=C svn up | grep -q At && test -d $JHALFSDIR/${PROGNAME}-commands && \ + test -f $JHALFSDIR/packages ; then echo -ne "done\n" # Set the canonical book version cd $JHALFSDIR @@ -627,94 +628,133 @@ extract_commands() { # get_sources } - #----------------------------# get_sources() { # Download file, write name to MISSING_FILES.DMP if an error #----------------------------# local saveIFS=$IFS - local IFS line URL1 URL2 FILE MD5 + local IFS line URL1 URL2 FILE MD5 HAVEMD5 fromARCHIVE - # Test if the packages must be downloaded - if [ "$GETPKG" = "1" ] ; then + # Test if the packages must be downloaded + [ ! "$GETPKG" = "1" ] && return - [[ ! -d $BUILDDIR/sources ]] && mkdir $BUILDDIR/sources - cd $BUILDDIR/sources - [[ -f MD5SUMS ]] && rm MD5SUMS - [[ -f MD5SUMS-$VERSION ]] && rm MD5SUMS-$VERSION - [[ -f MISSING_FILES.DMP ]] && rm MISSING_FILES.DMP - [[ -f urls.lst ]] && rm urls.lst + gs_wrt_message(){ + echo "${RED}$1${OFF}" + echo "$1" >> MISSING_FILES.DMP + } + # Housekeeping + [[ ! -d $BUILDDIR/sources ]] && mkdir $BUILDDIR/sources + cd $BUILDDIR/sources + [[ -f MD5SUMS ]] && rm MD5SUMS + [[ -f MD5SUMS-$VERSION ]] && rm MD5SUMS-$VERSION + [[ -f MISSING_FILES.DMP ]] && rm MISSING_FILES.DMP + [[ -f urls.lst ]] && rm urls.lst # Download a fresh MD5SUMS file - wget $SERVER/pub/lfs/conglomeration/MD5SUMS + wget $SERVER/pub/lfs/conglomeration/MD5SUMS # Generate URLs file - create_urls + create_urls - IFS=$'\x0A' # Modify the 'internal field separator' to break on 'LF' only - for line in `cat urls.lst`; do - IFS=$saveIFS # Restore the system defaults + IFS=$'\x0A' # Modify the 'internal field separator' to break on 'LF' only + for line in `cat urls.lst`; do + IFS=$saveIFS # Restore the system defaults # Skip some packages if they aren't needed - case $line in + case $line in */tcl* | */expect* | */dejagnu* | */tree* | */gcc-testsuite* ) [[ "$TEST" = "0" ]] && continue ;; */vim-*-lang* ) [[ "$VIMLANG" = "0" ]] && continue ;; - esac + esac - URL1=`echo $line | cut -d" " -f1` # Upstream URL - URL2=`echo $line | cut -d" " -f2` # Fallback URL - FILE=`basename $URL2` # File name - - # If the file exists in the archive copy it to the $BUILDDIR/sources dir - # MD5SUM is assumed to be correct from previous download - if [ ! -z ${SRC_ARCHIVE} ] && - [ -d ${SRC_ARCHIVE} ] && - [ -f ${SRC_ARCHIVE}/$FILE ]; then - cp ${SRC_ARCHIVE}/$FILE . - echo "$FILE: -- copied from $SRC_ARCHIVE" - fi + # Locations + URL1=`echo $line | cut -d" " -f2` # Preferred URL + URL2=`echo $line | cut -d" " -f1` # Fallback Upstream URL + FILE=`basename $URL2` # File name # Find the md5 sum for this package. - set +e - MD5=`grep " $FILE" MD5SUMS` - if [ $? -ne 0 ]; then - set -e - echo "${RED}$FILE not found in MD5SUMS${OFF}" - echo "$FILE not found in MD5SUMS" >> MISSING_FILES.DMP - fi + set +e + HAVEMD5=1 # Always assume we have a MD5SUM + MD5=`grep " $FILE" MD5SUMS` + if [ $? -ne 0 ]; then set -e - - if [ ! -f $FILE ] ; then - wget $URL1 || wget $URL2 - elif ! echo "$MD5" | md5sum -c - >/dev/null 2>/dev/null ; then - wget -N $URL1 || wget -N $URL2 - fi - - if ! echo "$MD5" | md5sum -c - ; then - echo "${RED}$FILE not match MD5SUMS value${OFF}" - echo "$FILE not match MD5SUMS value" >> MISSING_FILES.DMP - fi - echo `grep "$MD5" MD5SUMS` >> MD5SUMS-$VERSION - - # Copy the freshly downloaded file - # to the source archive. - if [ ! -z ${SRC_ARCHIVE} ] && - [ -d ${SRC_ARCHIVE} ] && - [ -w ${SRC_ARCHIVE} ] && - [ ! -f ${SRC_ARCHIVE}/$FILE ] ; then - echo "Store file:<$FILE> in package archive" - cp -v $FILE ${SRC_ARCHIVE} - fi - - done - - if [[ -s MISSING_FILES.DMP ]]; then - echo -e "\n\n${tab_}${RED} One or more files were not retrieved or have a bad MD5SUMS chechsum.\n${tab_} Check ${L_arrow}$BUILDDIR/sources/MISSING_FILES.DMP${R_arrow} for names ${OFF}\n\n" + gs_wrt_message "$FILE not found in MD5SUMS" + # IMPORTANT:: There is no MD5SUM for this file. + HAVEMD5=0 fi + set -e + # If the file exists in the archive copy it to the + # $BUILDDIR/sources dir. MD5SUM will be validated later. + if [ ! -z ${SRC_ARCHIVE} ] && + [ -d ${SRC_ARCHIVE} ] && + [ -f ${SRC_ARCHIVE}/$FILE ]; then + cp ${SRC_ARCHIVE}/$FILE . + echo "$FILE: -- copied from $SRC_ARCHIVE" + fromARCHIVE=1 + else + echo "${BOLD}${YELLOW}$FILE: not found in ${SRC_ARCHIVE}${OFF}" + fromARCHIVE=0 + # If the file does not exist in /sources download a fresh one + if [ ! -f $FILE ] ; then + if ! wget $URL1 && ! wget $URL2 ; then + gs_wrt_message "$FILE not found in the SRC_ARCHIVE or on any server..SKIPPING" + continue + fi + fi + fi + + # Is there a MD5SUM to validate the file against. + if [[ "$HAVEMD5" = "1" ]] ; then + # IF the md5sum does not match the existing files + if ! echo "$MD5" | md5sum -c - >/dev/null ; then + [[ $fromARCHIVE = "1" ]] && echo "${BOLD}${YELLOW}MD5SUM did not match SRC_ARCHIVE copy${OFF}" + [[ $fromARCHIVE = "0" ]] && echo "${BOLD}${YELLOW}MD5SUM did not match REMOTE copy${OFF}" + # Remove the old file and download a new one + rm -fv $FILE + # Force generation of MD5SUM and storage in SRC_ARCHIVE + fromARCHIVE=0; HAVEMD5=0 + # Try and retrieve the file + if ! wget -N $URL1 && ! wget -N $URL2 ; then + gs_wrt_message "$FILE not found on the servers.. SKIPPING" + continue + fi + fi + fi + + # Validate the MD5SUM one last time + if [[ "$HAVEMD5" = "1" ]] && ! echo "$MD5" | md5sum -c - >/dev/null ; then + gs_wrt_message "$FILE does not match MD5SUMS value" + fi + + # Generate a fresh MD5SUM for this file + if [[ "$HAVEMD5" = "0" ]] ; then + echo "${BOLD}${YELLOW}Generating a new MD5SUM for ${OFF}$FILE" + echo "NEW MD5SUM $(md5sum $FILE)" >> MISSING_FILES.DMP + fi + + # Good or bad we write the original md5sum to a file + echo "$MD5" >> MD5SUMS-$VERSION + + # Copy the freshly downloaded file + # to the source archive. + if [ ! -z ${SRC_ARCHIVE} ] && + [ -d ${SRC_ARCHIVE} ] && + [ -w ${SRC_ARCHIVE} ] && + [ "$fromARCHIVE" = "0" ] ; then + echo "Storing file:<$FILE> in the package archive" + cp -f $FILE ${SRC_ARCHIVE} + fi + + done + + if [[ -s MISSING_FILES.DMP ]]; then + echo -e "\n\n${tab_}${RED} One or more files were not retrieved or have a bad MD5SUMS chechsum.\n${tab_} Check ${L_arrow}$BUILDDIR/sources/MISSING_FILES.DMP${R_arrow} for names ${OFF}\n" + # Do not allow the automatic exection of the Makefile. + echo "${tab_}${BOLD}${RED}*** ${YELLOW}Automatic execution of the generated makefile has been inhibited. ${RED}***${OFF}${nl_}" + RUNMAKE=0 fi } diff --git a/common/config b/common/config index a7fdddc..cae590f 100644 --- a/common/config +++ b/common/config @@ -52,7 +52,7 @@ STRIP=1 PAGE=letter #--- set default timezone. -TIMEZONE=America/Toronto +TIMEZONE=GMT #--- install the optional vim-lang package 0(no)/1(yes) VIMLANG=1 diff --git a/common/create-sbu_du-report.sh b/common/create-sbu_du-report.sh index be918fe..07b45ee 100755 --- a/common/create-sbu_du-report.sh +++ b/common/create-sbu_du-report.sh @@ -14,7 +14,7 @@ VERSION=$2 [[ ! -f "$LOGSDIR"/000-masterscript.log ]] && \ echo -e "\nLooks like $LOGSDIR isn't a jhalfs logs directory.\n" && exit -# If this script is run manually, the book version may be unknow +# If this script is run manually, the book version may be unknown [[ -z "$VERSION" ]] && VERSION=unknown # If there is iteration logs directories, copy the logs inside iteration-1 @@ -74,10 +74,10 @@ for log in $BUILDLOGS ; do SBU2="$SBU2 + $SBU" #Start disk usage calculation - # Disk usage before unpack the package + # Disk usage before unpacking the package DU1=`grep "^KB: " $log | head -n1 | cut -f1 | sed -e 's/KB: //'` DU1MB=`echo "scale=2; $DU1 / 1024" | bc` - # Disk usage before delete sources and build dirs + # Disk usage before deleting the source and build dirs DU2=`grep "^KB: " $log | tail -n1 | cut -f1 | sed -e 's/KB: //'` DU2MB=`echo "scale=2; $DU2 / 1024" | bc` # Calculate disk space required to do the build @@ -108,8 +108,8 @@ for log in $BUILDLOGS ; do echo -e "Approximate SBU time is:\t$SBU" >> "$REPORT" # Dump disk usage values - echo -e "\nDisk usage before unpack the package:\t\t\t$DU1 KB or $DU1MB MB" >> "$REPORT" - echo -e "Disk usage before delete sources and build dirs:\t$DU2 KB or $DU2MB MB" >> "$REPORT" + echo -e "\nDisk usage before unpacking the package:\t\t\t$DU1 KB or $DU1MB MB" >> "$REPORT" + echo -e "Disk usage before deleting the source and build dirs:\t$DU2 KB or $DU2MB MB" >> "$REPORT" echo -e "Required space to build the package:\t\t\t$REQUIRED1 KB or $REQUIRED2 MB\n" >> "$REPORT" done diff --git a/common/func_compare.sh b/common/func_compare.sh index e4ff4f7..50197a0 100644 --- a/common/func_compare.sh +++ b/common/func_compare.sh @@ -108,6 +108,7 @@ wrt_logs() { # ( cat << EOF @pushd logs 1> /dev/null && \\ + rm $ITERATION && \\ mkdir $ITERATION && \\ mv ${LOGS} $ITERATION && \\ popd 1> /dev/null diff --git a/common/func_validate_configs.sh b/common/func_validate_configs.sh index 423e55c..4b4b20d 100644 --- a/common/func_validate_configs.sh +++ b/common/func_validate_configs.sh @@ -255,5 +255,5 @@ inline_doc esac done set -e - echo "$tab_***${BOLD}${GREEN} ${PARAM_GROUP%%_*T} config parameters look good${OFF} ***" + echo "${nl_}***${BOLD}${GREEN} ${PARAM_GROUP%%_*T} config parameters look good${OFF} ***${nl_}" } diff --git a/common/urls.xsl b/common/urls.xsl index 709da96..f7759d4 100644 --- a/common/urls.xsl +++ b/common/urls.xsl @@ -22,7 +22,7 @@ diff --git a/master.sh b/master.sh index 6c97e57..20331f3 100755 --- a/master.sh +++ b/master.sh @@ -71,7 +71,7 @@ source $MODULE [[ $? > 0 ]] && echo "$MODULE did not load.." && exit 2 [[ $VERBOSITY > 0 ]] && echo "OK" # -[[ $VERBOSITY > 0 ]] && echo "---------------${nl_}" +[[ $VERBOSITY > 0 ]] && echo "${SD_BORDER}${nl_}" #=========================================================== @@ -92,7 +92,7 @@ source $COMMON_DIR/func_check_version.sh source $COMMON_DIR/func_validate_configs.sh [[ $? > 0 ]] && echo " function module did not load.." && exit 2 [[ $VERBOSITY > 0 ]] && echo "OK" -[[ $VERBOSITY > 0 ]] && echo "---------------${nl_}" +[[ $VERBOSITY > 0 ]] && echo "${SD_BORDER}${nl_}" ################################### @@ -432,10 +432,10 @@ check_version "3.0" "$BASH_VERSION" "BASH" check_version "3.0" "`gcc -dumpversion`" "GCC" tarVer=`tar --version` check_version "1.15.0" "${tarVer##* }" "TAR" -echo "---------------${nl_}" +echo "${SD_BORDER}${nl_}" validate_config -echo "---------------${nl_}" +echo "${SD_BORDER}${nl_}" echo -n "Are you happy with these settings? yes/no (no): " read ANSWER @@ -443,7 +443,7 @@ if [ x$ANSWER != "xyes" ] ; then echo "${nl_}Fix the configuration options and rerun the script.${nl_}" exit 1 fi -echo "${nl_}---------------${nl_}" +echo "${nl_}${SD_BORDER}${nl_}" # Prevents setting "-d /" by mistake. @@ -495,10 +495,10 @@ fi >$LOGDIR/$LOG get_book -echo "---------------${nl_}" +echo "${SD_BORDER}${nl_}" build_Makefile -echo "---------------${nl_}" +echo "${SD_BORDER}${nl_}" run_make