Synchrnized experimental branch with trunk.

LFS/master.sh

@@ -161,10 +161,6 @@ chapter6_Makefiles() {
     done
     # Remove Bzip2 binaries before make install
     sed -e 's@make install@rm -vf /usr/bin/bz*\n&@' -i chapter06$N/*-bzip2
-    # Fix how Module-Init-Tools do the install target
-    sed -e 's@make install@make INSTALL=install install@' -i chapter06$N/*-module-init-tools
-    # Delete *old Readline libraries just after make install
-    sed -e 's@make install@&\nrm -v /lib/lib{history,readline}*old@' -i chapter06$N/*-readline
     # Let some Udev pre-installation commands to fail
     sed -e 's@/lib/udev/devices/fd@& || true@' \
         -e 's/mknod -m.*/& || true/' -i chapter06$N/*-udev

README

@@ -76,6 +76,22 @@ $Id$
     Q. "It doesn't work"
     A. Yes it does..
     
+    Q. "How do I specify the build location"
+    A. The original LFS document worked against the well know location /mnt/lfs.
+       This script automates the build of all of the LFS series of books and uses
+       a generic location $BUILDDIR with a default value a /mnt/build_dir.
+       You may change this value to suit your needs.
+ 
+       The layout below $BUILDDIR is as follows.
+       $BUILDDIR/
+            jhalfs      (makefile,cmd scripts,logs..etc)
+	    sources     (where packages reside
+	    temptools   (temporary bootstrap system
+	    cross-tools (CLFS only)
+	        ...
+	    FHS dir structure
+	        ...
+    
     Q. "What is the function of the SRC_ARCHIVE variable
     A. When then symlinked master.sh runs it creates a local copy of the
        necessary packages in BUILDDIR/sources by downloading the files. If 
@@ -84,6 +100,12 @@ $Id$
        If the files are not found in SRC_ARCHIVE _and_ you have write priv to 
        the directory any downloaded files will be mirrored there.
        
+    Q. "How do I set the SRC_ARCHIVE location"
+    A. The best way to set the value of SRC_ARCHIVE is
+       export SRC_ARCHIVE=/wherever/you/store/downloaded/packages
+                       OR
+       you can change the setting in .common/config. 
+    
     Q. "Why have 2 copies of the files."
     A. The package files must be visible during the chroot phase and this is a 
        simple and reliable method of doing so. This method also handles the CLFS

common/common-functions

@@ -24,9 +24,9 @@ declare -r  HIDDEN=$'\e[8m'
 declare -r  tab_=$'\t'
 declare -r  nl_=$'\n'
 
-declare -r   DD_BORDER="${BOLD}${WHITE}==============================================================================${OFF}"
+declare -r   DD_BORDER="${BOLD}==============================================================================${OFF}"
-declare -r   SD_BORDER="${BOLD}${WHITE}------------------------------------------------------------------------------${OFF}"
+declare -r   SD_BORDER="${BOLD}------------------------------------------------------------------------------${OFF}"
-declare -r STAR_BORDER="${BOLD}${WHITE}******************************************************************************${OFF}"
+declare -r STAR_BORDER="${BOLD}******************************************************************************${OFF}"
 
 # bold yellow > <  pair
 declare -r R_arrow=$'\e[1;33m>\e[0m'
@@ -53,6 +53,7 @@ ${BOLD}  -B, --book VER${OFF}
 ${BOLD}  -D  --directory DIR${OFF}
         use DIR directory for building ${BOLD}$(echo $PROGNAME | tr [a-z] [A-Z])${OFF}; all files jhalfs-X produces
         will be in the directory DIR/jhalfs.
+        The current setting for BUILDDIR is "$BUILDDIR"
 
 ${BOLD}  -G, --get-packages${OFF}
         download the packages and patches. This assumes that the server declared
@@ -244,9 +245,9 @@ EOF
 }
 
 
-#=============================#
+#----------------------------------#
 wrt_unpack3() {                    #
-#=============================#
+#----------------------------------#
   local FILE=$1
 (
 cat  << EOF
@@ -300,9 +301,9 @@ EOF
 }
 
 
-#=============================#
+#----------------------------------#
 wrt_run_as_root2() {               #
-#=============================#
+#----------------------------------#
   local this_script=$1
   local file=$2
 (
@@ -417,7 +418,7 @@ chapter789="$chapter789 create-sbu_du-report"
 
 
 #----------------------------#
-run_make() {
+run_make() {                 #
 #----------------------------#
   # Test if make must be run.
   if [ "$RUNMAKE" = "1" ] ; then
@@ -437,7 +438,7 @@ run_make() {
 
 
 #----------------------------#
-clean_builddir() {
+clean_builddir() {           #
 #----------------------------#
   # Test if the clean must be done.
   if [ "$CLEAN" = "1" ] ; then
@@ -464,7 +465,7 @@ clean_builddir() {
 }
 
 #----------------------------#
-get_book() {
+get_book() {                 #
 #----------------------------#
   cd $JHALFSDIR
 
@@ -487,8 +488,8 @@ get_book() {
     # sources.
     if [ -d ${PROGNAME}-$LFSVRS ] ; then
       cd ${PROGNAME}-$LFSVRS
-      if LC_ALL=C svn up | grep -q At && test -d $JHALFSDIR/commands && \
+      if LC_ALL=C svn up | grep -q At && test -d $JHALFSDIR/${PROGNAME}-commands && \
-      test -f $JHALFSDIR/packages && test -f $JHALFSDIR/patches ; then
+      test -f $JHALFSDIR/packages ; then
         echo -ne "done\n"
         # Set the canonical book version
         cd $JHALFSDIR
@@ -627,16 +628,20 @@ extract_commands() {         #
   get_sources
 }
 
-
 #----------------------------#
 get_sources() {              # Download file, write name to MISSING_FILES.DMP if an error
 #----------------------------#
   local saveIFS=$IFS
-  local IFS line URL1 URL2 FILE MD5
+  local IFS line URL1 URL2 FILE MD5 HAVEMD5 fromARCHIVE
 
     # Test if the packages must be downloaded
-  if [ "$GETPKG" = "1" ] ; then
+  [ ! "$GETPKG" = "1" ] && return
 
+  gs_wrt_message(){
+    echo "${RED}$1${OFF}"
+    echo "$1" >> MISSING_FILES.DMP
+  }
+    # Housekeeping
   [[ ! -d $BUILDDIR/sources ]] && mkdir $BUILDDIR/sources
   cd $BUILDDIR/sources
   [[ -f MD5SUMS ]] && rm MD5SUMS
@@ -664,57 +669,92 @@ get_sources() {              # Download file, write name to MISSING_FILES.DMP if
           ;;
     esac
 
-      URL1=`echo $line | cut -d" " -f1` # Upstream URL
+      # Locations
-      URL2=`echo $line | cut -d" " -f2` # Fallback URL
+    URL1=`echo $line | cut -d" " -f2` # Preferred URL
+    URL2=`echo $line | cut -d" " -f1` # Fallback Upstream URL
     FILE=`basename $URL2` # File name
 
-      # If the file exists in the archive copy it to the $BUILDDIR/sources dir
+      # Find the md5 sum for this package.
-      # MD5SUM is assumed to be correct from previous download
+    set +e
+    HAVEMD5=1  # Always assume we have a MD5SUM
+    MD5=`grep "  $FILE" MD5SUMS`
+    if [ $? -ne 0 ]; then
+      set -e
+      gs_wrt_message "$FILE not found in MD5SUMS"
+        # IMPORTANT:: There is no MD5SUM for this file.
+      HAVEMD5=0
+    fi
+
+    set -e
+      # If the file exists in the archive copy it to the
+      # $BUILDDIR/sources dir. MD5SUM will be validated later.
     if [ ! -z ${SRC_ARCHIVE} ] &&
        [   -d ${SRC_ARCHIVE} ] &&
        [   -f ${SRC_ARCHIVE}/$FILE ]; then
       cp ${SRC_ARCHIVE}/$FILE .
       echo "$FILE: -- copied from $SRC_ARCHIVE"
-      fi
+      fromARCHIVE=1
-
+    else
-      # Find the md5 sum for this package.
+      echo "${BOLD}${YELLOW}$FILE: not found in ${SRC_ARCHIVE}${OFF}"
-      set +e
+      fromARCHIVE=0
-      MD5=`grep "  $FILE" MD5SUMS`
+      # If the file does not exist in /sources download a fresh one
-      if [ $? -ne 0 ]; then
-        set -e
-        echo "${RED}$FILE not found in MD5SUMS${OFF}"
-        echo "$FILE not found in MD5SUMS" >> MISSING_FILES.DMP
-      fi
-      set -e
-
       if [ ! -f $FILE ] ; then
-        wget $URL1 || wget $URL2
+        if ! wget $URL1 && ! wget $URL2 ; then
-      elif ! echo "$MD5" | md5sum -c - >/dev/null 2>/dev/null ; then
+          gs_wrt_message "$FILE not found in the SRC_ARCHIVE or on any server..SKIPPING"
-        wget -N $URL1 || wget -N $URL2
+          continue
+        fi
+      fi
     fi
 
-      if ! echo "$MD5" | md5sum -c - ; then
+      # Is there a MD5SUM to validate the file against.
-        echo "${RED}$FILE not match MD5SUMS value${OFF}"
+    if [[ "$HAVEMD5" = "1" ]] ; then
-        echo "$FILE not match MD5SUMS value" >> MISSING_FILES.DMP
+        # IF the md5sum does not match the existing files
+      if ! echo "$MD5" | md5sum -c - >/dev/null ; then
+        [[ $fromARCHIVE = "1" ]] && echo "${BOLD}${YELLOW}MD5SUM did not match SRC_ARCHIVE copy${OFF}"
+        [[ $fromARCHIVE = "0" ]] && echo "${BOLD}${YELLOW}MD5SUM did not match REMOTE copy${OFF}"
+          # Remove the old file and download a new one
+        rm -fv $FILE
+          # Force generation of MD5SUM and storage in SRC_ARCHIVE
+        fromARCHIVE=0; HAVEMD5=0
+          # Try and retrieve the file
+        if ! wget -N $URL1 && ! wget -N $URL2 ; then
+          gs_wrt_message "$FILE not found on the servers.. SKIPPING"
+          continue
         fi
-      echo `grep "$MD5" MD5SUMS` >> MD5SUMS-$VERSION
+      fi
+    fi
+
+      # Validate the MD5SUM one last time
+    if [[ "$HAVEMD5" = "1" ]] && ! echo "$MD5" | md5sum -c - >/dev/null ; then
+      gs_wrt_message "$FILE does not match MD5SUMS value"
+    fi
+
+      # Generate a fresh MD5SUM for this file
+    if [[ "$HAVEMD5" = "0" ]] ; then
+      echo "${BOLD}${YELLOW}Generating a new MD5SUM for ${OFF}$FILE"
+      echo "NEW MD5SUM $(md5sum $FILE)" >> MISSING_FILES.DMP
+    fi
+
+      # Good or bad we write the original md5sum to a file
+    echo "$MD5" >> MD5SUMS-$VERSION
 
      # Copy the freshly downloaded file
      # to the source archive.
    if [ ! -z ${SRC_ARCHIVE}  ] &&
       [   -d ${SRC_ARCHIVE}  ] &&
       [   -w ${SRC_ARCHIVE}  ] &&
-         [ ! -f ${SRC_ARCHIVE}/$FILE ] ; then
+      [ "$fromARCHIVE" = "0" ] ; then
-        echo "Store file:<$FILE> in package archive"
+        echo "Storing file:<$FILE> in the package archive"
-        cp -v $FILE ${SRC_ARCHIVE}
+        cp -f $FILE ${SRC_ARCHIVE}
    fi
 
   done
 
   if [[ -s MISSING_FILES.DMP ]]; then
-      echo  -e "\n\n${tab_}${RED} One or more files were not retrieved or have a bad MD5SUMS chechsum.\n${tab_} Check ${L_arrow}$BUILDDIR/sources/MISSING_FILES.DMP${R_arrow} for names ${OFF}\n\n"
+    echo  -e "\n\n${tab_}${RED} One or more files were not retrieved or have a bad MD5SUMS chechsum.\n${tab_} Check ${L_arrow}$BUILDDIR/sources/MISSING_FILES.DMP${R_arrow} for names ${OFF}\n"
-    fi
+      # Do not allow the automatic exection of the Makefile.
-
+    echo "${tab_}${BOLD}${RED}*** ${YELLOW}Automatic execution of the generated makefile has been inhibited. ${RED}***${OFF}${nl_}"
+    RUNMAKE=0
   fi
 }
 

common/config

@@ -52,7 +52,7 @@ STRIP=1
 PAGE=letter
 
 #--- set default timezone.
-TIMEZONE=America/Toronto
+TIMEZONE=GMT
 
 #--- install the optional vim-lang package  0(no)/1(yes)
 VIMLANG=1

common/create-sbu_du-report.sh

@@ -14,7 +14,7 @@ VERSION=$2
 [[ ! -f "$LOGSDIR"/000-masterscript.log ]] && \
   echo -e "\nLooks like $LOGSDIR isn't a jhalfs logs directory.\n" && exit
 
-# If this script is run manually, the book version may be unknow
+# If this script is run manually, the book version may be unknown
 [[ -z "$VERSION" ]] && VERSION=unknown
 
 # If there is iteration logs directories, copy the logs inside iteration-1
@@ -74,10 +74,10 @@ for log in $BUILDLOGS ; do
   SBU2="$SBU2 + $SBU"
 
 #Start disk usage calculation
-  # Disk usage before unpack the package
+  # Disk usage before unpacking the package
   DU1=`grep "^KB: " $log | head -n1 | cut -f1 | sed -e 's/KB: //'`
   DU1MB=`echo "scale=2; $DU1 / 1024" | bc`
-  # Disk usage before delete sources and build dirs
+  # Disk usage before deleting the source and build dirs
   DU2=`grep "^KB: " $log | tail -n1 | cut -f1 | sed -e 's/KB: //'`
   DU2MB=`echo "scale=2; $DU2 / 1024" | bc`
   # Calculate disk space required to do the build
@@ -108,8 +108,8 @@ for log in $BUILDLOGS ; do
   echo -e "Approximate SBU time is:\t$SBU" >> "$REPORT"
 
   # Dump disk usage values
-  echo -e "\nDisk usage before unpack the package:\t\t\t$DU1 KB or $DU1MB MB" >> "$REPORT"
+  echo -e "\nDisk usage before unpacking the package:\t\t\t$DU1 KB or $DU1MB MB" >> "$REPORT"
-  echo -e "Disk usage before delete sources and build dirs:\t$DU2 KB or $DU2MB MB" >> "$REPORT"
+  echo -e "Disk usage before deleting the source and build dirs:\t$DU2 KB or $DU2MB MB" >> "$REPORT"
   echo -e "Required space to build the package:\t\t\t$REQUIRED1 KB or $REQUIRED2 MB\n" >> "$REPORT"
 
 done

common/func_compare.sh

@@ -108,6 +108,7 @@ wrt_logs() {             #
 (
     cat << EOF
 	@pushd logs 1> /dev/null && \\
+	rm $ITERATION && \\
 	mkdir $ITERATION && \\
 	mv ${LOGS} $ITERATION && \\
 	popd 1> /dev/null

common/func_validate_configs.sh

@@ -255,5 +255,5 @@ inline_doc
     esac
   done
   set -e
-  echo "$tab_***${BOLD}${GREEN} ${PARAM_GROUP%%_*T} config parameters look good${OFF} ***"
+  echo "${nl_}***${BOLD}${GREEN} ${PARAM_GROUP%%_*T} config parameters look good${OFF} ***${nl_}"
 }

common/urls.xsl

@@ -22,7 +22,7 @@
     <xsl:if test="(ancestor::varlistentry[@condition=$model]
                   or not(ancestor::varlistentry[@condition])) and
                   (contains(@url, '.tar.') or contains(@url, '.tgz')
-                  or contains(@url, '.patch') or contains(@url, '.rules')) and
+                  or contains(@url, '.patch')) and
                   not(ancestor-or-self::*/@condition = 'pdf')">
       <!-- Extract the package name -->
       <xsl:variable name="package">

master.sh

@@ -71,7 +71,7 @@ source $MODULE
 [[ $? > 0 ]] && echo "$MODULE did not load.." && exit 2
 [[ $VERBOSITY > 0 ]] && echo "OK"
 #
-[[ $VERBOSITY > 0 ]] && echo "---------------${nl_}"
+[[ $VERBOSITY > 0 ]] && echo "${SD_BORDER}${nl_}"
 
 
 #===========================================================
@@ -92,7 +92,7 @@ source $COMMON_DIR/func_check_version.sh
 source $COMMON_DIR/func_validate_configs.sh
 [[ $? > 0 ]] && echo " function module did not load.." && exit 2
 [[ $VERBOSITY > 0 ]] && echo "OK"
-[[ $VERBOSITY > 0 ]] && echo "---------------${nl_}"
+[[ $VERBOSITY > 0 ]] && echo "${SD_BORDER}${nl_}"
 
 
 ###################################
@@ -432,10 +432,10 @@ check_version "3.0"   "$BASH_VERSION"      "BASH"
 check_version "3.0"   "`gcc -dumpversion`" "GCC"
 tarVer=`tar --version`
 check_version "1.15.0" "${tarVer##* }"      "TAR"
-echo "---------------${nl_}"
+echo "${SD_BORDER}${nl_}"
 
 validate_config
-echo "---------------${nl_}"
+echo "${SD_BORDER}${nl_}"
 
 echo -n "Are you happy with these settings? yes/no (no): "
 read ANSWER
@@ -443,7 +443,7 @@ if [ x$ANSWER != "xyes" ] ; then
   echo "${nl_}Fix the configuration options and rerun the script.${nl_}"
   exit 1
 fi
-echo "${nl_}---------------${nl_}"
+echo "${nl_}${SD_BORDER}${nl_}"
 
 # Prevents setting "-d /" by mistake.
 
@@ -495,10 +495,10 @@ fi
 >$LOGDIR/$LOG
 
 get_book
-echo "---------------${nl_}"
+echo "${SD_BORDER}${nl_}"
 
 build_Makefile
-echo "---------------${nl_}"
+echo "${SD_BORDER}${nl_}"
 
 run_make