Merged optimize code from experimental branch.

CLFS/master.sh

@@ -184,6 +184,7 @@ temptools_Makefiles() {       #
     # Insert instructions for unpacking the package and to set the PKGDIR variable.
     #
     [[ "$vrs" != "" ]] && wrt_unpack "$name-$vrs.tar.*"
+    [[ "$vrs" != "" ]] && [[ "$OPTIMIZE" = "2" ]] &&  wrt_optimize "$name" && wrt_makeflags "$name"
     #
     wrt_run_as_su "${this_script}" "${file}"
     #
@@ -255,6 +256,7 @@ boot_Makefiles() {            #
     # Insert instructions for unpacking the package and changing directories
     #
     [[ "$vrs" != "" ]] && wrt_unpack "$name-$vrs.tar.*"
+    [[ "$vrs" != "" ]] && [[ "$OPTIMIZE" = "2" ]] &&  wrt_optimize "$name" && wrt_makeflags "$name"
     #
     # Select a script execution method
     case $this_script in
@@ -325,6 +327,7 @@ chroot_Makefiles() {          #
         *util-linux)    wrt_unpack  "$name-$vrs.tar.*"  ;;
         *)              wrt_unpack2 "$name-$vrs.tar.*"  ;;
       esac
+      [[ "$OPTIMIZE" = "2" ]] &&  wrt_optimize "$name" && wrt_makeflags "$name"
     fi
     #
     # Select a script execution method
@@ -385,6 +388,7 @@ testsuite_tools_Makefiles() { #
       tcl)    wrt_unpack2 "$name$vrs-src.tar.*" ;;
       *)      wrt_unpack2 "$name-$vrs.tar.*"    ;;
     esac
+    [[ "$OPTIMIZE" = "2" ]] &&  wrt_optimize "$name" && wrt_makeflags "$name"
     #
     wrt_run_as_chroot1 "${this_script}" "${file}"
     #
@@ -439,6 +443,7 @@ bm_testsuite_tools_Makefiles() { #
       tcl)    wrt_unpack3 "$name$vrs-src.tar.*" ;;
       *)      wrt_unpack3 "$name-$vrs.tar.*"    ;;
     esac
+    [[ "$OPTIMIZE" = "2" ]] &&  wrt_optimize "$name" && wrt_makeflags "$name"
     #
     wrt_run_as_root2 "${this_script}" "${file}"
     #
@@ -537,6 +542,7 @@ final_system_Makefiles() {    #
     if [ "$vrs" != "" ] ; then
       FILE="$name-$vrs.tar.*"
       wrt_unpack2 "$FILE"
+      [[ "$OPTIMIZE" != "0" ]] &&  wrt_optimize "$name" && wrt_makeflags "$name"
     fi
     #
     wrt_run_as_chroot1 "${this_script}" "${file}"
@@ -639,6 +645,7 @@ bm_final_system_Makefiles() { #
     if [ "$vrs" != "" ] ; then
       FILE="$name-$vrs.tar.*"
       wrt_unpack3 "$FILE"
+      [[ "$OPTIMIZE" != "0" ]] &&  wrt_optimize "$name" && wrt_makeflags "$name"
     fi
     #
     wrt_run_as_root2 "${this_script}" "${file}"

HLFS/master.sh

@@ -142,6 +142,7 @@ chapter5_Makefiles() {       # Bootstrap or temptools phase
       esac
       # Insert instructions for unpacking the package and to set the PKGDIR variable.
       wrt_unpack "$FILE"
+      [[ "$OPTIMIZE" = "2" ]] &&  wrt_optimize "$name" && wrt_makeflags "$name"
     fi
 
     case $this_script in
@@ -308,6 +309,7 @@ chapter6_Makefiles() {       # sysroot or chroot build phase
       esac
       wrt_unpack2 "$FILE"
       wrt_target_vars
+      [[ "$OPTIMIZE" != "0" ]] &&  wrt_optimize "$name" && wrt_makeflags "$name"
     fi
 
     case $this_script in

LFS/master.sh

@@ -112,6 +112,7 @@ chapter5_Makefiles() {
 
       # Insert instructions for unpacking the package and to set the PKGDIR variable.
       wrt_unpack "$FILE"
+      [[ "$OPTIMIZE" = "2" ]] &&  wrt_optimize "$name" && wrt_makeflags "$name"
     fi
 
     # Insert date and disk usage at the top of the log file, the script run
@@ -215,6 +216,7 @@ chapter6_Makefiles() {
     if [ "$vrs" != "" ] ; then
       FILE="$name-$vrs.tar.*"
       wrt_unpack2 "$FILE"
+      [[ "$OPTIMIZE" != "0" ]] &&  wrt_optimize "$name" && wrt_makeflags "$name"
     fi
 
     # In the mount of kernel filesystems we need to set LFS

common/common-functions

@@ -65,6 +65,13 @@ ${BOLD}  -G, --get-packages${OFF}
         in the configuration file has the proper packages and patches for the
         book version being processed.
 
+${BOLD}  -O, --optimize${OFF}
+        Optimize [0-2]
+          0 = no optimization
+          1 = optimize chapter06 only
+          2 = optimize both chapter05 and chapter06
+        Edit common/opt_config{,.d/*} and common/opt_override as desired.
+
 ${BOLD}  -T, --testsuites N ${OFF}
         Run test suites  [0-3]
           0 = none
@@ -267,7 +274,6 @@ EOF
 }
 
 
-
 #----------------------------------#
 wrt_target_vars() {                # Target vars for hlfs (cross-build method)
 #----------------------------------#

common/config

@@ -34,6 +34,14 @@ SERVER=ftp://ftp.lfs-matrix.net
 #    In BLFS the Makefile can't be run automatically
 RUNMAKE=0
 
+#--- Optimize the build  [0-2]
+#  0 = no optimization
+#  1 = optimize final system only
+#  2 = optimize both temporary tools and final system
+#
+# Optimization values are set in optimize/* files
+OPTIMIZE=0
+
 #--- Run test suites  [0-3]
 #  0 = none
 #  1 = only chapter06 Glibc, GCC and Binutils testsuites

common/func_validate_configs.sh

@@ -84,10 +84,10 @@ validate_config() {          # Are the config values sane (within reason)
 inline_doc
 
   # First internal variables, then the ones that change the book's flavour, and lastly system configuration variables
-  local -r blfs_PARAM_LIST="BOOK BUILDDIR SRC_ARCHIVE GETPKG         DEPEND                TEST"
+  local -r blfs_PARAM_LIST="BOOK BUILDDIR SRC_ARCHIVE GETPKG         DEPEND                TEST OPTIMIZE"
-  local -r hlfs_PARAM_LIST="BOOK BUILDDIR SRC_ARCHIVE GETPKG RUNMAKE MODEL GRSECURITY_HOST TEST REPORT COMPARE RUN_ICA RUN_FARCE ITERATIONS STRIP FSTAB             CONFIG KEYMAP         PAGE TIMEZONE LANG LC_ALL"
+  local -r hlfs_PARAM_LIST="BOOK BUILDDIR SRC_ARCHIVE GETPKG RUNMAKE MODEL GRSECURITY_HOST TEST OPTIMIZE REPORT COMPARE RUN_ICA RUN_FARCE ITERATIONS STRIP FSTAB             CONFIG KEYMAP         PAGE TIMEZONE LANG LC_ALL"
-  local -r clfs_PARAM_LIST="BOOK BUILDDIR SRC_ARCHIVE GETPKG RUNMAKE METHOD  ARCH  TARGET  TEST REPORT COMPARE RUN_ICA RUN_FARCE ITERATIONS STRIP FSTAB BOOT_CONFIG CONFIG KEYMAP VIMLANG PAGE TIMEZONE LANG"
+  local -r clfs_PARAM_LIST="BOOK BUILDDIR SRC_ARCHIVE GETPKG RUNMAKE METHOD  ARCH  TARGET  TEST OPTIMIZE REPORT COMPARE RUN_ICA RUN_FARCE ITERATIONS STRIP FSTAB BOOT_CONFIG CONFIG KEYMAP VIMLANG PAGE TIMEZONE LANG"
-  local -r  lfs_PARAM_LIST="BOOK BUILDDIR SRC_ARCHIVE GETPKG RUNMAKE                       TEST REPORT COMPARE RUN_ICA RUN_FARCE ITERATIONS STRIP FSTAB             CONFIG        VIMLANG PAGE TIMEZONE LANG"
+  local -r  lfs_PARAM_LIST="BOOK BUILDDIR SRC_ARCHIVE GETPKG RUNMAKE                       TEST OPTIMIZE REPORT COMPARE RUN_ICA RUN_FARCE ITERATIONS STRIP FSTAB             CONFIG        VIMLANG PAGE TIMEZONE LANG"
 
   local -r ERROR_MSG_pt1='The variable \"${L_arrow}${config_param}${R_arrow}\" value ${L_arrow}${BOLD}${!config_param}${R_arrow} is invalid,'
   local -r ERROR_MSG_pt2=' check the config file ${BOLD}${GREEN}\<$(echo $PROGNAME | tr [a-z] [A-Z])/config\> or \<common/config\>${OFF}'
@@ -205,6 +205,7 @@ inline_doc
       RUN_FARCE)  [[ "$COMPARE" = "1" ]] && validate_against_str "x0x x1x" ;;
       ITERATIONS) [[ "$COMPARE" = "1" ]] && validate_against_str "x2x x3x x4x x5x" ;;
       TEST)       validate_against_str "x0x x1x x2x x3x" ;;
+      OPTIMIZE)   validate_against_str "x0x x1x x2x" ;;
       STRIP)      validate_against_str "x0x x1x" ;;
       VIMLANG)    validate_against_str "x0x x1x" ;;
       DEPEND)     validate_against_str "x0x x1x x2x" ;;

master.sh

@@ -147,6 +147,21 @@ while test $# -gt 0 ; do
 
     --help | -h )  usage | more && exit  ;;
 
+    --optimize | -O )
+      test $# = 1 && eval "$exit_missing_arg"
+      shift
+      case $1 in
+        0 | 1 | 2 )
+          OPTIMIZE=$1
+          ;;
+        * )
+          echo -e "\n$1 isn't a valid optimize level value."
+          echo -e "You must use 0, 1, or 2.\n"
+          exit 1
+          ;;
+      esac
+      ;;
+
     --testsuites | -T )
       test $# = 1 && eval "$exit_missing_arg"
       shift
@@ -448,6 +463,26 @@ if [[ "$COMPARE" = "1" ]]; then
   [[ $? > 0 ]] && echo "$COMMON_DIR/func_compare.sh did not load.." && exit
   [[ $VERBOSITY > 0 ]] && echo "OK"
 fi
+#
+# optimize module
+if [[ "$OPTIMIZE" != "0" ]]; then
+  [[ $VERBOSITY > 0 ]] && echo -n "Loading optimization module..."
+  source optimize/optimize_functions
+  [[ $? > 0 ]] && echo " optimize/optimize_functions did not load.." && exit
+  [[ $VERBOSITY > 0 ]] && echo "OK"
+fi
+#
+# optimize configurations
+if [[ "$OPTIMIZE" != "0" ]]; then
+  [[ $VERBOSITY > 0 ]] && echo -n "Loading optimization config..."
+  source optimize/opt_config
+  [[ $? > 0 ]] && echo " optimize/opt_config did not load.." && exit
+  [[ $VERBOSITY > 0 ]] && echo "OK"
+fi
+#
+
+# Validate optimize settings, if required
+[[ "$OPTIMIZE" != "0" ]] && validate_opt_settings
 
 # Prevents setting "-d /" by mistake.
 
@@ -472,6 +507,7 @@ fi
 
 if [[ "$PWD" != "$JHALFSDIR" ]]; then
   cp $COMMON_DIR/makefile-functions $JHALFSDIR/
+  [[ "$OPTIMIZE" != "0" ]] && cp optimize/opt_override $JHALFSDIR/
   if [[ "$COMPARE" != "0" ]] ; then
     mkdir -p $JHALFSDIR/extras
     cp extras/* $JHALFSDIR/extras

optimize/opt_config

@@ -0,0 +1,51 @@
+#####
+#
+# optimization configuration file
+#
+#####
+#
+#$Id$
+#
+
+#--- Extra flags passed to the packages make commands
+#    "unset" will cause the variable to be not set
+#    instead of just set to null.
+MAKEFLAGS="-j3"
+
+#--- List of packages that don't support well MAKEFLAGS.
+#    This list may be different for you.
+BLACK_LIST="dejagnu gettext groff"
+
+#--- Default optimization mode
+#    This mode is overridden by definitions in opt_override;
+#    in this way, packages can be tuned independently.  For example,
+#    if you have trouble building a package in the mode set here, add
+#    the package to opt_override with a different mode.
+#
+#    Not set here combined modes of form defOpt_myMode.
+#    Combined modes of form modeA_modeB can be set here.
+DEF_OPT_MODE=O3pipe
+
+
+#--- Active optimization variables
+#    Variables listed here will be set as defined in the appropriate
+#    file in opt_config.d/; others will be ignored.
+ACTIVE_OPT_VARS="CFLAGS CXXFLAGS LDFLAGS \
+                 OTHER_CFLAGS OTHER_CXXFLAGS OTHER_LDFLAGS"
+
+#--- Load optimization modes
+#    To add modes, create a working template..
+#      cp opt_config.d/O3pipe opt_config.d/newMode
+#    ..and edit it as desired.
+#
+#    To inherit another mode and add to it, see opt_config.d/O3pipe_march
+#    or opt_config.d/defOpt_fPIC as examples for two different ways of
+#    doing it.  Given opt_config.d/modeA_modeB_modeC, modeB inherits
+#    the optimizations of modeA by sourcing it, and modeC inherits those
+#    of modeB (which includes those of modeA).
+#
+#    Add packages that are to use this mode to opt_override with:
+#      echo 'myPkg  newMode' >> opt_override
+for mode in optimize/opt_config.d/* ; do
+  source $mode
+done

optimize/opt_config.d/O3pipe

@@ -0,0 +1,22 @@
+#
+#$Id$
+#
+#####
+#
+# MODE O3pipe configuration
+#
+# Usage: - Apply optimization string to each variable
+#        - "unset" will cause the variable to be not set
+#          instead of just set to null.
+#
+# Use this as an example for cunstomized modes
+#
+#####
+
+CFLAGS_O3pipe="-O3 -pipe"
+CXXFLAGS_O3pipe=$CFLAGS_O3pipe
+LDFLAGS_O3pipe="unset"
+
+OTHER_CFLAGS_O3pipe=$CFLAGS_O3pipe
+OTHER_CXXFLAGS_O3pipe=$CXXFLAGS_O3pipe
+OTHER_LDFLAGS_O3pipe=$LDFLAGS_O3pipe

optimize/opt_config.d/O3pipe_march

@@ -0,0 +1,25 @@
+#
+#$Id$
+#
+#####
+#
+# MODE O3pipe_march configuration
+#
+# Usage: - Apply optimization string to each variable
+#        - "unset" will cause the variable to be not set
+#          instead of just set to null.
+#
+# Use this as an example for modeA_modeB combined modes
+#
+#####
+
+# Inherit optimizations from O3pipe
+source optimize/opt_config.d/O3pipe
+
+CFLAGS_O3pipe_march=$CFLAGS_O3pipe" -march=pentium4"
+CXXFLAGS_O3pipe_march=$CFLAGS_O3pipe"  -march=pentium4"
+LDFLAGS_O3pipe_march=$LDFLAGS_O3pipe
+
+OTHER_CFLAGS_O3pipe_march=$CFLAGS_O3pipe" -march=pentium4"
+OTHER_CXXFLAGS_O3pipe_march=$CXXFLAGS_O3pipe" -march=pentium4"
+OTHER_LDFLAGS_O3pipe_march=$LDFLAGS_O3pipe

optimize/opt_config.d/defOpt_fPIC

@@ -0,0 +1,36 @@
+#
+#$Id$
+#
+#####
+#
+# MODE defOpt_fPIC configuration
+#
+# WARNING: Do not edit this mode unless you know what you are
+#          doing. It's required to build Zlib.
+#
+# Usage: - Apply optimization string to each variable
+#        - "unset" will cause the variable to be not set
+#          instead of just set to null.
+#
+# Use this as an example for defOpt_myMode combined modes
+#
+#####
+
+# Inherit optimizations from DEF_OPT_MODE
+source optimize/opt_config.d/${DEF_OPT_MODE}
+
+# Append " -fPIC" to default *C{,XX}FLAGS and retain other defaults
+for OPT_VAR in $ACTIVE_OPT_VARS ; do
+  case $OPT_VAR in
+    CFLAGS | CXXFLAGS | OTHER_CFLAGS | OTHER_CXXFLAGS)
+      if [[ "$(eval "echo \$${OPT_VAR}_${DEF_OPT_MODE}")" != "unset" ]] ; then
+        eval ${OPT_VAR}_defOpt_fPIC=\$${OPT_VAR}_${DEF_OPT_MODE}\" -fPIC\"
+      else
+        eval ${OPT_VAR}_defOpt_fPIC="unset"
+      fi
+      ;;
+    *)
+      eval ${OPT_VAR}_defOpt_fPIC=\$${OPT_VAR}_${DEF_OPT_MODE}
+      ;;
+  esac
+done

optimize/opt_config.d/noOpt

@@ -0,0 +1,27 @@
+#
+#$Id$
+#
+#####
+#
+# MODE noOpt configuration
+#
+# WARNING: Do not edit this mode unless you know what you are
+#          doing.  By default, this is set to disable optimization
+#          of packages known to have build or runtime failure.
+#
+#          Also, if you what to use MAKEFLAGS but no build
+#          optimizations, set this mode in DEF_OPT_MODE
+#
+# Usage: - Apply optimization string to each variable
+#        - "unset" will cause the variable to be not set
+#          instead of being just set to null.
+#
+#####
+
+CFLAGS_noOpt="unset"
+CXXFLAGS_noOpt="unset"
+LDFLAGS_noOpt="unset"
+
+OTHER_CFLAGS_noOpt="unset"
+OTHER_CXXFLAGS_noOpt="unset"
+OTHER_LDFLAGS_noOpt="unset"

optimize/opt_config.d/noSymbols

@@ -0,0 +1,23 @@
+#
+#$Id$
+#
+#####
+#
+# MODE noOpt_noSymbols configuration
+#
+# Usage: - Apply optimization string to each variable
+#        - "unset" will cause the variable to be not set
+#          instead of just set to null.
+#
+# NOTE: If using this mode in DEF_OPT_MODE (alone or combined with
+#       other modes) please set STRIP=0 in common/config
+#
+#####
+
+CFLAGS_noSymbols="-s"
+CXXFLAGS_noSymbols=$CFLAGS_noSymbols
+LDFLAGS_noSymbols="-s"
+
+OTHER_CFLAGS_noSymbols=$CFLAGS_noSymbols
+OTHER_CXXFLAGS_noSymbols=$CXXFLAGS_noSymbols
+OTHER_LDFLAGS_noSymbols=$LDFLAGS_noSymbols

optimize/opt_override

@@ -0,0 +1,5 @@
+binutils       noOpt
+gcc            noOpt
+glibc          noOpt
+grub           noOpt
+zlib           defOpt_fPIC

optimize/optimize_functions

@@ -0,0 +1,91 @@
+#!/bin/bash
+
+# $Id$
+
+set +e
+
+
+#----------------------------------#
+validate_opt_settings() {          # Show optimize setting and wait user agreement
+#----------------------------------#
+  local OPT_VAR optVal
+
+  echo -e "\t\t${RED}${BOLD}WARNING:${OFF}\n"
+  echo -e "${BOLD}The use of build optimizations may be dangerous.\n"
+  echo -e "You should know what you are doing and be sure that the"
+  echo -e "optimization settings listed below are what you want.\n"
+  echo -e "If there are build issues or the system doesn't work as"
+  echo -e "expected, please rebuild without optimizations before"
+  echo -e "asking for support.${OFF}\n"
+
+  echo -e "MAKEFLAGS: ${L_arrow}${BOLD}${MAKEFLAGS}${OFF}${R_arrow}"
+  [[ "$MAKEFLAGS" = "unset" ]] && echo
+  [[ "$MAKEFLAGS" != "unset" ]] && \
+  echo -e "BLACK_LIST: ${L_arrow}${BOLD}${BLACK_LIST}${OFF}${R_arrow}\n"
+
+  echo -e "DEF_OPT_MODE: ${L_arrow}${BOLD}${DEF_OPT_MODE}${OFF}${R_arrow}\n"
+
+  for OPT_VAR in $ACTIVE_OPT_VARS ; do
+    eval optVal=\$${OPT_VAR}_${DEF_OPT_MODE}
+    echo -e "${OPT_VAR}: ${L_arrow}${BOLD}${optVal}${OFF}${R_arrow}"
+  done
+
+  echo -e "\nOverridden packages:"
+  cat optimize/opt_override
+  echo "${nl_}${SD_BORDER}${nl_}"
+
+  echo -n "Are you happy with these optimization settings? yes/no (no): "
+  read ANSWER
+  if [ x$ANSWER != "xyes" ] ; then
+    echo "${nl_}Fix the optimization options and rerun the script.${nl_}"
+    exit 1
+  fi
+  echo "${nl_}${SD_BORDER}${nl_}"
+}
+
+#----------------------------------#
+wrt_optimize() {                   # Apply pkg specific opt's to build
+#----------------------------------#
+  local pkg=$1
+  local optMode optVal OPT_VAR
+
+  optMode=`awk -v pkg="$pkg" '$1 == pkg { print $2 }' $JHALFSDIR/opt_override`
+  if [[ "$optMode" = "" ]] ; then
+    optMode=$DEF_OPT_MODE;
+  fi
+
+  for OPT_VAR in $ACTIVE_OPT_VARS ; do
+    eval optVal=\$${OPT_VAR}_$optMode
+
+    if [[ "$optVal" != "unset" ]]; then
+(
+cat << EOF
+	@echo "export $OPT_VAR=\"$optVal\"" >> envars
+EOF
+) >> $MKFILE.tmp
+    else
+      continue
+    fi
+  done
+}
+
+#----------------------------------#
+wrt_makeflags() {                  # Apply MAKEFLAGS to build
+#----------------------------------#
+  local pkg=$1
+  local MKF
+
+  if [[ "$BLACK_LIST" =~ "$pkg" ]]; then
+    MKF=unset
+  else
+    MKF=$MAKEFLAGS
+  fi
+
+  if [[ "$MKF" != "unset" ]]; then
+(
+cat << EOF
+	@echo "export MAKEFLAGS=\"$MAKEFLAGS\"" >> envars
+EOF
+) >> $MKFILE.tmp
+  fi
+}