From 1b65a847a59aa1f09893b15ddffe16612427e54f Mon Sep 17 00:00:00 2001 From: Manuel Canales Esparcia Date: Thu, 11 May 2006 20:21:14 +0000 Subject: [PATCH] Merged optimize code from experimental branch. --- CLFS/master.sh | 7 +++ HLFS/master.sh | 8 ++- LFS/master.sh | 2 + common/common-functions | 8 ++- common/config | 8 +++ common/func_validate_configs.sh | 9 +-- master.sh | 36 ++++++++++++ optimize/opt_config | 51 +++++++++++++++++ optimize/opt_config.d/O3pipe | 22 ++++++++ optimize/opt_config.d/O3pipe_march | 25 ++++++++ optimize/opt_config.d/defOpt_fPIC | 36 ++++++++++++ optimize/opt_config.d/noOpt | 27 +++++++++ optimize/opt_config.d/noSymbols | 23 ++++++++ optimize/opt_override | 5 ++ optimize/optimize_functions | 91 ++++++++++++++++++++++++++++++ 15 files changed, 350 insertions(+), 8 deletions(-) create mode 100644 optimize/opt_config create mode 100644 optimize/opt_config.d/O3pipe create mode 100644 optimize/opt_config.d/O3pipe_march create mode 100644 optimize/opt_config.d/defOpt_fPIC create mode 100644 optimize/opt_config.d/noOpt create mode 100644 optimize/opt_config.d/noSymbols create mode 100644 optimize/opt_override create mode 100644 optimize/optimize_functions diff --git a/CLFS/master.sh b/CLFS/master.sh index 6bdc38b..8d535ef 100755 --- a/CLFS/master.sh +++ b/CLFS/master.sh @@ -184,6 +184,7 @@ temptools_Makefiles() { # # Insert instructions for unpacking the package and to set the PKGDIR variable. # [[ "$vrs" != "" ]] && wrt_unpack "$name-$vrs.tar.*" + [[ "$vrs" != "" ]] && [[ "$OPTIMIZE" = "2" ]] && wrt_optimize "$name" && wrt_makeflags "$name" # wrt_run_as_su "${this_script}" "${file}" # @@ -255,6 +256,7 @@ boot_Makefiles() { # # Insert instructions for unpacking the package and changing directories # [[ "$vrs" != "" ]] && wrt_unpack "$name-$vrs.tar.*" + [[ "$vrs" != "" ]] && [[ "$OPTIMIZE" = "2" ]] && wrt_optimize "$name" && wrt_makeflags "$name" # # Select a script execution method case $this_script in @@ -325,6 +327,7 @@ chroot_Makefiles() { # *util-linux) wrt_unpack "$name-$vrs.tar.*" ;; *) wrt_unpack2 "$name-$vrs.tar.*" ;; esac + [[ "$OPTIMIZE" = "2" ]] && wrt_optimize "$name" && wrt_makeflags "$name" fi # # Select a script execution method @@ -385,6 +388,7 @@ testsuite_tools_Makefiles() { # tcl) wrt_unpack2 "$name$vrs-src.tar.*" ;; *) wrt_unpack2 "$name-$vrs.tar.*" ;; esac + [[ "$OPTIMIZE" = "2" ]] && wrt_optimize "$name" && wrt_makeflags "$name" # wrt_run_as_chroot1 "${this_script}" "${file}" # @@ -439,6 +443,7 @@ bm_testsuite_tools_Makefiles() { # tcl) wrt_unpack3 "$name$vrs-src.tar.*" ;; *) wrt_unpack3 "$name-$vrs.tar.*" ;; esac + [[ "$OPTIMIZE" = "2" ]] && wrt_optimize "$name" && wrt_makeflags "$name" # wrt_run_as_root2 "${this_script}" "${file}" # @@ -537,6 +542,7 @@ final_system_Makefiles() { # if [ "$vrs" != "" ] ; then FILE="$name-$vrs.tar.*" wrt_unpack2 "$FILE" + [[ "$OPTIMIZE" != "0" ]] && wrt_optimize "$name" && wrt_makeflags "$name" fi # wrt_run_as_chroot1 "${this_script}" "${file}" @@ -639,6 +645,7 @@ bm_final_system_Makefiles() { # if [ "$vrs" != "" ] ; then FILE="$name-$vrs.tar.*" wrt_unpack3 "$FILE" + [[ "$OPTIMIZE" != "0" ]] && wrt_optimize "$name" && wrt_makeflags "$name" fi # wrt_run_as_root2 "${this_script}" "${file}" diff --git a/HLFS/master.sh b/HLFS/master.sh index 03e3550..0b935ad 100755 --- a/HLFS/master.sh +++ b/HLFS/master.sh @@ -140,9 +140,10 @@ chapter5_Makefiles() { # Bootstrap or temptools phase gcc) FILE="gcc-core-$vrs.tar.*" ;; *) FILE="$name-$vrs.tar.*" ;; esac - # Insert instructions for unpacking the package and to set the PKGDIR variable. - wrt_unpack "$FILE" - fi + # Insert instructions for unpacking the package and to set the PKGDIR variable. + wrt_unpack "$FILE" + [[ "$OPTIMIZE" = "2" ]] && wrt_optimize "$name" && wrt_makeflags "$name" + fi case $this_script in *binutils* ) # Dump the path to sources directory for later removal @@ -308,6 +309,7 @@ chapter6_Makefiles() { # sysroot or chroot build phase esac wrt_unpack2 "$FILE" wrt_target_vars + [[ "$OPTIMIZE" != "0" ]] && wrt_optimize "$name" && wrt_makeflags "$name" fi case $this_script in diff --git a/LFS/master.sh b/LFS/master.sh index 7fb6966..b0a88a7 100755 --- a/LFS/master.sh +++ b/LFS/master.sh @@ -112,6 +112,7 @@ chapter5_Makefiles() { # Insert instructions for unpacking the package and to set the PKGDIR variable. wrt_unpack "$FILE" + [[ "$OPTIMIZE" = "2" ]] && wrt_optimize "$name" && wrt_makeflags "$name" fi # Insert date and disk usage at the top of the log file, the script run @@ -215,6 +216,7 @@ chapter6_Makefiles() { if [ "$vrs" != "" ] ; then FILE="$name-$vrs.tar.*" wrt_unpack2 "$FILE" + [[ "$OPTIMIZE" != "0" ]] && wrt_optimize "$name" && wrt_makeflags "$name" fi # In the mount of kernel filesystems we need to set LFS diff --git a/common/common-functions b/common/common-functions index b492da8..b2d8dac 100644 --- a/common/common-functions +++ b/common/common-functions @@ -65,6 +65,13 @@ ${BOLD} -G, --get-packages${OFF} in the configuration file has the proper packages and patches for the book version being processed. +${BOLD} -O, --optimize${OFF} + Optimize [0-2] + 0 = no optimization + 1 = optimize chapter06 only + 2 = optimize both chapter05 and chapter06 + Edit common/opt_config{,.d/*} and common/opt_override as desired. + ${BOLD} -T, --testsuites N ${OFF} Run test suites [0-3] 0 = none @@ -267,7 +274,6 @@ EOF } - #----------------------------------# wrt_target_vars() { # Target vars for hlfs (cross-build method) #----------------------------------# diff --git a/common/config b/common/config index 45c83b4..48a35df 100644 --- a/common/config +++ b/common/config @@ -34,6 +34,14 @@ SERVER=ftp://ftp.lfs-matrix.net # In BLFS the Makefile can't be run automatically RUNMAKE=0 +#--- Optimize the build [0-2] +# 0 = no optimization +# 1 = optimize final system only +# 2 = optimize both temporary tools and final system +# +# Optimization values are set in optimize/* files +OPTIMIZE=0 + #--- Run test suites [0-3] # 0 = none # 1 = only chapter06 Glibc, GCC and Binutils testsuites diff --git a/common/func_validate_configs.sh b/common/func_validate_configs.sh index 70373cc..baeeca6 100644 --- a/common/func_validate_configs.sh +++ b/common/func_validate_configs.sh @@ -84,10 +84,10 @@ validate_config() { # Are the config values sane (within reason) inline_doc # First internal variables, then the ones that change the book's flavour, and lastly system configuration variables - local -r blfs_PARAM_LIST="BOOK BUILDDIR SRC_ARCHIVE GETPKG DEPEND TEST" - local -r hlfs_PARAM_LIST="BOOK BUILDDIR SRC_ARCHIVE GETPKG RUNMAKE MODEL GRSECURITY_HOST TEST REPORT COMPARE RUN_ICA RUN_FARCE ITERATIONS STRIP FSTAB CONFIG KEYMAP PAGE TIMEZONE LANG LC_ALL" - local -r clfs_PARAM_LIST="BOOK BUILDDIR SRC_ARCHIVE GETPKG RUNMAKE METHOD ARCH TARGET TEST REPORT COMPARE RUN_ICA RUN_FARCE ITERATIONS STRIP FSTAB BOOT_CONFIG CONFIG KEYMAP VIMLANG PAGE TIMEZONE LANG" - local -r lfs_PARAM_LIST="BOOK BUILDDIR SRC_ARCHIVE GETPKG RUNMAKE TEST REPORT COMPARE RUN_ICA RUN_FARCE ITERATIONS STRIP FSTAB CONFIG VIMLANG PAGE TIMEZONE LANG" + local -r blfs_PARAM_LIST="BOOK BUILDDIR SRC_ARCHIVE GETPKG DEPEND TEST OPTIMIZE" + local -r hlfs_PARAM_LIST="BOOK BUILDDIR SRC_ARCHIVE GETPKG RUNMAKE MODEL GRSECURITY_HOST TEST OPTIMIZE REPORT COMPARE RUN_ICA RUN_FARCE ITERATIONS STRIP FSTAB CONFIG KEYMAP PAGE TIMEZONE LANG LC_ALL" + local -r clfs_PARAM_LIST="BOOK BUILDDIR SRC_ARCHIVE GETPKG RUNMAKE METHOD ARCH TARGET TEST OPTIMIZE REPORT COMPARE RUN_ICA RUN_FARCE ITERATIONS STRIP FSTAB BOOT_CONFIG CONFIG KEYMAP VIMLANG PAGE TIMEZONE LANG" + local -r lfs_PARAM_LIST="BOOK BUILDDIR SRC_ARCHIVE GETPKG RUNMAKE TEST OPTIMIZE REPORT COMPARE RUN_ICA RUN_FARCE ITERATIONS STRIP FSTAB CONFIG VIMLANG PAGE TIMEZONE LANG" local -r ERROR_MSG_pt1='The variable \"${L_arrow}${config_param}${R_arrow}\" value ${L_arrow}${BOLD}${!config_param}${R_arrow} is invalid,' local -r ERROR_MSG_pt2=' check the config file ${BOLD}${GREEN}\<$(echo $PROGNAME | tr [a-z] [A-Z])/config\> or \${OFF}' @@ -205,6 +205,7 @@ inline_doc RUN_FARCE) [[ "$COMPARE" = "1" ]] && validate_against_str "x0x x1x" ;; ITERATIONS) [[ "$COMPARE" = "1" ]] && validate_against_str "x2x x3x x4x x5x" ;; TEST) validate_against_str "x0x x1x x2x x3x" ;; + OPTIMIZE) validate_against_str "x0x x1x x2x" ;; STRIP) validate_against_str "x0x x1x" ;; VIMLANG) validate_against_str "x0x x1x" ;; DEPEND) validate_against_str "x0x x1x x2x" ;; diff --git a/master.sh b/master.sh index f5bc87f..78c4205 100755 --- a/master.sh +++ b/master.sh @@ -147,6 +147,21 @@ while test $# -gt 0 ; do --help | -h ) usage | more && exit ;; + --optimize | -O ) + test $# = 1 && eval "$exit_missing_arg" + shift + case $1 in + 0 | 1 | 2 ) + OPTIMIZE=$1 + ;; + * ) + echo -e "\n$1 isn't a valid optimize level value." + echo -e "You must use 0, 1, or 2.\n" + exit 1 + ;; + esac + ;; + --testsuites | -T ) test $# = 1 && eval "$exit_missing_arg" shift @@ -448,6 +463,26 @@ if [[ "$COMPARE" = "1" ]]; then [[ $? > 0 ]] && echo "$COMMON_DIR/func_compare.sh did not load.." && exit [[ $VERBOSITY > 0 ]] && echo "OK" fi +# +# optimize module +if [[ "$OPTIMIZE" != "0" ]]; then + [[ $VERBOSITY > 0 ]] && echo -n "Loading optimization module..." + source optimize/optimize_functions + [[ $? > 0 ]] && echo " optimize/optimize_functions did not load.." && exit + [[ $VERBOSITY > 0 ]] && echo "OK" +fi +# +# optimize configurations +if [[ "$OPTIMIZE" != "0" ]]; then + [[ $VERBOSITY > 0 ]] && echo -n "Loading optimization config..." + source optimize/opt_config + [[ $? > 0 ]] && echo " optimize/opt_config did not load.." && exit + [[ $VERBOSITY > 0 ]] && echo "OK" +fi +# + +# Validate optimize settings, if required +[[ "$OPTIMIZE" != "0" ]] && validate_opt_settings # Prevents setting "-d /" by mistake. @@ -472,6 +507,7 @@ fi if [[ "$PWD" != "$JHALFSDIR" ]]; then cp $COMMON_DIR/makefile-functions $JHALFSDIR/ + [[ "$OPTIMIZE" != "0" ]] && cp optimize/opt_override $JHALFSDIR/ if [[ "$COMPARE" != "0" ]] ; then mkdir -p $JHALFSDIR/extras cp extras/* $JHALFSDIR/extras diff --git a/optimize/opt_config b/optimize/opt_config new file mode 100644 index 0000000..e9a9d5e --- /dev/null +++ b/optimize/opt_config @@ -0,0 +1,51 @@ +##### +# +# optimization configuration file +# +##### +# +#$Id$ +# + +#--- Extra flags passed to the packages make commands +# "unset" will cause the variable to be not set +# instead of just set to null. +MAKEFLAGS="-j3" + +#--- List of packages that don't support well MAKEFLAGS. +# This list may be different for you. +BLACK_LIST="dejagnu gettext groff" + +#--- Default optimization mode +# This mode is overridden by definitions in opt_override; +# in this way, packages can be tuned independently. For example, +# if you have trouble building a package in the mode set here, add +# the package to opt_override with a different mode. +# +# Not set here combined modes of form defOpt_myMode. +# Combined modes of form modeA_modeB can be set here. +DEF_OPT_MODE=O3pipe + + +#--- Active optimization variables +# Variables listed here will be set as defined in the appropriate +# file in opt_config.d/; others will be ignored. +ACTIVE_OPT_VARS="CFLAGS CXXFLAGS LDFLAGS \ + OTHER_CFLAGS OTHER_CXXFLAGS OTHER_LDFLAGS" + +#--- Load optimization modes +# To add modes, create a working template.. +# cp opt_config.d/O3pipe opt_config.d/newMode +# ..and edit it as desired. +# +# To inherit another mode and add to it, see opt_config.d/O3pipe_march +# or opt_config.d/defOpt_fPIC as examples for two different ways of +# doing it. Given opt_config.d/modeA_modeB_modeC, modeB inherits +# the optimizations of modeA by sourcing it, and modeC inherits those +# of modeB (which includes those of modeA). +# +# Add packages that are to use this mode to opt_override with: +# echo 'myPkg newMode' >> opt_override +for mode in optimize/opt_config.d/* ; do + source $mode +done diff --git a/optimize/opt_config.d/O3pipe b/optimize/opt_config.d/O3pipe new file mode 100644 index 0000000..517f39f --- /dev/null +++ b/optimize/opt_config.d/O3pipe @@ -0,0 +1,22 @@ +# +#$Id$ +# +##### +# +# MODE O3pipe configuration +# +# Usage: - Apply optimization string to each variable +# - "unset" will cause the variable to be not set +# instead of just set to null. +# +# Use this as an example for cunstomized modes +# +##### + +CFLAGS_O3pipe="-O3 -pipe" +CXXFLAGS_O3pipe=$CFLAGS_O3pipe +LDFLAGS_O3pipe="unset" + +OTHER_CFLAGS_O3pipe=$CFLAGS_O3pipe +OTHER_CXXFLAGS_O3pipe=$CXXFLAGS_O3pipe +OTHER_LDFLAGS_O3pipe=$LDFLAGS_O3pipe diff --git a/optimize/opt_config.d/O3pipe_march b/optimize/opt_config.d/O3pipe_march new file mode 100644 index 0000000..7f0f50e --- /dev/null +++ b/optimize/opt_config.d/O3pipe_march @@ -0,0 +1,25 @@ +# +#$Id$ +# +##### +# +# MODE O3pipe_march configuration +# +# Usage: - Apply optimization string to each variable +# - "unset" will cause the variable to be not set +# instead of just set to null. +# +# Use this as an example for modeA_modeB combined modes +# +##### + +# Inherit optimizations from O3pipe +source optimize/opt_config.d/O3pipe + +CFLAGS_O3pipe_march=$CFLAGS_O3pipe" -march=pentium4" +CXXFLAGS_O3pipe_march=$CFLAGS_O3pipe" -march=pentium4" +LDFLAGS_O3pipe_march=$LDFLAGS_O3pipe + +OTHER_CFLAGS_O3pipe_march=$CFLAGS_O3pipe" -march=pentium4" +OTHER_CXXFLAGS_O3pipe_march=$CXXFLAGS_O3pipe" -march=pentium4" +OTHER_LDFLAGS_O3pipe_march=$LDFLAGS_O3pipe diff --git a/optimize/opt_config.d/defOpt_fPIC b/optimize/opt_config.d/defOpt_fPIC new file mode 100644 index 0000000..f114ab8 --- /dev/null +++ b/optimize/opt_config.d/defOpt_fPIC @@ -0,0 +1,36 @@ +# +#$Id$ +# +##### +# +# MODE defOpt_fPIC configuration +# +# WARNING: Do not edit this mode unless you know what you are +# doing. It's required to build Zlib. +# +# Usage: - Apply optimization string to each variable +# - "unset" will cause the variable to be not set +# instead of just set to null. +# +# Use this as an example for defOpt_myMode combined modes +# +##### + +# Inherit optimizations from DEF_OPT_MODE +source optimize/opt_config.d/${DEF_OPT_MODE} + +# Append " -fPIC" to default *C{,XX}FLAGS and retain other defaults +for OPT_VAR in $ACTIVE_OPT_VARS ; do + case $OPT_VAR in + CFLAGS | CXXFLAGS | OTHER_CFLAGS | OTHER_CXXFLAGS) + if [[ "$(eval "echo \$${OPT_VAR}_${DEF_OPT_MODE}")" != "unset" ]] ; then + eval ${OPT_VAR}_defOpt_fPIC=\$${OPT_VAR}_${DEF_OPT_MODE}\" -fPIC\" + else + eval ${OPT_VAR}_defOpt_fPIC="unset" + fi + ;; + *) + eval ${OPT_VAR}_defOpt_fPIC=\$${OPT_VAR}_${DEF_OPT_MODE} + ;; + esac +done diff --git a/optimize/opt_config.d/noOpt b/optimize/opt_config.d/noOpt new file mode 100644 index 0000000..39abdb3 --- /dev/null +++ b/optimize/opt_config.d/noOpt @@ -0,0 +1,27 @@ +# +#$Id$ +# +##### +# +# MODE noOpt configuration +# +# WARNING: Do not edit this mode unless you know what you are +# doing. By default, this is set to disable optimization +# of packages known to have build or runtime failure. +# +# Also, if you what to use MAKEFLAGS but no build +# optimizations, set this mode in DEF_OPT_MODE +# +# Usage: - Apply optimization string to each variable +# - "unset" will cause the variable to be not set +# instead of being just set to null. +# +##### + +CFLAGS_noOpt="unset" +CXXFLAGS_noOpt="unset" +LDFLAGS_noOpt="unset" + +OTHER_CFLAGS_noOpt="unset" +OTHER_CXXFLAGS_noOpt="unset" +OTHER_LDFLAGS_noOpt="unset" diff --git a/optimize/opt_config.d/noSymbols b/optimize/opt_config.d/noSymbols new file mode 100644 index 0000000..7089f02 --- /dev/null +++ b/optimize/opt_config.d/noSymbols @@ -0,0 +1,23 @@ +# +#$Id$ +# +##### +# +# MODE noOpt_noSymbols configuration +# +# Usage: - Apply optimization string to each variable +# - "unset" will cause the variable to be not set +# instead of just set to null. +# +# NOTE: If using this mode in DEF_OPT_MODE (alone or combined with +# other modes) please set STRIP=0 in common/config +# +##### + +CFLAGS_noSymbols="-s" +CXXFLAGS_noSymbols=$CFLAGS_noSymbols +LDFLAGS_noSymbols="-s" + +OTHER_CFLAGS_noSymbols=$CFLAGS_noSymbols +OTHER_CXXFLAGS_noSymbols=$CXXFLAGS_noSymbols +OTHER_LDFLAGS_noSymbols=$LDFLAGS_noSymbols diff --git a/optimize/opt_override b/optimize/opt_override new file mode 100644 index 0000000..05f8637 --- /dev/null +++ b/optimize/opt_override @@ -0,0 +1,5 @@ +binutils noOpt +gcc noOpt +glibc noOpt +grub noOpt +zlib defOpt_fPIC diff --git a/optimize/optimize_functions b/optimize/optimize_functions new file mode 100644 index 0000000..456e752 --- /dev/null +++ b/optimize/optimize_functions @@ -0,0 +1,91 @@ +#!/bin/bash + +# $Id$ + +set +e + + +#----------------------------------# +validate_opt_settings() { # Show optimize setting and wait user agreement +#----------------------------------# + local OPT_VAR optVal + + echo -e "\t\t${RED}${BOLD}WARNING:${OFF}\n" + echo -e "${BOLD}The use of build optimizations may be dangerous.\n" + echo -e "You should know what you are doing and be sure that the" + echo -e "optimization settings listed below are what you want.\n" + echo -e "If there are build issues or the system doesn't work as" + echo -e "expected, please rebuild without optimizations before" + echo -e "asking for support.${OFF}\n" + + echo -e "MAKEFLAGS: ${L_arrow}${BOLD}${MAKEFLAGS}${OFF}${R_arrow}" + [[ "$MAKEFLAGS" = "unset" ]] && echo + [[ "$MAKEFLAGS" != "unset" ]] && \ + echo -e "BLACK_LIST: ${L_arrow}${BOLD}${BLACK_LIST}${OFF}${R_arrow}\n" + + echo -e "DEF_OPT_MODE: ${L_arrow}${BOLD}${DEF_OPT_MODE}${OFF}${R_arrow}\n" + + for OPT_VAR in $ACTIVE_OPT_VARS ; do + eval optVal=\$${OPT_VAR}_${DEF_OPT_MODE} + echo -e "${OPT_VAR}: ${L_arrow}${BOLD}${optVal}${OFF}${R_arrow}" + done + + echo -e "\nOverridden packages:" + cat optimize/opt_override + echo "${nl_}${SD_BORDER}${nl_}" + + echo -n "Are you happy with these optimization settings? yes/no (no): " + read ANSWER + if [ x$ANSWER != "xyes" ] ; then + echo "${nl_}Fix the optimization options and rerun the script.${nl_}" + exit 1 + fi + echo "${nl_}${SD_BORDER}${nl_}" +} + +#----------------------------------# +wrt_optimize() { # Apply pkg specific opt's to build +#----------------------------------# + local pkg=$1 + local optMode optVal OPT_VAR + + optMode=`awk -v pkg="$pkg" '$1 == pkg { print $2 }' $JHALFSDIR/opt_override` + if [[ "$optMode" = "" ]] ; then + optMode=$DEF_OPT_MODE; + fi + + for OPT_VAR in $ACTIVE_OPT_VARS ; do + eval optVal=\$${OPT_VAR}_$optMode + + if [[ "$optVal" != "unset" ]]; then +( +cat << EOF + @echo "export $OPT_VAR=\"$optVal\"" >> envars +EOF +) >> $MKFILE.tmp + else + continue + fi + done +} + +#----------------------------------# +wrt_makeflags() { # Apply MAKEFLAGS to build +#----------------------------------# + local pkg=$1 + local MKF + + if [[ "$BLACK_LIST" =~ "$pkg" ]]; then + MKF=unset + else + MKF=$MAKEFLAGS + fi + + if [[ "$MKF" != "unset" ]]; then +( +cat << EOF + @echo "export MAKEFLAGS=\"$MAKEFLAGS\"" >> envars +EOF +) >> $MKFILE.tmp + fi +}