From 5126ee366339d18f8a3d167c336f29e0c685bf49 Mon Sep 17 00:00:00 2001 From: Pierre Labastie Date: Tue, 7 Mar 2023 22:23:37 +0100 Subject: [PATCH 01/51] porg: fix wrapInstall command --- pkgmngt/packInstall.sh.porg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgmngt/packInstall.sh.porg b/pkgmngt/packInstall.sh.porg index b9d3f32..7aea2c0 100644 --- a/pkgmngt/packInstall.sh.porg +++ b/pkgmngt/packInstall.sh.porg @@ -121,7 +121,7 @@ PACKAGE=${PACKAGE,,} # name is not normalized, several hacks are necessary (now in function # extract_version)... VERSION=$(extract_version $PCKGVRS) -porg -+ -lp ${PACKAGE}-${VERSION} sh -e << PORG_EOF +porg -+ -lp ${PACKAGE}-${VERSION} -- sh -e << PORG_EOF $* PORG_EOF } From 24ad4fd4f4119781b39eace0d0be78250f7de530 Mon Sep 17 00:00:00 2001 From: Pierre Labastie Date: Sat, 18 Mar 2023 22:44:03 +0100 Subject: [PATCH 02/51] Fix outputting too many sh < + the end-root has not been called, except if the preceding "screen" + tag is itself preceded by a containing and to + bootscripts. So close it only if needed --> + test="preceding-sibling::screen[1][@role='root'] and + @role='root' and + not(preceding-sibling::screen[1]/preceding-sibling::para[1]/xref[@linkend='bootscripts'])"> @@ -733,17 +737,21 @@ cd $BOOTUNPACKDIR select="id('bootscripts')//itemizedlist//ulink/@url"/> + the begin-root will not be called. So do it.--> + test="preceding-sibling::screen[1][@role='root'] and + @role='root'"> + the end-root has not been called. So do it, except if it was already a + unit install --> + test="preceding-sibling::screen[1][@role='root'] and + @role='root' and + not(preceding-sibling::screen[1]/preceding-sibling::para[1]/xref[@linkend='systemd-units'])"> @@ -752,9 +760,10 @@ cd $BOOTUNPACKDIR select="id('systemd-units')//itemizedlist//ulink/@url"/> + the begin-root will not be called. So do it. --> + test="preceding-sibling::screen[1][@role='root'] and + @role='root'"> @@ -762,7 +771,7 @@ cd $BOOTUNPACKDIR + the end-root has not been called. --> @@ -770,9 +779,12 @@ cd $BOOTUNPACKDIR popd + the begin-root will not be called. So do it, except if the next + is itself a unit or bootscript install --> + test="following-sibling::screen[1][@role='root'] and + @role='root' and + not(following-sibling::screen[1]/preceding-sibling::para[1]/xref[@linkend='bootscripts' or @linkend='systemd-units'])"> From 6ac0d96300ab74cd7c21d8cd73ed327425d9e07b Mon Sep 17 00:00:00 2001 From: Pierre Labastie Date: Sun, 19 Mar 2023 10:10:24 +0100 Subject: [PATCH 03/51] outputpkgdest: fix the case of several We output text()[1], literal (literally), text(2). This does not work if there are several tags. Change it to outputing preceding-sibling::text()[1], literal for each literal tag, then output text()[last()] (which works also when there are no literal tag, so it removes a choose. --- BLFS/xsl/scripts.xsl | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/BLFS/xsl/scripts.xsl b/BLFS/xsl/scripts.xsl index eb04609..d5d0fa3 100644 --- a/BLFS/xsl/scripts.xsl +++ b/BLFS/xsl/scripts.xsl @@ -930,22 +930,15 @@ echo Size after install: $(sudo du -skx --exclude home $BUILD_DIR) >> $INFOLOG - - - - - - - - - - - - - - - - + + + + + + + + + From 3fa41475157bd8481cc13473509fee252df8e2e6 Mon Sep 17 00:00:00 2001 From: Pierre Labastie Date: Sun, 19 Mar 2023 10:15:18 +0100 Subject: [PATCH 04/51] outputpkgdest: don't mix make and qmake For outputting DESTDIR=, we need to test whether the text contains "make". But when it contains "qmake" (or any different word containing "make" but not exactly "make), we don't want to output anything more than the text. So strengthen the test. --- BLFS/xsl/scripts.xsl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/BLFS/xsl/scripts.xsl b/BLFS/xsl/scripts.xsl index d5d0fa3..78f61ad 100644 --- a/BLFS/xsl/scripts.xsl +++ b/BLFS/xsl/scripts.xsl @@ -944,7 +944,8 @@ echo Size after install: $(sudo du -skx --exclude home $BUILD_DIR) >> $INFOLOG - + From 0a1943912e226fce18244a19dd155e3a66addab8 Mon Sep 17 00:00:00 2001 From: Pierre Labastie Date: Thu, 23 Mar 2023 18:32:24 +0100 Subject: [PATCH 05/51] Implementation of dependency checking 1 Generate a switch for configuration --- BLFS/xsl/gen_config.xsl | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/BLFS/xsl/gen_config.xsl b/BLFS/xsl/gen_config.xsl index f112926..63b3768 100644 --- a/BLFS/xsl/gen_config.xsl +++ b/BLFS/xsl/gen_config.xsl @@ -89,21 +89,42 @@ menu "Build settings" path. config DEL_LA_FILES - bool "Remove libtool .la files after package installation" - default y - help + bool "Remove libtool .la files after package installation" + default y + help This option should be active on any system mixing libtool and meson build systems. ImageMagick .la files are preserved. config STATS - bool "Generate statistics for the requested package(s)" - default n - help + bool "Generate statistics for the requested package(s)" + default n + help If you want timing and memory footprint statistics to be generated for the packages you build (not their dependencies), set this option to y. Due to the book layout, several scripts are not functional in this case. Please review them. + config DEP_CHECK + bool "Check dependencies of the requested package(s)" + default n + depends on WRAP_INSTALL + help + Setting this option does not work if more than one package + is selected. It will do the following: + - Build the dependency tree and generate a build ordered list + disregarding already installed packages + - Generate the scripts for the dependencies not already + installed (as usual) + - Generate a stript that: + + removes all unneeded packages using porg + (at this point the blfs_tools cannot be used anymore, + and your system may be non functional, so use a console + for that, not a graphical environment) + + installs the package + + restores all the previously removed packages + Note that this script may not be the last one, if there are runtime + dependencies + endmenu menu "Build Layout" From dedd502636180eb143caa01a1b6ff6e2955f7e77 Mon Sep 17 00:00:00 2001 From: Pierre Labastie Date: Thu, 23 Mar 2023 18:34:50 +0100 Subject: [PATCH 06/51] Implementation of dependency checking 2 Check the configuration variable in gen_pkg_book --- BLFS/gen_pkg_book.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/BLFS/gen_pkg_book.sh b/BLFS/gen_pkg_book.sh index e6aab30..44877d4 100755 --- a/BLFS/gen_pkg_book.sh +++ b/BLFS/gen_pkg_book.sh @@ -38,6 +38,7 @@ declare WRAP_INSTALL declare PACK_INSTALL declare DEL_LA_FILES declare STATS +declare DEP_CHECK declare SRC_ARCHIVE declare SRC_SUBDIRS declare BUILD_ROOT @@ -69,6 +70,7 @@ parse_configuration() { # PACK_INSTALL=* | \ DEL_LA_FILES=* | \ STATS=* | \ + DEP_CHECK=* | \ LANGUAGE=* | \ SUDO=* | \ SRC_ARCHIVE=* | \ @@ -108,7 +110,7 @@ parse_configuration() { # validate_configuration() { # #--------------------------# local -r dotSTR=".................." - local -r PARAM_LIST="DEP_LEVEL SUDO LANGUAGE MAIL_SERVER WRAP_INSTALL PACK_INSTALL DEL_LA_FILES STATS SRC_ARCHIVE SRC_SUBDIRS BUILD_ROOT BUILD_SUBDIRS KEEP_FILES JOBS CFG_CFLAGS CFG_CXXFLAGS CFG_LDFLAGS" + local -r PARAM_LIST="DEP_LEVEL SUDO LANGUAGE MAIL_SERVER WRAP_INSTALL PACK_INSTALL DEL_LA_FILES STATS DEP_CHECK SRC_ARCHIVE SRC_SUBDIRS BUILD_ROOT BUILD_SUBDIRS KEEP_FILES JOBS CFG_CFLAGS CFG_CXXFLAGS CFG_LDFLAGS" local -r PARAM_VALS='${config_param}${dotSTR:${#config_param}} ${L_arrow}${BOLD}${!config_param}${OFF}${R_arrow}' local config_param local -i index From 42dfc6ef21cbb2c7b5a2b4b60d399e1de0413f77 Mon Sep 17 00:00:00 2001 From: Pierre Labastie Date: Thu, 23 Mar 2023 18:37:20 +0100 Subject: [PATCH 07/51] Implementation of dependency checking 3 Add an XSL stylesheet for listing packages in LFS. Those are not listed anywhere else, but they are considered dependencies of everything, so should never be erased. --- BLFS/gen_pkg_book.sh | 1 + BLFS/xsl/list_lfs.xsl | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 BLFS/xsl/list_lfs.xsl diff --git a/BLFS/gen_pkg_book.sh b/BLFS/gen_pkg_book.sh index 44877d4..dd8f633 100755 --- a/BLFS/gen_pkg_book.sh +++ b/BLFS/gen_pkg_book.sh @@ -28,6 +28,7 @@ declare BookXml="${TOPDIR}/book.xml" declare MakeBook="${TOPDIR}/xsl/make_book.xsl" declare GetVersion="${TOPDIR}/xsl/get_version.xsl" declare MakeScripts="${TOPDIR}/xsl/scripts.xsl" +declare ListLFS="${TOPDIR}/xsl/list_lfs.xsl" declare BookHtml="${TOPDIR}/book-html" declare BLFS_XML="${TOPDIR}/blfs-xml" declare -a TARGET diff --git a/BLFS/xsl/list_lfs.xsl b/BLFS/xsl/list_lfs.xsl new file mode 100644 index 0000000..87fe253 --- /dev/null +++ b/BLFS/xsl/list_lfs.xsl @@ -0,0 +1,20 @@ + + + + + + + + bootscripts + lfs-bootscripts + kernel + + + + + + + + + From 1bf09c15a7bc85a01a996453bc9d11fa36ba1331 Mon Sep 17 00:00:00 2001 From: Pierre Labastie Date: Thu, 23 Mar 2023 18:38:51 +0100 Subject: [PATCH 08/51] Implementation of dependency checking 4 Implement generation of a modified script if dependency checking is requested. --- BLFS/gen_pkg_book.sh | 46 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/BLFS/gen_pkg_book.sh b/BLFS/gen_pkg_book.sh index dd8f633..e953013 100755 --- a/BLFS/gen_pkg_book.sh +++ b/BLFS/gen_pkg_book.sh @@ -297,4 +297,50 @@ xsltproc --xinclude --nonet \ chmod -R +x scripts echo -e "done\n" +if [ -n "$DEP_CHECK" ]; then + if (( ${#TARGET[*]} != 1 )); then + printf "\nWARNING: If dependencies are checked, only one package\n" + printf " shoud be selected. Not generating check code.\n" + exit + fi + + LIST_LFS="$(xsltproc $ListLFS $LFS_FULL)" + LIST_NEEDED="$(echo $FULL_LIST)" + LIST_INSTALLED="$(porg -a | sed 's/-[[:digit:]].*//')" + LIST_UNNEEDED= + for p in $LIST_INSTALLED; do + case " $LIST_LFS " in *" $p "*) continue ;; esac + case " $LIST_NEEDED " in *" $p "*) continue ;; esac + LIST_UNNEEDED="$LIST_UNNEEDED $p" + done + cat >head.tmp <tail.tmp <script.tmp + cat head.tmp script.tmp tail.tmp >scripts/*${TARGET} + rm *.tmp +fi #clean_configuration From 4dfe8f99c42a2041657305493103a2e0da77b688 Mon Sep 17 00:00:00 2001 From: Pierre Labastie Date: Thu, 23 Mar 2023 22:21:26 +0100 Subject: [PATCH 09/51] gen_pkg_book: fixes for dependency checkings Fix a typo in function definition use sudo for porg -r and porgball -e --- BLFS/gen_pkg_book.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/BLFS/gen_pkg_book.sh b/BLFS/gen_pkg_book.sh index e953013..e4dc8bb 100755 --- a/BLFS/gen_pkg_book.sh +++ b/BLFS/gen_pkg_book.sh @@ -321,13 +321,13 @@ set -e VERSIONED_LIST= for p in $LIST_UNNEEDED; do VERSIONED_LIST="\$VERSIONED_LIST \$(porg \$p)" - porg -rb \$p + sudo porg -rb \$p done # Function to restore packages -restore_pack { +restore_pack () { for p in \$VERSIONED_LIST; do - porgball -e -l /var/lib/packages/\${p}.porg.tar.gz + sudo porgball -e -l /var/lib/packages/\${p}.porg.tar.gz done } From 72711ab1cd6138ca05269c32a8c73ab2e25fa25d Mon Sep 17 00:00:00 2001 From: Pierre Labastie Date: Thu, 23 Mar 2023 22:24:06 +0100 Subject: [PATCH 10/51] list_lfs.xsl: fixes for dependency checking Add several packages that shouldn't be removed when doing dependency checking, two of which are actually not in lfs, but needed for script: porg tzdata sudo wget --- BLFS/xsl/list_lfs.xsl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/BLFS/xsl/list_lfs.xsl b/BLFS/xsl/list_lfs.xsl index 87fe253..0ce91c6 100644 --- a/BLFS/xsl/list_lfs.xsl +++ b/BLFS/xsl/list_lfs.xsl @@ -9,6 +9,11 @@ bootscripts lfs-bootscripts kernel + porg + tzdata + + sudo + wget From 78d04f3f791d0ac058492197f1e4dff05ab318c7 Mon Sep 17 00:00:00 2001 From: Pierre Labastie Date: Fri, 21 Apr 2023 10:49:40 +0200 Subject: [PATCH 11/51] Fix "ambiguous redirect" in func_dependencies We use grep -l ^"${otherlink[*]"\$ to find parentNode, since the parentNode is the only file with a link line that matches. Problem is that sometimes the priority line may match too. This makes a parentNode variable containing several lines, and when redirecting to $parentNode, creates those "ambiguous redirect" messages. Fix: use a loop, and only grep on the first line of each file. --- BLFS/libs/func_dependencies | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/BLFS/libs/func_dependencies b/BLFS/libs/func_dependencies index b2c082d..673d9f8 100644 --- a/BLFS/libs/func_dependencies +++ b/BLFS/libs/func_dependencies @@ -522,7 +522,13 @@ while read prio_of_dep build_of_dep id_of_dep; do # The parent has the same link without the last entry. # We do not need otherlink anymore so just destroy the last element unset otherlink[-1] - parentNode=$(grep ^"${otherlink[*]}"\$ -l *) +# We cannot use grep -l, because we need to restrict to the first line, +# since the prio line may match + for f in *.tree; do + if head -n1 $f | grep -q ^"${otherlink[*]}"\$; then + parentNode=$f; break + fi + done return $p2 fi else # not circular: prune tree (but not .dep, since it may happen that From 4ba94ad556320f641b78af0abafe92649df93328 Mon Sep 17 00:00:00 2001 From: Pierre Labastie Date: Wed, 24 May 2023 10:46:14 +0200 Subject: [PATCH 12/51] Update copyright year --- jhalfs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jhalfs b/jhalfs index fc425af..2166769 100755 --- a/jhalfs +++ b/jhalfs @@ -102,7 +102,7 @@ git_commit=$(git log -1 --format=format:"%h %ad") version=" ${BOLD} \"jhalfs\"${OFF} builder tool (development) $git_commit - Copyright (C) 2005-2021, the jhalfs team: + Copyright (C) 2005-2023, the jhalfs team: Jeremy Huntwork George Boudreau Manuel Canales Esparcia From 9eb3dbe1c4a7cab8f9c22edbba911ce4375550a0 Mon Sep 17 00:00:00 2001 From: Pierre Labastie Date: Wed, 30 Aug 2023 20:04:38 +0200 Subject: [PATCH 13/51] Don't build Xorg components that are commented out gen-special.sh uses the list of files and md5 from the cat instructions to generate an xml page for each of the individual components. Until now, there were no commented out component, so gen-special.sh was not prepared to handle comments. Now, it removes lines containing comments. --- BLFS/gen-special.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/BLFS/gen-special.sh b/BLFS/gen-special.sh index adaac4c..4dcd904 100755 --- a/BLFS/gen-special.sh +++ b/BLFS/gen-special.sh @@ -116,7 +116,7 @@ EOF # So we have to read that command too, since it may be assumed # that the preceding package is a dependency of the following, # except the first. - list_cat="$(sed -n '/>cat/,/EOF Date: Wed, 30 Aug 2023 20:20:59 +0200 Subject: [PATCH 14/51] Hide DEL_LA_FILES in internal settings This prevent normal users to change the default while preserving the possibility to set it to "n" for tests or whatever. Fixes ticket #1734 --- Config.in | 15 ++++++++------- common/libs/func_validate_configs.sh | 2 +- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/Config.in b/Config.in index 4bf1091..c44bebb 100644 --- a/Config.in +++ b/Config.in @@ -556,13 +556,6 @@ depends on !BOOK_BLFS bool "Strip Installed Binaries/Libraries" default n - config DEL_LA_FILES - bool "Remove libtool .la files" - default y - help - #-- Remove files libxxx.la installed by libtool. For a rationale - # see https://blog.flameeyes.eu/tags/lafiles/ - config NO_PROGRESS_BAR bool "DO NOT use/display progress_bar" default n @@ -913,6 +906,14 @@ depends on !BOOK_BLFS string "Package contents list" default "unpacked" + config DEL_LA_FILES + bool "Remove libtool .la files" + default y + help + #-- Remove files libxxx.la installed by libtool. Only set to + # "n" if you know what you are doing. For a rationale + # see https://blog.flameeyes.eu/tags/lafiles/ + #--- End Internal Settings endmenu diff --git a/common/libs/func_validate_configs.sh b/common/libs/func_validate_configs.sh index b14cf81..ec21656 100644 --- a/common/libs/func_validate_configs.sh +++ b/common/libs/func_validate_configs.sh @@ -29,7 +29,7 @@ inline_doc local -r LFS_book="$BOOK_common INITSYS BLFS_TOOL" # Build Settings by book - local -r LFS_build="$BUILD_chroot NCURSES5 DEL_LA_FILES $BUILD_common PKGMNGT FULL_LOCALE WRAP_INSTALL" + local -r LFS_build="$BUILD_chroot NCURSES5 $BUILD_common PKGMNGT FULL_LOCALE WRAP_INSTALL" # System Settings by book local -r LFS_system="HOSTNAME INTERFACE IP_ADDR GATEWAY PREFIX BROADCAST DOMAIN DNS1 DNS2 FONT KEYMAP LOCAL LOG_LEVEL" From 45f82b13263570f04ac4ef91b5002035e9b58503 Mon Sep 17 00:00:00 2001 From: Pierre Labastie Date: Sun, 3 Sep 2023 23:35:33 +0200 Subject: [PATCH 15/51] Use /usr for XORG_PREFIX unconditionally: This is possible now that instruction for non /usr XORG_PREFIX have been made role="nodump". --- BLFS/xsl/process-replaceable.xsl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/BLFS/xsl/process-replaceable.xsl b/BLFS/xsl/process-replaceable.xsl index e3e2331..6a9bdd1 100644 --- a/BLFS/xsl/process-replaceable.xsl +++ b/BLFS/xsl/process-replaceable.xsl @@ -123,6 +123,11 @@ $USER + + + /usr + **EDITME From 06b731490f368817ed489800d3409419b2156452 Mon Sep 17 00:00:00 2001 From: Pierre Labastie Date: Tue, 5 Sep 2023 13:14:26 +0200 Subject: [PATCH 16/51] Allow two names for scriptlet containing "userdel" It used to be named "revised_chroot", it is now named "cleanup" Allow both names by testing whether revised something is present. --- common/libs/func_compare.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/common/libs/func_compare.sh b/common/libs/func_compare.sh index fb368b8..2b6f9d1 100644 --- a/common/libs/func_compare.sh +++ b/common/libs/func_compare.sh @@ -22,9 +22,11 @@ wrt_compare_targets() { # local dir printf -v dir chapter%02d "$1" - sed -i '/userdel/d' $dir/*revised* + REVISED=cleanup + if ls $dir/*revised* 2>/dev/null; then REVISED=revised; fi + sed -i '/userdel/d' $dir/*$REVISED* for (( N = 2; N < ITERATIONS; N++ )); do - sed -i '/userdel/d' $dir-build_$N/*revised* + sed -i '/userdel/d' $dir-build_$N/*$REVISED* done } From 3b27f9376431be4add70a47315769c4bf8c2a848 Mon Sep 17 00:00:00 2001 From: Pierre Labastie Date: Fri, 8 Sep 2023 16:50:05 +0200 Subject: [PATCH 17/51] Apply a timeout to kernel config in LFS If the kernel-config file is not up to date (missing options or whatever), the kernel "make" will hang, waiting for input. So run "make oldconfig" explicitly, with a timeout. Fixes: #1736 --- LFS/lfs.xsl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/LFS/lfs.xsl b/LFS/lfs.xsl index c6075d6..782e9ee 100644 --- a/LFS/lfs.xsl +++ b/LFS/lfs.xsl @@ -415,6 +415,8 @@ echo -e "\n\nTotalseconds: $SECONDS\n" make mrproper cp -v ../kernel-config .config + timeout 60 make oldconfig ||\ + { echo kernel config is not up to date; exit 124; } From 1b02a9acff9b511fc0b563ade142600723dc5a97 Mon Sep 17 00:00:00 2001 From: Xi Ruoyao Date: Mon, 11 Sep 2023 14:38:15 +0800 Subject: [PATCH 18/51] Replace $(nproc || echo 1) and $(nproc) --- LFS/lfs.xsl | 18 ++++++++++++++++++ common/chroot.xsl | 9 +++++++++ common/libs/func_book_parser | 12 +++++++++++- 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/LFS/lfs.xsl b/LFS/lfs.xsl index c6075d6..99174fc 100644 --- a/LFS/lfs.xsl +++ b/LFS/lfs.xsl @@ -28,6 +28,10 @@ --> + + + + @@ -615,6 +619,12 @@ unset OLD_PKGDIR + + + + + + **EDITME @@ -1080,6 +1090,14 @@ LOGLEVEL=" + + + + + + + + @@ -55,6 +57,13 @@ + + + + + diff --git a/common/libs/func_book_parser b/common/libs/func_book_parser index 10115b8..5dbd502 100644 --- a/common/libs/func_book_parser +++ b/common/libs/func_book_parser @@ -63,6 +63,13 @@ extract_commands() { # fi popd > /dev/null + JOBS_1=1 + JOBS_2=1 + case "$OPTIMIZE" in + "1") JOBS_2="$N_PARALLEL";; + "2") JOBS_1="$N_PARALLEL"; JOBS_2="$N_PARALLEL";; + esac + # First profile the book, for revision and arch. Note that # MULTIBLIB is set to "default" if pure 64 bit book. In this case, # profiling on arch is useless, but does not hurt either. @@ -102,6 +109,8 @@ extract_commands() { # --stringparam local "$LOCAL" \ --stringparam log-level "$LOG_LEVEL" \ --stringparam script-root "$SCRIPT_ROOT" \ + --stringparam jobs_1 "$JOBS_1" \ + --stringparam jobs_2 "$JOBS_2" \ --output "./${COMMANDS}/" \ $XSL \ prbook.xml >> $LOGDIR/$LOG 2>&1 @@ -141,7 +150,8 @@ create_chroot_scripts() { # if [ ! -z $ARCH ] ; then echo -n " $ARCH" ; fi echo -n "... " xsltproc --nonet --xinclude \ - -o chroot-scripts/ chroot.xsl \ + --stringparam jobs_2 "$JOBS_2" \ + -o chroot-scripts/ chroot.xsl \ $BOOK/chapter0?/*chroot*.xml >> $LOGDIR/$LOG 2>&1 echo "done" From 74601a7f0673b4c60147db2bb4749d04a710734b Mon Sep 17 00:00:00 2001 From: Pierre Labastie Date: Mon, 11 Sep 2023 19:22:13 +0200 Subject: [PATCH 19/51] Separate parallelization from optimization Also, add cpu set choice and the possibility to use nproc --- Config.in | 82 +++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 65 insertions(+), 17 deletions(-) diff --git a/Config.in b/Config.in index 4bf1091..cfe4166 100644 --- a/Config.in +++ b/Config.in @@ -1,3 +1,11 @@ +# Check for "nproc" presence: +config HAVE_NPROC + def_bool $(shell,if nproc &>/dev/null; then echo y; else echo n; fi) + +# Check for "cpuset controller in cgroup v2" presence: +config HAVE_CGROUP + def_bool $(shell,if grep -q cpuset /sys/fs/cgroup/cgroup.controllers 2>/dev/null; then echo y; else echo n; fi) + menu "BOOK Settings" #--- BOOK/script @@ -811,7 +819,7 @@ depends on !BOOK_BLFS #--- Optimizations config CONFIG_OPTIMIZE - bool "Optimization and parallelization" + bool "Optimization" default n help # Opens a menu for various optimization settings: @@ -821,16 +829,9 @@ depends on !BOOK_BLFS # WARNING: The use of build optimizations may lead to build issues. # If the system doesn't work as expected, please rebuild # without optimizations before asking for support. - menu "Parallelization and Optimization settings" + menu "Optimization settings" depends on CONFIG_OPTIMIZE - config N_PARALLEL - int "Number of parallel `make' jobs" - default 1 - help - #-- The usual recommandation is (number of CPU cores)+1 - # Do not set for meaningful SBU calculations. - choice prompt "Optimization level" default OPT_1 @@ -845,14 +846,6 @@ depends on !BOOK_BLFS endchoice - config REALSBU - bool "Build Binutls pass1 without optimization (Real SBU)" - depends on OPT_2 - default n - help - #-- Use -j1 in make invokation for Binutils pass1 to - # get a valid SBU value. - endmenu # Optimization settings config OPTIMIZE int @@ -861,6 +854,61 @@ depends on !BOOK_BLFS default "2" if OPT_2 #--- End Optimizations + #--- Parallelism + menu "Parallelism settings" + if HAVE_NPROC + config ALL_CORES + bool "Use all cores as in the book" + default y + help + Use book instructions as written for parallelism. If you + answer no, then jhalfs will fall back to a static number of + cores, defined below. If you answer y, and your system + supports it, you'll be asked for a cpu set to use. + + if HAVE_CGROUP + config CPUSET + string "set of cpus to use, or 'all' for all cpus" + default "all" + help + See "List format" in cpuset(7). Choosing cpus depend + on the topology of your processors. Sometimes two + hyperthreads on the same core are numbered consecutively. + For example for using all cores and no hyperthreading on + a Haswell, use "0,2,4,6". Other brands may have a different + topology, and may require e.g. "0-3" to use the first 4 cores. + If not sure, keep the default. + endif + if !HAVE_CGROUP + config CPUSET + string + default "all" + endif + + endif # HAVE_NPROC + if !HAVE_NPROC + config ALL_CORES + bool + default n + endif + + config N_PARALLEL + int "Number of parallel `make' jobs" + depends on !ALL_CORES + default 1 + help + #-- The usual recommandation is (number of CPU cores)+1 + # Do not set for meaningful SBU calculations. + + config REALSBU + bool "Build Binutls pass1 without parallelism (Real SBU)" + depends on ALL_CORES != 1 + default n + help + #-- Use -j1 in make invokation for Binutils pass1 to + # get a valid SBU value. + endmenu # parallelism + #-- Internal Settings menu "Internal Settings (WARNING: for jhalfs developers only)" From aceacf2f8596069e6e66e18fb6fa828efa2feecb Mon Sep 17 00:00:00 2001 From: Pierre Labastie Date: Tue, 12 Sep 2023 22:54:51 +0200 Subject: [PATCH 20/51] Modify kernfs scripts to be able to run them twice Those scripts could only be run once, because mount errors out if it tries to mount an already mounted fs (or unmount a non mounted fs). This changes generation of the scripts so that any mount is preceded with "mountpoint -q || ", and any umount is preceded with "mountpoint -q && ". --- common/kernfs.xsl | 62 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/common/kernfs.xsl b/common/kernfs.xsl index f47f665..d7043a0 100644 --- a/common/kernfs.xsl +++ b/common/kernfs.xsl @@ -16,9 +16,69 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + mountpoint -q + + || + + + + + + + + + mountpoint -q + + && + + + + + + + + + + + + + + + + + + + + + From 200cc0151e7be09cd85ad2ae9b06f2f1e619a763 Mon Sep 17 00:00:00 2001 From: Pierre Labastie Date: Tue, 12 Sep 2023 22:58:55 +0200 Subject: [PATCH 21/51] All the targets after mk_SUDO depend on "devices" Now that devices.sh can be run even if the fs are already mounted all the targets after mounting the virtual kernel fs can depend on "devices". This way, if for some reason a partial build is restarted after a reboot, the virtual kernel fs are automatically re-mounted. Fixes #1737 --- LFS/master.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/LFS/master.sh b/LFS/master.sh index ba06e7e..7e4e919 100644 --- a/LFS/master.sh +++ b/LFS/master.sh @@ -368,24 +368,24 @@ mk_SUDO: mk_LUSER @sudo make BREAKPOINT=\$(BREAKPOINT) SUDO @touch \$@ -mk_CHROOT: mk_SUDO +mk_CHROOT: mk_SUDO devices @\$(call echo_CHROOT_request) @( sudo \$(CHROOT1) -c "cd \$(SCRIPT_ROOT) && make BREAKPOINT=\$(BREAKPOINT) CHROOT") @touch \$@ -mk_BOOT: mk_CHROOT +mk_BOOT: mk_CHROOT devices @\$(call echo_CHROOT_request) @( sudo \$(CHROOT1) -c "cd \$(SCRIPT_ROOT) && make BREAKPOINT=\$(BREAKPOINT) BOOT") @touch \$@ -mk_BLFS_TOOL: create-sbu_du-report +mk_BLFS_TOOL: create-sbu_du-report devices @if [ "\$(ADD_BLFS_TOOLS)" = "y" ]; then \\ \$(call sh_echo_PHASE,Building BLFS_TOOL); \\ (sudo \$(CHROOT1) -c "make -C $BLFS_ROOT/work"); \\ fi; @touch \$@ -mk_CUSTOM_TOOLS: mk_BLFS_TOOL +mk_CUSTOM_TOOLS: mk_BLFS_TOOL devices @if [ "\$(ADD_CUSTOM_TOOLS)" = "y" ]; then \\ \$(call sh_echo_PHASE,Building CUSTOM_TOOLS); \\ sudo mkdir -p ${BUILDDIR}${TRACKING_DIR}; \\ From c552eabe636ca27aa33c04c083a27c51c2958ffe Mon Sep 17 00:00:00 2001 From: Pierre Labastie Date: Thu, 14 Sep 2023 16:03:37 +0200 Subject: [PATCH 22/51] Always run "teardown" after chroot If the shell exits with an error, teardown is not run in target chroot, which may lead to unwanted effects, such as preventing unmounting the $BUILD_DIR. Tell make to ignore the error. --- LFS/master.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/LFS/master.sh b/LFS/master.sh index 7e4e919..2aa9843 100644 --- a/LFS/master.sh +++ b/LFS/master.sh @@ -437,11 +437,11 @@ fi cat << EOF chroot1: devices - sudo \$(CHROOT1) + -sudo \$(CHROOT1) \$(MAKE) teardown chroot: devices - sudo \$(CHROOT1) + -sudo \$(CHROOT1) \$(MAKE) teardown SETUP: $SETUP_TGT From 9a89ba79defccb5011b128ab412b90771ab182fa Mon Sep 17 00:00:00 2001 From: Pierre Labastie Date: Sat, 16 Sep 2023 09:39:53 +0200 Subject: [PATCH 23/51] ablfs: new pattern for config recognition chgrp -v mail ... is found in mutt and shouldn't considered an install command. --- BLFS/xsl/process-install.xsl | 1 + 1 file changed, 1 insertion(+) diff --git a/BLFS/xsl/process-install.xsl b/BLFS/xsl/process-install.xsl index 619c72a..1f15028 100644 --- a/BLFS/xsl/process-install.xsl +++ b/BLFS/xsl/process-install.xsl @@ -16,6 +16,7 @@ contains(string($current-instr),'tracker-miner') or contains(string($current-instr),'gtweak') or contains(string($current-instr),'query-immodules') or + contains(string($current-instr),'chgrp -v mail') or contains(string($current-instr),'gnome-control-center')"> ]> From 1c22535b2e2da8b27484cc2b1ec6e375a78bcd38 Mon Sep 17 00:00:00 2001 From: Pierre Labastie Date: Sat, 16 Sep 2023 13:39:39 +0200 Subject: [PATCH 24/51] Process two more tags - one for libreoffice (use the xreflabel to set LO_PREFIX) - one for abiword (installation of normal.awt-ll_CC files) --- BLFS/xsl/process-replaceable.xsl | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/BLFS/xsl/process-replaceable.xsl b/BLFS/xsl/process-replaceable.xsl index 6a9bdd1..ca61b60 100644 --- a/BLFS/xsl/process-replaceable.xsl +++ b/BLFS/xsl/process-replaceable.xsl @@ -128,6 +128,28 @@ ancestor::sect1[@id='xorg-env']"> /usr + + + /opt/ + + + + + + + + en_GB + + + + + _ + + + + **EDITME From 6d9bd8e8e87190ac8f57cd007c8ad34d24504911 Mon Sep 17 00:00:00 2001 From: Pierre Labastie Date: Sun, 17 Sep 2023 09:39:52 +0200 Subject: [PATCH 25/51] Add help about kernel config Now, there is a timeout if the config file is not up to date. Say so in the help. --- Config.in | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Config.in b/Config.in index c44bebb..b420534 100644 --- a/Config.in +++ b/Config.in @@ -604,6 +604,13 @@ menu "System configuration" #-- Fully qualified path to a kernel config file # The config file will be copied to ${BUILD_DIR}/sources # and renamed 'kernel-config' + # + # Important: if the config file is out of date (missing + # option), the kernel build will timeout instead of + # waiting forever for an input. This will generate error + # 124. In this case, update your config file, + # copy it to $BUILD_DIR/sources/kernel-config, and + # restart the build. #--- End Kernel config NCURSES5 From 5ed69f823f28db12a5f56eb709ed8446f5070df0 Mon Sep 17 00:00:00 2001 From: Pierre Labastie Date: Thu, 19 Oct 2023 17:35:25 +0200 Subject: [PATCH 26/51] Dependency on xorg7-* only on last package Our gen-special.sh transforms a dependency on compound packages (xorg7-*) on a dependency on the list of packages. This generates a lot of trials when removing circular dependencies, because if the dependency on xorg7-* is optional, all packages in the list are tried in turn, while it is sure that anyway the dependecy must be eliminated. If the dependency is only on the last package, the circular dep is eliminated at first try. --- BLFS/gen-special.sh | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/BLFS/gen-special.sh b/BLFS/gen-special.sh index 4dcd904..8b61d68 100755 --- a/BLFS/gen-special.sh +++ b/BLFS/gen-special.sh @@ -170,7 +170,25 @@ EOF EOF - cat >> tmpfile << EOF +# cat >> tmpfile << EOF +# +# +# +# +# +# +# +# $packname +# ref +# +#EOF + precpack=$packname + done + cat >>$SPECIAL_FILE << EOF + + +EOF + cat >> tmpfile << EOF @@ -181,13 +199,6 @@ EOF $packname ref -EOF - done - cat >>$SPECIAL_FILE << EOF - - -EOF - cat >> tmpfile << EOF EOF done From 645ec4780e533fec30b56ae6e1fca4f0ba88207d Mon Sep 17 00:00:00 2001 From: Pierre Labastie Date: Wed, 15 Nov 2023 18:04:05 +0100 Subject: [PATCH 27/51] Reorganize advanced options in Config.in - Put parallelization menu first - Fix the logic for variable definitions in this menu - change jhalfs to reflect the separation of optimization and parallelization - validate new variables - remove MAKEFLAGS from the validation of optimize variables - also remove the black list, which has not been used in years. --- Config.in | 182 +++++++++++++-------------- common/libs/func_validate_configs.sh | 6 +- jhalfs | 9 +- optimize/optimize_functions | 5 +- 4 files changed, 100 insertions(+), 102 deletions(-) diff --git a/Config.in b/Config.in index 8bc7161..a0bfc0f 100644 --- a/Config.in +++ b/Config.in @@ -771,6 +771,95 @@ endmenu #--- System configuration menu "Advanced Features" depends on !BOOK_BLFS + #--- Parallelism + menu "Parallelism settings" + if HAVE_NPROC + config ALL_CORES + bool "Use all cores as in new books (say n for old books)" + default y + help + Use book instructions as written for parallelism since version + r12.0-87 included. If you answer no, then jhalfs will fall back + to a static number of cores, defined below. If you answer y, and + your system supports it, you'll be asked for a cpu set to use. + There is no detection of book version. If the book version + is lower than r12.0-87, say no! + + if ALL_CORES && HAVE_CGROUP + config CPUSET + string "set of cpus to use, or 'all' for all cpus" + default "all" + help + See "List format" in cpuset(7). Choosing cpus depend + on the topology of your processors. Sometimes two + hyperthreads on the same core are numbered consecutively. + For example for using all cores and no hyperthreading on + a Haswell, use "0,2,4,6". Other brands may have a different + topology, and may require e.g. "0-3" to use the first 4 cores. + If not sure, keep the default. + endif + + endif # HAVE_NPROC + if !HAVE_NPROC + config ALL_CORES + bool + default n + endif + + config N_PARALLEL + int "Number of parallel `make' jobs" + depends on !ALL_CORES + default 1 + help + #-- The usual recommandation is (number of CPU cores)+1 + # Do not set for meaningful SBU calculations. + + config REALSBU + bool "Build Binutils pass1 without parallelism (Real SBU)" + default n + help + #-- Use -j1 in make invokation for Binutils pass1 to + # get a valid SBU value. + endmenu # parallelism + + #--- Optimizations + config CONFIG_OPTIMIZE + bool "Optimization" + default n + help + # Opens a menu for various optimization settings: + # Actual optimization flags MUST be defined in ./optimize/* + # before activating this option. + # + # WARNING: The use of build optimizations may lead to build issues. + # If the system doesn't work as expected, please rebuild + # without optimizations before asking for support. + menu "Optimization settings" + depends on CONFIG_OPTIMIZE + + choice + prompt "Optimization level" + default OPT_1 + help + #-- Optimization values are set in optimize/* files + + config OPT_1 + bool "Final system only" + + config OPT_2 + bool "Both temp tools and final system" + + endchoice + + endmenu # Optimization settings + config OPTIMIZE + int + default "0" if !CONFIG_OPTIMIZE + default "1" if OPT_1 + default "2" if OPT_2 + + #--- End Optimizations + config REPORT bool "Create SBU and disk usage report" default y @@ -817,99 +906,6 @@ depends on !BOOK_BLFS #--- End ICA - #--- Optimizations - config CONFIG_OPTIMIZE - bool "Optimization" - default n - help - # Opens a menu for various optimization settings: - # Actual optimization flags MUST be defined in ./optimize/* - # before activating this option. - # - # WARNING: The use of build optimizations may lead to build issues. - # If the system doesn't work as expected, please rebuild - # without optimizations before asking for support. - menu "Optimization settings" - depends on CONFIG_OPTIMIZE - - choice - prompt "Optimization level" - default OPT_1 - help - #-- Optimization values are set in optimize/* files - - config OPT_1 - bool "Final system only" - - config OPT_2 - bool "Both temp tools and final system" - - endchoice - - endmenu # Optimization settings - config OPTIMIZE - int - default "0" if !CONFIG_OPTIMIZE - default "1" if OPT_1 - default "2" if OPT_2 - - #--- End Optimizations - #--- Parallelism - menu "Parallelism settings" - if HAVE_NPROC - config ALL_CORES - bool "Use all cores as in the book" - default y - help - Use book instructions as written for parallelism. If you - answer no, then jhalfs will fall back to a static number of - cores, defined below. If you answer y, and your system - supports it, you'll be asked for a cpu set to use. - - if HAVE_CGROUP - config CPUSET - string "set of cpus to use, or 'all' for all cpus" - default "all" - help - See "List format" in cpuset(7). Choosing cpus depend - on the topology of your processors. Sometimes two - hyperthreads on the same core are numbered consecutively. - For example for using all cores and no hyperthreading on - a Haswell, use "0,2,4,6". Other brands may have a different - topology, and may require e.g. "0-3" to use the first 4 cores. - If not sure, keep the default. - endif - if !HAVE_CGROUP - config CPUSET - string - default "all" - endif - - endif # HAVE_NPROC - if !HAVE_NPROC - config ALL_CORES - bool - default n - endif - - config N_PARALLEL - int "Number of parallel `make' jobs" - depends on !ALL_CORES - default 1 - help - #-- The usual recommandation is (number of CPU cores)+1 - # Do not set for meaningful SBU calculations. - - config REALSBU - bool "Build Binutls pass1 without parallelism (Real SBU)" - depends on ALL_CORES != 1 - default n - help - #-- Use -j1 in make invokation for Binutils pass1 to - # get a valid SBU value. - endmenu # parallelism - - #-- Internal Settings menu "Internal Settings (WARNING: for jhalfs developers only)" diff --git a/common/libs/func_validate_configs.sh b/common/libs/func_validate_configs.sh index ec21656..34d07ec 100644 --- a/common/libs/func_validate_configs.sh +++ b/common/libs/func_validate_configs.sh @@ -35,7 +35,7 @@ inline_doc local -r LFS_system="HOSTNAME INTERFACE IP_ADDR GATEWAY PREFIX BROADCAST DOMAIN DNS1 DNS2 FONT KEYMAP LOCAL LOG_LEVEL" # Full list of books settings - local -r lfs_PARAM_LIST="$LFS_book $GENERAL_common $LFS_build $LFS_system $ADVANCED_chroot N_PARALLEL REALSBU SAVE_CH5 $ADVANCED_common" + local -r lfs_PARAM_LIST="$LFS_book $GENERAL_common $LFS_build $LFS_system $ADVANCED_chroot ALL_CORES CPUSET N_PARALLEL REALSBU SAVE_CH5 $ADVANCED_common" # local -r blfs_PARAM_LIST="BRANCH_ID BLFS_ROOT BLFS_XML TRACKING_DIR" # Additional variables @@ -133,8 +133,8 @@ inline_doc RETRYSRCDOWNLOAD) [[ "$GETPKG" = "y" ]] && echo -e "`eval echo $PARAM_VALS`" ;; RETRYDOWNLOADCNT) [[ "$GETPKG" = "y" ]] && echo -e "`eval echo $PARAM_VALS`" ;; DOWNLOADTIMEOUT) [[ "$GETPKG" = "y" ]] && echo -e "`eval echo $PARAM_VALS`" ;; - N_PARALLEL) [[ "$OPTIMIZE" -gt "0" ]] && echo -e "`eval echo $PARAM_VALS`" ;; - REALSBU) [[ "$OPTIMIZE" = "2" ]] && echo -e "`eval echo $PARAM_VALS`" ;; + CPUSET) [[ "$HAVE_CGROUP" = "y" ]] && [[ "$ALL_CORES" = "y" ]] && echo -e "`eval echo $PARAM_VALS`" ;; + N_PARALLEL) [[ "$ALL_CORES" = "n" ]] && echo -e "`eval echo $PARAM_VALS`" ;; # Envars that requires some validation LUSER) echo -e "`eval echo $PARAM_VALS`" diff --git a/jhalfs b/jhalfs index 2166769..fe2c30c 100755 --- a/jhalfs +++ b/jhalfs @@ -175,6 +175,7 @@ SET_MISC=${SET_MISC:=n} SET_BLOWFISH=${SET_BLOWFISH:=n} UNICODE=${UNICODE:=n} LOCAL=${LOCAL:=n} +ALL_CORES=${ALL_CORES:=n} REALSBU=${REALSBU:=n} SAVE_CH5=${SAVE_CH5:=n} @@ -287,12 +288,14 @@ if [[ "$OPTIMIZE" != "0" ]]; then # # optimize configurations load_file optimize/opt_config 'Loading optimization config' - # The number of parallel jobs is taken from configuration now - # shellcheck disable=SC2034 - JH_MAKEFLAGS="-j${N_PARALLEL}" # Validate optimize settings, if required validate_opt_settings fi +# Parallelization is outside optimization, because it is now in the book +if [[ "$ALL_CORES" = "n" ]]; then + # shellcheck disable=SC2034 + JH_MAKEFLAGS="-j${N_PARALLEL}" +fi # if [[ "$REBUILD_MAKEFILE" = "n" ]] ; then diff --git a/optimize/optimize_functions b/optimize/optimize_functions index 26d5043..1a73cd3 100644 --- a/optimize/optimize_functions +++ b/optimize/optimize_functions @@ -16,9 +16,8 @@ validate_opt_settings() { # Show optimize setting and wait user agreeme echo -e "expected, please rebuild without optimizations before" echo -e "asking for support.${OFF}\n" - echo -e "MAKEFLAGS: ${L_arrow}${BOLD}${JH_MAKEFLAGS}${OFF}${R_arrow}" - [ -n "$JH_MAKEFLAGS" ] && \ - echo -e "BLACK_LIST: ${L_arrow}${BOLD}${BLACK_LIST}${OFF}${R_arrow}\n" +# [ -n "$JH_MAKEFLAGS" ] && \ +# echo -e "BLACK_LIST: ${L_arrow}${BOLD}${BLACK_LIST}${OFF}${R_arrow}\n" echo -e "DEF_OPT_MODE: ${L_arrow}${BOLD}${DEF_OPT_MODE}${OFF}${R_arrow}\n" From 3c43655ec646c94ab3da44b72e856bc890468f25 Mon Sep 17 00:00:00 2001 From: Pierre Labastie Date: Wed, 15 Nov 2023 18:10:07 +0100 Subject: [PATCH 28/51] Only replace $(nproc) in scriptlets There is no more $(nproc || echo 1). Note that we may replace it with $(nproc) if using all cores, so don't replace it in the comment-test template (otherwise we have infinite recursion). --- LFS/lfs.xsl | 16 ++-------------- common/libs/func_book_parser | 14 ++++++-------- 2 files changed, 8 insertions(+), 22 deletions(-) diff --git a/LFS/lfs.xsl b/LFS/lfs.xsl index 3b575ec..49fbdf2 100644 --- a/LFS/lfs.xsl +++ b/LFS/lfs.xsl @@ -29,8 +29,7 @@ - - + @@ -621,11 +620,8 @@ unset OLD_PKGDIR - - - - + **EDITME @@ -1092,14 +1088,6 @@ LOGLEVEL=" - - - - - - /dev/null - JOBS_1=1 - JOBS_2=1 - case "$OPTIMIZE" in - "1") JOBS_2="$N_PARALLEL";; - "2") JOBS_1="$N_PARALLEL"; JOBS_2="$N_PARALLEL";; - esac + if [ "$ALL_CORES" = "y" ]; then + JOBS="\$(nproc)" + else + JOBS="$N_PARALLEL" + fi # First profile the book, for revision and arch. Note that # MULTIBLIB is set to "default" if pure 64 bit book. In this case, # profiling on arch is useless, but does not hurt either. @@ -109,8 +108,7 @@ extract_commands() { # --stringparam local "$LOCAL" \ --stringparam log-level "$LOG_LEVEL" \ --stringparam script-root "$SCRIPT_ROOT" \ - --stringparam jobs_1 "$JOBS_1" \ - --stringparam jobs_2 "$JOBS_2" \ + --stringparam jobs "$JOBS" \ --output "./${COMMANDS}/" \ $XSL \ prbook.xml >> $LOGDIR/$LOG 2>&1 From 63190e6547cbe5f05d6f3db106f8ed81bd730b0c Mon Sep 17 00:00:00 2001 From: Pierre Labastie Date: Wed, 15 Nov 2023 18:16:10 +0100 Subject: [PATCH 29/51] Only pass MAKEFLAGS and NINJAJOBS to old books Also pass them to binutils-pass1 if calculating the SBU --- LFS/master.sh | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/LFS/master.sh b/LFS/master.sh index 2aa9843..d28b5cb 100644 --- a/LFS/master.sh +++ b/LFS/master.sh @@ -127,11 +127,19 @@ chapter_targets() { # # If using optimizations, write the instructions case "${OPTIMIZE}$1${nb_chaps}${this_script}${REALSBU}" in - 0* | *binutils-pass1y | 15* | 167* | 177*) + 0* | *binutils-pass1y | 15* | 167* | 177*) ;; + *kernel*) ;; # No CFLAGS for kernel + *) wrt_optimize "$name" ;; + esac + # MAKEFLAGS is set even if no optimization now. + # For new books, the scripts contain the MAKEFLAGS, so no + # need to set them in the envars except if binutils-pass1 + # and REALSBU=y. For Old books, N_PARALLEL and JH_MAKEFLAGS + # are set and we must set the envars. + case "${mkf_included}${this_script}${REALSBU}" in + *binutils-pass1y) wrt_makeflags "$name" "-j1" "1" ;; - *kernel*) # No CFLAGS for kernel - wrt_makeflags "$name" "$JH_MAKEFLAGS" "$N_PARALLEL" ;; - *) wrt_optimize "$name" && + n*) wrt_makeflags "$name" "$JH_MAKEFLAGS" "$N_PARALLEL" ;; esac fi @@ -252,6 +260,14 @@ build_Makefile() { # # echo nb_chaps: $nb_chaps # end DEBUG + # We need to know if we have an old or a new book (with included MAKEFLAGS) + # we grep for MAKEFLAGS in chapter04 to find out. + if grep -q MAKEFLAGS chapter04/*; then + mkf_included=y + else + mkf_included=n + fi + # Make a temporary file with all script targets for (( i = 4; i < nb_chaps+4; i++ )); do chapter_targets $i From cacc6c5433c5c2e2b52eee16cb88f661899a2a6f Mon Sep 17 00:00:00 2001 From: Pierre Labastie Date: Wed, 15 Nov 2023 18:19:14 +0100 Subject: [PATCH 30/51] Allow users to pass a cpuset If using a new book, MAKEFLAGS is set to -j$(nproc), so that if we want to easure timings with 4 processors and our computer has more than that, we need to restrict the number of processors. Jhalfs allows to do that using th cpuset cgroup controller, so that it only restrits the make job and not the whole machine. For that use a small scripts written by Xi Ruoyao that restarts make this the cpuset set. --- LFS/master.sh | 11 +++++++++++ common/libs/func_wrt_Makefile | 1 + jhalfs | 2 +- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/LFS/master.sh b/LFS/master.sh index d28b5cb..8402fc1 100644 --- a/LFS/master.sh +++ b/LFS/master.sh @@ -302,6 +302,17 @@ build_Makefile() { # i=`expr $i + 1` done + # If CPUSET is defined and not equal to "all", then we define a first target + # that calls a script which re-enters make calling target all + if [ -n "$CPUSET" ] && [ "$CPUSET" != all ]; then +( + cat << EOF + +all-with-cpuset: + @CPUSPEC="\$(CPUSET)" ./run-in-cgroup \$(MAKE) all +EOF +) >> $MKFILE + fi # Drop in the main target 'all:' and the chapter targets with each sub-target # as a dependency. Also prevent running targets in parallel. ( diff --git a/common/libs/func_wrt_Makefile b/common/libs/func_wrt_Makefile index 78bb3e1..c694e76 100644 --- a/common/libs/func_wrt_Makefile +++ b/common/libs/func_wrt_Makefile @@ -44,6 +44,7 @@ ADD_CUSTOM_TOOLS = $CUSTOM_TOOLS ADD_BLFS_TOOLS = $BLFS_TOOL PKGMNGT = $PKGMNGT WRAP_INSTALL = $WRAP_INSTALL +CPUSET = $CPUSET export PATH := \${PATH}:/usr/sbin diff --git a/jhalfs b/jhalfs index fe2c30c..9f2518c 100755 --- a/jhalfs +++ b/jhalfs @@ -323,7 +323,7 @@ if [[ "$REBUILD_MAKEFILE" = "n" ]] ; then true >"$LOGDIR/$LOG" # Copy common helper files - cp "$COMMON_DIR"/{makefile-functions,progress_bar.sh} "$JHALFSDIR/" + cp "$COMMON_DIR"/{makefile-functions,progress_bar.sh,run-in-cgroup.sh} "$JHALFSDIR/" # Copy needed stylesheets cp "$COMMON_DIR"/{packages.xsl,chroot.xsl,kernfs.xsl} "$JHALFSDIR/" From 640cd5746bfe3262c22f5d14dc00d91f411568c4 Mon Sep 17 00:00:00 2001 From: Pierre Labastie Date: Wed, 15 Nov 2023 18:23:42 +0100 Subject: [PATCH 31/51] Fix a typo --- common/libs/func_validate_configs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/libs/func_validate_configs.sh b/common/libs/func_validate_configs.sh index 34d07ec..5eb63b5 100644 --- a/common/libs/func_validate_configs.sh +++ b/common/libs/func_validate_configs.sh @@ -167,7 +167,7 @@ inline_doc BOOT_CONFIG) [[ "${METHOD}" = "boot" ]] && validate_file -z -e -s ;; # Treatment of LANG parameter - LANG ) # See it the locale value has been set + LANG ) # See if the locale value has been set echo -n "`eval echo $PARAM_VALS`" [[ -z "${!config_param}" ]] && echo " -- Variable $config_param cannot be empty!" && From 8e9343b6323ae0792535c13289fc70de2f538c03 Mon Sep 17 00:00:00 2001 From: Pierre Labastie Date: Wed, 15 Nov 2023 19:37:58 +0100 Subject: [PATCH 32/51] Add forgotten run-with-cgroup.sh script --- common/run-in-cgroup.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100755 common/run-in-cgroup.sh diff --git a/common/run-in-cgroup.sh b/common/run-in-cgroup.sh new file mode 100755 index 0000000..9891d72 --- /dev/null +++ b/common/run-in-cgroup.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +if [ -z "$CPUSPEC" ] || [ "$#" -lt 1 ]; then + echo "usage: CPUSPEC=... $0 command" + exit 1 +fi + +set +e + +if type systemd-run >/dev/null 2>&1 ; then + sudo systemd-run -G --pty -d --uid=$(whoami) -p AllowedCPUs="$CPUSPEC" "$@" +else + sudo mkdir /sys/fs/cgroup/jhalfs + sudo sh -c "echo +cpuset > /sys/fs/cgroup/cgroup.subtree_control" + sudo sh -c "echo \"$CPUSPEC\" > /sys/fs/cgroup/jhalfs/cpuset.cpus" + (sudo sh -c "echo $BASHPID > /sys/fs/cgroup/jhalfs/cgroup.procs" && + exec "$@") + sudo rmdir /sys/fs/cgroup/jhalfs +fi From cff36a77ba586d41f2e62c1f5eadd7625621aad1 Mon Sep 17 00:00:00 2001 From: Pierre Labastie Date: Thu, 16 Nov 2023 14:37:58 +0100 Subject: [PATCH 33/51] Fix session management in run-in-cgroup.sh Still WIP but some progress... When moving a shell to a cgroup not associated with a session, then subsequent calls to pam_systemd or pam_elogind create a new session, and a new cgroup for that session, so that the cgroup of the calling process is not used (this is a problem with both systemd and elogind). For systemd, the problem can be solved by passing --slice to systemd-run. For elogind, we need to first move the shell to a non session cgroup, then run sudo so that a new session is created, then pass the cpuset to that session's cgroup. Hopefully, if neither systemd nor elgind is used, then the former solution should work (to be tested!!!). --- common/run-in-cgroup.sh | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/common/run-in-cgroup.sh b/common/run-in-cgroup.sh index 9891d72..8794cd4 100755 --- a/common/run-in-cgroup.sh +++ b/common/run-in-cgroup.sh @@ -7,13 +7,29 @@ fi set +e -if type systemd-run >/dev/null 2>&1 ; then - sudo systemd-run -G --pty -d --uid=$(whoami) -p AllowedCPUs="$CPUSPEC" "$@" -else - sudo mkdir /sys/fs/cgroup/jhalfs - sudo sh -c "echo +cpuset > /sys/fs/cgroup/cgroup.subtree_control" - sudo sh -c "echo \"$CPUSPEC\" > /sys/fs/cgroup/jhalfs/cpuset.cpus" - (sudo sh -c "echo $BASHPID > /sys/fs/cgroup/jhalfs/cgroup.procs" && - exec "$@") - sudo rmdir /sys/fs/cgroup/jhalfs +if type systemd-run >/dev/null 2>&1 ; then # systemd + sudo systemd-run -G --pty -d --uid=$(whoami) \ + -p AllowedCPUs="$CPUSPEC" \ + --slice "user-$(whoami).slice" \ + "$@" +elif type loginctl >/dev/null 2>&1 ; then #elogind + sudo mkdir /sys/fs/cgroup/jhalfs + sudo sh -c "echo +cpuset > /sys/fs/cgroup/cgroup.subtree_control" + ( + sudo sh -c "echo $BASHPID > /sys/fs/cgroup/jhalfs/cgroup.procs" + sudo -u $(whoami) sh < \$SESS_CGROUP/cpuset.cpus" + (sudo sh -c "echo \$BASHPID > \$SESS_CGROUP/cgroup.procs" && + exec $@) +EOF + ) + sudo rmdir /sys/fs/cgroup/jhalfs +else # no session manager + sudo mkdir /sys/fs/cgroup/jhalfs + sudo sh -c "echo +cpuset > /sys/fs/cgroup/cgroup.subtree_control" + sudo sh -c "echo \"$CPUSPEC\" > /sys/fs/cgroup/jhalfs/cpuset.cpus" + (sudo sh -c "echo $BASHPID > /sys/fs/cgroup/jhalfs/cgroup.procs" && + exec "$@") + sudo rmdir /sys/fs/cgroup/jhalfs fi From 09ac7fc548e3de4f0afdde85eb25a70e3e6f33d6 Mon Sep 17 00:00:00 2001 From: Pierre Labastie Date: Thu, 16 Nov 2023 16:10:07 +0100 Subject: [PATCH 34/51] Typo The name of the script is run-in-cgroup.sh, not run-in-cgroup ... --- LFS/master.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LFS/master.sh b/LFS/master.sh index 8402fc1..a7f4429 100644 --- a/LFS/master.sh +++ b/LFS/master.sh @@ -309,7 +309,7 @@ build_Makefile() { # cat << EOF all-with-cpuset: - @CPUSPEC="\$(CPUSET)" ./run-in-cgroup \$(MAKE) all + @CPUSPEC="\$(CPUSET)" ./run-in-cgroup.sh \$(MAKE) all EOF ) >> $MKFILE fi From 249d4ade208ff645ee5d5d409a4dba6f08561fa4 Mon Sep 17 00:00:00 2001 From: Pierre Labastie Date: Thu, 16 Nov 2023 16:10:46 +0100 Subject: [PATCH 35/51] Fix (again) the run-in-cgroup.sh script When using "sh << EOF", what is between this line and EOF runs with no attached terminal. But we need a terminal for running the build. So we have to use "sh -c ". But then nesting of double quotes, $, etc is too complicated. So run as root, and become back user only when exec'ing. For some reason $@ does not work, so use a variable set to $@. --- common/run-in-cgroup.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/common/run-in-cgroup.sh b/common/run-in-cgroup.sh index 8794cd4..3f57857 100755 --- a/common/run-in-cgroup.sh +++ b/common/run-in-cgroup.sh @@ -5,6 +5,8 @@ if [ -z "$CPUSPEC" ] || [ "$#" -lt 1 ]; then exit 1 fi +ARGS="$@" + set +e if type systemd-run >/dev/null 2>&1 ; then # systemd @@ -17,12 +19,11 @@ elif type loginctl >/dev/null 2>&1 ; then #elogind sudo sh -c "echo +cpuset > /sys/fs/cgroup/cgroup.subtree_control" ( sudo sh -c "echo $BASHPID > /sys/fs/cgroup/jhalfs/cgroup.procs" - sudo -u $(whoami) sh < \$SESS_CGROUP/cpuset.cpus" - (sudo sh -c "echo \$BASHPID > \$SESS_CGROUP/cgroup.procs" && - exec $@) -EOF + echo $CPUSPEC > \$SESS_CGROUP/cpuset.cpus + ( echo \$BASHPID > \$SESS_CGROUP/cgroup.procs && + exec sudo -u $(whoami) $ARGS )" ) sudo rmdir /sys/fs/cgroup/jhalfs else # no session manager From 078ef3cf122130dfc69fd1c1948a5583104986ce Mon Sep 17 00:00:00 2001 From: Pierre Labastie Date: Fri, 17 Nov 2023 13:18:12 +0100 Subject: [PATCH 36/51] Pass sanitized MAKEFLAGS to scriptlets Now that we have MAKEFLAGS in the environment, "make" in chroot is run with MAKEFLAGS set. This cause it to pass the -j value into MAKEFLAGS to inferiors (scriptlets), but it is passed with a --jobserver-auth value added. If we do not precede the command running the scriptlet with a +, it passes --jobserver-auth=-2,-2 meaning that the jobserver is disabled, and "make" in the scriptlet understands that it should only run one job at a time (defeating the value of the -j flag). But even if we do precede the command running the scriptlet with a +, it passes --jobserver-auth=3,4, which shows it is running, but this jobserver will authorize only one job at a time since the parent make contains the target .NOTPARALLEL! To work around this problem, tje only solution is to change the MAKEFLAGS at the beginning of the scriptlet. Fortunately, lfs.xsl knows what we want as job number (either $(nproc) or a fixed number), so use it to set MAKEFLAGS. --- LFS/lfs.xsl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/LFS/lfs.xsl b/LFS/lfs.xsl index 49fbdf2..26982b0 100644 --- a/LFS/lfs.xsl +++ b/LFS/lfs.xsl @@ -1249,6 +1249,11 @@ tar -xf $PACKAGE cd $PKGDIR + +export MAKEFLAGS="-j + + " + SECONDS=${PREV_SEC} # Start of LFS book script From d18fc24d5be19579426b6da1f5db75beb3d9a6da Mon Sep 17 00:00:00 2001 From: Pierre Labastie Date: Fri, 17 Nov 2023 13:57:15 +0100 Subject: [PATCH 37/51] blfs tools: update JOBS for the new config Now, we have ALL_CORES set to y or n, and depending on this, we want either JOBS to be set to $(nproc) or to the value of N_PARALLEL. --- common/libs/func_install_blfs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/libs/func_install_blfs b/common/libs/func_install_blfs index 8a87b1b..4261932 100644 --- a/common/libs/func_install_blfs +++ b/common/libs/func_install_blfs @@ -136,7 +136,7 @@ STATS=n SRC_ARCHIVE=/sources BUILD_ROOT=/sources BUILD_SUBDIRS=y -JOBS=$(if [ -n "$N_PARALLEL" ]; then echo $N_PARALLEL; else echo 1; fi) +JOBS=$(if [ "$ALL_CORES" = y ]; then nproc; else echo $N_PARALLEL; fi) EOF for OPT_VAR in CFLAGS CXXFLAGS LDFLAGS; do eval optVal=\$${OPT_VAR}_$DEF_OPT_MODE From da4bce358df104cc3e0ed59503eafb8c92c0db1e Mon Sep 17 00:00:00 2001 From: Pierre Labastie Date: Fri, 17 Nov 2023 13:59:06 +0100 Subject: [PATCH 38/51] Fix blfs tools Makefile now that MAKEFLAGS is set Previously, the "update" target was just added to the end of the "all" target prerequisites, so that it was run at the end. But now, we enter make with some -j value in MAKEFLAGS, so that the "update" target may well be run before the other targets, which is wrong: for example if it is run before the "xxx-z-libxslt" target, xsltproc is not found and the update target fails. To fix, define "update" as the default target, and have it depend on "all". --- common/libs/func_install_blfs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/libs/func_install_blfs b/common/libs/func_install_blfs index 4261932..32a7851 100644 --- a/common/libs/func_install_blfs +++ b/common/libs/func_install_blfs @@ -218,10 +218,10 @@ pushd $BUILDDIR$BLFS_ROOT/work # installation, using libxslt, which is not installed yet. So move # updating to the end of the process, adding an 'update' target sed -i -e '/xsltproc/,+6d' \ - -e '/^all/s@$@ update@' \ + -e '/^all/i update:' \ -e 's/touch/@touch/' Makefile cat >> Makefile << EOF -update: +update: all @echo Updating the tracking file @for pack in \$\$(grep '\([^<]*\)<.*/\1/' | \\ From a9360e10a757f6832b631c9ee4ae3bbe7e500f75 Mon Sep 17 00:00:00 2001 From: Pierre Labastie Date: Fri, 17 Nov 2023 15:55:36 +0100 Subject: [PATCH 39/51] Fix MAKEFLAGS for binutils-pass1 We need a special case for binutils-pass1 in case REALSBU is set to y. Use variables JOBSBP1 in shell and jobs-bp1 in .xsl. --- LFS/lfs.xsl | 12 +++++++++++- common/libs/func_book_parser | 4 +++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/LFS/lfs.xsl b/LFS/lfs.xsl index 26982b0..d96aa60 100644 --- a/LFS/lfs.xsl +++ b/LFS/lfs.xsl @@ -31,6 +31,9 @@ + + + @@ -1251,7 +1254,14 @@ cd $PKGDIR export MAKEFLAGS="-j - + + + + + + + + " SECONDS=${PREV_SEC} diff --git a/common/libs/func_book_parser b/common/libs/func_book_parser index 6b7435b..e754666 100644 --- a/common/libs/func_book_parser +++ b/common/libs/func_book_parser @@ -67,8 +67,9 @@ extract_commands() { # JOBS="\$(nproc)" else JOBS="$N_PARALLEL" - fi + if [ "REALSBU" = y ]; then JOBSBP1=1; else JOBSBP1="$JOBS"; fi + # First profile the book, for revision and arch. Note that # MULTIBLIB is set to "default" if pure 64 bit book. In this case, # profiling on arch is useless, but does not hurt either. @@ -109,6 +110,7 @@ extract_commands() { # --stringparam log-level "$LOG_LEVEL" \ --stringparam script-root "$SCRIPT_ROOT" \ --stringparam jobs "$JOBS" \ + --stringparam jobs-bp1 "$JOBSBP1" \ --output "./${COMMANDS}/" \ $XSL \ prbook.xml >> $LOGDIR/$LOG 2>&1 From 04a608ba8e512fc2458dd2be1d5f75b4e92e0ea6 Mon Sep 17 00:00:00 2001 From: Pierre Labastie Date: Fri, 17 Nov 2023 15:57:55 +0100 Subject: [PATCH 40/51] Pass NINJAJOBS if N_PARALLEL is defined Now that MAKEFLAGS is defined at the beginning of the scriptlet, there is no need to pass MAKEFLAGS as an envar to the scriptlet. But still NINJAJOBS is not defined. In most cases, when we want to use all cores, this is not a problem. But if N_PARALLEL is defined, it needs to be passed. Note that we also pass MAKEFLAGS in this case, because we use the old "wrt_xxx" function, but it is not used. --- LFS/master.sh | 27 +++++++-------------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/LFS/master.sh b/LFS/master.sh index a7f4429..e3334d0 100644 --- a/LFS/master.sh +++ b/LFS/master.sh @@ -131,18 +131,13 @@ chapter_targets() { # *kernel*) ;; # No CFLAGS for kernel *) wrt_optimize "$name" ;; esac - # MAKEFLAGS is set even if no optimization now. - # For new books, the scripts contain the MAKEFLAGS, so no - # need to set them in the envars except if binutils-pass1 - # and REALSBU=y. For Old books, N_PARALLEL and JH_MAKEFLAGS - # are set and we must set the envars. - case "${mkf_included}${this_script}${REALSBU}" in - *binutils-pass1y) - wrt_makeflags "$name" "-j1" "1" ;; - n*) - wrt_makeflags "$name" "$JH_MAKEFLAGS" "$N_PARALLEL" ;; - esac - fi + # There is no need to tweak MAKEFLAGS anymore, this is done + # by lfs.xsl. But still, NINJAJOBS needs to be set if + # N_PARALLEL is defined. + if [ -n "N_PARALLEL" ]; then + wrt_makeflags "$name" "$JH_MAKEFLAGS" "$N_PARALLEL" + fi + fi # end of package specific instructions # Some scriptlet have a special treatment; otherwise standard case "${this_script}" in @@ -260,14 +255,6 @@ build_Makefile() { # # echo nb_chaps: $nb_chaps # end DEBUG - # We need to know if we have an old or a new book (with included MAKEFLAGS) - # we grep for MAKEFLAGS in chapter04 to find out. - if grep -q MAKEFLAGS chapter04/*; then - mkf_included=y - else - mkf_included=n - fi - # Make a temporary file with all script targets for (( i = 4; i < nb_chaps+4; i++ )); do chapter_targets $i From 924076f0b5003e5a73b711afafbbd88c83225d69 Mon Sep 17 00:00:00 2001 From: Pierre Labastie Date: Fri, 17 Nov 2023 16:10:01 +0100 Subject: [PATCH 41/51] blfs tools: fix JOBS If all cores are used, it should be set to 0, which will use nproc after the cpuset has been defined, and not nproc (which is the number of cores/threads on the machine). --- common/libs/func_install_blfs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/common/libs/func_install_blfs b/common/libs/func_install_blfs index 32a7851..191878b 100644 --- a/common/libs/func_install_blfs +++ b/common/libs/func_install_blfs @@ -136,8 +136,9 @@ STATS=n SRC_ARCHIVE=/sources BUILD_ROOT=/sources BUILD_SUBDIRS=y -JOBS=$(if [ "$ALL_CORES" = y ]; then nproc; else echo $N_PARALLEL; fi) +JOBS=$(if [ "$ALL_CORES" = y ]; then echo 0; else echo $N_PARALLEL; fi) EOF +# The 0 value above is for using all cores for OPT_VAR in CFLAGS CXXFLAGS LDFLAGS; do eval optVal=\$${OPT_VAR}_$DEF_OPT_MODE if [ -n "$optVal" ] && [ "$optVal" != unset ]; then From 67ca47e5f8ac3208b2bf0a3fde64b42c908d4ba3 Mon Sep 17 00:00:00 2001 From: Pierre Labastie Date: Fri, 17 Nov 2023 16:31:22 +0100 Subject: [PATCH 42/51] Stupid typo --- common/libs/func_book_parser | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/libs/func_book_parser b/common/libs/func_book_parser index e754666..3d83470 100644 --- a/common/libs/func_book_parser +++ b/common/libs/func_book_parser @@ -68,7 +68,7 @@ extract_commands() { # else JOBS="$N_PARALLEL" fi - if [ "REALSBU" = y ]; then JOBSBP1=1; else JOBSBP1="$JOBS"; fi + if [ "$REALSBU" = y ]; then JOBSBP1=1; else JOBSBP1="$JOBS"; fi # First profile the book, for revision and arch. Note that # MULTIBLIB is set to "default" if pure 64 bit book. In this case, From c5f4ef3d5e73bf8652d05d8a9ca7f30b1eefd816 Mon Sep 17 00:00:00 2001 From: Pierre Labastie Date: Sat, 18 Nov 2023 07:47:29 +0100 Subject: [PATCH 43/51] BLFS tools: update qt?-prefix for recent books --- BLFS/xsl/scripts.xsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/BLFS/xsl/scripts.xsl b/BLFS/xsl/scripts.xsl index 78f61ad..be8a4db 100644 --- a/BLFS/xsl/scripts.xsl +++ b/BLFS/xsl/scripts.xsl @@ -302,7 +302,7 @@ cd $SRC_DIR - + From 23b0161d6c3d5c5f8eebbd31c56c5708fb4148ed Mon Sep 17 00:00:00 2001 From: Pierre Labastie Date: Sat, 18 Nov 2023 07:47:29 +0100 Subject: [PATCH 44/51] BLFS tools: update qt?-prefix for recent books --- BLFS/xsl/scripts.xsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/BLFS/xsl/scripts.xsl b/BLFS/xsl/scripts.xsl index 78f61ad..be8a4db 100644 --- a/BLFS/xsl/scripts.xsl +++ b/BLFS/xsl/scripts.xsl @@ -302,7 +302,7 @@ cd $SRC_DIR - + From 05b955b62c1cf4afda76c08587303cd95e4eb4c6 Mon Sep 17 00:00:00 2001 From: Pierre Labastie Date: Tue, 21 Nov 2023 10:12:28 +0100 Subject: [PATCH 45/51] BLFS: remove support for ftp and alternate server ftp urls have been removed from book, and in case you have to use an alternate server, better download directly to /sources/yourself. --- BLFS/xsl/scripts.xsl | 25 +++---------------------- 1 file changed, 3 insertions(+), 22 deletions(-) diff --git a/BLFS/xsl/scripts.xsl b/BLFS/xsl/scripts.xsl index be8a4db..ad6612a 100644 --- a/BLFS/xsl/scripts.xsl +++ b/BLFS/xsl/scripts.xsl @@ -441,10 +441,6 @@ echo Start Time: ${SECONDS} >> $INFOLOG - = @@ -460,24 +456,9 @@ echo Start Time: ${SECONDS} >> $INFOLOG " "$ " - else - - - wget -T 30 -t 5 " - - " || - - - - wget -T 30 -t 5 " - - " || - - - wget -T 30 -t 5 "${JH_FTP_SERVER}svn/ - - /$ - + else + wget -T 30 -t 5 " + " fi fi From c1f02f262e6c39e6b40adcbbeee1a38532c9c20b Mon Sep 17 00:00:00 2001 From: Pierre Labastie Date: Tue, 21 Nov 2023 10:19:20 +0100 Subject: [PATCH 46/51] LFS: remove alternate server code It's been broken for years, and nobody has complained, so I'd say it is not useful. If a user wants to use a mirror instead of upstream, better use a wget-list directly on this server... --- Config.in | 10 ---------- common/libs/func_download_pkgs | 2 -- common/libs/func_install_blfs | 1 - common/libs/func_validate_configs.sh | 3 +-- common/urls.xsl | 3 --- 5 files changed, 1 insertion(+), 18 deletions(-) diff --git a/Config.in b/Config.in index b420534..d03ae88 100644 --- a/Config.in +++ b/Config.in @@ -421,16 +421,6 @@ depends on !BOOK_BLFS #-- Number of seconds to wait for a download to start before # timing out. - config SERVER - string "FTP mirror" - default "http://ftp.osuosl.org" - depends on GETPKG - help - #-- FTP mirror to download packages and patches if not found - # in $SRC_ARCHIVE - # As a last resort, the files will downloaded from upstream, - # if possible. - config RUNMAKE bool "Run the makefile" default n diff --git a/common/libs/func_download_pkgs b/common/libs/func_download_pkgs index 3d369d2..a6d9c7e 100644 --- a/common/libs/func_download_pkgs +++ b/common/libs/func_download_pkgs @@ -132,8 +132,6 @@ create_urls() { # echo -n "Creating URLs file... " xsltproc --nonet --xinclude \ - --stringparam server "$SERVER" \ - --stringparam family lfs \ --stringparam pkgmngt "$PKGMNGT" \ --stringparam revision "$INITSYS" \ --output ../sources/urls.lst \ diff --git a/common/libs/func_install_blfs b/common/libs/func_install_blfs index 8a87b1b..45c25a8 100644 --- a/common/libs/func_install_blfs +++ b/common/libs/func_install_blfs @@ -179,7 +179,6 @@ pushd $BUILDDIR/sources # Remove `unpacked' files if some have been left sudo find . -name unpacked -exec rm \{\} \; if [ "$GETPKG" = "y" ]; then - JH_FTP_SERVER=$SERVER/pub/blfs/ \ JH_SRC_ARCHIVE=${SRC_ARCHIVE:-/dev/null} \ $BUILDDIR$BLFS_ROOT/download_script else # Save the download script in case the user wants to run it later diff --git a/common/libs/func_validate_configs.sh b/common/libs/func_validate_configs.sh index ec21656..fa8e8fb 100644 --- a/common/libs/func_validate_configs.sh +++ b/common/libs/func_validate_configs.sh @@ -18,7 +18,7 @@ inline_doc # Common settings by Config.in sections and books family local -r BOOK_common="COMMIT BOOK CUSTOM_TOOLS" local -r GENERAL_common="LUSER LGROUP LHOME BUILDDIR CLEAN GETPKG SRC_ARCHIVE \ - SERVER RETRYSRCDOWNLOAD RETRYDOWNLOADCNT DOWNLOADTIMEOUT \ + RETRYSRCDOWNLOAD RETRYDOWNLOADCNT DOWNLOADTIMEOUT \ RUNMAKE" local -r BUILD_chroot="TEST STRIP" local -r BUILD_common="FSTAB CONFIG TIMEZONE PAGE LANG INSTALL_LOG" @@ -129,7 +129,6 @@ inline_doc ITERATIONS) [[ "$COMPARE" = "y" ]] && echo -e "`eval echo $PARAM_VALS`" ;; TARGET32) [[ -n "${TARGET32}" ]] && echo -e "`eval echo $PARAM_VALS`" ;; MIPS_LEVEL) [[ "${ARCH}" = "mips" ]] && echo -e "`eval echo $PARAM_VALS`" ;; - SERVER) [[ "$GETPKG" = "y" ]] && echo -e "`eval echo $PARAM_VALS`" ;; RETRYSRCDOWNLOAD) [[ "$GETPKG" = "y" ]] && echo -e "`eval echo $PARAM_VALS`" ;; RETRYDOWNLOADCNT) [[ "$GETPKG" = "y" ]] && echo -e "`eval echo $PARAM_VALS`" ;; DOWNLOADTIMEOUT) [[ "$GETPKG" = "y" ]] && echo -e "`eval echo $PARAM_VALS`" ;; diff --git a/common/urls.xsl b/common/urls.xsl index 3bfa072..185f07d 100644 --- a/common/urls.xsl +++ b/common/urls.xsl @@ -5,9 +5,6 @@ - - ftp://ftp.osuosl.org - From bc64795e637cff64b6cef151704d7a50ee870293 Mon Sep 17 00:00:00 2001 From: Pierre Labastie Date: Tue, 21 Nov 2023 10:36:30 +0100 Subject: [PATCH 47/51] Remove a spurious menu in Config.in --- Config.in | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Config.in b/Config.in index d03ae88..d5c3fd5 100644 --- a/Config.in +++ b/Config.in @@ -457,9 +457,8 @@ depends on !BOOK_BLFS # out. If you select 'n' here, the commented test instructions # do not stop on test suite failures. - menu "Test settings" - depends on CONFIG_TESTS choice + depends on CONFIG_TESTS prompt "Tests level" default TST_1 @@ -480,8 +479,6 @@ depends on !BOOK_BLFS endchoice - endmenu # test settings - config TEST int default "0" if !CONFIG_TESTS From 10756d05d74b1faaa1609487c0aa77fc2a4253e5 Mon Sep 17 00:00:00 2001 From: Pierre Labastie Date: Tue, 21 Nov 2023 10:40:52 +0100 Subject: [PATCH 48/51] Move the parallelism menu to "Build Settings" Now that parallelism has been included into the book, this is no more an advanced configuration... --- Config.in | 102 +++++++++++++++++++++++++++--------------------------- 1 file changed, 51 insertions(+), 51 deletions(-) diff --git a/Config.in b/Config.in index 1a36cb5..e15ce87 100644 --- a/Config.in +++ b/Config.in @@ -449,6 +449,57 @@ endmenu menu "Build Settings" depends on !BOOK_BLFS + #--- Parallelism + menu "Parallelism settings" + if HAVE_NPROC + config ALL_CORES + bool "Use all cores as in new books (say n for old books)" + default y + help + Use book instructions as written for parallelism since version + r12.0-87 included. If you answer no, then jhalfs will fall back + to a static number of cores, defined below. If you answer y, and + your system supports it, you'll be asked for a cpu set to use. + There is no detection of book version. If the book version + is lower than r12.0-87, say no! + + if ALL_CORES && HAVE_CGROUP + config CPUSET + string "set of cpus to use, or 'all' for all cpus" + default "all" + help + See "List format" in cpuset(7). Choosing cpus depend + on the topology of your processors. Sometimes two + hyperthreads on the same core are numbered consecutively. + For example for using all cores and no hyperthreading on + a Haswell, use "0,2,4,6". Other brands may have a different + topology, and may require e.g. "0-3" to use the first 4 cores. + If not sure, keep the default. + endif + + endif # HAVE_NPROC + if !HAVE_NPROC + config ALL_CORES + bool + default n + endif + + config N_PARALLEL + int "Number of parallel `make' jobs" + depends on !ALL_CORES + default 1 + help + #-- The usual recommandation is (number of CPU cores)+1 + # Do not set for meaningful SBU calculations. + + config REALSBU + bool "Build Binutils pass1 without parallelism (Real SBU)" + default n + help + #-- Use -j1 in make invokation for Binutils pass1 to + # get a valid SBU value. + endmenu # parallelism + #--- Test Suites config CONFIG_TESTS bool "Run testsuites" @@ -758,57 +809,6 @@ endmenu #--- System configuration menu "Advanced Features" depends on !BOOK_BLFS - #--- Parallelism - menu "Parallelism settings" - if HAVE_NPROC - config ALL_CORES - bool "Use all cores as in new books (say n for old books)" - default y - help - Use book instructions as written for parallelism since version - r12.0-87 included. If you answer no, then jhalfs will fall back - to a static number of cores, defined below. If you answer y, and - your system supports it, you'll be asked for a cpu set to use. - There is no detection of book version. If the book version - is lower than r12.0-87, say no! - - if ALL_CORES && HAVE_CGROUP - config CPUSET - string "set of cpus to use, or 'all' for all cpus" - default "all" - help - See "List format" in cpuset(7). Choosing cpus depend - on the topology of your processors. Sometimes two - hyperthreads on the same core are numbered consecutively. - For example for using all cores and no hyperthreading on - a Haswell, use "0,2,4,6". Other brands may have a different - topology, and may require e.g. "0-3" to use the first 4 cores. - If not sure, keep the default. - endif - - endif # HAVE_NPROC - if !HAVE_NPROC - config ALL_CORES - bool - default n - endif - - config N_PARALLEL - int "Number of parallel `make' jobs" - depends on !ALL_CORES - default 1 - help - #-- The usual recommandation is (number of CPU cores)+1 - # Do not set for meaningful SBU calculations. - - config REALSBU - bool "Build Binutils pass1 without parallelism (Real SBU)" - default n - help - #-- Use -j1 in make invokation for Binutils pass1 to - # get a valid SBU value. - endmenu # parallelism - #--- Optimizations config CONFIG_OPTIMIZE bool "Optimization" From 6ffc37b9026331b5cc81c28111093187c433b3b9 Mon Sep 17 00:00:00 2001 From: Pierre Labastie Date: Tue, 21 Nov 2023 11:51:29 +0100 Subject: [PATCH 49/51] Confg.in: fix help for ALL_CORES The settings don't depend on book version, and you can answer y even with old books. --- Config.in | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/Config.in b/Config.in index e15ce87..d1d7750 100644 --- a/Config.in +++ b/Config.in @@ -453,15 +453,20 @@ depends on !BOOK_BLFS menu "Parallelism settings" if HAVE_NPROC config ALL_CORES - bool "Use all cores as in new books (say n for old books)" + bool "Use all cores" default y help - Use book instructions as written for parallelism since version - r12.0-87 included. If you answer no, then jhalfs will fall back - to a static number of cores, defined below. If you answer y, and - your system supports it, you'll be asked for a cpu set to use. - There is no detection of book version. If the book version - is lower than r12.0-87, say no! + If you answer y, MAKEFLAGS will be set to "-j$(nproc)" at the + beginning of each package script, and book instructions will not + be changed otherwise. This will run the scripts with the same + settings as in the book since version r12.0-87 included. Before + that version, this will run the builds with all cores, but some + test suites may still be run sequentially. You'll be asked for + a CPU set to use, so that the number of jobs can still be limited + in this way (useful for measuring SBU values at -j4, for + example). + If you answer n, then jhalfs will fall back to a static number + of cores, defined below. if ALL_CORES && HAVE_CGROUP config CPUSET From cd87e527fe152ed8e50e44437853dd6a60c48cd0 Mon Sep 17 00:00:00 2001 From: Pierre Labastie Date: Thu, 23 Nov 2023 10:05:27 +0100 Subject: [PATCH 50/51] Fix configuration about saving temporary work It is no more at the end of chapter five now... Also the tar file is not compressed --- Config.in | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Config.in b/Config.in index d1d7750..ef3779e 100644 --- a/Config.in +++ b/Config.in @@ -857,15 +857,16 @@ depends on !BOOK_BLFS default y config SAVE_CH5 - bool "Save Chapter 5 work" + bool "Save temporary system work" depends on BOOK_LFS || BOOK_LFS_SYSD default n help - Save the state of jhalfs at the end of chapter 5: + Save the state of jhalfs after finishing building the temporary + system if you tick this item, the whole $LFS directory is - saved when chapter 5 is finished. It'll be in an xz - compressed tarball in the $LFS/jhalfs directory + saved when chapter 7 is finished. It'll be in a .tar + file in the $BUILD_DIR/jhalfs directory #--- ICA config COMPARE From 35094eda67103de7a8380b8c2744ee3ebeee0036 Mon Sep 17 00:00:00 2001 From: Pierre Labastie Date: Thu, 23 Nov 2023 10:07:45 +0100 Subject: [PATCH 51/51] Major refactoring of README Describe each item in the menu, fix layout, and try to follow the order of tasks to do to run jhalfs. --- README | 365 ++++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 234 insertions(+), 131 deletions(-) diff --git a/README b/README index 13d6f61..038c6f3 100644 --- a/README +++ b/README @@ -1,150 +1,268 @@ 1. INTRODUCTION:: - The scripts in this directory implement an automation of the building - of a GNU/LInux system, as described in the Linux From Scratch book series. - The name of the project is jhalfs: in that name, "alfs" stands for - "automated linux from scratch", and the initials "jh" have been kept since - the original "jhalfs-0.2" code developed by Jeremy Huntwork. + The scripts in this directory implement an automation of the building + of a GNU/LInux system, as described in the Linux From Scratch book series. + The name of the project is jhalfs: in that name, "alfs" stands for + "automated linux from scratch", and the initials "jh" have been kept since + the original "jhalfs-0.2" code developed by Jeremy Huntwork. - The list of supported books can be found at - http://wiki.linuxfromscratch.org/alfs/wiki/SupportedBooks. + The list of supported books can be found at + http://wiki.linuxfromscratch.org/alfs/wiki/SupportedBooks (maybe outdated, + current develoment books and latest version are always supported). - The documentation is split among various README.* files. Here is a list - of what is in which: - - README (this file): instructions to use the LFS book. This should be - enough if you just want to build a base system as per the LFS book. It is - also a required reading for all the other projects. - - README.BLFS: instructions to install an automated build infrastructure - for the BLFS book. There are two ways to do so: (i) install the - tools at the end of an LFS build, or - (ii) install the tools on an already running system. Both methods are - described in that file. - - README.CUSTOM: instructions to run custom commands either during the xLFS - build, at the end of a xLFS build. Note that you will not find - instructions on how to write those commands, but some examples are - available. - - README.PACKAGE_MANAGEMENT: instructions to use package management during - the build + The documentation is split among various README.* files. Here is a list + of what is in which: + - README (this file): instructions to use the LFS book. This should be + enough if you just want to build a base system as per the LFS book. It + is also a required reading for all the other projects. + - README.BLFS: instructions to install an automated build infrastructure + for the BLFS book. There are two ways to do so: (i) install the + tools at the end of an LFS build, or + (ii) install the tools on an already running system. Both methods are + described in that file. + - README.CUSTOM: instructions to run custom commands either during the LFS + build, or at the end of a LFS build. Note that you will not find + instructions on how to write those commands, but some examples are + available. + - README.PACKAGE_MANAGEMENT: instructions to use package management during + the build (Note: the only package manager that is regularly tested is + porg) - Other sources of information are the context help in the menu interface, - and the xLFS books themselves. + Other sources of information are the context help in the menu interface, + and the LFS books themselves (both required readings of course!). 2. PREREQUISITES:: - As said elsewhere, it is strongly advised that you first build manually - a complete system before attempting to automate the build. + It is strongly advised that you first build manually a complete system + before attempting to automate the build. - Of course the "Host System Requirements" should be fulfilled. The needed - supplementary packages are detailed at the bottom of the page: - https://www.linuxfromscratch.org/alfs/download.html. In short, you need - wget, sudo, libxml2, libxslt, docbook-4.5-xml, and docbook-xsl-nons. + Of course the "Host System Requirements" should be fulfilled. Some + supplementary packages are needed for using jhalfs. They are detailed + at the bottom of the page: + https://www.linuxfromscratch.org/alfs/download.html. In short, you need + wget, sudo, libxml2, libxslt, docbook-4.5-xml, and docbook-xsl-nons. 3. INSTALLATION:: - No installation is required. You may want to move the files in this - directory to a convenient location, and then follow the instructions below. + No installation is required. You may want to move the files in this + directory to a convenient location, and then follow the instructions below. 4. CONFIGURATION:: - 4.1. CONFIGURATION OF THE TOOLS: - There is no configuration of the tools themselves. The various - parameters for the build are set through a menu driven interface. See - the section RUNNING below for details. + 4.1. CONFIGURATION OF THE TOOLS: + There is no configuration of the tools themselves. The various + parameters for the build are set through a menu driven interface. See + the section RUNNING below for details. - 4.2. PRELIMINARY TASKS: - This tool has no support at all for creating a partition and a mount - point for the built system. You should follow the book up to the section - "Mounting the new partition". Note that the default name for the - partition mount point is "/mnt/build_dir", instead of /mnt/{c,}lfs. - You can change that default to anything you'd like in the menu, so you - may name it /mnt/lfs, or whatever you like. We'll use the name - /mnt/build_dir in the sequel. + 4.2. PRELIMINARY TASKS: + This tool has no support at all for creating a partition and a mount + point for the built system. You should follow the book up to the section + "Mounting the new partition". Note that the default name for the + partition mount point is "/mnt/build_dir", instead of /mnt/lfs. + You can change that default to anything you'd like in the menu, so you + may name it /mnt/lfs if you prefer . We'll use the name /mnt/build_dir + in the sequel. - The tool can download the needed packages for you, or you may download - them yourself. The tool may optionally use a package archive directory - where the downloaded packages are stored. That directory name may be made - available to the tool in two ways: (i) export the SRC_ARCHIVE variable, - for example SRC_ARCHIVE=/usr/src, (ii) enter the name at the "Package - Archive Directory" menu prompt. Note that the user should have write - permission to that directory. If a needed package is found in that - directory, it is copied to /mnt/build_dir/sources, if not, it is - downloaded to that directory and copied to /mnt/build_dir/sources, - except if found in /mnt/build_dir/sources, in which case, it is just - copied to $SRC_ARCHIVE. If you want the tool to download packages and you - do not want to archive them, just unset SRC_ARCHIVE, and keep the - default entry for "Package Archive Directory". If you choose to download - the packages by yourself, you should download (or copy) them to - /mnt/build_dir/sources directly. + For downloading packages, you can use the tool or download them + yourself. Even if using the tool, it is recommended to set up a source + repository where you store already downloaded packages. The tool will + automatically search a package in this repository before downloading it + if it is not found there. This repository cannot be the same as + /mnt/build_dir/sources. As an example, we'll use /usr/src. You should + arrange for the user running the tool to have write access to this + directory. - If you want to build the kernel as part of the automated build, select - "Build the kernel" in the menu. Then, a configuration file must be - provided. In order to do so, it is recommended to download the kernel - tarball, unpack it, run (or any other *config), - configure the kernel as per - the book, and save the resulting .config file to a location where it can - be retrieved later on (a convenient location and name is - $SRC_ARCHIVE/config---). + If you want to build the kernel as part of the automated build, + a configuration file must be provided. In order to do so, it is + recommended to download the kernel tarball, unpack it, run + (or any other *config), configure the kernel as per + the book, and save the resulting .config file to a location where it can + be retrieved later on. It is suggested to put it into the source + repository, with a versioned name, e.g. + /usr/src/config---. - Another file you may provide is the fstab file. To use it, select - "Use a custom fstab file" in the menu interface, and enter the name of - the file where asked. As for the kernel configuration, this file has to - be prepared before running the menu. A convenient location and name is - $SRC_ARCHIVE/fstablfs. + Another file you may provide is the fstab file. As for the kernel + configuration, this file has to be prepared before running the menu. + You can copy-paste the file from the "Creating the /etc/fstab File" + page, then edit to suit the future lfs system layout, then save the + file. A convenient location and name is /usr/src/fstablfs. - At a more advanced level, you may want to supply custom commands - to be run at the end of (C)LFS build. Scripts containing those commands - are located in the ./custom/config directory. Examples are given in - ./custom/examples. A template is provided as ./custom/template. See - README.CUSTOM for more details. + At a more advanced level, you may want to supply custom commands + to be run at the end of LFS build. Scripts containing those commands + are located in the ./custom/config directory. Examples are given in + ./custom/examples. A template is provided as ./custom/template. See + README.CUSTOM for more details. 5. RUNNING:: - IMPORTANT:: - You must be logged as a normal user with sudo privileges to run + IMPORTANT:: + You must be logged as a normal user with sudo privileges to run the Makefile. Furthermore, you are supposed to have enough privilege to become any user. If you are not bothered about security issues, - the entry for the user "jhalfs_user" in /etc/sudoers could be - jhalfs_user ALL=(ALL) NOPASSWD:ALL + the entry for the user running the tool in /etc/sudoers could be + ALL=(ALL) NOPASSWD:ALL - The command will launch a menu based configuration program. The - underlying menu code was borrowed from BusyBox and slightly modified for - our use. + The command will launch a menu based configuration program, + similar to the kernel "menuconfig" configuration tool. - Help on parameter function is available from the on-line help. Please - make use of that feature: it may contain additional information not - duplicated in this file. + Help on parameter function is available from the on-line help (type the + character `?' after highlighting the parameter). Please do use the help: + it may contain additional information not duplicated in this file. - You should first choose which book and flavour you want to build. Note - that when you choose the BLFS book, the tool will just install the BLFS - tool to your system. You'll have to run that installed tool to build - packages in BLFS. See README.BLFS to know how. If you choose any other - book, you'll have to configure the settings and the build parameters - from the menu. Note that you may choose to install the blfs tools onto - the newly built system. It is not the same thing as choosing - the BLFS book in the menu, which will install the blfs tools on the - currently running system. + MENU "BOOK Settings" - The "General Settings" menu is where the "Build Directory" name is to be - entered. Other entries in that menu select what the tool should do. The - "Run the Makefile" entry selects whether the tool will start the build - automatically after generating the needed files. The "Rebuild files" selects - whether to clean the build directory before doing anything else. To protect - against removing important files, this can only be done in an empty directory, - or a directory previously populated by the tool. + Use BOOK: You have three choices: LFS System V, LFS systemd, BLFS. + The BLFS part is described in README.BLFS - The "Build Settings" menu is where various options for the build can be - selected. Two options, "Use a custom fstab file" and "Build the kernel", - have been described above. "Do not use/display progress_bar", if set, will - prevent a progress bar to be displayed during the build. That may be useful - on slow machine. The other options should be self explanatory, using either - the online help or book reading. + Book version: You have two choices: "Branch" or "Working Copy" + Branch will have the tool clone the book's git repository. The + choice of the branch (actually any git commit) or of the file + location for the working copy is done in the next menu entry. - The "Advanced Features" menu is for various maintenance tasks, like - testing the build instructions or reporting build statistics. One useful - option is "Optimization and parallelisation". It is not recommended to use - it for setting compiler optimization flags, although it is possible, but - if you select it, you'll be able to select the number of parallel `make' - jobs, which allows much faster builds on modern multicore CPUs. + Multilib: Four choices: Normal LFS, Multilib with i686 libraries, + multilib with x32 libraries, multilib with all libraries. + It is recommended to use "Normal LFS" unless you know what you + are doing + + Build method: two choices: chroot (as in book), boot + Presently, the "boot" method is not implemented, so keep the default. + + Add blfs-tools support (see README.BLFS) + This will install the blfs tools onto the newly built system. It + is not the same thing as choosing the BLFS book in the menu, which + will install the blfs tools on the currently running system. + + Add custom tools support (see README.CUSTOM) + + MENU "General Settings" + + Build Directory: the name of the root of the LFS system + This is the equivalent of the LFS variable in the book. Set it + to "/mnt/lfs" if you have followed the book for creating the LFS + partition and mount point. + + Retrieve source files: Say y to have jhalfs download the packages + If you say no, you must download the packages yourself and put + them into the /mnt/build_dir/sources directory. Follow book's + chapter 3 instructions. + If you say yes, you'll be asked several other questions: + - Package Archive Directory: Repository of downloaded packages + This directory, which is on the host and should be writable + by the user running the tool, is for storing downloaded packages. + If you keep the default "$SRC_ARCHIVE", you can set this variable + to the absolute path of the repository and export it. Or if the + variable is not set, jhalfs downloads the sources directly to + /mnt/build_dir/sources. + Instead of using the SRC_ARCHIVE envar, you can also enter the + path of the repository directory into this field. + - Retry on 'connection refused' failure: self explanatory + - Number of retry attempts on download failures: self explanatory + - Download timeout (in seconds): self explanatory + + Run the makefile: start the build immediately after running the tool + This is not the preferred method: it is recommended to rather + run "make -C /mnt/build_dir/jhalfs" after the tool has finished + setting up the build. But this may be handy if you are sure everything + is well, and want to leave the tool and the build run without + supervision. + + Rebuild files: clean up the /mnt/build_dir directory + Say n if you want to rerun the tool (to update generated scripts + for example) without removing what has already been done. Otherwise, + say y. Note that there are some guards against removing a directory + containing useful things, but double check that the /mnt/build_dir + directory is really what you want to erase. + + MENU "Build Settings" + + MENU Parallelism settings + - Use all cores: + If you say y, MAKEFLAGS will be set to "-j$(nproc)" at the + beginning of each script. Other envars are supposed to be passed + from the environment, as done in new books. Note that for old books, + this means the scripts using make or ninja will be run with all + cores, but not when this needs to set special envars like + TESTSUITEFLAGS. You can still define the number of cores used + in next field. + If you say n, you'll be asked for a static number of threads + to use. + - set of cpus to use, or 'all' for all cpus (only if using all cores): + You can define here the cores you want to use. See help for + details. This is the preferred way of reducing the number of cores + rather than using a static thread number. + - Number of parallel `make' jobs (only if not using all cores): + Every occurrence of $(nproc) in new books will be replaced with + the number entered here. Also MAKEFLAGS will be set to "-jN" (where + N is the number entered) at the beginning of each scripts. Furthermore + NINJAJOBS will be set to N in the environment. This allows to run all + books with N threads, except for paarts that need other envars to be + set + - Build Binutils pass1 without parallelism (Real SBU) + The standard SBU is defined as the time to run the binutils-pass1 + build with only one thread. Saying y here allows to get a value for + it. If you say n, the value is not meaningful for SBU measurements. + + Run testsuites: say y to run the test suites + You'll have the choice between running all the test suites, or only + those deemed critical (binutils, gmp, mpfr, mpc, and gcc). + + Package management: see README.PACKAGE_MANAGEMENT + + Create a log of installed files for each package: self explanatory + + Strip Installed Binaries/Libraries: use the book instructions for + stripping + + DO NOT use/display progress_bar (self explanatory) + + MENU System configuration + + Use a custom fstab file: + If you say y, you'll have to provide a file containing the fstab + for the LFS system. See above "preliminary tasks". + + Build the kernel: + If you say y, you'll be asked for a file containing the kernel + configuration. See above "preliminary tasks". + + Install non-wide-character ncurses (rarely used nowadays): + If you say y, the system will use instructions in the note on the + ncurses page to install those libraries. + + TimeZone: set to the result of "tzselect" + + Language: set to the result of the instructions on "The Bash Shell + Startup Files" page. + + Install the full set of locales: installs all the locales known to + glibc. + + Groff page size: choice between "A4" and "Letter". + + Hostname: self explanatory + + Network configuration: various fields for setting network. Look at + chapter 9 for background. + + Console configuration: various fields for setting console, as described + in chapter 9. + + MENU Advanced features: + + Optimization: Optimization settings are done by editing files in the + "optimize" directory. The menu just allows you to choose between applying + optimizations only to the final chapter or to all the book. Say n for + a normal build + + Create SBU and disk usage report: self explanatory + + Save temporary system work: self explanatory (see help) + + Run comparison analysis on final stage: build the system several times + using the preceding one, to test whether it is able to rebuild itself + identically. Don't use normally... + + Internal Settings (WARNING: for jhalfs developers only): says it all Once you have set the parameters and saved the configuration, the script is launched. Its aim is to extract instructions from the selected book @@ -275,21 +393,6 @@ boot build method where the final build may be done on a separate machine. - Q. "What is the function of "User account" and "Group account" menu - settings?" - A. If you are running jhalfs from a low or non-privileged account you may - not have the priv to create/delete the user needed to build temporary - tools. - These settings allow you to use your own user and group name to do those - build steps. - - These variables are adjustable also when invoking make: - - cd $BUILDDIR; make LUSER=myaccount LGROUP=mygroup - - The only changes to your account will be the creation of a NEW .bashrc - after saving your original to .bashrc.XXX - Q. "How could I stop the build at a predefined chosen point?" A. Launch the Makefile manually passing the last numbered target to be build as the break point. For example: