From fe30c61a75339c31eb350f8e67f2fb66263e7d3c Mon Sep 17 00:00:00 2001 From: Manuel Canales Esparcia Date: Sat, 11 Nov 2006 19:06:46 +0000 Subject: [PATCH] Splitted common-functions. --- README | 2 +- common/common-functions | 887 +-------------------------------- common/libs/func_blfs_deps | 178 +++++++ common/libs/func_book_parser | 209 ++++++++ common/libs/func_download_pkgs | 189 +++++++ common/libs/func_wrt_Makefile | 320 ++++++++++++ jhalfs | 4 +- 7 files changed, 919 insertions(+), 870 deletions(-) create mode 100644 common/libs/func_blfs_deps create mode 100644 common/libs/func_book_parser create mode 100644 common/libs/func_download_pkgs create mode 100644 common/libs/func_wrt_Makefile diff --git a/README b/README index 6db471b..732b4b1 100644 --- a/README +++ b/README @@ -129,7 +129,7 @@ $Id$ /create-sbu_du-report.sh /progress_bar.sh /blfs-tool-deps/9xx-* - /libs/* + /libs/func_* /contrib/jhalfs-paco.patch diff --git a/common/common-functions b/common/common-functions index 020066d..43ce0b2 100644 --- a/common/common-functions +++ b/common/common-functions @@ -23,387 +23,12 @@ ${DD_BORDER} } -HEADER="# This file is automatically generated by jhalfs -# DO NOT EDIT THIS FILE MANUALLY -# -# Generated on `date \"+%F %X %Z\"`" - - -#------------------------------------------------------# -# NEW Makefile scripting functions # -#------------------------------------------------------# - - -unset get_package_tarball_name -#----------------------------------# -get_package_tarball_name() { # -#----------------------------------# - local script_name=`echo ${1} | sed -e 's@[0-9]\{1\}-@@'` - - # The use of 'head' is necessary to limit the return value to the FIRST match.. - # hopefully this will not cause problems. - # - case $script_name in - tcl) echo $(grep "^tcl" $JHALFSDIR/pkg_tarball_list | head -n1 ) - ;; - linux-headers) - if [ "${PROGNAME}" = "lfs" ]; then - # Uses kernel headers directly - echo $(grep "^linux-[[:digit:]]" $JHALFSDIR/pkg_tarball_list | head -n1 ) - else - # CLFS/HLFS use massaged headers package - echo $(grep "^linux-headers-[[:digit:]]" $JHALFSDIR/pkg_tarball_list | head -n1 ) - fi - ;; - *) echo $(grep "^$script_name-[[:digit:]]" $JHALFSDIR/pkg_tarball_list | head -n1 ) - ;; - esac - -} - - -unset wrt_RunaAsRoot -#----------------------------------# -wrt_RunAsRoot() { # Some scripts must be run as root.. -#----------------------------------# - local MOUNT_ENV - local this_script=$1 - local file=$2 - - case ${PROGNAME} in - lfs ) MOUNT_ENV="LFS" ;; - blfs ) MOUNT_ENV="BLFS" ;; - clfs ) MOUNT_ENV="CLFS" ;; - clfs2 ) MOUNT_ENV="CLFS" ;; - clfs3 ) MOUNT_ENV="CLFS" ;; - hlfs ) MOUNT_ENV="HLFS" ;; - *) echo "undefined progname $PROGNAME"; exit 1 ;; - esac - -( -cat << EOF - @( time { export ${MOUNT_ENV}=\$(MOUNT_PT) && ${PROGNAME}-commands/`dirname $file`/\$@ >>logs/\$@ 2>&1 ; } ) 2>>logs/\$@ && \\ - \$(PRT_DU) >>logs/\$@ -EOF -) >> $MKFILE.tmp -} - - -#------------------------------------------------------# -#------------------------------------------------------# - -#----------------------------------# -ROOT_RunAsRoot() { # -#----------------------------------# - local file=$1 -( -cat << EOF - @( time { source envars && \$(CMDSDIR)/$file >>logs/\$@ 2>&1 ; } ) 2>>logs/\$@ && \\ - \$(PRT_DU_CR) >>logs/\$@ -EOF -) >> $MKFILE.tmp -} - -#----------------------------------# -ROOT_Unpack() { # An alias, for clairity -#----------------------------------# - local FILE=$1 - local optSAVE_PREVIOUS=$2 - - if [[ "${optSAVE_PREVIOUS}" != "1" ]]; then -( -cat << EOF - @\$(call remove_existing_dirs,$FILE) -EOF -) >> $MKFILE.tmp - fi - -( -cat << EOF - @\$(call unpack,$FILE) - @\$(call get_pkg_root_LUSER) -EOF -) >> $MKFILE.tmp -} - -#------------------------------------------------------# -#------------------------------------------------------# - -#----------------------------------# -LUSER_wrt_target() { # Create target and initialize log file -#----------------------------------# - local i=$1 - local PREV=$2 -( -cat << EOF - -$i: $PREV - @\$(call echo_message, Building) - @./progress_bar.sh \$@ \$\$PPID & - @echo "\$(nl_)\`date\`\$(nl_)" >logs/\$@ - @\$(PRT_DU) >>logs/\$@ -EOF -) >> $MKFILE.tmp -} - - -#----------------------------------# -LUSER_wrt_RunAsUser() { # Execute script inside time { }, footer to log file -#----------------------------------# - local file=$1 - -( -cat << EOF - @( time { source ~/.bashrc && \$(CMDSDIR)/`dirname $file`/\$@ >> logs/\$@ 2>&1; } ) 2>> logs/\$@ && \\ - \$(PRT_DU) >> logs/\$@ -EOF -) >> $MKFILE.tmp -} - - -#----------------------------------# -LUSER_wrt_unpack() { # Unpack and set 'ROOT' var -#----------------------------------# - local FILE=$1 - local optSAVE_PREVIOUS=$2 - - if [[ "${optSAVE_PREVIOUS}" != "1" ]]; then -( -cat << EOF - @\$(call remove_existing_dirs,$FILE) -EOF -) >> $MKFILE.tmp - fi - -( -cat << EOF - @\$(call unpack,$FILE) - @\$(call get_pkg_root_LUSER) -EOF -) >> $MKFILE.tmp - -} - - -#----------------------------------# -LUSER_wrt_CopyFstab() { # -#----------------------------------# -( -cat << EOF - @( time { cp -v \$(MOUNT_PT)/sources/fstab \$(MOUNT_PT)/etc/fstab >>logs/\$@ 2>&1 ; } ) 2>>logs/\$@ -EOF -) >> $MKFILE.tmp -} - - -#----------------------------------# -LUSER_wrt_test_log() { # Initialize testsuite log file -#----------------------------------# - local TESTLOGFILE=$1 -( -cat << EOF - @echo "export TEST_LOG=$TESTLOGDIR/$TESTLOGFILE" >> envars && \\ - echo '\$(nl_)\`date\`\$(nl_)' >$TESTLOGDIR/$TESTLOGFILE -EOF -) >> $MKFILE.tmp -} - - -#----------------------------------# -LUSER_RemoveBuildDirs() { # -#----------------------------------# - local name=$1 -( -cat << EOF - @\$(call remove_build_dirs,$name) -EOF -) >> $MKFILE.tmp -} - -#-----------------------------------------------------------------# -#-----------------------------------------------------------------# - -#----------------------------------# -CHROOT_wrt_target() { # Create target and initialize log file -#----------------------------------# - local i=$1 - local PREV=$2 - case $i in - iteration* ) local LOGFILE="${this_script}.log" ;; - * ) local LOGFILE="\$@" ;; - esac -( -cat << EOF - -$i: $PREV - @\$(call echo_message, Building) - @./progress_bar.sh \$@ \$\$PPID & - @echo "\$(nl_)\`date\`\$(nl_)" >logs/$LOGFILE - @\$(PRT_DU_CR) >>logs/$LOGFILE -EOF -) >> $MKFILE.tmp -} - - -#----------------------------------# -CHROOT_Unpack() { # -#----------------------------------# - local FILE=$1 - local optSAVE_PREVIOUS=$2 - - if [ "${optSAVE_PREVIOUS}" != "1" ]; then -( -cat << EOF - @\$(call remove_existing_dirs2,$FILE) -EOF -) >> $MKFILE.tmp - fi -( -cat << EOF - @\$(call unpack3,$FILE) - @\$(call get_pkg_root2) -EOF -) >> $MKFILE.tmp -} - - -#----------------------------------# -CHROOT_wrt_test_log() { # -#----------------------------------# - local TESTLOGFILE=$1 -( -cat << EOF - @echo "export TEST_LOG=/\$(SCRIPT_ROOT)/test-logs/$TESTLOGFILE" >> envars && \\ - echo "\$(nl_)\`date\`\$(nl_)" >test-logs/$TESTLOGFILE -EOF -) >> $MKFILE.tmp -} - - -#----------------------------------# -CHROOT_wrt_RunAsRoot() { # -#----------------------------------# - local file=$1 -( -cat << EOF - @( time { source envars && \$(crCMDSDIR)/$file >>logs/\$@ 2>&1 ; } ) 2>>logs/\$@ && \\ - \$(PRT_DU_CR) >>logs/\$@ -EOF -) >> $MKFILE.tmp - -} - - -#----------------------------------# -CHROOT_wrt_CopyFstab() { # -#----------------------------------# -( -cat << EOF - @( time { cp -v /sources/fstab /etc/fstab >>logs/\$@ 2>&1 ; } ) 2>>logs/\$@ -EOF -) >> $MKFILE.tmp -} - - -#----------------------------------# -CHROOT_wrt_RemoveBuildDirs() { # -#----------------------------------# - local name=$1 -( -cat << EOF - @\$(call remove_build_dirs2,$name) -EOF -) >> $MKFILE.tmp -} - - -unset wrt_touch -#----------------------------------# -wrt_touch() { # -#----------------------------------# -( -cat << EOF - @\$(call housekeeping) -EOF -) >> $MKFILE.tmp -} - - -#----------------------------------# -wrt_blfs_tool_targets() { # -#----------------------------------# - - echo "${tab_}${GREEN}Processing... ${L_arrow}BLFS_TOOL ${R_arrow}" - - for file in blfs-tool-deps/* ; do - # Keep the script file name - this_script=`basename $file` - - # Grab the name of the target - name=`echo ${this_script} | sed -e 's@[0-9]\{3\}-@@'` - - # Find the package. - case $name in - tidy ) pkg_tarball=${TIDY_PKG} ;; - unzip ) pkg_tarball=${UNZIP_PKG} ;; - * ) pkg_tarball=$(get_package_tarball_name $name) ;; - esac - - # Append each name of the script files to a list (this will become - # the names of the targets in the Makefile) - blfs_tool="$blfs_tool ${this_script}" - - #--------------------------------------------------------------------# - # >>>>>>>> START BUILDING A Makefile ENTRY <<<<<<<< # - #--------------------------------------------------------------------# - # - # Drop in the name of the target on a new line, and the previous target - # as a dependency. Also call the echo_message function. - CHROOT_wrt_target "${this_script}" "$PREV" - - # Insert instructions for unpacking the package and changing directories - # DocBook-XML is a zip, the build script will handle that. - [[ ! "$name" = "docbook-xml" ]] && CHROOT_Unpack "$pkg_tarball" - - # Run the script. - CHROOT_wrt_RunAsRoot "$file" - - # Remove the build directory(ies) except if the package build fails. - [[ ! "$name" = "docbook-xml" ]] && CHROOT_wrt_RemoveBuildDirs "$name" - - # Touch the tracking file. - case $name in - tidy ) pkg_ver=tidy-051026 ;; - unzip ) pkg_ver=unzip-5.52 ;; - * ) pkg_ver=$(echo $pkg_tarball | sed -e 's/.tar.*//;s/.tgz//;s/.zip//') ;; - esac - echo "\$(tab_)@touch $TRACKING_DIR/$pkg_ver" >> $MKFILE.tmp - - # Include a touch of the target name so make can check - # if it's already been made. - wrt_touch - # - #--------------------------------------------------------------------# - # >>>>>>>> END OF Makefile ENTRY <<<<<<<< # - #--------------------------------------------------------------------# - - # Keep the script file name for Makefile dependencies. - PREV=${this_script} - done -} - -#------------------------------------------------------# -# END Makefile scripting functions # -#------------------------------------------------------# - - - #----------------------------# run_make() { # #----------------------------# # Test if make must be run. if [ "$RUNMAKE" = "y" ] ; then - # Test to make sure we're running the build as root + # Test to make sure we're not running the build as root if [ "$UID" = "0" ] ; then echo "You must not be logged in as root to build the system." exit 1 @@ -442,503 +67,31 @@ clean_builddir() { # fi } -#----------------------------# -get_book() { # -#----------------------------# - cd $JHALFSDIR +VERBOSITY2=$VERBOSITY - if [ -z $WORKING_COPY ] ; then - # Check for Subversion instead of just letting the script hit 'svn' and fail. - test `type -p svn` || eval "echo \"This feature requires Subversion.\" - exit 1" - echo -n "Downloading the $PROGNAME document, $LFSVRS version... " +[[ $VERBOSITY2 > 0 ]] && echo "" - case $PROGNAME in - lfs) svn_root="LFS" ;; - hlfs) svn_root="HLFS" ;; - clfs) svn_root="cross-lfs" ;; - clfs2) svn_root="cross-lfs" ;; - clfs3) svn_root="cross-lfs" ;; - *) echo "BOOK not defined in function " - exit 1 ;; - esac - # Grab a fresh book if it's missing, otherwise, update it from the - # repo. If we've already extracted the commands, move on to getting the - # sources. - if [ -d ${PROGNAME}-$LFSVRS ] ; then - cd ${PROGNAME}-$LFSVRS - if LC_ALL=C svn up | grep -q At && \ - test -d $JHALFSDIR/${PROGNAME}-commands && \ - test -f $JHALFSDIR/pkg_tarball_list ; then - # Set the canonical book version - echo -ne "done\n" - cd $JHALFSDIR - case $PROGNAME in - clfs | clfs2 | clfs3 ) - VERSION=$(xmllint --noent $BOOK/prologue/$ARCH/bookinfo.xml 2>/dev/null | grep subtitle | sed -e 's/^.*ion //' -e 's/<\/.*//') ;; - *) - VERSION=$(xmllint --noent $BOOK/prologue/bookinfo.xml 2>/dev/null | grep subtitle | sed -e 's/^.*ion //' -e 's/<\/.*//') ;; - esac - get_sources - else - echo -ne "done\n" - extract_commands - fi - else - svn co $SVN/${svn_root}/${TREE} ${PROGNAME}-$LFSVRS >>$LOGDIR/$LOG 2>&1 - echo -ne "done\n" - extract_commands - fi +[[ $VERBOSITY2 > 0 ]] && echo -n "Loading ..." +source $COMMON_DIR/libs/func_book_parser +[[ $? > 0 ]] && echo "file libs/func_book_parser did not load.." && exit 1 +[[ $VERBOSITY2 > 0 ]] && echo "OK" - else - echo -ne "Using $BOOK as book's sources ...\n" - extract_commands - fi - echo -ne " Document version ${L_arrow}${BOLD}${VERSION}${R_arrow}\n" -} -#----------------------------# -extract_commands() { # -#----------------------------# +[[ $VERBOSITY2 > 0 ]] && echo -n "Loading ..." +source $COMMON_DIR/libs/func_download_pkgs +[[ $? > 0 ]] && echo "file libs/func_download_pkgs did not load.." && exit 1 +[[ $VERBOSITY2 > 0 ]] && echo "OK" - # Check for libxslt instead of just letting the script hit 'xsltproc' and fail. - test `type -p xsltproc` || eval "echo \"This feature requires libxslt.\" - exit 1" - cd $JHALFSDIR - case $PROGNAME in - clfs | clfs2 ) - VERSION=$(xmllint --noent $BOOK/prologue/$ARCH/bookinfo.xml 2>/dev/null | grep subtitle | sed -e 's/^.*ion //' -e 's/<\/.*//') ;; - *) - VERSION=$(xmllint --noent $BOOK/prologue/bookinfo.xml 2>/dev/null | grep subtitle | sed -e 's/^.*ion //' -e 's/<\/.*//') ;; - esac +[[ $VERBOSITY2 > 0 ]] && echo -n "Loading ..." +source $COMMON_DIR/libs/func_wrt_Makefile +[[ $? > 0 ]] && echo "file libs/func_wrt_Makefile did not load.." && exit 1 +[[ $VERBOSITY2 > 0 ]] && echo "OK" - # Start clean - if [ -d ${PROGNAME}-commands ]; then - rm -rf ${PROGNAME}-commands - mkdir -v ${PROGNAME}-commands - fi - echo -n "Extracting commands for" - # Dump the commands in shell script form from the HLFS book. - case ${PROGNAME} in - clfs) - echo -n " ${L_arrow}${BOLD}$ARCH${R_arrow} target architecture" - xsltproc --nonet \ - --xinclude \ - --stringparam method $METHOD \ - --stringparam testsuite $TEST \ - --stringparam bomb-testsuite $BOMB_TEST \ - --stringparam vim-lang $VIMLANG \ - --stringparam timezone $TIMEZONE \ - --stringparam page $PAGE \ - --stringparam lang $LANG \ - -o ./${PROGNAME}-commands/ $XSL $BOOK/$ARCH-index.xml >>$LOGDIR/$LOG 2>&1 - ;; +[[ $VERBOSITY2 > 0 ]] && echo -n "Loading ..." +source $COMMON_DIR/libs/func_blfs_deps +[[ $? > 0 ]] && echo "file libs/func_blfs_deps did not load.." && exit 1 +[[ $VERBOSITY2 > 0 ]] && echo "OK" - clfs2) - echo -n " ${L_arrow}${BOLD}$ARCH${R_arrow} target architecture" - xsltproc --nonet \ - --xinclude \ - --stringparam vim-lang $VIMLANG \ - --stringparam timezone $TIMEZONE \ - --stringparam page $PAGE \ - --stringparam lang $LANG \ - -o ./${PROGNAME}-commands/ $XSL $BOOK/$ARCH-index.xml >>$LOGDIR/$LOG 2>&1 - ;; - - clfs3) - echo -n " ${L_arrow}${BOLD}$ARCH${R_arrow} target architecture" - xsltproc --nonet \ - --xinclude \ - --stringparam vim-lang $VIMLANG \ - --stringparam timezone $TIMEZONE \ - --stringparam page $PAGE \ - --stringparam lang $LANG \ - -o ./${PROGNAME}-commands/ $XSL $BOOK/$ARCH-index.xml >>$LOGDIR/$LOG 2>&1 - ;; - - hlfs) - echo -n " ${L_arrow}${BOLD}$MODEL${R_arrow} HLFS libc implementation" - xsltproc --nonet \ - --xinclude \ - --stringparam model $MODEL \ - --stringparam testsuite $TEST \ - --stringparam bomb-testsuite $BOMB_TEST \ - --stringparam timezone $TIMEZONE \ - --stringparam page $PAGE \ - --stringparam lang $LANG \ - --stringparam lc_all $LC_ALL \ - --stringparam grsecurity_host $GRSECURITY_HOST \ - -o ./${PROGNAME}-commands/ $XSL $BOOK/index.xml >>$LOGDIR/$LOG 2>&1 - ;; - lfs) - echo -n " ${L_arrow}${BOLD}LFS${R_arrow} build" - xsltproc --nonet \ - --xinclude \ - --stringparam testsuite $TEST \ - --stringparam bomb-testsuite $BOMB_TEST \ - --stringparam vim-lang $VIMLANG \ - --stringparam timezone $TIMEZONE \ - --stringparam page $PAGE \ - --stringparam lang $LANG \ - -o ./${PROGNAME}-commands/ $XSL $BOOK/index.xml >>$LOGDIR/$LOG 2>&1 - ;; - *) echo -n " ${L_arrow}${BOLD}${PROGNAME}${R_arrow} book invalid, terminate build" - exit 1 ;; - esac - - [[ "${BLFS_TOOL}" = "y" ]] && copy_blfs_deps_scripts - - echo " ...OK" - - # Make the scripts executable. - chmod -R +x $JHALFSDIR/${PROGNAME}-commands - - # Create the packages file. We need it for proper Makefile creation - create_package_list - - # Done. Moving on... - get_sources - -} - -#----------------------------# -create_package_list() { # -#----------------------------# - - # Create the packages file. We need it for proper Makefile creation - rm -f pkg_tarball_list - echo -n "Creating <${PROGNAME}> list of tarball names for $BOOK $ARCH" - case ${PROGNAME} in - clfs | clfs2 | clfs3 ) - xsltproc --nonet --xinclude -o pkg_tarball_list packages.xsl \ - $BOOK/materials/${ARCH}-chapter.xml >>$LOGDIR/$LOG 2>&1 - ;; - hlfs) - xsltproc --nonet --xinclude -o pkg_tarball_list packages.xsl \ - $BOOK/chapter04/chapter04.xml >>$LOGDIR/$LOG 2>&1 - ;; - lfs) - xsltproc --nonet --xinclude -o pkg_tarball_list packages.xsl \ - $BOOK/chapter03/chapter03.xml >>$LOGDIR/$LOG 2>&1 - ;; - esac - - if [[ "${BLFS_TOOL}" = "y" ]] ; then -( -cat << EOF -$LIBXML_PKG -$LIBXSLT_PKG -$TIDY_PKG -$UNZIP_PKG -$DBXML_PKG -$DBXSL_PKG -$LINKS_PKG -$SUDO_PKG -$WGET_PKG -$SVN_PKG -$GPM_PKG -EOF -) >> pkg_tarball_list - fi - - echo " ...OK" - -} - - -#----------------------------# -get_sources() { # Download file, write name to MISSING_FILES.DMP if an error -#----------------------------# - local saveIFS=$IFS - local IFS line URL1 URL2 FILE BOOKMD5 MD5 HAVEMD5 fromARCHIVE - - # Test if the packages must be downloaded - [ ! "$GETPKG" = "y" ] && return - - gs_wrt_message(){ - echo "${RED}$1${OFF}" - echo "$1" >> MISSING_FILES.DMP - } - # Housekeeping - [[ ! -d $BUILDDIR/sources ]] && mkdir $BUILDDIR/sources - cd $BUILDDIR/sources - [[ -f MD5SUMS ]] && rm MD5SUMS - [[ -f MISSING_FILES.DMP ]] && rm MISSING_FILES.DMP - [[ -f urls.lst ]] && rm urls.lst - - # Generate URLs file - create_urls - - IFS=$'\x0A' # Modify the 'internal field separator' to break on 'LF' only - for line in `cat urls.lst`; do - IFS=$saveIFS # Restore the system defaults - - # Skip some packages if they aren't needed - case $line in - */tcl* | */expect* | */dejagnu* | */tree* | */gcc-testsuite* ) - [[ "$TEST" = "0" ]] && continue - ;; - */vim-*-lang* ) - [[ "$VIMLANG" = "0" ]] && continue - ;; - *linux/linux-* ) - [[ -z "$CONFIG" ]] && [[ -z "$BOOT_CONFIG" ]] && \ - [[ "$GETKERNEL" = "n" ]] && continue - ;; - esac - - # Locations - URL1=`echo $line | cut -d" " -f2` # Preferred URL - URL2=`echo $line | cut -d" " -f1` # Fallback Upstream URL - FILE=`basename $URL1` # File name - BOOKMD5=`echo $line | cut -d" " -f3` # MD5 book value - - # Validation pair - MD5="$BOOKMD5 $FILE" - HAVEMD5=1 - - set -e - # If the file exists in the archive copy it to the - # $BUILDDIR/sources dir. MD5SUM will be validated later. - if [ ! -z ${SRC_ARCHIVE} ] && - [ -d ${SRC_ARCHIVE} ] && - [ -f ${SRC_ARCHIVE}/$FILE ]; then - cp ${SRC_ARCHIVE}/$FILE . - echo "$FILE: -- copied from $SRC_ARCHIVE" - fromARCHIVE=1 - else - echo "${BOLD}${YELLOW}$FILE: not found in ${SRC_ARCHIVE}${OFF}" - fromARCHIVE=0 - # If the file does not exist yet in /sources download a fresh one - if [ ! -f $FILE ] ; then - if ! wget $URL1 && ! wget $URL2 ; then - gs_wrt_message "$FILE not found in the SRC_ARCHIVE or on any server..SKIPPING" - continue - fi - fi - fi - - # IF the md5sum does not match the existing files - if ! echo "$MD5" | md5sum -c - >/dev/null ; then - [[ $fromARCHIVE = "1" ]] && echo "${BOLD}${YELLOW}MD5SUM did not match SRC_ARCHIVE copy${OFF}" - [[ $fromARCHIVE = "0" ]] && echo "${BOLD}${YELLOW}MD5SUM did not match REMOTE copy${OFF}" - # Remove the old file and download a new one - rm -fv $FILE - # Force storage in SRC_ARCHIVE - fromARCHIVE=0; - # Try to retrieve again the file. Servers in reverse order. - if ! wget $URL2 && ! wget $URL1 ; then - gs_wrt_message "$FILE not found on the servers.. SKIPPING" - continue - fi - fi - - # Validate the MD5SUM one last time - if ! echo "$MD5" | md5sum -c - >/dev/null ; then - gs_wrt_message "$FILE does not match MD5SUMS value" - # Force generation of MD5SUM - HAVEMD5=0 - fi - - # Generate a fresh MD5SUM for this file - if [[ "$HAVEMD5" = "0" ]] ; then - echo "${BOLD}${YELLOW}Generating a new MD5SUM for ${OFF}$FILE" - echo "NEW MD5SUM: $(md5sum $FILE)" >> MISSING_FILES.DMP - fi - - # Good or bad we write the original md5sum to a file - echo "$MD5" >> MD5SUMS - - # Copy the freshly downloaded file - # to the source archive. - if [ ! -z ${SRC_ARCHIVE} ] && - [ -d ${SRC_ARCHIVE} ] && - [ -w ${SRC_ARCHIVE} ] && - [ "$fromARCHIVE" = "0" ] ; then - echo "Storing file:<$FILE> in the package archive" - cp -f $FILE ${SRC_ARCHIVE} - fi - - done - - if [[ -s MISSING_FILES.DMP ]]; then - echo -e "\n\n${tab_}${RED} One or more files were not retrieved or have bad MD5SUMS.\n${tab_} Check ${L_arrow}$BUILDDIR/sources/MISSING_FILES.DMP${R_arrow} for names ${OFF}\n" - # Do not allow the automatic execution of the Makefile. - echo "${tab_}${BOLD}${RED}*** ${YELLOW}Automatic execution of the generated makefile has been inhibited. ${RED}***${OFF}${nl_}" - RUNMAKE="n" - fi -} - -#----------------------------# -create_urls() { # -#----------------------------# - cd $JHALFSDIR - - case ${PROGNAME} in - clfs) - echo -n "Creating CLFS <${ARCH}> specific URLs file" - xsltproc --nonet --xinclude \ - --stringparam server $SERVER \ - -o $BUILDDIR/sources/urls.lst urls.xsl \ - $BOOK/materials/$ARCH-chapter.xml >>$LOGDIR/$LOG 2>&1 - echo " ...OK" - ;; - clfs2) - echo -n "Creating CLFS2 <${ARCH}> specific URLs file" - xsltproc --nonet --xinclude \ - --stringparam server $SERVER \ - -o $BUILDDIR/sources/urls.lst urls.xsl \ - $BOOK/materials/$ARCH-chapter.xml >>$LOGDIR/$LOG 2>&1 - echo " ...OK" - ;; - clfs3) - echo -n "Creating CLFS3 <${ARCH}> specific URLs file" - xsltproc --nonet --xinclude \ - --stringparam server $SERVER \ - -o $BUILDDIR/sources/urls.lst urls.xsl \ - $BOOK/materials/$ARCH-chapter.xml >>$LOGDIR/$LOG 2>&1 - echo " ...OK" - ;; - hlfs) - echo -n "Creating HLFS <${MODEL}> specific URLs file" - xsltproc --nonet --xinclude \ - --stringparam server $SERVER \ - --stringparam model $MODEL \ - -o $BUILDDIR/sources/urls.lst urls.xsl \ - $BOOK/chapter04/chapter04.xml >>$LOGDIR/$LOG 2>&1 - echo " ...OK" - ;; - lfs) - echo -n "Creating LFS specific URLs file" - xsltproc --nonet --xinclude \ - --stringparam server $SERVER \ - -o ../sources/urls.lst urls.xsl \ - $BOOK/chapter03/chapter03.xml >>$LOGDIR/$LOG 2>&1 - echo " ...OK" - ;; - esac - - cd $BUILDDIR/sources - - if [[ "${BLFS_TOOL}" = "y" ]]; then - add_blfs_deps_urls - fi - - if [[ "${CUSTOM_TOOLS}" = "y" ]]; then - add_CustomToolsURLS - fi - -} - -#----------------------------# Hardcoded URLs and MD5. -add_blfs_deps_urls() { # No easy way to extract it. -#----------------------------# Some FTP mirrors may not work - - local BLFS_SERVER="${SERVER}/pub/blfs/conglomeration/" - - if [[ "${DEP_LIBXML}" = "y" ]] ; then - echo "${LIBXML_URL} ${BLFS_SERVER}libxml2/${LIBXML_PKG} ${LIBXML_MD5}" >> urls.lst - fi - - if [[ "${DEP_LIBXSLT}" = "y" ]] ; then - echo "${LIBXSLT_URL} ${BLFS_SERVER}libxslt/${LIBXSLT_PKG} ${LIBXSLT_MD5}" >> urls.lst - fi - - if [[ "${DEP_TIDY}" = "y" ]] ; then - echo "${TIDY_URL} ${BLFS_SERVER}tidy/${TIDY_PKG} ${TIDY_MD5}" >> urls.lst - echo "${TIDY_PATCH_URL} ${BLFS_SERVER}tidy/${TIDY_PATCH} ${TIDY_PATCH_MD5}" >> urls.lst - echo "${TIDY_DOCS_URL} ${BLFS_SERVER}tidy/${TIDY_DOCS} ${TIDY_DOCS_MD5}" >> urls.lst - fi - - if [[ "${DEP_UNZIP}" = "y" ]] ; then - echo "${UNZIP_URL} ${BLFS_SERVER}unzip/${UNZIP_PKG} ${UNZIP_MD5}" >> urls.lst - fi - - if [[ "${DEP_DBXML}" = "y" ]] ; then - echo "${DBXML_URL} ${BLFS_SERVER}docbook-xml/${DBXML_PKG} ${DBXML_MD5}" >> urls.lst - fi - - if [[ "${DEP_DBXSL}" = "y" ]] ; then - echo "${DBXSL_URL} ${BLFS_SERVER}docbook-xsl/${DBXSL_PKG} ${DBXSL_MD5}" >> urls.lst - fi - - if [[ "${DEP_LINKS}" = "y" ]] ; then - echo "${LINKS_URL} ${BLFS_SERVER}links/${LINKS_PKG} ${LINKS_MD5}" >> urls.lst - fi - - if [[ "${DEP_SUDO}" = "y" ]] ; then - echo "${SUDO_URL} ${BLFS_SERVER}sudo/${SUDO_PKG} ${SUDO_MD5}" >> urls.lst - echo "${SUDO_PATCH_URL} ${BLFS_SERVER}sudo/${SUDO_PATCH} ${SUDO_PATCH_MD5}" >> urls.lst - fi - - if [[ "${DEP_WGET}" = "y" ]] ; then - echo "${WGET_URL} ${BLFS_SERVER}wget/${WGET_PKG} ${WGET_MD5}" >> urls.lst - fi - - if [[ "${DEP_SVN}" = "y" ]] ; then - echo "${SVN_URL} ${BLFS_SERVER}subversion/${SVN_PKG} ${SVN_MD5}" >> urls.lst - fi - - if [[ "${DEP_GPM}" = "y" ]] ; then - echo "${GPM_URL} ${BLFS_SERVER}gpm/${GPM_PKG} ${GPM_MD5}" >> urls.lst - echo "${GPM_PATCH_1_URL} ${BLFS_SERVER}gpm/${GPM_PATCH_1} ${GPM_PATCH_1_MD5}" >> urls.lst - echo "${GPM_PATCH_2_URL} ${BLFS_SERVER}gpm/${GPM_PATCH_2} ${GPM_PATCH_2_MD5}" >> urls.lst - fi - -} - -#----------------------------# Maybe there is a better way to do this, but this -copy_blfs_deps_scripts() { # method avoid to place the test on all -#----------------------------# $PROGNAME/master.sh scripts. - - mkdir -p ${PROGNAME}-commands/blfs-tool-deps - - if [[ "${DEP_LIBXML}" = "y" ]] ; then - mv blfs-tool-deps/901-libxml2 ${PROGNAME}-commands/blfs-tool-deps - fi - - if [[ "${DEP_LIBXSLT}" = "y" ]] ; then - mv blfs-tool-deps/902-libxslt ${PROGNAME}-commands/blfs-tool-deps - fi - - if [[ "${DEP_TIDY}" = "y" ]] ; then - mv blfs-tool-deps/903-tidy ${PROGNAME}-commands/blfs-tool-deps - fi - - if [[ "${DEP_UNZIP}" = "y" ]] ; then - mv blfs-tool-deps/904-unzip ${PROGNAME}-commands/blfs-tool-deps - fi - - if [[ "${DEP_DBXML}" = "y" ]] ; then - mv blfs-tool-deps/905-docbook-xml ${PROGNAME}-commands/blfs-tool-deps - fi - - if [[ "${DEP_DBXSL}" = "y" ]] ; then - mv blfs-tool-deps/906-docbook-xsl ${PROGNAME}-commands/blfs-tool-deps - fi - - if [[ "${DEP_LINKS}" = "y" ]] ; then - mv blfs-tool-deps/908-links ${PROGNAME}-commands/blfs-tool-deps - fi - - if [[ "${DEP_SUDO}" = "y" ]] ; then - mv blfs-tool-deps/909-sudo ${PROGNAME}-commands/blfs-tool-deps - fi - - if [[ "${DEP_WGET}" = "y" ]] ; then - mv blfs-tool-deps/910-wget ${PROGNAME}-commands/blfs-tool-deps - fi - - if [[ "${DEP_SVN}" = "y" ]] ; then - mv blfs-tool-deps/911-subversion ${PROGNAME}-commands/blfs-tool-deps - fi - - if [[ "${DEP_GPM}" = "y" ]] ; then - mv blfs-tool-deps/907-gpm ${PROGNAME}-commands/blfs-tool-deps - fi - - rm -rf blfs-tool-deps - -} +[[ $VERBOSITY2 > 0 ]] && echo -n " ..." \ No newline at end of file diff --git a/common/libs/func_blfs_deps b/common/libs/func_blfs_deps new file mode 100644 index 0000000..b82b9b6 --- /dev/null +++ b/common/libs/func_blfs_deps @@ -0,0 +1,178 @@ +#!/bin/bash + +# $Id$ + +#----------------------------# Hardcoded URLs and MD5. +add_blfs_deps_urls() { # No easy way to extract it. +#----------------------------# Some FTP mirrors may not work + + local BLFS_SERVER="${SERVER}/pub/blfs/conglomeration/" + + if [[ "${DEP_LIBXML}" = "y" ]] ; then + echo "${LIBXML_URL} ${BLFS_SERVER}libxml2/${LIBXML_PKG} ${LIBXML_MD5}" >> urls.lst + fi + + if [[ "${DEP_LIBXSLT}" = "y" ]] ; then + echo "${LIBXSLT_URL} ${BLFS_SERVER}libxslt/${LIBXSLT_PKG} ${LIBXSLT_MD5}" >> urls.lst + fi + + if [[ "${DEP_TIDY}" = "y" ]] ; then + echo "${TIDY_URL} ${BLFS_SERVER}tidy/${TIDY_PKG} ${TIDY_MD5}" >> urls.lst + echo "${TIDY_PATCH_URL} ${BLFS_SERVER}tidy/${TIDY_PATCH} ${TIDY_PATCH_MD5}" >> urls.lst + echo "${TIDY_DOCS_URL} ${BLFS_SERVER}tidy/${TIDY_DOCS} ${TIDY_DOCS_MD5}" >> urls.lst + fi + + if [[ "${DEP_UNZIP}" = "y" ]] ; then + echo "${UNZIP_URL} ${BLFS_SERVER}unzip/${UNZIP_PKG} ${UNZIP_MD5}" >> urls.lst + fi + + if [[ "${DEP_DBXML}" = "y" ]] ; then + echo "${DBXML_URL} ${BLFS_SERVER}docbook-xml/${DBXML_PKG} ${DBXML_MD5}" >> urls.lst + fi + + if [[ "${DEP_DBXSL}" = "y" ]] ; then + echo "${DBXSL_URL} ${BLFS_SERVER}docbook-xsl/${DBXSL_PKG} ${DBXSL_MD5}" >> urls.lst + fi + + if [[ "${DEP_LINKS}" = "y" ]] ; then + echo "${LINKS_URL} ${BLFS_SERVER}links/${LINKS_PKG} ${LINKS_MD5}" >> urls.lst + fi + + if [[ "${DEP_SUDO}" = "y" ]] ; then + echo "${SUDO_URL} ${BLFS_SERVER}sudo/${SUDO_PKG} ${SUDO_MD5}" >> urls.lst + echo "${SUDO_PATCH_URL} ${BLFS_SERVER}sudo/${SUDO_PATCH} ${SUDO_PATCH_MD5}" >> urls.lst + fi + + if [[ "${DEP_WGET}" = "y" ]] ; then + echo "${WGET_URL} ${BLFS_SERVER}wget/${WGET_PKG} ${WGET_MD5}" >> urls.lst + fi + + if [[ "${DEP_SVN}" = "y" ]] ; then + echo "${SVN_URL} ${BLFS_SERVER}subversion/${SVN_PKG} ${SVN_MD5}" >> urls.lst + fi + + if [[ "${DEP_GPM}" = "y" ]] ; then + echo "${GPM_URL} ${BLFS_SERVER}gpm/${GPM_PKG} ${GPM_MD5}" >> urls.lst + echo "${GPM_PATCH_1_URL} ${BLFS_SERVER}gpm/${GPM_PATCH_1} ${GPM_PATCH_1_MD5}" >> urls.lst + echo "${GPM_PATCH_2_URL} ${BLFS_SERVER}gpm/${GPM_PATCH_2} ${GPM_PATCH_2_MD5}" >> urls.lst + fi + +} + +#----------------------------# Maybe there is a better way to do this, but this +copy_blfs_deps_scripts() { # method avoid to place the test on all +#----------------------------# $PROGNAME/master.sh scripts. + + mkdir -p ${PROGNAME}-commands/blfs-tool-deps + + if [[ "${DEP_LIBXML}" = "y" ]] ; then + mv blfs-tool-deps/901-libxml2 ${PROGNAME}-commands/blfs-tool-deps + fi + + if [[ "${DEP_LIBXSLT}" = "y" ]] ; then + mv blfs-tool-deps/902-libxslt ${PROGNAME}-commands/blfs-tool-deps + fi + + if [[ "${DEP_TIDY}" = "y" ]] ; then + mv blfs-tool-deps/903-tidy ${PROGNAME}-commands/blfs-tool-deps + fi + + if [[ "${DEP_UNZIP}" = "y" ]] ; then + mv blfs-tool-deps/904-unzip ${PROGNAME}-commands/blfs-tool-deps + fi + + if [[ "${DEP_DBXML}" = "y" ]] ; then + mv blfs-tool-deps/905-docbook-xml ${PROGNAME}-commands/blfs-tool-deps + fi + + if [[ "${DEP_DBXSL}" = "y" ]] ; then + mv blfs-tool-deps/906-docbook-xsl ${PROGNAME}-commands/blfs-tool-deps + fi + + if [[ "${DEP_LINKS}" = "y" ]] ; then + mv blfs-tool-deps/908-links ${PROGNAME}-commands/blfs-tool-deps + fi + + if [[ "${DEP_SUDO}" = "y" ]] ; then + mv blfs-tool-deps/909-sudo ${PROGNAME}-commands/blfs-tool-deps + fi + + if [[ "${DEP_WGET}" = "y" ]] ; then + mv blfs-tool-deps/910-wget ${PROGNAME}-commands/blfs-tool-deps + fi + + if [[ "${DEP_SVN}" = "y" ]] ; then + mv blfs-tool-deps/911-subversion ${PROGNAME}-commands/blfs-tool-deps + fi + + if [[ "${DEP_GPM}" = "y" ]] ; then + mv blfs-tool-deps/907-gpm ${PROGNAME}-commands/blfs-tool-deps + fi + + rm -rf blfs-tool-deps + +} + + +#----------------------------------# +wrt_blfs_tool_targets() { # +#----------------------------------# + + echo "${tab_}${GREEN}Processing... ${L_arrow}BLFS_TOOL ${R_arrow}" + + for file in blfs-tool-deps/* ; do + # Keep the script file name + this_script=`basename $file` + + # Grab the name of the target + name=`echo ${this_script} | sed -e 's@[0-9]\{3\}-@@'` + + # Find the package. + case $name in + tidy ) pkg_tarball=${TIDY_PKG} ;; + unzip ) pkg_tarball=${UNZIP_PKG} ;; + * ) pkg_tarball=$(get_package_tarball_name $name) ;; + esac + + # Append each name of the script files to a list (this will become + # the names of the targets in the Makefile) + blfs_tool="$blfs_tool ${this_script}" + + #--------------------------------------------------------------------# + # >>>>>>>> START BUILDING A Makefile ENTRY <<<<<<<< # + #--------------------------------------------------------------------# + # + # Drop in the name of the target on a new line, and the previous target + # as a dependency. Also call the echo_message function. + CHROOT_wrt_target "${this_script}" "$PREV" + + # Insert instructions for unpacking the package and changing directories + # DocBook-XML is a zip, the build script will handle that. + [[ ! "$name" = "docbook-xml" ]] && CHROOT_Unpack "$pkg_tarball" + + # Run the script. + CHROOT_wrt_RunAsRoot "$file" + + # Remove the build directory(ies) except if the package build fails. + [[ ! "$name" = "docbook-xml" ]] && CHROOT_wrt_RemoveBuildDirs "$name" + + # Touch the tracking file. + case $name in + tidy ) pkg_ver=tidy-051026 ;; + unzip ) pkg_ver=unzip-5.52 ;; + * ) pkg_ver=$(echo $pkg_tarball | sed -e 's/.tar.*//;s/.tgz//;s/.zip//') ;; + esac + echo "\$(tab_)@touch $TRACKING_DIR/$pkg_ver" >> $MKFILE.tmp + + # Include a touch of the target name so make can check + # if it's already been made. + wrt_touch + # + #--------------------------------------------------------------------# + # >>>>>>>> END OF Makefile ENTRY <<<<<<<< # + #--------------------------------------------------------------------# + + # Keep the script file name for Makefile dependencies. + PREV=${this_script} + done +} diff --git a/common/libs/func_book_parser b/common/libs/func_book_parser new file mode 100644 index 0000000..00abc0f --- /dev/null +++ b/common/libs/func_book_parser @@ -0,0 +1,209 @@ +#!/bin/bash + +# $Id$ + +#----------------------------# +get_book() { # +#----------------------------# + cd $JHALFSDIR + + if [ -z $WORKING_COPY ] ; then + # Check for Subversion instead of just letting the script hit 'svn' and fail. + test `type -p svn` || eval "echo \"This feature requires Subversion.\" + exit 1" + echo -n "Downloading the $PROGNAME document, $LFSVRS version... " + + case $PROGNAME in + lfs) svn_root="LFS" ;; + hlfs) svn_root="HLFS" ;; + clfs) svn_root="cross-lfs" ;; + clfs2) svn_root="cross-lfs" ;; + clfs3) svn_root="cross-lfs" ;; + *) echo "BOOK not defined in function " + exit 1 ;; + esac + # Grab a fresh book if it's missing, otherwise, update it from the + # repo. If we've already extracted the commands, move on to getting the + # sources. + if [ -d ${PROGNAME}-$LFSVRS ] ; then + cd ${PROGNAME}-$LFSVRS + if LC_ALL=C svn up | grep -q At && \ + test -d $JHALFSDIR/${PROGNAME}-commands && \ + test -f $JHALFSDIR/pkg_tarball_list ; then + # Set the canonical book version + echo -ne "done\n" + cd $JHALFSDIR + case $PROGNAME in + clfs | clfs2 | clfs3 ) + VERSION=$(xmllint --noent $BOOK/prologue/$ARCH/bookinfo.xml 2>/dev/null | grep subtitle | sed -e 's/^.*ion //' -e 's/<\/.*//') ;; + *) + VERSION=$(xmllint --noent $BOOK/prologue/bookinfo.xml 2>/dev/null | grep subtitle | sed -e 's/^.*ion //' -e 's/<\/.*//') ;; + esac + get_sources + else + echo -ne "done\n" + extract_commands + fi + else + svn co $SVN/${svn_root}/${TREE} ${PROGNAME}-$LFSVRS >>$LOGDIR/$LOG 2>&1 + echo -ne "done\n" + extract_commands + fi + + else + echo -ne "Using $BOOK as book's sources ...\n" + extract_commands + fi + echo -ne " Document version ${L_arrow}${BOLD}${VERSION}${R_arrow}\n" +} + +#----------------------------# +extract_commands() { # +#----------------------------# + + # Check for libxslt instead of just letting the script hit 'xsltproc' and fail. + test `type -p xsltproc` || eval "echo \"This feature requires libxslt.\" + exit 1" + + cd $JHALFSDIR + case $PROGNAME in + clfs | clfs2 ) + VERSION=$(xmllint --noent $BOOK/prologue/$ARCH/bookinfo.xml 2>/dev/null | grep subtitle | sed -e 's/^.*ion //' -e 's/<\/.*//') ;; + *) + VERSION=$(xmllint --noent $BOOK/prologue/bookinfo.xml 2>/dev/null | grep subtitle | sed -e 's/^.*ion //' -e 's/<\/.*//') ;; + esac + + # Start clean + if [ -d ${PROGNAME}-commands ]; then + rm -rf ${PROGNAME}-commands + mkdir -v ${PROGNAME}-commands + fi + echo -n "Extracting commands for" + + # Dump the commands in shell script form from the HLFS book. + case ${PROGNAME} in + clfs) + echo -n " ${L_arrow}${BOLD}$ARCH${R_arrow} target architecture" + xsltproc --nonet \ + --xinclude \ + --stringparam method $METHOD \ + --stringparam testsuite $TEST \ + --stringparam bomb-testsuite $BOMB_TEST \ + --stringparam vim-lang $VIMLANG \ + --stringparam timezone $TIMEZONE \ + --stringparam page $PAGE \ + --stringparam lang $LANG \ + -o ./${PROGNAME}-commands/ $XSL $BOOK/$ARCH-index.xml >>$LOGDIR/$LOG 2>&1 + ;; + + clfs2) + echo -n " ${L_arrow}${BOLD}$ARCH${R_arrow} target architecture" + xsltproc --nonet \ + --xinclude \ + --stringparam vim-lang $VIMLANG \ + --stringparam timezone $TIMEZONE \ + --stringparam page $PAGE \ + --stringparam lang $LANG \ + -o ./${PROGNAME}-commands/ $XSL $BOOK/$ARCH-index.xml >>$LOGDIR/$LOG 2>&1 + ;; + + clfs3) + echo -n " ${L_arrow}${BOLD}$ARCH${R_arrow} target architecture" + xsltproc --nonet \ + --xinclude \ + --stringparam vim-lang $VIMLANG \ + --stringparam timezone $TIMEZONE \ + --stringparam page $PAGE \ + --stringparam lang $LANG \ + -o ./${PROGNAME}-commands/ $XSL $BOOK/$ARCH-index.xml >>$LOGDIR/$LOG 2>&1 + ;; + + hlfs) + echo -n " ${L_arrow}${BOLD}$MODEL${R_arrow} HLFS libc implementation" + xsltproc --nonet \ + --xinclude \ + --stringparam model $MODEL \ + --stringparam testsuite $TEST \ + --stringparam bomb-testsuite $BOMB_TEST \ + --stringparam timezone $TIMEZONE \ + --stringparam page $PAGE \ + --stringparam lang $LANG \ + --stringparam lc_all $LC_ALL \ + --stringparam grsecurity_host $GRSECURITY_HOST \ + -o ./${PROGNAME}-commands/ $XSL $BOOK/index.xml >>$LOGDIR/$LOG 2>&1 + ;; + lfs) + echo -n " ${L_arrow}${BOLD}LFS${R_arrow} build" + xsltproc --nonet \ + --xinclude \ + --stringparam testsuite $TEST \ + --stringparam bomb-testsuite $BOMB_TEST \ + --stringparam vim-lang $VIMLANG \ + --stringparam timezone $TIMEZONE \ + --stringparam page $PAGE \ + --stringparam lang $LANG \ + -o ./${PROGNAME}-commands/ $XSL $BOOK/index.xml >>$LOGDIR/$LOG 2>&1 + ;; + *) echo -n " ${L_arrow}${BOLD}${PROGNAME}${R_arrow} book invalid, terminate build" + exit 1 ;; + esac + + [[ "${BLFS_TOOL}" = "y" ]] && copy_blfs_deps_scripts + + echo " ...OK" + + # Make the scripts executable. + chmod -R +x $JHALFSDIR/${PROGNAME}-commands + + # Create the packages file. We need it for proper Makefile creation + create_package_list + + # Done. Moving on... + get_sources + +} + +#----------------------------# +create_package_list() { # +#----------------------------# + + # Create the packages file. We need it for proper Makefile creation + rm -f pkg_tarball_list + echo -n "Creating <${PROGNAME}> list of tarball names for $BOOK $ARCH" + case ${PROGNAME} in + clfs | clfs2 | clfs3 ) + xsltproc --nonet --xinclude -o pkg_tarball_list packages.xsl \ + $BOOK/materials/${ARCH}-chapter.xml >>$LOGDIR/$LOG 2>&1 + ;; + hlfs) + xsltproc --nonet --xinclude -o pkg_tarball_list packages.xsl \ + $BOOK/chapter04/chapter04.xml >>$LOGDIR/$LOG 2>&1 + ;; + lfs) + xsltproc --nonet --xinclude -o pkg_tarball_list packages.xsl \ + $BOOK/chapter03/chapter03.xml >>$LOGDIR/$LOG 2>&1 + ;; + esac + + if [[ "${BLFS_TOOL}" = "y" ]] ; then +( +cat << EOF +$LIBXML_PKG +$LIBXSLT_PKG +$TIDY_PKG +$UNZIP_PKG +$DBXML_PKG +$DBXSL_PKG +$LINKS_PKG +$SUDO_PKG +$WGET_PKG +$SVN_PKG +$GPM_PKG +EOF +) >> pkg_tarball_list + fi + + echo " ...OK" + +} + diff --git a/common/libs/func_download_pkgs b/common/libs/func_download_pkgs new file mode 100644 index 0000000..989fad1 --- /dev/null +++ b/common/libs/func_download_pkgs @@ -0,0 +1,189 @@ +#!/bin/bash + +# $Id$ + + +#----------------------------# +get_sources() { # Download file, write name to MISSING_FILES.DMP if an error +#----------------------------# + local saveIFS=$IFS + local IFS line URL1 URL2 FILE BOOKMD5 MD5 HAVEMD5 fromARCHIVE + + # Test if the packages must be downloaded + [ ! "$GETPKG" = "y" ] && return + + gs_wrt_message(){ + echo "${RED}$1${OFF}" + echo "$1" >> MISSING_FILES.DMP + } + # Housekeeping + [[ ! -d $BUILDDIR/sources ]] && mkdir $BUILDDIR/sources + cd $BUILDDIR/sources + [[ -f MD5SUMS ]] && rm MD5SUMS + [[ -f MISSING_FILES.DMP ]] && rm MISSING_FILES.DMP + [[ -f urls.lst ]] && rm urls.lst + + # Generate URLs file + create_urls + + IFS=$'\x0A' # Modify the 'internal field separator' to break on 'LF' only + for line in `cat urls.lst`; do + IFS=$saveIFS # Restore the system defaults + + # Skip some packages if they aren't needed + case $line in + */tcl* | */expect* | */dejagnu* | */tree* | */gcc-testsuite* ) + [[ "$TEST" = "0" ]] && continue + ;; + */vim-*-lang* ) + [[ "$VIMLANG" = "0" ]] && continue + ;; + *linux/linux-* ) + [[ -z "$CONFIG" ]] && [[ -z "$BOOT_CONFIG" ]] && \ + [[ "$GETKERNEL" = "n" ]] && continue + ;; + esac + + # Locations + URL1=`echo $line | cut -d" " -f2` # Preferred URL + URL2=`echo $line | cut -d" " -f1` # Fallback Upstream URL + FILE=`basename $URL1` # File name + BOOKMD5=`echo $line | cut -d" " -f3` # MD5 book value + + # Validation pair + MD5="$BOOKMD5 $FILE" + HAVEMD5=1 + + set -e + # If the file exists in the archive copy it to the + # $BUILDDIR/sources dir. MD5SUM will be validated later. + if [ ! -z ${SRC_ARCHIVE} ] && + [ -d ${SRC_ARCHIVE} ] && + [ -f ${SRC_ARCHIVE}/$FILE ]; then + cp ${SRC_ARCHIVE}/$FILE . + echo "$FILE: -- copied from $SRC_ARCHIVE" + fromARCHIVE=1 + else + echo "${BOLD}${YELLOW}$FILE: not found in ${SRC_ARCHIVE}${OFF}" + fromARCHIVE=0 + # If the file does not exist yet in /sources download a fresh one + if [ ! -f $FILE ] ; then + if ! wget $URL1 && ! wget $URL2 ; then + gs_wrt_message "$FILE not found in the SRC_ARCHIVE or on any server..SKIPPING" + continue + fi + fi + fi + + # IF the md5sum does not match the existing files + if ! echo "$MD5" | md5sum -c - >/dev/null ; then + [[ $fromARCHIVE = "1" ]] && echo "${BOLD}${YELLOW}MD5SUM did not match SRC_ARCHIVE copy${OFF}" + [[ $fromARCHIVE = "0" ]] && echo "${BOLD}${YELLOW}MD5SUM did not match REMOTE copy${OFF}" + # Remove the old file and download a new one + rm -fv $FILE + # Force storage in SRC_ARCHIVE + fromARCHIVE=0; + # Try to retrieve again the file. Servers in reverse order. + if ! wget $URL2 && ! wget $URL1 ; then + gs_wrt_message "$FILE not found on the servers.. SKIPPING" + continue + fi + fi + + # Validate the MD5SUM one last time + if ! echo "$MD5" | md5sum -c - >/dev/null ; then + gs_wrt_message "$FILE does not match MD5SUMS value" + # Force generation of MD5SUM + HAVEMD5=0 + fi + + # Generate a fresh MD5SUM for this file + if [[ "$HAVEMD5" = "0" ]] ; then + echo "${BOLD}${YELLOW}Generating a new MD5SUM for ${OFF}$FILE" + echo "NEW MD5SUM: $(md5sum $FILE)" >> MISSING_FILES.DMP + fi + + # Good or bad we write the original md5sum to a file + echo "$MD5" >> MD5SUMS + + # Copy the freshly downloaded file + # to the source archive. + if [ ! -z ${SRC_ARCHIVE} ] && + [ -d ${SRC_ARCHIVE} ] && + [ -w ${SRC_ARCHIVE} ] && + [ "$fromARCHIVE" = "0" ] ; then + echo "Storing file:<$FILE> in the package archive" + cp -f $FILE ${SRC_ARCHIVE} + fi + + done + + if [[ -s MISSING_FILES.DMP ]]; then + echo -e "\n\n${tab_}${RED} One or more files were not retrieved or have bad MD5SUMS.\n${tab_} Check ${L_arrow}$BUILDDIR/sources/MISSING_FILES.DMP${R_arrow} for names ${OFF}\n" + # Do not allow the automatic execution of the Makefile. + echo "${tab_}${BOLD}${RED}*** ${YELLOW}Automatic execution of the generated makefile has been inhibited. ${RED}***${OFF}${nl_}" + RUNMAKE="n" + fi +} + +#----------------------------# +create_urls() { # +#----------------------------# + cd $JHALFSDIR + + case ${PROGNAME} in + clfs) + echo -n "Creating CLFS <${ARCH}> specific URLs file" + xsltproc --nonet --xinclude \ + --stringparam server $SERVER \ + -o $BUILDDIR/sources/urls.lst urls.xsl \ + $BOOK/materials/$ARCH-chapter.xml >>$LOGDIR/$LOG 2>&1 + echo " ...OK" + ;; + clfs2) + echo -n "Creating CLFS2 <${ARCH}> specific URLs file" + xsltproc --nonet --xinclude \ + --stringparam server $SERVER \ + -o $BUILDDIR/sources/urls.lst urls.xsl \ + $BOOK/materials/$ARCH-chapter.xml >>$LOGDIR/$LOG 2>&1 + echo " ...OK" + ;; + clfs3) + echo -n "Creating CLFS3 <${ARCH}> specific URLs file" + xsltproc --nonet --xinclude \ + --stringparam server $SERVER \ + -o $BUILDDIR/sources/urls.lst urls.xsl \ + $BOOK/materials/$ARCH-chapter.xml >>$LOGDIR/$LOG 2>&1 + echo " ...OK" + ;; + hlfs) + echo -n "Creating HLFS <${MODEL}> specific URLs file" + xsltproc --nonet --xinclude \ + --stringparam server $SERVER \ + --stringparam model $MODEL \ + -o $BUILDDIR/sources/urls.lst urls.xsl \ + $BOOK/chapter04/chapter04.xml >>$LOGDIR/$LOG 2>&1 + echo " ...OK" + ;; + lfs) + echo -n "Creating LFS specific URLs file" + xsltproc --nonet --xinclude \ + --stringparam server $SERVER \ + -o ../sources/urls.lst urls.xsl \ + $BOOK/chapter03/chapter03.xml >>$LOGDIR/$LOG 2>&1 + echo " ...OK" + ;; + esac + + cd $BUILDDIR/sources + + if [[ "${BLFS_TOOL}" = "y" ]]; then + add_blfs_deps_urls + fi + + if [[ "${CUSTOM_TOOLS}" = "y" ]]; then + add_CustomToolsURLS + fi + +} + diff --git a/common/libs/func_wrt_Makefile b/common/libs/func_wrt_Makefile new file mode 100644 index 0000000..ab68443 --- /dev/null +++ b/common/libs/func_wrt_Makefile @@ -0,0 +1,320 @@ +#!/bin/bash + +# $Id$ + +#=== MAKEFILE HEADER === + +HEADER="# This file is automatically generated by jhalfs +# DO NOT EDIT THIS FILE MANUALLY +# +# Generated on `date \"+%F %X %Z\"`" + +#======================= + + +#==== PACKAGES NAMES === + +#----------------------------------# +get_package_tarball_name() { # +#----------------------------------# + local script_name=`echo ${1} | sed -e 's@[0-9]\{1\}-@@'` + + # The use of 'head' is necessary to limit the return value to the FIRST match.. + # hopefully this will not cause problems. + # + case $script_name in + tcl) echo $(grep "^tcl" $JHALFSDIR/pkg_tarball_list | head -n1 ) + ;; + linux-headers) + if [ "${PROGNAME}" = "lfs" ]; then + # Uses kernel headers directly + echo $(grep "^linux-[[:digit:]]" $JHALFSDIR/pkg_tarball_list | head -n1 ) + else + # CLFS/HLFS use massaged headers package + echo $(grep "^linux-headers-[[:digit:]]" $JHALFSDIR/pkg_tarball_list | head -n1 ) + fi + ;; + *) echo $(grep "^$script_name-[[:digit:]]" $JHALFSDIR/pkg_tarball_list | head -n1 ) + ;; + esac + +} + +#======================= + + +#==== TARGET HEADER ==== + +#----------------------------------# +LUSER_wrt_target() { # Create target and initialize log file +#----------------------------------# + local i=$1 + local PREV=$2 +( +cat << EOF + +$i: $PREV + @\$(call echo_message, Building) + @./progress_bar.sh \$@ \$\$PPID & + @echo "\$(nl_)\`date\`\$(nl_)" >logs/\$@ + @\$(PRT_DU) >>logs/\$@ +EOF +) >> $MKFILE.tmp +} + +#----------------------------------# +CHROOT_wrt_target() { # Create target and initialize log file +#----------------------------------# + local i=$1 + local PREV=$2 + case $i in + iteration* ) local LOGFILE="${this_script}.log" ;; + * ) local LOGFILE="\$@" ;; + esac +( +cat << EOF + +$i: $PREV + @\$(call echo_message, Building) + @./progress_bar.sh \$@ \$\$PPID & + @echo "\$(nl_)\`date\`\$(nl_)" >logs/$LOGFILE + @\$(PRT_DU_CR) >>logs/$LOGFILE +EOF +) >> $MKFILE.tmp +} + +#======================= + + +#======== UNPACK ======= + +#----------------------------------# +ROOT_Unpack() { # An alias, for clairity +#----------------------------------# + local FILE=$1 + local optSAVE_PREVIOUS=$2 + + if [[ "${optSAVE_PREVIOUS}" != "1" ]]; then +( +cat << EOF + @\$(call remove_existing_dirs,$FILE) +EOF +) >> $MKFILE.tmp + fi + +( +cat << EOF + @\$(call unpack,$FILE) + @\$(call get_pkg_root_LUSER) +EOF +) >> $MKFILE.tmp +} + +#----------------------------------# +LUSER_wrt_unpack() { # Unpack and set 'ROOT' var +#----------------------------------# + local FILE=$1 + local optSAVE_PREVIOUS=$2 + + if [[ "${optSAVE_PREVIOUS}" != "1" ]]; then +( +cat << EOF + @\$(call remove_existing_dirs,$FILE) +EOF +) >> $MKFILE.tmp + fi + +( +cat << EOF + @\$(call unpack,$FILE) + @\$(call get_pkg_root_LUSER) +EOF +) >> $MKFILE.tmp +} + +#----------------------------------# +CHROOT_Unpack() { # +#----------------------------------# + local FILE=$1 + local optSAVE_PREVIOUS=$2 + + if [ "${optSAVE_PREVIOUS}" != "1" ]; then +( +cat << EOF + @\$(call remove_existing_dirs2,$FILE) +EOF +) >> $MKFILE.tmp + fi +( +cat << EOF + @\$(call unpack3,$FILE) + @\$(call get_pkg_root2) +EOF +) >> $MKFILE.tmp +} + +#======================= + + +#===== TESTS LOGS ====== + +#----------------------------------# +LUSER_wrt_test_log() { # Initialize testsuite log file +#----------------------------------# + local TESTLOGFILE=$1 +( +cat << EOF + @echo "export TEST_LOG=$TESTLOGDIR/$TESTLOGFILE" >> envars && \\ + echo '\$(nl_)\`date\`\$(nl_)' >$TESTLOGDIR/$TESTLOGFILE +EOF +) >> $MKFILE.tmp +} + +#----------------------------------# +CHROOT_wrt_test_log() { # +#----------------------------------# + local TESTLOGFILE=$1 +( +cat << EOF + @echo "export TEST_LOG=/\$(SCRIPT_ROOT)/test-logs/$TESTLOGFILE" >> envars && \\ + echo "\$(nl_)\`date\`\$(nl_)" >test-logs/$TESTLOGFILE +EOF +) >> $MKFILE.tmp +} + +#======================= + + +#======== RUN AS ======= + +#----------------------------------# +wrt_RunAsRoot() { # Some scripts must be run as root.. +#----------------------------------# + local MOUNT_ENV + local this_script=$1 + local file=$2 + + case ${PROGNAME} in + lfs ) MOUNT_ENV="LFS" ;; + blfs ) MOUNT_ENV="BLFS" ;; + clfs ) MOUNT_ENV="CLFS" ;; + clfs2 ) MOUNT_ENV="CLFS" ;; + clfs3 ) MOUNT_ENV="CLFS" ;; + hlfs ) MOUNT_ENV="HLFS" ;; + *) echo "undefined progname $PROGNAME"; exit 1 ;; + esac + +( +cat << EOF + @( time { export ${MOUNT_ENV}=\$(MOUNT_PT) && ${PROGNAME}-commands/`dirname $file`/\$@ >>logs/\$@ 2>&1 ; } ) 2>>logs/\$@ && \\ + \$(PRT_DU) >>logs/\$@ +EOF +) >> $MKFILE.tmp +} + +#----------------------------------# +ROOT_RunAsRoot() { # +#----------------------------------# + local file=$1 +( +cat << EOF + @( time { source envars && \$(CMDSDIR)/$file >>logs/\$@ 2>&1 ; } ) 2>>logs/\$@ && \\ + \$(PRT_DU_CR) >>logs/\$@ +EOF +) >> $MKFILE.tmp +} + +#----------------------------------# +LUSER_wrt_RunAsUser() { # Execute script inside time { }, footer to log file +#----------------------------------# + local file=$1 + +( +cat << EOF + @( time { source ~/.bashrc && \$(CMDSDIR)/`dirname $file`/\$@ >> logs/\$@ 2>&1; } ) 2>> logs/\$@ && \\ + \$(PRT_DU) >> logs/\$@ +EOF +) >> $MKFILE.tmp +} + +#----------------------------------# +CHROOT_wrt_RunAsRoot() { # +#----------------------------------# + local file=$1 +( +cat << EOF + @( time { source envars && \$(crCMDSDIR)/$file >>logs/\$@ 2>&1 ; } ) 2>>logs/\$@ && \\ + \$(PRT_DU_CR) >>logs/\$@ +EOF +) >> $MKFILE.tmp + +} + +#======================= + + +#====== COPY FSTAB ===== + +#----------------------------------# +LUSER_wrt_CopyFstab() { # +#----------------------------------# +( +cat << EOF + @( time { cp -v \$(MOUNT_PT)/sources/fstab \$(MOUNT_PT)/etc/fstab >>logs/\$@ 2>&1 ; } ) 2>>logs/\$@ +EOF +) >> $MKFILE.tmp +} + +#----------------------------------# +CHROOT_wrt_CopyFstab() { # +#----------------------------------# +( +cat << EOF + @( time { cp -v /sources/fstab /etc/fstab >>logs/\$@ 2>&1 ; } ) 2>>logs/\$@ +EOF +) >> $MKFILE.tmp +} + +#======================= + + +#==== RM BUILD DIRS ==== + +#----------------------------------# +LUSER_RemoveBuildDirs() { # +#----------------------------------# + local name=$1 +( +cat << EOF + @\$(call remove_build_dirs,$name) +EOF +) >> $MKFILE.tmp +} + +#----------------------------------# +CHROOT_wrt_RemoveBuildDirs() { # +#----------------------------------# + local name=$1 +( +cat << EOF + @\$(call remove_build_dirs2,$name) +EOF +) >> $MKFILE.tmp +} + +#======================= + + +#======== TOUCH ======== + +#----------------------------------# +wrt_touch() { # +#----------------------------------# +( +cat << EOF + @\$(call housekeeping) +EOF +) >> $MKFILE.tmp +} + +#======================= diff --git a/jhalfs b/jhalfs index 562ac6a..bba312d 100755 --- a/jhalfs +++ b/jhalfs @@ -264,8 +264,8 @@ source $COMMON_DIR/libs/func_validate_configs.sh [[ $? > 0 ]] && echo " function module did not load.." && exit 2 [[ $VERBOSITY > 0 ]] && echo "OK" -[[ $VERBOSITY > 0 ]] && echo -n "Loading function ..." -source $COMMON_DIR/libs/custom_pkgs +[[ $VERBOSITY > 0 ]] && echo -n "Loading function ..." +source $COMMON_DIR/libs/func_custom_pkgs [[ $? > 0 ]] && echo " function module did not load.." && exit 2 [[ $VERBOSITY > 0 ]] && echo "OK"