From 45f827182973f04af51ea8992f754ffa408a2373 Mon Sep 17 00:00:00 2001 From: Manuel Canales Esparcia Date: Sat, 29 Apr 2006 14:44:29 +0000 Subject: [PATCH] Merged ICA/farce support from experimental branch. --- CLFS/master.sh | 143 +++-- HLFS/master.sh | 67 ++- LFS/master.sh | 66 ++- common/common-functions | 9 + common/config | 23 +- common/func_ICA.sh | 179 ------- common/func_compare.sh | 117 +++++ common/func_validate_configs.sh | 280 +++++----- extras/do_copy_files | 36 ++ extras/do_ica_prep | 77 +++ extras/do_ica_work | 55 ++ extras/farce | 896 ++++++++++++++++++++++++++++++++ extras/filelist | 60 +++ master.sh | 35 +- 14 files changed, 1658 insertions(+), 385 deletions(-) delete mode 100644 common/func_ICA.sh create mode 100644 common/func_compare.sh create mode 100755 extras/do_copy_files create mode 100755 extras/do_ica_prep create mode 100755 extras/do_ica_work create mode 100755 extras/farce create mode 100755 extras/filelist diff --git a/CLFS/master.sh b/CLFS/master.sh index 0603c5b..6bdc38b 100755 --- a/CLFS/master.sh +++ b/CLFS/master.sh @@ -461,22 +461,42 @@ bm_testsuite_tools_Makefiles() { # #-----------------------------# final_system_Makefiles() { # #-----------------------------# - echo "${tab_}${GREEN}Processing... ${L_arrow}(chroot) final system${R_arrow}" + # Set envars and scripts for iteration targets + LOGS="" # Start with an empty global LOGS envar + if [[ -z "$1" ]] ; then + local N="" + else + local N=-build_$1 + local basicsystem="" + mkdir final-system$N + cp final-system/* final-system$N + for script in final-system$N/* ; do + # Overwrite existing symlinks, files, and dirs + sed -e 's/ln -sv/&f/g' \ + -e 's/mv -v/&f/g' \ + -e 's/mkdir -v/&p/g' -i ${script} + done + # Remove Bzip2 binaries before make install + sed -e 's@make install@rm -vf /usr/bin/bz*\n&@' -i final-system$N/*-bzip2 + # Fix how Module-Init-Tools do the install target + sed -e 's@make install@make INSTALL=install install@' -i final-system$N/*-module-init-tools + # Delete *old Readline libraries just after make install + sed -e 's@make install@&\nrm -v /lib/lib{history,readline}*old@' -i final-system$N/*-readline + fi - for file in final-system/* ; do + echo "${tab_}${GREEN}Processing... ${L_arrow}(chroot) final system$N${R_arrow}" + + for file in final-system$N/* ; do # Keep the script file name this_script=`basename $file` - # Test if the stripping phase must be skipped + # Test if the stripping phase must be skipped. + # Skip alsp temp-perl for iterative runs case $this_script in - *stripping*) [[ "$STRIP" = "0" ]] && continue - ;; + *stripping*) [[ "$STRIP" = "0" ]] && continue ;; + *temp-perl*) [[ -n "$N" ]] && continue ;; esac - # First append each name of the script files to a list (this will become - # the names of the targets in the Makefile - basicsystem="$basicsystem $this_script" - # Grab the name of the target, strip id number, XXX-script name=`echo $this_script | sed -e 's@[0-9]\{3\}-@@' \ -e 's@temp-@@' \ @@ -485,22 +505,38 @@ final_system_Makefiles() { # -e 's@64@@' \ -e 's@n32@@'` + # Find the version of the command files, if it corresponds with the building of + # a specific package. We need this here to can skip scripts not needed for + # iterations rebuilds vrs=`grep "^$name-version" $JHALFSDIR/packages | sed -e 's/.* //' -e 's/"//g'` + if [[ "$vrs" = "" ]] && [[ -n "$N" ]] ; then + case "${this_script}" in + *stripping*) ;; + *) continue ;; + esac + fi + + # Append each name of the script files to a list (this will become + # the names of the targets in the Makefile + basicsystem="$basicsystem ${this_script}${N}" + + # Append each name of the script files to a list (this will become + # the names of the logs to be moved for each iteration) + LOGS="$LOGS ${this_script}" + #--------------------------------------------------------------------# # >>>>>>>> START BUILDING A Makefile ENTRY <<<<<<<< # #--------------------------------------------------------------------# # # Drop in the name of the target on a new line, and the previous target # as a dependency. Also call the echo_message function. - wrt_target "${this_script}" "$PREV" + wrt_target "${this_script}${N}" "$PREV" # If $vrs isn't empty, we've got a package... if [ "$vrs" != "" ] ; then - case $name in - temp-perl) wrt_unpack2 "perl-$vrs.tar.*" ;; - *) wrt_unpack2 "$name-$vrs.tar.*" ;; - esac + FILE="$name-$vrs.tar.*" + wrt_unpack2 "$FILE" fi # wrt_run_as_chroot1 "${this_script}" "${file}" @@ -515,8 +551,9 @@ final_system_Makefiles() { # #--------------------------------------------------------------------# # # Keep the script file name for Makefile dependencies. - PREV=$this_script - + PREV=${this_script}${N} + # Set system_build envar for iteration targets + system_build=$basicsystem done # for file in final-system/* ... } @@ -524,25 +561,44 @@ final_system_Makefiles() { # #-----------------------------# bm_final_system_Makefiles() { # #-----------------------------# - echo "${tab_}${GREEN}Processing... ${L_arrow}(boot) final system${R_arrow}" + # Set envars and scripts for iteration targets + LOGS="" # Start with an empty global LOGS envar + if [[ -z "$1" ]] ; then + local N="" + # The makesys phase was initiated in bm_testsuite_tools_makefile + [[ "$TEST" = 0 ]] && PREV="" + else + local N=-build_$1 + local basicsystem="" + mkdir final-system$N + cp final-system/* final-system$N + for script in final-system$N/* ; do + # Overwrite existing symlinks, files, and dirs + sed -e 's/ln -sv/&f/g' \ + -e 's/mv -v/&f/g' \ + -e 's/mkdir -v/&p/g' -i ${script} + done + # Remove Bzip2 binaries before make install + sed -e 's@make install@rm -vf /usr/bin/bz*\n&@' -i final-system$N/*-bzip2 + # Fix how Module-Init-Tools do the install target + sed -e 's@make install@make INSTALL=install install@' -i final-system$N/*-module-init-tools + # Delete *old Readline libraries just after make install + sed -e 's@make install@&\nrm -v /lib/lib{history,readline}*old@' -i final-system$N/*-readline + fi - # The makesys phase was initiated in bm_testsuite_tools_makefile - [[ "$TEST" = 0 ]] && PREV="" + echo "${tab_}${GREEN}Processing... ${L_arrow}(boot) final system$N${R_arrow}" - for file in final-system/* ; do + for file in final-system$N/* ; do # Keep the script file name this_script=`basename $file` # Test if the stripping phase must be skipped + # Skip alsp temp-perl for iterative runs case $this_script in - *stripping*) [[ "$STRIP" = "0" ]] && continue - ;; + *stripping*) [[ "$STRIP" = "0" ]] && continue ;; + *temp-perl*) [[ -n "$N" ]] && continue ;; esac - # First append each name of the script files to a list (this will become - # the names of the targets in the Makefile - basicsystem="$basicsystem $this_script" - # Grab the name of the target, strip id number, XXX-script name=`echo $this_script | sed -e 's@[0-9]\{3\}-@@' \ -e 's@temp-@@' \ @@ -551,22 +607,38 @@ bm_final_system_Makefiles() { # -e 's@64@@' \ -e 's@n32@@'` + # Find the version of the command files, if it corresponds with the building of + # a specific package. We need this here to can skip scripts not needed for + # iterations rebuilds vrs=`grep "^$name-version" $JHALFSDIR/packages | sed -e 's/.* //' -e 's/"//g'` + if [[ "$vrs" = "" ]] && [[ -n "$N" ]] ; then + case "${this_script}" in + *stripping*) ;; + *) continue ;; + esac + fi + + # Append each name of the script files to a list (this will become + # the names of the targets in the Makefile + basicsystem="$basicsystem ${this_script}${N}" + + # Append each name of the script files to a list (this will become + # the names of the logs to be moved for each iteration) + LOGS="$LOGS ${this_script}" + #--------------------------------------------------------------------# # >>>>>>>> START BUILDING A Makefile ENTRY <<<<<<<< # #--------------------------------------------------------------------# # # Drop in the name of the target on a new line, and the previous target # as a dependency. Also call the echo_message function. - wrt_target "${this_script}" "$PREV" + wrt_target "${this_script}${N}" "$PREV" # If $vrs isn't empty, we've got a package... if [ "$vrs" != "" ] ; then - case $name in - temp-perl) wrt_unpack3 "perl-$vrs.tar.*" ;; - *) wrt_unpack3 "$name-$vrs.tar.*" ;; - esac + FILE="$name-$vrs.tar.*" + wrt_unpack3 "$FILE" fi # wrt_run_as_root2 "${this_script}" "${file}" @@ -581,8 +653,9 @@ bm_final_system_Makefiles() { # #--------------------------------------------------------------------# # # Keep the script file name for Makefile dependencies. - PREV=$this_script - + PREV=${this_script}${N} + # Set system_build envar for iteration targets + system_build=$basicsystem done # for file in final-system/* ... } @@ -885,6 +958,8 @@ build_Makefile() { # Construct a Makefile from the book scripts testsuite_tools_Makefiles # $testsuitetools fi final_system_Makefiles # $basicsystem + # Add the iterations targets, if needed + [[ "$COMPARE" != "0" ]] && wrt_compare_targets bootscripts_Makefiles # $bootscripttools bootable_Makefiles # $bootabletools else @@ -893,6 +968,8 @@ build_Makefile() { # Construct a Makefile from the book scripts bm_testsuite_tools_Makefiles # $testsuitetools fi bm_final_system_Makefiles # $basicsystem + # Add the iterations targets, if needed + [[ "$COMPARE" != "0" ]] && wrt_compare_targets bm_bootscripts_Makefiles # $bootscipttools bm_bootable_Makefiles # $bootabletoosl fi diff --git a/HLFS/master.sh b/HLFS/master.sh index a929a58..8a3956e 100755 --- a/HLFS/master.sh +++ b/HLFS/master.sh @@ -295,8 +295,31 @@ chapter6_Makefiles() { # sysroot or chroot build phase local TARGET LOADER local file local this_script + # Set envars and scripts for iteration targets + LOGS="" # Start with an empty global LOGS envar + if [[ -z "$1" ]] ; then + local N="" + else + local N=-build_$1 + local chapter6="" + mkdir chapter06$N + cp chapter06/* chapter06$N + for script in chapter06$N/* ; do + # Overwrite existing symlinks, files, and dirs + sed -e 's/ln -s /ln -sf /g' \ + -e 's/^mv /&-f/g' -i ${script} + done + # Remove Bzip2 binaries before make install + sed -e 's@make install@rm -vf /usr/bin/bz*\n&@' -i chapter06$N/*-bzip2 + # Fix how Module-Init-Tools do the install target + sed -e 's@make install@make INSTALL=install install@' -i chapter06$N/*-module-init-tools + # Delete *old Readline libraries just after make install + sed -e 's@make install@&\nrm -v /lib/lib{history,readline}*old@' -i chapter06$N/*-readline + # Don't readd already existing groups + sed -e '/groupadd/d' -i chapter06$N/*-udev + fi - echo "${tab_}${GREEN}Processing... ${L_arrow}Chapter6${R_arrow}" + echo "${tab_}${GREEN}Processing... ${L_arrow}Chapter6$N${R_arrow}" # # Set these definitions early and only once # @@ -306,7 +329,7 @@ chapter6_Makefiles() { # sysroot or chroot build phase TARGET="pc-linux-gnu"; LOADER="ld-linux.so.2" fi - for file in chapter06/* ; do + for file in chapter06$N/* ; do # Keep the script file name this_script=`basename $file` @@ -317,26 +340,40 @@ chapter6_Makefiles() { # sysroot or chroot build phase *chroot* ) continue ;; # Test if the stripping phase must be skipped *-stripping* ) [[ "$STRIP" = "0" ]] && continue ;; - *) ;; esac - # First append each name of the script files to a list (this will become - # the names of the targets in the Makefile - chapter6="$chapter6 $this_script" - # Grab the name of the target name=`echo $this_script | sed -e 's@[0-9]\{3\}-@@'` + # Find the version of the command files, if it corresponds with the building of + # a specific package + vrs=`grep "^$name-version" $JHALFSDIR/packages | sed -e 's/.* //' -e 's/"//g'` + + if [[ "$vrs" = "" ]] && [[ -n "$N" ]] ; then + case "${this_script}" in + *stripping*) ;; + *) continue ;; + esac + fi + + # Append each name of the script files to a list (this will become + # the names of the targets in the Makefile + chapter6="$chapter6 ${this_script}${N}" + + # Append each name of the script files to a list (this will become + # the names of the logs to be moved for each iteration) + LOGS="$LOGS ${this_script}" + # # Sed replacement to fix some rm command that could fail. # That should be fixed in the book sources. # case $name in glibc) - sed 's/rm /rm -f /' -i chapter06/$this_script + sed 's/rm /rm -f /' -i chapter06$N/$this_script ;; gcc) - sed 's/rm /rm -f /' -i chapter06/$this_script + sed 's/rm /rm -f /' -i chapter06$N/$this_script ;; esac @@ -346,11 +383,7 @@ chapter6_Makefiles() { # sysroot or chroot build phase # # Drop in the name of the target on a new line, and the previous target # as a dependency. Also call the echo_message function. - wrt_target "$this_script" "$PREV" - - # Find the version of the command files, if it corresponds with the building of - # a specific package - vrs=`grep "^$name-version" $JHALFSDIR/packages | sed -e 's/.* //' -e 's/"//g'` + wrt_target "${this_script}${N}" "$PREV" # If $vrs isn't empty, we've got a package... # Insert instructions for unpacking the package and changing directories @@ -408,7 +441,9 @@ EOF #--------------------------------------------------------------------# # Keep the script file name for Makefile dependencies. - PREV=$this_script + PREV=${this_script}${N} + # Set system_build envar for iteration targets + system_build=$chapter6 done # end for file in chapter06/* } @@ -518,6 +553,8 @@ build_Makefile() { # Construct a Makefile from the book scripts chapter3_Makefiles chapter5_Makefiles chapter6_Makefiles + # Add the iterations targets, if needed + [[ "$COMPARE" != "0" ]] && wrt_compare_targets chapter7_Makefiles # Add a header, some variables and include the function file diff --git a/LFS/master.sh b/LFS/master.sh index 144a3b0..9ab8a92 100755 --- a/LFS/master.sh +++ b/LFS/master.sh @@ -144,9 +144,35 @@ chapter5_Makefiles() { #----------------------------# chapter6_Makefiles() { #----------------------------# - echo "${tab_}${GREEN}Processing... ${L_arrow}Chapter6${R_arrow}" + # Set envars and scripts for iteration targets + LOGS="" # Start with an empty global LOGS envar + if [[ -z "$1" ]] ; then + local N="" + else + local N=-build_$1 + local chapter6="" + mkdir chapter06$N + cp chapter06/* chapter06$N + for script in chapter06$N/* ; do + # Overwrite existing symlinks, files, and dirs + sed -e 's/ln -sv/&f/g' \ + -e 's/mv -v/&f/g' \ + -e 's/mkdir -v/&p/g' -i ${script} + done + # Remove Bzip2 binaries before make install + sed -e 's@make install@rm -vf /usr/bin/bz*\n&@' -i chapter06$N/*-bzip2 + # Fix how Module-Init-Tools do the install target + sed -e 's@make install@make INSTALL=install install@' -i chapter06$N/*-module-init-tools + # Delete *old Readline libraries just after make install + sed -e 's@make install@&\nrm -v /lib/lib{history,readline}*old@' -i chapter06$N/*-readline + # Let some Udev pre-installation commands to fail + sed -e 's@/lib/udev/devices/fd@& || true@' \ + -e 's/mknod -m.*/& || true/' -i chapter06$N/*-udev + fi - for file in chapter06/* ; do + echo "${tab_}${GREEN}Processing... ${L_arrow}Chapter6$N${R_arrow}" + + for file in chapter06$N/* ; do # Keep the script file name this_script=`basename $file` @@ -157,24 +183,36 @@ chapter6_Makefiles() { *stripping*) [[ "${STRIP}" = "0" ]] && continue ;; esac - # First append each name of the script files to a list (this will become - # the names of the targets in the Makefile - chapter6="$chapter6 ${this_script}" - # Grab the name of the target name=`echo ${this_script} | sed -e 's@[0-9]\{3\}-@@'` + # Find the version of the command files, if it corresponds with the building of + # a specific package. We need this here to can skip scripts not needed for + # iterations rebuilds + vrs=`grep "^$name-version" $JHALFSDIR/packages | sed -e 's/.* //' -e 's/"//g'` + + if [[ "$vrs" = "" ]] && [[ -n "$N" ]] ; then + case "${this_script}" in + *stripping*) ;; + *) continue ;; + esac + fi + + # Append each name of the script files to a list (this will become + # the names of the targets in the Makefile) + chapter6="$chapter6 ${this_script}${N}" + + # Append each name of the script files to a list (this will become + # the names of the logs to be moved for each iteration) + LOGS="$LOGS ${this_script}" + #--------------------------------------------------------------------# # >>>>>>>> START BUILDING A Makefile ENTRY <<<<<<<< # #--------------------------------------------------------------------# # # Drop in the name of the target on a new line, and the previous target # as a dependency. Also call the echo_message function. - wrt_target "${this_script}" "$PREV" - - # Find the version of the command files, if it corresponds with the building of - # a specific package - vrs=`grep "^$name-version" $JHALFSDIR/packages | sed -e 's/.* //' -e 's/"//g'` + wrt_target "${this_script}${N}" "$PREV" # If $vrs isn't empty, we've got a package... # Insert instructions for unpacking the package and changing directories @@ -204,7 +242,9 @@ chapter6_Makefiles() { #--------------------------------------------------------------------# # Keep the script file name for Makefile dependencies. - PREV=${this_script} + PREV=${this_script}${N} + # Set system_build envar for iteration targets + system_build=$chapter6 done # end for file in chapter06/* } @@ -304,6 +344,8 @@ build_Makefile() { chapter4_Makefiles chapter5_Makefiles chapter6_Makefiles + # Add the iterations targets, if needed + [[ "$COMPARE" != "0" ]] && wrt_compare_targets chapter789_Makefiles diff --git a/common/common-functions b/common/common-functions index 7744d3d..7778e5d 100644 --- a/common/common-functions +++ b/common/common-functions @@ -76,6 +76,15 @@ ${BOLD} -W, --working-copy DIR${OFF} [[ ${PROGNAME} != "blfs" ]] && cat <<- -EOF- +${BOLD} -C, --comparasion TYPE${OFF} + do iterative comparison analysis. That will take several time due that + the final system packages wil be rebuilded the times defined in the + ITERATIONS config option. + Analisys types allowed are: + ICA = do the ICA analisys designed by Greg Schafer + farce = do the farce analisys designed by Ken Moffat + both = perfom both ICA and farce analisys + ${BOLD} -F, --fstab FILE${OFF} use FILE as the /etc/fstab file for the ${BOLD}$(echo $PROGNAME | tr [a-z] [A-Z])${OFF} system. If not specified, a default /etc/fstab file with dummy values is created. diff --git a/common/config b/common/config index 4c73501..67d8f46 100644 --- a/common/config +++ b/common/config @@ -62,15 +62,18 @@ LC_ALL=$LC_ALL # (not used in LFS) # Not used in LFS KEYMAP=none -#=== Variables needed by ICA (Not implemented yet :-/)=== -#--- Run ICA testing 0/1 0(no)/1(yes) +#=== Variables needed by iterative comparison analysis === +#--- Should some iterative comparison analysis by made? 0(no)/1(yes) +COMPARE=0 + +#--- The number of final stage builds to create and compare, min. 2, max. 5 +ITERATIONS=3 + +#--- Run ICA testing 0(no)/1(yes) RUN_ICA=0 -#--- The number of final stage builds to create and compare -ITERATIONS= - -#--- ICA report log directory -ICALOGDIR=$JHALFSDIR/logs/ICA +#--- Run farce testing 0(no)/1(yes) +RUN_FARCE=0 #==== INTERNAL VARIABLES ==== # Don't edit it unless you know what you are doing @@ -78,3 +81,9 @@ ICALOGDIR=$JHALFSDIR/logs/ICA #--- Working directories JHALFSDIR=$BUILDDIR/jhalfs LOGDIR=$JHALFSDIR/logs + +#--- ICA report log directory +ICALOGDIR=$LOGDIR/ICA + +#--- farce report log directory +FARCELOGDIR=$LOGDIR/farce diff --git a/common/func_ICA.sh b/common/func_ICA.sh deleted file mode 100644 index c648517..0000000 --- a/common/func_ICA.sh +++ /dev/null @@ -1,179 +0,0 @@ -# $Id$ - -# Acknowledgment: -# The following code is a modified version of an original work written by -# Greg Schafer for the "DIY Linux" project and is included here with his -# permission. -# ref: http://www.diy-linux.org -# -# -# ---------------------------------------------------------------------------- # -# Here are the ICA functions. -# ---------------------------------------------------------------------------- # -# -# Here we prepare for the Iterative Comparison Analysis (ICA). Essentially, we -# copy most of our chroot phase files to a new location then perform some -# manipulations on the copied files to make diff comparisons easier. The steps -# involved are:- -# (1) copy the whole tree (minus the PRUNEPATH defined below) to the CMP_DIR -# location. Use tar as it seems like the most appropriate tool for copying -# large directory trees around. -# (2) delete all symlinks. -# (3) gunzip all `*.gz' files. -# (4) delete all hardlinked files (of course trying to leaving the "master" -# intact) -# (5) convert all `*.a' files (ar archives) into a directory of the same name -# containing the unpacked object files. -# (6) fully strip the whole lot (but being careful to strip only the debug -# symbols from object `*.o' files). - -#----------------------------------# -do_ica_prep() { # -#----------------------------------# -: <> $TMP_FILE - done - - echo -n "Copying files to ${CMP_DIR}... " - cd / - tar -X $TMP_FILE -cf - . | tar -C $CMP_DIR -xf - || { - echo -e "\n\n${RED}ERROR:${OFF} tar copy failed!\n" >&2 - exit 1 - } - echo "done." - rm -f $TMP_FILE - - echo -n "Removing symbolic links in ${CMP_DIR}... " - find $CMP_DIR -type l | xargs rm -f - echo "done." - - echo -n "Gunzipping \".gz\" files in ${CMP_DIR}... " - find $CMP_DIR -name '*.gz' | xargs gunzip - echo "done." - - # This was a bit tricky. You'll probably have to do it by hand - # to see what's actually going on. The "sort/uniq" part is - # inspired from the example DirCmp script from the book "Shell - # Programming Examples" by Bruce Blinn, published by Prentice - # Hall. We are essentially using the `-ls' option of the find - # utility to allow manipulations based on inode numbers. - # - # FIXME - this is a bit unreliable - rem out for now - #echo -n "Removing hardlinked file copies in ${CMP_DIR}... " - #find $CMP_DIR -ls | sort -n > $ALL_FILES - #find $CMP_DIR -ls | sort -n -u > $UNIQUE_FILES - #cat $UNIQUE_FILES $ALL_FILES | sort | uniq -u | awk '{ print $11 }' | xargs rm -f - #rm -f $ALL_FILES $UNIQUE_FILES - #echo "done." - - # ar archives contain date & time stamp info that causes us - # grief when trying to find differences. Here we perform some - # hackery to allow easy diffing. Essentially, replace each - # archive with a dir of the same name and extract the object - # files from the archive into this dir. Despite their names, - # libieee.a & libmcheck.a are not actual ar archives. - # - echo -n "Extracting object files from \".a\" files in ${CMP_DIR}... " - L=$(find $CMP_DIR -name '*.a' ! -name 'libieee.a' ! -name 'libmcheck.a') - - for F in $L; do - mv $F ${F}.XX - mkdir $F - cd $F - BN=${F##*/} - ar x ../${BN}.XX || { - echo -e "\n\n${RED}ERROR:${OFF} ar archive extraction failed!\n" >&2 - exit 1 - } - rm -f ../${BN}.XX - done - echo "done." - - echo -n "Stripping (debug) symbols from \".o\" files in ${CMP_DIR}... " - find $CMP_DIR -name '*.o' | xargs strip -p -g 2>/dev/null - echo "done." - - echo -n "Stripping (all) symbols from files OTHER THAN \".o\" files in ${CMP_DIR}... " - find $CMP_DIR ! -name '*.o' | xargs strip -p 2>/dev/null || : - echo "done." - - echo -e "\n${CYAN}[ICA] - ICA preparation for Iteration ${ITER}\c" - echo -e " complete.${OFF}\n${BORDER}" - do_stamp icaprep - fi -} - - -#----------------------------------# -do_ica_work() { # Do the ICA grunt work. -#----------------------------------# -: < $RAWDIFF || : - echo "done." - - echo -e "The list of binary files that differ:\n" > $REPORT - grep "iles.*differ$" $RAWDIFF >> $REPORT - echo -e "\n" >> $REPORT - - echo -e "The list of files that exist \"only in\" 1 of the directories:\n" >> $REPORT - - if grep "^Only in" $RAWDIFF >/dev/null 2>&1; then - grep "^Only in" $RAWDIFF >> $REPORT - else - echo NONE >> $REPORT - fi - - grep -v "iles.*differ$" $RAWDIFF | grep -v "^Only in" > ${SCRATCH_DIR}/logs/${1}V${2}.ASCII.DIFF - rm -f $RAWDIFF - -} diff --git a/common/func_compare.sh b/common/func_compare.sh new file mode 100644 index 0000000..e4ff4f7 --- /dev/null +++ b/common/func_compare.sh @@ -0,0 +1,117 @@ +# $Id$ + +#----------------------------------# +wrt_compare_targets() { # +#----------------------------------# + + for ((N=1; N <= ITERATIONS ; N++)) ; do # Double parentheses, + # and "ITERATIONS" with no "$". + ITERATION=iteration-$N + if [ "$N" != "1" ] ; then + wrt_system_build "$N" "$PREV_IT" + fi + this_script=$ITERATION + wrt_target "$ITERATION" "$PREV" + wrt_compare_work "$ITERATION" "$PREV_IT" + wrt_logs "$N" + PREV_IT=$ITERATION + PREV=$ITERATION + done +} + +#----------------------------------# +wrt_system_build() { # +#----------------------------------# + local RUN=$1 + local PREV_IT=$2 + + if [[ "$PROGNAME" = "clfs" ]] && [[ "$METHOD" = "chroot" ]] ; then + final_system_Makefiles $RUN + elif [[ "$PROGNAME" = "clfs" ]] && [[ "$METHOD" = "boot" ]] ; then + bm_final_system_Makefiles $RUN + else + chapter6_Makefiles $RUN + fi + + echo -e "\nsystem_build_$RUN: $PREV_IT $system_build" >> $MKFILE.tmp + PREV=system_build_$RUN + +} + +#----------------------------------# +wrt_compare_work() { # +#----------------------------------# + local ITERATION=$1 + local PREV_IT=$2 + local PRUNEPATH="/dev /home /jhalfs /lost+found /media /mnt /opt /proc \ +/sources /root /srv /sys /tmp /tools /usr/local /usr/src /var/log/paco" + + if [[ "$PROGNAME" = "clfs" ]] && [[ "$METHOD" = "boot" ]] ; then + local ROOT_DIR=/ + local DEST_TOPDIR=/jhalfs + local ICALOGDIR=/jhalfs/logs/ICA + local FARCELOGDIR=/jhalfs/logs/farce + else + local ROOT_DIR=$BUILDDIR + local DEST_TOPDIR=$BUILDDIR/jhalfs + fi + + if [[ "$RUN_ICA" = "1" ]] ; then + local DEST_ICA=$DEST_TOPDIR/ICA && \ +( + cat << EOF + @extras/do_copy_files "$PRUNEPATH" $ROOT_DIR $DEST_ICA/$ITERATION >>logs/$ITERATION.log 2>&1 && \\ + extras/do_ica_prep $DEST_ICA/$ITERATION >>logs/$ITERATION.log 2>&1 +EOF +) >> $MKFILE.tmp + if [[ "$ITERATION" != "iteration-1" ]] ; then + wrt_do_ica_work "$PREV_IT" "$ITERATION" "$DEST_ICA" + fi + fi + + if [[ "$RUN_FARCE" = "1" ]] ; then + local DEST_FARCE=$DEST_TOPDIR/farce && \ +( + cat << EOF + @extras/do_copy_files "$PRUNEPATH" $ROOT_DIR $DEST_FARCE/$ITERATION >>logs/$ITERATION.log 2>&1 && \\ + extras/filelist $DEST_FARCE/$ITERATION $DEST_FARCE/filelist-$ITERATION >>logs/$ITERATION.log 2>&1 +EOF +) >> $MKFILE.tmp + if [[ "$ITERATION" != "iteration-1" ]] ; then + wrt_do_farce_work "$PREV_IT" "$ITERATION" "$DEST_FARCE" + fi + fi +} + +#----------------------------------# +wrt_do_ica_work() { # +#----------------------------------# + echo -e "\t@extras/do_ica_work $1 $2 $ICALOGDIR $3 >>logs/$ITERATION.log 2>&1" >> $MKFILE.tmp +} + +#----------------------------------# +wrt_do_farce_work() { # +#----------------------------------# + local OUTPUT=$FARCELOGDIR/${1}_V_${2} + local PREDIR=$3/$1 + local PREFILE=$3/filelist-$1 + local ITEDIR=$3/$2 + local ITEFILE=$3/filelist-$2 + echo -e "\t@extras/farce --directory $OUTPUT $PREDIR $PREFILE $ITEDIR $ITEFILE >>logs/$ITERATION.log 2>&1" >> $MKFILE.tmp +} + +#----------------------------------# +wrt_logs() { # +#----------------------------------# + local ITERATION=iteration-$1 + +( + cat << EOF + @pushd logs 1> /dev/null && \\ + mkdir $ITERATION && \\ + mv ${LOGS} $ITERATION && \\ + popd 1> /dev/null + @touch \$@ +EOF +) >> $MKFILE.tmp +} diff --git a/common/func_validate_configs.sh b/common/func_validate_configs.sh index cef4e03..801750b 100644 --- a/common/func_validate_configs.sh +++ b/common/func_validate_configs.sh @@ -1,7 +1,8 @@ # $Id$ -validate_target() { - +#----------------------------# +validate_target() { # +#----------------------------# local -r ERROR_MSG_pt1='The variable \"${L_arrow}TARGET${R_arrow}\" value ${L_arrow}${BOLD}${TARGET}${R_arrow} is invalid for the ${L_arrow}${BOLD}${ARCH}${R_arrow} architecture' local -r ERROR_MSG_pt2=' check the config file ${BOLD}${GREEN}\<$(echo $PROGNAME | tr [a-z] [A-Z])/config\> or \${OFF}' @@ -67,7 +68,7 @@ validate_target() { #----------------------------# -validate_config() { # Are the config values sane (within reason) +validate_config() { # Are the config values sane (within reason) #----------------------------# : < or root mount point - echo -e "`eval echo $PARAM_VALS`" - if [[ "xx x/x" =~ "x${!config_param}x" ]]; then - write_error_and_die - fi - continue ;; - TIMEZONE) continue;; - MKFILE) continue;; - HPKG) validation_str="x0x x1x"; validate_str; continue ;; - RUNMAKE) validation_str="x0x x1x"; validate_str; continue ;; - TEST) validation_str="x0x x1x x2x x3x"; validate_str; continue ;; - REPORT) validation_str="x0x x1x"; validate_str; - if [[ "${!config_param}" = "1" ]]; then - if [[ `type -p bc` ]]; then - continue - else - echo -e " ${BOLD}The bc binary was not found${OFF}" - echo -e " The SBU and disk usage report creation will be skiped" - REPORT=0 - continue - fi - fi ;; - STRIP) validation_str="x0x x1x"; validate_str; continue ;; - VIMLANG) validation_str="x0x x1x"; validate_str; continue ;; - DEPEND) validation_str="x0x x1x x2x"; validate_str; continue ;; - MODEL) validation_str="xglibcx xuclibcx"; validate_str; continue ;; - PAGE) validation_str="xletterx xA4x"; validate_str; continue ;; - GRSECURITY_HOST) validation_str="x0x x1x"; validate_str; continue ;; - METHOD) validation_str="xchrootx xbootx"; validate_str; continue ;; - ARCH) validation_str="xx86x xx86_64x xx86_64-64x xsparcx xsparc64x xsparc64-64x xmipsx xmips64x xmips64-64x xppcx xppc64x xalphax"; validate_str; continue ;; - TARGET) validate_target; continue ;; - esac + PARAM_GROUP=${PROGNAME}_PARAM_LIST + for config_param in ${!PARAM_GROUP}; do + # This is a tricky little piece of code.. executes a cmd string. + case $config_param in + BUILDDIR) # We cannot have an or root mount point + echo -e "`eval echo $PARAM_VALS`" + [[ "xx x/x" =~ "x${!config_param}x" ]] && + write_error_and_die + ;; + TIMEZONE) ;; + # Validate general parameters.. + HPKG) validate_against_str "x0x x1x" ;; + RUNMAKE) validate_against_str "x0x x1x" ;; + REPORT) validate_against_str "x0x x1x" + if [[ "${!config_param}" = "1" ]]; then + if [[ `type -p bc` ]]; then + continue + else + echo -e " ${BOLD}The bc binary was not found${OFF}" + echo -e " The SBU and disk usage report creation will be skiped" + REPORT=0 + continue + fi + fi ;; + COMPARE) if [[ ! "$COMPARE" = "1" ]]; then + validate_against_str "x0x x1x" + else + if [[ ! "${RUN_ICA}" = "1" ]] && [[ ! "${RUN_FARCE}" = "1" ]]; then + echo "${nl_}${DD_BORDER}" + echo "You have elected to analyse your build but have failed to select a tool." >&2 + echo "Edit /common/config and set ${L_arrow}${BOLD}RUN_ICA${R_arrow} and/or ${L_arrow}${BOLD}RUN_FARCE${R_arrow} to the required values" >&2 + echo "${DD_BORDER}${nl_}" + exit 1 + fi + fi ;; + RUN_ICA) [[ "$COMPARE" = "1" ]] && validate_against_str "x0x x1x" ;; + RUN_FARCE) [[ "$COMPARE" = "1" ]] && validate_against_str "x0x x1x" ;; + ITERATIONS) [[ "$COMPARE" = "1" ]] && validate_against_str "x2x x3x x4x x5x" ;; + TEST) validate_against_str "x0x x1x x2x x3x" ;; + STRIP) validate_against_str "x0x x1x" ;; + VIMLANG) validate_against_str "x0x x1x" ;; + DEPEND) validate_against_str "x0x x1x x2x" ;; + MODEL) validate_against_str "xglibcx xuclibcx" ;; + PAGE) validate_against_str "xletterx xA4x" ;; + METHOD) validate_against_str "xchrootx xbootx" ;; + ARCH) validate_against_str "xx86x xx86_64x xx86_64-64x xsparcx xsparc64x xsparc64-64x xmipsx xmips64x xmips64-64x xppcx xppc64x xalphax" ;; + TARGET) validate_target ;; + GRSECURITY_HOST) validate_against_str "x0x x1x" ;; - if [[ "${config_param}" = "LC_ALL" ]]; then - echo "`eval echo $PARAM_VALS`" - [[ -z "${!config_param}" ]] && echo -e "\nVariable LC_ALL cannot be empty!" && write_error_and_die - # See it the locale values exist on this machine - if [[ "`locale -a | grep -c ${!config_param}`" > 0 ]]; then - continue - else # If you make it this far then there is a problem - write_error_and_die - fi - fi + # BOOK validation. Very ugly, need be fixed + BOOK) if [[ "${WC}" = "1" ]] ; then + validate_dir -z -d + else + validate_against_str "x${PROGNAME}-developmentx xlfs-udev_updatex" + fi + ;; - if [[ "${config_param}" = "LANG" ]]; then - echo "`eval echo $PARAM_VALS`" - [[ -z "${!config_param}" ]] && echo -e "\nVariable LANG cannot be empty!" && write_error_and_die - # See if the locale values exist on this machine - if [[ "`locale -a | grep -c ${!config_param}`" > 0 ]]; then - continue - else # If you make it this far then there is a problem - write_error_and_die - fi - fi + # Validate directories, testable states: + # fatal -z -d -w, + # warning -z+ -w+ + SRC_ARCHIVE) validate_dir -z+ -d -w+ ;; + # Validate files, testable states: + # fatal -z -e -s -w -x -r, + # warning -z+ + FSTAB) validate_file -z+ -e -s ;; + CONFIG) validate_file -z+ -e -s ;; + BOOT_CONFIG) [[ "${METHOD}" = "boot" ]] && validate_file -z -e -s ;; - if [[ "${config_param}" = "KEYMAP" ]]; then - echo "`eval echo $PARAM_VALS`" - [[ "${!config_param}" = "none" ]] && continue - if [[ -e "/usr/share/kbd/keymaps/${!config_param}" ]] && - [[ -s "/usr/share/kbd/keymaps/${!config_param}" ]]; then - continue - else - write_error_and_die - fi - fi - - if [[ "${config_param}" = "SRC_ARCHIVE" ]]; then - echo -n "`eval echo $PARAM_VALS`" - if [ ! -z ${SRC_ARCHIVE} ]; then - if [ ! -d ${SRC_ARCHIVE} ]; then - echo " -- is NOT a directory" - write_error_and_die - fi - if [ ! -w ${SRC_ARCHIVE} ]; then - echo -n "${nl_} [${BOLD}${YELLOW}WARN$OFF] You do not have access to this directory, ${nl_}${tab_}downloaded files can not be saved in this archive" - fi - fi - echo - continue - fi - - if [[ "${config_param}" = "FSTAB" ]]; then - echo "`eval echo $PARAM_VALS`" - [[ -z "${!config_param}" ]] && continue - if [[ -e "${!config_param}" ]] && - [[ -s "${!config_param}" ]]; then - continue - else - write_error_and_die - fi - fi - - if [[ "${config_param}" = "BOOK" ]]; then - echo "`eval echo $PARAM_VALS`" - [[ ! "${WC}" = 1 ]] && continue - [[ -z "${!config_param}" ]] && continue - if [[ -e "${!config_param}" ]] && - [[ -s "${!config_param}" ]]; then - continue - else - write_error_and_die - fi - fi - - if [[ "${config_param}" = "CONFIG" ]]; then - echo "`eval echo $PARAM_VALS`" - [[ -z "${!config_param}" ]] && continue - if [[ -e "${!config_param}" ]] && - [[ -s "${!config_param}" ]]; then - continue - else - write_error_and_die - fi - fi - - if [[ "${config_param}" = "BOOT_CONFIG" ]]; then - if [[ "${METHOD}" = "boot" ]]; then - echo "`eval echo $PARAM_VALS`" - # There must be a config file when the build method is 'boot' - [[ -e "${!config_param}" ]] && [[ -s "${!config_param}" ]] && continue - # If you make it this far then there is a problem - write_error_and_die - fi - fi - done + # Treatment of 'special' parameters + LANG | \ + LC_ALL) # See it the locale values exist on this machine + echo -n "`eval echo $PARAM_VALS`" + [[ -z "${!config_param}" ]] && + echo " -- Variable $config_param cannot be empty!" && + write_error_and_die + [[ ! "`locale -a | grep -c ${!config_param}`" > 0 ]] && + write_error_and_die + echo + ;; + KEYMAP) echo "`eval echo $PARAM_VALS`" + save_param=${KEYMAP} + [[ ! "${!config_param}" = "none" ]] && + KEYMAP="/usr/share/kbd/keymaps/${KEYMAP}" && + validate_file -z -e -s + KEYMAP=${save_param} + ;; + esac done - set -e echo "$tab_***${BOLD}${GREEN} ${PARAM_GROUP%%_*T} config parameters look good${OFF} ***" } diff --git a/extras/do_copy_files b/extras/do_copy_files new file mode 100755 index 0000000..a176d7b --- /dev/null +++ b/extras/do_copy_files @@ -0,0 +1,36 @@ +#!/bin/bash +# $Id$ +set -e + +: <> $TMP_FILE +done + +mkdir -p $3 +cd $2 +tar -X $TMP_FILE -cf - . | tar -C $3 -xf - + +# Clear out the temporary file +rm -f ${TMP_FILE} + +echo "done." diff --git a/extras/do_ica_prep b/extras/do_ica_prep new file mode 100755 index 0000000..1bbfaba --- /dev/null +++ b/extras/do_ica_prep @@ -0,0 +1,77 @@ +#!/bin/bash +# $Id$ + +# Acknowledgment: +# The following code is a modified version of an original work written by +# Greg Schafer for the "DIY Linux" project and is included here with his +# permission. +# ref: http://www.diy-linux.org +# + +set -e + +: <&2 + exit 1 + } + rm -f ../${BN}.XX + done + echo "done." + + echo -n "Stripping (debug) symbols from \".o\" files in ${CMP_DIR}... " + find $CMP_DIR -name '*.o' | xargs strip -p -g 2>/dev/null + echo "done." + + echo -n "Stripping (all) symbols from files OTHER THAN \".o\" files in ${CMP_DIR}... " + find $CMP_DIR ! -name '*.o' | xargs strip -p 2>/dev/null || : + echo "done." + + # We're all done + echo -en "\nSuccess: ICA preparation for " + echo -e "${CMP_DIR} complete." + touch $CMP_DIR/icaprep +else + echo -e "\n$CMP_DIR was already processed\n" +fi + diff --git a/extras/do_ica_work b/extras/do_ica_work new file mode 100755 index 0000000..fde1fd1 --- /dev/null +++ b/extras/do_ica_work @@ -0,0 +1,55 @@ +#!/bin/bash +# $Id$ + +# Acknowledgment: +# The following code is a modified version of an original work written by +# Greg Schafer for the "DIY Linux" project and is included here with his +# permission. +# ref: http://www.diy-linux.org +# + +set -e + +: < $REPORT +diff -ur ${1} ${2} > $RAWDIFF || : + +echo -e "The list of binary files that differ:\n" > $REPORT +grep "iles.*differ$" $RAWDIFF >> $REPORT + +echo -e "The list of files that exist \"only in\" 1 of the directories:\n" >> $REPORT + +if grep "^Only in" $RAWDIFF >/dev/null 2>&1; then + grep "^Only in" $RAWDIFF >> $REPORT +else + echo NONE >> $REPORT +fi + +grep -v "iles.*differ$" $RAWDIFF | \ +grep -v "^Only in" > ${3}/${1}V${2}.ASCII.DIFF + +rm -f $RAWDIFF + +echo "done." diff --git a/extras/farce b/extras/farce new file mode 100755 index 0000000..776aed4 --- /dev/null +++ b/extras/farce @@ -0,0 +1,896 @@ +#!/bin/bash + +# Acknowledgment: +# The following code is a no-modified version (except for this comment +# and the inline_doc fragment) of an original work written by +# Ken Moffat and is included here with his permission. +# + +# FARCE: Farce Assists Rebuild Comparison Evaluation ;) +# +# to answer the question "can it rebuild itself?" +# +# We expect four arguments - first directory path, filelist +# containing the files in this directory which we wish to compare, +# second directory path, filelist for second directory. +# +# Yes, we could just compare everything in each tree, but the +# filelist script knows about files it can reasonably ignore, +# and this also allows us to build a sytem, boot it and get a +# list of files, build a full desktop environment, and only then +# build and boot the "can it build itself" test system and get +# _its_ filelist. +# +# What this script aims to do: +# ____________________________ +# +# First, report files not in both builds. +# +# Then, confirm symlinks point to same targets. +# +# After that, compare individual files - +# if different, run the file name through 'expected' +# to pick out files that are unlikely to match (logs, +# pids, fstab [assumes '/' is a different device each time], +# count these as 'expected'. +# +# For whatever is left, check the file type - ar archives +# have their members extraced and compared (every member has +# a timestamp), gzipped files are compared beyond their +# timestamp, binaries, at least those using shared libs or +# which are shared objects, are copied and subjected to +# --strip-debug. If files match at this stage, count them as +# 'accepted'. +# +# As a last step for any file that doesn't match, copy it +# through some perl regexps to "process" it (convert any +# date, time, kernel-version information from standard formats +# into tokens, then see if the tokensi match. +# +# For details of the regexps, see the tokenize function. +# Those files that match after this are also counted as +# 'accepted'. Note that I don't always start from the kernel +# version that I'm going to build, so this copes with e.g. perl +# files that hardcode the kernel version. +# +# We now have files that don't match. A few of these seem to be +# common to all builds - some (members of) c++ libraries or ar +# archives, a few programs which perhaps use some sort of c++ code). +# The file name # is passed to the 'failure' function - these +# recognized filenames are labelled as 'predictable FAIL:', +# anything else is labelled as 'unexpected FAIL:'. +# +# output: +# stderr - files only in one of the builds, failure messages, +# and totals. +# +# farce-results - more details, including which files were treated +# as expected differences, files where neither copy could be read, +# files treated as accepted, with the reason (and member for ar +# archives). This data is typically up to 100 characters wide - +# sometimes it's a bit more, but it doesn't wrap too badly in a +# 100 character xterm. +# +# farce-extras - diffs for the files, or members, that didn't +# match. This file is to establish new regexps for picking up +# date/time/kernel-version formats. +# +# farce-identical - the names of the files which are identical +# +# farce-substitutions - whenever using tokenizeanddiff results in a +# difference being accepted, for both versions diff the before and +# after versions to show what got changed. If the file is a binary, +# the output may still be hard to read. Note that I _know_ glibc +# version strings pass one of the regexps looking for a kernel version +# - since I expect you to use the same version of glibc for each +# build, this is not a problem. +# +# farce-differ - the names of the files which could not be treated +# as matching (whether or not I regard the failure as predictable) +# for possible input to ICA processing. +# +# Copyright (C) 2005, 2006 Ken Moffat +# +# All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or (at +# your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or +# NON INFRINGEMENT. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +# + +: <&2 + echo "$@" >&5 +} + +function expected() { + # if we expect it to differ because of its name, + # allow it and report, return true ; else return false + case $1 in + /boot/grub/menu.lst) + # just in case somebody puts this into the main filesystem + true;; + /etc/aliases.db) + # some sort of database for postfix, not parsable + true;; + /etc/blkid.tab) + # includes dev name for rootfs + true;; + /etc/fstab) + # fstab, e.g. ' / ' will differ + true;; + /etc/group*) + true;; + /etc/hosts) + # with dhcp client, I add current ip address to this in a hook + true;; + /etc/ld.so.*) + # .conf and .cache can vary, + # particularly if one system has a full build when I run this + true;; + /etc/lilo.conf|/etc/yaboot.conf) + # bootloader control, I assume grub will all be on a separate + true;; + /etc/mtab) + # at a minimum, different '/' + true;; + /etc/ntp.drift) + true;; + /etc/passwd*) + true;; + /etc/shadow*) + true;; + /etc/ssh/*key|/etc/ssh/*pub) + # openssh keys + true;; + /misc/*) + # where I put buildscripts (which mostly won't change) + # and stamps containing name/time/space which will differ in the times + true;; + /root/*) + # expect .bash_history etc to differ - if we can read them + true;; + /usr/bin/lynx) + # part of my inital builds, I guess this uses anonymous namespaces + true;; + /usr/include/c++/*/*/bits/stdc++.h.gch/*) + # precompiled headers + true;; + /usr/lib*/libstdc++.a|/usr/lib*/libstdc++.so*|/usr/lib*/libsupc++.a) + # probably, anonymous namespaces + # libstdc++.a, libstdc++.so.n.n.n, libsupc++.a + true;; + /usr/share/info/dir) + # if one system has had extra stuff built, this will likely be bigger + true;; + /usr/share/man/whatis) + # if one system has had extra stuff built, this will likely be bigger + true;; + /var/lib/locate/locatedb) + # if one system has had extra stuff built, this will likely be bigger + true;; + /var/lib/nfs/*) + # allow nfs bookkeeping + true;; + /var/log/*) + true;; + /var/run/utmp) + true;; + /var/spool/fcron*) + true;; + /var/state/*) + # allow dhcp leases + true;; + /var/tmp/random-seed) + true;; + # following start with wildcards + *Image*|*.PPCBoot*|*vmlinuz*|*lfskernel*) + # compressed kernels, sometimes just building at a different + # date/time is enough to change the length of them, because the + # long format date and time is part of the compressed data + true;; + *pid*) + # pids, including e.g. /var/spool/postfix/pid/* + true;; + *) + # nothing else is expected to be different + false;; + esac + if [ $? -eq 0 ]; then + message "expected difference in $1" + let expected=$expected+1 + case $TYPE in + AR) + let EXPAR=$EXPAR+1 + ;; + ELF) + let EXPELF=$EXPELF+1 + ;; + UNK) + let EXPUNK=$EXPUNK+1 + ;; + # so far, no other valid types, so don't accumulate them + *) + emessage "internal error, expected difference for $1 of type $TYPE not allowed" + exit 2 + ;; + esac + true + else + false + fi +} + +function failure() { + # first parm is filename or token + # second parm is the error message + # update the appropriate total + # and write to both stderr and the results + # by using emessage + + let different=$different+1 + case $TYPE in + AR) + let DIFAR=$DIFAR+1 + ;; + ELF) + let DIFELF=$DIFELF+1 + ;; + GZ) + let DIFGZ=$DIFGZ+1 + ;; + SYM) + let DIFSYM=$DIFSYM+1 + ;; + UNK) + let DIFUNK=$DIFUNK+1 + ;; + *) + emessage "internal error in failure() for TYPE $TYPE" + exit 2 + ;; + esac + test -f ${P1}$1 && echo $1 >&9 + emessage "FAIL: $2" +} + +function fatal() { + # unrecoverable error + echo $* + exit 1 +} + +function filetype() { + TYPE=`file ${P1}${FILE}` + case $TYPE in + *'current ar archive'*) + let TOTAR=$TOTAR+1 + TYPE=AR + ;; + *' ELF '*) + let TOTELF=$TOTELF+1 + TYPE=ELF + ;; + *'gzip compressed data'*) + let TOTGZ=$TOTGZ+1 + TYPE=GZ + ;; + *) + let TOTUNK=$TOTUNK+1 + TYPE=UNK + ;; + esac +} + +function message() { + # write a string to $RESULT + echo $* >&5 +} + +function onlyone() { + #report files only in one build + # text should go to both stderr and the results, + # but blank lines only go to the results + if [ $1 == '<' ]; then + emessage "File(s) only in the first build" + else + emessage "File(s) only in the second build" + fi + message "" + FILES=`cat $DIFF | grep "^$1" | cut -d ' ' -f 2` + for F in $FILES; do + emessage $F + let only=$only+1 + done + message "" +} + +# 'test' functions are called with three arguments: +# the two pathes and the filename +# - we know the file is of this type, so see if we +# can get it to match by reasonalbe means. +# if not, treat it as different. +# +# NB if pathes are absolute, we need to prefix them +# with the original $PWD to access the .a files +# +function testar() { + # ar archives include timestamps for the members, + # but diff doesn't show file timestamps unless the data differs + # put out a message to help locate which archive any messages + # about the members refer to. + + # try just stripping them U1,2 undebuggable + U1=`mktemp` || fatal "cannot create a temporary file" + U2=`mktemp` || fatal "cannot create a temporary file" + cp ${1}${3} $U1 + cp ${2}${3} $U2 + strip --strip-debug $U1 + strip --strip-debug $U2 + cmp -s $U1 $U2 + rm $U1 $U2 + if [ $? -eq 0 ]; then + let accepted=$accepted+1 + let ACCAR=$ACCAR+1 + message "archive $3 matches after strip --strip-debug" + return + fi + # rest of this function retained primarily for pathologically bad builds + # put out a message in the log to help identify which archive has issues. + message "examining ar archive $3" + D1=`mktemp -d` || fatal "cannot create a temporary directory" + D2=`mktemp -d` || fatal "cannot create a temporary directory" + cd $D1 + ar -x ${OP1}${1}${3} + cd $D2 + ar -x ${OP2}${2}${3} + cd + # diff the members - true means they match + diff -Na $D1 $D2 >/dev/null + if [ $? -eq 0 ]; then + message "accept: $3 after diffing the members" + let accepted=$accepted+1 + let ACCAR=$ACCAR+1 + else + # process individual members to eliminate date/time/kernel-version + # first, check the members are the same + M1=`mktemp` || fatal "cannot create a temporary file" + M2=`mktemp` || fatal "cannot create a temporary file" + cd $D1 + MEMBERS= + for F in *; do + MEMBERS="$MEMBERS $F" + done + cd + echo $MEMBERS | sort >$M1 + cd $D2 + MEMBERS= + for F in *; do + MEMBERS="$MEMBERS $F" + done + cd + echo $MEMBERS | sort >$M2 + cmp -s $M1 $M2 + if [ $? -ne 0 ]; then + # oh dear, different members + echo "list of members differs for archive $3" >&6 + diff $M1 $M2 >&6 + failure $3 "$3 list of members differs" + else + # members (names) are same, + # process each one + STATUS=0 + for M in $MEMBERS; do + #avoid firing up perl on matching members + cmp -s $D1/$M $D2/$M + if [ $? -ne 0 ]; then + tokenizeanddiff $D1/$M $D2/$M $FILE:$M + if [ $? -eq 0 ]; then + message "member $M matches after processing" + else + message "member $M DIFFERS after processing" + STATUS=1 + fi + fi + done + if [ $STATUS -eq 0 ]; then + let accepted=$accepted+1 + let ACCAR=$ACCAR+1 + else + let different=$different+1 + let DIFAR=$DIFAR+1 + echo $3 >&9 + emessage "FAIL: in $3" + fi + fi + rm $M1 $M2 + fi + rm -rf $D1 $D2 +} + +function testgzip() { + # bytes 4,5,6,7 are the timestamp, so ignore these + cmp -s -i 8 ${1}${3} ${2}${3} + if [ $? -eq 0 ]; then + message "accept: $3 after ignoring gzip timestamp" + let accepted=$accepted+1 + let ACCGZ=$ACCGZ+1 + else + failure $3 " $3 even after ignoring gzip timestamp" + fi +} + +function testso() { + # shared object - first try stripping it + # in fact, this now handles ALL ELF files + S1=`mktemp` || fatal "cannot create a temporary file" + S2=`mktemp` || fatal "cannot create a temporary file" + cp ${1}${3} $S1 + strip --strip-debug $S1 + cp ${2}${3} $S2 + strip --strip-debug $S2 + cmp -s $S1 $S2 + if [ $? -eq 0 ]; then + message "accept: $3 after --strip-debug" + let accepted=$accepted+1 + let ACCELF=$ACCELF+1 + else + tokenizeanddiff $S1 $S2 $3 + if [ $? -ne 0 ]; then + failure $3 " $3 differs after stripping and processing" + else + message "accept: $3 after --strip-debug and processing" + let accepted=$accepted+1 + let ACCELF=$ACCELF+1 + fi + fi + rm $S1 $S2 +} + +function tokenize() { + # use regexes to replace date/time/kernel-version text + # with tokens which may allow files to match even though + # they have hardcoded date/time/kernel-version. + # arguments are file to process, and where to put it. + # these regexes are somewhat long, and the order they + # are applied in is important (to stop short ones being + # used when a longer version would match). + # KV00 linux version date (e.g. as in the kernel itself) + # allow 2 or 3 groups of three alphas here - optional smp, with day, mon + # KV01 kernel version, including possible cpu details (that is for cdda2wav) + # KV02 just the version, in quotes e.g. "2.6.12.6" or '2.6.13', for perl stuff + # except that "|' gives me grif, so try a boundary + # also, it might need local version on the end, I really want + # quote2.\d+.\d+.{0,32}quote - it is the quotes that don't work. + # DT00 Day Mon .d+ hh:mm:ss TZN CCYY variations include non-caps and 'mon d' + # DT01 Mon .d+ CCYY hh:mm:ss + # DT02 hh:mm:ss Mon .d CCYY + # DT03 Mon .d CCYY + # DT04 Day Mon { ,d}d hh:mm:ss CCYY - for groff example postscript files + # (somewhat similar to DT00, but ' d' or ' dd' for day of month and no TZN ) + # DT05 hh:mm:ss + # DT06 ISO date using space as separator + # DT07 ISO date using dash as separator + # DT08 ISO date using slash as separator + # DT09 fullmonth (capitalised), day number, comma, 4-digit year (groff 1.18.1 ps) + # DT10 dd, fullmonth (capitalised), 4-digit year (groff 1.18.1 manpages) + # DT11 '(xample comma space digit(s) backslash ) in groff memef.ps which is + # quite clearly the day of the month when it was compiled, preceded by 'example' + # with something weird between the e and the x. + + if [ $# -ne 2 ]; then + fatal "tokenizing called with $# arguments : $*" + fi + + cat $1 | perl -p \ + -e 's/(L|l)inux.*\d\.\d\.\d+.* \#\d+( [A-Za-z][a-z]{2}){2,3} \d+ \d\d:\d\d:\d\d [A-Za-z]{3} \d{4}\b/%KV00%/g;' \ + -e 's/(L|l)inux( (\w|_)+)?(-| |_)\d\.\d(\.\d+){1,2}((-|_)?(\w|_)+)?( |\000)*/%KV01%/g;' \ + -e 's/\W2(\.\d+){2,3}(-|_)?((\w|_)+)?\s*\W/%KV02%/g;' \ + -e 's/\b([A-Za-z][a-z]{2} ){2}( |\d)?\d \d\d:\d\d:\d\d [A-Za-z]{3} \d{4}\b/%DT00%/g;' \ + -e 's/\b[A-Z][a-z]{2} ( |\d)\d \d{4} \d\d:\d\d:\d\d\b/%DT01%/g;' \ + -e 's/\b\d\d:\d\d:\d\d [A-Z][a-z]{2} ( |\d)\d \d{4}\b/%DT02%/g;' \ + -e 's/\b[A-Z][a-z]{2} ( |\d)\d \d{4}\b/%DT03%/g;' \ + -e 's/\b([A-Z][a-z]{2} ){2}( |\d)\d \d\d:\d\d:\d\d \d{4}/%DT04%/g;' \ + -e 's/\b\d\d:\d\d:\d\d\b/%DT05%/g;' \ + -e 's/\b\d{4} \d\d \d\d\b/%DT06%/g;' \ + -e 's/\b\d{4}-\d\d-\d\d\b/%DT07%/g;' \ + -e 's/\b\d{4}\/\d\d\/\d\d\b/%DT08%/g;' \ + -e 's/\b[A-Z][a-z]{2,} \d{1,2}, \d{4}/%DT09%/g;' \ + -e 's/\b\d\d [A-Z][a-z]{2,} \d{4}/%DT10%/g;' \ + -e 's/\(xample, \d{1,2}\\\)/%DT11%/g;' \ + >$2 +} + +function tokenizeanddiff() { + # Call tokenize for the inputs, then compare the results + # Input arguments are path/filename for old and new versions + # third parm is readable name (filename, or archivename:member) + # to help understand what is in the extras output. + # - sometimes called for files, but other times called for + # members of ar archives extracted into temporary directories + #message tokenizeanddiff called for $1 $2 $3 + F1=`mktemp` || fatal "cannot create a temporary file" + F2=`mktemp` || fatal "cannot create a temporary file" + tokenize $1 $F1 + tokenize $2 $F2 + + # actually, cmp is probably more efficient + # but for picking up the pieces it will be better to + # use diff to see what got through. + cmp -s $F1 $F2 + TOKENRESULT=$? + if [ $TOKENRESULT -ne 0 ]; then + echo "failure in $3..." >&6 + diff -a $F1 $F2 >&6 + rm $F1 $F2 + false + else + # show what we did + echo "substitutions for $3" >&8 + echo "build one" >&8 + diff -a $1 $F1 >&8 + echo "build two" >&8 + diff -a $2 $F2 >&8 + rm $F1 $F2 + true + fi +} + +function validateargs() { +# validate the arguments +BAD=0 +if ! [ -d $1 ]; then + echo "Error: first argument is not a directory" >&2 + let BAD=$BAD+1 +fi +NAME=`basename ${2%%-*}` +if [ $NAME != filelist ]; then + echo "Error: second argument is not a recognized filelist" >&2 + let BAD=$BAD+1 +fi +if ! [ -d $3 ]; then + echo "Error: third argument is not a directory" >&2 + let BAD=$BAD+1 +fi +NAME=`basename ${4%%-*}` +if [ $NAME != filelist ]; then + echo "Error: fourth argument is not a recognized filelist" >&2 + let BAD=$BAD+1 +fi +for I in $1 $2 $3 $4; do + if ! [ -r $I ]; then + echo "Error: cannot read $I" >&2 + let BAD=$BAD+1 + fi +done +if [ $1 == $3 ]; then + echo "Error: directory pathes are identical" >&2 + let BAD=$BAD+1 +fi +if [ $2 == $4 ]; then + echo "Error: filelist names are identical" >&2 + let BAD=$BAD+1 +fi +if [ $BAD -eq 0 ]; then + ARGS=valid +fi +} + +# Mainline +ARGS=unproven +OUTDIR= +if [ $# -eq 1 ]; then + case $1 in + -version|--version) + echo "`basename $0` version $VERSION" + exit 0 + ;; + -help|--help) + dohelp + exit 0 + ;; + esac +fi +if [ $1 = "--directory" ]; then + OUTDIR=$2 + shift 2 + grep '/$' $OUTDIR >/dev/null 2>&1 || OUTDIR=`echo $OUTDIR | sed 's%$%/%'` + echo "creating directory $OUTDIR" + mkdir -p $OUTDIR + if [ $? -ne 0 ]; then + echo "cannot mkdir $OUTDIR" + exit 1 + fi +fi +if [ $# -eq 4 ]; then + validateargs $* +fi +if ! [ $ARGS == valid ]; then + dohelp + fatal "`basename $0`: error in arguments" +fi + +# ok, we're happy, lets hit these files +exec 5>${OUTDIR}$RESULT +exec 6>${OUTDIR}$EXTRAS +exec 7>${OUTDIR}$IDENTICAL +exec 8>${OUTDIR}$SUBS +exec 9>${OUTDIR}$DIFFER + +>${OUTDIR}$RESULT +if [ $? -ne 0 ]; then + fatal "cannot write to ${OUTDIR}$RESULT" +fi + +emessage "will compare:" +emessage " first build at $1 with files listed in $2" +emessage "second build at $3 with files listed in $4" + +let accepted=0 +let different=0 +let expected=0 +let matched=0 +let only=0 +let predictable=0 +let unreadable=0 +let total=0 + +# break down the accepted +let ACCAR=0 +let ACCELF=0 +let ACCGZ=0 +let ACCUNK=0 + +# break down definitely different +let DIFAR=0 +let DIFELF=0 +let DIFGZ=0 +let DIFSYM=0 +let DIFUNK=0 + +# break down the expected differences +let EXPAR=0 +let EXPELF=0 +let EXPGZ=0 +let EXPUNK=0 + +# break down the identical files +let MATAR=0 +let MATELF=0 +let MATGZ=0 +let MATSYM=0 +let MATUNK=0 + +# break down how many of each type +let TOTAR=0 +let TOTELF=0 +let TOTGZ=0 +let TOTSYM=0 +let TOTUNK=0 + +# now identify differences between the two trees +DIFF=`mktemp` || fatal "cannot create a temporary file" +diff $2 $4 >$DIFF + +for RUN in '<' '>' ; do + grep -q "$RUN" $DIFF && onlyone "$RUN" +done + +rm $DIFF + +# and compare them +message "Results of file comparison:" +message "" + +# Strip any trailing slash from the path for tidyness, +# because the filenames all start with a slash. Unfortunately, +# unfortunately, '/' becomes empty, which breaks subroutines, +# so special case it. +# also, to process ar archives we need to extract them in temp +# directories - that means that after cd'ing we've broken any +# relative path, so save original pwd as necessary. +P1=`echo $1 | sed 's%/$%%'` +echo $1 | grep '^/' >/dev/null +if [ $? -ne 0 ]; then + # relative path + OP1=${PWD}/ + #echo "setting OP1 to $OP1" +else + OP1= + #echo "$1 is an absolute path" +fi +test -z "$P1" && P1='/' +P2=`echo $3 | sed 's%/$%%'` +echo $3 | grep '^/' >/dev/null +if [ $? -ne 0 ]; then + # relative path + OP2=${PWD}/ + #echo "setting OP2 to $OP2" +else + OP2= + #echo "$3 is an absolute path" +fi +test -z "$P2" && P2='/' + +echo "about to read $2" +while read FILE ; do +#echo "process $FILE" +#echo "test existence of ${P2}${FILE}" + # confirm it exists in second build + # we have already reported files only in one build + if [ -f ${P2}"${FILE}" ]; then + let total=$total+1 + # check we can read both of them + # or count as unreadable - I used to separate only-one-unreadable, + # but if you compre '/' and a _copy_ of /mnt/lfs that assumption + # breaks, so be less picky. + if ! [ -r "${P1}${FILE}" ] || ! [ -r "${P2}${FILE}" ]; then + message "cannot read one or both versions of $FILE" + let unreadable=$unreadable+1 + continue + fi + if [ -h "${P1}${FILE}" ]; then + # for symlink, look at what it points to + # exceptionally, do not call filetype + TYPE=SYM + let TOTSYM=$TOTSYM+1 + SL1=`ls -l "${P1}${FILE}" | awk '{ print $11 }'` + SL2=`ls -l "${P2}${FILE}" | awk '{ print $11 }'` + if [ "$SL1" = "$SL2" ]; then + echo "symlink $FILE matches for $SL1" >&5 + let matched=$matched+1 + let MATSYM=$MATSYM+1 + else + failure TARGET " symlink $FILE points to $SL1 and $SL2" + echo $FILE >&9 + fi + else + # regular file, start by typing it for accounting, + # then compare it + filetype ${P1}${FILE} + cmp -s "${P1}${FILE}" "${P2}${FILE}" + if [ $? -eq 0 ]; then + let matched=$matched+1 + case $TYPE in + AR) + let MATAR=$MATAR+1 + ;; + ELF) + let MATELF=$MATELF+1 + ;; + GZ) + let MATGZ=$MATGZ+1 + ;; + UNK) + let MATUNK=$MATUNK+1 + ;; + *) + echo "unexpected TYPE of $TYPE for $FILE" >&2 + exit 2 + ;; + esac + echo ${FILE} >&7 + else + # seems different, can we do better ? + # test if we expect it to differ + expected $FILE + if [ $? -ne 0 ]; then + case $TYPE in + GZ) + testgzip $P1 $P2 $FILE ;; + AR) + testar $P1 $P2 $FILE ;; + ELF) + testso $P1 $P2 $FILE ;; + *) + # long-stop - strip dates from text files + tokenizeanddiff "${P1}${FILE}" "${P2}${FILE}" "$FILE" + if [ $? -eq 0 ]; then + message "accepted $FILE after processing" + let accepted=$accepted+1 + let ACCUNK=$ACCUNK+1 + else + failure "$FILE" " $FILE is different" + fi + ;; + esac + fi + fi + fi + fi +done < $2 + +message "" +# write totals to stderr as well as the results file +emessage "$only files in only one of the builds" +emessage "$total files compared, of which" +emessage "$unreadable files could not be read, skipped" +emessage "$matched files are identical" +emessage "$expected files differed as expected" +emessage "$accepted files had allowable differences" +#emessage "$predictable files differed as they normally do" +emessage "$different files differed" + +# totals of different file types +emessage "" +emessage "$TOTAR ar archives" +emessage " of which $MATAR are identical" +emessage " of which $ACCAR are accepted after strip-debug or extracting, diffing, tokenizing" +emessage " of which $EXPAR differed as expected" +emessage " of which $DIFAR differed" +emessage "$TOTELF ELF executables or shared libraries" +emessage " of which $MATELF are identical" +emessage " of which $ACCELF are accepted after stripping and tokenizing" +emessage " of which $EXPELF differed as expected" +emessage " of which $DIFELF differed" +emessage "$TOTGZ gzipped files" +emessage " of which $MATGZ are identical" +emessage " of which $ACCGZ are accepted after comparing beyond timestamp" +emessage " of which $DIFGZ are different" +emessage "$TOTSYM symbolic links" +emessage " of which $MATSYM are identical" +emessage " of which $DIFSYM have different targets" +emessage "$TOTUNK other files" +emessage " of which $MATUNK are identical" +emessage " of which $ACCUNK are accepted after tokenizing" +emessage " of which $EXPUNK differed as expected" +emessage " of which $DIFUNK differed" + diff --git a/extras/filelist b/extras/filelist new file mode 100755 index 0000000..f9b1b08 --- /dev/null +++ b/extras/filelist @@ -0,0 +1,60 @@ +#!/bin/bash +#$Id$ + +# Acknowledgment: +# The following code is a modified version of an original work written by +# Ken Moffat for their "farce" project and is included here with his +# permission. +# + +set -e + +: <$OUTFILE +if [ $? -ne 0 ]; then + echo "error, cannot write to $OUTFILE" + exit 1 +fi + +find $LOC -xdev -xtype f | sed "s%^${LOC}%/%" | sort >$OUTFILE + +echo -e "done.\n" + +exit diff --git a/master.sh b/master.sh index 24363a7..50c2c20 100755 --- a/master.sh +++ b/master.sh @@ -51,12 +51,16 @@ source $COMMON_DIR/common-functions [[ $? > 0 ]] && echo " $COMMON_DIR/common-functions did not load.." && exit [[ $VERBOSITY > 0 ]] && echo "OK" # - [[ $VERBOSITY > 0 ]] && echo -n "Loading masterscript conf..." source $COMMON_DIR/config [[ $? > 0 ]] && echo "$COMMON_DIR/conf did not load.." && exit [[ $VERBOSITY > 0 ]] && echo "OK" # +[[ $VERBOSITY > 0 ]] && echo -n "Loading compare module..." +source $COMMON_DIR/func_compare.sh +[[ $? > 0 ]] && echo "$COMMON_DIR/func_compare.sh did not load.." && exit +[[ $VERBOSITY > 0 ]] && echo "OK" +# [[ $VERBOSITY > 0 ]] && echo -n "Loading config module <$MODULE_CONFIG>..." source $MODULE_CONFIG [[ $? > 0 ]] && echo "$MODULE_CONFIG did not load.." && exit 1 @@ -92,7 +96,7 @@ source $COMMON_DIR/func_validate_configs.sh ################################### -### MAIN ### +### MAIN ### ################################### # Evaluate any command line switches @@ -213,6 +217,29 @@ while test $# -gt 0 ; do ;; # Common options for LFS, CLFS and HLFS + --comparasion | -C ) + test $# = 1 && eval "$exit_missing_arg" + shift + case $1 in + ICA) RUN_ICA=1 + RUN_FARCE=0 + COMPARE=1 + ;; + farce) RUN_ICA=0 + RUN_FARCE=1 + COMPARE=1 + ;; + both) RUN_ICA=1 + RUN_FARCE=1 + COMPARE=1 + ;; + *) + echo -e "\n$1 is an unknown analisys method." + exit 1 + ;; + esac + ;; + --fstab | -F ) test $# = 1 && eval "$exit_missing_arg" shift @@ -454,6 +481,10 @@ fi if [[ "$PWD" != "$JHALFSDIR" ]]; then cp $COMMON_DIR/makefile-functions $JHALFSDIR/ + if [[ "$COMPARE" != "0" ]] ; then + mkdir -p $JHALFSDIR/extras + cp extras/* $JHALFSDIR/extras + fi if [[ -n "$FILES" ]]; then # pushd/popd necessary to deal with mulitiple files pushd $PACKAGE_DIR 1> /dev/null