zeroday0619/MahiroOS-jhalfs
Archived
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merged ICA/farce support from experimental branch.

Browse source
This commit is contained in:
Manuel Canales Esparcia 2006-04-29 14:44:29 +00:00
parent c7c5a53734
commit 45f8271829
14 changed files with 1658 additions and 385 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

143
CLFS/master.sh
View file

@ -461,22 +461,42 @@ bm_testsuite_tools_Makefiles() { #
#-----------------------------# #-----------------------------#
final_system_Makefiles() { # final_system_Makefiles() { #
#-----------------------------# #-----------------------------#
echo "${tab_}${GREEN}Processing... ${L_arrow}(chroot) final system${R_arrow}" # Set envars and scripts for iteration targets
LOGS="" # Start with an empty global LOGS envar
if [[ -z "$1" ]] ; then
local N=""
else
local N=-build_$1
local basicsystem=""
mkdir final-system$N
cp final-system/* final-system$N
for script in final-system$N/* ; do
# Overwrite existing symlinks, files, and dirs
sed -e 's/ln -sv/&f/g' \
-e 's/mv -v/&f/g' \
-e 's/mkdir -v/&p/g' -i ${script}
done
# Remove Bzip2 binaries before make install
sed -e 's@make install@rm -vf /usr/bin/bz*\n&@' -i final-system$N/*-bzip2
# Fix how Module-Init-Tools do the install target
sed -e 's@make install@make INSTALL=install install@' -i final-system$N/*-module-init-tools
# Delete *old Readline libraries just after make install
sed -e 's@make install@&\nrm -v /lib/lib{history,readline}*old@' -i final-system$N/*-readline
fi
for file in final-system/* ; do echo "${tab_}${GREEN}Processing... ${L_arrow}(chroot) final system$N${R_arrow}"
for file in final-system$N/* ; do
# Keep the script file name # Keep the script file name
this_script=`basename $file` this_script=`basename $file`
# Test if the stripping phase must be skipped # Test if the stripping phase must be skipped.
# Skip alsp temp-perl for iterative runs
case $this_script in case $this_script in
*stripping*) [[ "$STRIP" = "0" ]] && continue *stripping*) [[ "$STRIP" = "0" ]] && continue ;;
;; *temp-perl*) [[ -n "$N" ]] && continue ;;
esac esac
# First append each name of the script files to a list (this will become
# the names of the targets in the Makefile
basicsystem="$basicsystem $this_script"
# Grab the name of the target, strip id number, XXX-script # Grab the name of the target, strip id number, XXX-script
name=`echo $this_script | sed -e 's@[0-9]\{3\}-@@' \ name=`echo $this_script | sed -e 's@[0-9]\{3\}-@@' \
-e 's@temp-@@' \ -e 's@temp-@@' \
@ -485,22 +505,38 @@ final_system_Makefiles() { #
-e 's@64@@' \ -e 's@64@@' \
-e 's@n32@@'` -e 's@n32@@'`
# Find the version of the command files, if it corresponds with the building of
# a specific package. We need this here to can skip scripts not needed for
# iterations rebuilds
vrs=`grep "^$name-version" $JHALFSDIR/packages | sed -e 's/.* //' -e 's/"//g'` vrs=`grep "^$name-version" $JHALFSDIR/packages | sed -e 's/.* //' -e 's/"//g'`
if [[ "$vrs" = "" ]] && [[ -n "$N" ]] ; then
case "${this_script}" in
*stripping*) ;;
*) continue ;;
esac
fi
# Append each name of the script files to a list (this will become
# the names of the targets in the Makefile
basicsystem="$basicsystem ${this_script}${N}"
# Append each name of the script files to a list (this will become
# the names of the logs to be moved for each iteration)
LOGS="$LOGS ${this_script}"
#--------------------------------------------------------------------# #--------------------------------------------------------------------#
# >>>>>>>> START BUILDING A Makefile ENTRY <<<<<<<< # # >>>>>>>> START BUILDING A Makefile ENTRY <<<<<<<< #
#--------------------------------------------------------------------# #--------------------------------------------------------------------#
# #
# Drop in the name of the target on a new line, and the previous target # Drop in the name of the target on a new line, and the previous target
# as a dependency. Also call the echo_message function. # as a dependency. Also call the echo_message function.
wrt_target "${this_script}" "$PREV" wrt_target "${this_script}${N}" "$PREV"
# If $vrs isn't empty, we've got a package... # If $vrs isn't empty, we've got a package...
if [ "$vrs" != "" ] ; then if [ "$vrs" != "" ] ; then
case $name in FILE="$name-$vrs.tar.*"
temp-perl) wrt_unpack2 "perl-$vrs.tar.*" ;; wrt_unpack2 "$FILE"
*) wrt_unpack2 "$name-$vrs.tar.*" ;;
esac
fi fi
# #
wrt_run_as_chroot1 "${this_script}" "${file}" wrt_run_as_chroot1 "${this_script}" "${file}"
@ -515,8 +551,9 @@ final_system_Makefiles() { #
#--------------------------------------------------------------------# #--------------------------------------------------------------------#
# #
# Keep the script file name for Makefile dependencies. # Keep the script file name for Makefile dependencies.
PREV=$this_script PREV=${this_script}${N}
# Set system_build envar for iteration targets
system_build=$basicsystem
done # for file in final-system/* ... done # for file in final-system/* ...
} }
@ -524,25 +561,44 @@ final_system_Makefiles() { #
#-----------------------------# #-----------------------------#
bm_final_system_Makefiles() { # bm_final_system_Makefiles() { #
#-----------------------------# #-----------------------------#
echo "${tab_}${GREEN}Processing... ${L_arrow}(boot) final system${R_arrow}" # Set envars and scripts for iteration targets
LOGS="" # Start with an empty global LOGS envar
if [[ -z "$1" ]] ; then
local N=""
# The makesys phase was initiated in bm_testsuite_tools_makefile
[[ "$TEST" = 0 ]] && PREV=""
else
local N=-build_$1
local basicsystem=""
mkdir final-system$N
cp final-system/* final-system$N
for script in final-system$N/* ; do
# Overwrite existing symlinks, files, and dirs
sed -e 's/ln -sv/&f/g' \
-e 's/mv -v/&f/g' \
-e 's/mkdir -v/&p/g' -i ${script}
done
# Remove Bzip2 binaries before make install
sed -e 's@make install@rm -vf /usr/bin/bz*\n&@' -i final-system$N/*-bzip2
# Fix how Module-Init-Tools do the install target
sed -e 's@make install@make INSTALL=install install@' -i final-system$N/*-module-init-tools
# Delete *old Readline libraries just after make install
sed -e 's@make install@&\nrm -v /lib/lib{history,readline}*old@' -i final-system$N/*-readline
fi
# The makesys phase was initiated in bm_testsuite_tools_makefile echo "${tab_}${GREEN}Processing... ${L_arrow}(boot) final system$N${R_arrow}"
[[ "$TEST" = 0 ]] && PREV=""
for file in final-system/* ; do for file in final-system$N/* ; do
# Keep the script file name # Keep the script file name
this_script=`basename $file` this_script=`basename $file`
# Test if the stripping phase must be skipped # Test if the stripping phase must be skipped
# Skip alsp temp-perl for iterative runs
case $this_script in case $this_script in
*stripping*) [[ "$STRIP" = "0" ]] && continue *stripping*) [[ "$STRIP" = "0" ]] && continue ;;
;; *temp-perl*) [[ -n "$N" ]] && continue ;;
esac esac
# First append each name of the script files to a list (this will become
# the names of the targets in the Makefile
basicsystem="$basicsystem $this_script"
# Grab the name of the target, strip id number, XXX-script # Grab the name of the target, strip id number, XXX-script
name=`echo $this_script | sed -e 's@[0-9]\{3\}-@@' \ name=`echo $this_script | sed -e 's@[0-9]\{3\}-@@' \
-e 's@temp-@@' \ -e 's@temp-@@' \
@ -551,22 +607,38 @@ bm_final_system_Makefiles() { #
-e 's@64@@' \ -e 's@64@@' \
-e 's@n32@@'` -e 's@n32@@'`
# Find the version of the command files, if it corresponds with the building of
# a specific package. We need this here to can skip scripts not needed for
# iterations rebuilds
vrs=`grep "^$name-version" $JHALFSDIR/packages | sed -e 's/.* //' -e 's/"//g'` vrs=`grep "^$name-version" $JHALFSDIR/packages | sed -e 's/.* //' -e 's/"//g'`
if [[ "$vrs" = "" ]] && [[ -n "$N" ]] ; then
case "${this_script}" in
*stripping*) ;;
*) continue ;;
esac
fi
# Append each name of the script files to a list (this will become
# the names of the targets in the Makefile
basicsystem="$basicsystem ${this_script}${N}"
# Append each name of the script files to a list (this will become
# the names of the logs to be moved for each iteration)
LOGS="$LOGS ${this_script}"
#--------------------------------------------------------------------# #--------------------------------------------------------------------#
# >>>>>>>> START BUILDING A Makefile ENTRY <<<<<<<< # # >>>>>>>> START BUILDING A Makefile ENTRY <<<<<<<< #
#--------------------------------------------------------------------# #--------------------------------------------------------------------#
# #
# Drop in the name of the target on a new line, and the previous target # Drop in the name of the target on a new line, and the previous target
# as a dependency. Also call the echo_message function. # as a dependency. Also call the echo_message function.
wrt_target "${this_script}" "$PREV" wrt_target "${this_script}${N}" "$PREV"
# If $vrs isn't empty, we've got a package... # If $vrs isn't empty, we've got a package...
if [ "$vrs" != "" ] ; then if [ "$vrs" != "" ] ; then
case $name in FILE="$name-$vrs.tar.*"
temp-perl) wrt_unpack3 "perl-$vrs.tar.*" ;; wrt_unpack3 "$FILE"
*) wrt_unpack3 "$name-$vrs.tar.*" ;;
esac
fi fi
# #
wrt_run_as_root2 "${this_script}" "${file}" wrt_run_as_root2 "${this_script}" "${file}"
@ -581,8 +653,9 @@ bm_final_system_Makefiles() { #
#--------------------------------------------------------------------# #--------------------------------------------------------------------#
# #
# Keep the script file name for Makefile dependencies. # Keep the script file name for Makefile dependencies.
PREV=$this_script PREV=${this_script}${N}
# Set system_build envar for iteration targets
system_build=$basicsystem
done # for file in final-system/* ... done # for file in final-system/* ...
} }
@ -885,6 +958,8 @@ build_Makefile() { # Construct a Makefile from the book scripts
testsuite_tools_Makefiles # $testsuitetools testsuite_tools_Makefiles # $testsuitetools
fi fi
final_system_Makefiles # $basicsystem final_system_Makefiles # $basicsystem
# Add the iterations targets, if needed
[[ "$COMPARE" != "0" ]] && wrt_compare_targets
bootscripts_Makefiles # $bootscripttools bootscripts_Makefiles # $bootscripttools
bootable_Makefiles # $bootabletools bootable_Makefiles # $bootabletools
else else
@ -893,6 +968,8 @@ build_Makefile() { # Construct a Makefile from the book scripts
bm_testsuite_tools_Makefiles # $testsuitetools bm_testsuite_tools_Makefiles # $testsuitetools
fi fi
bm_final_system_Makefiles # $basicsystem bm_final_system_Makefiles # $basicsystem
# Add the iterations targets, if needed
[[ "$COMPARE" != "0" ]] && wrt_compare_targets
bm_bootscripts_Makefiles # $bootscipttools bm_bootscripts_Makefiles # $bootscipttools
bm_bootable_Makefiles # $bootabletoosl bm_bootable_Makefiles # $bootabletoosl
fi fi

67
HLFS/master.sh
View file

@ -295,8 +295,31 @@ chapter6_Makefiles() { # sysroot or chroot build phase
local TARGET LOADER local TARGET LOADER
local file local file
local this_script local this_script
# Set envars and scripts for iteration targets
LOGS="" # Start with an empty global LOGS envar
if [[ -z "$1" ]] ; then
local N=""
else
local N=-build_$1
local chapter6=""
mkdir chapter06$N
cp chapter06/* chapter06$N
for script in chapter06$N/* ; do
# Overwrite existing symlinks, files, and dirs
sed -e 's/ln -s /ln -sf /g' \
-e 's/^mv /&-f/g' -i ${script}
done
# Remove Bzip2 binaries before make install
sed -e 's@make install@rm -vf /usr/bin/bz*\n&@' -i chapter06$N/*-bzip2
# Fix how Module-Init-Tools do the install target
sed -e 's@make install@make INSTALL=install install@' -i chapter06$N/*-module-init-tools
# Delete *old Readline libraries just after make install
sed -e 's@make install@&\nrm -v /lib/lib{history,readline}*old@' -i chapter06$N/*-readline
# Don't readd already existing groups
sed -e '/groupadd/d' -i chapter06$N/*-udev
fi
echo "${tab_}${GREEN}Processing... ${L_arrow}Chapter6${R_arrow}" echo "${tab_}${GREEN}Processing... ${L_arrow}Chapter6$N${R_arrow}"
# #
# Set these definitions early and only once # Set these definitions early and only once
# #
@ -306,7 +329,7 @@ chapter6_Makefiles() { # sysroot or chroot build phase
TARGET="pc-linux-gnu"; LOADER="ld-linux.so.2" TARGET="pc-linux-gnu"; LOADER="ld-linux.so.2"
fi fi
for file in chapter06/* ; do for file in chapter06$N/* ; do
# Keep the script file name # Keep the script file name
this_script=`basename $file` this_script=`basename $file`
@ -317,26 +340,40 @@ chapter6_Makefiles() { # sysroot or chroot build phase
*chroot* ) continue ;; *chroot* ) continue ;;
# Test if the stripping phase must be skipped # Test if the stripping phase must be skipped
*-stripping* ) [[ "$STRIP" = "0" ]] && continue ;; *-stripping* ) [[ "$STRIP" = "0" ]] && continue ;;
*) ;;
esac esac
# First append each name of the script files to a list (this will become
# the names of the targets in the Makefile
chapter6="$chapter6 $this_script"
# Grab the name of the target # Grab the name of the target
name=`echo $this_script | sed -e 's@[0-9]\{3\}-@@'` name=`echo $this_script | sed -e 's@[0-9]\{3\}-@@'`
# Find the version of the command files, if it corresponds with the building of
# a specific package
vrs=`grep "^$name-version" $JHALFSDIR/packages | sed -e 's/.* //' -e 's/"//g'`
if [[ "$vrs" = "" ]] && [[ -n "$N" ]] ; then
case "${this_script}" in
*stripping*) ;;
*) continue ;;
esac
fi
# Append each name of the script files to a list (this will become
# the names of the targets in the Makefile
chapter6="$chapter6 ${this_script}${N}"
# Append each name of the script files to a list (this will become
# the names of the logs to be moved for each iteration)
LOGS="$LOGS ${this_script}"
# #
# Sed replacement to fix some rm command that could fail. # Sed replacement to fix some rm command that could fail.
# That should be fixed in the book sources. # That should be fixed in the book sources.
# #
case $name in case $name in
glibc) glibc)
sed 's/rm /rm -f /' -i chapter06/$this_script sed 's/rm /rm -f /' -i chapter06$N/$this_script
;; ;;
gcc) gcc)
sed 's/rm /rm -f /' -i chapter06/$this_script sed 's/rm /rm -f /' -i chapter06$N/$this_script
;; ;;
esac esac
@ -346,11 +383,7 @@ chapter6_Makefiles() { # sysroot or chroot build phase
# #
# Drop in the name of the target on a new line, and the previous target # Drop in the name of the target on a new line, and the previous target
# as a dependency. Also call the echo_message function. # as a dependency. Also call the echo_message function.
wrt_target "$this_script" "$PREV" wrt_target "${this_script}${N}" "$PREV"
# Find the version of the command files, if it corresponds with the building of
# a specific package
vrs=`grep "^$name-version" $JHALFSDIR/packages | sed -e 's/.* //' -e 's/"//g'`
# If $vrs isn't empty, we've got a package... # If $vrs isn't empty, we've got a package...
# Insert instructions for unpacking the package and changing directories # Insert instructions for unpacking the package and changing directories
@ -408,7 +441,9 @@ EOF
#--------------------------------------------------------------------# #--------------------------------------------------------------------#
# Keep the script file name for Makefile dependencies. # Keep the script file name for Makefile dependencies.
PREV=$this_script PREV=${this_script}${N}
# Set system_build envar for iteration targets
system_build=$chapter6
done # end for file in chapter06/* done # end for file in chapter06/*
} }
@ -518,6 +553,8 @@ build_Makefile() { # Construct a Makefile from the book scripts
chapter3_Makefiles chapter3_Makefiles
chapter5_Makefiles chapter5_Makefiles
chapter6_Makefiles chapter6_Makefiles
# Add the iterations targets, if needed
[[ "$COMPARE" != "0" ]] && wrt_compare_targets
chapter7_Makefiles chapter7_Makefiles
# Add a header, some variables and include the function file # Add a header, some variables and include the function file

66
LFS/master.sh
View file

@ -144,9 +144,35 @@ chapter5_Makefiles() {
#----------------------------# #----------------------------#
chapter6_Makefiles() { chapter6_Makefiles() {
#----------------------------# #----------------------------#
echo "${tab_}${GREEN}Processing... ${L_arrow}Chapter6${R_arrow}" # Set envars and scripts for iteration targets
LOGS="" # Start with an empty global LOGS envar
if [[ -z "$1" ]] ; then
local N=""
else
local N=-build_$1
local chapter6=""
mkdir chapter06$N
cp chapter06/* chapter06$N
for script in chapter06$N/* ; do
# Overwrite existing symlinks, files, and dirs
sed -e 's/ln -sv/&f/g' \
-e 's/mv -v/&f/g' \
-e 's/mkdir -v/&p/g' -i ${script}
done
# Remove Bzip2 binaries before make install
sed -e 's@make install@rm -vf /usr/bin/bz*\n&@' -i chapter06$N/*-bzip2
# Fix how Module-Init-Tools do the install target
sed -e 's@make install@make INSTALL=install install@' -i chapter06$N/*-module-init-tools
# Delete *old Readline libraries just after make install
sed -e 's@make install@&\nrm -v /lib/lib{history,readline}*old@' -i chapter06$N/*-readline
# Let some Udev pre-installation commands to fail
sed -e 's@/lib/udev/devices/fd@& || true@' \
-e 's/mknod -m.*/& || true/' -i chapter06$N/*-udev
fi
for file in chapter06/* ; do echo "${tab_}${GREEN}Processing... ${L_arrow}Chapter6$N${R_arrow}"
for file in chapter06$N/* ; do
# Keep the script file name # Keep the script file name
this_script=`basename $file` this_script=`basename $file`
@ -157,24 +183,36 @@ chapter6_Makefiles() {
*stripping*) [[ "${STRIP}" = "0" ]] && continue ;; *stripping*) [[ "${STRIP}" = "0" ]] && continue ;;
esac esac
# First append each name of the script files to a list (this will become
# the names of the targets in the Makefile
chapter6="$chapter6 ${this_script}"
# Grab the name of the target # Grab the name of the target
name=`echo ${this_script} | sed -e 's@[0-9]\{3\}-@@'` name=`echo ${this_script} | sed -e 's@[0-9]\{3\}-@@'`
# Find the version of the command files, if it corresponds with the building of
# a specific package. We need this here to can skip scripts not needed for
# iterations rebuilds
vrs=`grep "^$name-version" $JHALFSDIR/packages | sed -e 's/.* //' -e 's/"//g'`
if [[ "$vrs" = "" ]] && [[ -n "$N" ]] ; then
case "${this_script}" in
*stripping*) ;;
*) continue ;;
esac
fi
# Append each name of the script files to a list (this will become
# the names of the targets in the Makefile)
chapter6="$chapter6 ${this_script}${N}"
# Append each name of the script files to a list (this will become
# the names of the logs to be moved for each iteration)
LOGS="$LOGS ${this_script}"
#--------------------------------------------------------------------# #--------------------------------------------------------------------#
# >>>>>>>> START BUILDING A Makefile ENTRY <<<<<<<< # # >>>>>>>> START BUILDING A Makefile ENTRY <<<<<<<< #
#--------------------------------------------------------------------# #--------------------------------------------------------------------#
# #
# Drop in the name of the target on a new line, and the previous target # Drop in the name of the target on a new line, and the previous target
# as a dependency. Also call the echo_message function. # as a dependency. Also call the echo_message function.
wrt_target "${this_script}" "$PREV" wrt_target "${this_script}${N}" "$PREV"
# Find the version of the command files, if it corresponds with the building of
# a specific package
vrs=`grep "^$name-version" $JHALFSDIR/packages | sed -e 's/.* //' -e 's/"//g'`
# If $vrs isn't empty, we've got a package... # If $vrs isn't empty, we've got a package...
# Insert instructions for unpacking the package and changing directories # Insert instructions for unpacking the package and changing directories
@ -204,7 +242,9 @@ chapter6_Makefiles() {
#--------------------------------------------------------------------# #--------------------------------------------------------------------#
# Keep the script file name for Makefile dependencies. # Keep the script file name for Makefile dependencies.
PREV=${this_script} PREV=${this_script}${N}
# Set system_build envar for iteration targets
system_build=$chapter6
done # end for file in chapter06/* done # end for file in chapter06/*
} }
@ -304,6 +344,8 @@ build_Makefile() {
chapter4_Makefiles chapter4_Makefiles
chapter5_Makefiles chapter5_Makefiles
chapter6_Makefiles chapter6_Makefiles
# Add the iterations targets, if needed
[[ "$COMPARE" != "0" ]] && wrt_compare_targets
chapter789_Makefiles chapter789_Makefiles

9
common/common-functions
View file

@ -76,6 +76,15 @@ ${BOLD} -W, --working-copy DIR${OFF}
[[ ${PROGNAME} != "blfs" ]] && [[ ${PROGNAME} != "blfs" ]] &&
cat <<- -EOF- cat <<- -EOF-
${BOLD} -C, --comparasion TYPE${OFF}
do iterative comparison analysis. That will take several time due that
the final system packages wil be rebuilded the times defined in the
ITERATIONS config option.
Analisys types allowed are:
ICA = do the ICA analisys designed by Greg Schafer
farce = do the farce analisys designed by Ken Moffat
both = perfom both ICA and farce analisys
${BOLD} -F, --fstab FILE${OFF} ${BOLD} -F, --fstab FILE${OFF}
use FILE as the /etc/fstab file for the ${BOLD}$(echo $PROGNAME | tr [a-z] [A-Z])${OFF} system. If not specified, use FILE as the /etc/fstab file for the ${BOLD}$(echo $PROGNAME | tr [a-z] [A-Z])${OFF} system. If not specified,
a default /etc/fstab file with dummy values is created. a default /etc/fstab file with dummy values is created.

23
common/config
View file

@ -62,15 +62,18 @@ LC_ALL=$LC_ALL # (not used in LFS)
# Not used in LFS # Not used in LFS
KEYMAP=none KEYMAP=none
#=== Variables needed by ICA (Not implemented yet :-/)=== #=== Variables needed by iterative comparison analysis ===
#--- Run ICA testing 0/1 0(no)/1(yes) #--- Should some iterative comparison analysis by made? 0(no)/1(yes)
COMPARE=0
#--- The number of final stage builds to create and compare, min. 2, max. 5
ITERATIONS=3
#--- Run ICA testing 0(no)/1(yes)
RUN_ICA=0 RUN_ICA=0
#--- The number of final stage builds to create and compare #--- Run farce testing 0(no)/1(yes)
ITERATIONS= RUN_FARCE=0
#--- ICA report log directory
ICALOGDIR=$JHALFSDIR/logs/ICA
#==== INTERNAL VARIABLES ==== #==== INTERNAL VARIABLES ====
# Don't edit it unless you know what you are doing # Don't edit it unless you know what you are doing
@ -78,3 +81,9 @@ ICALOGDIR=$JHALFSDIR/logs/ICA
#--- Working directories #--- Working directories
JHALFSDIR=$BUILDDIR/jhalfs JHALFSDIR=$BUILDDIR/jhalfs
LOGDIR=$JHALFSDIR/logs LOGDIR=$JHALFSDIR/logs
#--- ICA report log directory
ICALOGDIR=$LOGDIR/ICA
#--- farce report log directory
FARCELOGDIR=$LOGDIR/farce

179
common/func_ICA.sh
View file

@ -1,179 +0,0 @@
# $Id$
# Acknowledgment:
# The following code is a modified version of an original work written by
# Greg Schafer for the "DIY Linux" project and is included here with his
# permission.
# ref: http://www.diy-linux.org
#
#
# ---------------------------------------------------------------------------- #
# Here are the ICA functions.
# ---------------------------------------------------------------------------- #
#
# Here we prepare for the Iterative Comparison Analysis (ICA). Essentially, we
# copy most of our chroot phase files to a new location then perform some
# manipulations on the copied files to make diff comparisons easier. The steps
# involved are:-
# (1) copy the whole tree (minus the PRUNEPATH defined below) to the CMP_DIR
# location. Use tar as it seems like the most appropriate tool for copying
# large directory trees around.
# (2) delete all symlinks.
# (3) gunzip all `*.gz' files.
# (4) delete all hardlinked files (of course trying to leaving the "master"
# intact)
# (5) convert all `*.a' files (ar archives) into a directory of the same name
# containing the unpacked object files.
# (6) fully strip the whole lot (but being careful to strip only the debug
# symbols from object `*.o' files).
#----------------------------------#
do_ica_prep() { #
#----------------------------------#
: <<inline_doc
desc:
usage:
input vars:
externals: --
modifies: --
returns: --
on error:
on success:
inline_doc
local CMP_DIR F L BN
local ALL_FILES=/tmp/allfiles.$$
local UNIQUE_FILES=/tmp/uniquefiles.$$
local PRUNEPATH="$TT_PFX $PATCHES_DIR $SCRATCH_DIR $TARBALLS_DIR \
/dev /home /mnt /proc /root /sys /tmp /usr/src /lost+found"
if [ ! -f "${STAMP_DIR}/icaprep" ]; then
CMP_DIR="${SCRATCH_DIR}/cmp/iter${ITER}"
test -d "$CMP_DIR" || mkdir -p $CMP_DIR
echo -e "\n${BORDER}\n${CYAN}[ICA] - starting ICA preparation for\c"
echo -e " Iteration ${ITER}.${OFF}\n"
# Create a file that we can pass to tar as an "exclude list".
# There might be an easier way to achieve tar exclusions? Strip
# the leading /.
for F in $PRUNEPATH; do
echo ${F#*/} >> $TMP_FILE
done
echo -n "Copying files to ${CMP_DIR}... "
cd /
tar -X $TMP_FILE -cf - . | tar -C $CMP_DIR -xf - || {
echo -e "\n\n${RED}ERROR:${OFF} tar copy failed!\n" >&2
exit 1
}
echo "done."
rm -f $TMP_FILE
echo -n "Removing symbolic links in ${CMP_DIR}... "
find $CMP_DIR -type l | xargs rm -f
echo "done."
echo -n "Gunzipping \".gz\" files in ${CMP_DIR}... "
find $CMP_DIR -name '*.gz' | xargs gunzip
echo "done."
# This was a bit tricky. You'll probably have to do it by hand
# to see what's actually going on. The "sort/uniq" part is
# inspired from the example DirCmp script from the book "Shell
# Programming Examples" by Bruce Blinn, published by Prentice
# Hall. We are essentially using the `-ls' option of the find
# utility to allow manipulations based on inode numbers.
#
# FIXME - this is a bit unreliable - rem out for now
#echo -n "Removing hardlinked file copies in ${CMP_DIR}... "
#find $CMP_DIR -ls | sort -n > $ALL_FILES
#find $CMP_DIR -ls | sort -n -u > $UNIQUE_FILES
#cat $UNIQUE_FILES $ALL_FILES | sort | uniq -u | awk '{ print $11 }' | xargs rm -f
#rm -f $ALL_FILES $UNIQUE_FILES
#echo "done."
# ar archives contain date & time stamp info that causes us
# grief when trying to find differences. Here we perform some
# hackery to allow easy diffing. Essentially, replace each
# archive with a dir of the same name and extract the object
# files from the archive into this dir. Despite their names,
# libieee.a & libmcheck.a are not actual ar archives.
#
echo -n "Extracting object files from \".a\" files in ${CMP_DIR}... "
L=$(find $CMP_DIR -name '*.a' ! -name 'libieee.a' ! -name 'libmcheck.a')
for F in $L; do
mv $F ${F}.XX
mkdir $F
cd $F
BN=${F##*/}
ar x ../${BN}.XX || {
echo -e "\n\n${RED}ERROR:${OFF} ar archive extraction failed!\n" >&2
exit 1
}
rm -f ../${BN}.XX
done
echo "done."
echo -n "Stripping (debug) symbols from \".o\" files in ${CMP_DIR}... "
find $CMP_DIR -name '*.o' | xargs strip -p -g 2>/dev/null
echo "done."
echo -n "Stripping (all) symbols from files OTHER THAN \".o\" files in ${CMP_DIR}... "
find $CMP_DIR ! -name '*.o' | xargs strip -p 2>/dev/null || :
echo "done."
echo -e "\n${CYAN}[ICA] - ICA preparation for Iteration ${ITER}\c"
echo -e " complete.${OFF}\n${BORDER}"
do_stamp icaprep
fi
}
#----------------------------------#
do_ica_work() { # Do the ICA grunt work.
#----------------------------------#
: <<inline_doc
desc:
usage: do_ica_work 1 2
do_ica_work 2 3
input vars: $1 iteration number to use
$2 iteration number to use
externals: --
modifies: --
returns: --
on error:
on success:
inline_doc
local ICA_DIR="${SCRATCH_DIR}/cmp"
local RAWDIFF=/tmp/rawdiff.$$
local REPORT="${SCRATCH_DIR}/logs/REPORT.${1}V${2}"
cd $ICA_DIR
echo -n "Diffing iter${1} and iter${2}... "
diff -ur iter${1} iter${2} > $RAWDIFF || :
echo "done."
echo -e "The list of binary files that differ:\n" > $REPORT
grep "iles.*differ$" $RAWDIFF >> $REPORT
echo -e "\n" >> $REPORT
echo -e "The list of files that exist \"only in\" 1 of the directories:\n" >> $REPORT
if grep "^Only in" $RAWDIFF >/dev/null 2>&1; then
grep "^Only in" $RAWDIFF >> $REPORT
else
echo NONE >> $REPORT
fi
grep -v "iles.*differ$" $RAWDIFF | grep -v "^Only in" > ${SCRATCH_DIR}/logs/${1}V${2}.ASCII.DIFF
rm -f $RAWDIFF
}

117
common/func_compare.sh Normal file
View file

@ -0,0 +1,117 @@
# $Id$
#----------------------------------#
wrt_compare_targets() { #
#----------------------------------#
for ((N=1; N <= ITERATIONS ; N++)) ; do # Double parentheses,
# and "ITERATIONS" with no "$".
ITERATION=iteration-$N
if [ "$N" != "1" ] ; then
wrt_system_build "$N" "$PREV_IT"
fi
this_script=$ITERATION
wrt_target "$ITERATION" "$PREV"
wrt_compare_work "$ITERATION" "$PREV_IT"
wrt_logs "$N"
PREV_IT=$ITERATION
PREV=$ITERATION
done
}
#----------------------------------#
wrt_system_build() { #
#----------------------------------#
local RUN=$1
local PREV_IT=$2
if [[ "$PROGNAME" = "clfs" ]] && [[ "$METHOD" = "chroot" ]] ; then
final_system_Makefiles $RUN
elif [[ "$PROGNAME" = "clfs" ]] && [[ "$METHOD" = "boot" ]] ; then
bm_final_system_Makefiles $RUN
else
chapter6_Makefiles $RUN
fi
echo -e "\nsystem_build_$RUN: $PREV_IT $system_build" >> $MKFILE.tmp
PREV=system_build_$RUN
}
#----------------------------------#
wrt_compare_work() { #
#----------------------------------#
local ITERATION=$1
local PREV_IT=$2
local PRUNEPATH="/dev /home /jhalfs /lost+found /media /mnt /opt /proc \
/sources /root /srv /sys /tmp /tools /usr/local /usr/src /var/log/paco"
if [[ "$PROGNAME" = "clfs" ]] && [[ "$METHOD" = "boot" ]] ; then
local ROOT_DIR=/
local DEST_TOPDIR=/jhalfs
local ICALOGDIR=/jhalfs/logs/ICA
local FARCELOGDIR=/jhalfs/logs/farce
else
local ROOT_DIR=$BUILDDIR
local DEST_TOPDIR=$BUILDDIR/jhalfs
fi
if [[ "$RUN_ICA" = "1" ]] ; then
local DEST_ICA=$DEST_TOPDIR/ICA && \
(
cat << EOF
@extras/do_copy_files "$PRUNEPATH" $ROOT_DIR $DEST_ICA/$ITERATION >>logs/$ITERATION.log 2>&1 && \\
extras/do_ica_prep $DEST_ICA/$ITERATION >>logs/$ITERATION.log 2>&1
EOF
) >> $MKFILE.tmp
if [[ "$ITERATION" != "iteration-1" ]] ; then
wrt_do_ica_work "$PREV_IT" "$ITERATION" "$DEST_ICA"
fi
fi
if [[ "$RUN_FARCE" = "1" ]] ; then
local DEST_FARCE=$DEST_TOPDIR/farce && \
(
cat << EOF
@extras/do_copy_files "$PRUNEPATH" $ROOT_DIR $DEST_FARCE/$ITERATION >>logs/$ITERATION.log 2>&1 && \\
extras/filelist $DEST_FARCE/$ITERATION $DEST_FARCE/filelist-$ITERATION >>logs/$ITERATION.log 2>&1
EOF
) >> $MKFILE.tmp
if [[ "$ITERATION" != "iteration-1" ]] ; then
wrt_do_farce_work "$PREV_IT" "$ITERATION" "$DEST_FARCE"
fi
fi
}
#----------------------------------#
wrt_do_ica_work() { #
#----------------------------------#
echo -e "\t@extras/do_ica_work $1 $2 $ICALOGDIR $3 >>logs/$ITERATION.log 2>&1" >> $MKFILE.tmp
}
#----------------------------------#
wrt_do_farce_work() { #
#----------------------------------#
local OUTPUT=$FARCELOGDIR/${1}_V_${2}
local PREDIR=$3/$1
local PREFILE=$3/filelist-$1
local ITEDIR=$3/$2
local ITEFILE=$3/filelist-$2
echo -e "\t@extras/farce --directory $OUTPUT $PREDIR $PREFILE $ITEDIR $ITEFILE >>logs/$ITERATION.log 2>&1" >> $MKFILE.tmp
}
#----------------------------------#
wrt_logs() { #
#----------------------------------#
local ITERATION=iteration-$1
(
cat << EOF
@pushd logs 1> /dev/null && \\
mkdir $ITERATION && \\
mv ${LOGS} $ITERATION && \\
popd 1> /dev/null
@touch \$@
EOF
) >> $MKFILE.tmp
}

280
common/func_validate_configs.sh
View file

@ -1,7 +1,8 @@
# $Id$ # $Id$
validate_target() { #----------------------------#
validate_target() { #
#----------------------------#
local -r ERROR_MSG_pt1='The variable \"${L_arrow}TARGET${R_arrow}\" value ${L_arrow}${BOLD}${TARGET}${R_arrow} is invalid for the ${L_arrow}${BOLD}${ARCH}${R_arrow} architecture' local -r ERROR_MSG_pt1='The variable \"${L_arrow}TARGET${R_arrow}\" value ${L_arrow}${BOLD}${TARGET}${R_arrow} is invalid for the ${L_arrow}${BOLD}${ARCH}${R_arrow} architecture'
local -r ERROR_MSG_pt2=' check the config file ${BOLD}${GREEN}\<$(echo $PROGNAME | tr [a-z] [A-Z])/config\> or \<common/config\>${OFF}' local -r ERROR_MSG_pt2=' check the config file ${BOLD}${GREEN}\<$(echo $PROGNAME | tr [a-z] [A-Z])/config\> or \<common/config\>${OFF}'
@ -67,7 +68,7 @@ validate_target() {
#----------------------------# #----------------------------#
validate_config() { # Are the config values sane (within reason) validate_config() { # Are the config values sane (within reason)
#----------------------------# #----------------------------#
: <<inline_doc : <<inline_doc
Validates the configuration parameters. The global var PROGNAME selects the Validates the configuration parameters. The global var PROGNAME selects the
@ -78,24 +79,24 @@ validate_config() { # Are the config values sane (within reason)
PROGNAME (lfs,clfs,hlfs,blfs) PROGNAME (lfs,clfs,hlfs,blfs)
modifies: none modifies: none
returns: nothing returns: nothing
on error: write text to console and dies on error: write text to console and dies
on success: write text to console and returns on success: write text to console and returns
inline_doc inline_doc
# First internal variables, then the ones that change the book's flavour, and lastly system configuration variables # First internal variables, then the ones that change the book's flavour, and lastly system configuration variables
local -r blfs_PARAM_LIST="BOOK BUILDDIR SRC_ARCHIVE HPKG DEPEND TEST" local -r blfs_PARAM_LIST="BOOK BUILDDIR SRC_ARCHIVE HPKG DEPEND TEST"
local -r hlfs_PARAM_LIST="BOOK BUILDDIR SRC_ARCHIVE HPKG RUNMAKE MODEL GRSECURITY_HOST TEST REPORT STRIP FSTAB CONFIG KEYMAP PAGE TIMEZONE LANG LC_ALL" local -r hlfs_PARAM_LIST="BOOK BUILDDIR SRC_ARCHIVE HPKG RUNMAKE MODEL GRSECURITY_HOST TEST REPORT COMPARE RUN_ICA RUN_FARCE ITERATIONS STRIP FSTAB CONFIG KEYMAP PAGE TIMEZONE LANG LC_ALL"
local -r clfs_PARAM_LIST="BOOK BUILDDIR SRC_ARCHIVE HPKG RUNMAKE METHOD ARCH TARGET TEST REPORT STRIP FSTAB BOOT_CONFIG CONFIG KEYMAP VIMLANG PAGE TIMEZONE LANG" local -r clfs_PARAM_LIST="BOOK BUILDDIR SRC_ARCHIVE HPKG RUNMAKE METHOD ARCH TARGET TEST REPORT COMPARE RUN_ICA RUN_FARCE ITERATIONS STRIP FSTAB BOOT_CONFIG CONFIG KEYMAP VIMLANG PAGE TIMEZONE LANG"
local -r lfs_PARAM_LIST="BOOK BUILDDIR SRC_ARCHIVE HPKG RUNMAKE TEST REPORT STRIP FSTAB CONFIG VIMLANG PAGE TIMEZONE LANG" local -r lfs_PARAM_LIST="BOOK BUILDDIR SRC_ARCHIVE HPKG RUNMAKE TEST REPORT COMPARE RUN_ICA RUN_FARCE ITERATIONS STRIP FSTAB CONFIG VIMLANG PAGE TIMEZONE LANG"
local -r ERROR_MSG_pt1='The variable \"${L_arrow}${config_param}${R_arrow}\" value ${L_arrow}${BOLD}${!config_param}${R_arrow} is invalid,' local -r ERROR_MSG_pt1='The variable \"${L_arrow}${config_param}${R_arrow}\" value ${L_arrow}${BOLD}${!config_param}${R_arrow} is invalid,'
local -r ERROR_MSG_pt2=' check the config file ${BOLD}${GREEN}\<$(echo $PROGNAME | tr [a-z] [A-Z])/config\> or \<common/config\>${OFF}' local -r ERROR_MSG_pt2=' check the config file ${BOLD}${GREEN}\<$(echo $PROGNAME | tr [a-z] [A-Z])/config\> or \<common/config\>${OFF}'
local -r PARAM_VALS='${config_param}: ${L_arrow}${BOLD}${!config_param}${OFF}${R_arrow}' local -r PARAM_VALS='${config_param}: ${L_arrow}${BOLD}${!config_param}${OFF}${R_arrow}'
local PARAM_LIST= local PARAM_LIST=
local config_param local config_param
local validation_str local validation_str
local save_param
write_error_and_die() { write_error_and_die() {
echo -e "\n${DD_BORDER}" echo -e "\n${DD_BORDER}"
@ -105,150 +106,155 @@ inline_doc
exit 1 exit 1
} }
validate_str() { validate_against_str() {
# This is the 'regexp' test available in bash-3.0.. # This is the 'regexp' test available in bash-3.0..
# using it as a poor man's test for substring # using it as a poor man's test for substring
echo -e "`eval echo $PARAM_VALS`" echo -e "`eval echo $PARAM_VALS`"
if [[ ! "${validation_str}" =~ "x${!config_param}x" ]] ; then if [[ ! "$1" =~ "x${!config_param}x" ]] ; then
# parameter value entered is no good # parameter value entered is no good
write_error_and_die write_error_and_die
fi fi
} }
validate_file() {
# For parameters ending with a '+' failure causes a warning message only
echo -n "`eval echo $PARAM_VALS`"
while test $# -gt 0 ; do
case $1 in
# Failures caused program exit
"-z") [[ -z "${!config_param}" ]] && echo "${tab_}<-- NO file name given" && write_error_and_die ;;
"-e") [[ ! -e "${!config_param}" ]] && echo "${tab_}<-- file does not exist" && write_error_and_die ;;
"-s") [[ ! -s "${!config_param}" ]] && echo "${tab_}<-- file has zero bytes" && write_error_and_die ;;
"-r") [[ ! -r "${!config_param}" ]] && echo "${tab_}<-- no read permission " && write_error_and_die ;;
"-w") [[ ! -w "${!config_param}" ]] && echo "${tab_}<-- no write permission" && write_error_and_die ;;
"-x") [[ ! -x "${!config_param}" ]] && echo "${tab_}<-- file cannot be executed" && write_error_and_die ;;
# Warning messages only
"-z+") [[ -z "${!config_param}" ]] && echo && return ;;
esac
shift 1
done
echo
}
validate_dir() {
# For parameters ending with a '+' failure causes a warning message only
echo -n "`eval echo $PARAM_VALS`"
while test $# -gt 0 ; do
case $1 in
"-z") [[ -z "${!config_param}" ]] && echo "${tab_}NO directory name given" && write_error_and_die ;;
"-d") [[ ! -d "${!config_param}" ]] && echo "${tab_}This is NOT a directory" && write_error_and_die ;;
"-w") if [[ ! -w "${!config_param}" ]]; then
echo "${nl_}${DD_BORDER}"
echo "${tab_}${RED}You do not have ${L_arrow}write${R_arrow}${RED} access to the directory${OFF}"
echo "${tab_}${BOLD}${!config_param}${OFF}"
echo "${DD_BORDER}${nl_}"
exit 1
fi ;;
# Warnings only
"-w+") if [[ ! -w "${!config_param}" ]]; then
echo "${nl_}${DD_BORDER}"
echo "${tab_}WARNING-- You do not have ${L_arrow}write${R_arrow} access to the directory${OFF}"
echo "${tab_} -- ${BOLD}${!config_param}${OFF}"
echo "${DD_BORDER}"
fi ;;
"-z+") [[ -z "${!config_param}" ]] && echo "${tab_}<-- NO directory name given" && return
esac
shift 1
done
echo
}
set +e set +e
for PARAM_GROUP in ${PROGNAME}_PARAM_LIST; do PARAM_GROUP=${PROGNAME}_PARAM_LIST
for config_param in ${!PARAM_GROUP}; do for config_param in ${!PARAM_GROUP}; do
# This is a tricky little piece of code.. executes a cmd string. # This is a tricky little piece of code.. executes a cmd string.
case $config_param in case $config_param in
BUILDDIR) # We cannot have an <empty> or </> root mount point BUILDDIR) # We cannot have an <empty> or </> root mount point
echo -e "`eval echo $PARAM_VALS`" echo -e "`eval echo $PARAM_VALS`"
if [[ "xx x/x" =~ "x${!config_param}x" ]]; then [[ "xx x/x" =~ "x${!config_param}x" ]] &&
write_error_and_die write_error_and_die
fi ;;
continue ;; TIMEZONE) ;;
TIMEZONE) continue;;
MKFILE) continue;;
HPKG) validation_str="x0x x1x"; validate_str; continue ;;
RUNMAKE) validation_str="x0x x1x"; validate_str; continue ;;
TEST) validation_str="x0x x1x x2x x3x"; validate_str; continue ;;
REPORT) validation_str="x0x x1x"; validate_str;
if [[ "${!config_param}" = "1" ]]; then
if [[ `type -p bc` ]]; then
continue
else
echo -e " ${BOLD}The bc binary was not found${OFF}"
echo -e " The SBU and disk usage report creation will be skiped"
REPORT=0
continue
fi
fi ;;
STRIP) validation_str="x0x x1x"; validate_str; continue ;;
VIMLANG) validation_str="x0x x1x"; validate_str; continue ;;
DEPEND) validation_str="x0x x1x x2x"; validate_str; continue ;;
MODEL) validation_str="xglibcx xuclibcx"; validate_str; continue ;;
PAGE) validation_str="xletterx xA4x"; validate_str; continue ;;
GRSECURITY_HOST) validation_str="x0x x1x"; validate_str; continue ;;
METHOD) validation_str="xchrootx xbootx"; validate_str; continue ;;
ARCH) validation_str="xx86x xx86_64x xx86_64-64x xsparcx xsparc64x xsparc64-64x xmipsx xmips64x xmips64-64x xppcx xppc64x xalphax"; validate_str; continue ;;
TARGET) validate_target; continue ;;
esac
# Validate general parameters..
HPKG) validate_against_str "x0x x1x" ;;
RUNMAKE) validate_against_str "x0x x1x" ;;
REPORT) validate_against_str "x0x x1x"
if [[ "${!config_param}" = "1" ]]; then
if [[ `type -p bc` ]]; then
continue
else
echo -e " ${BOLD}The bc binary was not found${OFF}"
echo -e " The SBU and disk usage report creation will be skiped"
REPORT=0
continue
fi
fi ;;
COMPARE) if [[ ! "$COMPARE" = "1" ]]; then
validate_against_str "x0x x1x"
else
if [[ ! "${RUN_ICA}" = "1" ]] && [[ ! "${RUN_FARCE}" = "1" ]]; then
echo "${nl_}${DD_BORDER}"
echo "You have elected to analyse your build but have failed to select a tool." >&2
echo "Edit /common/config and set ${L_arrow}${BOLD}RUN_ICA${R_arrow} and/or ${L_arrow}${BOLD}RUN_FARCE${R_arrow} to the required values" >&2
echo "${DD_BORDER}${nl_}"
exit 1
fi
fi ;;
RUN_ICA) [[ "$COMPARE" = "1" ]] && validate_against_str "x0x x1x" ;;
RUN_FARCE) [[ "$COMPARE" = "1" ]] && validate_against_str "x0x x1x" ;;
ITERATIONS) [[ "$COMPARE" = "1" ]] && validate_against_str "x2x x3x x4x x5x" ;;
TEST) validate_against_str "x0x x1x x2x x3x" ;;
STRIP) validate_against_str "x0x x1x" ;;
VIMLANG) validate_against_str "x0x x1x" ;;
DEPEND) validate_against_str "x0x x1x x2x" ;;
MODEL) validate_against_str "xglibcx xuclibcx" ;;
PAGE) validate_against_str "xletterx xA4x" ;;
METHOD) validate_against_str "xchrootx xbootx" ;;
ARCH) validate_against_str "xx86x xx86_64x xx86_64-64x xsparcx xsparc64x xsparc64-64x xmipsx xmips64x xmips64-64x xppcx xppc64x xalphax" ;;
TARGET) validate_target ;;
GRSECURITY_HOST) validate_against_str "x0x x1x" ;;
if [[ "${config_param}" = "LC_ALL" ]]; then # BOOK validation. Very ugly, need be fixed
echo "`eval echo $PARAM_VALS`" BOOK) if [[ "${WC}" = "1" ]] ; then
[[ -z "${!config_param}" ]] && echo -e "\nVariable LC_ALL cannot be empty!" && write_error_and_die validate_dir -z -d
# See it the locale values exist on this machine else
if [[ "`locale -a | grep -c ${!config_param}`" > 0 ]]; then validate_against_str "x${PROGNAME}-developmentx xlfs-udev_updatex"
continue fi
else # If you make it this far then there is a problem ;;
write_error_and_die
fi
fi
if [[ "${config_param}" = "LANG" ]]; then # Validate directories, testable states:
echo "`eval echo $PARAM_VALS`" # fatal -z -d -w,
[[ -z "${!config_param}" ]] && echo -e "\nVariable LANG cannot be empty!" && write_error_and_die # warning -z+ -w+
# See if the locale values exist on this machine SRC_ARCHIVE) validate_dir -z+ -d -w+ ;;
if [[ "`locale -a | grep -c ${!config_param}`" > 0 ]]; then
continue
else # If you make it this far then there is a problem
write_error_and_die
fi
fi
# Validate files, testable states:
# fatal -z -e -s -w -x -r,
# warning -z+
FSTAB) validate_file -z+ -e -s ;;
CONFIG) validate_file -z+ -e -s ;;
BOOT_CONFIG) [[ "${METHOD}" = "boot" ]] && validate_file -z -e -s ;;
if [[ "${config_param}" = "KEYMAP" ]]; then # Treatment of 'special' parameters
echo "`eval echo $PARAM_VALS`" LANG | \
[[ "${!config_param}" = "none" ]] && continue LC_ALL) # See it the locale values exist on this machine
if [[ -e "/usr/share/kbd/keymaps/${!config_param}" ]] && echo -n "`eval echo $PARAM_VALS`"
[[ -s "/usr/share/kbd/keymaps/${!config_param}" ]]; then [[ -z "${!config_param}" ]] &&
continue echo " -- Variable $config_param cannot be empty!" &&
else write_error_and_die
write_error_and_die [[ ! "`locale -a | grep -c ${!config_param}`" > 0 ]] &&
fi write_error_and_die
fi echo
;;
if [[ "${config_param}" = "SRC_ARCHIVE" ]]; then KEYMAP) echo "`eval echo $PARAM_VALS`"
echo -n "`eval echo $PARAM_VALS`" save_param=${KEYMAP}
if [ ! -z ${SRC_ARCHIVE} ]; then [[ ! "${!config_param}" = "none" ]] &&
if [ ! -d ${SRC_ARCHIVE} ]; then KEYMAP="/usr/share/kbd/keymaps/${KEYMAP}" &&
echo " -- is NOT a directory" validate_file -z -e -s
write_error_and_die KEYMAP=${save_param}
fi ;;
if [ ! -w ${SRC_ARCHIVE} ]; then esac
echo -n "${nl_} [${BOLD}${YELLOW}WARN$OFF] You do not have <write> access to this directory, ${nl_}${tab_}downloaded files can not be saved in this archive"
fi
fi
echo
continue
fi
if [[ "${config_param}" = "FSTAB" ]]; then
echo "`eval echo $PARAM_VALS`"
[[ -z "${!config_param}" ]] && continue
if [[ -e "${!config_param}" ]] &&
[[ -s "${!config_param}" ]]; then
continue
else
write_error_and_die
fi
fi
if [[ "${config_param}" = "BOOK" ]]; then
echo "`eval echo $PARAM_VALS`"
[[ ! "${WC}" = 1 ]] && continue
[[ -z "${!config_param}" ]] && continue
if [[ -e "${!config_param}" ]] &&
[[ -s "${!config_param}" ]]; then
continue
else
write_error_and_die
fi
fi
if [[ "${config_param}" = "CONFIG" ]]; then
echo "`eval echo $PARAM_VALS`"
[[ -z "${!config_param}" ]] && continue
if [[ -e "${!config_param}" ]] &&
[[ -s "${!config_param}" ]]; then
continue
else
write_error_and_die
fi
fi
if [[ "${config_param}" = "BOOT_CONFIG" ]]; then
if [[ "${METHOD}" = "boot" ]]; then
echo "`eval echo $PARAM_VALS`"
# There must be a config file when the build method is 'boot'
[[ -e "${!config_param}" ]] && [[ -s "${!config_param}" ]] && continue
# If you make it this far then there is a problem
write_error_and_die
fi
fi
done
done done
set -e set -e
echo "$tab_***${BOLD}${GREEN} ${PARAM_GROUP%%_*T} config parameters look good${OFF} ***" echo "$tab_***${BOLD}${GREEN} ${PARAM_GROUP%%_*T} config parameters look good${OFF} ***"
} }

36
extras/do_copy_files Executable file
View file

@ -0,0 +1,36 @@
#!/bin/bash
# $Id$
set -e
: <<inline_doc
desc: Copy files from one dir to another dir using tar
usage: do_copy_files $PRUNEPATH $ROOT_DIR $DEST_DIR
input vars: $1 list of dirs that must be skipped by tar
$2 the root dir of the files that will be copied
$3 the dir where the copied files will be placed
externals: --
modifies: --
returns: --
on error:
on success:
inline_doc
TMP_FILE=/tmp/prunelist
echo -en "\nCopying system files to $3 ..."
# Create a file that we can pass to tar as an "exclude list".
# There might be an easier way to achieve tar exclusions? Strip
# the leading /.
for F in $1 ; do
echo .${F} >> $TMP_FILE
done
mkdir -p $3
cd $2
tar -X $TMP_FILE -cf - . | tar -C $3 -xf -
# Clear out the temporary file
rm -f ${TMP_FILE}
echo "done."

77
extras/do_ica_prep Executable file
View file

@ -0,0 +1,77 @@
#!/bin/bash
# $Id$
# Acknowledgment:
# The following code is a modified version of an original work written by
# Greg Schafer for the "DIY Linux" project and is included here with his
# permission.
# ref: http://www.diy-linux.org
#
set -e
: <<inline_doc
desc: prepare current iteration files for ICA report
usage: do_ica_prep $DEST_ICA/$ITERATION
input vars: $1 directory where files from current iteration are stored
externals: --
modifies: --
returns: --
on error:
on success:
inline_doc
CMP_DIR=$1
# Run ica_prep if it hasn't been done already
if [ ! -f "$CMP_DIR/icaprep" ]; then
echo -en "\nRemoving symbolic links in ${CMP_DIR}... "
find $CMP_DIR -type l | xargs rm -f
echo "done."
echo -n "Gunzipping \".gz\" files in ${CMP_DIR}... "
find $CMP_DIR -name '*.gz' | xargs gunzip
echo "done."
#echo -n "Bunzipping \".bz2\" files in ${CMP_DIR}... "
#find $CMP_DIR -name '*.bz2' | xargs bunzip2
#echo "done."
# ar archives contain date & time stamp info that causes us
# grief when trying to find differences. Here we perform some
# hackery to allow easy diffing. Essentially, replace each
# archive with a dir of the same name and extract the object
# files from the archive into this dir. Despite their names,
# libieee.a & libmcheck.a are not actual ar archives.
echo -n "Extracting object files from \".a\" files in ${CMP_DIR}... "
L=$(find $CMP_DIR -name '*.a' ! -name 'libieee.a' ! -name 'libmcheck.a')
for F in $L; do
mv $F ${F}.XX
mkdir $F
cd $F
BN=${F##*/}
ar x ../${BN}.XX || {
echo -e "\nError: ar archive extraction failed!\n" >&2
exit 1
}
rm -f ../${BN}.XX
done
echo "done."
echo -n "Stripping (debug) symbols from \".o\" files in ${CMP_DIR}... "
find $CMP_DIR -name '*.o' | xargs strip -p -g 2>/dev/null
echo "done."
echo -n "Stripping (all) symbols from files OTHER THAN \".o\" files in ${CMP_DIR}... "
find $CMP_DIR ! -name '*.o' | xargs strip -p 2>/dev/null || :
echo "done."
# We're all done
echo -en "\nSuccess: ICA preparation for "
echo -e "${CMP_DIR} complete."
touch $CMP_DIR/icaprep
else
echo -e "\n$CMP_DIR was already processed\n"
fi

55
extras/do_ica_work Executable file
View file

@ -0,0 +1,55 @@
#!/bin/bash
# $Id$
# Acknowledgment:
# The following code is a modified version of an original work written by
# Greg Schafer for the "DIY Linux" project and is included here with his
# permission.
# ref: http://www.diy-linux.org
#
set -e
: <<inline_doc
desc: creates the ICA report
usage: do_ica_work $PREV $ITERATION $ICALOGDIR $DEST_ICA
input vars: $1 previous iteration
$2 current iteration
$3 ICA log dir
$4 directory where ICA iterations are stored
externals: --
modifies: --
returns: --
on error:
on success:
inline_doc
RAWDIFF=/tmp/rawdiff.$$
REPORT="${3}/REPORT.${1}V${2}"
echo -en "\nGenerating ICA analysis report $1 versus $2 ..."
mkdir -p $3
cd $4
echo -e "Diffing ${1} and ${2}... " > $REPORT
diff -ur ${1} ${2} > $RAWDIFF || :
echo -e "The list of binary files that differ:\n" > $REPORT
grep "iles.*differ$" $RAWDIFF >> $REPORT
echo -e "The list of files that exist \"only in\" 1 of the directories:\n" >> $REPORT
if grep "^Only in" $RAWDIFF >/dev/null 2>&1; then
grep "^Only in" $RAWDIFF >> $REPORT
else
echo NONE >> $REPORT
fi
grep -v "iles.*differ$" $RAWDIFF | \
grep -v "^Only in" > ${3}/${1}V${2}.ASCII.DIFF
rm -f $RAWDIFF
echo "done."

896
extras/farce Executable file
View file

@ -0,0 +1,896 @@
#!/bin/bash
# Acknowledgment:
# The following code is a no-modified version (except for this comment
# and the inline_doc fragment) of an original work written by
# Ken Moffat and is included here with his permission.
#
# FARCE: Farce Assists Rebuild Comparison Evaluation ;)
#
# to answer the question "can it rebuild itself?"
#
# We expect four arguments - first directory path, filelist
# containing the files in this directory which we wish to compare,
# second directory path, filelist for second directory.
#
# Yes, we could just compare everything in each tree, but the
# filelist script knows about files it can reasonably ignore,
# and this also allows us to build a sytem, boot it and get a
# list of files, build a full desktop environment, and only then
# build and boot the "can it build itself" test system and get
# _its_ filelist.
#
# What this script aims to do:
# ____________________________
#
# First, report files not in both builds.
#
# Then, confirm symlinks point to same targets.
#
# After that, compare individual files -
# if different, run the file name through 'expected'
# to pick out files that are unlikely to match (logs,
# pids, fstab [assumes '/' is a different device each time],
# count these as 'expected'.
#
# For whatever is left, check the file type - ar archives
# have their members extraced and compared (every member has
# a timestamp), gzipped files are compared beyond their
# timestamp, binaries, at least those using shared libs or
# which are shared objects, are copied and subjected to
# --strip-debug. If files match at this stage, count them as
# 'accepted'.
#
# As a last step for any file that doesn't match, copy it
# through some perl regexps to "process" it (convert any
# date, time, kernel-version information from standard formats
# into tokens, then see if the tokensi match.
#
# For details of the regexps, see the tokenize function.
# Those files that match after this are also counted as
# 'accepted'. Note that I don't always start from the kernel
# version that I'm going to build, so this copes with e.g. perl
# files that hardcode the kernel version.
#
# We now have files that don't match. A few of these seem to be
# common to all builds - some (members of) c++ libraries or ar
# archives, a few programs which perhaps use some sort of c++ code).
# The file name # is passed to the 'failure' function - these
# recognized filenames are labelled as 'predictable FAIL:',
# anything else is labelled as 'unexpected FAIL:'.
#
# output:
# stderr - files only in one of the builds, failure messages,
# and totals.
#
# farce-results - more details, including which files were treated
# as expected differences, files where neither copy could be read,
# files treated as accepted, with the reason (and member for ar
# archives). This data is typically up to 100 characters wide -
# sometimes it's a bit more, but it doesn't wrap too badly in a
# 100 character xterm.
#
# farce-extras - diffs for the files, or members, that didn't
# match. This file is to establish new regexps for picking up
# date/time/kernel-version formats.
#
# farce-identical - the names of the files which are identical
#
# farce-substitutions - whenever using tokenizeanddiff results in a
# difference being accepted, for both versions diff the before and
# after versions to show what got changed. If the file is a binary,
# the output may still be hard to read. Note that I _know_ glibc
# version strings pass one of the regexps looking for a kernel version
# - since I expect you to use the same version of glibc for each
# build, this is not a problem.
#
# farce-differ - the names of the files which could not be treated
# as matching (whether or not I regard the failure as predictable)
# for possible input to ICA processing.
#
# Copyright (C) 2005, 2006 Ken Moffat <ken@linuxfromscratch.org>
#
# All rights reserved.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or (at
# your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
# NON INFRINGEMENT. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
#
: <<inline_doc
desc: do farce analisys and report
usage: farce -directory $FARCELOGDIR/dir path1 list1 path2 list2
input vars: $1 farce log dir for this comparation
$2 full path to previous iteration
$3 full path to filelist for previos iteration
$4 full path to current iteration
$5 full path to filelist for current iteration
externals: --
modifies: --
returns: --
on error:
on success:
inline_doc
VERSION="002"
# variables for output files
RESULT=farce-results
EXTRAS=farce-extras
IDENTICAL=farce-identical
SUBS=farce-substitutions
DIFFER=farce-differ
# documenting the variables
# C1, C2 temp files to hold disassembled code
# D1, D2 temp directories for extracting member of ar archive
# DIFF temp file for diffing the filelists
# F1, F2 temp files for tokenizeanddiff
# FTYPE obsolete, commented out
# M1, M2 temp files to hold members of an ar archive
# MEMBERS temp file to list members of ar archive
# OP1, OP2 the original $PWD, needed when extracting members of ar
# archives
# P1, P2 paths to first and second build
# S1, S2 temp files for shared objects
# functions
function dohelp() {
echo "`basename $0`: compare trees of files from a build"
echo "and lists of the files they contained"
echo ""
echo "`basename $0` [ -help | -version ] || path1 list1 path2 list2"
}
function emessage() {
# write a string to both stderr and $RESULT
echo "$@" >&2
echo "$@" >&5
}
function expected() {
# if we expect it to differ because of its name,
# allow it and report, return true ; else return false
case $1 in
/boot/grub/menu.lst)
# just in case somebody puts this into the main filesystem
true;;
/etc/aliases.db)
# some sort of database for postfix, not parsable
true;;
/etc/blkid.tab)
# includes dev name for rootfs
true;;
/etc/fstab)
# fstab, e.g. ' / ' will differ
true;;
/etc/group*)
true;;
/etc/hosts)
# with dhcp client, I add current ip address to this in a hook
true;;
/etc/ld.so.*)
# .conf and .cache can vary,
# particularly if one system has a full build when I run this
true;;
/etc/lilo.conf|/etc/yaboot.conf)
# bootloader control, I assume grub will all be on a separate
true;;
/etc/mtab)
# at a minimum, different '/'
true;;
/etc/ntp.drift)
true;;
/etc/passwd*)
true;;
/etc/shadow*)
true;;
/etc/ssh/*key|/etc/ssh/*pub)
# openssh keys
true;;
/misc/*)
# where I put buildscripts (which mostly won't change)
# and stamps containing name/time/space which will differ in the times
true;;
/root/*)
# expect .bash_history etc to differ - if we can read them
true;;
/usr/bin/lynx)
# part of my inital builds, I guess this uses anonymous namespaces
true;;
/usr/include/c++/*/*/bits/stdc++.h.gch/*)
# precompiled headers
true;;
/usr/lib*/libstdc++.a|/usr/lib*/libstdc++.so*|/usr/lib*/libsupc++.a)
# probably, anonymous namespaces
# libstdc++.a, libstdc++.so.n.n.n, libsupc++.a
true;;
/usr/share/info/dir)
# if one system has had extra stuff built, this will likely be bigger
true;;
/usr/share/man/whatis)
# if one system has had extra stuff built, this will likely be bigger
true;;
/var/lib/locate/locatedb)
# if one system has had extra stuff built, this will likely be bigger
true;;
/var/lib/nfs/*)
# allow nfs bookkeeping
true;;
/var/log/*)
true;;
/var/run/utmp)
true;;
/var/spool/fcron*)
true;;
/var/state/*)
# allow dhcp leases
true;;
/var/tmp/random-seed)
true;;
# following start with wildcards
*Image*|*.PPCBoot*|*vmlinuz*|*lfskernel*)
# compressed kernels, sometimes just building at a different
# date/time is enough to change the length of them, because the
# long format date and time is part of the compressed data
true;;
*pid*)
# pids, including e.g. /var/spool/postfix/pid/*
true;;
*)
# nothing else is expected to be different
false;;
esac
if [ $? -eq 0 ]; then
message "expected difference in $1"
let expected=$expected+1
case $TYPE in
AR)
let EXPAR=$EXPAR+1
;;
ELF)
let EXPELF=$EXPELF+1
;;
UNK)
let EXPUNK=$EXPUNK+1
;;
# so far, no other valid types, so don't accumulate them
*)
emessage "internal error, expected difference for $1 of type $TYPE not allowed"
exit 2
;;
esac
true
else
false
fi
}
function failure() {
# first parm is filename or token
# second parm is the error message
# update the appropriate total
# and write to both stderr and the results
# by using emessage
let different=$different+1
case $TYPE in
AR)
let DIFAR=$DIFAR+1
;;
ELF)
let DIFELF=$DIFELF+1
;;
GZ)
let DIFGZ=$DIFGZ+1
;;
SYM)
let DIFSYM=$DIFSYM+1
;;
UNK)
let DIFUNK=$DIFUNK+1
;;
*)
emessage "internal error in failure() for TYPE $TYPE"
exit 2
;;
esac
test -f ${P1}$1 && echo $1 >&9
emessage "FAIL: $2"
}
function fatal() {
# unrecoverable error
echo $*
exit 1
}
function filetype() {
TYPE=`file ${P1}${FILE}`
case $TYPE in
*'current ar archive'*)
let TOTAR=$TOTAR+1
TYPE=AR
;;
*' ELF '*)
let TOTELF=$TOTELF+1
TYPE=ELF
;;
*'gzip compressed data'*)
let TOTGZ=$TOTGZ+1
TYPE=GZ
;;
*)
let TOTUNK=$TOTUNK+1
TYPE=UNK
;;
esac
}
function message() {
# write a string to $RESULT
echo $* >&5
}
function onlyone() {
#report files only in one build
# text should go to both stderr and the results,
# but blank lines only go to the results
if [ $1 == '<' ]; then
emessage "File(s) only in the first build"
else
emessage "File(s) only in the second build"
fi
message ""
FILES=`cat $DIFF | grep "^$1" | cut -d ' ' -f 2`
for F in $FILES; do
emessage $F
let only=$only+1
done
message ""
}
# 'test' functions are called with three arguments:
# the two pathes and the filename
# - we know the file is of this type, so see if we
# can get it to match by reasonalbe means.
# if not, treat it as different.
#
# NB if pathes are absolute, we need to prefix them
# with the original $PWD to access the .a files
#
function testar() {
# ar archives include timestamps for the members,
# but diff doesn't show file timestamps unless the data differs
# put out a message to help locate which archive any messages
# about the members refer to.
# try just stripping them U1,2 undebuggable
U1=`mktemp` || fatal "cannot create a temporary file"
U2=`mktemp` || fatal "cannot create a temporary file"
cp ${1}${3} $U1
cp ${2}${3} $U2
strip --strip-debug $U1
strip --strip-debug $U2
cmp -s $U1 $U2
rm $U1 $U2
if [ $? -eq 0 ]; then
let accepted=$accepted+1
let ACCAR=$ACCAR+1
message "archive $3 matches after strip --strip-debug"
return
fi
# rest of this function retained primarily for pathologically bad builds
# put out a message in the log to help identify which archive has issues.
message "examining ar archive $3"
D1=`mktemp -d` || fatal "cannot create a temporary directory"
D2=`mktemp -d` || fatal "cannot create a temporary directory"
cd $D1
ar -x ${OP1}${1}${3}
cd $D2
ar -x ${OP2}${2}${3}
cd
# diff the members - true means they match
diff -Na $D1 $D2 >/dev/null
if [ $? -eq 0 ]; then
message "accept: $3 after diffing the members"
let accepted=$accepted+1
let ACCAR=$ACCAR+1
else
# process individual members to eliminate date/time/kernel-version
# first, check the members are the same
M1=`mktemp` || fatal "cannot create a temporary file"
M2=`mktemp` || fatal "cannot create a temporary file"
cd $D1
MEMBERS=
for F in *; do
MEMBERS="$MEMBERS $F"
done
cd
echo $MEMBERS | sort >$M1
cd $D2
MEMBERS=
for F in *; do
MEMBERS="$MEMBERS $F"
done
cd
echo $MEMBERS | sort >$M2
cmp -s $M1 $M2
if [ $? -ne 0 ]; then
# oh dear, different members
echo "list of members differs for archive $3" >&6
diff $M1 $M2 >&6
failure $3 "$3 list of members differs"
else
# members (names) are same,
# process each one
STATUS=0
for M in $MEMBERS; do
#avoid firing up perl on matching members
cmp -s $D1/$M $D2/$M
if [ $? -ne 0 ]; then
tokenizeanddiff $D1/$M $D2/$M $FILE:$M
if [ $? -eq 0 ]; then
message "member $M matches after processing"
else
message "member $M DIFFERS after processing"
STATUS=1
fi
fi
done
if [ $STATUS -eq 0 ]; then
let accepted=$accepted+1
let ACCAR=$ACCAR+1
else
let different=$different+1
let DIFAR=$DIFAR+1
echo $3 >&9
emessage "FAIL: in $3"
fi
fi
rm $M1 $M2
fi
rm -rf $D1 $D2
}
function testgzip() {
# bytes 4,5,6,7 are the timestamp, so ignore these
cmp -s -i 8 ${1}${3} ${2}${3}
if [ $? -eq 0 ]; then
message "accept: $3 after ignoring gzip timestamp"
let accepted=$accepted+1
let ACCGZ=$ACCGZ+1
else
failure $3 " $3 even after ignoring gzip timestamp"
fi
}
function testso() {
# shared object - first try stripping it
# in fact, this now handles ALL ELF files
S1=`mktemp` || fatal "cannot create a temporary file"
S2=`mktemp` || fatal "cannot create a temporary file"
cp ${1}${3} $S1
strip --strip-debug $S1
cp ${2}${3} $S2
strip --strip-debug $S2
cmp -s $S1 $S2
if [ $? -eq 0 ]; then
message "accept: $3 after --strip-debug"
let accepted=$accepted+1
let ACCELF=$ACCELF+1
else
tokenizeanddiff $S1 $S2 $3
if [ $? -ne 0 ]; then
failure $3 " $3 differs after stripping and processing"
else
message "accept: $3 after --strip-debug and processing"
let accepted=$accepted+1
let ACCELF=$ACCELF+1
fi
fi
rm $S1 $S2
}
function tokenize() {
# use regexes to replace date/time/kernel-version text
# with tokens which may allow files to match even though
# they have hardcoded date/time/kernel-version.
# arguments are file to process, and where to put it.
# these regexes are somewhat long, and the order they
# are applied in is important (to stop short ones being
# used when a longer version would match).
# KV00 linux version date (e.g. as in the kernel itself)
# allow 2 or 3 groups of three alphas here - optional smp, with day, mon
# KV01 kernel version, including possible cpu details (that is for cdda2wav)
# KV02 just the version, in quotes e.g. "2.6.12.6" or '2.6.13', for perl stuff
# except that "|' gives me grif, so try a boundary
# also, it might need local version on the end, I really want
# quote2.\d+.\d+.{0,32}quote - it is the quotes that don't work.
# DT00 Day Mon .d+ hh:mm:ss TZN CCYY variations include non-caps and 'mon d'
# DT01 Mon .d+ CCYY hh:mm:ss
# DT02 hh:mm:ss Mon .d CCYY
# DT03 Mon .d CCYY
# DT04 Day Mon { ,d}d hh:mm:ss CCYY - for groff example postscript files
# (somewhat similar to DT00, but ' d' or ' dd' for day of month and no TZN )
# DT05 hh:mm:ss
# DT06 ISO date using space as separator
# DT07 ISO date using dash as separator
# DT08 ISO date using slash as separator
# DT09 fullmonth (capitalised), day number, comma, 4-digit year (groff 1.18.1 ps)
# DT10 dd, fullmonth (capitalised), 4-digit year (groff 1.18.1 manpages)
# DT11 '(xample comma space digit(s) backslash ) in groff memef.ps which is
# quite clearly the day of the month when it was compiled, preceded by 'example'
# with something weird between the e and the x.
if [ $# -ne 2 ]; then
fatal "tokenizing called with $# arguments : $*"
fi
cat $1 | perl -p \
-e 's/(L|l)inux.*\d\.\d\.\d+.* \#\d+( [A-Za-z][a-z]{2}){2,3} \d+ \d\d:\d\d:\d\d [A-Za-z]{3} \d{4}\b/%KV00%/g;' \
-e 's/(L|l)inux( (\w|_)+)?(-| |_)\d\.\d(\.\d+){1,2}((-|_)?(\w|_)+)?( |\000)*/%KV01%/g;' \
-e 's/\W2(\.\d+){2,3}(-|_)?((\w|_)+)?\s*\W/%KV02%/g;' \
-e 's/\b([A-Za-z][a-z]{2} ){2}( |\d)?\d \d\d:\d\d:\d\d [A-Za-z]{3} \d{4}\b/%DT00%/g;' \
-e 's/\b[A-Z][a-z]{2} ( |\d)\d \d{4} \d\d:\d\d:\d\d\b/%DT01%/g;' \
-e 's/\b\d\d:\d\d:\d\d [A-Z][a-z]{2} ( |\d)\d \d{4}\b/%DT02%/g;' \
-e 's/\b[A-Z][a-z]{2} ( |\d)\d \d{4}\b/%DT03%/g;' \
-e 's/\b([A-Z][a-z]{2} ){2}( |\d)\d \d\d:\d\d:\d\d \d{4}/%DT04%/g;' \
-e 's/\b\d\d:\d\d:\d\d\b/%DT05%/g;' \
-e 's/\b\d{4} \d\d \d\d\b/%DT06%/g;' \
-e 's/\b\d{4}-\d\d-\d\d\b/%DT07%/g;' \
-e 's/\b\d{4}\/\d\d\/\d\d\b/%DT08%/g;' \
-e 's/\b[A-Z][a-z]{2,} \d{1,2}, \d{4}/%DT09%/g;' \
-e 's/\b\d\d [A-Z][a-z]{2,} \d{4}/%DT10%/g;' \
-e 's/\(xample, \d{1,2}\\\)/%DT11%/g;' \
>$2
}
function tokenizeanddiff() {
# Call tokenize for the inputs, then compare the results
# Input arguments are path/filename for old and new versions
# third parm is readable name (filename, or archivename:member)
# to help understand what is in the extras output.
# - sometimes called for files, but other times called for
# members of ar archives extracted into temporary directories
#message tokenizeanddiff called for $1 $2 $3
F1=`mktemp` || fatal "cannot create a temporary file"
F2=`mktemp` || fatal "cannot create a temporary file"
tokenize $1 $F1
tokenize $2 $F2
# actually, cmp is probably more efficient
# but for picking up the pieces it will be better to
# use diff to see what got through.
cmp -s $F1 $F2
TOKENRESULT=$?
if [ $TOKENRESULT -ne 0 ]; then
echo "failure in $3..." >&6
diff -a $F1 $F2 >&6
rm $F1 $F2
false
else
# show what we did
echo "substitutions for $3" >&8
echo "build one" >&8
diff -a $1 $F1 >&8
echo "build two" >&8
diff -a $2 $F2 >&8
rm $F1 $F2
true
fi
}
function validateargs() {
# validate the arguments
BAD=0
if ! [ -d $1 ]; then
echo "Error: first argument is not a directory" >&2
let BAD=$BAD+1
fi
NAME=`basename ${2%%-*}`
if [ $NAME != filelist ]; then
echo "Error: second argument is not a recognized filelist" >&2
let BAD=$BAD+1
fi
if ! [ -d $3 ]; then
echo "Error: third argument is not a directory" >&2
let BAD=$BAD+1
fi
NAME=`basename ${4%%-*}`
if [ $NAME != filelist ]; then
echo "Error: fourth argument is not a recognized filelist" >&2
let BAD=$BAD+1
fi
for I in $1 $2 $3 $4; do
if ! [ -r $I ]; then
echo "Error: cannot read $I" >&2
let BAD=$BAD+1
fi
done
if [ $1 == $3 ]; then
echo "Error: directory pathes are identical" >&2
let BAD=$BAD+1
fi
if [ $2 == $4 ]; then
echo "Error: filelist names are identical" >&2
let BAD=$BAD+1
fi
if [ $BAD -eq 0 ]; then
ARGS=valid
fi
}
# Mainline
ARGS=unproven
OUTDIR=
if [ $# -eq 1 ]; then
case $1 in
-version|--version)
echo "`basename $0` version $VERSION"
exit 0
;;
-help|--help)
dohelp
exit 0
;;
esac
fi
if [ $1 = "--directory" ]; then
OUTDIR=$2
shift 2
grep '/$' $OUTDIR >/dev/null 2>&1 || OUTDIR=`echo $OUTDIR | sed 's%$%/%'`
echo "creating directory $OUTDIR"
mkdir -p $OUTDIR
if [ $? -ne 0 ]; then
echo "cannot mkdir $OUTDIR"
exit 1
fi
fi
if [ $# -eq 4 ]; then
validateargs $*
fi
if ! [ $ARGS == valid ]; then
dohelp
fatal "`basename $0`: error in arguments"
fi
# ok, we're happy, lets hit these files
exec 5>${OUTDIR}$RESULT
exec 6>${OUTDIR}$EXTRAS
exec 7>${OUTDIR}$IDENTICAL
exec 8>${OUTDIR}$SUBS
exec 9>${OUTDIR}$DIFFER
>${OUTDIR}$RESULT
if [ $? -ne 0 ]; then
fatal "cannot write to ${OUTDIR}$RESULT"
fi
emessage "will compare:"
emessage " first build at $1 with files listed in $2"
emessage "second build at $3 with files listed in $4"
let accepted=0
let different=0
let expected=0
let matched=0
let only=0
let predictable=0
let unreadable=0
let total=0
# break down the accepted
let ACCAR=0
let ACCELF=0
let ACCGZ=0
let ACCUNK=0
# break down definitely different
let DIFAR=0
let DIFELF=0
let DIFGZ=0
let DIFSYM=0
let DIFUNK=0
# break down the expected differences
let EXPAR=0
let EXPELF=0
let EXPGZ=0
let EXPUNK=0
# break down the identical files
let MATAR=0
let MATELF=0
let MATGZ=0
let MATSYM=0
let MATUNK=0
# break down how many of each type
let TOTAR=0
let TOTELF=0
let TOTGZ=0
let TOTSYM=0
let TOTUNK=0
# now identify differences between the two trees
DIFF=`mktemp` || fatal "cannot create a temporary file"
diff $2 $4 >$DIFF
for RUN in '<' '>' ; do
grep -q "$RUN" $DIFF && onlyone "$RUN"
done
rm $DIFF
# and compare them
message "Results of file comparison:"
message ""
# Strip any trailing slash from the path for tidyness,
# because the filenames all start with a slash. Unfortunately,
# unfortunately, '/' becomes empty, which breaks subroutines,
# so special case it.
# also, to process ar archives we need to extract them in temp
# directories - that means that after cd'ing we've broken any
# relative path, so save original pwd as necessary.
P1=`echo $1 | sed 's%/$%%'`
echo $1 | grep '^/' >/dev/null
if [ $? -ne 0 ]; then
# relative path
OP1=${PWD}/
#echo "setting OP1 to $OP1"
else
OP1=
#echo "$1 is an absolute path"
fi
test -z "$P1" && P1='/'
P2=`echo $3 | sed 's%/$%%'`
echo $3 | grep '^/' >/dev/null
if [ $? -ne 0 ]; then
# relative path
OP2=${PWD}/
#echo "setting OP2 to $OP2"
else
OP2=
#echo "$3 is an absolute path"
fi
test -z "$P2" && P2='/'
echo "about to read $2"
while read FILE ; do
#echo "process $FILE"
#echo "test existence of ${P2}${FILE}"
# confirm it exists in second build
# we have already reported files only in one build
if [ -f ${P2}"${FILE}" ]; then
let total=$total+1
# check we can read both of them
# or count as unreadable - I used to separate only-one-unreadable,
# but if you compre '/' and a _copy_ of /mnt/lfs that assumption
# breaks, so be less picky.
if ! [ -r "${P1}${FILE}" ] || ! [ -r "${P2}${FILE}" ]; then
message "cannot read one or both versions of $FILE"
let unreadable=$unreadable+1
continue
fi
if [ -h "${P1}${FILE}" ]; then
# for symlink, look at what it points to
# exceptionally, do not call filetype
TYPE=SYM
let TOTSYM=$TOTSYM+1
SL1=`ls -l "${P1}${FILE}" | awk '{ print $11 }'`
SL2=`ls -l "${P2}${FILE}" | awk '{ print $11 }'`
if [ "$SL1" = "$SL2" ]; then
echo "symlink $FILE matches for $SL1" >&5
let matched=$matched+1
let MATSYM=$MATSYM+1
else
failure TARGET " symlink $FILE points to $SL1 and $SL2"
echo $FILE >&9
fi
else
# regular file, start by typing it for accounting,
# then compare it
filetype ${P1}${FILE}
cmp -s "${P1}${FILE}" "${P2}${FILE}"
if [ $? -eq 0 ]; then
let matched=$matched+1
case $TYPE in
AR)
let MATAR=$MATAR+1
;;
ELF)
let MATELF=$MATELF+1
;;
GZ)
let MATGZ=$MATGZ+1
;;
UNK)
let MATUNK=$MATUNK+1
;;
*)
echo "unexpected TYPE of $TYPE for $FILE" >&2
exit 2
;;
esac
echo ${FILE} >&7
else
# seems different, can we do better ?
# test if we expect it to differ
expected $FILE
if [ $? -ne 0 ]; then
case $TYPE in
GZ)
testgzip $P1 $P2 $FILE ;;
AR)
testar $P1 $P2 $FILE ;;
ELF)
testso $P1 $P2 $FILE ;;
*)
# long-stop - strip dates from text files
tokenizeanddiff "${P1}${FILE}" "${P2}${FILE}" "$FILE"
if [ $? -eq 0 ]; then
message "accepted $FILE after processing"
let accepted=$accepted+1
let ACCUNK=$ACCUNK+1
else
failure "$FILE" " $FILE is different"
fi
;;
esac
fi
fi
fi
fi
done < $2
message ""
# write totals to stderr as well as the results file
emessage "$only files in only one of the builds"
emessage "$total files compared, of which"
emessage "$unreadable files could not be read, skipped"
emessage "$matched files are identical"
emessage "$expected files differed as expected"
emessage "$accepted files had allowable differences"
#emessage "$predictable files differed as they normally do"
emessage "$different files differed"
# totals of different file types
emessage ""
emessage "$TOTAR ar archives"
emessage " of which $MATAR are identical"
emessage " of which $ACCAR are accepted after strip-debug or extracting, diffing, tokenizing"
emessage " of which $EXPAR differed as expected"
emessage " of which $DIFAR differed"
emessage "$TOTELF ELF executables or shared libraries"
emessage " of which $MATELF are identical"
emessage " of which $ACCELF are accepted after stripping and tokenizing"
emessage " of which $EXPELF differed as expected"
emessage " of which $DIFELF differed"
emessage "$TOTGZ gzipped files"
emessage " of which $MATGZ are identical"
emessage " of which $ACCGZ are accepted after comparing beyond timestamp"
emessage " of which $DIFGZ are different"
emessage "$TOTSYM symbolic links"
emessage " of which $MATSYM are identical"
emessage " of which $DIFSYM have different targets"
emessage "$TOTUNK other files"
emessage " of which $MATUNK are identical"
emessage " of which $ACCUNK are accepted after tokenizing"
emessage " of which $EXPUNK differed as expected"
emessage " of which $DIFUNK differed"

60
extras/filelist Executable file
View file

@ -0,0 +1,60 @@
#!/bin/bash
#$Id$
# Acknowledgment:
# The following code is a modified version of an original work written by
# Ken Moffat for their "farce" project and is included here with his
# permission.
#
set -e
: <<inline_doc
desc: creates farce file lists
usage: filelist $DEST_FARCE/$ITERATION $DEST_FARCE/$ITERATION.filelist
input vars: $1 directory where files from current iteration are stored
$2 name of the file list to be created
externals: --
modifies: --
returns: --
on error:
on success:
inline_doc
if [ $# -eq 2 ]; then
OUTFILE=$2
if [ -e $2 ]; then
echo -e "\nOutput $2 already exists\n"
exit
fi
else
echo -e "\nMissing argument\n"
exit 2
fi
if [ "$1" == "/" ]; then
LOC=$1
else
# ensure the path or mountpoint ends with a slash
# because of the seds after the 'find'
LOC=`echo $1 | sed 's%[^/]$%&/%'`
fi
echo -en "\nCreating file list for farce amalysis in $OUTFILE ..."
if [ -f $OUTFILE ]; then
echo "refusing to overwrite $OUTFILE"
exit 1
fi
# check we can indeed do this
>$OUTFILE
if [ $? -ne 0 ]; then
echo "error, cannot write to $OUTFILE"
exit 1
fi
find $LOC -xdev -xtype f | sed "s%^${LOC}%/%" | sort >$OUTFILE
echo -e "done.\n"
exit

35
master.sh
View file

@ -51,12 +51,16 @@ source $COMMON_DIR/common-functions
[[ $? > 0 ]] && echo " $COMMON_DIR/common-functions did not load.." && exit [[ $? > 0 ]] && echo " $COMMON_DIR/common-functions did not load.." && exit
[[ $VERBOSITY > 0 ]] && echo "OK" [[ $VERBOSITY > 0 ]] && echo "OK"
# #
[[ $VERBOSITY > 0 ]] && echo -n "Loading masterscript conf..." [[ $VERBOSITY > 0 ]] && echo -n "Loading masterscript conf..."
source $COMMON_DIR/config source $COMMON_DIR/config
[[ $? > 0 ]] && echo "$COMMON_DIR/conf did not load.." && exit [[ $? > 0 ]] && echo "$COMMON_DIR/conf did not load.." && exit
[[ $VERBOSITY > 0 ]] && echo "OK" [[ $VERBOSITY > 0 ]] && echo "OK"
# #
[[ $VERBOSITY > 0 ]] && echo -n "Loading compare module..."
source $COMMON_DIR/func_compare.sh
[[ $? > 0 ]] && echo "$COMMON_DIR/func_compare.sh did not load.." && exit
[[ $VERBOSITY > 0 ]] && echo "OK"
#
[[ $VERBOSITY > 0 ]] && echo -n "Loading config module <$MODULE_CONFIG>..." [[ $VERBOSITY > 0 ]] && echo -n "Loading config module <$MODULE_CONFIG>..."
source $MODULE_CONFIG source $MODULE_CONFIG
[[ $? > 0 ]] && echo "$MODULE_CONFIG did not load.." && exit 1 [[ $? > 0 ]] && echo "$MODULE_CONFIG did not load.." && exit 1
@ -92,7 +96,7 @@ source $COMMON_DIR/func_validate_configs.sh
################################### ###################################
### MAIN ### ### MAIN ###
################################### ###################################
# Evaluate any command line switches # Evaluate any command line switches
@ -213,6 +217,29 @@ while test $# -gt 0 ; do
;; ;;
# Common options for LFS, CLFS and HLFS # Common options for LFS, CLFS and HLFS
--comparasion | -C )
test $# = 1 && eval "$exit_missing_arg"
shift
case $1 in
ICA) RUN_ICA=1
RUN_FARCE=0
COMPARE=1
;;
farce) RUN_ICA=0
RUN_FARCE=1
COMPARE=1
;;
both) RUN_ICA=1
RUN_FARCE=1
COMPARE=1
;;
*)
echo -e "\n$1 is an unknown analisys method."
exit 1
;;
esac
;;
--fstab | -F ) --fstab | -F )
test $# = 1 && eval "$exit_missing_arg" test $# = 1 && eval "$exit_missing_arg"
shift shift
@ -454,6 +481,10 @@ fi
if [[ "$PWD" != "$JHALFSDIR" ]]; then if [[ "$PWD" != "$JHALFSDIR" ]]; then
cp $COMMON_DIR/makefile-functions $JHALFSDIR/ cp $COMMON_DIR/makefile-functions $JHALFSDIR/
if [[ "$COMPARE" != "0" ]] ; then
mkdir -p $JHALFSDIR/extras
cp extras/* $JHALFSDIR/extras
fi
if [[ -n "$FILES" ]]; then if [[ -n "$FILES" ]]; then
# pushd/popd necessary to deal with mulitiple files # pushd/popd necessary to deal with mulitiple files
pushd $PACKAGE_DIR 1> /dev/null pushd $PACKAGE_DIR 1> /dev/null