From 9e4b9a1d0db454464cb7deaf185cd92a75f6df37 Mon Sep 17 00:00:00 2001 From: Manuel Canales Esparcia Date: Wed, 26 Apr 2006 20:24:58 +0000 Subject: [PATCH] Merged r2575 and r2576 from trunk. --- CLFS/master.sh | 6 ++ HLFS/master.sh | 3 + LFS/master.sh | 3 + common/common-functions | 18 +++++ common/config | 4 ++ common/create-sbu_du-report.sh | 121 ++++++++++++++++++++++++++++++++ common/func_validate_configs.sh | 17 +++-- common/makefile-functions | 11 +++ master.sh | 1 + 9 files changed, 180 insertions(+), 4 deletions(-) create mode 100755 common/create-sbu_du-report.sh diff --git a/CLFS/master.sh b/CLFS/master.sh index a1dcc80..bffecbb 100755 --- a/CLFS/master.sh +++ b/CLFS/master.sh @@ -784,6 +784,9 @@ bootable_Makefiles() { # done + # Add SBU-disk_usage report target if required + if [[ "$REPORT" = "1" ]] ; then wrt_report ; fi + } @@ -861,6 +864,9 @@ bm_bootable_Makefiles() { # done + # Add SBU-disk_usage report target if required + if [[ "$REPORT" = "1" ]] ; then wrt_report ; fi + } diff --git a/HLFS/master.sh b/HLFS/master.sh index a105a41..8dd37bd 100755 --- a/HLFS/master.sh +++ b/HLFS/master.sh @@ -498,6 +498,9 @@ EOF # Keep the script file name for Makefile dependencies. PREV=$this_script done # for file in chapter07/* + + # Add SBU-disk_usage report target if required + if [[ "$REPORT" = "1" ]] ; then wrt_report ; fi } diff --git a/LFS/master.sh b/LFS/master.sh index 200d4f8..802c0d6 100755 --- a/LFS/master.sh +++ b/LFS/master.sh @@ -327,6 +327,9 @@ chapter789_Makefiles() { # Keep the script file name for Makefile dependencies. PREV=${this_script} done # for file in chapter0{7,8,9}/* + + # Add SBU-disk_usage report target if required + if [[ "$REPORT" = "1" ]] ; then wrt_report ; fi } diff --git a/common/common-functions b/common/common-functions index b266241..094d55c 100644 --- a/common/common-functions +++ b/common/common-functions @@ -399,6 +399,24 @@ EOF } +#----------------------------------# +wrt_report() { # +#----------------------------------# +( +cat << EOF + +create-sbu_du-report: $PREV + @\$(call echo_message, Building) + @./create-sbu_du-report.sh logs $VERSION $TEST + @\$(call echo_report,$VERSION-SBU_DU-$(date --iso-8601).report) + @touch \$@ +EOF +) >> $MKFILE.tmp + +chapter789="$chapter789 create-sbu_du-report" +} + + #----------------------------# run_make() { #----------------------------# diff --git a/common/config b/common/config index 1e0c9a7..67d8f46 100644 --- a/common/config +++ b/common/config @@ -36,6 +36,10 @@ RUNMAKE=0 # (in CLFS, alias to 2) TEST=1 +# Create SBU and disk usage report 0(no)/1(yes) +# NOTE: requires to have bc installed on the host +REPORT=1 + #--- Run the stripping phases 0(no)/1(yes) STRIP=1 diff --git a/common/create-sbu_du-report.sh b/common/create-sbu_du-report.sh new file mode 100755 index 0000000..4e4ba9e --- /dev/null +++ b/common/create-sbu_du-report.sh @@ -0,0 +1,121 @@ +#!/bin/bash +#$Id$ + +set -e + +LOGSDIR=$1 +VERSION=$2 +TESTLEVEL=$3 + +# Make sure that we have a directory as first argument +[[ ! -d "$LOGSDIR" ]] && \ + echo -e "\nUSAGE: create-sbu_du-report.sh logs_directory [book_version testsuites_level]\n" && exit + +# Make sure that the first argument is a jhalfs logs directory +[[ ! -f "$LOGSDIR"/000-masterscript.log ]] && \ + echo -e "\nLooks like $LOGSDIR isn't a jhalfs logs directory.\n" && exit + +# If this script is run manually, the book version and testsuite levels +# may be unknow +[[ -z "$VERSION" ]] && VERSION=unknown +[[ -z "$TESTLEVEL" ]] && TESTLEVEL=unknown + +# If there is iteration logs directories, copy the logs inside iteration-1 +# to the top level dir +[[ -d "$LOGSDIR"/iteration-1 ]] && \ + cp $LOGSDIR/iteration-1/* $LOGSDIR + +# Set the report file +REPORT="$VERSION"-SBU_DU-$(date --iso-8601).report + +# Dump generation time stamp, book version, and testsuites level +echo -e "\n`date`\n" > "$REPORT" +echo -e "Book version is:\t$VERSION\n" >> "$REPORT" +echo -e "Test suites level:\t$TESTLEVEL\n" >> "$REPORT" + +# Dump CPU and memory info +echo -e "\n\t\tCPU type:\n" >> "$REPORT" +cat /proc/cpuinfo >> "$REPORT" +echo -e "\n\t\tMemory info:\n" >> "$REPORT" +free >> "$REPORT" + +# Parse only that logs that have time data +BUILDLOGS=`grep -l "^real\>" $LOGSDIR/*` + +# Match the first timed log to extract the SBU unit value from it +BASELOG=`grep -l "^real\>" $LOGSDIR/* | head -n1` +echo -e "\n\nUsing $BASELOG to obtain the SBU unit value." >> "$REPORT" +BASEMINUTES=`grep "^real\>" $BASELOG | cut -f2 | sed -e 's/m.*//'` +BASESECONDS=`grep "^real\>" $BASELOG | cut -f2 | sed -e 's/.*m//;s/s//'` +SBU_UNIT=`echo "scale=3; $BASEMINUTES * 60 + $BASESECONDS" | bc` +echo -e "The SBU unit value is equal to $SBU_UNIT seconds.\n" >> "$REPORT" + +# Set the first value to 0 for grand totals calculation +SBU2=0 +INSTALL2=0 +INSTALLMB2=0 + +for log in $BUILDLOGS ; do + +#Start SBU calculation + # Build time + BUILDTIME=`grep "^real\>" $log | cut -f2` + # Build time in seconds + MINUTES=`grep "^real\>" $log | cut -f2 | sed -e 's/m.*//'` + SECS=`grep "^real\>" $log | cut -f2 | sed -e 's/.*m//;s/s//'` + TIME=`echo "scale=3; $MINUTES * 60 + $SECS" | bc` + # Calculate build time in SBU + SBU=`echo "scale=3; $TIME / $SBU_UNIT" | bc` + # Append SBU value to SBU2 for grand total + SBU2="$SBU2 + $SBU" + +#Start disk usage calculation + # Disk usage before unpack the package + DU1=`grep "^KB: " $log | head -n1 | cut -f1 | sed -e 's/KB: //'` + DU1MB=`echo "scale=2; $DU1 / 1024" | bc` + # Disk usage before delete sources and build dirs + DU2=`grep "^KB: " $log | tail -n1 | cut -f1 | sed -e 's/KB: //'` + DU2MB=`echo "scale=2; $DU2 / 1024" | bc` + # Calculate disk space required to do the build + REQUIRED1=`echo "$DU2 - $DU1" | bc` + REQUIRED2=`echo "scale=2; $DU2MB - $DU1MB" | bc` + + # Append installed files disk usage to the previous entry, + # except for the first parsed log + if [ "$log" != "$BASELOG" ] ; then + INSTALL=`echo "$DU1 - $DU1PREV" | bc` + INSTALLMB=`echo "scale=2; $DU1MB - $DU1MBPREV" | bc` + echo -e "Installed files disk usage:\t\t\t\t$INSTALL KB or $INSTALLMB MB\n" >> "$REPORT" + # Append install values for grand total + INSTALL2="$INSTALL2 + $INSTALL" + INSTALLMB2="$INSTALLMB2 + $INSTALLMB" + fi + + # Set variables to calculate installed files disk usage + DU1PREV=$DU1 + DU1MBPREV=$DU1MB + + # Append log name + echo -e "\n\t$log" >> "$REPORT" + + # Dump time values + echo -e "Build time is:\t\t\t$BUILDTIME" >> "$REPORT" + echo -e "Build time in seconds is\t$TIME" >> "$REPORT" + echo -e "Approximate SBU time is:\t$SBU" >> "$REPORT" + + # Dump disk usage values + echo -e "\nDisk usage before unpack the package:\t\t\t$DU1 KB or $DU1MB MB" >> "$REPORT" + echo -e "Disk usage before delete sources and build dirs:\t$DU2 KB or $DU2MB MB" >> "$REPORT" + echo -e "Required space to build the package:\t\t\t$REQUIRED1 KB or $REQUIRED2 MB\n" >> "$REPORT" + +done + +# Dump grand totals +TOTALSBU=`echo "scale=3; ${SBU2}" | bc` +echo -e "\nTotal time required to build the systen:\t$TOTALSBU SBU\n" >> "$REPORT" +TOTALINSTALL=`echo "${INSTALL2}" | bc` +TOTALINSTALLMB=`echo "scale=2; ${INSTALLMB2}" | bc` +echo -e "Total Installed files disk usage + (including /tools but not /sources):\t$TOTALINSTALL KB or $TOTALINSTALLMB MB\n" >> "$REPORT" + + diff --git a/common/func_validate_configs.sh b/common/func_validate_configs.sh index e2ae5b1..078a08c 100644 --- a/common/func_validate_configs.sh +++ b/common/func_validate_configs.sh @@ -87,9 +87,9 @@ inline_doc # First internal variables, then the ones that change the book's flavour, and lastly system configuration variables local -r blfs_PARAM_LIST="BOOK BUILDDIR SRC_ARCHIVE HPKG DEPEND TEST" - local -r hlfs_PARAM_LIST="BOOK BUILDDIR SRC_ARCHIVE HPKG RUNMAKE MODEL GRSECURITY_HOST TEST COMPARE RUN_ICA RUN_FARCE ITERATIONS STRIP FSTAB CONFIG KEYMAP PAGE TIMEZONE LANG LC_ALL" - local -r clfs_PARAM_LIST="BOOK BUILDDIR SRC_ARCHIVE HPKG RUNMAKE METHOD ARCH TARGET TEST COMPARE RUN_ICA RUN_FARCE ITERATIONS STRIP FSTAB BOOT_CONFIG CONFIG KEYMAP VIMLANG PAGE TIMEZONE LANG" - local -r lfs_PARAM_LIST="BOOK BUILDDIR SRC_ARCHIVE HPKG RUNMAKE TEST COMPARE RUN_ICA RUN_FARCE ITERATIONS STRIP FSTAB CONFIG VIMLANG PAGE TIMEZONE LANG" + local -r hlfs_PARAM_LIST="BOOK BUILDDIR SRC_ARCHIVE HPKG RUNMAKE MODEL GRSECURITY_HOST TEST REPORT COMPARE RUN_ICA RUN_FARCE ITERATIONS STRIP FSTAB CONFIG KEYMAP PAGE TIMEZONE LANG LC_ALL" + local -r clfs_PARAM_LIST="BOOK BUILDDIR SRC_ARCHIVE HPKG RUNMAKE METHOD ARCH TARGET TEST REPORT COMPARE RUN_ICA RUN_FARCE ITERATIONS STRIP FSTAB BOOT_CONFIG CONFIG KEYMAP VIMLANG PAGE TIMEZONE LANG" + local -r lfs_PARAM_LIST="BOOK BUILDDIR SRC_ARCHIVE HPKG RUNMAKE TEST REPORT COMPARE RUN_ICA RUN_FARCE ITERATIONS STRIP FSTAB CONFIG VIMLANG PAGE TIMEZONE LANG" local -r ERROR_MSG_pt1='The variable \"${L_arrow}${config_param}${R_arrow}\" value ${L_arrow}${BOLD}${!config_param}${R_arrow} is invalid,' local -r ERROR_MSG_pt2=' check the config file ${BOLD}${GREEN}\<$(echo $PROGNAME | tr [a-z] [A-Z])/config\> or \${OFF}' @@ -99,7 +99,7 @@ inline_doc local config_param local validation_str local save_param - + write_error_and_die() { echo -e "\n${DD_BORDER}" echo -e "`eval echo ${ERROR_MSG_pt1}`" >&2 @@ -181,6 +181,15 @@ inline_doc # Validate general parameters.. HPKG) validate_against_str "x0x x1x" ;; RUNMAKE) validate_against_str "x0x x1x" ;; + REPORT) validate_against_str "x0x x1x" + if [[ "${!config_param}" = "1" ]] && [[ `type -p bc` ]]; then + continue + else + echo -e " ${BOLD}The bc binary was not found${OFF}" + echo -e " The SBU and disk usage report creation will be skiped" + REPORT=0 + continue + fi ;; COMPARE) if [[ ! "$COMPARE" = "1" ]]; then validate_against_str "x0x x1x" else diff --git a/common/makefile-functions b/common/makefile-functions index 2a62397..e0975d6 100644 --- a/common/makefile-functions +++ b/common/makefile-functions @@ -30,6 +30,17 @@ define unpack3 tar -xvf `ls -t $(1) | head -n1` > /tmp/unpacked endef +define echo_report + @echo + @echo $(BOLD) The report file $(BLUE)$(1)$(BOLD) has been created + @echo + @echo ${WHITE}Please send the $(BOLD)$(MOUNT_PT)/jhalfs/$(1)$(WHITE) + @echo file to $(BOLD)manuel@linuxfromscratch.org$(WHITE) + @echo + @echo That will help us to keep more accurate SBU and + @echo disk usage values into the book. Thanks. +endef + define echo_finished @echo $(BOLD) @echo -------------------------------------------------------------------------------- diff --git a/master.sh b/master.sh index ba018e0..87e28dd 100755 --- a/master.sh +++ b/master.sh @@ -495,6 +495,7 @@ if [[ "$PWD" != "$JHALFSDIR" ]]; then cp $FILES $JHALFSDIR/ popd 1> /dev/null fi + [[ "$REPORT" = "1" ]] && cp $COMMON_DIR/create-sbu_du-report.sh $JHALFSDIR/ sed 's,FAKEDIR,'$BOOK',' $PACKAGE_DIR/$XSL > $JHALFSDIR/${XSL} export XSL=$JHALFSDIR/${XSL} fi