2006-11-11 20:06:46 +01:00
#!/bin/bash
# $Id$
#----------------------------#
get_sources() { # Download file, write name to MISSING_FILES.DMP if an error
#----------------------------#
local saveIFS=$IFS
2010-03-12 04:32:25 +01:00
local IFS line URL1 URL2 FILE BOOKMD5 MD5 HAVEMD5 fromARCHIVE WGETPARAM
WGETPARAM=""
if [[ "${RETRYSRCDOWNLOAD}" = "y" ]] ; then
WGETPARAM+="--retry-connrefused"
fi
WGETPARAM+=" --tries ${RETRYDOWNLOADCNT}"
WGETPARAM+=" --timeout ${DOWNLOADTIMEOUT}"
2006-11-11 20:06:46 +01:00
# Test if the packages must be downloaded
[ ! "$GETPKG" = "y" ] && return
gs_wrt_message(){
echo "${RED}$1${OFF}"
echo "$1" >> MISSING_FILES.DMP
}
# Housekeeping
[[ ! -d $BUILDDIR/sources ]] && mkdir $BUILDDIR/sources
cd $BUILDDIR/sources
2013-11-08 23:10:11 +01:00
# If using CLFS, /sources is writable by all, but with sticky bit,
# and user does not hold MD5SUMS nor the other files, so use sudo
[[ -f MD5SUMS ]] && sudo rm MD5SUMS
[[ -f MISSING_FILES.DMP ]] && sudo rm MISSING_FILES.DMP
[[ -f urls.lst ]] && sudo rm urls.lst
2006-11-11 20:06:46 +01:00
# Generate URLs file
create_urls
IFS=$'\x0A' # Modify the 'internal field separator' to break on 'LF' only
for line in `cat urls.lst`; do
IFS=$saveIFS # Restore the system defaults
# Skip some packages if they aren't needed
case $line in
2013-09-19 17:44:49 +02:00
*/tree* | */gcc-testsuite* ) #unneeded now?
2006-11-11 20:06:46 +01:00
[[ "$TEST" = "0" ]] && continue
;;
2012-02-02 00:29:37 +01:00
*/vim-*-lang* )
2006-11-11 20:06:46 +01:00
[[ "$VIMLANG" = "0" ]] && continue
;;
esac
# Locations
URL1=`echo $line | cut -d" " -f2` # Preferred URL
URL2=`echo $line | cut -d" " -f1` # Fallback Upstream URL
FILE=`basename $URL1` # File name
BOOKMD5=`echo $line | cut -d" " -f3` # MD5 book value
# Validation pair
MD5="$BOOKMD5 $FILE"
HAVEMD5=1
set -e
# If the file exists in the archive copy it to the
# $BUILDDIR/sources dir. MD5SUM will be validated later.
if [ ! -z ${SRC_ARCHIVE} ] &&
[ -d ${SRC_ARCHIVE} ] &&
[ -f ${SRC_ARCHIVE}/$FILE ]; then
cp ${SRC_ARCHIVE}/$FILE .
echo "$FILE: -- copied from $SRC_ARCHIVE"
fromARCHIVE=1
else
fromARCHIVE=0
# If the file does not exist yet in /sources download a fresh one
if [ ! -f $FILE ] ; then
2010-03-12 04:46:12 +01:00
if [[ ! ("$SRC_ARCHIVE" = "") ]] ; then
2012-02-02 00:29:37 +01:00
echo "${BOLD}${YELLOW}$FILE: not found in ${SRC_ARCHIVE} or ${BUILDDIR}/sources${OFF}"
2010-03-12 04:46:12 +01:00
else
2012-02-02 00:29:37 +01:00
echo "${BOLD}${YELLOW}$FILE: not found in ${BUILDDIR}/sources${OFF}"
2010-03-12 04:46:12 +01:00
fi
2010-03-12 04:32:25 +01:00
if ! wget $URL1 $WGETPARAM && ! wget $URL2 $WGETPARAM ; then
2006-11-11 20:06:46 +01:00
gs_wrt_message "$FILE not found in the SRC_ARCHIVE or on any server..SKIPPING"
continue
fi
2012-02-02 00:29:37 +01:00
else
echo "${BOLD}${YELLOW}$FILE: using cached file in ${BUILDDIR}/sources${OFF}"
2006-11-11 20:06:46 +01:00
fi
fi
2008-08-26 16:16:37 +02:00
# Deal with udev and bootscripts m5sum issue
2013-05-18 22:46:10 +02:00
[[ $BOOKMD5 = "BOOTSCRIPTS-MD5SUM" ]] && continue
[[ $BOOKMD5 = "UDEV-MD5SUM" ]] && continue
[[ $BOOKMD5 = "LFS-NETSCRIPTS-MD5SUM" ]] && continue
2008-08-26 16:16:37 +02:00
2006-11-11 20:06:46 +01:00
# IF the md5sum does not match the existing files
if ! echo "$MD5" | md5sum -c - >/dev/null ; then
[[ $fromARCHIVE = "1" ]] && echo "${BOLD}${YELLOW}MD5SUM did not match SRC_ARCHIVE copy${OFF}"
[[ $fromARCHIVE = "0" ]] && echo "${BOLD}${YELLOW}MD5SUM did not match REMOTE copy${OFF}"
# Remove the old file and download a new one
rm -fv $FILE
# Force storage in SRC_ARCHIVE
fromARCHIVE=0;
# Try to retrieve again the file. Servers in reverse order.
2010-03-12 04:32:25 +01:00
if ! wget $URL2 $WGETPARAM && ! wget $URL1 $WGETPARAM ; then
2006-11-11 20:06:46 +01:00
gs_wrt_message "$FILE not found on the servers.. SKIPPING"
continue
fi
fi
# Validate the MD5SUM one last time
if ! echo "$MD5" | md5sum -c - >/dev/null ; then
gs_wrt_message "$FILE does not match MD5SUMS value"
# Force generation of MD5SUM
HAVEMD5=0
fi
# Generate a fresh MD5SUM for this file
if [[ "$HAVEMD5" = "0" ]] ; then
echo "${BOLD}${YELLOW}Generating a new MD5SUM for ${OFF}$FILE"
echo "NEW MD5SUM: $(md5sum $FILE)" >> MISSING_FILES.DMP
fi
# Good or bad we write the original md5sum to a file
2012-02-07 23:30:26 +01:00
echo "$MD5" >> MD5SUMS
2006-11-11 20:06:46 +01:00
# Copy the freshly downloaded file
# to the source archive.
if [ ! -z ${SRC_ARCHIVE} ] &&
[ -d ${SRC_ARCHIVE} ] &&
[ -w ${SRC_ARCHIVE} ] &&
[ "$fromARCHIVE" = "0" ] ; then
echo "Storing file:<$FILE> in the package archive"
cp -f $FILE ${SRC_ARCHIVE}
fi
done
if [[ -s MISSING_FILES.DMP ]]; then
echo -e "\n\n${tab_}${RED} One or more files were not retrieved or have bad MD5SUMS.\n${tab_} Check ${L_arrow}$BUILDDIR/sources/MISSING_FILES.DMP${R_arrow} for names ${OFF}\n"
# Do not allow the automatic execution of the Makefile.
echo "${tab_}${BOLD}${RED}*** ${YELLOW}Automatic execution of the generated makefile has been inhibited. ${RED}***${OFF}${nl_}"
RUNMAKE="n"
fi
}
#----------------------------#
create_urls() { #
#----------------------------#
cd $JHALFSDIR
case ${PROGNAME} in
2013-09-14 23:23:16 +02:00
clfs*)
2006-11-11 20:06:46 +01:00
echo -n "Creating CLFS <${ARCH}> specific URLs file"
2017-06-29 16:15:36 +02:00
xsltproc --nonet --xinclude \
--stringparam server "$SERVER" \
--stringparam family clfs \
-o $BUILDDIR/sources/urls.lst \
urls.xsl \
2013-09-14 23:23:16 +02:00
$BOOK/BOOK/materials/$ARCH-chapter.xml >>$LOGDIR/$LOG 2>&1
2006-11-11 20:06:46 +01:00
echo " ...OK"
;;
hlfs)
2007-06-03 12:54:07 +02:00
echo -n "Creating HLFS <${MODEL}> + <${KERNEL}> specific URLs file"
2017-06-29 16:15:36 +02:00
xsltproc --nonet --xinclude \
--stringparam server "$SERVER" \
--stringparam family lfs \
--stringparam model "$MODEL" \
--stringparam kernel "$KERNEL" \
-o $BUILDDIR/sources/urls.lst \
urls.xsl \
2006-11-11 20:06:46 +01:00
$BOOK/chapter04/chapter04.xml >>$LOGDIR/$LOG 2>&1
echo " ...OK"
;;
lfs)
echo -n "Creating LFS specific URLs file"
2017-06-29 16:15:36 +02:00
xsltproc --nonet --xinclude \
--stringparam server "$SERVER" \
--stringparam family lfs \
--stringparam pkgmngt "$PKGMNGT" \
--stringparam revision "$INITSYS" \
--output ../sources/urls.lst \
urls.xsl \
2006-11-11 20:06:46 +01:00
$BOOK/chapter03/chapter03.xml >>$LOGDIR/$LOG 2>&1
echo " ...OK"
;;
esac
cd $BUILDDIR/sources
if [[ "${CUSTOM_TOOLS}" = "y" ]]; then
add_CustomToolsURLS
fi
}