blob: 69cfc969d8315ed20852913b2061ea4d2a0f0d53 [file] [log] [blame]
#!/bin/bash
# Copyright 1999-2011 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
# Author: Karl Trygve Kalleberg <karltk@gentoo.org>
# Rewritten from the old, Perl-based emerge-webrsync script
# Author: Alon Bar-Lev <alon.barlev@gmail.com>
# Major rewrite from Karl's scripts.
# TODO:
# - all output should prob be converted to e* funcs
# - add support for ROOT
#
# gpg key import
# KEY_ID=0x239C75C4
# gpg --homedir /etc/portage/gnupg --keyserver subkeys.pgp.net --recv-keys $KEY_ID
# gpg --homedir /etc/portage/gnupg --edit-key $KEY_ID trust
#
# Only echo if in verbose mode
vvecho() { [[ ${do_verbose} -eq 1 ]] && echo "$@" ; }
# Only echo if not in verbose mode
nvecho() { [[ ${do_verbose} -eq 0 ]] && echo "$@" ; }
# warning echos
wecho() { echo "${argv0}: warning: $*" 1>&2 ; }
# error echos
eecho() { echo "${argv0}: error: $*" 1>&2 ; }
argv0=$0
if ! type -P portageq > /dev/null ; then
eecho "could not find 'portageq'; aborting"
exit 1
fi
eval $(portageq envvar -v FEATURES FETCHCOMMAND GENTOO_MIRRORS \
PORTAGE_BIN_PATH PORTAGE_GPG_DIR \
PORTAGE_NICENESS PORTAGE_RSYNC_EXTRA_OPTS PORTAGE_TMPDIR PORTDIR \
SYNC http_proxy ftp_proxy)
DISTDIR="${PORTAGE_TMPDIR}/emerge-webrsync"
export http_proxy ftp_proxy
# If PORTAGE_NICENESS is overriden via the env then it will
# still pass through the portageq call and override properly.
if [ -n "${PORTAGE_NICENESS}" ]; then
renice $PORTAGE_NICENESS $$ > /dev/null
fi
source "${PORTAGE_BIN_PATH}"/isolated-functions.sh || exit 1
do_verbose=0
do_debug=0
if hasq webrsync-gpg ${FEATURES} ; then
WEBSYNC_VERIFY_SIGNATURE=1
else
WEBSYNC_VERIFY_SIGNATURE=0
fi
if [ ${WEBSYNC_VERIFY_SIGNATURE} != 0 -a -z "${PORTAGE_GPG_DIR}" ]; then
eecho "please set PORTAGE_GPG_DIR in make.conf"
exit 1
fi
do_tar() {
local file=$1; shift
local decompressor
case ${file} in
*.xz) decompressor="xzcat" ;;
*.bz2) decompressor="bzcat" ;;
*.gz) decompressor="zcat" ;;
*) decompressor="cat" ;;
esac
${decompressor} "${file}" | tar "$@"
_pipestatus=${PIPESTATUS[*]}
[[ ${_pipestatus// /} -eq 0 ]]
}
get_utc_date_in_seconds() {
date -u +"%s"
}
get_date_part() {
local utc_time_in_secs="$1"
local part="$2"
if [[ ${USERLAND} == BSD ]] ; then
date -r ${utc_time_in_secs} -u +"${part}"
else
date -d @${utc_time_in_secs} -u +"${part}"
fi
}
get_utc_second_from_string() {
local s="$1"
if [[ ${USERLAND} == BSD ]] ; then
date -juf "%Y%m%d" "$s" +"%s"
else
date -d "${s:0:4}-${s:4:2}-${s:6:2}" -u +"%s"
fi
}
get_portage_timestamp() {
local portage_current_timestamp=0
if [ -f "${PORTDIR}/metadata/timestamp.x" ]; then
portage_current_timestamp=$(cut -f 1 -d " " "${PORTDIR}/metadata/timestamp.x" )
fi
echo "${portage_current_timestamp}"
}
fetch_file() {
local URI="$1"
local FILE="$2"
local opts
if [ "${FETCHCOMMAND/wget/}" != "${FETCHCOMMAND}" ]; then
opts="--continue $(nvecho -q)"
elif [ "${FETCHCOMMAND/curl/}" != "${FETCHCOMMAND}" ]; then
opts="--continue-at - $(nvecho -s -f)"
else
rm -f "${FILE}"
fi
vecho "Fetching file ${FILE} ..."
# already set DISTDIR=
eval "${FETCHCOMMAND}" ${opts}
[ -s "${FILE}" ]
}
check_file_digest() {
local digest="$1"
local file="$2"
local r=1
vecho "Checking digest ..."
if type -P md5sum > /dev/null; then
md5sum -c $digest && r=0
elif type -P md5 > /dev/null; then
[ "$(md5 -q "${file}")" == "$(cut -d ' ' -f 1 "${digest}")" ] && r=0
else
eecho "cannot check digest: no suitable md5/md5sum binaries found"
fi
return "${r}"
}
check_file_signature() {
local signature="$1"
local file="$2"
local r=1
if [ ${WEBSYNC_VERIFY_SIGNATURE} != 0 ]; then
vecho "Checking signature ..."
if type -P gpg > /dev/null; then
gpg --homedir "${PORTAGE_GPG_DIR}" --verify "$signature" "$file" && r=0
else
eecho "cannot check signature: gpg binary not found"
fi
else
r=0
fi
return "${r}"
}
get_snapshot_timestamp() {
local file="$1"
do_tar "${file}" --to-stdout -xf - portage/metadata/timestamp.x | cut -f 1 -d " "
}
sync_local() {
local file="$1"
vecho "Syncing local tree ..."
if type -P tarsync > /dev/null ; then
local chown_opts="-o portage -g portage"
chown portage:portage portage > /dev/null 2>&1 || chown_opts=""
if ! tarsync $(vvecho -v) -s 1 ${chown_opts} \
-e /distfiles -e /packages -e /local "${file}" "${PORTDIR}"; then
eecho "tarsync failed; tarball is corrupt? (${file})"
return 1
fi
else
if ! do_tar "${file}" xf -; then
eecho "tar failed to extract the image. tarball is corrupt? (${file})"
rm -fr portage
return 1
fi
# Free disk space
rm -f "${file}"
chown portage:portage portage > /dev/null 2>&1 && \
chown -R portage:portage portage
cd portage
rsync -av --progress --stats --delete --delete-after \
--exclude='/distfiles' --exclude='/packages' \
--exclude='/local' ${PORTAGE_RSYNC_EXTRA_OPTS} . "${PORTDIR%%/}"
cd ..
vecho "Cleaning up ..."
rm -fr portage
fi
if hasq metadata-transfer ${FEATURES} ; then
vecho "Updating cache ..."
emerge --metadata
fi
[ -x /etc/portage/bin/post_sync ] && /etc/portage/bin/post_sync
return 0
}
do_snapshot() {
local ignore_timestamp="$1"
local date="$2"
local r=1
local base_file="portage-${date}.tar"
local have_files=0
local mirror
local compressions=""
# xz is not supported in app-arch/tarsync, so use
# bz2 format if we have tarsync.
if ! type -P tarsync > /dev/null ; then
type -P xzcat > /dev/null && compressions="${compressions} xz"
fi
type -P bzcat > /dev/null && compressions="${compressions} bz2"
type -P zcat > /dev/null && compressions="${compressions} gz"
if [[ -z ${compressions} ]] ; then
eecho "unable to locate any decompressors (xzcat or bzcat or zcat)"
exit 1
fi
for mirror in ${GENTOO_MIRRORS} ; do
vecho "Trying to retrieve ${date} snapshot from ${mirror} ..."
for compression in ${compressions} ; do
local file="portage-${date}.tar.${compression}"
local digest="${file}.md5sum"
local signature="${file}.gpgsig"
if [ -s "${file}" -a -s "${digest}" -a -s "${signature}" ] ; then
check_file_digest "${DISTDIR}/${digest}" "${DISTDIR}/${file}" && \
check_file_signature "${DISTDIR}/${signature}" "${DISTDIR}/${file}" && \
have_files=1
fi
if [ ${have_files} -eq 0 ] ; then
fetch_file "${mirror}/snapshots/${digest}" "${digest}" && \
fetch_file "${mirror}/snapshots/${signature}" "${signature}" && \
fetch_file "${mirror}/snapshots/${file}" "${file}" && \
check_file_digest "${DISTDIR}/${digest}" "${DISTDIR}/${file}" && \
check_file_signature "${DISTDIR}/${signature}" "${DISTDIR}/${file}" && \
have_files=1
fi
#
# If timestamp is invalid
# we want to try and retrieve
# from a different mirror
#
if [ ${have_files} -eq 1 ]; then
vecho "Getting snapshot timestamp ..."
local snapshot_timestamp=$(get_snapshot_timestamp "${file}")
if [ ${ignore_timestamp} == 0 ]; then
if [ ${snapshot_timestamp} -lt $(get_portage_timestamp) ]; then
wecho "portage is newer than snapshot"
have_files=0
fi
else
local utc_seconds=$(get_utc_second_from_string "${date}")
#
# Check that this snapshot
# is what it claims to be ...
#
if [ ${snapshot_timestamp} -lt ${utc_seconds} ] || \
[ ${snapshot_timestamp} -gt $((${utc_seconds}+ 2*86400)) ]; then
wecho "snapshot timestamp is not in acceptable period"
have_files=0
fi
fi
fi
if [ ${have_files} -eq 1 ]; then
break
else
#
# Remove files and use a different mirror
#
rm -f "${file}" "${digest}" "${signature}"
fi
done
[ ${have_files} -eq 1 ] && break
done
if [ ${have_files} -eq 1 ]; then
sync_local "${file}" && r=0
else
vecho "${date} snapshot was not found"
fi
rm -f "${file}" "${digest}" "${signature}"
return "${r}"
}
do_latest_snapshot() {
local attempts=0
local r=1
vecho "Fetching most recent snapshot ..."
# The snapshot for a given day is generated at 01:45 UTC on the following
# day, so the current day's snapshot (going by UTC time) hasn't been
# generated yet. Therefore, always start by looking for the previous day's
# snapshot (for attempts=1, subtract 1 day from the current UTC time).
# Timestamps that differ by less than 2 hours
# are considered to be approximately equal.
local min_time_diff=$(( 2 * 60 * 60 ))
local existing_timestamp=$(get_portage_timestamp)
local timestamp_difference
local timestamp_problem
local approx_snapshot_time
local start_time=$(get_utc_date_in_seconds)
local start_hour=$(get_date_part ${start_time} "%H")
# Daily snapshots are created at 1:45 AM and are not
# available until after 2 AM. Don't waste time trying
# to fetch a snapshot before it's been created.
if [ ${start_hour} -lt 2 ] ; then
(( start_time -= 86400 ))
fi
local snapshot_date=$(get_date_part ${start_time} "%Y%m%d")
local snapshot_date_seconds=$(get_utc_second_from_string ${snapshot_date})
while (( ${attempts} < 40 )) ; do
(( attempts++ ))
(( snapshot_date_seconds -= 86400 ))
# snapshots are created at 1:45 AM
(( approx_snapshot_time = snapshot_date_seconds + 86400 + 6300 ))
(( timestamp_difference = existing_timestamp - approx_snapshot_time ))
[ ${timestamp_difference} -lt 0 ] && (( timestamp_difference = -1 * timestamp_difference ))
snapshot_date=$(get_date_part ${snapshot_date_seconds} "%Y%m%d")
timestamp_problem=""
if [ ${timestamp_difference} -eq 0 ]; then
timestamp_problem="is identical to"
elif [ ${timestamp_difference} -lt ${min_time_diff} ]; then
timestamp_problem="is possibly identical to"
elif [ ${approx_snapshot_time} -lt ${existing_timestamp} ] ; then
timestamp_problem="is newer than"
fi
if [ -n "${timestamp_problem}" ]; then
ewarn "Latest snapshot date: ${snapshot_date}"
ewarn
ewarn "Approximate snapshot timestamp: ${approx_snapshot_time}"
ewarn " Current local timestamp: ${existing_timestamp}"
ewarn
echo -e "The current local timestamp" \
"${timestamp_problem} the" \
"timestamp of the latest" \
"snapshot. In order to force sync," \
"use the --revert option or remove" \
"the timestamp file located at" \
"'${PORTDIR}/metadata/timestamp.x'." | fmt -w 70 | \
while read -r line ; do
ewarn "${line}"
done
r=0
break
fi
if do_snapshot 0 "${snapshot_date}"; then
r=0
break;
fi
done
return "${r}"
}
usage() {
cat <<-EOF
Usage: $0 [options]
Options:
--revert=yyyymmdd Revert to snapshot
-q, --quiet Only output errors
-v, --verbose Enable verbose output
-x, --debug Enable debug output
-h, --help This help screen (duh!)
EOF
if [[ -n $* ]] ; then
printf "\nError: %s\n" "$*" 1>&2
exit 1
else
exit 0
fi
}
main() {
local arg
local revert_date
[ ! -d "${DISTDIR}" ] && mkdir -p "${DISTDIR}"
cd "${DISTDIR}"
for arg in "$@" ; do
local v=${arg#*=}
case ${arg} in
-h|--help) usage ;;
-q|--quiet) PORTAGE_QUIET=1 ;;
-v|--verbose) do_verbose=1 ;;
-x|--debug) do_debug=1 ;;
--revert=*) revert_date=${v} ;;
*) usage "Invalid option '${arg}'" ;;
esac
done
# This is a sanity check to help prevent people like funtoo users
# from accidentally wiping out their git tree.
if [[ -n $SYNC && ${SYNC#rsync:} = $SYNC ]] ; then
echo "The current SYNC variable setting does not refer to an rsync URI:" >&2
echo >&2
echo " SYNC=$SYNC" >&2
echo >&2
echo "If you intend to use emerge-webrsync then please" >&2
echo "adjust SYNC to refer to an rsync URI." >&2
echo "emerge-webrsync exiting due to abnormal SYNC setting." >&2
exit 1
fi
[[ ${do_debug} -eq 1 ]] && set -x
if [[ -n ${revert_date} ]] ; then
do_snapshot 1 "${revert_date}"
else
do_latest_snapshot
fi
}
main "$@"