Browse Source

silently skip acquire of empty index files

There is just no point in taking the time to acquire empty files –
especially as it will be tiny non-empty compressed files usually.
tags/debian/1.2.11
David Kalnischkies 5 years ago
parent
commit
b2fd852459
8 changed files with 41 additions and 47 deletions
  1. +9
    -3
      apt-pkg/acquire-item.cc
  2. +6
    -0
      test/integration/framework
  3. +2
    -1
      test/integration/test-apt-acquire-additional-files
  4. +1
    -0
      test/integration/test-apt-get-build-dep-file
  5. +5
    -1
      test/integration/test-apt-update-empty-files
  6. +15
    -35
      test/integration/test-bug-595691-empty-and-broken-archive-files
  7. +1
    -4
      test/integration/test-compressed-indexes
  8. +2
    -3
      test/integration/test-handle-redirect-as-used-mirror-change

+ 9
- 3
apt-pkg/acquire-item.cc View File

@@ -1151,11 +1151,17 @@ void pkgAcqMetaBase::QueueIndexes(bool const verify) /*{{{*/
else
{
auto const hashes = GetExpectedHashesFor(Target->MetaKey);
if (hashes.usable() == false && hashes.empty() == false)
if (hashes.empty() == false)
{
_error->Warning(_("Skipping acquire of configured file '%s' as repository '%s' provides only weak security information for it"),
if (hashes.usable() == false)
{
_error->Warning(_("Skipping acquire of configured file '%s' as repository '%s' provides only weak security information for it"),
Target->MetaKey.c_str(), TransactionManager->Target.Description.c_str());
continue;
continue;
}
// empty files are skipped as acquiring the very small compressed files is a waste of time
else if (hashes.FileSize() == 0)
continue;
}
}



+ 6
- 0
test/integration/framework View File

@@ -566,6 +566,11 @@ forcecompressor() {
local CONFFILE="${TMPWORKINGDIRECTORY}/rootdir/etc/apt/apt.conf.d/00force-compressor"
echo "Acquire::CompressionTypes::Order { \"${COMPRESS}\"; };
Dir::Bin::uncompressed \"/does/not/exist\";" > "$CONFFILE"
for COMP in $(aptconfig dump 'APT::Compressor' --format '%f%n' | cut -d':' -f 5 | uniq); do
if [ -z "$COMP" -o "$COMP" = '.' -o "$COMP" = "$COMPRESSOR" ]; then continue; fi
echo "Dir::Bin::${COMP} \"/does/not/exist\";" >> "$CONFFILE"
echo "APT::Compressor::${COMP}::Name \"${COMP}-disabled\";" >> "$CONFFILE"
done
}

setupsimplenativepackage() {
@@ -1843,6 +1848,7 @@ listcurrentlistsdirectory() {
} | sort
}
forallsupportedcompressors() {
rm -f "${TMPWORKINGDIRECTORY}/rootdir/etc/apt/apt.conf.d/00force-compressor"
for COMP in $(aptconfig dump 'APT::Compressor' --format '%f%n' | cut -d':' -f 5 | uniq); do
if [ -z "$COMP" -o "$COMP" = '.' ]; then continue; fi
"$@" "$COMP"


+ 2
- 1
test/integration/test-apt-acquire-additional-files View File

@@ -15,6 +15,7 @@ configcompression '.' 'gz'
LOWCOSTEXT='lz4'

buildsimplenativepackage 'foo' 'amd64' '1' 'unstable'
buildsimplenativepackage 'bar' 'all' '1' 'unstable'

setupaptarchive --no-update
changetowebserver
@@ -110,7 +111,7 @@ testequal "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_mai
testequal "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all.$LOWCOSTEXT" find rootdir/var/lib/apt/lists -name '*Contents-all*'
testequal "$(readfile Contents-amd64.$LOWCOSTEXT Contents-all.$LOWCOSTEXT)" aptget indextargets --format '$(FILENAME)' 'Created-By: Contents'
testequal "$(apthelper cat-file rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64.$LOWCOSTEXT)" apthelper cat-file 'aptarchive/dists/unstable/main/Contents-amd64.gz'
testempty apthelper cat-file rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all.$LOWCOSTEXT
testequal "$(apthelper cat-file rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all.$LOWCOSTEXT)" apthelper cat-file 'aptarchive/dists/unstable/main/Contents-all.gz'

rm ./rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64.$LOWCOSTEXT
rm ./rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all.$LOWCOSTEXT


+ 1
- 0
test/integration/test-apt-get-build-dep-file View File

@@ -10,6 +10,7 @@ configarchitecture 'i386'
insertpackage 'stable' 'debhelper' 'i386' '7'
insertpackage 'stable' 'build-essential' 'i386' '1'
insertpackage 'stable' 'build-depends' 'i386' '1'
insertsource 'stable' 'unrelated' 'any' '1'
insertinstalledpackage 'build-conflict' 'i386' '1'

setupaptarchive


+ 5
- 1
test/integration/test-apt-update-empty-files View File

@@ -17,9 +17,13 @@ msgmsg 'Test with file'
rm -rf rootdir/var/lib/apt/lists
testsuccess apt update -o Debug::pkgAcquire::Worker=1
cp rootdir/tmp/testsuccess.output apt.output
testsuccess grep '%0aAlt-Filename:%20/.*/Sources%0a' apt.output
testfailure grep '%0a\(Alt\)\?Filename:%20/.*/Sources\(\.gz\)\?%0a' apt.output
testempty find rootdir/var/lib/apt/lists -name '*_Sources'

msgmsg 'Test with http'
changetowebserver
rm -rf rootdir/var/lib/apt/lists
testsuccess apt update -o Debug::pkgAcquire::Worker=1
cp rootdir/tmp/testsuccess.output apt.output
testfailure grep 'http:600.*Sources' apt.output
testempty find rootdir/var/lib/apt/lists -name '*_Sources'

+ 15
- 35
test/integration/test-bug-595691-empty-and-broken-archive-files View File

@@ -10,35 +10,28 @@ buildaptarchive
touch aptarchive/Packages
setupflataptarchive

testaptgetupdate() {
failureupdate() {
rm -rf rootdir/var/lib/apt
aptget update >testaptgetupdate.diff 2>&1 || true
sed -i -e '/Ign /,+1d' -e '/Release/ d' -e 's#\[[0-9]* [kMGTPY]*B\]#\[\]#' testaptgetupdate.diff
GIVEN="$1"
shift
msgtest "Test for correctness of" "apt-get update with $*"
if [ -z "$GIVEN" ]; then
echo -n "" | checkdiff - testaptgetupdate.diff && msgpass || msgfail
else
echo "$GIVEN" | checkdiff - testaptgetupdate.diff && msgpass || msgfail
fi
rm testaptgetupdate.diff
testfailure aptget update --allow-insecure-repositories
testsuccess grep "^E: Failed to fetch store:${1} Empty files can't be valid archives$" rootdir/tmp/testfailure.output
}
successupdate() {
rm -rf rootdir/var/lib/apt
testwarning aptget update --allow-insecure-repositories
}

createemptyarchive() {
find aptarchive/ \( -name "Packages*" -o -name "en*" \) -type f -delete
touch aptarchive/Packages
echo -n "" | $COMPRESSOR_CMD > aptarchive/${1}.$COMPRESS
generatereleasefiles
signreleasefiles
find aptarchive -name '*Release*' -delete
rm -f aptarchive/Packages
}

createemptyfile() {
find aptarchive/ \( -name "Packages*" -o -name "en*" \) -type f -delete
touch aptarchive/Packages aptarchive/${1}.$COMPRESS
generatereleasefiles
signreleasefiles
find aptarchive -name '*Release*' -delete
rm -f aptarchive/Packages
}

@@ -46,34 +39,21 @@ testoverfile() {
local APTARCHIVE="$(readlink -f ./aptarchive)"
forcecompressor "$1"

msgmsg 'archive over file' "Packages.$COMPRESS"
createemptyarchive 'Packages'
testaptgetupdate "Get:2 file:$APTARCHIVE Packages []
Reading package lists..." "empty archive Packages.$COMPRESS over file"

successupdate
createemptyfile 'Packages'
testaptgetupdate "Get:2 file:$APTARCHIVE Packages
Err:2 file:$APTARCHIVE Packages
Empty files can't be valid archives
Reading package lists...
E: Failed to fetch store:$(readlink -f rootdir/var/lib/apt/lists/partial/$(echo "$APTARCHIVE" | sed -e 's#/#_#g')_Packages.${COMPRESS}) Empty files can't be valid archives
E: Some index files failed to download. They have been ignored, or old ones used instead." "empty file Packages.$COMPRESS over file"
failureupdate "$(readlink -f rootdir/var/lib/apt/lists/partial/$(echo "$APTARCHIVE" | sed -e 's#/#_#g')_Packages.${COMPRESS})"
}

testoverhttp() {
forcecompressor "$1"

msgmsg 'archive over http' "Packages.$COMPRESS"
createemptyarchive 'Packages'
testaptgetupdate "Get:2 http://localhost:${APTHTTPPORT} Packages []
Reading package lists..." "empty archive Packages.$COMPRESS over http"

successupdate
createemptyfile 'Packages'
#FIXME: we should response with a good error message instead
testaptgetupdate "Get:2 http://localhost:${APTHTTPPORT} Packages
Err:2 http://localhost:${APTHTTPPORT} Packages
Empty files can't be valid archives
Reading package lists...
E: Failed to fetch store:$(readlink -f rootdir/var/lib/apt/lists/partial/localhost:${APTHTTPPORT}_Packages.${COMPRESS}) Empty files can't be valid archives
E: Some index files failed to download. They have been ignored, or old ones used instead." "empty file Packages.$COMPRESS over http"
failureupdate "$(readlink -f rootdir/var/lib/apt/lists/partial/localhost:${APTHTTPPORT}_Packages.${COMPRESS})"
}

forallsupportedcompressors testoverfile


+ 1
- 4
test/integration/test-compressed-indexes View File

@@ -33,11 +33,9 @@ testrun() {
msgtest 'Check if all index files are' "${1:-uncompressed}"
if [ "$1" = 'compressed' ]; then
! test -e rootdir/var/lib/apt/lists/*i386_Packages || F=1
! test -e rootdir/var/lib/apt/lists/*all_Packages || F=1
! test -e rootdir/var/lib/apt/lists/*_Sources || F=1
! test -e rootdir/var/lib/apt/lists/*_Translation-en || F=1
test -e rootdir/var/lib/apt/lists/*i386_Packages.$LOWCOSTEXT || F=1
test -e rootdir/var/lib/apt/lists/*all_Packages.$LOWCOSTEXT || F=1
test -e rootdir/var/lib/apt/lists/*_Sources.$LOWCOSTEXT || F=1
test -e rootdir/var/lib/apt/lists/*_Translation-en.$LOWCOSTEXT || F=1
# there is no point in trying pdiff if we have compressed indexes
@@ -47,11 +45,9 @@ testrun() {
# clear the faked pdiff indexes so the glob below works
rm -f rootdir/var/lib/apt/lists/*diff_Index
test -e rootdir/var/lib/apt/lists/*i386_Packages || F=1
test -e rootdir/var/lib/apt/lists/*all_Packages || F=1
test -e rootdir/var/lib/apt/lists/*_Sources || F=1
test -e rootdir/var/lib/apt/lists/*_Translation-en || F=1
! test -e rootdir/var/lib/apt/lists/*i386_Packages.* || F=1
! test -e rootdir/var/lib/apt/lists/*all_Packages.* || F=1
! test -e rootdir/var/lib/apt/lists/*_Sources.* || F=1
! test -e rootdir/var/lib/apt/lists/*_Translation-en.* || F=1
fi
@@ -112,6 +108,7 @@ testovermethod() {
INDCOMP='uncompressed'
else
INDCOMP='compressed'
echo 'APT::Compressor::lz4::Name "lz4";' >> rootdir/etc/apt/apt.conf.d/02compressindex
fi

msgmsg "${1}: ${COMPRESSOR}: Test with $INDCOMP indexes gzip=$INDEX"


+ 2
- 3
test/integration/test-handle-redirect-as-used-mirror-change View File

@@ -16,9 +16,8 @@ rewritesourceslist "http://localhost:${APTHTTPPORT}/redirectme"

testsuccessequal "Get:1 http://0.0.0.0:${APTHTTPPORT} unstable InRelease [$(stat -c %s aptarchive/dists/unstable/InRelease) B]
Get:2 http://0.0.0.0:${APTHTTPPORT} unstable/main Sources [$(stat -c %s aptarchive/dists/unstable/main/source/Sources.gz) B]
Get:3 http://0.0.0.0:${APTHTTPPORT} unstable/main amd64 Packages [$(stat -c %s aptarchive/dists/unstable/main/binary-amd64/Packages.gz) B]
Get:4 http://0.0.0.0:${APTHTTPPORT} unstable/main all Packages [$(stat -c %s aptarchive/dists/unstable/main/binary-all/Packages.gz) B]
Get:5 http://0.0.0.0:${APTHTTPPORT} unstable/main Translation-en [$(stat -c %s aptarchive/dists/unstable/main/i18n/Translation-en.gz) B]
Get:3 http://0.0.0.0:${APTHTTPPORT} unstable/main all Packages [$(stat -c %s aptarchive/dists/unstable/main/binary-all/Packages.gz) B]
Get:4 http://0.0.0.0:${APTHTTPPORT} unstable/main Translation-en [$(stat -c %s aptarchive/dists/unstable/main/i18n/Translation-en.gz) B]
Reading package lists..." aptget update

testsuccessequal "Hit:1 http://0.0.0.0:${APTHTTPPORT} unstable InRelease


Loading…
Cancel
Save