Browse Source

require $(HASH)-Download field in .diff/Index files

Now that we ignore SHA1-only files it makes sense to require also the
provision of hashes for the compressed patches as this was introduced in
the same patchset as support for non-SHA1 hashes in the file itself in
dak and adding support in other archive creators (if they support pdiffs
at all) will likely be in the same batch.

The reason for the change itself is simple: If you are 'scared' enough
about the security of SHA1, you shouldn't uncompress a file you haven't
verified at all – after all, it could be exploiting a bug or a zip bomb.
tags/debian/1.2.7
David Kalnischkies 5 years ago
parent
commit
4a808deaac
2 changed files with 58 additions and 60 deletions
  1. +17
    -24
      apt-pkg/acquire-item.cc
  2. +41
    -36
      test/integration/test-pdiff-usage

+ 17
- 24
apt-pkg/acquire-item.cc View File

@@ -206,12 +206,12 @@ HashStringList pkgAcqMetaBase::GetExpectedHashes() const

APT_CONST bool pkgAcqIndexDiffs::HashesRequired() const
{
/* We don't always have the diff of the downloaded pdiff file.
What we have for sure is hashes for the uncompressed file,
but rred uncompresses them on the fly while parsing, so not handled here.
Hashes are (also) checked while searching for (next) patch to apply. */
/* We can't check hashes of rred result as we don't know what the
hash of the file will be. We just know the hash of the patch(es),
the hash of the file they will apply on and the hash of the resulting
file. */
if (State == StateFetchDiff)
return available_patches[0].download_hashes.empty() == false;
return true;
return false;
}
HashStringList pkgAcqIndexDiffs::GetExpectedHashes() const
@@ -227,7 +227,7 @@ APT_CONST bool pkgAcqIndexMergeDiffs::HashesRequired() const
we can check the rred result after all patches are applied as
we know the expected result rather than potentially apply more patches */
if (State == StateFetchDiff)
return patch.download_hashes.empty() == false;
return true;
return State == StateApplyDiff;
}
HashStringList pkgAcqIndexMergeDiffs::GetExpectedHashes() const
@@ -2022,6 +2022,17 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/
return false;
}

for (auto const &patch: available_patches)
if (patch.result_hashes.usable() == false ||
patch.patch_hashes.usable() == false ||
patch.download_hashes.usable() == false)
{
if (Debug)
std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": provides no usable hashes for " << patch.file
<< " so fallback to complete download" << std::endl;
return false;
}

// patching with too many files is rather slow compared to a fast download
unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0);
if (fileLimit != 0 && fileLimit < available_patches.size())
@@ -2036,7 +2047,6 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/
unsigned short const sizeLimitPercent = _config->FindI("Acquire::PDiffs::SizeLimit", 100);
if (sizeLimitPercent > 0 && TransactionManager->MetaIndexParser != nullptr)
{
// compressed case
unsigned long long downloadSize = std::accumulate(available_patches.begin(),
available_patches.end(), 0llu, [](unsigned long long const T, DiffInfo const &I) {
return T + I.download_hashes.FileSize();
@@ -2065,23 +2075,6 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/
return false;
}
}
// uncompressed case
downloadSize = std::accumulate(available_patches.begin(),
available_patches.end(), 0llu, [](unsigned long long const T, DiffInfo const &I) {
return T + I.patch_hashes.FileSize();
});
if (downloadSize != 0)
{
unsigned long long const downloadSizeIdx = ServerSize;
unsigned long long const sizeLimit = downloadSizeIdx * sizeLimitPercent;
if ((sizeLimit/100) < downloadSize)
{
if (Debug)
std::clog << "Need " << downloadSize << " uncompressed bytes (Limit is " << (sizeLimit/100) << ", "
<< "original is " << downloadSizeIdx << ") so fallback to complete download" << std::endl;
return false;
}
}
}

// we have something, queue the diffs


+ 41
- 36
test/integration/test-pdiff-usage View File

@@ -30,7 +30,7 @@ wasmergeused() {
if echo "$*" | grep -q -- '-o test::cannot-use-pdiff=1'; then
msgtest 'Check if pdiff was' 'not used'
cp -a rootdir/tmp/testsuccess.output rootdir/tmp/aptupdate.output
testsuccess --nomsg grep 'diff_Index: Did not find a good hashsum in the index' rootdir/tmp/aptupdate.output
testsuccess --nomsg grep "diff/Index with Message: Couldn't parse pdiff index" rootdir/tmp/aptupdate.output
return;
fi

@@ -51,8 +51,6 @@ wasmergeused() {
testrun() {
configcompression '.' 'xz'
msgmsg "Testcase: setup the base with: $*"
local DOWNLOADHASH=true
if [ "$1" = 'nohash' ]; then DOWNLOADHASH=false; shift; fi
find aptarchive -name 'Packages*' -type f -delete
cp "${PKGFILE}" aptarchive/Packages
compressfile 'aptarchive/Packages'
@@ -84,12 +82,10 @@ SHA256-History:
$(sha256sum "$PKGFILE" | cut -d' ' -f 1) $(stat -c%s "$PKGFILE") $(basename "$PATCHFILE")
SHA256-Patches:
e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28
$(sha256sum "$PATCHFILE" | cut -d' ' -f 1) $(stat -c%s "$PATCHFILE") $(basename "$PATCHFILE")" > "$PATCHINDEX"
if $DOWNLOADHASH; then
echo "SHA256-Download:
$(sha256sum "$PATCHFILE" | cut -d' ' -f 1) $(stat -c%s "$PATCHFILE") $(basename "$PATCHFILE")
SHA256-Download:
d2a1b33187ed2d248eeae3b1223ea71791ea35f2138a713ed371332a6421f467 197 2010-08-18-2013.28.gz
$(sha256sum "${PATCHFILE}.gz" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE}.gz") $(basename "${PATCHFILE}.gz")" >> "$PATCHINDEX"
fi
$(sha256sum "${PATCHFILE}.gz" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE}.gz") $(basename "${PATCHFILE}.gz")" > "$PATCHINDEX"

generatereleasefiles '+1hour'
signreleasefiles
@@ -99,7 +95,7 @@ SHA256-Patches:
testsuccessequal "$(cat "${PKGFILE}-new")
" aptcache show apt newstuff

msgmsg "Testcase: apply with one patch and SHA1 only: $*"
msgmsg "Testcase: SHA1-only patches are not used: $*"
find aptarchive -name 'Packages*' -type f -delete
cp "${PKGFILE}-new" aptarchive/Packages
compressfile 'aptarchive/Packages'
@@ -114,13 +110,35 @@ SHA1-History:
$(sha1sum "$PKGFILE" | cut -d' ' -f 1) $(stat -c%s "$PKGFILE") $(basename "$PATCHFILE")
SHA1-Patches:
7651fc0ac57cd83d41c63195a9342e2db5650257 19722 2010-08-18-2013.28
$(sha1sum "$PATCHFILE" | cut -d' ' -f 1) $(stat -c%s "$PATCHFILE") $(basename "$PATCHFILE")" > "$PATCHINDEX"
if $DOWNLOADHASH; then
echo "SHA1-Download:
$(sha1sum "$PATCHFILE" | cut -d' ' -f 1) $(stat -c%s "$PATCHFILE") $(basename "$PATCHFILE")
SHA1-Download:
2365ac0ac57cde3d43c63145e8251a3bd5410213 197 2010-08-18-2013.28.gz
$(sha1sum "${PATCHFILE}.gz" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE}.gz") $(basename "${PATCHFILE}.gz")" >> "$PATCHINDEX"
fi
$(sha1sum "${PATCHFILE}.gz" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE}.gz") $(basename "${PATCHFILE}.gz")" > "$PATCHINDEX"
generatereleasefiles '+1hour'
signreleasefiles
rm -rf rootdir/var/lib/apt/lists
cp -a rootdir/var/lib/apt/lists-bak rootdir/var/lib/apt/lists
wasmergeused "$@" -o test::cannot-use-pdiff=1
testnopackage oldstuff
testsuccessequal "$(cat "${PKGFILE}-new")
" aptcache show apt newstuff

msgmsg "Testcase: no download-hashes patches are not used: $*"
find aptarchive -name 'Packages*' -type f -delete
cp "${PKGFILE}-new" aptarchive/Packages
compressfile 'aptarchive/Packages'
mkdir -p aptarchive/Packages.diff
PATCHFILE="aptarchive/Packages.diff/$(date +%Y-%m-%d-%H%M.%S)"
diff -e "${PKGFILE}" "${PKGFILE}-new" > "${PATCHFILE}" || true
cat "$PATCHFILE" | gzip > "${PATCHFILE}.gz"
PATCHINDEX='aptarchive/Packages.diff/Index'
echo "SHA256-Current: $(sha256sum "${PKGFILE}-new" | cut -d' ' -f 1) $(stat -c%s "${PKGFILE}-new")
SHA256-History:
01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b 33053002 2010-08-18-2013.28
$(sha256sum "$PKGFILE" | cut -d' ' -f 1) $(stat -c%s "$PKGFILE") $(basename "$PATCHFILE")
SHA256-Patches:
e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28
$(sha256sum "$PATCHFILE" | cut -d' ' -f 1) $(stat -c%s "$PATCHFILE") $(basename "$PATCHFILE")" > "$PATCHINDEX"
generatereleasefiles '+1hour'
signreleasefiles
rm -rf rootdir/var/lib/apt/lists
@@ -157,13 +175,11 @@ SHA256-History:
SHA256-Patches:
e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28
$(sha256sum "$PATCHFILE" | cut -d' ' -f 1) $(stat -c%s "$PATCHFILE") $(basename "$PATCHFILE")
$(sha256sum "${PATCHFILE2}" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE2}") $(basename "${PATCHFILE2}")" > "$PATCHINDEX"
if $DOWNLOADHASH; then
echo "SHA256-Download:
$(sha256sum "${PATCHFILE2}" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE2}") $(basename "${PATCHFILE2}")
SHA256-Download:
d2a1b33187ed2d248eeae3b1223ea71791ea35f2138a713ed371332a6421f467 197 2010-08-18-2013.28.gz
$(sha256sum "${PATCHFILE}.gz" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE}.gz") $(basename "${PATCHFILE}.gz")
$(sha256sum "${PATCHFILE2}.gz" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE2}.gz") $(basename "${PATCHFILE2}.gz")" >> "$PATCHINDEX"
fi
$(sha256sum "${PATCHFILE2}.gz" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE2}.gz") $(basename "${PATCHFILE2}.gz")" > "$PATCHINDEX"

generatereleasefiles '+2hour'
signreleasefiles
@@ -205,12 +221,10 @@ SHA256-History:
$(sha256sum "$PKGFILE" | cut -d' ' -f 1) $(stat -c%s "$PKGFILE") $(basename "$PATCHFILE")
SHA256-Patches:
e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28
$(sha256sum "$PATCHFILE" | cut -d' ' -f 1) $(stat -c%s "$PATCHFILE") $(basename "$PATCHFILE")" > $PATCHINDEX
if $DOWNLOADHASH; then
echo "SHA256-Download:
$(sha256sum "$PATCHFILE" | cut -d' ' -f 1) $(stat -c%s "$PATCHFILE") $(basename "$PATCHFILE")
SHA256-Download:
d2a1b33187ed2d248eeae3b1223ea71791ea35f2138a713ed371332a6421f467 197 2010-08-18-2013.28.gz
$(sha256sum "${PATCHFILE}.gz" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE}.gz") $(basename "${PATCHFILE}.gz")" >> $PATCHINDEX
fi
$(sha256sum "${PATCHFILE}.gz" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE}.gz") $(basename "${PATCHFILE}.gz")" > "$PATCHINDEX"
# needs to look like a valid command, otherwise the parser will fail before hashes are checked
echo '1d' > "$PATCHFILE"
cat "$PATCHFILE" | gzip > "${PATCHFILE}.gz"
@@ -236,22 +250,16 @@ SHA256-Patches:
diff -e "${PKGFILE}" "${PKGFILE}-new" > "${PATCHFILE}" || true
cat "$PATCHFILE" | gzip > "${PATCHFILE}.gz"
PATCHINDEX='aptarchive/Packages.diff/Index'
BIGSIZE="$(stat -c%s "$PATCHFILE")"
if ! $DOWNLOADHASH; then
BIGSIZE="${BIGSIZE}000"
fi
echo "SHA256-Current: $(sha256sum "${PKGFILE}-new" | cut -d' ' -f 1) $(stat -c%s "${PKGFILE}-new")
SHA256-History:
01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b 33053002 2010-08-18-2013.28
$(sha256sum "$PKGFILE" | cut -d' ' -f 1) $(stat -c%s "$PKGFILE") $(basename "$PATCHFILE")
SHA256-Patches:
e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28
$(sha256sum "$PATCHFILE" | cut -d' ' -f 1) $BIGSIZE $(basename "$PATCHFILE")" > "$PATCHINDEX"
if $DOWNLOADHASH; then
echo "SHA256-Download:
$(sha256sum "$PATCHFILE" | cut -d' ' -f 1) $(stat -c%s "$PATCHFILE") $(basename "$PATCHFILE")
SHA256-Download:
d2a1b33187ed2d248eeae3b1223ea71791ea35f2138a713ed371332a6421f467 197 2010-08-18-2013.28.gz
$(sha256sum "${PATCHFILE}.gz" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE}.gz")000 $(basename "${PATCHFILE}.gz")" >> "$PATCHINDEX"
fi
$(sha256sum "${PATCHFILE}.gz" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE}.gz")000 $(basename "${PATCHFILE}.gz")" > "$PATCHINDEX"
generatereleasefiles '+1hour'
signreleasefiles
testsuccess apt update -o Debug::pkgAcquire::Diffs=1 "$@"
@@ -266,9 +274,6 @@ Debug::Acquire::Transaction "true";
Debug::pkgAcquire::rred "true";' > rootdir/etc/apt/apt.conf.d/rreddebug.conf

testcase() {
testrun nohash -o Acquire::PDiffs::Merge=0 -o APT::Get::List-Cleanup=1 "$@"
testrun nohash -o Acquire::PDiffs::Merge=1 -o APT::Get::List-Cleanup=1 "$@"

testrun -o Acquire::PDiffs::Merge=0 -o APT::Get::List-Cleanup=1 "$@"
testrun -o Acquire::PDiffs::Merge=1 -o APT::Get::List-Cleanup=1 "$@"
testrun -o Acquire::PDiffs::Merge=0 -o APT::Get::List-Cleanup=0 "$@"


Loading…
Cancel
Save