Browse Source

support hashes for compressed pdiff files

At the moment we only have hashes for the uncompressed pdiff files, but
via the new '$HASH-Download' field in the .diff/Index hashes can be
provided for the .gz compressed pdiff file, which apt will pick up now
and use to verify the download. Now, we "just" need a buy in from the
creators of repositories…
tags/debian/1.1.exp9
David Kalnischkies 6 years ago
parent
commit
4f51fd8636
7 changed files with 125 additions and 28 deletions
  1. +60
    -12
      apt-pkg/acquire-item.cc
  2. +4
    -7
      apt-pkg/acquire-item.h
  3. +8
    -3
      apt-pkg/contrib/hashes.cc
  4. +7
    -0
      apt-pkg/contrib/hashes.h
  5. +1
    -3
      apt-pkg/indexrecords.cc
  6. +1
    -1
      methods/rred.cc
  7. +44
    -2
      test/integration/test-pdiff-usage

+ 60
- 12
apt-pkg/acquire-item.cc View File

@@ -152,14 +152,18 @@ HashStringList pkgAcqMetaBase::GetExpectedHashes() const

APT_CONST bool pkgAcqIndexDiffs::HashesRequired() const
{
/* FIXME: We have only hashes for uncompressed pdiffs.
rred uncompresses them on the fly while parsing.
In StateFetchDiff state we also uncompress on the fly for hash check.
Hashes are checked while searching for (next) patch to apply. */
/* We don't always have the diff of the downloaded pdiff file.
What we have for sure is hashes for the uncompressed file,
but rred uncompresses them on the fly while parsing, so not handled here.
Hashes are (also) checked while searching for (next) patch to apply. */
if (State == StateFetchDiff)
return available_patches[0].download_hashes.empty() == false;
return false;
}
HashStringList pkgAcqIndexDiffs::GetExpectedHashes() const
{
if (State == StateFetchDiff)
return available_patches[0].download_hashes;
return HashStringList();
}

@@ -168,11 +172,15 @@ APT_CONST bool pkgAcqIndexMergeDiffs::HashesRequired() const
/* @see #pkgAcqIndexDiffs::HashesRequired, with the difference that
we can check the rred result after all patches are applied as
we know the expected result rather than potentially apply more patches */
if (State == StateFetchDiff)
return patch.download_hashes.empty() == false;
return State == StateApplyDiff;
}
HashStringList pkgAcqIndexMergeDiffs::GetExpectedHashes() const
{
if (State == StateApplyDiff)
if (State == StateFetchDiff)
return patch.download_hashes;
else if (State == StateApplyDiff)
return GetExpectedHashesFor(Target->MetaKey);
return HashStringList();
}
@@ -1618,7 +1626,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/
std::vector<DiffInfo>::iterator cur = available_patches.begin();
for (; cur != available_patches.end(); ++cur)
{
if (cur->file != filename || unlikely(cur->result_size != size))
if (cur->file != filename)
continue;
cur->result_hashes.push_back(HashString(*type, hash));
break;
@@ -1630,8 +1638,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/
DiffInfo next;
next.file = filename;
next.result_hashes.push_back(HashString(*type, hash));
next.result_size = size;
next.patch_size = 0;
next.result_hashes.FileSize(size);
available_patches.push_back(next);
}
else
@@ -1679,10 +1686,9 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/
{
if (cur->file != filename)
continue;
if (unlikely(cur->patch_size != 0 && cur->patch_size != size))
continue;
if (cur->patch_hashes.empty())
cur->patch_hashes.FileSize(size);
cur->patch_hashes.push_back(HashString(*type, hash));
cur->patch_size = size;
break;
}
if (cur != available_patches.end())
@@ -1694,6 +1700,48 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/
}
}

for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type)
{
std::string tagname = *type;
tagname.append("-Download");
std::string const tmp = Tags.FindS(tagname.c_str());
if (tmp.empty() == true)
continue;

string hash, filename;
unsigned long long size;
std::stringstream ss(tmp);

// FIXME: all of pdiff supports only .gz compressed patches
while (ss >> hash >> size >> filename)
{
if (unlikely(hash.empty() == true || filename.empty() == true))
continue;
if (unlikely(APT::String::Endswith(filename, ".gz") == false))
continue;
filename.erase(filename.length() - 3);

// see if we have a record for this file already
std::vector<DiffInfo>::iterator cur = available_patches.begin();
for (; cur != available_patches.end(); ++cur)
{
if (cur->file != filename)
continue;
if (cur->download_hashes.empty())
cur->download_hashes.FileSize(size);
cur->download_hashes.push_back(HashString(*type, hash));
break;
}
if (cur != available_patches.end())
continue;
if (Debug == true)
std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": File " << filename
<< " wasn't in the list for the first parsed hash! (download)" << std::endl;
break;
}
}


bool foundStart = false;
for (std::vector<DiffInfo>::iterator cur = available_patches.begin();
cur != available_patches.end(); ++cur)
@@ -1729,7 +1777,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/
unsigned long long patchesSize = 0;
for (std::vector<DiffInfo>::const_iterator cur = available_patches.begin();
cur != available_patches.end(); ++cur)
patchesSize += cur->patch_size;
patchesSize += cur->patch_hashes.FileSize();
unsigned long long const sizeLimit = ServerSize * _config->FindI("Acquire::PDiffs::SizeLimit", 100);
if (sizeLimit > 0 && (sizeLimit/100) < patchesSize)
{


+ 4
- 7
apt-pkg/acquire-item.h View File

@@ -705,17 +705,14 @@ struct APT_HIDDEN DiffInfo { /*{{{*/
/** The filename of the diff. */
std::string file;

/** The hashes of the diff */
/** The hashes of the file after the diff is applied */
HashStringList result_hashes;

/** The hashes of the file after the diff is applied */
/** The hashes of the diff */
HashStringList patch_hashes;

/** The size of the file after the diff is applied */
unsigned long long result_size;

/** The size of the diff itself */
unsigned long long patch_size;
/** The hashes of the compressed diff */
HashStringList download_hashes;
};
/*}}}*/
/** \brief An item that is responsible for fetching client-merge patches {{{


+ 8
- 3
apt-pkg/contrib/hashes.cc View File

@@ -188,6 +188,13 @@ unsigned long long HashStringList::FileSize() const /*{{{*/
return strtoull(hv.c_str(), NULL, 10);
}
/*}}}*/
bool HashStringList::FileSize(unsigned long long const Size) /*{{{*/
{
std::string size;
strprintf(size, "%llu", Size);
return push_back(HashString("Checksum-FileSize", size));
}
/*}}}*/
bool HashStringList::supported(char const * const type) /*{{{*/
{
for (char const * const * t = HashString::SupportedHashes(); *t != NULL; ++t)
@@ -361,9 +368,7 @@ APT_IGNORE_DEPRECATED_PUSH
if ((d->CalcHashes & SHA512SUM) == SHA512SUM)
hashes.push_back(HashString("SHA512", SHA512.Result().Value()));
APT_IGNORE_DEPRECATED_POP
std::string SizeStr;
strprintf(SizeStr, "%llu", d->FileSize);
hashes.push_back(HashString("Checksum-FileSize", SizeStr));
hashes.FileSize(d->FileSize);
return hashes;
}
APT_IGNORE_DEPRECATED_PUSH


+ 7
- 0
apt-pkg/contrib/hashes.h View File

@@ -96,6 +96,13 @@ class HashStringList
*/
unsigned long long FileSize() const;

/** sets the filesize hash
*
* @param Size of the file
* @return @see #push_back
*/
bool FileSize(unsigned long long const Size);

/** check if the given hash type is supported
*
* @param type to check


+ 1
- 3
apt-pkg/indexrecords.cc View File

@@ -121,9 +121,7 @@ bool indexRecords::Load(const string Filename) /*{{{*/
indexRecords::checkSum *Sum = new indexRecords::checkSum;
Sum->MetaKeyFilename = Name;
Sum->Size = Size;
std::string SizeStr;
strprintf(SizeStr, "%llu", Size);
Sum->Hashes.push_back(HashString("Checksum-FileSize", SizeStr));
Sum->Hashes.FileSize(Size);
APT_IGNORE_DEPRECATED(Sum->Hash = HashString(HashString::SupportedHashes()[i],Hash);)
Entries[Name] = Sum;
}


+ 1
- 1
methods/rred.cc View File

@@ -627,7 +627,7 @@ class RredMethod : public pkgAcqMethod {
p.Close();
HashStringList const hsl = patch_hash.GetHashStringList();
if (hsl != I->ExpectedHashes)
return _error->Error("Patch %s doesn't have the expected hashsum", patch_name.c_str());
return _error->Error("Hash Sum mismatch for uncompressed patch %s", patch_name.c_str());
}

if (Debug == true)


+ 44
- 2
test/integration/test-pdiff-usage View File

@@ -42,6 +42,8 @@ wasmergeused() {

testrun() {
msgmsg "Testcase: setup the base with: $*"
local DOWNLOADHASH=true
if [ "$1" = 'nohash' ]; then DOWNLOADHASH=false; shift; fi
find aptarchive -name 'Packages*' -type f -delete
cp ${PKGFILE} aptarchive/Packages
compressfile 'aptarchive/Packages'
@@ -76,6 +78,15 @@ SHA256-History:
SHA256-Patches:
e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28
$(sha256sum $PATCHFILE | cut -d' ' -f 1) $(stat -c%s $PATCHFILE) $(basename $PATCHFILE)" > $PATCHINDEX
if $DOWNLOADHASH; then
echo "SHA1-Download:
2365ac0ac57cde3d43c63145e8251a3bd5410213 197 2010-08-18-2013.28.gz
$(sha1sum ${PATCHFILE}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE}.gz) $(basename ${PATCHFILE}.gz)
SHA256-Download:
d2a1b33187ed2d248eeae3b1223ea71791ea35f2138a713ed371332a6421f467 197 2010-08-18-2013.28.gz
$(sha256sum ${PATCHFILE}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE}.gz) $(basename ${PATCHFILE}.gz)" >> $PATCHINDEX
fi

generatereleasefiles '+1hour'
signreleasefiles
find aptarchive -name 'Packages*' -type f -delete
@@ -131,6 +142,17 @@ SHA256-Patches:
e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28
$(sha256sum $PATCHFILE | cut -d' ' -f 1) $(stat -c%s $PATCHFILE) $(basename $PATCHFILE)
$(sha256sum ${PATCHFILE2} | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE2}) $(basename ${PATCHFILE2})" > $PATCHINDEX
if $DOWNLOADHASH; then
echo "SHA1-Download:
2365ac0ac57cde3d43c63145e8251a3bd5410213 197 2010-08-18-2013.28.gz
$(sha1sum ${PATCHFILE}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE}.gz) $(basename ${PATCHFILE}.gz)
$(sha1sum ${PATCHFILE2}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE2}.gz) $(basename ${PATCHFILE2}.gz)
SHA256-Download:
d2a1b33187ed2d248eeae3b1223ea71791ea35f2138a713ed371332a6421f467 197 2010-08-18-2013.28.gz
$(sha256sum ${PATCHFILE}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE}.gz) $(basename ${PATCHFILE}.gz)
$(sha256sum ${PATCHFILE2}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE2}.gz) $(basename ${PATCHFILE2}.gz)" >> $PATCHINDEX
fi

generatereleasefiles '+2hour'
signreleasefiles
cp -a aptarchive/Packages Packages-future
@@ -150,6 +172,7 @@ SHA256-Patches:
mkdir -p aptarchive/Packages.diff
PATCHFILE="aptarchive/Packages.diff/$(date +%Y-%m-%d-%H%M.%S)"
diff -e ${PKGFILE} ${PKGFILE}-new > ${PATCHFILE} || true
cat $PATCHFILE | gzip > ${PATCHFILE}.gz
PATCHINDEX='aptarchive/Packages.diff/Index'
echo "SHA1-Current: $(sha1sum ${PKGFILE}-new | cut -d' ' -f 1) $(stat -c%s ${PKGFILE}-new)
SHA1-History:
@@ -165,14 +188,22 @@ SHA256-History:
SHA256-Patches:
e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28
$(sha256sum $PATCHFILE | cut -d' ' -f 1) $(stat -c%s $PATCHFILE) $(basename $PATCHFILE)" > $PATCHINDEX
if $DOWNLOADHASH; then
echo "SHA1-Download:
2365ac0ac57cde3d43c63145e8251a3bd5410213 197 2010-08-18-2013.28.gz
$(sha1sum ${PATCHFILE}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE}.gz) $(basename ${PATCHFILE}.gz)
SHA256-Download:
d2a1b33187ed2d248eeae3b1223ea71791ea35f2138a713ed371332a6421f467 197 2010-08-18-2013.28.gz
$(sha256sum ${PATCHFILE}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE}.gz) $(basename ${PATCHFILE}.gz)" >> $PATCHINDEX
fi
# needs to look like a valid command, otherwise the parser will fail before hashes are checked
echo '1d' >> $PATCHFILE
echo '1d' > $PATCHFILE
cat $PATCHFILE | gzip > ${PATCHFILE}.gz
generatereleasefiles '+1hour'
signreleasefiles
testsuccess aptget update "$@"
cp -f rootdir/tmp/testsuccess.output rootdir/tmp/aptgetupdate.output
testsuccess grep 'have the expected hashsum' rootdir/tmp/aptgetupdate.output
testsuccess grep 'Hash Sum mismatch' rootdir/tmp/aptgetupdate.output
testnopackage oldstuff
testsuccessequal "$(cat ${PKGFILE}-new)
" aptcache show apt newstuff
@@ -201,6 +232,14 @@ SHA256-History:
SHA256-Patches:
e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28
$(sha256sum $PATCHFILE | cut -d' ' -f 1) $(stat -c%s $PATCHFILE)000 $(basename $PATCHFILE)" > $PATCHINDEX
if $DOWNLOADHASH; then
echo "SHA1-Download:
2365ac0ac57cde3d43c63145e8251a3bd5410213 197 2010-08-18-2013.28.gz
$(sha1sum ${PATCHFILE}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE}.gz)000 $(basename ${PATCHFILE}.gz)
SHA256-Download:
d2a1b33187ed2d248eeae3b1223ea71791ea35f2138a713ed371332a6421f467 197 2010-08-18-2013.28.gz
$(sha256sum ${PATCHFILE}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE}.gz)000 $(basename ${PATCHFILE}.gz)" >> $PATCHINDEX
fi
generatereleasefiles '+1hour'
signreleasefiles
#find aptarchive -name 'Packages*' -type f -delete
@@ -215,6 +254,9 @@ echo 'Debug::pkgAcquire::Diffs "true";
Debug::Acquire::Transaction "true";
Debug::pkgAcquire::rred "true";' > rootdir/etc/apt/apt.conf.d/rreddebug.conf

testrun nohash -o Acquire::PDiffs::Merge=0 -o APT::Get::List-Cleanup=1
testrun nohash -o Acquire::PDiffs::Merge=1 -o APT::Get::List-Cleanup=1

testrun -o Acquire::PDiffs::Merge=0 -o APT::Get::List-Cleanup=1
testrun -o Acquire::PDiffs::Merge=1 -o APT::Get::List-Cleanup=1
testrun -o Acquire::PDiffs::Merge=0 -o APT::Get::List-Cleanup=0


Loading…
Cancel
Save