Browse Source

don't ask server if we have entire file in partial/

We have this situation in cases were parts of the transaction are
refused (e.g. in a hashsum mismatch) and rerun the update (e.g. in the
hope that we get a mirror which is synced this time).

Previously we would ask the server with an if-range and in the best case
recieve a 416 in response (less featureful server might end up giving us
the entire file again or we get the wrong file this time giving us a
hashsum mismatch…), which is a waste of time if we know already by
checking the hashsums that we got the complete and correct file.
tags/debian/1.3_exp1
David Kalnischkies 5 years ago
parent
commit
742f67eaed
4 changed files with 81 additions and 31 deletions
  1. +54
    -24
      methods/server.cc
  2. +12
    -4
      test/integration/framework
  3. +2
    -1
      test/integration/test-apt-update-transactions
  4. +13
    -2
      test/integration/test-pdiff-usage

+ 54
- 24
methods/server.cc View File

@@ -419,36 +419,66 @@ void ServerMethod::SigTerm(int)
depth. */
bool ServerMethod::Fetch(FetchItem *)
{
if (Server == 0)
if (Server == nullptr || QueueBack == nullptr)
return true;

// Queue the requests
int Depth = -1;
for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth;
I = I->Next, Depth++)
{
if (Depth >= 0)
{
// If pipelining is disabled, we only queue 1 request
if (Server->Pipeline == false)
break;
// if we have no hashes, do at most one such request
// as we can't fixup pipeling misbehaviors otherwise
else if (I->ExpectedHashes.usable() == false)
break;
}
// If pipelining is disabled, we only queue 1 request
auto const AllowedDepth = Server->Pipeline ? PipelineDepth : 0;
// how deep is our pipeline currently?
decltype(PipelineDepth) CurrentDepth = 0;
for (FetchItem const *I = Queue; I != QueueBack; I = I->Next)
++CurrentDepth;

do {
// Make sure we stick with the same server
if (Server->Comp(I->Uri) == false)
if (Server->Comp(QueueBack->Uri) == false)
break;

bool const UsableHashes = QueueBack->ExpectedHashes.usable();
// if we have no hashes, do at most one such request
// as we can't fixup pipeling misbehaviors otherwise
if (CurrentDepth != 0 && UsableHashes == false)
break;
if (QueueBack == I)

if (UsableHashes && FileExists(QueueBack->DestFile))
{
QueueBack = I->Next;
SendReq(I);
continue;
FileFd partial(QueueBack->DestFile, FileFd::ReadOnly);
Hashes wehave(QueueBack->ExpectedHashes);
if (QueueBack->ExpectedHashes.FileSize() == partial.FileSize())
{
if (wehave.AddFD(partial) &&
wehave.GetHashStringList() == QueueBack->ExpectedHashes)
{
FetchResult Res;
Res.Filename = QueueBack->DestFile;
Res.ResumePoint = QueueBack->ExpectedHashes.FileSize();
URIStart(Res);
// move item to the start of the queue as URIDone will
// always dequeued the first item in the queue
if (Queue != QueueBack)
{
FetchItem *Prev = Queue;
for (; Prev->Next != QueueBack; Prev = Prev->Next)
/* look for the previous queue item */;
Prev->Next = QueueBack->Next;
QueueBack->Next = Queue;
Queue = QueueBack;
QueueBack = Prev->Next;
}
Res.TakeHashes(wehave);
URIDone(Res);
continue;
}
else
RemoveFile("Fetch-Partial", QueueBack->DestFile);
}
}
}
auto const Tmp = QueueBack;
QueueBack = QueueBack->Next;
SendReq(Tmp);
++CurrentDepth;
} while (CurrentDepth <= AllowedDepth && QueueBack != nullptr);

return true;
}
/*}}}*/


+ 12
- 4
test/integration/framework View File

@@ -1505,6 +1505,14 @@ testmarkedmanual() {
msggroup
}

catfile() {
if [ "${1##*.}" = 'deb' ]; then
stat >&2 "$1" || true
file >&2 "$1" || true
else
cat >&2 "$1" || true
fi
}
msgfailoutput() {
msgreportheader 'msgfailoutput'
local MSG="$1"
@@ -1514,7 +1522,7 @@ msgfailoutput() {
echo >&2
while [ -n "$2" ]; do shift; done
echo "#### Complete file: $1 ####"
cat >&2 "$1" || true
catfile "$1"
echo '#### grep output ####'
elif [ "$1" = 'test' ]; then
echo >&2
@@ -1529,7 +1537,7 @@ msgfailoutput() {
ls >&2 "$2" || true
elif test -e "$2"; then
echo "#### Complete file: $2 ####"
cat >&2 "$2" || true
catfile "$2"
fi
fi
}
@@ -1543,12 +1551,12 @@ msgfailoutput() {
echo >&2
while [ -n "$2" ]; do
echo "#### Complete file: $2 ####"
cat >&2 "$2" || true
catfile "$2"
shift
done
echo '#### cmp output ####'
fi
cat >&2 "$OUTPUT"
catfile "$OUTPUT"
msgfail "$MSG"
}



+ 2
- 1
test/integration/test-apt-update-transactions View File

@@ -44,7 +44,8 @@ testrun() {
signreleasefiles

onehashbroken() {
testfailure aptget update
rm -rf rootdir/var/lib/apt/lists/partial
testfailure aptget update -o Debug::pkgAcquire::Worker=1 -o Debug::Acquire::http=1
# each file generates two messages with this string
testequal '2' grep --count 'Hash Sum mismatch' rootdir/tmp/testfailure.output
testfileequal "$1" "$(listcurrentlistsdirectory)"


+ 13
- 2
test/integration/test-pdiff-usage View File

@@ -30,7 +30,7 @@ echo 'hacked' > aptarchive/hacked-i386
compressfile aptarchive/hacked-i386

wasmergeused() {
testsuccess apt update "$@" -o Debug::pkgAcquire::Worker=1 -o Debug::Acquire::http=1 -o Debug::pkgAcquire=1
testsuccess apt update "$@"

msgtest 'No intermediate patch files' 'still exist'
local EDS="$(find rootdir/var/lib/apt/lists -name '*.ed' -o -name '*.ed.*')"
@@ -224,6 +224,14 @@ SHA256-Download:
testsuccessequal "$(cat "${PKGFILE}")
" aptcache show apt oldstuff

# we reuse the entire state of the previous test here
msgmsg "Testcase: good files from previous fails are picked up from partial: $*"
wasmergeused "$@"
testfailure grep '^GET /Packages.diff/Index HTTP/1.1' rootdir/tmp/testsuccess.output
testnopackage oldstuff
testsuccessequal "$(cat Packages-future)
" aptcache show apt newstuff futurestuff

# we reuse the archive state of the previous test here
msgmsg "Testcase: downloading a patch fails, but successful fallback: $*"
rm -rf rootdir/var/lib/apt/lists
@@ -294,7 +302,7 @@ SHA256-Download:
$(sha256sum "${PATCHFILE}.gz" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE}.gz")000 $(basename "${PATCHFILE}.gz")" > "$PATCHINDEX"
generatereleasefiles '+1hour'
signreleasefiles
testsuccess apt update -o Debug::pkgAcquire::Diffs=1 "$@"
testsuccess apt update "$@"
cp -f rootdir/tmp/testsuccess.output rootdir/tmp/aptupdate.output
testsuccess grep 'bytes (Limit is' rootdir/tmp/aptupdate.output
testnopackage oldstuff
@@ -303,6 +311,9 @@ SHA256-Download:
}
echo 'Debug::pkgAcquire::Diffs "true";
Debug::Acquire::Transaction "true";
Debug::pkgAcquire::Worker "true";
Debug::Acquire::http "true";
Debug::pkgAcquire "true";
Debug::pkgAcquire::rred "true";' > rootdir/etc/apt/apt.conf.d/rreddebug.conf

testcase() {


Loading…
Cancel
Save