Browse Source

implement fallback to alternative URIs for all items

For deb files we always supported falling back from one server to the
other if one failed to download the deb, but that was hardwired in the
handling of this specific item. Moving this alongside the retry
infrastructure we can implement it for all items and allow methods to
use this as well by providing additional URIs in a redirect.
tags/debian/1.6_alpha6
David Kalnischkies 3 years ago
parent
commit
355e1aceac
7 changed files with 230 additions and 160 deletions
  1. +156
    -152
      apt-pkg/acquire-item.cc
  2. +4
    -0
      apt-pkg/acquire-item.h
  3. +46
    -7
      apt-pkg/acquire-worker.cc
  4. +4
    -0
      apt-pkg/indexfile.cc
  5. +1
    -0
      apt-pkg/indexfile.h
  6. +1
    -1
      test/integration/framework
  7. +18
    -0
      test/integration/test-bug-869859-retry-downloads

+ 156
- 152
apt-pkg/acquire-item.cc View File

@@ -683,6 +683,13 @@ class APT_HIDDEN CleanupItem : public pkgAcqTransactionItem /*{{{*/
class pkgAcquire::Item::Private
{
public:
struct AlternateURI
{
std::string const URI;
std::unordered_map<std::string, std::string> changefields;
AlternateURI(std::string &&u, decltype(changefields) &&cf) : URI(u), changefields(cf) {}
};
std::list<AlternateURI> AlternativeURIs;
std::vector<std::string> PastRedirections;
std::unordered_map<std::string, std::string> CustomFields;
unsigned int Retries;
@@ -723,6 +730,32 @@ std::unordered_map<std::string, std::string> &pkgAcquire::Item::ModifyCustomFiel
return d->CustomFields;
}
/*}}}*/
bool pkgAcquire::Item::PopAlternativeURI(std::string &NewURI) /*{{{*/
{
if (d->AlternativeURIs.empty())
return false;
auto const AltUri = d->AlternativeURIs.front();
d->AlternativeURIs.pop_front();
NewURI = AltUri.URI;
auto &CustomFields = ModifyCustomFields();
for (auto const &f : AltUri.changefields)
{
if (f.second.empty())
CustomFields.erase(f.first);
else
CustomFields[f.first] = f.second;
}
return true;
}
/*}}}*/
void pkgAcquire::Item::PushAlternativeURI(std::string &&NewURI, std::unordered_map<std::string, std::string> &&fields, bool const at_the_back) /*{{{*/
{
if (at_the_back)
d->AlternativeURIs.emplace_back(std::move(NewURI), std::move(fields));
else
d->AlternativeURIs.emplace_front(std::move(NewURI), std::move(fields));
}
/*}}}*/
unsigned int &pkgAcquire::Item::ModifyRetries() /*{{{*/
{
return d->Retries;
@@ -1070,6 +1103,7 @@ pkgAcqTransactionItem::pkgAcqTransactionItem(pkgAcquire * const Owner, /*{{{*/
{"Target-Release", Target.Option(IndexTarget::RELEASE)},
{"Target-Architecture", Target.Option(IndexTarget::ARCHITECTURE)},
{"Target-Language", Target.Option(IndexTarget::LANGUAGE)},
{"Target-Type", "index"},
};
}
/*}}}*/
@@ -3244,12 +3278,12 @@ pkgAcqIndex::~pkgAcqIndex() {}
// ---------------------------------------------------------------------
/* This just sets up the initial fetch environment and queues the first
possibilitiy */
pkgAcqArchive::pkgAcqArchive(pkgAcquire * const Owner,pkgSourceList * const Sources,
pkgRecords * const Recs,pkgCache::VerIterator const &Version,
string &StoreFilename) :
Item(Owner), d(NULL), LocalSource(false), Version(Version), Sources(Sources), Recs(Recs),
StoreFilename(StoreFilename), Vf(Version.FileList()),
Trusted(false)
APT_IGNORE_DEPRECATED_PUSH
pkgAcqArchive::pkgAcqArchive(pkgAcquire *const Owner, pkgSourceList *const Sources,
pkgRecords *const Recs, pkgCache::VerIterator const &Version,
string &StoreFilename) : Item(Owner), d(NULL), LocalSource(false), Version(Version), Sources(Sources), Recs(Recs),
StoreFilename(StoreFilename), Vf(),
Trusted(false)
{
if (Version.Arch() == 0)
{
@@ -3259,32 +3293,6 @@ pkgAcqArchive::pkgAcqArchive(pkgAcquire * const Owner,pkgSourceList * const Sour
Version.ParentPkg().FullName().c_str());
return;
}
/* We need to find a filename to determine the extension. We make the
assumption here that all the available sources for this version share
the same extension.. */
// Skip not source sources, they do not have file fields.
for (; Vf.end() == false; ++Vf)
{
if (Vf.File().Flagged(pkgCache::Flag::NotSource))
continue;
break;
}
// Does not really matter here.. we are going to fail out below
if (Vf.end() != true)
{
// If this fails to get a file name we will bomb out below.
pkgRecords::Parser &Parse = Recs->Lookup(Vf);
if (_error->PendingError() == true)
return;
// Generate the final file name as: package_version_arch.foo
StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
QuoteString(Version.VerStr(),"_:") + '_' +
QuoteString(Version.Arch(),"_:.") +
"." + flExtension(Parse.FileName());
}

// check if we have one trusted source for the package. if so, switch
// to "TrustedOnly" mode - but only if not in AllowUnauthenticated mode
@@ -3317,58 +3325,60 @@ pkgAcqArchive::pkgAcqArchive(pkgAcquire * const Owner,pkgSourceList * const Sour
if (allowUnauth == true && seenUntrusted == true)
Trusted = false;

// Select a source
if (QueueNext() == false && _error->PendingError() == false)
_error->Error(_("Can't find a source to download version '%s' of '%s'"),
Version.VerStr(), Version.ParentPkg().FullName(false).c_str());
}
/*}}}*/
// AcqArchive::QueueNext - Queue the next file source /*{{{*/
// ---------------------------------------------------------------------
/* This queues the next available file version for download. It checks if
the archive is already available in the cache and stashs the MD5 for
checking later. */
bool pkgAcqArchive::QueueNext()
{
for (; Vf.end() == false; ++Vf)
StoreFilename.clear();
std::set<string> targetComponents, targetCodenames, targetSuites;
for (auto Vf = Version.FileList(); Vf.end() == false; ++Vf)
{
pkgCache::PkgFileIterator const PkgF = Vf.File();
// Ignore not source sources
auto const PkgF = Vf.File();
if (unlikely(PkgF.end()))
continue;
if (PkgF.Flagged(pkgCache::Flag::NotSource))
continue;

// Try to cross match against the source list
pkgIndexFile *Index;
if (Sources->FindIndex(PkgF, Index) == false)
continue;
LocalSource = PkgF.Flagged(pkgCache::Flag::LocalSource);

// only try to get a trusted package from another source if that source
// is also trusted
if(Trusted && !Index->IsTrusted())
continue;
if (Trusted && Index->IsTrusted() == false)
continue;

// Grab the text package record
pkgRecords::Parser &Parse = Recs->Lookup(Vf);
if (_error->PendingError() == true)
return false;

string PkgFile = Parse.FileName();
ExpectedHashes = Parse.Hashes();

if (PkgFile.empty() == true)
return _error->Error(_("The package index files are corrupted. No Filename: "
"field for package %s."),
Version.ParentPkg().Name());
// collect the hashes from the indexes
auto hsl = Parse.Hashes();
if (ExpectedHashes.empty())
ExpectedHashes = hsl;
else
{
// bad things will likely happen, but the user might be "lucky" still
// if the sources provide the same hashtypes (so that they aren't mixed)
for (auto const &hs : hsl)
if (ExpectedHashes.push_back(hs) == false)
{
_error->Warning("Sources disagree on hashes for supposely identical version '%s' of '%s'.",
Version.VerStr(), Version.ParentPkg().FullName(false).c_str());
break;
}
}
// only allow local volatile sources to have no hashes
if (PkgF.Flagged(pkgCache::Flag::LocalSource))
LocalSource = true;
else if (hsl.empty())
continue;

Desc.URI = Index->ArchiveURI(PkgFile);
Desc.Description = Index->ArchiveInfo(Version);
Desc.Owner = this;
Desc.ShortDesc = Version.ParentPkg().FullName(true);
std::string poolfilename = Parse.FileName();
if (poolfilename.empty())
continue;

auto fields = ModifyCustomFields();
if (PkgF->Architecture != 0)
fields.emplace("Target-Architecture", PkgF.Architecture());
std::remove_reference<decltype(ModifyCustomFields())>::type fields;
{
auto const debIndex = dynamic_cast<pkgDebianIndexTargetFile const *const>(Index);
if (debIndex != nullptr)
{
auto const IT = debIndex->GetIndexTarget();
fields.emplace("Target-Repo-URI", IT.Option(IndexTarget::REPO_URI));
fields.emplace("Target-Release", IT.Option(IndexTarget::RELEASE));
fields.emplace("Target-Site", IT.Option(IndexTarget::SITE));
}
}
fields.emplace("Target-Base-URI", Index->ArchiveURI(""));
if (PkgF->Component != 0)
fields.emplace("Target-Component", PkgF.Component());
auto const RelF = PkgF.ReleaseFile();
@@ -3379,78 +3389,91 @@ bool pkgAcqArchive::QueueNext()
if (RelF->Archive != 0)
fields.emplace("Target-Suite", RelF.Archive());
}
fields.emplace("Target-Architecture", Version.Arch());
fields.emplace("Target-Type", flExtension(poolfilename));

// See if we already have the file. (Legacy filenames)
FileSize = Version->Size;
string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
struct stat Buf;
if (stat(FinalFile.c_str(),&Buf) == 0)
if (StoreFilename.empty())
{
// Make sure the size matches
if ((unsigned long long)Buf.st_size == Version->Size)
{
Complete = true;
Local = true;
Status = StatDone;
StoreFilename = DestFile = FinalFile;
return true;
}
/* Hmm, we have a file and its size does not match, this means it is
an old style mismatched arch */
RemoveFile("pkgAcqArchive::QueueNext", FinalFile);
}

// Check it again using the new style output filenames
FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
if (stat(FinalFile.c_str(),&Buf) == 0)
{
// Make sure the size matches
if ((unsigned long long)Buf.st_size == Version->Size)
{
Complete = true;
Local = true;
Status = StatDone;
StoreFilename = DestFile = FinalFile;
return true;
}
/* Hmm, we have a file and its size does not match, this shouldn't
happen.. */
RemoveFile("pkgAcqArchive::QueueNext", FinalFile);
}

DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
// Check the destination file
if (stat(DestFile.c_str(),&Buf) == 0)
{
// Hmm, the partial file is too big, erase it
if ((unsigned long long)Buf.st_size > Version->Size)
RemoveFile("pkgAcqArchive::QueueNext", DestFile);
else
PartialSize = Buf.st_size;
/* We pick a filename based on the information we have for the version,
but we don't know what extension such a file should have, so we look
at the filenames used online and assume that they are the same for
all repositories containing this file */
StoreFilename = QuoteString(Version.ParentPkg().Name(), "_:") + '_' +
QuoteString(Version.VerStr(), "_:") + '_' +
QuoteString(Version.Arch(), "_:.") +
"." + flExtension(poolfilename);

Desc.URI = Index->ArchiveURI(poolfilename);
Desc.Description = Index->ArchiveInfo(Version);
Desc.Owner = this;
Desc.ShortDesc = Version.ParentPkg().FullName(true);
auto &customfields = ModifyCustomFields();
for (auto const &f : fields)
customfields[f.first] = f.second;
FileSize = Version->Size;
}
else
PushAlternativeURI(Index->ArchiveURI(poolfilename), std::move(fields), true);
}
if (StoreFilename.empty())
{
_error->Error(_("Can't find a source to download version '%s' of '%s'"),
Version.VerStr(), Version.ParentPkg().FullName(false).c_str());
return;
}

// Disables download of archives - useful if no real installation follows,
// e.g. if we are just interested in proposed installation order
if (_config->FindB("Debug::pkgAcqArchive::NoQueue", false) == true)
// Check if we already downloaded the file
struct stat Buf;
auto FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
if (stat(FinalFile.c_str(), &Buf) == 0)
{
// Make sure the size matches
if ((unsigned long long)Buf.st_size == Version->Size)
{
Complete = true;
Local = true;
Status = StatDone;
StoreFilename = DestFile = FinalFile;
return true;
return;
}

// Create the item
Local = false;
++Vf;
QueueURI(Desc);
return true;
/* Hmm, we have a file and its size does not match, this shouldn't
happen.. */
RemoveFile("pkgAcqArchive::QueueNext", FinalFile);
}

// Check the destination file
DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
if (stat(DestFile.c_str(), &Buf) == 0)
{
// Hmm, the partial file is too big, erase it
if ((unsigned long long)Buf.st_size > Version->Size)
RemoveFile("pkgAcqArchive::QueueNext", DestFile);
else
PartialSize = Buf.st_size;
}

// Disables download of archives - useful if no real installation follows,
// e.g. if we are just interested in proposed installation order
if (_config->FindB("Debug::pkgAcqArchive::NoQueue", false) == true)
{
Complete = true;
Local = true;
Status = StatDone;
StoreFilename = DestFile = FinalFile;
return;
}

// Create the item
Local = false;
QueueURI(Desc);
}
APT_IGNORE_DEPRECATED_POP
/*}}}*/
bool pkgAcqArchive::QueueNext() /*{{{*/
{
return false;
}
}
/*}}}*/
// AcqArchive::Done - Finished fetching /*{{{*/
// ---------------------------------------------------------------------
@@ -3483,25 +3506,6 @@ void pkgAcqArchive::Done(string const &Message, HashStringList const &Hashes,
void pkgAcqArchive::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)
{
Item::Failed(Message,Cnf);

/* We don't really want to retry on failed media swaps, this prevents
that. An interesting observation is that permanent failures are not
recorded. */
if (Cnf->Removable == true &&
StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
{
// Vf = Version.FileList();
while (Vf.end() == false) ++Vf;
StoreFilename = string();
return;
}

Status = StatIdle;
if (QueueNext() == false)
{
StoreFilename = string();
Status = StatError;
}
}
/*}}}*/
APT_PURE bool pkgAcqArchive::IsTrusted() const /*{{{*/


+ 4
- 0
apt-pkg/acquire-item.h View File

@@ -244,6 +244,9 @@ class pkgAcquire::Item : public WeakPointable /*{{{*/
APT_HIDDEN unsigned int &ModifyRetries();
// this is more a hack than a proper external interface, hence hidden
APT_HIDDEN std::unordered_map<std::string, std::string> &ModifyCustomFields();
// this isn't the super nicest interface either…
APT_HIDDEN bool PopAlternativeURI(std::string &NewURI);
APT_HIDDEN void PushAlternativeURI(std::string &&NewURI, std::unordered_map<std::string, std::string> &&fields, bool const at_the_back);

/** \brief A "descriptive" URI-like string.
*
@@ -993,6 +996,7 @@ class pkgAcqArchive : public pkgAcquire::Item
std::string &StoreFilename;

/** \brief The next file for this version to try to download. */
APT_DEPRECATED_MSG("Unused member")
pkgCache::VerFileIterator Vf;

/** \brief How many (more) times to try to find a new source from


+ 46
- 7
apt-pkg/acquire-worker.cc View File

@@ -211,6 +211,39 @@ static bool isDoomedItem(pkgAcquire::Item const * const Itm)
return false;
return TransItm->TransactionManager->State != pkgAcqTransactionItem::TransactionStarted;
}
static HashStringList GetHashesFromMessage(std::string const &Prefix, std::string const &Message)
{
HashStringList hsl;
for (char const *const *type = HashString::SupportedHashes(); *type != NULL; ++type)
{
std::string const tagname = Prefix + *type + "-Hash";
std::string const hashsum = LookupTag(Message, tagname.c_str());
if (hashsum.empty() == false)
hsl.push_back(HashString(*type, hashsum));
}
return hsl;
}
static void APT_NONNULL(3) ChangeSiteIsMirrorChange(std::string const &NewURI, pkgAcquire::ItemDesc &desc, pkgAcquire::Item *const Owner)
{
if (URI::SiteOnly(NewURI) == URI::SiteOnly(desc.URI))
return;

auto const firstSpace = desc.Description.find(" ");
if (firstSpace != std::string::npos)
{
std::string const OldSite = desc.Description.substr(0, firstSpace);
if (likely(APT::String::Startswith(desc.URI, OldSite)))
{
std::string const OldExtra = desc.URI.substr(OldSite.length() + 1);
if (likely(APT::String::Endswith(NewURI, OldExtra)))
{
std::string const NewSite = NewURI.substr(0, NewURI.length() - OldExtra.length());
Owner->UsedMirror = URI::ArchiveOnly(NewSite);
desc.Description.replace(0, firstSpace, Owner->UsedMirror);
}
}
}
}
bool pkgAcquire::Worker::RunMessages()
{
while (MessageQueue.empty() == false)
@@ -377,13 +410,7 @@ bool pkgAcquire::Worker::RunMessages()
std::string const givenfilename = LookupTag(Message, "Filename");
std::string const filename = givenfilename.empty() ? Itm->Owner->DestFile : givenfilename;
// see if we got hashes to verify
for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type)
{
std::string const tagname = std::string(*type) + "-Hash";
std::string const hashsum = LookupTag(Message, tagname.c_str());
if (hashsum.empty() == false)
ReceivedHashes.push_back(HashString(*type, hashsum));
}
ReceivedHashes = GetHashesFromMessage("", Message);
// not all methods always sent Hashes our way
if (ReceivedHashes.usable() == false)
{
@@ -525,6 +552,7 @@ bool pkgAcquire::Worker::RunMessages()

for (auto const Owner: ItmOwners)
{
std::string NewURI;
if (errTransient == true && Config->LocalOnly == false && Owner->ModifyRetries() != 0)
{
--Owner->ModifyRetries();
@@ -535,6 +563,17 @@ bool pkgAcquire::Worker::RunMessages()
if (isDoomedItem(Owner) == false)
OwnerQ->Owner->Enqueue(SavedDesc);
}
else if (Owner->PopAlternativeURI(NewURI))
{
Owner->FailMessage(Message);
auto &desc = Owner->GetItemDesc();
if (Log != nullptr)
Log->Fail(desc);
ChangeSiteIsMirrorChange(NewURI, desc, Owner);
desc.URI = NewURI;
if (isDoomedItem(Owner) == false)
OwnerQ->Owner->Enqueue(desc);
}
else
{
if (errAuthErr && Owner->GetExpectedHashes().empty() == false)


+ 4
- 0
apt-pkg/indexfile.cc View File

@@ -273,6 +273,10 @@ std::string pkgDebianIndexTargetFile::GetProgressDescription() const
{
return Target.Description;
}
IndexTarget pkgDebianIndexTargetFile::GetIndexTarget() const
{
return Target;
}

pkgDebianIndexRealFile::pkgDebianIndexRealFile(std::string const &pFile, bool const Trusted) :/*{{{*/
pkgDebianIndexFile(Trusted), d(NULL)


+ 1
- 0
apt-pkg/indexfile.h View File

@@ -199,6 +199,7 @@ public:
virtual std::string Describe(bool const Short = false) const APT_OVERRIDE;
virtual bool Exists() const APT_OVERRIDE;
virtual unsigned long Size() const APT_OVERRIDE;
IndexTarget GetIndexTarget() const APT_HIDDEN;

pkgDebianIndexTargetFile(IndexTarget const &Target, bool const Trusted);
virtual ~pkgDebianIndexTargetFile();


+ 1
- 1
test/integration/framework View File

@@ -1298,7 +1298,7 @@ webserverconfig() {
rewritesourceslist() {
local APTARCHIVE="file://$(readlink -f "${TMPWORKINGDIRECTORY}/aptarchive" | sed 's# #%20#g')"
local APTARCHIVE2="copy://$(readlink -f "${TMPWORKINGDIRECTORY}/aptarchive" | sed 's# #%20#g')"
for LIST in $(find rootdir/etc/apt/sources.list.d/ -name 'apt-test-*.list'); do
for LIST in $(find "${TMPWORKINGDIRECTORY}/rootdir/etc/apt/sources.list.d/" -name 'apt-test-*.list'); do
sed -i $LIST -e "s#$APTARCHIVE#${1}#" -e "s#$APTARCHIVE2#${1}#" \
-e "s#http://[^@]*@\?localhost:${APTHTTPPORT}/\?#${1}#" \
-e "s#https://[^@]*@\?localhost:${APTHTTPSPORT}/\?#${1}#"


+ 18
- 0
test/integration/test-bug-869859-retry-downloads View File

@@ -35,3 +35,21 @@ webserverconfig 'aptwebserver::failrequest' '404'
webserverconfig 'aptwebserver::failrequest::pool/testpkg_1_all.deb' '2'
testfailure apt download testpkg -o acquire::retries=3
testfailure test -f testpkg_1_all.deb

cat ../rootdir/etc/apt/sources.list.d/apt-test-*.list > ../rootdir/etc/apt/sources.list.d/00http-source.list
changetohttpswebserver

msgmsg 'Check download from alternative sources if first failed'
webserverconfig 'aptwebserver::failrequest::pool/testpkg_1_all.deb' '0'
testsuccess apt update
testsuccess apt download testpkg -o acquire::retries=0
testsuccess test -f testpkg_1_all.deb
rm -f testpkg_1_all.deb

# we make the first source fail by disabling http support
webserverconfig 'aptwebserver::support::http' 'false'
testsuccess apt download testpkg -o acquire::retries=0
cp ../rootdir/tmp/testsuccess.output alt.output
testsuccess grep '^ 400 Bad Request' alt.output
testsuccess test -f testpkg_1_all.deb
rm -f testpkg_1_all.deb

Loading…
Cancel
Save