Browse Source

reimplement and simplify mirror:// method

Embedding an entire acquire stack and HTTP logic in the mirror method
made it rather heavy weight and fragile. This reimplement goes the other
way by doing only the bare minimum in the method itself and instead
redirect the actual download of files to their proper methods.

The reimplementation drops the (in the real world) unused query-string
feature as it isn't really implementable in the new architecture.
tags/debian/1.6_alpha6
David Kalnischkies 3 years ago
parent
commit
57fa854e4c
8 changed files with 258 additions and 498 deletions
  1. +6
    -19
      apt-pkg/acquire-worker.cc
  2. +3
    -5
      methods/CMakeLists.txt
  3. +10
    -0
      methods/aptmethod.h
  4. +13
    -1
      methods/http.cc
  5. +0
    -17
      methods/http_main.cc
  6. +225
    -398
      methods/mirror.cc
  7. +0
    -57
      methods/mirror.h
  8. +1
    -1
      po/CMakeLists.txt

+ 6
- 19
apt-pkg/acquire-worker.cc View File

@@ -308,6 +308,7 @@ bool pkgAcquire::Worker::RunMessages()

std::string const NewURI = LookupTag(Message,"New-URI",URI.c_str());
Itm->URI = NewURI;
auto const AltUris = VectorizeString(LookupTag(Message, "Alternate-URIs"), '\n');

ItemDone();

@@ -335,28 +336,14 @@ bool pkgAcquire::Worker::RunMessages()
if (Log != nullptr)
Log->Done(desc);

// if we change site, treat it as a mirror change
if (URI::SiteOnly(NewURI) != URI::SiteOnly(desc.URI))
{
auto const firstSpace = desc.Description.find(" ");
if (firstSpace != std::string::npos)
{
std::string const OldSite = desc.Description.substr(0, firstSpace);
if (likely(APT::String::Startswith(desc.URI, OldSite)))
{
std::string const OldExtra = desc.URI.substr(OldSite.length() + 1);
if (likely(APT::String::Endswith(NewURI, OldExtra)))
{
std::string const NewSite = NewURI.substr(0, NewURI.length() - OldExtra.length());
Owner->UsedMirror = URI::ArchiveOnly(NewSite);
desc.Description.replace(0, firstSpace, Owner->UsedMirror);
}
}
}
}
ChangeSiteIsMirrorChange(NewURI, desc, Owner);
desc.URI = NewURI;
if (isDoomedItem(Owner) == false)
{
for (auto alt = AltUris.crbegin(); alt != AltUris.crend(); ++alt)
Owner->PushAlternativeURI(std::string(*alt), {}, false);
OwnerQ->Owner->Enqueue(desc);
}
}
break;
}


+ 3
- 5
methods/CMakeLists.txt View File

@@ -2,7 +2,6 @@
include_directories($<$<BOOL:${SECCOMP_FOUND}>:${SECCOMP_INCLUDE_DIR}>)
link_libraries(apt-pkg $<$<BOOL:${SECCOMP_FOUND}>:${SECCOMP_LIBRARIES}>)

add_library(httplib OBJECT http.cc basehttp.cc)
add_library(connectlib OBJECT connect.cc rfc2553emu.cc)

add_executable(file file.cc)
@@ -10,8 +9,8 @@ add_executable(copy copy.cc)
add_executable(store store.cc)
add_executable(gpgv gpgv.cc)
add_executable(cdrom cdrom.cc)
add_executable(http http_main.cc $<TARGET_OBJECTS:httplib> $<TARGET_OBJECTS:connectlib>)
add_executable(mirror mirror.cc $<TARGET_OBJECTS:httplib> $<TARGET_OBJECTS:connectlib>)
add_executable(http http.cc basehttp.cc $<TARGET_OBJECTS:connectlib>)
add_executable(mirror mirror.cc)
add_executable(ftp ftp.cc $<TARGET_OBJECTS:connectlib>)
add_executable(rred rred.cc)
add_executable(rsh rsh.cc)
@@ -21,14 +20,13 @@ target_include_directories(connectlib PRIVATE ${GNUTLS_INCLUDE_DIR})

# Additional libraries to link against for networked stuff
target_link_libraries(http ${GNUTLS_LIBRARIES})
target_link_libraries(mirror ${RESOLV_LIBRARIES} ${GNUTLS_LIBRARIES})
target_link_libraries(ftp ${GNUTLS_LIBRARIES})

# Install the library
install(TARGETS file copy store gpgv cdrom http ftp rred rsh mirror
RUNTIME DESTINATION ${CMAKE_INSTALL_LIBEXECDIR}/apt/methods)

add_slaves(${CMAKE_INSTALL_LIBEXECDIR}/apt/methods store)
add_slaves(${CMAKE_INSTALL_LIBEXECDIR}/apt/methods mirror mirror+http mirror+https mirror+file)
add_slaves(${CMAKE_INSTALL_LIBEXECDIR}/apt/methods rsh ssh)




+ 10
- 0
methods/aptmethod.h View File

@@ -448,6 +448,16 @@ protected:
return true;
}

// This is a copy of #pkgAcqMethod::Dequeue which is private & hidden
void Dequeue()
{
FetchItem const *const Tmp = Queue;
Queue = Queue->Next;
if (Tmp == QueueBack)
QueueBack = Queue;
delete Tmp;
}

aptMethod(std::string &&Binary, char const *const Ver, unsigned long const Flags) APT_NONNULL(3)
: pkgAcqMethod(Ver, Flags), Binary(Binary), SeccompFlags(0), methodNames({Binary})
{


+ 13
- 1
methods/http.cc View File

@@ -31,6 +31,7 @@
#include <sstream>
#include <arpa/inet.h>
#include <errno.h>
#include <signal.h>
#include <stddef.h>
#include <stdio.h>
#include <stdlib.h>
@@ -1034,7 +1035,7 @@ BaseHttpMethod::DealWithHeadersResult HttpMethod::DealWithHeaders(FetchResult &R
return FILE_IS_OPEN;
}
/*}}}*/
HttpMethod::HttpMethod(std::string &&pProg) : BaseHttpMethod(pProg.c_str(), "1.2", Pipeline | SendConfig)/*{{{*/
HttpMethod::HttpMethod(std::string &&pProg) : BaseHttpMethod(std::move(pProg), "1.2", Pipeline | SendConfig) /*{{{*/
{
SeccompFlags = aptMethod::BASE | aptMethod::NETWORK;

@@ -1051,3 +1052,14 @@ HttpMethod::HttpMethod(std::string &&pProg) : BaseHttpMethod(pProg.c_str(), "1.2
}
}
/*}}}*/

int main(int, const char *argv[])
{
// ignore SIGPIPE, this can happen on write() if the socket
// closes the connection (this is dealt with via ServerDie())
signal(SIGPIPE, SIG_IGN);
std::string Binary = flNotDir(argv[0]);
if (Binary.find('+') == std::string::npos && Binary != "https" && Binary != "http")
Binary.append("+http");
return HttpMethod(std::move(Binary)).Loop();
}

+ 0
- 17
methods/http_main.cc View File

@@ -1,17 +0,0 @@
#include <config.h>
#include <apt-pkg/error.h>
#include <apt-pkg/fileutl.h>
#include <signal.h>

#include "http.h"

int main(int, const char *argv[])
{
// ignore SIGPIPE, this can happen on write() if the socket
// closes the connection (this is dealt with via ServerDie())
signal(SIGPIPE, SIG_IGN);
std::string Binary = flNotDir(argv[0]);
if (Binary.find('+') == std::string::npos && Binary != "https" && Binary != "http")
Binary.append("+http");
return HttpMethod(std::move(Binary)).Loop();
}

+ 225
- 398
methods/mirror.cc View File

@@ -1,18 +1,17 @@
// -*- mode: cpp; mode: fold -*-
// Description /*{{{*/
// $Id: mirror.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $
/* ######################################################################

Mirror Acquire Method - This is the Mirror acquire method for APT.
Mirror URI – This method helps avoiding hardcoding of mirrors in the
sources.lists by looking up a list of mirrors first to which the
following requests are redirected.

##################################################################### */
/*}}}*/
// Include Files /*{{{*/
#include <config.h>

#include <apt-pkg/acquire-item.h>
#include <apt-pkg/acquire.h>
#include <apt-pkg/aptconfiguration.h>
#include "aptmethod.h"
#include <apt-pkg/configuration.h>
#include <apt-pkg/error.h>
#include <apt-pkg/fileutl.h>
@@ -20,447 +19,275 @@
#include <apt-pkg/sourcelist.h>
#include <apt-pkg/strutl.h>

#include <algorithm>
#include <fstream>
#include <iostream>
#include <functional>
#include <random>
#include <string>
#include <unordered_map>

#include <dirent.h>
#include <fcntl.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/utsname.h>
#include <unistd.h>

using namespace std;

#include <sstream>

#include "http.h"
#include "mirror.h"
#include <apti18n.h>
/*}}}*/

/* Done:
* - works with http (only!)
* - always picks the first mirror from the list
* - call out to problem reporting script
* - supports "deb mirror://host/path/to/mirror-list/// dist component"
* - uses pkgAcqMethod::FailReason() to have a string representation
* of the failure that is also send to LP
*
* TODO:
* - deal with running as non-root because we can't write to the lists
dir then -> use the cached mirror file
* - better method to download than having a pkgAcquire interface here
* and better error handling there!
* - support more than http
* - testing :)
*/

MirrorMethod::MirrorMethod()
: HttpMethod("mirror"), DownloadedMirrorFile(false), Debug(false)
static void sortByLength(std::vector<std::string> &vec) /*{{{*/
{
}

// HttpMethod::Configuration - Handle a configuration message /*{{{*/
// ---------------------------------------------------------------------
/* We stash the desired pipeline depth */
bool MirrorMethod::Configuration(string Message)
{
if (HttpMethod::Configuration(Message) == false)
return false;
Debug = DebugEnabled();
return true;
// this ensures having mirror://foo/ and mirror://foo/bar/ works as expected
// by checking for the longest matches first
std::sort(vec.begin(), vec.end(), [](std::string const &a, std::string const &b) {
return a.length() > b.length();
});
}
/*}}}*/

// clean the mirrors dir based on ttl information
bool MirrorMethod::Clean(string Dir)
class MirrorMethod : public aptMethod /*{{{*/
{
vector<metaIndex *>::const_iterator I;

if(Debug)
clog << "MirrorMethod::Clean(): " << Dir << endl;

if(Dir == "/")
return _error->Error("will not clean: '/'");

// read sources.list
pkgSourceList list;
list.ReadMainList();

int const dirfd = open(Dir.c_str(), O_RDONLY | O_DIRECTORY | O_CLOEXEC);
if (dirfd == -1)
return _error->Errno("open",_("Unable to read %s"), Dir.c_str());
DIR * const D = fdopendir(dirfd);
if (D == nullptr)
return _error->Errno("fdopendir",_("Unable to read %s"),Dir.c_str());

for (struct dirent *Dir = readdir(D); Dir != 0; Dir = readdir(D))
std::vector<std::string> sourceslist;
enum MirrorFileState
{
// Skip some files..
if (strcmp(Dir->d_name,"lock") == 0 ||
strcmp(Dir->d_name,"partial") == 0 ||
strcmp(Dir->d_name,"lost+found") == 0 ||
strcmp(Dir->d_name,".") == 0 ||
strcmp(Dir->d_name,"..") == 0)
continue;

// see if we have that uri
for(I=list.begin(); I != list.end(); ++I)
REQUESTED,
FAILED,
AVAILABLE
};
struct MirrorInfo
{
MirrorFileState state;
std::string baseuri;
std::vector<std::string> list;
};
std::unordered_map<std::string, MirrorInfo> mirrorfilestate;
unsigned int seedvalue;

virtual bool URIAcquire(std::string const &Message, FetchItem *Itm) APT_OVERRIDE;

void RedirectItem(MirrorInfo const &info, FetchItem *const Itm);
bool MirrorListFileRecieved(MirrorInfo &info, FetchItem *const Itm);
std::string GetMirrorFileURI(std::string const &Message, FetchItem *const Itm);
void DealWithPendingItems(std::vector<std::string> const &baseuris, MirrorInfo const &info, FetchItem *const Itm, std::function<void()> handler);

public:
MirrorMethod(std::string &&pProg) : aptMethod(std::move(pProg), "2.0", SingleInstance | Pipeline | SendConfig)
{
SeccompFlags = aptMethod::BASE | aptMethod::DIRECTORY;

// we want the file to be random for each different machine, but also
// "stable" on the same machine to avoid issues like picking different
// mirrors in different states for indexes and deb downloads
struct utsname buf;
seedvalue = 1;
if (uname(&buf) == 0)
{
string uri = (*I)->GetURI();
if(uri.compare(0, strlen("mirror://"), "mirror://") != 0)
continue;
string BaseUri = uri.substr(0,uri.size()-1);
if (URItoFileName(BaseUri) == Dir->d_name)
break;
for (size_t i = 0; buf.nodename[i] != '\0'; ++i)
seedvalue = seedvalue * 31 + buf.nodename[i];
}
// nothing found, nuke it
if (I == list.end())
RemoveFileAt("mirror", dirfd, Dir->d_name);
}
closedir(D);
return true;
}


bool MirrorMethod::DownloadMirrorFile(string /*mirror_uri_str*/)
};
/*}}}*/
void MirrorMethod::RedirectItem(MirrorInfo const &info, FetchItem *const Itm) /*{{{*/
{
// not that great to use pkgAcquire here, but we do not have
// any other way right now
string fetch = BaseUri;
fetch.replace(0,strlen("mirror://"),"http://");

#if 0 // no need for this, the getArchitectures() will also include the main
// arch
// append main architecture
fetch += "?arch=" + _config->Find("Apt::Architecture");
#endif

// append all architectures
std::vector<std::string> vec = APT::Configuration::getArchitectures();
for (std::vector<std::string>::const_iterator I = vec.begin();
I != vec.end(); ++I)
if (I == vec.begin())
fetch += "?arch=" + (*I);
std::string const path = Itm->Uri.substr(info.baseuri.length());
std::string altMirrors;
std::unordered_map<std::string, std::string> fields;
fields.emplace("URI", Queue->Uri);
for (auto curMirror = info.list.cbegin(); curMirror != info.list.cend(); ++curMirror)
{
std::string mirror = *curMirror;
if (APT::String::Endswith(mirror, "/") == false)
mirror.append("/");
mirror.append(path);
if (curMirror == info.list.cbegin())
fields.emplace("New-URI", mirror);
else if (altMirrors.empty())
altMirrors.append(mirror);
else
fetch += "&arch=" + (*I);

// append the dist as a query string
if (Dist != "")
fetch += "&dist=" + Dist;

if(Debug)
clog << "MirrorMethod::DownloadMirrorFile(): '" << fetch << "'"
<< " to " << MirrorFile << endl;

pkgAcquire Fetcher;
new pkgAcqFile(&Fetcher, fetch, "", 0, "", "", "", MirrorFile);
bool res = (Fetcher.Run() == pkgAcquire::Continue);
if(res) {
DownloadedMirrorFile = true;
chmod(MirrorFile.c_str(), 0644);
}
Fetcher.Shutdown();

if(Debug)
clog << "MirrorMethod::DownloadMirrorFile() success: " << res << endl;

return res;
}

// Randomizes the lines in the mirror file, this is used so that
// we spread the load on the mirrors evenly
bool MirrorMethod::RandomizeMirrorFile(string mirror_file)
{
vector<string> content;
string line;

if (!FileExists(mirror_file))
return false;

// read
ifstream in(mirror_file.c_str());
while ( !in.eof() ) {
getline(in, line);
content.push_back(line);
altMirrors.append("\n").append(mirror);
}

// we want the file to be random for each different machine, but also
// "stable" on the same machine. this is to avoid running into out-of-sync
// issues (i.e. Release/Release.gpg different on each mirror)
struct utsname buf;
int seed=1;
if(uname(&buf) == 0) {
for(int i=0,seed=1; buf.nodename[i] != 0; ++i) {
seed = seed * 31 + buf.nodename[i];
}
}
srand( seed );
random_shuffle(content.begin(), content.end());

// write
ofstream out(mirror_file.c_str());
while ( !content.empty()) {
line = content.back();
content.pop_back();
out << line << "\n";
}

return true;
fields.emplace("Alternate-URIs", altMirrors);
SendMessage("103 Redirect", std::move(fields));
Dequeue();
}

/* convert a the Queue->Uri back to the mirror base uri and look
* at all mirrors we have for this, this is needed as queue->uri
* may point to different mirrors (if TryNextMirror() was run)
*/
void MirrorMethod::CurrentQueueUriToMirror()
/*}}}*/
void MirrorMethod::DealWithPendingItems(std::vector<std::string> const &baseuris, /*{{{*/
MirrorInfo const &info, FetchItem *const Itm,
std::function<void()> handler)
{
// already in mirror:// style so nothing to do
if(Queue->Uri.find("mirror://") == 0)
return;

// find current mirror and select next one
for (vector<string>::const_iterator mirror = AllMirrors.begin();
mirror != AllMirrors.end(); ++mirror)
FetchItem **LastItm = &Itm->Next;
while (*LastItm != nullptr)
LastItm = &((*LastItm)->Next);
while (Queue != Itm)
{
if (Queue->Uri.find(*mirror) == 0)
if (APT::String::Startswith(Queue->Uri, info.baseuri) == false ||
std::any_of(baseuris.cbegin(), baseuris.cend(), [&](std::string const &b) { return APT::String::Startswith(Queue->Uri, b); }))
{
// move the item behind the aux file not related to it
*LastItm = Queue;
Queue = QueueBack = Queue->Next;
(*LastItm)->Next = nullptr;
LastItm = &((*LastItm)->Next);
}
else
{
Queue->Uri.replace(0, mirror->length(), BaseUri);
return;
handler();
}
}
_error->Error("Internal error: Failed to convert %s back to %s",
Queue->Uri.c_str(), BaseUri.c_str());
// now remove out trigger
QueueBack = Queue = Queue->Next;
delete Itm;
}

bool MirrorMethod::TryNextMirror()
/*}}}*/
bool MirrorMethod::MirrorListFileRecieved(MirrorInfo &info, FetchItem *const Itm) /*{{{*/
{
// find current mirror and select next one
for (vector<string>::const_iterator mirror = AllMirrors.begin();
mirror != AllMirrors.end(); ++mirror)
std::vector<std::string> baseuris;
for (auto const &i : mirrorfilestate)
if (info.baseuri.length() < i.second.baseuri.length() &&
i.second.state == REQUESTED &&
APT::String::Startswith(i.second.baseuri, info.baseuri))
baseuris.push_back(i.second.baseuri);
sortByLength(baseuris);

FileFd mirrorlist;
if (FileExists(Itm->DestFile) && mirrorlist.Open(Itm->DestFile, FileFd::ReadOnly, FileFd::Extension))
{
if (Queue->Uri.find(*mirror) != 0)
continue;

vector<string>::const_iterator nextmirror = mirror + 1;
if (nextmirror == AllMirrors.end())
break;
Queue->Uri.replace(0, mirror->length(), *nextmirror);
if (Debug)
clog << "TryNextMirror: " << Queue->Uri << endl;

// inform parent
UsedMirror = *nextmirror;
Log("Switching mirror");
return true;
}

if (Debug)
clog << "TryNextMirror could not find another mirror to try" << endl;

return false;
}
std::string mirror;
while (mirrorlist.ReadLine(mirror))
{
if (mirror.empty() || mirror[0] == '#')
continue;
info.list.push_back(mirror);
}
mirrorlist.Close();
// we reseed each time to avoid "races" with multiple mirror://s
std::mt19937 g(seedvalue);
std::shuffle(info.list.begin(), info.list.end(), g);

bool MirrorMethod::InitMirrors()
{
// if we do not have a MirrorFile, fallback
if(!FileExists(MirrorFile))
{
// FIXME: fallback to a default mirror here instead
// and provide a config option to define that default
return _error->Error(_("No mirror file '%s' found "), MirrorFile.c_str());
if (info.list.empty())
{
info.state = FAILED;
DealWithPendingItems(baseuris, info, Itm, [&]() {
std::string msg;
strprintf(msg, "Mirror list %s is empty for %s", Itm->DestFile.c_str(), Queue->Uri.c_str());
Fail(msg, false);
});
}
else
{
info.state = AVAILABLE;
DealWithPendingItems(baseuris, info, Itm, [&]() {
RedirectItem(info, Queue);
});
}
}

if (access(MirrorFile.c_str(), R_OK) != 0)
{
// FIXME: fallback to a default mirror here instead
// and provide a config option to define that default
return _error->Error(_("Can not read mirror file '%s'"), MirrorFile.c_str());
}

// FIXME: make the mirror selection more clever, do not
// just use the first one!
// BUT: we can not make this random, the mirror has to be
// stable across session, because otherwise we can
// get into sync issues (got indexfiles from mirror A,
// but packages from mirror B - one might be out of date etc)
ifstream in(MirrorFile.c_str());
string s;
while (!in.eof())
else
{
getline(in, s);

// ignore lines that start with #
if (s.find("#") == 0)
continue;
// ignore empty lines
if (s.size() == 0)
continue;
// ignore non http lines
if (s.compare(0, strlen("http://"), "http://") != 0)
continue;

AllMirrors.push_back(s);
}
if (AllMirrors.empty()) {
return _error->Error(_("No entry found in mirror file '%s'"), MirrorFile.c_str());
info.state = FAILED;
DealWithPendingItems(baseuris, info, Itm, [&]() {
std::string msg;
strprintf(msg, "Downloading mirror file %s failed for %s", Itm->DestFile.c_str(), Queue->Uri.c_str());
Fail(msg, false);
});
}
Mirror = AllMirrors[0];
UsedMirror = Mirror;
return true;
}
string MirrorMethod::GetMirrorFileName(string mirror_uri_str)
/*}}}*/
std::string MirrorMethod::GetMirrorFileURI(std::string const &Message, FetchItem *const Itm) /*{{{*/
{
/*
- a mirror_uri_str looks like this:
mirror://people.ubuntu.com/~mvo/apt/mirror/mirrors/dists/feisty/Release.gpg
- the matching source.list entry
deb mirror://people.ubuntu.com/~mvo/apt/mirror/mirrors feisty main
- we actually want to go after:
http://people.ubuntu.com/~mvo/apt/mirror/mirrors

And we need to save the BaseUri for later:
- mirror://people.ubuntu.com/~mvo/apt/mirror/mirrors

FIXME: what if we have two similar prefixes?
mirror://people.ubuntu.com/~mvo/mirror
mirror://people.ubuntu.com/~mvo/mirror2
then mirror_uri_str looks like:
mirror://people.ubuntu.com/~mvo/apt/mirror/dists/feisty/Release.gpg
mirror://people.ubuntu.com/~mvo/apt/mirror2/dists/feisty/Release.gpg
we search sources.list and find:
mirror://people.ubuntu.com/~mvo/apt/mirror
in both cases! So we need to apply some domain knowledge here :( and
check for /dists/ or /Release.gpg as suffixes
*/
string name;
if(Debug)
std::cerr << "GetMirrorFileName: " << mirror_uri_str << std::endl;

// read sources.list and find match
vector<metaIndex *>::const_iterator I;
pkgSourceList list;
list.ReadMainList();
for(I=list.begin(); I != list.end(); ++I)
if (APT::String::Startswith(Itm->Uri, Binary))
{
std::string const repouri = LookupTag(Message, "Target-Repo-Uri");
if (repouri.empty() == false && std::find(sourceslist.cbegin(), sourceslist.cend(), repouri) == sourceslist.cend())
sourceslist.push_back(repouri);
}
if (sourceslist.empty())
{
string uristr = (*I)->GetURI();
if(Debug)
std::cerr << "Checking: " << uristr << std::endl;
if(uristr.substr(0,strlen("mirror://")) != string("mirror://"))
continue;
// find matching uri in sources.list
if(mirror_uri_str.substr(0,uristr.size()) == uristr)
// read sources.list and find the matching base uri
pkgSourceList sl;
if (sl.ReadMainList() == false)
{
if(Debug)
std::cerr << "found BaseURI: " << uristr << std::endl;
BaseUri = uristr.substr(0,uristr.size()-1);
Dist = (*I)->GetDist();
_error->Error(_("The list of sources could not be read."));
return "";
}
std::string const needle = Binary + ":";
for (auto const &SL : sl)
{
std::string uristr = SL->GetURI();
if (APT::String::Startswith(uristr, needle))
sourceslist.push_back(uristr);
}
sortByLength(sourceslist);
}
// get new file
name = _config->FindDir("Dir::State::mirrors") + URItoFileName(BaseUri);

if(Debug)
for (auto uristr : sourceslist)
{
cerr << "base-uri: " << BaseUri << endl;
cerr << "mirror-file: " << name << endl;
if (APT::String::Startswith(Itm->Uri, uristr))
{
uristr.erase(uristr.length() - 1); // remove the ending '/'
auto const colon = uristr.find(':');
if (unlikely(colon == std::string::npos))
continue;
auto const plus = uristr.find("+");
if (plus < colon)
return uristr.substr(plus + 1);
else
{
uristr.replace(0, strlen("mirror"), "http");
return uristr;
}
}
}
return name;
return "";
}

// MirrorMethod::Fetch - Fetch an item /*{{{*/
// ---------------------------------------------------------------------
/* This adds an item to the pipeline. We keep the pipeline at a fixed
depth. */
bool MirrorMethod::Fetch(FetchItem *Itm)
/*}}}*/
bool MirrorMethod::URIAcquire(std::string const &Message, FetchItem *Itm) /*{{{*/
{
if(Debug)
clog << "MirrorMethod::Fetch()" << endl;
auto mirrorinfo = mirrorfilestate.find(Itm->Uri);
if (mirrorinfo != mirrorfilestate.end())
return MirrorListFileRecieved(mirrorinfo->second, Itm);

// the http method uses Fetch(0) as a way to update the pipeline,
// just let it do its work in this case - Fetch() with a valid
// Itm will always run before the first Fetch(0)
if(Itm == NULL)
return HttpMethod::Fetch(Itm);

// if we don't have the name of the mirror file on disk yet,
// calculate it now (can be derived from the uri)
if(MirrorFile.empty())
MirrorFile = GetMirrorFileName(Itm->Uri);

// download mirror file once (if we are after index files)
if(Itm->IndexFile && !DownloadedMirrorFile)
std::string const mirrorfileuri = GetMirrorFileURI(Message, Itm);
if (mirrorfileuri.empty())
{
Clean(_config->FindDir("Dir::State::mirrors"));
if (DownloadMirrorFile(Itm->Uri))
RandomizeMirrorFile(MirrorFile);
_error->Error("Couldn't determine mirror list to query for %s", Itm->Uri.c_str());
return false;
}
if (DebugEnabled())
std::clog << "Mirror-URI: " << mirrorfileuri << " for " << Itm->Uri << std::endl;

if(AllMirrors.empty()) {
if(!InitMirrors()) {
// no valid mirror selected, something went wrong downloading
// from the master mirror site most likely and there is
// no old mirror file availalbe
// have we requested this mirror file already?
auto const state = mirrorfilestate.find(mirrorfileuri);
if (state == mirrorfilestate.end())
{
MirrorInfo info;
info.state = REQUESTED;
info.baseuri = mirrorfileuri + '/';
auto const colon = info.baseuri.find(':');
if (unlikely(colon == std::string::npos))
return false;
}
info.baseuri.replace(0, colon, Binary);
mirrorfilestate[mirrorfileuri] = info;
std::unordered_map<std::string, std::string> fields;
fields.emplace("URI", Itm->Uri);
fields.emplace("MaximumSize", std::to_string(1 * 1024 * 1024)); //FIXME: 1 MB is enough for everyone
fields.emplace("Aux-ShortDesc", "Mirrorlist");
fields.emplace("Aux-Description", mirrorfileuri + " Mirrorlist");
fields.emplace("Aux-Uri", mirrorfileuri);
SendMessage("351 Aux Request", std::move(fields));
return true;
}

if(Itm->Uri.find("mirror://") != string::npos)
Itm->Uri.replace(0,BaseUri.size(), Mirror);

if(Debug)
clog << "Fetch: " << Itm->Uri << endl << endl;

// now run the real fetcher
return HttpMethod::Fetch(Itm);
}

void MirrorMethod::Fail(string Err,bool Transient)
{
// FIXME: TryNextMirror is not ideal for indexfile as we may
// run into auth issues

if (Debug)
clog << "Failure to get " << Queue->Uri << endl;

// try the next mirror on fail (if its not a expected failure,
// e.g. translations are ok to ignore)
if (!Queue->FailIgnore && TryNextMirror())
return;

// all mirrors failed, so bail out
string s;
strprintf(s, _("[Mirror: %s]"), Mirror.c_str());
SetIP(s);

CurrentQueueUriToMirror();
pkgAcqMethod::Fail(Err, Transient);
}

void MirrorMethod::URIStart(FetchResult &Res)
{
CurrentQueueUriToMirror();
pkgAcqMethod::URIStart(Res);
}

void MirrorMethod::URIDone(FetchResult &Res,FetchResult *Alt)
{
CurrentQueueUriToMirror();
pkgAcqMethod::URIDone(Res, Alt);
switch (state->second.state)
{
case REQUESTED:
// lets wait for the requested mirror file
return true;
case FAILED:
Fail("Downloading mirror file failed", false);
return true;
case AVAILABLE:
RedirectItem(state->second, Itm);
return true;
}
return false;
}
/*}}}*/


int main()
int main(int, const char *argv[])
{
return MirrorMethod().Loop();
return MirrorMethod(flNotDir(argv[0])).Run();
}



+ 0
- 57
methods/mirror.h View File

@@ -1,57 +0,0 @@
// -*- mode: cpp; mode: fold -*-
// Description /*{{{*/
/* ######################################################################

MIRROR Acquire Method - This is the MIRROR acquire method for APT.

##################################################################### */
/*}}}*/

#ifndef APT_MIRROR_H
#define APT_MIRROR_H

#include <iostream>
#include <string>
#include <vector>

using std::cout;
using std::cerr;
using std::endl;

#include "http.h"

class MirrorMethod : public HttpMethod
{
FetchResult Res;
// we simply transform between BaseUri and Mirror
std::string BaseUri; // the original mirror://... url
std::string Mirror; // the selected mirror uri (http://...)
std::vector<std::string> AllMirrors; // all available mirrors
std::string MirrorFile; // the file that contains the list of mirrors
bool DownloadedMirrorFile; // already downloaded this session
std::string Dist; // the target distrubtion (e.g. sid, oneiric)

bool Debug;

protected:
bool DownloadMirrorFile(std::string uri);
bool RandomizeMirrorFile(std::string file);
std::string GetMirrorFileName(std::string uri);
bool InitMirrors();
bool TryNextMirror();
void CurrentQueueUriToMirror();
bool Clean(std::string dir);
// we need to overwrite those to transform the url back
virtual void Fail(std::string Why, bool Transient = false) APT_OVERRIDE;
virtual void URIStart(FetchResult &Res) APT_OVERRIDE;
virtual void URIDone(FetchResult &Res,FetchResult *Alt = 0) APT_OVERRIDE;
virtual bool Configuration(std::string Message) APT_OVERRIDE;

public:
MirrorMethod();
virtual bool Fetch(FetchItem *Itm) APT_OVERRIDE;
};


#endif

+ 1
- 1
po/CMakeLists.txt View File

@@ -15,7 +15,7 @@ apt_add_translation_domain(
TARGETS apt apt-cache apt-get apt-config apt-cdrom apt-helper apt-mark
apt-private
# Methods
connectlib httplib file copy store gpgv cdrom http ftp rred rsh mirror
connectlib file copy store gpgv cdrom http ftp rred rsh mirror
SCRIPTS ../dselect/install ../dselect/update
EXCLUDE_LANGUAGES ${languages_excluded}
)


Loading…
Cancel
Save