Browse Source

generalize Acquire::GzipIndex

tags/debian/1.0.9.2
Michael Vogt 7 years ago
parent
commit
b0f4b486e6
7 changed files with 90 additions and 58 deletions
  1. +29
    -15
      apt-pkg/acquire-item.cc
  2. +38
    -22
      apt-pkg/deb/debindexfile.cc
  3. +8
    -1
      apt-private/private-update.cc
  4. +1
    -1
      cmdline/apt-get.cc
  5. +4
    -7
      methods/copy.cc
  6. +1
    -1
      test/integration/test-bug-595691-empty-and-broken-archive-files
  7. +9
    -11
      test/integration/test-compressed-indexes

+ 29
- 15
apt-pkg/acquire-item.cc View File

@@ -969,8 +969,10 @@ void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &S
std::string const comprExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
if (comprExt == "uncompressed")
Desc.URI = URI;
else
else {
Desc.URI = URI + '.' + comprExt;
DestFile = DestFile + '.' + comprExt;
}

Desc.Description = URIDesc;
Desc.Owner = this;
@@ -984,10 +986,11 @@ void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &S
/* The only header we use is the last-modified header. */
string pkgAcqIndex::Custom600Headers()
{
std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
string Final = _config->FindDir("Dir::State::lists");
Final += URItoFileName(RealURI);
if (_config->FindB("Acquire::GzipIndexes",false))
Final += ".gz";
Final += compExt;
string msg = "\nIndex-File: true";
// FIXME: this really should use "IndexTarget::IsOptional()" but that
@@ -1027,8 +1030,8 @@ std::string pkgAcqIndex::GetFinalFilename(std::string const &URI,
{
std::string FinalFile = _config->FindDir("Dir::State::lists");
FinalFile += URItoFileName(URI);
if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz")
FinalFile += ".gz";
if (_config->FindB("Acquire::GzipIndexes",false) == true)
FinalFile += '.' + compExt;
return FinalFile;
}
/*}}}*/
@@ -1036,8 +1039,8 @@ std::string pkgAcqIndex::GetFinalFilename(std::string const &URI,
void pkgAcqIndex::ReverifyAfterIMS(std::string const &FileName)
{
std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz")
DestFile += ".gz";
if (_config->FindB("Acquire::GzipIndexes",false) == true)
DestFile += compExt;

string FinalFile = GetFinalFilename(RealURI, compExt);
Rename(FinalFile, FileName);
@@ -1080,7 +1083,7 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,string Hash,
/* Always verify the index file for correctness (all indexes must
* have a Package field) (LP: #346386) (Closes: #627642)
*/
FileFd fd(DestFile, FileFd::ReadOnlyGzip);
FileFd fd(DestFile, FileFd::ReadOnly, FileFd::Extension);
// Only test for correctness if the file is not empty (empty is ok)
if (fd.Size() > 0)
{
@@ -1104,8 +1107,8 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,string Hash,
will work OK */
DestFile = _config->FindDir("Dir::State::lists") + "partial/";
DestFile += URItoFileName(RealURI);
if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz")
DestFile += ".gz";
if (_config->FindB("Acquire::GzipIndexes",false))
DestFile += '.' + compExt;

// Remove the compressed version.
if (Erase == true)
@@ -1146,16 +1149,23 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,string Hash,
// matching the Release file
if (!Local && StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
{
// set destfile to the final destfile
if(_config->FindB("Acquire::GzipIndexes",false) == false)
{
DestFile = _config->FindDir("Dir::State::lists") + "partial/";
DestFile += URItoFileName(RealURI);
}

ReverifyAfterIMS(FileName);
return;
}
string decompProg;

// If we enable compressed indexes, queue for hash verification
if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz" && !Local)
if (_config->FindB("Acquire::GzipIndexes",false))
{
DestFile = _config->FindDir("Dir::State::lists");
DestFile += URItoFileName(RealURI) + ".gz";
DestFile += URItoFileName(RealURI) + '.' + compExt;

Decompression = true;
Desc.URI = "copy:" + FileName;
@@ -1201,11 +1211,11 @@ pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, IndexTarget const *Target,
// ---------------------------------------------------------------------
string pkgAcqIndexTrans::Custom600Headers()
{
std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
string Final = _config->FindDir("Dir::State::lists");
Final += URItoFileName(RealURI);

if (_config->FindB("Acquire::GzipIndexes",false))
Final += ".gz";
Final += compExt;

struct stat Buf;
if (stat(Final.c_str(),&Buf) != 0)
@@ -1552,8 +1562,12 @@ void pkgAcqMetaIndex::AuthDone(string Message) /*{{{*/
URItoFileName((*Target)->URI);
unlink(index.c_str());
// and also old gzipindexes
index += ".gz";
unlink(index.c_str());
std::vector<std::string> types = APT::Configuration::getCompressionTypes();
for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
{
index += '.' + (*t);
unlink(index.c_str());
}
}
}



+ 38
- 22
apt-pkg/deb/debindexfile.cc View File

@@ -80,14 +80,18 @@ pkgSrcRecords::Parser *debSourcesIndex::CreateSrcParser() const
{
string SourcesURI = _config->FindDir("Dir::State::lists") +
URItoFileName(IndexURI("Sources"));
string SourcesURIgzip = SourcesURI + ".gz";

if (!FileExists(SourcesURI) && !FileExists(SourcesURIgzip))
return NULL;
else if (!FileExists(SourcesURI) && FileExists(SourcesURIgzip))
SourcesURI = SourcesURIgzip;

return new debSrcRecordParser(SourcesURI,this);
std::vector<std::string> types = APT::Configuration::getCompressionTypes();
for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
{
string p;
p = SourcesURI + '.' + *t;
if (FileExists(p))
return new debSrcRecordParser(p, this);
}
if (FileExists(SourcesURI))
return new debSrcRecordParser(SourcesURI, this);
return NULL;
}
/*}}}*/
// SourcesIndex::Describe - Give a descriptive path to the index /*{{{*/
@@ -129,11 +133,15 @@ string debSourcesIndex::Info(const char *Type) const
inline string debSourcesIndex::IndexFile(const char *Type) const
{
string s = URItoFileName(IndexURI(Type));
string sgzip = s + ".gz";
if (!FileExists(s) && FileExists(sgzip))
return sgzip;
else
return s;

std::vector<std::string> types = APT::Configuration::getCompressionTypes();
for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
{
string p = s + '.' + *t;
if (FileExists(p))
return p;
}
return s;
}

string debSourcesIndex::IndexURI(const char *Type) const
@@ -259,11 +267,15 @@ string debPackagesIndex::Info(const char *Type) const
inline string debPackagesIndex::IndexFile(const char *Type) const
{
string s =_config->FindDir("Dir::State::lists") + URItoFileName(IndexURI(Type));
string sgzip = s + ".gz";
if (!FileExists(s) && FileExists(sgzip))
return sgzip;
else
return s;

std::vector<std::string> types = APT::Configuration::getCompressionTypes();
for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
{
string p = s + '.' + *t;
if (FileExists(p))
return p;
}
return s;
}
string debPackagesIndex::IndexURI(const char *Type) const
{
@@ -411,11 +423,15 @@ debTranslationsIndex::debTranslationsIndex(string URI,string Dist,string Section
inline string debTranslationsIndex::IndexFile(const char *Type) const
{
string s =_config->FindDir("Dir::State::lists") + URItoFileName(IndexURI(Type));
string sgzip = s + ".gz";
if (!FileExists(s) && FileExists(sgzip))
return sgzip;
else
return s;

std::vector<std::string> types = APT::Configuration::getCompressionTypes();
for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
{
string p = s + '.' + *t;
if (FileExists(p))
return p;
}
return s;
}
string debTranslationsIndex::IndexURI(const char *Type) const
{


+ 8
- 1
apt-private/private-update.cc View File

@@ -56,10 +56,17 @@ bool DoUpdate(CommandLine &CmdL)
if (List->GetIndexes(&Fetcher,true) == false)
return false;

std::string compExt = APT::Configuration::getCompressionTypes()[0];
pkgAcquire::UriIterator I = Fetcher.UriBegin();
for (; I != Fetcher.UriEnd(); ++I)
c1out << '\'' << I->URI << "' " << flNotDir(I->Owner->DestFile) << ' ' <<
{
std::string FileName = flNotDir(I->Owner->DestFile);
if(compExt.empty() == false &&
APT::String::Endswith(FileName, compExt))
FileName = FileName.substr(0, FileName.size() - compExt.size() - 1);
c1out << '\'' << I->URI << "' " << FileName << ' ' <<
I->Owner->FileSize << ' ' << I->Owner->HashSum() << std::endl;
}
return true;
}



+ 1
- 1
cmdline/apt-get.cc View File

@@ -665,7 +665,7 @@ static bool DoDownload(CommandLine &CmdL)
{
pkgAcquire::UriIterator I = Fetcher.UriBegin();
for (; I != Fetcher.UriEnd(); ++I)
cout << '\'' << I->URI << "' " << flNotDir(I->Owner->DestFile) << ' ' <<
cout << '\'' << I->URI << "' " << flNotDir(I->Owner->DestFile) << ' ' <<
I->Owner->FileSize << ' ' << I->Owner->HashSum() << endl;
return true;
}


+ 4
- 7
methods/copy.cc View File

@@ -37,15 +37,12 @@ class CopyMethod : public pkgAcqMethod

void CopyMethod::CalculateHashes(FetchResult &Res)
{
// For gzip indexes we need to look inside the gzip for the hash
// We can not use the extension here as its not used in partial
// on a IMS hit
FileFd::OpenMode OpenMode = FileFd::ReadOnly;
Hashes Hash;
FileFd::CompressMode CompressMode = FileFd::None;
if (_config->FindB("Acquire::GzipIndexes", false) == true)
OpenMode = FileFd::ReadOnlyGzip;
CompressMode = FileFd::Extension;

Hashes Hash;
FileFd Fd(Res.Filename, OpenMode);
FileFd Fd(Res.Filename, FileFd::ReadOnly, CompressMode);
Hash.AddFD(Fd);
Res.TakeHashes(Hash);
}


+ 1
- 1
test/integration/test-bug-595691-empty-and-broken-archive-files View File

@@ -121,7 +121,7 @@ Reading package lists..." "empty archive Packages.$COMPRESS over http"
testaptgetupdate "Get: http://localhost:8080 Packages
Err http://localhost:8080 Packages
Empty files can't be valid archives
W: Failed to fetch ${COMPRESSOR}:$(readlink -f rootdir/var/lib/apt/lists/partial/localhost:8080_Packages) Empty files can't be valid archives
W: Failed to fetch ${COMPRESSOR}:$(readlink -f rootdir/var/lib/apt/lists/partial/localhost:8080_Packages.${COMPRESS}) Empty files can't be valid archives

E: Some index files failed to download. They have been ignored, or old ones used instead." "empty file Packages.$COMPRESS over http"
}


+ 9
- 11
test/integration/test-compressed-indexes View File

@@ -5,7 +5,7 @@ TESTDIR=$(readlink -f $(dirname $0))
. $TESTDIR/framework

setupenvironment
configcompression '.' 'gz' # only gz is supported for this, so ensure it is used
configcompression '.' 'xz' 'gz'
configarchitecture "i386"

buildsimplenativepackage "testpkg" "i386" "1.0"
@@ -32,13 +32,13 @@ testrun() {
if [ "$1" = "compressed" ]; then
! test -e rootdir/var/lib/apt/lists/*_Packages || F=1
! test -e rootdir/var/lib/apt/lists/*_Sources || F=1
test -e rootdir/var/lib/apt/lists/*_Packages.gz || F=1
test -e rootdir/var/lib/apt/lists/*_Sources.gz || F=1
test -e rootdir/var/lib/apt/lists/*_Packages.xz || F=1
test -e rootdir/var/lib/apt/lists/*_Sources.xz || F=1
else
test -e rootdir/var/lib/apt/lists/*_Packages || F=1
test -e rootdir/var/lib/apt/lists/*_Sources || F=1
! test -e rootdir/var/lib/apt/lists/*_Packages.gz || F=1
! test -e rootdir/var/lib/apt/lists/*_Sources.gz || F=1
! test -e rootdir/var/lib/apt/lists/*_Packages.xz || F=1
! test -e rootdir/var/lib/apt/lists/*_Sources.xz || F=1
fi
if [ -n "$F" ]; then
ls -laR rootdir/var/lib/apt/lists/
@@ -84,10 +84,9 @@ msgmsg "File: Test with uncompressed indexes (update unchanged without pdiffs)"
testrun

rm -rf rootdir/var/lib/apt/lists
echo 'Acquire::CompressionTypes::Order:: "gz";
Acquire::GzipIndexes "true";' > rootdir/etc/apt/apt.conf.d/02compressindex
echo 'Acquire::GzipIndexes "true";' > rootdir/etc/apt/apt.conf.d/02compressindex

testsuccess aptget update
testsuccess aptget update -o Debug::pkgAcquire::worker=1
msgmsg "File: Test with compressed indexes"
testrun "compressed"

@@ -118,14 +117,13 @@ msgmsg "HTTP: Test with uncompressed indexes (update unchanged without pdiffs)"
testrun

rm -rf rootdir/var/lib/apt/lists
echo 'Acquire::CompressionTypes::Order:: "gz";
Acquire::GzipIndexes "true";' > rootdir/etc/apt/apt.conf.d/02compressindex
echo 'Acquire::GzipIndexes "true";' > rootdir/etc/apt/apt.conf.d/02compressindex

testsuccess aptget update
msgmsg "HTTP: Test with compressed indexes"
testrun "compressed"

testsuccess aptget update -o Acquire::Pdiffs=1
testsuccess aptget update -o Acquire::Pdiffs=1 -o debug::pkgAcquire::Worker=1 -o debug::pkgAcquire::Auth=1
msgmsg "HTTP: Test with compressed indexes (update unchanged with pdiffs)"
testrun "compressed"



Loading…
Cancel
Save