Browse Source

generalize Acquire::GzipIndex

debian/1.8.y
Michael Vogt 8 years ago
parent
commit
b0f4b486e6
  1. 44
      apt-pkg/acquire-item.cc
  2. 60
      apt-pkg/deb/debindexfile.cc
  3. 9
      apt-private/private-update.cc
  4. 2
      cmdline/apt-get.cc
  5. 11
      methods/copy.cc
  6. 2
      test/integration/test-bug-595691-empty-and-broken-archive-files
  7. 20
      test/integration/test-compressed-indexes

44
apt-pkg/acquire-item.cc

@ -969,8 +969,10 @@ void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &S
std::string const comprExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
if (comprExt == "uncompressed")
Desc.URI = URI;
else
else {
Desc.URI = URI + '.' + comprExt;
DestFile = DestFile + '.' + comprExt;
}
Desc.Description = URIDesc;
Desc.Owner = this;
@ -984,10 +986,11 @@ void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &S
/* The only header we use is the last-modified header. */
string pkgAcqIndex::Custom600Headers()
{
std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
string Final = _config->FindDir("Dir::State::lists");
Final += URItoFileName(RealURI);
if (_config->FindB("Acquire::GzipIndexes",false))
Final += ".gz";
Final += compExt;
string msg = "\nIndex-File: true";
// FIXME: this really should use "IndexTarget::IsOptional()" but that
@ -1027,8 +1030,8 @@ std::string pkgAcqIndex::GetFinalFilename(std::string const &URI,
{
std::string FinalFile = _config->FindDir("Dir::State::lists");
FinalFile += URItoFileName(URI);
if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz")
FinalFile += ".gz";
if (_config->FindB("Acquire::GzipIndexes",false) == true)
FinalFile += '.' + compExt;
return FinalFile;
}
/*}}}*/
@ -1036,8 +1039,8 @@ std::string pkgAcqIndex::GetFinalFilename(std::string const &URI,
void pkgAcqIndex::ReverifyAfterIMS(std::string const &FileName)
{
std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz")
DestFile += ".gz";
if (_config->FindB("Acquire::GzipIndexes",false) == true)
DestFile += compExt;
string FinalFile = GetFinalFilename(RealURI, compExt);
Rename(FinalFile, FileName);
@ -1080,7 +1083,7 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,string Hash,
/* Always verify the index file for correctness (all indexes must
* have a Package field) (LP: #346386) (Closes: #627642)
*/
FileFd fd(DestFile, FileFd::ReadOnlyGzip);
FileFd fd(DestFile, FileFd::ReadOnly, FileFd::Extension);
// Only test for correctness if the file is not empty (empty is ok)
if (fd.Size() > 0)
{
@ -1104,8 +1107,8 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,string Hash,
will work OK */
DestFile = _config->FindDir("Dir::State::lists") + "partial/";
DestFile += URItoFileName(RealURI);
if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz")
DestFile += ".gz";
if (_config->FindB("Acquire::GzipIndexes",false))
DestFile += '.' + compExt;
// Remove the compressed version.
if (Erase == true)
@ -1146,16 +1149,23 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,string Hash,
// matching the Release file
if (!Local && StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
{
// set destfile to the final destfile
if(_config->FindB("Acquire::GzipIndexes",false) == false)
{
DestFile = _config->FindDir("Dir::State::lists") + "partial/";
DestFile += URItoFileName(RealURI);
}
ReverifyAfterIMS(FileName);
return;
}
string decompProg;
// If we enable compressed indexes, queue for hash verification
if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz" && !Local)
if (_config->FindB("Acquire::GzipIndexes",false))
{
DestFile = _config->FindDir("Dir::State::lists");
DestFile += URItoFileName(RealURI) + ".gz";
DestFile += URItoFileName(RealURI) + '.' + compExt;
Decompression = true;
Desc.URI = "copy:" + FileName;
@ -1201,11 +1211,11 @@ pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, IndexTarget const *Target,
// ---------------------------------------------------------------------
string pkgAcqIndexTrans::Custom600Headers()
{
std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
string Final = _config->FindDir("Dir::State::lists");
Final += URItoFileName(RealURI);
if (_config->FindB("Acquire::GzipIndexes",false))
Final += ".gz";
Final += compExt;
struct stat Buf;
if (stat(Final.c_str(),&Buf) != 0)
@ -1552,8 +1562,12 @@ void pkgAcqMetaIndex::AuthDone(string Message) /*{{{*/
URItoFileName((*Target)->URI);
unlink(index.c_str());
// and also old gzipindexes
index += ".gz";
unlink(index.c_str());
std::vector<std::string> types = APT::Configuration::getCompressionTypes();
for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
{
index += '.' + (*t);
unlink(index.c_str());
}
}
}

60
apt-pkg/deb/debindexfile.cc

@ -80,14 +80,18 @@ pkgSrcRecords::Parser *debSourcesIndex::CreateSrcParser() const
{
string SourcesURI = _config->FindDir("Dir::State::lists") +
URItoFileName(IndexURI("Sources"));
string SourcesURIgzip = SourcesURI + ".gz";
if (!FileExists(SourcesURI) && !FileExists(SourcesURIgzip))
return NULL;
else if (!FileExists(SourcesURI) && FileExists(SourcesURIgzip))
SourcesURI = SourcesURIgzip;
return new debSrcRecordParser(SourcesURI,this);
std::vector<std::string> types = APT::Configuration::getCompressionTypes();
for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
{
string p;
p = SourcesURI + '.' + *t;
if (FileExists(p))
return new debSrcRecordParser(p, this);
}
if (FileExists(SourcesURI))
return new debSrcRecordParser(SourcesURI, this);
return NULL;
}
/*}}}*/
// SourcesIndex::Describe - Give a descriptive path to the index /*{{{*/
@ -129,11 +133,15 @@ string debSourcesIndex::Info(const char *Type) const
inline string debSourcesIndex::IndexFile(const char *Type) const
{
string s = URItoFileName(IndexURI(Type));
string sgzip = s + ".gz";
if (!FileExists(s) && FileExists(sgzip))
return sgzip;
else
return s;
std::vector<std::string> types = APT::Configuration::getCompressionTypes();
for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
{
string p = s + '.' + *t;
if (FileExists(p))
return p;
}
return s;
}
string debSourcesIndex::IndexURI(const char *Type) const
@ -259,11 +267,15 @@ string debPackagesIndex::Info(const char *Type) const
inline string debPackagesIndex::IndexFile(const char *Type) const
{
string s =_config->FindDir("Dir::State::lists") + URItoFileName(IndexURI(Type));
string sgzip = s + ".gz";
if (!FileExists(s) && FileExists(sgzip))
return sgzip;
else
return s;
std::vector<std::string> types = APT::Configuration::getCompressionTypes();
for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
{
string p = s + '.' + *t;
if (FileExists(p))
return p;
}
return s;
}
string debPackagesIndex::IndexURI(const char *Type) const
{
@ -411,11 +423,15 @@ debTranslationsIndex::debTranslationsIndex(string URI,string Dist,string Section
inline string debTranslationsIndex::IndexFile(const char *Type) const
{
string s =_config->FindDir("Dir::State::lists") + URItoFileName(IndexURI(Type));
string sgzip = s + ".gz";
if (!FileExists(s) && FileExists(sgzip))
return sgzip;
else
return s;
std::vector<std::string> types = APT::Configuration::getCompressionTypes();
for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
{
string p = s + '.' + *t;
if (FileExists(p))
return p;
}
return s;
}
string debTranslationsIndex::IndexURI(const char *Type) const
{

9
apt-private/private-update.cc

@ -56,10 +56,17 @@ bool DoUpdate(CommandLine &CmdL)
if (List->GetIndexes(&Fetcher,true) == false)
return false;
std::string compExt = APT::Configuration::getCompressionTypes()[0];
pkgAcquire::UriIterator I = Fetcher.UriBegin();
for (; I != Fetcher.UriEnd(); ++I)
c1out << '\'' << I->URI << "' " << flNotDir(I->Owner->DestFile) << ' ' <<
{
std::string FileName = flNotDir(I->Owner->DestFile);
if(compExt.empty() == false &&
APT::String::Endswith(FileName, compExt))
FileName = FileName.substr(0, FileName.size() - compExt.size() - 1);
c1out << '\'' << I->URI << "' " << FileName << ' ' <<
I->Owner->FileSize << ' ' << I->Owner->HashSum() << std::endl;
}
return true;
}

2
cmdline/apt-get.cc

@ -665,7 +665,7 @@ static bool DoDownload(CommandLine &CmdL)
{
pkgAcquire::UriIterator I = Fetcher.UriBegin();
for (; I != Fetcher.UriEnd(); ++I)
cout << '\'' << I->URI << "' " << flNotDir(I->Owner->DestFile) << ' ' <<
cout << '\'' << I->URI << "' " << flNotDir(I->Owner->DestFile) << ' ' <<
I->Owner->FileSize << ' ' << I->Owner->HashSum() << endl;
return true;
}

11
methods/copy.cc

@ -37,15 +37,12 @@ class CopyMethod : public pkgAcqMethod
void CopyMethod::CalculateHashes(FetchResult &Res)
{
// For gzip indexes we need to look inside the gzip for the hash
// We can not use the extension here as its not used in partial
// on a IMS hit
FileFd::OpenMode OpenMode = FileFd::ReadOnly;
Hashes Hash;
FileFd::CompressMode CompressMode = FileFd::None;
if (_config->FindB("Acquire::GzipIndexes", false) == true)
OpenMode = FileFd::ReadOnlyGzip;
CompressMode = FileFd::Extension;
Hashes Hash;
FileFd Fd(Res.Filename, OpenMode);
FileFd Fd(Res.Filename, FileFd::ReadOnly, CompressMode);
Hash.AddFD(Fd);
Res.TakeHashes(Hash);
}

2
test/integration/test-bug-595691-empty-and-broken-archive-files

@ -121,7 +121,7 @@ Reading package lists..." "empty archive Packages.$COMPRESS over http"
testaptgetupdate "Get: http://localhost:8080 Packages
Err http://localhost:8080 Packages
Empty files can't be valid archives
W: Failed to fetch ${COMPRESSOR}:$(readlink -f rootdir/var/lib/apt/lists/partial/localhost:8080_Packages) Empty files can't be valid archives
W: Failed to fetch ${COMPRESSOR}:$(readlink -f rootdir/var/lib/apt/lists/partial/localhost:8080_Packages.${COMPRESS}) Empty files can't be valid archives
E: Some index files failed to download. They have been ignored, or old ones used instead." "empty file Packages.$COMPRESS over http"
}

20
test/integration/test-compressed-indexes

@ -5,7 +5,7 @@ TESTDIR=$(readlink -f $(dirname $0))
. $TESTDIR/framework
setupenvironment
configcompression '.' 'gz' # only gz is supported for this, so ensure it is used
configcompression '.' 'xz' 'gz'
configarchitecture "i386"
buildsimplenativepackage "testpkg" "i386" "1.0"
@ -32,13 +32,13 @@ testrun() {
if [ "$1" = "compressed" ]; then
! test -e rootdir/var/lib/apt/lists/*_Packages || F=1
! test -e rootdir/var/lib/apt/lists/*_Sources || F=1
test -e rootdir/var/lib/apt/lists/*_Packages.gz || F=1
test -e rootdir/var/lib/apt/lists/*_Sources.gz || F=1
test -e rootdir/var/lib/apt/lists/*_Packages.xz || F=1
test -e rootdir/var/lib/apt/lists/*_Sources.xz || F=1
else
test -e rootdir/var/lib/apt/lists/*_Packages || F=1
test -e rootdir/var/lib/apt/lists/*_Sources || F=1
! test -e rootdir/var/lib/apt/lists/*_Packages.gz || F=1
! test -e rootdir/var/lib/apt/lists/*_Sources.gz || F=1
! test -e rootdir/var/lib/apt/lists/*_Packages.xz || F=1
! test -e rootdir/var/lib/apt/lists/*_Sources.xz || F=1
fi
if [ -n "$F" ]; then
ls -laR rootdir/var/lib/apt/lists/
@ -84,10 +84,9 @@ msgmsg "File: Test with uncompressed indexes (update unchanged without pdiffs)"
testrun
rm -rf rootdir/var/lib/apt/lists
echo 'Acquire::CompressionTypes::Order:: "gz";
Acquire::GzipIndexes "true";' > rootdir/etc/apt/apt.conf.d/02compressindex
echo 'Acquire::GzipIndexes "true";' > rootdir/etc/apt/apt.conf.d/02compressindex
testsuccess aptget update
testsuccess aptget update -o Debug::pkgAcquire::worker=1
msgmsg "File: Test with compressed indexes"
testrun "compressed"
@ -118,14 +117,13 @@ msgmsg "HTTP: Test with uncompressed indexes (update unchanged without pdiffs)"
testrun
rm -rf rootdir/var/lib/apt/lists
echo 'Acquire::CompressionTypes::Order:: "gz";
Acquire::GzipIndexes "true";' > rootdir/etc/apt/apt.conf.d/02compressindex
echo 'Acquire::GzipIndexes "true";' > rootdir/etc/apt/apt.conf.d/02compressindex
testsuccess aptget update
msgmsg "HTTP: Test with compressed indexes"
testrun "compressed"
testsuccess aptget update -o Acquire::Pdiffs=1
testsuccess aptget update -o Acquire::Pdiffs=1 -o debug::pkgAcquire::Worker=1 -o debug::pkgAcquire::Auth=1
msgmsg "HTTP: Test with compressed indexes (update unchanged with pdiffs)"
testrun "compressed"

Loading…
Cancel
Save