Browse Source

keep compressed indexes in a low-cost format

Downloading and storing are two different operations were different
compression types can be preferred. For downloading we provide the
choice via Acquire::CompressionTypes::Order as there is a choice to
be made between download size and speed – and limited by whats available
in the repository.

Storage on the other hand has all compressions currently supported by
apt available and to reduce runtime of tools accessing these files the
compression type should be a low-cost format in terms of decompression.

apt traditionally stores its indexes uncompressed on disk, but has
options to keep them compressed. Now that apt downloads additional files
we also deal with files which simply can't be stored uncompressed as
they are just too big (like Contents for apt-file). Traditionally they
are downloaded in a low-cost format (gz) as repositories do not provide
other formats, but there might be even lower-cost formats and for
download we could introduce higher-cost in the repositories.

Downloading an entire index potentially requires recompression to
another format, so an update takes potentially longer – but big files
are usually updated via pdiffs which has to de- and re-compress anyhow
and does it on the fly anyhow, so there is no extra time needed and in
general it seems to be benefitial to invest the time in update to save
time later on file access.
tags/debian/1.2_exp1
David Kalnischkies 5 years ago
parent
commit
0179cfa83c
14 changed files with 157 additions and 128 deletions
  1. +78
    -99
      apt-pkg/acquire-item.cc
  2. +2
    -9
      apt-pkg/acquire-item.h
  3. +2
    -0
      apt-pkg/aptconfiguration.cc
  4. +40
    -0
      apt-pkg/deb/debmetaindex.cc
  5. +1
    -0
      apt-pkg/indexfile.cc
  6. +1
    -0
      apt-pkg/indexfile.h
  7. +2
    -0
      cmdline/apt-get.cc
  8. +4
    -0
      doc/acquire-additional-files.txt
  9. +1
    -1
      test/integration/framework
  10. +8
    -7
      test/integration/test-apt-acquire-additional-files
  11. +4
    -4
      test/integration/test-apt-update-ims
  12. +7
    -3
      test/integration/test-apt-update-unauth
  13. +5
    -4
      test/integration/test-compressed-indexes
  14. +2
    -1
      test/integration/test-pdiff-usage

+ 78
- 99
apt-pkg/acquire-item.cc View File

@@ -85,32 +85,16 @@ static std::string GetKeepCompressedFileName(std::string file, IndexTarget const
if (Target.KeepCompressed == false)
return file;

std::string const CompressionTypes = Target.Option(IndexTarget::COMPRESSIONTYPES);
if (CompressionTypes.empty() == false)
std::string const KeepCompressedAs = Target.Option(IndexTarget::KEEPCOMPRESSEDAS);
if (KeepCompressedAs.empty() == false)
{
std::string const ext = CompressionTypes.substr(0, CompressionTypes.find(' '));
std::string const ext = KeepCompressedAs.substr(0, KeepCompressedAs.find(' '));
if (ext != "uncompressed")
file.append(".").append(ext);
}
return file;
}
/*}}}*/
static std::string GetCompressedFileName(IndexTarget const &Target, std::string const &Name, std::string const &Ext) /*{{{*/
{
if (Ext.empty() || Ext == "uncompressed")
return Name;

// do not reverify cdrom sources as apt-cdrom may rewrite the Packages
// file when its doing the indexcopy
if (Target.URI.substr(0,6) == "cdrom:")
return Name;

// adjust DestFile if its compressed on disk
if (Target.KeepCompressed == true)
return Name + '.' + Ext;
return Name;
}
/*}}}*/
static std::string GetMergeDiffsPatchFileName(std::string const &Final, std::string const &Patch)/*{{{*/
{
// rred expects the patch as $FinalFile.ed.$patchname.gz
@@ -326,7 +310,7 @@ std::string pkgAcqDiffIndex::GetFinalFilename() const
std::string pkgAcqIndex::GetFinalFilename() const
{
std::string const FinalFile = GetFinalFileNameFromURI(Target.URI);
return GetCompressedFileName(Target, FinalFile, CurrentCompressionExtension);
return GetKeepCompressedFileName(FinalFile, Target);
}
std::string pkgAcqMetaSig::GetFinalFilename() const
{
@@ -379,7 +363,30 @@ bool pkgAcqTransactionItem::TransactionState(TransactionStates const state)
case TransactionCommit:
if(PartialFile.empty() == false)
{
if (PartialFile != DestFile)
bool sameFile = (PartialFile == DestFile);
// we use symlinks on IMS-Hit to avoid copies
if (RealFileExists(DestFile))
{
struct stat Buf;
if (lstat(PartialFile.c_str(), &Buf) != -1)
{
if (S_ISLNK(Buf.st_mode) && Buf.st_size > 0)
{
char partial[Buf.st_size + 1];
ssize_t const sp = readlink(PartialFile.c_str(), partial, Buf.st_size);
if (sp == -1)
_error->Errno("pkgAcqTransactionItem::TransactionState-sp", _("Failed to readlink %s"), PartialFile.c_str());
else
{
partial[sp] = '\0';
sameFile = (DestFile == partial);
}
}
}
else
_error->Errno("pkgAcqTransactionItem::TransactionState-stat", _("Failed to stat %s"), PartialFile.c_str());
}
if (sameFile == false)
{
// ensure that even without lists-cleanup all compressions are nuked
std::string FinalFile = GetFinalFileNameFromURI(Target.URI);
@@ -2621,20 +2628,6 @@ void pkgAcqIndex::Failed(string const &Message,pkgAcquire::MethodConfig const *
TransactionManager->AbortTransaction();
}
/*}}}*/
// AcqIndex::ReverifyAfterIMS - Reverify index after an ims-hit /*{{{*/
void pkgAcqIndex::ReverifyAfterIMS()
{
// update destfile to *not* include the compression extension when doing
// a reverify (as its uncompressed on disk already)
DestFile = GetCompressedFileName(Target, GetPartialFileNameFromURI(Target.URI), CurrentCompressionExtension);

// copy FinalFile into partial/ so that we check the hash again
string FinalFile = GetFinalFilename();
Stage = STAGE_DECOMPRESS_AND_VERIFY;
Desc.URI = "copy:" + FinalFile;
QueueURI(Desc);
}
/*}}}*/
// AcqIndex::Done - Finished a fetch /*{{{*/
// ---------------------------------------------------------------------
/* This goes through a number of states.. On the initial fetch the
@@ -2651,107 +2644,93 @@ void pkgAcqIndex::Done(string const &Message,
switch(Stage)
{
case STAGE_DOWNLOAD:
StageDownloadDone(Message, Hashes, Cfg);
StageDownloadDone(Message);
break;
case STAGE_DECOMPRESS_AND_VERIFY:
StageDecompressDone(Message, Hashes, Cfg);
StageDecompressDone();
break;
}
}
/*}}}*/
// AcqIndex::StageDownloadDone - Queue for decompress and verify /*{{{*/
void pkgAcqIndex::StageDownloadDone(string const &Message, HashStringList const &,
pkgAcquire::MethodConfig const * const)
void pkgAcqIndex::StageDownloadDone(string const &Message)
{
Local = true;
Complete = true;

// Handle the unzipd case
std::string FileName = LookupTag(Message,"Alt-Filename");
if (FileName.empty() == false)
std::string const AltFilename = LookupTag(Message,"Alt-Filename");
std::string Filename = LookupTag(Message,"Filename");

// we need to verify the file against the current Release file again
// on if-modfied-since hit to avoid a stale attack against us
if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
{
// copy FinalFile into partial/ so that we check the hash again
string const FinalFile = GetExistingFilename(GetFinalFileNameFromURI(Target.URI));
if (symlink(FinalFile.c_str(), DestFile.c_str()) != 0)
_error->WarningE("pkgAcqIndex::StageDownloadDone", "Symlinking final file %s back to %s failed", FinalFile.c_str(), DestFile.c_str());
else
{
EraseFileName = DestFile;
Filename = DestFile;
}
Stage = STAGE_DECOMPRESS_AND_VERIFY;
Local = true;
if (CurrentCompressionExtension != "uncompressed")
DestFile.erase(DestFile.length() - (CurrentCompressionExtension.length() + 1));
Desc.URI = "copy:" + FileName;
Desc.URI = "store:" + Filename;
QueueURI(Desc);
SetActiveSubprocess("copy");
SetActiveSubprocess(::URI(Desc.URI).Access);
return;
}
FileName = LookupTag(Message,"Filename");

// methods like file:// give us an alternative (uncompressed) file
else if (Target.KeepCompressed == false && AltFilename.empty() == false)
{
if (CurrentCompressionExtension != "uncompressed")
DestFile.erase(DestFile.length() - (CurrentCompressionExtension.length() + 1));
Filename = AltFilename;
}
// Methods like e.g. "file:" will give us a (compressed) FileName that is
// not the "DestFile" we set, in this case we uncompress from the local file
if (FileName != DestFile && RealFileExists(DestFile) == false)
else if (Filename != DestFile && RealFileExists(DestFile) == false)
{
Local = true;
if (Target.KeepCompressed == true)
{
// but if we don't keep the uncompress we copy the compressed file first
Stage = STAGE_DOWNLOAD;
Desc.URI = "copy:" + FileName;
QueueURI(Desc);
SetActiveSubprocess("copy");
return;
}
// symlinking ensures that the filename can be used for compression detection
// that is e.g. needed for by-hash which has no extension over file
if (symlink(Filename.c_str(),DestFile.c_str()) != 0)
_error->WarningE("pkgAcqIndex::StageDownloadDone", "Symlinking file %s to %s failed", Filename.c_str(), DestFile.c_str());
else
{
// symlinking ensures that the filename can be used for compression detection
// that is e.g. needed for by-hash over file
if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
_error->WarningE("pkgAcqIndex::StageDownloadDone", "Symlinking file %s to %s failed", FileName.c_str(), DestFile.c_str());
else
{
EraseFileName = DestFile;
FileName = DestFile;
}
EraseFileName = DestFile;
Filename = DestFile;
}
}
else
EraseFileName = FileName;

// we need to verify the file against the current Release file again
// on if-modfied-since hit to avoid a stale attack against us
if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
{
// The files timestamp matches, reverify by copy into partial/
EraseFileName = "";
ReverifyAfterIMS();
return;
}

string decompProg = "store";
if (Target.KeepCompressed == true)
{
DestFile = "/dev/null";
EraseFileName.clear();
}
Stage = STAGE_DECOMPRESS_AND_VERIFY;
DestFile = GetKeepCompressedFileName(GetPartialFileNameFromURI(Target.URI), Target);
if (Filename != DestFile && flExtension(Filename) == flExtension(DestFile))
Desc.URI = "copy:" + Filename;
else
Desc.URI = "store:" + Filename;
if (DestFile == Filename)
{
if (CurrentCompressionExtension == "uncompressed")
decompProg = "copy";
else
DestFile.erase(DestFile.length() - (CurrentCompressionExtension.length() + 1));
return StageDecompressDone();
DestFile = "/dev/null";
}

if (EraseFileName.empty())
EraseFileName = Filename;

// queue uri for the next stage
Stage = STAGE_DECOMPRESS_AND_VERIFY;
Desc.URI = decompProg + ":" + FileName;
QueueURI(Desc);
SetActiveSubprocess(decompProg);
SetActiveSubprocess(::URI(Desc.URI).Access);
}
/*}}}*/
// AcqIndex::StageDecompressDone - Final verification /*{{{*/
void pkgAcqIndex::StageDecompressDone(string const &,
HashStringList const &,
pkgAcquire::MethodConfig const * const)
void pkgAcqIndex::StageDecompressDone()
{
if (Target.KeepCompressed == true && DestFile == "/dev/null")
DestFile = GetPartialFileNameFromURI(Target.URI + '.' + CurrentCompressionExtension);
if (DestFile == "/dev/null")
DestFile = GetKeepCompressedFileName(GetPartialFileNameFromURI(Target.URI), Target);

// Done, queue for rename on transaction finished
TransactionManager->TransactionStageCopy(this, DestFile, GetFinalFilename());
return;
}
/*}}}*/
pkgAcqIndex::~pkgAcqIndex() {}


+ 2
- 9
apt-pkg/acquire-item.h View File

@@ -914,16 +914,12 @@ class APT_HIDDEN pkgAcqIndex : public pkgAcqBaseIndex
AllStages Stage;

/** \brief Handle what needs to be done when the download is done */
void StageDownloadDone(std::string const &Message,
HashStringList const &Hashes,
pkgAcquire::MethodConfig const * const Cfg);
void StageDownloadDone(std::string const &Message);

/** \brief Handle what needs to be done when the decompression/copy is
* done
*/
void StageDecompressDone(std::string const &Message,
HashStringList const &Hashes,
pkgAcquire::MethodConfig const * const Cfg);
void StageDecompressDone();

/** \brief If \b set, this partially downloaded file will be
* removed when the download completes.
@@ -941,9 +937,6 @@ class APT_HIDDEN pkgAcqIndex : public pkgAcqBaseIndex
/** \brief Do the changes needed to fetch via AptByHash (if needed) */
void InitByHashIfNeeded();

/** \brief Schedule file for verification after a IMS hit */
void ReverifyAfterIMS();

/** \brief Get the full pathname of the final file for the current URI */
virtual std::string GetFinalFilename() const APT_OVERRIDE;



+ 2
- 0
apt-pkg/aptconfiguration.cc View File

@@ -37,6 +37,7 @@ static void setDefaultConfigurationForCompressors() {
// Set default application paths to check for optional compression types
_config->CndSet("Dir::Bin::bzip2", "/bin/bzip2");
_config->CndSet("Dir::Bin::xz", "/usr/bin/xz");
_config->CndSet("Dir::Bin::lz4", "/usr/bin/lz4");
if (FileExists(_config->FindFile("Dir::Bin::xz")) == true) {
_config->Set("Dir::Bin::lzma", _config->FindFile("Dir::Bin::xz"));
_config->Set("APT::Compressor::lzma::Binary", "xz");
@@ -79,6 +80,7 @@ const Configuration::getCompressionTypes(bool const &Cached) {
_config->CndSet("Acquire::CompressionTypes::bz2","bzip2");
_config->CndSet("Acquire::CompressionTypes::lzma","lzma");
_config->CndSet("Acquire::CompressionTypes::gz","gzip");
_config->CndSet("Acquire::CompressionTypes::lz4","lz4");

setDefaultConfigurationForCompressors();
std::vector<APT::Configuration::Compressor> const compressors = getCompressors();


+ 40
- 0
apt-pkg/deb/debmetaindex.cc View File

@@ -138,6 +138,21 @@ static void GetIndexTargetsFor(char const * const Type, std::string const &URI,
DefCompressionTypes = os.str();
}
}
std::string DefKeepCompressedAs;
{
std::vector<APT::Configuration::Compressor> comps = APT::Configuration::getCompressors();
if (comps.empty() == false)
{
std::sort(comps.begin(), comps.end(),
[](APT::Configuration::Compressor const &a, APT::Configuration::Compressor const &b) { return a.Cost < b.Cost; });
std::ostringstream os;
for (auto const &c : comps)
if (c.Cost != 0)
os << c.Extension.substr(1) << ' ';
DefKeepCompressedAs = os.str();
}
DefKeepCompressedAs += "uncompressed";
}
std::string const NativeArch = _config->Find("APT::Architecture");
bool const GzipIndex = _config->FindB("Acquire::GzipIndexes", false);
for (std::vector<debReleaseIndexPrivate::debSectionEntry>::const_iterator E = entries.begin(); E != entries.end(); ++E)
@@ -155,11 +170,35 @@ static void GetIndexTargetsFor(char const * const Type, std::string const &URI,
bool const UsePDiffs = APT_T_CONFIG_BOOL("PDiffs", E->UsePDiffs);
std::string const UseByHash = APT_T_CONFIG_STR("By-Hash", E->UseByHash);
std::string const CompressionTypes = APT_T_CONFIG_STR("CompressionTypes", DefCompressionTypes);
std::string KeepCompressedAs = APT_T_CONFIG_STR("KeepCompressedAs", "");
#undef APT_T_CONFIG_BOOL
#undef APT_T_CONFIG_STR
if (tplMetaKey.empty())
continue;

if (KeepCompressedAs.empty())
KeepCompressedAs = DefKeepCompressedAs;
else
{
std::vector<std::string> const defKeep = VectorizeString(DefKeepCompressedAs, ' ');
std::vector<std::string> const valKeep = VectorizeString(KeepCompressedAs, ' ');
std::vector<std::string> keep;
for (auto const &val : valKeep)
{
if (val.empty())
continue;
if (std::find(defKeep.begin(), defKeep.end(), val) == defKeep.end())
continue;
keep.push_back(val);
}
if (std::find(keep.begin(), keep.end(), "uncompressed") == keep.end())
keep.push_back("uncompressed");
std::ostringstream os;
std::copy(keep.begin(), keep.end()-1, std::ostream_iterator<std::string>(os, " "));
os << *keep.rbegin();
KeepCompressedAs = os.str();
}

for (std::vector<std::string>::const_iterator L = E->Languages.begin(); L != E->Languages.end(); ++L)
{
if (*L == "none" && tplMetaKey.find("$(LANGUAGE)") != std::string::npos)
@@ -253,6 +292,7 @@ static void GetIndexTargetsFor(char const * const Type, std::string const &URI,
Options.insert(std::make_pair("BY_HASH", UseByHash));
Options.insert(std::make_pair("DEFAULTENABLED", DefaultEnabled ? "yes" : "no"));
Options.insert(std::make_pair("COMPRESSIONTYPES", CompressionTypes));
Options.insert(std::make_pair("KEEPCOMPRESSEDAS", KeepCompressedAs));
Options.insert(std::make_pair("SOURCESENTRY", E->sourcesEntry));

bool IsOpt = IsOptional;


+ 1
- 0
apt-pkg/indexfile.cc View File

@@ -144,6 +144,7 @@ std::string IndexTarget::Option(OptionKeys const EnumKey) const /*{{{*/
APT_CASE(COMPRESSIONTYPES);
APT_CASE(SOURCESENTRY);
APT_CASE(BY_HASH);
APT_CASE(KEEPCOMPRESSEDAS);
#undef APT_CASE
case FILENAME: return _config->FindDir("Dir::State::lists") + URItoFileName(URI);
case EXISTING_FILENAME:


+ 1
- 0
apt-pkg/indexfile.h View File

@@ -90,6 +90,7 @@ class IndexTarget /*{{{*/
DEFAULTENABLED,
SOURCESENTRY,
BY_HASH,
KEEPCOMPRESSEDAS,
};
std::string Option(OptionKeys const Key) const;
bool OptionBool(OptionKeys const Key) const;


+ 2
- 0
cmdline/apt-get.cc View File

@@ -299,6 +299,8 @@ static bool DoIndexTargets(CommandLine &CmdL)
stanza << "PDiffs: " << O->second << "\n";
else if (O->first == "COMPRESSIONTYPES")
stanza << "CompressionTypes: " << O->second << "\n";
else if (O->first == "KEEPCOMPRESSEDAS")
stanza << "KeepCompressedAs: " << O->second << "\n";
else if (O->first == "DEFAULTENABLED")
stanza << "DefaultEnabled: " << O->second << "\n";
else


+ 4
- 0
doc/acquire-additional-files.txt View File

@@ -124,6 +124,10 @@ aren't accidentally used by front-ends:
type a front-end can't open transparently. This should always be
a temporary workaround through and a bug should be reported against
the front-end in question.
* KeepCompressedAs: The default value is a space separated list of
compression types supported by apt (see previous option) which is
sorted by the cost-value of the compression in ascending order,
except that cost=0 "compressions" (like uncompressed) are listed last.


# More examples


+ 1
- 1
test/integration/framework View File

@@ -1826,7 +1826,7 @@ listcurrentlistsdirectory() {
}
forallsupportedcompressors() {
for COMP in $(aptconfig dump 'APT::Compressor' --format '%f%n' | cut -d':' -f 5 | uniq); do
if [ -z "$COMP" -o "$COMP" = '.' -o "$COMP" = 'lz4' ]; then continue; fi
if [ -z "$COMP" -o "$COMP" = '.' ]; then continue; fi
"$@" "$COMP"
done
}


+ 8
- 7
test/integration/test-apt-acquire-additional-files View File

@@ -10,6 +10,7 @@ configarchitecture 'amd64'
# note that in --print-uri we talk about .xz because that is the default.
# This doesn't mean it is actually attempt to download it.
configcompression '.' 'gz'
LOWCOSTEXT='lz4'

buildsimplenativepackage 'foo' 'amd64' '1' 'unstable'

@@ -101,14 +102,14 @@ Get:2 http://localhost:${APTHTTPPORT} unstable/main amd64 Contents [$(stat -c%s
Get:3 http://localhost:${APTHTTPPORT} unstable/main all Contents [$(stat -c%s aptarchive/dists/unstable/main/Contents-all.gz) B]
Reading package lists..." aptget update

testequal "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64.gz" find rootdir/var/lib/apt/lists -name '*Contents-amd64*'
testequal "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all.gz" find rootdir/var/lib/apt/lists -name '*Contents-all*'
testequal "$(readfile Contents-amd64.gz Contents-all.gz)" aptget indextargets --format '$(FILENAME)' 'Created-By: Contents'
testsuccess cmp "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64.gz" 'aptarchive/dists/unstable/main/Contents-amd64.gz'
testsuccess cmp "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all.gz" 'aptarchive/dists/unstable/main/Contents-all.gz'
testequal "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64.$LOWCOSTEXT" find rootdir/var/lib/apt/lists -name '*Contents-amd64*'
testequal "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all.$LOWCOSTEXT" find rootdir/var/lib/apt/lists -name '*Contents-all*'
testequal "$(readfile Contents-amd64.$LOWCOSTEXT Contents-all.$LOWCOSTEXT)" aptget indextargets --format '$(FILENAME)' 'Created-By: Contents'
testequal "$(apthelper cat-file rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64.$LOWCOSTEXT)" apthelper cat-file 'aptarchive/dists/unstable/main/Contents-amd64.gz'
testempty apthelper cat-file rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all.$LOWCOSTEXT

rm ./rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64.gz
rm ./rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all.gz
rm ./rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64.$LOWCOSTEXT
rm ./rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all.$LOWCOSTEXT
testempty aptget indextargets --format '$(FILENAME)' 'Created-By: Contents'

msgmsg "Compressed Contents file"


+ 4
- 4
test/integration/test-apt-update-ims View File

@@ -24,15 +24,15 @@ runtest() {
rm -rf rootdir/var/lib/apt/lists/

local TEST="test${1:-success}"
$TEST aptget update $APTOPT
$TEST aptget update $APTOPT -o Debug::pkgAcquire::Worker=1
if [ "$1" = 'failure' ]; then
# accept the outdated Release file so we can check Hit behaviour
"test${2:-success}" aptget update -o Acquire::Min-ValidTime=9999999 $APTOPT
fi
listcurrentlistsdirectory > listsdir.lst
testsuccess grep '_Packages\(\.gz\)\?$' listsdir.lst
testsuccess grep '_Sources\(\.gz\)\?$' listsdir.lst
testsuccess grep '_Translation-en\(\.gz\)\?$' listsdir.lst
testsuccess grep '_Packages\(\.[0-9a-z]\+\)\?$' listsdir.lst
testsuccess grep '_Sources\(\.[0-9a-z]\+\)\?$' listsdir.lst
testsuccess grep '_Translation-en\(\.[0-9a-z]\+\)\?$' listsdir.lst

# ensure no leftovers in partial
testfailure ls 'rootdir/var/lib/apt/lists/partial/*'


+ 7
- 3
test/integration/test-apt-update-unauth View File

@@ -12,6 +12,7 @@ umask 022

setupenvironment
configarchitecture "i386"
configcompression '.' 'xz'

insertpackage 'unstable' 'foo' 'i386' '1.0'
insertsource 'unstable' 'foo' 'any' '1.0'
@@ -45,7 +46,7 @@ runtest() {
listcurrentlistsdirectory > lists.before

# update and ensure all is reverted on the hashsum failure
testfailure aptget update -o Debug::Acquire::Transaction=0 -o Debug::pkgAcquire::Auth=1 -o Debug::pkgAcquire::worker=0 -o Debug::acquire::http=0
testfailure aptget update -o Debug::Acquire::Transaction=1 -o Debug::pkgAcquire::Auth=1 -o Debug::pkgAcquire::worker=0 -o Debug::acquire::http=0

# ensure we have before what we have after
msgtest 'Check rollback on going from' 'unauth -> auth'
@@ -53,10 +54,13 @@ runtest() {
if cmp lists.before lists.after; then
msgpass
else
echo >&2
echo >&2 '### Output of previous apt-get update ###'
cat >&2 rootdir/tmp/testfailure.output
cat >&2 rootdir/tmp/testfailure.output || true
echo >&2 '### Changes in the lists-directory: ###'
diff -u >&2 lists.before lists.after
diff -u >&2 lists.before lists.after || true
echo >&2 '### Contents of the lists-directory: ###'
ls -l rootdir/var/lib/apt/lists rootdir/var/lib/apt/lists/partial || true
msgfail
fi



+ 5
- 4
test/integration/test-compressed-indexes View File

@@ -7,6 +7,7 @@ TESTDIR="$(readlink -f "$(dirname "$0")")"
setupenvironment
configcompression '.' $(aptconfig dump APT::Compressor --format '%t %v%n' | sed -n 's#^Extension \.\(.*\)$#\1#p')
configarchitecture 'i386'
LOWCOSTEXT='lz4'

buildsimplenativepackage 'testpkg' 'i386' '1.0'

@@ -35,10 +36,10 @@ testrun() {
! test -e rootdir/var/lib/apt/lists/*all_Packages || F=1
! test -e rootdir/var/lib/apt/lists/*_Sources || F=1
! test -e rootdir/var/lib/apt/lists/*_Translation-en || F=1
test -e rootdir/var/lib/apt/lists/*i386_Packages.${COMPRESS} || F=1
test -e rootdir/var/lib/apt/lists/*all_Packages.${COMPRESS} || F=1
test -e rootdir/var/lib/apt/lists/*_Sources.${COMPRESS} || F=1
test -e rootdir/var/lib/apt/lists/*_Translation-en.${COMPRESS} || F=1
test -e rootdir/var/lib/apt/lists/*i386_Packages.$LOWCOSTEXT || F=1
test -e rootdir/var/lib/apt/lists/*all_Packages.$LOWCOSTEXT || F=1
test -e rootdir/var/lib/apt/lists/*_Sources.$LOWCOSTEXT || F=1
test -e rootdir/var/lib/apt/lists/*_Translation-en.$LOWCOSTEXT || F=1
# there is no point in trying pdiff if we have compressed indexes
# as we can't patch compressed files (well, we can, but what is the point?)
! test -e rootdir/var/lib/apt/lists/*diff_Index || F=1


+ 2
- 1
test/integration/test-pdiff-usage View File

@@ -7,6 +7,7 @@ TESTDIR="$(readlink -f "$(dirname "$0")")"
setupenvironment
configarchitecture 'i386'
confighashes 'SHA1' 'SHA256'
LOWCOSTEXT='lz4'

buildaptarchive
setupflataptarchive
@@ -274,5 +275,5 @@ testcase() {
}
aptautotest_apt_update() { aptautotest_aptget_update "$@"; testsuccess test -e "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_Packages"; }
testcase -o Acquire::IndexTargets::deb::Packages::KeepCompressed=false
aptautotest_apt_update() { aptautotest_aptget_update "$@"; testsuccess test -e "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_Packages.gz"; }
aptautotest_apt_update() { aptautotest_aptget_update "$@"; testsuccess test -e "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_Packages.$LOWCOSTEXT"; }
testcase -o Acquire::IndexTargets::deb::Packages::KeepCompressed=true

Loading…
Cancel
Save