Browse Source

allow individual targets to be kept compressed

There is an option to keep all targets (Packages, Sources, …) compressed
for a while now, but the all-or-nothing approach is a bit limited for
our purposes with additional targets as some of them are very big
(Contents) and rarely used in comparison, so keeping them compressed by
default can make sense, while others are still unpacked.

Most interesting is the copy-change maybe: Copy is used by the acquire
system as an uncompressor and it is hence expected that it returns the
hashes for the "output", not the input. Now, in the case of keeping a
file compressed, the output is never written to disk, but generated in
memory and we should still validated it, so for compressed files copy is
expected to return the hashes of the uncompressed file. We used to use
the config option to enable on-the-fly decompress in the method, but in
reality copy is never used in a way where it shouldn't decompress a
compressed file to get its hashes, so we can save us the trouble of
sending this information to the method and just do it always.
tags/debian/1.1.exp9
David Kalnischkies 6 years ago
parent
commit
653ef26c70
7 changed files with 39 additions and 20 deletions
  1. +7
    -7
      apt-pkg/acquire-item.cc
  2. +5
    -2
      apt-pkg/deb/debmetaindex.cc
  3. +3
    -2
      apt-pkg/indexfile.cc
  4. +4
    -1
      apt-pkg/indexfile.h
  5. +2
    -6
      methods/copy.cc
  6. +17
    -1
      test/integration/test-apt-acquire-additional-files
  7. +1
    -1
      test/integration/test-compressed-indexes

+ 7
- 7
apt-pkg/acquire-item.cc View File

@@ -80,18 +80,18 @@ static std::string GetFinalFileNameFromURI(std::string const &uri) /*{{{*/
return _config->FindDir("Dir::State::lists") + URItoFileName(uri);
}
/*}}}*/
static std::string GetCompressedFileName(std::string const &URI, std::string const &Name, std::string const &Ext) /*{{{*/
static std::string GetCompressedFileName(IndexTarget const &Target, std::string const &Name, std::string const &Ext) /*{{{*/
{
if (Ext.empty() || Ext == "uncompressed")
return Name;

// do not reverify cdrom sources as apt-cdrom may rewrite the Packages
// file when its doing the indexcopy
if (URI.substr(0,6) == "cdrom:")
if (Target.URI.substr(0,6) == "cdrom:")
return Name;

// adjust DestFile if its compressed on disk
if (_config->FindB("Acquire::GzipIndexes",false) == true)
if (Target.KeepCompressed == true)
return Name + '.' + Ext;
return Name;
}
@@ -269,7 +269,7 @@ std::string pkgAcqDiffIndex::GetFinalFilename() const
std::string pkgAcqIndex::GetFinalFilename() const
{
std::string const FinalFile = GetFinalFileNameFromURI(Target.URI);
return GetCompressedFileName(Target.URI, FinalFile, CurrentCompressionExtension);
return GetCompressedFileName(Target, FinalFile, CurrentCompressionExtension);
}
std::string pkgAcqMetaSig::GetFinalFilename() const
{
@@ -2456,7 +2456,7 @@ void pkgAcqIndex::ReverifyAfterIMS()
{
// update destfile to *not* include the compression extension when doing
// a reverify (as its uncompressed on disk already)
DestFile = GetCompressedFileName(Target.URI, GetPartialFileNameFromURI(Target.URI), CurrentCompressionExtension);
DestFile = GetCompressedFileName(Target, GetPartialFileNameFromURI(Target.URI), CurrentCompressionExtension);

// copy FinalFile into partial/ so that we check the hash again
string FinalFile = GetFinalFilename();
@@ -2532,8 +2532,8 @@ void pkgAcqIndex::StageDownloadDone(string const &Message, HashStringList const
return;
}

// If we have compressed indexes enabled, queue for hash verification
if (_config->FindB("Acquire::GzipIndexes",false))
// If we want compressed indexes, just copy in place for hash verification
if (Target.KeepCompressed == true)
{
DestFile = GetPartialFileNameFromURI(Target.URI + '.' + CurrentCompressionExtension);
EraseFileName = "";


+ 5
- 2
apt-pkg/deb/debmetaindex.cc View File

@@ -123,6 +123,7 @@ static void GetIndexTargetsFor(char const * const Type, std::string const &URI,
std::string const Release = (Dist == "/") ? "" : Dist;
std::string const Site = ::URI::ArchiveOnly(URI);

bool const GzipIndex = _config->FindB("Acquire::GzipIndexes", false);
for (std::vector<debReleaseIndexPrivate::debSectionEntry>::const_iterator E = entries.begin(); E != entries.end(); ++E)
{
for (std::vector<std::string>::const_iterator T = E->Targets.begin(); T != E->Targets.end(); ++T)
@@ -131,7 +132,8 @@ static void GetIndexTargetsFor(char const * const Type, std::string const &URI,
std::string const tplMetaKey = APT_T_CONFIG(flatArchive ? "flatMetaKey" : "MetaKey");
std::string const tplShortDesc = APT_T_CONFIG("ShortDescription");
std::string const tplLongDesc = APT_T_CONFIG(flatArchive ? "flatDescription" : "Description");
bool const IsOptional = _config->FindB(std::string("APT::Acquire::Targets::deb-src::") + *T + "::Optional", true);
bool const IsOptional = _config->FindB(std::string("APT::Acquire::Targets::") + Type + "::" + *T + "::Optional", true);
bool const KeepCompressed = _config->FindB(std::string("APT::Acquire::Targets::") + Type + "::" + *T + "::KeepCompressed", GzipIndex);
#undef APT_T_CONFIG
if (tplMetaKey.empty())
continue;
@@ -173,6 +175,7 @@ static void GetIndexTargetsFor(char const * const Type, std::string const &URI,
LongDesc,
Options.find("BASE_URI")->second + MetaKey,
IsOptional,
KeepCompressed,
Options
);
IndexTargets.push_back(Target);
@@ -413,7 +416,7 @@ bool debReleaseIndex::parseSumData(const char *&Start, const char *End, /*{{{*/
bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll)/*{{{*/
{
std::vector<IndexTarget> const targets = GetIndexTargets();
#define APT_TARGET(X) IndexTarget("", X, MetaIndexInfo(X), MetaIndexURI(X), false, std::map<std::string,std::string>())
#define APT_TARGET(X) IndexTarget("", X, MetaIndexInfo(X), MetaIndexURI(X), false, false, std::map<std::string,std::string>())
pkgAcqMetaClearSig * const TransactionManager = new pkgAcqMetaClearSig(Owner,
APT_TARGET("InRelease"), APT_TARGET("Release"), APT_TARGET("Release.gpg"),
targets, this);


+ 3
- 2
apt-pkg/indexfile.cc View File

@@ -115,8 +115,9 @@ APT_DEPRECATED std::string pkgIndexFile::LanguageCode() {
// IndexTarget - Constructor /*{{{*/
IndexTarget::IndexTarget(std::string const &MetaKey, std::string const &ShortDesc,
std::string const &LongDesc, std::string const &URI, bool const IsOptional,
std::map<std::string, std::string> const &Options) :
URI(URI), Description(LongDesc), ShortDesc(ShortDesc), MetaKey(MetaKey), IsOptional(IsOptional), Options(Options)
bool const KeepCompressed, std::map<std::string, std::string> const &Options) :
URI(URI), Description(LongDesc), ShortDesc(ShortDesc), MetaKey(MetaKey),
IsOptional(IsOptional), KeepCompressed(KeepCompressed), Options(Options)
{
}
/*}}}*/


+ 4
- 1
apt-pkg/indexfile.h View File

@@ -60,6 +60,9 @@ class IndexTarget /*{{{*/
/** \brief Is it okay if the file isn't found in the meta index */
bool IsOptional;

/** \brief If the file is downloaded compressed, do not unpack it */
bool KeepCompressed;

/** \brief options with which this target was created
Prefer the usage of #Option if at all possible.
Beware: Not all of these options are intended for public use */
@@ -67,7 +70,7 @@ class IndexTarget /*{{{*/

IndexTarget(std::string const &MetaKey, std::string const &ShortDesc,
std::string const &LongDesc, std::string const &URI, bool const IsOptional,
std::map<std::string, std::string> const &Options);
bool const KeepCompressed, std::map<std::string, std::string> const &Options);

enum OptionKeys {
SITE,


+ 2
- 6
methods/copy.cc View File

@@ -38,11 +38,7 @@ class CopyMethod : public pkgAcqMethod
void CopyMethod::CalculateHashes(FetchItem const * const Itm, FetchResult &Res)
{
Hashes Hash(Itm->ExpectedHashes);
FileFd::CompressMode CompressMode = FileFd::None;
if (_config->FindB("Acquire::GzipIndexes", false) == true)
CompressMode = FileFd::Extension;

FileFd Fd(Res.Filename, FileFd::ReadOnly, CompressMode);
FileFd Fd(Res.Filename, FileFd::ReadOnly, FileFd::Extension);
Hash.AddFD(Fd);
Res.TakeHashes(Hash);
}
@@ -53,7 +49,7 @@ void CopyMethod::CalculateHashes(FetchItem const * const Itm, FetchResult &Res)
bool CopyMethod::Fetch(FetchItem *Itm)
{
// this ensures that relative paths work in copy
std::string File = Itm->Uri.substr(Itm->Uri.find(':')+1);
std::string const File = Itm->Uri.substr(Itm->Uri.find(':')+1);

// Stat the file and send a start message
struct stat Buf;


+ 17
- 1
test/integration/test-apt-acquire-additional-files View File

@@ -57,7 +57,23 @@ testequal 'rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents
testequal "$(readlink -f ./rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64)" aptget files --format '$(FILENAME)' 'Created-By: Contents'
testsuccess cmp 'rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64' 'aptarchive/dists/unstable/main/Contents-amd64'

# no automatic uncompress based on the name please,
rm ./rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64
testempty aptget files --format '$(FILENAME)' 'Created-By: Contents'

# if we asked for keeping it compressed, keep it
echo 'APT::Acquire::Targets::deb::Contents::KeepCompressed "true";' >> rootdir/etc/apt/apt.conf.d/content-target.conf
testsuccessequal "Hit:1 http://localhost:8080 unstable InRelease
Get:2 http://localhost:8080 unstable/main amd64 Contents [$(stat -c%s aptarchive/dists/unstable/main/Contents-amd64.gz) B]
Reading package lists..." aptget update

testequal 'rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz' find rootdir/var/lib/apt/lists -name '*Contents*'
testequal "$(readlink -f ./rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz)" aptget files --format '$(FILENAME)' 'Created-By: Contents'
testsuccess cmp 'rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz' 'aptarchive/dists/unstable/main/Contents-amd64.gz'

rm ./rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz
testempty aptget files --format '$(FILENAME)' 'Created-By: Contents'

# and no automatic uncompress based on the name please,
# only if we downloaded a compressed file, but target was uncompressed
cat > rootdir/etc/apt/apt.conf.d/content-target.conf <<EOF
APT::Acquire::Targets::deb::Contents {


+ 1
- 1
test/integration/test-compressed-indexes View File

@@ -60,7 +60,7 @@ testrun() {
msgtest 'Check if package is downloadable'
cd downloaded
testsuccess --nomsg aptget download testpkg
msgtest '\tdeb file is present'; testsuccess --nomsg test -f testpkg_1.0_i386.deb
msgtest 'deb file is present'; testsuccess --nomsg test -f testpkg_1.0_i386.deb
rm -f testpkg_1.0_i386.deb
cd - >/dev/null
testsuccessequal 'Reading package lists...


Loading…
Cancel
Save