Browse Source

support compression and by-hash for .diff/Index files

In af81ab9030 by-hash got implemented as a
special compression type for our usual index files like Packages.
Missing in this scheme was the special .diff/Index index file containing
the info about individual patches for this index file. Deriving from the
index file class directly we inherent the compression handling
infrastructure and in this way also by-hash nearly for free.

Closes: #824926
tags/debian/1.3_rc2
David Kalnischkies 5 years ago
parent
commit
77e274f5ad
4 changed files with 151 additions and 114 deletions
  1. +47
    -41
      apt-pkg/acquire-item.cc
  2. +74
    -73
      apt-pkg/acquire-item.h
  3. +1
    -0
      ftparchive/writer.cc
  4. +29
    -0
      test/integration/test-pdiff-usage

+ 47
- 41
apt-pkg/acquire-item.cc View File

@@ -451,7 +451,11 @@ std::string pkgAcquire::Item::GetFinalFilename() const
}
std::string pkgAcqDiffIndex::GetFinalFilename() const
{
return GetFinalFileNameFromURI(GetDiffIndexURI(Target));
std::string const FinalFile = GetFinalFileNameFromURI(GetDiffIndexURI(Target));
// we don't want recompress, so lets keep whatever we got
if (CurrentCompressionExtension == "uncompressed")
return FinalFile;
return FinalFile + "." + CurrentCompressionExtension;
}
std::string pkgAcqIndex::GetFinalFilename() const
{
@@ -488,7 +492,10 @@ std::string pkgAcqIndex::GetMetaKey() const
}
std::string pkgAcqDiffIndex::GetMetaKey() const
{
return GetDiffIndexFileName(Target.MetaKey);
auto const metakey = GetDiffIndexFileName(Target.MetaKey);
if (CurrentCompressionExtension == "uncompressed")
return metakey;
return metakey + "." + CurrentCompressionExtension;
}
/*}}}*/
//pkgAcqTransactionItem::TransactionState and specialisations for child classes /*{{{*/
@@ -1971,44 +1978,33 @@ pkgAcqBaseIndex::~pkgAcqBaseIndex() {}
pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire * const Owner,
pkgAcqMetaClearSig * const TransactionManager,
IndexTarget const &Target)
: pkgAcqBaseIndex(Owner, TransactionManager, Target), d(NULL), diffs(NULL)
: pkgAcqIndex(Owner, TransactionManager, Target, true), d(NULL), diffs(NULL)
{
// FIXME: Magic number as an upper bound on pdiffs we will reasonably acquire
ExpectedAdditionalItems = 40;

Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);

Desc.Owner = this;
Desc.Description = GetDiffIndexFileName(Target.Description);
Desc.ShortDesc = Target.ShortDesc;
Desc.URI = GetDiffIndexURI(Target);

DestFile = GetPartialFileNameFromURI(Desc.URI);
CompressionExtensions.clear();
{
std::vector<std::string> types = APT::Configuration::getCompressionTypes();
if (types.empty() == false)
{
std::ostringstream os;
std::copy_if(types.begin(), types.end()-1, std::ostream_iterator<std::string>(os, " "), [&](std::string const type) {
if (type == "uncompressed")
return true;
return TransactionManager->MetaIndexParser->Exists(GetDiffIndexFileName(Target.MetaKey) + '.' + type);
});
os << *types.rbegin();
CompressionExtensions = os.str();
}
}
if (Target.Option(IndexTarget::COMPRESSIONTYPES).find("by-hash") != std::string::npos)
CompressionExtensions = "by-hash " + CompressionExtensions;
Init(GetDiffIndexURI(Target), GetDiffIndexFileName(Target.Description), Target.ShortDesc);

if(Debug)
std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;

QueueURI(Desc);
}
/*}}}*/
// AcqIndex::Custom600Headers - Insert custom request headers /*{{{*/
// ---------------------------------------------------------------------
/* The only header we use is the last-modified header. */
string pkgAcqDiffIndex::Custom600Headers() const
{
if (TransactionManager->LastMetaIndexParser != NULL)
return "\nIndex-File: true";

string const Final = GetFinalFilename();

if(Debug)
std::clog << "Custom600Header-IMS: " << Final << std::endl;

struct stat Buf;
if (stat(Final.c_str(),&Buf) != 0)
return "\nIndex-File: true";
return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime, false);
}
/*}}}*/
void pkgAcqDiffIndex::QueueOnIMSHit() const /*{{{*/
@@ -2039,7 +2035,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/
std::clog << "pkgAcqDiffIndex::ParseIndexDiff() " << IndexDiffFile
<< std::endl;

FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
FileFd Fd(IndexDiffFile, FileFd::ReadOnly, FileFd::Extension);
pkgTagFile TF(&Fd);
if (Fd.IsOpen() == false || Fd.Failed())
return false;
@@ -2411,7 +2407,9 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/
/*}}}*/
void pkgAcqDiffIndex::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)/*{{{*/
{
pkgAcqBaseIndex::Failed(Message,Cnf);
if (CommonFailed(GetDiffIndexURI(Target), GetDiffIndexFileName(Target.Description), Message, Cnf))
return;

Status = StatDone;
ExpectedAdditionalItems = 0;

@@ -2829,10 +2827,12 @@ pkgAcqIndexMergeDiffs::~pkgAcqIndexMergeDiffs() {}
// AcqIndex::AcqIndex - Constructor /*{{{*/
pkgAcqIndex::pkgAcqIndex(pkgAcquire * const Owner,
pkgAcqMetaClearSig * const TransactionManager,
IndexTarget const &Target)
IndexTarget const &Target, bool const Derived)
: pkgAcqBaseIndex(Owner, TransactionManager, Target), d(NULL), Stage(STAGE_DOWNLOAD),
CompressionExtensions(Target.Option(IndexTarget::COMPRESSIONTYPES))
{
if (Derived)
return;
Init(Target.URI, Target.Description, Target.ShortDesc);

if(_config->FindB("Debug::Acquire::Transaction", false) == true)
@@ -2938,7 +2938,8 @@ string pkgAcqIndex::Custom600Headers() const
}
/*}}}*/
// AcqIndex::Failed - getting the indexfile failed /*{{{*/
void pkgAcqIndex::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)
bool pkgAcqIndex::CommonFailed(std::string const &TargetURI, std::string const TargetDesc,
std::string const &Message, pkgAcquire::MethodConfig const * const Cnf)
{
pkgAcqBaseIndex::Failed(Message,Cnf);

@@ -2950,10 +2951,9 @@ void pkgAcqIndex::Failed(string const &Message,pkgAcquire::MethodConfig const *
CompressionExtensions = "by-hash " + CompressionExtensions;
else
CompressionExtensions = CurrentCompressionExtension + ' ' + CompressionExtensions;
Desc.Description = Target.Description;
Init(Target.URI, Desc.Description, Desc.ShortDesc);
Init(TargetURI, TargetDesc, Desc.ShortDesc);
Status = StatIdle;
return;
return true;
}

// authorisation matches will not be fixed by other compression types
@@ -2961,11 +2961,17 @@ void pkgAcqIndex::Failed(string const &Message,pkgAcquire::MethodConfig const *
{
if (CompressionExtensions.empty() == false)
{
Init(Target.URI, Desc.Description, Desc.ShortDesc);
Init(TargetURI, Desc.Description, Desc.ShortDesc);
Status = StatIdle;
return;
return true;
}
}
return false;
}
void pkgAcqIndex::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)
{
if (CommonFailed(Target.URI, Target.Description, Message, Cnf))
return;

if(Target.IsOptional && GetExpectedHashes().empty() && Stage == STAGE_DOWNLOAD)
Status = StatDone;


+ 74
- 73
apt-pkg/acquire-item.h View File

@@ -611,6 +611,79 @@ class APT_HIDDEN pkgAcqBaseIndex : public pkgAcqTransactionItem
virtual ~pkgAcqBaseIndex();
};
/*}}}*/
/** \brief An acquire item that is responsible for fetching an index {{{
* file (e.g., Packages or Sources).
*
* \sa pkgAcqDiffIndex, pkgAcqIndexDiffs, pkgAcqIndexTrans
*
* \todo Why does pkgAcqIndex have protected members?
*/
class APT_HIDDEN pkgAcqIndex : public pkgAcqBaseIndex
{
void * const d;

protected:

/** \brief The stages the method goes through
*
* The method first downloads the indexfile, then its decompressed (or
* copied) and verified
*/
enum AllStages {
STAGE_DOWNLOAD,
STAGE_DECOMPRESS_AND_VERIFY,
};
AllStages Stage;

/** \brief Handle what needs to be done when the download is done */
void StageDownloadDone(std::string const &Message);

/** \brief Handle what needs to be done when the decompression/copy is
* done
*/
void StageDecompressDone();

/** \brief If \b set, this partially downloaded file will be
* removed when the download completes.
*/
std::string EraseFileName;

/** \brief The compression-related file extensions that are being
* added to the downloaded file one by one if first fails (e.g., "gz bz2").
*/
std::string CompressionExtensions;

/** \brief The actual compression extension currently used */
std::string CurrentCompressionExtension;

/** \brief Do the changes needed to fetch via AptByHash (if needed) */
void InitByHashIfNeeded();

/** \brief Get the full pathname of the final file for the current URI */
virtual std::string GetFinalFilename() const APT_OVERRIDE;

virtual bool TransactionState(TransactionStates const state) APT_OVERRIDE;

public:
// Specialized action members
virtual void Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE;
virtual void Done(std::string const &Message, HashStringList const &Hashes,
pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE;
virtual std::string Custom600Headers() const APT_OVERRIDE;
virtual std::string DescURI() const APT_OVERRIDE {return Desc.URI;};
virtual std::string GetMetaKey() const APT_OVERRIDE;

pkgAcqIndex(pkgAcquire * const Owner, pkgAcqMetaClearSig * const TransactionManager,
IndexTarget const &Target, bool const Derived = false) APT_NONNULL(2, 3);
virtual ~pkgAcqIndex();

protected:
APT_HIDDEN void Init(std::string const &URI, std::string const &URIDesc,
std::string const &ShortDesc);
APT_HIDDEN bool CommonFailed(std::string const &TargetURI, std::string const TargetDesc,
std::string const &Message, pkgAcquire::MethodConfig const * const Cnf);
};
/*}}}*/
/** \brief An item that is responsible for fetching an index file of {{{
* package list diffs and starting the package list's download.
*
@@ -620,7 +693,7 @@ class APT_HIDDEN pkgAcqBaseIndex : public pkgAcqTransactionItem
*
* \sa pkgAcqIndexDiffs, pkgAcqIndex
*/
class APT_HIDDEN pkgAcqDiffIndex : public pkgAcqBaseIndex
class APT_HIDDEN pkgAcqDiffIndex : public pkgAcqIndex
{
void * const d;
std::vector<pkgAcqIndexMergeDiffs*> * diffs;
@@ -646,7 +719,6 @@ class APT_HIDDEN pkgAcqDiffIndex : public pkgAcqBaseIndex
virtual void Done(std::string const &Message, HashStringList const &Hashes,
pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE;
virtual std::string DescURI() const APT_OVERRIDE {return Target.URI + "Index";};
virtual std::string Custom600Headers() const APT_OVERRIDE;
virtual std::string GetMetaKey() const APT_OVERRIDE;

/** \brief Parse the Index file for a set of Packages diffs.
@@ -887,77 +959,6 @@ class APT_HIDDEN pkgAcqIndexDiffs : public pkgAcqBaseIndex
virtual ~pkgAcqIndexDiffs();
};
/*}}}*/
/** \brief An acquire item that is responsible for fetching an index {{{
* file (e.g., Packages or Sources).
*
* \sa pkgAcqDiffIndex, pkgAcqIndexDiffs, pkgAcqIndexTrans
*
* \todo Why does pkgAcqIndex have protected members?
*/
class APT_HIDDEN pkgAcqIndex : public pkgAcqBaseIndex
{
void * const d;

protected:

/** \brief The stages the method goes through
*
* The method first downloads the indexfile, then its decompressed (or
* copied) and verified
*/
enum AllStages {
STAGE_DOWNLOAD,
STAGE_DECOMPRESS_AND_VERIFY,
};
AllStages Stage;

/** \brief Handle what needs to be done when the download is done */
void StageDownloadDone(std::string const &Message);

/** \brief Handle what needs to be done when the decompression/copy is
* done
*/
void StageDecompressDone();

/** \brief If \b set, this partially downloaded file will be
* removed when the download completes.
*/
std::string EraseFileName;

/** \brief The compression-related file extensions that are being
* added to the downloaded file one by one if first fails (e.g., "gz bz2").
*/
std::string CompressionExtensions;

/** \brief The actual compression extension currently used */
std::string CurrentCompressionExtension;

/** \brief Do the changes needed to fetch via AptByHash (if needed) */
void InitByHashIfNeeded();

/** \brief Get the full pathname of the final file for the current URI */
virtual std::string GetFinalFilename() const APT_OVERRIDE;

virtual bool TransactionState(TransactionStates const state) APT_OVERRIDE;

public:
// Specialized action members
virtual void Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE;
virtual void Done(std::string const &Message, HashStringList const &Hashes,
pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE;
virtual std::string Custom600Headers() const APT_OVERRIDE;
virtual std::string DescURI() const APT_OVERRIDE {return Desc.URI;};
virtual std::string GetMetaKey() const APT_OVERRIDE;

pkgAcqIndex(pkgAcquire * const Owner, pkgAcqMetaClearSig * const TransactionManager,
IndexTarget const &Target) APT_NONNULL(2, 3);
virtual ~pkgAcqIndex();

private:
APT_HIDDEN void Init(std::string const &URI, std::string const &URIDesc,
std::string const &ShortDesc);
};
/*}}}*/
/** \brief An item that is responsible for fetching a package file. {{{
*
* If the package file already exists in the cache, nothing will be


+ 1
- 0
ftparchive/writer.cc View File

@@ -992,6 +992,7 @@ ReleaseWriter::ReleaseWriter(FileFd * const GivenOutput, string const &/*DB*/) :
AddPattern("Release");
AddPattern("Contents-*");
AddPattern("Index");
AddPattern("Index.*");
AddPattern("icons-*.tar");
AddPattern("icons-*.tar.*");
AddPattern("Components-*.yml");


+ 29
- 0
test/integration/test-pdiff-usage View File

@@ -35,6 +35,7 @@ wasmergeused() {
fi

testsuccess apt update "$@"
#apt update "$@" 2>&1 | tee rootdir/tmp/testsuccess.output

msgtest 'No intermediate patch files' 'still exist'
local EDS="$(find rootdir/var/lib/apt/lists -name '*.ed' -o -name '*.ed.*')"
@@ -116,6 +117,34 @@ SHA256-Download:
testsuccessequal "$(cat "${PKGFILE}-new")
" aptcache show apt newstuff

msgmsg "Testcase: apply with compressed Index and one patch: $*"
cp "${PKGFILE}-new" aptarchive/Packages
compressfile 'aptarchive/Packages'
cat "$PATCHINDEX" | gzip > "${PATCHINDEX}.gz"
generatereleasefiles '+1hour'
signreleasefiles
find aptarchive -name 'Packages*' -type f -delete
rm "$PATCHINDEX"
rm -rf rootdir/var/lib/apt/lists
cp -a rootdir/var/lib/apt/lists-bak rootdir/var/lib/apt/lists
wasmergeused "$@"
testnopackage oldstuff
testsuccessequal "$(cat "${PKGFILE}-new")
" aptcache show apt newstuff

msgmsg "Testcase: apply with by-hash compressed Index and one patch: $*"
local NORMAL="$(readlink -f "./aptarchive/Packages.diff")"
local BYHASH="${NORMAL}/by-hash/SHA256"
mkdir -p "${BYHASH}"
find "${NORMAL}/" -maxdepth 1 -name "Index*" -exec mv '{}' "$BYHASH" \;
ln -s "${BYHASH}/Index.gz" "${BYHASH}/$(sha256sum "${BYHASH}/Index.gz" | cut -f1 -d' ')"
rm -rf rootdir/var/lib/apt/lists
cp -a rootdir/var/lib/apt/lists-bak rootdir/var/lib/apt/lists
wasmergeused "$@" -o Acquire::By-Hash=force
testnopackage oldstuff
testsuccessequal "$(cat "${PKGFILE}-new")
" aptcache show apt newstuff

msgmsg "Testcase: SHA1-only patches are not used: $*"
find aptarchive -name 'Packages*' -type f -delete
cp "${PKGFILE}-new" aptarchive/Packages


Loading…
Cancel
Save