Browse Source

add by-hash sources.list option and document all of by-hash

This changes the semantics of the option (which is renamed too) to be a
yes/no value with the special additional value "force" as this allows
by-hash to be disabled even if the repository indicates it would be
supported and is more in line with our other yes/no options like pdiff
which disable themselves if no support can be detected.

The feature wasn't documented so far and hasn't reached a (un)stable
release yet, so changing it without trying too hard to keep
compatibility seems okay.
tags/debian/1.1.exp13
David Kalnischkies 6 years ago
parent
commit
24e8f24e1e
10 changed files with 90 additions and 27 deletions
  1. +5
    -8
      apt-pkg/acquire-item.cc
  2. +18
    -3
      apt-pkg/deb/debmetaindex.cc
  3. +1
    -1
      apt-pkg/deb/debmetaindex.h
  4. +1
    -0
      apt-pkg/indexfile.cc
  5. +1
    -0
      apt-pkg/indexfile.h
  6. +1
    -0
      apt-pkg/sourcelist.cc
  7. +3
    -0
      doc/acquire-additional-files.txt
  8. +10
    -0
      doc/apt.conf.5.xml
  9. +16
    -0
      doc/sources.list.5.xml
  10. +34
    -15
      test/integration/test-apt-by-hash-update

+ 5
- 8
apt-pkg/acquire-item.cc View File

@@ -1247,7 +1247,7 @@ string pkgAcqMetaClearSig::Custom600Headers() const
return Header;
}
/*}}}*/
bool pkgAcqMetaClearSig::VerifyDone(std::string const &Message,
bool pkgAcqMetaClearSig::VerifyDone(std::string const &Message, /*{{{*/
pkgAcquire::MethodConfig const * const Cnf)
{
Item::VerifyDone(Message, Cnf);
@@ -1257,6 +1257,7 @@ bool pkgAcqMetaClearSig::VerifyDone(std::string const &Message,

return true;
}
/*}}}*/
// pkgAcqMetaClearSig::Done - We got a file /*{{{*/
void pkgAcqMetaClearSig::Done(std::string const &Message,
HashStringList const &Hashes,
@@ -2480,13 +2481,9 @@ void pkgAcqIndex::Init(string const &URI, string const &URIDesc,
// AcqIndex::AdjustForByHash - modify URI for by-hash support /*{{{*/
void pkgAcqIndex::InitByHashIfNeeded()
{
// TODO:
// - (maybe?) add support for by-hash into the sources.list as flag
// - make apt-ftparchive generate the hashes (and expire?)
std::string HostKnob = "APT::Acquire::" + ::URI(Desc.URI).Host + "::By-Hash";
if(_config->FindB("APT::Acquire::By-Hash", false) == true ||
_config->FindB(HostKnob, false) == true ||
TransactionManager->MetaIndexParser->GetSupportsAcquireByHash())
std::string const useByHash = Target.Option(IndexTarget::BY_HASH);
if(useByHash == "force" || (StringToBool(useByHash) == true &&
TransactionManager->MetaIndexParser->GetSupportsAcquireByHash()))
{
HashStringList const Hashes = GetExpectedHashes();
if(Hashes.usable())


+ 18
- 3
apt-pkg/deb/debmetaindex.cc View File

@@ -39,6 +39,7 @@ class APT_HIDDEN debReleaseIndexPrivate /*{{{*/
std::vector<std::string> Architectures;
std::vector<std::string> Languages;
bool UsePDiffs;
std::string UseByHash;
};

std::vector<debSectionEntry> DebEntries;
@@ -149,6 +150,7 @@ static void GetIndexTargetsFor(char const * const Type, std::string const &URI,
bool const KeepCompressed = APT_T_CONFIG_BOOL("KeepCompressed", GzipIndex);
bool const DefaultEnabled = APT_T_CONFIG_BOOL("DefaultEnabled", true);
bool const UsePDiffs = APT_T_CONFIG_BOOL("PDiffs", E->UsePDiffs);
std::string const UseByHash = APT_T_CONFIG_STR("By-Hash", E->UseByHash);
std::string const CompressionTypes = APT_T_CONFIG_STR("CompressionTypes", DefCompressionTypes);
#undef APT_T_CONFIG_BOOL
#undef APT_T_CONFIG_STR
@@ -245,6 +247,7 @@ static void GetIndexTargetsFor(char const * const Type, std::string const &URI,
Options.insert(std::make_pair("TARGET_OF", Type));
Options.insert(std::make_pair("CREATED_BY", *T));
Options.insert(std::make_pair("PDIFFS", UsePDiffs ? "yes" : "no"));
Options.insert(std::make_pair("BY_HASH", UseByHash));
Options.insert(std::make_pair("DEFAULTENABLED", DefaultEnabled ? "yes" : "no"));
Options.insert(std::make_pair("COMPRESSIONTYPES", CompressionTypes));
Options.insert(std::make_pair("SOURCESENTRY", E->sourcesEntry));
@@ -286,12 +289,12 @@ void debReleaseIndex::AddComponent(std::string const &sourcesEntry, /*{{{*/
std::vector<std::string> const &Targets,
std::vector<std::string> const &Architectures,
std::vector<std::string> Languages,
bool const usePDiffs)
bool const usePDiffs, std::string const &useByHash)
{
if (Languages.empty() == true)
Languages.push_back("none");
debReleaseIndexPrivate::debSectionEntry const entry = {
sourcesEntry, Name, Targets, Architectures, Languages, usePDiffs
sourcesEntry, Name, Targets, Architectures, Languages, usePDiffs, useByHash
};
if (isSrc)
d->DebSrcEntries.push_back(entry);
@@ -822,6 +825,17 @@ class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type /*{{{*/
UsePDiffs = StringToBool(opt->second);
}

std::string UseByHash = _config->Find("APT::Acquire::By-Hash", "yes");
UseByHash = _config->Find("Acquire::By-Hash", UseByHash);
{
std::string const host = ::URI(URI).Host;
UseByHash = _config->Find("APT::Acquire::" + host + "::By-Hash", UseByHash);
UseByHash = _config->Find("Acquire::" + host + "::By-Hash", UseByHash);
std::map<std::string, std::string>::const_iterator const opt = Options.find("by-hash");
if (opt != Options.end())
UseByHash = opt->second;
}

auto const entry = Options.find("sourceslist-entry");
Deb->AddComponent(
entry->second,
@@ -830,7 +844,8 @@ class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type /*{{{*/
mytargets,
parsePlusMinusOptions("arch", Options, APT::Configuration::getArchitectures()),
parsePlusMinusOptions("lang", Options, APT::Configuration::getLanguages(true)),
UsePDiffs
UsePDiffs,
UseByHash
);

if (Deb->SetTrusted(GetTriStateOption(Options, "trusted")) == false ||


+ 1
- 1
apt-pkg/deb/debmetaindex.h View File

@@ -64,7 +64,7 @@ class APT_HIDDEN debReleaseIndex : public metaIndex
std::vector<std::string> const &Targets,
std::vector<std::string> const &Architectures,
std::vector<std::string> Languages,
bool const usePDiffs);
bool const usePDiffs, std::string const &useByHash);
};

#endif

+ 1
- 0
apt-pkg/indexfile.cc View File

@@ -148,6 +148,7 @@ std::string IndexTarget::Option(OptionKeys const EnumKey) const /*{{{*/
APT_CASE(DEFAULTENABLED);
APT_CASE(COMPRESSIONTYPES);
APT_CASE(SOURCESENTRY);
APT_CASE(BY_HASH);
#undef APT_CASE
case FILENAME: return _config->FindDir("Dir::State::lists") + URItoFileName(URI);
case EXISTING_FILENAME:


+ 1
- 0
apt-pkg/indexfile.h View File

@@ -89,6 +89,7 @@ class IndexTarget /*{{{*/
COMPRESSIONTYPES,
DEFAULTENABLED,
SOURCESENTRY,
BY_HASH,
};
std::string Option(OptionKeys const Key) const;
bool OptionBool(OptionKeys const Key) const;


+ 1
- 0
apt-pkg/sourcelist.cc View File

@@ -108,6 +108,7 @@ bool pkgSourceList::Type::ParseStanza(vector<metaIndex *> &List, /*{{{*/
mapping.insert(std::make_pair("Valid-Until-Max", std::make_pair("valid-until-max", false)));
mapping.insert(std::make_pair("Signed-By", std::make_pair("signed-by", false)));
mapping.insert(std::make_pair("PDiffs", std::make_pair("pdiffs", false)));
mapping.insert(std::make_pair("By-Hash", std::make_pair("by-hash", false)));

for (std::map<char const * const, std::pair<char const * const, bool> >::const_iterator m = mapping.begin(); m != mapping.end(); ++m)
if (Tags.Exists(m->first))


+ 3
- 0
doc/acquire-additional-files.txt View File

@@ -114,6 +114,9 @@ aren't accidentally used by frontends:
Defaults to the value of Acquire::PDiffs which is true by default.
Can be overridden per-source by the sources.list option of the same
name. See the documentation for both of these for details.
* By-Hash: controls if apt will try to use an URI constructed from
a hashsum of the file to download. See the documentation for config
option Acquire::By-Hash and sources.list option By-Hash for details.
* CompressionTypes: The default value is a space separated list of
compression types supported by apt (see Acquire::CompressionTypes).
You can set this option to prevent apt from downloading a compression


+ 10
- 0
doc/apt.conf.5.xml View File

@@ -347,6 +347,16 @@ DPkg::Pre-Install-Pkgs {"/usr/sbin/dpkg-preconfigure --apt";};
</para></listitem>
</varlistentry>

<varlistentry><term><option>By-Hash</option></term>
<listitem><para>Try to download indexes via an URI constructed from a
hashsum of the expected file rather than downloaded via a well-known
stable filename. True by default, but automatically disabled if the
source indicates no support for it. Usage can be forced with the special
value "force". Preferably, this can be set for specific &sources-list; entries
or index files by using the <option>By-Hash</option> option there.
</para></listitem>
</varlistentry>

<varlistentry><term><option>Queue-Mode</option></term>
<listitem><para>Queuing mode; <literal>Queue-Mode</literal> can be one of <literal>host</literal> or
<literal>access</literal> which determines how APT parallelizes outgoing


+ 16
- 0
doc/sources.list.5.xml View File

@@ -239,6 +239,22 @@ deb-src [ option1=value1 option2=value2 ] uri suite [component1] [component2] [.
<literal>yes</literal>.
</para></listitem>

<listitem><para><option>By-Hash</option> (<option>by-hash</option>)
can have the value "yes", "no" or "force" and controls if APT
should try to acquire indexes via an URI constructed from a
hashsum of the expected file instead of using the well-known
stable filename of the index. Using this can avoid hashsum
mismatches, but requires a supporting mirror. The value
"yes"/"no" activates/disables the use of this feature if this
source indicates support for it, while "force" will enable the
feature regardless of what the source indicates.
Defaults to the value of the option of the same name for a
specific index file defined in the
<option>Acquire::IndexTargets</option> scope, which itself
defaults to the value of configuration option
<option>Acquire::By-Hash</option> which defaults to
<literal>yes</literal>.
</para></listitem>

</itemizedlist>



+ 34
- 15
test/integration/test-apt-by-hash-update View File

@@ -5,8 +5,9 @@ TESTDIR=$(readlink -f $(dirname $0))
. $TESTDIR/framework

setupenvironment
configarchitecture "i386"
configarchitecture 'i386'
confighashes 'SHA512'
configcompression '.' 'gz'

insertpackage 'unstable' 'foo' 'all' '1.0'

@@ -24,29 +25,47 @@ mkdir -p aptarchive/dists/unstable/main/source/by-hash/SHA512
ln -s ../../Sources.gz $(sha512sum ../../Sources.gz|cut -f1 -d' ')
)

# we moved the Packages file away, normal update won't work
testfailure aptget update

# ensure we do not know about "foo"
testfailureequal "Reading package lists...
ensureitsbroken() {
rm -rf rootdir/var/lib/apt/lists
# we moved the Packages file away, normal update won't work
testfailure aptget update "$@"
# ensure we do not know about "foo"
testfailureequal "Reading package lists...
Building dependency tree...
E: Unable to locate package foo" aptget install -q -s foo
# ensure we can apt-get update by hash
testsuccess aptget update -o APT::Acquire::By-Hash=1 -o Acquire::Languages=none
}
ensureitsbroken
ensureitsbroken -o Acquire::By-Hash=1

ensureitworks() {
rm -rf rootdir/var/lib/apt/lists
testsuccess aptget update -o Acquire::Languages=none "$@"
testsuccessequal "Inst foo (1.0 unstable [all])
Conf foo (1.0 unstable [all])" aptget install -qq -s foo
}
ensureitworks
msgmsg 'Test by-hash via' 'config option'
ensureitworks -o Acquire::By-Hash=force

msgmsg 'Test by-hash via' 'release option'
cp -a aptarchive/dists aptarchive/dists.bak
# add magic string to Release file ...
MAGIC="Acquire-By-Hash: true"
sed -i "s#Suite: unstable#Suite: unstable\n$MAGIC#" aptarchive/dists/unstable/Release
sed -i '/^Suite: / a \
Acquire-By-Hash: yes' aptarchive/dists/unstable/Release
signreleasefiles
# ... and verify that it fetches by hash now
rm -rf rootdir/var/lib/apt/lists
testsuccess aptget update -o Acquire::Languages=none
ensureitworks
ensureitsbroken -o Acquire::By-Hash=0

msgmsg 'Test by-hash via' 'sources option'
sed -i "s#^\(deb\(-src\)\?\) #\1 [by-hash=yes] #" rootdir/etc/apt/sources.list.d/*
ensureitworks
#ensureitsbroken -o Acquire::By-Hash=0

rm -rf aptarchive/dists
cp -a aptarchive/dists.bak aptarchive/dists
#ensureitworks -o Acquire::By-Hash=force
ensureitsbroken -o Acquire::By-Hash=1
ensureitsbroken -o Acquire::By-Hash=0

sed -i "s#^\(deb\(-src\)\?\) \[by-hash=yes\] #\1 [by-hash=force] #" rootdir/etc/apt/sources.list.d/*
ensureitworks
#ensureitsbroken -o Acquire::By-Hash=0

Loading…
Cancel
Save