Browse Source

allow multivalue fields in deb822 sources to be folded

The documentation said "spaces", but there is no real reason to be so
strict and only allow spaces to separate values as that only leads to
very long lines if e.g. multiple URIs are specified which are again hard
to deal with from a user PoV which the deb822 format is supposed to
avoid. It also deals with multiple consecutive spaces and strange things
like tabs users will surely end up using in the real world.

The old behviour on encountering folded lines is the generation of URIs
which end up containing all these whitespace characters which tends to
mess really bad with output and further processing.

Closes: 881875
tags/debian/1.6_alpha6
David Kalnischkies 3 years ago
parent
commit
7fba4e0df1
3 changed files with 143 additions and 47 deletions
  1. +46
    -44
      apt-pkg/sourcelist.cc
  2. +5
    -3
      doc/sources.list.5.xml
  3. +92
    -0
      test/integration/test-apt-sources-deb822

+ 46
- 44
apt-pkg/sourcelist.cc View File

@@ -43,6 +43,17 @@ static pkgSourceList::Type *ItmList[10];
pkgSourceList::Type **pkgSourceList::Type::GlobalList = ItmList;
unsigned long pkgSourceList::Type::GlobalListLen = 0;

static std::vector<std::string> FindMultiValue(pkgTagSection &Tags, char const *const Field) /*{{{*/
{
auto values = Tags.FindS(Field);
// we ignore duplicate spaces by removing empty values
std::replace_if(values.begin(), values.end(), isspace_ascii, ' ');
auto vect = VectorizeString(values, ' ');
vect.erase(std::remove_if(vect.begin(), vect.end(), [](std::string const &s) { return s.empty(); }), vect.end());
return vect;
}
/*}}}*/

// Type::Type - Constructor /*{{{*/
// ---------------------------------------------------------------------
/* Link this to the global list of items*/
@@ -115,11 +126,13 @@ bool pkgSourceList::Type::ParseStanza(vector<metaIndex *> &List, /*{{{*/
for (std::map<char const * const, std::pair<char const * const, bool> >::const_iterator m = mapping.begin(); m != mapping.end(); ++m)
if (Tags.Exists(m->first))
{
std::string option = Tags.FindS(m->first);
// for deb822 the " " is the delimiter, but the backend expects ","
if (m->second.second == true)
std::replace(option.begin(), option.end(), ' ', ',');
Options[m->second.first] = option;
if (m->second.second)
{
auto const values = FindMultiValue(Tags, m->first);
Options[m->second.first] = APT::String::Join(values, ",");
}
else
Options[m->second.first] = Tags.FindS(m->first);
}

{
@@ -129,37 +142,34 @@ bool pkgSourceList::Type::ParseStanza(vector<metaIndex *> &List, /*{{{*/
}

// now create one item per suite/section
string Suite = Tags.FindS("Suites");
Suite = SubstVar(Suite,"$(ARCH)",_config->Find("APT::Architecture"));
string const Component = Tags.FindS("Components");
string const URIS = Tags.FindS("URIs");
std::vector<std::string> const list_uris = VectorizeString(URIS, ' ');
std::vector<std::string> const list_suite = VectorizeString(Suite, ' ');
std::vector<std::string> const list_comp = VectorizeString(Component, ' ');
auto const list_uris = FindMultiValue(Tags, "URIs");
auto const list_comp = FindMultiValue(Tags, "Components");
auto list_suite = FindMultiValue(Tags, "Suites");
{
auto const nativeArch = _config->Find("APT::Architecture");
std::transform(list_suite.begin(), list_suite.end(), list_suite.begin(),
[&](std::string const &suite) { return SubstVar(suite, "$(ARCH)", nativeArch); });
}

if (list_uris.empty())
// TRANSLATOR: %u is a line number, the first %s is a filename of a file with the extension "second %s" and the third %s is a unique identifier for bugreports
return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "URI");

for (std::vector<std::string>::const_iterator U = list_uris.begin();
U != list_uris.end(); ++U)
if (list_suite.empty())
return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "Suite");

for (auto URI : list_uris)
{
std::string URI = *U;
if (U->empty() || FixupURI(URI) == false)
if (FixupURI(URI) == false)
return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "URI parse");

if (list_suite.empty())
return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "Suite");

for (std::vector<std::string>::const_iterator S = list_suite.begin();
S != list_suite.end(); ++S)
for (auto const &S : list_suite)
{
if (S->empty() == false && (*S)[S->size() - 1] == '/')
if (likely(S.empty() == false) && S[S.size() - 1] == '/')
{
if (list_comp.empty() == false)
return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "absolute Suite Component");
if (CreateItem(List, URI, *S, "", Options) == false)
if (CreateItem(List, URI, S, "", Options) == false)
return false;
}
else
@@ -167,14 +177,9 @@ bool pkgSourceList::Type::ParseStanza(vector<metaIndex *> &List, /*{{{*/
if (list_comp.empty())
return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "Component");

for (std::vector<std::string>::const_iterator C = list_comp.begin();
C != list_comp.end(); ++C)
{
if (CreateItem(List, URI, *S, *C, Options) == false)
{
for (auto const &C : list_comp)
if (CreateItem(List, URI, S, C, Options) == false)
return false;
}
}
}
}
}
@@ -433,20 +438,17 @@ bool pkgSourceList::ParseFileDeb822(string const &File)
if(Tags.Exists("Types") == false)
return _error->Error(_("Malformed stanza %u in source list %s (type)"),i,File.c_str());

string const types = Tags.FindS("Types");
std::vector<std::string> const list_types = VectorizeString(types, ' ');
for (std::vector<std::string>::const_iterator I = list_types.begin();
I != list_types.end(); ++I)
for (auto const &type : FindMultiValue(Tags, "Types"))
{
Type *Parse = Type::GetType((*I).c_str());
if (Parse == 0)
{
_error->Error(_("Type '%s' is not known on stanza %u in source list %s"), (*I).c_str(),i,Fd.Name().c_str());
return false;
}
if (!Parse->ParseStanza(SrcList, Tags, i, Fd))
return false;
Type *Parse = Type::GetType(type.c_str());
if (Parse == 0)
{
_error->Error(_("Type '%s' is not known on stanza %u in source list %s"), type.c_str(), i, Fd.Name().c_str());
return false;
}
if (!Parse->ParseStanza(SrcList, Tags, i, Fd))
return false;
}
}
return true;


+ 5
- 3
doc/sources.list.5.xml View File

@@ -101,9 +101,11 @@

Options have the same syntax as every other field: A fieldname separated by
a colon (<literal>:</literal>) and optionally spaces from its value(s).
Note especially that multiple values are separated by spaces, not by
commas as in the one-line format. Multivalue fields like <literal>Architectures</literal>
also have <literal>Architectures-Add</literal> and <literal>Architectures-Remove</literal>
Note especially that multiple values are separated by whitespaces (like spaces,
tabs and newlines), not by commas as in the one-line format.

Multivalue fields like <literal>Architectures</literal> also have
<literal>Architectures-Add</literal> and <literal>Architectures-Remove</literal>
to modify the default value rather than replacing it.
</para><para>
This is a new format supported by apt itself since version 1.1. Previous


+ 92
- 0
test/integration/test-apt-sources-deb822 View File

@@ -199,3 +199,95 @@ testsuccessequal --nomsg "'http://ftp.debian.org/debian/dists/stable/InRelease'
'http://ftp.debian.org/debian/dists/stable/main/binary-i386/Packages.xz' ftp.debian.org_debian_dists_stable_main_binary-i386_Packages 0
'http://ftp.debian.org/debian/dists/stable/main/binary-armel/Packages.xz' ftp.debian.org_debian_dists_stable_main_binary-armel_Packages 0
'http://ftp.debian.org/debian/dists/stable/main/binary-all/Packages.xz' ftp.debian.org_debian_dists_stable_main_binary-all_Packages 0 " aptget update --print-uris

msgcleantest 'Test deb822 sources with' 'multiline multivalues'
cat > "$SOURCES" <<EOF
Types: deb
deb-src
URIs:
http://ftp.debian.org/debian
http://ftp.debian.org/debian2
Suites: stable
sid
Components:
main
contrib non-free
Architectures: amd64
armhf powerpc
EOF
testsuccessequal --nomsg "'http://ftp.debian.org/debian/dists/stable/InRelease' ftp.debian.org_debian_dists_stable_InRelease 0
'http://ftp.debian.org/debian/dists/sid/InRelease' ftp.debian.org_debian_dists_sid_InRelease 0
'http://ftp.debian.org/debian2/dists/stable/InRelease' ftp.debian.org_debian2_dists_stable_InRelease 0
'http://ftp.debian.org/debian2/dists/sid/InRelease' ftp.debian.org_debian2_dists_sid_InRelease 0
'http://ftp.debian.org/debian/dists/stable/main/source/Sources.xz' ftp.debian.org_debian_dists_stable_main_source_Sources 0
'http://ftp.debian.org/debian/dists/stable/contrib/source/Sources.xz' ftp.debian.org_debian_dists_stable_contrib_source_Sources 0
'http://ftp.debian.org/debian/dists/stable/non-free/source/Sources.xz' ftp.debian.org_debian_dists_stable_non-free_source_Sources 0
'http://ftp.debian.org/debian/dists/stable/main/binary-amd64/Packages.xz' ftp.debian.org_debian_dists_stable_main_binary-amd64_Packages 0
'http://ftp.debian.org/debian/dists/stable/main/binary-armhf/Packages.xz' ftp.debian.org_debian_dists_stable_main_binary-armhf_Packages 0
'http://ftp.debian.org/debian/dists/stable/main/binary-powerpc/Packages.xz' ftp.debian.org_debian_dists_stable_main_binary-powerpc_Packages 0
'http://ftp.debian.org/debian/dists/stable/main/binary-all/Packages.xz' ftp.debian.org_debian_dists_stable_main_binary-all_Packages 0
'http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.xz' ftp.debian.org_debian_dists_stable_main_i18n_Translation-en 0
'http://ftp.debian.org/debian/dists/stable/contrib/binary-amd64/Packages.xz' ftp.debian.org_debian_dists_stable_contrib_binary-amd64_Packages 0
'http://ftp.debian.org/debian/dists/stable/contrib/binary-armhf/Packages.xz' ftp.debian.org_debian_dists_stable_contrib_binary-armhf_Packages 0
'http://ftp.debian.org/debian/dists/stable/contrib/binary-powerpc/Packages.xz' ftp.debian.org_debian_dists_stable_contrib_binary-powerpc_Packages 0
'http://ftp.debian.org/debian/dists/stable/contrib/binary-all/Packages.xz' ftp.debian.org_debian_dists_stable_contrib_binary-all_Packages 0
'http://ftp.debian.org/debian/dists/stable/contrib/i18n/Translation-en.xz' ftp.debian.org_debian_dists_stable_contrib_i18n_Translation-en 0
'http://ftp.debian.org/debian/dists/stable/non-free/binary-amd64/Packages.xz' ftp.debian.org_debian_dists_stable_non-free_binary-amd64_Packages 0
'http://ftp.debian.org/debian/dists/stable/non-free/binary-armhf/Packages.xz' ftp.debian.org_debian_dists_stable_non-free_binary-armhf_Packages 0
'http://ftp.debian.org/debian/dists/stable/non-free/binary-powerpc/Packages.xz' ftp.debian.org_debian_dists_stable_non-free_binary-powerpc_Packages 0
'http://ftp.debian.org/debian/dists/stable/non-free/binary-all/Packages.xz' ftp.debian.org_debian_dists_stable_non-free_binary-all_Packages 0
'http://ftp.debian.org/debian/dists/stable/non-free/i18n/Translation-en.xz' ftp.debian.org_debian_dists_stable_non-free_i18n_Translation-en 0
'http://ftp.debian.org/debian/dists/sid/main/source/Sources.xz' ftp.debian.org_debian_dists_sid_main_source_Sources 0
'http://ftp.debian.org/debian/dists/sid/contrib/source/Sources.xz' ftp.debian.org_debian_dists_sid_contrib_source_Sources 0
'http://ftp.debian.org/debian/dists/sid/non-free/source/Sources.xz' ftp.debian.org_debian_dists_sid_non-free_source_Sources 0
'http://ftp.debian.org/debian/dists/sid/main/binary-amd64/Packages.xz' ftp.debian.org_debian_dists_sid_main_binary-amd64_Packages 0
'http://ftp.debian.org/debian/dists/sid/main/binary-armhf/Packages.xz' ftp.debian.org_debian_dists_sid_main_binary-armhf_Packages 0
'http://ftp.debian.org/debian/dists/sid/main/binary-powerpc/Packages.xz' ftp.debian.org_debian_dists_sid_main_binary-powerpc_Packages 0
'http://ftp.debian.org/debian/dists/sid/main/binary-all/Packages.xz' ftp.debian.org_debian_dists_sid_main_binary-all_Packages 0
'http://ftp.debian.org/debian/dists/sid/main/i18n/Translation-en.xz' ftp.debian.org_debian_dists_sid_main_i18n_Translation-en 0
'http://ftp.debian.org/debian/dists/sid/contrib/binary-amd64/Packages.xz' ftp.debian.org_debian_dists_sid_contrib_binary-amd64_Packages 0
'http://ftp.debian.org/debian/dists/sid/contrib/binary-armhf/Packages.xz' ftp.debian.org_debian_dists_sid_contrib_binary-armhf_Packages 0
'http://ftp.debian.org/debian/dists/sid/contrib/binary-powerpc/Packages.xz' ftp.debian.org_debian_dists_sid_contrib_binary-powerpc_Packages 0
'http://ftp.debian.org/debian/dists/sid/contrib/binary-all/Packages.xz' ftp.debian.org_debian_dists_sid_contrib_binary-all_Packages 0
'http://ftp.debian.org/debian/dists/sid/contrib/i18n/Translation-en.xz' ftp.debian.org_debian_dists_sid_contrib_i18n_Translation-en 0
'http://ftp.debian.org/debian/dists/sid/non-free/binary-amd64/Packages.xz' ftp.debian.org_debian_dists_sid_non-free_binary-amd64_Packages 0
'http://ftp.debian.org/debian/dists/sid/non-free/binary-armhf/Packages.xz' ftp.debian.org_debian_dists_sid_non-free_binary-armhf_Packages 0
'http://ftp.debian.org/debian/dists/sid/non-free/binary-powerpc/Packages.xz' ftp.debian.org_debian_dists_sid_non-free_binary-powerpc_Packages 0
'http://ftp.debian.org/debian/dists/sid/non-free/binary-all/Packages.xz' ftp.debian.org_debian_dists_sid_non-free_binary-all_Packages 0
'http://ftp.debian.org/debian/dists/sid/non-free/i18n/Translation-en.xz' ftp.debian.org_debian_dists_sid_non-free_i18n_Translation-en 0
'http://ftp.debian.org/debian2/dists/stable/main/source/Sources.xz' ftp.debian.org_debian2_dists_stable_main_source_Sources 0
'http://ftp.debian.org/debian2/dists/stable/contrib/source/Sources.xz' ftp.debian.org_debian2_dists_stable_contrib_source_Sources 0
'http://ftp.debian.org/debian2/dists/stable/non-free/source/Sources.xz' ftp.debian.org_debian2_dists_stable_non-free_source_Sources 0
'http://ftp.debian.org/debian2/dists/stable/main/binary-amd64/Packages.xz' ftp.debian.org_debian2_dists_stable_main_binary-amd64_Packages 0
'http://ftp.debian.org/debian2/dists/stable/main/binary-armhf/Packages.xz' ftp.debian.org_debian2_dists_stable_main_binary-armhf_Packages 0
'http://ftp.debian.org/debian2/dists/stable/main/binary-powerpc/Packages.xz' ftp.debian.org_debian2_dists_stable_main_binary-powerpc_Packages 0
'http://ftp.debian.org/debian2/dists/stable/main/binary-all/Packages.xz' ftp.debian.org_debian2_dists_stable_main_binary-all_Packages 0
'http://ftp.debian.org/debian2/dists/stable/main/i18n/Translation-en.xz' ftp.debian.org_debian2_dists_stable_main_i18n_Translation-en 0
'http://ftp.debian.org/debian2/dists/stable/contrib/binary-amd64/Packages.xz' ftp.debian.org_debian2_dists_stable_contrib_binary-amd64_Packages 0
'http://ftp.debian.org/debian2/dists/stable/contrib/binary-armhf/Packages.xz' ftp.debian.org_debian2_dists_stable_contrib_binary-armhf_Packages 0
'http://ftp.debian.org/debian2/dists/stable/contrib/binary-powerpc/Packages.xz' ftp.debian.org_debian2_dists_stable_contrib_binary-powerpc_Packages 0
'http://ftp.debian.org/debian2/dists/stable/contrib/binary-all/Packages.xz' ftp.debian.org_debian2_dists_stable_contrib_binary-all_Packages 0
'http://ftp.debian.org/debian2/dists/stable/contrib/i18n/Translation-en.xz' ftp.debian.org_debian2_dists_stable_contrib_i18n_Translation-en 0
'http://ftp.debian.org/debian2/dists/stable/non-free/binary-amd64/Packages.xz' ftp.debian.org_debian2_dists_stable_non-free_binary-amd64_Packages 0
'http://ftp.debian.org/debian2/dists/stable/non-free/binary-armhf/Packages.xz' ftp.debian.org_debian2_dists_stable_non-free_binary-armhf_Packages 0
'http://ftp.debian.org/debian2/dists/stable/non-free/binary-powerpc/Packages.xz' ftp.debian.org_debian2_dists_stable_non-free_binary-powerpc_Packages 0
'http://ftp.debian.org/debian2/dists/stable/non-free/binary-all/Packages.xz' ftp.debian.org_debian2_dists_stable_non-free_binary-all_Packages 0
'http://ftp.debian.org/debian2/dists/stable/non-free/i18n/Translation-en.xz' ftp.debian.org_debian2_dists_stable_non-free_i18n_Translation-en 0
'http://ftp.debian.org/debian2/dists/sid/main/source/Sources.xz' ftp.debian.org_debian2_dists_sid_main_source_Sources 0
'http://ftp.debian.org/debian2/dists/sid/contrib/source/Sources.xz' ftp.debian.org_debian2_dists_sid_contrib_source_Sources 0
'http://ftp.debian.org/debian2/dists/sid/non-free/source/Sources.xz' ftp.debian.org_debian2_dists_sid_non-free_source_Sources 0
'http://ftp.debian.org/debian2/dists/sid/main/binary-amd64/Packages.xz' ftp.debian.org_debian2_dists_sid_main_binary-amd64_Packages 0
'http://ftp.debian.org/debian2/dists/sid/main/binary-armhf/Packages.xz' ftp.debian.org_debian2_dists_sid_main_binary-armhf_Packages 0
'http://ftp.debian.org/debian2/dists/sid/main/binary-powerpc/Packages.xz' ftp.debian.org_debian2_dists_sid_main_binary-powerpc_Packages 0
'http://ftp.debian.org/debian2/dists/sid/main/binary-all/Packages.xz' ftp.debian.org_debian2_dists_sid_main_binary-all_Packages 0
'http://ftp.debian.org/debian2/dists/sid/main/i18n/Translation-en.xz' ftp.debian.org_debian2_dists_sid_main_i18n_Translation-en 0
'http://ftp.debian.org/debian2/dists/sid/contrib/binary-amd64/Packages.xz' ftp.debian.org_debian2_dists_sid_contrib_binary-amd64_Packages 0
'http://ftp.debian.org/debian2/dists/sid/contrib/binary-armhf/Packages.xz' ftp.debian.org_debian2_dists_sid_contrib_binary-armhf_Packages 0
'http://ftp.debian.org/debian2/dists/sid/contrib/binary-powerpc/Packages.xz' ftp.debian.org_debian2_dists_sid_contrib_binary-powerpc_Packages 0
'http://ftp.debian.org/debian2/dists/sid/contrib/binary-all/Packages.xz' ftp.debian.org_debian2_dists_sid_contrib_binary-all_Packages 0
'http://ftp.debian.org/debian2/dists/sid/contrib/i18n/Translation-en.xz' ftp.debian.org_debian2_dists_sid_contrib_i18n_Translation-en 0
'http://ftp.debian.org/debian2/dists/sid/non-free/binary-amd64/Packages.xz' ftp.debian.org_debian2_dists_sid_non-free_binary-amd64_Packages 0
'http://ftp.debian.org/debian2/dists/sid/non-free/binary-armhf/Packages.xz' ftp.debian.org_debian2_dists_sid_non-free_binary-armhf_Packages 0
'http://ftp.debian.org/debian2/dists/sid/non-free/binary-powerpc/Packages.xz' ftp.debian.org_debian2_dists_sid_non-free_binary-powerpc_Packages 0
'http://ftp.debian.org/debian2/dists/sid/non-free/binary-all/Packages.xz' ftp.debian.org_debian2_dists_sid_non-free_binary-all_Packages 0
'http://ftp.debian.org/debian2/dists/sid/non-free/i18n/Translation-en.xz' ftp.debian.org_debian2_dists_sid_non-free_i18n_Translation-en 0 " aptget update --print-uris

Loading…
Cancel
Save