Remove extra space

-----BEGIN PGP SIGNATURE-----
 
 iQJPBAABCgA5FiEET7WIqEwt3nmnTHeHb6RY3R2wP3EFAmDB040bHGp1bGlhbi5r
 bG9kZUBjYW5vbmljYWwuY29tAAoJEG+kWN0dsD9xvRsP/j3YPDrUiUh2Gc8fsezh
 /MZ328rjFQLtd8BzCpdHnq/8i/HhMOWgxdep+13PJTQfjL79s8qGPSAdFKvdc8Kq
 hTEF6ZhR0voDug7o/kuiigkbb7xuT5kfAkuZLqkTto38Fc7HL6rSEcCLQAebIU7N
 uPBFLcjkeaGKLI0z7t78k1qIPzI6JINktTvVBosDIrBg3btlq5brTC1+qDIb12XM
 PJ8joSkGoqdc7zLQOJImdq4hgQODCWPwJkNf6C3EBu9HLk7yi5+w4kUxg0HaguRg
 ooQ8TdJSiVg8qvhw0FyQr6bSHqeJoa8WS6cNTjcuzIpHo5hFMvJ2clno32uawffl
 vxmCVIBW1HNFz6iRjRFQtsTjSdwpOeql7EULgT5xbB7wJGq7kCzSc1IwRqbhCkT7
 DdUVXg7Mu1AZ4lqsoxB75uhwALs58JNCSizqMT8B6CrGBEksc7IC2JsKvvidk4Qy
 HdEyEdgvumLJomPH3qz+P8ZvIz5JY1xwa+U06tZQGsD2riifqQRsX94leIp4aLTv
 EEkjMuRM28eEjcbdO3sb9oIqjYophPPkqSxFGZHxMOqWdHaVUT6PCovtbPjZmCHY
 ptnu7SfgVLlgP+7k5WKQQp5tvy2PMt+jfdBY1wFAJ0kjBdZxs29REgSAWyMe7qS/
 B/VK6c3UvVy+UxTIGuZhSS6t
 =nnY9
 -----END PGP SIGNATURE-----

Merge tag 'tags/debian/2.2.4'

apt Debian release 2.2.4

CMakeLists.txt

@@ -200,7 +200,7 @@ endif()
 # Configure some variables like package, version and architecture.
 set(PACKAGE ${PROJECT_NAME})
 set(PACKAGE_MAIL "APT Development Team <deity@lists.debian.org>")
-set(PACKAGE_VERSION "2.2.1+devuan1")
+set(PACKAGE_VERSION "2.2.4+devuan1")
 string(REGEX MATCH "^[0-9.]+" PROJECT_VERSION ${PACKAGE_VERSION})
 
 if (NOT DEFINED DPKG_DATADIR)

apt-pkg/acquire-item.cc

@@ -2564,26 +2564,16 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile)	/*{{{*/
       }
    }
 
-
-   bool foundStart = false;
-   for (std::vector<DiffInfo>::iterator cur = available_patches.begin();
-	 cur != available_patches.end(); ++cur)
    {
-      if (LocalHashes != cur->result_hashes)
-	 continue;
-
-      available_patches.erase(available_patches.begin(), cur);
-      foundStart = true;
-      break;
-   }
-
-   if (foundStart == false || unlikely(available_patches.empty() == true))
-   {
-      ErrorText = "Couldn't find the start of the patch series";
-      return false;
-   }
+      auto const foundStart = std::find_if(available_patches.rbegin(), available_patches.rend(),
+	    [&](auto const &cur) { return LocalHashes == cur.result_hashes; });
+      if (foundStart == available_patches.rend() || unlikely(available_patches.empty()))
+      {
+	 ErrorText = "Couldn't find the start of the patch series";
+	 return false;
+      }
+      available_patches.erase(available_patches.begin(), std::prev(foundStart.base()));
 
-   {
       auto const patch = std::find_if(available_patches.cbegin(), available_patches.cend(), [](auto const &patch) {
 	 return not patch.result_hashes.usable() ||
 		not patch.patch_hashes.usable() ||
@@ -3050,14 +3040,11 @@ void pkgAcqIndexMergeDiffs::Done(string const &Message, HashStringList const &Ha
       State = StateErrorDiff;
       return;
    }
-   std::string const PatchFile = GetMergeDiffsPatchFileName(UnpatchedFile, patch.file);
    std::string const PatchedFile = GetKeepCompressedFileName(UncompressedUnpatchedFile, Target);
 
    switch (State)
    {
       case StateFetchDiff:
-	 Rename(DestFile, PatchFile);
-
 	 // check if this is the last completed diff
 	 State = StateDoneDiff;
 	 for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
@@ -3068,6 +3055,8 @@ void pkgAcqIndexMergeDiffs::Done(string const &Message, HashStringList const &Ha
 		  std::clog << "Not the last done diff in the batch: " << Desc.URI << std::endl;
 	       return;
 	    }
+	 for (auto * diff : *allPatches)
+	    Rename(diff->DestFile, GetMergeDiffsPatchFileName(UnpatchedFile, diff->patch.file));
 	 // this is the last completed diff, so we are ready to apply now
 	 DestFile = GetKeepCompressedFileName(UncompressedUnpatchedFile + "-patched", Target);
 	 if(Debug)
@@ -3098,8 +3087,8 @@ void pkgAcqIndexMergeDiffs::Done(string const &Message, HashStringList const &Ha
 	 if(Debug)
 	    std::clog << "allDone: " << DestFile << "\n" << std::endl;
 	 return;
-      case StateDoneDiff: _error->Fatal("Done called for %s which is in an invalid Done state", PatchFile.c_str()); break;
-      case StateErrorDiff: _error->Fatal("Done called for %s which is in an invalid Error state", PatchFile.c_str()); break;
+      case StateDoneDiff: _error->Fatal("Done called for %s which is in an invalid Done state", patch.file.c_str()); break;
+      case StateErrorDiff: _error->Fatal("Done called for %s which is in an invalid Error state", patch.file.c_str()); break;
    }
 }
 									/*}}}*/
@@ -3188,8 +3177,8 @@ void pkgAcqIndex::Init(string const &URI, string const &URIDesc,
 /* The only header we use is the last-modified header. */
 string pkgAcqIndex::Custom600Headers() const
 {
-
-   string msg = "\nIndex-File: true";
+   std::string msg = pkgAcqBaseIndex::Custom600Headers();
+   msg.append("\nIndex-File: true");
 
    if (TransactionManager->LastMetaIndexParser == NULL)
    {
@@ -3480,6 +3469,12 @@ pkgAcqArchive::pkgAcqArchive(pkgAcquire *const Owner, pkgSourceList *const Sourc
 		    Version.VerStr(), Version.ParentPkg().FullName(false).c_str());
       return;
    }
+   if (FileSize == 0 && not _config->FindB("Acquire::AllowUnsizedPackages", false))
+   {
+      _error->Warning("Repository is broken: %s (= %s) has no Size information",
+		      Version.ParentPkg().FullName(false).c_str(),
+		      Version.VerStr());
+   }
 
    // Check if we already downloaded the file
    struct stat Buf;
@@ -3857,16 +3852,16 @@ pkgAcqFile::pkgAcqFile(pkgAcquire *const Owner, string const &URI, HashStringLis
 		       const string &DestDir, const string &DestFilename,
 		       bool const IsIndexFile) : Item(Owner), d(NULL), IsIndexFile(IsIndexFile), ExpectedHashes(Hashes)
 {
+   ::URI url{URI};
+   if (url.Path.find(' ') != std::string::npos || url.Path.find('%') == std::string::npos)
+      url.Path = pkgAcquire::URIEncode(url.Path);
+
    if(!DestFilename.empty())
       DestFile = DestFilename;
    else if(!DestDir.empty())
-      DestFile = DestDir + "/" + flNotDir(URI);
+      DestFile = DestDir + "/" + DeQuoteString(flNotDir(url.Path));
    else
-      DestFile = flNotDir(URI);
-
-   ::URI url{URI};
-   if (url.Path.find(' ') != std::string::npos || url.Path.find('%') == std::string::npos)
-      url.Path = pkgAcquire::URIEncode(url.Path);
+      DestFile = DeQuoteString(flNotDir(url.Path));
 
    // Create the item
    Desc.URI = std::string(url);
@@ -3941,9 +3936,10 @@ void pkgAcqFile::Done(string const &Message,HashStringList const &CalcHashes,
 									/*}}}*/
 string pkgAcqFile::Custom600Headers() const				/*{{{*/
 {
-   if (IsIndexFile)
-      return "\nIndex-File: true";
-   return "";
+   string Header = pkgAcquire::Item::Custom600Headers();
+   if (not IsIndexFile)
+      return Header;
+   return Header + "\nIndex-File: true";
 }
 									/*}}}*/
 pkgAcqFile::~pkgAcqFile() {}

apt-pkg/deb/dpkgpm.cc

@@ -1765,8 +1765,8 @@ bool pkgDPkgPM::Go(APT::Progress::PackageManager *progress)
       if (pipe(fd) != 0)
 	 return _error->Errno("pipe","Failed to create IPC pipe to dpkg");
 
-#define ADDARG(X) Args.push_back(X); Size += strlen(X)
-#define ADDARGC(X) Args.push_back(X); Size += sizeof(X) - 1
+#define ADDARG(X) do { const char *arg = (X); Args.push_back(arg); Size += strlen(arg); } while (0)
+#define ADDARGC(X) ADDARG(X)
 
       ADDARGC("--status-fd");
       char status_fd_buf[20];

apt-pkg/indexfile.cc

@@ -9,6 +9,7 @@
 // Include Files							/*{{{*/
 #include <config.h>
 
+#include <apt-pkg/acquire.h>
 #include <apt-pkg/aptconfiguration.h>
 #include <apt-pkg/configuration.h>
 #include <apt-pkg/deblistparser.h>
@@ -174,7 +175,7 @@ pkgDebianIndexTargetFile::pkgDebianIndexTargetFile(IndexTarget const &Target, bo
 									/*}}}*/
 std::string pkgDebianIndexTargetFile::ArchiveURI(std::string const &File) const/*{{{*/
 {
-   return Target.Option(IndexTarget::REPO_URI) + File;
+   return Target.Option(IndexTarget::REPO_URI) + pkgAcquire::URIEncode(File);
 }
 									/*}}}*/
 std::string pkgDebianIndexTargetFile::Describe(bool const Short) const	/*{{{*/
@@ -281,7 +282,7 @@ std::string pkgDebianIndexRealFile::Describe(bool const /*Short*/) const/*{{{*/
 									/*}}}*/
 std::string pkgDebianIndexRealFile::ArchiveURI(std::string const &/*File*/) const/*{{{*/
 {
-   return "file:" + File;
+   return "file:" + pkgAcquire::URIEncode(File);
 }
 									/*}}}*/
 std::string pkgDebianIndexRealFile::IndexFileName() const			/*{{{*/

apt-pkg/policy.cc

@@ -310,7 +310,7 @@ static inline bool ExcludePhased(std::string machineID, pkgCache::VerIterator co
 }
 APT_PURE signed short pkgPolicy::GetPriority(pkgCache::VerIterator const &Ver, bool ConsiderFiles)
 {
-   if (Ver.ParentPkg()->CurrentVer && Ver.PhasedUpdatePercentage() != 100)
+   if (Ver.PhasedUpdatePercentage() != 100)
    {
       if (ExcludePhased(d->machineID, Ver))
 	 return 1;

apt-private/acqprogress.cc

@@ -322,8 +322,10 @@ bool AcqTextStatus::MediaChange(std::string Media, std::string Drive)
    while (C != '\n' && C != '\r')
    {
       int len = read(STDIN_FILENO,&C,1);
-      if(C == 'c' || len <= 0)
+      if(C == 'c' || len <= 0) {
 	 bStatus = false;
+	 break;
+      }
    }
 
    if(bStatus)

apt-private/private-json-hooks.cc

@@ -11,7 +11,9 @@
 #include <apt-pkg/macros.h>
 #include <apt-pkg/strutl.h>
 #include <apt-private/private-json-hooks.h>
+#include <apt-private/private-output.h>
 
+#include <iomanip>
 #include <ostream>
 #include <sstream>
 #include <stack>
@@ -23,7 +25,7 @@
 /**
  * @brief Simple JSON writer
  *
- * This performs no error checking, or string escaping, be careful.
+ * This performs no error checking, so be careful.
  */
 class APT_HIDDEN JsonWriter
 {
@@ -78,6 +80,7 @@ class APT_HIDDEN JsonWriter
    void popState()
    {
       this->state = old_states.top();
+      old_states.pop();
    }
 
    public:
@@ -109,22 +112,40 @@ class APT_HIDDEN JsonWriter
       os << '}';
       return *this;
    }
+   std::ostream &encodeString(std::ostream &out, std::string const &str)
+   {
+      out << '"';
+
+      for (std::string::const_iterator c = str.begin(); c != str.end(); c++)
+      {
+	 if (*c <= 0x1F || *c == '"' || *c == '\\')
+	    ioprintf(out, "\\u%04X", *c);
+	 else
+	    out << *c;
+      }
+
+      out << '"';
+      return out;
+   }
    JsonWriter &name(std::string const &name)
    {
       maybeComma();
-      os << '"' << name << '"' << ':';
+      encodeString(os, name) << ':';
       return *this;
    }
    JsonWriter &value(std::string const &value)
    {
       maybeComma();
-      os << '"' << value << '"';
+      encodeString(os, value);
       return *this;
    }
    JsonWriter &value(const char *value)
    {
       maybeComma();
-      os << '"' << value << '"';
+      if (value == nullptr)
+	 os << "null";
+      else
+	 encodeString(os, value);
       return *this;
    }
    JsonWriter &value(int value)
@@ -315,6 +336,13 @@ bool RunJsonHook(std::string const &option, std::string const &method, const cha
       return true;
    Opts = Opts->Child;
 
+   // Flush output before calling hooks
+   std::clog.flush();
+   std::cerr.flush();
+   std::cout.flush();
+   c2out.flush();
+   c1out.flush();
+
    sighandler_t old_sigpipe = signal(SIGPIPE, SIG_IGN);
    sighandler_t old_sigint = signal(SIGINT, SIG_IGN);
    sighandler_t old_sigquit = signal(SIGQUIT, SIG_IGN);

debian/changelog

@@ -1,3 +1,76 @@
+apt (2.2.4+devuan1) unstable; urgency=medium
+
+  * Merge upstream 2.2.4
+
+ -- Andreas Messer <andi@bastelmap.de>  Mon, 21 Jun 2021 15:14:12 +0200
+
+apt (2.2.4) unstable; urgency=medium
+
+  [ Julian Andres Klode ]
+  * Various bugfixes to the JSON hooks:
+    - encoder fixes:
+      + json: Escape strings using \u escape sequences, add test
+      + json: Actually pop states
+      + json: Encode NULL strings as null
+    - json: Flush standard file descriptors before calling hooks
+      (this avoids output from hooks in middle of apt output)
+    - Non-installed JSON changes:
+      + test/json: Make the test hook more reliable
+      + Fix a typo in json-hooks-protocol.md (thanks to Brian Murray)
+  * Avoid infinite loop on EOF on media change prompt (LP: #1928687)
+  * Turn TLS handshake issues into transient errors (LP: #1928100),
+    this makes behavior consistent with TCP and enables Acquire::Retries
+  * policy: Apply phasing to uninstalled packages too (LP: #1925745),
+    this prevents inconsistencies when installing new binaries that depend
+    on the same version of an already installed binary.
+
+  [ David Kalnischkies ]
+  * URI encode Filename field of Packages files (again). This fixes a
+    regression introduced in 2.1.15 that causes download failures of
+    packages with an epoch included in their filename.
+
+ -- Julian Andres Klode <jak@debian.org>  Thu, 10 Jun 2021 10:53:34 +0200
+
+apt (2.2.3+devuan1) unstable; urgency=medium
+
+  * Merge upstream 2.2.3
+
+ -- Andreas Messer <andi@bastelmap.de>  Sun, 30 May 2021 10:20:55 +0200
+
+apt (2.2.3) unstable; urgency=medium
+
+  * tests: Check for and discard expected warning from MaybeAddAuth. For some
+    reason, this was only noticed with LTO enabled, but should be a general
+    issue.
+  * Fix downloads of unsized files that are largest in pipeline (LP: #1921626)
+  * Warn on packages without a Size field. Such repositories are broken and
+    need to be fixed, as we do not test apt against them, see the bug above
+    for more details. Set Acquire::AllowUnsizedPackages to disable the
+    warning.
+
+ -- Julian Andres Klode <jak@debian.org>  Tue, 13 Apr 2021 17:53:32 +0200
+
+apt (2.2.2+devuan1) unstable; urgency=medium
+
+  * Merge upstream version 2.2.2
+
+ -- Andreas Messer <andi@bastelmap.de>  Fri, 19 Mar 2021 18:46:07 +0100
+
+apt (2.2.2) unstable; urgency=medium
+
+  [ David Kalnischkies ]
+  * Deal with rred shortcomings around empty patch files (LP: #1918112)
+    - Allow merging with empty pdiff patches
+    - Rename pdiff merge patches only after they are all downloaded
+    - Start pdiff patching from the last possible starting point
+    - Ensure all index files sent custom tags to the methods
+  * Harden test for no new acquires after transaction abort (Closes: #984966)
+
+  [ Julian Andres Klode ]
+  * Make ADDARG{,C}() macros expand to single statements
+
+ -- Julian Andres Klode <jak@debian.org>  Fri, 12 Mar 2021 09:15:59 +0100
+
 apt (2.2.1+devuan1) unstable; urgency=medium
 
   * Merge upstream version 2.2.1

doc/apt-verbatim.ent

@@ -274,7 +274,7 @@
 ">
 
 <!-- this will be updated by 'prepare-release' -->
-<!ENTITY apt-product-version "2.2.1+devuan1">
+<!ENTITY apt-product-version "2.2.4+devuan1">
 
 <!-- (Code)names for various things used all over the place -->
 <!ENTITY debian-oldstable-codename "buster">

doc/examples/configure-index

@@ -257,6 +257,7 @@ Acquire
   AllowInsecureRepositories "<BOOL>";
   AllowWeakRepositories "<BOOL>";
   AllowDowngradeToInsecureRepositories "<BOOL>";
+  AllowUnsizedPackages "<BOOL>";
   // allow repositories to change information potentially breaking user config like pinning
   AllowReleaseInfoChange "<BOOL>"
   {

doc/json-hooks-protocol.md

@@ -23,7 +23,7 @@ The general life of a hook is as following.
 1. Hook is started
 2. Hello handshake is exchanged
 3. One or more calls or notifications are sent from apt to the hook
-4. Bye notification is send
+4. Bye notification is sent
 
 It is unspecified whether a hook is sent one or more messages. For
 example, a hook may be started only once for the lifetime of the apt

doc/po/apt-doc.pot

@@ -5,9 +5,9 @@
 #, fuzzy
 msgid ""
 msgstr ""
-"Project-Id-Version: apt-doc 2.2.1+devuan1\n"
+"Project-Id-Version: apt-doc 2.2.4+devuan1\n"
 "Report-Msgid-Bugs-To: APT Development Team <deity@lists.debian.org>\n"
-"POT-Creation-Date: 2021-03-17 18:55+0000\n"
+"POT-Creation-Date: 2021-06-21 13:22+0000\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"

methods/basehttp.cc

@@ -882,7 +882,11 @@ unsigned long long BaseHttpMethod::FindMaximumObjectSizeInQueue() const	/*{{{*/
 {
    unsigned long long MaxSizeInQueue = 0;
    for (FetchItem *I = Queue; I != 0 && I != QueueBack; I = I->Next)
+   {
+      if (I->MaximumSize == 0)
+	 return 0;
       MaxSizeInQueue = std::max(MaxSizeInQueue, I->MaximumSize);
+   }
    return MaxSizeInQueue;
 }
 									/*}}}*/

methods/connect.cc

@@ -1045,7 +1045,7 @@ ResultState UnwrapTLS(std::string const &Host, std::unique_ptr<MethodFd> &Fd,
    err = tlsFd->DoTLSHandshake();
 
    if (err < 0)
-      return ResultState::FATAL_ERROR;
+      return ResultState::TRANSIENT_ERROR;
 
    return ResultState::SUCCESSFUL;
 }

methods/rred.cc

@@ -441,8 +441,12 @@ class Patch {
       bool cmdwanted = true;
 
       Change ch(std::numeric_limits<size_t>::max());
-      if (f.ReadLine(buffer, sizeof(buffer)) == NULL)
+      if (f.ReadLine(buffer, sizeof(buffer)) == nullptr)
+      {
+	 if (f.Eof())
+	    return true;
 	 return _error->Error("Reading first line of patchfile %s failed", f.Name().c_str());
+      }
       do {
 	 if (h != NULL)
 	    h->Add(buffer);

po/apt-all.pot

@@ -5,9 +5,9 @@
 #, fuzzy
 msgid ""
 msgstr ""
-"Project-Id-Version: apt 2.2.1+devuan1\n"
+"Project-Id-Version: apt 2.2.4+devuan1\n"
 "Report-Msgid-Bugs-To: APT Development Team <deity@lists.debian.org>\n"
-"POT-Creation-Date: 2021-03-17 18:55+0000\n"
+"POT-Creation-Date: 2021-06-21 13:22+0000\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"

test/integration/framework

@@ -486,6 +486,10 @@ EOF
 
 	echo 'APT::Machine-ID "912e43bd1c1d4ba481f9f8ccab25f9ee";' > rootdir/etc/apt/apt.conf.d/machine-id
 
+	# XXX: We really ought to make the packages in the test suite all have Size fields.
+	# But this needs a lot more changes, so let's keep it simple for stable updates.
+	echo 'Acquire::AllowUnsizedPackages "true";' >> rootdir/etc/apt/apt.conf.d/temporary-allow-unsized-packages
+
 	configcompression '.' 'gz' #'bz2' 'lzma' 'xz'
 	confighashes 'SHA256' # these are tests, not security best-practices
 
@@ -2126,6 +2130,9 @@ aptautotest() {
 	fi
 }
 
+cdfind() {
+	( cd /; find "$@" )
+}
 aptautotest_aptget_update() {
 	local TESTCALL="$1"
 	while [ -n "$2" ]; do
@@ -2135,24 +2142,21 @@ aptautotest_aptget_update() {
 	if ! test -d "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists"; then return; fi
 	testfilestats "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt" '%U:%G:%a' '=' "${TEST_DEFAULT_USER}:${TEST_DEFAULT_GROUP}:755"
 	testfilestats "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists" '%U:%G:%a' '=' "${TEST_DEFAULT_USER}:${TEST_DEFAULT_GROUP}:755"
-	(
-	cd /
 	# all copied files are properly chmodded
 	local backupIFS="$IFS"
 	IFS="$(printf "\n\b")"
-	find "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists" -type f ! -name 'lock' | while read file; do
+	cdfind "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists" -type f ! -name 'lock' | while read file; do
 		testfilestats "$file" '%U:%G:%a' '=' "${TEST_DEFAULT_USER}:${TEST_DEFAULT_GROUP}:644"
 	done
 	IFS="$backupIFS"
 	if [ "$TESTCALL" = 'testsuccess' ]; then
 		# failure cases can retain partial files and such
-		testempty find "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists/partial" -mindepth 1 ! \( -name 'lock' -o -name '*.FAILED' \)
+		testempty cdfind "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists/partial" -mindepth 1 ! \( -name 'lock' -o -name '*.FAILED' \)
 	fi
 	if [ -s "${TMPWORKINGDIRECTORY}/rootdir/var/log/aptgetupdate.before.lst" ]; then
 		testfileequal "${TMPWORKINGDIRECTORY}/rootdir/var/log/aptgetupdate.before.lst" \
-			"$(find "${TMPWORKINGDIRECTORY}/aptarchive/dists" -type f | while read line; do stat --format '%U:%G:%a:%n' "$line"; done | sort)"
+			"$(cdfind "${TMPWORKINGDIRECTORY}/aptarchive/dists" -type f | while read line; do stat --format '%U:%G:%a:%n' "$line"; done | sort)"
 	fi
-	)
 }
 aptautotest_apt_update() { aptautotest_aptget_update "$@"; }
 aptautotest_aptcdrom_add() { aptautotest_aptget_update "$@"; }

test/integration/test-apt-cli-json-hooks

@@ -21,13 +21,12 @@ APTARCHIVE="$(readlink -f ./aptarchive)"
 
 cat >> json-hook.sh << EOF
 #!/bin/bash
+trap '' SIGPIPE
 while true; do
-	read request <&\$APT_HOOK_SOCKET
-	read empty <&\$APT_HOOK_SOCKET
+	read request <&\$APT_HOOK_SOCKET || exit 1
 
 	if echo "\$request" | grep -q ".hello"; then
 		echo "HOOK: HELLO"
-		printf '{"jsonrpc": "2.0", "result": {"version": "0.1"}, "id": 0}\n\n' >&\$APT_HOOK_SOCKET
 	fi
 
 	if echo "\$request" | grep -q ".bye"; then
@@ -36,7 +35,16 @@ while true; do
 	fi
 
 	echo HOOK: request \$request
+
+	read empty <&\$APT_HOOK_SOCKET || exit 1
+
 	echo HOOK: empty \$empty
+
+	if echo "\$request" | grep -q ".hello"; then
+		printf '{"jsonrpc": "2.0", "result": {"version": "0.1"}, "id": 0}\n\n' >&\$APT_HOOK_SOCKET 2>/dev/null || exit 1
+	fi
+
+
 done
 EOF
 

test/integration/test-apt-get-install-deb

@@ -209,30 +209,31 @@ echo "Package: pkg-as-it-should-be
 Architecture: all
 Version: 0
 Installed-Size: 2903
-Filename: incoming/pkg-as-it-should-be_0_all.deb
+Filename: incoming/pkg-as-it-should-be_0%3a0+0_all.deb
 Size: $(stat -c %s incoming/pkg-as-it-should-be_0_all.deb)
 SHA256: $(sha256sum incoming/pkg-as-it-should-be_0_all.deb | cut -d' ' -f 1)
 " > Packages
+ln -s pkg-as-it-should-be_0_all.deb incoming/pkg-as-it-should-be_0%3a0+0_all.deb
 testdpkgnotinstalled 'pkg-as-it-should-be'
 testnopackage pkg-as-it-should-be
 testsuccess apt install --with-source ./Packages pkg-as-it-should-be -s
 testsuccess apt install --with-source ./Packages pkg-as-it-should-be --print-uris
 testsuccess apt show --with-source ./Packages pkg-as-it-should-be
 testequal 'Package: pkg-as-it-should-be' head -n1 rootdir/tmp/testsuccess.output
-testsuccess apt install -y --with-source ./Packages pkg-as-it-should-be
+testsuccess apt install -y --with-source ./Packages pkg-as-it-should-be -o Debug::pkgAcquire::Worker=1
 testdpkginstalled 'pkg-as-it-should-be'
 rm -f ./Packages
 
 echo 'dpkg::install::recursive "true";
 dpkg::install::recursive::force "true";
 dpkg::install::recursive::minimum "0";' > rootdir/etc/apt/apt.conf.d/lowerminimum.conf
-mv ./incoming/pkg-as-it-should-be_0_all.deb ./incoming/pkg-as-it-should-be_0_all.ddeb
-testsuccess aptget install -y ./incoming/pkg-as-it-should-be_0_all.ddeb --reinstall
+mv ./incoming/pkg-as-it-should-be_0_all.deb ./incoming/pkg-as-it-should-be_0%3a0+0_all.ddeb
+testsuccess aptget install -y ./incoming/pkg-as-it-should-be_0%3a0+0_all.ddeb --reinstall
 testfailure grep 'is already the newest version' rootdir/tmp/testsuccess.output
 testsuccess apt purge -y pkg-as-it-should-be
 testdpkgnotinstalled 'pkg-as-it-should-be'
 
-mv ./incoming/pkg-as-it-should-be_0_all.ddeb ./incoming/pkg-as-it-should-be_0_all.foobar
+mv ./incoming/pkg-as-it-should-be_0%3a0+0_all.ddeb ./incoming/pkg-as-it-should-be_0_all.foobar
 echo "Package: pkg-as-it-should-be
 Architecture: all
 Version: 0

test/integration/test-apt-https-transient

@@ -0,0 +1,43 @@
+#!/bin/sh
+set -e
+
+TESTDIR="$(readlink -f "$(dirname "$0")")"
+. "$TESTDIR/framework"
+
+setupenvironment
+configarchitecture "i386"
+
+# Disable sandbox to avoid W: down below
+echo 'APT::Sandbox::User "root";' > rootdir/etc/apt/apt.conf.d/no-acquire-sandbox
+
+echo 'alright' > aptarchive/working
+changetohttpswebserver
+
+msgtest 'download of a file works via' 'http'
+testsuccess --nomsg downloadfile "http://localhost:${APTHTTPPORT}/working" httpsfile
+testfileequal httpsfile 'alright'
+rm -f httpfile httpsfile
+
+msgtest 'download of a file works via' 'https'
+testsuccess --nomsg downloadfile "https://localhost:${APTHTTPSPORT}/working" httpfile
+testfileequal httpfile 'alright'
+rm -f httpfile httpsfile
+
+# Speak wrong protocols (https on http port and vice versa). We check that they can be retried.
+
+msgtest 'protocol negotiation error is transient for' 'https'
+testfailureequal "Ign:1 https://localhost:${APTHTTPPORT}/working
+  Could not wait for server fd - select (11: Resource temporarily unavailable)
+Err:1 https://localhost:${APTHTTPPORT}/working
+  Could not wait for server fd - select (11: Resource temporarily unavailable)
+E: Failed to fetch https://localhost:${APTHTTPPORT}/working  Could not wait for server fd - select (11: Resource temporarily unavailable)
+E: Download Failed" apthelper download-file "https://localhost:${APTHTTPPORT}/working" httpfile -oAcquire::https::Timeout=1 -oAcquire::Retries=1
+
+# Speak wrong protocols (https on http port and vice versa)
+msgtest 'protocol negotiation error is transient for' 'http'
+testfailureequal "Ign:1 http://localhost:${APTHTTPSPORT}/working
+  Connection failed
+Err:1 http://localhost:${APTHTTPSPORT}/working
+  Connection failed
+E: Failed to fetch http://localhost:${APTHTTPSPORT}/working  Connection failed
+E: Download Failed" apthelper download-file "http://localhost:${APTHTTPSPORT}/working" httpfile -oAcquire::https::Timeout=1 -oAcquire::Retries=1

test/integration/test-bug-723705-tagfile-truncates-fields

@@ -26,8 +26,8 @@ After this operation, 19.8 MB of additional disk space will be used.
 'file:///tmp/aptarchive/pool/main/c/cdebconf/cdebconf-gtk-udeb_0.185_amd64.udeb' cdebconf-gtk-udeb_0.185_amd64.udeb 27278 MD5Sum:a1bbbc1d4fb8e0615b5621abac021924
 'file:///tmp/aptarchive/pool/main/c/cdebconf/cdebconf-newt-udeb_0.185_amd64.udeb' cdebconf-newt-udeb_0.185_amd64.udeb 19192 MD5Sum:de27807f56dae2f2403b3322d5fe6bd2
 'file:///tmp/aptarchive/pool/main/g/glib2.0/libglib2.0-udeb_2.36.4-1_amd64.udeb' libglib2.0-udeb_2.36.4-1_amd64.udeb 1714604 MD5Sum:72da029f1bbb36057d874f1f82a5d00a
-'file:///tmp/aptarchive/pool/main/e/eglibc/libc6-udeb_2.17-92+b1_amd64.udeb' libc6-udeb_2.17-92+b1_amd64.udeb 1056000 MD5Sum:7fd7032eeeecf7f76eff79a0543fbd72
-'file:///tmp/aptarchive/pool/main/g/gtk+2.0/libgtk2.0-0-udeb_2.24.20-1_amd64.udeb' libgtk2.0-0-udeb_2.24.20-1_amd64.udeb 1643046 MD5Sum:25513478eb2e02e5766c0eea0b411ca9
+'file:///tmp/aptarchive/pool/main/e/eglibc/libc6-udeb_2.17-92%2bb1_amd64.udeb' libc6-udeb_2.17-92+b1_amd64.udeb 1056000 MD5Sum:7fd7032eeeecf7f76eff79a0543fbd72
+'file:///tmp/aptarchive/pool/main/g/gtk%2b2.0/libgtk2.0-0-udeb_2.24.20-1_amd64.udeb' libgtk2.0-0-udeb_2.24.20-1_amd64.udeb 1643046 MD5Sum:25513478eb2e02e5766c0eea0b411ca9
 'file:///tmp/aptarchive/pool/main/v/vte/libvte9-udeb_0.28.2-5_amd64.udeb' libvte9-udeb_1%3a0.28.2-5_amd64.udeb 216968 MD5Sum:7da7201effaf5ced19abd9d0b45aa2c6
 'file:///tmp/aptarchive/pool/main/c/cdebconf-terminal/cdebconf-gtk-terminal_0.22_amd64.udeb' cdebconf-gtk-terminal_0.22_amd64.udeb 14734 MD5Sum:f9c3a7354560cb88e0396e2b7ba54363
 'file:///tmp/aptarchive/pool/main/c/cdebconf-terminal/cdebconf-newt-terminal_0.22_amd64.udeb' cdebconf-newt-terminal_0.22_amd64.udeb 4538 MD5Sum:20db6152fce5081fcbf49c7c08f21246"

test/integration/test-method-mirror

@@ -192,6 +192,16 @@ http://localhost:${APTHTTPPORT}/redirectme	type:deb
 testfailure apt update
 testrundownload 'foo=2'
 
+msgmsg 'Mirrors can be filtered by' 'by-hash type'
+echo "http://localhost:${APTHTTPPORT}/failure	type:foobar priority:1
+http://localhost:${APTHTTPPORT}/redirectme	type:index type:deb
+" > aptarchive/mirror.txt
+rm -rf rootdir/var/lib/apt/lists
+testsuccess apt update -o Acquire::By-Hash=force #-o Debug::pkgAcquire::Worker=1
+cp rootdir/tmp/testsuccess.output forcedbyhash.output
+testfailure grep "localhost:${APTHTTPPORT}/failure" forcedbyhash.output
+testrundownload 'foo=2'
+
 msgmsg 'The prefix for the mirrorlist is' 'passed on'
 echo 'Dir::Bin::Methods::foo+mirror+file "mirror";
 Dir::Bin::Methods::foo+mirror+http "mirror";

test/integration/test-method-rred

@@ -146,6 +146,7 @@ testrred 'Multi line change' 'more' '5,7c
  - even more good stuff
  - bonus good stuff
 $(tail -n 12 ./Packages)"
+testrred 'Patch file' 'empty' '' "$(cat ./Packages)"
 
 failrred() {
 	msgtest 'Failure caused by' "$1"
@@ -161,7 +162,6 @@ failrred 'Bogus content' '<html>
 </html>'
 
 # not a problem per-se, but we want our parser to be really strict
-failrred 'Empty patch file' ''
 failrred 'Empty line patch file' '
 '
 failrred 'Empty line before command' '
@@ -222,7 +222,8 @@ createpatch 'Remove more stuff and fix later' '23d,
 testsuccess apthelper cat-file --compress gzip Packages.ed-1
 mv rootdir/tmp/testsuccess.output Packages.ed-1.gz
 testsuccess rm Packages.ed-1
-createpatch 'Remove (old) dog paragraph' '10,19d' > Packages.ed-2
+touch Packages.ed-2 # an empty patch
+createpatch 'Remove (old) dog paragraph' '10,19d' > Packages.ed-3
 mergepatches '11,19c
 Package: extra-kittens
 Version: unavailable

test/integration/test-pdiff-usage

@@ -100,12 +100,15 @@ testrun() {
 	PATCHINDEX='aptarchive/Packages.diff/Index'
 	echo "SHA256-Current: $(sha256sum "${PKGFILE}-new" | cut -d' ' -f 1) $(stat -c%s "${PKGFILE}-new")
 SHA256-History:
+ $(sha256sum "$PKGFILE" | cut -d' ' -f 1) $(stat -c%s "$PKGFILE") 2000-08-18-2013.28
  01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b 33053002 2010-08-18-2013.28
  $(sha256sum "$PKGFILE" | cut -d' ' -f 1) $(stat -c%s "$PKGFILE") $(basename "$PATCHFILE")
 SHA256-Patches:
+ $(sha256sum "$PATCHFILE" | cut -d' ' -f 1) $(stat -c%s "$PATCHFILE") 2000-08-18-2013.28
  e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28
  $(sha256sum "$PATCHFILE" | cut -d' ' -f 1) $(stat -c%s "$PATCHFILE") $(basename "$PATCHFILE")
 SHA256-Download:
+ $(sha256sum "${PATCHFILE}.gz" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE}.gz") 2000-08-18-2013.28.gz
  d2a1b33187ed2d248eeae3b1223ea71791ea35f2138a713ed371332a6421f467 197 2010-08-18-2013.28.gz
  $(sha256sum "${PATCHFILE}.gz" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE}.gz") $(basename "${PATCHFILE}.gz")" > "$PATCHINDEX"
 
@@ -251,16 +254,10 @@ SHA256-Download:
 	cp Packages-future aptarchive/Packages
 	rm -f rootdir/var/lib/apt/lists/*_Contents-*
 	webserverconfig 'aptwebserver::overwrite::.*Contents-.*::filename' '/hacked-i386.gz'
-	# This should work in at least 4% of the cases...
-	for i in $(seq 25); do
-		testfailure apt update "$@"
-		if ! grep 'rred:600' rootdir/tmp/testfailure.output; then
-			break
-		fi
-	done
-	webserverconfig 'aptwebserver::overwrite::.*Contents-.*::filename' '/Contents-i386.gz'
+	testfailure apt update "$@"
 	cp rootdir/tmp/testfailure.output patchdownload.output
-	testfailure grep 'rred:600' patchdownload.output
+	webserverconfig 'aptwebserver::overwrite::.*Contents-.*::filename' '/Contents-i386.gz'
+	testfailure grep -q -z 'AbortTransaction.* [a-zA-Z0-9+-]\+:600%20' patchdownload.output
 	testnopackage newstuff futurestuff
 	testsuccessequal "$(cat "${PKGFILE}")
 " aptcache show apt oldstuff
@@ -281,7 +278,8 @@ SHA256-Download:
 	rm "${PATCHFILE}.gz"
 	testsuccess apt update "$@"
 	cp rootdir/tmp/testsuccess.output patchdownload.output
-	testsuccess grep '^Falling back to normal index file acquire' patchdownload.output
+	# it should be anchored on line start, but multiple processes on the same output stream…
+	testsuccess grep 'Falling back to normal index file acquire' patchdownload.output
 	testnopackage oldstuff
 	testsuccessequal "$(cat Packages-future)
 " aptcache show apt newstuff futurestuff

test/integration/test-phased-updates

@@ -59,7 +59,7 @@ phased3:
         500 file:${TMPWORKINGDIRECTORY}/aptarchive unstable/main all Packages
      50 500 (phased 50%)
         500 file:${TMPWORKINGDIRECTORY}/aptarchive unstable/main all Packages
-     10 500 (phased 10%)
+     10 1 (phased 10%)
         500 file:${TMPWORKINGDIRECTORY}/aptarchive unstable/main all Packages" aptcache policy phased1 phased2 phased3
 
 msgmsg "Test for always-include-phased-updates"
@@ -132,9 +132,9 @@ phased3:
   Version table:
      100 500
         500 file:${TMPWORKINGDIRECTORY}/aptarchive unstable/main all Packages
-     50 500 (phased 50%)
+     50 1 (phased 50%)
         500 file:${TMPWORKINGDIRECTORY}/aptarchive unstable/main all Packages
-     10 500 (phased 10%)
+     10 1 (phased 10%)
         500 file:${TMPWORKINGDIRECTORY}/aptarchive unstable/main all Packages" aptcache policy phased1 phased2 phased3 -o $never=true
 done
 

test/integration/test-ubuntu-bug-1921626-unsized-packages

@@ -0,0 +1,46 @@
+#!/bin/sh
+set -e
+
+TESTDIR="$(readlink -f "$(dirname "$0")")"
+. "$TESTDIR/framework"
+
+setupenvironment
+configarchitecture "i386"
+confighashes 'SHA512'
+
+mkdir tree
+head -c $((5*1024)) /dev/urandom > tree/data-file
+
+buildsimplenativepackage 'a' 'all' '1.0' 'stable'
+buildsimplenativepackage 'b' 'all' '1.0' 'stable' '' '' '' '' "$PWD/tree"
+buildsimplenativepackage 'c' 'all' '1.0' 'stable'
+
+setupaptarchive
+changetowebserver
+
+# Disable sandbox
+echo 'APT::Sandbox::User "root";' > rootdir/etc/apt/apt.conf.d/no-acquire-sandbox
+# XXX: Remove temporary hack
+rm rootdir/etc/apt/apt.conf.d/temporary-allow-unsized-packages
+
+testsuccess apt update
+
+for file in rootdir/var/lib/apt/lists/*Packages; do
+  awk '!(/^Size/ && ++cnt==2)'  $file > $file.new
+  mv $file.new $file
+done
+
+
+size_a=$(wc -c aptarchive/pool/a_1.0_all.deb | awk '{print $1}')
+size_b=$(wc -c aptarchive/pool/b_1.0_all.deb | awk '{print $1}')
+size_c=$(wc -c aptarchive/pool/c_1.0_all.deb | awk '{print $1}')
+
+testsuccessequal "Get:1 http://localhost:${APTHTTPPORT} stable/main all a all 1.0 [$size_a B]
+Get:2 http://localhost:${APTHTTPPORT} stable/main all b all 1.0 [$size_b B]
+Get:3 http://localhost:${APTHTTPPORT} stable/main all c all 1.0 [$size_c B]" apt download a b c -o Acquire::AllowUnsizedPackages=true
+
+rm *.deb
+testwarningequal "Get:1 http://localhost:${APTHTTPPORT} stable/main all a all 1.0 [$size_a B]
+Get:2 http://localhost:${APTHTTPPORT} stable/main all b all 1.0 [$size_b B]
+Get:3 http://localhost:${APTHTTPPORT} stable/main all c all 1.0 [$size_c B]
+W: Repository is broken: b:i386 (= 1.0) has no Size information" apt download a b c

test/integration/test-uri-encode-filename-field

@@ -0,0 +1,42 @@
+#!/bin/sh
+set -e
+
+TESTDIR="$(readlink -f "$(dirname "$0")")"
+. "$TESTDIR/framework"
+setupenvironment
+configarchitecture 'amd64'
+
+buildsimplenativepackage 'foo' 'all' '0+0~0'
+setupaptarchive --no-update
+
+runtest() {
+	rm -rf rootdir/var/lib/apt/lists
+	testsuccess apt update
+
+	cd downloaded
+	testsuccess apt download foo
+	testsuccess rm 'foo_0+0~0_all.deb'
+	testsuccess apt install foo
+
+	mv '../aptarchive/pool/foo_0+0~0_all.deb' '../aptarchive/pool/foo_0%3a0+0~0_all.deb'
+	testsuccess apt purge foo -y
+	testfailure apt download foo
+	testfailure apt install foo
+
+	sed -i -e 's#_0+0~0_#_0%3a0+0~0_#' ../rootdir/var/lib/apt/lists/*Packages
+	testsuccess apt download foo
+	# FIXME: we shouldn't take filename from file:/ in 'apt download'
+	if [ "$1" = 'file' ]; then
+		testsuccess rm 'foo_0%3a0+0~0_all.deb'
+	else
+		testsuccess rm 'foo_0+0~0_all.deb'
+	fi
+	testsuccess apt install foo
+
+	cd "$TMPWORKINGDIRECTORY" >/dev/null
+	mv 'aptarchive/pool/foo_0%3a0+0~0_all.deb' 'aptarchive/pool/foo_0+0~0_all.deb'
+}
+
+runtest 'file'
+changetowebserver
+runtest 'http'

test/libapt/authconf_test.cc

@@ -1,5 +1,6 @@
 #include <config.h>
 
+#include <apt-pkg/error.h>
 #include <apt-pkg/fileutl.h>
 #include <apt-pkg/netrc.h>
 #include <apt-pkg/strutl.h>
@@ -235,11 +236,15 @@ machine http://http.example login foo1 password bar
    EXPECT_EQ("foo1", U.User);
    EXPECT_EQ("bar", U.Password);
 
+   _error->PushToStack();
    EXPECT_TRUE(fd.Seek(0));
    U = URI("http://https.example/foo");
    EXPECT_TRUE(MaybeAddAuth(fd, U));
    EXPECT_TRUE(U.User.empty());
    EXPECT_TRUE(U.Password.empty());
+   EXPECT_FALSE(_error->empty());
+   EXPECT_TRUE(U.Password.empty());
+   _error->RevertToStack();
 
    EXPECT_TRUE(fd.Seek(0));
    U = URI("http://http.example/foo");

test/libapt/json_test.cc

@@ -0,0 +1,69 @@
+#include <config.h>
+#include "../../apt-private/private-cachefile.cc"
+#include "../../apt-private/private-json-hooks.cc"
+#include <gtest/gtest.h>
+#include <string>
+
+TEST(JsonTest, JsonString)
+{
+   std::ostringstream os;
+
+   // Check for escaping backslash and quotation marks, and ensure that we do not change number formatting
+   JsonWriter(os).value("H al\"l\\o").value(17);
+
+   EXPECT_EQ("\"H al\\u0022l\\u005Co\"17", os.str());
+
+   for (int i = 0; i <= 0x1F; i++)
+   {
+      os.str("");
+
+      JsonWriter(os).encodeString(os, std::string("X") + char(i) + "Z");
+
+      std::string exp;
+      strprintf(exp, "\"X\\u%04XZ\"", i);
+
+      EXPECT_EQ(exp, os.str());
+   }
+}
+
+TEST(JsonTest, JsonObject)
+{
+   std::ostringstream os;
+
+   JsonWriter(os).beginObject().name("key").value("value").endObject();
+
+   EXPECT_EQ("{\"key\":\"value\"}", os.str());
+}
+
+TEST(JsonTest, JsonArrayAndValues)
+{
+   std::ostringstream os;
+
+   JsonWriter(os).beginArray().value(0).value("value").value(1).value(true).endArray();
+
+   EXPECT_EQ("[0,\"value\",1,true]", os.str());
+}
+TEST(JsonTest, JsonStackRegression)
+{
+   std::ostringstream os;
+
+   JsonWriter w(os);
+
+   // Nest those things deeply such that we transition states:
+   //    object -> array -> object; -> array -> object
+   // Older versions never popped back and got stuck on array state.
+   w.beginObject();
+   w.name("a").beginArray().beginObject().endObject().endArray();
+   w.name("b").beginArray().beginObject().endObject().endArray();
+   w.endObject();
+
+   EXPECT_EQ("{\"a\":[{}],\"b\":[{}]}", os.str());
+}
+TEST(JsonTest, JsonNull)
+{
+   std::ostringstream os;
+
+   JsonWriter(os).value(nullptr);
+
+   EXPECT_EQ("null", os.str());
+}