- if ExecGPGV deals with a clear-signed file it will split this file

into data and signatures, pass it to gpgv for verification and
    recombines it after that in a known-good way without unsigned blocks
    and whitespaces resulting usually in more or less the same file as
    before, but later code can be sure about the format
* apt-pkg/deb/debmetaindex.cc:
  - reenable InRelease by default

apt-pkg/contrib/gpgv.cc

@@ -2,58 +2,54 @@
 // Include Files							/*{{{*/
 #include<config.h>
 
+#include <errno.h>
 #include <stdio.h>
+#include <string.h>
 #include <stdlib.h>
+#include <fcntl.h>
 #include <sys/stat.h>
 #include <sys/types.h>
-#include <fcntl.h>
-
-#include <vector>
+#include <sys/wait.h>
 
-#include <apt-pkg/error.h>
-#include <apt-pkg/strutl.h>
-#include <apt-pkg/fileutl.h>
-#include <apt-pkg/configuration.h>
+#include<apt-pkg/configuration.h>
+#include<apt-pkg/error.h>
+#include<apt-pkg/strutl.h>
+#include<apt-pkg/fileutl.h>
+#include<apt-pkg/gpgv.h>
 
 #include <apti18n.h>
 									/*}}}*/
+char * GenerateTemporaryFileTemplate(const char *basename)		/*{{{*/
+{
+   const char *tmpdir = getenv("TMPDIR");
+#ifdef P_tmpdir
+   if (!tmpdir)
+      tmpdir = P_tmpdir;
+#endif
+   if (!tmpdir)
+      tmpdir = "/tmp";
 
-using namespace std;
-
-// RunGPGV - returns the command needed for verify			/*{{{*/
+   std::string out;
+   strprintf(out,  "%s/%s.XXXXXX", tmpdir, basename);
+   return strdup(out.c_str());
+}
+									/*}}}*/
+// ExecGPGV - returns the command needed for verify			/*{{{*/
 // ---------------------------------------------------------------------
 /* Generating the commandline for calling gpgv is somehow complicated as
-   we need to add multiple keyrings and user supplied options. */
+   we need to add multiple keyrings and user supplied options.
+   Also, as gpgv has no options to enforce a certain reduced style of
+   clear-signed files (=the complete content of the file is signed and
+   the content isn't encoded) we do a divide and conquer approach here
+*/
 void ExecGPGV(std::string const &File, std::string const &FileGPG,
-			int const &statusfd, int fd[2])
+             int const &statusfd, int fd[2])
 {
    #define EINTERNAL 111
-
-   if (File == FileGPG)
-   {
-      #define SIGMSG "-----BEGIN PGP SIGNED MESSAGE-----\n"
-      char buffer[sizeof(SIGMSG)];
-      FILE* gpg = fopen(File.c_str(), "r");
-      if (gpg == NULL)
-      {
-	 ioprintf(std::cerr, _("Could not open file %s"), File.c_str());
-	 exit(EINTERNAL);
-      }
-      char const * const test = fgets(buffer, sizeof(buffer), gpg);
-      fclose(gpg);
-      if (test == NULL || strcmp(buffer, SIGMSG) != 0)
-      {
-	 ioprintf(std::cerr, _("File %s doesn't start with a clearsigned message"), File.c_str());
-	 exit(EINTERNAL);
-      }
-      #undef SIGMSG
-   }
-
-
-   string const gpgvpath = _config->Find("Dir::Bin::gpg", "/usr/bin/gpgv");
+   std::string const gpgvpath = _config->Find("Dir::Bin::gpg", "/usr/bin/gpgv");
    // FIXME: remove support for deprecated APT::GPGV setting
-   string const trustedFile = _config->Find("APT::GPGV::TrustedKeyring", _config->FindFile("Dir::Etc::Trusted"));
-   string const trustedPath = _config->FindDir("Dir::Etc::TrustedParts");
+   std::string const trustedFile = _config->Find("APT::GPGV::TrustedKeyring", _config->FindFile("Dir::Etc::Trusted"));
+   std::string const trustedPath = _config->FindDir("Dir::Etc::TrustedParts");
 
    bool const Debug = _config->FindB("Debug::Acquire::gpgv", false);
 
@@ -64,7 +60,7 @@ void ExecGPGV(std::string const &File, std::string const &FileGPG,
       std::clog << "Keyring path: " << trustedPath << std::endl;
    }
 
-   std::vector<string> keyrings;
+   std::vector<std::string> keyrings;
    if (DirectoryExists(trustedPath))
      keyrings = GetListOfFilesInDir(trustedPath, "gpg", false, true);
    if (RealFileExists(trustedFile) == true)
@@ -88,11 +84,11 @@ void ExecGPGV(std::string const &File, std::string const &FileGPG,
    if (statusfd != -1)
    {
       Args.push_back("--status-fd");
-      snprintf(statusfdstr, sizeof(fd), "%i", statusfd);
+      snprintf(statusfdstr, sizeof(statusfdstr), "%i", statusfd);
       Args.push_back(statusfdstr);
    }
 
-   for (vector<string>::const_iterator K = keyrings.begin();
+   for (std::vector<std::string>::const_iterator K = keyrings.begin();
 	K != keyrings.end(); ++K)
    {
       Args.push_back("--keyring");
@@ -112,9 +108,49 @@ void ExecGPGV(std::string const &File, std::string const &FileGPG,
       }
    }
 
-   Args.push_back(FileGPG.c_str());
+   int sigFd = -1;
+   int dataFd = -1;
+   std::vector<std::string> dataHeader;
+   char * sig = NULL;
+   char * data = NULL;
+
+   // file with detached signature
    if (FileGPG != File)
+   {
+      Args.push_back(FileGPG.c_str());
       Args.push_back(File.c_str());
+   }
+   else // clear-signed file
+   {
+      sig = GenerateTemporaryFileTemplate("apt.sig");
+      data = GenerateTemporaryFileTemplate("apt.data");
+      if (sig == NULL || data == NULL)
+      {
+	 ioprintf(std::cerr, "Couldn't create tempfiles for splitting up %s", File.c_str());
+	 exit(EINTERNAL);
+      }
+
+      sigFd = mkstemp(sig);
+      dataFd = mkstemp(data);
+      int const duppedSigFd = dup(sigFd);
+      int const duppedDataFd = dup(dataFd);
+
+      if (dataFd == -1 || sigFd == -1 || duppedDataFd == -1 || duppedSigFd == -1 ||
+	    SplitClearSignedFile(File, duppedDataFd, &dataHeader, duppedSigFd) == false)
+      {
+	 if (dataFd != -1)
+	    unlink(sig);
+	 if (sigFd != -1)
+	    unlink(data);
+	 ioprintf(std::cerr, "Splitting up %s into data and signature failed", File.c_str());
+	 exit(EINTERNAL);
+      }
+      lseek(dataFd, 0, SEEK_SET);
+      lseek(sigFd, 0, SEEK_SET);
+      Args.push_back(sig);
+      Args.push_back(data);
+   }
+
    Args.push_back(NULL);
 
    if (Debug == true)
@@ -142,8 +178,186 @@ void ExecGPGV(std::string const &File, std::string const &FileGPG,
       putenv((char *)"LC_MESSAGES=");
    }
 
-   execvp(gpgvpath.c_str(), (char **) &Args[0]);
-   ioprintf(std::cerr, "Couldn't execute %s to check %s", Args[0], File.c_str());
-   exit(EINTERNAL);
+   if (FileGPG != File)
+   {
+      execvp(gpgvpath.c_str(), (char **) &Args[0]);
+      ioprintf(std::cerr, "Couldn't execute %s to check %s", Args[0], File.c_str());
+      exit(EINTERNAL);
+   }
+   else
+   {
+//#define UNLINK_EXIT(X) exit(X)
+#define UNLINK_EXIT(X) unlink(sig);unlink(data);exit(X)
+
+      // for clear-signed files we have created tempfiles we have to clean up
+      // and we do an additional check, so fork yet another time …
+      pid_t pid = ExecFork();
+      if(pid < 0) {
+	 ioprintf(std::cerr, "Fork failed for %s to check %s", Args[0], File.c_str());
+	 UNLINK_EXIT(EINTERNAL);
+      }
+      if(pid == 0)
+      {
+	 if (statusfd != -1)
+	    dup2(fd[1], statusfd);
+	 execvp(gpgvpath.c_str(), (char **) &Args[0]);
+	 ioprintf(std::cerr, "Couldn't execute %s to check %s", Args[0], File.c_str());
+	 UNLINK_EXIT(EINTERNAL);
+      }
+
+      // Wait and collect the error code - taken from WaitPid as we need the exact Status
+      int Status;
+      while (waitpid(pid,&Status,0) != pid)
+      {
+	 if (errno == EINTR)
+	    continue;
+	 ioprintf(std::cerr, _("Waited for %s but it wasn't there"), "gpgv");
+	 UNLINK_EXIT(EINTERNAL);
+      }
+
+      // check if it exit'ed normally …
+      if (WIFEXITED(Status) == false)
+      {
+	 ioprintf(std::cerr, _("Sub-process %s exited unexpectedly"), "gpgv");
+	 UNLINK_EXIT(EINTERNAL);
+      }
+
+      // … and with a good exit code
+      if (WEXITSTATUS(Status) != 0)
+      {
+	 ioprintf(std::cerr, _("Sub-process %s returned an error code (%u)"), "gpgv", WEXITSTATUS(Status));
+	 UNLINK_EXIT(WEXITSTATUS(Status));
+      }
+
+      /* looks like its fine. Our caller will check the status fd,
+	 but we construct a good-known clear-signed file without garbage
+	 and other non-sense. In a perfect world, we get the same file,
+	 but empty lines, trailing whitespaces and stuff makes it inperfect … */
+      if (RecombineToClearSignedFile(File, dataFd, dataHeader, sigFd) == false)
+      {
+	 _error->DumpErrors(std::cerr);
+	 UNLINK_EXIT(EINTERNAL);
+      }
+
+      // everything fine, we have a clean file now!
+      UNLINK_EXIT(0);
+#undef UNLINK_EXIT
+   }
+   exit(EINTERNAL); // unreachable safe-guard
+}
+									/*}}}*/
+// RecombineToClearSignedFile - combine data/signature to message	/*{{{*/
+bool RecombineToClearSignedFile(std::string const &OutFile, int const ContentFile,
+      std::vector<std::string> const &ContentHeader, int const SignatureFile)
+{
+   FILE *clean_file = fopen(OutFile.c_str(), "w");
+   fputs("-----BEGIN PGP SIGNED MESSAGE-----\n", clean_file);
+   for (std::vector<std::string>::const_iterator h = ContentHeader.begin(); h != ContentHeader.end(); ++h)
+      fprintf(clean_file, "%s\n", h->c_str());
+   fputs("\n", clean_file);
+
+   FILE *data_file = fdopen(ContentFile, "r");
+   FILE *sig_file = fdopen(SignatureFile, "r");
+   if (data_file == NULL || sig_file == NULL)
+      return _error->Error("Couldn't open splitfiles to recombine them into %s", OutFile.c_str());
+   char *buf = NULL;
+   size_t buf_size = 0;
+   while (getline(&buf, &buf_size, data_file) != -1)
+      fputs(buf, clean_file);
+   fclose(data_file);
+   fputs("\n", clean_file);
+   while (getline(&buf, &buf_size, sig_file) != -1)
+      fputs(buf, clean_file);
+   fclose(sig_file);
+   fclose(clean_file);
+   return true;
 }
 									/*}}}*/
+// SplitClearSignedFile - split message into data/signature		/*{{{*/
+bool SplitClearSignedFile(std::string const &InFile, int const ContentFile,
+      std::vector<std::string> * const ContentHeader, int const SignatureFile)
+{
+   FILE *in = fopen(InFile.c_str(), "r");
+   if (in == NULL)
+      return _error->Errno("fopen", "can not open %s", InFile.c_str());
+
+   FILE *out_content = NULL;
+   FILE *out_signature = NULL;
+   if (ContentFile != -1)
+   {
+      out_content = fdopen(ContentFile, "w");
+      if (out_content == NULL)
+	 return _error->Errno("fdopen", "Failed to open file to write content to from %s", InFile.c_str());
+   }
+   if (SignatureFile != -1)
+   {
+      out_signature = fdopen(SignatureFile, "w");
+      if (out_signature == NULL)
+	 return _error->Errno("fdopen", "Failed to open file to write signature to from %s", InFile.c_str());
+   }
+
+   bool found_message_start = false;
+   bool found_message_end = false;
+   bool skip_until_empty_line = false;
+   bool found_signature = false;
+   bool first_line = true;
+
+   char *buf = NULL;
+   size_t buf_size = 0;
+   while (getline(&buf, &buf_size, in) != -1)
+   {
+      _strrstrip(buf);
+      if (found_message_start == false)
+      {
+	 if (strcmp(buf, "-----BEGIN PGP SIGNED MESSAGE-----") == 0)
+	 {
+	    found_message_start = true;
+	    skip_until_empty_line = true;
+	 }
+      }
+      else if (skip_until_empty_line == true)
+      {
+	 if (strlen(buf) == 0)
+	    skip_until_empty_line = false;
+	 // save "Hash" Armor Headers, others aren't allowed
+	 else if (ContentHeader != NULL && strncmp(buf, "Hash: ", strlen("Hash: ")) == 0)
+	    ContentHeader->push_back(buf);
+      }
+      else if (found_signature == false)
+      {
+	 if (strcmp(buf, "-----BEGIN PGP SIGNATURE-----") == 0)
+	 {
+	    found_signature = true;
+	    found_message_end = true;
+	    if (out_signature != NULL)
+	       fprintf(out_signature, "%s\n", buf);
+	 }
+	 else if (found_message_end == false)
+	 {
+	    // we are in the message block
+	    if(first_line == true) // first line does not need a newline
+	    {
+	       if (out_content != NULL)
+		  fprintf(out_content, "%s", buf);
+	       first_line = false;
+	    }
+	    else if (out_content != NULL)
+	       fprintf(out_content, "\n%s", buf);
+	 }
+      }
+      else if (found_signature == true)
+      {
+	 if (out_signature != NULL)
+	    fprintf(out_signature, "%s\n", buf);
+	 if (strcmp(buf, "-----END PGP SIGNATURE-----") == 0)
+	    found_signature = false; // look for other signatures
+      }
+      // all the rest is whitespace, unsigned garbage or additional message blocks we ignore
+   }
+   if (out_content != NULL)
+      fclose(out_content);
+   if (out_signature != NULL)
+      fclose(out_signature);
+
+   return true;
+}

apt-pkg/contrib/gpgv.h

@@ -10,6 +10,7 @@
 #define CONTRIB_GPGV_H
 
 #include <string>
+#include <vector>
 
 #if __GNUC__ >= 4
 	#define APT_noreturn	__attribute__ ((noreturn))
@@ -20,7 +21,9 @@
 /** \brief generates and run the command to verify a file with gpgv
  *
  * If File and FileSig specify the same file it is assumed that we
- * deal with a clear-signed message.
+ * deal with a clear-signed message. In that case the file will be
+ * rewritten to be in a good-known format without uneeded whitespaces
+ * and additional messages (unsigned or signed).
  *
  * @param File is the message (unsigned or clear-signed)
  * @param FileSig is the signature (detached or clear-signed)
@@ -35,4 +38,40 @@ inline void ExecGPGV(std::string const &File, std::string const &FileSig,
 
 #undef APT_noreturn
 
+/** \brief Split an inline signature into message and signature
+ *
+ *  Takes a clear-signed message and puts the first signed message
+ *  in the content file and all signatures following it into the
+ *  second. Unsigned messages, additional messages as well as
+ *  whitespaces are discarded. The resulting files are suitable to
+ *  be checked with gpgv.
+ *
+ *  If one or all Fds are -1 they will not be used and the content
+ *  which would have been written to them is discarded.
+ *
+ *  The code doesn't support dash-encoded lines as these are not
+ *  expected to be present in files we have to deal with.
+ *
+ *  @param InFile is the clear-signed file
+ *  @param ContentFile is the Fd the message will be written to
+ *  @param ContentHeader is a list of all required Amored Headers for the message
+ *  @param SignatureFile is the Fd all signatures will be written to
+ */
+bool SplitClearSignedFile(std::string const &InFile, int const ContentFile,
+      std::vector<std::string> * const ContentHeader, int const SignatureFile);
+
+/** \brief recombines message and signature to an inline signature
+ *
+ *  Reverses the splitting down by #SplitClearSignedFile by writing
+ *  a well-formed clear-signed message without unsigned messages,
+ *  additional signed messages or just trailing whitespaces
+ *
+ *  @param OutFile will be clear-signed file
+ *  @param ContentFile is the Fd the message will be read from
+ *  @param ContentHeader is a list of all required Amored Headers for the message
+ *  @param SignatureFile is the Fd all signatures will be read from
+ */
+bool RecombineToClearSignedFile(std::string const &OutFile, int const ContentFile,
+      std::vector<std::string> const &ContentHeader, int const SignatureFile);
+
 #endif

apt-pkg/deb/debmetaindex.cc

@@ -229,6 +229,8 @@ vector <struct IndexTarget *>* debReleaseIndex::ComputeIndexTargets() const {
 									/*}}}*/
 bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll) const
 {
+   bool const tryInRelease = _config->FindB("Acquire::TryInRelease", true);
+
    // special case for --print-uris
    if (GetAll) {
       vector <struct IndexTarget *> *targets = ComputeIndexTargets();
@@ -239,18 +241,27 @@ bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll) const
 
       // this is normally created in pkgAcqMetaSig, but if we run
       // in --print-uris mode, we add it here
-      new pkgAcqMetaIndex(Owner, MetaIndexURI("Release"),
-			  MetaIndexInfo("Release"), "Release",
-			  MetaIndexURI("Release.gpg"),
-			  ComputeIndexTargets(),
-			  new indexRecords (Dist));
+      if (tryInRelease == false)
+	 new pkgAcqMetaIndex(Owner, MetaIndexURI("Release"),
+	       MetaIndexInfo("Release"), "Release",
+	       MetaIndexURI("Release.gpg"),
+	       ComputeIndexTargets(),
+	       new indexRecords (Dist));
    }
 
-   new pkgAcqMetaSig(Owner, MetaIndexURI("Release.gpg"),
-		     MetaIndexInfo("Release.gpg"), "Release.gpg",
-		     MetaIndexURI("Release"), MetaIndexInfo("Release"), "Release",
-		     ComputeIndexTargets(),
-		     new indexRecords (Dist));
+   if (tryInRelease == true)
+      new pkgAcqMetaClearSig(Owner, MetaIndexURI("InRelease"),
+	    MetaIndexInfo("InRelease"), "InRelease",
+	    MetaIndexURI("Release"), MetaIndexInfo("Release"), "Release",
+	    MetaIndexURI("Release.gpg"), MetaIndexInfo("Release.gpg"), "Release.gpg",
+	    ComputeIndexTargets(),
+	    new indexRecords (Dist));
+   else
+      new pkgAcqMetaSig(Owner, MetaIndexURI("Release.gpg"),
+	    MetaIndexInfo("Release.gpg"), "Release.gpg",
+	    MetaIndexURI("Release"), MetaIndexInfo("Release"), "Release",
+	    ComputeIndexTargets(),
+	    new indexRecords (Dist));
 
    return true;
 }

debian/changelog

@@ -7,9 +7,16 @@ apt (0.9.7.9) UNRELEASED; urgency=low
     - ExecGPGV is a method which should never return, so mark it as such
       and fix the inconsistency of returning in error cases
     - don't close stdout/stderr if it is also the statusfd
+    - if ExecGPGV deals with a clear-signed file it will split this file
+      into data and signatures, pass it to gpgv for verification and
+      recombines it after that in a known-good way without unsigned blocks
+      and whitespaces resulting usually in more or less the same file as
+      before, but later code can be sure about the format
   * apt-pkg/acquire-item.cc:
     - keep the last good InRelease file around just as we do it with
       Release.gpg in case the new one we download isn't good for us
+  * apt-pkg/deb/debmetaindex.cc:
+    - reenable InRelease by default
 
  -- David Kalnischkies <kalnischkies@gmail.com>  Fri, 15 Mar 2013 14:15:43 +0100
 

test/integration/test-ubuntu-bug-784473-InRelease-one-message-only

@@ -26,6 +26,14 @@ MD5Sum:
  2182897e0a2a0c09e760beaae117a015     2023 Packages.diff/Index
  1b895931853981ad8204d2439821b999     4144 Packages.gz'; echo; cat ${RELEASE}.old;) > ${RELEASE}
 done
-aptget update -qq > /dev/null 2> starts-with-unsigned.msg
-sed -i 's#File .*InRelease#File InRelease#' starts-with-unsigned.msg
-testfileequal starts-with-unsigned.msg "W: GPG error: file: unstable InRelease: File InRelease doesn't start with a clearsigned message"
+
+msgtest 'The unsigned garbage before signed block is' 'ignored'
+aptget update -qq > /dev/null 2>&1 && msgpass || msgfail
+
+ROOTDIR="$(readlink -f .)"
+testequal "Package files:
+ 100 ${ROOTDIR}/rootdir/var/lib/dpkg/status
+     release a=now
+ 500 file:${ROOTDIR}/aptarchive/ unstable/main i386 Packages
+     release a=unstable,n=unstable,c=main
+Pinned packages:" aptcache policy