* merge the remaining Ubuntu change:

- on gpg verification failure warn and restore the last known
    good state
  - on failure display the IP of the server (useful for servers
    that use round robin DNS)
  - support Original-Maintainer in RewritePackageOrder
  - enable cdrom autodetection via libudev by default
  - show messsage about Vcs in use when apt-get source is run for
    packages maintained in a Vcs
  - better support transitional packages with mark auto-installed. 
    when the transitional package is in "oldlibs" the new package
    is not marked auto installed (same is true for section
    metapackages)
  - provide new "deb mirror://archive.foo/mirrors.list sid main"
    method expects a list of mirrors (generated on the server e.g.
    via geoip) and will use that, including cycle on failure
  - write apport crash file on package failure (disabled by default
    on debian until apport is available)
  - support mirror failure reporting (disabled by default on debian)

apt-pkg/acquire-item.cc

@@ -64,6 +64,7 @@ void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 {
    Status = StatIdle;
    ErrorText = LookupTag(Message,"Message");
+   UsedMirror =  LookupTag(Message,"UsedMirror");
    if (QueueCounter <= 1)
    {
       /* This indicates that the file is not available right now but might
@@ -76,10 +77,17 @@ void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 	 Dequeue();
 	 return;
       }
-      
+
       Status = StatError;
       Dequeue();
    }   
+
+   // report mirror failure back to LP if we actually use a mirror
+   string FailReason = LookupTag(Message, "FailReason");
+   if(FailReason.size() != 0)
+      ReportMirrorFailure(FailReason);
+   else
+      ReportMirrorFailure(ErrorText);
 }
 									/*}}}*/
 // Acquire::Item::Start - Item has begun to download			/*{{{*/
@@ -101,7 +109,7 @@ void pkgAcquire::Item::Done(string Message,unsigned long Size,string Hash,
 {
    // We just downloaded something..
    string FileName = LookupTag(Message,"Filename");
-   // we only inform the Log class if it was actually not a local thing
+   UsedMirror =  LookupTag(Message,"UsedMirror");
    if (Complete == false && !Local && FileName == DestFile)
    {
       if (Owner->Log != 0)
@@ -110,7 +118,6 @@ void pkgAcquire::Item::Done(string Message,unsigned long Size,string Hash,
 
    if (FileSize == 0)
       FileSize= Size;
-   
    Status = StatDone;
    ErrorText = string();
    Owner->Dequeue(this);
@@ -132,6 +139,49 @@ void pkgAcquire::Item::Rename(string From,string To)
    }   
 }
 									/*}}}*/
+
+void pkgAcquire::Item::ReportMirrorFailure(string FailCode)
+{
+   // we only act if a mirror was used at all
+   if(UsedMirror.empty())
+      return;
+#if 0
+   std::cerr << "\nReportMirrorFailure: " 
+	     << UsedMirror
+	     << " Uri: " << DescURI()
+	     << " FailCode: " 
+	     << FailCode << std::endl;
+#endif
+   const char *Args[40];
+   unsigned int i = 0;
+   string report = _config->Find("Methods::Mirror::ProblemReporting", 
+				 "/usr/lib/apt/apt-report-mirror-failure");
+   if(!FileExists(report))
+      return;
+   Args[i++] = report.c_str();
+   Args[i++] = UsedMirror.c_str();
+   Args[i++] = DescURI().c_str();
+   Args[i++] = FailCode.c_str();
+   Args[i++] = NULL;
+   pid_t pid = ExecFork();
+   if(pid < 0) 
+   {
+      _error->Error("ReportMirrorFailure Fork failed");
+      return;
+   }
+   else if(pid == 0) 
+   {
+      execvp(Args[0], (char**)Args);
+      std::cerr << "Could not exec " << Args[0] << std::endl;
+      _exit(100);
+   }
+   if(!ExecWait(pid, "report-mirror-failure")) 
+   {
+      _error->Warning("Couldn't report problem to '%s'",
+		      _config->Find("Methods::Mirror::ProblemReporting").c_str());
+   }
+}
+
 // AcqDiffIndex::AcqDiffIndex - Constructor				/*{{{*/
 // ---------------------------------------------------------------------
 /* Get the DiffIndex file first and see if there are patches availabe 
@@ -624,7 +674,6 @@ string pkgAcqIndex::Custom600Headers()
    struct stat Buf;
    if (stat(Final.c_str(),&Buf) != 0)
       return "\nIndex-File: true";
-   
    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 }
 									/*}}}*/
@@ -692,6 +741,7 @@ void pkgAcqIndex::Done(string Message,unsigned long Size,string Hash,
          Status = StatAuthError;
          ErrorText = _("Hash Sum mismatch");
          Rename(DestFile,DestFile + ".FAILED");
+	 ReportMirrorFailure("HashChecksumFailure");
          return;
       }
       // Done, move it into position
@@ -882,8 +932,9 @@ void pkgAcqMetaSig::Done(string Message,unsigned long Size,string MD5,
       Rename(LastGoodSig, DestFile);
 
    // queue a pkgAcqMetaIndex to be verified against the sig we just retrieved
-   new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
-		       DestFile, IndexTargets, MetaIndexParser);
+   new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, 
+		       MetaIndexShortDesc,  DestFile, IndexTargets, 
+		       MetaIndexParser);
 
 }
 									/*}}}*/
@@ -896,7 +947,7 @@ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
    {
       Item::Failed(Message,Cnf);
       // move the sigfile back on transient network failures 
-      if(FileExists(DestFile))
+      if(FileExists(LastGoodSig))
  	 Rename(LastGoodSig,Final);
 
       // set the status back to , Item::Failed likes to reset it
@@ -971,6 +1022,15 @@ void pkgAcqMetaIndex::Done(string Message,unsigned long Size,string Hash,	/*{{{*
    if (AuthPass == true)
    {
       AuthDone(Message);
+
+      // all cool, move Release file into place
+      Complete = true;
+
+      string FinalFile = _config->FindDir("Dir::State::lists");
+      FinalFile += URItoFileName(RealURI);
+      Rename(DestFile,FinalFile);
+      chmod(FinalFile.c_str(),0644);
+      DestFile = FinalFile;
    }
    else
    {
@@ -1022,22 +1082,15 @@ void pkgAcqMetaIndex::RetrievalDone(string Message)			/*{{{*/
       return;
    }
 
-   // see if the download was a IMSHit
+   // make sure to verify against the right file on I-M-S hit
    IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false);
+   if(IMSHit)
+   {
+      string FinalFile = _config->FindDir("Dir::State::lists");
+      FinalFile += URItoFileName(RealURI);
+      DestFile = FinalFile;
+   }
    Complete = true;
-
-   string FinalFile = _config->FindDir("Dir::State::lists");
-   FinalFile += URItoFileName(RealURI);
-
-   // If we get a IMS hit we can remove the empty file in partial
-   // othersie we move the file in place
-   if (IMSHit)
-      unlink(DestFile.c_str());
-   else
-      Rename(DestFile,FinalFile);
-
-   chmod(FinalFile.c_str(),0644);
-   DestFile = FinalFile;
 }
 									/*}}}*/
 void pkgAcqMetaIndex::AuthDone(string Message)				/*{{{*/
@@ -1067,7 +1120,6 @@ void pkgAcqMetaIndex::AuthDone(string Message)				/*{{{*/
    QueueIndexes(true);
 
    // Done, move signature file into position
-
    string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
       URItoFileName(RealURI) + ".gpg";
    Rename(SigFile,VerifiedSigFile);
@@ -1108,7 +1160,7 @@ void pkgAcqMetaIndex::QueueIndexes(bool verify)				/*{{{*/
       
       // Queue Packages file (either diff or full packages files, depending
       // on the users option)
-      if(_config->FindB("Acquire::PDiffs",true) == true) 
+      if(_config->FindB("Acquire::PDiffs", true) == true) 
 	 new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
 			     (*Target)->ShortDesc, ExpectedIndexHash);
       else 
@@ -1211,30 +1263,30 @@ void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 {
    if (AuthPass == true)
    {
-      // if we fail the authentication but got the file via a IMS-Hit 
-      // this means that the file wasn't downloaded and that it might be
-      // just stale (server problem, proxy etc). we delete what we have
-      // queue it again without i-m-s 
-      // alternatively we could just unlink the file and let the user try again
-      if (IMSHit)
+      // gpgv method failed, if we have a good signature 
+      string LastGoodSigFile = _config->FindDir("Dir::State::lists") +
+	 "partial/" + URItoFileName(RealURI) + ".gpg.reverify";
+      if(FileExists(LastGoodSigFile))
       {
-	 Complete = false;
-	 Local = false;
-	 AuthPass = false;
-	 unlink(DestFile.c_str());
-
-	 DestFile = _config->FindDir("Dir::State::lists") + "partial/";
-	 DestFile += URItoFileName(RealURI);
-	 Desc.URI = RealURI;
-	 QueueURI(Desc);
+	 string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
+	    URItoFileName(RealURI) + ".gpg";
+	 Rename(LastGoodSigFile,VerifiedSigFile);
+	 Status = StatTransientNetworkError;
+	 _error->Warning(_("A error occurred during the signature "
+			   "verification. The repository is not updated "
+			   "and the previous index files will be used."
+			   "GPG error: %s: %s\n"),
+			 Desc.Description.c_str(),
+			 LookupTag(Message,"Message").c_str());
+	 RunScripts("APT::Update::Auth-Failure");
 	 return;
+      } else {
+	 _error->Warning(_("GPG error: %s: %s"),
+			 Desc.Description.c_str(),
+			 LookupTag(Message,"Message").c_str());
       }
-
       // gpgv method failed 
-      _error->Warning("GPG error: %s: %s",
-                      Desc.Description.c_str(),
-                      LookupTag(Message,"Message").c_str());
-
+      ReportMirrorFailure("GPGFailure");
    }
 
    // No Release file was present, or verification failed, so fall

apt-pkg/acquire-item.h

@@ -143,6 +143,7 @@ class pkgAcquire::Item : public WeakPointable
     *  download progress indicator's overall statistics.
     */
    bool Local;
+   string UsedMirror;
 
    /** \brief The number of fetch queues into which this item has been
     *  inserted.
@@ -243,6 +244,17 @@ class pkgAcquire::Item : public WeakPointable
 
    /** \return \b true if this object is being fetched from a trusted source. */
    virtual bool IsTrusted() {return false;};
+   
+   // report mirror problems
+   /** \brief Report mirror problem
+    * 
+    *  This allows reporting mirror failures back to a centralized
+    *  server. The apt-report-mirror-failure script is called for this
+    * 
+    *  \param FailCode A short failure string that is send
+    */
+   void ReportMirrorFailure(string FailCode);
+
 
    /** \brief Initialize an item.
     *
@@ -551,7 +563,8 @@ class pkgAcqIndex : public pkgAcquire::Item
     *  fallback is ".gz" or none.
     */
    pkgAcqIndex(pkgAcquire *Owner,string URI,string URIDesc,
-	       string ShortDesc, HashString ExpectedHash, string compressExt="");
+	       string ShortDesc, HashString ExpectedHash, 
+	       string compressExt="");
 };
 									/*}}}*/
 /** \brief An acquire item that is responsible for fetching a		{{{
@@ -614,7 +627,6 @@ class pkgAcqMetaSig : public pkgAcquire::Item
    /** \brief The last good signature file */
    string LastGoodSig;
 
-
    /** \brief The fetch request that is currently being processed. */
    pkgAcquire::ItemDesc Desc;
 

apt-pkg/acquire-method.cc

@@ -96,12 +96,11 @@ void pkgAcqMethod::Fail(string Err,bool Transient)
    }
    
    char S[1024];
+   char *End = S;
    if (Queue != 0)
    {
-      snprintf(S,sizeof(S)-50,"400 URI Failure\nURI: %s\n"
-	       "Message: %s %s\n",Queue->Uri.c_str(),Err.c_str(),
-	       FailExtra.c_str());
-
+      End += snprintf(S,sizeof(S)-50,"400 URI Failure\nURI: %s\n"
+		      "Message: %s %s\n",Queue->Uri.c_str(), Err.c_str(), IP.c_str());
       // Dequeue
       FetchItem *Tmp = Queue;
       Queue = Queue->Next;
@@ -110,10 +109,14 @@ void pkgAcqMethod::Fail(string Err,bool Transient)
 	 QueueBack = Queue;
    }
    else
-      snprintf(S,sizeof(S)-50,"400 URI Failure\nURI: <UNKNOWN>\n"
-	       "Message: %s %s\n",Err.c_str(),
-	       FailExtra.c_str());
-      
+   {
+      End += snprintf(S,sizeof(S)-50,"400 URI Failure\nURI: <UNKNOWN>\n"
+		      "Message: %s\n",Err.c_str());
+   }
+   if(FailReason.empty() == false)
+      End += snprintf(End,sizeof(S)-50 - (End - S),"FailReason: %s\n",FailReason.c_str());
+   if (UsedMirror.empty() == false)
+      End += snprintf(End,sizeof(S)-50 - (End - S),"UsedMirror: %s\n",UsedMirror.c_str());
    // Set the transient flag 
    if (Transient == true)
       strcat(S,"Transient-Failure: true\n\n");
@@ -184,6 +187,8 @@ void pkgAcqMethod::URIDone(FetchResult &Res, FetchResult *Alt)
       End += snprintf(End,sizeof(S)-50 - (End - S),"SHA1-Hash: %s\n",Res.SHA1Sum.c_str());
    if (Res.SHA256Sum.empty() == false)
       End += snprintf(End,sizeof(S)-50 - (End - S),"SHA256-Hash: %s\n",Res.SHA256Sum.c_str());
+   if (UsedMirror.empty() == false)
+      End += snprintf(End,sizeof(S)-50 - (End - S),"UsedMirror: %s\n",UsedMirror.c_str());
    if (Res.GPGVOutput.size() > 0)
       End += snprintf(End,sizeof(S)-50 - (End - S),"GPGVOutput:\n");     
    for (vector<string>::iterator I = Res.GPGVOutput.begin();

apt-pkg/acquire-method.h

@@ -59,7 +59,9 @@ class pkgAcqMethod
    vector<string> Messages;
    FetchItem *Queue;
    FetchItem *QueueBack;
-   string FailExtra;
+   string FailReason;
+   string UsedMirror;
+   string IP;
    
    // Handlers for messages
    virtual bool Configuration(string Message);
@@ -68,14 +70,14 @@ class pkgAcqMethod
    // Outgoing messages
    void Fail(bool Transient = false);
    inline void Fail(const char *Why, bool Transient = false) {Fail(string(Why),Transient);};
-   void Fail(string Why, bool Transient = false);
-   void URIStart(FetchResult &Res);
-   void URIDone(FetchResult &Res,FetchResult *Alt = 0);
+   virtual void Fail(string Why, bool Transient = false);
+   virtual void URIStart(FetchResult &Res);
+   virtual void URIDone(FetchResult &Res,FetchResult *Alt = 0);
+
    bool MediaFail(string Required,string Drive);
    virtual void Exit() {};
 
    public:
-
    enum CnfFlags {SingleInstance = (1<<0),
                   Pipeline = (1<<1), SendConfig = (1<<2),
                   LocalOnly = (1<<3), NeedsCleanup = (1<<4), 
@@ -87,7 +89,8 @@ class pkgAcqMethod
    void Redirect(const string &NewURI);
  
    int Run(bool Single = false);
-   inline void SetFailExtraMsg(string Msg) {FailExtra = Msg;};
+   inline void SetFailReason(string Msg) {FailReason = Msg;};
+   inline void SetIP(string aIP) {IP = aIP;};
    
    pkgAcqMethod(const char *Ver,unsigned long Flags = 0);
    virtual ~pkgAcqMethod() {};

apt-pkg/algorithms.cc

@@ -1184,8 +1184,7 @@ bool pkgProblemResolver::Resolve(bool BrokenFix)
       return _error->Error(_("Unable to correct problems, you have held broken packages."));
    }
    
-   // set the auto-flags (mvo: I'm not sure if we _really_ need this, but
-   // I didn't managed 
+   // set the auto-flags (mvo: I'm not sure if we _really_ need this)
    pkgCache::PkgIterator I = Cache.PkgBegin();
    for (;I.end() != true; I++) {
       if (Cache[I].NewInstall() && !(Flags[I->ID] & PreInstalled)) {

apt-pkg/deb/debsystem.cc

@@ -18,7 +18,6 @@
 #include <apt-pkg/error.h>
 #include <apt-pkg/fileutl.h>
 #include <apti18n.h>
-    
 #include <sys/types.h>
 #include <unistd.h>
 #include <dirent.h>
@@ -79,8 +78,15 @@ bool debSystem::Lock()
    {
       close(LockFD);
       LockFD = -1;
+      const char *cmd;
+      if (getenv("SUDO_USER") != NULL)
+	 cmd = "sudo dpkg --configure -a";
+      else
+	 cmd = "dpkg --configure -a";
+      // TRANSLATORS: the %s contains the recovery command, usually
+      //              dpkg --configure -a
       return _error->Error(_("dpkg was interrupted, you must manually "
-                             "run 'dpkg --configure -a' to correct the problem. "));
+                             "run '%s' to correct the problem. "), cmd);
    }
 
 	 LockCount++;

apt-pkg/deb/dpkgpm.cc

@@ -12,6 +12,7 @@
 #include <apt-pkg/error.h>
 #include <apt-pkg/configuration.h>
 #include <apt-pkg/depcache.h>
+#include <apt-pkg/pkgrecords.h>
 #include <apt-pkg/strutl.h>
 #include <apti18n.h>
 #include <apt-pkg/fileutl.h>
@@ -24,6 +25,7 @@
 #include <sys/wait.h>
 #include <signal.h>
 #include <errno.h>
+#include <string.h>
 #include <stdio.h>
 #include <string.h>
 #include <algorithm>
@@ -354,7 +356,6 @@ bool pkgDPkgPM::RunScriptsWithPkgs(const char *Cnf)
 
    return true;
 }
-
 									/*}}}*/
 // DPkgPM::DoStdin - Read stdin and pass to slave pty			/*{{{*/
 // ---------------------------------------------------------------------
@@ -425,7 +426,7 @@ void pkgDPkgPM::ProcessDpkgStatusLine(int OutStatusFd, char *line)
       'processing: trigproc: trigger'
 	    
    */
-   char* list[5];
+   char* list[6];
    //        dpkg sends multiline error messages sometimes (see
    //        #374195 for a example. we should support this by
    //        either patching dpkg to not send multiline over the
@@ -476,6 +477,14 @@ void pkgDPkgPM::ProcessDpkgStatusLine(int OutStatusFd, char *line)
 
    if(strncmp(action,"error",strlen("error")) == 0)
    {
+      // urgs, sometime has ":" in its error string so that we
+      // end up with the error message split between list[3]
+      // and list[4], e.g. the message: 
+      // "failed in buffer_write(fd) (10, ret=-1): backend dpkg-deb ..."
+      // concat them again
+      if( list[4] != NULL )
+	 list[3][strlen(list[3])] = ':';
+
       status << "pmerror:" << list[1]
 	     << ":"  << (PackagesDone/float(PackagesTotal)*100.0) 
 	     << ":" << list[3]
@@ -484,6 +493,8 @@ void pkgDPkgPM::ProcessDpkgStatusLine(int OutStatusFd, char *line)
 	 write(OutStatusFd, status.str().c_str(), status.str().size());
       if (Debug == true)
 	 std::clog << "send: '" << status.str() << "'" << endl;
+      pkgFailures++;
+      WriteApportReport(list[1], list[3]);
       return;
    }
    else if(strncmp(action,"conffile",strlen("conffile")) == 0)
@@ -1175,3 +1186,186 @@ void pkgDPkgPM::Reset()
    List.erase(List.begin(),List.end());
 }
 									/*}}}*/
+// pkgDpkgPM::WriteApportReport - write out error report pkg failure	/*{{{*/
+// ---------------------------------------------------------------------
+/* */
+void pkgDPkgPM::WriteApportReport(const char *pkgpath, const char *errormsg) 
+{
+   string pkgname, reportfile, srcpkgname, pkgver, arch;
+   string::size_type pos;
+   FILE *report;
+
+   if (_config->FindB("Dpkg::ApportFailureReport", false) == false)
+   {
+      std::clog << "configured to not write apport reports" << std::endl;
+      return;
+   }
+
+   // only report the first errors
+   if(pkgFailures > _config->FindI("APT::Apport::MaxReports", 3))
+   {
+      std::clog << _("No apport report written because MaxReports is reached already") << std::endl;
+      return;
+   }
+
+   // check if its not a follow up error 
+   const char *needle = dgettext("dpkg", "dependency problems - leaving unconfigured");
+   if(strstr(errormsg, needle) != NULL) {
+      std::clog << _("No apport report written because the error message indicates its a followup error from a previous failure.") << std::endl;
+      return;
+   }
+
+   // do not report disk-full failures 
+   if(strstr(errormsg, strerror(ENOSPC)) != NULL) {
+      std::clog << _("No apport report written because the error message indicates a disk full error") << std::endl;
+      return;
+   }
+
+   // do not report out-of-memory failures 
+   if(strstr(errormsg, strerror(ENOMEM)) != NULL) {
+      std::clog << _("No apport report written because the error message indicates a out of memory error") << std::endl;
+      return;
+   }
+
+   // do not report dpkg I/O errors
+   // XXX - this message is localized, but this only matches the English version.  This is better than nothing.
+   if(strstr(errormsg, "short read in buffer_copy (")) {
+      std::clog << _("No apport report written because the error message indicates a dpkg I/O error") << std::endl;
+      return;
+   }
+
+   // get the pkgname and reportfile
+   pkgname = flNotDir(pkgpath);
+   pos = pkgname.find('_');
+   if(pos != string::npos)
+      pkgname = pkgname.substr(0, pos);
+
+   // find the package versin and source package name
+   pkgCache::PkgIterator Pkg = Cache.FindPkg(pkgname);
+   if (Pkg.end() == true)
+      return;
+   pkgCache::VerIterator Ver = Cache.GetCandidateVer(Pkg);
+   if (Ver.end() == true)
+      return;
+   pkgver = Ver.VerStr() == NULL ? "unknown" : Ver.VerStr();
+   pkgRecords Recs(Cache);
+   pkgRecords::Parser &Parse = Recs.Lookup(Ver.FileList());
+   srcpkgname = Parse.SourcePkg();
+   if(srcpkgname.empty())
+      srcpkgname = pkgname;
+
+   // if the file exists already, we check:
+   // - if it was reported already (touched by apport). 
+   //   If not, we do nothing, otherwise
+   //    we overwrite it. This is the same behaviour as apport
+   // - if we have a report with the same pkgversion already
+   //   then we skip it
+   reportfile = flCombine("/var/crash",pkgname+".0.crash");
+   if(FileExists(reportfile))
+   {
+      struct stat buf;
+      char strbuf[255];
+
+      // check atime/mtime
+      stat(reportfile.c_str(), &buf);
+      if(buf.st_mtime > buf.st_atime)
+	 return;
+
+      // check if the existing report is the same version
+      report = fopen(reportfile.c_str(),"r");
+      while(fgets(strbuf, sizeof(strbuf), report) != NULL)
+      {
+	 if(strstr(strbuf,"Package:") == strbuf)
+	 {
+	    char pkgname[255], version[255];
+	    if(sscanf(strbuf, "Package: %s %s", pkgname, version) == 2)
+	       if(strcmp(pkgver.c_str(), version) == 0)
+	       {
+		  fclose(report);
+		  return;
+	       }
+	 }
+      }
+      fclose(report);
+   }
+
+   // now write the report
+   arch = _config->Find("APT::Architecture");
+   report = fopen(reportfile.c_str(),"w");
+   if(report == NULL)
+      return;
+   if(_config->FindB("DPkgPM::InitialReportOnly",false) == true)
+      chmod(reportfile.c_str(), 0);
+   else
+      chmod(reportfile.c_str(), 0600);
+   fprintf(report, "ProblemType: Package\n");
+   fprintf(report, "Architecture: %s\n", arch.c_str());
+   time_t now = time(NULL);
+   fprintf(report, "Date: %s" , ctime(&now));
+   fprintf(report, "Package: %s %s\n", pkgname.c_str(), pkgver.c_str());
+   fprintf(report, "SourcePackage: %s\n", srcpkgname.c_str());
+   fprintf(report, "ErrorMessage:\n %s\n", errormsg);
+
+   // ensure that the log is flushed
+   if(term_out)
+      fflush(term_out);
+
+   // attach terminal log it if we have it
+   string logfile_name = _config->FindFile("Dir::Log::Terminal");
+   if (!logfile_name.empty())
+   {
+      FILE *log = NULL;
+      char buf[1024];
+
+      fprintf(report, "DpkgTerminalLog:\n");
+      log = fopen(logfile_name.c_str(),"r");
+      if(log != NULL)
+      {
+	 while( fgets(buf, sizeof(buf), log) != NULL)
+	    fprintf(report, " %s", buf);
+	 fclose(log);
+      }
+   }
+
+   // log the ordering 
+   const char *ops_str[] = {"Install", "Configure","Remove","Purge"};
+   fprintf(report, "AptOrdering:\n");
+   for (vector<Item>::iterator I = List.begin(); I != List.end(); I++)
+      fprintf(report, " %s: %s\n", (*I).Pkg.Name(), ops_str[(*I).Op]);
+
+   // attach dmesg log (to learn about segfaults)
+   if (FileExists("/bin/dmesg"))
+   {
+      FILE *log = NULL;
+      char buf[1024];
+
+      fprintf(report, "Dmesg:\n");
+      log = popen("/bin/dmesg","r");
+      if(log != NULL)
+      {
+	 while( fgets(buf, sizeof(buf), log) != NULL)
+	    fprintf(report, " %s", buf);
+	 fclose(log);
+      }
+   }
+
+   // attach df -l log (to learn about filesystem status)
+   if (FileExists("/bin/df"))
+   {
+      FILE *log = NULL;
+      char buf[1024];
+
+      fprintf(report, "Df:\n");
+      log = popen("/bin/df -l","r");
+      if(log != NULL)
+      {
+	 while( fgets(buf, sizeof(buf), log) != NULL)
+	    fprintf(report, " %s", buf);
+	 fclose(log);
+      }
+   }
+
+   fclose(report);
+
+}
+									/*}}}*/

apt-pkg/deb/dpkgpm.h

@@ -33,6 +33,7 @@ class pkgDPkgPM : public pkgPackageManager
    string dpkg_error;
 
    protected:
+   int pkgFailures;
 
    // progress reporting
    struct DpkgState 
@@ -49,6 +50,7 @@ class pkgDPkgPM : public pkgPackageManager
    // the int is the state that is already done (e.g. a package that is
    // going to be install is already in state "half-installed")
    map<string,unsigned int> PackageOpsDone;
+
    // progress reporting
    unsigned int PackagesDone;
    unsigned int PackagesTotal;
@@ -70,6 +72,9 @@ class pkgDPkgPM : public pkgPackageManager
    bool SendV2Pkgs(FILE *F);
    void WriteHistoryTag(string tag, string value);
 
+   // apport integration
+   void WriteApportReport(const char *pkgpath, const char *errormsg);
+
    // dpkg log
    bool OpenLog();
    bool CloseLog();

apt-pkg/init.cc

@@ -51,6 +51,7 @@ bool pkgInitConfig(Configuration &Cnf)
        
    Cnf.Set("Dir::State::lists","lists/");
    Cnf.Set("Dir::State::cdroms","cdroms.list");
+   Cnf.Set("Dir::State::mirrors","mirrors/");
    
    // Cache
    Cnf.Set("Dir::Cache","var/cache/apt/");

apt-pkg/tagfile.cc

@@ -409,6 +409,7 @@ static const char *iTFRewritePackageOrder[] = {
                           "Section",
                           "Installed-Size",
                           "Maintainer",
+                          "Original-Maintainer",
                           "Architecture",
                           "Source",
                           "Version",
@@ -438,6 +439,7 @@ static const char *iTFRewriteSourceOrder[] = {"Package",
                                       "Priority",
                                       "Section",
                                       "Maintainer",
+				      "Original-Maintainer",
                                       "Build-Depends",
                                       "Build-Depends-Indep",
                                       "Build-Conflicts",

cmdline/apt-cdrom.cc

@@ -147,7 +147,7 @@ bool DoAdd(CommandLine &)
    pkgCdrom cdrom;
    bool res = true;
 
-   bool AutoDetect = _config->FindB("Acquire::cdrom::AutoDetect");
+   bool AutoDetect = _config->FindB("Acquire::cdrom::AutoDetect", true);
    unsigned int count = 0;
    
    if (AutoDetect && UdevCdroms.Dlopen())

cmdline/apt-get.cc

@@ -2306,6 +2306,33 @@ bool DoSource(CommandLine &CmdL)
       if (Last == 0)
 	 return _error->Error(_("Unable to find a source package for %s"),Src.c_str());
       
+      string srec = Last->AsStr();
+      string::size_type pos = srec.find("\nVcs-");
+      while (pos != string::npos)
+      {
+	 pos += strlen("\nVcs-");
+	 string vcs = srec.substr(pos,srec.find(":",pos)-pos);
+	 if(vcs == "Browser") 
+	 {
+	    pos = srec.find("\nVcs-", pos);
+	    continue;
+	 }
+	 pos += vcs.length()+2;
+	 string::size_type epos = srec.find("\n", pos);
+	 string uri = srec.substr(pos,epos-pos).c_str();
+	 ioprintf(c1out, _("NOTICE: '%s' packaging is maintained in "
+			   "the '%s' version control system at:\n"
+			   "%s\n"),
+		  Src.c_str(), vcs.c_str(), uri.c_str());
+	 if(vcs == "Bzr") 
+	    ioprintf(c1out,_("Please use:\n"
+			     "bzr get %s\n"
+			     "to retrieve the latest (possibly unreleased) "
+			     "updates to the package.\n"),
+		     uri.c_str());
+	 break;
+      }
+
       // Back track
       vector<pkgSrcRecords::File> Lst;
       if (Last->Files(Lst) == false)
@@ -2968,7 +2995,6 @@ int main(int argc,const char *argv[])					/*{{{*/
                                    {"remove",&DoInstall},
                                    {"purge",&DoInstall},
 				   {"autoremove",&DoInstall},
-				   {"purge",&DoInstall},
 				   {"markauto",&DoMarkAuto},
 				   {"unmarkauto",&DoMarkAuto},
                                    {"dist-upgrade",&DoDistUpgrade},

cmdline/apt-report-mirror-failure

@@ -0,0 +1,29 @@
+#!/usr/bin/python
+#
+# This is a stub that is meant to support failure reporting of 
+# mirrors to a central database
+#
+# its currently not used
+
+import sys
+import urllib
+import apt_pkg
+
+apt_pkg.init()
+url = apt_pkg.Config.find("Acquire::Mirror::ReportFailures", "")
+                          #"http://people.ubuntu.com:9000/mirror-failure")
+                          #"http://localhost:9000/mirror-failure")
+if not url:
+    sys.exit(0)
+
+print "Reporting mirror failure to '%s'" % url
+
+data = {}
+data['mirror'] = sys.argv[1]
+data['failurl'] = sys.argv[2]
+data['error'] = sys.argv[3]
+f = urllib.urlopen(url, urllib.urlencode(data))
+f.read()
+f.close()
+
+

cmdline/makefile

@@ -58,3 +58,9 @@ SOURCE=apt-mark
 TO=$(BIN)
 TARGET=program
 include $(COPY_H)
+
+# The apt-report-mirror-failure program
+#SOURCE=apt-report-mirror-failure
+#TO=$(BIN)
+#TARGET=program
+#include $(COPY_H)

configure.in

@@ -18,7 +18,7 @@ AC_CONFIG_AUX_DIR(buildlib)
 AC_CONFIG_HEADER(include/config.h:buildlib/config.h.in include/apti18n.h:buildlib/apti18n.h.in)
 
 dnl -- SET THIS TO THE RELEASE VERSION --
-AC_DEFINE_UNQUOTED(VERSION,"0.7.26~exp4")
+AC_DEFINE_UNQUOTED(VERSION,"0.7.26~exp6")
 PACKAGE="apt"
 AC_DEFINE_UNQUOTED(PACKAGE,"$PACKAGE")
 AC_SUBST(PACKAGE)

debian/apt.conf.autoremove

@@ -1,9 +1,23 @@
 APT
 {
   NeverAutoRemove  
-  { 
+  {
+        "^linux-firmware$";
 	"^linux-image.*";  
 	"^linux-restricted-modules.*";
-	"^kfreebsd-image.*";  
+	"^linux-ubuntu-modules-.*";
+  };
+
+  Never-MarkAuto-Sections
+  {
+	"metapackages";
+        "restricted/metapackages";
+        "universe/metapackages";
+        "multiverse/metapackages";
+	"oldlibs";
+        "restricted/oldlibs";
+        "universe/oldlibs";
+        "multiverse/oldlibs";
+
   };
 };

debian/apt.dirs

@@ -9,6 +9,7 @@ etc/apt/trusted.gpg.d
 etc/logrotate.d
 var/cache/apt/archives/partial
 var/lib/apt/lists/partial
+var/lib/apt/mirrors/partial
 var/lib/apt/periodic
 var/log/apt
 usr/share/bug/apt

debian/changelog

@@ -1,3 +1,28 @@
+apt (0.7.26~exp6) experimental; urgency=low
+
+  [ Michael Vogt ]
+  * merge the remaining Ubuntu change:
+    - on gpg verification failure warn and restore the last known
+      good state
+    - on failure display the IP of the server (useful for servers
+      that use round robin DNS)
+    - support Original-Maintainer in RewritePackageOrder
+    - enable cdrom autodetection via libudev by default
+    - show messsage about Vcs in use when apt-get source is run for
+      packages maintained in a Vcs
+    - better support transitional packages with mark auto-installed. 
+      when the transitional package is in "oldlibs" the new package
+      is not marked auto installed (same is true for section
+      metapackages)
+    - provide new "deb mirror://archive.foo/mirrors.list sid main"
+      method expects a list of mirrors (generated on the server e.g.
+      via geoip) and will use that, including cycle on failure
+    - write apport crash file on package failure (disabled by default
+      on debian until apport is available)
+    - support mirror failure reporting (disabled by default on debian)
+
+ -- Michael Vogt <mvo@debian.org>  Wed, 09 Jun 2010 10:50:17 +0200
+
 apt (0.7.26~exp5) experimental; urgency=low
 
   [ David Kalnischkies ]

debian/control

@@ -6,7 +6,7 @@ Uploaders: Michael Vogt <mvo@debian.org>, Otavio Salvador <otavio@debian.org>,
  Christian Perrier <bubulle@debian.org>, Daniel Burrows <dburrows@debian.org>,
  Luca Bruno <lethalman88@gmail.com>, Julian Andres Klode <jak@debian.org>
 Standards-Version: 3.8.4
-Build-Depends: debhelper (>= 5.0), libdb-dev, gettext (>= 0.12), libcurl4-gnutls-dev | libcurl3-gnutls-dev (>= 7.15.5), debiandoc-sgml, xsltproc, docbook-xsl, po4a (>= 0.34-2), autotools-dev, autoconf, automake, doxygen
+Build-Depends: debhelper (>= 5.0), libdb-dev, gettext (>= 0.12), libcurl4-gnutls-dev | libcurl3-gnutls-dev (>= 7.15.5), debiandoc-sgml, xsltproc, docbook-xsl, po4a (>= 0.34-2), autotools-dev, autoconf, automake, doxygen, intltool
 Build-Conflicts: autoconf2.13, automake1.4
 
 Package: apt

debian/rules

@@ -215,6 +215,8 @@ apt: build build-doc debian/shlibs.local
 	cp debian/bugscript debian/$@/usr/share/bug/apt/script
 	cp debian/apt.logrotate debian/$@/etc/logrotate.d/apt
 
+	cp share/ubuntu-archive.gpg debian/$@/usr/share/$@
+	sed 's/^_//' share/apt-auth-failure.note > debian/$@/usr/share/$@/apt-auth-failure.note
 	cp debian/apt.conf.autoremove debian/$@/etc/apt/apt.conf.d/01autoremove
 
 	# copy lintian override
@@ -225,6 +227,10 @@ apt: build build-doc debian/shlibs.local
 	rm -f build/po/*.pot
 	rm -f po/*.pot
 
+	# move the mirror failure script in place
+	#mv debian/$@/usr/bin/apt-report-mirror-failure \
+	#   debian/$@/usr/lib/apt/apt-report-mirror-failure \
+
 	dh_installexamples -p$@ $(BLD)/docs/examples/*
 	dh_installman -p$@ $(wildcard $(patsubst %,doc/%.[158],$(apt_MANPAGES)) $(patsubst %,doc/*/%.*.[158],$(apt_MANPAGES)))
 	dh_installcron -p$@

methods/connect.cc

@@ -18,6 +18,7 @@
 #include <stdio.h>
 #include <errno.h>
 #include <unistd.h>
+#include <sstream>
 
 #include<set>
 #include<string>
@@ -70,19 +71,17 @@ static bool DoConnect(struct addrinfo *Addr,string Host,
    Owner->Status(_("Connecting to %s (%s)"),Host.c_str(),Name);
 
    // if that addr did timeout before, we do not try it again
-   if(bad_addr.find(string(Name)) != bad_addr.end()) 
+   if(bad_addr.find(string(Name)) != bad_addr.end())
       return false;
 
    /* If this is an IP rotation store the IP we are using.. If something goes
       wrong this will get tacked onto the end of the error message */
    if (LastHostAddr->ai_next != 0)
    {
-      char Name2[NI_MAXHOST + NI_MAXSERV + 10];
-      snprintf(Name2,sizeof(Name2),_("[IP: %s %s]"),Name,Service);
-      Owner->SetFailExtraMsg(string(Name2));
-   }   
-   else
-      Owner->SetFailExtraMsg("");
+      std::stringstream ss;
+      ioprintf(ss, _("[IP: %s %s]"),Name,Service);
+      Owner->SetIP(ss.str());
+   }
       
    // Get a socket
    if ((Fd = socket(Addr->ai_family,Addr->ai_socktype,
@@ -100,7 +99,7 @@ static bool DoConnect(struct addrinfo *Addr,string Host,
       nonblocking */
    if (WaitFd(Fd,true,TimeOut) == false) {
       bad_addr.insert(bad_addr.begin(), string(Name));
-      Owner->SetFailExtraMsg("\nFailReason: Timeout");
+      Owner->SetFailReason("Timeout");
       return _error->Error(_("Could not connect to %s:%s (%s), "
 			   "connection timed out"),Host.c_str(),Service,Name);
    }
@@ -115,9 +114,9 @@ static bool DoConnect(struct addrinfo *Addr,string Host,
    {
       errno = Err;
       if(errno == ECONNREFUSED)
-         Owner->SetFailExtraMsg("\nFailReason: ConnectionRefused");
+         Owner->SetFailReason("ConnectionRefused");
       else if (errno == ETIMEDOUT)
-	 Owner->SetFailExtraMsg("\nFailReason: ConnectionTimedOut");
+	 Owner->SetFailReason("ConnectionTimedOut");
       bad_addr.insert(bad_addr.begin(), string(Name));
       return _error->Errno("connect",_("Could not connect to %s:%s (%s)."),Host.c_str(),
 			   Service,Name);
@@ -184,13 +183,13 @@ bool Connect(string Host,int Port,const char *Service,int DefPort,int &Fd,
 		  continue;
 	       }
 	       bad_addr.insert(bad_addr.begin(), Host);
-	       Owner->SetFailExtraMsg("\nFailReason: ResolveFailure");
+	       Owner->SetFailReason("ResolveFailure");
 	       return _error->Error(_("Could not resolve '%s'"),Host.c_str());
 	    }
 	    
 	    if (Res == EAI_AGAIN)
 	    {
-	       Owner->SetFailExtraMsg("\nFailReason: TmpResolveFailure");
+	       Owner->SetFailReason("TmpResolveFailure");
 	       return _error->Error(_("Temporary failure resolving '%s'"),
 				    Host.c_str());
 	    }

methods/copy.cc

@@ -84,6 +84,7 @@ bool CopyMethod::Fetch(FetchItem *Itm)
    FileFd Fd(Res.Filename, FileFd::ReadOnly);
    Hash.AddFD(Fd.Fd(), Fd.Size());
    Res.TakeHashes(Hash);
+
    URIDone(Res);
    return true;
 }

methods/http.cc

@@ -953,6 +953,9 @@ HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
       failure */
    if (Srv->Result < 200 || Srv->Result >= 300)
    {
+      char err[255];
+      snprintf(err,sizeof(err)-1,"HttpError%i",Srv->Result);
+      SetFailReason(err);
       _error->Error("%u %s",Srv->Result,Srv->Code);
       if (Srv->HaveContent == true)
 	 return ERROR_WITH_CONTENT_PAGE;
@@ -1366,15 +1369,4 @@ bool HttpMethod::AutoDetectProxy()
 }
 									/*}}}*/
 
-int main()
-{
-   setlocale(LC_ALL, "");
-   // ignore SIGPIPE, this can happen on write() if the socket
-   // closes the connection (this is dealt with via ServerDie())
-   signal(SIGPIPE, SIG_IGN);
-
-   HttpMethod Mth;
-   return Mth.Loop();
-}
-
 

methods/http.h

@@ -13,7 +13,7 @@
 
 #define MAXLEN 360
 
-#include <iostream>
+
 
 using std::cout;
 using std::endl;
@@ -167,7 +167,6 @@ class HttpMethod : public pkgAcqMethod
    /** \brief Try to AutoDetect the proxy */
    bool AutoDetectProxy();
 
-   virtual bool Fetch(FetchItem *);
    virtual bool Configuration(string Message);
    
    // In the event of a fatal signal this file will be closed and timestamped.
@@ -175,6 +174,9 @@ class HttpMethod : public pkgAcqMethod
    static int FailFd;
    static time_t FailTime;
    static void SigTerm(int);
+
+   protected:
+   virtual bool Fetch(FetchItem *);
    
    string NextURI;
    string AutoDetectProxyCmd;

methods/http_main.cc

@@ -0,0 +1,20 @@
+#include <apt-pkg/fileutl.h>
+#include <apt-pkg/acquire-method.h>
+#include <signal.h>
+
+#include "connect.h"
+#include "rfc2553emu.h"
+#include "http.h"
+
+
+int main()
+{
+   setlocale(LC_ALL, "");
+
+   // ignore SIGPIPE, this can happen on write() if the socket
+   // closes the connection (this is dealt with via ServerDie())
+   signal(SIGPIPE, SIG_IGN);
+
+   HttpMethod Mth;
+   return Mth.Loop();
+}

methods/makefile

@@ -48,7 +48,7 @@ include $(PROGRAM_H)
 PROGRAM=http
 SLIBS = -lapt-pkg $(SOCKETLIBS) $(INTLLIBS)
 LIB_MAKES = apt-pkg/makefile
-SOURCE = http.cc rfc2553emu.cc connect.cc
+SOURCE = http.cc http_main.cc rfc2553emu.cc connect.cc
 include $(PROGRAM_H)
 
 # The https method
@@ -79,9 +79,17 @@ LIB_MAKES = apt-pkg/makefile
 SOURCE = rsh.cc
 include $(PROGRAM_H)
 
+# The mirror method
+PROGRAM=mirror
+SLIBS = -lapt-pkg $(SOCKETLIBS)
+LIB_MAKES = apt-pkg/makefile
+SOURCE = mirror.cc http.cc rfc2553emu.cc connect.cc
+include $(PROGRAM_H)
+
 # SSH and bzip2 method symlink
 binary: $(BIN)/ssh $(BIN)/bzip2 $(BIN)/lzma
 veryclean: clean-$(BIN)/ssh clean-$(BIN)/bzip2 clean-$(BIN)/lzma
+
 $(BIN)/ssh:
 	echo "Installing ssh method link"
 	ln -fs rsh $(BIN)/ssh

methods/mirror.cc

@@ -0,0 +1,330 @@
+// -*- mode: cpp; mode: fold -*-
+// Description								/*{{{*/
+// $Id: mirror.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $
+/* ######################################################################
+
+   Mirror Aquire Method - This is the Mirror aquire method for APT.
+   
+   ##################################################################### */
+									/*}}}*/
+// Include Files							/*{{{*/
+#include <apt-pkg/fileutl.h>
+#include <apt-pkg/acquire-method.h>
+#include <apt-pkg/acquire-item.h>
+#include <apt-pkg/acquire.h>
+#include <apt-pkg/error.h>
+#include <apt-pkg/hashes.h>
+#include <apt-pkg/sourcelist.h>
+
+#include <fstream>
+#include <iostream>
+#include <stdarg.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <dirent.h>
+
+using namespace std;
+
+#include "mirror.h"
+#include "http.h"
+#include "apti18n.h"
+									/*}}}*/
+
+/* Done:
+ * - works with http (only!)
+ * - always picks the first mirror from the list
+ * - call out to problem reporting script
+ * - supports "deb mirror://host/path/to/mirror-list/// dist component"
+ * - uses pkgAcqMethod::FailReason() to have a string representation
+ *   of the failure that is also send to LP
+ * 
+ * TODO: 
+ * - deal with runing as non-root because we can't write to the lists 
+     dir then -> use the cached mirror file
+ * - better method to download than having a pkgAcquire interface here
+ *   and better error handling there!
+ * - support more than http
+ * - testing :)
+ */
+
+MirrorMethod::MirrorMethod()
+   : HttpMethod(), DownloadedMirrorFile(false)
+{
+};
+
+// HttpMethod::Configuration - Handle a configuration message		/*{{{*/
+// ---------------------------------------------------------------------
+/* We stash the desired pipeline depth */
+bool MirrorMethod::Configuration(string Message)
+{
+   if (pkgAcqMethod::Configuration(Message) == false)
+      return false;
+   Debug = _config->FindB("Debug::Acquire::mirror",false);
+   
+   return true;
+}
+									/*}}}*/
+
+// clean the mirrors dir based on ttl information
+bool MirrorMethod::Clean(string Dir)
+{
+   vector<metaIndex *>::const_iterator I;
+
+   if(Debug)
+      clog << "MirrorMethod::Clean(): " << Dir << endl;
+
+   if(Dir == "/")
+      return _error->Error("will not clean: '/'");
+
+   // read sources.list
+   pkgSourceList list;
+   list.ReadMainList();
+
+   DIR *D = opendir(Dir.c_str());   
+   if (D == 0)
+      return _error->Errno("opendir",_("Unable to read %s"),Dir.c_str());
+   
+   string StartDir = SafeGetCWD();
+   if (chdir(Dir.c_str()) != 0)
+   {
+      closedir(D);
+      return _error->Errno("chdir",_("Unable to change to %s"),Dir.c_str());
+   }
+   
+   for (struct dirent *Dir = readdir(D); Dir != 0; Dir = readdir(D))
+   {
+      // Skip some files..
+      if (strcmp(Dir->d_name,"lock") == 0 ||
+	  strcmp(Dir->d_name,"partial") == 0 ||
+	  strcmp(Dir->d_name,".") == 0 ||
+	  strcmp(Dir->d_name,"..") == 0)
+	 continue;
+
+      // see if we have that uri
+      for(I=list.begin(); I != list.end(); I++)
+      {
+	 string uri = (*I)->GetURI();
+	 if(uri.substr(0,strlen("mirror://")) != string("mirror://"))
+	    continue;
+	 string BaseUri = uri.substr(0,uri.size()-1);
+	 if (URItoFileName(BaseUri) == Dir->d_name)
+	    break;
+      }
+      // nothing found, nuke it
+      if (I == list.end())
+	 unlink(Dir->d_name);
+   };
+   
+   chdir(StartDir.c_str());
+   closedir(D);
+   return true;   
+}
+
+
+bool MirrorMethod::DownloadMirrorFile(string mirror_uri_str)
+{
+   if(Debug)
+      clog << "MirrorMethod::DownloadMirrorFile(): " << endl;
+
+   // check the file, if it is not older than RefreshInterval just use it
+   // otherwise try to get a new one
+   if(FileExists(MirrorFile)) 
+   {
+      struct stat buf;
+      time_t t,now,refresh;
+      if(stat(MirrorFile.c_str(), &buf) != 0)
+	 return false;
+      t = std::max(buf.st_mtime, buf.st_ctime);
+      now = time(NULL);
+      refresh = 60*_config->FindI("Acquire::Mirror::RefreshInterval",360);
+      if(t + refresh > now)
+      {
+	 if(Debug)
+	    clog << "Mirror file is in RefreshInterval" << endl;
+	 DownloadedMirrorFile = true;
+	 return true;
+      }
+      if(Debug)
+	 clog << "Mirror file " << MirrorFile << " older than " << refresh << "min, re-download it" << endl;
+   }
+
+   // not that great to use pkgAcquire here, but we do not have 
+   // any other way right now
+   string fetch = BaseUri;
+   fetch.replace(0,strlen("mirror://"),"http://");
+
+   pkgAcquire Fetcher;
+   new pkgAcqFile(&Fetcher, fetch, "", 0, "", "", "", MirrorFile);
+   bool res = (Fetcher.Run() == pkgAcquire::Continue);
+   if(res)
+      DownloadedMirrorFile = true;
+   Fetcher.Shutdown();
+   return res;
+}
+
+bool MirrorMethod::SelectMirror()
+{
+   // if we do not have a MirrorFile, fallback
+   if(!FileExists(MirrorFile))
+   {
+      // FIXME: fallback to a default mirror here instead 
+      //        and provide a config option to define that default
+      return _error->Error(_("No mirror file '%s' found "), MirrorFile.c_str());
+   }
+
+   // FIXME: make the mirror selection more clever, do not 
+   //        just use the first one!
+   // BUT: we can not make this random, the mirror has to be
+   //      stable accross session, because otherwise we can
+   //      get into sync issues (got indexfiles from mirror A,
+   //      but packages from mirror B - one might be out of date etc)
+   ifstream in(MirrorFile.c_str());
+   getline(in, Mirror);
+   if(Debug)
+      cerr << "Using mirror: " << Mirror << endl;
+
+   UsedMirror = Mirror;
+   return true;
+}
+
+string MirrorMethod::GetMirrorFileName(string mirror_uri_str)
+{
+   /* 
+    - a mirror_uri_str looks like this:
+    mirror://people.ubuntu.com/~mvo/apt/mirror/mirrors/dists/feisty/Release.gpg
+   
+    - the matching source.list entry
+    deb mirror://people.ubuntu.com/~mvo/apt/mirror/mirrors feisty main
+   
+    - we actually want to go after:
+    http://people.ubuntu.com/~mvo/apt/mirror/mirrors
+
+    And we need to save the BaseUri for later:
+    - mirror://people.ubuntu.com/~mvo/apt/mirror/mirrors
+
+   FIXME: what if we have two similar prefixes?
+     mirror://people.ubuntu.com/~mvo/mirror
+     mirror://people.ubuntu.com/~mvo/mirror2
+   then mirror_uri_str looks like:
+     mirror://people.ubuntu.com/~mvo/apt/mirror/dists/feisty/Release.gpg
+     mirror://people.ubuntu.com/~mvo/apt/mirror2/dists/feisty/Release.gpg
+   we search sources.list and find:
+     mirror://people.ubuntu.com/~mvo/apt/mirror
+   in both cases! So we need to apply some domain knowledge here :( and
+   check for /dists/ or /Release.gpg as suffixes
+   */
+   string name;
+   if(Debug)
+      std::cerr << "GetMirrorFileName: " << mirror_uri_str << std::endl;
+
+   // read sources.list and find match
+   vector<metaIndex *>::const_iterator I;
+   pkgSourceList list;
+   list.ReadMainList();
+   for(I=list.begin(); I != list.end(); I++)
+   {
+      string uristr = (*I)->GetURI();
+      if(Debug)
+	 std::cerr << "Checking: " << uristr << std::endl;
+      if(uristr.substr(0,strlen("mirror://")) != string("mirror://"))
+	 continue;
+      // find matching uri in sources.list
+      if(mirror_uri_str.substr(0,uristr.size()) == uristr)
+      {
+	 if(Debug)
+	    std::cerr << "found BaseURI: " << uristr << std::endl;
+	 BaseUri = uristr.substr(0,uristr.size()-1);
+      }
+   }
+   // get new file
+   name = _config->FindDir("Dir::State::mirrors") + URItoFileName(BaseUri);
+
+   if(Debug) 
+   {
+      cerr << "base-uri: " << BaseUri << endl;
+      cerr << "mirror-file: " << name << endl;
+   }
+   return name;
+}
+
+// MirrorMethod::Fetch - Fetch an item					/*{{{*/
+// ---------------------------------------------------------------------
+/* This adds an item to the pipeline. We keep the pipeline at a fixed
+   depth. */
+bool MirrorMethod::Fetch(FetchItem *Itm)
+{
+   if(Debug)
+      clog << "MirrorMethod::Fetch()" << endl;
+
+   // the http method uses Fetch(0) as a way to update the pipeline,
+   // just let it do its work in this case - Fetch() with a valid
+   // Itm will always run before the first Fetch(0)
+   if(Itm == NULL) 
+      return HttpMethod::Fetch(Itm);
+
+   // if we don't have the name of the mirror file on disk yet,
+   // calculate it now (can be derived from the uri)
+   if(MirrorFile.empty())
+      MirrorFile = GetMirrorFileName(Itm->Uri);
+
+  // download mirror file once (if we are after index files)
+   if(Itm->IndexFile && !DownloadedMirrorFile)
+   {
+      Clean(_config->FindDir("Dir::State::mirrors"));
+      DownloadMirrorFile(Itm->Uri);
+   }
+
+   if(Mirror.empty()) {
+      if(!SelectMirror()) {
+	 // no valid mirror selected, something went wrong downloading
+	 // from the master mirror site most likely and there is
+	 // no old mirror file availalbe
+	 return false;
+      }
+   }
+   if(Debug)
+      clog << "selected mirror: " << Mirror << endl;
+
+
+   for (FetchItem *I = Queue; I != 0; I = I->Next)
+   {
+      if(I->Uri.find("mirror://") != string::npos)
+	 I->Uri.replace(0,BaseUri.size(), Mirror);
+   }
+   
+   // now run the real fetcher
+   return HttpMethod::Fetch(Itm);
+};
+
+void MirrorMethod::Fail(string Err,bool Transient)
+{
+   if(Queue->Uri.find("http://") != string::npos)
+      Queue->Uri.replace(0,Mirror.size(), BaseUri);
+   pkgAcqMethod::Fail(Err, Transient);
+}
+
+void MirrorMethod::URIStart(FetchResult &Res)
+{
+   if(Queue->Uri.find("http://") != string::npos)
+      Queue->Uri.replace(0,Mirror.size(), BaseUri);
+   pkgAcqMethod::URIStart(Res);
+}
+
+void MirrorMethod::URIDone(FetchResult &Res,FetchResult *Alt)
+{
+   if(Queue->Uri.find("http://") != string::npos)
+      Queue->Uri.replace(0,Mirror.size(), BaseUri);
+   pkgAcqMethod::URIDone(Res, Alt);
+}
+
+
+int main()
+{
+   setlocale(LC_ALL, "");
+
+   MirrorMethod Mth;
+
+   return Mth.Loop();
+}
+
+

methods/mirror.h

@@ -0,0 +1,52 @@
+// -*- mode: cpp; mode: fold -*-
+// Description								/*{{{*/// $Id: http.h,v 1.12 2002/04/18 05:09:38 jgg Exp $
+// $Id: http.h,v 1.12 2002/04/18 05:09:38 jgg Exp $
+/* ######################################################################
+
+   MIRROR Aquire Method - This is the MIRROR aquire method for APT.
+
+   ##################################################################### */
+									/*}}}*/
+
+#ifndef APT_MIRROR_H
+#define APT_MIRROR_H
+
+
+#include <iostream>
+
+using std::cout;
+using std::cerr;
+using std::endl;
+
+#include "http.h"
+
+class MirrorMethod : public HttpMethod
+{
+   FetchResult Res;
+   // we simply transform between BaseUri and Mirror
+   string BaseUri;    // the original mirror://... url
+   string Mirror;     // the selected mirror uri (http://...)
+   string MirrorFile; // the file that contains the list of mirrors
+   bool DownloadedMirrorFile; // already downloaded this session
+
+   bool Debug;
+
+ protected:
+   bool DownloadMirrorFile(string uri);
+   string GetMirrorFileName(string uri);
+   bool SelectMirror();
+   bool Clean(string dir);
+   
+   // we need to overwrite those to transform the url back
+   virtual void Fail(string Why, bool Transient = false);
+   virtual void URIStart(FetchResult &Res);
+   virtual void URIDone(FetchResult &Res,FetchResult *Alt = 0);
+   virtual bool Configuration(string Message);
+
+ public:
+   MirrorMethod();
+   virtual bool Fetch(FetchItem *Itm);
+};
+
+
+#endif

mirror-failure.py

@@ -0,0 +1,23 @@
+# File: cgihttpserver-example-1.py
+
+import CGIHTTPServer
+import BaseHTTPServer
+
+class Handler(CGIHTTPServer.CGIHTTPRequestHandler):
+    #cgi_directories = ["/cgi"]
+    def do_POST(self):
+	print "do_POST"
+        #print self.command
+        #print self.path
+        #print self.headers
+        print self.client_address
+        data = self.rfile.read(int(self.headers["content-length"]))
+        print data
+        self.wfile.write("200 Ok\n");
+
+PORT = 8000
+
+httpd = BaseHTTPServer.HTTPServer(("", PORT), Handler)
+print "serving at port", PORT
+httpd.serve_forever()
+

test/authReliability/lists/people.ubuntu.com_%7emvo_apt_auth-test-suit_gpg-package-broken_Packages

@@ -0,0 +1,25 @@
+Package: libglib2.0-data
+Priority: optional
+Section: misc
+Installed-Size: 2288
+Maintainer: Ubuntu Desktop Team <ubuntu-desktop@lists.ubuntu.com>
+Original-Maintainer: Loic Minier <lool@dooz.org>
+Architecture: all
+Source: glib2.0
+Version: 2.13.6-1ubuntu1
+Replaces: libglib1.3, libglib1.3-data
+Depends: libglib2.0-0 (>= 2.13.6-1ubuntu1)
+Conflicts: libglib1.3-data
+Filename: ./libglib2.0-data_2.13.6-1ubuntu1_all.deb
+Size: 958
+MD5sum: 803fc5e2e31a4345b3e9c771e1eae49f
+SHA1: 75b2c62b21bae60c58e694dd40ed6d4df946e304
+SHA256: 142d8466eac252f06bc957d76fe1bb87f86f2d3512b99c8d4b08c1ad79fbe59e
+Description: Common files for GLib library
+ GLib is a library containing many useful C routines for things such
+ as trees, hashes, lists, and strings.  It is a useful general-purpose
+ C library used by projects such as GTK+, GIMP, and GNOME.
+ .
+ This package is needed for the runtime libraries to display messages in
+ languages other than English.
+

test/authReliability/lists/people.ubuntu.com_%7emvo_apt_auth-test-suit_gpg-package-broken_Release

@@ -0,0 +1,13 @@
+Date: Fri, 27 Jul 2007 14:39:41 UTC
+MD5Sum:
+ 4672dadea6a144839f823c9f3d5fd44b              934 Packages
+ 82ebcf09a8d78a2b9cf7759349da4936              603 Packages.gz
+ d41d8cd98f00b204e9800998ecf8427e                0 Release
+SHA1:
+ fa0f294aa30789529371066b10e9497be1284d26              934 Packages
+ f4032808663b2810d87b4a4dab6f5ae4a1e8fa8e              603 Packages.gz
+ da39a3ee5e6b4b0d3255bfef95601890afd80709                0 Release
+SHA256:
+ 92c9b605480dc74e6be79c0ddc24738bfcbd6dd3148af531acd68717de528049              934 Packages
+ 659ccc0d07ff21f0247f9fa5abe149221c90d5e17da52c7afddb035b93c23d39              603 Packages.gz
+ e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855                0 Release

test/authReliability/lists/people.ubuntu.com_%7emvo_apt_auth-test-suit_gpg-package-broken_Release.gpg

@@ -0,0 +1,7 @@
+-----BEGIN PGP SIGNATURE-----
+Version: GnuPG v1.4.6 (GNU/Linux)
+
+iD8DBQBGqgOwliSD4VZixzQRAs6jAJ9p7Aiob9gzkUNCtoW8UPrBo0E/YwCdEaz0
+CQJszU6fRYX5jGWXSWzfc5c=
+=ugH0
+-----END PGP SIGNATURE-----

test/authReliability/lists/people.ubuntu.com_%7emvo_apt_auth-test-suit_gpg-package-ok_Packages

@@ -0,0 +1,25 @@
+Package: libglib2.0-data
+Priority: optional
+Section: misc
+Installed-Size: 2288
+Maintainer: Ubuntu Desktop Team <ubuntu-desktop@lists.ubuntu.com>
+Original-Maintainer: Loic Minier <lool@dooz.org>
+Architecture: all
+Source: glib2.0
+Version: 2.13.6-1ubuntu1
+Replaces: libglib1.3, libglib1.3-data
+Depends: libglib2.0-0 (>= 2.13.6-1ubuntu1)
+Conflicts: libglib1.3-data
+Filename: ./libglib2.0-data_2.13.6-1ubuntu1_all.deb
+Size: 958
+MD5sum: 803fc5e2e31a4345b3e9c771e1eae49f
+SHA1: 75b2c62b21bae60c58e694dd40ed6d4df946e304
+SHA256: 142d8466eac252f06bc957d76fe1bb87f86f2d3512b99c8d4b08c1ad79fbe59e
+Description: Common files for GLib library
+ GLib is a library containing many useful C routines for things such
+ as trees, hashes, lists, and strings.  It is a useful general-purpose
+ C library used by projects such as GTK+, GIMP, and GNOME.
+ .
+ This package is needed for the runtime libraries to display messages in
+ languages other than English.
+

test/authReliability/lists/people.ubuntu.com_%7emvo_apt_auth-test-suit_gpg-package-ok_Release

@@ -0,0 +1,13 @@
+Date: Fri, 27 Jul 2007 14:39:41 UTC
+MD5Sum:
+ 4672dadea6a144839f823c9f3d5fd44b              934 Packages
+ 82ebcf09a8d78a2b9cf7759349da4936              603 Packages.gz
+ d41d8cd98f00b204e9800998ecf8427e                0 Release
+SHA1:
+ fa0f294aa30789529371066b10e9497be1284d26              934 Packages
+ f4032808663b2810d87b4a4dab6f5ae4a1e8fa8e              603 Packages.gz
+ da39a3ee5e6b4b0d3255bfef95601890afd80709                0 Release
+SHA256:
+ 92c9b605480dc74e6be79c0ddc24738bfcbd6dd3148af531acd68717de528049              934 Packages
+ 659ccc0d07ff21f0247f9fa5abe149221c90d5e17da52c7afddb035b93c23d39              603 Packages.gz
+ e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855                0 Release

test/authReliability/lists/people.ubuntu.com_%7emvo_apt_auth-test-suit_gpg-package-ok_Release.gpg

@@ -0,0 +1,7 @@
+-----BEGIN PGP SIGNATURE-----
+Version: GnuPG v1.4.6 (GNU/Linux)
+
+iD8DBQBGqgOwliSD4VZixzQRAs6jAJ9p7Aiob9gzkUNCtoW8UPrBo0E/YwCdEaz0
+CQJszU6fRYX5jGWXSWzfc5c=
+=ugH0
+-----END PGP SIGNATURE-----

test/authReliability/sources.list.failure

@@ -0,0 +1,2 @@
+deb http://people.ubuntu.com/~mvo/apt/auth-test-suit/gpg-package-broken/ /
+

test/authReliability/sources.list.good

@@ -0,0 +1,2 @@
+deb http://people.ubuntu.com/~mvo/apt/auth-test-suit/gpg-package-ok/ /
+

test/pre-upload-check.py

@@ -4,6 +4,8 @@ import sys
 import os
 import glob
 import os.path
+import shutil
+import time
 from subprocess import call, PIPE
 
 import unittest
@@ -11,7 +13,102 @@ import unittest
 stdout = os.open("/dev/null",0) #sys.stdout
 stderr = os.open("/dev/null",0) # sys.stderr
 
-apt_args = []  # ["-o","Debug::pkgAcquire::Auth=true"]
+apt_args = [] 
+#apt_args = ["-o","Debug::pkgAcquire::Auth=true"]
+
+class testAptAuthenticationReliability(unittest.TestCase):
+    """
+    test if the spec https://wiki.ubuntu.com/AptAuthenticationReliability 
+    is properly implemented
+    """
+    #apt = "../bin/apt-get"
+    apt = "apt-get"
+
+    def setUp(self):
+        if os.path.exists("/tmp/autFailure"):
+            os.unlink("/tmp/authFailure");
+        if os.path.exists("/tmp/autFailure2"):
+            os.unlink("/tmp/authFailure2");
+    def testRepositorySigFailure(self):
+        """
+        test if a repository that used to be authenticated and fails on
+        apt-get update refuses to update and uses the old state
+        """
+        # copy valid signatures into lists (those are ok, even
+        # if the name is "-broken-" ...
+        for f in glob.glob("./authReliability/lists/*"):
+            shutil.copy(f,"/var/lib/apt/lists")
+            # ensure we do *not* get a I-M-S hit
+            os.utime("/var/lib/apt/lists/%s" % os.path.basename(f), (0,0))
+        res = call([self.apt,
+                    "update",
+                    "-o","Dir::Etc::sourcelist=./authReliability/sources.list.failure", 
+                    "-o",'APT::Update::Auth-Failure::=touch /tmp/authFailure',
+                   ] + apt_args,
+                   stdout=stdout, stderr=stderr)
+        self.assert_(os.path.exists("/var/lib/apt/lists/people.ubuntu.com_%7emvo_apt_auth-test-suit_gpg-package-broken_Release.gpg"),
+                     "The gpg file disappeared, this should not happen")
+        self.assert_(os.path.exists("/var/lib/apt/lists/people.ubuntu.com_%7emvo_apt_auth-test-suit_gpg-package-broken_Packages"),
+                     "The Packages file disappeared, this should not happen")
+        self.assert_(os.path.exists("/tmp/authFailure"),
+                     "The APT::Update::Auth-Failure script did not run (1)")
+        # the same with i-m-s hit this time
+        for f in glob.glob("./authReliability/lists/*"):
+            shutil.copy(f,"/var/lib/apt/lists")
+            os.utime("/var/lib/apt/lists/%s" % os.path.basename(f), (time.time(),time.time()))
+        res = call([self.apt,
+                    "update",
+                    "-o","Dir::Etc::sourcelist=./authReliability/sources.list.failure",
+                    "-o",'APT::Update::Auth-Failure::=touch /tmp/authFailure2',
+                   ] + apt_args,
+                   stdout=stdout, stderr=stderr)
+        self.assert_(os.path.exists("/var/lib/apt/lists/people.ubuntu.com_%7emvo_apt_auth-test-suit_gpg-package-broken_Release.gpg"),
+                     "The gpg file disappeared, this should not happen")
+        self.assert_(os.path.exists("/var/lib/apt/lists/people.ubuntu.com_%7emvo_apt_auth-test-suit_gpg-package-broken_Packages"),
+                     "The Packages file disappeared, this should not happen")
+        self.assert_(os.path.exists("/tmp/authFailure2"),
+                     "The APT::Update::Auth-Failure script did not run (2)")
+    def testRepositorySigGood(self):
+        """
+        test that a regular repository with good data stays good
+        """
+        res = call([self.apt,
+                    "update",
+                    "-o","Dir::Etc::sourcelist=./authReliability/sources.list.good"
+                   ] + apt_args,
+                   stdout=stdout, stderr=stderr)
+        self.assert_(os.path.exists("/var/lib/apt/lists/people.ubuntu.com_%7emvo_apt_auth-test-suit_gpg-package-ok_Release.gpg"),
+                     "The gpg file disappeared after a regular download, this should not happen")
+        self.assert_(os.path.exists("/var/lib/apt/lists/people.ubuntu.com_%7emvo_apt_auth-test-suit_gpg-package-ok_Packages"),
+                     "The Packages file disappeared, this should not happen")
+        # test good is still good after non I-M-S hit and a previous files in lists/
+        for f in glob.glob("./authReliability/lists/*"):
+            shutil.copy(f,"/var/lib/apt/lists")
+            # ensure we do *not* get a I-M-S hit
+            os.utime("/var/lib/apt/lists/%s" % os.path.basename(f), (0,0))
+        res = call([self.apt,
+                    "update",
+                    "-o","Dir::Etc::sourcelist=./authReliability/sources.list.good"
+                   ] + apt_args,
+                   stdout=stdout, stderr=stderr)
+        self.assert_(os.path.exists("/var/lib/apt/lists/people.ubuntu.com_%7emvo_apt_auth-test-suit_gpg-package-ok_Release.gpg"),
+                     "The gpg file disappeared after a I-M-S hit, this should not happen")
+        self.assert_(os.path.exists("/var/lib/apt/lists/people.ubuntu.com_%7emvo_apt_auth-test-suit_gpg-package-ok_Packages"),
+                     "The Packages file disappeared, this should not happen")
+        # test good is still good after I-M-S hit
+        for f in glob.glob("./authReliability/lists/*"):
+            shutil.copy(f,"/var/lib/apt/lists")
+            # ensure we do get a I-M-S hit
+            os.utime("/var/lib/apt/lists/%s" % os.path.basename(f), (time.time(),time.time()))
+        res = call([self.apt,
+                    "update",
+                    "-o","Dir::Etc::sourcelist=./authReliability/sources.list.good"
+                   ] + apt_args,
+                   stdout=stdout, stderr=stderr)
+        self.assert_(os.path.exists("/var/lib/apt/lists/people.ubuntu.com_%7emvo_apt_auth-test-suit_gpg-package-ok_Release.gpg"),
+                     "The gpg file disappeared, this should not happen")
+        self.assert_(os.path.exists("/var/lib/apt/lists/people.ubuntu.com_%7emvo_apt_auth-test-suit_gpg-package-ok_Packages"),
+                     "The Packages file disappeared, this should not happen")
 
 
 class testAuthentication(unittest.TestCase):
@@ -149,6 +246,7 @@ if __name__ == "__main__":
     if len(sys.argv) > 1 and sys.argv[1] == "-v":
         stdout = sys.stdout
         stderr = sys.stderr
+    
+    # run only one for now
+    #unittest.main(defaultTest="testAptAuthenticationReliability")
     unittest.main()
-
-