Browse Source

New http method

Author: jgg
Date: 1998-11-01 05:27:29 GMT
New http method
debian/1.8.y
Arch Librarian 18 years ago
parent
commit
be4401bfa4
  1. 2
      Makefile
  2. 129
      apt-pkg/acquire-method.cc
  3. 33
      apt-pkg/acquire-method.h
  4. 6
      apt-pkg/acquire-worker.cc
  5. 19
      apt-pkg/acquire.cc
  6. 3
      apt-pkg/acquire.h
  7. 4
      apt-pkg/contrib/md5.cc
  8. 6
      apt-pkg/contrib/strutl.cc
  9. 15
      apt-pkg/contrib/strutl.h
  10. 33
      doc/Bugs
  11. 13
      methods/copy.cc
  12. 11
      methods/file.cc
  13. 14
      methods/gzip.cc
  14. 892
      methods/http.cc
  15. 141
      methods/http.h
  16. 6
      methods/makefile

2
Makefile

@ -10,7 +10,7 @@ endif
all headers library clean veryclean binary program doc:
$(MAKE) -C apt-pkg $@
$(MAKE) -C methods $@
$(MAKE) -C methods/ftp $@
# $(MAKE) -C methods/ftp $@
$(MAKE) -C cmdline $@
$(MAKE) -C deity $@
$(MAKE) -C gui $@

129
apt-pkg/acquire-method.cc

@ -1,6 +1,6 @@
// -*- mode: cpp; mode: fold -*-
// Description /*{{{*/
// $Id: acquire-method.cc,v 1.1 1998/10/30 07:53:35 jgg Exp $
// $Id: acquire-method.cc,v 1.2 1998/11/01 05:27:30 jgg Exp $
/* ######################################################################
Acquire Method
@ -45,6 +45,8 @@ pkgAcqMethod::pkgAcqMethod(const char *Ver,unsigned long Flags)
if (write(STDOUT_FILENO,S,strlen(S)) != (signed)strlen(S))
exit(100);
SetNonBlock(STDIN_FILENO,true);
}
/*}}}*/
// AcqMethod::Fail - A fetch has failed /*{{{*/
@ -65,9 +67,20 @@ void pkgAcqMethod::Fail()
void pkgAcqMethod::Fail(string Err)
{
char S[1024];
snprintf(S,sizeof(S),"400 URI Failure\nURI: %s\n"
"Message %s\n\n",CurrentURI.c_str(),Err.c_str());
if (Queue != 0)
{
snprintf(S,sizeof(S),"400 URI Failure\nURI: %s\n"
"Message: %s\n\n",Queue->Uri.c_str(),Err.c_str());
// Dequeue
FetchItem *Tmp = Queue;
Queue = Queue->Next;
delete Tmp;
}
else
snprintf(S,sizeof(S),"400 URI Failure\nURI: <UNKNOWN>\n"
"Message: %s\n\n",Err.c_str());
if (write(STDOUT_FILENO,S,strlen(S)) != (signed)strlen(S))
exit(100);
}
@ -75,12 +88,15 @@ void pkgAcqMethod::Fail(string Err)
// AcqMethod::URIStart - Indicate a download is starting /*{{{*/
// ---------------------------------------------------------------------
/* */
void pkgAcqMethod::URIStart(FetchResult &Res,unsigned long Resume = 0)
void pkgAcqMethod::URIStart(FetchResult &Res)
{
if (Queue == 0)
abort();
char S[1024] = "";
char *End = S;
End += snprintf(S,sizeof(S),"200 URI Start\nURI: %s\n",CurrentURI.c_str());
End += snprintf(S,sizeof(S),"200 URI Start\nURI: %s\n",Queue->Uri.c_str());
if (Res.Size != 0)
End += snprintf(End,sizeof(S) - (End - S),"Size: %u\n",Res.Size);
@ -88,9 +104,9 @@ void pkgAcqMethod::URIStart(FetchResult &Res,unsigned long Resume = 0)
End += snprintf(End,sizeof(S) - (End - S),"Last-Modified: %s\n",
TimeRFC1123(Res.LastModified).c_str());
if (Resume != 0)
if (Res.ResumePoint != 0)
End += snprintf(End,sizeof(S) - (End - S),"Resume-Point: %u\n",
Resume);
Res.ResumePoint);
strcat(End,"\n");
if (write(STDOUT_FILENO,S,strlen(S)) != (signed)strlen(S))
@ -102,10 +118,13 @@ void pkgAcqMethod::URIStart(FetchResult &Res,unsigned long Resume = 0)
/* */
void pkgAcqMethod::URIDone(FetchResult &Res, FetchResult *Alt)
{
if (Queue == 0)
abort();
char S[1024] = "";
char *End = S;
End += snprintf(S,sizeof(S),"201 URI Done\nURI: %s\n",CurrentURI.c_str());
End += snprintf(S,sizeof(S),"201 URI Done\nURI: %s\n",Queue->Uri.c_str());
if (Res.Filename.empty() == false)
End += snprintf(End,sizeof(S) - (End - S),"Filename: %s\n",Res.Filename.c_str());
@ -147,6 +166,11 @@ void pkgAcqMethod::URIDone(FetchResult &Res, FetchResult *Alt)
strcat(End,"\n");
if (write(STDOUT_FILENO,S,strlen(S)) != (signed)strlen(S))
exit(100);
// Dequeue
FetchItem *Tmp = Queue;
Queue = Queue->Next;
delete Tmp;
}
/*}}}*/
// AcqMethod::Configuration - Handle the configuration message /*{{{*/
@ -186,19 +210,25 @@ bool pkgAcqMethod::Configuration(string Message)
// AcqMethod::Run - Run the message engine /*{{{*/
// ---------------------------------------------------------------------
/* */
int pkgAcqMethod::Run()
int pkgAcqMethod::Run(bool Single)
{
SetNonBlock(STDIN_FILENO,true);
while (1)
{
// Block if the message queue is empty
if (Messages.empty() == true)
if (WaitFd(STDIN_FILENO) == false)
return 0;
{
if (Single == false)
if (WaitFd(STDIN_FILENO) == false)
return 0;
}
if (ReadMessages(STDIN_FILENO,Messages) == false)
return 0;
// Single mode exits if the message queue is empty
if (Single == true && Messages.empty() == true)
return 0;
string Message = Messages.front();
Messages.erase(Messages.begin());
@ -219,12 +249,20 @@ int pkgAcqMethod::Run()
break;
case 600:
{
CurrentURI = LookupTag(Message,"URI");
DestFile = LookupTag(Message,"FileName");
StrToTime(LookupTag(Message,"Last-Modified"),LastModified);
if (Fetch(Message,CurrentURI) == false)
{
FetchItem *Tmp = new FetchItem;
Tmp->Uri = LookupTag(Message,"URI");
Tmp->DestFile = LookupTag(Message,"FileName");
StrToTime(LookupTag(Message,"Last-Modified"),Tmp->LastModified);
Tmp->Next = 0;
// Append it to the list
FetchItem **I = &Queue;
for (; *I != 0 && (*I)->Next != 0; I = &(*I)->Next);
*I = Tmp;
if (Fetch(Tmp) == false)
Fail();
break;
}
@ -234,6 +272,55 @@ int pkgAcqMethod::Run()
return 0;
}
/*}}}*/
// AcqMethod::Log - Send a log message /*{{{*/
// ---------------------------------------------------------------------
/* */
void pkgAcqMethod::Log(const char *Format,...)
{
string CurrentURI = "<UNKNOWN>";
if (Queue != 0)
CurrentURI = Queue->Uri;
va_list args;
va_start(args,Format);
// sprintf the description
char S[1024];
unsigned int Len = snprintf(S,sizeof(S),"101 Log\nURI: %s\n"
"Message: ",CurrentURI.c_str());
vsnprintf(S+Len,sizeof(S)-Len,Format,args);
strcat(S,"\n\n");
if (write(STDOUT_FILENO,S,strlen(S)) != (signed)strlen(S))
exit(100);
}
/*}}}*/
// AcqMethod::Status - Send a status message /*{{{*/
// ---------------------------------------------------------------------
/* */
void pkgAcqMethod::Status(const char *Format,...)
{
string CurrentURI = "<UNKNOWN>";
if (Queue != 0)
CurrentURI = Queue->Uri;
va_list args;
va_start(args,Format);
// sprintf the description
char S[1024];
unsigned int Len = snprintf(S,sizeof(S),"101 Log\nURI: %s\n"
"Message: ",CurrentURI.c_str());
vsnprintf(S+Len,sizeof(S)-Len,Format,args);
strcat(S,"\n\n");
if (write(STDOUT_FILENO,S,strlen(S)) != (signed)strlen(S))
exit(100);
}
/*}}}*/
// AcqMethod::FetchResult::FetchResult - Constructor /*{{{*/
// ---------------------------------------------------------------------
/* */

33
apt-pkg/acquire-method.h

@ -1,6 +1,6 @@
// -*- mode: cpp; mode: fold -*-
// Description /*{{{*/
// $Id: acquire-method.h,v 1.1 1998/10/30 07:53:35 jgg Exp $
// $Id: acquire-method.h,v 1.2 1998/11/01 05:27:32 jgg Exp $
/* ######################################################################
Acquire Method - Method helper class + functions
@ -23,12 +23,16 @@
class pkgAcqMethod
{
protected:
string CurrentURI;
string DestFile;
time_t LastModified;
vector<string> Messages;
struct FetchItem
{
FetchItem *Next;
string Uri;
string DestFile;
time_t LastModified;
};
struct FetchResult
{
@ -37,18 +41,22 @@ class pkgAcqMethod
bool IMSHit;
string Filename;
unsigned long Size;
unsigned long ResumePoint;
FetchResult();
};
// State
vector<string> Messages;
FetchItem *Queue;
// Handlers for messages
virtual bool Configuration(string Message);
virtual bool Fetch(string Message,URI Get) {return true;};
virtual bool Fetch(FetchItem *Item) {return true;};
// Outgoing messages
void Fail();
void Fail(string Why);
// void Log(const char *Format,...);
void URIStart(FetchResult &Res,unsigned long Resume = 0);
void URIStart(FetchResult &Res);
void URIDone(FetchResult &Res,FetchResult *Alt = 0);
public:
@ -56,7 +64,10 @@ class pkgAcqMethod
enum CnfFlags {SingleInstance = (1<<0), PreScan = (1<<1),
Pipeline = (1<<2), SendConfig = (1<<3)};
int Run();
void Log(const char *Format,...);
void Status(const char *Format,...);
int Run(bool Single = false);
pkgAcqMethod(const char *Ver,unsigned long Flags = 0);
virtual ~pkgAcqMethod() {};

6
apt-pkg/acquire-worker.cc

@ -1,6 +1,6 @@
// -*- mode: cpp; mode: fold -*-
// Description /*{{{*/
// $Id: acquire-worker.cc,v 1.8 1998/10/30 07:53:35 jgg Exp $
// $Id: acquire-worker.cc,v 1.9 1998/11/01 05:27:33 jgg Exp $
/* ######################################################################
Acquire Worker
@ -237,9 +237,9 @@ bool pkgAcquire::Worker::RunMessages()
break;
}
OwnerQ->ItemDone(Itm);
Itm->Owner->Done(Message,atoi(LookupTag(Message,"Size","0").c_str()),
LookupTag(Message,"MD5-Hash"));
OwnerQ->ItemDone(Itm);
break;
}
@ -252,8 +252,8 @@ bool pkgAcquire::Worker::RunMessages()
break;
}
Itm->Owner->Failed(Message);
OwnerQ->ItemDone(Itm);
Itm->Owner->Failed(Message);
break;
}

19
apt-pkg/acquire.cc

@ -1,6 +1,6 @@
// -*- mode: cpp; mode: fold -*-
// Description /*{{{*/
// $Id: acquire.cc,v 1.6 1998/10/30 07:53:37 jgg Exp $
// $Id: acquire.cc,v 1.7 1998/11/01 05:27:34 jgg Exp $
/* ######################################################################
Acquire - File Acquiration
@ -290,21 +290,23 @@ bool pkgAcquire::Run()
if (_error->PendingError() == true)
break;
}
// Shut down the acquire bits
Running = false;
for (Queue *I = Queues; I != 0; I = I->Next)
I->Shutdown();
Running = false;
return _error->PendingError();
}
/*}}}*/
// pkgAcquire::Bump - Called when an item is dequeued /*{{{*/
// Acquire::Bump - Called when an item is dequeued /*{{{*/
// ---------------------------------------------------------------------
/* This routine bumps idle queues in hopes that they will be able to fetch
the dequeued item */
void pkgAcquire::Bump()
{
for (Queue *I = Queues; I != 0; I = I->Next)
I->Bump();
}
/*}}}*/
@ -476,3 +478,10 @@ bool pkgAcquire::Queue::Cycle()
return Workers->QueueItem(I);
}
/*}}}*/
// Queue::Bump - Fetch any pending objects if we are idle /*{{{*/
// ---------------------------------------------------------------------
/* */
void pkgAcquire::Queue::Bump()
{
}
/*}}}*/

3
apt-pkg/acquire.h

@ -1,6 +1,6 @@
// -*- mode: cpp; mode: fold -*-
// Description /*{{{*/
// $Id: acquire.h,v 1.6 1998/10/30 07:53:38 jgg Exp $
// $Id: acquire.h,v 1.7 1998/11/01 05:27:35 jgg Exp $
/* ######################################################################
Acquire - File Acquiration
@ -133,6 +133,7 @@ class pkgAcquire::Queue
bool Startup();
bool Shutdown();
bool Cycle();
void Bump();
Queue(string Name,pkgAcquire *Owner);
~Queue();

4
apt-pkg/contrib/md5.cc

@ -1,6 +1,6 @@
// -*- mode: cpp; mode: fold -*-
// Description /*{{{*/
// $Id: md5.cc,v 1.1 1998/10/31 05:19:59 jgg Exp $
// $Id: md5.cc,v 1.2 1998/11/01 05:27:36 jgg Exp $
/* ######################################################################
MD5Sum - MD5 Message Digest Algorithm.
@ -40,7 +40,7 @@
/*}}}*/
// Include Files /*{{{*/
#ifdef __GNUG__
#pragma interface "apt-pkg/md5.h"
#pragma implementation "apt-pkg/md5.h"
#endif
#include <apt-pkg/md5.h>

6
apt-pkg/contrib/strutl.cc

@ -1,6 +1,6 @@
// -*- mode: cpp; mode: fold -*-
// Description /*{{{*/
// $Id: strutl.cc,v 1.9 1998/10/30 07:53:45 jgg Exp $
// $Id: strutl.cc,v 1.10 1998/11/01 05:27:37 jgg Exp $
/* ######################################################################
String Util - Some usefull string functions.
@ -609,10 +609,10 @@ bool StrToTime(string Val,time_t &Result)
}
/*}}}*/
// URI::URI - Constructor /*{{{*/
// URI::CopyFrom - Copy from an object /*{{{*/
// ---------------------------------------------------------------------
/* This parses the URI into all of its components */
URI::URI(string U)
void URI::CopyFrom(string U)
{
string::const_iterator I = U.begin();

15
apt-pkg/contrib/strutl.h

@ -1,6 +1,6 @@
// -*- mode: cpp; mode: fold -*-
// Description /*{{{*/
// $Id: strutl.h,v 1.8 1998/10/30 07:53:46 jgg Exp $
// $Id: strutl.h,v 1.9 1998/11/01 05:27:38 jgg Exp $
/* ######################################################################
String Util - These are some usefull string functions
@ -13,8 +13,6 @@
##################################################################### */
/*}}}*/
// This is a private header
// Header section: /
#ifndef STRUTL_H
#define STRUTL_H
@ -40,11 +38,15 @@ bool ReadMessages(int Fd, vector<string> &List);
int stringcmp(const char *A,const char *AEnd,const char *B,const char *BEnd);
inline int stringcmp(const char *A,const char *AEnd,const char *B) {return stringcmp(A,AEnd,B,B+strlen(B));};
inline int stringcmp(string A,const char *B) {return stringcmp(A.begin(),A.end(),B,B+strlen(B));};
int stringcasecmp(const char *A,const char *AEnd,const char *B,const char *BEnd);
inline int stringcasecmp(const char *A,const char *AEnd,const char *B) {return stringcasecmp(A,AEnd,B,B+strlen(B));};
inline int stringcasecmp(string A,const char *B) {return stringcasecmp(A.begin(),A.end(),B,B+strlen(B));};
class URI
{
void CopyFrom(string From);
public:
string Access;
@ -54,9 +56,12 @@ class URI
string Path;
unsigned int Port;
operator string();
inline operator string();
inline operator =(string From) {CopyFrom(From);};
inline bool empty() {return Access.empty();};
URI(string Path);
URI(string Path) {CopyFrom(Path);};
URI() : Port(0) {};
};
#endif

33
doc/Bugs

@ -2,9 +2,6 @@
#24000: Bug in apt Version: 0.0.17-1bo0
Summary: Couldn't locate an archive source for a package
Status: Require Status file.
#27601: srange errors from dselect
Summary: Couldn't locate an archive source
Status: Require status file
#26670: apt: apt-get dumps core after checking integrity
Summary: Some terminal environments cause APT to crash
Win95 telnet and emacs term-in-a-buffer are two at least
@ -29,12 +26,15 @@
Status: Probable that 0.3.x will have support for configuing some
parameters
#22892: Apt improvements
#28184: apt could be smarted regarding mirrors
Summary: Make use of redundant sources
Status: 0.3.0 will likely do this, already the information for it is stored.
#24799: Some suggestions for the apt method in dselect
Summary: Wants to be able to specifiy -d from dselect
Status: Likely a APT_OPTIONS enviornment variable will be created, -d can
be put there.
There is already an APT_CONFIG in 0.3, APT_OPTIONS may also
appear..
#25104: APT should retry failed downloads
Summary: FTP method has problems with busy servers
Status: The 0.3.0 ftp method should probably use the configuration mechanism
@ -44,6 +44,14 @@
Summary: FTP method has no controls for firewalls
Status: The 0.3.0 ftp method should probably use the configuration mechanism
to control this desired behavoir.
#28373: apt package is missing information on ftp.conf
Summary: The man pages have references to several non-existent items,
ftp.conf is only one of them.
Status: Fix the man pages. This certainly will be done in 0.3.0
#28391: apt-get install without upgrading
Summary: Make install leave the package in the keep state if it is already
installed
Status: Will be implemented in 0.3.0
-- Fixed but unclosed things
#25026: apt: Why do you list the packages you're _not_ doing anything to instead of the ones you are?
@ -59,7 +67,14 @@
Status: 0.3.0 has substantially better support for this to the point
that it is doable by using a seperate configuration file and
the -c option
#27601: srange errors from dselect
Summary: Couldn't locate an archive source
Status: Require status file
Believed to be fixed in 0.1.9, was not reproducable w/ given
status file
#27841: apt: apt depends on a missing library
Status: New versions of APT in slink have been compiled with libstdc++2.9
-- Silly things
#26592: apt: Problems with ftpd in SunOS 5.6
Summary: SunOS ftpd does not support the SIZE command
@ -74,6 +89,7 @@
Status: Impossible to do without an index file for all source archives.
#26019: apt may report wrong line speed
#26433: apt: claims to fetch things it doesn't actually fetch (wishlist)
#28778: apt: apt's fetched message is wrong for update of packages files
Summary: APT includes the fetch time from the local cache in its
calculations
Status: Probably will be fixed with new acquire code
@ -93,3 +109,12 @@
Summary: Wants to know what package files were not updated
Status: There is no place for this in the current apt-get design,
probably won't make the GUI either.
#28172: HTTP Proxy cache refresh should be forced for corrupted packages
Summary: Some problem resulted in a corrupted package
Status: I belive this reflects a deeper problem and the suggested solution
is only a band-aide patch. I intend to close this bug when #24685
is fixed with a configuration directive.
#27646: Apt: dpkg --merge-avail
Summary: Suggestion to call merge avail after each update operation
Status: Unlikely. The dpkg --print-avail functions should be obsoleted
by the apt-query program which should be written.

13
methods/copy.cc

@ -1,6 +1,6 @@
// -*- mode: cpp; mode: fold -*-
// Description /*{{{*/
// $Id: copy.cc,v 1.4 1998/10/30 07:53:51 jgg Exp $
// $Id: copy.cc,v 1.5 1998/11/01 05:27:40 jgg Exp $
/* ######################################################################
Copy URI - This method takes a uri like a file: uri and copies it
@ -20,7 +20,7 @@
class CopyMethod : public pkgAcqMethod
{
virtual bool Fetch(string Message,URI Get);
virtual bool Fetch(FetchItem *Itm);
public:
@ -30,13 +30,14 @@ class CopyMethod : public pkgAcqMethod
// CopyMethod::Fetch - Fetch a file /*{{{*/
// ---------------------------------------------------------------------
/* */
bool CopyMethod::Fetch(string Message,URI Get)
bool CopyMethod::Fetch(FetchItem *Itm)
{
URI Get = Itm->Uri;
string File = Get.Path;
// See if the file exists
FileFd From(File,FileFd::ReadOnly);
FileFd To(DestFile,FileFd::WriteEmpty);
FileFd To(Itm->DestFile,FileFd::WriteEmpty);
To.EraseOnFailure();
if (_error->PendingError() == true)
return false;
@ -59,7 +60,7 @@ bool CopyMethod::Fetch(string Message,URI Get)
struct utimbuf TimeBuf;
TimeBuf.actime = Buf.st_atime;
TimeBuf.modtime = Buf.st_mtime;
if (utime(DestFile.c_str(),&TimeBuf) != 0)
if (utime(Itm->DestFile.c_str(),&TimeBuf) != 0)
{
To.OpFail();
return _error->Errno("utime","Failed to set modification time");
@ -68,7 +69,7 @@ bool CopyMethod::Fetch(string Message,URI Get)
// Forumulate a result
FetchResult Res;
Res.Size = Buf.st_size;
Res.Filename = DestFile;
Res.Filename = Itm->DestFile;
Res.LastModified = Buf.st_mtime;
Res.IMSHit = false;

11
methods/file.cc

@ -1,6 +1,6 @@
// -*- mode: cpp; mode: fold -*-
// Description /*{{{*/
// $Id: file.cc,v 1.4 1998/10/30 07:53:52 jgg Exp $
// $Id: file.cc,v 1.5 1998/11/01 05:27:41 jgg Exp $
/* ######################################################################
File URI method for APT
@ -22,7 +22,7 @@
class FileMethod : public pkgAcqMethod
{
virtual bool Fetch(string Message,URI Get);
virtual bool Fetch(FetchItem *Itm);
public:
@ -32,8 +32,9 @@ class FileMethod : public pkgAcqMethod
// FileMethod::Fetch - Fetch a file /*{{{*/
// ---------------------------------------------------------------------
/* */
bool FileMethod::Fetch(string Message,URI Get)
bool FileMethod::Fetch(FetchItem *Itm)
{
URI Get = Itm->Uri;
string File = Get.Path;
FetchResult Res;
@ -45,7 +46,7 @@ bool FileMethod::Fetch(string Message,URI Get)
Res.Filename = File;
Res.LastModified = Buf.st_mtime;
Res.IMSHit = false;
if (LastModified == Buf.st_mtime)
if (Itm->LastModified == Buf.st_mtime)
Res.IMSHit = true;
}
@ -61,7 +62,7 @@ bool FileMethod::Fetch(string Message,URI Get)
AltRes.Filename = File;
AltRes.LastModified = Buf.st_mtime;
AltRes.IMSHit = false;
if (LastModified == Buf.st_mtime)
if (Itm->LastModified == Buf.st_mtime)
AltRes.IMSHit = true;
URIDone(Res,&AltRes);

14
methods/gzip.cc

@ -1,6 +1,6 @@
// -*- mode: cpp; mode: fold -*-
// Description /*{{{*/
// $Id: gzip.cc,v 1.3 1998/10/30 07:53:53 jgg Exp $
// $Id: gzip.cc,v 1.4 1998/11/01 05:27:43 jgg Exp $
/* ######################################################################
GZip method - Take a file URI in and decompress it into the target
@ -23,7 +23,7 @@
class GzipMethod : public pkgAcqMethod
{
virtual bool Fetch(string Message,URI Get);
virtual bool Fetch(FetchItem *Itm);
public:
@ -33,11 +33,13 @@ class GzipMethod : public pkgAcqMethod
// GzipMethod::Fetch - Decompress the passed URI /*{{{*/
// ---------------------------------------------------------------------
/* */
bool GzipMethod::Fetch(string Message,URI Get)
bool GzipMethod::Fetch(FetchItem *Itm)
{
URI Get = Itm->Uri;
// Open the source and destintation files
FileFd From(Get.Path,FileFd::ReadOnly);
FileFd To(DestFile,FileFd::WriteEmpty);
FileFd To(Itm->DestFile,FileFd::WriteEmpty);
To.EraseOnFailure();
if (_error->PendingError() == true)
return false;
@ -90,13 +92,13 @@ bool GzipMethod::Fetch(string Message,URI Get)
struct utimbuf TimeBuf;
TimeBuf.actime = Buf.st_atime;
TimeBuf.modtime = Buf.st_mtime;
if (utime(DestFile.c_str(),&TimeBuf) != 0)
if (utime(Itm->DestFile.c_str(),&TimeBuf) != 0)
return _error->Errno("utime","Failed to set modification time");
// Return a Done response
FetchResult Res;
Res.LastModified = Buf.st_mtime;
Res.Filename = DestFile;
Res.Filename = Itm->DestFile;
URIDone(Res);
return true;

892
methods/http.cc

@ -0,0 +1,892 @@
// -*- mode: cpp; mode: fold -*-
// Description /*{{{*/
// $Id: http.cc,v 1.1 1998/11/01 05:30:47 jgg Exp $
/* ######################################################################
HTTP Aquire Method - This is the HTTP aquire method for APT.
It uses HTTP/1.1 and many of the fancy options there-in, such as
pipelining, range, if-range and so on. It accepts on the command line
a list of url destination pairs and writes to stdout the status of the
operation as defined in the APT method spec.
It is based on a doubly buffered select loop. All the requests are
fed into a single output buffer that is constantly fed out the
socket. This provides ideal pipelining as in many cases all of the
requests will fit into a single packet. The input socket is buffered
the same way and fed into the fd for the file.
This double buffering provides fairly substantial transfer rates,
compared to wget the http method is about 4% faster. Most importantly,
when HTTP is compared with FTP as a protocol the speed difference is
huge. In tests over the internet from two sites to llug (via ATM) this
program got 230k/s sustained http transfer rates. FTP on the other
hand topped out at 170k/s. That combined with the time to setup the
FTP connection makes HTTP a vastly superior protocol.
##################################################################### */
/*}}}*/
// Include Files /*{{{*/
#include <apt-pkg/fileutl.h>
#include <apt-pkg/acquire-method.h>
#include <apt-pkg/error.h>
#include <apt-pkg/md5.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <utime.h>
#include <unistd.h>
#include <stdio.h>
// Internet stuff
#include <netinet/in.h>
#include <sys/socket.h>
#include <arpa/inet.h>
#include <netdb.h>
#include "http.h"
/*}}}*/
// CircleBuf::CircleBuf - Circular input buffer /*{{{*/
// ---------------------------------------------------------------------
/* */
CircleBuf::CircleBuf(unsigned long Size) : Size(Size), MD5(0)
{
Buf = new unsigned char[Size];
Reset();
}
/*}}}*/
// CircleBuf::Reset - Reset to the default state /*{{{*/
// ---------------------------------------------------------------------
/* */
void CircleBuf::Reset()
{
InP = 0;
OutP = 0;
StrPos = 0;
MaxGet = (unsigned int)-1;
OutQueue = string();
if (MD5 != 0)
{
delete MD5;
MD5 = new MD5Summation;
}
};
/*}}}*/
// CircleBuf::Read - Read from a FD into the circular buffer /*{{{*/
// ---------------------------------------------------------------------
/* This fills up the buffer with as much data as is in the FD, assuming it
is non-blocking.. */
bool CircleBuf::Read(int Fd)
{
while (1)
{
// Woops, buffer is full
if (InP - OutP == Size)
return true;
// Write the buffer segment
int Res;
Res = read(Fd,Buf + (InP%Size),LeftRead());
if (Res == 0)
return false;
if (Res < 0)
{
if (errno == EAGAIN)
return true;
return false;
}
if (InP == 0)
gettimeofday(&Start,0);
InP += Res;
}
}
/*}}}*/
// CircleBuf::Read - Put the string into the buffer /*{{{*/
// ---------------------------------------------------------------------
/* This will hold the string in and fill the buffer with it as it empties */
bool CircleBuf::Read(string Data)
{
OutQueue += Data;
FillOut();
return true;
}
/*}}}*/
// CircleBuf::FillOut - Fill the buffer from the output queue /*{{{*/
// ---------------------------------------------------------------------
/* */
void CircleBuf::FillOut()
{
if (OutQueue.empty() == true)
return;
while (1)
{
// Woops, buffer is full
if (InP - OutP == Size)
return;
// Write the buffer segment
unsigned long Sz = LeftRead();
if (OutQueue.length() - StrPos < Sz)
Sz = OutQueue.length() - StrPos;
memcpy(Buf + (InP%Size),OutQueue.begin() + StrPos,Sz);
// Advance
StrPos += Sz;
InP += Sz;
if (OutQueue.length() == StrPos)
{
StrPos = 0;
OutQueue = "";
return;
}
}
}
/*}}}*/
// CircleBuf::Write - Write from the buffer into a FD /*{{{*/
// ---------------------------------------------------------------------
/* This empties the buffer into the FD. */
bool CircleBuf::Write(int Fd)
{
while (1)
{
FillOut();
// Woops, buffer is empty
if (OutP == InP)
return true;
if (OutP == MaxGet)
return true;
// Write the buffer segment
int Res;
Res = write(Fd,Buf + (OutP%Size),LeftWrite());
if (Res == 0)
return false;
if (Res < 0)
{
if (errno == EAGAIN)
return true;
return false;
}
if (MD5 != 0)
MD5->Add(Buf + (OutP%Size),Res);
OutP += Res;
}
}
/*}}}*/
// CircleBuf::WriteTillEl - Write from the buffer to a string /*{{{*/
// ---------------------------------------------------------------------
/* This copies till the first empty line */
bool CircleBuf::WriteTillEl(string &Data,bool Single)
{
// We cheat and assume it is unneeded to have more than one buffer load
for (unsigned long I = OutP; I < InP; I++)
{
if (Buf[I%Size] != '\n')
continue;
for (I++; I < InP && Buf[I%Size] == '\r'; I++);
if (Single == false)
{
if (Buf[I%Size] != '\n')
continue;
for (I++; I < InP && Buf[I%Size] == '\r'; I++);
}
if (I > InP)
I = InP;
Data = "";
while (OutP < I)
{
unsigned long Sz = LeftWrite();
if (Sz == 0)
return false;
if (I - OutP < LeftWrite())
Sz = I - OutP;
Data += string((char *)(Buf + (OutP%Size)),Sz);
OutP += Sz;
}
return true;
}
return false;
}
/*}}}*/
// CircleBuf::Stats - Print out stats information /*{{{*/
// ---------------------------------------------------------------------
/* */
void CircleBuf::Stats()
{
if (InP == 0)
return;
struct timeval Stop;
gettimeofday(&Stop,0);
/* float Diff = Stop.tv_sec - Start.tv_sec +
(float)(Stop.tv_usec - Start.tv_usec)/1000000;
clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
}
/*}}}*/
// ServerState::ServerState - Constructor /*{{{*/
// ---------------------------------------------------------------------
/* */
ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
In(64*1024), Out(1*1024),
ServerName(Srv)
{
Reset();
}
/*}}}*/
// ServerState::Open - Open a connection to the server /*{{{*/
// ---------------------------------------------------------------------
/* This opens a connection to the server. */
string LastHost;
in_addr LastHostA;
bool ServerState::Open()
{
Close();
int Port;
string Host;
if (Proxy.empty() == false)
{
Port = ServerName.Port;
Host = ServerName.Host;
}
else
{
Port = Proxy.Port;
Host = Proxy.Host;
}
if (LastHost != Host)
{
Owner->Status("Connecting to %s",Host.c_str());
// Lookup the host
hostent *Addr = gethostbyname(Host.c_str());
if (Addr == 0)
return _error->Errno("gethostbyname","Could not lookup host %s",Host.c_str());
LastHost = Host;
LastHostA = *(in_addr *)(Addr->h_addr_list[0]);
}
Owner->Status("Connecting to %s (%s)",Host.c_str(),inet_ntoa(LastHostA));
// Get a socket
if ((ServerFd = socket(AF_INET,SOCK_STREAM,0)) < 0)
return _error->Errno("socket","Could not create a socket");
// Connect to the server
struct sockaddr_in server;
server.sin_family = AF_INET;
server.sin_port = htons(Port);
server.sin_addr = LastHostA;
if (connect(ServerFd,(sockaddr *)&server,sizeof(server)) < 0)
return _error->Errno("socket","Could not create a socket");
SetNonBlock(ServerFd,true);
return true;
}
/*}}}*/
// ServerState::Close - Close a connection to the server /*{{{*/
// ---------------------------------------------------------------------
/* */
bool ServerState::Close()
{
close(ServerFd);
ServerFd = -1;
In.Reset();
Out.Reset();
return true;
}
/*}}}*/
// ServerState::RunHeaders - Get the headers before the data /*{{{*/
// ---------------------------------------------------------------------
/* */
bool ServerState::RunHeaders()
{
State = Header;
Owner->Status("Waiting for file");
Major = 0;
Minor = 0;
Result = 0;
Size = 0;
StartPos = 0;
Encoding = Closes;
time(&Date);
do
{
string Data;
if (In.WriteTillEl(Data) == false)
continue;
for (string::const_iterator I = Data.begin(); I < Data.end(); I++)
{
string::const_iterator J = I;
for (; J != Data.end() && *J != '\n' && *J != '\r';J++);
if (HeaderLine(string(I,J-I)) == false)
return false;
I = J;
}
return true;
}
while (Owner->Go(false,this) == true);
return false;
}
/*}}}*/
// ServerState::RunData - Transfer the data from the socket /*{{{*/
// ---------------------------------------------------------------------
/* */
bool ServerState::RunData()
{
State = Data;
// Chunked transfer encoding is fun..
if (Encoding == Chunked)
{
while (1)
{
// Grab the block size
bool Last = true;
string Data;
In.Limit(-1);
do
{
if (In.WriteTillEl(Data,true) == true)
break;
}
while ((Last = Owner->Go(false,this)) == true);
if (Last == false)
return false;
// See if we are done
unsigned long Len = strtol(Data.c_str(),0,16);
if (Len == 0)
{
In.Limit(-1);
// We have to remove the entity trailer
Last = true;
do
{
if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
break;
}
while ((Last = Owner->Go(false,this)) == true);
if (Last == false)
return false;
return true;
}
// Transfer the block
In.Limit(Len);
while (Owner->Go(true,this) == true)
if (In.IsLimit() == true)
break;
// Error
if (In.IsLimit() == false)
return false;
// The server sends an extra new line before the next block specifier..
In.Limit(-1);
Last = true;
do
{
if (In.WriteTillEl(Data,true) == true)
break;
}
while ((Last = Owner->Go(false,this)) == true);
if (Last == false)
return false;
}
}
else
{
/* Closes encoding is used when the server did not specify a size, the
loss of the connection means we are done */
if (Encoding == Closes)
In.Limit(-1);
else
In.Limit(Size - StartPos);
// Just transfer the whole block.
do
{
if (In.IsLimit() == false)
continue;
In.Limit(-1);
return true;
}
while (Owner->Go(true,this) == true);
}
return Owner->Flush(this);
}
/*}}}*/
// ServerState::HeaderLine - Process a header line /*{{{*/
// ---------------------------------------------------------------------
/* */
bool ServerState::HeaderLine(string Line)
{
if (Line.empty() == true)
return true;
// The http server might be trying to do something evil.
if (Line.length() >= MAXLEN)
return _error->Error("Got a single header line over %u chars",MAXLEN);
string::size_type Pos = Line.find(' ');
if (Pos == string::npos || Pos+1 > Line.length())
return _error->Error("Bad header line");
string Tag = string(Line,0,Pos);
string Val = string(Line,Pos+1);
if (stringcasecmp(Tag,"HTTP") == 0)
{
// Evil servers return no version
if (Line[4] == '/')
{
if (sscanf(Line.c_str(),"HTTP/%u.%u %u %[^\n]",&Major,&Minor,
&Result,Code) != 4)
return _error->Error("The http server sent an invalid reply header");
}
else
{
Major = 0;
Minor = 9;
if (sscanf(Line.c_str(),"HTTP %u %[^\n]",&Result,Code) != 2)
return _error->Error("The http server sent an invalid reply header");
}
return true;
}
if (stringcasecmp(Tag,"Content-Length:"))
{
if (Encoding == Closes)
Encoding = Stream;
// The length is already set from the Content-Range header
if (StartPos != 0)
return true;
if (sscanf(Val.c_str(),"%lu",&Size) != 1)
return _error->Error("The http server sent an invalid Content-Length header");
return true;
}
if (stringcasecmp(Tag,"Content-Range:"))
{
if (sscanf(Val.c_str(),"bytes %lu-%*u/%lu",&StartPos,&Size) != 2)
return _error->Error("The http server sent an invalid Content-Range header");
if ((unsigned)StartPos > Size)
return _error->Error("This http server has broken range support");
return true;
}
if (stringcasecmp(Tag,"Transfer-Encoding:"))
{
if (stringcasecmp(Val,"chunked"))
Encoding = Chunked;
return true;
}
if (stringcasecmp(Tag,"Last-Modified:"))
{
if (StrToTime(Val,Date) == false)
return _error->Error("Unknown date format");
return true;
}
return true;
}
/*}}}*/
// HttpMethod::SendReq - Send the HTTP request /*{{{*/
// ---------------------------------------------------------------------
/* This places the http request in the outbound buffer */
void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
{
URI Uri = Itm->Uri;
// The HTTP server expects a hostname with a trailing :port
char Buf[300];
string ProperHost = Uri.Host;
if (Uri.Port != 0)
{
sprintf(Buf,":%u",Uri.Port);
ProperHost += Buf;
}
// Build the request
if (Proxy.empty() == true)
sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",
Uri.Path.c_str(),ProperHost.c_str());
else
sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
Itm->Uri.c_str(),ProperHost.c_str());
string Req = Buf;
// Check for a partial file
struct stat SBuf;
if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
{
// In this case we send an if-range query with a range header
sprintf(Buf,"Range: bytes=%li-\r\nIf-Range: %s\r\n",SBuf.st_size - 1,
TimeRFC1123(SBuf.st_mtime).c_str());
Req += Buf;
}
else
{
if (Itm->LastModified != 0)
{
sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str());
Req += Buf;
}
}
/* if (ProxyAuth.empty() == false)
Req += string("Proxy-Authorization: Basic ") + Base64Encode(ProxyAuth) + "\r\n";*/
Req += "User-Agent: Debian APT-HTTP/1.2\r\n\r\n";
Out.Read(Req);
}
/*}}}*/
// HttpMethod::Go - Run a single loop /*{{{*/
// ---------------------------------------------------------------------
/* This runs the select loop over the server FDs, Output file FDs and
stdin. */
bool HttpMethod::Go(bool ToFile,ServerState *Srv)
{
// Server has closed the connection
if (Srv->ServerFd == -1 && Srv->In.WriteSpace() == false)
return false;
fd_set rfds,wfds,efds;
FD_ZERO(&rfds);
FD_ZERO(&wfds);
FD_ZERO(&efds);
// Add the server
if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1)
FD_SET(Srv->ServerFd,&wfds);
if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1)
FD_SET(Srv->ServerFd,&rfds);
// Add the file
int FileFD = -1;
if (File != 0)
FileFD = File->Fd();
if (Srv->In.WriteSpace() == true && ToFile == true && FileFD != -1)
FD_SET(FileFD,&wfds);
// Add stdin
FD_SET(STDIN_FILENO,&rfds);
// Error Set
if (FileFD != -1)
FD_SET(FileFD,&efds);
if (Srv->ServerFd != -1)
FD_SET(Srv->ServerFd,&efds);
// Figure out the max fd
int MaxFd = FileFD;
if (MaxFd < Srv->ServerFd)
MaxFd = Srv->ServerFd;
// Select
struct timeval tv;
tv.tv_sec = 120;
tv.tv_usec = 0;
int Res = 0;
if ((Res = select(MaxFd+1,&rfds,&wfds,&efds,&tv)) < 0)
return _error->Errno("select","Select failed");
if (Res == 0)
{
_error->Error("Connection timed out");
return ServerDie(Srv);
}
// Some kind of exception (error) on the sockets, die
if ((FileFD != -1 && FD_ISSET(FileFD,&efds)) ||
(Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&efds)))
return _error->Error("Socket Exception");
// Handle server IO
if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&rfds))
{
errno = 0;
if (Srv->In.Read(Srv->ServerFd) == false)
return ServerDie(Srv);
}
if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&wfds))
{
errno = 0;
if (Srv->Out.Write(Srv->ServerFd) == false)
return ServerDie(Srv);
}
// Send data to the file
if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
{
if (Srv->In.Write(FileFD) == false)
return _error->Errno("write","Error writing to output file");
}
// Handle commands from APT
if (FD_ISSET(STDIN_FILENO,&rfds))
{
if (Run(true) != 0)
exit(100);
}
return true;
}
/*}}}*/
// HttpMethod::Flush - Dump the buffer into the file /*{{{*/
// ---------------------------------------------------------------------
/* This takes the current input buffer from the Server FD and writes it
into the file */
bool HttpMethod::Flush(ServerState *Srv)
{
if (File != 0)
{
SetNonBlock(File->Fd(),false);
if (Srv->In.WriteSpace() == false)
return true;
while (Srv->In.WriteSpace() == true)
{
if (Srv->In.Write(File->Fd()) == false)
return _error->Errno("write","Error writing to file");
}
if (Srv->In.IsLimit() == true || Srv->Encoding == ServerState::Closes)
return true;
}
return false;
}
/*}}}*/
// HttpMethod::ServerDie - The server has closed the connection. /*{{{*/
// ---------------------------------------------------------------------
/* */
bool HttpMethod::ServerDie(ServerState *Srv)
{
// Dump the buffer to the file
if (Srv->State == ServerState::Data)
{
SetNonBlock(File->Fd(),false);
while (Srv->In.WriteSpace() == true)
{
if (Srv->In.Write(File->Fd()) == false)
return _error->Errno("write","Error writing to the file");
}
}
// See if this is because the server finished the data stream
if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header &&
Srv->Encoding != ServerState::Closes)
{
if (errno == 0)
return _error->Error("Error reading from server Remote end closed connection");
return _error->Errno("read","Error reading from server");
}
else
{
Srv->In.Limit(-1);
// Nothing left in the buffer
if (Srv->In.WriteSpace() == false)
return false;
// We may have got multiple responses back in one packet..
Srv->Close();
return true;
}
return false;
}
/*}}}*/
// HttpMethod::DealWithHeaders - Handle the retrieved header data /*{{{*/
// ---------------------------------------------------------------------
/* We look at the header data we got back from the server and decide what
to do. Returns
0 - File is open,
1 - IMS hit
3 - Unrecoverable error */
int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
{
// Not Modified
if (Srv->Result == 304)
{
unlink(Queue->DestFile.c_str());
Res.IMSHit = true;
Res.LastModified = Queue->LastModified;
return 1;
}
/* We have a reply we dont handle. This should indicate a perm server
failure */
if (Srv->Result < 200 || Srv->Result >= 300)
{
_error->Error("%u %s",Srv->Result,Srv->Code);
return 3;
}
// This is some sort of 2xx 'data follows' reply
Res.LastModified = Srv->Date;
Res.Size = Srv->Size;
// Open the file
delete File;
File = new FileFd(Queue->DestFile,FileFd::WriteAny);
if (_error->PendingError() == true)
return 3;
// Set the expected size
if (Srv->StartPos >= 0)
{
Res.ResumePoint = Srv->StartPos;
ftruncate(File->Fd(),Srv->StartPos);
}
// Set the start point
lseek(File->Fd(),0,SEEK_END);
delete Srv->In.MD5;
Srv->In.MD5 = new MD5Summation;
// Fill the MD5 Hash if the file is non-empty (resume)
if (Srv->StartPos > 0)
{
lseek(File->Fd(),0,SEEK_SET);
if (Srv->In.MD5->AddFD(File->Fd(),Srv->StartPos) == false)
{
_error->Errno("read","Problem hashing file");
return 3;
}