You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 

892 lines
22 KiB

// -*- mode: cpp; mode: fold -*-
// Description /*{{{*/
// $Id: http.cc,v 1.1 1998/11/01 05:30:47 jgg Exp $
/* ######################################################################
HTTP Aquire Method - This is the HTTP aquire method for APT.
It uses HTTP/1.1 and many of the fancy options there-in, such as
pipelining, range, if-range and so on. It accepts on the command line
a list of url destination pairs and writes to stdout the status of the
operation as defined in the APT method spec.
It is based on a doubly buffered select loop. All the requests are
fed into a single output buffer that is constantly fed out the
socket. This provides ideal pipelining as in many cases all of the
requests will fit into a single packet. The input socket is buffered
the same way and fed into the fd for the file.
This double buffering provides fairly substantial transfer rates,
compared to wget the http method is about 4% faster. Most importantly,
when HTTP is compared with FTP as a protocol the speed difference is
huge. In tests over the internet from two sites to llug (via ATM) this
program got 230k/s sustained http transfer rates. FTP on the other
hand topped out at 170k/s. That combined with the time to setup the
FTP connection makes HTTP a vastly superior protocol.
##################################################################### */
/*}}}*/
// Include Files /*{{{*/
#include <apt-pkg/fileutl.h>
#include <apt-pkg/acquire-method.h>
#include <apt-pkg/error.h>
#include <apt-pkg/md5.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <utime.h>
#include <unistd.h>
#include <stdio.h>
// Internet stuff
#include <netinet/in.h>
#include <sys/socket.h>
#include <arpa/inet.h>
#include <netdb.h>
#include "http.h"
/*}}}*/
// CircleBuf::CircleBuf - Circular input buffer /*{{{*/
// ---------------------------------------------------------------------
/* */
CircleBuf::CircleBuf(unsigned long Size) : Size(Size), MD5(0)
{
Buf = new unsigned char[Size];
Reset();
}
/*}}}*/
// CircleBuf::Reset - Reset to the default state /*{{{*/
// ---------------------------------------------------------------------
/* */
void CircleBuf::Reset()
{
InP = 0;
OutP = 0;
StrPos = 0;
MaxGet = (unsigned int)-1;
OutQueue = string();
if (MD5 != 0)
{
delete MD5;
MD5 = new MD5Summation;
}
};
/*}}}*/
// CircleBuf::Read - Read from a FD into the circular buffer /*{{{*/
// ---------------------------------------------------------------------
/* This fills up the buffer with as much data as is in the FD, assuming it
is non-blocking.. */
bool CircleBuf::Read(int Fd)
{
while (1)
{
// Woops, buffer is full
if (InP - OutP == Size)
return true;
// Write the buffer segment
int Res;
Res = read(Fd,Buf + (InP%Size),LeftRead());
if (Res == 0)
return false;
if (Res < 0)
{
if (errno == EAGAIN)
return true;
return false;
}
if (InP == 0)
gettimeofday(&Start,0);
InP += Res;
}
}
/*}}}*/
// CircleBuf::Read - Put the string into the buffer /*{{{*/
// ---------------------------------------------------------------------
/* This will hold the string in and fill the buffer with it as it empties */
bool CircleBuf::Read(string Data)
{
OutQueue += Data;
FillOut();
return true;
}
/*}}}*/
// CircleBuf::FillOut - Fill the buffer from the output queue /*{{{*/
// ---------------------------------------------------------------------
/* */
void CircleBuf::FillOut()
{
if (OutQueue.empty() == true)
return;
while (1)
{
// Woops, buffer is full
if (InP - OutP == Size)
return;
// Write the buffer segment
unsigned long Sz = LeftRead();
if (OutQueue.length() - StrPos < Sz)
Sz = OutQueue.length() - StrPos;
memcpy(Buf + (InP%Size),OutQueue.begin() + StrPos,Sz);
// Advance
StrPos += Sz;
InP += Sz;
if (OutQueue.length() == StrPos)
{
StrPos = 0;
OutQueue = "";
return;
}
}
}
/*}}}*/
// CircleBuf::Write - Write from the buffer into a FD /*{{{*/
// ---------------------------------------------------------------------
/* This empties the buffer into the FD. */
bool CircleBuf::Write(int Fd)
{
while (1)
{
FillOut();
// Woops, buffer is empty
if (OutP == InP)
return true;
if (OutP == MaxGet)
return true;
// Write the buffer segment
int Res;
Res = write(Fd,Buf + (OutP%Size),LeftWrite());
if (Res == 0)
return false;
if (Res < 0)
{
if (errno == EAGAIN)
return true;
return false;
}
if (MD5 != 0)
MD5->Add(Buf + (OutP%Size),Res);
OutP += Res;
}
}
/*}}}*/
// CircleBuf::WriteTillEl - Write from the buffer to a string /*{{{*/
// ---------------------------------------------------------------------
/* This copies till the first empty line */
bool CircleBuf::WriteTillEl(string &Data,bool Single)
{
// We cheat and assume it is unneeded to have more than one buffer load
for (unsigned long I = OutP; I < InP; I++)
{
if (Buf[I%Size] != '\n')
continue;
for (I++; I < InP && Buf[I%Size] == '\r'; I++);
if (Single == false)
{
if (Buf[I%Size] != '\n')
continue;
for (I++; I < InP && Buf[I%Size] == '\r'; I++);
}
if (I > InP)
I = InP;
Data = "";
while (OutP < I)
{
unsigned long Sz = LeftWrite();
if (Sz == 0)
return false;
if (I - OutP < LeftWrite())
Sz = I - OutP;
Data += string((char *)(Buf + (OutP%Size)),Sz);
OutP += Sz;
}
return true;
}
return false;
}
/*}}}*/
// CircleBuf::Stats - Print out stats information /*{{{*/
// ---------------------------------------------------------------------
/* */
void CircleBuf::Stats()
{
if (InP == 0)
return;
struct timeval Stop;
gettimeofday(&Stop,0);
/* float Diff = Stop.tv_sec - Start.tv_sec +
(float)(Stop.tv_usec - Start.tv_usec)/1000000;
clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
}
/*}}}*/
// ServerState::ServerState - Constructor /*{{{*/
// ---------------------------------------------------------------------
/* */
ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
In(64*1024), Out(1*1024),
ServerName(Srv)
{
Reset();
}
/*}}}*/
// ServerState::Open - Open a connection to the server /*{{{*/
// ---------------------------------------------------------------------
/* This opens a connection to the server. */
string LastHost;
in_addr LastHostA;
bool ServerState::Open()
{
Close();
int Port;
string Host;
if (Proxy.empty() == false)
{
Port = ServerName.Port;
Host = ServerName.Host;
}
else
{
Port = Proxy.Port;
Host = Proxy.Host;
}
if (LastHost != Host)
{
Owner->Status("Connecting to %s",Host.c_str());
// Lookup the host
hostent *Addr = gethostbyname(Host.c_str());
if (Addr == 0)
return _error->Errno("gethostbyname","Could not lookup host %s",Host.c_str());
LastHost = Host;
LastHostA = *(in_addr *)(Addr->h_addr_list[0]);
}
Owner->Status("Connecting to %s (%s)",Host.c_str(),inet_ntoa(LastHostA));
// Get a socket
if ((ServerFd = socket(AF_INET,SOCK_STREAM,0)) < 0)
return _error->Errno("socket","Could not create a socket");
// Connect to the server
struct sockaddr_in server;
server.sin_family = AF_INET;
server.sin_port = htons(Port);
server.sin_addr = LastHostA;
if (connect(ServerFd,(sockaddr *)&server,sizeof(server)) < 0)
return _error->Errno("socket","Could not create a socket");
SetNonBlock(ServerFd,true);
return true;
}
/*}}}*/
// ServerState::Close - Close a connection to the server /*{{{*/
// ---------------------------------------------------------------------
/* */
bool ServerState::Close()
{
close(ServerFd);
ServerFd = -1;
In.Reset();
Out.Reset();
return true;
}
/*}}}*/
// ServerState::RunHeaders - Get the headers before the data /*{{{*/
// ---------------------------------------------------------------------
/* */
bool ServerState::RunHeaders()
{
State = Header;
Owner->Status("Waiting for file");
Major = 0;
Minor = 0;
Result = 0;
Size = 0;
StartPos = 0;
Encoding = Closes;
time(&Date);
do
{
string Data;
if (In.WriteTillEl(Data) == false)
continue;
for (string::const_iterator I = Data.begin(); I < Data.end(); I++)
{
string::const_iterator J = I;
for (; J != Data.end() && *J != '\n' && *J != '\r';J++);
if (HeaderLine(string(I,J-I)) == false)
return false;
I = J;
}
return true;
}
while (Owner->Go(false,this) == true);
return false;
}
/*}}}*/
// ServerState::RunData - Transfer the data from the socket /*{{{*/
// ---------------------------------------------------------------------
/* */
bool ServerState::RunData()
{
State = Data;
// Chunked transfer encoding is fun..
if (Encoding == Chunked)
{
while (1)
{
// Grab the block size
bool Last = true;
string Data;
In.Limit(-1);
do
{
if (In.WriteTillEl(Data,true) == true)
break;
}
while ((Last = Owner->Go(false,this)) == true);
if (Last == false)
return false;
// See if we are done
unsigned long Len = strtol(Data.c_str(),0,16);
if (Len == 0)
{
In.Limit(-1);
// We have to remove the entity trailer
Last = true;
do
{
if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
break;
}
while ((Last = Owner->Go(false,this)) == true);
if (Last == false)
return false;
return true;
}
// Transfer the block
In.Limit(Len);
while (Owner->Go(true,this) == true)
if (In.IsLimit() == true)
break;
// Error
if (In.IsLimit() == false)
return false;
// The server sends an extra new line before the next block specifier..
In.Limit(-1);
Last = true;
do
{
if (In.WriteTillEl(Data,true) == true)
break;
}
while ((Last = Owner->Go(false,this)) == true);
if (Last == false)
return false;
}
}
else
{
/* Closes encoding is used when the server did not specify a size, the
loss of the connection means we are done */
if (Encoding == Closes)
In.Limit(-1);
else
In.Limit(Size - StartPos);
// Just transfer the whole block.
do
{
if (In.IsLimit() == false)
continue;
In.Limit(-1);
return true;
}
while (Owner->Go(true,this) == true);
}
return Owner->Flush(this);
}
/*}}}*/
// ServerState::HeaderLine - Process a header line /*{{{*/
// ---------------------------------------------------------------------
/* */
bool ServerState::HeaderLine(string Line)
{
if (Line.empty() == true)
return true;
// The http server might be trying to do something evil.
if (Line.length() >= MAXLEN)
return _error->Error("Got a single header line over %u chars",MAXLEN);
string::size_type Pos = Line.find(' ');
if (Pos == string::npos || Pos+1 > Line.length())
return _error->Error("Bad header line");
string Tag = string(Line,0,Pos);
string Val = string(Line,Pos+1);
if (stringcasecmp(Tag,"HTTP") == 0)
{
// Evil servers return no version
if (Line[4] == '/')
{
if (sscanf(Line.c_str(),"HTTP/%u.%u %u %[^\n]",&Major,&Minor,
&Result,Code) != 4)
return _error->Error("The http server sent an invalid reply header");
}
else
{
Major = 0;
Minor = 9;
if (sscanf(Line.c_str(),"HTTP %u %[^\n]",&Result,Code) != 2)
return _error->Error("The http server sent an invalid reply header");
}
return true;
}
if (stringcasecmp(Tag,"Content-Length:"))
{
if (Encoding == Closes)
Encoding = Stream;
// The length is already set from the Content-Range header
if (StartPos != 0)
return true;
if (sscanf(Val.c_str(),"%lu",&Size) != 1)
return _error->Error("The http server sent an invalid Content-Length header");
return true;
}
if (stringcasecmp(Tag,"Content-Range:"))
{
if (sscanf(Val.c_str(),"bytes %lu-%*u/%lu",&StartPos,&Size) != 2)
return _error->Error("The http server sent an invalid Content-Range header");
if ((unsigned)StartPos > Size)
return _error->Error("This http server has broken range support");
return true;
}
if (stringcasecmp(Tag,"Transfer-Encoding:"))
{
if (stringcasecmp(Val,"chunked"))
Encoding = Chunked;
return true;
}
if (stringcasecmp(Tag,"Last-Modified:"))
{
if (StrToTime(Val,Date) == false)
return _error->Error("Unknown date format");
return true;
}
return true;
}
/*}}}*/
// HttpMethod::SendReq - Send the HTTP request /*{{{*/
// ---------------------------------------------------------------------
/* This places the http request in the outbound buffer */
void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
{
URI Uri = Itm->Uri;
// The HTTP server expects a hostname with a trailing :port
char Buf[300];
string ProperHost = Uri.Host;
if (Uri.Port != 0)
{
sprintf(Buf,":%u",Uri.Port);
ProperHost += Buf;
}
// Build the request
if (Proxy.empty() == true)
sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",
Uri.Path.c_str(),ProperHost.c_str());
else
sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
Itm->Uri.c_str(),ProperHost.c_str());
string Req = Buf;
// Check for a partial file
struct stat SBuf;
if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
{
// In this case we send an if-range query with a range header
sprintf(Buf,"Range: bytes=%li-\r\nIf-Range: %s\r\n",SBuf.st_size - 1,
TimeRFC1123(SBuf.st_mtime).c_str());
Req += Buf;
}
else
{
if (Itm->LastModified != 0)
{
sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str());
Req += Buf;
}
}
/* if (ProxyAuth.empty() == false)
Req += string("Proxy-Authorization: Basic ") + Base64Encode(ProxyAuth) + "\r\n";*/
Req += "User-Agent: Debian APT-HTTP/1.2\r\n\r\n";
Out.Read(Req);
}
/*}}}*/
// HttpMethod::Go - Run a single loop /*{{{*/
// ---------------------------------------------------------------------
/* This runs the select loop over the server FDs, Output file FDs and
stdin. */
bool HttpMethod::Go(bool ToFile,ServerState *Srv)
{
// Server has closed the connection
if (Srv->ServerFd == -1 && Srv->In.WriteSpace() == false)
return false;
fd_set rfds,wfds,efds;
FD_ZERO(&rfds);
FD_ZERO(&wfds);
FD_ZERO(&efds);
// Add the server
if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1)
FD_SET(Srv->ServerFd,&wfds);
if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1)
FD_SET(Srv->ServerFd,&rfds);
// Add the file
int FileFD = -1;
if (File != 0)
FileFD = File->Fd();
if (Srv->In.WriteSpace() == true && ToFile == true && FileFD != -1)
FD_SET(FileFD,&wfds);
// Add stdin
FD_SET(STDIN_FILENO,&rfds);
// Error Set
if (FileFD != -1)
FD_SET(FileFD,&efds);
if (Srv->ServerFd != -1)
FD_SET(Srv->ServerFd,&efds);
// Figure out the max fd
int MaxFd = FileFD;
if (MaxFd < Srv->ServerFd)
MaxFd = Srv->ServerFd;
// Select
struct timeval tv;
tv.tv_sec = 120;
tv.tv_usec = 0;
int Res = 0;
if ((Res = select(MaxFd+1,&rfds,&wfds,&efds,&tv)) < 0)
return _error->Errno("select","Select failed");
if (Res == 0)
{
_error->Error("Connection timed out");
return ServerDie(Srv);
}
// Some kind of exception (error) on the sockets, die
if ((FileFD != -1 && FD_ISSET(FileFD,&efds)) ||
(Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&efds)))
return _error->Error("Socket Exception");
// Handle server IO
if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&rfds))
{
errno = 0;
if (Srv->In.Read(Srv->ServerFd) == false)
return ServerDie(Srv);
}
if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&wfds))
{
errno = 0;
if (Srv->Out.Write(Srv->ServerFd) == false)
return ServerDie(Srv);
}
// Send data to the file
if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
{
if (Srv->In.Write(FileFD) == false)
return _error->Errno("write","Error writing to output file");
}
// Handle commands from APT
if (FD_ISSET(STDIN_FILENO,&rfds))
{
if (Run(true) != 0)
exit(100);
}
return true;
}
/*}}}*/
// HttpMethod::Flush - Dump the buffer into the file /*{{{*/
// ---------------------------------------------------------------------
/* This takes the current input buffer from the Server FD and writes it
into the file */
bool HttpMethod::Flush(ServerState *Srv)
{
if (File != 0)
{
SetNonBlock(File->Fd(),false);
if (Srv->In.WriteSpace() == false)
return true;
while (Srv->In.WriteSpace() == true)
{
if (Srv->In.Write(File->Fd()) == false)
return _error->Errno("write","Error writing to file");
}
if (Srv->In.IsLimit() == true || Srv->Encoding == ServerState::Closes)
return true;
}
return false;
}
/*}}}*/
// HttpMethod::ServerDie - The server has closed the connection. /*{{{*/
// ---------------------------------------------------------------------
/* */
bool HttpMethod::ServerDie(ServerState *Srv)
{
// Dump the buffer to the file
if (Srv->State == ServerState::Data)
{
SetNonBlock(File->Fd(),false);
while (Srv->In.WriteSpace() == true)
{
if (Srv->In.Write(File->Fd()) == false)
return _error->Errno("write","Error writing to the file");
}
}
// See if this is because the server finished the data stream
if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header &&
Srv->Encoding != ServerState::Closes)
{
if (errno == 0)
return _error->Error("Error reading from server Remote end closed connection");
return _error->Errno("read","Error reading from server");
}
else
{
Srv->In.Limit(-1);
// Nothing left in the buffer
if (Srv->In.WriteSpace() == false)
return false;
// We may have got multiple responses back in one packet..
Srv->Close();
return true;
}
return false;
}
/*}}}*/
// HttpMethod::DealWithHeaders - Handle the retrieved header data /*{{{*/
// ---------------------------------------------------------------------
/* We look at the header data we got back from the server and decide what
to do. Returns
0 - File is open,
1 - IMS hit
3 - Unrecoverable error */
int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
{
// Not Modified
if (Srv->Result == 304)
{
unlink(Queue->DestFile.c_str());
Res.IMSHit = true;
Res.LastModified = Queue->LastModified;
return 1;
}
/* We have a reply we dont handle. This should indicate a perm server
failure */
if (Srv->Result < 200 || Srv->Result >= 300)
{
_error->Error("%u %s",Srv->Result,Srv->Code);
return 3;
}
// This is some sort of 2xx 'data follows' reply
Res.LastModified = Srv->Date;
Res.Size = Srv->Size;
// Open the file
delete File;
File = new FileFd(Queue->DestFile,FileFd::WriteAny);
if (_error->PendingError() == true)
return 3;
// Set the expected size
if (Srv->StartPos >= 0)
{
Res.ResumePoint = Srv->StartPos;
ftruncate(File->Fd(),Srv->StartPos);
}
// Set the start point
lseek(File->Fd(),0,SEEK_END);
delete Srv->In.MD5;
Srv->In.MD5 = new MD5Summation;
// Fill the MD5 Hash if the file is non-empty (resume)
if (Srv->StartPos > 0)
{
lseek(File->Fd(),0,SEEK_SET);
if (Srv->In.MD5->AddFD(File->Fd(),Srv->StartPos) == false)
{
_error->Errno("read","Problem hashing file");
return 3;
}
lseek(File->Fd(),0,SEEK_END);
}
SetNonBlock(File->Fd(),true);
return 0;
}
/*}}}*/
// HttpMethod::Loop /*{{{*/
// ---------------------------------------------------------------------
/* */
int HttpMethod::Loop()
{
ServerState *Server = 0;
while (1)
{
// We have no commands, wait for some to arrive
if (Queue == 0)
{
if (WaitFd(STDIN_FILENO) == false)
return 0;
}
// Run messages
if (Run(true) != 0)
return 100;
if (Queue == 0)
continue;
// Connect to the server
if (Server == 0 || Server->Comp(Queue->Uri) == false)
{
delete Server;
Server = new ServerState(Queue->Uri,this);
}
// Connnect to the host
if (Server->Open() == false)
{
Fail();
continue;
}
// Queue the request
SendReq(Queue,Server->In);
// Handle the header data
if (Server->RunHeaders() == false)
{
Fail();
continue;
}
// Decide what to do.
FetchResult Res;
switch (DealWithHeaders(Res,Server))
{
// Ok, the file is Open
case 0:
{
URIStart(Res);
// Run the data
if (Server->RunData() == false)
Fail();
Res.MD5Sum = Srv->In.MD5->Result();
delete File;
File = 0;
break;
}
// IMS hit
case 1:
{
URIDone(Res);
break;
}
// Hard server error, not found or something
case 3:
{
Fail();
break;
}
default:
Fail("Internal error");
break;
}
}
return 0;
}
/*}}}*/
int main()
{
HttpMethod Mth;
return Mth.Loop();
}