Browse Source

Support large files in the complete toolset. Indexes of this

size are pretty unlikely for now, but we need it for deb
packages which could become bigger than 4GB now (LP: #815895)
tags/debian/0.9.0
David Kalnischkies 10 years ago
parent
commit
650faab016
52 changed files with 270 additions and 215 deletions
  1. +1
    -1
      apt-inst/contrib/arfile.h
  2. +4
    -4
      apt-pkg/acquire-worker.cc
  3. +1
    -1
      apt-pkg/contrib/crc-16.cc
  4. +1
    -1
      apt-pkg/contrib/crc-16.h
  5. +17
    -16
      apt-pkg/contrib/fileutl.cc
  6. +25
    -10
      apt-pkg/contrib/fileutl.h
  7. +5
    -5
      apt-pkg/contrib/hashes.cc
  8. +3
    -3
      apt-pkg/contrib/hashes.h
  9. +5
    -5
      apt-pkg/contrib/hashsum.cc
  10. +3
    -3
      apt-pkg/contrib/hashsum_template.h
  11. +1
    -1
      apt-pkg/contrib/md5.cc
  12. +1
    -1
      apt-pkg/contrib/md5.h
  13. +8
    -8
      apt-pkg/contrib/mmap.cc
  14. +4
    -4
      apt-pkg/contrib/mmap.h
  15. +4
    -4
      apt-pkg/contrib/progress.cc
  16. +8
    -8
      apt-pkg/contrib/progress.h
  17. +1
    -1
      apt-pkg/contrib/sha1.cc
  18. +1
    -1
      apt-pkg/contrib/sha1.h
  19. +3
    -3
      apt-pkg/contrib/sha2.h
  20. +28
    -0
      apt-pkg/contrib/strutl.cc
  21. +1
    -0
      apt-pkg/contrib/strutl.h
  22. +2
    -2
      apt-pkg/depcache.cc
  23. +12
    -11
      apt-pkg/indexcopy.cc
  24. +3
    -3
      apt-pkg/indexcopy.h
  25. +3
    -3
      apt-pkg/indexrecords.cc
  26. +2
    -2
      apt-pkg/indexrecords.h
  27. +0
    -1
      apt-pkg/pkgrecords.h
  28. +9
    -9
      apt-pkg/tagfile.cc
  29. +2
    -2
      apt-pkg/tagfile.h
  30. +3
    -0
      buildlib/config.h.in
  31. +6
    -6
      cmdline/acqprogress.cc
  32. +0
    -6
      cmdline/apt-get.cc
  33. +5
    -1
      configure.in
  34. +9
    -0
      debian/changelog
  35. +5
    -5
      ftparchive/apt-ftparchive.cc
  36. +3
    -3
      ftparchive/cachedb.h
  37. +1
    -1
      ftparchive/contents.cc
  38. +3
    -3
      ftparchive/contents.h
  39. +3
    -3
      ftparchive/multicompress.cc
  40. +1
    -1
      ftparchive/multicompress.h
  41. +8
    -8
      ftparchive/override.cc
  42. +9
    -9
      ftparchive/writer.cc
  43. +3
    -3
      ftparchive/writer.h
  44. +6
    -6
      methods/ftp.cc
  45. +2
    -2
      methods/ftp.h
  46. +1
    -1
      methods/gzip.cc
  47. +16
    -16
      methods/http.cc
  48. +15
    -15
      methods/http.h
  49. +1
    -1
      methods/https.cc
  50. +1
    -1
      methods/rred.cc
  51. +8
    -8
      methods/rsh.cc
  52. +3
    -3
      methods/rsh.h

+ 1
- 1
apt-inst/contrib/arfile.h View File

@@ -54,7 +54,7 @@ struct ARArchive::Member
unsigned long UID;
unsigned long GID;
unsigned long Mode;
unsigned long Size;
unsigned long long Size;
// Location of the data.
unsigned long Start;


+ 4
- 4
apt-pkg/acquire-worker.cc View File

@@ -258,9 +258,9 @@ bool pkgAcquire::Worker::RunMessages()
CurrentItem = Itm;
CurrentSize = 0;
TotalSize = atoi(LookupTag(Message,"Size","0").c_str());
ResumePoint = atoi(LookupTag(Message,"Resume-Point","0").c_str());
Itm->Owner->Start(Message,atoi(LookupTag(Message,"Size","0").c_str()));
TotalSize = strtoull(LookupTag(Message,"Size","0").c_str(), NULL, 10);
ResumePoint = strtoull(LookupTag(Message,"Resume-Point","0").c_str(), NULL, 10);
Itm->Owner->Start(Message,strtoull(LookupTag(Message,"Size","0").c_str(), NULL, 10));

// Display update before completion
if (Log != 0 && Log->MorePulses == true)
@@ -289,7 +289,7 @@ bool pkgAcquire::Worker::RunMessages()
Log->Pulse(Owner->GetOwner());
OwnerQ->ItemDone(Itm);
unsigned long long const ServerSize = atoll(LookupTag(Message,"Size","0").c_str());
unsigned long long const ServerSize = strtoull(LookupTag(Message,"Size","0").c_str(), NULL, 10);
if (TotalSize != 0 && ServerSize != TotalSize)
_error->Warning("Size of file %s is not what the server reported %s %llu",
Owner->DestFile.c_str(), LookupTag(Message,"Size","0").c_str(),TotalSize);


+ 1
- 1
apt-pkg/contrib/crc-16.cc View File

@@ -65,7 +65,7 @@ static unsigned short const crc16_table[256] =
/* Recompute the FCS with one more character appended. */
#define CalcFCS(fcs, c) (((fcs) >> 8) ^ crc16_table[((fcs) ^ (c)) & 0xff])
unsigned short AddCRC16(unsigned short fcs, void const *Buf,
unsigned long len)
unsigned long long len)
{
unsigned char const *buf = (unsigned char const *)Buf;
while (len--)


+ 1
- 1
apt-pkg/contrib/crc-16.h View File

@@ -12,6 +12,6 @@

#define INIT_FCS 0xffff
unsigned short AddCRC16(unsigned short fcs, void const *buf,
unsigned long len);
unsigned long long len);

#endif

+ 17
- 16
apt-pkg/contrib/fileutl.cc View File

@@ -133,10 +133,10 @@ bool CopyFile(FileFd &From,FileFd &To)
// Buffered copy between fds
SPtrArray<unsigned char> Buf = new unsigned char[64000];
unsigned long Size = From.Size();
unsigned long long Size = From.Size();
while (Size != 0)
{
unsigned long ToRead = Size;
unsigned long long ToRead = Size;
if (Size > 64000)
ToRead = 64000;
@@ -800,7 +800,7 @@ FileFd::~FileFd()
// ---------------------------------------------------------------------
/* We are carefull to handle interruption by a signal while reading
gracefully. */
bool FileFd::Read(void *To,unsigned long Size,unsigned long *Actual)
bool FileFd::Read(void *To,unsigned long long Size,unsigned long long *Actual)
{
int Res;
errno = 0;
@@ -839,13 +839,13 @@ bool FileFd::Read(void *To,unsigned long Size,unsigned long *Actual)
}
Flags |= Fail;
return _error->Error(_("read, still have %lu to read but none left"),Size);
return _error->Error(_("read, still have %llu to read but none left"), Size);
}
/*}}}*/
// FileFd::Write - Write to the file /*{{{*/
// ---------------------------------------------------------------------
/* */
bool FileFd::Write(const void *From,unsigned long Size)
bool FileFd::Write(const void *From,unsigned long long Size)
{
int Res;
errno = 0;
@@ -872,13 +872,13 @@ bool FileFd::Write(const void *From,unsigned long Size)
return true;
Flags |= Fail;
return _error->Error(_("write, still have %lu to write but couldn't"),Size);
return _error->Error(_("write, still have %llu to write but couldn't"), Size);
}
/*}}}*/
// FileFd::Seek - Seek in the file /*{{{*/
// ---------------------------------------------------------------------
/* */
bool FileFd::Seek(unsigned long To)
bool FileFd::Seek(unsigned long long To)
{
int res;
if (gz)
@@ -888,7 +888,7 @@ bool FileFd::Seek(unsigned long To)
if (res != (signed)To)
{
Flags |= Fail;
return _error->Error("Unable to seek to %lu",To);
return _error->Error("Unable to seek to %llu", To);
}
return true;
@@ -897,7 +897,7 @@ bool FileFd::Seek(unsigned long To)
// FileFd::Skip - Seek in the file /*{{{*/
// ---------------------------------------------------------------------
/* */
bool FileFd::Skip(unsigned long Over)
bool FileFd::Skip(unsigned long long Over)
{
int res;
if (gz)
@@ -907,7 +907,7 @@ bool FileFd::Skip(unsigned long Over)
if (res < 0)
{
Flags |= Fail;
return _error->Error("Unable to seek ahead %lu",Over);
return _error->Error("Unable to seek ahead %llu",Over);
}
return true;
@@ -916,7 +916,7 @@ bool FileFd::Skip(unsigned long Over)
// FileFd::Truncate - Truncate the file /*{{{*/
// ---------------------------------------------------------------------
/* */
bool FileFd::Truncate(unsigned long To)
bool FileFd::Truncate(unsigned long long To)
{
if (gz)
{
@@ -926,7 +926,7 @@ bool FileFd::Truncate(unsigned long To)
if (ftruncate(iFd,To) != 0)
{
Flags |= Fail;
return _error->Error("Unable to truncate to %lu",To);
return _error->Error("Unable to truncate to %llu",To);
}
return true;
@@ -935,7 +935,7 @@ bool FileFd::Truncate(unsigned long To)
// FileFd::Tell - Current seek position /*{{{*/
// ---------------------------------------------------------------------
/* */
unsigned long FileFd::Tell()
unsigned long long FileFd::Tell()
{
off_t Res;
if (gz)
@@ -950,7 +950,7 @@ unsigned long FileFd::Tell()
// FileFd::FileSize - Return the size of the file /*{{{*/
// ---------------------------------------------------------------------
/* */
unsigned long FileFd::FileSize()
unsigned long long FileFd::FileSize()
{
struct stat Buf;

@@ -962,9 +962,9 @@ unsigned long FileFd::FileSize()
// FileFd::Size - Return the size of the content in the file /*{{{*/
// ---------------------------------------------------------------------
/* */
unsigned long FileFd::Size()
unsigned long long FileFd::Size()
{
unsigned long size = FileSize();
unsigned long long size = FileSize();

// only check gzsize if we are actually a gzip file, just checking for
// "gz" is not sufficient as uncompressed files will be opened with
@@ -974,6 +974,7 @@ unsigned long FileFd::Size()
/* unfortunately zlib.h doesn't provide a gzsize(), so we have to do
* this ourselves; the original (uncompressed) file size is the last 32
* bits of the file */
// FIXME: Size for gz-files is limited by 32bit… no largefile support
off_t orig_pos = lseek(iFd, 0, SEEK_CUR);
if (lseek(iFd, -4, SEEK_END) < 0)
return _error->Errno("lseek","Unable to seek to end of gzipped file");


+ 25
- 10
apt-pkg/contrib/fileutl.h View File

@@ -49,21 +49,36 @@ class FileFd
enum OpenMode {ReadOnly,WriteEmpty,WriteExists,WriteAny,WriteTemp,ReadOnlyGzip,
WriteAtomic};
inline bool Read(void *To,unsigned long Size,bool AllowEof)
inline bool Read(void *To,unsigned long long Size,bool AllowEof)
{
unsigned long Jnk;
unsigned long long Jnk;
if (AllowEof)
return Read(To,Size,&Jnk);
return Read(To,Size);
}
bool Read(void *To,unsigned long Size,unsigned long *Actual = 0);
bool Write(const void *From,unsigned long Size);
bool Seek(unsigned long To);
bool Skip(unsigned long To);
bool Truncate(unsigned long To);
unsigned long Tell();
unsigned long Size();
unsigned long FileSize();
bool Read(void *To,unsigned long long Size,unsigned long long *Actual = 0);
bool Write(const void *From,unsigned long long Size);
bool Seek(unsigned long long To);
bool Skip(unsigned long long To);
bool Truncate(unsigned long long To);
unsigned long long Tell();
unsigned long long Size();
unsigned long long FileSize();

/* You want to use 'unsigned long long' if you are talking about a file
to be able to support large files (>2 or >4 GB) properly.
This shouldn't happen all to often for the indexes, but deb's might be…
And as the auto-conversation converts a 'unsigned long *' to a 'bool'
instead of 'unsigned long long *' we need to provide this explicitely -
otherwise applications magically start to fail… */
__deprecated bool Read(void *To,unsigned long long Size,unsigned long *Actual)
{
unsigned long long R;
bool const T = Read(To, Size, &R);
*Actual = R;
return T;
}

bool Open(string FileName,OpenMode Mode,unsigned long Perms = 0666);
bool OpenDescriptor(int Fd, OpenMode Mode, bool AutoClose=false);
bool Close();


+ 5
- 5
apt-pkg/contrib/hashes.cc View File

@@ -108,18 +108,18 @@ string HashString::toStr() const
// Hashes::AddFD - Add the contents of the FD /*{{{*/
// ---------------------------------------------------------------------
/* */
bool Hashes::AddFD(int const Fd,unsigned long Size, bool const addMD5,
bool Hashes::AddFD(int const Fd,unsigned long long Size, bool const addMD5,
bool const addSHA1, bool const addSHA256, bool const addSHA512)
{
unsigned char Buf[64*64];
int Res = 0;
ssize_t Res = 0;
int ToEOF = (Size == 0);
while (Size != 0 || ToEOF)
{
unsigned n = sizeof(Buf);
if (!ToEOF) n = min(Size,(unsigned long)n);
unsigned long long n = sizeof(Buf);
if (!ToEOF) n = min(Size, n);
Res = read(Fd,Buf,n);
if (Res < 0 || (!ToEOF && (unsigned) Res != n)) // error, or short read
if (Res < 0 || (!ToEOF && Res != (ssize_t) n)) // error, or short read
return false;
if (ToEOF && Res == 0) // EOF
break;


+ 3
- 3
apt-pkg/contrib/hashes.h View File

@@ -62,14 +62,14 @@ class Hashes
SHA256Summation SHA256;
SHA512Summation SHA512;
inline bool Add(const unsigned char *Data,unsigned long Size)
inline bool Add(const unsigned char *Data,unsigned long long Size)
{
return MD5.Add(Data,Size) && SHA1.Add(Data,Size) && SHA256.Add(Data,Size) && SHA512.Add(Data,Size);
};
inline bool Add(const char *Data) {return Add((unsigned char *)Data,strlen(Data));};
inline bool AddFD(int const Fd,unsigned long Size = 0)
inline bool AddFD(int const Fd,unsigned long long Size = 0)
{ return AddFD(Fd, Size, true, true, true, true); };
bool AddFD(int const Fd, unsigned long Size, bool const addMD5,
bool AddFD(int const Fd, unsigned long long Size, bool const addMD5,
bool const addSHA1, bool const addSHA256, bool const addSHA512);
inline bool Add(const unsigned char *Beg,const unsigned char *End)
{return Add(Beg,End-Beg);};


+ 5
- 5
apt-pkg/contrib/hashsum.cc View File

@@ -7,16 +7,16 @@
// Summation::AddFD - Add content of file into the checksum /*{{{*/
// ---------------------------------------------------------------------
/* */
bool SummationImplementation::AddFD(int const Fd, unsigned long Size) {
bool SummationImplementation::AddFD(int const Fd, unsigned long long Size) {
unsigned char Buf[64 * 64];
int Res = 0;
ssize_t Res = 0;
int ToEOF = (Size == 0);
while (Size != 0 || ToEOF)
{
unsigned n = sizeof(Buf);
if (!ToEOF) n = min(Size,(unsigned long)n);
unsigned long long n = sizeof(Buf);
if (!ToEOF) n = min(Size, n);
Res = read(Fd, Buf, n);
if (Res < 0 || (!ToEOF && (unsigned) Res != n)) // error, or short read
if (Res < 0 || (!ToEOF && Res != (ssize_t) n)) // error, or short read
return false;
if (ToEOF && Res == 0) // EOF
break;


+ 3
- 3
apt-pkg/contrib/hashsum_template.h View File

@@ -87,8 +87,8 @@ class HashSumValue
class SummationImplementation
{
public:
virtual bool Add(const unsigned char *inbuf, unsigned long inlen) = 0;
inline bool Add(const char *inbuf, unsigned long const inlen)
virtual bool Add(const unsigned char *inbuf, unsigned long long inlen) = 0;
inline bool Add(const char *inbuf, unsigned long long const inlen)
{ return Add((unsigned char *)inbuf, inlen); };

inline bool Add(const unsigned char *Data)
@@ -101,7 +101,7 @@ class SummationImplementation
inline bool Add(const char *Beg, const char *End)
{ return Add((const unsigned char *)Beg, End - Beg); };

bool AddFD(int Fd, unsigned long Size = 0);
bool AddFD(int Fd, unsigned long long Size = 0);
};

#endif

+ 1
- 1
apt-pkg/contrib/md5.cc View File

@@ -187,7 +187,7 @@ MD5Summation::MD5Summation()
// MD5Summation::Add - 'Add' a data set to the hash /*{{{*/
// ---------------------------------------------------------------------
/* */
bool MD5Summation::Add(const unsigned char *data,unsigned long len)
bool MD5Summation::Add(const unsigned char *data,unsigned long long len)
{
if (Done == true)
return false;


+ 1
- 1
apt-pkg/contrib/md5.h View File

@@ -45,7 +45,7 @@ class MD5Summation : public SummationImplementation

public:

bool Add(const unsigned char *inbuf, unsigned long inlen);
bool Add(const unsigned char *inbuf, unsigned long long inlen);
using SummationImplementation::Add;

MD5SumValue Result();


+ 8
- 8
apt-pkg/contrib/mmap.cc View File

@@ -95,7 +95,7 @@ bool MMap::Map(FileFd &Fd)
return false;
}
else
return _error->Errno("mmap",_("Couldn't make mmap of %lu bytes"),
return _error->Errno("mmap",_("Couldn't make mmap of %llu bytes"),
iSize);
}

@@ -166,7 +166,7 @@ bool MMap::Sync(unsigned long Start,unsigned long Stop)
return true;
#ifdef _POSIX_SYNCHRONIZED_IO
unsigned long PSize = sysconf(_SC_PAGESIZE);
unsigned long long PSize = sysconf(_SC_PAGESIZE);
if ((Flags & ReadOnly) != ReadOnly)
{
if (SyncToFd != 0)
@@ -177,7 +177,7 @@ bool MMap::Sync(unsigned long Start,unsigned long Stop)
}
else
{
if (msync((char *)Base+(int)(Start/PSize)*PSize,Stop - Start,MS_SYNC) < 0)
if (msync((char *)Base+(unsigned long long)(Start/PSize)*PSize,Stop - Start,MS_SYNC) < 0)
return _error->Errno("msync", _("Unable to synchronize mmap"));
}
}
@@ -197,7 +197,7 @@ DynamicMMap::DynamicMMap(FileFd &F,unsigned long Flags,unsigned long const &Work
if (_error->PendingError() == true)
return;
unsigned long EndOfFile = Fd->Size();
unsigned long long EndOfFile = Fd->Size();
if (EndOfFile > WorkSpace)
WorkSpace = EndOfFile;
else if(WorkSpace > 0)
@@ -285,7 +285,7 @@ DynamicMMap::~DynamicMMap()
return;
}
unsigned long EndOfFile = iSize;
unsigned long long EndOfFile = iSize;
iSize = WorkSpace;
Close(false);
if(ftruncate(Fd->Fd(),EndOfFile) < 0)
@@ -295,9 +295,9 @@ DynamicMMap::~DynamicMMap()
// DynamicMMap::RawAllocate - Allocate a raw chunk of unaligned space /*{{{*/
// ---------------------------------------------------------------------
/* This allocates a block of memory aligned to the given size */
unsigned long DynamicMMap::RawAllocate(unsigned long Size,unsigned long Aln)
unsigned long DynamicMMap::RawAllocate(unsigned long long Size,unsigned long Aln)
{
unsigned long Result = iSize;
unsigned long long Result = iSize;
if (Aln != 0)
Result += Aln - (iSize%Aln);

@@ -412,7 +412,7 @@ bool DynamicMMap::Grow() {
if (GrowFactor <= 0)
return _error->Error(_("Unable to increase size of the MMap as automatic growing is disabled by user."));

unsigned long const newSize = WorkSpace + GrowFactor;
unsigned long long const newSize = WorkSpace + GrowFactor;

if(Fd != 0) {
Fd->Seek(newSize - 1);


+ 4
- 4
apt-pkg/contrib/mmap.h View File

@@ -41,7 +41,7 @@ class MMap
protected:
unsigned long Flags;
unsigned long iSize;
unsigned long long iSize;
void *Base;

// In case mmap can not be used, we keep a dup of the file
@@ -60,8 +60,8 @@ class MMap
// Simple accessors
inline operator void *() {return Base;};
inline void *Data() {return Base;};
inline unsigned long Size() {return iSize;};
inline void AddSize(unsigned long const size) {iSize += size;};
inline unsigned long long Size() {return iSize;};
inline void AddSize(unsigned long long const size) {iSize += size;};
inline bool validData() const { return Base != (void *)-1 && Base != 0; };
// File manipulators
@@ -99,7 +99,7 @@ class DynamicMMap : public MMap
public:

// Allocation
unsigned long RawAllocate(unsigned long Size,unsigned long Aln = 0);
unsigned long RawAllocate(unsigned long long Size,unsigned long Aln = 0);
unsigned long Allocate(unsigned long ItemSize);
unsigned long WriteString(const char *String,unsigned long Len = (unsigned long)-1);
inline unsigned long WriteString(const string &S) {return WriteString(S.c_str(),S.length());};


+ 4
- 4
apt-pkg/contrib/progress.cc View File

@@ -37,7 +37,7 @@ OpProgress::OpProgress() : Current(0), Total(0), Size(0), SubTotal(1),
/* Current is the Base Overall progress in units of Total. Cur is the sub
progress in units of SubTotal. Size is a scaling factor that says what
percent of Total SubTotal is. */
void OpProgress::Progress(unsigned long Cur)
void OpProgress::Progress(unsigned long long Cur)
{
if (Total == 0 || Size == 0 || SubTotal == 0)
Percent = 0;
@@ -49,8 +49,8 @@ void OpProgress::Progress(unsigned long Cur)
// OpProgress::OverallProgress - Set the overall progress /*{{{*/
// ---------------------------------------------------------------------
/* */
void OpProgress::OverallProgress(unsigned long Current, unsigned long Total,
unsigned long Size,const string &Op)
void OpProgress::OverallProgress(unsigned long long Current, unsigned long long Total,
unsigned long long Size,const string &Op)
{
this->Current = Current;
this->Total = Total;
@@ -67,7 +67,7 @@ void OpProgress::OverallProgress(unsigned long Current, unsigned long Total,
// OpProgress::SubProgress - Set the sub progress state /*{{{*/
// ---------------------------------------------------------------------
/* */
void OpProgress::SubProgress(unsigned long SubTotal,const string &Op,
void OpProgress::SubProgress(unsigned long long SubTotal,const string &Op,
float const Percent)
{
this->SubTotal = SubTotal;


+ 8
- 8
apt-pkg/contrib/progress.h View File

@@ -30,10 +30,10 @@ using std::string;
class Configuration;
class OpProgress
{
unsigned long Current;
unsigned long Total;
unsigned long Size;
unsigned long SubTotal;
unsigned long long Current;
unsigned long long Total;
unsigned long long Size;
unsigned long long SubTotal;
float LastPercent;
// Change reduction code
@@ -54,10 +54,10 @@ class OpProgress
public:
void Progress(unsigned long Current);
void SubProgress(unsigned long SubTotal, const string &Op = "", float const Percent = -1);
void OverallProgress(unsigned long Current,unsigned long Total,
unsigned long Size,const string &Op);
void Progress(unsigned long long Current);
void SubProgress(unsigned long long SubTotal, const string &Op = "", float const Percent = -1);
void OverallProgress(unsigned long long Current,unsigned long long Total,
unsigned long long Size,const string &Op);
virtual void Done() {};
OpProgress();


+ 1
- 1
apt-pkg/contrib/sha1.cc View File

@@ -243,7 +243,7 @@ SHA1SumValue SHA1Summation::Result()
// SHA1Summation::Add - Adds content of buffer into the checksum /*{{{*/
// ---------------------------------------------------------------------
/* May not be called after Result() is called */
bool SHA1Summation::Add(const unsigned char *data,unsigned long len)
bool SHA1Summation::Add(const unsigned char *data,unsigned long long len)
{
if (Done)
return false;


+ 1
- 1
apt-pkg/contrib/sha1.h View File

@@ -34,7 +34,7 @@ class SHA1Summation : public SummationImplementation
bool Done;
public:
bool Add(const unsigned char *inbuf, unsigned long inlen);
bool Add(const unsigned char *inbuf, unsigned long long inlen);
using SummationImplementation::Add;

SHA1SumValue Result();


+ 3
- 3
apt-pkg/contrib/sha2.h View File

@@ -30,7 +30,7 @@ class SHA2SummationBase : public SummationImplementation
protected:
bool Done;
public:
bool Add(const unsigned char *inbuf, unsigned long len) = 0;
bool Add(const unsigned char *inbuf, unsigned long long len) = 0;

void Result();
};
@@ -41,7 +41,7 @@ class SHA256Summation : public SHA2SummationBase
unsigned char Sum[32];

public:
bool Add(const unsigned char *inbuf, unsigned long len)
bool Add(const unsigned char *inbuf, unsigned long long len)
{
if (Done)
return false;
@@ -73,7 +73,7 @@ class SHA512Summation : public SHA2SummationBase
unsigned char Sum[64];

public:
bool Add(const unsigned char *inbuf, unsigned long len)
bool Add(const unsigned char *inbuf, unsigned long long len)
{
if (Done)
return false;


+ 28
- 0
apt-pkg/contrib/strutl.cc View File

@@ -970,6 +970,34 @@ bool StrToNum(const char *Str,unsigned long &Res,unsigned Len,unsigned Base)
return true;
}
/*}}}*/
// StrToNum - Convert a fixed length string to a number /*{{{*/
// ---------------------------------------------------------------------
/* This is used in decoding the crazy fixed length string headers in
tar and ar files. */
bool StrToNum(const char *Str,unsigned long long &Res,unsigned Len,unsigned Base)
{
char S[30];
if (Len >= sizeof(S))
return false;
memcpy(S,Str,Len);
S[Len] = 0;
// All spaces is a zero
Res = 0;
unsigned I;
for (I = 0; S[I] == ' '; I++);
if (S[I] == 0)
return true;
char *End;
Res = strtoull(S,&End,Base);
if (End == S)
return false;
return true;
}
/*}}}*/

// Base256ToNum - Convert a fixed length binary to a number /*{{{*/
// ---------------------------------------------------------------------
/* This is used in decoding the 256bit encoded fixed length fields in


+ 1
- 0
apt-pkg/contrib/strutl.h View File

@@ -52,6 +52,7 @@ string LookupTag(const string &Message,const char *Tag,const char *Default = 0);
int StringToBool(const string &Text,int Default = -1);
bool ReadMessages(int Fd, vector<string> &List);
bool StrToNum(const char *Str,unsigned long &Res,unsigned Len,unsigned Base = 0);
bool StrToNum(const char *Str,unsigned long long &Res,unsigned Len,unsigned Base = 0);
bool Base256ToNum(const char *Str,unsigned long &Res,unsigned int Len);
bool Hex2Num(const string &Str,unsigned char *Num,unsigned int Length);
bool TokSplitString(char Tok,char *Input,char **List,


+ 2
- 2
apt-pkg/depcache.cc View File

@@ -171,14 +171,14 @@ bool pkgDepCache::readStateFile(OpProgress *Prog) /*{{{*/
string const state = _config->FindFile("Dir::State::extended_states");
if(RealFileExists(state)) {
state_file.Open(state, FileFd::ReadOnly);
int const file_size = state_file.Size();
off_t const file_size = state_file.Size();
if(Prog != NULL)
Prog->OverallProgress(0, file_size, 1,
_("Reading state information"));

pkgTagFile tagfile(&state_file);
pkgTagSection section;
int amt = 0;
off_t amt = 0;
bool const debug_autoremove = _config->FindB("Debug::pkgAutoRemove",false);
while(tagfile.Step(section)) {
string const pkgname = section.FindS("Package");


+ 12
- 11
apt-pkg/indexcopy.cc View File

@@ -29,6 +29,7 @@
#include <sys/types.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>

#include "indexcopy.h"
#include <apti18n.h>
@@ -55,7 +56,7 @@ bool IndexCopy::CopyPackages(string CDROM,string Name,vector<string> &List,
bool Debug = _config->FindB("Debug::aptcdrom",false);
// Prepare the progress indicator
unsigned long TotalSize = 0;
off_t TotalSize = 0;
for (vector<string>::iterator I = List.begin(); I != List.end(); I++)
{
struct stat Buf;
@@ -66,14 +67,14 @@ bool IndexCopy::CopyPackages(string CDROM,string Name,vector<string> &List,
TotalSize += Buf.st_size;
}

unsigned long CurrentSize = 0;
off_t CurrentSize = 0;
unsigned int NotFound = 0;
unsigned int WrongSize = 0;
unsigned int Packages = 0;
for (vector<string>::iterator I = List.begin(); I != List.end(); I++)
{
string OrigPath = string(*I,CDROM.length());
unsigned long FileSize = 0;
off_t FileSize = 0;
// Open the package file
FileFd Pkg;
@@ -166,7 +167,7 @@ bool IndexCopy::CopyPackages(string CDROM,string Name,vector<string> &List,
if(Progress)
Progress->Progress(Parser.Offset());
string File;
unsigned long Size;
unsigned long long Size;
if (GetFile(File,Size) == false)
{
fclose(TargetFl);
@@ -221,7 +222,7 @@ bool IndexCopy::CopyPackages(string CDROM,string Name,vector<string> &List,
}
// Size match
if ((unsigned)Buf.st_size != Size)
if ((unsigned long long)Buf.st_size != Size)
{
if (Debug == true)
clog << "Wrong Size: " << File << endl;
@@ -455,7 +456,7 @@ bool IndexCopy::GrabFirst(string Path,string &To,unsigned int Depth)
// PackageCopy::GetFile - Get the file information from the section /*{{{*/
// ---------------------------------------------------------------------
/* */
bool PackageCopy::GetFile(string &File,unsigned long &Size)
bool PackageCopy::GetFile(string &File,unsigned long long &Size)
{
File = Section->FindS("Filename");
Size = Section->FindI("Size");
@@ -481,7 +482,7 @@ bool PackageCopy::RewriteEntry(FILE *Target,string File)
// SourceCopy::GetFile - Get the file information from the section /*{{{*/
// ---------------------------------------------------------------------
/* */
bool SourceCopy::GetFile(string &File,unsigned long &Size)
bool SourceCopy::GetFile(string &File,unsigned long long &Size)
{
string Files = Section->FindS("Files");
if (Files.empty() == true)
@@ -504,7 +505,7 @@ bool SourceCopy::GetFile(string &File,unsigned long &Size)
return _error->Error("Error parsing file record");
// Parse the size and append the directory
Size = atoi(sSize.c_str());
Size = strtoull(sSize.c_str(), NULL, 10);
File = Base + File;
return true;
}
@@ -787,7 +788,7 @@ bool TranslationsCopy::CopyTranslations(string CDROM,string Name, /*{{{*/
bool Debug = _config->FindB("Debug::aptcdrom",false);
// Prepare the progress indicator
unsigned long TotalSize = 0;
off_t TotalSize = 0;
for (vector<string>::iterator I = List.begin(); I != List.end(); I++)
{
struct stat Buf;
@@ -798,14 +799,14 @@ bool TranslationsCopy::CopyTranslations(string CDROM,string Name, /*{{{*/
TotalSize += Buf.st_size;
}

unsigned long CurrentSize = 0;
off_t CurrentSize = 0;
unsigned int NotFound = 0;
unsigned int WrongSize = 0;
unsigned int Packages = 0;
for (vector<string>::iterator I = List.begin(); I != List.end(); I++)
{
string OrigPath = string(*I,CDROM.length());
unsigned long FileSize = 0;
off_t FileSize = 0;
// Open the package file
FileFd Pkg;


+ 3
- 3
apt-pkg/indexcopy.h View File

@@ -37,7 +37,7 @@ class IndexCopy /*{{{*/
bool ReconstructChop(unsigned long &Chop,string Dir,string File);
void ConvertToSourceList(string CD,string &Path);
bool GrabFirst(string Path,string &To,unsigned int Depth);
virtual bool GetFile(string &Filename,unsigned long &Size) = 0;
virtual bool GetFile(string &Filename,unsigned long long &Size) = 0;
virtual bool RewriteEntry(FILE *Target,string File) = 0;
virtual const char *GetFileName() = 0;
virtual const char *Type() = 0;
@@ -53,7 +53,7 @@ class PackageCopy : public IndexCopy /*{{{*/
{
protected:
virtual bool GetFile(string &Filename,unsigned long &Size);
virtual bool GetFile(string &Filename,unsigned long long &Size);
virtual bool RewriteEntry(FILE *Target,string File);
virtual const char *GetFileName() {return "Packages";};
virtual const char *Type() {return "Package";};
@@ -64,7 +64,7 @@ class SourceCopy : public IndexCopy /*{{{*/
{
protected:
virtual bool GetFile(string &Filename,unsigned long &Size);
virtual bool GetFile(string &Filename,unsigned long long &Size);
virtual bool RewriteEntry(FILE *Target,string File);
virtual const char *GetFileName() {return "Sources";};
virtual const char *Type() {return "Source";};


+ 3
- 3
apt-pkg/indexrecords.cc View File

@@ -80,7 +80,7 @@ bool indexRecords::Load(const string Filename) /*{{{*/

string Name;
string Hash;
size_t Size;
unsigned long long Size;
while (Start < End)
{
if (!parseSumData(Start, End, Name, Hash, Size))
@@ -147,7 +147,7 @@ vector<string> indexRecords::MetaKeys() /*{{{*/
}
/*}}}*/
bool indexRecords::parseSumData(const char *&Start, const char *End, /*{{{*/
string &Name, string &Hash, size_t &Size)
string &Name, string &Hash, unsigned long long &Size)
{
Name = "";
Hash = "";
@@ -184,7 +184,7 @@ bool indexRecords::parseSumData(const char *&Start, const char *End, /*{{{*/
if (EntryEnd == End)
return false;
Size = strtol (Start, NULL, 10);
Size = strtoull (Start, NULL, 10);
/* Skip over intermediate blanks */
Start = EntryEnd;


+ 2
- 2
apt-pkg/indexrecords.h View File

@@ -17,7 +17,7 @@
class indexRecords
{
bool parseSumData(const char *&Start, const char *End, string &Name,
string &Hash, size_t &Size);
string &Hash, unsigned long long &Size);
public:
struct checkSum;
string ErrorText;
@@ -53,7 +53,7 @@ struct indexRecords::checkSum
{
string MetaKeyFilename;
HashString Hash;
size_t Size;
unsigned long long Size;
};

#endif

+ 0
- 1
apt-pkg/pkgrecords.h View File

@@ -19,7 +19,6 @@


#include <apt-pkg/pkgcache.h>
#include <apt-pkg/fileutl.h>
#include <vector>

class pkgRecords /*{{{*/


+ 9
- 9
apt-pkg/tagfile.cc View File

@@ -29,7 +29,7 @@ using std::string;
class pkgTagFilePrivate
{
public:
pkgTagFilePrivate(FileFd *pFd, unsigned long Size) : Fd(*pFd), Size(Size)
pkgTagFilePrivate(FileFd *pFd, unsigned long long Size) : Fd(*pFd), Size(Size)
{
}
FileFd &Fd;
@@ -37,14 +37,14 @@ public:
char *Start;
char *End;
bool Done;
unsigned long iOffset;
unsigned long Size;
unsigned long long iOffset;
unsigned long long Size;
};

// TagFile::pkgTagFile - Constructor /*{{{*/
// ---------------------------------------------------------------------
/* */
pkgTagFile::pkgTagFile(FileFd *pFd,unsigned long Size)
pkgTagFile::pkgTagFile(FileFd *pFd,unsigned long long Size)
{
d = new pkgTagFilePrivate(pFd, Size);

@@ -86,7 +86,7 @@ unsigned long pkgTagFile::Offset()
bool pkgTagFile::Resize()
{
char *tmp;
unsigned long EndSize = d->End - d->Start;
unsigned long long EndSize = d->End - d->Start;

// fail is the buffer grows too big
if(d->Size > 1024*1024+1)
@@ -138,8 +138,8 @@ bool pkgTagFile::Step(pkgTagSection &Tag)
then fills the rest from the file */
bool pkgTagFile::Fill()
{
unsigned long EndSize = d->End - d->Start;
unsigned long Actual = 0;
unsigned long long EndSize = d->End - d->Start;
unsigned long long Actual = 0;
memmove(d->Buffer,d->Start,EndSize);
d->Start = d->Buffer;
@@ -180,12 +180,12 @@ bool pkgTagFile::Fill()
// ---------------------------------------------------------------------
/* This jumps to a pre-recorded file location and reads the record
that is there */
bool pkgTagFile::Jump(pkgTagSection &Tag,unsigned long Offset)
bool pkgTagFile::Jump(pkgTagSection &Tag,unsigned long long Offset)
{
// We are within a buffer space of the next hit..
if (Offset >= d->iOffset && d->iOffset + (d->End - d->Start) > Offset)
{
unsigned long Dist = Offset - d->iOffset;
unsigned long long Dist = Offset - d->iOffset;
d->Start += Dist;
d->iOffset += Dist;
return Step(Tag);


+ 2
- 2
apt-pkg/tagfile.h View File

@@ -94,9 +94,9 @@ class pkgTagFile

bool Step(pkgTagSection &Section);
unsigned long Offset();
bool Jump(pkgTagSection &Tag,unsigned long Offset);
bool Jump(pkgTagSection &Tag,unsigned long long Offset);

pkgTagFile(FileFd *F,unsigned long Size = 32*1024);
pkgTagFile(FileFd *F,unsigned long long Size = 32*1024);
virtual ~pkgTagFile();
};



+ 3
- 0
buildlib/config.h.in View File

@@ -33,6 +33,9 @@
/* If there is no socklen_t, define this for the netdb shim */
#undef NEED_SOCKLEN_T_DEFINE

/* Define to the size of the filesize containing structures */
#undef _FILE_OFFSET_BITS

/* Define the arch name string */
#undef COMMON_ARCH



+ 6
- 6
cmdline/acqprogress.cc View File

@@ -163,7 +163,7 @@ bool AcqTextStatus::Pulse(pkgAcquire *Owner)
ScreenWidth = sizeof(Buffer)-1;

// Put in the percent done
sprintf(S,"%ld%%",long(double((CurrentBytes + CurrentItems)*100.0)/double(TotalBytes+TotalItems)));
sprintf(S,"%.0f%%",((CurrentBytes + CurrentItems)*100.0)/(TotalBytes+TotalItems));

bool Shown = false;
for (pkgAcquire::Worker *I = Owner->WorkersBegin(); I != 0;
@@ -214,11 +214,11 @@ bool AcqTextStatus::Pulse(pkgAcquire *Owner)
if (I->TotalSize > 0 && I->CurrentItem->Owner->Complete == false)
{
if (Mode == Short)
snprintf(S,End-S," %lu%%",
long(double(I->CurrentSize*100.0)/double(I->TotalSize)));
snprintf(S,End-S," %.0f%%",
(I->CurrentSize*100.0)/I->TotalSize);
else
snprintf(S,End-S,"/%sB %lu%%",SizeToStr(I->TotalSize).c_str(),
long(double(I->CurrentSize*100.0)/double(I->TotalSize)));
snprintf(S,End-S,"/%sB %.0f%%",SizeToStr(I->TotalSize).c_str(),
(I->CurrentSize*100.0)/I->TotalSize);
}
S += strlen(S);
snprintf(S,End-S,"]");
@@ -238,7 +238,7 @@ bool AcqTextStatus::Pulse(pkgAcquire *Owner)
if (CurrentCPS != 0)
{
char Tmp[300];
unsigned long ETA = (unsigned long)((TotalBytes - CurrentBytes)/CurrentCPS);
unsigned long long ETA = (TotalBytes - CurrentBytes)/CurrentCPS;
sprintf(Tmp," %sB/s %s",SizeToStr(CurrentCPS).c_str(),TimeToStr(ETA).c_str());
unsigned int Len = strlen(Buffer);
unsigned int LenT = strlen(Tmp);


+ 0
- 6
cmdline/apt-get.cc View File

@@ -27,9 +27,6 @@
// Include Files /*{{{*/
#include <config.h>

#define _LARGEFILE_SOURCE
#define _LARGEFILE64_SOURCE

#include <apt-pkg/aptconfiguration.h>
#include <apt-pkg/error.h>
#include <apt-pkg/cmndline.h>
@@ -68,9 +65,6 @@
#include <sstream>

#include <apti18n.h>

#define statfs statfs64
#define statvfs statvfs64
/*}}}*/

#define RAMFS_MAGIC 0x858458f6


+ 5
- 1
configure.in View File

@@ -31,6 +31,10 @@ AC_CHECK_TOOL_PREFIX
AC_PROG_CC
AC_ISC_POSIX

dnl check for large file support and enable it if possible
dnl do this early as other stuff might depend on it
AC_SYS_LARGEFILE

dnl Check for other programs
AC_PROG_CXX
AC_PROG_CPP
@@ -151,7 +155,7 @@ if test "$cross_compiling" = "yes" -a "x$archline" = "x"; then
AC_MSG_ERROR(When cross compiling, architecture must be present in sizetable)
fi
AC_C_BIGENDIAN
dnl We do not need this if we have inttypes!
HAVE_C9X=yes
if test x"$apt_cv_c9x_ints" = x"no"; then


+ 9
- 0
debian/changelog View File

@@ -1,3 +1,12 @@
apt (0.8.16~exp3+nmu1) experimental; urgency=low

[ David Kalnischkies ]
* Support large files in the complete toolset. Indexes of this
size are pretty unlikely for now, but we need it for deb
packages which could become bigger than 4GB now (LP: #815895)

-- David Kalnischkies <kalnischkies@gmail.com> Tue, 13 Sep 2011 17:44:00 +0200

apt (0.8.16~exp3) experimental; urgency=low

[ David Kalnischkies ]


+ 5
- 5
ftparchive/apt-ftparchive.cc View File

@@ -206,7 +206,7 @@ bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
Packages.Output = 0; // Just in case
// Finish compressing
unsigned long Size;
unsigned long long Size;
if (Comp.Finalize(Size) == false)
{
c0out << endl;
@@ -292,7 +292,7 @@ bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
Sources.Output = 0; // Just in case
// Finish compressing
unsigned long Size;
unsigned long long Size;
if (Comp.Finalize(Size) == false)
{
c0out << endl;
@@ -363,11 +363,11 @@ bool PackageMap::GenContents(Configuration &Setup,
if (_error->PendingError() == true)
return false;
unsigned long Size = Head.Size();
unsigned long long Size = Head.Size();
unsigned char Buf[4096];
while (Size != 0)
{
unsigned long ToRead = Size;
unsigned long long ToRead = Size;
if (Size > sizeof(Buf))
ToRead = sizeof(Buf);
@@ -401,7 +401,7 @@ bool PackageMap::GenContents(Configuration &Setup,
Contents.Finish();
// Finish compressing
unsigned long Size;
unsigned long long Size;
if (Comp.Finalize(Size) == false || _error->PendingError() == true)
{
c0out << endl;


+ 3
- 3
ftparchive/cachedb.h View File

@@ -81,7 +81,7 @@ class CacheDB
{
uint32_t Flags;
uint32_t mtime;
uint32_t FileSize;
uint64_t FileSize;
uint8_t MD5[16];
uint8_t SHA1[20];
uint8_t SHA256[32];
@@ -114,7 +114,7 @@ class CacheDB
double SHA512Bytes;
unsigned long Packages;
unsigned long Misses;
unsigned long DeLinkBytes;
unsigned long long DeLinkBytes;
inline void Add(const Stats &S) {
Bytes += S.Bytes;
@@ -133,7 +133,7 @@ class CacheDB
inline bool DBFailed() {return Dbp != 0 && DBLoaded == false;};
inline bool Loaded() {return DBLoaded == true;};
inline off_t GetFileSize(void) {return CurStat.FileSize;}
inline unsigned long long GetFileSize(void) {return CurStat.FileSize;}
bool SetFile(string const &FileName,struct stat St,FileFd *Fd);
bool GetFileInfo(string const &FileName, bool const &DoControl, bool const &DoContents, bool const &GenContentsOnly,


+ 1
- 1
ftparchive/contents.cc View File

@@ -349,7 +349,7 @@ bool ContentsExtract::DoItem(Item &Itm,int &Fd)
// ContentsExtract::TakeContents - Load the contents data /*{{{*/
// ---------------------------------------------------------------------
/* */
bool ContentsExtract::TakeContents(const void *NewData,unsigned long Length)
bool ContentsExtract::TakeContents(const void *NewData,unsigned long long Length)
{
if (Length == 0)
{


+ 3
- 3
ftparchive/contents.h View File

@@ -71,15 +71,15 @@ class ContentsExtract : public pkgDirStream

// The Data Block
char *Data;
unsigned long MaxSize;
unsigned long CurSize;
unsigned long long MaxSize;
unsigned long long CurSize;
void AddData(const char *Text);
bool Read(debDebFile &Deb);
virtual bool DoItem(Item &Itm,int &Fd);
void Reset() {CurSize = 0;};
bool TakeContents(const void *Data,unsigned long Length);
bool TakeContents(const void *Data,unsigned long long Length);
void Add(GenContents &Contents,string const &Package);
ContentsExtract() : Data(0), MaxSize(0), CurSize(0) {};


+ 3
- 3
ftparchive/multicompress.cc View File

@@ -215,7 +215,7 @@ bool MultiCompress::Die()
// MultiCompress::Finalize - Finish up writing /*{{{*/
// ---------------------------------------------------------------------
/* This is only necessary for statistics reporting. */
bool MultiCompress::Finalize(unsigned long &OutSize)
bool MultiCompress::Finalize(unsigned long long &OutSize)
{
OutSize = 0;
if (Input == 0 || Die() == false)
@@ -383,7 +383,7 @@ bool MultiCompress::Child(int const &FD)
stash a hash of the data to use later. */
SetNonBlock(FD,false);
unsigned char Buffer[32*1024];
unsigned long FileSize = 0;
unsigned long long FileSize = 0;
MD5Summation MD5;
while (1)
{
@@ -445,7 +445,7 @@ bool MultiCompress::Child(int const &FD)
// Compute the hash
MD5Summation OldMD5;
unsigned long NewFileSize = 0;
unsigned long long NewFileSize = 0;
while (1)
{
int Res = read(CompFd,Buffer,sizeof(Buffer));


+ 1
- 1
ftparchive/multicompress.h View File

@@ -54,7 +54,7 @@ class MultiCompress
FILE *Input;
unsigned long UpdateMTime;
bool Finalize(unsigned long &OutSize);
bool Finalize(unsigned long long &OutSize);
bool OpenOld(int &Fd,pid_t &Proc);
bool CloseOld(int Fd,pid_t Proc);
static bool GetStat(string const &Output,string const &Compress,struct stat &St);


+ 8
- 8
ftparchive/override.cc View File

@@ -34,7 +34,7 @@ bool Override::ReadOverride(string const &File,bool const &Source)
return _error->Errno("fopen",_("Unable to open %s"),File.c_str());
char Line[500];
unsigned long Counter = 0;
unsigned long long Counter = 0;
while (fgets(Line,sizeof(Line),F) != 0)
{
Counter++;
@@ -57,7 +57,7 @@ bool Override::ReadOverride(string const &File,bool const &Source)
for (; isspace(*End) == 0 && *End != 0; End++);
if (*End == 0)
{
_error->Warning(_("Malformed override %s line %lu #1"),File.c_str(),
_error->Warning(_("Malformed override %s line %llu #1"),File.c_str(),
Counter);
continue;
}
@@ -71,7 +71,7 @@ bool Override::ReadOverride(string const &File,bool const &Source)
for (; isspace(*End) == 0 && *End != 0; End++);
if (*End == 0)
{
_error->Warning(_("Malformed override %s line %lu #2"),File.c_str(),
_error->Warning(_("Malformed override %s line %llu #2"),File.c_str(),
Counter);
continue;
}
@@ -85,7 +85,7 @@ bool Override::ReadOverride(string const &File,bool const &Source)
for (; isspace(*End) == 0 && *End != 0; End++);
if (*End == 0)
{
_error->Warning(_("Malformed override %s line %lu #3"),File.c_str(),
_error->Warning(_("Malformed override %s line %llu #3"),File.c_str(),
Counter);
continue;
}
@@ -142,7 +142,7 @@ bool Override::ReadExtraOverride(string const &File,bool const &Source)
return _error->Errno("fopen",_("Unable to open %s"),File.c_str());
char Line[500];
unsigned long Counter = 0;
unsigned long long Counter = 0;
while (fgets(Line,sizeof(Line),F) != 0)
{
Counter++;
@@ -163,7 +163,7 @@ bool Override::ReadExtraOverride(string const &File,bool const &Source)
for (; isspace(*End) == 0 && *End != 0; End++);
if (*End == 0)
{
_error->Warning(_("Malformed override %s line %lu #1"),File.c_str(),
_error->Warning(_("Malformed override %s line %llu #1"),File.c_str(),
Counter);
continue;
}
@@ -175,7 +175,7 @@ bool Override::ReadExtraOverride(string const &File,bool const &Source)
for (; isspace(*End) == 0 && *End != 0; End++);
if (*End == 0)
{
_error->Warning(_("Malformed override %s line %lu #2"),File.c_str(),
_error->Warning(_("Malformed override %s line %llu #2"),File.c_str(),
Counter);
continue;
}
@@ -188,7 +188,7 @@ bool Override::ReadExtraOverride(string const &File,bool const &Source)
for (; isspace(*(End-1)) && End > Value; End--);
if (End == Value)
{
_error->Warning(_("Malformed override %s line %lu #3"),File.c_str(),
_error->Warning(_("Malformed override %s line %llu #3"),File.c_str(),
Counter);
continue;
}


+ 9
- 9
ftparchive/writer.cc View File

@@ -248,8 +248,8 @@ bool FTWScanner::LoadFileList(string const &Dir, string const &File)
// ---------------------------------------------------------------------
/* */
bool FTWScanner::Delink(string &FileName,const char *OriginalPath,
unsigned long &DeLinkBytes,
off_t const &FileSize)
unsigned long long &DeLinkBytes,
unsigned long long const &FileSize)
{
// See if this isn't an internaly prefix'd file name.
if (InternalPrefix.empty() == false &&
@@ -379,7 +379,7 @@ bool PackagesWriter::DoPackage(string FileName)
return false;
}

off_t FileSize = Db.GetFileSize();
unsigned long long FileSize = Db.GetFileSize();
if (Delink(FileName,OriginalPath,Stats.DeLinkBytes,FileSize) == false)
return false;
@@ -415,7 +415,7 @@ bool PackagesWriter::DoPackage(string FileName)
}

char Size[40];
sprintf(Size,"%lu", (unsigned long) FileSize);
sprintf(Size,"%llu", (unsigned long long) FileSize);
// Strip the DirStrip prefix from the FileName and add the PathPrefix
string NewFileName;
@@ -612,7 +612,7 @@ bool SourcesWriter::DoPackage(string FileName)
if (St.st_size > 128*1024)
return _error->Error("DSC file '%s' is too large!",FileName.c_str());
if (BufSize < (unsigned)St.st_size+1)
if (BufSize < (unsigned long long)St.st_size+1)
{
BufSize = St.st_size+1;
Buffer = (char *)realloc(Buffer,St.st_size+1);
@@ -1067,7 +1067,7 @@ void ReleaseWriter::Finish()
for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
I != CheckSums.end(); ++I)
{
fprintf(Output, " %s %16ld %s\n",
fprintf(Output, " %s %16llu %s\n",
(*I).second.MD5.c_str(),
(*I).second.size,
(*I).first.c_str());
@@ -1079,7 +1079,7 @@ void ReleaseWriter::Finish()
for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
I != CheckSums.end(); ++I)
{
fprintf(Output, " %s %16ld %s\n",
fprintf(Output, " %s %16llu %s\n",
(*I).second.SHA1.c_str(),
(*I).second.size,
(*I).first.c_str());
@@ -1091,7 +1091,7 @@ void ReleaseWriter::Finish()
for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
I != CheckSums.end(); ++I)
{
fprintf(Output, " %s %16ld %s\n",
fprintf(Output, " %s %16llu %s\n",
(*I).second.SHA256.c_str(),
(*I).second.size,
(*I).first.c_str());
@@ -1103,7 +1103,7 @@ void ReleaseWriter::Finish()
I != CheckSums.end();
++I)
{
fprintf(Output, " %s %32ld %s\n",
fprintf(Output, " %s %16llu %s\n",
(*I).second.SHA512.c_str(),
(*I).second.size,
(*I).first.c_str());


+ 3
- 3
ftparchive/writer.h View File

@@ -48,7 +48,7 @@ class FTWScanner
static int ScannerFile(const char *File, bool const &ReadLink);

bool Delink(string &FileName,const char *OriginalPath,
unsigned long &Bytes,off_t const &FileSize);
unsigned long long &Bytes,unsigned long long const &FileSize);

inline void NewLine(unsigned const &Priority)
{
@@ -159,7 +159,7 @@ class SourcesWriter : public FTWScanner
Override BOver;
Override SOver;
char *Buffer;
unsigned long BufSize;
unsigned long long BufSize;
public:

@@ -198,7 +198,7 @@ protected:
string SHA256;
string SHA512;
// Limited by FileFd::Size()
unsigned long size;
unsigned long long size;
~CheckSum() {};
};
map<string,struct CheckSum> CheckSums;


+ 6
- 6
methods/ftp.cc View File

@@ -629,7 +629,7 @@ bool FTPConn::ExtGoPasv()
// FTPConn::Size - Return the size of a file /*{{{*/
// ---------------------------------------------------------------------
/* Grab the file size from the server, 0 means no size or empty file */
bool FTPConn::Size(const char *Path,unsigned long &Size)
bool FTPConn::Size(const char *Path,unsigned long long &Size)
{
// Query the size
unsigned int Tag;
@@ -639,7 +639,7 @@ bool FTPConn::Size(const char *Path,unsigned long &Size)
return false;
char *End;
Size = strtol(Msg.c_str(),&End,10);
Size = strtoull(Msg.c_str(),&End,10);
if (Tag >= 400 || End == Msg.c_str())
Size = 0;
return true;
@@ -841,7 +841,7 @@ bool FTPConn::Finalize()
// ---------------------------------------------------------------------
/* This opens a data connection, sends REST and RETR and then
transfers the file over. */
bool FTPConn::Get(const char *Path,FileFd &To,unsigned long Resume,
bool FTPConn::Get(const char *Path,FileFd &To,unsigned long long Resume,
Hashes &Hash,bool &Missing)
{
Missing = false;
@@ -1004,7 +1004,7 @@ bool FtpMethod::Fetch(FetchItem *Itm)
// Get the files information
Status(_("Query"));
unsigned long Size;
unsigned long long Size;
if (Server->Size(File,Size) == false ||
Server->ModTime(File,FailTime) == false)
{
@@ -1026,7 +1026,7 @@ bool FtpMethod::Fetch(FetchItem *Itm)
struct stat Buf;
if (stat(Itm->DestFile.c_str(),&Buf) == 0)
{
if (Size == (unsigned)Buf.st_size && FailTime == Buf.st_mtime)
if (Size == (unsigned long long)Buf.st_size && FailTime == Buf.st_mtime)
{
Res.Size = Buf.st_size;
Res.LastModified = Buf.st_mtime;
@@ -1036,7 +1036,7 @@ bool FtpMethod::Fetch(FetchItem *Itm)
}
// Resume?
if (FailTime == Buf.st_mtime && Size > (unsigned)Buf.st_size)
if (FailTime == Buf.st_mtime && Size > (unsigned long long)Buf.st_size)
Res.ResumePoint = Buf.st_size;
}


+ 2
- 2
methods/ftp.h View File

@@ -53,9 +53,9 @@ class FTPConn
bool ExtGoPasv();
// Query
bool Size(const char *Path,unsigned long &Size);
bool Size(const char *Path,unsigned long long &Size);
bool ModTime(const char *Path, time_t &Time);
bool Get(const char *Path,FileFd &To,unsigned long Resume,
bool Get(const char *Path,FileFd &To,unsigned long long Resume,
Hashes &MD5,bool &Missing);
FTPConn(URI Srv);


+ 1
- 1
methods/gzip.cc View File

@@ -64,7 +64,7 @@ bool GzipMethod::Fetch(FetchItem *Itm)
while (1)
{
unsigned char Buffer[4*1024];
unsigned long Count;
unsigned long long Count = 0;
if (!From.Read(Buffer,sizeof(Buffer),&Count))
{


+ 16
- 16
methods/http.cc View File

@@ -65,15 +65,15 @@ bool AllowRedirect = false;
bool Debug = false;
URI Proxy;

unsigned long CircleBuf::BwReadLimit=0;
unsigned long CircleBuf::BwTickReadData=0;
unsigned long long CircleBuf::BwReadLimit=0;
unsigned long long CircleBuf::BwTickReadData=0;
struct timeval CircleBuf::BwReadTick={0,0};
const unsigned int CircleBuf::BW_HZ=10;
// CircleBuf::CircleBuf - Circular input buffer /*{{{*/
// ---------------------------------------------------------------------
/* */
CircleBuf::CircleBuf(unsigned long Size) : Size(Size), Hash(0)
CircleBuf::CircleBuf(unsigned long long Size) : Size(Size), Hash(0)
{
Buf = new unsigned char[Size];
Reset();
@@ -89,7 +89,7 @@ void CircleBuf::Reset()
InP = 0;
OutP = 0;
StrPos = 0;
MaxGet = (unsigned int)-1;
MaxGet = (unsigned long long)-1;
OutQueue = string();
if (Hash != 0)
{
@@ -104,7 +104,7 @@ void CircleBuf::Reset()
is non-blocking.. */
bool CircleBuf::Read(int Fd)
{
unsigned long BwReadMax;
unsigned long long BwReadMax;

while (1)
{
@@ -119,7 +119,7 @@ bool CircleBuf::Read(int Fd)
struct timeval now;
gettimeofday(&now,0);

unsigned long d = (now.tv_sec-CircleBuf::BwReadTick.tv_sec)*1000000 +
unsigned long long d = (now.tv_sec-CircleBuf::BwReadTick.tv_sec)*1000000 +
now.tv_usec-CircleBuf::BwReadTick.tv_usec;
if(d > 1000000/BW_HZ) {
CircleBuf::BwReadTick = now;
@@ -133,7 +133,7 @@ bool CircleBuf::Read(int Fd)
}

// Write the buffer segment
int Res;
ssize_t Res;
if(CircleBuf::BwReadLimit) {
Res = read(Fd,Buf + (InP%Size),
BwReadMax > LeftRead() ? LeftRead() : BwReadMax);
@@ -182,7 +182,7 @@ void CircleBuf::FillOut()
return;
// Write the buffer segment
unsigned long Sz = LeftRead();
unsigned long long Sz = LeftRead();
if (OutQueue.length() - StrPos < Sz)
Sz = OutQueue.length() - StrPos;
memcpy(Buf + (InP%Size),OutQueue.c_str() + StrPos,Sz);
@@ -216,7 +216,7 @@ bool CircleBuf::Write(int Fd)
return true;
// Write the buffer segment
int Res;
ssize_t Res;
Res = write(Fd,Buf + (OutP%Size),LeftWrite());

if (Res == 0)
@@ -242,7 +242,7 @@ bool CircleBuf::Write(int Fd)
bool CircleBuf::WriteTillEl(string &Data,bool Single)
{
// We cheat and assume it is unneeded to have more than one buffer load
for (unsigned long I = OutP; I < InP; I++)
for (unsigned long long I = OutP; I < InP; I++)
{
if (Buf[I%Size] != '\n')
continue;
@@ -260,7 +260,7 @@ bool CircleBuf::WriteTillEl(string &Data,bool Single)
Data = "";
while (OutP < I)
{
unsigned long Sz = LeftWrite();
unsigned long long Sz = LeftWrite();
if (Sz == 0)
return false;
if (I - OutP < Sz)
@@ -455,7 +455,7 @@ bool ServerState::RunData()
return false;
// See if we are done
unsigned long Len = strtol(Data.c_str(),0,16);
unsigned long long Len = strtoull(Data.c_str(),0,16);
if (Len == 0)
{
In.Limit(-1);
@@ -598,7 +598,7 @@ bool ServerState::HeaderLine(string Line)
if (StartPos != 0)
return true;
if (sscanf(Val.c_str(),"%lu",&Size) != 1)
if (sscanf(Val.c_str(),"%llu",&Size) != 1)
return _error->Error(_("The HTTP server sent an invalid Content-Length header"));
return true;
}
@@ -613,9 +613,9 @@ bool ServerState::HeaderLine(string Line)
{
HaveContent = true;
if (sscanf(Val.c_str(),"bytes %lu-%*u/%lu",&StartPos,&Size) != 2)
if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&Size) != 2)
return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
if ((unsigned)StartPos > Size)
if ((unsigned long long)StartPos > Size)
return _error->Error(_("This HTTP server has broken range support"));
return true;
}
@@ -718,7 +718,7 @@ void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
{
// In this case we send an if-range query with a range header
sprintf(Buf,"Range: bytes=%li-\r\nIf-Range: %s\r\n",(long)SBuf.st_size - 1,
sprintf(Buf,"Range: bytes=%lli-\r\nIf-Range: %s\r\n",(long long)SBuf.st_size - 1,
TimeRFC1123(SBuf.st_mtime).c_str());
Req += Buf;
}


+ 15
- 15
methods/http.h View File

@@ -23,29 +23,29 @@ class HttpMethod;
class CircleBuf
{
unsigned char *Buf;
unsigned long Size;
unsigned long InP;
unsigned long OutP;
unsigned long long Size;
unsigned long long InP;
unsigned long long OutP;
string OutQueue;
unsigned long StrPos;
unsigned long MaxGet;
unsigned long long StrPos;
unsigned long long MaxGet;
struct timeval Start;
static unsigned long BwReadLimit;
static unsigned long BwTickReadData;
static unsigned long long BwReadLimit;
static unsigned long long BwTickReadData;
static struct timeval BwReadTick;
static const unsigned int BW_HZ;

unsigned long LeftRead()
unsigned long long LeftRead()
{
unsigned long Sz = Size - (InP - OutP);
unsigned long long Sz = Size - (InP - OutP);
if (Sz > Size - (InP%Size))
Sz = Size - (InP%Size);
return Sz;
}
unsigned long LeftWrite()
unsigned long long LeftWrite()
{
unsigned long Sz = InP - OutP;
unsigned long long Sz = InP - OutP;
if (InP > MaxGet)
Sz = MaxGet - OutP;
if (Sz > Size - (OutP%Size))
@@ -67,7 +67,7 @@ class CircleBuf
bool WriteTillEl(string &Data,bool Single = false);
// Control the write limit
void Limit(long Max) {if (Max == -1) MaxGet = 0-1; else MaxGet = OutP + Max;}
void Limit(long long Max) {if (Max == -1) MaxGet = 0-1; else MaxGet = OutP + Max;}
bool IsLimit() {return MaxGet == OutP;};
void Print() {cout << MaxGet << ',' << OutP << endl;};

@@ -79,7 +79,7 @@ class CircleBuf
void Reset();
void Stats();

CircleBuf(unsigned long Size);
CircleBuf(unsigned long long Size);
~CircleBuf() {delete [] Buf; delete Hash;};
};

@@ -92,8 +92,8 @@ struct ServerState
char Code[MAXLEN];
// These are some statistics from the last parsed header lines
unsigned long Size;
signed long StartPos;
unsigned long long Size;
signed long long StartPos;
time_t Date;
bool HaveContent;
enum {Chunked,Stream,Closes} Encoding;


+ 1
- 1
methods/https.cc View File

@@ -52,7 +52,7 @@ HttpsMethod::progress_callback(void *clientp, double dltotal, double dlnow,
{
HttpsMethod *me = (HttpsMethod *)clientp;
if(dltotal > 0 && me->Res.Size == 0) {
me->Res.Size = (unsigned long)dltotal;
me->Res.Size = (unsigned long long)dltotal;
me->URIStart(me->Res);
}
return 0;


+ 1
- 1
methods/rred.cc View File

@@ -256,7 +256,7 @@ RredMethod::State RredMethod::patchMMap(FileFd &Patch, FileFd &From, /*{{{*/
#ifdef _POSIX_MAPPED_FILES
MMap ed_cmds(MMap::ReadOnly);
if (Patch.gzFd() != NULL) {
unsigned long mapSize = Patch.Size();
unsigned long long mapSize = Patch.Size();
DynamicMMap* dyn = new DynamicMMap(0, mapSize, 0);
if (dyn->validData() == false) {
delete dyn;


+ 8
- 8
methods/rsh.cc View File

@@ -251,7 +251,7 @@ bool RSHConn::WriteMsg(string &Text,bool Sync,const char *Fmt,...)
// ---------------------------------------------------------------------
/* Right now for successfull transfer the file size must be known in
advance. */
bool RSHConn::Size(const char *Path,unsigned long &Size)
bool RSHConn::Size(const char *Path,unsigned long long &Size)
{
// Query the size
string Msg;
@@ -263,7 +263,7 @@ bool RSHConn::Size(const char *Path,unsigned long &Size)
// FIXME: Sense if the bad reply is due to a File Not Found.
char *End;
Size = strtoul(Msg.c_str(),&End,10);
Size = strtoull(Msg.c_str(),&End,10);
if (End == Msg.c_str())
return _error->Error(_("File not found"));
return true;
@@ -288,8 +288,8 @@ bool RSHConn::ModTime(const char *Path, time_t &Time)
// RSHConn::Get - Get a file /*{{{*/
// ---------------------------------------------------------------------
/* */
bool RSHConn::Get(const char *Path,FileFd &To,unsigned long Resume,
Hashes &Hash,bool &Missing, unsigned long Size)
bool RSHConn::Get(const char *Path,FileFd &To,unsigned long long Resume,
Hashes &Hash,bool &Missing, unsigned long long Size)
{
Missing = false;

@@ -314,7 +314,7 @@ bool RSHConn::Get(const char *Path,FileFd &To,unsigned long Resume,
return false;

// Copy loop
unsigned int MyLen = Resume;
unsigned long long MyLen = Resume;
unsigned char Buffer[4096];
while (MyLen < Size)
{
@@ -428,7 +428,7 @@ bool RSHMethod::Fetch(FetchItem *Itm)
Status(_("Connecting to %s"), Get.Host.c_str());

// Get the files information
unsigned long Size;
unsigned long long Size;
if (Server->Size(File,Size) == false ||
Server->ModTime(File,FailTime) == false)
{
@@ -449,7 +449,7 @@ bool RSHMethod::Fetch(FetchItem *Itm)
// See if the file exists
struct stat Buf;
if (stat(Itm->DestFile.c_str(),&Buf) == 0) {
if (Size == (unsigned)Buf.st_size && FailTime == Buf.st_mtime) {
if (Size == (unsigned long long)Buf.st_size && FailTime == Buf.st_mtime) {
Res.Size = Buf.st_size;
Res.LastModified = Buf.st_mtime;
Res.ResumePoint = Buf.st_size;
@@ -458,7 +458,7 @@ bool RSHMethod::Fetch(FetchItem *Itm)
}

// Resume?
if (FailTime == Buf.st_mtime && Size > (unsigned)Buf.st_size)
if (FailTime == Buf.st_mtime && Size > (unsigned long long)Buf.st_size)
Res.ResumePoint = Buf.st_size;
}



+ 3
- 3
methods/rsh.h View File

@@ -41,10 +41,10 @@ class RSHConn
void Close();

// Query
bool Size(const char *Path,unsigned long &Size);
bool Size(const char *Path,unsigned long long &Size);
bool ModTime(const char *Path, time_t &Time);
bool Get(const char *Path,FileFd &To,unsigned long Resume,
Hashes &Hash,bool &Missing, unsigned long Size);
bool Get(const char *Path,FileFd &To,unsigned long long Resume,
Hashes &Hash,bool &Missing, unsigned long long Size);

RSHConn(URI Srv);
~RSHConn();


Loading…
Cancel
Save