Browse Source

calculate only expected hashes in methods

Methods get told which hashes are expected by the acquire system, which
means we can use this list to restrict what we calculate in the methods
as any extra we are calculating is wasted effort as we can't compare it
with anything anyway.

Adding support for a new hash algorithm is therefore 'free' now and if a
algorithm is no longer provided in a repository for a file, we
automatically stop calculating it.

In practice this results in a speed-up in Debian as we don't have SHA512
here (so far), so we practically stop calculating it.
tags/debian/1.1.exp9
David Kalnischkies 6 years ago
parent
commit
9224ce3d4d
18 changed files with 116 additions and 46 deletions
  1. +48
    -15
      apt-pkg/contrib/hashes.cc
  2. +17
    -4
      apt-pkg/contrib/hashes.h
  3. +2
    -2
      ftparchive/cachedb.cc
  4. +2
    -2
      ftparchive/writer.cc
  5. +1
    -1
      methods/cdrom.cc
  6. +5
    -5
      methods/copy.cc
  7. +1
    -1
      methods/file.cc
  8. +1
    -1
      methods/ftp.cc
  9. +1
    -1
      methods/gzip.cc
  10. +7
    -7
      methods/http.cc
  11. +1
    -1
      methods/http.h
  12. +1
    -1
      methods/https.cc
  13. +1
    -1
      methods/https.h
  14. +1
    -1
      methods/rred.cc
  15. +1
    -1
      methods/rsh.cc
  16. +1
    -1
      methods/server.cc
  17. +1
    -1
      methods/server.h
  18. +24
    -0
      test/libapt/hashsums_test.cc

+ 48
- 15
apt-pkg/contrib/hashes.cc View File

@@ -250,28 +250,34 @@ bool HashStringList::operator!=(HashStringList const &other) const
class PrivateHashes {
public:
unsigned long long FileSize;
unsigned int CalcHashes;

PrivateHashes() : FileSize(0) {}
PrivateHashes(unsigned int const CalcHashes) : FileSize(0), CalcHashes(CalcHashes) {}
};
/*}}}*/
// Hashes::Add* - Add the contents of data or FD /*{{{*/
bool Hashes::Add(const unsigned char * const Data,unsigned long long const Size, unsigned int const Hashes)
bool Hashes::Add(const unsigned char * const Data, unsigned long long const Size)
{
bool Res = true;
APT_IGNORE_DEPRECATED_PUSH
if ((Hashes & MD5SUM) == MD5SUM)
if ((d->CalcHashes & MD5SUM) == MD5SUM)
Res &= MD5.Add(Data, Size);
if ((Hashes & SHA1SUM) == SHA1SUM)
if ((d->CalcHashes & SHA1SUM) == SHA1SUM)
Res &= SHA1.Add(Data, Size);
if ((Hashes & SHA256SUM) == SHA256SUM)
if ((d->CalcHashes & SHA256SUM) == SHA256SUM)
Res &= SHA256.Add(Data, Size);
if ((Hashes & SHA512SUM) == SHA512SUM)
if ((d->CalcHashes & SHA512SUM) == SHA512SUM)
Res &= SHA512.Add(Data, Size);
APT_IGNORE_DEPRECATED_POP
d->FileSize += Size;
return Res;
}
bool Hashes::AddFD(int const Fd,unsigned long long Size, unsigned int const Hashes)
bool Hashes::Add(const unsigned char * const Data, unsigned long long const Size, unsigned int const Hashes)
{
d->CalcHashes = Hashes;
return Add(Data, Size);
}
bool Hashes::AddFD(int const Fd,unsigned long long Size)
{
unsigned char Buf[64*64];
bool const ToEOF = (Size == UntilEOF);
@@ -285,12 +291,17 @@ bool Hashes::AddFD(int const Fd,unsigned long long Size, unsigned int const Hash
if (ToEOF && Res == 0) // EOF
break;
Size -= Res;
if (Add(Buf, Res, Hashes) == false)
if (Add(Buf, Res) == false)
return false;
}
return true;
}
bool Hashes::AddFD(FileFd &Fd,unsigned long long Size, unsigned int const Hashes)
bool Hashes::AddFD(int const Fd,unsigned long long Size, unsigned int const Hashes)
{
d->CalcHashes = Hashes;
return AddFD(Fd, Size);
}
bool Hashes::AddFD(FileFd &Fd,unsigned long long Size)
{
unsigned char Buf[64*64];
bool const ToEOF = (Size == 0);
@@ -309,20 +320,29 @@ bool Hashes::AddFD(FileFd &Fd,unsigned long long Size, unsigned int const Hashes
else if (a == 0) // EOF
break;
Size -= a;
if (Add(Buf, a, Hashes) == false)
if (Add(Buf, a) == false)
return false;
}
return true;
}
bool Hashes::AddFD(FileFd &Fd,unsigned long long Size, unsigned int const Hashes)
{
d->CalcHashes = Hashes;
return AddFD(Fd, Size);
}
/*}}}*/
HashStringList Hashes::GetHashStringList()
{
HashStringList hashes;
APT_IGNORE_DEPRECATED_PUSH
hashes.push_back(HashString("MD5Sum", MD5.Result().Value()));
hashes.push_back(HashString("SHA1", SHA1.Result().Value()));
hashes.push_back(HashString("SHA256", SHA256.Result().Value()));
hashes.push_back(HashString("SHA512", SHA512.Result().Value()));
if ((d->CalcHashes & MD5SUM) == MD5SUM)
hashes.push_back(HashString("MD5Sum", MD5.Result().Value()));
if ((d->CalcHashes & SHA1SUM) == SHA1SUM)
hashes.push_back(HashString("SHA1", SHA1.Result().Value()));
if ((d->CalcHashes & SHA256SUM) == SHA256SUM)
hashes.push_back(HashString("SHA256", SHA256.Result().Value()));
if ((d->CalcHashes & SHA512SUM) == SHA512SUM)
hashes.push_back(HashString("SHA512", SHA512.Result().Value()));
APT_IGNORE_DEPRECATED_POP
std::string SizeStr;
strprintf(SizeStr, "%llu", d->FileSize);
@@ -330,6 +350,19 @@ APT_IGNORE_DEPRECATED_POP
return hashes;
}
APT_IGNORE_DEPRECATED_PUSH
Hashes::Hashes() { d = new PrivateHashes(); }
Hashes::Hashes() { d = new PrivateHashes(~0); }
Hashes::Hashes(unsigned int const Hashes) { d = new PrivateHashes(Hashes); }
Hashes::Hashes(HashStringList const &Hashes) {
unsigned int calcHashes = Hashes.usable() ? 0 : ~0;
if (Hashes.find("MD5Sum") != NULL)
calcHashes |= MD5SUM;
if (Hashes.find("SHA1") != NULL)
calcHashes |= SHA1SUM;
if (Hashes.find("SHA256") != NULL)
calcHashes |= SHA256SUM;
if (Hashes.find("SHA512") != NULL)
calcHashes |= SHA512SUM;
d = new PrivateHashes(calcHashes);
}
Hashes::~Hashes() { delete d; }
APT_IGNORE_DEPRECATED_POP

+ 17
- 4
apt-pkg/contrib/hashes.h View File

@@ -178,7 +178,8 @@ class Hashes

static const int UntilEOF = 0;

bool Add(const unsigned char * const Data, unsigned long long const Size, unsigned int const Hashes = ~0);
bool Add(const unsigned char * const Data, unsigned long long const Size);
APT_DEPRECATED bool Add(const unsigned char * const Data, unsigned long long const Size, unsigned int const Hashes);
inline bool Add(const char * const Data)
{return Add((unsigned char const * const)Data,strlen(Data));};
inline bool Add(const unsigned char * const Beg,const unsigned char * const End)
@@ -186,13 +187,24 @@ class Hashes

enum SupportedHashes { MD5SUM = (1 << 0), SHA1SUM = (1 << 1), SHA256SUM = (1 << 2),
SHA512SUM = (1 << 3) };
bool AddFD(int const Fd,unsigned long long Size = 0, unsigned int const Hashes = ~0);
bool AddFD(FileFd &Fd,unsigned long long Size = 0, unsigned int const Hashes = ~0);
bool AddFD(int const Fd,unsigned long long Size = 0);
APT_DEPRECATED bool AddFD(int const Fd,unsigned long long Size, unsigned int const Hashes);
bool AddFD(FileFd &Fd,unsigned long long Size = 0);
APT_DEPRECATED bool AddFD(FileFd &Fd,unsigned long long Size, unsigned int const Hashes);

HashStringList GetHashStringList();

APT_IGNORE_DEPRECATED_PUSH
/** create a Hashes object to calculate all supported hashes
*
* If ALL is too much, you can limit which Hashes are calculated
* with the following other constructors which mention explicitly
* which hashes to generate. */
Hashes();
/** @param Hashes bitflag composed of #SupportedHashes */
Hashes(unsigned int const Hashes);
/** @param Hashes is a list of hashes */
Hashes(HashStringList const &Hashes);
virtual ~Hashes();
APT_IGNORE_DEPRECATED_POP

@@ -208,15 +220,16 @@ APT_IGNORE_DEPRECATED_POP
}

public:
APT_IGNORE_DEPRECATED_PUSH
APT_DEPRECATED bool AddFD(int const Fd, unsigned long long Size, bool const addMD5,
bool const addSHA1, bool const addSHA256, bool const addSHA512) {
return AddFD(Fd, Size, boolsToFlag(addMD5, addSHA1, addSHA256, addSHA512));
};

APT_DEPRECATED bool AddFD(FileFd &Fd, unsigned long long Size, bool const addMD5,
bool const addSHA1, bool const addSHA256, bool const addSHA512) {
return AddFD(Fd, Size, boolsToFlag(addMD5, addSHA1, addSHA256, addSHA512));
};
APT_IGNORE_DEPRECATED_POP
};

#endif

+ 2
- 2
ftparchive/cachedb.cc View File

@@ -441,8 +441,8 @@ bool CacheDB::GetHashes(bool const GenOnly, unsigned int const DoHashes)
if (OpenFile() == false)
return false;

Hashes hashes;
if (Fd->Seek(0) == false || hashes.AddFD(*Fd, CurStat.FileSize, FlHashes) == false)
Hashes hashes(FlHashes);
if (Fd->Seek(0) == false || hashes.AddFD(*Fd, CurStat.FileSize) == false)
return false;

HashStringList hl = hashes.GetHashStringList();


+ 2
- 2
ftparchive/writer.cc View File

@@ -1075,8 +1075,8 @@ bool ReleaseWriter::DoPackage(string FileName)

CheckSums[NewFileName].size = fd.Size();

Hashes hs;
hs.AddFD(fd, 0, DoHashes);
Hashes hs(DoHashes);
hs.AddFD(fd);
CheckSums[NewFileName].Hashes = hs.GetHashStringList();
fd.Close();



+ 1
- 1
methods/cdrom.cc View File

@@ -266,7 +266,7 @@ bool CDROMMethod::Fetch(FetchItem *Itm)
Res.LastModified = Buf.st_mtime;
Res.Size = Buf.st_size;

Hashes Hash;
Hashes Hash(Itm->ExpectedHashes);
FileFd Fd(Res.Filename, FileFd::ReadOnly);
Hash.AddFD(Fd);
Res.TakeHashes(Hash);


+ 5
- 5
methods/copy.cc View File

@@ -28,16 +28,16 @@
class CopyMethod : public pkgAcqMethod
{
virtual bool Fetch(FetchItem *Itm);
void CalculateHashes(FetchResult &Res);
void CalculateHashes(FetchItem const * const Itm, FetchResult &Res);
public:
CopyMethod() : pkgAcqMethod("1.0",SingleInstance | SendConfig) {};
};

void CopyMethod::CalculateHashes(FetchResult &Res)
void CopyMethod::CalculateHashes(FetchItem const * const Itm, FetchResult &Res)
{
Hashes Hash;
Hashes Hash(Itm->ExpectedHashes);
FileFd::CompressMode CompressMode = FileFd::None;
if (_config->FindB("Acquire::GzipIndexes", false) == true)
CompressMode = FileFd::Extension;
@@ -71,7 +71,7 @@ bool CopyMethod::Fetch(FetchItem *Itm)
// just calc the hashes if the source and destination are identical
if (File == Itm->DestFile)
{
CalculateHashes(Res);
CalculateHashes(Itm, Res);
URIDone(Res);
return true;
}
@@ -104,7 +104,7 @@ bool CopyMethod::Fetch(FetchItem *Itm)
if (utimes(Res.Filename.c_str(), times) != 0)
return _error->Errno("utimes",_("Failed to set modification time"));

CalculateHashes(Res);
CalculateHashes(Itm, Res);

URIDone(Res);
return true;


+ 1
- 1
methods/file.cc View File

@@ -87,7 +87,7 @@ bool FileMethod::Fetch(FetchItem *Itm)
if (Res.Filename.empty() == true)
return _error->Error(_("File not found"));

Hashes Hash;
Hashes Hash(Itm->ExpectedHashes);
FileFd Fd(Res.Filename, FileFd::ReadOnly);
Hash.AddFD(Fd);
Res.TakeHashes(Hash);


+ 1
- 1
methods/ftp.cc View File

@@ -1064,7 +1064,7 @@ bool FtpMethod::Fetch(FetchItem *Itm)
}
// Open the file
Hashes Hash;
Hashes Hash(Itm->ExpectedHashes);
{
FileFd Fd(Itm->DestFile,FileFd::WriteAny);
if (_error->PendingError() == true)


+ 1
- 1
methods/gzip.cc View File

@@ -91,7 +91,7 @@ bool GzipMethod::Fetch(FetchItem *Itm)
return false;

// Read data from source, generate checksums and write
Hashes Hash;
Hashes Hash(Itm->ExpectedHashes);
bool Failed = false;
while (1)
{


+ 7
- 7
methods/http.cc View File

@@ -64,8 +64,8 @@ const unsigned int CircleBuf::BW_HZ=10;
// CircleBuf::CircleBuf - Circular input buffer /*{{{*/
// ---------------------------------------------------------------------
/* */
CircleBuf::CircleBuf(unsigned long long Size)
: Size(Size), Hash(0), TotalWriten(0)
CircleBuf::CircleBuf(unsigned long long Size)
: Size(Size), Hash(NULL), TotalWriten(0)
{
Buf = new unsigned char[Size];
Reset();
@@ -84,10 +84,10 @@ void CircleBuf::Reset()
TotalWriten = 0;
MaxGet = (unsigned long long)-1;
OutQueue = string();
if (Hash != 0)
if (Hash != NULL)
{
delete Hash;
Hash = new Hashes;
Hash = NULL;
}
}
/*}}}*/
@@ -222,7 +222,7 @@ bool CircleBuf::Write(int Fd)

TotalWriten += Res;
if (Hash != 0)
if (Hash != NULL)
Hash->Add(Buf + (OutP%Size),Res);
OutP += Res;
@@ -484,10 +484,10 @@ APT_PURE bool HttpServerState::IsOpen() /*{{{*/
return (ServerFd != -1);
}
/*}}}*/
bool HttpServerState::InitHashes(FileFd &File) /*{{{*/
bool HttpServerState::InitHashes(FileFd &File, HashStringList const &ExpectedHashes)/*{{{*/
{
delete In.Hash;
In.Hash = new Hashes;
In.Hash = new Hashes(ExpectedHashes);

// Set the expected size and read file for the hashes
File.Truncate(StartPos);


+ 1
- 1
methods/http.h View File

@@ -111,7 +111,7 @@ struct HttpServerState: public ServerState
virtual bool Open();
virtual bool IsOpen();
virtual bool Close();
virtual bool InitHashes(FileFd &File);
virtual bool InitHashes(FileFd &File, HashStringList const &ExpectedHashes);
virtual Hashes * GetHashes();
virtual bool Die(FileFd &File);
virtual bool Flush(FileFd * const File);


+ 1
- 1
methods/https.cc View File

@@ -443,7 +443,7 @@ bool HttpsMethod::Fetch(FetchItem *Itm)
Res.LastModified = resultStat.st_mtime;

// take hashes
Hashes Hash;
Hashes Hash(Itm->ExpectedHashes);
FileFd Fd(Res.Filename, FileFd::ReadOnly);
Hash.AddFD(Fd);
Res.TakeHashes(Hash);


+ 1
- 1
methods/https.h View File

@@ -42,7 +42,7 @@ class HttpsServerState : public ServerState
virtual bool Open() { return false; }
virtual bool IsOpen() { return false; }
virtual bool Close() { return false; }
virtual bool InitHashes(FileFd &/*File*/) { return false; }
virtual bool InitHashes(FileFd &/*File*/, HashStringList const &/*ExpectedHashes*/) { return false; }
virtual Hashes * GetHashes() { return NULL; }
virtual bool Die(FileFd &/*File*/) { return false; }
virtual bool Flush(FileFd * const /*File*/) { return false; }


+ 1
- 1
methods/rred.cc View File

@@ -581,7 +581,7 @@ class RredMethod : public pkgAcqMethod {
FILE *inp = fopen(Path.c_str(), "r");
FILE *out = fopen(Itm->DestFile.c_str(), "w");

Hashes hash;
Hashes hash(Itm->ExpectedHashes);

patch.apply_against_file(out, inp, &hash);



+ 1
- 1
methods/rsh.cc View File

@@ -477,7 +477,7 @@ bool RSHMethod::Fetch(FetchItem *Itm)
}

// Open the file
Hashes Hash;
Hashes Hash(Itm->ExpectedHashes);
{
FileFd Fd(Itm->DestFile,FileFd::WriteAny);
if (_error->PendingError() == true)


+ 1
- 1
methods/server.cc View File

@@ -357,7 +357,7 @@ ServerMethod::DealWithHeaders(FetchResult &Res)
FailFd = File->Fd();
FailTime = Server->Date;

if (Server->InitHashes(*File) == false)
if (Server->InitHashes(*File, Queue->ExpectedHashes) == false)
{
_error->Errno("read",_("Problem hashing file"));
return ERROR_NOT_FROM_SERVER;


+ 1
- 1
methods/server.h View File

@@ -85,7 +85,7 @@ struct ServerState
virtual bool Open() = 0;
virtual bool IsOpen() = 0;
virtual bool Close() = 0;
virtual bool InitHashes(FileFd &File) = 0;
virtual bool InitHashes(FileFd &File, HashStringList const &ExpectedHashes) = 0;
virtual Hashes * GetHashes() = 0;
virtual bool Die(FileFd &File) = 0;
virtual bool Flush(FileFd * const File) = 0;


+ 24
- 0
test/libapt/hashsums_test.cc View File

@@ -193,6 +193,30 @@ TEST(HashSumsTest, FileBased)
EXPECT_EQ(FileSize, list.find("Checksum-FileSize")->HashValue());
}
fd.Seek(0);
{
Hashes hashes(Hashes::MD5SUM | Hashes::SHA512SUM);
hashes.AddFD(fd);
HashStringList list = hashes.GetHashStringList();
EXPECT_FALSE(list.empty());
EXPECT_EQ(3, list.size());
EXPECT_EQ(md5.Value(), list.find("MD5Sum")->HashValue());
EXPECT_EQ(NULL, list.find("SHA1"));
EXPECT_EQ(NULL, list.find("SHA256"));
EXPECT_EQ(sha512.Value(), list.find("SHA512")->HashValue());
EXPECT_EQ(FileSize, list.find("Checksum-FileSize")->HashValue());
fd.Seek(0);
Hashes hashes2(list);
hashes2.AddFD(fd);
list = hashes2.GetHashStringList();
EXPECT_FALSE(list.empty());
EXPECT_EQ(3, list.size());
EXPECT_EQ(md5.Value(), list.find("MD5Sum")->HashValue());
EXPECT_EQ(NULL, list.find("SHA1"));
EXPECT_EQ(NULL, list.find("SHA256"));
EXPECT_EQ(sha512.Value(), list.find("SHA512")->HashValue());
EXPECT_EQ(FileSize, list.find("Checksum-FileSize")->HashValue());
}
fd.Seek(0);
{
MD5Summation MD5;
MD5.AddFD(fd.Fd());


Loading…
Cancel
Save