Browse Source

add volatile sources support in libapt-pkg

Sources are usually defined in sources.list (and co) and are pretty
stable, but once in a while a frontend might want to add an additional
"source" like a local .deb file to install this package (No support for
'real' sources being added this way as this is a multistep process).

We had a hack in place to allow apt-get and apt to pull this of for a
short while now, but other frontends are either left in the cold by this
and/or the code for it looks dirty with FIXMEs plastering it and has on
top of this also some problems (like including these 'volatile' sources
in the srcpkgcache.bin file).

So the biggest part in this commit is actually the rewrite of the cache
generation as it is now potentially a three step process. The biggest
problem with adding support now through is that this makes a bunch of
previously mostly unusable by externs and therefore hidden classes
public, so a bit of further tuneing on this now public API is in order…
tags/debian/1.1.exp9
David Kalnischkies 6 years ago
parent
commit
5465192b9a
15 changed files with 309 additions and 362 deletions
  1. +11
    -16
      apt-pkg/deb/debindexfile.cc
  2. +9
    -9
      apt-pkg/deb/debindexfile.h
  3. +1
    -1
      apt-pkg/deb/deblistparser.cc
  4. +0
    -29
      apt-pkg/deb/debmetaindex.cc
  5. +0
    -39
      apt-pkg/deb/debmetaindex.h
  6. +1
    -2
      apt-pkg/edsp/edspsystem.cc
  7. +188
    -213
      apt-pkg/pkgcachegen.cc
  8. +0
    -1
      apt-pkg/pkgcachegen.h
  9. +3
    -2
      apt-pkg/pkgsystem.h
  10. +25
    -3
      apt-pkg/sourcelist.cc
  11. +19
    -0
      apt-pkg/sourcelist.h
  12. +0
    -21
      apt-private/private-cachefile.h
  13. +3
    -18
      apt-private/private-install.cc
  14. +9
    -3
      test/integration/framework
  15. +40
    -5
      test/integration/test-apt-get-install-deb

+ 11
- 16
apt-pkg/deb/debindexfile.cc View File

@@ -339,18 +339,16 @@ APT_CONST bool debStatusIndex::Exists() const
}
/*}}}*/

// debDebPkgFile - Single .deb file /*{{{*/
debDebPkgFileIndex::debDebPkgFileIndex(std::string DebFile)
// debDebPkgFileIndex - Single .deb file /*{{{*/
debDebPkgFileIndex::debDebPkgFileIndex(std::string const &DebFile)
: pkgIndexFile(true), d(NULL), DebFile(DebFile)
{
DebFileFullPath = flAbsPath(DebFile);
}

std::string debDebPkgFileIndex::ArchiveURI(std::string /*File*/) const
{
return "file:" + DebFileFullPath;
}

bool debDebPkgFileIndex::Exists() const
{
return FileExists(DebFile);
@@ -403,6 +401,8 @@ bool debDebPkgFileIndex::Merge(pkgCacheGenerator& Gen, OpProgress* Prog) const
if (GetContent(content, DebFile) == false)
return false;
std::string const contentstr = content.str();
if (contentstr.empty())
return true;
DebControl->Write(contentstr.c_str(), contentstr.length());
// rewind for the listparser
DebControl->Seek(0);
@@ -431,7 +431,7 @@ pkgCache::PkgFileIterator debDebPkgFileIndex::FindInCache(pkgCache &Cache) const

return File;
}
return File;
}
unsigned long debDebPkgFileIndex::Size() const
@@ -443,12 +443,11 @@ unsigned long debDebPkgFileIndex::Size() const
}
/*}}}*/

// debDscFileIndex stuff
debDscFileIndex::debDscFileIndex(std::string &DscFile)
// debDscFileIndex - a .dsc file /*{{{*/
debDscFileIndex::debDscFileIndex(std::string const &DscFile)
: pkgIndexFile(true), d(NULL), DscFile(DscFile)
{
}

bool debDscFileIndex::Exists() const
{
return FileExists(DscFile);
@@ -461,8 +460,6 @@ unsigned long debDscFileIndex::Size() const
return buf.st_size;
return 0;
}

// DscFileIndex::CreateSrcParser - Get a parser for the .dsc file /*{{{*/
pkgSrcRecords::Parser *debDscFileIndex::CreateSrcParser() const
{
if (!FileExists(DscFile))
@@ -471,18 +468,17 @@ pkgSrcRecords::Parser *debDscFileIndex::CreateSrcParser() const
return new debDscRecordParser(DscFile,this);
}
/*}}}*/

// Index File types for Debian /*{{{*/
class APT_HIDDEN debIFTypeSrc : public pkgIndexFile::Type
{
public:
debIFTypeSrc() {Label = "Debian Source Index";};
};
class APT_HIDDEN debIFTypePkg : public pkgIndexFile::Type
{
public:
virtual pkgRecords::Parser *CreatePkgParser(pkgCache::PkgFileIterator File) const APT_OVERRIDE
virtual pkgRecords::Parser *CreatePkgParser(pkgCache::PkgFileIterator File) const APT_OVERRIDE
{
return new debRecordParser(File.FileName(),*File.Cache());
};
@@ -496,8 +492,7 @@ class APT_HIDDEN debIFTypeTrans : public debIFTypePkg
class APT_HIDDEN debIFTypeStatus : public pkgIndexFile::Type
{
public:
virtual pkgRecords::Parser *CreatePkgParser(pkgCache::PkgFileIterator File) const APT_OVERRIDE
virtual pkgRecords::Parser *CreatePkgParser(pkgCache::PkgFileIterator File) const APT_OVERRIDE
{
return new debRecordParser(File.FileName(),*File.Cache());
};
@@ -506,7 +501,7 @@ class APT_HIDDEN debIFTypeStatus : public pkgIndexFile::Type
class APT_HIDDEN debIFTypeDebPkgFile : public pkgIndexFile::Type
{
public:
virtual pkgRecords::Parser *CreatePkgParser(pkgCache::PkgFileIterator File) const APT_OVERRIDE
virtual pkgRecords::Parser *CreatePkgParser(pkgCache::PkgFileIterator File) const APT_OVERRIDE
{
return new debDebFileRecordParser(File.FileName());
};


+ 9
- 9
apt-pkg/deb/debindexfile.h View File

@@ -27,7 +27,7 @@ class pkgAcquire;
class pkgCacheGenerator;


class APT_HIDDEN debStatusIndex : public pkgIndexFile
class debStatusIndex : public pkgIndexFile
{
void * const d;
protected:
@@ -51,7 +51,7 @@ class APT_HIDDEN debStatusIndex : public pkgIndexFile
virtual ~debStatusIndex();
};

class APT_HIDDEN debPackagesIndex : public pkgIndexTargetFile
class debPackagesIndex : public pkgIndexTargetFile
{
void * const d;
public:
@@ -70,7 +70,7 @@ class APT_HIDDEN debPackagesIndex : public pkgIndexTargetFile
virtual ~debPackagesIndex();
};

class APT_HIDDEN debTranslationsIndex : public pkgIndexTargetFile
class debTranslationsIndex : public pkgIndexTargetFile
{
void * const d;
public:
@@ -86,7 +86,7 @@ class APT_HIDDEN debTranslationsIndex : public pkgIndexTargetFile
virtual ~debTranslationsIndex();
};

class APT_HIDDEN debSourcesIndex : public pkgIndexTargetFile
class debSourcesIndex : public pkgIndexTargetFile
{
void * const d;
public:
@@ -107,7 +107,7 @@ class APT_HIDDEN debSourcesIndex : public pkgIndexTargetFile
virtual ~debSourcesIndex();
};

class APT_HIDDEN debDebPkgFileIndex : public pkgIndexFile
class debDebPkgFileIndex : public pkgIndexFile
{
private:
void * const d;
@@ -141,11 +141,11 @@ class APT_HIDDEN debDebPkgFileIndex : public pkgIndexFile
// Interface for acquire
virtual std::string ArchiveURI(std::string /*File*/) const APT_OVERRIDE;

debDebPkgFileIndex(std::string DebFile);
debDebPkgFileIndex(std::string const &DebFile);
virtual ~debDebPkgFileIndex();
};

class APT_HIDDEN debDscFileIndex : public pkgIndexFile
class debDscFileIndex : public pkgIndexFile
{
private:
void * const d;
@@ -160,11 +160,11 @@ class APT_HIDDEN debDscFileIndex : public pkgIndexFile
return DscFile;
};

debDscFileIndex(std::string &DscFile);
debDscFileIndex(std::string const &DscFile);
virtual ~debDscFileIndex();
};

class APT_HIDDEN debDebianSourceDirIndex : public debDscFileIndex
class debDebianSourceDirIndex : public debDscFileIndex
{
public:
virtual const Type *GetType() const APT_CONST;


+ 1
- 1
apt-pkg/deb/deblistparser.cc View File

@@ -959,7 +959,7 @@ bool debDebFileParser::UsePackage(pkgCache::PkgIterator &Pkg,
bool res = debListParser::UsePackage(Pkg, Ver);
// we use the full file path as a provides so that the file is found
// by its name
if(NewProvidesAllArch(Ver, DebFile, Ver.VerStr(), 0) == false)
if(NewProvides(Ver, DebFile, Pkg.Cache()->NativeArch(), Ver.VerStr(), 0) == false)
return false;
return res;
}


+ 0
- 29
apt-pkg/deb/debmetaindex.cc View File

@@ -785,34 +785,5 @@ class APT_HIDDEN debSLTypeDebSrc : public debSLTypeDebian /*{{{*/
};
/*}}}*/

debDebFileMetaIndex::debDebFileMetaIndex(std::string const &DebFile) /*{{{*/
: metaIndex(DebFile, "local-uri", "deb-dist"), d(NULL), DebFile(DebFile)
{
DebIndex = new debDebPkgFileIndex(DebFile);
Indexes = new std::vector<pkgIndexFile *>();
Indexes->push_back(DebIndex);
}
debDebFileMetaIndex::~debDebFileMetaIndex() {}
/*}}}*/
class APT_HIDDEN debSLTypeDebFile : public pkgSourceList::Type /*{{{*/
{
public:

bool CreateItem(std::vector<metaIndex *> &List, std::string const &URI,
std::string const &/*Dist*/, std::string const &/*Section*/,
std::map<std::string, std::string> const &/*Options*/) const APT_OVERRIDE
{
metaIndex *mi = new debDebFileMetaIndex(URI);
List.push_back(mi);
return true;
}

debSLTypeDebFile() : Type("deb-file", "Debian local deb file")
{
}
};
/*}}}*/

APT_HIDDEN debSLTypeDeb _apt_DebType;
APT_HIDDEN debSLTypeDebSrc _apt_DebSrcType;
APT_HIDDEN debSLTypeDebFile _apt_DebFileType;

+ 0
- 39
apt-pkg/deb/debmetaindex.h View File

@@ -17,7 +17,6 @@

class pkgAcquire;
class pkgIndexFile;
class debDebPkgFileIndex;
class IndexTarget;
class pkgCacheGenerator;
class OpProgress;
@@ -66,42 +65,4 @@ class APT_HIDDEN debReleaseIndex : public metaIndex
std::vector<std::string> Languages);
};

class APT_HIDDEN debDebFileMetaIndex : public metaIndex
{
private:
void * const d;
std::string DebFile;
debDebPkgFileIndex *DebIndex;
public:
virtual std::string ArchiveURI(std::string const& /*File*/) const APT_OVERRIDE {
return DebFile;
}
virtual bool GetIndexes(pkgAcquire* /*Owner*/, const bool& /*GetAll=false*/) APT_OVERRIDE {
return true;
}
virtual std::vector<IndexTarget> GetIndexTargets() const APT_OVERRIDE {
return std::vector<IndexTarget>();
}
virtual std::vector<pkgIndexFile *> *GetIndexFiles() APT_OVERRIDE {
return Indexes;
}
virtual bool IsTrusted() const APT_OVERRIDE {
return true;
}
virtual bool Load(std::string const &, std::string * const ErrorText) APT_OVERRIDE
{
LoadedSuccessfully = TRI_NO;
if (ErrorText != NULL)
strprintf(*ErrorText, "Unparseable metaindex as it represents the standalone deb file %s", DebFile.c_str());
return false;
}
virtual metaIndex * UnloadedClone() const APT_OVERRIDE
{
return NULL;
}
debDebFileMetaIndex(std::string const &DebFile);
virtual ~debDebFileMetaIndex();

};

#endif

+ 1
- 2
apt-pkg/edsp/edspsystem.cc View File

@@ -91,8 +91,7 @@ signed edspSystem::Score(Configuration const &Cnf)
return -1000;
}
/*}}}*/
// System::AddStatusFiles - Register the status files /*{{{*/
bool edspSystem::AddStatusFiles(std::vector<pkgIndexFile *> &List)
bool edspSystem::AddStatusFiles(std::vector<pkgIndexFile *> &List) /*{{{*/
{
if (StatusFile == 0)
{


+ 188
- 213
apt-pkg/pkgcachegen.cc View File

@@ -36,6 +36,8 @@
#include <iostream>
#include <string>
#include <vector>
#include <memory>
#include <algorithm>
#include <sys/stat.h>
#include <unistd.h>

@@ -1332,10 +1334,10 @@ map_stringitem_t pkgCacheGenerator::StoreString(enum StringType const type, cons
/* This just verifies that each file in the list of index files exists,
has matching attributes with the cache and the cache does not have
any extra files. */
static bool CheckValidity(const string &CacheFile,
static bool CheckValidity(const string &CacheFile,
pkgSourceList &List,
FileIterator Start,
FileIterator End,
FileIterator const Start,
FileIterator const End,
MMap **OutMap = 0)
{
bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
@@ -1343,7 +1345,7 @@ static bool CheckValidity(const string &CacheFile,
if (CacheFile.empty() == true || FileExists(CacheFile) == false)
{
if (Debug == true)
std::clog << "CacheFile doesn't exist" << std::endl;
std::clog << "CacheFile " << CacheFile << " doesn't exist" << std::endl;
return false;
}

@@ -1361,7 +1363,7 @@ static bool CheckValidity(const string &CacheFile,
if (_error->PendingError() == true || Map->Size() == 0)
{
if (Debug == true)
std::clog << "Errors are pending or Map is empty()" << std::endl;
std::clog << "Errors are pending or Map is empty() for " << CacheFile << std::endl;
_error->Discard();
return false;
}
@@ -1385,10 +1387,9 @@ static bool CheckValidity(const string &CacheFile,
if (Debug == true)
std::clog << "with ID " << RlsFile->ID << " is valid" << std::endl;

std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j)
if ((*j)->HasPackages())
Files.push_back (*j);
std::vector <pkgIndexFile *> const * const Indexes = (*i)->GetIndexFiles();
std::copy_if(Indexes->begin(), Indexes->end(), std::back_inserter(Files),
[](pkgIndexFile const * const I) { return I->HasPackages(); });
}
for (unsigned I = 0; I != Cache.HeaderP->ReleaseFileCount; ++I)
if (RlsVisited[I] == false)
@@ -1398,8 +1399,7 @@ static bool CheckValidity(const string &CacheFile,
return false;
}

for (; Start != End; ++Start)
Files.push_back(*Start);
std::copy(Start, End, std::back_inserter(Files));

/* Now we check every index file, see if it is in the cache,
verify the IMS data and check that it is on the disk too.. */
@@ -1482,16 +1482,41 @@ static map_filesize_t ComputeSize(pkgSourceList const * const List, FileIterator
}
/*}}}*/
// BuildCache - Merge the list of index files into the cache /*{{{*/
// ---------------------------------------------------------------------
/* */
static bool BuildCache(pkgCacheGenerator &Gen,
OpProgress *Progress,
OpProgress * const Progress,
map_filesize_t &CurrentSize,map_filesize_t TotalSize,
pkgSourceList const * const List,
FileIterator Start, FileIterator End)
FileIterator const Start, FileIterator const End)
{
std::vector<pkgIndexFile *> Files;
bool const HasFileDeps = Gen.HasFileDeps();
bool mergeFailure = false;

auto const indexFileMerge = [&](pkgIndexFile * const I) {
if (HasFileDeps)
Files.push_back(I);

if (I->HasPackages() == false || mergeFailure)
return;

if (I->Exists() == false)
return;

if (I->FindInCache(Gen.GetCache()).end() == false)
{
_error->Warning("Duplicate sources.list entry %s",
I->Describe().c_str());
return;
}

map_filesize_t const Size = I->Size();
if (Progress != NULL)
Progress->OverallProgress(CurrentSize, TotalSize, Size, _("Reading package lists"));
CurrentSize += Size;

if (I->Merge(Gen,Progress) == false)
mergeFailure = true;
};

if (List != NULL)
{
@@ -1508,63 +1533,20 @@ static bool BuildCache(pkgCacheGenerator &Gen,
return false;

std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
for (std::vector<pkgIndexFile *>::const_iterator I = Indexes->begin(); I != Indexes->end(); ++I)
{
if (HasFileDeps)
Files.push_back(*I);

if ((*I)->HasPackages() == false)
continue;

if ((*I)->Exists() == false)
continue;

if ((*I)->FindInCache(Gen.GetCache()).end() == false)
{
_error->Warning("Duplicate sources.list entry %s",
(*I)->Describe().c_str());
continue;
}

map_filesize_t Size = (*I)->Size();
if (Progress != NULL)
Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
CurrentSize += Size;

if ((*I)->Merge(Gen,Progress) == false)
return false;
}
if (Indexes != NULL)
std::for_each(Indexes->begin(), Indexes->end(), indexFileMerge);
if (mergeFailure)
return false;
}
}

Gen.SelectReleaseFile("", "");
FileIterator I;
for (I = Start; I != End; ++I)
if (Start != End)
{
if (HasFileDeps)
Files.push_back(*I);

if ((*I)->HasPackages() == false)
continue;
if ((*I)->Exists() == false)
continue;

if ((*I)->FindInCache(Gen.GetCache()).end() == false)
{
_error->Warning("Duplicate sources.list entry %s",
(*I)->Describe().c_str());
continue;
}
map_filesize_t Size = (*I)->Size();
if (Progress != NULL)
Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
CurrentSize += Size;
if ((*I)->Merge(Gen,Progress) == false)
Gen.SelectReleaseFile("", "");
std::for_each(Start, End, indexFileMerge);
if (mergeFailure)
return false;
}
}

if (HasFileDeps == true)
{
@@ -1582,12 +1564,20 @@ static bool BuildCache(pkgCacheGenerator &Gen,
return false;
}
}
return true;
}
/*}}}*/
// CacheGenerator::CreateDynamicMMap - load an mmap with configuration options /*{{{*/
DynamicMMap* pkgCacheGenerator::CreateDynamicMMap(FileFd *CacheF, unsigned long Flags) {
// CacheGenerator::MakeStatusCache - Construct the status cache /*{{{*/
// ---------------------------------------------------------------------
/* This makes sure that the status cache (the cache that has all
index files from the sources list and all local ones) is ready
to be mmaped. If OutMap is not zero then a MMap object representing
the cache will be stored there. This is pretty much mandetory if you
are using AllowMem. AllowMem lets the function be run as non-root
where it builds the cache 'fast' into a memory buffer. */
static DynamicMMap* CreateDynamicMMap(FileFd * const CacheF, unsigned long Flags)
{
map_filesize_t const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
map_filesize_t const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
map_filesize_t const MapLimit = _config->FindI("APT::Cache-Limit", 0);
@@ -1599,15 +1589,42 @@ DynamicMMap* pkgCacheGenerator::CreateDynamicMMap(FileFd *CacheF, unsigned long
else
return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
}
/*}}}*/
// CacheGenerator::MakeStatusCache - Construct the status cache /*{{{*/
// ---------------------------------------------------------------------
/* This makes sure that the status cache (the cache that has all
index files from the sources list and all local ones) is ready
to be mmaped. If OutMap is not zero then a MMap object representing
the cache will be stored there. This is pretty much mandetory if you
are using AllowMem. AllowMem lets the function be run as non-root
where it builds the cache 'fast' into a memory buffer. */
static bool writeBackMMapToFile(pkgCacheGenerator * const Gen, DynamicMMap * const Map,
std::string const &FileName)
{
FileFd SCacheF(FileName, FileFd::WriteAtomic);
if (_error->PendingError() == true)
return false;

fchmod(SCacheF.Fd(),0644);

// Write out the main data
if (SCacheF.Write(Map->Data(),Map->Size()) == false)
return _error->Error(_("IO Error saving source cache"));
SCacheF.Sync();

// Write out the proper header
Gen->GetCache().HeaderP->Dirty = false;
if (SCacheF.Seek(0) == false ||
SCacheF.Write(Map->Data(),sizeof(*Gen->GetCache().HeaderP)) == false)
return _error->Error(_("IO Error saving source cache"));
Gen->GetCache().HeaderP->Dirty = true;
SCacheF.Sync();
return true;
}
static bool loadBackMMapFromFile(std::unique_ptr<pkgCacheGenerator> &Gen,
SPtr<DynamicMMap> &Map, OpProgress * const Progress, std::string const &FileName)
{
Map = CreateDynamicMMap(NULL, 0);
FileFd CacheF(FileName, FileFd::ReadOnly);
map_pointer_t const alloc = Map->RawAllocate(CacheF.Size());
if ((alloc == 0 && _error->PendingError())
|| CacheF.Read((unsigned char *)Map->Data() + alloc,
CacheF.Size()) == false)
return false;
Gen.reset(new pkgCacheGenerator(Map.Get(),Progress));
return true;
}
APT_DEPRECATED bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
MMap **OutMap, bool AllowMem)
{ return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
@@ -1617,18 +1634,6 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress
bool const Debug = _config->FindB("Debug::pkgCacheGen", false);

std::vector<pkgIndexFile *> Files;
/*
for (std::vector<metaIndex *>::const_iterator i = List.begin();
i != List.end();
++i)
{
std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin();
j != Indexes->end();
++j)
Files.push_back (*j);
}
*/
if (_system->AddStatusFiles(Files) == false)
return false;

@@ -1649,160 +1654,130 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress
CreateDirectory(dir, flNotFile(SrcCacheFile));
}

// Decide if we can write to the cache
bool Writeable = false;
if (CacheFile.empty() == false)
Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
else
if (SrcCacheFile.empty() == false)
Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
if (Debug == true)
std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;

if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());

if (Progress != NULL)
Progress->OverallProgress(0,1,1,_("Reading package lists"));

// Cache is OK, Fin.
if (CheckValidity(CacheFile, List, Files.begin(),Files.end(),OutMap) == true)
bool pkgcache_fine = false;
bool srcpkgcache_fine = false;
bool volatile_fine = List.GetVolatileFiles().empty();

if (CheckValidity(CacheFile, List, Files.begin(), Files.end(), volatile_fine ? OutMap : NULL) == true)
{
if (Progress != NULL)
Progress->OverallProgress(1,1,1,_("Reading package lists"));
if (Debug == true)
std::clog << "pkgcache.bin is valid - no need to build anything" << std::endl;
return true;
std::clog << "pkgcache.bin is valid - no need to build any cache" << std::endl;
pkgcache_fine = true;
srcpkgcache_fine = true;
}
else if (Debug == true)
std::clog << "pkgcache.bin is NOT valid" << std::endl;
/* At this point we know we need to reconstruct the package cache,
begin. */
SPtr<FileFd> CacheF;
SPtr<DynamicMMap> Map;
if (Writeable == true && CacheFile.empty() == false)
if (pkgcache_fine == false)
{
_error->PushToStack();
unlink(CacheFile.c_str());
CacheF = new FileFd(CacheFile,FileFd::WriteAtomic);
fchmod(CacheF->Fd(),0644);
Map = CreateDynamicMMap(CacheF, MMap::Public);
if (_error->PendingError() == true)
if (CheckValidity(SrcCacheFile, List, Files.end(), Files.end()) == true)
{
delete CacheF.UnGuard();
delete Map.UnGuard();
if (Debug == true)
std::clog << "Open filebased MMap FAILED" << std::endl;
Writeable = false;
if (AllowMem == false)
{
_error->MergeWithStack();
return false;
}
_error->RevertToStack();
}
else
{
_error->MergeWithStack();
if (Debug == true)
std::clog << "Open filebased MMap" << std::endl;
std::clog << "srcpkgcache.bin is valid - it can be reused" << std::endl;
srcpkgcache_fine = true;
}
}
if (Writeable == false || CacheFile.empty() == true)

if (volatile_fine == true && srcpkgcache_fine == true && pkgcache_fine == true)
{
// Just build it in memory..
Map = CreateDynamicMMap(NULL);
if (Debug == true)
std::clog << "Open memory Map (not filebased)" << std::endl;
if (Progress != NULL)
Progress->OverallProgress(1,1,1,_("Reading package lists"));
return true;
}
// Lets try the source cache.
map_filesize_t CurrentSize = 0;
map_filesize_t TotalSize = 0;
if (CheckValidity(SrcCacheFile, List, Files.end(),
Files.end()) == true)

bool Writeable = false;
if (srcpkgcache_fine == false || pkgcache_fine == false)
{
if (CacheFile.empty() == false)
Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
else if (SrcCacheFile.empty() == false)
Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;

if (Debug == true)
std::clog << "srcpkgcache.bin is valid - populate MMap with it." << std::endl;
// Preload the map with the source cache
FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
map_pointer_t const alloc = Map->RawAllocate(SCacheF.Size());
if ((alloc == 0 && _error->PendingError())
|| SCacheF.Read((unsigned char *)Map->Data() + alloc,
SCacheF.Size()) == false)
return false;
std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;

if (Writeable == false && AllowMem == false)
{
if (CacheFile.empty() == false)
return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
else if (SrcCacheFile.empty() == false)
return _error->Error(_("Unable to write to %s"),flNotFile(SrcCacheFile).c_str());
else
return _error->Error("Unable to create caches as file usage is disabled, but memory not allowed either!");
}
}

TotalSize = ComputeSize(NULL, Files.begin(), Files.end());
// At this point we know we need to construct something, so get storage ready
SPtr<DynamicMMap> Map = CreateDynamicMMap(NULL, 0);
if (Debug == true)
std::clog << "Open memory Map (not filebased)" << std::endl;

// Build the status cache
pkgCacheGenerator Gen(Map.Get(),Progress);
if (_error->PendingError() == true)
return false;
if (BuildCache(Gen, Progress, CurrentSize, TotalSize, NULL,
Files.begin(),Files.end()) == false)
std::unique_ptr<pkgCacheGenerator> Gen{nullptr};
map_filesize_t CurrentSize = 0;
std::vector<pkgIndexFile*> VolatileFiles = List.GetVolatileFiles();
map_filesize_t TotalSize = ComputeSize(NULL, VolatileFiles.begin(), VolatileFiles.end());
if (srcpkgcache_fine == true && pkgcache_fine == false)
{
if (Debug == true)
std::clog << "srcpkgcache.bin was valid - populate MMap with it" << std::endl;
if (loadBackMMapFromFile(Gen, Map, Progress, SrcCacheFile) == false)
return false;
srcpkgcache_fine = true;
TotalSize += ComputeSize(NULL, Files.begin(), Files.end());
}
else
else if (srcpkgcache_fine == false)
{
if (Debug == true)
std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
TotalSize = ComputeSize(&List, Files.begin(),Files.end());
// Build the source cache
pkgCacheGenerator Gen(Map.Get(),Progress);
if (_error->PendingError() == true)
return false;
if (BuildCache(Gen, Progress, CurrentSize, TotalSize, &List,
Files.end(),Files.end()) == false)
Gen.reset(new pkgCacheGenerator(Map.Get(),Progress));

TotalSize += ComputeSize(&List, Files.begin(),Files.end());
if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, &List,
Files.end(),Files.end()) == false)
return false;
// Write it back

if (Writeable == true && SrcCacheFile.empty() == false)
{
FileFd SCacheF(SrcCacheFile,FileFd::WriteAtomic);
if (_error->PendingError() == true)
if (writeBackMMapToFile(Gen.get(), Map.Get(), SrcCacheFile) == false)
return false;
fchmod(SCacheF.Fd(),0644);
// Write out the main data
if (SCacheF.Write(Map->Data(),Map->Size()) == false)
return _error->Error(_("IO Error saving source cache"));
SCacheF.Sync();
// Write out the proper header
Gen.GetCache().HeaderP->Dirty = false;
if (SCacheF.Seek(0) == false ||
SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
return _error->Error(_("IO Error saving source cache"));
Gen.GetCache().HeaderP->Dirty = true;
SCacheF.Sync();
}
// Build the status cache
if (BuildCache(Gen, Progress, CurrentSize, TotalSize, NULL,
Files.begin(), Files.end()) == false)
}

if (pkgcache_fine == false)
{
if (Debug == true)
std::clog << "Building status cache in pkgcache.bin now" << std::endl;
if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
Files.begin(), Files.end()) == false)
return false;

if (Writeable == true && CacheFile.empty() == false)
if (writeBackMMapToFile(Gen.get(), Map.Get(), CacheFile) == false)
return false;
}

if (Debug == true)
std::clog << "Caches are ready for shipping" << std::endl;
std::clog << "Caches done. Now bring in the volatile files (if any)" << std::endl;

if (_error->PendingError() == true)
return false;
if (OutMap != 0)
if (volatile_fine == false)
{
if (CacheF != 0)
if (Gen == nullptr)
{
delete Map.UnGuard();
*OutMap = new MMap(*CacheF,0);
if (Debug == true)
std::clog << "Populate new MMap with cachefile contents" << std::endl;
if (loadBackMMapFromFile(Gen, Map, Progress, CacheFile) == false)
return false;
}
else
{
*OutMap = Map.UnGuard();
}

Files = List.GetVolatileFiles();
if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
Files.begin(), Files.end()) == false)
return false;
}

if (OutMap != nullptr)
*OutMap = Map.UnGuard();

if (Debug == true)
std::clog << "Everything is ready for shipping" << std::endl;
return true;
}
/*}}}*/
@@ -1817,7 +1792,7 @@ bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **O
if (_system->AddStatusFiles(Files) == false)
return false;

SPtr<DynamicMMap> Map = CreateDynamicMMap(NULL);
SPtr<DynamicMMap> Map = CreateDynamicMMap(NULL, 0);
map_filesize_t CurrentSize = 0;
map_filesize_t TotalSize = 0;


+ 0
- 1
apt-pkg/pkgcachegen.h View File

@@ -116,7 +116,6 @@ class APT_HIDDEN pkgCacheGenerator /*{{{*/
APT_PUBLIC static bool MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
MMap **OutMap = 0,bool AllowMem = false);
APT_PUBLIC static bool MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap);
APT_PUBLIC static DynamicMMap* CreateDynamicMMap(FileFd *CacheF, unsigned long Flags = 0);

void ReMap(void const * const oldMap, void const * const newMap);



+ 3
- 2
apt-pkg/pkgsystem.h View File

@@ -52,7 +52,7 @@ class Configuration;
class pkgIndexFile;

class pkgSystem
{
{
public:

// Global list of supported systems
@@ -81,7 +81,8 @@ class pkgSystem
virtual bool ArchiveSupported(const char *Type) = 0;

// Return a list of system index files..
virtual bool AddStatusFiles(std::vector<pkgIndexFile *> &List) = 0;
virtual bool AddStatusFiles(std::vector<pkgIndexFile *> &List) = 0;

virtual bool FindIndex(pkgCache::PkgFileIterator File,
pkgIndexFile *&Found) const = 0;



+ 25
- 3
apt-pkg/sourcelist.cc View File

@@ -274,6 +274,10 @@ pkgSourceList::~pkgSourceList()
{
for (const_iterator I = SrcList.begin(); I != SrcList.end(); ++I)
delete *I;
SrcList.clear();
for (pkgIndexFile * const File : VolatileFiles)
delete File;
VolatileFiles.clear();
}
/*}}}*/
// SourceList::ReadMainList - Read the main source list from etc /*{{{*/
@@ -339,7 +343,7 @@ bool pkgSourceList::ReadAppend(string const &File)
else
return ParseFileOldStyle(File);
}
/*}}}*/
// SourceList::ReadFileOldStyle - Read Traditional style sources.list /*{{{*/
// ---------------------------------------------------------------------
/* */
@@ -454,7 +458,15 @@ bool pkgSourceList::FindIndex(pkgCache::PkgFileIterator File,
}
}
}

for (vector<pkgIndexFile *>::const_iterator J = VolatileFiles.begin();
J != VolatileFiles.end(); ++J)
{
if ((*J)->FindInCache(*File.Cache()) == File)
{
Found = (*J);
return true;
}
}
return false;
}
/*}}}*/
@@ -511,4 +523,14 @@ time_t pkgSourceList::GetLastModifiedTime()
return mtime_sources;
}
/*}}}*/

std::vector<pkgIndexFile*> pkgSourceList::GetVolatileFiles() const /*{{{*/
{
return VolatileFiles;
}
/*}}}*/
void pkgSourceList::AddVolatileFile(pkgIndexFile * const File) /*{{{*/
{
if (File != NULL)
VolatileFiles.push_back(File);
}
/*}}}*/

+ 19
- 0
apt-pkg/sourcelist.h View File

@@ -50,6 +50,7 @@ class metaIndex;
class pkgSourceList
{
void * const d;
std::vector<pkgIndexFile*> VolatileFiles;
public:

// List of supported source list types
@@ -113,6 +114,24 @@ class pkgSourceList
// query last-modified time
time_t GetLastModifiedTime();

/** \brief add file for parsing, but not to the cache
*
* pkgIndexFiles origining from pkgSourcesList are included in
* srcpkgcache, the status files added via #AddStatusFiles are
* included in pkgcache, but these files here are not included in
* any cache to have the possibility of having a file included just
* for a single run like a local .deb/.dsc file.
*
* The volatile files do not count as "normal" sourceslist entries,
* can't be iterated over with #begin and #end and can't be
* downloaded, but they can be found via #FindIndex.
*
* @param File is an index file; pointer-ownership is transferred
*/
void AddVolatileFile(pkgIndexFile * const File);
/** @return list of files registered with #AddVolatileFile */
std::vector<pkgIndexFile*> GetVolatileFiles() const;

pkgSourceList();
virtual ~pkgSourceList();
};


+ 0
- 21
apt-private/private-cachefile.h View File

@@ -11,17 +11,6 @@

#include <apti18n.h>

// FIXME: we need to find a way to export this
class APT_PUBLIC SourceList : public pkgSourceList
{
public:
// Add custom metaIndex (e.g. local files)
void AddMetaIndex(metaIndex *mi) {
SrcList.push_back(mi);
}

};

// class CacheFile - Cover class for some dependency cache functions /*{{{*/
class APT_PUBLIC CacheFile : public pkgCacheFile
{
@@ -36,16 +25,6 @@ class APT_PUBLIC CacheFile : public pkgCacheFile
return false;
return true;
}
// FIXME: this can go once the "libapt-pkg" pkgSourceList has a way
// to add custom metaIndexes (or custom local files or so)
bool BuildSourceList(OpProgress */*Progress*/ = NULL) {
if (SrcList != NULL)
return true;
SrcList = new SourceList();
if (SrcList->ReadMainList() == false)
return _error->Error(_("The list of sources could not be read."));
return true;
}
bool Open(bool WithLock = true)
{
OpTextProgress Prog(*_config);


+ 3
- 18
apt-private/private-install.cc View File

@@ -21,6 +21,7 @@
#include <apt-pkg/pkgcache.h>
#include <apt-pkg/upgrade.h>
#include <apt-pkg/install-progress.h>
#include <apt-pkg/debindexfile.h>

#include <stdlib.h>
#include <string.h>
@@ -647,24 +648,8 @@ bool DoInstall(CommandLine &CmdL)
// first check for local pkgs and add them to the cache
for (const char **I = CmdL.FileList; *I != 0; I++)
{
if(FileExists(*I))
{
// FIXME: make this more elegant
std::string TypeStr = flExtension(*I) + "-file";
pkgSourceList::Type *Type = pkgSourceList::Type::GetType(TypeStr.c_str());
if(Type != 0)
{
std::vector<metaIndex *> List;
std::map<std::string, std::string> Options;
if(Type->CreateItem(List, *I, "", "", Options))
{
// we have our own CacheFile that gives us a SourceList
// with superpowerz
SourceList *sources = (SourceList*)Cache.GetSourceList();
sources->AddMetaIndex(List[0]);
}
}
}
if(FileExists(*I) && flExtension(*I) == "deb")
Cache.GetSourceList()->AddVolatileFile(new debDebPkgFileIndex(*I));
}

// then open the cache


+ 9
- 3
test/integration/framework View File

@@ -99,10 +99,14 @@ msgdone() {
}
getaptconfig() {
if [ -f ./aptconfig.conf ]; then
echo "./aptconfig.conf"
echo "$(readlink -f ./aptconfig.conf)"
elif [ -f ../aptconfig.conf ]; then
echo "../aptconfig.conf"
fi
echo "$(readlink -f ../aptconfig.conf)"
elif [ -f ../../aptconfig.conf ]; then
echo "$(readlink -f ../../aptconfig.conf)"
elif [ -f "${TMPWORKINGDIRECTORY}/aptconfig.conf" ]; then
echo "$(readlink -f "${TMPWORKINGDIRECTORY}/aptconfig.conf")"
fi
}
runapt() {
msgdebug "Executing: ${CCMD}$*${CDEBUG} "
@@ -141,6 +145,8 @@ gdb() {
case "$1" in
aptget) CMD="apt-get";;
aptcache) CMD="apt-cache";;
aptcdrom) CMD="apt-cdrom";;
aptconfig) CMD="apt-config";;
aptmark) CMD="apt-mark";;
apthelper) CMD="apt-helper";;
aptftparchive) CMD="apt-ftparchive";;


+ 40
- 5
test/integration/test-apt-get-install-deb View File

@@ -5,15 +5,21 @@ TESTDIR=$(readlink -f $(dirname $0))
. $TESTDIR/framework

setupenvironment
configarchitecture "i386"
configarchitecture 'amd64' 'i386'

# regression test for #754904
testfailureequal 'E: Unable to locate package /dev/null' aptget install -qq /dev/null

# and ensure we fail for invalid debs
cat > foo.deb <<EOF
# only consider .deb files
cat > foo.rpm <<EOF
I'm not a deb, I'm a teapot.
EOF
testfailureequal "E: Unable to locate package ./foo.rpm
E: Couldn't find any package by glob './foo.rpm'
E: Couldn't find any package by regex './foo.rpm'" aptget install -qq ./foo.rpm

# and ensure we fail for invalid debs
mv foo.rpm foo.deb
testfailure aptget install ./foo.deb
testsuccess grep '^E: Sub-process Popen returned an error code' rootdir/tmp/testfailure.output
testequal 'E: Encountered a section with no Package: header
@@ -21,7 +27,36 @@ E: Problem with MergeLister for ./foo.deb
E: The package lists or status file could not be parsed or opened.' tail -n 3 rootdir/tmp/testfailure.output

# fakeroot is currently not found, framwork needs updating
buildsimplenativepackage 'foo' 'all' '1.0'
buildsimplenativepackage 'foo' 'i386,amd64' '1.0'

testfailureequal "Reading package lists...
Building dependency tree...
Note, selecting 'foo:i386' instead of './incoming/foo_1.0_i386.deb'
Note, selecting 'foo' instead of './incoming/foo_1.0_amd64.deb'
Some packages could not be installed. This may mean that you have
requested an impossible situation or if you are using the unstable
distribution that some required packages have not yet been created
or been moved out of Incoming.
The following information may help to resolve the situation:

The following packages have unmet dependencies:
foo:i386 : Conflicts: foo but 1.0 is to be installed
foo : Conflicts: foo:i386 but 1.0 is to be installed
E: Unable to correct problems, you have held broken packages." aptget install ./incoming/foo_1.0_i386.deb ./incoming/foo_1.0_amd64.deb -s -q=0

testdpkgnotinstalled 'foo'
testsuccess aptget install ./incoming/foo_1.0_all.deb
testsuccess aptget install ./incoming/foo_1.0_i386.deb
testdpkginstalled 'foo'

testsuccessequal "Reading package lists...
Building dependency tree...
Reading state information...
Note, selecting 'foo' instead of './incoming/foo_1.0_amd64.deb'
The following packages will be REMOVED:
foo:i386
The following NEW packages will be installed:
foo
0 upgraded, 1 newly installed, 1 to remove and 0 not upgraded.
Remv foo:i386 [1.0]
Inst foo (1.0 now [amd64])
Conf foo (1.0 now [amd64])" aptget install ./incoming/foo_1.0_amd64.deb -s -q=0

Loading…
Cancel
Save