Browse Source

integrate Anthonys rred with POC for client-side merge

Providing the benefits of both without the downsides :)
(ABI breaks or external dependencies)

For this Anthonys rred is equipped with:
- magic-filename-pickup of patches rather than explicit messages
- use of FileFd instead of FILE* to get on-the-fly uncompress
  of the gzip compressed pdiff patches

The acquire code in turn stops checking for apt-file's helper
as our own rred is now clever enough for our needs.
tags/debian/0.9.14.3.exp2
David Kalnischkies 7 years ago
parent
commit
50bd6fd379
3 changed files with 45 additions and 83 deletions
  1. +3
    -8
      apt-pkg/acquire-item.cc
  2. +26
    -68
      methods/rred.cc
  3. +16
    -7
      test/integration/test-pdiff-usage

+ 3
- 8
apt-pkg/acquire-item.cc View File

@@ -513,14 +513,9 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) /*{{{*/
bool pdiff_merge = _config->FindB("Acquire::PDiffs::Merge", true);
if (pdiff_merge == true)
{
// this perl script is provided by apt-file
pdiff_merge = FileExists(_config->FindFile("Dir::Bin::rred", "/usr/bin/diffindex-rred"));
if (pdiff_merge == true)
{
// reprepro adds this flag if it has merged patches on the server
std::string const precedence = Tags.FindS("X-Patch-Precedence");
pdiff_merge = (precedence != "merged");
}
// reprepro adds this flag if it has merged patches on the server
std::string const precedence = Tags.FindS("X-Patch-Precedence");
pdiff_merge = (precedence != "merged");
}

if (pdiff_merge == false)


+ 26
- 68
methods/rred.cc View File

@@ -407,13 +407,13 @@ class Patch {

public:

void read_diff(FILE *f)
void read_diff(FileFd &f)
{
char buffer[BLOCK_SIZE];
bool cmdwanted = true;

Change ch(0);
while(fgets(buffer, sizeof(buffer), f))
while(f.ReadLine(buffer, sizeof(buffer)))
{
if (cmdwanted) {
char *m, *c;
@@ -534,66 +534,11 @@ class Patch {
}
};

bool LookupPatches(const std::string &Message, std::vector<std::string> &lines)
{
const char *Tag = "Patches";
const size_t Length = strlen(Tag);

std::string::const_iterator I, J;

std::clog << "Looking for \"Patches:\" section in message:\n\n" << Message << "\n\n";
std::clog.flush();

for (I = Message.begin(); I + Length < Message.end(); ++I)
{
if (I[Length] == ':' && stringcasecmp(I, I+Length, Tag) == 0)
{
// found the tag, now read the patches
for(;;) {
for (; I < Message.end() && *I != '\n'; ++I);
if (I < Message.end()) I++;
if (I == Message.end() || *I != ' ')
break;
while (I < Message.end() && isspace(*I)) I++;
for (J = I; J < Message.end() && *J != '\n'; ++J)
;
do
J--;
while (I < J && isspace(*J));
if (I < J)
lines.push_back(std::string(I,++J));
else
break;
I = J;
}
std::clog << "Found " << lines.size() << " patches!\n";
std::clog.flush();
return true;
}
}
std::clog << "Found no patches! :(\n";
std::clog.flush();
return false;
}


class RredMethod : public pkgAcqMethod {
private:
bool Debug;
std::vector<std::string> patchpaths;

protected:
virtual bool HandleMessage(int Number, std::string Message) {
if (Number == 600)
{
patchpaths.clear();
LookupPatches(Message, patchpaths);
std::clog << "Ended up with " << patchpaths.size() << " patches!\n";
std::clog.flush();
}
return pkgAcqMethod::HandleMessage(Number, Message);
}

virtual bool Fetch(FetchItem *Itm) {
Debug = _config->FindB("Debug::pkgAcquire::RRed", false);
URI Get = Itm->Uri;
@@ -601,17 +546,29 @@ class RredMethod : public pkgAcqMethod {

FetchResult Res;
Res.Filename = Itm->DestFile;
if (Itm->Uri.empty()) {
if (Itm->Uri.empty())
{
Path = Itm->DestFile;
Itm->DestFile.append(".result");
} else
URIStart(Res);

std::vector<std::string> patchpaths;
Patch patch;

if (patchpaths.empty())
{
if (FileExists(Path + ".ed") == true)
patchpaths.push_back(Path + ".ed");
else
{
_error->PushToStack();
std::vector<std::string> patches = GetListOfFilesInDir(flNotFile(Path), "gz", true, false);
_error->RevertToStack();

std::string const baseName = Path + ".ed.";
for (std::vector<std::string>::const_iterator p = patches.begin();
p != patches.end(); ++p)
if (p->compare(0, baseName.length(), baseName) == 0)
patchpaths.push_back(*p);
}

std::string patch_name;
@@ -624,13 +581,15 @@ class RredMethod : public pkgAcqMethod {
std::clog << "Patching " << Path << " with " << patch_name
<< std::endl;

FILE *p = fopen(patch_name.c_str(), "r");
if (p == NULL) {
std::clog << "Could not open patch file " << patch_name << std::endl;
FileFd p;
// all patches are compressed, even if the name doesn't reflect it
if (p.Open(patch_name, FileFd::ReadOnly, FileFd::Gzip) == false) {
std::cerr << "Could not open patch file " << patch_name << std::endl;
_error->DumpErrors(std::cerr);
abort();
}
patch.read_diff(p);
fclose(p);
p.Close();
}

if (Debug == true)
@@ -697,10 +656,9 @@ int main(int argc, char **argv)
}

for (; i < argc; i++) {
FILE *p;
p = fopen(argv[i], "r");
if (!p) {
perror(argv[i]);
FileFd p;
if (p.Open(argv[i], FileFd::ReadOnly) == false) {
_error->DumpErrors(std::cerr);
exit(1);
}
patch.read_diff(p);


+ 16
- 7
test/integration/test-pdiff-usage View File

@@ -20,10 +20,19 @@ chmod +x extrred
echo 'Dir::Bin::rred "./extrred";' > rootdir/etc/apt/apt.conf.d/99rred

wasmergeused() {
testsuccess aptget update "$@"
msgtest 'Test for successful execution of' "$*"
local OUTPUT=$(mktemp)
addtrap "rm $OUTPUT;"
if aptget update "$@" >${OUTPUT} 2>&1; then
msgpass
else
echo
cat $OUTPUT
msgfail
fi

msgtest 'Check if the right pdiff merger was used'
if [ -e ./merge-was-used ]; then
rm -f ./merge-was-used
if grep -q '^pkgAcqIndexMergeDiffs::Done(): rred' $OUTPUT; then
if echo "$*" | grep -q -- '-o Acquire::PDiffs::Merge=1'; then
msgpass
else
@@ -50,7 +59,7 @@ testrun() {
testequal "$(cat ${PKGFILE})
" aptcache show apt oldstuff

# apply with one patch
msgmsg 'Testcase: apply with one patch'
cp ${PKGFILE}-new aptarchive/Packages
compressfile 'aptarchive/Packages'
mkdir -p aptarchive/Packages.diff
@@ -73,13 +82,13 @@ SHA1-Patches:
testequal "$(cat ${PKGFILE}-new)
" aptcache show apt newstuff

# index is already up-to-date
msgmsg 'Testcase: index is already up-to-date'
find rootdir/var/lib/apt/lists -name '*.IndexDiff' -type f -delete
testsuccess aptget update "$@"
testequal "$(cat ${PKGFILE}-new)
" aptcache show apt newstuff

# apply with two patches
msgmsg 'Testcase: apply with two patches'
cp ${PKGFILE}-new aptarchive/Packages
echo '
Package: futurestuff
@@ -120,7 +129,7 @@ SHA1-Patches:
testequal "$(cat Packages-future)
" aptcache show apt newstuff futurestuff

# patch applying fails, but successful fallback
msgmsg 'Testcase: patch applying fails, but successful fallback'
rm -rf rootdir/var/lib/apt/lists
cp -a rootdir/var/lib/apt/lists-bak rootdir/var/lib/apt/lists
cp ${PKGFILE}-new aptarchive/Packages


Loading…
Cancel
Save