Browse Source

Rewrite prepop.pl/popcon-process.sh to handle encrypted reports

prepop.pl now extract encrypted report to files in popcon-gpg.
Then popcon-process.sh decrypt them, concatenated them, and
reprocess the result with prepop.pl.
This round-about way allows to decrypt report in parallel
safely. (parallelism is not implemented yet, p.d.o being single core).
suites/ascii
Bill Allombert 10 years ago
parent
commit
7108082189
  1. 2
      debian/changelog
  2. 6
      examples/bin/popcon-process.sh
  3. 139
      examples/bin/prepop.pl

2
debian/changelog

@ -21,6 +21,8 @@ popularity-contest (1.58) UNRELEASED; urgency=low
subsequent release.
+ examples/cgi-bin/popcon.cgi:
- Accept encrypted report.
+ examples/bin/prepop.pl, examples/bin/popcon-process.sh
- Add support for decrypting encrypted report
-- Bill Allombert <ballombe@debian.org> Sun, 19 May 2013 21:31:39 +0200

6
examples/bin/popcon-process.sh

@ -26,7 +26,13 @@ if [ true = "$READMAIL" ] ; then
chmod go-rwx $MAILDIR/survey
# process entries, splitting them into individual reports
rm -fr popcon-gpg
mkdir popcon-gpg
$BINDIR/prepop.pl <new-popcon-entries >$LOGDIR/prepop.out 2>&1
#decrypt reports (to be parallelized)
find popcon-gpg -type f -name '*.gpg' -execdir gpg {} \;
#process decrypted reports
find popcon-gpg -type f -name '*.txt'| xargs cat | $BINDIR/prepop.pl
fi
# delete outdated entries

139
examples/bin/prepop.pl

@ -6,74 +6,91 @@
#
$dirname = 'popcon-entries';
$gpgdir = 'popcon-gpg';
$now = time;
$state='initial'; # one of ('initial','accept','reject')
$filenum=0;
my($file,$mtime);
while(<>)
sub get_report
{
$state eq 'initial' and do
{
/^POPULARITY-CONTEST-0/ or next;
my @line=split(/ +/);
my %field;
for (@line)
{
my ($key, $value) = split(':', $_, 2);
$field{$key}=$value;
};
$id=$field{'ID'};
if (!defined($id) || $id !~ /^([a-f0-9]{32})$/)
{
print STDERR "Bad hostid: $id\n";
$state='reject'; next;
}
$id=$1; #untaint $id
$mtime=$field{'TIME'};
if (!defined($mtime) || $mtime!~/^([0-9]+)$/)
{
print STDERR "Bad mtime $mtime\n";
$state='reject'; next;
}
$mtime=int $1; #untaint $mtime;
$mtime=$now if ($mtime > $now);
my $dir=substr($id,0,2);
unless (-d "$dirname/$dir") {
mkdir("$dirname/$dir",0755) or do {$state='reject';next;};
};
$file="$dirname/$dir/$id";
open REPORT, ">",$file or do {$state='reject';next;};
print REPORT $_;
$state='accept'; next;
};
$state eq 'reject' and do
my($id,$file,$mtime,$vers);
my $line=$_;
chomp $line;
my @line=split(/ +/, $line);
my %field;
for (@line)
{
my ($key, $value) = split(':', $_, 2);
$field{$key}=$value;
};
$id=$field{'ID'};
if (!defined($id) || $id !~ /^([a-f0-9]{32})$/)
{
print STDERR "Bad hostid: $id\n";
return 'reject';
}
$id=$1; #untaint $id
$vers=$field{'POPCONVER'};
if (defined($vers) && $vers =~ /^1\.56ubuntu1/)
{
print STDERR "Report rejected: $vers: $id\n";
return 'reject';
}
$mtime=$field{'TIME'};
if (!defined($mtime) || $mtime!~/^([0-9]+)$/)
{
print STDERR "Bad mtime $mtime\n";
return 'reject';
}
$mtime=int $1; #untaint $mtime;
$mtime=$now if ($mtime > $now);
my $dir=substr($id,0,2);
unless (-d "$dirname/$dir") {
mkdir("$dirname/$dir",0755) or return 'reject';
}
$file="$dirname/$dir/$id";
open REPORT, ">",$file or return 'reject';
print REPORT $_;
while(<>)
{
/^From/ and last;
print REPORT $_; #accept line.
/^END-POPULARITY-CONTEST-0/ and do
{
/^From/ or next;
$state='initial';next;
close REPORT;
utime $mtime, $mtime, $file;
return 'accept';
};
$state eq 'accept' and do
}
close REPORT;
unlink $file;
print STDERR "Bad report $file\n";
return 'reject';
}
sub get_gpg
{
my $file="$gpgdir/$filenum.txt.gpg";
$filenum++;
open REPORT, ">",$file or return 'reject';
print REPORT $_;
while(<>)
{
/^From/ and last;
print REPORT $_; #accept line.
/^-----END PGP MESSAGE-----/ and do
{
/^From/ and do
{
close REPORT;
unlink $file;
print STDERR "Bad report $file\n";
$state='initial';
next;
};
print REPORT $_; #accept line.
/^END-POPULARITY-CONTEST-0/ and do
{
close REPORT;
utime $mtime, $mtime, $file;
$state='initial';
next;
};
close REPORT;
return 'accept';
};
}
close REPORT;
unlink $file;
print STDERR "Bad report $file\n";
return 'reject';
}
if ($state eq 'accept')
while(<>)
{
close REPORT;
unlink $file; #Reject
print STDERR "Bad last report $file\n";
/^POPULARITY-CONTEST-0/ and get_report();
/^-----BEGIN PGP MESSAGE-----/ and get_gpg();
}

Loading…
Cancel
Save