Browse Source

tag popcon-1.70

-----BEGIN PGP SIGNATURE-----
 
 iQIzBAABCgAdFiEEQgKOpASi6dgKxFMUjw58K0Ui44cFAl6CXG8ACgkQjw58K0Ui
 44cRuxAAguPPu1SmwIhFvgNn51eLeUmaZp4OCW8N28zprCDJMJQjw+qXC7dKZmFf
 +gDAyTXnoxBWESzvqRyoRcMjyKfLxplxvENUJSp305UxCyHTNbIPK/RAsyOWxX32
 7+Ye/h8GVfg0aY+LxpqkltZPRT5gh6r7vSF1zCDWoI3LWM/pTvPJJuBoQmzP1QHH
 mLGJ5pfe7KTtoYMQYKKVTcNwgiCgS8QU6QwnLaiIXxbUat5yXGZHZFDaM1wWiXZq
 D5r6FFcwmOSF7vhLnTfVtby3l0tcKryZfahSuIZy6cX5s08Rnh4kZ6hgIJqG9DHd
 sJw0j3s65a9LqW+52spWEnTCVCbJDc6lBfwpH00iz3hTxrgi1Ig50txUgF0pmlKx
 R4J+pHDFqpgQ8F7IHKYWcl9jtw7fUFiwD6yfHC9yykBYlxKrKCVUP6iMUb0V1U4C
 fcBOaT1c1MVOR2FPBTd7PW2tO8i7n/XwN+0+iv97azLeH9AqCAw/ol80/Ncav/1s
 DUfCrPJtcmD/3IF+lP5zPSjA93W9oKgNME6+p4Gi2fnCMr6S5dfhUt6fAS9mEEpl
 2OPn4LaWylnDF7WikyojBfmFbv6IKcol/8s0HvYTNi1WDX1nyj67Pvzp5Mr5eSAY
 rTiE3zevfqklwoPBobQXFe2vgrNY1Fotmbo4PTdDWLVWst7+p5I=
 =evng
 -----END PGP SIGNATURE-----

Merge tag 'popcon-1.70' into suites/unstable

tag popcon-1.70
suites/unstable
Mark Hindley 2 years ago
parent
commit
d3f46db47f
  1. 27
      debian/changelog
  2. 1
      debian/compat
  3. 4
      debian/control
  4. 36
      debian/cron.daily
  5. 3
      default.conf
  6. 2
      examples/bin/popanal.py
  7. 279
      examples/bin/popanal3.py
  8. 2
      examples/bin/popcon.pl
  9. 56
      popularity-contest

27
debian/changelog

@ -1,3 +1,30 @@
popularity-contest (1.70) unstable; urgency=low
* debian-popcon.gpg: use new submission key
* debian/cron.daily:
- fix reporting logic to avoid double submissions. Closes: #930446
- store last successful http submission timestamp in
/var/lib/popularity-contest/lastsub.
- run 'popularity-contest --su-nobody' as root. This allows
popcon to read the configuration file and /proc/*/maps files.
Closes: #865730. Thanks Robert Luberda.
- rename /var/log/popularity-contest.new.gpg to
/var/log/popularity-contest.gpg
* debian/control:
- Updated Standards-Version from 4.4.0 to 4.5.0. No change needed.
- Build-Depends on debhelper-compat (=12)
* debian/compat: removed
* Update example server-side scripts to popcon.d.o version:
- popanal.py: bump stable version to 1.67
- popcon.pl: update URL from Alioth to Salsa
* popularity-contest:
- add private option --su-nobody to drop privileges after reading
the configuration file and /proc.
* examples/bin/popanal3.py:
- Python 3 version of popanal.py (experimental), will replace popanal.py
-- Bill Allombert <ballombe@debian.org> Mon, 30 Mar 2020 19:47:56 +0200
popularity-contest (1.69+devuan1) unstable; urgency=medium
* New Debian version 1.69.

1
debian/compat

@ -1 +0,0 @@
9

4
debian/control

@ -5,9 +5,9 @@ Maintainer: Devuan Developers <devuan-dev@lists.dyne.org>
Uploaders: Vincenzo (KatolaZ) Nicosia <katolaz@freaknet.org>,
Daniel Reurich <daniel@centurion.net.nz>,
Mark (LeePen) Hindley <mark@hindley.org.uk>
Build-Depends: debhelper (>=9)
Build-Depends: debhelper-compat (=12)
Homepage: https://popcon.debian.org/
Standards-Version: 4.4.0
Standards-Version: 4.5.0
Vcs-Browser: https://git.devuan.org/devuan-packages/popularity-contest
Vcs-Git: git://git.devuan.org/devuan-packages/popularity-contest.git
Origin: Devuan

36
debian/cron.daily

@ -76,22 +76,37 @@ if [ -n "$HTTP_PROXY" ]; then
export http_proxy="$HTTP_PROXY";
fi
POPCONVAR=/var/lib/popularity-contest
POPCONOLD=/var/log/popularity-contest
POPCONNEW=/var/log/popularity-contest.new
POPCON="$POPCONNEW"
last_sub()
{
if [ -r "$POPCONVAR/lastsub" ] ; then
cat "$POPCONVAR/lastsub"
else
date -r "$POPCONOLD" +%s
fi
}
set_sub()
{
test -d "$POPCONVAR" || mkdir -p "$POPCONVAR"
date +%s > "$POPCONVAR/lastsub"
}
# Only run on the given day, to spread the load on the server a bit
if [ "$DAY" ] && [ "$DAY" != "$(date +%w)" ] ; then
if [ "$MODE" != "--crond" ] || ( [ "$DAY" ] && [ "$DAY" != "$(date +%w)" ] ) ; then
# Ensure that popcon runs at least once in the last week
if [ -f "$POPCONOLD" ] ; then
now=$(date +%s)
lastrun=$(date -r $POPCONOLD +%s)
lastrun=$(last_sub)
if [ "$MODE" = "--crond" ]; then
# 6.5 days, in seconds
week=561600
else
# 7.5 days, in seconds
week=648000
else
# 6.5 days, in seconds
week=561600
fi
if [ "$(( $now - $lastrun ))" -le "$week" ]; then
exit 0
@ -104,11 +119,6 @@ cd /var/log
umask 022
savelog -c 7 popularity-contest >/dev/null
run_popcon()
{
runuser -s /bin/sh -c "/usr/sbin/popularity-contest" nobody
}
do_sendmail()
{
if [ -n "$MAILFROM" ]; then
@ -120,7 +130,7 @@ do_sendmail()
# generate the popularity contest data
run_popcon > $POPCON
/usr/sbin/popularity-contest --su-nobody > $POPCON
GPG=/usr/bin/gpg
@ -149,6 +159,7 @@ if [ "$SUBMITURLS" ] && [ "yes" = "$USEHTTP" ]; then
if setsid $TORIFY /usr/share/popularity-contest/popcon-upload \
-u $URL -f $POPCON 2>/dev/null ; then
SUBMITTED=yes
set_sub
else
logger -t popularity-contest "unable to submit report to $URL."
fi
@ -182,4 +193,7 @@ if [ "yes" != "$SUBMITTED" ] ; then
logger -t popularity-contest "unable to submit report."
else
mv $POPCONNEW $POPCONOLD
if [ -n "$POPCONGPG" ]; then
mv $POPCONNEW.gpg $POPCONOLD.gpg
fi
fi

3
default.conf

@ -23,9 +23,6 @@ ENCRYPT="maybe"
# They should not be changed for proper operation with
# popcon.debian.org.
#
#KEYRING="/usr/share/popularity-contest/debian-popcon.gpg"
#POPCONKEY="6672B9765BDF38A3"
KEYRING="/usr/share/popularity-contest/devuan-popcon.gpg"
POPCONKEY="983D40FCDA1FE63C"

2
examples/bin/popanal.py

@ -6,7 +6,7 @@
import sys, string, time, glob, lzma
mirrorbase = "/srv/mirrors/debian"
stable_version = "1.64"
stable_version = "1.67"
def ewrite(s):
sys.stderr.write("%s\n" % s)

279
examples/bin/popanal3.py

@ -0,0 +1,279 @@
#!/usr/bin/python3
#
# Read Debian popularity-contest submission data on stdin and produce
# some statistics about it.
#
import sys, string, time, glob, lzma
mirrorbase = "/srv/mirrors/debian"
stable_version = "1.67"
def ewrite(s):
sys.stderr.write("%s\n" % s)
class Vote:
yes = 0
old_unused = 0
too_recent = 0
empty_package = 0
def vote_for(vote, package, entry):
now = time.time()
if entry.atime == 0: # no atime: empty package
vote.empty_package = vote.empty_package + 1
elif now - entry.atime > 30 * 24*3600: # 30 days since last use: old
vote.old_unused = vote.old_unused + 1
elif now - entry.ctime < 30 * 24* 3600 \
and entry.atime - entry.ctime < 24*3600: # upgraded too recently
vote.too_recent = vote.too_recent + 1
else: # otherwise, vote for this package
vote.yes = vote.yes + 1
deplist = {}
provlist = {}
class Stat:
def __init__(self):
self.vote = {}
self.vendor = {}
self.release = {}
self.arch = {}
self.count = 0
def output(self,filename):
out = open(filename, 'w')
out.write("Submissions: %8d\n" % self.count)
releaselist = list(self.release.keys())
releaselist.sort()
for release in releaselist:
out.write("Release: %-30s %5d\n"
% (release, self.release[release]))
archlist = list(self.arch.keys())
archlist.sort()
for arch in archlist:
out.write("Architecture: %-30s %5d\n"
% (arch, self.arch[arch]))
vendorlist = list(self.vendor.keys())
vendorlist.sort()
for vendor in vendorlist:
out.write("Vendor: %-30s %5d\n"
% (vendor, self.vendor[vendor]))
pkglist = list(self.vote.keys())
pkglist.sort()
for package in pkglist:
fv = self.vote[package]
out.write("Package: %-30s %5d %5d %5d %5d\n"
% (package, fv.yes, fv.old_unused,
fv.too_recent, fv.empty_package))
out.close()
stat = Stat()
stat_stable = Stat()
def parse_depends(depline):
l = []
split = depline.split(',')
for d in split:
x = d.split()
if (x):
l.append(x[0])
return l
def read_depends(filename):
file = lzma.LZMAFile(filename, "r")
package = None
while 1:
try:
line = str(file.readline(),encoding='latin_1')
except:
line = False
if line:
if line[0]==' ' or line[0]=='\t': continue # continuation
split = line.split(':')
if not line or split[0]=='Package':
if package and (len(dep) > 0 or len(prov) > 0):
deplist[package] = dep
for d in prov:
if not (d in provlist):
provlist[d] = []
provlist[d].append(package)
if package:
package = None
if line:
package = split[1].strip()
dep = []
prov = []
elif split[0]=='Depends' or split[0]=='Requires':
dep = dep + parse_depends(split[1])
elif split[0]=='Provides':
prov = parse_depends(split[1])
if not line: break
class Entry:
atime = 0;
ctime = 0;
mru_file = '';
def __init__(self, atime, ctime, mru_file):
try:
self.atime = int(atime)
self.ctime = int(ctime)
except:
self.atime = self.ctime = 0
self.mru_file = mru_file
def istimestamp(s):
return s.isdigit() or (s[0]=='-' and s[1:].isdigit())
class Submission:
# format: {package: [atime, ctime, mru_file]}
entries = {}
start_date = 0
arch = "unknown"
release= "unknown"
vendor= "Debian"
# initialize a new entry with known data
def __init__(self, version, owner_id, date):
self.entries = {}
self.start_date = int(date)
self.id = owner_id
# process a line of input from the survey
def addinfo(self, split):
if (len(split) < 4 or not istimestamp(split[0])
or not istimestamp(split[1])):
ewrite(self.id + ': Invalid input line: ' + repr(split))
return
self.entries[split[2]] = Entry(split[0], split[1], split[3])
# update the atime of dependency to that of dependant, if newer
def update_atime(self, dependency, dependant):
if not (dependency in self.entries): return
e = self.entries[dependency]
f = self.entries[dependant]
if e.atime < f.atime:
e.atime = f.atime
e.ctime = f.ctime
# we found the last line of the survey: finish it
def done(self, date, st):
st.count = st.count + 1
for package in self.entries.keys():
if package in deplist:
for d in deplist[package]:
self.update_atime(d, package)
if d in provlist:
for dd in provlist[d]:
self.update_atime(dd, package)
for package in self.entries.keys():
if not (package in st.vote):
st.vote[package] = Vote()
st.vote[package].vote_for(package, self.entries[package])
if not (self.vendor in st.vendor):
st.vendor[self.vendor] = 1
else:
st.vendor[self.vendor] = st.vendor[self.vendor] + 1
if not (self.release in st.release):
st.release[self.release] = 1
else:
st.release[self.release] = st.release[self.release] + 1
ewrite("#%s %s" % (st.release[self.release], self.release))
if not (self.arch in st.arch):
st.arch[self.arch] = 1
else:
st.arch[self.arch] = st.arch[self.arch] + 1
def headersplit(pairs):
header = {}
for d in pairs:
list = d.split(':')
try:
key, value = list
header[key] = value
except:
pass
return header
def read_submissions(stream):
e = None
while 1:
line = str(stream.readline(),encoding='latin_1')
if not line: break
split = line.split()
if not split: continue
if split[0]=='POPULARITY-CONTEST-0':
header = headersplit(split[1:])
if not ('ID' in header) or not ('TIME' in header):
ewrite('Invalid header: ' + split[1])
continue
e = None
try:
e = Submission(0, header['ID'], header['TIME'])
except:
ewrite('Invalid date: ' + header['TIME'] + ' for ID ' + header['ID'])
continue
if 'VENDOR' in header:
if header['VENDOR']=='':
e.vendor = 'unknown'
else:
e.vendor = header['VENDOR']
if 'POPCONVER' in header:
if header['POPCONVER']=='':
e.release = 'unknown'
else:
e.release = header['POPCONVER']
if 'ARCH' in header:
if header['ARCH']=='x86_64':
e.arch = 'amd64'
elif header['ARCH']=='i386-gnu':
e.arch = 'hurd-i386'
elif header['ARCH']=='':
e.arch = 'unknown'
else:
e.arch = header['ARCH']
elif split[0]=='END-POPULARITY-CONTEST-0' and e != None:
header = headersplit(split[1:])
if 'TIME' in header:
try:
date = int(header['TIME'])
except:
ewrite('Invalid date: ' + header['TIME'])
continue
e.done(date,stat)
if e.release==stable_version:
e.done(date,stat_stable)
e = None
elif e != None:
e.addinfo(split)
# end of while loop
# main program
for d in glob.glob('%s/dists/stable/*/binary-i386/Packages.xz' % mirrorbase):
read_depends(d)
for d in glob.glob('%s/dists/unstable/*/binary-i386/Packages.xz' % mirrorbase):
read_depends(d)
read_submissions(sys.stdin.buffer)
stat.output("results3")
stat_stable.output("results3.stable")

2
examples/bin/popcon.pl

@ -221,7 +221,7 @@ sub htmlfooter
print HTML <<EOH;
<div id="footer">
Made by <a href="mailto:ballombe\@debian.org">Bill Allombert</a>. Last generated on $date UTC. <br>
<a href="https://popcon.alioth.debian.org" > Popularity-contest project </a> by Avery Pennarun, Bill Allombert and Petter Reinholdtsen.
<a href="https://salsa.debian.org/popularity-contest-team/popularity-contest" > Popularity-contest project </a> by Avery Pennarun, Bill Allombert and Petter Reinholdtsen.
<br>
Debian theme by St&eacute;phane Blondon, based on
Debian template Copyright &copy; 1997-2013

56
popularity-contest

@ -53,6 +53,40 @@ if ( $HOSTID !~ /^([a-f0-9]{32})$/)
exit 1;
}
# List all mapped files
my %mapped;
if (opendir(PROC, "/proc"))
{
my @procfiles = readdir(PROC);
closedir(PROC);
foreach (@procfiles)
{
-d "/proc/$_" or next;
m{^[0-9]+$} or next;
open MAPS, "/proc/$_/maps" or next;
while (<MAPS>)
{
m{(/.*)} or next;
$mapped{$1} = 1;
}
close MAPS;
}
}
if (defined($ARGV[0]) && $ARGV[0] eq "--su-nobody")
{
my $user="nobody";
my ($uid, $gid, $home, $shell) = (getpwnam($user))[2,3,7,8];
$( = $) = $gid;
$< = $> = $uid;
$ENV{USER} = $user;
$ENV{LOGNAME} = $user;
$ENV{HOME} = $home;
$ENV{SHELL} = $shell;
}
# Architecture.
my $debarch = `dpkg --print-architecture`;
chomp $debarch;
@ -85,28 +119,6 @@ sub trunc_time {
my %popcon=();
# List all mapped files
my %mapped;
if (opendir(PROC, "/proc"))
{
my @procfiles = readdir(PROC);
closedir(PROC);
foreach (@procfiles)
{
-d "/proc/$_" or next;
m{^[0-9]+$} or next;
open MAPS, "/proc/$_/maps" or next;
while (<MAPS>)
{
m{(/.*)} or next;
$mapped{$1} = 1;
}
close MAPS;
}
}
# List files diverted by dpkg
my %diverted;
if (open DIVERSIONS, "env LC_ALL=C dpkg-divert --list|")

Loading…
Cancel
Save