onefang
/
apt-panopticon
2
2
Fork 0
Code Issues Releases Wiki Activity
Browse Source

Major refactor, especially of the downloading and processing code. 

Make the code more readable, less scattered.
Use a coroutine to multitask better.
Plugable functions for parsing the download results, and figuring out
what to download next.
Track timeouts at a finer level.
Dig for IPs in the forked apt-panopticons, not all at once at the
beginning.

Various cleanups and tweaks.
master
onefang 3 years ago
parent
23e955a33a
commit
5e8b0adf2a
5 changed files with 471 additions and 299 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 56
      apt-panopticommon.lua
  2. 1
      apt-panopticon-report-RRD.lua
  3. 31
      apt-panopticon-report-email-web.lua
  4. 2
      apt-panopticon-update-data.lua
  5. 680
      apt-panopticon.lua

56
apt-panopticommon.lua
View File

@ -53,7 +53,19 @@ APT.options =
{
typ = "number",
help = "",
value = 15,
value = 5,
},
timeouts =
{
typ = "number",
help = "",
value = 3,
},
retries =
{
typ = "number",
help = "",
value = 3,
},
reports =
{
@ -290,14 +302,16 @@ local log = function(v, t, s, prot, test, host)
t = t .. "(" .. x .. ")"
if "" ~= prot then
if "" == test then
if nil == APT.results[prot] then APT.results[prot] = {errors = 0; warnings = 0} end
if nil == APT.results[prot] then APT.results[prot] = {errors = 0; warnings = 0; timeouts = 0} end
if v == 0 then APT.results[prot].errors = APT.results[prot].errors + 1 end
if v == 1 then APT.results[prot].warnings = APT.results[prot].warnings + 1 end
if v == 2 then APT.results[prot].timeouts = APT.results[prot].timeouts + 1 end
else
if nil == APT.results[prot] then APT.results[prot] = {errors = 0; warnings = 0} end
if nil == APT.results[prot][test] then APT.results[prot][test] = {errors = 0; warnings = 0} end
if nil == APT.results[prot] then APT.results[prot] = {errors = 0; warnings = 0; timeouts = 0} end
if nil == APT.results[prot][test] then APT.results[prot][test] = {errors = 0; warnings = 0; timeouts = 0} end
if v == 0 then APT.results[prot][test].errors = APT.results[prot][test].errors + 1 end
if v == 1 then APT.results[prot][test].warnings = APT.results[prot][test].warnings + 1 end
if v == 2 then APT.results[prot][test].timeouts = APT.results[prot][test].timeouts + 1 end
end
end
end
@ -311,8 +325,9 @@ local log = function(v, t, s, prot, test, host)
if -1 == v then colour = "fuchsia" end -- CRITICAL
if 0 == v then colour = "red " end -- ERROR
if 1 == v then colour = "yellow " end -- WARNING
if 2 == v then colour = "white " end -- INFO
if 3 == v then colour = "gray " end -- DEBUG
if 2 == v then colour = "blue " end -- TIMEOUT
if 3 == v then colour = "white " end -- INFO
if 4 == v then colour = "gray " end -- DEBUG
APT.logFile:write(os.date() .. " <font color='" .. colour .. "'><b>" .. t .. "</b></font>: " .. s .. "</br>\n")
else
APT.logFile:write(os.date() .. " " .. t .. ": " .. s .. "\n")
@ -320,13 +335,15 @@ local log = function(v, t, s, prot, test, host)
APT.logFile:flush()
end
end
APT.D = function(s) log(3, "DEBUG ", s) end
APT.I = function(s) log(2, "INFO ", s) end
APT.D = function(s) log(4, "DEBUG ", s) end
APT.I = function(s) log(3, "INFO ", s) end
APT.T = function(s, p, t, h) log(2, "TIMEOUT ", s, p, t, h) end
APT.W = function(s, p, t, h) log(1, "WARNING ", s, p, t, h) end
APT.E = function(s, p, t, h) log(0, "ERROR ", s, p, t, h) end
APT.C = function(s) log(-1, "CRITICAL", s) end
local D = APT.D
local I = APT.I
local T = APT.T
local W = APT.W
local E = APT.E
local C = APT.C
@ -380,7 +397,7 @@ APT.checkFile = function(f)
if nil == h then return false else h:close(); return true end
end
APT.plurals = function(e, w)
APT.plurals = function(e, w, t)
local result = ""
if 1 == e then
result = e .. " error"
@ -397,18 +414,29 @@ APT.plurals = function(e, w)
end
if ("" ~= result) and APT.html then result = "<font color='yellow'><b>" .. result .. "</b></font>" end
end
if 0 < t then
if (0 < e) or (0 < w) then result = result .. ", " end
if 1 == t then
result = result .. t .. " timeout"
else
result = result .. t .. " timeouts"
end
if ("" ~= result) and APT.html then result = "<font color='blue'><b>" .. result .. "</b></font>" end
end
if "" ~= result then result = " (" .. result .. ")" end
return result
end
APT.padResults = function(results)
local c = 0
if nil == results then results = {}; c = 999 end
for k, v in pairs(APT.protocols) do
tests = results[v]
if nil == tests then tests = {errors = 0; warnings = 0} end
if nil == tests.Integrity then tests.Integrity = {errors = 0; warnings = 0} end
if nil == tests.Protocol then tests.Protocol = {errors = 0; warnings = 0} end
if nil == tests.Updated then tests.Updated = {errors = 0; warnings = 0} end
if nil == tests.URLSanity then tests.URLSanity = {errors = 0; warnings = 0} end
if nil == tests then tests = {errors = c; warnings = c; timeouts = c} end
if nil == tests.Integrity then tests.Integrity = {errors = c; warnings = c; timeouts = c} end
if nil == tests.Protocol then tests.Protocol = {errors = c; warnings = c; timeouts = c} end
if nil == tests.Updated then tests.Updated = {errors = c; warnings = c; timeouts = c} end
if nil == tests.URLSanity then tests.URLSanity = {errors = c; warnings = c; timeouts = c} end
results[v] = tests
end
if nil == results.timeout then results.timeout = false end

1
apt-panopticon-report-RRD.lua
View File

@ -3,6 +3,7 @@
local APT = require 'apt-panopticommon'
local D = APT.D
local I = APT.I
local T = APT.T
local W = APT.W
local E = APT.E
local C = APT.C

31
apt-panopticon-report-email-web.lua
View File

@ -3,6 +3,7 @@
local APT = require 'apt-panopticommon'
local D = APT.D
local I = APT.I
local T = APT.T
local W = APT.W
local E = APT.E
local C = APT.C
@ -47,16 +48,19 @@ local status = function(host, results, typ)
local result = ""
local e = 0
local w = 0
local t = 0
local s = nil ~= APT.mirrors[host].Protocols[typ]
local to = results.timeout
if not APT.search(APT.protocols, typ) then s = true end
if nil ~= results[typ] then
e = results[typ].errors
w = results[typ].warnings
t = results[typ].timeouts
for k, v in pairs(results[typ]) do
if "table" == type(v) then
if 0 <= v.errors then e = e + v.errors else to = true end
if 0 <= v.warnings then w = w + v.warnings else to = true end
if 0 <= v.timeouts then t = t + v.timeouts else to = true end
end
end
else
@ -67,6 +71,7 @@ local status = function(host, results, typ)
if typ == i then
if 0 <= u.errors then e = e + u.errors end
if 0 <= u.warnings then w = w + u.warnings end
if 0 <= u.timeouts then t = t + u.timeouts end
end
end
end
@ -99,6 +104,21 @@ local status = function(host, results, typ)
else
faulty = faulty .. host .. " (" .. typ .. ")\n"
end
elseif 0 < t then
result = "[timeout"
if not s then result = result .. "*" end
if APT.html then
if s then
result = "[<font color='blue'><b>timeout</b></font>"
else
result = "[<font color='darkblue'><b>timeout*</b></font>"
end
end
if APT.html then
faulty = faulty .. host .. " (" .. typ .. ")<br>\n"
else
faulty = faulty .. host .. " (" .. typ .. ")\n"
end
else
result = "[OK"
if not s then result = result .. "*" end
@ -110,7 +130,7 @@ local status = function(host, results, typ)
end
end
end
return result .. APT.plurals(e, w) .. "]"
return result .. APT.plurals(e, w, t) .. "]"
end
local m = {}
@ -125,9 +145,11 @@ local logCount = function(domain, ip)
if nil ~= rfile then
local errors = 0
local warnings = 0
local timeouts = 0
for l in rfile:lines() do
if nil ~= l:match("><b>ERROR ") then errors = errors + 1 end
if nil ~= l:match("><b>WARNING ") then warnings = warnings + 1 end
if nil ~= l:match("><b>TIMEOUT ") then timeouts = timeouts + 1 end
end
rfile:close()
if APT.html then
@ -137,7 +159,7 @@ local logCount = function(domain, ip)
log = "<a href='" .. nm .. "'>" .. ip .. "</a>"
end
end
log = log .. APT.plurals(errors, warnings)
log = log .. APT.plurals(errors, warnings, timeouts)
end
return log
end
@ -303,8 +325,11 @@ if nil == web then C("opening mirrors file - " .. e) else
" means the tested thing is supported for that mirror.</p>\n" ..
"<p>[<font color='darkred'><b>FAILED*</b></font>] or [<font color='darkgreen'><b>OK*</b></font>]" ..
" means the tested thing is unsupported for that mirror, but might have been tested anyway.</p>\n" ..
"<p>[<font color='blue'><b>timeout</b></font>] or [<font color='darkblue'><b>timeout</b></font>]" ..
" means the mirror had some timeouts, and tests where not yet aborted. The darker colour means unsupported by the mirror, but tested anyway.</p>" ..
"<p>[<font color='blue'><b>TIMEOUT</b></font>] or [<font color='darkblue'><b>TIMEOUT</b></font>]" ..
" means the server had too many timeouts, and tests where aborted, so there is no result for this test.</p>" ..
" means the mirror had too many timeouts, and tests where aborted, so there is no result for this test.</p>" ..
"<p>NOTE: timeouts may be due to a problem on the testing computer.</p>" ..
"<p>The DNS round robin (DNS-RR) column shows the IPs for that mirror, or [<font color='grey'><b>no</b></font>] if it isn't part of the DNS-RR. &nbsp; " ..
"The IPs link to the testing log for that IP accessed via the DNS-RR. &nbsp; " ..
"deb.devuan.org is the DNS-RR itself, so it doesn't get tested directly.</p>\n" ..

2
apt-panopticon-update-data.lua
View File

@ -3,7 +3,9 @@
local APT = require 'apt-panopticommon'
local D = APT.D
local I = APT.I
local T = APT.T
local W = APT.W
local E = APT.E
local C = APT.C
local arg, sendArgs = APT.parseArgs({...})

680
apt-panopticon.lua
View File

@ -5,6 +5,7 @@ local now = os.time()
local APT = require 'apt-panopticommon'
local D = APT.D
local I = APT.I
local T = APT.T
local W = APT.W
local E = APT.E
local C = APT.C
@ -23,14 +24,14 @@ local releases = {"jessie", "ascii", "beowulf", "ceres"}
local releaseFiles =
{
-- Release file.
"/Release", -- 3.7 MB
"/Release.gpg", --
-- "/InRelease", -- 3.7 MB
-- "/main/binary-all/Packages.xz", -- 2.6 GB for all that changed recently.
"Release", -- 3.7 MB
"Release.gpg", --
-- "InRelease", -- 3.7 MB
-- "main/binary-all/Packages.xz", -- 2.6 GB for all that changed recently.
-- Contents files. -- 3.3 GB
-- "/main/Contents-all.xz",
-- "/main/Contents-amd64.xz",
-- "/main/Contents-arm64.xz",
-- "main/Contents-all.xz",
-- "main/Contents-amd64.xz",
-- "main/Contents-arm64.xz",
-- "-security/main/Contents-all.xz",
-- "-security/main/Contents-amd64.xz",
-- "-security/main/Contents-arm64.xz",
@ -160,6 +161,7 @@ local url = require 'socket.url'
local ip = ""
local cor = nil
local repoExists = function (r)
@ -197,6 +199,7 @@ gatherIPs = function (host)
end
until nil == IPs
end
return IP[host]
end
-- Returns FTP directory listing
@ -229,7 +232,7 @@ checkHEAD = function (host, URL, r, retry, sanity)
E("too many timeouts! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host)
return
end
if 4 <= (totalTimeouts) then
if APT.options.timeouts.value <= (totalTimeouts) then
E("Way too many timeouts!", PU.scheme, "", host)
return
end
@ -237,7 +240,7 @@ checkHEAD = function (host, URL, r, retry, sanity)
E("too many redirects! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host)
return
end
if 4 <= retry then
if APT.options.retries.value <= retry then
E("too many retries! " .. check .. " " .. host .. " -> " .. URL, PU.scheme, "", host)
return
end
@ -343,6 +346,8 @@ checkHEAD = function (host, URL, r, retry, sanity)
Still duplicates a tiny bit, but much less than the previous find based method.
TODO - maybe we can switch to using flock like we do with the other curl calls?
]]
C('Commented out code being called in checkHEAD() for redirects that should no longer happen!')
--[[
local file = pu.host .. "://" .. pu.path
local f = io.popen(string.format('if [ ! -f results/%s.check ] ; then touch results/%s.check; echo -n "check"; fi', file:gsub("/", "_"), file:gsub("/", "_") )):read("*a")
if (nil == f) or ("check" == f) then
@ -351,28 +356,27 @@ TODO - maybe we can switch to using flock like we do with the other curl calls?
else
D(" Already checking " .. file)
end
]]
end
end
end
end
local checkTimeouts = function(host, scheme, URL)
if APT.testing(scheme) then
totalTimeouts = totalTimeouts + timeouts; timeouts = 0
checkHEAD(host, scheme .. "://" .. URL)
if 4 <= (totalTimeouts) then
E("Way too many timeouts!", scheme, "", host)
return true
end
end
totalTimeouts = totalTimeouts + timeouts; timeouts = 0
checkHEAD(host, scheme .. "://" .. URL)
if APT.testing("URLSanity") then
URL = URL:gsub("merged/", "merged///")
totalTimeouts = totalTimeouts + timeouts; timeouts = 0
checkHEAD(host, scheme .. "://" .. URL, 0, 0, true)
if 4 <= (totalTimeouts) then
E("Way too many timeouts!", scheme, "URLSanity", host)
return true
end
end
if nil ~= cor then
D('*&gt;* About to resume coroutine after checkHEAD(' .. host .. ' , ' .. scheme .. ' :// ' .. URL .. ')')
local ok, message = coroutine.resume(cor)
if not ok then cor = nil; print(message) end
end
if APT.options.timeouts.value <= (totalTimeouts) then
E("Way too many timeouts!", scheme, "URLSanity", host)
return true
end
return false
end
@ -387,19 +391,18 @@ local checkFiles = function (host, ip, path, file)
if checkTimeouts(host, "https", ip .. path .. "/" .. file) then return end
else
I(" Checking IP " .. host .. " -> " .. ip .. " " .. path)
for i, s in pairs(referenceDevs) do
if checkTimeouts(host, "http", ip .. path .. "/" .. s) then return end
if checkTimeouts(host, "https", ip .. path .. "/" .. s) then return end
end
for i, s in pairs(releases) do
for j, k in pairs(releaseFiles) do
if repoExists(s .. k) then
if checkTimeouts(host, "http", ip .. path .. "/merged/dists/" .. s .. k) then return end
if checkTimeouts(host, "https", ip .. path .. "/merged/dists/" .. s .. k) then return end
if checkTimeouts(host, "http", ip .. path .. "/merged/dists/" .. s .. '/' .. k) then return end
if checkTimeouts(host, "https", ip .. path .. "/merged/dists/" .. s .. '/' .. k) then return end
end
end
end
for i, s in pairs(referenceDevs) do
if checkTimeouts(host, "http", ip .. path .. "/" .. s) then return end
if checkTimeouts(host, "https", ip .. path .. "/" .. s) then return end
end
end
end
@ -421,127 +424,51 @@ checkHost = function (orig, host, path, ip, file)
D("checkHost " .. orig .. "" .. file)
if APT.testing("IPv4") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file) end
else D("checkHost " .. orig .. " -> " .. host) end
local h = APT.mirrors[ph.host]
if nil == h then return end
for k, v in pairs(h.IPs) do
if "table" == type(v) then
for k1, v1 in pairs(v) do
if v1 == "A" then
if APT.testing("IPv4") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k1 .. " " .. file) end
elseif v1 == "AAAA" then
if APT.testing("IPv6") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k1 .. " " .. file) end
end
end
else
if v == "A" then
if APT.testing("IPv4") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k .. " " .. file) end
elseif v == "AAAA" then
if APT.testing("IPv6") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. orig .. path .. " " .. k .. " " .. file) end
end
end
end
end
end
local addDownload = function(host, URL, f, r, k)
local file = k:match(".*/([%w%.%+%-_]*)$") -- Get the filename.
if APT.checkFile("results/" .. host .. "/merged/dists/" .. r .. k) then
if APT.checkFile("results/" .. host .. "/merged/dists/" .. r .. '/' .. k) then
-- Curls "check timestamp and overwrite file" stuff sucks.
-- -R means the destination file gets the timestamp of the remote file.
-- Can only do ONE timestamp check per command.
-- This doesn't work either. All downloads get all these headers. Pffft
-- local status, ts = APT.execute('TZ="GMT" ls -l --time-style="+%a, %d %b %Y %T %Z" results/' .. host .. "/merged/dists/" .. r .. k .. ' | cut -d " " -f 6-11')
-- local status, ts = APT.execute('TZ="GMT" ls -l --time-style="+%a, %d %b %Y %T %Z" results/' .. host .. "/merged/dists/" .. r .. '/' .. k .. ' | cut -d " " -f 6-11')
-- f:write('header "If-Modified-Since: ' .. ts:sub(2, -2) .. '"\n')
-- Curl will DELETE the existing file if the timestamp fails to download a new one, unless we change directory first,
-- which wont work with multiple files in multiple directories. WTF?
os.execute(" mv results/" .. host .. "/merged/dists/" .. r .. k ..
" results/" .. host .. "/merged/dists/" .. r .. k .. ".old")
os.execute(" mv results/" .. host .. "/merged/dists/" .. r .. '/' .. k ..
" results/" .. host .. "/merged/dists/" .. r .. '/' .. k .. ".old")
end
D('Downloading http://' .. host .. URL .. '/merged/dists/' .. r .. k)
f:write('url "' .. 'http://' .. host .. URL .. '/merged/dists/' .. r .. k .. '"\n')
f:write('output "results/' .. host .. '/merged/dists/' .. r .. k .. '"\n')
D('Downloading http://' .. host .. URL .. '/merged/dists/' .. r .. '/' .. k)
f:write('url "' .. 'http://' .. host .. URL .. '/merged/dists/' .. r .. '/' .. k .. '"\n')
f:write('output "results/' .. host .. '/merged/dists/' .. r .. '/' .. k .. '"\n')
end
local postDownload = function(host, r, k)
local file = k:match(".*/([%w%.%+%-_]*)$") -- Get the filename.
local dir = k:sub(1, 0 - (#file + 1))
os.execute("if [ -f results/" .. host .. "/merged/dists/" .. r .. k .. ".old ]" ..
" && [ ! -f results/" .. host .. "/merged/dists/" .. r .. k .. " ]; then cp -a" ..
" results/" .. host .. "/merged/dists/" .. r .. k .. ".old" ..
" results/" .. host .. "/merged/dists/" .. r .. k .. "; fi")
if ".gz" == k:sub(-3, -1) then APT.execute("ionice -c3 nice -n 19 gzip -dfk results/" .. host .. "/merged/dists/" .. r .. k) end
if ".xz" == k:sub(-3, -1) then APT.execute("ionice -c3 nice -n 19 xz -dfk results/" .. host .. "/merged/dists/" .. r .. k .. " 2>/dev/null") end
if APT.testing("Integrity") then
if ".gpg" == k:sub(-4, -1) then
local status, out = APT.execute("gpgv --keyring /usr/share/keyrings/devuan-keyring.gpg results/" .. host .. "/merged/dists/" .. r .. k ..
" results/" .. host .. "/merged/dists/" .. r .. k:sub(1, -5) .. " 2>/dev/null")
if "0" ~= status then E("GPG check failed - " .. host .. "/merged/dists/" .. r .. k, "http", "Integrity", host) end
os.execute('rm results/' .. host .. '/merged/dists/' .. r .. k)
end
-- TODO - should check the PGP sig of InRelease as well.
end
if APT.testing("Integrity") or APT.testing("Updated") then
if "Packages." == file:sub(1, 9) then
-- TODO - compare the SHA256 sums in pkgmaster's Release for both the packed and unpacked versions.
-- Also note that this might get only a partial download due to maxtime.
if APT.options.referenceSite.value == host then
local Pp, e = io.open('results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed', "w+")
if nil == Pp then W('opening results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed' .. ' file - ' .. e) else
local pp = {}
for l in io.lines('results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages') do
if "Package: " == l:sub(1, 9) then
if 0 ~= #pp then
for i = 1, 5 do
if nil == pp[i] then print(host .. " " .. r .. " " .. dir .. " " .. i) else Pp:write(pp[i] .. " | ") end
end
Pp:write("\n")
end
pp = {}
pp[1] = l:sub(10, -1)
elseif "Version: " == l:sub(1, 9) then
pp[2] = l:sub(10, -1)
elseif "Filename: " == l:sub(1, 10) then
pp[3] = l:sub(11, -1)
elseif "Size: " == l:sub(1, 6) then
pp[4] = l:sub(7, -1)
elseif "SHA256: " == l:sub(1, 8) then
pp[5] = l:sub(9, -1)
end
end
Pp:close()
os.execute('sort results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages.parsed >results/' .. host .. '/merged/dists/'.. r .. dir .. 'Packages_parsed-sorted')
if APT.checkFile('Packages/' .. r .. dir .. 'Packages_parsed-sorted') then
os.execute('diff -U 0 Packages/' .. r .. dir .. 'Packages_parsed-sorted ' ..
'results/pkgmaster.devuan.org/merged/dists/' .. r .. dir .. 'Packages_parsed-sorted ' ..
' | grep -E "^-" | grep -Ev "^\\+\\+\\+|^---" >>results/OLD_PACKAGES_' .. r .. '.txt')
os.execute('diff -U 0 Packages/' .. r .. dir .. 'Packages_parsed-sorted ' ..
'results/pkgmaster.devuan.org/merged/dists/' .. r .. dir .. 'Packages_parsed-sorted ' ..
' | grep -E "^\\+" | grep -Ev "^\\+\\+\\+|^---" >>results/NEW_Packages_' .. r .. '.txt')
-- Find the smallest new package for each release.
os.execute('sort -b -k 9,9 -n results/NEW_Packages_' .. r .. '.txt >results/NEW_Packages_' .. r .. '.sorted.txt')
os.execute('grep -s " | pool/DEBIAN/" results/NEW_Packages_' .. r .. '.sorted.txt 2>/dev/null | head -n 1 >results/NEW_Packages_' .. r .. '.test.tmp')
os.execute('grep -s " | pool/DEBIAN-SECURITY/" results/NEW_Packages_' .. r .. '.sorted.txt 2>/dev/null | head -n 1 >>results/NEW_Packages_' .. r .. '.test.tmp')
os.execute('grep -s " | pool/DEVUAN/" results/NEW_Packages_' .. r .. '.sorted.txt 2>/dev/null | head -n 1 >>results/NEW_Packages_' .. r .. '.test.tmp')
os.execute('mv results/NEW_Packages_' .. r .. '.test.tmp results/NEW_Packages_' .. r .. '.test.txt')
else
C("Can't find file Packages/" .. r .. dir .. "Packages_parsed-sorted")
end
os.execute('mkdir -p Packages/' .. r .. dir)
os.execute('mv -f results/pkgmaster.devuan.org/merged/dists/' .. r .. dir .. 'Packages_parsed-sorted Packages/' .. r .. dir .. 'Packages_parsed-sorted')
end
else
while not APT.checkFile('results/NEW_Packages_' .. r .. '.test.txt') do os.execute('sleep 10') end
end
os.execute('rm -f results/' .. host .. '/merged/dists/' .. r .. dir .. 'Packages 2>/dev/null')
os.execute('rm -f results/' .. host .. '/merged/dists/' .. r .. dir .. 'Packages.* 2>/dev/null')
end
if nil == file then file = k end
os.execute("if [ -f results/" .. host .. "/merged/dists/" .. r .. '/' .. k .. ".old ]" ..
" && [ ! -f results/" .. host .. "/merged/dists/" .. r .. '/' .. k .. " ]; then cp -a" ..
" results/" .. host .. "/merged/dists/" .. r .. '/' .. k .. ".old" ..
" results/" .. host .. "/merged/dists/" .. r .. '/' .. k .. "; fi")
if APT.checkFile('results/' .. host .. '/merged/dists/' .. r .. '/' .. k) then
if ".gz" == k:sub(-3, -1) then APT.execute("ionice -c3 nice -n 19 gzip -dfk results/" .. host .. "/merged/dists/" .. r .. '/' .. k) end
if ".xz" == k:sub(-3, -1) then APT.execute("ionice -c3 nice -n 19 xz -dfk results/" .. host .. "/merged/dists/" .. r .. '/' .. k .. " 2>/dev/null") end
end
end
local downloadLock = "flock -n results/curl-"
local download = "curl --connect-timeout " .. APT.options.timeout.value .. " --create-dirs -f -L --max-time " .. APT.options.maxtime.value .. " -z 'results/stamp.old' -v -R "
local download = "curl" ..
" --connect-timeout " .. APT.options.timeout.value ..
" --create-dirs -f -L" ..
" --fail-early" ..
" --max-time " .. APT.options.maxtime.value ..
" --retry " .. APT.options.retries.value ..
" -R -v -z 'results/stamp.old' "
local downloads = function(host, URL, meta, release, list)
if nil == URL then URL = "" end
local lock = meta .. "-" .. host .. ".log"
@ -600,6 +527,7 @@ local validateURL = function(m)
m.BaseURL = p.authority .. p.path
return m
end
local getMirrors = function ()
local mirrors = {}
local host = ""
@ -621,8 +549,6 @@ local getMirrors = function ()
end
host = d
m[t] = d
gatherIPs(host)
m["IPs"] = IP[host]
elseif "Protocols" == t then
local prot = {}
for w in d:gmatch("(%w+)") do
@ -643,19 +569,305 @@ local getMirrors = function ()
end
end
if APT.testing("DNSRR") then
mirrors[APT.options.roundRobin.value] = { ["Protocols"] = { ["http"] = true; ["https"] = true; }; ["FQDN"] = 'deb.devuan.org'; ["Active"] = 'yes'; ["BaseURL"] = 'deb.devuan.org'; }
gatherIPs(APT.options.roundRobin.value)
mirrors[APT.options.roundRobin.value].IPs = IP[APT.options.roundRobin.value]
end
local file, e = io.open("results/mirrors.lua", "w+")
if nil == file then C("opening mirrors file - " .. e) else
file:write(APT.dumpTable(mirrors, "", "mirrors") .. "\nreturn mirrors\n")
file:close()
mirrors[APT.options.roundRobin.value] = { ["Protocols"] = { ["http"] = true; ["https"] = true; };
["FQDN"] = APT.options.roundRobin.value; ["Active"] = 'yes'; ["BaseURL"] = APT.options.roundRobin.value; }
end
return mirrors
end
local postParse = function(host, list)
if APT.options.referenceSite.value == host then
if nil ~= list then
local sem = 'results/NEW_' .. list.out .. '_%s.txt'
for i, n in pairs(releases) do
local f = sem:format(n)
if APT.checkFile(f .. '.tmp') then
os.execute('mv ' .. f .. '.tmp ' .. f)
else
os.execute('touch ' .. f)
end
end
end
end
end
local parseDebs = function(host)
for i, n in pairs(releases) do
local inFile = 'results/NEW_debs_' .. n .. '.txt'
local nfile, e = io.open(inFile, "r")
if nil == nfile then W("opening " .. inFile .. " file - " .. e) else
for l in nfile:lines() do
local v, p, sz, sha = l:match(' | (.+) | (pool/.+%.deb) | (%d.+) | (%x.+) |')
if nil ~= p then
if APT.checkFile('results/' .. host .. "/merged/" .. p) then
local status, fsz = APT.execute('ls -l results/' .. host .. "/merged/" .. p .. ' | cut -d " " -f 5-5')
if APT.testing("Integrity") then
if sz ~= fsz:sub(2, -2) then -- The sub bit is to slice off the EOLs at each end.
E('Package size mismatch - results/' .. host .. "/merged/" .. p .. ' should be ' .. sz .. ', but is ' .. fsz:sub(2, -2) .. '.', 'http', 'Integrity', host)
else
local status, fsha = APT.execute('sha256sum results/' .. host .. "/merged/" .. p .. ' | cut -d " " -f 1')
if sha ~= fsha:sub(2, -2) then E('Package SHA256 sum mismatch - results/' .. host .. "/merged/" .. p, 'http', 'Integrity', host) end
-- TODO - maybe check the PGP key, though packages are mostly not signed.
end
end
if APT.testing("Updated") then
if sz ~= fsz:sub(2, -2) then
E('Package size mismatch - results/' .. host .. "/merged/" .. p, 'http', 'Updated', host)
end
end
os.execute('rm -f results/' .. host .. "/merged/" .. p)
else
E('Failed to download - results/' .. host .. "/merged/" .. p, 'http', 'Updated', host)
end
end
end
end
end
return nil
end
local parsePackages = function(host)
local list = {inf = 'Packages', parser = parseDebs, out = 'debs', files = {}, nextf = ''}
for i, n in pairs(releases) do
local inFile = 'results/NEW_' .. list.inf .. '_' .. n .. '.txt'
local outFile = 'results/NEW_' .. list.out .. '_' .. n .. '.txt'
if APT.options.referenceSite.value == host then
outFile = outFile .. '.tmp'
end
local dFile, e = io.open(inFile, "r")
if nil == dFile then W("opening " .. inFile .. " file - " .. e) else
for l in dFile:lines() do
postDownload(host, n, l)
l = '/' .. l
local file = l:match(".*/([%w%.%+%-_]*)$") -- Get the filename.
local dir = l:sub(1, 0 - (#file + 1))
if "Packages." == file:sub(1, 9) then
-- TODO - compare the SHA256 sums in pkgmaster's Release for both the packed and unpacked versions.
-- Also note that this might get only a partial download due to maxtime.
if APT.options.referenceSite.value == host then
local Pp, e = io.open('results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages.parsed', "w+")
if nil == Pp then W('opening results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages.parsed' .. ' file - ' .. e) else
local pp = {}
for l in io.lines('results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages') do
if "Package: " == l:sub(1, 9) then
if 0 ~= #pp then
for i = 1, 5 do
if nil == pp[i] then print(host .. " " .. n .. " " .. dir .. " " .. i) else Pp:write(pp[i] .. " | ") end
end
Pp:write("\n")
end
pp = {}
pp[1] = l:sub(10, -1)
elseif "Version: " == l:sub(1, 9) then
pp[2] = l:sub(10, -1)
elseif "Filename: " == l:sub(1, 10) then
pp[3] = l:sub(11, -1)
elseif "Size: " == l:sub(1, 6) then
pp[4] = l:sub(7, -1)
elseif "SHA256: " == l:sub(1, 8) then
pp[5] = l:sub(9, -1)
end
end
Pp:close()
os.execute('sort results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages.parsed >results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages_parsed-sorted')
if APT.checkFile('Packages/' .. n .. dir .. 'Packages_parsed-sorted') then
os.execute('diff -U 0 Packages/' .. n .. dir .. 'Packages_parsed-sorted ' ..
'results/' .. APT.options.referenceSite.value .. '/merged/dists/' .. n .. dir .. 'Packages_parsed-sorted ' ..
' | grep -E "^-" | grep -Ev "^\\+\\+\\+|^---" >>results/OLD_' .. list.out .. '_' .. n .. '.txt')
os.execute('diff -U 0 Packages/' .. n .. dir .. 'Packages_parsed-sorted ' ..
'results/' .. APT.options.referenceSite.value .. '/merged/dists/' .. n .. dir .. 'Packages_parsed-sorted ' ..
' | grep -E "^\\+" | grep -Ev "^\\+\\+\\+|^---" >>results/NEW_' .. list.out .. '_TMP_' .. n .. '.txt')
else
C("Can't find file Packages/" .. n .. dir .. "Packages_parsed-sorted")
end
os.execute('mkdir -p Packages/' .. n .. dir)
os.execute('mv -f results/' .. APT.options.referenceSite.value .. '/merged/dists/' .. n .. dir .. 'Packages_parsed-sorted Packages/' .. n .. dir .. 'Packages_parsed-sorted')
end
else
end
os.execute('rm -fr results/' .. host .. '/merged/dists/' .. n .. dir .. ' 2>/dev/null')
end
end
if APT.checkFile('results/NEW_' .. list.out .. '_TMP_' .. n .. '.txt') then
-- Sort by size.
os.execute('sort -b -k 9,9 -n results/NEW_' .. list.out .. '_TMP_' .. n .. '.txt >results/NEW_' .. list.out .. '_' .. n .. '.sorted.txt')
os.execute('grep -s " | pool/DEBIAN/" results/NEW_' .. list.out .. '_' .. n .. '.sorted.txt 2>/dev/null | head -n 1 >' .. outFile)
os.execute('grep -s " | pool/DEBIAN-SECURITY/" results/NEW_' .. list.out .. '_' .. n .. '.sorted.txt 2>/dev/null | head -n 1 >>' .. outFile)
os.execute('grep -s " | pool/DEVUAN/" results/NEW_' .. list.out .. '_' .. n .. '.sorted.txt 2>/dev/null | head -n 1 >>' .. outFile)
os.execute('rm -f results/NEW_' .. list.out .. '_TMP_' .. n .. '.txt')
end
end
local nfile, e = io.open(outFile, "r")
if nil ~= nfile then
-- for l in nfile:lines() do
local l = nfile:read('*l')
if nil ~= l then
local p = l:match('(pool/.*%.deb)')
if nil ~= p then
table.insert(list.files, p)
end
end
-- end
end
end
postParse(host, list)
return list
end
local parseRelease = function(host)
local list = {inf = 'Release', parser = parsePackages, out = 'Packages', files = {}, nextf = 'debs'}
for i, n in pairs(releases) do
for l, o in pairs(releaseFiles) do
if repoExists(i .. o) then
postDownload(host, n, o)
if (".gpg" == o:sub(-4, -1)) and (APT.checkFile('results/' .. host .. '/merged/dists/' .. n .. '/' .. o)) then
if APT.testing("Integrity") then
local status, out = APT.execute("gpgv --keyring /usr/share/keyrings/devuan-keyring.gpg results/" .. host .. "/merged/dists/" .. n .. '/' .. o ..
" results/" .. host .. "/merged/dists/" .. n .. '/' .. o:sub(1, -5) .. " 2>/dev/null")
if "0" ~= status then E("GPG check failed - " .. host .. "/merged/dists/" .. n .. '/' .. o, "http", "Integrity", host) end
-- TODO - should check the PGP sig of InRelease as well.
end
os.execute('rm results/' .. host .. '/merged/dists/' .. n .. '/' .. o)
end
end
end
if APT.checkFile('results/' .. host .. '/merged/dists/' .. n .. '/Release') then
os.execute('sort -k 3 results/' .. host .. '/merged/dists/' .. n .. '/Release >results/' .. host .. '/merged/dists/' .. n .. '/Release.SORTED')
local outFile = 'results/NEW_' .. list.out .. '_' .. n .. '.txt'
if APT.checkFile('results_old/' .. APT.options.referenceSite.value .. '/merged/dists/' .. n .. '/Release.SORTED') then
if APT.options.referenceSite.value == host then
outFile = outFile .. '.tmp'
os.execute('diff -U 0 results_old/' .. APT.options.referenceSite.value .. '/merged/dists/' .. n .. '/Release.SORTED ' ..
'results/' .. APT.options.referenceSite.value .. '/merged/dists/' .. n .. '/Release.SORTED ' ..
'| grep -v "@@" | grep "^+" | grep "Packages.xz$" | cut -c 77- >' .. outFile)
-- TODO - Maybe check the date in Release, though since they are updated daily, is there any point? Perhaps it's for checking amprolla got run?
-- Also check if that date is in the future, apt recently got a check for that, though not sure why.
os.execute('rm -f results/' .. host .. '/merged/dists/' .. n .. '/Release 2>/dev/null; ')
else
-- TODO - compare to the pkgmaster copy.
end
-- TODO - if it's not Integrity and not reference, then just do a HEAD check and compare file times?
if APT.testing("Integrity") or (APT.options.referenceSite.value == host) then
local dfile, e = io.open(outFile, "r")
if nil == dfile then W("opening " .. outFile .. " file - " .. e) else
for l in dfile:lines() do
table.insert(list.files, 'dists/' .. n .. '/' .. l)
end
end
end
end
end
end
postParse(host, list)
return list
end
local parseStart = function(host)
local list = {inf = '', parser = parseRelease, out = 'Release', files = {}, nextf = 'Packages'}
for i, n in pairs(releases) do
local outFile = 'results/NEW_' .. list.out .. '_' .. n .. '.txt'
for l, o in pairs(releaseFiles) do
if repoExists(n .. o) then
if APT.options.referenceSite.value == host then
local dfile, e = io.open(outFile .. '.tmp', "a+")
if nil == dfile then W("opening " .. outFile .. ".tmp file - " .. e) else
dfile:write(o .. '\n')
end
end
table.insert(list.files, 'dists/' .. n .. '/' .. o)
end
end
end
postParse(host, list)
return list
end
local doDownloads = function(host, path, list)
while nil ~= list do
if 0 ~= #(list.files) then
for j, f in pairs(list.files) do
downloads(host, path, list.out, nil, f)
end
downloads(host, path, list.out, nil, '')
--[[ I've seen flock & curl act oddly. Perhaps flock didn't have time to start up?
/var/www/html/apt-panopticon/apt-panopticon/results_2019-12-22-15-00
Mon Dec 23 01:02:54 2019 DEBUG : forking
ionice -c3 nice -n 19 flock -n results/curl-debs-pkgmaster.devuan.org.log curl --connect-timeout 5 --create-dirs -f -L --fail-early --max-time 300 --retry 3 -R -v -z 'results/stamp.old' --stderr results/curl-debs-pkgmaster.devuan.org.log -K results/curl-debs-pkgmaster.devuan.org.files
Mon Dec 23 01:02:54 2019 DEBUG : 0 flock -n results/curl-debs-pkgmaster.devuan.org.log commands still running.
Mon Dec 23 01:02:54 2019 DEBUG : *>* Resumed coroutine NO LONGER waiting on - 0 < APT.checkExes(flock -n results/curl-debs-pkgmaster.devuan.org.log
Mon Dec 23 01:02:54 2019 DEBUG : *** Doing list.parser() for debs
Mon Dec 23 01:02:54 2019 ERROR (http Updated pkgmaster.devuan.org): Failed to download - results/pkgmaster.devuan.org/merged/pool/DEBIAN/main/a/aptly/aptly_1.3.0+ds1-4_amd64.deb
drwxr-x--- 2 www-data www-data 4096 2019-12-23 01:02:57.000000000 +1000 aptly
-rw-r--r-- 1 www-data www-data 7129 2019-12-23 01:03:54.000000000 +1000 curl-debs-pkgmaster.devuan.org.log
]]
os.execute('sleep 2')
while 0 < APT.checkExes(downloadLock .. list.out .. "-" .. host .. ".log") do
D('*&lt;* About to yield coroutine while waiting on - 0 < APT.checkExes(' .. downloadLock .. list.out .. '-' .. host .. '.log')
coroutine.yield()
D('*&gt;* Resumed coroutine while waiting on - 0 < APT.checkExes(' .. downloadLock .. list.out .. '-' .. host .. '.log')
end
D('*&gt;* Resumed coroutine NO LONGER waiting on - 0 < APT.checkExes(' .. downloadLock .. list.out .. '-' .. host .. '.log')
local min, max, spd = 999999999999, 0
if APT.checkFile("results/curl-" .. list.out .. "-" .. host .. ".log") then
for l in io.lines("results/curl-" .. list.out .. "-" .. host .. ".log") do
local speed, crrnt = l:match('^%c *%d+ +%d+k? +%d+ +%d+k? +%d+ +%d+ +(%d+k?) +%d+ +[%d%-]+:[%d%-]+:[%d%-]+ +[%d%-]+:[%d%-]+:[%d%-]+ +[%d%-]+:[%d%-]+:[%d%-]+ +(%d+k?)')
if nil ~= speed then
if 'k' == speed:sub(-1, -1) then speed = speed:sub(1, -2) .. '000' end
if 'k' == crrnt:sub(-1, -1) then crrnt = crrnt:sub(1, -2) .. '000' end
speed = tonumber(speed)
crrnt = tonumber(crrnt)
if speed < min and speed ~= 0 then min = speed end
if speed > max then max = speed end
if crrnt < min and crrnt ~= 0 then min = crrnt end
if crrnt > max then max = crrnt end
end
if l:find('timed out') ~= nil then
E(" TIMEOUT " .. timeouts + 1 .. ', details in curl-' .. list.out .. '-' .. host .. '.log', 'http', '', host)
timeouts = timeouts + 1
APT.results["timeout"] = true
end
end
end
APT.results["speed"] = {min = min, max = max}
end
if (APT.options.referenceSite.value ~= host) and ('' ~= list.nextf) then
local sem = 'results/NEW_' .. list.nextf .. '_%s.txt'
for i, n in pairs(releases) do
local f = sem:format(n)
while not APT.checkFile(f) do
D('*&lt;* About to yield coroutine while waiting on - not APT.checkFile(' .. f .. ')')
coroutine.yield()
D('*&gt;* Resumed coroutine while waiting on - not APT.checkFile(' .. f .. ')')
end
end
end
D('*** Doing list.parser() for ' .. list.out)
list = list.parser(host)
if APT.options.timeouts.value <= (totalTimeouts) then break end
end
D('*&lt;&lt;* About to end coroutine.')
cor = nil
end
if 0 < #arg then
if "/" == arg[1]:sub(-1, -1) then
W("slash at end of path! " .. arg[1])
@ -673,6 +885,8 @@ if 0 < #arg then
end
end
if APT.origin then APT.results["IPs"] = gatherIPs(pu.host) end
if nil ~= arg[2] then
APT.logFile, e = io.open("results/LOG_" .. pu.host .. "_" .. arg[2] .. ".html", "a+")
else
@ -681,165 +895,57 @@ if 0 < #arg then
if nil == APT.logFile then C("opening log file - " .. e); return end
APT.logPre()
I("Starting tests for " .. arg[1] .. " with these tests - " .. table.concat(APT.options.tests.value, ", "))
APT.mirrors = loadfile("results/mirrors.lua")()
if nil ~= arg[2] then I(" Using IP " .. arg[2]); ip = arg[2] end
if nil ~= arg[3] then I(" Using file " .. arg[3]); end
APT.results = APT.padResults(APT.results)
if APT.origin then
local file = arg[3]
if nil == file then file = '' end
local path = pu.path
if nil == path then path = '' end
local ips = APT.results["IPs"]
for k, v in pairs(ips) do
if "table" == type(v) then
for k1, v1 in pairs(v) do
if v1 == "A" then
if APT.testing("IPv4") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k1 .. " " .. file) end
elseif v1 == "AAAA" then
if APT.testing("IPv6") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k1 .. " " .. file) end
end
end
else
if v == "A" then
if APT.testing("IPv4") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k .. " " .. file) end
elseif v == "AAAA" then
if APT.testing("IPv6") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k .. " " .. file) end
end
end
end
if APT.testing("Integrity") or APT.testing("Updated") then
if APT.origin and (APT.options.roundRobin.value ~= pu.host) then
I("Starting file downloads for " .. pu.host)
downloads(pu.host, pu.path, 'Release')
D('*&gt;* About to create coroutine.')
cor = coroutine.create(doDownloads)
local ok, message = coroutine.resume(cor, pu.host, pu.path, parseStart(pu.host))
if not ok then cor = nil; print(message) end
end
end
checkFiles(pu.host, pu.host, pu.path);
else
checkHost(pu.host, pu.host, pu.path, arg[2], arg[3])
end
if APT.testing("Integrity") or APT.testing("Updated") then
if 4 > (totalTimeouts) then
if APT.origin and (APT.options.roundRobin.value ~= pu.host) then
while 0 < APT.checkExes(downloadLock .. "Release-" .. pu.host .. ".log") do os.execute("sleep 10") end
local fcount = 0
for i, n in pairs(releases) do
for l, o in pairs(releaseFiles) do
if repoExists(i .. o) then
postDownload(pu.host, n, o)
end
end
if APT.checkFile('results/' .. pu.host .. '/merged/dists/' .. n .. '/Release') then
os.execute('sort -k 3 results/' .. pu.host .. '/merged/dists/' .. n .. '/Release >results/' .. pu.host .. '/merged/dists/' .. n .. '/Release.SORTED')
if APT.checkFile('results_old/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED') then
if APT.options.referenceSite.value == pu.host then
os.execute('diff -U 0 results_old/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' ..
'results/pkgmaster.devuan.org/merged/dists/' .. n .. '/Release.SORTED ' ..
'| grep -v "@@" | grep "^+" | grep "Packages.xz$" | cut -c 77- >results/NEW_Release_' .. n .. '.tmp')
-- TODO - Maybe check the date in Release, though since they are updated daily, is there any point? Perhaps it's for checking amprolla got run?
os.execute('rm -f results/' .. pu.host .. '/merged/dists/' .. n .. '/Release 2>/dev/null; ' ..
'mv results/NEW_Release_' .. n .. '.tmp results/NEW_Release_' .. n .. '.txt')
else
-- TODO - compare to the pkgmaster copy.
while not APT.checkFile('results/NEW_Release_' .. n .. '.txt') do os.execute('sleep 10') end
end
local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r")
if nil == dfile then W("opening results/NEW_Release_" .. n .. " file - " .. e) else
local diff = dfile:read("*a")
if "" ~= diff then
downloads(pu.host, pu.path, 'Packages', n, diff)
fcount = fcount + 1
end
end
end
end
end
if 0 ~= fcount then
downloads(pu.host, pu.path, 'Packages', "", "")
while 0 < APT.checkExes(downloadLock .. "Packages-" .. pu.host .. ".log") do os.execute("sleep 10") end
for i, n in pairs(releases) do
local dfile, e = io.open('results/NEW_Release_' .. n .. '.txt', "r")
if nil == dfile then W("opening results/NEW_Release_" .. n .. ".txt file - " .. e) else
local diff = dfile:read("*a")
for l in diff:gmatch("\n*([^\n]+)\n*") do
postDownload(pu.host, n, "/" .. l)
end
end
if APT.options.referenceSite.value == pu.host then
-- In case it wasn't dealt with already.
os.execute('touch results/NEW_Packages_' .. n .. '.test.txt')
end
end
fcount = 0
for i, n in pairs(releases) do
local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r")
if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else
for l in nfile:lines() do
local p = l:match('(pool/.*%.deb)')
if nil ~= p then
downloads(pu.host, pu.path, 'package', nil, p)
fcount = fcount + 1
end
end
end
end
if 0 ~= fcount then
downloads(pu.host, pu.path, 'package', nil, "")
while 0 < APT.checkExes(downloadLock .. "package-" .. pu.host .. ".log") do os.execute("sleep 10") end
for i, n in pairs(releases) do
local nfile, e = io.open('results/NEW_Packages_' .. n .. '.test.txt', "r")
if nil == nfile then W("opening results/NEW_Packages_" .. n .. ".test.txt file - " .. e) else
for l in nfile:lines() do
local v, p, sz, sha = l:match(' | (.+) | (pool/.+%.deb) | (%d.+) | (%x.+) |')
if nil ~= p then
if APT.checkFile('results/' .. pu.host .. "/merged/" .. p) then
local status, fsz = APT.execute('ls -l results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 5-5')
if APT.testing("Integrity") then
if sz ~= fsz:sub(2, -2) then -- The sub bit is to slice off the EOLs at each end.
E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p .. ' should be ' .. sz .. ', but is ' .. fsz:sub(2, -2) .. '.', 'http', 'Integrity', pu.host)
else
local status, fsha = APT.execute('sha256sum results/' .. pu.host .. "/merged/" .. p .. ' | cut -d " " -f 1')
if sha ~= fsha:sub(2, -2) then E('Package SHA256 sum mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Integrity', pu.host) end
-- TODO - maybe check the PGP key, though packages are mostly not signed.
end
end
if APT.testing("Updated") then
if sz ~= fsz:sub(2, -2) then
E('Package size mismatch - results/' .. pu.host .. "/merged/" .. p, 'http', 'Updated', pu.host)
end
end
else
E('Failed to download - results/' .. pu.host .. "/merged/" .. p, 'http', 'Updated', pu.host)
end
end
end
end
end
end
else
for i, n in pairs(releases) do
os.execute('touch results/NEW_Packages_' .. n .. '.test.txt')
end
end
end
while nil ~= cor do
os.execute('sleep 10')
D('*&gt;* About to resume coroutine before writing results.')
APT.results["timeout"] = false
else
APT.results["timeout"] = true
end
end
local min, max, spd = 999999999999, 0
for i, mt in pairs({'Release', 'Packages', 'package'}) do
if APT.checkFile("results/curl-" .. mt .. "-" .. pu.host .. ".log") then
for l in io.lines("results/curl-" .. mt .. "-" .. pu.host .. ".log") do
local speed, crrnt = l:match('^%c *%d+ +%d+k? +%d+ +%d+k? +%d+ +%d+ +(%d+k?) +%d+ +[%d%-]+:[%d%-]+:[%d%-]+ +[%d%-]+:[%d%-]+:[%d%-]+ +[%d%-]+:[%d%-]+:[%d%-]+ +(%d+k?)')
if nil ~= speed then
if 'k' == speed:sub(-1, -1) then speed = speed:sub(1, -2) .. '000' end
if 'k' == crrnt:sub(-1, -1) then crrnt = crrnt:sub(1, -2) .. '000' end
speed = tonumber(speed)
crrnt = tonumber(crrnt)
if speed < min and speed ~= 0 then min = speed end
if speed > max then max = speed end
if crrnt < min and crrnt ~= 0 then min = crrnt end
if crrnt > max then max = crrnt end
end
if l:find('timed out') ~= nil then
E(" TIMEOUT " .. timeouts + 1 .. ', details in curl-' .. mt .. '-' .. pu.host .. '.log', 'http', '', pu.host)
timeouts = timeouts + 1
APT.results["timeout"] = true
end
end
end
local ok, message = coroutine.resume(cor)
if not ok then cor = nil; print(message); break end
end
APT.results["speed"] = {min = min, max = max}
local f = pu.host
if "" ~= ip then f = f .. "_" .. ip end
@ -891,13 +997,23 @@ else
local pu = url.parse("http://" .. m.BaseURL)
if APT.options.referenceSite.value ~= pu.host then
checkHost(m.BaseURL)
APT.checkExes("apt-panopticon.lua " .. sendArgs)
if APT.testing("Integrity") or APT.testing("Updated") then APT.checkExes(downloadLock) end
end
end
while 1 <= APT.checkExes("apt-panopticon.lua " .. sendArgs) do os.execute("sleep 10") end
os.execute('rm -f results/*.check; rm STATUS_* 2>/dev/null')
for k, v in pairs(APT.mirrors) do
local f = 'results/' .. k .. '.lua'
if APT.checkFile(f) then
results = loadfile(f)()
APT.mirrors[k]['IPs'] = results.IPs
end
end
local file, e = io.open("results/mirrors.lua", "w+")
if nil == file then C("opening mirrors file - " .. e) else
file:write(APT.dumpTable(APT.mirrors, "", "mirrors") .. "\nreturn mirrors\n")
file:close()
end
-- Create the reports.
for n, r in pairs(APT.options.reports.value) do

Write Preview
Loading…
Cancel
Save