Browse Source

Track, store, and report info from the hosts that get redirected to.

master
onefang 3 years ago
parent
commit
0f43ca83d6
  1. 38
      apt-panopticommon.lua
  2. 6
      apt-panopticon-report-RRD.lua
  3. 201
      apt-panopticon-report-email-web.lua
  4. 103
      apt-panopticon.lua

38
apt-panopticommon.lua

@ -53,7 +53,7 @@ APT.options =
"DNSRR",
"Protocol",
"URLSanity",
"Integrity",
-- "Integrity",
"Updated",
},
},
@ -411,6 +411,7 @@ local E = APT.E
local C = APT.C
APT.debians = {}
APT.mirrors = {}
APT.testing = function(t, host)
@ -498,6 +499,7 @@ APT.padResults = function(results)
if nil == tests.Integrity then tests.Integrity = {errors = c; warnings = c; timeouts = c} end
if nil == tests.Protocol then tests.Protocol = {errors = c; warnings = c; timeouts = c} end
if nil == tests.Updated then tests.Updated = {errors = c; warnings = c; timeouts = c} end
if nil == tests.redirects then tests.redirects = {} end
if nil == tests.URLSanity then tests.URLSanity = {errors = c; warnings = c; timeouts = c} end
results[v] = tests
end
@ -517,23 +519,25 @@ APT.collate = function(l, host, ip, results)
if ("speed" == k) and (nil ~= results.speed) then
if v.min < results.speed.min then results.speed.min = v.min end
if v.max > results.speed.max then results.speed.max = v.max end
else
elseif 'IPs' ~= k then
for i, u in pairs(v) do
if "table" == type(u) then
for h, t in pairs(u) do
if 'redirects' ~= i then
if "table" == type(u) then
for h, t in pairs(u) do
local a = results[k]
if nil == a then results[k] = {i = {}}; a = results[k] end
a = a[i]
if nil == a then results[k][i] = {h = {}}; a = results[k][i] end
a = a[h]
if nil == a then a = 0 end
results[k][i][h] = a + t
end
else
local a = results[k]
if nil == a then results[k] = {i = {}}; a = results[k] end
a = a[i]
if nil == a then results[k][i] = {h = {}}; a = results[k][i] end
a = a[h]
if nil == a then a = 0; results[k] = {} else a = a[i] end
if nil == a then a = 0 end
results[k][i][h] = a + t
results[k][i] = a + u
end
else
local a = results[k]
if nil == a then a = 0; results[k] = {} else a = a[i] end
if nil == a then a = 0 end
results[k][i] = a + u
end
end
end
@ -546,14 +550,14 @@ APT.collate = function(l, host, ip, results)
end
return results
end
APT.collateAll = function(l, host, func)
APT.collateAll = function(hosts, l, host, func)
results = {}
local f = l .. "/" .. host .. ".lua"
if APT.checkFile(f) then
results = loadfile(f)()
results = APT.padResults(results)
if nil ~= func then func(results) end
local v = APT.mirrors[host]
local v = hosts[host]
if nil ~= v then
local IPs = results.IPs
if nil == IPs then W('No IPs for ' .. host .. ' in ' .. l) else
@ -662,7 +666,7 @@ APT.updateRRD = function(results, host, ip)
end
APT.doRRD = function(l, k, v, o)
APT.collateAll(l, k,
APT.collateAll(APT.mirrors, l, k,
function(results, ip)
APT.createRRD(k, ip, o)
APT.updateRRD(results, k, ip)

6
apt-panopticon-report-RRD.lua

@ -15,3 +15,9 @@ APT.html = false
for k, v in APT.orderedPairs(APT.mirrors) do
APT.doRRD('results', k, v)
end
APT.debians = loadfile("results/debians.lua")()
APT.html = false
for k, v in APT.orderedPairs(APT.debians) do
APT.doRRD('results', k, v)
end

201
apt-panopticon-report-email-web.lua

@ -12,11 +12,12 @@ local arg, sendArgs = APT.parseArgs({...})
local results = {}
APT.mirrors = loadfile("results/mirrors.lua")()
APT.debians = loadfile("results/debians.lua")()
local revDNS = function(dom, IP)
local revDNS = function(hosts, dom, IP)
if "deb.devuan.org" ~= dom then
if nil ~= APT.mirrors["deb.devuan.org"] then
if nil ~= APT.mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][IP] then
if nil ~= hosts["deb.devuan.org"] then
if nil ~= hosts["deb.devuan.org"].IPs["deb.roundr.devuan.org"][IP] then
if APT.html then
return "<font color='purple'><b>DNS-RR</b></font>"
else
@ -25,7 +26,7 @@ local revDNS = function(dom, IP)
end
end
else
for k, v in pairs(APT.mirrors) do
for k, v in pairs(hosts) do
if "deb.devuan.org" ~= k then
local IPs = v.IPs
for i, u in pairs(IPs) do
@ -44,12 +45,12 @@ local revDNS = function(dom, IP)
end
local faulty = ""
local status = function(host, results, typ)
local status = function(hosts, host, results, typ)
local result = ""
local e = 0
local w = 0
local t = 0
local s = nil ~= APT.mirrors[host].Protocols[typ]
local s = nil ~= hosts[host].Protocols[typ]
local to = results.timeout
if not APT.search(APT.protocols, typ) then s = true end
if nil ~= results[typ] then
@ -57,7 +58,7 @@ local status = function(host, results, typ)
w = results[typ].warnings
t = results[typ].timeouts
for k, v in pairs(results[typ]) do
if "table" == type(v) then
if ("table" == type(v)) and ('redirects' ~= k) then
if 0 <= v.errors then e = e + v.errors else to = true end
if 0 <= v.warnings then w = w + v.warnings else to = true end
if 0 <= v.timeouts then t = t + v.timeouts else to = true end
@ -139,31 +140,49 @@ local logCount = function(domain, ip)
local nm = "LOG_" .. domain
local log = ""
local extra = ""
local errors = 0
local warnings = 0
local timeouts = 0
if nil ~= ip then nm = nm .. "_" .. ip end
nm = nm .. ".html"
local rfile, e = io.open("results/" .. nm, "r")
if nil ~= rfile then
local errors = 0
local warnings = 0
local timeouts = 0
for l in rfile:lines() do
if nil ~= l:match("><b>ERROR ") then errors = errors + 1 end
if nil ~= l:match("><b>WARNING ") then warnings = warnings + 1 end
if nil ~= l:match("><b>TIMEOUT ") then timeouts = timeouts + 1 end
end
rfile:close()
if APT.html then
if nil == ip then
log = "<a href='" .. nm .. "'>" .. domain .. "</a>"
else
log = "<a href='" .. nm .. "'>" .. ip .. "</a>"
end
end
if APT.html then
if nil == ip then
log = "<a href='" .. nm .. "'>" .. domain .. "</a>"
else
log = "<a href='" .. nm .. "'>" .. ip .. "</a>"
end
log = log .. APT.plurals(errors, warnings, timeouts)
end
log = log .. APT.plurals(errors, warnings, timeouts)
return log
end
local redirs = function(hosts, host)
local results = APT.collateAll(hosts, 'results', host)
local rdr = {}
local redirs = ''
for p, pt in pairs(APT.protocols) do
if 0 ~= #(results[pt].redirects) then
table.sort(results[pt].redirects)
for r, rd in pairs(results[pt].redirects) do
rdr[rd] = rd
end
end
end
for r, rd in pairs(rdr) do
redirs = redirs .. ', &nbsp ' .. rd
end
if '' ~= redirs then redirs = '<br>\n &nbsp; &nbsp; (Redirects some packages to - ' .. redirs:sub(3) .. ')' end
return redirs
end
APT.html = false
local email, e = io.open("results/Report-email.txt", "w+")
@ -183,16 +202,16 @@ if nil == email then C("opening mirrors file - " .. e) else
"[skip] means that the test hasn't been written yet.\n\n")
for k, v in APT.orderedPairs(APT.mirrors) do
email:write(k .. "....\n")
local results = APT.collateAll('results', k)
local results = APT.collateAll(APT.mirrors, 'results', k)
local ftp = "[skip]"
local http = status(k, results, "http")
local https = status(k, results, "https")
local http = status(APT.mirrors, k, results, "http")
local https = status(APT.mirrors, k, results, "https")
local rsync = "[skip]"
local dns = ""
local protocol = status(k, results, "Protocol")
local sanity = status(k, results, "URLSanity")
local integrity = status(k, results, "Integrity")
local updated = status(k, results, "Updated")
local protocol = status(APT.mirrors, k, results, "Protocol")
local sanity = status(APT.mirrors, k, results, "URLSanity")
local integrity = status(APT.mirrors, k, results, "Integrity")
local updated = status(APT.mirrors, k, results, "Updated")
-- DNS-RR test.
if ("deb.devuan.org" ~= k) and (nil ~= APT.mirrors["deb.devuan.org"]) then
@ -347,7 +366,7 @@ if nil == web then C("opening mirrors file - " .. e) else
"<th>Protocol</th><th>URL sanity</th><th>Integrity</th><th>Updated</th><th colspan='2'>Speed range</th></tr>\n"
)
for k, v in APT.orderedPairs(APT.mirrors) do
local results = APT.collateAll('results', k)
local results = APT.collateAll(APT.mirrors, 'results', k)
local active = ""
if "yes" == v.Active then
web:write(" <tr><th>" .. k .. "</th> ")
@ -356,14 +375,14 @@ if nil == web then C("opening mirrors file - " .. e) else
web:write(" <tr style='background-color:dimgrey'><th>" .. k .. "</th> ")
end
local ftp = "[<font color='grey'><b>skip</b></font>]"
local http = status(k, results, "http")
local https = status(k, results, "https")
local http = status(APT.mirrors, k, results, "http")
local https = status(APT.mirrors, k, results, "https")
local rsync = "[<font color='grey'><b>skip</b></font>]"
local dns = ""
local protocol = status(k, results, "Protocol")
local sanity = status(k, results, "URLSanity")
local integrity = status(k, results, "Integrity")
local updated = status(k, results, "Updated")
local protocol = status(APT.mirrors, k, results, "Protocol")
local sanity = status(APT.mirrors, k, results, "URLSanity")
local integrity = status(APT.mirrors, k, results, "Integrity")
local updated = status(APT.mirrors, k, results, "Updated")
local rate = v.Rate
if nil ~= rate then updated = updated .. ' ' .. rate end
local min = tonumber(results.speed.min)
@ -434,14 +453,14 @@ if nil == web then C("opening mirrors file - " .. e) else
n[l] = {}
for i, u in pairs(w) do
local log = logCount(k, i)
if "" == log then n[l][i] = u else n[l][log .. " " .. revDNS(k, i)] = u end
if "" == log then n[l][i] = u else n[l][log .. " " .. revDNS(APT.mirrors, k, i)] = u end
end
else
local log = logCount(k, l)
if "" == log then n[l] = w else n[log .. " " .. revDNS(k, l)] = w end
if "" == log then n[l] = w else n[log .. " " .. revDNS(APT.mirrors, k, l)] = w end
end
end
m[log .. " DNS entries -"] = n
m[log .. " DNS entries -" .. redirs(APT.mirrors, k)] = n
end
web:write( "<p>This lists each mirror, and the DNS entries for that mirror. &nbsp; " ..
"The links point to the testing log files for " .. logCount("apt-panopticon") .. " for each domain name / IP combination that was tested. &nbsp; " ..
@ -453,8 +472,118 @@ if nil == web then C("opening mirrors file - " .. e) else
)
web:write(APT.dumpTableHTML(m, "", ""))
web:write( "\n<br>\n<br>\n<h2>==== graphs: ====</h2>\n" ..
"<img src='speed.png'>\n<br>\n<p><a href='../apt-panopticon_cgp/'>More graphs.</a> with greater detail.</p><hr>\n\n" ..
"<p>The <a href='Report-email.txt'>email report</a>. &nbsp; " ..
"<img src='speed.png'>\n<br>\n<p><a href='../apt-panopticon_cgp/'>More graphs.</a> with greater detail.</p><hr>\n\n")
results = {}
m = {}
faulty = ""
web:write( "<hr>\n<h2>==== Debian mirror status ====</h2>\n" ..
"<p>NOTE - This is not fully probing the Debian mirrors, we just collect some data from any redirects to other servers. &nbsp; " ..
"So this isn't a full set of tests.</p>\n" ..
"<p><font style='background-color:red; color:black'>EXPERIMENTAL CODE - this is even more experimental than the rest.</font></p>\n" ..
"<table>\n<tr><th></th><th>FTP</th><th>HTTP</th><th>HTTPS</th><th>RSYNC</th><th>DNS round robin</th>" ..
"<th>Protocol</th><th>URL sanity</th><th>Integrity</th><th>Updated</th><th colspan='2'>Speed range</th></tr>\n")
for k, v in APT.orderedPairs(APT.debians) do
local results = APT.collateAll(APT.debians, 'results', k)
local active = ""
if "yes" == v.Active then
web:write(" <tr><th>" .. k .. "</th> ")
else
if nil == v.Active then active = 'nil' else active = v.Active end
web:write(" <tr style='background-color:dimgrey'><th>" .. k .. "</th> ")
end
local ftp = "[<font color='grey'><b>skip</b></font>]"
local http = status(APT.debians, k, results, "http")
local https = status(APT.debians, k, results, "https")
local rsync = "[<font color='grey'><b>skip</b></font>]"
local dns = ""
local protocol = status(APT.debians, k, results, "Protocol")
local sanity = status(APT.debians, k, results, "URLSanity")
local integrity = status(APT.debians, k, results, "Integrity")
local updated = status(APT.debians, k, results, "Updated")
local rate = v.Rate
if nil ~= rate then updated = updated .. ' ' .. rate end
local min = tonumber(results.speed.min)
local max = tonumber(results.speed.max)
local spd = ''
-- DNS-RR test.
if ("deb.devuan.org" ~= k) and (nil ~= APT.debians["deb.devuan.org"]) then
for l, w in pairs(APT.debians[k].IPs) do
if type(w) == "table" then
for i, u in pairs(w) do
if nil ~= APT.debians["deb.devuan.org"].IPs["deb.roundr.devuan.org"][i] then
local log = logCount("deb.devuan.org", i)
if "" ~= log then
if "" == dns then dns = " " else dns = dns .. " &nbsp; " end
dns = dns .. logCount("deb.devuan.org", i)
else
if "" == dns then dns = " " else dns = dns .. " &nbsp; " end
dns = dns .. "<font color='maroon'><b>" .. i .. "</b></font>"
end
end
end
else
if nil ~= APT.debians["deb.devuan.org"].IPs["deb.roundr.devuan.org"][l] then
local log = logCount("deb.devuan.org", l)
if "" ~= log then
if "" == dns then dns = " " else dns = dns .. " &nbsp; " end
dns = dns .. log
else
if "" == dns then dns = " " else dns = dns .. " &nbsp; " end
dns = dns .. "<font color='maroon'><b>" .. l .. "</b></font>"
end
end
end
end
if "" == dns then dns = "[<font color='grey'><b>no</b></font>]" end
if 0 == max then
spd = '<td></td><td></td>'
else
spd = string.format('<td align="right">%d -></td><td align="right">%d</td>', min, max)
end
end
web:write("<td>" .. ftp .. "&nbsp;</td><td>" .. http .. "&nbsp;</td><td>" .. https .. "&nbsp;</td><td>" .. rsync .. "&nbsp;</td><td>" .. dns ..
"&nbsp;</td><td>" .. protocol .. "&nbsp;</td><td>" .. sanity ..
"&nbsp;</td><td>" .. integrity .. "&nbsp;</td><td>" .. updated .. "&nbsp;</td>" .. spd .. "</tr>\n")
if "" ~= active then
web:write("<tr><td style='background-color:dimgrey'>" .. active .. "</td></tr>\n")
end
end
web:write( "</table>\n<br>\n\n")
web:write( "<br>\n<br>\n<h2>==== Debian DNS and logs: ====</h2>\n")
for k, v in pairs(APT.debians) do
local log = k
local n = {}
log = logCount(k)
APT.debians[k].Protocols = nil
APT.debians[k].FQDN = nil
APT.debians[k].Active = nil
APT.debians[k].Rate = nil
APT.debians[k].BaseURL = nil
APT.debians[k].Country = nil
APT.debians[k].Bandwidth = nil
for l, w in pairs(APT.debians[k].IPs) do
if type(w) == "table" then
n[l] = {}
for i, u in pairs(w) do
local log = logCount(k, i)
if "" == log then n[l][i] = u else n[l][log .. " " .. revDNS(APT.debians, k, i)] = u end
end
else
local log = logCount(k, l)
if "" == log then n[l] = w else n[log .. " " .. revDNS(APT.debians, k, l)] = w end
end
end
m[log .. " DNS entries -" .. redirs(APT.mirrors, k)] = n
end
web:write(APT.dumpTableHTML(m, "", ""))
web:write( "<hr>\n<hr>\n<p>The <a href='Report-email.txt'>email report</a>. &nbsp; " ..
"All <a href='../results'>the logs and other output</a>. &nbsp; " ..
"You can get the <a href='https://sledjhamr.org/cgit/apt-panopticon/about/'>source code here</a>.</p>\n")
local status, whn = APT.execute('TZ="GMT" ls -l1 --time-style="+%s" results/stamp | cut -d " " -f 6-6')

103
apt-panopticon.lua

@ -376,6 +376,8 @@ checkHEAD = function (host, URL, r, retry, sanity)
local extraArgs = sendArgs .. ' -o -r '
if 'https' == pu.scheme then extraArgs = extraArgs .. ' --tests=-http' end
if 'http' == pu.scheme then extraArgs = extraArgs .. ' --tests=-https' end
local pth = path:match('^(.*/pool/).*$')
if nil ~= pth then table.insert(APT.results[PU.scheme].redirects, pu.host .. "/" .. pth) else E('Odd redirect path ' .. path) end
I(" Now checking redirected host " .. u)
APT.fork("ionice -c3 nice -n 19 " .. downloadLock .. "REDIR-" .. check .. ".log.txt" .. " ./apt-panopticon.lua " .. extraArgs .. ' ' .. pu.host .. "/" .. path .. " " .. file)
D('logging to ' .. APT.logName(pu.host, nil, file)[2])
@ -422,12 +424,10 @@ local checkFiles = function (host, ip, path, file)
end
end
end
for i, s in pairs(referenceDebs) do
if checkTimeouts(host, "http", ip .. path .. "/" .. s) then return end
if checkTimeouts(host, "https", ip .. path .. "/" .. s) then return end
end
for i, s in pairs(referenceDevs) do
if checkTimeouts(host, "http", ip .. path .. "/" .. s) then return end
if checkTimeouts(host, "https", ip .. path .. "/" .. s) then return end
@ -994,7 +994,7 @@ if 0 < #arg then
rfile:close()
end
if APT.origin and APT.options.referenceSite.value ~= pu.host then
if APT.origin and (not APT.redir) and (APT.options.referenceSite.value ~= pu.host) then
os.execute('sleep 1') -- Wait for things to start up before checking for them.
while 0 < APT.checkExes(downloadLock .. "Release-" .. pu.host .. ".log.txt") do os.execute("sleep 10") end
while 0 < APT.checkExes(downloadLock .. "Packages-" .. pu.host .. ".log.txt") do os.execute("sleep 10") end
@ -1038,6 +1038,103 @@ else
os.execute('sleep 1') -- Wait for things to start up before checking for them.
while 1 <= APT.checkExes("apt-panopticon.lua " .. sendArgs) do os.execute("sleep 10") end
local APT_args = APT.args
local APT_logFile = APT.logFile
local debians = {}
local srvs = io.popen('ls -1 results/*.lua')
for l in srvs:lines() do
local hst = l:sub(9, -5)
if (l:find('_') == nil) and (nil == APT.mirrors[hst]) then
local ips = loadfile(l)().IPs
if nil ~= ips then
debians[hst] = {Country = '', FDQN = hst, Active = 'yes', Rate = '', BaseURL = hst, Protocols = {http = true, https = true}, Bandwidth = '', IPs = ips}
local baseFiles = {}
local IPfiles = {}
for i, a in pairs(ips) do
IPfiles[i] = {}
if type(a) == 'table' then
for j, b in pairs(a) do
IPfiles[i][j] = {}
end
else
end
end
local files = io.popen('ls -1 results/LOG_' .. hst .. '_*.html')
for ll in files:lines() do
local dn = false
for i, a in pairs(ips) do
if type(a) == 'table' then
for j, b in pairs(a) do
if nil ~= ll:match('(results/LOG_' .. hst .. '_' .. j .. '_.*)') then
table.insert(IPfiles[i][j], ll)
dn = true
end
end
else
if nil ~= ll:match('(results/LOG_' .. hst .. '_' .. i .. '_.*)') then
table.insert(IPfiles[i], ll)
dn = true
end
end
end
if not dn then table.insert(baseFiles, ll) end
end
APT.logOpen(hst)
APT.logFile:write('<h1>Note log lines will be out of order, this is a bunch of other log files combined.</h1>\n')
for i, f in pairs(baseFiles) do
f = f:sub(9, -1)
APT.logFile:write('<hr>\n<hr>\n<h2><a href="' .. f .. '">' .. f .. '</a></h2>\n')
for l in io.lines('results/' .. f) do
if l:match('^' .. os.date('%a %b %d ') .. '.*$') then APT.logFile:write(l .. '\n') end
end
end
APT.logPost()
APT.args = APT_args
APT.logFile = APT_logFile
for ip, a in pairs(IPfiles) do
if nil == a[1] then
for i, f in pairs(a) do
if not APT.logOpen(hst, i) then print('PROBLEM OPENING LOG FILE ' .. hst .. ' ' .. i) end
APT.logFile:write('<h1>Note log lines will be out of order, this is a bunch of other log files combined.</h1>\n')
for j, g in pairs(f) do
g = g:sub(9, -1)
APT.logFile:write('<hr>\n<hr>\n<h2><a href="' .. g .. '">' .. g .. '</a></h2>\n')
for l in io.lines('results/' .. g) do
if l:match('^' .. os.date('%a %b %d ') .. '.*$') then APT.logFile:write(l .. '\n') end
end
end
APT.logPost()
APT.args = APT_args
APT.logFile = APT_logFile
end
else
if not APT.logOpen(hst, ip) then print('PROBLEM OPENING LOG FILE ' .. hst .. ' ' .. ip) end
APT.logFile:write('<h1>Note log lines will be out of order, this is a bunch of other log files combined.</h1>\n')
for i, f in pairs(a) do
f = f:sub(9, -1)
APT.logFile:write('<hr>\n<hr>\n<h2><a href="' .. f .. '">' .. f .. '</a></h2>\n')
for l in io.lines('results/' .. f) do
if l:match('^' .. os.date('%a %b %d ') .. '.*$') then APT.logFile:write(l .. '\n') end
end
end
APT.logPost()
APT.args = APT_args
APT.logFile = APT_logFile
end
end
end
end
end
local file, e = io.open("results/debians.lua", "w+")
if nil == file then C("opening debians file - " .. e) else
file:write(APT.dumpTable(debians, "", "debians") .. "\nreturn debians\n")
file:close()
end
for k, v in pairs(APT.mirrors) do
local f = 'results/' .. k .. '.lua'
if APT.checkFile(f) then

Loading…
Cancel
Save