onefang
/
apt-panopticon
2
2
Fork 0
Code Issues Releases Wiki Activity
Browse Source

Finish the executable refactor, by using it and removing the original.

master
onefang 3 years ago
parent
6e868b6382
commit
cafd4c5d4a
4 changed files with 60 additions and 65 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 54
      apt-panopticommon.lua
  2. 4
      apt-panopticon-report-email-web.lua
  3. 6
      apt-panopticon-update-data.lua
  4. 61
      apt-panopticon.lua

54
apt-panopticommon.lua
View File

@ -424,43 +424,24 @@ APT.testing = function(t, host)
return false
end
APT.execute = function (s, logit)
if nil == logit then logit = true end
if logit then D(" executing - &nbsp; <code>" .. s .. "</code>") end
--[[ Damn os.execute()
Lua 5.1 says it returns "a status code, which is system-dependent"
Lua 5.2 says it returns true/nil, "exit"/"signal", the status code.
I'm getting 7168 or 0. No idea what the fuck that is.
local ok, rslt, status = os.execute(s)
]]
local f = io.popen(s .. ' ; echo "$?"', 'r')
local status = ""
local result = ""
-- The last line will be the command's returned status, collect everything else in result.
for l in f:lines() do
result = result .. status .. "\n"
status = l
end
return status, result
end
APT.fork = function(s)
D(" forking - &nbsp; <code>" .. s .. "</code>")
os.execute(s .. " &")
end
APT.exe = function(c)
local exe = {status = 0, result = ''}
exe.cmd = c .. ' '
exe.log = false
local exe = {status = 0, result = '', log = true, cmd = c .. ' '}
function exe:log()
self.log = true
return self
end
function exe:Nice(c)
self.cmd = self.cmd .. 'ionice -c3 nice -n 19 ' .. c .. ' '
if nil == c then
self.cmd = 'ionice -c3 nice -n 19 ' .. self.cmd
else
self.cmd = self.cmd .. 'ionice -c3 nice -n 19 ' .. c .. ' '
end
return self
end
function exe:also(c)
if nil == c then c = '' else c = ' ' .. c end
self.cmd = self.cmd .. ';' .. c .. ' '
return self
end
function exe:And(c)
@ -482,7 +463,12 @@ APT.exe = function(c)
return self
end
function exe:Do()
if self.log then D(" executing - &nbsp; <code>" .. self.cmd .. "</code>") end
--[[ "The condition expression of a control structure can return any
value. Both false and nil are considered false. All values different
from nil and false are considered true (in particular, the number 0
and the empty string are also true)."
says the docs, I beg to differ.]]
if true == self.log then D(" executing - &nbsp; <code>" .. self.cmd .. "</code>") end
--[[ Damn os.execute()
Lua 5.1 says it returns "a status code, which is system-dependent"
Lua 5.2 says it returns true/nil, "exit"/"signal", the status code.
@ -491,6 +477,7 @@ APT.exe = function(c)
]]
local f = io.popen(self.cmd .. ' ; echo "$?"', 'r')
-- The last line will be the command's returned status, collect everything else in result.
self.status = '' -- Otherwise the result starts with 0.
for l in f:lines() do
self.result = self.result .. self.status .. "\n"
self.status = l
@ -500,14 +487,13 @@ APT.exe = function(c)
end
function exe:fork()
self.cmd = '{ ' .. self.cmd .. '; } &'
if self.log then D(" forking - &nbsp; <code>" .. self.cmd .. "</code>") end
if true == self.log then D(" forking - &nbsp; <code>" .. self.cmd .. "</code>") end
os.execute(self.cmd)
return self
end
return exe
end
APT.checkExes = function (exe)
local count = io.popen('ps x | grep "' .. exe .. '" | grep -v " grep " | grep -v "flock -n apt-panopticon.lock " | wc -l'):read("*l")
D(count .. " " .. exe .. " commands still running.")
@ -650,7 +636,7 @@ end
APT.now = 0
local status
status, APT.now = APT.execute('TZ="GMT" ls -l --time-style="+%s" results/stamp | cut -d " " -f 6-6')
APT.now = APT.exe('TZ="GMT" ls -l --time-style="+%s" results/stamp | cut -d " " -f 6-6'):Do().result
APT.now = tonumber(APT.now)
local start = 'now-1month'
local step = '10min'

4
apt-panopticon-report-email-web.lua
View File

@ -645,8 +645,8 @@ if nil == web then C("opening mirrors file - " .. e) else
"You can get the cgp graphing <a href='https://sledjhamr.org/cgit/apt-panopticon_cgp/about/'>source code here (main repo)</a>" ..
"and <a href='https://git.devuan.org/onefang/apt-panopticon_cgp'>here (Devuan repo)</a>.</p>\n"
)
local status, whn = APT.execute('TZ="GMT" ls -l1 --time-style="+%s" results/stamp | cut -d " " -f 6-6')
web:write( "<p>This run took " .. (os.time() - tonumber("0" .. whn:sub(2, -2))) .. " seconds.</p>" ..
local whn = APT.exe('TZ="GMT" ls -l1 --time-style="+%s" results/stamp | cut -d " " -f 6-6'):Do().result:sub(2, -2)
web:write( "<p>This run took " .. (os.time() - tonumber("0" .. whn)) .. " seconds.</p>" ..
"\n</body></html>\n")
web:close()
end

6
apt-panopticon-update-data.lua
View File

@ -106,9 +106,11 @@ local files = io.popen('ls -d1 results_*')
for l in files:lines() do
if ('results_old' ~= l) and ('.tar.xz' ~= l:sub(25, -1)) then
if APT.checkFile(l .. '/stamp') then
local status, whn = APT.execute('TZ="GMT" ls -d1 --time-style="+%s" ' .. l .. ' | cut -d " " -f 6-6')
-- local status, whn = APT.execute('TZ="GMT" ls -d1 --time-style="+%s" ' .. l .. ' | cut -d " " -f 6-6')
local whn = APT.exe('TZ="GMT" ls -d1 --time-style="+%s" ' .. l .. ' | cut -d " " -f 6-6'):Do()
whn = whn:sub(2, -2)
local status, new = APT.execute('TZ="GMT" date -d "' .. whn:sub(9, 18) .. ' ' .. whn:sub(20, 21) .. ':' .. whn:sub(23, 24) .. '" "+%s"')
-- local status, new = APT.execute('TZ="GMT" date -d "' .. whn:sub(9, 18) .. ' ' .. whn:sub(20, 21) .. ':' .. whn:sub(23, 24) .. '" "+%s"')
local new = APT.exe('TZ="GMT" date -d "' .. whn:sub(9, 18) .. ' ' .. whn:sub(20, 21) .. ':' .. whn:sub(23, 24) .. '" "+%s"'):Do()
APT.now = tonumber("0" .. new:sub(2, -2))
if 0 ~= APT.now then
if APT.checkFile(l .. '/mirrors.lua') then

61
apt-panopticon.lua
View File

@ -272,15 +272,16 @@ checkHEAD = function (host, URL, r, retry, sanity)
end
IP = '--connect-to "' .. pu.host .. '::' .. PU.host .. ':"'
end
local cmd = 'ionice -c3 nice -n 19 curl -I --retry 0 -s --path-as-is --connect-timeout ' .. APT.options.timeout.value .. ' --max-redirs 0 ' ..
IP .. ' ' .. '-o /dev/null -D results/"HEADERS_' .. fname .. '" ' ..
hdr .. ' -w "#%{http_code} %{ssl_verify_result} %{url_effective}\\n" ' .. PU.scheme .. '://' .. host .. PU.path .. ' >>results/"STATUS_' .. fname .. '"'
local status, result = APT.execute(cmd)
local status = APT.exe(
'curl -I --retry 0 -s --path-as-is --connect-timeout ' .. APT.options.timeout.value .. ' --max-redirs 0 ' ..
IP .. ' ' .. '-o /dev/null -D results/"HEADERS_' .. fname .. '" ' ..
hdr .. ' -w "#%{http_code} %{ssl_verify_result} %{url_effective}\\n" ' .. PU.scheme .. '://' .. host .. PU.path .. ' >>results/"STATUS_' .. fname .. '"'
):Nice():log():Do().status
os.execute('cat results/"HEADERS_' .. fname .. '" >>results/"STATUS_' .. fname .. '" 2>/dev/null; rm -f results/"HEADERS_' .. fname .. '" 2>/dev/null')
if "0" ~= status then
local msg = curlStatus[0 + status]
if 0 ~= status then
local msg = curlStatus[status]
if nil == msg then msg = "UNKNOWN CURL STATUS CODE!" end
if ("28" == status) or ("7" == status) then
if (28 == status) or (7 == status) then
if sanity then
T(" TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1, PU.scheme, "URLSanity", host)
else
@ -385,7 +386,8 @@ checkHEAD = function (host, URL, r, retry, sanity)
local pth = path:match('^(.*/pool/).*$')
if nil ~= pth then table.insert(APT.results[PU.scheme].redirects, pu.host .. "/" .. pth) else E('Odd redirect path ' .. path) end
I(" Now checking redirected host " .. u)
APT.fork("ionice -c3 nice -n 19 " .. downloadLock .. "REDIR-" .. check .. ".log.txt" .. " ./apt-panopticon.lua " .. extraArgs .. ' ' .. pu.host .. "/" .. path .. " " .. file)
-- APT.fork("ionice -c3 nice -n 19 " .. downloadLock .. "REDIR-" .. check .. ".log.txt" .. " ./apt-panopticon.lua " .. extraArgs .. ' ' .. pu.host .. "/" .. path .. " " .. file)
APT.exe(downloadLock .. "REDIR-" .. check .. ".log.txt" .. " ./apt-panopticon.lua " .. extraArgs .. ' ' .. pu.host .. "/" .. path .. " " .. file):Nice():log():fork()
D('logging to ' .. APT.logName(pu.host, nil, file)[2])
end
end
@ -457,7 +459,7 @@ checkHost = function (orig, host, path, ip, file)
else
if orig == host then
D("checkHost " .. orig .. "" .. file)
APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file)
APT.exe("./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file):Nice():log():fork()
D('logging to ' .. APT.logName(ph.host, nil, file)[2])
else D("checkHost " .. orig .. " -> " .. host) end
end
@ -492,8 +494,8 @@ local postDownload = function(host, r, k)
" results/" .. host .. "/merged/dists/" .. r .. '/' .. k .. ".old" ..
" results/" .. host .. "/merged/dists/" .. r .. '/' .. k .. "; fi")
if APT.checkFile('results/' .. host .. '/merged/dists/' .. r .. '/' .. k) then
if ".gz" == k:sub(-3, -1) then APT.execute("ionice -c3 nice -n 19 gzip -dfk results/" .. host .. "/merged/dists/" .. r .. '/' .. k, false) end
if ".xz" == k:sub(-3, -1) then APT.execute("ionice -c3 nice -n 19 xz -dfk results/" .. host .. "/merged/dists/" .. r .. '/' .. k .. " 2>/dev/null", false) end
if ".gz" == k:sub(-3, -1) then APT.exe("gzip -dfk results/" .. host .. "/merged/dists/" .. r .. '/' .. k):Nice():noErr():Do() end
if ".xz" == k:sub(-3, -1) then APT.exe("xz -dfk results/" .. host .. "/merged/dists/" .. r .. '/' .. k):Nice():noErr():Do() end
end
end
@ -509,7 +511,7 @@ local downloads = function(host, URL, meta, release, list)
local files = 'curl-' .. meta .. '-' .. host .. '.files.txt'
local lock = meta .. "-" .. host .. ".log.txt"
local log = "curl-" .. meta .. "-" .. host .. ".log.txt"
local cm = "ionice -c3 nice -n 19 " .. downloadLock .. lock .. " " .. download .. log .. " -K results/" .. files
local cm = downloadLock .. lock .. " " .. download .. log .. " -K results/" .. files
if APT.testing("IPv4") and (not APT.testing("IPv6")) then cm = cm .. ' -4' end
if (not APT.testing("IPv4")) and APT.testing("IPv6") then cm = cm .. ' -6' end
f, e = io.open("results/curl-" .. meta .. '-' .. host .. ".files.txt", "a+")
@ -539,7 +541,7 @@ local downloads = function(host, URL, meta, release, list)
end
end
f:close()
APT.fork(cm)
APT.exe(cm):Nice():log():fork()
D('logging to <a href="' .. log .. '">' .. log .. '</a>, with <a href="' .. files .. '">these files</a>')
end
@ -656,12 +658,14 @@ local parseDebs = function(host)
local v, p, sz, sha = l:match(' | (.+) | (pool/.+%.deb) | (%d.+) | (%x.+) |')
if nil ~= p then
if APT.checkFile('results/' .. host .. "/merged/" .. p) then
local status, fsz = APT.execute('ls -l results/' .. host .. "/merged/" .. p .. ' | cut -d " " -f 5-5', false)
-- local status, fsz = APT.execute('ls -l results/' .. host .. "/merged/" .. p .. ' | cut -d " " -f 5-5', false)
local fsz = APT.exe('ls -l results/' .. host .. "/merged/" .. p .. ' | cut -d " " -f 5-5'):Do().result
if APT.testing("Integrity") then
if sz ~= fsz:sub(2, -2) then -- The sub bit is to slice off the EOLs at each end.
E('Package size mismatch - results/' .. host .. "/merged/" .. p .. ' should be ' .. sz .. ', but is ' .. fsz:sub(2, -2) .. '.', 'http', 'Integrity', host)
else
local status, fsha = APT.execute('sha256sum results/' .. host .. "/merged/" .. p .. ' | cut -d " " -f 1')
-- local status, fsha = APT.execute('sha256sum results/' .. host .. "/merged/" .. p .. ' | cut -d " " -f 1')
local fsha = APT.exe('sha256sum results/' .. host .. "/merged/" .. p .. ' | cut -d " " -f 1'):Do().result
if sha ~= fsha:sub(2, -2) then E('Package SHA256 sum mismatch - results/' .. host .. "/merged/" .. p, 'http', 'Integrity', host) end
-- TODO - maybe check the PGP key, though packages are mostly not signed.
end
@ -779,7 +783,8 @@ end
local parseRelease = function(host)
local list = {inf = 'Release', parser = parsePackages, out = 'Packages', files = {}, nextf = 'debs'}
local updated = false
local status, now = APT.execute('TZ="GMT" date "+%s"', false)
-- local now = APT.execute('TZ="GMT" date "+%s"', false)
local now = os.date('!%s')
now = tonumber(now:sub(2, -2))
for i, n in pairs(releases) do
for l, o in pairs(releaseFiles) do
@ -787,9 +792,9 @@ local parseRelease = function(host)
postDownload(host, n, o)
if (".gpg" == o:sub(-4, -1)) and (APT.checkFile('results/' .. host .. '/merged/dists/' .. n .. '/' .. o)) then
if APT.testing("Integrity") then
local status, out = APT.execute("gpgv --keyring /usr/share/keyrings/devuan-keyring.gpg results/" .. host .. "/merged/dists/" .. n .. '/' .. o ..
" results/" .. host .. "/merged/dists/" .. n .. '/' .. o:sub(1, -5) .. " 2>/dev/null")
if "0" ~= status then E("GPG check failed - " .. host .. "/merged/dists/" .. n .. '/' .. o, "http", "Integrity", host) end
local status = APT.exe( "gpgv --keyring /usr/share/keyrings/devuan-keyring.gpg results/" .. host .. "/merged/dists/" .. n .. '/' .. o ..
" results/" .. host .. "/merged/dists/" .. n .. '/' .. o:sub(1, -5)):Nice():noErr():log():Do().status
if 0 ~= status then E("GPG check failed - " .. host .. "/merged/dists/" .. n .. '/' .. o, "http", "Integrity", host) end
-- TODO - should check the PGP sig of InRelease as well.
end
os.execute('rm results/' .. host .. '/merged/dists/' .. n .. '/' .. o)
@ -812,9 +817,11 @@ local parseRelease = function(host)
else
-- TODO - compare to the pkgmaster copy.
if APT.testing('Updated') then
local status, pkt = APT.execute([[TZ="GMT" date -d "$(grep '^Date:' results/pkgmaster.devuan.org/merged/dists/]] .. n .. [[/Release.SORTED | cut -d ' ' -f 2-)" '+%s']], false)
-- local status, pkt = APT.execute([[TZ="GMT" date -d "$(grep '^Date:' results/pkgmaster.devuan.org/merged/dists/]] .. n .. [[/Release.SORTED | cut -d ' ' -f 2-)" '+%s']], false)
local pkt = APT.exe([[TZ="GMT" date -d "$(grep '^Date:' results/pkgmaster.devuan.org/merged/dists/]] .. n .. [[/Release.SORTED | cut -d ' ' -f 2-)" '+%s']]):Do().result
pkt = tonumber(pkt:sub(2, -2))
local status, new = APT.execute([[TZ="GMT" date -d "$(grep '^Date:' results/]] .. host .. [[/merged/dists/]] .. n .. [[/Release.SORTED | cut -d ' ' -f 2-)" '+%s']], false)
-- local status, new = APT.execute([[TZ="GMT" date -d "$(grep '^Date:' results/]] .. host .. [[/merged/dists/]] .. n .. [[/Release.SORTED | cut -d ' ' -f 2-)" '+%s']], false)
local new = APT.exe([[TZ="GMT" date -d "$(grep '^Date:' results/]] .. host .. [[/merged/dists/]] .. n .. [[/Release.SORTED | cut -d ' ' -f 2-)" '+%s']]):Do().result
new = tonumber(new:sub(2, -2))
local upd = pkt + APT.mirrors[host].Updated
local updd = pkt + (APT.mirrors[host].Updated * 1.5) -- Give the mirror time to actually do the update.
@ -883,7 +890,7 @@ local doDownloads = function(host, path, list)
Mon Dec 23 01:02:54 2019 DEBUG : forking
ionice -c3 nice -n 19 flock -n results/curl-debs-pkgmaster.devuan.org.log curl --connect-timeout 5 --create-dirs -f -L --fail-early --max-time 300 --retry 3 -R -v -z 'results/stamp.old' --stderr results/curl-debs-pkgmaster.devuan.org.log -K results/curl-debs-pkgmaster.devuan.org.files
flock -n results/curl-debs-pkgmaster.devuan.org.log curl --connect-timeout 5 --create-dirs -f -L --fail-early --max-time 300 --retry 3 -R -v -z 'results/stamp.old' --stderr results/curl-debs-pkgmaster.devuan.org.log -K results/curl-debs-pkgmaster.devuan.org.files
Mon Dec 23 01:02:54 2019 DEBUG : 0 flock -n results/curl-debs-pkgmaster.devuan.org.log commands still running.
@ -998,17 +1005,17 @@ if 0 < #arg then
if "table" == type(v) then
for k1, v1 in pairs(v) do
if v1 == "A" then
if APT.testing("IPv4") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k1 .. " " .. file) end
if APT.testing("IPv4") then APT.exe("./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k1 .. " " .. file):Nice():log():fork() end
elseif v1 == "AAAA" then
if APT.testing("IPv6") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k1 .. " " .. file) end
if APT.testing("IPv6") then APT.exe("./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k1 .. " " .. file):Nice():log():fork() end
end
D('logging to ' .. APT.logName(pu.host, k1, file)[2])
end
else
if v == "A" then
if APT.testing("IPv4") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k .. " " .. file) end
if APT.testing("IPv4") then APT.exe("./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k .. " " .. file):Nice():log():fork() end
elseif v == "AAAA" then
if APT.testing("IPv6") then APT.fork("ionice -c3 ./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k .. " " .. file) end
if APT.testing("IPv6") then APT.exe("./apt-panopticon.lua " .. sendArgs .. " " .. pu.host .. path .. " " .. k .. " " .. file):Nice():log():fork() end
end
D('logging to ' .. APT.logName(pu.host, k, file)[2])
end
@ -1212,7 +1219,7 @@ os.execute('sleep 1') -- Wait for things to start up before checking for them.
for n, r in pairs(APT.options.reports.value) do
if APT.checkFile("apt-panopticon-report-" .. r .. ".lua") then
I("Creating " .. r .. " report.")
APT.execute("./apt-panopticon-report-" .. r .. ".lua " .. sendArgs)
APT.exe("./apt-panopticon-report-" .. r .. ".lua " .. sendArgs):log():Do()
end
end

Write Preview
Loading…
Cancel
Save