apt-panopticon/apt-panopticon-update-data.lua

#!/usr/bin/env luajit

local APT = require 'apt-panopticommon'
local D = APT.D
local I = APT.I
local T = APT.T
local W = APT.W
local E = APT.E
local C = APT.C
local arg, sendArgs = APT.parseArgs({...})

local oldest = os.date('%s') - (60 * 60)


local lock = 'apt-panopticon.lock'
while APT.checkFile(lock) do os.execute('sleep 10') end
os.execute('touch ' .. lock)


if -1 < APT.verbosity then print('\nUpacking tarballs and cleaning them.') end
local files = io.popen('ls -1 results_*/results_*.tar.xz 2>/dev/null; ls -1 results_*.tar.xz 2>/dev/null')
for l in files:lines() do
    local lsub = l:sub(1, -8)

    local dt = os.date('%s', os.time{year = l:sub(-23, -20), month = l:sub(-18, -17), day = l:sub(-15, -14), hour = l:sub(-12, -11), min = l:sub(-9, -8)}) - (60 * 60)
    if dt < oldest then oldest = dt; print('\nOldest time is ' .. oldest) end

    if -1 < APT.verbosity then io.stdout:write('\r' .. 'tar -xf ' .. l);  io.stdout:flush() end
    os.execute('tar -xf ' .. l)

    os.execute('rm -f ' .. lsub .. '/*.curl')
    os.execute('rm -f ' .. lsub .. '/STATUS_*')

    if APT.checkFile(lsub .. '/mirrors.lua') then
	APT.mirrors = loadfile(lsub .. "/mirrors.lua")()
	for k, v in pairs(APT.mirrors) do
	    if APT.options.referenceSite.value ~= k then
		if APT.checkFile(lsub .. '/' .. k) then
		    os.execute('rm -fr ' .. lsub .. '/' .. k)
		end
	    else
		for i, n in pairs(APT.releases) do
		    os.execute( 'rm -f  ' .. lsub .. '/' .. k .. '/merged/dists/' .. n .. '/Release;' ..
				'rm -f  ' .. lsub .. '/' .. k .. '/merged/dists/' .. n .. '/Release.gpg;' ..
				'rm -fr ' .. lsub .. '/' .. k .. '/merged/dists/' .. n .. '/contrib 2>/dev/null;' ..
				'rm -fr ' .. lsub .. '/' .. k .. '/merged/dists/' .. n .. '/main 2>/dev/null;' ..
				'rm -fr ' .. lsub .. '/' .. k .. '/merged/dists/' .. n .. '/non-free 2>/dev/null')
		end
		os.execute('rm -fr ' .. lsub .. '/' .. k .. '/merged/pool')
	    end

	    local results = {}
	    local f = l:sub(1, -8) .. "/" .. k .. ".lua"
	    if APT.checkFile(f) then
		results = loadfile(f)()
		results['IPs'] = v.IPs
		local rfile, e = io.open(f, "w+")
		if nil == rfile then C("opening results file - " .. e) else
		    rfile:write(APT.dumpTable(results, "results") .. "\nreturn results\n")
		    rfile:close()
		end
	    end

	end
    end

    local fls = io.popen('ls -1 ' .. lsub .. '/*.lua 2>/dev/null')
    for k in fls:lines() do
	if APT.checkFile(k) then
	    local results = loadfile(k)()
	    results = APT.padResults(results)
	    for i, w in pairs(APT.protocols) do
		tests = results[w]
		if nil ~= tests.Redirects then
		    tests.redirects = tests.Redirects
		    tests.Redirects = nil
		elseif nil == tests.redirects then
		    tests.redirects = {}
		end
		results[w] = tests
	    end
	    if nil ~= results.Redirects then
		for i, w in pairs(APT.protocols) do
		    results[w].redirects  = results.Redirects
		end
		results.Redirects = nil
	    end
	    local rfile, e = io.open(k, "w+")
	    if nil == rfile then C("opening results file - " .. e) else
		rfile:write(APT.dumpTable(results, "results") .. "\nreturn results\n")
		rfile:close()
	    end
	end
    end
end


if -1 < APT.verbosity then print('\nDeleting old RRD data.') end
os.execute('rm -fr rrd')
if -1 < APT.verbosity then print('\nRecreating RRD data.') end
local files = io.popen('ls -d1 results_*')
for l in files:lines() do
    if ('results_old' ~= l) and ('.tar.xz' ~= l:sub(25, -1)) then
	if APT.checkFile(l .. '/stamp') then
	    APT.now = os.date('%s', os.time{year = l:sub(9, 12), month = l:sub(14, 15), day = l:sub(17, 18), hour = l:sub(20, 21), min = l:sub(23, 24)})
	    if 0 ~= APT.now then
		if APT.checkFile(l .. '/mirrors.lua') then
		if -1 < APT.verbosity then io.stdout:write('\r' .. l .. ' -> ' .. APT.now);  io.stdout:flush() end
		    APT.mirrors = loadfile(l .. "/mirrors.lua")()
		    for k, v in pairs(APT.mirrors) do
			APT.doRRD(l, k, v, oldest)
		    end
		else
		    if -1 < APT.verbosity then io.stdout:write('\r' .. l);  io.stdout:flush() end
		end
	    else
		if -1 < APT.verbosity then io.stdout:write('\r' .. l);  io.stdout:flush() end
	    end
	else
	    if -1 < APT.verbosity then io.stdout:write('\r' .. l);  io.stdout:flush() end
	end
    end
end


os.execute('rm ' .. lock)


if -1 < APT.verbosity then print('\nRecreating tarballs.') end

local files = io.popen('ls -1 results_*/results_*.tar.xz 2>/dev/null')
for l in files:lines() do
    if -1 < APT.verbosity then io.stdout:write('\r' .. 'tar -c --xz  ' .. l:sub(20, 43) .. ' -f ' .. l);  io.stdout:flush() end
    os.execute('mkdir -p ' .. l:sub(1, 18) .. '; tar -c --xz  ' .. l:sub(20, 43) .. ' -f ' .. l .. '; rm -fr ' .. l:sub(20, 43))
end
local files = io.popen('ls -1 results_*.tar.xz 2>/dev/null')
for l in files:lines() do
    if -1 < APT.verbosity then io.stdout:write('\r' .. 'tar -c --xz  ' .. l:sub(1, 24) .. ' -f ' .. l:sub(1, 18) .. '/' .. l);  io.stdout:flush() end
    os.execute('mkdir -p ' .. l:sub(1, 18) .. '; tar -c --xz  ' .. l:sub(1, 24) .. ' -f ' .. l:sub(1, 18) .. '/' .. l .. '; rm -fr ' .. l:sub(1, 24) .. '; rm -f ' .. l)
end

local rslt  = APT.exe( 'ls -dl results | cut -d " " -f 11'):Do().result:sub(2, -2)
local rslto = APT.exe( 'ls -dl results_old | cut -d " " -f 11'):Do().result:sub(2, -2)
local files = io.popen('ls -d1 results_*-*-*-*-* 2>/dev/null')
for l in files:lines() do
    if (rslt ~= l) and (rslto ~= l) then
	if -1 < APT.verbosity then io.stdout:write('\r' .. 'tar -c --xz  ' .. l:sub(1, 24) .. ' -f ' .. l:sub(1, 18) .. '/' .. l .. '.tar.xz');  io.stdout:flush() end
	os.execute('mkdir -p ' .. l:sub(1, 18) .. '; tar -c --xz  ' .. l:sub(1, 24) .. ' -f ' .. l:sub(1, 18) .. '/' .. l .. '; rm -fr ' .. l:sub(1, 24) .. '; rm -f ' .. l)
    end
end


if -1 < APT.verbosity then print('\nFinished updating data.') end