apt-panopticon/apt-panopticommon.lua

local APT = {}

-- https://oss.oetiker.ch/rrdtool/prog/rrdlua.en.html
APT.rrd = require 'rrd'

APT.version = '0.3 alpha'

APT.protocols = {"ftp", "http", "https", "rsync"}
APT.tests = {'raw', 'Integrity', 'Protocol', 'Redirects', 'Updated', 'URLSanity', 'Speed'}
--APT.releases = {"jessie", "ascii", "beowulf", "chimaera", "daedalus", "ceres"}
APT.releases = {"beowulf", "chimaera", "daedalus", "ceres"}
APT.subRels = {'backports', 'proposed-updates', 'security', 'updates'}
APT.notExist =
{
    'ceres-backports',		-- These will never exist, it's our code name for the testing suite.
    'ceres-proposed-updates',
    'ceres-updates',
    'ceres-security',
}

APT.verbosity = -1
APT.origin = false
APT.redir = false
APT.keep = false
APT.IPv46 = ''

APT.options =
{
    bandwidth =
	{
	    typ = "number",
	    help =	'\n   0 = low    - HTTP tests for all IPs of all mirrors (HEAD tests), but not URL sanity tests.\n' ..
			'   1 = medium - Also HTTP(S) redirect tests, other protocols, download Release files over HTTP, and check them.\n' ..
			'   2 = high   - Also download Packages.xz files that changed, unpack and check them.\n' ..
			'                Also download and check InRelease files.\n' .. 
			'                Pick a few small packages, download them, check their SHA512.\n' ..
			'   3 = more   - Also pick more files and packages, some for each release / arch.\n' ..
			'   4 = all    - Do absolutely everything.\n' ..
			'                Actually download some Contents files, and some more Packages.xz, and package files.',
	    value = 2,
	},
    referenceSite =
	{
	    typ = "string",
	    help = "The mirror that is used as a reference, coz the others sync to it.",
	    value = "pkgmaster.devuan.org",
	},
    roundRobin =
	{
	    typ = "string",
	    help = "The round robin DNS name.",
	    value = "deb.devuan.org",
	},
    roundRobinCname =
	{
	    typ = "string",
	    help = "The round robin DNS name.",
	    value = "deb.rr.devuan.org",
	},
    tests =
	{
	    typ = "table",
	    help = "A list of the tests, see the documentation for details.",
	    value =
	    {
	        "IPv4",
	        "IPv6",
--	        "ftp",
	        "http",
	        "https",
--	        "rsync",
	        "DNSRR",
	        "Protocol",
	        "Redirects",
	        "URLSanity",
	        "Integrity",
	        "Updated",
	    },
	},
    maxtime =
	{
	    typ = "number",
	    help = "",
	    value = 240,
	},
    timeout =
	{
	    typ = "number",
	    help = "",
	    value = 5,
	},
    timeouts =
	{
	    typ = "number",
	    help = "",
	    value = 3,
	},
    refresh =
	{
	    typ = "number",
	    help = "",
	    value = 300,
	},
    retries =
	{
	    typ = "number",
	    help = "",
	    value = 2,
	},
    reports =
	{
	    typ = "table",
	    help = "A list of the reports generated.",
	    value =
	    {
		"RRD",		-- RRD has to be before web, coz web creates a graph from the RRD data.
		"email-web",
--		"Prometheus",
	    },
	},
    cgi =
	{
	    typ = "boolean",
	    help = "",
	    value = false,
	},
}

APT.args = {}
APT.parseArgs = function(args)
    APT.args = args
    local arg = {}
    local sendArgs = ""
    -- A special test to disable IPv6 tests if IPv6 isn't available.
    if 1 == APT.exe('ip -6 addr | grep inet6 | grep " global"'):timeout():Do().status then
	table.insert(args, '--tests=-IPv6')
    end
    if 0 ~= #(args) then
	local option = ""
	for i, a in pairs(args) do
	    if ("--help" == a) or ("-h" == a) then
		print("Check the health of Devuan Linux package mirrors.")
		for k, v in pairs(APT.options) do
		    print("")
		    print('--' .. k .. '   ' .. v.help)
		end
		os.exit()
	    elseif "--version" == a then
		print("apt-panopticon version " .. APT.version)
		os.exit()
	    elseif "-v" == a then
		APT.verbosity = APT.verbosity + 1
		sendArgs = sendArgs .. a .. " "
	    elseif "-q" == a then
		APT.verbosity = -1
		sendArgs = sendArgs .. a .. " "
	    elseif ("-4" == a) or ("-6" == a) then
		APT.IPv46 = a
		sendArgs = sendArgs .. a .. " "
	    elseif "-k" == a then
		APT.keep = true
		sendArgs = sendArgs .. a .. " "
	    elseif "-o" == a then
		APT.origin = true
		-- Not adding to sendArgs.
	    elseif "-r" == a then
		APT.redir = true
		sendArgs = sendArgs .. a .. " "
	    elseif "--cgi" == a then
		APT.options.cgi.value = true
		sendArgs = sendArgs .. a .. " "
	    elseif "--low" == a then
		APT.options.bandwidth.value = 0
		APT.options.timeout.value = 2
		APT.options.timeouts.value = 1
		APT.options.retries.value = 1
		sendArgs = sendArgs .. a .. " "
	    elseif "--medium" == a then
		APT.options.bandwidth.value = 1
		sendArgs = sendArgs .. a .. " "
	    elseif "--high" == a then
		APT.options.bandwidth.value = 2
		sendArgs = sendArgs .. a .. " "
	    elseif "--more" == a then
		APT.options.bandwidth.value = 3
		sendArgs = sendArgs .. a .. " "
	    elseif "--all" == a then
		APT.options.bandwidth.value = 4
		sendArgs = sendArgs .. a .. " "
	    elseif "--" == a:sub(1, 2) then
		local s, e = a:find("=")
		if nil == s then e = -1 end
		option = a:sub(3, e - 1)
		local o = APT.options[option]
		if nil == o then
		    print("Unknown option --" .. option)
		    option = ""
		else
		    option = a
		    sendArgs = sendArgs .. a .. " "
		    local s, e = a:find("=")
		    if nil == s then e = 0 end
		    option = a:sub(3, e - 1)
		    if "table" == APT.options[option].typ then
			local result = {}
			for t in (a:sub(e + 1) .. ","):gmatch("([+%-]?%w*),") do
			    local f = t:sub(1, 1)
			    local n = t:sub(2, -1)
			    if ("+" ~= f) and ("-" ~= f) then
				table.insert(result, t)
			    end
			end
			if 0 ~= #result then
			    APT.options[option].value = result
			else
			    for t in (a:sub(e + 1) .. ","):gmatch("([+%-]?%w*),") do
				local f = t:sub(1, 1)
				local n = t:sub(2, -1)
				if "+" == f then
				    table.insert(APT.options[option].value, n)
				elseif "-" == f then
				    local r = {}
				    for i, k in pairs(APT.options[option].value) do
					if k ~= n then table.insert(r, k) end
				    end
				    APT.options[option].value = r
				end
			    end
			end
		    else
			APT.options[option].value = a:sub(e + 1, -1)
		    end
		    option = ""
		end
	    elseif "-" == a:sub(1, 1) then
		print("Unknown option " .. a)
	    else
		table.insert(arg, a)
	    end
	end
    end
    return arg, sendArgs
end



--[[ Ordered table iterator, allow to iterate on the natural order of the keys of a table.
    From http://lua-users.org/wiki/SortedIteration
  ]]
function __genOrderedIndex( t )
    local orderedIndex = {}
    for key in pairs(t) do
        table.insert( orderedIndex, key )
    end
    table.sort( orderedIndex )
    return orderedIndex
end
function orderedNext(t, state)
    -- Equivalent of the next function, but returns the keys in the alphabetic
    -- order. We use a temporary ordered key table that is stored in the
    -- table being iterated.

    local key = nil
    --print("orderedNext: state = "..tostring(state) )
    if state == nil then
        -- the first time, generate the index
        t.__orderedIndex = __genOrderedIndex( t )
        key = t.__orderedIndex[1]
    else
        -- fetch the next value
        for i = 1,table.getn(t.__orderedIndex) do
            if t.__orderedIndex[i] == state then
                key = t.__orderedIndex[i+1]
            end
        end
    end

    if key then
        return key, t[key]
    end

    -- no more value to return, cleanup
    t.__orderedIndex = nil
    return
end
function APT.orderedPairs(t)
    -- Equivalent of the pairs() function on tables. Allows to iterate
    -- in order
    return orderedNext, t, nil
end


-- Use this to dump a table to a string, with HTML.
APT.dumpTableHTML = function (table, name, space)
    if nil == space then space = '' end
    local r = name .. "\n"
    r = r .. dumpTableHTMLSub(table, space .. " ")
    r = r .. space .. ""
    return r
end
dumpTableHTMLSub = function (table, space)
    local r = ""
    for k, v in APT.orderedPairs(table) do 
	if type(v) == "table" then
	    if " " == space then
		r = r ..           space .. APT.dumpTableHTML(v, k .. "<ul>", space) .. "</ul>\n"
	    else
		r = r .. "<li>" .. space .. APT.dumpTableHTML(v, k .. "<ul>", space) .. "</ul></li>\n"
	    end
	else
	    r = r .. space .. "<li>" .. k .. "</li>\n"
	end
    end
    return r
end

-- Use this to dump a table to a string.
APT.dumpTable = function (table, name, space)
    if nil == space then space = '' end
    local r = ""
    if "" == space then r = r .. space .. name .. " =\n" else r = r .. space .. "[" .. name .. "] =\n" end
    r = r .. space .. "{\n"
    r = r .. dumpTableSub(table, space .. "  ")
    if "" == space then r = r .. space .. "}\n" else r = r .. space .. "},\n" end
    return r
end
dumpTableSub = function (table, space)
    local r = ""
    for k, v in pairs(table) do 
        if type(k) == "string" then k = '"' .. k .. '"' end
	if type(v) == "table" then
	    r = r .. APT.dumpTable(v, k, space)
	elseif type(v) == "string" then
	    local bq = '"'
	    local eq = '"'
	    if nil ~= v:match(bq) then
		bq, eq = '"', '"'
		if nil ~= v:match(bq) then
		    bq, eq = '[[', ']]'
		    mbq, meq = '%[%[', '%]%]'
		    while (nil ~= v:match(mbq)) or (nil ~= v:match(meq)) do
			bq = '[' .. '=' .. bq:sub(2, -1)
			eq = ']' .. '=' .. eq:sub(2, -1)
			mbq = '%[' .. '=' .. bq:sub(3, -1)
			meq = '%]' .. '=' .. eq:sub(3, -1)
		    end
		end
	    end
	    r = r .. space .. "[" .. k .. "] = " .. bq .. v .. eq .. ";\n"
	elseif type(v) == "function" then
	    r = r .. space .. "[" .. k .. "] = function ();\n"
	elseif type(v) == "userdata" then
	    r = r .. space .. "userdata " .. "[" .. k .. "];\n"
	elseif type(v) == "boolean" then
	    if (v) then
		r = r .. space .. "[" .. k .. "] = true;\n"
	    else
		r = r .. space .. "[" .. k .. "] = false;\n"
	    end
	else
	    r = r .. space .. "[" .. k .. "] = " .. v .. ";\n"
	end
    end
    return r
end


APT.allpairs = function(tbl, func)
    for k, v in pairs(tbl) do
	if 'table' == type(v) then
	    for i, w in pairs(v) do
		func(i, w, k, v)
	    end
	else
	    func(k, v)
	end
    end
end

APT.lnk = function(URL)
    return '<a href="' .. URL .. '">' .. URL .. '</a>'
end


APT.search = function(t, s)
    for i, v in pairs(t) do
	if v == s then return true end
    end
    return false
end

APT.results = {}
APT.logFile = nil
APT.html = false

APT.logName = function(host, a2, a3)
    local name = host
    if (nil ~= a2) and ('' ~= a2) then name = name .. "_" .. a2 end
    if (nil ~= a3) and ('' ~= a3) then name = name .. "_" .. a3 end
    name = 'LOG_' .. name .. '.html'
    return {'results/' .. name, '<a href="' .. name:gsub("/", "_") .. '">' .. name:gsub("/", "_") .. '</a>'}
end

APT.logOpen = function(host, a2, a3)
    local name = APT.logName(host, a2, a3)[1]
    if APT.checkFile(name) then return false end
    APT.logFile, e = io.open(name, "a+")
    if nil == APT.logFile then print('CRITICAL - opening log file (' .. name .. ') - ' .. e); return false end
    if nil ~= APT.logFile then
	APT.logFile:write("<html><head>\n")
	APT.logFile:write("</head><body bgcolor='black' text='white' alink='red' link='aqua' vlink='fuchsia'>\n")
	APT.logFile:write("<pre>\n")
	APT.logFile:write(APT.dumpTable(APT.args, 'Arguments'))
	APT.logFile:write("</pre>\n")
    else
	return false
    end
    return true
end

APT.logPost = function()
    if nil ~= APT.logFile then
	APT.logFile:write("</body></html> \n")
	APT.logFile:close()
	APT.logFile = nil
    end
end

local log = function(v, t, s, prot, test, host)
    local x = ""
    if nil == prot then prot = "" end
    if nil == test then test = "" end
    x = x .. prot
    if "" ~= test then
	if #x > 0 then x = x .. " " end
	x = x .. test
    end
    if nil ~= host then
	if #x > 0 then x = x .. " " end
	x = x .. host
    end
    if #x > 0 then
	t = t .. "(" .. x ..  ")"
	if "" ~= prot then
	    if "" == test then
		if nil == APT.results[prot] then APT.results[prot] = {tested = 0; errors = 0; warnings = 0; timeouts = 0} end
		if v == 0 then APT.results[prot].errors = APT.results[prot].errors + 1 end
		if v == 1 then APT.results[prot].warnings = APT.results[prot].warnings + 1 end
		if v == 2 then APT.results[prot].timeouts = APT.results[prot].timeouts + 1 end
	    else
		if nil == APT.results[prot] then APT.results[prot] = {tested = 0; errors = 0; warnings = 0; timeouts = 0} end
		if nil == APT.results[prot][test] then APT.results[prot][test] = {tested = 0; errors = 0; warnings = 0; timeouts = 0} end
		if v == 0 then APT.results[prot][test].errors = APT.results[prot][test].errors + 1 end
		if v == 1 then APT.results[prot][test].warnings = APT.results[prot][test].warnings + 1 end
		if v == 2 then APT.results[prot][test].timeouts = APT.results[prot][test].timeouts + 1 end
	    end
	end
    end
    if v <= APT.verbosity then
	if 3 <= APT.verbosity then t = os.date('!%F %T') .. " " .. t end
	print(t .. ": " .. s)
    end
    if nil ~= APT.logFile then
	if APT.html then
	    local colour = "white"
	    if -1 == v then colour = "fuchsia"; print(t .. " " .. s) end	-- CRITICAL
	    if  0 == v then colour = "red    " end	-- ERROR
	    if  1 == v then colour = "yellow " end	-- WARNING
	    if  2 == v then colour = "blue   " end	-- TIMEOUT
	    if  3 == v then colour = "white  " end	-- INFO
	    if  4 == v then colour = "gray   " end	-- DEBUG
	    APT.logFile:write(os.date('!%F %T') .. " <font color='" .. colour .. "'><b>" .. t .. "</b></font>: " .. s .. "</br>\n")
	else
	    APT.logFile:write(os.date('!%F %T') .. " " .. t .. ": " .. s .. "\n")
	end
	APT.logFile:flush()
    end
end
APT.D = function(s, h)		log(4,  "DEBUG   ", s, nil, nil, h) end
APT.I = function(s, h)		log(3,  "INFO    ", s, nil, nil, h) end
APT.T = function(s, p, t, h)	log(2,  "TIMEOUT ", s, p, t, h) end
APT.W = function(s, p, t, h)	log(1,  "WARNING ", s, p, t, h) end
APT.E = function(s, p, t, h)	log(0,  "ERROR   ", s, p, t, h) end
APT.C = function(s)		log(-1, "CRITICAL", s) end
local D = APT.D
local I = APT.I
local T = APT.T
local W = APT.W
local E = APT.E
local C = APT.C


APT.readCmd = function(cmd)
    local result = {}
    local output = io.popen(cmd)
    if nil ~= output then
	for l in output:lines() do
	    table.insert(result, l)
	end
    end
    return result
end

APT.debians = {}
APT.mirrors = {}

APT.testing = function(t, host)
    for i, v in pairs(APT.options.tests.value) do
	if t == v then
	    local h = APT.mirrors[host]
	    if nil == h then return true end
	    if true == h["Protocols"][t] then return true else I("Skipping " .. t .. " checks for " .. host) end
	end
    end
    return false
end

APT.tested = function(prot, test, host)
    if "" == test then
	if nil == APT.results[prot] then APT.results[prot] = {tested = 0; errors = 0; warnings = 0; timeouts = 0} end
	APT.results[prot].tested = APT.results[prot].tested + 1;
    else
	if nil == APT.results[prot] then APT.results[prot] = {tested = 0; errors = 0; warnings = 0; timeouts = 0} end
	if nil == APT.results[prot][test] then APT.results[prot][test] = {tested = 0; errors = 0; warnings = 0; timeouts = 0} end
	APT.results[prot][test].tested = APT.results[prot][test].tested + 1;
    end
end

APT.exe = function(c)
    local exe = {status = 0, result = '', log = true, cmd = c .. ' ', command = c}

    function exe:log()
	self.log = true
	return self
    end
    function exe:Nice(c)
	if nil == c then
	    self.cmd = 'ionice -c3 nice -n 19 ' .. self.cmd
	else
	    self.cmd = self.cmd .. 'ionice -c3 nice -n 19 ' .. c .. ' '
	end
	return self
    end
    function exe:timeout(c)
	-- timeout returns a status of - command status if --preserve-status; "128+9" (actually 137) if --kill-after ends up being done; 124 if it had to TERM; command status if all went well.
	-- --kill-after means "send KILL after TERM fails.
	if nil == c then
	    self.cmd = 'timeout --kill-after=10.0 --foreground 42.0s ' .. self.cmd
	else
	    self.cmd = 'timeout --kill-after=10.0 --foreground ' .. c .. ' ' .. self.cmd
	end
	return self
    end
    function exe:also(c)
	if nil == c then c = '' else c = ' ' .. c end
	self.cmd = self.cmd .. ';' .. c .. ' '
	return self
    end
    function exe:And(c)
	if nil == c then c = '' else c = ' ' .. c end
	self.cmd = self.cmd .. '&&' .. c .. ' '
	return self
    end
    function exe:Or(c)
	if nil == c then c = '' end
	self.cmd = self.cmd .. '|| ' .. c .. ' '
	return self
    end
    function exe:noErr()
	self.cmd = self.cmd .. '2>/dev/null '
	return self
    end
    function exe:wait(w)
	self.cmd = self.cmd .. '&& touch ' .. w .. ' '
	return self
    end
    function exe:Do()
    --[[    "The condition expression of a control structure can return any
	    value. Both false and nil are considered false. All values different
	    from nil and false are considered true (in particular, the number 0
	    and the empty string are also true)."
	says the docs, I beg to differ.]]
	if true == self.log then D("  executing - &nbsp; <code>" .. self.cmd .. "</code>") end
	--[[ Damn os.execute()
	    Lua 5.1 says it returns "a status code, which is system-dependent"
	    Lua 5.2 says it returns true/nil, "exit"/"signal", the status code.
	    I'm getting 7168 or 0.  No idea what the fuck that is.
	local ok, rslt, status = os.execute(s)
	]]
	local f = APT.readCmd(self.cmd, 'r')
	-- The last line will be the command's returned status, collect everything else in result.
	self.status = ''	-- Otherwise the result starts with 0.
	self.result = '\n'
	for i,l in ipairs(f) do
	    self.result = self.result .. l .. "\n"
	end
	f = APT.readCmd('echo "$?"', 'r')
	for i,l in ipairs(f) do
	    self.status = tonumber(l)
	    if (137 == self.status) or (124 == self.status) then
		print("timeout killed " .. self.status .. ' ' .. self.command)
		E("timeout killed " .. self.status .. ' ' .. self.command)
	    elseif (0 ~= self.status) then
		print("status |" .. self.status .. '| ' .. self.command)
		E("status |" .. self.status .. '| ' .. self.command)
	    end
	end
	return self
    end
    function exe:fork(host)
	if nil ~= host then self.cmd = self.cmd .. ';  r=$?; if [ $r -ge 124 ]; then echo "$r ' .. host .. ' failed forked command ' .. string.gsub(self.cmd, '"', "'") .. '"; fi' end
	self.cmd = '{ ' .. self.cmd .. '; } &'
	if true == self.log then D("  forking - &nbsp; <code>" .. self.cmd .. "</code>") end
	os.execute(self.cmd)
	return self
    end
    return exe
end

APT.checkExes = function (exe)
    local count = 0
    local ps = io.popen('ps x | grep "' .. exe .. '" | grep -v " grep " | grep -v "flock -n apt-panopticon.lock " | wc -l')
    if nil ~= ps then count = ps:read("*l") end
    D(count .. " " .. exe .. " commands still running.")
    return tonumber(count)
end

APT.checkFile = function(f)
    local h, e = io.open(f, "r")
    if nil == h then return false else h:close();  return true end
end

APT.plurals = function(e, w, t)
    local result = ""
    if 1 == e then
	result = e .. " error"
    elseif e ~= 0 then
	result = e .. " errors"
    end
    if ("" ~= result) and APT.html then result = "<font color='red'><b>" .. result .. "</b></font>" end
    if 0 < w then
	if 0 < e then result = result .. ", " end
	if 1 == w then 
	    result = result .. w .. " warning"
	else
	    result = result .. w .. " warnings"
	end
	if ("" ~= result) and APT.html then result = "<font color='yellow'><b>" .. result .. "</b></font>" end
    end
    if 0 < t then
	if (0 < e) or (0 < w) then result = result .. ", " end
	if 1 == t then 
	    result = result .. t .. " timeout"
	else
	    result = result .. t .. " timeouts"
	end
	if ("" ~= result) and APT.html then result = "<font color='blue'><b>" .. result .. "</b></font>" end
    end
    if "" ~= result then result = "  " .. result end
    return result
end

local typs = {'tested', 'errors', 'warnings', 'timeouts'}
APT.padResults = function(results)
    local c = 0
    if nil == results then results = {}; c = 999 end
    for k, v in pairs(APT.protocols) do
	tests = results[v]
	if nil == tests then tests = {} end
	for m, x in pairs(typs) do
	    if nil == tests[x] then tests[x] = c end
	end
	for l, w in pairs(APT.tests) do
	    if ('raw' ~= w) and ('Speed' ~= w) then
		if nil == tests[w] then tests[w] = {} end
		for m, x in pairs(typs) do
		    if nil == tests[w][x] then tests[w][x] = c end
		end
	    end
	end
	if nil == tests.redirects then
	    tests.redirects = {}
	end
	results[v] = tests
    end
    if nil == results.timeout then results.timeout = false end
    if nil == results.speed   then results.speed   = {min = 999999999999; max = 0} end
    return results
end

APT.collate = function(l, host, ip, results)
    results = APT.padResults(results)
    local f = l .. "/" .. host .. "_" .. ip .. ".lua"
    if APT.checkFile(f) then
	local rs = loadfile(f)()
	rs = APT.padResults(rs)
	for k, v in pairs(rs) do
	    if "table" == type(v) then
		if ("speed" == k) and (nil ~= results.speed) then
		    if v.min < results.speed.min then results.speed.min = v.min end
		    if v.max > results.speed.max then results.speed.max = v.max end
		elseif 'IPs' ~= k then
		    for i, u in pairs(v) do
			if 'redirects' ~= i then
			    if "table" == type(u) then
				for h, t in pairs(u) do
				        local a = results[k]
				    if nil == a then results[k]    = {i = {}}; a = results[k]    end
				    a = a[i]
				    if nil == a then results[k][i] = {h = {}}; a = results[k][i] end
				    a = a[h]
				    if nil == a then a = 0 end
				    results[k][i][h] = a + t
				end
			    else
				local a = results[k]
				if nil == a then a = 0; results[k] = {} else a = a[i] end
				if nil == a then a = 0 end
				results[k][i] = a + u
			    end
			end
		    end
		end
	    elseif "timeout" ~= k then
		local a = results[k]
		if nil == a then a = 0 end
		results[k] = a + v
	    end
	end
    end
    return results
end
APT.collateAll = function(hosts, l, host, func)
    results = {}
    local f = l .. "/" .. host .. ".lua"
    if APT.checkFile(f) then
	results = loadfile(f)()
	results = APT.padResults(results)
	if nil ~= func then func(results) end
	results = APT.collate(l, host, 'R', results)
	local v = hosts[host]
	if nil ~= v then
	    local IPs = results.IPs
	    if nil == IPs then W('No IPs for ' .. host .. ' in ' .. l) else
		for i, u in pairs(IPs) do
		    if "table" == type(u) then
			for h, t in pairs(u) do
			    results = APT.collate(l, host, h, results)
			    results = APT.collate(l, host, h .. '_R', results)
			    if nil ~= func then func(results, h) end
			end
		    else
			results = APT.collate(l, host, i .. '_R', results)
			if nil ~= func then func(results, i) end
		    end
		end
	    end
	end
    end
    results = APT.padResults(results)
    return results
end


APT.now = 0
local status
APT.now = tonumber(APT.exe('TZ="GMT" ls -l --time-style="+%s" results/stamp | cut -d " " -f 6-6'):Do().result)
local start = 'now-1month'
local step = '10min'
local hb = '150min'
local DSIe = 'DS:IntegrityErrors:GAUGE:' .. hb .. ':0:U'
local DSIw = 'DS:IntegrityWarnings:GAUGE:' .. hb .. ':0:U'
local DSIt = 'DS:IntegrityTimeouts:GAUGE:' .. hb .. ':0:U'
local DSPe = 'DS:ProtocolErrors:GAUGE:' .. hb .. ':0:U'
local DSPw = 'DS:ProtocolWarnings:GAUGE:' .. hb .. ':0:U'
local DSPt = 'DS:ProtocolTimeouts:GAUGE:' .. hb .. ':0:U'
local DSRe = 'DS:RedirectsErrors:GAUGE:' .. hb .. ':0:U'
local DSRw = 'DS:RedirectsWarnings:GAUGE:' .. hb .. ':0:U'
local DSRt = 'DS:RedirectsTimeouts:GAUGE:' .. hb .. ':0:U'
local DSUe = 'DS:UpdatedErrors:GAUGE:' .. hb .. ':0:U'
local DSUw = 'DS:UpdatedWarnings:GAUGE:' .. hb .. ':0:U'
local DSUt = 'DS:UpdatedTimeouts:GAUGE:' .. hb .. ':0:U'
local DSSe = 'DS:URLSanityErrors:GAUGE:' .. hb .. ':0:U'
local DSSw = 'DS:URLSanityWarnings:GAUGE:' .. hb .. ':0:U'
local DSSt = 'DS:URLSanityTimeouts:GAUGE:' .. hb .. ':0:U'
local DSx = 'DS:max:GAUGE:' .. hb .. ':0:U'
local DSn = 'DS:min:GAUGE:' .. hb .. ':0:U'
-- What Collectd uses.
local RRAc0  = 'RRA:AVERAGE:0.9:1:1200'
local RRAc1  = 'RRA:MIN:0.9:1:1200'
local RRAc2  = 'RRA:MAX:0.9:1:1200'
local RRAc3  = 'RRA:AVERAGE:0.9:7:1235'
local RRAc4  = 'RRA:MIN:0.9:7:1235'
local RRAc5  = 'RRA:MAX:0.9:7:1235'
local RRAc6  = 'RRA:AVERAGE:0.9:50:1210'
local RRAc7  = 'RRA:MIN:0.9:50:1210'
local RRAc8  = 'RRA:MAX:0.9:50:1210'
local RRAc9  = 'RRA:AVERAGE:0.9:223:1202'
local RRAc10 = 'RRA:MIN:0.9:223:1202'
local RRAc11 = 'RRA:MAX:0.9:223:1202'
local RRAc12 = 'RRA:AVERAGE:0.9:2635:1201'
local RRAc13 = 'RRA:MIN:0.9:2635:1201'
local RRAc14 = 'RRA:MAX:0.9:2635:1201'
-- Try LAST.
local RRAl0  = 'RRA:LAST:0.9:1:1200'
local RRAl1  = 'RRA:LAST:0.9:7:1235'
local RRAl2  = 'RRA:LAST:0.9:50:1210'
local RRAl3  = 'RRA:LAST:0.9:223:1202'
local RRAl4  = 'RRA:LAST:0.9:2635:1201'

APT.createRRD = function(host, ip, o)
    if nil ~= o then start = o end
    if nil ~= ip then host = host .. '_' .. ip end
    for i, p in pairs(APT.protocols) do
	os.execute( 'mkdir -p rrd/' .. host .. '/' .. p:upper())
	if not APT.checkFile('rrd/' .. host .. '/' .. p:upper() .. '/Tests.rrd') then
	    D('Creating ' .. 'rrd/' .. host .. '/' .. p:upper() .. '/Tests.rrd')
	    APT.rrd.create(  'rrd/' .. host .. '/' .. p:upper() .. '/Tests.rrd', '--start', start, '--step', step,
			    DSIe, DSIw, DSIt, DSPe, DSPw, DSPt, DSRe, DSRw, DSRt, DSUe, DSUw, DSUt, DSSe, DSSw, DSSt,
			    RRAc0, RRAc1, RRAc2, RRAl0, RRAc3, RRAc4, RRAc5, RRAl1, RRAc6, RRAc7, RRAc8, RRAl2, RRAc9, RRAc10, RRAc11, RRAl3, RRAc12, RRAc13, RRAc14, RRAl4)
	    -- Start them at 0 so the average has something to work on.
	    APT.rrd.update(      'rrd/' .. host .. '/' .. p:upper() .. '/Tests.rrd', (APT.now - 600) .. ':0:0:0:0:0:0:0:0:0:0:0:0:0:0:0')
	end
    end
    os.execute( 'mkdir -p rrd/' .. host .. '/Speed')
    if not APT.checkFile('rrd/' .. host .. '/Speed/Speed.rrd') then
	D('Creating ' .. 'rrd/' .. host .. '/Speed/Speed.rrd')
	APT.rrd.create(  'rrd/' .. host .. '/Speed/Speed.rrd', '--start', start, '--step', step, DSx, DSn,
			RRAc0, RRAc1, RRAc2, RRAl0, RRAc3, RRAc4, RRAc5, RRAl1, RRAc6, RRAc7, RRAc8, RRAl2, RRAc9, RRAc10, RRAc11, RRAl3, RRAc12, RRAc13, RRAc14, RRAl4)
	APT.rrd.update(      'rrd/' .. host .. '/Speed/Speed.rrd', (APT.now - 600) .. ':0:0')
    end
end

APT.updateRRD = function(results, host, ip)
    if nil ~= ip then host = host .. '_' .. ip end
    for i, p in pairs(APT.protocols) do
	if nil ~= results[p] then
	    APT.rrd.update('rrd/' .. host .. '/' .. p:upper() .. '/Tests.rrd', APT.now .. ':' ..
	    results[p]['Integrity'].errors .. ':'  .. results[p]['Integrity'].warnings .. ':' .. results[p]['Integrity'].timeouts .. ':' ..
	    results[p]['Protocol'].errors  .. ':'  .. results[p]['Protocol'].warnings  .. ':' .. results[p]['Protocol'].timeouts  .. ':' ..
	    results[p]['Redirects'].errors .. ':'  .. results[p]['Redirects'].warnings .. ':' .. results[p]['Redirects'].timeouts   .. ':' ..
	    results[p]['Updated'].errors   .. ':'  .. results[p]['Updated'].warnings   .. ':' .. results[p]['Updated'].timeouts   .. ':' ..
	    results[p]['URLSanity'].errors .. ':'  .. results[p]['URLSanity'].warnings .. ':' .. results[p]['URLSanity'].timeouts)
	else
	    APT.rrd.update('rrd/' .. host .. '/' .. p:upper() .. '/Tests.rrd', APT.now .. ':U:U:U:U:U:U:U:U:U:U:U:U:U:U:U')
	end
    end
    if nil ~= results.speed then
	if 0 ~= results.speed.max then
	    APT.rrd.update('rrd/' .. host .. '/Speed/Speed.rrd', APT.now .. ':' .. results.speed.max .. ':'  .. results.speed.min)
	else
	    APT.rrd.update('rrd/' .. host .. '/Speed/Speed.rrd', APT.now .. ':0:0')
	end
    else
	APT.rrd.update(    'rrd/' .. host .. '/Speed/Speed.rrd', APT.now .. ':U:U')
    end
end

APT.doRRD = function(l, k, v, o)
-- TODO - Quick hack, fix it later.
if k == APT.options.roundRobin.value then return end
    APT.collateAll(APT.mirrors, l, k,
	function(results, ip)
	    APT.createRRD(k, ip, o)
	    APT.updateRRD(results, k, ip)
	end)
end


return APT