onefang
/
apt-panopticon
2
2
Fork 0
Code Issues Releases Wiki Activity
Browse Source

Track, store, and report info from the hosts that get redirected to.

master
onefang 3 years ago
parent
09f41b9003
commit
0f43ca83d6
4 changed files with 292 additions and 56 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 38
      apt-panopticommon.lua
  2. 6
      apt-panopticon-report-RRD.lua
  3. 201
      apt-panopticon-report-email-web.lua
  4. 103
      apt-panopticon.lua

38
apt-panopticommon.lua
View File

@ -53,7 +53,7 @@ APT.options =
"DNSRR",
"Protocol",
"URLSanity",
"Integrity",
-- "Integrity",
"Updated",
},
},
@ -411,6 +411,7 @@ local E = APT.E
local C = APT.C
APT.debians = {}
APT.mirrors = {}
APT.testing = function(t, host)
@ -498,6 +499,7 @@ APT.padResults = function(results)
if nil == tests.Integrity then tests.Integrity = {errors = c; warnings = c; timeouts = c} end
if nil == tests.Protocol then tests.Protocol = {errors = c; warnings = c; timeouts = c} end
if nil == tests.Updated then tests.Updated = {errors = c; warnings = c; timeouts = c} end
if nil == tests.redirects then tests.redirects = {} end
if nil == tests.URLSanity then tests.URLSanity = {errors = c; warnings = c; timeouts = c} end
results[v] = tests
end
@ -517,23 +519,25 @@ APT.collate = function(l, host, ip, results)
if ("speed" == k) and (nil ~= results.speed) then
if v.min < results.speed.min then results.speed.min = v.min end
if v.max > results.speed.max then results.speed.max = v.max end
else
elseif 'IPs' ~= k then
for i, u in pairs(v) do
if "table" == type(u) then
for h, t in pairs(u) do
if 'redirects' ~= i then
if "table" == type(u) then
for h, t in pairs(u) do
local a = results[k]
if nil == a then results[k] = {i = {}}; a = results[k] end
a = a[i]
if nil == a then results[k][i] = {h = {}}; a = results[k][i] end
a = a[h]
if nil == a then a = 0 end
results[k][i][h] = a + t
end
else
local a = results[k]
if nil == a then results[k] = {i = {}}; a = results[k] end
a = a[i]
if nil == a then results[k][i] = {h = {}}; a = results[k][i] end
a = a[h]
if nil == a then a = 0; results[k] = {} else a = a[i] end
if nil == a then a = 0 end
results[k][i][h] = a + t
results[k][i] = a + u
end
else
local a = results[k]
if nil == a then a = 0; results[k] = {} else a = a[i] end
if nil == a then a = 0 end
results[k][i] = a + u
end
end
end
@ -546,14 +550,14 @@ APT.collate = function(l, host, ip, results)
end
return results
end
APT.collateAll = function(l, host, func)
APT.collateAll = function(hosts, l, host, func)
results = {}
local f = l .. "/" .. host .. ".lua"
if APT.checkFile(f) then
results = loadfile(f)()
results = APT.padResults(results)
if nil ~= func then func(results) end
local v = APT.mirrors[host]
local v = hosts[host]
if nil ~= v then
local IPs = results.IPs
if nil == IPs then W('No IPs for ' .. host .. ' in ' .. l) else
@ -662,7 +666,7 @@ APT.updateRRD = function(results, host, ip)
end
APT.doRRD = function(l, k, v, o)
APT.collateAll(l, k,
APT.collateAll(APT.mirrors, l, k,
function(results, ip)
APT.createRRD(k, ip, o)
APT.updateRRD(results, k, ip)

6
apt-panopticon-report-RRD.lua
View File

@ -15,3 +15,9 @@ APT.html = false
for k, v in APT.orderedPairs(APT.mirrors) do
APT.doRRD('results', k, v)
end
APT.debians = loadfile("results/debians.lua")()
APT.html = false
for k, v in APT.orderedPairs(APT.debians) do
APT.doRRD('results', k, v)
end

201
apt-panopticon-report-email-web.lua
View File

@ -12,11 +12,12 @@ local arg, sendArgs = APT.parseArgs({...})
local results = {}
APT.mirrors = loadfile("results/mirrors.lua")()
APT.debians = loadfile("results/debians.lua")()
local revDNS = function(dom, IP)
local revDNS = function(hosts, dom, IP)
if "deb.devuan.org" ~= dom then
if nil ~= APT.mirrors["deb.devuan.org"] then
if nil ~= APT.mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][IP] then
if nil ~= hosts["deb.devuan.org"] then
if nil ~= hosts["deb.devuan.org"].IPs["deb.roundr.devuan.org"][IP] then
if APT.html then
return "<font color='purple'><b>DNS-RR</b></font>"
else
@ -25,7 +26,7 @@ local revDNS = function(dom, IP)
end
end
else
for k, v in pairs(APT.mirrors) do
for k, v in pairs(hosts) do
if "deb.devuan.org" ~= k then
local IPs = v.IPs
for i, u in pairs(IPs) do
@ -44,12 +45,12 @@ local revDNS = function(dom, IP)
end
local faulty = ""
local status = function(host, results, typ)
local status = function(hosts, host, results, typ)
local result = ""
local e = 0
local w = 0
local t = 0
local s = nil ~= APT.mirrors[host].Protocols[typ]
local s = nil ~= hosts[host].Protocols[typ]
local to = results.timeout
if not APT.search(APT.protocols, typ) then s = true end
if nil ~= results[typ] then
@ -57,7 +58,7 @@ local status = function(host, results, typ)
w = results[typ].warnings
t = results[typ].timeouts
for k, v in pairs(results[typ]) do
if "table" == type(v) then
if ("table" == type(v)) and ('redirects' ~= k) then
if 0 <= v.errors then e = e + v.errors else to = true end
if 0 <= v.warnings then w = w + v.warnings else to = true end
if 0 <= v.timeouts then t = t + v.timeouts else to = true end
@ -139,31 +140,49 @@ local logCount = function(domain, ip)
local nm = "LOG_" .. domain
local log = ""
local extra = ""
local errors = 0
local warnings = 0
local timeouts = 0
if nil ~= ip then nm = nm .. "_" .. ip end
nm = nm .. ".html"
local rfile, e = io.open("results/" .. nm, "r")
if nil ~= rfile then
local errors = 0
local warnings = 0
local timeouts = 0
for l in rfile:lines() do
if nil ~= l:match("><b>ERROR ") then errors = errors + 1 end
if nil ~= l:match("><b>WARNING ") then warnings = warnings + 1 end
if nil ~= l:match("><b>TIMEOUT ") then timeouts = timeouts + 1 end
end
rfile:close()
if APT.html then
if nil == ip then
log = "<a href='" .. nm .. "'>" .. domain .. "</a>"
else
log = "<a href='" .. nm .. "'>" .. ip .. "</a>"
end
end
if APT.html then
if nil == ip then
log = "<a href='" .. nm .. "'>" .. domain .. "</a>"
else
log = "<a href='" .. nm .. "'>" .. ip .. "</a>"
end
log = log .. APT.plurals(errors, warnings, timeouts)
end
log = log .. APT.plurals(errors, warnings, timeouts)
return log
end
local redirs = function(hosts, host)
local results = APT.collateAll(hosts, 'results', host)
local rdr = {}
local redirs = ''
for p, pt in pairs(APT.protocols) do
if 0 ~= #(results[pt].redirects) then
table.sort(results[pt].redirects)
for r, rd in pairs(results[pt].redirects) do
rdr[rd] = rd
end
end
end
for r, rd in pairs(rdr) do
redirs = redirs .. ', &nbsp ' .. rd
end
if '' ~= redirs then redirs = '<br>\n &nbsp; &nbsp; (Redirects some packages to - ' .. redirs:sub(3) .. ')' end
return redirs
end
APT.html = false
local email, e = io.open("results/Report-email.txt", "w+")
@ -183,16 +202,16 @@ if nil == email then C("opening mirrors file - " .. e) else
"[skip] means that the test hasn't been written yet.\n\n")
for k, v in APT.orderedPairs(APT.mirrors) do
email:write(k .. "....\n")
local results = APT.collateAll('results', k)
local results = APT.collateAll(APT.mirrors, 'results', k)
local ftp = "[skip]"
local http = status(k, results, "http")
local https = status(k, results, "https")
local http = status(APT.mirrors, k, results, "http")
local https = status(APT.mirrors, k, results, "https")
local rsync = "[skip]"
local dns = ""
local protocol = status(k, results, "Protocol")
local sanity = status(k, results, "URLSanity")
local integrity = status(k, results, "Integrity")
local updated = status(k, results, "Updated")
local protocol = status(APT.mirrors, k, results, "Protocol")
local sanity = status(APT.mirrors, k, results, "URLSanity")
local integrity = status(APT.mirrors, k, results, "Integrity")
local updated = status(APT.mirrors, k, results, "Updated")
-- DNS-RR test.
if ("deb.devuan.org" ~= k) and (nil ~= APT.mirrors["deb.devuan.org"]) then
@ -347,7 +366,7 @@ if nil == web then C("opening mirrors file - " .. e) else
"<th>Protocol</th><th>URL sanity</th><th>Integrity</th><th>Updated</th><th colspan='2'>Speed range</th></tr>\n"
)
for k, v in APT.orderedPairs(APT.mirrors) do
local results = APT.collateAll('results', k)
local results = APT.collateAll(APT.mirrors, 'results', k)
local active = ""
if "yes" == v.Active then
web:write(" <tr><th>" .. k .. "</th> ")
@ -356,14 +375,14 @@ if nil == web then C("opening mirrors file - " .. e) else
web:write(" <tr style='background-color:dimgrey'><th>" .. k .. "</th> ")
end
local ftp = "[<font color='grey'><b>skip</b></font>]"
local http = status(k, results, "http")
local https = status(k, results, "https")
local http = status(APT.mirrors, k, results, "http")
local https = status(APT.mirrors, k, results, "https")
local rsync = "[<font color='grey'><b>skip</b></font>]"
local dns = ""
local protocol = status(k, results, "Protocol")
local sanity = status(k, results, "URLSanity")
local integrity = status(k, results, "Integrity")
local updated = status(k, results, "Updated")
local protocol = status(APT.mirrors, k, results, "Protocol")
local sanity = status(APT.mirrors, k, results, "URLSanity")
local integrity = status(APT.mirrors, k, results, "Integrity")
local updated = status(APT.mirrors, k, results, "Updated")
local rate = v.Rate
if nil ~= rate then updated = updated .. ' ' .. rate end
local min = tonumber(results.speed.min)
@ -434,14 +453,14 @@ if nil == web then C("opening mirrors file - " .. e) else
n[l] = {}
for i, u in pairs(w) do
local log = logCount(k, i)
if "" == log then n[l][i] = u else n[l][log .. " " .. revDNS(k, i)] = u end
if "" == log then n[l][i] = u else n[l][log .. " " .. revDNS(APT.mirrors, k, i)] = u end
end
else
local log = logCount(k, l)
if "" == log then n[l] = w else n[log .. " " .. revDNS(k, l)] = w end
if "" == log then n[l] = w else n[log .. " " .. revDNS(APT.mirrors, k, l)] = w end
end
end
m[log .. " DNS entries -"] = n
m[log .. " DNS entries -" .. redirs(APT.mirrors, k)] = n
end
web:write( "<p>This lists each mirror, and the DNS entries for that mirror. &nbsp; " ..
"The links point to the testing log files for " .. logCount("apt-panopticon") .. " for each domain name / IP combination that was tested. &nbsp; " ..
@ -453,8 +472,118 @@ if nil == web then C("opening mirrors file - " .. e) else
)
web:write(APT.dumpTableHTML(m, "", ""))
web:write( "\n<br>\n<br>\n<h2>==== graphs: ====</h2>\n" ..
"<img src='speed.png'>\n<br>\n<p><a href='../apt-panopticon_cgp/'>More graphs.</a> with greater detail.</p><hr>\n\n" ..
"<p>The <a href='Report-email.txt'>email report</a>. &nbsp; " ..
"<img src='speed.png'>\n<br>\n<p><a href='../apt-panopticon_cgp/'>More graphs.</a> with greater detail.</p><hr>\n\n")
results = {}
m = {}
faulty = ""
web:write( "<hr>\n<h2>==== Debian mirror status ====</h2>\n" ..
"<p>NOTE - This is not fully probing the Debian mirrors, we just collect some data from any redirects to other servers. &nbsp; " ..
"So this isn't a full set of tests.</p>\n" ..
"<p><font style='background-color:red; color:black'>EXPERIMENTAL CODE - this is even more experimental than the rest.</font></p>\n" ..
"<table>\n<tr><th></th><th>FTP</th><th>HTTP</th><th>HTTPS</th><th>RSYNC</th><th>DNS round robin</th>" ..
"<th>Protocol</th><th>URL sanity</th><th>Integrity</th><th>Updated</th><th colspan='2'>Speed range</th></tr>\n")
for k, v in APT.orderedPairs(APT.debians) do
local results = APT.collateAll(APT.debians, 'results', k)
local active = ""
if "yes" == v.Active then
web:write(" <tr><th>" .. k .. "</th> ")
else
if nil == v.Active then active = 'nil' else active = v.Active end
web:write(" <tr style='background-color:dimgrey'><th>" .. k .. "</th> ")
end
local ftp = "[<font color='grey'><b>skip</b></font>]"
local http = status(APT.debians, k, results, "http")
local https = status(APT.debians, k, results, "https")
local rsync = "[<font color='grey'><b>skip</b></font>]"
local dns = ""
local protocol = status(APT.debians, k, results, "Protocol")
local sanity = status(APT.debians, k, results, "URLSanity")
local integrity = status(APT.debians, k, results, "Integrity")
local updated = status(APT.debians, k, results, "Updated")
local rate = v.Rate
if nil ~= rate then updated = updated .. ' ' .. rate end
local min = tonumber(results.speed.min)
local max = tonumber(results.speed.max)
local spd = ''
-- DNS-RR test.
if ("deb.devuan.org" ~= k) and (nil ~= APT.debians["deb.devuan.org"]) then
for l, w in pairs(APT.debians[k].IPs) do
if type(w) == "table" then
for i, u in pairs(w) do
if nil ~= APT.debians["deb.devuan.org"].IPs["deb.roundr.devuan.org"][i] then
local log = logCount("deb.devuan.org", i)
if "" ~= log then
if "" == dns then dns = " " else dns = dns .. " &nbsp; " end
dns = dns .. logCount("deb.devuan.org", i)
else
if "" == dns then dns = " " else dns = dns .. " &nbsp; " end
dns = dns .. "<font color='maroon'><b>" .. i .. "</b></font>"
end
end
end
else
if nil ~= APT.debians["deb.devuan.org"].IPs["deb.roundr.devuan.org"][l] then
local log = logCount("deb.devuan.org", l)
if "" ~= log then
if "" == dns then dns = " " else dns = dns .. " &nbsp; " end
dns = dns .. log
else
if "" == dns then dns = " " else dns = dns .. " &nbsp; " end
dns = dns .. "<font color='maroon'><b>" .. l .. "</b></font>"
end
end
end
end
if "" == dns then dns = "[<font color='grey'><b>no</b></font>]" end
if 0 == max then
spd = '<td></td><td></td>'
else
spd = string.format('<td align="right">%d -></td><td align="right">%d</td>', min, max)
end
end
web:write("<td>" .. ftp .. "&nbsp;</td><td>" .. http .. "&nbsp;</td><td>" .. https .. "&nbsp;</td><td>" .. rsync .. "&nbsp;</td><td>" .. dns ..
"&nbsp;</td><td>" .. protocol .. "&nbsp;</td><td>" .. sanity ..
"&nbsp;</td><td>" .. integrity .. "&nbsp;</td><td>" .. updated .. "&nbsp;</td>" .. spd .. "</tr>\n")
if "" ~= active then
web:write("<tr><td style='background-color:dimgrey'>" .. active .. "</td></tr>\n")
end
end
web:write( "</table>\n<br>\n\n")
web:write( "<br>\n<br>\n<h2>==== Debian DNS and logs: ====</h2>\n")
for k, v in pairs(APT.debians) do
local log = k
local n = {}
log = logCount(k)
APT.debians[k].Protocols = nil
APT.debians[k].FQDN = nil
APT.debians[k].Active = nil
APT.debians[k].Rate = nil
APT.debians[k].BaseURL = nil
APT.debians[k].Country = nil
APT.debians[k].Bandwidth = nil
for l, w in pairs(APT.debians[k].IPs) do
if type(w) == "table" then
n[l] = {}
for i, u in pairs(w) do
local log = logCount(k, i)
if "" == log then n[l][i] = u else n[l][log .. " " .. revDNS(APT.debians, k, i)] = u end
end
else
local log = logCount(k, l)
if "" == log then n[l] = w else n[log .. " " .. revDNS(APT.debians, k, l)] = w end
end
end
m[log .. " DNS entries -" .. redirs(APT.mirrors, k)] = n
end
web:write(APT.dumpTableHTML(m, "", ""))
web:write( "<hr>\n<hr>\n<p>The <a href='Report-email.txt'>email report</a>. &nbsp; " ..
"All <a href='../results'>the logs and other output</a>. &nbsp; " ..
"You can get the <a href='https://sledjhamr.org/cgit/apt-panopticon/about/'>source code here</a>.</p>\n")
local status, whn = APT.execute('TZ="GMT" ls -l1 --time-style="+%s" results/stamp | cut -d " " -f 6-6')

103
apt-panopticon.lua
View File

@ -376,6 +376,8 @@ checkHEAD = function (host, URL, r, retry, sanity)
local extraArgs = sendArgs .. ' -o -r '
if 'https' == pu.scheme then extraArgs = extraArgs .. ' --tests=-http' end
if 'http' == pu.scheme then extraArgs = extraArgs .. ' --tests=-https' end
local pth = path:match('^(.*/pool/).*$')
if nil ~= pth then table.insert(APT.results[PU.scheme].redirects, pu.host .. "/" .. pth) else E('Odd redirect path ' .. path) end
I(" Now checking redirected host " .. u)
APT.fork("ionice -c3 nice -n 19 " .. downloadLock .. "REDIR-" .. check .. ".log.txt" .. " ./apt-panopticon.lua " .. extraArgs .. ' ' .. pu.host .. "/" .. path .. " " .. file)
D('logging to ' .. APT.logName(pu.host, nil, file)[2])
@ -422,12 +424,10 @@ local checkFiles = function (host, ip, path, file)
end
end
end
for i, s in pairs(referenceDebs) do
if checkTimeouts(host, "http", ip .. path .. "/" .. s) then return end
if checkTimeouts(host, "https", ip .. path .. "/" .. s) then return end
end
for i, s in pairs(referenceDevs) do
if checkTimeouts(host, "http", ip .. path .. "/" .. s) then return end
if checkTimeouts(host, "https", ip .. path .. "/" .. s) then return end
@ -994,7 +994,7 @@ if 0 < #arg then
rfile:close()
end
if APT.origin and APT.options.referenceSite.value ~= pu.host then
if APT.origin and (not APT.redir) and (APT.options.referenceSite.value ~= pu.host) then
os.execute('sleep 1') -- Wait for things to start up before checking for them.
while 0 < APT.checkExes(downloadLock .. "Release-" .. pu.host .. ".log.txt") do os.execute("sleep 10") end
while 0 < APT.checkExes(downloadLock .. "Packages-" .. pu.host .. ".log.txt") do os.execute("sleep 10") end
@ -1038,6 +1038,103 @@ else
os.execute('sleep 1') -- Wait for things to start up before checking for them.
while 1 <= APT.checkExes("apt-panopticon.lua " .. sendArgs) do os.execute("sleep 10") end
local APT_args = APT.args
local APT_logFile = APT.logFile
local debians = {}
local srvs = io.popen('ls -1 results/*.lua')
for l in srvs:lines() do
local hst = l:sub(9, -5)
if (l:find('_') == nil) and (nil == APT.mirrors[hst]) then
local ips = loadfile(l)().IPs
if nil ~= ips then
debians[hst] = {Country = '', FDQN = hst, Active = 'yes', Rate = '', BaseURL = hst, Protocols = {http = true, https = true}, Bandwidth = '', IPs = ips}
local baseFiles = {}
local IPfiles = {}
for i, a in pairs(ips) do
IPfiles[i] = {}
if type(a) == 'table' then
for j, b in pairs(a) do
IPfiles[i][j] = {}
end
else
end
end
local files = io.popen('ls -1 results/LOG_' .. hst .. '_*.html')
for ll in files:lines() do
local dn = false
for i, a in pairs(ips) do
if type(a) == 'table' then
for j, b in pairs(a) do
if nil ~= ll:match('(results/LOG_' .. hst .. '_' .. j .. '_.*)') then
table.insert(IPfiles[i][j], ll)
dn = true
end
end
else
if nil ~= ll:match('(results/LOG_' .. hst .. '_' .. i .. '_.*)') then
table.insert(IPfiles[i], ll)
dn = true
end
end
end
if not dn then table.insert(baseFiles, ll) end
end
APT.logOpen(hst)
APT.logFile:write('<h1>Note log lines will be out of order, this is a bunch of other log files combined.</h1>\n')
for i, f in pairs(baseFiles) do
f = f:sub(9, -1)
APT.logFile:write('<hr>\n<hr>\n<h2><a href="' .. f .. '">' .. f .. '</a></h2>\n')
for l in io.lines('results/' .. f) do
if l:match('^' .. os.date('%a %b %d ') .. '.*$') then APT.logFile:write(l .. '\n') end
end
end
APT.logPost()
APT.args = APT_args
APT.logFile = APT_logFile
for ip, a in pairs(IPfiles) do
if nil == a[1] then
for i, f in pairs(a) do
if not APT.logOpen(hst, i) then print('PROBLEM OPENING LOG FILE ' .. hst .. ' ' .. i) end
APT.logFile:write('<h1>Note log lines will be out of order, this is a bunch of other log files combined.</h1>\n')
for j, g in pairs(f) do
g = g:sub(9, -1)
APT.logFile:write('<hr>\n<hr>\n<h2><a href="' .. g .. '">' .. g .. '</a></h2>\n')
for l in io.lines('results/' .. g) do
if l:match('^' .. os.date('%a %b %d ') .. '.*$') then APT.logFile:write(l .. '\n') end
end
end
APT.logPost()
APT.args = APT_args
APT.logFile = APT_logFile
end
else
if not APT.logOpen(hst, ip) then print('PROBLEM OPENING LOG FILE ' .. hst .. ' ' .. ip) end
APT.logFile:write('<h1>Note log lines will be out of order, this is a bunch of other log files combined.</h1>\n')
for i, f in pairs(a) do
f = f:sub(9, -1)
APT.logFile:write('<hr>\n<hr>\n<h2><a href="' .. f .. '">' .. f .. '</a></h2>\n')
for l in io.lines('results/' .. f) do
if l:match('^' .. os.date('%a %b %d ') .. '.*$') then APT.logFile:write(l .. '\n') end
end
end
APT.logPost()
APT.args = APT_args
APT.logFile = APT_logFile
end
end
end
end
end
local file, e = io.open("results/debians.lua", "w+")
if nil == file then C("opening debians file - " .. e) else
file:write(APT.dumpTable(debians, "", "debians") .. "\nreturn debians\n")
file:close()
end
for k, v in pairs(APT.mirrors) do
local f = 'results/' .. k .. '.lua'
if APT.checkFile(f) then

Write Preview
Loading…
Cancel
Save