mirrors/prosody
1
0
Fork 0
mirror of https://github.com/bjc/prosody.git synced 2025-04-04 05:37:39 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

mod_http_file_share: Always measure total disk usage for statistics!

Browse source 
Metrics available or not depending on configuration is weird, even tho
it might be expensive to calculate and it's only really needed when
there is a global quota.

Default quota is set to infinity, which is essentially what it was.

Reports NaN if there is an error, which should count as over the
infinite default quota.
This commit is contained in:
Kim Alvefur 2022-01-11 04:15:29 +01:00
parent 2a0311098c
commit ec9ec1b9a0
1 changed files with 46 additions and 49 deletions
Download patch file Download diff file Expand all files Collapse all files

95
plugins/mod_http_file_share.lua
View file

@ -20,6 +20,9 @@ local hi = require "util.human.units";
local cache = require "util.cache";
local lfs = require "lfs";
local unknown = math.abs(0/0);
local unlimited = math.huge;
local namespace = "urn:xmpp:http:upload:0";
module:depends("disco");
@ -38,7 +41,7 @@ local file_types = module:get_option_set(module.name .. "_allowed_file_types", {
local safe_types = module:get_option_set(module.name .. "_safe_file_types", {"image/*","video/*","audio/*","text/plain"});
local expiry = module:get_option_number(module.name .. "_expires_after", 7 * 86400);
local daily_quota = module:get_option_number(module.name .. "_daily_quota", file_size_limit*10); -- 100 MB / day
local total_storage_limit = module:get_option_number(module.name.."_global_quota", nil);
local total_storage_limit = module:get_option_number(module.name.."_global_quota", unlimited);
local access = module:get_option_set(module.name .. "_access", {});
@ -60,30 +63,29 @@ local upload_errors = errors.init(module.name, namespace, {
};
filesizefmt = { type = "modify"; condition = "bad-request"; text = "File size must be positive integer"; };
quota = { type = "wait"; condition = "resource-constraint"; text = "Daily quota reached"; };
unknowntotal = { type = "wait"; condition = "undefined-condition"; text = "Server storage usage not yet calculated" };
outofdisk = { type = "wait"; condition = "resource-constraint"; text = "Server global storage quota reached" };
});
local upload_cache = cache.new(1024);
local quota_cache = cache.new(1024);
local total_storage_usage = nil;
local total_storage_usage = unknown;
local measure_upload_cache_size = module:measure("upload_cache", "amount");
local measure_quota_cache_size = module:measure("quota_cache", "amount");
local measure_total_storage_usage = nil;
if total_storage_limit then
local measure_total_storage_usage = module:measure("total_storage", "amount", { unit = "bytes" });
do
local total, err = persist_stats:get(nil, "total");
if not err then total_storage_usage = tonumber(total) or 0; end
measure_total_storage_usage = module:measure("total_storage", "amount", { unit = "bytes" });
if not err then
total_storage_usage = tonumber(total) or 0;
end
end
module:hook_global("stats-update", function ()
measure_upload_cache_size(upload_cache:count());
measure_quota_cache_size(quota_cache:count());
if total_storage_limit and total_storage_usage then
measure_total_storage_usage(total_storage_usage);
end
measure_total_storage_usage(total_storage_usage);
end);
local buckets = {};
@ -95,7 +97,14 @@ end
local measure_uploads = module:measure("upload", "sizes", {buckets = buckets});
-- Convenience wrapper for logging file sizes
local function B(bytes) return hi.format(bytes, "B", "b"); end
local function B(bytes)
if bytes ~= bytes then
return "unknown"
elseif bytes == unlimited then
return "unlimited";
end
return hi.format(bytes, "B", "b");
end
local function get_filename(slot, create)
return dm.getpath(slot, module.host, module.name, "bin", create)
@ -141,13 +150,9 @@ function may_upload(uploader, filename, filesize, filetype) -- > boolean, error
return false, upload_errors.new("filesize");
end
if total_storage_limit then
if not total_storage_usage then
return false, upload_errors.new("unknowntotal");
elseif total_storage_usage + filesize > total_storage_limit then
module:log("warn", "Global storage quota reached, at %s!", B(total_storage_usage));
return false, upload_errors.new("outofdisk");
end
if total_storage_usage + filesize > total_storage_limit then
module:log("warn", "Global storage quota reached, at %s / %s!", B(total_storage_usage), B(total_storage_limit));
return false, upload_errors.new("outofdisk");
end
local uploader_quota = get_daily_quota(uploader);
@ -217,10 +222,8 @@ function handle_slot_request(event)
return true;
end
if total_storage_usage then
total_storage_usage = total_storage_usage + filesize;
module:log("debug", "Global quota %s / %s", B(total_storage_usage), B(total_storage_limit));
end
total_storage_usage = total_storage_usage + filesize;
module:log("debug", "Total storage usage: %s / %s", B(total_storage_usage), B(total_storage_limit));
local cached_quota = quota_cache:get(uploader);
if cached_quota and cached_quota.time > os.time()-86400 then
@ -472,11 +475,7 @@ if expiry >= 0 and not external_base_url then
end
module:log("info", "Pruning expired files uploaded earlier than %s", dt.datetime(boundary_time));
if total_storage_usage then
module:log("debug", "Global quota %s / %s", B(total_storage_usage), B(total_storage_limit));
elseif total_storage_limit then
module:log("debug", "Global quota %s / %s", "not yet calculated", B(total_storage_limit));
end
module:log("debug", "Total storage usage: %s / %s", B(total_storage_usage), B(total_storage_limit));
local obsolete_uploads = array();
local num_expired = 0;
@ -513,11 +512,9 @@ if expiry >= 0 and not external_base_url then
deletion_query = {ids = obsolete_uploads};
end
if total_storage_usage then
total_storage_usage = total_storage_usage - size_sum;
module:log("debug", "Global quota %s / %s", B(total_storage_usage), B(total_storage_limit));
persist_stats:set(nil, "total", total_storage_usage);
end
total_storage_usage = total_storage_usage - size_sum;
module:log("debug", "Total storage usage: %s / %s", B(total_storage_usage), B(total_storage_limit));
persist_stats:set(nil, "total", total_storage_usage);
if #obsolete_uploads == 0 then
module:log("debug", "No metadata to remove");
@ -535,27 +532,27 @@ if expiry >= 0 and not external_base_url then
end);
end
if total_storage_limit then
local summary_start = module:measure("summary", "times");
local summary_start = module:measure("summary", "times");
module:weekly("Global quota check", function()
local summary_done = summary_start();
local iter = assert(uploads:find(nil));
module:weekly("Calculate total storage usage", function()
local summary_done = summary_start();
local iter = assert(uploads:find(nil));
local count, sum = 0, 0;
for _, file in iter do
sum = sum + tonumber(file.attr.size);
count = count + 1;
end
local count, sum = 0, 0;
for _, file in iter do
sum = sum + tonumber(file.attr.size);
count = count + 1;
end
module:log("info", "Uploaded files total: %s in %d files", B(sum), count);
module:log("info", "Uploaded files total: %s in %d files", B(sum), count);
if persist_stats:set(nil, "total", sum) then
total_storage_usage = sum;
module:log("debug", "Global quota %s / %s", B(total_storage_usage), B(total_storage_limit));
persist_stats:set(nil, "total", sum);
summary_done();
end);
end
else
total_storage_usage = unknown;
end
module:log("debug", "Total storage usage: %s / %s", B(total_storage_usage), B(total_storage_limit));
summary_done();
end);
-- Reachable from the console
function check_files(query)