diff --git a/README.md b/README.md index 07ae584..4acffda 100644 --- a/README.md +++ b/README.md @@ -161,6 +161,61 @@ auto_ssl:set("allow_domain", function(domain, auto_ssl, ssl_options, renewal) end) ``` +#### `get_failures` + +The optional `get_failures` function accepts a domain name argument, and can be used to retrieve statistics about failed certificate requests concerning the domain. The function will return a table with fields `first` (timestamp of first failure encountered), `last` (timestamp of most recent failure encountered), `num` (number of failures). The function will instead return `nil` if no error has been encountered. + +Note: the statistics are only kept for as long as the nginx instance is running. There is no sharing across multiple servers (as in a load-balanced environment) implemented. + +To make use of the `get_failures` function, add the following to the `http` configuration block: + +```nginx + lua_shared_dict auto_ssl_failures 1m; +``` + +When this shm-based dictionary exists, `lua-resty-auto-ssl` will use it to update a record it keeps for the domain whenever a Let's Encrypt certificate request fails (for both new domains, as well as renewing ones). When a certificate request is successful, `lua-resty-auto-ssl` will delete the record it has for the domain, so that future invocations will return `nil`. + +The `get_failures` function can be used inside `allow_domain` to implement per-domain rate-limiting, and similar rule sets. + +*Example:* + +```lua +auto_ssl:set("allow_domain", function(domain, auto_ssl, ssl_options, renewal) + local failures = auto_ssl:get_failures(domain) + -- only attempt one certificate request per hour + if not failures or 3600 < ngx.now() - failures["last"] then + return true + else + return false + end +end) +``` + +#### `track_failure` + +The optional `track_failure` function accepts a domain name argument and records a failure for this domain. This can be used to avoid repeated lookups of a domain in `allow_domain`. + +*Example:* + +```lua +auto_ssl:set("allow_domain", function(domain, auto_ssl, ssl_options, renewal) + local failures = auto_ssl:get_failures(domain) + -- only attempt one lookup or certificate request per hour + if failures and ngx.now() - failures["last"] <= 3600 then + return false + end + + local allow + -- (external lookup to check domain, e.g. via http) + if not allow then + auto_ssl:track_failure(domain) + return false + else + return true + end +end) +``` + ### `dir` *Default:* `/etc/resty-auto-ssl` @@ -183,6 +238,11 @@ How frequently (in seconds) all of the domains should be checked for certificate auto_ssl:set("renew_check_interval", 172800) ``` +### `renewals_per_hour` +*Default:* `60` + +How many renewal requests to issue per hour at most. The ACME v2 protocol limits each account to 300 new orders per 3 hours. This setting will throttle the renewal job so that a sufficient margin remains available for new domains at all times. You might consider lowering this setting when the same Let's Encrypt account credentials are shared across multiple servers (in a load-balanced environment). + ### `storage_adapter` *Default:* `resty.auto-ssl.storage_adapters.file`
*Options:* `resty.auto-ssl.storage_adapters.file`, `resty.auto-ssl.storage_adapters.redis` diff --git a/lib/resty/auto-ssl.lua b/lib/resty/auto-ssl.lua index 4640bd7..094f522 100644 --- a/lib/resty/auto-ssl.lua +++ b/lib/resty/auto-ssl.lua @@ -44,6 +44,10 @@ function _M.new(options) options["renew_check_interval"] = 86400 -- 1 day end + if not options["renewals_per_hour"] then + options["renewals_per_hour"] = 60 + end + if not options["hook_server_port"] then options["hook_server_port"] = 8999 end @@ -95,4 +99,62 @@ function _M.hook_server(self) server(self) end +function _M.get_failures(self, domain) + if not ngx.shared.auto_ssl_failures then + ngx.log(ngx.ERR, "auto-ssl: dict auto_ssl_failures could not be found. Please add it to your configuration: `lua_shared_dict auto_ssl_failures 1m;`") + return + end + + local string = ngx.shared.auto_ssl_failures:get("domain:" .. domain) + if string then + local failures, json_err = self.storage.json_adapter:decode(string) + if json_err then + ngx.log(ngx.ERR, json_err, domain) + end + if failures then + local mt = { + __concat = function(op1, op2) + return tostring(op1) .. tostring(op2) + end, + __tostring = function(f) + return "first: " .. f["first"] .. ", last: " .. f["last"] .. ", num: " .. f["num"] + end + } + setmetatable(failures, mt) + return failures + end + end +end + +function _M.track_failure(self, domain) + if not ngx.shared.auto_ssl_failures then + return + end + + local failures + local string = ngx.shared.auto_ssl_failures:get("domain:" .. domain) + if string then + failures = self.storage.json_adapter:decode(string) + end + if not failures then + failures = {} + failures["first"] = ngx.now() + failures["last"] = failures["first"] + failures["num"] = 1 + else + failures["last"] = ngx.now() + failures["num"] = failures["num"] + 1 + end + string = self.storage.json_adapter:encode(failures) + ngx.shared.auto_ssl_failures:set("domain:" .. domain, string, 2592000) +end + +function _M.track_success(_, domain) + if not ngx.shared.auto_ssl_failures then + return + end + + ngx.shared.auto_ssl_failures:delete("domain:" .. domain) +end + return _M diff --git a/lib/resty/auto-ssl/init_worker.lua b/lib/resty/auto-ssl/init_worker.lua index e2fce9b..d273721 100644 --- a/lib/resty/auto-ssl/init_worker.lua +++ b/lib/resty/auto-ssl/init_worker.lua @@ -2,6 +2,7 @@ local random_seed = require "resty.auto-ssl.utils.random_seed" local renewal_job = require "resty.auto-ssl.jobs.renewal" local shell_blocking = require "shell-games" local start_sockproc = require "resty.auto-ssl.utils.start_sockproc" +local timer_rand = math.random() return function(auto_ssl_instance) local base_dir = auto_ssl_instance:get("dir") @@ -37,5 +38,5 @@ return function(auto_ssl_instance) storage_adapter:setup_worker() end - renewal_job.spawn(auto_ssl_instance) + renewal_job.spawn(auto_ssl_instance, timer_rand) end diff --git a/lib/resty/auto-ssl/jobs/renewal.lua b/lib/resty/auto-ssl/jobs/renewal.lua index 4a74a6e..53112a3 100644 --- a/lib/resty/auto-ssl/jobs/renewal.lua +++ b/lib/resty/auto-ssl/jobs/renewal.lua @@ -5,6 +5,8 @@ local shuffle_table = require "resty.auto-ssl.utils.shuffle_table" local ssl_provider = require "resty.auto-ssl.ssl_providers.lets_encrypt" local _M = {} +local min_renewal_seconds +local last_renewal -- Based on lua-rest-upstream-healthcheck's lock: -- https://github.com/openresty/lua-resty-upstream-healthcheck/blob/v0.03/lib/resty/upstream/healthcheck.lua#L423-L440 @@ -125,12 +127,22 @@ local function renew_check_cert(auto_ssl_instance, storage, domain) end -- If expiry date is known, attempt renewal if it's within 30 days. + -- Between 30 and 15 days out, only attempt renewal of a subset of domains (with + -- increasing likelihood of renewal being attempted). if cert["expiry"] then local now = ngx.now() if now + (30 * 24 * 60 * 60) < cert["expiry"] then ngx.log(ngx.NOTICE, "auto-ssl: expiry date is more than 30 days out, skipping renewal: ", domain) renew_check_cert_unlock(domain, storage, local_lock, distributed_lock_value) return + elseif now + (15 * 24 * 60 * 60) < cert["expiry"] then + local rand_value = math.random(cert["expiry"] - (30 * 24 * 60 * 60), cert["expiry"] - (15 * 24 * 60 * 60)) + local rand_renewal_threshold = now + if rand_value < rand_renewal_threshold then + ngx.log(ngx.NOTICE, "auto-ssl: expiry date is more than 15 days out, randomly not picked for renewal: ", domain) + renew_check_cert_unlock(domain, storage, local_lock, distributed_lock_value) + return + end end end @@ -181,9 +193,25 @@ local function renew_check_cert(auto_ssl_instance, storage, domain) ngx.log(ngx.WARN, "auto-ssl: existing certificate is expired, deleting: ", domain) storage:delete_cert(domain) end + + auto_ssl_instance:track_failure(domain) + else + auto_ssl_instance:track_success(domain) end renew_check_cert_unlock(domain, storage, local_lock, distributed_lock_value) + + -- Throttle renewal requests based on renewals_per_hour setting. + if last_renewal and ngx.now() - last_renewal < min_renewal_seconds then + local to_sleep = min_renewal_seconds - (ngx.now() - last_renewal) + ngx.log(ngx.NOTICE, "auto-ssl: pausing renewal job for " .. to_sleep .. " seconds") + ngx.sleep(to_sleep) + end + if last_renewal then + last_renewal = last_renewal + min_renewal_seconds + else + last_renewal = ngx.now() + end end local function renew_all_domains(auto_ssl_instance) @@ -199,6 +227,10 @@ local function renew_all_domains(auto_ssl_instance) -- renewal attempts). shuffle_table(domains) + -- Set up renewal request throttling. + min_renewal_seconds = 3600 / auto_ssl_instance:get("renewals_per_hour") + last_renewal = ngx.now() + for _, domain in ipairs(domains) do renew_check_cert(auto_ssl_instance, storage, domain) end @@ -236,12 +268,14 @@ end local function renew(premature, auto_ssl_instance) if premature then return end + local start = ngx.now() local renew_ok, renew_err = pcall(do_renew, auto_ssl_instance) if not renew_ok then ngx.log(ngx.ERR, "auto-ssl: failed to run do_renew cycle: ", renew_err) end - local timer_ok, timer_err = ngx.timer.at(auto_ssl_instance:get("renew_check_interval"), renew, auto_ssl_instance) + local delay = math.max(0, auto_ssl_instance:get("renew_check_interval") - (ngx.now() - start)) + local timer_ok, timer_err = ngx.timer.at(delay, renew, auto_ssl_instance) if not timer_ok then if timer_err ~= "process exiting" then ngx.log(ngx.ERR, "auto-ssl: failed to create timer: ", timer_err) @@ -250,8 +284,8 @@ local function renew(premature, auto_ssl_instance) end end -function _M.spawn(auto_ssl_instance) - local ok, err = ngx.timer.at(auto_ssl_instance:get("renew_check_interval"), renew, auto_ssl_instance) +function _M.spawn(auto_ssl_instance, timer_rand) + local ok, err = ngx.timer.at(timer_rand * auto_ssl_instance:get("renew_check_interval"), renew, auto_ssl_instance) if not ok then ngx.log(ngx.ERR, "auto-ssl: failed to create timer: ", err) return diff --git a/lib/resty/auto-ssl/ssl_certificate.lua b/lib/resty/auto-ssl/ssl_certificate.lua index ab71cfd..69f838e 100644 --- a/lib/resty/auto-ssl/ssl_certificate.lua +++ b/lib/resty/auto-ssl/ssl_certificate.lua @@ -95,6 +95,9 @@ local function issue_cert(auto_ssl_instance, storage, domain) cert, err = ssl_provider.issue_cert(auto_ssl_instance, domain) if err then ngx.log(ngx.ERR, "auto-ssl: issuing new certificate failed: ", err) + auto_ssl_instance:track_failure(domain) + else + auto_ssl_instance:track_success(domain) end issue_cert_unlock(domain, storage, local_lock, distributed_lock_value) diff --git a/spec/config/nginx.conf.etlua b/spec/config/nginx.conf.etlua index 56c45a3..a7b5119 100644 --- a/spec/config/nginx.conf.etlua +++ b/spec/config/nginx.conf.etlua @@ -33,6 +33,7 @@ http { allow_domain = function(domain) return true end, + renewals_per_hour = 3600, } <%- auto_ssl_pre_new or "" %> auto_ssl = (require "resty.auto-ssl").new(options)