From 553c035a20cae5f99f1bc5dbf602ac3100a2de87 Mon Sep 17 00:00:00 2001 From: Paul Moeller-Friedrich Date: Thu, 20 Aug 2026 09:24:02 +0200 Subject: [PATCH] Change interval --- README.md | 47 ++++++++++++++++++++++++------- app/checker.py | 22 +++++++++------ app/main.py | 25 ++--------------- app/scheduler.py | 67 +++++++++++++++++++++++++++++++++++++++++++++ config.example.yaml | 15 ++++++---- 5 files changed, 129 insertions(+), 47 deletions(-) create mode 100644 app/scheduler.py diff --git a/README.md b/README.md index 3a775b7..3795c55 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,12 @@ A small single-container Python 3 uptime monitor. - **DNS resolution never uses the system/router-provided resolver** — every lookup is sent explicitly and only to the DNS server configured for that endpoint (see "How DNS enforcement works" below). -- Runs a full rolling pass over all endpoints every 10 minutes (configurable). +- Checks are spread evenly across `interval_seconds` rather than all + firing at once: with N endpoints, one is checked every + `interval_seconds / N` seconds, rotating through all of them (see + "Scheduling" below). - If any endpoint fails, all endpoints are immediately re-checked once, - instead of waiting for the next scheduled pass. + instead of waiting for their normal rotation slot. - Writes every check result to an InfluxDB 2.x bucket (`uptime`) over HTTPS, with certificate verification disabled for InfluxDB specifically (self-signed cert). @@ -47,7 +50,8 @@ influx: verify_ssl: false # keep false: influx uses a self-signed cert check: - interval_seconds: 600 # 10 min rolling interval + interval_seconds: 600 # each endpoint checked once per this many seconds, + # spread evenly across N endpoints (see Scheduling below) timeout_seconds: 10 # default per-check timeout immediate_recheck_delay_seconds: 5 # pause before the failure-triggered recheck @@ -106,15 +110,38 @@ from(bucket: "uptime") |> filter(fn: (r) => r._field == "success") ``` +## Scheduling + +Endpoints are **not** all checked together and then slept on as a batch. +Each endpoint gets its own slot, spread evenly across `interval_seconds`: +with N endpoints configured, one is checked every +`interval_seconds / N` seconds, rotating through all of them. So +`interval_seconds: 3600` with 6 endpoints means one endpoint is checked +every 10 minutes — each individual endpoint is still only checked once +per hour, but the 6 checks are spread across that hour instead of firing +all at once, then waiting an hour. + +Example with 3 endpoints and `interval_seconds: 600` (so ~200s apart): + +``` +t=0s endpoint A checked +t=200s endpoint B checked +t=400s endpoint C checked +t=600s endpoint A checked again (600s after its previous check) +t=800s endpoint B checked again +... +``` + ## Failure / immediate recheck behavior -- Every `check.interval_seconds`, all endpoints are checked in sequence. -- If any endpoint fails that pass, after `immediate_recheck_delay_seconds` - all endpoints are checked again immediately (one extra pass). -- The monitor then returns to the normal `interval_seconds` schedule. - If the outage continues, the next scheduled pass will detect it again - and trigger another single immediate recheck — it does not hammer the - endpoint in a tight loop. +- If any single endpoint check fails, **all** endpoints are checked once, + immediately, after `immediate_recheck_delay_seconds` — instead of + waiting for their normal rotation slot. +- The rotation is then re-spread evenly from that point, so it doesn't + bunch endpoints back up onto the same slot, and resumes normally. +- If the outage continues, that endpoint's next regular slot will detect + it again and trigger another single immediate recheck of everyone — + it does not hammer the endpoint in a tight loop. ## Local dev (without Docker) diff --git a/app/checker.py b/app/checker.py index 155f1f5..3bd5870 100644 --- a/app/checker.py +++ b/app/checker.py @@ -75,18 +75,22 @@ def check_endpoint(ep, default_timeout: float, timestamp_ns: int) -> CheckResult ) +def log_result(result: CheckResult) -> None: + level = logging.INFO if result.success else logging.WARNING + logger.log( + level, + "endpoint=%s success=%s status=%s expected=%s ip=%s time_ms=%s error=%s", + result.endpoint_name, result.success, result.status_code, + result.expected_status, result.resolved_ip, + None if result.response_time_ms is None else round(result.response_time_ms, 1), + result.error, + ) + + def check_all(endpoints, default_timeout: float, timestamp_ns: int) -> List[CheckResult]: results = [] for ep in endpoints: result = check_endpoint(ep, default_timeout, timestamp_ns) - level = logging.INFO if result.success else logging.WARNING - logger.log( - level, - "endpoint=%s success=%s status=%s expected=%s ip=%s time_ms=%s error=%s", - result.endpoint_name, result.success, result.status_code, - result.expected_status, result.resolved_ip, - None if result.response_time_ms is None else round(result.response_time_ms, 1), - result.error, - ) + log_result(result) results.append(result) return results diff --git a/app/main.py b/app/main.py index 262b06e..5999587 100644 --- a/app/main.py +++ b/app/main.py @@ -3,11 +3,9 @@ from __future__ import annotations import logging import os import sys -import time -from app.checker import check_all from app.config import ConfigError, load_config -from app.influx_writer import write_results +from app.scheduler import run_forever logger = logging.getLogger("uptime_monitor") @@ -20,13 +18,6 @@ def setup_logging() -> None: ) -def run_cycle(cfg) -> list: - timestamp_ns = time.time_ns() - results = check_all(cfg.endpoints, cfg.check.timeout_seconds, timestamp_ns) - write_results(cfg.influx, results) - return results - - def main() -> None: setup_logging() config_path = os.environ.get("CONFIG_PATH", "/config/config.yaml") @@ -42,19 +33,7 @@ def main() -> None: len(cfg.endpoints), config_path, cfg.check.interval_seconds, ) - while True: - results = run_cycle(cfg) - - if any(not r.success for r in results): - logger.warning( - "at least one endpoint failed this cycle; " - "re-checking all endpoints immediately in %.0fs", - cfg.check.immediate_recheck_delay_seconds, - ) - time.sleep(cfg.check.immediate_recheck_delay_seconds) - run_cycle(cfg) - - time.sleep(cfg.check.interval_seconds) + run_forever(cfg) if __name__ == "__main__": diff --git a/app/scheduler.py b/app/scheduler.py new file mode 100644 index 0000000..dd83148 --- /dev/null +++ b/app/scheduler.py @@ -0,0 +1,67 @@ +"""Rolling scheduler: spreads endpoint checks evenly across the interval. + +With N endpoints and `interval_seconds` = T, each endpoint gets its own +slot, T/N seconds apart, and is checked once every T seconds — e.g. 6 +endpoints with interval_seconds: 3600 means one endpoint is checked every +10 minutes, rotating through all six over the course of an hour. This is +NOT "check everything, then sleep T seconds": individual endpoint checks +are spread out, not bunched together. + +If any single check fails, ALL endpoints are checked once immediately +(out of band) as an extra confirmation pass, and the rotation is then +re-staggered from that point so it stays evenly spread afterward. +""" + +from __future__ import annotations + +import logging +import time +from typing import List + +from .checker import check_all, check_endpoint, log_result +from .influx_writer import write_results + +logger = logging.getLogger("uptime_monitor.scheduler") + + +def run_forever(cfg) -> None: + endpoints = cfg.endpoints + interval = cfg.check.interval_seconds + n = len(endpoints) + stagger = interval / n + + now = time.monotonic() + next_at: List[float] = [now + i * stagger for i in range(n)] + + logger.info( + "rolling schedule: %d endpoint(s) spread across %ss (~%.1fs apart)", + n, interval, stagger, + ) + + while True: + idx = min(range(n), key=lambda i: next_at[i]) + wait = next_at[idx] - time.monotonic() + if wait > 0: + time.sleep(wait) + + ep = endpoints[idx] + result = check_endpoint(ep, cfg.check.timeout_seconds, time.time_ns()) + log_result(result) + write_results(cfg.influx, [result]) + + # schedule this endpoint's next regular slot + next_at[idx] += interval + + if not result.success: + logger.warning( + "endpoint=%s failed; re-checking all endpoints immediately in %.0fs", + ep.name, cfg.check.immediate_recheck_delay_seconds, + ) + time.sleep(cfg.check.immediate_recheck_delay_seconds) + results = check_all(endpoints, cfg.check.timeout_seconds, time.time_ns()) + write_results(cfg.influx, results) + + # Re-stagger everyone from now so the rotation stays evenly + # spread out afterward, instead of bunching back up. + now = time.monotonic() + next_at = [now + (i + 1) * stagger for i in range(n)] diff --git a/config.example.yaml b/config.example.yaml index 19a5f5d..7c3d6f6 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -24,17 +24,22 @@ influx: verify_ssl: false check: - # How often to run the full rolling check of all endpoints, in seconds. - # 600 = 10 minutes. + # Each endpoint is checked once every interval_seconds, but checks are + # spread evenly across that window rather than all fired at once — with + # N endpoints configured, one endpoint is checked every + # (interval_seconds / N) seconds, rotating through all of them. + # Example: 6 endpoints + interval_seconds: 3600 (1 hour) -> one endpoint + # checked every 10 minutes. interval_seconds: 600 # Default per-request timeout (DNS lookup + HTTP request), in seconds. # Can be overridden per endpoint. timeout_seconds: 10 - # If any endpoint fails during a cycle, all endpoints are re-checked - # once, immediately, after this short delay (in seconds) — instead of - # waiting for the next scheduled cycle. Regular scheduling then resumes. + # If any single endpoint check fails, ALL endpoints are checked once, + # immediately, after this short delay (in seconds) — instead of waiting + # for their normal rotation slot. The rotation is then re-spread evenly + # from that point and resumes normally. immediate_recheck_delay_seconds: 5 endpoints: