Change interval

README.md

@@ -8,9 +8,12 @@ A small single-container Python 3 uptime monitor.
 - **DNS resolution never uses the system/router-provided resolver** —
   every lookup is sent explicitly and only to the DNS server configured
   for that endpoint (see "How DNS enforcement works" below).
-- Runs a full rolling pass over all endpoints every 10 minutes (configurable).
+- Checks are spread evenly across `interval_seconds` rather than all
+  firing at once: with N endpoints, one is checked every
+  `interval_seconds / N` seconds, rotating through all of them (see
+  "Scheduling" below).
 - If any endpoint fails, all endpoints are immediately re-checked once,
-  instead of waiting for the next scheduled pass.
+  instead of waiting for their normal rotation slot.
 - Writes every check result to an InfluxDB 2.x bucket (`uptime`) over
   HTTPS, with certificate verification disabled for InfluxDB specifically
   (self-signed cert).
@@ -47,7 +50,8 @@ influx:
   verify_ssl: false      # keep false: influx uses a self-signed cert
 
 check:
-  interval_seconds: 600                  # 10 min rolling interval
+  interval_seconds: 600                  # each endpoint checked once per this many seconds,
+                                          # spread evenly across N endpoints (see Scheduling below)
   timeout_seconds: 10                     # default per-check timeout
   immediate_recheck_delay_seconds: 5      # pause before the failure-triggered recheck
 
@@ -106,15 +110,38 @@ from(bucket: "uptime")
   |> filter(fn: (r) => r._field == "success")
 ```
 
+## Scheduling
+
+Endpoints are **not** all checked together and then slept on as a batch.
+Each endpoint gets its own slot, spread evenly across `interval_seconds`:
+with N endpoints configured, one is checked every
+`interval_seconds / N` seconds, rotating through all of them. So
+`interval_seconds: 3600` with 6 endpoints means one endpoint is checked
+every 10 minutes — each individual endpoint is still only checked once
+per hour, but the 6 checks are spread across that hour instead of firing
+all at once, then waiting an hour.
+
+Example with 3 endpoints and `interval_seconds: 600` (so ~200s apart):
+
+```
+t=0s     endpoint A checked
+t=200s   endpoint B checked
+t=400s   endpoint C checked
+t=600s   endpoint A checked again   (600s after its previous check)
+t=800s   endpoint B checked again
+...
+```
+
 ## Failure / immediate recheck behavior
 
-- Every `check.interval_seconds`, all endpoints are checked in sequence.
-- If any endpoint fails that pass, after `immediate_recheck_delay_seconds`
-  all endpoints are checked again immediately (one extra pass).
-- The monitor then returns to the normal `interval_seconds` schedule.
-  If the outage continues, the next scheduled pass will detect it again
-  and trigger another single immediate recheck — it does not hammer the
-  endpoint in a tight loop.
+- If any single endpoint check fails, **all** endpoints are checked once,
+  immediately, after `immediate_recheck_delay_seconds` — instead of
+  waiting for their normal rotation slot.
+- The rotation is then re-spread evenly from that point, so it doesn't
+  bunch endpoints back up onto the same slot, and resumes normally.
+- If the outage continues, that endpoint's next regular slot will detect
+  it again and trigger another single immediate recheck of everyone —
+  it does not hammer the endpoint in a tight loop.
 
 ## Local dev (without Docker)
 

app/checker.py

@@ -75,18 +75,22 @@ def check_endpoint(ep, default_timeout: float, timestamp_ns: int) -> CheckResult
         )
 
 
+def log_result(result: CheckResult) -> None:
+    level = logging.INFO if result.success else logging.WARNING
+    logger.log(
+        level,
+        "endpoint=%s success=%s status=%s expected=%s ip=%s time_ms=%s error=%s",
+        result.endpoint_name, result.success, result.status_code,
+        result.expected_status, result.resolved_ip,
+        None if result.response_time_ms is None else round(result.response_time_ms, 1),
+        result.error,
+    )
+
+
 def check_all(endpoints, default_timeout: float, timestamp_ns: int) -> List[CheckResult]:
     results = []
     for ep in endpoints:
         result = check_endpoint(ep, default_timeout, timestamp_ns)
-        level = logging.INFO if result.success else logging.WARNING
-        logger.log(
-            level,
-            "endpoint=%s success=%s status=%s expected=%s ip=%s time_ms=%s error=%s",
-            result.endpoint_name, result.success, result.status_code,
-            result.expected_status, result.resolved_ip,
-            None if result.response_time_ms is None else round(result.response_time_ms, 1),
-            result.error,
-        )
+        log_result(result)
         results.append(result)
     return results

app/main.py

@@ -3,11 +3,9 @@ from __future__ import annotations
 import logging
 import os
 import sys
-import time
 
-from app.checker import check_all
 from app.config import ConfigError, load_config
-from app.influx_writer import write_results
+from app.scheduler import run_forever
 
 logger = logging.getLogger("uptime_monitor")
 
@@ -20,13 +18,6 @@ def setup_logging() -> None:
     )
 
 
-def run_cycle(cfg) -> list:
-    timestamp_ns = time.time_ns()
-    results = check_all(cfg.endpoints, cfg.check.timeout_seconds, timestamp_ns)
-    write_results(cfg.influx, results)
-    return results
-
-
 def main() -> None:
     setup_logging()
     config_path = os.environ.get("CONFIG_PATH", "/config/config.yaml")
@@ -42,19 +33,7 @@ def main() -> None:
         len(cfg.endpoints), config_path, cfg.check.interval_seconds,
     )
 
-    while True:
-        results = run_cycle(cfg)
-
-        if any(not r.success for r in results):
-            logger.warning(
-                "at least one endpoint failed this cycle; "
-                "re-checking all endpoints immediately in %.0fs",
-                cfg.check.immediate_recheck_delay_seconds,
-            )
-            time.sleep(cfg.check.immediate_recheck_delay_seconds)
-            run_cycle(cfg)
-
-        time.sleep(cfg.check.interval_seconds)
+    run_forever(cfg)
 
 
 if __name__ == "__main__":

app/scheduler.py

@@ -0,0 +1,67 @@
+"""Rolling scheduler: spreads endpoint checks evenly across the interval.
+
+With N endpoints and `interval_seconds` = T, each endpoint gets its own
+slot, T/N seconds apart, and is checked once every T seconds — e.g. 6
+endpoints with interval_seconds: 3600 means one endpoint is checked every
+10 minutes, rotating through all six over the course of an hour. This is
+NOT "check everything, then sleep T seconds": individual endpoint checks
+are spread out, not bunched together.
+
+If any single check fails, ALL endpoints are checked once immediately
+(out of band) as an extra confirmation pass, and the rotation is then
+re-staggered from that point so it stays evenly spread afterward.
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+from typing import List
+
+from .checker import check_all, check_endpoint, log_result
+from .influx_writer import write_results
+
+logger = logging.getLogger("uptime_monitor.scheduler")
+
+
+def run_forever(cfg) -> None:
+    endpoints = cfg.endpoints
+    interval = cfg.check.interval_seconds
+    n = len(endpoints)
+    stagger = interval / n
+
+    now = time.monotonic()
+    next_at: List[float] = [now + i * stagger for i in range(n)]
+
+    logger.info(
+        "rolling schedule: %d endpoint(s) spread across %ss (~%.1fs apart)",
+        n, interval, stagger,
+    )
+
+    while True:
+        idx = min(range(n), key=lambda i: next_at[i])
+        wait = next_at[idx] - time.monotonic()
+        if wait > 0:
+            time.sleep(wait)
+
+        ep = endpoints[idx]
+        result = check_endpoint(ep, cfg.check.timeout_seconds, time.time_ns())
+        log_result(result)
+        write_results(cfg.influx, [result])
+
+        # schedule this endpoint's next regular slot
+        next_at[idx] += interval
+
+        if not result.success:
+            logger.warning(
+                "endpoint=%s failed; re-checking all endpoints immediately in %.0fs",
+                ep.name, cfg.check.immediate_recheck_delay_seconds,
+            )
+            time.sleep(cfg.check.immediate_recheck_delay_seconds)
+            results = check_all(endpoints, cfg.check.timeout_seconds, time.time_ns())
+            write_results(cfg.influx, results)
+
+            # Re-stagger everyone from now so the rotation stays evenly
+            # spread out afterward, instead of bunching back up.
+            now = time.monotonic()
+            next_at = [now + (i + 1) * stagger for i in range(n)]

config.example.yaml

@@ -24,17 +24,22 @@ influx:
   verify_ssl: false
 
 check:
-  # How often to run the full rolling check of all endpoints, in seconds.
-  # 600 = 10 minutes.
+  # Each endpoint is checked once every interval_seconds, but checks are
+  # spread evenly across that window rather than all fired at once — with
+  # N endpoints configured, one endpoint is checked every
+  # (interval_seconds / N) seconds, rotating through all of them.
+  # Example: 6 endpoints + interval_seconds: 3600 (1 hour) -> one endpoint
+  # checked every 10 minutes.
   interval_seconds: 600
 
   # Default per-request timeout (DNS lookup + HTTP request), in seconds.
   # Can be overridden per endpoint.
   timeout_seconds: 10
 
-  # If any endpoint fails during a cycle, all endpoints are re-checked
-  # once, immediately, after this short delay (in seconds) — instead of
-  # waiting for the next scheduled cycle. Regular scheduling then resumes.
+  # If any single endpoint check fails, ALL endpoints are checked once,
+  # immediately, after this short delay (in seconds) — instead of waiting
+  # for their normal rotation slot. The rotation is then re-spread evenly
+  # from that point and resumes normally.
   immediate_recheck_delay_seconds: 5
 
 endpoints: