uptime-monitor

app/checker.py

@@ -0,0 +1,84 @@
+from __future__ import annotations
+
+import dataclasses
+import logging
+import time
+from typing import List, Optional
+
+import dns.exception
+import requests
+
+from . import dns_resolver
+
+logger = logging.getLogger("uptime_monitor.checker")
+
+
+@dataclasses.dataclass
+class CheckResult:
+    endpoint_name: str
+    host: str
+    scheme: str
+    port: int
+    success: bool
+    status_code: Optional[int]
+    expected_status: int
+    response_time_ms: Optional[float]
+    resolved_ip: Optional[str]
+    error: Optional[str]
+    timestamp_ns: int
+
+
+def check_endpoint(ep, default_timeout: float, timestamp_ns: int) -> CheckResult:
+    timeout = ep.timeout_seconds or default_timeout
+    start = time.monotonic()
+
+    try:
+        resolved_ip = dns_resolver.resolve(ep.host, ep.dns_server, timeout)
+    except dns.exception.DNSException as e:
+        return CheckResult(
+            endpoint_name=ep.name, host=ep.host, scheme=ep.scheme, port=ep.port,
+            success=False, status_code=None, expected_status=ep.expected_status,
+            response_time_ms=None, resolved_ip=None,
+            error=f"dns_error: {e}", timestamp_ns=timestamp_ns,
+        )
+
+    url = f"{ep.scheme}://{ep.host}:{ep.port}{ep.path}"
+    try:
+        with dns_resolver.pin_resolution(ep.host, resolved_ip):
+            resp = requests.get(
+                url, timeout=timeout, verify=ep.verify_tls, allow_redirects=True
+            )
+        elapsed_ms = (time.monotonic() - start) * 1000
+        success = resp.status_code == ep.expected_status
+        return CheckResult(
+            endpoint_name=ep.name, host=ep.host, scheme=ep.scheme, port=ep.port,
+            success=success, status_code=resp.status_code, expected_status=ep.expected_status,
+            response_time_ms=elapsed_ms, resolved_ip=resolved_ip,
+            error=None if success else f"unexpected_status_code:{resp.status_code}",
+            timestamp_ns=timestamp_ns,
+        )
+    except requests.RequestException as e:
+        elapsed_ms = (time.monotonic() - start) * 1000
+        return CheckResult(
+            endpoint_name=ep.name, host=ep.host, scheme=ep.scheme, port=ep.port,
+            success=False, status_code=None, expected_status=ep.expected_status,
+            response_time_ms=elapsed_ms, resolved_ip=resolved_ip,
+            error=str(e), timestamp_ns=timestamp_ns,
+        )
+
+
+def check_all(endpoints, default_timeout: float, timestamp_ns: int) -> List[CheckResult]:
+    results = []
+    for ep in endpoints:
+        result = check_endpoint(ep, default_timeout, timestamp_ns)
+        level = logging.INFO if result.success else logging.WARNING
+        logger.log(
+            level,
+            "endpoint=%s success=%s status=%s expected=%s ip=%s time_ms=%s error=%s",
+            result.endpoint_name, result.success, result.status_code,
+            result.expected_status, result.resolved_ip,
+            None if result.response_time_ms is None else round(result.response_time_ms, 1),
+            result.error,
+        )
+        results.append(result)
+    return results

app/config.py

@@ -0,0 +1,121 @@
+from __future__ import annotations
+
+import dataclasses
+from typing import List, Optional
+
+import yaml
+
+
+class ConfigError(Exception):
+    """Raised for any problem with the config file's contents."""
+
+
+@dataclasses.dataclass
+class Endpoint:
+    name: str
+    host: str
+    scheme: str  # "http" or "https"
+    port: int
+    dns_server: str  # DNS server used to resolve `host` — the system/router
+    # resolver is never consulted, see app/dns_resolver.py
+    expected_status: int
+    path: str = "/"
+    timeout_seconds: Optional[float] = None
+    verify_tls: bool = True  # set False for endpoints with self-signed certs
+
+
+@dataclasses.dataclass
+class InfluxConfig:
+    url: str
+    token: str
+    org: str
+    bucket: str
+    verify_ssl: bool = False  # default off: influx uses a self-signed cert
+
+
+@dataclasses.dataclass
+class CheckConfig:
+    interval_seconds: int = 600
+    timeout_seconds: float = 10.0
+    immediate_recheck_delay_seconds: float = 5.0
+
+
+@dataclasses.dataclass
+class AppConfig:
+    influx: InfluxConfig
+    check: CheckConfig
+    endpoints: List[Endpoint]
+
+
+def _require(d: dict, key: str, ctx: str):
+    value = d.get(key)
+    if value in (None, ""):
+        raise ConfigError(f"Missing required field '{key}' in {ctx}")
+    return value
+
+
+def load_config(path: str) -> AppConfig:
+    try:
+        with open(path, "r", encoding="utf-8") as f:
+            raw = yaml.safe_load(f)
+    except FileNotFoundError:
+        raise ConfigError(f"Config file not found: {path}")
+    except yaml.YAMLError as e:
+        raise ConfigError(f"Invalid YAML in config file '{path}': {e}")
+
+    if not raw:
+        raise ConfigError(f"Config file '{path}' is empty")
+
+    influx_raw = raw.get("influx") or {}
+    influx = InfluxConfig(
+        url=_require(influx_raw, "url", "influx"),
+        token=_require(influx_raw, "token", "influx"),
+        org=_require(influx_raw, "org", "influx"),
+        bucket=_require(influx_raw, "bucket", "influx"),
+        verify_ssl=bool(influx_raw.get("verify_ssl", False)),
+    )
+
+    check_raw = raw.get("check") or {}
+    check = CheckConfig(
+        interval_seconds=int(check_raw.get("interval_seconds", 600)),
+        timeout_seconds=float(check_raw.get("timeout_seconds", 10.0)),
+        immediate_recheck_delay_seconds=float(
+            check_raw.get("immediate_recheck_delay_seconds", 5.0)
+        ),
+    )
+
+    endpoints_raw = raw.get("endpoints") or []
+    if not endpoints_raw:
+        raise ConfigError("No endpoints configured under 'endpoints'")
+
+    endpoints: List[Endpoint] = []
+    seen_names = set()
+    for i, ep in enumerate(endpoints_raw):
+        ctx = f"endpoints[{i}]"
+        name = _require(ep, "name", ctx)
+        if name in seen_names:
+            raise ConfigError(f"Duplicate endpoint name: '{name}'")
+        seen_names.add(name)
+
+        scheme = str(_require(ep, "scheme", ctx)).lower()
+        if scheme not in ("http", "https"):
+            raise ConfigError(
+                f"{ctx} ('{name}'): scheme must be 'http' or 'https', got '{scheme}'"
+            )
+
+        timeout = ep.get("timeout_seconds")
+        endpoints.append(
+            Endpoint(
+                name=name,
+                host=_require(ep, "host", ctx),
+                scheme=scheme,
+                port=int(_require(ep, "port", ctx)),
+                dns_server=_require(ep, "dns_server", ctx),
+                expected_status=int(_require(ep, "expected_status", ctx)),
+                path=ep.get("path", "/"),
+                timeout_seconds=float(timeout) if timeout else None,
+                verify_tls=bool(ep.get("verify_tls", True)),
+            )
+        )
+
+    return AppConfig(influx=influx, check=check, endpoints=endpoints)

app/dns_resolver.py

@@ -0,0 +1,60 @@
+"""DNS resolution that always uses an explicitly configured server.
+
+The system/router-provided resolver (as configured via /etc/resolv.conf,
+usually pointing at your router) is never consulted. Each lookup opens a
+fresh, unconfigured dnspython resolver and points it only at the DNS
+server given for that specific endpoint.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import socket
+import threading
+
+import dns.resolver
+
+# Guards socket.getaddrinfo patching below. Checks must run sequentially
+# while a patch is active since getaddrinfo is a process-global function;
+# the monitor's main loop does exactly that (one endpoint at a time).
+_patch_lock = threading.Lock()
+
+
+def resolve(host: str, dns_server: str, timeout: float) -> str:
+    """Resolve `host` to an IPv4 address using *only* `dns_server`.
+
+    Raises dns.exception.DNSException (or a subclass) on failure —
+    NXDOMAIN, timeout, no answer, etc.
+    """
+    resolver = dns.resolver.Resolver(configure=False)
+    resolver.nameservers = [dns_server]
+    resolver.lifetime = timeout
+    resolver.timeout = timeout
+    answer = resolver.resolve(host, "A")
+    return str(answer[0])
+
+
+@contextlib.contextmanager
+def pin_resolution(hostname: str, ip: str):
+    """Force socket.getaddrinfo(hostname, ...) to return `ip` for the
+    duration of the block.
+
+    This lets `requests`/urllib3 connect to the IP we already resolved via
+    the explicit DNS server above, without doing any resolution of its
+    own (which would otherwise go through the system resolver). The
+    hostname in the URL is left untouched, so the Host header and TLS
+    SNI/certificate verification behave exactly as normal.
+    """
+    original_getaddrinfo = socket.getaddrinfo
+
+    def patched(host, *args, **kwargs):
+        if host == hostname:
+            host = ip
+        return original_getaddrinfo(host, *args, **kwargs)
+
+    with _patch_lock:
+        socket.getaddrinfo = patched
+        try:
+            yield
+        finally:
+            socket.getaddrinfo = original_getaddrinfo

app/influx_writer.py

@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+import logging
+from typing import List
+
+import requests
+import urllib3
+
+from .checker import CheckResult
+
+logger = logging.getLogger("uptime_monitor.influx")
+
+# The InfluxDB endpoint uses a self-signed certificate by design (per
+# config, verify_ssl defaults to False for it) — silence the resulting
+# urllib3 warning so it doesn't spam the logs on every write.
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+
+def _escape_tag(value) -> str:
+    return (
+        str(value)
+        .replace("\\", "\\\\")
+        .replace(",", "\\,")
+        .replace(" ", "\\ ")
+        .replace("=", "\\=")
+    )
+
+
+def _escape_field_string(value) -> str:
+    return str(value).replace("\\", "\\\\").replace('"', '\\"')
+
+
+def result_to_line(result: CheckResult, measurement: str = "uptime_check") -> str:
+    tags = (
+        f"endpoint={_escape_tag(result.endpoint_name)},"
+        f"host={_escape_tag(result.host)},"
+        f"scheme={_escape_tag(result.scheme)},"
+        f"port={result.port}"
+    )
+    fields = [
+        f"success={1 if result.success else 0}i",
+        f"expected_status={result.expected_status}i",
+    ]
+    if result.status_code is not None:
+        fields.append(f"status_code={result.status_code}i")
+    if result.response_time_ms is not None:
+        fields.append(f"response_time_ms={result.response_time_ms}")
+    if result.resolved_ip:
+        fields.append(f'resolved_ip="{_escape_field_string(result.resolved_ip)}"')
+    if result.error:
+        fields.append(f'error="{_escape_field_string(result.error)}"')
+
+    return f"{measurement},{tags} {','.join(fields)} {result.timestamp_ns}"
+
+
+def write_results(influx_cfg, results: List[CheckResult]) -> None:
+    if not results:
+        return
+
+    lines = "\n".join(result_to_line(r) for r in results)
+    url = f"{influx_cfg.url.rstrip('/')}/api/v2/write"
+    params = {"org": influx_cfg.org, "bucket": influx_cfg.bucket, "precision": "ns"}
+    headers = {
+        "Authorization": f"Token {influx_cfg.token}",
+        "Content-Type": "text/plain; charset=utf-8",
+    }
+
+    try:
+        resp = requests.post(
+            url,
+            params=params,
+            headers=headers,
+            data=lines.encode("utf-8"),
+            timeout=10,
+            verify=influx_cfg.verify_ssl,
+        )
+        resp.raise_for_status()
+        logger.info("wrote %d point(s) to influx bucket=%s", len(results), influx_cfg.bucket)
+    except requests.RequestException as e:
+        logger.error("failed to write to influx: %s", e)

app/main.py

@@ -0,0 +1,61 @@
+from __future__ import annotations
+
+import logging
+import os
+import sys
+import time
+
+from app.checker import check_all
+from app.config import ConfigError, load_config
+from app.influx_writer import write_results
+
+logger = logging.getLogger("uptime_monitor")
+
+
+def setup_logging() -> None:
+    logging.basicConfig(
+        level=os.environ.get("LOG_LEVEL", "INFO"),
+        format="%(asctime)s %(levelname)s %(name)s: %(message)s",
+        stream=sys.stdout,
+    )
+
+
+def run_cycle(cfg) -> list:
+    timestamp_ns = time.time_ns()
+    results = check_all(cfg.endpoints, cfg.check.timeout_seconds, timestamp_ns)
+    write_results(cfg.influx, results)
+    return results
+
+
+def main() -> None:
+    setup_logging()
+    config_path = os.environ.get("CONFIG_PATH", "/config/config.yaml")
+
+    try:
+        cfg = load_config(config_path)
+    except ConfigError as e:
+        logger.error("configuration error: %s", e)
+        sys.exit(1)
+
+    logger.info(
+        "loaded %d endpoint(s) from %s; interval=%ss, immediate recheck on failure enabled",
+        len(cfg.endpoints), config_path, cfg.check.interval_seconds,
+    )
+
+    while True:
+        results = run_cycle(cfg)
+
+        if any(not r.success for r in results):
+            logger.warning(
+                "at least one endpoint failed this cycle; "
+                "re-checking all endpoints immediately in %.0fs",
+                cfg.check.immediate_recheck_delay_seconds,
+            )
+            time.sleep(cfg.check.immediate_recheck_delay_seconds)
+            run_cycle(cfg)
+
+        time.sleep(cfg.check.interval_seconds)
+
+
+if __name__ == "__main__":
+    main()