diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..1f8977b --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +config.yaml +config.example.yaml +README.md +.git +__pycache__ +*.pyc diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..4b4a83f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,17 @@ +FROM python:3.12-slim + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY app/ ./app/ + +ENV CONFIG_PATH=/config/config.yaml +ENV LOG_LEVEL=INFO +ENV PYTHONUNBUFFERED=1 + +# config.yaml is intentionally NOT copied into the image — it is bind +# mounted at runtime (see docker-compose.yml) so it can be edited without +# rebuilding this image. A container restart is enough to pick up changes. +ENTRYPOINT ["python", "-m", "app.main"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..6e85923 --- /dev/null +++ b/README.md @@ -0,0 +1,117 @@ +# uptime-monitor + +A small single-container Python 3 uptime monitor. + +- Monitors a configurable list of HTTP/HTTPS endpoints. +- Each endpoint independently specifies: scheme (http/https), port, the + DNS server used to resolve it, and the expected HTTP status code. +- **DNS resolution never uses the system/router-provided resolver** — + every lookup is sent explicitly and only to the DNS server configured + for that endpoint (see "How DNS enforcement works" below). +- Runs a full rolling pass over all endpoints every 10 minutes (configurable). +- If any endpoint fails, all endpoints are immediately re-checked once, + instead of waiting for the next scheduled pass. +- Writes every check result to an InfluxDB 2.x bucket (`uptime`) over + HTTPS, with certificate verification disabled for InfluxDB specifically + (self-signed cert). +- Everything — endpoint list, InfluxDB URL/token/org/bucket, intervals — + lives in one YAML file that's bind-mounted into the container. Edit it + and restart the container; no image rebuild required. + +## Setup + +```bash +cp config.example.yaml config.yaml +# edit config.yaml: fill in your InfluxDB url/token/org, and your endpoints +docker compose up -d --build +``` + +## Editing endpoints / InfluxDB settings later + +```bash +vim config.yaml +docker compose restart uptime-monitor +``` + +No rebuild needed — `config.yaml` is bind-mounted (`docker-compose.yml`), +never baked into the image (see `.dockerignore`). + +## Config reference (`config.yaml`) + +```yaml +influx: + url: "https://influxdb.example.local:8086" + token: "..." # InfluxDB 2.x API token with write access + org: "..." + bucket: "uptime" + verify_ssl: false # keep false: influx uses a self-signed cert + +check: + interval_seconds: 600 # 10 min rolling interval + timeout_seconds: 10 # default per-check timeout + immediate_recheck_delay_seconds: 5 # pause before the failure-triggered recheck + +endpoints: + - name: "my-service" + host: "service.example.com" + scheme: "https" + port: 443 + dns_server: "1.1.1.1" # required, per-endpoint — no default DNS is ever used + expected_status: 200 + path: "/" # optional, default "/" + timeout_seconds: 5 # optional, overrides check.timeout_seconds + verify_tls: true # optional, set false if THIS endpoint has a self-signed cert +``` + +Add/remove/edit entries under `endpoints:` freely. + +## How DNS enforcement works + +For each check: + +1. A fresh `dnspython` resolver is created with `configure=False` (so it + never reads `/etc/resolv.conf`) and pointed at only the endpoint's + `dns_server`. +2. That resolver performs the A-record lookup. +3. The resulting IP is pinned for the single outgoing HTTP(S) request via + a scoped patch of `socket.getaddrinfo` — so `requests`/urllib3 connects + directly to that IP instead of resolving the hostname itself. The + hostname in the URL is left untouched, so the `Host` header and TLS + SNI/certificate checks behave exactly as they normally would. + +At no point is the container's own (router-provided) DNS resolver +consulted for endpoint hostnames. + +## What gets written to InfluxDB + +Measurement `uptime_check`, one point per endpoint per check: + +- tags: `endpoint`, `host`, `scheme`, `port` +- fields: `success` (0/1), `expected_status`, `status_code` (if a response + was received), `response_time_ms`, `resolved_ip`, `error` (if any) + +Example Flux query: + +```flux +from(bucket: "uptime") + |> range(start: -24h) + |> filter(fn: (r) => r._measurement == "uptime_check") + |> filter(fn: (r) => r._field == "success") +``` + +## Failure / immediate recheck behavior + +- Every `check.interval_seconds`, all endpoints are checked in sequence. +- If any endpoint fails that pass, after `immediate_recheck_delay_seconds` + all endpoints are checked again immediately (one extra pass). +- The monitor then returns to the normal `interval_seconds` schedule. + If the outage continues, the next scheduled pass will detect it again + and trigger another single immediate recheck — it does not hammer the + endpoint in a tight loop. + +## Local dev (without Docker) + +```bash +pip install -r requirements.txt +CONFIG_PATH=./config.yaml python -m app.main +``` diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/checker.py b/app/checker.py new file mode 100644 index 0000000..773c190 --- /dev/null +++ b/app/checker.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +import dataclasses +import logging +import time +from typing import List, Optional + +import dns.exception +import requests + +from . import dns_resolver + +logger = logging.getLogger("uptime_monitor.checker") + + +@dataclasses.dataclass +class CheckResult: + endpoint_name: str + host: str + scheme: str + port: int + success: bool + status_code: Optional[int] + expected_status: int + response_time_ms: Optional[float] + resolved_ip: Optional[str] + error: Optional[str] + timestamp_ns: int + + +def check_endpoint(ep, default_timeout: float, timestamp_ns: int) -> CheckResult: + timeout = ep.timeout_seconds or default_timeout + start = time.monotonic() + + try: + resolved_ip = dns_resolver.resolve(ep.host, ep.dns_server, timeout) + except dns.exception.DNSException as e: + return CheckResult( + endpoint_name=ep.name, host=ep.host, scheme=ep.scheme, port=ep.port, + success=False, status_code=None, expected_status=ep.expected_status, + response_time_ms=None, resolved_ip=None, + error=f"dns_error: {e}", timestamp_ns=timestamp_ns, + ) + + url = f"{ep.scheme}://{ep.host}:{ep.port}{ep.path}" + try: + with dns_resolver.pin_resolution(ep.host, resolved_ip): + resp = requests.get( + url, timeout=timeout, verify=ep.verify_tls, allow_redirects=True + ) + elapsed_ms = (time.monotonic() - start) * 1000 + success = resp.status_code == ep.expected_status + return CheckResult( + endpoint_name=ep.name, host=ep.host, scheme=ep.scheme, port=ep.port, + success=success, status_code=resp.status_code, expected_status=ep.expected_status, + response_time_ms=elapsed_ms, resolved_ip=resolved_ip, + error=None if success else f"unexpected_status_code:{resp.status_code}", + timestamp_ns=timestamp_ns, + ) + except requests.RequestException as e: + elapsed_ms = (time.monotonic() - start) * 1000 + return CheckResult( + endpoint_name=ep.name, host=ep.host, scheme=ep.scheme, port=ep.port, + success=False, status_code=None, expected_status=ep.expected_status, + response_time_ms=elapsed_ms, resolved_ip=resolved_ip, + error=str(e), timestamp_ns=timestamp_ns, + ) + + +def check_all(endpoints, default_timeout: float, timestamp_ns: int) -> List[CheckResult]: + results = [] + for ep in endpoints: + result = check_endpoint(ep, default_timeout, timestamp_ns) + level = logging.INFO if result.success else logging.WARNING + logger.log( + level, + "endpoint=%s success=%s status=%s expected=%s ip=%s time_ms=%s error=%s", + result.endpoint_name, result.success, result.status_code, + result.expected_status, result.resolved_ip, + None if result.response_time_ms is None else round(result.response_time_ms, 1), + result.error, + ) + results.append(result) + return results diff --git a/app/config.py b/app/config.py new file mode 100644 index 0000000..03653f6 --- /dev/null +++ b/app/config.py @@ -0,0 +1,121 @@ +from __future__ import annotations + +import dataclasses +from typing import List, Optional + +import yaml + + +class ConfigError(Exception): + """Raised for any problem with the config file's contents.""" + + +@dataclasses.dataclass +class Endpoint: + name: str + host: str + scheme: str # "http" or "https" + port: int + dns_server: str # DNS server used to resolve `host` — the system/router + # resolver is never consulted, see app/dns_resolver.py + expected_status: int + path: str = "/" + timeout_seconds: Optional[float] = None + verify_tls: bool = True # set False for endpoints with self-signed certs + + +@dataclasses.dataclass +class InfluxConfig: + url: str + token: str + org: str + bucket: str + verify_ssl: bool = False # default off: influx uses a self-signed cert + + +@dataclasses.dataclass +class CheckConfig: + interval_seconds: int = 600 + timeout_seconds: float = 10.0 + immediate_recheck_delay_seconds: float = 5.0 + + +@dataclasses.dataclass +class AppConfig: + influx: InfluxConfig + check: CheckConfig + endpoints: List[Endpoint] + + +def _require(d: dict, key: str, ctx: str): + value = d.get(key) + if value in (None, ""): + raise ConfigError(f"Missing required field '{key}' in {ctx}") + return value + + +def load_config(path: str) -> AppConfig: + try: + with open(path, "r", encoding="utf-8") as f: + raw = yaml.safe_load(f) + except FileNotFoundError: + raise ConfigError(f"Config file not found: {path}") + except yaml.YAMLError as e: + raise ConfigError(f"Invalid YAML in config file '{path}': {e}") + + if not raw: + raise ConfigError(f"Config file '{path}' is empty") + + influx_raw = raw.get("influx") or {} + influx = InfluxConfig( + url=_require(influx_raw, "url", "influx"), + token=_require(influx_raw, "token", "influx"), + org=_require(influx_raw, "org", "influx"), + bucket=_require(influx_raw, "bucket", "influx"), + verify_ssl=bool(influx_raw.get("verify_ssl", False)), + ) + + check_raw = raw.get("check") or {} + check = CheckConfig( + interval_seconds=int(check_raw.get("interval_seconds", 600)), + timeout_seconds=float(check_raw.get("timeout_seconds", 10.0)), + immediate_recheck_delay_seconds=float( + check_raw.get("immediate_recheck_delay_seconds", 5.0) + ), + ) + + endpoints_raw = raw.get("endpoints") or [] + if not endpoints_raw: + raise ConfigError("No endpoints configured under 'endpoints'") + + endpoints: List[Endpoint] = [] + seen_names = set() + for i, ep in enumerate(endpoints_raw): + ctx = f"endpoints[{i}]" + name = _require(ep, "name", ctx) + if name in seen_names: + raise ConfigError(f"Duplicate endpoint name: '{name}'") + seen_names.add(name) + + scheme = str(_require(ep, "scheme", ctx)).lower() + if scheme not in ("http", "https"): + raise ConfigError( + f"{ctx} ('{name}'): scheme must be 'http' or 'https', got '{scheme}'" + ) + + timeout = ep.get("timeout_seconds") + endpoints.append( + Endpoint( + name=name, + host=_require(ep, "host", ctx), + scheme=scheme, + port=int(_require(ep, "port", ctx)), + dns_server=_require(ep, "dns_server", ctx), + expected_status=int(_require(ep, "expected_status", ctx)), + path=ep.get("path", "/"), + timeout_seconds=float(timeout) if timeout else None, + verify_tls=bool(ep.get("verify_tls", True)), + ) + ) + + return AppConfig(influx=influx, check=check, endpoints=endpoints) diff --git a/app/dns_resolver.py b/app/dns_resolver.py new file mode 100644 index 0000000..de6b338 --- /dev/null +++ b/app/dns_resolver.py @@ -0,0 +1,60 @@ +"""DNS resolution that always uses an explicitly configured server. + +The system/router-provided resolver (as configured via /etc/resolv.conf, +usually pointing at your router) is never consulted. Each lookup opens a +fresh, unconfigured dnspython resolver and points it only at the DNS +server given for that specific endpoint. +""" + +from __future__ import annotations + +import contextlib +import socket +import threading + +import dns.resolver + +# Guards socket.getaddrinfo patching below. Checks must run sequentially +# while a patch is active since getaddrinfo is a process-global function; +# the monitor's main loop does exactly that (one endpoint at a time). +_patch_lock = threading.Lock() + + +def resolve(host: str, dns_server: str, timeout: float) -> str: + """Resolve `host` to an IPv4 address using *only* `dns_server`. + + Raises dns.exception.DNSException (or a subclass) on failure — + NXDOMAIN, timeout, no answer, etc. + """ + resolver = dns.resolver.Resolver(configure=False) + resolver.nameservers = [dns_server] + resolver.lifetime = timeout + resolver.timeout = timeout + answer = resolver.resolve(host, "A") + return str(answer[0]) + + +@contextlib.contextmanager +def pin_resolution(hostname: str, ip: str): + """Force socket.getaddrinfo(hostname, ...) to return `ip` for the + duration of the block. + + This lets `requests`/urllib3 connect to the IP we already resolved via + the explicit DNS server above, without doing any resolution of its + own (which would otherwise go through the system resolver). The + hostname in the URL is left untouched, so the Host header and TLS + SNI/certificate verification behave exactly as normal. + """ + original_getaddrinfo = socket.getaddrinfo + + def patched(host, *args, **kwargs): + if host == hostname: + host = ip + return original_getaddrinfo(host, *args, **kwargs) + + with _patch_lock: + socket.getaddrinfo = patched + try: + yield + finally: + socket.getaddrinfo = original_getaddrinfo diff --git a/app/influx_writer.py b/app/influx_writer.py new file mode 100644 index 0000000..f1fde52 --- /dev/null +++ b/app/influx_writer.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +import logging +from typing import List + +import requests +import urllib3 + +from .checker import CheckResult + +logger = logging.getLogger("uptime_monitor.influx") + +# The InfluxDB endpoint uses a self-signed certificate by design (per +# config, verify_ssl defaults to False for it) — silence the resulting +# urllib3 warning so it doesn't spam the logs on every write. +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + + +def _escape_tag(value) -> str: + return ( + str(value) + .replace("\\", "\\\\") + .replace(",", "\\,") + .replace(" ", "\\ ") + .replace("=", "\\=") + ) + + +def _escape_field_string(value) -> str: + return str(value).replace("\\", "\\\\").replace('"', '\\"') + + +def result_to_line(result: CheckResult, measurement: str = "uptime_check") -> str: + tags = ( + f"endpoint={_escape_tag(result.endpoint_name)}," + f"host={_escape_tag(result.host)}," + f"scheme={_escape_tag(result.scheme)}," + f"port={result.port}" + ) + fields = [ + f"success={1 if result.success else 0}i", + f"expected_status={result.expected_status}i", + ] + if result.status_code is not None: + fields.append(f"status_code={result.status_code}i") + if result.response_time_ms is not None: + fields.append(f"response_time_ms={result.response_time_ms}") + if result.resolved_ip: + fields.append(f'resolved_ip="{_escape_field_string(result.resolved_ip)}"') + if result.error: + fields.append(f'error="{_escape_field_string(result.error)}"') + + return f"{measurement},{tags} {','.join(fields)} {result.timestamp_ns}" + + +def write_results(influx_cfg, results: List[CheckResult]) -> None: + if not results: + return + + lines = "\n".join(result_to_line(r) for r in results) + url = f"{influx_cfg.url.rstrip('/')}/api/v2/write" + params = {"org": influx_cfg.org, "bucket": influx_cfg.bucket, "precision": "ns"} + headers = { + "Authorization": f"Token {influx_cfg.token}", + "Content-Type": "text/plain; charset=utf-8", + } + + try: + resp = requests.post( + url, + params=params, + headers=headers, + data=lines.encode("utf-8"), + timeout=10, + verify=influx_cfg.verify_ssl, + ) + resp.raise_for_status() + logger.info("wrote %d point(s) to influx bucket=%s", len(results), influx_cfg.bucket) + except requests.RequestException as e: + logger.error("failed to write to influx: %s", e) diff --git a/app/main.py b/app/main.py new file mode 100644 index 0000000..262b06e --- /dev/null +++ b/app/main.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +import logging +import os +import sys +import time + +from app.checker import check_all +from app.config import ConfigError, load_config +from app.influx_writer import write_results + +logger = logging.getLogger("uptime_monitor") + + +def setup_logging() -> None: + logging.basicConfig( + level=os.environ.get("LOG_LEVEL", "INFO"), + format="%(asctime)s %(levelname)s %(name)s: %(message)s", + stream=sys.stdout, + ) + + +def run_cycle(cfg) -> list: + timestamp_ns = time.time_ns() + results = check_all(cfg.endpoints, cfg.check.timeout_seconds, timestamp_ns) + write_results(cfg.influx, results) + return results + + +def main() -> None: + setup_logging() + config_path = os.environ.get("CONFIG_PATH", "/config/config.yaml") + + try: + cfg = load_config(config_path) + except ConfigError as e: + logger.error("configuration error: %s", e) + sys.exit(1) + + logger.info( + "loaded %d endpoint(s) from %s; interval=%ss, immediate recheck on failure enabled", + len(cfg.endpoints), config_path, cfg.check.interval_seconds, + ) + + while True: + results = run_cycle(cfg) + + if any(not r.success for r in results): + logger.warning( + "at least one endpoint failed this cycle; " + "re-checking all endpoints immediately in %.0fs", + cfg.check.immediate_recheck_delay_seconds, + ) + time.sleep(cfg.check.immediate_recheck_delay_seconds) + run_cycle(cfg) + + time.sleep(cfg.check.interval_seconds) + + +if __name__ == "__main__": + main() diff --git a/config.example.yaml b/config.example.yaml new file mode 100644 index 0000000..72f8b9d --- /dev/null +++ b/config.example.yaml @@ -0,0 +1,61 @@ +# Copy this file to config.yaml and adjust it to your setup. +# +# This file is bind-mounted into the container (see docker-compose.yml), +# NOT baked into the image. You can edit it freely at any time — just +# restart the container afterwards (`docker compose restart`) to pick up +# the changes. No rebuild is ever needed for changes in this file. + +influx: + # Base URL of your InfluxDB 2.x instance. Reachable directly (host/IP + + # port), no shared docker network required. + url: "https://influxdb.example.local:8086" + + # InfluxDB 2.x API token with write access to the bucket below. + token: "REPLACE_WITH_YOUR_INFLUX_API_TOKEN" + + # InfluxDB organization name (or ID) that owns the bucket. + org: "REPLACE_WITH_YOUR_ORG" + + bucket: "uptime" + + # InfluxDB uses a self-signed certificate -> skip certificate + # verification for it specifically. This does NOT affect certificate + # verification of the monitored endpoints below (see verify_tls there). + verify_ssl: false + +check: + # How often to run the full rolling check of all endpoints, in seconds. + # 600 = 10 minutes. + interval_seconds: 600 + + # Default per-request timeout (DNS lookup + HTTP request), in seconds. + # Can be overridden per endpoint. + timeout_seconds: 10 + + # If any endpoint fails during a cycle, all endpoints are re-checked + # once, immediately, after this short delay (in seconds) — instead of + # waiting for the next scheduled cycle. Regular scheduling then resumes. + immediate_recheck_delay_seconds: 5 + +endpoints: + - name: "example-https" + host: "example.com" + scheme: "https" + port: 443 + # DNS server used to resolve `host` for THIS endpoint. The + # system/router-provided DNS resolver is never used — every lookup + # goes explicitly to this server only. + dns_server: "1.1.1.1" + expected_status: 200 + # path: "/" # optional, defaults to "/" + # timeout_seconds: 5 # optional, overrides check.timeout_seconds + # verify_tls: true # optional, set false for self-signed endpoint certs + + - name: "example-internal-http" + host: "internal.example.local" + scheme: "http" + port: 8080 + dns_server: "9.9.9.9" + expected_status: 200 + + # Add as many endpoints as you like below. diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..d4554cd --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,11 @@ +services: + uptime-monitor: + build: . + image: uptime-monitor:latest + container_name: uptime-monitor + restart: unless-stopped + volumes: + - ./config.yaml:/config/config.yaml:ro + # No network wiring to InfluxDB is needed here — config.yaml's + # influx.url just needs to be reachable (hostname/IP + port) from + # this container, e.g. because it's exposed on the host/LAN already. diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..bcb9e28 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +requests>=2.31,<3 +dnspython>=2.6,<3 +PyYAML>=6.0,<7