uptime-monitor
This commit is contained in:
@@ -0,0 +1,84 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
import logging
|
||||
import time
|
||||
from typing import List, Optional
|
||||
|
||||
import dns.exception
|
||||
import requests
|
||||
|
||||
from . import dns_resolver
|
||||
|
||||
logger = logging.getLogger("uptime_monitor.checker")
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class CheckResult:
|
||||
endpoint_name: str
|
||||
host: str
|
||||
scheme: str
|
||||
port: int
|
||||
success: bool
|
||||
status_code: Optional[int]
|
||||
expected_status: int
|
||||
response_time_ms: Optional[float]
|
||||
resolved_ip: Optional[str]
|
||||
error: Optional[str]
|
||||
timestamp_ns: int
|
||||
|
||||
|
||||
def check_endpoint(ep, default_timeout: float, timestamp_ns: int) -> CheckResult:
|
||||
timeout = ep.timeout_seconds or default_timeout
|
||||
start = time.monotonic()
|
||||
|
||||
try:
|
||||
resolved_ip = dns_resolver.resolve(ep.host, ep.dns_server, timeout)
|
||||
except dns.exception.DNSException as e:
|
||||
return CheckResult(
|
||||
endpoint_name=ep.name, host=ep.host, scheme=ep.scheme, port=ep.port,
|
||||
success=False, status_code=None, expected_status=ep.expected_status,
|
||||
response_time_ms=None, resolved_ip=None,
|
||||
error=f"dns_error: {e}", timestamp_ns=timestamp_ns,
|
||||
)
|
||||
|
||||
url = f"{ep.scheme}://{ep.host}:{ep.port}{ep.path}"
|
||||
try:
|
||||
with dns_resolver.pin_resolution(ep.host, resolved_ip):
|
||||
resp = requests.get(
|
||||
url, timeout=timeout, verify=ep.verify_tls, allow_redirects=True
|
||||
)
|
||||
elapsed_ms = (time.monotonic() - start) * 1000
|
||||
success = resp.status_code == ep.expected_status
|
||||
return CheckResult(
|
||||
endpoint_name=ep.name, host=ep.host, scheme=ep.scheme, port=ep.port,
|
||||
success=success, status_code=resp.status_code, expected_status=ep.expected_status,
|
||||
response_time_ms=elapsed_ms, resolved_ip=resolved_ip,
|
||||
error=None if success else f"unexpected_status_code:{resp.status_code}",
|
||||
timestamp_ns=timestamp_ns,
|
||||
)
|
||||
except requests.RequestException as e:
|
||||
elapsed_ms = (time.monotonic() - start) * 1000
|
||||
return CheckResult(
|
||||
endpoint_name=ep.name, host=ep.host, scheme=ep.scheme, port=ep.port,
|
||||
success=False, status_code=None, expected_status=ep.expected_status,
|
||||
response_time_ms=elapsed_ms, resolved_ip=resolved_ip,
|
||||
error=str(e), timestamp_ns=timestamp_ns,
|
||||
)
|
||||
|
||||
|
||||
def check_all(endpoints, default_timeout: float, timestamp_ns: int) -> List[CheckResult]:
|
||||
results = []
|
||||
for ep in endpoints:
|
||||
result = check_endpoint(ep, default_timeout, timestamp_ns)
|
||||
level = logging.INFO if result.success else logging.WARNING
|
||||
logger.log(
|
||||
level,
|
||||
"endpoint=%s success=%s status=%s expected=%s ip=%s time_ms=%s error=%s",
|
||||
result.endpoint_name, result.success, result.status_code,
|
||||
result.expected_status, result.resolved_ip,
|
||||
None if result.response_time_ms is None else round(result.response_time_ms, 1),
|
||||
result.error,
|
||||
)
|
||||
results.append(result)
|
||||
return results
|
||||
+121
@@ -0,0 +1,121 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
from typing import List, Optional
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
class ConfigError(Exception):
|
||||
"""Raised for any problem with the config file's contents."""
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class Endpoint:
|
||||
name: str
|
||||
host: str
|
||||
scheme: str # "http" or "https"
|
||||
port: int
|
||||
dns_server: str # DNS server used to resolve `host` — the system/router
|
||||
# resolver is never consulted, see app/dns_resolver.py
|
||||
expected_status: int
|
||||
path: str = "/"
|
||||
timeout_seconds: Optional[float] = None
|
||||
verify_tls: bool = True # set False for endpoints with self-signed certs
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class InfluxConfig:
|
||||
url: str
|
||||
token: str
|
||||
org: str
|
||||
bucket: str
|
||||
verify_ssl: bool = False # default off: influx uses a self-signed cert
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class CheckConfig:
|
||||
interval_seconds: int = 600
|
||||
timeout_seconds: float = 10.0
|
||||
immediate_recheck_delay_seconds: float = 5.0
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class AppConfig:
|
||||
influx: InfluxConfig
|
||||
check: CheckConfig
|
||||
endpoints: List[Endpoint]
|
||||
|
||||
|
||||
def _require(d: dict, key: str, ctx: str):
|
||||
value = d.get(key)
|
||||
if value in (None, ""):
|
||||
raise ConfigError(f"Missing required field '{key}' in {ctx}")
|
||||
return value
|
||||
|
||||
|
||||
def load_config(path: str) -> AppConfig:
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
raw = yaml.safe_load(f)
|
||||
except FileNotFoundError:
|
||||
raise ConfigError(f"Config file not found: {path}")
|
||||
except yaml.YAMLError as e:
|
||||
raise ConfigError(f"Invalid YAML in config file '{path}': {e}")
|
||||
|
||||
if not raw:
|
||||
raise ConfigError(f"Config file '{path}' is empty")
|
||||
|
||||
influx_raw = raw.get("influx") or {}
|
||||
influx = InfluxConfig(
|
||||
url=_require(influx_raw, "url", "influx"),
|
||||
token=_require(influx_raw, "token", "influx"),
|
||||
org=_require(influx_raw, "org", "influx"),
|
||||
bucket=_require(influx_raw, "bucket", "influx"),
|
||||
verify_ssl=bool(influx_raw.get("verify_ssl", False)),
|
||||
)
|
||||
|
||||
check_raw = raw.get("check") or {}
|
||||
check = CheckConfig(
|
||||
interval_seconds=int(check_raw.get("interval_seconds", 600)),
|
||||
timeout_seconds=float(check_raw.get("timeout_seconds", 10.0)),
|
||||
immediate_recheck_delay_seconds=float(
|
||||
check_raw.get("immediate_recheck_delay_seconds", 5.0)
|
||||
),
|
||||
)
|
||||
|
||||
endpoints_raw = raw.get("endpoints") or []
|
||||
if not endpoints_raw:
|
||||
raise ConfigError("No endpoints configured under 'endpoints'")
|
||||
|
||||
endpoints: List[Endpoint] = []
|
||||
seen_names = set()
|
||||
for i, ep in enumerate(endpoints_raw):
|
||||
ctx = f"endpoints[{i}]"
|
||||
name = _require(ep, "name", ctx)
|
||||
if name in seen_names:
|
||||
raise ConfigError(f"Duplicate endpoint name: '{name}'")
|
||||
seen_names.add(name)
|
||||
|
||||
scheme = str(_require(ep, "scheme", ctx)).lower()
|
||||
if scheme not in ("http", "https"):
|
||||
raise ConfigError(
|
||||
f"{ctx} ('{name}'): scheme must be 'http' or 'https', got '{scheme}'"
|
||||
)
|
||||
|
||||
timeout = ep.get("timeout_seconds")
|
||||
endpoints.append(
|
||||
Endpoint(
|
||||
name=name,
|
||||
host=_require(ep, "host", ctx),
|
||||
scheme=scheme,
|
||||
port=int(_require(ep, "port", ctx)),
|
||||
dns_server=_require(ep, "dns_server", ctx),
|
||||
expected_status=int(_require(ep, "expected_status", ctx)),
|
||||
path=ep.get("path", "/"),
|
||||
timeout_seconds=float(timeout) if timeout else None,
|
||||
verify_tls=bool(ep.get("verify_tls", True)),
|
||||
)
|
||||
)
|
||||
|
||||
return AppConfig(influx=influx, check=check, endpoints=endpoints)
|
||||
@@ -0,0 +1,60 @@
|
||||
"""DNS resolution that always uses an explicitly configured server.
|
||||
|
||||
The system/router-provided resolver (as configured via /etc/resolv.conf,
|
||||
usually pointing at your router) is never consulted. Each lookup opens a
|
||||
fresh, unconfigured dnspython resolver and points it only at the DNS
|
||||
server given for that specific endpoint.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import socket
|
||||
import threading
|
||||
|
||||
import dns.resolver
|
||||
|
||||
# Guards socket.getaddrinfo patching below. Checks must run sequentially
|
||||
# while a patch is active since getaddrinfo is a process-global function;
|
||||
# the monitor's main loop does exactly that (one endpoint at a time).
|
||||
_patch_lock = threading.Lock()
|
||||
|
||||
|
||||
def resolve(host: str, dns_server: str, timeout: float) -> str:
|
||||
"""Resolve `host` to an IPv4 address using *only* `dns_server`.
|
||||
|
||||
Raises dns.exception.DNSException (or a subclass) on failure —
|
||||
NXDOMAIN, timeout, no answer, etc.
|
||||
"""
|
||||
resolver = dns.resolver.Resolver(configure=False)
|
||||
resolver.nameservers = [dns_server]
|
||||
resolver.lifetime = timeout
|
||||
resolver.timeout = timeout
|
||||
answer = resolver.resolve(host, "A")
|
||||
return str(answer[0])
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def pin_resolution(hostname: str, ip: str):
|
||||
"""Force socket.getaddrinfo(hostname, ...) to return `ip` for the
|
||||
duration of the block.
|
||||
|
||||
This lets `requests`/urllib3 connect to the IP we already resolved via
|
||||
the explicit DNS server above, without doing any resolution of its
|
||||
own (which would otherwise go through the system resolver). The
|
||||
hostname in the URL is left untouched, so the Host header and TLS
|
||||
SNI/certificate verification behave exactly as normal.
|
||||
"""
|
||||
original_getaddrinfo = socket.getaddrinfo
|
||||
|
||||
def patched(host, *args, **kwargs):
|
||||
if host == hostname:
|
||||
host = ip
|
||||
return original_getaddrinfo(host, *args, **kwargs)
|
||||
|
||||
with _patch_lock:
|
||||
socket.getaddrinfo = patched
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
socket.getaddrinfo = original_getaddrinfo
|
||||
@@ -0,0 +1,80 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import List
|
||||
|
||||
import requests
|
||||
import urllib3
|
||||
|
||||
from .checker import CheckResult
|
||||
|
||||
logger = logging.getLogger("uptime_monitor.influx")
|
||||
|
||||
# The InfluxDB endpoint uses a self-signed certificate by design (per
|
||||
# config, verify_ssl defaults to False for it) — silence the resulting
|
||||
# urllib3 warning so it doesn't spam the logs on every write.
|
||||
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
||||
|
||||
|
||||
def _escape_tag(value) -> str:
|
||||
return (
|
||||
str(value)
|
||||
.replace("\\", "\\\\")
|
||||
.replace(",", "\\,")
|
||||
.replace(" ", "\\ ")
|
||||
.replace("=", "\\=")
|
||||
)
|
||||
|
||||
|
||||
def _escape_field_string(value) -> str:
|
||||
return str(value).replace("\\", "\\\\").replace('"', '\\"')
|
||||
|
||||
|
||||
def result_to_line(result: CheckResult, measurement: str = "uptime_check") -> str:
|
||||
tags = (
|
||||
f"endpoint={_escape_tag(result.endpoint_name)},"
|
||||
f"host={_escape_tag(result.host)},"
|
||||
f"scheme={_escape_tag(result.scheme)},"
|
||||
f"port={result.port}"
|
||||
)
|
||||
fields = [
|
||||
f"success={1 if result.success else 0}i",
|
||||
f"expected_status={result.expected_status}i",
|
||||
]
|
||||
if result.status_code is not None:
|
||||
fields.append(f"status_code={result.status_code}i")
|
||||
if result.response_time_ms is not None:
|
||||
fields.append(f"response_time_ms={result.response_time_ms}")
|
||||
if result.resolved_ip:
|
||||
fields.append(f'resolved_ip="{_escape_field_string(result.resolved_ip)}"')
|
||||
if result.error:
|
||||
fields.append(f'error="{_escape_field_string(result.error)}"')
|
||||
|
||||
return f"{measurement},{tags} {','.join(fields)} {result.timestamp_ns}"
|
||||
|
||||
|
||||
def write_results(influx_cfg, results: List[CheckResult]) -> None:
|
||||
if not results:
|
||||
return
|
||||
|
||||
lines = "\n".join(result_to_line(r) for r in results)
|
||||
url = f"{influx_cfg.url.rstrip('/')}/api/v2/write"
|
||||
params = {"org": influx_cfg.org, "bucket": influx_cfg.bucket, "precision": "ns"}
|
||||
headers = {
|
||||
"Authorization": f"Token {influx_cfg.token}",
|
||||
"Content-Type": "text/plain; charset=utf-8",
|
||||
}
|
||||
|
||||
try:
|
||||
resp = requests.post(
|
||||
url,
|
||||
params=params,
|
||||
headers=headers,
|
||||
data=lines.encode("utf-8"),
|
||||
timeout=10,
|
||||
verify=influx_cfg.verify_ssl,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
logger.info("wrote %d point(s) to influx bucket=%s", len(results), influx_cfg.bucket)
|
||||
except requests.RequestException as e:
|
||||
logger.error("failed to write to influx: %s", e)
|
||||
+61
@@ -0,0 +1,61 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
from app.checker import check_all
|
||||
from app.config import ConfigError, load_config
|
||||
from app.influx_writer import write_results
|
||||
|
||||
logger = logging.getLogger("uptime_monitor")
|
||||
|
||||
|
||||
def setup_logging() -> None:
|
||||
logging.basicConfig(
|
||||
level=os.environ.get("LOG_LEVEL", "INFO"),
|
||||
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
|
||||
stream=sys.stdout,
|
||||
)
|
||||
|
||||
|
||||
def run_cycle(cfg) -> list:
|
||||
timestamp_ns = time.time_ns()
|
||||
results = check_all(cfg.endpoints, cfg.check.timeout_seconds, timestamp_ns)
|
||||
write_results(cfg.influx, results)
|
||||
return results
|
||||
|
||||
|
||||
def main() -> None:
|
||||
setup_logging()
|
||||
config_path = os.environ.get("CONFIG_PATH", "/config/config.yaml")
|
||||
|
||||
try:
|
||||
cfg = load_config(config_path)
|
||||
except ConfigError as e:
|
||||
logger.error("configuration error: %s", e)
|
||||
sys.exit(1)
|
||||
|
||||
logger.info(
|
||||
"loaded %d endpoint(s) from %s; interval=%ss, immediate recheck on failure enabled",
|
||||
len(cfg.endpoints), config_path, cfg.check.interval_seconds,
|
||||
)
|
||||
|
||||
while True:
|
||||
results = run_cycle(cfg)
|
||||
|
||||
if any(not r.success for r in results):
|
||||
logger.warning(
|
||||
"at least one endpoint failed this cycle; "
|
||||
"re-checking all endpoints immediately in %.0fs",
|
||||
cfg.check.immediate_recheck_delay_seconds,
|
||||
)
|
||||
time.sleep(cfg.check.immediate_recheck_delay_seconds)
|
||||
run_cycle(cfg)
|
||||
|
||||
time.sleep(cfg.check.interval_seconds)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user