Eduardo David Paredes Vara
721 lines
26 KiB
Python
Executable File
721 lines
26 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import base64
|
|
import datetime as dt
|
|
import http.client
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import re
|
|
import ssl
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.parse
|
|
from collections import Counter
|
|
|
|
|
|
DEFAULT_ENDPOINTS = [
|
|
"https://traefik.sherlockhomeless.net",
|
|
"https://auth.sherlockhomeless.net",
|
|
"https://n8n.sherlockhomeless.net/healthz",
|
|
"https://nextcloud.sherlockhomeless.net",
|
|
"https://paperless.sherlockhomeless.net",
|
|
"https://onlyoffice.sherlockhomeless.net",
|
|
"https://memos.sherlockhomeless.net",
|
|
"https://vikunja.sherlockhomeless.net",
|
|
"https://karakeep.sherlockhomeless.net",
|
|
"https://rss.sherlockhomeless.net",
|
|
"https://ots.sherlockhomeless.net",
|
|
"https://dozzle.sherlockhomeless.net",
|
|
"https://beszel.sherlockhomeless.net",
|
|
]
|
|
|
|
DEFAULT_ACCEPTED_STATUS_CODES = {
|
|
"https://traefik.sherlockhomeless.net": {200, 301, 302, 303, 307, 308, 401, 403},
|
|
"https://rss.sherlockhomeless.net": {200, 301, 302, 303, 307, 308, 401, 403},
|
|
"https://ots.sherlockhomeless.net": {200, 301, 302, 303, 307, 308, 401, 403},
|
|
"https://dozzle.sherlockhomeless.net": {200, 301, 302, 303, 307, 308, 401, 403},
|
|
}
|
|
|
|
DEFAULT_TARGET_CONTAINERS = [
|
|
"traefik",
|
|
"n8n",
|
|
"nextcloud",
|
|
"paperless",
|
|
"mail-relay",
|
|
"openclaw-openclaw-gateway-1",
|
|
]
|
|
|
|
ERROR_PATTERN = re.compile(
|
|
r"error|exception|fatal|panic|critical|denied|timeout|refused|unhealthy|failed|oauth|401",
|
|
re.IGNORECASE,
|
|
)
|
|
HOST_RULE_RE = re.compile(r"Host\(([^)]*)\)")
|
|
QUOTED_RE = re.compile(r"`([^`]+)`|'([^']+)'|\"([^\"]+)\"")
|
|
SABLIER_RE = re.compile(r"sablier-([A-Za-z0-9_.-]+)@file")
|
|
|
|
|
|
def env_int(name: str, default: int, minimum: int = 0) -> int:
|
|
raw = os.environ.get(name)
|
|
try:
|
|
value = int(raw) if raw is not None else default
|
|
except ValueError:
|
|
return default
|
|
return value if value >= minimum else default
|
|
|
|
|
|
def env_bool(name: str, default: bool) -> bool:
|
|
raw = os.environ.get(name)
|
|
if raw is None:
|
|
return default
|
|
return raw.strip().lower() in {"1", "true", "yes", "on"}
|
|
|
|
|
|
def now_utc() -> str:
|
|
return dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
|
|
|
|
def section(title: str) -> None:
|
|
print(f"\n## {title}")
|
|
|
|
|
|
def run(cmd, *, timeout=30, check=False, input_text=None):
|
|
try:
|
|
return subprocess.run(
|
|
cmd,
|
|
text=True,
|
|
input=input_text,
|
|
capture_output=True,
|
|
timeout=timeout,
|
|
check=check,
|
|
)
|
|
except Exception as exc:
|
|
completed = subprocess.CompletedProcess(cmd, 1, "", str(exc))
|
|
return completed
|
|
|
|
|
|
def command_exists(name: str) -> bool:
|
|
return run(["sh", "-lc", f"command -v {name} >/dev/null 2>&1"]).returncode == 0
|
|
|
|
|
|
def print_command(cmd, *, timeout=30) -> None:
|
|
proc = run(cmd, timeout=timeout)
|
|
out = proc.stdout.rstrip("\n")
|
|
err = proc.stderr.rstrip("\n")
|
|
if out:
|
|
print(out)
|
|
if err:
|
|
print(err)
|
|
|
|
|
|
def parse_list_env(raw: str | None, default: list[str]) -> list[str]:
|
|
if not raw:
|
|
return default
|
|
items = []
|
|
for part in re.split(r"[,;\n]", raw):
|
|
part = part.strip()
|
|
if part:
|
|
items.append(part)
|
|
return items
|
|
|
|
|
|
def parse_expected_codes(raw: str | None) -> dict[str, set[int]]:
|
|
expected = {endpoint: set(codes) for endpoint, codes in DEFAULT_ACCEPTED_STATUS_CODES.items()}
|
|
if not raw:
|
|
return expected
|
|
for item in parse_list_env(raw, []):
|
|
if "=" not in item:
|
|
continue
|
|
endpoint, codes_raw = item.split("=", 1)
|
|
codes = set()
|
|
for code_raw in re.split(r"[|: ]+", codes_raw.strip()):
|
|
if not code_raw:
|
|
continue
|
|
try:
|
|
codes.add(int(code_raw))
|
|
except ValueError:
|
|
continue
|
|
if endpoint.strip() and codes:
|
|
expected[endpoint.strip()] = codes
|
|
return expected
|
|
|
|
|
|
def resolve_docker_since(since: str) -> str:
|
|
if re.match(r"^\d+[smhd]$", since):
|
|
return since
|
|
proc = run(["date", "-d", since, "--iso-8601=seconds"])
|
|
if proc.returncode == 0 and proc.stdout.strip():
|
|
return proc.stdout.strip()
|
|
return "24h"
|
|
|
|
|
|
def docker_names(all_containers=False) -> list[str]:
|
|
cmd = ["docker", "ps", "--format", "{{.Names}}"]
|
|
if all_containers:
|
|
cmd.insert(2, "-a")
|
|
proc = run(cmd)
|
|
if proc.returncode != 0:
|
|
return []
|
|
return [line.strip() for line in proc.stdout.splitlines() if line.strip()]
|
|
|
|
|
|
def docker_inspect_labels(container: str) -> dict:
|
|
proc = run(["docker", "inspect", "--format", "{{json .Config.Labels}}", container])
|
|
if proc.returncode != 0 or not proc.stdout.strip():
|
|
return {}
|
|
try:
|
|
return json.loads(proc.stdout)
|
|
except json.JSONDecodeError:
|
|
return {}
|
|
|
|
|
|
def add_host_rule(rule: str, domains: set[str]) -> None:
|
|
if not isinstance(rule, str):
|
|
return
|
|
for host_match in HOST_RULE_RE.finditer(rule):
|
|
body = host_match.group(1)
|
|
for quoted_match in QUOTED_RE.finditer(body):
|
|
domain = next((item for item in quoted_match.groups() if item), "")
|
|
domain = domain.strip().lower()
|
|
if domain and "*" not in domain and "{" not in domain and "}" not in domain:
|
|
domains.add(domain)
|
|
|
|
|
|
def dynamic_files():
|
|
for base in (pathlib.Path("/opt/traefik/dynamic"), pathlib.Path("/etc/traefik/dynamic")):
|
|
if not base.exists():
|
|
continue
|
|
for path in base.rglob("*"):
|
|
if path.is_file() and path.suffix.lower() in {".yml", ".yaml", ".toml"}:
|
|
yield path
|
|
|
|
|
|
def active_tls_domains() -> set[str]:
|
|
domains: set[str] = set()
|
|
for container in docker_names():
|
|
labels = docker_inspect_labels(container)
|
|
for key, value in labels.items():
|
|
if key.startswith("traefik.http.routers.") and key.endswith(".rule"):
|
|
add_host_rule(value, domains)
|
|
for path in dynamic_files():
|
|
try:
|
|
add_host_rule(path.read_text(encoding="utf-8", errors="ignore"), domains)
|
|
except Exception:
|
|
continue
|
|
return domains
|
|
|
|
|
|
def sablier_managed_containers() -> set[str]:
|
|
names = set()
|
|
for path in dynamic_files():
|
|
try:
|
|
content = path.read_text(encoding="utf-8", errors="ignore")
|
|
except Exception:
|
|
continue
|
|
for match in SABLIER_RE.finditer(content):
|
|
names.add(match.group(1))
|
|
return names
|
|
|
|
|
|
def health_request(url: str, max_time: int, follow_redirects: bool, max_redirects=8) -> dict:
|
|
current_url = url
|
|
redirects = 0
|
|
start = time.monotonic()
|
|
last_code = 0
|
|
context = ssl.create_default_context()
|
|
|
|
while True:
|
|
parsed = urllib.parse.urlparse(current_url)
|
|
scheme = parsed.scheme or "https"
|
|
host = parsed.hostname
|
|
if not host:
|
|
raise ValueError(f"invalid URL: {current_url}")
|
|
port = parsed.port or (443 if scheme == "https" else 80)
|
|
path = urllib.parse.urlunparse(("", "", parsed.path or "/", parsed.params, parsed.query, ""))
|
|
conn_cls = http.client.HTTPSConnection if scheme == "https" else http.client.HTTPConnection
|
|
kwargs = {"timeout": max_time}
|
|
if scheme == "https":
|
|
kwargs["context"] = context
|
|
conn = conn_cls(host, port, **kwargs)
|
|
try:
|
|
conn.request("GET", path, headers={"User-Agent": "ops-report/1.0"})
|
|
response = conn.getresponse()
|
|
last_code = response.status
|
|
response.read()
|
|
if follow_redirects and last_code in {301, 302, 303, 307, 308} and redirects < max_redirects:
|
|
location = response.getheader("Location")
|
|
if location:
|
|
current_url = urllib.parse.urljoin(current_url, location)
|
|
redirects += 1
|
|
continue
|
|
elapsed = time.monotonic() - start
|
|
return {
|
|
"http_code": last_code,
|
|
"curl_time_total_seconds": f"{elapsed:.6f}",
|
|
"effective_url": current_url,
|
|
}
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def cmd_health(_args) -> int:
|
|
max_time = env_int("HEALTH_MAX_TIME", 12, 1)
|
|
retries = env_int("HEALTH_RETRIES", 2, 1)
|
|
retry_delay = env_int("HEALTH_RETRY_DELAY", 8, 0)
|
|
follow_redirects = env_bool("HEALTH_FOLLOW_REDIRECTS", True)
|
|
endpoints = parse_list_env(os.environ.get("HEALTH_ENDPOINTS"), DEFAULT_ENDPOINTS)
|
|
expected_codes = parse_expected_codes(os.environ.get("HEALTH_EXPECTED_CODES"))
|
|
|
|
results = []
|
|
for endpoint in endpoints:
|
|
start_ms = int(time.time() * 1000)
|
|
status = "down"
|
|
http_code = 0
|
|
effective_url = ""
|
|
time_total = ""
|
|
error = ""
|
|
attempts = 0
|
|
for attempt in range(1, retries + 1):
|
|
attempts = attempt
|
|
try:
|
|
response = health_request(endpoint, max_time, follow_redirects)
|
|
http_code = int(response["http_code"])
|
|
effective_url = response["effective_url"]
|
|
time_total = response["curl_time_total_seconds"]
|
|
accepted = expected_codes.get(endpoint)
|
|
if 200 <= http_code < 400 or (accepted is not None and http_code in accepted):
|
|
status = "up"
|
|
error = ""
|
|
break
|
|
error = f"HTTP {http_code}"
|
|
except Exception as exc:
|
|
http_code = 0
|
|
effective_url = ""
|
|
time_total = ""
|
|
error = str(exc)
|
|
if attempt < retries:
|
|
time.sleep(retry_delay)
|
|
latency_ms = int(time.time() * 1000) - start_ms
|
|
item = {
|
|
"endpoint": endpoint,
|
|
"status": status,
|
|
"http_code": http_code,
|
|
"latency_ms": latency_ms,
|
|
"attempts": attempts,
|
|
}
|
|
if time_total:
|
|
item["curl_time_total_seconds"] = time_total
|
|
if effective_url:
|
|
item["effective_url"] = effective_url
|
|
if status == "down":
|
|
item["error"] = error
|
|
elif http_code >= 400:
|
|
item["accepted_http_code"] = True
|
|
item["status_note"] = "reachable_but_access_restricted"
|
|
results.append(item)
|
|
|
|
up_count = sum(1 for item in results if item["status"] == "up")
|
|
down_count = sum(1 for item in results if item["status"] == "down")
|
|
endpoints_count = up_count + down_count
|
|
overall = "up"
|
|
if endpoints_count == 0:
|
|
overall = "down"
|
|
elif down_count:
|
|
overall = "degraded"
|
|
print(json.dumps({
|
|
"generated_at_utc": now_utc(),
|
|
"overall_status": overall,
|
|
"endpoints_count": endpoints_count,
|
|
"up_count": up_count,
|
|
"down_count": down_count,
|
|
"max_time_seconds": max_time,
|
|
"retries": retries,
|
|
"retry_delay_seconds": retry_delay,
|
|
"results": results,
|
|
}, indent=2))
|
|
return 0
|
|
|
|
|
|
def proc_text(path: pathlib.Path) -> str:
|
|
try:
|
|
return path.read_text(errors="ignore")
|
|
except Exception:
|
|
return ""
|
|
|
|
|
|
def top_swap_processes(limit: int) -> list[tuple[int, str, str]]:
|
|
rows = []
|
|
if limit <= 0:
|
|
return rows
|
|
for status_file in pathlib.Path("/proc").glob("[0-9]*/status"):
|
|
data = proc_text(status_file)
|
|
if not data:
|
|
continue
|
|
name = ""
|
|
pid = status_file.parent.name
|
|
swap = 0
|
|
for line in data.splitlines():
|
|
if line.startswith("Name:"):
|
|
name = line.split(None, 1)[1]
|
|
elif line.startswith("Pid:"):
|
|
pid = line.split(None, 1)[1]
|
|
elif line.startswith("VmSwap:"):
|
|
parts = line.split()
|
|
if len(parts) >= 2:
|
|
try:
|
|
swap = int(parts[1])
|
|
except ValueError:
|
|
swap = 0
|
|
if swap > 0:
|
|
rows.append((swap, name, pid))
|
|
rows.sort(reverse=True)
|
|
return rows[:limit]
|
|
|
|
|
|
def cert_end_date_from_file(path: pathlib.Path) -> str:
|
|
proc = run(["openssl", "x509", "-enddate", "-noout", "-in", str(path)])
|
|
if proc.returncode != 0:
|
|
return ""
|
|
return proc.stdout.strip().split("=", 1)[-1]
|
|
|
|
|
|
def cert_end_date_from_pem(pem: str) -> str:
|
|
proc = run(["openssl", "x509", "-enddate", "-noout"], input_text=pem)
|
|
if proc.returncode != 0:
|
|
return ""
|
|
return proc.stdout.strip().split("=", 1)[-1]
|
|
|
|
|
|
def parse_cert_epoch(end_raw: str) -> int | None:
|
|
if not end_raw:
|
|
return None
|
|
proc = run(["date", "-d", end_raw, "+%s"])
|
|
if proc.returncode != 0:
|
|
return None
|
|
try:
|
|
return int(proc.stdout.strip())
|
|
except ValueError:
|
|
return None
|
|
|
|
|
|
def print_cert_line(domain: str, end_raw: str, warn_days: int) -> None:
|
|
if not end_raw:
|
|
print(f"{domain}: unable to parse cert")
|
|
return
|
|
epoch = parse_cert_epoch(end_raw)
|
|
if epoch is None:
|
|
print(f"{domain}: enddate={end_raw}")
|
|
return
|
|
days_left = int((epoch - time.time()) / 86400)
|
|
warn = " [WARN]" if days_left <= warn_days else ""
|
|
print(f"{domain}: expires={end_raw} days_left={days_left}{warn}")
|
|
|
|
|
|
def acme_entries(paths: list[str]) -> list[tuple[str, str]]:
|
|
entries = []
|
|
for raw_path in paths:
|
|
path = pathlib.Path(raw_path)
|
|
if not path.exists():
|
|
continue
|
|
try:
|
|
data = json.loads(path.read_text())
|
|
except Exception:
|
|
continue
|
|
for payload in data.values():
|
|
if not isinstance(payload, dict):
|
|
continue
|
|
for cert in payload.get("Certificates", []):
|
|
domain = (cert.get("domain") or {}).get("main")
|
|
encoded = cert.get("certificate")
|
|
if domain and encoded:
|
|
entries.append((domain, encoded))
|
|
return entries
|
|
|
|
|
|
def cmd_summary(_args) -> int:
|
|
since_hours = env_int("SINCE_HOURS", 24, 1)
|
|
top_error_lines = env_int("TOP_ERROR_LINES", 20, 0)
|
|
cert_warn_days = env_int("CERT_WARN_DAYS", 30, 0)
|
|
top_swap = env_int("TOP_SWAP_PROCESSES", 10, 0)
|
|
show_overlays = env_bool("SHOW_DF_OVERLAYS", False)
|
|
show_running = env_bool("SHOW_RUNNING_CONTAINERS", False)
|
|
active_only = env_bool("ACTIVE_CERT_DOMAINS_ONLY", True)
|
|
exclude_domains = {d.lower() for d in parse_list_env(
|
|
os.environ.get("CERT_EXCLUDE_DOMAINS", "portainer.thehomelesssherlock.com uptimekuma.sherlockhomeless.net notas.sherlockhomeless.net").replace(" ", "\n"),
|
|
[],
|
|
)}
|
|
|
|
print("# Server Daily Summary")
|
|
print(f"generated_at_utc: {now_utc()}")
|
|
hostname = run(["hostname", "-f"]).stdout.strip() or run(["hostname"]).stdout.strip()
|
|
print(f"host: {hostname}")
|
|
print(f"kernel: {run(['uname', '-r']).stdout.strip()}")
|
|
|
|
section("Uptime & Load")
|
|
uptime = run(["uptime", "-p"]).stdout.strip() or run(["uptime"]).stdout.strip()
|
|
print(f"uptime: {uptime}")
|
|
loadavg = pathlib.Path("/proc/loadavg").read_text().split()[:3]
|
|
print(f"loadavg: {' '.join(loadavg)}")
|
|
|
|
section("Memory")
|
|
print_command(["free", "-h"])
|
|
if top_swap > 0:
|
|
print()
|
|
print("top_swap_processes:")
|
|
print("SWAP_KIB\tCOMMAND\tPID")
|
|
for swap, name, pid in top_swap_processes(top_swap):
|
|
print(f"{swap}\t{name}\t{pid}")
|
|
|
|
section("Disk")
|
|
df_cmd = ["df", "-hT", "-x", "tmpfs", "-x", "devtmpfs"]
|
|
if not show_overlays:
|
|
df_cmd += ["-x", "overlay"]
|
|
print_command(df_cmd)
|
|
if not show_overlays:
|
|
print("overlay_mounts: omitted set SHOW_DF_OVERLAYS=1 to include")
|
|
|
|
section("Docker Overview")
|
|
if not command_exists("docker"):
|
|
print("docker command not available")
|
|
else:
|
|
sablier = sablier_managed_containers()
|
|
running = docker_names()
|
|
unhealthy = run(["docker", "ps", "--filter", "health=unhealthy", "-q"]).stdout.splitlines()
|
|
exited = run(["docker", "ps", "-a", "--filter", "status=exited", "-q"]).stdout.splitlines()
|
|
restarting = run(["docker", "ps", "-a", "--filter", "status=restarting", "-q"]).stdout.splitlines()
|
|
print(f"containers_running: {len(running)}")
|
|
print(f"containers_unhealthy: {len(unhealthy)}")
|
|
print(f"containers_exited: {len(exited)}")
|
|
print(f"containers_restarting: {len(restarting)}")
|
|
print()
|
|
if show_running:
|
|
print("running_containers:")
|
|
print_command(["docker", "ps", "--format", "table {{.Names}}\t{{.Status}}\t{{.Networks}}"])
|
|
else:
|
|
print(f"running_containers: omitted count={len(running)} set SHOW_RUNNING_CONTAINERS=1 to include")
|
|
if unhealthy:
|
|
print()
|
|
print("unhealthy_containers:")
|
|
print_command(["docker", "ps", "--filter", "health=unhealthy", "--format", "table {{.Names}}\t{{.Status}}\t{{.Image}}"])
|
|
if exited or restarting:
|
|
print()
|
|
print("non_running_containers:")
|
|
non_running = run(["docker", "ps", "-a", "--filter", "status=exited", "--filter", "status=restarting", "--format", "{{.Names}}\t{{.Status}}\t{{.Image}}"]).stdout.splitlines()
|
|
print("NAMES\tSTATUS\tIMAGE")
|
|
parsed = []
|
|
for line in non_running:
|
|
parts = line.split("\t")
|
|
if len(parts) == 3:
|
|
parsed.append(tuple(parts))
|
|
print(line)
|
|
if sablier:
|
|
print()
|
|
print("non_running_sablier_managed_containers:")
|
|
print("NAMES\tSTATUS\tIMAGE")
|
|
for name, status, image in parsed:
|
|
if name in sablier:
|
|
print(f"{name}\t{status}\t{image}")
|
|
print()
|
|
print("non_running_other_containers:")
|
|
print("NAMES\tSTATUS\tIMAGE")
|
|
for name, status, image in parsed:
|
|
if name not in sablier:
|
|
print(f"{name}\t{status}\t{image}")
|
|
|
|
section("TLS Certificates")
|
|
if not command_exists("openssl"):
|
|
print("openssl not available")
|
|
else:
|
|
found = False
|
|
seen = set()
|
|
active = active_tls_domains() if active_only else set()
|
|
if active_only:
|
|
if active:
|
|
print(f"tls_scope: active_domains_only active_domains={len(active)} excluded_domains={len(exclude_domains)}")
|
|
else:
|
|
print(f"tls_scope: active_domains_only requested but no active domains detected; fallback=all_discovered excluded_domains={len(exclude_domains)}")
|
|
live = pathlib.Path("/etc/letsencrypt/live")
|
|
if live.exists():
|
|
for cert in sorted(live.glob("*/cert.pem")):
|
|
domain = cert.parent.name
|
|
domain_lc = domain.lower()
|
|
if domain_lc in exclude_domains:
|
|
continue
|
|
if active_only and active and domain_lc not in active:
|
|
continue
|
|
if domain in seen:
|
|
continue
|
|
seen.add(domain)
|
|
found = True
|
|
print_cert_line(domain, cert_end_date_from_file(cert), cert_warn_days)
|
|
for domain, encoded in acme_entries(["/opt/traefik/letsencrypt/acme.json", "/opt/traefik/acme/acme.json"]):
|
|
domain_lc = domain.lower()
|
|
if domain_lc in exclude_domains:
|
|
continue
|
|
if active_only and active and domain_lc not in active:
|
|
continue
|
|
if domain in seen:
|
|
continue
|
|
seen.add(domain)
|
|
found = True
|
|
try:
|
|
pem = base64.b64decode(encoded).decode()
|
|
except Exception:
|
|
print(f"{domain}: unable to decode traefik cert")
|
|
continue
|
|
print_cert_line(domain, cert_end_date_from_pem(pem), cert_warn_days)
|
|
if not found:
|
|
print("no local TLS certificates found")
|
|
|
|
section("SSH Protection")
|
|
if not command_exists("fail2ban-client"):
|
|
print("fail2ban-client not available")
|
|
else:
|
|
proc = run(["fail2ban-client", "status", "sshd"])
|
|
text = proc.stdout
|
|
if text:
|
|
def extract(label):
|
|
m = re.search(rf"{re.escape(label)}:\s*(\d+)", text)
|
|
return m.group(1) if m else "0"
|
|
print(f"fail2ban_sshd: active total_failed={extract('Total failed')} current_banned={extract('Currently banned')} total_banned={extract('Total banned')}")
|
|
print("note: SSH probes can still generate auth/journal log entries before or while bans are applied; that alone does not mean fail2ban is inactive.")
|
|
else:
|
|
print("fail2ban_sshd: no status available")
|
|
|
|
section("OpenClaw OAuth")
|
|
report_openclaw_oauth()
|
|
|
|
section(f"Top Journal Errors (last {since_hours}h)")
|
|
if command_exists("journalctl") and top_error_lines > 0:
|
|
proc = run(["journalctl", "-p", "err..alert", "--since", f"{since_hours} hours ago", "--no-pager", "-o", "short-iso"], timeout=30)
|
|
for line in proc.stdout.splitlines()[-top_error_lines:]:
|
|
print(line)
|
|
elif not command_exists("journalctl"):
|
|
print("journalctl not available")
|
|
return 0
|
|
|
|
|
|
def report_openclaw_oauth() -> None:
|
|
auth_file = pathlib.Path(os.environ.get("OPENCLAW_AUTH_FILE", "/home/felidae/.openclaw/agents/main/agent/auth-profiles.json"))
|
|
warn_hours = env_int("OPENCLAW_WARN_HOURS", 72, 0)
|
|
critical_hours = env_int("OPENCLAW_CRITICAL_HOURS", 24, 0)
|
|
action = os.environ.get("OPENCLAW_REAUTH_CMD", "codex-reauth")
|
|
if not auth_file.exists():
|
|
print("openclaw_codex_oauth: auth file or jq unavailable")
|
|
return
|
|
try:
|
|
data = json.loads(auth_file.read_text())
|
|
profiles = data.get("profiles", {})
|
|
expiries = [
|
|
int(value.get("expires") or 0)
|
|
for value in profiles.values()
|
|
if isinstance(value, dict) and value.get("provider") == "openai-codex"
|
|
]
|
|
expiry_ms = max(expiries) if expiries else 0
|
|
except Exception:
|
|
expiry_ms = 0
|
|
if expiry_ms <= 0:
|
|
print("openclaw_codex_oauth: no oauth profile expiry found")
|
|
return
|
|
now_ms = int(time.time() * 1000)
|
|
diff_ms = expiry_ms - now_ms
|
|
expiry_iso = dt.datetime.fromtimestamp(expiry_ms / 1000, dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
if diff_ms <= 0:
|
|
print(f'openclaw_codex_oauth: CRITICAL expires_utc={expiry_iso} reason=expired action="{action}"')
|
|
return
|
|
hours_left = diff_ms // 3_600_000
|
|
mins_left = (diff_ms % 3_600_000) // 60_000
|
|
severity = "OK"
|
|
if hours_left <= critical_hours:
|
|
severity = "CRITICAL"
|
|
elif hours_left <= warn_hours:
|
|
severity = "WARN"
|
|
print(f'openclaw_codex_oauth: {severity} expires_utc={expiry_iso} hours_left={hours_left}h{mins_left}m critical_hours={critical_hours} warn_hours={warn_hours} action="{action}"')
|
|
|
|
|
|
def cmd_logs(_args) -> int:
|
|
since = os.environ.get("SINCE", "24 hours ago")
|
|
max_journal = env_int("MAX_JOURNAL_LINES", 120, 0)
|
|
max_docker = env_int("MAX_DOCKER_LINES_PER_SERVICE", 40, 0)
|
|
targets = parse_list_env(os.environ.get("TARGET_CONTAINERS"), DEFAULT_TARGET_CONTAINERS)
|
|
docker_since = resolve_docker_since(since)
|
|
|
|
print("# Log Digest")
|
|
print(f"generated_at_utc: {now_utc()}")
|
|
print(f"window: {since}")
|
|
print(f"docker_since: {docker_since}")
|
|
|
|
section("System Errors (journalctl err..alert)")
|
|
if command_exists("journalctl"):
|
|
proc = run(["journalctl", "--since", since, "-p", "err..alert", "--no-pager", "-o", "short-iso"], timeout=30)
|
|
for line in proc.stdout.splitlines()[-max_journal:]:
|
|
print(line)
|
|
else:
|
|
print("journalctl not available")
|
|
|
|
section("System Warnings (journalctl warning)")
|
|
if command_exists("journalctl"):
|
|
proc = run(["journalctl", "--since", since, "-p", "warning", "--no-pager", "-o", "short-iso"], timeout=30)
|
|
for line in proc.stdout.splitlines()[-max_journal:]:
|
|
print(line)
|
|
else:
|
|
print("journalctl not available")
|
|
|
|
section("Docker Service Error Highlights")
|
|
if command_exists("docker"):
|
|
all_names = set(docker_names(all_containers=True))
|
|
for svc in targets:
|
|
if svc not in all_names:
|
|
continue
|
|
print()
|
|
print(f"### {svc}")
|
|
proc = run(["docker", "logs", "--since", docker_since, svc], timeout=30)
|
|
lines = (proc.stdout + proc.stderr).splitlines()
|
|
for line in [line for line in lines if ERROR_PATTERN.search(line)][-max_docker:]:
|
|
print(line)
|
|
else:
|
|
print("docker command not available")
|
|
|
|
section("Top Error Patterns")
|
|
if command_exists("journalctl"):
|
|
proc = run(["journalctl", "--since", since, "--no-pager", "-o", "cat"], timeout=30)
|
|
counts = Counter()
|
|
for line in proc.stdout.splitlines():
|
|
if re.search(r"error|exception|fatal|panic|critical|denied|timeout|failed", line, re.IGNORECASE):
|
|
normalized = re.sub(r"\s+", " ", line.strip())[:180]
|
|
counts[normalized] += 1
|
|
for line, count in counts.most_common(20):
|
|
print(f"{count:7d} {line}")
|
|
else:
|
|
print("journalctl not available")
|
|
return 0
|
|
|
|
|
|
def cmd_all(_args) -> int:
|
|
print("__N8N_LOG_DIGEST_BEGIN__")
|
|
rc_logs = cmd_logs(_args)
|
|
print("\n__N8N_SERVER_SUMMARY_BEGIN__")
|
|
rc_summary = cmd_summary(_args)
|
|
print("\n__N8N_HEALTH_RESULTS_BEGIN__")
|
|
rc_health = cmd_health(_args)
|
|
print(f"\n__N8N_STATUS__ log={rc_logs} summary={rc_summary} health={rc_health}")
|
|
return 0 if rc_logs == rc_summary == rc_health == 0 else 1
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description="Personal ops report helper")
|
|
sub = parser.add_subparsers(dest="command", required=True)
|
|
sub.add_parser("health").set_defaults(func=cmd_health)
|
|
sub.add_parser("summary").set_defaults(func=cmd_summary)
|
|
sub.add_parser("logs").set_defaults(func=cmd_logs)
|
|
sub.add_parser("all").set_defaults(func=cmd_all)
|
|
args = parser.parse_args()
|
|
return args.func(args)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|