Phase 2: caddy_manager — Caddyfile generation, health monitor, DNS-01 support

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

api/app.py

@@ -41,6 +41,7 @@ from managers import (
     email_manager, calendar_manager, file_manager,
     routing_manager, vault_manager, container_manager,
     cell_link_manager, auth_manager, setup_manager,
+    caddy_manager,
     firewall_manager, EventType,
 )
 # Re-exports: tests do `from app import CellManager` and `from app import _resolve_peer_dns`
@@ -556,6 +557,21 @@ def health_monitor_loop():
             # Re-anchor stateful rule every cycle: wg0 PostUp uses -I FORWARD which
             # pushes ESTABLISHED,RELATED down below per-peer DROPs on restart.
             firewall_manager.ensure_forward_stateful()
+            # Caddy health monitor: 3 consecutive failures triggers a restart.
+            try:
+                if caddy_manager.check_caddy_health():
+                    caddy_manager.reset_health_failures()
+                else:
+                    count = caddy_manager.increment_health_failure()
+                    if count >= 3:
+                        logger.warning(
+                            "Caddy health check failed %d times \u2014 restarting",
+                            count,
+                        )
+                        container_manager.restart_container('cell-caddy')
+                        caddy_manager.reset_health_failures()
+            except Exception as _caddy_err:
+                logger.error("Caddy health monitor error: %s", _caddy_err)
         time.sleep(60)  # Check every 60 seconds
 
 # Start health monitor thread

api/caddy_manager.py

@@ -0,0 +1,391 @@
+#!/usr/bin/env python3
+"""
+Caddy Manager for Personal Internet Cell.
+
+Generates a Caddyfile based on the current identity (domain mode, cell name,
+domain) and the list of installed services that contribute reverse-proxy
+routes.  Uses Caddy's admin API on http://127.0.0.1:2019 to hot-reload the
+config without restarting the container.
+
+Domain modes supported:
+    lan        — local-only, internal CA, HTTP + self-signed HTTPS via
+                 /etc/caddy/internal/{cert,key}.pem
+    pic_ngo    — DNS-01 ACME via the pic_ngo Caddy plugin (wildcard cert)
+    cloudflare — DNS-01 ACME via the cloudflare Caddy plugin (wildcard cert)
+    duckdns    — DNS-01 ACME via the duckdns Caddy plugin
+    http01     — HTTP-01 ACME (no wildcard); each subdomain gets its own
+                 server block (used by No-IP, FreeDNS, etc.)
+
+For all ACME modes ``acme_ca`` is read from the ``ACME_CA_URL`` env var so
+tests / staging can point at Pebble or LE-staging without a code change.
+Routes for installed services are inserted before the catch-all ``handle``
+in the main server block (or, for ``http01``, written as their own per-host
+blocks).
+"""
+
+import logging
+import os
+from typing import Any, Dict, List, Optional
+
+import requests
+
+from base_service_manager import BaseServiceManager
+
+logger = logging.getLogger(__name__)
+
+# Live Caddyfile path inside the cell-api container (host path is
+# ./config/caddy/Caddyfile, mounted at /app/config-caddy).  May be overridden
+# in tests via the CADDYFILE_PATH env var.
+LIVE_CADDYFILE = os.environ.get('CADDYFILE_PATH', '/app/config-caddy/Caddyfile')
+
+# Caddy admin API base — local to the cell-api container only because Caddy
+# binds 2019 on 127.0.0.1.  In production the API and Caddy both run with
+# host networking via the bridge, so this hostname must be set to the Caddy
+# container hostname (or admin enabled cluster-wide).  We default to
+# localhost to match the dev/test wiring.
+CADDY_ADMIN_URL = os.environ.get('CADDY_ADMIN_URL', 'http://cell-caddy:2019')
+
+
+class CaddyManager(BaseServiceManager):
+    """Manages Caddy reverse-proxy configuration and runtime health."""
+
+    def __init__(self, config_manager=None,
+                 data_dir: str = '/app/data',
+                 config_dir: str = '/app/config'):
+        super().__init__('caddy', data_dir, config_dir)
+        self.config_manager = config_manager
+        self.container_name = 'cell-caddy'
+        self.caddyfile_path = LIVE_CADDYFILE
+        # Consecutive health-check failure counter (reset on success or when
+        # the caller restarts the container).
+        self._health_failures = 0
+
+    # ── BaseServiceManager required ───────────────────────────────────────
+
+    def get_status(self) -> Dict[str, Any]:
+        """Return basic Caddy status (running + admin-API reachable)."""
+        healthy = self.check_caddy_health()
+        return {
+            'service': self.service_name,
+            'running': healthy,
+            'admin_url': CADDY_ADMIN_URL,
+            'caddyfile_path': self.caddyfile_path,
+            'consecutive_failures': self._health_failures,
+        }
+
+    def test_connectivity(self) -> Dict[str, Any]:
+        """Ping the Caddy admin API."""
+        ok = self.check_caddy_health()
+        return {
+            'success': ok,
+            'admin_url': CADDY_ADMIN_URL,
+        }
+
+    # ── Caddyfile generation ──────────────────────────────────────────────
+
+    def generate_caddyfile(self, identity: Dict[str, Any],
+                           installed_services: List[Dict[str, Any]]) -> str:
+        """Generate a complete Caddyfile based on identity and services.
+
+        Args:
+            identity: identity dict from ``ConfigManager.get_identity()``.
+                      Expected keys: ``cell_name``, ``domain_mode``, optional
+                      ``custom_domain``, ``acme_email``.
+            installed_services: list of service dicts; each may have a
+                                ``caddy_route`` string with one or more
+                                Caddyfile directives (e.g.
+                                ``"handle /calendar* {\\n  reverse_proxy ..."``).
+
+        Returns:
+            Caddyfile text.
+        """
+        identity = identity or {}
+        cell_name = identity.get('cell_name', 'cell')
+        domain_mode = identity.get('domain_mode', 'lan')
+
+        # Aggregate the per-service route snippets that go inside the main
+        # server block (everything except http01 mode).  Each route is
+        # indented to four spaces to keep the Caddyfile readable.
+        service_routes = self._collect_service_routes(installed_services)
+
+        # Core routes always present in the main server block.  Inserted
+        # *after* installed-service routes so a more specific /api/* on a
+        # service can never shadow the API itself (no service should use
+        # /api anyway, but this protects us from misconfigured plugins).
+        core_routes = (
+            "    handle /api/* {\n"
+            "        reverse_proxy cell-api:3000\n"
+            "    }\n"
+            "    handle {\n"
+            "        reverse_proxy cell-webui:80\n"
+            "    }"
+        )
+
+        if domain_mode == 'lan':
+            return self._caddyfile_lan(cell_name, service_routes, core_routes)
+        if domain_mode == 'pic_ngo':
+            return self._caddyfile_pic_ngo(cell_name, service_routes, core_routes)
+        if domain_mode == 'cloudflare':
+            custom_domain = identity.get('custom_domain', f'{cell_name}.local')
+            return self._caddyfile_cloudflare(
+                custom_domain, service_routes, core_routes
+            )
+        if domain_mode == 'duckdns':
+            return self._caddyfile_duckdns(cell_name, service_routes, core_routes)
+        if domain_mode == 'http01':
+            host = identity.get('custom_domain', f'{cell_name}.noip.me')
+            return self._caddyfile_http01(host, installed_services, core_routes)
+
+        # Fallback to lan so we always emit a valid Caddyfile.
+        logger.warning("Unknown domain_mode %r; falling back to 'lan'", domain_mode)
+        return self._caddyfile_lan(cell_name, service_routes, core_routes)
+
+    # ── per-mode generators ───────────────────────────────────────────────
+
+    @staticmethod
+    def _global_acme_block(email: Optional[str]) -> str:
+        """Return the ``{ ... }`` global block for an ACME-enabled mode."""
+        lines = ["{"]
+        # Bind admin API on all interfaces so cell-api can reach cell-caddy
+        # across the Docker bridge (default 127.0.0.1 is unreachable cross-container).
+        lines.append("    admin 0.0.0.0:2019")
+        if email:
+            lines.append(f"    email {email}")
+        # Always allow tests to override the ACME directory via env var.
+        lines.append("    acme_ca {$ACME_CA_URL}")
+        lines.append("}")
+        return "\n".join(lines)
+
+    @staticmethod
+    def _indent_routes(routes: str, spaces: int = 4) -> str:
+        """Indent a multi-line route block by ``spaces`` columns."""
+        if not routes:
+            return ""
+        prefix = " " * spaces
+        return "\n".join(prefix + line if line.strip() else line
+                         for line in routes.splitlines())
+
+    def _collect_service_routes(self,
+                                installed_services: List[Dict[str, Any]]) -> str:
+        """Concatenate ``caddy_route`` strings from installed services."""
+        chunks: List[str] = []
+        for svc in installed_services or []:
+            route = (svc or {}).get('caddy_route')
+            if route:
+                chunks.append(route.strip("\n"))
+        return "\n".join(chunks)
+
+    def _caddyfile_lan(self, cell_name: str,
+                       service_routes: str, core_routes: str) -> str:
+        """LAN mode: HTTP only + internal-CA TLS, no ACME."""
+        body = []
+        if service_routes:
+            body.append(self._indent_routes(service_routes))
+        body.append(core_routes)
+        inner = "\n".join(body)
+        return (
+            "{\n"
+            "    admin 0.0.0.0:2019\n"
+            "    auto_https off\n"
+            "}\n"
+            "\n"
+            f"http://{cell_name}.cell, http://172.20.0.2:80 {{\n"
+            "    tls /etc/caddy/internal/cert.pem /etc/caddy/internal/key.pem\n"
+            f"{inner}\n"
+            "}\n"
+        )
+
+    def _caddyfile_pic_ngo(self, cell_name: str,
+                           service_routes: str, core_routes: str) -> str:
+        """pic_ngo mode: wildcard DNS-01 via the pic_ngo plugin."""
+        body = []
+        if service_routes:
+            body.append(self._indent_routes(service_routes))
+        body.append(core_routes)
+        inner = "\n".join(body)
+        email = f"admin@{cell_name}.pic.ngo"
+        return (
+            f"{self._global_acme_block(email)}\n"
+            "\n"
+            f"*.{cell_name}.pic.ngo, {cell_name}.pic.ngo {{\n"
+            "    tls {\n"
+            "        dns pic_ngo {\n"
+            "            token {$PIC_NGO_DDNS_TOKEN}\n"
+            "            api_base_url {$PIC_NGO_DDNS_API}\n"
+            "        }\n"
+            "    }\n"
+            f"{inner}\n"
+            "}\n"
+        )
+
+    def _caddyfile_cloudflare(self, custom_domain: str,
+                              service_routes: str, core_routes: str) -> str:
+        """cloudflare mode: wildcard DNS-01 via the cloudflare plugin."""
+        body = []
+        if service_routes:
+            body.append(self._indent_routes(service_routes))
+        body.append(core_routes)
+        inner = "\n".join(body)
+        return (
+            f"{self._global_acme_block('{$ACME_EMAIL}')}\n"
+            "\n"
+            f"*.{custom_domain}, {custom_domain} {{\n"
+            "    tls {\n"
+            "        dns cloudflare {$CF_API_TOKEN}\n"
+            "    }\n"
+            f"{inner}\n"
+            "}\n"
+        )
+
+    def _caddyfile_duckdns(self, cell_name: str,
+                           service_routes: str, core_routes: str) -> str:
+        """duckdns mode: DNS-01 via the duckdns plugin."""
+        body = []
+        if service_routes:
+            body.append(self._indent_routes(service_routes))
+        body.append(core_routes)
+        inner = "\n".join(body)
+        return (
+            f"{self._global_acme_block(None)}\n"
+            "\n"
+            f"*.{cell_name}.duckdns.org {{\n"
+            "    tls {\n"
+            "        dns duckdns {$DUCKDNS_TOKEN}\n"
+            "    }\n"
+            f"{inner}\n"
+            "}\n"
+        )
+
+    def _caddyfile_http01(self, host: str,
+                          installed_services: List[Dict[str, Any]],
+                          core_routes: str) -> str:
+        """http01 mode: no wildcard.  Each service gets its own block."""
+        # Main host block — only the core routes (api + webui).  Service
+        # routes that could otherwise be served as path-prefixes are NOT
+        # placed here because in http01 mode each service is intended to
+        # live on its own subdomain (otherwise it could also use a path
+        # prefix here, but the spec calls for separate blocks).
+        out = [self._global_acme_block('{$ACME_EMAIL}'), ""]
+        out.append(f"{host} {{")
+        out.append(core_routes)
+        out.append("}")
+
+        # One block per installed service that has a caddy_route.
+        for svc in installed_services or []:
+            if not svc:
+                continue
+            route = svc.get('caddy_route')
+            name = svc.get('name') or svc.get('subdomain')
+            if not route or not name:
+                continue
+            out.append("")
+            out.append(f"{name}.{host} {{")
+            out.append(self._indent_routes(route))
+            out.append("}")
+        return "\n".join(out) + "\n"
+
+    # ── filesystem + admin-API operations ─────────────────────────────────
+
+    def write_caddyfile(self, caddyfile_content: str) -> bool:
+        """Write the Caddyfile and reload Caddy via the admin API.
+
+        Writes in-place (same inode) so Docker bind-mounts continue to see
+        the file.  Returns True if both write and reload succeed.
+        """
+        try:
+            os.makedirs(os.path.dirname(os.path.abspath(self.caddyfile_path)),
+                        exist_ok=True)
+        except (PermissionError, OSError) as e:
+            logger.warning("Could not create Caddyfile dir: %s", e)
+
+        try:
+            with open(self.caddyfile_path, 'w') as f:
+                f.write(caddyfile_content)
+                f.flush()
+                try:
+                    os.fsync(f.fileno())
+                except OSError:
+                    pass
+            logger.info("Wrote Caddyfile to %s (%d bytes)",
+                        self.caddyfile_path, len(caddyfile_content))
+        except Exception as e:
+            logger.error("Failed to write Caddyfile: %s", e)
+            return False
+
+        return self.reload_caddy()
+
+    def reload_caddy(self) -> bool:
+        """POST the current Caddyfile to the Caddy admin API for a hot reload.
+
+        Returns True on HTTP 200, False otherwise.
+        """
+        try:
+            with open(self.caddyfile_path, 'r') as f:
+                caddyfile = f.read()
+        except Exception as e:
+            logger.error("Cannot read Caddyfile for reload: %s", e)
+            return False
+
+        url = f"{CADDY_ADMIN_URL}/load"
+        try:
+            resp = requests.post(
+                url,
+                data=caddyfile,
+                headers={'Content-Type': 'text/caddyfile'},
+                timeout=10,
+            )
+        except requests.RequestException as e:
+            logger.error("Caddy admin reload failed: %s", e)
+            return False
+
+        if resp.status_code == 200:
+            logger.info("Caddy reload succeeded (status=200)")
+            return True
+        logger.error(
+            "Caddy reload failed: status=%s body=%s",
+            resp.status_code, resp.text[:500],
+        )
+        return False
+
+    def check_caddy_health(self) -> bool:
+        """GET the Caddy admin API root.  Returns True on HTTP 200."""
+        try:
+            resp = requests.get(CADDY_ADMIN_URL + "/", timeout=5)
+        except requests.RequestException as e:
+            logger.debug("Caddy health check error: %s", e)
+            return False
+        return resp.status_code == 200
+
+    # ── consecutive-failure bookkeeping ───────────────────────────────────
+
+    def get_health_failure_count(self) -> int:
+        """Return the current consecutive failure count."""
+        return self._health_failures
+
+    def increment_health_failure(self) -> int:
+        """Increment and return the consecutive failure count."""
+        self._health_failures += 1
+        return self._health_failures
+
+    def reset_health_failures(self) -> None:
+        """Reset the consecutive failure counter to zero."""
+        self._health_failures = 0
+
+    # ── certificate status ────────────────────────────────────────────────
+
+    def get_cert_status(self) -> Dict[str, Any]:
+        """Return TLS cert status from identity['tls'] if present."""
+        default = {'status': 'unknown', 'expiry': None, 'days_remaining': None}
+        if not self.config_manager:
+            return default
+        try:
+            ident = self.config_manager.get_identity() or {}
+        except Exception as e:
+            logger.error("get_cert_status: failed to read identity: %s", e)
+            return default
+        tls = ident.get('tls') or {}
+        return {
+            'status': tls.get('status', 'unknown'),
+            'expiry': tls.get('expiry'),
+            'days_remaining': tls.get('days_remaining'),
+        }

api/managers.py

@@ -28,6 +28,7 @@ from cell_link_manager import CellLinkManager
 import firewall_manager
 from auth_manager import AuthManager
 from setup_manager import SetupManager
+from caddy_manager import CaddyManager
 
 DATA_DIR   = os.environ.get('DATA_DIR',   '/app/data')
 CONFIG_DIR = os.environ.get('CONFIG_DIR', '/app/config')
@@ -55,6 +56,7 @@ cell_link_manager  = CellLinkManager(
 )
 auth_manager = AuthManager(data_dir=DATA_DIR, config_dir=CONFIG_DIR)
 setup_manager = SetupManager(config_manager=config_manager, auth_manager=auth_manager)
+caddy_manager = CaddyManager(config_manager=config_manager, data_dir=DATA_DIR, config_dir=CONFIG_DIR)
 
 # Service logger configuration
 _service_log_configs = {
@@ -88,7 +90,7 @@ __all__ = [
     'network_manager', 'wireguard_manager', 'peer_registry',
     'email_manager', 'calendar_manager', 'file_manager',
     'routing_manager', 'vault_manager', 'container_manager',
-    'cell_link_manager', 'auth_manager', 'setup_manager',
+    'cell_link_manager', 'auth_manager', 'setup_manager', 'caddy_manager',
     'firewall_manager', 'EventType',
     'DATA_DIR', 'CONFIG_DIR',
 ]

api/setup_manager.py

@@ -55,7 +55,7 @@ AVAILABLE_SERVICES = [
     'wireguard',
 ]
 
-VALID_DOMAIN_MODES = {'pic_ngo', 'custom', 'lan'}
+VALID_DOMAIN_MODES = {'pic_ngo', 'cloudflare', 'duckdns', 'http01', 'lan'}
 
 CELL_NAME_RE = re.compile(r'^[a-z][a-z0-9-]{1,30}$')
 

docker-compose.yml

@@ -3,7 +3,7 @@ version: '3.3'
 services:
   # Reverse Proxy - Caddy for routing all .cell traffic
   caddy:
-    image: caddy:2-alpine
+    image: git.pic.ngo/roof/pic-caddy:latest
     container_name: cell-caddy
     profiles: ["core", "full"]
     ports:

tests/test_caddy_manager.py

@@ -0,0 +1,228 @@
+"""Tests for CaddyManager — Caddyfile generation per domain mode plus
+admin-API reload, health check, and consecutive-failure bookkeeping.
+"""
+import os
+import sys
+import unittest
+from unittest.mock import MagicMock, patch
+
+import requests
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'api'))
+
+from caddy_manager import CaddyManager  # noqa: E402
+
+
+def _mgr(tmpdir=None, identity=None):
+    """Build a CaddyManager backed by a mock config_manager."""
+    cm = MagicMock()
+    cm.get_identity.return_value = identity or {}
+    mgr = CaddyManager(
+        config_manager=cm,
+        data_dir=tmpdir or '/tmp/pic-test-data',
+        config_dir=tmpdir or '/tmp/pic-test-config',
+    )
+    return mgr
+
+
+CALENDAR_ROUTE = (
+    "handle /calendar* {\n"
+    "    reverse_proxy cell-radicale:5232\n"
+    "}"
+)
+FILES_ROUTE = (
+    "handle /files* {\n"
+    "    reverse_proxy cell-filegator:8080\n"
+    "}"
+)
+
+
+class TestGenerateCaddyfileLan(unittest.TestCase):
+    def test_lan_mode_has_auto_https_off_and_no_acme(self):
+        mgr = _mgr()
+        identity = {'cell_name': 'mycell', 'domain_mode': 'lan'}
+        out = mgr.generate_caddyfile(identity, [])
+        self.assertIn('auto_https off', out)
+        # No ACME anywhere
+        self.assertNotIn('acme_ca', out)
+        self.assertNotIn('acme_email', out)
+        self.assertNotIn('dns pic_ngo', out)
+        self.assertNotIn('dns cloudflare', out)
+        # Internal-CA TLS pair
+        self.assertIn('tls /etc/caddy/internal/cert.pem '
+                      '/etc/caddy/internal/key.pem', out)
+        # Cell hostname plus virtual IP listener
+        self.assertIn('http://mycell.cell', out)
+        self.assertIn('http://172.20.0.2:80', out)
+
+
+class TestGenerateCaddyfilePicNgo(unittest.TestCase):
+    def test_pic_ngo_has_dns_plugin_and_wildcard(self):
+        mgr = _mgr()
+        identity = {'cell_name': 'alpha', 'domain_mode': 'pic_ngo'}
+        out = mgr.generate_caddyfile(identity, [])
+        self.assertIn('dns pic_ngo', out)
+        self.assertIn('*.alpha.pic.ngo', out)
+        self.assertIn('alpha.pic.ngo', out)
+        self.assertIn('{$PIC_NGO_DDNS_TOKEN}', out)
+        self.assertIn('{$PIC_NGO_DDNS_API}', out)
+        self.assertIn('email admin@alpha.pic.ngo', out)
+        # ACME staging hook
+        self.assertIn('acme_ca {$ACME_CA_URL}', out)
+
+
+class TestGenerateCaddyfileCloudflare(unittest.TestCase):
+    def test_cloudflare_has_dns_cloudflare(self):
+        mgr = _mgr()
+        identity = {
+            'cell_name': 'beta',
+            'domain_mode': 'cloudflare',
+            'custom_domain': 'example.com',
+        }
+        out = mgr.generate_caddyfile(identity, [])
+        self.assertIn('dns cloudflare {$CF_API_TOKEN}', out)
+        self.assertIn('*.example.com', out)
+        self.assertIn('email {$ACME_EMAIL}', out)
+        self.assertIn('acme_ca {$ACME_CA_URL}', out)
+
+
+class TestGenerateCaddyfileDuckDns(unittest.TestCase):
+    def test_duckdns_has_dns_duckdns(self):
+        mgr = _mgr()
+        identity = {'cell_name': 'gamma', 'domain_mode': 'duckdns'}
+        out = mgr.generate_caddyfile(identity, [])
+        self.assertIn('dns duckdns {$DUCKDNS_TOKEN}', out)
+        self.assertIn('*.gamma.duckdns.org', out)
+
+
+class TestGenerateCaddyfileHttp01(unittest.TestCase):
+    def test_http01_no_tls_block_and_per_service_blocks(self):
+        mgr = _mgr()
+        identity = {
+            'cell_name': 'delta',
+            'domain_mode': 'http01',
+            'custom_domain': 'delta.noip.me',
+        }
+        services = [
+            {'name': 'calendar', 'caddy_route':
+             'reverse_proxy cell-radicale:5232'},
+            {'name': 'files', 'caddy_route':
+             'reverse_proxy cell-filegator:8080'},
+        ]
+        out = mgr.generate_caddyfile(identity, services)
+        # No wildcard, no DNS-01 plugins.
+        self.assertNotIn('*.delta', out)
+        self.assertNotIn('dns ', out)
+        # No explicit tls block (no internal CA, no plugin) — the host block
+        # itself is left empty so Caddy uses HTTP-01 by default.
+        self.assertNotIn('tls {', out)
+        # Per-service blocks
+        self.assertIn('calendar.delta.noip.me {', out)
+        self.assertIn('files.delta.noip.me {', out)
+        self.assertIn('reverse_proxy cell-radicale:5232', out)
+        self.assertIn('reverse_proxy cell-filegator:8080', out)
+
+
+class TestServiceRoutesIncluded(unittest.TestCase):
+    def test_installed_service_route_appears_in_output(self):
+        mgr = _mgr()
+        identity = {'cell_name': 'eps', 'domain_mode': 'lan'}
+        services = [
+            {'name': 'calendar', 'caddy_route': CALENDAR_ROUTE},
+            {'name': 'files',    'caddy_route': FILES_ROUTE},
+        ]
+        out = mgr.generate_caddyfile(identity, services)
+        self.assertIn('handle /calendar*', out)
+        self.assertIn('reverse_proxy cell-radicale:5232', out)
+        self.assertIn('handle /files*', out)
+        self.assertIn('reverse_proxy cell-filegator:8080', out)
+        # Core routes still emitted
+        self.assertIn('reverse_proxy cell-api:3000', out)
+        self.assertIn('reverse_proxy cell-webui:80', out)
+
+
+class TestReloadCaddyAdminAPI(unittest.TestCase):
+    def test_reload_calls_admin_api_load_endpoint(self):
+        mgr = _mgr()
+        # Point at a tmp Caddyfile so we can read it back during reload.
+        import tempfile
+        tmp = tempfile.NamedTemporaryFile('w', delete=False, suffix='.caddyfile')
+        tmp.write(":80 { reverse_proxy cell-webui:80 }\n")
+        tmp.close()
+        mgr.caddyfile_path = tmp.name
+
+        with patch('caddy_manager.requests.post') as mock_post:
+            mock_post.return_value = MagicMock(status_code=200, text='ok')
+            ok = mgr.reload_caddy()
+
+        self.assertTrue(ok)
+        mock_post.assert_called_once()
+        args, kwargs = mock_post.call_args
+        # First positional arg is the URL
+        self.assertEqual(args[0], 'http://cell-caddy:2019/load')
+        self.assertEqual(kwargs['headers']['Content-Type'], 'text/caddyfile')
+        self.assertIn('cell-webui:80', kwargs['data'])
+        os.unlink(tmp.name)
+
+
+class TestHealthCheck(unittest.TestCase):
+    def test_returns_true_on_200(self):
+        mgr = _mgr()
+        with patch('caddy_manager.requests.get') as mock_get:
+            mock_get.return_value = MagicMock(status_code=200)
+            self.assertTrue(mgr.check_caddy_health())
+        mock_get.assert_called_once()
+        # URL must be the admin API root
+        self.assertIn('cell-caddy:2019', mock_get.call_args[0][0])
+
+    def test_returns_false_on_connection_error(self):
+        mgr = _mgr()
+        with patch('caddy_manager.requests.get',
+                   side_effect=requests.ConnectionError('refused')):
+            self.assertFalse(mgr.check_caddy_health())
+
+    def test_returns_false_on_non_200(self):
+        mgr = _mgr()
+        with patch('caddy_manager.requests.get') as mock_get:
+            mock_get.return_value = MagicMock(status_code=500)
+            self.assertFalse(mgr.check_caddy_health())
+
+
+class TestFailureCounter(unittest.TestCase):
+    def test_increments_and_resets(self):
+        mgr = _mgr()
+        self.assertEqual(mgr.get_health_failure_count(), 0)
+        self.assertEqual(mgr.increment_health_failure(), 1)
+        self.assertEqual(mgr.increment_health_failure(), 2)
+        self.assertEqual(mgr.increment_health_failure(), 3)
+        self.assertEqual(mgr.get_health_failure_count(), 3)
+        mgr.reset_health_failures()
+        self.assertEqual(mgr.get_health_failure_count(), 0)
+
+
+class TestCertStatus(unittest.TestCase):
+    def test_returns_default_when_no_tls_in_identity(self):
+        mgr = _mgr(identity={'cell_name': 'x', 'domain_mode': 'lan'})
+        out = mgr.get_cert_status()
+        self.assertEqual(out['status'], 'unknown')
+        self.assertIsNone(out['expiry'])
+        self.assertIsNone(out['days_remaining'])
+
+    def test_returns_tls_block_when_present(self):
+        mgr = _mgr(identity={
+            'cell_name': 'x',
+            'domain_mode': 'pic_ngo',
+            'tls': {
+                'status': 'valid',
+                'expiry': '2026-08-01T00:00:00Z',
+                'days_remaining': 84,
+            },
+        })
+        out = mgr.get_cert_status()
+        self.assertEqual(out['status'], 'valid')
+        self.assertEqual(out['expiry'], '2026-08-01T00:00:00Z')
+        self.assertEqual(out['days_remaining'], 84)
+
+
+if __name__ == '__main__':
+    unittest.main()