feat: replace hardcoded service names with ServiceRegistry-driven Caddy and CoreDNS config

Previously, CaddyManager and NetworkManager contained hardcoded lists of
service names (calendar, files, mail, webdav, etc.), meaning every new
service required a code change to appear in Caddy routes and DNS records.
Now both managers accept a service_registry parameter and derive their
service lists dynamically from the registry at runtime.

- CaddyManager: new _build_registry_service_routes() and
  _http01_service_pairs() methods pull routes from the registry
- NetworkManager: new _get_service_subdomains() method returns registry
  subdomains with a hardcoded fallback when no registry is wired in;
  _build_dns_records, stale-record detection, and service name sets all
  use the registry
- managers.py: service_registry constructed before network_manager so it
  can be injected into both CaddyManager and NetworkManager
- service_registry.py: validation chokepoint in get_caddy_routes() rejects
  invalid subdomain/backend values and reserved service names
- service_store_manager.py: _validate_manifest now validates top-level
  subdomain, backend, extra_subdomains, and extra_backends fields
- tests: 24 new tests covering registry-driven routing and DNS subdomain
  generation (test_caddy_registry_integration.py)

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

api/app.py

@@ -44,6 +44,9 @@ from managers import (
     caddy_manager,
     ddns_manager, service_store_manager,
     connectivity_manager,
+    service_registry,
+    service_composer,
+    account_manager,
     firewall_manager, EventType,
 )
 # Re-exports: tests do `from app import CellManager` and `from app import _resolve_peer_dns`

api/caddy_manager.py

@@ -52,11 +52,13 @@ class CaddyManager(BaseServiceManager):
     def __init__(self, config_manager=None,
                  data_dir: str = '/app/data',
                  config_dir: str = '/app/config',
-                 service_bus=None):
+                 service_bus=None,
+                 service_registry=None):
         super().__init__('caddy', data_dir, config_dir)
         self.config_manager = config_manager
         self.container_name = 'cell-caddy'
         self.caddyfile_path = LIVE_CADDYFILE
+        self._service_registry = service_registry
         # Consecutive health-check failure counter (reset on success or when
         # the caller restarts the container).
         self._health_failures = 0
@@ -187,6 +189,69 @@ class CaddyManager(BaseServiceManager):
             f"    }}"
         )
 
+    def _build_registry_service_routes(self, domain: str) -> str:
+        """Build named-matcher + handle blocks from the service registry.
+
+        Falls back to the hardcoded ``_build_core_service_routes`` when no
+        registry is wired or the registry returns nothing, so the method is
+        always safe to call even in tests that don't supply a registry.
+        """
+        routes: List[Dict] = []
+        if self._service_registry is not None:
+            try:
+                routes = self._service_registry.get_caddy_routes()
+            except Exception as exc:
+                logger.warning('_build_registry_service_routes: registry error: %s', exc)
+
+        if not routes:
+            return self._build_core_service_routes(domain)
+
+        # Pre-seed with reserved names so no registry entry can squat them.
+        seen_matchers: set = {'api', 'webui'}
+
+        blocks: List[str] = []
+        for route in routes:
+            primary_sub = route['subdomain']
+            backend = route['backend']
+            extra_subs: List[str] = route.get('extra_subdomains') or []
+            extra_backends: Dict[str, str] = route.get('extra_backends') or {}
+
+            if primary_sub in seen_matchers:
+                logger.warning('Caddy: skipping duplicate/reserved matcher %r', primary_sub)
+                continue
+            seen_matchers.add(primary_sub)
+
+            # Subdomains that share the primary backend go in one matcher block.
+            shared = [primary_sub] + [s for s in extra_subs if s not in extra_backends]
+            host_list = ' '.join(f'{s}.{domain}' for s in shared)
+            blocks.append(
+                f'    @{primary_sub} host {host_list}\n'
+                f'    handle @{primary_sub} {{\n'
+                f'        reverse_proxy {backend}\n'
+                f'    }}'
+            )
+            # Extra subdomains with their own backends each get their own block.
+            for sub, sub_backend in extra_backends.items():
+                if sub in seen_matchers:
+                    logger.warning('Caddy: skipping duplicate/reserved matcher %r', sub)
+                    continue
+                seen_matchers.add(sub)
+                blocks.append(
+                    f'    @{sub} host {sub}.{domain}\n'
+                    f'    handle @{sub} {{\n'
+                    f'        reverse_proxy {sub_backend}\n'
+                    f'    }}'
+                )
+
+        # The api subdomain is always infrastructure — not delegated to the registry.
+        blocks.append(
+            f'    @api host api.{domain}\n'
+            f'    handle @api {{\n'
+            f'        reverse_proxy cell-api:3000\n'
+            f'    }}'
+        )
+        return '\n'.join(blocks)
+
     @staticmethod
     def _indent_routes(routes: str, spaces: int = 4) -> str:
         """Indent a multi-line route block by ``spaces`` columns."""
@@ -230,7 +295,7 @@ class CaddyManager(BaseServiceManager):
                            service_routes: str, core_routes: str) -> str:
         """pic_ngo mode: wildcard DNS-01 via the pic_ngo plugin."""
         domain = f"{cell_name}.pic.ngo"
-        body = [self._build_core_service_routes(domain)]
+        body = [self._build_registry_service_routes(domain)]
         if service_routes:
             body.append(self._indent_routes(service_routes))
         body.append(core_routes)
@@ -253,7 +318,7 @@ class CaddyManager(BaseServiceManager):
     def _caddyfile_cloudflare(self, custom_domain: str,
                               service_routes: str, core_routes: str) -> str:
         """cloudflare mode: wildcard DNS-01 via the cloudflare plugin."""
-        body = [self._build_core_service_routes(custom_domain)]
+        body = [self._build_registry_service_routes(custom_domain)]
         if service_routes:
             body.append(self._indent_routes(service_routes))
         body.append(core_routes)
@@ -273,7 +338,7 @@ class CaddyManager(BaseServiceManager):
                            service_routes: str, core_routes: str) -> str:
         """duckdns mode: DNS-01 via the duckdns plugin."""
         domain = f"{cell_name}.duckdns.org"
-        body = [self._build_core_service_routes(domain)]
+        body = [self._build_registry_service_routes(domain)]
         if service_routes:
             body.append(self._indent_routes(service_routes))
         body.append(core_routes)
@@ -299,15 +364,8 @@ class CaddyManager(BaseServiceManager):
         out.append(core_routes)
         out.append("}")
 
-        # One block per core service subdomain.
-        _core_services = [
-            ('calendar', 'cell-radicale:5232'),
-            ('mail',     'cell-rainloop:8888'),
-            ('webmail',  'cell-rainloop:8888'),
-            ('files',    'cell-filegator:8080'),
-            ('webdav',   'cell-webdav:80'),
-            ('api',      'cell-api:3000'),
-        ]
+        # Build (subdomain, backend) pairs from registry when available.
+        _core_services = self._http01_service_pairs()
         for subdomain, backend in _core_services:
             out.append("")
             out.append(f"{subdomain}.{host} {{")
@@ -330,6 +388,32 @@ class CaddyManager(BaseServiceManager):
             out.append("}")
         return "\n".join(out) + "\n"
 
+    def _http01_service_pairs(self) -> List[tuple]:
+        """Return (subdomain, backend) pairs for http01 per-host blocks."""
+        pairs: List[tuple] = []
+        if self._service_registry is not None:
+            try:
+                for route in self._service_registry.get_caddy_routes():
+                    pairs.append((route['subdomain'], route['backend']))
+                    extra_subs: List[str] = route.get('extra_subdomains') or []
+                    extra_backends: Dict[str, str] = route.get('extra_backends') or {}
+                    for sub in extra_subs:
+                        backend = extra_backends.get(sub, route['backend'])
+                        pairs.append((sub, backend))
+            except Exception as exc:
+                logger.warning('_http01_service_pairs: registry error: %s', exc)
+                pairs = []
+        if not pairs:
+            pairs = [
+                ('calendar', 'cell-radicale:5232'),
+                ('mail',     'cell-rainloop:8888'),
+                ('webmail',  'cell-rainloop:8888'),
+                ('files',    'cell-filegator:8080'),
+                ('webdav',   'cell-webdav:80'),
+            ]
+        pairs.append(('api', 'cell-api:3000'))
+        return pairs
+
     # ── filesystem + admin-API operations ─────────────────────────────────
 
     def write_caddyfile(self, caddyfile_content: str) -> bool:

api/config_manager.py

@@ -6,6 +6,8 @@ Centralized configuration management for all services
 
 import os
 import json
+import re
+import subprocess
 import yaml
 import shutil
 import hashlib
@@ -14,6 +16,9 @@ from typing import Dict, List, Optional, Any
 from pathlib import Path
 import logging
 
+_SAFE_CONTAINER_RE = re.compile(r'^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,63}$')
+_SAFE_VOL_NAME_RE  = re.compile(r'^[a-zA-Z0-9_.-]{1,64}$')
+
 # The Caddyfile lives on a separate volume mount from the rest of config
 LIVE_CADDYFILE = os.environ.get('CADDYFILE_PATH', '/app/config-caddy/Caddyfile')
 
@@ -228,8 +233,128 @@ class ConfigManager:
             "warnings": warnings
         }
     
-    def backup_config(self) -> str:
-        """Create a backup of cell_config.json, secrets, Caddyfile, .env, Corefile, and DNS zones."""
+    @staticmethod
+    def _validate_vol_entry(service_id: str, vol: dict) -> bool:
+        """Return True if a backup volume entry is safe to use; log and return False otherwise."""
+        container = vol.get('container', '')
+        path = vol.get('path', '')
+        name = vol.get('name', '')
+        if not _SAFE_CONTAINER_RE.match(container):
+            logger.warning('Backup: unsafe container name %r for %s — skipping', container, service_id)
+            return False
+        if not path.startswith('/') or '..' in path.split('/') or '\x00' in path:
+            logger.warning('Backup: unsafe volume path %r for %s — skipping', path, service_id)
+            return False
+        if not _SAFE_VOL_NAME_RE.match(name):
+            logger.warning('Backup: unsafe volume name %r for %s — skipping', name, service_id)
+            return False
+        return True
+
+    def _backup_service_volumes(self, backup_path: Path, service_registry) -> None:
+        """Stream service data out of each container via 'docker exec tar'.
+
+        Archives are relative (created with -C <path> .) so they can be safely
+        restored with -C <path> without risk of path traversal outside the volume.
+        Writes to a .partial temp file then renames atomically on success.
+        """
+        try:
+            plan = service_registry.get_backup_plan()
+        except Exception as e:
+            logger.warning('_backup_service_volumes: could not get backup plan: %s', e)
+            return
+        for entry in plan:
+            service_id = entry['service_id']
+            volumes = entry.get('volumes') or []
+            if not volumes:
+                continue
+            svc_dir = backup_path / 'service_data' / service_id
+            svc_dir.mkdir(parents=True, exist_ok=True)
+            for vol in volumes:
+                if not self._validate_vol_entry(service_id, vol):
+                    continue
+                container = vol['container']
+                path = vol['path']
+                name = vol['name']
+                archive_path = svc_dir / f'{name}.tar.gz'
+                tmp_path = svc_dir / f'{name}.tar.gz.partial'
+                try:
+                    with open(tmp_path, 'wb') as af:
+                        result = subprocess.run(
+                            # -C path; then '.' archives the whole dir with relative entries.
+                            # '--' prevents path/container from being parsed as options.
+                            ['docker', 'exec', '--', container,
+                             'tar', '-C', path, '-czf', '-', '.'],
+                            stdout=af,
+                            stderr=subprocess.PIPE,
+                            timeout=300,
+                        )
+                    if result.returncode != 0:
+                        logger.warning(
+                            'Backup: docker exec tar failed for %s/%s: %s',
+                            service_id, name, result.stderr.decode(errors='replace'),
+                        )
+                        tmp_path.unlink(missing_ok=True)
+                    else:
+                        os.replace(tmp_path, archive_path)
+                        logger.info('Backup: archived %s/%s', service_id, name)
+                except subprocess.TimeoutExpired:
+                    logger.warning('Backup: timed out streaming %s/%s', service_id, name)
+                    tmp_path.unlink(missing_ok=True)
+                except Exception as e:
+                    logger.warning('Backup: failed to archive %s/%s: %s', service_id, name, e)
+                    tmp_path.unlink(missing_ok=True)
+
+    def _restore_service_volumes(self, backup_path: Path, service_registry) -> None:
+        """Pipe archived service data back into containers via 'docker exec -i tar'.
+
+        Extracts with -C <path>, matching how archives were created (relative paths).
+        This bounds extraction to within the declared volume directory.
+        """
+        svc_data_dir = backup_path / 'service_data'
+        if not svc_data_dir.is_dir():
+            return
+        for svc_dir in svc_data_dir.iterdir():
+            if not svc_dir.is_dir():
+                continue
+            service_id = svc_dir.name
+            svc = service_registry.get(service_id)
+            if not svc:
+                logger.warning('Restore: unknown service %s in backup, skipping', service_id)
+                continue
+            volumes = (svc.get('backup') or {}).get('volumes') or []
+            for vol in volumes:
+                if not self._validate_vol_entry(service_id, vol):
+                    continue
+                container = vol['container']
+                path = vol['path']
+                name = vol['name']
+                archive_path = svc_dir / f'{name}.tar.gz'
+                if not archive_path.exists():
+                    continue
+                try:
+                    with open(archive_path, 'rb') as af:
+                        result = subprocess.run(
+                            ['docker', 'exec', '-i', '--', container,
+                             'tar', '-C', path, '-xzf', '-'],
+                            stdin=af,
+                            stderr=subprocess.PIPE,
+                            timeout=300,
+                        )
+                    if result.returncode != 0:
+                        logger.warning(
+                            'Restore: docker exec tar failed for %s/%s: %s',
+                            service_id, name, result.stderr.decode(errors='replace'),
+                        )
+                    else:
+                        logger.info('Restore: restored %s/%s', service_id, name)
+                except subprocess.TimeoutExpired:
+                    logger.warning('Restore: timed out restoring %s/%s', service_id, name)
+                except Exception as e:
+                    logger.warning('Restore: failed to restore %s/%s: %s', service_id, name, e)
+
+    def backup_config(self, service_registry=None) -> str:
+        """Create a backup of cell_config.json, secrets, Caddyfile, .env, Corefile, DNS zones,
+        and (when service_registry is provided) live service data volumes."""
         try:
             timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
             backup_id = f"backup_{timestamp}"
@@ -278,12 +403,17 @@ class ConfigManager:
                     except (PermissionError, OSError) as e:
                         logger.warning(f"Could not back up {src.name}: {e} (skipping)")
 
+            # Live service data volumes (streamed via docker exec)
+            if service_registry is not None:
+                self._backup_service_volumes(backup_path, service_registry)
+
             services = ['identity'] + list(self.service_schemas.keys())
             manifest = {
                 "backup_id": backup_id,
                 "timestamp": datetime.now().isoformat(),
                 "services": services,
                 "files": [f.name for f in backup_path.iterdir()],
+                "includes_service_data": service_registry is not None,
             }
             with open(backup_path / 'manifest.json', 'w') as f:
                 json.dump(manifest, f, indent=2)
@@ -294,8 +424,9 @@ class ConfigManager:
         except Exception as e:
             logger.error(f"Error creating backup: {e}")
             raise
-    
-    def restore_config(self, backup_id: str, services: list = None) -> bool:
+
+    def restore_config(self, backup_id: str, services: list = None,
+                       service_registry=None) -> bool:
         """Restore from backup. If services list given, only restore those service configs (selective)."""
         try:
             backup_path = self.backup_dir / backup_id
@@ -373,6 +504,10 @@ class ConfigManager:
                     except (PermissionError, OSError) as e:
                         logger.warning(f"Could not restore {dest.name}: {e} (skipping)")
 
+            # Live service data volumes
+            if service_registry is not None:
+                self._restore_service_volumes(backup_path, service_registry)
+
             self.configs = self._load_all_configs()
             logger.info(f"Restored configuration from backup: {backup_id}")
             return True

api/managers.py

@@ -31,6 +31,9 @@ from setup_manager import SetupManager
 from caddy_manager import CaddyManager
 from ddns_manager import DDNSManager
 from connectivity_manager import ConnectivityManager
+from service_registry import ServiceRegistry
+from service_composer import ServiceComposer
+from account_manager import AccountManager
 
 DATA_DIR   = os.environ.get('DATA_DIR',   '/app/data')
 CONFIG_DIR = os.environ.get('CONFIG_DIR', '/app/config')
@@ -42,7 +45,13 @@ config_manager = ConfigManager(
 service_bus  = ServiceBus()
 log_manager  = LogManager(log_dir='./data/logs')
 
-network_manager    = NetworkManager(data_dir=DATA_DIR, config_dir=CONFIG_DIR)
+# ServiceRegistry depends only on config_manager; create it early so
+# NetworkManager and CaddyManager can derive subdomains from manifests
+# instead of hardcoding service names.
+service_registry = ServiceRegistry(config_manager=config_manager)
+
+network_manager    = NetworkManager(data_dir=DATA_DIR, config_dir=CONFIG_DIR,
+                                    service_registry=service_registry)
 wireguard_manager  = WireGuardManager(data_dir=DATA_DIR, config_dir=CONFIG_DIR)
 peer_registry      = PeerRegistry(data_dir=DATA_DIR, config_dir=CONFIG_DIR)
 email_manager      = EmailManager(data_dir=DATA_DIR, config_dir=CONFIG_DIR, service_bus=service_bus)
@@ -58,7 +67,8 @@ cell_link_manager  = CellLinkManager(
 )
 auth_manager = AuthManager(data_dir=DATA_DIR, config_dir=CONFIG_DIR)
 setup_manager = SetupManager(config_manager=config_manager, auth_manager=auth_manager)
-caddy_manager = CaddyManager(config_manager=config_manager, data_dir=DATA_DIR, config_dir=CONFIG_DIR, service_bus=service_bus)
+caddy_manager = CaddyManager(config_manager=config_manager, data_dir=DATA_DIR, config_dir=CONFIG_DIR,
+                              service_bus=service_bus, service_registry=service_registry)
 ddns_manager = DDNSManager(config_manager=config_manager, data_dir=DATA_DIR, config_dir=CONFIG_DIR)
 connectivity_manager = ConnectivityManager(
     config_manager=config_manager,
@@ -67,6 +77,15 @@ connectivity_manager = ConnectivityManager(
     config_dir=CONFIG_DIR,
 )
 
+service_composer = ServiceComposer(config_manager=config_manager, data_dir=DATA_DIR)
+account_manager  = AccountManager(
+    service_registry=service_registry,
+    data_dir=DATA_DIR,
+    email_manager=email_manager,
+    calendar_manager=calendar_manager,
+    file_manager=file_manager,
+)
+
 from service_store_manager import ServiceStoreManager
 service_store_manager = ServiceStoreManager(
     config_manager=config_manager,
@@ -110,6 +129,7 @@ __all__ = [
     'routing_manager', 'vault_manager', 'container_manager',
     'cell_link_manager', 'auth_manager', 'setup_manager', 'caddy_manager',
     'ddns_manager', 'service_store_manager', 'connectivity_manager',
+    'service_registry', 'service_composer', 'account_manager',
     'firewall_manager', 'EventType',
     'DATA_DIR', 'CONFIG_DIR',
 ]

api/network_manager.py

@@ -18,11 +18,13 @@ logger = logging.getLogger(__name__)
 class NetworkManager(BaseServiceManager):
     """Manages network services (DNS, DHCP, NTP)"""
     
-    def __init__(self, data_dir: str = '/app/data', config_dir: str = '/app/config'):
+    def __init__(self, data_dir: str = '/app/data', config_dir: str = '/app/config',
+                 service_registry=None):
         super().__init__('network', data_dir, config_dir)
         self.dns_zones_dir = os.path.join(data_dir, 'dns')
         self.dhcp_leases_file = os.path.join(data_dir, 'dhcp', 'leases')
-        
+        self._service_registry = service_registry
+
         # Ensure directories exist
         self.safe_makedirs(self.dns_zones_dir)
         self.safe_makedirs(os.path.dirname(self.dhcp_leases_file))
@@ -201,7 +203,7 @@ class NetworkManager(BaseServiceManager):
         # domain (e.g. primary_domain='pic.ngo', effective_domain='pic2.pic.ngo'),
         # bootstrap service records like 'api', 'calendar' etc. would pollute the
         # zone display and shadow the public domain.  Remove them.
-        _stale = {'api', 'webui', 'calendar', 'files', 'mail', 'webmail', 'webdav'}
+        _stale = {'api', 'webui'} | set(self._get_service_subdomains())
         if effective_domain.endswith('.' + primary_domain):
             existing = self._load_dns_records(primary_domain)
             cleaned = [r for r in existing if r.get('name', '') not in _stale]
@@ -249,6 +251,25 @@ class NetworkManager(BaseServiceManager):
             pass
         return '10.0.0.1'
 
+    _SUBDOMAIN_RE = re.compile(r'^[a-z][a-z0-9-]{0,30}$')
+
+    def _get_service_subdomains(self) -> List[str]:
+        """Return all service subdomains from the registry, or a hardcoded fallback."""
+        registry = getattr(self, "_service_registry", None)
+        if registry is not None:
+            try:
+                subs: List[str] = []
+                for route in registry.get_caddy_routes():
+                    for sub in [route['subdomain']] + list(route.get('extra_subdomains') or []):
+                        if self._SUBDOMAIN_RE.match(sub):
+                            subs.append(sub)
+                        else:
+                            logger.warning('_get_service_subdomains: skipping invalid subdomain %r', sub)
+                return subs
+            except Exception as exc:
+                logger.warning('_get_service_subdomains: registry error: %s', exc)
+        return ['calendar', 'files', 'mail', 'webmail', 'webdav']
+
     def _build_dns_records(self, cell_name: str, ip_range: str) -> List[Dict]:
         """Build the standard set of DNS A records.
 
@@ -258,16 +279,14 @@ class NetworkManager(BaseServiceManager):
         routes requests to the correct backend by Host header.
         """
         wg_ip = self._get_wg_server_ip()
-        return [
-            {'name': cell_name,  'type': 'A', 'value': wg_ip},
-            {'name': 'api',      'type': 'A', 'value': wg_ip},
-            {'name': 'webui',    'type': 'A', 'value': wg_ip},
-            {'name': 'calendar', 'type': 'A', 'value': wg_ip},
-            {'name': 'files',    'type': 'A', 'value': wg_ip},
-            {'name': 'mail',     'type': 'A', 'value': wg_ip},
-            {'name': 'webmail',  'type': 'A', 'value': wg_ip},
-            {'name': 'webdav',   'type': 'A', 'value': wg_ip},
+        records = [
+            {'name': cell_name, 'type': 'A', 'value': wg_ip},
+            {'name': 'api',     'type': 'A', 'value': wg_ip},
+            {'name': 'webui',   'type': 'A', 'value': wg_ip},
         ]
+        for sub in self._get_service_subdomains():
+            records.append({'name': sub, 'type': 'A', 'value': wg_ip})
+        return records
 
     def get_dns_records(self, zone: str = 'cell') -> List[Dict]:
         """Get all DNS records across all zones"""
@@ -595,7 +614,7 @@ class NetworkManager(BaseServiceManager):
         if not new_name:
             return {'restarted': restarted, 'warnings': warnings}
         # Exclude service names, wildcard, and apex from cell-hostname detection.
-        _service_names = {'api', 'webui', 'calendar', 'files', 'mail', 'webmail', 'webdav'}
+        _service_names = {'api', 'webui'} | set(self._get_service_subdomains())
         _reserved = _service_names | {'@', '*'}
         changed = False
         try:

api/routes/config.py

@@ -784,8 +784,8 @@ def apply_pending_config():
 @bp.route('/api/config/backup', methods=['POST'])
 def create_config_backup():
     try:
-        from app import config_manager, service_bus, EventType
-        backup_id = config_manager.backup_config()
+        from app import config_manager, service_bus, service_registry, EventType
+        backup_id = config_manager.backup_config(service_registry=service_registry)
         service_bus.publish_event(EventType.BACKUP_CREATED, 'api', {
             'backup_id': backup_id,
             'timestamp': datetime.utcnow().isoformat()
@@ -809,9 +809,14 @@ def list_config_backups():
 @bp.route('/api/config/restore/<backup_id>', methods=['POST'])
 def restore_config(backup_id):
     try:
-        from app import config_manager, service_bus, EventType
+        from app import config_manager, service_bus, service_registry, EventType
         data = request.get_json(silent=True) or {}
-        success = config_manager.restore_config(backup_id, services=data.get('services'))
+        services = data.get('services')
+        success = config_manager.restore_config(
+            backup_id,
+            services=services,
+            service_registry=service_registry if services is None else None,
+        )
         if success:
             service_bus.publish_event(EventType.RESTORE_COMPLETED, 'api', {
                 'backup_id': backup_id,

api/routes/peers.py

@@ -125,6 +125,17 @@ def add_peer():
                 return jsonify({"error": f"Peer {peer_name} already exists"}), 400
             peer_added_to_registry = True
 
+            # Store credentials only after the peer is committed — avoids orphaned
+            # credential entries if peer_registry.add_peer rejects a duplicate name.
+            try:
+                from app import account_manager
+                _svc_names = {'email', 'calendar', 'files'}
+                for svc in provisioned:
+                    if svc in _svc_names:
+                        account_manager.store_credentials(svc, peer_name, {'password': password})
+            except Exception as _am_err:
+                logger.warning(f"Peer {peer_name}: credential storage failed (non-fatal): {_am_err}")
+
             firewall_manager.apply_peer_rules(peer_info['ip'], peer_info,
                                               wg_subnet=_wg_subnet, cell_subnets=_cell_subnets)
             firewall_applied = True
@@ -320,12 +331,46 @@ def remove_peer(peer_name):
                 _cleanup()
             except Exception:
                 pass
+        try:
+            from app import account_manager
+            account_manager.deprovision_peer(peer_name)
+        except Exception as _am_err:
+            logger.warning(f"Peer {peer_name}: account_manager cleanup failed (non-fatal): {_am_err}")
         return jsonify({"message": f"Peer {peer_name} removed successfully"})
     except Exception as e:
         logger.error(f"Error removing peer: {e}")
         return jsonify({"error": str(e)}), 500
 
 
+@bp.route('/api/peers/<peer_name>/service-credentials', methods=['GET'])
+def get_peer_service_credentials(peer_name: str):
+    """Return service credentials for a peer across all provisioned services (admin only).
+
+    Returns filled peer_config_template values for each service the peer is provisioned on.
+    Intended for an admin to view or copy credentials to share with the peer during
+    device setup. The global enforce_auth gate already restricts this to admin sessions.
+
+    Phase 2 note: a peer-self-service variant should live at /api/peer/service-credentials
+    (no path arg) and restrict to session['username'] to prevent cross-peer enumeration.
+    """
+    try:
+        from app import peer_registry, account_manager, service_registry, config_manager
+        peer = peer_registry.get_peer(peer_name)
+        if not peer:
+            return jsonify({'error': f'Peer {peer_name!r} not found'}), 404
+        raw_creds = account_manager.get_all_credentials(peer_name)
+        identity = config_manager.get_identity()
+        domain = config_manager.get_effective_domain() or identity.get('domain', '')
+        result = {}
+        for service_id, cred in raw_creds.items():
+            svc_info = service_registry.get_peer_service_info(service_id, peer_name, domain, cred)
+            result[service_id] = svc_info if svc_info is not None else cred
+        return jsonify({'peer': peer_name, 'services': result})
+    except Exception as e:
+        logger.error('get_peer_service_credentials(%s): %s', peer_name, e)
+        return jsonify({'error': str(e)}), 500
+
+
 @bp.route('/api/peers/register', methods=['POST'])
 def register_peer():
     try:

api/routes/services.py

@@ -6,6 +6,174 @@ from flask import Blueprint, request, jsonify
 logger = logging.getLogger('picell')
 bp = Blueprint('services', __name__)
 
+@bp.route('/api/services/catalog', methods=['GET'])
+def get_services_catalog():
+    """
+    Return all services (builtins + installed store packages) with merged config.
+    Used by the frontend to build navigation and service pages dynamically.
+    """
+    try:
+        from app import service_registry
+        return jsonify({'services': service_registry.list_all()})
+    except Exception as e:
+        logger.error('get_services_catalog: %s', e)
+        return jsonify({'error': str(e)}), 500
+
+
+@bp.route('/api/services/catalog/<service_id>', methods=['GET'])
+def get_service_catalog_entry(service_id: str):
+    """Return a single service manifest+config, or 404 if unknown."""
+    try:
+        from app import service_registry
+        svc = service_registry.get(service_id)
+        if svc is None:
+            return jsonify({'error': f'Service {service_id!r} not found'}), 404
+        return jsonify(svc)
+    except Exception as e:
+        logger.error('get_service_catalog_entry(%s): %s', service_id, e)
+        return jsonify({'error': str(e)}), 500
+
+
+@bp.route('/api/services/catalog/<service_id>/status', methods=['GET'])
+def get_service_container_status(service_id: str):
+    """
+    Return container status for a service.
+    Builtins query the main compose stack; store services query their own compose project.
+    """
+    try:
+        from app import service_registry, service_composer
+        svc = service_registry.get(service_id)
+        if svc is None:
+            return jsonify({'error': f'Service {service_id!r} not found'}), 404
+        result = service_composer.status_service(service_id, svc)
+        return jsonify(result)
+    except ValueError as e:
+        return jsonify({'error': str(e)}), 400
+    except Exception as e:
+        logger.error('get_service_container_status(%s): %s', service_id, e)
+        return jsonify({'error': str(e)}), 500
+
+
+@bp.route('/api/services/catalog/<service_id>/restart', methods=['POST'])
+def restart_service_containers(service_id: str):
+    """
+    Restart containers for a service.
+    Builtins restart via the main compose stack; store services via their own compose project.
+    """
+    try:
+        from app import service_registry, service_composer
+        svc = service_registry.get(service_id)
+        if svc is None:
+            return jsonify({'error': f'Service {service_id!r} not found'}), 404
+        result = service_composer.restart_service(service_id, svc)
+        if result['ok']:
+            return jsonify({'message': f'Service {service_id!r} restarted', **result})
+        return jsonify({'error': result.get('stderr') or result.get('error', 'restart failed')}), 500
+    except ValueError as e:
+        return jsonify({'error': str(e)}), 400
+    except Exception as e:
+        logger.error('restart_service_containers(%s): %s', service_id, e)
+        return jsonify({'error': str(e)}), 500
+
+
+@bp.route('/api/services/catalog/<service_id>/reconfigure', methods=['POST'])
+def reconfigure_service(service_id: str):
+    """
+    Re-apply the stored compose file for a store service (rolling `up -d`).
+    The compose template must already exist on disk from the original install —
+    accepting templates from the request body is deliberately not supported
+    (arbitrary compose files can mount host paths or request privileged mode).
+    """
+    try:
+        from app import service_registry, service_composer
+        svc = service_registry.get(service_id)
+        if svc is None:
+            return jsonify({'error': f'Service {service_id!r} not found'}), 404
+        if svc.get('kind') == 'builtin':
+            return jsonify({'error': 'Builtins are reconfigured via their settings routes'}), 400
+        if not service_composer.has_compose_file(service_id):
+            return jsonify({'error': f'No compose file for {service_id!r} — install it first'}), 400
+
+        result = service_composer.up(service_id)
+        if result['ok']:
+            return jsonify({'message': f'Service {service_id!r} reconfigured', **result})
+        return jsonify({'error': result.get('stderr') or result.get('error', 'reconfigure failed')}), 500
+    except ValueError as e:
+        return jsonify({'error': str(e)}), 400
+    except Exception as e:
+        logger.error('reconfigure_service(%s): %s', service_id, e)
+        return jsonify({'error': str(e)}), 500
+
+
+@bp.route('/api/services/catalog/<service_id>/accounts', methods=['GET'])
+def list_service_accounts(service_id: str):
+    """Return peer usernames provisioned on a service."""
+    try:
+        from app import account_manager
+        accounts = account_manager.list_accounts(service_id)
+        return jsonify({'service_id': service_id, 'accounts': accounts})
+    except Exception as e:
+        logger.error('list_service_accounts(%s): %s', service_id, e)
+        return jsonify({'error': str(e)}), 500
+
+
+@bp.route('/api/services/catalog/<service_id>/accounts', methods=['POST'])
+def provision_service_account(service_id: str):
+    """Provision a peer account on a service. Generates a password if none is given.
+
+    The generated or provided password is NOT echoed in this response — retrieve it
+    separately via GET /api/services/catalog/<id>/accounts/<username>/credentials.
+    This keeps passwords out of HTTP logs and browser network panels.
+    """
+    try:
+        from app import account_manager
+        data = request.get_json(silent=True) or {}
+        peer_username = data.get('username')
+        if not peer_username:
+            return jsonify({'error': 'username is required'}), 400
+        account_manager.provision(service_id, peer_username,
+                                  password=data.get('password'))
+        return jsonify({'service_id': service_id, 'username': peer_username,
+                        'provisioned': True}), 201
+    except ValueError as e:
+        return jsonify({'error': str(e)}), 400
+    except RuntimeError as e:
+        return jsonify({'error': str(e)}), 500
+    except Exception as e:
+        logger.error('provision_service_account(%s): %s', service_id, e)
+        return jsonify({'error': str(e)}), 500
+
+
+@bp.route('/api/services/catalog/<service_id>/accounts/<username>', methods=['DELETE'])
+def deprovision_service_account(service_id: str, username: str):
+    """Remove a peer's account from a service."""
+    try:
+        from app import account_manager
+        ok = account_manager.deprovision(service_id, username)
+        if ok:
+            return jsonify({'message': f'{username!r} deprovisioned from {service_id!r}'})
+        return jsonify({'error': 'deprovision failed'}), 500
+    except ValueError as e:
+        return jsonify({'error': str(e)}), 400
+    except Exception as e:
+        logger.error('deprovision_service_account(%s, %s): %s', service_id, username, e)
+        return jsonify({'error': str(e)}), 500
+
+
+@bp.route('/api/services/catalog/<service_id>/accounts/<username>/credentials', methods=['GET'])
+def get_service_account_credentials(service_id: str, username: str):
+    """Return stored credentials for a peer on a service."""
+    try:
+        from app import account_manager
+        creds = account_manager.get_credentials(service_id, username)
+        if creds is None:
+            return jsonify({'error': f'{username!r} not provisioned on {service_id!r}'}), 404
+        return jsonify({'service_id': service_id, 'username': username, **creds})
+    except Exception as e:
+        logger.error('get_service_account_credentials(%s, %s): %s', service_id, username, e)
+        return jsonify({'error': str(e)}), 500
+
+
 @bp.route('/api/services/bus/status', methods=['GET'])
 def get_service_bus_status():
     try:

api/service_registry.py

@@ -0,0 +1,218 @@
+"""
+ServiceRegistry — single source of truth for all PIC services.
+
+Merges three layers:
+  1. Manifest defaults (config_schema.*.default)
+  2. Admin-saved config from ConfigManager (cell_config.json)
+  3. Runtime state from installed store records
+
+All consumers (CaddyManager, backup, peer services endpoint) read from here
+rather than hardcoding service names or subdomains.
+"""
+
+import json
+import logging
+import os
+import re
+from typing import Dict, List, Optional
+from urllib.parse import quote as _urlquote
+
+logger = logging.getLogger('picell')
+
+# Built-ins are baked into the container image at build time.
+# Do not bind-mount this path read-write in docker-compose.
+_BUILTINS_DIR = os.path.join(os.path.dirname(__file__), 'services', 'builtins')
+
+_SUBDOMAIN_RE = re.compile(r'^[a-z][a-z0-9-]{0,30}$')
+_BACKEND_RE   = re.compile(r'^[A-Za-z0-9._-]+:\d{1,5}$')
+_RESERVED_SUBS = frozenset({'api', 'webui', 'admin', 'www', 'ns1', 'ns2', 'git', 'registry', 'install'})
+
+
+class ServiceRegistry:
+
+    def __init__(self, config_manager):
+        self._cm = config_manager
+
+    # ── Manifest loading ──────────────────────────────────────────────────
+
+    def _load_manifest(self, path: str) -> Optional[Dict]:
+        try:
+            with open(path) as f:
+                return json.load(f)
+        except Exception as e:
+            logger.warning('ServiceRegistry: failed to load manifest %s: %s', path, e)
+            return None
+
+    def _builtin_ids(self) -> List[str]:
+        if not os.path.isdir(_BUILTINS_DIR):
+            return []
+        return sorted(
+            d for d in os.listdir(_BUILTINS_DIR)
+            if os.path.isfile(os.path.join(_BUILTINS_DIR, d, 'manifest.json'))
+        )
+
+    def _builtin_manifest(self, service_id: str) -> Optional[Dict]:
+        return self._load_manifest(
+            os.path.join(_BUILTINS_DIR, service_id, 'manifest.json')
+        )
+
+    # ── Config merging ────────────────────────────────────────────────────
+
+    _TYPE_COERCIONS = {'integer': int, 'string': str, 'boolean': bool}
+
+    def _merged_config(self, manifest: Dict) -> Dict:
+        """Return manifest defaults overridden by admin-saved values, type-coerced."""
+        svc_id = manifest.get('id', '')
+        saved = self._cm.configs.get(svc_id, {})
+        schema = manifest.get('config_schema') or {}
+        merged = {k: v['default'] for k, v in schema.items() if 'default' in v}
+        for k, spec in schema.items():
+            if k not in saved:
+                continue
+            raw = saved[k]
+            coerce = self._TYPE_COERCIONS.get(spec.get('type', ''))
+            if coerce is not None:
+                try:
+                    raw = coerce(raw)
+                except (TypeError, ValueError):
+                    raw = merged.get(k, raw)
+            merged[k] = raw
+        return merged
+
+    # ── Public API ────────────────────────────────────────────────────────
+
+    def get(self, service_id: str) -> Optional[Dict]:
+        """Return manifest + merged config for one service, or None if unknown."""
+        manifest = self._builtin_manifest(service_id)
+        if manifest is None:
+            record = self._cm.get_installed_services().get(service_id)
+            if record:
+                manifest = record.get('manifest')
+        if not manifest:
+            return None
+        return {**manifest, 'config': self._merged_config(manifest)}
+
+    def list_all(self) -> List[Dict]:
+        """
+        Return all services — builtins first, then installed store services —
+        each with merged config attached as the 'config' key.
+        """
+        results: List[Dict] = []
+        seen: set = set()
+
+        for svc_id in self._builtin_ids():
+            manifest = self._builtin_manifest(svc_id)
+            if manifest:
+                results.append({**manifest, 'config': self._merged_config(manifest)})
+                seen.add(svc_id)
+
+        for svc_id, record in self._cm.get_installed_services().items():
+            if svc_id in seen:
+                continue
+            manifest = record.get('manifest') or {}
+            if manifest.get('id'):
+                results.append({**manifest, 'config': self._merged_config(manifest)})
+
+        return results
+
+    def get_caddy_routes(self) -> List[Dict]:
+        """
+        Return routing info for all services that have a subdomain.
+        Used by CaddyManager to build service blocks without hardcoding.
+
+        Values are validated here as a chokepoint so Caddyfile/DNS builders
+        can safely interpolate them regardless of how manifests reached disk.
+        """
+        routes = []
+        for svc in self.list_all():
+            caps = svc.get('capabilities') or {}
+            if not caps.get('has_subdomain'):
+                continue
+            sub = svc.get('subdomain', '')
+            bknd = svc.get('backend', '')
+            if not sub or not bknd:
+                continue
+            svc_id = svc.get('id', '?')
+            if not _SUBDOMAIN_RE.match(sub) or sub in _RESERVED_SUBS:
+                logger.warning('ServiceRegistry: skipping %s — invalid/reserved subdomain %r', svc_id, sub)
+                continue
+            if not _BACKEND_RE.match(bknd):
+                logger.warning('ServiceRegistry: skipping %s — invalid backend %r', svc_id, bknd)
+                continue
+            extra_subs = [
+                s for s in (svc.get('extra_subdomains') or [])
+                if isinstance(s, str) and _SUBDOMAIN_RE.match(s) and s not in _RESERVED_SUBS
+            ]
+            extra_backends = {
+                k: v for k, v in (svc.get('extra_backends') or {}).items()
+                if (isinstance(k, str) and _SUBDOMAIN_RE.match(k) and k not in _RESERVED_SUBS
+                    and isinstance(v, str) and _BACKEND_RE.match(v))
+            }
+            routes.append({
+                'service_id': svc_id,
+                'subdomain': sub,
+                'backend': bknd,
+                'extra_subdomains': extra_subs,
+                'extra_backends': extra_backends,
+            })
+        return routes
+
+    def get_backup_plan(self) -> List[Dict]:
+        """
+        Return backup declarations for all services that have storage.
+        Used by the backup system instead of hardcoded file lists.
+
+        Each entry:
+          service_id   — service identifier
+          volumes      — list of {container, path, name} for docker-exec streaming
+          config_paths — host-relative paths copied directly (config files)
+        """
+        plan = []
+        for svc in self.list_all():
+            caps = svc.get('capabilities') or {}
+            if not caps.get('has_storage'):
+                continue
+            backup = svc.get('backup') or {}
+            volumes = backup.get('volumes') or []
+            config_paths = backup.get('config_paths') or []
+            if not volumes and not config_paths:
+                continue
+            plan.append({
+                'service_id': svc['id'],
+                'volumes': volumes,
+                'config_paths': config_paths,
+            })
+        return plan
+
+    def get_peer_service_info(self, service_id: str, peer_username: str,
+                              domain: str, credentials: Dict) -> Optional[Dict]:
+        """
+        Fill peer_config_template for one service+peer combination.
+        credentials: dict of {field_name: value} for that peer+service.
+        Returns None if service unknown or has no peer template.
+        """
+        svc = self.get(service_id)
+        if not svc:
+            return None
+        template = svc.get('peer_config_template')
+        if not template:
+            return None
+
+        # URL-safe peer username (safe='') — prevents path traversal in CalDAV/WebDAV URLs
+        safe_username = _urlquote(peer_username, safe='')
+
+        result = {}
+        for key, raw in template.items():
+            val = raw
+            val = val.replace('{domain}', domain)
+            val = val.replace('{peer.username}', safe_username)
+            for field, cred_val in credentials.items():
+                val = val.replace(
+                    '{peer.service_credentials.' + service_id + '.' + field + '}',
+                    str(cred_val) if cred_val is not None else '',
+                )
+            cfg = svc.get('config') or {}
+            for cfg_key, cfg_val in cfg.items():
+                val = val.replace('{config.' + cfg_key + '}', str(cfg_val) if cfg_val is not None else '')
+            result[key] = val
+        return result

api/service_store_manager.py

@@ -51,6 +51,8 @@ RESERVED_SUBDOMAINS = frozenset([
     'git', 'registry', 'install',
 ])
 ENV_VALUE_RE = re.compile(r'^[A-Za-z0-9._@:/+\-= ]*$')
+SUBDOMAIN_RE  = re.compile(r'^[a-z][a-z0-9-]{0,30}$')
+BACKEND_RE    = re.compile(r'^[A-Za-z0-9._-]+:\d{1,5}$')
 
 
 # ---------------------------------------------------------------------------
@@ -141,19 +143,55 @@ class ServiceStoreManager(BaseServiceManager):
                     f'iptables_rules[].proto must be tcp or udp, got: {proto}'
                 )
 
-        # Caddy route subdomain
+        # Legacy caddy_route dict subdomain (for store manifests using the old format)
         caddy_route = m.get('caddy_route') or {}
         if isinstance(caddy_route, dict):
-            subdomain = caddy_route.get('subdomain', '')
+            legacy_sub = caddy_route.get('subdomain', '')
         else:
-            subdomain = ''
-        if subdomain:
-            if subdomain in RESERVED_SUBDOMAINS:
-                errors.append(f'caddy_route.subdomain is reserved: {subdomain}')
-            elif not re.match(r'^[a-z][a-z0-9-]{0,30}$', subdomain):
+            legacy_sub = ''
+        if legacy_sub:
+            if legacy_sub in RESERVED_SUBDOMAINS:
+                errors.append(f'caddy_route.subdomain is reserved: {legacy_sub}')
+            elif not SUBDOMAIN_RE.match(legacy_sub):
                 errors.append(
                     f'caddy_route.subdomain must match ^[a-z][a-z0-9-]{{0,30}}$, '
-                    f'got: {subdomain}'
+                    f'got: {legacy_sub}'
+                )
+
+        # Top-level subdomain + backend (consumed by ServiceRegistry.get_caddy_routes)
+        subdomain = m.get('subdomain', '')
+        if subdomain:
+            if subdomain in RESERVED_SUBDOMAINS:
+                errors.append(f'subdomain is reserved: {subdomain}')
+            elif not SUBDOMAIN_RE.match(subdomain):
+                errors.append(
+                    f'subdomain must match ^[a-z][a-z0-9-]{{0,30}}$, got: {subdomain}'
+                )
+
+        backend = m.get('backend', '')
+        if backend and not BACKEND_RE.match(backend):
+            errors.append(f'backend must be host:port (e.g. cell-foo:8080), got: {backend}')
+
+        for sub in m.get('extra_subdomains') or []:
+            if not isinstance(sub, str):
+                errors.append('extra_subdomains entries must be strings')
+            elif sub in RESERVED_SUBDOMAINS:
+                errors.append(f'extra_subdomains entry is reserved: {sub}')
+            elif not SUBDOMAIN_RE.match(sub):
+                errors.append(
+                    f'extra_subdomains entry must match ^[a-z][a-z0-9-]{{0,30}}$, got: {sub}'
+                )
+
+        for sub, bknd in (m.get('extra_backends') or {}).items():
+            if not isinstance(sub, str) or not SUBDOMAIN_RE.match(sub):
+                errors.append(
+                    f'extra_backends key must match ^[a-z][a-z0-9-]{{0,30}}$, got: {sub!r}'
+                )
+            elif sub in RESERVED_SUBDOMAINS:
+                errors.append(f'extra_backends key is reserved: {sub}')
+            if not isinstance(bknd, str) or not BACKEND_RE.match(bknd):
+                errors.append(
+                    f'extra_backends[{sub!r}] value must be host:port, got: {bknd!r}'
                 )
 
         # Env value safety