roof
f7bb2cc962
Unit Tests / test (push) Successful in 11m25s
Two manifest validation bugs blocked all store service installs: 1. service_store_manager.RESERVED_SUBDOMAINS included 'mail', which prevented the email service from using its required subdomain. Removed mail/calendar/files/webmail — they belong to official PIC store services and must be claimable by them. 2. manifest_validator required @sha256 digest pins on ALL images, including first-party git.pic.ngo/roof/* images that the PIC team builds and controls. service_store_manager._validate_manifest already only warned for first-party images; the secondary validator was stricter than intended, causing a hard reject on :latest tags. Aligned to warn-not-reject for first-party; malformed digests (when provided) are still a hard error. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
439 lines
18 KiB
Python
439 lines
18 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Service Store Manager for Personal Internet Cell.
|
|
|
|
Manages installation, removal, and lifecycle of third-party services from the
|
|
PIC service store index. Each installed service runs as a Docker container
|
|
declared in a compose override file and has:
|
|
- An allocated IP in the service pool (172.20.0.20–254 by default)
|
|
- Optional iptables FORWARD rules declared in its manifest
|
|
- Optional Caddy reverse-proxy route declared in its manifest
|
|
"""
|
|
|
|
import logging
|
|
import os
|
|
import re
|
|
import threading
|
|
from datetime import datetime
|
|
from typing import Any, Dict, List, Optional, Tuple
|
|
|
|
import json
|
|
|
|
import requests
|
|
|
|
from base_service_manager import BaseServiceManager
|
|
from manifest_validator import validate_manifest, validate_provision_hook
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Constants
|
|
# ---------------------------------------------------------------------------
|
|
|
|
INDEX_URL_DEFAULT = (
|
|
'https://git.pic.ngo/roof/pic-services/raw/branch/main/index.json'
|
|
)
|
|
MANIFEST_URL_TPL = (
|
|
'https://git.pic.ngo/roof/pic-services/raw/branch/main/services/{id}/manifest.json'
|
|
)
|
|
TEMPLATE_URL_TPL = (
|
|
'https://git.pic.ngo/roof/pic-services/raw/branch/main/services/{id}/compose-template.yml'
|
|
)
|
|
|
|
IMAGE_ALLOWLIST_RE = re.compile(
|
|
r'^git\.pic\.ngo/roof/[a-z0-9._/-]+(:[a-zA-Z0-9._-]+)?(@sha256:[a-f0-9]{64})?$'
|
|
)
|
|
|
|
# Images from well-known vendors that pre-date digest pinning in PIC.
|
|
# These are allowed to ship without a @sha256 digest; all others require one
|
|
# or must come from git.pic.ngo/roof/*.
|
|
TRUSTED_IMAGES_NO_DIGEST = frozenset({
|
|
'mailserver/docker-mailserver',
|
|
'tomsquest/docker-radicale',
|
|
'bytemark/webdav',
|
|
'filegator/filegator',
|
|
'hardware/rainloop',
|
|
})
|
|
FORBIDDEN_MOUNTS = frozenset([
|
|
'/', '/etc', '/var', '/proc', '/sys', '/dev', '/app', '/run', '/boot',
|
|
])
|
|
RESERVED_SUBDOMAINS = frozenset([
|
|
'api', 'webui', 'admin', 'www', 'ns1', 'ns2',
|
|
'git', 'registry', 'install',
|
|
# mail, calendar, files, webmail are intentionally absent:
|
|
# they are claimed by official PIC store services.
|
|
])
|
|
ENV_VALUE_RE = re.compile(r'^[A-Za-z0-9._@:/+\-= ]*$')
|
|
SUBDOMAIN_RE = re.compile(r'^[a-z][a-z0-9-]{0,30}$')
|
|
BACKEND_RE = re.compile(r'^[A-Za-z0-9._-]+:\d{1,5}$')
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ServiceStoreManager
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class ServiceStoreManager(BaseServiceManager):
|
|
"""Manages service store: install, remove, and list available/installed services."""
|
|
|
|
def __init__(self, config_manager, caddy_manager, container_manager,
|
|
data_dir: str = '', config_dir: str = '',
|
|
service_composer=None, egress_manager=None):
|
|
super().__init__('service_store', data_dir, config_dir)
|
|
self.config_manager = config_manager
|
|
self.caddy_manager = caddy_manager
|
|
self.container_manager = container_manager
|
|
self.service_composer = service_composer
|
|
self.egress_manager = egress_manager
|
|
self.compose_override = os.environ.get(
|
|
'COMPOSE_SERVICES_PATH', '/app/docker-compose.services.yml'
|
|
)
|
|
self.index_url = os.environ.get('PIC_STORE_INDEX_URL', INDEX_URL_DEFAULT)
|
|
self._lock = threading.Lock()
|
|
self._index_cache: Optional[list] = None
|
|
self._index_cache_time: float = 0
|
|
self._cache_ttl: int = 300 # 5 min
|
|
|
|
# ── BaseServiceManager required ───────────────────────────────────────
|
|
|
|
def get_status(self) -> Dict[str, Any]:
|
|
installed = self.config_manager.get_installed_services()
|
|
return {
|
|
'service': self.service_name,
|
|
'running': True,
|
|
'installed_count': len(installed),
|
|
}
|
|
|
|
def test_connectivity(self) -> Dict[str, Any]:
|
|
try:
|
|
resp = requests.get(self.index_url, timeout=5)
|
|
return {'success': resp.status_code == 200}
|
|
except Exception as e:
|
|
return {'success': False, 'error': str(e)}
|
|
|
|
# ── Manifest validation ───────────────────────────────────────────────
|
|
|
|
@staticmethod
|
|
def _validate_manifest(m: dict) -> Tuple[bool, List[str]]:
|
|
"""Validate a service manifest. Returns (ok, [errors])."""
|
|
errors: List[str] = []
|
|
|
|
# Required top-level fields
|
|
for field in ('id', 'name', 'version', 'author', 'image', 'container_name'):
|
|
if not m.get(field):
|
|
errors.append(f'Missing required field: {field}')
|
|
|
|
# Image allowlist
|
|
image = m.get('image', '')
|
|
if image and not IMAGE_ALLOWLIST_RE.match(image):
|
|
errors.append(
|
|
f'image must match git.pic.ngo/roof/* pattern, got: {image}'
|
|
)
|
|
elif image:
|
|
# Warn when a digest pin is absent so operators know exact-version
|
|
# tracking is not guaranteed. Images in TRUSTED_IMAGES_NO_DIGEST
|
|
# and images from our own git.pic.ngo/roof/* registry (which we
|
|
# build and tag) get warnings rather than hard errors; any other
|
|
# image that somehow passes the allowlist gets a hard error.
|
|
if '@sha256:' not in image:
|
|
image_base = image.split(':')[0].split('@')[0]
|
|
is_own_registry = image_base.startswith('git.pic.ngo/roof/')
|
|
if image_base in TRUSTED_IMAGES_NO_DIGEST or is_own_registry:
|
|
logger.warning('image %s has no digest pin', image)
|
|
else:
|
|
errors.append(
|
|
f'image {image!r} must include a @sha256:<digest> pin'
|
|
)
|
|
|
|
# Volume mount safety
|
|
for vol in m.get('volumes', []):
|
|
mount = vol.get('mount', '')
|
|
if mount in FORBIDDEN_MOUNTS:
|
|
errors.append(f'Forbidden volume mount: {mount}')
|
|
elif mount.startswith('/home/roof/pic'):
|
|
errors.append(f'Volume mount cannot be a prefix of /home/roof/pic: {mount}')
|
|
|
|
# iptables rules
|
|
for rule in m.get('iptables_rules', []):
|
|
if rule.get('type') != 'ACCEPT':
|
|
errors.append(
|
|
f'iptables_rules[].type must be ACCEPT, got: {rule.get("type")}'
|
|
)
|
|
if rule.get('dest_ip') != '${SERVICE_IP}':
|
|
errors.append(
|
|
f'iptables_rules[].dest_ip must be exactly ${{SERVICE_IP}}, '
|
|
f'got: {rule.get("dest_ip")}'
|
|
)
|
|
port = rule.get('dest_port')
|
|
if not isinstance(port, int) or not (1 <= port <= 65535):
|
|
errors.append(
|
|
f'iptables_rules[].dest_port must be an integer 1-65535, got: {port}'
|
|
)
|
|
proto = rule.get('proto', 'tcp')
|
|
if proto not in ('tcp', 'udp'):
|
|
errors.append(
|
|
f'iptables_rules[].proto must be tcp or udp, got: {proto}'
|
|
)
|
|
|
|
# Legacy caddy_route dict subdomain (for store manifests using the old format)
|
|
caddy_route = m.get('caddy_route') or {}
|
|
if isinstance(caddy_route, dict):
|
|
legacy_sub = caddy_route.get('subdomain', '')
|
|
else:
|
|
legacy_sub = ''
|
|
if legacy_sub:
|
|
if legacy_sub in RESERVED_SUBDOMAINS:
|
|
errors.append(f'caddy_route.subdomain is reserved: {legacy_sub}')
|
|
elif not SUBDOMAIN_RE.match(legacy_sub):
|
|
errors.append(
|
|
f'caddy_route.subdomain must match ^[a-z][a-z0-9-]{{0,30}}$, '
|
|
f'got: {legacy_sub}'
|
|
)
|
|
|
|
# Top-level subdomain + backend (consumed by ServiceRegistry.get_caddy_routes)
|
|
subdomain = m.get('subdomain', '')
|
|
if subdomain:
|
|
if subdomain in RESERVED_SUBDOMAINS:
|
|
errors.append(f'subdomain is reserved: {subdomain}')
|
|
elif not SUBDOMAIN_RE.match(subdomain):
|
|
errors.append(
|
|
f'subdomain must match ^[a-z][a-z0-9-]{{0,30}}$, got: {subdomain}'
|
|
)
|
|
|
|
backend = m.get('backend', '')
|
|
if backend and not BACKEND_RE.match(backend):
|
|
errors.append(f'backend must be host:port (e.g. cell-foo:8080), got: {backend}')
|
|
|
|
for sub in m.get('extra_subdomains') or []:
|
|
if not isinstance(sub, str):
|
|
errors.append('extra_subdomains entries must be strings')
|
|
elif sub in RESERVED_SUBDOMAINS:
|
|
errors.append(f'extra_subdomains entry is reserved: {sub}')
|
|
elif not SUBDOMAIN_RE.match(sub):
|
|
errors.append(
|
|
f'extra_subdomains entry must match ^[a-z][a-z0-9-]{{0,30}}$, got: {sub}'
|
|
)
|
|
|
|
for sub, bknd in (m.get('extra_backends') or {}).items():
|
|
if not isinstance(sub, str) or not SUBDOMAIN_RE.match(sub):
|
|
errors.append(
|
|
f'extra_backends key must match ^[a-z][a-z0-9-]{{0,30}}$, got: {sub!r}'
|
|
)
|
|
elif sub in RESERVED_SUBDOMAINS:
|
|
errors.append(f'extra_backends key is reserved: {sub}')
|
|
if not isinstance(bknd, str) or not BACKEND_RE.match(bknd):
|
|
errors.append(
|
|
f'extra_backends[{sub!r}] value must be host:port, got: {bknd!r}'
|
|
)
|
|
|
|
# Env value safety
|
|
for env_entry in m.get('env', []):
|
|
val = str(env_entry.get('value', ''))
|
|
if not ENV_VALUE_RE.match(val):
|
|
errors.append(
|
|
f'env[].value contains disallowed characters: {val!r}'
|
|
)
|
|
|
|
# Security layer: delegate to manifest_validator for cap_add, backend
|
|
# denylist, provision_hook, reserved container names, and kind guard.
|
|
ok, sec_errs = validate_manifest(m)
|
|
if not ok:
|
|
errors.extend(sec_errs)
|
|
|
|
return (len(errors) == 0, errors)
|
|
|
|
# ── Index / manifest fetching ─────────────────────────────────────────
|
|
|
|
def fetch_index(self) -> list:
|
|
"""Fetch and cache the service index."""
|
|
import time
|
|
_SIZE_LIMIT = 256 * 1024
|
|
now = time.time()
|
|
if self._index_cache is not None and (now - self._index_cache_time) < self._cache_ttl:
|
|
return self._index_cache
|
|
try:
|
|
resp = requests.get(self.index_url, timeout=10, stream=True)
|
|
resp.raise_for_status()
|
|
content = resp.raw.read(_SIZE_LIMIT + 1, decode_content=True)
|
|
if len(content) > _SIZE_LIMIT:
|
|
raise ValueError('Index response exceeds 256 KB limit')
|
|
data = json.loads(content)
|
|
self._index_cache = data if isinstance(data, list) else data.get('services', [])
|
|
self._index_cache_time = now
|
|
return self._index_cache
|
|
except Exception as e:
|
|
logger.warning(f'fetch_index failed: {e}')
|
|
return self._index_cache or []
|
|
|
|
def _fetch_manifest(self, service_id: str) -> dict:
|
|
"""Fetch a service manifest by ID."""
|
|
_SIZE_LIMIT = 256 * 1024
|
|
url = MANIFEST_URL_TPL.format(id=service_id)
|
|
resp = requests.get(url, timeout=10, stream=True)
|
|
resp.raise_for_status()
|
|
content = resp.raw.read(_SIZE_LIMIT + 1, decode_content=True)
|
|
if len(content) > _SIZE_LIMIT:
|
|
raise ValueError(
|
|
f'Manifest response for {service_id} exceeds 256 KB limit'
|
|
)
|
|
return json.loads(content)
|
|
|
|
def _fetch_template(self, service_id: str, manifest: dict) -> str:
|
|
"""Fetch the compose template for a service."""
|
|
_SIZE_LIMIT = 256 * 1024
|
|
url = TEMPLATE_URL_TPL.format(id=service_id)
|
|
resp = requests.get(url, timeout=10, stream=True)
|
|
resp.raise_for_status()
|
|
content = resp.raw.read(_SIZE_LIMIT + 1, decode_content=True)
|
|
if len(content) > _SIZE_LIMIT:
|
|
raise ValueError(f'Compose template for {service_id} exceeds 256 KB limit')
|
|
return content.decode('utf-8')
|
|
|
|
# ── Core operations ───────────────────────────────────────────────────
|
|
|
|
def install(self, service_id: str) -> dict:
|
|
"""Install a service from the store."""
|
|
with self._lock:
|
|
installed = self.config_manager.get_installed_services()
|
|
if service_id in installed:
|
|
return {'ok': True, 'already_installed': True}
|
|
|
|
# Fetch and validate manifest
|
|
try:
|
|
manifest = self._fetch_manifest(service_id)
|
|
except Exception as e:
|
|
return {'ok': False, 'error': f'Failed to fetch manifest: {e}'}
|
|
|
|
ok, errs = self._validate_manifest(manifest)
|
|
if not ok:
|
|
return {'ok': False, 'errors': errs}
|
|
|
|
ok2, errs2 = validate_manifest(manifest)
|
|
if not ok2:
|
|
return {'ok': False, 'errors': errs2}
|
|
|
|
# Dependency check
|
|
if self.service_composer is not None:
|
|
err = self.service_composer._resolve_requires(manifest, installed)
|
|
if err:
|
|
return {'ok': False, 'error': err}
|
|
|
|
# Fetch compose template
|
|
try:
|
|
template_content = self._fetch_template(service_id, manifest)
|
|
except Exception as e:
|
|
return {'ok': False, 'error': f'Failed to fetch compose template: {e}'}
|
|
|
|
# Write compose file and start containers (validation inside write_compose)
|
|
if self.service_composer is not None:
|
|
try:
|
|
result = self.service_composer.install(service_id, manifest, template_content)
|
|
except ValueError as e:
|
|
return {'ok': False, 'error': str(e)}
|
|
except Exception as e:
|
|
return {'ok': False, 'error': f'Failed to start service: {e}'}
|
|
if not result.get('ok'):
|
|
return {'ok': False, 'error': result.get('error') or result.get('stderr', 'docker up failed')}
|
|
|
|
# Persist minimal install record
|
|
record = {
|
|
'id': service_id,
|
|
'manifest': manifest,
|
|
'installed_at': datetime.utcnow().isoformat(),
|
|
}
|
|
self.config_manager.set_installed_service(service_id, record)
|
|
|
|
# Regenerate Caddy (registry now drives routes, no caddy_routes list needed)
|
|
try:
|
|
self.caddy_manager.regenerate_with_installed([])
|
|
except Exception as e:
|
|
logger.warning('install: caddy regenerate failed for %s (non-fatal): %s', service_id, e)
|
|
|
|
if self.egress_manager:
|
|
try:
|
|
self.egress_manager.apply_service(service_id)
|
|
except Exception as exc:
|
|
logger.warning('Egress apply failed for %s (non-fatal): %s', service_id, exc)
|
|
|
|
return {'ok': True}
|
|
|
|
def remove(self, service_id: str, purge_data: bool = False) -> dict:
|
|
"""Remove an installed service."""
|
|
with self._lock:
|
|
installed = self.config_manager.get_installed_services()
|
|
if service_id not in installed:
|
|
return {'ok': False, 'error': f'Service {service_id} is not installed'}
|
|
|
|
# Prevent removing a service that others depend on
|
|
if self.service_composer is not None:
|
|
dependents = self.service_composer._resolve_dependents(service_id, installed)
|
|
if dependents:
|
|
return {
|
|
'ok': False,
|
|
'error': f'Cannot remove {service_id}: required by {", ".join(sorted(dependents))}',
|
|
}
|
|
|
|
if self.egress_manager:
|
|
try:
|
|
self.egress_manager.clear_service(service_id)
|
|
except Exception as exc:
|
|
logger.warning('Egress clear failed for %s (non-fatal): %s', service_id, exc)
|
|
|
|
# Stop and remove containers (best-effort)
|
|
if self.service_composer is not None:
|
|
try:
|
|
self.service_composer.remove(service_id, purge_data=purge_data)
|
|
except Exception as e:
|
|
logger.warning('remove: composer.remove failed for %s (non-fatal): %s', service_id, e)
|
|
|
|
# Remove from config
|
|
self.config_manager.remove_installed_service(service_id)
|
|
|
|
# Regenerate Caddy
|
|
try:
|
|
self.caddy_manager.regenerate_with_installed([])
|
|
except Exception as e:
|
|
logger.warning('remove: caddy regenerate failed for %s (non-fatal): %s', service_id, e)
|
|
|
|
return {'ok': True}
|
|
|
|
def list_services(self) -> dict:
|
|
"""Return available (from index) and installed services."""
|
|
available = self.fetch_index()
|
|
installed = self.config_manager.get_installed_services()
|
|
return {'available': available, 'installed': installed}
|
|
|
|
def reapply_on_startup(self) -> None:
|
|
"""Re-apply firewall and Caddy rules for all installed services on startup."""
|
|
from firewall_manager import apply_service_rules
|
|
|
|
installed = self.config_manager.get_installed_services()
|
|
if not installed:
|
|
return
|
|
|
|
# Re-apply iptables rules
|
|
for svc_id, record in installed.items():
|
|
ip = record.get('service_ip', '')
|
|
rules = record.get('iptables_rules', [])
|
|
try:
|
|
apply_service_rules(svc_id, ip, rules)
|
|
except Exception as e:
|
|
logger.warning(f'reapply_on_startup: apply_service_rules({svc_id}) failed: {e}')
|
|
|
|
# Regenerate Caddyfile
|
|
try:
|
|
caddy_routes = [
|
|
r.get('caddy_route')
|
|
for r in installed.values()
|
|
if r.get('caddy_route')
|
|
]
|
|
self.caddy_manager.regenerate_with_installed(caddy_routes)
|
|
except Exception as e:
|
|
logger.warning(f'reapply_on_startup: caddy regenerate failed: {e}')
|
|
|
|
# Bring up per-service compose stacks
|
|
if self.service_composer is not None:
|
|
try:
|
|
self.service_composer.reapply_active_services()
|
|
except Exception as e:
|
|
logger.warning('reapply_on_startup: reapply_active_services failed: %s', e)
|