feat: add Steps 1-4 implementation files (AccountManager, ServiceComposer, builtins, tests)
Unit Tests / test (push) Successful in 11m24s

These files were created during Steps 1-4 of the services architecture but were
never staged: AccountManager (per-service credential provisioning), ServiceComposer
(docker-compose lifecycle), built-in service manifests for email/calendar/files,
and their test suites (158 tests). Also un-tracks .coverage binaries that were
accidentally committed.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-05-29 04:39:19 -04:00
parent dc7b316cbd
commit 2f5370bd98
12 changed files with 2443 additions and 0 deletions
BIN
View File
Binary file not shown.
+237
View File
@@ -0,0 +1,237 @@
"""
AccountManager — per-service credential provisioning for PIC peers.
Responsibilities:
- Dispatch account creation/deletion to each service's underlying manager
- Store per-peer per-service credentials securely (0o600 file)
- Provide credential retrieval for peer_config_template filling
- Bulk-deprovision a peer from all services on peer deletion
Credentials file format (data/peer_service_credentials.json):
{
"<service_id>": {
"<peer_username>": {"password": "..."}
}
}
Design note — plaintext passwords:
Credentials are stored in plaintext so the peer endpoint can return them to
the peer's device for one-time client configuration. The file is created with
0o600 so it is only readable by the process owner (same pattern used for
WireGuard keys and service_secrets.json).
"""
import json
import logging
import os
import secrets as _secrets_mod
import threading
from pathlib import Path
from typing import Dict, List, Optional
logger = logging.getLogger('picell')
_DISPATCH_PROVISION = {
'email_manager': '_provision_email',
'calendar_manager': '_provision_calendar',
'file_manager': '_provision_files',
}
_DISPATCH_DEPROVISION = {
'email_manager': '_deprovision_email',
'calendar_manager': '_deprovision_calendar',
'file_manager': '_deprovision_files',
}
class AccountManager:
def __init__(self, service_registry, data_dir: str, **managers):
"""
service_registry — ServiceRegistry instance
data_dir — host data directory (data/peer_service_credentials.json lives here)
**managers — named manager instances: email_manager=..., calendar_manager=...,
file_manager=...
"""
self._registry = service_registry
self._creds_path = Path(data_dir) / 'peer_service_credentials.json'
self._managers = managers
self._lock = threading.Lock()
# ── Credential storage (0o600) ────────────────────────────────────────
def _load_creds(self) -> Dict:
if not self._creds_path.exists():
return {}
try:
with open(self._creds_path) as f:
return json.load(f)
except (OSError, json.JSONDecodeError) as e:
logger.warning('AccountManager: failed to load credentials: %s', e)
return {}
def _save_creds(self, creds: Dict) -> None:
tmp = str(self._creds_path) + '.tmp'
with open(tmp, 'w', opener=lambda path, flags: os.open(path, flags, 0o600)) as f:
json.dump(creds, f, indent=2)
f.flush()
os.fsync(f.fileno())
os.replace(tmp, str(self._creds_path))
# ── Per-manager provision / deprovision ───────────────────────────────
def _provision_email(self, manager, svc: Dict, peer_username: str, password: str) -> bool:
domain = (svc.get('config') or {}).get('domain', '')
if not domain:
raise ValueError("Email service has no 'domain' configured")
return manager.create_email_user(peer_username, domain, password)
def _deprovision_email(self, manager, svc: Dict, peer_username: str) -> bool:
domain = (svc.get('config') or {}).get('domain', '')
return manager.delete_email_user(peer_username, domain)
@staticmethod
def _provision_calendar(manager, _svc: Dict, peer_username: str, password: str) -> bool:
return manager.create_calendar_user(peer_username, password)
@staticmethod
def _deprovision_calendar(manager, _svc: Dict, peer_username: str) -> bool:
return manager.delete_calendar_user(peer_username)
@staticmethod
def _provision_files(manager, _svc: Dict, peer_username: str, password: str) -> bool:
return manager.create_user(peer_username, password)
@staticmethod
def _deprovision_files(manager, _svc: Dict, peer_username: str) -> bool:
return manager.delete_user(peer_username)
# ── Service validation helper ─────────────────────────────────────────
def _resolve_service(self, service_id: str):
"""Return (svc, manager_name, manager) or raise ValueError."""
svc = self._registry.get(service_id)
if svc is None:
raise ValueError(f'Unknown service: {service_id!r}')
accounts_cfg = svc.get('accounts') or {}
manager_name = accounts_cfg.get('manager')
if not manager_name:
raise ValueError(f'Service {service_id!r} does not support accounts')
manager = self._managers.get(manager_name)
if manager is None:
raise ValueError(f'Manager {manager_name!r} is not registered with AccountManager')
return svc, manager_name, manager
# ── Public API ────────────────────────────────────────────────────────
def provision(self, service_id: str, peer_username: str,
password: str = None) -> Dict:
"""Create an account on the service for the peer; store and return credentials.
Raises ValueError if the service doesn't support accounts.
Raises RuntimeError if the underlying manager fails.
"""
svc, manager_name, manager = self._resolve_service(service_id)
if password is None:
password = _secrets_mod.token_urlsafe(16)
dispatch = _DISPATCH_PROVISION.get(manager_name)
if dispatch is None:
raise ValueError(f'No provision dispatch for manager: {manager_name!r}')
fn = getattr(self, dispatch)
ok = fn(manager, svc, peer_username, password)
if not ok:
raise RuntimeError(
f'Provision of {peer_username!r} on {service_id!r} returned False — '
'check underlying service manager logs'
)
cred = {'password': password}
with self._lock:
all_creds = self._load_creds()
all_creds.setdefault(service_id, {})[peer_username] = cred
self._save_creds(all_creds)
logger.info('AccountManager: provisioned %s on %s', peer_username, service_id)
return cred
def deprovision(self, service_id: str, peer_username: str) -> bool:
"""Delete the peer's account on the service and clear stored credentials."""
svc, manager_name, manager = self._resolve_service(service_id)
dispatch = _DISPATCH_DEPROVISION.get(manager_name)
if dispatch is None:
raise ValueError(f'No deprovision dispatch for manager: {manager_name!r}')
fn = getattr(self, dispatch)
ok = fn(manager, svc, peer_username)
with self._lock:
all_creds = self._load_creds()
svc_creds = all_creds.get(service_id, {})
if peer_username in svc_creds:
del svc_creds[peer_username]
if not svc_creds:
del all_creds[service_id]
self._save_creds(all_creds)
logger.info('AccountManager: deprovisioned %s from %s', peer_username, service_id)
return bool(ok)
def get_credentials(self, service_id: str, peer_username: str) -> Optional[Dict]:
"""Return stored credentials for peer+service, or None if not provisioned."""
with self._lock:
return self._load_creds().get(service_id, {}).get(peer_username)
def list_accounts(self, service_id: str) -> List[str]:
"""Return peer usernames provisioned on a service."""
with self._lock:
return list(self._load_creds().get(service_id, {}).keys())
def list_peer_services(self, peer_username: str) -> List[str]:
"""Return service IDs where this peer has a provisioned account."""
with self._lock:
creds = self._load_creds()
return [svc_id for svc_id, peers in creds.items() if peer_username in peers]
def is_provisioned(self, service_id: str, peer_username: str) -> bool:
return self.get_credentials(service_id, peer_username) is not None
def deprovision_peer(self, peer_username: str) -> Dict[str, bool]:
"""Remove a peer from every service they are provisioned on.
Called on peer deletion. Continues even if individual services fail.
Returns {service_id: success} for each service attempted.
"""
results: Dict[str, bool] = {}
for service_id in self.list_peer_services(peer_username):
try:
results[service_id] = self.deprovision(service_id, peer_username)
except Exception as e:
logger.warning('AccountManager: deprovision %s from %s failed: %s',
peer_username, service_id, e)
results[service_id] = False
return results
def get_all_credentials(self, peer_username: str) -> Dict[str, Dict]:
"""Return {service_id: {field: value}} for all services the peer is provisioned on."""
with self._lock:
creds = self._load_creds()
return {
svc_id: peers[peer_username]
for svc_id, peers in creds.items()
if peer_username in peers
}
def store_credentials(self, service_id: str, peer_username: str,
cred: Dict) -> None:
"""Directly store credentials without calling the underlying manager.
Used when a peer was provisioned through the legacy peers-POST route
so that their credentials become retrievable via AccountManager.
"""
with self._lock:
all_creds = self._load_creds()
all_creds.setdefault(service_id, {})[peer_username] = cred
self._save_creds(all_creds)
+310
View File
@@ -0,0 +1,310 @@
"""
ServiceComposer — docker-compose generation and container lifecycle for PIC services.
Responsibilities:
- Render compose-template.yml → per-service docker-compose.yml with PIC_* substitution
- Manage store-service container lifecycle (up / down / restart / status / reconfigure)
- Manage builtin-service restarts and status via the main compose stack
- Generate and persist PIC_SECRET_* variables in a dedicated secrets file
Template variable reference (for compose-template.yml authors):
${PIC_CFG_<KEY>} — value from manifest config_schema, uppercased
${PIC_SECRET_<NAME>} — auto-generated random secret, persisted across reconfigures
${PIC_DOMAIN} — effective domain (e.g. cell.pic.ngo)
${PIC_CELL_NAME} — cell name (e.g. mycell)
${PIC_SERVICE_ID} — service identifier (e.g. nextcloud)
"""
import json
import logging
import os
import re
import secrets as _secrets_lib
import shutil
import subprocess
import threading
from typing import Dict, List, Optional
logger = logging.getLogger('picell')
_SECRET_RE = re.compile(r'\$\{(PIC_SECRET_\w+)\}')
_SAFE_ID_RE = re.compile(r'^[a-z0-9][a-z0-9_-]{0,63}$')
class ServiceComposer:
def __init__(self, config_manager, data_dir: str):
self.cm = config_manager
self.data_dir = data_dir
self._services_dir = os.path.join(data_dir, 'services')
self._secrets_path = os.path.join(data_dir, 'service_secrets.json')
self._lock = threading.Lock()
# ── Path helpers ──────────────────────────────────────────────────────
@staticmethod
def _validate_service_id(service_id: str) -> None:
"""Raise ValueError if service_id could be used for path traversal."""
if not _SAFE_ID_RE.match(service_id):
raise ValueError(
f'Invalid service_id {service_id!r}: '
'must match ^[a-z0-9][a-z0-9_-]{{0,63}}$'
)
def _svc_dir(self, service_id: str) -> str:
self._validate_service_id(service_id)
candidate = os.path.join(self._services_dir, service_id)
# Paranoia: ensure the resolved path stays inside _services_dir
real_base = os.path.realpath(self._services_dir)
real_cand = os.path.realpath(candidate)
if not real_cand.startswith(real_base + os.sep) and real_cand != real_base:
raise ValueError(f'service_id {service_id!r} escapes services directory')
return candidate
def _compose_path(self, service_id: str) -> str:
return os.path.join(self._svc_dir(service_id), 'docker-compose.yml')
def has_compose_file(self, service_id: str) -> bool:
try:
return os.path.exists(self._compose_path(service_id))
except ValueError:
return False
# ── Secrets management ────────────────────────────────────────────────
def _load_secrets(self) -> Dict:
if not os.path.exists(self._secrets_path):
return {}
try:
with open(self._secrets_path) as f:
return json.load(f)
except (OSError, json.JSONDecodeError) as e:
logger.warning('ServiceComposer: failed to load secrets: %s', e)
return {}
def _save_secrets(self, secrets: Dict) -> None:
tmp = self._secrets_path + '.tmp'
# 0o600: readable only by the process owner — secrets must not be world-readable
with open(tmp, 'w',
opener=lambda path, flags: os.open(path, flags, 0o600)) as f:
json.dump(secrets, f, indent=2)
f.flush()
os.fsync(f.fileno())
os.replace(tmp, self._secrets_path)
def _get_or_create_secret(self, service_id: str, var_name: str) -> str:
with self._lock:
secrets = self._load_secrets()
svc_secrets = secrets.setdefault(service_id, {})
if var_name not in svc_secrets:
svc_secrets[var_name] = _secrets_lib.token_urlsafe(24)
self._save_secrets(secrets)
return svc_secrets[var_name]
def _clear_secrets(self, service_id: str) -> None:
with self._lock:
secrets = self._load_secrets()
if service_id in secrets:
del secrets[service_id]
self._save_secrets(secrets)
# ── Template rendering ────────────────────────────────────────────────
def render_template(self, service_id: str, manifest: Dict,
template_content: str) -> str:
"""
Substitute all PIC_* variables in a compose-template.yml string.
Returns the rendered compose YAML.
"""
schema = manifest.get('config_schema') or {}
saved = self.cm.configs.get(service_id, {})
config: Dict = {k: v['default'] for k, v in schema.items() if 'default' in v}
config.update({k: saved[k] for k in schema if k in saved})
identity = self.cm.get_identity()
domain = self.cm.get_effective_domain() or identity.get('domain', 'cell.local')
cell_name = identity.get('cell_name', 'mycell')
result = template_content
for key, value in config.items():
# Strip newlines/tabs to prevent YAML injection (a config string containing
# \n could inject new YAML keys into the compose file)
safe_val = str(value).replace('\n', '').replace('\r', '').replace('\t', ' ')
result = result.replace(f'${{PIC_CFG_{key.upper()}}}', safe_val)
result = result.replace('${PIC_DOMAIN}', domain)
result = result.replace('${PIC_CELL_NAME}', cell_name)
result = result.replace('${PIC_SERVICE_ID}', service_id)
# PIC_SECRET_* — generate on first use, reuse on reconfigure
for match in _SECRET_RE.finditer(template_content):
var_name = match.group(1)
secret = self._get_or_create_secret(service_id, var_name)
result = result.replace(f'${{{var_name}}}', secret)
return result
def write_compose(self, service_id: str, manifest: Dict,
template_content: str) -> str:
"""Render and atomically write the per-service compose file. Returns rendered content."""
os.makedirs(self._svc_dir(service_id), exist_ok=True)
content = self.render_template(service_id, manifest, template_content)
path = self._compose_path(service_id)
tmp = path + '.tmp'
with open(tmp, 'w') as f:
f.write(content)
f.flush()
os.fsync(f.fileno())
os.replace(tmp, path)
logger.info('ServiceComposer: wrote compose file for %s', service_id)
return content
# ── Subprocess helper ─────────────────────────────────────────────────
def _run(self, cmd: List[str], timeout: int = 120) -> Dict:
try:
r = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout)
if r.returncode != 0 and r.stderr:
logger.warning('ServiceComposer command failed: %s', r.stderr.strip())
return {
'ok': r.returncode == 0,
'stdout': r.stdout.strip(),
'stderr': r.stderr.strip(),
}
except subprocess.TimeoutExpired:
return {'ok': False, 'error': 'docker compose command timed out'}
except Exception as e:
logger.error('ServiceComposer._run error: %s', e)
return {'ok': False, 'error': str(e)}
@staticmethod
def _parse_ps_json(output: str) -> List[Dict]:
"""Parse `docker compose ps --format json` output (one JSON object per line)."""
containers = []
for line in output.splitlines():
line = line.strip()
if not line:
continue
try:
containers.append(json.loads(line))
except json.JSONDecodeError:
pass
return containers
# ── Store-service lifecycle (per-service compose file) ────────────────
def _store_cmd(self, service_id: str, *args, timeout: int = 120) -> Dict:
compose_file = self._compose_path(service_id)
if not os.path.exists(compose_file):
return {'ok': False, 'error': f'No compose file found for service {service_id!r}'}
cmd = [
'docker', 'compose',
'-f', compose_file,
'--project-name', f'pic-{service_id}',
*args,
]
return self._run(cmd, timeout)
def up(self, service_id: str) -> Dict:
# 600s: image pulls on slow connections can take several minutes
return self._store_cmd(service_id, 'up', '-d', '--remove-orphans', timeout=600)
def down(self, service_id: str, remove_volumes: bool = False) -> Dict:
args = ['down']
if remove_volumes:
args.append('--volumes')
return self._store_cmd(service_id, *args)
def restart(self, service_id: str) -> Dict:
return self._store_cmd(service_id, 'restart')
def status(self, service_id: str) -> Dict:
result = self._store_cmd(service_id, 'ps', '--format', 'json')
result['containers'] = self._parse_ps_json(result.get('stdout', ''))
return result
def reconfigure(self, service_id: str, manifest: Dict,
template_content: str) -> Dict:
"""Re-render the compose file then re-apply with `up -d` (rolling update)."""
self.write_compose(service_id, manifest, template_content)
return self.up(service_id)
def install(self, service_id: str, manifest: Dict,
template_content: str) -> Dict:
"""Write compose file and start containers."""
self.write_compose(service_id, manifest, template_content)
return self.up(service_id)
def remove(self, service_id: str, purge_data: bool = False) -> Dict:
"""Stop containers, optionally delete compose file, secrets, and service data dir."""
result = self.down(service_id, remove_volumes=purge_data)
if purge_data:
self._clear_secrets(service_id)
svc_dir = self._svc_dir(service_id) # already validates service_id + realpath
if os.path.isdir(svc_dir):
# Final realpath check: reject symlinks that escape the services dir
real_svc = os.path.realpath(svc_dir)
real_base = os.path.realpath(self._services_dir)
if not real_svc.startswith(real_base + os.sep):
logger.error('ServiceComposer: refusing rmtree outside services dir: %s', svc_dir)
else:
try:
shutil.rmtree(svc_dir)
except OSError as e:
logger.warning('ServiceComposer: could not remove %s: %s', svc_dir, e)
elif os.path.exists(self._compose_path(service_id)):
# Remove compose file even without purge so stale file doesn't confuse future installs
try:
os.remove(self._compose_path(service_id))
except OSError:
pass
return result
# ── Builtin-service lifecycle (main compose stack) ─────────────────────
@staticmethod
def _main_compose() -> str:
return os.environ.get('COMPOSE_FILE', '/app/docker-compose.yml')
def restart_builtin(self, container_names: List[str]) -> Dict:
"""Restart one or more containers that live in the main docker-compose stack."""
if not container_names:
return {'ok': False, 'error': 'No container names provided'}
cmd = ['docker', 'compose', '-f', self._main_compose(),
'restart', *container_names]
return self._run(cmd)
def status_builtin(self, container_names: List[str]) -> Dict:
"""Return status of containers from the main compose stack."""
if not container_names:
return {'ok': False, 'error': 'No container names provided'}
cmd = ['docker', 'compose', '-f', self._main_compose(),
'ps', '--format', 'json', *container_names]
result = self._run(cmd)
result['containers'] = self._parse_ps_json(result.get('stdout', ''))
return result
# ── Unified lifecycle (dispatches based on service kind) ───────────────
def restart_service(self, service_id: str, manifest: Dict) -> Dict:
"""
Restart any service — builtin or store — using the right compose stack.
Builtin: uses manifest.containers + main docker-compose.yml.
Store: uses per-service compose file.
"""
if manifest.get('kind') == 'builtin':
containers = manifest.get('containers') or []
return self.restart_builtin(containers)
return self.restart(service_id)
def status_service(self, service_id: str, manifest: Dict) -> Dict:
"""
Return container status for any service.
Builtin: queries manifest.containers from main compose stack.
Store: queries per-service compose project.
"""
if manifest.get('kind') == 'builtin':
containers = manifest.get('containers') or []
return self.status_builtin(containers)
return self.status(service_id)
@@ -0,0 +1,68 @@
{
"schema_version": 3,
"id": "calendar",
"name": "Calendar & Contacts",
"description": "Radicale CalDAV / CardDAV server",
"version": "1.0.0",
"author": "pic",
"kind": "builtin",
"min_pic_version": "1.0",
"capabilities": {
"has_subdomain": true,
"has_accounts": true,
"has_admin_config": true,
"has_storage": true,
"has_egress": true,
"has_api_hooks": false
},
"subdomain": "calendar",
"extra_subdomains": [],
"backend": "cell-radicale:5232",
"containers": ["cell-radicale"],
"config_schema": {
"port": {
"type": "integer",
"label": "CalDAV port (internal)",
"default": 5232,
"min": 1,
"max": 65535
}
},
"peer_config_template": {
"caldav_url": "https://calendar.{domain}/{peer.username}/",
"carddav_url": "https://calendar.{domain}/{peer.username}/",
"username": "{peer.username}",
"password": "{peer.service_credentials.calendar.password}"
},
"accounts": {
"manager": "calendar_manager",
"credentials": ["password"]
},
"compose": null,
"backup": {
"volumes": [
{"container": "cell-radicale", "path": "/data", "name": "radicale_data"}
],
"config_paths": [
"config/radicale"
]
},
"egress": {
"default": "default",
"allowed": ["default", "wireguard_ext", "openvpn", "tor"]
},
"storage": {
"primary_path": "data/radicale",
"quota_mb": null
}
}
+99
View File
@@ -0,0 +1,99 @@
{
"schema_version": 3,
"id": "email",
"name": "Email",
"description": "Postfix (SMTP) + Dovecot (IMAP) email server with Rainloop webmail",
"version": "1.0.0",
"author": "pic",
"kind": "builtin",
"min_pic_version": "1.0",
"capabilities": {
"has_subdomain": true,
"has_accounts": true,
"has_admin_config": true,
"has_storage": true,
"has_egress": true,
"has_api_hooks": false
},
"subdomain": "mail",
"extra_subdomains": ["webmail"],
"backend": "cell-rainloop:8888",
"containers": ["cell-mail", "cell-rainloop"],
"config_schema": {
"domain": {
"type": "string",
"label": "Mail domain",
"required": true
},
"smtp_port": {
"type": "integer",
"label": "SMTP port",
"default": 25,
"min": 1,
"max": 65535
},
"submission_port": {
"type": "integer",
"label": "Submission port",
"default": 587,
"min": 1,
"max": 65535
},
"imap_port": {
"type": "integer",
"label": "IMAP port",
"default": 993,
"min": 1,
"max": 65535
},
"webmail_port": {
"type": "integer",
"label": "Webmail port (internal)",
"default": 8888,
"min": 1,
"max": 65535
}
},
"peer_config_template": {
"imap_server": "{domain}",
"imap_port": "{config.imap_port}",
"smtp_server": "{domain}",
"smtp_port": "{config.submission_port}",
"webmail_url": "https://mail.{domain}/",
"username": "{peer.username}@{domain}",
"password": "{peer.service_credentials.email.password}"
},
"accounts": {
"manager": "email_manager",
"credentials": ["password"]
},
"compose": null,
"backup": {
"volumes": [
{"container": "cell-mail", "path": "/var/mail", "name": "maildata"},
{"container": "cell-mail", "path": "/var/mail-state", "name": "mailstate"},
{"container": "cell-rainloop", "path": "/rainloop/data", "name": "rainloop"}
],
"config_paths": [
"config/mail"
]
},
"egress": {
"default": "default",
"allowed": ["default", "wireguard_ext", "openvpn", "tor"]
},
"storage": {
"primary_path": "data/maildata",
"quota_mb": null
}
}
+79
View File
@@ -0,0 +1,79 @@
{
"schema_version": 3,
"id": "files",
"name": "File Storage",
"description": "FileGator browser UI + WebDAV network drive",
"version": "1.0.0",
"author": "pic",
"kind": "builtin",
"min_pic_version": "1.0",
"capabilities": {
"has_subdomain": true,
"has_accounts": true,
"has_admin_config": true,
"has_storage": true,
"has_egress": true,
"has_api_hooks": false
},
"subdomain": "files",
"extra_subdomains": ["webdav"],
"backend": "cell-filegator:8080",
"extra_backends": {
"webdav": "cell-webdav:80"
},
"containers": ["cell-filegator", "cell-webdav"],
"config_schema": {
"manager_port": {
"type": "integer",
"label": "FileGator port (internal)",
"default": 8082,
"min": 1,
"max": 65535
},
"port": {
"type": "integer",
"label": "WebDAV port (internal)",
"default": 8080,
"min": 1,
"max": 65535
}
},
"peer_config_template": {
"files_url": "https://files.{domain}/",
"webdav_url": "https://webdav.{domain}/",
"username": "{peer.username}",
"password": "{peer.service_credentials.files.password}"
},
"accounts": {
"manager": "file_manager",
"credentials": ["password"]
},
"compose": null,
"backup": {
"volumes": [
{"container": "cell-filegator", "path": "/var/www/filegator/private", "name": "filegator"},
{"container": "cell-webdav", "path": "/var/lib/dav", "name": "files"}
],
"config_paths": [
"config/webdav"
]
},
"egress": {
"default": "default",
"allowed": ["default", "wireguard_ext", "openvpn", "tor"]
},
"storage": {
"primary_path": "data/files",
"quota_mb": null
}
}