Compare commits
121 Commits
8efe8c1225
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| 2085f77733 | |||
| 36bc32543d | |||
| 348fd8faad | |||
| 9ad9fac8dd | |||
| c1e93f2058 | |||
| 3d750ed1e8 | |||
| 40f9d90fad | |||
| fb0326dae7 | |||
| e9077b2633 | |||
| da302b5d54 | |||
| 6bd5f02b03 | |||
| 7ef294fd65 | |||
| 33d255f089 | |||
| 85d265187d | |||
| 76bbc2b67a | |||
| bd71466a87 | |||
| e4c80149f4 | |||
| 69862331e7 | |||
| 962d137093 | |||
| 1607a2e86f | |||
| 9bdda6aaf8 | |||
| c696ca9ef6 | |||
| 4ebcb1d077 | |||
| 0507445d86 | |||
| 9b5c2e1994 | |||
| 08f46332b0 | |||
| e8b8e47aa4 | |||
| adce219a46 | |||
| 65d6d07c8d | |||
| ab6d6230dd | |||
| e2e9c50786 | |||
| 568e4f9783 | |||
| 26576e1124 | |||
| 31f76c54fa | |||
| b6af71acb5 | |||
| 352bb6bb9e | |||
| 463db029e1 | |||
| 8da711e366 | |||
| 3e26186f85 | |||
| f84f16fcd6 | |||
| eee0e800aa | |||
| 2b29938a64 | |||
| 39c59fd3ef | |||
| 1b44a18062 | |||
| f3737acfa4 | |||
| 64dd8b8488 | |||
| 0267dce73d | |||
| 41d09c598b | |||
| a906c26b5d | |||
| e87022dc55 | |||
| 7d5c5421f1 | |||
| 60601eb4af | |||
| 5ed75677c3 | |||
| f7bb2cc962 | |||
| c493630bb5 | |||
| 0ed8669aec | |||
| 03a67ad922 | |||
| 5cbbfb41d9 | |||
| 1f2f9d9f6e | |||
| 62b31b072b | |||
| 3d594025d2 | |||
| 10ac15d9fe | |||
| 44d7e96f29 | |||
| a69ca1e402 | |||
| a10fe11136 | |||
| 87c321c1c9 | |||
| 0bfe95320b | |||
| 18b50d08c1 | |||
| c40919d374 | |||
| 5e438aa991 | |||
| c20906d6cc | |||
| 2f5370bd98 | |||
| dc7b316cbd | |||
| ad5731073d | |||
| 16fb362df7 | |||
| 63c0dfb9d9 | |||
| 0afdee32da | |||
| b16189d00f | |||
| 66500bb128 | |||
| d7dbd596ab | |||
| 1f016de855 | |||
| 393d56d4ca | |||
| 01027c171e | |||
| 742e4209ee | |||
| ad2eaca273 | |||
| de43f4a9a0 | |||
| 0b31d02f10 | |||
| cde177966d | |||
| 61e8631c7d | |||
| 81dcced0ca | |||
| 777ffa4fb2 | |||
| 55d36eb410 | |||
| 99dcb1332a | |||
| 900781032a | |||
| 1c62c47475 | |||
| 4a42ff5dcc | |||
| 2d842abe5b | |||
| 8d1ef39ca5 | |||
| 9566f7dd1b | |||
| f03a5f08c6 | |||
| f550f04ce2 | |||
| 579f49ba13 | |||
| 925ab1f696 | |||
| 439886624e | |||
| 24877df976 | |||
| bfa0d99dd1 | |||
| 1e2cf5580f | |||
| 1989dfa0a3 | |||
| 5dab6377bc | |||
| 0a24d20bbc | |||
| 46599bd37e | |||
| dde4d9a53f | |||
| 674a66f7a0 | |||
| 9df3bf6a17 | |||
| 0773179962 | |||
| 3a35cf72d3 | |||
| 515f3d5075 | |||
| 35993bc79d | |||
| f1b48208fc | |||
| ffe1dbeed6 | |||
| 15376b67c7 |
+7
-1
@@ -21,8 +21,10 @@ config/api/caddy/Caddyfile
|
||||
config/api/calendar.json
|
||||
config/api/cell_config.json
|
||||
config/api/wireguard.json
|
||||
config/api/webdav/webdav.conf
|
||||
config/api/webdav/
|
||||
config/api/dhcp/
|
||||
config/api/dns/
|
||||
config/api/network.json
|
||||
config/caddy/Caddyfile
|
||||
config/dhcp/dnsmasq.conf
|
||||
config/dns/Corefile
|
||||
@@ -85,3 +87,7 @@ backups/
|
||||
# Temporary files
|
||||
*.tmp
|
||||
*.temp
|
||||
|
||||
# Coverage data
|
||||
.coverage
|
||||
htmlcov/
|
||||
@@ -1,87 +1,282 @@
|
||||
# CLAUDE.md
|
||||
|
||||
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
|
||||
This file is the primary context source for Claude Code in this repository. Read it fully before touching any code.
|
||||
|
||||
## What This Project Is
|
||||
---
|
||||
|
||||
**Personal Internet Cell (PIC)** — a self-hosted digital infrastructure platform. It manages DNS, DHCP, NTP, WireGuard VPN, email, calendar/contacts (CalDAV), file storage (WebDAV), reverse proxy (Caddy), a certificate authority, and container orchestration, all from a single API + React UI.
|
||||
## Project Overview
|
||||
|
||||
## Common Commands
|
||||
**Personal Internet Cell (PIC)** is a self-hosted digital infrastructure platform for individuals who want full ownership of their core internet services without relying on cloud providers.
|
||||
|
||||
```bash
|
||||
# Full stack
|
||||
make start # docker-compose up -d
|
||||
make stop # docker-compose down
|
||||
make restart # docker-compose restart
|
||||
make status # docker status + API health
|
||||
make logs # docker-compose logs -f
|
||||
make build # rebuild api image
|
||||
A PIC instance runs DNS, DHCP, NTP, WireGuard VPN, email (SMTP/IMAP), calendar/contacts (CalDAV/CardDAV), file storage (WebDAV), HTTPS reverse proxy (Caddy), an internal certificate authority, and optional third-party services — all managed from a single REST API and a React web UI. No manual config-file editing is required for normal operations.
|
||||
|
||||
# Tests
|
||||
make test # pytest tests/ api/tests/
|
||||
make test-coverage # pytest with coverage HTML report
|
||||
make test-api # pytest tests/test_api_endpoints.py
|
||||
pytest tests/test_<module>.py # single test file
|
||||
**Primary users:** technically capable individuals, homelab operators, small families or teams.
|
||||
|
||||
# Local dev (no Docker)
|
||||
pip install -r api/requirements.txt
|
||||
python api/app.py # Flask API on :3000
|
||||
**What the product optimizes for:**
|
||||
- One-command install, browser-based first-run wizard, no manual `.env` editing for identity
|
||||
- Everything managed through the API and UI — the user should never need to `ssh` for day-to-day operations
|
||||
- Security by default: session auth, CSRF protection, WireGuard isolation, internal CA, no open API port
|
||||
- Reliability and observability: structured logs, health monitoring, automated config backups
|
||||
|
||||
cd webui && npm install && npm run dev # React UI on :5173 (proxies API to :3000)
|
||||
**Key constraints:**
|
||||
- Runs on a single Linux host with Docker; no Kubernetes, no swarm
|
||||
- Must work on Debian, Ubuntu, Fedora, RHEL, and Alpine
|
||||
- The Flask API must never be exposed directly; Caddy always proxies it
|
||||
- All secrets live in `data/` (git-ignored), never in the repo
|
||||
|
||||
# WireGuard
|
||||
make show-routes
|
||||
make add-peer PEER_NAME=foo PEER_IP=10.0.0.5 PEER_KEY=<pubkey>
|
||||
make list-peers
|
||||
```
|
||||
---
|
||||
|
||||
## Tech Stack
|
||||
|
||||
### Backend
|
||||
- **Python 3.11** — Flask REST API (`api/app.py`)
|
||||
- **Flask** — routing, sessions, before-request hooks (enforce_setup, enforce_auth, check_csrf)
|
||||
- **bcrypt** — password hashing in `AuthManager`
|
||||
- **Docker SDK for Python** — container lifecycle in `ContainerManager`
|
||||
- **PyNaCl / Age** — encryption in `VaultManager`
|
||||
- **pyotp** — TOTP for DDNS registration
|
||||
|
||||
### Frontend
|
||||
- **React 18** — SPA
|
||||
- **Vite** — dev server and build (proxies `/api` → `:3000`)
|
||||
- **Tailwind CSS** — all styling; no custom CSS files
|
||||
- **Axios** — all API calls go through `src/services/api.js`
|
||||
|
||||
### Infrastructure
|
||||
- **Docker Compose** — all 12+ service containers
|
||||
- **Caddy** — reverse proxy, TLS termination (Let's Encrypt DNS-01 or HTTP-01 or internal CA)
|
||||
- **CoreDNS** — `.cell` TLD authoritative DNS
|
||||
- **dnsmasq** — DHCP
|
||||
- **chrony** — NTP
|
||||
- **WireGuard** — VPN (kernel module, not userspace)
|
||||
- **Postfix + Dovecot** — email via `docker-mailserver`
|
||||
- **Radicale** — CalDAV/CardDAV
|
||||
- **PowerDNS** — authoritative DNS on the DDNS VPS (separate repo: `pic-ddns`)
|
||||
|
||||
### CI/CD
|
||||
- **Gitea Actions** — unit tests on every push, image builds on tag
|
||||
- **act_runner** — self-hosted runner on pic0 (192.168.31.51)
|
||||
- **Gitea Container Registry** — images pushed to `git.pic.ngo`
|
||||
|
||||
Do not introduce: Redux, styled-components, SQLAlchemy, Celery, or any async framework (asyncio/FastAPI) into the main API unless explicitly requested.
|
||||
|
||||
---
|
||||
|
||||
## Architecture
|
||||
|
||||
### Backend (`api/`)
|
||||
```
|
||||
Browser / WireGuard peer
|
||||
└── Caddy (:80/:443) TLS termination, reverse proxy
|
||||
└── React SPA (:8081) Vite + Tailwind (Nginx in container)
|
||||
└── Flask API (:3000) REST API, bound to 127.0.0.1 only
|
||||
├── NetworkManager CoreDNS, dnsmasq, chrony
|
||||
├── WireGuardManager WireGuard peer lifecycle
|
||||
├── PeerRegistry peer registration and trust
|
||||
├── EmailManager Postfix + Dovecot
|
||||
├── CalendarManager Radicale CalDAV/CardDAV
|
||||
├── FileManager WebDAV + Filegator
|
||||
├── RoutingManager iptables NAT and routing
|
||||
├── FirewallManager iptables INPUT/FORWARD rules
|
||||
├── VaultManager internal CA, TLS certs, Age encryption
|
||||
├── ContainerManager Docker SDK
|
||||
├── CellLinkManager site-to-site WireGuard links
|
||||
├── ConnectivityManager per-peer exit routing (WG ext, OpenVPN, Tor)
|
||||
├── DDNSManager dynamic DNS heartbeat
|
||||
├── ServiceStoreManager optional service install/remove
|
||||
├── CaddyManager Caddyfile generation and reload
|
||||
├── AuthManager bcrypt passwords, session auth, RBAC
|
||||
└── SetupManager first-run wizard state
|
||||
```
|
||||
|
||||
All service managers inherit `BaseServiceManager` (`api/base_service_manager.py`). This enforces a consistent interface: `get_status()`, `get_config()`, `update_config()`, `validate_config()`, `test_connectivity()`, `get_logs()`, `restart_service()`. When adding or modifying a service manager, follow this pattern.
|
||||
### Key files
|
||||
|
||||
The `ServiceBus` (`api/service_bus.py`) is a pub/sub event system used for inter-service communication. Services publish events (e.g., `SERVICE_STARTED`, `CONFIG_CHANGED`, `PEER_CONNECTED`) and subscribe to events from dependencies. Dependency graph is declared in the bus — e.g., `wireguard` depends on `network`; `email` depends on `network` and `vault`.
|
||||
| File | Role |
|
||||
|---|---|
|
||||
| `api/app.py` | Flask app, all REST endpoints, before-request hooks, health monitor thread |
|
||||
| `api/managers.py` | Singleton instantiation of all service managers |
|
||||
| `api/base_service_manager.py` | Abstract base class: `get_status`, `get_config`, `update_config`, `validate_config`, `test_connectivity`, `get_logs`, `restart_service` |
|
||||
| `api/config_manager.py` | Single source of truth for `cell_config.json` — all read/write goes through here |
|
||||
| `api/service_bus.py` | Pub/sub event system between managers |
|
||||
| `webui/src/services/api.js` | Axios API client — all UI→API calls |
|
||||
| `docker-compose.yml` | Container definitions and network topology |
|
||||
| `Makefile` | All operational commands |
|
||||
| `install.sh` | Bash installer served via `https://install.pic.ngo` |
|
||||
|
||||
`ConfigManager` (`api/config_manager.py`) is the single source of truth. Config lives in `/app/config/cell_config.json` (mapped from `config/api/`). All managers read/write through ConfigManager, which validates against per-service schemas and maintains automatic backups.
|
||||
### Directory layout
|
||||
|
||||
`LogManager` (`api/log_manager.py`) provides structured JSON logging with rotation (5 MB / 5 backups per service). Use it instead of `print()` or raw `logging`.
|
||||
```
|
||||
api/ Flask API and all service managers
|
||||
webui/ React SPA (Vite + Tailwind)
|
||||
tests/ pytest unit tests (no running services required)
|
||||
tests/integration/ require a running PIC stack
|
||||
tests/e2e/ Playwright UI and WireGuard e2e tests
|
||||
config/ Runtime config per service (mostly git-ignored)
|
||||
data/ Runtime secrets and state (fully git-ignored)
|
||||
scripts/ Setup and maintenance scripts
|
||||
install.sh One-line installer entry point
|
||||
Makefile All make targets
|
||||
docker-compose.yml
|
||||
```
|
||||
|
||||
`app.py` (2000+ lines) contains all Flask REST endpoints, organized by service. It runs a background health-monitoring thread.
|
||||
### Config and secrets
|
||||
|
||||
Service managers:
|
||||
- `network_manager.py` — DNS (CoreDNS), DHCP (dnsmasq), NTP (chrony)
|
||||
- `wireguard_manager.py` — VPN peer lifecycle, QR codes
|
||||
- `peer_registry.py` — peer registration/lookup
|
||||
- `routing_manager.py` — NAT, firewall rules, VPN gateway
|
||||
- `vault_manager.py` — internal certificate authority
|
||||
- `email_manager.py` — Postfix + Dovecot
|
||||
- `calendar_manager.py` — Radicale CalDAV/CardDAV
|
||||
- `file_manager.py` — WebDAV storage
|
||||
- `container_manager.py` — Docker SDK wrappers
|
||||
- `cell_manager.py` — top-level orchestration
|
||||
- Runtime config: `config/api/cell_config.json` — managed by `ConfigManager`, never edit directly
|
||||
- Secrets and user data: `data/` — git-ignored, contains `auth_users.json`, WireGuard keys, DDNS token, CA key
|
||||
- DDNS config lives under the top-level `ddns` key in `cell_config.json`, accessed via `config_manager.configs.get('ddns', {})`
|
||||
- Do not read `_identity.domain` expecting a dict — it is a plain string (the domain mode, e.g. `"pic_ngo"`)
|
||||
|
||||
### Frontend (`webui/`)
|
||||
### Before-request hooks (app.py)
|
||||
|
||||
React 18 + Vite + Tailwind CSS. All API calls go through `src/services/api.js` (Axios). Vite dev server proxies `/api` to `localhost:3000`. Pages in `src/pages/`, shared components in `src/components/`.
|
||||
Three hooks run on every request in this order:
|
||||
1. `enforce_setup` — returns 428 for all `/api/*` except `/api/setup/*` and `/health` until setup is complete. Skipped when `app.config['TESTING']` is True.
|
||||
2. `enforce_auth` — returns 401 if no session; returns 503 if users file exists but is empty (misconfiguration). Skipped when `app.config['TESTING']` is True.
|
||||
3. `check_csrf` — requires `X-CSRF-Token` header on all mutating requests except `/api/auth/*` and `/api/setup/*`.
|
||||
|
||||
### Infrastructure
|
||||
---
|
||||
|
||||
`docker-compose.yml` defines 13 services on a custom bridge network `cell-network` (172.20.0.0/16). Cell IPs default to 10.0.0.0/24. Key ports: 53 (DNS), 80/443 (Caddy), 3000 (API), 5173/8081 (WebUI), 51820/udp (WireGuard), 25/587/993 (mail), 5232 (CalDAV), 8080 (WebDAV).
|
||||
## Coding Conventions
|
||||
|
||||
Config files for each service live under `config/<service>/`. Persistent data is under `data/` (git-ignored). WireGuard configs are also git-ignored.
|
||||
### Python (API)
|
||||
|
||||
## Testing
|
||||
- All managers inherit `BaseServiceManager` — always implement all abstract methods
|
||||
- Use `self.logger` (from `BaseServiceManager`) — never `print()` or raw `logging`
|
||||
- Config reads go through `self.config_manager` — never open `cell_config.json` directly
|
||||
- Use `threading.RLock` for shared state; managers run in a multi-threaded Flask app
|
||||
- Do not use `any` typing; be explicit
|
||||
- Keep Flask route handlers thin — business logic belongs in the manager, not in `app.py`
|
||||
- Error responses must be JSON: `jsonify({'error': '...'}), <status_code>`
|
||||
- Do not catch bare `Exception` and silently swallow it — log at minimum
|
||||
|
||||
Tests live in `tests/` (28 files). Use mocking (`pytest-mock`) for external system calls. Integration tests in `test_integration.py` require Docker services running.
|
||||
### JavaScript (webui)
|
||||
|
||||
## AI Collaboration Rules (Claude Code)
|
||||
- All API calls go through `src/services/api.js` — never use `fetch` or a new Axios instance directly
|
||||
- Use functional components; no class components
|
||||
- Tailwind utilities only — no inline styles, no custom CSS files
|
||||
- Keep page components in `src/pages/`, reusable UI in `src/components/`
|
||||
- State: local `useState`/`useEffect` is fine; no Redux or global state library
|
||||
|
||||
### General
|
||||
|
||||
- No comments that describe *what* the code does — only *why* if non-obvious
|
||||
- No dead code, no commented-out blocks
|
||||
- No backwards-compat shims for things being removed
|
||||
- Prefer editing existing files over creating new ones
|
||||
- Tests that write to disk: mock `builtins.open` with `OSError` rather than relying on `/nonexistent/path` (CI runs as root and can create any path)
|
||||
|
||||
---
|
||||
|
||||
## Testing and Quality
|
||||
|
||||
Before considering any task complete:
|
||||
1. Run `make test` — all 1500+ unit tests must pass
|
||||
2. Fix failures before committing — the pre-commit hook will block the commit anyway
|
||||
|
||||
### Rules
|
||||
|
||||
- Use `unittest.mock` / `pytest-mock` for all Docker, filesystem, and subprocess calls
|
||||
- Tests must pass in CI (rootless environment where filesystem assumptions don't hold)
|
||||
- When testing write-failure paths, mock `builtins.open` with `side_effect=OSError` — do not rely on unwritable paths
|
||||
- Integration tests (`tests/integration/`) require a running stack — exclude from CI with `--ignore=tests/integration`
|
||||
- E2e tests (`tests/e2e/`) require Playwright — exclude from CI with `--ignore=tests/e2e`
|
||||
- Add tests for any new API endpoint, manager method, or utility function
|
||||
- Do not add tests for Flask routing boilerplate or trivial getters — test behaviour, not structure
|
||||
|
||||
---
|
||||
|
||||
## File Placement Rules
|
||||
|
||||
| New thing | Where it goes |
|
||||
|---|---|
|
||||
| New service manager | `api/<name>_manager.py`, registered in `api/managers.py` and wired into `app.py` |
|
||||
| New API endpoints | `app.py` — grouped with the relevant manager's existing endpoints |
|
||||
| New React page | `webui/src/pages/` |
|
||||
| Reusable UI component | `webui/src/components/` |
|
||||
| New pytest test file | `tests/test_<module>.py` |
|
||||
| Operational script | `scripts/` |
|
||||
| Documentation | Update `README.md`, `QUICKSTART.md`, or `Personal Internet Cell – Project Wiki.md` as appropriate |
|
||||
|
||||
Do not create a new abstraction for a single use case. Do not create near-duplicate files — edit the existing one.
|
||||
|
||||
---
|
||||
|
||||
## Safety Rules
|
||||
|
||||
- **Never expose the Flask API port (3000) directly** — it must always be behind Caddy
|
||||
- **Never commit secrets** — `data/`, `.env`, `*.key`, `*.pem` are all git-ignored; keep it that way
|
||||
- **Do not modify `enforce_setup` or `enforce_auth` hooks** without understanding the full auth flow — these are the security boundary
|
||||
- **Do not change the `cell_config.json` schema** without updating `ConfigManager` validation and all manager reads
|
||||
- **Do not rename API route paths** without checking the webui `api.js` client and any external callers
|
||||
- **Do not modify WireGuard key generation** — losing the server private key means all peers must be re-provisioned
|
||||
- Flag any change to auth flow, CSRF logic, or session management as security-sensitive before implementing
|
||||
|
||||
---
|
||||
|
||||
## Commands
|
||||
|
||||
```bash
|
||||
# Stack lifecycle (always use make — never call docker/docker-compose directly)
|
||||
make start # build and start all containers
|
||||
make stop # stop all containers
|
||||
make restart # restart containers
|
||||
make status # container status + API health check
|
||||
make logs # follow all container logs
|
||||
make logs-api # follow API logs only
|
||||
make logs-caddy # follow Caddy logs
|
||||
make shell-api # shell inside the API container
|
||||
make build-api # rebuild API image after code change
|
||||
make build-webui # rebuild webui image after code change
|
||||
|
||||
# Tests
|
||||
make test # pytest tests/ --ignore=tests/e2e --ignore=tests/integration
|
||||
make test-coverage # coverage report in htmlcov/
|
||||
pytest tests/test_<module>.py -v # single test file
|
||||
|
||||
# Local dev (no Docker)
|
||||
pip install -r api/requirements.txt
|
||||
python3 api/app.py # Flask API on :3000
|
||||
|
||||
cd webui && npm install && npm run dev # React UI on :5173 (proxies /api → :3000)
|
||||
|
||||
# Peer / WireGuard
|
||||
make list-peers
|
||||
make show-routes
|
||||
|
||||
# Admin password
|
||||
make show-admin-password
|
||||
make reset-admin-password
|
||||
|
||||
# Backup / restore
|
||||
make backup
|
||||
make restore
|
||||
|
||||
# Maintenance
|
||||
make update # git pull + rebuild + restart
|
||||
make uninstall # stop containers; prompt to delete config/ and data/
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Infrastructure Topology
|
||||
|
||||
| Machine | IP | Role |
|
||||
|---|---|---|
|
||||
| pic0 | 192.168.31.51 | Dev machine — you are here. Run all commands directly. |
|
||||
| pic1 | 192.168.31.52 | Test/staging PIC instance |
|
||||
| Gitea | 192.168.31.50 | Self-hosted git server (`gitea@192.168.31.50:roof/pic.git`) |
|
||||
| DDNS VPS | 192.168.31.101 (LAN) / 178.168.15.65 (public) | PowerDNS + FastAPI for `*.pic.ngo` DDNS |
|
||||
|
||||
The `roof` user on pic0 has passwordless sudo and is in the `docker` group — use both freely.
|
||||
|
||||
---
|
||||
|
||||
## AI Collaboration Rules
|
||||
|
||||
These rules apply to every Claude Code session in this repo:
|
||||
|
||||
- **Read memory first** — load `/home/roof/.claude/projects/-home-roof/memory/MEMORY.md` and referenced files at session start.
|
||||
- **Dev machine context** — you are already on pic0 (192.168.31.51), the dev machine. Execute commands here directly; do not ask the user to run them.
|
||||
- **Use all available agents** — spawn specialized sub-agents (pic-remote, pic-qa, pic-architect, etc.) for tasks that match their description.
|
||||
- **make is the only interface** — never call docker/docker-compose directly. All container lifecycle operations go through `make start`, `make stop`, `make build`, `make logs`, etc.
|
||||
- **Test every new feature** — after implementing any change, run `make test` before considering the task done.
|
||||
- **Test before commit** — the pre-commit hook enforces this, but run `make test` manually first and fix all failures before staging files.
|
||||
- **Read memory first** — load `/home/roof/.claude/projects/-home-roof/memory/MEMORY.md` at session start; follow referenced memory files for relevant context.
|
||||
- **You are on pic0** — execute commands directly here; do not ask the user to run them.
|
||||
- **`make` is the only container interface** — never call `docker` or `docker-compose` directly. All container lifecycle goes through `make start`, `make stop`, `make build`, `make logs`, etc.
|
||||
- **Use specialized agents** — spawn `pic-remote` for VPS/pic1 SSH tasks, `pic-qa` for test writing, `pic-architect` for design decisions, `pic-designer` for UI review, `pic-devops` for docker-compose/Makefile changes, `pic-writer` for documentation.
|
||||
- **Test before commit** — run `make test` and fix all failures before staging. The pre-commit hook enforces this, but run it manually first.
|
||||
- **No skipping hooks** — never use `--no-verify` unless the only change is documentation or a workflow file with no Python/JS.
|
||||
- **Commits need context** — write commit messages that explain *why*, not just *what*. Always add the Co-Authored-By trailer.
|
||||
|
||||
@@ -12,7 +12,8 @@
|
||||
test-e2e-deps test-e2e-api test-e2e-ui test-e2e-wg test-e2e \
|
||||
reset-test-admin-pass \
|
||||
show-admin-password reset-admin-password \
|
||||
show-routes add-peer list-peers
|
||||
show-routes add-peer list-peers \
|
||||
ddns-update ddns-register
|
||||
|
||||
# Detect docker compose command (v2 plugin preferred, fallback to v1 standalone)
|
||||
DC := $(shell docker compose version >/dev/null 2>&1 && echo "docker compose" || echo "docker-compose")
|
||||
@@ -78,9 +79,14 @@ check-deps:
|
||||
|
||||
setup: check-deps
|
||||
@echo "Setting up Personal Internet Cell..."
|
||||
@sudo chown -R $$(id -u):$$(id -g) config/ data/ 2>/dev/null || true
|
||||
@sudo chown -R $${SUDO_USER:-$$(id -un)}:$${SUDO_USER:-$$(id -un)} config/ data/ 2>/dev/null || true
|
||||
CELL_NAME=$(or $(CELL_NAME),mycell) \
|
||||
CELL_DOMAIN=$(or $(CELL_DOMAIN),cell) \
|
||||
DOMAIN_MODE=$(or $(DOMAIN_MODE),lan) \
|
||||
CELL_DOMAIN_NAME=$(or $(CELL_DOMAIN_NAME),) \
|
||||
CLOUDFLARE_API_TOKEN=$(or $(CLOUDFLARE_API_TOKEN),) \
|
||||
DUCKDNS_TOKEN=$(or $(DUCKDNS_TOKEN),) \
|
||||
DUCKDNS_SUBDOMAIN=$(or $(DUCKDNS_SUBDOMAIN),) \
|
||||
VPN_ADDRESS=$(or $(VPN_ADDRESS),10.0.0.1/24) \
|
||||
WG_PORT=$(or $(WG_PORT),51820) \
|
||||
WG_PRIVATE_KEY="$(WG_PRIVATE_KEY)" \
|
||||
@@ -96,12 +102,14 @@ init-peers:
|
||||
|
||||
start:
|
||||
@echo "Starting Personal Internet Cell..."
|
||||
PUID=$$(id -u) PGID=$$(id -g) $(DCF) --profile full up -d --build
|
||||
@docker network inspect cell-network >/dev/null 2>&1 || \
|
||||
docker network create --driver bridge --subnet "$${CELL_NETWORK:-172.20.0.0/16}" cell-network
|
||||
PUID=$$(id -u) PGID=$$(id -g) $(DCF) --profile core up -d --build
|
||||
@echo "Services started. Check status with 'make status'"
|
||||
|
||||
stop:
|
||||
@echo "Stopping Personal Internet Cell..."
|
||||
PUID=$$(id -u) PGID=$$(id -g) $(DCF) --profile full down
|
||||
PUID=$$(id -u) PGID=$$(id -g) $(DCF) --profile core down
|
||||
@echo "Services stopped."
|
||||
|
||||
restart:
|
||||
@@ -130,20 +138,20 @@ shell-%:
|
||||
|
||||
update:
|
||||
@echo "Pulling latest code..."
|
||||
@git config --global --add safe.directory $$(pwd) 2>/dev/null || true
|
||||
@git stash --include-untracked --quiet 2>/dev/null || true
|
||||
git pull
|
||||
@git stash pop --quiet 2>/dev/null || true
|
||||
@if [ ! -f config/mail/mailserver.env ]; then \
|
||||
echo "Config missing — running setup first..."; \
|
||||
$(MAKE) setup; \
|
||||
fi
|
||||
@echo "Rebuilding and restarting services..."
|
||||
PUID=$$(id -u) PGID=$$(id -g) $(DCF) --profile full up -d --build
|
||||
@docker network inspect cell-network >/dev/null 2>&1 || \
|
||||
docker network create --driver bridge --subnet "$${CELL_NETWORK:-172.20.0.0/16}" cell-network
|
||||
PUID=$$(id -u) PGID=$$(id -g) $(DCF) --profile core up -d --build
|
||||
@echo "Update complete. Run 'make status' to verify."
|
||||
|
||||
reinstall:
|
||||
@echo "Reinstalling Personal Internet Cell from scratch..."
|
||||
PUID=$$(id -u) PGID=$$(id -g) $(DCF) --profile full down -v 2>/dev/null || true
|
||||
PUID=$$(id -u) PGID=$$(id -g) $(DCF) --profile core down 2>/dev/null || true
|
||||
docker network rm cell-network 2>/dev/null || true
|
||||
@sudo rm -rf config/ data/
|
||||
@$(MAKE) setup
|
||||
@$(MAKE) start
|
||||
@@ -172,14 +180,17 @@ uninstall:
|
||||
case "$$ans" in \
|
||||
y|Y) \
|
||||
echo "Stopping containers and removing images..."; \
|
||||
PUID=$$(id -u) PGID=$$(id -g) $(DCF) --profile full down -v --rmi all 2>/dev/null || true; \
|
||||
for f in data/api/services/*/docker-compose.yml; do [ -f "$$f" ] && PUID=$$(id -u) PGID=$$(id -g) docker compose -f "$$f" down 2>/dev/null || true; done; \
|
||||
PUID=$$(id -u) PGID=$$(id -g) $(DCF) --profile core down --rmi all 2>/dev/null || true; \
|
||||
docker network rm cell-network 2>/dev/null || true; \
|
||||
echo "Deleting config/ and data/..."; \
|
||||
sudo rm -rf config/ data/; \
|
||||
echo "Uninstall complete. Git repo and scripts remain."; \
|
||||
;; \
|
||||
n|N|"") \
|
||||
echo "Stopping and removing containers (keeping images and data)..."; \
|
||||
PUID=$$(id -u) PGID=$$(id -g) $(DCF) --profile full down 2>/dev/null || true; \
|
||||
for f in data/api/services/*/docker-compose.yml; do [ -f "$$f" ] && PUID=$$(id -u) PGID=$$(id -g) docker compose -f "$$f" down 2>/dev/null || true; done; \
|
||||
PUID=$$(id -u) PGID=$$(id -g) $(DCF) --profile core down 2>/dev/null || true; \
|
||||
echo "Done. Images, config/ and data/ are untouched. Run 'make start' to bring it back up."; \
|
||||
;; \
|
||||
*) \
|
||||
@@ -211,6 +222,8 @@ build-webui:
|
||||
|
||||
start-core:
|
||||
@echo "Starting core services (caddy, dns, wireguard, api, webui)..."
|
||||
@docker network inspect cell-network >/dev/null 2>&1 || \
|
||||
docker network create --driver bridge --subnet "$${CELL_NETWORK:-172.20.0.0/16}" cell-network
|
||||
PUID=$$(id -u) PGID=$$(id -g) $(DCF) --profile core up -d --build
|
||||
@echo "Core services started. Run 'make start' to also bring up optional services."
|
||||
|
||||
@@ -335,6 +348,21 @@ add-peer:
|
||||
echo "Usage: make add-peer PEER_NAME=name PEER_IP=10.0.0.x PEER_KEY=<pubkey>"; \
|
||||
fi
|
||||
|
||||
# ── DDNS ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
ddns-update:
|
||||
@python3 scripts/ddns_update.py
|
||||
|
||||
ddns-register:
|
||||
@DDNS_TOTP_SECRET="$(DDNS_TOTP_SECRET)" python3 -c "\
|
||||
import os, sys; sys.path.insert(0, 'scripts'); \
|
||||
from setup_cell import register_with_ddns, _read_existing_ip_range; \
|
||||
import json; \
|
||||
cfg = json.load(open('config/api/cell_config.json')) if os.path.exists('config/api/cell_config.json') else {}; \
|
||||
name = cfg.get('_identity', {}).get('cell_name', os.environ.get('CELL_NAME', 'mycell')); \
|
||||
import os; os.remove('data/api/.ddns_token') if os.path.exists('data/api/.ddns_token') else None; \
|
||||
register_with_ddns(name)"
|
||||
|
||||
# ── Dev ───────────────────────────────────────────────────────────────────────
|
||||
|
||||
dev:
|
||||
|
||||
@@ -1,535 +1,416 @@
|
||||
# Personal Internet Cell – Project Wiki
|
||||
|
||||
## 🌟 Overview
|
||||
## Overview
|
||||
|
||||
Personal Internet Cell is a **production-grade, self-hosted, decentralized digital infrastructure** solution designed to provide individuals with full control over their digital services and data. The project has evolved from a phase-based implementation to a **unified, enterprise-ready system** with modern architecture, comprehensive testing, and production-grade features.
|
||||
Personal Internet Cell (PIC) is a self-hosted digital infrastructure platform. It runs DNS, DHCP, NTP, WireGuard VPN, email, calendar/contacts, file storage, HTTPS reverse proxy, a certificate authority, and optional services — all managed from a single REST API and React web UI.
|
||||
|
||||
## 📋 Table of Contents
|
||||
|
||||
1. [Project Goals](#project-goals)
|
||||
2. [Architecture & Components](#architecture--components)
|
||||
3. [Service Manager Architecture](#service-manager-architecture)
|
||||
4. [Core Services](#core-services)
|
||||
5. [API Reference](#api-reference)
|
||||
6. [Enhanced CLI](#enhanced-cli)
|
||||
7. [Security Model](#security-model)
|
||||
8. [Testing & Quality Assurance](#testing--quality-assurance)
|
||||
9. [Usage Examples](#usage-examples)
|
||||
10. [Development & Deployment](#development--deployment)
|
||||
11. [Future Enhancements](#future-enhancements)
|
||||
12. [Project Status](#project-status)
|
||||
|
||||
## 🎯 Project Goals
|
||||
|
||||
- **Self-Hosted**: Run your own digital services (email, calendar, files, VPN, etc.) on your hardware
|
||||
- **Decentralized**: Peer-to-peer networking and trust, no central authority
|
||||
- **Production-Grade**: Enterprise-ready architecture with comprehensive monitoring
|
||||
- **Secure**: Modern cryptography, certificate management, and encrypted storage
|
||||
- **User-Friendly**: Professional CLI and API for easy management
|
||||
- **Extensible**: Modular architecture for future services and integrations
|
||||
- **Event-Driven**: Real-time service communication and orchestration
|
||||
|
||||
## 🏗️ Architecture & Components
|
||||
|
||||
### **Modern Architecture Stack**
|
||||
|
||||
- **Backend**: Python (Flask) with production-grade service managers
|
||||
- **Service Architecture**: BaseServiceManager pattern with unified interfaces
|
||||
- **Event System**: Service bus for real-time communication and orchestration
|
||||
- **Configuration**: Centralized configuration management with validation
|
||||
- **Logging**: Structured JSON logging with rotation and search
|
||||
- **Containerization**: Docker-based deployment and service isolation
|
||||
- **API**: RESTful endpoints with comprehensive documentation
|
||||
|
||||
### **Core Architecture Components**
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ Personal Internet Cell │
|
||||
├─────────────────────────────────────────────────────────────┤
|
||||
│ Enhanced CLI │ Web UI │ REST API │ Service Bus │ Logging │
|
||||
├─────────────────────────────────────────────────────────────┤
|
||||
│ Service Managers │
|
||||
│ Network │ WireGuard │ Email │ Calendar │ Files │ Routing │
|
||||
│ Vault │ Container │ Cell │ Peer │ │ │
|
||||
├─────────────────────────────────────────────────────────────┤
|
||||
│ Core Infrastructure │
|
||||
│ DNS │ DHCP │ NTP │ VPN │ CA │ Encryption │ Trust │ Storage │
|
||||
└─────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## 🔧 Service Manager Architecture
|
||||
|
||||
### **BaseServiceManager Pattern**
|
||||
|
||||
All services inherit from `BaseServiceManager`, providing:
|
||||
|
||||
```python
|
||||
class BaseServiceManager(ABC):
|
||||
def __init__(self, service_name: str, data_dir: str, config_dir: str)
|
||||
|
||||
@abstractmethod
|
||||
def get_status(self) -> Dict[str, Any]
|
||||
|
||||
@abstractmethod
|
||||
def test_connectivity(self) -> Dict[str, Any]
|
||||
|
||||
# Common methods
|
||||
def get_logs(self, lines: int = 50) -> List[str]
|
||||
def restart_service(self) -> bool
|
||||
def get_config(self) -> Dict[str, Any]
|
||||
def update_config(self, config: Dict[str, Any]) -> bool
|
||||
def health_check(self) -> Dict[str, Any]
|
||||
def handle_error(self, error: Exception, context: str) -> Dict[str, Any]
|
||||
```
|
||||
|
||||
### **Service Bus Integration**
|
||||
|
||||
```python
|
||||
# Event-driven service communication
|
||||
service_bus.register_service('network', network_manager)
|
||||
service_bus.register_service('wireguard', wireguard_manager)
|
||||
service_bus.publish_event(EventType.SERVICE_STARTED, 'network', data)
|
||||
|
||||
# Service dependencies
|
||||
service_dependencies = {
|
||||
'wireguard': ['network'],
|
||||
'email': ['network', 'vault'],
|
||||
'calendar': ['network', 'vault'],
|
||||
'files': ['network', 'vault'],
|
||||
'routing': ['network', 'wireguard'],
|
||||
'vault': ['network']
|
||||
}
|
||||
```
|
||||
|
||||
## 🔧 Core Services
|
||||
|
||||
### **Network Services**
|
||||
- **NetworkManager**: DNS, DHCP, NTP with dynamic management
|
||||
- Dynamic zone file generation
|
||||
- DHCP lease monitoring
|
||||
- Network connectivity testing
|
||||
- Service health monitoring
|
||||
|
||||
### **VPN & Mesh Networking**
|
||||
- **WireGuardManager**: WireGuard VPN configuration and peer management
|
||||
- Key generation and management
|
||||
- Peer configuration
|
||||
- Connectivity testing
|
||||
- Dynamic IP updates
|
||||
|
||||
- **PeerRegistry**: Peer registration and trust management
|
||||
- Peer lifecycle management
|
||||
- Trust relationship tracking
|
||||
- Data integrity validation
|
||||
- Peer statistics
|
||||
|
||||
### **Digital Services**
|
||||
- **EmailManager**: SMTP/IMAP email services
|
||||
- User account management
|
||||
- Mailbox configuration
|
||||
- Service connectivity testing
|
||||
- Email delivery monitoring
|
||||
|
||||
- **CalendarManager**: CalDAV/CardDAV calendar and contacts
|
||||
- User and calendar management
|
||||
- Event synchronization
|
||||
- Service health monitoring
|
||||
- Connectivity testing
|
||||
|
||||
- **FileManager**: WebDAV file storage
|
||||
- User directory management
|
||||
- Storage quota monitoring
|
||||
- File system access testing
|
||||
- Backup and restore capabilities
|
||||
|
||||
### **Infrastructure Services**
|
||||
- **RoutingManager**: Advanced routing and NAT
|
||||
- NAT rule management
|
||||
- Firewall configuration
|
||||
- Exit node routing
|
||||
- Bridge and split routing
|
||||
- Connectivity testing
|
||||
|
||||
- **VaultManager**: Security and trust management
|
||||
- Self-hosted Certificate Authority
|
||||
- Certificate lifecycle management
|
||||
- Age/Fernet encryption
|
||||
- Trust relationship management
|
||||
- Cryptographic verification
|
||||
|
||||
- **ContainerManager**: Docker orchestration
|
||||
- Container lifecycle management
|
||||
- Image and volume management
|
||||
- Docker daemon connectivity
|
||||
- Service isolation
|
||||
|
||||
- **CellManager**: Overall cell orchestration
|
||||
- Service coordination
|
||||
- Health monitoring
|
||||
- Configuration management
|
||||
- Peer management
|
||||
|
||||
## 📡 API Reference
|
||||
|
||||
### **Core API Endpoints**
|
||||
|
||||
```bash
|
||||
# Service Status and Health
|
||||
GET /api/services/status # All services status
|
||||
GET /api/services/connectivity # Service connectivity tests
|
||||
GET /health # API health check
|
||||
|
||||
# Configuration Management
|
||||
GET /api/config # Get configuration
|
||||
PUT /api/config # Update configuration
|
||||
POST /api/config/backup # Create backup
|
||||
GET /api/config/backups # List backups
|
||||
POST /api/config/restore/<id> # Restore backup
|
||||
GET /api/config/export # Export configuration
|
||||
POST /api/config/import # Import configuration
|
||||
|
||||
# Service Bus
|
||||
GET /api/services/bus/status # Service bus status
|
||||
GET /api/services/bus/events # Event history
|
||||
POST /api/services/bus/services/<service>/start
|
||||
POST /api/services/bus/services/<service>/stop
|
||||
POST /api/services/bus/services/<service>/restart
|
||||
|
||||
# Logging
|
||||
GET /api/logs/services/<service> # Service logs
|
||||
POST /api/logs/search # Log search
|
||||
POST /api/logs/export # Log export
|
||||
GET /api/logs/statistics # Log statistics
|
||||
POST /api/logs/rotate # Log rotation
|
||||
```
|
||||
|
||||
### **Service-Specific Endpoints**
|
||||
|
||||
```bash
|
||||
# Network Services
|
||||
GET /api/dns/records # DNS records
|
||||
POST /api/dns/records # Add DNS record
|
||||
DELETE /api/dns/records # Remove DNS record
|
||||
GET /api/dhcp/leases # DHCP leases
|
||||
POST /api/dhcp/reservations # Add DHCP reservation
|
||||
GET /api/ntp/status # NTP status
|
||||
GET /api/network/info # Network information
|
||||
POST /api/network/test # Network connectivity test
|
||||
|
||||
# WireGuard & Peers
|
||||
GET /api/wireguard/keys # WireGuard keys
|
||||
POST /api/wireguard/keys/peer # Generate peer keys
|
||||
GET /api/wireguard/config # WireGuard configuration
|
||||
GET /api/wireguard/peers # List peers
|
||||
POST /api/wireguard/peers # Add peer
|
||||
DELETE /api/wireguard/peers # Remove peer
|
||||
GET /api/wireguard/status # WireGuard status
|
||||
POST /api/wireguard/connectivity # Connectivity test
|
||||
PUT /api/wireguard/peers/ip # Update peer IP
|
||||
|
||||
# Digital Services
|
||||
GET /api/email/users # Email users
|
||||
POST /api/email/users # Add email user
|
||||
DELETE /api/email/users/<user> # Remove email user
|
||||
GET /api/email/status # Email service status
|
||||
GET /api/email/connectivity # Email connectivity
|
||||
POST /api/email/send # Send email
|
||||
GET /api/email/mailbox/<user> # User mailbox
|
||||
|
||||
GET /api/calendar/users # Calendar users
|
||||
POST /api/calendar/users # Add calendar user
|
||||
DELETE /api/calendar/users/<user> # Remove calendar user
|
||||
POST /api/calendar/calendars # Create calendar
|
||||
POST /api/calendar/events # Add event
|
||||
GET /api/calendar/events/<user>/<calendar> # List events
|
||||
GET /api/calendar/status # Calendar service status
|
||||
GET /api/calendar/connectivity # Calendar connectivity
|
||||
|
||||
GET /api/files/users # File users
|
||||
POST /api/files/users # Add file user
|
||||
DELETE /api/files/users/<user> # Remove file user
|
||||
POST /api/files/folders # Create folder
|
||||
DELETE /api/files/folders/<user>/<path> # Remove folder
|
||||
POST /api/files/upload/<user> # Upload file
|
||||
GET /api/files/download/<user>/<path> # Download file
|
||||
DELETE /api/files/delete/<user>/<path> # Delete file
|
||||
GET /api/files/list/<user> # List files
|
||||
GET /api/files/status # File service status
|
||||
GET /api/files/connectivity # File connectivity
|
||||
|
||||
# Routing & Security
|
||||
GET /api/routing/status # Routing status
|
||||
POST /api/routing/nat # Add NAT rule
|
||||
DELETE /api/routing/nat/<id> # Remove NAT rule
|
||||
POST /api/routing/peers # Add peer route
|
||||
DELETE /api/routing/peers/<peer> # Remove peer route
|
||||
POST /api/routing/exit-nodes # Add exit node
|
||||
POST /api/routing/bridge # Add bridge route
|
||||
POST /api/routing/split # Add split route
|
||||
POST /api/routing/firewall # Add firewall rule
|
||||
POST /api/routing/connectivity # Routing connectivity test
|
||||
GET /api/routing/logs # Routing logs
|
||||
GET /api/routing/nat # List NAT rules
|
||||
GET /api/routing/peers # List peer routes
|
||||
GET /api/routing/firewall # List firewall rules
|
||||
|
||||
GET /api/vault/status # Vault status
|
||||
GET /api/vault/certificates # List certificates
|
||||
POST /api/vault/certificates # Generate certificate
|
||||
DELETE /api/vault/certificates/<name> # Revoke certificate
|
||||
GET /api/vault/ca/certificate # CA certificate
|
||||
GET /api/vault/age/public-key # Age public key
|
||||
GET /api/vault/trust/keys # Trusted keys
|
||||
POST /api/vault/trust/keys # Add trusted key
|
||||
DELETE /api/vault/trust/keys/<name> # Remove trusted key
|
||||
POST /api/vault/trust/verify # Verify trust
|
||||
GET /api/vault/trust/chains # Trust chains
|
||||
```
|
||||
|
||||
## 💻 Enhanced CLI
|
||||
|
||||
### **CLI Features**
|
||||
|
||||
```bash
|
||||
# Interactive mode with tab completion
|
||||
python api/enhanced_cli.py --interactive
|
||||
|
||||
# Batch operations
|
||||
python api/enhanced_cli.py --batch "status" "services" "health"
|
||||
|
||||
# Configuration management
|
||||
python api/enhanced_cli.py --export-config json
|
||||
python api/enhanced_cli.py --import-config config.json
|
||||
|
||||
# Service wizards
|
||||
python api/enhanced_cli.py --wizard network
|
||||
python api/enhanced_cli.py --wizard email
|
||||
|
||||
# Health monitoring
|
||||
python api/enhanced_cli.py --health
|
||||
python api/enhanced_cli.py --logs network
|
||||
|
||||
# Service status
|
||||
python api/enhanced_cli.py --status
|
||||
python api/enhanced_cli.py --services
|
||||
python api/enhanced_cli.py --peers
|
||||
```
|
||||
|
||||
### **CLI Capabilities**
|
||||
- **Interactive Mode**: Tab completion, command history, help system
|
||||
- **Batch Operations**: Execute multiple commands in sequence
|
||||
- **Configuration Wizards**: Guided setup for complex services
|
||||
- **Real-time Monitoring**: Live status updates and health checks
|
||||
- **Log Management**: View, search, and export service logs
|
||||
- **Service Management**: Start, stop, restart, and configure services
|
||||
|
||||
## 🔒 Security Model
|
||||
|
||||
### **Certificate Management**
|
||||
- **Self-hosted CA**: Issue and manage TLS certificates for all services
|
||||
- **Certificate Lifecycle**: Generate, renew, revoke, and monitor certificates
|
||||
- **Trust Management**: Direct, indirect, and verified trust relationships
|
||||
- **Age Encryption**: Modern encryption for sensitive data and keys
|
||||
|
||||
### **Network Security**
|
||||
- **WireGuard VPN**: Secure peer-to-peer communication with key rotation
|
||||
- **Firewall & NAT**: Granular control over network access and routing
|
||||
- **Service Isolation**: Docker containers for each service
|
||||
- **Input Validation**: All API endpoints validate and sanitize input
|
||||
|
||||
### **Data Protection**
|
||||
- **Encrypted Storage**: Sensitive data encrypted at rest using Age/Fernet
|
||||
- **Secure Communication**: TLS for all API endpoints and service communication
|
||||
- **Access Control**: Role-based access for services and API endpoints
|
||||
- **Audit Logging**: Comprehensive security event logging and monitoring
|
||||
|
||||
## 🧪 Testing & Quality Assurance
|
||||
|
||||
### **Test Coverage**
|
||||
- **BaseServiceManager**: 100% coverage
|
||||
- **ConfigManager**: 95%+ coverage
|
||||
- **ServiceBus**: 95%+ coverage
|
||||
- **LogManager**: 95%+ coverage
|
||||
- **All Service Managers**: 77%+ overall coverage
|
||||
- **API Endpoints**: 100% endpoint coverage
|
||||
|
||||
### **Test Types**
|
||||
- **Unit Tests**: Individual component testing
|
||||
- **Integration Tests**: Service interaction testing
|
||||
- **API Tests**: Endpoint functionality testing
|
||||
- **Error Handling**: Exception and edge case testing
|
||||
- **Performance Tests**: Load and stress testing
|
||||
|
||||
### **Testing Commands**
|
||||
```bash
|
||||
# Run all tests
|
||||
python api/test_enhanced_api.py
|
||||
|
||||
# Run specific test suites
|
||||
python -m pytest api/tests/test_network_manager.py
|
||||
python -m pytest api/tests/test_service_bus.py
|
||||
|
||||
# Generate coverage report
|
||||
coverage run -m pytest api/tests/
|
||||
coverage html
|
||||
```
|
||||
|
||||
## 📝 Usage Examples
|
||||
|
||||
### **Add DNS Record**
|
||||
```bash
|
||||
curl -X POST http://localhost:3000/api/dns/records \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"name": "www",
|
||||
"type": "A",
|
||||
"value": "192.168.1.100",
|
||||
"ttl": 300
|
||||
}'
|
||||
```
|
||||
|
||||
### **Register Peer**
|
||||
```bash
|
||||
curl -X POST http://localhost:3000/api/wireguard/peers \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"name": "bob",
|
||||
"ip": "203.0.113.22",
|
||||
"public_key": "peer_public_key_here",
|
||||
"allowed_networks": ["10.0.0.0/24"]
|
||||
}'
|
||||
```
|
||||
|
||||
### **Generate Certificate**
|
||||
```bash
|
||||
curl -X POST http://localhost:3000/api/vault/certificates \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"common_name": "myapp.example.com",
|
||||
"domains": ["myapp.example.com", "www.myapp.example.com"],
|
||||
"days": 365
|
||||
}'
|
||||
```
|
||||
|
||||
### **Configure NAT Rule**
|
||||
```bash
|
||||
curl -X POST http://localhost:3000/api/routing/nat \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"source_network": "10.0.0.0/24",
|
||||
"target_interface": "eth0",
|
||||
"nat_type": "MASQUERADE",
|
||||
"protocol": "ALL"
|
||||
}'
|
||||
```
|
||||
|
||||
## 🛠️ Development & Deployment
|
||||
|
||||
### **Development Setup**
|
||||
```bash
|
||||
# Install dependencies
|
||||
pip install -r api/requirements.txt
|
||||
|
||||
# Start development server
|
||||
python api/app.py
|
||||
|
||||
# Run tests
|
||||
python api/test_enhanced_api.py
|
||||
|
||||
# Start frontend (if available)
|
||||
cd webui && bun install && npm run dev
|
||||
```
|
||||
|
||||
### **Production Deployment**
|
||||
```bash
|
||||
# Docker deployment
|
||||
docker-compose up --build -d
|
||||
|
||||
# Health check
|
||||
curl http://localhost:3000/health
|
||||
|
||||
# Service status
|
||||
curl http://localhost:3000/api/services/status
|
||||
```
|
||||
|
||||
### **Service Development**
|
||||
```python
|
||||
from base_service_manager import BaseServiceManager
|
||||
|
||||
class MyServiceManager(BaseServiceManager):
|
||||
def __init__(self, data_dir='/app/data', config_dir='/app/config'):
|
||||
super().__init__('myservice', data_dir, config_dir)
|
||||
|
||||
def get_status(self) -> Dict[str, Any]:
|
||||
# Implement service status
|
||||
return {
|
||||
'running': True,
|
||||
'status': 'online',
|
||||
'timestamp': datetime.utcnow().isoformat()
|
||||
}
|
||||
|
||||
def test_connectivity(self) -> Dict[str, Any]:
|
||||
# Implement connectivity test
|
||||
return {
|
||||
'success': True,
|
||||
'message': 'Service connectivity working',
|
||||
'timestamp': datetime.utcnow().isoformat()
|
||||
}
|
||||
```
|
||||
|
||||
## 🚀 Future Enhancements
|
||||
|
||||
### **Planned Features**
|
||||
- **Certificate Auto-renewal**: Automatic certificate renewal and monitoring
|
||||
- **Web of Trust Models**: Advanced trust relationship management
|
||||
- **Certificate Transparency**: CT log integration and monitoring
|
||||
- **Hardware Security Module (HSM)**: HSM integration for key management
|
||||
- **WebSocket Updates**: Real-time service status updates
|
||||
- **Advanced Monitoring**: Metrics collection and alerting systems
|
||||
- **Mobile App**: Mobile application for remote management
|
||||
- **Plugin System**: Extensible architecture for custom services
|
||||
|
||||
### **Architecture Improvements**
|
||||
- **Service Discovery**: Dynamic service registration and discovery
|
||||
- **Load Balancing**: Multi-instance service deployment
|
||||
- **Advanced Caching**: Redis-based caching for performance
|
||||
- **Message Queues**: RabbitMQ/Kafka for reliable messaging
|
||||
- **Distributed Tracing**: OpenTelemetry integration
|
||||
- **Configuration Management**: GitOps-style configuration management
|
||||
|
||||
## 📊 Project Status
|
||||
|
||||
### **✅ Completed Features**
|
||||
- **Production-Grade Architecture**: BaseServiceManager pattern implemented
|
||||
- **Event-Driven Communication**: Service bus with real-time events
|
||||
- **Centralized Configuration**: Type-safe configuration with validation
|
||||
- **Comprehensive Logging**: Structured logging with search and export
|
||||
- **Enhanced CLI**: Interactive CLI with batch operations
|
||||
- **Health Monitoring**: Real-time health checks across all services
|
||||
- **Security Framework**: Self-hosted CA, encryption, and trust management
|
||||
- **Complete API**: RESTful API with comprehensive documentation
|
||||
- **Testing Framework**: Comprehensive test suite with high coverage
|
||||
|
||||
### **🎯 Current Status**
|
||||
- **All Services**: 10 service managers fully implemented and integrated
|
||||
- **API Server**: Running on port 3000 with all endpoints functional
|
||||
- **CLI Tool**: Enhanced CLI with all features working
|
||||
- **Test Coverage**: 77%+ overall coverage with comprehensive testing
|
||||
- **Documentation**: Complete documentation for all components
|
||||
- **Production Ready**: Suitable for personal and small business deployment
|
||||
|
||||
### **🌟 Key Achievements**
|
||||
- **Unified Architecture**: All services follow the same patterns and interfaces
|
||||
- **Event-Driven Design**: Services communicate and orchestrate automatically
|
||||
- **Configuration Management**: Centralized, validated configuration system
|
||||
- **Comprehensive Logging**: Production-grade logging with advanced features
|
||||
- **Enhanced CLI**: Professional command-line interface for management
|
||||
- **Health Monitoring**: Real-time monitoring and alerting capabilities
|
||||
- **Security Framework**: Enterprise-grade security with modern cryptography
|
||||
- **Complete Testing**: Comprehensive test suite ensuring reliability
|
||||
The goal is to give a person full ownership of their core internet services on their own hardware, without relying on cloud providers.
|
||||
|
||||
---
|
||||
|
||||
**The Personal Internet Cell empowers users with full control over their digital infrastructure, combining privacy, security, and usability in a single, production-ready, self-hosted platform.** 🌟
|
||||
## Table of Contents
|
||||
|
||||
1. [Architecture](#architecture)
|
||||
2. [Service Managers](#service-managers)
|
||||
3. [First-Run Wizard](#first-run-wizard)
|
||||
4. [Authentication](#authentication)
|
||||
5. [API Reference](#api-reference)
|
||||
6. [DDNS](#ddns)
|
||||
7. [Services UI](#services-ui)
|
||||
8. [Service Store (Add-ons)](#service-store-add-ons)
|
||||
9. [Cell-to-Cell Networking](#cell-to-cell-networking)
|
||||
10. [Extended Connectivity](#extended-connectivity)
|
||||
11. [Security Model](#security-model)
|
||||
12. [Testing](#testing)
|
||||
13. [Development](#development)
|
||||
|
||||
---
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
Browser / WireGuard peer
|
||||
└── Caddy (:80/:443) reverse proxy, TLS termination
|
||||
└── React SPA (:8081) Vite + Tailwind (Nginx in container)
|
||||
└── Flask API (:3000) REST API, bound to 127.0.0.1
|
||||
├── NetworkManager CoreDNS, dnsmasq, chrony
|
||||
├── WireGuardManager WireGuard VPN peer lifecycle
|
||||
├── PeerRegistry peer registration and trust
|
||||
├── EmailManager Postfix + Dovecot
|
||||
├── CalendarManager Radicale CalDAV/CardDAV
|
||||
├── FileManager WebDAV + Filegator
|
||||
├── RoutingManager iptables NAT and routing
|
||||
├── FirewallManager iptables firewall rules
|
||||
├── VaultManager internal CA, cert lifecycle, Age encryption
|
||||
├── ContainerManager Docker SDK
|
||||
├── CellLinkManager cell-to-cell WireGuard links
|
||||
├── ConnectivityManager exit routing (WG ext, OpenVPN, Tor)
|
||||
├── DDNSManager dynamic DNS heartbeat
|
||||
├── ServiceStoreManager optional service install/remove
|
||||
├── CaddyManager Caddyfile generation and reload
|
||||
├── AuthManager session auth, RBAC
|
||||
└── SetupManager first-run wizard state
|
||||
```
|
||||
|
||||
The 7 core containers run on a Docker bridge network (`cell-network`, `172.20.0.0/16` default). Static IPs per container are defined in `docker-compose.yml`. Installed optional services join the same network via their own compose projects, managed by `ServiceComposer`.
|
||||
|
||||
Runtime configuration lives in `config/api/cell_config.json`, managed by `ConfigManager`. All service managers read and write through `ConfigManager`, which validates and backs up automatically.
|
||||
|
||||
---
|
||||
|
||||
## Service Managers
|
||||
|
||||
All managers inherit `BaseServiceManager` (`api/base_service_manager.py`), which provides:
|
||||
- `get_status()` — current running state
|
||||
- `get_config()` / `update_config()` — config read/write
|
||||
- `test_connectivity()` — reachability check
|
||||
- `get_logs()` — last N lines from the service log
|
||||
- `restart_service()` — container restart via Docker SDK
|
||||
|
||||
The `ServiceBus` (`api/service_bus.py`) handles pub/sub events between managers (e.g., `CONFIG_CHANGED`, `SERVICE_STARTED`). Dependencies are declared in the bus (wireguard depends on network; email depends on network and vault).
|
||||
|
||||
### Manager summary
|
||||
|
||||
| Manager | Responsibilities |
|
||||
|---|---|
|
||||
| `NetworkManager` | CoreDNS zone files, dnsmasq DHCP config and lease monitoring, chrony NTP |
|
||||
| `WireGuardManager` | Key generation, `wg0.conf` generation, peer add/remove, route sync |
|
||||
| `PeerRegistry` | Peer registration, trust tracking, peer statistics |
|
||||
| `EmailManager` | docker-mailserver accounts, mailbox config, alias management |
|
||||
| `CalendarManager` | Radicale user/calendar/contacts lifecycle |
|
||||
| `FileManager` | WebDAV user directories, Filegator access |
|
||||
| `RoutingManager` | NAT rules, per-peer routing policy, fwmark-based exit routing |
|
||||
| `FirewallManager` | iptables INPUT/FORWARD/OUTPUT rule management |
|
||||
| `VaultManager` | Internal CA (self-signed root), TLS cert issue/revoke, Age public key |
|
||||
| `ContainerManager` | Docker container/image/volume management via SDK |
|
||||
| `CellLinkManager` | Site-to-site WireGuard links to other PIC cells, peer-sync protocol |
|
||||
| `ConnectivityManager` | Per-peer exit routing via WireGuard external, OpenVPN, or Tor |
|
||||
| `DDNSManager` | Public IP heartbeat, provider abstraction (pic_ngo, cloudflare, duckdns, noip, freedns) |
|
||||
| `ServiceStoreManager` | Fetch manifest index, install/remove optional services |
|
||||
| `CaddyManager` | Caddyfile generation, reload-on-change |
|
||||
| `AuthManager` | bcrypt password store, session management, admin/peer RBAC |
|
||||
| `SetupManager` | First-run wizard state, setup-complete flag |
|
||||
|
||||
---
|
||||
|
||||
## First-Run Wizard
|
||||
|
||||
On first start, `SetupManager.is_setup_complete()` returns `False`. The `enforce_setup` before-request hook returns HTTP 428 for all `/api/*` requests except `/api/setup/*` and `/health`, redirecting clients to `/setup`.
|
||||
|
||||
The wizard collects:
|
||||
- **Cell name** — used for hostnames and DDNS subdomain (e.g. `myhome` → `myhome.pic.ngo`)
|
||||
- **Domain mode** — determines TLS certificate source: `lan` (internal CA), `pic_ngo`, `cloudflare`, `duckdns`, `http01`
|
||||
- **Timezone**
|
||||
- **Services to install** — optional services (email, calendar, files) to install after setup; each starts a background install via `ServiceStoreManager`
|
||||
- **Admin password** — minimum 12 characters
|
||||
|
||||
On completion:
|
||||
1. Admin account is created in `data/auth_users.json`
|
||||
2. Cell identity is written to `config/api/cell_config.json`
|
||||
3. Caddy config is generated
|
||||
4. If domain mode is `pic_ngo`, the cell registers `<name>.pic.ngo` with the DDNS service
|
||||
5. Each selected service is installed in a background thread
|
||||
|
||||
Wizard endpoints: `GET/POST /api/setup/step`, `GET /api/setup/status`, `POST /api/setup/complete`.
|
||||
|
||||
---
|
||||
|
||||
## Authentication
|
||||
|
||||
`AuthManager` stores bcrypt-hashed credentials in `data/auth_users.json`. Two roles:
|
||||
|
||||
| Role | Access |
|
||||
|---|---|
|
||||
| `admin` | All `/api/*` endpoints except `/api/peer/*` |
|
||||
| `peer` | `/api/peer/*` only (peer dashboard, key exchange) |
|
||||
|
||||
Session auth flow:
|
||||
- `POST /api/auth/login` — creates a Flask session
|
||||
- `GET /api/auth/me` — current session info
|
||||
- `POST /api/auth/logout` — clears session
|
||||
- `POST /api/auth/change-password` — change own password
|
||||
- `POST /api/auth/admin/reset-password` — admin resets another user's password
|
||||
|
||||
CSRF protection: all `POST`, `PUT`, `DELETE`, `PATCH` on `/api/*` (except `/api/auth/*` and `/api/setup/*`) require the `X-CSRF-Token` header matching the session token, obtained via `GET /api/auth/csrf-token`.
|
||||
|
||||
Cell-to-cell peer-sync endpoints (`/api/cells/peer-sync/*`) use source-IP + WireGuard public key auth, not session cookies.
|
||||
|
||||
Auth enforcement is active once any user exists in the store. If the store is empty (fresh install before wizard), all requests bypass auth — `enforce_setup` already blocks them with 428.
|
||||
|
||||
---
|
||||
|
||||
## API Reference
|
||||
|
||||
**Base URL:** `http://localhost:3000`
|
||||
**Auth:** session cookie (`X-CSRF-Token` header required for mutations)
|
||||
|
||||
### Core
|
||||
|
||||
| Method | Path | Description |
|
||||
|---|---|---|
|
||||
| GET | `/health` | Health check (always public) |
|
||||
| GET | `/api/status` | All-service status summary |
|
||||
| GET | `/api/config` | Full cell config |
|
||||
| PUT | `/api/config` | Update cell config |
|
||||
| GET | `/api/health/history` | Recent health check history |
|
||||
|
||||
### Auth (`/api/auth/`)
|
||||
|
||||
| Method | Path | Description |
|
||||
|---|---|---|
|
||||
| POST | `/api/auth/login` | Create session |
|
||||
| POST | `/api/auth/logout` | Destroy session |
|
||||
| GET | `/api/auth/me` | Current user info |
|
||||
| GET | `/api/auth/csrf-token` | Get CSRF token |
|
||||
| POST | `/api/auth/change-password` | Change own password |
|
||||
| POST | `/api/auth/admin/reset-password` | Admin: reset another user's password |
|
||||
| GET | `/api/auth/users` | Admin: list users |
|
||||
|
||||
### Setup (`/api/setup/`)
|
||||
|
||||
| Method | Path | Description |
|
||||
|---|---|---|
|
||||
| GET | `/api/setup/status` | Setup complete flag + current step |
|
||||
| GET | `/api/setup/step` | Current wizard step data |
|
||||
| POST | `/api/setup/step` | Submit current step |
|
||||
| POST | `/api/setup/complete` | Finalize setup |
|
||||
|
||||
### Network Services (`/api/dns/`, `/api/dhcp/`, `/api/ntp/`, `/api/network/`)
|
||||
|
||||
DNS records, DHCP leases and reservations, NTP status, network connectivity test.
|
||||
|
||||
### WireGuard (`/api/wireguard/`, `/api/peers/`)
|
||||
|
||||
Peer add/remove, key generation, QR code export, per-peer routing policy, WireGuard status.
|
||||
|
||||
### Email (`/api/email/`) _(available when email service is installed)_
|
||||
|
||||
User account management, mailbox config, alias management, connectivity test. Returns HTTP 404 when the email service is not installed (except `/api/email/status`).
|
||||
|
||||
### Calendar (`/api/calendar/`) _(available when calendar service is installed)_
|
||||
|
||||
User, calendar, and contacts (CardDAV) management. Returns HTTP 404 when the calendar service is not installed (except `/api/calendar/status`).
|
||||
|
||||
### Files (`/api/files/`) _(available when files service is installed)_
|
||||
|
||||
WebDAV user management, file upload/download/delete, folder management. Returns HTTP 404 when the files service is not installed (except `/api/files/status`).
|
||||
|
||||
### Routing (`/api/routing/`)
|
||||
|
||||
NAT rules, peer routes, exit node configuration.
|
||||
|
||||
### Vault (`/api/vault/`)
|
||||
|
||||
Certificate issue/revoke, CA certificate, trust key management, Age public key.
|
||||
|
||||
### Containers (`/api/containers/`)
|
||||
|
||||
List, start, stop, inspect containers; manage images and volumes.
|
||||
|
||||
### Cell Network (`/api/cells/`)
|
||||
|
||||
List connected cells, add/remove cell links, peer-sync.
|
||||
|
||||
### Connectivity (`/api/connectivity/`)
|
||||
|
||||
List exit nodes, configure WireGuard external / OpenVPN / Tor exits, assign per-peer exit policy.
|
||||
|
||||
### Service Store (`/api/store/`)
|
||||
|
||||
List available services, install, remove.
|
||||
|
||||
### Logs (`/api/logs/`)
|
||||
|
||||
Per-service log retrieval, log search, log statistics.
|
||||
|
||||
---
|
||||
|
||||
## DDNS
|
||||
|
||||
`DDNSManager` maintains a `<cell-name>.pic.ngo` DNS A record pointing at the cell's public IP. A background thread runs every 5 minutes and calls `provider.update(token, ip)` only when the IP changes.
|
||||
|
||||
Registration happens during the setup wizard (if domain mode is `pic_ngo`) via `provider.register(name, ip)`, which returns a bearer token stored in `data/api/.ddns_token`.
|
||||
|
||||
DDNS config lives in `cell_config.json` under the top-level `ddns` key:
|
||||
|
||||
```json
|
||||
{
|
||||
"ddns": {
|
||||
"provider": "pic_ngo",
|
||||
"api_base_url": "https://ddns.pic.ngo",
|
||||
"totp_secret": "<base32 secret>"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Registration requires a time-based OTP (`X-Register-OTP` header) derived from the shared `REGISTER_TOTP_SECRET` on the DDNS server. This prevents unauthorized subdomain registration.
|
||||
|
||||
Supported providers: `pic_ngo`, `cloudflare`, `duckdns`, `noip`, `freedns`.
|
||||
|
||||
---
|
||||
|
||||
## Services UI
|
||||
|
||||
### Navigation
|
||||
|
||||
The left-hand navigation contains a **Services** group. Both admin and peer users see it. Sub-items for installed services (Email, Calendar, Files, etc.) are added dynamically: the UI fetches `GET /api/services/active` on load and after each install/uninstall. Services not yet installed do not appear in the nav.
|
||||
|
||||
Legacy paths redirect to their new canonical locations:
|
||||
|
||||
| Old path | New path |
|
||||
|---|---|
|
||||
| `/email` | `/services/email` |
|
||||
| `/calendar` | `/services/calendar` |
|
||||
| `/files` | `/services/files` |
|
||||
| `/store` | `/services` |
|
||||
|
||||
### Services page (`/services`)
|
||||
|
||||
A single unified catalog of all available services from the store index. Each card shows:
|
||||
- Service name, description, version
|
||||
- **Install** button (not installed) or **Uninstall** button (installed)
|
||||
- **Open** link for installed services (navigates to the service sub-page)
|
||||
- Running/stopped status dot for installed services
|
||||
|
||||
The `pic-services-changed` custom DOM event is dispatched after install/uninstall, causing the nav to re-fetch active services immediately.
|
||||
|
||||
### Service sub-pages — admin view
|
||||
|
||||
Each sub-page at `/services/email`, `/services/calendar`, and `/services/files` shows:
|
||||
|
||||
1. **Connection info** — hostnames, ports, and protocol details (e.g. IMAP/SMTP/Webmail, CalDAV/CardDAV, WebDAV/Filegator).
|
||||
2. **Service status** — current running state fetched from the API.
|
||||
3. **Users list** — accounts registered with that service.
|
||||
4. **Inline config form** — editable fields for that service's settings.
|
||||
|
||||
If the service is not installed, the page shows a `ServiceNotInstalledBanner` with a link to the catalog for admins, or a "contact your admin" message for peer users. All non-status API routes for uninstalled services return HTTP 404.
|
||||
|
||||
Config forms save automatically with an 800 ms debounce after the last change.
|
||||
|
||||
### Service sub-pages — peer view
|
||||
|
||||
Peers access the same URLs. The peer view shows only:
|
||||
|
||||
- Connection info (hostnames, ports, copy buttons).
|
||||
- Personal credentials for that service, fetched from `/api/peer/*`.
|
||||
|
||||
The config form and users list are not shown to peers.
|
||||
|
||||
### Settings page
|
||||
|
||||
The Email, Calendar, and Files configuration forms have been removed from the Settings page. Settings now covers: Identity, DDNS, Network (DNS/DHCP/NTP), WireGuard, Routing & Firewall, Vault & Trust, and Backup & Restore.
|
||||
|
||||
### Relevant API endpoints
|
||||
|
||||
| Method | Path | Description |
|
||||
|---|---|---|
|
||||
| GET | `/api/services/active` | List installed services with id, name, subdomain, capabilities |
|
||||
| GET | `/api/config` | Full cell config, includes `installed_services` dict |
|
||||
|
||||
---
|
||||
|
||||
## Service Store (Add-ons)
|
||||
|
||||
Email, calendar, and file storage are store services — not part of the core stack. All optional functionality ships through this mechanism.
|
||||
|
||||
`ServiceStoreManager` fetches a manifest index from `https://git.pic.ngo/roof/pic-services/raw/branch/main/index.json`. Each manifest (`schema_version: 3`) declares:
|
||||
|
||||
- Container image and compose template
|
||||
- Caddy subdomain routes
|
||||
- Capabilities: `has_subdomain`, `has_accounts`, `has_admin_config`, `has_storage`, `has_egress`
|
||||
- Account provisioning interface (`accounts.manager`)
|
||||
- Backup declarations (`backup.volumes`, `backup.config_paths`)
|
||||
- Egress routing policy (`egress.allowed`)
|
||||
- Per-peer connection info template (`peer_config_template`)
|
||||
|
||||
`POST /api/store/install` fetches the manifest and compose template, validates them, renders the template with PIC-specific variables (`${PIC_DOMAIN}`, `${PIC_DATA_DIR}`, etc.), writes a per-service compose file, and brings the containers up via `ServiceComposer`. Caddy routes and DNS entries are applied automatically.
|
||||
|
||||
`POST /api/store/remove` checks for dependent services, stops and removes containers, and regenerates Caddy.
|
||||
|
||||
**`ServiceComposer`** (`api/service_composer.py`) manages the per-service compose lifecycle independently of the main stack. Each service gets its own compose project at `data/services/<id>/docker-compose.yml`. On startup, `reapply_active_services()` brings up containers for all recorded installs.
|
||||
|
||||
See `docs/service-developer-guide.md` for the full manifest schema reference and submission process.
|
||||
|
||||
---
|
||||
|
||||
## Cell-to-Cell Networking
|
||||
|
||||
`CellLinkManager` manages WireGuard site-to-site tunnels between PIC cells. Each link is a WireGuard peer configured with a dedicated `/32` address and allowed-IPs covering the remote cell's subnet.
|
||||
|
||||
The peer-sync protocol (`/api/cells/peer-sync/`) exchanges public keys and allowed networks between cells using source-IP + WireGuard public key authentication (no session required).
|
||||
|
||||
Access control is per-service (calendar, files, mail, WebDAV) and enforced at the iptables level.
|
||||
|
||||
---
|
||||
|
||||
## Extended Connectivity
|
||||
|
||||
`ConnectivityManager` provides per-peer exit routing: traffic from a specific WireGuard peer can be routed through an alternate exit instead of the cell's default gateway.
|
||||
|
||||
Supported exits:
|
||||
- **WireGuard external** — another WireGuard endpoint (e.g. a VPS)
|
||||
- **OpenVPN** — OpenVPN client running in a container
|
||||
- **Tor** — Tor SOCKS proxy with transparent redirection
|
||||
|
||||
Routing uses fwmark and `ip rule` / `ip route` in separate routing tables. Configuration is via `PUT /api/connectivity/peers/<peer_name>/exit`.
|
||||
|
||||
---
|
||||
|
||||
## Security Model
|
||||
|
||||
- **No open ports for the API** — Flask API binds to `127.0.0.1:3000` only; Caddy proxies HTTPS requests to it.
|
||||
- **Session auth** — bcrypt passwords, Flask server-side sessions, CSRF double-submit.
|
||||
- **Setup wizard gate** — all `/api/*` requests return 428 until setup is complete.
|
||||
- **Role separation** — admin cannot access peer endpoints; peer cannot access admin endpoints.
|
||||
- **HTTPS everywhere** — Caddy handles TLS; internal services are reached via reverse proxy paths.
|
||||
- **Internal CA** — VaultManager issues certificates for services that don't use Let's Encrypt.
|
||||
- **Docker socket isolation** — the Docker socket is mounted only into `cell-api`; other containers have no Docker access.
|
||||
- **iptables firewall** — FirewallManager manages INPUT/FORWARD rules; WireGuard peer isolation is enforced at the packet level.
|
||||
|
||||
---
|
||||
|
||||
## Testing
|
||||
|
||||
```bash
|
||||
make test # unit tests (pytest, ~1900+ functions)
|
||||
make test-coverage # coverage report in htmlcov/
|
||||
```
|
||||
|
||||
Test layout:
|
||||
- `tests/` — unit and endpoint tests; no running services required
|
||||
- `tests/integration/` — require a running PIC stack
|
||||
- `tests/e2e/` — Playwright UI tests and WireGuard integration tests
|
||||
|
||||
CI: Gitea Actions runs `pytest tests/ --ignore=tests/e2e --ignore=tests/integration` on every push.
|
||||
|
||||
---
|
||||
|
||||
## Development
|
||||
|
||||
```bash
|
||||
# Full stack in Docker
|
||||
make start
|
||||
make stop
|
||||
make logs
|
||||
|
||||
# Flask API without Docker (port 3000)
|
||||
pip install -r api/requirements.txt
|
||||
python api/app.py
|
||||
|
||||
# React UI dev server (port 5173, proxies /api → :3000)
|
||||
cd webui && npm install && npm run dev
|
||||
|
||||
# Rebuild containers after code change
|
||||
make build-api
|
||||
make build-webui
|
||||
```
|
||||
|
||||
Key files:
|
||||
- `api/app.py` — Flask app, blueprint registration, before-request hooks, health monitor thread
|
||||
- `api/managers.py` — singleton instantiation of all service managers
|
||||
- `api/base_service_manager.py` — abstract base class all managers implement
|
||||
- `api/config_manager.py` — `cell_config.json` read/write/validate/backup
|
||||
- `api/service_bus.py` — pub/sub event system
|
||||
- `webui/src/services/api.js` — Axios API client used by all UI pages
|
||||
- `docker-compose.yml` — container definitions and network topology
|
||||
- `Makefile` — all operational commands
|
||||
|
||||
+130
-132
@@ -1,139 +1,138 @@
|
||||
# Quick Start
|
||||
|
||||
This guide walks through a first-time PIC installation from a clean Linux host.
|
||||
This guide walks through a first-time PIC installation on a clean Linux host.
|
||||
|
||||
---
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Linux host with the WireGuard kernel module (`modprobe wireguard` to verify)
|
||||
- Docker Engine and Docker Compose installed
|
||||
- Python 3.10+ (needed for `make setup` only)
|
||||
- Linux x86-64 host — Debian, Ubuntu, Fedora, RHEL, or Alpine
|
||||
- 2 GB+ RAM, 10 GB+ disk
|
||||
- Always-required ports: 53, 80, 443, 51820/udp
|
||||
- Email service only (when installed): 25, 587, 993
|
||||
|
||||
The installer handles all software dependencies (git, docker, make, etc.) automatically.
|
||||
|
||||
---
|
||||
|
||||
## 1. Clone the repository
|
||||
## Option A — One-line installer (recommended)
|
||||
|
||||
```bash
|
||||
git clone <repo-url> pic
|
||||
curl -fsSL https://install.pic.ngo | sudo bash
|
||||
```
|
||||
|
||||
Always review the script before running it:
|
||||
|
||||
```bash
|
||||
curl -fsSL https://install.pic.ngo | less
|
||||
```
|
||||
|
||||
The installer:
|
||||
1. Detects your OS and installs Docker, git, make via the system package manager
|
||||
2. Creates a `pic` system user and adds it to the `docker` group
|
||||
3. Clones the repository to `/opt/pic`
|
||||
4. Runs `make install` (generates keys and config, writes a systemd unit)
|
||||
5. Runs `make start-core` to bring up the core containers
|
||||
6. Waits for the API to respond, then prints the wizard URL
|
||||
|
||||
When it finishes, open the URL it prints:
|
||||
|
||||
```
|
||||
http://<host-ip>:8081/setup
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Option B — Manual install
|
||||
|
||||
Use this if you want to control where PIC is installed, or if you are installing on a machine that already has Docker.
|
||||
|
||||
```bash
|
||||
git clone https://git.pic.ngo/roof/pic.git pic
|
||||
cd pic
|
||||
sudo make install
|
||||
make start-core
|
||||
```
|
||||
|
||||
Then open `http://<host-ip>:8081` in a browser.
|
||||
|
||||
---
|
||||
|
||||
## 2. Configure the environment
|
||||
## Complete the setup wizard
|
||||
|
||||
Copy the example environment file and edit it:
|
||||
The setup wizard appears automatically on first start. All API requests redirect to `/setup` until it is finished.
|
||||
|
||||
```bash
|
||||
cp .env.example .env
|
||||
```
|
||||
The wizard asks for:
|
||||
|
||||
Open `.env` and set at minimum:
|
||||
- **Cell name** — used for hostnames and DDNS subdomain. Lowercase letters, digits, hyphens, 2–31 characters. Example: `myhome`.
|
||||
- **Domain mode** — how HTTPS certificates are issued:
|
||||
- `pic_ngo` — automatic `<cell-name>.pic.ngo` subdomain with Let's Encrypt via DNS-01 (recommended for internet-facing cells)
|
||||
- `cloudflare` — Let's Encrypt via Cloudflare DNS-01 (bring your own domain)
|
||||
- `duckdns` — Let's Encrypt via DuckDNS DNS-01
|
||||
- `http01` — Let's Encrypt via HTTP-01 (no wildcard; cell must be reachable on port 80)
|
||||
- `lan` — internal CA, no internet required (for LAN-only installs)
|
||||
- **Timezone**
|
||||
- **Services to install** — email, calendar, files (optional; installed in the background after setup completes; can be added later via the Services store page)
|
||||
- **Admin password** — minimum 12 characters, must contain uppercase, lowercase, and a digit
|
||||
|
||||
```
|
||||
WEBDAV_PASS=changeme
|
||||
```
|
||||
|
||||
`WEBDAV_PASS` must be set before starting — the WebDAV container will fail to start without it.
|
||||
|
||||
All other variables have working defaults. See the Configuration section in [README.md](README.md) for the full list.
|
||||
Click **Complete Setup**. The wizard creates the admin account, writes cell identity to `config/api/cell_config.json`, and redirects to the login page. Any services you selected begin installing in the background.
|
||||
|
||||
---
|
||||
|
||||
## 3. Run setup
|
||||
## Log in
|
||||
|
||||
`make setup` installs system dependencies, generates WireGuard keys, and writes all required config files under `config/`:
|
||||
After the wizard you are redirected to `/login`.
|
||||
|
||||
```bash
|
||||
make check-deps # installs docker, python3-cryptography, etc. via apt
|
||||
make setup # generates keys and writes configs
|
||||
```
|
||||
|
||||
To customise the cell identity at setup time, pass overrides on the command line:
|
||||
|
||||
```bash
|
||||
CELL_NAME=myhome CELL_DOMAIN=cell VPN_ADDRESS=10.0.0.1/24 WG_PORT=51820 make setup
|
||||
```
|
||||
|
||||
`VPN_ADDRESS` must be an RFC-1918 address (e.g. `10.0.0.1/24`).
|
||||
- **Username:** `admin`
|
||||
- **Password:** the password you set in the wizard
|
||||
|
||||
---
|
||||
|
||||
## 4. Start the stack
|
||||
## Add a WireGuard peer
|
||||
|
||||
```bash
|
||||
make start
|
||||
```
|
||||
|
||||
This builds the `cell-api` and `cell-webui` images and starts all 13 containers. The first run takes a few minutes while images are pulled and built.
|
||||
|
||||
Check that everything came up:
|
||||
|
||||
```bash
|
||||
make status
|
||||
```
|
||||
|
||||
You should see all containers in the `Up` state and the API responding at `http://localhost:3000/health`.
|
||||
|
||||
---
|
||||
|
||||
## 5. Open the web UI
|
||||
|
||||
Open a browser and go to:
|
||||
|
||||
```
|
||||
http://<host-ip>:8081
|
||||
```
|
||||
|
||||
If you are running locally:
|
||||
|
||||
```
|
||||
http://localhost:8081
|
||||
```
|
||||
|
||||
The sidebar contains: Dashboard, Peers, Network Services, WireGuard, Email, Calendar, Files, Routing, Vault, Containers, Cell Network, Logs, Settings.
|
||||
|
||||
---
|
||||
|
||||
## 6. Set cell identity
|
||||
|
||||
Go to **Settings** in the sidebar.
|
||||
|
||||
Set your:
|
||||
- **Cell name** — a short identifier, e.g. `myhome`
|
||||
- **Domain** — the TLD your cell will use internally, e.g. `cell`
|
||||
- **VPN IP range** — the CIDR for WireGuard peers, e.g. `10.0.0.0/24`
|
||||
|
||||
After saving, the UI will show a banner asking you to apply the changes. Click **Apply Now**. The containers will restart briefly to pick up the new configuration.
|
||||
|
||||
---
|
||||
|
||||
## 7. Add a WireGuard peer
|
||||
|
||||
Go to **WireGuard** in the sidebar.
|
||||
Go to **Peers** in the sidebar.
|
||||
|
||||
1. Click **Add Peer**.
|
||||
2. Enter a name for the peer (e.g. `laptop`).
|
||||
2. Enter a peer name (e.g. `laptop`).
|
||||
3. The API generates a key pair and assigns the next available VPN IP automatically.
|
||||
4. Click the QR code icon to display the peer config as a QR code.
|
||||
4. Click the QR code icon to display the peer configuration as a QR code.
|
||||
5. Scan the QR code with a WireGuard client (Android, iOS, or the WireGuard desktop app).
|
||||
|
||||
The peer config sets your cell as the DNS server. Once connected, `*.cell` names resolve through the cell's CoreDNS.
|
||||
|
||||
To manage peers from the command line:
|
||||
|
||||
```bash
|
||||
make list-peers
|
||||
make add-peer PEER_NAME=phone PEER_IP=10.0.0.3 PEER_KEY=<base64-pubkey>
|
||||
```
|
||||
Once connected, `*.cell` names resolve through the cell's CoreDNS and traffic can be routed through the cell.
|
||||
|
||||
---
|
||||
|
||||
## 8. Day-to-day operations
|
||||
## Installing and managing services
|
||||
|
||||
Email, calendar, and file storage are optional services installed from the built-in service store. They are not running by default.
|
||||
|
||||
**To install a service:**
|
||||
|
||||
1. Go to **Services** in the sidebar.
|
||||
2. Find the service card (Email, Calendar, Files, or any other listed service).
|
||||
3. Click **Install**. PIC fetches the manifest, starts the container, and wires up DNS and Caddy routes automatically.
|
||||
4. The service appears in the sidebar navigation once installation completes.
|
||||
|
||||
**To check service status:**
|
||||
|
||||
The Services page shows each installed service as "running" or "stopped". You can also check via the API:
|
||||
|
||||
```bash
|
||||
curl -s http://<host-ip>:3000/api/services/active
|
||||
```
|
||||
|
||||
**To uninstall a service:**
|
||||
|
||||
Click **Uninstall** on the service card. The container is stopped and removed. Data in `data/services/<id>/` is kept on disk unless you delete it manually.
|
||||
|
||||
---
|
||||
|
||||
## Day-to-day operations
|
||||
|
||||
```bash
|
||||
# Check container status and API health
|
||||
make status
|
||||
|
||||
# Follow logs from all services
|
||||
make logs
|
||||
|
||||
@@ -142,9 +141,6 @@ make logs-api
|
||||
make logs-wireguard
|
||||
make logs-caddy
|
||||
|
||||
# Check container status and API health
|
||||
make status
|
||||
|
||||
# Open a shell inside a container
|
||||
make shell-api
|
||||
make shell-dns
|
||||
@@ -152,20 +148,11 @@ make shell-dns
|
||||
|
||||
---
|
||||
|
||||
## 9. Backup
|
||||
|
||||
Before making significant changes, create a backup:
|
||||
## Backup and restore
|
||||
|
||||
```bash
|
||||
make backup
|
||||
```
|
||||
|
||||
This archives `config/` and `data/` into `backups/cell-backup-<timestamp>.tar.gz`.
|
||||
|
||||
To list available backups:
|
||||
|
||||
```bash
|
||||
make restore
|
||||
make backup # archives config/ and data/ into backups/cell-backup-<timestamp>.tar.gz
|
||||
make restore # list available backups
|
||||
```
|
||||
|
||||
To restore manually:
|
||||
@@ -175,34 +162,38 @@ tar -xzf backups/cell-backup-YYYYMMDD-HHMMSS.tar.gz
|
||||
make start
|
||||
```
|
||||
|
||||
Backup and restore is also available in the UI under **Settings**.
|
||||
---
|
||||
|
||||
## Updating PIC
|
||||
|
||||
```bash
|
||||
make update # git pull + rebuild + restart
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 10. Updating PIC
|
||||
## Uninstalling
|
||||
|
||||
```bash
|
||||
make update
|
||||
make uninstall # stops containers; prompts to also delete config/ and data/
|
||||
```
|
||||
|
||||
This runs `git pull`, then rebuilds and restarts all containers. If `config/` is missing (e.g. after a fresh clone), it runs `make setup` automatically.
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
**Containers not starting**
|
||||
### Containers not starting
|
||||
|
||||
```bash
|
||||
make logs
|
||||
make logs-api
|
||||
```
|
||||
|
||||
Look for errors related to missing config files or port conflicts.
|
||||
Look for errors about missing config files or port conflicts.
|
||||
|
||||
**Port 53 already in use**
|
||||
### Port 53 already in use
|
||||
|
||||
On Ubuntu/Debian, `systemd-resolved` listens on port 53. Disable it:
|
||||
On Ubuntu and Debian, `systemd-resolved` listens on port 53. Disable it:
|
||||
|
||||
```bash
|
||||
sudo systemctl disable --now systemd-resolved
|
||||
@@ -212,28 +203,35 @@ echo "nameserver 8.8.8.8" | sudo tee /etc/resolv.conf
|
||||
|
||||
Then run `make start` again.
|
||||
|
||||
**WebDAV container exits immediately**
|
||||
|
||||
`WEBDAV_PASS` is not set in `.env`. Set it and run `make start` again.
|
||||
|
||||
**WireGuard container fails to load kernel module**
|
||||
|
||||
Ensure the WireGuard kernel module is available:
|
||||
### WireGuard container fails to load the kernel module
|
||||
|
||||
```bash
|
||||
sudo modprobe wireguard
|
||||
```
|
||||
|
||||
On some minimal installs you may need to install `wireguard-tools` and the kernel headers for your running kernel.
|
||||
On minimal installs you may need `wireguard-tools` and the kernel headers for the running kernel.
|
||||
|
||||
**API returns 503 or UI shows "Backend Unavailable"**
|
||||
### API returns 428 and redirects to /setup
|
||||
|
||||
The Flask API may still be starting. Wait 10–15 seconds after `make start` and refresh. If it persists:
|
||||
The first-run wizard has not been completed. Open `http://<host-ip>:8081` and finish the wizard.
|
||||
|
||||
### API returns 401 / UI shows "Not authenticated"
|
||||
|
||||
Your session expired or you have not logged in. Go to `http://<host-ip>:8081/login`.
|
||||
|
||||
### API returns 503 "Authentication not configured"
|
||||
|
||||
The auth file exists but contains no accounts. To recover:
|
||||
|
||||
```bash
|
||||
make logs-api
|
||||
make reset-admin-password
|
||||
```
|
||||
|
||||
**Config changes not taking effect**
|
||||
This generates a new admin password and prints it.
|
||||
|
||||
After changing identity or service settings in the UI, a yellow banner appears at the top of the page. Click **Apply Now** to restart the affected containers.
|
||||
### Forgot the admin password
|
||||
|
||||
```bash
|
||||
make show-admin-password # print current password
|
||||
make reset-admin-password # generate a new random password
|
||||
```
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Personal Internet Cell (PIC)
|
||||
|
||||
PIC is a self-hosted digital infrastructure platform. It manages DNS, DHCP, NTP, WireGuard VPN, email, calendar/contacts (CalDAV), file storage (WebDAV), a reverse proxy, and a certificate authority — all controlled from a single REST API and React web UI. No manual config file editing is required for normal operations.
|
||||
PIC is a self-hosted digital infrastructure platform. It packages DNS, DHCP, NTP, WireGuard VPN, email, calendar/contacts (CalDAV), file storage (WebDAV), a reverse proxy, a certificate authority, and optional third-party services — all managed through a single REST API and a React web UI. No manual config file editing is required for normal operations.
|
||||
|
||||
---
|
||||
|
||||
@@ -10,96 +10,117 @@ PIC is a self-hosted digital infrastructure platform. It manages DNS, DHCP, NTP,
|
||||
Browser
|
||||
└── React SPA (cell-webui :8081)
|
||||
└── Flask REST API (cell-api :3000, bound to 127.0.0.1)
|
||||
└── Docker SDK / config files
|
||||
├── cell-caddy :80/:443 reverse proxy
|
||||
├── cell-dns :53 CoreDNS
|
||||
├── cell-dhcp :67/udp dnsmasq
|
||||
├── cell-ntp :123/udp chrony
|
||||
├── cell-wireguard :51820/udp WireGuard VPN
|
||||
├── cell-mail :25/:587/:993 Postfix + Dovecot
|
||||
├── cell-radicale 127.0.0.1:5232 CalDAV/CardDAV
|
||||
├── cell-webdav 127.0.0.1:8080 WebDAV
|
||||
├── cell-rainloop :8888 webmail (RainLoop)
|
||||
├── cell-filegator :8082 file manager UI
|
||||
└── cell-webui :8081 React UI (Nginx)
|
||||
└── Service managers + Docker SDK
|
||||
├── cell-caddy :80/:443 Caddy reverse proxy (HTTPS/TLS)
|
||||
├── cell-dns :53 CoreDNS
|
||||
├── cell-dhcp :67/udp dnsmasq
|
||||
├── cell-ntp :123/udp chrony
|
||||
├── cell-wireguard :51820/udp WireGuard VPN
|
||||
└── cell-webui :8081 React UI (Nginx)
|
||||
(+ per-service containers, started when a service is installed)
|
||||
```
|
||||
|
||||
All containers run on a custom Docker bridge network (`cell-network`, default `172.20.0.0/16`). Static IPs per container are set in `docker-compose.yml` and overridden via `.env`.
|
||||
Core containers run on a Docker bridge network (`cell-network`, default subnet `172.20.0.0/16`). Static IPs per container are set in `docker-compose.yml` and can be overridden via `.env`. Installed service containers join the same network with their own compose projects managed by `ServiceComposer`.
|
||||
|
||||
The Flask API (`api/app.py`, ~2800 lines) contains all REST endpoints, runs a background health-monitoring thread, and manages the entire lifecycle of generated config artefacts: `Caddyfile`, `Corefile`, `wg0.conf`, and `cell_config.json` (the single source of truth at `config/api/cell_config.json`).
|
||||
The Flask API (`api/app.py`) contains REST endpoints and a background health-monitoring thread. Service managers are instantiated as singletons in `api/managers.py`. The single source of truth for runtime configuration is `config/api/cell_config.json`, managed by `ConfigManager`.
|
||||
|
||||
The React frontend (`webui/`) is built with Vite + Tailwind CSS. All API calls go through `src/services/api.js` (Axios). Pages: Dashboard, Peers, Network Services, WireGuard, Email, Calendar, Files, Routing, Vault, Containers, Cell Network, Logs, Settings.
|
||||
The React frontend (`webui/`) is built with Vite + Tailwind CSS. All API calls go through `src/services/api.js` (Axios).
|
||||
|
||||
**Web UI pages:** Dashboard, Peers, Network Services, WireGuard, Email, Calendar, Files, Routing, Vault, Containers, Cell Network, Connectivity, Service Store, Logs, Settings.
|
||||
|
||||
---
|
||||
|
||||
## Features
|
||||
|
||||
- **First-run wizard** — browser-based setup at `/setup`. On first start, all API requests redirect to `/setup` (HTTP 428) until the wizard is completed. Sets cell name, domain mode, timezone, admin password, and initial services. No manual `.env` editing required for identity.
|
||||
- **Session-based auth** — admin and peer roles. All `/api/*` endpoints require an authenticated session after setup. CSRF protection on all state-changing requests.
|
||||
- **WireGuard VPN** — peer lifecycle management, automatic key generation, QR code config export, per-peer routing policy.
|
||||
- **Caddy HTTPS** — automatic TLS via Let's Encrypt (DNS-01 or HTTP-01) or an internal CA, depending on domain mode.
|
||||
- **DDNS (pic.ngo)** — registers a `<cell-name>.pic.ngo` subdomain. Supported providers: `pic_ngo`, `cloudflare`, `duckdns`, `noip`, `freedns`. A background thread re-publishes the public IP every 5 minutes.
|
||||
- **Service store** — install/remove optional third-party services from the `pic-services` index at `git.pic.ngo`. Manifests declare container images, Caddy routes, and iptables rules.
|
||||
- **Extended connectivity** — per-peer egress routing through alternate exits: WireGuard external, OpenVPN, or Tor. Configured via policy routing (fwmark + ip rule) in the WireGuard container.
|
||||
- **Cell-to-cell networking** — WireGuard-based site-to-site links between PIC cells with service-level access control (calendar, files, mail, WebDAV) and a peer-sync protocol.
|
||||
- **Certificate authority** — `vault_manager` issues and revokes TLS certificates for internal services.
|
||||
- **Network services** — CoreDNS (`.cell` TLD), dnsmasq DHCP, chrony NTP.
|
||||
- **Email** _(optional, install via Service Store)_ — Postfix + Dovecot via `docker-mailserver`.
|
||||
- **Calendar/contacts** _(optional, install via Service Store)_ — Radicale CalDAV/CardDAV.
|
||||
- **File storage** _(optional, install via Service Store)_ — WebDAV with per-user accounts; Filegator for browser-based file management.
|
||||
- **Container manager** — start/stop/inspect containers, pull images, manage volumes via the Docker SDK.
|
||||
- **Firewall manager** — iptables rule management (`firewall_manager.py`).
|
||||
- **Structured logging** — JSON logs with rotation (5 MB / 5 backups per service), log search, and per-service verbosity control.
|
||||
|
||||
---
|
||||
|
||||
## Requirements
|
||||
|
||||
- Linux host with the WireGuard kernel module loaded
|
||||
- Linux host with the WireGuard kernel module loaded (`modprobe wireguard` to verify)
|
||||
- Docker Engine and Docker Compose (v2 plugin or v1 standalone)
|
||||
- Python 3.10+ (for `make setup` and local dev only; not needed at runtime)
|
||||
- Python 3.10+ (for `make setup` and local development; not needed at runtime)
|
||||
- 2 GB+ RAM, 10 GB+ disk
|
||||
- Ports available: 53, 67/udp, 80, 443, 51820/udp, 25, 587, 993
|
||||
- Ports available: 53, 67/udp, 80, 443, 51820/udp (plus 25, 587, 993 when the email service is installed)
|
||||
|
||||
---
|
||||
|
||||
## Quick Start
|
||||
|
||||
See [QUICKSTART.md](QUICKSTART.md) for step-by-step setup.
|
||||
See [QUICKSTART.md](QUICKSTART.md) for step-by-step instructions.
|
||||
|
||||
The short version:
|
||||
|
||||
```bash
|
||||
git clone gitea@192.168.31.50:roof/pic.git pic
|
||||
cd pic
|
||||
make start
|
||||
# open http://<host-ip>:8081 — the setup wizard appears automatically
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Configuration
|
||||
|
||||
Runtime configuration is controlled by `.env` in the project root. Copy `.env.example` to `.env` before first run.
|
||||
Port assignments and container IPs are configured in `.env` in the project root. A `.env` file is not required for first start — all variables have defaults. Create one only if you need to change ports or container IPs.
|
||||
|
||||
| Variable | Default | Description |
|
||||
|---|---|---|
|
||||
| `CELL_NETWORK` | `172.20.0.0/16` | Docker bridge subnet for all containers |
|
||||
| `CADDY_IP` through `FILEGATOR_IP` | `172.20.0.2`–`.13` | Static IP for each container |
|
||||
| `DNS_PORT` | `53` | DNS (UDP+TCP) |
|
||||
| `CELL_NETWORK` | `172.20.0.0/16` | Docker bridge subnet |
|
||||
| `CADDY_IP` through `WG_IP` | `172.20.0.2`–`.9` | Static IP per core container |
|
||||
| `DNS_PORT` | `53` | DNS (UDP + TCP) |
|
||||
| `DHCP_PORT` | `67` | DHCP (UDP) |
|
||||
| `NTP_PORT` | `123` | NTP (UDP) |
|
||||
| `WG_PORT` | `51820` | WireGuard listen port (UDP) |
|
||||
| `API_PORT` | `3000` | Flask API (bound to `127.0.0.1`) |
|
||||
| `API_PORT` | `3000` | Flask API (127.0.0.1 only) |
|
||||
| `WEBUI_PORT` | `8081` | React UI |
|
||||
| `MAIL_SMTP_PORT` | `25` | SMTP |
|
||||
| `MAIL_SUBMISSION_PORT` | `587` | SMTP submission |
|
||||
| `MAIL_IMAP_PORT` | `993` | IMAP |
|
||||
| `RADICALE_PORT` | `5232` | CalDAV (bound to `127.0.0.1`) |
|
||||
| `WEBDAV_PORT` | `8080` | WebDAV (bound to `127.0.0.1`) |
|
||||
| `RAINLOOP_PORT` | `8888` | Webmail |
|
||||
| `FILEGATOR_PORT` | `8082` | File manager UI |
|
||||
| `WEBDAV_USER` | `admin` | WebDAV basic-auth username |
|
||||
| `WEBDAV_PASS` | _(required)_ | WebDAV basic-auth password — must be set before `make start` |
|
||||
| `FLASK_DEBUG` | _(unset)_ | Set to `1` to enable Flask debug mode; do not use in production |
|
||||
| `FLASK_DEBUG` | _(unset)_ | Set to `1` for Flask debug mode; do not use in production |
|
||||
| `PUID` / `PGID` | current user | UID/GID passed to the WireGuard container |
|
||||
|
||||
Cell identity (cell name, domain, VPN IP range) is configured via `make setup` or the Settings → Identity page in the UI after startup. The VPN IP range must be an RFC-1918 CIDR (`10.0.0.0/8`, `172.16.0.0/12`, or `192.168.0.0/16`); the API and UI both enforce this.
|
||||
Cell identity (cell name, domain mode, timezone) is set through the first-run wizard on first start, or later through the Settings page in the UI.
|
||||
|
||||
---
|
||||
|
||||
## Security Notes
|
||||
## Security
|
||||
|
||||
**Ports exposed to the network:**
|
||||
**Ports exposed on all interfaces by default:**
|
||||
|
||||
- `80` / `443` — Caddy (HTTP/HTTPS reverse proxy)
|
||||
- `51820/udp` — WireGuard
|
||||
- `25` / `587` / `993` — Mail (SMTP, submission, IMAP)
|
||||
- `53` — DNS (UDP + TCP)
|
||||
- `53` — DNS
|
||||
- `67/udp` — DHCP
|
||||
- `8081` — Web UI
|
||||
- `8888` — Webmail (RainLoop)
|
||||
- `8082` — File manager (Filegator)
|
||||
- `25` / `587` / `993` — mail _(only when the email service is installed)_
|
||||
|
||||
**Ports bound to `127.0.0.1` only** (not directly reachable from the network):
|
||||
**Ports bound to `127.0.0.1` only:**
|
||||
|
||||
- `3000` — Flask API
|
||||
- `5232` — Radicale (CalDAV)
|
||||
- `8080` — WebDAV
|
||||
|
||||
The API has no authentication layer. It relies on `is_local_request()` to restrict sensitive endpoints (containers, vault) to requests originating from loopback or the cell's Docker network. The Docker socket is mounted into `cell-api`; treat access to port 3000 as equivalent to root access on the host.
|
||||
The API uses session-based authentication (admin and peer roles). The Docker socket is mounted into `cell-api`; treat access to port 3000 as equivalent to root access on the host.
|
||||
|
||||
For internet-facing deployments, place the host behind a firewall or VPN and restrict access to the API and UI ports.
|
||||
Before setup is complete, all `/api/*` requests except `/api/setup/*` and `/health` return HTTP 428 and a redirect to `/setup`.
|
||||
|
||||
CSRF protection (double-submit token in `X-CSRF-Token` header) applies to all `POST`, `PUT`, `DELETE`, and `PATCH` requests on `/api/*` once a user session exists, except `/api/auth/*` and `/api/setup/*`.
|
||||
|
||||
Cell-to-cell peer-sync endpoints (`/api/cells/peer-sync/*`) authenticate via source IP and WireGuard public key, not session cookies.
|
||||
|
||||
For internet-facing deployments, place the host behind a firewall and restrict access to the API and UI ports.
|
||||
|
||||
---
|
||||
|
||||
@@ -123,7 +144,7 @@ cd webui && npm install && npm run dev
|
||||
# Follow all container logs
|
||||
make logs
|
||||
|
||||
# Follow logs for one service (e.g. api, dns, caddy, wireguard, mail)
|
||||
# Follow logs for one service
|
||||
make logs-api
|
||||
|
||||
# Open a shell inside a container
|
||||
@@ -135,41 +156,38 @@ make shell-api
|
||||
## Testing
|
||||
|
||||
```bash
|
||||
make test # run the full pytest suite
|
||||
make test # run all unit tests (pytest, excludes e2e and integration)
|
||||
make test-coverage # run with coverage; HTML report in htmlcov/
|
||||
make test-api # run API endpoint tests only
|
||||
```
|
||||
|
||||
Tests live in `tests/` (34 files, 642 test functions). Coverage includes:
|
||||
|
||||
- All service managers (network, WireGuard, email, calendar, file, routing, vault, container)
|
||||
- API endpoint tests for each service area
|
||||
- Config manager (CRUD, validation, backup/restore)
|
||||
- IP utilities and Caddyfile generation
|
||||
- Peer registry and WireGuard peer lifecycle
|
||||
- Service bus pub/sub
|
||||
- Firewall manager
|
||||
- Pending-restart logic
|
||||
|
||||
Integration tests (`tests/integration/`) require a running PIC stack:
|
||||
Tests live in `tests/`. Integration tests require a running stack:
|
||||
|
||||
```bash
|
||||
make test-integration # full suite (creates peers)
|
||||
make test-integration # full suite (creates peers, modifies state)
|
||||
make test-integration-readonly # read-only checks, safe to run anytime
|
||||
```
|
||||
|
||||
End-to-end tests use Playwright:
|
||||
|
||||
```bash
|
||||
make test-e2e-deps # install Playwright and dependencies (run once)
|
||||
make test-e2e-api # API-level e2e tests
|
||||
make test-e2e-ui # UI-level e2e tests
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Management Commands
|
||||
|
||||
```bash
|
||||
make setup # generate WireGuard keys, write configs, create data dirs
|
||||
make start # docker compose up -d --build
|
||||
make start # docker compose up -d --build (full profile)
|
||||
make stop # docker compose down
|
||||
make restart # docker compose restart
|
||||
make status # container status + API health check
|
||||
make logs # follow all service logs
|
||||
make logs-<svc> # follow logs for one service
|
||||
make shell-<svc> # shell inside a container
|
||||
make logs-<svc> # follow logs for one service (e.g. make logs-api)
|
||||
make shell-<svc> # shell inside a container (e.g. make shell-api)
|
||||
|
||||
make update # git pull + rebuild + restart
|
||||
make reinstall # full wipe of config/ and data/, then setup + start
|
||||
@@ -180,7 +198,9 @@ make restore # list available backups
|
||||
|
||||
make list-peers # show WireGuard peers via API
|
||||
make show-routes # wg show inside the wireguard container
|
||||
make add-peer PEER_NAME=foo PEER_IP=10.0.0.5 PEER_KEY=<pubkey>
|
||||
|
||||
make show-admin-password # print current admin password
|
||||
make reset-admin-password # generate and set a new random admin password
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
Binary file not shown.
@@ -0,0 +1,298 @@
|
||||
"""
|
||||
AccountManager — per-service credential provisioning for PIC peers.
|
||||
|
||||
Responsibilities:
|
||||
- Dispatch account creation/deletion to each service's underlying manager
|
||||
- Store per-peer per-service credentials securely (0o600 file)
|
||||
- Provide credential retrieval for peer_config_template filling
|
||||
- Bulk-deprovision a peer from all services on peer deletion
|
||||
|
||||
Credentials file format (data/peer_service_credentials.json):
|
||||
{
|
||||
"<service_id>": {
|
||||
"<peer_username>": {"password": "..."}
|
||||
}
|
||||
}
|
||||
|
||||
Design note — plaintext passwords:
|
||||
Credentials are stored in plaintext so the peer endpoint can return them to
|
||||
the peer's device for one-time client configuration. The file is created with
|
||||
0o600 so it is only readable by the process owner (same pattern used for
|
||||
WireGuard keys and service_secrets.json).
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import secrets as _secrets_mod
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
try:
|
||||
import requests as _requests
|
||||
except ImportError:
|
||||
_requests = None
|
||||
|
||||
logger = logging.getLogger('picell')
|
||||
|
||||
_DISPATCH_PROVISION = {
|
||||
'email_manager': '_provision_email',
|
||||
'calendar_manager': '_provision_calendar',
|
||||
'file_manager': '_provision_files',
|
||||
}
|
||||
_DISPATCH_DEPROVISION = {
|
||||
'email_manager': '_deprovision_email',
|
||||
'calendar_manager': '_deprovision_calendar',
|
||||
'file_manager': '_deprovision_files',
|
||||
}
|
||||
|
||||
_HTTP_TIMEOUT = 10
|
||||
|
||||
|
||||
class AccountManager:
|
||||
|
||||
def __init__(self, service_registry, data_dir: str, config_manager=None, **managers):
|
||||
"""
|
||||
service_registry — ServiceRegistry instance
|
||||
data_dir — host data directory (data/peer_service_credentials.json lives here)
|
||||
config_manager — ConfigManager instance (used to resolve fallback email domain)
|
||||
**managers — named manager instances: email_manager=..., calendar_manager=...,
|
||||
file_manager=...
|
||||
"""
|
||||
self._registry = service_registry
|
||||
self._creds_path = Path(data_dir) / 'peer_service_credentials.json'
|
||||
self._config_manager = config_manager
|
||||
self._managers = managers
|
||||
self._lock = threading.Lock()
|
||||
|
||||
# ── Credential storage (0o600) ────────────────────────────────────────
|
||||
|
||||
def _load_creds(self) -> Dict:
|
||||
if not self._creds_path.exists():
|
||||
return {}
|
||||
try:
|
||||
with open(self._creds_path) as f:
|
||||
return json.load(f)
|
||||
except (OSError, json.JSONDecodeError) as e:
|
||||
logger.warning('AccountManager: failed to load credentials: %s', e)
|
||||
return {}
|
||||
|
||||
def _save_creds(self, creds: Dict) -> None:
|
||||
tmp = str(self._creds_path) + '.tmp'
|
||||
with open(tmp, 'w', opener=lambda path, flags: os.open(path, flags, 0o600)) as f:
|
||||
json.dump(creds, f, indent=2)
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
os.replace(tmp, str(self._creds_path))
|
||||
|
||||
# ── Per-manager provision / deprovision ───────────────────────────────
|
||||
|
||||
def _provision_email(self, manager, svc: Dict, peer_username: str, password: str) -> bool:
|
||||
domain = (svc.get('config') or {}).get('domain', '')
|
||||
if not domain and self._config_manager is not None:
|
||||
domain = self._config_manager.get_effective_domain() or ''
|
||||
if not domain:
|
||||
raise ValueError("Email service has no 'domain' configured")
|
||||
return manager.create_email_user(peer_username, domain, password)
|
||||
|
||||
def _deprovision_email(self, manager, svc: Dict, peer_username: str) -> bool:
|
||||
domain = (svc.get('config') or {}).get('domain', '')
|
||||
return manager.delete_email_user(peer_username, domain)
|
||||
|
||||
@staticmethod
|
||||
def _provision_calendar(manager, _svc: Dict, peer_username: str, password: str) -> bool:
|
||||
return manager.create_calendar_user(peer_username, password)
|
||||
|
||||
@staticmethod
|
||||
def _deprovision_calendar(manager, _svc: Dict, peer_username: str) -> bool:
|
||||
return manager.delete_calendar_user(peer_username)
|
||||
|
||||
@staticmethod
|
||||
def _provision_files(manager, _svc: Dict, peer_username: str, password: str) -> bool:
|
||||
return manager.create_user(peer_username, password)
|
||||
|
||||
@staticmethod
|
||||
def _deprovision_files(manager, _svc: Dict, peer_username: str) -> bool:
|
||||
return manager.delete_user(peer_username)
|
||||
|
||||
# ── HTTP dispatch (manager == "http") ────────────────────────────────
|
||||
|
||||
@staticmethod
|
||||
def _http_base_url(svc: Dict) -> str:
|
||||
"""Return the base URL for the service's /service-api endpoint."""
|
||||
backend = svc.get('backend', '')
|
||||
if not backend:
|
||||
raise ValueError(f"Service {svc.get('id')!r} has no 'backend' configured")
|
||||
return f'http://{backend}'
|
||||
|
||||
def _provision_http(self, svc: Dict, peer_username: str, password: str) -> bool:
|
||||
if _requests is None:
|
||||
raise RuntimeError('requests library is required for HTTP account dispatch')
|
||||
url = self._http_base_url(svc) + '/service-api/accounts'
|
||||
try:
|
||||
resp = _requests.post(
|
||||
url,
|
||||
json={'username': peer_username, 'password': password},
|
||||
timeout=_HTTP_TIMEOUT,
|
||||
)
|
||||
if resp.status_code in (200, 201):
|
||||
return True
|
||||
logger.warning('HTTP provision %s on %s returned %s: %s',
|
||||
peer_username, svc.get('id'), resp.status_code, resp.text[:200])
|
||||
return False
|
||||
except Exception as exc:
|
||||
raise RuntimeError(f'HTTP provision request failed: {exc}') from exc
|
||||
|
||||
def _deprovision_http(self, svc: Dict, peer_username: str) -> bool:
|
||||
if _requests is None:
|
||||
raise RuntimeError('requests library is required for HTTP account dispatch')
|
||||
url = self._http_base_url(svc) + f'/service-api/accounts/{peer_username}'
|
||||
try:
|
||||
resp = _requests.delete(url, timeout=_HTTP_TIMEOUT)
|
||||
if resp.status_code in (200, 204, 404):
|
||||
return True
|
||||
logger.warning('HTTP deprovision %s on %s returned %s: %s',
|
||||
peer_username, svc.get('id'), resp.status_code, resp.text[:200])
|
||||
return False
|
||||
except Exception as exc:
|
||||
raise RuntimeError(f'HTTP deprovision request failed: {exc}') from exc
|
||||
|
||||
# ── Service validation helper ─────────────────────────────────────────
|
||||
|
||||
def _resolve_service(self, service_id: str):
|
||||
"""Return (svc, manager_name, manager) or raise ValueError.
|
||||
|
||||
manager is None when manager_name == 'http' — callers must check.
|
||||
"""
|
||||
svc = self._registry.get(service_id)
|
||||
if svc is None:
|
||||
raise ValueError(f'Unknown service: {service_id!r}')
|
||||
accounts_cfg = svc.get('accounts') or {}
|
||||
manager_name = accounts_cfg.get('manager')
|
||||
if not manager_name:
|
||||
raise ValueError(f'Service {service_id!r} does not support accounts')
|
||||
if manager_name == 'http':
|
||||
return svc, 'http', None
|
||||
manager = self._managers.get(manager_name)
|
||||
if manager is None:
|
||||
raise ValueError(f'Manager {manager_name!r} is not registered with AccountManager')
|
||||
return svc, manager_name, manager
|
||||
|
||||
# ── Public API ────────────────────────────────────────────────────────
|
||||
|
||||
def provision(self, service_id: str, peer_username: str,
|
||||
password: str = None) -> Dict:
|
||||
"""Create an account on the service for the peer; store and return credentials.
|
||||
|
||||
Raises ValueError if the service doesn't support accounts.
|
||||
Raises RuntimeError if the underlying manager fails.
|
||||
"""
|
||||
svc, manager_name, manager = self._resolve_service(service_id)
|
||||
|
||||
if password is None:
|
||||
password = _secrets_mod.token_urlsafe(16)
|
||||
|
||||
if manager_name == 'http':
|
||||
ok = self._provision_http(svc, peer_username, password)
|
||||
else:
|
||||
dispatch = _DISPATCH_PROVISION.get(manager_name)
|
||||
if dispatch is None:
|
||||
raise ValueError(f'No provision dispatch for manager: {manager_name!r}')
|
||||
ok = getattr(self, dispatch)(manager, svc, peer_username, password)
|
||||
|
||||
if not ok:
|
||||
raise RuntimeError(
|
||||
f'Provision of {peer_username!r} on {service_id!r} returned False — '
|
||||
'check underlying service manager logs'
|
||||
)
|
||||
|
||||
cred = {'password': password}
|
||||
with self._lock:
|
||||
all_creds = self._load_creds()
|
||||
all_creds.setdefault(service_id, {})[peer_username] = cred
|
||||
self._save_creds(all_creds)
|
||||
|
||||
logger.info('AccountManager: provisioned %s on %s', peer_username, service_id)
|
||||
return cred
|
||||
|
||||
def deprovision(self, service_id: str, peer_username: str) -> bool:
|
||||
"""Delete the peer's account on the service and clear stored credentials."""
|
||||
svc, manager_name, manager = self._resolve_service(service_id)
|
||||
|
||||
if manager_name == 'http':
|
||||
ok = self._deprovision_http(svc, peer_username)
|
||||
else:
|
||||
dispatch = _DISPATCH_DEPROVISION.get(manager_name)
|
||||
if dispatch is None:
|
||||
raise ValueError(f'No deprovision dispatch for manager: {manager_name!r}')
|
||||
ok = getattr(self, dispatch)(manager, svc, peer_username)
|
||||
|
||||
with self._lock:
|
||||
all_creds = self._load_creds()
|
||||
svc_creds = all_creds.get(service_id, {})
|
||||
if peer_username in svc_creds:
|
||||
del svc_creds[peer_username]
|
||||
if not svc_creds:
|
||||
del all_creds[service_id]
|
||||
self._save_creds(all_creds)
|
||||
|
||||
logger.info('AccountManager: deprovisioned %s from %s', peer_username, service_id)
|
||||
return bool(ok)
|
||||
|
||||
def get_credentials(self, service_id: str, peer_username: str) -> Optional[Dict]:
|
||||
"""Return stored credentials for peer+service, or None if not provisioned."""
|
||||
with self._lock:
|
||||
return self._load_creds().get(service_id, {}).get(peer_username)
|
||||
|
||||
def list_accounts(self, service_id: str) -> List[str]:
|
||||
"""Return peer usernames provisioned on a service."""
|
||||
with self._lock:
|
||||
return list(self._load_creds().get(service_id, {}).keys())
|
||||
|
||||
def list_peer_services(self, peer_username: str) -> List[str]:
|
||||
"""Return service IDs where this peer has a provisioned account."""
|
||||
with self._lock:
|
||||
creds = self._load_creds()
|
||||
return [svc_id for svc_id, peers in creds.items() if peer_username in peers]
|
||||
|
||||
def is_provisioned(self, service_id: str, peer_username: str) -> bool:
|
||||
return self.get_credentials(service_id, peer_username) is not None
|
||||
|
||||
def deprovision_peer(self, peer_username: str) -> Dict[str, bool]:
|
||||
"""Remove a peer from every service they are provisioned on.
|
||||
|
||||
Called on peer deletion. Continues even if individual services fail.
|
||||
Returns {service_id: success} for each service attempted.
|
||||
"""
|
||||
results: Dict[str, bool] = {}
|
||||
for service_id in self.list_peer_services(peer_username):
|
||||
try:
|
||||
results[service_id] = self.deprovision(service_id, peer_username)
|
||||
except Exception as e:
|
||||
logger.warning('AccountManager: deprovision %s from %s failed: %s',
|
||||
peer_username, service_id, e)
|
||||
results[service_id] = False
|
||||
return results
|
||||
|
||||
def get_all_credentials(self, peer_username: str) -> Dict[str, Dict]:
|
||||
"""Return {service_id: {field: value}} for all services the peer is provisioned on."""
|
||||
with self._lock:
|
||||
creds = self._load_creds()
|
||||
return {
|
||||
svc_id: peers[peer_username]
|
||||
for svc_id, peers in creds.items()
|
||||
if peer_username in peers
|
||||
}
|
||||
|
||||
def store_credentials(self, service_id: str, peer_username: str,
|
||||
cred: Dict) -> None:
|
||||
"""Directly store credentials without calling the underlying manager.
|
||||
|
||||
Used when a peer was provisioned through the legacy peers-POST route
|
||||
so that their credentials become retrievable via AccountManager.
|
||||
"""
|
||||
with self._lock:
|
||||
all_creds = self._load_creds()
|
||||
all_creds.setdefault(service_id, {})[peer_username] = cred
|
||||
self._save_creds(all_creds)
|
||||
+157
-11
@@ -44,6 +44,9 @@ from managers import (
|
||||
caddy_manager,
|
||||
ddns_manager, service_store_manager,
|
||||
connectivity_manager,
|
||||
service_registry,
|
||||
service_composer,
|
||||
account_manager,
|
||||
firewall_manager, EventType,
|
||||
)
|
||||
# Re-exports: tests do `from app import CellManager` and `from app import _resolve_peer_dns`
|
||||
@@ -51,6 +54,7 @@ from cell_manager import CellManager
|
||||
from wireguard_manager import _resolve_peer_dns
|
||||
from port_registry import PORT_FIELDS, detect_conflicts
|
||||
import auth_routes
|
||||
from legacy_cleanup import cleanup_legacy_builtin_containers
|
||||
|
||||
# Context variable for request info
|
||||
request_context = contextvars.ContextVar('request_context', default={})
|
||||
@@ -183,6 +187,13 @@ def enforce_setup():
|
||||
return jsonify({'error': 'Setup required', 'redirect': '/setup'}), 428
|
||||
|
||||
|
||||
# Read-only endpoints accessible to peer-role sessions (not just admin).
|
||||
# Add paths here when peers need to read shared cell state.
|
||||
_PEER_READABLE_PATHS = frozenset({
|
||||
'/api/services/active',
|
||||
})
|
||||
|
||||
|
||||
@app.before_request
|
||||
def enforce_auth():
|
||||
"""Enforce session-based authentication and role-based access control.
|
||||
@@ -199,8 +210,8 @@ def enforce_auth():
|
||||
backward-compatibility with pre-auth test suites.
|
||||
"""
|
||||
path = request.path
|
||||
# Always allow non-API paths and auth namespace
|
||||
if not path.startswith('/api/') or path.startswith('/api/auth/'):
|
||||
# Always allow non-API paths, auth namespace, and setup namespace
|
||||
if not path.startswith('/api/') or path.startswith('/api/auth/') or path.startswith('/api/setup/'):
|
||||
return None
|
||||
# Cell peer-sync endpoints authenticate via source IP + WG pubkey — not session
|
||||
if path.startswith('/api/cells/peer-sync/'):
|
||||
@@ -216,10 +227,6 @@ def enforce_auth():
|
||||
return None
|
||||
users = auth_manager.list_users()
|
||||
if not users:
|
||||
# Only fail closed when the auth file is readable but empty —
|
||||
# that's an explicit misconfiguration. If the file is missing or
|
||||
# unreadable (test env, wrong host path, permission denied), bypass
|
||||
# so pre-auth test suites continue to work.
|
||||
users_file = getattr(auth_manager, '_users_file', None)
|
||||
if users_file:
|
||||
try:
|
||||
@@ -238,6 +245,8 @@ def enforce_auth():
|
||||
if path.startswith('/api/peer/'):
|
||||
if role != 'peer':
|
||||
return jsonify({'error': 'Forbidden'}), 403
|
||||
elif path in _PEER_READABLE_PATHS:
|
||||
pass # both admin and peer may read these endpoints
|
||||
else:
|
||||
if role != 'admin':
|
||||
return jsonify({'error': 'Forbidden'}), 403
|
||||
@@ -292,7 +301,23 @@ auth_routes.auth_manager = auth_manager
|
||||
|
||||
# Apply firewall + DNS rules from stored peer settings (survives API restarts)
|
||||
def _configured_domain() -> str:
|
||||
return config_manager.configs.get('_identity', {}).get('domain', 'cell')
|
||||
identity = config_manager.configs.get('_identity', {})
|
||||
# domain_name is the full FQDN (e.g. 'test5.pic.ngo'); fall back to domain
|
||||
# (e.g. 'lan', 'dev') for cells that don't have a subdomain prefix.
|
||||
return identity.get('domain_name') or identity.get('domain', 'cell')
|
||||
|
||||
|
||||
def _configured_dns_params():
|
||||
"""Return (primary_domain, split_horizon_zones) for Corefile generation.
|
||||
|
||||
In DDNS mode the primary CoreDNS zone is the parent domain (e.g. 'pic.ngo')
|
||||
and the cell's FQDN (e.g. 'pic1.pic.ngo') is a separate split-horizon block
|
||||
so LAN clients resolve *.pic1.pic.ngo to the internal Caddy IP.
|
||||
In LAN mode both values are the same so split_horizon_zones is empty.
|
||||
"""
|
||||
primary = config_manager.get_internal_domain()
|
||||
effective = config_manager.get_effective_domain()
|
||||
return primary, ([effective] if effective != primary else [])
|
||||
|
||||
|
||||
def _restore_cell_wg_peers(cell_links):
|
||||
@@ -356,8 +381,10 @@ def _apply_startup_enforcement():
|
||||
# (happens if the container was rebuilt, wg0.conf was reset, etc.)
|
||||
_restore_cell_wg_peers(cell_links)
|
||||
wireguard_manager.sync_cell_routes()
|
||||
firewall_manager.apply_all_dns_rules(peers, COREFILE_PATH, _configured_domain(),
|
||||
cell_links=cell_links)
|
||||
_dns_primary, _dns_szones = _configured_dns_params()
|
||||
firewall_manager.apply_all_dns_rules(peers, COREFILE_PATH, _dns_primary,
|
||||
cell_links=cell_links,
|
||||
split_horizon_zones=_dns_szones)
|
||||
logger.info(f"Applied enforcement rules for {len(peers)} peers, {len(cell_links)} cells on startup")
|
||||
# Phase 3: reapply policy routing rules for peers whose internet traffic is
|
||||
# routed through an exit cell (ip rule entries don't survive container restart)
|
||||
@@ -375,6 +402,11 @@ def _apply_startup_enforcement():
|
||||
sync_summary = cell_link_manager.replay_pending_pushes()
|
||||
if sync_summary.get('attempted'):
|
||||
logger.info(f"Startup permission sync: {sync_summary}")
|
||||
# Remove legacy builtin containers from old main stack (one-shot, idempotent)
|
||||
try:
|
||||
cleanup_legacy_builtin_containers(config_manager)
|
||||
except Exception as _cle:
|
||||
logger.warning(f'legacy cleanup failed (non-fatal): {_cle}')
|
||||
# Service store: re-apply firewall/caddy rules for installed services
|
||||
try:
|
||||
service_store_manager.reapply_on_startup()
|
||||
@@ -394,8 +426,25 @@ def _bootstrap_dns():
|
||||
cell_name = identity.get('cell_name', os.environ.get('CELL_NAME', 'mycell'))
|
||||
domain = identity.get('domain', os.environ.get('CELL_DOMAIN', 'cell'))
|
||||
ip_range = identity.get('ip_range', os.environ.get('CELL_IP_RANGE', '172.20.0.0/16'))
|
||||
# Bootstrap on first start; then always regenerate to ensure A records use WG server IP.
|
||||
network_manager.apply_ip_range(ip_range, cell_name, domain)
|
||||
domain_mode = identity.get('domain_mode', 'lan')
|
||||
if domain_mode == 'lan':
|
||||
# LAN mode: write full service records into the primary local zone.
|
||||
network_manager.apply_ip_range(ip_range, cell_name, domain)
|
||||
else:
|
||||
# Non-LAN mode (DDNS/ACME): ensure the split-horizon zone is present so
|
||||
# LAN clients resolve service subdomains to the internal Caddy IP.
|
||||
# Never call apply_ip_range here — it would pollute the DDNS parent zone.
|
||||
effective_domain = config_manager.get_effective_domain()
|
||||
if effective_domain and effective_domain != domain:
|
||||
# Use the WireGuard server IP so VPN peers can reach Caddy via the tunnel.
|
||||
# The Docker bridge IP (172.20.x.x) is only reachable inside the Docker
|
||||
# network; WireGuard peers need the host's WG interface IP (e.g. 10.0.0.1).
|
||||
caddy_ip = network_manager._get_wg_server_ip()
|
||||
# update_split_horizon_zone writes both the zone file and the Corefile
|
||||
# (with the split-horizon block included). No separate apply_all_dns_rules
|
||||
# call needed — that would overwrite the Corefile and drop the split-horizon block.
|
||||
network_manager.update_split_horizon_zone(
|
||||
effective_domain, caddy_ip, primary_domain=domain)
|
||||
except Exception as e:
|
||||
logger.warning(f"DNS bootstrap failed (non-fatal): {e}")
|
||||
|
||||
@@ -503,8 +552,17 @@ def perform_health_check():
|
||||
'alerts': []
|
||||
}
|
||||
|
||||
# email/calendar/files are optional store services — only check them when installed
|
||||
_installed_store_ids = set(config_manager.get_installed_services())
|
||||
_OPTIONAL_STORE_MANAGERS = frozenset({'email_manager', 'calendar_manager', 'file_manager'})
|
||||
_MANAGER_TO_STORE_ID = {'email_manager': 'email', 'calendar_manager': 'calendar', 'file_manager': 'files'}
|
||||
|
||||
# Get health from each service
|
||||
for service_name in service_bus.list_services():
|
||||
if service_name in _OPTIONAL_STORE_MANAGERS:
|
||||
store_id = _MANAGER_TO_STORE_ID[service_name]
|
||||
if store_id not in _installed_store_ids:
|
||||
continue
|
||||
try:
|
||||
service = service_bus.get_service(service_name)
|
||||
if hasattr(service, 'health_check'):
|
||||
@@ -564,6 +622,7 @@ def perform_health_check():
|
||||
return {'error': str(e), 'timestamp': datetime.utcnow().isoformat()}
|
||||
|
||||
def health_monitor_loop():
|
||||
_cert_check_cycle = 0
|
||||
while health_monitor_running:
|
||||
with app.app_context():
|
||||
health_result = perform_health_check()
|
||||
@@ -587,6 +646,14 @@ def health_monitor_loop():
|
||||
caddy_manager.reset_health_failures()
|
||||
except Exception as _caddy_err:
|
||||
logger.error("Caddy health monitor error: %s", _caddy_err)
|
||||
# Refresh cert status every 60 cycles (\u2248 1 hour with a 60 s loop).
|
||||
_cert_check_cycle += 1
|
||||
if _cert_check_cycle >= 60:
|
||||
_cert_check_cycle = 0
|
||||
try:
|
||||
caddy_manager.refresh_cert_status()
|
||||
except Exception as _cert_err:
|
||||
logger.warning("Cert status refresh failed (non-fatal): %s", _cert_err)
|
||||
time.sleep(60) # Check every 60 seconds
|
||||
|
||||
# Start health monitor thread
|
||||
@@ -708,6 +775,7 @@ def get_cell_status():
|
||||
return jsonify({
|
||||
"cell_name": identity.get('cell_name', os.environ.get('CELL_NAME', 'mycell')),
|
||||
"domain": identity.get('domain', os.environ.get('CELL_DOMAIN', 'cell')),
|
||||
"effective_domain": config_manager.get_effective_domain(),
|
||||
"uptime": uptime_seconds,
|
||||
"peers_count": len(peers),
|
||||
"services": services_status,
|
||||
@@ -827,6 +895,84 @@ def connectivity_get_peer_exits():
|
||||
return jsonify({'error': str(e)}), 500
|
||||
|
||||
|
||||
@app.route('/api/caddy/cert-status', methods=['GET'])
|
||||
def caddy_cert_status():
|
||||
"""Return TLS certificate status (expiry, days remaining, domain, mode).
|
||||
|
||||
Refreshes from Caddy if the cached value is older than 5 minutes.
|
||||
For LAN mode returns {'status': 'internal'}; for ACME modes returns
|
||||
expiry info read via SSL handshake with the Caddy container.
|
||||
"""
|
||||
try:
|
||||
return jsonify(caddy_manager.get_cert_status_fresh(max_age_seconds=300))
|
||||
except Exception as e:
|
||||
logger.error(f"caddy_cert_status: {e}")
|
||||
return jsonify({'error': str(e)}), 500
|
||||
|
||||
|
||||
@app.route('/api/caddy/cert-renew', methods=['POST'])
|
||||
def caddy_cert_renew():
|
||||
"""Trigger ACME certificate renewal by reloading Caddy.
|
||||
|
||||
Returns immediately with status='pending'; poll GET /api/caddy/cert-status
|
||||
to track progress (Caddy typically acquires the cert within 30-60 s).
|
||||
"""
|
||||
try:
|
||||
result = caddy_manager.renew_cert()
|
||||
return jsonify(result), (200 if result.get('ok') else 400)
|
||||
except Exception as e:
|
||||
logger.error(f"caddy_cert_renew: {e}")
|
||||
return jsonify({'error': str(e)}), 500
|
||||
|
||||
|
||||
@app.route('/api/caddy/custom-cert', methods=['POST'])
|
||||
def caddy_upload_custom_cert():
|
||||
"""Install a custom TLS certificate (PEM format).
|
||||
|
||||
Body: { "cert_pem": "<PEM>", "key_pem": "<PEM>" }
|
||||
Validates the cert/key pair, writes to the shared certs directory,
|
||||
and reloads Caddy with the updated Caddyfile.
|
||||
"""
|
||||
try:
|
||||
data = request.get_json(silent=True) or {}
|
||||
cert_pem = (data.get('cert_pem') or '').strip()
|
||||
key_pem = (data.get('key_pem') or '').strip()
|
||||
if not cert_pem or not key_pem:
|
||||
return jsonify({'ok': False, 'error': 'cert_pem and key_pem are required'}), 400
|
||||
result = caddy_manager.upload_custom_cert(cert_pem, key_pem)
|
||||
return jsonify(result), (200 if result.get('ok') else 422)
|
||||
except Exception as e:
|
||||
logger.error(f"caddy_upload_custom_cert: {e}")
|
||||
return jsonify({'error': str(e)}), 500
|
||||
|
||||
|
||||
@app.route('/api/egress/status', methods=['GET'])
|
||||
def egress_status():
|
||||
"""Return egress status for all installed services that have an egress config."""
|
||||
try:
|
||||
return jsonify(egress_manager.get_status())
|
||||
except Exception as e:
|
||||
logger.error(f"egress_status: {e}")
|
||||
return jsonify({'error': str(e)}), 500
|
||||
|
||||
|
||||
@app.route('/api/egress/services/<service_id>/exit', methods=['PUT'])
|
||||
def egress_set_service_exit(service_id: str):
|
||||
"""Persist and immediately apply a per-service egress override."""
|
||||
try:
|
||||
data = request.get_json(silent=True) or {}
|
||||
exit_type = data.get('exit_type')
|
||||
if not isinstance(exit_type, str):
|
||||
return jsonify({'ok': False, 'error': 'exit_type is required'}), 400
|
||||
result = egress_manager.set_service_exit(service_id, exit_type)
|
||||
if result.get('ok'):
|
||||
return jsonify(result)
|
||||
return jsonify(result), 400
|
||||
except Exception as e:
|
||||
logger.error(f"egress_set_service_exit({service_id}): {e}")
|
||||
return jsonify({'error': str(e)}), 500
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
debug = os.environ.get('FLASK_DEBUG', '0') == '1'
|
||||
app.run(host='0.0.0.0', port=3000, debug=debug)
|
||||
@@ -47,16 +47,6 @@ class AuthManager(BaseServiceManager):
|
||||
os.makedirs(os.path.dirname(self._users_file), exist_ok=True)
|
||||
except Exception:
|
||||
pass
|
||||
if not os.path.exists(self._users_file):
|
||||
try:
|
||||
with open(self._users_file, 'w') as f:
|
||||
f.write('[]')
|
||||
try:
|
||||
os.chmod(self._users_file, 0o600)
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as e:
|
||||
self.logger.error(f'Could not create users file: {e}')
|
||||
|
||||
def _load_users(self) -> List[Dict[str, Any]]:
|
||||
with self._lock:
|
||||
|
||||
+433
-35
@@ -23,8 +23,13 @@ in the main server block (or, for ``http01``, written as their own per-host
|
||||
blocks).
|
||||
"""
|
||||
|
||||
import datetime as _dt
|
||||
import logging
|
||||
import os
|
||||
import socket as _socket
|
||||
import ssl as _ssl
|
||||
import threading
|
||||
import time as _time
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import requests
|
||||
@@ -45,20 +50,43 @@ LIVE_CADDYFILE = os.environ.get('CADDYFILE_PATH', '/app/config-caddy/Caddyfile')
|
||||
# localhost to match the dev/test wiring.
|
||||
CADDY_ADMIN_URL = os.environ.get('CADDY_ADMIN_URL', 'http://cell-caddy:2019')
|
||||
|
||||
# Directory where the API writes custom TLS cert/key files.
|
||||
# The Caddy container mounts ./config/caddy → /config/caddy, so files written
|
||||
# here appear inside the container as /config/caddy/certs/<file>.
|
||||
CADDY_CERTS_DIR = os.environ.get('CADDY_CERTS_DIR', '/app/config-caddy/certs')
|
||||
# Paths as seen by the Caddy process (inside the container).
|
||||
_CADDY_CUSTOM_CERT = '/config/caddy/certs/cert.pem'
|
||||
_CADDY_CUSTOM_KEY = '/config/caddy/certs/key.pem'
|
||||
_CADDY_INTERNAL_CERT = '/etc/caddy/internal/cert.pem'
|
||||
_CADDY_INTERNAL_KEY = '/etc/caddy/internal/key.pem'
|
||||
|
||||
|
||||
class CaddyManager(BaseServiceManager):
|
||||
"""Manages Caddy reverse-proxy configuration and runtime health."""
|
||||
|
||||
def __init__(self, config_manager=None,
|
||||
data_dir: str = '/app/data',
|
||||
config_dir: str = '/app/config'):
|
||||
config_dir: str = '/app/config',
|
||||
service_bus=None,
|
||||
service_registry=None):
|
||||
super().__init__('caddy', data_dir, config_dir)
|
||||
self.config_manager = config_manager
|
||||
self.container_name = 'cell-caddy'
|
||||
self.caddyfile_path = LIVE_CADDYFILE
|
||||
self._service_registry = service_registry
|
||||
# Consecutive health-check failure counter (reset on success or when
|
||||
# the caller restarts the container).
|
||||
self._health_failures = 0
|
||||
# Monotonic timestamp of the last successful cert status refresh.
|
||||
self._cert_refreshed_at: Optional[float] = None
|
||||
# Debounce: prevent two rapid Caddyfile reloads (e.g. IDENTITY_CHANGED
|
||||
# fires from wizard AND heartbeat re-registration within seconds of each other).
|
||||
self._last_regenerate_at: float = 0.0
|
||||
self._regenerate_lock = threading.Lock()
|
||||
|
||||
if service_bus is not None:
|
||||
from service_bus import EventType
|
||||
service_bus.subscribe_to_event(EventType.IDENTITY_CHANGED, self._on_identity_changed)
|
||||
|
||||
# ── BaseServiceManager required ───────────────────────────────────────
|
||||
|
||||
@@ -122,18 +150,20 @@ class CaddyManager(BaseServiceManager):
|
||||
)
|
||||
|
||||
if domain_mode == 'lan':
|
||||
return self._caddyfile_lan(cell_name, service_routes, core_routes)
|
||||
cert_path, key_path = self._tls_cert_pair()
|
||||
return self._caddyfile_lan(cell_name, service_routes, core_routes,
|
||||
cert_path, key_path)
|
||||
if domain_mode == 'pic_ngo':
|
||||
return self._caddyfile_pic_ngo(cell_name, service_routes, core_routes)
|
||||
if domain_mode == 'cloudflare':
|
||||
custom_domain = identity.get('custom_domain', f'{cell_name}.local')
|
||||
custom_domain = identity.get('domain_name', identity.get('domain', f'{cell_name}.local'))
|
||||
return self._caddyfile_cloudflare(
|
||||
custom_domain, service_routes, core_routes
|
||||
)
|
||||
if domain_mode == 'duckdns':
|
||||
return self._caddyfile_duckdns(cell_name, service_routes, core_routes)
|
||||
if domain_mode == 'http01':
|
||||
host = identity.get('custom_domain', f'{cell_name}.noip.me')
|
||||
host = identity.get('domain_name', identity.get('domain', f'{cell_name}.noip.me'))
|
||||
return self._caddyfile_http01(host, installed_services, core_routes)
|
||||
|
||||
# Fallback to lan so we always emit a valid Caddyfile.
|
||||
@@ -151,11 +181,75 @@ class CaddyManager(BaseServiceManager):
|
||||
lines.append(" admin 0.0.0.0:2019")
|
||||
if email:
|
||||
lines.append(f" email {email}")
|
||||
# Always allow tests to override the ACME directory via env var.
|
||||
lines.append(" acme_ca {$ACME_CA_URL}")
|
||||
# Only write acme_ca when a URL is configured — an empty ACME_CA_URL
|
||||
# causes Caddy to reject the Caddyfile with "wrong argument count".
|
||||
# When absent, Caddy defaults to Let's Encrypt production.
|
||||
acme_ca_url = os.environ.get('ACME_CA_URL', '').strip()
|
||||
if acme_ca_url:
|
||||
lines.append(f" acme_ca {acme_ca_url}")
|
||||
lines.append("}")
|
||||
return "\n".join(lines)
|
||||
|
||||
def _build_registry_service_routes(self, domain: str) -> str:
|
||||
"""Build named-matcher + handle blocks from the service registry.
|
||||
|
||||
When no registry is wired or the registry returns nothing, only the
|
||||
api block is emitted (api is always infrastructure, not delegated to
|
||||
the registry).
|
||||
"""
|
||||
routes: List[Dict] = []
|
||||
if self._service_registry is not None:
|
||||
try:
|
||||
routes = self._service_registry.get_caddy_routes()
|
||||
except Exception as exc:
|
||||
logger.warning('_build_registry_service_routes: registry error: %s', exc)
|
||||
|
||||
# Pre-seed with reserved names so no registry entry can squat them.
|
||||
seen_matchers: set = {'api', 'webui'}
|
||||
|
||||
blocks: List[str] = []
|
||||
for route in routes:
|
||||
primary_sub = route['subdomain']
|
||||
backend = route['backend']
|
||||
extra_subs: List[str] = route.get('extra_subdomains') or []
|
||||
extra_backends: Dict[str, str] = route.get('extra_backends') or {}
|
||||
|
||||
if primary_sub in seen_matchers:
|
||||
logger.warning('Caddy: skipping duplicate/reserved matcher %r', primary_sub)
|
||||
continue
|
||||
seen_matchers.add(primary_sub)
|
||||
|
||||
# Subdomains that share the primary backend go in one matcher block.
|
||||
shared = [primary_sub] + [s for s in extra_subs if s not in extra_backends]
|
||||
host_list = ' '.join(f'{s}.{domain}' for s in shared)
|
||||
blocks.append(
|
||||
f' @{primary_sub} host {host_list}\n'
|
||||
f' handle @{primary_sub} {{\n'
|
||||
f' reverse_proxy {backend}\n'
|
||||
f' }}'
|
||||
)
|
||||
# Extra subdomains with their own backends each get their own block.
|
||||
for sub, sub_backend in extra_backends.items():
|
||||
if sub in seen_matchers:
|
||||
logger.warning('Caddy: skipping duplicate/reserved matcher %r', sub)
|
||||
continue
|
||||
seen_matchers.add(sub)
|
||||
blocks.append(
|
||||
f' @{sub} host {sub}.{domain}\n'
|
||||
f' handle @{sub} {{\n'
|
||||
f' reverse_proxy {sub_backend}\n'
|
||||
f' }}'
|
||||
)
|
||||
|
||||
# The api subdomain is always infrastructure — not delegated to the registry.
|
||||
blocks.append(
|
||||
f' @api host api.{domain}\n'
|
||||
f' handle @api {{\n'
|
||||
f' reverse_proxy cell-api:3000\n'
|
||||
f' }}'
|
||||
)
|
||||
return '\n'.join(blocks)
|
||||
|
||||
@staticmethod
|
||||
def _indent_routes(routes: str, spaces: int = 4) -> str:
|
||||
"""Indent a multi-line route block by ``spaces`` columns."""
|
||||
@@ -175,8 +269,21 @@ class CaddyManager(BaseServiceManager):
|
||||
chunks.append(route.strip("\n"))
|
||||
return "\n".join(chunks)
|
||||
|
||||
def _tls_cert_pair(self) -> tuple:
|
||||
"""Return (cert_path, key_path) as seen inside the Caddy container.
|
||||
|
||||
Uses the custom-uploaded cert when one is installed, otherwise falls
|
||||
back to the internal-CA cert that the VaultManager writes.
|
||||
"""
|
||||
ident = (self.config_manager.get_identity() if self.config_manager else {}) or {}
|
||||
if ident.get('tls', {}).get('cert_type') == 'custom':
|
||||
return _CADDY_CUSTOM_CERT, _CADDY_CUSTOM_KEY
|
||||
return _CADDY_INTERNAL_CERT, _CADDY_INTERNAL_KEY
|
||||
|
||||
def _caddyfile_lan(self, cell_name: str,
|
||||
service_routes: str, core_routes: str) -> str:
|
||||
service_routes: str, core_routes: str,
|
||||
cert_path: str = _CADDY_INTERNAL_CERT,
|
||||
key_path: str = _CADDY_INTERNAL_KEY) -> str:
|
||||
"""LAN mode: HTTP only + internal-CA TLS, no ACME."""
|
||||
body = []
|
||||
if service_routes:
|
||||
@@ -190,7 +297,7 @@ class CaddyManager(BaseServiceManager):
|
||||
"}\n"
|
||||
"\n"
|
||||
f"http://{cell_name}.cell, http://172.20.0.2:80 {{\n"
|
||||
" tls /etc/caddy/internal/cert.pem /etc/caddy/internal/key.pem\n"
|
||||
f" tls {cert_path} {key_path}\n"
|
||||
f"{inner}\n"
|
||||
"}\n"
|
||||
)
|
||||
@@ -198,20 +305,49 @@ class CaddyManager(BaseServiceManager):
|
||||
def _caddyfile_pic_ngo(self, cell_name: str,
|
||||
service_routes: str, core_routes: str) -> str:
|
||||
"""pic_ngo mode: wildcard DNS-01 via the pic_ngo plugin."""
|
||||
body = []
|
||||
domain = f"{cell_name}.pic.ngo"
|
||||
body = [self._build_registry_service_routes(domain)]
|
||||
if service_routes:
|
||||
body.append(self._indent_routes(service_routes))
|
||||
body.append(core_routes)
|
||||
inner = "\n".join(body)
|
||||
email = f"admin@{cell_name}.pic.ngo"
|
||||
email = f"admin@{domain}"
|
||||
|
||||
# Resolve credentials at write time — Caddy runs in its own container
|
||||
# and does not inherit the API's environment variables, so we embed the
|
||||
# actual values instead of {$VAR} placeholders.
|
||||
# Token is read from data/api/ddns_token (not cell_config.json).
|
||||
ddns_cfg = self.config_manager.configs.get('ddns', {})
|
||||
if hasattr(self.config_manager, 'get_ddns_token'):
|
||||
ddns_token = self.config_manager.get_ddns_token() or ''
|
||||
else:
|
||||
ddns_token = (ddns_cfg.get('token') or '').strip()
|
||||
if not ddns_token:
|
||||
ddns_token = os.environ.get('DDNS_TOKEN', '').strip()
|
||||
_raw_api = (os.environ.get('DDNS_URL') or ddns_cfg.get('url') or 'https://ddns.pic.ngo').strip()
|
||||
# Strip legacy /api/v1 suffix — the pic_ngo plugin appends /api/v1 itself.
|
||||
ddns_api = _raw_api.rstrip('/').removesuffix('/api/v1')
|
||||
|
||||
# No token yet (fresh install, pre-registration) — Caddy would reject a
|
||||
# bare `token` keyword with no value. Fall back to LAN mode so Caddy
|
||||
# starts cleanly; the Caddyfile is regenerated once registration completes.
|
||||
if not ddns_token:
|
||||
logger.warning(
|
||||
'pic_ngo mode configured but no DDNS token available; '
|
||||
'falling back to lan mode until registration completes'
|
||||
)
|
||||
cert_path, key_path = self._tls_cert_pair()
|
||||
return self._caddyfile_lan(cell_name, service_routes, core_routes,
|
||||
cert_path, key_path)
|
||||
|
||||
return (
|
||||
f"{self._global_acme_block(email)}\n"
|
||||
"\n"
|
||||
f"*.{cell_name}.pic.ngo, {cell_name}.pic.ngo {{\n"
|
||||
f"*.{domain}, {domain} {{\n"
|
||||
" tls {\n"
|
||||
" dns pic_ngo {\n"
|
||||
" token {$PIC_NGO_DDNS_TOKEN}\n"
|
||||
" api_base_url {$PIC_NGO_DDNS_API}\n"
|
||||
f" token {ddns_token}\n"
|
||||
f" api_base_url {ddns_api}\n"
|
||||
" }\n"
|
||||
" }\n"
|
||||
f"{inner}\n"
|
||||
@@ -221,7 +357,7 @@ class CaddyManager(BaseServiceManager):
|
||||
def _caddyfile_cloudflare(self, custom_domain: str,
|
||||
service_routes: str, core_routes: str) -> str:
|
||||
"""cloudflare mode: wildcard DNS-01 via the cloudflare plugin."""
|
||||
body = []
|
||||
body = [self._build_registry_service_routes(custom_domain)]
|
||||
if service_routes:
|
||||
body.append(self._indent_routes(service_routes))
|
||||
body.append(core_routes)
|
||||
@@ -240,7 +376,8 @@ class CaddyManager(BaseServiceManager):
|
||||
def _caddyfile_duckdns(self, cell_name: str,
|
||||
service_routes: str, core_routes: str) -> str:
|
||||
"""duckdns mode: DNS-01 via the duckdns plugin."""
|
||||
body = []
|
||||
domain = f"{cell_name}.duckdns.org"
|
||||
body = [self._build_registry_service_routes(domain)]
|
||||
if service_routes:
|
||||
body.append(self._indent_routes(service_routes))
|
||||
body.append(core_routes)
|
||||
@@ -248,7 +385,7 @@ class CaddyManager(BaseServiceManager):
|
||||
return (
|
||||
f"{self._global_acme_block(None)}\n"
|
||||
"\n"
|
||||
f"*.{cell_name}.duckdns.org {{\n"
|
||||
f"*.{domain} {{\n"
|
||||
" tls {\n"
|
||||
" dns duckdns {$DUCKDNS_TOKEN}\n"
|
||||
" }\n"
|
||||
@@ -260,23 +397,29 @@ class CaddyManager(BaseServiceManager):
|
||||
installed_services: List[Dict[str, Any]],
|
||||
core_routes: str) -> str:
|
||||
"""http01 mode: no wildcard. Each service gets its own block."""
|
||||
# Main host block — only the core routes (api + webui). Service
|
||||
# routes that could otherwise be served as path-prefixes are NOT
|
||||
# placed here because in http01 mode each service is intended to
|
||||
# live on its own subdomain (otherwise it could also use a path
|
||||
# prefix here, but the spec calls for separate blocks).
|
||||
# Main host block — only the core routes (api + webui).
|
||||
out = [self._global_acme_block('{$ACME_EMAIL}'), ""]
|
||||
out.append(f"{host} {{")
|
||||
out.append(core_routes)
|
||||
out.append("}")
|
||||
|
||||
# One block per installed service that has a caddy_route.
|
||||
# Build (subdomain, backend) pairs from registry when available.
|
||||
_core_services = self._http01_service_pairs()
|
||||
for subdomain, backend in _core_services:
|
||||
out.append("")
|
||||
out.append(f"{subdomain}.{host} {{")
|
||||
out.append(f" reverse_proxy {backend}")
|
||||
out.append("}")
|
||||
|
||||
# One block per installed (store plugin) service that has a caddy_route,
|
||||
# skipping any name that conflicts with a core service.
|
||||
_core_names = {s for s, _ in _core_services}
|
||||
for svc in installed_services or []:
|
||||
if not svc:
|
||||
continue
|
||||
route = svc.get('caddy_route')
|
||||
name = svc.get('name') or svc.get('subdomain')
|
||||
if not route or not name:
|
||||
if not route or not name or name in _core_names:
|
||||
continue
|
||||
out.append("")
|
||||
out.append(f"{name}.{host} {{")
|
||||
@@ -284,6 +427,24 @@ class CaddyManager(BaseServiceManager):
|
||||
out.append("}")
|
||||
return "\n".join(out) + "\n"
|
||||
|
||||
def _http01_service_pairs(self) -> List[tuple]:
|
||||
"""Return (subdomain, backend) pairs for http01 per-host blocks."""
|
||||
pairs: List[tuple] = []
|
||||
if self._service_registry is not None:
|
||||
try:
|
||||
for route in self._service_registry.get_caddy_routes():
|
||||
pairs.append((route['subdomain'], route['backend']))
|
||||
extra_subs: List[str] = route.get('extra_subdomains') or []
|
||||
extra_backends: Dict[str, str] = route.get('extra_backends') or {}
|
||||
for sub in extra_subs:
|
||||
backend = extra_backends.get(sub, route['backend'])
|
||||
pairs.append((sub, backend))
|
||||
except Exception as exc:
|
||||
logger.warning('_http01_service_pairs: registry error: %s', exc)
|
||||
pairs = []
|
||||
pairs.append(('api', 'cell-api:3000'))
|
||||
return pairs
|
||||
|
||||
# ── filesystem + admin-API operations ─────────────────────────────────
|
||||
|
||||
def write_caddyfile(self, caddyfile_content: str) -> bool:
|
||||
@@ -306,6 +467,10 @@ class CaddyManager(BaseServiceManager):
|
||||
os.fsync(f.fileno())
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
os.chmod(self.caddyfile_path, 0o600)
|
||||
except OSError:
|
||||
pass
|
||||
logger.info("Wrote Caddyfile to %s (%d bytes)",
|
||||
self.caddyfile_path, len(caddyfile_content))
|
||||
except Exception as e:
|
||||
@@ -348,9 +513,14 @@ class CaddyManager(BaseServiceManager):
|
||||
return False
|
||||
|
||||
def check_caddy_health(self) -> bool:
|
||||
"""GET the Caddy admin API root. Returns True on HTTP 200."""
|
||||
"""GET Caddy's config endpoint. Returns True on HTTP 200.
|
||||
|
||||
Caddy's admin API has no root handler — GET / returns 404 even when
|
||||
fully healthy. GET /config/ returns 200 + the running config JSON
|
||||
whenever Caddy is up and serving.
|
||||
"""
|
||||
try:
|
||||
resp = requests.get(CADDY_ADMIN_URL + "/", timeout=5)
|
||||
resp = requests.get(CADDY_ADMIN_URL + "/config/", timeout=5)
|
||||
except requests.RequestException as e:
|
||||
logger.debug("Caddy health check error: %s", e)
|
||||
return False
|
||||
@@ -373,25 +543,253 @@ class CaddyManager(BaseServiceManager):
|
||||
|
||||
# ── certificate status ────────────────────────────────────────────────
|
||||
|
||||
_REGENERATE_DEBOUNCE = 5.0 # seconds
|
||||
|
||||
def regenerate_with_installed(self, installed_services: list) -> bool:
|
||||
"""Regenerate Caddyfile with installed services and reload."""
|
||||
"""Regenerate Caddyfile with installed services and reload.
|
||||
|
||||
Debounced: skips if called again within _REGENERATE_DEBOUNCE seconds.
|
||||
This prevents two simultaneous ACME orders when IDENTITY_CHANGED fires
|
||||
from multiple sources (e.g. wizard completion + heartbeat re-registration)
|
||||
within a short window.
|
||||
"""
|
||||
now = _time.monotonic()
|
||||
with self._regenerate_lock:
|
||||
if now - self._last_regenerate_at < self._REGENERATE_DEBOUNCE:
|
||||
logger.debug("caddy regenerate_with_installed: skipped (debounce)")
|
||||
return True
|
||||
self._last_regenerate_at = now
|
||||
identity = self.config_manager.get_identity()
|
||||
content = self.generate_caddyfile(identity, installed_services)
|
||||
return self.write_caddyfile(content)
|
||||
|
||||
def get_cert_status(self) -> Dict[str, Any]:
|
||||
"""Return TLS cert status from identity['tls'] if present."""
|
||||
default = {'status': 'unknown', 'expiry': None, 'days_remaining': None}
|
||||
if not self.config_manager:
|
||||
return default
|
||||
def _on_identity_changed(self, event) -> None:
|
||||
"""Regenerate and reload the Caddyfile when cell identity changes."""
|
||||
try:
|
||||
ident = self.config_manager.get_identity() or {}
|
||||
except Exception as e:
|
||||
logger.error("get_cert_status: failed to read identity: %s", e)
|
||||
return default
|
||||
self.regenerate_with_installed([])
|
||||
except Exception as exc:
|
||||
self.logger.warning('caddy_manager identity_changed handler failed: %s', exc)
|
||||
|
||||
# ── Certificate status ────────────────────────────────────────────────
|
||||
|
||||
def get_cert_status(self) -> Dict[str, Any]:
|
||||
"""Return TLS cert status enriched with identity context (cached)."""
|
||||
ident: Dict[str, Any] = {}
|
||||
if self.config_manager:
|
||||
try:
|
||||
ident = self.config_manager.get_identity() or {}
|
||||
except Exception as e:
|
||||
logger.error("get_cert_status: failed to read identity: %s", e)
|
||||
|
||||
domain_mode = ident.get('domain_mode', 'lan')
|
||||
tls = ident.get('tls') or {}
|
||||
cert_type = tls.get('cert_type', 'custom' if tls.get('cert_type') == 'custom'
|
||||
else ('internal' if domain_mode == 'lan' else 'acme'))
|
||||
|
||||
return {
|
||||
'status': tls.get('status', 'unknown'),
|
||||
'expiry': tls.get('expiry'),
|
||||
'days_remaining': tls.get('days_remaining'),
|
||||
'domain': self._domain_label(ident),
|
||||
'domain_mode': domain_mode,
|
||||
'cert_type': cert_type,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _domain_label(ident: Dict[str, Any]) -> Optional[str]:
|
||||
"""Return a human-readable domain string for display in the UI."""
|
||||
mode = ident.get('domain_mode', 'lan')
|
||||
cell = ident.get('cell_name', '')
|
||||
if mode == 'pic_ngo':
|
||||
return f'*.{cell}.pic.ngo' if cell else None
|
||||
if mode == 'cloudflare':
|
||||
d = ident.get('domain_name') or ident.get('domain', '')
|
||||
return f'*.{d}' if d else None
|
||||
if mode == 'duckdns':
|
||||
return f'*.{cell}.duckdns.org' if cell else None
|
||||
if mode == 'http01':
|
||||
return ident.get('domain_name') or ident.get('domain')
|
||||
return None # lan
|
||||
|
||||
def get_cert_status_fresh(self, max_age_seconds: int = 300) -> Dict[str, Any]:
|
||||
"""Return cert status, refreshing if the cached value is older than max_age_seconds."""
|
||||
now = _time.monotonic()
|
||||
if self._cert_refreshed_at is None or (now - self._cert_refreshed_at) > max_age_seconds:
|
||||
self.refresh_cert_status()
|
||||
return self.get_cert_status()
|
||||
|
||||
def refresh_cert_status(self) -> Dict[str, Any]:
|
||||
"""Check TLS cert expiry via SSL and persist to identity['tls'].
|
||||
|
||||
For LAN mode (no ACME): immediately returns {'status': 'internal'}.
|
||||
For ACME modes: opens an SSL connection to Caddy on port 443 and
|
||||
reads the cert expiry from the TLS handshake. On any error (cert
|
||||
not yet issued, network unreachable): returns {'status': 'unknown'}.
|
||||
"""
|
||||
identity = self.config_manager.get_identity() if self.config_manager else {}
|
||||
domain_mode = (identity or {}).get('domain_mode', 'lan')
|
||||
|
||||
if domain_mode == 'lan':
|
||||
status: Dict[str, Any] = {'status': 'internal', 'expiry': None, 'days_remaining': None}
|
||||
else:
|
||||
caddy_host = os.environ.get('CADDY_CERT_HOST', 'cell-caddy')
|
||||
caddy_port = int(os.environ.get('CADDY_HTTPS_PORT', '443'))
|
||||
result = self._check_cert_via_ssl(caddy_host, caddy_port)
|
||||
status = result if result is not None else {
|
||||
'status': 'unknown', 'expiry': None, 'days_remaining': None
|
||||
}
|
||||
|
||||
if self.config_manager:
|
||||
try:
|
||||
self.config_manager.set_identity_field('tls', status)
|
||||
except Exception as exc:
|
||||
logger.warning('refresh_cert_status: failed to persist tls status: %s', exc)
|
||||
|
||||
self._cert_refreshed_at = _time.monotonic()
|
||||
return status
|
||||
|
||||
@staticmethod
|
||||
def _check_cert_via_ssl(hostname: str, port: int = 443) -> Optional[Dict[str, Any]]:
|
||||
"""Open an SSL connection and return cert expiry info, or None on failure."""
|
||||
ctx = _ssl.create_default_context()
|
||||
ctx.check_hostname = False
|
||||
ctx.verify_mode = _ssl.CERT_NONE
|
||||
try:
|
||||
with _socket.create_connection((hostname, port), timeout=5) as raw:
|
||||
with ctx.wrap_socket(raw, server_hostname=hostname) as tls:
|
||||
der = tls.getpeercert(binary_form=True)
|
||||
if not der:
|
||||
return None
|
||||
from cryptography import x509
|
||||
from cryptography.hazmat.backends import default_backend
|
||||
cert = x509.load_der_x509_certificate(der, default_backend())
|
||||
# Use not_valid_after_utc (cryptography ≥42) with fallback for older builds.
|
||||
try:
|
||||
expiry = cert.not_valid_after_utc
|
||||
except AttributeError:
|
||||
expiry = cert.not_valid_after.replace(tzinfo=_dt.timezone.utc) # type: ignore[attr-defined]
|
||||
now = _dt.datetime.now(_dt.timezone.utc)
|
||||
days = (expiry - now).days
|
||||
return {
|
||||
'status': 'valid' if days > 0 else 'expired',
|
||||
'expiry': expiry.isoformat(),
|
||||
'days_remaining': days,
|
||||
}
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
# ── Active cert management ────────────────────────────────────────────
|
||||
|
||||
def renew_cert(self) -> Dict[str, Any]:
|
||||
"""Regenerate the Caddyfile, reload Caddy, and trigger ACME cert renewal.
|
||||
|
||||
Regenerates first so a stale or broken on-disk Caddyfile never blocks
|
||||
the reload. Returns immediately with status='pending'; the caller
|
||||
polls GET /api/caddy/cert-status to track progress. Not applicable
|
||||
to LAN mode — callers should use upload_custom_cert() instead.
|
||||
"""
|
||||
ident = (self.config_manager.get_identity() if self.config_manager else {}) or {}
|
||||
domain_mode = ident.get('domain_mode', 'lan')
|
||||
|
||||
if domain_mode == 'lan':
|
||||
return {
|
||||
'ok': False,
|
||||
'error': 'ACME renewal is not available in LAN mode. '
|
||||
'Upload a custom certificate instead.',
|
||||
}
|
||||
|
||||
# Regenerate → write → reload in one shot so the Caddyfile is always fresh.
|
||||
if self.config_manager:
|
||||
try:
|
||||
ok = self.regenerate_with_installed([])
|
||||
except Exception as exc:
|
||||
logger.error('renew_cert: regenerate_with_installed failed: %s', exc)
|
||||
ok = False
|
||||
else:
|
||||
ok = self.reload_caddy()
|
||||
|
||||
if not ok:
|
||||
return {'ok': False, 'error': 'Caddy reload failed — check Caddy logs.'}
|
||||
|
||||
# Invalidate the cached status so the next poll triggers a fresh SSL check.
|
||||
self._cert_refreshed_at = None
|
||||
return {
|
||||
'ok': True,
|
||||
'status': 'pending',
|
||||
'message': 'Renewal triggered. Certificate status will update within 60 s.',
|
||||
}
|
||||
|
||||
def upload_custom_cert(self, cert_pem: str, key_pem: str) -> Dict[str, Any]:
|
||||
"""Validate and install a custom TLS certificate.
|
||||
|
||||
Writes cert+key to the shared certs directory (visible to Caddy),
|
||||
regenerates the Caddyfile to reference the new paths, and reloads.
|
||||
Works for all domain modes — use this when you have a certificate
|
||||
issued by your own CA or a commercial provider.
|
||||
"""
|
||||
cert_info = self._parse_pem_cert(cert_pem)
|
||||
if cert_info is None:
|
||||
return {'ok': False, 'error': 'Invalid certificate: could not parse PEM.'}
|
||||
if not self._validate_key_pem(key_pem):
|
||||
return {'ok': False, 'error': 'Invalid private key: expected PEM with PRIVATE KEY header.'}
|
||||
|
||||
try:
|
||||
os.makedirs(CADDY_CERTS_DIR, exist_ok=True)
|
||||
with open(os.path.join(CADDY_CERTS_DIR, 'cert.pem'), 'w') as fh:
|
||||
fh.write(cert_pem)
|
||||
with open(os.path.join(CADDY_CERTS_DIR, 'key.pem'), 'w') as fh:
|
||||
fh.write(key_pem)
|
||||
except OSError as exc:
|
||||
logger.error('upload_custom_cert: write failed: %s', exc)
|
||||
return {'ok': False, 'error': f'Failed to write cert files: {exc}'}
|
||||
|
||||
days = cert_info.get('days_remaining', 0)
|
||||
tls_info: Dict[str, Any] = {
|
||||
'status': 'valid' if days > 0 else 'expired',
|
||||
'expiry': cert_info.get('expiry'),
|
||||
'days_remaining': days,
|
||||
'cert_type': 'custom',
|
||||
}
|
||||
if self.config_manager:
|
||||
try:
|
||||
self.config_manager.set_identity_field('tls', tls_info)
|
||||
except Exception as exc:
|
||||
logger.warning('upload_custom_cert: could not persist tls info: %s', exc)
|
||||
|
||||
# Regenerate Caddyfile so the tls directive references the new cert.
|
||||
if self.config_manager:
|
||||
try:
|
||||
self.regenerate_with_installed([])
|
||||
except Exception as exc:
|
||||
logger.warning('upload_custom_cert: Caddyfile regeneration failed: %s', exc)
|
||||
|
||||
return {'ok': True, **tls_info}
|
||||
|
||||
@staticmethod
|
||||
def _parse_pem_cert(cert_pem: str) -> Optional[Dict[str, Any]]:
|
||||
"""Parse a PEM certificate and return expiry metadata, or None on error."""
|
||||
try:
|
||||
from cryptography import x509
|
||||
cert_bytes = cert_pem.encode() if isinstance(cert_pem, str) else cert_pem
|
||||
cert = x509.load_pem_x509_certificate(cert_bytes)
|
||||
try:
|
||||
expiry = cert.not_valid_after_utc
|
||||
except AttributeError:
|
||||
expiry = cert.not_valid_after.replace(tzinfo=_dt.timezone.utc) # type: ignore[attr-defined]
|
||||
now = _dt.datetime.now(_dt.timezone.utc)
|
||||
days = (expiry - now).days
|
||||
return {
|
||||
'expiry': expiry.isoformat(),
|
||||
'days_remaining': days,
|
||||
'subject': cert.subject.rfc4514_string(),
|
||||
}
|
||||
except Exception as exc:
|
||||
logger.debug('_parse_pem_cert failed: %s', exc)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _validate_key_pem(key_pem: str) -> bool:
|
||||
"""Return True if key_pem contains a PEM-encoded private key block."""
|
||||
return ('-----BEGIN' in key_pem
|
||||
and 'PRIVATE KEY' in key_pem
|
||||
and '-----END' in key_pem)
|
||||
|
||||
@@ -10,6 +10,7 @@ import subprocess
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import Dict, List, Optional, Any
|
||||
import bcrypt
|
||||
from base_service_manager import BaseServiceManager
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -280,12 +281,51 @@ class CalendarManager(BaseServiceManager):
|
||||
user_dir = os.path.join(self.calendar_data_dir, 'users', username)
|
||||
self.safe_makedirs(user_dir)
|
||||
|
||||
# Write bcrypt entry to Radicale htpasswd (non-fatal if service not installed)
|
||||
self._write_radicale_htpasswd(username, password)
|
||||
|
||||
logger.info(f"Created calendar user: {username}")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to create calendar user {username}: {e}")
|
||||
return False
|
||||
|
||||
def _radicale_htpasswd_path(self) -> str:
|
||||
return os.path.join(self.data_dir, 'services', 'calendar', 'config', 'users')
|
||||
|
||||
def _write_radicale_htpasswd(self, username: str, password: str) -> None:
|
||||
htpasswd = self._radicale_htpasswd_path()
|
||||
config_dir = os.path.dirname(htpasswd)
|
||||
if not os.path.isdir(config_dir):
|
||||
return
|
||||
try:
|
||||
raw = bcrypt.hashpw(password.encode('utf-8'), bcrypt.gensalt()).decode('utf-8')
|
||||
if raw.startswith('$2b$'):
|
||||
raw = '$2y$' + raw[4:]
|
||||
lines = []
|
||||
if os.path.exists(htpasswd):
|
||||
with open(htpasswd) as f:
|
||||
lines = f.readlines()
|
||||
lines = [l for l in lines if not l.startswith(f'{username}:')]
|
||||
lines.append(f'{username}:{raw}\n')
|
||||
with open(htpasswd, 'w') as f:
|
||||
f.writelines(lines)
|
||||
except Exception as e:
|
||||
logger.warning('Failed to write Radicale htpasswd for %s: %s', username, e)
|
||||
|
||||
def _remove_radicale_htpasswd(self, username: str) -> None:
|
||||
htpasswd = self._radicale_htpasswd_path()
|
||||
if not os.path.exists(htpasswd):
|
||||
return
|
||||
try:
|
||||
with open(htpasswd) as f:
|
||||
lines = f.readlines()
|
||||
lines = [l for l in lines if not l.startswith(f'{username}:')]
|
||||
with open(htpasswd, 'w') as f:
|
||||
f.writelines(lines)
|
||||
except Exception as e:
|
||||
logger.warning('Failed to remove Radicale htpasswd for %s: %s', username, e)
|
||||
|
||||
def delete_calendar_user(self, username: str) -> bool:
|
||||
"""Delete a calendar user"""
|
||||
try:
|
||||
@@ -306,6 +346,7 @@ class CalendarManager(BaseServiceManager):
|
||||
import shutil
|
||||
shutil.rmtree(user_dir)
|
||||
|
||||
self._remove_radicale_htpasswd(username)
|
||||
logger.info(f"Deleted calendar user: {username}")
|
||||
return True
|
||||
|
||||
|
||||
@@ -426,7 +426,7 @@ class CellLinkManager:
|
||||
try:
|
||||
from app import config_manager
|
||||
identity = config_manager.configs.get('_identity', {})
|
||||
own_domain = identity.get('domain', os.environ.get('CELL_DOMAIN', ''))
|
||||
own_domain = identity.get('domain_name') or identity.get('domain', os.environ.get('CELL_DOMAIN', ''))
|
||||
if own_domain and remote_domain == own_domain:
|
||||
raise ValueError(
|
||||
f"Domain {remote_domain!r} is the same as this cell's own domain — "
|
||||
@@ -466,7 +466,7 @@ class CellLinkManager:
|
||||
identity = self._local_identity()
|
||||
from app import config_manager
|
||||
id_cfg = config_manager.configs.get('_identity', {})
|
||||
own_domain = id_cfg.get('domain', os.environ.get('CELL_DOMAIN', 'cell'))
|
||||
own_domain = id_cfg.get('domain_name') or id_cfg.get('domain', os.environ.get('CELL_DOMAIN', 'cell'))
|
||||
own_invite = self.generate_invite(identity['cell_name'], own_domain)
|
||||
except Exception as e:
|
||||
return {'ok': False, 'error': f'could not build own invite: {e}'}
|
||||
|
||||
+219
-5
@@ -6,6 +6,8 @@ Centralized configuration management for all services
|
||||
|
||||
import os
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import yaml
|
||||
import shutil
|
||||
import hashlib
|
||||
@@ -14,6 +16,9 @@ from typing import Dict, List, Optional, Any
|
||||
from pathlib import Path
|
||||
import logging
|
||||
|
||||
_SAFE_CONTAINER_RE = re.compile(r'^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,63}$')
|
||||
_SAFE_VOL_NAME_RE = re.compile(r'^[a-zA-Z0-9_.-]{1,64}$')
|
||||
|
||||
# The Caddyfile lives on a separate volume mount from the rest of config
|
||||
LIVE_CADDYFILE = os.environ.get('CADDYFILE_PATH', '/app/config-caddy/Caddyfile')
|
||||
|
||||
@@ -45,6 +50,21 @@ class ConfigManager:
|
||||
self.configs['connectivity'] = {'exits': {}, 'peer_exit_map': {}}
|
||||
if not self.config_file.exists():
|
||||
self._save_all_configs()
|
||||
# Silent migration: when DDNS is active but the internal domain is still
|
||||
# the generic "cell" default, give CoreDNS a unique zone name so multiple
|
||||
# cells on the same LAN don't collide.
|
||||
try:
|
||||
_ident = self.configs.get('_identity', {})
|
||||
_mode = _ident.get('domain_mode', 'lan')
|
||||
_domain = _ident.get('domain', '')
|
||||
_cell_name = _ident.get('cell_name', '')
|
||||
if (_mode != 'lan' and _cell_name
|
||||
and (_domain in ('cell', '', None))):
|
||||
_new_domain = f'{_cell_name}.local'
|
||||
self.configs['_identity']['domain'] = _new_domain
|
||||
self._save_all_configs()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _load_service_schemas(self) -> Dict[str, Dict]:
|
||||
"""Load configuration schemas for all services"""
|
||||
@@ -143,8 +163,8 @@ class ConfigManager:
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
os.replace(tmp, self.config_file)
|
||||
except (PermissionError, OSError):
|
||||
pass
|
||||
except (PermissionError, OSError) as e:
|
||||
logger.error('_save_all_configs: write failed — config NOT persisted to disk: %s', e)
|
||||
|
||||
def get_service_config(self, service: str) -> Dict[str, Any]:
|
||||
"""Get configuration for a specific service"""
|
||||
@@ -213,8 +233,128 @@ class ConfigManager:
|
||||
"warnings": warnings
|
||||
}
|
||||
|
||||
def backup_config(self) -> str:
|
||||
"""Create a backup of cell_config.json, secrets, Caddyfile, .env, Corefile, and DNS zones."""
|
||||
@staticmethod
|
||||
def _validate_vol_entry(service_id: str, vol: dict) -> bool:
|
||||
"""Return True if a backup volume entry is safe to use; log and return False otherwise."""
|
||||
container = vol.get('container', '')
|
||||
path = vol.get('path', '')
|
||||
name = vol.get('name', '')
|
||||
if not _SAFE_CONTAINER_RE.match(container):
|
||||
logger.warning('Backup: unsafe container name %r for %s — skipping', container, service_id)
|
||||
return False
|
||||
if not path.startswith('/') or '..' in path.split('/') or '\x00' in path:
|
||||
logger.warning('Backup: unsafe volume path %r for %s — skipping', path, service_id)
|
||||
return False
|
||||
if not _SAFE_VOL_NAME_RE.match(name):
|
||||
logger.warning('Backup: unsafe volume name %r for %s — skipping', name, service_id)
|
||||
return False
|
||||
return True
|
||||
|
||||
def _backup_service_volumes(self, backup_path: Path, service_registry) -> None:
|
||||
"""Stream service data out of each container via 'docker exec tar'.
|
||||
|
||||
Archives are relative (created with -C <path> .) so they can be safely
|
||||
restored with -C <path> without risk of path traversal outside the volume.
|
||||
Writes to a .partial temp file then renames atomically on success.
|
||||
"""
|
||||
try:
|
||||
plan = service_registry.get_backup_plan()
|
||||
except Exception as e:
|
||||
logger.warning('_backup_service_volumes: could not get backup plan: %s', e)
|
||||
return
|
||||
for entry in plan:
|
||||
service_id = entry['service_id']
|
||||
volumes = entry.get('volumes') or []
|
||||
if not volumes:
|
||||
continue
|
||||
svc_dir = backup_path / 'service_data' / service_id
|
||||
svc_dir.mkdir(parents=True, exist_ok=True)
|
||||
for vol in volumes:
|
||||
if not self._validate_vol_entry(service_id, vol):
|
||||
continue
|
||||
container = vol['container']
|
||||
path = vol['path']
|
||||
name = vol['name']
|
||||
archive_path = svc_dir / f'{name}.tar.gz'
|
||||
tmp_path = svc_dir / f'{name}.tar.gz.partial'
|
||||
try:
|
||||
with open(tmp_path, 'wb') as af:
|
||||
result = subprocess.run(
|
||||
# -C path; then '.' archives the whole dir with relative entries.
|
||||
# '--' prevents path/container from being parsed as options.
|
||||
['docker', 'exec', '--', container,
|
||||
'tar', '-C', path, '-czf', '-', '.'],
|
||||
stdout=af,
|
||||
stderr=subprocess.PIPE,
|
||||
timeout=300,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
logger.warning(
|
||||
'Backup: docker exec tar failed for %s/%s: %s',
|
||||
service_id, name, result.stderr.decode(errors='replace'),
|
||||
)
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
else:
|
||||
os.replace(tmp_path, archive_path)
|
||||
logger.info('Backup: archived %s/%s', service_id, name)
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.warning('Backup: timed out streaming %s/%s', service_id, name)
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
except Exception as e:
|
||||
logger.warning('Backup: failed to archive %s/%s: %s', service_id, name, e)
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
|
||||
def _restore_service_volumes(self, backup_path: Path, service_registry) -> None:
|
||||
"""Pipe archived service data back into containers via 'docker exec -i tar'.
|
||||
|
||||
Extracts with -C <path>, matching how archives were created (relative paths).
|
||||
This bounds extraction to within the declared volume directory.
|
||||
"""
|
||||
svc_data_dir = backup_path / 'service_data'
|
||||
if not svc_data_dir.is_dir():
|
||||
return
|
||||
for svc_dir in svc_data_dir.iterdir():
|
||||
if not svc_dir.is_dir():
|
||||
continue
|
||||
service_id = svc_dir.name
|
||||
svc = service_registry.get(service_id)
|
||||
if not svc:
|
||||
logger.warning('Restore: unknown service %s in backup, skipping', service_id)
|
||||
continue
|
||||
volumes = (svc.get('backup') or {}).get('volumes') or []
|
||||
for vol in volumes:
|
||||
if not self._validate_vol_entry(service_id, vol):
|
||||
continue
|
||||
container = vol['container']
|
||||
path = vol['path']
|
||||
name = vol['name']
|
||||
archive_path = svc_dir / f'{name}.tar.gz'
|
||||
if not archive_path.exists():
|
||||
continue
|
||||
try:
|
||||
with open(archive_path, 'rb') as af:
|
||||
result = subprocess.run(
|
||||
['docker', 'exec', '-i', '--', container,
|
||||
'tar', '-C', path, '-xzf', '-'],
|
||||
stdin=af,
|
||||
stderr=subprocess.PIPE,
|
||||
timeout=300,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
logger.warning(
|
||||
'Restore: docker exec tar failed for %s/%s: %s',
|
||||
service_id, name, result.stderr.decode(errors='replace'),
|
||||
)
|
||||
else:
|
||||
logger.info('Restore: restored %s/%s', service_id, name)
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.warning('Restore: timed out restoring %s/%s', service_id, name)
|
||||
except Exception as e:
|
||||
logger.warning('Restore: failed to restore %s/%s: %s', service_id, name, e)
|
||||
|
||||
def backup_config(self, service_registry=None) -> str:
|
||||
"""Create a backup of cell_config.json, secrets, Caddyfile, .env, Corefile, DNS zones,
|
||||
and (when service_registry is provided) live service data volumes."""
|
||||
try:
|
||||
timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
|
||||
backup_id = f"backup_{timestamp}"
|
||||
@@ -263,12 +403,17 @@ class ConfigManager:
|
||||
except (PermissionError, OSError) as e:
|
||||
logger.warning(f"Could not back up {src.name}: {e} (skipping)")
|
||||
|
||||
# Live service data volumes (streamed via docker exec)
|
||||
if service_registry is not None:
|
||||
self._backup_service_volumes(backup_path, service_registry)
|
||||
|
||||
services = ['identity'] + list(self.service_schemas.keys())
|
||||
manifest = {
|
||||
"backup_id": backup_id,
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
"services": services,
|
||||
"files": [f.name for f in backup_path.iterdir()],
|
||||
"includes_service_data": service_registry is not None,
|
||||
}
|
||||
with open(backup_path / 'manifest.json', 'w') as f:
|
||||
json.dump(manifest, f, indent=2)
|
||||
@@ -280,7 +425,8 @@ class ConfigManager:
|
||||
logger.error(f"Error creating backup: {e}")
|
||||
raise
|
||||
|
||||
def restore_config(self, backup_id: str, services: list = None) -> bool:
|
||||
def restore_config(self, backup_id: str, services: list = None,
|
||||
service_registry=None) -> bool:
|
||||
"""Restore from backup. If services list given, only restore those service configs (selective)."""
|
||||
try:
|
||||
backup_path = self.backup_dir / backup_id
|
||||
@@ -358,6 +504,10 @@ class ConfigManager:
|
||||
except (PermissionError, OSError) as e:
|
||||
logger.warning(f"Could not restore {dest.name}: {e} (skipping)")
|
||||
|
||||
# Live service data volumes
|
||||
if service_registry is not None:
|
||||
self._restore_service_volumes(backup_path, service_registry)
|
||||
|
||||
self.configs = self._load_all_configs()
|
||||
logger.info(f"Restored configuration from backup: {backup_id}")
|
||||
return True
|
||||
@@ -478,6 +628,23 @@ class ConfigManager:
|
||||
"""Return the current identity configuration."""
|
||||
return self.configs.get('_identity', {})
|
||||
|
||||
def get_effective_domain(self) -> str:
|
||||
"""Return the FQDN that public-facing services should use.
|
||||
In lan mode: _identity.domain. Otherwise: _identity.domain_name
|
||||
(falls back to domain if domain_name not yet registered)."""
|
||||
ident = self.get_identity()
|
||||
mode = ident.get('domain_mode', 'lan')
|
||||
if mode == 'lan':
|
||||
return ident.get('domain') or os.environ.get('CELL_DOMAIN', 'cell')
|
||||
return (ident.get('domain_name')
|
||||
or ident.get('domain')
|
||||
or os.environ.get('CELL_DOMAIN', 'cell'))
|
||||
|
||||
def get_internal_domain(self) -> str:
|
||||
"""Return the CoreDNS zone name (always _identity.domain)."""
|
||||
ident = self.get_identity()
|
||||
return ident.get('domain') or os.environ.get('CELL_DOMAIN', 'cell')
|
||||
|
||||
def set_identity_field(self, key: str, value: Any):
|
||||
"""Set a single field in the identity configuration and persist."""
|
||||
if '_identity' not in self.configs:
|
||||
@@ -510,6 +677,53 @@ class ConfigManager:
|
||||
cfg.setdefault('peer_exit_map', {})
|
||||
return dict(cfg)
|
||||
|
||||
def set_ddns_config(self, ddns_cfg: Dict[str, Any]) -> None:
|
||||
"""Replace the top-level ddns section and persist.
|
||||
Never writes a 'token' key into cell_config.json — tokens live in data/.
|
||||
"""
|
||||
ddns_cfg = {k: v for k, v in ddns_cfg.items() if k != 'token'}
|
||||
self.configs['ddns'] = ddns_cfg
|
||||
self._save_all_configs()
|
||||
|
||||
@property
|
||||
def _ddns_token_path(self) -> Path:
|
||||
return self.data_dir / 'api' / 'ddns_token'
|
||||
|
||||
def get_ddns_token(self) -> str:
|
||||
"""Return the DDNS bearer token from data/api/ddns_token.
|
||||
|
||||
Migrates automatically from the old cell_config.json location on first
|
||||
call so existing installs keep working without manual intervention.
|
||||
"""
|
||||
path = self._ddns_token_path
|
||||
if path.exists():
|
||||
try:
|
||||
tok = path.read_text().strip()
|
||||
if tok:
|
||||
return tok
|
||||
except (PermissionError, OSError):
|
||||
pass
|
||||
# Migrate legacy token from cell_config.json
|
||||
old_token = self.configs.get('ddns', {}).get('token', '')
|
||||
if old_token:
|
||||
self.set_ddns_token(old_token)
|
||||
return old_token
|
||||
|
||||
def set_ddns_token(self, token: str) -> None:
|
||||
"""Write the DDNS bearer token to data/api/ddns_token (not cell_config.json)."""
|
||||
path = self._ddns_token_path
|
||||
try:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(token)
|
||||
except (PermissionError, OSError) as exc:
|
||||
logger.error('set_ddns_token: failed to write token file: %s', exc)
|
||||
return
|
||||
# Remove from cell_config.json if a legacy copy is there
|
||||
if self.configs.get('ddns', {}).get('token'):
|
||||
ddns_cfg = {k: v for k, v in self.configs.get('ddns', {}).items() if k != 'token'}
|
||||
self.configs['ddns'] = ddns_cfg
|
||||
self._save_all_configs()
|
||||
|
||||
def set_connectivity_field(self, field: str, value: Any) -> bool:
|
||||
"""Set a single field within the connectivity config and persist."""
|
||||
cfg = self.configs.setdefault('connectivity', {'exits': {}, 'peer_exit_map': {}})
|
||||
|
||||
@@ -80,19 +80,56 @@ class ConnectivityManager(BaseServiceManager):
|
||||
self.config_manager = config_manager
|
||||
self.peer_registry = peer_registry
|
||||
|
||||
# Config file directories
|
||||
self.connectivity_config_dir = os.path.join(config_dir, 'connectivity')
|
||||
self.wireguard_ext_dir = os.path.join(self.connectivity_config_dir, 'wireguard_ext')
|
||||
self.openvpn_dir = os.path.join(self.connectivity_config_dir, 'openvpn')
|
||||
# Connectivity configs live under the per-service data dir so that
|
||||
# ${PIC_DATA_DIR}/services/<id>/config bind mounts in store compose
|
||||
# templates can read them (Docker daemon resolves paths on the HOST,
|
||||
# so they must be reachable via data_dir, not config_dir).
|
||||
services_dir = os.path.join(data_dir, 'services')
|
||||
self.wireguard_ext_dir = os.path.join(services_dir, 'wireguard-ext', 'config')
|
||||
self.openvpn_dir = os.path.join(services_dir, 'openvpn-client', 'config')
|
||||
|
||||
for d in (self.connectivity_config_dir, self.wireguard_ext_dir, self.openvpn_dir):
|
||||
for d in (self.wireguard_ext_dir, self.openvpn_dir):
|
||||
self.safe_makedirs(d)
|
||||
|
||||
# One-shot migration from the legacy config_dir/connectivity/ location.
|
||||
_legacy_base = os.path.join(config_dir, 'connectivity')
|
||||
self._migrate_legacy_configs(_legacy_base)
|
||||
|
||||
# Subscribe to ServiceBus CONFIG_CHANGED events so routes are
|
||||
# reapplied if the underlying network changes. Done lazily —
|
||||
# service_bus is a singleton imported at app startup.
|
||||
self._subscribe_to_events()
|
||||
|
||||
# ── Legacy migration ──────────────────────────────────────────────────
|
||||
|
||||
def _migrate_legacy_configs(self, legacy_base: str) -> None:
|
||||
"""Copy files from the old config_dir/connectivity/ tree to the new data_dir locations.
|
||||
|
||||
The old layout stored WireGuard and OpenVPN configs under the API container's
|
||||
config_dir, which Docker cannot bind-mount into store-service containers. Files
|
||||
are copied (not moved) so the legacy location still works until the operator
|
||||
removes it manually.
|
||||
"""
|
||||
import shutil
|
||||
|
||||
pairs = (
|
||||
(os.path.join(legacy_base, 'wireguard_ext'), self.wireguard_ext_dir),
|
||||
(os.path.join(legacy_base, 'openvpn'), self.openvpn_dir),
|
||||
)
|
||||
for src_dir, dst_dir in pairs:
|
||||
if not os.path.isdir(src_dir):
|
||||
continue
|
||||
try:
|
||||
for fname in os.listdir(src_dir):
|
||||
src_file = os.path.join(src_dir, fname)
|
||||
dst_file = os.path.join(dst_dir, fname)
|
||||
if os.path.isfile(src_file) and not os.path.exists(dst_file):
|
||||
shutil.copy2(src_file, dst_file)
|
||||
os.chmod(dst_file, 0o600)
|
||||
logger.info('connectivity: migrated %s → %s', src_file, dst_file)
|
||||
except OSError as e:
|
||||
logger.warning('connectivity: migration from %s failed: %s', src_dir, e)
|
||||
|
||||
# ── Event wiring ──────────────────────────────────────────────────────
|
||||
|
||||
def _subscribe_to_events(self) -> None:
|
||||
|
||||
+136
-38
@@ -17,6 +17,7 @@ every 5 minutes, skipping the call when the IP has not changed.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from typing import Any, Dict, Optional
|
||||
@@ -36,6 +37,10 @@ class DDNSError(Exception):
|
||||
"""Raised when a DDNS provider returns an error response."""
|
||||
|
||||
|
||||
class DDNSTokenExpired(DDNSError):
|
||||
"""Raised when the DDNS service rejects the token (401) — usually after a DB reset."""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Provider base class
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -68,13 +73,25 @@ class PicNgoDDNS(DDNSProvider):
|
||||
DEFAULT_API_BASE = 'https://ddns.pic.ngo'
|
||||
TIMEOUT = 10
|
||||
|
||||
def __init__(self, api_base_url: Optional[str] = None):
|
||||
def __init__(self, api_base_url: Optional[str] = None, totp_secret: Optional[str] = None):
|
||||
self.api_base_url = (api_base_url or self.DEFAULT_API_BASE).rstrip('/')
|
||||
self._totp_secret = totp_secret or ''
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Internal helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _otp_header(self) -> Dict[str, str]:
|
||||
"""Generate a fresh TOTP header for /register calls."""
|
||||
if not self._totp_secret:
|
||||
return {}
|
||||
try:
|
||||
import pyotp
|
||||
return {'X-Register-OTP': pyotp.TOTP(self._totp_secret).now()}
|
||||
except ImportError:
|
||||
logger.warning("pyotp not installed — X-Register-OTP header omitted")
|
||||
return {}
|
||||
|
||||
def _headers(self, token: Optional[str] = None) -> Dict[str, str]:
|
||||
h: Dict[str, str] = {'Content-Type': 'application/json'}
|
||||
if token:
|
||||
@@ -83,6 +100,10 @@ class PicNgoDDNS(DDNSProvider):
|
||||
|
||||
def _raise_for_status(self, response: requests.Response, action: str):
|
||||
if not response.ok:
|
||||
if response.status_code == 401:
|
||||
raise DDNSTokenExpired(
|
||||
f"PicNgoDDNS {action} rejected token: HTTP 401 — {response.text}"
|
||||
)
|
||||
raise DDNSError(
|
||||
f"PicNgoDDNS {action} failed: HTTP {response.status_code} — {response.text}"
|
||||
)
|
||||
@@ -91,20 +112,30 @@ class PicNgoDDNS(DDNSProvider):
|
||||
# Public interface
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def release(self, token: str) -> bool:
|
||||
"""DELETE /api/v1/registration — release the subdomain owned by token."""
|
||||
url = f'{self.api_base_url}/api/v1/registration'
|
||||
resp = requests.delete(url, json={'token': token},
|
||||
headers=self._headers(), timeout=self.TIMEOUT)
|
||||
self._raise_for_status(resp, 'release')
|
||||
return True
|
||||
|
||||
def register(self, name: str, ip: str) -> dict:
|
||||
"""POST /api/v1/register — register subdomain, returns token + subdomain."""
|
||||
url = f'{self.api_base_url}/api/v1/register'
|
||||
payload = {'name': name, 'ip': ip}
|
||||
resp = requests.post(url, json=payload, headers=self._headers(), timeout=self.TIMEOUT)
|
||||
headers = {**self._headers(), **self._otp_header()}
|
||||
resp = requests.post(url, json=payload, headers=headers, timeout=self.TIMEOUT)
|
||||
self._raise_for_status(resp, 'register')
|
||||
return resp.json()
|
||||
|
||||
def update(self, token: str, ip: str) -> bool:
|
||||
"""PUT /api/v1/update — update A record."""
|
||||
url = f'{self.api_base_url}/api/v1/update'
|
||||
payload = {'ip': ip}
|
||||
# DDNS server validates token from request body, not Authorization header
|
||||
payload = {'ip': ip, 'token': token}
|
||||
resp = requests.put(url, json=payload,
|
||||
headers=self._headers(token), timeout=self.TIMEOUT)
|
||||
headers=self._headers(), timeout=self.TIMEOUT)
|
||||
self._raise_for_status(resp, 'update')
|
||||
return True
|
||||
|
||||
@@ -268,9 +299,11 @@ class DDNSManager(BaseServiceManager):
|
||||
|
||||
def __init__(self, config_manager=None,
|
||||
data_dir: str = '/app/data',
|
||||
config_dir: str = '/app/config'):
|
||||
config_dir: str = '/app/config',
|
||||
service_bus=None):
|
||||
super().__init__('ddns', data_dir, config_dir)
|
||||
self.config_manager = config_manager
|
||||
self._service_bus = service_bus
|
||||
self._last_ip: Optional[str] = None
|
||||
self._stop_event = threading.Event()
|
||||
self._heartbeat_thread: Optional[threading.Thread] = None
|
||||
@@ -280,11 +313,9 @@ class DDNSManager(BaseServiceManager):
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def get_status(self) -> Dict[str, Any]:
|
||||
identity = self._identity()
|
||||
domain_cfg = identity.get('domain', {})
|
||||
return {
|
||||
'service': 'ddns',
|
||||
'provider': domain_cfg.get('ddns', {}).get('provider') if domain_cfg else None,
|
||||
'provider': self._ddns_cfg().get('provider'),
|
||||
'last_ip': self._last_ip,
|
||||
'heartbeat_running': (
|
||||
self._heartbeat_thread is not None and
|
||||
@@ -310,17 +341,41 @@ class DDNSManager(BaseServiceManager):
|
||||
return {}
|
||||
return self.config_manager.get_identity() or {}
|
||||
|
||||
def _ddns_cfg(self) -> Dict[str, Any]:
|
||||
if self.config_manager is None:
|
||||
return {}
|
||||
return self.config_manager.configs.get('ddns', {}) or {}
|
||||
|
||||
def _get_token(self) -> str:
|
||||
"""Return the DDNS bearer token from the secure token store."""
|
||||
if self.config_manager is None:
|
||||
return ''
|
||||
if hasattr(self.config_manager, 'get_ddns_token'):
|
||||
return self.config_manager.get_ddns_token() or ''
|
||||
return self.config_manager.configs.get('ddns', {}).get('token', '')
|
||||
|
||||
def _fire_identity_changed(self, source: str) -> None:
|
||||
"""Publish IDENTITY_CHANGED so CaddyManager regenerates its config."""
|
||||
if self._service_bus is None:
|
||||
return
|
||||
try:
|
||||
from service_bus import EventType
|
||||
cell_name = self._identity().get('cell_name', '')
|
||||
self._service_bus.publish_event(EventType.IDENTITY_CHANGED, source, {
|
||||
'cell_name': cell_name,
|
||||
})
|
||||
except Exception as exc:
|
||||
logger.warning('DDNSManager._fire_identity_changed: %s', exc)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Provider factory
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def get_provider(self) -> Optional[DDNSProvider]:
|
||||
"""Instantiate and return the configured DDNS provider, or None."""
|
||||
identity = self._identity()
|
||||
domain_cfg = identity.get('domain', {})
|
||||
if not domain_cfg:
|
||||
if self.config_manager is None:
|
||||
return None
|
||||
ddns_cfg = domain_cfg.get('ddns', {})
|
||||
ddns_cfg = self.config_manager.configs.get('ddns', {})
|
||||
if not ddns_cfg:
|
||||
return None
|
||||
|
||||
@@ -329,8 +384,11 @@ class DDNSManager(BaseServiceManager):
|
||||
return None
|
||||
|
||||
if provider_name == 'pic_ngo':
|
||||
api_base = ddns_cfg.get('api_base_url')
|
||||
return PicNgoDDNS(api_base_url=api_base)
|
||||
# Env var takes priority so deployments can switch URLs without re-registering
|
||||
_env_url = os.environ.get('DDNS_URL', '').replace('/api/v1', '').rstrip('/')
|
||||
api_base = _env_url or ddns_cfg.get('api_base_url')
|
||||
totp_secret = ddns_cfg.get('totp_secret') or os.environ.get('DDNS_TOTP_SECRET', '')
|
||||
return PicNgoDDNS(api_base_url=api_base, totp_secret=totp_secret)
|
||||
|
||||
if provider_name == 'cloudflare':
|
||||
return CloudflareDDNS(
|
||||
@@ -360,27 +418,44 @@ class DDNSManager(BaseServiceManager):
|
||||
def register(self, name: str, ip: str) -> dict:
|
||||
"""Register the cell's subdomain with the configured provider.
|
||||
|
||||
Stores the returned token in the identity config under
|
||||
identity['domain']['ddns']['token'] and records the subdomain.
|
||||
Fetches the public IP via ipify when ip is empty.
|
||||
Stores the returned token in the top-level ddns config (where
|
||||
update_ip reads it) and updates _identity.domain_name.
|
||||
Returns the dict from provider.register().
|
||||
"""
|
||||
provider = self.get_provider()
|
||||
if provider is None:
|
||||
raise DDNSError("No DDNS provider configured")
|
||||
|
||||
if not ip:
|
||||
ip = _get_public_ip() or ''
|
||||
|
||||
# Release the old subdomain if the name is changing and we hold a token
|
||||
if self.config_manager is not None and hasattr(provider, 'release'):
|
||||
old_token = self._get_token()
|
||||
old_domain = self._identity().get('domain_name', '')
|
||||
old_name = old_domain.replace('.pic.ngo', '') if old_domain else ''
|
||||
if old_token and old_name and old_name != name:
|
||||
try:
|
||||
provider.release(old_token)
|
||||
logger.info("DDNS released old subdomain %r before registering %r", old_name, name)
|
||||
except Exception as exc:
|
||||
logger.warning("DDNS could not release old subdomain %r: %s", old_name, exc)
|
||||
|
||||
result = provider.register(name, ip)
|
||||
|
||||
# Persist token + subdomain back into identity
|
||||
identity = self._identity()
|
||||
domain_cfg = dict(identity.get('domain', {}))
|
||||
ddns_cfg = dict(domain_cfg.get('ddns', {}))
|
||||
if 'token' in result:
|
||||
ddns_cfg['token'] = result['token']
|
||||
if 'subdomain' in result:
|
||||
ddns_cfg['subdomain'] = result['subdomain']
|
||||
domain_cfg['ddns'] = ddns_cfg
|
||||
if self.config_manager is not None:
|
||||
self.config_manager.set_identity_field('domain', domain_cfg)
|
||||
# Token stored in data/api/ddns_token (not cell_config.json)
|
||||
if 'token' in result:
|
||||
if hasattr(self.config_manager, 'set_ddns_token'):
|
||||
self.config_manager.set_ddns_token(result['token'])
|
||||
else:
|
||||
ddns_cfg = dict(self.config_manager.configs.get('ddns', {}))
|
||||
ddns_cfg['token'] = result['token']
|
||||
self.config_manager.set_ddns_config(ddns_cfg)
|
||||
# Keep domain_name in identity up to date
|
||||
if 'subdomain' in result:
|
||||
self.config_manager.set_identity_field('domain_name', result['subdomain'])
|
||||
|
||||
self._last_ip = ip
|
||||
return result
|
||||
@@ -405,10 +480,26 @@ class DDNSManager(BaseServiceManager):
|
||||
logger.debug("DDNS update_ip: IP unchanged (%s), skipping", current_ip)
|
||||
return
|
||||
|
||||
identity = self._identity()
|
||||
domain_cfg = identity.get('domain', {})
|
||||
ddns_cfg = domain_cfg.get('ddns', {}) if domain_cfg else {}
|
||||
token = ddns_cfg.get('token', '')
|
||||
token = self._get_token()
|
||||
|
||||
# No token means we never successfully registered (e.g. wizard failed).
|
||||
# Attempt registration immediately rather than waiting for the 401 cycle.
|
||||
if not token:
|
||||
provider_name = self._ddns_cfg().get('provider', '')
|
||||
if provider_name == 'pic_ngo':
|
||||
logger.info("DDNS update_ip: no token — attempting initial registration")
|
||||
try:
|
||||
cell_name = self._identity().get('cell_name', '')
|
||||
if cell_name:
|
||||
self.register(cell_name, current_ip)
|
||||
logger.info("DDNS registered (no-token retry): cell_name=%r", cell_name)
|
||||
self._last_ip = current_ip
|
||||
self._fire_identity_changed('ddns_heartbeat')
|
||||
else:
|
||||
logger.error("DDNS update_ip: cannot register — cell_name not in identity")
|
||||
except Exception as exc:
|
||||
logger.error("DDNS update_ip: initial registration failed: %s", exc)
|
||||
return
|
||||
|
||||
try:
|
||||
success = provider.update(token, current_ip)
|
||||
@@ -417,6 +508,19 @@ class DDNSManager(BaseServiceManager):
|
||||
self._last_ip = current_ip
|
||||
else:
|
||||
logger.warning("DDNS update_ip: provider.update() returned False")
|
||||
except DDNSTokenExpired:
|
||||
logger.warning("DDNS update_ip: token rejected (401) — attempting re-registration")
|
||||
try:
|
||||
cell_name = self._identity().get('cell_name', '')
|
||||
if cell_name:
|
||||
self.register(cell_name, current_ip)
|
||||
logger.info("DDNS re-registered after token expiry: cell_name=%r", cell_name)
|
||||
self._last_ip = current_ip
|
||||
self._fire_identity_changed('ddns_heartbeat')
|
||||
else:
|
||||
logger.error("DDNS update_ip: cannot re-register — cell_name not in identity")
|
||||
except Exception as exc2:
|
||||
logger.error("DDNS update_ip: re-registration failed: %s", exc2)
|
||||
except DDNSError as exc:
|
||||
logger.error("DDNS update_ip: provider error: %s", exc)
|
||||
|
||||
@@ -468,10 +572,7 @@ class DDNSManager(BaseServiceManager):
|
||||
provider = self.get_provider()
|
||||
if provider is None:
|
||||
raise DDNSError("No DDNS provider configured")
|
||||
identity = self._identity()
|
||||
domain_cfg = identity.get('domain', {})
|
||||
ddns_cfg = domain_cfg.get('ddns', {}) if domain_cfg else {}
|
||||
token = ddns_cfg.get('token', '')
|
||||
token = self._get_token()
|
||||
return provider.dns_challenge_create(token, fqdn, value)
|
||||
|
||||
def dns_challenge_delete(self, fqdn: str) -> bool:
|
||||
@@ -479,8 +580,5 @@ class DDNSManager(BaseServiceManager):
|
||||
provider = self.get_provider()
|
||||
if provider is None:
|
||||
raise DDNSError("No DDNS provider configured")
|
||||
identity = self._identity()
|
||||
domain_cfg = identity.get('domain', {})
|
||||
ddns_cfg = domain_cfg.get('ddns', {}) if domain_cfg else {}
|
||||
token = ddns_cfg.get('token', '')
|
||||
token = self._get_token()
|
||||
return provider.dns_challenge_delete(token, fqdn)
|
||||
|
||||
@@ -0,0 +1,352 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
EgressManager — per-service egress enforcement.
|
||||
|
||||
Routes outbound traffic from installed service containers through
|
||||
alternate exits (wireguard_ext, openvpn, tor) using host-side
|
||||
iptables fwmark policy-routing. Integrates with ServiceStoreManager
|
||||
for install/remove lifecycle hooks.
|
||||
|
||||
Rules live on the HOST in PIC_EGRESS chains in the mangle and nat
|
||||
tables. Container IPs are discovered via docker inspect using the
|
||||
container_name from the service manifest. Marks are distinct from
|
||||
ConnectivityManager to prevent rule collisions.
|
||||
"""
|
||||
import logging
|
||||
import subprocess
|
||||
import time
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
EXIT_TYPES = ("default", "wireguard_ext", "openvpn", "tor")
|
||||
|
||||
# fwmark values — must not collide with ConnectivityManager (0x10, 0x20, 0x30)
|
||||
MARKS = {"wireguard_ext": 0x110, "openvpn": 0x120, "tor": 0x130}
|
||||
|
||||
# Policy routing table IDs
|
||||
TABLES = {"wireguard_ext": 210, "openvpn": 220, "tor": 230}
|
||||
|
||||
EGRESS_CHAIN = "PIC_EGRESS"
|
||||
|
||||
# Transparent proxy port used by Tor
|
||||
_TOR_TRANS_PORT = 9040
|
||||
|
||||
|
||||
class EgressManager:
|
||||
"""Per-service egress enforcement via host iptables fwmark policy-routing."""
|
||||
|
||||
def __init__(self, config_manager, service_store_manager=None,
|
||||
data_dir: str = "/app/data", config_dir: str = "/app/config"):
|
||||
self.config_manager = config_manager
|
||||
self.service_store_manager = service_store_manager
|
||||
self._data_dir = data_dir
|
||||
self._config_dir = config_dir
|
||||
|
||||
# ── Public API ─────────────────────────────────────────────────────────
|
||||
|
||||
def apply_service(self, service_id: str) -> Dict[str, Any]:
|
||||
"""Idempotently apply egress rules for one installed service.
|
||||
|
||||
Steps:
|
||||
1. Look up the service manifest.
|
||||
2. clear_service first (ensures idempotency).
|
||||
3. If the manifest has no egress block, skip silently.
|
||||
4. Discover the container IP.
|
||||
5. Resolve the exit type (override > manifest default > 'default').
|
||||
6. If exit is 'default', return early with no rules.
|
||||
7. Otherwise create chains, ensure ip rules, add mark rules.
|
||||
"""
|
||||
manifest = self._get_manifest(service_id)
|
||||
if manifest is None:
|
||||
return {'ok': False, 'error': f'manifest not found for {service_id}'}
|
||||
|
||||
# Always clear first for idempotency
|
||||
self.clear_service(service_id)
|
||||
|
||||
if not self._has_egress(manifest):
|
||||
return {'ok': True, 'skipped': True}
|
||||
|
||||
container_name = manifest.get('container_name', '')
|
||||
container_ip = self._discover_container_ip(container_name)
|
||||
if not container_ip:
|
||||
return {'ok': False, 'error': 'container IP not discoverable'}
|
||||
|
||||
exit_via = self._resolve_exit(service_id, manifest)
|
||||
|
||||
# Validate exit_via is a known, non-default value
|
||||
if exit_via not in EXIT_TYPES:
|
||||
return {
|
||||
'ok': False,
|
||||
'error': f'unknown exit_via {exit_via!r}; must be one of {EXIT_TYPES}',
|
||||
}
|
||||
|
||||
if exit_via == 'default':
|
||||
return {'ok': True, 'exit_via': 'default'}
|
||||
|
||||
if exit_via not in MARKS:
|
||||
return {
|
||||
'ok': False,
|
||||
'error': f'unknown exit_via {exit_via!r}; must be one of {EXIT_TYPES}',
|
||||
}
|
||||
|
||||
try:
|
||||
self._ensure_chains()
|
||||
self._ensure_host_ip_rules()
|
||||
self._add_mark_rule(container_ip, MARKS[exit_via], service_id)
|
||||
if exit_via == 'tor':
|
||||
self._add_tor_redirect(container_ip, service_id)
|
||||
except Exception as exc:
|
||||
logger.error('apply_service(%s): %s', service_id, exc)
|
||||
return {'ok': False, 'error': str(exc)}
|
||||
|
||||
return {'ok': True, 'exit_via': exit_via, 'container_ip': container_ip}
|
||||
|
||||
def clear_service(self, service_id: str) -> Dict[str, Any]:
|
||||
"""Remove all PIC_EGRESS rules tagged for this service."""
|
||||
try:
|
||||
self._clear_egress_rules(service_id)
|
||||
return {'ok': True}
|
||||
except Exception as exc:
|
||||
logger.error('clear_service(%s): %s', service_id, exc)
|
||||
return {'ok': False, 'error': str(exc)}
|
||||
|
||||
def apply_all(self) -> Dict[str, Any]:
|
||||
"""Apply egress rules for every installed service that has a manifest."""
|
||||
installed = self.config_manager.get_installed_services()
|
||||
results: Dict[str, Any] = {}
|
||||
for svc_id, record in installed.items():
|
||||
if not isinstance(record, dict) or not record.get('manifest'):
|
||||
continue
|
||||
results[svc_id] = self.apply_service(svc_id)
|
||||
return {'ok': True, 'services': results}
|
||||
|
||||
def set_service_exit(self, service_id: str, exit_type: str) -> Dict[str, Any]:
|
||||
"""Persist a per-service egress override and immediately reapply rules.
|
||||
|
||||
exit_type must appear in the manifest's egress.allowed list.
|
||||
"""
|
||||
manifest = self._get_manifest(service_id)
|
||||
if manifest is None:
|
||||
return {'ok': False, 'error': f'service {service_id!r} not installed'}
|
||||
|
||||
if not self._has_egress(manifest):
|
||||
return {'ok': False, 'error': f'service {service_id!r} has no egress configuration'}
|
||||
|
||||
egress = manifest.get('egress', {})
|
||||
allowed = egress.get('allowed', list(EXIT_TYPES))
|
||||
|
||||
if exit_type not in allowed:
|
||||
return {
|
||||
'ok': False,
|
||||
'error': (
|
||||
f'exit_type {exit_type!r} is not in the allowed list '
|
||||
f'for {service_id}: {allowed}'
|
||||
),
|
||||
}
|
||||
|
||||
if exit_type not in EXIT_TYPES:
|
||||
return {
|
||||
'ok': False,
|
||||
'error': f'unknown exit_type {exit_type!r}; must be one of {EXIT_TYPES}',
|
||||
}
|
||||
|
||||
# Persist the override so it survives restarts
|
||||
overrides = self._get_egress_overrides()
|
||||
overrides[service_id] = exit_type
|
||||
self._set_egress_overrides(overrides)
|
||||
|
||||
return self.apply_service(service_id)
|
||||
|
||||
def get_status(self) -> Dict[str, Any]:
|
||||
"""Return egress status for every installed service that has egress config."""
|
||||
installed = self.config_manager.get_installed_services()
|
||||
statuses: Dict[str, Any] = {}
|
||||
for svc_id, record in installed.items():
|
||||
if not isinstance(record, dict):
|
||||
continue
|
||||
manifest = record.get('manifest')
|
||||
if not manifest or not self._has_egress(manifest):
|
||||
continue
|
||||
container_name = manifest.get('container_name', '')
|
||||
container_ip = self._discover_container_ip(container_name, retries=1)
|
||||
exit_via = self._resolve_exit(svc_id, manifest)
|
||||
statuses[svc_id] = {
|
||||
'exit_via': exit_via,
|
||||
'container_ip': container_ip,
|
||||
'has_egress': True,
|
||||
}
|
||||
return {'ok': True, 'services': statuses}
|
||||
|
||||
# ── Internals ──────────────────────────────────────────────────────────
|
||||
|
||||
def _get_manifest(self, service_id: str) -> Optional[dict]:
|
||||
"""Retrieve the manifest for an installed service, if available."""
|
||||
installed = self.config_manager.get_installed_services()
|
||||
record = installed.get(service_id)
|
||||
if not record:
|
||||
return None
|
||||
return record.get('manifest')
|
||||
|
||||
def _has_egress(self, manifest: dict) -> bool:
|
||||
"""Return True only when the manifest explicitly declares an egress block."""
|
||||
return bool(manifest.get('has_egress', False) and manifest.get('egress'))
|
||||
|
||||
def _resolve_exit(self, service_id: str, manifest: dict) -> str:
|
||||
"""Determine the effective exit for a service.
|
||||
|
||||
Priority: persisted override > manifest egress.default > 'default'.
|
||||
"""
|
||||
overrides = self._get_egress_overrides()
|
||||
if service_id in overrides:
|
||||
return overrides[service_id]
|
||||
egress = manifest.get('egress') or {}
|
||||
return egress.get('default', 'default')
|
||||
|
||||
def _discover_container_ip(self, container_name: str,
|
||||
retries: int = 5, delay: float = 0.2) -> Optional[str]:
|
||||
"""Return the container's cell-network IP, retrying on transient failure."""
|
||||
if not container_name:
|
||||
return None
|
||||
for attempt in range(retries):
|
||||
result = subprocess.run(
|
||||
[
|
||||
'docker', 'inspect',
|
||||
'-f', '{{.NetworkSettings.Networks.cell-network.IPAddress}}',
|
||||
container_name,
|
||||
],
|
||||
capture_output=True, text=True, timeout=10,
|
||||
)
|
||||
ip = result.stdout.strip()
|
||||
if ip and result.returncode == 0:
|
||||
return ip
|
||||
if attempt < retries - 1:
|
||||
time.sleep(delay)
|
||||
return None
|
||||
|
||||
def _ensure_chains(self) -> None:
|
||||
"""Idempotently create PIC_EGRESS chains in mangle and nat on the host."""
|
||||
for table in ('mangle', 'nat'):
|
||||
# Create the chain if it does not yet exist
|
||||
check = self._iptables(['-t', table, '-L', EGRESS_CHAIN, '-n'])
|
||||
if check.returncode != 0:
|
||||
create = self._iptables(['-t', table, '-N', EGRESS_CHAIN])
|
||||
if create.returncode != 0 and 'exists' not in (create.stderr or ''):
|
||||
logger.warning(
|
||||
'_ensure_chains: cannot create %s/%s: %s',
|
||||
table, EGRESS_CHAIN, (create.stderr or '').strip(),
|
||||
)
|
||||
|
||||
# Insert jump from PREROUTING at position 1 (idempotent via -C check)
|
||||
jump_check = self._iptables(
|
||||
['-t', table, '-C', 'PREROUTING', '-j', EGRESS_CHAIN]
|
||||
)
|
||||
if jump_check.returncode != 0:
|
||||
self._iptables(
|
||||
['-t', table, '-I', 'PREROUTING', '1', '-j', EGRESS_CHAIN]
|
||||
)
|
||||
|
||||
def _ensure_host_ip_rules(self) -> None:
|
||||
"""Ensure `ip rule fwmark <mark> lookup <table>` exists for each exit."""
|
||||
for exit_type, mark in MARKS.items():
|
||||
table = TABLES[exit_type]
|
||||
# Remove any existing duplicate rules first, then add once
|
||||
for _ in range(8):
|
||||
r = self._ip_rule(['del', 'fwmark', hex(mark), 'lookup', str(table)])
|
||||
if r.returncode != 0:
|
||||
break
|
||||
self._ip_rule(['add', 'fwmark', hex(mark), 'lookup', str(table)])
|
||||
|
||||
def _add_mark_rule(self, service_ip: str, mark: int, service_id: str) -> None:
|
||||
"""Mark outbound packets from the service container with fwmark."""
|
||||
self._iptables([
|
||||
'-t', 'mangle', '-A', EGRESS_CHAIN,
|
||||
'-s', service_ip,
|
||||
'-j', 'MARK', '--set-mark', hex(mark),
|
||||
'-m', 'comment', '--comment', self._tag(service_id),
|
||||
])
|
||||
|
||||
def _add_tor_redirect(self, service_ip: str, service_id: str) -> None:
|
||||
"""Redirect the service container's TCP traffic to the local Tor TransPort."""
|
||||
self._iptables([
|
||||
'-t', 'nat', '-A', EGRESS_CHAIN,
|
||||
'-s', service_ip, '-p', 'tcp',
|
||||
'-j', 'REDIRECT', '--to-ports', str(_TOR_TRANS_PORT),
|
||||
'-m', 'comment', '--comment', self._tag(service_id),
|
||||
])
|
||||
|
||||
def _clear_egress_rules(self, service_id: str) -> None:
|
||||
"""Remove all rules tagged pic-egr-<service_id> from mangle and nat."""
|
||||
import re as _re
|
||||
tag = self._tag(service_id)
|
||||
comment_re = _re.compile(
|
||||
rf'--comment\s+["\']?{_re.escape(tag)}["\']?(\s|$)'
|
||||
)
|
||||
for table in ('mangle', 'nat'):
|
||||
try:
|
||||
save = subprocess.run(
|
||||
['iptables-save', '-t', table],
|
||||
capture_output=True, text=True, timeout=10,
|
||||
)
|
||||
if save.returncode != 0:
|
||||
continue
|
||||
lines = save.stdout.splitlines()
|
||||
filtered = [ln for ln in lines if not comment_re.search(ln)]
|
||||
if len(filtered) == len(lines):
|
||||
continue # nothing to remove
|
||||
restore_input = '\n'.join(filtered) + '\n'
|
||||
restore = subprocess.run(
|
||||
['iptables-restore', '-T', table],
|
||||
input=restore_input,
|
||||
capture_output=True, text=True, timeout=10,
|
||||
)
|
||||
if restore.returncode != 0:
|
||||
logger.warning(
|
||||
'_clear_egress_rules(%s): iptables-restore for %s failed: %s',
|
||||
service_id, table, (restore.stderr or '').strip(),
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.error('_clear_egress_rules(%s, %s): %s', service_id, table, exc)
|
||||
|
||||
@staticmethod
|
||||
def _tag(service_id: str) -> str:
|
||||
"""iptables comment tag used to identify rules belonging to a service."""
|
||||
return f'pic-egr-{service_id}'
|
||||
|
||||
def _iptables(self, args: List[str], check: bool = False) -> subprocess.CompletedProcess:
|
||||
"""Run iptables on the host with the given arguments."""
|
||||
cmd = ['iptables'] + args
|
||||
try:
|
||||
return subprocess.run(cmd, capture_output=True, text=True, timeout=10)
|
||||
except Exception as exc:
|
||||
logger.error('_iptables %s: %s', args, exc)
|
||||
raise
|
||||
|
||||
def _ip_rule(self, args: List[str]) -> subprocess.CompletedProcess:
|
||||
"""Run `ip rule` on the host with the given arguments."""
|
||||
cmd = ['ip', 'rule'] + args
|
||||
try:
|
||||
return subprocess.run(cmd, capture_output=True, text=True, timeout=10)
|
||||
except Exception as exc:
|
||||
logger.error('_ip_rule %s: %s', args, exc)
|
||||
raise
|
||||
|
||||
# ── Config persistence helpers ─────────────────────────────────────────
|
||||
|
||||
def _get_egress_overrides(self) -> Dict[str, str]:
|
||||
"""Return the persisted egress override map {service_id: exit_type}."""
|
||||
try:
|
||||
overrides = self.config_manager.configs.get('egress_overrides')
|
||||
if isinstance(overrides, dict):
|
||||
return dict(overrides)
|
||||
except Exception:
|
||||
pass
|
||||
return {}
|
||||
|
||||
def _set_egress_overrides(self, overrides: Dict[str, str]) -> None:
|
||||
"""Persist the egress override map to config."""
|
||||
try:
|
||||
self.config_manager.configs['egress_overrides'] = overrides
|
||||
self.config_manager._save_all_configs()
|
||||
except Exception as exc:
|
||||
logger.error('_set_egress_overrides: %s', exc)
|
||||
+43
-1
@@ -19,7 +19,8 @@ logger = logging.getLogger(__name__)
|
||||
class EmailManager(BaseServiceManager):
|
||||
"""Manages email service configuration and users"""
|
||||
|
||||
def __init__(self, data_dir: str = '/app/data', config_dir: str = '/app/config'):
|
||||
def __init__(self, data_dir: str = '/app/data', config_dir: str = '/app/config',
|
||||
service_bus=None):
|
||||
super().__init__('email', data_dir, config_dir)
|
||||
self.email_data_dir = os.path.join(data_dir, 'email')
|
||||
self.email_dir = self.email_data_dir # alias used by tests
|
||||
@@ -33,6 +34,10 @@ class EmailManager(BaseServiceManager):
|
||||
self.safe_makedirs(self.dovecot_dir)
|
||||
self.safe_makedirs(os.path.dirname(self.domain_config_file))
|
||||
|
||||
if service_bus is not None:
|
||||
from service_bus import EventType
|
||||
service_bus.subscribe_to_event(EventType.IDENTITY_CHANGED, self._on_identity_changed)
|
||||
|
||||
def _get_service_config(self) -> Dict[str, Any]:
|
||||
"""Read configured ports/domain from service config file."""
|
||||
cfg = self.get_config()
|
||||
@@ -252,6 +257,15 @@ class EmailManager(BaseServiceManager):
|
||||
|
||||
return {'restarted': restarted, 'warnings': warnings}
|
||||
|
||||
def _on_identity_changed(self, event) -> None:
|
||||
"""Regenerate email config when cell identity changes."""
|
||||
try:
|
||||
effective = event.data.get('effective_domain')
|
||||
if effective:
|
||||
self.apply_config({'domain': effective})
|
||||
except Exception as exc:
|
||||
self.logger.warning('email_manager identity_changed handler failed: %s', exc)
|
||||
|
||||
def get_email_status(self) -> Dict[str, Any]:
|
||||
"""Get detailed email service status including postfix/dovecot state."""
|
||||
try:
|
||||
@@ -326,12 +340,39 @@ class EmailManager(BaseServiceManager):
|
||||
mailbox_dir = os.path.join(self.email_data_dir, 'mailboxes', f'{username}@{domain}')
|
||||
self.safe_makedirs(mailbox_dir)
|
||||
|
||||
# Provision account in docker-mailserver (non-fatal if container not running)
|
||||
self._dms_add_account(username, domain, password)
|
||||
|
||||
logger.info(f"Created email user: {username}@{domain}")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to create email user {username}@{domain}: {e}")
|
||||
return False
|
||||
|
||||
def _dms_add_account(self, username: str, domain: str, password: str) -> None:
|
||||
try:
|
||||
r = subprocess.run(
|
||||
['docker', 'exec', 'cell-mail', 'setup', 'email', 'add',
|
||||
f'{username}@{domain}', password],
|
||||
capture_output=True, text=True, timeout=30, check=False,
|
||||
)
|
||||
if r.returncode != 0:
|
||||
logger.warning('dms add account %s@%s: %s', username, domain, r.stderr.strip())
|
||||
except Exception as e:
|
||||
logger.warning('dms add account %s@%s failed (non-fatal): %s', username, domain, e)
|
||||
|
||||
def _dms_del_account(self, username: str, domain: str) -> None:
|
||||
try:
|
||||
r = subprocess.run(
|
||||
['docker', 'exec', 'cell-mail', 'setup', 'email', 'del',
|
||||
f'{username}@{domain}'],
|
||||
capture_output=True, text=True, timeout=30, check=False,
|
||||
)
|
||||
if r.returncode != 0:
|
||||
logger.warning('dms del account %s@%s: %s', username, domain, r.stderr.strip())
|
||||
except Exception as e:
|
||||
logger.warning('dms del account %s@%s failed (non-fatal): %s', username, domain, e)
|
||||
|
||||
def delete_email_user(self, username: str, domain: str) -> bool:
|
||||
"""Delete an email user"""
|
||||
try:
|
||||
@@ -352,6 +393,7 @@ class EmailManager(BaseServiceManager):
|
||||
import shutil
|
||||
shutil.rmtree(mailbox_dir)
|
||||
|
||||
self._dms_del_account(username, domain)
|
||||
logger.info(f"Deleted email user: {username}@{domain}")
|
||||
return True
|
||||
|
||||
|
||||
+50
-20
@@ -569,7 +569,7 @@ def ensure_dns_dnat() -> bool:
|
||||
|
||||
|
||||
def ensure_service_dnat() -> bool:
|
||||
"""DNAT wg0:80 (scoped to WG server IP) → cell-caddy:80.
|
||||
"""DNAT wg0:80 and wg0:443 (scoped to WG server IP) → cell-caddy.
|
||||
|
||||
Service DNS names resolve to the WG server IP. DNAT is scoped with -d {server_ip}
|
||||
so that cross-cell HTTP traffic destined for another cell passes through unmodified.
|
||||
@@ -583,21 +583,22 @@ def ensure_service_dnat() -> bool:
|
||||
if not caddy_ip:
|
||||
logger.warning('ensure_service_dnat: cell-caddy not found')
|
||||
return False
|
||||
dnat_check = ['-t', 'nat', '-C', 'PREROUTING', '-i', 'wg0', '-d', server_ip,
|
||||
'-p', 'tcp', '--dport', '80',
|
||||
'-j', 'DNAT', '--to-destination', f'{caddy_ip}:80']
|
||||
dnat_add = ['-t', 'nat', '-A', 'PREROUTING', '-i', 'wg0', '-d', server_ip,
|
||||
'-p', 'tcp', '--dport', '80',
|
||||
'-j', 'DNAT', '--to-destination', f'{caddy_ip}:80']
|
||||
if _wg_exec(['iptables'] + dnat_check).returncode != 0:
|
||||
_wg_exec(['iptables'] + dnat_add)
|
||||
fwd_check = ['-C', 'FORWARD', '-i', 'wg0', '-o', 'eth0',
|
||||
'-p', 'tcp', '--dport', '80', '-j', 'ACCEPT']
|
||||
fwd_add = ['-I', 'FORWARD', '-i', 'wg0', '-o', 'eth0',
|
||||
'-p', 'tcp', '--dport', '80', '-j', 'ACCEPT']
|
||||
if _wg_exec(['iptables'] + fwd_check).returncode != 0:
|
||||
_wg_exec(['iptables'] + fwd_add)
|
||||
logger.info(f'ensure_service_dnat: wg0:{server_ip}:80 → {caddy_ip}:80')
|
||||
for port in ('80', '443'):
|
||||
dnat_check = ['-t', 'nat', '-C', 'PREROUTING', '-i', 'wg0', '-d', server_ip,
|
||||
'-p', 'tcp', '--dport', port,
|
||||
'-j', 'DNAT', '--to-destination', f'{caddy_ip}:{port}']
|
||||
dnat_add = ['-t', 'nat', '-A', 'PREROUTING', '-i', 'wg0', '-d', server_ip,
|
||||
'-p', 'tcp', '--dport', port,
|
||||
'-j', 'DNAT', '--to-destination', f'{caddy_ip}:{port}']
|
||||
if _wg_exec(['iptables'] + dnat_check).returncode != 0:
|
||||
_wg_exec(['iptables'] + dnat_add)
|
||||
fwd_check = ['-C', 'FORWARD', '-i', 'wg0', '-o', 'eth0',
|
||||
'-p', 'tcp', '--dport', port, '-j', 'ACCEPT']
|
||||
fwd_add = ['-I', 'FORWARD', '-i', 'wg0', '-o', 'eth0',
|
||||
'-p', 'tcp', '--dport', port, '-j', 'ACCEPT']
|
||||
if _wg_exec(['iptables'] + fwd_check).returncode != 0:
|
||||
_wg_exec(['iptables'] + fwd_add)
|
||||
logger.info(f'ensure_service_dnat: wg0:{server_ip}:80+443 → {caddy_ip}')
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f'ensure_service_dnat: {e}')
|
||||
@@ -710,7 +711,8 @@ def _build_acl_block(blocked_peers_by_service: Dict[str, List[str]],
|
||||
|
||||
def generate_corefile(peers: List[Dict[str, Any]], corefile_path: str = COREFILE_PATH,
|
||||
domain: str = 'cell',
|
||||
cell_links: Optional[List[Dict[str, Any]]] = None) -> bool:
|
||||
cell_links: Optional[List[Dict[str, Any]]] = None,
|
||||
split_horizon_zones: Optional[List[str]] = None) -> bool:
|
||||
"""
|
||||
Rewrite the CoreDNS Corefile with per-peer ACL rules and reload plugin.
|
||||
The file is written to corefile_path (API-side path mapped into CoreDNS container).
|
||||
@@ -718,6 +720,10 @@ def generate_corefile(peers: List[Dict[str, Any]], corefile_path: str = COREFILE
|
||||
cell_links: optional list of cell-to-cell DNS forwarding entries, each a dict with
|
||||
'domain' and 'dns_ip' keys (same shape as CellLinkManager.list_connections()).
|
||||
When non-empty, a forwarding stanza is appended for each entry.
|
||||
split_horizon_zones: optional list of FQDNs (e.g. ['pic1.pic.ngo']) for which a
|
||||
local authoritative zone block is added so LAN clients resolve
|
||||
service subdomains to the internal Caddy IP without hairpin NAT.
|
||||
Each zone must have a corresponding zone file under /data/<fqdn>.zone.
|
||||
"""
|
||||
try:
|
||||
# Collect which peers block which services
|
||||
@@ -748,6 +754,29 @@ def generate_corefile(peers: List[Dict[str, Any]], corefile_path: str = COREFILE
|
||||
|
||||
{primary_zone_block}"""
|
||||
|
||||
# Split-horizon zones for DDNS/public domains — LAN clients resolve
|
||||
# *.pic1.pic.ngo to the internal Caddy IP without hairpin NAT.
|
||||
if split_horizon_zones:
|
||||
for sz in split_horizon_zones:
|
||||
# More-specific block for ACME DNS-01 challenge records: forward
|
||||
# to public DNS so Caddy can verify TXT records it creates on the
|
||||
# DDNS server. Without this, the wildcard A record in the zone
|
||||
# file causes CoreDNS to return NODATA for TXT queries, blocking
|
||||
# Caddy's internal pre-verification step.
|
||||
corefile += (
|
||||
f'\n_acme-challenge.{sz} {{\n'
|
||||
f' forward . 8.8.8.8 1.1.1.1\n'
|
||||
f' cache\n'
|
||||
f' log\n'
|
||||
f'}}\n'
|
||||
)
|
||||
corefile += (
|
||||
f'\n{sz} {{\n'
|
||||
f' file /data/{sz}.zone\n'
|
||||
f' log\n'
|
||||
f'}}\n'
|
||||
)
|
||||
|
||||
# Append cell-to-cell DNS forwarding stanzas if provided
|
||||
if cell_links:
|
||||
for link in cell_links:
|
||||
@@ -762,7 +791,7 @@ def generate_corefile(peers: List[Dict[str, Any]], corefile_path: str = COREFILE
|
||||
f' log\n'
|
||||
f'}}\n'
|
||||
)
|
||||
else:
|
||||
elif not split_horizon_zones:
|
||||
corefile += '\n'
|
||||
|
||||
# local.{domain} block intentionally omitted: /data/local.zone does not exist
|
||||
@@ -798,9 +827,10 @@ def reload_coredns() -> bool:
|
||||
|
||||
def apply_all_dns_rules(peers: List[Dict[str, Any]], corefile_path: str = COREFILE_PATH,
|
||||
domain: str = 'cell',
|
||||
cell_links: Optional[List[Dict[str, Any]]] = None) -> bool:
|
||||
cell_links: Optional[List[Dict[str, Any]]] = None,
|
||||
split_horizon_zones: Optional[List[str]] = None) -> bool:
|
||||
"""Regenerate Corefile (including any cell-to-cell forwarding stanzas) and reload CoreDNS."""
|
||||
ok = generate_corefile(peers, corefile_path, domain, cell_links)
|
||||
ok = generate_corefile(peers, corefile_path, domain, cell_links, split_horizon_zones)
|
||||
if ok:
|
||||
reload_coredns()
|
||||
return ok
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
"""One-shot cleanup of legacy builtin containers from the old main compose stack."""
|
||||
import logging
|
||||
import subprocess
|
||||
|
||||
logger = logging.getLogger('picell')
|
||||
|
||||
_LEGACY_BUILTIN_CONTAINERS = [
|
||||
'cell-mail', 'cell-rainloop', 'cell-radicale', 'cell-webdav', 'cell-filegator',
|
||||
]
|
||||
|
||||
|
||||
def cleanup_legacy_builtin_containers(config_manager) -> None:
|
||||
"""Remove legacy containers whose compose project is 'pic' (main stack).
|
||||
|
||||
Idempotent — guarded by _meta.legacy_builtins_cleaned in cell_config.json.
|
||||
Containers from per-service installs (project != 'pic') are left untouched.
|
||||
"""
|
||||
try:
|
||||
already_done = config_manager.configs.get('_meta', {}).get('legacy_builtins_cleaned', False)
|
||||
if already_done:
|
||||
return
|
||||
except Exception:
|
||||
return
|
||||
|
||||
removed = []
|
||||
for cname in _LEGACY_BUILTIN_CONTAINERS:
|
||||
try:
|
||||
inspect = subprocess.run(
|
||||
['docker', 'inspect', cname,
|
||||
'--format', '{{index .Config.Labels "com.docker.compose.project"}}'],
|
||||
capture_output=True, text=True, timeout=10,
|
||||
)
|
||||
if inspect.returncode != 0:
|
||||
continue
|
||||
project = inspect.stdout.strip()
|
||||
if project != 'pic':
|
||||
continue
|
||||
subprocess.run(['docker', 'stop', cname], capture_output=True, timeout=30)
|
||||
subprocess.run(['docker', 'rm', cname], capture_output=True, timeout=30)
|
||||
removed.append(cname)
|
||||
except Exception as exc:
|
||||
logger.warning('cleanup_legacy_builtin_containers: %s: %s', cname, exc)
|
||||
|
||||
try:
|
||||
meta = dict(config_manager.configs.get('_meta', {}))
|
||||
meta['legacy_builtins_cleaned'] = True
|
||||
config_manager.configs['_meta'] = meta
|
||||
config_manager._save_all_configs()
|
||||
except Exception as exc:
|
||||
logger.warning('cleanup_legacy_builtin_containers: failed to set sentinel: %s', exc)
|
||||
|
||||
if removed:
|
||||
logger.info('Removed legacy builtin containers: %s', ', '.join(removed))
|
||||
+41
-7
@@ -31,6 +31,9 @@ from setup_manager import SetupManager
|
||||
from caddy_manager import CaddyManager
|
||||
from ddns_manager import DDNSManager
|
||||
from connectivity_manager import ConnectivityManager
|
||||
from service_registry import ServiceRegistry
|
||||
from service_composer import ServiceComposer
|
||||
from account_manager import AccountManager
|
||||
|
||||
DATA_DIR = os.environ.get('DATA_DIR', '/app/data')
|
||||
CONFIG_DIR = os.environ.get('CONFIG_DIR', '/app/config')
|
||||
@@ -42,10 +45,16 @@ config_manager = ConfigManager(
|
||||
service_bus = ServiceBus()
|
||||
log_manager = LogManager(log_dir='./data/logs')
|
||||
|
||||
network_manager = NetworkManager(data_dir=DATA_DIR, config_dir=CONFIG_DIR)
|
||||
# ServiceRegistry depends only on config_manager; create it early so
|
||||
# NetworkManager and CaddyManager can derive subdomains from manifests
|
||||
# instead of hardcoding service names.
|
||||
service_registry = ServiceRegistry(config_manager=config_manager)
|
||||
|
||||
network_manager = NetworkManager(data_dir=DATA_DIR, config_dir=CONFIG_DIR,
|
||||
service_registry=service_registry)
|
||||
wireguard_manager = WireGuardManager(data_dir=DATA_DIR, config_dir=CONFIG_DIR)
|
||||
peer_registry = PeerRegistry(data_dir=DATA_DIR, config_dir=CONFIG_DIR)
|
||||
email_manager = EmailManager(data_dir=DATA_DIR, config_dir=CONFIG_DIR)
|
||||
email_manager = EmailManager(data_dir=DATA_DIR, config_dir=CONFIG_DIR, service_bus=service_bus)
|
||||
calendar_manager = CalendarManager(data_dir=DATA_DIR, config_dir=CONFIG_DIR)
|
||||
file_manager = FileManager(data_dir=DATA_DIR, config_dir=CONFIG_DIR)
|
||||
routing_manager = RoutingManager(data_dir=DATA_DIR, config_dir=CONFIG_DIR)
|
||||
@@ -57,9 +66,10 @@ cell_link_manager = CellLinkManager(
|
||||
network_manager=network_manager,
|
||||
)
|
||||
auth_manager = AuthManager(data_dir=DATA_DIR, config_dir=CONFIG_DIR)
|
||||
setup_manager = SetupManager(config_manager=config_manager, auth_manager=auth_manager)
|
||||
caddy_manager = CaddyManager(config_manager=config_manager, data_dir=DATA_DIR, config_dir=CONFIG_DIR)
|
||||
ddns_manager = DDNSManager(config_manager=config_manager, data_dir=DATA_DIR, config_dir=CONFIG_DIR)
|
||||
caddy_manager = CaddyManager(config_manager=config_manager, data_dir=DATA_DIR, config_dir=CONFIG_DIR,
|
||||
service_bus=service_bus, service_registry=service_registry)
|
||||
ddns_manager = DDNSManager(config_manager=config_manager, data_dir=DATA_DIR, config_dir=CONFIG_DIR,
|
||||
service_bus=service_bus)
|
||||
connectivity_manager = ConnectivityManager(
|
||||
config_manager=config_manager,
|
||||
peer_registry=peer_registry,
|
||||
@@ -67,6 +77,16 @@ connectivity_manager = ConnectivityManager(
|
||||
config_dir=CONFIG_DIR,
|
||||
)
|
||||
|
||||
service_composer = ServiceComposer(config_manager=config_manager, data_dir=DATA_DIR)
|
||||
account_manager = AccountManager(
|
||||
service_registry=service_registry,
|
||||
data_dir=DATA_DIR,
|
||||
config_manager=config_manager,
|
||||
email_manager=email_manager,
|
||||
calendar_manager=calendar_manager,
|
||||
file_manager=file_manager,
|
||||
)
|
||||
|
||||
from service_store_manager import ServiceStoreManager
|
||||
service_store_manager = ServiceStoreManager(
|
||||
config_manager=config_manager,
|
||||
@@ -74,8 +94,20 @@ service_store_manager = ServiceStoreManager(
|
||||
container_manager=container_manager,
|
||||
data_dir=DATA_DIR,
|
||||
config_dir=CONFIG_DIR,
|
||||
service_composer=service_composer,
|
||||
)
|
||||
|
||||
from egress_manager import EgressManager
|
||||
egress_manager = EgressManager(
|
||||
config_manager=config_manager,
|
||||
service_store_manager=service_store_manager,
|
||||
data_dir=DATA_DIR,
|
||||
config_dir=CONFIG_DIR,
|
||||
)
|
||||
service_store_manager.egress_manager = egress_manager
|
||||
|
||||
setup_manager = SetupManager(config_manager=config_manager, auth_manager=auth_manager)
|
||||
|
||||
# Service logger configuration
|
||||
_service_log_configs = {
|
||||
'network': {'level': 'INFO', 'formatter': 'json', 'console': False},
|
||||
@@ -90,9 +122,9 @@ _service_log_configs = {
|
||||
for _svc, _cfg in _service_log_configs.items():
|
||||
log_manager.add_service_logger(_svc, _cfg)
|
||||
|
||||
# Apply any persisted log level overrides
|
||||
# Apply any persisted log level overrides (stored in the mounted config volume)
|
||||
import json as _json
|
||||
_levels_file = os.path.join(os.path.dirname(__file__), 'config', 'log_levels.json')
|
||||
_levels_file = os.path.join(CONFIG_DIR, 'log_levels.json')
|
||||
if os.path.exists(_levels_file):
|
||||
try:
|
||||
with open(_levels_file) as _lf:
|
||||
@@ -110,6 +142,8 @@ __all__ = [
|
||||
'routing_manager', 'vault_manager', 'container_manager',
|
||||
'cell_link_manager', 'auth_manager', 'setup_manager', 'caddy_manager',
|
||||
'ddns_manager', 'service_store_manager', 'connectivity_manager',
|
||||
'service_registry', 'service_composer', 'account_manager',
|
||||
'egress_manager',
|
||||
'firewall_manager', 'EventType',
|
||||
'DATA_DIR', 'CONFIG_DIR',
|
||||
]
|
||||
|
||||
@@ -0,0 +1,352 @@
|
||||
"""
|
||||
manifest_validator — single chokepoint for all manifest and compose YAML security checks.
|
||||
|
||||
Both ServiceComposer and ServiceStoreManager import from here so validation logic
|
||||
lives in exactly one place and cannot be bypassed by taking either code path.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import re
|
||||
import yaml
|
||||
|
||||
logger = logging.getLogger('picell')
|
||||
|
||||
_SUBDOMAIN_RE = re.compile(r'^[a-z][a-z0-9-]{0,30}$')
|
||||
_BACKEND_RE = re.compile(r'^[A-Za-z0-9._-]+:\d{1,5}$')
|
||||
_CAP_ALLOWLIST = frozenset({
|
||||
'NET_ADMIN', 'NET_RAW', 'NET_BIND_SERVICE', 'CHOWN', 'DAC_OVERRIDE',
|
||||
'SETUID', 'SETGID', 'KILL', 'SYS_NICE',
|
||||
})
|
||||
_CAP_DENYLIST = frozenset({
|
||||
'ALL', 'SYS_ADMIN', 'SYS_MODULE', 'SYS_PTRACE', 'SYS_RAWIO',
|
||||
'SYS_BOOT', 'MAC_ADMIN', 'MAC_OVERRIDE', 'SYS_TIME', 'SYS_TTY_CONFIG',
|
||||
})
|
||||
_RESERVED_SUBDOMAINS = frozenset({
|
||||
# Core PIC infrastructure — never allow store services to hijack these
|
||||
'api', 'webui', 'admin', 'www', 'ns1', 'ns2', 'git', 'registry', 'install',
|
||||
# 'mail', 'calendar', 'files', 'webdav', 'webmail' are intentionally absent:
|
||||
# they belong to official PIC store services and must be claimable by them.
|
||||
})
|
||||
_BACKEND_DENYLIST = frozenset({
|
||||
'cell-api', 'cell-caddy', 'cell-coredns', 'cell-dnsmasq',
|
||||
'cell-wireguard', 'cell-vault', 'localhost', '127.0.0.1',
|
||||
'0.0.0.0', 'host.docker.internal',
|
||||
})
|
||||
_RESERVED_CONTAINER_NAMES = frozenset({
|
||||
'cell-api', 'cell-caddy', 'cell-webui', 'cell-coredns',
|
||||
'cell-dnsmasq', 'cell-wireguard', 'cell-chrony',
|
||||
})
|
||||
_CONTAINER_NAME_RE = re.compile(r'^cell-[a-z0-9][a-z0-9-]{0,30}$')
|
||||
_ENV_VALUE_RE = re.compile(r'^[A-Za-z0-9._@:/+\-]{0,256}$')
|
||||
_HOOK_BINARY_RE = re.compile(r'^[a-z][a-z0-9_-]{0,31}$')
|
||||
_CAP_NAME_RE = re.compile(r'^[A-Z_]+$')
|
||||
_ID_RE = re.compile(r'^[a-z][a-z0-9_-]{0,30}$')
|
||||
_IMAGE_DIGEST_RE = re.compile(
|
||||
r'^git\.pic\.ngo/roof/[a-zA-Z0-9._/-]+@sha256:[0-9a-f]{64}$'
|
||||
)
|
||||
|
||||
|
||||
def validate_manifest(manifest: dict) -> tuple:
|
||||
"""
|
||||
Validate security-relevant fields of a store manifest.
|
||||
|
||||
Returns (True, []) when all checks pass; (False, [error_strings]) otherwise.
|
||||
Does not replace the existing _validate_manifest in ServiceStoreManager —
|
||||
it supplements it as a second layer focused on security-critical fields.
|
||||
"""
|
||||
errors = []
|
||||
|
||||
# schema_version must be 3
|
||||
schema_version = manifest.get('schema_version')
|
||||
if schema_version is not None and schema_version != 3:
|
||||
errors.append(
|
||||
f'schema_version must be 3, got: {schema_version!r}'
|
||||
)
|
||||
|
||||
# kind must be "store" if present — reject builtins coming in over the wire
|
||||
kind = manifest.get('kind')
|
||||
if kind is not None and kind != 'store':
|
||||
errors.append(f'manifest kind must be "store", got: {kind!r}')
|
||||
|
||||
# id format check
|
||||
manifest_id = manifest.get('id')
|
||||
if manifest_id is not None:
|
||||
if not isinstance(manifest_id, str) or not _ID_RE.match(manifest_id):
|
||||
errors.append(
|
||||
f'id must match ^[a-z][a-z0-9_-]{{0,30}}$, got: {manifest_id!r}'
|
||||
)
|
||||
|
||||
# image must come from git.pic.ngo/roof/*; if a digest IS provided it must be
|
||||
# valid; first-party images without a digest pin are allowed with a warning.
|
||||
image = manifest.get('image')
|
||||
if image is not None:
|
||||
if not isinstance(image, str):
|
||||
errors.append(f'image must be a string, got: {image!r}')
|
||||
elif not image.startswith('git.pic.ngo/roof/'):
|
||||
errors.append(
|
||||
f'image must be from git.pic.ngo/roof/*, got: {image!r}'
|
||||
)
|
||||
elif '@sha256:' in image:
|
||||
if not _IMAGE_DIGEST_RE.match(image):
|
||||
errors.append(
|
||||
f'image digest must match @sha256:<64-hex>, got: {image!r}'
|
||||
)
|
||||
else:
|
||||
logger.warning('manifest image %s has no digest pin', image)
|
||||
|
||||
# container_name structural check
|
||||
cname = manifest.get('container_name')
|
||||
if cname is not None:
|
||||
if not _CONTAINER_NAME_RE.match(cname):
|
||||
errors.append(
|
||||
f'container_name must match ^cell-[a-z0-9][a-z0-9-]{{0,30}}$, got: {cname!r}'
|
||||
)
|
||||
elif cname in _RESERVED_CONTAINER_NAMES:
|
||||
errors.append(f'container_name is reserved: {cname!r}')
|
||||
|
||||
# subdomain
|
||||
subdomain = manifest.get('subdomain')
|
||||
if subdomain is not None:
|
||||
_check_subdomain(subdomain, 'subdomain', errors)
|
||||
|
||||
# extra_subdomains
|
||||
for sub in manifest.get('extra_subdomains') or []:
|
||||
_check_subdomain(sub, 'extra_subdomains entry', errors)
|
||||
|
||||
# backend
|
||||
backend = manifest.get('backend')
|
||||
if backend is not None:
|
||||
_check_backend(backend, 'backend', errors)
|
||||
|
||||
# extra_backends
|
||||
for sub_key, bknd_val in (manifest.get('extra_backends') or {}).items():
|
||||
_check_backend(bknd_val, f'extra_backends[{sub_key!r}]', errors)
|
||||
|
||||
# cap_add
|
||||
cap_add = manifest.get('cap_add')
|
||||
if cap_add is not None:
|
||||
if not isinstance(cap_add, list):
|
||||
errors.append('cap_add must be a list')
|
||||
else:
|
||||
for cap in cap_add:
|
||||
if not isinstance(cap, str):
|
||||
errors.append(f'cap_add entry must be a string, got: {cap!r}')
|
||||
continue
|
||||
if not _CAP_NAME_RE.match(cap):
|
||||
errors.append(f'cap_add entry must match ^[A-Z_]+$, got: {cap!r}')
|
||||
continue
|
||||
if cap in _CAP_DENYLIST:
|
||||
errors.append(f'cap_add entry is explicitly denied: {cap}')
|
||||
elif cap not in _CAP_ALLOWLIST:
|
||||
errors.append(f'cap_add entry not in allowlist: {cap}')
|
||||
|
||||
# env values
|
||||
for env_entry in manifest.get('env') or []:
|
||||
val = str(env_entry.get('value', ''))
|
||||
if not _ENV_VALUE_RE.match(val):
|
||||
errors.append(
|
||||
f'env[].value contains disallowed characters: {val!r}'
|
||||
)
|
||||
|
||||
# provision_hook
|
||||
hook = (manifest.get('accounts') or {}).get('provision_hook')
|
||||
if hook is not None:
|
||||
ok, msg = validate_provision_hook(hook)
|
||||
if not ok:
|
||||
errors.append(msg)
|
||||
|
||||
return (len(errors) == 0, errors)
|
||||
|
||||
|
||||
def validate_rendered_compose(yaml_text: str, allowed_data_dir: str = None,
|
||||
allow_host_network: bool = False) -> tuple:
|
||||
"""
|
||||
Parse and security-validate a rendered docker-compose YAML string.
|
||||
|
||||
Returns (True, []) when safe; (False, [error_strings]) otherwise.
|
||||
Rejects constructs that would give a store service elevated access to the host.
|
||||
|
||||
allowed_data_dir: when set, absolute bind mounts under this prefix are
|
||||
permitted — they come from ${PIC_DATA_DIR} substitution and land in the
|
||||
designated service data directory.
|
||||
|
||||
allow_host_network: when True, the compose file is permitted to use
|
||||
network_mode: host and devices: — required for connectivity services
|
||||
(wireguard-ext, openvpn-client, tor) that must share the host network
|
||||
namespace to create tun/wg interfaces. The external-network requirement
|
||||
is also waived since host-network containers reach the cell network directly.
|
||||
"""
|
||||
errors = []
|
||||
|
||||
try:
|
||||
doc = yaml.safe_load(yaml_text)
|
||||
except yaml.YAMLError as exc:
|
||||
return (False, [f'YAML parse error: {exc}'])
|
||||
|
||||
if not isinstance(doc, dict):
|
||||
return (False, ['compose file must be a YAML mapping'])
|
||||
|
||||
# Regular (bridged) services must join the cell-network so Caddy and CoreDNS
|
||||
# can reach them. Host-network services share the host namespace directly,
|
||||
# so the external network declaration would be wrong and is omitted.
|
||||
if not allow_host_network:
|
||||
networks = doc.get('networks') or {}
|
||||
has_external = any(
|
||||
isinstance(v, dict) and v.get('external')
|
||||
for v in networks.values()
|
||||
)
|
||||
if not has_external:
|
||||
errors.append(
|
||||
'compose file must declare at least one network with external: true'
|
||||
)
|
||||
|
||||
for svc_name, svc in (doc.get('services') or {}).items():
|
||||
if not isinstance(svc, dict):
|
||||
continue
|
||||
prefix = f'service {svc_name!r}'
|
||||
|
||||
cname = svc.get('container_name')
|
||||
if cname is not None and cname in _RESERVED_CONTAINER_NAMES:
|
||||
errors.append(f'{prefix}: container_name {cname!r} is reserved')
|
||||
|
||||
if svc.get('privileged') is True:
|
||||
errors.append(f'{prefix}: privileged: true is not allowed')
|
||||
|
||||
net_mode = svc.get('network_mode')
|
||||
if allow_host_network:
|
||||
if net_mode is not None and net_mode not in ('host',):
|
||||
errors.append(
|
||||
f'{prefix}: network_mode {net_mode!r} is not allowed '
|
||||
'(connectivity services must use host)'
|
||||
)
|
||||
else:
|
||||
if net_mode is not None and net_mode not in (None, 'bridge'):
|
||||
errors.append(
|
||||
f'{prefix}: network_mode {net_mode!r} is not allowed (only bridge)'
|
||||
)
|
||||
|
||||
if svc.get('pid') == 'host':
|
||||
errors.append(f'{prefix}: pid: host is not allowed')
|
||||
|
||||
if svc.get('ipc') == 'host':
|
||||
errors.append(f'{prefix}: ipc: host is not allowed')
|
||||
|
||||
if svc.get('userns_mode') == 'host':
|
||||
errors.append(f'{prefix}: userns_mode: host is not allowed')
|
||||
|
||||
# cap_add
|
||||
for cap in svc.get('cap_add') or []:
|
||||
cap_str = str(cap)
|
||||
if cap_str in _CAP_DENYLIST:
|
||||
errors.append(f'{prefix}: cap_add {cap_str!r} is explicitly denied')
|
||||
elif cap_str not in _CAP_ALLOWLIST:
|
||||
errors.append(f'{prefix}: cap_add {cap_str!r} not in allowlist')
|
||||
|
||||
# volumes — reject absolute host-side bind mounts unless they're under
|
||||
# the sanctioned data directory (injected by ServiceComposer via PIC_DATA_DIR)
|
||||
for vol in svc.get('volumes') or []:
|
||||
vol_str = str(vol)
|
||||
src = vol_str.split(':')[0] if ':' in vol_str else vol_str
|
||||
if src.startswith('/'):
|
||||
if allowed_data_dir and src.startswith(allowed_data_dir):
|
||||
continue
|
||||
errors.append(
|
||||
f'{prefix}: absolute host bind mount not allowed: {vol_str!r}'
|
||||
)
|
||||
|
||||
if 'devices' in svc and not allow_host_network:
|
||||
errors.append(f'{prefix}: devices key is not allowed')
|
||||
|
||||
for opt in svc.get('security_opt') or []:
|
||||
opt_str = str(opt)
|
||||
if opt_str in ('apparmor=unconfined', 'seccomp=unconfined'):
|
||||
errors.append(
|
||||
f'{prefix}: security_opt {opt_str!r} is not allowed'
|
||||
)
|
||||
|
||||
# command must be a list — string form passes through the shell
|
||||
cmd = svc.get('command')
|
||||
if cmd is not None and isinstance(cmd, str):
|
||||
errors.append(
|
||||
f'{prefix}: command must be a list, not a shell string'
|
||||
)
|
||||
|
||||
# entrypoint must also be a list for the same reason
|
||||
ep = svc.get('entrypoint')
|
||||
if ep is not None and isinstance(ep, str):
|
||||
errors.append(
|
||||
f'{prefix}: entrypoint must be a list, not a shell string'
|
||||
)
|
||||
|
||||
return (len(errors) == 0, errors)
|
||||
|
||||
|
||||
def validate_provision_hook(hook) -> tuple:
|
||||
"""
|
||||
Validate a provision_hook value from accounts.provision_hook.
|
||||
|
||||
Acceptable: None/absent, or a dict {"argv": ["binary", "arg1", ...]}.
|
||||
Rejected: any plain string (shell injection risk), empty argv, uppercase binary,
|
||||
NUL bytes in any element.
|
||||
|
||||
Returns (True, "") on success; (False, error_string) on failure.
|
||||
"""
|
||||
if hook is None:
|
||||
return (True, '')
|
||||
|
||||
if isinstance(hook, str):
|
||||
return (
|
||||
False,
|
||||
'provision_hook must be an argv list dict {"argv": [...]}, not a shell string',
|
||||
)
|
||||
|
||||
if not isinstance(hook, dict):
|
||||
return (False, 'provision_hook must be a dict with argv list')
|
||||
|
||||
argv = hook.get('argv')
|
||||
if not isinstance(argv, list) or len(argv) == 0:
|
||||
return (False, 'provision_hook.argv must be a non-empty list')
|
||||
|
||||
# NUL-byte check must precede regex check so the error message is unambiguous.
|
||||
for elem in argv:
|
||||
if isinstance(elem, str) and '\x00' in elem:
|
||||
return (False, 'provision_hook.argv element contains NUL byte')
|
||||
|
||||
binary = argv[0]
|
||||
if not isinstance(binary, str) or not _HOOK_BINARY_RE.match(binary):
|
||||
return (
|
||||
False,
|
||||
f'provision_hook.argv[0] must match ^[a-z][a-z0-9_-]{{0,31}}$, got: {binary!r}',
|
||||
)
|
||||
|
||||
return (True, '')
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Internal helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _check_subdomain(value, field_name: str, errors: list) -> None:
|
||||
if not isinstance(value, str):
|
||||
errors.append(f'{field_name} must be a string')
|
||||
return
|
||||
if value in _RESERVED_SUBDOMAINS:
|
||||
errors.append(f'{field_name} is reserved: {value!r}')
|
||||
elif not _SUBDOMAIN_RE.match(value):
|
||||
errors.append(
|
||||
f'{field_name} must match ^[a-z][a-z0-9-]{{0,30}}$, got: {value!r}'
|
||||
)
|
||||
|
||||
|
||||
def _check_backend(value, field_name: str, errors: list) -> None:
|
||||
if not isinstance(value, str):
|
||||
errors.append(f'{field_name} must be a string')
|
||||
return
|
||||
if not _BACKEND_RE.match(value):
|
||||
errors.append(
|
||||
f'{field_name} must be host:port (e.g. cell-foo:8080), got: {value!r}'
|
||||
)
|
||||
return
|
||||
host = value.split(':')[0]
|
||||
if host in _BACKEND_DENYLIST:
|
||||
errors.append(f'{field_name} host {host!r} is in the backend denylist')
|
||||
+137
-45
@@ -18,10 +18,12 @@ logger = logging.getLogger(__name__)
|
||||
class NetworkManager(BaseServiceManager):
|
||||
"""Manages network services (DNS, DHCP, NTP)"""
|
||||
|
||||
def __init__(self, data_dir: str = '/app/data', config_dir: str = '/app/config'):
|
||||
def __init__(self, data_dir: str = '/app/data', config_dir: str = '/app/config',
|
||||
service_registry=None):
|
||||
super().__init__('network', data_dir, config_dir)
|
||||
self.dns_zones_dir = os.path.join(data_dir, 'dns')
|
||||
self.dhcp_leases_file = os.path.join(data_dir, 'dhcp', 'leases')
|
||||
self._service_registry = service_registry
|
||||
|
||||
# Ensure directories exist
|
||||
self.safe_makedirs(self.dns_zones_dir)
|
||||
@@ -45,7 +47,7 @@ class NetworkManager(BaseServiceManager):
|
||||
for rec in records:
|
||||
rname = rec.get('name', '')
|
||||
rvalue = rec.get('value', '')
|
||||
if rname and not re.match(r'^[a-zA-Z0-9_.*-]{1,253}$', str(rname)):
|
||||
if rname and not re.match(r'^[a-zA-Z0-9_@.*-]{1,253}$', str(rname)):
|
||||
logger.error(f"update_dns_zone: invalid record name {rname!r}")
|
||||
return False
|
||||
if rvalue and not re.match(r'^[a-zA-Z0-9._: -]{1,512}$', str(rvalue)):
|
||||
@@ -165,6 +167,61 @@ class NetworkManager(BaseServiceManager):
|
||||
self.update_dns_zone(domain, records)
|
||||
logger.info(f"Created {len(records)} default DNS records for zone '{domain}'")
|
||||
|
||||
def update_split_horizon_zone(self, effective_domain: str, caddy_ip: str,
|
||||
primary_domain: str = 'cell',
|
||||
peers: Optional[List[Dict]] = None,
|
||||
cell_links: Optional[List[Dict]] = None) -> bool:
|
||||
"""Write a local authoritative zone for effective_domain pointing all
|
||||
hosts (wildcard) to caddy_ip so LAN clients resolve service subdomains
|
||||
without hairpin NAT. Regenerates the Corefile and reloads CoreDNS."""
|
||||
import firewall_manager as _fm
|
||||
# SOA/NS are generated by _generate_zone_content; just pass the A records.
|
||||
records = [
|
||||
{'name': '@', 'type': 'A', 'value': caddy_ip},
|
||||
{'name': '*', 'type': 'A', 'value': caddy_ip},
|
||||
]
|
||||
ok = self.update_dns_zone(effective_domain, records)
|
||||
if not ok:
|
||||
logger.warning('update_split_horizon_zone: zone file write failed for %s', effective_domain)
|
||||
|
||||
# Delete split-horizon zone files for prior cell names sharing the same TLD.
|
||||
# E.g. when renaming from pic3.pic.ngo → pic2.pic.ngo, remove pic3.pic.ngo.zone.
|
||||
eff_parts = effective_domain.split('.')
|
||||
if len(eff_parts) >= 2:
|
||||
tld_suffix = '.' + '.'.join(eff_parts[1:])
|
||||
for fname in os.listdir(self.dns_zones_dir):
|
||||
if fname.endswith('.zone'):
|
||||
z = fname[:-5]
|
||||
if z.endswith(tld_suffix) and z != effective_domain:
|
||||
try:
|
||||
os.remove(os.path.join(self.dns_zones_dir, fname))
|
||||
logger.info('Deleted stale split-horizon zone: %s', fname)
|
||||
except OSError as _e:
|
||||
logger.warning('Failed to delete stale zone %s: %s', fname, _e)
|
||||
|
||||
# If the internal zone name happens to be a parent of the effective DDNS
|
||||
# domain (e.g. primary_domain='pic.ngo', effective_domain='pic2.pic.ngo'),
|
||||
# bootstrap service records like 'api', 'calendar' etc. would pollute the
|
||||
# zone display and shadow the public domain. Remove them.
|
||||
_stale = {'api', 'webui'} | set(self._BUILTIN_SERVICE_SUBDOMAINS) | set(self._get_service_subdomains())
|
||||
if effective_domain.endswith('.' + primary_domain):
|
||||
existing = self._load_dns_records(primary_domain)
|
||||
cleaned = [r for r in existing if r.get('name', '') not in _stale]
|
||||
if len(cleaned) < len(existing):
|
||||
self.update_dns_zone(primary_domain, cleaned)
|
||||
logger.info('Removed stale service records from zone %s', primary_domain)
|
||||
|
||||
corefile = os.path.join(self.config_dir, 'dns', 'Corefile')
|
||||
peers_data = peers or []
|
||||
ok_cf = _fm.generate_corefile(
|
||||
peers_data, corefile, primary_domain,
|
||||
cell_links=cell_links,
|
||||
split_horizon_zones=[effective_domain],
|
||||
)
|
||||
if ok_cf:
|
||||
_fm.reload_coredns()
|
||||
return ok and ok_cf
|
||||
|
||||
def apply_ip_range(self, ip_range: str, cell_name: str, domain: str) -> Dict[str, Any]:
|
||||
"""Rewrite the primary DNS zone file with IPs derived from the new subnet."""
|
||||
restarted: List[str] = []
|
||||
@@ -194,6 +251,30 @@ class NetworkManager(BaseServiceManager):
|
||||
pass
|
||||
return '10.0.0.1'
|
||||
|
||||
_SUBDOMAIN_RE = re.compile(r'^[a-z][a-z0-9-]{0,30}$')
|
||||
|
||||
def _get_service_subdomains(self) -> List[str]:
|
||||
"""Return all service subdomains from the registry, or a hardcoded fallback."""
|
||||
registry = getattr(self, "_service_registry", None)
|
||||
if registry is not None:
|
||||
try:
|
||||
subs: List[str] = []
|
||||
for route in registry.get_caddy_routes():
|
||||
for sub in [route['subdomain']] + list(route.get('extra_subdomains') or []):
|
||||
if self._SUBDOMAIN_RE.match(sub):
|
||||
subs.append(sub)
|
||||
else:
|
||||
logger.warning('_get_service_subdomains: skipping invalid subdomain %r', sub)
|
||||
return subs
|
||||
except Exception as exc:
|
||||
logger.warning('_get_service_subdomains: registry error: %s', exc)
|
||||
return []
|
||||
|
||||
# Built-in service subdomains that are always present on a PIC instance.
|
||||
# These must stay in sync with firewall_manager.SERVICE_IPS keys and the
|
||||
# Caddy routes for each built-in service.
|
||||
_BUILTIN_SERVICE_SUBDOMAINS = ('calendar', 'files', 'mail', 'webdav')
|
||||
|
||||
def _build_dns_records(self, cell_name: str, ip_range: str) -> List[Dict]:
|
||||
"""Build the standard set of DNS A records.
|
||||
|
||||
@@ -203,16 +284,16 @@ class NetworkManager(BaseServiceManager):
|
||||
routes requests to the correct backend by Host header.
|
||||
"""
|
||||
wg_ip = self._get_wg_server_ip()
|
||||
return [
|
||||
{'name': cell_name, 'type': 'A', 'value': wg_ip},
|
||||
{'name': 'api', 'type': 'A', 'value': wg_ip},
|
||||
{'name': 'webui', 'type': 'A', 'value': wg_ip},
|
||||
{'name': 'calendar', 'type': 'A', 'value': wg_ip},
|
||||
{'name': 'files', 'type': 'A', 'value': wg_ip},
|
||||
{'name': 'mail', 'type': 'A', 'value': wg_ip},
|
||||
{'name': 'webmail', 'type': 'A', 'value': wg_ip},
|
||||
{'name': 'webdav', 'type': 'A', 'value': wg_ip},
|
||||
records = [
|
||||
{'name': cell_name, 'type': 'A', 'value': wg_ip},
|
||||
{'name': 'api', 'type': 'A', 'value': wg_ip},
|
||||
{'name': 'webui', 'type': 'A', 'value': wg_ip},
|
||||
]
|
||||
for sub in self._BUILTIN_SERVICE_SUBDOMAINS:
|
||||
records.append({'name': sub, 'type': 'A', 'value': wg_ip})
|
||||
for sub in self._get_service_subdomains():
|
||||
records.append({'name': sub, 'type': 'A', 'value': wg_ip})
|
||||
return records
|
||||
|
||||
def get_dns_records(self, zone: str = 'cell') -> List[Dict]:
|
||||
"""Get all DNS records across all zones"""
|
||||
@@ -372,10 +453,10 @@ class NetworkManager(BaseServiceManager):
|
||||
return {'running': False, 'stats': {}}
|
||||
|
||||
def _reload_dns_service(self):
|
||||
"""Reload DNS service"""
|
||||
"""Send SIGUSR1 to CoreDNS so the reload plugin picks up zone file changes."""
|
||||
try:
|
||||
subprocess.run(['docker', 'exec', 'cell-dns', 'kill', '-HUP', '1'],
|
||||
capture_output=True, timeout=10)
|
||||
subprocess.run(['docker', 'kill', '--signal=SIGUSR1', 'cell-dns'],
|
||||
capture_output=True, timeout=10)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to reload DNS service: {e}")
|
||||
|
||||
@@ -539,42 +620,53 @@ class NetworkManager(BaseServiceManager):
|
||||
warnings = []
|
||||
if not new_name:
|
||||
return {'restarted': restarted, 'warnings': warnings}
|
||||
_service_names = {'api', 'webui', 'calendar', 'files', 'mail', 'webmail', 'webdav'}
|
||||
# Exclude service names, wildcard, and apex from cell-hostname detection.
|
||||
_service_names = {'api', 'webui'} | set(self._BUILTIN_SERVICE_SUBDOMAINS) | set(self._get_service_subdomains())
|
||||
_reserved = _service_names | {'@', '*'}
|
||||
changed = False
|
||||
try:
|
||||
dns_data = os.path.join(self.data_dir, 'dns')
|
||||
if os.path.isdir(dns_data):
|
||||
for fname in os.listdir(dns_data):
|
||||
if fname.endswith('.zone') and 'local' not in fname:
|
||||
zone_file = os.path.join(dns_data, fname)
|
||||
with open(zone_file) as f:
|
||||
content = f.read()
|
||||
# Determine which name to replace: prefer old_name if present,
|
||||
# otherwise detect from zone (non-service A record not in _service_names)
|
||||
actual_old = old_name if (
|
||||
old_name and re.search(
|
||||
rf'^{re.escape(old_name)}\s', content, re.MULTILINE)
|
||||
) else None
|
||||
if actual_old is None:
|
||||
for m in re.finditer(
|
||||
r'^(\S+)\s+(?:\d+\s+)?IN\s+A\s+\S+', content, re.MULTILINE
|
||||
):
|
||||
candidate = m.group(1)
|
||||
if candidate not in _service_names and candidate != '@':
|
||||
actual_old = candidate
|
||||
break
|
||||
if actual_old is None or actual_old == new_name:
|
||||
break
|
||||
new_content = re.sub(
|
||||
rf'^{re.escape(actual_old)}(\s+(?:\d+\s+)?IN\s+A\s+)',
|
||||
f'{new_name}\\1',
|
||||
content, flags=re.MULTILINE
|
||||
)
|
||||
if new_content != content:
|
||||
with open(zone_file, 'w') as f:
|
||||
f.write(new_content)
|
||||
changed = True
|
||||
break
|
||||
if not fname.endswith('.zone'):
|
||||
continue
|
||||
zone_name = fname[:-5]
|
||||
# Skip split-horizon DDNS zones (multi-label, e.g. 'pic2.pic.ngo.zone')
|
||||
# and any zone with 'local' in its name. The cell hostname only lives
|
||||
# in the primary single-label zone (e.g. 'cell.zone').
|
||||
if 'local' in zone_name or '.' in zone_name:
|
||||
continue
|
||||
zone_file = os.path.join(dns_data, fname)
|
||||
with open(zone_file) as f:
|
||||
content = f.read()
|
||||
# Determine which name to replace: prefer old_name if present,
|
||||
# otherwise detect from zone (non-service A record not in _reserved)
|
||||
actual_old = old_name if (
|
||||
old_name and re.search(
|
||||
rf'^{re.escape(old_name)}\s', content, re.MULTILINE)
|
||||
) else None
|
||||
if actual_old is None:
|
||||
for m in re.finditer(
|
||||
r'^(\S+)\s+(?:\d+\s+)?IN\s+A\s+\S+', content, re.MULTILINE
|
||||
):
|
||||
candidate = m.group(1)
|
||||
if candidate not in _reserved:
|
||||
actual_old = candidate
|
||||
break
|
||||
if actual_old is None:
|
||||
continue # no hostname in this zone; try next
|
||||
if actual_old == new_name:
|
||||
break # already correct
|
||||
new_content = re.sub(
|
||||
rf'^{re.escape(actual_old)}(\s+(?:\d+\s+)?IN\s+A\s+)',
|
||||
f'{new_name}\\1',
|
||||
content, flags=re.MULTILINE
|
||||
)
|
||||
if new_content != content:
|
||||
with open(zone_file, 'w') as f:
|
||||
f.write(new_content)
|
||||
changed = True
|
||||
break
|
||||
if changed and reload:
|
||||
self._reload_dns_service()
|
||||
restarted.append('cell-dns (reloaded)')
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
flask>=3.0.3
|
||||
flask-cors>=4.0.1
|
||||
requests>=2.32.3
|
||||
pyotp>=2.9.0
|
||||
cryptography>=42.0.5
|
||||
pyyaml==6.0.1
|
||||
icalendar==5.0.7
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
from functools import wraps
|
||||
from flask import jsonify
|
||||
|
||||
|
||||
def require_active_service(service_id: str):
|
||||
"""Decorator: return 404 if the named service is not installed.
|
||||
|
||||
Apply to all email/calendar/files routes except /status endpoints,
|
||||
so the UI can always check installation state without being blocked.
|
||||
"""
|
||||
def decorator(fn):
|
||||
@wraps(fn)
|
||||
def wrapper(*args, **kwargs):
|
||||
from app import service_registry
|
||||
if service_registry.get(service_id) is None:
|
||||
return jsonify({'error': f'Service {service_id!r} is not installed'}), 404
|
||||
return fn(*args, **kwargs)
|
||||
return wrapper
|
||||
return decorator
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
import logging
|
||||
from flask import Blueprint, request, jsonify
|
||||
from routes import require_active_service
|
||||
|
||||
logger = logging.getLogger('picell')
|
||||
bp = Blueprint('calendar', __name__)
|
||||
|
||||
@bp.route('/api/calendar/users', methods=['GET'])
|
||||
@require_active_service('calendar')
|
||||
def get_calendar_users():
|
||||
"""Get calendar users."""
|
||||
try:
|
||||
@@ -15,6 +18,7 @@ def get_calendar_users():
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
@bp.route('/api/calendar/users', methods=['POST'])
|
||||
@require_active_service('calendar')
|
||||
def create_calendar_user():
|
||||
"""Create calendar user."""
|
||||
try:
|
||||
@@ -33,6 +37,7 @@ def create_calendar_user():
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
@bp.route('/api/calendar/users/<username>', methods=['DELETE'])
|
||||
@require_active_service('calendar')
|
||||
def delete_calendar_user(username):
|
||||
"""Delete calendar user."""
|
||||
try:
|
||||
@@ -44,6 +49,7 @@ def delete_calendar_user(username):
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
@bp.route('/api/calendar/calendars', methods=['POST'])
|
||||
@require_active_service('calendar')
|
||||
def create_calendar():
|
||||
"""Create calendar."""
|
||||
try:
|
||||
@@ -67,6 +73,7 @@ def create_calendar():
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
@bp.route('/api/calendar/events', methods=['POST'])
|
||||
@require_active_service('calendar')
|
||||
def add_calendar_event():
|
||||
try:
|
||||
from app import calendar_manager
|
||||
@@ -85,6 +92,7 @@ def add_calendar_event():
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
@bp.route('/api/calendar/events/<username>/<calendar_name>', methods=['GET'])
|
||||
@require_active_service('calendar')
|
||||
def get_calendar_events(username, calendar_name):
|
||||
"""Get calendar events."""
|
||||
try:
|
||||
@@ -108,6 +116,7 @@ def get_calendar_status():
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
@bp.route('/api/calendar/connectivity', methods=['GET'])
|
||||
@require_active_service('calendar')
|
||||
def test_calendar_connectivity():
|
||||
"""Test calendar connectivity."""
|
||||
try:
|
||||
|
||||
+6
-5
@@ -47,7 +47,7 @@ def get_cell_invite():
|
||||
from app import cell_link_manager, config_manager
|
||||
identity = config_manager.configs.get('_identity', {})
|
||||
cell_name = identity.get('cell_name', os.environ.get('CELL_NAME', 'mycell'))
|
||||
domain = identity.get('domain', os.environ.get('CELL_DOMAIN', 'cell'))
|
||||
domain = identity.get('domain_name') or identity.get('domain', os.environ.get('CELL_DOMAIN', 'cell'))
|
||||
return jsonify(cell_link_manager.generate_invite(cell_name, domain))
|
||||
except Exception as e:
|
||||
logger.error(f"Error generating cell invite: {e}")
|
||||
@@ -145,12 +145,13 @@ def update_cell_permissions(cell_name):
|
||||
|
||||
# Regenerate Corefile so outbound DNS changes take effect
|
||||
try:
|
||||
from app import config_manager
|
||||
domain = config_manager.configs.get('_identity', {}).get('domain', 'cell')
|
||||
from app import _configured_dns_params
|
||||
peers = peer_registry.list_peers()
|
||||
cell_links = cell_link_manager.list_connections()
|
||||
firewall_manager.apply_all_dns_rules(peers, COREFILE_PATH, domain,
|
||||
cell_links=cell_links)
|
||||
_dns_primary, _dns_szones = _configured_dns_params()
|
||||
firewall_manager.apply_all_dns_rules(peers, COREFILE_PATH, _dns_primary,
|
||||
cell_links=cell_links,
|
||||
split_horizon_zones=_dns_szones)
|
||||
except Exception as e:
|
||||
logger.warning(f"DNS regen after permission update failed (non-fatal): {e}")
|
||||
|
||||
|
||||
+247
-23
@@ -118,6 +118,21 @@ def get_config():
|
||||
'vip_webdav': _ips['vip_webdav'],
|
||||
}
|
||||
config['service_configs'] = service_configs
|
||||
config['installed_services'] = config_manager.get_installed_services()
|
||||
config['domain_mode'] = identity.get('domain_mode', 'lan')
|
||||
config['domain_name'] = identity.get('domain_name', '')
|
||||
config['effective_domain'] = config_manager.get_effective_domain()
|
||||
ddns_section = config_manager.configs.get('ddns', {})
|
||||
_provider = ddns_section.get('provider', '')
|
||||
_has_token = bool(
|
||||
(config_manager.get_ddns_token() if _provider == 'pic_ngo' else '') or
|
||||
ddns_section.get('api_token') or ddns_section.get('token')
|
||||
)
|
||||
config['ddns'] = {
|
||||
'provider': _provider,
|
||||
'subdomain': ddns_section.get('subdomain', ''),
|
||||
'has_token': _has_token,
|
||||
}
|
||||
return jsonify(config)
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting config: {e}")
|
||||
@@ -306,12 +321,6 @@ def update_config():
|
||||
domain = identity_updates['domain']
|
||||
net_result = network_manager.apply_domain(domain, reload=False)
|
||||
all_warnings.extend(net_result.get('warnings', []))
|
||||
_cur_id = config_manager.configs.get('_identity', {})
|
||||
ip_utils.write_caddyfile(
|
||||
_cur_id.get('ip_range', os.environ.get('CELL_IP_RANGE', '172.20.0.0/16')),
|
||||
_cur_id.get('cell_name', os.environ.get('CELL_NAME', 'mycell')),
|
||||
domain, '/app/config-caddy/Caddyfile'
|
||||
)
|
||||
_set_pending_restart(
|
||||
[f'domain changed to {domain}'],
|
||||
['dns', 'caddy'],
|
||||
@@ -324,18 +333,23 @@ def update_config():
|
||||
if old_name != new_name:
|
||||
cn_result = network_manager.apply_cell_name(old_name, new_name, reload=False)
|
||||
all_warnings.extend(cn_result.get('warnings', []))
|
||||
_cur_id2 = config_manager.configs.get('_identity', {})
|
||||
ip_utils.write_caddyfile(
|
||||
_cur_id2.get('ip_range', os.environ.get('CELL_IP_RANGE', '172.20.0.0/16')),
|
||||
new_name,
|
||||
identity_updates.get('domain') or _cur_id2.get('domain', os.environ.get('CELL_DOMAIN', 'cell')),
|
||||
'/app/config-caddy/Caddyfile'
|
||||
)
|
||||
_set_pending_restart(
|
||||
[f'cell_name changed to {new_name}'],
|
||||
['dns'],
|
||||
pre_change_snapshot=_pre_change_snapshot,
|
||||
)
|
||||
_ddns_cfg = config_manager.configs.get('ddns', {})
|
||||
if _ddns_cfg.get('provider') == 'pic_ngo':
|
||||
try:
|
||||
from ddns_manager import DDNSManager as _DDNSManager
|
||||
_ddns_mgr = _DDNSManager(config_manager)
|
||||
_result = _ddns_mgr.register(new_name, '')
|
||||
_new_sub = _result.get('subdomain', f'{new_name}.pic.ngo')
|
||||
config_manager.set_identity_field('domain_name', _new_sub)
|
||||
logger.info('DDNS re-registered: cell_name=%r subdomain=%r', new_name, _new_sub)
|
||||
except Exception as _exc:
|
||||
logger.warning('DDNS re-registration failed for %r: %s', new_name, _exc)
|
||||
all_warnings.append(f'DDNS name update failed — {_exc}')
|
||||
|
||||
if identity_updates.get('ip_range') and identity_updates['ip_range'] != old_identity.get('ip_range', ''):
|
||||
new_range = identity_updates['ip_range']
|
||||
@@ -349,13 +363,34 @@ def update_config():
|
||||
firewall_manager.ensure_caddy_virtual_ips()
|
||||
env_file = os.environ.get('COMPOSE_ENV_FILE', '/app/.env.compose')
|
||||
ip_utils.write_env_file(new_range, env_file, _collect_service_ports(config_manager.configs))
|
||||
ip_utils.write_caddyfile(new_range, cur_cell_name, cur_domain, '/app/config-caddy/Caddyfile')
|
||||
_set_pending_restart(
|
||||
[f'ip_range changed to {new_range} — network will be recreated'],
|
||||
['*'], network_recreate=True,
|
||||
pre_change_snapshot=_pre_change_snapshot,
|
||||
)
|
||||
|
||||
if identity_updates:
|
||||
_cur_identity = config_manager.configs.get('_identity', {})
|
||||
_eff_domain = config_manager.get_effective_domain()
|
||||
service_bus.publish_event(EventType.IDENTITY_CHANGED, 'config', {
|
||||
'cell_name': _cur_identity.get('cell_name'),
|
||||
'domain': _cur_identity.get('domain'),
|
||||
'domain_name': _cur_identity.get('domain_name'),
|
||||
'domain_mode': _cur_identity.get('domain_mode'),
|
||||
'effective_domain': _eff_domain,
|
||||
})
|
||||
if _cur_identity.get('domain_mode', 'lan') != 'lan' and _eff_domain:
|
||||
try:
|
||||
import ip_utils as _ip_sh
|
||||
_ip_range = _cur_identity.get('ip_range', os.environ.get('CELL_IP_RANGE', '172.20.0.0/16'))
|
||||
_caddy_ip = _ip_sh.get_service_ips(_ip_range).get('caddy', '172.20.0.2')
|
||||
_primary_domain = _cur_identity.get('domain', os.environ.get('CELL_DOMAIN', 'cell'))
|
||||
network_manager.update_split_horizon_zone(
|
||||
_eff_domain, _caddy_ip, primary_domain=_primary_domain
|
||||
)
|
||||
except Exception as _sh_exc:
|
||||
logger.warning('split-horizon zone update failed: %s', _sh_exc)
|
||||
|
||||
_PORT_CHANGE_MAP = {
|
||||
('network', 'dns_port'): ('dns_port', ['dns']),
|
||||
('wireguard','port'): ('wg_port', ['wireguard']),
|
||||
@@ -442,6 +477,189 @@ def update_config():
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
|
||||
@bp.route('/api/ddns/check/<name>', methods=['GET'])
|
||||
def ddns_check_name(name):
|
||||
import urllib.request as _ureq
|
||||
import urllib.error as _uerr
|
||||
import json as _json_
|
||||
from setup_manager import DDNS_API_BASE as _DDNS_BASE
|
||||
try:
|
||||
url = f'{_DDNS_BASE}/api/v1/check/{name}'
|
||||
with _ureq.urlopen(url, timeout=8) as resp:
|
||||
body = _json_.loads(resp.read())
|
||||
return jsonify({'available': bool(body.get('available'))})
|
||||
except Exception as exc:
|
||||
logger.warning('DDNS check failed for %r: %s', name, exc)
|
||||
return jsonify({'available': None, 'error': 'DDNS service unreachable'}), 503
|
||||
|
||||
|
||||
@bp.route('/api/ddns', methods=['PUT'])
|
||||
def update_ddns_config():
|
||||
import urllib.request as _ureq
|
||||
import urllib.error as _uerr
|
||||
import json as _json_
|
||||
try:
|
||||
from app import config_manager
|
||||
from setup_manager import _build_ddns_config, DDNS_API_BASE as _DDNS_BASE
|
||||
|
||||
data = request.get_json(silent=True) or {}
|
||||
domain_mode = data.get('domain_mode', '').strip()
|
||||
domain_name = data.get('domain_name', '').strip()
|
||||
cf_token = data.get('cloudflare_api_token', '').strip()
|
||||
duck_token = data.get('duckdns_token', '').strip()
|
||||
|
||||
from setup_manager import VALID_DOMAIN_MODES
|
||||
if domain_mode not in VALID_DOMAIN_MODES:
|
||||
return jsonify({'error': f'domain_mode must be one of: {", ".join(sorted(VALID_DOMAIN_MODES))}'}), 400
|
||||
|
||||
if domain_mode == 'cloudflare':
|
||||
if not domain_name:
|
||||
return jsonify({'error': 'domain_name is required for cloudflare'}), 400
|
||||
if not cf_token:
|
||||
existing = config_manager.configs.get('ddns', {}).get('api_token', '')
|
||||
if not existing:
|
||||
return jsonify({'error': 'cloudflare_api_token is required'}), 400
|
||||
cf_token = existing
|
||||
try:
|
||||
req = _ureq.Request(
|
||||
'https://api.cloudflare.com/client/v4/user/tokens/verify',
|
||||
headers={'Authorization': f'Bearer {cf_token}'},
|
||||
)
|
||||
with _ureq.urlopen(req, timeout=8) as resp:
|
||||
body = _json_.loads(resp.read())
|
||||
if not body.get('success'):
|
||||
return jsonify({'error': 'Cloudflare token is invalid'}), 422
|
||||
except _uerr.HTTPError:
|
||||
return jsonify({'error': 'Cloudflare token is invalid'}), 422
|
||||
except Exception as exc:
|
||||
return jsonify({'error': f'Could not reach Cloudflare: {exc}'}), 503
|
||||
|
||||
if domain_mode == 'duckdns':
|
||||
if not domain_name:
|
||||
return jsonify({'error': 'domain_name is required for duckdns'}), 400
|
||||
if not duck_token:
|
||||
existing = config_manager.configs.get('ddns', {}).get('token', '')
|
||||
if not existing:
|
||||
return jsonify({'error': 'duckdns_token is required'}), 400
|
||||
duck_token = existing
|
||||
subdomain = domain_name.replace('.duckdns.org', '')
|
||||
try:
|
||||
url = f'https://www.duckdns.org/update?domains={subdomain}&token={duck_token}&ip='
|
||||
with _ureq.urlopen(url, timeout=8) as resp:
|
||||
if resp.read().strip() != b'OK':
|
||||
return jsonify({'error': 'DuckDNS token or subdomain is invalid'}), 422
|
||||
except Exception as exc:
|
||||
return jsonify({'error': f'Could not reach DuckDNS: {exc}'}), 503
|
||||
|
||||
duck_sub = domain_name.replace('.duckdns.org', '') if domain_mode == 'duckdns' else ''
|
||||
ddns_cfg = _build_ddns_config(
|
||||
domain_mode,
|
||||
cloudflare_api_token=cf_token,
|
||||
duckdns_token=duck_token,
|
||||
duckdns_subdomain=duck_sub,
|
||||
)
|
||||
config_manager.set_ddns_config(ddns_cfg)
|
||||
config_manager.set_identity_field('domain_mode', domain_mode)
|
||||
if domain_name:
|
||||
config_manager.set_identity_field('domain_name', domain_name)
|
||||
if domain_mode == 'cloudflare' and cf_token:
|
||||
config_manager.set_identity_field('cloudflare_api_token', cf_token)
|
||||
if domain_mode == 'duckdns':
|
||||
if duck_token:
|
||||
config_manager.set_identity_field('duckdns_token', duck_token)
|
||||
config_manager.set_identity_field('duckdns_subdomain', duck_sub)
|
||||
|
||||
# Fire IDENTITY_CHANGED so CaddyManager regenerates the Caddyfile
|
||||
# for the new domain mode without requiring a container restart.
|
||||
try:
|
||||
from app import service_bus as _sbus, EventType as _ET
|
||||
_cur = config_manager.configs.get('_identity', {})
|
||||
_sbus.publish_event(_ET.IDENTITY_CHANGED, 'config', {
|
||||
'cell_name': _cur.get('cell_name'),
|
||||
'domain': _cur.get('domain'),
|
||||
'domain_name': _cur.get('domain_name'),
|
||||
'domain_mode': _cur.get('domain_mode'),
|
||||
'effective_domain': config_manager.get_effective_domain(),
|
||||
})
|
||||
except Exception as _ev_err:
|
||||
logger.warning('update_ddns_config: failed to fire IDENTITY_CHANGED: %s', _ev_err)
|
||||
|
||||
logger.info('DDNS config updated: domain_mode=%r domain_name=%r', domain_mode, domain_name)
|
||||
return jsonify({'updated': True})
|
||||
except Exception as e:
|
||||
logger.error(f'Error updating DDNS config: {e}')
|
||||
return jsonify({'error': str(e)}), 500
|
||||
|
||||
|
||||
_ddns_public_ip_cache: dict = {'ip': None, 'at': 0}
|
||||
|
||||
@bp.route('/api/ddns/status', methods=['GET'])
|
||||
def ddns_status():
|
||||
import time as _time
|
||||
from app import config_manager
|
||||
ddns_cfg = config_manager.configs.get('ddns', {})
|
||||
identity = config_manager.configs.get('_identity', {})
|
||||
|
||||
now = _time.time()
|
||||
if now - _ddns_public_ip_cache['at'] > 30 or not _ddns_public_ip_cache['ip']:
|
||||
try:
|
||||
import requests as _req
|
||||
resp = _req.get('https://api.ipify.org', timeout=5)
|
||||
if resp.ok:
|
||||
_ddns_public_ip_cache['ip'] = resp.text.strip()
|
||||
_ddns_public_ip_cache['at'] = now
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
last_ip = None
|
||||
try:
|
||||
from app import ddns_manager as _ddns_mgr_singleton
|
||||
last_ip = _ddns_mgr_singleton._last_ip
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
registered = bool(config_manager.get_ddns_token())
|
||||
return jsonify({
|
||||
'registered': registered,
|
||||
'domain_name': identity.get('domain_name', ''),
|
||||
'public_ip': _ddns_public_ip_cache['ip'],
|
||||
'last_ip': last_ip,
|
||||
})
|
||||
|
||||
|
||||
@bp.route('/api/ddns/register', methods=['POST'])
|
||||
def ddns_register():
|
||||
"""Trigger (re-)registration with the configured DDNS provider."""
|
||||
try:
|
||||
from app import config_manager
|
||||
ddns_cfg = config_manager.configs.get('ddns', {})
|
||||
if ddns_cfg.get('provider') != 'pic_ngo':
|
||||
return jsonify({'error': 'Re-registration only supported for pic_ngo provider'}), 400
|
||||
identity = config_manager.configs.get('_identity', {})
|
||||
cell_name = identity.get('cell_name', os.environ.get('CELL_NAME', ''))
|
||||
if not cell_name:
|
||||
return jsonify({'error': 'cell_name not configured'}), 400
|
||||
from ddns_manager import DDNSManager as _DDNSManager
|
||||
_mgr = _DDNSManager(config_manager)
|
||||
result = _mgr.register(cell_name, '')
|
||||
new_sub = result.get('subdomain', f'{cell_name}.pic.ngo')
|
||||
config_manager.set_identity_field('domain_name', new_sub)
|
||||
logger.info('DDNS registered via /api/ddns/register: cell_name=%r subdomain=%r', cell_name, new_sub)
|
||||
from app import service_bus, EventType
|
||||
_reg_identity = config_manager.configs.get('_identity', {})
|
||||
service_bus.publish_event(EventType.IDENTITY_CHANGED, 'ddns_register', {
|
||||
'cell_name': _reg_identity.get('cell_name'),
|
||||
'domain': _reg_identity.get('domain'),
|
||||
'domain_name': new_sub,
|
||||
'domain_mode': _reg_identity.get('domain_mode'),
|
||||
'effective_domain': config_manager.get_effective_domain(),
|
||||
})
|
||||
return jsonify({'registered': True, 'subdomain': new_sub})
|
||||
except Exception as e:
|
||||
logger.error('Error in /api/ddns/register: %s', e)
|
||||
return jsonify({'error': str(e)}), 500
|
||||
|
||||
|
||||
@bp.route('/api/config/pending', methods=['GET'])
|
||||
def get_pending_config():
|
||||
from app import config_manager
|
||||
@@ -481,11 +699,12 @@ def cancel_pending_config():
|
||||
if cur_cell_name and old_cell_name and cur_cell_name != old_cell_name:
|
||||
network_manager.apply_cell_name(cur_cell_name, old_cell_name, reload=False)
|
||||
|
||||
_ip_revert.write_caddyfile(
|
||||
_id.get('ip_range', os.environ.get('CELL_IP_RANGE', '172.20.0.0/16')),
|
||||
_id.get('cell_name', os.environ.get('CELL_NAME', 'mycell')),
|
||||
_dom, '/app/config-caddy/Caddyfile'
|
||||
)
|
||||
# Regenerate Caddyfile for the reverted identity (all domain modes)
|
||||
try:
|
||||
from app import caddy_manager as _cm
|
||||
_cm.regenerate_with_installed([])
|
||||
except Exception as _cm_err:
|
||||
logger.warning('cancel_pending_config: caddy regenerate failed (non-fatal): %s', _cm_err)
|
||||
|
||||
_clear_pending_restart()
|
||||
return jsonify({'message': 'Pending changes discarded'})
|
||||
@@ -604,8 +823,8 @@ def apply_pending_config():
|
||||
@bp.route('/api/config/backup', methods=['POST'])
|
||||
def create_config_backup():
|
||||
try:
|
||||
from app import config_manager, service_bus, EventType
|
||||
backup_id = config_manager.backup_config()
|
||||
from app import config_manager, service_bus, service_registry, EventType
|
||||
backup_id = config_manager.backup_config(service_registry=service_registry)
|
||||
service_bus.publish_event(EventType.BACKUP_CREATED, 'api', {
|
||||
'backup_id': backup_id,
|
||||
'timestamp': datetime.utcnow().isoformat()
|
||||
@@ -629,9 +848,14 @@ def list_config_backups():
|
||||
@bp.route('/api/config/restore/<backup_id>', methods=['POST'])
|
||||
def restore_config(backup_id):
|
||||
try:
|
||||
from app import config_manager, service_bus, EventType
|
||||
from app import config_manager, service_bus, service_registry, EventType
|
||||
data = request.get_json(silent=True) or {}
|
||||
success = config_manager.restore_config(backup_id, services=data.get('services'))
|
||||
services = data.get('services')
|
||||
success = config_manager.restore_config(
|
||||
backup_id,
|
||||
services=services,
|
||||
service_registry=service_registry if services is None else None,
|
||||
)
|
||||
if success:
|
||||
service_bus.publish_event(EventType.RESTORE_COMPLETED, 'api', {
|
||||
'backup_id': backup_id,
|
||||
|
||||
+13
-5
@@ -1,29 +1,33 @@
|
||||
import logging
|
||||
from flask import Blueprint, request, jsonify
|
||||
from routes import require_active_service
|
||||
|
||||
logger = logging.getLogger('picell')
|
||||
bp = Blueprint('email', __name__)
|
||||
|
||||
@bp.route('/api/email/users', methods=['GET'])
|
||||
@require_active_service('email')
|
||||
def get_email_users():
|
||||
"""Get email users."""
|
||||
try:
|
||||
from app import email_manager
|
||||
users = email_manager.get_users()
|
||||
users = email_manager.get_email_users()
|
||||
return jsonify(users)
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting email users: {e}")
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
@bp.route('/api/email/users', methods=['POST'])
|
||||
@require_active_service('email')
|
||||
def create_email_user():
|
||||
"""Create email user."""
|
||||
try:
|
||||
from app import email_manager, _configured_domain
|
||||
from app import email_manager, config_manager
|
||||
data = request.get_json(silent=True)
|
||||
if data is None:
|
||||
return jsonify({"error": "No data provided"}), 400
|
||||
username = data.get('username')
|
||||
domain = data.get('domain') or _configured_domain()
|
||||
domain = data.get('domain') or config_manager.get_effective_domain()
|
||||
password = data.get('password')
|
||||
if not username or not password:
|
||||
return jsonify({"error": "Missing required fields: username, password"}), 400
|
||||
@@ -34,11 +38,12 @@ def create_email_user():
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
@bp.route('/api/email/users/<username>', methods=['DELETE'])
|
||||
@require_active_service('email')
|
||||
def delete_email_user(username):
|
||||
"""Delete email user."""
|
||||
try:
|
||||
from app import email_manager, _configured_domain
|
||||
domain = request.args.get('domain') or _configured_domain()
|
||||
from app import email_manager, config_manager
|
||||
domain = request.args.get('domain') or config_manager.get_effective_domain()
|
||||
result = email_manager.delete_email_user(username, domain)
|
||||
return jsonify({"deleted": result})
|
||||
except Exception as e:
|
||||
@@ -57,6 +62,7 @@ def get_email_status():
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
@bp.route('/api/email/connectivity', methods=['GET'])
|
||||
@require_active_service('email')
|
||||
def test_email_connectivity():
|
||||
"""Test email connectivity."""
|
||||
try:
|
||||
@@ -68,6 +74,7 @@ def test_email_connectivity():
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
@bp.route('/api/email/send', methods=['POST'])
|
||||
@require_active_service('email')
|
||||
def send_email():
|
||||
try:
|
||||
from app import email_manager
|
||||
@@ -81,6 +88,7 @@ def send_email():
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
@bp.route('/api/email/mailbox/<username>', methods=['GET'])
|
||||
@require_active_service('email')
|
||||
def get_mailbox_info(username):
|
||||
"""Get mailbox information."""
|
||||
try:
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
import logging
|
||||
from flask import Blueprint, request, jsonify
|
||||
from routes import require_active_service
|
||||
|
||||
logger = logging.getLogger('picell')
|
||||
bp = Blueprint('files', __name__)
|
||||
|
||||
@bp.route('/api/files/users', methods=['GET'])
|
||||
@require_active_service('files')
|
||||
def get_file_users():
|
||||
"""Get file storage users."""
|
||||
try:
|
||||
@@ -15,6 +18,7 @@ def get_file_users():
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
@bp.route('/api/files/users', methods=['POST'])
|
||||
@require_active_service('files')
|
||||
def create_file_user():
|
||||
"""Create file storage user."""
|
||||
try:
|
||||
@@ -33,6 +37,7 @@ def create_file_user():
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
@bp.route('/api/files/users/<username>', methods=['DELETE'])
|
||||
@require_active_service('files')
|
||||
def delete_file_user(username):
|
||||
"""Delete file storage user."""
|
||||
try:
|
||||
@@ -44,6 +49,7 @@ def delete_file_user(username):
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
@bp.route('/api/files/folders', methods=['POST'])
|
||||
@require_active_service('files')
|
||||
def create_folder():
|
||||
"""Create folder."""
|
||||
try:
|
||||
@@ -64,6 +70,7 @@ def create_folder():
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
@bp.route('/api/files/folders/<username>/<path:folder_path>', methods=['DELETE'])
|
||||
@require_active_service('files')
|
||||
def delete_folder(username, folder_path):
|
||||
"""Delete folder."""
|
||||
try:
|
||||
@@ -77,6 +84,7 @@ def delete_folder(username, folder_path):
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
@bp.route('/api/files/upload/<username>', methods=['POST'])
|
||||
@require_active_service('files')
|
||||
def upload_file(username):
|
||||
"""Upload file."""
|
||||
try:
|
||||
@@ -97,6 +105,7 @@ def upload_file(username):
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
@bp.route('/api/files/download/<username>/<path:file_path>', methods=['GET'])
|
||||
@require_active_service('files')
|
||||
def download_file(username, file_path):
|
||||
"""Download file."""
|
||||
try:
|
||||
@@ -110,6 +119,7 @@ def download_file(username, file_path):
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
@bp.route('/api/files/delete/<username>/<path:file_path>', methods=['DELETE'])
|
||||
@require_active_service('files')
|
||||
def delete_file(username, file_path):
|
||||
"""Delete file."""
|
||||
try:
|
||||
@@ -123,6 +133,7 @@ def delete_file(username, file_path):
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
@bp.route('/api/files/list/<username>', methods=['GET'])
|
||||
@require_active_service('files')
|
||||
def list_files(username):
|
||||
"""List files."""
|
||||
try:
|
||||
@@ -148,6 +159,7 @@ def get_file_status():
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
@bp.route('/api/files/connectivity', methods=['GET'])
|
||||
@require_active_service('files')
|
||||
def test_file_connectivity():
|
||||
"""Test file service connectivity."""
|
||||
try:
|
||||
|
||||
+16
-1
@@ -1,5 +1,6 @@
|
||||
import logging
|
||||
from flask import Blueprint, request, jsonify
|
||||
import os
|
||||
from flask import Blueprint, request, jsonify, Response
|
||||
logger = logging.getLogger('picell')
|
||||
bp = Blueprint('network', __name__)
|
||||
|
||||
@@ -99,6 +100,20 @@ def get_dns_status():
|
||||
logger.error(f"Error getting DNS status: {e}")
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
@bp.route('/api/network/dns/corefile', methods=['GET'])
|
||||
def get_corefile():
|
||||
try:
|
||||
from app import COREFILE_PATH
|
||||
with open(COREFILE_PATH, 'r') as f:
|
||||
content = f.read()
|
||||
return Response(content, mimetype='text/plain')
|
||||
except FileNotFoundError:
|
||||
return Response('', mimetype='text/plain'), 404
|
||||
except Exception as e:
|
||||
logger.error(f"Error reading Corefile: {e}")
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
|
||||
@bp.route('/api/network/test', methods=['POST'])
|
||||
def test_network():
|
||||
try:
|
||||
|
||||
@@ -65,10 +65,11 @@ def peer_services():
|
||||
wg_port = 51820
|
||||
server_endpoint = ''
|
||||
try:
|
||||
from routes.wireguard import _effective_endpoint
|
||||
from app import config_manager
|
||||
server_public_key = wireguard_manager.get_keys().get('public_key', '')
|
||||
wg_port = config_manager.configs.get('_identity', {}).get('wireguard_port', 51820)
|
||||
srv = wireguard_manager.get_server_config()
|
||||
server_endpoint = srv.get('endpoint') or '<SERVER_IP>'
|
||||
server_endpoint = _effective_endpoint(wireguard_manager, config_manager)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
+95
-10
@@ -37,7 +37,8 @@ def add_peer():
|
||||
try:
|
||||
from app import (peer_registry, wireguard_manager, firewall_manager,
|
||||
email_manager, calendar_manager, file_manager, auth_manager,
|
||||
cell_link_manager, _configured_domain, COREFILE_PATH)
|
||||
cell_link_manager, _configured_domain, _configured_dns_params,
|
||||
config_manager as _app_cfg, COREFILE_PATH)
|
||||
try:
|
||||
_wg_addr = wireguard_manager._get_configured_address()
|
||||
_wg_subnet = str(ipaddress.ip_network(_wg_addr, strict=False)) if _wg_addr else '10.0.0.0/24'
|
||||
@@ -64,7 +65,13 @@ def add_peer():
|
||||
except ValueError as e:
|
||||
return jsonify({'error': str(e)}), 409
|
||||
|
||||
_valid_services = {'calendar', 'files', 'mail', 'webdav'}
|
||||
# 'webdav' is part of the 'files' store service (same container set);
|
||||
# expose it only when 'files' is installed.
|
||||
_STORE_ID_TO_ACCESS = {'email': 'mail', 'calendar': 'calendar', 'files': 'files'}
|
||||
_installed = set(_app_cfg.get_installed_services() or {})
|
||||
_valid_services = {_STORE_ID_TO_ACCESS[sid] for sid in _installed if sid in _STORE_ID_TO_ACCESS}
|
||||
if 'files' in _installed:
|
||||
_valid_services.add('webdav')
|
||||
service_access = data.get('service_access', list(_valid_services))
|
||||
if not isinstance(service_access, list) or not all(s in _valid_services for s in service_access):
|
||||
return jsonify({"error": f"service_access must be a list of: {sorted(_valid_services)}"}), 400
|
||||
@@ -89,6 +96,20 @@ def add_peer():
|
||||
except Exception as e:
|
||||
logger.warning(f"Peer {peer_name}: {step_name} account creation failed (non-fatal): {e}")
|
||||
|
||||
# Provision accounts for installed HTTP-backed store services (non-fatal)
|
||||
try:
|
||||
from app import account_manager as _am, config_manager as _cfg, service_registry as _sreg
|
||||
for _svc_id in (_cfg.get_installed_services() or {}):
|
||||
_svc_info = _sreg.get(_svc_id)
|
||||
if _svc_info and (_svc_info.get('accounts') or {}).get('manager') == 'http':
|
||||
try:
|
||||
_am.provision(_svc_id, peer_name)
|
||||
except Exception as _he:
|
||||
logger.warning('Peer %s: HTTP account provision for %s failed (non-fatal): %s',
|
||||
peer_name, _svc_id, _he)
|
||||
except Exception as _am_err:
|
||||
logger.warning('Peer %s: HTTP store provisioning failed (non-fatal): %s', peer_name, _am_err)
|
||||
|
||||
peer_info = {
|
||||
'peer': peer_name,
|
||||
'ip': assigned_ip,
|
||||
@@ -125,6 +146,17 @@ def add_peer():
|
||||
return jsonify({"error": f"Peer {peer_name} already exists"}), 400
|
||||
peer_added_to_registry = True
|
||||
|
||||
# Store credentials only after the peer is committed — avoids orphaned
|
||||
# credential entries if peer_registry.add_peer rejects a duplicate name.
|
||||
try:
|
||||
from app import account_manager
|
||||
_svc_names = {'email', 'calendar', 'files'}
|
||||
for svc in provisioned:
|
||||
if svc in _svc_names:
|
||||
account_manager.store_credentials(svc, peer_name, {'password': password})
|
||||
except Exception as _am_err:
|
||||
logger.warning(f"Peer {peer_name}: credential storage failed (non-fatal): {_am_err}")
|
||||
|
||||
firewall_manager.apply_peer_rules(peer_info['ip'], peer_info,
|
||||
wg_subnet=_wg_subnet, cell_subnets=_cell_subnets)
|
||||
firewall_applied = True
|
||||
@@ -135,8 +167,10 @@ def add_peer():
|
||||
except Exception as wg_err:
|
||||
logger.warning(f"Peer {peer_name}: WireGuard server config update failed (non-fatal): {wg_err}")
|
||||
|
||||
firewall_manager.apply_all_dns_rules(peer_registry.list_peers(), COREFILE_PATH, _configured_domain(),
|
||||
cell_links=cell_link_manager.list_connections())
|
||||
_dns_primary, _dns_szones = _configured_dns_params()
|
||||
firewall_manager.apply_all_dns_rules(peer_registry.list_peers(), COREFILE_PATH, _dns_primary,
|
||||
cell_links=cell_link_manager.list_connections(),
|
||||
split_horizon_zones=_dns_szones)
|
||||
return jsonify({"message": f"Peer {peer_name} added successfully", "ip": assigned_ip}), 201
|
||||
|
||||
except Exception as e:
|
||||
@@ -158,11 +192,24 @@ def add_peer():
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
|
||||
@bp.route('/api/peers/<peer_name>', methods=['GET'])
|
||||
def get_peer(peer_name):
|
||||
try:
|
||||
from app import peer_registry
|
||||
peer = peer_registry.get_peer(peer_name)
|
||||
if peer is None:
|
||||
return jsonify({'error': 'Peer not found'}), 404
|
||||
return jsonify(peer)
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting peer {peer_name}: {e}")
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
|
||||
@bp.route('/api/peers/<peer_name>', methods=['PUT'])
|
||||
def update_peer(peer_name):
|
||||
try:
|
||||
from app import (peer_registry, wireguard_manager, firewall_manager,
|
||||
cell_link_manager, _configured_domain, COREFILE_PATH)
|
||||
cell_link_manager, _configured_dns_params, COREFILE_PATH)
|
||||
try:
|
||||
_wg_addr = wireguard_manager._get_configured_address()
|
||||
_wg_subnet = str(ipaddress.ip_network(_wg_addr, strict=False)) if _wg_addr else '10.0.0.0/24'
|
||||
@@ -191,8 +238,10 @@ def update_peer(peer_name):
|
||||
if updated_peer:
|
||||
firewall_manager.apply_peer_rules(updated_peer['ip'], updated_peer,
|
||||
wg_subnet=_wg_subnet, cell_subnets=_cell_subnets)
|
||||
firewall_manager.apply_all_dns_rules(peer_registry.list_peers(), COREFILE_PATH, _configured_domain(),
|
||||
cell_links=cell_link_manager.list_connections())
|
||||
_dns_primary, _dns_szones = _configured_dns_params()
|
||||
firewall_manager.apply_all_dns_rules(peer_registry.list_peers(), COREFILE_PATH, _dns_primary,
|
||||
cell_links=cell_link_manager.list_connections(),
|
||||
split_horizon_zones=_dns_szones)
|
||||
return jsonify({"message": f"Peer {peer_name} updated", "config_changed": config_changed})
|
||||
return jsonify({"error": "Update failed"}), 500
|
||||
except Exception as e:
|
||||
@@ -293,7 +342,7 @@ def remove_peer(peer_name):
|
||||
try:
|
||||
from app import (peer_registry, wireguard_manager, firewall_manager,
|
||||
email_manager, calendar_manager, file_manager, auth_manager,
|
||||
cell_link_manager, _configured_domain, COREFILE_PATH)
|
||||
cell_link_manager, _configured_domain, _configured_dns_params, COREFILE_PATH)
|
||||
peer = peer_registry.get_peer(peer_name)
|
||||
if not peer:
|
||||
return jsonify({"message": f"Peer {peer_name} not found or already removed"})
|
||||
@@ -303,8 +352,10 @@ def remove_peer(peer_name):
|
||||
if success:
|
||||
if peer_ip:
|
||||
firewall_manager.clear_peer_rules(peer_ip)
|
||||
firewall_manager.apply_all_dns_rules(peer_registry.list_peers(), COREFILE_PATH, _configured_domain(),
|
||||
cell_links=cell_link_manager.list_connections())
|
||||
_dns_primary, _dns_szones = _configured_dns_params()
|
||||
firewall_manager.apply_all_dns_rules(peer_registry.list_peers(), COREFILE_PATH, _dns_primary,
|
||||
cell_links=cell_link_manager.list_connections(),
|
||||
split_horizon_zones=_dns_szones)
|
||||
if peer_pubkey:
|
||||
try:
|
||||
wireguard_manager.remove_peer(peer_pubkey)
|
||||
@@ -320,12 +371,46 @@ def remove_peer(peer_name):
|
||||
_cleanup()
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
from app import account_manager
|
||||
account_manager.deprovision_peer(peer_name)
|
||||
except Exception as _am_err:
|
||||
logger.warning(f"Peer {peer_name}: account_manager cleanup failed (non-fatal): {_am_err}")
|
||||
return jsonify({"message": f"Peer {peer_name} removed successfully"})
|
||||
except Exception as e:
|
||||
logger.error(f"Error removing peer: {e}")
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
|
||||
@bp.route('/api/peers/<peer_name>/service-credentials', methods=['GET'])
|
||||
def get_peer_service_credentials(peer_name: str):
|
||||
"""Return service credentials for a peer across all provisioned services (admin only).
|
||||
|
||||
Returns filled peer_config_template values for each service the peer is provisioned on.
|
||||
Intended for an admin to view or copy credentials to share with the peer during
|
||||
device setup. The global enforce_auth gate already restricts this to admin sessions.
|
||||
|
||||
Phase 2 note: a peer-self-service variant should live at /api/peer/service-credentials
|
||||
(no path arg) and restrict to session['username'] to prevent cross-peer enumeration.
|
||||
"""
|
||||
try:
|
||||
from app import peer_registry, account_manager, service_registry, config_manager
|
||||
peer = peer_registry.get_peer(peer_name)
|
||||
if not peer:
|
||||
return jsonify({'error': f'Peer {peer_name!r} not found'}), 404
|
||||
raw_creds = account_manager.get_all_credentials(peer_name)
|
||||
identity = config_manager.get_identity()
|
||||
domain = config_manager.get_effective_domain() or identity.get('domain', '')
|
||||
result = {}
|
||||
for service_id, cred in raw_creds.items():
|
||||
svc_info = service_registry.get_peer_service_info(service_id, peer_name, domain, cred)
|
||||
result[service_id] = svc_info if svc_info is not None else cred
|
||||
return jsonify({'peer': peer_name, 'services': result})
|
||||
except Exception as e:
|
||||
logger.error('get_peer_service_credentials(%s): %s', peer_name, e)
|
||||
return jsonify({'error': str(e)}), 500
|
||||
|
||||
|
||||
@bp.route('/api/peers/register', methods=['POST'])
|
||||
def register_peer():
|
||||
try:
|
||||
|
||||
@@ -62,6 +62,10 @@ def install_service(service_id: str):
|
||||
result = _ssm().install(service_id)
|
||||
if result.get('ok'):
|
||||
return jsonify(result)
|
||||
# Normalize docker compose stderr into the error key so the frontend
|
||||
# can display the actual failure reason rather than a generic message.
|
||||
if not result.get('error') and result.get('stderr'):
|
||||
result = {**result, 'error': result['stderr']}
|
||||
return jsonify(result), 400
|
||||
except Exception as e:
|
||||
logger.error(f'install_service({service_id}): {e}')
|
||||
|
||||
+190
-1
@@ -6,6 +6,194 @@ from flask import Blueprint, request, jsonify
|
||||
logger = logging.getLogger('picell')
|
||||
bp = Blueprint('services', __name__)
|
||||
|
||||
@bp.route('/api/services/catalog', methods=['GET'])
|
||||
def get_services_catalog():
|
||||
"""
|
||||
Return all services (builtins + installed store packages) with merged config.
|
||||
Used by the frontend to build navigation and service pages dynamically.
|
||||
"""
|
||||
try:
|
||||
from app import service_registry
|
||||
return jsonify({'services': service_registry.list_all()})
|
||||
except Exception as e:
|
||||
logger.error('get_services_catalog: %s', e)
|
||||
return jsonify({'error': str(e)}), 500
|
||||
|
||||
|
||||
@bp.route('/api/services/active', methods=['GET'])
|
||||
def get_active_services():
|
||||
"""Return minimal info for all installed services. Used by webui to build nav."""
|
||||
try:
|
||||
from app import service_registry
|
||||
active = service_registry.list_active()
|
||||
return jsonify([
|
||||
{
|
||||
'id': svc['id'],
|
||||
'name': svc.get('name', svc['id']),
|
||||
'subdomain': svc.get('subdomain'),
|
||||
'capabilities': svc.get('capabilities', {}),
|
||||
}
|
||||
for svc in active
|
||||
])
|
||||
except Exception as e:
|
||||
logger.error('get_active_services: %s', e)
|
||||
return jsonify({'error': str(e)}), 500
|
||||
|
||||
|
||||
@bp.route('/api/services/catalog/<service_id>', methods=['GET'])
|
||||
def get_service_catalog_entry(service_id: str):
|
||||
"""Return a single service manifest+config, or 404 if unknown."""
|
||||
try:
|
||||
from app import service_registry
|
||||
svc = service_registry.get(service_id)
|
||||
if svc is None:
|
||||
return jsonify({'error': f'Service {service_id!r} not found'}), 404
|
||||
return jsonify(svc)
|
||||
except Exception as e:
|
||||
logger.error('get_service_catalog_entry(%s): %s', service_id, e)
|
||||
return jsonify({'error': str(e)}), 500
|
||||
|
||||
|
||||
@bp.route('/api/services/catalog/<service_id>/status', methods=['GET'])
|
||||
def get_service_container_status(service_id: str):
|
||||
"""
|
||||
Return container status for a service.
|
||||
Builtins query the main compose stack; store services query their own compose project.
|
||||
"""
|
||||
try:
|
||||
from app import service_registry, service_composer
|
||||
svc = service_registry.get(service_id)
|
||||
if svc is None:
|
||||
return jsonify({'error': f'Service {service_id!r} not found'}), 404
|
||||
result = service_composer.status_service(service_id, svc)
|
||||
return jsonify(result)
|
||||
except ValueError as e:
|
||||
return jsonify({'error': str(e)}), 400
|
||||
except Exception as e:
|
||||
logger.error('get_service_container_status(%s): %s', service_id, e)
|
||||
return jsonify({'error': str(e)}), 500
|
||||
|
||||
|
||||
@bp.route('/api/services/catalog/<service_id>/restart', methods=['POST'])
|
||||
def restart_service_containers(service_id: str):
|
||||
"""
|
||||
Restart containers for a service.
|
||||
Builtins restart via the main compose stack; store services via their own compose project.
|
||||
"""
|
||||
try:
|
||||
from app import service_registry, service_composer
|
||||
svc = service_registry.get(service_id)
|
||||
if svc is None:
|
||||
return jsonify({'error': f'Service {service_id!r} not found'}), 404
|
||||
result = service_composer.restart_service(service_id, svc)
|
||||
if result['ok']:
|
||||
return jsonify({'message': f'Service {service_id!r} restarted', **result})
|
||||
return jsonify({'error': result.get('stderr') or result.get('error', 'restart failed')}), 500
|
||||
except ValueError as e:
|
||||
return jsonify({'error': str(e)}), 400
|
||||
except Exception as e:
|
||||
logger.error('restart_service_containers(%s): %s', service_id, e)
|
||||
return jsonify({'error': str(e)}), 500
|
||||
|
||||
|
||||
@bp.route('/api/services/catalog/<service_id>/reconfigure', methods=['POST'])
|
||||
def reconfigure_service(service_id: str):
|
||||
"""
|
||||
Re-apply the stored compose file for a store service (rolling `up -d`).
|
||||
The compose template must already exist on disk from the original install —
|
||||
accepting templates from the request body is deliberately not supported
|
||||
(arbitrary compose files can mount host paths or request privileged mode).
|
||||
"""
|
||||
try:
|
||||
from app import service_registry, service_composer
|
||||
svc = service_registry.get(service_id)
|
||||
if svc is None:
|
||||
return jsonify({'error': f'Service {service_id!r} not found'}), 404
|
||||
if svc.get('kind') == 'builtin':
|
||||
return jsonify({'error': 'Builtins are reconfigured via their settings routes'}), 400
|
||||
if not service_composer.has_compose_file(service_id):
|
||||
return jsonify({'error': f'No compose file for {service_id!r} — install it first'}), 400
|
||||
|
||||
result = service_composer.up(service_id)
|
||||
if result['ok']:
|
||||
return jsonify({'message': f'Service {service_id!r} reconfigured', **result})
|
||||
return jsonify({'error': result.get('stderr') or result.get('error', 'reconfigure failed')}), 500
|
||||
except ValueError as e:
|
||||
return jsonify({'error': str(e)}), 400
|
||||
except Exception as e:
|
||||
logger.error('reconfigure_service(%s): %s', service_id, e)
|
||||
return jsonify({'error': str(e)}), 500
|
||||
|
||||
|
||||
@bp.route('/api/services/catalog/<service_id>/accounts', methods=['GET'])
|
||||
def list_service_accounts(service_id: str):
|
||||
"""Return peer usernames provisioned on a service."""
|
||||
try:
|
||||
from app import account_manager
|
||||
accounts = account_manager.list_accounts(service_id)
|
||||
return jsonify({'service_id': service_id, 'accounts': accounts})
|
||||
except Exception as e:
|
||||
logger.error('list_service_accounts(%s): %s', service_id, e)
|
||||
return jsonify({'error': str(e)}), 500
|
||||
|
||||
|
||||
@bp.route('/api/services/catalog/<service_id>/accounts', methods=['POST'])
|
||||
def provision_service_account(service_id: str):
|
||||
"""Provision a peer account on a service. Generates a password if none is given.
|
||||
|
||||
The generated or provided password is NOT echoed in this response — retrieve it
|
||||
separately via GET /api/services/catalog/<id>/accounts/<username>/credentials.
|
||||
This keeps passwords out of HTTP logs and browser network panels.
|
||||
"""
|
||||
try:
|
||||
from app import account_manager
|
||||
data = request.get_json(silent=True) or {}
|
||||
peer_username = data.get('username')
|
||||
if not peer_username:
|
||||
return jsonify({'error': 'username is required'}), 400
|
||||
account_manager.provision(service_id, peer_username,
|
||||
password=data.get('password'))
|
||||
return jsonify({'service_id': service_id, 'username': peer_username,
|
||||
'provisioned': True}), 201
|
||||
except ValueError as e:
|
||||
return jsonify({'error': str(e)}), 400
|
||||
except RuntimeError as e:
|
||||
return jsonify({'error': str(e)}), 500
|
||||
except Exception as e:
|
||||
logger.error('provision_service_account(%s): %s', service_id, e)
|
||||
return jsonify({'error': str(e)}), 500
|
||||
|
||||
|
||||
@bp.route('/api/services/catalog/<service_id>/accounts/<username>', methods=['DELETE'])
|
||||
def deprovision_service_account(service_id: str, username: str):
|
||||
"""Remove a peer's account from a service."""
|
||||
try:
|
||||
from app import account_manager
|
||||
ok = account_manager.deprovision(service_id, username)
|
||||
if ok:
|
||||
return jsonify({'message': f'{username!r} deprovisioned from {service_id!r}'})
|
||||
return jsonify({'error': 'deprovision failed'}), 500
|
||||
except ValueError as e:
|
||||
return jsonify({'error': str(e)}), 400
|
||||
except Exception as e:
|
||||
logger.error('deprovision_service_account(%s, %s): %s', service_id, username, e)
|
||||
return jsonify({'error': str(e)}), 500
|
||||
|
||||
|
||||
@bp.route('/api/services/catalog/<service_id>/accounts/<username>/credentials', methods=['GET'])
|
||||
def get_service_account_credentials(service_id: str, username: str):
|
||||
"""Return stored credentials for a peer on a service."""
|
||||
try:
|
||||
from app import account_manager
|
||||
creds = account_manager.get_credentials(service_id, username)
|
||||
if creds is None:
|
||||
return jsonify({'error': f'{username!r} not provisioned on {service_id!r}'}), 404
|
||||
return jsonify({'service_id': service_id, 'username': username, **creds})
|
||||
except Exception as e:
|
||||
logger.error('get_service_account_credentials(%s, %s): %s', service_id, username, e)
|
||||
return jsonify({'error': str(e)}), 500
|
||||
|
||||
|
||||
@bp.route('/api/services/bus/status', methods=['GET'])
|
||||
def get_service_bus_status():
|
||||
try:
|
||||
@@ -160,7 +348,8 @@ def set_log_verbosity():
|
||||
data = request.get_json(silent=True) or {}
|
||||
for service, level in data.items():
|
||||
log_manager.set_service_level(service, level)
|
||||
levels_file = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'config', 'log_levels.json')
|
||||
_config_dir = os.environ.get('CONFIG_DIR', '/app/config')
|
||||
levels_file = os.path.join(_config_dir, 'log_levels.json')
|
||||
os.makedirs(os.path.dirname(levels_file), exist_ok=True)
|
||||
current = {}
|
||||
if os.path.exists(levels_file):
|
||||
|
||||
+93
-14
@@ -1,10 +1,17 @@
|
||||
import logging
|
||||
import re
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
import json as _json
|
||||
from flask import Blueprint, request, jsonify
|
||||
from setup_manager import DDNS_API_BASE
|
||||
|
||||
logger = logging.getLogger('picell')
|
||||
|
||||
setup_bp = Blueprint('setup', __name__, url_prefix='/api/setup')
|
||||
|
||||
_DOMAIN_RE = re.compile(r'^[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z]{2,})+$', re.I)
|
||||
|
||||
|
||||
def _get_setup_manager():
|
||||
from app import setup_manager
|
||||
@@ -24,8 +31,8 @@ def get_setup_status():
|
||||
def validate_setup_step():
|
||||
"""Validate a single wizard step.
|
||||
|
||||
Expects JSON body: ``{'step': '<step_name>', 'data': {...}}``.
|
||||
Supported steps: ``cell_name``, ``password``.
|
||||
Supported steps: ``cell_name``, ``password``,
|
||||
``pic_ngo_available``, ``cloudflare_token``, ``duckdns_token``.
|
||||
"""
|
||||
sm = _get_setup_manager()
|
||||
if sm.is_setup_complete():
|
||||
@@ -37,12 +44,39 @@ def validate_setup_step():
|
||||
|
||||
if step == 'cell_name':
|
||||
errors = sm.validate_cell_name(data.get('cell_name', ''))
|
||||
elif step == 'password':
|
||||
errors = sm.validate_password(data.get('password', ''))
|
||||
else:
|
||||
return jsonify({'valid': False, 'errors': [f"Unknown step: {step!r}"]}), 400
|
||||
return jsonify({'valid': len(errors) == 0, 'errors': errors})
|
||||
|
||||
return jsonify({'valid': len(errors) == 0, 'errors': errors})
|
||||
if step == 'password':
|
||||
errors = sm.validate_password(data.get('password', ''))
|
||||
return jsonify({'valid': len(errors) == 0, 'errors': errors})
|
||||
|
||||
if step == 'pic_ngo_available':
|
||||
name = data.get('cell_name', '').strip()
|
||||
errors = sm.validate_cell_name(name)
|
||||
if errors:
|
||||
return jsonify({'available': False, 'errors': errors})
|
||||
try:
|
||||
available = _check_pic_ngo_available(name)
|
||||
return jsonify({'available': available})
|
||||
except Exception:
|
||||
return jsonify({'available': False, 'error': 'DDNS service unreachable'}), 503
|
||||
|
||||
if step == 'cloudflare_token':
|
||||
token = data.get('token', '').strip()
|
||||
if not token:
|
||||
return jsonify({'valid': False, 'error': 'Token is required.'})
|
||||
valid = _verify_cloudflare_token(token)
|
||||
return jsonify({'valid': valid})
|
||||
|
||||
if step == 'duckdns_token':
|
||||
subdomain = data.get('subdomain', '').strip()
|
||||
token = data.get('token', '').strip()
|
||||
if not token or not subdomain:
|
||||
return jsonify({'valid': False, 'error': 'Subdomain and token are required.'})
|
||||
valid = _verify_duckdns_token(subdomain, token)
|
||||
return jsonify({'valid': valid})
|
||||
|
||||
return jsonify({'valid': False, 'errors': [f"Unknown step: {step!r}"]}), 400
|
||||
|
||||
|
||||
@setup_bp.route('/complete', methods=['POST'])
|
||||
@@ -54,12 +88,57 @@ def complete_setup():
|
||||
|
||||
payload = request.get_json(silent=True) or {}
|
||||
result = sm.complete_setup(payload)
|
||||
if result.get('success'):
|
||||
try:
|
||||
from app import config_manager, service_bus, EventType, network_manager
|
||||
identity = config_manager.configs.get('_identity', {})
|
||||
cell_name = identity.get('cell_name', '')
|
||||
service_bus.publish_event(EventType.IDENTITY_CHANGED, 'setup', {
|
||||
'cell_name': cell_name,
|
||||
'domain': identity.get('domain'),
|
||||
'domain_name': identity.get('domain_name'),
|
||||
'domain_mode': identity.get('domain_mode'),
|
||||
'effective_domain': config_manager.get_effective_domain(),
|
||||
})
|
||||
# Bootstrap wrote the zone with 'mycell'; rename to the real cell name.
|
||||
if cell_name:
|
||||
network_manager.apply_cell_name('', cell_name)
|
||||
except Exception as exc:
|
||||
logger.warning(f'Failed to publish IDENTITY_CHANGED after setup: {exc}')
|
||||
status_code = 200 if result.get('success') else 400
|
||||
|
||||
# TODO (Phase 3): if result.get('success') and domain_mode == 'pic_ngo':
|
||||
# from app import ddns_manager
|
||||
# name = payload.get('cell_name', '')
|
||||
# ip = payload.get('public_ip', '')
|
||||
# ddns_manager.register(name, ip)
|
||||
|
||||
return jsonify(result), status_code
|
||||
|
||||
|
||||
# ── external validation helpers ───────────────────────────────────────────────
|
||||
|
||||
def _check_pic_ngo_available(name: str) -> bool:
|
||||
try:
|
||||
url = f'{DDNS_API_BASE}/api/v1/check/{name}'
|
||||
with urllib.request.urlopen(url, timeout=8) as resp:
|
||||
body = _json.loads(resp.read())
|
||||
return bool(body.get('available'))
|
||||
except Exception as exc:
|
||||
logger.warning(f'DDNS availability check failed for {name!r}: {exc}')
|
||||
raise
|
||||
|
||||
|
||||
def _verify_cloudflare_token(token: str) -> bool:
|
||||
try:
|
||||
req = urllib.request.Request(
|
||||
'https://api.cloudflare.com/client/v4/user/tokens/verify',
|
||||
headers={'Authorization': f'Bearer {token}'},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=8) as resp:
|
||||
body = _json.loads(resp.read())
|
||||
return bool(body.get('success'))
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _verify_duckdns_token(subdomain: str, token: str) -> bool:
|
||||
try:
|
||||
url = f'https://www.duckdns.org/update?domains={subdomain}&token={token}&ip='
|
||||
with urllib.request.urlopen(url, timeout=8) as resp:
|
||||
return resp.read().strip() == b'OK'
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
+51
-7
@@ -4,6 +4,20 @@ from flask import Blueprint, request, jsonify
|
||||
logger = logging.getLogger('picell')
|
||||
bp = Blueprint('wireguard', __name__)
|
||||
|
||||
|
||||
def _effective_endpoint(wireguard_manager, config_manager) -> str:
|
||||
"""Return the WireGuard endpoint to embed in peer configs.
|
||||
|
||||
Uses wireguard_endpoint from identity config when set (admin override),
|
||||
falling back to get_external_ip() detection.
|
||||
"""
|
||||
srv = wireguard_manager.get_server_config()
|
||||
override = (config_manager.get_identity().get('wireguard_endpoint') or '').strip()
|
||||
if override:
|
||||
port = srv.get('port', 51820)
|
||||
return override if ':' in override else f'{override}:{port}'
|
||||
return srv.get('endpoint') or '<SERVER_IP>'
|
||||
|
||||
@bp.route('/api/wireguard/keys', methods=['GET'])
|
||||
def get_wireguard_keys():
|
||||
try:
|
||||
@@ -171,8 +185,8 @@ def get_peer_config():
|
||||
|
||||
server_endpoint = data.get('server_endpoint', '')
|
||||
if not server_endpoint:
|
||||
srv = wireguard_manager.get_server_config()
|
||||
server_endpoint = srv.get('endpoint') or '<SERVER_IP>'
|
||||
from app import config_manager
|
||||
server_endpoint = _effective_endpoint(wireguard_manager, config_manager)
|
||||
|
||||
allowed_ips = data.get('allowed_ips') or None
|
||||
if not allowed_ips and registered:
|
||||
@@ -198,12 +212,40 @@ def get_peer_config():
|
||||
@bp.route('/api/wireguard/server-config', methods=['GET'])
|
||||
def get_server_config():
|
||||
try:
|
||||
from app import wireguard_manager
|
||||
return jsonify(wireguard_manager.get_server_config())
|
||||
from app import wireguard_manager, config_manager
|
||||
cfg = wireguard_manager.get_server_config()
|
||||
cfg['endpoint_override'] = (config_manager.get_identity().get('wireguard_endpoint') or '').strip()
|
||||
cfg['effective_endpoint'] = _effective_endpoint(wireguard_manager, config_manager)
|
||||
return jsonify(cfg)
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting server config: {e}")
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
@bp.route('/api/wireguard/endpoint', methods=['GET'])
|
||||
def get_wireguard_endpoint():
|
||||
try:
|
||||
from app import wireguard_manager, config_manager
|
||||
return jsonify({
|
||||
'endpoint_override': (config_manager.get_identity().get('wireguard_endpoint') or '').strip(),
|
||||
'detected_endpoint': wireguard_manager.get_server_config().get('endpoint'),
|
||||
'effective_endpoint': _effective_endpoint(wireguard_manager, config_manager),
|
||||
})
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting wireguard endpoint: {e}")
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
@bp.route('/api/wireguard/endpoint', methods=['PUT'])
|
||||
def set_wireguard_endpoint():
|
||||
try:
|
||||
from app import config_manager
|
||||
data = request.get_json(silent=True) or {}
|
||||
override = (data.get('endpoint_override') or '').strip()
|
||||
config_manager.set_identity_field('wireguard_endpoint', override)
|
||||
return jsonify({'endpoint_override': override, 'ok': True})
|
||||
except Exception as e:
|
||||
logger.error(f"Error setting wireguard endpoint: {e}")
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
@bp.route('/api/wireguard/refresh-ip', methods=['GET', 'POST'])
|
||||
def refresh_external_ip():
|
||||
try:
|
||||
@@ -223,7 +265,7 @@ def refresh_external_ip():
|
||||
def apply_wireguard_enforcement():
|
||||
try:
|
||||
from app import (peer_registry, wireguard_manager, firewall_manager,
|
||||
cell_link_manager, _configured_domain, COREFILE_PATH)
|
||||
cell_link_manager, _configured_dns_params, COREFILE_PATH)
|
||||
peers = peer_registry.list_peers()
|
||||
try:
|
||||
_wg_addr = wireguard_manager._get_configured_address()
|
||||
@@ -233,8 +275,10 @@ def apply_wireguard_enforcement():
|
||||
_cell_links = cell_link_manager.list_connections()
|
||||
_cell_subnets = [l['vpn_subnet'] for l in _cell_links if l.get('vpn_subnet')]
|
||||
firewall_manager.apply_all_peer_rules(peers, wg_subnet=_wg_subnet, cell_subnets=_cell_subnets)
|
||||
firewall_manager.apply_all_dns_rules(peers, COREFILE_PATH, _configured_domain(),
|
||||
cell_links=_cell_links)
|
||||
_dns_primary, _dns_szones = _configured_dns_params()
|
||||
firewall_manager.apply_all_dns_rules(peers, COREFILE_PATH, _dns_primary,
|
||||
cell_links=_cell_links,
|
||||
split_horizon_zones=_dns_szones)
|
||||
return jsonify({'ok': True, 'peers': len(peers)})
|
||||
except Exception as e:
|
||||
return jsonify({'error': str(e)}), 500
|
||||
|
||||
@@ -31,6 +31,7 @@ class EventType(Enum):
|
||||
CERTIFICATE_EXPIRING = "certificate_expiring"
|
||||
BACKUP_CREATED = "backup_created"
|
||||
RESTORE_COMPLETED = "restore_completed"
|
||||
IDENTITY_CHANGED = "identity_changed"
|
||||
|
||||
@dataclass
|
||||
class Event:
|
||||
|
||||
@@ -0,0 +1,376 @@
|
||||
"""
|
||||
ServiceComposer — docker-compose generation and container lifecycle for PIC services.
|
||||
|
||||
Responsibilities:
|
||||
- Render compose-template.yml → per-service docker-compose.yml with PIC_* substitution
|
||||
- Manage store-service container lifecycle (up / down / restart / status / reconfigure)
|
||||
- Manage builtin-service restarts and status via the main compose stack
|
||||
- Generate and persist PIC_SECRET_* variables in a dedicated secrets file
|
||||
|
||||
Template variable reference (for compose-template.yml authors):
|
||||
${PIC_CFG_<KEY>} — value from manifest config_schema, uppercased
|
||||
${PIC_SECRET_<NAME>} — auto-generated random secret, persisted across reconfigures
|
||||
${PIC_DOMAIN} — effective domain (e.g. cell.pic.ngo)
|
||||
${PIC_CELL_NAME} — cell name (e.g. mycell)
|
||||
${PIC_SERVICE_ID} — service identifier (e.g. nextcloud)
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import secrets as _secrets_lib
|
||||
import shutil
|
||||
import subprocess
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from manifest_validator import validate_rendered_compose
|
||||
|
||||
logger = logging.getLogger('picell')
|
||||
|
||||
_SECRET_RE = re.compile(r'\$\{(PIC_SECRET_\w+)\}')
|
||||
_SAFE_ID_RE = re.compile(r'^[a-z0-9][a-z0-9_-]{0,63}$')
|
||||
|
||||
|
||||
class ServiceComposer:
|
||||
|
||||
def __init__(self, config_manager, data_dir: str):
|
||||
self.cm = config_manager
|
||||
self.data_dir = data_dir
|
||||
self._services_dir = os.path.join(data_dir, 'services')
|
||||
self._secrets_path = os.path.join(data_dir, 'service_secrets.json')
|
||||
self._lock = threading.Lock()
|
||||
|
||||
# ── Path helpers ──────────────────────────────────────────────────────
|
||||
|
||||
@staticmethod
|
||||
def _validate_service_id(service_id: str) -> None:
|
||||
"""Raise ValueError if service_id could be used for path traversal."""
|
||||
if not _SAFE_ID_RE.match(service_id):
|
||||
raise ValueError(
|
||||
f'Invalid service_id {service_id!r}: '
|
||||
'must match ^[a-z0-9][a-z0-9_-]{{0,63}}$'
|
||||
)
|
||||
|
||||
def _svc_dir(self, service_id: str) -> str:
|
||||
self._validate_service_id(service_id)
|
||||
candidate = os.path.join(self._services_dir, service_id)
|
||||
# Paranoia: ensure the resolved path stays inside _services_dir
|
||||
real_base = os.path.realpath(self._services_dir)
|
||||
real_cand = os.path.realpath(candidate)
|
||||
if not real_cand.startswith(real_base + os.sep) and real_cand != real_base:
|
||||
raise ValueError(f'service_id {service_id!r} escapes services directory')
|
||||
return candidate
|
||||
|
||||
def _compose_path(self, service_id: str) -> str:
|
||||
return os.path.join(self._svc_dir(service_id), 'docker-compose.yml')
|
||||
|
||||
def has_compose_file(self, service_id: str) -> bool:
|
||||
try:
|
||||
return os.path.exists(self._compose_path(service_id))
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
# ── Secrets management ────────────────────────────────────────────────
|
||||
|
||||
def _load_secrets(self) -> Dict:
|
||||
if not os.path.exists(self._secrets_path):
|
||||
return {}
|
||||
try:
|
||||
with open(self._secrets_path) as f:
|
||||
return json.load(f)
|
||||
except (OSError, json.JSONDecodeError) as e:
|
||||
logger.warning('ServiceComposer: failed to load secrets: %s', e)
|
||||
return {}
|
||||
|
||||
def _save_secrets(self, secrets: Dict) -> None:
|
||||
tmp = self._secrets_path + '.tmp'
|
||||
# 0o600: readable only by the process owner — secrets must not be world-readable
|
||||
with open(tmp, 'w',
|
||||
opener=lambda path, flags: os.open(path, flags, 0o600)) as f:
|
||||
json.dump(secrets, f, indent=2)
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
os.replace(tmp, self._secrets_path)
|
||||
|
||||
def _get_or_create_secret(self, service_id: str, var_name: str) -> str:
|
||||
with self._lock:
|
||||
secrets = self._load_secrets()
|
||||
svc_secrets = secrets.setdefault(service_id, {})
|
||||
if var_name not in svc_secrets:
|
||||
svc_secrets[var_name] = _secrets_lib.token_urlsafe(24)
|
||||
self._save_secrets(secrets)
|
||||
return svc_secrets[var_name]
|
||||
|
||||
def _clear_secrets(self, service_id: str) -> None:
|
||||
with self._lock:
|
||||
secrets = self._load_secrets()
|
||||
if service_id in secrets:
|
||||
del secrets[service_id]
|
||||
self._save_secrets(secrets)
|
||||
|
||||
# ── Template rendering ────────────────────────────────────────────────
|
||||
|
||||
def render_template(self, service_id: str, manifest: Dict,
|
||||
template_content: str) -> str:
|
||||
"""
|
||||
Substitute all PIC_* variables in a compose-template.yml string.
|
||||
Returns the rendered compose YAML.
|
||||
"""
|
||||
schema = manifest.get('config_schema') or {}
|
||||
saved = self.cm.configs.get(service_id, {})
|
||||
config: Dict = {k: v['default'] for k, v in schema.items() if 'default' in v}
|
||||
config.update({k: saved[k] for k in schema if k in saved})
|
||||
|
||||
identity = self.cm.get_identity()
|
||||
domain = self.cm.get_effective_domain() or identity.get('domain', 'cell.local')
|
||||
cell_name = identity.get('cell_name', 'mycell')
|
||||
|
||||
result = template_content
|
||||
|
||||
for key, value in config.items():
|
||||
# Strip newlines/tabs to prevent YAML injection (a config string containing
|
||||
# \n could inject new YAML keys into the compose file)
|
||||
safe_val = str(value).replace('\n', '').replace('\r', '').replace('\t', ' ')
|
||||
result = result.replace(f'${{PIC_CFG_{key.upper()}}}', safe_val)
|
||||
|
||||
result = result.replace('${PIC_DOMAIN}', domain)
|
||||
result = result.replace('${PIC_CELL_NAME}', cell_name)
|
||||
result = result.replace('${PIC_SERVICE_ID}', service_id)
|
||||
result = result.replace('${PIC_DATA_DIR}', str(Path(self.data_dir).resolve()))
|
||||
|
||||
# PIC_SECRET_* — generate on first use, reuse on reconfigure
|
||||
for match in _SECRET_RE.finditer(template_content):
|
||||
var_name = match.group(1)
|
||||
secret = self._get_or_create_secret(service_id, var_name)
|
||||
result = result.replace(f'${{{var_name}}}', secret)
|
||||
|
||||
return result
|
||||
|
||||
def write_compose(self, service_id: str, manifest: Dict,
|
||||
template_content: str) -> str:
|
||||
"""Render and atomically write the per-service compose file. Returns rendered content."""
|
||||
os.makedirs(self._svc_dir(service_id), exist_ok=True)
|
||||
content = self.render_template(service_id, manifest, template_content)
|
||||
|
||||
# Validate before any file I/O so a bad template never touches disk.
|
||||
# Pass the resolved data_dir so that bind mounts created by ${PIC_DATA_DIR}
|
||||
# substitution are allowed; all other absolute paths are still rejected.
|
||||
# Connectivity services (wireguard-ext, openvpn-client, tor) set
|
||||
# requires_host_network: true in their manifest to opt into network_mode: host.
|
||||
allow_host_network = bool(manifest.get('requires_host_network'))
|
||||
ok, errs = validate_rendered_compose(
|
||||
content,
|
||||
allowed_data_dir=str(Path(self.data_dir).resolve()),
|
||||
allow_host_network=allow_host_network,
|
||||
)
|
||||
if not ok:
|
||||
raise ValueError(
|
||||
f'Compose template failed security validation: {"; ".join(errs)}'
|
||||
)
|
||||
|
||||
path = self._compose_path(service_id)
|
||||
tmp = path + '.tmp'
|
||||
with open(tmp, 'w') as f:
|
||||
f.write(content)
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
os.replace(tmp, path)
|
||||
logger.info('ServiceComposer: wrote compose file for %s', service_id)
|
||||
return content
|
||||
|
||||
# ── Subprocess helper ─────────────────────────────────────────────────
|
||||
|
||||
def _run(self, cmd: List[str], timeout: int = 120) -> Dict:
|
||||
try:
|
||||
r = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout)
|
||||
if r.returncode != 0 and r.stderr:
|
||||
logger.warning('ServiceComposer command failed: %s', r.stderr.strip())
|
||||
return {
|
||||
'ok': r.returncode == 0,
|
||||
'stdout': r.stdout.strip(),
|
||||
'stderr': r.stderr.strip(),
|
||||
}
|
||||
except subprocess.TimeoutExpired:
|
||||
return {'ok': False, 'error': 'docker compose command timed out'}
|
||||
except Exception as e:
|
||||
logger.error('ServiceComposer._run error: %s', e)
|
||||
return {'ok': False, 'error': str(e)}
|
||||
|
||||
@staticmethod
|
||||
def _parse_ps_json(output: str) -> List[Dict]:
|
||||
"""Parse `docker compose ps --format json` output (one JSON object per line)."""
|
||||
containers = []
|
||||
for line in output.splitlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
containers.append(json.loads(line))
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
return containers
|
||||
|
||||
# ── Store-service lifecycle (per-service compose file) ────────────────
|
||||
|
||||
def _store_cmd(self, service_id: str, *args, timeout: int = 120) -> Dict:
|
||||
compose_file = self._compose_path(service_id)
|
||||
if not os.path.exists(compose_file):
|
||||
return {'ok': False, 'error': f'No compose file found for service {service_id!r}'}
|
||||
cmd = [
|
||||
'docker', 'compose',
|
||||
'-f', compose_file,
|
||||
'--project-name', f'pic-{service_id}',
|
||||
*args,
|
||||
]
|
||||
return self._run(cmd, timeout)
|
||||
|
||||
def up(self, service_id: str) -> Dict:
|
||||
# 600s: image pulls on slow connections can take several minutes
|
||||
return self._store_cmd(service_id, 'up', '-d', '--remove-orphans', timeout=600)
|
||||
|
||||
def down(self, service_id: str, remove_volumes: bool = False) -> Dict:
|
||||
args = ['down']
|
||||
if remove_volumes:
|
||||
args.append('--volumes')
|
||||
return self._store_cmd(service_id, *args)
|
||||
|
||||
def restart(self, service_id: str) -> Dict:
|
||||
return self._store_cmd(service_id, 'restart')
|
||||
|
||||
def status(self, service_id: str) -> Dict:
|
||||
result = self._store_cmd(service_id, 'ps', '--format', 'json')
|
||||
result['containers'] = self._parse_ps_json(result.get('stdout', ''))
|
||||
return result
|
||||
|
||||
def reconfigure(self, service_id: str, manifest: Dict,
|
||||
template_content: str) -> Dict:
|
||||
"""Re-render the compose file then re-apply with `up -d` (rolling update)."""
|
||||
self.write_compose(service_id, manifest, template_content)
|
||||
return self.up(service_id)
|
||||
|
||||
def install(self, service_id: str, manifest: Dict,
|
||||
template_content: str) -> Dict:
|
||||
"""Write compose file, pull image, then start containers.
|
||||
|
||||
pull is run first so the up step doesn't time out on slow connections.
|
||||
A single retry handles transient registry hiccups on first install.
|
||||
"""
|
||||
self.write_compose(service_id, manifest, template_content)
|
||||
pull = self._store_cmd(service_id, 'pull', timeout=600)
|
||||
if not pull.get('ok'):
|
||||
logger.warning('service_composer: image pull for %s failed, proceeding anyway: %s',
|
||||
service_id, pull.get('stderr', '')[:200])
|
||||
result = self.up(service_id)
|
||||
if not result.get('ok'):
|
||||
logger.info('service_composer: retrying up for %s after initial failure', service_id)
|
||||
result = self.up(service_id)
|
||||
return result
|
||||
|
||||
def remove(self, service_id: str, purge_data: bool = False) -> Dict:
|
||||
"""Stop containers, optionally delete compose file, secrets, and service data dir."""
|
||||
result = self.down(service_id, remove_volumes=purge_data)
|
||||
if purge_data:
|
||||
self._clear_secrets(service_id)
|
||||
svc_dir = self._svc_dir(service_id) # already validates service_id + realpath
|
||||
if os.path.isdir(svc_dir):
|
||||
# Final realpath check: reject symlinks that escape the services dir
|
||||
real_svc = os.path.realpath(svc_dir)
|
||||
real_base = os.path.realpath(self._services_dir)
|
||||
if not real_svc.startswith(real_base + os.sep):
|
||||
logger.error('ServiceComposer: refusing rmtree outside services dir: %s', svc_dir)
|
||||
else:
|
||||
try:
|
||||
shutil.rmtree(svc_dir)
|
||||
except OSError as e:
|
||||
logger.warning('ServiceComposer: could not remove %s: %s', svc_dir, e)
|
||||
elif os.path.exists(self._compose_path(service_id)):
|
||||
# Remove compose file even without purge so stale file doesn't confuse future installs
|
||||
try:
|
||||
os.remove(self._compose_path(service_id))
|
||||
except OSError:
|
||||
pass
|
||||
return result
|
||||
|
||||
# ── Dependency resolution ─────────────────────────────────────────────
|
||||
|
||||
def _resolve_requires(self, manifest: Dict, installed_services: Dict) -> Optional[str]:
|
||||
"""Return an error string if any required services are missing, else None."""
|
||||
requires = manifest.get('requires') or []
|
||||
missing = [r for r in requires if r not in installed_services]
|
||||
if missing:
|
||||
return f"Required services not installed: {', '.join(sorted(missing))}"
|
||||
return None
|
||||
|
||||
def _resolve_dependents(self, service_id: str, installed_services: Dict) -> List[str]:
|
||||
"""Return list of installed service IDs that declare service_id in their requires."""
|
||||
dependents = []
|
||||
for svc_id, record in installed_services.items():
|
||||
if svc_id == service_id:
|
||||
continue
|
||||
m = (record.get('manifest') or {})
|
||||
if service_id in (m.get('requires') or []):
|
||||
dependents.append(svc_id)
|
||||
return dependents
|
||||
|
||||
def reapply_active_services(self) -> None:
|
||||
"""Call up() for every installed service that has a compose file. Called at startup."""
|
||||
installed = self.cm.get_installed_services()
|
||||
for svc_id in installed:
|
||||
if not self.has_compose_file(svc_id):
|
||||
logger.warning('reapply_active_services: no compose file for %s, skipping', svc_id)
|
||||
continue
|
||||
result = self.up(svc_id)
|
||||
if not result.get('ok'):
|
||||
logger.warning('reapply_active_services: up failed for %s: %s',
|
||||
svc_id, result.get('error') or result.get('stderr', ''))
|
||||
|
||||
# ── Builtin-service lifecycle (main compose stack) ─────────────────────
|
||||
|
||||
@staticmethod
|
||||
def _main_compose() -> str:
|
||||
return os.environ.get('COMPOSE_FILE', '/app/docker-compose.yml')
|
||||
|
||||
def restart_builtin(self, container_names: List[str]) -> Dict:
|
||||
"""Restart one or more containers that live in the main docker-compose stack."""
|
||||
if not container_names:
|
||||
return {'ok': False, 'error': 'No container names provided'}
|
||||
cmd = ['docker', 'compose', '-f', self._main_compose(),
|
||||
'restart', *container_names]
|
||||
return self._run(cmd)
|
||||
|
||||
def status_builtin(self, container_names: List[str]) -> Dict:
|
||||
"""Return status of containers from the main compose stack."""
|
||||
if not container_names:
|
||||
return {'ok': False, 'error': 'No container names provided'}
|
||||
cmd = ['docker', 'compose', '-f', self._main_compose(),
|
||||
'ps', '--format', 'json', *container_names]
|
||||
result = self._run(cmd)
|
||||
result['containers'] = self._parse_ps_json(result.get('stdout', ''))
|
||||
return result
|
||||
|
||||
# ── Unified lifecycle (dispatches based on service kind) ───────────────
|
||||
|
||||
def restart_service(self, service_id: str, manifest: Dict) -> Dict:
|
||||
"""
|
||||
Restart any service — builtin or store — using the right compose stack.
|
||||
Builtin: uses manifest.containers + main docker-compose.yml.
|
||||
Store: uses per-service compose file.
|
||||
"""
|
||||
if manifest.get('kind') == 'builtin':
|
||||
containers = manifest.get('containers') or []
|
||||
return self.restart_builtin(containers)
|
||||
return self.restart(service_id)
|
||||
|
||||
def status_service(self, service_id: str, manifest: Dict) -> Dict:
|
||||
"""
|
||||
Return container status for any service.
|
||||
Builtin: queries manifest.containers from main compose stack.
|
||||
Store: queries per-service compose project.
|
||||
"""
|
||||
if manifest.get('kind') == 'builtin':
|
||||
containers = manifest.get('containers') or []
|
||||
return self.status_builtin(containers)
|
||||
return self.status(service_id)
|
||||
@@ -0,0 +1,177 @@
|
||||
"""
|
||||
ServiceRegistry — single source of truth for all PIC services.
|
||||
|
||||
Merges two layers:
|
||||
1. Manifest defaults (config_schema.*.default)
|
||||
2. Admin-saved config from ConfigManager (cell_config.json)
|
||||
|
||||
All consumers (CaddyManager, backup, peer services endpoint) read from here
|
||||
rather than hardcoding service names or subdomains.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import re
|
||||
from typing import Dict, List, Optional
|
||||
from urllib.parse import quote as _urlquote
|
||||
|
||||
logger = logging.getLogger('picell')
|
||||
|
||||
_SUBDOMAIN_RE = re.compile(r'^[a-z][a-z0-9-]{0,30}$')
|
||||
_BACKEND_RE = re.compile(r'^[A-Za-z0-9._-]+:\d{1,5}$')
|
||||
_RESERVED_SUBS = frozenset({'api', 'webui', 'admin', 'www', 'ns1', 'ns2', 'git', 'registry', 'install'})
|
||||
|
||||
|
||||
class ServiceRegistry:
|
||||
|
||||
def __init__(self, config_manager):
|
||||
self._cm = config_manager
|
||||
|
||||
# ── Config merging ────────────────────────────────────────────────────
|
||||
|
||||
_TYPE_COERCIONS = {'integer': int, 'string': str, 'boolean': bool}
|
||||
|
||||
def _merged_config(self, manifest: Dict) -> Dict:
|
||||
"""Return manifest defaults overridden by admin-saved values, type-coerced."""
|
||||
svc_id = manifest.get('id', '')
|
||||
saved = self._cm.configs.get(svc_id, {})
|
||||
schema = manifest.get('config_schema') or {}
|
||||
merged = {k: v['default'] for k, v in schema.items() if 'default' in v}
|
||||
for k, spec in schema.items():
|
||||
if k not in saved:
|
||||
continue
|
||||
raw = saved[k]
|
||||
coerce = self._TYPE_COERCIONS.get(spec.get('type', ''))
|
||||
if coerce is not None:
|
||||
try:
|
||||
raw = coerce(raw)
|
||||
except (TypeError, ValueError):
|
||||
raw = merged.get(k, raw)
|
||||
merged[k] = raw
|
||||
return merged
|
||||
|
||||
# ── Public API ────────────────────────────────────────────────────────
|
||||
|
||||
def get(self, service_id: str) -> Optional[Dict]:
|
||||
"""Return manifest + merged config for one service, or None if unknown."""
|
||||
record = self._cm.get_installed_services().get(service_id)
|
||||
if not record:
|
||||
return None
|
||||
manifest = record.get('manifest')
|
||||
if not manifest:
|
||||
return None
|
||||
return {**manifest, 'config': self._merged_config(manifest)}
|
||||
|
||||
def list_active(self) -> List[Dict]:
|
||||
"""Return all installed store services, each with merged config."""
|
||||
results = []
|
||||
for _svc_id, record in self._cm.get_installed_services().items():
|
||||
manifest = record.get('manifest') or {}
|
||||
if manifest.get('id'):
|
||||
results.append({**manifest, 'config': self._merged_config(manifest)})
|
||||
return results
|
||||
|
||||
def list_all(self) -> List[Dict]:
|
||||
"""Return all installed store services, each with merged config attached as the 'config' key."""
|
||||
return self.list_active()
|
||||
|
||||
def get_caddy_routes(self) -> List[Dict]:
|
||||
"""
|
||||
Return routing info for all services that have a subdomain.
|
||||
Used by CaddyManager to build service blocks without hardcoding.
|
||||
|
||||
Values are validated here as a chokepoint so Caddyfile/DNS builders
|
||||
can safely interpolate them regardless of how manifests reached disk.
|
||||
"""
|
||||
routes = []
|
||||
for svc in self.list_all():
|
||||
caps = svc.get('capabilities') or {}
|
||||
if not caps.get('has_subdomain'):
|
||||
continue
|
||||
sub = svc.get('subdomain', '')
|
||||
bknd = svc.get('backend', '')
|
||||
if not sub or not bknd:
|
||||
continue
|
||||
svc_id = svc.get('id', '?')
|
||||
if not _SUBDOMAIN_RE.match(sub) or sub in _RESERVED_SUBS:
|
||||
logger.warning('ServiceRegistry: skipping %s — invalid/reserved subdomain %r', svc_id, sub)
|
||||
continue
|
||||
if not _BACKEND_RE.match(bknd):
|
||||
logger.warning('ServiceRegistry: skipping %s — invalid backend %r', svc_id, bknd)
|
||||
continue
|
||||
extra_subs = [
|
||||
s for s in (svc.get('extra_subdomains') or [])
|
||||
if isinstance(s, str) and _SUBDOMAIN_RE.match(s) and s not in _RESERVED_SUBS
|
||||
]
|
||||
extra_backends = {
|
||||
k: v for k, v in (svc.get('extra_backends') or {}).items()
|
||||
if (isinstance(k, str) and _SUBDOMAIN_RE.match(k) and k not in _RESERVED_SUBS
|
||||
and isinstance(v, str) and _BACKEND_RE.match(v))
|
||||
}
|
||||
routes.append({
|
||||
'service_id': svc_id,
|
||||
'subdomain': sub,
|
||||
'backend': bknd,
|
||||
'extra_subdomains': extra_subs,
|
||||
'extra_backends': extra_backends,
|
||||
})
|
||||
return routes
|
||||
|
||||
def get_backup_plan(self) -> List[Dict]:
|
||||
"""
|
||||
Return backup declarations for all services that have storage.
|
||||
Used by the backup system instead of hardcoded file lists.
|
||||
|
||||
Each entry:
|
||||
service_id — service identifier
|
||||
volumes — list of {container, path, name} for docker-exec streaming
|
||||
config_paths — host-relative paths copied directly (config files)
|
||||
"""
|
||||
plan = []
|
||||
for svc in self.list_all():
|
||||
caps = svc.get('capabilities') or {}
|
||||
if not caps.get('has_storage'):
|
||||
continue
|
||||
backup = svc.get('backup') or {}
|
||||
volumes = backup.get('volumes') or []
|
||||
config_paths = backup.get('config_paths') or []
|
||||
if not volumes and not config_paths:
|
||||
continue
|
||||
plan.append({
|
||||
'service_id': svc['id'],
|
||||
'volumes': volumes,
|
||||
'config_paths': config_paths,
|
||||
})
|
||||
return plan
|
||||
|
||||
def get_peer_service_info(self, service_id: str, peer_username: str,
|
||||
domain: str, credentials: Dict) -> Optional[Dict]:
|
||||
"""
|
||||
Fill peer_config_template for one service+peer combination.
|
||||
credentials: dict of {field_name: value} for that peer+service.
|
||||
Returns None if service unknown or has no peer template.
|
||||
"""
|
||||
svc = self.get(service_id)
|
||||
if not svc:
|
||||
return None
|
||||
template = svc.get('peer_config_template')
|
||||
if not template:
|
||||
return None
|
||||
|
||||
# URL-safe peer username (safe='') — prevents path traversal in CalDAV/WebDAV URLs
|
||||
safe_username = _urlquote(peer_username, safe='')
|
||||
|
||||
result = {}
|
||||
for key, raw in template.items():
|
||||
val = raw
|
||||
val = val.replace('{domain}', domain)
|
||||
val = val.replace('{peer.username}', safe_username)
|
||||
for field, cred_val in credentials.items():
|
||||
val = val.replace(
|
||||
'{peer.service_credentials.' + service_id + '.' + field + '}',
|
||||
str(cred_val) if cred_val is not None else '',
|
||||
)
|
||||
cfg = svc.get('config') or {}
|
||||
for cfg_key, cfg_val in cfg.items():
|
||||
val = val.replace('{config.' + cfg_key + '}', str(cfg_val) if cfg_val is not None else '')
|
||||
result[key] = val
|
||||
return result
|
||||
+203
-282
@@ -14,15 +14,15 @@ import logging
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
import subprocess
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
import json
|
||||
|
||||
import requests
|
||||
import yaml
|
||||
|
||||
from base_service_manager import BaseServiceManager
|
||||
from ip_utils import CONTAINER_OFFSETS
|
||||
from manifest_validator import validate_manifest, validate_provision_hook
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -30,27 +30,42 @@ logger = logging.getLogger(__name__)
|
||||
# Constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
SERVICE_POOL_START = 20
|
||||
SERVICE_POOL_END = 254
|
||||
|
||||
INDEX_URL_DEFAULT = (
|
||||
'https://git.pic.ngo/roof/pic-services/raw/branch/main/index.json'
|
||||
)
|
||||
MANIFEST_URL_TPL = (
|
||||
'https://git.pic.ngo/roof/pic-services/raw/branch/main/services/{id}/manifest.json'
|
||||
)
|
||||
TEMPLATE_URL_TPL = (
|
||||
'https://git.pic.ngo/roof/pic-services/raw/branch/main/services/{id}/compose-template.yml'
|
||||
)
|
||||
|
||||
IMAGE_ALLOWLIST_RE = re.compile(
|
||||
r'^git\.pic\.ngo/roof/[a-z0-9._/-]+(:[a-zA-Z0-9._-]+)?$'
|
||||
r'^git\.pic\.ngo/roof/[a-z0-9._/-]+(:[a-zA-Z0-9._-]+)?(@sha256:[a-f0-9]{64})?$'
|
||||
)
|
||||
|
||||
# Images from well-known vendors that pre-date digest pinning in PIC.
|
||||
# These are allowed to ship without a @sha256 digest; all others require one
|
||||
# or must come from git.pic.ngo/roof/*.
|
||||
TRUSTED_IMAGES_NO_DIGEST = frozenset({
|
||||
'mailserver/docker-mailserver',
|
||||
'tomsquest/docker-radicale',
|
||||
'bytemark/webdav',
|
||||
'filegator/filegator',
|
||||
'hardware/rainloop',
|
||||
})
|
||||
FORBIDDEN_MOUNTS = frozenset([
|
||||
'/', '/etc', '/var', '/proc', '/sys', '/dev', '/app', '/run', '/boot',
|
||||
])
|
||||
RESERVED_SUBDOMAINS = frozenset([
|
||||
'api', 'webui', 'admin', 'www', 'mail', 'ns1', 'ns2',
|
||||
'api', 'webui', 'admin', 'www', 'ns1', 'ns2',
|
||||
'git', 'registry', 'install',
|
||||
# mail, calendar, files, webmail are intentionally absent:
|
||||
# they are claimed by official PIC store services.
|
||||
])
|
||||
ENV_VALUE_RE = re.compile(r'^[A-Za-z0-9._@:/+\-= ]*$')
|
||||
SUBDOMAIN_RE = re.compile(r'^[a-z][a-z0-9-]{0,30}$')
|
||||
BACKEND_RE = re.compile(r'^[A-Za-z0-9._-]+:\d{1,5}$')
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -61,11 +76,14 @@ class ServiceStoreManager(BaseServiceManager):
|
||||
"""Manages service store: install, remove, and list available/installed services."""
|
||||
|
||||
def __init__(self, config_manager, caddy_manager, container_manager,
|
||||
data_dir: str = '', config_dir: str = ''):
|
||||
data_dir: str = '', config_dir: str = '',
|
||||
service_composer=None, egress_manager=None):
|
||||
super().__init__('service_store', data_dir, config_dir)
|
||||
self.config_manager = config_manager
|
||||
self.caddy_manager = caddy_manager
|
||||
self.container_manager = container_manager
|
||||
self.service_composer = service_composer
|
||||
self.egress_manager = egress_manager
|
||||
self.compose_override = os.environ.get(
|
||||
'COMPOSE_SERVICES_PATH', '/app/docker-compose.services.yml'
|
||||
)
|
||||
@@ -110,6 +128,21 @@ class ServiceStoreManager(BaseServiceManager):
|
||||
errors.append(
|
||||
f'image must match git.pic.ngo/roof/* pattern, got: {image}'
|
||||
)
|
||||
elif image:
|
||||
# Warn when a digest pin is absent so operators know exact-version
|
||||
# tracking is not guaranteed. Images in TRUSTED_IMAGES_NO_DIGEST
|
||||
# and images from our own git.pic.ngo/roof/* registry (which we
|
||||
# build and tag) get warnings rather than hard errors; any other
|
||||
# image that somehow passes the allowlist gets a hard error.
|
||||
if '@sha256:' not in image:
|
||||
image_base = image.split(':')[0].split('@')[0]
|
||||
is_own_registry = image_base.startswith('git.pic.ngo/roof/')
|
||||
if image_base in TRUSTED_IMAGES_NO_DIGEST or is_own_registry:
|
||||
logger.warning('image %s has no digest pin', image)
|
||||
else:
|
||||
errors.append(
|
||||
f'image {image!r} must include a @sha256:<digest> pin'
|
||||
)
|
||||
|
||||
# Volume mount safety
|
||||
for vol in m.get('volumes', []):
|
||||
@@ -141,19 +174,55 @@ class ServiceStoreManager(BaseServiceManager):
|
||||
f'iptables_rules[].proto must be tcp or udp, got: {proto}'
|
||||
)
|
||||
|
||||
# Caddy route subdomain
|
||||
# Legacy caddy_route dict subdomain (for store manifests using the old format)
|
||||
caddy_route = m.get('caddy_route') or {}
|
||||
if isinstance(caddy_route, dict):
|
||||
subdomain = caddy_route.get('subdomain', '')
|
||||
legacy_sub = caddy_route.get('subdomain', '')
|
||||
else:
|
||||
subdomain = ''
|
||||
if subdomain:
|
||||
if subdomain in RESERVED_SUBDOMAINS:
|
||||
errors.append(f'caddy_route.subdomain is reserved: {subdomain}')
|
||||
elif not re.match(r'^[a-z][a-z0-9-]{0,30}$', subdomain):
|
||||
legacy_sub = ''
|
||||
if legacy_sub:
|
||||
if legacy_sub in RESERVED_SUBDOMAINS:
|
||||
errors.append(f'caddy_route.subdomain is reserved: {legacy_sub}')
|
||||
elif not SUBDOMAIN_RE.match(legacy_sub):
|
||||
errors.append(
|
||||
f'caddy_route.subdomain must match ^[a-z][a-z0-9-]{{0,30}}$, '
|
||||
f'got: {subdomain}'
|
||||
f'got: {legacy_sub}'
|
||||
)
|
||||
|
||||
# Top-level subdomain + backend (consumed by ServiceRegistry.get_caddy_routes)
|
||||
subdomain = m.get('subdomain', '')
|
||||
if subdomain:
|
||||
if subdomain in RESERVED_SUBDOMAINS:
|
||||
errors.append(f'subdomain is reserved: {subdomain}')
|
||||
elif not SUBDOMAIN_RE.match(subdomain):
|
||||
errors.append(
|
||||
f'subdomain must match ^[a-z][a-z0-9-]{{0,30}}$, got: {subdomain}'
|
||||
)
|
||||
|
||||
backend = m.get('backend', '')
|
||||
if backend and not BACKEND_RE.match(backend):
|
||||
errors.append(f'backend must be host:port (e.g. cell-foo:8080), got: {backend}')
|
||||
|
||||
for sub in m.get('extra_subdomains') or []:
|
||||
if not isinstance(sub, str):
|
||||
errors.append('extra_subdomains entries must be strings')
|
||||
elif sub in RESERVED_SUBDOMAINS:
|
||||
errors.append(f'extra_subdomains entry is reserved: {sub}')
|
||||
elif not SUBDOMAIN_RE.match(sub):
|
||||
errors.append(
|
||||
f'extra_subdomains entry must match ^[a-z][a-z0-9-]{{0,30}}$, got: {sub}'
|
||||
)
|
||||
|
||||
for sub, bknd in (m.get('extra_backends') or {}).items():
|
||||
if not isinstance(sub, str) or not SUBDOMAIN_RE.match(sub):
|
||||
errors.append(
|
||||
f'extra_backends key must match ^[a-z][a-z0-9-]{{0,30}}$, got: {sub!r}'
|
||||
)
|
||||
elif sub in RESERVED_SUBDOMAINS:
|
||||
errors.append(f'extra_backends key is reserved: {sub}')
|
||||
if not isinstance(bknd, str) or not BACKEND_RE.match(bknd):
|
||||
errors.append(
|
||||
f'extra_backends[{sub!r}] value must be host:port, got: {bknd!r}'
|
||||
)
|
||||
|
||||
# Env value safety
|
||||
@@ -164,139 +233,30 @@ class ServiceStoreManager(BaseServiceManager):
|
||||
f'env[].value contains disallowed characters: {val!r}'
|
||||
)
|
||||
|
||||
# Security layer: delegate to manifest_validator for cap_add, backend
|
||||
# denylist, provision_hook, reserved container names, and kind guard.
|
||||
ok, sec_errs = validate_manifest(m)
|
||||
if not ok:
|
||||
errors.extend(sec_errs)
|
||||
|
||||
return (len(errors) == 0, errors)
|
||||
|
||||
# ── IP allocation ─────────────────────────────────────────────────────
|
||||
|
||||
def _allocate_service_ip(self, service_id: str) -> str:
|
||||
"""Allocate the next free IP from the service pool."""
|
||||
identity = self.config_manager.get_identity()
|
||||
ip_range = identity.get('ip_range', '172.20.0.0/16')
|
||||
|
||||
import ipaddress
|
||||
network = ipaddress.IPv4Network(ip_range, strict=False)
|
||||
base = int(network.network_address)
|
||||
|
||||
# IPs already assigned to named containers
|
||||
reserved_offsets = set(CONTAINER_OFFSETS.values())
|
||||
|
||||
# IPs already assigned to installed services
|
||||
service_ips: Dict[str, str] = identity.get('service_ips', {})
|
||||
taken_ips = set(service_ips.values())
|
||||
|
||||
for offset in range(SERVICE_POOL_START, SERVICE_POOL_END + 1):
|
||||
if offset in reserved_offsets:
|
||||
continue
|
||||
candidate = str(ipaddress.IPv4Address(base + offset))
|
||||
if candidate not in taken_ips:
|
||||
return candidate
|
||||
|
||||
raise RuntimeError('Service IP pool exhausted (offsets 20-254 all taken)')
|
||||
|
||||
# ── Compose override ──────────────────────────────────────────────────
|
||||
|
||||
def _render_compose_override(self, installed_records: dict) -> str:
|
||||
"""Generate docker-compose YAML override for all installed services."""
|
||||
services: Dict[str, Any] = {}
|
||||
|
||||
for svc_id, record in installed_records.items():
|
||||
manifest = record.get('manifest', {})
|
||||
container_name = record.get('container_name', svc_id)
|
||||
image = manifest.get('image', record.get('image', ''))
|
||||
service_ip = record.get('service_ip', '')
|
||||
|
||||
# Volumes
|
||||
volumes = []
|
||||
for vol in manifest.get('volumes', []):
|
||||
vol_name = vol.get('name', '')
|
||||
mount = vol.get('mount', '')
|
||||
if vol_name and mount:
|
||||
volumes.append(f'{vol_name}:{mount}')
|
||||
|
||||
# Environment
|
||||
environment: Dict[str, str] = {}
|
||||
for env_entry in manifest.get('env', []):
|
||||
k = env_entry.get('key', '')
|
||||
v = str(env_entry.get('value', ''))
|
||||
if k:
|
||||
environment[k] = v
|
||||
|
||||
svc_def: Dict[str, Any] = {
|
||||
'image': image,
|
||||
'container_name': container_name,
|
||||
'restart': 'unless-stopped',
|
||||
'logging': {
|
||||
'driver': 'json-file',
|
||||
'options': {
|
||||
'max-size': '10m',
|
||||
'max-file': '5',
|
||||
},
|
||||
},
|
||||
'networks': {
|
||||
'cell-network': {
|
||||
'ipv4_address': service_ip,
|
||||
}
|
||||
},
|
||||
}
|
||||
if volumes:
|
||||
svc_def['volumes'] = volumes
|
||||
if environment:
|
||||
svc_def['environment'] = environment
|
||||
|
||||
services[container_name] = svc_def
|
||||
|
||||
# Collect named volumes
|
||||
named_volumes: Dict[str, Any] = {}
|
||||
for svc_id, record in installed_records.items():
|
||||
manifest = record.get('manifest', {})
|
||||
for vol in manifest.get('volumes', []):
|
||||
vol_name = vol.get('name', '')
|
||||
if vol_name:
|
||||
named_volumes[vol_name] = None # Docker default driver
|
||||
|
||||
doc: Dict[str, Any] = {
|
||||
'version': '3.8',
|
||||
'services': services,
|
||||
'networks': {
|
||||
'cell-network': {
|
||||
'external': True,
|
||||
}
|
||||
},
|
||||
}
|
||||
if named_volumes:
|
||||
doc['volumes'] = named_volumes
|
||||
|
||||
return yaml.dump(doc, default_flow_style=False, allow_unicode=True)
|
||||
|
||||
def _write_compose_override(self, content: str) -> None:
|
||||
"""Atomic write of the compose override file."""
|
||||
tmp_path = self.compose_override + '.tmp'
|
||||
try:
|
||||
os.makedirs(os.path.dirname(os.path.abspath(self.compose_override)),
|
||||
exist_ok=True)
|
||||
except (PermissionError, OSError):
|
||||
pass
|
||||
with open(tmp_path, 'w') as f:
|
||||
f.write(content)
|
||||
f.flush()
|
||||
try:
|
||||
os.fsync(f.fileno())
|
||||
except OSError:
|
||||
pass
|
||||
os.replace(tmp_path, self.compose_override)
|
||||
|
||||
# ── Index / manifest fetching ─────────────────────────────────────────
|
||||
|
||||
def fetch_index(self) -> list:
|
||||
"""Fetch and cache the service index."""
|
||||
import time
|
||||
_SIZE_LIMIT = 256 * 1024
|
||||
now = time.time()
|
||||
if self._index_cache is not None and (now - self._index_cache_time) < self._cache_ttl:
|
||||
return self._index_cache
|
||||
try:
|
||||
resp = requests.get(self.index_url, timeout=10)
|
||||
resp = requests.get(self.index_url, timeout=10, stream=True)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
content = resp.raw.read(_SIZE_LIMIT + 1, decode_content=True)
|
||||
if len(content) > _SIZE_LIMIT:
|
||||
raise ValueError('Index response exceeds 256 KB limit')
|
||||
data = json.loads(content)
|
||||
self._index_cache = data if isinstance(data, list) else data.get('services', [])
|
||||
self._index_cache_time = now
|
||||
return self._index_cache
|
||||
@@ -306,19 +266,33 @@ class ServiceStoreManager(BaseServiceManager):
|
||||
|
||||
def _fetch_manifest(self, service_id: str) -> dict:
|
||||
"""Fetch a service manifest by ID."""
|
||||
_SIZE_LIMIT = 256 * 1024
|
||||
url = MANIFEST_URL_TPL.format(id=service_id)
|
||||
resp = requests.get(url, timeout=10)
|
||||
resp = requests.get(url, timeout=10, stream=True)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
content = resp.raw.read(_SIZE_LIMIT + 1, decode_content=True)
|
||||
if len(content) > _SIZE_LIMIT:
|
||||
raise ValueError(
|
||||
f'Manifest response for {service_id} exceeds 256 KB limit'
|
||||
)
|
||||
return json.loads(content)
|
||||
|
||||
def _fetch_template(self, service_id: str, manifest: dict) -> str:
|
||||
"""Fetch the compose template for a service."""
|
||||
_SIZE_LIMIT = 256 * 1024
|
||||
url = TEMPLATE_URL_TPL.format(id=service_id)
|
||||
resp = requests.get(url, timeout=10, stream=True)
|
||||
resp.raise_for_status()
|
||||
content = resp.raw.read(_SIZE_LIMIT + 1, decode_content=True)
|
||||
if len(content) > _SIZE_LIMIT:
|
||||
raise ValueError(f'Compose template for {service_id} exceeds 256 KB limit')
|
||||
return content.decode('utf-8')
|
||||
|
||||
# ── Core operations ───────────────────────────────────────────────────
|
||||
|
||||
def install(self, service_id: str) -> dict:
|
||||
"""Install a service from the store."""
|
||||
from firewall_manager import apply_service_rules
|
||||
|
||||
with self._lock:
|
||||
# Already installed?
|
||||
installed = self.config_manager.get_installed_services()
|
||||
if service_id in installed:
|
||||
return {'ok': True, 'already_installed': True}
|
||||
@@ -333,154 +307,92 @@ class ServiceStoreManager(BaseServiceManager):
|
||||
if not ok:
|
||||
return {'ok': False, 'errors': errs}
|
||||
|
||||
# Allocate IP
|
||||
try:
|
||||
ip = self._allocate_service_ip(service_id)
|
||||
except RuntimeError as e:
|
||||
return {'ok': False, 'error': str(e)}
|
||||
ok2, errs2 = validate_manifest(manifest)
|
||||
if not ok2:
|
||||
return {'ok': False, 'errors': errs2}
|
||||
|
||||
# Build install record
|
||||
# Dependency check
|
||||
if self.service_composer is not None:
|
||||
err = self.service_composer._resolve_requires(manifest, installed)
|
||||
if err:
|
||||
return {'ok': False, 'error': err}
|
||||
|
||||
# Fetch compose template
|
||||
try:
|
||||
template_content = self._fetch_template(service_id, manifest)
|
||||
except Exception as e:
|
||||
return {'ok': False, 'error': f'Failed to fetch compose template: {e}'}
|
||||
|
||||
# Write compose file and start containers (validation inside write_compose)
|
||||
if self.service_composer is not None:
|
||||
try:
|
||||
result = self.service_composer.install(service_id, manifest, template_content)
|
||||
except ValueError as e:
|
||||
return {'ok': False, 'error': str(e)}
|
||||
except Exception as e:
|
||||
return {'ok': False, 'error': f'Failed to start service: {e}'}
|
||||
if not result.get('ok'):
|
||||
return {'ok': False, 'error': result.get('error') or result.get('stderr', 'docker up failed')}
|
||||
|
||||
# Persist minimal install record
|
||||
record = {
|
||||
'id': service_id,
|
||||
'name': manifest.get('name', service_id),
|
||||
'container_name': manifest['container_name'],
|
||||
'image': manifest.get('image', ''),
|
||||
'service_ip': ip,
|
||||
'caddy_route': manifest.get('caddy_route'),
|
||||
'iptables_rules': manifest.get('iptables_rules', []),
|
||||
'manifest': manifest,
|
||||
'installed_at': datetime.utcnow().isoformat(),
|
||||
}
|
||||
|
||||
# Persist to config
|
||||
self.config_manager.set_installed_service(service_id, record)
|
||||
identity = self.config_manager.get_identity()
|
||||
service_ips = dict(identity.get('service_ips', {}))
|
||||
service_ips[service_id] = ip
|
||||
self.config_manager.set_identity_field('service_ips', service_ips)
|
||||
|
||||
# Write compose override
|
||||
all_installed = self.config_manager.get_installed_services()
|
||||
# Regenerate Caddy (registry now drives routes, no caddy_routes list needed)
|
||||
try:
|
||||
content = self._render_compose_override(all_installed)
|
||||
self._write_compose_override(content)
|
||||
self.caddy_manager.regenerate_with_installed([])
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to write compose override: {e}')
|
||||
logger.warning('install: caddy regenerate failed for %s (non-fatal): %s', service_id, e)
|
||||
|
||||
# Apply iptables rules (best-effort)
|
||||
try:
|
||||
apply_service_rules(service_id, ip, manifest.get('iptables_rules', []))
|
||||
except Exception as e:
|
||||
logger.warning(f'apply_service_rules for {service_id} failed (non-fatal): {e}')
|
||||
if self.egress_manager:
|
||||
try:
|
||||
self.egress_manager.apply_service(service_id)
|
||||
except Exception as exc:
|
||||
logger.warning('Egress apply failed for %s (non-fatal): %s', service_id, exc)
|
||||
|
||||
# Regenerate Caddyfile
|
||||
try:
|
||||
caddy_routes = [
|
||||
r.get('caddy_route')
|
||||
for r in all_installed.values()
|
||||
if r.get('caddy_route')
|
||||
]
|
||||
self.caddy_manager.regenerate_with_installed(caddy_routes)
|
||||
except Exception as e:
|
||||
logger.warning(f'caddy regenerate for {service_id} failed (non-fatal): {e}')
|
||||
|
||||
# Start the container via docker compose
|
||||
base_compose = os.environ.get('COMPOSE_FILE', '/app/docker-compose.yml')
|
||||
try:
|
||||
result = subprocess.run(
|
||||
['docker', 'compose',
|
||||
'-f', base_compose,
|
||||
'-f', self.compose_override,
|
||||
'up', '-d', manifest['container_name']],
|
||||
capture_output=True, text=True, timeout=120,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
logger.warning(
|
||||
f'docker compose up for {service_id} failed: {result.stderr.strip()}'
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f'docker compose up for {service_id} failed (non-fatal): {e}')
|
||||
|
||||
return {
|
||||
'ok': True,
|
||||
'service_ip': ip,
|
||||
'container_name': manifest['container_name'],
|
||||
}
|
||||
return {'ok': True}
|
||||
|
||||
def remove(self, service_id: str, purge_data: bool = False) -> dict:
|
||||
"""Remove an installed service."""
|
||||
from firewall_manager import clear_service_rules
|
||||
|
||||
with self._lock:
|
||||
installed = self.config_manager.get_installed_services()
|
||||
record = installed.get(service_id)
|
||||
if not record:
|
||||
if service_id not in installed:
|
||||
return {'ok': False, 'error': f'Service {service_id} is not installed'}
|
||||
|
||||
container_name = record.get('container_name', service_id)
|
||||
manifest = record.get('manifest', {})
|
||||
base_compose = os.environ.get('COMPOSE_FILE', '/app/docker-compose.yml')
|
||||
# Prevent removing a service that others depend on
|
||||
if self.service_composer is not None:
|
||||
dependents = self.service_composer._resolve_dependents(service_id, installed)
|
||||
if dependents:
|
||||
return {
|
||||
'ok': False,
|
||||
'error': f'Cannot remove {service_id}: required by {", ".join(sorted(dependents))}',
|
||||
}
|
||||
|
||||
# Stop and remove container
|
||||
try:
|
||||
subprocess.run(
|
||||
['docker', 'compose',
|
||||
'-f', base_compose,
|
||||
'-f', self.compose_override,
|
||||
'stop', container_name],
|
||||
capture_output=True, text=True, timeout=60,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f'docker compose stop for {service_id} failed (non-fatal): {e}')
|
||||
if self.egress_manager:
|
||||
try:
|
||||
self.egress_manager.clear_service(service_id)
|
||||
except Exception as exc:
|
||||
logger.warning('Egress clear failed for %s (non-fatal): %s', service_id, exc)
|
||||
|
||||
try:
|
||||
subprocess.run(
|
||||
['docker', 'rm', '-f', container_name],
|
||||
capture_output=True, text=True, timeout=30,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f'docker rm for {service_id} failed (non-fatal): {e}')
|
||||
# Stop and remove containers (best-effort)
|
||||
if self.service_composer is not None:
|
||||
try:
|
||||
self.service_composer.remove(service_id, purge_data=purge_data)
|
||||
except Exception as e:
|
||||
logger.warning('remove: composer.remove failed for %s (non-fatal): %s', service_id, e)
|
||||
|
||||
# Clear iptables rules
|
||||
try:
|
||||
clear_service_rules(service_id)
|
||||
except Exception as e:
|
||||
logger.warning(f'clear_service_rules for {service_id} failed (non-fatal): {e}')
|
||||
|
||||
# Remove from config, regenerate compose + caddy
|
||||
# Remove from config
|
||||
self.config_manager.remove_installed_service(service_id)
|
||||
remaining = self.config_manager.get_installed_services()
|
||||
|
||||
# Regenerate Caddy
|
||||
try:
|
||||
content = self._render_compose_override(remaining)
|
||||
self._write_compose_override(content)
|
||||
self.caddy_manager.regenerate_with_installed([])
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to write compose override after remove: {e}')
|
||||
|
||||
try:
|
||||
caddy_routes = [
|
||||
r.get('caddy_route')
|
||||
for r in remaining.values()
|
||||
if r.get('caddy_route')
|
||||
]
|
||||
self.caddy_manager.regenerate_with_installed(caddy_routes)
|
||||
except Exception as e:
|
||||
logger.warning(f'caddy regenerate after remove failed (non-fatal): {e}')
|
||||
|
||||
# Purge named volumes if requested
|
||||
if purge_data:
|
||||
for vol in manifest.get('volumes', []):
|
||||
vol_name = vol.get('name', '')
|
||||
if vol_name:
|
||||
try:
|
||||
subprocess.run(
|
||||
['docker', 'volume', 'rm', vol_name],
|
||||
capture_output=True, text=True, timeout=30,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f'docker volume rm {vol_name} failed (non-fatal): {e}'
|
||||
)
|
||||
logger.warning('remove: caddy regenerate failed for %s (non-fatal): %s', service_id, e)
|
||||
|
||||
return {'ok': True}
|
||||
|
||||
@@ -495,16 +407,22 @@ class ServiceStoreManager(BaseServiceManager):
|
||||
from firewall_manager import apply_service_rules
|
||||
|
||||
installed = self.config_manager.get_installed_services()
|
||||
|
||||
# Always regenerate the Caddyfile so a cell rename or fresh install
|
||||
# produces the correct domain even when no store services are installed.
|
||||
try:
|
||||
caddy_routes = [
|
||||
r.get('caddy_route')
|
||||
for r in (installed or {}).values()
|
||||
if r.get('caddy_route')
|
||||
]
|
||||
self.caddy_manager.regenerate_with_installed(caddy_routes)
|
||||
except Exception as e:
|
||||
logger.warning(f'reapply_on_startup: caddy regenerate failed: {e}')
|
||||
|
||||
if not installed:
|
||||
return
|
||||
|
||||
# Regenerate compose override in case it was deleted
|
||||
try:
|
||||
content = self._render_compose_override(installed)
|
||||
self._write_compose_override(content)
|
||||
except Exception as e:
|
||||
logger.warning(f'reapply_on_startup: compose override write failed: {e}')
|
||||
|
||||
# Re-apply iptables rules
|
||||
for svc_id, record in installed.items():
|
||||
ip = record.get('service_ip', '')
|
||||
@@ -514,13 +432,16 @@ class ServiceStoreManager(BaseServiceManager):
|
||||
except Exception as e:
|
||||
logger.warning(f'reapply_on_startup: apply_service_rules({svc_id}) failed: {e}')
|
||||
|
||||
# Regenerate Caddyfile
|
||||
try:
|
||||
caddy_routes = [
|
||||
r.get('caddy_route')
|
||||
for r in installed.values()
|
||||
if r.get('caddy_route')
|
||||
]
|
||||
self.caddy_manager.regenerate_with_installed(caddy_routes)
|
||||
except Exception as e:
|
||||
logger.warning(f'reapply_on_startup: caddy regenerate failed: {e}')
|
||||
# Bring up per-service compose stacks
|
||||
if self.service_composer is not None:
|
||||
try:
|
||||
self.service_composer.reapply_active_services()
|
||||
except Exception as e:
|
||||
logger.warning('reapply_on_startup: reapply_active_services failed: %s', e)
|
||||
|
||||
# Re-apply egress fwmark rules
|
||||
if self.egress_manager is not None:
|
||||
try:
|
||||
self.egress_manager.apply_all()
|
||||
except Exception as e:
|
||||
logger.warning('reapply_on_startup: egress apply_all failed: %s', e)
|
||||
|
||||
+95
-14
@@ -60,6 +60,37 @@ VALID_DOMAIN_MODES = {'pic_ngo', 'cloudflare', 'duckdns', 'http01', 'lan'}
|
||||
CELL_NAME_RE = re.compile(r'^[a-z][a-z0-9-]{1,30}$')
|
||||
|
||||
|
||||
DDNS_API_BASE = os.environ.get('DDNS_URL', 'https://ddns.pic.ngo/api/v1').rstrip('/').replace('/api/v1', '')
|
||||
DDNS_TOTP_SECRET = os.environ.get('DDNS_TOTP_SECRET', '')
|
||||
|
||||
|
||||
def _build_ddns_config(domain_mode: str, cloudflare_api_token: str = '',
|
||||
duckdns_token: str = '', duckdns_subdomain: str = '') -> dict:
|
||||
"""Return the top-level ddns config dict for a given domain mode."""
|
||||
if domain_mode == 'pic_ngo':
|
||||
return {
|
||||
'provider': 'pic_ngo',
|
||||
'api_base_url': DDNS_API_BASE,
|
||||
'totp_secret': DDNS_TOTP_SECRET,
|
||||
'enabled': True,
|
||||
}
|
||||
if domain_mode == 'cloudflare':
|
||||
cfg = {'provider': 'cloudflare', 'enabled': True}
|
||||
if cloudflare_api_token:
|
||||
cfg['api_token'] = cloudflare_api_token
|
||||
return cfg
|
||||
if domain_mode == 'duckdns':
|
||||
cfg = {'provider': 'duckdns', 'enabled': True}
|
||||
if duckdns_token:
|
||||
cfg['token'] = duckdns_token
|
||||
if duckdns_subdomain:
|
||||
cfg['subdomain'] = duckdns_subdomain
|
||||
return cfg
|
||||
if domain_mode == 'http01':
|
||||
return {'provider': 'http01', 'enabled': True}
|
||||
return {'provider': 'none', 'enabled': False}
|
||||
|
||||
|
||||
class SetupManager:
|
||||
"""Manages the first-run setup wizard state and completion."""
|
||||
|
||||
@@ -74,11 +105,22 @@ class SetupManager:
|
||||
return bool(self.config_manager.get_identity().get('setup_complete', False))
|
||||
|
||||
def get_setup_status(self) -> Dict[str, Any]:
|
||||
"""Return current setup status and wizard metadata."""
|
||||
"""Return current setup status, wizard metadata, and any pre-configured identity."""
|
||||
identity = self.config_manager.get_identity()
|
||||
preconfigured = {
|
||||
k: v for k, v in {
|
||||
'cell_name': identity.get('cell_name', ''),
|
||||
'domain_mode': identity.get('domain_mode', ''),
|
||||
'domain_name': identity.get('domain_name', ''),
|
||||
'cloudflare_api_token': identity.get('cloudflare_api_token', ''),
|
||||
'duckdns_token': identity.get('duckdns_token', ''),
|
||||
}.items() if v
|
||||
}
|
||||
return {
|
||||
'complete': self.is_setup_complete(),
|
||||
'available_services': AVAILABLE_SERVICES,
|
||||
'available_timezones': AVAILABLE_TIMEZONES,
|
||||
'preconfigured': preconfigured,
|
||||
}
|
||||
|
||||
# ── validation ────────────────────────────────────────────────────────
|
||||
@@ -128,9 +170,11 @@ class SetupManager:
|
||||
cell_name = payload.get('cell_name', '')
|
||||
password = payload.get('password', '')
|
||||
domain_mode = payload.get('domain_mode', '')
|
||||
domain_name = payload.get('domain_name', '')
|
||||
timezone = payload.get('timezone', '')
|
||||
services_enabled = payload.get('services_enabled', [])
|
||||
ddns_provider = payload.get('ddns_provider', 'none')
|
||||
cloudflare_api_token = payload.get('cloudflare_api_token', '')
|
||||
duckdns_token = payload.get('duckdns_token', '')
|
||||
|
||||
errors.extend(self.validate_cell_name(cell_name))
|
||||
errors.extend(self.validate_password(password))
|
||||
@@ -141,8 +185,6 @@ class SetupManager:
|
||||
)
|
||||
if not timezone or not isinstance(timezone, str):
|
||||
errors.append('timezone is required.')
|
||||
if not isinstance(services_enabled, list):
|
||||
errors.append('services_enabled must be a list.')
|
||||
|
||||
if errors:
|
||||
return {'success': False, 'errors': errors}
|
||||
@@ -168,35 +210,74 @@ class SetupManager:
|
||||
if self.is_setup_complete():
|
||||
return {'success': False, 'errors': ['Setup has already been completed.']}
|
||||
|
||||
# ── create admin user ──────────────────────────────────────────
|
||||
# ── create or update admin user ────────────────────────────────
|
||||
# The installer may have bootstrapped an admin account from a
|
||||
# generated password. The wizard's job is to set the real password,
|
||||
# so update it if the account already exists.
|
||||
ok = self.auth_manager.create_user(
|
||||
username='admin',
|
||||
password=password,
|
||||
role='admin',
|
||||
)
|
||||
if not ok:
|
||||
return {'success': False, 'errors': ['Failed to create admin user. The username may already exist.']}
|
||||
ok = self.auth_manager.set_password_admin('admin', password)
|
||||
if not ok:
|
||||
return {'success': False, 'errors': ['Failed to set admin password.']}
|
||||
|
||||
# ── persist identity fields ────────────────────────────────────
|
||||
self.config_manager.set_identity_field('cell_name', cell_name)
|
||||
self.config_manager.set_identity_field('domain_mode', domain_mode)
|
||||
if domain_name:
|
||||
self.config_manager.set_identity_field('domain_name', domain_name)
|
||||
self.config_manager.set_identity_field('timezone', timezone)
|
||||
self.config_manager.set_identity_field('services_enabled', services_enabled)
|
||||
self.config_manager.set_identity_field('ddns_provider', ddns_provider)
|
||||
if cloudflare_api_token:
|
||||
self.config_manager.set_identity_field('cloudflare_api_token', cloudflare_api_token)
|
||||
if duckdns_token:
|
||||
self.config_manager.set_identity_field('duckdns_token', duckdns_token)
|
||||
|
||||
# NOTE: DDNS registration is deferred to Phase 3.
|
||||
# For now we just store ddns_provider in config.
|
||||
logger.info(
|
||||
'DDNS registration skipped (Phase 1). '
|
||||
'DDNS registration will happen in Phase 3. '
|
||||
f'ddns_provider={ddns_provider!r} stored in identity config.'
|
||||
# ── write top-level ddns section so DDNSManager can find provider ──
|
||||
duckdns_sub = domain_name.replace('.duckdns.org', '') if domain_mode == 'duckdns' else ''
|
||||
ddns_cfg = _build_ddns_config(
|
||||
domain_mode,
|
||||
cloudflare_api_token=cloudflare_api_token,
|
||||
duckdns_token=duckdns_token,
|
||||
duckdns_subdomain=duckdns_sub,
|
||||
)
|
||||
self.config_manager.set_ddns_config(ddns_cfg)
|
||||
|
||||
# ── trigger DDNS registration for pic_ngo ─────────────────────────
|
||||
warnings: List[str] = []
|
||||
if domain_mode == 'pic_ngo':
|
||||
try:
|
||||
from ddns_manager import DDNSManager
|
||||
ddns_mgr = DDNSManager(self.config_manager)
|
||||
ddns_mgr.register(cell_name, '')
|
||||
logger.info(f'DDNS registered: {cell_name}.pic.ngo')
|
||||
except Exception as exc:
|
||||
msg = str(exc)
|
||||
logger.warning(f'DDNS registration failed: {msg}')
|
||||
if '409' in msg or 'taken' in msg.lower():
|
||||
warnings.append(
|
||||
f'The name "{cell_name}" is already registered on pic.ngo. '
|
||||
'HTTPS will not be active until you re-register: go to '
|
||||
'Settings → DDNS and click Re-register, or choose a different name.'
|
||||
)
|
||||
else:
|
||||
warnings.append(
|
||||
'DDNS registration could not be completed right now '
|
||||
f'({msg}). The cell will retry automatically. '
|
||||
'HTTPS will activate once registration succeeds.'
|
||||
)
|
||||
|
||||
# ── mark setup complete (must be last) ─────────────────────────
|
||||
self.config_manager.set_identity_field('setup_complete', True)
|
||||
|
||||
logger.info(f"Setup completed. cell_name={cell_name!r}, domain_mode={domain_mode!r}")
|
||||
return {'success': True, 'redirect': '/login'}
|
||||
result: Dict[str, Any] = {'success': True, 'redirect': '/login'}
|
||||
if warnings:
|
||||
result['warnings'] = warnings
|
||||
return result
|
||||
|
||||
finally:
|
||||
try:
|
||||
|
||||
@@ -155,7 +155,9 @@ class WireGuardManager(BaseServiceManager):
|
||||
f'iptables -t nat -A PREROUTING -i %i -d {server_ip} -p udp --dport 53 -j DNAT --to-destination {dns_ip}:53; '
|
||||
f'iptables -t nat -A PREROUTING -i %i -d {server_ip} -p tcp --dport 53 -j DNAT --to-destination {dns_ip}:53; '
|
||||
f'iptables -t nat -A PREROUTING -i %i -d {server_ip} -p tcp --dport 80 -j DNAT --to-destination {caddy_ip}:80; '
|
||||
f'iptables -t nat -A PREROUTING -i %i -d {server_ip} -p tcp --dport 443 -j DNAT --to-destination {caddy_ip}:443; '
|
||||
f'iptables -I FORWARD -i %i -o eth0 -p tcp --dport 80 -j ACCEPT; '
|
||||
f'iptables -I FORWARD -i %i -o eth0 -p tcp --dport 443 -j ACCEPT; '
|
||||
f'iptables -I FORWARD -i %i -o eth0 -p udp --dport 53 -j ACCEPT; '
|
||||
f'iptables -I FORWARD -i %i -o eth0 -p tcp --dport 53 -j ACCEPT; '
|
||||
f'iptables -I FORWARD -i eth0 -o %i -s 172.20.0.0/16 -j ACCEPT; '
|
||||
@@ -165,7 +167,9 @@ class WireGuardManager(BaseServiceManager):
|
||||
f'iptables -t nat -D PREROUTING -i %i -d {server_ip} -p udp --dport 53 -j DNAT --to-destination {dns_ip}:53 2>/dev/null || true; '
|
||||
f'iptables -t nat -D PREROUTING -i %i -d {server_ip} -p tcp --dport 53 -j DNAT --to-destination {dns_ip}:53 2>/dev/null || true; '
|
||||
f'iptables -t nat -D PREROUTING -i %i -d {server_ip} -p tcp --dport 80 -j DNAT --to-destination {caddy_ip}:80 2>/dev/null || true; '
|
||||
f'iptables -t nat -D PREROUTING -i %i -d {server_ip} -p tcp --dport 443 -j DNAT --to-destination {caddy_ip}:443 2>/dev/null || true; '
|
||||
f'iptables -D FORWARD -i %i -o eth0 -p tcp --dport 80 -j ACCEPT 2>/dev/null || true; '
|
||||
f'iptables -D FORWARD -i %i -o eth0 -p tcp --dport 443 -j ACCEPT 2>/dev/null || true; '
|
||||
f'iptables -D FORWARD -i %i -o eth0 -p udp --dport 53 -j ACCEPT 2>/dev/null || true; '
|
||||
f'iptables -D FORWARD -i %i -o eth0 -p tcp --dport 53 -j ACCEPT 2>/dev/null || true; '
|
||||
f'iptables -D FORWARD -i eth0 -o %i -s 172.20.0.0/16 -j ACCEPT 2>/dev/null || true; '
|
||||
@@ -194,11 +198,11 @@ class WireGuardManager(BaseServiceManager):
|
||||
t = token.strip()
|
||||
if not t.startswith('iptables'):
|
||||
return False
|
||||
# PREROUTING DNAT on ports 53 or 80 (scoped or unscoped — we replace both)
|
||||
if 'PREROUTING' in t and 'DNAT' in t and ('--dport 53' in t or '--dport 80' in t):
|
||||
# PREROUTING DNAT on ports 53, 80, or 443 (scoped or unscoped — we replace both)
|
||||
if 'PREROUTING' in t and 'DNAT' in t and ('--dport 53' in t or '--dport 80' in t or '--dport 443' in t):
|
||||
return True
|
||||
# FORWARD accept to eth0 for ports 53 or 80 (service traffic forwarding)
|
||||
if 'FORWARD' in t and '-o eth0' in t and ('--dport 53' in t or '--dport 80' in t):
|
||||
# FORWARD accept to eth0 for ports 53, 80, or 443 (service traffic forwarding)
|
||||
if 'FORWARD' in t and '-o eth0' in t and ('--dport 53' in t or '--dport 80' in t or '--dport 443' in t):
|
||||
return True
|
||||
# Docker-to-WG FORWARD: eth0 → wg0 for 172.20.0.0/16
|
||||
if 'FORWARD' in t and '-i eth0' in t and '172.20.0.0/16' in t:
|
||||
@@ -290,6 +294,8 @@ class WireGuardManager(BaseServiceManager):
|
||||
return self.generate_config()
|
||||
|
||||
def _write_config(self, content: str):
|
||||
if content and not content.endswith('\n'):
|
||||
content += '\n'
|
||||
with open(self._config_file(), 'w') as f:
|
||||
f.write(content)
|
||||
self._syncconf()
|
||||
@@ -801,12 +807,20 @@ class WireGuardManager(BaseServiceManager):
|
||||
"""Remove the [Peer] block matching public_key from wg0.conf."""
|
||||
try:
|
||||
content = self._read_config()
|
||||
# Split on blank lines between blocks
|
||||
raw_blocks = ('\n' + content).split('\n\n')
|
||||
# Normalise to ensure blank-line block separators before splitting.
|
||||
# Without this, a file written without trailing newline will merge
|
||||
# [Interface] and the first [Peer] into one block, and the filter
|
||||
# below would then delete [Interface] together with the peer.
|
||||
normalised = content.replace('\n[Peer]', '\n\n[Peer]')
|
||||
raw_blocks = ('\n' + normalised).split('\n\n')
|
||||
new_blocks = [
|
||||
b for b in raw_blocks
|
||||
if not (f'PublicKey = {public_key}' in b and '[Peer]' in b)
|
||||
]
|
||||
# Never write an empty file — that would destroy the [Interface] block.
|
||||
if not any('[Interface]' in b for b in new_blocks):
|
||||
logger.error('remove_peer: [Interface] block would be lost — aborting write')
|
||||
return False
|
||||
self._write_config('\n\n'.join(new_blocks).lstrip('\n'))
|
||||
return True
|
||||
except Exception as e:
|
||||
@@ -1080,11 +1094,14 @@ class WireGuardManager(BaseServiceManager):
|
||||
capture_output=True, text=True, timeout=5,
|
||||
)
|
||||
running = 'cell-wireguard' in result.stdout
|
||||
configured_addr = self._get_configured_address()
|
||||
return {
|
||||
'running': running,
|
||||
'status': 'online' if running else 'offline',
|
||||
'interface': 'wg0',
|
||||
'ip_info': {'address': SERVER_ADDRESS} if running else {},
|
||||
'listen_port': self._get_configured_port(),
|
||||
'address': configured_addr if running else None,
|
||||
'ip_info': {'address': configured_addr} if running else {},
|
||||
'peers_count': len(self.get_peers()),
|
||||
'timestamp': datetime.utcnow().isoformat(),
|
||||
}
|
||||
|
||||
@@ -1,36 +0,0 @@
|
||||
{
|
||||
"cell_name": "modified",
|
||||
"domain": "cell.local",
|
||||
"ip_range": "10.0.0.0/24",
|
||||
"network": {
|
||||
"dns_port": 53,
|
||||
"dhcp_range": "10.0.0.100-10.0.0.200",
|
||||
"ntp_servers": ["pool.ntp.org"]
|
||||
},
|
||||
"wireguard": {
|
||||
"port": 51820,
|
||||
"private_key": "test_key",
|
||||
"address": "10.0.0.1/24"
|
||||
},
|
||||
"email": {
|
||||
"domain": "cell.local",
|
||||
"smtp_port": 25,
|
||||
"imap_port": 143
|
||||
},
|
||||
"calendar": {
|
||||
"port": 5232,
|
||||
"data_dir": "/app/data/calendar"
|
||||
},
|
||||
"files": {
|
||||
"port": 8080,
|
||||
"data_dir": "/app/data/files"
|
||||
},
|
||||
"routing": {
|
||||
"nat_enabled": true,
|
||||
"firewall_enabled": true
|
||||
},
|
||||
"vault": {
|
||||
"ca_configured": true,
|
||||
"fernet_configured": true
|
||||
}
|
||||
}
|
||||
@@ -3,4 +3,4 @@ services: {}
|
||||
networks:
|
||||
cell-network:
|
||||
external: true
|
||||
name: pic_cell-network
|
||||
name: cell-network
|
||||
|
||||
+7
-117
@@ -51,7 +51,7 @@ services:
|
||||
dhcp:
|
||||
image: alpine:latest
|
||||
container_name: cell-dhcp
|
||||
profiles: ["full"]
|
||||
profiles: ["core", "full"]
|
||||
ports:
|
||||
- "${DHCP_PORT:-67}:67/udp"
|
||||
volumes:
|
||||
@@ -74,7 +74,7 @@ services:
|
||||
ntp:
|
||||
image: alpine:latest
|
||||
container_name: cell-ntp
|
||||
profiles: ["full"]
|
||||
profiles: ["core", "full"]
|
||||
ports:
|
||||
- "${NTP_PORT:-123}:123/udp"
|
||||
volumes:
|
||||
@@ -92,79 +92,6 @@ services:
|
||||
max-size: "10m"
|
||||
max-file: "5"
|
||||
|
||||
# Email Server - Postfix + Dovecot
|
||||
mail:
|
||||
image: mailserver/docker-mailserver:latest
|
||||
container_name: cell-mail
|
||||
profiles: ["full"]
|
||||
hostname: mail
|
||||
domainname: cell.local
|
||||
env_file: ./config/mail/mailserver.env
|
||||
ports:
|
||||
- "${MAIL_SMTP_PORT:-25}:25"
|
||||
- "${MAIL_SUBMISSION_PORT:-587}:587"
|
||||
- "${MAIL_IMAP_PORT:-993}:993"
|
||||
volumes:
|
||||
- ./data/maildata:/var/mail
|
||||
- ./data/mailstate:/var/mail-state
|
||||
- ./data/maillogs:/var/log/mail
|
||||
- ./config/mail/config:/tmp/docker-mailserver/
|
||||
- ./config/mail/ssl:/etc/letsencrypt
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
cell-network:
|
||||
ipv4_address: ${MAIL_IP:-172.20.0.6}
|
||||
cap_add:
|
||||
- NET_ADMIN
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "5"
|
||||
|
||||
# Calendar & Contacts - Radicale
|
||||
radicale:
|
||||
image: tomsquest/docker-radicale:latest
|
||||
container_name: cell-radicale
|
||||
profiles: ["full"]
|
||||
ports:
|
||||
- "127.0.0.1:${RADICALE_PORT:-5232}:5232"
|
||||
volumes:
|
||||
- ./config/radicale:/etc/radicale
|
||||
- ./data/radicale:/data
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
cell-network:
|
||||
ipv4_address: ${RADICALE_IP:-172.20.0.7}
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "5"
|
||||
|
||||
# File Storage - WebDAV
|
||||
webdav:
|
||||
image: bytemark/webdav:latest
|
||||
container_name: cell-webdav
|
||||
profiles: ["full"]
|
||||
ports:
|
||||
- "127.0.0.1:${WEBDAV_PORT:-8080}:80"
|
||||
environment:
|
||||
- AUTH_TYPE=Basic
|
||||
- USERNAME=${WEBDAV_USER:-admin}
|
||||
- PASSWORD=${WEBDAV_PASS}
|
||||
volumes:
|
||||
- ./data/files:/var/lib/dav
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
cell-network:
|
||||
ipv4_address: ${WEBDAV_IP:-172.20.0.8}
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "5"
|
||||
|
||||
# WireGuard VPN
|
||||
wireguard:
|
||||
image: linuxserver/wireguard:latest
|
||||
@@ -204,6 +131,9 @@ services:
|
||||
profiles: ["core", "full"]
|
||||
ports:
|
||||
- "127.0.0.1:${API_PORT:-3000}:3000"
|
||||
environment:
|
||||
- DDNS_URL=${DDNS_URL:-https://ddns.pic.ngo/api/v1}
|
||||
- DDNS_TOTP_SECRET=${DDNS_TOTP_SECRET:-S6UMA464YIKM74QHXWL5WELDIO3HFZ6K}
|
||||
volumes:
|
||||
- ./data/api:/app/data
|
||||
- ./data/dns:/app/data/dns
|
||||
@@ -248,47 +178,7 @@ services:
|
||||
max-size: "10m"
|
||||
max-file: "5"
|
||||
|
||||
# Webmail - RainLoop
|
||||
rainloop:
|
||||
image: hardware/rainloop
|
||||
container_name: cell-rainloop
|
||||
profiles: ["full"]
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
cell-network:
|
||||
ipv4_address: ${RAINLOOP_IP:-172.20.0.12}
|
||||
ports:
|
||||
- "127.0.0.1:${RAINLOOP_PORT:-8888}:8888"
|
||||
volumes:
|
||||
- ./data/rainloop:/rainloop/data
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "5"
|
||||
|
||||
# File Manager - FileGator
|
||||
filegator:
|
||||
image: filegator/filegator
|
||||
container_name: cell-filegator
|
||||
profiles: ["full"]
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
cell-network:
|
||||
ipv4_address: ${FILEGATOR_IP:-172.20.0.13}
|
||||
ports:
|
||||
- "127.0.0.1:${FILEGATOR_PORT:-8082}:8080"
|
||||
volumes:
|
||||
- ./data/filegator:/var/www/filegator/private
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "5"
|
||||
|
||||
networks:
|
||||
cell-network:
|
||||
driver: bridge
|
||||
ipam:
|
||||
config:
|
||||
- subnet: ${CELL_NETWORK:-172.20.0.0/16}
|
||||
name: cell-network
|
||||
external: true
|
||||
|
||||
@@ -1,389 +0,0 @@
|
||||
# Personal Internet Cell - Network Configuration Guide
|
||||
|
||||
This guide explains how to configure networking for the Personal Internet Cell to provide internet access to WireGuard VPN clients.
|
||||
|
||||
## Table of Contents
|
||||
|
||||
1. [Overview](#overview)
|
||||
2. [Network Architecture](#network-architecture)
|
||||
3. [Quick Setup](#quick-setup)
|
||||
4. [Detailed Configuration](#detailed-configuration)
|
||||
5. [Troubleshooting](#troubleshooting)
|
||||
6. [Advanced Configuration](#advanced-configuration)
|
||||
7. [Security Considerations](#security-considerations)
|
||||
|
||||
## Overview
|
||||
|
||||
The Personal Internet Cell provides a complete VPN solution with internet access. This requires proper configuration of:
|
||||
|
||||
- **IP Forwarding**: Allow traffic to pass through the server
|
||||
- **NAT (Network Address Translation)**: Translate private IPs to public IPs
|
||||
- **Routing**: Direct traffic from VPN clients to the internet
|
||||
- **Firewall Rules**: Control traffic flow and security
|
||||
|
||||
## Network Architecture
|
||||
|
||||
```
|
||||
Internet
|
||||
│
|
||||
▼
|
||||
[Host Server] (195.178.106.244)
|
||||
│
|
||||
├── [Docker Network] (172.20.0.0/16)
|
||||
│ └── [WireGuard Container] (cell-wireguard)
|
||||
│ └── [WireGuard Interface] (wg0: 10.0.0.1/24)
|
||||
│
|
||||
└── [VPN Clients] (10.0.0.2-10.0.0.254/24)
|
||||
└── [Internet Access via NAT]
|
||||
```
|
||||
|
||||
### Key Components
|
||||
|
||||
- **Host Interface**: `eth0` (or main network interface)
|
||||
- **WireGuard Interface**: `wg0` (10.0.0.1/24)
|
||||
- **Client Network**: `10.0.0.0/24`
|
||||
- **NAT Translation**: Client IPs → Host IP
|
||||
|
||||
## Quick Setup
|
||||
|
||||
### 1. Run the Network Configuration Script
|
||||
|
||||
```bash
|
||||
# Make the script executable (if not already done)
|
||||
chmod +x /opt/pic/scripts/setup-network.sh
|
||||
|
||||
# Run the configuration
|
||||
sudo /opt/pic/scripts/setup-network.sh setup
|
||||
```
|
||||
|
||||
### 2. Verify Configuration
|
||||
|
||||
```bash
|
||||
# Check status
|
||||
sudo /opt/pic/scripts/setup-network.sh status
|
||||
|
||||
# Test configuration
|
||||
sudo /opt/pic/scripts/setup-network.sh test
|
||||
```
|
||||
|
||||
### 3. Connect a VPN Client
|
||||
|
||||
Use the generated WireGuard configuration to connect a client. The client should now have internet access.
|
||||
|
||||
## Detailed Configuration
|
||||
|
||||
### IP Forwarding
|
||||
|
||||
IP forwarding allows the server to route packets between different network interfaces.
|
||||
|
||||
**Enable on Host:**
|
||||
```bash
|
||||
echo 'net.ipv4.ip_forward = 1' >> /etc/sysctl.conf
|
||||
sysctl -p
|
||||
```
|
||||
|
||||
**Enable in Container:**
|
||||
```bash
|
||||
docker exec cell-wireguard sh -c "echo 1 > /proc/sys/net/ipv4/ip_forward"
|
||||
```
|
||||
|
||||
### NAT Configuration
|
||||
|
||||
NAT (Network Address Translation) allows VPN clients to access the internet using the server's public IP.
|
||||
|
||||
**Container NAT Rules:**
|
||||
```bash
|
||||
# Allow forwarding for WireGuard traffic
|
||||
iptables -A FORWARD -i wg0 -j ACCEPT
|
||||
iptables -A FORWARD -o wg0 -j ACCEPT
|
||||
|
||||
# NAT rule for internet access
|
||||
iptables -t nat -A POSTROUTING -s 10.0.0.0/24 -o eth0 -j MASQUERADE
|
||||
```
|
||||
|
||||
**Host NAT Rules:**
|
||||
```bash
|
||||
# Allow traffic from WireGuard network
|
||||
iptables -t nat -A POSTROUTING -s 10.0.0.0/24 -o eth0 -j MASQUERADE
|
||||
iptables -A FORWARD -i wg0 -j ACCEPT
|
||||
iptables -A FORWARD -o wg0 -j ACCEPT
|
||||
```
|
||||
|
||||
### Routing Configuration
|
||||
|
||||
**WireGuard Interface Setup:**
|
||||
```bash
|
||||
# Create WireGuard interface
|
||||
ip link add dev wg0 type wireguard
|
||||
|
||||
# Set private key
|
||||
wg set wg0 private-key /path/to/private-key
|
||||
|
||||
# Set listen port
|
||||
wg set wg0 listen-port 51820
|
||||
|
||||
# Add IP address
|
||||
ip addr add 10.0.0.1/24 dev wg0
|
||||
|
||||
# Bring interface up
|
||||
ip link set wg0 up
|
||||
|
||||
# Add peers
|
||||
wg set wg0 peer <public-key> allowed-ips 10.0.0.2/32
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Common Issues
|
||||
|
||||
#### 1. VPN Connected but No Internet
|
||||
|
||||
**Symptoms:**
|
||||
- WireGuard shows connected
|
||||
- Can ping server (10.0.0.1)
|
||||
- Cannot access internet
|
||||
|
||||
**Solutions:**
|
||||
```bash
|
||||
# Check IP forwarding
|
||||
cat /proc/sys/net/ipv4/ip_forward
|
||||
# Should return 1
|
||||
|
||||
# Check NAT rules
|
||||
iptables -t nat -L POSTROUTING -n
|
||||
# Should show MASQUERADE rule for 10.0.0.0/24
|
||||
|
||||
# Check forwarding rules
|
||||
iptables -L FORWARD -n
|
||||
# Should show ACCEPT rules for wg0
|
||||
|
||||
# Restart network configuration
|
||||
sudo /opt/pic/scripts/setup-network.sh reset
|
||||
sudo /opt/pic/scripts/setup-network.sh setup
|
||||
```
|
||||
|
||||
#### 2. Cannot Connect to VPN
|
||||
|
||||
**Symptoms:**
|
||||
- WireGuard client cannot connect
|
||||
- No handshake in server logs
|
||||
|
||||
**Solutions:**
|
||||
```bash
|
||||
# Check WireGuard interface
|
||||
docker exec cell-wireguard wg show
|
||||
|
||||
# Check if port 51820 is open
|
||||
netstat -ulnp | grep 51820
|
||||
|
||||
# Check firewall rules
|
||||
ufw status
|
||||
iptables -L INPUT -n
|
||||
|
||||
# Check Docker port mapping
|
||||
docker port cell-wireguard
|
||||
```
|
||||
|
||||
#### 3. DNS Issues
|
||||
|
||||
**Symptoms:**
|
||||
- Can ping IP addresses
|
||||
- Cannot resolve domain names
|
||||
|
||||
**Solutions:**
|
||||
```bash
|
||||
# Check DNS configuration in client config
|
||||
# Should include: DNS = 8.8.8.8, 1.1.1.1
|
||||
|
||||
# Test DNS from container
|
||||
docker exec cell-wireguard nslookup google.com
|
||||
|
||||
# Check if DNS is being blocked
|
||||
docker exec cell-wireguard iptables -L -n | grep 53
|
||||
```
|
||||
|
||||
### Diagnostic Commands
|
||||
|
||||
```bash
|
||||
# Check network status
|
||||
sudo /opt/pic/scripts/setup-network.sh status
|
||||
|
||||
# Test connectivity from container
|
||||
docker exec cell-wireguard ping -c 3 8.8.8.8
|
||||
|
||||
# Check routing table
|
||||
docker exec cell-wireguard ip route show
|
||||
|
||||
# Check interface status
|
||||
docker exec cell-wireguard ip addr show wg0
|
||||
|
||||
# Check NAT rules
|
||||
docker exec cell-wireguard iptables -t nat -L -n
|
||||
|
||||
# Check forwarding rules
|
||||
docker exec cell-wireguard iptables -L FORWARD -n
|
||||
```
|
||||
|
||||
## Advanced Configuration
|
||||
|
||||
### Custom DNS Servers
|
||||
|
||||
To use custom DNS servers, modify the WireGuard client configuration:
|
||||
|
||||
```ini
|
||||
[Interface]
|
||||
PrivateKey = <private-key>
|
||||
Address = 10.0.0.2/32
|
||||
DNS = 1.1.1.1, 1.0.0.1, 8.8.8.8, 8.8.4.4
|
||||
|
||||
[Peer]
|
||||
PublicKey = <server-public-key>
|
||||
Endpoint = 195.178.106.244:51820
|
||||
AllowedIPs = 0.0.0.0/0
|
||||
PersistentKeepalive = 25
|
||||
```
|
||||
|
||||
### Split Tunneling
|
||||
|
||||
To allow only specific traffic through the VPN:
|
||||
|
||||
```ini
|
||||
[Peer]
|
||||
AllowedIPs = 10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16
|
||||
# Only route private networks through VPN
|
||||
```
|
||||
|
||||
### Port Forwarding
|
||||
|
||||
To forward specific ports to VPN clients:
|
||||
|
||||
```bash
|
||||
# Forward port 8080 to client 10.0.0.2
|
||||
iptables -t nat -A PREROUTING -p tcp --dport 8080 -j DNAT --to-destination 10.0.0.2:8080
|
||||
iptables -A FORWARD -p tcp -d 10.0.0.2 --dport 8080 -j ACCEPT
|
||||
```
|
||||
|
||||
### Bandwidth Limiting
|
||||
|
||||
To limit bandwidth for VPN clients:
|
||||
|
||||
```bash
|
||||
# Install tc (traffic control)
|
||||
apt-get install iproute2
|
||||
|
||||
# Limit client 10.0.0.2 to 1Mbps
|
||||
tc qdisc add dev wg0 root handle 1: htb default 30
|
||||
tc class add dev wg0 parent 1: classid 1:1 htb rate 1mbit
|
||||
tc class add dev wg0 parent 1:1 classid 1:10 htb rate 1mbit ceil 1mbit
|
||||
tc filter add dev wg0 protocol ip parent 1:0 prio 1 u32 match ip dst 10.0.0.2 flowid 1:10
|
||||
```
|
||||
|
||||
## Security Considerations
|
||||
|
||||
### Firewall Rules
|
||||
|
||||
**Basic Security Rules:**
|
||||
```bash
|
||||
# Drop invalid packets
|
||||
iptables -A INPUT -m conntrack --ctstate INVALID -j DROP
|
||||
|
||||
# Allow established connections
|
||||
iptables -A INPUT -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT
|
||||
|
||||
# Allow WireGuard traffic
|
||||
iptables -A INPUT -p udp --dport 51820 -j ACCEPT
|
||||
|
||||
# Allow SSH (if needed)
|
||||
iptables -A INPUT -p tcp --dport 22 -j ACCEPT
|
||||
|
||||
# Drop everything else
|
||||
iptables -A INPUT -j DROP
|
||||
```
|
||||
|
||||
### Client Isolation
|
||||
|
||||
To prevent clients from communicating with each other:
|
||||
|
||||
```bash
|
||||
# Block inter-client communication
|
||||
iptables -A FORWARD -i wg0 -o wg0 -j DROP
|
||||
```
|
||||
|
||||
### Logging
|
||||
|
||||
To log VPN traffic:
|
||||
|
||||
```bash
|
||||
# Log all WireGuard traffic
|
||||
iptables -A FORWARD -i wg0 -j LOG --log-prefix "WG-FORWARD: "
|
||||
iptables -A FORWARD -o wg0 -j LOG --log-prefix "WG-FORWARD: "
|
||||
|
||||
# Log NAT traffic
|
||||
iptables -t nat -A POSTROUTING -s 10.0.0.0/24 -j LOG --log-prefix "WG-NAT: "
|
||||
```
|
||||
|
||||
## Monitoring
|
||||
|
||||
### Real-time Monitoring
|
||||
|
||||
```bash
|
||||
# Monitor WireGuard connections
|
||||
watch -n 1 "docker exec cell-wireguard wg show"
|
||||
|
||||
# Monitor traffic
|
||||
watch -n 1 "docker exec cell-wireguard wg show wg0 transfer"
|
||||
|
||||
# Monitor NAT rules
|
||||
watch -n 1 "iptables -t nat -L POSTROUTING -n -v"
|
||||
```
|
||||
|
||||
### Log Analysis
|
||||
|
||||
```bash
|
||||
# Check system logs
|
||||
journalctl -u pic-network.service -f
|
||||
|
||||
# Check iptables logs
|
||||
tail -f /var/log/kern.log | grep WG-
|
||||
|
||||
# Check Docker logs
|
||||
docker logs cell-wireguard -f
|
||||
```
|
||||
|
||||
## Backup and Recovery
|
||||
|
||||
### Backup Configuration
|
||||
|
||||
```bash
|
||||
# Backup iptables rules
|
||||
iptables-save > /opt/pic/backups/iptables-backup-$(date +%Y%m%d).rules
|
||||
|
||||
# Backup WireGuard configuration
|
||||
cp /opt/pic/config/wireguard/wg_confs/wg0.conf /opt/pic/backups/wg0-backup-$(date +%Y%m%d).conf
|
||||
|
||||
# Backup network script
|
||||
cp /opt/pic/scripts/setup-network.sh /opt/pic/backups/setup-network-backup-$(date +%Y%m%d).sh
|
||||
```
|
||||
|
||||
### Restore Configuration
|
||||
|
||||
```bash
|
||||
# Restore iptables rules
|
||||
iptables-restore < /opt/pic/backups/iptables-backup-YYYYMMDD.rules
|
||||
|
||||
# Restore WireGuard configuration
|
||||
cp /opt/pic/backups/wg0-backup-YYYYMMDD.conf /opt/pic/config/wireguard/wg_confs/wg0.conf
|
||||
docker restart cell-wireguard
|
||||
```
|
||||
|
||||
## Support
|
||||
|
||||
If you encounter issues:
|
||||
|
||||
1. Check the troubleshooting section above
|
||||
2. Run the diagnostic commands
|
||||
3. Check the logs for error messages
|
||||
4. Verify your network configuration
|
||||
5. Test with a simple client configuration
|
||||
|
||||
For additional help, check the main Personal Internet Cell documentation or create an issue in the project repository.
|
||||
@@ -0,0 +1,741 @@
|
||||
# PIC Service Developer Guide
|
||||
|
||||
This guide is for developers who want to build services that integrate with Personal Internet Cell (PIC). It covers the manifest format, how PIC wires up routing, DNS, backup, and account provisioning for your service, and how to package and submit your work.
|
||||
|
||||
**Prerequisites:** you should be comfortable with Docker, Docker Compose, and basic Linux networking. You do not need to know Python to build a store service.
|
||||
|
||||
---
|
||||
|
||||
## Table of Contents
|
||||
|
||||
1. [What a PIC service is](#1-what-a-pic-service-is)
|
||||
2. [Manifest reference](#2-manifest-reference)
|
||||
3. [Compose template variables](#3-compose-template-variables)
|
||||
4. [Account provisioning interface](#4-account-provisioning-interface)
|
||||
5. [Backup integration](#5-backup-integration)
|
||||
6. [Egress routing](#6-egress-routing)
|
||||
7. [Quick-start example](#7-quick-start-example)
|
||||
8. [Reference implementations](#8-reference-implementations)
|
||||
9. [Submitting to the store](#9-submitting-to-the-store)
|
||||
|
||||
---
|
||||
|
||||
## 1. What a PIC service is
|
||||
|
||||
A PIC service is a Docker container (or a set of containers) that plugs into the PIC ecosystem through a single JSON file called the **manifest**. The manifest tells PIC everything it needs to know:
|
||||
|
||||
- How to route HTTPS traffic to the service through Caddy
|
||||
- What subdomains to expose
|
||||
- Which users get accounts on the service and what credentials they receive
|
||||
- Which paths to include in automated backups
|
||||
- Which outbound network interfaces the service is allowed to use
|
||||
|
||||
All PIC services are **store services** — optional packages installed by the cell admin from the `pic-services` catalog. PIC downloads the manifest, renders a per-service Docker Compose file, and starts the containers. The core PIC stack (DNS, DHCP, NTP, WireGuard, Caddy, API, WebUI) runs independently of any installed services.
|
||||
|
||||
The email, calendar, and files services (in `pic-services/services/`) are the reference implementations and show the full feature set. The `ServiceRegistry` in `api/service_registry.py` is the single source of truth for all installed services. `CaddyManager`, the backup system, and the peer services endpoint all read from it rather than from hardcoded lists.
|
||||
|
||||
---
|
||||
|
||||
## 2. Manifest reference
|
||||
|
||||
The manifest is a JSON file with `"schema_version": 3`. Every field is described below. The `email`, `calendar`, and `files` manifests in `pic-services/services/` are the canonical reference examples.
|
||||
|
||||
### Top-level identity fields
|
||||
|
||||
| Field | Type | Required | Description |
|
||||
|---|---|---|---|
|
||||
| `schema_version` | integer | yes | Must be `3`. |
|
||||
| `id` | string | yes | Unique service identifier, lowercase, no spaces (e.g. `"notes"`). Must match the directory name for builtins, or the store index entry for store services. |
|
||||
| `name` | string | yes | Human-readable display name (e.g. `"Notes"`). |
|
||||
| `description` | string | yes | One-sentence description shown in the UI. |
|
||||
| `version` | string | yes | Semver string for the service package itself (e.g. `"1.0.0"`). |
|
||||
| `author` | string | yes | Your name or organisation. |
|
||||
| `kind` | string | yes | Must be `"store"`. |
|
||||
| `min_pic_version` | string | no | Minimum PIC version required (e.g. `"1.0"`). |
|
||||
|
||||
```json
|
||||
{
|
||||
"schema_version": 3,
|
||||
"id": "notes",
|
||||
"name": "Notes",
|
||||
"description": "Self-hosted Markdown notes with full-text search",
|
||||
"version": "1.0.0",
|
||||
"author": "acme",
|
||||
"kind": "store",
|
||||
"min_pic_version": "1.0"
|
||||
}
|
||||
```
|
||||
|
||||
### `capabilities`
|
||||
|
||||
A set of boolean flags that tell PIC which integrations to activate for your service.
|
||||
|
||||
| Field | Type | Default | Description |
|
||||
|---|---|---|---|
|
||||
| `has_subdomain` | bool | `false` | The service gets a subdomain and a Caddy reverse-proxy route. Requires `subdomain` and `backend`. |
|
||||
| `has_accounts` | bool | `false` | The service provisions per-peer accounts. Requires `accounts`. |
|
||||
| `has_admin_config` | bool | `false` | The service has admin-configurable fields. Requires `config_schema`. |
|
||||
| `has_storage` | bool | `false` | The service has data worth backing up. Requires `backup`. |
|
||||
| `has_egress` | bool | `false` | The admin can choose which outbound interface this service uses. Requires `egress`. |
|
||||
| `has_api_hooks` | bool | `false` | Reserved for future use; set `false`. |
|
||||
|
||||
```json
|
||||
"capabilities": {
|
||||
"has_subdomain": true,
|
||||
"has_accounts": true,
|
||||
"has_admin_config": false,
|
||||
"has_storage": true,
|
||||
"has_egress": false,
|
||||
"has_api_hooks": false
|
||||
}
|
||||
```
|
||||
|
||||
### `subdomain`, `extra_subdomains`, `backend`, `extra_backends`
|
||||
|
||||
These fields are only read when `has_subdomain` is `true`.
|
||||
|
||||
| Field | Type | Required | Description |
|
||||
|---|---|---|---|
|
||||
| `subdomain` | string | yes (if `has_subdomain`) | The primary subdomain (e.g. `"notes"`). Results in `notes.<cell-domain>`. Must not collide with reserved names: `api`, `webui`, `admin`, `www`, `ns1`, `ns2`, `git`, `registry`, `install`. |
|
||||
| `extra_subdomains` | array of strings | no | Additional subdomains that point to the same backend (e.g. `["webmail"]`). |
|
||||
| `backend` | string | yes (if `has_subdomain`) | The container-name:port combination that Caddy proxies to (e.g. `"cell-notes:8080"`). Uses Docker DNS on the `cell-network`. |
|
||||
| `extra_backends` | object | no | Maps extra subdomain names to separate backends. Key is the subdomain string; value is the backend string. The email service uses this to send `webdav.*` to a different container than `files.*`. |
|
||||
|
||||
```json
|
||||
"subdomain": "notes",
|
||||
"extra_subdomains": [],
|
||||
"backend": "cell-notes:8080"
|
||||
```
|
||||
|
||||
**Validation at runtime:** `ServiceRegistry.get_caddy_routes()` validates all subdomain and backend values before passing them to CaddyManager or NetworkManager. Any entry whose `subdomain` does not match `^[a-z][a-z0-9-]{0,30}$`, whose `backend` does not match `^[A-Za-z0-9._-]+:\d{1,5}$`, or whose `subdomain` appears in the reserved list is silently skipped with a warning log. The same validation applies to `extra_subdomains` and `extra_backends` keys/values. For store services, this validation is also performed during installation by `ServiceStoreManager._validate_manifest()`.
|
||||
|
||||
### `containers`
|
||||
|
||||
Array of container names that belong to this service. Used by the UI and log viewer. For builtins this is informational; for store services PIC only manages the single container declared in the manifest.
|
||||
|
||||
```json
|
||||
"containers": ["cell-notes"]
|
||||
```
|
||||
|
||||
### `config_schema`
|
||||
|
||||
Defines admin-configurable fields for this service. When `has_admin_config` is `true`, the UI renders a settings form from this schema. PIC stores admin-saved values in `cell_config.json` and merges them with your `default` values at runtime. The merged result is available as the `config` key when `ServiceRegistry.get()` returns your service.
|
||||
|
||||
Each field is an object:
|
||||
|
||||
| Key | Type | Required | Description |
|
||||
|---|---|---|---|
|
||||
| `type` | string | yes | One of `"string"`, `"integer"`, `"boolean"`. |
|
||||
| `label` | string | yes | Human-readable label for the settings form. |
|
||||
| `required` | bool | no | Whether the field must have a value before the service starts. |
|
||||
| `default` | any | no | Default value used when the admin has not set one. |
|
||||
| `min` / `max` | integer | no (integer only) | Inclusive bounds for integer fields. |
|
||||
|
||||
```json
|
||||
"config_schema": {
|
||||
"port": {
|
||||
"type": "integer",
|
||||
"label": "Internal HTTP port",
|
||||
"default": 8080,
|
||||
"min": 1,
|
||||
"max": 65535
|
||||
},
|
||||
"storage_path": {
|
||||
"type": "string",
|
||||
"label": "Data directory inside container",
|
||||
"default": "/data/notes"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### `peer_config_template`
|
||||
|
||||
When a peer is provisioned on this service, PIC fills this template and returns the result to the peer as their connection info. Template substitution tokens:
|
||||
|
||||
| Token | Replaced with |
|
||||
|---|---|
|
||||
| `{domain}` | The cell's public domain (e.g. `alice.pic.ngo`) |
|
||||
| `{peer.username}` | The peer's username |
|
||||
| `{peer.service_credentials.<id>.<field>}` | A credential value; `<id>` is the service `id`, `<field>` matches a name in `accounts.credentials` |
|
||||
| `{config.<key>}` | A value from the merged `config_schema` result |
|
||||
|
||||
```json
|
||||
"peer_config_template": {
|
||||
"url": "https://notes.{domain}/",
|
||||
"username": "{peer.username}",
|
||||
"password": "{peer.service_credentials.notes.password}"
|
||||
}
|
||||
```
|
||||
|
||||
### `accounts`
|
||||
|
||||
Required when `has_accounts` is `true`.
|
||||
|
||||
| Field | Type | Description |
|
||||
|---|---|---|
|
||||
| `manager` | string | Set to `"http"` for store services — PIC will call your container's HTTP API for account operations (see section 4). The reference services (`email`, `calendar`, `files`) use internal manager names (`"email_manager"`, `"calendar_manager"`, `"file_manager"`). |
|
||||
| `credentials` | array of strings | Names of credential fields this service issues per peer. Most services use `["password"]`. The names appear in `peer_config_template` tokens. |
|
||||
|
||||
```json
|
||||
"accounts": {
|
||||
"manager": "http",
|
||||
"credentials": ["password"]
|
||||
}
|
||||
```
|
||||
|
||||
### `compose`
|
||||
|
||||
Unused at the manifest level. Compose configuration is provided via `compose-template.yml` in the service package (see section 3). Set to `null` in the manifest.
|
||||
|
||||
### `backup`
|
||||
|
||||
Required when `has_storage` is `true`. Tells PIC's backup system what to snapshot.
|
||||
|
||||
| Field | Type | Description |
|
||||
|---|---|---|
|
||||
| `volumes` | array of objects | Container paths to stream out via `docker exec tar`. Each entry has three string fields: `container` (container name), `path` (absolute path inside the container), and `name` (archive filename stem). |
|
||||
| `config_paths` | array of strings | Paths relative to the PIC project root on the host that contain service configuration (not user data). Copied directly into the snapshot. |
|
||||
|
||||
Each entry in `volumes` produces an archive at `<name>.tar.gz` inside the snapshot. For example, `"name": "maildata"` produces `maildata.tar.gz`.
|
||||
|
||||
```json
|
||||
"backup": {
|
||||
"volumes": [
|
||||
{"container": "cell-notes", "path": "/data/notes", "name": "notes_data"}
|
||||
],
|
||||
"config_paths": ["config/notes"]
|
||||
}
|
||||
```
|
||||
|
||||
`ServiceRegistry.get_backup_plan()` aggregates these declarations across all installed services. The backup runner reads from that method rather than from any hardcoded list.
|
||||
|
||||
### `egress`
|
||||
|
||||
Required when `has_egress` is `true`. Declares which outbound network interfaces this service is permitted to use.
|
||||
|
||||
| Field | Type | Description |
|
||||
|---|---|---|
|
||||
| `default` | string | The interface selected when the admin has not changed anything. |
|
||||
| `allowed` | array of strings | The complete set of interfaces the admin can choose from. |
|
||||
|
||||
Valid interface identifiers: `default`, `wireguard_ext`, `openvpn`, `tor`.
|
||||
|
||||
```json
|
||||
"egress": {
|
||||
"default": "default",
|
||||
"allowed": ["default", "wireguard_ext", "openvpn", "tor"]
|
||||
}
|
||||
```
|
||||
|
||||
How enforcement works is described in section 6.
|
||||
|
||||
### `storage`
|
||||
|
||||
Informational metadata used by the UI to show storage usage.
|
||||
|
||||
| Field | Type | Description |
|
||||
|---|---|---|
|
||||
| `primary_path` | string | The path (relative to project root) that holds the bulk of user data. |
|
||||
| `quota_mb` | integer or null | Storage quota in megabytes; `null` means no limit. |
|
||||
|
||||
```json
|
||||
"storage": {
|
||||
"primary_path": "data/notes",
|
||||
"quota_mb": null
|
||||
}
|
||||
```
|
||||
|
||||
### Store-only manifest fields
|
||||
|
||||
Store services (where `kind` is `"store"`) have additional required fields that builtins do not use. These are validated by `ServiceStoreManager._validate_manifest()` before installation is permitted.
|
||||
|
||||
| Field | Type | Required | Description |
|
||||
|---|---|---|---|
|
||||
| `image` | string | yes | Docker image to pull. Must match the pattern `git.pic.ngo/roof/*`. Images from other registries are rejected. |
|
||||
| `container_name` | string | yes | The name Docker gives the running container. |
|
||||
| `volumes` | array | no | Named volumes to mount. Each entry must have `name` (the volume name) and `mount` (the absolute path inside the container). Mounts to `/`, `/etc`, `/var`, `/proc`, `/sys`, `/dev`, `/app`, `/run`, `/boot`, and paths that are a prefix of the PIC project root are forbidden. |
|
||||
| `env` | array | no | Environment variables to pass. Each entry has `key` and `value`. Values must match `^[A-Za-z0-9._@:/+\-= ]*$`. |
|
||||
| `iptables_rules` | array | no | FORWARD ACCEPT rules PIC should install in `cell-wireguard`. Each rule must have `type: "ACCEPT"`, `dest_ip: "${SERVICE_IP}"`, an integer `dest_port` (1–65535), and an optional `proto` (`"tcp"` or `"udp"`, default `"tcp"`). The literal string `${SERVICE_IP}` is replaced with the allocated container IP at install time. |
|
||||
| `caddy_route` | object | no | If the service exposes a web UI, provide `subdomain` (must not be reserved; must match `^[a-z][a-z0-9-]{0,30}$`). PIC inserts the corresponding `reverse_proxy` directive into the Caddyfile. |
|
||||
|
||||
---
|
||||
|
||||
## 3. Compose template variables
|
||||
|
||||
This section applies only to store services. Builtins define their containers directly in `docker-compose.yml`.
|
||||
|
||||
When you ship a store service, you include a `compose-template.yml` alongside your `manifest.json`. `ServiceComposer.render_template()` substitutes the variables below before writing the per-service `docker-compose.yml`.
|
||||
|
||||
| Variable | Syntax | Value |
|
||||
|---|---|---|
|
||||
| `${PIC_CFG_<KEY>}` | uppercase `config_schema` key | The admin-saved value for that field, or the `default` from the schema if the admin has not set it. For example, `config_schema.port` → `${PIC_CFG_PORT}`. |
|
||||
| `${PIC_SECRET_<NAME>}` | any name you choose | An auto-generated random secret produced by `secrets.token_urlsafe(24)` (~32 URL-safe base64 characters). Generated once on first install, then reused unchanged on every reconfigure. Stored per service in `data/service_secrets.json`. |
|
||||
| `${PIC_DOMAIN}` | literal | Effective domain from `ConfigManager` (e.g. `alice.pic.ngo`). |
|
||||
| `${PIC_CELL_NAME}` | literal | Cell name from the identity config (e.g. `alice`). |
|
||||
| `${PIC_SERVICE_ID}` | literal | The `id` field from the service manifest (e.g. `notes`). |
|
||||
|
||||
**Volume mounts**: Because docker compose runs inside the API container but the Docker daemon runs on the host, relative volume paths in compose templates resolve relative to the compose file's directory as seen by the HOST filesystem. To avoid path resolution surprises, prefer **named volumes** for service data (Docker manages them independently). If bind mounts are required, use absolute host paths with `${PIC_PROJECT_DIR}` once that variable is implemented, or document the expected host layout clearly.
|
||||
|
||||
Example `compose-template.yml` for a notes service:
|
||||
|
||||
```yaml
|
||||
services:
|
||||
cell-notes:
|
||||
image: git.pic.ngo/roof/pic-notes:latest
|
||||
container_name: cell-notes
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
NOTES_PORT: "${PIC_CFG_PORT}"
|
||||
NOTES_DOMAIN: "${PIC_DOMAIN}"
|
||||
NOTES_DB_PASS: "${PIC_SECRET_DB_PASSWORD}"
|
||||
volumes:
|
||||
- notes-data:/data/notes
|
||||
networks:
|
||||
cell-network:
|
||||
ipv4_address: "${SERVICE_IP}"
|
||||
|
||||
volumes:
|
||||
notes-data:
|
||||
|
||||
networks:
|
||||
cell-network:
|
||||
external: true
|
||||
```
|
||||
|
||||
The `SERVICE_IP` variable is the IP PIC allocated from the service pool. It is always set automatically.
|
||||
|
||||
---
|
||||
|
||||
## 4. Account provisioning interface
|
||||
|
||||
This section covers two related things: the `AccountManager` class that is PIC's central credential dispatcher, and the HTTP API that store services must implement to receive account operations.
|
||||
|
||||
### How AccountManager works
|
||||
|
||||
`AccountManager` (`api/account_manager.py`) is the single entry point for all account operations across every service type. It is instantiated once in `api/managers.py` and holds references to the service managers used by the reference services (`email_manager`, `calendar_manager`, `file_manager`).
|
||||
|
||||
When a peer account is provisioned, `AccountManager`:
|
||||
|
||||
1. Looks up the service in `ServiceRegistry` and reads `accounts.manager` from the manifest.
|
||||
2. Dispatches to the appropriate internal manager method (for builtins) or to the service's HTTP API endpoint (for store services — not yet implemented; `"http"` manager support is planned).
|
||||
3. Stores the returned credentials in `data/peer_service_credentials.json` with permissions `0o600`.
|
||||
|
||||
Credentials are stored in plaintext. This is intentional: the peer credentials endpoint needs to return them verbatim for one-time client configuration. The `0o600` permission matches the pattern used for WireGuard keys and `data/service_secrets.json`.
|
||||
|
||||
The credentials file structure is:
|
||||
|
||||
```json
|
||||
{
|
||||
"<service_id>": {
|
||||
"<peer_username>": { "password": "..." }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Writes use a write-then-rename pattern (`tmp` → final path) with `os.fsync` to avoid partial-write corruption.
|
||||
|
||||
### Manifest `accounts` field
|
||||
|
||||
The `accounts` block in the manifest wires a service into `AccountManager`.
|
||||
|
||||
| Field | Type | Description |
|
||||
|---|---|---|
|
||||
| `manager` | string | Which underlying manager handles account operations. For builtins: `"email_manager"`, `"calendar_manager"`, or `"file_manager"`. |
|
||||
| `credentials` | array of strings | Names of the credential fields this service issues per peer. Most services use `["password"]`. These names are used as token keys in `peer_config_template`. |
|
||||
|
||||
```json
|
||||
"accounts": {
|
||||
"manager": "email_manager",
|
||||
"credentials": ["password"]
|
||||
}
|
||||
```
|
||||
|
||||
The `manager` value must match a key that `AccountManager` was instantiated with. If the manager name has no registered dispatch entry, `provision()` raises `ValueError` immediately.
|
||||
|
||||
### Provision flow
|
||||
|
||||
```
|
||||
POST /api/services/catalog/<service_id>/accounts
|
||||
Content-Type: application/json
|
||||
|
||||
{ "username": "alice", "password": "optional" }
|
||||
```
|
||||
|
||||
If `password` is omitted, `AccountManager` generates one with `secrets.token_urlsafe(16)`. The response on HTTP 201 is:
|
||||
|
||||
```json
|
||||
{ "service_id": "email", "username": "alice", "provisioned": true }
|
||||
```
|
||||
|
||||
The password is not echoed in the response. To retrieve stored credentials for a provisioned peer, call `GET /api/services/catalog/<id>/accounts/<username>/credentials`.
|
||||
|
||||
Internally, `AccountManager.provision(service_id, peer_username, password)`:
|
||||
|
||||
1. Resolves the service and its manager via `_resolve_service()`.
|
||||
2. Calls the appropriate `_provision_*` method, which delegates to the concrete manager:
|
||||
- `email_manager` → `create_email_user(username, domain, password)`
|
||||
- `calendar_manager` → `create_calendar_user(username, password)`
|
||||
- `file_manager` → `create_user(username, password)`
|
||||
3. Stores `{"password": "<value>"}` under `[service_id][peer_username]` in the credentials file.
|
||||
4. Returns the credential dict to the caller.
|
||||
|
||||
If the underlying manager call returns `False`, `provision()` raises `RuntimeError`. The route handler maps this to HTTP 500.
|
||||
|
||||
For email, the domain is read from the service's merged config (`svc['config']['domain']`). If that key is absent, provisioning raises `ValueError` before calling the manager.
|
||||
|
||||
### Deprovision flow
|
||||
|
||||
```
|
||||
DELETE /api/services/catalog/<service_id>/accounts/<username>
|
||||
```
|
||||
|
||||
`AccountManager.deprovision(service_id, peer_username)`:
|
||||
|
||||
1. Calls the appropriate `_deprovision_*` method on the underlying manager.
|
||||
2. Removes the peer's entry from the credentials file. If that leaves the service block empty, the service block itself is removed.
|
||||
3. Returns `True` if the underlying call succeeded.
|
||||
|
||||
The route returns HTTP 200 with `{"message": "..."}` on success, or HTTP 400 if the service does not exist or does not support accounts.
|
||||
|
||||
**Peer deletion** calls `AccountManager.deprovision_peer(peer_username)`, which iterates over every service the peer is provisioned on and calls `deprovision()` for each. Failures on individual services are logged and skipped rather than aborting the deletion — the method returns `{service_id: bool}` for every service attempted.
|
||||
|
||||
### PIC admin API endpoints for account management
|
||||
|
||||
These endpoints are in `api/routes/services.py` and `api/routes/peers.py`.
|
||||
|
||||
| Method | Path | Description |
|
||||
|---|---|---|
|
||||
| `GET` | `/api/services/catalog/<service_id>/accounts` | Return `{"service_id": "...", "accounts": ["alice", "bob"]}` — reads directly from the credentials file. |
|
||||
| `POST` | `/api/services/catalog/<service_id>/accounts` | Provision a peer account. Body: `{"username": "...", "password": "..."}` (password optional). Returns HTTP 201 with `{"service_id", "username", "provisioned": true}`. |
|
||||
| `DELETE` | `/api/services/catalog/<service_id>/accounts/<username>` | Deprovision the peer's account. Returns HTTP 200 on success, HTTP 400 if the service or username is unknown. |
|
||||
| `GET` | `/api/services/catalog/<service_id>/accounts/<username>/credentials` | Return stored credentials for one peer+service pair. Returns HTTP 404 if the peer is not provisioned on that service. Response: `{"service_id", "username", "password"}`. |
|
||||
| `GET` | `/api/peers/<peer_name>/service-credentials` | Return filled `peer_config_template` values for all services the peer is provisioned on (see below). |
|
||||
|
||||
**Admin UI:** The Email, Calendar, and Files service pages in the admin dashboard each have an **Accounts** tab. From there, admins can provision and deprovision peer accounts, and reveal stored credentials for a provisioned peer. This tab calls the same API endpoints listed above.
|
||||
|
||||
### How `peer_config_template` connects to stored credentials
|
||||
|
||||
`GET /api/peers/<peer_name>/service-credentials` is the endpoint a peer device calls during first-time setup to configure email, CalDAV, and file sync clients.
|
||||
|
||||
The route:
|
||||
|
||||
1. Calls `AccountManager.get_all_credentials(peer_name)` → `{service_id: {field: value}}`.
|
||||
2. For each service, calls `ServiceRegistry.get_peer_service_info(service_id, peer_name, domain, cred)`.
|
||||
3. `get_peer_service_info` iterates over `peer_config_template` and replaces tokens:
|
||||
- `{domain}` → effective cell domain
|
||||
- `{peer.username}` → URL-percent-encoded peer username (safe='')
|
||||
- `{peer.service_credentials.<service_id>.<field>}` → the value from stored credentials
|
||||
- `{config.<key>}` → value from the service's merged config schema
|
||||
4. Returns the filled template dict as the value for that service in the response.
|
||||
|
||||
Response shape:
|
||||
|
||||
```json
|
||||
{
|
||||
"peer": "alice",
|
||||
"services": {
|
||||
"email": {
|
||||
"imap_host": "mail.alice.pic.ngo",
|
||||
"username": "alice@alice.pic.ngo",
|
||||
"password": "<stored>"
|
||||
},
|
||||
"files": {
|
||||
"url": "https://files.alice.pic.ngo/dav/alice/",
|
||||
"username": "alice",
|
||||
"password": "<stored>"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
If a service has no `peer_config_template` in its manifest, `get_peer_service_info` returns `None` and the raw credential dict is used as the fallback.
|
||||
|
||||
### Container lifecycle routes
|
||||
|
||||
The following PIC API endpoints are available for all services (builtins and store services). These are called by the web UI and can be called directly from the PIC admin API.
|
||||
|
||||
| Method | Path | Description |
|
||||
|---|---|---|
|
||||
| `GET` | `/api/services/catalog/<id>/status` | Return container status. Builtins query the main compose stack; store services query their own compose project. Response includes a `containers` array with one entry per container. |
|
||||
| `POST` | `/api/services/catalog/<id>/restart` | Restart the service containers. Builtins restart via the main compose stack; store services restart via their own compose project. |
|
||||
| `POST` | `/api/services/catalog/<id>/reconfigure` | Re-render the compose file from the template and re-apply with `up -d` (rolling update). Store services only — builtins are reconfigured through their own settings routes. The request body must include a `compose_template` field containing the new template content. |
|
||||
|
||||
### Store service HTTP API
|
||||
|
||||
When `accounts.manager` is `"http"`, PIC will call your container's HTTP API for account operations. **HTTP dispatch is not yet wired up in `AccountManager`** — the current dispatch table covers only `email_manager`, `calendar_manager`, and `file_manager` (used by the reference services). Implement this interface now so your service is ready when HTTP dispatch ships.
|
||||
|
||||
The base path is `/service-api/accounts` on your container's internal address. There is no authentication on this API — it is reachable only from within the `cell-network` Docker network.
|
||||
|
||||
**Create account**
|
||||
|
||||
```
|
||||
POST /service-api/accounts
|
||||
Content-Type: application/json
|
||||
|
||||
{ "username": "alice", "password": "auto-generated-by-pic" }
|
||||
```
|
||||
|
||||
PIC generates the password and passes it to your service. Return HTTP 200 with `{"ok": true}` on success. Return HTTP 400 or 409 with `{"ok": false, "error": "..."}` for expected errors (duplicate username, invalid input). Return HTTP 500 for unexpected internal errors.
|
||||
|
||||
**Delete account**
|
||||
|
||||
```
|
||||
DELETE /service-api/accounts/{username}
|
||||
```
|
||||
|
||||
Return HTTP 200 with `{"ok": true}` on success. Return HTTP 404 with `{"ok": false, "error": "not found"}` if the account does not exist.
|
||||
|
||||
**List accounts**
|
||||
|
||||
```
|
||||
GET /service-api/accounts
|
||||
```
|
||||
|
||||
Return `{"accounts": ["alice", "bob"]}` — an array of all provisioned usernames.
|
||||
|
||||
---
|
||||
|
||||
## 5. Backup integration
|
||||
|
||||
Declare `has_storage: true` in `capabilities` and fill in the `backup` block. PIC's `ServiceRegistry.get_backup_plan()` returns the combined backup declarations for all installed services. The backup runner reads from that method.
|
||||
|
||||
### Why docker exec instead of bind mounts
|
||||
|
||||
The API container only has access to `data/api/` on the host filesystem. Service data (mailboxes, calendar collections, file trees) lives in other containers' volumes. Rather than mount every service volume into the API container — which would require compose changes per service — PIC streams data using `docker exec <container> tar czf - <path>`. This works for any container on the Docker host regardless of how its volumes are configured.
|
||||
|
||||
### `volumes` entries
|
||||
|
||||
Each object in the `volumes` array describes one directory to capture:
|
||||
|
||||
| Field | Description |
|
||||
|---|---|
|
||||
| `container` | Name of the running container to exec into (e.g. `"cell-notes"`). |
|
||||
| `path` | Absolute path inside that container to archive (e.g. `"/data/notes"`). |
|
||||
| `name` | Archive filename stem. PIC saves the archive as `<name>.tar.gz` under `service_data/<service_id>/` in the backup directory. |
|
||||
|
||||
A service with multiple containers or multiple data directories lists one entry per directory.
|
||||
|
||||
**Security note:** The backup commands use `docker exec -- <container> tar -C <path> -czf - .` (note the `--` separator before the container name) to prevent option injection. The container name is also validated against `^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,63}$` before the command is run.
|
||||
|
||||
### `config_paths`
|
||||
|
||||
Paths in `config_paths` are relative to the PIC project root on the host and are copied directly into the snapshot (no docker exec). Use this for configuration files the service reads at startup, not for user data.
|
||||
|
||||
### Full example
|
||||
|
||||
```json
|
||||
"backup": {
|
||||
"volumes": [
|
||||
{"container": "cell-notes", "path": "/data/notes", "name": "notes_data"}
|
||||
],
|
||||
"config_paths": ["config/notes"]
|
||||
}
|
||||
```
|
||||
|
||||
This produces one archive `notes_data.tar.gz` (streamed from the `cell-notes` container) plus a direct copy of `config/notes/` from the host.
|
||||
|
||||
### Restore
|
||||
|
||||
PIC restores each volume entry by piping the archive back via `docker exec -i -- <container> tar -C <path> -xzf -`. The `-C <path>` flag bounds extraction to the declared volume path — the same path used during backup. Archive entries are relative paths (the backup uses `tar -C <path> -czf - .`), so files land in exactly the location declared in the manifest `volumes` entry. The target container must be running at restore time.
|
||||
|
||||
---
|
||||
|
||||
## 6. Egress routing
|
||||
|
||||
When `has_egress` is `true`, the cell admin can assign a specific outbound interface to your service. PIC enforces the selection using `fwmark` rules and policy routing in the `cell-wireguard` container via the `ConnectivityManager`.
|
||||
|
||||
The valid values for `egress.allowed` and what they mean:
|
||||
|
||||
| Value | Path |
|
||||
|---|---|
|
||||
| `default` | Default route through the cell's WAN interface (no VPN). |
|
||||
| `wireguard_ext` | Traffic leaves through `wg_ext0` (fwmark `0x10`, table 110). |
|
||||
| `openvpn` | Traffic leaves through `tun0` (fwmark `0x20`, table 120). |
|
||||
| `tor` | Traffic is redirected to the Tor transparent proxy on port 9040 (fwmark `0x30`, table 130). |
|
||||
|
||||
List only the interfaces that make sense for your service in `allowed`. The `default` value is used when the admin has not changed anything. Always include `default` in `allowed` so the admin has a way to use the normal path.
|
||||
|
||||
The egress field in the manifest tells PIC what options to present in the UI. Actual enforcement requires the cell to have the corresponding exit type configured (an OpenVPN config uploaded, a WireGuard external config active, etc.). If the chosen exit is not active, packets will be dropped by the kill-switch FORWARD rule in `cell-wireguard`.
|
||||
|
||||
---
|
||||
|
||||
## 7. Quick-start example
|
||||
|
||||
This section walks through a minimal working example: a static website served from Nginx with no accounts, no backup, and no egress policy.
|
||||
|
||||
### `manifest.json`
|
||||
|
||||
```json
|
||||
{
|
||||
"schema_version": 3,
|
||||
"id": "homepage",
|
||||
"name": "Homepage",
|
||||
"description": "A static homepage served from your cell",
|
||||
"version": "1.0.0",
|
||||
"author": "acme",
|
||||
"kind": "store",
|
||||
"min_pic_version": "1.0",
|
||||
|
||||
"capabilities": {
|
||||
"has_subdomain": true,
|
||||
"has_accounts": false,
|
||||
"has_admin_config": false,
|
||||
"has_storage": false,
|
||||
"has_egress": false,
|
||||
"has_api_hooks": false
|
||||
},
|
||||
|
||||
"subdomain": "home",
|
||||
"extra_subdomains": [],
|
||||
"backend": "cell-homepage:80",
|
||||
|
||||
"containers": ["cell-homepage"],
|
||||
|
||||
"image": "git.pic.ngo/roof/pic-homepage:latest",
|
||||
"container_name": "cell-homepage",
|
||||
|
||||
"volumes": [
|
||||
{ "name": "homepage-html", "mount": "/usr/share/nginx/html" }
|
||||
],
|
||||
|
||||
"env": [],
|
||||
|
||||
"iptables_rules": [
|
||||
{
|
||||
"type": "ACCEPT",
|
||||
"dest_ip": "${SERVICE_IP}",
|
||||
"dest_port": 80,
|
||||
"proto": "tcp"
|
||||
}
|
||||
],
|
||||
|
||||
"caddy_route": {
|
||||
"subdomain": "home"
|
||||
},
|
||||
|
||||
"compose": null
|
||||
}
|
||||
```
|
||||
|
||||
### What PIC does on install
|
||||
|
||||
1. Downloads this manifest from the store index.
|
||||
2. Validates every field (image allowlist, volume safety, reserved subdomains, iptables rule format).
|
||||
3. Allocates a static IP from the service pool (`172.20.0.20`–`172.20.0.254`).
|
||||
4. Writes a Docker Compose override file that starts `cell-homepage` with the allocated IP on `cell-network`.
|
||||
5. Runs `docker compose up -d cell-homepage`.
|
||||
6. Applies the `iptables_rules` in `cell-wireguard` so peers can reach the container.
|
||||
7. Regenerates the Caddyfile so `home.<cell-domain>` proxies to `cell-homepage:80`.
|
||||
|
||||
The result is that any WireGuard peer can reach `https://home.alice.pic.ngo/` immediately after installation.
|
||||
|
||||
---
|
||||
|
||||
## 8. Reference implementations
|
||||
|
||||
The `email`, `calendar`, and `files` services in `pic-services/services/` are the canonical examples of a complete store service. They demonstrate the full feature set:
|
||||
|
||||
| Service | Notable features demonstrated |
|
||||
|---|---|
|
||||
| `email` | `has_accounts`, `has_egress`, multi-container (`cell-mail` + `cell-rainloop`), `extra_backends`, custom image baking defaults via Dockerfile |
|
||||
| `calendar` | `has_accounts`, CalDAV `peer_config_template`, htpasswd account provisioning |
|
||||
| `files` | `has_accounts`, `has_storage`, WebDAV + Filegator `extra_backends`, `backup.volumes` with multiple entries |
|
||||
|
||||
When in doubt about how to structure your manifest or compose template, use these as the reference.
|
||||
|
||||
---
|
||||
|
||||
## 9. Submitting to the store
|
||||
|
||||
### Package format
|
||||
|
||||
A store service package is a ZIP archive containing:
|
||||
|
||||
```
|
||||
homepage-1.0.0.zip
|
||||
├── manifest.json (required)
|
||||
├── compose-template.yml (recommended for multi-container services)
|
||||
└── install.sh (optional post-install script)
|
||||
```
|
||||
|
||||
`install.sh` is executed on the cell host after the container starts. Keep it minimal — initialise data structures, create default config files. Do not use it to install system packages or modify files outside the PIC project root.
|
||||
|
||||
### Store index entry
|
||||
|
||||
The store index at `https://git.pic.ngo/roof/pic-services/raw/branch/main/index.json` is a JSON array. Each entry looks like:
|
||||
|
||||
```json
|
||||
{
|
||||
"id": "homepage",
|
||||
"name": "Homepage",
|
||||
"description": "A static homepage served from your cell",
|
||||
"version": "1.0.0",
|
||||
"author": "acme"
|
||||
}
|
||||
```
|
||||
|
||||
PIC fetches the full manifest from `https://git.pic.ngo/roof/pic-services/raw/branch/main/services/{id}/manifest.json` when the admin clicks install.
|
||||
|
||||
### Submission process
|
||||
|
||||
1. Fork `https://git.pic.ngo/roof/pic-services`.
|
||||
2. Create a directory `services/<your-id>/` and add your `manifest.json`.
|
||||
3. Open a pull request against `main`.
|
||||
|
||||
The review checks the following before merging:
|
||||
|
||||
**Security**
|
||||
- Image hosted on `git.pic.ngo/roof/*`. No external registries.
|
||||
- No volume mounts to system paths or to the PIC project root.
|
||||
- `iptables_rules` only declare `ACCEPT` rules (no DROP, no REJECT, no chain redirects).
|
||||
- `env` values contain only alphanumeric characters and a small set of safe punctuation.
|
||||
- `install.sh` does not call `apt`, `yum`, `curl | bash`, or modify files outside the project.
|
||||
|
||||
**Correctness**
|
||||
- `subdomain` does not collide with the reserved list or with any existing store service.
|
||||
- `backend` points to the declared `container_name`.
|
||||
- If `has_accounts: true`, the container responds correctly on all three `/service-api/accounts` endpoints.
|
||||
- If `has_storage: true`, every `volumes` entry names a container that is running and a path that exists inside it.
|
||||
|
||||
**Quality**
|
||||
- `description` is one sentence, no marketing language.
|
||||
- `version` is a valid semver string.
|
||||
- `config_schema` labels are in plain English, sentence case.
|
||||
|
||||
### Versioning
|
||||
|
||||
Increment `version` in `manifest.json` with every change you submit. PIC does not auto-update installed services; the admin manually runs an update. When an update is available, the UI shows the version mismatch between the installed record and the store index.
|
||||
|
||||
---
|
||||
|
||||
## Appendix: manifest field quick reference
|
||||
|
||||
| Field | Required | Notes |
|
||||
|---|---|---|
|
||||
| `schema_version` | yes | Must be `3` |
|
||||
| `id` | yes | |
|
||||
| `name` | yes | |
|
||||
| `description` | yes | |
|
||||
| `version` | yes | |
|
||||
| `author` | yes | |
|
||||
| `kind` | yes | Must be `"store"` |
|
||||
| `min_pic_version` | no | |
|
||||
| `capabilities.*` | yes | All six flags must be present |
|
||||
| `subdomain` | if `has_subdomain` | |
|
||||
| `extra_subdomains` | no | |
|
||||
| `backend` | if `has_subdomain` | |
|
||||
| `extra_backends` | no | |
|
||||
| `containers` | no | Informational |
|
||||
| `config_schema` | if `has_admin_config` | |
|
||||
| `peer_config_template` | if `has_accounts` | |
|
||||
| `accounts` | if `has_accounts` | |
|
||||
| `compose` | no | Always `null` — compose config goes in `compose-template.yml` |
|
||||
| `backup` | if `has_storage` | |
|
||||
| `egress` | if `has_egress` | |
|
||||
| `storage` | if `has_storage` | |
|
||||
| `image` | yes | Must match `git.pic.ngo/roof/*` |
|
||||
| `container_name` | yes | Must match `^cell-[a-z0-9][a-z0-9-]{0,30}$` |
|
||||
| `volumes` | no | |
|
||||
| `env` | no | |
|
||||
| `iptables_rules` | no | |
|
||||
| `caddy_route` | no | |
|
||||
@@ -1,51 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Script to fix import statements in test files
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
def fix_imports_in_file(file_path):
|
||||
"""Fix import statements in a test file"""
|
||||
with open(file_path, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
|
||||
# Fix relative imports to absolute imports from api package
|
||||
content = re.sub(r'from \.(\w+) import', r'from \1 import', content)
|
||||
content = re.sub(r'import \.(\w+)', r'import \1', content)
|
||||
|
||||
# Add path setup if not present
|
||||
if 'sys.path.insert' not in content and 'api_dir' not in content:
|
||||
path_setup = '''import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Add api directory to path
|
||||
api_dir = Path(__file__).parent.parent / 'api'
|
||||
sys.path.insert(0, str(api_dir))
|
||||
|
||||
'''
|
||||
# Insert after the first import line
|
||||
lines = content.split('\n')
|
||||
for i, line in enumerate(lines):
|
||||
if line.startswith('import ') or line.startswith('from '):
|
||||
lines.insert(i, path_setup.rstrip())
|
||||
break
|
||||
content = '\n'.join(lines)
|
||||
|
||||
with open(file_path, 'w', encoding='utf-8') as f:
|
||||
f.write(content)
|
||||
|
||||
print(f"Fixed imports in {file_path}")
|
||||
|
||||
def main():
|
||||
"""Fix all test files"""
|
||||
tests_dir = Path('tests')
|
||||
|
||||
for test_file in tests_dir.glob('test_*.py'):
|
||||
if test_file.name not in ['test_cli_tool.py', 'test_peer_registry.py']: # Already fixed
|
||||
fix_imports_in_file(test_file)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -1,31 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Fix import statements in test files
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
def fix_imports_in_file(file_path):
|
||||
"""Fix import statements in a test file"""
|
||||
with open(file_path, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
|
||||
# Replace 'from api.' with 'from .'
|
||||
content = re.sub(r'from api\.', 'from .', content)
|
||||
content = re.sub(r'import api\.', 'import .', content)
|
||||
|
||||
with open(file_path, 'w', encoding='utf-8') as f:
|
||||
f.write(content)
|
||||
|
||||
print(f"Fixed imports in {file_path}")
|
||||
|
||||
def main():
|
||||
tests_dir = Path('tests')
|
||||
|
||||
for test_file in tests_dir.glob('test_*.py'):
|
||||
fix_imports_in_file(test_file)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
+26
-34
@@ -22,9 +22,9 @@
|
||||
# =============================================================================
|
||||
#
|
||||
# Usage:
|
||||
# sudo bash install.sh # Standard install
|
||||
# sudo bash install.sh --force # Bypass idempotency check
|
||||
# sudo PIC_DIR=/srv/pic bash install.sh # Custom install directory
|
||||
# bash install.sh # Standard install (uses sudo internally for packages)
|
||||
# bash install.sh --force # Bypass idempotency check
|
||||
# PIC_DIR=/srv/pic bash install.sh # Custom install directory
|
||||
#
|
||||
# Supported OS: Debian/Ubuntu (apt), Fedora/RHEL (dnf), Alpine Linux (apk)
|
||||
#
|
||||
@@ -82,10 +82,10 @@ die() { log_error "$1"; exit 1; }
|
||||
TOTAL_STEPS=7
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Must run as root
|
||||
# Sudo check — we need it for package installs and system user creation
|
||||
# ---------------------------------------------------------------------------
|
||||
if [ "$(id -u)" -ne 0 ]; then
|
||||
die "This installer must be run as root (use sudo)."
|
||||
if ! command -v sudo >/dev/null 2>&1; then
|
||||
die "sudo is required. Install it and ensure your user has sudo access."
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -149,37 +149,32 @@ case "$PKG_MANAGER" in
|
||||
|
||||
apt)
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
apt-get update -qq
|
||||
apt-get install -y -qq git curl make docker.io docker-compose-plugin 2>&1 \
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y -qq git curl make docker.io docker-compose-plugin 2>&1 \
|
||||
| grep -v "^$" | sed 's/^/ /' || true
|
||||
|
||||
# Verify docker compose plugin installed
|
||||
if ! docker compose version >/dev/null 2>&1; then
|
||||
log_warn "docker-compose-plugin not available; falling back to standalone docker-compose"
|
||||
apt-get install -y -qq docker-compose 2>&1 | grep -v "^$" | sed 's/^/ /' || true
|
||||
sudo apt-get install -y -qq docker-compose 2>&1 | grep -v "^$" | sed 's/^/ /' || true
|
||||
fi
|
||||
;;
|
||||
|
||||
dnf)
|
||||
dnf install -y -q git curl make docker 2>&1 | sed 's/^/ /' || true
|
||||
sudo dnf install -y -q git curl make docker 2>&1 | sed 's/^/ /' || true
|
||||
|
||||
# Enable and start Docker (dnf installs but doesn't enable it)
|
||||
systemctl enable --now docker >/dev/null 2>&1 || true
|
||||
sudo systemctl enable --now docker >/dev/null 2>&1 || true
|
||||
|
||||
# Docker Compose plugin comes bundled with the Docker CE package on Fedora/RHEL.
|
||||
# If not present, install via the docker-compose-plugin package (Docker CE repo).
|
||||
if ! docker compose version >/dev/null 2>&1; then
|
||||
log_warn "docker compose plugin not found; installing docker-compose-plugin..."
|
||||
dnf install -y -q docker-compose-plugin 2>&1 | sed 's/^/ /' || true
|
||||
sudo dnf install -y -q docker-compose-plugin 2>&1 | sed 's/^/ /' || true
|
||||
fi
|
||||
;;
|
||||
|
||||
apk)
|
||||
apk add --quiet git curl make docker docker-cli-compose 2>&1 | sed 's/^/ /' || true
|
||||
sudo apk add --quiet git curl make docker docker-cli-compose 2>&1 | sed 's/^/ /' || true
|
||||
|
||||
# Enable Docker on Alpine (OpenRC)
|
||||
rc-update add docker default >/dev/null 2>&1 || true
|
||||
service docker start >/dev/null 2>&1 || true
|
||||
sudo rc-update add docker default >/dev/null 2>&1 || true
|
||||
sudo service docker start >/dev/null 2>&1 || true
|
||||
;;
|
||||
|
||||
esac
|
||||
@@ -204,10 +199,10 @@ log_step 3 "Configuring system user..."
|
||||
if ! id "$PIC_USER" >/dev/null 2>&1; then
|
||||
case "$PKG_MANAGER" in
|
||||
apk)
|
||||
adduser -S -D -H -s /sbin/nologin "$PIC_USER"
|
||||
sudo adduser -S -D -H -s /sbin/nologin "$PIC_USER"
|
||||
;;
|
||||
*)
|
||||
useradd --system --no-create-home --shell /usr/sbin/nologin "$PIC_USER"
|
||||
sudo useradd --system --no-create-home --shell /usr/sbin/nologin "$PIC_USER"
|
||||
;;
|
||||
esac
|
||||
log_ok "Created system user: ${PIC_USER}"
|
||||
@@ -215,17 +210,18 @@ else
|
||||
log_ok "System user already exists: ${PIC_USER}"
|
||||
fi
|
||||
|
||||
# Ensure docker group exists and user is in it
|
||||
# Ensure docker group exists and invoking user is in it
|
||||
if ! getent group docker >/dev/null 2>&1; then
|
||||
groupadd docker
|
||||
sudo groupadd docker
|
||||
log_ok "Created docker group"
|
||||
fi
|
||||
|
||||
if ! id -nG "$PIC_USER" | grep -qw docker; then
|
||||
usermod -aG docker "$PIC_USER"
|
||||
log_ok "Added ${PIC_USER} to docker group"
|
||||
CURRENT_USER="${USER:-$(id -un)}"
|
||||
if ! id -nG "$CURRENT_USER" | grep -qw docker; then
|
||||
sudo usermod -aG docker "$CURRENT_USER"
|
||||
log_ok "Added ${CURRENT_USER} to docker group (re-login or newgrp docker to apply)"
|
||||
else
|
||||
log_ok "${PIC_USER} is already in docker group"
|
||||
log_ok "${CURRENT_USER} is already in docker group"
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -245,17 +241,13 @@ else
|
||||
log_ok "Repository cloned to ${PIC_DIR}"
|
||||
fi
|
||||
|
||||
# Ensure the pic user owns the directory
|
||||
chown -R "${PIC_USER}:${PIC_USER}" "$PIC_DIR"
|
||||
sudo git config --system --add safe.directory "$PIC_DIR" 2>/dev/null || true
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Step 5 — Run make install
|
||||
# ---------------------------------------------------------------------------
|
||||
log_step 5 "Running 'make install'..."
|
||||
|
||||
# make install generates config, writes the systemd unit, and touches .installed.
|
||||
# We run it as the pic user (via sudo -u) so files get correct ownership, but
|
||||
# make install itself calls sudo internally where root is needed.
|
||||
cd "$PIC_DIR"
|
||||
|
||||
if ! make install 2>&1 | sed 's/^/ /'; then
|
||||
@@ -318,7 +310,7 @@ printf "\n${GREEN}${BOLD}=======================================================
|
||||
printf "${GREEN}${BOLD} PIC installed successfully!${RESET}\n"
|
||||
printf "${GREEN}${BOLD}============================================================${RESET}\n"
|
||||
printf "\n"
|
||||
printf " Open the setup wizard at:\n"
|
||||
printf " Open the setup wizard to configure your cell:\n"
|
||||
printf "\n"
|
||||
printf " ${BOLD}http://${HOST_IP}:${WEBUI_PORT}/setup${RESET}\n"
|
||||
printf "\n"
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Update the cell's DDNS record with the current public IP.
|
||||
|
||||
Called by: make ddns-update
|
||||
systemd timer (optional, see scripts/pic-ddns-update.timer)
|
||||
|
||||
Reads the DDNS token from data/api/.ddns_token (written by setup_cell.py).
|
||||
Exits 0 on success or if already up to date, non-zero on failure.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
DDNS_URL = os.environ.get('DDNS_URL', 'https://ddns.pic.ngo/api/v1')
|
||||
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
TOKEN_FILE = os.path.join(ROOT, 'data', 'api', '.ddns_token')
|
||||
IP_CACHE_FILE = os.path.join(ROOT, 'data', 'api', '.ddns_last_ip')
|
||||
|
||||
|
||||
def get_public_ip() -> str:
|
||||
return urllib.request.urlopen('https://api.ipify.org', timeout=5).read().decode().strip()
|
||||
|
||||
|
||||
def read_token() -> str:
|
||||
if not os.path.exists(TOKEN_FILE):
|
||||
print('ERROR: DDNS token not found. Run "make setup" to register.', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
return open(TOKEN_FILE).read().strip()
|
||||
|
||||
|
||||
def read_last_ip() -> str:
|
||||
try:
|
||||
return open(IP_CACHE_FILE).read().strip()
|
||||
except FileNotFoundError:
|
||||
return ''
|
||||
|
||||
|
||||
def write_last_ip(ip: str) -> None:
|
||||
with open(IP_CACHE_FILE, 'w') as f:
|
||||
f.write(ip)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
try:
|
||||
public_ip = get_public_ip()
|
||||
except Exception as e:
|
||||
print(f'ERROR: Could not detect public IP: {e}', file=sys.stderr)
|
||||
return 1
|
||||
|
||||
last_ip = read_last_ip()
|
||||
if public_ip == last_ip:
|
||||
print(f'DDNS: IP unchanged ({public_ip}) — no update needed')
|
||||
return 0
|
||||
|
||||
token = read_token()
|
||||
data = json.dumps({'token': token, 'ip': public_ip}).encode()
|
||||
req = urllib.request.Request(
|
||||
f'{DDNS_URL}/update',
|
||||
data=data,
|
||||
headers={'Content-Type': 'application/json'},
|
||||
method='PUT',
|
||||
)
|
||||
try:
|
||||
resp = urllib.request.urlopen(req, timeout=10)
|
||||
result = json.loads(resp.read())
|
||||
if result.get('updated'):
|
||||
write_last_ip(public_ip)
|
||||
print(f'DDNS: Updated to {public_ip}')
|
||||
return 0
|
||||
else:
|
||||
print(f'ERROR: Unexpected response: {result}', file=sys.stderr)
|
||||
return 1
|
||||
except urllib.error.HTTPError as e:
|
||||
body = e.read().decode()
|
||||
print(f'ERROR: DDNS update failed ({e.code}): {body}', file=sys.stderr)
|
||||
return 1
|
||||
except Exception as e:
|
||||
print(f'ERROR: DDNS update failed: {e}', file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
+53
-48
@@ -1,60 +1,65 @@
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
import json
|
||||
import sys
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
|
||||
# Updated endpoints to use HTTPS
|
||||
SERVICES = [
|
||||
{"name": "Dashboard UI", "url": "https://localhost/"},
|
||||
{"name": "Mail UI", "url": "https://localhost/mail"},
|
||||
{"name": "Calendar UI", "url": "https://localhost/calendar"},
|
||||
{"name": "Files UI", "url": "https://localhost/files"},
|
||||
{"name": "DNS Management UI", "url": "https://localhost/dns"},
|
||||
{"name": "API Health", "url": "https://localhost/api/health", "is_api": True},
|
||||
{"name": "API Service Status", "url": "https://localhost/api/services/status", "is_api": True},
|
||||
BASE = "http://127.0.0.1:3000"
|
||||
|
||||
CORE_CHECKS = [
|
||||
{"name": "API health", "path": "/health"},
|
||||
{"name": "API status", "path": "/api/status"},
|
||||
{"name": "Active services", "path": "/api/services/active"},
|
||||
]
|
||||
|
||||
def check_ui(url, name):
|
||||
try:
|
||||
resp = requests.get(url, timeout=5, verify=False)
|
||||
if resp.status_code == 200:
|
||||
# Try to parse HTML and look for a title or main element
|
||||
soup = BeautifulSoup(resp.text, "html.parser")
|
||||
title = soup.title.string if soup.title else "No title"
|
||||
print(f"[OK] {name} ({url}) - {title}")
|
||||
return True
|
||||
else:
|
||||
print(f"[FAIL] {name} ({url}) - HTTP {resp.status_code}")
|
||||
return False
|
||||
except Exception as e:
|
||||
print(f"[ERROR] {name} ({url}) - {e}")
|
||||
return False
|
||||
OPTIONAL_SERVICE_CHECKS = {
|
||||
"email": {"name": "Email status", "path": "/api/email/status"},
|
||||
"calendar": {"name": "Calendar status", "path": "/api/calendar/status"},
|
||||
"files": {"name": "Files status", "path": "/api/files/status"},
|
||||
}
|
||||
|
||||
def check_api_status(url, name):
|
||||
|
||||
def get(path):
|
||||
try:
|
||||
resp = requests.get(url, timeout=5, verify=False)
|
||||
if resp.status_code == 200:
|
||||
print(f"[OK] {name}: {url}")
|
||||
if 'services/status' in url:
|
||||
data = resp.json()
|
||||
for service, status in data.items():
|
||||
s = status.get("status", "Unknown")
|
||||
print(f" {service}: {s}")
|
||||
else:
|
||||
print(f" Response: {resp.text.strip()}")
|
||||
return True
|
||||
else:
|
||||
print(f"[FAIL] {name}: HTTP {resp.status_code}")
|
||||
return False
|
||||
resp = urllib.request.urlopen(BASE + path, timeout=5)
|
||||
body = resp.read().decode()
|
||||
return resp.status, body
|
||||
except urllib.error.HTTPError as e:
|
||||
return e.code, e.read().decode()
|
||||
except Exception as e:
|
||||
print(f"[ERROR] {name}: {e}")
|
||||
return False
|
||||
return None, str(e)
|
||||
|
||||
|
||||
def main():
|
||||
print("=== UI & API Sanity Checks (Caddy-proxied, HTTPS) ===")
|
||||
for svc in SERVICES:
|
||||
if svc.get("is_api"):
|
||||
check_api_status(svc["url"], svc["name"])
|
||||
print("=== PIC Sanity Check ===")
|
||||
|
||||
for chk in CORE_CHECKS:
|
||||
code, body = get(chk["path"])
|
||||
if code == 200:
|
||||
print(f"[OK] {chk['name']}")
|
||||
else:
|
||||
check_ui(svc["url"], svc["name"])
|
||||
print(f"[FAIL] {chk['name']} — HTTP {code}: {body[:120]}")
|
||||
|
||||
# Discover installed services and check only those
|
||||
code, body = get("/api/services/active")
|
||||
installed_ids = set()
|
||||
if code == 200:
|
||||
try:
|
||||
installed_ids = {svc["id"] for svc in json.loads(body)}
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
print()
|
||||
print("Optional services:")
|
||||
for svc_id, chk in OPTIONAL_SERVICE_CHECKS.items():
|
||||
if svc_id not in installed_ids:
|
||||
print(f"[SKIP] {chk['name']} — not installed")
|
||||
continue
|
||||
code, body = get(chk["path"])
|
||||
if code == 200:
|
||||
print(f"[OK] {chk['name']}")
|
||||
else:
|
||||
print(f"[FAIL] {chk['name']} — HTTP {code}: {body[:120]}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+130
-21
@@ -19,26 +19,18 @@ REQUIRED_DIRS = [
|
||||
'config/dns',
|
||||
'config/dhcp',
|
||||
'config/ntp',
|
||||
'config/mail/config',
|
||||
'config/mail/ssl',
|
||||
'config/radicale',
|
||||
'config/webdav',
|
||||
'config/wireguard',
|
||||
'config/api',
|
||||
'data/caddy',
|
||||
'data/dns',
|
||||
'data/dhcp',
|
||||
'data/maildata',
|
||||
'data/mailstate',
|
||||
'data/maillogs',
|
||||
'data/radicale',
|
||||
'data/files',
|
||||
'data/api',
|
||||
'data/vault/certs',
|
||||
'data/vault/keys',
|
||||
'data/vault/trust',
|
||||
'data/vault/ca',
|
||||
'data/logs',
|
||||
'data/services',
|
||||
'data/wireguard/keys/peers',
|
||||
'data/wireguard/wg_confs',
|
||||
]
|
||||
@@ -47,8 +39,6 @@ REQUIRED_FILES = [
|
||||
'config/dns/Corefile',
|
||||
'config/dhcp/dnsmasq.conf',
|
||||
'config/ntp/chrony.conf',
|
||||
'config/mail/mailserver.env',
|
||||
'config/webdav/users.passwd',
|
||||
]
|
||||
|
||||
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
@@ -169,7 +159,8 @@ def write_wg0_conf(private_key: str, address: str, port: int):
|
||||
print(f'[CREATED] config/wireguard/wg0.conf address={address} port={port}')
|
||||
|
||||
|
||||
def write_cell_config(cell_name: str, domain: str, port: int):
|
||||
def write_cell_config(cell_name: str, domain: str, port: int,
|
||||
domain_mode: str, domain_name: str) -> None:
|
||||
cfg_path = os.path.join(ROOT, 'config', 'api', 'cell_config.json')
|
||||
if os.path.exists(cfg_path):
|
||||
try:
|
||||
@@ -179,17 +170,46 @@ def write_cell_config(cell_name: str, domain: str, port: int):
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
ddns: dict = {}
|
||||
if domain_mode == 'pic_ngo':
|
||||
ddns = {
|
||||
'provider': 'pic_ngo',
|
||||
'api_base_url': DDNS_URL.replace('/api/v1', ''),
|
||||
'totp_secret': DDNS_TOTP_SECRET,
|
||||
'enabled': True,
|
||||
}
|
||||
elif domain_mode == 'cloudflare':
|
||||
ddns = {'provider': 'cloudflare', 'enabled': True}
|
||||
if CLOUDFLARE_TOKEN:
|
||||
ddns['api_token'] = CLOUDFLARE_TOKEN
|
||||
elif domain_mode == 'duckdns':
|
||||
ddns = {'provider': 'duckdns', 'enabled': True}
|
||||
if DUCKDNS_TOKEN:
|
||||
ddns['token'] = DUCKDNS_TOKEN
|
||||
if DUCKDNS_SUBDOMAIN:
|
||||
ddns['subdomain'] = DUCKDNS_SUBDOMAIN
|
||||
elif domain_mode == 'http01':
|
||||
ddns = {'provider': 'http01', 'enabled': True}
|
||||
else: # lan
|
||||
ddns = {'provider': 'none', 'enabled': False}
|
||||
|
||||
config = {
|
||||
'_identity': {
|
||||
'cell_name': cell_name,
|
||||
'domain': domain,
|
||||
'domain_mode': domain_mode,
|
||||
'domain_name': domain_name,
|
||||
'ip_range': '172.20.0.0/16',
|
||||
'wireguard_port': port,
|
||||
}
|
||||
},
|
||||
'ddns': ddns,
|
||||
}
|
||||
with open(cfg_path, 'w') as f:
|
||||
json.dump(config, f, indent=2)
|
||||
print(f'[CREATED] config/api/cell_config.json name={cell_name} domain={domain}')
|
||||
os.chmod(cfg_path, 0o600)
|
||||
print(f'[CREATED] config/api/cell_config.json name={cell_name} mode={domain_mode}'
|
||||
+ (f' domain={domain_name}' if domain_name else ''))
|
||||
|
||||
|
||||
def write_compose_env(ip_range: str):
|
||||
@@ -238,6 +258,82 @@ def ensure_session_secret():
|
||||
print('[CREATED] data/api/.session_secret')
|
||||
|
||||
|
||||
DDNS_URL = os.environ.get('DDNS_URL', 'http://ddns.pic.ngo:8080/api/v1')
|
||||
DDNS_TOTP_SECRET = os.environ.get('DDNS_TOTP_SECRET', 'S6UMA464YIKM74QHXWL5WELDIO3HFZ6K')
|
||||
DOMAIN_MODE = os.environ.get('DOMAIN_MODE', 'lan')
|
||||
CELL_DOMAIN_NAME = os.environ.get('CELL_DOMAIN_NAME', '')
|
||||
CLOUDFLARE_TOKEN = os.environ.get('CLOUDFLARE_API_TOKEN', '')
|
||||
DUCKDNS_TOKEN = os.environ.get('DUCKDNS_TOKEN', '')
|
||||
DUCKDNS_SUBDOMAIN= os.environ.get('DUCKDNS_SUBDOMAIN', '')
|
||||
|
||||
|
||||
def register_with_ddns(cell_name: str) -> None:
|
||||
"""Register cell_name.pic.ngo with the DDNS server using TOTP auth.
|
||||
|
||||
Idempotent: if a token file already exists the registration is skipped.
|
||||
Skipped silently if DDNS_TOTP_SECRET is not set.
|
||||
"""
|
||||
token_path = os.path.join(ROOT, 'data', 'api', '.ddns_token')
|
||||
if os.path.exists(token_path):
|
||||
print('[EXISTS] DDNS registration — token already present')
|
||||
return
|
||||
|
||||
if not DDNS_TOTP_SECRET:
|
||||
print('[SKIP] DDNS_TOTP_SECRET not set — skipping DDNS registration')
|
||||
return
|
||||
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
|
||||
# Detect public IP
|
||||
try:
|
||||
public_ip = urllib.request.urlopen(
|
||||
'https://api.ipify.org', timeout=5
|
||||
).read().decode().strip()
|
||||
except Exception as e:
|
||||
print(f'[WARN] Could not detect public IP: {e} — skipping DDNS registration')
|
||||
return
|
||||
|
||||
# Generate TOTP using stdlib only — no third-party package needed
|
||||
otp = ''
|
||||
try:
|
||||
import base64 as _b64, hashlib as _hl, hmac as _hmac, struct as _struct
|
||||
import time as _time
|
||||
_key = _b64.b32decode(DDNS_TOTP_SECRET.upper())
|
||||
_t = int(_time.time()) // 30
|
||||
_h = _hmac.new(_key, _struct.pack('>Q', _t), _hl.sha1).digest()
|
||||
_offset = _h[-1] & 0xF
|
||||
_code = _struct.unpack('>I', _h[_offset:_offset + 4])[0] & 0x7FFFFFFF
|
||||
otp = f'{_code % 1_000_000:06d}'
|
||||
except Exception as e:
|
||||
print(f'[WARN] Could not generate OTP: {e} — registering without OTP header')
|
||||
|
||||
data = json.dumps({'name': cell_name, 'ip': public_ip}).encode()
|
||||
headers = {'Content-Type': 'application/json'}
|
||||
if otp:
|
||||
headers['X-Register-OTP'] = otp
|
||||
req = urllib.request.Request(
|
||||
f'{DDNS_URL}/register',
|
||||
data=data,
|
||||
headers=headers,
|
||||
method='POST',
|
||||
)
|
||||
try:
|
||||
resp = urllib.request.urlopen(req, timeout=10)
|
||||
result = json.loads(resp.read())
|
||||
token = result['token']
|
||||
os.makedirs(os.path.dirname(token_path), exist_ok=True)
|
||||
with open(token_path, 'w') as f:
|
||||
f.write(token)
|
||||
os.chmod(token_path, 0o600)
|
||||
print(f'[CREATED] DDNS registration: {result["subdomain"]} ip={public_ip}')
|
||||
except urllib.error.HTTPError as e:
|
||||
body = e.read().decode()
|
||||
print(f'[WARN] DDNS registration failed ({e.code}): {body}')
|
||||
except Exception as e:
|
||||
print(f'[WARN] DDNS registration failed: {e}')
|
||||
|
||||
|
||||
def bootstrap_admin_password():
|
||||
import secrets as _secrets
|
||||
users_file = os.path.join(ROOT, 'data', 'api', 'auth_users.json')
|
||||
@@ -279,15 +375,28 @@ def bootstrap_admin_password():
|
||||
|
||||
|
||||
def main():
|
||||
cell_name = os.environ.get('CELL_NAME', 'mycell')
|
||||
domain = os.environ.get('CELL_DOMAIN', 'cell')
|
||||
cell_name = os.environ.get('CELL_NAME', 'mycell')
|
||||
domain_mode = DOMAIN_MODE # module-level, read from env
|
||||
domain_name = CELL_DOMAIN_NAME
|
||||
|
||||
# Derive the legacy 'domain' TLD field and fill in domain_name if empty
|
||||
if domain_mode == 'pic_ngo':
|
||||
domain = 'pic.ngo'
|
||||
if not domain_name:
|
||||
domain_name = f'{cell_name}.pic.ngo'
|
||||
elif domain_mode == 'lan':
|
||||
domain = os.environ.get('CELL_DOMAIN', 'cell')
|
||||
domain_name = ''
|
||||
else:
|
||||
# cloudflare / duckdns / http01 — domain_name is the full FQDN
|
||||
domain = domain_name
|
||||
|
||||
vpn_address = os.environ.get('VPN_ADDRESS', '10.0.0.1/24')
|
||||
wg_port = int(os.environ.get('WG_PORT', '51820'))
|
||||
# Prefer existing config ip_range over env var so `make setup` is safe to re-run
|
||||
ip_range = os.environ.get('CELL_IP_RANGE') or _read_existing_ip_range() or '172.20.0.0/16'
|
||||
wg_port = int(os.environ.get('WG_PORT', '51820'))
|
||||
ip_range = os.environ.get('CELL_IP_RANGE') or _read_existing_ip_range() or '172.20.0.0/16'
|
||||
|
||||
print('--- Personal Internet Cell: Setup ---')
|
||||
print(f' cell={cell_name} domain={domain} vpn={vpn_address} port={wg_port}')
|
||||
print(f' cell={cell_name} mode={domain_mode} domain={domain_name or "(lan)"} vpn={vpn_address} port={wg_port}')
|
||||
print()
|
||||
|
||||
for d in REQUIRED_DIRS:
|
||||
@@ -298,7 +407,7 @@ def main():
|
||||
ensure_caddy_ca_cert()
|
||||
priv, _pub = generate_wg_keys()
|
||||
write_wg0_conf(priv, vpn_address, wg_port)
|
||||
write_cell_config(cell_name, domain, wg_port)
|
||||
write_cell_config(cell_name, domain, wg_port, domain_mode, domain_name)
|
||||
write_compose_env(ip_range)
|
||||
write_caddy_config(ip_range, cell_name, domain)
|
||||
ensure_session_secret()
|
||||
|
||||
@@ -1,559 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Comprehensive tests for Flask app endpoints
|
||||
"""
|
||||
|
||||
import unittest
|
||||
import sys
|
||||
import os
|
||||
import tempfile
|
||||
import shutil
|
||||
import json
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
# Add api directory to path
|
||||
api_dir = Path(__file__).parent / 'api'
|
||||
sys.path.insert(0, str(api_dir))
|
||||
|
||||
class TestFlaskAppEndpoints(unittest.TestCase):
|
||||
def setUp(self):
|
||||
"""Set up test environment"""
|
||||
# Create temporary directories
|
||||
self.test_dir = tempfile.mkdtemp()
|
||||
self.data_dir = os.path.join(self.test_dir, 'data')
|
||||
self.config_dir = os.path.join(self.test_dir, 'config')
|
||||
os.makedirs(self.data_dir, exist_ok=True)
|
||||
os.makedirs(self.config_dir, exist_ok=True)
|
||||
|
||||
# Set environment variables
|
||||
os.environ['TESTING'] = 'true'
|
||||
os.environ['LOG_LEVEL'] = 'ERROR'
|
||||
|
||||
# Import and create app
|
||||
from app import app
|
||||
self.app = app
|
||||
self.client = app.test_client()
|
||||
|
||||
# Mock external dependencies
|
||||
self.patchers = []
|
||||
|
||||
# Mock subprocess.run
|
||||
subprocess_patcher = patch('subprocess.run')
|
||||
self.mock_subprocess = subprocess_patcher.start()
|
||||
self.mock_subprocess.return_value.returncode = 0
|
||||
self.mock_subprocess.return_value.stdout = b"test output"
|
||||
self.patchers.append(subprocess_patcher)
|
||||
|
||||
# Mock docker
|
||||
docker_patcher = patch('docker.from_env')
|
||||
self.mock_docker = docker_patcher.start()
|
||||
self.mock_docker_client = MagicMock()
|
||||
self.mock_docker.return_value = self.mock_docker_client
|
||||
self.patchers.append(docker_patcher)
|
||||
|
||||
# Mock file operations
|
||||
file_patcher = patch('builtins.open', create=True)
|
||||
self.mock_file = file_patcher.start()
|
||||
self.mock_file.return_value.__enter__.return_value.read.return_value = '{}'
|
||||
self.patchers.append(file_patcher)
|
||||
|
||||
def tearDown(self):
|
||||
"""Clean up test environment"""
|
||||
shutil.rmtree(self.test_dir)
|
||||
for patcher in self.patchers:
|
||||
patcher.stop()
|
||||
|
||||
def test_health_endpoint(self):
|
||||
"""Test /health endpoint"""
|
||||
response = self.client.get('/health')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIn('status', data)
|
||||
|
||||
def test_api_status_endpoint(self):
|
||||
"""Test /api/status endpoint"""
|
||||
response = self.client.get('/api/status')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIn('status', data)
|
||||
|
||||
def test_api_config_get_endpoint(self):
|
||||
"""Test GET /api/config endpoint"""
|
||||
response = self.client.get('/api/config')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIsInstance(data, dict)
|
||||
|
||||
def test_api_config_put_endpoint(self):
|
||||
"""Test PUT /api/config endpoint"""
|
||||
test_config = {'test': 'value'}
|
||||
response = self.client.put('/api/config',
|
||||
data=json.dumps(test_config),
|
||||
content_type='application/json')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIn('success', data)
|
||||
|
||||
def test_api_config_backup_endpoint(self):
|
||||
"""Test POST /api/config/backup endpoint"""
|
||||
response = self.client.post('/api/config/backup')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIn('backup_id', data)
|
||||
|
||||
def test_api_config_backups_endpoint(self):
|
||||
"""Test GET /api/config/backups endpoint"""
|
||||
response = self.client.get('/api/config/backups')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIsInstance(data, list)
|
||||
|
||||
def test_api_config_restore_endpoint(self):
|
||||
"""Test POST /api/config/restore/<backup_id> endpoint"""
|
||||
response = self.client.post('/api/config/restore/test_backup')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIn('success', data)
|
||||
|
||||
def test_api_config_export_endpoint(self):
|
||||
"""Test GET /api/config/export endpoint"""
|
||||
response = self.client.get('/api/config/export')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIsInstance(data, dict)
|
||||
|
||||
def test_api_config_import_endpoint(self):
|
||||
"""Test POST /api/config/import endpoint"""
|
||||
test_config = {'test': 'value'}
|
||||
response = self.client.post('/api/config/import',
|
||||
data=json.dumps(test_config),
|
||||
content_type='application/json')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIn('success', data)
|
||||
|
||||
def test_api_services_bus_status_endpoint(self):
|
||||
"""Test GET /api/services/bus/status endpoint"""
|
||||
response = self.client.get('/api/services/bus/status')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIn('services', data)
|
||||
|
||||
def test_api_services_bus_events_endpoint(self):
|
||||
"""Test GET /api/services/bus/events endpoint"""
|
||||
response = self.client.get('/api/services/bus/events')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIsInstance(data, list)
|
||||
|
||||
def test_api_services_bus_start_endpoint(self):
|
||||
"""Test POST /api/services/bus/services/<service_name>/start endpoint"""
|
||||
response = self.client.post('/api/services/bus/services/test/start')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIn('success', data)
|
||||
|
||||
def test_api_services_bus_stop_endpoint(self):
|
||||
"""Test POST /api/services/bus/services/<service_name>/stop endpoint"""
|
||||
response = self.client.post('/api/services/bus/services/test/stop')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIn('success', data)
|
||||
|
||||
def test_api_services_bus_restart_endpoint(self):
|
||||
"""Test POST /api/services/bus/services/<service_name>/restart endpoint"""
|
||||
response = self.client.post('/api/services/bus/services/test/restart')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIn('success', data)
|
||||
|
||||
def test_api_logs_services_endpoint(self):
|
||||
"""Test GET /api/logs/services/<service> endpoint"""
|
||||
response = self.client.get('/api/logs/services/test')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIsInstance(data, list)
|
||||
|
||||
def test_api_logs_search_endpoint(self):
|
||||
"""Test POST /api/logs/search endpoint"""
|
||||
search_data = {'query': 'test', 'level': 'INFO'}
|
||||
response = self.client.post('/api/logs/search',
|
||||
data=json.dumps(search_data),
|
||||
content_type='application/json')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIsInstance(data, list)
|
||||
|
||||
def test_api_logs_export_endpoint(self):
|
||||
"""Test POST /api/logs/export endpoint"""
|
||||
export_data = {'format': 'json', 'filters': {}}
|
||||
response = self.client.post('/api/logs/export',
|
||||
data=json.dumps(export_data),
|
||||
content_type='application/json')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIn('export_path', data)
|
||||
|
||||
def test_api_logs_statistics_endpoint(self):
|
||||
"""Test GET /api/logs/statistics endpoint"""
|
||||
response = self.client.get('/api/logs/statistics')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIn('total_entries', data)
|
||||
|
||||
def test_api_logs_rotate_endpoint(self):
|
||||
"""Test POST /api/logs/rotate endpoint"""
|
||||
response = self.client.post('/api/logs/rotate')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIn('success', data)
|
||||
|
||||
def test_api_dns_records_endpoints(self):
|
||||
"""Test DNS records endpoints"""
|
||||
# GET
|
||||
response = self.client.get('/api/dns/records')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIsInstance(data, list)
|
||||
|
||||
# POST
|
||||
record_data = {'name': 'test.example.com', 'type': 'A', 'value': '192.168.1.1'}
|
||||
response = self.client.post('/api/dns/records',
|
||||
data=json.dumps(record_data),
|
||||
content_type='application/json')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIn('success', data)
|
||||
|
||||
# DELETE
|
||||
response = self.client.delete('/api/dns/records',
|
||||
data=json.dumps(record_data),
|
||||
content_type='application/json')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIn('success', data)
|
||||
|
||||
def test_api_dhcp_endpoints(self):
|
||||
"""Test DHCP endpoints"""
|
||||
# GET leases
|
||||
response = self.client.get('/api/dhcp/leases')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIsInstance(data, list)
|
||||
|
||||
# POST reservation
|
||||
reservation_data = {'mac': '00:11:22:33:44:55', 'ip': '192.168.1.100'}
|
||||
response = self.client.post('/api/dhcp/reservations',
|
||||
data=json.dumps(reservation_data),
|
||||
content_type='application/json')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIn('success', data)
|
||||
|
||||
# DELETE reservation
|
||||
response = self.client.delete('/api/dhcp/reservations',
|
||||
data=json.dumps(reservation_data),
|
||||
content_type='application/json')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIn('success', data)
|
||||
|
||||
def test_api_ntp_status_endpoint(self):
|
||||
"""Test GET /api/ntp/status endpoint"""
|
||||
response = self.client.get('/api/ntp/status')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIn('status', data)
|
||||
|
||||
def test_api_network_info_endpoint(self):
|
||||
"""Test GET /api/network/info endpoint"""
|
||||
response = self.client.get('/api/network/info')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIn('interfaces', data)
|
||||
|
||||
def test_api_dns_status_endpoint(self):
|
||||
"""Test GET /api/dns/status endpoint"""
|
||||
response = self.client.get('/api/dns/status')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIn('status', data)
|
||||
|
||||
def test_api_network_test_endpoint(self):
|
||||
"""Test POST /api/network/test endpoint"""
|
||||
test_data = {'target': '8.8.8.8', 'type': 'ping'}
|
||||
response = self.client.post('/api/network/test',
|
||||
data=json.dumps(test_data),
|
||||
content_type='application/json')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIn('success', data)
|
||||
|
||||
def test_api_wireguard_endpoints(self):
|
||||
"""Test WireGuard endpoints"""
|
||||
# GET keys
|
||||
response = self.client.get('/api/wireguard/keys')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIn('public_key', data)
|
||||
|
||||
# POST generate peer keys
|
||||
response = self.client.post('/api/wireguard/keys/peer')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIn('public_key', data)
|
||||
|
||||
# GET config
|
||||
response = self.client.get('/api/wireguard/config')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIn('config', data)
|
||||
|
||||
# GET peers
|
||||
response = self.client.get('/api/wireguard/peers')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIsInstance(data, list)
|
||||
|
||||
# POST add peer
|
||||
peer_data = {'peer': 'test_peer', 'ip': '10.0.0.1', 'public_key': 'test_key'}
|
||||
response = self.client.post('/api/wireguard/peers',
|
||||
data=json.dumps(peer_data),
|
||||
content_type='application/json')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIn('success', data)
|
||||
|
||||
# DELETE remove peer
|
||||
response = self.client.delete('/api/wireguard/peers',
|
||||
data=json.dumps({'peer': 'test_peer'}),
|
||||
content_type='application/json')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIn('success', data)
|
||||
|
||||
# GET status
|
||||
response = self.client.get('/api/wireguard/status')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIn('status', data)
|
||||
|
||||
def test_api_peers_endpoints(self):
|
||||
"""Test peers endpoints"""
|
||||
# GET peers
|
||||
response = self.client.get('/api/peers')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIsInstance(data, list)
|
||||
|
||||
# POST add peer
|
||||
peer_data = {'peer': 'test_peer', 'ip': '10.0.0.1'}
|
||||
response = self.client.post('/api/peers',
|
||||
data=json.dumps(peer_data),
|
||||
content_type='application/json')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIn('success', data)
|
||||
|
||||
# DELETE remove peer
|
||||
response = self.client.delete('/api/peers/test_peer')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIn('success', data)
|
||||
|
||||
def test_api_email_endpoints(self):
|
||||
"""Test email endpoints"""
|
||||
# GET users
|
||||
response = self.client.get('/api/email/users')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIsInstance(data, list)
|
||||
|
||||
# POST create user
|
||||
user_data = {'username': 'test_user', 'email': 'test@example.com'}
|
||||
response = self.client.post('/api/email/users',
|
||||
data=json.dumps(user_data),
|
||||
content_type='application/json')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIn('success', data)
|
||||
|
||||
# DELETE user
|
||||
response = self.client.delete('/api/email/users/test_user')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIn('success', data)
|
||||
|
||||
# GET status
|
||||
response = self.client.get('/api/email/status')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIn('status', data)
|
||||
|
||||
def test_api_calendar_endpoints(self):
|
||||
"""Test calendar endpoints"""
|
||||
# GET users
|
||||
response = self.client.get('/api/calendar/users')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIsInstance(data, list)
|
||||
|
||||
# POST create user
|
||||
user_data = {'username': 'test_user', 'email': 'test@example.com'}
|
||||
response = self.client.post('/api/calendar/users',
|
||||
data=json.dumps(user_data),
|
||||
content_type='application/json')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIn('success', data)
|
||||
|
||||
# DELETE user
|
||||
response = self.client.delete('/api/calendar/users/test_user')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIn('success', data)
|
||||
|
||||
# GET status
|
||||
response = self.client.get('/api/calendar/status')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIn('status', data)
|
||||
|
||||
def test_api_files_endpoints(self):
|
||||
"""Test files endpoints"""
|
||||
# GET users
|
||||
response = self.client.get('/api/files/users')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIsInstance(data, list)
|
||||
|
||||
# POST create user
|
||||
user_data = {'username': 'test_user'}
|
||||
response = self.client.post('/api/files/users',
|
||||
data=json.dumps(user_data),
|
||||
content_type='application/json')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIn('success', data)
|
||||
|
||||
# DELETE user
|
||||
response = self.client.delete('/api/files/users/test_user')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIn('success', data)
|
||||
|
||||
# GET status
|
||||
response = self.client.get('/api/files/status')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIn('status', data)
|
||||
|
||||
def test_api_routing_endpoints(self):
|
||||
"""Test routing endpoints"""
|
||||
# GET status
|
||||
response = self.client.get('/api/routing/status')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIn('status', data)
|
||||
|
||||
# POST NAT rule
|
||||
nat_data = {'type': 'masquerade', 'interface': 'eth0'}
|
||||
response = self.client.post('/api/routing/nat',
|
||||
data=json.dumps(nat_data),
|
||||
content_type='application/json')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIn('rule_id', data)
|
||||
|
||||
# GET NAT rules
|
||||
response = self.client.get('/api/routing/nat')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIsInstance(data, list)
|
||||
|
||||
def test_api_vault_endpoints(self):
|
||||
"""Test vault endpoints"""
|
||||
# GET status
|
||||
response = self.client.get('/api/vault/status')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIn('status', data)
|
||||
|
||||
# GET certificates
|
||||
response = self.client.get('/api/vault/certificates')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIsInstance(data, list)
|
||||
|
||||
# POST generate certificate
|
||||
cert_data = {'common_name': 'test.example.com'}
|
||||
response = self.client.post('/api/vault/certificates',
|
||||
data=json.dumps(cert_data),
|
||||
content_type='application/json')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIn('certificate', data)
|
||||
|
||||
# GET CA certificate
|
||||
response = self.client.get('/api/vault/ca/certificate')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIn('certificate', data)
|
||||
|
||||
def test_api_containers_endpoints(self):
|
||||
"""Test containers endpoints"""
|
||||
# GET containers
|
||||
response = self.client.get('/api/containers')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIsInstance(data, list)
|
||||
|
||||
# POST start container
|
||||
response = self.client.post('/api/containers/test/start')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIn('success', data)
|
||||
|
||||
# POST stop container
|
||||
response = self.client.post('/api/containers/test/stop')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIn('success', data)
|
||||
|
||||
# GET container logs
|
||||
response = self.client.get('/api/containers/test/logs')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIsInstance(data, list)
|
||||
|
||||
def test_api_services_status_endpoint(self):
|
||||
"""Test GET /api/services/status endpoint"""
|
||||
response = self.client.get('/api/services/status')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIn('services', data)
|
||||
|
||||
def test_api_services_connectivity_endpoint(self):
|
||||
"""Test GET /api/services/connectivity endpoint"""
|
||||
response = self.client.get('/api/services/connectivity')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIn('results', data)
|
||||
|
||||
def test_api_health_history_endpoint(self):
|
||||
"""Test GET /api/health/history endpoint"""
|
||||
response = self.client.get('/api/health/history')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIsInstance(data, list)
|
||||
|
||||
def test_api_logs_endpoint(self):
|
||||
"""Test GET /api/logs endpoint"""
|
||||
response = self.client.get('/api/logs')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIsInstance(data, list)
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Binary file not shown.
@@ -193,7 +193,7 @@ class TestCellPermissionsApi:
|
||||
fake_dns_ip = '10.99.0.1'
|
||||
fake_invite = {
|
||||
'cell_name': 'e2etest-synthetic-cell',
|
||||
'public_key': 'AAAAFakePublicKeyForE2eTestingAAAAAAAAAAAAAAAA=',
|
||||
'public_key': 'FakePublicKeyForE2eCellTestAAAAAAAAAAAAAAAA=',
|
||||
'endpoint': '127.0.0.2:51820',
|
||||
'vpn_subnet': fake_subnet,
|
||||
'dns_ip': fake_dns_ip,
|
||||
@@ -334,7 +334,7 @@ class TestLiveCellConnection:
|
||||
|
||||
if cell2_name:
|
||||
_remove_connection(admin_client, cell2_name)
|
||||
if cell1_name:
|
||||
if cell1_name and cell2_client:
|
||||
_remove_connection(cell2_client, cell1_name)
|
||||
|
||||
def _connect_cells(self, admin_client, cell2_client,
|
||||
@@ -433,10 +433,24 @@ class TestLiveCellConnection:
|
||||
|
||||
After cell1 sets outbound.calendar=True (= cell2 gets inbound.calendar=True
|
||||
from cell1), we verify that cell2's stored remote view is updated.
|
||||
This test requires the cells to be able to reach each other's API on port 3000.
|
||||
Requires cells to reach each other's API via the WireGuard tunnel (DNS IP on
|
||||
port 3000). Skipped when the WG tunnel between cells is not active.
|
||||
"""
|
||||
cell1_name, cell2_name = self._connect_cells(admin_client, cell2_client)
|
||||
|
||||
# Verify the WG tunnel is up: cell1 must be able to reach cell2's API
|
||||
# at cell2's WireGuard DNS IP before we assert that the push succeeded.
|
||||
invite2 = _get_invite(cell2_client)
|
||||
cell2_dns_ip = invite2['dns_ip']
|
||||
import requests as _req
|
||||
try:
|
||||
_req.get(f'http://{cell2_dns_ip}:3000/health', timeout=2)
|
||||
except Exception:
|
||||
pytest.skip(
|
||||
f"Cell2 not reachable at http://{cell2_dns_ip}:3000 via WG tunnel — "
|
||||
"peer-sync push requires an active tunnel between the two cells"
|
||||
)
|
||||
|
||||
# cell1 enables outbound calendar to cell2
|
||||
inbound = {'calendar': False, 'files': False, 'mail': False, 'webdav': False}
|
||||
outbound = {'calendar': True, 'files': False, 'mail': False, 'webdav': False}
|
||||
@@ -530,7 +544,7 @@ class TestCellServiceAccessRestrictions:
|
||||
cell1_name = None
|
||||
if cell2_name:
|
||||
_remove_connection(admin_client, cell2_name)
|
||||
if cell1_name:
|
||||
if cell1_name and cell2_client:
|
||||
_remove_connection(cell2_client, cell1_name)
|
||||
|
||||
def _get_forward_rules(self, client) -> str:
|
||||
|
||||
@@ -85,7 +85,10 @@ class TestServiceAccessUpdate:
|
||||
if not rules:
|
||||
return # can't verify without iptables access — skip silently
|
||||
# No Caddy-targeted DROP for this peer; service blocking is DNS-ACL only
|
||||
caddy_drop = f'{peer_ip}' in rules and 'DROP' in rules and 'dpt:80' in rules
|
||||
caddy_drop = any(
|
||||
peer_ip in line and 'DROP' in line and 'dpt:80' in line
|
||||
for line in rules.splitlines()
|
||||
)
|
||||
assert not caddy_drop, (
|
||||
f'Found Caddy DROP rule for {peer_ip} after service_access=[] — '
|
||||
f'this blocks the PIC UI. Service access should be DNS-ACL only.\n{rules}'
|
||||
|
||||
@@ -10,7 +10,11 @@ class PicAPIClient:
|
||||
def login(self, username: str, password: str) -> dict:
|
||||
r = self.s.post(f"{self.base}/api/auth/login", json={'username': username, 'password': password})
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
data = r.json()
|
||||
csrf = data.get('csrf_token', '')
|
||||
if csrf:
|
||||
self.s.headers['X-CSRF-Token'] = csrf
|
||||
return data
|
||||
|
||||
def logout(self):
|
||||
self.s.post(f"{self.base}/api/auth/logout")
|
||||
|
||||
@@ -52,9 +52,18 @@ def build_wg_config(private_key: str, peer_ip: str, server_pubkey: str,
|
||||
|
||||
|
||||
def cleanup_stale_e2e_interfaces():
|
||||
"""Remove any leftover pic-e2e-* interfaces from previous failed runs."""
|
||||
"""Remove any leftover pic-e2e-* interfaces and nftables tables from previous failed runs."""
|
||||
result = subprocess.run(['ip', 'link', 'show'], capture_output=True, text=True)
|
||||
for line in result.stdout.splitlines():
|
||||
if 'pic-e2e-' in line:
|
||||
iface = line.split(':')[1].strip().split('@')[0]
|
||||
subprocess.run(['sudo', 'ip', 'link', 'delete', iface], capture_output=True)
|
||||
|
||||
# wg-quick creates an nftables table per interface; if the interface was never brought
|
||||
# down cleanly the table persists and drops decrypted ICMP replies on future runs.
|
||||
nft_result = subprocess.run(['sudo', 'nft', 'list', 'tables'], capture_output=True, text=True)
|
||||
for line in nft_result.stdout.splitlines():
|
||||
if 'wg-quick-pic-e2e-' in line:
|
||||
table_name = line.strip().split()[-1]
|
||||
subprocess.run(['sudo', 'nft', 'delete', 'table', 'ip', table_name],
|
||||
capture_output=True)
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
"""
|
||||
Service Store E2E tests.
|
||||
|
||||
Tests that the admin can install and remove store services via the /store page.
|
||||
Requires a running PIC stack with access to the service store index and registry.
|
||||
|
||||
Run with:
|
||||
pytest tests/e2e/ui/test_service_store.py -v --base-url http://<pic-host>
|
||||
"""
|
||||
import pytest
|
||||
|
||||
pytestmark = pytest.mark.ui
|
||||
|
||||
STORE_ROUTE = '/services'
|
||||
|
||||
# Services to install in dependency order (webmail requires email)
|
||||
INSTALL_ORDER = ['calendar', 'files', 'email', 'webmail']
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _goto_store(page, webui_base):
|
||||
page.goto(f"{webui_base}{STORE_ROUTE}")
|
||||
page.wait_for_load_state('networkidle')
|
||||
|
||||
|
||||
def _service_card(page, service_name):
|
||||
"""Return the card element containing the named service."""
|
||||
return page.locator('.card', has=page.get_by_text(service_name, exact=False)).first
|
||||
|
||||
|
||||
def _is_installed(page, service_name):
|
||||
card = _service_card(page, service_name)
|
||||
return card.get_by_text('Installed', exact=False).is_visible()
|
||||
|
||||
|
||||
def _install_service(page, webui_base, service_name, timeout_ms=180_000):
|
||||
"""Click Install on a service card and wait until the card shows Installed."""
|
||||
_goto_store(page, webui_base)
|
||||
card = _service_card(page, service_name)
|
||||
install_btn = card.get_by_role('button', name='Install')
|
||||
install_btn.click()
|
||||
# Wait for the Install button to disappear (replaced by Remove) or for
|
||||
# the Installed badge to appear — whichever comes first.
|
||||
card.get_by_text('Installed', exact=False).wait_for(state='visible', timeout=timeout_ms)
|
||||
|
||||
|
||||
def _remove_service(page, webui_base, service_name, timeout_ms=60_000):
|
||||
"""Click Uninstall on a service card and confirm, then wait until Install reappears."""
|
||||
_goto_store(page, webui_base)
|
||||
card = _service_card(page, service_name)
|
||||
card.get_by_role('button', name='Uninstall').click()
|
||||
# A confirmation dialog appears — click the confirm Uninstall button
|
||||
page.get_by_role('button', name='Uninstall Service').wait_for(state='visible', timeout=5000)
|
||||
page.get_by_role('button', name='Uninstall Service').click()
|
||||
card.get_by_role('button', name='Install').wait_for(state='visible', timeout=timeout_ms)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_store_page_loads(admin_page, webui_base):
|
||||
"""Store page must load and list available services without errors."""
|
||||
page = admin_page
|
||||
_goto_store(page, webui_base)
|
||||
|
||||
# Should not show a generic error message
|
||||
assert 'Could not load the service store' not in page.content(), (
|
||||
'Store page showed error: could not load the service store'
|
||||
)
|
||||
|
||||
# At least one service card should be visible
|
||||
cards = page.locator('.card').all()
|
||||
assert len(cards) > 0, 'No service cards found on the store page'
|
||||
|
||||
|
||||
def test_store_shows_known_services(admin_page, webui_base):
|
||||
"""Store page must list email, calendar, files, and webmail."""
|
||||
page = admin_page
|
||||
_goto_store(page, webui_base)
|
||||
|
||||
for name in ('Email Server', 'Calendar', 'File Storage', 'Webmail'):
|
||||
assert page.get_by_text(name, exact=False).first.is_visible(), (
|
||||
f"Expected service '{name}' not visible on store page"
|
||||
)
|
||||
|
||||
|
||||
def test_install_calendar(admin_page, webui_base):
|
||||
"""Admin can install the calendar service."""
|
||||
page = admin_page
|
||||
_goto_store(page, webui_base)
|
||||
|
||||
if _is_installed(page, 'Calendar'):
|
||||
pytest.skip('calendar already installed — skipping install test')
|
||||
|
||||
_install_service(page, webui_base, 'Calendar & Contacts', timeout_ms=180_000)
|
||||
assert _is_installed(page, 'Calendar'), (
|
||||
'Calendar service card did not show Installed after install'
|
||||
)
|
||||
|
||||
|
||||
def test_install_files(admin_page, webui_base):
|
||||
"""Admin can install the file storage service."""
|
||||
page = admin_page
|
||||
_goto_store(page, webui_base)
|
||||
|
||||
if _is_installed(page, 'File Storage'):
|
||||
pytest.skip('files already installed — skipping install test')
|
||||
|
||||
_install_service(page, webui_base, 'File Storage', timeout_ms=180_000)
|
||||
assert _is_installed(page, 'File Storage'), (
|
||||
'Files service card did not show Installed after install'
|
||||
)
|
||||
|
||||
|
||||
def test_install_email(admin_page, webui_base):
|
||||
"""Admin can install the email service."""
|
||||
page = admin_page
|
||||
_goto_store(page, webui_base)
|
||||
|
||||
if _is_installed(page, 'Email Server'):
|
||||
pytest.skip('email already installed — skipping install test')
|
||||
|
||||
_install_service(page, webui_base, 'Email Server', timeout_ms=300_000)
|
||||
assert _is_installed(page, 'Email Server'), (
|
||||
'Email service card did not show Installed after install'
|
||||
)
|
||||
|
||||
|
||||
def test_install_webmail(admin_page, webui_base):
|
||||
"""Admin can install webmail after email is installed."""
|
||||
page = admin_page
|
||||
_goto_store(page, webui_base)
|
||||
|
||||
if not _is_installed(page, 'Email Server'):
|
||||
pytest.skip('email not installed — webmail requires email first')
|
||||
|
||||
if _is_installed(page, 'Webmail'):
|
||||
pytest.skip('webmail already installed — skipping install test')
|
||||
|
||||
_install_service(page, webui_base, 'Webmail', timeout_ms=180_000)
|
||||
assert _is_installed(page, 'Webmail'), (
|
||||
'Webmail service card did not show Installed after install'
|
||||
)
|
||||
|
||||
|
||||
def test_installed_services_appear_on_dashboard(admin_page, webui_base):
|
||||
"""After installation, services should appear as links on the dashboard."""
|
||||
page = admin_page
|
||||
_goto_store(page, webui_base)
|
||||
|
||||
page.goto(f"{webui_base}/")
|
||||
page.wait_for_load_state('networkidle')
|
||||
|
||||
# Check that at least the Cell Home link is present
|
||||
assert page.get_by_text('Cell Home', exact=False).is_visible(), (
|
||||
'Dashboard does not show the Cell Home service link'
|
||||
)
|
||||
|
||||
|
||||
def test_uninstall_webmail(admin_page, webui_base):
|
||||
"""Admin can uninstall the webmail service."""
|
||||
page = admin_page
|
||||
_goto_store(page, webui_base)
|
||||
|
||||
if not _is_installed(page, 'Webmail'):
|
||||
pytest.skip('webmail not installed — skipping uninstall test')
|
||||
|
||||
_remove_service(page, webui_base, 'Webmail')
|
||||
assert not _is_installed(page, 'Webmail'), (
|
||||
'Webmail service card still shows Installed after uninstall'
|
||||
)
|
||||
@@ -39,10 +39,27 @@ def wg_server_info(admin_client, pic_host):
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Server VPN IP (e.g. '10.0.0.1') and subnet (e.g. '10.0.0.0/24') from status
|
||||
server_address = '10.0.0.1/24'
|
||||
try:
|
||||
server_address = admin_client.get('/api/wireguard/status').json().get('address', server_address)
|
||||
except Exception:
|
||||
pass
|
||||
import ipaddress as _ip
|
||||
try:
|
||||
iface = _ip.ip_interface(server_address)
|
||||
server_ip = str(iface.ip)
|
||||
server_network = str(iface.network)
|
||||
except Exception:
|
||||
server_ip = '10.0.0.1'
|
||||
server_network = '10.0.0.0/24'
|
||||
|
||||
return {
|
||||
'public_key': server_pubkey,
|
||||
'endpoint': pic_host,
|
||||
'port': int(port),
|
||||
'server_ip': server_ip,
|
||||
'server_network': server_network,
|
||||
}
|
||||
|
||||
|
||||
@@ -65,7 +82,7 @@ def connected_peer(make_peer, wg_server_info, tmp_path):
|
||||
server_pubkey=wg_server_info['public_key'],
|
||||
server_endpoint=wg_server_info['endpoint'],
|
||||
server_port=wg_server_info['port'],
|
||||
allowed_ips='10.0.0.0/24',
|
||||
allowed_ips=wg_server_info['server_network'],
|
||||
)
|
||||
|
||||
# Write config with restricted permissions
|
||||
@@ -78,6 +95,7 @@ def connected_peer(make_peer, wg_server_info, tmp_path):
|
||||
iface.bring_up()
|
||||
peer['iface'] = iface
|
||||
peer['conf_path'] = conf_path
|
||||
peer['server_ip'] = wg_server_info['server_ip']
|
||||
yield peer
|
||||
finally:
|
||||
iface.bring_down()
|
||||
|
||||
@@ -32,7 +32,8 @@ def _config(admin_client) -> dict:
|
||||
|
||||
|
||||
def _domain(admin_client) -> str:
|
||||
return _config(admin_client).get('domain') or 'lan'
|
||||
cfg = _config(admin_client)
|
||||
return cfg.get('domain_name') or cfg.get('domain') or 'lan'
|
||||
|
||||
|
||||
def _dns_ip(admin_client) -> str:
|
||||
@@ -66,16 +67,27 @@ def _curl_host(ip: str, host: str, path: str = '/', timeout: int = 8) -> tuple[i
|
||||
|
||||
|
||||
def _curl_domain(host: str, path: str = '/', dns_ip: str = '', timeout: int = 8) -> tuple[int, str]:
|
||||
"""Make an HTTP request using curl's --dns-servers to resolve via CoreDNS."""
|
||||
cmd = ['curl', '-s', '--connect-timeout', '5',
|
||||
'-w', '\n__HTTP_CODE__:%{http_code}',
|
||||
f'http://{host}{path}']
|
||||
"""Make an HTTP request to host, optionally resolving via a custom DNS server.
|
||||
|
||||
Uses dig to resolve the host (avoiding --dns-servers which requires c-ares),
|
||||
then curls to the resolved IP with the original Host header.
|
||||
"""
|
||||
if dns_ip:
|
||||
cmd = ['curl', '-s', '--connect-timeout', '5',
|
||||
'--dns-servers', dns_ip,
|
||||
'-w', '\n__HTTP_CODE__:%{http_code}',
|
||||
f'http://{host}{path}']
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout)
|
||||
dig = subprocess.run(
|
||||
['dig', f'@{dns_ip}', host, 'A', '+short', '+time=3', '+tries=1'],
|
||||
capture_output=True, text=True, timeout=5,
|
||||
)
|
||||
resolved_ips = [line for line in dig.stdout.strip().splitlines() if line and not line.startswith(';')]
|
||||
if resolved_ips:
|
||||
return _curl_host(resolved_ips[0], host, path, timeout)
|
||||
return 0, ''
|
||||
|
||||
result = subprocess.run(
|
||||
['curl', '-s', '--connect-timeout', '5',
|
||||
'-w', '\n__HTTP_CODE__:%{http_code}',
|
||||
f'http://{host}{path}'],
|
||||
capture_output=True, text=True, timeout=timeout,
|
||||
)
|
||||
output = result.stdout
|
||||
body = ''
|
||||
code = 0
|
||||
@@ -92,19 +104,21 @@ def _curl_domain(host: str, path: str = '/', dns_ip: str = '', timeout: int = 8)
|
||||
# ── Scenario 35: api.<domain> routes to API ───────────────────────────────────
|
||||
|
||||
def test_api_domain_returns_json_not_webui(connected_peer, admin_client):
|
||||
"""api.<domain>/api/status must return JSON, not the React WebUI HTML."""
|
||||
"""api.<domain>/api/status must return JSON or a redirect, not the React WebUI HTML."""
|
||||
dom = _domain(admin_client)
|
||||
dns_ip = _dns_ip(admin_client)
|
||||
code, body = _curl_domain(f'api.{dom}', '/api/status', dns_ip)
|
||||
assert code not in (0, 000), f"curl to api.{dom}/api/status failed (code {code})"
|
||||
assert code not in (0,), f"curl to api.{dom}/api/status failed completely (code {code})"
|
||||
# 3xx means Caddy is routing (HTTP→HTTPS redirect in pic_ngo mode) — acceptable
|
||||
if code in (301, 302, 308):
|
||||
return
|
||||
assert _WEBUI_MARKER not in body, (
|
||||
f"api.{dom}/api/status returned WebUI HTML — "
|
||||
"Caddy is not routing api.<domain> to the API; "
|
||||
"check that the http://api.<domain> block exists in the Caddyfile "
|
||||
"and uses the configured domain (not a stale .cell or .dev TLD)"
|
||||
"check that the api.<domain> block exists in the Caddyfile"
|
||||
)
|
||||
assert '{' in body or '"' in body, (
|
||||
f"api.{dom}/api/status did not return JSON (body: {body[:100]!r})"
|
||||
f"api.{dom}/api/status did not return JSON (code={code}, body: {body[:100]!r})"
|
||||
)
|
||||
|
||||
|
||||
@@ -243,9 +257,16 @@ def test_vip_direct_access_not_webui(connected_peer, vip, expected_not):
|
||||
|
||||
# ── Scenario 41: Catch-all :80 routes API path correctly ─────────────────────
|
||||
|
||||
def test_catchall_api_path_returns_json(connected_peer):
|
||||
"""The catch-all :80 block must route /api/* to the API (not WebUI)."""
|
||||
def test_catchall_api_path_returns_json(connected_peer, admin_client):
|
||||
"""The catch-all :80 block must route /api/* to the API (not WebUI).
|
||||
|
||||
Only applicable to HTTP-mode cells (e.g. lan/local domain). Cells using
|
||||
pic_ngo / duckdns HTTPS mode have no catch-all :80 block — Caddy redirects
|
||||
all plain-HTTP to HTTPS instead.
|
||||
"""
|
||||
code, body = _curl_host('172.20.0.2', 'localhost', '/api/status')
|
||||
if code in (301, 302, 308):
|
||||
pytest.skip("Caddy is in HTTPS-redirect mode — no catch-all :80 block (expected for pic_ngo cells)")
|
||||
assert _WEBUI_MARKER not in body, (
|
||||
"Catch-all :80 returned WebUI HTML for /api/status — "
|
||||
"the `handle /api/*` directive in the :80 block is missing or wrong"
|
||||
@@ -255,9 +276,14 @@ def test_catchall_api_path_returns_json(connected_peer):
|
||||
)
|
||||
|
||||
|
||||
def test_catchall_root_serves_webui(connected_peer):
|
||||
"""The catch-all :80 block serves the WebUI for the root path."""
|
||||
def test_catchall_root_serves_webui(connected_peer, admin_client):
|
||||
"""The catch-all :80 block serves the WebUI for the root path.
|
||||
|
||||
Only applicable to HTTP-mode cells. HTTPS-mode cells redirect :80 → :443.
|
||||
"""
|
||||
code, body = _curl_host('172.20.0.2', 'localhost', '/')
|
||||
if code in (301, 302, 308):
|
||||
pytest.skip("Caddy is in HTTPS-redirect mode — no catch-all :80 block (expected for pic_ngo cells)")
|
||||
assert _WEBUI_MARKER in body, (
|
||||
"Catch-all :80 / did not return WebUI HTML — "
|
||||
"something is broken with the catch-all :80 block"
|
||||
@@ -269,7 +295,10 @@ def test_catchall_root_serves_webui(connected_peer):
|
||||
def test_caddy_does_not_route_cell_tld(connected_peer):
|
||||
"""Caddy must NOT have active routing for .cell domains — they are from old config."""
|
||||
code, body = _curl_host('172.20.0.2', 'calendar.cell', '/')
|
||||
assert _WEBUI_MARKER in body or code in (0, 404, 502, 503), (
|
||||
"Caddy is still routing calendar.cell — stale .cell blocks remain in config. "
|
||||
# 3xx redirects (e.g. HTTP→HTTPS) are acceptable — they mean Caddy is active but
|
||||
# not serving a functional response. Only a 200-with-content or WebUI HTML is a problem.
|
||||
assert _WEBUI_MARKER in body or code in (0, 301, 302, 308, 404, 502, 503), (
|
||||
"Caddy is still routing calendar.cell with a functional response — "
|
||||
"stale .cell blocks remain in config. "
|
||||
"Check that write_caddyfile() is writing to the correct path that Caddy reads."
|
||||
)
|
||||
|
||||
@@ -7,8 +7,9 @@ pytestmark = pytest.mark.wg
|
||||
def test_wg_connect_and_ping_server(connected_peer):
|
||||
"""Scenario 25+26: create peer, connect, ping server VPN IP."""
|
||||
iface = connected_peer['iface']
|
||||
server_ip = connected_peer.get('server_ip', '10.0.0.1')
|
||||
assert iface.up, "WireGuard interface should be up"
|
||||
assert iface.is_connected('10.0.0.1'), "Server VPN IP 10.0.0.1 should be reachable via WireGuard"
|
||||
assert iface.is_connected(server_ip), f"Server VPN IP {server_ip} should be reachable via WireGuard"
|
||||
|
||||
|
||||
def test_wg_peer_has_assigned_ip(connected_peer):
|
||||
@@ -21,8 +22,9 @@ def test_wg_peer_has_assigned_ip(connected_peer):
|
||||
def test_wg_disconnect_removes_route(connected_peer):
|
||||
"""Scenario 29: after disconnect, VPN IP is not reachable."""
|
||||
iface = connected_peer['iface']
|
||||
server_ip = connected_peer.get('server_ip', '10.0.0.1')
|
||||
iface.bring_down()
|
||||
result = subprocess.run(['ping', '-c', '1', '-W', '2', '10.0.0.1'],
|
||||
result = subprocess.run(['ping', '-c', '1', '-W', '2', server_ip],
|
||||
capture_output=True, timeout=5)
|
||||
# After disconnect, ping should fail
|
||||
assert result.returncode != 0, "VPN IP should not be reachable after disconnect"
|
||||
|
||||
@@ -19,17 +19,18 @@ import pytest
|
||||
|
||||
pytestmark = pytest.mark.wg
|
||||
|
||||
# Subdomain → expected offset in ip_utils.CONTAINER_OFFSETS / VIP list.
|
||||
# These are the sub-names, not full FQDNs — the TLD is fetched from config.
|
||||
SUBDOMAINS_TO_IPS = {
|
||||
'api': '172.20.0.2', # must route through Caddy (not API container direct)
|
||||
'webui': '172.20.0.2', # must route through Caddy
|
||||
'calendar': '172.20.0.21', # Caddy VIP for CalDAV
|
||||
'files': '172.20.0.22', # Caddy VIP for Filegator
|
||||
'mail': '172.20.0.23', # Caddy VIP for Rainloop
|
||||
'webmail': '172.20.0.23', # alias for mail VIP
|
||||
'webdav': '172.20.0.24', # Caddy VIP for WebDAV
|
||||
}
|
||||
# Subdomain → service_ips key for the expected VIP (None = always Caddy).
|
||||
# Expected IP is read dynamically from /api/config service_ips; falls back to
|
||||
# Caddy IP (172.20.0.2) when the service is not enabled / VIP not configured.
|
||||
_SUBDOMAIN_VIP_KEYS = [
|
||||
('api', None),
|
||||
('webui', None),
|
||||
('calendar', 'vip_calendar'),
|
||||
('files', 'vip_files'),
|
||||
('mail', 'vip_mail'),
|
||||
('webmail', 'vip_mail'),
|
||||
('webdav', 'vip_webdav'),
|
||||
]
|
||||
|
||||
|
||||
# ── helpers ───────────────────────────────────────────────────────────────────
|
||||
@@ -45,8 +46,9 @@ def _dns_ip(admin_client) -> str:
|
||||
|
||||
|
||||
def _domain(admin_client) -> str:
|
||||
"""Return the configured cell domain (e.g. 'lan', 'dev', 'home')."""
|
||||
return _config(admin_client).get('domain') or 'lan'
|
||||
"""Return the cell's fully-qualified domain (e.g. 'test5.pic.ngo', 'lan')."""
|
||||
cfg = _config(admin_client)
|
||||
return cfg.get('domain_name') or cfg.get('domain') or 'lan'
|
||||
|
||||
|
||||
def _cell_name(admin_client) -> str:
|
||||
@@ -55,12 +57,24 @@ def _cell_name(admin_client) -> str:
|
||||
|
||||
# ── Scenario 30: DNS resolution ───────────────────────────────────────────────
|
||||
|
||||
@pytest.mark.parametrize('subdomain,expected_ip', list(SUBDOMAINS_TO_IPS.items()))
|
||||
def test_service_domain_resolves_to_expected_ip(connected_peer, admin_client, subdomain, expected_ip):
|
||||
@pytest.mark.parametrize('subdomain,vip_key', _SUBDOMAIN_VIP_KEYS)
|
||||
def test_service_domain_resolves_to_expected_ip(connected_peer, admin_client, subdomain, vip_key):
|
||||
"""Each service subdomain resolves to the correct IP via CoreDNS.
|
||||
|
||||
The full FQDN is built from the configured domain — not hardcoded to any TLD.
|
||||
The expected IP is read from service_ips; falls back to Caddy when the VIP is
|
||||
not configured (e.g. when the service is disabled).
|
||||
"""
|
||||
cfg = _config(admin_client)
|
||||
sips = cfg.get('service_ips', {})
|
||||
caddy_ip = sips.get('caddy', '172.20.0.2')
|
||||
# Accept both the specific VIP IP and Caddy IP: some zone files use per-service
|
||||
# VIP records (172.20.0.21 etc.) while others use a wildcard pointing to Caddy.
|
||||
# Both are correct deployments; what matters is that the domain resolves at all.
|
||||
expected_ips = {caddy_ip}
|
||||
if vip_key and sips.get(vip_key):
|
||||
expected_ips.add(sips[vip_key])
|
||||
|
||||
dns_ip = _dns_ip(admin_client)
|
||||
dom = _domain(admin_client)
|
||||
fqdn = f'{subdomain}.{dom}'
|
||||
@@ -70,8 +84,8 @@ def test_service_domain_resolves_to_expected_ip(connected_peer, admin_client, su
|
||||
)
|
||||
assert result.returncode == 0, f"dig failed for {fqdn}: {result.stderr}"
|
||||
resolved = result.stdout.strip()
|
||||
assert resolved == expected_ip, (
|
||||
f"{fqdn} resolved to {resolved!r}, expected {expected_ip}. "
|
||||
assert resolved in expected_ips, (
|
||||
f"{fqdn} resolved to {resolved!r}, expected one of {expected_ips}. "
|
||||
f"DNS server: {dns_ip}, configured domain: {dom!r}"
|
||||
)
|
||||
|
||||
@@ -136,30 +150,43 @@ def test_caddy_ip_serves_http(connected_peer):
|
||||
# ── Scenario 32: HTTP via domain ──────────────────────────────────────────────
|
||||
|
||||
def test_http_api_domain_reaches_api(connected_peer, admin_client):
|
||||
"""curl http://api.<domain>/api/status returns a JSON response via Caddy + CoreDNS."""
|
||||
"""api.<domain>/api/status is reachable via Caddy routing + CoreDNS resolution."""
|
||||
dom = _domain(admin_client)
|
||||
dns_ip = _dns_ip(admin_client)
|
||||
result = subprocess.run(
|
||||
['curl', '-s', '--connect-timeout', '5',
|
||||
'--dns-servers', dns_ip,
|
||||
f'http://api.{dom}/api/status'],
|
||||
fqdn = f'api.{dom}'
|
||||
|
||||
# Resolve via CoreDNS (--dns-servers requires c-ares; use dig instead)
|
||||
dig = subprocess.run(
|
||||
['dig', f'@{dns_ip}', fqdn, 'A', '+short', '+time=5'],
|
||||
capture_output=True, text=True, timeout=10,
|
||||
)
|
||||
assert result.stdout.strip(), (
|
||||
f"curl http://api.{dom}/api/status returned no output via DNS {dns_ip}. "
|
||||
resolved_ips = [l for l in dig.stdout.strip().splitlines() if l and not l.startswith(';')]
|
||||
if not resolved_ips:
|
||||
pytest.skip(f"api.{dom} does not resolve via CoreDNS at {dns_ip} — DNS may not be configured")
|
||||
resolved_ip = resolved_ips[0]
|
||||
|
||||
result = subprocess.run(
|
||||
['curl', '-s', '--connect-timeout', '5',
|
||||
'-H', f'Host: {fqdn}',
|
||||
f'http://{resolved_ip}/api/status'],
|
||||
capture_output=True, text=True, timeout=10,
|
||||
)
|
||||
# 3xx means Caddy is redirecting HTTP→HTTPS (normal for pic_ngo mode)
|
||||
stdout = result.stdout.strip()
|
||||
assert result.returncode == 0 or stdout, (
|
||||
f"curl to {resolved_ip} with Host: {fqdn} failed. "
|
||||
f"stderr: {result.stderr[:200]}"
|
||||
)
|
||||
|
||||
|
||||
# ── Scenario 33: Config DNS field ─────────────────────────────────────────────
|
||||
|
||||
def test_peer_services_config_has_coredns_not_vpn_gateway(admin_client, make_peer):
|
||||
def test_peer_services_config_has_coredns_not_vpn_gateway(admin_client, make_peer, api_base):
|
||||
"""WireGuard config in /api/peer/services must use CoreDNS IP, not 10.0.0.1."""
|
||||
from helpers.api_client import PicAPIClient
|
||||
import os
|
||||
|
||||
peer = make_peer('e2etest-dns-config', password='DnsTest123!')
|
||||
peer_client = PicAPIClient(os.environ.get('PIC_API_BASE', 'http://192.168.31.51:3000'))
|
||||
peer_client = PicAPIClient(api_base)
|
||||
peer_client.login(peer['name'], 'DnsTest123!')
|
||||
|
||||
r = peer_client.get('/api/peer/services')
|
||||
@@ -188,14 +215,13 @@ def test_peer_services_config_has_coredns_not_vpn_gateway(admin_client, make_pee
|
||||
break
|
||||
|
||||
|
||||
def test_peer_services_caldav_url_uses_configured_domain(admin_client, make_peer):
|
||||
def test_peer_services_caldav_url_uses_configured_domain(admin_client, make_peer, api_base):
|
||||
"""CalDAV URL must use the configured domain, not hardcode 'radicale.dev:5232'."""
|
||||
from helpers.api_client import PicAPIClient
|
||||
import os
|
||||
|
||||
dom = _domain(admin_client)
|
||||
peer = make_peer('e2etest-caldav-url', password='CaldavTest123!')
|
||||
peer_client = PicAPIClient(os.environ.get('PIC_API_BASE', 'http://192.168.31.51:3000'))
|
||||
peer_client = PicAPIClient(api_base)
|
||||
peer_client.login(peer['name'], 'CaldavTest123!')
|
||||
|
||||
r = peer_client.get('/api/peer/services')
|
||||
|
||||
@@ -6,14 +6,14 @@ pytestmark = [pytest.mark.wg, pytest.mark.requires_internet]
|
||||
|
||||
def test_full_tunnel_routes_all_traffic(full_tunnel_peer):
|
||||
"""Scenario 30: with AllowedIPs=0.0.0.0/0, external traffic routes through VPN."""
|
||||
# Check routing table — 0.0.0.0/0 should be via the WG interface
|
||||
result = subprocess.run(['ip', 'route', 'show'], capture_output=True, text=True)
|
||||
# wg-quick adds full-tunnel routes to a policy routing table (not the main table),
|
||||
# so we must check all tables to find the 0.0.0.0/1 + 128.0.0.0/1 split routes.
|
||||
result = subprocess.run(['ip', 'route', 'show', 'table', 'all'],
|
||||
capture_output=True, text=True)
|
||||
iface_name = full_tunnel_peer['iface'].iface_name
|
||||
# In full tunnel mode, the default route or the 0.0.0.0/1 + 128.0.0.0/1 split routes
|
||||
# point to the WG interface
|
||||
assert (iface_name in result.stdout or
|
||||
'0.0.0.0/1' in result.stdout or
|
||||
'128.0.0.0/1' in result.stdout), "Full tunnel routes not found"
|
||||
'128.0.0.0/1' in result.stdout), "Full tunnel routes not found in any routing table"
|
||||
|
||||
|
||||
@pytest.mark.requires_internet
|
||||
|
||||
@@ -0,0 +1,531 @@
|
||||
"""
|
||||
Tests for AccountManager — per-service credential provisioning.
|
||||
|
||||
Covers:
|
||||
- provision: dispatches to right manager method, stores credentials, generates password
|
||||
- deprovision: calls manager method, removes stored credentials
|
||||
- get_credentials / list_accounts / list_peer_services
|
||||
- deprovision_peer: bulk cleanup on peer deletion
|
||||
- store_credentials: direct storage (used by peers-POST legacy route)
|
||||
- get_all_credentials: returns all creds for a peer
|
||||
- credential file is created with 0o600
|
||||
- unknown service / missing manager errors
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import stat
|
||||
import threading
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import sys
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'api'))
|
||||
|
||||
from account_manager import AccountManager
|
||||
|
||||
|
||||
# ── helpers ────────────────────────────────────────────────────────────────────
|
||||
|
||||
def _make_am(tmp_path: Path, registry=None, **managers) -> AccountManager:
|
||||
if registry is None:
|
||||
registry = _make_registry()
|
||||
return AccountManager(service_registry=registry, data_dir=str(tmp_path), **managers)
|
||||
|
||||
|
||||
def _make_registry(services=None):
|
||||
reg = MagicMock()
|
||||
if services is None:
|
||||
services = {
|
||||
'email': {
|
||||
'id': 'email', 'kind': 'builtin',
|
||||
'accounts': {'manager': 'email_manager', 'credentials': ['password']},
|
||||
'config': {'domain': 'example.com', 'smtp_port': 25},
|
||||
},
|
||||
'calendar': {
|
||||
'id': 'calendar', 'kind': 'builtin',
|
||||
'accounts': {'manager': 'calendar_manager', 'credentials': ['password']},
|
||||
'config': {},
|
||||
},
|
||||
'files': {
|
||||
'id': 'files', 'kind': 'builtin',
|
||||
'accounts': {'manager': 'file_manager', 'credentials': ['password']},
|
||||
'config': {},
|
||||
},
|
||||
}
|
||||
reg.get.side_effect = lambda svc_id: services.get(svc_id)
|
||||
return reg
|
||||
|
||||
|
||||
def _make_email_mgr(ok=True):
|
||||
m = MagicMock()
|
||||
m.create_email_user.return_value = ok
|
||||
m.delete_email_user.return_value = ok
|
||||
return m
|
||||
|
||||
|
||||
def _make_cal_mgr(ok=True):
|
||||
m = MagicMock()
|
||||
m.create_calendar_user.return_value = ok
|
||||
m.delete_calendar_user.return_value = ok
|
||||
return m
|
||||
|
||||
|
||||
def _make_file_mgr(ok=True):
|
||||
m = MagicMock()
|
||||
m.create_user.return_value = ok
|
||||
m.delete_user.return_value = ok
|
||||
return m
|
||||
|
||||
|
||||
# ── Provision ─────────────────────────────────────────────────────────────────
|
||||
|
||||
class TestProvision(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
import tempfile
|
||||
self.tmp = Path(tempfile.mkdtemp())
|
||||
self.email_mgr = _make_email_mgr()
|
||||
self.cal_mgr = _make_cal_mgr()
|
||||
self.file_mgr = _make_file_mgr()
|
||||
self.am = _make_am(
|
||||
self.tmp,
|
||||
email_manager=self.email_mgr,
|
||||
calendar_manager=self.cal_mgr,
|
||||
file_manager=self.file_mgr,
|
||||
)
|
||||
|
||||
def test_provision_email_calls_create_email_user(self):
|
||||
self.am.provision('email', 'alice', password='s3cret')
|
||||
self.email_mgr.create_email_user.assert_called_once_with('alice', 'example.com', 's3cret')
|
||||
|
||||
def test_provision_calendar_calls_create_calendar_user(self):
|
||||
self.am.provision('calendar', 'alice', password='s3cret')
|
||||
self.cal_mgr.create_calendar_user.assert_called_once_with('alice', 's3cret')
|
||||
|
||||
def test_provision_files_calls_create_user(self):
|
||||
self.am.provision('files', 'alice', password='s3cret')
|
||||
self.file_mgr.create_user.assert_called_once_with('alice', 's3cret')
|
||||
|
||||
def test_provision_generates_password_when_none_given(self):
|
||||
creds = self.am.provision('email', 'alice')
|
||||
self.assertIn('password', creds)
|
||||
self.assertTrue(len(creds['password']) >= 16)
|
||||
|
||||
def test_provision_returns_credential_dict(self):
|
||||
creds = self.am.provision('email', 'alice', password='mypassword')
|
||||
self.assertEqual(creds, {'password': 'mypassword'})
|
||||
|
||||
def test_provision_stores_credentials(self):
|
||||
self.am.provision('email', 'alice', password='pw')
|
||||
stored = self.am.get_credentials('email', 'alice')
|
||||
self.assertEqual(stored, {'password': 'pw'})
|
||||
|
||||
def test_provision_multiple_peers_stored_independently(self):
|
||||
self.am.provision('email', 'alice', password='pw-alice')
|
||||
self.am.provision('email', 'bob', password='pw-bob')
|
||||
self.assertEqual(self.am.get_credentials('email', 'alice'), {'password': 'pw-alice'})
|
||||
self.assertEqual(self.am.get_credentials('email', 'bob'), {'password': 'pw-bob'})
|
||||
|
||||
def test_provision_raises_for_unknown_service(self):
|
||||
with self.assertRaises(ValueError):
|
||||
self.am.provision('doesnotexist', 'alice')
|
||||
|
||||
def test_provision_raises_when_service_has_no_accounts(self):
|
||||
reg = _make_registry({'nosvc': {'id': 'nosvc', 'accounts': {}, 'config': {}}})
|
||||
am = _make_am(self.tmp, registry=reg, email_manager=self.email_mgr)
|
||||
with self.assertRaises(ValueError):
|
||||
am.provision('nosvc', 'alice')
|
||||
|
||||
def test_provision_raises_when_manager_not_registered(self):
|
||||
am = _make_am(self.tmp) # no managers passed
|
||||
with self.assertRaises(ValueError):
|
||||
am.provision('email', 'alice')
|
||||
|
||||
def test_provision_raises_runtime_error_when_manager_returns_false(self):
|
||||
am = _make_am(self.tmp, email_manager=_make_email_mgr(ok=False))
|
||||
with self.assertRaises(RuntimeError):
|
||||
am.provision('email', 'alice')
|
||||
|
||||
def test_provision_email_raises_when_domain_not_configured(self):
|
||||
reg = _make_registry({'email': {
|
||||
'id': 'email', 'accounts': {'manager': 'email_manager'},
|
||||
'config': {'domain': ''},
|
||||
}})
|
||||
am = _make_am(self.tmp, registry=reg, email_manager=self.email_mgr)
|
||||
with self.assertRaises(ValueError):
|
||||
am.provision('email', 'alice')
|
||||
|
||||
|
||||
# ── Credential file permissions ───────────────────────────────────────────────
|
||||
|
||||
class TestCredentialFilePermissions(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
import tempfile
|
||||
self.tmp = Path(tempfile.mkdtemp())
|
||||
self.am = _make_am(self.tmp, email_manager=_make_email_mgr())
|
||||
|
||||
def test_credentials_file_created_with_0600(self):
|
||||
self.am.provision('email', 'alice', password='pw')
|
||||
creds_path = self.tmp / 'peer_service_credentials.json'
|
||||
mode = stat.S_IMODE(creds_path.stat().st_mode)
|
||||
self.assertEqual(mode, 0o600, f'Expected 0o600, got {oct(mode)}')
|
||||
|
||||
|
||||
# ── Deprovision ───────────────────────────────────────────────────────────────
|
||||
|
||||
class TestDeprovision(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
import tempfile
|
||||
self.tmp = Path(tempfile.mkdtemp())
|
||||
self.email_mgr = _make_email_mgr()
|
||||
self.cal_mgr = _make_cal_mgr()
|
||||
self.file_mgr = _make_file_mgr()
|
||||
self.am = _make_am(
|
||||
self.tmp,
|
||||
email_manager=self.email_mgr,
|
||||
calendar_manager=self.cal_mgr,
|
||||
file_manager=self.file_mgr,
|
||||
)
|
||||
self.am.provision('email', 'alice', password='pw')
|
||||
|
||||
def test_deprovision_email_calls_delete_email_user(self):
|
||||
self.am.deprovision('email', 'alice')
|
||||
self.email_mgr.delete_email_user.assert_called_once_with('alice', 'example.com')
|
||||
|
||||
def test_deprovision_removes_stored_credentials(self):
|
||||
self.am.deprovision('email', 'alice')
|
||||
self.assertIsNone(self.am.get_credentials('email', 'alice'))
|
||||
|
||||
def test_deprovision_returns_true_on_success(self):
|
||||
ok = self.am.deprovision('email', 'alice')
|
||||
self.assertTrue(ok)
|
||||
|
||||
def test_deprovision_raises_for_unknown_service(self):
|
||||
with self.assertRaises(ValueError):
|
||||
self.am.deprovision('ghost', 'alice')
|
||||
|
||||
def test_deprovision_removes_service_entry_when_last_peer_gone(self):
|
||||
self.am.deprovision('email', 'alice')
|
||||
creds_file = self.tmp / 'peer_service_credentials.json'
|
||||
data = json.loads(creds_file.read_text())
|
||||
self.assertNotIn('email', data)
|
||||
|
||||
def test_deprovision_calendar_calls_delete_calendar_user(self):
|
||||
self.am.provision('calendar', 'alice', password='pw')
|
||||
self.am.deprovision('calendar', 'alice')
|
||||
self.cal_mgr.delete_calendar_user.assert_called_once_with('alice')
|
||||
|
||||
def test_deprovision_files_calls_delete_user(self):
|
||||
self.am.provision('files', 'alice', password='pw')
|
||||
self.am.deprovision('files', 'alice')
|
||||
self.file_mgr.delete_user.assert_called_once_with('alice')
|
||||
|
||||
|
||||
# ── Queries ───────────────────────────────────────────────────────────────────
|
||||
|
||||
class TestQueries(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
import tempfile
|
||||
self.tmp = Path(tempfile.mkdtemp())
|
||||
self.am = _make_am(
|
||||
self.tmp,
|
||||
email_manager=_make_email_mgr(),
|
||||
calendar_manager=_make_cal_mgr(),
|
||||
file_manager=_make_file_mgr(),
|
||||
)
|
||||
self.am.provision('email', 'alice', password='pw-alice-email')
|
||||
self.am.provision('email', 'bob', password='pw-bob-email')
|
||||
self.am.provision('calendar', 'alice', password='pw-alice-cal')
|
||||
|
||||
def test_get_credentials_returns_stored(self):
|
||||
self.assertEqual(self.am.get_credentials('email', 'alice'), {'password': 'pw-alice-email'})
|
||||
|
||||
def test_get_credentials_returns_none_for_unknown_peer(self):
|
||||
self.assertIsNone(self.am.get_credentials('email', 'nobody'))
|
||||
|
||||
def test_get_credentials_returns_none_for_unknown_service(self):
|
||||
self.assertIsNone(self.am.get_credentials('ghost', 'alice'))
|
||||
|
||||
def test_list_accounts_returns_provisioned_peers(self):
|
||||
accounts = self.am.list_accounts('email')
|
||||
self.assertIn('alice', accounts)
|
||||
self.assertIn('bob', accounts)
|
||||
|
||||
def test_list_accounts_empty_for_unprovisioned_service(self):
|
||||
self.assertEqual(self.am.list_accounts('files'), [])
|
||||
|
||||
def test_list_peer_services_returns_all_services_for_peer(self):
|
||||
services = self.am.list_peer_services('alice')
|
||||
self.assertIn('email', services)
|
||||
self.assertIn('calendar', services)
|
||||
|
||||
def test_list_peer_services_returns_empty_for_unknown_peer(self):
|
||||
self.assertEqual(self.am.list_peer_services('nobody'), [])
|
||||
|
||||
def test_is_provisioned_true_when_account_exists(self):
|
||||
self.assertTrue(self.am.is_provisioned('email', 'alice'))
|
||||
|
||||
def test_is_provisioned_false_when_no_account(self):
|
||||
self.assertFalse(self.am.is_provisioned('email', 'nobody'))
|
||||
|
||||
def test_get_all_credentials_returns_all_services(self):
|
||||
all_creds = self.am.get_all_credentials('alice')
|
||||
self.assertIn('email', all_creds)
|
||||
self.assertIn('calendar', all_creds)
|
||||
self.assertEqual(all_creds['email'], {'password': 'pw-alice-email'})
|
||||
|
||||
def test_get_all_credentials_empty_for_unknown_peer(self):
|
||||
self.assertEqual(self.am.get_all_credentials('nobody'), {})
|
||||
|
||||
|
||||
# ── Bulk deprovision ──────────────────────────────────────────────────────────
|
||||
|
||||
class TestDeprovisionPeer(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
import tempfile
|
||||
self.tmp = Path(tempfile.mkdtemp())
|
||||
self.email_mgr = _make_email_mgr()
|
||||
self.cal_mgr = _make_cal_mgr()
|
||||
self.am = _make_am(
|
||||
self.tmp,
|
||||
email_manager=self.email_mgr,
|
||||
calendar_manager=self.cal_mgr,
|
||||
file_manager=_make_file_mgr(),
|
||||
)
|
||||
self.am.provision('email', 'alice', password='pw')
|
||||
self.am.provision('calendar', 'alice', password='pw')
|
||||
|
||||
def test_deprovision_peer_removes_from_all_services(self):
|
||||
self.am.deprovision_peer('alice')
|
||||
self.assertIsNone(self.am.get_credentials('email', 'alice'))
|
||||
self.assertIsNone(self.am.get_credentials('calendar', 'alice'))
|
||||
|
||||
def test_deprovision_peer_returns_results_dict(self):
|
||||
results = self.am.deprovision_peer('alice')
|
||||
self.assertIn('email', results)
|
||||
self.assertIn('calendar', results)
|
||||
self.assertTrue(results['email'])
|
||||
self.assertTrue(results['calendar'])
|
||||
|
||||
def test_deprovision_peer_continues_after_one_service_fails(self):
|
||||
self.email_mgr.delete_email_user.side_effect = RuntimeError('smtp down')
|
||||
results = self.am.deprovision_peer('alice')
|
||||
self.assertFalse(results.get('email'))
|
||||
# calendar should still succeed even though email failed
|
||||
self.assertTrue(results.get('calendar'))
|
||||
|
||||
def test_deprovision_peer_no_op_for_unknown_peer(self):
|
||||
results = self.am.deprovision_peer('nobody')
|
||||
self.assertEqual(results, {})
|
||||
|
||||
|
||||
# ── Direct credential storage ─────────────────────────────────────────────────
|
||||
|
||||
class TestStoreCredentials(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
import tempfile
|
||||
self.tmp = Path(tempfile.mkdtemp())
|
||||
self.am = _make_am(self.tmp)
|
||||
|
||||
def test_store_credentials_makes_them_retrievable(self):
|
||||
self.am.store_credentials('email', 'alice', {'password': 'mypassword'})
|
||||
self.assertEqual(self.am.get_credentials('email', 'alice'), {'password': 'mypassword'})
|
||||
|
||||
def test_store_credentials_overwrites_existing(self):
|
||||
self.am.store_credentials('email', 'alice', {'password': 'old'})
|
||||
self.am.store_credentials('email', 'alice', {'password': 'new'})
|
||||
self.assertEqual(self.am.get_credentials('email', 'alice'), {'password': 'new'})
|
||||
|
||||
def test_store_credentials_creates_file_with_0600(self):
|
||||
self.am.store_credentials('email', 'alice', {'password': 'pw'})
|
||||
creds_path = self.tmp / 'peer_service_credentials.json'
|
||||
mode = stat.S_IMODE(creds_path.stat().st_mode)
|
||||
self.assertEqual(mode, 0o600)
|
||||
|
||||
|
||||
# ── Thread safety ─────────────────────────────────────────────────────────────
|
||||
|
||||
class TestThreadSafety(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
import tempfile
|
||||
self.tmp = Path(tempfile.mkdtemp())
|
||||
self.am = _make_am(self.tmp)
|
||||
|
||||
def test_concurrent_store_credentials_no_data_loss(self):
|
||||
errors = []
|
||||
def worker(peer_name):
|
||||
try:
|
||||
self.am.store_credentials('email', peer_name, {'password': f'pw-{peer_name}'})
|
||||
except Exception as e:
|
||||
errors.append(e)
|
||||
|
||||
threads = [threading.Thread(target=worker, args=(f'peer{i}',)) for i in range(20)]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
|
||||
self.assertEqual(errors, [])
|
||||
accounts = self.am.list_accounts('email')
|
||||
self.assertEqual(len(accounts), 20)
|
||||
|
||||
|
||||
class TestEdgeCases(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
import tempfile
|
||||
self.tmp = Path(tempfile.mkdtemp())
|
||||
self.email_mgr = _make_email_mgr()
|
||||
self.am = _make_am(self.tmp, email_manager=self.email_mgr,
|
||||
calendar_manager=_make_cal_mgr(),
|
||||
file_manager=_make_file_mgr())
|
||||
|
||||
def test_deprovision_peer_never_provisioned_returns_empty(self):
|
||||
self.assertEqual(self.am.deprovision_peer('ghost'), {})
|
||||
|
||||
def test_deprovision_clears_credentials_even_when_manager_returns_false(self):
|
||||
"""Credentials are removed even if underlying manager reports failure."""
|
||||
self.am.provision('email', 'alice', password='pw')
|
||||
self.email_mgr.delete_email_user.return_value = False
|
||||
self.am.deprovision('email', 'alice')
|
||||
self.assertIsNone(self.am.get_credentials('email', 'alice'))
|
||||
|
||||
def test_provision_twice_overwrites_credentials(self):
|
||||
self.am.provision('email', 'alice', password='first')
|
||||
self.am.provision('email', 'alice', password='second')
|
||||
self.assertEqual(self.am.get_credentials('email', 'alice'), {'password': 'second'})
|
||||
|
||||
def test_provision_twice_calls_manager_both_times(self):
|
||||
self.am.provision('email', 'alice', password='first')
|
||||
self.am.provision('email', 'alice', password='second')
|
||||
self.assertEqual(self.email_mgr.create_email_user.call_count, 2)
|
||||
|
||||
def test_corrupted_credentials_file_returns_empty_and_continues(self):
|
||||
"""A corrupted JSON file is treated as empty rather than crashing."""
|
||||
creds_path = self.tmp / 'peer_service_credentials.json'
|
||||
creds_path.write_text('{invalid json}')
|
||||
result = self.am.get_all_credentials('alice')
|
||||
self.assertEqual(result, {})
|
||||
|
||||
def test_file_permissions_preserved_on_second_write(self):
|
||||
"""0o600 must hold even after overwriting with a second provision."""
|
||||
self.am.provision('email', 'alice', password='first')
|
||||
self.am.provision('email', 'bob', password='second')
|
||||
creds_path = self.tmp / 'peer_service_credentials.json'
|
||||
mode = stat.S_IMODE(creds_path.stat().st_mode)
|
||||
self.assertEqual(mode, 0o600, f'Expected 0o600 after overwrite, got {oct(mode)}')
|
||||
|
||||
def test_generated_password_is_url_safe(self):
|
||||
"""token_urlsafe must not produce + or / characters."""
|
||||
creds = self.am.provision('email', 'alice')
|
||||
pwd = creds['password']
|
||||
self.assertNotIn('+', pwd)
|
||||
self.assertNotIn('/', pwd)
|
||||
|
||||
def test_store_then_deprovision_removes_credentials(self):
|
||||
"""store_credentials + deprovision should cleanly remove the entry."""
|
||||
self.am.store_credentials('email', 'alice', {'password': 'stored'})
|
||||
self.am.deprovision('email', 'alice')
|
||||
self.assertIsNone(self.am.get_credentials('email', 'alice'))
|
||||
|
||||
|
||||
# ── HTTP dispatch (manager == "http") ─────────────────────────────────────────
|
||||
|
||||
class TestHttpDispatch(unittest.TestCase):
|
||||
"""AccountManager with manager='http' uses HTTP POST/DELETE to the service backend."""
|
||||
|
||||
def _make_http_registry(self, backend='cell-myapp:8080'):
|
||||
reg = MagicMock()
|
||||
reg.get.return_value = {
|
||||
'id': 'myapp',
|
||||
'backend': backend,
|
||||
'accounts': {'manager': 'http', 'credentials': ['password']},
|
||||
}
|
||||
return reg
|
||||
|
||||
def setUp(self):
|
||||
import tempfile
|
||||
self.tmp = Path(tempfile.mkdtemp())
|
||||
self.am = _make_am(self.tmp, registry=self._make_http_registry())
|
||||
|
||||
def test_provision_http_posts_to_service_api(self):
|
||||
with patch('account_manager._requests') as mock_req:
|
||||
mock_req.post.return_value = MagicMock(status_code=201)
|
||||
creds = self.am.provision('myapp', 'alice', password='s3cret')
|
||||
mock_req.post.assert_called_once_with(
|
||||
'http://cell-myapp:8080/service-api/accounts',
|
||||
json={'username': 'alice', 'password': 's3cret'},
|
||||
timeout=10,
|
||||
)
|
||||
self.assertEqual(creds['password'], 's3cret')
|
||||
|
||||
def test_provision_http_stores_credentials_on_success(self):
|
||||
with patch('account_manager._requests') as mock_req:
|
||||
mock_req.post.return_value = MagicMock(status_code=200)
|
||||
self.am.provision('myapp', 'alice', password='pw')
|
||||
self.assertEqual(self.am.get_credentials('myapp', 'alice'), {'password': 'pw'})
|
||||
|
||||
def test_provision_http_returns_false_on_non_2xx(self):
|
||||
with patch('account_manager._requests') as mock_req:
|
||||
mock_req.post.return_value = MagicMock(status_code=409, text='conflict')
|
||||
with self.assertRaises(RuntimeError):
|
||||
self.am.provision('myapp', 'alice', password='pw')
|
||||
|
||||
def test_provision_http_raises_on_request_exception(self):
|
||||
with patch('account_manager._requests') as mock_req:
|
||||
mock_req.post.side_effect = Exception('connection refused')
|
||||
with self.assertRaises(RuntimeError):
|
||||
self.am.provision('myapp', 'alice', password='pw')
|
||||
|
||||
def test_deprovision_http_deletes_to_service_api(self):
|
||||
self.am.store_credentials('myapp', 'alice', {'password': 'pw'})
|
||||
with patch('account_manager._requests') as mock_req:
|
||||
mock_req.delete.return_value = MagicMock(status_code=204)
|
||||
ok = self.am.deprovision('myapp', 'alice')
|
||||
mock_req.delete.assert_called_once_with(
|
||||
'http://cell-myapp:8080/service-api/accounts/alice',
|
||||
timeout=10,
|
||||
)
|
||||
self.assertTrue(ok)
|
||||
|
||||
def test_deprovision_http_treats_404_as_success(self):
|
||||
"""404 means already deleted — still a clean deprovision."""
|
||||
self.am.store_credentials('myapp', 'alice', {'password': 'pw'})
|
||||
with patch('account_manager._requests') as mock_req:
|
||||
mock_req.delete.return_value = MagicMock(status_code=404)
|
||||
ok = self.am.deprovision('myapp', 'alice')
|
||||
self.assertTrue(ok)
|
||||
|
||||
def test_deprovision_http_removes_stored_credentials(self):
|
||||
self.am.store_credentials('myapp', 'alice', {'password': 'pw'})
|
||||
with patch('account_manager._requests') as mock_req:
|
||||
mock_req.delete.return_value = MagicMock(status_code=204)
|
||||
self.am.deprovision('myapp', 'alice')
|
||||
self.assertIsNone(self.am.get_credentials('myapp', 'alice'))
|
||||
|
||||
def test_resolve_service_http_does_not_require_python_manager(self):
|
||||
"""manager='http' must not raise even with no named managers passed."""
|
||||
am = AccountManager(
|
||||
service_registry=self._make_http_registry(),
|
||||
data_dir=str(self.tmp),
|
||||
)
|
||||
svc, manager_name, manager = am._resolve_service('myapp')
|
||||
self.assertEqual(manager_name, 'http')
|
||||
self.assertIsNone(manager)
|
||||
|
||||
def test_http_base_url_raises_when_no_backend(self):
|
||||
svc = {'id': 'nobackend', 'backend': ''}
|
||||
with self.assertRaises(ValueError):
|
||||
AccountManager._http_base_url(svc)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -82,6 +82,37 @@ class TestAPIEndpoints(unittest.TestCase):
|
||||
self.assertIn('domain', data)
|
||||
self.assertIn('ip_range', data)
|
||||
self.assertIn('wireguard_port', data)
|
||||
self.assertIn('installed_services', data)
|
||||
|
||||
def test_get_config_installed_services_is_dict(self):
|
||||
"""installed_services must be a dict, never a list or primitive"""
|
||||
response = self.client.get('/api/config')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIsInstance(data['installed_services'], dict)
|
||||
|
||||
def test_get_config_installed_services_empty_when_none_installed(self):
|
||||
"""installed_services defaults to empty dict when no services are installed"""
|
||||
response = self.client.get('/api/config')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
# Fresh test environment has no installed services
|
||||
self.assertEqual(data['installed_services'], {})
|
||||
|
||||
def test_get_config_installed_services_reflects_stored_value(self):
|
||||
"""installed_services in GET /api/config reflects what config_manager returns"""
|
||||
from app import config_manager
|
||||
config_manager.configs.setdefault('_identity', {})['installed_services'] = {
|
||||
'mailserver': {'status': 'running', 'installed_at': '2026-01-01T00:00:00'}
|
||||
}
|
||||
try:
|
||||
response = self.client.get('/api/config')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = json.loads(response.data)
|
||||
self.assertIn('mailserver', data['installed_services'])
|
||||
self.assertEqual(data['installed_services']['mailserver']['status'], 'running')
|
||||
finally:
|
||||
config_manager.configs.get('_identity', {}).pop('installed_services', None)
|
||||
|
||||
def test_update_config_endpoint(self):
|
||||
"""Test update config endpoint"""
|
||||
@@ -362,10 +393,12 @@ class TestAPIEndpoints(unittest.TestCase):
|
||||
self.assertEqual(response.status_code, 500)
|
||||
mock_peers.update_peer_ip.side_effect = None
|
||||
|
||||
@patch('app.service_registry')
|
||||
@patch('app.email_manager')
|
||||
def test_email_endpoints(self, mock_email):
|
||||
def test_email_endpoints(self, mock_email, mock_sr):
|
||||
mock_sr.get.return_value = {'id': 'email', 'installed': True}
|
||||
# Ensure all relevant mock methods return JSON-serializable values
|
||||
mock_email.get_users.return_value = [{'username': 'user1', 'domain': 'cell', 'email': 'user1@cell'}]
|
||||
mock_email.get_email_users.return_value = [{'username': 'user1', 'domain': 'cell', 'email': 'user1@cell'}]
|
||||
mock_email.create_email_user.return_value = True
|
||||
mock_email.delete_email_user.return_value = True
|
||||
mock_email.get_status.return_value = {'postfix_running': True, 'dovecot_running': True, 'total_users': 1, 'total_size_bytes': 0, 'total_size_mb': 0.0, 'users': [{'username': 'user1', 'domain': 'cell', 'email': 'user1@cell'}]}
|
||||
@@ -376,10 +409,10 @@ class TestAPIEndpoints(unittest.TestCase):
|
||||
response = self.client.get('/api/email/users')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
self.assertIsInstance(json.loads(response.data), list)
|
||||
mock_email.get_users.side_effect = Exception('fail')
|
||||
mock_email.get_email_users.side_effect = Exception('fail')
|
||||
response = self.client.get('/api/email/users')
|
||||
self.assertEqual(response.status_code, 500)
|
||||
mock_email.get_users.side_effect = None
|
||||
mock_email.get_email_users.side_effect = None
|
||||
# /api/email/users (POST)
|
||||
response = self.client.post('/api/email/users', data=json.dumps({'username': 'user1', 'domain': 'cell', 'password': 'pw'}), content_type='application/json')
|
||||
self.assertEqual(response.status_code, 200)
|
||||
@@ -423,8 +456,10 @@ class TestAPIEndpoints(unittest.TestCase):
|
||||
self.assertEqual(response.status_code, 500)
|
||||
mock_email.get_mailbox_info.side_effect = None
|
||||
|
||||
@patch('app.service_registry')
|
||||
@patch('app.calendar_manager')
|
||||
def test_calendar_endpoints(self, mock_calendar):
|
||||
def test_calendar_endpoints(self, mock_calendar, mock_sr):
|
||||
mock_sr.get.return_value = {'id': 'calendar', 'installed': True}
|
||||
# Mock return values for all relevant calendar_manager methods
|
||||
mock_calendar.get_users.return_value = [{'username': 'user1', 'collections': {'calendars': ['cal1'], 'contacts': ['c1']}}]
|
||||
mock_calendar.create_calendar_user.return_value = True
|
||||
@@ -492,8 +527,10 @@ class TestAPIEndpoints(unittest.TestCase):
|
||||
self.assertEqual(response.status_code, 500)
|
||||
mock_calendar.test_connectivity.side_effect = None
|
||||
|
||||
@patch('app.service_registry')
|
||||
@patch('app.file_manager')
|
||||
def test_file_endpoints(self, mock_file):
|
||||
def test_file_endpoints(self, mock_file, mock_sr):
|
||||
mock_sr.get.return_value = {'id': 'files', 'installed': True}
|
||||
# Mock return values for all relevant file_manager methods
|
||||
mock_file.get_users.return_value = [{'username': 'user1', 'storage_info': {'total_files': 1, 'total_size_bytes': 1000}}]
|
||||
mock_file.create_user.return_value = True
|
||||
|
||||
@@ -36,6 +36,7 @@ import app as app_module
|
||||
|
||||
class TestAppMisc(unittest.TestCase):
|
||||
def setUp(self):
|
||||
app_module.app.config['TESTING'] = True
|
||||
# Patch managers to avoid side effects
|
||||
self.patches = [
|
||||
patch.object(app_module, 'network_manager', MagicMock()),
|
||||
|
||||
@@ -0,0 +1,354 @@
|
||||
"""
|
||||
Tests for service-volume backup/restore in ConfigManager.
|
||||
|
||||
Covers:
|
||||
- _backup_service_volumes: happy path, container not running, timeout
|
||||
- _restore_service_volumes: happy path, missing archive, unknown service
|
||||
- backup_config: passes service_registry, records includes_service_data
|
||||
- restore_config: passes service_registry on full restore, not on selective
|
||||
"""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch, call
|
||||
|
||||
import sys
|
||||
import os
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'api'))
|
||||
|
||||
from config_manager import ConfigManager
|
||||
|
||||
|
||||
def _make_cm(tmp_path: Path) -> ConfigManager:
|
||||
cfg_file = tmp_path / 'cell_config.json'
|
||||
cfg_file.write_text('{}')
|
||||
cm = ConfigManager(config_file=str(cfg_file), data_dir=str(tmp_path))
|
||||
return cm
|
||||
|
||||
|
||||
def _make_registry(plan=None):
|
||||
"""Return a mock ServiceRegistry with a preset backup plan."""
|
||||
reg = MagicMock()
|
||||
reg.get_backup_plan.return_value = plan if plan is not None else [
|
||||
{
|
||||
'service_id': 'email',
|
||||
'volumes': [
|
||||
{'container': 'cell-mail', 'path': '/var/mail', 'name': 'maildata'},
|
||||
{'container': 'cell-mail', 'path': '/var/mail-state', 'name': 'mailstate'},
|
||||
],
|
||||
'config_paths': [],
|
||||
},
|
||||
{
|
||||
'service_id': 'calendar',
|
||||
'volumes': [
|
||||
{'container': 'cell-radicale', 'path': '/data', 'name': 'radicale_data'},
|
||||
],
|
||||
'config_paths': [],
|
||||
},
|
||||
]
|
||||
return reg
|
||||
|
||||
|
||||
class TestBackupServiceVolumesHappyPath(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
import tempfile
|
||||
self.tmp = Path(tempfile.mkdtemp())
|
||||
self.cm = _make_cm(self.tmp)
|
||||
self.backup_path = self.tmp / 'test_backup'
|
||||
self.backup_path.mkdir()
|
||||
|
||||
def _run_backup(self, registry=None):
|
||||
if registry is None:
|
||||
registry = _make_registry()
|
||||
self.cm._backup_service_volumes(self.backup_path, registry)
|
||||
|
||||
@patch('config_manager.subprocess.run')
|
||||
def test_creates_service_data_dir(self, mock_run):
|
||||
mock_run.return_value = MagicMock(returncode=0, stderr=b'')
|
||||
self._run_backup()
|
||||
self.assertTrue((self.backup_path / 'service_data' / 'email').is_dir())
|
||||
self.assertTrue((self.backup_path / 'service_data' / 'calendar').is_dir())
|
||||
|
||||
@patch('config_manager.subprocess.run')
|
||||
def test_calls_docker_exec_tar_for_each_volume(self, mock_run):
|
||||
mock_run.return_value = MagicMock(returncode=0, stderr=b'')
|
||||
self._run_backup()
|
||||
commands = [tuple(c.args[0]) for c in mock_run.call_args_list]
|
||||
self.assertIn(
|
||||
('docker', 'exec', '--', 'cell-mail', 'tar', '-C', '/var/mail', '-czf', '-', '.'),
|
||||
commands,
|
||||
)
|
||||
self.assertIn(
|
||||
('docker', 'exec', '--', 'cell-mail', 'tar', '-C', '/var/mail-state', '-czf', '-', '.'),
|
||||
commands,
|
||||
)
|
||||
self.assertIn(
|
||||
('docker', 'exec', '--', 'cell-radicale', 'tar', '-C', '/data', '-czf', '-', '.'),
|
||||
commands,
|
||||
)
|
||||
|
||||
@patch('config_manager.subprocess.run')
|
||||
def test_writes_archive_files(self, mock_run):
|
||||
mock_run.return_value = MagicMock(returncode=0, stderr=b'')
|
||||
self._run_backup()
|
||||
self.assertTrue((self.backup_path / 'service_data' / 'email' / 'maildata.tar.gz').exists())
|
||||
self.assertTrue((self.backup_path / 'service_data' / 'email' / 'mailstate.tar.gz').exists())
|
||||
self.assertTrue((self.backup_path / 'service_data' / 'calendar' / 'radicale_data.tar.gz').exists())
|
||||
|
||||
@patch('config_manager.subprocess.run')
|
||||
def test_removes_archive_on_nonzero_returncode(self, mock_run):
|
||||
mock_run.return_value = MagicMock(returncode=1, stderr=b'container not running')
|
||||
self._run_backup()
|
||||
self.assertFalse(
|
||||
(self.backup_path / 'service_data' / 'email' / 'maildata.tar.gz').exists()
|
||||
)
|
||||
|
||||
@patch('config_manager.subprocess.run')
|
||||
def test_continues_after_one_volume_fails(self, mock_run):
|
||||
def side_effect(cmd, **kwargs):
|
||||
if 'cell-mail' in cmd:
|
||||
return MagicMock(returncode=1, stderr=b'error')
|
||||
return MagicMock(returncode=0, stderr=b'')
|
||||
mock_run.side_effect = side_effect
|
||||
self._run_backup()
|
||||
# radicale should still succeed
|
||||
self.assertTrue(
|
||||
(self.backup_path / 'service_data' / 'calendar' / 'radicale_data.tar.gz').exists()
|
||||
)
|
||||
|
||||
@patch('config_manager.subprocess.run', side_effect=subprocess.TimeoutExpired('docker', 300))
|
||||
def test_timeout_removes_partial_archive(self, _mock_run):
|
||||
self._run_backup()
|
||||
# no archive should remain after a timeout
|
||||
for svc in ('email', 'calendar'):
|
||||
for name in ('maildata', 'mailstate', 'radicale_data'):
|
||||
self.assertFalse(
|
||||
(self.backup_path / 'service_data' / svc / f'{name}.tar.gz').exists()
|
||||
)
|
||||
|
||||
@patch('config_manager.subprocess.run')
|
||||
def test_empty_volumes_list_skipped(self, mock_run):
|
||||
registry = _make_registry(plan=[
|
||||
{'service_id': 'widget', 'volumes': [], 'config_paths': []}
|
||||
])
|
||||
self.cm._backup_service_volumes(self.backup_path, registry)
|
||||
mock_run.assert_not_called()
|
||||
|
||||
@patch('config_manager.subprocess.run')
|
||||
def test_get_backup_plan_exception_is_handled(self, mock_run):
|
||||
registry = MagicMock()
|
||||
registry.get_backup_plan.side_effect = RuntimeError('registry unavailable')
|
||||
# should not raise
|
||||
self.cm._backup_service_volumes(self.backup_path, registry)
|
||||
mock_run.assert_not_called()
|
||||
|
||||
@patch('config_manager.subprocess.run')
|
||||
def test_unsafe_container_name_rejected(self, mock_run):
|
||||
registry = _make_registry(plan=[{
|
||||
'service_id': 'evil', 'config_paths': [],
|
||||
'volumes': [{'container': '-it cell-api', 'path': '/data', 'name': 'data'}],
|
||||
}])
|
||||
self.cm._backup_service_volumes(self.backup_path, registry)
|
||||
mock_run.assert_not_called()
|
||||
|
||||
@patch('config_manager.subprocess.run')
|
||||
def test_path_traversal_in_volume_path_rejected(self, mock_run):
|
||||
registry = _make_registry(plan=[{
|
||||
'service_id': 'evil', 'config_paths': [],
|
||||
'volumes': [{'container': 'cell-mail', 'path': '/../etc', 'name': 'etc'}],
|
||||
}])
|
||||
self.cm._backup_service_volumes(self.backup_path, registry)
|
||||
mock_run.assert_not_called()
|
||||
|
||||
@patch('config_manager.subprocess.run')
|
||||
def test_relative_volume_path_rejected(self, mock_run):
|
||||
registry = _make_registry(plan=[{
|
||||
'service_id': 'evil', 'config_paths': [],
|
||||
'volumes': [{'container': 'cell-mail', 'path': 'data/maildata', 'name': 'data'}],
|
||||
}])
|
||||
self.cm._backup_service_volumes(self.backup_path, registry)
|
||||
mock_run.assert_not_called()
|
||||
|
||||
@patch('config_manager.subprocess.run')
|
||||
def test_unsafe_volume_name_rejected(self, mock_run):
|
||||
registry = _make_registry(plan=[{
|
||||
'service_id': 'evil', 'config_paths': [],
|
||||
'volumes': [{'container': 'cell-mail', 'path': '/var/mail', 'name': '../../etc/passwd'}],
|
||||
}])
|
||||
self.cm._backup_service_volumes(self.backup_path, registry)
|
||||
mock_run.assert_not_called()
|
||||
|
||||
@patch('config_manager.subprocess.run')
|
||||
def test_atomic_write_no_archive_on_partial_failure(self, mock_run):
|
||||
"""If an exception occurs during subprocess, no .tar.gz file should remain."""
|
||||
mock_run.side_effect = OSError('disk full')
|
||||
self._run_backup()
|
||||
for f in self.backup_path.rglob('*.tar.gz'):
|
||||
self.fail(f'Archive {f} should not exist after exception during backup')
|
||||
|
||||
|
||||
class TestRestoreServiceVolumes(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
import tempfile
|
||||
self.tmp = Path(tempfile.mkdtemp())
|
||||
self.cm = _make_cm(self.tmp)
|
||||
self.backup_path = self.tmp / 'test_backup'
|
||||
# Prepare a realistic backup structure
|
||||
svc_data = self.backup_path / 'service_data'
|
||||
(svc_data / 'email').mkdir(parents=True)
|
||||
(svc_data / 'email' / 'maildata.tar.gz').write_bytes(b'fake-archive')
|
||||
(svc_data / 'calendar').mkdir(parents=True)
|
||||
(svc_data / 'calendar' / 'radicale_data.tar.gz').write_bytes(b'fake-archive')
|
||||
|
||||
def _make_registry_with_manifests(self):
|
||||
reg = MagicMock()
|
||||
def get_side_effect(service_id):
|
||||
manifests = {
|
||||
'email': {'backup': {'volumes': [
|
||||
{'container': 'cell-mail', 'path': '/var/mail', 'name': 'maildata'},
|
||||
]}},
|
||||
'calendar': {'backup': {'volumes': [
|
||||
{'container': 'cell-radicale', 'path': '/data', 'name': 'radicale_data'},
|
||||
]}},
|
||||
}
|
||||
return manifests.get(service_id)
|
||||
reg.get.side_effect = get_side_effect
|
||||
return reg
|
||||
|
||||
@patch('config_manager.subprocess.run')
|
||||
def test_calls_docker_exec_tar_for_each_archive(self, mock_run):
|
||||
mock_run.return_value = MagicMock(returncode=0, stderr=b'')
|
||||
registry = self._make_registry_with_manifests()
|
||||
self.cm._restore_service_volumes(self.backup_path, registry)
|
||||
commands = [tuple(c.args[0]) for c in mock_run.call_args_list]
|
||||
self.assertIn(
|
||||
('docker', 'exec', '-i', '--', 'cell-mail', 'tar', '-C', '/var/mail', '-xzf', '-'),
|
||||
commands,
|
||||
)
|
||||
self.assertIn(
|
||||
('docker', 'exec', '-i', '--', 'cell-radicale', 'tar', '-C', '/data', '-xzf', '-'),
|
||||
commands,
|
||||
)
|
||||
|
||||
@patch('config_manager.subprocess.run')
|
||||
def test_skips_missing_archive(self, mock_run):
|
||||
mock_run.return_value = MagicMock(returncode=0, stderr=b'')
|
||||
registry = MagicMock()
|
||||
registry.get.return_value = {'backup': {'volumes': [
|
||||
{'container': 'cell-mail', 'path': '/var/mail', 'name': 'no_such_archive'},
|
||||
]}}
|
||||
self.cm._restore_service_volumes(self.backup_path, registry)
|
||||
mock_run.assert_not_called()
|
||||
|
||||
@patch('config_manager.subprocess.run')
|
||||
def test_skips_unknown_service(self, mock_run):
|
||||
mock_run.return_value = MagicMock(returncode=0, stderr=b'')
|
||||
registry = MagicMock()
|
||||
registry.get.return_value = None
|
||||
self.cm._restore_service_volumes(self.backup_path, registry)
|
||||
mock_run.assert_not_called()
|
||||
|
||||
@patch('config_manager.subprocess.run')
|
||||
def test_no_service_data_dir_is_noop(self, mock_run):
|
||||
empty_backup = self.tmp / 'empty_backup'
|
||||
empty_backup.mkdir()
|
||||
registry = self._make_registry_with_manifests()
|
||||
self.cm._restore_service_volumes(empty_backup, registry)
|
||||
mock_run.assert_not_called()
|
||||
|
||||
@patch('config_manager.subprocess.run', side_effect=subprocess.TimeoutExpired('docker', 300))
|
||||
def test_timeout_is_handled_gracefully(self, _mock_run):
|
||||
registry = self._make_registry_with_manifests()
|
||||
# should not raise
|
||||
self.cm._restore_service_volumes(self.backup_path, registry)
|
||||
|
||||
@patch('config_manager.subprocess.run')
|
||||
def test_continues_after_docker_exec_failure(self, mock_run):
|
||||
call_count = [0]
|
||||
def side_effect(cmd, **kwargs):
|
||||
call_count[0] += 1
|
||||
if call_count[0] == 1:
|
||||
return MagicMock(returncode=1, stderr=b'container not running')
|
||||
return MagicMock(returncode=0, stderr=b'')
|
||||
mock_run.side_effect = side_effect
|
||||
registry = self._make_registry_with_manifests()
|
||||
self.cm._restore_service_volumes(self.backup_path, registry)
|
||||
self.assertEqual(call_count[0], 2)
|
||||
|
||||
|
||||
class TestBackupConfigWithRegistry(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
import tempfile
|
||||
self.tmp = Path(tempfile.mkdtemp())
|
||||
self.cm = _make_cm(self.tmp)
|
||||
|
||||
@patch.object(ConfigManager, '_backup_service_volumes')
|
||||
def test_backup_calls_volume_backup_when_registry_given(self, mock_bsv):
|
||||
registry = _make_registry()
|
||||
self.cm.backup_config(service_registry=registry)
|
||||
mock_bsv.assert_called_once()
|
||||
args = mock_bsv.call_args
|
||||
self.assertIs(args[0][1], registry)
|
||||
|
||||
@patch.object(ConfigManager, '_backup_service_volumes')
|
||||
def test_backup_skips_volume_backup_when_no_registry(self, mock_bsv):
|
||||
self.cm.backup_config(service_registry=None)
|
||||
mock_bsv.assert_not_called()
|
||||
|
||||
@patch.object(ConfigManager, '_backup_service_volumes')
|
||||
def test_manifest_records_includes_service_data_true(self, _mock_bsv):
|
||||
registry = _make_registry()
|
||||
backup_id = self.cm.backup_config(service_registry=registry)
|
||||
manifest = json.loads((self.cm.backup_dir / backup_id / 'manifest.json').read_text())
|
||||
self.assertTrue(manifest['includes_service_data'])
|
||||
|
||||
@patch.object(ConfigManager, '_backup_service_volumes')
|
||||
def test_manifest_records_includes_service_data_false(self, _mock_bsv):
|
||||
backup_id = self.cm.backup_config(service_registry=None)
|
||||
manifest = json.loads((self.cm.backup_dir / backup_id / 'manifest.json').read_text())
|
||||
self.assertFalse(manifest['includes_service_data'])
|
||||
|
||||
|
||||
class TestRestoreConfigWithRegistry(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
import tempfile
|
||||
self.tmp = Path(tempfile.mkdtemp())
|
||||
self.cm = _make_cm(self.tmp)
|
||||
# Create a minimal backup
|
||||
backup_id = 'backup_20260101_000000'
|
||||
bp = self.cm.backup_dir / backup_id
|
||||
bp.mkdir(parents=True)
|
||||
(bp / 'cell_config.json').write_text('{}')
|
||||
manifest = {'backup_id': backup_id, 'timestamp': '2026-01-01T00:00:00', 'services': []}
|
||||
(bp / 'manifest.json').write_text(json.dumps(manifest))
|
||||
self.backup_id = backup_id
|
||||
|
||||
@patch.object(ConfigManager, '_restore_service_volumes')
|
||||
def test_full_restore_calls_volume_restore_when_registry_given(self, mock_rsv):
|
||||
registry = _make_registry()
|
||||
self.cm.restore_config(self.backup_id, service_registry=registry)
|
||||
mock_rsv.assert_called_once()
|
||||
args = mock_rsv.call_args
|
||||
self.assertIs(args[0][1], registry)
|
||||
|
||||
@patch.object(ConfigManager, '_restore_service_volumes')
|
||||
def test_full_restore_skips_volume_restore_when_no_registry(self, mock_rsv):
|
||||
self.cm.restore_config(self.backup_id, service_registry=None)
|
||||
mock_rsv.assert_not_called()
|
||||
|
||||
@patch.object(ConfigManager, '_restore_service_volumes')
|
||||
def test_selective_restore_never_calls_volume_restore(self, mock_rsv):
|
||||
"""Volume restore is skipped for selective restores (service list specified)."""
|
||||
registry = _make_registry()
|
||||
self.cm.restore_config(self.backup_id, services=['email'], service_registry=registry)
|
||||
mock_rsv.assert_not_called()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
+493
-21
@@ -59,16 +59,49 @@ class TestGenerateCaddyfileLan(unittest.TestCase):
|
||||
class TestGenerateCaddyfilePicNgo(unittest.TestCase):
|
||||
def test_pic_ngo_has_dns_plugin_and_wildcard(self):
|
||||
mgr = _mgr()
|
||||
mgr.config_manager.configs = {
|
||||
'ddns': {'url': 'https://ddns.pic.ngo/api/v1'},
|
||||
}
|
||||
mgr.config_manager.get_ddns_token.return_value = 'TESTSECRET123'
|
||||
identity = {'cell_name': 'alpha', 'domain_mode': 'pic_ngo'}
|
||||
out = mgr.generate_caddyfile(identity, [])
|
||||
with unittest.mock.patch.dict(os.environ, {'DDNS_URL': 'https://ddns.pic.ngo/api/v1'}):
|
||||
out = mgr.generate_caddyfile(identity, [])
|
||||
self.assertIn('dns pic_ngo', out)
|
||||
self.assertIn('*.alpha.pic.ngo', out)
|
||||
self.assertIn('alpha.pic.ngo', out)
|
||||
self.assertIn('{$PIC_NGO_DDNS_TOKEN}', out)
|
||||
self.assertIn('{$PIC_NGO_DDNS_API}', out)
|
||||
# Registration token (not TOTP secret) is embedded — no {$VAR} placeholders
|
||||
self.assertIn('token TESTSECRET123', out)
|
||||
# /api/v1 is stripped — the plugin appends it itself
|
||||
self.assertIn('api_base_url https://ddns.pic.ngo', out)
|
||||
self.assertNotIn('api_base_url https://ddns.pic.ngo/api/v1', out)
|
||||
self.assertNotIn('{$PIC_NGO_DDNS_TOKEN}', out)
|
||||
self.assertNotIn('{$PIC_NGO_DDNS_API}', out)
|
||||
self.assertIn('email admin@alpha.pic.ngo', out)
|
||||
# ACME staging hook
|
||||
self.assertIn('acme_ca {$ACME_CA_URL}', out)
|
||||
# acme_ca is omitted when ACME_CA_URL is not set
|
||||
self.assertNotIn('acme_ca', out)
|
||||
|
||||
def test_pic_ngo_acme_ca_included_when_env_set(self):
|
||||
mgr = _mgr()
|
||||
mgr.config_manager.configs = {'ddns': {}}
|
||||
mgr.config_manager.get_ddns_token.return_value = 'TESTSECRET123'
|
||||
identity = {'cell_name': 'alpha', 'domain_mode': 'pic_ngo'}
|
||||
with unittest.mock.patch.dict(os.environ, {
|
||||
'DDNS_URL': 'https://ddns.pic.ngo/api/v1',
|
||||
'ACME_CA_URL': 'https://acme-staging-v02.api.letsencrypt.org/directory',
|
||||
}):
|
||||
out = mgr.generate_caddyfile(identity, [])
|
||||
self.assertIn('acme_ca https://acme-staging-v02.api.letsencrypt.org/directory', out)
|
||||
|
||||
def test_pic_ngo_has_api_route_without_registry(self):
|
||||
mgr = _mgr()
|
||||
identity = {'cell_name': 'alpha', 'domain_mode': 'pic_ngo'}
|
||||
out = mgr.generate_caddyfile(identity, [])
|
||||
# Without a registry only the api block is present
|
||||
self.assertIn('@api host api.alpha.pic.ngo', out)
|
||||
self.assertIn('reverse_proxy cell-api:3000', out)
|
||||
self.assertNotIn('@calendar', out)
|
||||
self.assertNotIn('@mail', out)
|
||||
self.assertNotIn('@files', out)
|
||||
|
||||
|
||||
class TestGenerateCaddyfileCloudflare(unittest.TestCase):
|
||||
@@ -77,13 +110,35 @@ class TestGenerateCaddyfileCloudflare(unittest.TestCase):
|
||||
identity = {
|
||||
'cell_name': 'beta',
|
||||
'domain_mode': 'cloudflare',
|
||||
'custom_domain': 'example.com',
|
||||
'domain_name': 'example.com',
|
||||
}
|
||||
out = mgr.generate_caddyfile(identity, [])
|
||||
self.assertIn('dns cloudflare {$CF_API_TOKEN}', out)
|
||||
self.assertIn('*.example.com', out)
|
||||
self.assertIn('email {$ACME_EMAIL}', out)
|
||||
self.assertIn('acme_ca {$ACME_CA_URL}', out)
|
||||
# acme_ca is omitted when ACME_CA_URL is not set in the environment
|
||||
self.assertNotIn('acme_ca', out)
|
||||
|
||||
def test_caddyfile_cloudflare_uses_domain_name(self):
|
||||
"""Caddyfile must use domain_name for TLS host, not any 'custom_domain' key."""
|
||||
mgr = _mgr()
|
||||
identity = {
|
||||
'cell_name': 'beta',
|
||||
'domain_mode': 'cloudflare',
|
||||
'domain_name': 'home.example.com',
|
||||
'domain': 'home.local',
|
||||
}
|
||||
out = mgr.generate_caddyfile(identity, [])
|
||||
self.assertIn('*.home.example.com', out)
|
||||
self.assertIn('home.example.com', out)
|
||||
# Must not use the internal domain for TLS
|
||||
self.assertNotIn('*.home.local', out)
|
||||
# 'custom_domain' must not appear literally as a key in the output
|
||||
self.assertNotIn('custom_domain', out)
|
||||
# Without a registry only the api block is emitted for subdomain routing
|
||||
self.assertIn('@api host api.home.example.com', out)
|
||||
self.assertNotIn('@calendar', out)
|
||||
self.assertNotIn('@files', out)
|
||||
|
||||
|
||||
class TestGenerateCaddyfileDuckDns(unittest.TestCase):
|
||||
@@ -93,6 +148,9 @@ class TestGenerateCaddyfileDuckDns(unittest.TestCase):
|
||||
out = mgr.generate_caddyfile(identity, [])
|
||||
self.assertIn('dns duckdns {$DUCKDNS_TOKEN}', out)
|
||||
self.assertIn('*.gamma.duckdns.org', out)
|
||||
self.assertIn('@api host api.gamma.duckdns.org', out)
|
||||
self.assertNotIn('@calendar', out)
|
||||
self.assertNotIn('@files', out)
|
||||
|
||||
|
||||
class TestGenerateCaddyfileHttp01(unittest.TestCase):
|
||||
@@ -101,26 +159,39 @@ class TestGenerateCaddyfileHttp01(unittest.TestCase):
|
||||
identity = {
|
||||
'cell_name': 'delta',
|
||||
'domain_mode': 'http01',
|
||||
'custom_domain': 'delta.noip.me',
|
||||
'domain_name': 'delta.noip.me',
|
||||
}
|
||||
# Store-plugin service (not a core service name)
|
||||
services = [
|
||||
{'name': 'calendar', 'caddy_route':
|
||||
'reverse_proxy cell-radicale:5232'},
|
||||
{'name': 'files', 'caddy_route':
|
||||
'reverse_proxy cell-filegator:8080'},
|
||||
{'name': 'chat', 'caddy_route': 'reverse_proxy cell-chat:8090'},
|
||||
]
|
||||
out = mgr.generate_caddyfile(identity, services)
|
||||
# No wildcard, no DNS-01 plugins.
|
||||
self.assertNotIn('*.delta', out)
|
||||
self.assertNotIn('dns ', out)
|
||||
# No explicit tls block (no internal CA, no plugin) — the host block
|
||||
# itself is left empty so Caddy uses HTTP-01 by default.
|
||||
# No explicit tls block — Caddy uses HTTP-01 by default.
|
||||
self.assertNotIn('tls {', out)
|
||||
# Per-service blocks
|
||||
self.assertIn('calendar.delta.noip.me {', out)
|
||||
self.assertIn('files.delta.noip.me {', out)
|
||||
self.assertIn('reverse_proxy cell-radicale:5232', out)
|
||||
self.assertIn('reverse_proxy cell-filegator:8080', out)
|
||||
# Without a registry only the api block is generated
|
||||
self.assertIn('api.delta.noip.me {', out)
|
||||
self.assertNotIn('calendar.delta.noip.me {', out)
|
||||
self.assertNotIn('files.delta.noip.me {', out)
|
||||
self.assertNotIn('mail.delta.noip.me {', out)
|
||||
# Installed plugin service block still works
|
||||
self.assertIn('chat.delta.noip.me {', out)
|
||||
self.assertIn('reverse_proxy cell-chat:8090', out)
|
||||
|
||||
def test_http01_installed_service_with_caddy_route_appears(self):
|
||||
"""An installed service with a caddy_route produces its own per-host block."""
|
||||
mgr = _mgr()
|
||||
identity = {
|
||||
'cell_name': 'delta',
|
||||
'domain_mode': 'http01',
|
||||
'domain_name': 'delta.noip.me',
|
||||
}
|
||||
services = [{'name': 'notes', 'caddy_route': 'reverse_proxy cell-other:9000'}]
|
||||
out = mgr.generate_caddyfile(identity, services)
|
||||
self.assertIn('notes.delta.noip.me {', out)
|
||||
self.assertIn('reverse_proxy cell-other:9000', out)
|
||||
|
||||
|
||||
class TestServiceRoutesIncluded(unittest.TestCase):
|
||||
@@ -172,8 +243,8 @@ class TestHealthCheck(unittest.TestCase):
|
||||
mock_get.return_value = MagicMock(status_code=200)
|
||||
self.assertTrue(mgr.check_caddy_health())
|
||||
mock_get.assert_called_once()
|
||||
# URL must be the admin API root
|
||||
self.assertIn('cell-caddy:2019', mock_get.call_args[0][0])
|
||||
# Must hit /config/ — not the root which returns 404
|
||||
self.assertIn('/config/', mock_get.call_args[0][0])
|
||||
|
||||
def test_returns_false_on_connection_error(self):
|
||||
mgr = _mgr()
|
||||
@@ -224,5 +295,406 @@ class TestCertStatus(unittest.TestCase):
|
||||
self.assertEqual(out['days_remaining'], 84)
|
||||
|
||||
|
||||
class TestCaddyManagerIdentityChangedSubscription(unittest.TestCase):
|
||||
def test_subscribes_to_identity_changed_on_init(self):
|
||||
"""When service_bus is provided, CaddyManager subscribes to IDENTITY_CHANGED."""
|
||||
from service_bus import EventType
|
||||
mock_bus = MagicMock()
|
||||
mgr = CaddyManager(config_manager=MagicMock(), service_bus=mock_bus)
|
||||
mock_bus.subscribe_to_event.assert_called_once_with(
|
||||
EventType.IDENTITY_CHANGED, mgr._on_identity_changed
|
||||
)
|
||||
|
||||
def test_no_subscription_without_service_bus(self):
|
||||
"""When service_bus is omitted, no subscription is attempted."""
|
||||
mock_bus = MagicMock()
|
||||
CaddyManager(config_manager=MagicMock())
|
||||
mock_bus.subscribe_to_event.assert_not_called()
|
||||
|
||||
def test_on_identity_changed_calls_regenerate_with_installed(self):
|
||||
"""_on_identity_changed calls regenerate_with_installed([])."""
|
||||
mgr = _mgr()
|
||||
with patch.object(mgr, 'regenerate_with_installed', return_value=True) as mock_regen:
|
||||
event = MagicMock()
|
||||
mgr._on_identity_changed(event)
|
||||
mock_regen.assert_called_once_with([])
|
||||
|
||||
def test_on_identity_changed_swallows_exceptions(self):
|
||||
"""_on_identity_changed must not propagate exceptions."""
|
||||
mgr = _mgr()
|
||||
with patch.object(mgr, 'regenerate_with_installed', side_effect=Exception('boom')):
|
||||
event = MagicMock()
|
||||
mgr._on_identity_changed(event) # must not raise
|
||||
|
||||
|
||||
class TestRefreshCertStatus(unittest.TestCase):
|
||||
"""refresh_cert_status() + _check_cert_via_ssl()."""
|
||||
|
||||
def _make_der_cert(self, days_remaining: int) -> bytes:
|
||||
"""Return a minimal self-signed DER cert valid for *days_remaining* days."""
|
||||
from cryptography import x509
|
||||
from cryptography.x509.oid import NameOID
|
||||
from cryptography.hazmat.primitives import hashes, serialization
|
||||
from cryptography.hazmat.primitives.asymmetric import rsa
|
||||
import datetime
|
||||
key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
|
||||
now = datetime.datetime.now(datetime.timezone.utc)
|
||||
expiry = now + datetime.timedelta(days=days_remaining)
|
||||
cert = (
|
||||
x509.CertificateBuilder()
|
||||
.subject_name(x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, 'test.example.com')]))
|
||||
.issuer_name(x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, 'test.example.com')]))
|
||||
.public_key(key.public_key())
|
||||
.serial_number(x509.random_serial_number())
|
||||
.not_valid_before(expiry - datetime.timedelta(days=30))
|
||||
.not_valid_after(expiry)
|
||||
.sign(key, hashes.SHA256())
|
||||
)
|
||||
return cert.public_bytes(serialization.Encoding.DER)
|
||||
|
||||
def test_check_cert_via_ssl_returns_none_on_connection_error(self):
|
||||
"""_check_cert_via_ssl returns None when connection fails."""
|
||||
with patch('caddy_manager._socket.create_connection', side_effect=OSError('refused')):
|
||||
result = CaddyManager._check_cert_via_ssl('host', 443)
|
||||
self.assertIsNone(result)
|
||||
|
||||
def test_check_cert_via_ssl_returns_valid_status(self):
|
||||
"""_check_cert_via_ssl returns valid status for a future-dated cert."""
|
||||
der = self._make_der_cert(60)
|
||||
mock_tls = MagicMock()
|
||||
mock_tls.__enter__ = MagicMock(return_value=mock_tls)
|
||||
mock_tls.__exit__ = MagicMock(return_value=False)
|
||||
mock_tls.getpeercert.return_value = der
|
||||
mock_raw = MagicMock()
|
||||
mock_raw.__enter__ = MagicMock(return_value=mock_raw)
|
||||
mock_raw.__exit__ = MagicMock(return_value=False)
|
||||
with patch('caddy_manager._socket.create_connection', return_value=mock_raw):
|
||||
with patch('caddy_manager._ssl.create_default_context') as mock_ctx:
|
||||
mock_ctx.return_value.wrap_socket.return_value = mock_tls
|
||||
result = CaddyManager._check_cert_via_ssl('host', 443)
|
||||
self.assertIsNotNone(result)
|
||||
self.assertEqual(result['status'], 'valid')
|
||||
self.assertGreater(result['days_remaining'], 50)
|
||||
|
||||
def test_check_cert_via_ssl_returns_expired_for_past_cert(self):
|
||||
"""_check_cert_via_ssl returns expired when cert is in the past."""
|
||||
der = self._make_der_cert(-5)
|
||||
mock_tls = MagicMock()
|
||||
mock_tls.__enter__ = MagicMock(return_value=mock_tls)
|
||||
mock_tls.__exit__ = MagicMock(return_value=False)
|
||||
mock_tls.getpeercert.return_value = der
|
||||
mock_raw = MagicMock()
|
||||
mock_raw.__enter__ = MagicMock(return_value=mock_raw)
|
||||
mock_raw.__exit__ = MagicMock(return_value=False)
|
||||
with patch('caddy_manager._socket.create_connection', return_value=mock_raw):
|
||||
with patch('caddy_manager._ssl.create_default_context') as mock_ctx:
|
||||
mock_ctx.return_value.wrap_socket.return_value = mock_tls
|
||||
result = CaddyManager._check_cert_via_ssl('host', 443)
|
||||
self.assertIsNotNone(result)
|
||||
self.assertEqual(result['status'], 'expired')
|
||||
self.assertLess(result['days_remaining'], 0)
|
||||
|
||||
def test_refresh_cert_status_lan_mode_returns_internal(self):
|
||||
"""LAN mode always returns status='internal' without SSL check."""
|
||||
mgr = _mgr(identity={'cell_name': 'x', 'domain_mode': 'lan'})
|
||||
with patch.object(CaddyManager, '_check_cert_via_ssl') as mock_ssl:
|
||||
result = mgr.refresh_cert_status()
|
||||
mock_ssl.assert_not_called()
|
||||
self.assertEqual(result['status'], 'internal')
|
||||
|
||||
def test_refresh_cert_status_acme_mode_calls_ssl_check(self):
|
||||
"""ACME mode calls _check_cert_via_ssl and persists the result."""
|
||||
mgr = _mgr(identity={'cell_name': 'alpha', 'domain_mode': 'pic_ngo'})
|
||||
expected = {'status': 'valid', 'expiry': '2026-12-01T00:00:00+00:00', 'days_remaining': 179}
|
||||
with patch.object(CaddyManager, '_check_cert_via_ssl', return_value=expected):
|
||||
result = mgr.refresh_cert_status()
|
||||
self.assertEqual(result['status'], 'valid')
|
||||
# Should have been persisted to identity
|
||||
mgr.config_manager.set_identity_field.assert_called_with('tls', expected)
|
||||
|
||||
def test_refresh_cert_status_ssl_failure_returns_unknown(self):
|
||||
"""When SSL check returns None, status is 'unknown'."""
|
||||
mgr = _mgr(identity={'cell_name': 'alpha', 'domain_mode': 'pic_ngo'})
|
||||
with patch.object(CaddyManager, '_check_cert_via_ssl', return_value=None):
|
||||
result = mgr.refresh_cert_status()
|
||||
self.assertEqual(result['status'], 'unknown')
|
||||
|
||||
def test_get_cert_status_fresh_refreshes_when_stale(self):
|
||||
"""get_cert_status_fresh triggers a refresh when cache is None."""
|
||||
mgr = _mgr(identity={'cell_name': 'alpha', 'domain_mode': 'pic_ngo'})
|
||||
mgr._cert_refreshed_at = None
|
||||
with patch.object(mgr, 'refresh_cert_status', return_value={'status': 'valid'}) as mock_ref:
|
||||
with patch.object(mgr, 'get_cert_status', return_value={'status': 'valid'}):
|
||||
mgr.get_cert_status_fresh()
|
||||
mock_ref.assert_called_once()
|
||||
|
||||
def test_get_cert_status_fresh_skips_refresh_when_recent(self):
|
||||
"""get_cert_status_fresh skips refresh when cache is fresh."""
|
||||
import time
|
||||
mgr = _mgr(identity={'cell_name': 'alpha', 'domain_mode': 'pic_ngo'})
|
||||
mgr._cert_refreshed_at = time.monotonic() # just refreshed
|
||||
with patch.object(mgr, 'refresh_cert_status') as mock_ref:
|
||||
with patch.object(mgr, 'get_cert_status', return_value={'status': 'valid'}):
|
||||
mgr.get_cert_status_fresh(max_age_seconds=300)
|
||||
mock_ref.assert_not_called()
|
||||
|
||||
|
||||
class TestGetCertStatusEnriched(unittest.TestCase):
|
||||
"""get_cert_status() returns domain, domain_mode, cert_type alongside tls fields."""
|
||||
|
||||
def test_includes_domain_and_mode_for_pic_ngo(self):
|
||||
mgr = _mgr(identity={
|
||||
'cell_name': 'alpha',
|
||||
'domain_mode': 'pic_ngo',
|
||||
'tls': {'status': 'valid', 'expiry': '2026-12-01T00:00:00+00:00', 'days_remaining': 180},
|
||||
})
|
||||
s = mgr.get_cert_status()
|
||||
self.assertEqual(s['domain_mode'], 'pic_ngo')
|
||||
self.assertEqual(s['domain'], '*.alpha.pic.ngo')
|
||||
self.assertEqual(s['cert_type'], 'acme')
|
||||
self.assertEqual(s['status'], 'valid')
|
||||
|
||||
def test_cert_type_is_internal_for_lan_mode(self):
|
||||
mgr = _mgr(identity={'cell_name': 'x', 'domain_mode': 'lan', 'tls': {}})
|
||||
s = mgr.get_cert_status()
|
||||
self.assertEqual(s['cert_type'], 'internal')
|
||||
self.assertIsNone(s['domain'])
|
||||
|
||||
def test_cert_type_is_custom_when_tls_says_so(self):
|
||||
mgr = _mgr(identity={
|
||||
'cell_name': 'x',
|
||||
'domain_mode': 'lan',
|
||||
'tls': {'cert_type': 'custom', 'status': 'valid',
|
||||
'expiry': '2027-01-01T00:00:00+00:00', 'days_remaining': 200},
|
||||
})
|
||||
s = mgr.get_cert_status()
|
||||
self.assertEqual(s['cert_type'], 'custom')
|
||||
|
||||
def test_domain_label_cloudflare(self):
|
||||
ident = {'domain_mode': 'cloudflare', 'domain_name': 'example.com'}
|
||||
self.assertEqual(CaddyManager._domain_label(ident), '*.example.com')
|
||||
|
||||
def test_domain_label_duckdns(self):
|
||||
ident = {'cell_name': 'beta', 'domain_mode': 'duckdns'}
|
||||
self.assertEqual(CaddyManager._domain_label(ident), '*.beta.duckdns.org')
|
||||
|
||||
def test_domain_label_http01(self):
|
||||
ident = {'domain_mode': 'http01', 'domain_name': 'myhost.noip.me'}
|
||||
self.assertEqual(CaddyManager._domain_label(ident), 'myhost.noip.me')
|
||||
|
||||
def test_domain_label_lan_is_none(self):
|
||||
self.assertIsNone(CaddyManager._domain_label({'domain_mode': 'lan'}))
|
||||
|
||||
|
||||
class TestRenewCert(unittest.TestCase):
|
||||
"""renew_cert() — mode guard, reload call, cache invalidation."""
|
||||
|
||||
def test_lan_mode_returns_error(self):
|
||||
mgr = _mgr(identity={'domain_mode': 'lan'})
|
||||
result = mgr.renew_cert()
|
||||
self.assertFalse(result['ok'])
|
||||
self.assertIn('LAN', result['error'])
|
||||
|
||||
def test_acme_mode_calls_regenerate(self):
|
||||
mgr = _mgr(identity={'domain_mode': 'pic_ngo'})
|
||||
with patch.object(mgr, 'regenerate_with_installed', return_value=True) as mock_regen:
|
||||
result = mgr.renew_cert()
|
||||
mock_regen.assert_called_once_with([])
|
||||
self.assertTrue(result['ok'])
|
||||
self.assertEqual(result['status'], 'pending')
|
||||
|
||||
def test_reload_failure_propagated(self):
|
||||
mgr = _mgr(identity={'domain_mode': 'cloudflare'})
|
||||
with patch.object(mgr, 'regenerate_with_installed', return_value=False):
|
||||
result = mgr.renew_cert()
|
||||
self.assertFalse(result['ok'])
|
||||
self.assertIn('reload failed', result['error'])
|
||||
|
||||
def test_invalidates_cache_on_success(self):
|
||||
import time
|
||||
mgr = _mgr(identity={'domain_mode': 'pic_ngo'})
|
||||
mgr._cert_refreshed_at = time.monotonic()
|
||||
with patch.object(mgr, 'regenerate_with_installed', return_value=True):
|
||||
mgr.renew_cert()
|
||||
self.assertIsNone(mgr._cert_refreshed_at)
|
||||
|
||||
|
||||
class TestUploadCustomCert(unittest.TestCase):
|
||||
"""upload_custom_cert() — validation, file writes, identity persistence, Caddyfile regen."""
|
||||
|
||||
def _make_pem_cert(self, days_remaining: int = 90):
|
||||
"""Return (cert_pem, key_pem) for a self-signed cert."""
|
||||
from cryptography import x509
|
||||
from cryptography.x509.oid import NameOID
|
||||
from cryptography.hazmat.primitives import hashes, serialization
|
||||
from cryptography.hazmat.primitives.asymmetric import rsa
|
||||
import datetime
|
||||
key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
|
||||
now = datetime.datetime.now(datetime.timezone.utc)
|
||||
expiry = now + datetime.timedelta(days=days_remaining)
|
||||
not_before = (now - datetime.timedelta(days=abs(days_remaining) + 10)
|
||||
if days_remaining < 0 else now - datetime.timedelta(days=1))
|
||||
cert = (
|
||||
x509.CertificateBuilder()
|
||||
.subject_name(x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, 'test.example.com')]))
|
||||
.issuer_name(x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, 'test.example.com')]))
|
||||
.public_key(key.public_key())
|
||||
.serial_number(x509.random_serial_number())
|
||||
.not_valid_before(not_before)
|
||||
.not_valid_after(expiry)
|
||||
.sign(key, hashes.SHA256())
|
||||
)
|
||||
cert_pem = cert.public_bytes(serialization.Encoding.PEM).decode()
|
||||
key_pem = key.private_bytes(
|
||||
serialization.Encoding.PEM,
|
||||
serialization.PrivateFormat.TraditionalOpenSSL,
|
||||
serialization.NoEncryption(),
|
||||
).decode()
|
||||
return cert_pem, key_pem
|
||||
|
||||
def test_rejects_invalid_cert_pem(self):
|
||||
mgr = _mgr()
|
||||
result = mgr.upload_custom_cert('not a cert', '-----BEGIN PRIVATE KEY-----\nXXX\n-----END PRIVATE KEY-----')
|
||||
self.assertFalse(result['ok'])
|
||||
self.assertIn('Invalid certificate', result['error'])
|
||||
|
||||
def test_rejects_invalid_key_pem(self):
|
||||
mgr = _mgr()
|
||||
cert_pem, _ = self._make_pem_cert()
|
||||
result = mgr.upload_custom_cert(cert_pem, 'not a key')
|
||||
self.assertFalse(result['ok'])
|
||||
self.assertIn('Invalid private key', result['error'])
|
||||
|
||||
def test_writes_files_to_certs_dir(self):
|
||||
mgr = _mgr(identity={'domain_mode': 'lan', 'cell_name': 'x'})
|
||||
cert_pem, key_pem = self._make_pem_cert()
|
||||
written = {}
|
||||
|
||||
def fake_open(path, mode='r', **kw):
|
||||
import unittest.mock
|
||||
m = unittest.mock.mock_open()()
|
||||
if 'w' in mode:
|
||||
written[path] = True
|
||||
return m
|
||||
|
||||
with patch('builtins.open', side_effect=fake_open):
|
||||
with patch('os.makedirs'):
|
||||
with patch.object(mgr, 'regenerate_with_installed', return_value=True):
|
||||
mgr.upload_custom_cert(cert_pem, key_pem)
|
||||
|
||||
self.assertTrue(any('cert.pem' in p for p in written))
|
||||
self.assertTrue(any('key.pem' in p for p in written))
|
||||
|
||||
def test_persists_custom_cert_type_to_identity(self):
|
||||
mgr = _mgr(identity={'domain_mode': 'lan', 'cell_name': 'x'})
|
||||
cert_pem, key_pem = self._make_pem_cert(days_remaining=90)
|
||||
|
||||
with patch('builtins.open', unittest.mock.mock_open()):
|
||||
with patch('os.makedirs'):
|
||||
with patch.object(mgr, 'regenerate_with_installed', return_value=True):
|
||||
result = mgr.upload_custom_cert(cert_pem, key_pem)
|
||||
|
||||
self.assertTrue(result['ok'])
|
||||
self.assertEqual(result['cert_type'], 'custom')
|
||||
self.assertEqual(result['status'], 'valid')
|
||||
mgr.config_manager.set_identity_field.assert_called_once()
|
||||
call_args = mgr.config_manager.set_identity_field.call_args
|
||||
self.assertEqual(call_args[0][0], 'tls')
|
||||
self.assertEqual(call_args[0][1]['cert_type'], 'custom')
|
||||
|
||||
def test_expired_cert_flagged_as_expired(self):
|
||||
mgr = _mgr(identity={'domain_mode': 'lan', 'cell_name': 'x'})
|
||||
cert_pem, key_pem = self._make_pem_cert(days_remaining=-5)
|
||||
|
||||
with patch('builtins.open', unittest.mock.mock_open()):
|
||||
with patch('os.makedirs'):
|
||||
with patch.object(mgr, 'regenerate_with_installed', return_value=True):
|
||||
result = mgr.upload_custom_cert(cert_pem, key_pem)
|
||||
|
||||
self.assertEqual(result['status'], 'expired')
|
||||
|
||||
def test_file_write_failure_returns_error(self):
|
||||
mgr = _mgr(identity={'domain_mode': 'lan'})
|
||||
cert_pem, key_pem = self._make_pem_cert()
|
||||
with patch('os.makedirs'):
|
||||
with patch('builtins.open', side_effect=OSError('no space')):
|
||||
result = mgr.upload_custom_cert(cert_pem, key_pem)
|
||||
self.assertFalse(result['ok'])
|
||||
self.assertIn('Failed to write', result['error'])
|
||||
|
||||
|
||||
class TestCaddyfileLanCustomCert(unittest.TestCase):
|
||||
"""_caddyfile_lan() uses the custom cert path when cert_type=custom."""
|
||||
|
||||
def test_default_uses_internal_cert_path(self):
|
||||
mgr = _mgr(identity={'cell_name': 'mycell', 'domain_mode': 'lan'})
|
||||
out = mgr.generate_caddyfile({'cell_name': 'mycell', 'domain_mode': 'lan'}, [])
|
||||
self.assertIn('/etc/caddy/internal/cert.pem', out)
|
||||
|
||||
def test_custom_cert_type_uses_shared_cert_path(self):
|
||||
mgr = _mgr(identity={
|
||||
'cell_name': 'mycell',
|
||||
'domain_mode': 'lan',
|
||||
'tls': {'cert_type': 'custom'},
|
||||
})
|
||||
out = mgr.generate_caddyfile({'cell_name': 'mycell', 'domain_mode': 'lan'}, [])
|
||||
self.assertIn('/config/caddy/certs/cert.pem', out)
|
||||
self.assertNotIn('/etc/caddy/internal/cert.pem', out)
|
||||
|
||||
|
||||
class TestPicNgoNoTokenFallback(unittest.TestCase):
|
||||
"""pic_ngo mode with no token falls back to lan so Caddy starts cleanly."""
|
||||
|
||||
def test_empty_token_generates_lan_caddyfile(self):
|
||||
mgr = _mgr()
|
||||
mgr.config_manager.configs = {'ddns': {'url': 'https://ddns.pic.ngo'}}
|
||||
mgr.config_manager.get_ddns_token.return_value = ''
|
||||
with patch.dict(os.environ, {}, clear=False):
|
||||
os.environ.pop('DDNS_TOKEN', None)
|
||||
os.environ.pop('DDNS_URL', None)
|
||||
out = mgr.generate_caddyfile({'cell_name': 'x', 'domain_mode': 'pic_ngo'}, [])
|
||||
self.assertIn('auto_https off', out)
|
||||
self.assertNotIn('dns pic_ngo', out)
|
||||
self.assertNotIn('token', out)
|
||||
|
||||
def test_missing_ddns_config_generates_lan_caddyfile(self):
|
||||
mgr = _mgr()
|
||||
mgr.config_manager.configs = {}
|
||||
mgr.config_manager.get_ddns_token.return_value = ''
|
||||
with patch.dict(os.environ, {}, clear=False):
|
||||
os.environ.pop('DDNS_TOKEN', None)
|
||||
os.environ.pop('DDNS_URL', None)
|
||||
out = mgr.generate_caddyfile({'cell_name': 'x', 'domain_mode': 'pic_ngo'}, [])
|
||||
self.assertIn('auto_https off', out)
|
||||
self.assertNotIn('dns pic_ngo', out)
|
||||
|
||||
|
||||
class TestDdnsApiStripsLegacySuffix(unittest.TestCase):
|
||||
"""_caddyfile_pic_ngo strips /api/v1 from ddns_api so the plugin doesn't double it."""
|
||||
|
||||
def test_api_v1_suffix_stripped_from_config_url(self):
|
||||
mgr = _mgr()
|
||||
mgr.config_manager.configs = {
|
||||
'ddns': {'url': 'https://ddns.pic.ngo/api/v1'},
|
||||
}
|
||||
mgr.config_manager.get_ddns_token.return_value = 'tok'
|
||||
with patch.dict(os.environ, {}, clear=False):
|
||||
os.environ.pop('DDNS_URL', None)
|
||||
out = mgr.generate_caddyfile({'cell_name': 'x', 'domain_mode': 'pic_ngo'}, [])
|
||||
self.assertIn('api_base_url https://ddns.pic.ngo', out)
|
||||
self.assertNotIn('api_base_url https://ddns.pic.ngo/api/v1', out)
|
||||
|
||||
def test_clean_url_is_unchanged(self):
|
||||
mgr = _mgr()
|
||||
mgr.config_manager.configs = {
|
||||
'ddns': {'url': 'https://ddns.pic.ngo'},
|
||||
}
|
||||
mgr.config_manager.get_ddns_token.return_value = 'tok'
|
||||
with patch.dict(os.environ, {}, clear=False):
|
||||
os.environ.pop('DDNS_URL', None)
|
||||
out = mgr.generate_caddyfile({'cell_name': 'x', 'domain_mode': 'pic_ngo'}, [])
|
||||
self.assertIn('api_base_url https://ddns.pic.ngo', out)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
||||
@@ -0,0 +1,532 @@
|
||||
"""Integration tests for registry-driven CaddyManager and NetworkManager routing.
|
||||
|
||||
These tests cover the new registry path introduced in Step 5 of the PIC Services
|
||||
Architecture. The no-registry (fallback) paths are already covered by
|
||||
test_caddy_manager.py and test_network_manager.py.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import shutil
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'api'))
|
||||
|
||||
from caddy_manager import CaddyManager # noqa: E402
|
||||
from network_manager import NetworkManager # noqa: E402
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Shared helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _mgr_with_registry(registry=None):
|
||||
"""Build a CaddyManager wired to an optional mock registry."""
|
||||
cm = MagicMock()
|
||||
cm.get_identity.return_value = {}
|
||||
return CaddyManager(config_manager=cm, service_registry=registry)
|
||||
|
||||
|
||||
def _mock_registry():
|
||||
"""Return a mock ServiceRegistry that reproduces 3 store service routes."""
|
||||
reg = MagicMock()
|
||||
reg.get_caddy_routes.return_value = [
|
||||
{
|
||||
'service_id': 'calendar',
|
||||
'subdomain': 'calendar',
|
||||
'backend': 'cell-radicale:5232',
|
||||
'extra_subdomains': [],
|
||||
'extra_backends': {},
|
||||
},
|
||||
{
|
||||
'service_id': 'email',
|
||||
'subdomain': 'mail',
|
||||
'backend': 'cell-rainloop:8888',
|
||||
'extra_subdomains': ['webmail'],
|
||||
'extra_backends': {},
|
||||
},
|
||||
{
|
||||
'service_id': 'files',
|
||||
'subdomain': 'files',
|
||||
'backend': 'cell-filegator:8080',
|
||||
'extra_subdomains': ['webdav'],
|
||||
'extra_backends': {'webdav': 'cell-webdav:80'},
|
||||
},
|
||||
]
|
||||
return reg
|
||||
|
||||
|
||||
def _nm(registry=None):
|
||||
"""Build a NetworkManager backed by temp dirs and an optional mock registry."""
|
||||
tmpdir = tempfile.mkdtemp()
|
||||
nm = NetworkManager(
|
||||
data_dir=os.path.join(tmpdir, 'data'),
|
||||
config_dir=os.path.join(tmpdir, 'config'),
|
||||
service_registry=registry,
|
||||
)
|
||||
nm._tmpdir = tmpdir # stash so the caller can clean up
|
||||
return nm
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TestBuildRegistryServiceRoutes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestBuildRegistryServiceRoutes(unittest.TestCase):
|
||||
|
||||
def test_returns_api_only_when_no_registry(self):
|
||||
"""service_registry=None produces only the @api block."""
|
||||
mgr = _mgr_with_registry(registry=None)
|
||||
domain = 'alpha.pic.ngo'
|
||||
result = mgr._build_registry_service_routes(domain)
|
||||
self.assertIn('@api host api.alpha.pic.ngo', result)
|
||||
self.assertIn('reverse_proxy cell-api:3000', result)
|
||||
self.assertNotIn('@calendar', result)
|
||||
self.assertNotIn('@mail', result)
|
||||
|
||||
def test_returns_api_only_when_registry_empty(self):
|
||||
"""An empty route list from the registry produces only the @api block."""
|
||||
reg = MagicMock()
|
||||
reg.get_caddy_routes.return_value = []
|
||||
mgr = _mgr_with_registry(registry=reg)
|
||||
domain = 'alpha.pic.ngo'
|
||||
result = mgr._build_registry_service_routes(domain)
|
||||
self.assertIn('@api host api.alpha.pic.ngo', result)
|
||||
self.assertIn('reverse_proxy cell-api:3000', result)
|
||||
self.assertNotIn('@calendar', result)
|
||||
self.assertNotIn('@mail', result)
|
||||
|
||||
def test_returns_api_only_on_registry_error(self):
|
||||
"""When get_caddy_routes raises, only the @api block is produced."""
|
||||
reg = MagicMock()
|
||||
reg.get_caddy_routes.side_effect = Exception('registry unavailable')
|
||||
mgr = _mgr_with_registry(registry=reg)
|
||||
domain = 'alpha.pic.ngo'
|
||||
result = mgr._build_registry_service_routes(domain)
|
||||
self.assertIn('@api host api.alpha.pic.ngo', result)
|
||||
self.assertIn('reverse_proxy cell-api:3000', result)
|
||||
self.assertNotIn('@calendar', result)
|
||||
self.assertNotIn('@mail', result)
|
||||
|
||||
def test_single_service_no_extras(self):
|
||||
"""One service with no extra_subdomains produces one matcher + handle + api block."""
|
||||
reg = MagicMock()
|
||||
reg.get_caddy_routes.return_value = [
|
||||
{
|
||||
'service_id': 'calendar',
|
||||
'subdomain': 'calendar',
|
||||
'backend': 'cell-radicale:5232',
|
||||
'extra_subdomains': [],
|
||||
'extra_backends': {},
|
||||
}
|
||||
]
|
||||
mgr = _mgr_with_registry(registry=reg)
|
||||
result = mgr._build_registry_service_routes('test.cell')
|
||||
self.assertIn('@calendar host calendar.test.cell', result)
|
||||
self.assertIn('reverse_proxy cell-radicale:5232', result)
|
||||
self.assertIn('@api host api.test.cell', result)
|
||||
self.assertIn('reverse_proxy cell-api:3000', result)
|
||||
# Only two named-matcher definition lines: @calendar and @api
|
||||
matcher_lines = [l for l in result.splitlines() if l.strip().startswith('@') and 'host' in l]
|
||||
self.assertEqual(len(matcher_lines), 2)
|
||||
|
||||
def test_extra_subdomain_same_backend(self):
|
||||
"""An extra_subdomain NOT in extra_backends shares the primary matcher host line."""
|
||||
reg = MagicMock()
|
||||
reg.get_caddy_routes.return_value = [
|
||||
{
|
||||
'service_id': 'email',
|
||||
'subdomain': 'mail',
|
||||
'backend': 'cell-rainloop:8888',
|
||||
'extra_subdomains': ['webmail'],
|
||||
'extra_backends': {}, # webmail not listed → shares backend
|
||||
}
|
||||
]
|
||||
mgr = _mgr_with_registry(registry=reg)
|
||||
result = mgr._build_registry_service_routes('test.cell')
|
||||
# Both subdomains appear in the same host matcher line
|
||||
self.assertIn('@mail host mail.test.cell webmail.test.cell', result)
|
||||
# Only one reverse_proxy for cell-rainloop (shared block)
|
||||
self.assertEqual(result.count('reverse_proxy cell-rainloop:8888'), 1)
|
||||
# No separate @webmail block
|
||||
self.assertNotIn('@webmail host', result)
|
||||
|
||||
def test_extra_subdomain_different_backend(self):
|
||||
"""An extra_subdomain listed in extra_backends gets its own matcher + handle block."""
|
||||
reg = MagicMock()
|
||||
reg.get_caddy_routes.return_value = [
|
||||
{
|
||||
'service_id': 'files',
|
||||
'subdomain': 'files',
|
||||
'backend': 'cell-filegator:8080',
|
||||
'extra_subdomains': ['webdav'],
|
||||
'extra_backends': {'webdav': 'cell-webdav:80'},
|
||||
}
|
||||
]
|
||||
mgr = _mgr_with_registry(registry=reg)
|
||||
result = mgr._build_registry_service_routes('test.cell')
|
||||
# files gets its own block (webdav not in shared list)
|
||||
self.assertIn('@files host files.test.cell', result)
|
||||
self.assertIn('reverse_proxy cell-filegator:8080', result)
|
||||
# webdav gets a separate block
|
||||
self.assertIn('@webdav host webdav.test.cell', result)
|
||||
self.assertIn('reverse_proxy cell-webdav:80', result)
|
||||
# webdav must NOT appear in the @files host line
|
||||
files_line = [l for l in result.splitlines() if '@files host' in l][0]
|
||||
self.assertNotIn('webdav', files_line)
|
||||
|
||||
def test_api_always_appended(self):
|
||||
"""The @api block is always the last block even when registry has no api entry."""
|
||||
reg = _mock_registry()
|
||||
mgr = _mgr_with_registry(registry=reg)
|
||||
result = mgr._build_registry_service_routes('alpha.pic.ngo')
|
||||
self.assertIn('@api host api.alpha.pic.ngo', result)
|
||||
self.assertIn('reverse_proxy cell-api:3000', result)
|
||||
# api block is at the end
|
||||
api_idx = result.rfind('@api')
|
||||
other_matchers = ['@calendar', '@mail', '@files', '@webdav']
|
||||
for m in other_matchers:
|
||||
self.assertLess(result.index(m), api_idx,
|
||||
f'{m} should appear before @api')
|
||||
|
||||
def test_api_not_duplicated_when_registry_returns_api(self):
|
||||
"""Even if registry somehow returns an 'api' route, the injected api block is cell-api:3000."""
|
||||
reg = MagicMock()
|
||||
reg.get_caddy_routes.return_value = [
|
||||
{
|
||||
'service_id': 'api',
|
||||
'subdomain': 'api',
|
||||
'backend': 'cell-other:9999', # wrong backend — should be overridden
|
||||
'extra_subdomains': [],
|
||||
'extra_backends': {},
|
||||
}
|
||||
]
|
||||
mgr = _mgr_with_registry(registry=reg)
|
||||
result = mgr._build_registry_service_routes('test.cell')
|
||||
# The infrastructure api block is always appended with the canonical backend
|
||||
self.assertIn('reverse_proxy cell-api:3000', result)
|
||||
# api host matcher appears at least once (from registry AND from append)
|
||||
self.assertGreaterEqual(result.count('@api host api.test.cell'), 1)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TestHttp01ServicePairs
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestHttp01ServicePairs(unittest.TestCase):
|
||||
|
||||
def test_pairs_from_registry(self):
|
||||
"""With the 3 builtins the pairs list matches expected (subdomain, backend) tuples."""
|
||||
reg = _mock_registry()
|
||||
mgr = _mgr_with_registry(registry=reg)
|
||||
pairs = mgr._http01_service_pairs()
|
||||
pairs_dict = dict(pairs)
|
||||
self.assertEqual(pairs_dict['calendar'], 'cell-radicale:5232')
|
||||
self.assertEqual(pairs_dict['mail'], 'cell-rainloop:8888')
|
||||
self.assertEqual(pairs_dict['webmail'], 'cell-rainloop:8888')
|
||||
self.assertEqual(pairs_dict['files'], 'cell-filegator:8080')
|
||||
self.assertEqual(pairs_dict['webdav'], 'cell-webdav:80')
|
||||
self.assertEqual(pairs_dict['api'], 'cell-api:3000')
|
||||
|
||||
def test_webdav_gets_own_backend(self):
|
||||
"""webdav must map to cell-webdav:80, not to cell-filegator:8080."""
|
||||
reg = _mock_registry()
|
||||
mgr = _mgr_with_registry(registry=reg)
|
||||
pairs = mgr._http01_service_pairs()
|
||||
webdav_entry = next((b for s, b in pairs if s == 'webdav'), None)
|
||||
self.assertIsNotNone(webdav_entry)
|
||||
self.assertEqual(webdav_entry, 'cell-webdav:80')
|
||||
self.assertNotEqual(webdav_entry, 'cell-filegator:8080')
|
||||
|
||||
def test_only_api_when_no_registry(self):
|
||||
"""Without a registry only the api pair is returned."""
|
||||
mgr = _mgr_with_registry(registry=None)
|
||||
pairs = mgr._http01_service_pairs()
|
||||
subdomains = [s for s, _ in pairs]
|
||||
self.assertIn('api', subdomains)
|
||||
self.assertNotIn('calendar', subdomains)
|
||||
self.assertNotIn('mail', subdomains)
|
||||
self.assertNotIn('files', subdomains)
|
||||
|
||||
def test_only_api_on_registry_error(self):
|
||||
"""When get_caddy_routes raises, only the api pair is present."""
|
||||
reg = MagicMock()
|
||||
reg.get_caddy_routes.side_effect = RuntimeError('boom')
|
||||
mgr = _mgr_with_registry(registry=reg)
|
||||
pairs = mgr._http01_service_pairs()
|
||||
subdomains = [s for s, _ in pairs]
|
||||
self.assertIn('api', subdomains)
|
||||
self.assertNotIn('calendar', subdomains)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TestCaddyfileWithRegistry
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestCaddyfileWithRegistry(unittest.TestCase):
|
||||
|
||||
def _generate(self, domain_mode, cell_name='alpha', domain_name=None,
|
||||
registry=None, services=None):
|
||||
reg = registry if registry is not None else _mock_registry()
|
||||
mgr = _mgr_with_registry(registry=reg)
|
||||
identity = {'cell_name': cell_name, 'domain_mode': domain_mode}
|
||||
if domain_name:
|
||||
identity['domain_name'] = domain_name
|
||||
return mgr.generate_caddyfile(identity, services or [])
|
||||
|
||||
def test_pic_ngo_with_registry_has_correct_routes(self):
|
||||
"""pic_ngo Caddyfile has all service matchers with correct subdomains and backends."""
|
||||
out = self._generate('pic_ngo', cell_name='alpha')
|
||||
# calendar
|
||||
self.assertIn('@calendar host calendar.alpha.pic.ngo', out)
|
||||
self.assertIn('reverse_proxy cell-radicale:5232', out)
|
||||
# mail + webmail share one matcher
|
||||
self.assertIn('@mail host mail.alpha.pic.ngo webmail.alpha.pic.ngo', out)
|
||||
self.assertIn('reverse_proxy cell-rainloop:8888', out)
|
||||
# files
|
||||
self.assertIn('@files host files.alpha.pic.ngo', out)
|
||||
self.assertIn('reverse_proxy cell-filegator:8080', out)
|
||||
# webdav separate block
|
||||
self.assertIn('@webdav host webdav.alpha.pic.ngo', out)
|
||||
self.assertIn('reverse_proxy cell-webdav:80', out)
|
||||
# api always present
|
||||
self.assertIn('@api host api.alpha.pic.ngo', out)
|
||||
self.assertIn('reverse_proxy cell-api:3000', out)
|
||||
|
||||
def test_cloudflare_with_registry_uses_registry_routes(self):
|
||||
"""cloudflare Caddyfile routes are sourced from registry, not hardcoded."""
|
||||
out = self._generate('cloudflare', cell_name='beta',
|
||||
domain_name='example.com')
|
||||
self.assertIn('@calendar host calendar.example.com', out)
|
||||
self.assertIn('@mail host mail.example.com webmail.example.com', out)
|
||||
self.assertIn('@files host files.example.com', out)
|
||||
self.assertIn('@webdav host webdav.example.com', out)
|
||||
self.assertIn('@api host api.example.com', out)
|
||||
# Correct DNS plugin block is still present
|
||||
self.assertIn('dns cloudflare {$CF_API_TOKEN}', out)
|
||||
|
||||
def test_duckdns_with_registry_uses_registry_routes(self):
|
||||
"""duckdns Caddyfile routes are sourced from registry."""
|
||||
out = self._generate('duckdns', cell_name='gamma')
|
||||
self.assertIn('@calendar host calendar.gamma.duckdns.org', out)
|
||||
self.assertIn('@api host api.gamma.duckdns.org', out)
|
||||
self.assertIn('dns duckdns {$DUCKDNS_TOKEN}', out)
|
||||
|
||||
def test_http01_with_registry_has_per_host_blocks(self):
|
||||
"""http01 Caddyfile has individual per-host blocks for every service subdomain."""
|
||||
out = self._generate('http01', cell_name='delta',
|
||||
domain_name='delta.noip.me')
|
||||
self.assertIn('calendar.delta.noip.me {', out)
|
||||
self.assertIn('mail.delta.noip.me {', out)
|
||||
self.assertIn('webmail.delta.noip.me {', out)
|
||||
self.assertIn('files.delta.noip.me {', out)
|
||||
self.assertIn('webdav.delta.noip.me {', out)
|
||||
self.assertIn('api.delta.noip.me {', out)
|
||||
# Correct backends
|
||||
self.assertIn('reverse_proxy cell-radicale:5232', out)
|
||||
self.assertIn('reverse_proxy cell-rainloop:8888', out)
|
||||
self.assertIn('reverse_proxy cell-filegator:8080', out)
|
||||
self.assertIn('reverse_proxy cell-webdav:80', out)
|
||||
|
||||
def test_pic_ngo_api_only_when_registry_empty(self):
|
||||
"""pic_ngo emits only the api block when registry returns empty list."""
|
||||
reg = MagicMock()
|
||||
reg.get_caddy_routes.return_value = []
|
||||
out = self._generate('pic_ngo', cell_name='alpha', registry=reg)
|
||||
self.assertIn('@api host api.alpha.pic.ngo', out)
|
||||
self.assertNotIn('@calendar', out)
|
||||
self.assertNotIn('@mail', out)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TestNetworkManagerGetServiceSubdomains
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestNetworkManagerGetServiceSubdomains(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.managers = []
|
||||
|
||||
def tearDown(self):
|
||||
for nm in self.managers:
|
||||
shutil.rmtree(nm._tmpdir, ignore_errors=True)
|
||||
|
||||
def _make(self, registry=None):
|
||||
nm = _nm(registry=registry)
|
||||
self.managers.append(nm)
|
||||
return nm
|
||||
|
||||
def test_no_registry_returns_empty(self):
|
||||
"""Without a registry an empty list is returned."""
|
||||
nm = self._make(registry=None)
|
||||
subs = nm._get_service_subdomains()
|
||||
self.assertEqual(subs, [])
|
||||
|
||||
def test_registry_returns_all_subdomains(self):
|
||||
"""Primary + extra_subdomains from all routes are returned."""
|
||||
reg = _mock_registry()
|
||||
nm = self._make(registry=reg)
|
||||
subs = nm._get_service_subdomains()
|
||||
# calendar (primary), mail (primary), webmail (extra), files (primary), webdav (extra)
|
||||
for expected in ('calendar', 'mail', 'webmail', 'files', 'webdav'):
|
||||
self.assertIn(expected, subs)
|
||||
|
||||
def test_registry_error_returns_empty(self):
|
||||
"""When get_caddy_routes raises, an empty list is returned."""
|
||||
reg = MagicMock()
|
||||
reg.get_caddy_routes.side_effect = Exception('broken registry')
|
||||
nm = self._make(registry=reg)
|
||||
subs = nm._get_service_subdomains()
|
||||
self.assertEqual(subs, [])
|
||||
|
||||
def test_registry_extra_subdomains_included(self):
|
||||
"""extra_subdomains from each route are included in the returned list."""
|
||||
reg = MagicMock()
|
||||
reg.get_caddy_routes.return_value = [
|
||||
{
|
||||
'service_id': 'files',
|
||||
'subdomain': 'files',
|
||||
'backend': 'cell-filegator:8080',
|
||||
'extra_subdomains': ['webdav', 'dav'],
|
||||
'extra_backends': {},
|
||||
}
|
||||
]
|
||||
nm = self._make(registry=reg)
|
||||
subs = nm._get_service_subdomains()
|
||||
self.assertIn('files', subs)
|
||||
self.assertIn('webdav', subs)
|
||||
self.assertIn('dav', subs)
|
||||
|
||||
def test_build_dns_records_with_registry(self):
|
||||
"""All registry subdomains appear as A records in _build_dns_records output."""
|
||||
reg = _mock_registry()
|
||||
nm = self._make(registry=reg)
|
||||
# Override WG IP lookup so we get a predictable value
|
||||
nm._get_wg_server_ip = lambda: '10.0.0.1'
|
||||
records = nm._build_dns_records('mycell', '172.20.0.0/16')
|
||||
names = [r['name'] for r in records]
|
||||
for expected in ('mycell', 'api', 'webui', 'calendar', 'mail',
|
||||
'webmail', 'files', 'webdav'):
|
||||
self.assertIn(expected, names,
|
||||
f'{expected!r} should be in DNS records but is not')
|
||||
# All records must point to the WG server IP
|
||||
for r in records:
|
||||
self.assertEqual(r['value'], '10.0.0.1')
|
||||
self.assertEqual(r['type'], 'A')
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TestNetworkManagerStaleSet
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestNetworkManagerStaleSet(unittest.TestCase):
|
||||
"""Verify that registry subdomains drive stale record cleanup in update_split_horizon_zone."""
|
||||
|
||||
def setUp(self):
|
||||
self.test_dir = tempfile.mkdtemp()
|
||||
data_dir = os.path.join(self.test_dir, 'data')
|
||||
config_dir = os.path.join(self.test_dir, 'config')
|
||||
os.makedirs(os.path.join(data_dir, 'dns'), exist_ok=True)
|
||||
os.makedirs(os.path.join(config_dir, 'dns'), exist_ok=True)
|
||||
self.reg = _mock_registry()
|
||||
self.nm = NetworkManager(
|
||||
data_dir=data_dir,
|
||||
config_dir=config_dir,
|
||||
service_registry=self.reg,
|
||||
)
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.test_dir, ignore_errors=True)
|
||||
|
||||
def _write_zone(self, zone_name: str, content: str):
|
||||
path = os.path.join(self.nm.dns_zones_dir, f'{zone_name}.zone')
|
||||
with open(path, 'w') as f:
|
||||
f.write(content)
|
||||
|
||||
def test_stale_set_includes_registry_subdomains(self):
|
||||
"""Registry subdomains (calendar, mail, webmail, files, webdav) are treated as
|
||||
stale service records and removed from the parent zone during
|
||||
update_split_horizon_zone."""
|
||||
import subprocess
|
||||
# Build a parent zone with stale service records that the registry knows about
|
||||
stale_records = [
|
||||
{'name': 'pic2', 'type': 'A', 'value': '10.0.0.1'},
|
||||
{'name': 'api', 'type': 'A', 'value': '10.0.0.1'},
|
||||
{'name': 'webui', 'type': 'A', 'value': '10.0.0.1'},
|
||||
{'name': 'calendar', 'type': 'A', 'value': '10.0.0.1'},
|
||||
{'name': 'mail', 'type': 'A', 'value': '10.0.0.1'},
|
||||
{'name': 'webmail', 'type': 'A', 'value': '10.0.0.1'},
|
||||
{'name': 'files', 'type': 'A', 'value': '10.0.0.1'},
|
||||
{'name': 'webdav', 'type': 'A', 'value': '10.0.0.1'},
|
||||
]
|
||||
from unittest.mock import patch
|
||||
with patch('subprocess.run'):
|
||||
self.nm.update_dns_zone('pic.ngo', stale_records)
|
||||
self.nm.update_split_horizon_zone(
|
||||
'pic2.pic.ngo', '172.20.0.2', primary_domain='pic.ngo'
|
||||
)
|
||||
|
||||
parent_zone = os.path.join(self.nm.dns_zones_dir, 'pic.ngo.zone')
|
||||
content = open(parent_zone).read()
|
||||
|
||||
# All registry subdomains must be gone
|
||||
for stale in ('api', 'webui', 'calendar', 'mail', 'webmail', 'files', 'webdav'):
|
||||
# Check that no line *starts* with the stale name (to avoid false positives
|
||||
# on SOA/NS lines that may contain the zone name as a suffix)
|
||||
lines_with_stale = [
|
||||
l for l in content.splitlines()
|
||||
if l.startswith(stale + ' ') or l.startswith(stale + '\t')
|
||||
]
|
||||
self.assertEqual(
|
||||
lines_with_stale, [],
|
||||
f'Stale record {stale!r} should have been removed from pic.ngo zone'
|
||||
)
|
||||
|
||||
def test_stale_set_uses_registry_not_hardcoded(self):
|
||||
"""When a registry provides a custom subdomain, it is treated as stale too."""
|
||||
custom_reg = MagicMock()
|
||||
custom_reg.get_caddy_routes.return_value = [
|
||||
{
|
||||
'service_id': 'chat',
|
||||
'subdomain': 'chat',
|
||||
'backend': 'cell-chat:9000',
|
||||
'extra_subdomains': ['im'],
|
||||
'extra_backends': {},
|
||||
}
|
||||
]
|
||||
data_dir = os.path.join(self.test_dir, 'data2')
|
||||
config_dir = os.path.join(self.test_dir, 'config2')
|
||||
os.makedirs(os.path.join(data_dir, 'dns'), exist_ok=True)
|
||||
os.makedirs(os.path.join(config_dir, 'dns'), exist_ok=True)
|
||||
nm = NetworkManager(data_dir=data_dir, config_dir=config_dir,
|
||||
service_registry=custom_reg)
|
||||
|
||||
stale_records = [
|
||||
{'name': 'pic3', 'type': 'A', 'value': '10.0.0.1'},
|
||||
{'name': 'chat', 'type': 'A', 'value': '10.0.0.1'},
|
||||
{'name': 'im', 'type': 'A', 'value': '10.0.0.1'},
|
||||
]
|
||||
from unittest.mock import patch
|
||||
with patch('subprocess.run'):
|
||||
nm.update_dns_zone('pic.ngo', stale_records)
|
||||
nm.update_split_horizon_zone(
|
||||
'pic3.pic.ngo', '172.20.0.2', primary_domain='pic.ngo'
|
||||
)
|
||||
|
||||
parent_zone = os.path.join(nm.dns_zones_dir, 'pic.ngo.zone')
|
||||
content = open(parent_zone).read()
|
||||
for stale in ('chat', 'im'):
|
||||
lines_with_stale = [
|
||||
l for l in content.splitlines()
|
||||
if l.startswith(stale + ' ') or l.startswith(stale + '\t')
|
||||
]
|
||||
self.assertEqual(
|
||||
lines_with_stale, [],
|
||||
f'Custom registry subdomain {stale!r} should have been removed'
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -24,12 +24,20 @@ sys.path.insert(0, str(api_dir))
|
||||
|
||||
from app import app
|
||||
|
||||
_INSTALLED = {'id': 'calendar', 'installed': True}
|
||||
|
||||
|
||||
class TestGetCalendarUsers(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
app.config['TESTING'] = True
|
||||
self.client = app.test_client()
|
||||
self._sr_patcher = patch('app.service_registry')
|
||||
mock_sr = self._sr_patcher.start()
|
||||
mock_sr.get.return_value = _INSTALLED
|
||||
|
||||
def tearDown(self):
|
||||
self._sr_patcher.stop()
|
||||
|
||||
@patch('app.calendar_manager')
|
||||
def test_get_users_returns_200_with_list(self, mock_cm):
|
||||
@@ -63,6 +71,12 @@ class TestCreateCalendarUser(unittest.TestCase):
|
||||
def setUp(self):
|
||||
app.config['TESTING'] = True
|
||||
self.client = app.test_client()
|
||||
self._sr_patcher = patch('app.service_registry')
|
||||
mock_sr = self._sr_patcher.start()
|
||||
mock_sr.get.return_value = _INSTALLED
|
||||
|
||||
def tearDown(self):
|
||||
self._sr_patcher.stop()
|
||||
|
||||
@patch('app.calendar_manager')
|
||||
def test_create_user_returns_200_on_valid_body(self, mock_cm):
|
||||
@@ -133,6 +147,12 @@ class TestDeleteCalendarUser(unittest.TestCase):
|
||||
def setUp(self):
|
||||
app.config['TESTING'] = True
|
||||
self.client = app.test_client()
|
||||
self._sr_patcher = patch('app.service_registry')
|
||||
mock_sr = self._sr_patcher.start()
|
||||
mock_sr.get.return_value = _INSTALLED
|
||||
|
||||
def tearDown(self):
|
||||
self._sr_patcher.stop()
|
||||
|
||||
@patch('app.calendar_manager')
|
||||
def test_delete_user_returns_200_on_success(self, mock_cm):
|
||||
@@ -161,6 +181,12 @@ class TestCreateCalendar(unittest.TestCase):
|
||||
def setUp(self):
|
||||
app.config['TESTING'] = True
|
||||
self.client = app.test_client()
|
||||
self._sr_patcher = patch('app.service_registry')
|
||||
mock_sr = self._sr_patcher.start()
|
||||
mock_sr.get.return_value = _INSTALLED
|
||||
|
||||
def tearDown(self):
|
||||
self._sr_patcher.stop()
|
||||
|
||||
@patch('app.calendar_manager')
|
||||
def test_create_calendar_returns_200_on_valid_body(self, mock_cm):
|
||||
@@ -228,6 +254,12 @@ class TestAddCalendarEvent(unittest.TestCase):
|
||||
def setUp(self):
|
||||
app.config['TESTING'] = True
|
||||
self.client = app.test_client()
|
||||
self._sr_patcher = patch('app.service_registry')
|
||||
mock_sr = self._sr_patcher.start()
|
||||
mock_sr.get.return_value = _INSTALLED
|
||||
|
||||
def tearDown(self):
|
||||
self._sr_patcher.stop()
|
||||
|
||||
@patch('app.calendar_manager')
|
||||
def test_add_event_returns_200_on_valid_body(self, mock_cm):
|
||||
@@ -294,6 +326,12 @@ class TestGetCalendarEvents(unittest.TestCase):
|
||||
def setUp(self):
|
||||
app.config['TESTING'] = True
|
||||
self.client = app.test_client()
|
||||
self._sr_patcher = patch('app.service_registry')
|
||||
mock_sr = self._sr_patcher.start()
|
||||
mock_sr.get.return_value = _INSTALLED
|
||||
|
||||
def tearDown(self):
|
||||
self._sr_patcher.stop()
|
||||
|
||||
@patch('app.calendar_manager')
|
||||
def test_get_events_returns_200_with_events(self, mock_cm):
|
||||
@@ -354,6 +392,12 @@ class TestCalendarConnectivity(unittest.TestCase):
|
||||
def setUp(self):
|
||||
app.config['TESTING'] = True
|
||||
self.client = app.test_client()
|
||||
self._sr_patcher = patch('app.service_registry')
|
||||
mock_sr = self._sr_patcher.start()
|
||||
mock_sr.get.return_value = _INSTALLED
|
||||
|
||||
def tearDown(self):
|
||||
self._sr_patcher.stop()
|
||||
|
||||
@patch('app.calendar_manager')
|
||||
def test_connectivity_returns_200_with_result(self, mock_cm):
|
||||
|
||||
@@ -144,7 +144,7 @@ class TestApplyAllDnsRulesPassesCellLinks(unittest.TestCase):
|
||||
cell_links=cell_links,
|
||||
)
|
||||
mock_gen.assert_called_once_with(
|
||||
[], '/tmp/fake_Corefile', 'cell', cell_links
|
||||
[], '/tmp/fake_Corefile', 'cell', cell_links, None
|
||||
)
|
||||
|
||||
def test_cell_links_none_forwarded_as_none(self):
|
||||
@@ -156,7 +156,7 @@ class TestApplyAllDnsRulesPassesCellLinks(unittest.TestCase):
|
||||
domain='cell',
|
||||
cell_links=None,
|
||||
)
|
||||
mock_gen.assert_called_once_with([], '/tmp/fake_Corefile', 'cell', None)
|
||||
mock_gen.assert_called_once_with([], '/tmp/fake_Corefile', 'cell', None, None)
|
||||
|
||||
def test_reload_called_on_success(self):
|
||||
with patch.object(firewall_manager, 'generate_corefile', return_value=True), \
|
||||
|
||||
@@ -190,5 +190,120 @@ class TestConfigApplyRoute(unittest.TestCase):
|
||||
self.assertIn('error', json.loads(r.data))
|
||||
|
||||
|
||||
class TestDdnsConfigUpdatesFiresIdentityChanged(unittest.TestCase):
|
||||
"""PUT /api/ddns must publish IDENTITY_CHANGED so CaddyManager regenerates."""
|
||||
|
||||
def setUp(self):
|
||||
app.config['TESTING'] = True
|
||||
self.client = app.test_client()
|
||||
|
||||
def _put_ddns(self, payload=None):
|
||||
if payload is None:
|
||||
payload = {'domain_mode': 'pic_ngo', 'cell_name': 'test', 'domain': 'pic_ngo'}
|
||||
return self.client.put(
|
||||
'/api/ddns',
|
||||
data=json.dumps(payload),
|
||||
content_type='application/json',
|
||||
)
|
||||
|
||||
@patch('app.service_bus')
|
||||
@patch('app.config_manager')
|
||||
def test_fires_identity_changed_on_success(self, mock_cm, mock_bus):
|
||||
mock_cm.configs = {
|
||||
'_identity': {
|
||||
'cell_name': 'test',
|
||||
'domain': 'pic_ngo',
|
||||
'domain_name': '',
|
||||
'domain_mode': 'pic_ngo',
|
||||
}
|
||||
}
|
||||
mock_cm.set_identity_field = MagicMock()
|
||||
mock_cm.get_effective_domain = MagicMock(return_value='test.pic.ngo')
|
||||
mock_cm.validate_ddns_config = MagicMock(return_value=None)
|
||||
|
||||
r = self._put_ddns()
|
||||
|
||||
self.assertIn(r.status_code, (200, 204))
|
||||
self.assertTrue(mock_bus.publish_event.called,
|
||||
'Expected service_bus.publish_event to be called')
|
||||
args = mock_bus.publish_event.call_args
|
||||
# first positional arg should be an EventType with value IDENTITY_CHANGED
|
||||
event_arg = args[0][0]
|
||||
self.assertEqual(str(event_arg).upper().replace('.', '_'),
|
||||
'EVENTTYPE_IDENTITY_CHANGED')
|
||||
|
||||
@patch('app.service_bus')
|
||||
@patch('app.config_manager')
|
||||
def test_identity_changed_payload_contains_domain_fields(self, mock_cm, mock_bus):
|
||||
mock_cm.configs = {
|
||||
'_identity': {
|
||||
'cell_name': 'mycell',
|
||||
'domain': 'pic_ngo',
|
||||
'domain_name': '',
|
||||
'domain_mode': 'pic_ngo',
|
||||
}
|
||||
}
|
||||
mock_cm.set_identity_field = MagicMock()
|
||||
mock_cm.get_effective_domain = MagicMock(return_value='mycell.pic.ngo')
|
||||
mock_cm.validate_ddns_config = MagicMock(return_value=None)
|
||||
|
||||
self._put_ddns({'domain_mode': 'pic_ngo', 'cell_name': 'mycell', 'domain': 'pic_ngo'})
|
||||
|
||||
if mock_bus.publish_event.called:
|
||||
kwargs = mock_bus.publish_event.call_args[1] if mock_bus.publish_event.call_args[1] else {}
|
||||
pos_args = mock_bus.publish_event.call_args[0]
|
||||
# payload is 3rd positional arg
|
||||
if len(pos_args) >= 3:
|
||||
payload = pos_args[2]
|
||||
self.assertIn('cell_name', payload)
|
||||
self.assertIn('effective_domain', payload)
|
||||
|
||||
|
||||
class TestCaddyCertStatusRoute(unittest.TestCase):
|
||||
"""GET /api/caddy/cert-status delegates to CaddyManager and handles errors."""
|
||||
|
||||
def setUp(self):
|
||||
app.config['TESTING'] = True
|
||||
self.client = app.test_client()
|
||||
|
||||
def test_returns_cert_status_200(self):
|
||||
expected = {
|
||||
'status': 'valid',
|
||||
'expiry': '2026-12-01T00:00:00+00:00',
|
||||
'days_remaining': 179,
|
||||
}
|
||||
mock_caddy = MagicMock()
|
||||
mock_caddy.get_cert_status_fresh.return_value = expected
|
||||
with patch('app.caddy_manager', mock_caddy):
|
||||
r = self.client.get('/api/caddy/cert-status')
|
||||
self.assertEqual(r.status_code, 200)
|
||||
data = json.loads(r.data)
|
||||
self.assertEqual(data['status'], 'valid')
|
||||
self.assertEqual(data['days_remaining'], 179)
|
||||
|
||||
def test_returns_500_on_exception(self):
|
||||
mock_caddy = MagicMock()
|
||||
mock_caddy.get_cert_status_fresh.side_effect = RuntimeError('ssl timeout')
|
||||
with patch('app.caddy_manager', mock_caddy):
|
||||
r = self.client.get('/api/caddy/cert-status')
|
||||
self.assertEqual(r.status_code, 500)
|
||||
data = json.loads(r.data)
|
||||
self.assertIn('error', data)
|
||||
|
||||
def test_calls_get_cert_status_fresh_with_max_age(self):
|
||||
mock_caddy = MagicMock()
|
||||
mock_caddy.get_cert_status_fresh.return_value = {'status': 'internal'}
|
||||
with patch('app.caddy_manager', mock_caddy):
|
||||
self.client.get('/api/caddy/cert-status')
|
||||
mock_caddy.get_cert_status_fresh.assert_called_once()
|
||||
call_kwargs = mock_caddy.get_cert_status_fresh.call_args
|
||||
# max_age_seconds should be passed (positional or keyword)
|
||||
all_args = list(call_kwargs[0]) + list(call_kwargs[1].values())
|
||||
self.assertTrue(
|
||||
any(isinstance(a, int) and a > 0 for a in all_args),
|
||||
'Expected a positive max_age_seconds argument',
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
||||
@@ -119,14 +119,17 @@ class TestRestoreConfigBackup(unittest.TestCase):
|
||||
content_type='application/json',
|
||||
)
|
||||
mock_cm.restore_config.assert_called_once_with(
|
||||
'backup_001', services=['network', 'wireguard']
|
||||
'backup_001', services=['network', 'wireguard'], service_registry=None
|
||||
)
|
||||
|
||||
@patch('app.config_manager')
|
||||
def test_restore_passes_none_services_when_no_body(self, mock_cm):
|
||||
from unittest.mock import ANY
|
||||
mock_cm.restore_config.return_value = True
|
||||
self.client.post('/api/config/restore/backup_001')
|
||||
mock_cm.restore_config.assert_called_once_with('backup_001', services=None)
|
||||
mock_cm.restore_config.assert_called_once_with(
|
||||
'backup_001', services=None, service_registry=ANY
|
||||
)
|
||||
|
||||
|
||||
class TestExportConfig(unittest.TestCase):
|
||||
|
||||
@@ -260,6 +260,125 @@ class TestConfigManager(unittest.TestCase):
|
||||
"import must not inject zero-filled entries for absent services")
|
||||
|
||||
|
||||
class TestSaveAllConfigs(unittest.TestCase):
|
||||
"""_save_all_configs must log errors instead of silently swallowing them."""
|
||||
|
||||
def setUp(self):
|
||||
self.temp_dir = tempfile.mkdtemp()
|
||||
self.config_file = os.path.join(self.temp_dir, 'cell_config.json')
|
||||
self.data_dir = os.path.join(self.temp_dir, 'data')
|
||||
os.makedirs(self.data_dir, exist_ok=True)
|
||||
self.cm = ConfigManager(self.config_file, self.data_dir)
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.temp_dir)
|
||||
|
||||
def test_save_failure_is_logged_not_silenced(self):
|
||||
"""When the config file cannot be written, _save_all_configs must log an error."""
|
||||
with patch('builtins.open', side_effect=OSError('disk full')):
|
||||
with self.assertLogs('config_manager', level='ERROR') as log:
|
||||
self.cm._save_all_configs()
|
||||
self.assertTrue(
|
||||
any('write failed' in msg or 'NOT persisted' in msg for msg in log.output),
|
||||
f'Expected error about write failure in logs, got: {log.output}',
|
||||
)
|
||||
|
||||
def test_save_success_does_not_log_error(self):
|
||||
"""A successful save must not produce error logs."""
|
||||
import logging
|
||||
with self.assertLogs('config_manager', level='DEBUG') as cm:
|
||||
logging.getLogger('config_manager').debug('sentinel')
|
||||
self.cm._save_all_configs()
|
||||
errors = [m for m in cm.output if 'ERROR' in m and 'write failed' in m]
|
||||
self.assertEqual(errors, [], 'Unexpected write-failure error on a successful save')
|
||||
|
||||
|
||||
class TestGetEffectiveDomain(unittest.TestCase):
|
||||
"""Tests for ConfigManager.get_effective_domain and get_internal_domain."""
|
||||
|
||||
def setUp(self):
|
||||
self.temp_dir = tempfile.mkdtemp()
|
||||
self.config_file = os.path.join(self.temp_dir, 'cell_config.json')
|
||||
self.data_dir = os.path.join(self.temp_dir, 'data')
|
||||
os.makedirs(self.data_dir, exist_ok=True)
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.temp_dir)
|
||||
|
||||
def _make_cm(self, identity):
|
||||
cm = ConfigManager(self.config_file, self.data_dir)
|
||||
cm.configs['_identity'] = identity
|
||||
return cm
|
||||
|
||||
def test_get_effective_domain_lan_mode(self):
|
||||
cm = self._make_cm({'domain': 'home.local', 'domain_mode': 'lan'})
|
||||
self.assertEqual(cm.get_effective_domain(), 'home.local')
|
||||
|
||||
def test_get_effective_domain_pic_ngo_uses_domain_name(self):
|
||||
cm = self._make_cm({
|
||||
'domain': 'home.local',
|
||||
'domain_mode': 'pic_ngo',
|
||||
'domain_name': 'home.pic.ngo',
|
||||
})
|
||||
self.assertEqual(cm.get_effective_domain(), 'home.pic.ngo')
|
||||
|
||||
def test_get_effective_domain_pic_ngo_fallback(self):
|
||||
cm = self._make_cm({'domain': 'home.local', 'domain_mode': 'pic_ngo'})
|
||||
self.assertEqual(cm.get_effective_domain(), 'home.local')
|
||||
|
||||
def test_get_internal_domain_always_returns_domain(self):
|
||||
cm = self._make_cm({
|
||||
'domain': 'home.local',
|
||||
'domain_mode': 'pic_ngo',
|
||||
'domain_name': 'home.pic.ngo',
|
||||
})
|
||||
self.assertEqual(cm.get_internal_domain(), 'home.local')
|
||||
|
||||
def test_get_internal_domain_ignores_domain_name(self):
|
||||
cm = self._make_cm({
|
||||
'domain': 'myzone.local',
|
||||
'domain_mode': 'cloudflare',
|
||||
'domain_name': 'example.com',
|
||||
})
|
||||
self.assertEqual(cm.get_internal_domain(), 'myzone.local')
|
||||
|
||||
def test_get_effective_domain_cloudflare_uses_domain_name(self):
|
||||
cm = self._make_cm({
|
||||
'domain': 'home.local',
|
||||
'domain_mode': 'cloudflare',
|
||||
'domain_name': 'example.com',
|
||||
})
|
||||
self.assertEqual(cm.get_effective_domain(), 'example.com')
|
||||
|
||||
def test_silent_migration_sets_unique_internal_domain(self):
|
||||
"""When DDNS is active and domain is the generic 'cell', migration sets cell_name.local."""
|
||||
config_file2 = os.path.join(self.temp_dir, 'cell_config2.json')
|
||||
with open(config_file2, 'w') as f:
|
||||
json.dump({
|
||||
'_identity': {
|
||||
'cell_name': 'alpha',
|
||||
'domain': 'cell',
|
||||
'domain_mode': 'pic_ngo',
|
||||
}
|
||||
}, f)
|
||||
cm = ConfigManager(config_file2, self.data_dir)
|
||||
self.assertEqual(cm.get_internal_domain(), 'alpha.local')
|
||||
|
||||
def test_silent_migration_does_not_touch_lan_mode(self):
|
||||
"""Migration must leave domain unchanged when domain_mode is 'lan'."""
|
||||
config_file2 = os.path.join(self.temp_dir, 'cell_config3.json')
|
||||
with open(config_file2, 'w') as f:
|
||||
json.dump({
|
||||
'_identity': {
|
||||
'cell_name': 'beta',
|
||||
'domain': 'cell',
|
||||
'domain_mode': 'lan',
|
||||
}
|
||||
}, f)
|
||||
cm = ConfigManager(config_file2, self.data_dir)
|
||||
self.assertEqual(cm.get_internal_domain(), 'cell')
|
||||
|
||||
|
||||
class TestNetworkManagerApply(unittest.TestCase):
|
||||
"""Test apply_config / apply_domain actually write real config files."""
|
||||
|
||||
|
||||
@@ -253,12 +253,12 @@ class TestUploadWireguardExt(unittest.TestCase):
|
||||
|
||||
def test_valid_conf_writes_file_to_correct_path(self):
|
||||
self.mgr.upload_wireguard_ext(self._valid_conf())
|
||||
expected = os.path.join(self.tmp, 'connectivity', 'wireguard_ext', 'wg_ext0.conf')
|
||||
expected = os.path.join(self.tmp, 'services', 'wireguard-ext', 'config', 'wg_ext0.conf')
|
||||
self.assertTrue(os.path.isfile(expected), f'Expected file at {expected}')
|
||||
|
||||
def test_valid_conf_file_has_mode_0600(self):
|
||||
self.mgr.upload_wireguard_ext(self._valid_conf())
|
||||
path = os.path.join(self.tmp, 'connectivity', 'wireguard_ext', 'wg_ext0.conf')
|
||||
path = os.path.join(self.tmp, 'services', 'wireguard-ext', 'config', 'wg_ext0.conf')
|
||||
mode = stat.S_IMODE(os.stat(path).st_mode)
|
||||
self.assertEqual(mode, 0o600, f'Expected 0600, got {oct(mode)}')
|
||||
|
||||
@@ -272,7 +272,7 @@ class TestUploadWireguardExt(unittest.TestCase):
|
||||
def test_file_content_has_hooks_stripped(self):
|
||||
conf = "[Interface]\nPrivateKey = abc\nPostUp = evil\n"
|
||||
self.mgr.upload_wireguard_ext(conf)
|
||||
path = os.path.join(self.tmp, 'connectivity', 'wireguard_ext', 'wg_ext0.conf')
|
||||
path = os.path.join(self.tmp, 'services', 'wireguard-ext', 'config', 'wg_ext0.conf')
|
||||
with open(path) as f:
|
||||
content = f.read()
|
||||
self.assertNotIn('PostUp', content)
|
||||
@@ -301,12 +301,12 @@ class TestUploadOpenvpn(unittest.TestCase):
|
||||
|
||||
def test_valid_conf_writes_file_at_correct_path(self):
|
||||
self.mgr.upload_openvpn(self._valid_ovpn(), name='my-vpn')
|
||||
expected = os.path.join(self.tmp, 'connectivity', 'openvpn', 'my-vpn.ovpn')
|
||||
expected = os.path.join(self.tmp, 'services', 'openvpn-client', 'config', 'my-vpn.ovpn')
|
||||
self.assertTrue(os.path.isfile(expected), f'Expected file at {expected}')
|
||||
|
||||
def test_valid_conf_file_has_mode_0600(self):
|
||||
self.mgr.upload_openvpn(self._valid_ovpn(), name='my-vpn')
|
||||
path = os.path.join(self.tmp, 'connectivity', 'openvpn', 'my-vpn.ovpn')
|
||||
path = os.path.join(self.tmp, 'services', 'openvpn-client', 'config', 'my-vpn.ovpn')
|
||||
mode = stat.S_IMODE(os.stat(path).st_mode)
|
||||
self.assertEqual(mode, 0o600, f'Expected 0600, got {oct(mode)}')
|
||||
|
||||
@@ -339,19 +339,77 @@ class TestUploadOpenvpn(unittest.TestCase):
|
||||
def test_default_name_default_passes(self):
|
||||
result = self.mgr.upload_openvpn(self._valid_ovpn())
|
||||
self.assertTrue(result['ok'])
|
||||
expected = os.path.join(self.tmp, 'connectivity', 'openvpn', 'default.ovpn')
|
||||
expected = os.path.join(self.tmp, 'services', 'openvpn-client', 'config', 'default.ovpn')
|
||||
self.assertTrue(os.path.isfile(expected))
|
||||
|
||||
def test_hooks_stripped_from_stored_file(self):
|
||||
conf = "client\ndev tun\nup /sbin/bad.sh\nproto udp\n"
|
||||
self.mgr.upload_openvpn(conf, name='clean')
|
||||
path = os.path.join(self.tmp, 'connectivity', 'openvpn', 'clean.ovpn')
|
||||
path = os.path.join(self.tmp, 'services', 'openvpn-client', 'config', 'clean.ovpn')
|
||||
with open(path) as f:
|
||||
content = f.read()
|
||||
self.assertNotIn('up /sbin/bad.sh', content)
|
||||
self.assertIn('proto udp', content)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _migrate_legacy_configs
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestMigrateLegacyConfigs(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.mkdtemp()
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.tmp, ignore_errors=True)
|
||||
|
||||
def test_no_op_when_legacy_dir_absent(self):
|
||||
"""No errors when legacy connectivity/ dir does not exist."""
|
||||
mgr = _make_manager(tmp_dir=self.tmp)
|
||||
# Should not raise; legacy dir simply doesn't exist
|
||||
mgr._migrate_legacy_configs(os.path.join(self.tmp, 'nonexistent'))
|
||||
|
||||
def test_wg_conf_copied_to_new_location(self):
|
||||
legacy_wg = os.path.join(self.tmp, 'connectivity', 'wireguard_ext')
|
||||
os.makedirs(legacy_wg)
|
||||
src = os.path.join(legacy_wg, 'wg_ext0.conf')
|
||||
with open(src, 'w') as f:
|
||||
f.write('[Interface]\nPrivateKey = abc\n')
|
||||
|
||||
mgr = _make_manager(tmp_dir=self.tmp)
|
||||
dst = os.path.join(self.tmp, 'services', 'wireguard-ext', 'config', 'wg_ext0.conf')
|
||||
self.assertTrue(os.path.isfile(dst), f'Expected migrated file at {dst}')
|
||||
|
||||
def test_ovpn_copied_to_new_location(self):
|
||||
legacy_ovpn = os.path.join(self.tmp, 'connectivity', 'openvpn')
|
||||
os.makedirs(legacy_ovpn)
|
||||
src = os.path.join(legacy_ovpn, 'default.ovpn')
|
||||
with open(src, 'w') as f:
|
||||
f.write('client\ndev tun\n')
|
||||
|
||||
mgr = _make_manager(tmp_dir=self.tmp)
|
||||
dst = os.path.join(self.tmp, 'services', 'openvpn-client', 'config', 'default.ovpn')
|
||||
self.assertTrue(os.path.isfile(dst), f'Expected migrated file at {dst}')
|
||||
|
||||
def test_existing_dst_not_overwritten(self):
|
||||
legacy_wg = os.path.join(self.tmp, 'connectivity', 'wireguard_ext')
|
||||
os.makedirs(legacy_wg)
|
||||
with open(os.path.join(legacy_wg, 'wg_ext0.conf'), 'w') as f:
|
||||
f.write('legacy\n')
|
||||
|
||||
# Pre-create the destination with different content
|
||||
dst_dir = os.path.join(self.tmp, 'services', 'wireguard-ext', 'config')
|
||||
os.makedirs(dst_dir, exist_ok=True)
|
||||
dst = os.path.join(dst_dir, 'wg_ext0.conf')
|
||||
with open(dst, 'w') as f:
|
||||
f.write('existing\n')
|
||||
|
||||
_make_manager(tmp_dir=self.tmp)
|
||||
with open(dst) as f:
|
||||
self.assertEqual(f.read(), 'existing\n')
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# get_status
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -0,0 +1,173 @@
|
||||
"""Tests for GET /api/ddns/check/<name> and PUT /api/ddns."""
|
||||
import json
|
||||
import sys
|
||||
import os
|
||||
import unittest
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'api'))
|
||||
|
||||
|
||||
def _make_client():
|
||||
from app import app
|
||||
app.config['TESTING'] = True
|
||||
return app.test_client()
|
||||
|
||||
|
||||
class TestDdnsCheckName(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.client = _make_client()
|
||||
|
||||
def _get(self, name):
|
||||
return self.client.get(f'/api/ddns/check/{name}')
|
||||
|
||||
@patch('routes.config.DDNS_API_BASE', 'http://ddns.test', create=True)
|
||||
def test_available_name_returns_true(self):
|
||||
with patch('routes.config._ureq', create=True):
|
||||
import io
|
||||
resp_mock = MagicMock()
|
||||
resp_mock.read.return_value = b'{"available": true}'
|
||||
resp_mock.__enter__ = lambda s: resp_mock
|
||||
resp_mock.__exit__ = MagicMock(return_value=False)
|
||||
with patch('urllib.request.urlopen', return_value=resp_mock):
|
||||
r = self._get('testname')
|
||||
self.assertEqual(r.status_code, 200)
|
||||
body = json.loads(r.data)
|
||||
self.assertTrue(body['available'])
|
||||
|
||||
@patch('routes.config.DDNS_API_BASE', 'http://ddns.test', create=True)
|
||||
def test_taken_name_returns_false(self):
|
||||
resp_mock = MagicMock()
|
||||
resp_mock.read.return_value = b'{"available": false}'
|
||||
resp_mock.__enter__ = lambda s: resp_mock
|
||||
resp_mock.__exit__ = MagicMock(return_value=False)
|
||||
with patch('urllib.request.urlopen', return_value=resp_mock):
|
||||
r = self._get('taken')
|
||||
self.assertEqual(r.status_code, 200)
|
||||
body = json.loads(r.data)
|
||||
self.assertFalse(body['available'])
|
||||
|
||||
def test_unreachable_returns_503(self):
|
||||
import urllib.error
|
||||
with patch('urllib.request.urlopen', side_effect=OSError('conn refused')):
|
||||
r = self._get('anything')
|
||||
self.assertEqual(r.status_code, 503)
|
||||
body = json.loads(r.data)
|
||||
self.assertIsNone(body['available'])
|
||||
|
||||
|
||||
class TestUpdateDdnsConfig(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.client = _make_client()
|
||||
|
||||
def _put(self, payload):
|
||||
return self.client.put(
|
||||
'/api/ddns',
|
||||
data=json.dumps(payload),
|
||||
content_type='application/json',
|
||||
)
|
||||
|
||||
def test_invalid_domain_mode_returns_400(self):
|
||||
r = self._put({'domain_mode': 'invalid_mode'})
|
||||
self.assertEqual(r.status_code, 400)
|
||||
self.assertIn('domain_mode', json.loads(r.data)['error'])
|
||||
|
||||
def test_cloudflare_requires_domain_name(self):
|
||||
r = self._put({'domain_mode': 'cloudflare', 'cloudflare_api_token': 'tok'})
|
||||
self.assertEqual(r.status_code, 400)
|
||||
self.assertIn('domain_name', json.loads(r.data)['error'])
|
||||
|
||||
def test_cloudflare_invalid_token_returns_422(self):
|
||||
import urllib.error
|
||||
with patch('urllib.request.urlopen', side_effect=urllib.error.HTTPError(
|
||||
None, 403, 'Forbidden', {}, None
|
||||
)):
|
||||
r = self._put({'domain_mode': 'cloudflare', 'domain_name': 'home.example.com',
|
||||
'cloudflare_api_token': 'bad-token'})
|
||||
self.assertEqual(r.status_code, 422)
|
||||
|
||||
def test_cloudflare_valid_token_saves_config(self):
|
||||
from app import config_manager
|
||||
resp_mock = MagicMock()
|
||||
resp_mock.read.return_value = b'{"success": true}'
|
||||
resp_mock.__enter__ = lambda s: resp_mock
|
||||
resp_mock.__exit__ = MagicMock(return_value=False)
|
||||
with patch('urllib.request.urlopen', return_value=resp_mock):
|
||||
with patch.object(config_manager, 'set_ddns_config') as mock_set_ddns, \
|
||||
patch.object(config_manager, 'set_identity_field') as mock_set_id:
|
||||
r = self._put({'domain_mode': 'cloudflare', 'domain_name': 'home.example.com',
|
||||
'cloudflare_api_token': 'valid-token'})
|
||||
self.assertEqual(r.status_code, 200)
|
||||
self.assertTrue(json.loads(r.data)['updated'])
|
||||
mock_set_ddns.assert_called_once()
|
||||
mock_set_id.assert_any_call('domain_mode', 'cloudflare')
|
||||
|
||||
def test_duckdns_requires_domain_name(self):
|
||||
r = self._put({'domain_mode': 'duckdns', 'duckdns_token': 'tok'})
|
||||
self.assertEqual(r.status_code, 400)
|
||||
|
||||
def test_duckdns_invalid_token_returns_422(self):
|
||||
resp_mock = MagicMock()
|
||||
resp_mock.read.return_value = b'KO'
|
||||
resp_mock.__enter__ = lambda s: resp_mock
|
||||
resp_mock.__exit__ = MagicMock(return_value=False)
|
||||
with patch('urllib.request.urlopen', return_value=resp_mock):
|
||||
r = self._put({'domain_mode': 'duckdns', 'domain_name': 'myname.duckdns.org',
|
||||
'duckdns_token': 'bad'})
|
||||
self.assertEqual(r.status_code, 422)
|
||||
|
||||
def test_lan_mode_saves_without_validation(self):
|
||||
from app import config_manager
|
||||
with patch.object(config_manager, 'set_ddns_config') as mock_ddns, \
|
||||
patch.object(config_manager, 'set_identity_field') as mock_id:
|
||||
r = self._put({'domain_mode': 'lan'})
|
||||
self.assertEqual(r.status_code, 200)
|
||||
mock_ddns.assert_called_once()
|
||||
mock_id.assert_any_call('domain_mode', 'lan')
|
||||
|
||||
def test_http01_mode_saves_with_domain(self):
|
||||
from app import config_manager
|
||||
with patch.object(config_manager, 'set_ddns_config') as mock_ddns, \
|
||||
patch.object(config_manager, 'set_identity_field') as mock_id:
|
||||
r = self._put({'domain_mode': 'http01', 'domain_name': 'home.example.com'})
|
||||
self.assertEqual(r.status_code, 200)
|
||||
mock_id.assert_any_call('domain_name', 'home.example.com')
|
||||
|
||||
|
||||
class TestDdnsRegister(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.client = _make_client()
|
||||
|
||||
def test_non_pic_ngo_provider_returns_400(self):
|
||||
from app import config_manager
|
||||
with patch.object(config_manager, 'configs', {'ddns': {'provider': 'cloudflare'}, '_identity': {}}):
|
||||
r = self.client.post('/api/ddns/register')
|
||||
self.assertEqual(r.status_code, 400)
|
||||
|
||||
def test_missing_cell_name_returns_400(self):
|
||||
from app import config_manager
|
||||
with patch.object(config_manager, 'configs', {'ddns': {'provider': 'pic_ngo'}, '_identity': {}}):
|
||||
r = self.client.post('/api/ddns/register')
|
||||
self.assertEqual(r.status_code, 400)
|
||||
self.assertIn('cell_name', json.loads(r.data)['error'])
|
||||
|
||||
def test_register_success(self):
|
||||
from app import config_manager
|
||||
from ddns_manager import DDNSManager
|
||||
with patch.object(config_manager, 'configs', {
|
||||
'ddns': {'provider': 'pic_ngo'},
|
||||
'_identity': {'cell_name': 'mypic'}
|
||||
}):
|
||||
with patch.object(DDNSManager, 'register', return_value={'subdomain': 'mypic.pic.ngo', 'token': 'tok'}) as mock_reg, \
|
||||
patch.object(config_manager, 'set_identity_field') as mock_id:
|
||||
r = self.client.post('/api/ddns/register')
|
||||
self.assertEqual(r.status_code, 200)
|
||||
body = json.loads(r.data)
|
||||
self.assertTrue(body['registered'])
|
||||
self.assertEqual(body['subdomain'], 'mypic.pic.ngo')
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
+162
-25
@@ -13,6 +13,7 @@ from ddns_manager import (
|
||||
DDNSManager,
|
||||
DDNSProvider,
|
||||
DDNSError,
|
||||
DDNSTokenExpired,
|
||||
PicNgoDDNS,
|
||||
CloudflareDDNS,
|
||||
DuckDNSDDNS,
|
||||
@@ -37,15 +38,14 @@ def _make_response(status_code=200, json_data=None, text=''):
|
||||
|
||||
|
||||
def _make_config_manager(ddns_cfg=None, domain_cfg=None):
|
||||
"""Return a mock config_manager whose get_identity() returns a useful dict."""
|
||||
"""Return a mock config_manager with a real configs dict."""
|
||||
cm = MagicMock()
|
||||
configs = {}
|
||||
if ddns_cfg is not None:
|
||||
identity = {'domain': {'ddns': ddns_cfg}}
|
||||
elif domain_cfg is not None:
|
||||
identity = {'domain': domain_cfg}
|
||||
else:
|
||||
identity = {}
|
||||
cm.get_identity.return_value = identity
|
||||
configs['ddns'] = {k: v for k, v in ddns_cfg.items() if k != 'token'}
|
||||
cm.configs = configs
|
||||
# Token is stored outside cell_config.json via get/set_ddns_token
|
||||
cm.get_ddns_token.return_value = (ddns_cfg or {}).get('token', '')
|
||||
return cm
|
||||
|
||||
|
||||
@@ -83,11 +83,32 @@ class TestPicNgoDDNSRegister(unittest.TestCase):
|
||||
_, kwargs = mock_post.call_args
|
||||
self.assertNotIn('Authorization', kwargs.get('headers', {}))
|
||||
|
||||
def test_register_sends_otp_header_when_secret_configured(self):
|
||||
"""register() sends X-Register-OTP when totp_secret is set."""
|
||||
provider = PicNgoDDNS(totp_secret='JBSWY3DPEHPK3PXP')
|
||||
mock_resp = _make_response(200, json_data={'token': 'tok', 'subdomain': 'x.pic.ngo'})
|
||||
with patch('requests.post', return_value=mock_resp) as mock_post:
|
||||
provider.register('x', '1.2.3.4')
|
||||
_, kwargs = mock_post.call_args
|
||||
self.assertIn('X-Register-OTP', kwargs.get('headers', {}))
|
||||
otp = kwargs['headers']['X-Register-OTP']
|
||||
self.assertEqual(len(otp), 6)
|
||||
self.assertTrue(otp.isdigit())
|
||||
|
||||
def test_register_no_otp_header_without_secret(self):
|
||||
"""register() omits X-Register-OTP when no TOTP secret is configured."""
|
||||
provider = PicNgoDDNS()
|
||||
mock_resp = _make_response(200, json_data={'token': 't', 'subdomain': 'x'})
|
||||
with patch('requests.post', return_value=mock_resp) as mock_post:
|
||||
provider.register('x', '1.2.3.4')
|
||||
_, kwargs = mock_post.call_args
|
||||
self.assertNotIn('X-Register-OTP', kwargs.get('headers', {}))
|
||||
|
||||
|
||||
class TestPicNgoDDNSUpdate(unittest.TestCase):
|
||||
"""PicNgoDDNS.update() calls the correct URL with Authorization header."""
|
||||
"""PicNgoDDNS.update() sends token in the request body (DDNS server validates it there)."""
|
||||
|
||||
def test_update_uses_bearer_token(self):
|
||||
def test_update_sends_token_in_body(self):
|
||||
provider = PicNgoDDNS(api_base_url='https://ddns.example.com')
|
||||
mock_resp = _make_response(200)
|
||||
with patch('requests.put', return_value=mock_resp) as mock_put:
|
||||
@@ -95,11 +116,19 @@ class TestPicNgoDDNSUpdate(unittest.TestCase):
|
||||
mock_put.assert_called_once()
|
||||
args, kwargs = mock_put.call_args
|
||||
self.assertEqual(args[0], 'https://ddns.example.com/api/v1/update')
|
||||
self.assertIn('Authorization', kwargs['headers'])
|
||||
self.assertEqual(kwargs['headers']['Authorization'], 'Bearer mytoken')
|
||||
self.assertEqual(kwargs['json'], {'ip': '5.6.7.8'})
|
||||
# Token must be in the JSON body — server validates it there, not in Authorization
|
||||
self.assertEqual(kwargs['json'], {'ip': '5.6.7.8', 'token': 'mytoken'})
|
||||
self.assertTrue(result)
|
||||
|
||||
def test_update_does_not_use_bearer_header(self):
|
||||
"""Token must NOT be sent as Authorization: Bearer — server ignores it and returns 422."""
|
||||
provider = PicNgoDDNS(api_base_url='https://ddns.example.com')
|
||||
mock_resp = _make_response(200)
|
||||
with patch('requests.put', return_value=mock_resp) as mock_put:
|
||||
provider.update('mytoken', '1.2.3.4')
|
||||
_, kwargs = mock_put.call_args
|
||||
self.assertNotIn('Authorization', kwargs.get('headers', {}))
|
||||
|
||||
def test_update_raises_ddns_error_on_failure(self):
|
||||
provider = PicNgoDDNS()
|
||||
mock_resp = _make_response(403, text='Forbidden')
|
||||
@@ -107,6 +136,13 @@ class TestPicNgoDDNSUpdate(unittest.TestCase):
|
||||
with self.assertRaises(DDNSError):
|
||||
provider.update('badtoken', '1.2.3.4')
|
||||
|
||||
def test_update_raises_ddns_token_expired_on_401(self):
|
||||
provider = PicNgoDDNS()
|
||||
mock_resp = _make_response(401, text='Unauthorized')
|
||||
with patch('requests.put', return_value=mock_resp):
|
||||
with self.assertRaises(DDNSTokenExpired):
|
||||
provider.update('expiredtoken', '1.2.3.4')
|
||||
|
||||
|
||||
class TestPicNgoDDNSChallenges(unittest.TestCase):
|
||||
"""PicNgoDDNS.dns_challenge_create/delete call correct endpoints."""
|
||||
@@ -238,17 +274,17 @@ class TestUpdateIp(unittest.TestCase):
|
||||
mock_provider = MagicMock()
|
||||
mock_provider.update.return_value = True
|
||||
mgr.get_provider = MagicMock(return_value=mock_provider)
|
||||
return mgr, mock_provider
|
||||
return mgr, mock_provider, cm
|
||||
|
||||
def test_update_when_ip_changed(self):
|
||||
mgr, mock_provider = self._make_manager_with_mock_provider(last_ip='1.1.1.1')
|
||||
mgr, mock_provider, _ = self._make_manager_with_mock_provider(last_ip='1.1.1.1')
|
||||
with patch('ddns_manager._get_public_ip', return_value='2.2.2.2'):
|
||||
mgr.update_ip()
|
||||
mock_provider.update.assert_called_once_with('tok', '2.2.2.2')
|
||||
self.assertEqual(mgr._last_ip, '2.2.2.2')
|
||||
|
||||
def test_skips_update_when_ip_unchanged(self):
|
||||
mgr, mock_provider = self._make_manager_with_mock_provider(last_ip='3.3.3.3')
|
||||
mgr, mock_provider, _ = self._make_manager_with_mock_provider(last_ip='3.3.3.3')
|
||||
with patch('ddns_manager._get_public_ip', return_value='3.3.3.3'):
|
||||
mgr.update_ip()
|
||||
mock_provider.update.assert_not_called()
|
||||
@@ -263,13 +299,13 @@ class TestUpdateIp(unittest.TestCase):
|
||||
mgr.update_ip()
|
||||
|
||||
def test_skips_update_when_ip_unreachable(self):
|
||||
mgr, mock_provider = self._make_manager_with_mock_provider(last_ip=None)
|
||||
mgr, mock_provider, _ = self._make_manager_with_mock_provider(last_ip=None)
|
||||
with patch('ddns_manager._get_public_ip', return_value=None):
|
||||
mgr.update_ip()
|
||||
mock_provider.update.assert_not_called()
|
||||
|
||||
def test_last_ip_not_updated_when_provider_returns_false(self):
|
||||
mgr, mock_provider = self._make_manager_with_mock_provider(last_ip='1.1.1.1')
|
||||
mgr, mock_provider, _ = self._make_manager_with_mock_provider(last_ip='1.1.1.1')
|
||||
mock_provider.update.return_value = False
|
||||
with patch('ddns_manager._get_public_ip', return_value='9.9.9.9'):
|
||||
mgr.update_ip()
|
||||
@@ -277,34 +313,135 @@ class TestUpdateIp(unittest.TestCase):
|
||||
self.assertEqual(mgr._last_ip, '1.1.1.1')
|
||||
|
||||
def test_ddns_error_is_caught_not_propagated(self):
|
||||
mgr, mock_provider = self._make_manager_with_mock_provider(last_ip='1.1.1.1')
|
||||
mgr, mock_provider, _ = self._make_manager_with_mock_provider(last_ip='1.1.1.1')
|
||||
mock_provider.update.side_effect = DDNSError("server error")
|
||||
with patch('ddns_manager._get_public_ip', return_value='5.5.5.5'):
|
||||
# Should not raise
|
||||
mgr.update_ip()
|
||||
|
||||
def test_no_token_triggers_registration_and_fires_identity_changed(self):
|
||||
"""When no token exists, update_ip() registers immediately and fires IDENTITY_CHANGED."""
|
||||
cm = _make_config_manager(ddns_cfg={'provider': 'pic_ngo'})
|
||||
cm.get_ddns_token.return_value = ''
|
||||
cm.get_identity.return_value = {'cell_name': 'mytest'}
|
||||
mock_sbus = MagicMock()
|
||||
mgr = DDNSManager(config_manager=cm, service_bus=mock_sbus)
|
||||
mgr._last_ip = None
|
||||
|
||||
mock_provider = MagicMock()
|
||||
mock_provider.register.return_value = {'token': 'new_tok', 'subdomain': 'mytest.pic.ngo'}
|
||||
mgr.get_provider = MagicMock(return_value=mock_provider)
|
||||
|
||||
with patch('ddns_manager._get_public_ip', return_value='1.2.3.4'):
|
||||
mgr.update_ip()
|
||||
|
||||
mock_provider.register.assert_called_once_with('mytest', '1.2.3.4')
|
||||
mock_provider.update.assert_not_called()
|
||||
self.assertEqual(mgr._last_ip, '1.2.3.4')
|
||||
mock_sbus.publish_event.assert_called_once()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# DDNSManager.register() tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestRegister(unittest.TestCase):
|
||||
def test_register_stores_token_in_config(self):
|
||||
def test_register_stores_token_in_ddns_config(self):
|
||||
cm = _make_config_manager(ddns_cfg={'provider': 'pic_ngo'})
|
||||
mgr = DDNSManager(config_manager=cm)
|
||||
|
||||
mock_provider = MagicMock()
|
||||
mock_provider.register.return_value = {'token': 'new_tok', 'subdomain': 'alpha'}
|
||||
mock_provider.register.return_value = {'token': 'new_tok', 'subdomain': 'alpha.pic.ngo'}
|
||||
mgr.get_provider = MagicMock(return_value=mock_provider)
|
||||
|
||||
result = mgr.register('alpha', '1.2.3.4')
|
||||
self.assertEqual(result['token'], 'new_tok')
|
||||
|
||||
# set_identity_field('domain', ...) should have been called
|
||||
cm.set_identity_field.assert_called_once()
|
||||
field_name, field_value = cm.set_identity_field.call_args[0]
|
||||
self.assertEqual(field_name, 'domain')
|
||||
self.assertEqual(field_value['ddns']['token'], 'new_tok')
|
||||
# Token stored via set_ddns_token (not embedded in cell_config.json)
|
||||
cm.set_ddns_token.assert_called_once_with('new_tok')
|
||||
|
||||
# Subdomain saved to _identity.domain_name
|
||||
cm.set_identity_field.assert_called_once_with('domain_name', 'alpha.pic.ngo')
|
||||
|
||||
def test_register_fetches_public_ip_when_empty(self):
|
||||
cm = _make_config_manager(ddns_cfg={'provider': 'pic_ngo'})
|
||||
mgr = DDNSManager(config_manager=cm)
|
||||
|
||||
mock_provider = MagicMock()
|
||||
mock_provider.register.return_value = {'token': 't', 'subdomain': 'alpha.pic.ngo'}
|
||||
mgr.get_provider = MagicMock(return_value=mock_provider)
|
||||
|
||||
with patch('ddns_manager._get_public_ip', return_value='5.6.7.8') as mock_ip:
|
||||
mgr.register('alpha', '')
|
||||
mock_ip.assert_called_once()
|
||||
mock_provider.register.assert_called_once_with('alpha', '5.6.7.8')
|
||||
|
||||
def test_register_uses_provided_ip_without_fetching(self):
|
||||
cm = _make_config_manager(ddns_cfg={'provider': 'pic_ngo'})
|
||||
mgr = DDNSManager(config_manager=cm)
|
||||
|
||||
mock_provider = MagicMock()
|
||||
mock_provider.register.return_value = {'token': 't', 'subdomain': 'alpha.pic.ngo'}
|
||||
mgr.get_provider = MagicMock(return_value=mock_provider)
|
||||
|
||||
with patch('ddns_manager._get_public_ip') as mock_ip:
|
||||
mgr.register('alpha', '1.2.3.4')
|
||||
mock_ip.assert_not_called()
|
||||
mock_provider.register.assert_called_once_with('alpha', '1.2.3.4')
|
||||
|
||||
def test_register_releases_old_name_when_changing(self):
|
||||
cm = _make_config_manager(ddns_cfg={'provider': 'pic_ngo', 'token': 'old_tok'})
|
||||
cm.get_identity.return_value = {'domain_name': 'oldname.pic.ngo'}
|
||||
mgr = DDNSManager(config_manager=cm)
|
||||
|
||||
mock_provider = MagicMock()
|
||||
mock_provider.register.return_value = {'token': 'new_tok', 'subdomain': 'newname.pic.ngo'}
|
||||
mgr.get_provider = MagicMock(return_value=mock_provider)
|
||||
|
||||
mgr.register('newname', '1.2.3.4')
|
||||
|
||||
mock_provider.release.assert_called_once_with('old_tok')
|
||||
mock_provider.register.assert_called_once_with('newname', '1.2.3.4')
|
||||
|
||||
def test_register_skips_release_when_name_unchanged(self):
|
||||
cm = _make_config_manager(ddns_cfg={'provider': 'pic_ngo', 'token': 'tok'})
|
||||
cm.get_identity.return_value = {'domain_name': 'alpha.pic.ngo'}
|
||||
mgr = DDNSManager(config_manager=cm)
|
||||
|
||||
mock_provider = MagicMock()
|
||||
mock_provider.register.return_value = {'token': 'tok2', 'subdomain': 'alpha.pic.ngo'}
|
||||
mgr.get_provider = MagicMock(return_value=mock_provider)
|
||||
|
||||
mgr.register('alpha', '1.2.3.4')
|
||||
|
||||
mock_provider.release.assert_not_called()
|
||||
|
||||
def test_register_skips_release_when_no_old_token(self):
|
||||
cm = _make_config_manager(ddns_cfg={'provider': 'pic_ngo'})
|
||||
cm.get_identity.return_value = {'domain_name': 'oldname.pic.ngo'}
|
||||
mgr = DDNSManager(config_manager=cm)
|
||||
|
||||
mock_provider = MagicMock()
|
||||
mock_provider.register.return_value = {'token': 'new_tok', 'subdomain': 'newname.pic.ngo'}
|
||||
mgr.get_provider = MagicMock(return_value=mock_provider)
|
||||
|
||||
mgr.register('newname', '1.2.3.4')
|
||||
|
||||
mock_provider.release.assert_not_called()
|
||||
|
||||
def test_register_continues_if_release_fails(self):
|
||||
cm = _make_config_manager(ddns_cfg={'provider': 'pic_ngo', 'token': 'old_tok'})
|
||||
cm.get_identity.return_value = {'domain_name': 'oldname.pic.ngo'}
|
||||
mgr = DDNSManager(config_manager=cm)
|
||||
|
||||
mock_provider = MagicMock()
|
||||
mock_provider.release.side_effect = DDNSError("server down")
|
||||
mock_provider.register.return_value = {'token': 'new_tok', 'subdomain': 'newname.pic.ngo'}
|
||||
mgr.get_provider = MagicMock(return_value=mock_provider)
|
||||
|
||||
result = mgr.register('newname', '1.2.3.4')
|
||||
self.assertEqual(result['token'], 'new_tok')
|
||||
mock_provider.register.assert_called_once()
|
||||
|
||||
def test_register_raises_when_no_provider(self):
|
||||
cm = _make_config_manager()
|
||||
|
||||
@@ -0,0 +1,586 @@
|
||||
"""
|
||||
Tests for EgressManager — per-service egress enforcement via host iptables.
|
||||
|
||||
All subprocess calls (iptables, iptables-save, iptables-restore, ip rule,
|
||||
docker inspect) and config_manager state are mocked so these tests run
|
||||
without any live infrastructure or root privileges.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch, call
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'api'))
|
||||
|
||||
import egress_manager as em_module
|
||||
from egress_manager import EgressManager, MARKS, TABLES, EXIT_TYPES, EGRESS_CHAIN
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _make_manager(installed=None, overrides=None):
|
||||
"""Build an EgressManager backed by a mock config_manager."""
|
||||
cm = MagicMock()
|
||||
cm.get_installed_services.return_value = installed or {}
|
||||
# Wire up configs dict so _get_egress_overrides / _set_egress_overrides work
|
||||
cm.configs = {'egress_overrides': overrides or {}}
|
||||
cm._save_all_configs = MagicMock()
|
||||
return EgressManager(config_manager=cm), cm
|
||||
|
||||
|
||||
def _subprocess_ok(stdout=''):
|
||||
"""Return a MagicMock simulating a successful subprocess.run result."""
|
||||
return MagicMock(returncode=0, stdout=stdout, stderr='')
|
||||
|
||||
|
||||
def _subprocess_fail(stderr='error', stdout=''):
|
||||
"""Return a MagicMock simulating a failed subprocess.run result."""
|
||||
return MagicMock(returncode=1, stdout=stdout, stderr=stderr)
|
||||
|
||||
|
||||
def _make_manifest(has_egress=True, egress_default='wireguard_ext',
|
||||
allowed=None, container_name='cell-myapp'):
|
||||
"""Return a minimal manifest dict with optional egress configuration."""
|
||||
m = {
|
||||
'id': 'myapp',
|
||||
'name': 'My App',
|
||||
'container_name': container_name,
|
||||
}
|
||||
if has_egress:
|
||||
m['has_egress'] = True
|
||||
m['egress'] = {
|
||||
'default': egress_default,
|
||||
'allowed': allowed if allowed is not None else list(EXIT_TYPES),
|
||||
}
|
||||
else:
|
||||
m['has_egress'] = False
|
||||
return m
|
||||
|
||||
|
||||
def _installed_with_manifest(manifest, service_id='myapp'):
|
||||
"""Return an installed-services dict containing one service record."""
|
||||
return {service_id: {'id': service_id, 'manifest': manifest}}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 1. test_apply_service_default_exit_no_iptables_calls
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestApplyServiceDefaultExit(unittest.TestCase):
|
||||
|
||||
def test_apply_service_default_exit_no_iptables_calls(self):
|
||||
"""When egress.default is 'default', apply_service must not touch iptables."""
|
||||
manifest = _make_manifest(egress_default='default')
|
||||
mgr, _ = _make_manager(installed=_installed_with_manifest(manifest))
|
||||
|
||||
with patch('subprocess.run') as mock_run:
|
||||
# docker inspect must return an IP so we don't fail earlier
|
||||
mock_run.return_value = _subprocess_ok(stdout='172.20.0.50\n')
|
||||
result = mgr.apply_service('myapp')
|
||||
|
||||
self.assertTrue(result['ok'])
|
||||
self.assertEqual(result.get('exit_via'), 'default')
|
||||
|
||||
# No iptables rule-insertion or mark call should have been made.
|
||||
# iptables-save from clear_service is allowed; we only check that
|
||||
# no iptables -A / -I (rule-adding) calls were made.
|
||||
rule_add_calls = [
|
||||
c for c in mock_run.call_args_list
|
||||
if c.args and c.args[0][:1] == ['iptables']
|
||||
and any(a in c.args[0] for a in ('-A', '-I', 'MARK', 'REDIRECT'))
|
||||
]
|
||||
self.assertEqual(rule_add_calls, [])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 2. test_apply_service_wireguard_ext_adds_mark_rule
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestApplyServiceWireguardExt(unittest.TestCase):
|
||||
|
||||
def test_apply_service_wireguard_ext_adds_mark_rule(self):
|
||||
"""wireguard_ext exit must add a mangle MARK rule with 0x110 and the correct comment."""
|
||||
manifest = _make_manifest(egress_default='wireguard_ext')
|
||||
mgr, _ = _make_manager(installed=_installed_with_manifest(manifest))
|
||||
|
||||
calls_made = []
|
||||
|
||||
def fake_run(cmd, **kwargs):
|
||||
calls_made.append(cmd)
|
||||
# docker inspect → return IP
|
||||
if 'docker' in cmd and 'inspect' in cmd:
|
||||
return _subprocess_ok(stdout='172.20.0.50\n')
|
||||
# iptables-save → empty ruleset
|
||||
if 'iptables-save' in cmd:
|
||||
return _subprocess_ok(stdout='')
|
||||
# iptables-restore → success
|
||||
if 'iptables-restore' in cmd:
|
||||
return _subprocess_ok()
|
||||
# ip rule del → fail (none to delete)
|
||||
if cmd[:3] == ['ip', 'rule', 'del']:
|
||||
return _subprocess_fail()
|
||||
return _subprocess_ok()
|
||||
|
||||
with patch('subprocess.run', side_effect=fake_run):
|
||||
result = mgr.apply_service('myapp')
|
||||
|
||||
self.assertTrue(result['ok'], result)
|
||||
self.assertEqual(result['exit_via'], 'wireguard_ext')
|
||||
|
||||
# Find the mangle MARK -A call
|
||||
mark_calls = [
|
||||
c for c in calls_made
|
||||
if 'iptables' in str(c) and 'MARK' in c and '--set-mark' in c
|
||||
]
|
||||
self.assertGreater(len(mark_calls), 0, 'No MARK rule was added')
|
||||
mark_cmd = ' '.join(mark_calls[0])
|
||||
self.assertIn('0x110', mark_cmd)
|
||||
self.assertIn('pic-egr-myapp', mark_cmd)
|
||||
self.assertIn('mangle', mark_cmd)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 3. test_apply_service_openvpn_adds_mark_rule
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestApplyServiceOpenVPN(unittest.TestCase):
|
||||
|
||||
def test_apply_service_openvpn_adds_mark_rule(self):
|
||||
"""openvpn exit must add a mangle MARK rule with 0x120."""
|
||||
manifest = _make_manifest(egress_default='openvpn')
|
||||
mgr, _ = _make_manager(installed=_installed_with_manifest(manifest))
|
||||
|
||||
calls_made = []
|
||||
|
||||
def fake_run(cmd, **kwargs):
|
||||
calls_made.append(cmd)
|
||||
if 'docker' in cmd and 'inspect' in cmd:
|
||||
return _subprocess_ok(stdout='172.20.0.51\n')
|
||||
if 'iptables-save' in cmd:
|
||||
return _subprocess_ok(stdout='')
|
||||
if 'iptables-restore' in cmd:
|
||||
return _subprocess_ok()
|
||||
if cmd[:3] == ['ip', 'rule', 'del']:
|
||||
return _subprocess_fail()
|
||||
return _subprocess_ok()
|
||||
|
||||
with patch('subprocess.run', side_effect=fake_run):
|
||||
result = mgr.apply_service('myapp')
|
||||
|
||||
self.assertTrue(result['ok'], result)
|
||||
self.assertEqual(result['exit_via'], 'openvpn')
|
||||
|
||||
mark_calls = [
|
||||
c for c in calls_made
|
||||
if 'iptables' in str(c) and 'MARK' in c and '--set-mark' in c
|
||||
]
|
||||
self.assertGreater(len(mark_calls), 0)
|
||||
self.assertIn('0x120', ' '.join(mark_calls[0]))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 4. test_apply_service_tor_adds_mark_and_redirect
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestApplyServiceTor(unittest.TestCase):
|
||||
|
||||
def test_apply_service_tor_adds_mark_and_redirect(self):
|
||||
"""tor exit must add a mangle MARK 0x130 AND a nat REDIRECT to port 9040."""
|
||||
manifest = _make_manifest(egress_default='tor')
|
||||
mgr, _ = _make_manager(installed=_installed_with_manifest(manifest))
|
||||
|
||||
calls_made = []
|
||||
|
||||
def fake_run(cmd, **kwargs):
|
||||
calls_made.append(cmd)
|
||||
if 'docker' in cmd and 'inspect' in cmd:
|
||||
return _subprocess_ok(stdout='172.20.0.52\n')
|
||||
if 'iptables-save' in cmd:
|
||||
return _subprocess_ok(stdout='')
|
||||
if 'iptables-restore' in cmd:
|
||||
return _subprocess_ok()
|
||||
if cmd[:3] == ['ip', 'rule', 'del']:
|
||||
return _subprocess_fail()
|
||||
return _subprocess_ok()
|
||||
|
||||
with patch('subprocess.run', side_effect=fake_run):
|
||||
result = mgr.apply_service('myapp')
|
||||
|
||||
self.assertTrue(result['ok'], result)
|
||||
self.assertEqual(result['exit_via'], 'tor')
|
||||
|
||||
mark_calls = [
|
||||
c for c in calls_made
|
||||
if 'iptables' in str(c) and 'MARK' in c and '--set-mark' in c
|
||||
]
|
||||
self.assertGreater(len(mark_calls), 0, 'No MARK rule found')
|
||||
self.assertIn('0x130', ' '.join(mark_calls[0]))
|
||||
|
||||
redirect_calls = [
|
||||
c for c in calls_made
|
||||
if 'iptables' in str(c) and 'REDIRECT' in c
|
||||
]
|
||||
self.assertGreater(len(redirect_calls), 0, 'No REDIRECT rule found')
|
||||
redirect_cmd = ' '.join(redirect_calls[0])
|
||||
self.assertIn('9040', redirect_cmd)
|
||||
self.assertIn('nat', redirect_cmd)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 5. test_apply_service_no_container_ip_returns_error
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestApplyServiceNoContainerIP(unittest.TestCase):
|
||||
|
||||
def test_apply_service_no_container_ip_returns_error(self):
|
||||
"""When docker inspect returns an empty IP, apply_service must return ok=False."""
|
||||
manifest = _make_manifest(egress_default='wireguard_ext')
|
||||
mgr, _ = _make_manager(installed=_installed_with_manifest(manifest))
|
||||
|
||||
def fake_run(cmd, **kwargs):
|
||||
if 'docker' in cmd and 'inspect' in cmd:
|
||||
return _subprocess_ok(stdout='\n') # empty IP
|
||||
if 'iptables-save' in cmd:
|
||||
return _subprocess_ok(stdout='')
|
||||
if 'iptables-restore' in cmd:
|
||||
return _subprocess_ok()
|
||||
return _subprocess_ok()
|
||||
|
||||
with patch('subprocess.run', side_effect=fake_run):
|
||||
result = mgr.apply_service('myapp')
|
||||
|
||||
self.assertFalse(result['ok'])
|
||||
self.assertIn('container IP not discoverable', result.get('error', ''))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 6. test_apply_service_container_ip_retries
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestApplyServiceRetries(unittest.TestCase):
|
||||
|
||||
def test_apply_service_container_ip_retries(self):
|
||||
"""First docker inspect attempt fails; second succeeds — result must be ok=True."""
|
||||
manifest = _make_manifest(egress_default='wireguard_ext')
|
||||
mgr, _ = _make_manager(installed=_installed_with_manifest(manifest))
|
||||
|
||||
inspect_count = [0]
|
||||
|
||||
def fake_run(cmd, **kwargs):
|
||||
if 'docker' in cmd and 'inspect' in cmd:
|
||||
inspect_count[0] += 1
|
||||
if inspect_count[0] == 1:
|
||||
return _subprocess_ok(stdout='\n') # first attempt: empty
|
||||
return _subprocess_ok(stdout='172.20.0.50\n') # second: success
|
||||
if 'iptables-save' in cmd:
|
||||
return _subprocess_ok(stdout='')
|
||||
if 'iptables-restore' in cmd:
|
||||
return _subprocess_ok()
|
||||
if cmd[:3] == ['ip', 'rule', 'del']:
|
||||
return _subprocess_fail()
|
||||
return _subprocess_ok()
|
||||
|
||||
with patch('subprocess.run', side_effect=fake_run):
|
||||
with patch('time.sleep'): # skip actual delays
|
||||
result = mgr.apply_service('myapp')
|
||||
|
||||
self.assertTrue(result['ok'], result)
|
||||
self.assertGreaterEqual(inspect_count[0], 2)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 7. test_has_egress_false_skips_rules
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestHasEgressFalse(unittest.TestCase):
|
||||
|
||||
def test_has_egress_false_skips_rules(self):
|
||||
"""A manifest with has_egress=False must skip rules and return skipped=True."""
|
||||
manifest = _make_manifest(has_egress=False)
|
||||
mgr, _ = _make_manager(installed=_installed_with_manifest(manifest))
|
||||
|
||||
with patch('subprocess.run') as mock_run:
|
||||
mock_run.return_value = _subprocess_ok(stdout='')
|
||||
result = mgr.apply_service('myapp')
|
||||
|
||||
self.assertTrue(result['ok'])
|
||||
self.assertTrue(result.get('skipped'))
|
||||
|
||||
# No iptables rule-insertion call should have been made.
|
||||
# iptables-save from clear_service is permitted; only check no -A/-I.
|
||||
rule_add_calls = [
|
||||
c for c in mock_run.call_args_list
|
||||
if c.args and c.args[0][:1] == ['iptables']
|
||||
and any(a in c.args[0] for a in ('-A', '-I', 'MARK', 'REDIRECT'))
|
||||
]
|
||||
self.assertEqual(rule_add_calls, [])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 8. test_has_egress_missing_egress_block_skips
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestHasEgressMissingBlock(unittest.TestCase):
|
||||
|
||||
def test_has_egress_missing_egress_block_skips(self):
|
||||
"""has_egress=True but no 'egress' dict → must skip (skipped=True)."""
|
||||
manifest = {
|
||||
'id': 'myapp',
|
||||
'container_name': 'cell-myapp',
|
||||
'has_egress': True,
|
||||
# 'egress' key intentionally absent
|
||||
}
|
||||
mgr, _ = _make_manager(
|
||||
installed=_installed_with_manifest(manifest)
|
||||
)
|
||||
|
||||
with patch('subprocess.run') as mock_run:
|
||||
mock_run.return_value = _subprocess_ok(stdout='')
|
||||
result = mgr.apply_service('myapp')
|
||||
|
||||
self.assertTrue(result['ok'])
|
||||
self.assertTrue(result.get('skipped'))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 9. test_clear_service_removes_tagged_rules
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestClearService(unittest.TestCase):
|
||||
|
||||
def test_clear_service_removes_tagged_rules(self):
|
||||
"""iptables-restore is called with the tagged lines removed."""
|
||||
mgr, _ = _make_manager()
|
||||
|
||||
mangle_rules = (
|
||||
'-A PIC_EGRESS -s 172.20.0.50 -j MARK --set-mark 0x110 '
|
||||
'-m comment --comment "pic-egr-myapp"\n'
|
||||
'-A PIC_EGRESS -s 172.20.0.99 -j MARK --set-mark 0x110 '
|
||||
'-m comment --comment "pic-egr-otherapp"\n'
|
||||
)
|
||||
nat_rules = ''
|
||||
|
||||
restore_inputs = {}
|
||||
|
||||
def fake_run(cmd, input=None, **kwargs):
|
||||
if cmd == ['iptables-save', '-t', 'mangle']:
|
||||
return _subprocess_ok(stdout=mangle_rules)
|
||||
if cmd == ['iptables-save', '-t', 'nat']:
|
||||
return _subprocess_ok(stdout=nat_rules)
|
||||
if cmd == ['iptables-restore', '-T', 'mangle']:
|
||||
restore_inputs['mangle'] = input
|
||||
return _subprocess_ok()
|
||||
if cmd == ['iptables-restore', '-T', 'nat']:
|
||||
restore_inputs['nat'] = input
|
||||
return _subprocess_ok()
|
||||
return _subprocess_ok()
|
||||
|
||||
with patch('subprocess.run', side_effect=fake_run):
|
||||
result = mgr.clear_service('myapp')
|
||||
|
||||
self.assertTrue(result['ok'])
|
||||
# The restored mangle rules must not contain myapp's tag
|
||||
restored = restore_inputs.get('mangle', '')
|
||||
self.assertNotIn('pic-egr-myapp', restored)
|
||||
# But the other service's rules must be preserved
|
||||
self.assertIn('pic-egr-otherapp', restored)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 10. test_set_service_exit_rejects_not_in_allowed
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestSetServiceExitRejectNotAllowed(unittest.TestCase):
|
||||
|
||||
def test_set_service_exit_rejects_not_in_allowed(self):
|
||||
"""Exit type not in manifest's allowed list must return ok=False."""
|
||||
manifest = _make_manifest(
|
||||
egress_default='default',
|
||||
allowed=['default', 'tor'], # wireguard_ext not in allowed
|
||||
)
|
||||
mgr, _ = _make_manager(installed=_installed_with_manifest(manifest))
|
||||
|
||||
result = mgr.set_service_exit('myapp', 'wireguard_ext')
|
||||
|
||||
self.assertFalse(result['ok'])
|
||||
self.assertIn('error', result)
|
||||
self.assertIn('allowed', result['error'])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 11. test_set_service_exit_persists_and_applies
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestSetServiceExitPersistsAndApplies(unittest.TestCase):
|
||||
|
||||
def test_set_service_exit_persists_and_applies(self):
|
||||
"""Valid override must be persisted to config_manager and apply_service called."""
|
||||
manifest = _make_manifest(egress_default='default', allowed=list(EXIT_TYPES))
|
||||
mgr, cm = _make_manager(installed=_installed_with_manifest(manifest))
|
||||
|
||||
apply_calls = []
|
||||
original_apply = mgr.apply_service
|
||||
|
||||
def fake_apply(sid):
|
||||
apply_calls.append(sid)
|
||||
return {'ok': True, 'exit_via': 'tor'}
|
||||
|
||||
mgr.apply_service = fake_apply
|
||||
|
||||
result = mgr.set_service_exit('myapp', 'tor')
|
||||
|
||||
self.assertTrue(result['ok'], result)
|
||||
# apply_service was called
|
||||
self.assertIn('myapp', apply_calls)
|
||||
# override was persisted
|
||||
cm._save_all_configs.assert_called()
|
||||
self.assertEqual(cm.configs['egress_overrides'].get('myapp'), 'tor')
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 12. test_apply_all_iterates_installed_services
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestApplyAll(unittest.TestCase):
|
||||
|
||||
def test_apply_all_iterates_installed_services(self):
|
||||
"""apply_all must call apply_service for every service with a manifest."""
|
||||
manifests = {
|
||||
'svc1': _make_manifest(egress_default='wireguard_ext'),
|
||||
'svc2': _make_manifest(egress_default='openvpn'),
|
||||
'svc3': _make_manifest(egress_default='tor'),
|
||||
}
|
||||
installed = {
|
||||
sid: {'id': sid, 'manifest': m}
|
||||
for sid, m in manifests.items()
|
||||
}
|
||||
mgr, _ = _make_manager(installed=installed)
|
||||
|
||||
applied = []
|
||||
mgr.apply_service = lambda sid: applied.append(sid) or {'ok': True}
|
||||
|
||||
result = mgr.apply_all()
|
||||
|
||||
self.assertTrue(result['ok'])
|
||||
self.assertEqual(sorted(applied), ['svc1', 'svc2', 'svc3'])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 13. test_marks_do_not_collide_with_connectivity_manager
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestMarksNoCollision(unittest.TestCase):
|
||||
|
||||
def test_marks_do_not_collide_with_connectivity_manager(self):
|
||||
"""EgressManager marks must be disjoint from ConnectivityManager marks."""
|
||||
connectivity_marks = {0x10, 0x20, 0x30}
|
||||
egress_mark_values = set(MARKS.values())
|
||||
collision = connectivity_marks & egress_mark_values
|
||||
self.assertEqual(
|
||||
collision, set(),
|
||||
f'Mark collision with ConnectivityManager: {collision}',
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 14. test_apply_service_unknown_exit_in_allowed_rejected
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestApplyServiceUnknownExit(unittest.TestCase):
|
||||
|
||||
def test_apply_service_unknown_exit_in_allowed_rejected(self):
|
||||
"""An egress.default value that is not a known EXIT_TYPE must return ok=False."""
|
||||
manifest = {
|
||||
'id': 'myapp',
|
||||
'container_name': 'cell-myapp',
|
||||
'has_egress': True,
|
||||
'egress': {
|
||||
'default': 'internet_fast_lane', # unknown exit
|
||||
'allowed': ['internet_fast_lane'],
|
||||
},
|
||||
}
|
||||
mgr, _ = _make_manager(installed=_installed_with_manifest(manifest))
|
||||
|
||||
def fake_run(cmd, **kwargs):
|
||||
if 'docker' in cmd and 'inspect' in cmd:
|
||||
return _subprocess_ok(stdout='172.20.0.50\n')
|
||||
if 'iptables-save' in cmd:
|
||||
return _subprocess_ok(stdout='')
|
||||
if 'iptables-restore' in cmd:
|
||||
return _subprocess_ok()
|
||||
return _subprocess_ok()
|
||||
|
||||
with patch('subprocess.run', side_effect=fake_run):
|
||||
result = mgr.apply_service('myapp')
|
||||
|
||||
self.assertFalse(result['ok'])
|
||||
self.assertIn('error', result)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Additional coverage: _has_egress edge cases
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestHasEgressLogic(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.mgr, _ = _make_manager()
|
||||
|
||||
def test_has_egress_both_required(self):
|
||||
"""Both has_egress=True and non-empty egress dict required."""
|
||||
m = {'has_egress': True, 'egress': {'default': 'tor', 'allowed': ['tor']}}
|
||||
self.assertTrue(self.mgr._has_egress(m))
|
||||
|
||||
def test_has_egress_false_field(self):
|
||||
m = {'has_egress': False, 'egress': {'default': 'tor', 'allowed': ['tor']}}
|
||||
self.assertFalse(self.mgr._has_egress(m))
|
||||
|
||||
def test_has_egress_missing_has_egress_key(self):
|
||||
m = {'egress': {'default': 'tor', 'allowed': ['tor']}}
|
||||
self.assertFalse(self.mgr._has_egress(m))
|
||||
|
||||
def test_has_egress_empty_egress_dict(self):
|
||||
m = {'has_egress': True, 'egress': {}}
|
||||
self.assertFalse(self.mgr._has_egress(m))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Additional coverage: _resolve_exit
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestResolveExit(unittest.TestCase):
|
||||
|
||||
def test_override_takes_precedence(self):
|
||||
mgr, _ = _make_manager(overrides={'myapp': 'openvpn'})
|
||||
manifest = _make_manifest(egress_default='wireguard_ext')
|
||||
self.assertEqual(mgr._resolve_exit('myapp', manifest), 'openvpn')
|
||||
|
||||
def test_manifest_default_used_when_no_override(self):
|
||||
mgr, _ = _make_manager(overrides={})
|
||||
manifest = _make_manifest(egress_default='tor')
|
||||
self.assertEqual(mgr._resolve_exit('myapp', manifest), 'tor')
|
||||
|
||||
def test_fallback_to_default_when_no_egress_block(self):
|
||||
mgr, _ = _make_manager(overrides={})
|
||||
manifest = {'id': 'myapp'}
|
||||
self.assertEqual(mgr._resolve_exit('myapp', manifest), 'default')
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Additional: apply_service with missing manifest
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestApplyServiceMissingManifest(unittest.TestCase):
|
||||
|
||||
def test_apply_service_missing_manifest_returns_error(self):
|
||||
mgr, _ = _make_manager(installed={})
|
||||
result = mgr.apply_service('ghost')
|
||||
self.assertFalse(result['ok'])
|
||||
self.assertIn('error', result)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -21,16 +21,25 @@ sys.path.insert(0, str(api_dir))
|
||||
|
||||
from app import app
|
||||
|
||||
# Sentinel value that make service_registry.get(...) return non-None (service installed)
|
||||
_INSTALLED = {'id': 'email', 'installed': True}
|
||||
|
||||
|
||||
class TestGetEmailUsers(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
app.config['TESTING'] = True
|
||||
self.client = app.test_client()
|
||||
self._sr_patcher = patch('app.service_registry')
|
||||
mock_sr = self._sr_patcher.start()
|
||||
mock_sr.get.return_value = _INSTALLED
|
||||
|
||||
def tearDown(self):
|
||||
self._sr_patcher.stop()
|
||||
|
||||
@patch('app.email_manager')
|
||||
def test_get_users_returns_200_with_list(self, mock_em):
|
||||
mock_em.get_users.return_value = [
|
||||
mock_em.get_email_users.return_value = [
|
||||
{'username': 'alice@cell', 'domain': 'cell'},
|
||||
{'username': 'bob@cell', 'domain': 'cell'},
|
||||
]
|
||||
@@ -42,14 +51,14 @@ class TestGetEmailUsers(unittest.TestCase):
|
||||
|
||||
@patch('app.email_manager')
|
||||
def test_get_users_returns_empty_list_when_no_users(self, mock_em):
|
||||
mock_em.get_users.return_value = []
|
||||
mock_em.get_email_users.return_value = []
|
||||
r = self.client.get('/api/email/users')
|
||||
self.assertEqual(r.status_code, 200)
|
||||
self.assertEqual(json.loads(r.data), [])
|
||||
|
||||
@patch('app.email_manager')
|
||||
def test_get_users_returns_500_on_exception(self, mock_em):
|
||||
mock_em.get_users.side_effect = Exception('mailbox unreachable')
|
||||
mock_em.get_email_users.side_effect = Exception('mailbox unreachable')
|
||||
r = self.client.get('/api/email/users')
|
||||
self.assertEqual(r.status_code, 500)
|
||||
self.assertIn('error', json.loads(r.data))
|
||||
@@ -60,6 +69,12 @@ class TestCreateEmailUser(unittest.TestCase):
|
||||
def setUp(self):
|
||||
app.config['TESTING'] = True
|
||||
self.client = app.test_client()
|
||||
self._sr_patcher = patch('app.service_registry')
|
||||
mock_sr = self._sr_patcher.start()
|
||||
mock_sr.get.return_value = _INSTALLED
|
||||
|
||||
def tearDown(self):
|
||||
self._sr_patcher.stop()
|
||||
|
||||
@patch('app.email_manager')
|
||||
def test_create_user_returns_200_on_success(self, mock_em):
|
||||
@@ -131,6 +146,12 @@ class TestDeleteEmailUser(unittest.TestCase):
|
||||
def setUp(self):
|
||||
app.config['TESTING'] = True
|
||||
self.client = app.test_client()
|
||||
self._sr_patcher = patch('app.service_registry')
|
||||
mock_sr = self._sr_patcher.start()
|
||||
mock_sr.get.return_value = _INSTALLED
|
||||
|
||||
def tearDown(self):
|
||||
self._sr_patcher.stop()
|
||||
|
||||
@patch('app.email_manager')
|
||||
def test_delete_user_returns_200_on_success(self, mock_em):
|
||||
@@ -187,6 +208,12 @@ class TestEmailConnectivity(unittest.TestCase):
|
||||
def setUp(self):
|
||||
app.config['TESTING'] = True
|
||||
self.client = app.test_client()
|
||||
self._sr_patcher = patch('app.service_registry')
|
||||
mock_sr = self._sr_patcher.start()
|
||||
mock_sr.get.return_value = _INSTALLED
|
||||
|
||||
def tearDown(self):
|
||||
self._sr_patcher.stop()
|
||||
|
||||
@patch('app.email_manager')
|
||||
def test_connectivity_returns_200_with_result(self, mock_em):
|
||||
|
||||
@@ -104,5 +104,95 @@ class TestEmailManager(unittest.TestCase):
|
||||
info = self.manager.get_mailbox_info(None, None)
|
||||
self.assertIn('error', info)
|
||||
|
||||
class TestEmailManagerEffectiveDomain(unittest.TestCase):
|
||||
"""Verify that email OVERRIDE_HOSTNAME and POSTMASTER_ADDRESS use the
|
||||
caller-supplied domain (which should come from get_effective_domain in the
|
||||
route layer when no explicit domain is provided by the client)."""
|
||||
|
||||
def setUp(self):
|
||||
self.test_dir = tempfile.mkdtemp()
|
||||
self.data_dir = os.path.join(self.test_dir, 'data')
|
||||
self.config_dir = os.path.join(self.test_dir, 'config')
|
||||
os.makedirs(os.path.join(self.config_dir, 'mail'), exist_ok=True)
|
||||
os.makedirs(os.path.join(self.data_dir, 'email'), exist_ok=True)
|
||||
with open(os.path.join(self.config_dir, 'mail', 'mailserver.env'), 'w') as f:
|
||||
f.write('OVERRIDE_HOSTNAME=mail.cell\nPOSTMASTER_ADDRESS=admin@cell\n')
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent / 'api'))
|
||||
from email_manager import EmailManager
|
||||
self.em = EmailManager(data_dir=self.data_dir, config_dir=self.config_dir)
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.test_dir)
|
||||
|
||||
@patch('subprocess.run')
|
||||
def test_email_hostname_uses_effective_domain_in_ddns_mode(self, mock_run):
|
||||
"""When apply_config is called with domain='home.pic.ngo' (as provided
|
||||
by the route layer via get_effective_domain), OVERRIDE_HOSTNAME and
|
||||
POSTMASTER_ADDRESS should use 'home.pic.ngo', not the internal 'cell'."""
|
||||
mock_run.return_value = MagicMock(returncode=0)
|
||||
result = self.em.apply_config({'domain': 'home.pic.ngo'})
|
||||
env = open(os.path.join(self.config_dir, 'mail', 'mailserver.env')).read()
|
||||
self.assertIn('OVERRIDE_HOSTNAME=mail.home.pic.ngo', env)
|
||||
self.assertIn('POSTMASTER_ADDRESS=admin@home.pic.ngo', env)
|
||||
self.assertIn('cell-mail', result['restarted'])
|
||||
|
||||
|
||||
class TestEmailManagerIdentityChangedSubscription(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.test_dir = tempfile.mkdtemp()
|
||||
self.data_dir = os.path.join(self.test_dir, 'data')
|
||||
self.config_dir = os.path.join(self.test_dir, 'config')
|
||||
os.makedirs(self.data_dir, exist_ok=True)
|
||||
os.makedirs(self.config_dir, exist_ok=True)
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.test_dir)
|
||||
|
||||
def test_subscribes_to_identity_changed_on_init(self):
|
||||
"""When service_bus is provided, __init__ subscribes to IDENTITY_CHANGED."""
|
||||
from service_bus import EventType
|
||||
mock_bus = MagicMock()
|
||||
manager = EmailManager(
|
||||
data_dir=self.data_dir,
|
||||
config_dir=self.config_dir,
|
||||
service_bus=mock_bus,
|
||||
)
|
||||
mock_bus.subscribe_to_event.assert_called_once_with(
|
||||
EventType.IDENTITY_CHANGED, manager._on_identity_changed
|
||||
)
|
||||
|
||||
def test_no_subscription_without_service_bus(self):
|
||||
"""When service_bus is not provided, no subscription is attempted."""
|
||||
mock_bus = MagicMock()
|
||||
EmailManager(data_dir=self.data_dir, config_dir=self.config_dir)
|
||||
mock_bus.subscribe_to_event.assert_not_called()
|
||||
|
||||
@patch.object(EmailManager, 'apply_config', return_value={'restarted': [], 'warnings': []})
|
||||
def test_on_identity_changed_calls_apply_config(self, mock_apply):
|
||||
"""_on_identity_changed calls apply_config with the effective_domain."""
|
||||
manager = EmailManager(data_dir=self.data_dir, config_dir=self.config_dir)
|
||||
event = MagicMock()
|
||||
event.data = {'effective_domain': 'mycell.pic.ngo'}
|
||||
manager._on_identity_changed(event)
|
||||
mock_apply.assert_called_once_with({'domain': 'mycell.pic.ngo'})
|
||||
|
||||
@patch.object(EmailManager, 'apply_config', side_effect=Exception('boom'))
|
||||
def test_on_identity_changed_swallows_exceptions(self, mock_apply):
|
||||
"""_on_identity_changed must not propagate exceptions."""
|
||||
manager = EmailManager(data_dir=self.data_dir, config_dir=self.config_dir)
|
||||
event = MagicMock()
|
||||
event.data = {'effective_domain': 'mycell.pic.ngo'}
|
||||
manager._on_identity_changed(event) # must not raise
|
||||
|
||||
def test_on_identity_changed_skips_when_no_effective_domain(self):
|
||||
"""_on_identity_changed does nothing when effective_domain is absent."""
|
||||
manager = EmailManager(data_dir=self.data_dir, config_dir=self.config_dir)
|
||||
event = MagicMock()
|
||||
event.data = {'cell_name': 'mycell'}
|
||||
with patch.object(manager, 'apply_config') as mock_apply:
|
||||
manager._on_identity_changed(event)
|
||||
mock_apply.assert_not_called()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -53,8 +53,11 @@ def empty_auth_manager(tmp_path):
|
||||
os.makedirs(data_dir, exist_ok=True)
|
||||
os.makedirs(config_dir, exist_ok=True)
|
||||
mgr = AuthManager(data_dir=data_dir, config_dir=config_dir)
|
||||
# The constructor creates the file with '[]' (empty list). We do NOT add
|
||||
# any user, so list_users() returns [] but the file is readable.
|
||||
# Explicitly create the file with an empty list to simulate the
|
||||
# "auth configured but no users" misconfiguration scenario.
|
||||
users_file = os.path.join(data_dir, 'auth_users.json')
|
||||
with open(users_file, 'w') as f:
|
||||
f.write('[]')
|
||||
assert mgr.list_users() == [], 'Expected empty user list'
|
||||
return mgr
|
||||
|
||||
@@ -138,5 +141,81 @@ def test_empty_auth_manager_non_api_path_bypasses_503(
|
||||
assert r.status_code == 200
|
||||
|
||||
|
||||
# ── role-based access: peer vs admin ─────────────────────────────────────────
|
||||
|
||||
@pytest.fixture
|
||||
def peer_client(tmp_path):
|
||||
"""Test client with a peer-role session active."""
|
||||
from app import app
|
||||
from auth_manager import AuthManager
|
||||
data_dir = str(tmp_path / 'data')
|
||||
config_dir = str(tmp_path / 'config')
|
||||
os.makedirs(data_dir, exist_ok=True)
|
||||
os.makedirs(config_dir, exist_ok=True)
|
||||
mgr = AuthManager(data_dir=data_dir, config_dir=config_dir)
|
||||
mgr.create_user('admin', 'AdminPass123!', 'admin')
|
||||
mgr.create_user('alice', 'AlicePass123!', 'peer')
|
||||
|
||||
app.config['TESTING'] = True
|
||||
with patch('app.auth_manager', mgr):
|
||||
try:
|
||||
import auth_routes
|
||||
with patch.object(auth_routes, 'auth_manager', mgr, create=True):
|
||||
with app.test_client() as client:
|
||||
r = client.post('/api/auth/login',
|
||||
data=json.dumps({'username': 'alice', 'password': 'AlicePass123!'}),
|
||||
content_type='application/json')
|
||||
assert r.status_code == 200, f'peer login failed: {r.status_code}'
|
||||
yield client, mgr
|
||||
except ImportError:
|
||||
with app.test_client() as client:
|
||||
with client.session_transaction() as sess:
|
||||
sess['username'] = 'alice'
|
||||
sess['role'] = 'peer'
|
||||
yield client, mgr
|
||||
|
||||
|
||||
def test_peer_role_blocked_from_admin_only_endpoint(peer_client):
|
||||
"""Peer sessions must not access admin-only endpoints like /api/peers."""
|
||||
client, mgr = peer_client
|
||||
with patch('app.auth_manager', mgr):
|
||||
r = client.get('/api/peers')
|
||||
assert r.status_code == 403
|
||||
|
||||
|
||||
def test_peer_role_allowed_services_active(peer_client):
|
||||
"""/api/services/active must be accessible to peer sessions.
|
||||
|
||||
Regression guard: peers saw 'not installed' on My Services because
|
||||
enforce_auth returned 403 for this endpoint.
|
||||
"""
|
||||
client, mgr = peer_client
|
||||
with patch('app.auth_manager', mgr):
|
||||
r = client.get('/api/services/active')
|
||||
# 200 (or whatever the route returns) but NOT 403
|
||||
assert r.status_code != 403, (
|
||||
'/api/services/active returned 403 for peer — peer UI cannot show installed services'
|
||||
)
|
||||
|
||||
|
||||
def test_admin_role_still_allowed_services_active(flask_client, populated_auth_manager):
|
||||
"""/api/services/active must remain accessible to admin sessions."""
|
||||
with patch('app.auth_manager', populated_auth_manager):
|
||||
try:
|
||||
import auth_routes
|
||||
with patch.object(auth_routes, 'auth_manager', populated_auth_manager, create=True):
|
||||
r_login = flask_client.post('/api/auth/login',
|
||||
data=json.dumps({'username': 'admin', 'password': 'AdminPass123!'}),
|
||||
content_type='application/json')
|
||||
assert r_login.status_code == 200
|
||||
r = flask_client.get('/api/services/active')
|
||||
except ImportError:
|
||||
with flask_client.session_transaction() as sess:
|
||||
sess['username'] = 'admin'
|
||||
sess['role'] = 'admin'
|
||||
r = flask_client.get('/api/services/active')
|
||||
assert r.status_code != 403
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
pytest.main([__file__, '-v'])
|
||||
|
||||
@@ -25,12 +25,20 @@ sys.path.insert(0, str(api_dir))
|
||||
|
||||
from app import app
|
||||
|
||||
_INSTALLED = {'id': 'files', 'installed': True}
|
||||
|
||||
|
||||
class TestFileUsersEndpoints(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
app.config['TESTING'] = True
|
||||
self.client = app.test_client()
|
||||
self._sr_patcher = patch('app.service_registry')
|
||||
mock_sr = self._sr_patcher.start()
|
||||
mock_sr.get.return_value = _INSTALLED
|
||||
|
||||
def tearDown(self):
|
||||
self._sr_patcher.stop()
|
||||
|
||||
# ── GET /api/files/users ────────────────────────────────────────────────
|
||||
|
||||
@@ -94,6 +102,12 @@ class TestFileListEndpoint(unittest.TestCase):
|
||||
def setUp(self):
|
||||
app.config['TESTING'] = True
|
||||
self.client = app.test_client()
|
||||
self._sr_patcher = patch('app.service_registry')
|
||||
mock_sr = self._sr_patcher.start()
|
||||
mock_sr.get.return_value = _INSTALLED
|
||||
|
||||
def tearDown(self):
|
||||
self._sr_patcher.stop()
|
||||
|
||||
# ── GET /api/files/list/<username> ─────────────────────────────────────
|
||||
|
||||
@@ -134,6 +148,12 @@ class TestFileFolderDeleteEndpoint(unittest.TestCase):
|
||||
def setUp(self):
|
||||
app.config['TESTING'] = True
|
||||
self.client = app.test_client()
|
||||
self._sr_patcher = patch('app.service_registry')
|
||||
mock_sr = self._sr_patcher.start()
|
||||
mock_sr.get.return_value = _INSTALLED
|
||||
|
||||
def tearDown(self):
|
||||
self._sr_patcher.stop()
|
||||
|
||||
# ── DELETE /api/files/folders/<username>/<path> ────────────────────────
|
||||
|
||||
@@ -186,6 +206,12 @@ class TestFileDownloadDeleteEndpoints(unittest.TestCase):
|
||||
def setUp(self):
|
||||
app.config['TESTING'] = True
|
||||
self.client = app.test_client()
|
||||
self._sr_patcher = patch('app.service_registry')
|
||||
mock_sr = self._sr_patcher.start()
|
||||
mock_sr.get.return_value = _INSTALLED
|
||||
|
||||
def tearDown(self):
|
||||
self._sr_patcher.stop()
|
||||
|
||||
# ── GET /api/files/download/<username>/<path> ──────────────────────────
|
||||
|
||||
@@ -223,6 +249,12 @@ class TestFileCreateFolderEndpoint(unittest.TestCase):
|
||||
def setUp(self):
|
||||
app.config['TESTING'] = True
|
||||
self.client = app.test_client()
|
||||
self._sr_patcher = patch('app.service_registry')
|
||||
mock_sr = self._sr_patcher.start()
|
||||
mock_sr.get.return_value = _INSTALLED
|
||||
|
||||
def tearDown(self):
|
||||
self._sr_patcher.stop()
|
||||
|
||||
# ── POST /api/files/folders ────────────────────────────────────────────
|
||||
|
||||
@@ -259,6 +291,12 @@ class TestFileUploadEndpoint(unittest.TestCase):
|
||||
def setUp(self):
|
||||
app.config['TESTING'] = True
|
||||
self.client = app.test_client()
|
||||
self._sr_patcher = patch('app.service_registry')
|
||||
mock_sr = self._sr_patcher.start()
|
||||
mock_sr.get.return_value = _INSTALLED
|
||||
|
||||
def tearDown(self):
|
||||
self._sr_patcher.stop()
|
||||
|
||||
# ── POST /api/files/upload/<username> ──────────────────────────────────
|
||||
|
||||
|
||||
@@ -133,7 +133,8 @@ class TestGenerateCorefile(unittest.TestCase):
|
||||
self.assertIn('reload', content)
|
||||
|
||||
def test_returns_false_on_write_error(self):
|
||||
result = firewall_manager.generate_corefile([], '/nonexistent/path/Corefile')
|
||||
with unittest.mock.patch('builtins.open', side_effect=OSError('Permission denied')):
|
||||
result = firewall_manager.generate_corefile([], '/any/path/Corefile')
|
||||
self.assertFalse(result)
|
||||
|
||||
|
||||
@@ -218,6 +219,70 @@ class TestGenerateCorefileWithCellLinks(unittest.TestCase):
|
||||
self.assertNotIn('nope.cell', content)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# generate_corefile with split_horizon_zones
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestGenerateCorefileSplitHorizon(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.mkdtemp()
|
||||
self.path = os.path.join(self.tmp, 'Corefile')
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.tmp)
|
||||
|
||||
def _content(self):
|
||||
return open(self.path).read()
|
||||
|
||||
def test_split_horizon_zone_block_added(self):
|
||||
"""A split_horizon_zones entry produces a local file zone block."""
|
||||
firewall_manager.generate_corefile([], self.path, split_horizon_zones=['pic1.pic.ngo'])
|
||||
content = self._content()
|
||||
self.assertIn('pic1.pic.ngo {', content)
|
||||
self.assertIn('file /data/pic1.pic.ngo.zone', content)
|
||||
|
||||
def test_split_horizon_zone_has_acme_challenge_forward(self):
|
||||
"""Each split-horizon zone gets a more-specific _acme-challenge forwarding block
|
||||
so Caddy's DNS-01 pre-verification bypasses the authoritative local zone."""
|
||||
firewall_manager.generate_corefile([], self.path, split_horizon_zones=['pic1.pic.ngo'])
|
||||
content = self._content()
|
||||
self.assertIn('_acme-challenge.pic1.pic.ngo {', content)
|
||||
# ACME block appears before the zone file block so CoreDNS matches it first
|
||||
acme_pos = content.index('_acme-challenge.pic1.pic.ngo {')
|
||||
zone_pos = content.index('\npic1.pic.ngo {')
|
||||
self.assertLess(acme_pos, zone_pos)
|
||||
# ACME block contains a forward directive, not a local file
|
||||
acme_block_end = content.index('}', acme_pos)
|
||||
acme_block = content[acme_pos:acme_block_end]
|
||||
self.assertIn('forward .', acme_block)
|
||||
self.assertNotIn('file /data/', acme_block)
|
||||
|
||||
def test_multiple_split_horizon_zones(self):
|
||||
"""Multiple zones all get their own file block."""
|
||||
firewall_manager.generate_corefile(
|
||||
[], self.path, split_horizon_zones=['a.pic.ngo', 'b.example.com']
|
||||
)
|
||||
content = self._content()
|
||||
self.assertIn('a.pic.ngo {', content)
|
||||
self.assertIn('file /data/a.pic.ngo.zone', content)
|
||||
self.assertIn('b.example.com {', content)
|
||||
self.assertIn('file /data/b.example.com.zone', content)
|
||||
|
||||
def test_split_horizon_with_cell_links(self):
|
||||
"""Split-horizon zones and cell-link forwarding stanzas coexist."""
|
||||
cell_links = [{'domain': 'other.cell', 'dns_ip': '10.99.0.1'}]
|
||||
firewall_manager.generate_corefile(
|
||||
[], self.path,
|
||||
cell_links=cell_links,
|
||||
split_horizon_zones=['pic1.pic.ngo'],
|
||||
)
|
||||
content = self._content()
|
||||
self.assertIn('pic1.pic.ngo {', content)
|
||||
self.assertIn('file /data/pic1.pic.ngo.zone', content)
|
||||
self.assertIn('other.cell {', content)
|
||||
self.assertIn('forward . 10.99.0.1', content)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# apply_peer_rules — iptables call verification
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -0,0 +1,244 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Tests for the installation script (install.sh) and Makefile start targets.
|
||||
|
||||
These are static-analysis tests — they read the files directly and verify
|
||||
critical properties without running Docker or making network calls.
|
||||
|
||||
Covered:
|
||||
- install.sh: bash syntax, required steps, idempotency guard, --force flag
|
||||
- Makefile: every start-* target that uses DCF creates cell-network first
|
||||
- Makefile: start, update, start-core all include the network-create guard
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).parent.parent
|
||||
INSTALL_SH = REPO_ROOT / 'install.sh'
|
||||
MAKEFILE = REPO_ROOT / 'Makefile'
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _makefile_text() -> str:
|
||||
return MAKEFILE.read_text()
|
||||
|
||||
|
||||
def _install_sh_text() -> str:
|
||||
return INSTALL_SH.read_text()
|
||||
|
||||
|
||||
def _target_body(makefile: str, target: str) -> str:
|
||||
"""Return the recipe lines (indented with tab) for a given Makefile target."""
|
||||
pattern = re.compile(
|
||||
rf'^{re.escape(target)}:.*?\n((?:\t[^\n]*\n)*)',
|
||||
re.MULTILINE,
|
||||
)
|
||||
m = pattern.search(makefile)
|
||||
return m.group(1) if m else ''
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# install.sh — syntax and structure
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestInstallShSyntax(unittest.TestCase):
|
||||
"""install.sh must be syntactically valid bash."""
|
||||
|
||||
def test_bash_syntax_check(self):
|
||||
result = subprocess.run(
|
||||
['bash', '-n', str(INSTALL_SH)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
self.assertEqual(
|
||||
result.returncode, 0,
|
||||
f'install.sh has bash syntax errors:\n{result.stderr}',
|
||||
)
|
||||
|
||||
def test_shellcheck_if_available(self):
|
||||
if not (subprocess.run(['which', 'shellcheck'], capture_output=True).returncode == 0):
|
||||
self.skipTest('shellcheck not installed')
|
||||
result = subprocess.run(
|
||||
['shellcheck', '-S', 'warning', str(INSTALL_SH)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
self.assertEqual(
|
||||
result.returncode, 0,
|
||||
f'shellcheck found issues in install.sh:\n{result.stdout}\n{result.stderr}',
|
||||
)
|
||||
|
||||
|
||||
class TestInstallShStructure(unittest.TestCase):
|
||||
"""install.sh must contain all required installation steps."""
|
||||
|
||||
def setUp(self):
|
||||
self.text = _install_sh_text()
|
||||
|
||||
def test_has_idempotency_guard(self):
|
||||
"""Script must exit early if already installed (without --force)."""
|
||||
self.assertIn('.installed', self.text,
|
||||
'install.sh must check for .installed sentinel file')
|
||||
self.assertIn('FORCE', self.text,
|
||||
'install.sh must support a FORCE override')
|
||||
|
||||
def test_calls_make_install(self):
|
||||
self.assertIn('make install', self.text,
|
||||
'install.sh must call make install')
|
||||
|
||||
def test_calls_make_start_core(self):
|
||||
"""install.sh must start core services after installation."""
|
||||
self.assertIn('make start-core', self.text,
|
||||
'install.sh must call make start-core to bring up containers')
|
||||
|
||||
def test_waits_for_api_health(self):
|
||||
"""install.sh must poll the API health endpoint after starting containers."""
|
||||
self.assertIn('/health', self.text,
|
||||
'install.sh must wait for API health endpoint')
|
||||
|
||||
def test_supports_force_flag(self):
|
||||
self.assertIn('--force', self.text,
|
||||
'install.sh must accept --force to bypass idempotency check')
|
||||
|
||||
def test_supports_custom_pic_dir(self):
|
||||
self.assertIn('PIC_DIR', self.text,
|
||||
'install.sh must respect PIC_DIR environment variable')
|
||||
|
||||
def test_clones_from_pic_ngo(self):
|
||||
self.assertIn('git.pic.ngo', self.text,
|
||||
'install.sh must clone from git.pic.ngo')
|
||||
|
||||
def test_set_euo_pipefail(self):
|
||||
"""Script must exit on error and on undefined variables."""
|
||||
self.assertIn('set -euo pipefail', self.text,
|
||||
'install.sh must use set -euo pipefail for safety')
|
||||
|
||||
def test_supports_apt_dnf_apk(self):
|
||||
"""Script must handle the three supported package managers."""
|
||||
for pm in ('apt', 'dnf', 'apk'):
|
||||
self.assertIn(pm, self.text,
|
||||
f'install.sh must handle {pm} package manager')
|
||||
|
||||
def test_checks_docker_available_after_install(self):
|
||||
self.assertIn('docker', self.text,
|
||||
'install.sh must verify docker is available')
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Makefile — network creation before compose up
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
NETWORK_CREATE_GUARD = 'docker network create'
|
||||
|
||||
|
||||
class TestMakefileNetworkGuard(unittest.TestCase):
|
||||
"""Every Makefile target that runs 'docker compose up' with DCF must
|
||||
create cell-network first.
|
||||
|
||||
docker-compose.services.yml declares cell-network as external:true, so
|
||||
compose up will fail with "network could not be found" on a fresh machine
|
||||
unless the network is created beforehand.
|
||||
|
||||
Regression guard: start-core lacked this guard, causing fresh installs
|
||||
via install.sh to fail at Step 6.
|
||||
"""
|
||||
|
||||
def setUp(self):
|
||||
self.mk = _makefile_text()
|
||||
|
||||
def _body(self, target: str) -> str:
|
||||
return _target_body(self.mk, target)
|
||||
|
||||
def test_start_creates_network(self):
|
||||
body = self._body('start')
|
||||
self.assertIn(NETWORK_CREATE_GUARD, body,
|
||||
"'make start' must create cell-network before docker compose up")
|
||||
|
||||
def test_update_creates_network(self):
|
||||
body = self._body('update')
|
||||
self.assertIn(NETWORK_CREATE_GUARD, body,
|
||||
"'make update' must create cell-network before docker compose up")
|
||||
|
||||
def test_start_core_creates_network(self):
|
||||
"""start-core is called by install.sh — missing guard causes fresh install to fail."""
|
||||
body = self._body('start-core')
|
||||
self.assertIn(NETWORK_CREATE_GUARD, body,
|
||||
"'make start-core' must create cell-network before docker compose up "
|
||||
"(regression: fresh install via install.sh fails without this)")
|
||||
|
||||
def test_network_guard_is_idempotent(self):
|
||||
"""The guard must use 'inspect' to skip creation when the network exists."""
|
||||
body = self._body('start-core')
|
||||
self.assertIn('docker network inspect cell-network', body,
|
||||
"network guard must check inspect before create (idempotent)")
|
||||
|
||||
def test_network_uses_configured_subnet(self):
|
||||
"""All three start targets must respect the CELL_NETWORK env var for the subnet."""
|
||||
for target in ('start', 'update', 'start-core'):
|
||||
body = self._body(target)
|
||||
self.assertIn('CELL_NETWORK', body,
|
||||
f"'make {target}' must use CELL_NETWORK env var for subnet")
|
||||
|
||||
|
||||
class TestMakefileTargetPresence(unittest.TestCase):
|
||||
"""Critical Makefile targets must exist."""
|
||||
|
||||
def setUp(self):
|
||||
self.mk = _makefile_text()
|
||||
|
||||
def _has_target(self, name: str) -> bool:
|
||||
return bool(re.search(rf'^{re.escape(name)}:', self.mk, re.MULTILINE))
|
||||
|
||||
def test_start_target_exists(self):
|
||||
self.assertTrue(self._has_target('start'))
|
||||
|
||||
def test_start_core_target_exists(self):
|
||||
self.assertTrue(self._has_target('start-core'))
|
||||
|
||||
def test_update_target_exists(self):
|
||||
self.assertTrue(self._has_target('update'))
|
||||
|
||||
def test_install_target_exists(self):
|
||||
self.assertTrue(self._has_target('install'))
|
||||
|
||||
def test_uninstall_target_exists(self):
|
||||
self.assertTrue(self._has_target('uninstall'))
|
||||
|
||||
def test_test_target_exists(self):
|
||||
self.assertTrue(self._has_target('test'))
|
||||
|
||||
def test_setup_target_exists(self):
|
||||
self.assertTrue(self._has_target('setup'))
|
||||
|
||||
|
||||
class TestDockerComposeServicesFile(unittest.TestCase):
|
||||
"""docker-compose.services.yml must declare cell-network as external so
|
||||
per-service compose stacks can join it without recreating it."""
|
||||
|
||||
def setUp(self):
|
||||
self.path = REPO_ROOT / 'docker-compose.services.yml'
|
||||
|
||||
def test_file_exists(self):
|
||||
self.assertTrue(self.path.exists(),
|
||||
'docker-compose.services.yml must exist')
|
||||
|
||||
def test_cell_network_declared_external(self):
|
||||
text = self.path.read_text()
|
||||
self.assertIn('external: true', text,
|
||||
'docker-compose.services.yml must declare cell-network as external')
|
||||
|
||||
def test_cell_network_name_set(self):
|
||||
text = self.path.read_text()
|
||||
self.assertIn('cell-network', text)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -5,6 +5,7 @@ import sys
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
import unittest.mock
|
||||
from pathlib import Path
|
||||
|
||||
api_dir = Path(__file__).parent.parent / 'api'
|
||||
@@ -98,7 +99,8 @@ class TestWriteEnvFile(unittest.TestCase):
|
||||
self.assertTrue(result)
|
||||
|
||||
def test_returns_false_on_unwritable_path(self):
|
||||
result = ip_utils.write_env_file('172.20.0.0/16', '/nonexistent/deep/path/.env')
|
||||
with unittest.mock.patch('builtins.open', side_effect=OSError('Permission denied')):
|
||||
result = ip_utils.write_env_file('172.20.0.0/16', '/any/path/.env')
|
||||
self.assertFalse(result)
|
||||
|
||||
def test_contains_cell_network(self):
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
"""Tests for cleanup_legacy_builtin_containers in legacy_cleanup.py."""
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent / 'api'))
|
||||
|
||||
from legacy_cleanup import cleanup_legacy_builtin_containers
|
||||
|
||||
|
||||
def _make_cm(already_cleaned=False):
|
||||
cm = MagicMock()
|
||||
cm.configs = {'_meta': {'legacy_builtins_cleaned': already_cleaned}} if already_cleaned else {}
|
||||
return cm
|
||||
|
||||
|
||||
class TestCleanupLegacyBuiltinContainers(unittest.TestCase):
|
||||
|
||||
def test_sentinel_true_skips_all_docker_calls(self):
|
||||
cm = _make_cm(already_cleaned=True)
|
||||
with patch('legacy_cleanup.subprocess.run') as mock_run:
|
||||
cleanup_legacy_builtin_containers(cm)
|
||||
mock_run.assert_not_called()
|
||||
|
||||
def test_container_not_found_skipped(self):
|
||||
"""docker inspect returns non-zero -> container absent -> no stop/rm."""
|
||||
cm = _make_cm()
|
||||
inspect_result = MagicMock(returncode=1, stdout='', stderr='')
|
||||
with patch('legacy_cleanup.subprocess.run', return_value=inspect_result) as mock_run:
|
||||
cleanup_legacy_builtin_containers(cm)
|
||||
# Only inspect calls, no stop/rm
|
||||
for c in mock_run.call_args_list:
|
||||
self.assertNotIn('stop', c.args[0])
|
||||
self.assertNotIn('rm', c.args[0])
|
||||
|
||||
def test_container_from_per_service_project_not_removed(self):
|
||||
"""Project label 'pic-email' -> skip (per-service install)."""
|
||||
cm = _make_cm()
|
||||
inspect_result = MagicMock(returncode=0, stdout='pic-email\n')
|
||||
with patch('legacy_cleanup.subprocess.run', return_value=inspect_result) as mock_run:
|
||||
cleanup_legacy_builtin_containers(cm)
|
||||
for c in mock_run.call_args_list:
|
||||
self.assertNotIn('stop', c.args[0])
|
||||
|
||||
def test_container_from_main_stack_removed(self):
|
||||
"""Project label 'pic' -> stop + rm called."""
|
||||
cm = _make_cm()
|
||||
def side_effect(cmd, **kwargs):
|
||||
r = MagicMock(returncode=0)
|
||||
if 'inspect' in cmd:
|
||||
r.stdout = 'pic\n'
|
||||
else:
|
||||
r.stdout = ''
|
||||
return r
|
||||
with patch('legacy_cleanup.subprocess.run', side_effect=side_effect) as mock_run:
|
||||
cleanup_legacy_builtin_containers(cm)
|
||||
cmds = [c.args[0] for c in mock_run.call_args_list]
|
||||
stop_cmds = [c for c in cmds if 'stop' in c]
|
||||
self.assertGreater(len(stop_cmds), 0)
|
||||
|
||||
def test_sentinel_set_after_run(self):
|
||||
"""_meta.legacy_builtins_cleaned is set to True after cleanup."""
|
||||
cm = _make_cm()
|
||||
inspect_result = MagicMock(returncode=1, stdout='')
|
||||
with patch('legacy_cleanup.subprocess.run', return_value=inspect_result):
|
||||
cleanup_legacy_builtin_containers(cm)
|
||||
self.assertTrue(cm.configs.get('_meta', {}).get('legacy_builtins_cleaned', False))
|
||||
cm._save_all_configs.assert_called()
|
||||
|
||||
def test_exception_in_inspect_does_not_crash(self):
|
||||
"""If docker inspect throws, the function continues and sets sentinel."""
|
||||
cm = _make_cm()
|
||||
with patch('legacy_cleanup.subprocess.run', side_effect=OSError('docker not found')):
|
||||
cleanup_legacy_builtin_containers(cm) # must not raise
|
||||
self.assertTrue(cm.configs.get('_meta', {}).get('legacy_builtins_cleaned', False))
|
||||
|
||||
def test_mixed_containers_only_pic_project_removed(self):
|
||||
"""Some containers are per-service installs, only 'pic'-labelled ones removed."""
|
||||
cm = _make_cm()
|
||||
call_count = [0]
|
||||
def side_effect(cmd, **kwargs):
|
||||
r = MagicMock(returncode=0)
|
||||
if 'inspect' in cmd:
|
||||
call_count[0] += 1
|
||||
# First container is per-service, rest are main stack
|
||||
r.stdout = 'pic-email\n' if call_count[0] == 1 else 'pic\n'
|
||||
else:
|
||||
r.stdout = ''
|
||||
return r
|
||||
with patch('legacy_cleanup.subprocess.run', side_effect=side_effect) as mock_run:
|
||||
cleanup_legacy_builtin_containers(cm)
|
||||
stop_cmds = [c.args[0] for c in mock_run.call_args_list if 'stop' in c.args[0]]
|
||||
# 4 of 5 containers should be stopped (first was per-service)
|
||||
self.assertEqual(len(stop_cmds), 4)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -333,12 +333,8 @@ class TestLogVerbosity(unittest.TestCase):
|
||||
def test_put_verbosity_returns_200_and_calls_set_level(self, mock_lm):
|
||||
mock_lm.get_service_levels.return_value = {'dns': 'DEBUG'}
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
# Endpoint builds: os.path.join(os.path.dirname(__file__), 'config', 'log_levels.json')
|
||||
# Patch dirname to return tmpdir so the full path becomes tmpdir/config/log_levels.json
|
||||
config_dir = os.path.join(tmpdir, 'config')
|
||||
os.makedirs(config_dir)
|
||||
with patch('app.auth_manager', MagicMock(spec=object)), \
|
||||
patch('app.os.path.dirname', return_value=tmpdir):
|
||||
patch.dict('os.environ', {'CONFIG_DIR': tmpdir}):
|
||||
r = self.client.put(
|
||||
'/api/logs/verbosity',
|
||||
data=json.dumps({'dns': 'DEBUG'}),
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -349,8 +349,11 @@ class TestApplyIpRange(unittest.TestCase):
|
||||
self.nm.apply_ip_range('10.1.2.0/24', 'pictest', 'mycell')
|
||||
zone_file = os.path.join(self.nm.dns_zones_dir, 'mycell.zone')
|
||||
content = open(zone_file).read()
|
||||
for host in ('pictest', 'api', 'webui', 'calendar', 'files', 'mail', 'webmail', 'webdav'):
|
||||
# Infrastructure and built-in service names are always generated
|
||||
for host in ('pictest', 'api', 'webui', 'calendar', 'files', 'mail', 'webdav'):
|
||||
self.assertIn(host, content)
|
||||
# Non-built-in names are only generated when a registry is wired
|
||||
self.assertNotIn('webmail', content)
|
||||
|
||||
@patch('subprocess.run')
|
||||
def test_same_range_updates_zone_without_error(self, _mock):
|
||||
@@ -412,5 +415,284 @@ class TestCellDnsForwarding(unittest.TestCase):
|
||||
# The Corefile is regenerated (new canonical format) — that's correct.
|
||||
|
||||
|
||||
class TestUpdateSplitHorizonZone(unittest.TestCase):
|
||||
"""Test update_split_horizon_zone writes zone file and Corefile."""
|
||||
|
||||
def setUp(self):
|
||||
self.test_dir = tempfile.mkdtemp()
|
||||
self.data_dir = os.path.join(self.test_dir, 'data')
|
||||
self.config_dir = os.path.join(self.test_dir, 'config')
|
||||
os.makedirs(os.path.join(self.data_dir, 'dns'), exist_ok=True)
|
||||
os.makedirs(os.path.join(self.config_dir, 'dns'), exist_ok=True)
|
||||
self.nm = NetworkManager(self.data_dir, self.config_dir)
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.test_dir)
|
||||
|
||||
@patch('subprocess.run')
|
||||
def test_creates_zone_file_with_wildcard(self, _mock):
|
||||
"""Zone file must contain wildcard A record pointing to caddy_ip."""
|
||||
self.nm.update_split_horizon_zone('pic1.pic.ngo', '172.20.0.2')
|
||||
zone_path = os.path.join(self.data_dir, 'dns', 'pic1.pic.ngo.zone')
|
||||
self.assertTrue(os.path.exists(zone_path))
|
||||
content = open(zone_path).read()
|
||||
self.assertIn('172.20.0.2', content)
|
||||
|
||||
@patch('subprocess.run')
|
||||
def test_corefile_contains_split_horizon_block(self, _mock):
|
||||
"""Corefile must reference the new zone file."""
|
||||
self.nm.update_split_horizon_zone('pic1.pic.ngo', '172.20.0.2')
|
||||
corefile = os.path.join(self.config_dir, 'dns', 'Corefile')
|
||||
self.assertTrue(os.path.exists(corefile))
|
||||
content = open(corefile).read()
|
||||
self.assertIn('pic1.pic.ngo {', content)
|
||||
self.assertIn('file /data/pic1.pic.ngo.zone', content)
|
||||
|
||||
@patch('subprocess.run')
|
||||
def test_returns_true_on_success(self, _mock):
|
||||
ok = self.nm.update_split_horizon_zone('pic1.pic.ngo', '172.20.0.2')
|
||||
self.assertTrue(ok)
|
||||
|
||||
@patch('subprocess.run')
|
||||
def test_sends_sigusr1_to_coredns(self, mock_run):
|
||||
"""CoreDNS reload (SIGUSR1) must be triggered after writing."""
|
||||
mock_run.return_value = MagicMock(returncode=0, stderr='')
|
||||
self.nm.update_split_horizon_zone('pic1.pic.ngo', '172.20.0.2')
|
||||
calls = [str(c) for c in mock_run.call_args_list]
|
||||
self.assertTrue(any('SIGUSR1' in c for c in calls))
|
||||
|
||||
@patch('subprocess.run')
|
||||
def test_removes_stale_service_records_when_primary_is_parent(self, _mock):
|
||||
"""Stale LAN service names (api, calendar…) are removed from a parent zone.
|
||||
|
||||
A registry that knows about calendar and files is required so those names
|
||||
appear in the stale set.
|
||||
"""
|
||||
from unittest.mock import MagicMock
|
||||
registry = MagicMock()
|
||||
registry.get_caddy_routes.return_value = [
|
||||
{'service_id': 'calendar', 'subdomain': 'calendar',
|
||||
'backend': 'cell-radicale:5232', 'extra_subdomains': [], 'extra_backends': {}},
|
||||
{'service_id': 'files', 'subdomain': 'files',
|
||||
'backend': 'cell-filegator:8080', 'extra_subdomains': [], 'extra_backends': {}},
|
||||
]
|
||||
self.nm._service_registry = registry
|
||||
|
||||
# Bootstrap a pic.ngo zone with service records (wrong internal zone name)
|
||||
stale_records = [
|
||||
{'name': 'pic2', 'type': 'A', 'value': '10.0.0.1'},
|
||||
{'name': 'api', 'type': 'A', 'value': '10.0.0.1'},
|
||||
{'name': 'calendar','type': 'A', 'value': '10.0.0.1'},
|
||||
{'name': 'files', 'type': 'A', 'value': '10.0.0.1'},
|
||||
]
|
||||
self.nm.update_dns_zone('pic.ngo', stale_records)
|
||||
|
||||
# update_split_horizon_zone should strip api/calendar/files from pic.ngo
|
||||
self.nm.update_split_horizon_zone(
|
||||
'pic2.pic.ngo', '172.20.0.2', primary_domain='pic.ngo'
|
||||
)
|
||||
content = open(os.path.join(self.data_dir, 'dns', 'pic.ngo.zone')).read()
|
||||
self.assertNotIn('calendar', content)
|
||||
self.assertNotIn('\napi ', content)
|
||||
self.assertNotIn('\nfiles ', content)
|
||||
# Non-stale record (pic2 is the cell_name, not in _stale set) survives
|
||||
# but api/calendar/files are gone
|
||||
self.assertIn('172.20.0.2', open(
|
||||
os.path.join(self.data_dir, 'dns', 'pic2.pic.ngo.zone')).read())
|
||||
|
||||
@patch('subprocess.run')
|
||||
def test_no_stale_cleanup_when_primary_not_parent(self, _mock):
|
||||
"""When primary_domain is unrelated, no zone file is touched."""
|
||||
stale_records = [{'name': 'calendar', 'type': 'A', 'value': '10.0.0.1'}]
|
||||
self.nm.update_dns_zone('cell', stale_records)
|
||||
self.nm.update_split_horizon_zone(
|
||||
'pic2.pic.ngo', '172.20.0.2', primary_domain='cell'
|
||||
)
|
||||
# cell zone is untouched
|
||||
content = open(os.path.join(self.data_dir, 'dns', 'cell.zone')).read()
|
||||
self.assertIn('calendar', content)
|
||||
|
||||
|
||||
class TestApplyCellName(unittest.TestCase):
|
||||
"""Tests for apply_cell_name — hostname rename in primary DNS zone."""
|
||||
|
||||
def setUp(self):
|
||||
self.test_dir = tempfile.mkdtemp()
|
||||
self.data_dir = os.path.join(self.test_dir, 'data')
|
||||
self.config_dir = os.path.join(self.test_dir, 'config')
|
||||
os.makedirs(os.path.join(self.data_dir, 'dns'), exist_ok=True)
|
||||
os.makedirs(os.path.join(self.config_dir, 'dns'), exist_ok=True)
|
||||
self.nm = NetworkManager(self.data_dir, self.config_dir)
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.test_dir)
|
||||
|
||||
def _write_zone(self, name: str, content: str):
|
||||
path = os.path.join(self.data_dir, 'dns', f'{name}.zone')
|
||||
with open(path, 'w') as f:
|
||||
f.write(content)
|
||||
return path
|
||||
|
||||
@patch('subprocess.run')
|
||||
def test_renames_hostname_in_primary_zone(self, _mock):
|
||||
"""Old cell name is replaced with new name in the primary zone."""
|
||||
self._write_zone('cell', (
|
||||
'$ORIGIN cell.\n'
|
||||
'@ 300 IN SOA ns1 admin 1 3600 900 86400 300\n'
|
||||
'@ 300 IN NS ns1\n'
|
||||
'oldname 300 IN A 172.20.0.2\n'
|
||||
'api 300 IN A 172.20.0.10\n'
|
||||
))
|
||||
self.nm.apply_cell_name('oldname', 'newname', reload=False)
|
||||
content = open(os.path.join(self.data_dir, 'dns', 'cell.zone')).read()
|
||||
self.assertIn('newname', content)
|
||||
self.assertNotIn('oldname', content)
|
||||
|
||||
@patch('subprocess.run')
|
||||
def test_does_not_corrupt_split_horizon_zone(self, _mock):
|
||||
"""A multi-label DDNS zone (e.g. pic2.pic.ngo.zone) must not be touched."""
|
||||
sh_path = self._write_zone('pic2.pic.ngo', (
|
||||
'$ORIGIN pic2.pic.ngo.\n'
|
||||
'@ 300 IN SOA ns1 admin 1 3600 900 86400 300\n'
|
||||
'@ 300 IN NS ns1\n'
|
||||
'@ 300 IN A 172.20.0.2\n'
|
||||
'* 300 IN A 172.20.0.2\n'
|
||||
))
|
||||
self._write_zone('cell', (
|
||||
'$ORIGIN cell.\n'
|
||||
'@ 300 IN SOA ns1 admin 1 3600 900 86400 300\n'
|
||||
'oldname 300 IN A 172.20.0.2\n'
|
||||
))
|
||||
self.nm.apply_cell_name('oldname', 'newname', reload=False)
|
||||
# Split-horizon zone must be unchanged (wildcard not renamed)
|
||||
sh_content = open(sh_path).read()
|
||||
self.assertNotIn('newname', sh_content)
|
||||
self.assertIn('* 300 IN A 172.20.0.2', sh_content)
|
||||
|
||||
@patch('subprocess.run')
|
||||
def test_wildcard_not_treated_as_hostname(self, _mock):
|
||||
"""Wildcard record in a zone must never be detected as the cell hostname."""
|
||||
zone_path = self._write_zone('cell', (
|
||||
'$ORIGIN cell.\n'
|
||||
'@ 300 IN SOA ns1 admin 1 3600 900 86400 300\n'
|
||||
'@ 300 IN A 172.20.0.2\n'
|
||||
'* 300 IN A 172.20.0.2\n'
|
||||
))
|
||||
self.nm.apply_cell_name('', 'newname', reload=False)
|
||||
content = open(zone_path).read()
|
||||
# Wildcard must remain; 'newname' must not appear
|
||||
self.assertIn('* 300 IN A', content)
|
||||
self.assertNotIn('newname', content)
|
||||
|
||||
@patch('subprocess.run')
|
||||
def test_skips_zone_with_local_in_name(self, _mock):
|
||||
"""Zones with 'local' in the filename are ignored."""
|
||||
local_path = self._write_zone('home.local', (
|
||||
'$ORIGIN home.local.\n'
|
||||
'oldname 300 IN A 172.20.0.2\n'
|
||||
))
|
||||
self.nm.apply_cell_name('oldname', 'newname', reload=False)
|
||||
content = open(local_path).read()
|
||||
self.assertIn('oldname', content)
|
||||
self.assertNotIn('newname', content)
|
||||
|
||||
|
||||
class TestUpdateSplitHorizonZoneStaleCleanup(unittest.TestCase):
|
||||
"""Tests for stale split-horizon zone deletion in update_split_horizon_zone."""
|
||||
|
||||
def setUp(self):
|
||||
self.test_dir = tempfile.mkdtemp()
|
||||
self.data_dir = os.path.join(self.test_dir, 'data')
|
||||
self.config_dir = os.path.join(self.test_dir, 'config')
|
||||
os.makedirs(os.path.join(self.data_dir, 'dns'), exist_ok=True)
|
||||
os.makedirs(os.path.join(self.config_dir, 'dns'), exist_ok=True)
|
||||
self.nm = NetworkManager(self.data_dir, self.config_dir)
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.test_dir)
|
||||
|
||||
@patch('subprocess.run')
|
||||
def test_deletes_old_cell_zone_same_tld(self, _mock):
|
||||
"""When renaming pic3.pic.ngo → pic2.pic.ngo the old zone file is removed."""
|
||||
old_zone = os.path.join(self.data_dir, 'dns', 'pic3.pic.ngo.zone')
|
||||
with open(old_zone, 'w') as f:
|
||||
f.write('@ 300 IN A 172.20.0.2\n')
|
||||
self.nm.update_split_horizon_zone('pic2.pic.ngo', '172.20.0.2')
|
||||
self.assertFalse(os.path.exists(old_zone), 'stale pic3.pic.ngo.zone should be deleted')
|
||||
new_zone = os.path.join(self.data_dir, 'dns', 'pic2.pic.ngo.zone')
|
||||
self.assertTrue(os.path.exists(new_zone))
|
||||
|
||||
@patch('subprocess.run')
|
||||
def test_keeps_zone_for_different_tld(self, _mock):
|
||||
"""Zone files under a different TLD are not deleted."""
|
||||
other_zone = os.path.join(self.data_dir, 'dns', 'myhost.example.com.zone')
|
||||
with open(other_zone, 'w') as f:
|
||||
f.write('@ 300 IN A 1.2.3.4\n')
|
||||
self.nm.update_split_horizon_zone('pic2.pic.ngo', '172.20.0.2')
|
||||
self.assertTrue(os.path.exists(other_zone), 'unrelated zone must not be deleted')
|
||||
|
||||
@patch('subprocess.run')
|
||||
def test_keeps_current_effective_zone(self, _mock):
|
||||
"""The current effective_domain zone file is never deleted."""
|
||||
self.nm.update_split_horizon_zone('pic2.pic.ngo', '172.20.0.2')
|
||||
current_zone = os.path.join(self.data_dir, 'dns', 'pic2.pic.ngo.zone')
|
||||
self.assertTrue(os.path.exists(current_zone))
|
||||
|
||||
|
||||
class TestGetWgServerIp(unittest.TestCase):
|
||||
"""_get_wg_server_ip must read from wg0.conf and fall back to 10.0.0.1.
|
||||
|
||||
Regression guard: _bootstrap_dns used to pass 172.20.0.2 (Docker bridge IP)
|
||||
to update_split_horizon_zone. WireGuard peers cannot reach that IP; the zone
|
||||
must use the WireGuard server IP (e.g. 10.0.0.1) so VPN clients can reach Caddy.
|
||||
"""
|
||||
|
||||
def setUp(self):
|
||||
self.test_dir = tempfile.mkdtemp()
|
||||
self.data_dir = os.path.join(self.test_dir, 'data')
|
||||
self.config_dir = os.path.join(self.test_dir, 'config')
|
||||
os.makedirs(os.path.join(self.data_dir, 'dns'), exist_ok=True)
|
||||
os.makedirs(os.path.join(self.config_dir, 'dns'), exist_ok=True)
|
||||
self.nm = NetworkManager(self.data_dir, self.config_dir)
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.test_dir)
|
||||
|
||||
def _write_wg_conf(self, address: str) -> None:
|
||||
wg_dir = os.path.join(self.config_dir, 'wireguard', 'wg_confs')
|
||||
os.makedirs(wg_dir, exist_ok=True)
|
||||
with open(os.path.join(wg_dir, 'wg0.conf'), 'w') as f:
|
||||
f.write(f'[Interface]\nAddress = {address}\nListenPort = 51820\n')
|
||||
|
||||
def test_reads_address_from_wg0_conf(self):
|
||||
self._write_wg_conf('10.0.0.1/24')
|
||||
self.assertEqual(self.nm._get_wg_server_ip(), '10.0.0.1')
|
||||
|
||||
def test_reads_non_default_address(self):
|
||||
self._write_wg_conf('10.8.0.1/16')
|
||||
self.assertEqual(self.nm._get_wg_server_ip(), '10.8.0.1')
|
||||
|
||||
def test_falls_back_to_10_0_0_1_when_conf_missing(self):
|
||||
self.assertEqual(self.nm._get_wg_server_ip(), '10.0.0.1')
|
||||
|
||||
def test_split_horizon_zone_uses_wg_ip_not_docker_bridge(self):
|
||||
"""update_split_horizon_zone called with WG IP writes that IP in zone file.
|
||||
|
||||
This is the correct call pattern from _bootstrap_dns: pass the WireGuard
|
||||
server IP, not 172.20.0.x (Docker bridge IP unreachable from VPN peers).
|
||||
"""
|
||||
self._write_wg_conf('10.0.0.1/24')
|
||||
wg_ip = self.nm._get_wg_server_ip()
|
||||
self.assertEqual(wg_ip, '10.0.0.1',
|
||||
'WireGuard IP must be read from wg0.conf, not be a Docker bridge address')
|
||||
with patch('subprocess.run'):
|
||||
self.nm.update_split_horizon_zone('pic1.pic.ngo', wg_ip)
|
||||
zone_path = os.path.join(self.data_dir, 'dns', 'pic1.pic.ngo.zone')
|
||||
content = open(zone_path).read()
|
||||
self.assertIn('10.0.0.1', content)
|
||||
self.assertNotIn('172.20.0', content,
|
||||
'Zone must not contain Docker bridge IP — VPN peers cannot reach it')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,929 @@
|
||||
"""
|
||||
Tests for the optional-services feature: email/calendar/files moving from
|
||||
always-on builtins to installable store services.
|
||||
|
||||
Covers:
|
||||
1. ServiceRegistry.list_active() — zero installed, partial, full
|
||||
2. ServiceRegistry.get_caddy_routes() / get_service_subdomains() with list_active()
|
||||
3. ServiceRegistry.get() returns None for catalog-only (not installed) entries
|
||||
4. ServiceStoreManager.install() happy path, idempotency, fetch failure, compose failure
|
||||
5. ServiceStoreManager.uninstall() (remove()) happy path and not-installed error
|
||||
6. CaddyManager._build_registry_service_routes() with empty list_active()
|
||||
7. GET /api/services/active endpoint
|
||||
8. migrate_legacy_containers(): writes install records, idempotent on second call
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch, call
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent / 'api'))
|
||||
|
||||
from service_registry import ServiceRegistry
|
||||
from service_store_manager import ServiceStoreManager
|
||||
from caddy_manager import CaddyManager
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Shared manifest helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _store_manifest(service_id, subdomain=None, backend=None):
|
||||
"""Minimal valid store manifest for use in installed-services records."""
|
||||
m = {
|
||||
'id': service_id,
|
||||
'name': service_id.capitalize(),
|
||||
'kind': 'store',
|
||||
'capabilities': {
|
||||
'has_subdomain': bool(subdomain),
|
||||
'has_accounts': True,
|
||||
'has_admin_config': False,
|
||||
'has_storage': True,
|
||||
'has_egress': False,
|
||||
'has_api_hooks': False,
|
||||
},
|
||||
'config_schema': {},
|
||||
}
|
||||
if subdomain:
|
||||
m['subdomain'] = subdomain
|
||||
if backend:
|
||||
m['backend'] = backend
|
||||
if subdomain and backend:
|
||||
m['extra_subdomains'] = []
|
||||
m['extra_backends'] = {}
|
||||
return m
|
||||
|
||||
|
||||
_FIXTURE_DIGEST = 'a' * 64
|
||||
|
||||
|
||||
def _ssm_manifest(service_id='myapp', **overrides):
|
||||
"""Minimal manifest that passes ServiceStoreManager._validate_manifest."""
|
||||
m = {
|
||||
'id': service_id,
|
||||
'name': 'My App',
|
||||
'version': '1.0.0',
|
||||
'author': 'Test Author',
|
||||
'image': f'git.pic.ngo/roof/{service_id}@sha256:{_FIXTURE_DIGEST}',
|
||||
'container_name': f'cell-{service_id}',
|
||||
}
|
||||
m.update(overrides)
|
||||
return m
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 1. ServiceRegistry.list_active()
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestServiceRegistryListActive(unittest.TestCase):
|
||||
"""
|
||||
list_active() must return only services that appear in get_installed_services().
|
||||
When builtins are removed from the filesystem, only installed records count.
|
||||
"""
|
||||
|
||||
def _make_registry(self, installed=None):
|
||||
cm = MagicMock()
|
||||
cm.configs = {}
|
||||
cm.get_installed_services.return_value = installed or {}
|
||||
return ServiceRegistry(cm)
|
||||
|
||||
def test_list_active_zero_installed_returns_empty(self):
|
||||
"""With no installed records, list_active() is empty."""
|
||||
reg = self._make_registry(installed={})
|
||||
result = reg.list_active()
|
||||
self.assertEqual(result, [])
|
||||
|
||||
def test_list_active_one_installed_returns_only_that_service(self):
|
||||
"""Email installed, calendar not: only email appears in list_active()."""
|
||||
email_manifest = _store_manifest('email', subdomain='mail', backend='cell-rainloop:8888')
|
||||
installed = {
|
||||
'email': {'manifest': email_manifest},
|
||||
}
|
||||
reg = self._make_registry(installed=installed)
|
||||
result = reg.list_active()
|
||||
ids = [s['id'] for s in result]
|
||||
self.assertIn('email', ids)
|
||||
self.assertNotIn('calendar', ids)
|
||||
self.assertNotIn('files', ids)
|
||||
|
||||
def test_list_active_multiple_installed_returns_all(self):
|
||||
"""All three installed services appear in list_active()."""
|
||||
installed = {
|
||||
'email': {'manifest': _store_manifest('email', 'mail', 'cell-rainloop:8888')},
|
||||
'calendar': {'manifest': _store_manifest('calendar', 'calendar', 'cell-radicale:5232')},
|
||||
'files': {'manifest': _store_manifest('files', 'files', 'cell-filegator:8080')},
|
||||
}
|
||||
reg = self._make_registry(installed=installed)
|
||||
result = reg.list_active()
|
||||
ids = {s['id'] for s in result}
|
||||
self.assertEqual(ids, {'email', 'calendar', 'files'})
|
||||
|
||||
def test_list_active_each_entry_has_config_key(self):
|
||||
"""Each active service entry must carry the merged 'config' key."""
|
||||
installed = {
|
||||
'calendar': {'manifest': _store_manifest('calendar', 'calendar', 'cell-radicale:5232')},
|
||||
}
|
||||
reg = self._make_registry(installed=installed)
|
||||
result = reg.list_active()
|
||||
for svc in result:
|
||||
self.assertIn('config', svc, f'{svc["id"]} is missing the config key')
|
||||
|
||||
def test_list_active_record_without_manifest_skipped(self):
|
||||
"""An installed record with no manifest key must not appear (no KeyError either)."""
|
||||
installed = {
|
||||
'broken': {}, # no 'manifest' key at all
|
||||
}
|
||||
reg = self._make_registry(installed=installed)
|
||||
result = reg.list_active()
|
||||
self.assertEqual(result, [])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 2. ServiceRegistry.get_caddy_routes() only returns active services
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestServiceRegistryGetCaddyRoutesActiveOnly(unittest.TestCase):
|
||||
"""
|
||||
After the migration get_caddy_routes() must delegate to list_active(),
|
||||
not list_all(). This test class validates the behaviour that the
|
||||
implementation will need to satisfy — it patches list_active() on the
|
||||
registry so the tests don't depend on whether list_active() is already
|
||||
implemented or is still list_all().
|
||||
"""
|
||||
|
||||
def _make_registry(self, active_services):
|
||||
cm = MagicMock()
|
||||
cm.configs = {}
|
||||
cm.get_installed_services.return_value = {}
|
||||
reg = ServiceRegistry(cm)
|
||||
# Point get_caddy_routes' iteration at the active list only.
|
||||
# We do this by patching list_all to return only active services,
|
||||
# which mirrors the post-migration behaviour of list_all == list_active.
|
||||
reg.list_all = MagicMock(return_value=active_services)
|
||||
return reg
|
||||
|
||||
def test_no_active_services_produces_no_routes(self):
|
||||
"""When list_active returns empty, get_caddy_routes must return []."""
|
||||
reg = self._make_registry([])
|
||||
routes = reg.get_caddy_routes()
|
||||
self.assertEqual(routes, [])
|
||||
|
||||
def test_email_active_calendar_not_only_email_in_routes(self):
|
||||
"""Email installed; calendar and files not: only email route returned."""
|
||||
email_svc = {
|
||||
**_store_manifest('email', 'mail', 'cell-rainloop:8888'),
|
||||
'extra_subdomains': ['webmail'],
|
||||
'extra_backends': {},
|
||||
'config': {},
|
||||
}
|
||||
reg = self._make_registry([email_svc])
|
||||
routes = reg.get_caddy_routes()
|
||||
service_ids = [r['service_id'] for r in routes]
|
||||
self.assertIn('email', service_ids)
|
||||
self.assertNotIn('calendar', service_ids)
|
||||
self.assertNotIn('files', service_ids)
|
||||
|
||||
def test_route_shape_is_correct(self):
|
||||
"""Each route dict must have the expected keys with correct values."""
|
||||
svc = {
|
||||
**_store_manifest('calendar', 'calendar', 'cell-radicale:5232'),
|
||||
'extra_subdomains': [],
|
||||
'extra_backends': {},
|
||||
'config': {},
|
||||
}
|
||||
reg = self._make_registry([svc])
|
||||
routes = reg.get_caddy_routes()
|
||||
self.assertEqual(len(routes), 1)
|
||||
r = routes[0]
|
||||
self.assertEqual(r['service_id'], 'calendar')
|
||||
self.assertEqual(r['subdomain'], 'calendar')
|
||||
self.assertEqual(r['backend'], 'cell-radicale:5232')
|
||||
self.assertIn('extra_subdomains', r)
|
||||
self.assertIn('extra_backends', r)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 3. ServiceRegistry.get_service_subdomains() active services only
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestGetServiceSubdomainsActiveOnly(unittest.TestCase):
|
||||
"""
|
||||
The network manager calls registry.get_caddy_routes() via _get_service_subdomains.
|
||||
This test verifies that after the migration, a registry with only calendar
|
||||
installed does not include 'mail' or 'files' subdomains in its route output.
|
||||
"""
|
||||
|
||||
def test_only_installed_subdomains_returned(self):
|
||||
cm = MagicMock()
|
||||
cm.configs = {}
|
||||
cm.get_installed_services.return_value = {}
|
||||
|
||||
calendar_svc = {
|
||||
**_store_manifest('calendar', 'calendar', 'cell-radicale:5232'),
|
||||
'extra_subdomains': [],
|
||||
'extra_backends': {},
|
||||
'config': {},
|
||||
}
|
||||
reg = ServiceRegistry(cm)
|
||||
reg.list_all = MagicMock(return_value=[calendar_svc])
|
||||
|
||||
routes = reg.get_caddy_routes()
|
||||
subdomains = [r['subdomain'] for r in routes]
|
||||
extra = [s for r in routes for s in (r.get('extra_subdomains') or [])]
|
||||
all_subs = set(subdomains) | set(extra)
|
||||
|
||||
self.assertIn('calendar', all_subs)
|
||||
self.assertNotIn('mail', all_subs)
|
||||
self.assertNotIn('webmail', all_subs)
|
||||
self.assertNotIn('files', all_subs)
|
||||
self.assertNotIn('webdav', all_subs)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 4. ServiceRegistry.get() returns None for catalog-only (not installed) entries
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestServiceRegistryGetNotInstalled(unittest.TestCase):
|
||||
"""
|
||||
Once builtins are removed from the filesystem, get('email') must return None
|
||||
unless the service is in get_installed_services().
|
||||
"""
|
||||
|
||||
def test_get_returns_none_when_not_installed(self):
|
||||
cm = MagicMock()
|
||||
cm.configs = {}
|
||||
cm.get_installed_services.return_value = {}
|
||||
reg = ServiceRegistry(cm)
|
||||
result = reg.get('email')
|
||||
self.assertIsNone(result)
|
||||
|
||||
def test_get_returns_none_for_calendar_when_not_installed(self):
|
||||
cm = MagicMock()
|
||||
cm.configs = {}
|
||||
cm.get_installed_services.return_value = {}
|
||||
reg = ServiceRegistry(cm)
|
||||
self.assertIsNone(reg.get('calendar'))
|
||||
|
||||
def test_get_returns_none_for_files_when_not_installed(self):
|
||||
cm = MagicMock()
|
||||
cm.configs = {}
|
||||
cm.get_installed_services.return_value = {}
|
||||
reg = ServiceRegistry(cm)
|
||||
self.assertIsNone(reg.get('files'))
|
||||
|
||||
def test_get_returns_service_when_installed(self):
|
||||
"""Once email is in installed records it must be returned by get()."""
|
||||
email_manifest = _store_manifest('email', 'mail', 'cell-rainloop:8888')
|
||||
cm = MagicMock()
|
||||
cm.configs = {}
|
||||
cm.get_installed_services.return_value = {
|
||||
'email': {'manifest': email_manifest},
|
||||
}
|
||||
reg = ServiceRegistry(cm)
|
||||
result = reg.get('email')
|
||||
self.assertIsNotNone(result)
|
||||
self.assertEqual(result['id'], 'email')
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 5. ServiceStoreManager.install() — new scenarios
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _make_ssm(tmp_dir, installed=None, identity=None):
|
||||
cm = MagicMock()
|
||||
cm.get_installed_services.return_value = installed or {}
|
||||
cm.get_identity.return_value = identity or {
|
||||
'ip_range': '172.20.0.0/16',
|
||||
'service_ips': {},
|
||||
}
|
||||
caddy = MagicMock()
|
||||
container = MagicMock()
|
||||
composer = MagicMock()
|
||||
composer._resolve_requires.return_value = None
|
||||
composer._resolve_dependents.return_value = []
|
||||
composer.install.return_value = {'ok': True}
|
||||
composer.remove.return_value = {'ok': True}
|
||||
mgr = ServiceStoreManager(
|
||||
config_manager=cm,
|
||||
caddy_manager=caddy,
|
||||
container_manager=container,
|
||||
data_dir=tmp_dir,
|
||||
config_dir=tmp_dir,
|
||||
service_composer=composer,
|
||||
)
|
||||
return mgr
|
||||
|
||||
|
||||
class TestInstallHappyPath(unittest.TestCase):
|
||||
|
||||
def test_install_fetches_manifest_renders_compose_calls_docker_up(self):
|
||||
"""install() happy path: fetches manifest, calls service_composer.install, stores record."""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
mgr = _make_ssm(tmp)
|
||||
manifest = _ssm_manifest('email')
|
||||
mgr._fetch_manifest = MagicMock(return_value=manifest)
|
||||
mgr._fetch_template = MagicMock(return_value='version: "3"\nservices: {}\n')
|
||||
|
||||
result = mgr.install('email')
|
||||
|
||||
self.assertTrue(result['ok'])
|
||||
mgr._fetch_manifest.assert_called_once_with('email')
|
||||
mgr.config_manager.set_installed_service.assert_called_once()
|
||||
# service_composer.install must have been called
|
||||
mgr.service_composer.install.assert_called_once()
|
||||
|
||||
def test_install_persists_install_record_after_composer_install(self):
|
||||
"""Install record must be written after service_composer.install succeeds."""
|
||||
call_order = []
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
mgr = _make_ssm(tmp)
|
||||
manifest = _ssm_manifest('calendar')
|
||||
mgr._fetch_manifest = MagicMock(return_value=manifest)
|
||||
mgr._fetch_template = MagicMock(return_value='version: "3"\nservices: {}\n')
|
||||
mgr.config_manager.set_installed_service.side_effect = \
|
||||
lambda *a, **kw: call_order.append('set_installed')
|
||||
|
||||
def _composer_install(*a, **kw):
|
||||
call_order.append('composer_install')
|
||||
return {'ok': True}
|
||||
mgr.service_composer.install.side_effect = _composer_install
|
||||
mgr.install('calendar')
|
||||
|
||||
self.assertIn('composer_install', call_order)
|
||||
self.assertIn('set_installed', call_order)
|
||||
self.assertLess(
|
||||
call_order.index('composer_install'),
|
||||
call_order.index('set_installed'),
|
||||
'composer.install must be called before install record is persisted',
|
||||
)
|
||||
|
||||
|
||||
class TestInstallAlreadyInstalled(unittest.TestCase):
|
||||
|
||||
def test_install_already_installed_is_idempotent(self):
|
||||
"""Calling install() on an already-installed service returns ok=True, already_installed=True."""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
installed = {'email': {'id': 'email'}}
|
||||
mgr = _make_ssm(tmp, installed=installed)
|
||||
result = mgr.install('email')
|
||||
self.assertTrue(result['ok'])
|
||||
self.assertTrue(result.get('already_installed'))
|
||||
|
||||
def test_install_already_installed_does_not_fetch_manifest(self):
|
||||
"""No network call should be made when service is already installed."""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
installed = {'email': {'id': 'email'}}
|
||||
mgr = _make_ssm(tmp, installed=installed)
|
||||
mgr._fetch_manifest = MagicMock()
|
||||
mgr.install('email')
|
||||
mgr._fetch_manifest.assert_not_called()
|
||||
|
||||
def test_install_already_installed_does_not_write_config(self):
|
||||
"""set_installed_service must NOT be called for an idempotent re-install."""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
installed = {'calendar': {'id': 'calendar'}}
|
||||
mgr = _make_ssm(tmp, installed=installed)
|
||||
mgr.install('calendar')
|
||||
mgr.config_manager.set_installed_service.assert_not_called()
|
||||
|
||||
|
||||
class TestInstallManifestFetchFails(unittest.TestCase):
|
||||
|
||||
def test_install_fetch_failure_returns_error_with_message(self):
|
||||
"""A network error during manifest fetch must return ok=False with an error field."""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
mgr = _make_ssm(tmp)
|
||||
mgr._fetch_manifest = MagicMock(
|
||||
side_effect=Exception('connection refused')
|
||||
)
|
||||
result = mgr.install('email')
|
||||
self.assertFalse(result['ok'])
|
||||
self.assertIn('error', result)
|
||||
self.assertIn('fetch', result['error'].lower())
|
||||
|
||||
def test_install_fetch_failure_leaves_no_install_record(self):
|
||||
"""No install record must be written when the manifest fetch fails."""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
mgr = _make_ssm(tmp)
|
||||
mgr._fetch_manifest = MagicMock(side_effect=Exception('timeout'))
|
||||
mgr.install('email')
|
||||
mgr.config_manager.set_installed_service.assert_not_called()
|
||||
|
||||
def test_install_http_404_leaves_no_install_record(self):
|
||||
"""HTTP 404 from the manifest endpoint must not leave a partial install."""
|
||||
import requests as _requests
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
mgr = _make_ssm(tmp)
|
||||
response_404 = MagicMock()
|
||||
response_404.raise_for_status.side_effect = \
|
||||
_requests.HTTPError('404 Not Found')
|
||||
with patch('service_store_manager.requests.get', return_value=response_404):
|
||||
result = mgr.install('nonexistent-service')
|
||||
self.assertFalse(result['ok'])
|
||||
mgr.config_manager.set_installed_service.assert_not_called()
|
||||
|
||||
def test_install_invalid_manifest_does_not_write_record(self):
|
||||
"""Manifest validation failure must prevent any install record from being written."""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
mgr = _make_ssm(tmp)
|
||||
bad_manifest = {
|
||||
'id': 'email',
|
||||
# missing name, version, author, container_name; bad image
|
||||
'image': 'docker.io/bad-actor/email:latest',
|
||||
}
|
||||
mgr._fetch_manifest = MagicMock(return_value=bad_manifest)
|
||||
result = mgr.install('email')
|
||||
self.assertFalse(result['ok'])
|
||||
self.assertIn('errors', result)
|
||||
mgr.config_manager.set_installed_service.assert_not_called()
|
||||
|
||||
|
||||
class TestInstallComposeUpFails(unittest.TestCase):
|
||||
"""
|
||||
In the new architecture, a compose failure from service_composer.install returns
|
||||
ok=False immediately — the install record is NOT written when compose fails.
|
||||
"""
|
||||
|
||||
def test_install_compose_failure_returns_error(self):
|
||||
"""A failure from service_composer.install must return ok=False."""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
mgr = _make_ssm(tmp)
|
||||
manifest = _ssm_manifest('email')
|
||||
mgr._fetch_manifest = MagicMock(return_value=manifest)
|
||||
mgr._fetch_template = MagicMock(return_value='version: "3"\nservices: {}\n')
|
||||
mgr.service_composer.install.return_value = {'ok': False, 'stderr': 'image pull failed'}
|
||||
result = mgr.install('email')
|
||||
self.assertFalse(result['ok'])
|
||||
self.assertIn('error', result)
|
||||
|
||||
def test_install_record_not_written_when_compose_fails(self):
|
||||
"""Install record must NOT be written when service_composer.install fails."""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
mgr = _make_ssm(tmp)
|
||||
manifest = _ssm_manifest('email')
|
||||
mgr._fetch_manifest = MagicMock(return_value=manifest)
|
||||
mgr._fetch_template = MagicMock(return_value='version: "3"\nservices: {}\n')
|
||||
mgr.service_composer.install.return_value = {'ok': False, 'stderr': 'pull failed'}
|
||||
mgr.install('email')
|
||||
mgr.config_manager.set_installed_service.assert_not_called()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 6. ServiceStoreManager.uninstall() (remove())
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestUninstallHappyPath(unittest.TestCase):
|
||||
|
||||
def _make_mgr_with_email(self, tmp):
|
||||
record = {
|
||||
'id': 'email',
|
||||
'manifest': {
|
||||
'image': 'git.pic.ngo/roof/email:1.0',
|
||||
},
|
||||
}
|
||||
installed = {'email': record}
|
||||
mgr = _make_ssm(tmp, installed=installed)
|
||||
mgr.config_manager.remove_installed_service = MagicMock()
|
||||
return mgr
|
||||
|
||||
def test_uninstall_happy_path_returns_ok_true(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
mgr = self._make_mgr_with_email(tmp)
|
||||
result = mgr.remove('email')
|
||||
self.assertTrue(result['ok'])
|
||||
|
||||
def test_uninstall_removes_install_record(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
mgr = self._make_mgr_with_email(tmp)
|
||||
mgr.remove('email')
|
||||
mgr.config_manager.remove_installed_service.assert_called_once_with('email')
|
||||
|
||||
def test_uninstall_calls_service_composer_remove(self):
|
||||
"""New architecture: composer.remove() is called instead of subprocess directly."""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
mgr = self._make_mgr_with_email(tmp)
|
||||
mgr.remove('email')
|
||||
mgr.service_composer.remove.assert_called_once_with('email', purge_data=False)
|
||||
|
||||
def test_uninstall_regenerates_caddyfile(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
mgr = self._make_mgr_with_email(tmp)
|
||||
mgr.remove('email')
|
||||
mgr.caddy_manager.regenerate_with_installed.assert_called()
|
||||
|
||||
|
||||
class TestUninstallNotInstalled(unittest.TestCase):
|
||||
|
||||
def test_uninstall_service_not_installed_returns_error(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
mgr = _make_ssm(tmp, installed={})
|
||||
result = mgr.remove('email')
|
||||
self.assertFalse(result['ok'])
|
||||
self.assertIn('error', result)
|
||||
self.assertIn('not installed', result['error'].lower())
|
||||
|
||||
def test_uninstall_nonexistent_service_does_not_call_composer(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
mgr = _make_ssm(tmp, installed={})
|
||||
mgr.remove('email')
|
||||
mgr.service_composer.remove.assert_not_called()
|
||||
|
||||
def test_uninstall_nonexistent_service_does_not_remove_config(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
mgr = _make_ssm(tmp, installed={})
|
||||
mgr.config_manager.remove_installed_service = MagicMock()
|
||||
mgr.remove('email')
|
||||
mgr.config_manager.remove_installed_service.assert_not_called()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 7. CaddyManager._build_registry_service_routes() with empty registry
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestCaddyManagerEmptyActiveRegistry(unittest.TestCase):
|
||||
"""
|
||||
When the registry returns no active routes (empty list_active()), the
|
||||
registry-driven path produces only the @api block — no service matcher
|
||||
blocks for calendar/mail/files/webdav.
|
||||
|
||||
Phase 2: builtins removed, so there is no hardcoded fallback. An empty
|
||||
registry means no service routes at all (except the always-present api block).
|
||||
"""
|
||||
|
||||
def _mgr_with_empty_registry(self):
|
||||
cm = MagicMock()
|
||||
cm.get_identity.return_value = {}
|
||||
reg = MagicMock()
|
||||
reg.get_caddy_routes.return_value = [] # no active services
|
||||
return CaddyManager(config_manager=cm, service_registry=reg)
|
||||
|
||||
def test_empty_active_list_produces_no_service_matcher_blocks(self):
|
||||
"""Zero active services → no @calendar, @mail, @files, @webdav matchers.
|
||||
|
||||
Phase 2: builtins are gone so an empty registry produces only the @api block.
|
||||
"""
|
||||
mgr = self._mgr_with_empty_registry()
|
||||
result = mgr._build_registry_service_routes('mycell.pic.ngo')
|
||||
self.assertIn('@api host api.mycell.pic.ngo', result)
|
||||
self.assertNotIn('@calendar', result)
|
||||
self.assertNotIn('@mail', result)
|
||||
self.assertNotIn('@files', result)
|
||||
self.assertNotIn('@webdav', result)
|
||||
|
||||
def test_empty_registry_no_store_service_blocks_injected(self):
|
||||
"""An empty active list must not inject any store-service-specific matchers."""
|
||||
mgr = self._mgr_with_empty_registry()
|
||||
result = mgr._build_registry_service_routes('mycell.pic.ngo')
|
||||
# No store service names should appear that don't come from core services
|
||||
self.assertNotIn('@chat', result)
|
||||
self.assertNotIn('@nextcloud', result)
|
||||
self.assertNotIn('@wiki', result)
|
||||
|
||||
def test_registry_with_only_email_installed_produces_only_email_block(self):
|
||||
"""When only email is active the Caddyfile must have @mail but not @calendar or @files."""
|
||||
cm = MagicMock()
|
||||
cm.get_identity.return_value = {}
|
||||
reg = MagicMock()
|
||||
reg.get_caddy_routes.return_value = [
|
||||
{
|
||||
'service_id': 'email',
|
||||
'subdomain': 'mail',
|
||||
'backend': 'cell-rainloop:8888',
|
||||
'extra_subdomains': ['webmail'],
|
||||
'extra_backends': {},
|
||||
}
|
||||
]
|
||||
mgr = CaddyManager(config_manager=cm, service_registry=reg)
|
||||
result = mgr._build_registry_service_routes('mycell.pic.ngo')
|
||||
|
||||
self.assertIn('@mail host mail.mycell.pic.ngo', result)
|
||||
self.assertNotIn('@calendar host', result)
|
||||
self.assertNotIn('@files host', result)
|
||||
self.assertNotIn('@webdav host', result)
|
||||
# api block is always appended
|
||||
self.assertIn('@api host api.mycell.pic.ngo', result)
|
||||
|
||||
def test_caddyfile_with_no_active_services_still_has_api_and_webui(self):
|
||||
"""Even with no installed services the api and webui routes must appear."""
|
||||
mgr = self._mgr_with_empty_registry()
|
||||
identity = {'cell_name': 'alpha', 'domain_mode': 'pic_ngo'}
|
||||
caddyfile = mgr.generate_caddyfile(identity, [])
|
||||
self.assertIn('cell-api:3000', caddyfile)
|
||||
self.assertIn('cell-webui:80', caddyfile)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 8. GET /api/services/active endpoint
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestServicesActiveEndpoint(unittest.TestCase):
|
||||
"""
|
||||
Tests for GET /api/services/active.
|
||||
|
||||
The endpoint does not exist yet — these tests define the required contract
|
||||
so they can be run once the endpoint is implemented. They are marked with
|
||||
a skip decorator that references the missing route; remove the skip when
|
||||
the endpoint is added to api/routes/services.py.
|
||||
"""
|
||||
|
||||
def setUp(self):
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent / 'api'))
|
||||
from app import app
|
||||
app.config['TESTING'] = True
|
||||
self.client = app.test_client()
|
||||
|
||||
def _mock_registry(self, active_services):
|
||||
"""Patch app.service_registry.list_active to return active_services."""
|
||||
reg = MagicMock()
|
||||
reg.list_active = MagicMock(return_value=active_services)
|
||||
reg.list_all = MagicMock(return_value=active_services) # fallback
|
||||
return reg
|
||||
|
||||
@unittest.skip(
|
||||
'GET /api/services/active does not exist yet; add to routes/services.py then unskip'
|
||||
)
|
||||
def test_active_endpoint_returns_200(self):
|
||||
import app as app_module
|
||||
with patch.object(app_module, 'service_registry',
|
||||
self._mock_registry([])):
|
||||
resp = self.client.get('/api/services/active')
|
||||
self.assertEqual(resp.status_code, 200)
|
||||
|
||||
@unittest.skip(
|
||||
'GET /api/services/active does not exist yet; add to routes/services.py then unskip'
|
||||
)
|
||||
def test_active_endpoint_returns_empty_list_when_nothing_installed(self):
|
||||
import app as app_module
|
||||
with patch.object(app_module, 'service_registry',
|
||||
self._mock_registry([])):
|
||||
resp = self.client.get('/api/services/active')
|
||||
data = json.loads(resp.data)
|
||||
self.assertIn('services', data)
|
||||
self.assertEqual(data['services'], [])
|
||||
|
||||
@unittest.skip(
|
||||
'GET /api/services/active does not exist yet; add to routes/services.py then unskip'
|
||||
)
|
||||
def test_active_endpoint_only_returns_installed_services(self):
|
||||
email_svc = {**_store_manifest('email', 'mail', 'cell-rainloop:8888'), 'config': {}}
|
||||
import app as app_module
|
||||
with patch.object(app_module, 'service_registry',
|
||||
self._mock_registry([email_svc])):
|
||||
resp = self.client.get('/api/services/active')
|
||||
data = json.loads(resp.data)
|
||||
ids = [s['id'] for s in data['services']]
|
||||
self.assertIn('email', ids)
|
||||
self.assertNotIn('calendar', ids)
|
||||
self.assertNotIn('files', ids)
|
||||
|
||||
def test_catalog_endpoint_exists_and_returns_200(self):
|
||||
"""Smoke-test the existing /api/services/catalog endpoint for baseline health."""
|
||||
import app as app_module
|
||||
reg = MagicMock()
|
||||
reg.list_all.return_value = []
|
||||
with patch.object(app_module, 'service_registry', reg):
|
||||
resp = self.client.get('/api/services/catalog')
|
||||
self.assertEqual(resp.status_code, 200)
|
||||
data = json.loads(resp.data)
|
||||
self.assertIn('services', data)
|
||||
|
||||
def test_catalog_single_entry_returns_404_for_uninstalled(self):
|
||||
"""GET /api/services/catalog/<id> returns 404 when service is not installed."""
|
||||
import app as app_module
|
||||
reg = MagicMock()
|
||||
reg.get.return_value = None # simulates uninstalled
|
||||
with patch.object(app_module, 'service_registry', reg):
|
||||
resp = self.client.get('/api/services/catalog/email')
|
||||
self.assertEqual(resp.status_code, 404)
|
||||
data = json.loads(resp.data)
|
||||
self.assertIn('error', data)
|
||||
|
||||
def test_catalog_single_entry_returns_200_when_installed(self):
|
||||
"""GET /api/services/catalog/<id> returns 200 when service is installed."""
|
||||
import app as app_module
|
||||
email_svc = {**_store_manifest('email', 'mail', 'cell-rainloop:8888'), 'config': {}}
|
||||
reg = MagicMock()
|
||||
reg.get.return_value = email_svc
|
||||
with patch.object(app_module, 'service_registry', reg):
|
||||
resp = self.client.get('/api/services/catalog/email')
|
||||
self.assertEqual(resp.status_code, 200)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 9. migrate_legacy_containers()
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestMigrateLegacyContainers(unittest.TestCase):
|
||||
"""
|
||||
migrate_legacy_containers() is a new helper that should be called on startup
|
||||
to write install records for any of {email, calendar, files} whose containers
|
||||
are already running but have no install record yet (upgrade path).
|
||||
|
||||
The method does not exist yet; these tests define its required contract.
|
||||
When implemented, remove the @unittest.skip decorators.
|
||||
|
||||
Expected behaviour:
|
||||
- For each legacy service whose container is running and has no install
|
||||
record, call config_manager.set_installed_service with an appropriate
|
||||
record derived from the legacy manifest.
|
||||
- If the install record already exists, do not overwrite it (idempotent).
|
||||
- Calling migrate_legacy_containers() twice must produce the same number
|
||||
of set_installed_service calls as calling it once (idempotent on second call).
|
||||
"""
|
||||
|
||||
def _make_ssm_for_migration(self, tmp, running_containers, installed=None):
|
||||
"""
|
||||
Build a ServiceStoreManager whose container_manager mock reports
|
||||
the given running_containers list.
|
||||
"""
|
||||
cm = MagicMock()
|
||||
# First call: before migration. Second call (idempotency): after migration.
|
||||
installed_before = installed or {}
|
||||
installed_after = dict(installed_before)
|
||||
# Simulate that after migration the records are present
|
||||
cm.get_installed_services.side_effect = [installed_before, installed_after]
|
||||
cm.get_identity.return_value = {'ip_range': '172.20.0.0/16', 'service_ips': {}}
|
||||
|
||||
container_mgr = MagicMock()
|
||||
container_mgr.list_containers.return_value = running_containers
|
||||
|
||||
caddy = MagicMock()
|
||||
mgr = ServiceStoreManager(
|
||||
config_manager=cm,
|
||||
caddy_manager=caddy,
|
||||
container_manager=container_mgr,
|
||||
data_dir=tmp,
|
||||
config_dir=tmp,
|
||||
)
|
||||
mgr.compose_override = os.path.join(tmp, 'docker-compose.services.yml')
|
||||
return mgr
|
||||
|
||||
@unittest.skip(
|
||||
'migrate_legacy_containers() not yet implemented; '
|
||||
'add to ServiceStoreManager then unskip'
|
||||
)
|
||||
def test_migrate_writes_record_for_running_email_container(self):
|
||||
"""A running cell-mail container with no install record gets an install record written."""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
mgr = self._make_ssm_for_migration(
|
||||
tmp,
|
||||
running_containers=[
|
||||
{'name': 'cell-mail', 'status': 'running'},
|
||||
],
|
||||
installed={},
|
||||
)
|
||||
with patch('service_registry._BUILTINS_DIR',
|
||||
str(Path(__file__).parent.parent / 'api' / 'services' / 'builtins')):
|
||||
mgr.migrate_legacy_containers()
|
||||
mgr.config_manager.set_installed_service.assert_called()
|
||||
call_service_ids = [
|
||||
c[0][0] for c in mgr.config_manager.set_installed_service.call_args_list
|
||||
]
|
||||
self.assertIn('email', call_service_ids)
|
||||
|
||||
@unittest.skip(
|
||||
'migrate_legacy_containers() not yet implemented; '
|
||||
'add to ServiceStoreManager then unskip'
|
||||
)
|
||||
def test_migrate_does_not_overwrite_existing_record(self):
|
||||
"""If email already has an install record, migrate must not overwrite it."""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
existing_record = {'id': 'email', 'installed_at': '2026-01-01T00:00:00'}
|
||||
mgr = self._make_ssm_for_migration(
|
||||
tmp,
|
||||
running_containers=[{'name': 'cell-mail', 'status': 'running'}],
|
||||
installed={'email': existing_record},
|
||||
)
|
||||
with patch('service_registry._BUILTINS_DIR',
|
||||
str(Path(__file__).parent.parent / 'api' / 'services' / 'builtins')):
|
||||
mgr.migrate_legacy_containers()
|
||||
mgr.config_manager.set_installed_service.assert_not_called()
|
||||
|
||||
@unittest.skip(
|
||||
'migrate_legacy_containers() not yet implemented; '
|
||||
'add to ServiceStoreManager then unskip'
|
||||
)
|
||||
def test_migrate_is_idempotent_on_second_call(self):
|
||||
"""Calling migrate twice must not produce more set_installed_service calls than once."""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
mgr = self._make_ssm_for_migration(
|
||||
tmp,
|
||||
running_containers=[{'name': 'cell-mail', 'status': 'running'}],
|
||||
installed={},
|
||||
)
|
||||
# Simulate that after first migration the record is present
|
||||
# by making get_installed_services return {} first, then {'email': {...}}
|
||||
mgr.config_manager.get_installed_services.side_effect = [
|
||||
{}, # first call inside first migrate
|
||||
{'email': {}}, # second call inside second migrate
|
||||
]
|
||||
with patch('service_registry._BUILTINS_DIR',
|
||||
str(Path(__file__).parent.parent / 'api' / 'services' / 'builtins')):
|
||||
mgr.migrate_legacy_containers()
|
||||
first_call_count = mgr.config_manager.set_installed_service.call_count
|
||||
mgr.migrate_legacy_containers()
|
||||
second_call_count = mgr.config_manager.set_installed_service.call_count
|
||||
self.assertEqual(
|
||||
first_call_count, second_call_count,
|
||||
'Second migrate call must not write any additional install records',
|
||||
)
|
||||
|
||||
@unittest.skip(
|
||||
'migrate_legacy_containers() not yet implemented; '
|
||||
'add to ServiceStoreManager then unskip'
|
||||
)
|
||||
def test_migrate_only_migrates_known_legacy_services(self):
|
||||
"""Non-legacy containers (e.g. cell-caddy) must not receive install records."""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
mgr = self._make_ssm_for_migration(
|
||||
tmp,
|
||||
running_containers=[
|
||||
{'name': 'cell-caddy', 'status': 'running'},
|
||||
{'name': 'cell-coredns', 'status': 'running'},
|
||||
],
|
||||
installed={},
|
||||
)
|
||||
with patch('service_registry._BUILTINS_DIR',
|
||||
str(Path(__file__).parent.parent / 'api' / 'services' / 'builtins')):
|
||||
mgr.migrate_legacy_containers()
|
||||
mgr.config_manager.set_installed_service.assert_not_called()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 10. Phase 2 completion: verify builtins layer is fully removed
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestPhase2CompletionChecks(unittest.TestCase):
|
||||
"""
|
||||
Confirms that Phase 2 (builtins removal) is complete.
|
||||
|
||||
These tests verify the post-migration state: no builtins directory,
|
||||
no hardcoded fallbacks, and registry-only routing for all services.
|
||||
"""
|
||||
|
||||
def test_builtins_dir_does_not_exist(self):
|
||||
"""api/services/builtins/ must not exist after Phase 2."""
|
||||
import api.service_registry as sr_module
|
||||
self.assertFalse(hasattr(sr_module, '_BUILTINS_DIR'),
|
||||
'service_registry must not export _BUILTINS_DIR after Phase 2')
|
||||
|
||||
def test_list_all_empty_without_installed_services(self):
|
||||
"""list_all() returns [] when nothing is installed."""
|
||||
cm = MagicMock()
|
||||
cm.configs = {}
|
||||
cm.get_installed_services.return_value = {}
|
||||
reg = ServiceRegistry(cm)
|
||||
services = reg.list_all()
|
||||
ids = [s['id'] for s in services]
|
||||
self.assertNotIn('email', ids)
|
||||
self.assertNotIn('calendar', ids)
|
||||
self.assertNotIn('files', ids)
|
||||
self.assertEqual(ids, [])
|
||||
|
||||
def test_get_caddy_routes_empty_without_installed_services(self):
|
||||
"""get_caddy_routes() returns [] when nothing is installed."""
|
||||
cm = MagicMock()
|
||||
cm.configs = {}
|
||||
cm.get_installed_services.return_value = {}
|
||||
reg = ServiceRegistry(cm)
|
||||
routes = reg.get_caddy_routes()
|
||||
self.assertEqual(routes, [])
|
||||
|
||||
def test_backup_plan_empty_without_installed_services(self):
|
||||
"""get_backup_plan() returns [] when nothing is installed."""
|
||||
cm = MagicMock()
|
||||
cm.configs = {}
|
||||
cm.get_installed_services.return_value = {}
|
||||
reg = ServiceRegistry(cm)
|
||||
plan = reg.get_backup_plan()
|
||||
self.assertEqual(plan, [])
|
||||
|
||||
def test_get_returns_none_for_uninstalled_service(self):
|
||||
"""get('calendar') returns None when calendar is not installed."""
|
||||
cm = MagicMock()
|
||||
cm.configs = {'calendar': {}}
|
||||
cm.get_installed_services.return_value = {}
|
||||
reg = ServiceRegistry(cm)
|
||||
result = reg.get('calendar')
|
||||
self.assertIsNone(result)
|
||||
|
||||
def test_caddy_empty_registry_produces_only_api_block(self):
|
||||
"""Empty registry → no service matcher blocks (no hardcoded fallback)."""
|
||||
reg = MagicMock()
|
||||
reg.get_caddy_routes.return_value = []
|
||||
cm = MagicMock()
|
||||
cm.get_identity.return_value = {}
|
||||
mgr = CaddyManager(config_manager=cm, service_registry=reg)
|
||||
result = mgr._build_registry_service_routes('alpha.pic.ngo')
|
||||
self.assertIn('@api host api.alpha.pic.ngo', result)
|
||||
self.assertNotIn('@calendar', result)
|
||||
self.assertNotIn('@mail', result)
|
||||
self.assertNotIn('@files', result)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -500,35 +500,39 @@ class TestDNSZoneRecords:
|
||||
f"got {rec['value']}"
|
||||
)
|
||||
|
||||
def test_calendar_resolves_to_wg_server_ip(self):
|
||||
def test_service_records_absent_without_registry(self):
|
||||
"""Built-in services always get DNS records; optional services require a registry."""
|
||||
records = self._records()
|
||||
rec = next((r for r in records if r['name'] == 'calendar'), None)
|
||||
assert rec and rec['value'] == self._WG_SERVER_IP, \
|
||||
f"calendar.dev should resolve to WG server IP; got {rec}"
|
||||
names = {r['name'] for r in records}
|
||||
# Built-in services are always present
|
||||
for svc in ('calendar', 'files', 'mail', 'webdav'):
|
||||
assert svc in names, f'{svc} DNS record must always be generated'
|
||||
# Non-built-in names are only generated when a registry is wired
|
||||
assert 'webmail' not in names, \
|
||||
'webmail DNS record must not appear without a registry'
|
||||
|
||||
def test_files_resolves_to_wg_server_ip(self):
|
||||
records = self._records()
|
||||
rec = next((r for r in records if r['name'] == 'files'), None)
|
||||
assert rec and rec['value'] == self._WG_SERVER_IP, \
|
||||
f"files.dev should resolve to WG server IP; got {rec}"
|
||||
|
||||
def test_mail_resolves_to_wg_server_ip(self):
|
||||
records = self._records()
|
||||
rec = next((r for r in records if r['name'] == 'mail'), None)
|
||||
assert rec and rec['value'] == self._WG_SERVER_IP, \
|
||||
f"mail.dev should resolve to WG server IP; got {rec}"
|
||||
|
||||
def test_webmail_resolves_to_wg_server_ip(self):
|
||||
records = self._records()
|
||||
rec = next((r for r in records if r['name'] == 'webmail'), None)
|
||||
assert rec and rec['value'] == self._WG_SERVER_IP, \
|
||||
f"webmail.dev should resolve to WG server IP; got {rec}"
|
||||
|
||||
def test_webdav_resolves_to_wg_server_ip(self):
|
||||
records = self._records()
|
||||
rec = next((r for r in records if r['name'] == 'webdav'), None)
|
||||
assert rec and rec['value'] == self._WG_SERVER_IP, \
|
||||
f"webdav.dev should resolve to WG server IP; got {rec}"
|
||||
def test_service_records_present_with_registry(self):
|
||||
"""With a registry that provides calendar/mail/files, all resolve to WG IP."""
|
||||
from unittest.mock import MagicMock
|
||||
import network_manager as nm
|
||||
registry = MagicMock()
|
||||
registry.get_caddy_routes.return_value = [
|
||||
{'service_id': 'calendar', 'subdomain': 'calendar',
|
||||
'backend': 'cell-radicale:5232', 'extra_subdomains': [], 'extra_backends': {}},
|
||||
{'service_id': 'email', 'subdomain': 'mail',
|
||||
'backend': 'cell-rainloop:8888', 'extra_subdomains': ['webmail'], 'extra_backends': {}},
|
||||
{'service_id': 'files', 'subdomain': 'files',
|
||||
'backend': 'cell-filegator:8080', 'extra_subdomains': ['webdav'], 'extra_backends': {}},
|
||||
]
|
||||
mgr = nm.NetworkManager.__new__(nm.NetworkManager)
|
||||
mgr._service_registry = registry
|
||||
records = mgr._build_dns_records('pic0', '172.20.0.0/16')
|
||||
names = {r['name'] for r in records}
|
||||
for expected in ('calendar', 'mail', 'webmail', 'files', 'webdav'):
|
||||
assert expected in names, f'{expected} should be in DNS records with registry'
|
||||
for rec in records:
|
||||
assert rec['value'] == self._WG_SERVER_IP, \
|
||||
f"Record {rec['name']} should point to WG server IP"
|
||||
|
||||
def test_cell_name_resolves_to_wg_server_ip(self):
|
||||
records = self._records(cell_name='mypic')
|
||||
|
||||
@@ -372,6 +372,104 @@ def test_delete_nonexistent_peer_returns_gracefully(admin_client, mock_peer_regi
|
||||
assert r.status_code in (200, 404)
|
||||
|
||||
|
||||
# ── POST /api/peers — HTTP store service provisioning ────────────────────────
|
||||
|
||||
def test_create_peer_provisions_http_store_services(
|
||||
auth_mgr, mock_email_mgr, mock_calendar_mgr,
|
||||
mock_file_mgr, mock_wg_mgr, mock_peer_registry):
|
||||
"""When an installed store service has accounts.manager='http',
|
||||
account_manager.provision() must be called for the new peer."""
|
||||
app.config['TESTING'] = True
|
||||
app.config['SECRET_KEY'] = 'test-secret'
|
||||
|
||||
mock_am = MagicMock()
|
||||
mock_am.provision.return_value = {'password': 'generated'}
|
||||
mock_am.store_credentials = MagicMock()
|
||||
|
||||
mock_cfg = MagicMock()
|
||||
mock_cfg.get_installed_services.return_value = {'my-store-app': {}}
|
||||
|
||||
mock_sreg = MagicMock()
|
||||
mock_sreg.get.return_value = {'id': 'my-store-app', 'accounts': {'manager': 'http'}, 'backend': 'cell-my-store-app:8080'}
|
||||
|
||||
patches = [
|
||||
patch('app.auth_manager', auth_mgr),
|
||||
patch('app.email_manager', mock_email_mgr),
|
||||
patch('app.calendar_manager', mock_calendar_mgr),
|
||||
patch('app.file_manager', mock_file_mgr),
|
||||
patch('app.wireguard_manager', mock_wg_mgr),
|
||||
patch('app.peer_registry', mock_peer_registry),
|
||||
patch('app.firewall_manager'),
|
||||
patch('app.account_manager', mock_am),
|
||||
patch('app.config_manager', mock_cfg),
|
||||
patch('app.service_registry', mock_sreg),
|
||||
]
|
||||
try:
|
||||
import auth_routes
|
||||
patches.append(patch.object(auth_routes, 'auth_manager', auth_mgr, create=True))
|
||||
except (ImportError, AttributeError):
|
||||
pass
|
||||
|
||||
started = [p.start() for p in patches]
|
||||
try:
|
||||
with app.test_client() as client:
|
||||
r = _login(client)
|
||||
assert r.status_code == 200
|
||||
resp = _post_peer(client)
|
||||
assert resp.status_code == 201
|
||||
mock_am.provision.assert_called_once_with('my-store-app', 'alice')
|
||||
finally:
|
||||
for p in patches:
|
||||
p.stop()
|
||||
|
||||
|
||||
def test_create_peer_http_provision_failure_is_nonfatal(
|
||||
auth_mgr, mock_email_mgr, mock_calendar_mgr,
|
||||
mock_file_mgr, mock_wg_mgr, mock_peer_registry):
|
||||
"""HTTP account provisioning failure must not block peer creation."""
|
||||
app.config['TESTING'] = True
|
||||
app.config['SECRET_KEY'] = 'test-secret'
|
||||
|
||||
mock_am = MagicMock()
|
||||
mock_am.provision.side_effect = RuntimeError('service unavailable')
|
||||
mock_am.store_credentials = MagicMock()
|
||||
|
||||
mock_cfg = MagicMock()
|
||||
mock_cfg.get_installed_services.return_value = {'my-store-app': {}}
|
||||
|
||||
mock_sreg = MagicMock()
|
||||
mock_sreg.get.return_value = {'id': 'my-store-app', 'accounts': {'manager': 'http'}, 'backend': 'cell-my-store-app:8080'}
|
||||
|
||||
patches = [
|
||||
patch('app.auth_manager', auth_mgr),
|
||||
patch('app.email_manager', mock_email_mgr),
|
||||
patch('app.calendar_manager', mock_calendar_mgr),
|
||||
patch('app.file_manager', mock_file_mgr),
|
||||
patch('app.wireguard_manager', mock_wg_mgr),
|
||||
patch('app.peer_registry', mock_peer_registry),
|
||||
patch('app.firewall_manager'),
|
||||
patch('app.account_manager', mock_am),
|
||||
patch('app.config_manager', mock_cfg),
|
||||
patch('app.service_registry', mock_sreg),
|
||||
]
|
||||
try:
|
||||
import auth_routes
|
||||
patches.append(patch.object(auth_routes, 'auth_manager', auth_mgr, create=True))
|
||||
except (ImportError, AttributeError):
|
||||
pass
|
||||
|
||||
started = [p.start() for p in patches]
|
||||
try:
|
||||
with app.test_client() as client:
|
||||
r = _login(client)
|
||||
assert r.status_code == 200
|
||||
resp = _post_peer(client)
|
||||
assert resp.status_code == 201, 'HTTP provision failure must not block peer creation'
|
||||
finally:
|
||||
for p in patches:
|
||||
p.stop()
|
||||
|
||||
|
||||
# ── POST /api/peers — firewall rollback (A3) ──────────────────────────────────
|
||||
|
||||
def test_create_peer_rolls_back_firewall_on_dns_failure(
|
||||
@@ -417,3 +515,109 @@ def test_create_peer_rolls_back_firewall_on_dns_failure(
|
||||
finally:
|
||||
for p in patches:
|
||||
p.stop()
|
||||
|
||||
|
||||
# ── service_access webdav gating ──────────────────────────────────────────────
|
||||
|
||||
def _make_admin_client_with_installed(auth_mgr, mock_email_mgr, mock_calendar_mgr,
|
||||
mock_file_mgr, mock_wg_mgr, mock_peer_registry,
|
||||
installed_services):
|
||||
"""Return patch list for an admin client with get_installed_services pre-configured."""
|
||||
app.config['TESTING'] = True
|
||||
app.config['SECRET_KEY'] = 'test-secret'
|
||||
mock_cfg = MagicMock()
|
||||
mock_cfg.get_installed_services.return_value = installed_services
|
||||
|
||||
patches = [
|
||||
patch('app.auth_manager', auth_mgr),
|
||||
patch('app.config_manager', mock_cfg),
|
||||
patch('app.email_manager', mock_email_mgr),
|
||||
patch('app.calendar_manager', mock_calendar_mgr),
|
||||
patch('app.file_manager', mock_file_mgr),
|
||||
patch('app.wireguard_manager', mock_wg_mgr),
|
||||
patch('app.peer_registry', mock_peer_registry),
|
||||
patch('app.firewall_manager'),
|
||||
]
|
||||
try:
|
||||
import auth_routes
|
||||
patches.append(patch.object(auth_routes, 'auth_manager', auth_mgr, create=True))
|
||||
except (ImportError, AttributeError):
|
||||
pass
|
||||
return patches
|
||||
|
||||
|
||||
def test_webdav_not_offered_when_files_not_installed(
|
||||
auth_mgr, mock_email_mgr, mock_calendar_mgr,
|
||||
mock_file_mgr, mock_wg_mgr, mock_peer_registry):
|
||||
"""Requesting service_access=['webdav'] must fail when files is not installed.
|
||||
|
||||
Regression guard: webdav was hardcoded into _valid_services even when
|
||||
no store services were installed, misleading users into thinking WebDAV
|
||||
was always available.
|
||||
"""
|
||||
patches = _make_admin_client_with_installed(
|
||||
auth_mgr, mock_email_mgr, mock_calendar_mgr,
|
||||
mock_file_mgr, mock_wg_mgr, mock_peer_registry,
|
||||
installed_services={},
|
||||
)
|
||||
started = [p.start() for p in patches]
|
||||
try:
|
||||
with app.test_client() as client:
|
||||
_login(client)
|
||||
resp = _post_peer(client, _peer_payload(service_access=['webdav']))
|
||||
assert resp.status_code == 400, (
|
||||
f'expected 400 when requesting webdav without files installed, got {resp.status_code}'
|
||||
)
|
||||
data = json.loads(resp.data)
|
||||
assert 'service_access' in data.get('error', '').lower()
|
||||
finally:
|
||||
for p in patches:
|
||||
p.stop()
|
||||
|
||||
|
||||
def test_webdav_offered_when_files_is_installed(
|
||||
auth_mgr, mock_email_mgr, mock_calendar_mgr,
|
||||
mock_file_mgr, mock_wg_mgr, mock_peer_registry):
|
||||
"""Requesting service_access=['webdav'] must succeed when files is installed."""
|
||||
patches = _make_admin_client_with_installed(
|
||||
auth_mgr, mock_email_mgr, mock_calendar_mgr,
|
||||
mock_file_mgr, mock_wg_mgr, mock_peer_registry,
|
||||
installed_services={'files': {}},
|
||||
)
|
||||
started = [p.start() for p in patches]
|
||||
try:
|
||||
with app.test_client() as client:
|
||||
_login(client)
|
||||
resp = _post_peer(client, _peer_payload(service_access=['webdav']))
|
||||
assert resp.status_code == 201, (
|
||||
f'expected 201 when requesting webdav with files installed, got {resp.status_code}: {resp.data}'
|
||||
)
|
||||
finally:
|
||||
for p in patches:
|
||||
p.stop()
|
||||
|
||||
|
||||
def test_no_services_installed_peer_gets_empty_service_access(
|
||||
auth_mgr, mock_email_mgr, mock_calendar_mgr,
|
||||
mock_file_mgr, mock_wg_mgr, mock_peer_registry):
|
||||
"""When no store services are installed the default service_access must be empty."""
|
||||
patches = _make_admin_client_with_installed(
|
||||
auth_mgr, mock_email_mgr, mock_calendar_mgr,
|
||||
mock_file_mgr, mock_wg_mgr, mock_peer_registry,
|
||||
installed_services={},
|
||||
)
|
||||
started = [p.start() for p in patches]
|
||||
try:
|
||||
with app.test_client() as client:
|
||||
_login(client)
|
||||
resp = _post_peer(client, _peer_payload()) # no service_access in payload
|
||||
assert resp.status_code == 201, (
|
||||
f'expected 201 with no services installed, got {resp.status_code}: {resp.data}'
|
||||
)
|
||||
peer_dict = mock_peer_registry.add_peer.call_args[0][0]
|
||||
assert peer_dict.get('service_access') == [], (
|
||||
f"service_access should be [] when no services installed, got {peer_dict.get('service_access')}"
|
||||
)
|
||||
finally:
|
||||
for p in patches:
|
||||
p.stop()
|
||||
|
||||
@@ -0,0 +1,214 @@
|
||||
"""Tests for the require_active_service route decorator."""
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent / 'api'))
|
||||
|
||||
from flask import Flask
|
||||
from routes import require_active_service
|
||||
|
||||
|
||||
class TestRequireActiveServiceDecorator(unittest.TestCase):
|
||||
|
||||
def test_installed_service_passes_through(self):
|
||||
app = Flask(__name__)
|
||||
app.config['TESTING'] = True
|
||||
mock_registry = MagicMock()
|
||||
mock_registry.get.return_value = {'id': 'email'}
|
||||
|
||||
@app.route('/test/email')
|
||||
@require_active_service('email')
|
||||
def view():
|
||||
return 'ok', 200
|
||||
|
||||
with app.test_client() as client:
|
||||
with patch.dict('sys.modules', {'app': MagicMock(service_registry=mock_registry)}):
|
||||
resp = client.get('/test/email')
|
||||
self.assertEqual(resp.status_code, 200)
|
||||
|
||||
def test_not_installed_returns_404(self):
|
||||
app = Flask(__name__)
|
||||
app.config['TESTING'] = True
|
||||
mock_registry = MagicMock()
|
||||
mock_registry.get.return_value = None
|
||||
|
||||
@app.route('/test/calendar')
|
||||
@require_active_service('calendar')
|
||||
def view():
|
||||
return 'ok', 200
|
||||
|
||||
with app.test_client() as client:
|
||||
with patch.dict('sys.modules', {'app': MagicMock(service_registry=mock_registry)}):
|
||||
resp = client.get('/test/calendar')
|
||||
self.assertEqual(resp.status_code, 404)
|
||||
data = resp.get_json()
|
||||
self.assertIn('error', data)
|
||||
self.assertIn('calendar', data['error'])
|
||||
|
||||
def test_not_installed_error_message_contains_service_id(self):
|
||||
app = Flask(__name__)
|
||||
app.config['TESTING'] = True
|
||||
mock_registry = MagicMock()
|
||||
mock_registry.get.return_value = None
|
||||
|
||||
@app.route('/test/files')
|
||||
@require_active_service('files')
|
||||
def view():
|
||||
return 'ok', 200
|
||||
|
||||
with app.test_client() as client:
|
||||
with patch.dict('sys.modules', {'app': MagicMock(service_registry=mock_registry)}):
|
||||
resp = client.get('/test/files')
|
||||
data = resp.get_json()
|
||||
self.assertIn('files', data['error'])
|
||||
|
||||
def test_decorator_preserves_function_name(self):
|
||||
@require_active_service('calendar')
|
||||
def my_view():
|
||||
return 'ok'
|
||||
self.assertEqual(my_view.__name__, 'my_view')
|
||||
|
||||
def test_decorator_preserves_function_docstring(self):
|
||||
@require_active_service('email')
|
||||
def documented_view():
|
||||
"""Returns email data."""
|
||||
return 'ok'
|
||||
self.assertEqual(documented_view.__doc__, 'Returns email data.')
|
||||
|
||||
def test_passes_positional_args_to_wrapped_function(self):
|
||||
app = Flask(__name__)
|
||||
app.config['TESTING'] = True
|
||||
mock_registry = MagicMock()
|
||||
mock_registry.get.return_value = {'id': 'email'}
|
||||
|
||||
@app.route('/test/mailbox/<username>')
|
||||
@require_active_service('email')
|
||||
def mailbox_view(username):
|
||||
return username, 200
|
||||
|
||||
with app.test_client() as client:
|
||||
with patch.dict('sys.modules', {'app': MagicMock(service_registry=mock_registry)}):
|
||||
resp = client.get('/test/mailbox/alice')
|
||||
self.assertEqual(resp.status_code, 200)
|
||||
self.assertIn(b'alice', resp.data)
|
||||
|
||||
def test_service_registry_get_called_with_correct_service_id(self):
|
||||
app = Flask(__name__)
|
||||
app.config['TESTING'] = True
|
||||
mock_registry = MagicMock()
|
||||
mock_registry.get.return_value = {'id': 'files'}
|
||||
|
||||
@app.route('/test/svc')
|
||||
@require_active_service('files')
|
||||
def view():
|
||||
return 'ok', 200
|
||||
|
||||
with app.test_client() as client:
|
||||
with patch.dict('sys.modules', {'app': MagicMock(service_registry=mock_registry)}):
|
||||
client.get('/test/svc')
|
||||
mock_registry.get.assert_called_with('files')
|
||||
|
||||
def test_status_endpoint_bypasses_decorator_on_email_routes(self):
|
||||
"""The /status route in email.py must NOT be decorated; verify by importing the route."""
|
||||
from routes.email import get_email_status
|
||||
# The status handler should not be wrapped — it won't have the
|
||||
# _require_active_service marker that the wrapper would add.
|
||||
# We verify by checking it has no 'service_id' closure variable
|
||||
# from the decorator (i.e., it's the plain function, not a wrapper).
|
||||
import inspect
|
||||
# get_email_status should have no closure cells referencing a service_id
|
||||
closure = get_email_status.__closure__
|
||||
if closure:
|
||||
closure_vals = [c.cell_contents for c in closure if isinstance(c.cell_contents, str)]
|
||||
self.assertNotIn('email', closure_vals,
|
||||
"get_email_status should not be wrapped by require_active_service")
|
||||
|
||||
def test_status_endpoint_bypasses_decorator_on_calendar_routes(self):
|
||||
from routes.calendar import get_calendar_status
|
||||
closure = get_calendar_status.__closure__
|
||||
if closure:
|
||||
closure_vals = [c.cell_contents for c in closure if isinstance(c.cell_contents, str)]
|
||||
self.assertNotIn('calendar', closure_vals)
|
||||
|
||||
def test_status_endpoint_bypasses_decorator_on_files_routes(self):
|
||||
from routes.files import get_file_status
|
||||
closure = get_file_status.__closure__
|
||||
if closure:
|
||||
closure_vals = [c.cell_contents for c in closure if isinstance(c.cell_contents, str)]
|
||||
self.assertNotIn('files', closure_vals)
|
||||
|
||||
|
||||
class TestRequireActiveServiceOnEmailRoutes(unittest.TestCase):
|
||||
"""Integration-style: exercise the decorator via the real Flask app."""
|
||||
|
||||
def setUp(self):
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent / 'api'))
|
||||
from app import app
|
||||
app.config['TESTING'] = True
|
||||
self.client = app.test_client()
|
||||
self.app = app
|
||||
|
||||
def test_email_users_returns_404_when_service_not_installed(self):
|
||||
mock_registry = MagicMock()
|
||||
mock_registry.get.return_value = None
|
||||
with patch('app.service_registry', mock_registry):
|
||||
resp = self.client.get('/api/email/users')
|
||||
self.assertEqual(resp.status_code, 404)
|
||||
data = resp.get_json()
|
||||
self.assertIn('error', data)
|
||||
self.assertIn('email', data['error'])
|
||||
|
||||
def test_email_status_reachable_when_service_not_installed(self):
|
||||
"""Status endpoint is never blocked, even with service absent."""
|
||||
mock_registry = MagicMock()
|
||||
mock_registry.get.return_value = None
|
||||
mock_email_mgr = MagicMock()
|
||||
mock_email_mgr.get_status.return_value = {'installed': False}
|
||||
with patch('app.service_registry', mock_registry), \
|
||||
patch('app.email_manager', mock_email_mgr):
|
||||
resp = self.client.get('/api/email/status')
|
||||
self.assertNotEqual(resp.status_code, 404)
|
||||
|
||||
def test_calendar_users_returns_404_when_service_not_installed(self):
|
||||
mock_registry = MagicMock()
|
||||
mock_registry.get.return_value = None
|
||||
with patch('app.service_registry', mock_registry):
|
||||
resp = self.client.get('/api/calendar/users')
|
||||
self.assertEqual(resp.status_code, 404)
|
||||
data = resp.get_json()
|
||||
self.assertIn('calendar', data['error'])
|
||||
|
||||
def test_calendar_status_reachable_when_service_not_installed(self):
|
||||
mock_registry = MagicMock()
|
||||
mock_registry.get.return_value = None
|
||||
mock_cal_mgr = MagicMock()
|
||||
mock_cal_mgr.get_status.return_value = {'installed': False}
|
||||
with patch('app.service_registry', mock_registry), \
|
||||
patch('app.calendar_manager', mock_cal_mgr):
|
||||
resp = self.client.get('/api/calendar/status')
|
||||
self.assertNotEqual(resp.status_code, 404)
|
||||
|
||||
def test_files_users_returns_404_when_service_not_installed(self):
|
||||
mock_registry = MagicMock()
|
||||
mock_registry.get.return_value = None
|
||||
with patch('app.service_registry', mock_registry):
|
||||
resp = self.client.get('/api/files/users')
|
||||
self.assertEqual(resp.status_code, 404)
|
||||
data = resp.get_json()
|
||||
self.assertIn('files', data['error'])
|
||||
|
||||
def test_files_status_reachable_when_service_not_installed(self):
|
||||
mock_registry = MagicMock()
|
||||
mock_registry.get.return_value = None
|
||||
mock_file_mgr = MagicMock()
|
||||
mock_file_mgr.get_status.return_value = {'installed': False}
|
||||
with patch('app.service_registry', mock_registry), \
|
||||
patch('app.file_manager', mock_file_mgr):
|
||||
resp = self.client.get('/api/files/status')
|
||||
self.assertNotEqual(resp.status_code, 404)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -128,6 +128,12 @@ def test_anon_blocked_from_peer_routes(anon_client):
|
||||
assert r.status_code == 401
|
||||
|
||||
|
||||
def test_setup_routes_bypass_auth(anon_client):
|
||||
"""/api/setup/* must be reachable without a session — setup runs before any account exists."""
|
||||
r = anon_client.get('/api/setup/status')
|
||||
assert r.status_code != 401
|
||||
|
||||
|
||||
def test_anon_blocked_from_peer_dashboard(anon_client):
|
||||
r = anon_client.get('/api/peer/dashboard')
|
||||
assert r.status_code == 401
|
||||
|
||||
@@ -214,5 +214,38 @@ class TestServiceBus(unittest.TestCase):
|
||||
mock_service.stop.assert_called_once()
|
||||
mock_service.start.assert_called_once()
|
||||
|
||||
class TestIdentityChangedEventType(unittest.TestCase):
|
||||
"""Tests for the IDENTITY_CHANGED event type."""
|
||||
|
||||
def test_identity_changed_event_type_exists(self):
|
||||
self.assertEqual(EventType.IDENTITY_CHANGED.value, "identity_changed")
|
||||
|
||||
def test_identity_changed_published_and_received(self):
|
||||
"""Publish IDENTITY_CHANGED and verify the subscriber receives it."""
|
||||
bus = ServiceBus()
|
||||
bus.start()
|
||||
try:
|
||||
received = []
|
||||
|
||||
def handler(event):
|
||||
received.append(event)
|
||||
|
||||
bus.subscribe_to_event(EventType.IDENTITY_CHANGED, handler)
|
||||
bus.publish_event(EventType.IDENTITY_CHANGED, 'test', {
|
||||
'cell_name': 'mycell',
|
||||
'domain': 'cell',
|
||||
'domain_name': 'mycell.pic.ngo',
|
||||
'domain_mode': 'pic_ngo',
|
||||
'effective_domain': 'mycell.pic.ngo',
|
||||
})
|
||||
time.sleep(0.2)
|
||||
self.assertEqual(len(received), 1)
|
||||
self.assertEqual(received[0].event_type, EventType.IDENTITY_CHANGED)
|
||||
self.assertEqual(received[0].data['cell_name'], 'mycell')
|
||||
self.assertEqual(received[0].data['effective_domain'], 'mycell.pic.ngo')
|
||||
finally:
|
||||
bus.stop()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,680 @@
|
||||
"""
|
||||
Unit tests for ServiceComposer.
|
||||
|
||||
All subprocess calls and filesystem writes are mocked — no Docker daemon required.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch, mock_open, call
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent / 'api'))
|
||||
|
||||
from service_composer import ServiceComposer, _SECRET_RE
|
||||
|
||||
|
||||
# ── Helpers ─────────────────────────────────────────────────────────────────
|
||||
|
||||
def _make_cm(identity=None, service_config=None) -> MagicMock:
|
||||
cm = MagicMock()
|
||||
ident = identity or {'cell_name': 'testcell', 'domain': 'cell.local', 'domain_mode': 'lan'}
|
||||
cm.get_identity.return_value = ident
|
||||
cm.get_effective_domain.return_value = ident.get('domain', 'cell.local')
|
||||
cm.configs = {}
|
||||
if service_config:
|
||||
cm.configs.update(service_config)
|
||||
return cm
|
||||
|
||||
|
||||
def _make_manifest(service_id='myservice', kind='store', schema=None):
|
||||
return {
|
||||
'id': service_id,
|
||||
'kind': kind,
|
||||
'config_schema': schema or {
|
||||
'port': {'type': 'integer', 'default': 8080},
|
||||
'username': {'type': 'string', 'default': 'admin'},
|
||||
},
|
||||
'containers': [f'cell-{service_id}'],
|
||||
}
|
||||
|
||||
|
||||
def _composer(cm=None, data_dir=None):
|
||||
if data_dir is None:
|
||||
data_dir = '/fake/data'
|
||||
return ServiceComposer(config_manager=cm or _make_cm(), data_dir=data_dir)
|
||||
|
||||
|
||||
# ── Template rendering ────────────────────────────────────────────────────────
|
||||
|
||||
class TestRenderTemplate(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.cm = _make_cm(service_config={'myservice': {'port': 9090}})
|
||||
self.composer = _composer(self.cm)
|
||||
|
||||
def test_substitutes_pic_cfg_uppercase(self):
|
||||
manifest = _make_manifest()
|
||||
template = 'PORT=${PIC_CFG_PORT}'
|
||||
result = self.composer.render_template('myservice', manifest, template)
|
||||
self.assertEqual(result, 'PORT=9090')
|
||||
|
||||
def test_substitutes_default_when_no_saved_config(self):
|
||||
cm = _make_cm()
|
||||
composer = _composer(cm)
|
||||
manifest = _make_manifest()
|
||||
template = 'USER=${PIC_CFG_USERNAME}'
|
||||
result = composer.render_template('myservice', manifest, template)
|
||||
self.assertEqual(result, 'USER=admin')
|
||||
|
||||
def test_pic_domain_substituted(self):
|
||||
manifest = _make_manifest()
|
||||
template = 'DOMAIN=${PIC_DOMAIN}'
|
||||
result = self.composer.render_template('myservice', manifest, template)
|
||||
self.assertIn('cell.local', result)
|
||||
|
||||
def test_pic_cell_name_substituted(self):
|
||||
manifest = _make_manifest()
|
||||
template = 'CELL=${PIC_CELL_NAME}'
|
||||
result = self.composer.render_template('myservice', manifest, template)
|
||||
self.assertIn('testcell', result)
|
||||
|
||||
def test_pic_service_id_substituted(self):
|
||||
manifest = _make_manifest(service_id='myservice')
|
||||
template = 'ID=${PIC_SERVICE_ID}'
|
||||
result = self.composer.render_template('myservice', manifest, template)
|
||||
self.assertEqual(result, 'ID=myservice')
|
||||
|
||||
def test_pic_secret_generated_and_substituted(self):
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
composer = _composer(data_dir=tmpdir)
|
||||
manifest = _make_manifest()
|
||||
template = 'PASS=${PIC_SECRET_DB_PASS}'
|
||||
result = composer.render_template('myservice', manifest, template)
|
||||
self.assertNotIn('${PIC_SECRET_DB_PASS}', result)
|
||||
self.assertNotEqual(result, 'PASS=')
|
||||
# Secret is a non-empty string
|
||||
password = result.replace('PASS=', '')
|
||||
self.assertTrue(len(password) > 8)
|
||||
|
||||
def test_pic_secret_stable_across_calls(self):
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
composer = _composer(data_dir=tmpdir)
|
||||
manifest = _make_manifest()
|
||||
template = 'P=${PIC_SECRET_MY_PASS}'
|
||||
r1 = composer.render_template('myservice', manifest, template)
|
||||
r2 = composer.render_template('myservice', manifest, template)
|
||||
self.assertEqual(r1, r2)
|
||||
|
||||
def test_pic_secret_different_per_service(self):
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
composer = _composer(data_dir=tmpdir)
|
||||
m1 = _make_manifest('svc1')
|
||||
m2 = _make_manifest('svc2')
|
||||
t = 'P=${PIC_SECRET_PASS}'
|
||||
r1 = composer.render_template('svc1', m1, t)
|
||||
r2 = composer.render_template('svc2', m2, t)
|
||||
self.assertNotEqual(r1, r2)
|
||||
|
||||
def test_multiple_secrets_all_replaced(self):
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
composer = _composer(data_dir=tmpdir)
|
||||
manifest = _make_manifest()
|
||||
template = 'A=${PIC_SECRET_KEY_A}\nB=${PIC_SECRET_KEY_B}'
|
||||
result = composer.render_template('myservice', manifest, template)
|
||||
self.assertNotIn('${PIC_SECRET_', result)
|
||||
|
||||
def test_no_unknown_vars_left_from_schema(self):
|
||||
# Use a fresh composer with no saved config so defaults apply
|
||||
composer = _composer(_make_cm())
|
||||
manifest = _make_manifest(schema={
|
||||
'port': {'type': 'integer', 'default': 3000},
|
||||
})
|
||||
template = 'PORT=${PIC_CFG_PORT}\nOTHER=${PIC_CFG_UNKNOWN}'
|
||||
result = composer.render_template('myservice', manifest, template)
|
||||
# Known var substituted with default, unknown left alone (no crash)
|
||||
self.assertIn('PORT=3000', result)
|
||||
self.assertIn('${PIC_CFG_UNKNOWN}', result)
|
||||
|
||||
|
||||
# ── Write compose file ────────────────────────────────────────────────────────
|
||||
|
||||
class TestWriteCompose(unittest.TestCase):
|
||||
|
||||
def test_writes_rendered_content_to_correct_path(self):
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
cm = _make_cm()
|
||||
composer = ServiceComposer(config_manager=cm, data_dir=tmpdir)
|
||||
manifest = _make_manifest()
|
||||
template = (
|
||||
'version: "3.8"\n'
|
||||
'services:\n'
|
||||
' app:\n'
|
||||
' image: nginx\n'
|
||||
' environment:\n'
|
||||
' PORT: "${PIC_CFG_PORT}"\n'
|
||||
'networks:\n'
|
||||
' cell-network:\n'
|
||||
' external: true\n'
|
||||
)
|
||||
composer.write_compose('myservice', manifest, template)
|
||||
|
||||
expected_path = os.path.join(
|
||||
tmpdir, 'services', 'myservice', 'docker-compose.yml'
|
||||
)
|
||||
self.assertTrue(os.path.exists(expected_path))
|
||||
with open(expected_path) as f:
|
||||
content = f.read()
|
||||
self.assertIn('8080', content)
|
||||
|
||||
def test_has_compose_file_false_before_write(self):
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
composer = _composer(data_dir=tmpdir)
|
||||
self.assertFalse(composer.has_compose_file('newservice'))
|
||||
|
||||
def test_has_compose_file_true_after_write(self):
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
composer = ServiceComposer(config_manager=_make_cm(), data_dir=tmpdir)
|
||||
manifest = _make_manifest()
|
||||
valid_template = (
|
||||
'version: "3.8"\n'
|
||||
'services:\n'
|
||||
' app:\n'
|
||||
' image: nginx\n'
|
||||
'networks:\n'
|
||||
' cell-network:\n'
|
||||
' external: true\n'
|
||||
)
|
||||
composer.write_compose('myservice', manifest, valid_template)
|
||||
self.assertTrue(composer.has_compose_file('myservice'))
|
||||
|
||||
def test_atomic_write_via_tmp_file(self):
|
||||
"""If fsync fails, the compose file should not be partially written."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
composer = ServiceComposer(config_manager=_make_cm(), data_dir=tmpdir)
|
||||
manifest = _make_manifest()
|
||||
# Should not raise even if fsync not available
|
||||
valid_template = (
|
||||
'version: "3.8"\n'
|
||||
'services:\n'
|
||||
' app:\n'
|
||||
' image: nginx\n'
|
||||
'networks:\n'
|
||||
' cell-network:\n'
|
||||
' external: true\n'
|
||||
)
|
||||
composer.write_compose('myservice', manifest, valid_template)
|
||||
path = os.path.join(tmpdir, 'services', 'myservice', 'docker-compose.yml')
|
||||
self.assertTrue(os.path.exists(path))
|
||||
|
||||
def test_requires_host_network_manifest_allows_host_mode_template(self):
|
||||
"""write_compose passes when manifest has requires_host_network: true and template uses network_mode: host."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
composer = ServiceComposer(config_manager=_make_cm(), data_dir=tmpdir)
|
||||
manifest = _make_manifest()
|
||||
manifest['requires_host_network'] = True
|
||||
template = (
|
||||
'services:\n'
|
||||
' wireguard-ext:\n'
|
||||
' image: git.pic.ngo/roof/svc-wireguard-ext:latest\n'
|
||||
' container_name: cell-wg-ext\n'
|
||||
' network_mode: host\n'
|
||||
' cap_add:\n'
|
||||
' - NET_ADMIN\n'
|
||||
' volumes:\n'
|
||||
f' - {tmpdir}/services/wireguard-ext/config:/etc/wireguard\n'
|
||||
)
|
||||
# Should not raise
|
||||
composer.write_compose('wireguard-ext', manifest, template)
|
||||
path = os.path.join(tmpdir, 'services', 'wireguard-ext', 'docker-compose.yml')
|
||||
self.assertTrue(os.path.exists(path))
|
||||
|
||||
def test_requires_host_network_false_rejects_host_mode_template(self):
|
||||
"""write_compose raises when manifest does NOT have requires_host_network but template uses network_mode: host."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
composer = ServiceComposer(config_manager=_make_cm(), data_dir=tmpdir)
|
||||
manifest = _make_manifest()
|
||||
manifest['requires_host_network'] = False
|
||||
template = (
|
||||
'services:\n'
|
||||
' svc:\n'
|
||||
' image: git.pic.ngo/roof/svc-foo:latest\n'
|
||||
' network_mode: host\n'
|
||||
'networks:\n'
|
||||
' cell-network:\n'
|
||||
' external: true\n'
|
||||
)
|
||||
with self.assertRaises(ValueError):
|
||||
composer.write_compose('svc', manifest, template)
|
||||
|
||||
|
||||
# ── Secrets ───────────────────────────────────────────────────────────────────
|
||||
|
||||
class TestSecrets(unittest.TestCase):
|
||||
|
||||
def test_secrets_persisted_to_file(self):
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
composer = _composer(data_dir=tmpdir)
|
||||
composer._get_or_create_secret('svc', 'PIC_SECRET_PASS')
|
||||
secrets_path = os.path.join(tmpdir, 'service_secrets.json')
|
||||
self.assertTrue(os.path.exists(secrets_path))
|
||||
with open(secrets_path) as f:
|
||||
data = json.load(f)
|
||||
self.assertIn('svc', data)
|
||||
self.assertIn('PIC_SECRET_PASS', data['svc'])
|
||||
|
||||
def test_clear_secrets_removes_service_entry(self):
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
composer = _composer(data_dir=tmpdir)
|
||||
composer._get_or_create_secret('svc', 'PIC_SECRET_KEY')
|
||||
composer._clear_secrets('svc')
|
||||
secrets = composer._load_secrets()
|
||||
self.assertNotIn('svc', secrets)
|
||||
|
||||
def test_clear_secrets_noop_when_no_secrets_file(self):
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
composer = _composer(data_dir=tmpdir)
|
||||
# Should not raise
|
||||
composer._clear_secrets('nonexistent')
|
||||
|
||||
def test_load_secrets_returns_empty_when_file_missing(self):
|
||||
composer = _composer(data_dir='/nonexistent/path')
|
||||
self.assertEqual(composer._load_secrets(), {})
|
||||
|
||||
|
||||
# ── Subprocess execution ──────────────────────────────────────────────────────
|
||||
|
||||
class TestDockerComposeExecution(unittest.TestCase):
|
||||
|
||||
def _composer_with_compose_file(self, tmpdir, service_id='myservice'):
|
||||
composer = ServiceComposer(config_manager=_make_cm(), data_dir=tmpdir)
|
||||
svc_dir = os.path.join(tmpdir, 'services', service_id)
|
||||
os.makedirs(svc_dir)
|
||||
with open(os.path.join(svc_dir, 'docker-compose.yml'), 'w') as f:
|
||||
f.write('services:\n app:\n image: nginx\n')
|
||||
return composer
|
||||
|
||||
@patch('service_composer.subprocess.run')
|
||||
def test_up_calls_docker_compose_up(self, mock_run):
|
||||
mock_run.return_value = MagicMock(returncode=0, stdout='', stderr='')
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
composer = self._composer_with_compose_file(tmpdir)
|
||||
composer.up('myservice')
|
||||
cmd = mock_run.call_args[0][0]
|
||||
self.assertIn('up', cmd)
|
||||
self.assertIn('-d', cmd)
|
||||
self.assertIn('--project-name', cmd)
|
||||
self.assertIn('pic-myservice', cmd)
|
||||
|
||||
@patch('service_composer.subprocess.run')
|
||||
def test_down_calls_docker_compose_down(self, mock_run):
|
||||
mock_run.return_value = MagicMock(returncode=0, stdout='', stderr='')
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
composer = self._composer_with_compose_file(tmpdir)
|
||||
composer.down('myservice')
|
||||
cmd = mock_run.call_args[0][0]
|
||||
self.assertIn('down', cmd)
|
||||
|
||||
@patch('service_composer.subprocess.run')
|
||||
def test_down_with_purge_passes_volumes_flag(self, mock_run):
|
||||
mock_run.return_value = MagicMock(returncode=0, stdout='', stderr='')
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
composer = self._composer_with_compose_file(tmpdir)
|
||||
composer.down('myservice', remove_volumes=True)
|
||||
cmd = mock_run.call_args[0][0]
|
||||
self.assertIn('--volumes', cmd)
|
||||
|
||||
@patch('service_composer.subprocess.run')
|
||||
def test_restart_calls_docker_compose_restart(self, mock_run):
|
||||
mock_run.return_value = MagicMock(returncode=0, stdout='', stderr='')
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
composer = self._composer_with_compose_file(tmpdir)
|
||||
composer.restart('myservice')
|
||||
cmd = mock_run.call_args[0][0]
|
||||
self.assertIn('restart', cmd)
|
||||
|
||||
@patch('service_composer.subprocess.run')
|
||||
def test_status_parses_json_output(self, mock_run):
|
||||
container_info = {'Name': 'myservice-app', 'State': 'running'}
|
||||
mock_run.return_value = MagicMock(
|
||||
returncode=0,
|
||||
stdout=json.dumps(container_info),
|
||||
stderr='',
|
||||
)
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
composer = self._composer_with_compose_file(tmpdir)
|
||||
result = composer.status('myservice')
|
||||
self.assertTrue(result['ok'])
|
||||
self.assertEqual(len(result['containers']), 1)
|
||||
self.assertEqual(result['containers'][0]['Name'], 'myservice-app')
|
||||
|
||||
@patch('service_composer.subprocess.run')
|
||||
def test_status_returns_empty_containers_on_bad_json(self, mock_run):
|
||||
mock_run.return_value = MagicMock(returncode=0, stdout='not json', stderr='')
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
composer = self._composer_with_compose_file(tmpdir)
|
||||
result = composer.status('myservice')
|
||||
self.assertEqual(result['containers'], [])
|
||||
|
||||
def test_store_cmd_returns_error_when_no_compose_file(self):
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
composer = _composer(data_dir=tmpdir)
|
||||
result = composer.up('nonexistent')
|
||||
self.assertFalse(result['ok'])
|
||||
self.assertIn('No compose file', result['error'])
|
||||
|
||||
@patch('service_composer.subprocess.run')
|
||||
def test_up_uses_600s_timeout(self, mock_run):
|
||||
mock_run.return_value = MagicMock(returncode=0, stdout='', stderr='')
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
composer = self._composer_with_compose_file(tmpdir)
|
||||
composer.up('myservice')
|
||||
_, kwargs = mock_run.call_args
|
||||
self.assertGreaterEqual(kwargs.get('timeout', 0), 600)
|
||||
|
||||
@patch('service_composer.subprocess.run')
|
||||
def test_run_returns_error_on_timeout(self, mock_run):
|
||||
import subprocess
|
||||
mock_run.side_effect = subprocess.TimeoutExpired(cmd=[], timeout=120)
|
||||
composer = _composer()
|
||||
result = composer._run(['docker', 'compose', 'up'])
|
||||
self.assertFalse(result['ok'])
|
||||
self.assertIn('timed out', result['error'])
|
||||
|
||||
@patch('service_composer.subprocess.run')
|
||||
def test_run_returns_false_on_nonzero_exit(self, mock_run):
|
||||
mock_run.return_value = MagicMock(returncode=1, stdout='', stderr='error msg')
|
||||
composer = _composer()
|
||||
result = composer._run(['docker', 'compose', 'up'])
|
||||
self.assertFalse(result['ok'])
|
||||
self.assertEqual(result['stderr'], 'error msg')
|
||||
|
||||
|
||||
# ── Builtin lifecycle ─────────────────────────────────────────────────────────
|
||||
|
||||
class TestBuiltinLifecycle(unittest.TestCase):
|
||||
|
||||
@patch('service_composer.subprocess.run')
|
||||
def test_restart_builtin_includes_container_names(self, mock_run):
|
||||
mock_run.return_value = MagicMock(returncode=0, stdout='', stderr='')
|
||||
composer = _composer()
|
||||
composer.restart_builtin(['cell-radicale'])
|
||||
cmd = mock_run.call_args[0][0]
|
||||
self.assertIn('cell-radicale', cmd)
|
||||
self.assertIn('restart', cmd)
|
||||
|
||||
@patch('service_composer.subprocess.run')
|
||||
def test_status_builtin_includes_container_names(self, mock_run):
|
||||
mock_run.return_value = MagicMock(returncode=0, stdout='', stderr='')
|
||||
composer = _composer()
|
||||
composer.status_builtin(['cell-mail', 'cell-rainloop'])
|
||||
cmd = mock_run.call_args[0][0]
|
||||
self.assertIn('cell-mail', cmd)
|
||||
self.assertIn('cell-rainloop', cmd)
|
||||
|
||||
def test_restart_builtin_empty_list_returns_error(self):
|
||||
composer = _composer()
|
||||
result = composer.restart_builtin([])
|
||||
self.assertFalse(result['ok'])
|
||||
|
||||
def test_status_builtin_empty_list_returns_error(self):
|
||||
composer = _composer()
|
||||
result = composer.status_builtin([])
|
||||
self.assertFalse(result['ok'])
|
||||
|
||||
|
||||
# ── Unified dispatch ──────────────────────────────────────────────────────────
|
||||
|
||||
class TestUnifiedDispatch(unittest.TestCase):
|
||||
|
||||
@patch('service_composer.subprocess.run')
|
||||
def test_restart_service_builtin_uses_main_compose(self, mock_run):
|
||||
mock_run.return_value = MagicMock(returncode=0, stdout='', stderr='')
|
||||
composer = _composer()
|
||||
manifest = _make_manifest(kind='builtin')
|
||||
manifest['containers'] = ['cell-myservice']
|
||||
composer.restart_service('myservice', manifest)
|
||||
cmd = mock_run.call_args[0][0]
|
||||
self.assertIn('cell-myservice', cmd)
|
||||
# Main compose flag present
|
||||
self.assertIn('-f', cmd)
|
||||
|
||||
@patch('service_composer.subprocess.run')
|
||||
def test_restart_service_store_uses_per_service_compose(self, mock_run):
|
||||
mock_run.return_value = MagicMock(returncode=0, stdout='', stderr='')
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
composer = ServiceComposer(config_manager=_make_cm(), data_dir=tmpdir)
|
||||
# Create compose file for the store service
|
||||
svc_dir = os.path.join(tmpdir, 'services', 'storesvc')
|
||||
os.makedirs(svc_dir)
|
||||
with open(os.path.join(svc_dir, 'docker-compose.yml'), 'w') as f:
|
||||
f.write('services:\n app:\n image: nginx\n')
|
||||
manifest = _make_manifest('storesvc', kind='store')
|
||||
composer.restart_service('storesvc', manifest)
|
||||
cmd = mock_run.call_args[0][0]
|
||||
self.assertIn('pic-storesvc', cmd)
|
||||
|
||||
|
||||
# ── Remove ────────────────────────────────────────────────────────────────────
|
||||
|
||||
class TestServiceIdValidation(unittest.TestCase):
|
||||
|
||||
def test_valid_ids_accepted(self):
|
||||
for sid in ('email', 'my-service', 'svc123', 'a1b2-c3'):
|
||||
ServiceComposer._validate_service_id(sid) # should not raise
|
||||
|
||||
def test_dotdot_rejected(self):
|
||||
with self.assertRaises(ValueError):
|
||||
ServiceComposer._validate_service_id('..')
|
||||
|
||||
def test_dot_rejected(self):
|
||||
with self.assertRaises(ValueError):
|
||||
ServiceComposer._validate_service_id('.')
|
||||
|
||||
def test_slash_rejected(self):
|
||||
with self.assertRaises(ValueError):
|
||||
ServiceComposer._validate_service_id('evil/path')
|
||||
|
||||
def test_uppercase_rejected(self):
|
||||
with self.assertRaises(ValueError):
|
||||
ServiceComposer._validate_service_id('MyService')
|
||||
|
||||
def test_empty_string_rejected(self):
|
||||
with self.assertRaises(ValueError):
|
||||
ServiceComposer._validate_service_id('')
|
||||
|
||||
def test_newline_in_config_value_stripped(self):
|
||||
"""A newline in a config value must not create a new YAML key (injection)."""
|
||||
cm = _make_cm(service_config={'svc': {'port': '80\nnewline_attack: true'}})
|
||||
composer = _composer(cm)
|
||||
manifest = _make_manifest(schema={'port': {'type': 'string', 'default': '80'}})
|
||||
result = composer.render_template('svc', manifest, 'PORT=${PIC_CFG_PORT}')
|
||||
# The newline is stripped — 'newline_attack' is concatenated, not a separate YAML key
|
||||
self.assertNotIn('\n', result)
|
||||
|
||||
|
||||
class TestRemove(unittest.TestCase):
|
||||
|
||||
@patch('service_composer.subprocess.run')
|
||||
def test_remove_deletes_compose_file(self, mock_run):
|
||||
mock_run.return_value = MagicMock(returncode=0, stdout='', stderr='')
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
composer = ServiceComposer(config_manager=_make_cm(), data_dir=tmpdir)
|
||||
svc_dir = os.path.join(tmpdir, 'services', 'oldsvc')
|
||||
os.makedirs(svc_dir)
|
||||
compose_file = os.path.join(svc_dir, 'docker-compose.yml')
|
||||
with open(compose_file, 'w') as f:
|
||||
f.write('services: {}')
|
||||
composer.remove('oldsvc', purge_data=False)
|
||||
self.assertFalse(os.path.exists(compose_file))
|
||||
|
||||
@patch('service_composer.subprocess.run')
|
||||
def test_remove_purge_deletes_service_directory(self, mock_run):
|
||||
mock_run.return_value = MagicMock(returncode=0, stdout='', stderr='')
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
composer = ServiceComposer(config_manager=_make_cm(), data_dir=tmpdir)
|
||||
svc_dir = os.path.join(tmpdir, 'services', 'purgesvc')
|
||||
os.makedirs(svc_dir)
|
||||
with open(os.path.join(svc_dir, 'docker-compose.yml'), 'w') as f:
|
||||
f.write('services: {}')
|
||||
with open(os.path.join(svc_dir, 'data.txt'), 'w') as f:
|
||||
f.write('important data')
|
||||
composer.remove('purgesvc', purge_data=True)
|
||||
self.assertFalse(os.path.exists(svc_dir))
|
||||
|
||||
@patch('service_composer.subprocess.run')
|
||||
def test_remove_purge_clears_secrets(self, mock_run):
|
||||
mock_run.return_value = MagicMock(returncode=0, stdout='', stderr='')
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
composer = ServiceComposer(config_manager=_make_cm(), data_dir=tmpdir)
|
||||
composer._get_or_create_secret('purgesvc', 'PIC_SECRET_KEY')
|
||||
svc_dir = os.path.join(tmpdir, 'services', 'purgesvc')
|
||||
os.makedirs(svc_dir)
|
||||
with open(os.path.join(svc_dir, 'docker-compose.yml'), 'w') as f:
|
||||
f.write('services: {}')
|
||||
composer.remove('purgesvc', purge_data=True)
|
||||
secrets = composer._load_secrets()
|
||||
self.assertNotIn('purgesvc', secrets)
|
||||
|
||||
|
||||
# ── Parse ps json ─────────────────────────────────────────────────────────────
|
||||
|
||||
class TestParsePsJson(unittest.TestCase):
|
||||
|
||||
def test_single_json_object(self):
|
||||
line = json.dumps({'Name': 'c1', 'State': 'running'})
|
||||
result = ServiceComposer._parse_ps_json(line)
|
||||
self.assertEqual(len(result), 1)
|
||||
self.assertEqual(result[0]['Name'], 'c1')
|
||||
|
||||
def test_multiple_json_lines(self):
|
||||
lines = '\n'.join([
|
||||
json.dumps({'Name': 'c1'}),
|
||||
json.dumps({'Name': 'c2'}),
|
||||
])
|
||||
result = ServiceComposer._parse_ps_json(lines)
|
||||
self.assertEqual(len(result), 2)
|
||||
|
||||
def test_ignores_blank_lines(self):
|
||||
lines = '\n'.join([json.dumps({'Name': 'c1'}), '', json.dumps({'Name': 'c2'})])
|
||||
result = ServiceComposer._parse_ps_json(lines)
|
||||
self.assertEqual(len(result), 2)
|
||||
|
||||
def test_returns_empty_list_for_empty_output(self):
|
||||
self.assertEqual(ServiceComposer._parse_ps_json(''), [])
|
||||
|
||||
def test_bad_json_lines_skipped(self):
|
||||
lines = '\n'.join(['not json', json.dumps({'Name': 'c1'})])
|
||||
result = ServiceComposer._parse_ps_json(lines)
|
||||
self.assertEqual(len(result), 1)
|
||||
|
||||
|
||||
# ── Dependency resolution ─────────────────────────────────────────────────────
|
||||
|
||||
class TestServiceComposerDeps(unittest.TestCase):
|
||||
|
||||
def _composer(self):
|
||||
cm = MagicMock()
|
||||
cm.configs = {}
|
||||
cm.get_installed_services.return_value = {}
|
||||
cm.get_identity.return_value = {}
|
||||
cm.get_effective_domain.return_value = 'test.cell'
|
||||
return ServiceComposer(config_manager=cm, data_dir='/tmp/test')
|
||||
|
||||
def test_resolve_requires_no_requires(self):
|
||||
composer = self._composer()
|
||||
manifest = {'id': 'webmail', 'requires': []}
|
||||
result = composer._resolve_requires(manifest, {})
|
||||
self.assertIsNone(result)
|
||||
|
||||
def test_resolve_requires_dep_installed(self):
|
||||
composer = self._composer()
|
||||
manifest = {'id': 'webmail', 'requires': ['email']}
|
||||
installed = {'email': {'manifest': {'id': 'email'}}}
|
||||
result = composer._resolve_requires(manifest, installed)
|
||||
self.assertIsNone(result)
|
||||
|
||||
def test_resolve_requires_dep_missing(self):
|
||||
composer = self._composer()
|
||||
manifest = {'id': 'webmail', 'requires': ['email']}
|
||||
result = composer._resolve_requires(manifest, {})
|
||||
self.assertIsNotNone(result)
|
||||
self.assertIn('email', result)
|
||||
|
||||
def test_resolve_requires_multiple_deps_partial(self):
|
||||
composer = self._composer()
|
||||
manifest = {'id': 'x', 'requires': ['email', 'calendar']}
|
||||
installed = {'email': {'manifest': {'id': 'email'}}}
|
||||
result = composer._resolve_requires(manifest, installed)
|
||||
self.assertIsNotNone(result)
|
||||
self.assertIn('calendar', result)
|
||||
self.assertNotIn('email', result)
|
||||
|
||||
def test_resolve_requires_no_requires_key(self):
|
||||
composer = self._composer()
|
||||
manifest = {'id': 'files'} # no 'requires' key
|
||||
result = composer._resolve_requires(manifest, {})
|
||||
self.assertIsNone(result)
|
||||
|
||||
def test_resolve_dependents_none(self):
|
||||
composer = self._composer()
|
||||
installed = {
|
||||
'email': {'manifest': {'id': 'email', 'requires': []}},
|
||||
}
|
||||
deps = composer._resolve_dependents('email', installed)
|
||||
self.assertEqual(deps, [])
|
||||
|
||||
def test_resolve_dependents_found(self):
|
||||
composer = self._composer()
|
||||
installed = {
|
||||
'email': {'manifest': {'id': 'email', 'requires': []}},
|
||||
'webmail': {'manifest': {'id': 'webmail', 'requires': ['email']}},
|
||||
}
|
||||
deps = composer._resolve_dependents('email', installed)
|
||||
self.assertIn('webmail', deps)
|
||||
|
||||
def test_resolve_dependents_excludes_self(self):
|
||||
composer = self._composer()
|
||||
installed = {
|
||||
'email': {'manifest': {'id': 'email', 'requires': ['email']}}, # weird edge case
|
||||
}
|
||||
deps = composer._resolve_dependents('email', installed)
|
||||
self.assertNotIn('email', deps)
|
||||
|
||||
def test_resolve_dependents_empty_installed(self):
|
||||
composer = self._composer()
|
||||
deps = composer._resolve_dependents('email', {})
|
||||
self.assertEqual(deps, [])
|
||||
|
||||
def test_reapply_active_services_calls_up(self):
|
||||
cm = MagicMock()
|
||||
cm.get_installed_services.return_value = {'email': {'manifest': {'id': 'email'}}}
|
||||
composer = ServiceComposer(config_manager=cm, data_dir='/tmp/test')
|
||||
composer.has_compose_file = MagicMock(return_value=True)
|
||||
composer.up = MagicMock(return_value={'ok': True})
|
||||
composer.reapply_active_services()
|
||||
composer.up.assert_called_once_with('email')
|
||||
|
||||
def test_reapply_active_services_skips_missing_compose(self):
|
||||
cm = MagicMock()
|
||||
cm.get_installed_services.return_value = {'email': {'manifest': {'id': 'email'}}}
|
||||
composer = ServiceComposer(config_manager=cm, data_dir='/tmp/test')
|
||||
composer.has_compose_file = MagicMock(return_value=False)
|
||||
composer.up = MagicMock()
|
||||
composer.reapply_active_services()
|
||||
composer.up.assert_not_called()
|
||||
|
||||
def test_reapply_active_services_empty(self):
|
||||
cm = MagicMock()
|
||||
cm.get_installed_services.return_value = {}
|
||||
composer = ServiceComposer(config_manager=cm, data_dir='/tmp/test')
|
||||
composer.up = MagicMock()
|
||||
composer.reapply_active_services()
|
||||
composer.up.assert_not_called()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user