Compare commits
6 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 515f3d5075 | |||
| 35993bc79d | |||
| f1b48208fc | |||
| ffe1dbeed6 | |||
| 15376b67c7 | |||
| 8efe8c1225 |
+3
-1
@@ -21,8 +21,10 @@ config/api/caddy/Caddyfile
|
|||||||
config/api/calendar.json
|
config/api/calendar.json
|
||||||
config/api/cell_config.json
|
config/api/cell_config.json
|
||||||
config/api/wireguard.json
|
config/api/wireguard.json
|
||||||
config/api/webdav/webdav.conf
|
config/api/webdav/
|
||||||
config/api/dhcp/
|
config/api/dhcp/
|
||||||
|
config/api/dns/
|
||||||
|
config/api/network.json
|
||||||
config/caddy/Caddyfile
|
config/caddy/Caddyfile
|
||||||
config/dhcp/dnsmasq.conf
|
config/dhcp/dnsmasq.conf
|
||||||
config/dns/Corefile
|
config/dns/Corefile
|
||||||
|
|||||||
@@ -12,7 +12,8 @@
|
|||||||
test-e2e-deps test-e2e-api test-e2e-ui test-e2e-wg test-e2e \
|
test-e2e-deps test-e2e-api test-e2e-ui test-e2e-wg test-e2e \
|
||||||
reset-test-admin-pass \
|
reset-test-admin-pass \
|
||||||
show-admin-password reset-admin-password \
|
show-admin-password reset-admin-password \
|
||||||
show-routes add-peer list-peers
|
show-routes add-peer list-peers \
|
||||||
|
ddns-update ddns-register
|
||||||
|
|
||||||
# Detect docker compose command (v2 plugin preferred, fallback to v1 standalone)
|
# Detect docker compose command (v2 plugin preferred, fallback to v1 standalone)
|
||||||
DC := $(shell docker compose version >/dev/null 2>&1 && echo "docker compose" || echo "docker-compose")
|
DC := $(shell docker compose version >/dev/null 2>&1 && echo "docker compose" || echo "docker-compose")
|
||||||
@@ -335,6 +336,21 @@ add-peer:
|
|||||||
echo "Usage: make add-peer PEER_NAME=name PEER_IP=10.0.0.x PEER_KEY=<pubkey>"; \
|
echo "Usage: make add-peer PEER_NAME=name PEER_IP=10.0.0.x PEER_KEY=<pubkey>"; \
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# ── DDNS ─────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
ddns-update:
|
||||||
|
@python3 scripts/ddns_update.py
|
||||||
|
|
||||||
|
ddns-register:
|
||||||
|
@DDNS_TOTP_SECRET="$(DDNS_TOTP_SECRET)" python3 -c "\
|
||||||
|
import os, sys; sys.path.insert(0, 'scripts'); \
|
||||||
|
from setup_cell import register_with_ddns, _read_existing_ip_range; \
|
||||||
|
import json; \
|
||||||
|
cfg = json.load(open('config/api/cell_config.json')) if os.path.exists('config/api/cell_config.json') else {}; \
|
||||||
|
name = cfg.get('_identity', {}).get('cell_name', os.environ.get('CELL_NAME', 'mycell')); \
|
||||||
|
import os; os.remove('data/api/.ddns_token') if os.path.exists('data/api/.ddns_token') else None; \
|
||||||
|
register_with_ddns(name)"
|
||||||
|
|
||||||
# ── Dev ───────────────────────────────────────────────────────────────────────
|
# ── Dev ───────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
dev:
|
dev:
|
||||||
|
|||||||
@@ -1,535 +1,343 @@
|
|||||||
# Personal Internet Cell – Project Wiki
|
# Personal Internet Cell – Project Wiki
|
||||||
|
|
||||||
## 🌟 Overview
|
## Overview
|
||||||
|
|
||||||
Personal Internet Cell is a **production-grade, self-hosted, decentralized digital infrastructure** solution designed to provide individuals with full control over their digital services and data. The project has evolved from a phase-based implementation to a **unified, enterprise-ready system** with modern architecture, comprehensive testing, and production-grade features.
|
Personal Internet Cell (PIC) is a self-hosted digital infrastructure platform. It runs DNS, DHCP, NTP, WireGuard VPN, email, calendar/contacts, file storage, HTTPS reverse proxy, a certificate authority, and optional services — all managed from a single REST API and React web UI.
|
||||||
|
|
||||||
## 📋 Table of Contents
|
The goal is to give a person full ownership of their core internet services on their own hardware, without relying on cloud providers.
|
||||||
|
|
||||||
1. [Project Goals](#project-goals)
|
|
||||||
2. [Architecture & Components](#architecture--components)
|
|
||||||
3. [Service Manager Architecture](#service-manager-architecture)
|
|
||||||
4. [Core Services](#core-services)
|
|
||||||
5. [API Reference](#api-reference)
|
|
||||||
6. [Enhanced CLI](#enhanced-cli)
|
|
||||||
7. [Security Model](#security-model)
|
|
||||||
8. [Testing & Quality Assurance](#testing--quality-assurance)
|
|
||||||
9. [Usage Examples](#usage-examples)
|
|
||||||
10. [Development & Deployment](#development--deployment)
|
|
||||||
11. [Future Enhancements](#future-enhancements)
|
|
||||||
12. [Project Status](#project-status)
|
|
||||||
|
|
||||||
## 🎯 Project Goals
|
|
||||||
|
|
||||||
- **Self-Hosted**: Run your own digital services (email, calendar, files, VPN, etc.) on your hardware
|
|
||||||
- **Decentralized**: Peer-to-peer networking and trust, no central authority
|
|
||||||
- **Production-Grade**: Enterprise-ready architecture with comprehensive monitoring
|
|
||||||
- **Secure**: Modern cryptography, certificate management, and encrypted storage
|
|
||||||
- **User-Friendly**: Professional CLI and API for easy management
|
|
||||||
- **Extensible**: Modular architecture for future services and integrations
|
|
||||||
- **Event-Driven**: Real-time service communication and orchestration
|
|
||||||
|
|
||||||
## 🏗️ Architecture & Components
|
|
||||||
|
|
||||||
### **Modern Architecture Stack**
|
|
||||||
|
|
||||||
- **Backend**: Python (Flask) with production-grade service managers
|
|
||||||
- **Service Architecture**: BaseServiceManager pattern with unified interfaces
|
|
||||||
- **Event System**: Service bus for real-time communication and orchestration
|
|
||||||
- **Configuration**: Centralized configuration management with validation
|
|
||||||
- **Logging**: Structured JSON logging with rotation and search
|
|
||||||
- **Containerization**: Docker-based deployment and service isolation
|
|
||||||
- **API**: RESTful endpoints with comprehensive documentation
|
|
||||||
|
|
||||||
### **Core Architecture Components**
|
|
||||||
|
|
||||||
```
|
|
||||||
┌─────────────────────────────────────────────────────────────┐
|
|
||||||
│ Personal Internet Cell │
|
|
||||||
├─────────────────────────────────────────────────────────────┤
|
|
||||||
│ Enhanced CLI │ Web UI │ REST API │ Service Bus │ Logging │
|
|
||||||
├─────────────────────────────────────────────────────────────┤
|
|
||||||
│ Service Managers │
|
|
||||||
│ Network │ WireGuard │ Email │ Calendar │ Files │ Routing │
|
|
||||||
│ Vault │ Container │ Cell │ Peer │ │ │
|
|
||||||
├─────────────────────────────────────────────────────────────┤
|
|
||||||
│ Core Infrastructure │
|
|
||||||
│ DNS │ DHCP │ NTP │ VPN │ CA │ Encryption │ Trust │ Storage │
|
|
||||||
└─────────────────────────────────────────────────────────────┘
|
|
||||||
```
|
|
||||||
|
|
||||||
## 🔧 Service Manager Architecture
|
|
||||||
|
|
||||||
### **BaseServiceManager Pattern**
|
|
||||||
|
|
||||||
All services inherit from `BaseServiceManager`, providing:
|
|
||||||
|
|
||||||
```python
|
|
||||||
class BaseServiceManager(ABC):
|
|
||||||
def __init__(self, service_name: str, data_dir: str, config_dir: str)
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def get_status(self) -> Dict[str, Any]
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def test_connectivity(self) -> Dict[str, Any]
|
|
||||||
|
|
||||||
# Common methods
|
|
||||||
def get_logs(self, lines: int = 50) -> List[str]
|
|
||||||
def restart_service(self) -> bool
|
|
||||||
def get_config(self) -> Dict[str, Any]
|
|
||||||
def update_config(self, config: Dict[str, Any]) -> bool
|
|
||||||
def health_check(self) -> Dict[str, Any]
|
|
||||||
def handle_error(self, error: Exception, context: str) -> Dict[str, Any]
|
|
||||||
```
|
|
||||||
|
|
||||||
### **Service Bus Integration**
|
|
||||||
|
|
||||||
```python
|
|
||||||
# Event-driven service communication
|
|
||||||
service_bus.register_service('network', network_manager)
|
|
||||||
service_bus.register_service('wireguard', wireguard_manager)
|
|
||||||
service_bus.publish_event(EventType.SERVICE_STARTED, 'network', data)
|
|
||||||
|
|
||||||
# Service dependencies
|
|
||||||
service_dependencies = {
|
|
||||||
'wireguard': ['network'],
|
|
||||||
'email': ['network', 'vault'],
|
|
||||||
'calendar': ['network', 'vault'],
|
|
||||||
'files': ['network', 'vault'],
|
|
||||||
'routing': ['network', 'wireguard'],
|
|
||||||
'vault': ['network']
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## 🔧 Core Services
|
|
||||||
|
|
||||||
### **Network Services**
|
|
||||||
- **NetworkManager**: DNS, DHCP, NTP with dynamic management
|
|
||||||
- Dynamic zone file generation
|
|
||||||
- DHCP lease monitoring
|
|
||||||
- Network connectivity testing
|
|
||||||
- Service health monitoring
|
|
||||||
|
|
||||||
### **VPN & Mesh Networking**
|
|
||||||
- **WireGuardManager**: WireGuard VPN configuration and peer management
|
|
||||||
- Key generation and management
|
|
||||||
- Peer configuration
|
|
||||||
- Connectivity testing
|
|
||||||
- Dynamic IP updates
|
|
||||||
|
|
||||||
- **PeerRegistry**: Peer registration and trust management
|
|
||||||
- Peer lifecycle management
|
|
||||||
- Trust relationship tracking
|
|
||||||
- Data integrity validation
|
|
||||||
- Peer statistics
|
|
||||||
|
|
||||||
### **Digital Services**
|
|
||||||
- **EmailManager**: SMTP/IMAP email services
|
|
||||||
- User account management
|
|
||||||
- Mailbox configuration
|
|
||||||
- Service connectivity testing
|
|
||||||
- Email delivery monitoring
|
|
||||||
|
|
||||||
- **CalendarManager**: CalDAV/CardDAV calendar and contacts
|
|
||||||
- User and calendar management
|
|
||||||
- Event synchronization
|
|
||||||
- Service health monitoring
|
|
||||||
- Connectivity testing
|
|
||||||
|
|
||||||
- **FileManager**: WebDAV file storage
|
|
||||||
- User directory management
|
|
||||||
- Storage quota monitoring
|
|
||||||
- File system access testing
|
|
||||||
- Backup and restore capabilities
|
|
||||||
|
|
||||||
### **Infrastructure Services**
|
|
||||||
- **RoutingManager**: Advanced routing and NAT
|
|
||||||
- NAT rule management
|
|
||||||
- Firewall configuration
|
|
||||||
- Exit node routing
|
|
||||||
- Bridge and split routing
|
|
||||||
- Connectivity testing
|
|
||||||
|
|
||||||
- **VaultManager**: Security and trust management
|
|
||||||
- Self-hosted Certificate Authority
|
|
||||||
- Certificate lifecycle management
|
|
||||||
- Age/Fernet encryption
|
|
||||||
- Trust relationship management
|
|
||||||
- Cryptographic verification
|
|
||||||
|
|
||||||
- **ContainerManager**: Docker orchestration
|
|
||||||
- Container lifecycle management
|
|
||||||
- Image and volume management
|
|
||||||
- Docker daemon connectivity
|
|
||||||
- Service isolation
|
|
||||||
|
|
||||||
- **CellManager**: Overall cell orchestration
|
|
||||||
- Service coordination
|
|
||||||
- Health monitoring
|
|
||||||
- Configuration management
|
|
||||||
- Peer management
|
|
||||||
|
|
||||||
## 📡 API Reference
|
|
||||||
|
|
||||||
### **Core API Endpoints**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Service Status and Health
|
|
||||||
GET /api/services/status # All services status
|
|
||||||
GET /api/services/connectivity # Service connectivity tests
|
|
||||||
GET /health # API health check
|
|
||||||
|
|
||||||
# Configuration Management
|
|
||||||
GET /api/config # Get configuration
|
|
||||||
PUT /api/config # Update configuration
|
|
||||||
POST /api/config/backup # Create backup
|
|
||||||
GET /api/config/backups # List backups
|
|
||||||
POST /api/config/restore/<id> # Restore backup
|
|
||||||
GET /api/config/export # Export configuration
|
|
||||||
POST /api/config/import # Import configuration
|
|
||||||
|
|
||||||
# Service Bus
|
|
||||||
GET /api/services/bus/status # Service bus status
|
|
||||||
GET /api/services/bus/events # Event history
|
|
||||||
POST /api/services/bus/services/<service>/start
|
|
||||||
POST /api/services/bus/services/<service>/stop
|
|
||||||
POST /api/services/bus/services/<service>/restart
|
|
||||||
|
|
||||||
# Logging
|
|
||||||
GET /api/logs/services/<service> # Service logs
|
|
||||||
POST /api/logs/search # Log search
|
|
||||||
POST /api/logs/export # Log export
|
|
||||||
GET /api/logs/statistics # Log statistics
|
|
||||||
POST /api/logs/rotate # Log rotation
|
|
||||||
```
|
|
||||||
|
|
||||||
### **Service-Specific Endpoints**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Network Services
|
|
||||||
GET /api/dns/records # DNS records
|
|
||||||
POST /api/dns/records # Add DNS record
|
|
||||||
DELETE /api/dns/records # Remove DNS record
|
|
||||||
GET /api/dhcp/leases # DHCP leases
|
|
||||||
POST /api/dhcp/reservations # Add DHCP reservation
|
|
||||||
GET /api/ntp/status # NTP status
|
|
||||||
GET /api/network/info # Network information
|
|
||||||
POST /api/network/test # Network connectivity test
|
|
||||||
|
|
||||||
# WireGuard & Peers
|
|
||||||
GET /api/wireguard/keys # WireGuard keys
|
|
||||||
POST /api/wireguard/keys/peer # Generate peer keys
|
|
||||||
GET /api/wireguard/config # WireGuard configuration
|
|
||||||
GET /api/wireguard/peers # List peers
|
|
||||||
POST /api/wireguard/peers # Add peer
|
|
||||||
DELETE /api/wireguard/peers # Remove peer
|
|
||||||
GET /api/wireguard/status # WireGuard status
|
|
||||||
POST /api/wireguard/connectivity # Connectivity test
|
|
||||||
PUT /api/wireguard/peers/ip # Update peer IP
|
|
||||||
|
|
||||||
# Digital Services
|
|
||||||
GET /api/email/users # Email users
|
|
||||||
POST /api/email/users # Add email user
|
|
||||||
DELETE /api/email/users/<user> # Remove email user
|
|
||||||
GET /api/email/status # Email service status
|
|
||||||
GET /api/email/connectivity # Email connectivity
|
|
||||||
POST /api/email/send # Send email
|
|
||||||
GET /api/email/mailbox/<user> # User mailbox
|
|
||||||
|
|
||||||
GET /api/calendar/users # Calendar users
|
|
||||||
POST /api/calendar/users # Add calendar user
|
|
||||||
DELETE /api/calendar/users/<user> # Remove calendar user
|
|
||||||
POST /api/calendar/calendars # Create calendar
|
|
||||||
POST /api/calendar/events # Add event
|
|
||||||
GET /api/calendar/events/<user>/<calendar> # List events
|
|
||||||
GET /api/calendar/status # Calendar service status
|
|
||||||
GET /api/calendar/connectivity # Calendar connectivity
|
|
||||||
|
|
||||||
GET /api/files/users # File users
|
|
||||||
POST /api/files/users # Add file user
|
|
||||||
DELETE /api/files/users/<user> # Remove file user
|
|
||||||
POST /api/files/folders # Create folder
|
|
||||||
DELETE /api/files/folders/<user>/<path> # Remove folder
|
|
||||||
POST /api/files/upload/<user> # Upload file
|
|
||||||
GET /api/files/download/<user>/<path> # Download file
|
|
||||||
DELETE /api/files/delete/<user>/<path> # Delete file
|
|
||||||
GET /api/files/list/<user> # List files
|
|
||||||
GET /api/files/status # File service status
|
|
||||||
GET /api/files/connectivity # File connectivity
|
|
||||||
|
|
||||||
# Routing & Security
|
|
||||||
GET /api/routing/status # Routing status
|
|
||||||
POST /api/routing/nat # Add NAT rule
|
|
||||||
DELETE /api/routing/nat/<id> # Remove NAT rule
|
|
||||||
POST /api/routing/peers # Add peer route
|
|
||||||
DELETE /api/routing/peers/<peer> # Remove peer route
|
|
||||||
POST /api/routing/exit-nodes # Add exit node
|
|
||||||
POST /api/routing/bridge # Add bridge route
|
|
||||||
POST /api/routing/split # Add split route
|
|
||||||
POST /api/routing/firewall # Add firewall rule
|
|
||||||
POST /api/routing/connectivity # Routing connectivity test
|
|
||||||
GET /api/routing/logs # Routing logs
|
|
||||||
GET /api/routing/nat # List NAT rules
|
|
||||||
GET /api/routing/peers # List peer routes
|
|
||||||
GET /api/routing/firewall # List firewall rules
|
|
||||||
|
|
||||||
GET /api/vault/status # Vault status
|
|
||||||
GET /api/vault/certificates # List certificates
|
|
||||||
POST /api/vault/certificates # Generate certificate
|
|
||||||
DELETE /api/vault/certificates/<name> # Revoke certificate
|
|
||||||
GET /api/vault/ca/certificate # CA certificate
|
|
||||||
GET /api/vault/age/public-key # Age public key
|
|
||||||
GET /api/vault/trust/keys # Trusted keys
|
|
||||||
POST /api/vault/trust/keys # Add trusted key
|
|
||||||
DELETE /api/vault/trust/keys/<name> # Remove trusted key
|
|
||||||
POST /api/vault/trust/verify # Verify trust
|
|
||||||
GET /api/vault/trust/chains # Trust chains
|
|
||||||
```
|
|
||||||
|
|
||||||
## 💻 Enhanced CLI
|
|
||||||
|
|
||||||
### **CLI Features**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Interactive mode with tab completion
|
|
||||||
python api/enhanced_cli.py --interactive
|
|
||||||
|
|
||||||
# Batch operations
|
|
||||||
python api/enhanced_cli.py --batch "status" "services" "health"
|
|
||||||
|
|
||||||
# Configuration management
|
|
||||||
python api/enhanced_cli.py --export-config json
|
|
||||||
python api/enhanced_cli.py --import-config config.json
|
|
||||||
|
|
||||||
# Service wizards
|
|
||||||
python api/enhanced_cli.py --wizard network
|
|
||||||
python api/enhanced_cli.py --wizard email
|
|
||||||
|
|
||||||
# Health monitoring
|
|
||||||
python api/enhanced_cli.py --health
|
|
||||||
python api/enhanced_cli.py --logs network
|
|
||||||
|
|
||||||
# Service status
|
|
||||||
python api/enhanced_cli.py --status
|
|
||||||
python api/enhanced_cli.py --services
|
|
||||||
python api/enhanced_cli.py --peers
|
|
||||||
```
|
|
||||||
|
|
||||||
### **CLI Capabilities**
|
|
||||||
- **Interactive Mode**: Tab completion, command history, help system
|
|
||||||
- **Batch Operations**: Execute multiple commands in sequence
|
|
||||||
- **Configuration Wizards**: Guided setup for complex services
|
|
||||||
- **Real-time Monitoring**: Live status updates and health checks
|
|
||||||
- **Log Management**: View, search, and export service logs
|
|
||||||
- **Service Management**: Start, stop, restart, and configure services
|
|
||||||
|
|
||||||
## 🔒 Security Model
|
|
||||||
|
|
||||||
### **Certificate Management**
|
|
||||||
- **Self-hosted CA**: Issue and manage TLS certificates for all services
|
|
||||||
- **Certificate Lifecycle**: Generate, renew, revoke, and monitor certificates
|
|
||||||
- **Trust Management**: Direct, indirect, and verified trust relationships
|
|
||||||
- **Age Encryption**: Modern encryption for sensitive data and keys
|
|
||||||
|
|
||||||
### **Network Security**
|
|
||||||
- **WireGuard VPN**: Secure peer-to-peer communication with key rotation
|
|
||||||
- **Firewall & NAT**: Granular control over network access and routing
|
|
||||||
- **Service Isolation**: Docker containers for each service
|
|
||||||
- **Input Validation**: All API endpoints validate and sanitize input
|
|
||||||
|
|
||||||
### **Data Protection**
|
|
||||||
- **Encrypted Storage**: Sensitive data encrypted at rest using Age/Fernet
|
|
||||||
- **Secure Communication**: TLS for all API endpoints and service communication
|
|
||||||
- **Access Control**: Role-based access for services and API endpoints
|
|
||||||
- **Audit Logging**: Comprehensive security event logging and monitoring
|
|
||||||
|
|
||||||
## 🧪 Testing & Quality Assurance
|
|
||||||
|
|
||||||
### **Test Coverage**
|
|
||||||
- **BaseServiceManager**: 100% coverage
|
|
||||||
- **ConfigManager**: 95%+ coverage
|
|
||||||
- **ServiceBus**: 95%+ coverage
|
|
||||||
- **LogManager**: 95%+ coverage
|
|
||||||
- **All Service Managers**: 77%+ overall coverage
|
|
||||||
- **API Endpoints**: 100% endpoint coverage
|
|
||||||
|
|
||||||
### **Test Types**
|
|
||||||
- **Unit Tests**: Individual component testing
|
|
||||||
- **Integration Tests**: Service interaction testing
|
|
||||||
- **API Tests**: Endpoint functionality testing
|
|
||||||
- **Error Handling**: Exception and edge case testing
|
|
||||||
- **Performance Tests**: Load and stress testing
|
|
||||||
|
|
||||||
### **Testing Commands**
|
|
||||||
```bash
|
|
||||||
# Run all tests
|
|
||||||
python api/test_enhanced_api.py
|
|
||||||
|
|
||||||
# Run specific test suites
|
|
||||||
python -m pytest api/tests/test_network_manager.py
|
|
||||||
python -m pytest api/tests/test_service_bus.py
|
|
||||||
|
|
||||||
# Generate coverage report
|
|
||||||
coverage run -m pytest api/tests/
|
|
||||||
coverage html
|
|
||||||
```
|
|
||||||
|
|
||||||
## 📝 Usage Examples
|
|
||||||
|
|
||||||
### **Add DNS Record**
|
|
||||||
```bash
|
|
||||||
curl -X POST http://localhost:3000/api/dns/records \
|
|
||||||
-H "Content-Type: application/json" \
|
|
||||||
-d '{
|
|
||||||
"name": "www",
|
|
||||||
"type": "A",
|
|
||||||
"value": "192.168.1.100",
|
|
||||||
"ttl": 300
|
|
||||||
}'
|
|
||||||
```
|
|
||||||
|
|
||||||
### **Register Peer**
|
|
||||||
```bash
|
|
||||||
curl -X POST http://localhost:3000/api/wireguard/peers \
|
|
||||||
-H "Content-Type: application/json" \
|
|
||||||
-d '{
|
|
||||||
"name": "bob",
|
|
||||||
"ip": "203.0.113.22",
|
|
||||||
"public_key": "peer_public_key_here",
|
|
||||||
"allowed_networks": ["10.0.0.0/24"]
|
|
||||||
}'
|
|
||||||
```
|
|
||||||
|
|
||||||
### **Generate Certificate**
|
|
||||||
```bash
|
|
||||||
curl -X POST http://localhost:3000/api/vault/certificates \
|
|
||||||
-H "Content-Type: application/json" \
|
|
||||||
-d '{
|
|
||||||
"common_name": "myapp.example.com",
|
|
||||||
"domains": ["myapp.example.com", "www.myapp.example.com"],
|
|
||||||
"days": 365
|
|
||||||
}'
|
|
||||||
```
|
|
||||||
|
|
||||||
### **Configure NAT Rule**
|
|
||||||
```bash
|
|
||||||
curl -X POST http://localhost:3000/api/routing/nat \
|
|
||||||
-H "Content-Type: application/json" \
|
|
||||||
-d '{
|
|
||||||
"source_network": "10.0.0.0/24",
|
|
||||||
"target_interface": "eth0",
|
|
||||||
"nat_type": "MASQUERADE",
|
|
||||||
"protocol": "ALL"
|
|
||||||
}'
|
|
||||||
```
|
|
||||||
|
|
||||||
## 🛠️ Development & Deployment
|
|
||||||
|
|
||||||
### **Development Setup**
|
|
||||||
```bash
|
|
||||||
# Install dependencies
|
|
||||||
pip install -r api/requirements.txt
|
|
||||||
|
|
||||||
# Start development server
|
|
||||||
python api/app.py
|
|
||||||
|
|
||||||
# Run tests
|
|
||||||
python api/test_enhanced_api.py
|
|
||||||
|
|
||||||
# Start frontend (if available)
|
|
||||||
cd webui && bun install && npm run dev
|
|
||||||
```
|
|
||||||
|
|
||||||
### **Production Deployment**
|
|
||||||
```bash
|
|
||||||
# Docker deployment
|
|
||||||
docker-compose up --build -d
|
|
||||||
|
|
||||||
# Health check
|
|
||||||
curl http://localhost:3000/health
|
|
||||||
|
|
||||||
# Service status
|
|
||||||
curl http://localhost:3000/api/services/status
|
|
||||||
```
|
|
||||||
|
|
||||||
### **Service Development**
|
|
||||||
```python
|
|
||||||
from base_service_manager import BaseServiceManager
|
|
||||||
|
|
||||||
class MyServiceManager(BaseServiceManager):
|
|
||||||
def __init__(self, data_dir='/app/data', config_dir='/app/config'):
|
|
||||||
super().__init__('myservice', data_dir, config_dir)
|
|
||||||
|
|
||||||
def get_status(self) -> Dict[str, Any]:
|
|
||||||
# Implement service status
|
|
||||||
return {
|
|
||||||
'running': True,
|
|
||||||
'status': 'online',
|
|
||||||
'timestamp': datetime.utcnow().isoformat()
|
|
||||||
}
|
|
||||||
|
|
||||||
def test_connectivity(self) -> Dict[str, Any]:
|
|
||||||
# Implement connectivity test
|
|
||||||
return {
|
|
||||||
'success': True,
|
|
||||||
'message': 'Service connectivity working',
|
|
||||||
'timestamp': datetime.utcnow().isoformat()
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## 🚀 Future Enhancements
|
|
||||||
|
|
||||||
### **Planned Features**
|
|
||||||
- **Certificate Auto-renewal**: Automatic certificate renewal and monitoring
|
|
||||||
- **Web of Trust Models**: Advanced trust relationship management
|
|
||||||
- **Certificate Transparency**: CT log integration and monitoring
|
|
||||||
- **Hardware Security Module (HSM)**: HSM integration for key management
|
|
||||||
- **WebSocket Updates**: Real-time service status updates
|
|
||||||
- **Advanced Monitoring**: Metrics collection and alerting systems
|
|
||||||
- **Mobile App**: Mobile application for remote management
|
|
||||||
- **Plugin System**: Extensible architecture for custom services
|
|
||||||
|
|
||||||
### **Architecture Improvements**
|
|
||||||
- **Service Discovery**: Dynamic service registration and discovery
|
|
||||||
- **Load Balancing**: Multi-instance service deployment
|
|
||||||
- **Advanced Caching**: Redis-based caching for performance
|
|
||||||
- **Message Queues**: RabbitMQ/Kafka for reliable messaging
|
|
||||||
- **Distributed Tracing**: OpenTelemetry integration
|
|
||||||
- **Configuration Management**: GitOps-style configuration management
|
|
||||||
|
|
||||||
## 📊 Project Status
|
|
||||||
|
|
||||||
### **✅ Completed Features**
|
|
||||||
- **Production-Grade Architecture**: BaseServiceManager pattern implemented
|
|
||||||
- **Event-Driven Communication**: Service bus with real-time events
|
|
||||||
- **Centralized Configuration**: Type-safe configuration with validation
|
|
||||||
- **Comprehensive Logging**: Structured logging with search and export
|
|
||||||
- **Enhanced CLI**: Interactive CLI with batch operations
|
|
||||||
- **Health Monitoring**: Real-time health checks across all services
|
|
||||||
- **Security Framework**: Self-hosted CA, encryption, and trust management
|
|
||||||
- **Complete API**: RESTful API with comprehensive documentation
|
|
||||||
- **Testing Framework**: Comprehensive test suite with high coverage
|
|
||||||
|
|
||||||
### **🎯 Current Status**
|
|
||||||
- **All Services**: 10 service managers fully implemented and integrated
|
|
||||||
- **API Server**: Running on port 3000 with all endpoints functional
|
|
||||||
- **CLI Tool**: Enhanced CLI with all features working
|
|
||||||
- **Test Coverage**: 77%+ overall coverage with comprehensive testing
|
|
||||||
- **Documentation**: Complete documentation for all components
|
|
||||||
- **Production Ready**: Suitable for personal and small business deployment
|
|
||||||
|
|
||||||
### **🌟 Key Achievements**
|
|
||||||
- **Unified Architecture**: All services follow the same patterns and interfaces
|
|
||||||
- **Event-Driven Design**: Services communicate and orchestrate automatically
|
|
||||||
- **Configuration Management**: Centralized, validated configuration system
|
|
||||||
- **Comprehensive Logging**: Production-grade logging with advanced features
|
|
||||||
- **Enhanced CLI**: Professional command-line interface for management
|
|
||||||
- **Health Monitoring**: Real-time monitoring and alerting capabilities
|
|
||||||
- **Security Framework**: Enterprise-grade security with modern cryptography
|
|
||||||
- **Complete Testing**: Comprehensive test suite ensuring reliability
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
**The Personal Internet Cell empowers users with full control over their digital infrastructure, combining privacy, security, and usability in a single, production-ready, self-hosted platform.** 🌟
|
## Table of Contents
|
||||||
|
|
||||||
|
1. [Architecture](#architecture)
|
||||||
|
2. [Service Managers](#service-managers)
|
||||||
|
3. [First-Run Wizard](#first-run-wizard)
|
||||||
|
4. [Authentication](#authentication)
|
||||||
|
5. [API Reference](#api-reference)
|
||||||
|
6. [DDNS](#ddns)
|
||||||
|
7. [Service Store](#service-store)
|
||||||
|
8. [Cell-to-Cell Networking](#cell-to-cell-networking)
|
||||||
|
9. [Extended Connectivity](#extended-connectivity)
|
||||||
|
10. [Security Model](#security-model)
|
||||||
|
11. [Testing](#testing)
|
||||||
|
12. [Development](#development)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Architecture
|
||||||
|
|
||||||
|
```
|
||||||
|
Browser / WireGuard peer
|
||||||
|
└── Caddy (:80/:443) reverse proxy, TLS termination
|
||||||
|
└── React SPA (:8081) Vite + Tailwind (Nginx in container)
|
||||||
|
└── Flask API (:3000) REST API, bound to 127.0.0.1
|
||||||
|
├── NetworkManager CoreDNS, dnsmasq, chrony
|
||||||
|
├── WireGuardManager WireGuard VPN peer lifecycle
|
||||||
|
├── PeerRegistry peer registration and trust
|
||||||
|
├── EmailManager Postfix + Dovecot
|
||||||
|
├── CalendarManager Radicale CalDAV/CardDAV
|
||||||
|
├── FileManager WebDAV + Filegator
|
||||||
|
├── RoutingManager iptables NAT and routing
|
||||||
|
├── FirewallManager iptables firewall rules
|
||||||
|
├── VaultManager internal CA, cert lifecycle, Age encryption
|
||||||
|
├── ContainerManager Docker SDK
|
||||||
|
├── CellLinkManager cell-to-cell WireGuard links
|
||||||
|
├── ConnectivityManager exit routing (WG ext, OpenVPN, Tor)
|
||||||
|
├── DDNSManager dynamic DNS heartbeat
|
||||||
|
├── ServiceStoreManager optional service install/remove
|
||||||
|
├── CaddyManager Caddyfile generation and reload
|
||||||
|
├── AuthManager session auth, RBAC
|
||||||
|
└── SetupManager first-run wizard state
|
||||||
|
```
|
||||||
|
|
||||||
|
All 12 service containers run on a Docker bridge network (`cell-network`, `172.20.0.0/16` default). Static IPs per container are defined in `docker-compose.yml`.
|
||||||
|
|
||||||
|
Runtime configuration lives in `config/api/cell_config.json`, managed by `ConfigManager`. All service managers read and write through `ConfigManager`, which validates and backs up automatically.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Service Managers
|
||||||
|
|
||||||
|
All managers inherit `BaseServiceManager` (`api/base_service_manager.py`), which provides:
|
||||||
|
- `get_status()` — current running state
|
||||||
|
- `get_config()` / `update_config()` — config read/write
|
||||||
|
- `test_connectivity()` — reachability check
|
||||||
|
- `get_logs()` — last N lines from the service log
|
||||||
|
- `restart_service()` — container restart via Docker SDK
|
||||||
|
|
||||||
|
The `ServiceBus` (`api/service_bus.py`) handles pub/sub events between managers (e.g., `CONFIG_CHANGED`, `SERVICE_STARTED`). Dependencies are declared in the bus (wireguard depends on network; email depends on network and vault).
|
||||||
|
|
||||||
|
### Manager summary
|
||||||
|
|
||||||
|
| Manager | Responsibilities |
|
||||||
|
|---|---|
|
||||||
|
| `NetworkManager` | CoreDNS zone files, dnsmasq DHCP config and lease monitoring, chrony NTP |
|
||||||
|
| `WireGuardManager` | Key generation, `wg0.conf` generation, peer add/remove, route sync |
|
||||||
|
| `PeerRegistry` | Peer registration, trust tracking, peer statistics |
|
||||||
|
| `EmailManager` | docker-mailserver accounts, mailbox config, alias management |
|
||||||
|
| `CalendarManager` | Radicale user/calendar/contacts lifecycle |
|
||||||
|
| `FileManager` | WebDAV user directories, Filegator access |
|
||||||
|
| `RoutingManager` | NAT rules, per-peer routing policy, fwmark-based exit routing |
|
||||||
|
| `FirewallManager` | iptables INPUT/FORWARD/OUTPUT rule management |
|
||||||
|
| `VaultManager` | Internal CA (self-signed root), TLS cert issue/revoke, Age public key |
|
||||||
|
| `ContainerManager` | Docker container/image/volume management via SDK |
|
||||||
|
| `CellLinkManager` | Site-to-site WireGuard links to other PIC cells, peer-sync protocol |
|
||||||
|
| `ConnectivityManager` | Per-peer exit routing via WireGuard external, OpenVPN, or Tor |
|
||||||
|
| `DDNSManager` | Public IP heartbeat, provider abstraction (pic_ngo, cloudflare, duckdns, noip, freedns) |
|
||||||
|
| `ServiceStoreManager` | Fetch manifest index, install/remove optional services |
|
||||||
|
| `CaddyManager` | Caddyfile generation, reload-on-change |
|
||||||
|
| `AuthManager` | bcrypt password store, session management, admin/peer RBAC |
|
||||||
|
| `SetupManager` | First-run wizard state, setup-complete flag |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## First-Run Wizard
|
||||||
|
|
||||||
|
On first start, `SetupManager.is_setup_complete()` returns `False`. The `enforce_setup` before-request hook returns HTTP 428 for all `/api/*` requests except `/api/setup/*` and `/health`, redirecting clients to `/setup`.
|
||||||
|
|
||||||
|
The wizard collects:
|
||||||
|
- **Cell name** — used for hostnames and DDNS subdomain (e.g. `myhome` → `myhome.pic.ngo`)
|
||||||
|
- **Domain mode** — determines TLS certificate source: `lan` (internal CA), `pic_ngo`, `cloudflare`, `duckdns`, `http01`
|
||||||
|
- **Timezone**
|
||||||
|
- **Initial services to enable**
|
||||||
|
- **Admin password** — minimum 12 characters
|
||||||
|
|
||||||
|
On completion:
|
||||||
|
1. Admin account is created in `data/auth_users.json`
|
||||||
|
2. Cell identity is written to `config/api/cell_config.json`
|
||||||
|
3. Caddy config is generated
|
||||||
|
4. If domain mode is `pic_ngo`, the cell registers `<name>.pic.ngo` with the DDNS service
|
||||||
|
|
||||||
|
Wizard endpoints: `GET/POST /api/setup/step`, `GET /api/setup/status`, `POST /api/setup/complete`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Authentication
|
||||||
|
|
||||||
|
`AuthManager` stores bcrypt-hashed credentials in `data/auth_users.json`. Two roles:
|
||||||
|
|
||||||
|
| Role | Access |
|
||||||
|
|---|---|
|
||||||
|
| `admin` | All `/api/*` endpoints except `/api/peer/*` |
|
||||||
|
| `peer` | `/api/peer/*` only (peer dashboard, key exchange) |
|
||||||
|
|
||||||
|
Session auth flow:
|
||||||
|
- `POST /api/auth/login` — creates a Flask session
|
||||||
|
- `GET /api/auth/me` — current session info
|
||||||
|
- `POST /api/auth/logout` — clears session
|
||||||
|
- `POST /api/auth/change-password` — change own password
|
||||||
|
- `POST /api/auth/admin/reset-password` — admin resets another user's password
|
||||||
|
|
||||||
|
CSRF protection: all `POST`, `PUT`, `DELETE`, `PATCH` on `/api/*` (except `/api/auth/*` and `/api/setup/*`) require the `X-CSRF-Token` header matching the session token, obtained via `GET /api/auth/csrf-token`.
|
||||||
|
|
||||||
|
Cell-to-cell peer-sync endpoints (`/api/cells/peer-sync/*`) use source-IP + WireGuard public key auth, not session cookies.
|
||||||
|
|
||||||
|
Auth enforcement is active once any user exists in the store. If the store is empty (fresh install before wizard), all requests bypass auth — `enforce_setup` already blocks them with 428.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## API Reference
|
||||||
|
|
||||||
|
**Base URL:** `http://localhost:3000`
|
||||||
|
**Auth:** session cookie (`X-CSRF-Token` header required for mutations)
|
||||||
|
|
||||||
|
### Core
|
||||||
|
|
||||||
|
| Method | Path | Description |
|
||||||
|
|---|---|---|
|
||||||
|
| GET | `/health` | Health check (always public) |
|
||||||
|
| GET | `/api/status` | All-service status summary |
|
||||||
|
| GET | `/api/config` | Full cell config |
|
||||||
|
| PUT | `/api/config` | Update cell config |
|
||||||
|
| GET | `/api/health/history` | Recent health check history |
|
||||||
|
|
||||||
|
### Auth (`/api/auth/`)
|
||||||
|
|
||||||
|
| Method | Path | Description |
|
||||||
|
|---|---|---|
|
||||||
|
| POST | `/api/auth/login` | Create session |
|
||||||
|
| POST | `/api/auth/logout` | Destroy session |
|
||||||
|
| GET | `/api/auth/me` | Current user info |
|
||||||
|
| GET | `/api/auth/csrf-token` | Get CSRF token |
|
||||||
|
| POST | `/api/auth/change-password` | Change own password |
|
||||||
|
| POST | `/api/auth/admin/reset-password` | Admin: reset another user's password |
|
||||||
|
| GET | `/api/auth/users` | Admin: list users |
|
||||||
|
|
||||||
|
### Setup (`/api/setup/`)
|
||||||
|
|
||||||
|
| Method | Path | Description |
|
||||||
|
|---|---|---|
|
||||||
|
| GET | `/api/setup/status` | Setup complete flag + current step |
|
||||||
|
| GET | `/api/setup/step` | Current wizard step data |
|
||||||
|
| POST | `/api/setup/step` | Submit current step |
|
||||||
|
| POST | `/api/setup/complete` | Finalize setup |
|
||||||
|
|
||||||
|
### Network Services (`/api/dns/`, `/api/dhcp/`, `/api/ntp/`, `/api/network/`)
|
||||||
|
|
||||||
|
DNS records, DHCP leases and reservations, NTP status, network connectivity test.
|
||||||
|
|
||||||
|
### WireGuard (`/api/wireguard/`, `/api/peers/`)
|
||||||
|
|
||||||
|
Peer add/remove, key generation, QR code export, per-peer routing policy, WireGuard status.
|
||||||
|
|
||||||
|
### Email (`/api/email/`)
|
||||||
|
|
||||||
|
User account management, mailbox config, alias management, connectivity test.
|
||||||
|
|
||||||
|
### Calendar (`/api/calendar/`)
|
||||||
|
|
||||||
|
User, calendar, and contacts (CardDAV) management.
|
||||||
|
|
||||||
|
### Files (`/api/files/`)
|
||||||
|
|
||||||
|
WebDAV user management, file upload/download/delete, folder management.
|
||||||
|
|
||||||
|
### Routing (`/api/routing/`)
|
||||||
|
|
||||||
|
NAT rules, peer routes, exit node configuration.
|
||||||
|
|
||||||
|
### Vault (`/api/vault/`)
|
||||||
|
|
||||||
|
Certificate issue/revoke, CA certificate, trust key management, Age public key.
|
||||||
|
|
||||||
|
### Containers (`/api/containers/`)
|
||||||
|
|
||||||
|
List, start, stop, inspect containers; manage images and volumes.
|
||||||
|
|
||||||
|
### Cell Network (`/api/cells/`)
|
||||||
|
|
||||||
|
List connected cells, add/remove cell links, peer-sync.
|
||||||
|
|
||||||
|
### Connectivity (`/api/connectivity/`)
|
||||||
|
|
||||||
|
List exit nodes, configure WireGuard external / OpenVPN / Tor exits, assign per-peer exit policy.
|
||||||
|
|
||||||
|
### Service Store (`/api/store/`)
|
||||||
|
|
||||||
|
List available services, install, remove.
|
||||||
|
|
||||||
|
### Logs (`/api/logs/`)
|
||||||
|
|
||||||
|
Per-service log retrieval, log search, log statistics.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## DDNS
|
||||||
|
|
||||||
|
`DDNSManager` maintains a `<cell-name>.pic.ngo` DNS A record pointing at the cell's public IP. A background thread runs every 5 minutes and calls `provider.update(token, ip)` only when the IP changes.
|
||||||
|
|
||||||
|
Registration happens during the setup wizard (if domain mode is `pic_ngo`) via `provider.register(name, ip)`, which returns a bearer token stored in `data/api/.ddns_token`.
|
||||||
|
|
||||||
|
DDNS config lives in `cell_config.json` under the top-level `ddns` key:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"ddns": {
|
||||||
|
"provider": "pic_ngo",
|
||||||
|
"api_base_url": "https://ddns.pic.ngo",
|
||||||
|
"totp_secret": "<base32 secret>"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Registration requires a time-based OTP (`X-Register-OTP` header) derived from the shared `REGISTER_TOTP_SECRET` on the DDNS server. This prevents unauthorized subdomain registration.
|
||||||
|
|
||||||
|
Supported providers: `pic_ngo`, `cloudflare`, `duckdns`, `noip`, `freedns`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Service Store
|
||||||
|
|
||||||
|
`ServiceStoreManager` fetches a manifest index from `http://git.pic.ngo/roof/pic-services/raw/branch/main/index.json`. Each manifest declares:
|
||||||
|
- Container image
|
||||||
|
- Caddy routes (added to the Caddyfile)
|
||||||
|
- iptables rules
|
||||||
|
- Environment variables
|
||||||
|
- Health check endpoint
|
||||||
|
|
||||||
|
`POST /api/store/install` pulls the image, writes the Caddy route, applies iptables rules, and starts the container. `POST /api/store/remove` reverses this.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Cell-to-Cell Networking
|
||||||
|
|
||||||
|
`CellLinkManager` manages WireGuard site-to-site tunnels between PIC cells. Each link is a WireGuard peer configured with a dedicated `/32` address and allowed-IPs covering the remote cell's subnet.
|
||||||
|
|
||||||
|
The peer-sync protocol (`/api/cells/peer-sync/`) exchanges public keys and allowed networks between cells using source-IP + WireGuard public key authentication (no session required).
|
||||||
|
|
||||||
|
Access control is per-service (calendar, files, mail, WebDAV) and enforced at the iptables level.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Extended Connectivity
|
||||||
|
|
||||||
|
`ConnectivityManager` provides per-peer exit routing: traffic from a specific WireGuard peer can be routed through an alternate exit instead of the cell's default gateway.
|
||||||
|
|
||||||
|
Supported exits:
|
||||||
|
- **WireGuard external** — another WireGuard endpoint (e.g. a VPS)
|
||||||
|
- **OpenVPN** — OpenVPN client running in a container
|
||||||
|
- **Tor** — Tor SOCKS proxy with transparent redirection
|
||||||
|
|
||||||
|
Routing uses fwmark and `ip rule` / `ip route` in separate routing tables. Configuration is via `PUT /api/connectivity/peers/<peer_name>/exit`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Security Model
|
||||||
|
|
||||||
|
- **No open ports for the API** — Flask API binds to `127.0.0.1:3000` only; Caddy proxies HTTPS requests to it.
|
||||||
|
- **Session auth** — bcrypt passwords, Flask server-side sessions, CSRF double-submit.
|
||||||
|
- **Setup wizard gate** — all `/api/*` requests return 428 until setup is complete.
|
||||||
|
- **Role separation** — admin cannot access peer endpoints; peer cannot access admin endpoints.
|
||||||
|
- **HTTPS everywhere** — Caddy handles TLS; internal services are reached via reverse proxy paths.
|
||||||
|
- **Internal CA** — VaultManager issues certificates for services that don't use Let's Encrypt.
|
||||||
|
- **Docker socket isolation** — the Docker socket is mounted only into `cell-api`; other containers have no Docker access.
|
||||||
|
- **iptables firewall** — FirewallManager manages INPUT/FORWARD rules; WireGuard peer isolation is enforced at the packet level.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Testing
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make test # unit tests (pytest, ~1500 functions)
|
||||||
|
make test-coverage # coverage report in htmlcov/
|
||||||
|
```
|
||||||
|
|
||||||
|
Test layout:
|
||||||
|
- `tests/` — unit and endpoint tests; no running services required
|
||||||
|
- `tests/integration/` — require a running PIC stack
|
||||||
|
- `tests/e2e/` — Playwright UI tests and WireGuard integration tests
|
||||||
|
|
||||||
|
CI: Gitea Actions runs `pytest tests/ --ignore=tests/e2e --ignore=tests/integration` on every push.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Development
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Full stack in Docker
|
||||||
|
make start
|
||||||
|
make stop
|
||||||
|
make logs
|
||||||
|
|
||||||
|
# Flask API without Docker (port 3000)
|
||||||
|
pip install -r api/requirements.txt
|
||||||
|
python api/app.py
|
||||||
|
|
||||||
|
# React UI dev server (port 5173, proxies /api → :3000)
|
||||||
|
cd webui && npm install && npm run dev
|
||||||
|
|
||||||
|
# Rebuild containers after code change
|
||||||
|
make build-api
|
||||||
|
make build-webui
|
||||||
|
```
|
||||||
|
|
||||||
|
Key files:
|
||||||
|
- `api/app.py` — Flask app, blueprint registration, before-request hooks, health monitor thread
|
||||||
|
- `api/managers.py` — singleton instantiation of all service managers
|
||||||
|
- `api/base_service_manager.py` — abstract base class all managers implement
|
||||||
|
- `api/config_manager.py` — `cell_config.json` read/write/validate/backup
|
||||||
|
- `api/service_bus.py` — pub/sub event system
|
||||||
|
- `webui/src/services/api.js` — Axios API client used by all UI pages
|
||||||
|
- `docker-compose.yml` — container definitions and network topology
|
||||||
|
- `Makefile` — all operational commands
|
||||||
|
|||||||
+104
-132
@@ -1,139 +1,112 @@
|
|||||||
# Quick Start
|
# Quick Start
|
||||||
|
|
||||||
This guide walks through a first-time PIC installation from a clean Linux host.
|
This guide walks through a first-time PIC installation on a clean Linux host.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Prerequisites
|
## Prerequisites
|
||||||
|
|
||||||
- Linux host with the WireGuard kernel module (`modprobe wireguard` to verify)
|
- Linux x86-64 host — Debian, Ubuntu, Fedora, RHEL, or Alpine
|
||||||
- Docker Engine and Docker Compose installed
|
|
||||||
- Python 3.10+ (needed for `make setup` only)
|
|
||||||
- 2 GB+ RAM, 10 GB+ disk
|
- 2 GB+ RAM, 10 GB+ disk
|
||||||
|
- Ports 53, 80, 443, 51820/udp, 25, 587, 993 available
|
||||||
|
|
||||||
|
The installer handles all software dependencies (git, docker, make, etc.) automatically.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 1. Clone the repository
|
## Option A — One-line installer (recommended)
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
git clone <repo-url> pic
|
curl -fsSL https://install.pic.ngo | sudo bash
|
||||||
|
```
|
||||||
|
|
||||||
|
Always review the script before running it:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl -fsSL https://install.pic.ngo | less
|
||||||
|
```
|
||||||
|
|
||||||
|
The installer:
|
||||||
|
1. Detects your OS and installs Docker, git, make via the system package manager
|
||||||
|
2. Creates a `pic` system user and adds it to the `docker` group
|
||||||
|
3. Clones the repository to `/opt/pic`
|
||||||
|
4. Runs `make install` (generates keys and config, writes a systemd unit)
|
||||||
|
5. Runs `make start-core` to bring up the core containers
|
||||||
|
6. Waits for the API to respond, then prints the wizard URL
|
||||||
|
|
||||||
|
When it finishes, open the URL it prints:
|
||||||
|
|
||||||
|
```
|
||||||
|
http://<host-ip>:8081/setup
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Option B — Manual install
|
||||||
|
|
||||||
|
Use this if you want to control where PIC is installed, or if you are installing on a machine that already has Docker.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git clone https://git.pic.ngo/roof/pic.git pic
|
||||||
cd pic
|
cd pic
|
||||||
|
sudo make install
|
||||||
|
make start-core
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Then open `http://<host-ip>:8081` in a browser.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 2. Configure the environment
|
## Complete the setup wizard
|
||||||
|
|
||||||
Copy the example environment file and edit it:
|
The setup wizard appears automatically on first start. All API requests redirect to `/setup` until it is finished.
|
||||||
|
|
||||||
```bash
|
The wizard asks for:
|
||||||
cp .env.example .env
|
|
||||||
```
|
|
||||||
|
|
||||||
Open `.env` and set at minimum:
|
- **Cell name** — used for hostnames and DDNS subdomain. Lowercase letters, digits, hyphens, 2–31 characters. Example: `myhome`.
|
||||||
|
- **Domain mode** — how HTTPS certificates are issued:
|
||||||
|
- `pic_ngo` — automatic `<cell-name>.pic.ngo` subdomain with Let's Encrypt via DNS-01 (recommended for internet-facing cells)
|
||||||
|
- `cloudflare` — Let's Encrypt via Cloudflare DNS-01 (bring your own domain)
|
||||||
|
- `duckdns` — Let's Encrypt via DuckDNS DNS-01
|
||||||
|
- `http01` — Let's Encrypt via HTTP-01 (no wildcard; cell must be reachable on port 80)
|
||||||
|
- `lan` — internal CA, no internet required (for LAN-only installs)
|
||||||
|
- **Timezone**
|
||||||
|
- **Services to enable** — email, calendar, files, WireGuard
|
||||||
|
- **Admin password** — minimum 12 characters, must contain uppercase, lowercase, and a digit
|
||||||
|
|
||||||
```
|
Click **Complete Setup**. The wizard creates the admin account, writes cell identity to `config/api/cell_config.json`, and redirects to the login page.
|
||||||
WEBDAV_PASS=changeme
|
|
||||||
```
|
|
||||||
|
|
||||||
`WEBDAV_PASS` must be set before starting — the WebDAV container will fail to start without it.
|
|
||||||
|
|
||||||
All other variables have working defaults. See the Configuration section in [README.md](README.md) for the full list.
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 3. Run setup
|
## Log in
|
||||||
|
|
||||||
`make setup` installs system dependencies, generates WireGuard keys, and writes all required config files under `config/`:
|
After the wizard you are redirected to `/login`.
|
||||||
|
|
||||||
```bash
|
- **Username:** `admin`
|
||||||
make check-deps # installs docker, python3-cryptography, etc. via apt
|
- **Password:** the password you set in the wizard
|
||||||
make setup # generates keys and writes configs
|
|
||||||
```
|
|
||||||
|
|
||||||
To customise the cell identity at setup time, pass overrides on the command line:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
CELL_NAME=myhome CELL_DOMAIN=cell VPN_ADDRESS=10.0.0.1/24 WG_PORT=51820 make setup
|
|
||||||
```
|
|
||||||
|
|
||||||
`VPN_ADDRESS` must be an RFC-1918 address (e.g. `10.0.0.1/24`).
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 4. Start the stack
|
## Add a WireGuard peer
|
||||||
|
|
||||||
```bash
|
Go to **Peers** in the sidebar.
|
||||||
make start
|
|
||||||
```
|
|
||||||
|
|
||||||
This builds the `cell-api` and `cell-webui` images and starts all 13 containers. The first run takes a few minutes while images are pulled and built.
|
|
||||||
|
|
||||||
Check that everything came up:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
make status
|
|
||||||
```
|
|
||||||
|
|
||||||
You should see all containers in the `Up` state and the API responding at `http://localhost:3000/health`.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 5. Open the web UI
|
|
||||||
|
|
||||||
Open a browser and go to:
|
|
||||||
|
|
||||||
```
|
|
||||||
http://<host-ip>:8081
|
|
||||||
```
|
|
||||||
|
|
||||||
If you are running locally:
|
|
||||||
|
|
||||||
```
|
|
||||||
http://localhost:8081
|
|
||||||
```
|
|
||||||
|
|
||||||
The sidebar contains: Dashboard, Peers, Network Services, WireGuard, Email, Calendar, Files, Routing, Vault, Containers, Cell Network, Logs, Settings.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 6. Set cell identity
|
|
||||||
|
|
||||||
Go to **Settings** in the sidebar.
|
|
||||||
|
|
||||||
Set your:
|
|
||||||
- **Cell name** — a short identifier, e.g. `myhome`
|
|
||||||
- **Domain** — the TLD your cell will use internally, e.g. `cell`
|
|
||||||
- **VPN IP range** — the CIDR for WireGuard peers, e.g. `10.0.0.0/24`
|
|
||||||
|
|
||||||
After saving, the UI will show a banner asking you to apply the changes. Click **Apply Now**. The containers will restart briefly to pick up the new configuration.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 7. Add a WireGuard peer
|
|
||||||
|
|
||||||
Go to **WireGuard** in the sidebar.
|
|
||||||
|
|
||||||
1. Click **Add Peer**.
|
1. Click **Add Peer**.
|
||||||
2. Enter a name for the peer (e.g. `laptop`).
|
2. Enter a peer name (e.g. `laptop`).
|
||||||
3. The API generates a key pair and assigns the next available VPN IP automatically.
|
3. The API generates a key pair and assigns the next available VPN IP automatically.
|
||||||
4. Click the QR code icon to display the peer config as a QR code.
|
4. Click the QR code icon to display the peer configuration as a QR code.
|
||||||
5. Scan the QR code with a WireGuard client (Android, iOS, or the WireGuard desktop app).
|
5. Scan the QR code with a WireGuard client (Android, iOS, or the WireGuard desktop app).
|
||||||
|
|
||||||
The peer config sets your cell as the DNS server. Once connected, `*.cell` names resolve through the cell's CoreDNS.
|
Once connected, `*.cell` names resolve through the cell's CoreDNS and traffic can be routed through the cell.
|
||||||
|
|
||||||
To manage peers from the command line:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
make list-peers
|
|
||||||
make add-peer PEER_NAME=phone PEER_IP=10.0.0.3 PEER_KEY=<base64-pubkey>
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 8. Day-to-day operations
|
## Day-to-day operations
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
# Check container status and API health
|
||||||
|
make status
|
||||||
|
|
||||||
# Follow logs from all services
|
# Follow logs from all services
|
||||||
make logs
|
make logs
|
||||||
|
|
||||||
@@ -142,9 +115,6 @@ make logs-api
|
|||||||
make logs-wireguard
|
make logs-wireguard
|
||||||
make logs-caddy
|
make logs-caddy
|
||||||
|
|
||||||
# Check container status and API health
|
|
||||||
make status
|
|
||||||
|
|
||||||
# Open a shell inside a container
|
# Open a shell inside a container
|
||||||
make shell-api
|
make shell-api
|
||||||
make shell-dns
|
make shell-dns
|
||||||
@@ -152,20 +122,11 @@ make shell-dns
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 9. Backup
|
## Backup and restore
|
||||||
|
|
||||||
Before making significant changes, create a backup:
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
make backup
|
make backup # archives config/ and data/ into backups/cell-backup-<timestamp>.tar.gz
|
||||||
```
|
make restore # list available backups
|
||||||
|
|
||||||
This archives `config/` and `data/` into `backups/cell-backup-<timestamp>.tar.gz`.
|
|
||||||
|
|
||||||
To list available backups:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
make restore
|
|
||||||
```
|
```
|
||||||
|
|
||||||
To restore manually:
|
To restore manually:
|
||||||
@@ -175,34 +136,38 @@ tar -xzf backups/cell-backup-YYYYMMDD-HHMMSS.tar.gz
|
|||||||
make start
|
make start
|
||||||
```
|
```
|
||||||
|
|
||||||
Backup and restore is also available in the UI under **Settings**.
|
---
|
||||||
|
|
||||||
|
## Updating PIC
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make update # git pull + rebuild + restart
|
||||||
|
```
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 10. Updating PIC
|
## Uninstalling
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
make update
|
make uninstall # stops containers; prompts to also delete config/ and data/
|
||||||
```
|
```
|
||||||
|
|
||||||
This runs `git pull`, then rebuilds and restarts all containers. If `config/` is missing (e.g. after a fresh clone), it runs `make setup` automatically.
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Troubleshooting
|
## Troubleshooting
|
||||||
|
|
||||||
**Containers not starting**
|
### Containers not starting
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
make logs
|
make logs
|
||||||
make logs-api
|
make logs-api
|
||||||
```
|
```
|
||||||
|
|
||||||
Look for errors related to missing config files or port conflicts.
|
Look for errors about missing config files or port conflicts.
|
||||||
|
|
||||||
**Port 53 already in use**
|
### Port 53 already in use
|
||||||
|
|
||||||
On Ubuntu/Debian, `systemd-resolved` listens on port 53. Disable it:
|
On Ubuntu and Debian, `systemd-resolved` listens on port 53. Disable it:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
sudo systemctl disable --now systemd-resolved
|
sudo systemctl disable --now systemd-resolved
|
||||||
@@ -212,28 +177,35 @@ echo "nameserver 8.8.8.8" | sudo tee /etc/resolv.conf
|
|||||||
|
|
||||||
Then run `make start` again.
|
Then run `make start` again.
|
||||||
|
|
||||||
**WebDAV container exits immediately**
|
### WireGuard container fails to load the kernel module
|
||||||
|
|
||||||
`WEBDAV_PASS` is not set in `.env`. Set it and run `make start` again.
|
|
||||||
|
|
||||||
**WireGuard container fails to load kernel module**
|
|
||||||
|
|
||||||
Ensure the WireGuard kernel module is available:
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
sudo modprobe wireguard
|
sudo modprobe wireguard
|
||||||
```
|
```
|
||||||
|
|
||||||
On some minimal installs you may need to install `wireguard-tools` and the kernel headers for your running kernel.
|
On minimal installs you may need `wireguard-tools` and the kernel headers for the running kernel.
|
||||||
|
|
||||||
**API returns 503 or UI shows "Backend Unavailable"**
|
### API returns 428 and redirects to /setup
|
||||||
|
|
||||||
The Flask API may still be starting. Wait 10–15 seconds after `make start` and refresh. If it persists:
|
The first-run wizard has not been completed. Open `http://<host-ip>:8081` and finish the wizard.
|
||||||
|
|
||||||
|
### API returns 401 / UI shows "Not authenticated"
|
||||||
|
|
||||||
|
Your session expired or you have not logged in. Go to `http://<host-ip>:8081/login`.
|
||||||
|
|
||||||
|
### API returns 503 "Authentication not configured"
|
||||||
|
|
||||||
|
The auth file exists but contains no accounts. To recover:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
make logs-api
|
make reset-admin-password
|
||||||
```
|
```
|
||||||
|
|
||||||
**Config changes not taking effect**
|
This generates a new admin password and prints it.
|
||||||
|
|
||||||
After changing identity or service settings in the UI, a yellow banner appears at the top of the page. Click **Apply Now** to restart the affected containers.
|
### Forgot the admin password
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make show-admin-password # print current password
|
||||||
|
make reset-admin-password # generate a new random password
|
||||||
|
```
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
# Personal Internet Cell (PIC)
|
# Personal Internet Cell (PIC)
|
||||||
|
|
||||||
PIC is a self-hosted digital infrastructure platform. It manages DNS, DHCP, NTP, WireGuard VPN, email, calendar/contacts (CalDAV), file storage (WebDAV), a reverse proxy, and a certificate authority — all controlled from a single REST API and React web UI. No manual config file editing is required for normal operations.
|
PIC is a self-hosted digital infrastructure platform. It packages DNS, DHCP, NTP, WireGuard VPN, email, calendar/contacts (CalDAV), file storage (WebDAV), a reverse proxy, a certificate authority, and optional third-party services — all managed through a single REST API and a React web UI. No manual config file editing is required for normal operations.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -10,33 +10,56 @@ PIC is a self-hosted digital infrastructure platform. It manages DNS, DHCP, NTP,
|
|||||||
Browser
|
Browser
|
||||||
└── React SPA (cell-webui :8081)
|
└── React SPA (cell-webui :8081)
|
||||||
└── Flask REST API (cell-api :3000, bound to 127.0.0.1)
|
└── Flask REST API (cell-api :3000, bound to 127.0.0.1)
|
||||||
└── Docker SDK / config files
|
└── Service managers + Docker SDK
|
||||||
├── cell-caddy :80/:443 reverse proxy
|
├── cell-caddy :80/:443 Caddy reverse proxy (HTTPS/TLS)
|
||||||
├── cell-dns :53 CoreDNS
|
├── cell-dns :53 CoreDNS
|
||||||
├── cell-dhcp :67/udp dnsmasq
|
├── cell-dhcp :67/udp dnsmasq
|
||||||
├── cell-ntp :123/udp chrony
|
├── cell-ntp :123/udp chrony
|
||||||
├── cell-wireguard :51820/udp WireGuard VPN
|
├── cell-wireguard :51820/udp WireGuard VPN
|
||||||
├── cell-mail :25/:587/:993 Postfix + Dovecot
|
├── cell-mail :25/:587/:993 Postfix + Dovecot
|
||||||
├── cell-radicale 127.0.0.1:5232 CalDAV/CardDAV
|
├── cell-radicale 127.0.0.1:5232 CalDAV/CardDAV
|
||||||
├── cell-webdav 127.0.0.1:8080 WebDAV
|
├── cell-webdav 127.0.0.1:8080 WebDAV
|
||||||
├── cell-rainloop :8888 webmail (RainLoop)
|
├── cell-rainloop 127.0.0.1:8888 Webmail (RainLoop)
|
||||||
├── cell-filegator :8082 file manager UI
|
├── cell-filegator 127.0.0.1:8082 File manager (Filegator)
|
||||||
└── cell-webui :8081 React UI (Nginx)
|
└── cell-webui :8081 React UI (Nginx)
|
||||||
```
|
```
|
||||||
|
|
||||||
All containers run on a custom Docker bridge network (`cell-network`, default `172.20.0.0/16`). Static IPs per container are set in `docker-compose.yml` and overridden via `.env`.
|
All containers run on a custom Docker bridge network (`cell-network`, default subnet `172.20.0.0/16`). Static IPs per container are set in `docker-compose.yml` and can be overridden via `.env`.
|
||||||
|
|
||||||
The Flask API (`api/app.py`, ~2800 lines) contains all REST endpoints, runs a background health-monitoring thread, and manages the entire lifecycle of generated config artefacts: `Caddyfile`, `Corefile`, `wg0.conf`, and `cell_config.json` (the single source of truth at `config/api/cell_config.json`).
|
The Flask API (`api/app.py`) contains REST endpoints and a background health-monitoring thread. Service managers are instantiated as singletons in `api/managers.py`. The single source of truth for runtime configuration is `config/api/cell_config.json`, managed by `ConfigManager`.
|
||||||
|
|
||||||
The React frontend (`webui/`) is built with Vite + Tailwind CSS. All API calls go through `src/services/api.js` (Axios). Pages: Dashboard, Peers, Network Services, WireGuard, Email, Calendar, Files, Routing, Vault, Containers, Cell Network, Logs, Settings.
|
The React frontend (`webui/`) is built with Vite + Tailwind CSS. All API calls go through `src/services/api.js` (Axios).
|
||||||
|
|
||||||
|
**Web UI pages:** Dashboard, Peers, Network Services, WireGuard, Email, Calendar, Files, Routing, Vault, Containers, Cell Network, Connectivity, Service Store, Logs, Settings.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Features
|
||||||
|
|
||||||
|
- **First-run wizard** — browser-based setup at `/setup`. On first start, all API requests redirect to `/setup` (HTTP 428) until the wizard is completed. Sets cell name, domain mode, timezone, admin password, and initial services. No manual `.env` editing required for identity.
|
||||||
|
- **Session-based auth** — admin and peer roles. All `/api/*` endpoints require an authenticated session after setup. CSRF protection on all state-changing requests.
|
||||||
|
- **WireGuard VPN** — peer lifecycle management, automatic key generation, QR code config export, per-peer routing policy.
|
||||||
|
- **Caddy HTTPS** — automatic TLS via Let's Encrypt (DNS-01 or HTTP-01) or an internal CA, depending on domain mode.
|
||||||
|
- **DDNS (pic.ngo)** — registers a `<cell-name>.pic.ngo` subdomain. Supported providers: `pic_ngo`, `cloudflare`, `duckdns`, `noip`, `freedns`. A background thread re-publishes the public IP every 5 minutes.
|
||||||
|
- **Service store** — install/remove optional third-party services from the `pic-services` index at `git.pic.ngo`. Manifests declare container images, Caddy routes, and iptables rules.
|
||||||
|
- **Extended connectivity** — per-peer egress routing through alternate exits: WireGuard external, OpenVPN, or Tor. Configured via policy routing (fwmark + ip rule) in the WireGuard container.
|
||||||
|
- **Cell-to-cell networking** — WireGuard-based site-to-site links between PIC cells with service-level access control (calendar, files, mail, WebDAV) and a peer-sync protocol.
|
||||||
|
- **Certificate authority** — `vault_manager` issues and revokes TLS certificates for internal services.
|
||||||
|
- **Network services** — CoreDNS (`.cell` TLD), dnsmasq DHCP, chrony NTP.
|
||||||
|
- **Email** — Postfix + Dovecot via `docker-mailserver`.
|
||||||
|
- **Calendar/contacts** — Radicale CalDAV/CardDAV.
|
||||||
|
- **File storage** — WebDAV with per-user accounts; Filegator for browser-based file management.
|
||||||
|
- **Container manager** — start/stop/inspect containers, pull images, manage volumes via the Docker SDK.
|
||||||
|
- **Firewall manager** — iptables rule management (`firewall_manager.py`).
|
||||||
|
- **Structured logging** — JSON logs with rotation (5 MB / 5 backups per service), log search, and per-service verbosity control.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Requirements
|
## Requirements
|
||||||
|
|
||||||
- Linux host with the WireGuard kernel module loaded
|
- Linux host with the WireGuard kernel module loaded (`modprobe wireguard` to verify)
|
||||||
- Docker Engine and Docker Compose (v2 plugin or v1 standalone)
|
- Docker Engine and Docker Compose (v2 plugin or v1 standalone)
|
||||||
- Python 3.10+ (for `make setup` and local dev only; not needed at runtime)
|
- Python 3.10+ (for `make setup` and local development; not needed at runtime)
|
||||||
- 2 GB+ RAM, 10 GB+ disk
|
- 2 GB+ RAM, 10 GB+ disk
|
||||||
- Ports available: 53, 67/udp, 80, 443, 51820/udp, 25, 587, 993
|
- Ports available: 53, 67/udp, 80, 443, 51820/udp, 25, 587, 993
|
||||||
|
|
||||||
@@ -44,62 +67,77 @@ The React frontend (`webui/`) is built with Vite + Tailwind CSS. All API calls g
|
|||||||
|
|
||||||
## Quick Start
|
## Quick Start
|
||||||
|
|
||||||
See [QUICKSTART.md](QUICKSTART.md) for step-by-step setup.
|
See [QUICKSTART.md](QUICKSTART.md) for step-by-step instructions.
|
||||||
|
|
||||||
|
The short version:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git clone gitea@192.168.31.50:roof/pic.git pic
|
||||||
|
cd pic
|
||||||
|
make start
|
||||||
|
# open http://<host-ip>:8081 — the setup wizard appears automatically
|
||||||
|
```
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Configuration
|
## Configuration
|
||||||
|
|
||||||
Runtime configuration is controlled by `.env` in the project root. Copy `.env.example` to `.env` before first run.
|
Port assignments and container IPs are configured in `.env` in the project root. A `.env` file is not required for first start — all variables have defaults. Create one only if you need to change ports or container IPs.
|
||||||
|
|
||||||
| Variable | Default | Description |
|
| Variable | Default | Description |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| `CELL_NETWORK` | `172.20.0.0/16` | Docker bridge subnet for all containers |
|
| `CELL_NETWORK` | `172.20.0.0/16` | Docker bridge subnet |
|
||||||
| `CADDY_IP` through `FILEGATOR_IP` | `172.20.0.2`–`.13` | Static IP for each container |
|
| `CADDY_IP` through `FILEGATOR_IP` | `172.20.0.2`–`.13` | Static IP per container |
|
||||||
| `DNS_PORT` | `53` | DNS (UDP+TCP) |
|
| `DNS_PORT` | `53` | DNS (UDP + TCP) |
|
||||||
| `DHCP_PORT` | `67` | DHCP (UDP) |
|
| `DHCP_PORT` | `67` | DHCP (UDP) |
|
||||||
| `NTP_PORT` | `123` | NTP (UDP) |
|
| `NTP_PORT` | `123` | NTP (UDP) |
|
||||||
| `WG_PORT` | `51820` | WireGuard listen port (UDP) |
|
| `WG_PORT` | `51820` | WireGuard listen port (UDP) |
|
||||||
| `API_PORT` | `3000` | Flask API (bound to `127.0.0.1`) |
|
| `API_PORT` | `3000` | Flask API (127.0.0.1 only) |
|
||||||
| `WEBUI_PORT` | `8081` | React UI |
|
| `WEBUI_PORT` | `8081` | React UI |
|
||||||
| `MAIL_SMTP_PORT` | `25` | SMTP |
|
| `MAIL_SMTP_PORT` | `25` | SMTP |
|
||||||
| `MAIL_SUBMISSION_PORT` | `587` | SMTP submission |
|
| `MAIL_SUBMISSION_PORT` | `587` | SMTP submission |
|
||||||
| `MAIL_IMAP_PORT` | `993` | IMAP |
|
| `MAIL_IMAP_PORT` | `993` | IMAP |
|
||||||
| `RADICALE_PORT` | `5232` | CalDAV (bound to `127.0.0.1`) |
|
| `RADICALE_PORT` | `5232` | CalDAV (127.0.0.1 only) |
|
||||||
| `WEBDAV_PORT` | `8080` | WebDAV (bound to `127.0.0.1`) |
|
| `WEBDAV_PORT` | `8080` | WebDAV (127.0.0.1 only) |
|
||||||
| `RAINLOOP_PORT` | `8888` | Webmail |
|
| `RAINLOOP_PORT` | `8888` | Webmail |
|
||||||
| `FILEGATOR_PORT` | `8082` | File manager UI |
|
| `FILEGATOR_PORT` | `8082` | File manager UI |
|
||||||
| `WEBDAV_USER` | `admin` | WebDAV basic-auth username |
|
| `WEBDAV_USER` | `admin` | WebDAV basic-auth username |
|
||||||
| `WEBDAV_PASS` | _(required)_ | WebDAV basic-auth password — must be set before `make start` |
|
| `WEBDAV_PASS` | _(unset)_ | WebDAV basic-auth password |
|
||||||
| `FLASK_DEBUG` | _(unset)_ | Set to `1` to enable Flask debug mode; do not use in production |
|
| `FLASK_DEBUG` | _(unset)_ | Set to `1` for Flask debug mode; do not use in production |
|
||||||
| `PUID` / `PGID` | current user | UID/GID passed to the WireGuard container |
|
| `PUID` / `PGID` | current user | UID/GID passed to the WireGuard container |
|
||||||
|
|
||||||
Cell identity (cell name, domain, VPN IP range) is configured via `make setup` or the Settings → Identity page in the UI after startup. The VPN IP range must be an RFC-1918 CIDR (`10.0.0.0/8`, `172.16.0.0/12`, or `192.168.0.0/16`); the API and UI both enforce this.
|
Cell identity (cell name, domain mode, timezone) is set through the first-run wizard on first start, or later through the Settings page in the UI.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Security Notes
|
## Security
|
||||||
|
|
||||||
**Ports exposed to the network:**
|
**Ports exposed on all interfaces by default:**
|
||||||
|
|
||||||
- `80` / `443` — Caddy (HTTP/HTTPS reverse proxy)
|
- `80` / `443` — Caddy (HTTP/HTTPS reverse proxy)
|
||||||
- `51820/udp` — WireGuard
|
- `51820/udp` — WireGuard
|
||||||
- `25` / `587` / `993` — Mail (SMTP, submission, IMAP)
|
- `25` / `587` / `993` — mail
|
||||||
- `53` — DNS (UDP + TCP)
|
- `53` — DNS
|
||||||
- `67/udp` — DHCP
|
- `67/udp` — DHCP
|
||||||
- `8081` — Web UI
|
- `8081` — Web UI
|
||||||
- `8888` — Webmail (RainLoop)
|
|
||||||
- `8082` — File manager (Filegator)
|
|
||||||
|
|
||||||
**Ports bound to `127.0.0.1` only** (not directly reachable from the network):
|
**Ports bound to `127.0.0.1` only:**
|
||||||
|
|
||||||
- `3000` — Flask API
|
- `3000` — Flask API
|
||||||
- `5232` — Radicale (CalDAV)
|
- `5232` — Radicale (CalDAV)
|
||||||
- `8080` — WebDAV
|
- `8080` — WebDAV
|
||||||
|
- `8888` — Webmail
|
||||||
|
- `8082` — Filegator
|
||||||
|
|
||||||
The API has no authentication layer. It relies on `is_local_request()` to restrict sensitive endpoints (containers, vault) to requests originating from loopback or the cell's Docker network. The Docker socket is mounted into `cell-api`; treat access to port 3000 as equivalent to root access on the host.
|
The API uses session-based authentication (admin and peer roles). The Docker socket is mounted into `cell-api`; treat access to port 3000 as equivalent to root access on the host.
|
||||||
|
|
||||||
For internet-facing deployments, place the host behind a firewall or VPN and restrict access to the API and UI ports.
|
Before setup is complete, all `/api/*` requests except `/api/setup/*` and `/health` return HTTP 428 and a redirect to `/setup`.
|
||||||
|
|
||||||
|
CSRF protection (double-submit token in `X-CSRF-Token` header) applies to all `POST`, `PUT`, `DELETE`, and `PATCH` requests on `/api/*` once a user session exists, except `/api/auth/*` and `/api/setup/*`.
|
||||||
|
|
||||||
|
Cell-to-cell peer-sync endpoints (`/api/cells/peer-sync/*`) authenticate via source IP and WireGuard public key, not session cookies.
|
||||||
|
|
||||||
|
For internet-facing deployments, place the host behind a firewall and restrict access to the API and UI ports.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -123,7 +161,7 @@ cd webui && npm install && npm run dev
|
|||||||
# Follow all container logs
|
# Follow all container logs
|
||||||
make logs
|
make logs
|
||||||
|
|
||||||
# Follow logs for one service (e.g. api, dns, caddy, wireguard, mail)
|
# Follow logs for one service
|
||||||
make logs-api
|
make logs-api
|
||||||
|
|
||||||
# Open a shell inside a container
|
# Open a shell inside a container
|
||||||
@@ -135,41 +173,38 @@ make shell-api
|
|||||||
## Testing
|
## Testing
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
make test # run the full pytest suite
|
make test # run all unit tests (pytest, excludes e2e and integration)
|
||||||
make test-coverage # run with coverage; HTML report in htmlcov/
|
make test-coverage # run with coverage; HTML report in htmlcov/
|
||||||
|
make test-api # run API endpoint tests only
|
||||||
```
|
```
|
||||||
|
|
||||||
Tests live in `tests/` (34 files, 642 test functions). Coverage includes:
|
Tests live in `tests/`. Integration tests require a running stack:
|
||||||
|
|
||||||
- All service managers (network, WireGuard, email, calendar, file, routing, vault, container)
|
|
||||||
- API endpoint tests for each service area
|
|
||||||
- Config manager (CRUD, validation, backup/restore)
|
|
||||||
- IP utilities and Caddyfile generation
|
|
||||||
- Peer registry and WireGuard peer lifecycle
|
|
||||||
- Service bus pub/sub
|
|
||||||
- Firewall manager
|
|
||||||
- Pending-restart logic
|
|
||||||
|
|
||||||
Integration tests (`tests/integration/`) require a running PIC stack:
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
make test-integration # full suite (creates peers)
|
make test-integration # full suite (creates peers, modifies state)
|
||||||
make test-integration-readonly # read-only checks, safe to run anytime
|
make test-integration-readonly # read-only checks, safe to run anytime
|
||||||
```
|
```
|
||||||
|
|
||||||
|
End-to-end tests use Playwright:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make test-e2e-deps # install Playwright and dependencies (run once)
|
||||||
|
make test-e2e-api # API-level e2e tests
|
||||||
|
make test-e2e-ui # UI-level e2e tests
|
||||||
|
```
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Management Commands
|
## Management Commands
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
make setup # generate WireGuard keys, write configs, create data dirs
|
make start # docker compose up -d --build (full profile)
|
||||||
make start # docker compose up -d --build
|
|
||||||
make stop # docker compose down
|
make stop # docker compose down
|
||||||
make restart # docker compose restart
|
make restart # docker compose restart
|
||||||
make status # container status + API health check
|
make status # container status + API health check
|
||||||
make logs # follow all service logs
|
make logs # follow all service logs
|
||||||
make logs-<svc> # follow logs for one service
|
make logs-<svc> # follow logs for one service (e.g. make logs-api)
|
||||||
make shell-<svc> # shell inside a container
|
make shell-<svc> # shell inside a container (e.g. make shell-api)
|
||||||
|
|
||||||
make update # git pull + rebuild + restart
|
make update # git pull + rebuild + restart
|
||||||
make reinstall # full wipe of config/ and data/, then setup + start
|
make reinstall # full wipe of config/ and data/, then setup + start
|
||||||
@@ -180,7 +215,9 @@ make restore # list available backups
|
|||||||
|
|
||||||
make list-peers # show WireGuard peers via API
|
make list-peers # show WireGuard peers via API
|
||||||
make show-routes # wg show inside the wireguard container
|
make show-routes # wg show inside the wireguard container
|
||||||
make add-peer PEER_NAME=foo PEER_IP=10.0.0.5 PEER_KEY=<pubkey>
|
|
||||||
|
make show-admin-password # print current admin password
|
||||||
|
make reset-admin-password # generate and set a new random admin password
|
||||||
```
|
```
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|||||||
@@ -216,10 +216,6 @@ def enforce_auth():
|
|||||||
return None
|
return None
|
||||||
users = auth_manager.list_users()
|
users = auth_manager.list_users()
|
||||||
if not users:
|
if not users:
|
||||||
# Only fail closed when the auth file is readable but empty —
|
|
||||||
# that's an explicit misconfiguration. If the file is missing or
|
|
||||||
# unreadable (test env, wrong host path, permission denied), bypass
|
|
||||||
# so pre-auth test suites continue to work.
|
|
||||||
users_file = getattr(auth_manager, '_users_file', None)
|
users_file = getattr(auth_manager, '_users_file', None)
|
||||||
if users_file:
|
if users_file:
|
||||||
try:
|
try:
|
||||||
|
|||||||
@@ -47,16 +47,6 @@ class AuthManager(BaseServiceManager):
|
|||||||
os.makedirs(os.path.dirname(self._users_file), exist_ok=True)
|
os.makedirs(os.path.dirname(self._users_file), exist_ok=True)
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
if not os.path.exists(self._users_file):
|
|
||||||
try:
|
|
||||||
with open(self._users_file, 'w') as f:
|
|
||||||
f.write('[]')
|
|
||||||
try:
|
|
||||||
os.chmod(self._users_file, 0o600)
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
except Exception as e:
|
|
||||||
self.logger.error(f'Could not create users file: {e}')
|
|
||||||
|
|
||||||
def _load_users(self) -> List[Dict[str, Any]]:
|
def _load_users(self) -> List[Dict[str, Any]]:
|
||||||
with self._lock:
|
with self._lock:
|
||||||
|
|||||||
+29
-22
@@ -17,6 +17,7 @@ every 5 minutes, skipping the call when the IP has not changed.
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
import os
|
||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
from typing import Any, Dict, Optional
|
from typing import Any, Dict, Optional
|
||||||
@@ -68,13 +69,25 @@ class PicNgoDDNS(DDNSProvider):
|
|||||||
DEFAULT_API_BASE = 'https://ddns.pic.ngo'
|
DEFAULT_API_BASE = 'https://ddns.pic.ngo'
|
||||||
TIMEOUT = 10
|
TIMEOUT = 10
|
||||||
|
|
||||||
def __init__(self, api_base_url: Optional[str] = None):
|
def __init__(self, api_base_url: Optional[str] = None, totp_secret: Optional[str] = None):
|
||||||
self.api_base_url = (api_base_url or self.DEFAULT_API_BASE).rstrip('/')
|
self.api_base_url = (api_base_url or self.DEFAULT_API_BASE).rstrip('/')
|
||||||
|
self._totp_secret = totp_secret or ''
|
||||||
|
|
||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
# Internal helpers
|
# Internal helpers
|
||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _otp_header(self) -> Dict[str, str]:
|
||||||
|
"""Generate a fresh TOTP header for /register calls."""
|
||||||
|
if not self._totp_secret:
|
||||||
|
return {}
|
||||||
|
try:
|
||||||
|
import pyotp
|
||||||
|
return {'X-Register-OTP': pyotp.TOTP(self._totp_secret).now()}
|
||||||
|
except ImportError:
|
||||||
|
logger.warning("pyotp not installed — X-Register-OTP header omitted")
|
||||||
|
return {}
|
||||||
|
|
||||||
def _headers(self, token: Optional[str] = None) -> Dict[str, str]:
|
def _headers(self, token: Optional[str] = None) -> Dict[str, str]:
|
||||||
h: Dict[str, str] = {'Content-Type': 'application/json'}
|
h: Dict[str, str] = {'Content-Type': 'application/json'}
|
||||||
if token:
|
if token:
|
||||||
@@ -95,7 +108,8 @@ class PicNgoDDNS(DDNSProvider):
|
|||||||
"""POST /api/v1/register — register subdomain, returns token + subdomain."""
|
"""POST /api/v1/register — register subdomain, returns token + subdomain."""
|
||||||
url = f'{self.api_base_url}/api/v1/register'
|
url = f'{self.api_base_url}/api/v1/register'
|
||||||
payload = {'name': name, 'ip': ip}
|
payload = {'name': name, 'ip': ip}
|
||||||
resp = requests.post(url, json=payload, headers=self._headers(), timeout=self.TIMEOUT)
|
headers = {**self._headers(), **self._otp_header()}
|
||||||
|
resp = requests.post(url, json=payload, headers=headers, timeout=self.TIMEOUT)
|
||||||
self._raise_for_status(resp, 'register')
|
self._raise_for_status(resp, 'register')
|
||||||
return resp.json()
|
return resp.json()
|
||||||
|
|
||||||
@@ -280,11 +294,9 @@ class DDNSManager(BaseServiceManager):
|
|||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
def get_status(self) -> Dict[str, Any]:
|
def get_status(self) -> Dict[str, Any]:
|
||||||
identity = self._identity()
|
|
||||||
domain_cfg = identity.get('domain', {})
|
|
||||||
return {
|
return {
|
||||||
'service': 'ddns',
|
'service': 'ddns',
|
||||||
'provider': domain_cfg.get('ddns', {}).get('provider') if domain_cfg else None,
|
'provider': self._ddns_cfg().get('provider'),
|
||||||
'last_ip': self._last_ip,
|
'last_ip': self._last_ip,
|
||||||
'heartbeat_running': (
|
'heartbeat_running': (
|
||||||
self._heartbeat_thread is not None and
|
self._heartbeat_thread is not None and
|
||||||
@@ -310,17 +322,20 @@ class DDNSManager(BaseServiceManager):
|
|||||||
return {}
|
return {}
|
||||||
return self.config_manager.get_identity() or {}
|
return self.config_manager.get_identity() or {}
|
||||||
|
|
||||||
|
def _ddns_cfg(self) -> Dict[str, Any]:
|
||||||
|
if self.config_manager is None:
|
||||||
|
return {}
|
||||||
|
return self.config_manager.configs.get('ddns', {}) or {}
|
||||||
|
|
||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
# Provider factory
|
# Provider factory
|
||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
def get_provider(self) -> Optional[DDNSProvider]:
|
def get_provider(self) -> Optional[DDNSProvider]:
|
||||||
"""Instantiate and return the configured DDNS provider, or None."""
|
"""Instantiate and return the configured DDNS provider, or None."""
|
||||||
identity = self._identity()
|
if self.config_manager is None:
|
||||||
domain_cfg = identity.get('domain', {})
|
|
||||||
if not domain_cfg:
|
|
||||||
return None
|
return None
|
||||||
ddns_cfg = domain_cfg.get('ddns', {})
|
ddns_cfg = self.config_manager.configs.get('ddns', {})
|
||||||
if not ddns_cfg:
|
if not ddns_cfg:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
@@ -330,7 +345,8 @@ class DDNSManager(BaseServiceManager):
|
|||||||
|
|
||||||
if provider_name == 'pic_ngo':
|
if provider_name == 'pic_ngo':
|
||||||
api_base = ddns_cfg.get('api_base_url')
|
api_base = ddns_cfg.get('api_base_url')
|
||||||
return PicNgoDDNS(api_base_url=api_base)
|
totp_secret = ddns_cfg.get('totp_secret') or os.environ.get('DDNS_TOTP_SECRET', '')
|
||||||
|
return PicNgoDDNS(api_base_url=api_base, totp_secret=totp_secret)
|
||||||
|
|
||||||
if provider_name == 'cloudflare':
|
if provider_name == 'cloudflare':
|
||||||
return CloudflareDDNS(
|
return CloudflareDDNS(
|
||||||
@@ -405,10 +421,7 @@ class DDNSManager(BaseServiceManager):
|
|||||||
logger.debug("DDNS update_ip: IP unchanged (%s), skipping", current_ip)
|
logger.debug("DDNS update_ip: IP unchanged (%s), skipping", current_ip)
|
||||||
return
|
return
|
||||||
|
|
||||||
identity = self._identity()
|
token = self._ddns_cfg().get('token', '')
|
||||||
domain_cfg = identity.get('domain', {})
|
|
||||||
ddns_cfg = domain_cfg.get('ddns', {}) if domain_cfg else {}
|
|
||||||
token = ddns_cfg.get('token', '')
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
success = provider.update(token, current_ip)
|
success = provider.update(token, current_ip)
|
||||||
@@ -468,10 +481,7 @@ class DDNSManager(BaseServiceManager):
|
|||||||
provider = self.get_provider()
|
provider = self.get_provider()
|
||||||
if provider is None:
|
if provider is None:
|
||||||
raise DDNSError("No DDNS provider configured")
|
raise DDNSError("No DDNS provider configured")
|
||||||
identity = self._identity()
|
token = self._ddns_cfg().get('token', '')
|
||||||
domain_cfg = identity.get('domain', {})
|
|
||||||
ddns_cfg = domain_cfg.get('ddns', {}) if domain_cfg else {}
|
|
||||||
token = ddns_cfg.get('token', '')
|
|
||||||
return provider.dns_challenge_create(token, fqdn, value)
|
return provider.dns_challenge_create(token, fqdn, value)
|
||||||
|
|
||||||
def dns_challenge_delete(self, fqdn: str) -> bool:
|
def dns_challenge_delete(self, fqdn: str) -> bool:
|
||||||
@@ -479,8 +489,5 @@ class DDNSManager(BaseServiceManager):
|
|||||||
provider = self.get_provider()
|
provider = self.get_provider()
|
||||||
if provider is None:
|
if provider is None:
|
||||||
raise DDNSError("No DDNS provider configured")
|
raise DDNSError("No DDNS provider configured")
|
||||||
identity = self._identity()
|
token = self._ddns_cfg().get('token', '')
|
||||||
domain_cfg = identity.get('domain', {})
|
|
||||||
ddns_cfg = domain_cfg.get('ddns', {}) if domain_cfg else {}
|
|
||||||
token = ddns_cfg.get('token', '')
|
|
||||||
return provider.dns_challenge_delete(token, fqdn)
|
return provider.dns_challenge_delete(token, fqdn)
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
flask>=3.0.3
|
flask>=3.0.3
|
||||||
flask-cors>=4.0.1
|
flask-cors>=4.0.1
|
||||||
requests>=2.32.3
|
requests>=2.32.3
|
||||||
|
pyotp>=2.9.0
|
||||||
cryptography>=42.0.5
|
cryptography>=42.0.5
|
||||||
pyyaml==6.0.1
|
pyyaml==6.0.1
|
||||||
icalendar==5.0.7
|
icalendar==5.0.7
|
||||||
|
|||||||
@@ -0,0 +1,86 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
Update the cell's DDNS record with the current public IP.
|
||||||
|
|
||||||
|
Called by: make ddns-update
|
||||||
|
systemd timer (optional, see scripts/pic-ddns-update.timer)
|
||||||
|
|
||||||
|
Reads the DDNS token from data/api/.ddns_token (written by setup_cell.py).
|
||||||
|
Exits 0 on success or if already up to date, non-zero on failure.
|
||||||
|
"""
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
DDNS_URL = os.environ.get('DDNS_URL', 'https://ddns.pic.ngo/api/v1')
|
||||||
|
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||||
|
TOKEN_FILE = os.path.join(ROOT, 'data', 'api', '.ddns_token')
|
||||||
|
IP_CACHE_FILE = os.path.join(ROOT, 'data', 'api', '.ddns_last_ip')
|
||||||
|
|
||||||
|
|
||||||
|
def get_public_ip() -> str:
|
||||||
|
return urllib.request.urlopen('https://api.ipify.org', timeout=5).read().decode().strip()
|
||||||
|
|
||||||
|
|
||||||
|
def read_token() -> str:
|
||||||
|
if not os.path.exists(TOKEN_FILE):
|
||||||
|
print('ERROR: DDNS token not found. Run "make setup" to register.', file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
return open(TOKEN_FILE).read().strip()
|
||||||
|
|
||||||
|
|
||||||
|
def read_last_ip() -> str:
|
||||||
|
try:
|
||||||
|
return open(IP_CACHE_FILE).read().strip()
|
||||||
|
except FileNotFoundError:
|
||||||
|
return ''
|
||||||
|
|
||||||
|
|
||||||
|
def write_last_ip(ip: str) -> None:
|
||||||
|
with open(IP_CACHE_FILE, 'w') as f:
|
||||||
|
f.write(ip)
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
try:
|
||||||
|
public_ip = get_public_ip()
|
||||||
|
except Exception as e:
|
||||||
|
print(f'ERROR: Could not detect public IP: {e}', file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
last_ip = read_last_ip()
|
||||||
|
if public_ip == last_ip:
|
||||||
|
print(f'DDNS: IP unchanged ({public_ip}) — no update needed')
|
||||||
|
return 0
|
||||||
|
|
||||||
|
token = read_token()
|
||||||
|
data = json.dumps({'token': token, 'ip': public_ip}).encode()
|
||||||
|
req = urllib.request.Request(
|
||||||
|
f'{DDNS_URL}/update',
|
||||||
|
data=data,
|
||||||
|
headers={'Content-Type': 'application/json'},
|
||||||
|
method='PUT',
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
resp = urllib.request.urlopen(req, timeout=10)
|
||||||
|
result = json.loads(resp.read())
|
||||||
|
if result.get('updated'):
|
||||||
|
write_last_ip(public_ip)
|
||||||
|
print(f'DDNS: Updated to {public_ip}')
|
||||||
|
return 0
|
||||||
|
else:
|
||||||
|
print(f'ERROR: Unexpected response: {result}', file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
except urllib.error.HTTPError as e:
|
||||||
|
body = e.read().decode()
|
||||||
|
print(f'ERROR: DDNS update failed ({e.code}): {body}', file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
except Exception as e:
|
||||||
|
print(f'ERROR: DDNS update failed: {e}', file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
sys.exit(main())
|
||||||
+76
-1
@@ -185,7 +185,13 @@ def write_cell_config(cell_name: str, domain: str, port: int):
|
|||||||
'domain': domain,
|
'domain': domain,
|
||||||
'ip_range': '172.20.0.0/16',
|
'ip_range': '172.20.0.0/16',
|
||||||
'wireguard_port': port,
|
'wireguard_port': port,
|
||||||
}
|
},
|
||||||
|
'ddns': {
|
||||||
|
'provider': 'pic_ngo',
|
||||||
|
'api_base_url': DDNS_URL.replace('/api/v1', ''),
|
||||||
|
'totp_secret': DDNS_TOTP_SECRET,
|
||||||
|
'enabled': True,
|
||||||
|
},
|
||||||
}
|
}
|
||||||
with open(cfg_path, 'w') as f:
|
with open(cfg_path, 'w') as f:
|
||||||
json.dump(config, f, indent=2)
|
json.dump(config, f, indent=2)
|
||||||
@@ -238,6 +244,74 @@ def ensure_session_secret():
|
|||||||
print('[CREATED] data/api/.session_secret')
|
print('[CREATED] data/api/.session_secret')
|
||||||
|
|
||||||
|
|
||||||
|
DDNS_URL = os.environ.get('DDNS_URL', 'https://ddns.pic.ngo/api/v1')
|
||||||
|
DDNS_TOTP_SECRET = os.environ.get('DDNS_TOTP_SECRET', 'S6UMA464YIKM74QHXWL5WELDIO3HFZ6K')
|
||||||
|
|
||||||
|
|
||||||
|
def register_with_ddns(cell_name: str) -> None:
|
||||||
|
"""Register cell_name.pic.ngo with the DDNS server using TOTP auth.
|
||||||
|
|
||||||
|
Idempotent: if a token file already exists the registration is skipped.
|
||||||
|
Skipped silently if DDNS_TOTP_SECRET is not set.
|
||||||
|
"""
|
||||||
|
token_path = os.path.join(ROOT, 'data', 'api', '.ddns_token')
|
||||||
|
if os.path.exists(token_path):
|
||||||
|
print('[EXISTS] DDNS registration — token already present')
|
||||||
|
return
|
||||||
|
|
||||||
|
if not DDNS_TOTP_SECRET:
|
||||||
|
print('[SKIP] DDNS_TOTP_SECRET not set — skipping DDNS registration')
|
||||||
|
return
|
||||||
|
|
||||||
|
import urllib.request
|
||||||
|
import urllib.error
|
||||||
|
|
||||||
|
# Detect public IP
|
||||||
|
try:
|
||||||
|
public_ip = urllib.request.urlopen(
|
||||||
|
'https://api.ipify.org', timeout=5
|
||||||
|
).read().decode().strip()
|
||||||
|
except Exception as e:
|
||||||
|
print(f'[WARN] Could not detect public IP: {e} — skipping DDNS registration')
|
||||||
|
return
|
||||||
|
|
||||||
|
# Generate TOTP code (requires pyotp; if not available fall back gracefully)
|
||||||
|
try:
|
||||||
|
import pyotp
|
||||||
|
otp = pyotp.TOTP(DDNS_TOTP_SECRET).now()
|
||||||
|
except ImportError:
|
||||||
|
# Try python3 -c as a subprocess fallback
|
||||||
|
try:
|
||||||
|
otp = subprocess.check_output(
|
||||||
|
['python3', '-c', f"import pyotp; print(pyotp.TOTP('{DDNS_TOTP_SECRET}').now())"]
|
||||||
|
).decode().strip()
|
||||||
|
except Exception as e:
|
||||||
|
print(f'[WARN] pyotp not available and fallback failed: {e} — skipping DDNS')
|
||||||
|
return
|
||||||
|
|
||||||
|
data = json.dumps({'name': cell_name, 'ip': public_ip}).encode()
|
||||||
|
req = urllib.request.Request(
|
||||||
|
f'{DDNS_URL}/register',
|
||||||
|
data=data,
|
||||||
|
headers={'Content-Type': 'application/json', 'X-Register-OTP': otp},
|
||||||
|
method='POST',
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
resp = urllib.request.urlopen(req, timeout=10)
|
||||||
|
result = json.loads(resp.read())
|
||||||
|
token = result['token']
|
||||||
|
os.makedirs(os.path.dirname(token_path), exist_ok=True)
|
||||||
|
with open(token_path, 'w') as f:
|
||||||
|
f.write(token)
|
||||||
|
os.chmod(token_path, 0o600)
|
||||||
|
print(f'[CREATED] DDNS registration: {result["subdomain"]} ip={public_ip}')
|
||||||
|
except urllib.error.HTTPError as e:
|
||||||
|
body = e.read().decode()
|
||||||
|
print(f'[WARN] DDNS registration failed ({e.code}): {body}')
|
||||||
|
except Exception as e:
|
||||||
|
print(f'[WARN] DDNS registration failed: {e}')
|
||||||
|
|
||||||
|
|
||||||
def bootstrap_admin_password():
|
def bootstrap_admin_password():
|
||||||
import secrets as _secrets
|
import secrets as _secrets
|
||||||
users_file = os.path.join(ROOT, 'data', 'api', 'auth_users.json')
|
users_file = os.path.join(ROOT, 'data', 'api', 'auth_users.json')
|
||||||
@@ -303,6 +377,7 @@ def main():
|
|||||||
write_caddy_config(ip_range, cell_name, domain)
|
write_caddy_config(ip_range, cell_name, domain)
|
||||||
ensure_session_secret()
|
ensure_session_secret()
|
||||||
bootstrap_admin_password()
|
bootstrap_admin_password()
|
||||||
|
register_with_ddns(cell_name)
|
||||||
|
|
||||||
print()
|
print()
|
||||||
print('--- Setup complete! Run: make start ---')
|
print('--- Setup complete! Run: make start ---')
|
||||||
|
|||||||
@@ -36,6 +36,7 @@ import app as app_module
|
|||||||
|
|
||||||
class TestAppMisc(unittest.TestCase):
|
class TestAppMisc(unittest.TestCase):
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
|
app_module.app.config['TESTING'] = True
|
||||||
# Patch managers to avoid side effects
|
# Patch managers to avoid side effects
|
||||||
self.patches = [
|
self.patches = [
|
||||||
patch.object(app_module, 'network_manager', MagicMock()),
|
patch.object(app_module, 'network_manager', MagicMock()),
|
||||||
|
|||||||
@@ -37,15 +37,12 @@ def _make_response(status_code=200, json_data=None, text=''):
|
|||||||
|
|
||||||
|
|
||||||
def _make_config_manager(ddns_cfg=None, domain_cfg=None):
|
def _make_config_manager(ddns_cfg=None, domain_cfg=None):
|
||||||
"""Return a mock config_manager whose get_identity() returns a useful dict."""
|
"""Return a mock config_manager with a real configs dict."""
|
||||||
cm = MagicMock()
|
cm = MagicMock()
|
||||||
|
configs = {}
|
||||||
if ddns_cfg is not None:
|
if ddns_cfg is not None:
|
||||||
identity = {'domain': {'ddns': ddns_cfg}}
|
configs['ddns'] = ddns_cfg
|
||||||
elif domain_cfg is not None:
|
cm.configs = configs
|
||||||
identity = {'domain': domain_cfg}
|
|
||||||
else:
|
|
||||||
identity = {}
|
|
||||||
cm.get_identity.return_value = identity
|
|
||||||
return cm
|
return cm
|
||||||
|
|
||||||
|
|
||||||
@@ -83,6 +80,27 @@ class TestPicNgoDDNSRegister(unittest.TestCase):
|
|||||||
_, kwargs = mock_post.call_args
|
_, kwargs = mock_post.call_args
|
||||||
self.assertNotIn('Authorization', kwargs.get('headers', {}))
|
self.assertNotIn('Authorization', kwargs.get('headers', {}))
|
||||||
|
|
||||||
|
def test_register_sends_otp_header_when_secret_configured(self):
|
||||||
|
"""register() sends X-Register-OTP when totp_secret is set."""
|
||||||
|
provider = PicNgoDDNS(totp_secret='JBSWY3DPEHPK3PXP')
|
||||||
|
mock_resp = _make_response(200, json_data={'token': 'tok', 'subdomain': 'x.pic.ngo'})
|
||||||
|
with patch('requests.post', return_value=mock_resp) as mock_post:
|
||||||
|
provider.register('x', '1.2.3.4')
|
||||||
|
_, kwargs = mock_post.call_args
|
||||||
|
self.assertIn('X-Register-OTP', kwargs.get('headers', {}))
|
||||||
|
otp = kwargs['headers']['X-Register-OTP']
|
||||||
|
self.assertEqual(len(otp), 6)
|
||||||
|
self.assertTrue(otp.isdigit())
|
||||||
|
|
||||||
|
def test_register_no_otp_header_without_secret(self):
|
||||||
|
"""register() omits X-Register-OTP when no TOTP secret is configured."""
|
||||||
|
provider = PicNgoDDNS()
|
||||||
|
mock_resp = _make_response(200, json_data={'token': 't', 'subdomain': 'x'})
|
||||||
|
with patch('requests.post', return_value=mock_resp) as mock_post:
|
||||||
|
provider.register('x', '1.2.3.4')
|
||||||
|
_, kwargs = mock_post.call_args
|
||||||
|
self.assertNotIn('X-Register-OTP', kwargs.get('headers', {}))
|
||||||
|
|
||||||
|
|
||||||
class TestPicNgoDDNSUpdate(unittest.TestCase):
|
class TestPicNgoDDNSUpdate(unittest.TestCase):
|
||||||
"""PicNgoDDNS.update() calls the correct URL with Authorization header."""
|
"""PicNgoDDNS.update() calls the correct URL with Authorization header."""
|
||||||
|
|||||||
@@ -53,8 +53,11 @@ def empty_auth_manager(tmp_path):
|
|||||||
os.makedirs(data_dir, exist_ok=True)
|
os.makedirs(data_dir, exist_ok=True)
|
||||||
os.makedirs(config_dir, exist_ok=True)
|
os.makedirs(config_dir, exist_ok=True)
|
||||||
mgr = AuthManager(data_dir=data_dir, config_dir=config_dir)
|
mgr = AuthManager(data_dir=data_dir, config_dir=config_dir)
|
||||||
# The constructor creates the file with '[]' (empty list). We do NOT add
|
# Explicitly create the file with an empty list to simulate the
|
||||||
# any user, so list_users() returns [] but the file is readable.
|
# "auth configured but no users" misconfiguration scenario.
|
||||||
|
users_file = os.path.join(data_dir, 'auth_users.json')
|
||||||
|
with open(users_file, 'w') as f:
|
||||||
|
f.write('[]')
|
||||||
assert mgr.list_users() == [], 'Expected empty user list'
|
assert mgr.list_users() == [], 'Expected empty user list'
|
||||||
return mgr
|
return mgr
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user