roof
13074f56cb
Unit Tests / test (push) Successful in 12m34s
Root causes fixed:
- Dead LOG_LEVEL globals() lookup pinned root logger at INFO regardless of
PIC_LOG_LEVEL env or config; replaced with _resolve_root_log_level() +
apply_root_log_level() which sets both root logger and all attached handlers
at startup and on runtime re-apply.
- set_service_level() only set the named 'pic.<service>' logger; bare module
loggers (e.g. 'caddy_manager') were never reached, so per-service log files
stayed 0 bytes. Fixed via _SERVICE_MODULE_LOGGERS map covering all managers.
- Log viewer GET /api/logs had no level filter; added ?level= query param.
- Per-service log levels lived in an out-of-band config/api/log_levels.json
side-file with no validation; migrated into ConfigManager under a new
'logging' section ({python:{root,services}, containers:{caddy,coredns,
wireguard,mailserver,api}}) with get/set helpers, invalid-level rejection,
and one-time migration from the old file on first load.
New capabilities:
- Container log levels: Caddy (injects global log { level X } + hot reload),
CoreDNS (DEBUG enables log plugin, else errors-only), WireGuard/mailserver
via pending_restart path.
- PUT /api/logs/verbosity accepts {python, containers} dict; returns per-entry
applied:hot|pending_restart status.
- Webui Logs page gains two-section Verbosity tab (Python services + Container
services) with needs-restart badges.
- managers.py wires per-service loggers before manager instantiation and
re-applies persisted levels from ConfigManager; legacy log_levels.json read
removed.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
533 lines
23 KiB
Python
533 lines
23 KiB
Python
import logging
|
|
import json
|
|
import os
|
|
from datetime import datetime
|
|
from flask import Blueprint, request, jsonify
|
|
logger = logging.getLogger('picell')
|
|
bp = Blueprint('services', __name__)
|
|
|
|
@bp.route('/api/services/catalog', methods=['GET'])
|
|
def get_services_catalog():
|
|
"""
|
|
Return all services (builtins + installed store packages) with merged config.
|
|
Used by the frontend to build navigation and service pages dynamically.
|
|
"""
|
|
try:
|
|
from app import service_registry
|
|
return jsonify({'services': service_registry.list_all()})
|
|
except Exception as e:
|
|
logger.error('get_services_catalog: %s', e)
|
|
return jsonify({'error': str(e)}), 500
|
|
|
|
|
|
@bp.route('/api/services/active', methods=['GET'])
|
|
def get_active_services():
|
|
"""Return minimal info for all installed services. Used by webui to build nav."""
|
|
try:
|
|
from app import service_registry
|
|
active = service_registry.list_active()
|
|
return jsonify([
|
|
{
|
|
'id': svc['id'],
|
|
'name': svc.get('name', svc['id']),
|
|
'subdomain': svc.get('subdomain'),
|
|
'capabilities': svc.get('capabilities', {}),
|
|
}
|
|
for svc in active
|
|
])
|
|
except Exception as e:
|
|
logger.error('get_active_services: %s', e)
|
|
return jsonify({'error': str(e)}), 500
|
|
|
|
|
|
@bp.route('/api/services/catalog/<service_id>', methods=['GET'])
|
|
def get_service_catalog_entry(service_id: str):
|
|
"""Return a single service manifest+config, or 404 if unknown."""
|
|
try:
|
|
from app import service_registry
|
|
svc = service_registry.get(service_id)
|
|
if svc is None:
|
|
return jsonify({'error': f'Service {service_id!r} not found'}), 404
|
|
return jsonify(svc)
|
|
except Exception as e:
|
|
logger.error('get_service_catalog_entry(%s): %s', service_id, e)
|
|
return jsonify({'error': str(e)}), 500
|
|
|
|
|
|
@bp.route('/api/services/catalog/<service_id>/status', methods=['GET'])
|
|
def get_service_container_status(service_id: str):
|
|
"""
|
|
Return container status for a service.
|
|
Builtins query the main compose stack; store services query their own compose project.
|
|
"""
|
|
try:
|
|
from app import service_registry, service_composer
|
|
svc = service_registry.get(service_id)
|
|
if svc is None:
|
|
return jsonify({'error': f'Service {service_id!r} not found'}), 404
|
|
result = service_composer.status_service(service_id, svc)
|
|
return jsonify(result)
|
|
except ValueError as e:
|
|
return jsonify({'error': str(e)}), 400
|
|
except Exception as e:
|
|
logger.error('get_service_container_status(%s): %s', service_id, e)
|
|
return jsonify({'error': str(e)}), 500
|
|
|
|
|
|
@bp.route('/api/services/catalog/<service_id>/restart', methods=['POST'])
|
|
def restart_service_containers(service_id: str):
|
|
"""
|
|
Restart containers for a service.
|
|
Builtins restart via the main compose stack; store services via their own compose project.
|
|
"""
|
|
try:
|
|
from app import service_registry, service_composer
|
|
svc = service_registry.get(service_id)
|
|
if svc is None:
|
|
return jsonify({'error': f'Service {service_id!r} not found'}), 404
|
|
result = service_composer.restart_service(service_id, svc)
|
|
if result['ok']:
|
|
return jsonify({'message': f'Service {service_id!r} restarted', **result})
|
|
return jsonify({'error': result.get('stderr') or result.get('error', 'restart failed')}), 500
|
|
except ValueError as e:
|
|
return jsonify({'error': str(e)}), 400
|
|
except Exception as e:
|
|
logger.error('restart_service_containers(%s): %s', service_id, e)
|
|
return jsonify({'error': str(e)}), 500
|
|
|
|
|
|
@bp.route('/api/services/catalog/<service_id>/reconfigure', methods=['POST'])
|
|
def reconfigure_service(service_id: str):
|
|
"""
|
|
Re-apply the stored compose file for a store service (rolling `up -d`).
|
|
The compose template must already exist on disk from the original install —
|
|
accepting templates from the request body is deliberately not supported
|
|
(arbitrary compose files can mount host paths or request privileged mode).
|
|
"""
|
|
try:
|
|
from app import service_registry, service_composer
|
|
svc = service_registry.get(service_id)
|
|
if svc is None:
|
|
return jsonify({'error': f'Service {service_id!r} not found'}), 404
|
|
if svc.get('kind') == 'builtin':
|
|
return jsonify({'error': 'Builtins are reconfigured via their settings routes'}), 400
|
|
if not service_composer.has_compose_file(service_id):
|
|
return jsonify({'error': f'No compose file for {service_id!r} — install it first'}), 400
|
|
|
|
result = service_composer.up(service_id)
|
|
if result['ok']:
|
|
return jsonify({'message': f'Service {service_id!r} reconfigured', **result})
|
|
return jsonify({'error': result.get('stderr') or result.get('error', 'reconfigure failed')}), 500
|
|
except ValueError as e:
|
|
return jsonify({'error': str(e)}), 400
|
|
except Exception as e:
|
|
logger.error('reconfigure_service(%s): %s', service_id, e)
|
|
return jsonify({'error': str(e)}), 500
|
|
|
|
|
|
@bp.route('/api/services/catalog/<service_id>/accounts', methods=['GET'])
|
|
def list_service_accounts(service_id: str):
|
|
"""Return peer usernames provisioned on a service."""
|
|
try:
|
|
from app import account_manager
|
|
accounts = account_manager.list_accounts(service_id)
|
|
return jsonify({'service_id': service_id, 'accounts': accounts})
|
|
except Exception as e:
|
|
logger.error('list_service_accounts(%s): %s', service_id, e)
|
|
return jsonify({'error': str(e)}), 500
|
|
|
|
|
|
@bp.route('/api/services/catalog/<service_id>/accounts', methods=['POST'])
|
|
def provision_service_account(service_id: str):
|
|
"""Provision a peer account on a service. Generates a password if none is given.
|
|
|
|
The generated or provided password is NOT echoed in this response — retrieve it
|
|
separately via GET /api/services/catalog/<id>/accounts/<username>/credentials.
|
|
This keeps passwords out of HTTP logs and browser network panels.
|
|
"""
|
|
try:
|
|
from app import account_manager
|
|
data = request.get_json(silent=True) or {}
|
|
peer_username = data.get('username')
|
|
if not peer_username:
|
|
return jsonify({'error': 'username is required'}), 400
|
|
account_manager.provision(service_id, peer_username,
|
|
password=data.get('password'))
|
|
return jsonify({'service_id': service_id, 'username': peer_username,
|
|
'provisioned': True}), 201
|
|
except ValueError as e:
|
|
return jsonify({'error': str(e)}), 400
|
|
except RuntimeError as e:
|
|
return jsonify({'error': str(e)}), 500
|
|
except Exception as e:
|
|
logger.error('provision_service_account(%s): %s', service_id, e)
|
|
return jsonify({'error': str(e)}), 500
|
|
|
|
|
|
@bp.route('/api/services/catalog/<service_id>/accounts/<username>', methods=['DELETE'])
|
|
def deprovision_service_account(service_id: str, username: str):
|
|
"""Remove a peer's account from a service."""
|
|
try:
|
|
from app import account_manager
|
|
ok = account_manager.deprovision(service_id, username)
|
|
if ok:
|
|
return jsonify({'message': f'{username!r} deprovisioned from {service_id!r}'})
|
|
return jsonify({'error': 'deprovision failed'}), 500
|
|
except ValueError as e:
|
|
return jsonify({'error': str(e)}), 400
|
|
except Exception as e:
|
|
logger.error('deprovision_service_account(%s, %s): %s', service_id, username, e)
|
|
return jsonify({'error': str(e)}), 500
|
|
|
|
|
|
@bp.route('/api/services/catalog/<service_id>/accounts/<username>/credentials', methods=['GET'])
|
|
def get_service_account_credentials(service_id: str, username: str):
|
|
"""Return stored credentials for a peer on a service."""
|
|
try:
|
|
from app import account_manager
|
|
creds = account_manager.get_credentials(service_id, username)
|
|
if creds is None:
|
|
return jsonify({'error': f'{username!r} not provisioned on {service_id!r}'}), 404
|
|
return jsonify({'service_id': service_id, 'username': username, **creds})
|
|
except Exception as e:
|
|
logger.error('get_service_account_credentials(%s, %s): %s', service_id, username, e)
|
|
return jsonify({'error': str(e)}), 500
|
|
|
|
|
|
@bp.route('/api/services/bus/status', methods=['GET'])
|
|
def get_service_bus_status():
|
|
try:
|
|
from app import service_bus
|
|
return jsonify(service_bus.get_service_status_summary())
|
|
except Exception as e:
|
|
logger.error(f"Error getting service bus status: {e}")
|
|
return jsonify({"error": str(e)}), 500
|
|
|
|
@bp.route('/api/services/bus/events', methods=['GET'])
|
|
def get_service_bus_events():
|
|
try:
|
|
from app import service_bus
|
|
from service_bus import EventType
|
|
event_type = request.args.get('type')
|
|
source = request.args.get('source')
|
|
limit = int(request.args.get('limit', 100))
|
|
events = service_bus.get_event_history(
|
|
EventType(event_type) if event_type else None,
|
|
source,
|
|
limit
|
|
)
|
|
return jsonify([{
|
|
'event_id': e.event_id,
|
|
'event_type': e.event_type.value,
|
|
'source': e.source,
|
|
'data': e.data,
|
|
'timestamp': e.timestamp.isoformat()
|
|
} for e in events])
|
|
except Exception as e:
|
|
logger.error(f"Error getting service bus events: {e}")
|
|
return jsonify({"error": str(e)}), 500
|
|
|
|
@bp.route('/api/services/bus/services/<service_name>/start', methods=['POST'])
|
|
def start_service(service_name):
|
|
try:
|
|
from app import service_bus
|
|
success = service_bus.orchestrate_service_start(service_name)
|
|
if success:
|
|
return jsonify({"message": f"Service {service_name} started successfully"})
|
|
return jsonify({"error": f"Failed to start service {service_name}"}), 500
|
|
except Exception as e:
|
|
logger.error(f"Error starting service {service_name}: {e}")
|
|
return jsonify({"error": str(e)}), 500
|
|
|
|
@bp.route('/api/services/bus/services/<service_name>/stop', methods=['POST'])
|
|
def stop_service(service_name):
|
|
try:
|
|
from app import service_bus
|
|
success = service_bus.orchestrate_service_stop(service_name)
|
|
if success:
|
|
return jsonify({"message": f"Service {service_name} stopped successfully"})
|
|
return jsonify({"error": f"Failed to stop service {service_name}"}), 500
|
|
except Exception as e:
|
|
logger.error(f"Error stopping service {service_name}: {e}")
|
|
return jsonify({"error": str(e)}), 500
|
|
|
|
@bp.route('/api/services/bus/services/<service_name>/restart', methods=['POST'])
|
|
def restart_service(service_name):
|
|
try:
|
|
from app import service_bus
|
|
success = service_bus.orchestrate_service_restart(service_name)
|
|
if success:
|
|
return jsonify({"message": f"Service {service_name} restarted successfully"})
|
|
return jsonify({"error": f"Failed to restart service {service_name}"}), 500
|
|
except Exception as e:
|
|
logger.error(f"Error restarting service {service_name}: {e}")
|
|
return jsonify({"error": str(e)}), 500
|
|
|
|
@bp.route('/api/logs/services/<service>', methods=['GET'])
|
|
def get_service_logs(service):
|
|
try:
|
|
from app import log_manager
|
|
level = request.args.get('level', 'INFO')
|
|
lines = int(request.args.get('lines', 50))
|
|
logs = log_manager.get_service_logs(service, level, lines)
|
|
return jsonify({"service": service, "logs": logs})
|
|
except Exception as e:
|
|
logger.error(f"Error getting logs for {service}: {e}")
|
|
return jsonify({"error": str(e)}), 500
|
|
|
|
@bp.route('/api/logs/search', methods=['POST'])
|
|
def search_logs():
|
|
try:
|
|
from app import log_manager
|
|
data = request.get_json(silent=True) or {}
|
|
results = log_manager.search_logs(
|
|
data.get('query', ''),
|
|
data.get('time_range'),
|
|
data.get('services'),
|
|
data.get('level')
|
|
)
|
|
return jsonify({"results": results, "count": len(results)})
|
|
except Exception as e:
|
|
logger.error(f"Error searching logs: {e}")
|
|
return jsonify({"error": str(e)}), 500
|
|
|
|
@bp.route('/api/logs/export', methods=['POST'])
|
|
def export_logs():
|
|
try:
|
|
from app import log_manager
|
|
data = request.get_json(silent=True) or {}
|
|
format = data.get('format', 'json')
|
|
log_data = log_manager.export_logs(format, data.get('filters', {}))
|
|
return jsonify({"logs": log_data, "format": format})
|
|
except Exception as e:
|
|
logger.error(f"Error exporting logs: {e}")
|
|
return jsonify({"error": str(e)}), 500
|
|
|
|
@bp.route('/api/logs/statistics', methods=['GET'])
|
|
def get_log_statistics():
|
|
try:
|
|
from app import log_manager
|
|
return jsonify(log_manager.get_log_statistics(request.args.get('service')))
|
|
except Exception as e:
|
|
logger.error(f"Error getting log statistics: {e}")
|
|
return jsonify({"error": str(e)}), 500
|
|
|
|
@bp.route('/api/logs/rotate', methods=['POST'])
|
|
def rotate_logs():
|
|
try:
|
|
from app import log_manager
|
|
data = request.get_json(silent=True) or {}
|
|
log_manager.rotate_logs(data.get('service'))
|
|
return jsonify({"message": "Logs rotated successfully"})
|
|
except Exception as e:
|
|
logger.error(f"Error rotating logs: {e}")
|
|
return jsonify({"error": str(e)}), 500
|
|
|
|
@bp.route('/api/logs/files', methods=['GET'])
|
|
def get_log_file_infos():
|
|
try:
|
|
from app import log_manager
|
|
return jsonify(log_manager.get_all_log_file_infos())
|
|
except Exception as e:
|
|
logger.error(f"Error listing log files: {e}")
|
|
return jsonify({"error": str(e)}), 500
|
|
|
|
# Container-ENV driven services need a container recreate before a level change
|
|
# takes effect (the others — caddy/coredns/api — apply hot).
|
|
_RESTART_CONTAINERS = {'wireguard', 'mailserver'}
|
|
|
|
|
|
@bp.route('/api/logs/verbosity', methods=['GET'])
|
|
def get_log_verbosity():
|
|
"""Return both the python (per-service + root) and container log levels."""
|
|
try:
|
|
from app import config_manager
|
|
return jsonify(config_manager.get_logging_config())
|
|
except Exception as e:
|
|
logger.error(f"Error getting log verbosity: {e}")
|
|
return jsonify({"error": str(e)}), 500
|
|
|
|
|
|
@bp.route('/api/logs/verbosity', methods=['PUT'])
|
|
def set_log_verbosity():
|
|
"""Update python and/or container log levels.
|
|
|
|
Payload: {"python": {"root": "DEBUG", "services": {...}}, "containers": {...}}
|
|
Python levels apply hot to the running API. Container levels regenerate the
|
|
relevant config and hot-reload (caddy/coredns) or are queued for the next
|
|
container recreate (wireguard/mailserver). Returns an `applied` map of
|
|
"hot" | "pending_restart" per container entry.
|
|
"""
|
|
try:
|
|
from app import config_manager, log_manager, apply_root_log_level
|
|
data = request.get_json(silent=True) or {}
|
|
python = data.get('python', {}) or {}
|
|
containers = data.get('containers', {}) or {}
|
|
|
|
applied = {}
|
|
|
|
services = python.get('services', {}) or {}
|
|
for service, level in services.items():
|
|
config_manager.set_python_log_level(service, level)
|
|
log_manager.set_service_level(service, level)
|
|
|
|
if 'root' in python:
|
|
config_manager.set_python_log_level('root', python['root'])
|
|
apply_root_log_level(python['root'])
|
|
|
|
for container, level in containers.items():
|
|
config_manager.set_container_log_level(container, level)
|
|
applied[container] = _apply_container_level(container)
|
|
|
|
return jsonify({
|
|
"message": "Log levels updated",
|
|
"logging": config_manager.get_logging_config(),
|
|
"applied": applied,
|
|
})
|
|
except ValueError as e:
|
|
return jsonify({"error": str(e)}), 400
|
|
except Exception as e:
|
|
logger.error(f"Error setting log verbosity: {e}")
|
|
return jsonify({"error": str(e)}), 500
|
|
|
|
|
|
def _apply_container_level(container: str) -> str:
|
|
"""Apply a container's log level. Returns "hot" or "pending_restart"."""
|
|
if container == 'caddy':
|
|
from app import caddy_manager, config_manager
|
|
caddy_manager.regenerate_with_installed(
|
|
list(config_manager.get_installed_services().values())
|
|
)
|
|
return "hot"
|
|
if container == 'coredns':
|
|
from app import firewall_manager, peer_registry, config_manager, cell_link_manager
|
|
peers = peer_registry.list_peers() if peer_registry else []
|
|
cell_links = cell_link_manager.list_connections() if cell_link_manager else None
|
|
firewall_manager.generate_corefile(
|
|
peers, domain=config_manager.get_internal_domain(), cell_links=cell_links)
|
|
firewall_manager.reload_coredns()
|
|
return "hot"
|
|
if container == 'api':
|
|
# The API container's own root level is applied hot via apply_root_log_level
|
|
# when python.root changes; the container entry is informational.
|
|
return "hot"
|
|
if container in _RESTART_CONTAINERS:
|
|
return "pending_restart"
|
|
return "pending_restart"
|
|
|
|
@bp.route('/api/services/status', methods=['GET'])
|
|
def get_all_services_status():
|
|
try:
|
|
from app import service_bus
|
|
services_status = {}
|
|
for service_name in service_bus.list_services():
|
|
try:
|
|
service = service_bus.get_service(service_name)
|
|
status = service.get_status()
|
|
if isinstance(status, dict):
|
|
clean_status = {
|
|
'status': status.get('status', 'unknown'),
|
|
'running': status.get('running', False),
|
|
'timestamp': status.get('timestamp', datetime.utcnow().isoformat())
|
|
}
|
|
if service_name == 'network':
|
|
clean_status.update({
|
|
'dns_status': status.get('dns_running', False),
|
|
'ntp_status': status.get('ntp_running', False)
|
|
})
|
|
elif service_name == 'wireguard':
|
|
clean_status.update({
|
|
'peers_count': status.get('peers_count', 0),
|
|
'interface': status.get('interface', 'unknown')
|
|
})
|
|
elif service_name == 'email':
|
|
clean_status.update({
|
|
'users_count': status.get('users_count', 0),
|
|
'domain': status.get('domain', 'unknown')
|
|
})
|
|
elif service_name == 'calendar':
|
|
clean_status.update({
|
|
'users_count': status.get('users_count', 0),
|
|
'calendars_count': status.get('calendars_count', 0)
|
|
})
|
|
elif service_name == 'files':
|
|
clean_status.update({
|
|
'users_count': status.get('users_count', 0),
|
|
'storage_used': status.get('total_storage_used', {})
|
|
})
|
|
elif service_name == 'routing':
|
|
clean_status.update({
|
|
'nat_rules_count': status.get('nat_rules_count', 0),
|
|
'peer_routes_count': status.get('peer_routes_count', 0),
|
|
'firewall_rules_count': status.get('firewall_rules_count', 0)
|
|
})
|
|
elif service_name == 'vault':
|
|
clean_status.update({
|
|
'certificates_count': status.get('certificates_count', 0),
|
|
'trusted_keys_count': status.get('trusted_keys_count', 0)
|
|
})
|
|
services_status[service_name] = clean_status
|
|
else:
|
|
services_status[service_name] = {'status': str(status), 'running': bool(status)}
|
|
except Exception as e:
|
|
services_status[service_name] = {'error': str(e), 'status': 'offline', 'running': False}
|
|
return jsonify({
|
|
"network": services_status.get('network', {}),
|
|
"wireguard": services_status.get('wireguard', {}),
|
|
"email": services_status.get('email', {}),
|
|
"calendar": services_status.get('calendar', {}),
|
|
"files": services_status.get('files', {}),
|
|
"routing": services_status.get('routing', {}),
|
|
"vault": services_status.get('vault', {}),
|
|
"timestamp": datetime.utcnow().isoformat()
|
|
})
|
|
except Exception as e:
|
|
logger.error(f"Error getting all services status: {e}")
|
|
return jsonify({"error": str(e)}), 500
|
|
|
|
@bp.route('/api/services/connectivity', methods=['GET'])
|
|
def test_all_services_connectivity():
|
|
try:
|
|
from app import service_bus
|
|
connectivity_results = {}
|
|
for service_name in service_bus.list_services():
|
|
try:
|
|
service = service_bus.get_service(service_name)
|
|
if hasattr(service, 'test_connectivity'):
|
|
connectivity_results[service_name] = service.test_connectivity()
|
|
else:
|
|
connectivity_results[service_name] = {'status': 'ok', 'message': 'No connectivity test available'}
|
|
except Exception as e:
|
|
connectivity_results[service_name] = {'status': 'error', 'message': str(e)}
|
|
return jsonify({
|
|
"network": connectivity_results.get('network', {}),
|
|
"wireguard": connectivity_results.get('wireguard', {}),
|
|
"email": connectivity_results.get('email', {}),
|
|
"calendar": connectivity_results.get('calendar', {}),
|
|
"files": connectivity_results.get('files', {}),
|
|
"routing": connectivity_results.get('routing', {}),
|
|
"timestamp": datetime.utcnow().isoformat()
|
|
})
|
|
except Exception as e:
|
|
logger.error(f"Error testing all services connectivity: {e}")
|
|
return jsonify({"error": str(e)}), 500
|
|
|
|
@bp.route('/api/logs', methods=['GET'])
|
|
def get_backend_logs():
|
|
log_file = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'picell.log')
|
|
lines = int(request.args.get('lines', 100))
|
|
level = (request.args.get('level') or 'ALL').upper()
|
|
try:
|
|
if not os.path.exists(log_file):
|
|
return jsonify({"error": "Log file not found."}), 404
|
|
with open(log_file, 'r', encoding='utf-8', errors='ignore') as f:
|
|
all_lines = f.readlines()
|
|
if level != 'ALL':
|
|
from app import log_manager
|
|
all_lines = [ln for ln in all_lines if log_manager._is_log_level(ln, level)]
|
|
tail_lines = all_lines[-lines:] if lines > 0 else all_lines
|
|
return jsonify({"log": ''.join(tail_lines)})
|
|
except Exception as e:
|
|
logger.error(f"Error reading log file: {e}")
|
|
return jsonify({"error": str(e)}), 500
|