feat: proper logging architecture — Docker rotation, persisted service logs, verbosity config

docker-compose.yml:
- Add json-file logging driver (max-size: 10m, max-file: 5) to all 13 containers
- Docker now owns container stdout/stderr rotation automatically
- Add ./data/logs:/app/api/data/logs volume to API — service logs now persist across restarts

log_manager.py:
- Remove container log collection hack (Docker handles it natively)
- Add set_service_level(service, level) — change log level at runtime without restart
- Add get_service_levels() — return current per-service levels
- Simplify get_all_log_file_infos to return only service log files

app.py:
- Add GET /api/logs/verbosity — return current per-service log levels
- Add PUT /api/logs/verbosity — update levels at runtime, persist to config/log_levels.json
- Load persisted log level overrides at startup from log_levels.json
- Simplify rotate endpoint (service logs only, container logs owned by Docker)

wireguard_manager.py:
- get_keys(): return empty strings if key files don't exist (prevents get_status crash
  when wg0.conf is missing at startup and falls through to generate_config)

Logs page (4 tabs):
- API Service Logs: structured JSON logs from Python managers, with search/filter/rotate panel
- Container Logs: live docker logs (read via existing /api/containers/<name>/logs endpoint)
- Verbosity Config: per-service level dropdowns, apply immediately + persist
- Health History: existing health poll table

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

api/app.py

@@ -125,6 +125,16 @@ service_log_configs = {
 for service, config in service_log_configs.items():
     log_manager.add_service_logger(service, config)
 
+# Apply any persisted log level overrides
+_levels_file = os.path.join(os.path.dirname(__file__), 'config', 'log_levels.json')
+if os.path.exists(_levels_file):
+    try:
+        with open(_levels_file) as _f:
+            for _svc, _lvl in json.load(_f).items():
+                log_manager.set_service_level(_svc, _lvl)
+    except Exception:
+        pass
+
 # Start service bus
 service_bus.start()
 
@@ -619,17 +629,11 @@ def get_log_statistics():
 
 @app.route('/api/logs/rotate', methods=['POST'])
 def rotate_logs():
-    """Manually rotate logs (API service log or container log)."""
+    """Manually rotate an API service log file."""
     try:
         data = request.get_json(silent=True) or {}
-        name = data.get('name')   # e.g. 'wireguard' or 'container_cell-api'
-        kind = data.get('kind', 'service')  # 'service' or 'container'
-
-        if kind == 'container':
-            container_name = name[len('container_'):] if name and name.startswith('container_') else name
-            log_manager.rotate_container_log(container_name)
-        else:
-            log_manager.rotate_logs(name)
+        service = data.get('service')  # None = rotate all
+        log_manager.rotate_logs(service)
         return jsonify({"message": "Logs rotated successfully"})
     except Exception as e:
         logger.error(f"Error rotating logs: {e}")
@@ -637,23 +641,45 @@ def rotate_logs():
 
 @app.route('/api/logs/files', methods=['GET'])
 def get_log_file_infos():
-    """List all stored log files (service + container) with sizes."""
+    """List service log files with sizes."""
     try:
         return jsonify(log_manager.get_all_log_file_infos())
     except Exception as e:
         logger.error(f"Error listing log files: {e}")
         return jsonify({"error": str(e)}), 500
 
-@app.route('/api/logs/containers/<container_name>', methods=['GET'])
-def get_stored_container_logs(container_name):
-    """Collect latest docker logs into file and return last N lines."""
+@app.route('/api/logs/verbosity', methods=['GET'])
+def get_log_verbosity():
+    """Return current per-service log levels."""
     try:
-        tail = int(request.args.get('tail', 100))
-        log_manager.collect_container_logs(container_name)
-        lines = log_manager.get_container_log_lines(container_name, tail)
-        return jsonify({'container': container_name, 'lines': lines})
+        return jsonify(log_manager.get_service_levels())
     except Exception as e:
-        logger.error(f"Error getting stored container logs for {container_name}: {e}")
+        logger.error(f"Error getting log verbosity: {e}")
+        return jsonify({"error": str(e)}), 500
+
+@app.route('/api/logs/verbosity', methods=['PUT'])
+def set_log_verbosity():
+    """Update log levels for one or all services. Body: {service: level} map."""
+    try:
+        data = request.get_json(silent=True) or {}
+        for service, level in data.items():
+            log_manager.set_service_level(service, level)
+        # Persist to config so levels survive API restarts
+        levels_file = os.path.join(os.path.dirname(__file__), 'config', 'log_levels.json')
+        os.makedirs(os.path.dirname(levels_file), exist_ok=True)
+        current = {}
+        if os.path.exists(levels_file):
+            try:
+                with open(levels_file) as f:
+                    current = json.load(f)
+            except Exception:
+                pass
+        current.update(data)
+        with open(levels_file, 'w') as f:
+            json.dump(current, f, indent=2)
+        return jsonify({"message": "Log levels updated", "levels": log_manager.get_service_levels()})
+    except Exception as e:
+        logger.error(f"Error setting log verbosity: {e}")
         return jsonify({"error": str(e)}), 500
 
 # Network Services API

api/log_manager.py

@@ -498,88 +498,35 @@ class LogManager:
         except Exception as e:
             return {'error': str(e)}
     
-    def collect_container_logs(self, container_name: str, docker_client=None) -> int:
-        """Append new docker container stdout/stderr to a persistent log file.
-        Returns number of new lines written, or -1 on error."""
+    def set_service_level(self, service: str, level: str):
+        """Change log level for a service at runtime."""
         try:
-            import subprocess
-            log_file = self.log_dir / f'container_{container_name}.log'
-            # Determine --since timestamp from last line of existing file
-            since_arg = []
-            if log_file.exists() and log_file.stat().st_size > 0:
-                with open(log_file, 'r', encoding='utf-8', errors='ignore') as f:
-                    for line in f:
-                        pass
-                # Parse last timestamp from docker log line (format: 2006-01-02T15:04:05...)
-                ts_match = re.match(r'^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z)', line.strip())
-                if ts_match:
-                    since_arg = ['--since', ts_match.group(1)]
-
-            result = subprocess.run(
-                ['docker', 'logs', '--timestamps'] + since_arg + [container_name],
-                capture_output=True, text=True, timeout=30
-            )
-            output = result.stdout + result.stderr
-            lines = [l for l in output.splitlines() if l.strip()]
-            if lines:
-                with open(log_file, 'a', encoding='utf-8') as f:
-                    f.write('\n'.join(lines) + '\n')
-            return len(lines)
+            log_level = getattr(logging, level.upper(), logging.INFO)
+            if service in self.service_loggers:
+                self.service_loggers[service].setLevel(log_level)
+                if service in self.handlers and 'file' in self.handlers[service]:
+                    self.handlers[service]['file'].setLevel(log_level)
+                logger.info(f"Set log level for {service} to {level}")
+            else:
+                logger.warning(f"Service logger not found: {service}")
         except Exception as e:
-            logger.error(f"Error collecting container logs for {container_name}: {e}")
-            return -1
+            logger.error(f"Error setting log level for {service}: {e}")
 
-    def get_container_log_lines(self, container_name: str, lines: int = 100) -> List[str]:
-        """Read last N lines from stored container log file."""
-        try:
-            log_file = self.log_dir / f'container_{container_name}.log'
-            if not log_file.exists():
-                return []
-            with open(log_file, 'r', encoding='utf-8', errors='ignore') as f:
-                all_lines = f.readlines()
-            return [l.rstrip() for l in all_lines[-lines:]] if lines > 0 else [l.rstrip() for l in all_lines]
-        except Exception as e:
-            logger.error(f"Error reading container log for {container_name}: {e}")
-            return []
-
-    def rotate_container_log(self, container_name: str):
-        """Rotate a stored container log file."""
-        try:
-            log_file = self.log_dir / f'container_{container_name}.log'
-            if not log_file.exists():
-                return
-            # Find next available backup index
-            for i in range(1, self.backup_count + 1):
-                backup = self.log_dir / f'container_{container_name}.log.{i}'
-                if not backup.exists():
-                    log_file.rename(backup)
-                    logger.info(f"Rotated container log for {container_name} → {backup.name}")
-                    return
-            # All slots full — remove oldest, shift others
-            oldest = self.log_dir / f'container_{container_name}.log.{self.backup_count}'
-            oldest.unlink(missing_ok=True)
-            for i in range(self.backup_count - 1, 0, -1):
-                src = self.log_dir / f'container_{container_name}.log.{i}'
-                if src.exists():
-                    src.rename(self.log_dir / f'container_{container_name}.log.{i + 1}')
-            log_file.rename(self.log_dir / f'container_{container_name}.log.1')
-            logger.info(f"Rotated container log for {container_name}")
-        except Exception as e:
-            logger.error(f"Error rotating container log for {container_name}: {e}")
+    def get_service_levels(self) -> Dict[str, str]:
+        """Return current log level for each service logger."""
+        return {
+            svc: logging.getLevelName(lgr.level)
+            for svc, lgr in self.service_loggers.items()
+        }
 
     def get_all_log_file_infos(self) -> List[Dict[str, Any]]:
-        """Return size/mtime info for all log files (API service logs + container logs)."""
+        """Return size/mtime info for all service log files."""
         results = []
         for log_file in sorted(self.log_dir.glob('*.log')):
             try:
                 stat = log_file.stat()
-                name = log_file.stem  # e.g. 'wireguard' or 'container_cell-api'
-                kind = 'container' if name.startswith('container_') else 'service'
-                label = name[len('container_'):] if kind == 'container' else name
                 results.append({
-                    'name': name,
-                    'label': label,
-                    'kind': kind,
+                    'name': log_file.stem,
                     'file': log_file.name,
                     'size': stat.st_size,
                     'modified': datetime.fromtimestamp(stat.st_mtime).isoformat(),

api/wireguard_manager.py

@@ -74,9 +74,13 @@ class WireGuardManager(BaseServiceManager):
                 pass
 
     def get_keys(self) -> Dict[str, str]:
-        """Return server public/private keys as base64 strings."""
+        """Return server public/private keys as base64 strings. Generates them if missing."""
         priv_file = os.path.join(self.keys_dir, 'private.key')
         pub_file = os.path.join(self.keys_dir, 'public.key')
+        if not os.path.exists(priv_file):
+            self._ensure_server_keys()
+        if not os.path.exists(priv_file):
+            return {'private_key': '', 'public_key': ''}
         with open(priv_file, 'rb') as f:
             priv = f.read()
         with open(pub_file, 'rb') as f: