Implement connectivity store services (wireguard-ext, openvpn-client, tor)

- ConnectivityManager: move config dirs to data_dir/services/<id>/config so
  Docker can bind-mount them into store-service containers (Docker resolves
  bind-mount paths on the host, not inside the API container).  Add
  _migrate_legacy_configs to copy existing files from the old config_dir
  location on first boot.

- manifest_validator: add allow_host_network parameter to
  validate_rendered_compose.  When True, waives the external-network
  requirement, permits network_mode: host, and allows devices: — all needed
  by VPN/Tor containers that must share the host network namespace to create
  tun/wg interfaces.  Non-host services are unaffected.

- service_composer: read requires_host_network from the manifest and pass
  allow_host_network=True to validate_rendered_compose for connectivity
  services.

- Tests: update file-path assertions to new data_dir layout; add
  TestMigrateLegacyConfigs, TestValidateRenderedComposeHostNetwork, and
  two TestWriteCompose cases for the host-network path.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

api/connectivity_manager.py

@@ -80,19 +80,56 @@ class ConnectivityManager(BaseServiceManager):
         self.config_manager = config_manager
         self.peer_registry = peer_registry
 
-        # Config file directories
+        # Connectivity configs live under the per-service data dir so that
-        self.connectivity_config_dir = os.path.join(config_dir, 'connectivity')
+        # ${PIC_DATA_DIR}/services/<id>/config bind mounts in store compose
-        self.wireguard_ext_dir = os.path.join(self.connectivity_config_dir, 'wireguard_ext')
+        # templates can read them (Docker daemon resolves paths on the HOST,
-        self.openvpn_dir = os.path.join(self.connectivity_config_dir, 'openvpn')
+        # so they must be reachable via data_dir, not config_dir).
+        services_dir = os.path.join(data_dir, 'services')
+        self.wireguard_ext_dir = os.path.join(services_dir, 'wireguard-ext', 'config')
+        self.openvpn_dir       = os.path.join(services_dir, 'openvpn-client', 'config')
 
-        for d in (self.connectivity_config_dir, self.wireguard_ext_dir, self.openvpn_dir):
+        for d in (self.wireguard_ext_dir, self.openvpn_dir):
             self.safe_makedirs(d)
 
+        # One-shot migration from the legacy config_dir/connectivity/ location.
+        _legacy_base = os.path.join(config_dir, 'connectivity')
+        self._migrate_legacy_configs(_legacy_base)
+
         # Subscribe to ServiceBus CONFIG_CHANGED events so routes are
         # reapplied if the underlying network changes.  Done lazily —
         # service_bus is a singleton imported at app startup.
         self._subscribe_to_events()
 
+    # ── Legacy migration ──────────────────────────────────────────────────
+
+    def _migrate_legacy_configs(self, legacy_base: str) -> None:
+        """Copy files from the old config_dir/connectivity/ tree to the new data_dir locations.
+
+        The old layout stored WireGuard and OpenVPN configs under the API container's
+        config_dir, which Docker cannot bind-mount into store-service containers. Files
+        are copied (not moved) so the legacy location still works until the operator
+        removes it manually.
+        """
+        import shutil
+
+        pairs = (
+            (os.path.join(legacy_base, 'wireguard_ext'), self.wireguard_ext_dir),
+            (os.path.join(legacy_base, 'openvpn'), self.openvpn_dir),
+        )
+        for src_dir, dst_dir in pairs:
+            if not os.path.isdir(src_dir):
+                continue
+            try:
+                for fname in os.listdir(src_dir):
+                    src_file = os.path.join(src_dir, fname)
+                    dst_file = os.path.join(dst_dir, fname)
+                    if os.path.isfile(src_file) and not os.path.exists(dst_file):
+                        shutil.copy2(src_file, dst_file)
+                        os.chmod(dst_file, 0o600)
+                        logger.info('connectivity: migrated %s → %s', src_file, dst_file)
+            except OSError as e:
+                logger.warning('connectivity: migration from %s failed: %s', src_dir, e)
+
     # ── Event wiring ──────────────────────────────────────────────────────
 
     def _subscribe_to_events(self) -> None:

api/manifest_validator.py

@@ -158,7 +158,8 @@ def validate_manifest(manifest: dict) -> tuple:
     return (len(errors) == 0, errors)
 
 
-def validate_rendered_compose(yaml_text: str, allowed_data_dir: str = None) -> tuple:
+def validate_rendered_compose(yaml_text: str, allowed_data_dir: str = None,
+                               allow_host_network: bool = False) -> tuple:
     """
     Parse and security-validate a rendered docker-compose YAML string.
 
@@ -168,6 +169,12 @@ def validate_rendered_compose(yaml_text: str, allowed_data_dir: str = None) -> t
     allowed_data_dir: when set, absolute bind mounts under this prefix are
     permitted — they come from ${PIC_DATA_DIR} substitution and land in the
     designated service data directory.
+
+    allow_host_network: when True, the compose file is permitted to use
+    network_mode: host and devices: — required for connectivity services
+    (wireguard-ext, openvpn-client, tor) that must share the host network
+    namespace to create tun/wg interfaces. The external-network requirement
+    is also waived since host-network containers reach the cell network directly.
     """
     errors = []
 
@@ -179,17 +186,19 @@ def validate_rendered_compose(yaml_text: str, allowed_data_dir: str = None) -> t
     if not isinstance(doc, dict):
         return (False, ['compose file must be a YAML mapping'])
 
-    # At least one external network must exist so the container joins the cell network
+    # Regular (bridged) services must join the cell-network so Caddy and CoreDNS
-    # rather than an isolated bridge that would be invisible to Caddy and CoreDNS.
+    # can reach them. Host-network services share the host namespace directly,
-    networks = doc.get('networks') or {}
+    # so the external network declaration would be wrong and is omitted.
-    has_external = any(
+    if not allow_host_network:
-        isinstance(v, dict) and v.get('external')
+        networks = doc.get('networks') or {}
-        for v in networks.values()
+        has_external = any(
-    )
+            isinstance(v, dict) and v.get('external')
-    if not has_external:
+            for v in networks.values()
-        errors.append(
-            'compose file must declare at least one network with external: true'
         )
+        if not has_external:
+            errors.append(
+                'compose file must declare at least one network with external: true'
+            )
 
     for svc_name, svc in (doc.get('services') or {}).items():
         if not isinstance(svc, dict):
@@ -204,10 +213,17 @@ def validate_rendered_compose(yaml_text: str, allowed_data_dir: str = None) -> t
             errors.append(f'{prefix}: privileged: true is not allowed')
 
         net_mode = svc.get('network_mode')
-        if net_mode is not None and net_mode not in (None, 'bridge'):
+        if allow_host_network:
-            errors.append(
+            if net_mode is not None and net_mode not in ('host',):
-                f'{prefix}: network_mode {net_mode!r} is not allowed (only bridge)'
+                errors.append(
-            )
+                    f'{prefix}: network_mode {net_mode!r} is not allowed '
+                    '(connectivity services must use host)'
+                )
+        else:
+            if net_mode is not None and net_mode not in (None, 'bridge'):
+                errors.append(
+                    f'{prefix}: network_mode {net_mode!r} is not allowed (only bridge)'
+                )
 
         if svc.get('pid') == 'host':
             errors.append(f'{prefix}: pid: host is not allowed')
@@ -238,7 +254,7 @@ def validate_rendered_compose(yaml_text: str, allowed_data_dir: str = None) -> t
                     f'{prefix}: absolute host bind mount not allowed: {vol_str!r}'
                 )
 
-        if 'devices' in svc:
+        if 'devices' in svc and not allow_host_network:
             errors.append(f'{prefix}: devices key is not allowed')
 
         for opt in svc.get('security_opt') or []:

api/service_composer.py

@@ -158,8 +158,13 @@ class ServiceComposer:
         # Validate before any file I/O so a bad template never touches disk.
         # Pass the resolved data_dir so that bind mounts created by ${PIC_DATA_DIR}
         # substitution are allowed; all other absolute paths are still rejected.
+        # Connectivity services (wireguard-ext, openvpn-client, tor) set
+        # requires_host_network: true in their manifest to opt into network_mode: host.
+        allow_host_network = bool(manifest.get('requires_host_network'))
         ok, errs = validate_rendered_compose(
-            content, allowed_data_dir=str(Path(self.data_dir).resolve())
+            content,
+            allowed_data_dir=str(Path(self.data_dir).resolve()),
+            allow_host_network=allow_host_network,
         )
         if not ok:
             raise ValueError(

tests/test_connectivity_manager.py

@@ -253,12 +253,12 @@ class TestUploadWireguardExt(unittest.TestCase):
 
     def test_valid_conf_writes_file_to_correct_path(self):
         self.mgr.upload_wireguard_ext(self._valid_conf())
-        expected = os.path.join(self.tmp, 'connectivity', 'wireguard_ext', 'wg_ext0.conf')
+        expected = os.path.join(self.tmp, 'services', 'wireguard-ext', 'config', 'wg_ext0.conf')
         self.assertTrue(os.path.isfile(expected), f'Expected file at {expected}')
 
     def test_valid_conf_file_has_mode_0600(self):
         self.mgr.upload_wireguard_ext(self._valid_conf())
-        path = os.path.join(self.tmp, 'connectivity', 'wireguard_ext', 'wg_ext0.conf')
+        path = os.path.join(self.tmp, 'services', 'wireguard-ext', 'config', 'wg_ext0.conf')
         mode = stat.S_IMODE(os.stat(path).st_mode)
         self.assertEqual(mode, 0o600, f'Expected 0600, got {oct(mode)}')
 
@@ -272,7 +272,7 @@ class TestUploadWireguardExt(unittest.TestCase):
     def test_file_content_has_hooks_stripped(self):
         conf = "[Interface]\nPrivateKey = abc\nPostUp = evil\n"
         self.mgr.upload_wireguard_ext(conf)
-        path = os.path.join(self.tmp, 'connectivity', 'wireguard_ext', 'wg_ext0.conf')
+        path = os.path.join(self.tmp, 'services', 'wireguard-ext', 'config', 'wg_ext0.conf')
         with open(path) as f:
             content = f.read()
         self.assertNotIn('PostUp', content)
@@ -301,12 +301,12 @@ class TestUploadOpenvpn(unittest.TestCase):
 
     def test_valid_conf_writes_file_at_correct_path(self):
         self.mgr.upload_openvpn(self._valid_ovpn(), name='my-vpn')
-        expected = os.path.join(self.tmp, 'connectivity', 'openvpn', 'my-vpn.ovpn')
+        expected = os.path.join(self.tmp, 'services', 'openvpn-client', 'config', 'my-vpn.ovpn')
         self.assertTrue(os.path.isfile(expected), f'Expected file at {expected}')
 
     def test_valid_conf_file_has_mode_0600(self):
         self.mgr.upload_openvpn(self._valid_ovpn(), name='my-vpn')
-        path = os.path.join(self.tmp, 'connectivity', 'openvpn', 'my-vpn.ovpn')
+        path = os.path.join(self.tmp, 'services', 'openvpn-client', 'config', 'my-vpn.ovpn')
         mode = stat.S_IMODE(os.stat(path).st_mode)
         self.assertEqual(mode, 0o600, f'Expected 0600, got {oct(mode)}')
 
@@ -339,19 +339,77 @@ class TestUploadOpenvpn(unittest.TestCase):
     def test_default_name_default_passes(self):
         result = self.mgr.upload_openvpn(self._valid_ovpn())
         self.assertTrue(result['ok'])
-        expected = os.path.join(self.tmp, 'connectivity', 'openvpn', 'default.ovpn')
+        expected = os.path.join(self.tmp, 'services', 'openvpn-client', 'config', 'default.ovpn')
         self.assertTrue(os.path.isfile(expected))
 
     def test_hooks_stripped_from_stored_file(self):
         conf = "client\ndev tun\nup /sbin/bad.sh\nproto udp\n"
         self.mgr.upload_openvpn(conf, name='clean')
-        path = os.path.join(self.tmp, 'connectivity', 'openvpn', 'clean.ovpn')
+        path = os.path.join(self.tmp, 'services', 'openvpn-client', 'config', 'clean.ovpn')
         with open(path) as f:
             content = f.read()
         self.assertNotIn('up /sbin/bad.sh', content)
         self.assertIn('proto udp', content)
 
 
+# ---------------------------------------------------------------------------
+# _migrate_legacy_configs
+# ---------------------------------------------------------------------------
+
+class TestMigrateLegacyConfigs(unittest.TestCase):
+
+    def setUp(self):
+        self.tmp = tempfile.mkdtemp()
+
+    def tearDown(self):
+        shutil.rmtree(self.tmp, ignore_errors=True)
+
+    def test_no_op_when_legacy_dir_absent(self):
+        """No errors when legacy connectivity/ dir does not exist."""
+        mgr = _make_manager(tmp_dir=self.tmp)
+        # Should not raise; legacy dir simply doesn't exist
+        mgr._migrate_legacy_configs(os.path.join(self.tmp, 'nonexistent'))
+
+    def test_wg_conf_copied_to_new_location(self):
+        legacy_wg = os.path.join(self.tmp, 'connectivity', 'wireguard_ext')
+        os.makedirs(legacy_wg)
+        src = os.path.join(legacy_wg, 'wg_ext0.conf')
+        with open(src, 'w') as f:
+            f.write('[Interface]\nPrivateKey = abc\n')
+
+        mgr = _make_manager(tmp_dir=self.tmp)
+        dst = os.path.join(self.tmp, 'services', 'wireguard-ext', 'config', 'wg_ext0.conf')
+        self.assertTrue(os.path.isfile(dst), f'Expected migrated file at {dst}')
+
+    def test_ovpn_copied_to_new_location(self):
+        legacy_ovpn = os.path.join(self.tmp, 'connectivity', 'openvpn')
+        os.makedirs(legacy_ovpn)
+        src = os.path.join(legacy_ovpn, 'default.ovpn')
+        with open(src, 'w') as f:
+            f.write('client\ndev tun\n')
+
+        mgr = _make_manager(tmp_dir=self.tmp)
+        dst = os.path.join(self.tmp, 'services', 'openvpn-client', 'config', 'default.ovpn')
+        self.assertTrue(os.path.isfile(dst), f'Expected migrated file at {dst}')
+
+    def test_existing_dst_not_overwritten(self):
+        legacy_wg = os.path.join(self.tmp, 'connectivity', 'wireguard_ext')
+        os.makedirs(legacy_wg)
+        with open(os.path.join(legacy_wg, 'wg_ext0.conf'), 'w') as f:
+            f.write('legacy\n')
+
+        # Pre-create the destination with different content
+        dst_dir = os.path.join(self.tmp, 'services', 'wireguard-ext', 'config')
+        os.makedirs(dst_dir, exist_ok=True)
+        dst = os.path.join(dst_dir, 'wg_ext0.conf')
+        with open(dst, 'w') as f:
+            f.write('existing\n')
+
+        _make_manager(tmp_dir=self.tmp)
+        with open(dst) as f:
+            self.assertEqual(f.read(), 'existing\n')
+
+
 # ---------------------------------------------------------------------------
 # get_status
 # ---------------------------------------------------------------------------

tests/test_manifest_validator.py

@@ -967,6 +967,127 @@ class TestValidateRenderedCompose(unittest.TestCase):
         self.assertTrue(any('bad' in e for e in errs))
 
 
+# ---------------------------------------------------------------------------
+# TestValidateRenderedComposeHostNetwork
+# ---------------------------------------------------------------------------
+
+class TestValidateRenderedComposeHostNetwork(unittest.TestCase):
+    """Tests for allow_host_network=True — connectivity services."""
+
+    _HOST_NET_COMPOSE = (
+        'services:\n'
+        '  wireguard-ext:\n'
+        '    image: git.pic.ngo/roof/svc-wireguard-ext:latest\n'
+        '    container_name: cell-wg-ext\n'
+        '    restart: unless-stopped\n'
+        '    network_mode: host\n'
+        '    cap_add:\n'
+        '      - NET_ADMIN\n'
+        '    volumes:\n'
+        '      - /app/data/services/wireguard-ext/config:/etc/wireguard\n'
+    )
+
+    def test_host_network_compose_passes_with_flag(self):
+        ok, errs = validate_rendered_compose(
+            self._HOST_NET_COMPOSE,
+            allowed_data_dir='/app/data',
+            allow_host_network=True,
+        )
+        self.assertTrue(ok, errs)
+
+    def test_host_network_compose_fails_without_flag(self):
+        ok, errs = validate_rendered_compose(
+            self._HOST_NET_COMPOSE,
+            allowed_data_dir='/app/data',
+            allow_host_network=False,
+        )
+        self.assertFalse(ok)
+
+    def test_network_mode_host_rejected_without_flag(self):
+        yaml_text = (
+            'services:\n'
+            '  svc:\n'
+            '    image: git.pic.ngo/roof/foo:latest\n'
+            '    network_mode: host\n'
+            'networks:\n'
+            '  cell-network:\n'
+            '    external: true\n'
+        )
+        ok, errs = validate_rendered_compose(yaml_text)
+        self.assertFalse(ok)
+        self.assertTrue(any('network_mode' in e for e in errs))
+
+    def test_devices_allowed_with_flag(self):
+        yaml_text = (
+            'services:\n'
+            '  openvpn-client:\n'
+            '    image: git.pic.ngo/roof/svc-openvpn-client:latest\n'
+            '    container_name: cell-openvpn\n'
+            '    network_mode: host\n'
+            '    cap_add:\n'
+            '      - NET_ADMIN\n'
+            '    devices:\n'
+            '      - /dev/net/tun\n'
+            '    volumes:\n'
+            '      - /app/data/services/openvpn-client/config:/etc/openvpn\n'
+        )
+        ok, errs = validate_rendered_compose(
+            yaml_text,
+            allowed_data_dir='/app/data',
+            allow_host_network=True,
+        )
+        self.assertTrue(ok, errs)
+
+    def test_devices_rejected_without_flag(self):
+        yaml_text = (
+            'services:\n'
+            '  svc:\n'
+            '    image: git.pic.ngo/roof/foo:latest\n'
+            '    devices:\n'
+            '      - /dev/net/tun\n'
+            'networks:\n'
+            '  cell-network:\n'
+            '    external: true\n'
+        )
+        ok, errs = validate_rendered_compose(yaml_text)
+        self.assertFalse(ok)
+        self.assertTrue(any('devices' in e for e in errs))
+
+    def test_no_external_network_ok_with_flag(self):
+        yaml_text = (
+            'services:\n'
+            '  tor:\n'
+            '    image: git.pic.ngo/roof/svc-tor:latest\n'
+            '    network_mode: host\n'
+        )
+        ok, errs = validate_rendered_compose(yaml_text, allow_host_network=True)
+        self.assertTrue(ok, errs)
+
+    def test_privileged_still_rejected_with_flag(self):
+        yaml_text = (
+            'services:\n'
+            '  svc:\n'
+            '    image: git.pic.ngo/roof/foo:latest\n'
+            '    network_mode: host\n'
+            '    privileged: true\n'
+        )
+        ok, errs = validate_rendered_compose(yaml_text, allow_host_network=True)
+        self.assertFalse(ok)
+        self.assertTrue(any('privileged' in e for e in errs))
+
+    def test_non_host_network_mode_rejected_with_flag(self):
+        """When allow_host_network=True, only 'host' is accepted as network_mode."""
+        yaml_text = (
+            'services:\n'
+            '  svc:\n'
+            '    image: git.pic.ngo/roof/foo:latest\n'
+            '    network_mode: none\n'
+        )
+        ok, errs = validate_rendered_compose(yaml_text, allow_host_network=True)
+        self.assertFalse(ok)
+        self.assertTrue(any('network_mode' in e for e in errs))
+
+
 # ---------------------------------------------------------------------------
 # TestValidateProvisionHook
 # ---------------------------------------------------------------------------

tests/test_service_composer.py

@@ -210,6 +210,46 @@ class TestWriteCompose(unittest.TestCase):
             path = os.path.join(tmpdir, 'services', 'myservice', 'docker-compose.yml')
             self.assertTrue(os.path.exists(path))
 
+    def test_requires_host_network_manifest_allows_host_mode_template(self):
+        """write_compose passes when manifest has requires_host_network: true and template uses network_mode: host."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            composer = ServiceComposer(config_manager=_make_cm(), data_dir=tmpdir)
+            manifest = _make_manifest()
+            manifest['requires_host_network'] = True
+            template = (
+                'services:\n'
+                '  wireguard-ext:\n'
+                '    image: git.pic.ngo/roof/svc-wireguard-ext:latest\n'
+                '    container_name: cell-wg-ext\n'
+                '    network_mode: host\n'
+                '    cap_add:\n'
+                '      - NET_ADMIN\n'
+                '    volumes:\n'
+                f'      - {tmpdir}/services/wireguard-ext/config:/etc/wireguard\n'
+            )
+            # Should not raise
+            composer.write_compose('wireguard-ext', manifest, template)
+            path = os.path.join(tmpdir, 'services', 'wireguard-ext', 'docker-compose.yml')
+            self.assertTrue(os.path.exists(path))
+
+    def test_requires_host_network_false_rejects_host_mode_template(self):
+        """write_compose raises when manifest does NOT have requires_host_network but template uses network_mode: host."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            composer = ServiceComposer(config_manager=_make_cm(), data_dir=tmpdir)
+            manifest = _make_manifest()
+            manifest['requires_host_network'] = False
+            template = (
+                'services:\n'
+                '  svc:\n'
+                '    image: git.pic.ngo/roof/svc-foo:latest\n'
+                '    network_mode: host\n'
+                'networks:\n'
+                '  cell-network:\n'
+                '    external: true\n'
+            )
+            with self.assertRaises(ValueError):
+                composer.write_compose('svc', manifest, template)
+
 
 # ── Secrets ───────────────────────────────────────────────────────────────────