feat: connectivity redesign phase 3+4 — per-connection health, per-peer fallback, connection CRUD API

Health probes (probe_health/refresh_health) are type-aware: WireGuard
checks the last WG handshake timestamp, OpenVPN checks the tun/tap
interface, Tor checks the control-port GETINFO, and sshuttle/proxy
types do a TCP reachability probe to the remote endpoint. Results are
persisted via set_connection_status and wired into the health_monitor_loop
so the UI always has a current health snapshot without polling.

Per-peer fail-open semantics: VPN, SSH, and proxy connections default to
fail-closed (kill-switch stays active even when the tunnel is down).
Tor defaults to fail-open. The default can be overridden per-peer via
set_peer_failopen/effective_failopen. apply_routes skips the fwmark and
kill-switch rules for any fail-open peer whose connection health is not
"working", letting traffic fall back to direct routing transparently.

New generic admin-only connection CRUD endpoints (GET/POST/PUT/DELETE
/api/connectivity/connections, GET /<id>/health, PUT
/api/connectivity/peers/<peer>/failopen) are guarded by the existing
admin role check. connection.create, connection.update, connection.delete,
and peer.failopen are all registered in ROUTE_ACTION_MAP for the audit
hook so every change is recorded in the owner-visible change log.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

tests/test_audit_hook_routes.py

@@ -110,6 +110,39 @@ def test_unmapped_mutating_endpoint_gets_generic_action(auth_mgr, audit_mgr):
     assert match[0]['target_type'] == 'unknown'
 
 
+# ── connectivity v2 connection routes are audited ─────────────────────────────
+
+def test_connection_create_audited(auth_mgr, audit_mgr):
+    with _client(auth_mgr, audit_mgr, login_as='admin') as c:
+        with patch('app.connectivity_manager') as cm:
+            cm.create_connection.return_value = {'ok': True, 'connection': {'id': 'c'}}
+            c.post('/api/connectivity/connections',
+                   json={'type': 'tor', 'name': 'T'})
+    res = audit_mgr.query({'action': 'connection.create'})
+    assert res['total'] >= 1
+    assert res['entries'][0]['target_type'] == 'connection'
+
+
+def test_connection_delete_audited_with_id(auth_mgr, audit_mgr):
+    with _client(auth_mgr, audit_mgr, login_as='admin') as c:
+        with patch('app.connectivity_manager') as cm:
+            cm.delete_connection.return_value = {'ok': True}
+            c.delete('/api/connectivity/connections/conn_abc')
+    res = audit_mgr.query({'action': 'connection.delete'})
+    assert res['total'] >= 1
+    assert res['entries'][0]['target_id'] == 'conn_abc'
+
+
+def test_peer_failopen_audited(auth_mgr, audit_mgr):
+    with _client(auth_mgr, audit_mgr, login_as='admin') as c:
+        with patch('app.connectivity_manager') as cm:
+            cm.set_peer_failopen.return_value = {'ok': True, 'peer': 'bob'}
+            c.put('/api/connectivity/peers/bob/failopen', json={'failopen': True})
+    res = audit_mgr.query({'action': 'peer.failopen'})
+    assert res['total'] >= 1
+    assert res['entries'][0]['target_id'] == 'bob'
+
+
 # ── auth routes: never write password ─────────────────────────────────────────
 
 def test_change_password_audited_without_value(auth_mgr, audit_mgr):

tests/test_connectivity_api.py

@@ -0,0 +1,331 @@
+"""
+Phase 4 tests for the generic connection CRUD REST API and the per-peer
+fail-open endpoint. Logic lives in ConnectivityManager (mocked here); these
+tests assert the thin route layer: status codes, error mapping (404/409/400),
+that secrets are never echoed, and that admin-only enforcement applies.
+
+The `client` fixture sets TESTING=True (bypassing auth/CSRF) for happy-path
+status-code checks; admin-only enforcement is verified separately against a
+real seeded AuthManager with TESTING off.
+"""
+
+import os
+import sys
+import json
+from pathlib import Path
+from unittest.mock import patch, MagicMock
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).parent.parent / 'api'))
+
+import app as app_module
+from app import app
+from auth_manager import AuthManager
+
+
+@pytest.fixture
+def client():
+    app.config['TESTING'] = True
+    with app.test_client() as c:
+        yield c
+
+
+# ---------------------------------------------------------------------------
+# GET /api/connectivity/connections
+# ---------------------------------------------------------------------------
+
+class TestListConnections:
+    def test_list_returns_connections_with_status(self, client):
+        cm = MagicMock()
+        cm.list_connections.return_value = [
+            {'id': 'conn_a', 'type': 'proxy', 'name': 'P',
+             'secret_refs': ['conn_a_password'],
+             'status': {'state': 'configured', 'health': 'working'},
+             'config': {'host': 'p', 'port': 3128}},
+        ]
+        with patch.object(app_module, 'connectivity_manager', cm):
+            resp = client.get('/api/connectivity/connections')
+        assert resp.status_code == 200
+        body = resp.get_json()
+        assert body['connections'][0]['id'] == 'conn_a'
+        assert body['connections'][0]['status']['health'] == 'working'
+
+    def test_list_never_returns_secret_values(self, client):
+        cm = MagicMock()
+        # _public_record strips secret values; the manager is what enforces it.
+        cm.list_connections.return_value = [
+            {'id': 'conn_a', 'type': 'sshuttle', 'name': 'S',
+             'secret_refs': ['conn_a_private_key'],
+             'config': {'host': 'h', 'user': 'u', 'auth': 'key'},
+             'status': {}},
+        ]
+        with patch.object(app_module, 'connectivity_manager', cm):
+            resp = client.get('/api/connectivity/connections')
+        raw = resp.get_data(as_text=True)
+        assert 'PRIVATE KEY' not in raw
+        assert 'private_key' not in json.loads(raw)['connections'][0]['config']
+
+    def test_list_500_on_exception(self, client):
+        cm = MagicMock()
+        cm.list_connections.side_effect = RuntimeError('boom')
+        with patch.object(app_module, 'connectivity_manager', cm):
+            resp = client.get('/api/connectivity/connections')
+        assert resp.status_code == 500
+
+
+# ---------------------------------------------------------------------------
+# POST /api/connectivity/connections
+# ---------------------------------------------------------------------------
+
+class TestCreateConnection:
+    def test_create_201_with_record(self, client):
+        cm = MagicMock()
+        cm.create_connection.return_value = {
+            'ok': True, 'connection': {'id': 'conn_x', 'type': 'proxy',
+                                       'secret_refs': []}}
+        with patch.object(app_module, 'connectivity_manager', cm):
+            resp = client.post('/api/connectivity/connections',
+                               json={'type': 'proxy', 'name': 'My Proxy',
+                                     'config': {'scheme': 'http', 'host': 'p',
+                                                'port': 3128}})
+        assert resp.status_code == 201
+        assert resp.get_json()['connection']['id'] == 'conn_x'
+
+    def test_create_passes_secrets_to_manager(self, client):
+        cm = MagicMock()
+        cm.create_connection.return_value = {'ok': True, 'connection': {'id': 'c'}}
+        with patch.object(app_module, 'connectivity_manager', cm):
+            client.post('/api/connectivity/connections',
+                        json={'type': 'sshuttle', 'name': 'S',
+                              'config': {'host': 'h'},
+                              'secrets': {'private_key': 'SECRET'}})
+        _, kwargs = cm.create_connection.call_args
+        assert kwargs['secrets'] == {'private_key': 'SECRET'}
+
+    def test_create_does_not_echo_secret_value(self, client):
+        cm = MagicMock()
+        cm.create_connection.return_value = {
+            'ok': True, 'connection': {'id': 'c', 'secret_refs': ['c_private_key'],
+                                       'config': {}}}
+        with patch.object(app_module, 'connectivity_manager', cm):
+            resp = client.post('/api/connectivity/connections',
+                               json={'type': 'sshuttle', 'name': 'S',
+                                     'secrets': {'private_key': 'TOPSECRET'}})
+        assert 'TOPSECRET' not in resp.get_data(as_text=True)
+
+    def test_create_missing_type_400(self, client):
+        resp = client.post('/api/connectivity/connections', json={'name': 'x'})
+        assert resp.status_code == 400
+
+    def test_create_missing_name_400(self, client):
+        resp = client.post('/api/connectivity/connections', json={'type': 'tor'})
+        assert resp.status_code == 400
+
+    def test_create_validation_error_400(self, client):
+        cm = MagicMock()
+        cm.create_connection.return_value = {'ok': False, 'error': 'invalid host'}
+        with patch.object(app_module, 'connectivity_manager', cm):
+            resp = client.post('/api/connectivity/connections',
+                               json={'type': 'proxy', 'name': 'P', 'config': {}})
+        assert resp.status_code == 400
+        assert 'invalid host' in resp.get_json()['error']
+
+
+# ---------------------------------------------------------------------------
+# PUT /api/connectivity/connections/<id>
+# ---------------------------------------------------------------------------
+
+class TestUpdateConnection:
+    def test_update_200(self, client):
+        cm = MagicMock()
+        cm.update_connection.return_value = {
+            'ok': True, 'connection': {'id': 'conn_a', 'name': 'New'}}
+        with patch.object(app_module, 'connectivity_manager', cm):
+            resp = client.put('/api/connectivity/connections/conn_a',
+                              json={'name': 'New'})
+        assert resp.status_code == 200
+
+    def test_update_not_found_404(self, client):
+        cm = MagicMock()
+        cm.update_connection.return_value = {
+            'ok': False, 'error': "connection 'conn_z' not found"}
+        with patch.object(app_module, 'connectivity_manager', cm):
+            resp = client.put('/api/connectivity/connections/conn_z',
+                              json={'name': 'x'})
+        assert resp.status_code == 404
+
+    def test_update_validation_400(self, client):
+        cm = MagicMock()
+        cm.update_connection.return_value = {'ok': False, 'error': 'invalid name'}
+        with patch.object(app_module, 'connectivity_manager', cm):
+            resp = client.put('/api/connectivity/connections/conn_a',
+                              json={'name': '!!!'})
+        assert resp.status_code == 400
+
+
+# ---------------------------------------------------------------------------
+# DELETE /api/connectivity/connections/<id>
+# ---------------------------------------------------------------------------
+
+class TestDeleteConnection:
+    def test_delete_200(self, client):
+        cm = MagicMock()
+        cm.delete_connection.return_value = {'ok': True}
+        with patch.object(app_module, 'connectivity_manager', cm):
+            resp = client.delete('/api/connectivity/connections/conn_a')
+        assert resp.status_code == 200
+
+    def test_delete_referenced_409(self, client):
+        cm = MagicMock()
+        cm.delete_connection.return_value = {
+            'ok': False, 'error': "connection is in use by peer 'alice'; detach it first"}
+        with patch.object(app_module, 'connectivity_manager', cm):
+            resp = client.delete('/api/connectivity/connections/conn_a')
+        assert resp.status_code == 409
+        assert 'in use by' in resp.get_json()['error']
+
+    def test_delete_not_found_404(self, client):
+        cm = MagicMock()
+        cm.delete_connection.return_value = {
+            'ok': False, 'error': "connection 'conn_z' not found"}
+        with patch.object(app_module, 'connectivity_manager', cm):
+            resp = client.delete('/api/connectivity/connections/conn_z')
+        assert resp.status_code == 404
+
+
+# ---------------------------------------------------------------------------
+# GET /api/connectivity/connections/<id>/health
+# ---------------------------------------------------------------------------
+
+class TestConnectionHealthEndpoint:
+    def test_health_returns_probe_result(self, client):
+        cm = MagicMock()
+        cm.get_connection.return_value = {'id': 'conn_a', 'type': 'proxy'}
+        cm.probe_health.return_value = ('working', 'reachable')
+        with patch.object(app_module, 'connectivity_manager', cm):
+            resp = client.get('/api/connectivity/connections/conn_a/health')
+        assert resp.status_code == 200
+        body = resp.get_json()
+        assert body['health'] == 'working'
+        assert body['detail'] == 'reachable'
+
+    def test_health_unknown_connection_404(self, client):
+        cm = MagicMock()
+        cm.get_connection.return_value = None
+        with patch.object(app_module, 'connectivity_manager', cm):
+            resp = client.get('/api/connectivity/connections/conn_z/health')
+        assert resp.status_code == 404
+
+
+# ---------------------------------------------------------------------------
+# PUT /api/connectivity/peers/<peer>/failopen
+# ---------------------------------------------------------------------------
+
+class TestSetPeerFailopen:
+    def test_set_true_200(self, client):
+        cm = MagicMock()
+        cm.set_peer_failopen.return_value = {
+            'ok': True, 'peer': 'alice', 'exit_failopen': True}
+        with patch.object(app_module, 'connectivity_manager', cm):
+            resp = client.put('/api/connectivity/peers/alice/failopen',
+                              json={'failopen': True})
+        assert resp.status_code == 200
+        cm.set_peer_failopen.assert_called_once_with('alice', True)
+
+    def test_clear_with_null_200(self, client):
+        cm = MagicMock()
+        cm.set_peer_failopen.return_value = {
+            'ok': True, 'peer': 'alice', 'exit_failopen': None}
+        with patch.object(app_module, 'connectivity_manager', cm):
+            resp = client.put('/api/connectivity/peers/alice/failopen',
+                              json={'failopen': None})
+        assert resp.status_code == 200
+        cm.set_peer_failopen.assert_called_once_with('alice', None)
+
+    def test_non_bool_400(self, client):
+        resp = client.put('/api/connectivity/peers/alice/failopen',
+                          json={'failopen': 'yes'})
+        assert resp.status_code == 400
+
+    def test_unknown_peer_404(self, client):
+        cm = MagicMock()
+        cm.set_peer_failopen.return_value = {
+            'ok': False, 'error': "peer 'ghost' not found"}
+        with patch.object(app_module, 'connectivity_manager', cm):
+            resp = client.put('/api/connectivity/peers/ghost/failopen',
+                              json={'failopen': True})
+        assert resp.status_code == 404
+
+
+# ---------------------------------------------------------------------------
+# admin-only enforcement (mutating connection routes)
+# ---------------------------------------------------------------------------
+
+def _seed_auth(tmp_path):
+    data_dir = str(tmp_path / 'data')
+    config_dir = str(tmp_path / 'config')
+    os.makedirs(data_dir, exist_ok=True)
+    os.makedirs(config_dir, exist_ok=True)
+    mgr = AuthManager(data_dir=data_dir, config_dir=config_dir)
+    mgr.create_user('admin', 'AdminPass123!', 'admin')
+    mgr.create_user('alice', 'AlicePass123!', 'peer')
+    return mgr
+
+
+class TestAdminOnly:
+    def _login(self, c, user, pw):
+        return c.post('/api/auth/login',
+                      data=json.dumps({'username': user, 'password': pw}),
+                      content_type='application/json')
+
+    def test_peer_role_forbidden_on_create(self, tmp_path):
+        auth = _seed_auth(tmp_path)
+        app.config['TESTING'] = False
+        app.config['SECRET_KEY'] = 'test-secret'
+        try:
+            import auth_routes
+            with patch.object(app_module, 'auth_manager', auth), \
+                 patch.object(auth_routes, 'auth_manager', auth, create=True), \
+                 patch.object(app_module.setup_manager, 'is_setup_complete',
+                              return_value=True):
+                with app.test_client() as c:
+                    assert self._login(c, 'alice', 'AlicePass123!').status_code == 200
+                    # CSRF token from session for the mutating request.
+                    with c.session_transaction() as sess:
+                        token = sess.get('csrf_token')
+                    resp = c.post('/api/connectivity/connections',
+                                  json={'type': 'tor', 'name': 'T'},
+                                  headers={'X-CSRF-Token': token or ''})
+            assert resp.status_code == 403
+        finally:
+            app.config['TESTING'] = True
+
+    def test_admin_role_allowed_on_create(self, tmp_path):
+        auth = _seed_auth(tmp_path)
+        cm = MagicMock()
+        cm.create_connection.return_value = {'ok': True, 'connection': {'id': 'c'}}
+        app.config['TESTING'] = False
+        app.config['SECRET_KEY'] = 'test-secret'
+        try:
+            import auth_routes
+            with patch.object(app_module, 'auth_manager', auth), \
+                 patch.object(auth_routes, 'auth_manager', auth, create=True), \
+                 patch.object(app_module, 'connectivity_manager', cm), \
+                 patch.object(app_module.setup_manager, 'is_setup_complete',
+                              return_value=True):
+                with app.test_client() as c:
+                    assert self._login(c, 'admin', 'AdminPass123!').status_code == 200
+                    with c.session_transaction() as sess:
+                        token = sess.get('csrf_token')
+                    resp = c.post('/api/connectivity/connections',
+                                  json={'type': 'tor', 'name': 'T'},
+                                  headers={'X-CSRF-Token': token or ''})
+            assert resp.status_code == 201
+        finally:
+            app.config['TESTING'] = True
+
+
+if __name__ == '__main__':
+    import pytest as _pytest
+    _pytest.main([__file__, '-q'])

tests/test_connectivity_health.py

@@ -0,0 +1,453 @@
+"""
+Phase 3 tests for ConnectivityManager — per-connection health probing,
+refresh_health persistence + TTL cache, per-peer configurable fallback
+(exit_failopen + type defaults), and apply_routes fail-open/fail-closed
+behaviour while a connection is DOWN.
+
+All real subprocess/socket access is mocked via the small helper methods
+(_exec_in_container, _tcp_reachable, _container_running, _listener_reachable,
+_wg_ip) so no live infrastructure is touched.
+"""
+
+import os
+import sys
+import shutil
+import tempfile
+import time
+import unittest
+from unittest.mock import MagicMock, patch
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'api'))
+
+import connectivity_manager as cm_module
+from connectivity_manager import ConnectivityManager
+
+
+def _make_manager(config_manager=None, peer_registry=None, tmp_dir=None):
+    if tmp_dir is None:
+        tmp_dir = tempfile.mkdtemp()
+    if config_manager is None:
+        config_manager = MagicMock()
+        config_manager.get_identity.return_value = {
+            'cell_name': 'test', 'ip_range': '172.20.0.0/16'}
+        config_manager.list_connections.return_value = []
+    if peer_registry is None:
+        peer_registry = MagicMock()
+        peer_registry.list_peers.return_value = []
+    with patch.object(ConnectivityManager, '_subscribe_to_events',
+                      lambda self: None):
+        mgr = ConnectivityManager(
+            config_manager=config_manager,
+            peer_registry=peer_registry,
+            data_dir=tmp_dir,
+            config_dir=tmp_dir,
+        )
+    return mgr
+
+
+def _cp(returncode=0, stdout='', stderr=''):
+    return MagicMock(returncode=returncode, stdout=stdout, stderr=stderr)
+
+
+# ---------------------------------------------------------------------------
+# probe_health per type
+# ---------------------------------------------------------------------------
+
+class TestProbeHealth(unittest.TestCase):
+
+    def setUp(self):
+        self.tmp = tempfile.mkdtemp()
+        self.mgr = _make_manager(tmp_dir=self.tmp)
+
+    def tearDown(self):
+        shutil.rmtree(self.tmp, ignore_errors=True)
+
+    # wireguard_ext ---------------------------------------------------------
+
+    def test_wireguard_recent_handshake_working(self):
+        conn = {'id': 'c1', 'type': 'wireguard_ext', 'iface': 'wgext_a'}
+        recent = str(int(time.time()) - 10)
+        with patch.object(self.mgr, '_exec_in_container',
+                          return_value=_cp(stdout=f'PUBKEY\t{recent}\n')):
+            health, detail = self.mgr.probe_health(conn)
+        self.assertEqual(health, 'working')
+
+    def test_wireguard_stale_handshake_down(self):
+        conn = {'id': 'c1', 'type': 'wireguard_ext', 'iface': 'wgext_a'}
+        stale = str(int(time.time()) - 9999)
+        with patch.object(self.mgr, '_exec_in_container',
+                          return_value=_cp(stdout=f'PUBKEY\t{stale}\n')):
+            health, _ = self.mgr.probe_health(conn)
+        self.assertEqual(health, 'down')
+
+    def test_wireguard_no_handshake_down(self):
+        conn = {'id': 'c1', 'type': 'wireguard_ext', 'iface': 'wgext_a'}
+        with patch.object(self.mgr, '_exec_in_container',
+                          return_value=_cp(stdout='PUBKEY\t0\n')):
+            health, _ = self.mgr.probe_health(conn)
+        self.assertEqual(health, 'down')
+
+    def test_wireguard_exec_fails_down(self):
+        conn = {'id': 'c1', 'type': 'wireguard_ext', 'iface': 'wgext_a'}
+        with patch.object(self.mgr, '_exec_in_container', return_value=None):
+            health, _ = self.mgr.probe_health(conn)
+        self.assertEqual(health, 'down')
+
+    def test_wireguard_no_iface_unknown(self):
+        conn = {'id': 'c1', 'type': 'wireguard_ext', 'iface': None}
+        health, _ = self.mgr.probe_health(conn)
+        self.assertEqual(health, 'unknown')
+
+    # openvpn ---------------------------------------------------------------
+
+    def test_openvpn_tun_up_working(self):
+        conn = {'id': 'c2', 'type': 'openvpn', 'iface': 'ovpn_x'}
+        with patch.object(self.mgr, '_exec_in_container',
+                          return_value=_cp(stdout='5: ovpn_x: <UP,LOWER_UP>')):
+            health, _ = self.mgr.probe_health(conn)
+        self.assertEqual(health, 'working')
+
+    def test_openvpn_tun_down(self):
+        conn = {'id': 'c2', 'type': 'openvpn', 'iface': 'ovpn_x'}
+        with patch.object(self.mgr, '_exec_in_container',
+                          return_value=_cp(stdout='5: ovpn_x: <DOWN>')):
+            health, _ = self.mgr.probe_health(conn)
+        self.assertEqual(health, 'down')
+
+    def test_openvpn_falls_back_to_wg_container_tun(self):
+        conn = {'id': 'c2', 'type': 'openvpn', 'iface': 'ovpn_x'}
+        with patch.object(self.mgr, '_exec_in_container', return_value=None), \
+             patch.object(self.mgr, '_wg_ip',
+                          return_value=_cp(stdout='tun0: <UP>')):
+            health, _ = self.mgr.probe_health(conn)
+        self.assertEqual(health, 'working')
+
+    # tor -------------------------------------------------------------------
+
+    def test_tor_bootstrapped_working(self):
+        conn = {'id': 'c3', 'type': 'tor'}
+        with patch.object(self.mgr, '_container_running', return_value=True), \
+             patch.object(self.mgr, '_exec_in_container',
+                          return_value=_cp(stdout='Bootstrapped 100% (done)')):
+            health, _ = self.mgr.probe_health(conn)
+        self.assertEqual(health, 'working')
+
+    def test_tor_container_down(self):
+        conn = {'id': 'c3', 'type': 'tor'}
+        with patch.object(self.mgr, '_container_running', return_value=False):
+            health, _ = self.mgr.probe_health(conn)
+        self.assertEqual(health, 'down')
+
+    def test_tor_running_but_no_bootstrap_log_still_working(self):
+        conn = {'id': 'c3', 'type': 'tor'}
+        with patch.object(self.mgr, '_container_running', return_value=True), \
+             patch.object(self.mgr, '_exec_in_container',
+                          return_value=_cp(stdout='')):
+            health, _ = self.mgr.probe_health(conn)
+        self.assertEqual(health, 'working')
+
+    # sshuttle --------------------------------------------------------------
+
+    def test_sshuttle_host_and_listener_working(self):
+        conn = {'id': 'c4', 'type': 'sshuttle', 'redirect_port': 9100,
+                'config': {'host': 'ssh.example.com', 'port': 22}}
+        with patch.object(self.mgr, '_tcp_reachable', return_value=True), \
+             patch.object(self.mgr, '_listener_reachable', return_value=True):
+            health, _ = self.mgr.probe_health(conn)
+        self.assertEqual(health, 'working')
+
+    def test_sshuttle_host_unreachable_down(self):
+        conn = {'id': 'c4', 'type': 'sshuttle', 'redirect_port': 9100,
+                'config': {'host': 'ssh.example.com', 'port': 22}}
+        with patch.object(self.mgr, '_tcp_reachable', return_value=False):
+            health, _ = self.mgr.probe_health(conn)
+        self.assertEqual(health, 'down')
+
+    def test_sshuttle_listener_down(self):
+        conn = {'id': 'c4', 'type': 'sshuttle', 'redirect_port': 9100,
+                'config': {'host': 'ssh.example.com', 'port': 22}}
+        with patch.object(self.mgr, '_tcp_reachable', return_value=True), \
+             patch.object(self.mgr, '_listener_reachable', return_value=False):
+            health, _ = self.mgr.probe_health(conn)
+        self.assertEqual(health, 'down')
+
+    def test_sshuttle_no_host_unknown(self):
+        conn = {'id': 'c4', 'type': 'sshuttle', 'config': {}}
+        health, _ = self.mgr.probe_health(conn)
+        self.assertEqual(health, 'unknown')
+
+    # proxy -----------------------------------------------------------------
+
+    def test_proxy_reachable_working(self):
+        conn = {'id': 'c5', 'type': 'proxy',
+                'config': {'host': 'proxy.example.com', 'port': 3128}}
+        with patch.object(self.mgr, '_tcp_reachable', return_value=True):
+            health, _ = self.mgr.probe_health(conn)
+        self.assertEqual(health, 'working')
+
+    def test_proxy_unreachable_down(self):
+        conn = {'id': 'c5', 'type': 'proxy',
+                'config': {'host': 'proxy.example.com', 'port': 3128}}
+        with patch.object(self.mgr, '_tcp_reachable', return_value=False):
+            health, _ = self.mgr.probe_health(conn)
+        self.assertEqual(health, 'down')
+
+    def test_proxy_missing_config_unknown(self):
+        conn = {'id': 'c5', 'type': 'proxy', 'config': {}}
+        health, _ = self.mgr.probe_health(conn)
+        self.assertEqual(health, 'unknown')
+
+    def test_unknown_type_returns_unknown(self):
+        health, _ = self.mgr.probe_health({'id': 'x', 'type': 'bogus'})
+        self.assertEqual(health, 'unknown')
+
+    def test_probe_never_raises(self):
+        conn = {'id': 'c5', 'type': 'proxy',
+                'config': {'host': 'h', 'port': 1}}
+        with patch.object(self.mgr, '_tcp_reachable',
+                          side_effect=RuntimeError('boom')):
+            health, detail = self.mgr.probe_health(conn)
+        self.assertEqual(health, 'unknown')
+        self.assertIn('boom', detail)
+
+
+# ---------------------------------------------------------------------------
+# refresh_health — persistence + TTL cache
+# ---------------------------------------------------------------------------
+
+class TestRefreshHealth(unittest.TestCase):
+
+    def setUp(self):
+        self.tmp = tempfile.mkdtemp()
+        self.conns = [
+            {'id': 'c1', 'type': 'proxy', 'enabled': True,
+             'config': {'host': 'p', 'port': 3128}, 'status': {}},
+        ]
+        self.cm = MagicMock()
+        self.cm.list_connections.return_value = self.conns
+        self.mgr = _make_manager(config_manager=self.cm, tmp_dir=self.tmp)
+
+    def tearDown(self):
+        shutil.rmtree(self.tmp, ignore_errors=True)
+
+    def test_refresh_persists_status(self):
+        with patch.object(self.mgr, '_tcp_reachable', return_value=True):
+            result = self.mgr.refresh_health()
+        self.assertEqual(result['c1'], 'working')
+        self.cm.set_connection_status.assert_called_once()
+        cid, status = self.cm.set_connection_status.call_args.args
+        self.assertEqual(cid, 'c1')
+        self.assertEqual(status['health'], 'working')
+        self.assertIsNotNone(status['last_check'])
+
+    def test_refresh_single_connection_only(self):
+        self.conns.append({'id': 'c2', 'type': 'proxy', 'enabled': True,
+                           'config': {'host': 'p2', 'port': 3128}, 'status': {}})
+        with patch.object(self.mgr, '_tcp_reachable', return_value=True):
+            result = self.mgr.refresh_health(connection_id='c2')
+        self.assertEqual(list(result.keys()), ['c2'])
+
+    def test_ttl_skips_fresh_connections(self):
+        fresh = self.mgr._now_iso()
+        self.conns[0]['status'] = {'health': 'working', 'last_check': fresh}
+        with patch.object(self.mgr, '_tcp_reachable') as tcp:
+            result = self.mgr.refresh_health()
+        tcp.assert_not_called()
+        self.assertEqual(result['c1'], 'working')
+        self.cm.set_connection_status.assert_not_called()
+
+    def test_force_reprobes_even_when_fresh(self):
+        fresh = self.mgr._now_iso()
+        self.conns[0]['status'] = {'health': 'working', 'last_check': fresh}
+        with patch.object(self.mgr, '_tcp_reachable', return_value=False):
+            self.mgr.refresh_health(force=True)
+        self.cm.set_connection_status.assert_called_once()
+
+    def test_disabled_connection_skipped(self):
+        self.conns[0]['enabled'] = False
+        with patch.object(self.mgr, '_tcp_reachable', return_value=True) as tcp:
+            result = self.mgr.refresh_health()
+        tcp.assert_not_called()
+        self.assertEqual(result, {})
+
+
+# ---------------------------------------------------------------------------
+# per-peer fail-open resolution
+# ---------------------------------------------------------------------------
+
+class TestFailopenResolution(unittest.TestCase):
+
+    def setUp(self):
+        self.tmp = tempfile.mkdtemp()
+        self.pr = MagicMock()
+        self.mgr = _make_manager(peer_registry=self.pr, tmp_dir=self.tmp)
+
+    def tearDown(self):
+        shutil.rmtree(self.tmp, ignore_errors=True)
+
+    def test_type_default_tor_fails_open(self):
+        peer = {'peer': 'a'}
+        self.assertTrue(self.mgr.effective_failopen(peer, {'type': 'tor'}))
+
+    def test_type_default_wireguard_fails_closed(self):
+        peer = {'peer': 'a'}
+        self.assertFalse(
+            self.mgr.effective_failopen(peer, {'type': 'wireguard_ext'}))
+
+    def test_override_true_beats_type_default(self):
+        peer = {'peer': 'a', 'exit_failopen': True}
+        self.assertTrue(
+            self.mgr.effective_failopen(peer, {'type': 'wireguard_ext'}))
+
+    def test_override_false_beats_tor_default(self):
+        peer = {'peer': 'a', 'exit_failopen': False}
+        self.assertFalse(self.mgr.effective_failopen(peer, {'type': 'tor'}))
+
+    def test_none_override_uses_type_default(self):
+        peer = {'peer': 'a', 'exit_failopen': None}
+        self.assertTrue(self.mgr.effective_failopen(peer, {'type': 'tor'}))
+
+    def test_set_peer_failopen_updates_peer(self):
+        self.pr.get_peer.return_value = {'peer': 'a'}
+        with patch.object(self.mgr, 'apply_routes', return_value={'ok': True}):
+            result = self.mgr.set_peer_failopen('a', True)
+        self.assertTrue(result['ok'])
+        self.pr.update_peer.assert_called_once_with('a', {'exit_failopen': True})
+
+    def test_set_peer_failopen_clear_with_null(self):
+        self.pr.get_peer.return_value = {'peer': 'a'}
+        with patch.object(self.mgr, 'apply_routes', return_value={'ok': True}):
+            result = self.mgr.set_peer_failopen('a', None)
+        self.assertTrue(result['ok'])
+        self.pr.update_peer.assert_called_once_with('a', {'exit_failopen': None})
+
+    def test_set_peer_failopen_unknown_peer(self):
+        self.pr.get_peer.return_value = None
+        result = self.mgr.set_peer_failopen('ghost', True)
+        self.assertFalse(result['ok'])
+        self.assertIn('not found', result['error'])
+
+    def test_set_peer_failopen_rejects_non_bool(self):
+        self.pr.get_peer.return_value = {'peer': 'a'}
+        result = self.mgr.set_peer_failopen('a', 'yes')
+        self.assertFalse(result['ok'])
+
+
+# ---------------------------------------------------------------------------
+# apply_routes — fail-open / fail-closed under DOWN connections
+# ---------------------------------------------------------------------------
+
+class TestApplyRoutesFallback(unittest.TestCase):
+
+    def setUp(self):
+        self.tmp = tempfile.mkdtemp()
+
+    def tearDown(self):
+        shutil.rmtree(self.tmp, ignore_errors=True)
+
+    def _mgr(self, conns, peers, peer_ips):
+        cm = MagicMock()
+        cm.get_identity.return_value = {'cell_name': 't', 'ip_range': '172.20.0.0/16'}
+        cm.list_connections.return_value = conns
+        pr = MagicMock()
+        pr.list_peers.return_value = peers
+        pr.get_peer.side_effect = lambda n: peer_ips.get(n)
+        return _make_manager(config_manager=cm, peer_registry=pr, tmp_dir=self.tmp)
+
+    def test_down_failopen_peer_skips_mark_and_killswitch(self):
+        conns = [{'id': 'c1', 'type': 'wireguard_ext', 'enabled': True,
+                  'mark': 0x1000, 'table': 1000, 'iface': 'wgext_a',
+                  'redirect_port': None, 'status': {'health': 'down'}}]
+        peers = [{'peer': 'a', 'exit_via': 'c1', 'exit_failopen': True}]
+        ips = {'a': {'peer': 'a', 'ip': '172.20.0.50/32'}}
+        mgr = self._mgr(conns, peers, ips)
+
+        with patch.object(mgr, '_add_mark_rule') as mark, \
+             patch.object(mgr, '_add_killswitch') as ks, \
+             patch.object(cm_module, 'subprocess') as sp:
+            sp.run.return_value = _cp()
+            mgr.apply_routes()
+
+        mark.assert_not_called()
+        ks.assert_not_called()
+
+    def test_down_failclosed_peer_keeps_mark_and_killswitch(self):
+        conns = [{'id': 'c1', 'type': 'wireguard_ext', 'enabled': True,
+                  'mark': 0x1000, 'table': 1000, 'iface': 'wgext_a',
+                  'redirect_port': None, 'status': {'health': 'down'}}]
+        # No override → wireguard_ext default is fail-closed.
+        peers = [{'peer': 'a', 'exit_via': 'c1'}]
+        ips = {'a': {'peer': 'a', 'ip': '172.20.0.50/32'}}
+        mgr = self._mgr(conns, peers, ips)
+
+        with patch.object(mgr, '_add_mark_rule') as mark, \
+             patch.object(mgr, '_add_killswitch') as ks, \
+             patch.object(cm_module, 'subprocess') as sp:
+            sp.run.return_value = _cp()
+            mgr.apply_routes()
+
+        mark.assert_called_once_with('172.20.0.50', 0x1000)
+        ks.assert_called_once_with(0x1000, 'wgext_a')
+
+    def test_working_failopen_peer_routes_normally(self):
+        conns = [{'id': 'c1', 'type': 'wireguard_ext', 'enabled': True,
+                  'mark': 0x1000, 'table': 1000, 'iface': 'wgext_a',
+                  'redirect_port': None, 'status': {'health': 'working'}}]
+        peers = [{'peer': 'a', 'exit_via': 'c1', 'exit_failopen': True}]
+        ips = {'a': {'peer': 'a', 'ip': '172.20.0.50/32'}}
+        mgr = self._mgr(conns, peers, ips)
+
+        with patch.object(mgr, '_add_mark_rule') as mark, \
+             patch.object(mgr, '_add_killswitch') as ks, \
+             patch.object(cm_module, 'subprocess') as sp:
+            sp.run.return_value = _cp()
+            mgr.apply_routes()
+
+        mark.assert_called_once_with('172.20.0.50', 0x1000)
+        ks.assert_called_once_with(0x1000, 'wgext_a')
+
+    def test_unknown_health_routes_normally(self):
+        """A never-probed connection (health unknown) must not silently drop."""
+        conns = [{'id': 'c1', 'type': 'wireguard_ext', 'enabled': True,
+                  'mark': 0x1000, 'table': 1000, 'iface': 'wgext_a',
+                  'redirect_port': None, 'status': {'health': 'unknown'}}]
+        peers = [{'peer': 'a', 'exit_via': 'c1', 'exit_failopen': True}]
+        ips = {'a': {'peer': 'a', 'ip': '172.20.0.50/32'}}
+        mgr = self._mgr(conns, peers, ips)
+
+        with patch.object(mgr, '_add_mark_rule') as mark, \
+             patch.object(cm_module, 'subprocess') as sp:
+            sp.run.return_value = _cp()
+            mgr.apply_routes()
+
+        mark.assert_called_once_with('172.20.0.50', 0x1000)
+
+    def test_mixed_peers_failclosed_keeps_killswitch(self):
+        """When one peer fails open and another fails closed on the same DOWN
+        connection, the fail-closed peer keeps its mark and the killswitch
+        stays so its traffic is blocked while the tunnel is down."""
+        conns = [{'id': 'c1', 'type': 'wireguard_ext', 'enabled': True,
+                  'mark': 0x1000, 'table': 1000, 'iface': 'wgext_a',
+                  'redirect_port': None, 'status': {'health': 'down'}}]
+        peers = [
+            {'peer': 'a', 'exit_via': 'c1', 'exit_failopen': True},
+            {'peer': 'b', 'exit_via': 'c1', 'exit_failopen': False},
+        ]
+        ips = {'a': {'peer': 'a', 'ip': '172.20.0.50/32'},
+               'b': {'peer': 'b', 'ip': '172.20.0.51/32'}}
+        mgr = self._mgr(conns, peers, ips)
+
+        marked = []
+        with patch.object(mgr, '_add_mark_rule',
+                          side_effect=lambda ip, m: marked.append(ip)), \
+             patch.object(mgr, '_add_killswitch') as ks, \
+             patch.object(cm_module, 'subprocess') as sp:
+            sp.run.return_value = _cp()
+            mgr.apply_routes()
+
+        self.assertEqual(marked, ['172.20.0.51'])
+        ks.assert_called_once_with(0x1000, 'wgext_a')
+
+
+if __name__ == '__main__':
+    unittest.main()