#!/usr/bin/env python3 """ WebSocket API test suite for dmf-studio-manager. Usage: ./tests/.venv/bin/python3 tests/test_ws_api.py [options] Options: --url URL WebSocket server URL (default: ws://localhost:7070/ws) --skip-nodes Skip tests that require node binaries and MXL Prerequisites: - studio-manager running (no graph loaded, or will be reset) - Node binaries in the same dir as studio-manager (for node tests) """ import argparse import asyncio import json import os import signal import sys import websockets # ── Minimal graph: testpattern → fakesink (no hardware needed) ──────────────── SIMPLE_GRAPH = { "nodes": [ {"id": "src", "type": "testpattern", "params": {}}, {"id": "sink", "type": "fakesink", "params": {}}, ], "edges": [ { "from": "src", "from_port": "video_flow_id", "to": "sink", "to_port": "video_flow_id", "format": {"kind": "video", "width": 1920, "height": 1080, "fps_num": 25, "fps_den": 1}, } ], } EMPTY_GRAPH = {"nodes": [], "edges": []} # ── Suite / reporter ────────────────────────────────────────────────────────── class Suite: def __init__(self): self.passed = self.failed = self.skipped = 0 def ok(self, name): print(f" \033[32m✓\033[0m {name}") self.passed += 1 def fail(self, name, reason=""): msg = f": {reason}" if reason else "" print(f" \033[31m✗\033[0m {name}{msg}") self.failed += 1 def skip(self, name, reason=""): msg = f" ({reason})" if reason else "" print(f" \033[33m–\033[0m {name}{msg}") self.skipped += 1 def section(self, title): print(f"\n{title}") print("─" * len(title)) def summary(self): total = self.passed + self.failed + self.skipped parts = [f"{self.passed}/{total} passed"] if self.skipped: parts.append(f"{self.skipped} skipped") if self.failed: parts.append(f"\033[31m{self.failed} failed\033[0m") print("\n" + " ".join(parts)) return self.failed == 0 # ── Low-level helpers ───────────────────────────────────────────────────────── async def recv(ws, timeout=3.0): raw = await asyncio.wait_for(ws.recv(), timeout=timeout) return json.loads(raw) async def send_recv(ws, payload, timeout=3.0): await ws.send(json.dumps(payload)) return await recv(ws, timeout) async def open_ws(url): return await websockets.connect(url) async def reset(ws): """Load empty graph to clear all nodes between tests.""" r = await send_recv(ws, {"type": "load_graph", "graph": EMPTY_GRAPH}) return r def is_status(msg): return isinstance(msg, dict) and msg.get("type") == "status" def is_error(msg): return isinstance(msg, dict) and msg.get("type") == "error" def nodes_by_id(status): return {n["id"]: n for n in status.get("nodes", [])} # ── Test groups ─────────────────────────────────────────────────────────────── async def run_connection_tests(url, s): s.section("Connection") # 1. Connect → immediate status push ws = await open_ws(url) try: msg = await recv(ws, timeout=3.0) if is_status(msg) and "nodes" in msg: s.ok("connect → immediate status received") else: s.fail("connect → immediate status received", f"got {msg}") except asyncio.TimeoutError: s.fail("connect → immediate status received", "timeout") finally: await ws.close() # 2. Two clients connect → both receive status independently ws1 = await open_ws(url) ws2 = await open_ws(url) try: m1 = await recv(ws1, timeout=3.0) m2 = await recv(ws2, timeout=3.0) if is_status(m1) and is_status(m2): s.ok("two clients connect → both receive status") else: s.fail("two clients connect → both receive status", f"ws1={m1.get('type')} ws2={m2.get('type')}") except asyncio.TimeoutError: s.fail("two clients connect → both receive status", "timeout") finally: await ws1.close() await ws2.close() async def run_protocol_tests(url, s): s.section("Protocol robustness") ws = await open_ws(url) await recv(ws) # discard initial status # 3. Invalid JSON → error, connection stays open await ws.send("not json {{") try: msg = await recv(ws, timeout=3.0) if is_error(msg): s.ok("invalid JSON → error response") else: s.fail("invalid JSON → error response", f"got type={msg.get('type')}") except asyncio.TimeoutError: s.fail("invalid JSON → error response", "timeout") # 4. Unknown command type → error, connection stays open try: msg = await send_recv(ws, {"type": "explode"}) if is_error(msg): s.ok("unknown command → error response") else: s.fail("unknown command → error response", f"got type={msg.get('type')}") except asyncio.TimeoutError: s.fail("unknown command → error response", "timeout") # 5. Connection still alive after errors try: msg = await send_recv(ws, {"type": "get_status"}) if is_status(msg): s.ok("connection alive after error responses") else: s.fail("connection alive after error responses", f"got {msg}") except Exception as e: s.fail("connection alive after error responses", str(e)) # 6. load_graph with missing 'graph' key → error try: msg = await send_recv(ws, {"type": "load_graph"}) if is_error(msg): s.ok("load_graph missing 'graph' key → error") else: s.fail("load_graph missing 'graph' key → error", f"got {msg}") except asyncio.TimeoutError: s.fail("load_graph missing 'graph' key → error", "timeout") # 7. stop_node with missing 'id' key → error try: msg = await send_recv(ws, {"type": "stop_node"}) if is_error(msg): s.ok("stop_node missing 'id' key → error") else: s.fail("stop_node missing 'id' key → error", f"got {msg}") except asyncio.TimeoutError: s.fail("stop_node missing 'id' key → error", "timeout") await ws.close() async def run_status_tests(url, s): s.section("get_status") ws = await open_ws(url) push = await recv(ws) # 8. get_status response matches last push try: reply = await send_recv(ws, {"type": "get_status"}) if reply.get("nodes") == push.get("nodes"): s.ok("get_status matches initial push") else: s.fail("get_status matches initial push", f"push={push['nodes']} reply={reply['nodes']}") except asyncio.TimeoutError: s.fail("get_status matches initial push", "timeout") await ws.close() async def run_node_tests(url, s): s.section("Graph and node lifecycle (requires node binaries + MXL)") ws = await open_ws(url) await recv(ws) # discard initial status # 9. load_graph with valid simple graph → nodes Running try: r = await send_recv(ws, {"type": "load_graph", "graph": SIMPLE_GRAPH}, timeout=10.0) nodes = nodes_by_id(r) if (is_status(r) and "src" in nodes and nodes["src"]["state"] == "running" and "sink" in nodes and nodes["sink"]["state"] == "running"): s.ok("load valid graph → nodes running") else: s.fail("load valid graph → nodes running", f"states={[(n['id'], n['state']) for n in r.get('nodes', [])]}") except asyncio.TimeoutError: s.fail("load valid graph → nodes running", "timeout") # 10. get_status consistent with load_graph response try: r = await send_recv(ws, {"type": "get_status"}) nodes = nodes_by_id(r) if "src" in nodes and "sink" in nodes: s.ok("get_status consistent after load") else: s.fail("get_status consistent after load", f"nodes={list(nodes.keys())}") except asyncio.TimeoutError: s.fail("get_status consistent after load", "timeout") # 11. load_graph again → previous nodes stopped, new ones started try: r1 = await send_recv(ws, {"type": "get_status"}) old_pids = {n["id"]: n["pid"] for n in r1.get("nodes", [])} r2 = await send_recv(ws, {"type": "load_graph", "graph": SIMPLE_GRAPH}, timeout=10.0) new_nodes = nodes_by_id(r2) new_pids = {nid: n["pid"] for nid, n in new_nodes.items()} if (all(n["state"] == "running" for n in new_nodes.values()) and any(new_pids.get(k) != old_pids.get(k) for k in old_pids)): s.ok("reload graph → new PIDs, all running") else: s.fail("reload graph → new PIDs, all running", f"old={old_pids} new={new_pids}") except asyncio.TimeoutError: s.fail("reload graph → new PIDs, all running", "timeout") # 12. stop_node → state becomes stopped try: r = await send_recv(ws, {"type": "stop_node", "id": "src"}) nodes = nodes_by_id(r) if nodes.get("src", {}).get("state") == "stopped": s.ok("stop_node → state=stopped") else: s.fail("stop_node → state=stopped", f"state={nodes.get('src', {}).get('state')}") except asyncio.TimeoutError: s.fail("stop_node → state=stopped", "timeout") # 13. stop already-stopped node → no crash, state still stopped try: r = await send_recv(ws, {"type": "stop_node", "id": "src"}) nodes = nodes_by_id(r) if is_status(r) and nodes.get("src", {}).get("state") == "stopped": s.ok("stop already-stopped node → graceful no-op") else: s.fail("stop already-stopped node → graceful no-op", f"got {r}") except asyncio.TimeoutError: s.fail("stop already-stopped node → graceful no-op", "timeout") # 14. stop nonexistent node → error or status (not a crash) try: r = await send_recv(ws, {"type": "stop_node", "id": "does-not-exist"}) if is_status(r) or is_error(r): s.ok("stop nonexistent node → no crash") else: s.fail("stop nonexistent node → no crash", f"got {r}") except asyncio.TimeoutError: s.fail("stop nonexistent node → no crash", "timeout") # 15. start_node → state becomes running try: r = await send_recv(ws, {"type": "start_node", "id": "src"}, timeout=5.0) nodes = nodes_by_id(r) if nodes.get("src", {}).get("state") == "running": s.ok("start_node → state=running") else: s.fail("start_node → state=running", f"state={nodes.get('src', {}).get('state')}") except asyncio.TimeoutError: s.fail("start_node → state=running", "timeout") # 16. start already-running node → graceful no-op try: r = await send_recv(ws, {"type": "start_node", "id": "src"}) nodes = nodes_by_id(r) if is_status(r) and nodes.get("src", {}).get("state") == "running": s.ok("start already-running node → graceful no-op") else: s.fail("start already-running node → graceful no-op", f"got {r}") except asyncio.TimeoutError: s.fail("start already-running node → graceful no-op", "timeout") # 17. Node crash detection: kill a node externally → pushed as crashed try: status = await send_recv(ws, {"type": "get_status"}) src_pid = nodes_by_id(status).get("src", {}).get("pid", -1) if src_pid > 0: os.kill(src_pid, signal.SIGKILL) # Monitor polls every 500ms; wait up to 2s for crash push deadline = asyncio.get_event_loop().time() + 2.0 crashed = False while asyncio.get_event_loop().time() < deadline: try: push = await recv(ws, timeout=1.5) if is_status(push): node = nodes_by_id(push).get("src", {}) if node.get("state") == "crashed": crashed = True break except asyncio.TimeoutError: break if crashed: s.ok("crashed node detected and pushed within 2s") else: s.fail("crashed node detected and pushed within 2s", "no push received") else: s.fail("crashed node detected and pushed within 2s", "couldn't get src pid") except Exception as e: s.fail("crashed node detected and pushed within 2s", str(e)) # 18. Push sent to all connected clients simultaneously ws2 = await open_ws(url) await recv(ws2) # discard initial status try: r = await send_recv(ws, {"type": "load_graph", "graph": SIMPLE_GRAPH}, timeout=10.0) # ws gets the response; ws2 should also get a push if is_status(r): try: push2 = await recv(ws2, timeout=3.0) if is_status(push2) and len(push2.get("nodes", [])) > 0: s.ok("load_graph push reaches all connected clients") else: s.fail("load_graph push reaches all connected clients", f"ws2 got {push2}") except asyncio.TimeoutError: s.fail("load_graph push reaches all connected clients", "ws2 got no push") else: s.fail("load_graph push reaches all connected clients", f"load response was {r}") except asyncio.TimeoutError: s.fail("load_graph push reaches all connected clients", "timeout on load") finally: await ws2.close() # 19. Unknown node type → node quickly transitions to crashed try: r = await send_recv(ws, {"type": "load_graph", "graph": { "nodes": [{"id": "bad", "type": "nonexistent-node-type", "params": {}}], "edges": [] }}, timeout=5.0) # The load itself may succeed (fork happens), but the node should crash fast if is_status(r): nodes = nodes_by_id(r) initial_state = nodes.get("bad", {}).get("state") if initial_state == "crashed": s.ok("unknown node type → immediately crashed") else: # Wait for crash push (exec fails → child exits → monitor detects) try: push = await recv(ws, timeout=2.0) if is_status(push) and nodes_by_id(push).get("bad", {}).get("state") == "crashed": s.ok("unknown node type → crashes quickly") else: s.fail("unknown node type → crashes quickly", f"initial={initial_state}, push={push}") except asyncio.TimeoutError: s.fail("unknown node type → crashes quickly", f"initial state={initial_state}, no crash push") else: s.fail("unknown node type → crashes quickly", f"load returned {r}") except asyncio.TimeoutError: s.fail("unknown node type → crashes quickly", "timeout") # Cleanup await reset(ws) await ws.close() # ── Entry point ─────────────────────────────────────────────────────────────── async def main(url, skip_nodes): print(f"studio-manager WebSocket API tests") print(f"server: {url}") s = Suite() try: await run_connection_tests(url, s) await run_protocol_tests(url, s) await run_status_tests(url, s) if skip_nodes: s.section("Graph and node lifecycle (requires node binaries + MXL)") for name in [ "load valid graph → nodes running", "get_status consistent after load", "reload graph → new PIDs, all running", "stop_node → state=stopped", "stop already-stopped node → graceful no-op", "stop nonexistent node → no crash", "start_node → state=running", "start already-running node → graceful no-op", "crashed node detected and pushed within 2s", "load_graph push reaches all connected clients", "unknown node type → crashes quickly", ]: s.skip(name, "--skip-nodes") else: await run_node_tests(url, s) except OSError as e: print(f"\n\033[31mCannot connect to {url}: {e}\033[0m") print("Is studio-manager running?") sys.exit(2) s.summary() sys.exit(0 if s.failed == 0 else 1) if __name__ == "__main__": parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument("--url", default="ws://localhost:7070/ws", help="WebSocket server URL") parser.add_argument("--skip-nodes", action="store_true", help="Skip tests requiring node binaries and MXL") args = parser.parse_args() asyncio.run(main(args.url, args.skip_nodes))