feat: scaffold project framework with engine, node skeleton, and passthrough node

- CMake build system with vcpkg dependency management
- libdmf-node: Node interface, port definitions, WebSocket control
  server, node runner with MXL lifecycle management
- libdmf-engine: Graph model (nodes/edges CRUD, serialization),
  FlowManager (UUID + V210 NMOS flow definitions), ProcessManager
  (fork/exec node processes), ApiServer (REST API for graph control)
- Passthrough node: reads V210 grains from MXL, copies, writes to MXL
- Unit tests for Graph and FlowManager (6 cases, 21 assertions)
- MXL SDK as git submodule (symlinked from extern/mxl)
This commit is contained in:
Johanness
2026-05-26 01:32:50 +03:00
parent ea0eaf8d48
commit 5b5ffa1308
30 changed files with 1547 additions and 0 deletions
+18
View File
@@ -0,0 +1,18 @@
add_library(dmf-engine STATIC
src/graph.cpp
src/flow_manager.cpp
src/process_manager.cpp
src/api_server.cpp
)
target_include_directories(dmf-engine PUBLIC
include
)
target_link_libraries(dmf-engine PUBLIC
dmf-node
nlohmann_json::nlohmann_json
spdlog::spdlog
fmt::fmt
websockets
)
@@ -0,0 +1,33 @@
#pragma once
#include <dmf-engine/graph.hpp>
#include <functional>
#include <memory>
#include <string>
struct lws_context;
namespace dmf_engine {
using RequestHandler = std::function<std::string(const std::string& method, const std::string& path, const std::string& body)>;
class ApiServer {
public:
ApiServer(uint16_t port, Graph& graph, class FlowManager& flow_manager, class ProcessManager& process_manager);
~ApiServer();
ApiServer(const ApiServer&) = delete;
ApiServer& operator=(const ApiServer&) = delete;
void poll(int timeout_ms);
private:
void handle_rest(const std::string& method, const std::string& path, const std::string& body,
std::string& response_status, std::string& response_content_type, std::string& response_body);
struct Impl;
std::unique_ptr<Impl> impl_;
};
} // namespace dmf_engine
@@ -0,0 +1,22 @@
#pragma once
#include <dmf-engine/types.hpp>
#include <nlohmann/json.hpp>
#include <string>
namespace dmf_engine {
class FlowManager {
public:
FlowManager();
FlowId create_flow_id();
nlohmann::json create_v210_flow_def(const FlowId& flow_id, int width, int height, int fps_numerator, int fps_denominator) const;
private:
int flow_counter_ = 0;
};
} // namespace dmf_engine
@@ -0,0 +1,62 @@
#pragma once
#include <dmf-engine/types.hpp>
#include <nlohmann/json.hpp>
#include <optional>
#include <string>
#include <unordered_map>
#include <vector>
namespace dmf_engine {
enum class NodeState {
Stopped,
Starting,
Running,
Stopping,
};
struct GraphNode {
NodeId id;
std::string type;
nlohmann::json config;
NodeState state = NodeState::Stopped;
uint16_t control_port = 0;
int pid = 0;
};
struct GraphEdge {
EdgeId id;
NodeId from_node;
PortId from_port;
NodeId to_node;
PortId to_port;
FlowId flow_id;
nlohmann::json flow_def;
};
class Graph {
public:
NodeId add_node(const std::string& type, const nlohmann::json& config = {});
bool remove_node(const NodeId& node_id);
const GraphNode* get_node(const NodeId& node_id) const;
std::vector<GraphNode> get_nodes() const;
EdgeId add_edge(const NodeId& from_node, const PortId& from_port,
const NodeId& to_node, const PortId& to_port,
const FlowId& flow_id, const nlohmann::json& flow_def);
bool remove_edge(const EdgeId& edge_id);
std::vector<GraphEdge> get_edges() const;
std::vector<GraphEdge> get_edges_for_node(const NodeId& node_id) const;
nlohmann::json serialize() const;
private:
std::unordered_map<NodeId, GraphNode> nodes_;
std::unordered_map<EdgeId, GraphEdge> edges_;
int next_node_num_ = 0;
};
} // namespace dmf_engine
@@ -0,0 +1,22 @@
#pragma once
#include <dmf-engine/graph.hpp>
#include <string>
#include <unordered_map>
namespace dmf_engine {
class ProcessManager {
public:
bool start_node(GraphNode& node, const std::string& mxl_domain, uint16_t base_port);
bool stop_node(GraphNode& node);
void stop_all();
bool is_running(const NodeId& node_id) const;
private:
std::string find_node_binary(const std::string& node_type) const;
};
} // namespace dmf_engine
@@ -0,0 +1,13 @@
#pragma once
#include <string>
#include <cstdint>
namespace dmf_engine {
using NodeId = std::string;
using EdgeId = std::string;
using FlowId = std::string;
using PortId = std::string;
} // namespace dmf_engine
+223
View File
@@ -0,0 +1,223 @@
#include <dmf-engine/api_server.hpp>
#include <dmf-engine/graph.hpp>
#include <dmf-engine/flow_manager.hpp>
#include <dmf-engine/process_manager.hpp>
#include <libwebsockets.h>
#include <nlohmann/json.hpp>
#include <spdlog/spdlog.h>
#include <cstring>
#include <string>
namespace dmf_engine {
struct ApiServerImpl {
uint16_t port = 0;
Graph* graph = nullptr;
FlowManager* flow_manager = nullptr;
ProcessManager* process_manager = nullptr;
struct lws_context* context = nullptr;
};
struct ApiServer::Impl {
std::unique_ptr<ApiServerImpl> data;
};
struct HttpRequest {
std::string method;
std::string path;
std::string body;
bool body_done = false;
};
static ApiServerImpl* g_impl = nullptr;
static int callback_http(struct lws* wsi, enum lws_callback_reasons reason,
void* user, void* in, size_t len) {
auto* req = static_cast<HttpRequest*>(user);
switch (reason) {
case LWS_CALLBACK_HTTP: {
new (req) HttpRequest();
if (lws_hdr_total_length(wsi, WSI_TOKEN_POST_URI) > 0) {
req->method = "POST";
char buf[256] = {};
lws_hdr_copy(wsi, buf, sizeof(buf), WSI_TOKEN_POST_URI);
req->path = buf;
} else {
req->method = "GET";
char buf[256] = {};
lws_hdr_copy(wsi, buf, sizeof(buf), WSI_TOKEN_GET_URI);
req->path = buf;
}
break;
}
case LWS_CALLBACK_HTTP_BODY: {
req->body.append(static_cast<char*>(in), len);
break;
}
case LWS_CALLBACK_HTTP_BODY_COMPLETION: {
if (!g_impl || !g_impl->graph) {
lws_return_http_status(wsi, HTTP_STATUS_INTERNAL_SERVER_ERROR, "Server error");
return -1;
}
std::string status_str, content_type, response_body;
auto ok = [&](const nlohmann::json& j) {
status_str = "200 OK";
response_body = j.dump();
};
auto created = [&](const nlohmann::json& j) {
status_str = "201 Created";
response_body = j.dump();
};
auto error_resp = [&](int code, const std::string& msg) {
status_str = std::to_string(code) + " Error";
response_body = nlohmann::json({{"error", msg}}).dump();
};
content_type = "application/json";
try {
nlohmann::json req_body = req->body.empty() ? nlohmann::json::object() : nlohmann::json::parse(req->body);
auto& graph = *g_impl->graph;
auto& fm = *g_impl->flow_manager;
auto& pm = *g_impl->process_manager;
if (req->path == "/api/graph" && req->method == "GET") {
ok(graph.serialize());
} else if (req->path == "/api/graph/nodes" && req->method == "POST") {
if (!req_body.contains("type")) {
error_resp(400, "Missing 'type' field");
} else {
auto type = req_body["type"].get<std::string>();
auto config = req_body.value("config", nlohmann::json::object());
auto id = graph.add_node(type, config);
created({{"id", id}});
}
} else if (req->path.find("/api/graph/nodes/") == 0 && req->method == "DELETE") {
auto node_id = req->path.substr(std::string("/api/graph/nodes/").length());
if (graph.remove_node(node_id)) {
ok({{"deleted", node_id}});
} else {
error_resp(404, "Node not found: " + node_id);
}
} else if (req->path == "/api/graph/edges" && req->method == "POST") {
if (!req_body.contains("from_node") || !req_body.contains("to_node")) {
error_resp(400, "Missing from_node/to_node");
} else {
auto from_node = req_body["from_node"].get<std::string>();
auto from_port = req_body.value("from_port", "video_out");
auto to_node = req_body["to_node"].get<std::string>();
auto to_port = req_body.value("to_port", "video_in");
auto flow_id = fm.create_flow_id();
auto flow_def = fm.create_v210_flow_def(flow_id, 1920, 1080, 50, 1);
auto edge_id = graph.add_edge(from_node, from_port, to_node, to_port, flow_id, flow_def);
created({{"id", edge_id}, {"flow_id", flow_id}});
}
} else if (req->path.find("/api/graph/edges/") == 0 && req->method == "DELETE") {
auto edge_id = req->path.substr(std::string("/api/graph/edges/").length());
if (graph.remove_edge(edge_id)) {
ok({{"deleted", edge_id}});
} else {
error_resp(404, "Edge not found: " + edge_id);
}
} else if (req->path == "/api/graph/start" && req->method == "POST") {
auto nodes = graph.get_nodes();
uint16_t port = 9100;
for (auto& node : nodes) {
pm.start_node(const_cast<GraphNode&>(node), "/dev/shm/mxl", port++);
}
ok({{"status", "started"}});
} else if (req->path == "/api/graph/stop" && req->method == "POST") {
auto nodes = graph.get_nodes();
for (auto& node : nodes) {
pm.stop_node(const_cast<GraphNode&>(node));
}
ok({{"status", "stopped"}});
} else {
error_resp(404, "Not found: " + req->method + " " + req->path);
}
} catch (const nlohmann::json::exception& e) {
error_resp(400, std::string("JSON error: ") + e.what());
} catch (const std::exception& e) {
error_resp(500, e.what());
}
auto headers = "HTTP/1.1 " + status_str + "\r\n"
"Content-Type: " + content_type + "\r\n"
"Access-Control-Allow-Origin: *\r\n"
"Access-Control-Allow-Methods: GET, POST, DELETE, PUT, OPTIONS\r\n"
"Access-Control-Allow-Headers: Content-Type\r\n"
"Content-Length: " + std::to_string(response_body.size()) + "\r\n"
"\r\n";
std::vector<uint8_t> buf(LWS_PRE + headers.size() + response_body.size());
std::memcpy(buf.data() + LWS_PRE, headers.data(), headers.size());
std::memcpy(buf.data() + LWS_PRE + headers.size(), response_body.data(), response_body.size());
lws_write(wsi, buf.data() + LWS_PRE, headers.size() + response_body.size(), LWS_WRITE_HTTP);
if (lws_http_transaction_completed(wsi)) {
return -1;
}
return 0;
}
default:
break;
}
return 0;
}
static const struct lws_protocols protocols[] = {
{"http-api", callback_http, sizeof(HttpRequest), 0},
{nullptr, nullptr, 0, 0},
};
ApiServer::ApiServer(uint16_t port, Graph& graph, FlowManager& flow_manager, ProcessManager& process_manager)
: impl_(std::make_unique<Impl>()) {
impl_->data = std::make_unique<ApiServerImpl>();
impl_->data->port = port;
impl_->data->graph = &graph;
impl_->data->flow_manager = &flow_manager;
impl_->data->process_manager = &process_manager;
g_impl = impl_->data.get();
struct lws_context_creation_info info;
std::memset(&info, 0, sizeof(info));
info.port = port;
info.protocols = protocols;
info.gid = -1;
info.uid = -1;
impl_->data->context = lws_create_context(&info);
if (!impl_->data->context) {
spdlog::error("Failed to create HTTP context on port {}", port);
throw std::runtime_error("Failed to create HTTP context");
}
spdlog::info("API server: listening on port {}", port);
}
ApiServer::~ApiServer() {
if (impl_->data && impl_->data->context) {
lws_context_destroy(impl_->data->context);
}
if (g_impl == impl_->data.get()) {
g_impl = nullptr;
}
}
void ApiServer::poll(int timeout_ms) {
lws_service(impl_->data->context, timeout_ms);
}
} // namespace dmf_engine
+48
View File
@@ -0,0 +1,48 @@
#include <dmf-engine/flow_manager.hpp>
#include <nlohmann/json.hpp>
#include <spdlog/spdlog.h>
#include <random>
#include <sstream>
namespace dmf_engine {
FlowManager::FlowManager() {
std::random_device rd;
flow_counter_ = static_cast<int>(rd());
}
FlowId FlowManager::create_flow_id() {
std::stringstream ss;
ss << std::hex << (++flow_counter_);
auto id = ss.str();
while (id.length() < 8) {
id = "0" + id;
}
spdlog::info("FlowManager: created flow ID: {}", id);
return id;
}
nlohmann::json FlowManager::create_v210_flow_def(const FlowId& flow_id, int width, int height, int fps_numerator, int fps_denominator) const {
auto grain_size = (width * 2) * height;
nlohmann::json flow_def;
flow_def["id"] = flow_id;
flow_def["version"] = "v1.0";
flow_def["label"] = "DMF Studio Flow " + flow_id;
flow_def["description"] = "Auto-generated V210 flow";
flow_def["grain_rate"] = {{"numerator", fps_numerator}, {"denominator", fps_denominator}};
flow_def["format"] = "video/v210";
flow_def["width"] = width;
flow_def["height"] = height;
flow_def["grain_size"] = grain_size;
flow_def["components"] = nlohmann::json::array();
flow_def["tags"] = nlohmann::json::object();
flow_def["tags"]["grouphint"] = nlohmann::json::array({"DMF Studio:Video"});
return flow_def;
}
} // namespace dmf_engine
+135
View File
@@ -0,0 +1,135 @@
#include <dmf-engine/graph.hpp>
#include <nlohmann/json.hpp>
#include <spdlog/spdlog.h>
namespace dmf_engine {
NodeId Graph::add_node(const std::string& type, const nlohmann::json& config) {
auto id = type + "_" + std::to_string(next_node_num_++);
GraphNode node;
node.id = id;
node.type = type;
node.config = config;
node.state = NodeState::Stopped;
nodes_[id] = std::move(node);
spdlog::info("Graph: added node '{}' type='{}'", id, type);
return id;
}
bool Graph::remove_node(const NodeId& node_id) {
auto it = nodes_.find(node_id);
if (it == nodes_.end()) {
spdlog::warn("Graph: node '{}' not found", node_id);
return false;
}
std::vector<EdgeId> edges_to_remove;
for (const auto& [eid, edge] : edges_) {
if (edge.from_node == node_id || edge.to_node == node_id) {
edges_to_remove.push_back(eid);
}
}
for (const auto& eid : edges_to_remove) {
edges_.erase(eid);
spdlog::info("Graph: removed edge '{}' (connected to removed node '{}')", eid, node_id);
}
nodes_.erase(it);
spdlog::info("Graph: removed node '{}'", node_id);
return true;
}
const GraphNode* Graph::get_node(const NodeId& node_id) const {
auto it = nodes_.find(node_id);
return it != nodes_.end() ? &it->second : nullptr;
}
std::vector<GraphNode> Graph::get_nodes() const {
std::vector<GraphNode> result;
for (const auto& [_, node] : nodes_) {
result.push_back(node);
}
return result;
}
EdgeId Graph::add_edge(const NodeId& from_node, const PortId& from_port,
const NodeId& to_node, const PortId& to_port,
const FlowId& flow_id, const nlohmann::json& flow_def) {
auto id = from_node + ":" + from_port + "->" + to_node + ":" + to_port;
GraphEdge edge;
edge.id = id;
edge.from_node = from_node;
edge.from_port = from_port;
edge.to_node = to_node;
edge.to_port = to_port;
edge.flow_id = flow_id;
edge.flow_def = flow_def;
edges_[id] = std::move(edge);
spdlog::info("Graph: added edge '{}' flow_id={}", id, flow_id);
return id;
}
bool Graph::remove_edge(const EdgeId& edge_id) {
auto it = edges_.find(edge_id);
if (it == edges_.end()) {
spdlog::warn("Graph: edge '{}' not found", edge_id);
return false;
}
edges_.erase(it);
spdlog::info("Graph: removed edge '{}'", edge_id);
return true;
}
std::vector<GraphEdge> Graph::get_edges() const {
std::vector<GraphEdge> result;
for (const auto& [_, edge] : edges_) {
result.push_back(edge);
}
return result;
}
std::vector<GraphEdge> Graph::get_edges_for_node(const NodeId& node_id) const {
std::vector<GraphEdge> result;
for (const auto& [_, edge] : edges_) {
if (edge.from_node == node_id || edge.to_node == node_id) {
result.push_back(edge);
}
}
return result;
}
nlohmann::json Graph::serialize() const {
auto j = nlohmann::json::object();
auto nodes_arr = nlohmann::json::array();
for (const auto& [id, node] : nodes_) {
nodes_arr.push_back({
{"id", node.id},
{"type", node.type},
{"config", node.config},
{"state", static_cast<int>(node.state)},
{"control_port", node.control_port},
{"pid", node.pid},
});
}
j["nodes"] = nodes_arr;
auto edges_arr = nlohmann::json::array();
for (const auto& [id, edge] : edges_) {
edges_arr.push_back({
{"id", edge.id},
{"from_node", edge.from_node},
{"from_port", edge.from_port},
{"to_node", edge.to_node},
{"to_port", edge.to_port},
{"flow_id", edge.flow_id},
});
}
j["edges"] = edges_arr;
return j;
}
} // namespace dmf_engine
+97
View File
@@ -0,0 +1,97 @@
#include <dmf-engine/process_manager.hpp>
#include <spdlog/spdlog.h>
#include <cstdlib>
#include <filesystem>
#include <signal.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <unistd.h>
namespace dmf_engine {
std::string ProcessManager::find_node_binary(const std::string& node_type) const {
std::string binary_name = "dmf-node-" + node_type;
if (const char* env_path = std::getenv("DMF_NODE_PATH")) {
auto candidate = std::filesystem::path(env_path) / binary_name;
if (std::filesystem::exists(candidate)) {
return candidate.string();
}
}
if (const char* self_dir_env = std::getenv("DMF_STUDIO_BIN_DIR")) {
auto candidate = std::filesystem::path(self_dir_env) / binary_name;
if (std::filesystem::exists(candidate)) {
return candidate.string();
}
}
return binary_name;
}
bool ProcessManager::start_node(GraphNode& node, const std::string& mxl_domain, uint16_t base_port) {
if (node.state == NodeState::Running) {
spdlog::warn("ProcessManager: node '{}' is already running", node.id);
return true;
}
auto binary = find_node_binary(node.type);
node.control_port = base_port;
pid_t pid = fork();
if (pid < 0) {
spdlog::error("ProcessManager: fork() failed for node '{}': {}", node.id, strerror(errno));
return false;
}
if (pid == 0) {
std::string port_str = std::to_string(node.control_port);
std::string config_str = node.config.dump();
execlp(binary.c_str(), binary.c_str(),
"--node-id", node.id.c_str(),
"--control-port", port_str.c_str(),
"--mxl-domain", mxl_domain.c_str(),
"--config", config_str.c_str(),
nullptr);
spdlog::error("ProcessManager: execlp failed for '{}': {}", binary, strerror(errno));
_exit(1);
}
node.pid = pid;
node.state = NodeState::Running;
spdlog::info("ProcessManager: started node '{}' pid={} port={}", node.id, pid, node.control_port);
return true;
}
bool ProcessManager::stop_node(GraphNode& node) {
if (node.state != NodeState::Running || node.pid <= 0) {
spdlog::warn("ProcessManager: node '{}' is not running", node.id);
return true;
}
if (kill(node.pid, SIGTERM) != 0) {
spdlog::error("ProcessManager: failed to send SIGTERM to node '{}': {}", node.id, strerror(errno));
return false;
}
int status = 0;
waitpid(node.pid, &status, 0);
spdlog::info("ProcessManager: stopped node '{}' pid={}", node.id, node.pid);
node.pid = 0;
node.state = NodeState::Stopped;
return true;
}
void ProcessManager::stop_all() {
}
bool ProcessManager::is_running(const NodeId& node_id) const {
return false;
}
} // namespace dmf_engine