Add the renderer process

The renderer becomes two programs: the adapter (t5deRenderEngine.dll, 32-bit, in IMVU's process) and a 64-bit renderer process (t5de-renderer.exe) drawing with Diligent Engine, fetched with FetchContent and pinned. For now a spike: the adapter starts the renderer process for each engine and signals each frame through a shared memory channel, and the renderer process draws a box into a child window in the client's 3D view.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

adapter/renderer_process.cpp

@@ -0,0 +1,92 @@
+#include "renderer_process.hpp"
+
+#include <atomic>
+#include <filesystem>
+#include <format>
+#include <string>
+
+#include "log.hpp"
+
+namespace t5de {
+
+namespace {
+
+// The folder this DLL is in: the client's program folder.
+std::filesystem::path module_folder()
+{
+    HMODULE self = nullptr;
+    GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
+                       reinterpret_cast<LPCWSTR>(&module_folder), &self);
+    wchar_t path[MAX_PATH];
+    GetModuleFileNameW(self, path, MAX_PATH);
+    return std::filesystem::path(path).parent_path();
+}
+
+} // namespace
+
+std::unique_ptr<RendererProcess> RendererProcess::start(HWND window)
+{
+    static std::atomic<int> engines = 0;
+    const std::wstring name = std::format(L"Local\\t5de-renderer-{}-{}", GetCurrentProcessId(), engines++);
+
+    std::unique_ptr<RendererProcess> renderer(new RendererProcess());
+    renderer->mapping_ = CreateFileMappingW(INVALID_HANDLE_VALUE, nullptr, PAGE_READWRITE, 0,
+                                            sizeof(protocol::Channel), name.c_str());
+    if (renderer->mapping_ == nullptr) {
+        log::info("renderer process: can't create the channel: error {}", GetLastError());
+        return nullptr;
+    }
+    renderer->channel_ = static_cast<protocol::Channel*>(
+        MapViewOfFile(renderer->mapping_, FILE_MAP_ALL_ACCESS, 0, 0, sizeof(protocol::Channel)));
+    renderer->frameEvent_ = CreateEventW(nullptr, FALSE, FALSE, (name + protocol::frameEventSuffix).c_str());
+    if (renderer->channel_ == nullptr || renderer->frameEvent_ == nullptr) {
+        log::info("renderer process: can't set up the channel: error {}", GetLastError());
+        return nullptr;
+    }
+    *renderer->channel_ = {.version = protocol::version,
+                           .closing = 0,
+                           .window = reinterpret_cast<std::uintptr_t>(window),
+                           .frames = 0};
+
+    // The renderer's window goes on top of the client's 3D view: keep the client's drawing out of it.
+    SetWindowLongW(window, GWL_STYLE, GetWindowLongW(window, GWL_STYLE) | WS_CLIPCHILDREN);
+
+    const auto exe = module_folder() / L"t5de-renderer.exe";
+    std::wstring command = std::format(L"\"{}\" --channel {} --parent {}", exe.wstring(), name, GetCurrentProcessId());
+    STARTUPINFOW startup{.cb = sizeof(STARTUPINFOW)};
+    PROCESS_INFORMATION process{};
+    if (!CreateProcessW(exe.c_str(), command.data(), nullptr, nullptr, FALSE, 0, nullptr, nullptr, &startup,
+                        &process)) {
+        log::info("renderer process: can't start {}: error {}", exe.string(), GetLastError());
+        return nullptr;
+    }
+    CloseHandle(process.hThread);
+    renderer->process_ = process.hProcess;
+    log::info("renderer process: started {} (pid {}) on channel {}", exe.string(), process.dwProcessId,
+              std::filesystem::path(name).string());
+    return renderer;
+}
+
+RendererProcess::~RendererProcess()
+{
+    if (channel_ != nullptr) {
+        channel_->closing = 1;
+        if (frameEvent_ != nullptr) {
+            SetEvent(frameEvent_);
+        }
+        UnmapViewOfFile(channel_);
+    }
+    for (HANDLE handle : {frameEvent_, mapping_, process_}) {
+        if (handle != nullptr) {
+            CloseHandle(handle);
+        }
+    }
+}
+
+void RendererProcess::frameDrawn()
+{
+    ++channel_->frames;
+    SetEvent(frameEvent_);
+}
+
+} // namespace t5de