Draw all of a client's views in one renderer process

The adapter starts one renderer process per client, with the first engine
(again if it has exited), and attaches each engine's channel to it through a
control section (protocol version 5: Control, the -attach event). The renderer
process keeps a window, swap chain and scene per channel on one Direct3D
device, takes the views' frames in turns, and closes a view when its engine
does.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

adapter/renderer_process.cpp

@@ -1,10 +1,10 @@
 #include "renderer_process.hpp"
 
 #include <algorithm>
-#include <atomic>
 #include <cstring>
 #include <filesystem>
 #include <format>
+#include <mutex>
 #include <string>
 
 #include "log.hpp"
@@ -54,12 +54,92 @@ std::filesystem::path running_copy(const std::filesystem::path& exe)
     return copy;
 }
 
+// The client's one renderer process, and the control section (protocol::Control) engines attach their channels
+// through. Started with the first engine; started again by the next one if it has exited.
+class Host {
+public:
+    static Host& get()
+    {
+        static Host host;
+        return host;
+    }
+
+    std::mutex& mutex() { return mutex_; }
+
+    // The next channel's name.
+    std::wstring nextChannel() const { return std::format(L"{}-{}", name_, channels_); }
+
+    // Tells the renderer process about the channel just created (nextChannel()), starting it if it isn't running.
+    // Returns a handle to the renderer process, the caller's to close; null if it can't.
+    HANDLE attach()
+    {
+        if (control_ == nullptr || (!running() && !startProcess())) {
+            return nullptr;
+        }
+        control_->channels = ++channels_;
+        SetEvent(attach_);
+        HANDLE process = nullptr;
+        DuplicateHandle(GetCurrentProcess(), process_, GetCurrentProcess(), &process, 0, FALSE, DUPLICATE_SAME_ACCESS);
+        return process;
+    }
+
+private:
+    Host() : name_(std::format(L"Local\\t5de-renderer-{}", GetCurrentProcessId()))
+    {
+        mapping_ = CreateFileMappingW(INVALID_HANDLE_VALUE, nullptr, PAGE_READWRITE, 0, sizeof(protocol::Control),
+                                      name_.c_str());
+        attach_ = CreateEventW(nullptr, FALSE, FALSE, (name_ + protocol::attachEventSuffix).c_str());
+        if (mapping_ != nullptr && attach_ != nullptr) {
+            control_ = static_cast<protocol::Control*>(
+                MapViewOfFile(mapping_, FILE_MAP_ALL_ACCESS, 0, 0, sizeof(protocol::Control)));
+        }
+        if (control_ == nullptr) {
+            log::info("renderer process: can't create the control section: error {}", GetLastError());
+            return;
+        }
+        *control_ = {.version = protocol::version, .channels = 0};
+    }
+
+    [[nodiscard]] bool running() const { return process_ != nullptr && WaitForSingleObject(process_, 0) == WAIT_TIMEOUT; }
+
+    bool startProcess()
+    {
+        if (process_ != nullptr) {
+            CloseHandle(process_);
+            process_ = nullptr;
+        }
+        const auto exe = running_copy(module_folder() / L"t5de-renderer.exe");
+        std::wstring command =
+            std::format(L"\"{}\" --control {} --parent {}", exe.wstring(), name_, GetCurrentProcessId());
+        STARTUPINFOW startup{.cb = sizeof(STARTUPINFOW)};
+        PROCESS_INFORMATION process{};
+        if (!CreateProcessW(exe.c_str(), command.data(), nullptr, nullptr, FALSE, 0, nullptr, nullptr, &startup,
+                            &process)) {
+            log::info("renderer process: can't start {}: error {}", exe.string(), GetLastError());
+            return false;
+        }
+        CloseHandle(process.hThread);
+        process_ = process.hProcess;
+        log::info("renderer process: started {} (pid {})", exe.string(), process.dwProcessId);
+        return true;
+    }
+
+    std::mutex mutex_;
+    std::wstring name_;
+    HANDLE mapping_ = nullptr;
+    HANDLE attach_ = nullptr;
+    protocol::Control* control_ = nullptr;
+    HANDLE process_ = nullptr;
+    std::uint32_t channels_ = 0;
+};
+
 } // namespace
 
 std::unique_ptr<RendererProcess> RendererProcess::start(HWND window, protocol::View view)
 {
-    static std::atomic<int> engines = 0;
-    const std::wstring name = std::format(L"Local\\t5de-renderer-{}-{}", GetCurrentProcessId(), engines++);
+    Host& host = Host::get();
+    const std::scoped_lock lock(host.mutex());
+    const std::wstring name = host.nextChannel();
     constexpr DWORD size = sizeof(protocol::Channel) + protocol::channelCapacity;
 
     std::unique_ptr<RendererProcess> renderer(new RendererProcess());
@@ -95,18 +175,11 @@ std::unique_ptr<RendererProcess> RendererProcess::start(HWND window, protocol::V
     // WM_PAINT, which a window entirely covered by clipped children never gets (it then drew about once a second). The
     // renderer's swap chain is a flip model one, which the compositor shows above the client's drawing anyway.
     SetWindowLongW(window, GWL_STYLE, GetWindowLongW(window, GWL_STYLE) & ~WS_CLIPCHILDREN);
-    const auto exe = running_copy(module_folder() / L"t5de-renderer.exe");
-    std::wstring command = std::format(L"\"{}\" --channel {} --parent {}", exe.wstring(), name, GetCurrentProcessId());
-    STARTUPINFOW startup{.cb = sizeof(STARTUPINFOW)};
-    PROCESS_INFORMATION process{};
-    if (!CreateProcessW(exe.c_str(), command.data(), nullptr, nullptr, FALSE, 0, nullptr, nullptr, &startup,
-                        &process)) {
-        log::info("renderer process: can't start {}: error {}", exe.string(), GetLastError());
+    renderer->process_ = host.attach();
+    if (renderer->process_ == nullptr) {
         return nullptr;
     }
-    CloseHandle(process.hThread);
-    renderer->process_ = process.hProcess;
-    log::info("renderer process: started {} (pid {}) on channel {}", exe.string(), process.dwProcessId, narrow(name));
+    log::info("renderer process: drawing channel {}", narrow(name));
     return renderer;
 }
 

adapter/renderer_process.hpp

@@ -1,6 +1,7 @@
 #pragma once
-// The renderer process (t5de-renderer.exe, 64-bit) an engine draws through: started with the engine, sent the
-// command stream through a shared memory channel (include/t5de/protocol/channel.hpp), and told to exit with it.
+// The renderer process (t5de-renderer.exe, 64-bit) an engine draws through: one for the whole client, started with
+// the first engine; each engine sends it the command stream through a shared memory channel of its own
+// (include/t5de/protocol/channel.hpp), and closes the channel when it's destroyed.
 
 #include <windows.h>
 
@@ -14,7 +15,8 @@ namespace t5de {
 
 class RendererProcess {
 public:
-    // Starts t5de-renderer.exe from this DLL's folder, drawing into part of window; null if it can't.
+    // A channel to the renderer process (t5de-renderer.exe from this DLL's folder, started if it isn't running),
+    // drawing into part of window; null if it can't.
     static std::unique_ptr<RendererProcess> start(HWND window, protocol::View view);
 
     RendererProcess(const RendererProcess&) = delete;