t5de-renderer/adapter/renderer_process.cpp

#include "renderer_process.hpp"

#include <algorithm>
#include <atomic>
#include <cstring>
#include <filesystem>
#include <format>
#include <string>

#include "log.hpp"

namespace t5de {

namespace {

// How long to wait for the renderer process to take a chunk before giving up on it.
constexpr DWORD consumeTimeout = 5000;

// The folder this DLL is in: the client's program folder.
std::filesystem::path module_folder()
{
    HMODULE self = nullptr;
    GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
                       reinterpret_cast<LPCWSTR>(&module_folder), &self);
    wchar_t path[MAX_PATH];
    GetModuleFileNameW(self, path, MAX_PATH);
    return std::filesystem::path(path).parent_path();
}

std::string narrow(const std::wstring& text) { return std::filesystem::path(text).string(); }

} // namespace

std::unique_ptr<RendererProcess> RendererProcess::start(HWND window, protocol::View view)
{
    static std::atomic<int> engines = 0;
    const std::wstring name = std::format(L"Local\\t5de-renderer-{}-{}", GetCurrentProcessId(), engines++);
    constexpr DWORD size = sizeof(protocol::Channel) + protocol::channelCapacity;

    std::unique_ptr<RendererProcess> renderer(new RendererProcess());
    renderer->mapping_ = CreateFileMappingW(INVALID_HANDLE_VALUE, nullptr, PAGE_READWRITE, 0, size, name.c_str());
    if (renderer->mapping_ == nullptr) {
        log::info("renderer process: can't create the channel: error {}", GetLastError());
        return nullptr;
    }
    renderer->channel_ =
        static_cast<protocol::Channel*>(MapViewOfFile(renderer->mapping_, FILE_MAP_ALL_ACCESS, 0, 0, size));
    renderer->ready_ = CreateEventW(nullptr, FALSE, FALSE, (name + protocol::readyEventSuffix).c_str());
    renderer->consumed_ = CreateEventW(nullptr, FALSE, TRUE, (name + protocol::consumedEventSuffix).c_str());
    if (renderer->channel_ == nullptr || renderer->ready_ == nullptr || renderer->consumed_ == nullptr) {
        log::info("renderer process: can't set up the channel: error {}", GetLastError());
        return nullptr;
    }
    renderer->data_ = reinterpret_cast<std::byte*>(renderer->channel_ + 1);
    *renderer->channel_ = {.version = protocol::version,
                           .closing = 0,
                           .window = reinterpret_cast<std::uintptr_t>(window),
                           .frames = 0,
                           .view = view,
                           .capacity = protocol::channelCapacity,
                           .chunkSize = 0,
                           .chunkFlags = 0};

    // The renderer's window goes on top of the client's 3D view: keep the client's drawing out of it.
    SetWindowLongW(window, GWL_STYLE, GetWindowLongW(window, GWL_STYLE) | WS_CLIPCHILDREN);

    const auto exe = module_folder() / L"t5de-renderer.exe";
    std::wstring command = std::format(L"\"{}\" --channel {} --parent {}", exe.wstring(), name, GetCurrentProcessId());
    STARTUPINFOW startup{.cb = sizeof(STARTUPINFOW)};
    PROCESS_INFORMATION process{};
    if (!CreateProcessW(exe.c_str(), command.data(), nullptr, nullptr, FALSE, 0, nullptr, nullptr, &startup,
                        &process)) {
        log::info("renderer process: can't start {}: error {}", exe.string(), GetLastError());
        return nullptr;
    }
    CloseHandle(process.hThread);
    renderer->process_ = process.hProcess;
    log::info("renderer process: started {} (pid {}) on channel {}", exe.string(), process.dwProcessId, narrow(name));
    return renderer;
}

RendererProcess::~RendererProcess()
{
    if (channel_ != nullptr) {
        channel_->closing = 1;
        if (ready_ != nullptr) {
            SetEvent(ready_);
        }
        UnmapViewOfFile(channel_);
    }
    for (HANDLE handle : {ready_, consumed_, mapping_, process_}) {
        if (handle != nullptr) {
            CloseHandle(handle);
        }
    }
}

bool RendererProcess::send(std::span<const std::byte> commands, bool endsFrame)
{
    if (!alive_) {
        return false;
    }
    do {
        const HANDLE waits[] = {consumed_, process_};
        const DWORD result = WaitForMultipleObjects(2, waits, FALSE, consumeTimeout);
        if (result != WAIT_OBJECT_0) {
            log::info("renderer process: {}; giving up on it", result == WAIT_OBJECT_0 + 1 ? "exited" : "not responding");
            alive_ = false;
            return false;
        }
        const std::size_t size = std::min<std::size_t>(commands.size(), channel_->capacity);
        std::memcpy(data_, commands.data(), size);
        commands = commands.subspan(size);
        channel_->chunkSize = static_cast<std::uint32_t>(size);
        channel_->chunkFlags = endsFrame && commands.empty() ? protocol::chunkEndsFrame : 0;
        if (endsFrame && commands.empty()) {
            ++channel_->frames;
        }
        SetEvent(ready_);
    } while (!commands.empty());
    return true;
}

} // namespace t5de