t5de-renderer/adapter/renderer_process.cpp

#include "renderer_process.hpp"

#include <algorithm>
#include <atomic>
#include <cstring>
#include <filesystem>
#include <format>
#include <string>

#include "log.hpp"

namespace t5de {

namespace {

// How long to wait for the renderer process to take a chunk before giving up on it.
constexpr DWORD consumeTimeout = 5000;

// The folder this DLL is in: the client's program folder.
std::filesystem::path module_folder()
{
    HMODULE self = nullptr;
    GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
                       reinterpret_cast<LPCWSTR>(&module_folder), &self);
    wchar_t path[MAX_PATH];
    GetModuleFileNameW(self, path, MAX_PATH);
    return std::filesystem::path(path).parent_path();
}

std::string narrow(const std::wstring& text) { return std::filesystem::path(text).string(); }

// The renderer process runs from a copy in %TEMP%\t5de-renderer, one per version of the program (by size and
// modification time), so the program in the client's folder can be replaced (updated) while the client runs: the
// next engine runs the new one. Copies no longer running are removed. Falls back to the program itself.
std::filesystem::path running_copy(const std::filesystem::path& exe)
{
    std::error_code error;
    const auto size = std::filesystem::file_size(exe, error);
    const auto modified = std::filesystem::last_write_time(exe, error);
    if (error) {
        return exe;
    }
    const auto folder = std::filesystem::temp_directory_path(error) / L"t5de-renderer";
    std::filesystem::create_directories(folder, error);
    const auto copy = folder / std::format(L"t5de-renderer-{}-{}.exe", size, modified.time_since_epoch().count());
    if (!std::filesystem::exists(copy, error)) {
        for (const auto& old : std::filesystem::directory_iterator(folder, error)) {
            std::filesystem::remove(old.path(), error); // fails, harmlessly, for copies still running
        }
        if (!CopyFileW(exe.c_str(), copy.c_str(), FALSE)) {
            return exe;
        }
    }
    return copy;
}

} // namespace

std::unique_ptr<RendererProcess> RendererProcess::start(HWND window, protocol::View view)
{
    static std::atomic<int> engines = 0;
    const std::wstring name = std::format(L"Local\\t5de-renderer-{}-{}", GetCurrentProcessId(), engines++);
    constexpr DWORD size = sizeof(protocol::Channel) + protocol::channelCapacity;

    std::unique_ptr<RendererProcess> renderer(new RendererProcess());
    renderer->mapping_ = CreateFileMappingW(INVALID_HANDLE_VALUE, nullptr, PAGE_READWRITE, 0, size, name.c_str());
    if (renderer->mapping_ == nullptr) {
        log::info("renderer process: can't create the channel: error {}", GetLastError());
        return nullptr;
    }
    renderer->channel_ =
        static_cast<protocol::Channel*>(MapViewOfFile(renderer->mapping_, FILE_MAP_ALL_ACCESS, 0, 0, size));
    renderer->ready_ = CreateEventW(nullptr, FALSE, FALSE, (name + protocol::readyEventSuffix).c_str());
    renderer->consumed_ = CreateEventW(nullptr, FALSE, TRUE, (name + protocol::consumedEventSuffix).c_str());
    renderer->read_ = CreateEventW(nullptr, FALSE, FALSE, (name + protocol::readEventSuffix).c_str());
    if (renderer->channel_ == nullptr || renderer->ready_ == nullptr || renderer->consumed_ == nullptr
        || renderer->read_ == nullptr) {
        log::info("renderer process: can't set up the channel: error {}", GetLastError());
        return nullptr;
    }
    renderer->data_ = reinterpret_cast<std::byte*>(renderer->channel_ + 1);
    *renderer->channel_ = {.version = protocol::version,
                           .closing = 0,
                           .window = reinterpret_cast<std::uintptr_t>(window),
                           .frames = 0,
                           .view = view,
                           .capacity = protocol::channelCapacity,
                           .chunkSize = 0,
                           .chunkFlags = 0,
                           .covering = 0,
                           .readSize = 0};

    // The renderer's window goes on top of the client's 3D view, as a child window. The client's window mustn't clip
    // its children (WS_CLIPCHILDREN, which the client creates it with): the client draws a frame when its window gets
    // WM_PAINT, which a window entirely covered by clipped children never gets (it then drew about once a second). The
    // renderer's swap chain is a flip model one, which the compositor shows above the client's drawing anyway.
    SetWindowLongW(window, GWL_STYLE, GetWindowLongW(window, GWL_STYLE) & ~WS_CLIPCHILDREN);
    const auto exe = running_copy(module_folder() / L"t5de-renderer.exe");
    std::wstring command = std::format(L"\"{}\" --channel {} --parent {}", exe.wstring(), name, GetCurrentProcessId());
    STARTUPINFOW startup{.cb = sizeof(STARTUPINFOW)};
    PROCESS_INFORMATION process{};
    if (!CreateProcessW(exe.c_str(), command.data(), nullptr, nullptr, FALSE, 0, nullptr, nullptr, &startup,
                        &process)) {
        log::info("renderer process: can't start {}: error {}", exe.string(), GetLastError());
        return nullptr;
    }
    CloseHandle(process.hThread);
    renderer->process_ = process.hProcess;
    log::info("renderer process: started {} (pid {}) on channel {}", exe.string(), process.dwProcessId, narrow(name));
    return renderer;
}

RendererProcess::~RendererProcess()
{
    if (channel_ != nullptr) {
        channel_->closing = 1;
        if (ready_ != nullptr) {
            SetEvent(ready_);
        }
        UnmapViewOfFile(channel_);
    }
    for (HANDLE handle : {ready_, consumed_, read_, mapping_, process_}) {
        if (handle != nullptr) {
            CloseHandle(handle);
        }
    }
}

bool RendererProcess::send(std::span<const std::byte> commands, bool endsFrame)
{
    if (!alive_) {
        return false;
    }
    do {
        const HANDLE waits[] = {consumed_, process_};
        const DWORD result = WaitForMultipleObjects(2, waits, FALSE, consumeTimeout);
        if (result != WAIT_OBJECT_0) {
            log::info("renderer process: {}; giving up on it", result == WAIT_OBJECT_0 + 1 ? "exited" : "not responding");
            alive_ = false;
            return false;
        }
        const std::size_t size = std::min<std::size_t>(commands.size(), channel_->capacity);
        std::memcpy(data_, commands.data(), size);
        commands = commands.subspan(size);
        channel_->chunkSize = static_cast<std::uint32_t>(size);
        channel_->chunkFlags = endsFrame && commands.empty() ? protocol::chunkEndsFrame : 0;
        if (endsFrame && commands.empty()) {
            ++channel_->frames;
        }
        SetEvent(ready_);
    } while (!commands.empty());
    return true;
}

std::span<const std::byte> RendererProcess::readBack()
{
    if (!alive_) {
        return {};
    }
    const HANDLE waits[] = {read_, process_};
    const DWORD result = WaitForMultipleObjects(2, waits, FALSE, consumeTimeout);
    if (result != WAIT_OBJECT_0) {
        log::info("renderer process: {} reading back a frame; giving up on it",
                  result == WAIT_OBJECT_0 + 1 ? "exited" : "not responding");
        alive_ = false;
        return {};
    }
    return {data_, std::min<std::size_t>(channel_->readSize, channel_->capacity)};
}

} // namespace t5de