t5de-renderer/adapter/renderer_process.cpp

#include "renderer_process.hpp"

#include <algorithm>
#include <cstring>
#include <filesystem>
#include <format>
#include <mutex>
#include <string>

#include "log.hpp"

namespace t5de {

namespace {

// How long to wait for the renderer process to take a chunk before giving up on it.
constexpr DWORD consumeTimeout = 5000;

// The folder this DLL is in: the client's program folder.
std::filesystem::path module_folder()
{
    HMODULE self = nullptr;
    GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
                       reinterpret_cast<LPCWSTR>(&module_folder), &self);
    wchar_t path[MAX_PATH];
    GetModuleFileNameW(self, path, MAX_PATH);
    return std::filesystem::path(path).parent_path();
}

std::string narrow(const std::wstring& text) { return std::filesystem::path(text).string(); }

// The renderer process runs from a copy in %TEMP%\t5de-renderer, one per version of the program (by size and
// modification time), so the program in the client's folder can be replaced (updated) while the client runs: the
// next engine runs the new one. Copies no longer running are removed. Falls back to the program itself.
std::filesystem::path running_copy(const std::filesystem::path& exe)
{
    std::error_code error;
    const auto size = std::filesystem::file_size(exe, error);
    const auto modified = std::filesystem::last_write_time(exe, error);
    if (error) {
        return exe;
    }
    const auto folder = std::filesystem::temp_directory_path(error) / L"t5de-renderer";
    std::filesystem::create_directories(folder, error);
    const auto copy = folder / std::format(L"t5de-renderer-{}-{}.exe", size, modified.time_since_epoch().count());
    if (!std::filesystem::exists(copy, error)) {
        for (const auto& old : std::filesystem::directory_iterator(folder, error)) {
            std::filesystem::remove(old.path(), error); // fails, harmlessly, for copies still running
        }
        if (!CopyFileW(exe.c_str(), copy.c_str(), FALSE)) {
            return exe;
        }
    }
    return copy;
}

// The client's one renderer process, and the control section (protocol::Control) engines attach their channels
// through. Started with the first engine; started again by the next one if it has exited.
class Host {
public:
    static Host& get()
    {
        static Host host;
        return host;
    }

    std::mutex& mutex() { return mutex_; }

    // The next channel's name.
    std::wstring nextChannel() const { return std::format(L"{}-{}", name_, channels_); }

    // Tells the renderer process about the channel just created (nextChannel()), starting it if it isn't running.
    // Returns a handle to the renderer process, the caller's to close; null if it can't.
    HANDLE attach()
    {
        if (control_ == nullptr || (!running() && !startProcess())) {
            return nullptr;
        }
        control_->channels = ++channels_;
        SetEvent(attach_);
        HANDLE process = nullptr;
        DuplicateHandle(GetCurrentProcess(), process_, GetCurrentProcess(), &process, 0, FALSE, DUPLICATE_SAME_ACCESS);
        return process;
    }

private:
    Host() : name_(std::format(L"Local\\t5de-renderer-{}", GetCurrentProcessId()))
    {
        mapping_ = CreateFileMappingW(INVALID_HANDLE_VALUE, nullptr, PAGE_READWRITE, 0, sizeof(protocol::Control),
                                      name_.c_str());
        attach_ = CreateEventW(nullptr, FALSE, FALSE, (name_ + protocol::attachEventSuffix).c_str());
        if (mapping_ != nullptr && attach_ != nullptr) {
            control_ = static_cast<protocol::Control*>(
                MapViewOfFile(mapping_, FILE_MAP_ALL_ACCESS, 0, 0, sizeof(protocol::Control)));
        }
        if (control_ == nullptr) {
            log::info("renderer process: can't create the control section: error {}", GetLastError());
            return;
        }
        *control_ = {.version = protocol::version, .channels = 0};
    }

    [[nodiscard]] bool running() const { return process_ != nullptr && WaitForSingleObject(process_, 0) == WAIT_TIMEOUT; }

    bool startProcess()
    {
        if (process_ != nullptr) {
            CloseHandle(process_);
            process_ = nullptr;
        }
        const auto exe = running_copy(module_folder() / L"t5de-renderer.exe");
        std::wstring command =
            std::format(L"\"{}\" --control {} --parent {}", exe.wstring(), name_, GetCurrentProcessId());
        STARTUPINFOW startup{.cb = sizeof(STARTUPINFOW)};
        PROCESS_INFORMATION process{};
        if (!CreateProcessW(exe.c_str(), command.data(), nullptr, nullptr, FALSE, 0, nullptr, nullptr, &startup,
                            &process)) {
            log::info("renderer process: can't start {}: error {}", exe.string(), GetLastError());
            return false;
        }
        CloseHandle(process.hThread);
        process_ = process.hProcess;
        log::info("renderer process: started {} (pid {})", exe.string(), process.dwProcessId);
        return true;
    }

    std::mutex mutex_;
    std::wstring name_;
    HANDLE mapping_ = nullptr;
    HANDLE attach_ = nullptr;
    protocol::Control* control_ = nullptr;
    HANDLE process_ = nullptr;
    std::uint32_t channels_ = 0;
};

} // namespace

std::unique_ptr<RendererProcess> RendererProcess::start(HWND window, protocol::View view)
{
    Host& host = Host::get();
    const std::scoped_lock lock(host.mutex());
    const std::wstring name = host.nextChannel();
    constexpr DWORD size = sizeof(protocol::Channel) + protocol::channelCapacity;

    std::unique_ptr<RendererProcess> renderer(new RendererProcess());
    renderer->mapping_ = CreateFileMappingW(INVALID_HANDLE_VALUE, nullptr, PAGE_READWRITE, 0, size, name.c_str());
    if (renderer->mapping_ == nullptr) {
        log::info("renderer process: can't create the channel: error {}", GetLastError());
        return nullptr;
    }
    renderer->channel_ =
        static_cast<protocol::Channel*>(MapViewOfFile(renderer->mapping_, FILE_MAP_ALL_ACCESS, 0, 0, size));
    renderer->ready_ = CreateEventW(nullptr, FALSE, FALSE, (name + protocol::readyEventSuffix).c_str());
    renderer->consumed_ = CreateEventW(nullptr, FALSE, TRUE, (name + protocol::consumedEventSuffix).c_str());
    renderer->read_ = CreateEventW(nullptr, FALSE, FALSE, (name + protocol::readEventSuffix).c_str());
    if (renderer->channel_ == nullptr || renderer->ready_ == nullptr || renderer->consumed_ == nullptr
        || renderer->read_ == nullptr) {
        log::info("renderer process: can't set up the channel: error {}", GetLastError());
        return nullptr;
    }
    renderer->data_ = reinterpret_cast<std::byte*>(renderer->channel_ + 1);
    *renderer->channel_ = {.version = protocol::version,
                           .closing = 0,
                           .window = reinterpret_cast<std::uintptr_t>(window),
                           .frames = 0,
                           .view = view,
                           .capacity = protocol::channelCapacity,
                           .chunkSize = 0,
                           .chunkFlags = 0,
                           .covering = 0,
                           .readSize = 0};

    // The renderer's window goes on top of the client's 3D view, as a child window. The client's window mustn't clip
    // its children (WS_CLIPCHILDREN, which the client creates it with): the client draws a frame when its window gets
    // WM_PAINT, which a window entirely covered by clipped children never gets (it then drew about once a second). The
    // renderer's swap chain is a flip model one, which the compositor shows above the client's drawing anyway.
    SetWindowLongW(window, GWL_STYLE, GetWindowLongW(window, GWL_STYLE) & ~WS_CLIPCHILDREN);
    renderer->process_ = host.attach();
    if (renderer->process_ == nullptr) {
        return nullptr;
    }
    log::info("renderer process: drawing channel {}", narrow(name));
    return renderer;
}

RendererProcess::~RendererProcess()
{
    if (channel_ != nullptr) {
        channel_->closing = 1;
        if (ready_ != nullptr) {
            SetEvent(ready_);
        }
        UnmapViewOfFile(channel_);
    }
    for (HANDLE handle : {ready_, consumed_, read_, mapping_, process_}) {
        if (handle != nullptr) {
            CloseHandle(handle);
        }
    }
}

bool RendererProcess::send(std::span<const std::byte> commands, bool endsFrame)
{
    if (!alive_) {
        return false;
    }
    do {
        const HANDLE waits[] = {consumed_, process_};
        const DWORD result = WaitForMultipleObjects(2, waits, FALSE, consumeTimeout);
        if (result != WAIT_OBJECT_0) {
            log::info("renderer process: {}; giving up on it", result == WAIT_OBJECT_0 + 1 ? "exited" : "not responding");
            alive_ = false;
            return false;
        }
        const std::size_t size = std::min<std::size_t>(commands.size(), channel_->capacity);
        std::memcpy(data_, commands.data(), size);
        commands = commands.subspan(size);
        channel_->chunkSize = static_cast<std::uint32_t>(size);
        channel_->chunkFlags = endsFrame && commands.empty() ? protocol::chunkEndsFrame : 0;
        if (endsFrame && commands.empty()) {
            ++channel_->frames;
        }
        SetEvent(ready_);
    } while (!commands.empty());
    return true;
}

std::span<const std::byte> RendererProcess::readBack()
{
    if (!alive_) {
        return {};
    }
    const HANDLE waits[] = {read_, process_};
    const DWORD result = WaitForMultipleObjects(2, waits, FALSE, consumeTimeout);
    if (result != WAIT_OBJECT_0) {
        log::info("renderer process: {} reading back a frame; giving up on it",
                  result == WAIT_OBJECT_0 + 1 ? "exited" : "not responding");
        alive_ = false;
        return {};
    }
    return {data_, std::min<std::size_t>(channel_->readSize, channel_->capacity)};
}

} // namespace t5de