From 2d9da249fa45caaa877d02b6c5dc38781efb82e4 Mon Sep 17 00:00:00 2001 From: dhkatz Date: Wed, 28 Oct 2026 18:14:17 -0700 Subject: [PATCH] Draw the whole 3D view in place of the client's renderer The full view is now the default. While the renderer process covers the whole view (Channel::covering, set after each frame it presents), the adapter skips the client's renderer's clears and draws, decided per frame; the client's renderer still gets state and resources, and draws again as soon as the renderer process stops covering the view. The client's window no longer gets WS_CLIPCHILDREN: the client draws when its window gets WM_PAINT, which a window entirely covered by clipped children never gets (it fell to about one frame a second). The flip model swap chain shows above the client's drawing anyway. Co-Authored-By: Claude Opus 5.5 --- adapter/engine.cpp | 27 ++++++++++++++++++++++----- adapter/engine.hpp | 12 +++++++++--- adapter/plugin.cpp | 6 +++--- adapter/recorder.hpp | 3 +++ adapter/renderer_process.cpp | 11 +++++++---- adapter/renderer_process.hpp | 3 +++ docs/architecture.md | 24 +++++++++++++++--------- include/t5de/protocol/channel.hpp | 11 ++++++++--- renderer/main.cpp | 4 ++++ 9 files changed, 74 insertions(+), 27 deletions(-) diff --git a/adapter/engine.cpp b/adapter/engine.cpp index 5f8c524..eaa4568 100644 --- a/adapter/engine.cpp +++ b/adapter/engine.cpp @@ -47,7 +47,9 @@ void Engine::enable2dFrameBuffer() { inner_->enable2dFrameBuffer(); } void Engine::clear(std::uint32_t flags, imvu::Color color) { - inner_->clear(flags, color); + if (innerDraws()) { + inner_->clear(flags, color); + } recorder_->add(Op::Clear, protocol::Clear{flags, color}); } @@ -242,6 +244,13 @@ imvu::SharedPtr Engine::newTextureRenderTarget(imvu void Engine::beginRendering(imvu::PiRenderTarget* target) { inner_->beginRendering(target); + // Decided per frame: frames into render targets, and frames while the renderer process doesn't cover the view, are + // drawn by the client's renderer. + const bool takenOver = target == nullptr && recorder_->covering(); + if (takenOver != takenOver_ && target == nullptr) { + log::info("engine: frame {}: {}", frames_, takenOver ? "drawn by the renderer process" : "drawn by the client's renderer"); + } + takenOver_ = takenOver; if (target != nullptr && foreignLogged_ < 20) { ++foreignLogged_; log::info("engine: frame {}: drawing into a render target ({})", frames_, static_cast(target)); @@ -271,7 +280,9 @@ void Engine::drawTriangles(imvu::PiInternalIndexBuffer* indices, imvu::PiInterna imvu::PiInternalVertexBuffer* normals, imvu::PiInternalVertexBuffer* texCoords, imvu::PiInternalVertexBuffer* colors) { - inner_->drawTriangles(inner(indices), inner(positions), inner(normals), inner(texCoords), inner(colors)); + if (innerDraws()) { + inner_->drawTriangles(inner(indices), inner(positions), inner(normals), inner(texCoords), inner(colors)); + } ++(id(indices) != 0 && id(positions) != 0 ? stats_.triangles : stats_.foreignTriangles); recorder_->add(Op::DrawTriangles, protocol::DrawTriangles{id(indices), id(positions), id(normals), id(texCoords), id(colors)}); @@ -279,13 +290,17 @@ void Engine::drawTriangles(imvu::PiInternalIndexBuffer* indices, imvu::PiInterna void Engine::drawLine(imvu::Vec3f from, imvu::Vec3f to, imvu::Color color) { - inner_->drawLine(from, to, color); + if (innerDraws()) { + inner_->drawLine(from, to, color); + } ++stats_.lines; } void Engine::drawPointSprites(const imvu::PointSpriteVector& sprites) { - inner_->drawPointSprites(sprites); + if (innerDraws()) { + inner_->drawPointSprites(sprites); + } ++stats_.sprites; if (sprites.first == nullptr || sprites.last <= sprites.first) { return; @@ -304,7 +319,9 @@ void Engine::drawTrianglesFromMemory(std::uint32_t vertexCount, std::uint32_t in const imvu::VertexAttributeArray& texCoords, const imvu::VertexAttributeArray& colors) { - inner_->drawTrianglesFromMemory(vertexCount, indexCount, indices, positions, normals, texCoords, colors); + if (innerDraws()) { + inner_->drawTrianglesFromMemory(vertexCount, indexCount, indices, positions, normals, texCoords, colors); + } ++(indices != nullptr ? stats_.indexed : stats_.unindexed); if (positions.data == nullptr || vertexCount == 0) { return; diff --git a/adapter/engine.hpp b/adapter/engine.hpp index e186695..60a9e91 100644 --- a/adapter/engine.hpp +++ b/adapter/engine.hpp @@ -1,7 +1,10 @@ #pragma once -// The adapter's engine: what the client draws with. Everything goes to the client's own Direct3D renderer, which -// still draws the whole 3D view, and is recorded for the renderer process, which draws its part on top (the right -// half for now, to compare). If the renderer process fails, the client's renderer carries on alone. +// The adapter's engine: what the client draws with. Everything is recorded for the renderer process, which draws the +// 3D view. The client's own Direct3D renderer gets everything too (state and resources: it still draws into render +// targets and reads textures back, which the renderer process doesn't yet), but while the renderer process covers the +// whole view, the client's renderer skips its frames' clears and draws. When the renderer process doesn't cover it +// (it failed, or shows only part of the view, for comparing) the client's renderer draws everything, from the next +// frame on. #include #include @@ -63,10 +66,13 @@ public: private: void setState(protocol::State state, std::uint32_t value); + // Whether the client's renderer should draw this frame's clears and draws. + [[nodiscard]] bool innerDraws() const { return !takenOver_; } imvu::SharedPtr inner_; std::shared_ptr recorder_; std::uint64_t frames_ = 0; + bool takenOver_ = false; // this frame is drawn by the renderer process only int foreignLogged_ = 0; // diagnostics: textures the client renderer made, render targets // Diagnostics: what the client draws, summed over 300 frames and logged (the first few times). diff --git a/adapter/plugin.cpp b/adapter/plugin.cpp index ce9a9a8..4f7d001 100644 --- a/adapter/plugin.cpp +++ b/adapter/plugin.cpp @@ -33,7 +33,7 @@ imvu::SharedPtr create_imvu_engine(imvu::SharedPtralive(); } + // Whether the renderer process draws the whole 3D view, so the client's renderer needn't. + [[nodiscard]] bool covering() const { return process_ && process_->covering(); } + // Records a command (if active); returns its payload to fill, or null. template std::byte* add(protocol::Op op, const Body& body, std::size_t payloadSize = 0) diff --git a/adapter/renderer_process.cpp b/adapter/renderer_process.cpp index 732071d..ee66c03 100644 --- a/adapter/renderer_process.cpp +++ b/adapter/renderer_process.cpp @@ -84,11 +84,14 @@ std::unique_ptr RendererProcess::start(HWND window, protocol::V .view = view, .capacity = protocol::channelCapacity, .chunkSize = 0, - .chunkFlags = 0}; - - // The renderer's window goes on top of the client's 3D view: keep the client's drawing out of it. - SetWindowLongW(window, GWL_STYLE, GetWindowLongW(window, GWL_STYLE) | WS_CLIPCHILDREN); + .chunkFlags = 0, + .covering = 0, + .reserved = 0}; + // The renderer's window goes on top of the client's 3D view, as a child window. The client's window mustn't clip + // its children (WS_CLIPCHILDREN): the client draws a frame when its window gets WM_PAINT, which a window entirely + // covered by clipped children never gets (it then drew about once a second). The renderer's swap chain is a flip + // model one, which the compositor shows above the client's drawing anyway. const auto exe = running_copy(module_folder() / L"t5de-renderer.exe"); std::wstring command = std::format(L"\"{}\" --channel {} --parent {}", exe.wstring(), name, GetCurrentProcessId()); STARTUPINFOW startup{.cb = sizeof(STARTUPINFOW)}; diff --git a/adapter/renderer_process.hpp b/adapter/renderer_process.hpp index 9049021..5285b69 100644 --- a/adapter/renderer_process.hpp +++ b/adapter/renderer_process.hpp @@ -27,6 +27,9 @@ public: [[nodiscard]] bool alive() const { return alive_; } + // Whether the renderer process covers the whole 3D view (protocol::Channel::covering). + [[nodiscard]] bool covering() const { return alive_ && channel_->covering != 0; } + private: RendererProcess() = default; diff --git a/docs/architecture.md b/docs/architecture.md index ecc455a..9be0feb 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -29,19 +29,25 @@ creates (each 3D view, and again on room changes), the adapter: The renderer process draws into a child window it creates inside the client's window: window handles are valid in every process, and Windows allows child windows across processes (Chrome's GPU process works this way). The child -is disabled so mouse input goes to the client's window under it, and the adapter gives the client's window -`WS_CLIPCHILDREN` so the client's own drawing stays out of it. The process exits when the engine is destroyed or the -client exits. +is disabled so mouse input goes to the client's window under it. Its swap chain is a flip model one, which the +compositor shows above the client's own drawing. The client's window must not clip its children (`WS_CLIPCHILDREN`): +the client draws a frame when its window gets `WM_PAINT` (`SceneWindow.dll` invalidates it, and draws on the paint), +and a window entirely covered by clipped children never gets one, so the client fell to about one frame a second. +The process exits when the engine is destroyed or the client exits. ## Status -The renderer process draws the client's whole scene, over the right half of the 3D view, beside the client's own -Direct3D renderer on the left (the adapter still forwards everything to it). Checked in rooms with screenshots of -both renderers side by side (the compare view): they match, particles and the 2D overlay (name tags) included. +The renderer process draws the whole 3D view in place of the client's renderer (the full view, the default). The +adapter still forwards everything to the client's Direct3D renderer (state and resources: it draws render targets and +reads textures back, which the renderer process doesn't yet), but while the renderer process covers the whole view +(`Channel::covering`, set after each frame it presents) the client's renderer skips its clears and draws, decided per +frame; it draws again as soon as the renderer process stops covering the view (the view changes, or the process +fails). Checked in rooms with screenshots of both renderers side by side (the compare view): they match, particles +and the 2D overlay (name tags) included. For comparing while the client runs (development): -- Views: `T5DE_RENDERER_VIEW` (off, split, full or compare) when the client starts, or +- Views: `T5DE_RENDERER_VIEW` (off, split, full (the default) or compare) when the client starts, or `%TEMP%\t5de-renderer-view.txt` holding one of those at any time. Compare draws the left half's scene over the right half, so both renderers show the same thing side by side. - `%TEMP%\t5de-renderer-shaders\fixed_function.hlsl`, if present, replaces the built-in shaders and is reloaded @@ -63,8 +69,8 @@ For comparing while the client runs (development): the right half visibly trails, mostly from compositing another process's window over Remote Desktop. The renderer process draws only the newest frame when it falls behind, and keeps at most one frame queued. -Next: render targets (offscreen frames are skipped), the renderer process drawing the whole view in place of the -client's renderer, and one renderer process for all of a client's views. +Next: render targets (offscreen frames are skipped, and drawn by the client's renderer), and one renderer process for +all of a client's views. ## Building diff --git a/include/t5de/protocol/channel.hpp b/include/t5de/protocol/channel.hpp index 610a39d..9ab026a 100644 --- a/include/t5de/protocol/channel.hpp +++ b/include/t5de/protocol/channel.hpp @@ -7,9 +7,10 @@ namespace t5de::protocol { -inline constexpr std::uint32_t version = 2; +inline constexpr std::uint32_t version = 3; -// Which part of the client's 3D view the renderer process draws (the client's own renderer draws all of it, under). +// Which part of the client's 3D view the renderer process draws. Full is the default; the others are for comparing +// with the client's renderer, which then draws the whole view under the renderer process's window. enum class View : std::uint32_t { Off, // none Split, // the right half, to compare with the client's renderer on the left @@ -30,9 +31,13 @@ struct Channel { std::uint32_t capacity; std::uint32_t chunkSize; std::uint32_t chunkFlags; // chunkEndsFrame + // Set by the renderer process while its window covers the whole 3D view and it's presenting frames: the client's + // renderer then needn't draw (the adapter skips its draws; it draws again as soon as this is cleared). + std::uint32_t covering; + std::uint32_t reserved; }; -static_assert(sizeof(Channel) == 40, "same layout in 32-bit and 64-bit builds"); +static_assert(sizeof(Channel) == 48, "same layout in 32-bit and 64-bit builds"); inline constexpr std::uint32_t chunkEndsFrame = 1; inline constexpr std::uint32_t channelCapacity = 16 * 1024 * 1024; diff --git a/renderer/main.cpp b/renderer/main.cpp index 3f4eed9..29700bf 100644 --- a/renderer/main.cpp +++ b/renderer/main.cpp @@ -290,14 +290,17 @@ int WINAPI wWinMain(HINSTANCE, HINSTANCE, PWSTR, int) info("drawing {}x{} at {} in the client's 3D view, its scene from {}", width, height, place.left, offset); } else { ShowWindow(window, SW_HIDE); + channel->covering = 0; info("hidden"); } } reload_shaders(scene); LARGE_INTEGER start, end; QueryPerformanceCounter(&start); + const bool covers = current_view(channel->view) == View::Full && width > 0 && height > 0; if (scene.execute(std::span(frame).first(complete), offset, width > 0 && height > 0)) { swapChain->Present(0); + channel->covering = covers ? 1 : 0; // the client's renderer can stop drawing (or must start again) QueryPerformanceCounter(&end); drawTime += static_cast(end.QuadPart - start.QuadPart) / static_cast(frequency.QuadPart); latency += static_cast(end.QuadPart - static_cast(scene.lastSentAt())) / @@ -315,6 +318,7 @@ int WINAPI wWinMain(HINSTANCE, HINSTANCE, PWSTR, int) frame.erase(frame.begin(), frame.begin() + static_cast(complete)); } + channel->covering = 0; info("exiting"); DestroyWindow(window); return 0;