Never make the client wait for drawing, or hold up its steps

The renderer process now takes each view's chunks on a thread of its own as
they come, and draws the newest frame on its main thread: the client no longer
waits for the previous frame to be drawn (on a VM, 22 of a frame's 25 ms).

The adapter records the client's steps (SceneViewer::step, by replacing its
entry in the exported IFrameListener vtable, after checking it), and the frame
pacer doesn't ask for a frame that wouldn't end before the next step is due: a
late step made the client catch up two at once, which showed as hitches. While
the renderer process covers the view, the client's renderer doesn't begin or
present frames, which could wait for the display.

The engine's stats now include the time in the client renderer's methods.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

adapter/CMakeLists.txt

@@ -9,7 +9,8 @@ add_library(t5de_render_engine SHARED
     plugin.def
     recorder.cpp
     renderer_process.cpp
-    resources.cpp)
+    resources.cpp
+    scene_hooks.cpp)
 set_target_properties(t5de_render_engine PROPERTIES OUTPUT_NAME t5deRenderEngine)
 target_link_libraries(t5de_render_engine PRIVATE t5de::imvu t5de::protocol t5de::options)
 

adapter/engine.cpp

@@ -8,6 +8,7 @@
 #include <vector>
 
 #include "frame_pacer.hpp"
+#include "inner_timing.hpp"
 #include "log.hpp"
 #include "resources.hpp"
 
@@ -31,7 +32,7 @@ protocol::SetTransform transform_command(const imvu::Transform& t)
 } // namespace
 
 Engine::Engine(imvu::SharedPtr<imvu::PiRenderEngine> inner, std::shared_ptr<Recorder> recorder, HWND window)
-    : inner_(std::move(inner)), recorder_(std::move(recorder)), window_(window)
+    : inner_(std::move(inner)), recorder_(std::move(recorder)), window_(window), innerTicksAtStats_(innerTicks)
 {
     QueryPerformanceCounter(&statsStart_);
     if (window_ != nullptr) {
@@ -52,21 +53,21 @@ void Engine::setState(State state, std::uint32_t value)
     recorder_->add(Op::SetState, protocol::SetState{state, value});
 }
 
-void Engine::resizeFrameBuffer(imvu::Vec2i size) { inner_->resizeFrameBuffer(size); }
+void Engine::resizeFrameBuffer(imvu::Vec2i size) { const InnerCall timed; inner_->resizeFrameBuffer(size); }
 
-void Engine::enable2dFrameBuffer() { inner_->enable2dFrameBuffer(); }
+void Engine::enable2dFrameBuffer() { const InnerCall timed; inner_->enable2dFrameBuffer(); }
 
 void Engine::clear(std::uint32_t flags, imvu::Color color)
 {
     if (innerDraws()) {
-        inner_->clear(flags, color);
+        { const InnerCall timed; inner_->clear(flags, color); }
     }
     recorder_->add(Op::Clear, protocol::Clear{flags, color});
 }
 
 void Engine::setProjection(imvu::Vec2i viewport, const imvu::Mat4f& projection)
 {
-    inner_->setProjection(viewport, projection);
+    { const InnerCall timed; inner_->setProjection(viewport, projection); }
     if (projection.m[2][3] == 0.0f) {
         ++stats_.flat;
     }
@@ -77,19 +78,19 @@ void Engine::setProjection(imvu::Vec2i viewport, const imvu::Mat4f& projection)
 
 void Engine::setModelSpace(const imvu::Transform& model)
 {
-    inner_->setModelSpace(model);
+    { const InnerCall timed; inner_->setModelSpace(model); }
     recorder_->add(Op::SetModel, transform_command(model));
 }
 
 void Engine::setViewSpace(const imvu::Transform& view)
 {
-    inner_->setViewSpace(view);
+    { const InnerCall timed; inner_->setViewSpace(view); }
     recorder_->add(Op::SetView, transform_command(view));
 }
 
 void Engine::setTextures(imvu::PiRenderTexture* texture0, imvu::PiRenderTexture* texture1)
 {
-    inner_->setTextures(inner(texture0), inner(texture1));
+    { const InnerCall timed; inner_->setTextures(inner(texture0), inner(texture1)); }
     for (imvu::PiRenderTexture* texture : {texture0, texture1}) {
         if (texture != nullptr && id(texture) == 0 && foreignLogged_ < 20) {
             ++foreignLogged_;
@@ -101,61 +102,61 @@ void Engine::setTextures(imvu::PiRenderTexture* texture0, imvu::PiRenderTexture*
 
 void Engine::setDepthTestingOn(bool on)
 {
-    inner_->setDepthTestingOn(on);
+    { const InnerCall timed; inner_->setDepthTestingOn(on); }
     setState(State::DepthTest, on);
 }
 
 void Engine::setDepthWritingOn(bool on)
 {
-    inner_->setDepthWritingOn(on);
+    { const InnerCall timed; inner_->setDepthWritingOn(on); }
     setState(State::DepthWrite, on);
 }
 
 void Engine::setBlendMode(imvu::RenderBlendMode mode)
 {
-    inner_->setBlendMode(mode);
+    { const InnerCall timed; inner_->setBlendMode(mode); }
     setState(State::BlendMode, static_cast<std::uint32_t>(mode));
 }
 
 void Engine::setAlphaTestingOn(bool on)
 {
-    inner_->setAlphaTestingOn(on);
+    { const InnerCall timed; inner_->setAlphaTestingOn(on); }
     setState(State::AlphaTest, on);
 }
 
 void Engine::setAlphaTestValue(float value)
 {
-    inner_->setAlphaTestValue(value);
+    { const InnerCall timed; inner_->setAlphaTestValue(value); }
     setState(State::AlphaReference, static_cast<std::uint32_t>(std::lround(value * 255.0f)));
 }
 
 void Engine::setBlendingOn(bool on)
 {
-    inner_->setBlendingOn(on);
+    { const InnerCall timed; inner_->setBlendingOn(on); }
     setState(State::Blending, on);
 }
 
 void Engine::setBackFaceCullingOn(bool on)
 {
-    inner_->setBackFaceCullingOn(on);
+    { const InnerCall timed; inner_->setBackFaceCullingOn(on); }
     setState(State::Culling, on);
 }
 
 void Engine::setNormalInvertingOn(bool on)
 {
-    inner_->setNormalInvertingOn(on);
+    { const InnerCall timed; inner_->setNormalInvertingOn(on); }
     setState(State::NormalInverting, on);
 }
 
 void Engine::setLightingOn(bool on)
 {
-    inner_->setLightingOn(on);
+    { const InnerCall timed; inner_->setLightingOn(on); }
     setState(State::Lighting, on);
 }
 
 void Engine::setLightProperties(std::uint32_t index, const imvu::LightProperties& light)
 {
-    inner_->setLightProperties(index, light);
+    { const InnerCall timed; inner_->setLightProperties(index, light); }
     protocol::SetLight command{index, light.enabled, light.diffuse,
                                light.type == imvu::LightType::Directional,
                                {light.vector.x, light.vector.y, light.vector.z}};
@@ -168,25 +169,25 @@ void Engine::setLightProperties(std::uint32_t index, const imvu::LightProperties
 
 void Engine::setFogOn(bool on)
 {
-    inner_->setFogOn(on);
+    { const InnerCall timed; inner_->setFogOn(on); }
     setState(State::Fog, on);
 }
 
 void Engine::setFogAttributes(float start, float end, imvu::Color color)
 {
-    inner_->setFogAttributes(start, end, color);
+    { const InnerCall timed; inner_->setFogAttributes(start, end, color); }
     recorder_->add(Op::SetFog, protocol::SetFog{start, end, color});
 }
 
 void Engine::setTextureTransformMatrixIdentity(std::uint32_t stage)
 {
-    inner_->setTextureTransformMatrixIdentity(stage);
+    { const InnerCall timed; inner_->setTextureTransformMatrixIdentity(stage); }
     recorder_->add(Op::SetTextureTransform, protocol::SetTextureTransform{stage, 1, {1, 0, 0, 1, 0, 0}});
 }
 
 void Engine::setTextureTransformMatrix(std::uint32_t stage, const imvu::TextureTransform& transform)
 {
-    inner_->setTextureTransformMatrix(stage, transform);
+    { const InnerCall timed; inner_->setTextureTransformMatrix(stage, transform); }
     protocol::SetTextureTransform command{stage, 0, {}};
     std::memcpy(command.m, transform.m, sizeof(command.m));
     recorder_->add(Op::SetTextureTransform, command);
@@ -194,7 +195,7 @@ void Engine::setTextureTransformMatrix(std::uint32_t stage, const imvu::TextureT
 
 void Engine::setEnvironmentAmbient(imvu::Color color)
 {
-    inner_->setEnvironmentAmbient(color);
+    { const InnerCall timed; inner_->setEnvironmentAmbient(color); }
     setState(State::Ambient, color);
 }
 
@@ -264,12 +265,15 @@ void Engine::beginRendering(imvu::PiRenderTarget* target)
     windowFrame_ = target == nullptr;
     // Frames into our render targets are drawn by the renderer process only; the client's renderer doesn't know them.
     ourTarget_ = ours(target) ? target : nullptr;
-    if (ourTarget_ == nullptr) {
-        inner_->beginRendering(target);
-    }
     // Decided per frame: frames into the client renderer's render targets, and frames while the renderer process
     // doesn't cover the view, are drawn by the client's renderer.
     const bool takenOver = ourTarget_ != nullptr || (target == nullptr && recorder_->covering());
+    // Frames the client's renderer doesn't draw aren't begun in it either: its endRendering presents, which can wait
+    // for the display (and would show nothing: the renderer process's window covers it).
+    innerFrame_ = !takenOver;
+    if (innerFrame_) {
+        { const InnerCall timed; inner_->beginRendering(target); }
+    }
     if (takenOver != takenOver_ && target == nullptr) {
         log::info("engine: frame {}: {}", frames_, takenOver ? "drawn by the renderer process" : "drawn by the client's renderer");
     }
@@ -290,8 +294,8 @@ void Engine::beginRendering(imvu::PiRenderTarget* target)
 
 void Engine::endRendering()
 {
-    if (ourTarget_ == nullptr) {
+    if (innerFrame_) {
-        inner_->endRendering();
+        { const InnerCall timed; inner_->endRendering(); }
     }
     LARGE_INTEGER now;
     QueryPerformanceCounter(&now);
@@ -321,16 +325,19 @@ void Engine::endRendering()
         if (statsLogged_ < 6 || frames_ % 3000 == 0) {
             ++statsLogged_;
             log::info("engine: last 300 frames: {:.0f} per second; each {:.2f} ms beginRendering to endRendering, {:.2f} ms "
-                      "of it sending, {:.0f} KB sent, {:.1f} buffer updates ({:.0f} KB); drawTriangles {} (and {} with "
+                      "in the client renderer's methods, {:.2f} ms sending, {:.0f} KB sent, {:.1f} buffer updates ({:.0f} "
-                      "buffers not ours), drawTrianglesFromMemory {} (and {} without indices), point sprites {}, lines {}, "
+                      "KB); drawTriangles {} (and {} with buffers not ours), drawTrianglesFromMemory {} (and {} without "
-                      "2D projections {}",
+                      "indices), point sprites {}, lines {}, 2D projections {}",
-                      300 / seconds, frameSeconds_ * 1000 / 300, sent.sendSeconds * 1000 / 300,
+                      300 / seconds, frameSeconds_ * 1000 / 300,
+                      static_cast<double>(innerTicks - innerTicksAtStats_) * 1000 / static_cast<double>(frequency.QuadPart) / 300,
+                      sent.sendSeconds * 1000 / 300,
                       static_cast<double>(sent.bytes) / 1024 / 300, sent.bufferUpdates / 300.0,
                       static_cast<double>(sent.bufferBytes) / 1024 / 300, stats_.triangles, stats_.foreignTriangles,
                       stats_.indexed, stats_.unindexed, stats_.sprites, stats_.lines, stats_.flat);
         }
         stats_ = {};
         frameSeconds_ = 0;
+        innerTicksAtStats_ = innerTicks;
     }
 }
 
@@ -339,7 +346,7 @@ void Engine::drawTriangles(imvu::PiInternalIndexBuffer* indices, imvu::PiInterna
                            imvu::PiInternalVertexBuffer* colors)
 {
     if (innerDraws()) {
-        inner_->drawTriangles(inner(indices), inner(positions), inner(normals), inner(texCoords), inner(colors));
+        { const InnerCall timed; inner_->drawTriangles(inner(indices), inner(positions), inner(normals), inner(texCoords), inner(colors)); }
     }
     ++(id(indices) != 0 && id(positions) != 0 ? stats_.triangles : stats_.foreignTriangles);
     recorder_->add(Op::DrawTriangles,
@@ -349,7 +356,7 @@ void Engine::drawTriangles(imvu::PiInternalIndexBuffer* indices, imvu::PiInterna
 void Engine::drawLine(imvu::Vec3f from, imvu::Vec3f to, imvu::Color color)
 {
     if (innerDraws()) {
-        inner_->drawLine(from, to, color);
+        { const InnerCall timed; inner_->drawLine(from, to, color); }
     }
     ++stats_.lines;
 }
@@ -357,7 +364,7 @@ void Engine::drawLine(imvu::Vec3f from, imvu::Vec3f to, imvu::Color color)
 void Engine::drawPointSprites(const imvu::PointSpriteVector& sprites)
 {
     if (innerDraws()) {
-        inner_->drawPointSprites(sprites);
+        { const InnerCall timed; inner_->drawPointSprites(sprites); }
     }
     ++stats_.sprites;
     if (sprites.first == nullptr || sprites.last <= sprites.first) {
@@ -378,7 +385,7 @@ void Engine::drawTrianglesFromMemory(std::uint32_t vertexCount, std::uint32_t in
                                      const imvu::VertexAttributeArray<imvu::Color>& colors)
 {
     if (innerDraws()) {
-        inner_->drawTrianglesFromMemory(vertexCount, indexCount, indices, positions, normals, texCoords, colors);
+        { const InnerCall timed; inner_->drawTrianglesFromMemory(vertexCount, indexCount, indices, positions, normals, texCoords, colors); }
     }
     ++(indices != nullptr ? stats_.indexed : stats_.unindexed);
     if (positions.data == nullptr || vertexCount == 0) {

adapter/engine.hpp

@@ -93,6 +93,8 @@ private:
     LARGE_INTEGER frameStart_{}; // when the current frame began
     double frameSeconds_ = 0;    // the stats' window frames' time, beginRendering to endRendering
     bool windowFrame_ = false;   // the current frame is drawn into the window
+    bool innerFrame_ = false;    // the current frame is begun in the client's renderer
+    std::int64_t innerTicksAtStats_ = 0; // innerTicks when the stats' 300 frames started
     int statsLogged_ = 0;
 
 };

adapter/frame_pacer.cpp

@@ -1,12 +1,14 @@
 #include "frame_pacer.hpp"
 
 #include <algorithm>
+#include <cstdint>
 #include <mutex>
 #include <string>
 #include <thread>
 #include <vector>
 
 #include "log.hpp"
+#include "scene_hooks.hpp"
 
 namespace t5de::frame_pacer {
 
@@ -16,6 +18,10 @@ namespace {
 constexpr DWORD clientRate = 30;
 // The client is left idle at least this share of a frame's time between frames.
 constexpr double idleShare = 1.0 / 3.0;
+// The client's tick (SceneWindow.dll's millisecondsPerFrame).
+constexpr double tickSeconds = 0.033;
+// An extra frame must end at least this long before the next tick is due.
+constexpr double tickMargin = 0.001;
 
 // The user's frame rate, from the registry (unset or unreadable: the client's own).
 DWORD read_rate()
@@ -29,16 +35,21 @@ DWORD read_rate()
     return rate;
 }
 
-double now_seconds()
+double seconds(std::int64_t counter)
 {
     static const double frequency = [] {
         LARGE_INTEGER value;
         QueryPerformanceFrequency(&value);
         return static_cast<double>(value.QuadPart);
     }();
+    return static_cast<double>(counter) / frequency;
+}
+
+double now_seconds()
+{
     LARGE_INTEGER counter;
     QueryPerformanceCounter(&counter);
-    return static_cast<double>(counter.QuadPart) / frequency;
+    return seconds(counter.QuadPart);
 }
 
 class Pacer {
@@ -108,12 +119,19 @@ private:
                 continue;
             }
             const double interval = rate == 0 ? 0 : 1.0 / rate;
+            // The next tick, if known (scene_hooks.hpp): extra frames mustn't hold it up, or the client catches up two
+            // steps at once (visible hitches).
+            const std::int64_t lastStep = scene_hooks::lastStep();
+            const double untilTick = lastStep != 0 ? seconds(lastStep) + tickSeconds - now : tickSeconds;
             {
                 const std::scoped_lock lock(mutex_);
                 for (Window& w : windows_) {
                     if (w.pending && now - w.askedAt < 0.25) {
                         continue; // not drawn yet (asked again after a while: the client may have skipped it)
                     }
+                    if (untilTick > -tickSeconds && untilTick < w.frameTime + tickMargin) {
+                        continue; // the tick is due: it draws a frame of its own
+                    }
                     // Since the last frame ended: the rest of the interval since it began, and the idle time.
                     if (now - w.drawnAt >= std::max(interval - w.frameTime, w.frameTime * idleShare)) {
                         w.pending = true;

adapter/inner_timing.hpp

@@ -0,0 +1,31 @@
+#pragma once
+// Diagnostics: the time spent in the client renderer's methods (the engine's inner engine, and the resources it made),
+// summed, so the stats can tell it from the rest of a frame.
+
+#include <windows.h>
+
+#include <cstdint>
+
+namespace t5de {
+
+// QueryPerformanceCounter ticks spent in the client renderer's methods so far.
+inline std::int64_t innerTicks = 0;
+
+// Adds its lifetime to innerTicks.
+class InnerCall {
+public:
+    InnerCall() { QueryPerformanceCounter(&start_); }
+    InnerCall(const InnerCall&) = delete;
+    InnerCall& operator=(const InnerCall&) = delete;
+    ~InnerCall()
+    {
+        LARGE_INTEGER end;
+        QueryPerformanceCounter(&end);
+        innerTicks += end.QuadPart - start_.QuadPart;
+    }
+
+private:
+    LARGE_INTEGER start_;
+};
+
+} // namespace t5de

adapter/plugin.cpp

@@ -11,6 +11,7 @@
 #include "engine.hpp"
 #include "recorder.hpp"
 #include "renderer_process.hpp"
+#include "scene_hooks.hpp"
 #include "t5de/imvu/render_engine.hpp"
 
 namespace {
@@ -57,6 +58,7 @@ extern "C" imvu::SharedPtr<imvu::PiRenderEngine> __cdecl createRenderEngine(imvu
     try {
         log::info("createRenderEngine: window {}", static_cast<void*>(window.get()));
         HWND hwnd = window ? window->getPlatformWindow() : nullptr;
+        scene_hooks::install();
         auto inner = create_imvu_engine(std::move(window));
         if (!inner) {
             return {};

adapter/resources.cpp

@@ -6,6 +6,7 @@
 #include <new>
 #include <unordered_set>
 
+#include "inner_timing.hpp"
 #include "log.hpp"
 
 namespace t5de {
@@ -73,7 +74,7 @@ imvu::Vec2i Texture::size() const { return inner_ != nullptr ? inner_->size() :
 void Texture::updateRegions(const imvu::Pixmap& pixmap, const imvu::CompoundRegion& regions)
 {
     if (inner_ != nullptr) {
-        inner_->updateRegions(pixmap, regions);
+        { const InnerCall timed; inner_->updateRegions(pixmap, regions); }
         recorder_->updateTexture(id_, pixmap); // all of it: the regions' layout isn't known
     }
 }
@@ -81,7 +82,7 @@ void Texture::updateRegions(const imvu::Pixmap& pixmap, const imvu::CompoundRegi
 void Texture::update(const imvu::Pixmap& pixmap)
 {
     if (inner_ != nullptr) {
-        inner_->update(pixmap);
+        { const InnerCall timed; inner_->update(pixmap); }
         recorder_->updateTexture(id_, pixmap);
     }
 }
@@ -124,7 +125,7 @@ VertexBuffer::~VertexBuffer()
 
 void VertexBuffer::update(const void* data, std::uint32_t stride, std::uint32_t elementSize, std::uint32_t elementCount)
 {
-    inner_->update(data, stride, elementSize, elementCount);
+    { const InnerCall timed; inner_->update(data, stride, elementSize, elementCount); }
     recorder_->updateBuffer(id_, data, stride, elementSize, elementCount);
 }
 

adapter/scene_hooks.cpp

@@ -0,0 +1,76 @@
+#include "scene_hooks.hpp"
+
+#include <windows.h>
+
+#include <atomic>
+#include <mutex>
+
+#include "log.hpp"
+
+namespace t5de::scene_hooks {
+
+namespace {
+
+// IMVU's SceneViewer::step(), a virtual method: called with the scene viewer in ECX.
+using StepFunction = void(__thiscall*)(void* self);
+
+StepFunction originalStep = nullptr;
+std::atomic<std::int64_t> stepTime = 0;
+std::atomic<bool> hooked = false;
+
+// Our SceneViewer::step: records the tick, then steps as IMVU's does. __fastcall takes ECX like __thiscall (EDX unused).
+void __fastcall step(void* self, void* /*edx*/)
+{
+    LARGE_INTEGER now;
+    QueryPerformanceCounter(&now);
+    stepTime = now.QuadPart;
+    originalStep(self);
+}
+
+// Replaces the vtable entry at slot, if it holds expected; returns whether it did.
+bool replace(void** slot, void* expected, void* replacement)
+{
+    if (*slot != expected) {
+        return false;
+    }
+    DWORD protection = 0;
+    if (!VirtualProtect(slot, sizeof(void*), PAGE_READWRITE, &protection)) {
+        return false;
+    }
+    *slot = replacement;
+    VirtualProtect(slot, sizeof(void*), protection, &protection);
+    FlushInstructionCache(GetCurrentProcess(), slot, sizeof(void*));
+    return true;
+}
+
+} // namespace
+
+void install()
+{
+    static std::once_flag once;
+    std::call_once(once, [] {
+        HMODULE sceneWindow = GetModuleHandleW(L"SceneWindow.dll");
+        auto* vtable = sceneWindow == nullptr
+                           ? nullptr
+                           : reinterpret_cast<void**>(GetProcAddress(sceneWindow, "??_7SceneViewer@@6BIFrameListener@@@"));
+        auto* stepMethod = sceneWindow == nullptr ? nullptr : GetProcAddress(sceneWindow, "?step@SceneViewer@@UAEXXZ");
+        // IFrameListener's vtable: its destructor, then step (what stepFrame calls).
+        if (vtable == nullptr || stepMethod == nullptr) {
+            log::info("scene hooks: SceneViewer's vtable or step not found; not installed");
+            return;
+        }
+        originalStep = reinterpret_cast<StepFunction>(stepMethod);
+        if (!replace(&vtable[1], reinterpret_cast<void*>(stepMethod), reinterpret_cast<void*>(&step))) {
+            log::info("scene hooks: SceneViewer's vtable doesn't hold its step; not installed");
+            return;
+        }
+        hooked = true;
+        log::info("scene hooks: installed");
+    });
+}
+
+bool installed() { return hooked; }
+
+std::int64_t lastStep() { return stepTime; }
+
+} // namespace t5de::scene_hooks

adapter/scene_hooks.hpp

@@ -0,0 +1,22 @@
+#pragma once
+// Our hooks into SceneWindow.dll's scene code, installed once, the first time an engine is made. Each replaces a
+// pointer IMVU's code calls through (an entry of an exported vtable), after checking it holds the function expected,
+// with ours, which calls IMVU's: nothing else is changed, and nothing is if a check fails (an unexpected
+// SceneWindow.dll: the hooks' features are then off, and the client runs as it would without them).
+//
+// - SceneViewer::step (the IFrameListener vtable): the client's tick, 30 times a second (SceneWindow.dll's runApp
+//   calls stepFrame, which steps each scene viewer). Recorded, for the frame pacer and for drawing between ticks.
+
+#include <cstdint>
+
+namespace t5de::scene_hooks {
+
+void install();
+
+// Whether the hooks are installed.
+[[nodiscard]] bool installed();
+
+// When a scene viewer last stepped (QueryPerformanceCounter), or 0 if none has.
+[[nodiscard]] std::int64_t lastStep();
+
+} // namespace t5de::scene_hooks

docs/architecture.md

@@ -83,8 +83,15 @@ data area (16 MB: the client's 2048x2048 snapshots just do). Checked with outfit
 Higher frame rates (in progress): the client steps its scene 30 times a second (SceneWindow.dll's `runApp`), and
 draws a window's frame on WM_PAINT. The adapter's frame pacer (`adapter/frame_pacer.cpp`) asks the 3D windows to
 repaint between steps at the user's rate (T5DE's settings, Graphics: `HKCU\Software\T5DE\Graphics\FrameRate`), only
-when affordable: drawing shares the client's main thread with its input and interface. The extra frames show the
+leaving the client idle time, and never holding up the next step: the adapter records the client's steps by replacing
-same scene until animations, the camera and particles are drawn at the time between steps.
+`SceneViewer::step`'s entry in its exported vtable (`adapter/scene_hooks.cpp`, after checking it), and a frame that
+wouldn't end before the step is due isn't asked for (a late step makes the client catch up two at once: hitches).
+While the renderer process covers the view, the client's renderer doesn't begin or present frames (its present can
+wait for the display). The renderer process takes each view's chunks on a thread of its own as they come, so the
+client never waits for drawing; it draws the newest frame. On a PC with a graphics card a room frame takes the client
+about 2.5 ms (the client's own work: posing avatars, which it does every frame, and walking the scene), and the
+renderer process about 0.5 ms. The extra frames show the same scene until animations, the camera and particles are
+drawn at the time between steps.
 
 Next: sharing pipelines between views (each view's scene compiles its own).
 

renderer/main.cpp

@@ -9,14 +9,17 @@
 #include <shellapi.h>
 
 #include <algorithm>
+#include <atomic>
 #include <cstdint>
 #include <cstring>
 #include <filesystem>
 #include <format>
 #include <memory>
+#include <mutex>
 #include <optional>
 #include <string>
 #include <string_view>
+#include <thread>
 #include <vector>
 
 #include "EngineFactoryD3D11.h"
@@ -178,7 +181,9 @@ Placement placement(HWND parent, View view)
     return place;
 }
 
-// One of the client's 3D views: an engine's channel, and the child window its frames are drawn into.
+// One of the client's 3D views: an engine's channel, and the child window its frames are drawn into. A thread of its
+// own takes the channel's chunks as soon as they come (so the client never waits for drawing) and queues them; the main
+// thread draws the newest complete frame, carrying out older ones' changes without drawing them.
 class ViewWindow {
 public:
     // Opens the channel named name and sets up its window; null if the channel is gone or it can't.
@@ -230,6 +235,11 @@ public:
             SetEvent(self->read_);
         });
         QueryPerformanceFrequency(&view->frequency_);
+        view->framesReady_ = CreateEventW(nullptr, FALSE, FALSE, nullptr);
+        if (view->framesReady_ == nullptr) {
+            return nullptr;
+        }
+        view->receiver_ = std::thread([self] { self->receive(); });
         info("{}: drawing window {:#x}, view {}", view->name_, view->channel_->window,
              static_cast<int>(view->channel_->view));
         return view;
@@ -240,6 +250,10 @@ public:
 
     ~ViewWindow()
     {
+        stopping_ = true;
+        if (receiver_.joinable()) {
+            receiver_.join();
+        }
         if (channel_ != nullptr) {
             channel_->covering = 0;
             UnmapViewOfFile(channel_);
@@ -250,41 +264,42 @@ public:
             DestroyWindow(window_);
             info("{}: closed", name_);
         }
-        for (HANDLE handle : {ready_, consumed_, read_, mapping_}) {
+        for (HANDLE handle : {ready_, consumed_, read_, mapping_, framesReady_}) {
             if (handle != nullptr) {
                 CloseHandle(handle);
             }
         }
     }
 
-    [[nodiscard]] HANDLE ready() const { return ready_; }
+    // Set when complete frames are queued.
+    [[nodiscard]] HANDLE framesReady() const { return framesReady_; }
     // Its engine is gone.
     [[nodiscard]] bool closing() const { return channel_->closing != 0; }
 
     void setShaderSource(const std::string& source) { scene_->setShaderSource(source); }
 
-    // After "ready": takes the chunk waiting, and draws the newest frame once one is complete.
+    // After framesReady: draws the newest complete frame queued. If the client has sent more meanwhile (drawing fell
-    void onReady()
+    // behind), the older ones' changes are carried out without drawing them.
+    void onFramesReady()
     {
-        if (closing() || !takeChunk()) {
+        std::vector<std::size_t> ends;
+        {
+            const std::scoped_lock lock(mutex_);
+            if (frameEnds_.empty()) {
                 return;
             }
-        // If the client has sent more frames meanwhile (drawing fell behind), catch up: carry out the older ones'
+            const std::size_t complete = frameEnds_.back();
-        // changes without drawing them, and draw only the newest. (An incomplete frame stays for the next round.)
+            frame_.assign(received_.begin(), received_.begin() + static_cast<std::ptrdiff_t>(complete));
-        std::size_t complete = frame_.size();
+            received_.erase(received_.begin(), received_.begin() + static_cast<std::ptrdiff_t>(complete));
-        while (!closing() && WaitForSingleObject(ready_, 0) == WAIT_OBJECT_0) {
+            ends.swap(frameEnds_);
-            bool ended = takeChunk();
-            while (!ended && WaitForSingleObject(ready_, 1000) == WAIT_OBJECT_0) {
-                ended = takeChunk();
         }
-            if (!ended) {
+        std::size_t begin = 0;
-                break;
+        for (std::size_t i = 0; i + 1 < ends.size(); ++i) {
-            }
+            scene_->execute(std::span(frame_).subspan(begin, ends[i] - begin), 0, false);
-            scene_->execute(std::span(frame_).first(complete), 0, false);
+            begin = ends[i];
-            frame_.erase(frame_.begin(), frame_.begin() + static_cast<std::ptrdiff_t>(complete));
-            complete = frame_.size();
             ++skipped_;
         }
+        const std::size_t complete = ends.back();
 
         // Follow the client's 3D view's size.
         const View view = current_view(channel_->view);
@@ -308,7 +323,7 @@ public:
         LARGE_INTEGER start, end;
         QueryPerformanceCounter(&start);
         const bool covers = view == View::Full && width > 0 && height > 0;
-        if (scene_->execute(std::span(frame_).first(complete), offset, width > 0 && height > 0)) {
+        if (scene_->execute(std::span(frame_).subspan(begin, complete - begin), offset, width > 0 && height > 0)) {
             swapChain_->Present(0);
             channel_->covering = covers ? 1 : 0; // the client's renderer can stop drawing (or must start again)
             QueryPerformanceCounter(&end);
@@ -327,21 +342,34 @@ public:
                 skipped_ = 0;
             }
         }
-        frame_.erase(frame_.begin(), frame_.begin() + static_cast<std::ptrdiff_t>(complete));
     }
 
 private:
     ViewWindow() = default;
 
-    // Takes the chunk waiting in the channel (after "ready"), and lets the adapter send the next: appends it to the
+    // The receiver thread: takes each chunk as it comes (after "ready"), lets the adapter send the next, and queues it;
-    // frame, and returns whether it ends a frame.
+    // sets framesReady when a frame is complete. Ends when the view closes.
-    bool takeChunk()
+    void receive()
     {
+        while (!stopping_ && !closing()) {
+            if (WaitForSingleObject(ready_, 100) != WAIT_OBJECT_0 || closing()) {
+                continue;
+            }
             const std::uint32_t size = channel_->chunkSize <= channel_->capacity ? channel_->chunkSize : 0;
             const bool endsFrame = (channel_->chunkFlags & t5de::protocol::chunkEndsFrame) != 0;
-        frame_.insert(frame_.end(), data_, data_ + size);
+            {
+                const std::scoped_lock lock(mutex_);
+                received_.insert(received_.end(), data_, data_ + size);
+                if (endsFrame) {
+                    frameEnds_.push_back(received_.size());
+                }
+            }
             SetEvent(consumed_);
-        return endsFrame;
+            if (endsFrame) {
+                SetEvent(framesReady_);
+            }
+        }
+        SetEvent(framesReady_); // wakes the main thread to close the view
     }
 
     std::string name_;
@@ -355,9 +383,16 @@ private:
     HWND window_ = nullptr;
     RefCntAutoPtr<ISwapChain> swapChain_;
     std::unique_ptr<t5de::Scene> scene_;
-    std::vector<std::byte> frame_;
+    std::vector<std::byte> frame_; // the frames being drawn
     RECT placed_{};
 
+    std::thread receiver_;
+    std::atomic<bool> stopping_ = false;
+    HANDLE framesReady_ = nullptr;
+    std::mutex mutex_;                  // guards received_ and frameEnds_
+    std::vector<std::byte> received_;   // chunks taken, not yet drawn
+    std::vector<std::size_t> frameEnds_; // where complete frames end in received_
+
     LARGE_INTEGER frequency_{};
     double drawTime_ = 0;
     double latency_ = 0;
@@ -427,7 +462,7 @@ int WINAPI wWinMain(HINSTANCE, HINSTANCE, PWSTR, int)
         std::vector<HANDLE> handles{parent, attach};
         const std::size_t count = std::min<std::size_t>(views.size(), MAXIMUM_WAIT_OBJECTS - 3);
         for (std::size_t i = 0; i < count; ++i) {
-            handles.push_back(views[(first + i) % views.size()]->ready());
+            handles.push_back(views[(first + i) % views.size()]->framesReady());
         }
         const DWORD result = MsgWaitForMultipleObjects(static_cast<DWORD>(handles.size()), handles.data(), FALSE, 1000,
                                                        QS_ALLINPUT);
@@ -440,7 +475,7 @@ int WINAPI wWinMain(HINSTANCE, HINSTANCE, PWSTR, int)
         }
         if (result >= WAIT_OBJECT_0 + 2 && result < WAIT_OBJECT_0 + handles.size()) {
             const std::size_t index = (first + (result - WAIT_OBJECT_0 - 2)) % views.size();
-            views[index]->onReady();
+            views[index]->onFramesReady();
             first = (index + 1) % views.size();
         }
     }