Never make the client wait for drawing, or hold up its steps

The renderer process now takes each view's chunks on a thread of its own as
they come, and draws the newest frame on its main thread: the client no longer
waits for the previous frame to be drawn (on a VM, 22 of a frame's 25 ms).

The adapter records the client's steps (SceneViewer::step, by replacing its
entry in the exported IFrameListener vtable, after checking it), and the frame
pacer doesn't ask for a frame that wouldn't end before the next step is due: a
late step made the client catch up two at once, which showed as hitches. While
the renderer process covers the view, the client's renderer doesn't begin or
present frames, which could wait for the display.

The engine's stats now include the time in the client renderer's methods.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

renderer/main.cpp

@@ -9,14 +9,17 @@
 #include <shellapi.h>
 
 #include <algorithm>
+#include <atomic>
 #include <cstdint>
 #include <cstring>
 #include <filesystem>
 #include <format>
 #include <memory>
+#include <mutex>
 #include <optional>
 #include <string>
 #include <string_view>
+#include <thread>
 #include <vector>
 
 #include "EngineFactoryD3D11.h"
@@ -178,7 +181,9 @@ Placement placement(HWND parent, View view)
     return place;
 }
 
-// One of the client's 3D views: an engine's channel, and the child window its frames are drawn into.
+// One of the client's 3D views: an engine's channel, and the child window its frames are drawn into. A thread of its
+// own takes the channel's chunks as soon as they come (so the client never waits for drawing) and queues them; the main
+// thread draws the newest complete frame, carrying out older ones' changes without drawing them.
 class ViewWindow {
 public:
     // Opens the channel named name and sets up its window; null if the channel is gone or it can't.
@@ -230,6 +235,11 @@ public:
             SetEvent(self->read_);
         });
         QueryPerformanceFrequency(&view->frequency_);
+        view->framesReady_ = CreateEventW(nullptr, FALSE, FALSE, nullptr);
+        if (view->framesReady_ == nullptr) {
+            return nullptr;
+        }
+        view->receiver_ = std::thread([self] { self->receive(); });
         info("{}: drawing window {:#x}, view {}", view->name_, view->channel_->window,
              static_cast<int>(view->channel_->view));
         return view;
@@ -240,6 +250,10 @@ public:
 
     ~ViewWindow()
     {
+        stopping_ = true;
+        if (receiver_.joinable()) {
+            receiver_.join();
+        }
         if (channel_ != nullptr) {
             channel_->covering = 0;
             UnmapViewOfFile(channel_);
@@ -250,41 +264,42 @@ public:
             DestroyWindow(window_);
             info("{}: closed", name_);
         }
-        for (HANDLE handle : {ready_, consumed_, read_, mapping_}) {
+        for (HANDLE handle : {ready_, consumed_, read_, mapping_, framesReady_}) {
             if (handle != nullptr) {
                 CloseHandle(handle);
             }
         }
     }
 
-    [[nodiscard]] HANDLE ready() const { return ready_; }
+    // Set when complete frames are queued.
+    [[nodiscard]] HANDLE framesReady() const { return framesReady_; }
     // Its engine is gone.
     [[nodiscard]] bool closing() const { return channel_->closing != 0; }
 
     void setShaderSource(const std::string& source) { scene_->setShaderSource(source); }
 
-    // After "ready": takes the chunk waiting, and draws the newest frame once one is complete.
-    void onReady()
+    // After framesReady: draws the newest complete frame queued. If the client has sent more meanwhile (drawing fell
+    // behind), the older ones' changes are carried out without drawing them.
+    void onFramesReady()
     {
-        if (closing() || !takeChunk()) {
-            return;
+        std::vector<std::size_t> ends;
+        {
+            const std::scoped_lock lock(mutex_);
+            if (frameEnds_.empty()) {
+                return;
+            }
+            const std::size_t complete = frameEnds_.back();
+            frame_.assign(received_.begin(), received_.begin() + static_cast<std::ptrdiff_t>(complete));
+            received_.erase(received_.begin(), received_.begin() + static_cast<std::ptrdiff_t>(complete));
+            ends.swap(frameEnds_);
         }
-        // If the client has sent more frames meanwhile (drawing fell behind), catch up: carry out the older ones'
-        // changes without drawing them, and draw only the newest. (An incomplete frame stays for the next round.)
-        std::size_t complete = frame_.size();
-        while (!closing() && WaitForSingleObject(ready_, 0) == WAIT_OBJECT_0) {
-            bool ended = takeChunk();
-            while (!ended && WaitForSingleObject(ready_, 1000) == WAIT_OBJECT_0) {
-                ended = takeChunk();
-            }
-            if (!ended) {
-                break;
-            }
-            scene_->execute(std::span(frame_).first(complete), 0, false);
-            frame_.erase(frame_.begin(), frame_.begin() + static_cast<std::ptrdiff_t>(complete));
-            complete = frame_.size();
+        std::size_t begin = 0;
+        for (std::size_t i = 0; i + 1 < ends.size(); ++i) {
+            scene_->execute(std::span(frame_).subspan(begin, ends[i] - begin), 0, false);
+            begin = ends[i];
             ++skipped_;
         }
+        const std::size_t complete = ends.back();
 
         // Follow the client's 3D view's size.
         const View view = current_view(channel_->view);
@@ -308,7 +323,7 @@ public:
         LARGE_INTEGER start, end;
         QueryPerformanceCounter(&start);
         const bool covers = view == View::Full && width > 0 && height > 0;
-        if (scene_->execute(std::span(frame_).first(complete), offset, width > 0 && height > 0)) {
+        if (scene_->execute(std::span(frame_).subspan(begin, complete - begin), offset, width > 0 && height > 0)) {
             swapChain_->Present(0);
             channel_->covering = covers ? 1 : 0; // the client's renderer can stop drawing (or must start again)
             QueryPerformanceCounter(&end);
@@ -327,21 +342,34 @@ public:
                 skipped_ = 0;
             }
         }
-        frame_.erase(frame_.begin(), frame_.begin() + static_cast<std::ptrdiff_t>(complete));
     }
 
 private:
     ViewWindow() = default;
 
-    // Takes the chunk waiting in the channel (after "ready"), and lets the adapter send the next: appends it to the
-    // frame, and returns whether it ends a frame.
-    bool takeChunk()
+    // The receiver thread: takes each chunk as it comes (after "ready"), lets the adapter send the next, and queues it;
+    // sets framesReady when a frame is complete. Ends when the view closes.
+    void receive()
     {
-        const std::uint32_t size = channel_->chunkSize <= channel_->capacity ? channel_->chunkSize : 0;
-        const bool endsFrame = (channel_->chunkFlags & t5de::protocol::chunkEndsFrame) != 0;
-        frame_.insert(frame_.end(), data_, data_ + size);
-        SetEvent(consumed_);
-        return endsFrame;
+        while (!stopping_ && !closing()) {
+            if (WaitForSingleObject(ready_, 100) != WAIT_OBJECT_0 || closing()) {
+                continue;
+            }
+            const std::uint32_t size = channel_->chunkSize <= channel_->capacity ? channel_->chunkSize : 0;
+            const bool endsFrame = (channel_->chunkFlags & t5de::protocol::chunkEndsFrame) != 0;
+            {
+                const std::scoped_lock lock(mutex_);
+                received_.insert(received_.end(), data_, data_ + size);
+                if (endsFrame) {
+                    frameEnds_.push_back(received_.size());
+                }
+            }
+            SetEvent(consumed_);
+            if (endsFrame) {
+                SetEvent(framesReady_);
+            }
+        }
+        SetEvent(framesReady_); // wakes the main thread to close the view
     }
 
     std::string name_;
@@ -355,9 +383,16 @@ private:
     HWND window_ = nullptr;
     RefCntAutoPtr<ISwapChain> swapChain_;
     std::unique_ptr<t5de::Scene> scene_;
-    std::vector<std::byte> frame_;
+    std::vector<std::byte> frame_; // the frames being drawn
     RECT placed_{};
 
+    std::thread receiver_;
+    std::atomic<bool> stopping_ = false;
+    HANDLE framesReady_ = nullptr;
+    std::mutex mutex_;                  // guards received_ and frameEnds_
+    std::vector<std::byte> received_;   // chunks taken, not yet drawn
+    std::vector<std::size_t> frameEnds_; // where complete frames end in received_
+
     LARGE_INTEGER frequency_{};
     double drawTime_ = 0;
     double latency_ = 0;
@@ -427,7 +462,7 @@ int WINAPI wWinMain(HINSTANCE, HINSTANCE, PWSTR, int)
         std::vector<HANDLE> handles{parent, attach};
         const std::size_t count = std::min<std::size_t>(views.size(), MAXIMUM_WAIT_OBJECTS - 3);
         for (std::size_t i = 0; i < count; ++i) {
-            handles.push_back(views[(first + i) % views.size()]->ready());
+            handles.push_back(views[(first + i) % views.size()]->framesReady());
         }
         const DWORD result = MsgWaitForMultipleObjects(static_cast<DWORD>(handles.size()), handles.data(), FALSE, 1000,
                                                        QS_ALLINPUT);
@@ -440,7 +475,7 @@ int WINAPI wWinMain(HINSTANCE, HINSTANCE, PWSTR, int)
         }
         if (result >= WAIT_OBJECT_0 + 2 && result < WAIT_OBJECT_0 + handles.size()) {
             const std::size_t index = (first + (result - WAIT_OBJECT_0 - 2)) % views.size();
-            views[index]->onReady();
+            views[index]->onFramesReady();
             first = (index + 1) % views.size();
         }
     }