Ask the client for frames between its steps

The frame pacer repaints the client's 3D windows at the user's frame rate
(HKCU\Software\T5DE\Graphics\FrameRate: 30, the default, adds nothing; 60, 120,
240, 360; 0 uncapped), so the client draws frames between its 30-per-second
steps. Only when affordable: a window is asked once its last frame is drawn,
and only if its frames take under three quarters of the interval; uncapped,
the client is left idle a third of a frame between frames. The engine's stats
now log frames per second.

The extra frames show the same scene for now.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

adapter/CMakeLists.txt

@@ -2,6 +2,7 @@
 
 add_library(t5de_render_engine SHARED
     engine.cpp
+    frame_pacer.cpp
     log.cpp
     pixmap.cpp
     plugin.cpp

adapter/engine.cpp

@@ -7,6 +7,7 @@
 #include <cstring>
 #include <vector>
 
+#include "frame_pacer.hpp"
 #include "log.hpp"
 #include "resources.hpp"
 
@@ -29,12 +30,22 @@ protocol::SetTransform transform_command(const imvu::Transform& t)
 
 } // namespace
 
-Engine::Engine(imvu::SharedPtr<imvu::PiRenderEngine> inner, std::shared_ptr<Recorder> recorder)
+Engine::Engine(imvu::SharedPtr<imvu::PiRenderEngine> inner, std::shared_ptr<Recorder> recorder, HWND window)
-    : inner_(std::move(inner)), recorder_(std::move(recorder))
+    : inner_(std::move(inner)), recorder_(std::move(recorder)), window_(window)
 {
+    QueryPerformanceCounter(&statsStart_);
+    if (window_ != nullptr) {
+        frame_pacer::add(window_);
+    }
 }
 
-Engine::~Engine() { log::info("engine: destroyed after {} frames", frames_); }
+Engine::~Engine()
+{
+    if (window_ != nullptr) {
+        frame_pacer::remove(window_);
+    }
+    log::info("engine: destroyed after {} frames", frames_);
+}
 
 void Engine::setState(State state, std::uint32_t value)
 {
@@ -249,6 +260,8 @@ imvu::SharedPtr<imvu::PiTextureRenderTarget> Engine::newTextureRenderTarget(imvu
 
 void Engine::beginRendering(imvu::PiRenderTarget* target)
 {
+    QueryPerformanceCounter(&frameStart_);
+    windowFrame_ = target == nullptr;
     // Frames into our render targets are drawn by the renderer process only; the client's renderer doesn't know them.
     ourTarget_ = ours(target) ? target : nullptr;
     if (ourTarget_ == nullptr) {
@@ -286,15 +299,29 @@ void Engine::endRendering()
     recorder_->flush(true);
     if (auto* pixmap = dynamic_cast<PixmapTarget*>(ourTarget_)) {
         pixmap->setPixels(recorder_->readBack());
+    } else if (windowFrame_ && window_ != nullptr) {
+        LARGE_INTEGER end, frequency;
+        QueryPerformanceCounter(&end);
+        QueryPerformanceFrequency(&frequency);
+        frame_pacer::frameDrawn(window_, static_cast<double>(end.QuadPart - frameStart_.QuadPart) /
+                                             static_cast<double>(frequency.QuadPart));
     }
     ourTarget_ = nullptr;
     ++frames_;
-    if (frames_ % 300 == 0 && statsLogged_ < 6) {
+    if (frames_ % 300 == 0) {
-        ++statsLogged_;
+        LARGE_INTEGER ended, frequency;
-        log::info("engine: last 300 frames: drawTriangles {} (and {} with buffers not ours), drawTrianglesFromMemory {} "
+        QueryPerformanceCounter(&ended);
-                  "(and {} without indices), point sprites {}, lines {}, 2D projections {}",
+        QueryPerformanceFrequency(&frequency);
-                  stats_.triangles, stats_.foreignTriangles, stats_.indexed, stats_.unindexed, stats_.sprites, stats_.lines,
+        const double seconds =
-                  stats_.flat);
+            static_cast<double>(ended.QuadPart - statsStart_.QuadPart) / static_cast<double>(frequency.QuadPart);
+        statsStart_ = ended;
+        if (statsLogged_ < 6 || frames_ % 3000 == 0) {
+            ++statsLogged_;
+            log::info("engine: last 300 frames: {:.0f} per second; drawTriangles {} (and {} with buffers not ours), "
+                      "drawTrianglesFromMemory {} (and {} without indices), point sprites {}, lines {}, 2D projections {}",
+                      300 / seconds, stats_.triangles, stats_.foreignTriangles, stats_.indexed, stats_.unindexed,
+                      stats_.sprites, stats_.lines, stats_.flat);
+        }
         stats_ = {};
     }
 }

adapter/engine.hpp

@@ -1,10 +1,11 @@
 #pragma once
 // The adapter's engine: what the client draws with. Everything is recorded for the renderer process, which draws the
-// 3D view. The client's own Direct3D renderer gets everything too (state and resources: it still draws into render
+// 3D view and the render targets it makes. The client's own Direct3D renderer gets state and resources too (as a
-// targets and reads textures back, which the renderer process doesn't yet), but while the renderer process covers the
+// fallback, and for reading textures back), but while the renderer process covers the whole view, the client's
-// whole view, the client's renderer skips its frames' clears and draws. When the renderer process doesn't cover it
+// renderer skips its frames' clears and draws. When the renderer process doesn't cover it (it failed, or shows only
-// (it failed, or shows only part of the view, for comparing) the client's renderer draws everything, from the next
+// part of the view, for comparing) the client's renderer draws everything, from the next frame on.
-// frame on.
+
+#include <windows.h>
 
 #include <cstdint>
 #include <memory>
@@ -16,8 +17,8 @@ namespace t5de {
 
 class Engine final : public imvu::PiRenderEngine {
 public:
-    // recorder may be inactive (no renderer process).
+    // recorder may be inactive (no renderer process); window is the 3D window drawn into (frame_pacer.hpp), or null.
-    Engine(imvu::SharedPtr<imvu::PiRenderEngine> inner, std::shared_ptr<Recorder> recorder);
+    Engine(imvu::SharedPtr<imvu::PiRenderEngine> inner, std::shared_ptr<Recorder> recorder, HWND window);
     ~Engine() override;
 
     void resizeFrameBuffer(imvu::Vec2i size) override;
@@ -71,6 +72,7 @@ private:
 
     imvu::SharedPtr<imvu::PiRenderEngine> inner_;
     std::shared_ptr<Recorder> recorder_;
+    HWND window_;
     std::uint64_t frames_ = 0;
     bool takenOver_ = false; // this frame is drawn by the renderer process only
     imvu::PiRenderTarget* ourTarget_ = nullptr; // this frame's render target, if it's one of ours
@@ -87,6 +89,9 @@ private:
         std::uint32_t flat = 0; // setProjection with an orthographic projection (2D)
     };
     Stats stats_;
+    LARGE_INTEGER statsStart_{}; // when the stats' 300 frames started
+    LARGE_INTEGER frameStart_{}; // when the current frame began
+    bool windowFrame_ = false;   // the current frame is drawn into the window
     int statsLogged_ = 0;
 
 };

adapter/frame_pacer.cpp

@@ -0,0 +1,152 @@
+#include "frame_pacer.hpp"
+
+#include <algorithm>
+#include <mutex>
+#include <string>
+#include <thread>
+#include <vector>
+
+#include "log.hpp"
+
+namespace t5de::frame_pacer {
+
+namespace {
+
+// The client's own rate: its ticks already draw these frames.
+constexpr DWORD clientRate = 30;
+// Extra frames only if a frame takes less than this share of the interval.
+constexpr double affordable = 0.75;
+// Uncapped, the client is left idle this share of a frame's time between frames.
+constexpr double uncappedIdle = 1.0 / 3.0;
+
+// The user's frame rate, from the registry (unset or unreadable: the client's own).
+DWORD read_rate()
+{
+    DWORD rate = 0;
+    DWORD size = sizeof(rate);
+    if (RegGetValueW(HKEY_CURRENT_USER, L"Software\\T5DE\\Graphics", L"FrameRate", RRF_RT_REG_DWORD, nullptr, &rate,
+                     &size) != ERROR_SUCCESS) {
+        return clientRate;
+    }
+    return rate;
+}
+
+double now_seconds()
+{
+    static const double frequency = [] {
+        LARGE_INTEGER value;
+        QueryPerformanceFrequency(&value);
+        return static_cast<double>(value.QuadPart);
+    }();
+    LARGE_INTEGER counter;
+    QueryPerformanceCounter(&counter);
+    return static_cast<double>(counter.QuadPart) / frequency;
+}
+
+class Pacer {
+public:
+    static Pacer& get()
+    {
+        static Pacer pacer;
+        return pacer;
+    }
+
+    void add(HWND window)
+    {
+        const std::scoped_lock lock(mutex_);
+        windows_.push_back({.window = window});
+        if (!started_) {
+            started_ = true;
+            std::thread([this] { run(); }).detach(); // runs while the client does
+        }
+    }
+
+    void remove(HWND window)
+    {
+        const std::scoped_lock lock(mutex_);
+        std::erase_if(windows_, [window](const Window& w) { return w.window == window; });
+    }
+
+    void frameDrawn(HWND window, double seconds)
+    {
+        const std::scoped_lock lock(mutex_);
+        for (Window& w : windows_) {
+            if (w.window == window) {
+                w.pending = false;
+                w.frameTime = w.frameTime == 0 ? seconds : w.frameTime * 0.9 + seconds * 0.1;
+                w.drawnAt = now_seconds();
+            }
+        }
+    }
+
+private:
+    struct Window {
+        HWND window;
+        bool pending = false;  // a repaint asked for and not yet drawn
+        double frameTime = 0;  // seconds a frame takes the client, averaged
+        double drawnAt = 0;    // when its last frame was drawn
+        double askedAt = 0;    // when a repaint was last asked for
+    };
+
+    // Re-reads the setting now and then; above the client's rate, asks each window for a repaint at the rate, when
+    // affordable.
+    void run()
+    {
+        HANDLE timer = CreateWaitableTimerExW(nullptr, nullptr, CREATE_WAITABLE_TIMER_HIGH_RESOLUTION, TIMER_ALL_ACCESS);
+        double checked = 0;
+        DWORD rate = clientRate;
+        for (;;) {
+            const double now = now_seconds();
+            if (now - checked >= 0.5) {
+                checked = now;
+                const DWORD read = read_rate();
+                if (read != rate) {
+                    rate = read;
+                    log::info("frame pacer: frame rate {}", rate == 0 ? std::string("uncapped") : std::to_string(rate));
+                }
+            }
+            if (rate != 0 && rate <= clientRate) {
+                Sleep(100);
+                continue;
+            }
+            const double interval = rate == 0 ? 0 : 1.0 / rate;
+            {
+                const std::scoped_lock lock(mutex_);
+                for (Window& w : windows_) {
+                    if (w.pending && now - w.askedAt < 0.25) {
+                        continue; // not drawn yet (asked again after a while: the client may have skipped it)
+                    }
+                    const bool due = rate == 0 ? now - w.drawnAt >= w.frameTime * uncappedIdle
+                                               : now - w.askedAt >= interval && w.frameTime < interval * affordable;
+                    if (due) {
+                        w.pending = true;
+                        w.askedAt = now;
+                        InvalidateRect(w.window, nullptr, FALSE);
+                    }
+                }
+            }
+            // Wake often enough for the rate (1 ms at most when uncapped).
+            const double wait = std::clamp(interval / 4, 0.0005, 0.004);
+            LARGE_INTEGER due{.QuadPart = -static_cast<LONGLONG>(wait * 10'000'000)}; // relative, 100 ns units
+            if (timer != nullptr && SetWaitableTimer(timer, &due, 0, nullptr, nullptr, FALSE)) {
+                WaitForSingleObject(timer, INFINITE);
+            } else {
+                Sleep(1);
+            }
+        }
+    }
+
+    std::mutex mutex_;
+    std::vector<Window> windows_;
+    bool started_ = false;
+};
+
+} // namespace
+
+void add(HWND window) { Pacer::get().add(window); }
+
+void remove(HWND window) { Pacer::get().remove(window); }
+
+void frameDrawn(HWND window, double seconds) { Pacer::get().frameDrawn(window, seconds); }
+
+} // namespace t5de::frame_pacer

adapter/frame_pacer.hpp

@@ -0,0 +1,24 @@
+#pragma once
+// Extra frames between the client's 30-per-second ticks. The client draws a 3D window's frame when the window gets
+// WM_PAINT, and steps its scene on its own timer (SceneWindow.dll's runApp: a step, then the window is invalidated);
+// its message loop also wakes for WM_PAINT between steps. So asking the windows to repaint more often makes the client
+// draw more frames, without stepping the scene. The rate is the user's (T5DE's settings, Graphics: HKCU\Software\T5DE\
+// Graphics, FrameRate): 30, the client's own and the default, adds nothing; 60, 120, 240 or 360 add frames between
+// ticks; 0 is uncapped.
+//
+// Drawing runs on the client's main thread, with its input and interface: extra frames are only asked for when they're
+// affordable. A window gets one only once its last frame is drawn, and only if its frames take less than three
+// quarters of the interval; uncapped, the client is left idle a third of a frame's time between frames.
+
+#include <windows.h>
+
+namespace t5de::frame_pacer {
+
+// A 3D window an engine draws into: repainted at the user's frame rate while added.
+void add(HWND window);
+void remove(HWND window);
+
+// A frame drawn into window, taking the client this long (beginRendering to endRendering).
+void frameDrawn(HWND window, double seconds);
+
+} // namespace t5de::frame_pacer

adapter/plugin.cpp

@@ -64,7 +64,7 @@ extern "C" imvu::SharedPtr<imvu::PiRenderEngine> __cdecl createRenderEngine(imvu
         const protocol::View shown = view();
         auto renderer = hwnd != nullptr && shown != protocol::View::Off ? RendererProcess::start(hwnd, shown) : nullptr;
         return imvu::SharedPtr<imvu::PiRenderEngine>(std::make_unique<Engine>(
-            std::move(inner), std::make_shared<Recorder>(std::move(renderer))));
+            std::move(inner), std::make_shared<Recorder>(std::move(renderer)), hwnd));
     } catch (const std::exception& e) {
         // Exceptions mustn't cross into IMVU's code (a different C++ runtime).
         log::info("createRenderEngine failed: {}", e.what());

docs/architecture.md

@@ -80,7 +80,13 @@ pipelines draw both), and a pixmap target's frame is read back through the chann
 `-read` event) into its Pixmap, as the client's renderer does, synchronously. A read back must fit in the channel's
 data area (16 MB: the client's 2048x2048 snapshots just do). Checked with outfit pictures and photos.
 
-Next: sharing pipelines between views (each view's scene compiles its own), and higher frame rates.
+Higher frame rates (in progress): the client steps its scene 30 times a second (SceneWindow.dll's `runApp`), and
+draws a window's frame on WM_PAINT. The adapter's frame pacer (`adapter/frame_pacer.cpp`) asks the 3D windows to
+repaint between steps at the user's rate (T5DE's settings, Graphics: `HKCU\Software\T5DE\Graphics\FrameRate`), only
+when affordable: drawing shares the client's main thread with its input and interface. The extra frames show the
+same scene until animations, the camera and particles are drawn at the time between steps.
+
+Next: sharing pipelines between views (each view's scene compiles its own).
 
 ## Building