Version 0.3.0

Co-Authored-By: Claude Opus 5.5 <[email protected]>

.gitea/scripts/release.py

@@ -77,8 +77,9 @@ def notes(info: dict[str, str], files: dict[str, bytes]) -> str:
     nightly = ("A nightly build of the latest commit, for testing; it's replaced by the next one.\n\n"
                if info["channel"] == "nightly" else "")
     hashes = "\n".join(f"`{name}` sha256: `{hashlib.sha256(content).hexdigest()}`" for name, content in files.items())
-    return f"""{nightly}T5DE's renderer for IMVU Classic: copy both files into a T5DE install's folder, and choose OpenGL
+    return f"""{nightly}T5DE's renderer for IMVU Classic. T5DE's setup installs a pinned release with its T5DE Renderer
-in the client's Settings > Advanced (with T5DE's renderer patch).
+patch, used when the client's Settings > Advanced > Graphics is OpenGL. To try this build instead, copy both files
+into a T5DE install's folder (with that patch).
 
 Unofficial: not made by or affiliated with IMVU. Use at your own risk.
 

CMakeLists.txt

@@ -1,7 +1,7 @@
 cmake_minimum_required(VERSION 3.28)
 
 project(t5de-renderer
-    VERSION 0.1.0
+    VERSION 0.3.0
     DESCRIPTION "A new 3D renderer for the IMVU Classic client"
     LANGUAGES CXX)
 

adapter/CMakeLists.txt

@@ -2,13 +2,15 @@
 
 add_library(t5de_render_engine SHARED
     engine.cpp
+    frame_pacer.cpp
     log.cpp
     pixmap.cpp
     plugin.cpp
     plugin.def
     recorder.cpp
     renderer_process.cpp
-    resources.cpp)
+    resources.cpp
+    scene_hooks.cpp)
 set_target_properties(t5de_render_engine PROPERTIES OUTPUT_NAME t5deRenderEngine)
 target_link_libraries(t5de_render_engine PRIVATE t5de::imvu t5de::protocol t5de::options)
 

adapter/engine.cpp

@@ -2,11 +2,14 @@
 
 #include <windows.h>
 
+#include <algorithm>
 #include <bit>
 #include <cmath>
 #include <cstring>
 #include <vector>
 
+#include "frame_pacer.hpp"
+#include "inner_timing.hpp"
 #include "log.hpp"
 #include "resources.hpp"
 
@@ -29,33 +32,43 @@ protocol::SetTransform transform_command(const imvu::Transform& t)
 
 } // namespace
 
-Engine::Engine(imvu::SharedPtr<imvu::PiRenderEngine> inner, std::shared_ptr<Recorder> recorder)
+Engine::Engine(imvu::SharedPtr<imvu::PiRenderEngine> inner, std::shared_ptr<Recorder> recorder, HWND window)
-    : inner_(std::move(inner)), recorder_(std::move(recorder))
+    : inner_(std::move(inner)), recorder_(std::move(recorder)), window_(window), innerTicksAtStats_(innerTicks)
 {
+    QueryPerformanceCounter(&statsStart_);
+    if (window_ != nullptr) {
+        frame_pacer::add(window_);
+    }
 }
 
-Engine::~Engine() { log::info("engine: destroyed after {} frames", frames_); }
+Engine::~Engine()
+{
+    if (window_ != nullptr) {
+        frame_pacer::remove(window_);
+    }
+    log::info("engine: destroyed after {} frames", frames_);
+}
 
 void Engine::setState(State state, std::uint32_t value)
 {
     recorder_->add(Op::SetState, protocol::SetState{state, value});
 }
 
-void Engine::resizeFrameBuffer(imvu::Vec2i size) { inner_->resizeFrameBuffer(size); }
+void Engine::resizeFrameBuffer(imvu::Vec2i size) { const InnerCall timed; inner_->resizeFrameBuffer(size); }
 
-void Engine::enable2dFrameBuffer() { inner_->enable2dFrameBuffer(); }
+void Engine::enable2dFrameBuffer() { const InnerCall timed; inner_->enable2dFrameBuffer(); }
 
 void Engine::clear(std::uint32_t flags, imvu::Color color)
 {
     if (innerDraws()) {
-        inner_->clear(flags, color);
+        { const InnerCall timed; inner_->clear(flags, color); }
     }
     recorder_->add(Op::Clear, protocol::Clear{flags, color});
 }
 
 void Engine::setProjection(imvu::Vec2i viewport, const imvu::Mat4f& projection)
 {
-    inner_->setProjection(viewport, projection);
+    { const InnerCall timed; inner_->setProjection(viewport, projection); }
     if (projection.m[2][3] == 0.0f) {
         ++stats_.flat;
     }
@@ -66,19 +79,19 @@ void Engine::setProjection(imvu::Vec2i viewport, const imvu::Mat4f& projection)
 
 void Engine::setModelSpace(const imvu::Transform& model)
 {
-    inner_->setModelSpace(model);
+    { const InnerCall timed; inner_->setModelSpace(model); }
     recorder_->add(Op::SetModel, transform_command(model));
 }
 
 void Engine::setViewSpace(const imvu::Transform& view)
 {
-    inner_->setViewSpace(view);
+    { const InnerCall timed; inner_->setViewSpace(view); }
     recorder_->add(Op::SetView, transform_command(view));
 }
 
 void Engine::setTextures(imvu::PiRenderTexture* texture0, imvu::PiRenderTexture* texture1)
 {
-    inner_->setTextures(inner(texture0), inner(texture1));
+    { const InnerCall timed; inner_->setTextures(inner(texture0), inner(texture1)); }
     for (imvu::PiRenderTexture* texture : {texture0, texture1}) {
         if (texture != nullptr && id(texture) == 0 && foreignLogged_ < 20) {
             ++foreignLogged_;
@@ -90,61 +103,61 @@ void Engine::setTextures(imvu::PiRenderTexture* texture0, imvu::PiRenderTexture*
 
 void Engine::setDepthTestingOn(bool on)
 {
-    inner_->setDepthTestingOn(on);
+    { const InnerCall timed; inner_->setDepthTestingOn(on); }
     setState(State::DepthTest, on);
 }
 
 void Engine::setDepthWritingOn(bool on)
 {
-    inner_->setDepthWritingOn(on);
+    { const InnerCall timed; inner_->setDepthWritingOn(on); }
     setState(State::DepthWrite, on);
 }
 
 void Engine::setBlendMode(imvu::RenderBlendMode mode)
 {
-    inner_->setBlendMode(mode);
+    { const InnerCall timed; inner_->setBlendMode(mode); }
     setState(State::BlendMode, static_cast<std::uint32_t>(mode));
 }
 
 void Engine::setAlphaTestingOn(bool on)
 {
-    inner_->setAlphaTestingOn(on);
+    { const InnerCall timed; inner_->setAlphaTestingOn(on); }
     setState(State::AlphaTest, on);
 }
 
 void Engine::setAlphaTestValue(float value)
 {
-    inner_->setAlphaTestValue(value);
+    { const InnerCall timed; inner_->setAlphaTestValue(value); }
     setState(State::AlphaReference, static_cast<std::uint32_t>(std::lround(value * 255.0f)));
 }
 
 void Engine::setBlendingOn(bool on)
 {
-    inner_->setBlendingOn(on);
+    { const InnerCall timed; inner_->setBlendingOn(on); }
     setState(State::Blending, on);
 }
 
 void Engine::setBackFaceCullingOn(bool on)
 {
-    inner_->setBackFaceCullingOn(on);
+    { const InnerCall timed; inner_->setBackFaceCullingOn(on); }
     setState(State::Culling, on);
 }
 
 void Engine::setNormalInvertingOn(bool on)
 {
-    inner_->setNormalInvertingOn(on);
+    { const InnerCall timed; inner_->setNormalInvertingOn(on); }
     setState(State::NormalInverting, on);
 }
 
 void Engine::setLightingOn(bool on)
 {
-    inner_->setLightingOn(on);
+    { const InnerCall timed; inner_->setLightingOn(on); }
     setState(State::Lighting, on);
 }
 
 void Engine::setLightProperties(std::uint32_t index, const imvu::LightProperties& light)
 {
-    inner_->setLightProperties(index, light);
+    { const InnerCall timed; inner_->setLightProperties(index, light); }
     protocol::SetLight command{index, light.enabled, light.diffuse,
                                light.type == imvu::LightType::Directional,
                                {light.vector.x, light.vector.y, light.vector.z}};
@@ -157,25 +170,25 @@ void Engine::setLightProperties(std::uint32_t index, const imvu::LightProperties
 
 void Engine::setFogOn(bool on)
 {
-    inner_->setFogOn(on);
+    { const InnerCall timed; inner_->setFogOn(on); }
     setState(State::Fog, on);
 }
 
 void Engine::setFogAttributes(float start, float end, imvu::Color color)
 {
-    inner_->setFogAttributes(start, end, color);
+    { const InnerCall timed; inner_->setFogAttributes(start, end, color); }
     recorder_->add(Op::SetFog, protocol::SetFog{start, end, color});
 }
 
 void Engine::setTextureTransformMatrixIdentity(std::uint32_t stage)
 {
-    inner_->setTextureTransformMatrixIdentity(stage);
+    { const InnerCall timed; inner_->setTextureTransformMatrixIdentity(stage); }
     recorder_->add(Op::SetTextureTransform, protocol::SetTextureTransform{stage, 1, {1, 0, 0, 1, 0, 0}});
 }
 
 void Engine::setTextureTransformMatrix(std::uint32_t stage, const imvu::TextureTransform& transform)
 {
-    inner_->setTextureTransformMatrix(stage, transform);
+    { const InnerCall timed; inner_->setTextureTransformMatrix(stage, transform); }
     protocol::SetTextureTransform command{stage, 0, {}};
     std::memcpy(command.m, transform.m, sizeof(command.m));
     recorder_->add(Op::SetTextureTransform, command);
@@ -183,7 +196,7 @@ void Engine::setTextureTransformMatrix(std::uint32_t stage, const imvu::TextureT
 
 void Engine::setEnvironmentAmbient(imvu::Color color)
 {
-    inner_->setEnvironmentAmbient(color);
+    { const InnerCall timed; inner_->setEnvironmentAmbient(color); }
     setState(State::Ambient, color);
 }
 
@@ -232,47 +245,110 @@ imvu::PiInternalVertexBuffer* Engine::newDynamicVertexBuffer(std::uint32_t eleme
 imvu::SharedPtr<imvu::PiPixmapRenderTarget> Engine::newPixmapRenderTarget(imvu::Vec2i size)
 {
     log::info("engine: frame {}: a {}x{} pixmap render target", frames_, size.x, size.y);
+    if (!recorder_->active() || size.x <= 0 || size.y <= 0) {
         return inner_->newPixmapRenderTarget(size);
     }
+    return imvu::SharedPtr<PixmapTarget>(std::make_unique<PixmapTarget>(recorder_, size));
+}
 
 imvu::SharedPtr<imvu::PiTextureRenderTarget> Engine::newTextureRenderTarget(imvu::Vec2i size)
 {
     log::info("engine: frame {}: a {}x{} texture render target", frames_, size.x, size.y);
+    if (!recorder_->active() || size.x <= 0 || size.y <= 0) {
         return inner_->newTextureRenderTarget(size);
     }
+    return imvu::SharedPtr<TextureTarget>(std::make_unique<TextureTarget>(recorder_, size));
+}
 
 void Engine::beginRendering(imvu::PiRenderTarget* target)
 {
-    inner_->beginRendering(target);
+    QueryPerformanceCounter(&frameStart_);
-    // Decided per frame: frames into render targets, and frames while the renderer process doesn't cover the view, are
+    windowFrame_ = target == nullptr;
-    // drawn by the client's renderer.
+    if (windowFrame_) {
-    const bool takenOver = target == nullptr && recorder_->covering();
+        if (lastWindowFrame_ != 0) {
+            LARGE_INTEGER frequency;
+            QueryPerformanceFrequency(&frequency);
+            longestGap_ = std::max(longestGap_, static_cast<double>(frameStart_.QuadPart - lastWindowFrame_) /
+                                                    static_cast<double>(frequency.QuadPart));
+        }
+        lastWindowFrame_ = frameStart_.QuadPart;
+    }
+    // Frames into our render targets are drawn by the renderer process only; the client's renderer doesn't know them.
+    ourTarget_ = ours(target) ? target : nullptr;
+    // Decided per frame: frames into the client renderer's render targets, and frames while the renderer process
+    // doesn't cover the view, are drawn by the client's renderer.
+    const bool takenOver = ourTarget_ != nullptr || (target == nullptr && recorder_->covering());
+    // Frames the client's renderer doesn't draw aren't begun in it either: its endRendering presents, which can wait
+    // for the display (and would show nothing: the renderer process's window covers it).
+    innerFrame_ = !takenOver;
+    if (innerFrame_) {
+        { const InnerCall timed; inner_->beginRendering(target); }
+    }
     if (takenOver != takenOver_ && target == nullptr) {
         log::info("engine: frame {}: {}", frames_, takenOver ? "drawn by the renderer process" : "drawn by the client's renderer");
     }
     takenOver_ = takenOver;
     if (target != nullptr && foreignLogged_ < 20) {
         ++foreignLogged_;
-        log::info("engine: frame {}: drawing into a render target ({})", frames_, static_cast<void*>(target));
+        log::info("engine: frame {}: drawing into {} render target ({})", frames_,
+                  ourTarget_ != nullptr ? "our" : "the client renderer's", static_cast<void*>(target));
     }
-    recorder_->add(Op::BeginFrame, protocol::BeginFrame{target != nullptr});
+    protocol::BeginFrame begin{target == nullptr ? 0 : protocol::foreignTarget, 0};
+    if (const auto* pixmap = dynamic_cast<const PixmapTarget*>(ourTarget_)) {
+        begin = {pixmap->id(), 1};
+    } else if (const auto* texture = dynamic_cast<const TextureTarget*>(ourTarget_)) {
+        begin = {texture->id(), 0};
+    }
+    recorder_->add(Op::BeginFrame, begin);
 }
 
 void Engine::endRendering()
 {
-    inner_->endRendering();
+    if (innerFrame_) {
+        { const InnerCall timed; inner_->endRendering(); }
+    }
     LARGE_INTEGER now;
     QueryPerformanceCounter(&now);
     recorder_->add(Op::EndFrame, protocol::EndFrame{static_cast<std::uint64_t>(now.QuadPart)});
     recorder_->flush(true);
+    if (auto* pixmap = dynamic_cast<PixmapTarget*>(ourTarget_)) {
+        pixmap->setPixels(recorder_->readBack());
+    } else if (windowFrame_ && window_ != nullptr) {
+        LARGE_INTEGER end, frequency;
+        QueryPerformanceCounter(&end);
+        QueryPerformanceFrequency(&frequency);
+        const double seconds =
+            static_cast<double>(end.QuadPart - frameStart_.QuadPart) / static_cast<double>(frequency.QuadPart);
+        frameSeconds_ += seconds;
+        frame_pacer::frameDrawn(window_, seconds);
+    }
+    ourTarget_ = nullptr;
     ++frames_;
-    if (frames_ % 300 == 0 && statsLogged_ < 6) {
+    if (frames_ % 300 == 0) {
+        LARGE_INTEGER ended, frequency;
+        QueryPerformanceCounter(&ended);
+        QueryPerformanceFrequency(&frequency);
+        const double seconds =
+            static_cast<double>(ended.QuadPart - statsStart_.QuadPart) / static_cast<double>(frequency.QuadPart);
+        statsStart_ = ended;
+        const Recorder::Counters sent = recorder_->takeCounters();
+        if (statsLogged_ < 6 || frames_ % 3000 == 0) {
             ++statsLogged_;
-        log::info("engine: last 300 frames: drawTriangles {} (and {} with buffers not ours), drawTrianglesFromMemory {} "
+            log::info("engine: last 300 frames: {:.0f} per second, at most {:.1f} ms apart; each {:.2f} ms beginRendering to "
-                  "(and {} without indices), point sprites {}, lines {}, 2D projections {}",
+                      "endRendering, {:.2f} ms in the client renderer's methods, {:.2f} ms sending, {:.0f} KB sent, {:.1f} "
-                  stats_.triangles, stats_.foreignTriangles, stats_.indexed, stats_.unindexed, stats_.sprites, stats_.lines,
+                      "buffer updates ({:.0f} KB); drawTriangles {} (and {} with buffers not ours), drawTrianglesFromMemory "
-                  stats_.flat);
+                      "{} (and {} without indices), point sprites {}, lines {}, 2D projections {}",
+                      300 / seconds, longestGap_ * 1000, frameSeconds_ * 1000 / 300,
+                      static_cast<double>(innerTicks - innerTicksAtStats_) * 1000 / static_cast<double>(frequency.QuadPart) / 300,
+                      sent.sendSeconds * 1000 / 300,
+                      static_cast<double>(sent.bytes) / 1024 / 300, sent.bufferUpdates / 300.0,
+                      static_cast<double>(sent.bufferBytes) / 1024 / 300, stats_.triangles, stats_.foreignTriangles,
+                      stats_.indexed, stats_.unindexed, stats_.sprites, stats_.lines, stats_.flat);
+        }
         stats_ = {};
+        frameSeconds_ = 0;
+        longestGap_ = 0;
+        innerTicksAtStats_ = innerTicks;
     }
 }
 
@@ -281,7 +357,7 @@ void Engine::drawTriangles(imvu::PiInternalIndexBuffer* indices, imvu::PiInterna
                            imvu::PiInternalVertexBuffer* colors)
 {
     if (innerDraws()) {
-        inner_->drawTriangles(inner(indices), inner(positions), inner(normals), inner(texCoords), inner(colors));
+        { const InnerCall timed; inner_->drawTriangles(inner(indices), inner(positions), inner(normals), inner(texCoords), inner(colors)); }
     }
     ++(id(indices) != 0 && id(positions) != 0 ? stats_.triangles : stats_.foreignTriangles);
     recorder_->add(Op::DrawTriangles,
@@ -291,7 +367,7 @@ void Engine::drawTriangles(imvu::PiInternalIndexBuffer* indices, imvu::PiInterna
 void Engine::drawLine(imvu::Vec3f from, imvu::Vec3f to, imvu::Color color)
 {
     if (innerDraws()) {
-        inner_->drawLine(from, to, color);
+        { const InnerCall timed; inner_->drawLine(from, to, color); }
     }
     ++stats_.lines;
 }
@@ -299,7 +375,7 @@ void Engine::drawLine(imvu::Vec3f from, imvu::Vec3f to, imvu::Color color)
 void Engine::drawPointSprites(const imvu::PointSpriteVector& sprites)
 {
     if (innerDraws()) {
-        inner_->drawPointSprites(sprites);
+        { const InnerCall timed; inner_->drawPointSprites(sprites); }
     }
     ++stats_.sprites;
     if (sprites.first == nullptr || sprites.last <= sprites.first) {
@@ -320,7 +396,7 @@ void Engine::drawTrianglesFromMemory(std::uint32_t vertexCount, std::uint32_t in
                                      const imvu::VertexAttributeArray<imvu::Color>& colors)
 {
     if (innerDraws()) {
-        inner_->drawTrianglesFromMemory(vertexCount, indexCount, indices, positions, normals, texCoords, colors);
+        { const InnerCall timed; inner_->drawTrianglesFromMemory(vertexCount, indexCount, indices, positions, normals, texCoords, colors); }
     }
     ++(indices != nullptr ? stats_.indexed : stats_.unindexed);
     if (positions.data == nullptr || vertexCount == 0) {

adapter/engine.hpp

@@ -1,10 +1,11 @@
 #pragma once
 // The adapter's engine: what the client draws with. Everything is recorded for the renderer process, which draws the
-// 3D view. The client's own Direct3D renderer gets everything too (state and resources: it still draws into render
+// 3D view and the render targets it makes. The client's own Direct3D renderer gets state and resources too (as a
-// targets and reads textures back, which the renderer process doesn't yet), but while the renderer process covers the
+// fallback, and for reading textures back), but while the renderer process covers the whole view, the client's
-// whole view, the client's renderer skips its frames' clears and draws. When the renderer process doesn't cover it
+// renderer skips its frames' clears and draws. When the renderer process doesn't cover it (it failed, or shows only
-// (it failed, or shows only part of the view, for comparing) the client's renderer draws everything, from the next
+// part of the view, for comparing) the client's renderer draws everything, from the next frame on.
-// frame on.
+
+#include <windows.h>
 
 #include <cstdint>
 #include <memory>
@@ -16,8 +17,8 @@ namespace t5de {
 
 class Engine final : public imvu::PiRenderEngine {
 public:
-    // recorder may be inactive (no renderer process).
+    // recorder may be inactive (no renderer process); window is the 3D window drawn into (frame_pacer.hpp), or null.
-    Engine(imvu::SharedPtr<imvu::PiRenderEngine> inner, std::shared_ptr<Recorder> recorder);
+    Engine(imvu::SharedPtr<imvu::PiRenderEngine> inner, std::shared_ptr<Recorder> recorder, HWND window);
     ~Engine() override;
 
     void resizeFrameBuffer(imvu::Vec2i size) override;
@@ -71,8 +72,10 @@ private:
 
     imvu::SharedPtr<imvu::PiRenderEngine> inner_;
     std::shared_ptr<Recorder> recorder_;
+    HWND window_;
     std::uint64_t frames_ = 0;
     bool takenOver_ = false; // this frame is drawn by the renderer process only
+    imvu::PiRenderTarget* ourTarget_ = nullptr; // this frame's render target, if it's one of ours
     int foreignLogged_ = 0; // diagnostics: textures the client renderer made, render targets
 
     // Diagnostics: what the client draws, summed over 300 frames and logged (the first few times).
@@ -86,6 +89,14 @@ private:
         std::uint32_t flat = 0; // setProjection with an orthographic projection (2D)
     };
     Stats stats_;
+    LARGE_INTEGER statsStart_{}; // when the stats' 300 frames started
+    LARGE_INTEGER frameStart_{}; // when the current frame began
+    std::int64_t lastWindowFrame_ = 0; // when the last window frame began
+    double longestGap_ = 0;      // the stats' longest time between window frames' beginnings, in seconds
+    double frameSeconds_ = 0;    // the stats' window frames' time, beginRendering to endRendering
+    bool windowFrame_ = false;   // the current frame is drawn into the window
+    bool innerFrame_ = false;    // the current frame is begun in the client's renderer
+    std::int64_t innerTicksAtStats_ = 0; // innerTicks when the stats' 300 frames started
     int statsLogged_ = 0;
 
 };

adapter/frame_pacer.cpp

@@ -0,0 +1,167 @@
+#include "frame_pacer.hpp"
+
+#include <algorithm>
+#include <cstdint>
+#include <mutex>
+#include <string>
+#include <thread>
+#include <vector>
+
+#include "log.hpp"
+#include "scene_hooks.hpp"
+
+namespace t5de::frame_pacer {
+
+namespace {
+
+// The client's own rate: its ticks already draw these frames.
+constexpr DWORD clientRate = 30;
+// The client is left idle at least this share of a frame's time between frames.
+constexpr double idleShare = 1.0 / 3.0;
+// The client's tick (SceneWindow.dll's millisecondsPerFrame).
+constexpr double tickSeconds = 0.033;
+// An extra frame must end at least this long before the next tick is due.
+constexpr double tickMargin = 0.001;
+
+// The user's frame rate, from the registry (unset or unreadable: the client's own).
+DWORD read_rate()
+{
+    DWORD rate = 0;
+    DWORD size = sizeof(rate);
+    if (RegGetValueW(HKEY_CURRENT_USER, L"Software\\T5DE\\Graphics", L"FrameRate", RRF_RT_REG_DWORD, nullptr, &rate,
+                     &size) != ERROR_SUCCESS) {
+        return clientRate;
+    }
+    return rate;
+}
+
+double seconds(std::int64_t counter)
+{
+    static const double frequency = [] {
+        LARGE_INTEGER value;
+        QueryPerformanceFrequency(&value);
+        return static_cast<double>(value.QuadPart);
+    }();
+    return static_cast<double>(counter) / frequency;
+}
+
+double now_seconds()
+{
+    LARGE_INTEGER counter;
+    QueryPerformanceCounter(&counter);
+    return seconds(counter.QuadPart);
+}
+
+class Pacer {
+public:
+    static Pacer& get()
+    {
+        static Pacer pacer;
+        return pacer;
+    }
+
+    void add(HWND window)
+    {
+        const std::scoped_lock lock(mutex_);
+        windows_.push_back({.window = window});
+        if (!started_) {
+            started_ = true;
+            std::thread([this] { run(); }).detach(); // runs while the client does
+        }
+    }
+
+    void remove(HWND window)
+    {
+        const std::scoped_lock lock(mutex_);
+        std::erase_if(windows_, [window](const Window& w) { return w.window == window; });
+    }
+
+    void frameDrawn(HWND window, double seconds)
+    {
+        const std::scoped_lock lock(mutex_);
+        for (Window& w : windows_) {
+            if (w.window == window) {
+                w.pending = false;
+                w.frameTime = w.frameTime == 0 ? seconds : w.frameTime * 0.9 + seconds * 0.1;
+                w.drawnAt = now_seconds();
+            }
+        }
+    }
+
+private:
+    struct Window {
+        HWND window;
+        bool pending = false;  // a repaint asked for and not yet drawn
+        double frameTime = 0;  // seconds a frame takes the client, averaged
+        double drawnAt = 0;    // when its last frame was drawn
+        double askedAt = 0;    // when a repaint was last asked for
+    };
+
+    // Re-reads the setting now and then; above the client's rate, asks each window for repaints at the rate, or as often
+    // as leaves the client its idle time.
+    void run()
+    {
+        HANDLE timer = CreateWaitableTimerExW(nullptr, nullptr, CREATE_WAITABLE_TIMER_HIGH_RESOLUTION, TIMER_ALL_ACCESS);
+        double checked = 0;
+        DWORD rate = clientRate;
+        for (;;) {
+            const double now = now_seconds();
+            if (now - checked >= 0.5) {
+                checked = now;
+                const DWORD read = read_rate();
+                if (read != rate) {
+                    rate = read;
+                    log::info("frame pacer: frame rate {}", rate == 0 ? std::string("uncapped") : std::to_string(rate));
+                }
+            }
+            if (rate != 0 && rate <= clientRate) {
+                Sleep(100);
+                continue;
+            }
+            const double interval = rate == 0 ? 0 : 1.0 / rate;
+            // The next tick, if known (scene_hooks.hpp): extra frames mustn't hold it up, or the client catches up two
+            // steps at once (visible hitches).
+            const std::int64_t lastStep = scene_hooks::lastStep();
+            const double untilTick = lastStep != 0 ? seconds(lastStep) + tickSeconds - now : tickSeconds;
+            {
+                const std::scoped_lock lock(mutex_);
+                for (Window& w : windows_) {
+                    if (w.pending && now - w.askedAt < 0.25) {
+                        continue; // not drawn yet (asked again after a while: the client may have skipped it)
+                    }
+                    if (untilTick > -tickSeconds && untilTick < w.frameTime + tickMargin) {
+                        continue; // the tick is due: it draws a frame of its own
+                    }
+                    // Since the last frame ended: the rest of the interval since it began, and the idle time.
+                    if (now - w.drawnAt >= std::max(interval - w.frameTime, w.frameTime * idleShare)) {
+                        w.pending = true;
+                        w.askedAt = now;
+                        InvalidateRect(w.window, nullptr, FALSE);
+                    }
+                }
+            }
+            // Wake often enough for the rate (1 ms at most when uncapped).
+            const double wait = std::clamp(interval / 4, 0.0005, 0.004);
+            LARGE_INTEGER due{.QuadPart = -static_cast<LONGLONG>(wait * 10'000'000)}; // relative, 100 ns units
+            if (timer != nullptr && SetWaitableTimer(timer, &due, 0, nullptr, nullptr, FALSE)) {
+                WaitForSingleObject(timer, INFINITE);
+            } else {
+                Sleep(1);
+            }
+        }
+    }
+
+    std::mutex mutex_;
+    std::vector<Window> windows_;
+    bool started_ = false;
+};
+
+} // namespace
+
+void add(HWND window) { Pacer::get().add(window); }
+
+void remove(HWND window) { Pacer::get().remove(window); }
+
+void frameDrawn(HWND window, double seconds) { Pacer::get().frameDrawn(window, seconds); }
+
+} // namespace t5de::frame_pacer

adapter/frame_pacer.hpp

@@ -0,0 +1,25 @@
+#pragma once
+// Extra frames between the client's 30-per-second ticks. The client draws a 3D window's frame when the window gets
+// WM_PAINT, and steps its scene on its own timer (SceneWindow.dll's runApp: a step, then the window is invalidated);
+// its message loop also wakes for WM_PAINT between steps. So asking the windows to repaint more often makes the client
+// draw more frames, without stepping the scene. The rate is the user's (T5DE's settings, Graphics: HKCU\Software\T5DE\
+// Graphics, FrameRate): 30, the client's own and the default, adds nothing; 60, 120, 240 or 360 add frames between
+// ticks; 0 is uncapped.
+//
+// The rate counts all of a window's frames, the client's own included: a window is asked for one once its last frame
+// is drawn and the interval since it began has passed. Drawing runs on the client's main thread, with its input and
+// interface, so the client is also always left idle a third of a frame's time between frames: when the rate can't be
+// reached, the client draws as many frames as that allows.
+
+#include <windows.h>
+
+namespace t5de::frame_pacer {
+
+// A 3D window an engine draws into: repainted at the user's frame rate while added.
+void add(HWND window);
+void remove(HWND window);
+
+// A frame drawn into window, taking the client this long (beginRendering to endRendering).
+void frameDrawn(HWND window, double seconds);
+
+} // namespace t5de::frame_pacer

adapter/inner_timing.hpp

@@ -0,0 +1,31 @@
+#pragma once
+// Diagnostics: the time spent in the client renderer's methods (the engine's inner engine, and the resources it made),
+// summed, so the stats can tell it from the rest of a frame.
+
+#include <windows.h>
+
+#include <cstdint>
+
+namespace t5de {
+
+// QueryPerformanceCounter ticks spent in the client renderer's methods so far.
+inline std::int64_t innerTicks = 0;
+
+// Adds its lifetime to innerTicks.
+class InnerCall {
+public:
+    InnerCall() { QueryPerformanceCounter(&start_); }
+    InnerCall(const InnerCall&) = delete;
+    InnerCall& operator=(const InnerCall&) = delete;
+    ~InnerCall()
+    {
+        LARGE_INTEGER end;
+        QueryPerformanceCounter(&end);
+        innerTicks += end.QuadPart - start_.QuadPart;
+    }
+
+private:
+    LARGE_INTEGER start_;
+};
+
+} // namespace t5de

adapter/pixmap.cpp

@@ -8,12 +8,20 @@ namespace {
 
 using Getter = unsigned(__thiscall*)(const imvu::Pixmap*);
 using PixelsGetter = std::uint32_t*(__thiscall*)(const imvu::Pixmap*);
+using SizeConstructor = imvu::Pixmap*(__thiscall*)(void*, int, int);
+using DefaultConstructor = imvu::Pixmap*(__thiscall*)(void*);
+using CopyConstructor = imvu::Pixmap*(__thiscall*)(void*, const imvu::Pixmap*);
+using Destructor = void(__thiscall*)(imvu::Pixmap*);
 
 struct Methods {
     Getter width = nullptr;
     Getter height = nullptr;
     Getter rowBytes = nullptr;
     PixelsGetter pixels = nullptr;
+    SizeConstructor construct = nullptr;
+    DefaultConstructor constructEmpty = nullptr;
+    CopyConstructor copy = nullptr;
+    Destructor destroy = nullptr;
 };
 
 const Methods& methods()
@@ -25,6 +33,10 @@ const Methods& methods()
             found.height = reinterpret_cast<Getter>(GetProcAddress(module, "?height@Pixmap@@QBEIXZ"));
             found.rowBytes = reinterpret_cast<Getter>(GetProcAddress(module, "?rowBytes@Pixmap@@QBEIXZ"));
             found.pixels = reinterpret_cast<PixelsGetter>(GetProcAddress(module, "?pixels@Pixmap@@QBEPAIXZ"));
+            found.construct = reinterpret_cast<SizeConstructor>(GetProcAddress(module, "??0Pixmap@@QAE@HH@Z"));
+            found.constructEmpty = reinterpret_cast<DefaultConstructor>(GetProcAddress(module, "??0Pixmap@@QAE@XZ"));
+            found.copy = reinterpret_cast<CopyConstructor>(GetProcAddress(module, "??0Pixmap@@QAE@ABV0@@Z"));
+            found.destroy = reinterpret_cast<Destructor>(GetProcAddress(module, "??1Pixmap@@QAE@XZ"));
         }
         return found;
     }();
@@ -42,4 +54,30 @@ std::optional<PixmapPixels> pixmap_pixels(const imvu::Pixmap& pixmap)
     return PixmapPixels{m.width(&pixmap), m.height(&pixmap), m.rowBytes(&pixmap), m.pixels(&pixmap)};
 }
 
+OwnedPixmap::OwnedPixmap(int width, int height)
+{
+    const Methods& m = methods();
+    if (m.construct != nullptr && m.constructEmpty != nullptr && m.copy != nullptr && m.destroy != nullptr) {
+        m.construct(storage_, width, height);
+        valid_ = true;
+    }
+}
+
+OwnedPixmap::~OwnedPixmap()
+{
+    if (valid_) {
+        methods().destroy(reinterpret_cast<imvu::Pixmap*>(storage_));
+    }
+}
+
+imvu::Pixmap* OwnedPixmap::copyTo(imvu::Pixmap* result) const
+{
+    const Methods& m = methods();
+    if (valid_) {
+        return m.copy(result, &get());
+    }
+    // Without pixmap.dll's methods there's nothing to construct one with (IMVU's own code would have failed already).
+    return m.constructEmpty != nullptr ? m.constructEmpty(result) : result;
+}
+
 } // namespace t5de

adapter/pixmap.hpp

@@ -1,6 +1,8 @@
 #pragma once
-// Reading IMVU's images (Pixmap) through pixmap.dll's exported methods, rather than depending on their layout.
+// IMVU's images (Pixmap), through pixmap.dll's exported methods rather than depending on their layout (only their
+// size, for making our own).
 
+#include <cstddef>
 #include <cstdint>
 #include <optional>
 
@@ -12,10 +14,32 @@ struct PixmapPixels {
     std::uint32_t width;
     std::uint32_t height;
     std::uint32_t rowBytes;
-    const std::uint32_t* pixels; // 0xAARRGGBB, rows rowBytes apart
+    std::uint32_t* pixels; // 0xAARRGGBB, rows rowBytes apart
 };
 
 // Null if pixmap.dll's methods can't be found.
 std::optional<PixmapPixels> pixmap_pixels(const imvu::Pixmap& pixmap);
 
+// A Pixmap of our own, made and destroyed by pixmap.dll.
+class OwnedPixmap {
+public:
+    // Width by height; invalid if pixmap.dll's methods can't be found.
+    OwnedPixmap(int width, int height);
+    OwnedPixmap(const OwnedPixmap&) = delete;
+    OwnedPixmap& operator=(const OwnedPixmap&) = delete;
+    ~OwnedPixmap();
+
+    [[nodiscard]] bool valid() const { return valid_; }
+    [[nodiscard]] const imvu::Pixmap& get() const { return *reinterpret_cast<const imvu::Pixmap*>(storage_); }
+
+    // Constructs a copy in result (raw memory for a Pixmap, as for a Pixmap returned by value); an empty Pixmap if
+    // this one isn't valid.
+    imvu::Pixmap* copyTo(imvu::Pixmap* result) const;
+
+private:
+    // sizeof(Pixmap) in pixmap.dll: its constructors set 7 fields.
+    alignas(4) std::byte storage_[28];
+    bool valid_ = false;
+};
+
 } // namespace t5de

adapter/plugin.cpp

@@ -11,6 +11,7 @@
 #include "engine.hpp"
 #include "recorder.hpp"
 #include "renderer_process.hpp"
+#include "scene_hooks.hpp"
 #include "t5de/imvu/render_engine.hpp"
 
 namespace {
@@ -57,6 +58,7 @@ extern "C" imvu::SharedPtr<imvu::PiRenderEngine> __cdecl createRenderEngine(imvu
     try {
         log::info("createRenderEngine: window {}", static_cast<void*>(window.get()));
         HWND hwnd = window ? window->getPlatformWindow() : nullptr;
+        scene_hooks::install();
         auto inner = create_imvu_engine(std::move(window));
         if (!inner) {
             return {};
@@ -64,7 +66,7 @@ extern "C" imvu::SharedPtr<imvu::PiRenderEngine> __cdecl createRenderEngine(imvu
         const protocol::View shown = view();
         auto renderer = hwnd != nullptr && shown != protocol::View::Off ? RendererProcess::start(hwnd, shown) : nullptr;
         return imvu::SharedPtr<imvu::PiRenderEngine>(std::make_unique<Engine>(
-            std::move(inner), std::make_shared<Recorder>(std::move(renderer))));
+            std::move(inner), std::make_shared<Recorder>(std::move(renderer)), hwnd));
     } catch (const std::exception& e) {
         // Exceptions mustn't cross into IMVU's code (a different C++ runtime).
         log::info("createRenderEngine failed: {}", e.what());

adapter/recorder.cpp

@@ -35,7 +35,13 @@ void Recorder::flush(bool endsFrame)
         writer_.clear();
         return;
     }
+    LARGE_INTEGER start, end, frequency;
+    QueryPerformanceCounter(&start);
     process_->send(writer_.bytes(), endsFrame);
+    QueryPerformanceCounter(&end);
+    QueryPerformanceFrequency(&frequency);
+    counters_.bytes += writer_.size();
+    counters_.sendSeconds += static_cast<double>(end.QuadPart - start.QuadPart) / static_cast<double>(frequency.QuadPart);
     writer_.clear();
 }
 
@@ -69,6 +75,18 @@ void Recorder::addPixels(protocol::Op op, std::uint32_t id, const imvu::Pixmap&
     flushIfLarge();
 }
 
+std::uint32_t Recorder::createRenderTarget(std::uint32_t width, std::uint32_t height)
+{
+    const std::uint32_t id = nextId_++;
+    add(protocol::Op::CreateRenderTarget, protocol::CreateRenderTarget{id, width, height});
+    return id;
+}
+
+std::span<const std::byte> Recorder::readBack()
+{
+    return active() ? process_->readBack() : std::span<const std::byte>{};
+}
+
 std::uint32_t Recorder::createTexture(const imvu::Pixmap& pixmap, std::uint32_t flags)
 {
     const std::uint32_t id = nextId_++;
@@ -100,6 +118,8 @@ void Recorder::updateBuffer(std::uint32_t id, const void* data, std::uint32_t st
                             std::uint32_t count)
 {
     const std::size_t size = std::size_t{elementSize} * count;
+    ++counters_.bufferUpdates;
+    counters_.bufferBytes += size;
     if (std::byte* payload = add(protocol::Op::UpdateBuffer, protocol::UpdateBuffer{id, elementSize, count}, size);
         payload != nullptr && data != nullptr) {
         pack(payload, data, stride, elementSize, count);

adapter/recorder.hpp

@@ -5,6 +5,8 @@
 
 #include <cstdint>
 #include <memory>
+#include <span>
+#include <utility>
 
 #include "renderer_process.hpp"
 #include "t5de/imvu/types.hpp"
@@ -37,6 +39,10 @@ public:
     void flushIfLarge();
 
     std::uint32_t createTexture(const imvu::Pixmap& pixmap, std::uint32_t flags);
+    // Destroyed with destroyTexture.
+    std::uint32_t createRenderTarget(std::uint32_t width, std::uint32_t height);
+    // After flushing a frame that reads its render target back: its pixels (RendererProcess::readBack).
+    std::span<const std::byte> readBack();
     void updateTexture(std::uint32_t id, const imvu::Pixmap& pixmap);
     void destroyTexture(std::uint32_t id);
 
@@ -47,12 +53,23 @@ public:
                       std::uint32_t count);
     void destroyBuffer(std::uint32_t id);
 
+    // Diagnostics: what's been sent, and the time spent sending (waiting for the renderer process to take it).
+    struct Counters {
+        std::uint64_t bytes = 0;
+        double sendSeconds = 0;
+        std::uint32_t bufferUpdates = 0;
+        std::uint64_t bufferBytes = 0;
+    };
+    // Returns the counters since the last call.
+    Counters takeCounters() { return std::exchange(counters_, {}); }
+
 private:
     void addPixels(protocol::Op op, std::uint32_t id, const imvu::Pixmap& pixmap, std::uint32_t flags);
 
     std::unique_ptr<RendererProcess> process_;
     protocol::Writer writer_;
     std::uint32_t nextId_ = 1;
+    Counters counters_;
 };
 
 // Copies count elements of elementSize bytes, stride bytes apart (0: packed), into destination, packed.

adapter/renderer_process.cpp

@@ -1,10 +1,10 @@
 #include "renderer_process.hpp"
 
 #include <algorithm>
-#include <atomic>
 #include <cstring>
 #include <filesystem>
 #include <format>
+#include <mutex>
 #include <string>
 
 #include "log.hpp"
@@ -54,12 +54,92 @@ std::filesystem::path running_copy(const std::filesystem::path& exe)
     return copy;
 }
 
+// The client's one renderer process, and the control section (protocol::Control) engines attach their channels
+// through. Started with the first engine; started again by the next one if it has exited.
+class Host {
+public:
+    static Host& get()
+    {
+        static Host host;
+        return host;
+    }
+
+    std::mutex& mutex() { return mutex_; }
+
+    // The next channel's name.
+    std::wstring nextChannel() const { return std::format(L"{}-{}", name_, channels_); }
+
+    // Tells the renderer process about the channel just created (nextChannel()), starting it if it isn't running.
+    // Returns a handle to the renderer process, the caller's to close; null if it can't.
+    HANDLE attach()
+    {
+        if (control_ == nullptr || (!running() && !startProcess())) {
+            return nullptr;
+        }
+        control_->channels = ++channels_;
+        SetEvent(attach_);
+        HANDLE process = nullptr;
+        DuplicateHandle(GetCurrentProcess(), process_, GetCurrentProcess(), &process, 0, FALSE, DUPLICATE_SAME_ACCESS);
+        return process;
+    }
+
+private:
+    Host() : name_(std::format(L"Local\\t5de-renderer-{}", GetCurrentProcessId()))
+    {
+        mapping_ = CreateFileMappingW(INVALID_HANDLE_VALUE, nullptr, PAGE_READWRITE, 0, sizeof(protocol::Control),
+                                      name_.c_str());
+        attach_ = CreateEventW(nullptr, FALSE, FALSE, (name_ + protocol::attachEventSuffix).c_str());
+        if (mapping_ != nullptr && attach_ != nullptr) {
+            control_ = static_cast<protocol::Control*>(
+                MapViewOfFile(mapping_, FILE_MAP_ALL_ACCESS, 0, 0, sizeof(protocol::Control)));
+        }
+        if (control_ == nullptr) {
+            log::info("renderer process: can't create the control section: error {}", GetLastError());
+            return;
+        }
+        *control_ = {.version = protocol::version, .channels = 0};
+    }
+
+    [[nodiscard]] bool running() const { return process_ != nullptr && WaitForSingleObject(process_, 0) == WAIT_TIMEOUT; }
+
+    bool startProcess()
+    {
+        if (process_ != nullptr) {
+            CloseHandle(process_);
+            process_ = nullptr;
+        }
+        const auto exe = running_copy(module_folder() / L"t5de-renderer.exe");
+        std::wstring command =
+            std::format(L"\"{}\" --control {} --parent {}", exe.wstring(), name_, GetCurrentProcessId());
+        STARTUPINFOW startup{.cb = sizeof(STARTUPINFOW)};
+        PROCESS_INFORMATION process{};
+        if (!CreateProcessW(exe.c_str(), command.data(), nullptr, nullptr, FALSE, 0, nullptr, nullptr, &startup,
+                            &process)) {
+            log::info("renderer process: can't start {}: error {}", exe.string(), GetLastError());
+            return false;
+        }
+        CloseHandle(process.hThread);
+        process_ = process.hProcess;
+        log::info("renderer process: started {} (pid {})", exe.string(), process.dwProcessId);
+        return true;
+    }
+
+    std::mutex mutex_;
+    std::wstring name_;
+    HANDLE mapping_ = nullptr;
+    HANDLE attach_ = nullptr;
+    protocol::Control* control_ = nullptr;
+    HANDLE process_ = nullptr;
+    std::uint32_t channels_ = 0;
+};
+
 } // namespace
 
 std::unique_ptr<RendererProcess> RendererProcess::start(HWND window, protocol::View view)
 {
-    static std::atomic<int> engines = 0;
+    Host& host = Host::get();
-    const std::wstring name = std::format(L"Local\\t5de-renderer-{}-{}", GetCurrentProcessId(), engines++);
+    const std::scoped_lock lock(host.mutex());
+    const std::wstring name = host.nextChannel();
     constexpr DWORD size = sizeof(protocol::Channel) + protocol::channelCapacity;
 
     std::unique_ptr<RendererProcess> renderer(new RendererProcess());
@@ -72,7 +152,9 @@ std::unique_ptr<RendererProcess> RendererProcess::start(HWND window, protocol::V
         static_cast<protocol::Channel*>(MapViewOfFile(renderer->mapping_, FILE_MAP_ALL_ACCESS, 0, 0, size));
     renderer->ready_ = CreateEventW(nullptr, FALSE, FALSE, (name + protocol::readyEventSuffix).c_str());
     renderer->consumed_ = CreateEventW(nullptr, FALSE, TRUE, (name + protocol::consumedEventSuffix).c_str());
-    if (renderer->channel_ == nullptr || renderer->ready_ == nullptr || renderer->consumed_ == nullptr) {
+    renderer->read_ = CreateEventW(nullptr, FALSE, FALSE, (name + protocol::readEventSuffix).c_str());
+    if (renderer->channel_ == nullptr || renderer->ready_ == nullptr || renderer->consumed_ == nullptr
+        || renderer->read_ == nullptr) {
         log::info("renderer process: can't set up the channel: error {}", GetLastError());
         return nullptr;
     }
@@ -86,25 +168,18 @@ std::unique_ptr<RendererProcess> RendererProcess::start(HWND window, protocol::V
                            .chunkSize = 0,
                            .chunkFlags = 0,
                            .covering = 0,
-                           .reserved = 0};
+                           .readSize = 0};
 
     // The renderer's window goes on top of the client's 3D view, as a child window. The client's window mustn't clip
     // its children (WS_CLIPCHILDREN, which the client creates it with): the client draws a frame when its window gets
     // WM_PAINT, which a window entirely covered by clipped children never gets (it then drew about once a second). The
     // renderer's swap chain is a flip model one, which the compositor shows above the client's drawing anyway.
     SetWindowLongW(window, GWL_STYLE, GetWindowLongW(window, GWL_STYLE) & ~WS_CLIPCHILDREN);
-    const auto exe = running_copy(module_folder() / L"t5de-renderer.exe");
+    renderer->process_ = host.attach();
-    std::wstring command = std::format(L"\"{}\" --channel {} --parent {}", exe.wstring(), name, GetCurrentProcessId());
+    if (renderer->process_ == nullptr) {
-    STARTUPINFOW startup{.cb = sizeof(STARTUPINFOW)};
-    PROCESS_INFORMATION process{};
-    if (!CreateProcessW(exe.c_str(), command.data(), nullptr, nullptr, FALSE, 0, nullptr, nullptr, &startup,
-                        &process)) {
-        log::info("renderer process: can't start {}: error {}", exe.string(), GetLastError());
         return nullptr;
     }
-    CloseHandle(process.hThread);
+    log::info("renderer process: drawing channel {}", narrow(name));
-    renderer->process_ = process.hProcess;
-    log::info("renderer process: started {} (pid {}) on channel {}", exe.string(), process.dwProcessId, narrow(name));
     return renderer;
 }
 
@@ -117,7 +192,7 @@ RendererProcess::~RendererProcess()
         }
         UnmapViewOfFile(channel_);
     }
-    for (HANDLE handle : {ready_, consumed_, mapping_, process_}) {
+    for (HANDLE handle : {ready_, consumed_, read_, mapping_, process_}) {
         if (handle != nullptr) {
             CloseHandle(handle);
         }
@@ -150,4 +225,20 @@ bool RendererProcess::send(std::span<const std::byte> commands, bool endsFrame)
     return true;
 }
 
+std::span<const std::byte> RendererProcess::readBack()
+{
+    if (!alive_) {
+        return {};
+    }
+    const HANDLE waits[] = {read_, process_};
+    const DWORD result = WaitForMultipleObjects(2, waits, FALSE, consumeTimeout);
+    if (result != WAIT_OBJECT_0) {
+        log::info("renderer process: {} reading back a frame; giving up on it",
+                  result == WAIT_OBJECT_0 + 1 ? "exited" : "not responding");
+        alive_ = false;
+        return {};
+    }
+    return {data_, std::min<std::size_t>(channel_->readSize, channel_->capacity)};
+}
+
 } // namespace t5de

adapter/renderer_process.hpp

@@ -1,6 +1,7 @@
 #pragma once
-// The renderer process (t5de-renderer.exe, 64-bit) an engine draws through: started with the engine, sent the
+// The renderer process (t5de-renderer.exe, 64-bit) an engine draws through: one for the whole client, started with
-// command stream through a shared memory channel (include/t5de/protocol/channel.hpp), and told to exit with it.
+// the first engine; each engine sends it the command stream through a shared memory channel of its own
+// (include/t5de/protocol/channel.hpp), and closes the channel when it's destroyed.
 
 #include <windows.h>
 
@@ -14,7 +15,8 @@ namespace t5de {
 
 class RendererProcess {
 public:
-    // Starts t5de-renderer.exe from this DLL's folder, drawing into part of window; null if it can't.
+    // A channel to the renderer process (t5de-renderer.exe from this DLL's folder, started if it isn't running),
+    // drawing into part of window; null if it can't.
     static std::unique_ptr<RendererProcess> start(HWND window, protocol::View view);
 
     RendererProcess(const RendererProcess&) = delete;
@@ -25,6 +27,10 @@ public:
     // process is gone or stopped taking them (it's then given up on).
     bool send(std::span<const std::byte> commands, bool endsFrame);
 
+    // After sending a frame that reads its render target back: waits for the pixels (protocol::Channel), valid until
+    // the next send. Empty if the renderer process couldn't, or is gone.
+    std::span<const std::byte> readBack();
+
     [[nodiscard]] bool alive() const { return alive_; }
 
     // Whether the renderer process covers the whole 3D view (protocol::Channel::covering).
@@ -36,6 +42,7 @@ private:
     HANDLE mapping_ = nullptr;
     HANDLE ready_ = nullptr;
     HANDLE consumed_ = nullptr;
+    HANDLE read_ = nullptr;
     HANDLE process_ = nullptr;
     protocol::Channel* channel_ = nullptr;
     std::byte* data_ = nullptr;

adapter/resources.cpp

@@ -1,8 +1,14 @@
 #include "resources.hpp"
 
+#include <algorithm>
+#include <cstring>
 #include <mutex>
+#include <new>
 #include <unordered_set>
 
+#include "inner_timing.hpp"
+#include "log.hpp"
+
 namespace t5de {
 
 namespace {
@@ -48,28 +54,48 @@ Texture::Texture(imvu::PiRenderTexture* inner, std::shared_ptr<Recorder> recorde
     add(static_cast<imvu::PiRenderTexture*>(this));
 }
 
+Texture::Texture(std::shared_ptr<Recorder> recorder, std::uint32_t targetId, imvu::Vec2i size)
+    : inner_(nullptr), recorder_(std::move(recorder)), id_(targetId), size_(size), ownsId_(false)
+{
+    add(static_cast<imvu::PiRenderTexture*>(this));
+}
+
 Texture::~Texture()
 {
     remove(static_cast<imvu::PiRenderTexture*>(this));
     delete inner_; // through its virtual destructor: IMVU's code frees it
+    if (ownsId_) {
         recorder_->destroyTexture(id_);
     }
+}
 
-imvu::Vec2i Texture::size() const { return inner_->size(); }
+imvu::Vec2i Texture::size() const { return inner_ != nullptr ? inner_->size() : size_; }
 
 void Texture::updateRegions(const imvu::Pixmap& pixmap, const imvu::CompoundRegion& regions)
 {
-    inner_->updateRegions(pixmap, regions);
+    if (inner_ != nullptr) {
+        { const InnerCall timed; inner_->updateRegions(pixmap, regions); }
         recorder_->updateTexture(id_, pixmap); // all of it: the regions' layout isn't known
     }
+}
 
 void Texture::update(const imvu::Pixmap& pixmap)
 {
-    inner_->update(pixmap);
+    if (inner_ != nullptr) {
+        { const InnerCall timed; inner_->update(pixmap); }
         recorder_->updateTexture(id_, pixmap);
     }
+}
 
-imvu::Pixmap* Texture::read(imvu::Pixmap* result) const { return inner_->read(result); }
+imvu::Pixmap* Texture::read(imvu::Pixmap* result) const
+{
+    if (inner_ != nullptr) {
+        return inner_->read(result);
+    }
+    // A render target's texture: not read back (not seen used); an image of its size.
+    log::info("texture: a render target's texture read; not supported, returning a blank image");
+    return OwnedPixmap(size_.x, size_.y).copyTo(result);
+}
 
 IndexBuffer::IndexBuffer(imvu::PiInternalIndexBuffer* inner, std::shared_ptr<Recorder> recorder, std::uint32_t id)
     : inner_(inner), recorder_(std::move(recorder)), id_(id)
@@ -99,10 +125,68 @@ VertexBuffer::~VertexBuffer()
 
 void VertexBuffer::update(const void* data, std::uint32_t stride, std::uint32_t elementSize, std::uint32_t elementCount)
 {
-    inner_->update(data, stride, elementSize, elementCount);
+    { const InnerCall timed; inner_->update(data, stride, elementSize, elementCount); }
     recorder_->updateBuffer(id_, data, stride, elementSize, elementCount);
 }
 
 void VertexBuffer::unknown2() { inner_->unknown2(); }
 
+PixmapTarget::PixmapTarget(std::shared_ptr<Recorder> recorder, imvu::Vec2i size)
+    : recorder_(std::move(recorder)),
+      id_(recorder_->createRenderTarget(static_cast<std::uint32_t>(size.x), static_cast<std::uint32_t>(size.y))),
+      size_(size), pixmap_(size.x, size.y)
+{
+    add(static_cast<imvu::PiRenderTarget*>(this));
+}
+
+PixmapTarget::~PixmapTarget()
+{
+    remove(static_cast<imvu::PiRenderTarget*>(this));
+    recorder_->destroyTexture(id_);
+}
+
+imvu::Pixmap* PixmapTarget::getPixmap(imvu::Pixmap* result) const { return pixmap_.copyTo(result); }
+
+void PixmapTarget::setPixels(std::span<const std::byte> pixels)
+{
+    const auto into = pixmap_.valid() ? pixmap_pixels(pixmap_.get()) : std::nullopt;
+    const auto width = static_cast<std::uint32_t>(size_.x);
+    const auto height = static_cast<std::uint32_t>(size_.y);
+    const std::size_t rowSize = std::size_t{width} * 4;
+    if (!into || into->pixels == nullptr || pixels.size() < rowSize * height) {
+        log::info("render target: {}x{} frame not read back ({} bytes)", width, height, pixels.size());
+        return;
+    }
+    const std::size_t copied = std::min<std::size_t>(rowSize, std::size_t{into->width} * 4);
+    auto* rows = reinterpret_cast<std::byte*>(into->pixels);
+    for (std::uint32_t y = 0; y < std::min(into->height, height); ++y) {
+        std::memcpy(rows + std::size_t{y} * into->rowBytes, pixels.data() + y * rowSize, copied);
+    }
+}
+
+TextureTarget::TextureTarget(std::shared_ptr<Recorder> recorder, imvu::Vec2i size)
+    : recorder_(std::move(recorder)),
+      id_(recorder_->createRenderTarget(static_cast<std::uint32_t>(size.x), static_cast<std::uint32_t>(size.y))),
+      size_(size)
+{
+    add(static_cast<imvu::PiRenderTarget*>(this));
+}
+
+TextureTarget::~TextureTarget()
+{
+    remove(static_cast<imvu::PiRenderTarget*>(this));
+    recorder_->destroyTexture(id_);
+}
+
+imvu::TextureArea* TextureTarget::area(imvu::TextureArea* result) const
+{
+    return new (result) imvu::TextureArea{0, 0, 1, 1};
+}
+
+imvu::SharedPtr<imvu::PiRenderTexture>* TextureTarget::texture(imvu::SharedPtr<imvu::PiRenderTexture>* result) const
+{
+    return new (result) imvu::SharedPtr<imvu::PiRenderTexture>(
+        std::unique_ptr<imvu::PiRenderTexture>(new Texture(recorder_, id_, size_)));
+}
+
 } // namespace t5de

adapter/resources.hpp

@@ -3,8 +3,12 @@
 // client renderer still draws with) and mirrors it, by id, to the renderer process. The client deletes them through
 // their virtual destructors.
 
+#include <cstddef>
 #include <cstdint>
 #include <memory>
+#include <span>
+
+#include "pixmap.hpp"
 
 #include "recorder.hpp"
 #include "t5de/imvu/resources.hpp"
@@ -14,6 +18,9 @@ namespace t5de {
 class Texture final : public imvu::PiRenderTexture {
 public:
     Texture(imvu::PiRenderTexture* inner, std::shared_ptr<Recorder> recorder, std::uint32_t id);
+    // What's been drawn into one of our texture render targets (TextureTarget): no texture of the client renderer's,
+    // and the id stays the target's.
+    Texture(std::shared_ptr<Recorder> recorder, std::uint32_t targetId, imvu::Vec2i size);
     ~Texture() override;
 
     imvu::Vec2i size() const override;
@@ -28,6 +35,44 @@ private:
     imvu::PiRenderTexture* inner_;
     std::shared_ptr<Recorder> recorder_;
     std::uint32_t id_;
+    imvu::Vec2i size_{};
+    bool ownsId_ = true;
+};
+
+// Render targets the renderer process draws (made while it runs; otherwise the client renderer makes its own).
+class PixmapTarget final : public imvu::PiPixmapRenderTarget {
+public:
+    PixmapTarget(std::shared_ptr<Recorder> recorder, imvu::Vec2i size);
+    ~PixmapTarget() override;
+
+    imvu::Pixmap* getPixmap(imvu::Pixmap* result) const override;
+
+    // Copies a frame's pixels, read back from the renderer process (rows packed, 0xAARRGGBB), into the pixmap.
+    void setPixels(std::span<const std::byte> pixels);
+
+    [[nodiscard]] std::uint32_t id() const { return id_; }
+
+private:
+    std::shared_ptr<Recorder> recorder_;
+    std::uint32_t id_;
+    imvu::Vec2i size_;
+    OwnedPixmap pixmap_;
+};
+
+class TextureTarget final : public imvu::PiTextureRenderTarget {
+public:
+    TextureTarget(std::shared_ptr<Recorder> recorder, imvu::Vec2i size);
+    ~TextureTarget() override;
+
+    imvu::TextureArea* area(imvu::TextureArea* result) const override;
+    imvu::SharedPtr<imvu::PiRenderTexture>* texture(imvu::SharedPtr<imvu::PiRenderTexture>* result) const override;
+
+    [[nodiscard]] std::uint32_t id() const { return id_; }
+
+private:
+    std::shared_ptr<Recorder> recorder_;
+    std::uint32_t id_;
+    imvu::Vec2i size_;
 };
 
 class IndexBuffer final : public imvu::PiInternalIndexBuffer {
@@ -62,8 +107,9 @@ private:
     std::uint32_t id_;
 };
 
-// Whether an object is one of ours (live Texture, IndexBuffer or VertexBuffer): the client can also hand the engine
+// Whether an object is one of ours (live Texture, IndexBuffer, VertexBuffer, PixmapTarget or TextureTarget): the client
-// objects the client renderer made itself (render targets' textures), which must reach it unchanged.
+// can also hand the engine objects the client renderer made itself (its render targets, and their textures), which
+// must reach it unchanged.
 bool ours(const void* object);
 
 // The client renderer's object, and our id, for what the client passes in: ours unwrapped, others unchanged (id 0).

adapter/scene_hooks.cpp

@@ -0,0 +1,503 @@
+#include "scene_hooks.hpp"
+
+#include <windows.h>
+
+#include <algorithm>
+#include <atomic>
+#include <cmath>
+#include <cstring>
+#include <mutex>
+#include <unordered_map>
+
+#include "log.hpp"
+
+namespace t5de::scene_hooks {
+
+namespace {
+
+// IMVU's SceneViewer::step() and ModelInstance::updateCalSkeleton(): methods, called with the object in ECX.
+using Method = void(__thiscall*)(void* self);
+
+Method originalStep = nullptr;
+Method originalUpdateCalSkeleton = nullptr;
+// IMVU's CameraController::getCameraTransform(): returns a Transform by value (constructed in result).
+using GetTransform = void*(__thiscall*)(const void* self, void* result);
+GetTransform originalGetCameraTransform = nullptr;
+float cameraEasing = 0; // the share of the way to its target the camera moves at each tick
+std::atomic<std::int64_t> stepTime = 0;
+std::atomic<std::uint64_t> steps = 0;
+// How many ticks the last step covered: the client steps a scene several ticks at once when it falls behind.
+std::atomic<int> stepTicks = 1;
+std::atomic<bool> stepHooked = false;
+std::atomic<bool> poseHooked = false;
+bool crossfades = false; // whether ModelInstance::step's crossfades are as expected (checked when installing)
+std::atomic<bool> cameraHooked = false;
+std::atomic<bool> particlesHooked = false;
+
+// IMVU's ParticleSystem::render(RenderManager*, const Transform&, const Transform&): a method with 3 arguments.
+using RenderParticles = void(__thiscall*)(void* self, void* renderManager, const void* a, const void* b);
+RenderParticles originalRenderParticles = nullptr;
+// The particle library's (ParticleLib.dll: PAPI, the Particle System API) PContextParticleGroup_t::GetParticlePointer:
+// the current group's particles, as floats: returns their count, and sets the first's address, the stride between
+// them, and the offsets of their attributes (in floats): position, previous position, size, velocity, previous
+// velocity, color, alpha, age, up, rotational velocity, previous up, mass, data.
+using GetParticlePointer = unsigned(__thiscall*)(void* group, float*& particles, unsigned& stride, unsigned& position,
+                                                 unsigned& positionB, unsigned& size, unsigned& velocity,
+                                                 unsigned& velocityB, unsigned& color, unsigned& alpha, unsigned& age,
+                                                 unsigned& up, unsigned& rotationalVelocity, unsigned& upB,
+                                                 unsigned& mass, unsigned& data);
+GetParticlePointer getParticlePointer = nullptr;
+// PContextParticleGroup_t::GetGroupCount: how many particle groups there are (GetParticlePointer throws without one).
+using GetGroupCount = unsigned(__thiscall*)(void* group);
+GetGroupCount getGroupCount = nullptr;
+
+// zero.dll's constants: a tick, in milliseconds (33), and an animation frame, in seconds (1/30).
+const unsigned* millisecondsPerFrame = nullptr;
+const float* secondsPerFrame = nullptr;
+
+LARGE_INTEGER frequency()
+{
+    static const LARGE_INTEGER value = [] {
+        LARGE_INTEGER f;
+        QueryPerformanceFrequency(&f);
+        return f;
+    }();
+    return value;
+}
+
+// Our SceneViewer::step: records the tick, then steps as IMVU's does. __fastcall takes ECX like __thiscall (EDX unused).
+// Each scene viewer steps in turn at a tick: steps closer than 5 ms are the same tick.
+void __fastcall step(void* self, void* /*edx*/)
+{
+    LARGE_INTEGER now;
+    QueryPerformanceCounter(&now);
+    const std::int64_t last = stepTime.load();
+    if (now.QuadPart - last > frequency().QuadPart / 200) {
+        ++steps;
+        if (last != 0 && millisecondsPerFrame != nullptr && *millisecondsPerFrame != 0) {
+            const double ticks = static_cast<double>(now.QuadPart - last) * 1000.0 /
+                                 static_cast<double>(frequency().QuadPart) / *millisecondsPerFrame;
+            stepTicks = std::clamp(static_cast<int>(ticks + 0.5), 1, 5);
+        }
+        stepTime = now.QuadPart;
+    }
+    originalStep(self);
+}
+
+// IMVU's objects, as far as posing between ticks reads and writes them (from SceneWindow.dll's ModelInstance::step,
+// which copies each effect's time and weight into its Cal3D animation at every tick, and updateCalSkeleton):
+// - ModelInstance: +0x2c, a flag: its skeleton must be posed again; +0x120, its animation effects, a std::list (a pointer
+//   to the list's head node; a node: next, previous, then the effect).
+// - Effect: +0x34, playing (a flag); +0x3c, its time, in seconds; +0x90, its Cal3D animation.
+// - The Cal3D animation: +0x8, its time, which the mixer poses the skeleton at (CalMixer::updateSkeleton); +0xc, its
+//   weight in the pose.
+//
+// Crossfades (ModelInstance::step, at each tick): an effect fading in (+0x44 set, +0x18 nonzero) has its weight
+// (+0x38) ramped over 10 ticks, counted at +0x48: (n + 1) * 0.1 * weight, then n goes up. One fading out is in the
+// model's crossfade list (+0x138, a std::list; a node holds the effect at +0xc and its count at +0x14): m goes up, then
+// (1 - m * 0.1) * weight; after 10, it's removed. Between ticks, both are partway to the next tick's weight.
+constexpr std::size_t modelPoseDirty = 0x2c;
+constexpr std::size_t modelEffects = 0x120;
+constexpr std::size_t modelFadingOut = 0x138;
+constexpr std::size_t effectPlaying = 0x34;
+constexpr std::size_t effectTime = 0x3c;
+constexpr std::size_t effectAnimation = 0x90;
+constexpr std::size_t effectFadesIn = 0x44;
+constexpr std::size_t effectFadeSource = 0x18;
+constexpr std::size_t effectFadeCount = 0x48;
+constexpr std::size_t effectWeight = 0x38;
+constexpr std::size_t fadingOutEffect = 0xc;
+constexpr std::size_t fadingOutCount = 0x14;
+constexpr std::size_t animationTime = 0x8;
+constexpr std::size_t animationWeight = 0xc;
+constexpr unsigned fadeTicks = 10;
+
+template <class T>
+T& field(void* object, std::size_t offset)
+{
+    return *reinterpret_cast<T*>(static_cast<std::byte*>(object) + offset);
+}
+
+// Each effect's time at the last two ticks it was posed at, to tell an animation playing on (at any rate: the time it
+// moved per tick at the last step) from one held, looping back or just started (posed as it is).
+struct EffectHistory {
+    std::uint64_t step = 0;
+    float time = 0;
+    float previous = -1;
+};
+std::unordered_map<void*, EffectHistory> histories;
+
+// How far into the current tick the client is, from 0 to just under 1; 0 if unknown.
+float tick_fraction()
+{
+    const std::int64_t last = stepTime.load();
+    if (last == 0 || millisecondsPerFrame == nullptr || *millisecondsPerFrame == 0) {
+        return 0;
+    }
+    LARGE_INTEGER now;
+    QueryPerformanceCounter(&now);
+    const double elapsed = static_cast<double>(now.QuadPart - last) * 1000.0 / static_cast<double>(frequency().QuadPart);
+    return static_cast<float>(std::clamp(elapsed / *millisecondsPerFrame, 0.0, 0.999));
+}
+
+// Our ModelInstance::updateCalSkeleton, called where ModelInstance::render poses the model before drawing it: between
+// ticks, poses it at the time between them. Each playing animation that moved on at the last step is set ahead by the
+// fraction of a tick since, at the rate it moved (up to 4 frames a tick), and crossfading animations' weights partway
+// to the next tick's; the skeleton is posed (by IMVU's), and the times and weights put back, with the skeleton marked to
+// be posed again, so everything else sees the tick's pose.
+void __fastcall updateCalSkeleton(void* model, void* /*edx*/)
+{
+    const float fraction = tick_fraction();
+    if (fraction < 0.01f || secondsPerFrame == nullptr) {
+        originalUpdateCalSkeleton(model);
+        return;
+    }
+    const std::uint64_t tick = steps.load();
+    const float frame = *secondsPerFrame;
+
+    struct Saved {
+        float* value;
+        float was;
+    };
+    Saved saved[128];
+    std::size_t count = 0;
+    const auto set = [&](float& value, float to) {
+        saved[count++] = {&value, value};
+        value = to;
+    };
+    void* head = field<void*>(model, modelEffects);
+    for (void* node = head != nullptr ? field<void*>(head, 0) : nullptr; node != nullptr && node != head && count < 126;
+         node = field<void*>(node, 0)) {
+        void* effect = field<void*>(node, 8);
+        void* animation = effect != nullptr ? field<void*>(effect, effectAnimation) : nullptr;
+        if (animation == nullptr) {
+            continue;
+        }
+        const unsigned faded = field<std::uint32_t>(effect, effectFadeCount);
+        if (crossfades && field<std::uint8_t>(effect, effectFadesIn) != 0 && field<std::int32_t>(effect, effectFadeSource) != 0
+            && faded >= 1 && faded < fadeTicks) {
+            set(field<float>(animation, animationWeight),
+                (static_cast<float>(faded) + fraction) * 0.1f * field<float>(effect, effectWeight));
+        }
+        if (field<std::uint8_t>(effect, effectPlaying) == 0) {
+            continue;
+        }
+        const float time = field<float>(effect, effectTime);
+        EffectHistory& history = histories[effect];
+        if (history.step != tick) {
+            history.previous = history.step + 1 == tick ? history.time : -1;
+            history.time = time;
+            history.step = tick;
+        }
+        const float perTick = (history.time - history.previous) / static_cast<float>(stepTicks.load());
+        if (history.previous < 0 || perTick <= frame * 0.01f || perTick > frame * 4) {
+            continue; // held, looped back, or just started: as it is
+        }
+        set(field<float>(animation, animationTime), time + fraction * perTick);
+    }
+    void* fadingHead = crossfades ? field<void*>(model, modelFadingOut) : nullptr;
+    for (void* node = fadingHead != nullptr ? field<void*>(fadingHead, 0) : nullptr;
+         node != nullptr && node != fadingHead && count < 127; node = field<void*>(node, 0)) {
+        void* effect = field<void*>(node, fadingOutEffect);
+        void* animation = effect != nullptr ? field<void*>(effect, effectAnimation) : nullptr;
+        const unsigned faded = field<std::uint32_t>(node, fadingOutCount);
+        if (animation != nullptr && faded >= 1 && faded < fadeTicks) {
+            set(field<float>(animation, animationWeight),
+                (1.0f - (static_cast<float>(faded) + fraction) * 0.1f) * field<float>(effect, effectWeight));
+        }
+    }
+    if (count == 0) {
+        originalUpdateCalSkeleton(model);
+        return;
+    }
+    field<std::uint8_t>(model, modelPoseDirty) = 1;
+    originalUpdateCalSkeleton(model);
+    for (std::size_t i = count; i-- > 0;) {
+        *saved[i].value = saved[i].was;
+    }
+    field<std::uint8_t>(model, modelPoseDirty) = 1;
+
+    // Forget effects not posed for a while (they may be gone).
+    if (histories.size() > 1024) {
+        std::erase_if(histories, [tick](const auto& entry) { return entry.second.step + 30 < tick; });
+    }
+}
+
+// The camera (SceneWindow.dll's CameraController::step, at each tick): each of these values moves a share k of the way
+// to its target (k is 0.4): the camera's position (+0x118, 3 floats, to +0xf0; panning moves the target), and its
+// orbit angles and zoom, in degrees (+0x140, 3 floats, to +0x134; the difference taken between -180 and 180).
+// A fraction f of a tick on, the same easing has moved it 1 - (1 - k)^f of the way: exactly, nothing guessed. The point
+// the camera looks at (+0x160, 3 floats) is the position plus an offset (setCameraFocus, at each tick), so it moves with
+// the position.
+constexpr std::size_t cameraPosition = 0x118;
+constexpr std::size_t cameraPositionTarget = 0xf0;
+constexpr std::size_t cameraAngles = 0x140;
+constexpr std::size_t cameraAnglesTarget = 0x134;
+constexpr std::size_t cameraFocus = 0x160;
+// Nonzero: the camera shows a fixed view ("take"), not eased.
+constexpr std::size_t cameraTake = 0xec;
+
+float degrees180(float degrees)
+{
+    float wrapped = std::fmod(degrees + 180.0f, 360.0f);
+    if (wrapped < 0) {
+        wrapped += 360.0f;
+    }
+    return wrapped - 180.0f;
+}
+
+// Our CameraController::getCameraTransform: between ticks, the camera where its easing has brought it since the tick.
+// The eased values are set ahead for IMVU's code to make the transform from, then put back. __fastcall takes ECX like
+// __thiscall, and like it the callee pops the stack argument.
+void* __fastcall getCameraTransform(void* camera, void* /*edx*/, void* result)
+{
+    const float fraction = tick_fraction();
+    if (fraction < 0.01f || field<std::int32_t>(camera, cameraTake) != 0) {
+        return originalGetCameraTransform(camera, result);
+    }
+    const float moved = 1.0f - std::pow(1.0f - cameraEasing, fraction);
+    float saved[9];
+    for (int i = 0; i < 3; ++i) {
+        float& position = field<float>(camera, cameraPosition + i * 4);
+        float& focus = field<float>(camera, cameraFocus + i * 4);
+        saved[i] = position;
+        saved[6 + i] = focus;
+        const float step = (field<float>(camera, cameraPositionTarget + i * 4) - position) * moved;
+        position += step;
+        focus += step;
+        float& angle = field<float>(camera, cameraAngles + i * 4);
+        saved[3 + i] = angle;
+        angle = degrees180(angle + degrees180(field<float>(camera, cameraAnglesTarget + i * 4) - angle) * moved);
+    }
+    void* transform = originalGetCameraTransform(camera, result);
+    for (int i = 0; i < 3; ++i) {
+        field<float>(camera, cameraPosition + i * 4) = saved[i];
+        field<float>(camera, cameraAngles + i * 4) = saved[3 + i];
+        field<float>(camera, cameraFocus + i * 4) = saved[6 + i];
+    }
+    return transform;
+}
+
+// Particles (SceneWindow.dll's ParticleSystem::step, at each tick) move by their velocity times the tick's time
+// (PContextActionList_t::TimeStep, millisecondsPerFrame / 1000 seconds a tick, then Move). Between ticks, they're drawn
+// moved on by their velocity times the time since the tick, then moved back the same way. (ParticleSystem::render
+// sorts the particles by distance before drawing them, reordering them in memory: each is moved back by its own
+// velocity, wherever it now is.) A ParticleSystem's particle group is at +0x28 (what ParticleSystem::render reads
+// them from).
+constexpr std::size_t particleGroup = 0x28;
+
+void __fastcall renderParticles(void* system, void* /*edx*/, void* renderManager, const void* a, const void* b)
+{
+    const float fraction = tick_fraction();
+    void* group = static_cast<std::byte*>(system) + particleGroup;
+    float* particles = nullptr;
+    unsigned stride = 0, position = 0, positionB = 0, size = 0, velocity = 0, velocityB = 0, color = 0, alpha = 0,
+             age = 0, up = 0, rotationalVelocity = 0, upB = 0, mass = 0, data = 0;
+    // As ParticleSystem::render does: no particles without a group. The library reports errors by throwing; then the
+    // particles are drawn as they are.
+    const auto particle_pointer = [&] {
+        try {
+            if (getGroupCount(group) != 0) {
+                return getParticlePointer(group, particles, stride, position, positionB, size, velocity, velocityB,
+                                          color, alpha, age, up, rotationalVelocity, upB, mass, data);
+            }
+        } catch (...) {
+        }
+        return 0u;
+    };
+    const unsigned count = fraction >= 0.01f ? particle_pointer() : 0;
+    if (count == 0 || particles == nullptr || stride == 0) {
+        originalRenderParticles(system, renderManager, a, b);
+        return;
+    }
+    const float seconds = fraction * static_cast<float>(*millisecondsPerFrame) / 1000.0f;
+    const auto move = [&](float by) {
+        for (unsigned i = 0; i < count; ++i) {
+            float* particle = particles + std::size_t{i} * stride;
+            for (unsigned axis = 0; axis < 3; ++axis) {
+                particle[position + axis] += particle[velocity + axis] * by;
+            }
+        }
+    };
+    move(seconds);
+    originalRenderParticles(system, renderManager, a, b);
+    if (particle_pointer() == count && particles != nullptr) { // the same particles, maybe reordered
+        move(-seconds);
+    }
+}
+
+// Replaces the vtable entry at slot, if it holds expected; returns whether it did.
+bool replace(void** slot, void* expected, void* replacement)
+{
+    if (*slot != expected) {
+        return false;
+    }
+    DWORD protection = 0;
+    if (!VirtualProtect(slot, sizeof(void*), PAGE_READWRITE, &protection)) {
+        return false;
+    }
+    *slot = replacement;
+    VirtualProtect(slot, sizeof(void*), protection, &protection);
+    FlushInstructionCache(GetCurrentProcess(), slot, sizeof(void*));
+    return true;
+}
+
+// Redirects the call instruction (E8, then the target relative to the next instruction) at site, if it calls
+// expected; returns whether it did.
+bool redirect_call(std::byte* site, void* expected, void* replacement)
+{
+    std::int32_t offset = 0;
+    std::memcpy(&offset, site + 1, sizeof(offset));
+    if (site[0] != std::byte{0xe8} || site + 5 + offset != expected) {
+        return false;
+    }
+    const auto replacementOffset = static_cast<std::int32_t>(static_cast<std::byte*>(replacement) - (site + 5));
+    DWORD protection = 0;
+    if (!VirtualProtect(site, 5, PAGE_EXECUTE_READWRITE, &protection)) {
+        return false;
+    }
+    std::memcpy(site + 1, &replacementOffset, sizeof(replacementOffset));
+    VirtualProtect(site, 5, protection, &protection);
+    FlushInstructionCache(GetCurrentProcess(), site, 5);
+    return true;
+}
+
+void install_step(HMODULE sceneWindow)
+{
+    auto* vtable = reinterpret_cast<void**>(GetProcAddress(sceneWindow, "??_7SceneViewer@@6BIFrameListener@@@"));
+    auto* stepMethod = GetProcAddress(sceneWindow, "?step@SceneViewer@@UAEXXZ");
+    // IFrameListener's vtable: its destructor, then step (what stepFrame calls).
+    if (vtable == nullptr || stepMethod == nullptr) {
+        log::info("scene hooks: SceneViewer's vtable or step not found");
+        return;
+    }
+    originalStep = reinterpret_cast<Method>(stepMethod);
+    if (!replace(&vtable[1], reinterpret_cast<void*>(stepMethod), reinterpret_cast<void*>(&step))) {
+        log::info("scene hooks: SceneViewer's vtable doesn't hold its step");
+        return;
+    }
+    stepHooked = true;
+}
+
+void install_pose(HMODULE sceneWindow)
+{
+    HMODULE zero = GetModuleHandleW(L"zero.dll");
+    millisecondsPerFrame =
+        zero == nullptr ? nullptr : reinterpret_cast<const unsigned*>(GetProcAddress(zero, "?millisecondsPerFrame@@3IB"));
+    secondsPerFrame = zero == nullptr ? nullptr : reinterpret_cast<const float*>(GetProcAddress(zero, "?secondsPerFrame@@3MB"));
+    auto* render = reinterpret_cast<std::byte*>(GetProcAddress(
+        sceneWindow, "?render@ModelInstance@@QAEXPAVRenderManager@@ABVCullingFrustum@@ABVTransform@@2ABUFogState@@"
+                     "ABULightState@@_NAAVSceneStats@@ABVRenderSettings@@@Z"));
+    auto* update = GetProcAddress(sceneWindow, "?updateCalSkeleton@ModelInstance@@AAEXXZ");
+    if (millisecondsPerFrame == nullptr || secondsPerFrame == nullptr || render == nullptr || update == nullptr) {
+        log::info("scene hooks: ModelInstance::render, updateCalSkeleton or zero.dll's frame lengths not found");
+        return;
+    }
+    // ModelInstance::render poses the model first: its 5th instruction, at +0x12, calls updateCalSkeleton.
+    originalUpdateCalSkeleton = reinterpret_cast<Method>(update);
+    if (!redirect_call(render + 0x12, reinterpret_cast<void*>(update), reinterpret_cast<void*>(&updateCalSkeleton))) {
+        log::info("scene hooks: ModelInstance::render doesn't call updateCalSkeleton where expected");
+        return;
+    }
+    poseHooked = true;
+
+    // Crossfades: ModelInstance::step's instructions that count a fade in (cmp dword ptr [eax + 0x48], 10) and a fade
+    // out (cmp dword ptr [ebx + 0x14], 10), where expected.
+    auto* step = reinterpret_cast<const std::byte*>(GetProcAddress(sceneWindow, "?step@ModelInstance@@QAEXABVFrameInterval@@@Z"));
+    constexpr std::uint8_t fadeIn[] = {0x83, 0x78, 0x48, 0x0a};
+    constexpr std::uint8_t fadeOut[] = {0x83, 0x7b, 0x14, 0x0a};
+    crossfades = step != nullptr && std::memcmp(step + 0xc6, fadeIn, sizeof(fadeIn)) == 0
+                 && std::memcmp(step + 0xf1, fadeOut, sizeof(fadeOut)) == 0;
+    if (!crossfades) {
+        log::info("scene hooks: ModelInstance::step's crossfades not as expected; crossfades at the client's rate");
+    }
+}
+
+void install_particles(HMODULE sceneWindow)
+{
+    HMODULE particleLib = GetModuleHandleW(L"ParticleLib.dll");
+    getParticlePointer = particleLib == nullptr
+                             ? nullptr
+                             : reinterpret_cast<GetParticlePointer>(GetProcAddress(
+                                   particleLib, "?GetParticlePointer@PContextParticleGroup_t@PAPI@@QAEIAAPAMAAI1111111111111@Z"));
+    getGroupCount = particleLib == nullptr ? nullptr
+                                           : reinterpret_cast<GetGroupCount>(GetProcAddress(
+                                                 particleLib, "?GetGroupCount@PContextParticleGroup_t@PAPI@@QAEIXZ"));
+    auto* modelRender = reinterpret_cast<std::byte*>(GetProcAddress(
+        sceneWindow, "?render@ModelInstance@@QAEXPAVRenderManager@@ABVCullingFrustum@@ABVTransform@@2ABUFogState@@"
+                     "ABULightState@@_NAAVSceneStats@@ABVRenderSettings@@@Z"));
+    auto* render = GetProcAddress(sceneWindow, "?render@ParticleSystem@@QAEXPAVRenderManager@@ABVTransform@@1@Z");
+    if (getParticlePointer == nullptr || getGroupCount == nullptr || modelRender == nullptr || render == nullptr
+        || millisecondsPerFrame == nullptr) {
+        log::info("scene hooks: ParticleLib's GetParticlePointer, ModelInstance::render or ParticleSystem::render not found");
+        return;
+    }
+    // ModelInstance::render draws its particle systems with a call at +0x114.
+    originalRenderParticles = reinterpret_cast<RenderParticles>(render);
+    if (!redirect_call(modelRender + 0x114, reinterpret_cast<void*>(render), reinterpret_cast<void*>(&renderParticles))) {
+        log::info("scene hooks: ModelInstance::render doesn't call ParticleSystem::render where expected");
+        return;
+    }
+    particlesHooked = true;
+}
+
+void install_camera(HMODULE sceneWindow)
+{
+    auto* vtable = reinterpret_cast<void**>(GetProcAddress(sceneWindow, "??_7CameraController@@6B@"));
+    auto* get = GetProcAddress(sceneWindow, "?getCameraTransform@CameraController@@UBE?AVTransform@@XZ");
+    auto* step = reinterpret_cast<const std::byte*>(GetProcAddress(sceneWindow, "?step@CameraController@@UAEXXZ"));
+    if (vtable == nullptr || get == nullptr || step == nullptr) {
+        log::info("scene hooks: CameraController's vtable, getCameraTransform or step not found");
+        return;
+    }
+    // The easing share: a float constant CameraController::step reads (at SceneWindow.dll's 0x100cb934 when loaded at
+    // its preferred address), checked to be referenced by step's code and to be the expected 0.4.
+    const auto* base = reinterpret_cast<const std::byte*>(sceneWindow);
+    const auto* easing = reinterpret_cast<const float*>(base + 0xcb934);
+    const auto address = static_cast<std::uint32_t>(reinterpret_cast<std::uintptr_t>(easing));
+    bool referenced = false;
+    for (std::size_t i = 0; i + 4 <= 0x700 && !referenced; ++i) {
+        referenced = std::memcmp(step + i, &address, 4) == 0;
+    }
+    if (!referenced || *easing != 0.4f) {
+        log::info("scene hooks: CameraController::step's easing not where expected");
+        return;
+    }
+    cameraEasing = *easing;
+    // CameraController's vtable: getCameraTransform is its 6th entry.
+    originalGetCameraTransform = reinterpret_cast<GetTransform>(get);
+    if (!replace(&vtable[5], reinterpret_cast<void*>(get), reinterpret_cast<void*>(&getCameraTransform))) {
+        log::info("scene hooks: CameraController's vtable doesn't hold its getCameraTransform");
+        return;
+    }
+    cameraHooked = true;
+}
+
+} // namespace
+
+void install()
+{
+    static std::once_flag once;
+    std::call_once(once, [] {
+        HMODULE sceneWindow = GetModuleHandleW(L"SceneWindow.dll");
+        if (sceneWindow == nullptr) {
+            log::info("scene hooks: SceneWindow.dll not loaded");
+            return;
+        }
+        install_step(sceneWindow);
+        if (stepHooked) { // drawing between ticks needs the ticks
+            install_pose(sceneWindow);
+            install_particles(sceneWindow);
+            install_camera(sceneWindow);
+        }
+        log::info("scene hooks: ticks {}, posing between ticks {}, particles between ticks {}, camera between ticks {}",
+                  stepHooked ? "recorded" : "not recorded", poseHooked ? "on" : "off", particlesHooked ? "on" : "off",
+                  cameraHooked ? "on" : "off");
+    });
+}
+
+bool installed() { return stepHooked; }
+
+std::int64_t lastStep() { return stepTime; }
+
+} // namespace t5de::scene_hooks

adapter/scene_hooks.hpp

@@ -0,0 +1,30 @@
+#pragma once
+// Our hooks into SceneWindow.dll's scene code, installed once, the first time an engine is made. Each replaces where
+// IMVU's code calls a function (an entry of an exported vtable, or a call instruction's target), after checking it
+// calls the function expected, with ours, which calls IMVU's: nothing else is changed, and nothing is if a check
+// fails (an unexpected SceneWindow.dll: the hooks' features are then off, and the client runs as it would without).
+//
+// - SceneViewer::step (the IFrameListener vtable): the client's tick, 30 times a second (SceneWindow.dll's runApp
+//   calls stepFrame, which steps each scene viewer). Recorded, for the frame pacer and for drawing between ticks.
+// - ModelInstance::render's call to updateCalSkeleton, which poses the model before it's drawn: between ticks, the
+//   model is posed at the time between them (its animations' times moved on by the fraction of a frame since the
+//   tick; Cal3D poses between keyframes) and crossfading animations' weights partway to the next tick's, then its
+//   animations are put back as they were.
+// - ModelInstance::render's call to ParticleSystem::render: between ticks, particles drawn moved on by their velocity
+//   times the time since the tick (as the particle library moves them at each tick), then put back.
+// - CameraController::getCameraTransform (its exported vtable): between ticks, the camera where its easing toward its
+//   target has brought it since the tick.
+
+#include <cstdint>
+
+namespace t5de::scene_hooks {
+
+void install();
+
+// Whether the hooks are installed.
+[[nodiscard]] bool installed();
+
+// When a scene viewer last stepped (QueryPerformanceCounter), or 0 if none has.
+[[nodiscard]] std::int64_t lastStep();
+
+} // namespace t5de::scene_hooks

docs/architecture.md

@@ -19,13 +19,17 @@ Direct3D 11 backend is built for now.
 
 ## How they talk
 
-`include/t5de/protocol/` is shared by both builds, so it uses fixed-size fields only. For each engine the client
+`include/t5de/protocol/` is shared by both builds, so it uses fixed-size fields only. One renderer process draws all
-creates (each 3D view, and again on room changes), the adapter:
+of a client's 3D views, with one Direct3D device. The adapter starts `t5de-renderer.exe --control <name> --parent
+<pid>` from its own folder with the first engine (again with the next one if it has exited), and creates a control
+section (`Local\t5de-renderer-<pid>`, `protocol::Control`). For each engine the client creates (each 3D view: Home's
+avatar, the room, ...), the adapter:
 
-1. creates a shared memory channel (`Local\t5de-renderer-<pid>-<n>`) holding IMVU's window handle and a frame
+1. creates a shared memory channel (`<control name>-<n>`) holding IMVU's window handle, and its events;
-   count, and an event set after each frame;
+2. counts it in the control section and sets `<control name>-attach`, and the renderer process opens it, with a
-2. starts `t5de-renderer.exe --channel <name> --parent <pid>` from its own folder;
+   child window, swap chain and scene of its own;
-3. sets the event at each `endRendering`, and sets `closing` when the engine is destroyed.
+3. sends each frame's commands through it, and sets `closing` when the engine is destroyed (the renderer process
+   then closes the view).
 
 The renderer process draws into a child window it creates inside the client's window: window handles are valid in
 every process, and Windows allows child windows across processes (Chrome's GPU process works this way). The child
@@ -34,13 +38,13 @@ compositor shows above the client's own drawing. The client's window must not cl
 which the client creates it with, so the adapter clears it):
 the client draws a frame when its window gets `WM_PAINT` (`SceneWindow.dll` invalidates it, and draws on the paint),
 and a window entirely covered by clipped children never gets one, so the client fell to about one frame a second.
-The process exits when the engine is destroyed or the client exits.
+The process exits when the client exits.
 
 ## Status
 
 The renderer process draws the whole 3D view in place of the client's renderer (the full view, the default). The
-adapter still forwards everything to the client's Direct3D renderer (state and resources: it draws render targets and
+adapter still forwards state and resources to the client's Direct3D renderer (as a fallback, and for reading textures
-reads textures back, which the renderer process doesn't yet), but while the renderer process covers the whole view
+back, which the renderer process doesn't do), but while the renderer process covers the whole view
 (`Channel::covering`, set after each frame it presents) the client's renderer skips its clears and draws, decided per
 frame; it draws again as soon as the renderer process stops covering the view (the view changes, or the process
 fails). Checked in rooms with screenshots of both renderers side by side (the compare view): they match, particles
@@ -70,8 +74,35 @@ For comparing while the client runs (development):
   the right half visibly trails, mostly from compositing another process's window over Remote Desktop. The
   renderer process draws only the newest frame when it falls behind, and keeps at most one frame queued.
 
-Next: render targets (offscreen frames are skipped, and drawn by the client's renderer), and one renderer process for
+Render targets are the renderer process's too, while it runs: the adapter makes its own (`PixmapTarget`,
-all of a client's views.
+`TextureTarget`), the renderer process draws frames into them (textures in the window's formats, so the same
+pipelines draw both), and a pixmap target's frame is read back through the channel (`Channel::readSize`, the
+`-read` event) into its Pixmap, as the client's renderer does, synchronously. A read back must fit in the channel's
+data area (16 MB: the client's 2048x2048 snapshots just do). Checked with outfit pictures and photos.
+
+Higher frame rates (in progress): the client steps its scene 30 times a second (SceneWindow.dll's `runApp`), and
+draws a window's frame on WM_PAINT. The adapter's frame pacer (`adapter/frame_pacer.cpp`) asks the 3D windows to
+repaint between steps at the user's rate (T5DE's settings, Graphics: `HKCU\Software\T5DE\Graphics\FrameRate`), only
+leaving the client idle time, and never holding up the next step: the adapter records the client's steps by replacing
+`SceneViewer::step`'s entry in its exported vtable (`adapter/scene_hooks.cpp`, after checking it), and a frame that
+wouldn't end before the step is due isn't asked for (a late step makes the client catch up two at once: hitches).
+While the renderer process covers the view, the client's renderer doesn't begin or present frames (its present can
+wait for the display). The renderer process takes each view's chunks on a thread of its own as they come, so the
+client never waits for drawing; it draws the newest frame. On a PC with a graphics card a room frame takes the client
+about 2.5 ms (the client's own work: posing avatars, which it does every frame, and walking the scene), and the
+renderer process about 0.5 ms.
+
+Between steps, the frames show the scene at the time between them (`adapter/scene_hooks.cpp`): avatars posed with
+their animations a fraction of a frame on (ModelInstance::render's call to updateCalSkeleton, redirected; Cal3D poses
+between keyframes; animations held, looping or starting are posed as they are), and the camera where its easing (a
+share 0.4 of the way to its target per step) has brought it, `1 - 0.6^fraction` of the way (CameraController's
+getCameraTransform, in its exported vtable). Nothing is predicted: both are what the next step would give, at the
+fraction; and particles moved on by their velocity times the time since the step (ModelInstance::render's call to
+ParticleSystem::render, redirected), then back by the same (the client sorts them by distance as it draws them, which
+reorders them); and crossfades between animations (weights ramped in tenths over 10 steps by ModelInstance::step),
+partway to the next step's weights. Still at 30 per second: gaze (heads turning to whoever's talking).
+
+Next: sharing pipelines between views (each view's scene compiles its own).
 
 ## Building
 

docs/interface.md

@@ -83,8 +83,14 @@ virtual methods, IMVU's code only uses an engine through helpers that call its v
   practice: clamped to it they look much smaller and fainter than the client's.
 - Fog: linear table fog, start 1 and end 50 in rooms. Fogging by view depth matches the client; by depth buffer
   value (as Direct3D 9's documentation suggests for projections like IMVU's) doesn't.
-- Still unknown: render targets, `CompoundRegion` (texture update
+- Render targets (`include/t5de/imvu/resources.hpp`): the engine returns each as a pointer into its own object;
-  regions), `newRenderTexture`'s flags, and what `beginRendering` does with a null target. MSVC orders overloaded
+  the client calls only their interfaces' methods and passes them to `beginRendering`. A pixmap render target has one
+  method, `getPixmap()` (a Pixmap returned by value), with the pixels of the last frame drawn into it (IMVU's engine
+  copies them from the GPU at the frame's end). A texture render target has `area()` (always 0, 0, 1, 1) and
+  `texture()` (a new texture object for what's been drawn, as a shared pointer returned by value). The client uses
+  pixmap targets for snapshots (`takeSnapshot`: outfit pictures, the photo widget), 2048x2048; texture targets
+  haven't been seen used. Pixmap is 28 bytes; pixmap.dll exports its constructors and destructor.
+- Still unknown: `CompoundRegion` (texture update regions), `newRenderTexture`'s flags, and what `beginRendering` does with a null target. MSVC orders overloaded
   virtual methods unlike their declaration, so our declarations never overload virtual methods.
 - Memory crosses the boundary only through virtual methods (destructors, control blocks), so each side can use its
   own C runtime. Exceptions must not cross it.

include/t5de/imvu/resources.hpp

@@ -6,6 +6,7 @@
 
 #include <cstdint>
 
+#include "t5de/imvu/shared_ptr.hpp"
 #include "t5de/imvu/types.hpp"
 
 namespace t5de::imvu {
@@ -39,8 +40,36 @@ public:
     virtual Pixmap* read(Pixmap* result) const = 0;
 };
 
-class PiRenderTarget;
+// What frames can be drawn into besides the window (PiRenderEngine::beginRendering). IMVU's Direct3D engine returns
-class PiPixmapRenderTarget;
+// each as a pointer into its own object (single inheritance from these, so the PiRenderTarget pointer is the same).
-class PiTextureRenderTarget;
+class PiRenderTarget {
+public:
+    virtual ~PiRenderTarget() = default;
+};
+
+// Drawn into, then read: after each frame drawn into it, getPixmap() has its pixels (IMVU's engine copies them at the
+// end of the frame). Used for snapshots: outfit pictures, the photo widget.
+class PiPixmapRenderTarget : public PiRenderTarget {
+public:
+    // Constructs a copy of the pixels in result (a Pixmap returned by value).
+    virtual Pixmap* getPixmap(Pixmap* result) const = 0;
+};
+
+// The part of the texture the target covers, in texture coordinates. IMVU's Direct3D engine always returns 0, 0, 1, 1.
+struct TextureArea {
+    float left;
+    float top;
+    float right;
+    float bottom;
+};
+
+// Drawn into, then drawn with as a texture.
+class PiTextureRenderTarget : public PiRenderTarget {
+public:
+    // Constructs the area in result (returned by value).
+    virtual TextureArea* area(TextureArea* result) const = 0;
+    // Constructs, in result, a new texture object for what's been drawn (a shared pointer returned by value).
+    virtual SharedPtr<PiRenderTexture>* texture(SharedPtr<PiRenderTexture>* result) const = 0;
+};
 
 } // namespace t5de::imvu

include/t5de/protocol/channel.hpp

@@ -7,7 +7,19 @@
 
 namespace t5de::protocol {
 
-inline constexpr std::uint32_t version = 3;
+inline constexpr std::uint32_t version = 5;
+
+// One renderer process draws all of a client's 3D views. A shared memory section the adapter creates once per client
+// process, named by the renderer process's `--control <name>` argument, holds this; each engine's channel is named
+// "<name>-<n>", n counting from 0. Having created a channel, the adapter increments channels and sets the event
+// "<name>-attach" (auto-reset); the renderer process opens the channels it hasn't yet (those already gone are
+// skipped), and stops drawing a channel's view when its engine sets closing. It exits with the client.
+struct Control {
+    std::uint32_t version;
+    std::uint32_t channels; // channels created so far
+};
+
+inline constexpr wchar_t attachEventSuffix[] = L"-attach";
 
 // Which part of the client's 3D view the renderer process draws. Full is the default; the others are for comparing
 // with the client's renderer, which then draws the whole view under the renderer process's window.
@@ -18,10 +30,15 @@ enum class View : std::uint32_t {
     Compare, // over the right half, the left half's scene: both renderers side by side, showing the same
 };
 
-// A shared memory section the adapter creates for each engine, named by the renderer process's `--channel <name>`
+// A shared memory section the adapter creates for each engine (named as Control says): this header, then `capacity`
-// argument: this header, then `capacity` bytes of data. The adapter sends the command stream (commands.hpp) through
+// bytes of data. The adapter sends the command stream (commands.hpp) through
 // it in chunks: it waits for the event "<name>-consumed", writes a chunk and sets "<name>-ready"; the renderer
 // process copies the chunk out and sets "<name>-consumed" (both auto-reset; consumed starts set).
+//
+// A frame drawn into a render target can ask for its pixels back (BeginFrame::readBack): having sent it, the adapter
+// waits for "<name>-read" (auto-reset), which the renderer process sets once it has drawn the frame and written the
+// pixels into the data area (readSize bytes: rows packed, 0xAARRGGBB; 0 if it couldn't). The adapter sends nothing
+// meanwhile, so the data area is free.
 struct Channel {
     std::uint32_t version;
     std::uint32_t closing; // nonzero: the engine is gone, the renderer process should exit
@@ -34,7 +51,7 @@ struct Channel {
     // Set by the renderer process while its window covers the whole 3D view and it's presenting frames: the client's
     // renderer then needn't draw (the adapter skips its draws; it draws again as soon as this is cleared).
     std::uint32_t covering;
-    std::uint32_t reserved;
+    std::uint32_t readSize;
 };
 
 static_assert(sizeof(Channel) == 48, "same layout in 32-bit and 64-bit builds");
@@ -44,5 +61,6 @@ inline constexpr std::uint32_t channelCapacity = 16 * 1024 * 1024;
 
 inline constexpr wchar_t readyEventSuffix[] = L"-ready";
 inline constexpr wchar_t consumedEventSuffix[] = L"-consumed";
+inline constexpr wchar_t readEventSuffix[] = L"-read";
 
 } // namespace t5de::protocol

include/t5de/protocol/commands.hpp

@@ -33,6 +33,7 @@ enum class Op : std::uint16_t {
     DrawTriangles,
     DrawArrays,
     DrawPointSprites,
+    CreateRenderTarget,
 };
 
 struct Header {
@@ -44,9 +45,15 @@ struct Header {
 // Colors are 0xAARRGGBB. Resource ids are the adapter's, per engine; 0 means none.
 
 struct BeginFrame {
-    std::uint32_t offscreen; // drawn into a render target, not the window: nothing to show
+    // The render target drawn into (CreateRenderTarget's id), 0 for the window, or foreignTarget.
+    std::uint32_t target;
+    std::uint32_t readBack; // at the end of the frame, send the render target's pixels back (channel.hpp)
 };
 
+// A frame drawn into a render target of the client renderer's own (made while the renderer process wasn't running):
+// the client renderer draws it, the renderer process only follows its changes.
+inline constexpr std::uint32_t foreignTarget = 0xffffffff;
+
 struct EndFrame {
     std::uint64_t sentAt; // the adapter's QueryPerformanceCounter (the same clock in every process)
 };
@@ -131,6 +138,13 @@ struct DestroyTexture {
     std::uint32_t id;
 };
 
+// A texture frames can be drawn into (BeginFrame), and drawn with like any other; destroyed with DestroyTexture.
+struct CreateRenderTarget {
+    std::uint32_t id;
+    std::uint32_t width;
+    std::uint32_t height;
+};
+
 enum class BufferKind : std::uint32_t {
     Index,  // 16-bit indices
     Vertex, // positions and normals (3 floats), texture coordinates (2 floats) or colors (0xAARRGGBB)

renderer/main.cpp

@@ -1,7 +1,6 @@
-// t5de-renderer.exe: the renderer process. Started by the adapter (t5deRenderEngine.dll) for each of the client's
+// t5de-renderer.exe: the renderer process. Started by the adapter (t5deRenderEngine.dll), one for the whole client:
-// 3D views: receives the client's drawing as commands through a shared memory channel, and draws it with Diligent
+// receives each 3D view's drawing as commands through a shared memory channel of its own, and draws it with Diligent
-// into a child window it places in the client's window (over the right half, for now, to compare with the client's
+// into a child window it places over the view in the client's window.
-// own renderer on the left).
 
 #include <windows.h>
 
@@ -9,12 +8,18 @@
 #include <dxgi.h>
 #include <shellapi.h>
 
+#include <algorithm>
+#include <atomic>
 #include <cstdint>
 #include <cstring>
 #include <filesystem>
 #include <format>
+#include <memory>
+#include <mutex>
+#include <optional>
 #include <string>
 #include <string_view>
+#include <thread>
 #include <vector>
 
 #include "EngineFactoryD3D11.h"
@@ -42,7 +47,7 @@ void DILIGENT_CALL_TYPE diligent_message(DEBUG_MESSAGE_SEVERITY severity, const
 }
 
 struct Arguments {
-    std::wstring channel;
+    std::wstring control;
     DWORD parent = 0;
 };
 
@@ -53,8 +58,8 @@ Arguments parse_arguments()
     wchar_t** argv = CommandLineToArgvW(GetCommandLineW(), &count);
     for (int i = 1; i + 1 < count; i += 2) {
         const std::wstring_view name = argv[i];
-        if (name == L"--channel") {
+        if (name == L"--control") {
-            arguments.channel = argv[i + 1];
+            arguments.control = argv[i + 1];
         } else if (name == L"--parent") {
             arguments.parent = static_cast<DWORD>(std::wcstoul(argv[i + 1], nullptr, 10));
         }
@@ -63,6 +68,8 @@ Arguments parse_arguments()
     return arguments;
 }
 
+std::string narrow(const std::wstring& text) { return std::filesystem::path(text).string(); }
+
 LRESULT CALLBACK window_procedure(HWND window, UINT message, WPARAM wparam, LPARAM lparam)
 {
     if (message == WM_ERASEBKGND) {
@@ -74,71 +81,75 @@ LRESULT CALLBACK window_procedure(HWND window, UINT message, WPARAM wparam, LPAR
 // A child of the client's window. Disabled, so mouse input goes to the client's window under it.
 HWND create_child_window(HWND parent)
 {
-    WNDCLASSEXW windowClass{.cbSize = sizeof(WNDCLASSEXW),
+    static const ATOM windowClass = [] {
+        const WNDCLASSEXW description{.cbSize = sizeof(WNDCLASSEXW),
                                       .lpfnWndProc = window_procedure,
                                       .hInstance = GetModuleHandleW(nullptr),
                                       .hCursor = LoadCursorW(nullptr, IDC_ARROW),
                                       .lpszClassName = L"T5DERenderer"};
-    RegisterClassExW(&windowClass);
+        return RegisterClassExW(&description);
-    return CreateWindowExW(0, windowClass.lpszClassName, L"", WS_CHILD | WS_VISIBLE | WS_DISABLED | WS_CLIPSIBLINGS,
+    }();
-                           0, 0, 1, 1, parent, nullptr, windowClass.hInstance, nullptr);
+    return CreateWindowExW(0, MAKEINTATOM(windowClass), L"", WS_CHILD | WS_VISIBLE | WS_DISABLED | WS_CLIPSIBLINGS, 0,
+                           0, 1, 1, parent, nullptr, GetModuleHandleW(nullptr), nullptr);
 }
 
 // The view, as the adapter set it, or as %TEMP%\t5de-renderer-view.txt says while the client runs (off, split, full
 // or compare): for comparing the renderers without restarting the client. Read every second or so.
 View current_view(View adapter)
 {
-    static View view = adapter;
+    static std::optional<View> view;
     static ULONGLONG checked = 0;
-    if (GetTickCount64() - checked < 1000) {
+    if (GetTickCount64() - checked >= 1000) {
-        return view;
-    }
         checked = GetTickCount64();
-    view = adapter;
+        view.reset();
         wchar_t temp[MAX_PATH + 1];
         const DWORD length = GetTempPathW(MAX_PATH + 1, temp);
-    if (length == 0 || length > MAX_PATH) {
+        const HANDLE file = length == 0 || length > MAX_PATH
-        return view;
+                                ? INVALID_HANDLE_VALUE
-    }
+                                : CreateFileW((std::filesystem::path(temp) / L"t5de-renderer-view.txt").c_str(),
-    const HANDLE file = CreateFileW((std::filesystem::path(temp) / L"t5de-renderer-view.txt").c_str(), GENERIC_READ,
+                                              GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
-                                    FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, nullptr, OPEN_EXISTING,
+                                              nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr);
-                                    FILE_ATTRIBUTE_NORMAL, nullptr);
+        if (file != INVALID_HANDLE_VALUE) {
-    if (file == INVALID_HANDLE_VALUE) {
-        return view;
-    }
             char text[16] = {};
             DWORD read = 0;
             ReadFile(file, text, sizeof(text) - 1, &read, nullptr);
             CloseHandle(file);
             const std::string_view setting(text, std::strcspn(text, "\r\n "));
-    view = setting == "off"       ? View::Off
+            if (setting == "off") {
-           : setting == "split"   ? View::Split
+                view = View::Off;
-           : setting == "full"    ? View::Full
+            } else if (setting == "split") {
-           : setting == "compare" ? View::Compare
+                view = View::Split;
-                                  : adapter;
+            } else if (setting == "full") {
-    return view;
+                view = View::Full;
+            } else if (setting == "compare") {
+                view = View::Compare;
+            }
+        }
+    }
+    return view.value_or(adapter);
 }
 
 // Shader changes to try while the client runs: %TEMP%\t5de-renderer-shaders\fixed_function.hlsl, if there is
-// one, replaces the built-in shaders, and is read again when it changes (checked every second or so).
+// one, replaces the built-in shaders, and is read again when it changes (checked every second or so). Returns the
-void reload_shaders(t5de::Scene& scene)
+// new source when it has changed.
+std::optional<std::string> changed_shaders()
 {
     static ULONGLONG checked = 0;
     static std::filesystem::file_time_type loaded{};
     if (GetTickCount64() - checked < 1000) {
-        return;
+        return std::nullopt;
     }
     checked = GetTickCount64();
     std::error_code error;
     const auto path = std::filesystem::temp_directory_path(error) / L"t5de-renderer-shaders" / L"fixed_function.hlsl";
     const auto modified = std::filesystem::last_write_time(path, error);
     if (error || modified == loaded) {
-        return;
+        return std::nullopt;
     }
     const HANDLE file = CreateFileW(path.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr,
                                     OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr);
     if (file == INVALID_HANDLE_VALUE) {
-        return;
+        return std::nullopt;
     }
     std::string source(static_cast<std::size_t>(std::filesystem::file_size(path, error)), ' ');
     DWORD read = 0;
@@ -146,8 +157,8 @@ void reload_shaders(t5de::Scene& scene)
     CloseHandle(file);
     source.resize(read);
     loaded = modified;
-    scene.setShaderSource(std::move(source));
     info("shaders reloaded from {}", path.string());
+    return source;
 }
 
 // Where the child window goes in the client's 3D view, and where in the 3D view its scene starts (offset).
@@ -170,34 +181,244 @@ Placement placement(HWND parent, View view)
     return place;
 }
 
+// One of the client's 3D views: an engine's channel, and the child window its frames are drawn into. A thread of its
+// own takes the channel's chunks as soon as they come (so the client never waits for drawing) and queues them; the main
+// thread draws the newest complete frame, carrying out older ones' changes without drawing them.
+class ViewWindow {
+public:
+    // Opens the channel named name and sets up its window; null if the channel is gone or it can't.
+    static std::unique_ptr<ViewWindow> open(const std::wstring& name, IEngineFactoryD3D11* factory,
+                                            IRenderDevice* device, IDeviceContext* context,
+                                            const std::optional<std::string>& shaders)
+    {
+        std::unique_ptr<ViewWindow> view(new ViewWindow());
+        view->name_ = narrow(name.substr(name.rfind(L'-') + 1));
+        view->mapping_ = OpenFileMappingW(FILE_MAP_ALL_ACCESS, FALSE, name.c_str());
+        if (view->mapping_ == nullptr) {
+            return nullptr; // its engine is gone already
+        }
+        view->channel_ = static_cast<Channel*>(MapViewOfFile(view->mapping_, FILE_MAP_ALL_ACCESS, 0, 0, 0));
+        view->ready_ = OpenEventW(SYNCHRONIZE, FALSE, (name + t5de::protocol::readyEventSuffix).c_str());
+        view->consumed_ = OpenEventW(EVENT_MODIFY_STATE, FALSE, (name + t5de::protocol::consumedEventSuffix).c_str());
+        view->read_ = OpenEventW(EVENT_MODIFY_STATE, FALSE, (name + t5de::protocol::readEventSuffix).c_str());
+        if (view->channel_ == nullptr || view->ready_ == nullptr || view->consumed_ == nullptr
+            || view->read_ == nullptr || view->channel_->version != t5de::protocol::version) {
+            info("{}: can't open the channel", view->name_);
+            return nullptr;
+        }
+        view->data_ = reinterpret_cast<std::byte*>(view->channel_ + 1);
+        view->clientWindow_ = reinterpret_cast<HWND>(static_cast<std::uintptr_t>(view->channel_->window));
+        view->window_ = create_child_window(view->clientWindow_);
+        if (view->window_ == nullptr) {
+            info("{}: can't create a window in {:#x}", view->name_, view->channel_->window);
+            return nullptr;
+        }
+        SwapChainDesc swapChainDesc;
+        swapChainDesc.ColorBufferFormat = TEX_FORMAT_RGBA8_UNORM; // no sRGB conversion, like the client's renderer
+        swapChainDesc.DepthBufferFormat = TEX_FORMAT_D32_FLOAT;
+        factory->CreateSwapChainD3D11(device, context, swapChainDesc, FullScreenModeDesc{},
+                                      Win32NativeWindow{view->window_}, &view->swapChain_);
+        if (!view->swapChain_) {
+            info("{}: can't create a swap chain", view->name_);
+            return nullptr;
+        }
+        view->scene_ = std::make_unique<t5de::Scene>(device, context, view->swapChain_);
+        if (shaders) {
+            view->scene_->setShaderSource(*shaders);
+        }
+        // Frames read back: the adapter waits for them, sending nothing meanwhile, so the data area is free.
+        ViewWindow* self = view.get();
+        view->scene_->setReadBack([self](std::span<const std::byte> pixels) {
+            const std::size_t size = pixels.size() <= self->channel_->capacity ? pixels.size() : 0;
+            std::memcpy(self->data_, pixels.data(), size);
+            self->channel_->readSize = static_cast<std::uint32_t>(size);
+            SetEvent(self->read_);
+        });
+        QueryPerformanceFrequency(&view->frequency_);
+        view->framesReady_ = CreateEventW(nullptr, FALSE, FALSE, nullptr);
+        if (view->framesReady_ == nullptr) {
+            return nullptr;
+        }
+        view->receiver_ = std::thread([self] { self->receive(); });
+        info("{}: drawing window {:#x}, view {}", view->name_, view->channel_->window,
+             static_cast<int>(view->channel_->view));
+        return view;
+    }
+
+    ViewWindow(const ViewWindow&) = delete;
+    ViewWindow& operator=(const ViewWindow&) = delete;
+
+    ~ViewWindow()
+    {
+        stopping_ = true;
+        if (receiver_.joinable()) {
+            receiver_.join();
+        }
+        if (channel_ != nullptr) {
+            channel_->covering = 0;
+            UnmapViewOfFile(channel_);
+        }
+        scene_.reset();
+        swapChain_.Release();
+        if (window_ != nullptr) {
+            DestroyWindow(window_);
+            info("{}: closed", name_);
+        }
+        for (HANDLE handle : {ready_, consumed_, read_, mapping_, framesReady_}) {
+            if (handle != nullptr) {
+                CloseHandle(handle);
+            }
+        }
+    }
+
+    // Set when complete frames are queued.
+    [[nodiscard]] HANDLE framesReady() const { return framesReady_; }
+    // Its engine is gone.
+    [[nodiscard]] bool closing() const { return channel_->closing != 0; }
+
+    void setShaderSource(const std::string& source) { scene_->setShaderSource(source); }
+
+    // After framesReady: draws the newest complete frame queued. If the client has sent more meanwhile (drawing fell
+    // behind), the older ones' changes are carried out without drawing them.
+    void onFramesReady()
+    {
+        std::vector<std::size_t> ends;
+        {
+            const std::scoped_lock lock(mutex_);
+            if (frameEnds_.empty()) {
+                return;
+            }
+            const std::size_t complete = frameEnds_.back();
+            frame_.assign(received_.begin(), received_.begin() + static_cast<std::ptrdiff_t>(complete));
+            received_.erase(received_.begin(), received_.begin() + static_cast<std::ptrdiff_t>(complete));
+            ends.swap(frameEnds_);
+        }
+        std::size_t begin = 0;
+        for (std::size_t i = 0; i + 1 < ends.size(); ++i) {
+            scene_->execute(std::span(frame_).subspan(begin, ends[i] - begin), 0, false);
+            begin = ends[i];
+            ++skipped_;
+        }
+        const std::size_t complete = ends.back();
+
+        // Follow the client's 3D view's size.
+        const View view = current_view(channel_->view);
+        const auto [place, offset] = placement(clientWindow_, view);
+        const int width = place.right - place.left;
+        const int height = place.bottom - place.top;
+        if (std::memcmp(&place, &placed_, sizeof(RECT)) != 0) {
+            placed_ = place;
+            if (width > 0 && height > 0) {
+                SetWindowPos(window_, nullptr, place.left, place.top, width, height,
+                             SWP_NOZORDER | SWP_NOACTIVATE | SWP_SHOWWINDOW);
+                swapChain_->Resize(static_cast<Uint32>(width), static_cast<Uint32>(height));
+                info("{}: drawing {}x{} at {} in the client's 3D view, its scene from {}", name_, width, height,
+                     place.left, offset);
+            } else {
+                ShowWindow(window_, SW_HIDE);
+                channel_->covering = 0;
+                info("{}: hidden", name_);
+            }
+        }
+        LARGE_INTEGER start, end;
+        QueryPerformanceCounter(&start);
+        const bool covers = view == View::Full && width > 0 && height > 0;
+        if (scene_->execute(std::span(frame_).subspan(begin, complete - begin), offset, width > 0 && height > 0)) {
+            swapChain_->Present(0);
+            channel_->covering = covers ? 1 : 0; // the client's renderer can stop drawing (or must start again)
+            QueryPerformanceCounter(&end);
+            const auto seconds = [this](LONGLONG ticks) {
+                return static_cast<double>(ticks) / static_cast<double>(frequency_.QuadPart);
+            };
+            drawTime_ += seconds(end.QuadPart - start.QuadPart);
+            latency_ += seconds(end.QuadPart - static_cast<LONGLONG>(scene_->lastSentAt()));
+            if (++drawn_ == 300) {
+                info("{}: last 300 frames: {:.1f} ms each to draw and present, {:.1f} ms from the client's end of frame "
+                     "to presented, {} skipped to catch up",
+                     name_, drawTime_ * 1000 / drawn_, latency_ * 1000 / drawn_, skipped_);
+                drawTime_ = 0;
+                latency_ = 0;
+                drawn_ = 0;
+                skipped_ = 0;
+            }
+        }
+    }
+
+private:
+    ViewWindow() = default;
+
+    // The receiver thread: takes each chunk as it comes (after "ready"), lets the adapter send the next, and queues it;
+    // sets framesReady when a frame is complete. Ends when the view closes.
+    void receive()
+    {
+        while (!stopping_ && !closing()) {
+            if (WaitForSingleObject(ready_, 100) != WAIT_OBJECT_0 || closing()) {
+                continue;
+            }
+            const std::uint32_t size = channel_->chunkSize <= channel_->capacity ? channel_->chunkSize : 0;
+            const bool endsFrame = (channel_->chunkFlags & t5de::protocol::chunkEndsFrame) != 0;
+            {
+                const std::scoped_lock lock(mutex_);
+                received_.insert(received_.end(), data_, data_ + size);
+                if (endsFrame) {
+                    frameEnds_.push_back(received_.size());
+                }
+            }
+            SetEvent(consumed_);
+            if (endsFrame) {
+                SetEvent(framesReady_);
+            }
+        }
+        SetEvent(framesReady_); // wakes the main thread to close the view
+    }
+
+    std::string name_;
+    HANDLE mapping_ = nullptr;
+    HANDLE ready_ = nullptr;
+    HANDLE consumed_ = nullptr;
+    HANDLE read_ = nullptr;
+    Channel* channel_ = nullptr;
+    std::byte* data_ = nullptr;
+    HWND clientWindow_ = nullptr;
+    HWND window_ = nullptr;
+    RefCntAutoPtr<ISwapChain> swapChain_;
+    std::unique_ptr<t5de::Scene> scene_;
+    std::vector<std::byte> frame_; // the frames being drawn
+    RECT placed_{};
+
+    std::thread receiver_;
+    std::atomic<bool> stopping_ = false;
+    HANDLE framesReady_ = nullptr;
+    std::mutex mutex_;                  // guards received_ and frameEnds_
+    std::vector<std::byte> received_;   // chunks taken, not yet drawn
+    std::vector<std::size_t> frameEnds_; // where complete frames end in received_
+
+    LARGE_INTEGER frequency_{};
+    double drawTime_ = 0;
+    double latency_ = 0;
+    int drawn_ = 0;
+    int skipped_ = 0;
+};
+
 } // namespace
 
 int WINAPI wWinMain(HINSTANCE, HINSTANCE, PWSTR, int)
 {
     const Arguments arguments = parse_arguments();
-    if (arguments.channel.empty()) {
+    if (arguments.control.empty()) {
         return 2;
     }
-    HANDLE mapping = OpenFileMappingW(FILE_MAP_ALL_ACCESS, FALSE, arguments.channel.c_str());
+    HANDLE mapping = OpenFileMappingW(FILE_MAP_READ, FALSE, arguments.control.c_str());
-    auto* channel = mapping == nullptr ? nullptr
+    const auto* control = mapping == nullptr ? nullptr
-                                       : static_cast<Channel*>(MapViewOfFile(mapping, FILE_MAP_ALL_ACCESS, 0, 0, 0));
+                                             : static_cast<const t5de::protocol::Control*>(
-    HANDLE ready = OpenEventW(SYNCHRONIZE, FALSE, (arguments.channel + t5de::protocol::readyEventSuffix).c_str());
+                                                   MapViewOfFile(mapping, FILE_MAP_READ, 0, 0, 0));
-    HANDLE consumed =
+    HANDLE attach = OpenEventW(SYNCHRONIZE, FALSE, (arguments.control + t5de::protocol::attachEventSuffix).c_str());
-        OpenEventW(EVENT_MODIFY_STATE, FALSE, (arguments.channel + t5de::protocol::consumedEventSuffix).c_str());
     HANDLE parent = OpenProcess(SYNCHRONIZE, FALSE, arguments.parent);
-    if (channel == nullptr || ready == nullptr || consumed == nullptr || parent == nullptr
+    if (control == nullptr || attach == nullptr || parent == nullptr || control->version != t5de::protocol::version) {
-        || channel->version != t5de::protocol::version) {
         return 3;
     }
-    const auto* data = reinterpret_cast<const std::byte*>(channel + 1);
 
-    const HWND clientWindow = reinterpret_cast<HWND>(static_cast<std::uintptr_t>(channel->window));
+    info("started for client {}", arguments.parent);
-    const HWND window = create_child_window(clientWindow);
-    if (window == nullptr) {
-        return 4;
-    }
-
-    info("started for window {:#x}, view {}", channel->window, static_cast<int>(channel->view));
     IEngineFactoryD3D11* factory = GetEngineFactoryD3D11();
     factory->SetMessageCallback(diligent_message);
     RefCntAutoPtr<IRenderDevice> device;
@@ -216,110 +437,50 @@ int WINAPI wWinMain(HINSTANCE, HINSTANCE, PWSTR, int)
             dxgi->Release();
         }
     }
-    RefCntAutoPtr<ISwapChain> swapChain;
-    SwapChainDesc swapChainDesc;
-    swapChainDesc.ColorBufferFormat = TEX_FORMAT_RGBA8_UNORM; // no sRGB conversion, like the client's renderer
-    swapChainDesc.DepthBufferFormat = TEX_FORMAT_D32_FLOAT;
-    factory->CreateSwapChainD3D11(device, context, swapChainDesc, FullScreenModeDesc{}, Win32NativeWindow{window},
-                                  &swapChain);
-    if (!swapChain) {
-        info("can't create a swap chain");
-        return 6;
-    }
-    t5de::Scene scene(device, context, swapChain);
 
-    // Takes the chunk waiting in the channel (after "ready"), and lets the adapter send the next: appends it to
+    std::vector<std::unique_ptr<ViewWindow>> views;
-    // frame, and returns whether it ends a frame.
+    std::uint32_t attached = 0;
-    std::vector<std::byte> frame;
+    std::optional<std::string> shaders; // the override, if any
-    const auto take_chunk = [&] {
+    std::size_t first = 0;              // which view's event is waited on first, taking turns
-        const std::uint32_t size = channel->chunkSize <= channel->capacity ? channel->chunkSize : 0;
-        const bool endsFrame = (channel->chunkFlags & t5de::protocol::chunkEndsFrame) != 0;
-        frame.insert(frame.end(), data, data + size);
-        SetEvent(consumed);
-        return endsFrame;
-    };
-
-    LARGE_INTEGER frequency;
-    QueryPerformanceFrequency(&frequency);
-    double drawTime = 0;
-    double latency = 0;
-    int drawn = 0;
-    int skipped = 0;
-    RECT placed{};
     for (;;) {
-        const HANDLE handles[] = {ready, parent};
+        // New channels: opened in order; those whose engines are gone already are skipped.
-        const DWORD result = MsgWaitForMultipleObjects(2, handles, FALSE, 1000, QS_ALLINPUT);
+        while (attached < control->channels) {
+            if (auto view = ViewWindow::open(std::format(L"{}-{}", arguments.control, attached), factory, device,
+                                             context, shaders)) {
+                views.push_back(std::move(view));
+            }
+            ++attached;
+        }
+        std::erase_if(views, [](const auto& view) { return view->closing(); });
+        if (auto source = changed_shaders()) {
+            shaders = std::move(source);
+            for (const auto& view : views) {
+                view->setShaderSource(*shaders);
+            }
+        }
+
+        std::vector<HANDLE> handles{parent, attach};
+        const std::size_t count = std::min<std::size_t>(views.size(), MAXIMUM_WAIT_OBJECTS - 3);
+        for (std::size_t i = 0; i < count; ++i) {
+            handles.push_back(views[(first + i) % views.size()]->framesReady());
+        }
+        const DWORD result = MsgWaitForMultipleObjects(static_cast<DWORD>(handles.size()), handles.data(), FALSE, 1000,
+                                                       QS_ALLINPUT);
         MSG message;
         while (PeekMessageW(&message, nullptr, 0, 0, PM_REMOVE)) {
             DispatchMessageW(&message);
         }
-        if (result == WAIT_OBJECT_0 + 1 || channel->closing != 0) {
+        if (result == WAIT_OBJECT_0) {
-            break; // the client exited, or the engine is gone
+            break; // the client exited
+        }
+        if (result >= WAIT_OBJECT_0 + 2 && result < WAIT_OBJECT_0 + handles.size()) {
+            const std::size_t index = (first + (result - WAIT_OBJECT_0 - 2)) % views.size();
+            views[index]->onFramesReady();
+            first = (index + 1) % views.size();
         }
-        if (result != WAIT_OBJECT_0 || !take_chunk()) {
-            continue;
     }
 
-        // If the client has sent more frames meanwhile (drawing fell behind), catch up: carry out the older ones'
+    views.clear();
-        // changes without drawing them, and draw only the newest. (An incomplete frame stays for the next round.)
-        std::size_t complete = frame.size();
-        while (WaitForSingleObject(ready, 0) == WAIT_OBJECT_0) {
-            bool ended = take_chunk();
-            while (!ended && WaitForSingleObject(ready, 1000) == WAIT_OBJECT_0) {
-                ended = take_chunk();
-            }
-            if (!ended) {
-                break;
-            }
-            scene.execute(std::span(frame).first(complete), 0, false);
-            frame.erase(frame.begin(), frame.begin() + static_cast<std::ptrdiff_t>(complete));
-            complete = frame.size();
-            ++skipped;
-        }
-
-        // Follow the client's 3D view's size.
-        const auto [place, offset] = placement(clientWindow, current_view(channel->view));
-        const int width = place.right - place.left;
-        const int height = place.bottom - place.top;
-        if (std::memcmp(&place, &placed, sizeof(RECT)) != 0) {
-            placed = place;
-            if (width > 0 && height > 0) {
-                SetWindowPos(window, nullptr, place.left, place.top, width, height,
-                             SWP_NOZORDER | SWP_NOACTIVATE | SWP_SHOWWINDOW);
-                swapChain->Resize(static_cast<Uint32>(width), static_cast<Uint32>(height));
-                info("drawing {}x{} at {} in the client's 3D view, its scene from {}", width, height, place.left, offset);
-            } else {
-                ShowWindow(window, SW_HIDE);
-                channel->covering = 0;
-                info("hidden");
-            }
-        }
-        reload_shaders(scene);
-        LARGE_INTEGER start, end;
-        QueryPerformanceCounter(&start);
-        const bool covers = current_view(channel->view) == View::Full && width > 0 && height > 0;
-        if (scene.execute(std::span(frame).first(complete), offset, width > 0 && height > 0)) {
-            swapChain->Present(0);
-            channel->covering = covers ? 1 : 0; // the client's renderer can stop drawing (or must start again)
-            QueryPerformanceCounter(&end);
-            drawTime += static_cast<double>(end.QuadPart - start.QuadPart) / static_cast<double>(frequency.QuadPart);
-            latency += static_cast<double>(end.QuadPart - static_cast<LONGLONG>(scene.lastSentAt())) /
-                       static_cast<double>(frequency.QuadPart);
-            if (++drawn == 300) {
-                info("last 300 frames: {:.1f} ms each to draw and present, {:.1f} ms from the client's end of frame to "
-                     "presented, {} skipped to catch up",
-                     drawTime * 1000 / drawn, latency * 1000 / drawn, skipped);
-                drawTime = 0;
-                latency = 0;
-                drawn = 0;
-                skipped = 0;
-            }
-        }
-        frame.erase(frame.begin(), frame.begin() + static_cast<std::ptrdiff_t>(complete));
-    }
-
-    channel->covering = 0;
     info("exiting");
-    DestroyWindow(window);
     return 0;
 }

renderer/scene.cpp

@@ -102,12 +102,7 @@ bool Scene::execute(std::span<const std::byte> commands, int offsetX, bool draw)
         switch (command->op()) {
         case Op::BeginFrame:
             if (const auto begin = command->body<protocol::BeginFrame>()) {
-                onscreen_ = begin->offscreen == 0 && draw_;
+                beginFrame(*begin);
-                if (onscreen_) {
-                    ITextureView* target = swapChain_->GetCurrentBackBufferRTV();
-                    context_->SetRenderTargets(1, &target, swapChain_->GetDepthBufferDSV(),
-                                               RESOURCE_STATE_TRANSITION_MODE_TRANSITION);
-                }
             }
             break;
         case Op::EndFrame:
@@ -115,18 +110,17 @@ bool Scene::execute(std::span<const std::byte> commands, int offsetX, bool draw)
                 lastSentAt_ = end->sentAt;
             }
             drew = drew || onscreen_;
-            onscreen_ = false;
+            endFrame();
             break;
         case Op::Clear:
-            if (const auto clear = command->body<protocol::Clear>(); clear && onscreen_) {
+            if (const auto clear = command->body<protocol::Clear>(); clear && colorView_ != nullptr) {
                 if ((clear->flags & 1) != 0) {
                     float color[4];
                     set_color(color, clear->color);
-                    context_->ClearRenderTarget(swapChain_->GetCurrentBackBufferRTV(), color,
+                    context_->ClearRenderTarget(colorView_, color, RESOURCE_STATE_TRANSITION_MODE_TRANSITION);
-                                                RESOURCE_STATE_TRANSITION_MODE_TRANSITION);
                 }
                 if ((clear->flags & 2) != 0) {
-                    context_->ClearDepthStencil(swapChain_->GetDepthBufferDSV(), CLEAR_DEPTH_FLAG, 1.0f, 0,
+                    context_->ClearDepthStencil(depthView_, CLEAR_DEPTH_FLAG, 1.0f, 0,
                                                 RESOURCE_STATE_TRANSITION_MODE_TRANSITION);
                 }
             }
@@ -184,6 +178,11 @@ bool Scene::execute(std::span<const std::byte> commands, int offsetX, bool draw)
         case Op::UpdateTexture:
             updateTexture(*command);
             break;
+        case Op::CreateRenderTarget:
+            if (const auto create = command->body<protocol::CreateRenderTarget>()) {
+                createRenderTarget(*create);
+            }
+            break;
         case Op::DestroyTexture:
             if (const auto destroy = command->body<protocol::DestroyTexture>()) {
                 textures_.erase(destroy->id);
@@ -201,17 +200,17 @@ bool Scene::execute(std::span<const std::byte> commands, int offsetX, bool draw)
             }
             break;
         case Op::DrawTriangles:
-            if (const auto triangles = command->body<protocol::DrawTriangles>(); triangles && onscreen_) {
+            if (const auto triangles = command->body<protocol::DrawTriangles>(); triangles && colorView_ != nullptr) {
                 drawTriangles(*triangles);
             }
             break;
         case Op::DrawArrays:
-            if (onscreen_) {
+            if (colorView_ != nullptr) {
                 drawArrays(*command);
             }
             break;
         case Op::DrawPointSprites:
-            if (onscreen_) {
+            if (colorView_ != nullptr) {
                 drawPointSprites(*command);
             }
             break;
@@ -232,19 +231,107 @@ void Scene::setProjection(const protocol::SetProjection& projection)
     }
     constants_.viewport[0] = static_cast<float>(projection.width);
     constants_.viewport[1] = static_cast<float>(projection.height);
-    if (!onscreen_) {
+    projectionWidth_ = projection.width;
+    projectionHeight_ = projection.height;
+    setViewport();
+}
+
+void Scene::setViewport()
+{
+    if (colorView_ == nullptr || projectionWidth_ <= 0 || projectionHeight_ <= 0) {
         return;
     }
     // The client's projection covers its whole 3D view; the window shows part of it, from offsetX.
     const SwapChainDesc& window = swapChain_->GetDesc();
     Viewport viewport;
-    viewport.TopLeftX = static_cast<float>(-offsetX_);
+    viewport.TopLeftX = onscreen_ ? static_cast<float>(-offsetX_) : 0.0f;
     viewport.TopLeftY = 0;
-    viewport.Width = static_cast<float>(projection.width);
+    viewport.Width = static_cast<float>(projectionWidth_);
-    viewport.Height = static_cast<float>(projection.height);
+    viewport.Height = static_cast<float>(projectionHeight_);
     viewport.MinDepth = 0;
     viewport.MaxDepth = 1;
-    context_->SetViewports(1, &viewport, window.Width, window.Height);
+    context_->SetViewports(1, &viewport, onscreen_ ? window.Width : target_->width,
+                           onscreen_ ? window.Height : target_->height);
+}
+
+void Scene::beginFrame(const protocol::BeginFrame& begin)
+{
+    onscreen_ = false;
+    target_ = nullptr;
+    colorView_ = nullptr;
+    depthView_ = nullptr;
+    readsBack_ = begin.readBack != 0;
+    if (begin.target == 0) {
+        if (draw_) {
+            onscreen_ = true;
+            colorView_ = swapChain_->GetCurrentBackBufferRTV();
+            depthView_ = swapChain_->GetDepthBufferDSV();
+        }
+    } else if (const auto found = textures_.find(begin.target); found != textures_.end() && found->second.depth) {
+        target_ = &found->second;
+        colorView_ = target_->texture->GetDefaultView(TEXTURE_VIEW_RENDER_TARGET);
+        depthView_ = target_->depth->GetDefaultView(TEXTURE_VIEW_DEPTH_STENCIL);
+    }
+    if (colorView_ != nullptr) {
+        context_->SetRenderTargets(1, &colorView_, depthView_, RESOURCE_STATE_TRANSITION_MODE_TRANSITION);
+        setViewport();
+    }
+}
+
+void Scene::endFrame()
+{
+    if (readsBack_ && readBack_) {
+        readBack_(target_ != nullptr ? readTarget(*target_) : std::span<const std::byte>{});
+    }
+    if (target_ != nullptr) {
+        // Unbound, so it can be drawn with (a texture can't be both).
+        context_->SetRenderTargets(0, nullptr, nullptr, RESOURCE_STATE_TRANSITION_MODE_TRANSITION);
+    }
+    onscreen_ = false;
+    target_ = nullptr;
+    readsBack_ = false;
+    colorView_ = nullptr;
+    depthView_ = nullptr;
+}
+
+std::span<const std::byte> Scene::readTarget(const Texture& target)
+{
+    TextureDesc desc;
+    desc.Name = "Render target read back";
+    desc.Type = RESOURCE_DIM_TEX_2D;
+    desc.Width = target.width;
+    desc.Height = target.height;
+    desc.MipLevels = 1;
+    desc.Format = TEX_FORMAT_RGBA8_UNORM;
+    desc.Usage = USAGE_STAGING;
+    desc.CPUAccessFlags = CPU_ACCESS_READ;
+    RefCntAutoPtr<ITexture> staging;
+    device_->CreateTexture(desc, nullptr, &staging);
+    if (!staging) {
+        return {};
+    }
+    CopyTextureAttribs copy(target.texture, RESOURCE_STATE_TRANSITION_MODE_TRANSITION, staging,
+                            RESOURCE_STATE_TRANSITION_MODE_TRANSITION);
+    context_->CopyTexture(copy);
+    MappedTextureSubresource mapped;
+    context_->MapTextureSubresource(staging, 0, 0, MAP_READ, MAP_FLAG_NONE, nullptr, mapped); // waits for the GPU
+    if (mapped.pData == nullptr) {
+        return {};
+    }
+    // RGBA in memory, to the client's 0xAARRGGBB (BGRA in memory).
+    readPixels_.resize(std::size_t{target.width} * target.height * 4);
+    for (std::uint32_t y = 0; y < target.height; ++y) {
+        const auto* from = static_cast<const std::uint8_t*>(mapped.pData) + y * mapped.Stride;
+        auto* to = reinterpret_cast<std::uint8_t*>(readPixels_.data()) + std::size_t{y} * target.width * 4;
+        for (std::uint32_t x = 0; x < target.width; ++x) {
+            to[x * 4 + 0] = from[x * 4 + 2];
+            to[x * 4 + 1] = from[x * 4 + 1];
+            to[x * 4 + 2] = from[x * 4 + 0];
+            to[x * 4 + 3] = from[x * 4 + 3];
+        }
+    }
+    context_->UnmapTextureSubresource(staging, 0, 0);
+    return readPixels_;
 }
 
 void Scene::setState(const protocol::SetState& state)
@@ -322,6 +409,38 @@ void Scene::createTexture(const protocol::Command& command)
     textures_[create->id] = std::move(texture);
 }
 
+void Scene::createRenderTarget(const protocol::CreateRenderTarget& create)
+{
+    if (create.width == 0 || create.height == 0) {
+        return;
+    }
+    // The window's formats, so the same pipelines draw into both.
+    const SwapChainDesc& window = swapChain_->GetDesc();
+    TextureDesc desc;
+    desc.Name = "Client render target";
+    desc.Type = RESOURCE_DIM_TEX_2D;
+    desc.Width = create.width;
+    desc.Height = create.height;
+    desc.MipLevels = 1;
+    desc.Format = window.ColorBufferFormat;
+    desc.BindFlags = BIND_RENDER_TARGET | BIND_SHADER_RESOURCE;
+    desc.ClearValue.Format = desc.Format;
+    Texture target{.width = create.width, .height = create.height, .flags = 0};
+    device_->CreateTexture(desc, nullptr, &target.texture);
+    TextureDesc depth = desc;
+    depth.Name = "Client render target depth";
+    depth.Format = window.DepthBufferFormat;
+    depth.BindFlags = BIND_DEPTH_STENCIL;
+    depth.ClearValue.Format = depth.Format;
+    depth.ClearValue.DepthStencil.Depth = 1;
+    device_->CreateTexture(depth, nullptr, &target.depth);
+    if (!target.texture || !target.depth) {
+        return;
+    }
+    target.texture->GetDefaultView(TEXTURE_VIEW_SHADER_RESOURCE)->SetSampler(samplers_[0]);
+    textures_[create.id] = std::move(target);
+}
+
 void Scene::updateTexture(const protocol::Command& command)
 {
     const auto update = command.body<protocol::UpdateTexture>();

renderer/scene.hpp

@@ -5,9 +5,11 @@
 
 #include <cstddef>
 #include <cstdint>
+#include <functional>
 #include <span>
 #include <string>
 #include <unordered_map>
+#include <vector>
 
 #include "EngineFactoryD3D11.h"
 #include "RefCntAutoPtr.hpp"
@@ -20,10 +22,14 @@ public:
     Scene(Diligent::IRenderDevice* device, Diligent::IDeviceContext* context, Diligent::ISwapChain* swapChain);
 
     // Carries out one frame's commands. offsetX is where the window starts within the client's 3D view; without draw,
-    // only the frame's changes are made (textures, buffers, state). True if the frame was drawn into the window (to
+    // only the frame's changes are made (textures, buffers, state) and frames into the window aren't drawn (frames into
-    // present).
+    // render targets always are). True if the frame was drawn into the window (to present).
     bool execute(std::span<const std::byte> commands, int offsetX, bool draw);
 
+    // Receives the pixels of frames that read their render target back (protocol::BeginFrame::readBack): rows packed,
+    // 0xAARRGGBB; empty if they couldn't be read. Valid during the call.
+    void setReadBack(std::function<void(std::span<const std::byte>)> readBack) { readBack_ = std::move(readBack); }
+
     // Replaces the shaders' source (the built-in one by default): for trying changes while the client runs.
     void setShaderSource(std::string source);
 
@@ -33,6 +39,7 @@ public:
 private:
     struct Texture {
         Diligent::RefCntAutoPtr<Diligent::ITexture> texture;
+        Diligent::RefCntAutoPtr<Diligent::ITexture> depth; // render targets only
         std::uint32_t width = 0;
         std::uint32_t height = 0;
         std::uint32_t flags = 0;
@@ -86,6 +93,13 @@ private:
     };
 
     void createTexture(const protocol::Command& command);
+    void createRenderTarget(const protocol::CreateRenderTarget& create);
+    void beginFrame(const protocol::BeginFrame& begin);
+    void endFrame();
+    // Sets the viewport for the last projection, in what the frame draws into.
+    void setViewport();
+    // The render target's pixels, rows packed, 0xAARRGGBB; empty if it can't be read.
+    std::span<const std::byte> readTarget(const Texture& target);
     void updateTexture(const protocol::Command& command);
     void createBuffer(const protocol::Command& command);
     void updateBuffer(const protocol::Command& command);
@@ -119,7 +133,16 @@ private:
 
     Constants constants_{};
     State state_;
+    // What the current frame draws into: the window, a render target, or nothing (a frame not drawn).
     bool onscreen_ = false; // in a frame drawn into the window
+    Texture* target_ = nullptr; // in a frame drawn into this render target
+    bool readsBack_ = false;    // the frame's render target is read back at its end
+    Diligent::ITextureView* colorView_ = nullptr;
+    Diligent::ITextureView* depthView_ = nullptr;
+    std::int32_t projectionWidth_ = 0;
+    std::int32_t projectionHeight_ = 0;
+    std::function<void(std::span<const std::byte>)> readBack_;
+    std::vector<std::byte> readPixels_;
     bool draw_ = true;
     std::uint64_t lastSentAt_ = 0;
     int offsetX_ = 0;