Draw render targets in the renderer process

While the renderer process runs, the adapter makes the client's render targets
itself and the renderer process draws frames into them: texture targets are
drawn with like any texture, and a pixmap target's frame is read back through
the channel into its Pixmap (protocol version 4: BeginFrame names its target,
CreateRenderTarget, Channel::readSize and the -read event). Frames into render
targets are drawn even while the window is hidden.

The client uses pixmap targets for snapshots (outfit pictures, photos), checked
to look right. The render target interfaces are declared from IMVU's vtables.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

adapter/engine.cpp

@@ -232,39 +232,62 @@ imvu::PiInternalVertexBuffer* Engine::newDynamicVertexBuffer(std::uint32_t eleme
 imvu::SharedPtr<imvu::PiPixmapRenderTarget> Engine::newPixmapRenderTarget(imvu::Vec2i size)
 {
     log::info("engine: frame {}: a {}x{} pixmap render target", frames_, size.x, size.y);
+    if (!recorder_->active() || size.x <= 0 || size.y <= 0) {
         return inner_->newPixmapRenderTarget(size);
     }
+    return imvu::SharedPtr<PixmapTarget>(std::make_unique<PixmapTarget>(recorder_, size));
+}
 
 imvu::SharedPtr<imvu::PiTextureRenderTarget> Engine::newTextureRenderTarget(imvu::Vec2i size)
 {
     log::info("engine: frame {}: a {}x{} texture render target", frames_, size.x, size.y);
+    if (!recorder_->active() || size.x <= 0 || size.y <= 0) {
         return inner_->newTextureRenderTarget(size);
     }
+    return imvu::SharedPtr<TextureTarget>(std::make_unique<TextureTarget>(recorder_, size));
+}
 
 void Engine::beginRendering(imvu::PiRenderTarget* target)
 {
+    // Frames into our render targets are drawn by the renderer process only; the client's renderer doesn't know them.
+    ourTarget_ = ours(target) ? target : nullptr;
+    if (ourTarget_ == nullptr) {
         inner_->beginRendering(target);
-    // Decided per frame: frames into render targets, and frames while the renderer process doesn't cover the view, are
-    // drawn by the client's renderer.
-    const bool takenOver = target == nullptr && recorder_->covering();
+    }
+    // Decided per frame: frames into the client renderer's render targets, and frames while the renderer process
+    // doesn't cover the view, are drawn by the client's renderer.
+    const bool takenOver = ourTarget_ != nullptr || (target == nullptr && recorder_->covering());
     if (takenOver != takenOver_ && target == nullptr) {
         log::info("engine: frame {}: {}", frames_, takenOver ? "drawn by the renderer process" : "drawn by the client's renderer");
     }
     takenOver_ = takenOver;
     if (target != nullptr && foreignLogged_ < 20) {
         ++foreignLogged_;
-        log::info("engine: frame {}: drawing into a render target ({})", frames_, static_cast<void*>(target));
+        log::info("engine: frame {}: drawing into {} render target ({})", frames_,
+                  ourTarget_ != nullptr ? "our" : "the client renderer's", static_cast<void*>(target));
     }
-    recorder_->add(Op::BeginFrame, protocol::BeginFrame{target != nullptr});
+    protocol::BeginFrame begin{target == nullptr ? 0 : protocol::foreignTarget, 0};
+    if (const auto* pixmap = dynamic_cast<const PixmapTarget*>(ourTarget_)) {
+        begin = {pixmap->id(), 1};
+    } else if (const auto* texture = dynamic_cast<const TextureTarget*>(ourTarget_)) {
+        begin = {texture->id(), 0};
+    }
+    recorder_->add(Op::BeginFrame, begin);
 }
 
 void Engine::endRendering()
 {
+    if (ourTarget_ == nullptr) {
         inner_->endRendering();
+    }
     LARGE_INTEGER now;
     QueryPerformanceCounter(&now);
     recorder_->add(Op::EndFrame, protocol::EndFrame{static_cast<std::uint64_t>(now.QuadPart)});
     recorder_->flush(true);
+    if (auto* pixmap = dynamic_cast<PixmapTarget*>(ourTarget_)) {
+        pixmap->setPixels(recorder_->readBack());
+    }
+    ourTarget_ = nullptr;
     ++frames_;
     if (frames_ % 300 == 0 && statsLogged_ < 6) {
         ++statsLogged_;

adapter/engine.hpp

@@ -73,6 +73,7 @@ private:
     std::shared_ptr<Recorder> recorder_;
     std::uint64_t frames_ = 0;
     bool takenOver_ = false; // this frame is drawn by the renderer process only
+    imvu::PiRenderTarget* ourTarget_ = nullptr; // this frame's render target, if it's one of ours
     int foreignLogged_ = 0; // diagnostics: textures the client renderer made, render targets
 
     // Diagnostics: what the client draws, summed over 300 frames and logged (the first few times).

adapter/pixmap.cpp

@@ -8,12 +8,20 @@ namespace {
 
 using Getter = unsigned(__thiscall*)(const imvu::Pixmap*);
 using PixelsGetter = std::uint32_t*(__thiscall*)(const imvu::Pixmap*);
+using SizeConstructor = imvu::Pixmap*(__thiscall*)(void*, int, int);
+using DefaultConstructor = imvu::Pixmap*(__thiscall*)(void*);
+using CopyConstructor = imvu::Pixmap*(__thiscall*)(void*, const imvu::Pixmap*);
+using Destructor = void(__thiscall*)(imvu::Pixmap*);
 
 struct Methods {
     Getter width = nullptr;
     Getter height = nullptr;
     Getter rowBytes = nullptr;
     PixelsGetter pixels = nullptr;
+    SizeConstructor construct = nullptr;
+    DefaultConstructor constructEmpty = nullptr;
+    CopyConstructor copy = nullptr;
+    Destructor destroy = nullptr;
 };
 
 const Methods& methods()
@@ -25,6 +33,10 @@ const Methods& methods()
             found.height = reinterpret_cast<Getter>(GetProcAddress(module, "?height@Pixmap@@QBEIXZ"));
             found.rowBytes = reinterpret_cast<Getter>(GetProcAddress(module, "?rowBytes@Pixmap@@QBEIXZ"));
             found.pixels = reinterpret_cast<PixelsGetter>(GetProcAddress(module, "?pixels@Pixmap@@QBEPAIXZ"));
+            found.construct = reinterpret_cast<SizeConstructor>(GetProcAddress(module, "??0Pixmap@@QAE@HH@Z"));
+            found.constructEmpty = reinterpret_cast<DefaultConstructor>(GetProcAddress(module, "??0Pixmap@@QAE@XZ"));
+            found.copy = reinterpret_cast<CopyConstructor>(GetProcAddress(module, "??0Pixmap@@QAE@ABV0@@Z"));
+            found.destroy = reinterpret_cast<Destructor>(GetProcAddress(module, "??1Pixmap@@QAE@XZ"));
         }
         return found;
     }();
@@ -42,4 +54,30 @@ std::optional<PixmapPixels> pixmap_pixels(const imvu::Pixmap& pixmap)
     return PixmapPixels{m.width(&pixmap), m.height(&pixmap), m.rowBytes(&pixmap), m.pixels(&pixmap)};
 }
 
+OwnedPixmap::OwnedPixmap(int width, int height)
+{
+    const Methods& m = methods();
+    if (m.construct != nullptr && m.constructEmpty != nullptr && m.copy != nullptr && m.destroy != nullptr) {
+        m.construct(storage_, width, height);
+        valid_ = true;
+    }
+}
+
+OwnedPixmap::~OwnedPixmap()
+{
+    if (valid_) {
+        methods().destroy(reinterpret_cast<imvu::Pixmap*>(storage_));
+    }
+}
+
+imvu::Pixmap* OwnedPixmap::copyTo(imvu::Pixmap* result) const
+{
+    const Methods& m = methods();
+    if (valid_) {
+        return m.copy(result, &get());
+    }
+    // Without pixmap.dll's methods there's nothing to construct one with (IMVU's own code would have failed already).
+    return m.constructEmpty != nullptr ? m.constructEmpty(result) : result;
+}
+
 } // namespace t5de

adapter/pixmap.hpp

@@ -1,6 +1,8 @@
 #pragma once
-// Reading IMVU's images (Pixmap) through pixmap.dll's exported methods, rather than depending on their layout.
+// IMVU's images (Pixmap), through pixmap.dll's exported methods rather than depending on their layout (only their
+// size, for making our own).
 
+#include <cstddef>
 #include <cstdint>
 #include <optional>
 
@@ -12,10 +14,32 @@ struct PixmapPixels {
     std::uint32_t width;
     std::uint32_t height;
     std::uint32_t rowBytes;
-    const std::uint32_t* pixels; // 0xAARRGGBB, rows rowBytes apart
+    std::uint32_t* pixels; // 0xAARRGGBB, rows rowBytes apart
 };
 
 // Null if pixmap.dll's methods can't be found.
 std::optional<PixmapPixels> pixmap_pixels(const imvu::Pixmap& pixmap);
 
+// A Pixmap of our own, made and destroyed by pixmap.dll.
+class OwnedPixmap {
+public:
+    // Width by height; invalid if pixmap.dll's methods can't be found.
+    OwnedPixmap(int width, int height);
+    OwnedPixmap(const OwnedPixmap&) = delete;
+    OwnedPixmap& operator=(const OwnedPixmap&) = delete;
+    ~OwnedPixmap();
+
+    [[nodiscard]] bool valid() const { return valid_; }
+    [[nodiscard]] const imvu::Pixmap& get() const { return *reinterpret_cast<const imvu::Pixmap*>(storage_); }
+
+    // Constructs a copy in result (raw memory for a Pixmap, as for a Pixmap returned by value); an empty Pixmap if
+    // this one isn't valid.
+    imvu::Pixmap* copyTo(imvu::Pixmap* result) const;
+
+private:
+    // sizeof(Pixmap) in pixmap.dll: its constructors set 7 fields.
+    alignas(4) std::byte storage_[28];
+    bool valid_ = false;
+};
+
 } // namespace t5de

adapter/recorder.cpp

@@ -69,6 +69,18 @@ void Recorder::addPixels(protocol::Op op, std::uint32_t id, const imvu::Pixmap&
     flushIfLarge();
 }
 
+std::uint32_t Recorder::createRenderTarget(std::uint32_t width, std::uint32_t height)
+{
+    const std::uint32_t id = nextId_++;
+    add(protocol::Op::CreateRenderTarget, protocol::CreateRenderTarget{id, width, height});
+    return id;
+}
+
+std::span<const std::byte> Recorder::readBack()
+{
+    return active() ? process_->readBack() : std::span<const std::byte>{};
+}
+
 std::uint32_t Recorder::createTexture(const imvu::Pixmap& pixmap, std::uint32_t flags)
 {
     const std::uint32_t id = nextId_++;

adapter/recorder.hpp

@@ -5,6 +5,7 @@
 
 #include <cstdint>
 #include <memory>
+#include <span>
 
 #include "renderer_process.hpp"
 #include "t5de/imvu/types.hpp"
@@ -37,6 +38,10 @@ public:
     void flushIfLarge();
 
     std::uint32_t createTexture(const imvu::Pixmap& pixmap, std::uint32_t flags);
+    // Destroyed with destroyTexture.
+    std::uint32_t createRenderTarget(std::uint32_t width, std::uint32_t height);
+    // After flushing a frame that reads its render target back: its pixels (RendererProcess::readBack).
+    std::span<const std::byte> readBack();
     void updateTexture(std::uint32_t id, const imvu::Pixmap& pixmap);
     void destroyTexture(std::uint32_t id);
 

adapter/renderer_process.cpp

@@ -72,7 +72,9 @@ std::unique_ptr<RendererProcess> RendererProcess::start(HWND window, protocol::V
         static_cast<protocol::Channel*>(MapViewOfFile(renderer->mapping_, FILE_MAP_ALL_ACCESS, 0, 0, size));
     renderer->ready_ = CreateEventW(nullptr, FALSE, FALSE, (name + protocol::readyEventSuffix).c_str());
     renderer->consumed_ = CreateEventW(nullptr, FALSE, TRUE, (name + protocol::consumedEventSuffix).c_str());
-    if (renderer->channel_ == nullptr || renderer->ready_ == nullptr || renderer->consumed_ == nullptr) {
+    renderer->read_ = CreateEventW(nullptr, FALSE, FALSE, (name + protocol::readEventSuffix).c_str());
+    if (renderer->channel_ == nullptr || renderer->ready_ == nullptr || renderer->consumed_ == nullptr
+        || renderer->read_ == nullptr) {
         log::info("renderer process: can't set up the channel: error {}", GetLastError());
         return nullptr;
     }
@@ -86,7 +88,7 @@ std::unique_ptr<RendererProcess> RendererProcess::start(HWND window, protocol::V
                            .chunkSize = 0,
                            .chunkFlags = 0,
                            .covering = 0,
-                           .reserved = 0};
+                           .readSize = 0};
 
     // The renderer's window goes on top of the client's 3D view, as a child window. The client's window mustn't clip
     // its children (WS_CLIPCHILDREN, which the client creates it with): the client draws a frame when its window gets
@@ -117,7 +119,7 @@ RendererProcess::~RendererProcess()
         }
         UnmapViewOfFile(channel_);
     }
-    for (HANDLE handle : {ready_, consumed_, mapping_, process_}) {
+    for (HANDLE handle : {ready_, consumed_, read_, mapping_, process_}) {
         if (handle != nullptr) {
             CloseHandle(handle);
         }
@@ -150,4 +152,20 @@ bool RendererProcess::send(std::span<const std::byte> commands, bool endsFrame)
     return true;
 }
 
+std::span<const std::byte> RendererProcess::readBack()
+{
+    if (!alive_) {
+        return {};
+    }
+    const HANDLE waits[] = {read_, process_};
+    const DWORD result = WaitForMultipleObjects(2, waits, FALSE, consumeTimeout);
+    if (result != WAIT_OBJECT_0) {
+        log::info("renderer process: {} reading back a frame; giving up on it",
+                  result == WAIT_OBJECT_0 + 1 ? "exited" : "not responding");
+        alive_ = false;
+        return {};
+    }
+    return {data_, std::min<std::size_t>(channel_->readSize, channel_->capacity)};
+}
+
 } // namespace t5de

adapter/renderer_process.hpp

@@ -25,6 +25,10 @@ public:
     // process is gone or stopped taking them (it's then given up on).
     bool send(std::span<const std::byte> commands, bool endsFrame);
 
+    // After sending a frame that reads its render target back: waits for the pixels (protocol::Channel), valid until
+    // the next send. Empty if the renderer process couldn't, or is gone.
+    std::span<const std::byte> readBack();
+
     [[nodiscard]] bool alive() const { return alive_; }
 
     // Whether the renderer process covers the whole 3D view (protocol::Channel::covering).
@@ -36,6 +40,7 @@ private:
     HANDLE mapping_ = nullptr;
     HANDLE ready_ = nullptr;
     HANDLE consumed_ = nullptr;
+    HANDLE read_ = nullptr;
     HANDLE process_ = nullptr;
     protocol::Channel* channel_ = nullptr;
     std::byte* data_ = nullptr;

adapter/resources.cpp

@@ -1,8 +1,13 @@
 #include "resources.hpp"
 
+#include <algorithm>
+#include <cstring>
 #include <mutex>
+#include <new>
 #include <unordered_set>
 
+#include "log.hpp"
+
 namespace t5de {
 
 namespace {
@@ -48,28 +53,48 @@ Texture::Texture(imvu::PiRenderTexture* inner, std::shared_ptr<Recorder> recorde
     add(static_cast<imvu::PiRenderTexture*>(this));
 }
 
+Texture::Texture(std::shared_ptr<Recorder> recorder, std::uint32_t targetId, imvu::Vec2i size)
+    : inner_(nullptr), recorder_(std::move(recorder)), id_(targetId), size_(size), ownsId_(false)
+{
+    add(static_cast<imvu::PiRenderTexture*>(this));
+}
+
 Texture::~Texture()
 {
     remove(static_cast<imvu::PiRenderTexture*>(this));
     delete inner_; // through its virtual destructor: IMVU's code frees it
+    if (ownsId_) {
         recorder_->destroyTexture(id_);
     }
+}
 
-imvu::Vec2i Texture::size() const { return inner_->size(); }
+imvu::Vec2i Texture::size() const { return inner_ != nullptr ? inner_->size() : size_; }
 
 void Texture::updateRegions(const imvu::Pixmap& pixmap, const imvu::CompoundRegion& regions)
 {
+    if (inner_ != nullptr) {
         inner_->updateRegions(pixmap, regions);
         recorder_->updateTexture(id_, pixmap); // all of it: the regions' layout isn't known
     }
+}
 
 void Texture::update(const imvu::Pixmap& pixmap)
 {
+    if (inner_ != nullptr) {
         inner_->update(pixmap);
         recorder_->updateTexture(id_, pixmap);
     }
+}
 
-imvu::Pixmap* Texture::read(imvu::Pixmap* result) const { return inner_->read(result); }
+imvu::Pixmap* Texture::read(imvu::Pixmap* result) const
+{
+    if (inner_ != nullptr) {
+        return inner_->read(result);
+    }
+    // A render target's texture: not read back (not seen used); an image of its size.
+    log::info("texture: a render target's texture read; not supported, returning a blank image");
+    return OwnedPixmap(size_.x, size_.y).copyTo(result);
+}
 
 IndexBuffer::IndexBuffer(imvu::PiInternalIndexBuffer* inner, std::shared_ptr<Recorder> recorder, std::uint32_t id)
     : inner_(inner), recorder_(std::move(recorder)), id_(id)
@@ -105,4 +130,62 @@ void VertexBuffer::update(const void* data, std::uint32_t stride, std::uint32_t
 
 void VertexBuffer::unknown2() { inner_->unknown2(); }
 
+PixmapTarget::PixmapTarget(std::shared_ptr<Recorder> recorder, imvu::Vec2i size)
+    : recorder_(std::move(recorder)),
+      id_(recorder_->createRenderTarget(static_cast<std::uint32_t>(size.x), static_cast<std::uint32_t>(size.y))),
+      size_(size), pixmap_(size.x, size.y)
+{
+    add(static_cast<imvu::PiRenderTarget*>(this));
+}
+
+PixmapTarget::~PixmapTarget()
+{
+    remove(static_cast<imvu::PiRenderTarget*>(this));
+    recorder_->destroyTexture(id_);
+}
+
+imvu::Pixmap* PixmapTarget::getPixmap(imvu::Pixmap* result) const { return pixmap_.copyTo(result); }
+
+void PixmapTarget::setPixels(std::span<const std::byte> pixels)
+{
+    const auto into = pixmap_.valid() ? pixmap_pixels(pixmap_.get()) : std::nullopt;
+    const auto width = static_cast<std::uint32_t>(size_.x);
+    const auto height = static_cast<std::uint32_t>(size_.y);
+    const std::size_t rowSize = std::size_t{width} * 4;
+    if (!into || into->pixels == nullptr || pixels.size() < rowSize * height) {
+        log::info("render target: {}x{} frame not read back ({} bytes)", width, height, pixels.size());
+        return;
+    }
+    const std::size_t copied = std::min<std::size_t>(rowSize, std::size_t{into->width} * 4);
+    auto* rows = reinterpret_cast<std::byte*>(into->pixels);
+    for (std::uint32_t y = 0; y < std::min(into->height, height); ++y) {
+        std::memcpy(rows + std::size_t{y} * into->rowBytes, pixels.data() + y * rowSize, copied);
+    }
+}
+
+TextureTarget::TextureTarget(std::shared_ptr<Recorder> recorder, imvu::Vec2i size)
+    : recorder_(std::move(recorder)),
+      id_(recorder_->createRenderTarget(static_cast<std::uint32_t>(size.x), static_cast<std::uint32_t>(size.y))),
+      size_(size)
+{
+    add(static_cast<imvu::PiRenderTarget*>(this));
+}
+
+TextureTarget::~TextureTarget()
+{
+    remove(static_cast<imvu::PiRenderTarget*>(this));
+    recorder_->destroyTexture(id_);
+}
+
+imvu::TextureArea* TextureTarget::area(imvu::TextureArea* result) const
+{
+    return new (result) imvu::TextureArea{0, 0, 1, 1};
+}
+
+imvu::SharedPtr<imvu::PiRenderTexture>* TextureTarget::texture(imvu::SharedPtr<imvu::PiRenderTexture>* result) const
+{
+    return new (result) imvu::SharedPtr<imvu::PiRenderTexture>(
+        std::unique_ptr<imvu::PiRenderTexture>(new Texture(recorder_, id_, size_)));
+}
+
 } // namespace t5de

adapter/resources.hpp

@@ -3,8 +3,12 @@
 // client renderer still draws with) and mirrors it, by id, to the renderer process. The client deletes them through
 // their virtual destructors.
 
+#include <cstddef>
 #include <cstdint>
 #include <memory>
+#include <span>
+
+#include "pixmap.hpp"
 
 #include "recorder.hpp"
 #include "t5de/imvu/resources.hpp"
@@ -14,6 +18,9 @@ namespace t5de {
 class Texture final : public imvu::PiRenderTexture {
 public:
     Texture(imvu::PiRenderTexture* inner, std::shared_ptr<Recorder> recorder, std::uint32_t id);
+    // What's been drawn into one of our texture render targets (TextureTarget): no texture of the client renderer's,
+    // and the id stays the target's.
+    Texture(std::shared_ptr<Recorder> recorder, std::uint32_t targetId, imvu::Vec2i size);
     ~Texture() override;
 
     imvu::Vec2i size() const override;
@@ -28,6 +35,44 @@ private:
     imvu::PiRenderTexture* inner_;
     std::shared_ptr<Recorder> recorder_;
     std::uint32_t id_;
+    imvu::Vec2i size_{};
+    bool ownsId_ = true;
+};
+
+// Render targets the renderer process draws (made while it runs; otherwise the client renderer makes its own).
+class PixmapTarget final : public imvu::PiPixmapRenderTarget {
+public:
+    PixmapTarget(std::shared_ptr<Recorder> recorder, imvu::Vec2i size);
+    ~PixmapTarget() override;
+
+    imvu::Pixmap* getPixmap(imvu::Pixmap* result) const override;
+
+    // Copies a frame's pixels, read back from the renderer process (rows packed, 0xAARRGGBB), into the pixmap.
+    void setPixels(std::span<const std::byte> pixels);
+
+    [[nodiscard]] std::uint32_t id() const { return id_; }
+
+private:
+    std::shared_ptr<Recorder> recorder_;
+    std::uint32_t id_;
+    imvu::Vec2i size_;
+    OwnedPixmap pixmap_;
+};
+
+class TextureTarget final : public imvu::PiTextureRenderTarget {
+public:
+    TextureTarget(std::shared_ptr<Recorder> recorder, imvu::Vec2i size);
+    ~TextureTarget() override;
+
+    imvu::TextureArea* area(imvu::TextureArea* result) const override;
+    imvu::SharedPtr<imvu::PiRenderTexture>* texture(imvu::SharedPtr<imvu::PiRenderTexture>* result) const override;
+
+    [[nodiscard]] std::uint32_t id() const { return id_; }
+
+private:
+    std::shared_ptr<Recorder> recorder_;
+    std::uint32_t id_;
+    imvu::Vec2i size_;
 };
 
 class IndexBuffer final : public imvu::PiInternalIndexBuffer {
@@ -62,8 +107,9 @@ private:
     std::uint32_t id_;
 };
 
-// Whether an object is one of ours (live Texture, IndexBuffer or VertexBuffer): the client can also hand the engine
-// objects the client renderer made itself (render targets' textures), which must reach it unchanged.
+// Whether an object is one of ours (live Texture, IndexBuffer, VertexBuffer, PixmapTarget or TextureTarget): the client
+// can also hand the engine objects the client renderer made itself (its render targets, and their textures), which
+// must reach it unchanged.
 bool ours(const void* object);
 
 // The client renderer's object, and our id, for what the client passes in: ours unwrapped, others unchanged (id 0).

docs/architecture.md

@@ -39,8 +39,8 @@ The process exits when the engine is destroyed or the client exits.
 ## Status
 
 The renderer process draws the whole 3D view in place of the client's renderer (the full view, the default). The
-adapter still forwards everything to the client's Direct3D renderer (state and resources: it draws render targets and
-reads textures back, which the renderer process doesn't yet), but while the renderer process covers the whole view
+adapter still forwards state and resources to the client's Direct3D renderer (as a fallback, and for reading textures
+back, which the renderer process doesn't do), but while the renderer process covers the whole view
 (`Channel::covering`, set after each frame it presents) the client's renderer skips its clears and draws, decided per
 frame; it draws again as soon as the renderer process stops covering the view (the view changes, or the process
 fails). Checked in rooms with screenshots of both renderers side by side (the compare view): they match, particles
@@ -70,8 +70,13 @@ For comparing while the client runs (development):
   the right half visibly trails, mostly from compositing another process's window over Remote Desktop. The
   renderer process draws only the newest frame when it falls behind, and keeps at most one frame queued.
 
-Next: render targets (offscreen frames are skipped, and drawn by the client's renderer), and one renderer process for
-all of a client's views.
+Render targets are the renderer process's too, while it runs: the adapter makes its own (`PixmapTarget`,
+`TextureTarget`), the renderer process draws frames into them (textures in the window's formats, so the same
+pipelines draw both), and a pixmap target's frame is read back through the channel (`Channel::readSize`, the
+`-read` event) into its Pixmap, as the client's renderer does, synchronously. A read back must fit in the channel's
+data area (16 MB: the client's 2048x2048 snapshots just do). Checked with outfit pictures and photos.
+
+Next: one renderer process for all of a client's views.
 
 ## Building
 

docs/interface.md

@@ -83,8 +83,14 @@ virtual methods, IMVU's code only uses an engine through helpers that call its v
   practice: clamped to it they look much smaller and fainter than the client's.
 - Fog: linear table fog, start 1 and end 50 in rooms. Fogging by view depth matches the client; by depth buffer
   value (as Direct3D 9's documentation suggests for projections like IMVU's) doesn't.
-- Still unknown: render targets, `CompoundRegion` (texture update
-  regions), `newRenderTexture`'s flags, and what `beginRendering` does with a null target. MSVC orders overloaded
+- Render targets (`include/t5de/imvu/resources.hpp`): the engine returns each as a pointer into its own object;
+  the client calls only their interfaces' methods and passes them to `beginRendering`. A pixmap render target has one
+  method, `getPixmap()` (a Pixmap returned by value), with the pixels of the last frame drawn into it (IMVU's engine
+  copies them from the GPU at the frame's end). A texture render target has `area()` (always 0, 0, 1, 1) and
+  `texture()` (a new texture object for what's been drawn, as a shared pointer returned by value). The client uses
+  pixmap targets for snapshots (`takeSnapshot`: outfit pictures, the photo widget), 2048x2048; texture targets
+  haven't been seen used. Pixmap is 28 bytes; pixmap.dll exports its constructors and destructor.
+- Still unknown: `CompoundRegion` (texture update regions), `newRenderTexture`'s flags, and what `beginRendering` does with a null target. MSVC orders overloaded
   virtual methods unlike their declaration, so our declarations never overload virtual methods.
 - Memory crosses the boundary only through virtual methods (destructors, control blocks), so each side can use its
   own C runtime. Exceptions must not cross it.

include/t5de/imvu/resources.hpp

@@ -6,6 +6,7 @@
 
 #include <cstdint>
 
+#include "t5de/imvu/shared_ptr.hpp"
 #include "t5de/imvu/types.hpp"
 
 namespace t5de::imvu {
@@ -39,8 +40,36 @@ public:
     virtual Pixmap* read(Pixmap* result) const = 0;
 };
 
-class PiRenderTarget;
-class PiPixmapRenderTarget;
-class PiTextureRenderTarget;
+// What frames can be drawn into besides the window (PiRenderEngine::beginRendering). IMVU's Direct3D engine returns
+// each as a pointer into its own object (single inheritance from these, so the PiRenderTarget pointer is the same).
+class PiRenderTarget {
+public:
+    virtual ~PiRenderTarget() = default;
+};
+
+// Drawn into, then read: after each frame drawn into it, getPixmap() has its pixels (IMVU's engine copies them at the
+// end of the frame). Used for snapshots: outfit pictures, the photo widget.
+class PiPixmapRenderTarget : public PiRenderTarget {
+public:
+    // Constructs a copy of the pixels in result (a Pixmap returned by value).
+    virtual Pixmap* getPixmap(Pixmap* result) const = 0;
+};
+
+// The part of the texture the target covers, in texture coordinates. IMVU's Direct3D engine always returns 0, 0, 1, 1.
+struct TextureArea {
+    float left;
+    float top;
+    float right;
+    float bottom;
+};
+
+// Drawn into, then drawn with as a texture.
+class PiTextureRenderTarget : public PiRenderTarget {
+public:
+    // Constructs the area in result (returned by value).
+    virtual TextureArea* area(TextureArea* result) const = 0;
+    // Constructs, in result, a new texture object for what's been drawn (a shared pointer returned by value).
+    virtual SharedPtr<PiRenderTexture>* texture(SharedPtr<PiRenderTexture>* result) const = 0;
+};
 
 } // namespace t5de::imvu

include/t5de/protocol/channel.hpp

@@ -7,7 +7,7 @@
 
 namespace t5de::protocol {
 
-inline constexpr std::uint32_t version = 3;
+inline constexpr std::uint32_t version = 4;
 
 // Which part of the client's 3D view the renderer process draws. Full is the default; the others are for comparing
 // with the client's renderer, which then draws the whole view under the renderer process's window.
@@ -22,6 +22,11 @@ enum class View : std::uint32_t {
 // argument: this header, then `capacity` bytes of data. The adapter sends the command stream (commands.hpp) through
 // it in chunks: it waits for the event "<name>-consumed", writes a chunk and sets "<name>-ready"; the renderer
 // process copies the chunk out and sets "<name>-consumed" (both auto-reset; consumed starts set).
+//
+// A frame drawn into a render target can ask for its pixels back (BeginFrame::readBack): having sent it, the adapter
+// waits for "<name>-read" (auto-reset), which the renderer process sets once it has drawn the frame and written the
+// pixels into the data area (readSize bytes: rows packed, 0xAARRGGBB; 0 if it couldn't). The adapter sends nothing
+// meanwhile, so the data area is free.
 struct Channel {
     std::uint32_t version;
     std::uint32_t closing; // nonzero: the engine is gone, the renderer process should exit
@@ -34,7 +39,7 @@ struct Channel {
     // Set by the renderer process while its window covers the whole 3D view and it's presenting frames: the client's
     // renderer then needn't draw (the adapter skips its draws; it draws again as soon as this is cleared).
     std::uint32_t covering;
-    std::uint32_t reserved;
+    std::uint32_t readSize;
 };
 
 static_assert(sizeof(Channel) == 48, "same layout in 32-bit and 64-bit builds");
@@ -44,5 +49,6 @@ inline constexpr std::uint32_t channelCapacity = 16 * 1024 * 1024;
 
 inline constexpr wchar_t readyEventSuffix[] = L"-ready";
 inline constexpr wchar_t consumedEventSuffix[] = L"-consumed";
+inline constexpr wchar_t readEventSuffix[] = L"-read";
 
 } // namespace t5de::protocol

include/t5de/protocol/commands.hpp

@@ -33,6 +33,7 @@ enum class Op : std::uint16_t {
     DrawTriangles,
     DrawArrays,
     DrawPointSprites,
+    CreateRenderTarget,
 };
 
 struct Header {
@@ -44,9 +45,15 @@ struct Header {
 // Colors are 0xAARRGGBB. Resource ids are the adapter's, per engine; 0 means none.
 
 struct BeginFrame {
-    std::uint32_t offscreen; // drawn into a render target, not the window: nothing to show
+    // The render target drawn into (CreateRenderTarget's id), 0 for the window, or foreignTarget.
+    std::uint32_t target;
+    std::uint32_t readBack; // at the end of the frame, send the render target's pixels back (channel.hpp)
 };
 
+// A frame drawn into a render target of the client renderer's own (made while the renderer process wasn't running):
+// the client renderer draws it, the renderer process only follows its changes.
+inline constexpr std::uint32_t foreignTarget = 0xffffffff;
+
 struct EndFrame {
     std::uint64_t sentAt; // the adapter's QueryPerformanceCounter (the same clock in every process)
 };
@@ -131,6 +138,13 @@ struct DestroyTexture {
     std::uint32_t id;
 };
 
+// A texture frames can be drawn into (BeginFrame), and drawn with like any other; destroyed with DestroyTexture.
+struct CreateRenderTarget {
+    std::uint32_t id;
+    std::uint32_t width;
+    std::uint32_t height;
+};
+
 enum class BufferKind : std::uint32_t {
     Index,  // 16-bit indices
     Vertex, // positions and normals (3 floats), texture coordinates (2 floats) or colors (0xAARRGGBB)

renderer/main.cpp

@@ -184,12 +184,13 @@ int WINAPI wWinMain(HINSTANCE, HINSTANCE, PWSTR, int)
     HANDLE ready = OpenEventW(SYNCHRONIZE, FALSE, (arguments.channel + t5de::protocol::readyEventSuffix).c_str());
     HANDLE consumed =
         OpenEventW(EVENT_MODIFY_STATE, FALSE, (arguments.channel + t5de::protocol::consumedEventSuffix).c_str());
+    HANDLE read = OpenEventW(EVENT_MODIFY_STATE, FALSE, (arguments.channel + t5de::protocol::readEventSuffix).c_str());
     HANDLE parent = OpenProcess(SYNCHRONIZE, FALSE, arguments.parent);
-    if (channel == nullptr || ready == nullptr || consumed == nullptr || parent == nullptr
+    if (channel == nullptr || ready == nullptr || consumed == nullptr || read == nullptr || parent == nullptr
         || channel->version != t5de::protocol::version) {
         return 3;
     }
-    const auto* data = reinterpret_cast<const std::byte*>(channel + 1);
+    auto* data = reinterpret_cast<std::byte*>(channel + 1);
 
     const HWND clientWindow = reinterpret_cast<HWND>(static_cast<std::uintptr_t>(channel->window));
     const HWND window = create_child_window(clientWindow);
@@ -227,6 +228,13 @@ int WINAPI wWinMain(HINSTANCE, HINSTANCE, PWSTR, int)
         return 6;
     }
     t5de::Scene scene(device, context, swapChain);
+    // Frames read back: the adapter waits for them, sending nothing meanwhile, so the data area is free.
+    scene.setReadBack([&](std::span<const std::byte> pixels) {
+        const std::size_t size = pixels.size() <= channel->capacity ? pixels.size() : 0;
+        std::memcpy(data, pixels.data(), size);
+        channel->readSize = static_cast<std::uint32_t>(size);
+        SetEvent(read);
+    });
 
     // Takes the chunk waiting in the channel (after "ready"), and lets the adapter send the next: appends it to
     // frame, and returns whether it ends a frame.

renderer/scene.cpp

@@ -102,12 +102,7 @@ bool Scene::execute(std::span<const std::byte> commands, int offsetX, bool draw)
         switch (command->op()) {
         case Op::BeginFrame:
             if (const auto begin = command->body<protocol::BeginFrame>()) {
-                onscreen_ = begin->offscreen == 0 && draw_;
-                if (onscreen_) {
-                    ITextureView* target = swapChain_->GetCurrentBackBufferRTV();
-                    context_->SetRenderTargets(1, &target, swapChain_->GetDepthBufferDSV(),
-                                               RESOURCE_STATE_TRANSITION_MODE_TRANSITION);
-                }
+                beginFrame(*begin);
             }
             break;
         case Op::EndFrame:
@@ -115,18 +110,17 @@ bool Scene::execute(std::span<const std::byte> commands, int offsetX, bool draw)
                 lastSentAt_ = end->sentAt;
             }
             drew = drew || onscreen_;
-            onscreen_ = false;
+            endFrame();
             break;
         case Op::Clear:
-            if (const auto clear = command->body<protocol::Clear>(); clear && onscreen_) {
+            if (const auto clear = command->body<protocol::Clear>(); clear && colorView_ != nullptr) {
                 if ((clear->flags & 1) != 0) {
                     float color[4];
                     set_color(color, clear->color);
-                    context_->ClearRenderTarget(swapChain_->GetCurrentBackBufferRTV(), color,
-                                                RESOURCE_STATE_TRANSITION_MODE_TRANSITION);
+                    context_->ClearRenderTarget(colorView_, color, RESOURCE_STATE_TRANSITION_MODE_TRANSITION);
                 }
                 if ((clear->flags & 2) != 0) {
-                    context_->ClearDepthStencil(swapChain_->GetDepthBufferDSV(), CLEAR_DEPTH_FLAG, 1.0f, 0,
+                    context_->ClearDepthStencil(depthView_, CLEAR_DEPTH_FLAG, 1.0f, 0,
                                                 RESOURCE_STATE_TRANSITION_MODE_TRANSITION);
                 }
             }
@@ -184,6 +178,11 @@ bool Scene::execute(std::span<const std::byte> commands, int offsetX, bool draw)
         case Op::UpdateTexture:
             updateTexture(*command);
             break;
+        case Op::CreateRenderTarget:
+            if (const auto create = command->body<protocol::CreateRenderTarget>()) {
+                createRenderTarget(*create);
+            }
+            break;
         case Op::DestroyTexture:
             if (const auto destroy = command->body<protocol::DestroyTexture>()) {
                 textures_.erase(destroy->id);
@@ -201,17 +200,17 @@ bool Scene::execute(std::span<const std::byte> commands, int offsetX, bool draw)
             }
             break;
         case Op::DrawTriangles:
-            if (const auto triangles = command->body<protocol::DrawTriangles>(); triangles && onscreen_) {
+            if (const auto triangles = command->body<protocol::DrawTriangles>(); triangles && colorView_ != nullptr) {
                 drawTriangles(*triangles);
             }
             break;
         case Op::DrawArrays:
-            if (onscreen_) {
+            if (colorView_ != nullptr) {
                 drawArrays(*command);
             }
             break;
         case Op::DrawPointSprites:
-            if (onscreen_) {
+            if (colorView_ != nullptr) {
                 drawPointSprites(*command);
             }
             break;
@@ -232,19 +231,107 @@ void Scene::setProjection(const protocol::SetProjection& projection)
     }
     constants_.viewport[0] = static_cast<float>(projection.width);
     constants_.viewport[1] = static_cast<float>(projection.height);
-    if (!onscreen_) {
+    projectionWidth_ = projection.width;
+    projectionHeight_ = projection.height;
+    setViewport();
+}
+
+void Scene::setViewport()
+{
+    if (colorView_ == nullptr || projectionWidth_ <= 0 || projectionHeight_ <= 0) {
         return;
     }
     // The client's projection covers its whole 3D view; the window shows part of it, from offsetX.
     const SwapChainDesc& window = swapChain_->GetDesc();
     Viewport viewport;
-    viewport.TopLeftX = static_cast<float>(-offsetX_);
+    viewport.TopLeftX = onscreen_ ? static_cast<float>(-offsetX_) : 0.0f;
     viewport.TopLeftY = 0;
-    viewport.Width = static_cast<float>(projection.width);
-    viewport.Height = static_cast<float>(projection.height);
+    viewport.Width = static_cast<float>(projectionWidth_);
+    viewport.Height = static_cast<float>(projectionHeight_);
     viewport.MinDepth = 0;
     viewport.MaxDepth = 1;
-    context_->SetViewports(1, &viewport, window.Width, window.Height);
+    context_->SetViewports(1, &viewport, onscreen_ ? window.Width : target_->width,
+                           onscreen_ ? window.Height : target_->height);
+}
+
+void Scene::beginFrame(const protocol::BeginFrame& begin)
+{
+    onscreen_ = false;
+    target_ = nullptr;
+    colorView_ = nullptr;
+    depthView_ = nullptr;
+    readsBack_ = begin.readBack != 0;
+    if (begin.target == 0) {
+        if (draw_) {
+            onscreen_ = true;
+            colorView_ = swapChain_->GetCurrentBackBufferRTV();
+            depthView_ = swapChain_->GetDepthBufferDSV();
+        }
+    } else if (const auto found = textures_.find(begin.target); found != textures_.end() && found->second.depth) {
+        target_ = &found->second;
+        colorView_ = target_->texture->GetDefaultView(TEXTURE_VIEW_RENDER_TARGET);
+        depthView_ = target_->depth->GetDefaultView(TEXTURE_VIEW_DEPTH_STENCIL);
+    }
+    if (colorView_ != nullptr) {
+        context_->SetRenderTargets(1, &colorView_, depthView_, RESOURCE_STATE_TRANSITION_MODE_TRANSITION);
+        setViewport();
+    }
+}
+
+void Scene::endFrame()
+{
+    if (readsBack_ && readBack_) {
+        readBack_(target_ != nullptr ? readTarget(*target_) : std::span<const std::byte>{});
+    }
+    if (target_ != nullptr) {
+        // Unbound, so it can be drawn with (a texture can't be both).
+        context_->SetRenderTargets(0, nullptr, nullptr, RESOURCE_STATE_TRANSITION_MODE_TRANSITION);
+    }
+    onscreen_ = false;
+    target_ = nullptr;
+    readsBack_ = false;
+    colorView_ = nullptr;
+    depthView_ = nullptr;
+}
+
+std::span<const std::byte> Scene::readTarget(const Texture& target)
+{
+    TextureDesc desc;
+    desc.Name = "Render target read back";
+    desc.Type = RESOURCE_DIM_TEX_2D;
+    desc.Width = target.width;
+    desc.Height = target.height;
+    desc.MipLevels = 1;
+    desc.Format = TEX_FORMAT_RGBA8_UNORM;
+    desc.Usage = USAGE_STAGING;
+    desc.CPUAccessFlags = CPU_ACCESS_READ;
+    RefCntAutoPtr<ITexture> staging;
+    device_->CreateTexture(desc, nullptr, &staging);
+    if (!staging) {
+        return {};
+    }
+    CopyTextureAttribs copy(target.texture, RESOURCE_STATE_TRANSITION_MODE_TRANSITION, staging,
+                            RESOURCE_STATE_TRANSITION_MODE_TRANSITION);
+    context_->CopyTexture(copy);
+    MappedTextureSubresource mapped;
+    context_->MapTextureSubresource(staging, 0, 0, MAP_READ, MAP_FLAG_NONE, nullptr, mapped); // waits for the GPU
+    if (mapped.pData == nullptr) {
+        return {};
+    }
+    // RGBA in memory, to the client's 0xAARRGGBB (BGRA in memory).
+    readPixels_.resize(std::size_t{target.width} * target.height * 4);
+    for (std::uint32_t y = 0; y < target.height; ++y) {
+        const auto* from = static_cast<const std::uint8_t*>(mapped.pData) + y * mapped.Stride;
+        auto* to = reinterpret_cast<std::uint8_t*>(readPixels_.data()) + std::size_t{y} * target.width * 4;
+        for (std::uint32_t x = 0; x < target.width; ++x) {
+            to[x * 4 + 0] = from[x * 4 + 2];
+            to[x * 4 + 1] = from[x * 4 + 1];
+            to[x * 4 + 2] = from[x * 4 + 0];
+            to[x * 4 + 3] = from[x * 4 + 3];
+        }
+    }
+    context_->UnmapTextureSubresource(staging, 0, 0);
+    return readPixels_;
 }
 
 void Scene::setState(const protocol::SetState& state)
@@ -322,6 +409,38 @@ void Scene::createTexture(const protocol::Command& command)
     textures_[create->id] = std::move(texture);
 }
 
+void Scene::createRenderTarget(const protocol::CreateRenderTarget& create)
+{
+    if (create.width == 0 || create.height == 0) {
+        return;
+    }
+    // The window's formats, so the same pipelines draw into both.
+    const SwapChainDesc& window = swapChain_->GetDesc();
+    TextureDesc desc;
+    desc.Name = "Client render target";
+    desc.Type = RESOURCE_DIM_TEX_2D;
+    desc.Width = create.width;
+    desc.Height = create.height;
+    desc.MipLevels = 1;
+    desc.Format = window.ColorBufferFormat;
+    desc.BindFlags = BIND_RENDER_TARGET | BIND_SHADER_RESOURCE;
+    desc.ClearValue.Format = desc.Format;
+    Texture target{.width = create.width, .height = create.height, .flags = 0};
+    device_->CreateTexture(desc, nullptr, &target.texture);
+    TextureDesc depth = desc;
+    depth.Name = "Client render target depth";
+    depth.Format = window.DepthBufferFormat;
+    depth.BindFlags = BIND_DEPTH_STENCIL;
+    depth.ClearValue.Format = depth.Format;
+    depth.ClearValue.DepthStencil.Depth = 1;
+    device_->CreateTexture(depth, nullptr, &target.depth);
+    if (!target.texture || !target.depth) {
+        return;
+    }
+    target.texture->GetDefaultView(TEXTURE_VIEW_SHADER_RESOURCE)->SetSampler(samplers_[0]);
+    textures_[create.id] = std::move(target);
+}
+
 void Scene::updateTexture(const protocol::Command& command)
 {
     const auto update = command.body<protocol::UpdateTexture>();

renderer/scene.hpp

@@ -5,9 +5,11 @@
 
 #include <cstddef>
 #include <cstdint>
+#include <functional>
 #include <span>
 #include <string>
 #include <unordered_map>
+#include <vector>
 
 #include "EngineFactoryD3D11.h"
 #include "RefCntAutoPtr.hpp"
@@ -20,10 +22,14 @@ public:
     Scene(Diligent::IRenderDevice* device, Diligent::IDeviceContext* context, Diligent::ISwapChain* swapChain);
 
     // Carries out one frame's commands. offsetX is where the window starts within the client's 3D view; without draw,
-    // only the frame's changes are made (textures, buffers, state). True if the frame was drawn into the window (to
-    // present).
+    // only the frame's changes are made (textures, buffers, state) and frames into the window aren't drawn (frames into
+    // render targets always are). True if the frame was drawn into the window (to present).
     bool execute(std::span<const std::byte> commands, int offsetX, bool draw);
 
+    // Receives the pixels of frames that read their render target back (protocol::BeginFrame::readBack): rows packed,
+    // 0xAARRGGBB; empty if they couldn't be read. Valid during the call.
+    void setReadBack(std::function<void(std::span<const std::byte>)> readBack) { readBack_ = std::move(readBack); }
+
     // Replaces the shaders' source (the built-in one by default): for trying changes while the client runs.
     void setShaderSource(std::string source);
 
@@ -33,6 +39,7 @@ public:
 private:
     struct Texture {
         Diligent::RefCntAutoPtr<Diligent::ITexture> texture;
+        Diligent::RefCntAutoPtr<Diligent::ITexture> depth; // render targets only
         std::uint32_t width = 0;
         std::uint32_t height = 0;
         std::uint32_t flags = 0;
@@ -86,6 +93,13 @@ private:
     };
 
     void createTexture(const protocol::Command& command);
+    void createRenderTarget(const protocol::CreateRenderTarget& create);
+    void beginFrame(const protocol::BeginFrame& begin);
+    void endFrame();
+    // Sets the viewport for the last projection, in what the frame draws into.
+    void setViewport();
+    // The render target's pixels, rows packed, 0xAARRGGBB; empty if it can't be read.
+    std::span<const std::byte> readTarget(const Texture& target);
     void updateTexture(const protocol::Command& command);
     void createBuffer(const protocol::Command& command);
     void updateBuffer(const protocol::Command& command);
@@ -119,7 +133,16 @@ private:
 
     Constants constants_{};
     State state_;
+    // What the current frame draws into: the window, a render target, or nothing (a frame not drawn).
     bool onscreen_ = false; // in a frame drawn into the window
+    Texture* target_ = nullptr; // in a frame drawn into this render target
+    bool readsBack_ = false;    // the frame's render target is read back at its end
+    Diligent::ITextureView* colorView_ = nullptr;
+    Diligent::ITextureView* depthView_ = nullptr;
+    std::int32_t projectionWidth_ = 0;
+    std::int32_t projectionHeight_ = 0;
+    std::function<void(std::span<const std::byte>)> readBack_;
+    std::vector<std::byte> readPixels_;
     bool draw_ = true;
     std::uint64_t lastSentAt_ = 0;
     int offsetX_ = 0;