Draw render targets in the renderer process

While the renderer process runs, the adapter makes the client's render targets
itself and the renderer process draws frames into them: texture targets are
drawn with like any texture, and a pixmap target's frame is read back through
the channel into its Pixmap (protocol version 4: BeginFrame names its target,
CreateRenderTarget, Channel::readSize and the -read event). Frames into render
targets are drawn even while the window is hidden.

The client uses pixmap targets for snapshots (outfit pictures, photos), checked
to look right. The render target interfaces are declared from IMVU's vtables.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

renderer/main.cpp

@@ -184,12 +184,13 @@ int WINAPI wWinMain(HINSTANCE, HINSTANCE, PWSTR, int)
     HANDLE ready = OpenEventW(SYNCHRONIZE, FALSE, (arguments.channel + t5de::protocol::readyEventSuffix).c_str());
     HANDLE consumed =
         OpenEventW(EVENT_MODIFY_STATE, FALSE, (arguments.channel + t5de::protocol::consumedEventSuffix).c_str());
+    HANDLE read = OpenEventW(EVENT_MODIFY_STATE, FALSE, (arguments.channel + t5de::protocol::readEventSuffix).c_str());
     HANDLE parent = OpenProcess(SYNCHRONIZE, FALSE, arguments.parent);
-    if (channel == nullptr || ready == nullptr || consumed == nullptr || parent == nullptr
+    if (channel == nullptr || ready == nullptr || consumed == nullptr || read == nullptr || parent == nullptr
         || channel->version != t5de::protocol::version) {
         return 3;
     }
-    const auto* data = reinterpret_cast<const std::byte*>(channel + 1);
+    auto* data = reinterpret_cast<std::byte*>(channel + 1);
 
     const HWND clientWindow = reinterpret_cast<HWND>(static_cast<std::uintptr_t>(channel->window));
     const HWND window = create_child_window(clientWindow);
@@ -227,6 +228,13 @@ int WINAPI wWinMain(HINSTANCE, HINSTANCE, PWSTR, int)
         return 6;
     }
     t5de::Scene scene(device, context, swapChain);
+    // Frames read back: the adapter waits for them, sending nothing meanwhile, so the data area is free.
+    scene.setReadBack([&](std::span<const std::byte> pixels) {
+        const std::size_t size = pixels.size() <= channel->capacity ? pixels.size() : 0;
+        std::memcpy(data, pixels.data(), size);
+        channel->readSize = static_cast<std::uint32_t>(size);
+        SetEvent(read);
+    });
 
     // Takes the chunk waiting in the channel (after "ready"), and lets the adapter send the next: appends it to
     // frame, and returns whether it ends a frame.

renderer/scene.cpp

@@ -102,12 +102,7 @@ bool Scene::execute(std::span<const std::byte> commands, int offsetX, bool draw)
         switch (command->op()) {
         case Op::BeginFrame:
             if (const auto begin = command->body<protocol::BeginFrame>()) {
-                onscreen_ = begin->offscreen == 0 && draw_;
-                if (onscreen_) {
-                    ITextureView* target = swapChain_->GetCurrentBackBufferRTV();
-                    context_->SetRenderTargets(1, &target, swapChain_->GetDepthBufferDSV(),
-                                               RESOURCE_STATE_TRANSITION_MODE_TRANSITION);
-                }
+                beginFrame(*begin);
             }
             break;
         case Op::EndFrame:
@@ -115,18 +110,17 @@ bool Scene::execute(std::span<const std::byte> commands, int offsetX, bool draw)
                 lastSentAt_ = end->sentAt;
             }
             drew = drew || onscreen_;
-            onscreen_ = false;
+            endFrame();
             break;
         case Op::Clear:
-            if (const auto clear = command->body<protocol::Clear>(); clear && onscreen_) {
+            if (const auto clear = command->body<protocol::Clear>(); clear && colorView_ != nullptr) {
                 if ((clear->flags & 1) != 0) {
                     float color[4];
                     set_color(color, clear->color);
-                    context_->ClearRenderTarget(swapChain_->GetCurrentBackBufferRTV(), color,
-                                                RESOURCE_STATE_TRANSITION_MODE_TRANSITION);
+                    context_->ClearRenderTarget(colorView_, color, RESOURCE_STATE_TRANSITION_MODE_TRANSITION);
                 }
                 if ((clear->flags & 2) != 0) {
-                    context_->ClearDepthStencil(swapChain_->GetDepthBufferDSV(), CLEAR_DEPTH_FLAG, 1.0f, 0,
+                    context_->ClearDepthStencil(depthView_, CLEAR_DEPTH_FLAG, 1.0f, 0,
                                                 RESOURCE_STATE_TRANSITION_MODE_TRANSITION);
                 }
             }
@@ -184,6 +178,11 @@ bool Scene::execute(std::span<const std::byte> commands, int offsetX, bool draw)
         case Op::UpdateTexture:
             updateTexture(*command);
             break;
+        case Op::CreateRenderTarget:
+            if (const auto create = command->body<protocol::CreateRenderTarget>()) {
+                createRenderTarget(*create);
+            }
+            break;
         case Op::DestroyTexture:
             if (const auto destroy = command->body<protocol::DestroyTexture>()) {
                 textures_.erase(destroy->id);
@@ -201,17 +200,17 @@ bool Scene::execute(std::span<const std::byte> commands, int offsetX, bool draw)
             }
             break;
         case Op::DrawTriangles:
-            if (const auto triangles = command->body<protocol::DrawTriangles>(); triangles && onscreen_) {
+            if (const auto triangles = command->body<protocol::DrawTriangles>(); triangles && colorView_ != nullptr) {
                 drawTriangles(*triangles);
             }
             break;
         case Op::DrawArrays:
-            if (onscreen_) {
+            if (colorView_ != nullptr) {
                 drawArrays(*command);
             }
             break;
         case Op::DrawPointSprites:
-            if (onscreen_) {
+            if (colorView_ != nullptr) {
                 drawPointSprites(*command);
             }
             break;
@@ -232,19 +231,107 @@ void Scene::setProjection(const protocol::SetProjection& projection)
     }
     constants_.viewport[0] = static_cast<float>(projection.width);
     constants_.viewport[1] = static_cast<float>(projection.height);
-    if (!onscreen_) {
+    projectionWidth_ = projection.width;
+    projectionHeight_ = projection.height;
+    setViewport();
+}
+
+void Scene::setViewport()
+{
+    if (colorView_ == nullptr || projectionWidth_ <= 0 || projectionHeight_ <= 0) {
         return;
     }
     // The client's projection covers its whole 3D view; the window shows part of it, from offsetX.
     const SwapChainDesc& window = swapChain_->GetDesc();
     Viewport viewport;
-    viewport.TopLeftX = static_cast<float>(-offsetX_);
+    viewport.TopLeftX = onscreen_ ? static_cast<float>(-offsetX_) : 0.0f;
     viewport.TopLeftY = 0;
-    viewport.Width = static_cast<float>(projection.width);
-    viewport.Height = static_cast<float>(projection.height);
+    viewport.Width = static_cast<float>(projectionWidth_);
+    viewport.Height = static_cast<float>(projectionHeight_);
     viewport.MinDepth = 0;
     viewport.MaxDepth = 1;
-    context_->SetViewports(1, &viewport, window.Width, window.Height);
+    context_->SetViewports(1, &viewport, onscreen_ ? window.Width : target_->width,
+                           onscreen_ ? window.Height : target_->height);
+}
+
+void Scene::beginFrame(const protocol::BeginFrame& begin)
+{
+    onscreen_ = false;
+    target_ = nullptr;
+    colorView_ = nullptr;
+    depthView_ = nullptr;
+    readsBack_ = begin.readBack != 0;
+    if (begin.target == 0) {
+        if (draw_) {
+            onscreen_ = true;
+            colorView_ = swapChain_->GetCurrentBackBufferRTV();
+            depthView_ = swapChain_->GetDepthBufferDSV();
+        }
+    } else if (const auto found = textures_.find(begin.target); found != textures_.end() && found->second.depth) {
+        target_ = &found->second;
+        colorView_ = target_->texture->GetDefaultView(TEXTURE_VIEW_RENDER_TARGET);
+        depthView_ = target_->depth->GetDefaultView(TEXTURE_VIEW_DEPTH_STENCIL);
+    }
+    if (colorView_ != nullptr) {
+        context_->SetRenderTargets(1, &colorView_, depthView_, RESOURCE_STATE_TRANSITION_MODE_TRANSITION);
+        setViewport();
+    }
+}
+
+void Scene::endFrame()
+{
+    if (readsBack_ && readBack_) {
+        readBack_(target_ != nullptr ? readTarget(*target_) : std::span<const std::byte>{});
+    }
+    if (target_ != nullptr) {
+        // Unbound, so it can be drawn with (a texture can't be both).
+        context_->SetRenderTargets(0, nullptr, nullptr, RESOURCE_STATE_TRANSITION_MODE_TRANSITION);
+    }
+    onscreen_ = false;
+    target_ = nullptr;
+    readsBack_ = false;
+    colorView_ = nullptr;
+    depthView_ = nullptr;
+}
+
+std::span<const std::byte> Scene::readTarget(const Texture& target)
+{
+    TextureDesc desc;
+    desc.Name = "Render target read back";
+    desc.Type = RESOURCE_DIM_TEX_2D;
+    desc.Width = target.width;
+    desc.Height = target.height;
+    desc.MipLevels = 1;
+    desc.Format = TEX_FORMAT_RGBA8_UNORM;
+    desc.Usage = USAGE_STAGING;
+    desc.CPUAccessFlags = CPU_ACCESS_READ;
+    RefCntAutoPtr<ITexture> staging;
+    device_->CreateTexture(desc, nullptr, &staging);
+    if (!staging) {
+        return {};
+    }
+    CopyTextureAttribs copy(target.texture, RESOURCE_STATE_TRANSITION_MODE_TRANSITION, staging,
+                            RESOURCE_STATE_TRANSITION_MODE_TRANSITION);
+    context_->CopyTexture(copy);
+    MappedTextureSubresource mapped;
+    context_->MapTextureSubresource(staging, 0, 0, MAP_READ, MAP_FLAG_NONE, nullptr, mapped); // waits for the GPU
+    if (mapped.pData == nullptr) {
+        return {};
+    }
+    // RGBA in memory, to the client's 0xAARRGGBB (BGRA in memory).
+    readPixels_.resize(std::size_t{target.width} * target.height * 4);
+    for (std::uint32_t y = 0; y < target.height; ++y) {
+        const auto* from = static_cast<const std::uint8_t*>(mapped.pData) + y * mapped.Stride;
+        auto* to = reinterpret_cast<std::uint8_t*>(readPixels_.data()) + std::size_t{y} * target.width * 4;
+        for (std::uint32_t x = 0; x < target.width; ++x) {
+            to[x * 4 + 0] = from[x * 4 + 2];
+            to[x * 4 + 1] = from[x * 4 + 1];
+            to[x * 4 + 2] = from[x * 4 + 0];
+            to[x * 4 + 3] = from[x * 4 + 3];
+        }
+    }
+    context_->UnmapTextureSubresource(staging, 0, 0);
+    return readPixels_;
 }
 
 void Scene::setState(const protocol::SetState& state)
@@ -322,6 +409,38 @@ void Scene::createTexture(const protocol::Command& command)
     textures_[create->id] = std::move(texture);
 }
 
+void Scene::createRenderTarget(const protocol::CreateRenderTarget& create)
+{
+    if (create.width == 0 || create.height == 0) {
+        return;
+    }
+    // The window's formats, so the same pipelines draw into both.
+    const SwapChainDesc& window = swapChain_->GetDesc();
+    TextureDesc desc;
+    desc.Name = "Client render target";
+    desc.Type = RESOURCE_DIM_TEX_2D;
+    desc.Width = create.width;
+    desc.Height = create.height;
+    desc.MipLevels = 1;
+    desc.Format = window.ColorBufferFormat;
+    desc.BindFlags = BIND_RENDER_TARGET | BIND_SHADER_RESOURCE;
+    desc.ClearValue.Format = desc.Format;
+    Texture target{.width = create.width, .height = create.height, .flags = 0};
+    device_->CreateTexture(desc, nullptr, &target.texture);
+    TextureDesc depth = desc;
+    depth.Name = "Client render target depth";
+    depth.Format = window.DepthBufferFormat;
+    depth.BindFlags = BIND_DEPTH_STENCIL;
+    depth.ClearValue.Format = depth.Format;
+    depth.ClearValue.DepthStencil.Depth = 1;
+    device_->CreateTexture(depth, nullptr, &target.depth);
+    if (!target.texture || !target.depth) {
+        return;
+    }
+    target.texture->GetDefaultView(TEXTURE_VIEW_SHADER_RESOURCE)->SetSampler(samplers_[0]);
+    textures_[create.id] = std::move(target);
+}
+
 void Scene::updateTexture(const protocol::Command& command)
 {
     const auto update = command.body<protocol::UpdateTexture>();

renderer/scene.hpp

@@ -5,9 +5,11 @@
 
 #include <cstddef>
 #include <cstdint>
+#include <functional>
 #include <span>
 #include <string>
 #include <unordered_map>
+#include <vector>
 
 #include "EngineFactoryD3D11.h"
 #include "RefCntAutoPtr.hpp"
@@ -20,10 +22,14 @@ public:
     Scene(Diligent::IRenderDevice* device, Diligent::IDeviceContext* context, Diligent::ISwapChain* swapChain);
 
     // Carries out one frame's commands. offsetX is where the window starts within the client's 3D view; without draw,
-    // only the frame's changes are made (textures, buffers, state). True if the frame was drawn into the window (to
-    // present).
+    // only the frame's changes are made (textures, buffers, state) and frames into the window aren't drawn (frames into
+    // render targets always are). True if the frame was drawn into the window (to present).
     bool execute(std::span<const std::byte> commands, int offsetX, bool draw);
 
+    // Receives the pixels of frames that read their render target back (protocol::BeginFrame::readBack): rows packed,
+    // 0xAARRGGBB; empty if they couldn't be read. Valid during the call.
+    void setReadBack(std::function<void(std::span<const std::byte>)> readBack) { readBack_ = std::move(readBack); }
+
     // Replaces the shaders' source (the built-in one by default): for trying changes while the client runs.
     void setShaderSource(std::string source);
 
@@ -33,6 +39,7 @@ public:
 private:
     struct Texture {
         Diligent::RefCntAutoPtr<Diligent::ITexture> texture;
+        Diligent::RefCntAutoPtr<Diligent::ITexture> depth; // render targets only
         std::uint32_t width = 0;
         std::uint32_t height = 0;
         std::uint32_t flags = 0;
@@ -86,6 +93,13 @@ private:
     };
 
     void createTexture(const protocol::Command& command);
+    void createRenderTarget(const protocol::CreateRenderTarget& create);
+    void beginFrame(const protocol::BeginFrame& begin);
+    void endFrame();
+    // Sets the viewport for the last projection, in what the frame draws into.
+    void setViewport();
+    // The render target's pixels, rows packed, 0xAARRGGBB; empty if it can't be read.
+    std::span<const std::byte> readTarget(const Texture& target);
     void updateTexture(const protocol::Command& command);
     void createBuffer(const protocol::Command& command);
     void updateBuffer(const protocol::Command& command);
@@ -119,7 +133,16 @@ private:
 
     Constants constants_{};
     State state_;
+    // What the current frame draws into: the window, a render target, or nothing (a frame not drawn).
     bool onscreen_ = false; // in a frame drawn into the window
+    Texture* target_ = nullptr; // in a frame drawn into this render target
+    bool readsBack_ = false;    // the frame's render target is read back at its end
+    Diligent::ITextureView* colorView_ = nullptr;
+    Diligent::ITextureView* depthView_ = nullptr;
+    std::int32_t projectionWidth_ = 0;
+    std::int32_t projectionHeight_ = 0;
+    std::function<void(std::span<const std::byte>)> readBack_;
+    std::vector<std::byte> readPixels_;
     bool draw_ = true;
     std::uint64_t lastSentAt_ = 0;
     int offsetX_ = 0;