From 6e4fe589e553039b349bd2daa77cc71614fa4b9f Mon Sep 17 00:00:00 2001 From: dhkatz Date: Wed, 28 Oct 2026 19:09:48 -0700 Subject: [PATCH] Draw render targets in the renderer process While the renderer process runs, the adapter makes the client's render targets itself and the renderer process draws frames into them: texture targets are drawn with like any texture, and a pixmap target's frame is read back through the channel into its Pixmap (protocol version 4: BeginFrame names its target, CreateRenderTarget, Channel::readSize and the -read event). Frames into render targets are drawn even while the window is hidden. The client uses pixmap targets for snapshots (outfit pictures, photos), checked to look right. The render target interfaces are declared from IMVU's vtables. Co-Authored-By: Claude Opus 5.5 --- adapter/engine.cpp | 41 ++++++-- adapter/engine.hpp | 1 + adapter/pixmap.cpp | 38 +++++++ adapter/pixmap.hpp | 28 ++++- adapter/recorder.cpp | 12 +++ adapter/recorder.hpp | 5 + adapter/renderer_process.cpp | 24 ++++- adapter/renderer_process.hpp | 5 + adapter/resources.cpp | 97 ++++++++++++++++-- adapter/resources.hpp | 50 ++++++++- docs/architecture.md | 13 ++- docs/interface.md | 10 +- include/t5de/imvu/resources.hpp | 35 ++++++- include/t5de/protocol/channel.hpp | 10 +- include/t5de/protocol/commands.hpp | 16 ++- renderer/main.cpp | 12 ++- renderer/scene.cpp | 157 +++++++++++++++++++++++++---- renderer/scene.hpp | 27 ++++- 18 files changed, 523 insertions(+), 58 deletions(-) diff --git a/adapter/engine.cpp b/adapter/engine.cpp index eaa4568..b68a3d7 100644 --- a/adapter/engine.cpp +++ b/adapter/engine.cpp @@ -232,39 +232,62 @@ imvu::PiInternalVertexBuffer* Engine::newDynamicVertexBuffer(std::uint32_t eleme imvu::SharedPtr Engine::newPixmapRenderTarget(imvu::Vec2i size) { log::info("engine: frame {}: a {}x{} pixmap render target", frames_, size.x, size.y); - return inner_->newPixmapRenderTarget(size); + if (!recorder_->active() || size.x <= 0 || size.y <= 0) { + return inner_->newPixmapRenderTarget(size); + } + return imvu::SharedPtr(std::make_unique(recorder_, size)); } imvu::SharedPtr Engine::newTextureRenderTarget(imvu::Vec2i size) { log::info("engine: frame {}: a {}x{} texture render target", frames_, size.x, size.y); - return inner_->newTextureRenderTarget(size); + if (!recorder_->active() || size.x <= 0 || size.y <= 0) { + return inner_->newTextureRenderTarget(size); + } + return imvu::SharedPtr(std::make_unique(recorder_, size)); } void Engine::beginRendering(imvu::PiRenderTarget* target) { - inner_->beginRendering(target); - // Decided per frame: frames into render targets, and frames while the renderer process doesn't cover the view, are - // drawn by the client's renderer. - const bool takenOver = target == nullptr && recorder_->covering(); + // Frames into our render targets are drawn by the renderer process only; the client's renderer doesn't know them. + ourTarget_ = ours(target) ? target : nullptr; + if (ourTarget_ == nullptr) { + inner_->beginRendering(target); + } + // Decided per frame: frames into the client renderer's render targets, and frames while the renderer process + // doesn't cover the view, are drawn by the client's renderer. + const bool takenOver = ourTarget_ != nullptr || (target == nullptr && recorder_->covering()); if (takenOver != takenOver_ && target == nullptr) { log::info("engine: frame {}: {}", frames_, takenOver ? "drawn by the renderer process" : "drawn by the client's renderer"); } takenOver_ = takenOver; if (target != nullptr && foreignLogged_ < 20) { ++foreignLogged_; - log::info("engine: frame {}: drawing into a render target ({})", frames_, static_cast(target)); + log::info("engine: frame {}: drawing into {} render target ({})", frames_, + ourTarget_ != nullptr ? "our" : "the client renderer's", static_cast(target)); } - recorder_->add(Op::BeginFrame, protocol::BeginFrame{target != nullptr}); + protocol::BeginFrame begin{target == nullptr ? 0 : protocol::foreignTarget, 0}; + if (const auto* pixmap = dynamic_cast(ourTarget_)) { + begin = {pixmap->id(), 1}; + } else if (const auto* texture = dynamic_cast(ourTarget_)) { + begin = {texture->id(), 0}; + } + recorder_->add(Op::BeginFrame, begin); } void Engine::endRendering() { - inner_->endRendering(); + if (ourTarget_ == nullptr) { + inner_->endRendering(); + } LARGE_INTEGER now; QueryPerformanceCounter(&now); recorder_->add(Op::EndFrame, protocol::EndFrame{static_cast(now.QuadPart)}); recorder_->flush(true); + if (auto* pixmap = dynamic_cast(ourTarget_)) { + pixmap->setPixels(recorder_->readBack()); + } + ourTarget_ = nullptr; ++frames_; if (frames_ % 300 == 0 && statsLogged_ < 6) { ++statsLogged_; diff --git a/adapter/engine.hpp b/adapter/engine.hpp index 60a9e91..9acb441 100644 --- a/adapter/engine.hpp +++ b/adapter/engine.hpp @@ -73,6 +73,7 @@ private: std::shared_ptr recorder_; std::uint64_t frames_ = 0; bool takenOver_ = false; // this frame is drawn by the renderer process only + imvu::PiRenderTarget* ourTarget_ = nullptr; // this frame's render target, if it's one of ours int foreignLogged_ = 0; // diagnostics: textures the client renderer made, render targets // Diagnostics: what the client draws, summed over 300 frames and logged (the first few times). diff --git a/adapter/pixmap.cpp b/adapter/pixmap.cpp index cb2ee82..9245931 100644 --- a/adapter/pixmap.cpp +++ b/adapter/pixmap.cpp @@ -8,12 +8,20 @@ namespace { using Getter = unsigned(__thiscall*)(const imvu::Pixmap*); using PixelsGetter = std::uint32_t*(__thiscall*)(const imvu::Pixmap*); +using SizeConstructor = imvu::Pixmap*(__thiscall*)(void*, int, int); +using DefaultConstructor = imvu::Pixmap*(__thiscall*)(void*); +using CopyConstructor = imvu::Pixmap*(__thiscall*)(void*, const imvu::Pixmap*); +using Destructor = void(__thiscall*)(imvu::Pixmap*); struct Methods { Getter width = nullptr; Getter height = nullptr; Getter rowBytes = nullptr; PixelsGetter pixels = nullptr; + SizeConstructor construct = nullptr; + DefaultConstructor constructEmpty = nullptr; + CopyConstructor copy = nullptr; + Destructor destroy = nullptr; }; const Methods& methods() @@ -25,6 +33,10 @@ const Methods& methods() found.height = reinterpret_cast(GetProcAddress(module, "?height@Pixmap@@QBEIXZ")); found.rowBytes = reinterpret_cast(GetProcAddress(module, "?rowBytes@Pixmap@@QBEIXZ")); found.pixels = reinterpret_cast(GetProcAddress(module, "?pixels@Pixmap@@QBEPAIXZ")); + found.construct = reinterpret_cast(GetProcAddress(module, "??0Pixmap@@QAE@HH@Z")); + found.constructEmpty = reinterpret_cast(GetProcAddress(module, "??0Pixmap@@QAE@XZ")); + found.copy = reinterpret_cast(GetProcAddress(module, "??0Pixmap@@QAE@ABV0@@Z")); + found.destroy = reinterpret_cast(GetProcAddress(module, "??1Pixmap@@QAE@XZ")); } return found; }(); @@ -42,4 +54,30 @@ std::optional pixmap_pixels(const imvu::Pixmap& pixmap) return PixmapPixels{m.width(&pixmap), m.height(&pixmap), m.rowBytes(&pixmap), m.pixels(&pixmap)}; } +OwnedPixmap::OwnedPixmap(int width, int height) +{ + const Methods& m = methods(); + if (m.construct != nullptr && m.constructEmpty != nullptr && m.copy != nullptr && m.destroy != nullptr) { + m.construct(storage_, width, height); + valid_ = true; + } +} + +OwnedPixmap::~OwnedPixmap() +{ + if (valid_) { + methods().destroy(reinterpret_cast(storage_)); + } +} + +imvu::Pixmap* OwnedPixmap::copyTo(imvu::Pixmap* result) const +{ + const Methods& m = methods(); + if (valid_) { + return m.copy(result, &get()); + } + // Without pixmap.dll's methods there's nothing to construct one with (IMVU's own code would have failed already). + return m.constructEmpty != nullptr ? m.constructEmpty(result) : result; +} + } // namespace t5de diff --git a/adapter/pixmap.hpp b/adapter/pixmap.hpp index 8ac18fe..d918e41 100644 --- a/adapter/pixmap.hpp +++ b/adapter/pixmap.hpp @@ -1,6 +1,8 @@ #pragma once -// Reading IMVU's images (Pixmap) through pixmap.dll's exported methods, rather than depending on their layout. +// IMVU's images (Pixmap), through pixmap.dll's exported methods rather than depending on their layout (only their +// size, for making our own). +#include #include #include @@ -12,10 +14,32 @@ struct PixmapPixels { std::uint32_t width; std::uint32_t height; std::uint32_t rowBytes; - const std::uint32_t* pixels; // 0xAARRGGBB, rows rowBytes apart + std::uint32_t* pixels; // 0xAARRGGBB, rows rowBytes apart }; // Null if pixmap.dll's methods can't be found. std::optional pixmap_pixels(const imvu::Pixmap& pixmap); +// A Pixmap of our own, made and destroyed by pixmap.dll. +class OwnedPixmap { +public: + // Width by height; invalid if pixmap.dll's methods can't be found. + OwnedPixmap(int width, int height); + OwnedPixmap(const OwnedPixmap&) = delete; + OwnedPixmap& operator=(const OwnedPixmap&) = delete; + ~OwnedPixmap(); + + [[nodiscard]] bool valid() const { return valid_; } + [[nodiscard]] const imvu::Pixmap& get() const { return *reinterpret_cast(storage_); } + + // Constructs a copy in result (raw memory for a Pixmap, as for a Pixmap returned by value); an empty Pixmap if + // this one isn't valid. + imvu::Pixmap* copyTo(imvu::Pixmap* result) const; + +private: + // sizeof(Pixmap) in pixmap.dll: its constructors set 7 fields. + alignas(4) std::byte storage_[28]; + bool valid_ = false; +}; + } // namespace t5de diff --git a/adapter/recorder.cpp b/adapter/recorder.cpp index 3c72ef7..723f50e 100644 --- a/adapter/recorder.cpp +++ b/adapter/recorder.cpp @@ -69,6 +69,18 @@ void Recorder::addPixels(protocol::Op op, std::uint32_t id, const imvu::Pixmap& flushIfLarge(); } +std::uint32_t Recorder::createRenderTarget(std::uint32_t width, std::uint32_t height) +{ + const std::uint32_t id = nextId_++; + add(protocol::Op::CreateRenderTarget, protocol::CreateRenderTarget{id, width, height}); + return id; +} + +std::span Recorder::readBack() +{ + return active() ? process_->readBack() : std::span{}; +} + std::uint32_t Recorder::createTexture(const imvu::Pixmap& pixmap, std::uint32_t flags) { const std::uint32_t id = nextId_++; diff --git a/adapter/recorder.hpp b/adapter/recorder.hpp index fa114e7..d9f80cd 100644 --- a/adapter/recorder.hpp +++ b/adapter/recorder.hpp @@ -5,6 +5,7 @@ #include #include +#include #include "renderer_process.hpp" #include "t5de/imvu/types.hpp" @@ -37,6 +38,10 @@ public: void flushIfLarge(); std::uint32_t createTexture(const imvu::Pixmap& pixmap, std::uint32_t flags); + // Destroyed with destroyTexture. + std::uint32_t createRenderTarget(std::uint32_t width, std::uint32_t height); + // After flushing a frame that reads its render target back: its pixels (RendererProcess::readBack). + std::span readBack(); void updateTexture(std::uint32_t id, const imvu::Pixmap& pixmap); void destroyTexture(std::uint32_t id); diff --git a/adapter/renderer_process.cpp b/adapter/renderer_process.cpp index 2f96ebe..09ebef2 100644 --- a/adapter/renderer_process.cpp +++ b/adapter/renderer_process.cpp @@ -72,7 +72,9 @@ std::unique_ptr RendererProcess::start(HWND window, protocol::V static_cast(MapViewOfFile(renderer->mapping_, FILE_MAP_ALL_ACCESS, 0, 0, size)); renderer->ready_ = CreateEventW(nullptr, FALSE, FALSE, (name + protocol::readyEventSuffix).c_str()); renderer->consumed_ = CreateEventW(nullptr, FALSE, TRUE, (name + protocol::consumedEventSuffix).c_str()); - if (renderer->channel_ == nullptr || renderer->ready_ == nullptr || renderer->consumed_ == nullptr) { + renderer->read_ = CreateEventW(nullptr, FALSE, FALSE, (name + protocol::readEventSuffix).c_str()); + if (renderer->channel_ == nullptr || renderer->ready_ == nullptr || renderer->consumed_ == nullptr + || renderer->read_ == nullptr) { log::info("renderer process: can't set up the channel: error {}", GetLastError()); return nullptr; } @@ -86,7 +88,7 @@ std::unique_ptr RendererProcess::start(HWND window, protocol::V .chunkSize = 0, .chunkFlags = 0, .covering = 0, - .reserved = 0}; + .readSize = 0}; // The renderer's window goes on top of the client's 3D view, as a child window. The client's window mustn't clip // its children (WS_CLIPCHILDREN, which the client creates it with): the client draws a frame when its window gets @@ -117,7 +119,7 @@ RendererProcess::~RendererProcess() } UnmapViewOfFile(channel_); } - for (HANDLE handle : {ready_, consumed_, mapping_, process_}) { + for (HANDLE handle : {ready_, consumed_, read_, mapping_, process_}) { if (handle != nullptr) { CloseHandle(handle); } @@ -150,4 +152,20 @@ bool RendererProcess::send(std::span commands, bool endsFrame) return true; } +std::span RendererProcess::readBack() +{ + if (!alive_) { + return {}; + } + const HANDLE waits[] = {read_, process_}; + const DWORD result = WaitForMultipleObjects(2, waits, FALSE, consumeTimeout); + if (result != WAIT_OBJECT_0) { + log::info("renderer process: {} reading back a frame; giving up on it", + result == WAIT_OBJECT_0 + 1 ? "exited" : "not responding"); + alive_ = false; + return {}; + } + return {data_, std::min(channel_->readSize, channel_->capacity)}; +} + } // namespace t5de diff --git a/adapter/renderer_process.hpp b/adapter/renderer_process.hpp index 5285b69..4ff1a3c 100644 --- a/adapter/renderer_process.hpp +++ b/adapter/renderer_process.hpp @@ -25,6 +25,10 @@ public: // process is gone or stopped taking them (it's then given up on). bool send(std::span commands, bool endsFrame); + // After sending a frame that reads its render target back: waits for the pixels (protocol::Channel), valid until + // the next send. Empty if the renderer process couldn't, or is gone. + std::span readBack(); + [[nodiscard]] bool alive() const { return alive_; } // Whether the renderer process covers the whole 3D view (protocol::Channel::covering). @@ -36,6 +40,7 @@ private: HANDLE mapping_ = nullptr; HANDLE ready_ = nullptr; HANDLE consumed_ = nullptr; + HANDLE read_ = nullptr; HANDLE process_ = nullptr; protocol::Channel* channel_ = nullptr; std::byte* data_ = nullptr; diff --git a/adapter/resources.cpp b/adapter/resources.cpp index 872b48e..173e3ce 100644 --- a/adapter/resources.cpp +++ b/adapter/resources.cpp @@ -1,8 +1,13 @@ #include "resources.hpp" +#include +#include #include +#include #include +#include "log.hpp" + namespace t5de { namespace { @@ -48,28 +53,48 @@ Texture::Texture(imvu::PiRenderTexture* inner, std::shared_ptr recorde add(static_cast(this)); } +Texture::Texture(std::shared_ptr recorder, std::uint32_t targetId, imvu::Vec2i size) + : inner_(nullptr), recorder_(std::move(recorder)), id_(targetId), size_(size), ownsId_(false) +{ + add(static_cast(this)); +} + Texture::~Texture() { remove(static_cast(this)); delete inner_; // through its virtual destructor: IMVU's code frees it - recorder_->destroyTexture(id_); + if (ownsId_) { + recorder_->destroyTexture(id_); + } } -imvu::Vec2i Texture::size() const { return inner_->size(); } +imvu::Vec2i Texture::size() const { return inner_ != nullptr ? inner_->size() : size_; } void Texture::updateRegions(const imvu::Pixmap& pixmap, const imvu::CompoundRegion& regions) { - inner_->updateRegions(pixmap, regions); - recorder_->updateTexture(id_, pixmap); // all of it: the regions' layout isn't known + if (inner_ != nullptr) { + inner_->updateRegions(pixmap, regions); + recorder_->updateTexture(id_, pixmap); // all of it: the regions' layout isn't known + } } void Texture::update(const imvu::Pixmap& pixmap) { - inner_->update(pixmap); - recorder_->updateTexture(id_, pixmap); + if (inner_ != nullptr) { + inner_->update(pixmap); + recorder_->updateTexture(id_, pixmap); + } } -imvu::Pixmap* Texture::read(imvu::Pixmap* result) const { return inner_->read(result); } +imvu::Pixmap* Texture::read(imvu::Pixmap* result) const +{ + if (inner_ != nullptr) { + return inner_->read(result); + } + // A render target's texture: not read back (not seen used); an image of its size. + log::info("texture: a render target's texture read; not supported, returning a blank image"); + return OwnedPixmap(size_.x, size_.y).copyTo(result); +} IndexBuffer::IndexBuffer(imvu::PiInternalIndexBuffer* inner, std::shared_ptr recorder, std::uint32_t id) : inner_(inner), recorder_(std::move(recorder)), id_(id) @@ -105,4 +130,62 @@ void VertexBuffer::update(const void* data, std::uint32_t stride, std::uint32_t void VertexBuffer::unknown2() { inner_->unknown2(); } +PixmapTarget::PixmapTarget(std::shared_ptr recorder, imvu::Vec2i size) + : recorder_(std::move(recorder)), + id_(recorder_->createRenderTarget(static_cast(size.x), static_cast(size.y))), + size_(size), pixmap_(size.x, size.y) +{ + add(static_cast(this)); +} + +PixmapTarget::~PixmapTarget() +{ + remove(static_cast(this)); + recorder_->destroyTexture(id_); +} + +imvu::Pixmap* PixmapTarget::getPixmap(imvu::Pixmap* result) const { return pixmap_.copyTo(result); } + +void PixmapTarget::setPixels(std::span pixels) +{ + const auto into = pixmap_.valid() ? pixmap_pixels(pixmap_.get()) : std::nullopt; + const auto width = static_cast(size_.x); + const auto height = static_cast(size_.y); + const std::size_t rowSize = std::size_t{width} * 4; + if (!into || into->pixels == nullptr || pixels.size() < rowSize * height) { + log::info("render target: {}x{} frame not read back ({} bytes)", width, height, pixels.size()); + return; + } + const std::size_t copied = std::min(rowSize, std::size_t{into->width} * 4); + auto* rows = reinterpret_cast(into->pixels); + for (std::uint32_t y = 0; y < std::min(into->height, height); ++y) { + std::memcpy(rows + std::size_t{y} * into->rowBytes, pixels.data() + y * rowSize, copied); + } +} + +TextureTarget::TextureTarget(std::shared_ptr recorder, imvu::Vec2i size) + : recorder_(std::move(recorder)), + id_(recorder_->createRenderTarget(static_cast(size.x), static_cast(size.y))), + size_(size) +{ + add(static_cast(this)); +} + +TextureTarget::~TextureTarget() +{ + remove(static_cast(this)); + recorder_->destroyTexture(id_); +} + +imvu::TextureArea* TextureTarget::area(imvu::TextureArea* result) const +{ + return new (result) imvu::TextureArea{0, 0, 1, 1}; +} + +imvu::SharedPtr* TextureTarget::texture(imvu::SharedPtr* result) const +{ + return new (result) imvu::SharedPtr( + std::unique_ptr(new Texture(recorder_, id_, size_))); +} + } // namespace t5de diff --git a/adapter/resources.hpp b/adapter/resources.hpp index 6452426..d3e0e8d 100644 --- a/adapter/resources.hpp +++ b/adapter/resources.hpp @@ -3,8 +3,12 @@ // client renderer still draws with) and mirrors it, by id, to the renderer process. The client deletes them through // their virtual destructors. +#include #include #include +#include + +#include "pixmap.hpp" #include "recorder.hpp" #include "t5de/imvu/resources.hpp" @@ -14,6 +18,9 @@ namespace t5de { class Texture final : public imvu::PiRenderTexture { public: Texture(imvu::PiRenderTexture* inner, std::shared_ptr recorder, std::uint32_t id); + // What's been drawn into one of our texture render targets (TextureTarget): no texture of the client renderer's, + // and the id stays the target's. + Texture(std::shared_ptr recorder, std::uint32_t targetId, imvu::Vec2i size); ~Texture() override; imvu::Vec2i size() const override; @@ -28,6 +35,44 @@ private: imvu::PiRenderTexture* inner_; std::shared_ptr recorder_; std::uint32_t id_; + imvu::Vec2i size_{}; + bool ownsId_ = true; +}; + +// Render targets the renderer process draws (made while it runs; otherwise the client renderer makes its own). +class PixmapTarget final : public imvu::PiPixmapRenderTarget { +public: + PixmapTarget(std::shared_ptr recorder, imvu::Vec2i size); + ~PixmapTarget() override; + + imvu::Pixmap* getPixmap(imvu::Pixmap* result) const override; + + // Copies a frame's pixels, read back from the renderer process (rows packed, 0xAARRGGBB), into the pixmap. + void setPixels(std::span pixels); + + [[nodiscard]] std::uint32_t id() const { return id_; } + +private: + std::shared_ptr recorder_; + std::uint32_t id_; + imvu::Vec2i size_; + OwnedPixmap pixmap_; +}; + +class TextureTarget final : public imvu::PiTextureRenderTarget { +public: + TextureTarget(std::shared_ptr recorder, imvu::Vec2i size); + ~TextureTarget() override; + + imvu::TextureArea* area(imvu::TextureArea* result) const override; + imvu::SharedPtr* texture(imvu::SharedPtr* result) const override; + + [[nodiscard]] std::uint32_t id() const { return id_; } + +private: + std::shared_ptr recorder_; + std::uint32_t id_; + imvu::Vec2i size_; }; class IndexBuffer final : public imvu::PiInternalIndexBuffer { @@ -62,8 +107,9 @@ private: std::uint32_t id_; }; -// Whether an object is one of ours (live Texture, IndexBuffer or VertexBuffer): the client can also hand the engine -// objects the client renderer made itself (render targets' textures), which must reach it unchanged. +// Whether an object is one of ours (live Texture, IndexBuffer, VertexBuffer, PixmapTarget or TextureTarget): the client +// can also hand the engine objects the client renderer made itself (its render targets, and their textures), which +// must reach it unchanged. bool ours(const void* object); // The client renderer's object, and our id, for what the client passes in: ours unwrapped, others unchanged (id 0). diff --git a/docs/architecture.md b/docs/architecture.md index 19b1aab..9af4d96 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -39,8 +39,8 @@ The process exits when the engine is destroyed or the client exits. ## Status The renderer process draws the whole 3D view in place of the client's renderer (the full view, the default). The -adapter still forwards everything to the client's Direct3D renderer (state and resources: it draws render targets and -reads textures back, which the renderer process doesn't yet), but while the renderer process covers the whole view +adapter still forwards state and resources to the client's Direct3D renderer (as a fallback, and for reading textures +back, which the renderer process doesn't do), but while the renderer process covers the whole view (`Channel::covering`, set after each frame it presents) the client's renderer skips its clears and draws, decided per frame; it draws again as soon as the renderer process stops covering the view (the view changes, or the process fails). Checked in rooms with screenshots of both renderers side by side (the compare view): they match, particles @@ -70,8 +70,13 @@ For comparing while the client runs (development): the right half visibly trails, mostly from compositing another process's window over Remote Desktop. The renderer process draws only the newest frame when it falls behind, and keeps at most one frame queued. -Next: render targets (offscreen frames are skipped, and drawn by the client's renderer), and one renderer process for -all of a client's views. +Render targets are the renderer process's too, while it runs: the adapter makes its own (`PixmapTarget`, +`TextureTarget`), the renderer process draws frames into them (textures in the window's formats, so the same +pipelines draw both), and a pixmap target's frame is read back through the channel (`Channel::readSize`, the +`-read` event) into its Pixmap, as the client's renderer does, synchronously. A read back must fit in the channel's +data area (16 MB: the client's 2048x2048 snapshots just do). Checked with outfit pictures and photos. + +Next: one renderer process for all of a client's views. ## Building diff --git a/docs/interface.md b/docs/interface.md index 0b33705..41a3167 100644 --- a/docs/interface.md +++ b/docs/interface.md @@ -83,8 +83,14 @@ virtual methods, IMVU's code only uses an engine through helpers that call its v practice: clamped to it they look much smaller and fainter than the client's. - Fog: linear table fog, start 1 and end 50 in rooms. Fogging by view depth matches the client; by depth buffer value (as Direct3D 9's documentation suggests for projections like IMVU's) doesn't. -- Still unknown: render targets, `CompoundRegion` (texture update - regions), `newRenderTexture`'s flags, and what `beginRendering` does with a null target. MSVC orders overloaded +- Render targets (`include/t5de/imvu/resources.hpp`): the engine returns each as a pointer into its own object; + the client calls only their interfaces' methods and passes them to `beginRendering`. A pixmap render target has one + method, `getPixmap()` (a Pixmap returned by value), with the pixels of the last frame drawn into it (IMVU's engine + copies them from the GPU at the frame's end). A texture render target has `area()` (always 0, 0, 1, 1) and + `texture()` (a new texture object for what's been drawn, as a shared pointer returned by value). The client uses + pixmap targets for snapshots (`takeSnapshot`: outfit pictures, the photo widget), 2048x2048; texture targets + haven't been seen used. Pixmap is 28 bytes; pixmap.dll exports its constructors and destructor. +- Still unknown: `CompoundRegion` (texture update regions), `newRenderTexture`'s flags, and what `beginRendering` does with a null target. MSVC orders overloaded virtual methods unlike their declaration, so our declarations never overload virtual methods. - Memory crosses the boundary only through virtual methods (destructors, control blocks), so each side can use its own C runtime. Exceptions must not cross it. diff --git a/include/t5de/imvu/resources.hpp b/include/t5de/imvu/resources.hpp index 640c427..b20295d 100644 --- a/include/t5de/imvu/resources.hpp +++ b/include/t5de/imvu/resources.hpp @@ -6,6 +6,7 @@ #include +#include "t5de/imvu/shared_ptr.hpp" #include "t5de/imvu/types.hpp" namespace t5de::imvu { @@ -39,8 +40,36 @@ public: virtual Pixmap* read(Pixmap* result) const = 0; }; -class PiRenderTarget; -class PiPixmapRenderTarget; -class PiTextureRenderTarget; +// What frames can be drawn into besides the window (PiRenderEngine::beginRendering). IMVU's Direct3D engine returns +// each as a pointer into its own object (single inheritance from these, so the PiRenderTarget pointer is the same). +class PiRenderTarget { +public: + virtual ~PiRenderTarget() = default; +}; + +// Drawn into, then read: after each frame drawn into it, getPixmap() has its pixels (IMVU's engine copies them at the +// end of the frame). Used for snapshots: outfit pictures, the photo widget. +class PiPixmapRenderTarget : public PiRenderTarget { +public: + // Constructs a copy of the pixels in result (a Pixmap returned by value). + virtual Pixmap* getPixmap(Pixmap* result) const = 0; +}; + +// The part of the texture the target covers, in texture coordinates. IMVU's Direct3D engine always returns 0, 0, 1, 1. +struct TextureArea { + float left; + float top; + float right; + float bottom; +}; + +// Drawn into, then drawn with as a texture. +class PiTextureRenderTarget : public PiRenderTarget { +public: + // Constructs the area in result (returned by value). + virtual TextureArea* area(TextureArea* result) const = 0; + // Constructs, in result, a new texture object for what's been drawn (a shared pointer returned by value). + virtual SharedPtr* texture(SharedPtr* result) const = 0; +}; } // namespace t5de::imvu diff --git a/include/t5de/protocol/channel.hpp b/include/t5de/protocol/channel.hpp index 9ab026a..1f606c1 100644 --- a/include/t5de/protocol/channel.hpp +++ b/include/t5de/protocol/channel.hpp @@ -7,7 +7,7 @@ namespace t5de::protocol { -inline constexpr std::uint32_t version = 3; +inline constexpr std::uint32_t version = 4; // Which part of the client's 3D view the renderer process draws. Full is the default; the others are for comparing // with the client's renderer, which then draws the whole view under the renderer process's window. @@ -22,6 +22,11 @@ enum class View : std::uint32_t { // argument: this header, then `capacity` bytes of data. The adapter sends the command stream (commands.hpp) through // it in chunks: it waits for the event "-consumed", writes a chunk and sets "-ready"; the renderer // process copies the chunk out and sets "-consumed" (both auto-reset; consumed starts set). +// +// A frame drawn into a render target can ask for its pixels back (BeginFrame::readBack): having sent it, the adapter +// waits for "-read" (auto-reset), which the renderer process sets once it has drawn the frame and written the +// pixels into the data area (readSize bytes: rows packed, 0xAARRGGBB; 0 if it couldn't). The adapter sends nothing +// meanwhile, so the data area is free. struct Channel { std::uint32_t version; std::uint32_t closing; // nonzero: the engine is gone, the renderer process should exit @@ -34,7 +39,7 @@ struct Channel { // Set by the renderer process while its window covers the whole 3D view and it's presenting frames: the client's // renderer then needn't draw (the adapter skips its draws; it draws again as soon as this is cleared). std::uint32_t covering; - std::uint32_t reserved; + std::uint32_t readSize; }; static_assert(sizeof(Channel) == 48, "same layout in 32-bit and 64-bit builds"); @@ -44,5 +49,6 @@ inline constexpr std::uint32_t channelCapacity = 16 * 1024 * 1024; inline constexpr wchar_t readyEventSuffix[] = L"-ready"; inline constexpr wchar_t consumedEventSuffix[] = L"-consumed"; +inline constexpr wchar_t readEventSuffix[] = L"-read"; } // namespace t5de::protocol diff --git a/include/t5de/protocol/commands.hpp b/include/t5de/protocol/commands.hpp index 808156e..3c9b4ba 100644 --- a/include/t5de/protocol/commands.hpp +++ b/include/t5de/protocol/commands.hpp @@ -33,6 +33,7 @@ enum class Op : std::uint16_t { DrawTriangles, DrawArrays, DrawPointSprites, + CreateRenderTarget, }; struct Header { @@ -44,9 +45,15 @@ struct Header { // Colors are 0xAARRGGBB. Resource ids are the adapter's, per engine; 0 means none. struct BeginFrame { - std::uint32_t offscreen; // drawn into a render target, not the window: nothing to show + // The render target drawn into (CreateRenderTarget's id), 0 for the window, or foreignTarget. + std::uint32_t target; + std::uint32_t readBack; // at the end of the frame, send the render target's pixels back (channel.hpp) }; +// A frame drawn into a render target of the client renderer's own (made while the renderer process wasn't running): +// the client renderer draws it, the renderer process only follows its changes. +inline constexpr std::uint32_t foreignTarget = 0xffffffff; + struct EndFrame { std::uint64_t sentAt; // the adapter's QueryPerformanceCounter (the same clock in every process) }; @@ -131,6 +138,13 @@ struct DestroyTexture { std::uint32_t id; }; +// A texture frames can be drawn into (BeginFrame), and drawn with like any other; destroyed with DestroyTexture. +struct CreateRenderTarget { + std::uint32_t id; + std::uint32_t width; + std::uint32_t height; +}; + enum class BufferKind : std::uint32_t { Index, // 16-bit indices Vertex, // positions and normals (3 floats), texture coordinates (2 floats) or colors (0xAARRGGBB) diff --git a/renderer/main.cpp b/renderer/main.cpp index 29700bf..71af0a1 100644 --- a/renderer/main.cpp +++ b/renderer/main.cpp @@ -184,12 +184,13 @@ int WINAPI wWinMain(HINSTANCE, HINSTANCE, PWSTR, int) HANDLE ready = OpenEventW(SYNCHRONIZE, FALSE, (arguments.channel + t5de::protocol::readyEventSuffix).c_str()); HANDLE consumed = OpenEventW(EVENT_MODIFY_STATE, FALSE, (arguments.channel + t5de::protocol::consumedEventSuffix).c_str()); + HANDLE read = OpenEventW(EVENT_MODIFY_STATE, FALSE, (arguments.channel + t5de::protocol::readEventSuffix).c_str()); HANDLE parent = OpenProcess(SYNCHRONIZE, FALSE, arguments.parent); - if (channel == nullptr || ready == nullptr || consumed == nullptr || parent == nullptr + if (channel == nullptr || ready == nullptr || consumed == nullptr || read == nullptr || parent == nullptr || channel->version != t5de::protocol::version) { return 3; } - const auto* data = reinterpret_cast(channel + 1); + auto* data = reinterpret_cast(channel + 1); const HWND clientWindow = reinterpret_cast(static_cast(channel->window)); const HWND window = create_child_window(clientWindow); @@ -227,6 +228,13 @@ int WINAPI wWinMain(HINSTANCE, HINSTANCE, PWSTR, int) return 6; } t5de::Scene scene(device, context, swapChain); + // Frames read back: the adapter waits for them, sending nothing meanwhile, so the data area is free. + scene.setReadBack([&](std::span pixels) { + const std::size_t size = pixels.size() <= channel->capacity ? pixels.size() : 0; + std::memcpy(data, pixels.data(), size); + channel->readSize = static_cast(size); + SetEvent(read); + }); // Takes the chunk waiting in the channel (after "ready"), and lets the adapter send the next: appends it to // frame, and returns whether it ends a frame. diff --git a/renderer/scene.cpp b/renderer/scene.cpp index 3bc4344..60ded80 100644 --- a/renderer/scene.cpp +++ b/renderer/scene.cpp @@ -102,12 +102,7 @@ bool Scene::execute(std::span commands, int offsetX, bool draw) switch (command->op()) { case Op::BeginFrame: if (const auto begin = command->body()) { - onscreen_ = begin->offscreen == 0 && draw_; - if (onscreen_) { - ITextureView* target = swapChain_->GetCurrentBackBufferRTV(); - context_->SetRenderTargets(1, &target, swapChain_->GetDepthBufferDSV(), - RESOURCE_STATE_TRANSITION_MODE_TRANSITION); - } + beginFrame(*begin); } break; case Op::EndFrame: @@ -115,18 +110,17 @@ bool Scene::execute(std::span commands, int offsetX, bool draw) lastSentAt_ = end->sentAt; } drew = drew || onscreen_; - onscreen_ = false; + endFrame(); break; case Op::Clear: - if (const auto clear = command->body(); clear && onscreen_) { + if (const auto clear = command->body(); clear && colorView_ != nullptr) { if ((clear->flags & 1) != 0) { float color[4]; set_color(color, clear->color); - context_->ClearRenderTarget(swapChain_->GetCurrentBackBufferRTV(), color, - RESOURCE_STATE_TRANSITION_MODE_TRANSITION); + context_->ClearRenderTarget(colorView_, color, RESOURCE_STATE_TRANSITION_MODE_TRANSITION); } if ((clear->flags & 2) != 0) { - context_->ClearDepthStencil(swapChain_->GetDepthBufferDSV(), CLEAR_DEPTH_FLAG, 1.0f, 0, + context_->ClearDepthStencil(depthView_, CLEAR_DEPTH_FLAG, 1.0f, 0, RESOURCE_STATE_TRANSITION_MODE_TRANSITION); } } @@ -184,6 +178,11 @@ bool Scene::execute(std::span commands, int offsetX, bool draw) case Op::UpdateTexture: updateTexture(*command); break; + case Op::CreateRenderTarget: + if (const auto create = command->body()) { + createRenderTarget(*create); + } + break; case Op::DestroyTexture: if (const auto destroy = command->body()) { textures_.erase(destroy->id); @@ -201,17 +200,17 @@ bool Scene::execute(std::span commands, int offsetX, bool draw) } break; case Op::DrawTriangles: - if (const auto triangles = command->body(); triangles && onscreen_) { + if (const auto triangles = command->body(); triangles && colorView_ != nullptr) { drawTriangles(*triangles); } break; case Op::DrawArrays: - if (onscreen_) { + if (colorView_ != nullptr) { drawArrays(*command); } break; case Op::DrawPointSprites: - if (onscreen_) { + if (colorView_ != nullptr) { drawPointSprites(*command); } break; @@ -232,19 +231,107 @@ void Scene::setProjection(const protocol::SetProjection& projection) } constants_.viewport[0] = static_cast(projection.width); constants_.viewport[1] = static_cast(projection.height); - if (!onscreen_) { + projectionWidth_ = projection.width; + projectionHeight_ = projection.height; + setViewport(); +} + +void Scene::setViewport() +{ + if (colorView_ == nullptr || projectionWidth_ <= 0 || projectionHeight_ <= 0) { return; } // The client's projection covers its whole 3D view; the window shows part of it, from offsetX. const SwapChainDesc& window = swapChain_->GetDesc(); Viewport viewport; - viewport.TopLeftX = static_cast(-offsetX_); + viewport.TopLeftX = onscreen_ ? static_cast(-offsetX_) : 0.0f; viewport.TopLeftY = 0; - viewport.Width = static_cast(projection.width); - viewport.Height = static_cast(projection.height); + viewport.Width = static_cast(projectionWidth_); + viewport.Height = static_cast(projectionHeight_); viewport.MinDepth = 0; viewport.MaxDepth = 1; - context_->SetViewports(1, &viewport, window.Width, window.Height); + context_->SetViewports(1, &viewport, onscreen_ ? window.Width : target_->width, + onscreen_ ? window.Height : target_->height); +} + +void Scene::beginFrame(const protocol::BeginFrame& begin) +{ + onscreen_ = false; + target_ = nullptr; + colorView_ = nullptr; + depthView_ = nullptr; + readsBack_ = begin.readBack != 0; + if (begin.target == 0) { + if (draw_) { + onscreen_ = true; + colorView_ = swapChain_->GetCurrentBackBufferRTV(); + depthView_ = swapChain_->GetDepthBufferDSV(); + } + } else if (const auto found = textures_.find(begin.target); found != textures_.end() && found->second.depth) { + target_ = &found->second; + colorView_ = target_->texture->GetDefaultView(TEXTURE_VIEW_RENDER_TARGET); + depthView_ = target_->depth->GetDefaultView(TEXTURE_VIEW_DEPTH_STENCIL); + } + if (colorView_ != nullptr) { + context_->SetRenderTargets(1, &colorView_, depthView_, RESOURCE_STATE_TRANSITION_MODE_TRANSITION); + setViewport(); + } +} + +void Scene::endFrame() +{ + if (readsBack_ && readBack_) { + readBack_(target_ != nullptr ? readTarget(*target_) : std::span{}); + } + if (target_ != nullptr) { + // Unbound, so it can be drawn with (a texture can't be both). + context_->SetRenderTargets(0, nullptr, nullptr, RESOURCE_STATE_TRANSITION_MODE_TRANSITION); + } + onscreen_ = false; + target_ = nullptr; + readsBack_ = false; + colorView_ = nullptr; + depthView_ = nullptr; +} + +std::span Scene::readTarget(const Texture& target) +{ + TextureDesc desc; + desc.Name = "Render target read back"; + desc.Type = RESOURCE_DIM_TEX_2D; + desc.Width = target.width; + desc.Height = target.height; + desc.MipLevels = 1; + desc.Format = TEX_FORMAT_RGBA8_UNORM; + desc.Usage = USAGE_STAGING; + desc.CPUAccessFlags = CPU_ACCESS_READ; + RefCntAutoPtr staging; + device_->CreateTexture(desc, nullptr, &staging); + if (!staging) { + return {}; + } + CopyTextureAttribs copy(target.texture, RESOURCE_STATE_TRANSITION_MODE_TRANSITION, staging, + RESOURCE_STATE_TRANSITION_MODE_TRANSITION); + context_->CopyTexture(copy); + MappedTextureSubresource mapped; + context_->MapTextureSubresource(staging, 0, 0, MAP_READ, MAP_FLAG_NONE, nullptr, mapped); // waits for the GPU + if (mapped.pData == nullptr) { + return {}; + } + // RGBA in memory, to the client's 0xAARRGGBB (BGRA in memory). + readPixels_.resize(std::size_t{target.width} * target.height * 4); + for (std::uint32_t y = 0; y < target.height; ++y) { + const auto* from = static_cast(mapped.pData) + y * mapped.Stride; + auto* to = reinterpret_cast(readPixels_.data()) + std::size_t{y} * target.width * 4; + for (std::uint32_t x = 0; x < target.width; ++x) { + to[x * 4 + 0] = from[x * 4 + 2]; + to[x * 4 + 1] = from[x * 4 + 1]; + to[x * 4 + 2] = from[x * 4 + 0]; + to[x * 4 + 3] = from[x * 4 + 3]; + } + } + context_->UnmapTextureSubresource(staging, 0, 0); + return readPixels_; } void Scene::setState(const protocol::SetState& state) @@ -322,6 +409,38 @@ void Scene::createTexture(const protocol::Command& command) textures_[create->id] = std::move(texture); } +void Scene::createRenderTarget(const protocol::CreateRenderTarget& create) +{ + if (create.width == 0 || create.height == 0) { + return; + } + // The window's formats, so the same pipelines draw into both. + const SwapChainDesc& window = swapChain_->GetDesc(); + TextureDesc desc; + desc.Name = "Client render target"; + desc.Type = RESOURCE_DIM_TEX_2D; + desc.Width = create.width; + desc.Height = create.height; + desc.MipLevels = 1; + desc.Format = window.ColorBufferFormat; + desc.BindFlags = BIND_RENDER_TARGET | BIND_SHADER_RESOURCE; + desc.ClearValue.Format = desc.Format; + Texture target{.width = create.width, .height = create.height, .flags = 0}; + device_->CreateTexture(desc, nullptr, &target.texture); + TextureDesc depth = desc; + depth.Name = "Client render target depth"; + depth.Format = window.DepthBufferFormat; + depth.BindFlags = BIND_DEPTH_STENCIL; + depth.ClearValue.Format = depth.Format; + depth.ClearValue.DepthStencil.Depth = 1; + device_->CreateTexture(depth, nullptr, &target.depth); + if (!target.texture || !target.depth) { + return; + } + target.texture->GetDefaultView(TEXTURE_VIEW_SHADER_RESOURCE)->SetSampler(samplers_[0]); + textures_[create.id] = std::move(target); +} + void Scene::updateTexture(const protocol::Command& command) { const auto update = command.body(); diff --git a/renderer/scene.hpp b/renderer/scene.hpp index d4be637..c75e054 100644 --- a/renderer/scene.hpp +++ b/renderer/scene.hpp @@ -5,9 +5,11 @@ #include #include +#include #include #include #include +#include #include "EngineFactoryD3D11.h" #include "RefCntAutoPtr.hpp" @@ -20,10 +22,14 @@ public: Scene(Diligent::IRenderDevice* device, Diligent::IDeviceContext* context, Diligent::ISwapChain* swapChain); // Carries out one frame's commands. offsetX is where the window starts within the client's 3D view; without draw, - // only the frame's changes are made (textures, buffers, state). True if the frame was drawn into the window (to - // present). + // only the frame's changes are made (textures, buffers, state) and frames into the window aren't drawn (frames into + // render targets always are). True if the frame was drawn into the window (to present). bool execute(std::span commands, int offsetX, bool draw); + // Receives the pixels of frames that read their render target back (protocol::BeginFrame::readBack): rows packed, + // 0xAARRGGBB; empty if they couldn't be read. Valid during the call. + void setReadBack(std::function)> readBack) { readBack_ = std::move(readBack); } + // Replaces the shaders' source (the built-in one by default): for trying changes while the client runs. void setShaderSource(std::string source); @@ -33,6 +39,7 @@ public: private: struct Texture { Diligent::RefCntAutoPtr texture; + Diligent::RefCntAutoPtr depth; // render targets only std::uint32_t width = 0; std::uint32_t height = 0; std::uint32_t flags = 0; @@ -86,6 +93,13 @@ private: }; void createTexture(const protocol::Command& command); + void createRenderTarget(const protocol::CreateRenderTarget& create); + void beginFrame(const protocol::BeginFrame& begin); + void endFrame(); + // Sets the viewport for the last projection, in what the frame draws into. + void setViewport(); + // The render target's pixels, rows packed, 0xAARRGGBB; empty if it can't be read. + std::span readTarget(const Texture& target); void updateTexture(const protocol::Command& command); void createBuffer(const protocol::Command& command); void updateBuffer(const protocol::Command& command); @@ -119,7 +133,16 @@ private: Constants constants_{}; State state_; + // What the current frame draws into: the window, a render target, or nothing (a frame not drawn). bool onscreen_ = false; // in a frame drawn into the window + Texture* target_ = nullptr; // in a frame drawn into this render target + bool readsBack_ = false; // the frame's render target is read back at its end + Diligent::ITextureView* colorView_ = nullptr; + Diligent::ITextureView* depthView_ = nullptr; + std::int32_t projectionWidth_ = 0; + std::int32_t projectionHeight_ = 0; + std::function)> readBack_; + std::vector readPixels_; bool draw_ = true; std::uint64_t lastSentAt_ = 0; int offsetX_ = 0;