Show T5 products in the client's shop; keep float IDs and unknown paths off IMVU

The shop's T5 filter (maturity_rating "t5") is answered from the hub's products, fetched so they can be
shown and tried on; buying T5 products adds them to the hub library, and product.php details for
community products are answered here.

The client sends some product IDs as floats ("2000000008.0"): product.php passed the worn community
product to IMVU that way. Every ID is now read either way, and products.Guard refuses any request to
IMVU that still names a local ID given out. T5DE_PROXY_DUMP logs request and response bodies.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

README.md

@@ -44,7 +44,8 @@ itself:
   checksum): for a product deriving from an IMVU product, IMVU is asked about the parent and the proxy adds its own
   entry (the client needs the parent's derivation chain).
 - `productdata/<local ID>/<revision>`: the pack built into a CFL.
-- `api/shop/product.php` (putting a product on asks whether it's a bundle).
+- `api/shop/product.php`: putting a product on asks whether it's a bundle (not), and the shop asks for product
+  details (answered from the hub's listing, or the pack; IMVU is asked only about its own products).
 
 IMVU never sees community products: they're taken out of outfit saves (`api/base_look.php`) and outfit chat
 commands (`*use`, `*putOn`, `*putOnOutfit`, `*remove`, `*takeOff` on `/chat/` queues; a command left empty is
@@ -53,6 +54,9 @@ products with the outfit they were saved with, and adds them back when the clien
 (`base_look.php`, and `look` in the login response); an outfit changed elsewhere comes back without them.
 Requests it changes are re-signed (`X-imvu-auth`) with the session key from the login response, which never
 leaves the machine.
+The client sends some product IDs as floats (`2000000008.0`), so every ID is read either way (`parseID`). As a
+last check, `products.Guard` refuses any request to IMVU that still names a local ID given out, and logs it
+(`products: refused a request to ...`): such a request is a path not handled yet.
 
 ## The hub
 
@@ -69,6 +73,17 @@ and what they wear; the proxy gives the client those of their products it has pa
 from them, and adds them to the whole outfits they send. It also keeps community products out of the outfit the
 client posts to IMVU's chat participant endpoint.
 
+The shop (the client's `t5de-shop` patch adds a T5 choice to its rating filter): searches with
+`maturity_rating` `t5` (`api/shop/search_st.php`) are answered from the hub's products (`GET /api/v1/products`,
+filtered by the shop's categories against each product's IMVU parent's category path), fetching listed products
+that aren't here yet (up to 15 seconds) so they can be shown and tried on. Buying T5 products
+(`api/service/shop/svc_catalog_purchase.php`) adds them to the person's hub library; IMVU products in the same
+purchase go on to IMVU without them.
+
+For finding out what IMVU's APIs send, `T5DE_PROXY_DUMP=<text>` logs the request and response bodies of requests
+whose path contains it (test accounts only: bodies hold personal data). In Git Bash, keep the value from starting
+with `/` (or set `MSYS_NO_PATHCONV=1`), or it's turned into a Windows path.
+
 ## Building
 
     go test ./...

cmd/t5de-proxy/main.go

@@ -153,6 +153,10 @@ func run(logger *log.Logger, listen, ready string, parentPID int, trace bool, pa
 		fetcher.OnLibrary = prods.RefreshInventory
 		prods.Want = fetcher.Want
 		prods.HubDir, prods.Owned = hubProducts, fetcher.Owned
+		prods.Shop, prods.Fetch = client.Products, fetcher.Ensure
+		prods.AddToLibrary = func(ctx context.Context, id string) error {
+			return client.Call(ctx, "PUT", "/api/v1/library/"+id, map[string]any{}, nil)
+		}
 		prods.OnHide = func(id string) {
 			if fetch.IsHubID(id) {
 				go func() {
@@ -164,11 +168,15 @@ func run(logger *log.Logger, listen, ready string, parentPID int, trace bool, pa
 		}
 	}
 	manager.Start()
+	chain := proxy.Interceptors{manager}
 	if prods != nil {
-		handler.Interceptor = proxy.Interceptors{manager, prods}
-	} else {
-		handler.Interceptor = manager
+		chain = append(chain, prods)
 	}
+	chain = append(chain, proxy.Dump{Match: os.Getenv("T5DE_PROXY_DUMP"), Log: logger})
+	if prods != nil {
+		chain = append(chain, products.Guard{Products: prods}) // last: nothing naming a community product reaches IMVU
+	}
+	handler.Interceptor = chain
 	server := &http.Server{
 		Handler:           handler,
 		ReadHeaderTimeout: 30 * time.Second,

internal/hub/hub.go

@@ -15,6 +15,7 @@ import (
 	"io"
 	"log"
 	"net/http"
+	"net/url"
 	"os"
 	"path/filepath"
 	"strings"
@@ -491,3 +492,28 @@ func (c *Client) AccessToken(ctx context.Context) (string, error) {
 	err := c.Call(ctx, "POST", "/api/v1/access", nil, &out)
 	return out.Token, err
 }
+
+// Product is a product as the hub lists it (GET /api/v1/products).
+type Product struct {
+	ID         string     `json:"id"`
+	Name       string     `json:"name"`
+	Creator    string     `json:"creator"`
+	Rating     string     `json:"rating"`
+	Latest     int        `json:"latest"`
+	Parent     int64      `json:"parent"`
+	Image      string     `json:"image"`
+	Categories []Category `json:"categories"` // the parent's IMVU category path, top first
+}
+
+// Category is an IMVU shop category.
+type Category struct {
+	ID   int64  `json:"id"`
+	Name string `json:"name"`
+}
+
+// Products searches the hub's products (by name; all if query is empty), newest first.
+func (c *Client) Products(ctx context.Context, query string) ([]Product, error) {
+	var out struct{ Products []Product }
+	err := c.Call(ctx, "GET", "/api/v1/products?limit=200&q="+url.QueryEscape(query), nil, &out)
+	return out.Products, err
+}

internal/products/chat.go

@@ -2,7 +2,6 @@ package products
 
 import (
 	"encoding/json"
-	"strconv"
 	"strings"
 )
 
@@ -34,15 +33,15 @@ func stripCommand(text string, isCommunity func(int64) bool) strippedCommand {
 	r := strippedCommand{text: text}
 	kept := []string{fields[0]}
 	for _, f := range fields[1:] {
-		id, err := strconv.ParseInt(f, 10, 64)
-		if err == nil && id == bodyProduct && fields[0] == "*use" {
+		id, ok := parseID(f)
+		if ok && id == bodyProduct && fields[0] == "*use" {
 			r.outfit = true
 		}
-		if err == nil && isCommunity(id) {
+		if ok && isCommunity(id) {
 			r.products = append(r.products, id)
 			continue
 		}
-		if err == nil {
+		if ok {
 			r.kept = append(r.kept, id)
 		}
 		kept = append(kept, f)

internal/products/guard.go

@@ -0,0 +1,41 @@
+package products
+
+import (
+	"net/url"
+	"regexp"
+	"strings"
+
+	"git.t5de.dev/t5de/t5de-proxy/internal/proxy"
+)
+
+// Guard is the last check before a request goes to IMVU: one that still names a community product (a local ID the
+// proxy gave out, in its query or body, as an integer or a float) is refused, not sent. The interceptors take
+// community products out of every request known to carry them; Guard stops the ones not known yet, and logs them
+// so they can be handled.
+type Guard struct{ Products *Products }
+
+var localIDPattern = regexp.MustCompile(`(?:^|[^0-9.])(2[0-9]{9})(?:\.0+)?(?:[^0-9.]|$)`)
+
+func (g Guard) Intercept(x *proxy.Exchange) error {
+	if x.Response != nil || !strings.HasSuffix(strings.ToLower(x.URL.Hostname()), ".imvu.com") {
+		return nil
+	}
+	query, _ := url.QueryUnescape(x.URL.RawQuery)
+	for _, text := range []string{query, x.URL.Path, string(x.Body)} {
+		for _, m := range localIDPattern.FindAllStringSubmatch(text, -1) {
+			id, ok := parseID(m[1])
+			if ok && IsCommunity(id) && g.Products.given(id) {
+				g.Products.logf("products: refused a request to %s%s: it names community product %d", x.URL.Host, x.URL.Path, id)
+				x.Response = localResponse(403, "application/json", []byte(`{"error":"t5de: community products stay off IMVU"}`))
+				return nil
+			}
+		}
+	}
+	return nil
+}
+
+// given reports whether a local ID was given to a product.
+func (p *Products) given(local int64) bool {
+	_, ok := p.state.productID(local)
+	return ok
+}

internal/products/guard_test.go

@@ -0,0 +1,32 @@
+package products
+
+import (
+	"fmt"
+	"net/http"
+	"net/url"
+	"testing"
+
+	"git.t5de.dev/t5de/t5de-proxy/internal/proxy"
+)
+
+func TestGuard(t *testing.T) {
+	h, _ := newHarness(t)
+	local := h.p.byProductID("blue-coat").local
+	g := Guard{Products: h.p}
+	check := func(rawURL, body string, refused bool) {
+		t.Helper()
+		u, _ := url.Parse(rawURL)
+		req, _ := http.NewRequest("POST", rawURL, nil)
+		x := &proxy.Exchange{URL: u, Request: req, Body: []byte(body)}
+		g.Intercept(x)
+		if (x.Response != nil) != refused {
+			t.Errorf("%s %s: refused %v", rawURL, body, x.Response != nil)
+		}
+	}
+	check("https://client-dynamic.imvu.com/api/new.php", fmt.Sprintf(`{"pids": [80, %d.0]}`, local), true)
+	check(fmt.Sprintf("https://api.imvu.com/product/product-%d", local), "", true)
+	check(fmt.Sprintf("https://client-dynamic.imvu.com/api/x.php?ids=1%%2C%d", local), "", true)
+	check("https://client-dynamic.imvu.com/api/new.php", `{"pids": [80, 2000000777]}`, false) // not given out
+	check("https://client-dynamic.imvu.com/api/new.php", fmt.Sprintf(`{"n": "1%d"}`, local), false)
+	check(fmt.Sprintf("https://hub.example/%d", local), "", false) // only IMVU's hosts
+}

internal/products/ids.go

@@ -0,0 +1,22 @@
+package products
+
+import (
+	"math"
+	"strconv"
+	"strings"
+)
+
+// parseID reads a product ID as the client writes it: "2000000008", or as a float, "2000000008.0" (its JavaScript
+// numbers go through Python as floats). Every place that takes community products out of what IMVU gets uses it, so a
+// float can't slip through as "not a product ID".
+func parseID(s string) (int64, bool) {
+	s = strings.TrimSpace(s)
+	if id, err := strconv.ParseInt(s, 10, 64); err == nil {
+		return id, true
+	}
+	f, err := strconv.ParseFloat(s, 64)
+	if err != nil || f != math.Trunc(f) || math.Abs(f) > 1<<53 {
+		return 0, false
+	}
+	return int64(f), true
+}

internal/products/ids_test.go

@@ -0,0 +1,16 @@
+package products
+
+import "testing"
+
+func TestParseID(t *testing.T) {
+	for s, want := range map[string]int64{"2000000008": 2000000008, "2000000008.0": 2000000008, " 80 ": 80, "2e9": 2000000000} {
+		if got, ok := parseID(s); !ok || got != want {
+			t.Errorf("parseID(%q) = %d, %v", s, got, ok)
+		}
+	}
+	for _, s := range []string{"", "1.5", "abc", "1e300"} {
+		if _, ok := parseID(s); ok {
+			t.Errorf("parseID(%q) ok", s)
+		}
+	}
+}

internal/products/intercept.go

@@ -40,9 +40,15 @@ func (p *Products) Intercept(x *proxy.Exchange) error {
 		return p.interceptXMLRPC(x)
 	case "/api/shop/product.php":
 		// Putting a product on asks the shop whether it's a bundle; community products aren't.
-		if id, err := strconv.ParseInt(x.URL.Query().Get("pid"), 10, 64); err == nil && IsCommunity(id) {
+		if id, ok := parseID(x.URL.Query().Get("pid")); ok && IsCommunity(id) {
 			x.Response = localResponse(200, "application/json", []byte(`{"products":[]}`))
+		} else if len(x.Body) > 0 {
+			return p.interceptProductInfo(x)
 		}
+	case "/api/shop/search_st.php":
+		return p.interceptShopSearch(x)
+	case "/api/service/shop/svc_catalog_purchase.php":
+		return p.interceptPurchase(x)
 	case "/api/base_look.php":
 		return p.interceptBaseLook(x)
 	case "/api/service/inventory/inventory_delete.php":
@@ -166,11 +172,18 @@ func productList(v xmlrpc.Value) ([]int64, bool) {
 	}
 	ids := make([]int64, 0, len(list))
 	for _, v := range list {
-		n, ok := v.(int64)
-		if !ok {
+		switch n := v.(type) {
+		case int64:
+			ids = append(ids, n)
+		case float64:
+			id, ok := parseID(strconv.FormatFloat(n, 'f', -1, 64))
+			if !ok {
+				return nil, false
+			}
+			ids = append(ids, id)
+		default:
 			return nil, false
 		}
-		ids = append(ids, n)
 	}
 	return ids, true
 }
@@ -568,8 +581,8 @@ func (p *Products) interceptHide(x *proxy.Exchange) error {
 	if err != nil {
 		return nil
 	}
-	pid, err := strconv.ParseInt(form.Get("pid"), 10, 64)
-	if err != nil || !IsCommunity(pid) {
+	pid, ok := parseID(form.Get("pid"))
+	if !ok || !IsCommunity(pid) {
 		return nil
 	}
 	if id, ok := p.state.productID(pid); ok && p.OnHide != nil {
@@ -596,8 +609,8 @@ func (p *Products) interceptOutfits(x *proxy.Exchange) error {
 		var community []string
 		stripped := 0
 		for _, f := range strings.Split(products, ",") {
-			id, err := strconv.ParseInt(strings.TrimSpace(f), 10, 64)
-			if err == nil && IsCommunity(id) {
+			id, ok := parseID(f)
+			if ok && IsCommunity(id) {
 				stripped++ // always kept from IMVU, even if its product isn't known anymore
 				if pid, ok := p.state.productID(id); ok {
 					community = append(community, pid)

internal/products/products.go

@@ -11,6 +11,7 @@ package products
 
 import (
 	"bytes"
+	"context"
 	"crypto/md5"
 	"encoding/hex"
 	"errors"
@@ -58,10 +59,17 @@ type Products struct {
 	// library, for hub products).
 	OnHide func(id string)
 
+	// Shop, if set, searches the hub's products for the shop's T5 filter; Fetch gets one into the hub's products
+	// now; AddToLibrary adds one to the person's library (buying it in the shop).
+	Shop         func(ctx context.Context, query string) ([]hub.Product, error)
+	Fetch        func(ctx context.Context, id string) error
+	AddToLibrary func(ctx context.Context, id string) error
+
 	roomMu sync.Mutex
 	room   room
 
-	user string // the signed-in IMVU account (from the login answer), under mu
+	user     string                 // the signed-in IMVU account (from the login answer), under mu
+	shopSeen map[string]hub.Product // hub products as the shop last listed them, under mu
 }
 
 // Ready is told a product arrived (the fetcher wrote it): it's served from now on, and given to the client for

internal/products/products_test.go

@@ -73,6 +73,17 @@ func (f *fakeIMVU) RoundTrip(r *http.Request) (*http.Response, error) {
 		return reply(`{"status": "success"}`)
 	}
 	switch r.URL.Path {
+	case "/api/shop/product.php":
+		checkSigned(false)
+		f.record("product info", body)
+		return reply(`{"products": [{"id": "12345", "name": "IMVU's"}]}`)
+	case "/api/shop/search_st.php":
+		f.record("search", body)
+		return reply(`{"keys": [], "result": {}}`)
+	case "/api/service/shop/svc_catalog_purchase.php":
+		checkSigned(false)
+		f.record("purchase", body)
+		return reply(`{"status": 1, "balance_credits": 10, "balance_promo": 0}`)
 	case "/api/outfits.php":
 		if len(body) > 0 {
 			checkSigned(false)

internal/products/shop.go

@@ -0,0 +1,392 @@
+package products
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"git.t5de.dev/t5de/t5de-proxy/internal/hub"
+	"git.t5de.dev/t5de/t5de-proxy/internal/proxy"
+)
+
+// The client's shop shows T5 products under its T5 filter (the t5de-shop patch), which searches with
+// maturity_rating "t5": the proxy answers those searches from the hub instead of asking IMVU. Buying a T5 product
+// adds it to the person's library on the hub; it's free, and IMVU never hears of it.
+
+// ShopRating is the shop's maturity_rating for T5 products.
+const ShopRating = "t5"
+
+// shopKeys are search_st.php's columns, as IMVU sends them.
+var shopKeys = []string{"id", "name", "price", "discount_price", "visible", "ap", "applus", "image", "creator_id",
+	"creator_name", "pro", "owns_product", "in_wishlist", "category", "category_name", "parent_categories",
+	"sub_pids", "index", "new", "sponsored", "purchasable"}
+
+// shopFetchTime is how long a search waits for products to arrive from catalogs (the rest show on a later search).
+var shopFetchTime = 15 * time.Second
+
+func (p *Products) interceptShopSearch(x *proxy.Exchange) error {
+	var req struct {
+		Rating   string          `json:"maturity_rating"`
+		Search   *string         `json:"search"`
+		Category json.RawMessage `json:"category"`
+		Page     json.Number     `json:"page"`
+		Skip     json.Number     `json:"skip"`
+		PerPage  json.Number     `json:"results_per_page"`
+		Limit    json.Number     `json:"solr_limit"`
+	}
+	if json.Unmarshal(x.Body, &req) != nil || req.Rating != ShopRating {
+		return nil
+	}
+	if p.Shop == nil {
+		x.Response = localResponse(200, "application/json", shopAnswer(nil, 1, 0, 12, 24))
+		return nil
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), shopFetchTime+5*time.Second)
+	defer cancel()
+	query := ""
+	if req.Search != nil {
+		query = strings.TrimSpace(*req.Search)
+	}
+	list, err := p.Shop(ctx, query)
+	if err != nil {
+		p.logf("shop: searching the hub: %v", err)
+		x.Response = localResponse(502, "application/json", []byte(`{"error":"hub"}`))
+		return nil
+	}
+	p.mu.Lock()
+	if p.shopSeen == nil {
+		p.shopSeen = map[string]hub.Product{}
+	}
+	for _, hp := range list {
+		p.shopSeen[hp.ID] = hp
+	}
+	p.mu.Unlock()
+	list = inCategories(list, categoryIDs(req.Category))
+	p.fetchForShop(ctx, list)
+
+	var rows []shopRow
+	for _, hp := range list {
+		if s := p.byProductID(hp.ID); s != nil {
+			rows = append(rows, shopRow{Product: hp, local: s.local})
+		}
+	}
+	page, skip, perPage, limit := number(req.Page, 1), number(req.Skip, 0), number(req.PerPage, 12), number(req.Limit, 24)
+	if perPage < 1 {
+		perPage = 12
+	}
+	if page < 1 {
+		page = 1
+	}
+	if skip < 0 {
+		skip = (page - 1) * perPage
+	}
+	for i := range rows {
+		rows[i].owned = p.owns(rows[i].ID)
+	}
+	x.Response = localResponse(200, "application/json", shopAnswer(rows, page, skip, perPage, limit))
+	return nil
+}
+
+type shopRow struct {
+	hub.Product
+	local int64
+	owned bool
+}
+
+func (r shopRow) cells(index int) []any {
+	ap := 0
+	if r.Rating == "adult" {
+		ap = 1
+	}
+	category, categoryName := int64(0), ""
+	parents := []hub.Category{}
+	if n := len(r.Categories); n > 0 {
+		category, categoryName = r.Categories[n-1].ID, categoryText(r.Categories[n-1].Name)
+		for _, c := range r.Categories[:n-1] {
+			parents = append(parents, hub.Category{ID: c.ID, Name: categoryText(c.Name)})
+		}
+	}
+	return []any{strconv.FormatInt(r.local, 10), plain(r.Name), 0, 0, 1, ap, 0, r.Image, "0", plain(r.Creator), 0,
+		r.owned, 0, category, categoryName, parents, "", index, 0, 0, 1}
+}
+
+// categoryText is an IMVU category name (from the hub), kept to what plain allows plus "&" ("Coats & Jackets"):
+// a lone "&" can't start markup.
+func categoryText(s string) string {
+	parts := strings.Split(s, "&")
+	for i, part := range parts {
+		parts[i] = plain(part)
+	}
+	return strings.Join(parts, "&")
+}
+
+// owns reports whether the person has a product: local packs always, hub products in their library.
+func (p *Products) owns(id string) bool {
+	if p.inHubDir(id) {
+		return p.Owned != nil && p.Owned(id)
+	}
+	return true
+}
+
+// object is the row as one of product.php's products (keys to values).
+func (r shopRow) object() map[string]any {
+	cells := r.cells(0)
+	out := map[string]any{}
+	for i, k := range shopKeys {
+		out[k] = cells[i]
+	}
+	return out
+}
+
+// interceptProductInfo answers product.php's product details for community products, and asks IMVU only for its
+// own.
+func (p *Products) interceptProductInfo(x *proxy.Exchange) error {
+	var body map[string]json.RawMessage
+	if json.Unmarshal(x.Body, &body) != nil {
+		return nil
+	}
+	var pids []json.RawMessage
+	if json.Unmarshal(body["pids"], &pids) != nil {
+		return nil
+	}
+	var keep []json.RawMessage
+	var ours []any
+	community := 0
+	for _, raw := range pids {
+		id, ok := rawID(raw)
+		if !ok || !IsCommunity(id) {
+			keep = append(keep, raw)
+			continue
+		}
+		community++
+		if r, ok := p.shopRowFor(id); ok {
+			ours = append(ours, r.object())
+		}
+	}
+	if community == 0 {
+		return nil
+	}
+	if len(keep) == 0 {
+		b, _ := json.Marshal(map[string]any{"products": ours})
+		x.Response = localResponse(200, "application/json", b)
+		return nil
+	}
+	body["pids"], _ = json.Marshal(keep)
+	nb, _ := json.Marshal(body)
+	if !p.keys.resign(x.Request.Header, x.Body, x.URL.RawQuery, nb, x.URL.RawQuery) {
+		return errNoKey
+	}
+	x.Body = nb
+	x.ModifyResponse = func(resp *http.Response, b []byte) ([]byte, error) {
+		var r map[string]json.RawMessage
+		var list []any
+		if json.Unmarshal(b, &r) != nil || json.Unmarshal(r["products"], &list) != nil {
+			return b, nil
+		}
+		r["products"], _ = json.Marshal(append(list, ours...))
+		return json.Marshal(r)
+	}
+	return nil
+}
+
+// shopRowFor is a community product's shop entry: as the hub listed it, else from its pack.
+func (p *Products) shopRowFor(local int64) (shopRow, bool) {
+	s := p.lookup(local)
+	if s == nil {
+		return shopRow{}, false
+	}
+	m := s.pack.Manifest
+	p.mu.Lock()
+	hp, ok := p.shopSeen[m.ID]
+	p.mu.Unlock()
+	if !ok {
+		hp = hub.Product{ID: m.ID, Name: m.Name, Creator: m.Creator, Rating: m.Rating, Latest: m.Revision}
+	}
+	return shopRow{Product: hp, local: s.local, owned: p.owns(m.ID)}, true
+}
+
+// rawID reads a product ID sent as a number or a string.
+func rawID(raw json.RawMessage) (int64, bool) {
+	var s string
+	if json.Unmarshal(raw, &s) != nil {
+		var n json.Number
+		if json.Unmarshal(raw, &n) != nil {
+			return 0, false
+		}
+		s = n.String()
+	}
+	return parseID(s)
+}
+
+// shopAnswer is a search_st.php answer: the pages from page on, holding up to limit products from skip.
+func shopAnswer(rows []shopRow, page, skip, perPage, limit int) []byte {
+	result := map[string][]any{}
+	end := min(skip+limit, len(rows))
+	for i := skip; i < end; i++ {
+		key := strconv.Itoa(page + (i-skip)/perPage)
+		result[key] = append(result[key], rows[i].cells(i))
+	}
+	b, _ := json.Marshal(map[string]any{
+		"keys":                 shopKeys,
+		"startIndex":           skip,
+		"endIndex":             end - 1,
+		"result":               result,
+		"totalProductCount":    len(rows),
+		"thisPageProductCount": max(end-skip, 0),
+		"pageCount":            (len(rows) + perPage - 1) / perPage,
+		"thisPage":             page,
+	})
+	return b
+}
+
+// categoryIDs reads the shop's category field: IDs separated by commas, as a string or number (empty: all).
+func categoryIDs(raw json.RawMessage) map[int64]bool {
+	var s string
+	if json.Unmarshal(raw, &s) != nil {
+		var n json.Number
+		if json.Unmarshal(raw, &n) != nil {
+			return nil
+		}
+		s = n.String()
+	}
+	ids := map[int64]bool{}
+	for _, f := range strings.Split(s, ",") {
+		if id, ok := parseID(f); ok {
+			ids[id] = true
+		}
+	}
+	if len(ids) == 0 {
+		return nil
+	}
+	return ids
+}
+
+// inCategories keeps the products under any of the categories (all of them if there are none).
+func inCategories(list []hub.Product, ids map[int64]bool) []hub.Product {
+	if ids == nil {
+		return list
+	}
+	var out []hub.Product
+	for _, hp := range list {
+		for _, c := range hp.Categories {
+			if ids[c.ID] {
+				out = append(out, hp)
+				break
+			}
+		}
+	}
+	return out
+}
+
+// fetchForShop gets the listed products that aren't here (or are an older revision), a few at a time, so the shop
+// can show them and they can be tried on.
+func (p *Products) fetchForShop(ctx context.Context, list []hub.Product) {
+	if p.Fetch == nil {
+		return
+	}
+	ctx, cancel := context.WithTimeout(ctx, shopFetchTime)
+	defer cancel()
+	var wg sync.WaitGroup
+	slots := make(chan struct{}, 6)
+	for _, hp := range list {
+		if s := p.byProductID(hp.ID); s != nil && s.pack.Manifest.Revision >= hp.Latest {
+			continue
+		}
+		wg.Add(1)
+		go func(id string) {
+			defer wg.Done()
+			select {
+			case slots <- struct{}{}:
+			case <-ctx.Done():
+				return
+			}
+			defer func() { <-slots }()
+			if err := p.Fetch(ctx, id); err != nil {
+				p.logf("shop: %s: %v", id, err)
+			}
+		}(hp.ID)
+	}
+	wg.Wait()
+	p.refresh(true)
+}
+
+func number(n json.Number, fallback int) int {
+	if v, err := n.Int64(); err == nil {
+		return int(v)
+	}
+	return fallback
+}
+
+// interceptPurchase buys T5 products by adding them to the person's library. Products from IMVU in the same
+// purchase go on to IMVU without the T5 ones.
+func (p *Products) interceptPurchase(x *proxy.Exchange) error {
+	var body map[string]json.RawMessage
+	if json.Unmarshal(x.Body, &body) != nil {
+		return nil
+	}
+	var pids []json.RawMessage
+	if json.Unmarshal(body["products"], &pids) != nil {
+		var one json.RawMessage
+		if json.Unmarshal(body["products"], &one) != nil {
+			return nil
+		}
+		pids = []json.RawMessage{one}
+	}
+	var keep []json.RawMessage
+	var t5 []string
+	community := 0
+	for _, raw := range pids {
+		id, ok := rawID(raw)
+		if !ok || !IsCommunity(id) {
+			keep = append(keep, raw)
+			continue
+		}
+		community++
+		if pid, ok := p.state.productID(id); ok && p.HubDir != "" && p.inHubDir(pid) {
+			t5 = append(t5, pid)
+		} // local packs are always in the inventory: nothing to buy
+	}
+	if community == 0 {
+		return nil
+	}
+	// IMVU's part first (re-signed), so nothing is added to the library for a purchase that can't go on.
+	if len(keep) > 0 {
+		body["products"], _ = json.Marshal(keep)
+		nb, _ := json.Marshal(body)
+		if !p.keys.resign(x.Request.Header, x.Body, x.URL.RawQuery, nb, x.URL.RawQuery) {
+			return errNoKey
+		}
+		x.Body = nb
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	for _, pid := range t5 {
+		if p.AddToLibrary == nil {
+			break
+		}
+		if err := p.AddToLibrary(ctx, pid); err != nil {
+			p.logf("shop: adding %s to the library: %v", pid, err)
+			x.Response = localResponse(200, "application/json", []byte(`{"status":3}`))
+			return nil
+		}
+	}
+	if len(t5) > 0 {
+		p.logf("shop: added %d T5 products to the library", len(t5))
+	}
+	if len(keep) == 0 {
+		// No credits change hands: the client's patch leaves the balance alone when the answer has none.
+		x.Response = localResponse(200, "application/json", []byte(`{"status":1}`))
+	}
+	return nil
+}
+
+// inHubDir reports whether a product is served from the hub's products (not a local pack).
+func (p *Products) inHubDir(id string) bool {
+	s := p.byProductID(id)
+	return s != nil && filepath.Dir(s.dir) == filepath.Clean(p.HubDir)
+}

internal/products/shop_test.go

@@ -0,0 +1,151 @@
+package products
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+	"sync"
+	"testing"
+
+	"git.t5de.dev/t5de/t5de-proxy/internal/hub"
+)
+
+func TestShop(t *testing.T) {
+	h, _ := newHarness(t)
+	hubDir := t.TempDir()
+	h.p.dirs = append(h.p.dirs, hubDir)
+	h.p.HubDir = hubDir
+	library := map[string]bool{"coatcoat22": true}
+	h.p.Owned = func(id string) bool { return library[id] }
+	coat := []hub.Category{{ID: 106, Name: "Clothing"}, {ID: 40, Name: "Clothing for Female"}, {ID: 716, Name: "Coats & <Jackets>"}}
+	hair := []hub.Category{{ID: 3114, Name: "Body"}, {ID: 3115, Name: "Body for Female"}, {ID: 75, Name: "Hair"}}
+	listed := []hub.Product{
+		{ID: "coatcoat22", Name: "Green Coat", Creator: "erin", Rating: "general", Latest: 1, Image: "https://userimages-akm.imvu.com/a.gif", Categories: coat},
+		{ID: "hairhair22", Name: "Red Hair", Creator: "erin", Rating: "adult", Latest: 2, Categories: hair},
+		{ID: "slowslow22", Name: "Never Arrives", Creator: "erin", Rating: "general", Latest: 1, Categories: coat},
+	}
+	var queries []string
+	h.p.Shop = func(ctx context.Context, q string) ([]hub.Product, error) {
+		queries = append(queries, q)
+		return listed, nil
+	}
+	fetched := map[string]int{}
+	var mu sync.Mutex
+	h.p.Fetch = func(ctx context.Context, id string) error {
+		mu.Lock()
+		defer mu.Unlock()
+		fetched[id]++
+		if id == "slowslow22" {
+			return fmt.Errorf("no catalog hosts it")
+		}
+		for _, hp := range listed {
+			if hp.ID == id {
+				writePack(t, hubDir, id, hp.Latest)
+			}
+		}
+		return nil
+	}
+	var added []string
+	h.p.AddToLibrary = func(ctx context.Context, id string) error {
+		added = append(added, id)
+		library[id] = true
+		return nil
+	}
+
+	search := func(rating, category string, page int) map[string]any {
+		t.Helper()
+		body, _ := json.Marshal(map[string]any{"category": category, "search": "coat", "skip": (page - 1) * 1, "maturity_rating": rating,
+			"results_per_page": 1, "solr_limit": 2, "page": page})
+		status, b := h.do("POST", "http://client-dynamic.imvu.com/api/shop/search_st.php", body, true, false)
+		var out map[string]any
+		if status != 200 || json.Unmarshal(b, &out) != nil {
+			t.Fatalf("search: %d %s", status, b)
+		}
+		return out
+	}
+
+	// Other ratings go to IMVU.
+	search("all", "", 1)
+	if len(h.imvu.asked["search"]) != 1 || len(queries) != 0 {
+		t.Fatal("an IMVU search wasn't left to IMVU")
+	}
+
+	// The T5 filter: the hub's products, fetched so they can be shown and tried on, in IMVU's columns.
+	out := search(ShopRating, "", 1)
+	if queries[0] != "coat" || fetched["coatcoat22"] != 1 || fetched["hairhair22"] != 1 {
+		t.Fatalf("queries %v, fetched %v", queries, fetched)
+	}
+	if out["totalProductCount"].(float64) != 2 || out["pageCount"].(float64) != 2 || out["endIndex"].(float64) != 1 {
+		t.Fatalf("answer %v", out)
+	}
+	result := out["result"].(map[string]any)
+	first, second := result["1"].([]any)[0].([]any), result["2"].([]any)[0].([]any)
+	row := map[string]any{}
+	for i, k := range out["keys"].([]any) {
+		row[k.(string)] = first[i]
+	}
+	coatLocal := h.p.byProductID("coatcoat22").local
+	if row["id"] != fmt.Sprint(coatLocal) || row["name"] != "Green Coat" || row["owns_product"] != true || row["ap"].(float64) != 0 ||
+		row["category"].(float64) != 716 || row["category_name"] != "Coats & Jackets" || row["image"] != "https://userimages-akm.imvu.com/a.gif" {
+		t.Fatalf("row %v", row)
+	}
+	if second[5].(float64) != 1 || second[11] != false {
+		t.Fatalf("the adult product, not owned: %v", second)
+	}
+
+	// Products here at the listed revision aren't fetched again; categories filter.
+	out = search(ShopRating, "3115,75", 1)
+	if fetched["coatcoat22"] != 1 || fetched["hairhair22"] != 1 || out["totalProductCount"].(float64) != 1 {
+		t.Fatalf("fetched %v, answer %v", fetched, out)
+	}
+
+	// Before signing in, a mixed purchase can't be re-signed: it fails without touching the library.
+	hairLocal := h.p.byProductID("hairhair22").local
+	purchase := func(pids ...any) string {
+		body, _ := json.Marshal(map[string]any{"products": pids, "context": "sitc", "source": "client"})
+		_, b := h.do("POST", "http://client-dynamic.imvu.com/api/service/shop/svc_catalog_purchase.php", body, true, false)
+		return string(b)
+	}
+	if purchase(hairLocal, 12345); len(added) != 0 {
+		t.Fatal("added to the library for a failed purchase")
+	}
+	h.do("POST", "https://secure.imvu.com/api/xmlrpc/client.php", call("test.avatarInfoForLogin2"), false, false)
+
+	// Buying a T5 product adds it to the library; IMVU hears only of its own products.
+	// Product details: community ones answered here, IMVU asked only for its own.
+	info := func(pids ...any) []map[string]any {
+		body, _ := json.Marshal(map[string]any{"pids": pids})
+		_, b := h.do("POST", "http://client-dynamic.imvu.com/api/shop/product.php", body, true, false)
+		var out struct{ Products []map[string]any }
+		if json.Unmarshal(b, &out) != nil {
+			t.Fatalf("product info: %s", b)
+		}
+		return out.Products
+	}
+	if got := info(coatLocal); len(got) != 1 || got[0]["name"] != "Green Coat" || got[0]["owns_product"] != true || len(h.imvu.asked["product info"]) != 0 {
+		t.Fatalf("product info %v", got)
+	}
+	// The client sends some IDs as floats ("2000000008.0"): those are community products too.
+	body, _ := json.Marshal(map[string]any{"pids": []json.RawMessage{json.RawMessage(fmt.Sprintf("%d.0", coatLocal)), json.RawMessage("12345")}})
+	h.do("POST", "http://client-dynamic.imvu.com/api/shop/product.php", body, true, false)
+	if asked := h.imvu.asked["product info"]; len(asked) != 1 || strings.Contains(asked[0], fmt.Sprint(coatLocal)) {
+		t.Fatalf("a float community ID went to IMVU: %v", asked)
+	}
+	if got := info(fmt.Sprint(hairLocal), 12345); len(got) != 2 || got[0]["name"] != "IMVU's" || got[1]["ap"].(float64) != 1 ||
+		strings.Contains(h.imvu.asked["product info"][1], fmt.Sprint(hairLocal)) {
+		t.Fatalf("mixed product info %v, IMVU got %v", got, h.imvu.asked["product info"])
+	}
+
+	if got := purchase(fmt.Sprint(hairLocal)); got != `{"status":1}` || len(added) != 1 || added[0] != "hairhair22" || len(h.imvu.asked["purchase"]) != 0 {
+		t.Fatalf("T5 purchase: %s, added %v", got, added)
+	}
+	if got := purchase(hairLocal, 12345); !strings.Contains(got, "balance_credits") || len(h.imvu.asked["purchase"]) != 1 ||
+		strings.Contains(h.imvu.asked["purchase"][0], fmt.Sprint(hairLocal)) || !strings.Contains(h.imvu.asked["purchase"][0], "12345") {
+		t.Fatalf("mixed purchase: %s, IMVU got %v", got, h.imvu.asked["purchase"])
+	}
+	purchase(12345)
+	if len(h.imvu.asked["purchase"]) != 2 {
+		t.Fatal("an IMVU purchase wasn't left to IMVU")
+	}
+}

internal/proxy/dump.go

@@ -0,0 +1,33 @@
+package proxy
+
+import (
+	"log"
+	"net/http"
+	"strings"
+)
+
+// Dump logs the request and response bodies of requests whose path contains Match (T5DE_PROXY_DUMP, for finding
+// out what IMVU's APIs send). It may log personal data: only for test accounts.
+type Dump struct {
+	Match string
+	Log   *log.Logger
+}
+
+func (d Dump) Intercept(x *Exchange) error {
+	if d.Match == "" || !strings.Contains(x.URL.Path, d.Match) {
+		return nil
+	}
+	d.Log.Printf("dump: %s %s\n%s", x.Request.Method, x.URL, x.Body)
+	next := x.ModifyResponse
+	x.ModifyResponse = func(resp *http.Response, body []byte) ([]byte, error) {
+		if next != nil {
+			var err error
+			if body, err = next(resp, body); err != nil {
+				return nil, err
+			}
+		}
+		d.Log.Printf("dump: %s answered %d\n%s", x.URL.Path, resp.StatusCode, body)
+		return body, nil
+	}
+	return nil
+}