Serve community products from local packs

The proxy serves products from local packs (t5de-api's format) under local IDs from 2,000,000,000: product
info and permission answers (with the parent's derivation chain from IMVU), the product's CFL, and the shop's
bundle lookup. Community products are taken out of outfit saves and outfit chat commands before IMVU sees
them, remembered per account with the outfit they belong to, and added back when the client loads that
outfit. Changed requests are re-signed with the session key from the login response.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
dhkatzandClaude Opus 5.5 committed 2026-10-29 03:17:14 -07:00
1 parent b2eed27b99
commit 887e8a2364
7 files changed
+1177 -21

No files matched your search

+38 -3
View File
@@ -18,10 +18,13 @@ import (
"net"
"net/http"
"os"
"path/filepath"
"strings"
"time"
"git.t5de.dev/t5de/t5de-proxy/internal/buildinfo"
"git.t5de.dev/t5de/t5de-proxy/internal/parent"
"git.t5de.dev/t5de/t5de-proxy/internal/products"
"git.t5de.dev/t5de/t5de-proxy/internal/proxy"
)
@@ -31,6 +34,9 @@ func main() {
parentPID := flag.Int("parent", 0, "exit when this process exits")
logFile := flag.String("log", "", "log file (default: standard error)")
trace := flag.Bool("trace", false, "log full URLs and every IMQ frame")
packs := flag.String("packs", "", "community product packs: directories of packs, separated by "+string(filepath.ListSeparator)+
" (default: %APPDATA%/T5DE/packs and $T5DE_PACKS)")
data := flag.String("data", "", "where the proxy keeps its state (default: %APPDATA%/T5DE/proxy)")
version := flag.Bool("version", false, "print the version and exit")
flag.Parse()
if *version {
@@ -48,13 +54,35 @@ func main() {
out = f
}
logger := log.New(out, "", log.Ldate|log.Ltime|log.Lmicroseconds)
if err := run(logger, *listen, *ready, *parentPID, *trace); err != nil {
if err := run(logger, *listen, *ready, *parentPID, *trace, packDirs(*packs), dataDir(*data)); err != nil {
logger.Printf("t5de-proxy: %v", err)
os.Exit(1)
}
}
func run(logger *log.Logger, listen, ready string, parentPID int, trace bool) error {
// packDirs is where community product packs are: the -packs flag, else T5DE's packs folder and $T5DE_PACKS.
func packDirs(flagValue string) []string {
if flagValue != "" {
return filepath.SplitList(flagValue)
}
var dirs []string
if config, err := os.UserConfigDir(); err == nil {
dirs = append(dirs, filepath.Join(config, "T5DE", "packs"))
}
return append(dirs, filepath.SplitList(os.Getenv("T5DE_PACKS"))...)
}
func dataDir(flagValue string) string {
if flagValue != "" {
return flagValue
}
if config, err := os.UserConfigDir(); err == nil {
return filepath.Join(config, "T5DE", "proxy")
}
return "."
}
func run(logger *log.Logger, listen, ready string, parentPID int, trace bool, packs []string, data string) error {
secret := make([]byte, 24)
if _, err := rand.Read(secret); err != nil {
return err
@@ -78,8 +106,15 @@ func run(logger *log.Logger, listen, ready string, parentPID int, trace bool) er
os.Exit(0)
}()
}
handler := &proxy.Server{Token: token, Log: logger, Trace: trace}
if prods, err := products.New(packs, data, logger.Printf); err != nil {
logger.Printf("community products are off: %v", err)
} else {
handler.Interceptor, handler.IMQFilter = prods, prods
logger.Printf("community products from %s", strings.Join(packs, ", "))
}
server := &http.Server{
Handler: &proxy.Server{Token: token, Log: logger, Trace: trace},
Handler: handler,
ReadHeaderTimeout: 30 * time.Second,
IdleTimeout: 2 * time.Minute,
ErrorLog: logger,
+4
View File
@@ -1,3 +1,7 @@
module git.t5de.dev/t5de/t5de-proxy
go 1.27.0
require git.t5de.dev/t5de/t5de-api v0.0.0-00010101000000-000000000000
replace git.t5de.dev/t5de/t5de-api => ../t5de-api
+4
View File
@@ -22,6 +22,7 @@ type strippedCommand struct {
empty bool // nothing is left to send
outfit bool // the line was a whole outfit (*use with the body)
products []int64 // the community products it named
kept []int64 // the other products it named
}
// stripCommand takes products for which isCommunity is true out of an outfit command's arguments.
@@ -41,6 +42,9 @@ func stripCommand(text string, isCommunity func(int64) bool) strippedCommand {
r.products = append(r.products, id)
continue
}
if err == nil {
r.kept = append(r.kept, id)
}
kept = append(kept, f)
}
if len(r.products) == 0 {
+476
View File
@@ -0,0 +1,476 @@
package products
import (
"encoding/json"
"errors"
"net/http"
"strconv"
"strings"
"git.t5de.dev/t5de/t5de-proxy/internal/proxy"
"git.t5de.dev/t5de/t5de-proxy/internal/xmlrpc"
)
func localResponse(status int, contentType string, body []byte) *http.Response {
return proxy.LocalResponse(status, contentType, body)
}
// Intercept implements proxy.Interceptor.
func (p *Products) Intercept(x *proxy.Exchange) error {
host, path := strings.ToLower(x.URL.Hostname()), x.URL.Path
if strings.HasSuffix(host, ".imvu.com") && strings.HasPrefix(path, "/productdata/") {
if resp, ok := p.serveFile(path); ok {
x.Response = resp
}
return nil
}
if !strings.HasSuffix(host, ".imvu.com") {
return nil
}
switch path {
case "/api/xmlrpc/client.php":
return p.interceptXMLRPC(x)
case "/api/shop/product.php":
// Putting a product on asks the shop whether it's a bundle; community products aren't.
if id, err := strconv.ParseInt(x.URL.Query().Get("pid"), 10, 64); err == nil && IsCommunity(id) {
x.Response = localResponse(200, "application/json", []byte(`{"products":[]}`))
}
case "/api/base_look.php":
return p.interceptBaseLook(x)
}
return nil
}
func (p *Products) interceptXMLRPC(x *proxy.Exchange) error {
call, err := xmlrpc.ParseCall(x.Body)
if err != nil {
return nil // not ours to judge; IMVU answers
}
switch call.Method {
case "test.avatarInfoForLogin2":
x.ModifyResponse = func(resp *http.Response, body []byte) ([]byte, error) {
p.keys.learn(body)
return p.addToLook(body)
}
case "test.inventoryProductBatch":
return p.inventoryProductBatch(x, call)
case "test.authorizeProducts2":
return p.authorizeProducts2(x, call)
}
return nil
}
// replaceBody sends call instead of the client's, re-signed.
func (p *Products) replaceBody(x *proxy.Exchange, call *xmlrpc.Call) error {
body := call.Marshal()
if !p.keys.resign(x.Request.Header, x.Body, x.URL.RawQuery, body, x.URL.RawQuery) {
return errNoKey
}
x.Body = body
return nil
}
// addToLook adds the user's community products to the outfit ("look") in a login response.
func (p *Products) addToLook(body []byte) ([]byte, error) {
m := userIDPattern.FindSubmatch(body)
if m == nil {
return body, nil
}
user := string(m[1])
r, err := xmlrpc.ParseResponse(body)
if err != nil || r.Fault != nil {
return body, err
}
changed := false
var walk func(v xmlrpc.Value) xmlrpc.Value
walk = func(v xmlrpc.Value) xmlrpc.Value {
switch v := v.(type) {
case xmlrpc.Struct:
for i, mem := range v {
if mem.Name == "look" {
if look, ok := productList(mem.Value); ok {
if worn := p.wornLocal(user, look); len(worn) > 0 {
v[i].Value = appendIDs(mem.Value.([]xmlrpc.Value), worn)
changed = true
}
continue
}
}
v[i].Value = walk(mem.Value)
}
return v
case []xmlrpc.Value:
for i := range v {
v[i] = walk(v[i])
}
return v
}
return v
}
for i := range r.Params {
r.Params[i] = walk(r.Params[i])
}
if !changed {
return body, nil
}
return r.Marshal(), nil
}
func appendIDs(list []xmlrpc.Value, ids []int64) []xmlrpc.Value {
have := map[int64]bool{}
for _, v := range list {
if n, ok := v.(int64); ok {
have[n] = true
}
}
for _, id := range ids {
if !have[id] {
list = append(list, id)
}
}
return list
}
// productList reads a list of product IDs.
func productList(v xmlrpc.Value) ([]int64, bool) {
list, ok := v.([]xmlrpc.Value)
if !ok {
return nil, false
}
ids := make([]int64, 0, len(list))
for _, v := range list {
n, ok := v.(int64)
if !ok {
return nil, false
}
ids = append(ids, n)
}
return ids, true
}
// inventoryProductBatch answers product info for community products: the parent's info from IMVU with the
// product's own name, creator and ID, or (without a parent) info of its own.
func (p *Products) inventoryProductBatch(x *proxy.Exchange, call *xmlrpc.Call) error {
if len(call.Params) != 1 {
return nil
}
ids, ok := productList(call.Params[0])
if !ok {
return nil
}
var community []int64
var forward []xmlrpc.Value
asked := map[int64]bool{}
for _, id := range ids {
if !IsCommunity(id) {
if !asked[id] {
forward, asked[id] = append(forward, id), true
}
continue
}
community = append(community, id)
if s := p.lookup(id); s != nil && s.pack.Index.Parent > 0 && !asked[int64(s.pack.Index.Parent)] {
forward, asked[int64(s.pack.Index.Parent)] = append(forward, int64(s.pack.Index.Parent)), true
}
}
if len(community) == 0 {
return nil
}
// The answer, in the order asked, from IMVU's entries (by product ID) and our own.
answer := func(imvu map[int64]xmlrpc.Struct) []byte {
var out []xmlrpc.Value
for _, id := range ids {
if !IsCommunity(id) {
if e, ok := imvu[id]; ok {
out = append(out, e)
}
continue
}
if s := p.lookup(id); s != nil {
out = append(out, s.productInfo(imvu[int64(s.pack.Index.Parent)]))
}
}
return (&xmlrpc.Response{Params: []xmlrpc.Value{out}}).Marshal()
}
if len(forward) == 0 {
x.Response = localResponse(200, "text/xml", answer(nil))
return nil
}
if err := p.replaceBody(x, &xmlrpc.Call{Method: call.Method, Params: []xmlrpc.Value{forward}}); err != nil {
return err
}
x.ModifyResponse = func(resp *http.Response, body []byte) ([]byte, error) {
r, err := xmlrpc.ParseResponse(body)
if err != nil || r.Fault != nil || len(r.Params) != 1 {
return body, err
}
list, _ := r.Params[0].([]xmlrpc.Value)
imvu := map[int64]xmlrpc.Struct{}
for _, v := range list {
if e, ok := v.(xmlrpc.Struct); ok {
if id, ok := e.Get("products_id"); ok {
if n, ok := id.(int64); ok {
imvu[n] = e
}
}
}
}
return answer(imvu), nil
}
return nil
}
// productInfo is a community product's entry for inventoryProductBatch, based on its parent's if there is one.
func (s *served) productInfo(parent xmlrpc.Struct) xmlrpc.Struct {
m := s.pack.Manifest
mature := "N"
if m.Rating == "adult" {
mature = "Y"
}
own := map[string]xmlrpc.Value{
"products_id": s.local,
"manufacturers_id": int64(0),
"manufacturers_name": m.Creator,
"products_name": m.Name,
"products_mature": mature,
"products_bundle": "",
"is_published": "1",
"is_purchasable": false,
}
var out xmlrpc.Struct
done := map[string]bool{}
for _, mem := range parent {
if v, ok := own[mem.Name]; ok {
out = append(out, xmlrpc.Member{Name: mem.Name, Value: v})
done[mem.Name] = true
} else {
out = append(out, mem) // cPath, products_image, ... from the parent
}
}
for _, name := range []string{"products_id", "manufacturers_id", "manufacturers_name", "products_name", "cPath",
"products_mature", "products_bundle", "products_image", "is_published", "is_purchasable"} {
if done[name] {
continue
}
v, ok := own[name]
if !ok {
v = "" // cPath and products_image, without a parent
}
if _, inParent := parent.Get(name); !inParent {
out = append(out, xmlrpc.Member{Name: name, Value: v})
}
}
return out
}
// authColumns are authorizeProducts2's answer columns, as IMVU sends them.
var authColumns = []string{"userId", "parentProductId", "productId", "cflChecksum", "cflUrl", "sub_pids",
"usableDirect", "autoAuth", "usableDerivation", "usableTrial", "trialTimeout", "usablePeerReview", "info"}
// authorizeProducts2 answers permission and file addresses for community products, and asks IMVU for their
// parents' (the client needs the whole derivation chain).
func (p *Products) authorizeProducts2(x *proxy.Exchange, call *xmlrpc.Call) error {
if len(call.Params) != 1 {
return nil
}
args, ok := call.Params[0].(xmlrpc.Struct)
if !ok {
return nil
}
authsValue, _ := args.Get("auths")
table, ok := authsValue.([]xmlrpc.Value)
if !ok || len(table) < 1 {
return nil
}
header, ok := table[0].([]xmlrpc.Value)
if !ok || len(header) != 2 || header[0] != "userId" || header[1] != "productId" {
return nil
}
type auth struct {
user, product int64
}
var community []auth
forward := []xmlrpc.Value{table[0]}
asked := map[auth]bool{}
for _, row := range table[1:] {
r, ok := row.([]xmlrpc.Value)
if !ok || len(r) != 2 {
return nil
}
user, ok1 := r[0].(int64)
product, ok2 := r[1].(int64)
if !ok1 || !ok2 {
return nil
}
if !IsCommunity(product) {
if !asked[auth{user, product}] {
forward, asked[auth{user, product}] = append(forward, row), true
}
continue
}
community = append(community, auth{user, product})
if s := p.lookup(product); s != nil && s.pack.Index.Parent > 0 {
a := auth{user, int64(s.pack.Index.Parent)}
if !asked[a] {
forward, asked[a] = append(forward, []xmlrpc.Value{user, a.product}), true
}
}
}
if len(community) == 0 {
return nil
}
ours := func(columns []xmlrpc.Value) []xmlrpc.Value {
var rows []xmlrpc.Value
for _, a := range community {
s := p.lookup(a.product)
if s == nil {
continue // unknown: no row, as IMVU does for products it can't authorize
}
values := map[string]xmlrpc.Value{
"userId": a.user, "parentProductId": int64(s.pack.Index.Parent), "productId": s.local,
"cflChecksum": s.checksum, "cflUrl": cflURL(s.local, s.pack.Manifest.Revision), "sub_pids": "",
"usableDirect": true, "autoAuth": true, "usableDerivation": int64(1), "usableTrial": true,
"trialTimeout": int64(40), "usablePeerReview": true, "info": "",
}
row := make([]xmlrpc.Value, len(columns))
for i, c := range columns {
name, _ := c.(string)
if v, ok := values[name]; ok {
row[i] = v
} else {
row[i] = ""
}
}
rows = append(rows, row)
}
return rows
}
if len(forward) == 1 { // only community products, none deriving from IMVU's
columns := make([]xmlrpc.Value, len(authColumns))
for i, c := range authColumns {
columns[i] = c
}
out := append([]xmlrpc.Value{columns}, ours(columns)...)
x.Response = localResponse(200, "text/xml", (&xmlrpc.Response{Params: []xmlrpc.Value{out}}).Marshal())
return nil
}
changed := make(xmlrpc.Struct, len(args))
copy(changed, args)
for i := range changed {
if changed[i].Name == "auths" {
changed[i].Value = forward
}
}
if err := p.replaceBody(x, &xmlrpc.Call{Method: call.Method, Params: []xmlrpc.Value{changed}}); err != nil {
return err
}
x.ModifyResponse = func(resp *http.Response, body []byte) ([]byte, error) {
r, err := xmlrpc.ParseResponse(body)
if err != nil || r.Fault != nil || len(r.Params) != 1 {
return body, err
}
rows, ok := r.Params[0].([]xmlrpc.Value)
if !ok || len(rows) == 0 {
return body, errors.New("products: unexpected authorizeProducts2 answer")
}
columns, ok := rows[0].([]xmlrpc.Value)
if !ok {
return body, errors.New("products: unexpected authorizeProducts2 columns")
}
r.Params[0] = append(rows, ours(columns)...)
return r.Marshal(), nil
}
return nil
}
// interceptBaseLook keeps community products out of the outfit IMVU stores, and adds them back when the client
// loads it.
func (p *Products) interceptBaseLook(x *proxy.Exchange) error {
user := x.Request.Header.Get("X-imvu-userid")
if user == "" {
return nil
}
if len(x.Body) == 0 { // loading the outfit
x.ModifyResponse = func(resp *http.Response, body []byte) ([]byte, error) {
var look map[string]json.RawMessage
if err := json.Unmarshal(body, &look); err != nil {
return body, err
}
var ids []int64
if err := json.Unmarshal(look["result"], &ids); err != nil {
return body, err
}
worn := p.wornLocal(user, ids)
if len(worn) == 0 {
return body, nil
}
have := map[int64]bool{}
for _, id := range ids {
have[id] = true
}
for _, id := range worn {
if !have[id] {
ids = append(ids, id)
}
}
look["result"], _ = json.Marshal(ids)
return json.Marshal(look)
}
return nil
}
// Saving it: {"pids": [...]}.
var save map[string]json.RawMessage
if err := json.Unmarshal(x.Body, &save); err != nil {
return nil
}
var ids []int64
if err := json.Unmarshal(save["pids"], &ids); err != nil {
return nil
}
var kept, community []int64
for _, id := range ids {
if IsCommunity(id) {
community = append(community, id)
} else {
kept = append(kept, id)
}
}
p.recordWorn(user, kept, community)
if len(community) == 0 {
return nil
}
p.logf("products: kept %d community products out of the outfit saved to IMVU", len(community))
save["pids"], _ = json.Marshal(kept)
body, _ := json.Marshal(save)
if !p.keys.resign(x.Request.Header, x.Body, x.URL.RawQuery, body, x.URL.RawQuery) {
return errNoKey
}
x.Body = body
return nil
}
// Outgoing implements proxy.IMQFilter: community products are taken out of outfit chat commands (IMVU and other
// people see the defaults), and whole outfits are remembered.
func (p *Products) Outgoing(queue, mount string, message []byte) ([]byte, bool) {
if !strings.HasPrefix(queue, "/chat/") || mount != "messages" {
return message, false
}
m, ok := parseChatMessage(message)
if !ok {
return message, false
}
r := stripCommand(m.text, IsCommunity)
if r.outfit && m.userID != 0 {
p.recordWorn(strconv.FormatInt(m.userID, 10), r.kept, r.products)
}
if !r.changed {
return message, false
}
command, _, _ := strings.Cut(r.text, " ")
if r.empty {
p.logf("products: answered a %s of only community products without sending it", command)
return nil, true
}
p.logf("products: kept %d community products out of a %s sent to IMVU", len(r.products), command)
return m.withText(r.text), false
}
+282
View File
@@ -0,0 +1,282 @@
// Package products serves community products to the client from local packs (t5de-api's pack format), and keeps
// them away from IMVU.
//
// The client knows products by number, so each product gets a local ID from FirstLocalID up. For those IDs the
// proxy answers the client itself: product info (test.inventoryProductBatch), permission and file addresses
// (test.authorizeProducts2), the shop's product lookup, and the product's CFL. A product deriving from an IMVU
// product (its index.xml imports the parent's) gets the parent's answers from IMVU, asked for the parent. Outfits
// keep community products for the client but not for IMVU: they're taken out of outfit saves (base_look.php) and
// outfit chat commands (*use ...), remembered per account, and added back when the client loads the outfit.
package products
import (
"bytes"
"crypto/md5"
"encoding/hex"
"errors"
"fmt"
"io/fs"
"net/http"
"os"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"sync"
"time"
"git.t5de.dev/t5de/t5de-api/cfl"
"git.t5de.dev/t5de/t5de-api/pack"
)
// Products is the proxy's community products: an Interceptor for HTTP and an IMQFilter for chat.
type Products struct {
dirs []string
state *state
keys keys
logf func(format string, args ...any)
mu sync.Mutex
scanned time.Time
byLocal map[int64]*served
byID map[string]*served
problems map[string]string // pack directory -> last error logged
}
// served is a pack ready to serve.
type served struct {
dir string
fingerprint string
pack *pack.Pack
local int64
cfl []byte
checksum string // md5 of cfl, as IMVU's cflChecksum
}
// New serves the packs in dirs (each pack a subdirectory), remembering its state in dataDir.
func New(dirs []string, dataDir string, logf func(format string, args ...any)) (*Products, error) {
st, err := loadState(filepath.Join(dataDir, "products.json"))
if err != nil {
return nil, err
}
if logf == nil {
logf = func(string, ...any) {}
}
p := &Products{dirs: dirs, state: st, logf: logf, byLocal: map[int64]*served{}, byID: map[string]*served{},
problems: map[string]string{}}
p.refresh(true)
return p, nil
}
// IsCommunity reports whether the client's product ID is a community product's local ID.
func IsCommunity(id int64) bool { return id >= FirstLocalID }
// lookup is the pack served under a local ID, after picking up changed packs.
func (p *Products) lookup(local int64) *served {
p.refresh(false)
p.mu.Lock()
defer p.mu.Unlock()
return p.byLocal[local]
}
func (p *Products) byProductID(id string) *served {
p.refresh(false)
p.mu.Lock()
defer p.mu.Unlock()
return p.byID[id]
}
// refresh reloads packs that changed on disk, at most once a second unless forced. Creators editing a pack see the
// new files the next time the client loads the product.
func (p *Products) refresh(force bool) {
p.mu.Lock()
defer p.mu.Unlock()
if !force && time.Since(p.scanned) < time.Second {
return
}
p.scanned = time.Now()
seen := map[string]bool{}
for _, root := range p.dirs {
entries, err := os.ReadDir(root)
if err != nil {
continue
}
for _, e := range entries {
dir := filepath.Join(root, e.Name())
if !e.IsDir() {
continue
}
if _, err := os.Stat(filepath.Join(dir, "t5de-product.json")); err != nil {
continue
}
fp := fingerprint(dir)
if old := p.findDirLocked(dir); old != nil && old.fingerprint == fp {
seen[old.pack.Manifest.ID] = true
continue
}
s, err := p.loadLocked(dir, fp)
if err != nil {
if p.problems[dir] != err.Error() {
p.logf("products: %s: %v", dir, err)
p.problems[dir] = err.Error()
}
if old := p.findDirLocked(dir); old != nil {
seen[old.pack.Manifest.ID] = true // keep serving the last good one
}
continue
}
delete(p.problems, dir)
if other, ok := p.byID[s.pack.Manifest.ID]; ok && other.dir != dir && seen[s.pack.Manifest.ID] {
p.logf("products: %s: product %q is also in %s; ignoring this one", dir, s.pack.Manifest.ID, other.dir)
continue
}
p.byID[s.pack.Manifest.ID] = s
p.byLocal[s.local] = s
seen[s.pack.Manifest.ID] = true
p.logf("products: serving %q revision %d (%s) as %d", s.pack.Manifest.ID, s.pack.Manifest.Revision, s.pack.Manifest.Name, s.local)
}
}
for id, s := range p.byID {
if !seen[id] {
delete(p.byID, id)
delete(p.byLocal, s.local)
p.logf("products: %q is gone", id)
}
}
}
func (p *Products) findDirLocked(dir string) *served {
for _, s := range p.byID {
if s.dir == dir {
return s
}
}
return nil
}
func (p *Products) loadLocked(dir, fp string) (*served, error) {
pk, err := pack.Load(dir)
if err != nil {
return nil, err
}
local, err := p.state.localID(pk.Manifest.ID)
if err != nil {
return nil, err
}
entries := make([]cfl.Entry, 0, len(pk.Files))
for _, f := range pk.Files {
data := f.Data
if f.Name == "index.xml" {
data = withProductName(data, pk.Manifest.Name)
}
entries = append(entries, cfl.Entry{Name: f.Name, Data: data})
}
var buf bytes.Buffer
if err := cfl.Write(&buf, entries); err != nil {
return nil, err
}
sum := md5.Sum(buf.Bytes())
return &served{dir: dir, fingerprint: fp, pack: pk, local: local, cfl: buf.Bytes(), checksum: hex.EncodeToString(sum[:])}, nil
}
// fingerprint changes when anything in the pack directory (or its files/ directory) changes.
func fingerprint(dir string) string {
var b strings.Builder
filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return nil
}
if info, err := d.Info(); err == nil {
fmt.Fprintf(&b, "%s|%d|%d\n", path, info.Size(), info.ModTime().UnixNano())
}
return nil
})
return b.String()
}
var productNamePattern = regexp.MustCompile(`(?s)<ProductName>.*?</ProductName>`)
// withProductName puts the pack's name in index.xml, so the client shows it.
func withProductName(index []byte, name string) []byte {
var esc bytes.Buffer
xmlEscape(&esc, name)
repl := []byte("<ProductName>" + esc.String() + "</ProductName>")
if productNamePattern.Match(index) {
return productNamePattern.ReplaceAllLiteral(index, repl)
}
return index
}
func xmlEscape(b *bytes.Buffer, s string) {
r := strings.NewReplacer("&", "&amp;", "<", "&lt;", ">", "&gt;", `"`, "&quot;", "'", "&apos;")
b.WriteString(r.Replace(s))
}
// productdata matches IMVU's product file addresses: /productdata/<id>/<revision>[/<file>].
var productdataPath = regexp.MustCompile(`^/productdata/([0-9]+)/([0-9]+)(?:/(.+))?$`)
// cflURL is where the client fetches a community product's CFL: IMVU's address form, answered by the proxy.
func cflURL(local int64, revision int) string {
return fmt.Sprintf("http://userimages-akm.imvu.com/productdata/%d/%d", local, revision)
}
// serveFile answers a request for a community product's files.
func (p *Products) serveFile(path string) (*http.Response, bool) {
m := productdataPath.FindStringSubmatch(path)
if m == nil {
return nil, false
}
local, _ := strconv.ParseInt(m[1], 10, 64)
if !IsCommunity(local) {
return nil, false
}
s := p.lookup(local)
rev, _ := strconv.Atoi(m[2])
if s == nil || rev != s.pack.Manifest.Revision {
return notFound(), true
}
if m[3] == "" {
resp := localResponse(200, "application/octet-stream", s.cfl)
resp.Header.Set("Content-MD5", s.checksum)
return resp, true
}
for _, f := range s.pack.Files {
if f.Name == m[3] {
return localResponse(200, "application/octet-stream", f.Data), true
}
}
return notFound(), true
}
func notFound() *http.Response {
return localResponse(404, "text/plain", []byte("t5de: no such community product file"))
}
// wornLocal is the user's community products (local IDs, still served) that go with the IMVU outfit look.
func (p *Products) wornLocal(user string, look []int64) []int64 {
var ids []int64
for _, id := range p.state.worn(user, look) {
if s := p.byProductID(id); s != nil {
ids = append(ids, s.local)
}
}
sort.Slice(ids, func(i, j int) bool { return ids[i] < ids[j] })
return ids
}
// recordWorn remembers the user's outfit: the IMVU products sent to IMVU, and the community products (local IDs)
// taken out.
func (p *Products) recordWorn(user string, imvu, locals []int64) {
var ids []string
for _, local := range locals {
if id, ok := p.state.productID(local); ok {
ids = append(ids, id)
}
}
if err := p.state.setWorn(user, imvu, ids); err != nil {
p.logf("products: saving what %s wears: %v", user, err)
}
}
var errNoKey = errors.New("products: can't re-sign the request (no session key yet)")
+308
View File
@@ -0,0 +1,308 @@
package products
import (
"bufio"
"bytes"
"encoding/base64"
"encoding/json"
"fmt"
"io"
"net"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"git.t5de.dev/t5de/t5de-api/cfl"
"git.t5de.dev/t5de/t5de-proxy/internal/proxy"
"git.t5de.dev/t5de/t5de-proxy/internal/xmlrpc"
)
const (
user = "7"
key = "0123abcd"
parent = 500 // the IMVU product the test pack derives from
token = "tok"
)
// writePack makes a pack deriving from IMVU product 500.
func writePack(t *testing.T, root, id string, revision int) string {
dir := filepath.Join(root, id)
os.MkdirAll(filepath.Join(dir, "files"), 0o755)
manifest := fmt.Sprintf(`{"format": 1, "id": %q, "revision": %d, "name": "Blue <Coat>", "creator": "Tester", "rating": "general"}`, id, revision)
os.WriteFile(filepath.Join(dir, "t5de-product.json"), []byte(manifest), 0o644)
index := fmt.Sprintf(`<?xml version="1.0" encoding="UTF-8" ?><Template><ProductRating>GA</ProductRating><__DATAIMPORT>product://%d/index.xml</__DATAIMPORT><ProductName>Old Name</ProductName></Template>`, parent)
os.WriteFile(filepath.Join(dir, "files", "index.xml"), []byte(index), 0o644)
os.WriteFile(filepath.Join(dir, "files", "tex.png"), []byte("not really a png"), 0o644)
return dir
}
// fakeIMVU answers the calls the client makes, checking signatures, and records what it was asked.
type fakeIMVU struct {
t *testing.T
mu sync.Mutex
asked map[string][]string // method or path -> bodies
look []int64
}
func (f *fakeIMVU) record(name string, body []byte) {
f.mu.Lock()
defer f.mu.Unlock()
f.asked[name] = append(f.asked[name], string(body))
}
func (f *fakeIMVU) RoundTrip(r *http.Request) (*http.Response, error) {
var body []byte
if r.Body != nil {
body, _ = io.ReadAll(r.Body)
}
reply := func(s string) (*http.Response, error) {
return &http.Response{StatusCode: 200, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(s)), Request: r}, nil
}
checkSigned := func(withQuery bool) {
if r.Header.Get("X-imvu-auth") != sign(user, key, body, r.URL.RawQuery, withQuery) {
f.t.Errorf("%s: bad signature", r.URL)
}
}
switch r.URL.Path {
case "/api/base_look.php":
if len(body) > 0 {
checkSigned(true)
f.record("save look", body)
var save struct{ Pids []int64 }
json.Unmarshal(body, &save)
f.look = save.Pids
return reply(`{"success": true}`)
}
b, _ := json.Marshal(map[string]any{"result": f.look})
return reply(string(b))
case "/api/xmlrpc/client.php":
call, err := xmlrpc.ParseCall(body)
if err != nil {
f.t.Fatalf("bad call: %v", err)
}
f.record(call.Method, body)
switch call.Method {
case "test.avatarInfoForLogin2":
return reply(string((&xmlrpc.Response{Params: []xmlrpc.Value{xmlrpc.Struct{
{Name: "userId", Value: int64(7)}, {Name: "securityKey", Value: key},
{Name: "look", Value: []xmlrpc.Value{int64(80), int64(1)}}}}}).Marshal()))
case "test.inventoryProductBatch":
checkSigned(false)
var out []xmlrpc.Value
for _, v := range call.Params[0].([]xmlrpc.Value) {
out = append(out, xmlrpc.Struct{{Name: "products_id", Value: v}, {Name: "products_name", Value: "IMVU's"},
{Name: "cPath", Value: "106_40"}, {Name: "manufacturers_name", Value: "IMVU Creator"}})
}
return reply(string((&xmlrpc.Response{Params: []xmlrpc.Value{out}}).Marshal()))
case "test.authorizeProducts2":
checkSigned(false)
cols := []xmlrpc.Value{}
for _, c := range authColumns {
cols = append(cols, c)
}
rows := []xmlrpc.Value{cols}
auths, _ := call.Params[0].(xmlrpc.Struct).Get("auths")
for _, a := range auths.([]xmlrpc.Value)[1:] {
pid := a.([]xmlrpc.Value)[1].(int64)
rows = append(rows, []xmlrpc.Value{int64(7), int64(400), pid, "sum", fmt.Sprintf("http://x/productdata/%d/1", pid), "", false, false, int64(1), true, int64(40), true, ""},
[]xmlrpc.Value{int64(7), int64(0), int64(400), "sum2", "http://x/productdata/400/1", "", false, false, int64(1), true, int64(40), true, ""})
}
return reply(string((&xmlrpc.Response{Params: []xmlrpc.Value{rows}}).Marshal()))
}
}
f.t.Errorf("unexpected request to IMVU: %s", r.URL)
return reply("")
}
type harness struct {
t *testing.T
p *Products
imvu *fakeIMVU
proxy *httptest.Server
}
func newHarness(t *testing.T) (*harness, string) {
packs := t.TempDir()
writePack(t, packs, "blue-coat", 1)
p, err := New([]string{packs}, t.TempDir(), t.Logf)
if err != nil {
t.Fatal(err)
}
f := &fakeIMVU{t: t, asked: map[string][]string{}, look: []int64{80, 1}}
srv := httptest.NewServer(&proxy.Server{Token: token, Transport: f, Interceptor: p, IMQFilter: p})
t.Cleanup(srv.Close)
return &harness{t: t, p: p, imvu: f, proxy: srv}, packs
}
// do sends a request through the proxy as the client does: signed (if signed), in absolute form.
func (h *harness) do(method, url string, body []byte, signed, withQuery bool) (int, []byte) {
h.t.Helper()
req, _ := http.NewRequest(method, url, bytes.NewReader(body))
req.Header.Set("Proxy-Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte("t5de:"+token)))
if signed {
req.Header.Set("X-imvu-userid", user)
req.Header.Set("X-imvu-auth", sign(user, key, body, req.URL.RawQuery, withQuery))
}
conn, err := net.Dial("tcp", h.proxy.Listener.Addr().String())
if err != nil {
h.t.Fatal(err)
}
defer conn.Close()
req.WriteProxy(conn)
resp, err := http.ReadResponse(bufio.NewReader(conn), req)
if err != nil {
h.t.Fatal(err)
}
defer resp.Body.Close()
b, _ := io.ReadAll(resp.Body)
return resp.StatusCode, b
}
func call(method string, params ...xmlrpc.Value) []byte {
return (&xmlrpc.Call{Method: method, Params: params}).Marshal()
}
func TestCommunityProductEndToEnd(t *testing.T) {
h, _ := newHarness(t)
local := h.p.byProductID("blue-coat").local
if local != FirstLocalID {
t.Fatalf("local ID %d", local)
}
api := "http://client-dynamic.imvu.com/api/xmlrpc/client.php"
// Logging in teaches the proxy the session key.
h.do("POST", "https://secure.imvu.com/api/xmlrpc/client.php", call("test.avatarInfoForLogin2"), false, false)
// Product info: the community product's from its parent's, with its own name; IMVU is asked for the parent.
_, body := h.do("POST", api, call("test.inventoryProductBatch", []xmlrpc.Value{int64(1), local}), true, false)
r, err := xmlrpc.ParseResponse(body)
if err != nil {
t.Fatal(err, string(body))
}
list := r.Params[0].([]xmlrpc.Value)
if len(list) != 2 {
t.Fatalf("%d entries: %s", len(list), body)
}
ours := list[1].(xmlrpc.Struct)
if id, _ := ours.Get("products_id"); id != local {
t.Fatalf("entry %v", ours)
}
if name, _ := ours.Get("products_name"); name != "Blue <Coat>" {
t.Fatalf("name %v", name)
}
if path, _ := ours.Get("cPath"); path != "106_40" {
t.Fatalf("cPath %v", path)
}
if asked := h.imvu.asked["test.inventoryProductBatch"]; len(asked) != 1 || strings.Contains(asked[0], fmt.Sprint(local)) || !strings.Contains(asked[0], "<int>500</int>") {
t.Fatalf("IMVU was asked %v", asked)
}
// Permission: our row (with the CFL's address and checksum) and IMVU's rows for the parent's chain.
auth := xmlrpc.Struct{{Name: "auths", Value: []xmlrpc.Value{[]xmlrpc.Value{"userId", "productId"}, []xmlrpc.Value{int64(7), local}}},
{Name: "use_extracted_assets_if_available", Value: true}, {Name: "max_supported_format", Value: int64(9)}}
_, body = h.do("POST", api, call("test.authorizeProducts2", auth), true, false)
r, err = xmlrpc.ParseResponse(body)
if err != nil {
t.Fatal(err)
}
rows := r.Params[0].([]xmlrpc.Value)
var cflAddress string
for _, row := range rows[1:] {
values := row.([]xmlrpc.Value)
if values[2] == local {
cflAddress = values[4].(string)
if values[1] != int64(parent) || values[3] != h.p.byProductID("blue-coat").checksum || values[6] != true {
t.Fatalf("our row %v", values)
}
}
}
if cflAddress == "" || len(rows) != 4 {
t.Fatalf("rows %v", rows)
}
// The CFL: the pack's files, with the pack's name in index.xml. IMVU is never asked.
status, data := h.do("GET", cflAddress, nil, false, false)
entries, err := cfl.Read(data)
if status != 200 || err != nil || len(entries) != 2 {
t.Fatalf("%d %v %v", status, err, entries)
}
if !bytes.Contains(entries[0].Data, []byte("<ProductName>Blue &lt;Coat&gt;</ProductName>")) {
t.Fatalf("index.xml %s", entries[0].Data)
}
if status, _ := h.do("GET", strings.Replace(cflAddress, "/1", "/2", 1), nil, false, false); status != 404 {
t.Fatalf("another revision: %d", status)
}
// Saving an outfit with it: IMVU gets the outfit without it; loading the outfit adds it back.
h.do("POST", "http://client-dynamic.imvu.com/api/base_look.php", []byte(fmt.Sprintf(`{"pids": [80, 1, %d]}`, local)), true, true)
if saved := h.imvu.asked["save look"]; len(saved) != 1 || saved[0] != `{"pids":[80,1]}` {
t.Fatalf("IMVU saved %v", saved)
}
_, body = h.do("GET", "http://client-dynamic.imvu.com/api/base_look.php", nil, true, true)
if string(body) != fmt.Sprintf(`{"result":[80,1,%d]}`, local) {
t.Fatalf("look %s", body)
}
_, body = h.do("POST", "https://secure.imvu.com/api/xmlrpc/client.php", call("test.avatarInfoForLogin2"), false, false)
if !bytes.Contains(body, []byte(fmt.Sprintf("<int>%d</int>", local))) {
t.Fatalf("login look %s", body)
}
// If the outfit IMVU has changes (saved elsewhere), it comes back without community products.
h.imvu.look = []int64{80, 2}
_, body = h.do("GET", "http://client-dynamic.imvu.com/api/base_look.php", nil, true, true)
if string(body) != `{"result":[80,2]}` {
t.Fatalf("changed look %s", body)
}
// The shop's bundle lookup is answered without IMVU.
if _, body := h.do("GET", fmt.Sprintf("http://client-dynamic.imvu.com/api/shop/product.php?pid=%d", local), nil, true, true); string(body) != `{"products":[]}` {
t.Fatalf("product.php %s", body)
}
}
func TestOutgoingChat(t *testing.T) {
h, _ := newHarness(t)
local := h.p.byProductID("blue-coat").local
msg := func(text string) []byte {
return []byte(fmt.Sprintf(`{"to": 0, "message": %q, "userId": 7, "chatId": 9}`, text))
}
out, drop := h.p.Outgoing("/chat/9", "messages", msg(fmt.Sprintf("*use 80 1 %d", local)))
if drop || !bytes.Contains(out, []byte(`"message":"*use 80 1"`)) {
t.Fatalf("%s %v", out, drop)
}
if got := h.p.wornLocal(user, []int64{1, 80}); len(got) != 1 || got[0] != local {
t.Fatalf("worn %v", got)
}
// With a different IMVU outfit (changed elsewhere), community products aren't added to it.
if got := h.p.wornLocal(user, []int64{80, 2}); len(got) != 0 {
t.Fatalf("worn with another outfit %v", got)
}
if _, drop := h.p.Outgoing("/chat/9", "messages", msg(fmt.Sprintf("*use %d", local))); !drop {
t.Fatal("a command with only community products was sent")
}
if out, _ := h.p.Outgoing("/chat/9", "messages", msg("hello")); !bytes.Equal(out, msg("hello")) {
t.Fatal("plain chat changed")
}
if out, _ := h.p.Outgoing("room_occupancy/1", "occupants", []byte("x")); string(out) != "x" {
t.Fatal("another queue changed")
}
}
func TestPackChangesAreServed(t *testing.T) {
h, packs := newHarness(t)
before := h.p.byProductID("blue-coat")
os.WriteFile(filepath.Join(packs, "blue-coat", "files", "tex.png"), []byte("a new texture, longer"), 0o644)
h.p.refresh(true)
after := h.p.byProductID("blue-coat")
if after.checksum == before.checksum || after.local != before.local {
t.Fatalf("checksum %s -> %s, local %d -> %d", before.checksum, after.checksum, before.local, after.local)
}
os.RemoveAll(filepath.Join(packs, "blue-coat"))
h.p.refresh(true)
if h.p.byProductID("blue-coat") != nil {
t.Fatal("a removed pack is still served")
}
}
+65 -18
View File
@@ -24,12 +24,20 @@ type state struct {
}
type stateData struct {
IDs map[string]int64 `json:"ids"` // product ID -> local ID
Worn map[string][]string `json:"worn"` // IMVU user ID -> product IDs
IDs map[string]int64 `json:"ids"` // product ID -> local ID
Worn map[string]wornOutfit `json:"worn"` // IMVU user ID -> what they wear
}
// wornOutfit is an account's outfit as last saved or sent: the IMVU products IMVU got, and the community products
// taken out. The community products are added back only to that same outfit: if IMVU's outfit changed elsewhere
// (another client, the website), they'd no longer fit it.
type wornOutfit struct {
IMVU []int64 `json:"imvu"`
Products []string `json:"products"`
}
func loadState(path string) (*state, error) {
s := &state{path: path, data: stateData{IDs: map[string]int64{}, Worn: map[string][]string{}}}
s := &state{path: path, data: stateData{IDs: map[string]int64{}, Worn: map[string]wornOutfit{}}}
b, err := os.ReadFile(path)
if errors.Is(err, fs.ErrNotExist) {
return s, nil
@@ -44,7 +52,7 @@ func loadState(path string) (*state, error) {
s.data.IDs = map[string]int64{}
}
if s.data.Worn == nil {
s.data.Worn = map[string][]string{}
s.data.Worn = map[string]wornOutfit{}
}
return s, nil
}
@@ -67,30 +75,57 @@ func (s *state) localID(product string) (int64, error) {
return next, s.saveLocked()
}
// worn is the community products the user wears.
func (s *state) worn(user string) []string {
// worn is the community products the user wears with the IMVU outfit look, if they were recorded with it.
func (s *state) worn(user string, look []int64) []string {
s.mu.Lock()
defer s.mu.Unlock()
return append([]string(nil), s.data.Worn[user]...)
}
// setWorn records the community products the user wears.
func (s *state) setWorn(user string, products []string) error {
products = append([]string(nil), products...)
sort.Strings(products)
s.mu.Lock()
defer s.mu.Unlock()
if equal(s.data.Worn[user], products) {
w, ok := s.data.Worn[user]
if !ok || !sameSet(w.IMVU, look) {
return nil
}
if len(products) == 0 {
return append([]string(nil), w.Products...)
}
// setWorn records the user's outfit: the IMVU products IMVU got, and the community products taken out.
func (s *state) setWorn(user string, imvu []int64, products []string) error {
w := wornOutfit{IMVU: append([]int64(nil), imvu...), Products: append([]string(nil), products...)}
sort.Slice(w.IMVU, func(i, j int) bool { return w.IMVU[i] < w.IMVU[j] })
sort.Strings(w.Products)
s.mu.Lock()
defer s.mu.Unlock()
old, had := s.data.Worn[user]
if had && sameSet(old.IMVU, w.IMVU) && equal(old.Products, w.Products) {
return nil
}
if len(w.Products) == 0 {
if !had {
return nil
}
delete(s.data.Worn, user)
} else {
s.data.Worn[user] = products
s.data.Worn[user] = w
}
return s.saveLocked()
}
// sameSet reports whether a and b hold the same IDs, in any order.
func sameSet(a, b []int64) bool {
if len(a) != len(b) {
return false
}
count := map[int64]int{}
for _, v := range a {
count[v]++
}
for _, v := range b {
if count[v] == 0 {
return false
}
count[v]--
}
return true
}
func equal(a, b []string) bool {
if len(a) != len(b) {
return false
@@ -117,3 +152,15 @@ func (s *state) saveLocked() error {
}
return os.Rename(tmp, s.path)
}
// productID is the product a local ID was given to.
func (s *state) productID(local int64) (string, bool) {
s.mu.Lock()
defer s.mu.Unlock()
for id, l := range s.data.IDs {
if l == local {
return id, true
}
}
return "", false
}