diff --git a/cmd/t5de-proxy/main.go b/cmd/t5de-proxy/main.go index a8014cc..b6fc869 100644 --- a/cmd/t5de-proxy/main.go +++ b/cmd/t5de-proxy/main.go @@ -18,10 +18,13 @@ import ( "net" "net/http" "os" + "path/filepath" + "strings" "time" "git.t5de.dev/t5de/t5de-proxy/internal/buildinfo" "git.t5de.dev/t5de/t5de-proxy/internal/parent" + "git.t5de.dev/t5de/t5de-proxy/internal/products" "git.t5de.dev/t5de/t5de-proxy/internal/proxy" ) @@ -31,6 +34,9 @@ func main() { parentPID := flag.Int("parent", 0, "exit when this process exits") logFile := flag.String("log", "", "log file (default: standard error)") trace := flag.Bool("trace", false, "log full URLs and every IMQ frame") + packs := flag.String("packs", "", "community product packs: directories of packs, separated by "+string(filepath.ListSeparator)+ + " (default: %APPDATA%/T5DE/packs and $T5DE_PACKS)") + data := flag.String("data", "", "where the proxy keeps its state (default: %APPDATA%/T5DE/proxy)") version := flag.Bool("version", false, "print the version and exit") flag.Parse() if *version { @@ -48,13 +54,35 @@ func main() { out = f } logger := log.New(out, "", log.Ldate|log.Ltime|log.Lmicroseconds) - if err := run(logger, *listen, *ready, *parentPID, *trace); err != nil { + if err := run(logger, *listen, *ready, *parentPID, *trace, packDirs(*packs), dataDir(*data)); err != nil { logger.Printf("t5de-proxy: %v", err) os.Exit(1) } } -func run(logger *log.Logger, listen, ready string, parentPID int, trace bool) error { +// packDirs is where community product packs are: the -packs flag, else T5DE's packs folder and $T5DE_PACKS. +func packDirs(flagValue string) []string { + if flagValue != "" { + return filepath.SplitList(flagValue) + } + var dirs []string + if config, err := os.UserConfigDir(); err == nil { + dirs = append(dirs, filepath.Join(config, "T5DE", "packs")) + } + return append(dirs, filepath.SplitList(os.Getenv("T5DE_PACKS"))...) +} + +func dataDir(flagValue string) string { + if flagValue != "" { + return flagValue + } + if config, err := os.UserConfigDir(); err == nil { + return filepath.Join(config, "T5DE", "proxy") + } + return "." +} + +func run(logger *log.Logger, listen, ready string, parentPID int, trace bool, packs []string, data string) error { secret := make([]byte, 24) if _, err := rand.Read(secret); err != nil { return err @@ -78,8 +106,15 @@ func run(logger *log.Logger, listen, ready string, parentPID int, trace bool) er os.Exit(0) }() } + handler := &proxy.Server{Token: token, Log: logger, Trace: trace} + if prods, err := products.New(packs, data, logger.Printf); err != nil { + logger.Printf("community products are off: %v", err) + } else { + handler.Interceptor, handler.IMQFilter = prods, prods + logger.Printf("community products from %s", strings.Join(packs, ", ")) + } server := &http.Server{ - Handler: &proxy.Server{Token: token, Log: logger, Trace: trace}, + Handler: handler, ReadHeaderTimeout: 30 * time.Second, IdleTimeout: 2 * time.Minute, ErrorLog: logger, diff --git a/go.mod b/go.mod index 1f56b4c..bf81b2c 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,7 @@ module git.t5de.dev/t5de/t5de-proxy go 1.27.0 + +require git.t5de.dev/t5de/t5de-api v0.0.0-00010101000000-000000000000 + +replace git.t5de.dev/t5de/t5de-api => ../t5de-api diff --git a/internal/products/chat.go b/internal/products/chat.go index 26a2a97..0bd11db 100644 --- a/internal/products/chat.go +++ b/internal/products/chat.go @@ -22,6 +22,7 @@ type strippedCommand struct { empty bool // nothing is left to send outfit bool // the line was a whole outfit (*use with the body) products []int64 // the community products it named + kept []int64 // the other products it named } // stripCommand takes products for which isCommunity is true out of an outfit command's arguments. @@ -41,6 +42,9 @@ func stripCommand(text string, isCommunity func(int64) bool) strippedCommand { r.products = append(r.products, id) continue } + if err == nil { + r.kept = append(r.kept, id) + } kept = append(kept, f) } if len(r.products) == 0 { diff --git a/internal/products/intercept.go b/internal/products/intercept.go new file mode 100644 index 0000000..0ca01e6 --- /dev/null +++ b/internal/products/intercept.go @@ -0,0 +1,476 @@ +package products + +import ( + "encoding/json" + "errors" + "net/http" + "strconv" + "strings" + + "git.t5de.dev/t5de/t5de-proxy/internal/proxy" + "git.t5de.dev/t5de/t5de-proxy/internal/xmlrpc" +) + +func localResponse(status int, contentType string, body []byte) *http.Response { + return proxy.LocalResponse(status, contentType, body) +} + +// Intercept implements proxy.Interceptor. +func (p *Products) Intercept(x *proxy.Exchange) error { + host, path := strings.ToLower(x.URL.Hostname()), x.URL.Path + if strings.HasSuffix(host, ".imvu.com") && strings.HasPrefix(path, "/productdata/") { + if resp, ok := p.serveFile(path); ok { + x.Response = resp + } + return nil + } + if !strings.HasSuffix(host, ".imvu.com") { + return nil + } + switch path { + case "/api/xmlrpc/client.php": + return p.interceptXMLRPC(x) + case "/api/shop/product.php": + // Putting a product on asks the shop whether it's a bundle; community products aren't. + if id, err := strconv.ParseInt(x.URL.Query().Get("pid"), 10, 64); err == nil && IsCommunity(id) { + x.Response = localResponse(200, "application/json", []byte(`{"products":[]}`)) + } + case "/api/base_look.php": + return p.interceptBaseLook(x) + } + return nil +} + +func (p *Products) interceptXMLRPC(x *proxy.Exchange) error { + call, err := xmlrpc.ParseCall(x.Body) + if err != nil { + return nil // not ours to judge; IMVU answers + } + switch call.Method { + case "test.avatarInfoForLogin2": + x.ModifyResponse = func(resp *http.Response, body []byte) ([]byte, error) { + p.keys.learn(body) + return p.addToLook(body) + } + case "test.inventoryProductBatch": + return p.inventoryProductBatch(x, call) + case "test.authorizeProducts2": + return p.authorizeProducts2(x, call) + } + return nil +} + +// replaceBody sends call instead of the client's, re-signed. +func (p *Products) replaceBody(x *proxy.Exchange, call *xmlrpc.Call) error { + body := call.Marshal() + if !p.keys.resign(x.Request.Header, x.Body, x.URL.RawQuery, body, x.URL.RawQuery) { + return errNoKey + } + x.Body = body + return nil +} + +// addToLook adds the user's community products to the outfit ("look") in a login response. +func (p *Products) addToLook(body []byte) ([]byte, error) { + m := userIDPattern.FindSubmatch(body) + if m == nil { + return body, nil + } + user := string(m[1]) + r, err := xmlrpc.ParseResponse(body) + if err != nil || r.Fault != nil { + return body, err + } + changed := false + var walk func(v xmlrpc.Value) xmlrpc.Value + walk = func(v xmlrpc.Value) xmlrpc.Value { + switch v := v.(type) { + case xmlrpc.Struct: + for i, mem := range v { + if mem.Name == "look" { + if look, ok := productList(mem.Value); ok { + if worn := p.wornLocal(user, look); len(worn) > 0 { + v[i].Value = appendIDs(mem.Value.([]xmlrpc.Value), worn) + changed = true + } + continue + } + } + v[i].Value = walk(mem.Value) + } + return v + case []xmlrpc.Value: + for i := range v { + v[i] = walk(v[i]) + } + return v + } + return v + } + for i := range r.Params { + r.Params[i] = walk(r.Params[i]) + } + if !changed { + return body, nil + } + return r.Marshal(), nil +} + +func appendIDs(list []xmlrpc.Value, ids []int64) []xmlrpc.Value { + have := map[int64]bool{} + for _, v := range list { + if n, ok := v.(int64); ok { + have[n] = true + } + } + for _, id := range ids { + if !have[id] { + list = append(list, id) + } + } + return list +} + +// productList reads a list of product IDs. +func productList(v xmlrpc.Value) ([]int64, bool) { + list, ok := v.([]xmlrpc.Value) + if !ok { + return nil, false + } + ids := make([]int64, 0, len(list)) + for _, v := range list { + n, ok := v.(int64) + if !ok { + return nil, false + } + ids = append(ids, n) + } + return ids, true +} + +// inventoryProductBatch answers product info for community products: the parent's info from IMVU with the +// product's own name, creator and ID, or (without a parent) info of its own. +func (p *Products) inventoryProductBatch(x *proxy.Exchange, call *xmlrpc.Call) error { + if len(call.Params) != 1 { + return nil + } + ids, ok := productList(call.Params[0]) + if !ok { + return nil + } + var community []int64 + var forward []xmlrpc.Value + asked := map[int64]bool{} + for _, id := range ids { + if !IsCommunity(id) { + if !asked[id] { + forward, asked[id] = append(forward, id), true + } + continue + } + community = append(community, id) + if s := p.lookup(id); s != nil && s.pack.Index.Parent > 0 && !asked[int64(s.pack.Index.Parent)] { + forward, asked[int64(s.pack.Index.Parent)] = append(forward, int64(s.pack.Index.Parent)), true + } + } + if len(community) == 0 { + return nil + } + // The answer, in the order asked, from IMVU's entries (by product ID) and our own. + answer := func(imvu map[int64]xmlrpc.Struct) []byte { + var out []xmlrpc.Value + for _, id := range ids { + if !IsCommunity(id) { + if e, ok := imvu[id]; ok { + out = append(out, e) + } + continue + } + if s := p.lookup(id); s != nil { + out = append(out, s.productInfo(imvu[int64(s.pack.Index.Parent)])) + } + } + return (&xmlrpc.Response{Params: []xmlrpc.Value{out}}).Marshal() + } + if len(forward) == 0 { + x.Response = localResponse(200, "text/xml", answer(nil)) + return nil + } + if err := p.replaceBody(x, &xmlrpc.Call{Method: call.Method, Params: []xmlrpc.Value{forward}}); err != nil { + return err + } + x.ModifyResponse = func(resp *http.Response, body []byte) ([]byte, error) { + r, err := xmlrpc.ParseResponse(body) + if err != nil || r.Fault != nil || len(r.Params) != 1 { + return body, err + } + list, _ := r.Params[0].([]xmlrpc.Value) + imvu := map[int64]xmlrpc.Struct{} + for _, v := range list { + if e, ok := v.(xmlrpc.Struct); ok { + if id, ok := e.Get("products_id"); ok { + if n, ok := id.(int64); ok { + imvu[n] = e + } + } + } + } + return answer(imvu), nil + } + return nil +} + +// productInfo is a community product's entry for inventoryProductBatch, based on its parent's if there is one. +func (s *served) productInfo(parent xmlrpc.Struct) xmlrpc.Struct { + m := s.pack.Manifest + mature := "N" + if m.Rating == "adult" { + mature = "Y" + } + own := map[string]xmlrpc.Value{ + "products_id": s.local, + "manufacturers_id": int64(0), + "manufacturers_name": m.Creator, + "products_name": m.Name, + "products_mature": mature, + "products_bundle": "", + "is_published": "1", + "is_purchasable": false, + } + var out xmlrpc.Struct + done := map[string]bool{} + for _, mem := range parent { + if v, ok := own[mem.Name]; ok { + out = append(out, xmlrpc.Member{Name: mem.Name, Value: v}) + done[mem.Name] = true + } else { + out = append(out, mem) // cPath, products_image, ... from the parent + } + } + for _, name := range []string{"products_id", "manufacturers_id", "manufacturers_name", "products_name", "cPath", + "products_mature", "products_bundle", "products_image", "is_published", "is_purchasable"} { + if done[name] { + continue + } + v, ok := own[name] + if !ok { + v = "" // cPath and products_image, without a parent + } + if _, inParent := parent.Get(name); !inParent { + out = append(out, xmlrpc.Member{Name: name, Value: v}) + } + } + return out +} + +// authColumns are authorizeProducts2's answer columns, as IMVU sends them. +var authColumns = []string{"userId", "parentProductId", "productId", "cflChecksum", "cflUrl", "sub_pids", + "usableDirect", "autoAuth", "usableDerivation", "usableTrial", "trialTimeout", "usablePeerReview", "info"} + +// authorizeProducts2 answers permission and file addresses for community products, and asks IMVU for their +// parents' (the client needs the whole derivation chain). +func (p *Products) authorizeProducts2(x *proxy.Exchange, call *xmlrpc.Call) error { + if len(call.Params) != 1 { + return nil + } + args, ok := call.Params[0].(xmlrpc.Struct) + if !ok { + return nil + } + authsValue, _ := args.Get("auths") + table, ok := authsValue.([]xmlrpc.Value) + if !ok || len(table) < 1 { + return nil + } + header, ok := table[0].([]xmlrpc.Value) + if !ok || len(header) != 2 || header[0] != "userId" || header[1] != "productId" { + return nil + } + type auth struct { + user, product int64 + } + var community []auth + forward := []xmlrpc.Value{table[0]} + asked := map[auth]bool{} + for _, row := range table[1:] { + r, ok := row.([]xmlrpc.Value) + if !ok || len(r) != 2 { + return nil + } + user, ok1 := r[0].(int64) + product, ok2 := r[1].(int64) + if !ok1 || !ok2 { + return nil + } + if !IsCommunity(product) { + if !asked[auth{user, product}] { + forward, asked[auth{user, product}] = append(forward, row), true + } + continue + } + community = append(community, auth{user, product}) + if s := p.lookup(product); s != nil && s.pack.Index.Parent > 0 { + a := auth{user, int64(s.pack.Index.Parent)} + if !asked[a] { + forward, asked[a] = append(forward, []xmlrpc.Value{user, a.product}), true + } + } + } + if len(community) == 0 { + return nil + } + ours := func(columns []xmlrpc.Value) []xmlrpc.Value { + var rows []xmlrpc.Value + for _, a := range community { + s := p.lookup(a.product) + if s == nil { + continue // unknown: no row, as IMVU does for products it can't authorize + } + values := map[string]xmlrpc.Value{ + "userId": a.user, "parentProductId": int64(s.pack.Index.Parent), "productId": s.local, + "cflChecksum": s.checksum, "cflUrl": cflURL(s.local, s.pack.Manifest.Revision), "sub_pids": "", + "usableDirect": true, "autoAuth": true, "usableDerivation": int64(1), "usableTrial": true, + "trialTimeout": int64(40), "usablePeerReview": true, "info": "", + } + row := make([]xmlrpc.Value, len(columns)) + for i, c := range columns { + name, _ := c.(string) + if v, ok := values[name]; ok { + row[i] = v + } else { + row[i] = "" + } + } + rows = append(rows, row) + } + return rows + } + if len(forward) == 1 { // only community products, none deriving from IMVU's + columns := make([]xmlrpc.Value, len(authColumns)) + for i, c := range authColumns { + columns[i] = c + } + out := append([]xmlrpc.Value{columns}, ours(columns)...) + x.Response = localResponse(200, "text/xml", (&xmlrpc.Response{Params: []xmlrpc.Value{out}}).Marshal()) + return nil + } + changed := make(xmlrpc.Struct, len(args)) + copy(changed, args) + for i := range changed { + if changed[i].Name == "auths" { + changed[i].Value = forward + } + } + if err := p.replaceBody(x, &xmlrpc.Call{Method: call.Method, Params: []xmlrpc.Value{changed}}); err != nil { + return err + } + x.ModifyResponse = func(resp *http.Response, body []byte) ([]byte, error) { + r, err := xmlrpc.ParseResponse(body) + if err != nil || r.Fault != nil || len(r.Params) != 1 { + return body, err + } + rows, ok := r.Params[0].([]xmlrpc.Value) + if !ok || len(rows) == 0 { + return body, errors.New("products: unexpected authorizeProducts2 answer") + } + columns, ok := rows[0].([]xmlrpc.Value) + if !ok { + return body, errors.New("products: unexpected authorizeProducts2 columns") + } + r.Params[0] = append(rows, ours(columns)...) + return r.Marshal(), nil + } + return nil +} + +// interceptBaseLook keeps community products out of the outfit IMVU stores, and adds them back when the client +// loads it. +func (p *Products) interceptBaseLook(x *proxy.Exchange) error { + user := x.Request.Header.Get("X-imvu-userid") + if user == "" { + return nil + } + if len(x.Body) == 0 { // loading the outfit + x.ModifyResponse = func(resp *http.Response, body []byte) ([]byte, error) { + var look map[string]json.RawMessage + if err := json.Unmarshal(body, &look); err != nil { + return body, err + } + var ids []int64 + if err := json.Unmarshal(look["result"], &ids); err != nil { + return body, err + } + worn := p.wornLocal(user, ids) + if len(worn) == 0 { + return body, nil + } + have := map[int64]bool{} + for _, id := range ids { + have[id] = true + } + for _, id := range worn { + if !have[id] { + ids = append(ids, id) + } + } + look["result"], _ = json.Marshal(ids) + return json.Marshal(look) + } + return nil + } + // Saving it: {"pids": [...]}. + var save map[string]json.RawMessage + if err := json.Unmarshal(x.Body, &save); err != nil { + return nil + } + var ids []int64 + if err := json.Unmarshal(save["pids"], &ids); err != nil { + return nil + } + var kept, community []int64 + for _, id := range ids { + if IsCommunity(id) { + community = append(community, id) + } else { + kept = append(kept, id) + } + } + p.recordWorn(user, kept, community) + if len(community) == 0 { + return nil + } + p.logf("products: kept %d community products out of the outfit saved to IMVU", len(community)) + save["pids"], _ = json.Marshal(kept) + body, _ := json.Marshal(save) + if !p.keys.resign(x.Request.Header, x.Body, x.URL.RawQuery, body, x.URL.RawQuery) { + return errNoKey + } + x.Body = body + return nil +} + +// Outgoing implements proxy.IMQFilter: community products are taken out of outfit chat commands (IMVU and other +// people see the defaults), and whole outfits are remembered. +func (p *Products) Outgoing(queue, mount string, message []byte) ([]byte, bool) { + if !strings.HasPrefix(queue, "/chat/") || mount != "messages" { + return message, false + } + m, ok := parseChatMessage(message) + if !ok { + return message, false + } + r := stripCommand(m.text, IsCommunity) + if r.outfit && m.userID != 0 { + p.recordWorn(strconv.FormatInt(m.userID, 10), r.kept, r.products) + } + if !r.changed { + return message, false + } + command, _, _ := strings.Cut(r.text, " ") + if r.empty { + p.logf("products: answered a %s of only community products without sending it", command) + return nil, true + } + p.logf("products: kept %d community products out of a %s sent to IMVU", len(r.products), command) + return m.withText(r.text), false +} diff --git a/internal/products/products.go b/internal/products/products.go new file mode 100644 index 0000000..4ff0769 --- /dev/null +++ b/internal/products/products.go @@ -0,0 +1,282 @@ +// Package products serves community products to the client from local packs (t5de-api's pack format), and keeps +// them away from IMVU. +// +// The client knows products by number, so each product gets a local ID from FirstLocalID up. For those IDs the +// proxy answers the client itself: product info (test.inventoryProductBatch), permission and file addresses +// (test.authorizeProducts2), the shop's product lookup, and the product's CFL. A product deriving from an IMVU +// product (its index.xml imports the parent's) gets the parent's answers from IMVU, asked for the parent. Outfits +// keep community products for the client but not for IMVU: they're taken out of outfit saves (base_look.php) and +// outfit chat commands (*use ...), remembered per account, and added back when the client loads the outfit. +package products + +import ( + "bytes" + "crypto/md5" + "encoding/hex" + "errors" + "fmt" + "io/fs" + "net/http" + "os" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + "sync" + "time" + + "git.t5de.dev/t5de/t5de-api/cfl" + "git.t5de.dev/t5de/t5de-api/pack" +) + +// Products is the proxy's community products: an Interceptor for HTTP and an IMQFilter for chat. +type Products struct { + dirs []string + state *state + keys keys + logf func(format string, args ...any) + + mu sync.Mutex + scanned time.Time + byLocal map[int64]*served + byID map[string]*served + problems map[string]string // pack directory -> last error logged +} + +// served is a pack ready to serve. +type served struct { + dir string + fingerprint string + pack *pack.Pack + local int64 + cfl []byte + checksum string // md5 of cfl, as IMVU's cflChecksum +} + +// New serves the packs in dirs (each pack a subdirectory), remembering its state in dataDir. +func New(dirs []string, dataDir string, logf func(format string, args ...any)) (*Products, error) { + st, err := loadState(filepath.Join(dataDir, "products.json")) + if err != nil { + return nil, err + } + if logf == nil { + logf = func(string, ...any) {} + } + p := &Products{dirs: dirs, state: st, logf: logf, byLocal: map[int64]*served{}, byID: map[string]*served{}, + problems: map[string]string{}} + p.refresh(true) + return p, nil +} + +// IsCommunity reports whether the client's product ID is a community product's local ID. +func IsCommunity(id int64) bool { return id >= FirstLocalID } + +// lookup is the pack served under a local ID, after picking up changed packs. +func (p *Products) lookup(local int64) *served { + p.refresh(false) + p.mu.Lock() + defer p.mu.Unlock() + return p.byLocal[local] +} + +func (p *Products) byProductID(id string) *served { + p.refresh(false) + p.mu.Lock() + defer p.mu.Unlock() + return p.byID[id] +} + +// refresh reloads packs that changed on disk, at most once a second unless forced. Creators editing a pack see the +// new files the next time the client loads the product. +func (p *Products) refresh(force bool) { + p.mu.Lock() + defer p.mu.Unlock() + if !force && time.Since(p.scanned) < time.Second { + return + } + p.scanned = time.Now() + seen := map[string]bool{} + for _, root := range p.dirs { + entries, err := os.ReadDir(root) + if err != nil { + continue + } + for _, e := range entries { + dir := filepath.Join(root, e.Name()) + if !e.IsDir() { + continue + } + if _, err := os.Stat(filepath.Join(dir, "t5de-product.json")); err != nil { + continue + } + fp := fingerprint(dir) + if old := p.findDirLocked(dir); old != nil && old.fingerprint == fp { + seen[old.pack.Manifest.ID] = true + continue + } + s, err := p.loadLocked(dir, fp) + if err != nil { + if p.problems[dir] != err.Error() { + p.logf("products: %s: %v", dir, err) + p.problems[dir] = err.Error() + } + if old := p.findDirLocked(dir); old != nil { + seen[old.pack.Manifest.ID] = true // keep serving the last good one + } + continue + } + delete(p.problems, dir) + if other, ok := p.byID[s.pack.Manifest.ID]; ok && other.dir != dir && seen[s.pack.Manifest.ID] { + p.logf("products: %s: product %q is also in %s; ignoring this one", dir, s.pack.Manifest.ID, other.dir) + continue + } + p.byID[s.pack.Manifest.ID] = s + p.byLocal[s.local] = s + seen[s.pack.Manifest.ID] = true + p.logf("products: serving %q revision %d (%s) as %d", s.pack.Manifest.ID, s.pack.Manifest.Revision, s.pack.Manifest.Name, s.local) + } + } + for id, s := range p.byID { + if !seen[id] { + delete(p.byID, id) + delete(p.byLocal, s.local) + p.logf("products: %q is gone", id) + } + } +} + +func (p *Products) findDirLocked(dir string) *served { + for _, s := range p.byID { + if s.dir == dir { + return s + } + } + return nil +} + +func (p *Products) loadLocked(dir, fp string) (*served, error) { + pk, err := pack.Load(dir) + if err != nil { + return nil, err + } + local, err := p.state.localID(pk.Manifest.ID) + if err != nil { + return nil, err + } + entries := make([]cfl.Entry, 0, len(pk.Files)) + for _, f := range pk.Files { + data := f.Data + if f.Name == "index.xml" { + data = withProductName(data, pk.Manifest.Name) + } + entries = append(entries, cfl.Entry{Name: f.Name, Data: data}) + } + var buf bytes.Buffer + if err := cfl.Write(&buf, entries); err != nil { + return nil, err + } + sum := md5.Sum(buf.Bytes()) + return &served{dir: dir, fingerprint: fp, pack: pk, local: local, cfl: buf.Bytes(), checksum: hex.EncodeToString(sum[:])}, nil +} + +// fingerprint changes when anything in the pack directory (or its files/ directory) changes. +func fingerprint(dir string) string { + var b strings.Builder + filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return nil + } + if info, err := d.Info(); err == nil { + fmt.Fprintf(&b, "%s|%d|%d\n", path, info.Size(), info.ModTime().UnixNano()) + } + return nil + }) + return b.String() +} + +var productNamePattern = regexp.MustCompile(`(?s).*?`) + +// withProductName puts the pack's name in index.xml, so the client shows it. +func withProductName(index []byte, name string) []byte { + var esc bytes.Buffer + xmlEscape(&esc, name) + repl := []byte("" + esc.String() + "") + if productNamePattern.Match(index) { + return productNamePattern.ReplaceAllLiteral(index, repl) + } + return index +} + +func xmlEscape(b *bytes.Buffer, s string) { + r := strings.NewReplacer("&", "&", "<", "<", ">", ">", `"`, """, "'", "'") + b.WriteString(r.Replace(s)) +} + +// productdata matches IMVU's product file addresses: /productdata//[/]. +var productdataPath = regexp.MustCompile(`^/productdata/([0-9]+)/([0-9]+)(?:/(.+))?$`) + +// cflURL is where the client fetches a community product's CFL: IMVU's address form, answered by the proxy. +func cflURL(local int64, revision int) string { + return fmt.Sprintf("http://userimages-akm.imvu.com/productdata/%d/%d", local, revision) +} + +// serveFile answers a request for a community product's files. +func (p *Products) serveFile(path string) (*http.Response, bool) { + m := productdataPath.FindStringSubmatch(path) + if m == nil { + return nil, false + } + local, _ := strconv.ParseInt(m[1], 10, 64) + if !IsCommunity(local) { + return nil, false + } + s := p.lookup(local) + rev, _ := strconv.Atoi(m[2]) + if s == nil || rev != s.pack.Manifest.Revision { + return notFound(), true + } + if m[3] == "" { + resp := localResponse(200, "application/octet-stream", s.cfl) + resp.Header.Set("Content-MD5", s.checksum) + return resp, true + } + for _, f := range s.pack.Files { + if f.Name == m[3] { + return localResponse(200, "application/octet-stream", f.Data), true + } + } + return notFound(), true +} + +func notFound() *http.Response { + return localResponse(404, "text/plain", []byte("t5de: no such community product file")) +} + +// wornLocal is the user's community products (local IDs, still served) that go with the IMVU outfit look. +func (p *Products) wornLocal(user string, look []int64) []int64 { + var ids []int64 + for _, id := range p.state.worn(user, look) { + if s := p.byProductID(id); s != nil { + ids = append(ids, s.local) + } + } + sort.Slice(ids, func(i, j int) bool { return ids[i] < ids[j] }) + return ids +} + +// recordWorn remembers the user's outfit: the IMVU products sent to IMVU, and the community products (local IDs) +// taken out. +func (p *Products) recordWorn(user string, imvu, locals []int64) { + var ids []string + for _, local := range locals { + if id, ok := p.state.productID(local); ok { + ids = append(ids, id) + } + } + if err := p.state.setWorn(user, imvu, ids); err != nil { + p.logf("products: saving what %s wears: %v", user, err) + } +} + +var errNoKey = errors.New("products: can't re-sign the request (no session key yet)") diff --git a/internal/products/products_test.go b/internal/products/products_test.go new file mode 100644 index 0000000..79b37ea --- /dev/null +++ b/internal/products/products_test.go @@ -0,0 +1,308 @@ +package products + +import ( + "bufio" + "bytes" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "git.t5de.dev/t5de/t5de-api/cfl" + "git.t5de.dev/t5de/t5de-proxy/internal/proxy" + "git.t5de.dev/t5de/t5de-proxy/internal/xmlrpc" +) + +const ( + user = "7" + key = "0123abcd" + parent = 500 // the IMVU product the test pack derives from + token = "tok" +) + +// writePack makes a pack deriving from IMVU product 500. +func writePack(t *testing.T, root, id string, revision int) string { + dir := filepath.Join(root, id) + os.MkdirAll(filepath.Join(dir, "files"), 0o755) + manifest := fmt.Sprintf(`{"format": 1, "id": %q, "revision": %d, "name": "Blue ", "creator": "Tester", "rating": "general"}`, id, revision) + os.WriteFile(filepath.Join(dir, "t5de-product.json"), []byte(manifest), 0o644) + index := fmt.Sprintf(``, parent) + os.WriteFile(filepath.Join(dir, "files", "index.xml"), []byte(index), 0o644) + os.WriteFile(filepath.Join(dir, "files", "tex.png"), []byte("not really a png"), 0o644) + return dir +} + +// fakeIMVU answers the calls the client makes, checking signatures, and records what it was asked. +type fakeIMVU struct { + t *testing.T + mu sync.Mutex + asked map[string][]string // method or path -> bodies + look []int64 +} + +func (f *fakeIMVU) record(name string, body []byte) { + f.mu.Lock() + defer f.mu.Unlock() + f.asked[name] = append(f.asked[name], string(body)) +} + +func (f *fakeIMVU) RoundTrip(r *http.Request) (*http.Response, error) { + var body []byte + if r.Body != nil { + body, _ = io.ReadAll(r.Body) + } + reply := func(s string) (*http.Response, error) { + return &http.Response{StatusCode: 200, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(s)), Request: r}, nil + } + checkSigned := func(withQuery bool) { + if r.Header.Get("X-imvu-auth") != sign(user, key, body, r.URL.RawQuery, withQuery) { + f.t.Errorf("%s: bad signature", r.URL) + } + } + switch r.URL.Path { + case "/api/base_look.php": + if len(body) > 0 { + checkSigned(true) + f.record("save look", body) + var save struct{ Pids []int64 } + json.Unmarshal(body, &save) + f.look = save.Pids + return reply(`{"success": true}`) + } + b, _ := json.Marshal(map[string]any{"result": f.look}) + return reply(string(b)) + case "/api/xmlrpc/client.php": + call, err := xmlrpc.ParseCall(body) + if err != nil { + f.t.Fatalf("bad call: %v", err) + } + f.record(call.Method, body) + switch call.Method { + case "test.avatarInfoForLogin2": + return reply(string((&xmlrpc.Response{Params: []xmlrpc.Value{xmlrpc.Struct{ + {Name: "userId", Value: int64(7)}, {Name: "securityKey", Value: key}, + {Name: "look", Value: []xmlrpc.Value{int64(80), int64(1)}}}}}).Marshal())) + case "test.inventoryProductBatch": + checkSigned(false) + var out []xmlrpc.Value + for _, v := range call.Params[0].([]xmlrpc.Value) { + out = append(out, xmlrpc.Struct{{Name: "products_id", Value: v}, {Name: "products_name", Value: "IMVU's"}, + {Name: "cPath", Value: "106_40"}, {Name: "manufacturers_name", Value: "IMVU Creator"}}) + } + return reply(string((&xmlrpc.Response{Params: []xmlrpc.Value{out}}).Marshal())) + case "test.authorizeProducts2": + checkSigned(false) + cols := []xmlrpc.Value{} + for _, c := range authColumns { + cols = append(cols, c) + } + rows := []xmlrpc.Value{cols} + auths, _ := call.Params[0].(xmlrpc.Struct).Get("auths") + for _, a := range auths.([]xmlrpc.Value)[1:] { + pid := a.([]xmlrpc.Value)[1].(int64) + rows = append(rows, []xmlrpc.Value{int64(7), int64(400), pid, "sum", fmt.Sprintf("http://x/productdata/%d/1", pid), "", false, false, int64(1), true, int64(40), true, ""}, + []xmlrpc.Value{int64(7), int64(0), int64(400), "sum2", "http://x/productdata/400/1", "", false, false, int64(1), true, int64(40), true, ""}) + } + return reply(string((&xmlrpc.Response{Params: []xmlrpc.Value{rows}}).Marshal())) + } + } + f.t.Errorf("unexpected request to IMVU: %s", r.URL) + return reply("") +} + +type harness struct { + t *testing.T + p *Products + imvu *fakeIMVU + proxy *httptest.Server +} + +func newHarness(t *testing.T) (*harness, string) { + packs := t.TempDir() + writePack(t, packs, "blue-coat", 1) + p, err := New([]string{packs}, t.TempDir(), t.Logf) + if err != nil { + t.Fatal(err) + } + f := &fakeIMVU{t: t, asked: map[string][]string{}, look: []int64{80, 1}} + srv := httptest.NewServer(&proxy.Server{Token: token, Transport: f, Interceptor: p, IMQFilter: p}) + t.Cleanup(srv.Close) + return &harness{t: t, p: p, imvu: f, proxy: srv}, packs +} + +// do sends a request through the proxy as the client does: signed (if signed), in absolute form. +func (h *harness) do(method, url string, body []byte, signed, withQuery bool) (int, []byte) { + h.t.Helper() + req, _ := http.NewRequest(method, url, bytes.NewReader(body)) + req.Header.Set("Proxy-Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte("t5de:"+token))) + if signed { + req.Header.Set("X-imvu-userid", user) + req.Header.Set("X-imvu-auth", sign(user, key, body, req.URL.RawQuery, withQuery)) + } + conn, err := net.Dial("tcp", h.proxy.Listener.Addr().String()) + if err != nil { + h.t.Fatal(err) + } + defer conn.Close() + req.WriteProxy(conn) + resp, err := http.ReadResponse(bufio.NewReader(conn), req) + if err != nil { + h.t.Fatal(err) + } + defer resp.Body.Close() + b, _ := io.ReadAll(resp.Body) + return resp.StatusCode, b +} + +func call(method string, params ...xmlrpc.Value) []byte { + return (&xmlrpc.Call{Method: method, Params: params}).Marshal() +} + +func TestCommunityProductEndToEnd(t *testing.T) { + h, _ := newHarness(t) + local := h.p.byProductID("blue-coat").local + if local != FirstLocalID { + t.Fatalf("local ID %d", local) + } + api := "http://client-dynamic.imvu.com/api/xmlrpc/client.php" + + // Logging in teaches the proxy the session key. + h.do("POST", "https://secure.imvu.com/api/xmlrpc/client.php", call("test.avatarInfoForLogin2"), false, false) + + // Product info: the community product's from its parent's, with its own name; IMVU is asked for the parent. + _, body := h.do("POST", api, call("test.inventoryProductBatch", []xmlrpc.Value{int64(1), local}), true, false) + r, err := xmlrpc.ParseResponse(body) + if err != nil { + t.Fatal(err, string(body)) + } + list := r.Params[0].([]xmlrpc.Value) + if len(list) != 2 { + t.Fatalf("%d entries: %s", len(list), body) + } + ours := list[1].(xmlrpc.Struct) + if id, _ := ours.Get("products_id"); id != local { + t.Fatalf("entry %v", ours) + } + if name, _ := ours.Get("products_name"); name != "Blue " { + t.Fatalf("name %v", name) + } + if path, _ := ours.Get("cPath"); path != "106_40" { + t.Fatalf("cPath %v", path) + } + if asked := h.imvu.asked["test.inventoryProductBatch"]; len(asked) != 1 || strings.Contains(asked[0], fmt.Sprint(local)) || !strings.Contains(asked[0], "500") { + t.Fatalf("IMVU was asked %v", asked) + } + + // Permission: our row (with the CFL's address and checksum) and IMVU's rows for the parent's chain. + auth := xmlrpc.Struct{{Name: "auths", Value: []xmlrpc.Value{[]xmlrpc.Value{"userId", "productId"}, []xmlrpc.Value{int64(7), local}}}, + {Name: "use_extracted_assets_if_available", Value: true}, {Name: "max_supported_format", Value: int64(9)}} + _, body = h.do("POST", api, call("test.authorizeProducts2", auth), true, false) + r, err = xmlrpc.ParseResponse(body) + if err != nil { + t.Fatal(err) + } + rows := r.Params[0].([]xmlrpc.Value) + var cflAddress string + for _, row := range rows[1:] { + values := row.([]xmlrpc.Value) + if values[2] == local { + cflAddress = values[4].(string) + if values[1] != int64(parent) || values[3] != h.p.byProductID("blue-coat").checksum || values[6] != true { + t.Fatalf("our row %v", values) + } + } + } + if cflAddress == "" || len(rows) != 4 { + t.Fatalf("rows %v", rows) + } + + // The CFL: the pack's files, with the pack's name in index.xml. IMVU is never asked. + status, data := h.do("GET", cflAddress, nil, false, false) + entries, err := cfl.Read(data) + if status != 200 || err != nil || len(entries) != 2 { + t.Fatalf("%d %v %v", status, err, entries) + } + if !bytes.Contains(entries[0].Data, []byte("Blue <Coat>")) { + t.Fatalf("index.xml %s", entries[0].Data) + } + if status, _ := h.do("GET", strings.Replace(cflAddress, "/1", "/2", 1), nil, false, false); status != 404 { + t.Fatalf("another revision: %d", status) + } + + // Saving an outfit with it: IMVU gets the outfit without it; loading the outfit adds it back. + h.do("POST", "http://client-dynamic.imvu.com/api/base_look.php", []byte(fmt.Sprintf(`{"pids": [80, 1, %d]}`, local)), true, true) + if saved := h.imvu.asked["save look"]; len(saved) != 1 || saved[0] != `{"pids":[80,1]}` { + t.Fatalf("IMVU saved %v", saved) + } + _, body = h.do("GET", "http://client-dynamic.imvu.com/api/base_look.php", nil, true, true) + if string(body) != fmt.Sprintf(`{"result":[80,1,%d]}`, local) { + t.Fatalf("look %s", body) + } + _, body = h.do("POST", "https://secure.imvu.com/api/xmlrpc/client.php", call("test.avatarInfoForLogin2"), false, false) + if !bytes.Contains(body, []byte(fmt.Sprintf("%d", local))) { + t.Fatalf("login look %s", body) + } + // If the outfit IMVU has changes (saved elsewhere), it comes back without community products. + h.imvu.look = []int64{80, 2} + _, body = h.do("GET", "http://client-dynamic.imvu.com/api/base_look.php", nil, true, true) + if string(body) != `{"result":[80,2]}` { + t.Fatalf("changed look %s", body) + } + + // The shop's bundle lookup is answered without IMVU. + if _, body := h.do("GET", fmt.Sprintf("http://client-dynamic.imvu.com/api/shop/product.php?pid=%d", local), nil, true, true); string(body) != `{"products":[]}` { + t.Fatalf("product.php %s", body) + } +} + +func TestOutgoingChat(t *testing.T) { + h, _ := newHarness(t) + local := h.p.byProductID("blue-coat").local + msg := func(text string) []byte { + return []byte(fmt.Sprintf(`{"to": 0, "message": %q, "userId": 7, "chatId": 9}`, text)) + } + out, drop := h.p.Outgoing("/chat/9", "messages", msg(fmt.Sprintf("*use 80 1 %d", local))) + if drop || !bytes.Contains(out, []byte(`"message":"*use 80 1"`)) { + t.Fatalf("%s %v", out, drop) + } + if got := h.p.wornLocal(user, []int64{1, 80}); len(got) != 1 || got[0] != local { + t.Fatalf("worn %v", got) + } + // With a different IMVU outfit (changed elsewhere), community products aren't added to it. + if got := h.p.wornLocal(user, []int64{80, 2}); len(got) != 0 { + t.Fatalf("worn with another outfit %v", got) + } + if _, drop := h.p.Outgoing("/chat/9", "messages", msg(fmt.Sprintf("*use %d", local))); !drop { + t.Fatal("a command with only community products was sent") + } + if out, _ := h.p.Outgoing("/chat/9", "messages", msg("hello")); !bytes.Equal(out, msg("hello")) { + t.Fatal("plain chat changed") + } + if out, _ := h.p.Outgoing("room_occupancy/1", "occupants", []byte("x")); string(out) != "x" { + t.Fatal("another queue changed") + } +} + +func TestPackChangesAreServed(t *testing.T) { + h, packs := newHarness(t) + before := h.p.byProductID("blue-coat") + os.WriteFile(filepath.Join(packs, "blue-coat", "files", "tex.png"), []byte("a new texture, longer"), 0o644) + h.p.refresh(true) + after := h.p.byProductID("blue-coat") + if after.checksum == before.checksum || after.local != before.local { + t.Fatalf("checksum %s -> %s, local %d -> %d", before.checksum, after.checksum, before.local, after.local) + } + os.RemoveAll(filepath.Join(packs, "blue-coat")) + h.p.refresh(true) + if h.p.byProductID("blue-coat") != nil { + t.Fatal("a removed pack is still served") + } +} diff --git a/internal/products/state.go b/internal/products/state.go index 45467c6..9901c47 100644 --- a/internal/products/state.go +++ b/internal/products/state.go @@ -24,12 +24,20 @@ type state struct { } type stateData struct { - IDs map[string]int64 `json:"ids"` // product ID -> local ID - Worn map[string][]string `json:"worn"` // IMVU user ID -> product IDs + IDs map[string]int64 `json:"ids"` // product ID -> local ID + Worn map[string]wornOutfit `json:"worn"` // IMVU user ID -> what they wear +} + +// wornOutfit is an account's outfit as last saved or sent: the IMVU products IMVU got, and the community products +// taken out. The community products are added back only to that same outfit: if IMVU's outfit changed elsewhere +// (another client, the website), they'd no longer fit it. +type wornOutfit struct { + IMVU []int64 `json:"imvu"` + Products []string `json:"products"` } func loadState(path string) (*state, error) { - s := &state{path: path, data: stateData{IDs: map[string]int64{}, Worn: map[string][]string{}}} + s := &state{path: path, data: stateData{IDs: map[string]int64{}, Worn: map[string]wornOutfit{}}} b, err := os.ReadFile(path) if errors.Is(err, fs.ErrNotExist) { return s, nil @@ -44,7 +52,7 @@ func loadState(path string) (*state, error) { s.data.IDs = map[string]int64{} } if s.data.Worn == nil { - s.data.Worn = map[string][]string{} + s.data.Worn = map[string]wornOutfit{} } return s, nil } @@ -67,30 +75,57 @@ func (s *state) localID(product string) (int64, error) { return next, s.saveLocked() } -// worn is the community products the user wears. -func (s *state) worn(user string) []string { +// worn is the community products the user wears with the IMVU outfit look, if they were recorded with it. +func (s *state) worn(user string, look []int64) []string { s.mu.Lock() defer s.mu.Unlock() - return append([]string(nil), s.data.Worn[user]...) -} - -// setWorn records the community products the user wears. -func (s *state) setWorn(user string, products []string) error { - products = append([]string(nil), products...) - sort.Strings(products) - s.mu.Lock() - defer s.mu.Unlock() - if equal(s.data.Worn[user], products) { + w, ok := s.data.Worn[user] + if !ok || !sameSet(w.IMVU, look) { return nil } - if len(products) == 0 { + return append([]string(nil), w.Products...) +} + +// setWorn records the user's outfit: the IMVU products IMVU got, and the community products taken out. +func (s *state) setWorn(user string, imvu []int64, products []string) error { + w := wornOutfit{IMVU: append([]int64(nil), imvu...), Products: append([]string(nil), products...)} + sort.Slice(w.IMVU, func(i, j int) bool { return w.IMVU[i] < w.IMVU[j] }) + sort.Strings(w.Products) + s.mu.Lock() + defer s.mu.Unlock() + old, had := s.data.Worn[user] + if had && sameSet(old.IMVU, w.IMVU) && equal(old.Products, w.Products) { + return nil + } + if len(w.Products) == 0 { + if !had { + return nil + } delete(s.data.Worn, user) } else { - s.data.Worn[user] = products + s.data.Worn[user] = w } return s.saveLocked() } +// sameSet reports whether a and b hold the same IDs, in any order. +func sameSet(a, b []int64) bool { + if len(a) != len(b) { + return false + } + count := map[int64]int{} + for _, v := range a { + count[v]++ + } + for _, v := range b { + if count[v] == 0 { + return false + } + count[v]-- + } + return true +} + func equal(a, b []string) bool { if len(a) != len(b) { return false @@ -117,3 +152,15 @@ func (s *state) saveLocked() error { } return os.Rename(tmp, s.path) } + +// productID is the product a local ID was given to. +func (s *state) productID(local int64) (string, bool) { + s.mu.Lock() + defer s.mu.Unlock() + for id, l := range s.data.IDs { + if l == local { + return id, true + } + } + return "", false +}