From 9bdc3ff455526899d7dd259523150b8b34cdf1a5 Mon Sep 17 00:00:00 2001 From: dhkatz Date: Thu, 29 Oct 2026 02:52:59 -0700 Subject: [PATCH] Add pack, cfl and safehtml pack loads local product packs (format 1): t5de-product.json and the product's files, as product.chkn or files/, with index.xml's rating, type, gender and parent product. cfl writes and reads IMVU's product container, uncompressed. safehtml cleans product descriptions by an allowlist (bluemonday). Co-Authored-By: Claude Opus 5.5 --- .gitattributes | 2 + .gitea/workflows/ci.yml | 26 ++++ .gitignore | 1 + CLAUDE.md | 21 +++ README.md | 16 ++ cfl/cfl.go | 228 +++++++++++++++++++++++++++ cfl/cfl_test.go | 153 ++++++++++++++++++ docs/pack.md | 86 +++++++++++ go.mod | 13 ++ go.sum | 8 + pack/index.go | 137 +++++++++++++++++ pack/manifest.go | 102 ++++++++++++ pack/pack.go | 275 +++++++++++++++++++++++++++++++++ pack/pack_test.go | 315 ++++++++++++++++++++++++++++++++++++++ safehtml/safehtml.go | 141 +++++++++++++++++ safehtml/safehtml_test.go | 114 ++++++++++++++ 16 files changed, 1638 insertions(+) create mode 100644 .gitattributes create mode 100644 .gitea/workflows/ci.yml create mode 100644 .gitignore create mode 100644 CLAUDE.md create mode 100644 README.md create mode 100644 cfl/cfl.go create mode 100644 cfl/cfl_test.go create mode 100644 docs/pack.md create mode 100644 go.mod create mode 100644 go.sum create mode 100644 pack/index.go create mode 100644 pack/manifest.go create mode 100644 pack/pack.go create mode 100644 pack/pack_test.go create mode 100644 safehtml/safehtml.go create mode 100644 safehtml/safehtml_test.go diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..3a99698 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# gofmt requires LF; keep sources LF on every platform. +* text=auto eol=lf diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml new file mode 100644 index 0000000..c17885b --- /dev/null +++ b/.gitea/workflows/ci.yml @@ -0,0 +1,26 @@ +# Tests the module: formatting, vet and tests. +name: CI + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Test + run: | + unformatted=$(gofmt -l .) + if [ -n "$unformatted" ]; then echo "Not gofmt-formatted: $unformatted"; exit 1; fi + go vet ./... + GOOS=windows go vet ./... + go test ./... diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..178135c --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/dist/ diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..d74ef3d --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,21 @@ +# t5de-api + +Go definitions shared by T5DE's proxy, hub and catalogs (see README.md, docs/pack.md). Part of T5DE; the proxy is +`t5de-proxy` (git.t5de.dev/t5de/t5de-proxy) and the design is in `t5de-hub`'s `docs/design.md`, both usually +checked out beside this one. + +## Rules + +- Never commit IMVU's code or files: no decompiled source, no product files from IMVU (not even as test data). + Test fixtures are synthetic, built in the tests. Our own declarations of IMVU's formats are fine. +- Test with a throwaway IMVU account, never a real one. +- Commits: author "dhkatz "; never the user's real name or email. +- Dependencies: the standard library, and bluemonday for `safehtml` (with its own dependencies). Ask before adding + others. +- Descriptions are shown in the client's Gecko pages, which have chrome privileges: `safehtml` is an allowlist, + never a blocklist. Every bypass found gets a test. + +## Notes + +- `gofmt`, `go vet ./...` and `go test ./...` must pass (CI checks them). `go test ./cfl -fuzz FuzzRead` fuzzes + the CFL reader. diff --git a/README.md b/README.md new file mode 100644 index 0000000..e14a6a5 --- /dev/null +++ b/README.md @@ -0,0 +1,16 @@ +# t5de-api + +Definitions shared by T5DE's programs that serve community products to IMVU Classic: the local proxy +([t5de-proxy](https://git.t5de.dev/t5de/t5de-proxy)), the hub and catalogs. Community products are original work, +not from IMVU's shop; the client loads them like IMVU's own, in IMVU's own file formats. The first phase is local +product packs: a folder on the user's machine that the proxy serves. + +Unofficial: not made by or affiliated with IMVU. + +## Packages + +- `pack`: local product packs, format 1 ([docs/pack.md](docs/pack.md)). `pack.Load` reads and checks one. +- `cfl`: CFL files, IMVU's product container, as the client downloads products. Writes uncompressed CFLs and reads + them (LZMA-compressed ones aren't supported yet). +- `safehtml`: product descriptions cleaned by an allowlist, safe to show anywhere, including the client's + privileged pages. diff --git a/cfl/cfl.go b/cfl/cfl.go new file mode 100644 index 0000000..9beded9 --- /dev/null +++ b/cfl/cfl.go @@ -0,0 +1,228 @@ +// Package cfl reads and writes CFL files, IMVU's product container: a product's files in one archive, as the +// client downloads and caches them. +// +// Layout (little-endian): "CFL3", u32 directory offset, u32 directory size (uncompressed). Each entry's data is a +// u32 stored length and the stored bytes. At the directory offset: u32 compression flag, u32 stored length, the +// directory. Each directory entry: i32 size, i32 offset (of the entry's length field), i32 compression flag, i16 +// name length, the name (Latin-1). "DFL3" files are the same, with each directory entry followed by an i32 length +// and that many bytes of content hash. +package cfl + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "io" + "math" + "unicode/utf8" +) + +// Compression flags. +const ( + None = 0 + LZMA = 4 // pylzma's format: 5 property bytes, then the raw stream +) + +// Entry is one file in a CFL. +type Entry struct { + Name string + Data []byte +} + +// ErrLZMA is returned for CFLs with LZMA-compressed entries or directories. +var ErrLZMA = errors.New("cfl: LZMA-compressed CFLs aren't supported") + +const headerSize = 12 + +// Write writes entries as a CFL, in order and uncompressed. The output depends only on the entries. +func Write(w io.Writer, entries []Entry) error { + var body, dir bytes.Buffer + offset := int64(headerSize) + for _, e := range entries { + name, err := Latin1(e.Name) + if err != nil { + return err + } + if len(name) == 0 { + return errors.New("cfl: empty entry name") + } + if len(name) > math.MaxInt16 { + return fmt.Errorf("cfl: entry name %q is too long", e.Name) + } + if offset+4+int64(len(e.Data)) > math.MaxInt32 { + return errors.New("cfl: entries over 2 GiB") + } + le32(&dir, uint32(len(e.Data))) + le32(&dir, uint32(offset)) + le32(&dir, None) + binary.Write(&dir, binary.LittleEndian, int16(len(name))) + dir.Write(name) + + le32(&body, uint32(len(e.Data))) + body.Write(e.Data) + offset += 4 + int64(len(e.Data)) + } + if offset+8+int64(dir.Len()) > math.MaxInt32 { + return errors.New("cfl: entries over 2 GiB") + } + + var head bytes.Buffer + head.WriteString("CFL3") + le32(&head, uint32(offset)) + le32(&head, uint32(dir.Len())) + le32(&body, None) + le32(&body, uint32(dir.Len())) + for _, b := range [][]byte{head.Bytes(), body.Bytes(), dir.Bytes()} { + if _, err := w.Write(b); err != nil { + return err + } + } + return nil +} + +func le32(b *bytes.Buffer, v uint32) { + b.Write(binary.LittleEndian.AppendUint32(nil, v)) +} + +// Read reads a CFL (or DFL) whose entries and directory are uncompressed. Entries' Data share memory with data. +func Read(data []byte) ([]Entry, error) { + if len(data) < headerSize { + return nil, errors.New("cfl: too short") + } + var hashes bool + switch string(data[:4]) { + case "CFL3": + case "DFL3": + hashes = true + default: + return nil, fmt.Errorf("cfl: unknown header %q", data[:4]) + } + dirOffset := int64(binary.LittleEndian.Uint32(data[4:])) + dirSize := int64(binary.LittleEndian.Uint32(data[8:])) + + r := reader{data: data, pos: dirOffset} + flag := r.u32() + stored := int64(r.u32()) + if r.err != nil { + return nil, fmt.Errorf("cfl: directory offset %d is past the end", dirOffset) + } + switch flag { + case None: + case LZMA: + return nil, ErrLZMA + default: + return nil, fmt.Errorf("cfl: directory has unknown compression %d", flag) + } + if stored != dirSize { + return nil, fmt.Errorf("cfl: directory is %d bytes, header says %d", stored, dirSize) + } + dir := r.bytes(stored) + if r.err != nil { + return nil, errors.New("cfl: directory runs past the end") + } + + var entries []Entry + d := reader{data: dir} + for d.pos < int64(len(dir)) { + size := int64(int32(d.u32())) + offset := int64(int32(d.u32())) + flag := d.u32() + nameLen := int64(int16(d.u16())) + if nameLen < 0 { + return nil, fmt.Errorf("cfl: directory entry %d has a negative name length", len(entries)) + } + name := d.bytes(nameLen) + if hashes { + hashLen := int64(int32(d.u32())) + if hashLen < 0 { + return nil, fmt.Errorf("cfl: directory entry %d has a negative hash length", len(entries)) + } + d.bytes(hashLen) + } + if d.err != nil { + return nil, fmt.Errorf("cfl: directory entry %d is cut off", len(entries)) + } + if len(name) == 0 { + return nil, fmt.Errorf("cfl: directory entry %d has no name", len(entries)) + } + n := latin1String(name) + switch flag { + case None: + case LZMA: + return nil, ErrLZMA + default: + return nil, fmt.Errorf("cfl: %q has unknown compression %d", n, flag) + } + if size < 0 || offset < 0 { + return nil, fmt.Errorf("cfl: %q has a negative size or offset", n) + } + e := reader{data: data, pos: offset} + stored := int64(e.u32()) + body := e.bytes(stored) + if e.err != nil { + return nil, fmt.Errorf("cfl: %q runs past the end", n) + } + if stored != size { + return nil, fmt.Errorf("cfl: %q is %d bytes, directory says %d", n, stored, size) + } + entries = append(entries, Entry{Name: n, Data: body}) + } + return entries, nil +} + +// reader reads little-endian values from data, setting err instead of reading past the end. +type reader struct { + data []byte + pos int64 + err error +} + +func (r *reader) bytes(n int64) []byte { + if r.err != nil || n < 0 || r.pos < 0 || r.pos > int64(len(r.data)) || n > int64(len(r.data))-r.pos { + r.err = io.ErrUnexpectedEOF + return nil + } + b := r.data[r.pos : r.pos+n : r.pos+n] + r.pos += n + return b +} + +func (r *reader) u32() uint32 { + b := r.bytes(4) + if b == nil { + return 0 + } + return binary.LittleEndian.Uint32(b) +} + +func (r *reader) u16() uint16 { + b := r.bytes(2) + if b == nil { + return 0 + } + return binary.LittleEndian.Uint16(b) +} + +// Latin1 encodes s as Latin-1, as CFLs store names. +func Latin1(s string) ([]byte, error) { + if !utf8.ValidString(s) { + return nil, fmt.Errorf("cfl: name %q isn't valid UTF-8", s) + } + b := make([]byte, 0, len(s)) + for _, r := range s { + if r > 0xff { + return nil, fmt.Errorf("cfl: name %q has characters outside Latin-1", s) + } + b = append(b, byte(r)) + } + return b, nil +} + +func latin1String(b []byte) string { + r := make([]rune, len(b)) + for i, c := range b { + r[i] = rune(c) + } + return string(r) +} diff --git a/cfl/cfl_test.go b/cfl/cfl_test.go new file mode 100644 index 0000000..fa77710 --- /dev/null +++ b/cfl/cfl_test.go @@ -0,0 +1,153 @@ +package cfl + +import ( + "bytes" + "encoding/binary" + "encoding/hex" + "errors" + "reflect" + "testing" +) + +func write(t testing.TB, entries []Entry) []byte { + t.Helper() + var b bytes.Buffer + if err := Write(&b, entries); err != nil { + t.Fatal(err) + } + return b.Bytes() +} + +func TestWriteLayout(t *testing.T) { + got := write(t, []Entry{{Name: "a", Data: []byte("xy")}, {Name: "é", Data: nil}}) + want := "" + + "43464c33" + "16000000" + "1e000000" + // CFL3, directory at 22, 30 bytes + "02000000" + "7879" + // "a" + "00000000" + // "é" + "00000000" + "1e000000" + // directory: uncompressed, 30 bytes + "02000000" + "0c000000" + "00000000" + "0100" + "61" + + "00000000" + "12000000" + "00000000" + "0100" + "e9" + if hex.EncodeToString(got) != want { + t.Errorf("got %x\nwant %s", got, want) + } +} + +func TestRoundTrip(t *testing.T) { + entries := []Entry{ + {Name: "index.xml", Data: []byte("