Add pack, cfl and safehtml

pack loads local product packs (format 1): t5de-product.json and the product's files, as product.chkn or
files/, with index.xml's rating, type, gender and parent product. cfl writes and reads IMVU's product
container, uncompressed. safehtml cleans product descriptions by an allowlist (bluemonday).

Co-Authored-By: Claude Opus 5.5 <[email protected]>

.gitattributes

@@ -0,0 +1,2 @@
+# gofmt requires LF; keep sources LF on every platform.
+* text=auto eol=lf

.gitea/workflows/ci.yml

@@ -0,0 +1,26 @@
+# Tests the module: formatting, vet and tests.
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+
+      - name: Test
+        run: |
+          unformatted=$(gofmt -l .)
+          if [ -n "$unformatted" ]; then echo "Not gofmt-formatted: $unformatted"; exit 1; fi
+          go vet ./...
+          GOOS=windows go vet ./...
+          go test ./...

.gitignore

@@ -0,0 +1 @@
+/dist/

CLAUDE.md

@@ -0,0 +1,21 @@
+# t5de-api
+
+Go definitions shared by T5DE's proxy, hub and catalogs (see README.md, docs/pack.md). Part of T5DE; the proxy is
+`t5de-proxy` (git.t5de.dev/t5de/t5de-proxy) and the design is in `t5de-hub`'s `docs/design.md`, both usually
+checked out beside this one.
+
+## Rules
+
+- Never commit IMVU's code or files: no decompiled source, no product files from IMVU (not even as test data).
+  Test fixtures are synthetic, built in the tests. Our own declarations of IMVU's formats are fine.
+- Test with a throwaway IMVU account, never a real one.
+- Commits: author "dhkatz <dev@t5de.dev>"; never the user's real name or email.
+- Dependencies: the standard library, and bluemonday for `safehtml` (with its own dependencies). Ask before adding
+  others.
+- Descriptions are shown in the client's Gecko pages, which have chrome privileges: `safehtml` is an allowlist,
+  never a blocklist. Every bypass found gets a test.
+
+## Notes
+
+- `gofmt`, `go vet ./...` and `go test ./...` must pass (CI checks them). `go test ./cfl -fuzz FuzzRead` fuzzes
+  the CFL reader.

README.md

@@ -0,0 +1,16 @@
+# t5de-api
+
+Definitions shared by T5DE's programs that serve community products to IMVU Classic: the local proxy
+([t5de-proxy](https://git.t5de.dev/t5de/t5de-proxy)), the hub and catalogs. Community products are original work,
+not from IMVU's shop; the client loads them like IMVU's own, in IMVU's own file formats. The first phase is local
+product packs: a folder on the user's machine that the proxy serves.
+
+Unofficial: not made by or affiliated with IMVU.
+
+## Packages
+
+- `pack`: local product packs, format 1 ([docs/pack.md](docs/pack.md)). `pack.Load` reads and checks one.
+- `cfl`: CFL files, IMVU's product container, as the client downloads products. Writes uncompressed CFLs and reads
+  them (LZMA-compressed ones aren't supported yet).
+- `safehtml`: product descriptions cleaned by an allowlist, safe to show anywhere, including the client's
+  privileged pages.

cfl/cfl.go

@@ -0,0 +1,228 @@
+// Package cfl reads and writes CFL files, IMVU's product container: a product's files in one archive, as the
+// client downloads and caches them.
+//
+// Layout (little-endian): "CFL3", u32 directory offset, u32 directory size (uncompressed). Each entry's data is a
+// u32 stored length and the stored bytes. At the directory offset: u32 compression flag, u32 stored length, the
+// directory. Each directory entry: i32 size, i32 offset (of the entry's length field), i32 compression flag, i16
+// name length, the name (Latin-1). "DFL3" files are the same, with each directory entry followed by an i32 length
+// and that many bytes of content hash.
+package cfl
+
+import (
+	"bytes"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"io"
+	"math"
+	"unicode/utf8"
+)
+
+// Compression flags.
+const (
+	None = 0
+	LZMA = 4 // pylzma's format: 5 property bytes, then the raw stream
+)
+
+// Entry is one file in a CFL.
+type Entry struct {
+	Name string
+	Data []byte
+}
+
+// ErrLZMA is returned for CFLs with LZMA-compressed entries or directories.
+var ErrLZMA = errors.New("cfl: LZMA-compressed CFLs aren't supported")
+
+const headerSize = 12
+
+// Write writes entries as a CFL, in order and uncompressed. The output depends only on the entries.
+func Write(w io.Writer, entries []Entry) error {
+	var body, dir bytes.Buffer
+	offset := int64(headerSize)
+	for _, e := range entries {
+		name, err := Latin1(e.Name)
+		if err != nil {
+			return err
+		}
+		if len(name) == 0 {
+			return errors.New("cfl: empty entry name")
+		}
+		if len(name) > math.MaxInt16 {
+			return fmt.Errorf("cfl: entry name %q is too long", e.Name)
+		}
+		if offset+4+int64(len(e.Data)) > math.MaxInt32 {
+			return errors.New("cfl: entries over 2 GiB")
+		}
+		le32(&dir, uint32(len(e.Data)))
+		le32(&dir, uint32(offset))
+		le32(&dir, None)
+		binary.Write(&dir, binary.LittleEndian, int16(len(name)))
+		dir.Write(name)
+
+		le32(&body, uint32(len(e.Data)))
+		body.Write(e.Data)
+		offset += 4 + int64(len(e.Data))
+	}
+	if offset+8+int64(dir.Len()) > math.MaxInt32 {
+		return errors.New("cfl: entries over 2 GiB")
+	}
+
+	var head bytes.Buffer
+	head.WriteString("CFL3")
+	le32(&head, uint32(offset))
+	le32(&head, uint32(dir.Len()))
+	le32(&body, None)
+	le32(&body, uint32(dir.Len()))
+	for _, b := range [][]byte{head.Bytes(), body.Bytes(), dir.Bytes()} {
+		if _, err := w.Write(b); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func le32(b *bytes.Buffer, v uint32) {
+	b.Write(binary.LittleEndian.AppendUint32(nil, v))
+}
+
+// Read reads a CFL (or DFL) whose entries and directory are uncompressed. Entries' Data share memory with data.
+func Read(data []byte) ([]Entry, error) {
+	if len(data) < headerSize {
+		return nil, errors.New("cfl: too short")
+	}
+	var hashes bool
+	switch string(data[:4]) {
+	case "CFL3":
+	case "DFL3":
+		hashes = true
+	default:
+		return nil, fmt.Errorf("cfl: unknown header %q", data[:4])
+	}
+	dirOffset := int64(binary.LittleEndian.Uint32(data[4:]))
+	dirSize := int64(binary.LittleEndian.Uint32(data[8:]))
+
+	r := reader{data: data, pos: dirOffset}
+	flag := r.u32()
+	stored := int64(r.u32())
+	if r.err != nil {
+		return nil, fmt.Errorf("cfl: directory offset %d is past the end", dirOffset)
+	}
+	switch flag {
+	case None:
+	case LZMA:
+		return nil, ErrLZMA
+	default:
+		return nil, fmt.Errorf("cfl: directory has unknown compression %d", flag)
+	}
+	if stored != dirSize {
+		return nil, fmt.Errorf("cfl: directory is %d bytes, header says %d", stored, dirSize)
+	}
+	dir := r.bytes(stored)
+	if r.err != nil {
+		return nil, errors.New("cfl: directory runs past the end")
+	}
+
+	var entries []Entry
+	d := reader{data: dir}
+	for d.pos < int64(len(dir)) {
+		size := int64(int32(d.u32()))
+		offset := int64(int32(d.u32()))
+		flag := d.u32()
+		nameLen := int64(int16(d.u16()))
+		if nameLen < 0 {
+			return nil, fmt.Errorf("cfl: directory entry %d has a negative name length", len(entries))
+		}
+		name := d.bytes(nameLen)
+		if hashes {
+			hashLen := int64(int32(d.u32()))
+			if hashLen < 0 {
+				return nil, fmt.Errorf("cfl: directory entry %d has a negative hash length", len(entries))
+			}
+			d.bytes(hashLen)
+		}
+		if d.err != nil {
+			return nil, fmt.Errorf("cfl: directory entry %d is cut off", len(entries))
+		}
+		if len(name) == 0 {
+			return nil, fmt.Errorf("cfl: directory entry %d has no name", len(entries))
+		}
+		n := latin1String(name)
+		switch flag {
+		case None:
+		case LZMA:
+			return nil, ErrLZMA
+		default:
+			return nil, fmt.Errorf("cfl: %q has unknown compression %d", n, flag)
+		}
+		if size < 0 || offset < 0 {
+			return nil, fmt.Errorf("cfl: %q has a negative size or offset", n)
+		}
+		e := reader{data: data, pos: offset}
+		stored := int64(e.u32())
+		body := e.bytes(stored)
+		if e.err != nil {
+			return nil, fmt.Errorf("cfl: %q runs past the end", n)
+		}
+		if stored != size {
+			return nil, fmt.Errorf("cfl: %q is %d bytes, directory says %d", n, stored, size)
+		}
+		entries = append(entries, Entry{Name: n, Data: body})
+	}
+	return entries, nil
+}
+
+// reader reads little-endian values from data, setting err instead of reading past the end.
+type reader struct {
+	data []byte
+	pos  int64
+	err  error
+}
+
+func (r *reader) bytes(n int64) []byte {
+	if r.err != nil || n < 0 || r.pos < 0 || r.pos > int64(len(r.data)) || n > int64(len(r.data))-r.pos {
+		r.err = io.ErrUnexpectedEOF
+		return nil
+	}
+	b := r.data[r.pos : r.pos+n : r.pos+n]
+	r.pos += n
+	return b
+}
+
+func (r *reader) u32() uint32 {
+	b := r.bytes(4)
+	if b == nil {
+		return 0
+	}
+	return binary.LittleEndian.Uint32(b)
+}
+
+func (r *reader) u16() uint16 {
+	b := r.bytes(2)
+	if b == nil {
+		return 0
+	}
+	return binary.LittleEndian.Uint16(b)
+}
+
+// Latin1 encodes s as Latin-1, as CFLs store names.
+func Latin1(s string) ([]byte, error) {
+	if !utf8.ValidString(s) {
+		return nil, fmt.Errorf("cfl: name %q isn't valid UTF-8", s)
+	}
+	b := make([]byte, 0, len(s))
+	for _, r := range s {
+		if r > 0xff {
+			return nil, fmt.Errorf("cfl: name %q has characters outside Latin-1", s)
+		}
+		b = append(b, byte(r))
+	}
+	return b, nil
+}
+
+func latin1String(b []byte) string {
+	r := make([]rune, len(b))
+	for i, c := range b {
+		r[i] = rune(c)
+	}
+	return string(r)
+}

cfl/cfl_test.go

@@ -0,0 +1,153 @@
+package cfl
+
+import (
+	"bytes"
+	"encoding/binary"
+	"encoding/hex"
+	"errors"
+	"reflect"
+	"testing"
+)
+
+func write(t testing.TB, entries []Entry) []byte {
+	t.Helper()
+	var b bytes.Buffer
+	if err := Write(&b, entries); err != nil {
+		t.Fatal(err)
+	}
+	return b.Bytes()
+}
+
+func TestWriteLayout(t *testing.T) {
+	got := write(t, []Entry{{Name: "a", Data: []byte("xy")}, {Name: "é", Data: nil}})
+	want := "" +
+		"43464c33" + "16000000" + "1e000000" + // CFL3, directory at 22, 30 bytes
+		"02000000" + "7879" + // "a"
+		"00000000" + // "é"
+		"00000000" + "1e000000" + // directory: uncompressed, 30 bytes
+		"02000000" + "0c000000" + "00000000" + "0100" + "61" +
+		"00000000" + "12000000" + "00000000" + "0100" + "e9"
+	if hex.EncodeToString(got) != want {
+		t.Errorf("got  %x\nwant %s", got, want)
+	}
+}
+
+func TestRoundTrip(t *testing.T) {
+	entries := []Entry{
+		{Name: "index.xml", Data: []byte("<Template/>")},
+		{Name: "café.png", Data: []byte{0x89, 'P', 'N', 'G'}},
+		{Name: "empty", Data: []byte{}},
+	}
+	got, err := Read(write(t, entries))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(got, entries) {
+		t.Errorf("got %q, want %q", got, entries)
+	}
+
+	if got, err := Read(write(t, nil)); err != nil || len(got) != 0 {
+		t.Errorf("empty CFL: %v, %v", got, err)
+	}
+}
+
+func TestWriteErrors(t *testing.T) {
+	for _, name := range []string{"", "snow☃.png", "\xff"} {
+		var b bytes.Buffer
+		if err := Write(&b, []Entry{{Name: name}}); err == nil {
+			t.Errorf("%q: no error", name)
+		}
+	}
+}
+
+// TestDFL builds a DFL3 by hand: one entry with a content hash after its directory entry.
+func TestDFL(t *testing.T) {
+	var b []byte
+	le := binary.LittleEndian
+	b = append(b, "DFL3"...)
+	b = le.AppendUint32(b, 18)
+	dirSizeAt := len(b)
+	b = le.AppendUint32(b, 0)
+	b = le.AppendUint32(b, 2)
+	b = append(b, "hi"...)
+	var dir []byte
+	dir = le.AppendUint32(dir, 2)
+	dir = le.AppendUint32(dir, 12)
+	dir = le.AppendUint32(dir, 0)
+	dir = le.AppendUint16(dir, 1)
+	dir = append(dir, 'x')
+	dir = le.AppendUint32(dir, 3)
+	dir = append(dir, "abc"...)
+	le.PutUint32(b[dirSizeAt:], uint32(len(dir)))
+	b = le.AppendUint32(b, 0)
+	b = le.AppendUint32(b, uint32(len(dir)))
+	b = append(b, dir...)
+
+	got, err := Read(b)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if want := []Entry{{Name: "x", Data: []byte("hi")}}; !reflect.DeepEqual(got, want) {
+		t.Errorf("got %q, want %q", got, want)
+	}
+}
+
+func TestReadLZMA(t *testing.T) {
+	b := write(t, []Entry{{Name: "a", Data: []byte("xy")}})
+	dir := binary.LittleEndian.Uint32(b[4:])
+
+	c := bytes.Clone(b)
+	binary.LittleEndian.PutUint32(c[dir:], LZMA)
+	if _, err := Read(c); !errors.Is(err, ErrLZMA) {
+		t.Errorf("compressed directory: %v", err)
+	}
+
+	c = bytes.Clone(b)
+	binary.LittleEndian.PutUint32(c[dir+8+8:], LZMA) // the entry's flag
+	if _, err := Read(c); !errors.Is(err, ErrLZMA) {
+		t.Errorf("compressed entry: %v", err)
+	}
+}
+
+func TestReadMalformed(t *testing.T) {
+	good := write(t, []Entry{{Name: "a", Data: []byte("xy")}})
+	for i := range len(good) {
+		if _, err := Read(good[:i]); err == nil {
+			t.Errorf("cut at %d: no error", i)
+		}
+	}
+	bad := bytes.Clone(good)
+	bad[0] = 'X'
+	if _, err := Read(bad); err == nil {
+		t.Error("bad header: no error")
+	}
+	bad = bytes.Clone(good)
+	binary.LittleEndian.PutUint32(bad[12:], 1000) // the entry's stored length
+	if _, err := Read(bad); err == nil {
+		t.Error("bad entry length: no error")
+	}
+}
+
+func FuzzRead(f *testing.F) {
+	f.Add(write(f, []Entry{{Name: "a", Data: []byte("xy")}, {Name: "b.png", Data: []byte{1, 2, 3}}}))
+	f.Add(write(f, nil))
+	f.Add([]byte("DFL3\x0c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"))
+	f.Fuzz(func(t *testing.T, data []byte) {
+		entries, err := Read(data)
+		if err != nil {
+			return
+		}
+		again, err := Read(write(t, entries))
+		if err != nil {
+			t.Fatalf("rewritten CFL: %v", err)
+		}
+		if len(again) != len(entries) {
+			t.Fatalf("rewritten CFL has %d entries, want %d", len(again), len(entries))
+		}
+		for i := range entries {
+			if again[i].Name != entries[i].Name || !bytes.Equal(again[i].Data, entries[i].Data) {
+				t.Fatalf("entry %d changed", i)
+			}
+		}
+	})
+}

docs/pack.md

@@ -0,0 +1,86 @@
+# Product packs (format 1)
+
+A product pack is a folder on the user's machine holding one community product: its manifest,
+`t5de-product.json`, and the product's files. T5DE's proxy serves packs to the client as if they were IMVU
+products. Package `pack` reads and checks them (`pack.Load`).
+
+```
+blue-fur-coat/
+  t5de-product.json
+  product.chkn        the files, zipped (or files/, not both)
+```
+
+## Manifest
+
+`t5de-product.json`, UTF-8 JSON:
+
+```json
+{
+  "format": 1,
+  "id": "blue-fur-coat",
+  "revision": 1,
+  "name": "Blue Fur Coat",
+  "creator": "Some Creator",
+  "description": "<p>HTML description</p>",
+  "rating": "general"
+}
+```
+
+| Field | Rules |
+| --- | --- |
+| `format` | 1 |
+| `id` | 1-64 of `a-z`, `0-9` and `-`, not starting or ending with `-`. Names the product locally; the hub assigns global IDs later. |
+| `revision` | an integer, at least 1. Raise it when the files change, so the client doesn't keep the old ones cached. |
+| `name` | 1-64 characters after trimming spaces, no control characters |
+| `creator` | optional; up to 64 characters, no control characters |
+| `description` | optional; HTML, up to 20000 bytes. Cleaned by an allowlist (package `safehtml`) wherever it's shown. |
+| `rating` | `general` or `adult`; must agree with `index.xml` (below) |
+
+Other fields are an error, and so is anything after the object.
+
+## Files
+
+The product's files are IMVU's product files, as Create Mode makes them: `index.xml` (required), meshes
+(`.xmf`), materials (`.xrf`), textures (`.png`, `.jpg`), and so on. They come one of two ways:
+
+- `product.chkn`: a zip file, Create Mode's project format, with the files at its root. Folder entries are
+  ignored; files inside folders are an error.
+- `files/`: a folder with the files at its top level. Subfolders are an error.
+
+Having both is an error. Other files in the pack's folder are ignored.
+
+Rules for each file:
+
+- Names: not empty, no `/` or `\`, not `.` or `..`, no control characters, Latin-1 characters only (CFL files
+  store names as Latin-1), at most 200 bytes (UTF-8). No two files with the same name.
+- `_contents.json` isn't allowed: it makes the client download a product's files separately, which packs don't
+  support.
+- Limits: 512 files, 64 MiB per file, 256 MiB in all (uncompressed; sizes are counted while reading, not taken
+  from the zip's headers).
+
+`pack.Checksum` hashes the files' names and contents (sha256), so the same files give the same checksum however
+they're stored or compressed. The proxy uses it to notice changes.
+
+## index.xml
+
+IMVU's product template (root element `<Template>`). T5DE reads only these top-level elements and ignores the
+rest:
+
+| Element | Use |
+| --- | --- |
+| `ProductRating` | required: `GA` (general) or `AP` (adult), matching the manifest's `rating` |
+| `ProductName` | the name Create Mode gave it |
+| `ProductGender` | `female`, `male`, ... |
+| `ProductTypeID` | optional integer |
+| `BodyPatternProductId` | optional integer |
+| `__DATAIMPORT` | optional: `product://<id>/index.xml`, the IMVU product it derives from |
+
+A product derives from an IMVU parent product, as every IMVU product does: the client loads the parent's files
+first (from IMVU, as usual) and the pack's files on top. So a pack holds only what it changes, and the parent's
+files never pass through T5DE.
+
+## How the proxy serves a pack
+
+The proxy builds the files into a CFL, IMVU's product container (package `cfl`), uncompressed, files in name
+order, and serves it where the client asks for the product's files. The revision is part of the address the
+client caches the product under, so a new revision makes clients load the new files.

go.mod

@@ -0,0 +1,13 @@
+module git.t5de.dev/t5de/t5de-api
+
+go 1.27.0
+
+require (
+	github.com/microcosm-cc/bluemonday v1.0.27
+	golang.org/x/net v0.26.0
+)
+
+require (
+	github.com/aymerick/douceur v0.2.0 // indirect
+	github.com/gorilla/css v1.0.1 // indirect
+)

go.sum

@@ -0,0 +1,8 @@
+github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
+github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
+github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8=
+github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0=
+github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk=
+github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA=
+golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
+golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=

pack/index.go

@@ -0,0 +1,137 @@
+package pack
+
+import (
+	"bytes"
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"io"
+	"strconv"
+	"strings"
+)
+
+// Index is what T5DE reads from a product's index.xml, IMVU's product template. Other elements are ignored.
+type Index struct {
+	Rating      string // ProductRating: "GA" (general) or "AP" (adult)
+	Name        string // ProductName
+	Gender      string // ProductGender
+	TypeID      int64  // ProductTypeID, 0 if absent
+	BodyPattern int64  // BodyPatternProductId, 0 if absent
+	Parent      int64  // the product it derives from (__DATAIMPORT product://<id>/index.xml), 0 if none
+}
+
+// ParseIndex parses index.xml.
+func ParseIndex(data []byte) (*Index, error) {
+	d := xml.NewDecoder(bytes.NewReader(data))
+	d.Strict = false
+	d.Entity = xml.HTMLEntity
+	d.CharsetReader = charsetReader
+
+	values := map[string]string{}
+	depth := 0
+	root := false
+	var field string // the top-level element being read, if wanted
+	var text strings.Builder
+	for {
+		tok, err := d.Token()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
+		switch t := tok.(type) {
+		case xml.StartElement:
+			depth++
+			if depth == 1 && t.Name.Local != "Template" {
+				return nil, fmt.Errorf("root element is <%s>, not <Template>", t.Name.Local)
+			}
+			root = true
+			if depth == 2 && wanted[t.Name.Local] {
+				if _, ok := values[t.Name.Local]; ok {
+					return nil, fmt.Errorf("<%s> appears twice", t.Name.Local)
+				}
+				field = t.Name.Local
+				text.Reset()
+			}
+		case xml.EndElement:
+			if depth == 2 && field != "" {
+				values[field] = strings.TrimSpace(text.String())
+				field = ""
+			}
+			depth--
+		case xml.CharData:
+			if field != "" {
+				text.Write(t)
+			}
+		}
+	}
+	if !root {
+		return nil, errors.New("no <Template>")
+	}
+
+	var x Index
+	x.Rating = values["ProductRating"]
+	x.Name = values["ProductName"]
+	x.Gender = values["ProductGender"]
+	if x.Rating != "GA" && x.Rating != "AP" {
+		return nil, fmt.Errorf("ProductRating: %q must be GA or AP", x.Rating)
+	}
+	var err error
+	if x.TypeID, err = id(values, "ProductTypeID"); err != nil {
+		return nil, err
+	}
+	if x.BodyPattern, err = id(values, "BodyPatternProductId"); err != nil {
+		return nil, err
+	}
+	if imp := values["__DATAIMPORT"]; imp != "" {
+		s, ok := strings.CutPrefix(imp, "product://")
+		if ok {
+			s, ok = strings.CutSuffix(s, "/index.xml")
+		}
+		n, err := strconv.ParseInt(s, 10, 64)
+		if !ok || err != nil || n < 1 {
+			return nil, fmt.Errorf("__DATAIMPORT: %q isn't product://<id>/index.xml", imp)
+		}
+		x.Parent = n
+	}
+	return &x, nil
+}
+
+var wanted = map[string]bool{
+	"ProductRating":        true,
+	"ProductName":          true,
+	"ProductGender":        true,
+	"ProductTypeID":        true,
+	"BodyPatternProductId": true,
+	"__DATAIMPORT":         true,
+}
+
+func id(values map[string]string, name string) (int64, error) {
+	s := values[name]
+	if s == "" {
+		return 0, nil
+	}
+	n, err := strconv.ParseInt(s, 10, 64)
+	if err != nil || n < 0 {
+		return 0, fmt.Errorf("%s: %q isn't a number", name, s)
+	}
+	return n, nil
+}
+
+// charsetReader reads Latin-1 (and windows-1252, approximately) as well as UTF-8.
+func charsetReader(charset string, r io.Reader) (io.Reader, error) {
+	switch strings.ToLower(charset) {
+	case "iso-8859-1", "latin1", "latin-1", "windows-1252", "cp1252", "us-ascii", "ascii":
+		data, err := io.ReadAll(r)
+		if err != nil {
+			return nil, err
+		}
+		var b strings.Builder
+		for _, c := range data {
+			b.WriteRune(rune(c))
+		}
+		return strings.NewReader(b.String()), nil
+	}
+	return nil, fmt.Errorf("unsupported encoding %q", charset)
+}

pack/manifest.go

@@ -0,0 +1,102 @@
+package pack
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"strings"
+	"unicode"
+	"unicode/utf8"
+)
+
+// Format is the pack format this package reads.
+const Format = 1
+
+// Ratings.
+const (
+	General = "general"
+	Adult   = "adult"
+)
+
+// Manifest limits.
+const (
+	MaxIDLength          = 64
+	MaxNameLength        = 64 // characters
+	MaxCreatorLength     = 64 // characters
+	MaxDescriptionLength = 20000
+)
+
+// Manifest is t5de-product.json.
+type Manifest struct {
+	Format      int    `json:"format"`
+	ID          string `json:"id"`
+	Revision    int    `json:"revision"`
+	Name        string `json:"name"`
+	Creator     string `json:"creator,omitempty"`
+	Description string `json:"description,omitempty"` // HTML, cleaned by package safehtml when shown
+	Rating      string `json:"rating"`
+}
+
+// ParseManifest parses and checks t5de-product.json. Name and Creator come back trimmed.
+func ParseManifest(data []byte) (*Manifest, error) {
+	m, err := parseManifest(data)
+	if err != nil {
+		return nil, fmt.Errorf("%s: %w", ManifestName, err)
+	}
+	return m, nil
+}
+
+func parseManifest(data []byte) (*Manifest, error) {
+	d := json.NewDecoder(bytes.NewReader(data))
+	d.DisallowUnknownFields()
+	var m Manifest
+	if err := d.Decode(&m); err != nil {
+		return nil, err
+	}
+	if _, err := d.Token(); err != io.EOF {
+		return nil, errors.New("data after the object")
+	}
+	m.Name = strings.TrimSpace(m.Name)
+	m.Creator = strings.TrimSpace(m.Creator)
+	switch {
+	case m.Format != Format:
+		return nil, fmt.Errorf("format: is %d, must be %d", m.Format, Format)
+	case !validID(m.ID):
+		return nil, fmt.Errorf("id: %q must be 1-%d of a-z, 0-9 and -, not starting or ending with -", m.ID, MaxIDLength)
+	case m.Revision < 1:
+		return nil, fmt.Errorf("revision: is %d, must be at least 1", m.Revision)
+	case m.Name == "":
+		return nil, errors.New("name: missing")
+	case utf8.RuneCountInString(m.Name) > MaxNameLength:
+		return nil, fmt.Errorf("name: over %d characters", MaxNameLength)
+	case hasControl(m.Name):
+		return nil, errors.New("name: has control characters")
+	case utf8.RuneCountInString(m.Creator) > MaxCreatorLength:
+		return nil, fmt.Errorf("creator: over %d characters", MaxCreatorLength)
+	case hasControl(m.Creator):
+		return nil, errors.New("creator: has control characters")
+	case len(m.Description) > MaxDescriptionLength:
+		return nil, fmt.Errorf("description: over %d bytes", MaxDescriptionLength)
+	case m.Rating != General && m.Rating != Adult:
+		return nil, fmt.Errorf("rating: %q must be %q or %q", m.Rating, General, Adult)
+	}
+	return &m, nil
+}
+
+func validID(id string) bool {
+	if id == "" || len(id) > MaxIDLength || id[0] == '-' || id[len(id)-1] == '-' {
+		return false
+	}
+	for _, c := range []byte(id) {
+		if !(c >= 'a' && c <= 'z' || c >= '0' && c <= '9' || c == '-') {
+			return false
+		}
+	}
+	return true
+}
+
+func hasControl(s string) bool {
+	return strings.ContainsFunc(s, unicode.IsControl)
+}

pack/pack.go

@@ -0,0 +1,275 @@
+// Package pack loads local product packs (format 1): a folder with t5de-product.json and the product's files,
+// either zipped as product.chkn (IMVU Create Mode's project format) or in files/. See docs/pack.md.
+package pack
+
+import (
+	"archive/zip"
+	"bytes"
+	"crypto/sha256"
+	"encoding/binary"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+	"unicode/utf8"
+
+	"git.t5de.dev/t5de/t5de-api/cfl"
+)
+
+// Names in a pack folder.
+const (
+	ManifestName = "t5de-product.json"
+	ChknName     = "product.chkn"
+	FilesDir     = "files"
+	IndexName    = "index.xml"
+)
+
+// Limits.
+const (
+	MaxFiles     = 512
+	MaxFileSize  = 64 << 20
+	MaxTotalSize = 256 << 20
+	MaxNameBytes = 200
+	maxManifest  = 1 << 20
+)
+
+// Pack is a loaded product pack.
+type Pack struct {
+	Manifest Manifest
+	Files    []File // sorted by name
+	Index    Index  // from index.xml
+}
+
+// File is one of a product's files.
+type File struct {
+	Name string
+	Data []byte
+}
+
+// Load reads the pack in dir and checks it.
+func Load(dir string) (*Pack, error) {
+	data, err := readLimited(filepath.Join(dir, ManifestName), maxManifest)
+	if err != nil {
+		return nil, err
+	}
+	m, err := ParseManifest(data)
+	if err != nil {
+		return nil, err
+	}
+
+	chkn := filepath.Join(dir, ChknName)
+	filesDir := filepath.Join(dir, FilesDir)
+	_, chknErr := os.Stat(chkn)
+	_, dirErr := os.Stat(filesDir)
+	var files []File
+	switch {
+	case chknErr == nil && dirErr == nil:
+		return nil, fmt.Errorf("%s and %s/ both exist; keep one", ChknName, FilesDir)
+	case chknErr == nil:
+		files, err = readChkn(chkn)
+	case dirErr == nil:
+		files, err = readFilesDir(filesDir)
+	case errors.Is(chknErr, fs.ErrNotExist) && errors.Is(dirErr, fs.ErrNotExist):
+		return nil, fmt.Errorf("%s: no %s or %s/", dir, ChknName, FilesDir)
+	case !errors.Is(chknErr, fs.ErrNotExist):
+		return nil, chknErr
+	default:
+		return nil, dirErr
+	}
+	if err != nil {
+		return nil, err
+	}
+	slices.SortFunc(files, func(a, b File) int { return strings.Compare(a.Name, b.Name) })
+
+	p := &Pack{Manifest: *m, Files: files}
+	i, ok := slices.BinarySearchFunc(files, IndexName, func(f File, name string) int { return strings.Compare(f.Name, name) })
+	if !ok {
+		return nil, fmt.Errorf("no %s", IndexName)
+	}
+	index, err := ParseIndex(files[i].Data)
+	if err != nil {
+		return nil, fmt.Errorf("%s: %w", IndexName, err)
+	}
+	p.Index = *index
+	if want := ratings[m.Rating]; index.Rating != want {
+		return nil, fmt.Errorf("%s: ProductRating is %q, but %s says %q (%s)", IndexName, index.Rating, ManifestName, m.Rating, want)
+	}
+	return p, nil
+}
+
+// ratings maps manifest ratings to index.xml's ProductRating.
+var ratings = map[string]string{General: "GA", Adult: "AP"}
+
+func readLimited(path string, limit int64) ([]byte, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	data, err := io.ReadAll(io.LimitReader(f, limit+1))
+	if err != nil {
+		return nil, err
+	}
+	if int64(len(data)) > limit {
+		return nil, fmt.Errorf("%s: over %d bytes", filepath.Base(path), limit)
+	}
+	return data, nil
+}
+
+// collector checks files as they're added.
+type collector struct {
+	files []File
+	names map[string]bool
+	total int64
+}
+
+// add reads a file's data from r, at most the limits allow.
+func (c *collector) add(name string, r io.Reader) error {
+	if err := CheckName(name); err != nil {
+		return err
+	}
+	if c.names[name] {
+		return fmt.Errorf("%q: duplicate file", name)
+	}
+	if len(c.files) >= MaxFiles {
+		return fmt.Errorf("over %d files", MaxFiles)
+	}
+	data, err := io.ReadAll(io.LimitReader(r, MaxFileSize+1))
+	if err != nil {
+		return fmt.Errorf("%q: %w", name, err)
+	}
+	if len(data) > MaxFileSize {
+		return fmt.Errorf("%q: over %d MiB", name, MaxFileSize>>20)
+	}
+	c.total += int64(len(data))
+	if c.total > MaxTotalSize {
+		return fmt.Errorf("files over %d MiB in all", MaxTotalSize>>20)
+	}
+	if c.names == nil {
+		c.names = map[string]bool{}
+	}
+	c.names[name] = true
+	c.files = append(c.files, File{Name: name, Data: data})
+	return nil
+}
+
+func readChkn(path string) ([]File, error) {
+	z, err := zip.OpenReader(path)
+	if err != nil {
+		return nil, fmt.Errorf("%s: %w", ChknName, err)
+	}
+	defer z.Close()
+	var c collector
+	for _, f := range z.File {
+		if f.FileInfo().IsDir() {
+			continue
+		}
+		if strings.Contains(f.Name, "/") {
+			return nil, fmt.Errorf("%s: %q is in a folder; files must be at the top", ChknName, f.Name)
+		}
+		if err := addZipFile(&c, f); err != nil {
+			return nil, fmt.Errorf("%s: %w", ChknName, err)
+		}
+	}
+	return c.files, nil
+}
+
+func addZipFile(c *collector, f *zip.File) error {
+	rc, err := f.Open()
+	if err != nil {
+		return fmt.Errorf("%q: %w", f.Name, err)
+	}
+	defer rc.Close()
+	return c.add(f.Name, rc)
+}
+
+func readFilesDir(dir string) ([]File, error) {
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		return nil, err
+	}
+	var c collector
+	for _, e := range entries {
+		if e.IsDir() {
+			return nil, fmt.Errorf("%s/%s: folders aren't allowed in %s/", FilesDir, e.Name(), FilesDir)
+		}
+		if !e.Type().IsRegular() {
+			return nil, fmt.Errorf("%s/%s: not a regular file", FilesDir, e.Name())
+		}
+		if err := addDirFile(&c, filepath.Join(dir, e.Name()), e.Name()); err != nil {
+			return nil, fmt.Errorf("%s/: %w", FilesDir, err)
+		}
+	}
+	return c.files, nil
+}
+
+func addDirFile(c *collector, path, name string) error {
+	f, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	return c.add(name, f)
+}
+
+// CheckName checks a product file's name.
+func CheckName(name string) error {
+	switch {
+	case name == "":
+		return errors.New("empty file name")
+	case name == "." || name == "..":
+		return fmt.Errorf("%q isn't a file name", name)
+	case strings.ContainsAny(name, `/\`):
+		return fmt.Errorf("%q: file names can't have / or \\", name)
+	case !utf8.ValidString(name):
+		return fmt.Errorf("%q: file name isn't valid UTF-8", name)
+	case len(name) > MaxNameBytes:
+		return fmt.Errorf("%q: file name over %d bytes", name, MaxNameBytes)
+	case strings.EqualFold(name, "_contents.json"):
+		return fmt.Errorf("%q: packs can't have _contents.json", name)
+	}
+	for _, r := range name {
+		if r < 0x20 || (r >= 0x7f && r < 0xa0) {
+			return fmt.Errorf("%q: file name has control characters", name)
+		}
+		if r > 0xff {
+			return fmt.Errorf("%q: file name has characters outside Latin-1", name)
+		}
+	}
+	return nil
+}
+
+// Checksum returns a hex sha256 of the pack's files (names and contents), whatever their storage. It changes when
+// any file does.
+func (p *Pack) Checksum() string {
+	h := sha256.New()
+	h.Write([]byte("t5de-pack-files\n"))
+	var n [8]byte
+	for _, f := range p.Files {
+		binary.BigEndian.PutUint64(n[:], uint64(len(f.Name)))
+		h.Write(n[:])
+		h.Write([]byte(f.Name))
+		binary.BigEndian.PutUint64(n[:], uint64(len(f.Data)))
+		h.Write(n[:])
+		h.Write(f.Data)
+	}
+	return hex.EncodeToString(h.Sum(nil))
+}
+
+// CFL returns the pack's files as an uncompressed CFL, as the client downloads products.
+func (p *Pack) CFL() ([]byte, error) {
+	entries := make([]cfl.Entry, len(p.Files))
+	for i, f := range p.Files {
+		entries[i] = cfl.Entry{Name: f.Name, Data: f.Data}
+	}
+	var b bytes.Buffer
+	if err := cfl.Write(&b, entries); err != nil {
+		return nil, err
+	}
+	return b.Bytes(), nil
+}

pack/pack_test.go

@@ -0,0 +1,315 @@
+package pack
+
+import (
+	"archive/zip"
+	"bytes"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"git.t5de.dev/t5de/t5de-api/cfl"
+)
+
+// Synthetic test files: not IMVU's.
+const (
+	testManifest = `{"format": 1, "id": "blue-coat", "revision": 2, "name": " Blue Coat ", "creator": "Tester",
+		"description": "<p>A coat</p>", "rating": "general"}`
+	testIndex = `<?xml version="1.0" encoding="UTF-8" ?><Template><ProductRating>GA</ProductRating>` +
+		`<ProductTypeID>106</ProductTypeID><BodyPatternProductId>80</BodyPatternProductId>` +
+		`<ProductGender>female</ProductGender><ParentCategoryIDs>1,2</ParentCategoryIDs>` +
+		`<__DATAIMPORT>product://12345/index.xml</__DATAIMPORT><ProductName>Test Coat</ProductName>` +
+		`<Other><ProductName>nested, ignored</ProductName></Other></Template>`
+)
+
+var testPNG = []byte("\x89PNG\r\n\x1a\nfake")
+
+func testFiles() map[string][]byte {
+	return map[string][]byte{"index.xml": []byte(testIndex), "coat.png": testPNG, "coat.xmf": []byte("mesh")}
+}
+
+func writeFile(t *testing.T, path string, data []byte) {
+	t.Helper()
+	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(path, data, 0o644); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// dirPack makes a pack with its files in files/.
+func dirPack(t *testing.T, manifest string, files map[string][]byte) string {
+	dir := t.TempDir()
+	writeFile(t, filepath.Join(dir, ManifestName), []byte(manifest))
+	if err := os.Mkdir(filepath.Join(dir, FilesDir), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	for name, data := range files {
+		writeFile(t, filepath.Join(dir, FilesDir, name), data)
+	}
+	return dir
+}
+
+type zipEntry struct {
+	name   string
+	data   []byte
+	method uint16
+}
+
+func makeZip(t *testing.T, entries []zipEntry) []byte {
+	t.Helper()
+	var b bytes.Buffer
+	z := zip.NewWriter(&b)
+	for _, e := range entries {
+		w, err := z.CreateHeader(&zip.FileHeader{Name: e.name, Method: e.method})
+		if err != nil {
+			t.Fatal(err)
+		}
+		w.Write(e.data)
+	}
+	if err := z.Close(); err != nil {
+		t.Fatal(err)
+	}
+	return b.Bytes()
+}
+
+// chknPack makes a pack with its files in product.chkn.
+func chknPack(t *testing.T, manifest string, entries []zipEntry) string {
+	dir := t.TempDir()
+	writeFile(t, filepath.Join(dir, ManifestName), []byte(manifest))
+	writeFile(t, filepath.Join(dir, ChknName), makeZip(t, entries))
+	return dir
+}
+
+func zipEntries(method uint16) []zipEntry {
+	var entries []zipEntry
+	for name, data := range testFiles() {
+		entries = append(entries, zipEntry{name, data, method})
+	}
+	return entries
+}
+
+func TestLoad(t *testing.T) {
+	fromDir, err := Load(dirPack(t, testManifest, testFiles()))
+	if err != nil {
+		t.Fatal(err)
+	}
+	m := fromDir.Manifest
+	if m.ID != "blue-coat" || m.Revision != 2 || m.Name != "Blue Coat" || m.Creator != "Tester" || m.Rating != General {
+		t.Errorf("manifest: %+v", m)
+	}
+	want := Index{Rating: "GA", Name: "Test Coat", Gender: "female", TypeID: 106, BodyPattern: 80, Parent: 12345}
+	if fromDir.Index != want {
+		t.Errorf("index: got %+v, want %+v", fromDir.Index, want)
+	}
+	var names []string
+	for _, f := range fromDir.Files {
+		names = append(names, f.Name)
+	}
+	if got := strings.Join(names, " "); got != "coat.png coat.xmf index.xml" {
+		t.Errorf("files: %s", got)
+	}
+
+	// The same files zipped, stored or deflated, have the same checksum. Folder entries are ignored.
+	for _, method := range []uint16{zip.Store, zip.Deflate} {
+		entries := append(zipEntries(method), zipEntry{name: "folder/", method: method})
+		fromZip, err := Load(chknPack(t, testManifest, entries))
+		if err != nil {
+			t.Fatal(err)
+		}
+		if fromZip.Checksum() != fromDir.Checksum() {
+			t.Errorf("method %d: checksum differs", method)
+		}
+	}
+
+	changed := testFiles()
+	changed["coat.png"] = append(testPNG, 0)
+	other, err := Load(dirPack(t, testManifest, changed))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if other.Checksum() == fromDir.Checksum() {
+		t.Error("checksum didn't change")
+	}
+
+	data, err := fromDir.CFL()
+	if err != nil {
+		t.Fatal(err)
+	}
+	entries, err := cfl.Read(data)
+	if err != nil || len(entries) != 3 || entries[2].Name != "index.xml" {
+		t.Errorf("CFL: %q, %v", entries, err)
+	}
+}
+
+func TestLoadErrors(t *testing.T) {
+	adult := strings.Replace(testManifest, `"general"`, `"adult"`, 1)
+	with := func(name string, data []byte) map[string][]byte {
+		f := testFiles()
+		f[name] = data
+		return f
+	}
+	without := func(name string) map[string][]byte {
+		f := testFiles()
+		delete(f, name)
+		return f
+	}
+	extra := func(e ...zipEntry) []zipEntry { return append(zipEntries(zip.Store), e...) }
+
+	for _, c := range []struct {
+		name, want string
+		dir        func(t *testing.T) string
+	}{
+		{"rating mismatch", "ProductRating", func(t *testing.T) string { return dirPack(t, adult, testFiles()) }},
+		{"no index", "no index.xml", func(t *testing.T) string { return dirPack(t, testManifest, without("index.xml")) }},
+		{"contents json", "_contents.json", func(t *testing.T) string {
+			return dirPack(t, testManifest, with("_contents.json", []byte("{}")))
+		}},
+		{"bad index", "index.xml: ProductRating", func(t *testing.T) string {
+			return dirPack(t, testManifest, with("index.xml", []byte("<Template/>")))
+		}},
+		{"subfolder", "folders", func(t *testing.T) string {
+			dir := dirPack(t, testManifest, testFiles())
+			writeFile(t, filepath.Join(dir, FilesDir, "sub", "a.png"), testPNG)
+			return dir
+		}},
+		{"both", "both exist", func(t *testing.T) string {
+			dir := dirPack(t, testManifest, testFiles())
+			writeFile(t, filepath.Join(dir, ChknName), makeZip(t, zipEntries(zip.Store)))
+			return dir
+		}},
+		{"neither", "no product.chkn", func(t *testing.T) string {
+			dir := t.TempDir()
+			writeFile(t, filepath.Join(dir, ManifestName), []byte(testManifest))
+			return dir
+		}},
+		{"no manifest", ManifestName, func(t *testing.T) string { return t.TempDir() }},
+		{"bad manifest", "revision", func(t *testing.T) string {
+			return dirPack(t, strings.Replace(testManifest, `"revision": 2`, `"revision": 0`, 1), testFiles())
+		}},
+		{"nested zip entry", "in a folder", func(t *testing.T) string {
+			return chknPack(t, testManifest, extra(zipEntry{"sub/a.png", testPNG, zip.Store}))
+		}},
+		{"backslash in zip", `can't have`, func(t *testing.T) string {
+			return chknPack(t, testManifest, extra(zipEntry{`sub\a.png`, testPNG, zip.Store}))
+		}},
+		{"duplicate in zip", "duplicate", func(t *testing.T) string {
+			return chknPack(t, testManifest, extra(zipEntry{"coat.png", testPNG, zip.Store}))
+		}},
+		{"not latin-1", "Latin-1", func(t *testing.T) string {
+			return chknPack(t, testManifest, extra(zipEntry{"snow☃.png", testPNG, zip.Store}))
+		}},
+		{"too many files", "over 512 files", func(t *testing.T) string {
+			var e []zipEntry
+			for i := range MaxFiles {
+				e = append(e, zipEntry{name: fmt.Sprintf("f%d.png", i), method: zip.Store})
+			}
+			return chknPack(t, testManifest, extra(e...))
+		}},
+		{"zip bomb", "over 64 MiB", func(t *testing.T) string {
+			// Deflates to well under 1 MiB.
+			return chknPack(t, testManifest, extra(zipEntry{"big.png", make([]byte, MaxFileSize+1), zip.Deflate}))
+		}},
+		{"too big in all", "in all", func(t *testing.T) string {
+			var e []zipEntry
+			for i := range 5 {
+				e = append(e, zipEntry{fmt.Sprintf("big%d.png", i), make([]byte, MaxFileSize), zip.Deflate})
+			}
+			return chknPack(t, testManifest, extra(e...))
+		}},
+		{"not a zip", ChknName, func(t *testing.T) string {
+			dir := t.TempDir()
+			writeFile(t, filepath.Join(dir, ManifestName), []byte(testManifest))
+			writeFile(t, filepath.Join(dir, ChknName), []byte("not a zip"))
+			return dir
+		}},
+	} {
+		t.Run(c.name, func(t *testing.T) {
+			_, err := Load(c.dir(t))
+			if err == nil || !strings.Contains(err.Error(), c.want) {
+				t.Errorf("got %v, want an error with %q", err, c.want)
+			}
+		})
+	}
+}
+
+func TestParseManifest(t *testing.T) {
+	for _, c := range []struct{ json, want string }{
+		{`{"format": 2, "id": "a", "revision": 1, "name": "A", "rating": "general"}`, "format"},
+		{`{"id": "a", "revision": 1, "name": "A", "rating": "general"}`, "format"},
+		{`{"format": 1, "id": "-a", "revision": 1, "name": "A", "rating": "general"}`, "id"},
+		{`{"format": 1, "id": "a-", "revision": 1, "name": "A", "rating": "general"}`, "id"},
+		{`{"format": 1, "id": "A", "revision": 1, "name": "A", "rating": "general"}`, "id"},
+		{`{"format": 1, "id": "", "revision": 1, "name": "A", "rating": "general"}`, "id"},
+		{`{"format": 1, "id": "` + strings.Repeat("a", 65) + `", "revision": 1, "name": "A", "rating": "general"}`, "id"},
+		{`{"format": 1, "id": "a", "revision": 0, "name": "A", "rating": "general"}`, "revision"},
+		{`{"format": 1, "id": "a", "revision": 1.5, "name": "A", "rating": "general"}`, "revision"},
+		{`{"format": 1, "id": "a", "revision": 1, "name": "  ", "rating": "general"}`, "name"},
+		{`{"format": 1, "id": "a", "revision": 1, "name": "` + strings.Repeat("é", 65) + `", "rating": "general"}`, "name"},
+		{`{"format": 1, "id": "a", "revision": 1, "name": "A\u0007", "rating": "general"}`, "name"},
+		{`{"format": 1, "id": "a", "revision": 1, "name": "A", "creator": "` + strings.Repeat("c", 65) + `", "rating": "general"}`, "creator"},
+		{`{"format": 1, "id": "a", "revision": 1, "name": "A", "description": "` + strings.Repeat("d", 20001) + `", "rating": "general"}`, "description"},
+		{`{"format": 1, "id": "a", "revision": 1, "name": "A", "rating": "GA"}`, "rating"},
+		{`{"format": 1, "id": "a", "revision": 1, "name": "A"}`, "rating"},
+		{`{"format": 1, "id": "a", "revision": 1, "name": "A", "rating": "general", "price": 5}`, "price"},
+		{`{"format": 1, "id": "a", "revision": 1, "name": "A", "rating": "general"} {}`, "after"},
+	} {
+		_, err := ParseManifest([]byte(c.json))
+		if err == nil || !strings.Contains(err.Error(), c.want) || !strings.Contains(err.Error(), ManifestName) {
+			t.Errorf("%s: got %v, want an error with %q", c.json, err, c.want)
+		}
+	}
+
+	m, err := ParseManifest([]byte(`{"format": 1, "id": "a-1", "revision": 1, "name": "` + strings.Repeat("é", 64) + `", "rating": "adult"}`))
+	if err != nil || m.Rating != Adult {
+		t.Errorf("valid manifest: %+v, %v", m, err)
+	}
+}
+
+func TestParseIndex(t *testing.T) {
+	x, err := ParseIndex([]byte("<?xml version=\"1.0\" encoding=\"ISO-8859-1\"?><Template>" +
+		"<ProductRating> AP </ProductRating><ProductName>Caf\xe9 &amp; &nbsp;Bar</ProductName><Junk a=b>x<br></Junk></Template>"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if want := (Index{Rating: "AP", Name: "Café &  Bar"}); *x != want {
+		t.Errorf("got %+v, want %+v", *x, want)
+	}
+
+	if x, err := ParseIndex([]byte("\ufeff<Template><ProductRating>GA</ProductRating></Template>")); err != nil || x.Rating != "GA" {
+		t.Errorf("with a BOM: %+v, %v", x, err)
+	}
+
+	for _, c := range []struct{ xml, want string }{
+		{`<Product><ProductRating>GA</ProductRating></Product>`, "Template"},
+		{``, "Template"},
+		{`<Template></Template>`, "ProductRating"},
+		{`<Template><ProductRating>PG</ProductRating></Template>`, "ProductRating"},
+		{`<Template><ProductRating>GA</ProductRating><ProductTypeID>x</ProductTypeID></Template>`, "ProductTypeID"},
+		{`<Template><ProductRating>GA</ProductRating><BodyPatternProductId>-1</BodyPatternProductId></Template>`, "BodyPatternProductId"},
+		{`<Template><ProductRating>GA</ProductRating><__DATAIMPORT>http://e.com/index.xml</__DATAIMPORT></Template>`, "__DATAIMPORT"},
+		{`<Template><ProductRating>GA</ProductRating><__DATAIMPORT>product://0/index.xml</__DATAIMPORT></Template>`, "__DATAIMPORT"},
+		{`<Template><ProductRating>GA</ProductRating><ProductRating>AP</ProductRating></Template>`, "twice"},
+		{`<?xml version="1.0" encoding="EBCDIC"?><Template/>`, "encoding"},
+	} {
+		_, err := ParseIndex([]byte(c.xml))
+		if err == nil || !strings.Contains(err.Error(), c.want) {
+			t.Errorf("%s: got %v, want an error with %q", c.xml, err, c.want)
+		}
+	}
+}
+
+func TestCheckName(t *testing.T) {
+	for _, name := range []string{"index.xml", "a b.png", "café.jpg", strings.Repeat("a", 200)} {
+		if err := CheckName(name); err != nil {
+			t.Errorf("%q: %v", name, err)
+		}
+	}
+	for _, name := range []string{"", ".", "..", "a/b", `a\b`, "\xff", strings.Repeat("a", 201), "_CONTENTS.json", "a\x00b", "a\nb", "snow☃"} {
+		if err := CheckName(name); err == nil {
+			t.Errorf("%q: no error", name)
+		}
+	}
+}

safehtml/safehtml.go

@@ -0,0 +1,141 @@
+// Package safehtml makes product descriptions safe to show anywhere, including the client's privileged pages.
+//
+// Descriptions are cleaned by an allowlist: text formatting, lists, simple tables, headings, links and images,
+// and a few styles with checked values. Everything else is removed.
+package safehtml
+
+import (
+	"net/url"
+	"regexp"
+	"strings"
+
+	"github.com/microcosm-cc/bluemonday"
+	"golang.org/x/net/html"
+)
+
+// MaxDescription is the most bytes a description may have. Description doesn't check it: callers validate
+// descriptions when they're saved.
+const MaxDescription = 20000
+
+// LinkRel is the rel attribute every link gets.
+const LinkRel = "nofollow ugc noopener noreferrer"
+
+var (
+	color      = `#[0-9a-f]{3,4}|#[0-9a-f]{6}|#[0-9a-f]{8}|[a-z]{1,30}|rgba?\(\s*\d{1,3}%?\s*,\s*\d{1,3}%?\s*,\s*\d{1,3}%?\s*(,\s*(0|1|0?\.\d{1,3})\s*)?\)`
+	colorValue = regexp.MustCompile(`^(?i)(` + color + `)$`)
+	// font's color attribute: CSS colors, and hex without '#', which browsers also take.
+	fontColor      = regexp.MustCompile(`^(?i)(` + color + `|[0-9a-f]{6})$`)
+	fontSize       = regexp.MustCompile(`^[+-]?[1-7]$`)
+	cssFontSize    = regexp.MustCompile(`^(?i)(\d{1,3}(\.\d{1,2})?(px|pt|em|%)|0|xx-small|x-small|small|medium|large|x-large|xx-large|smaller|larger)$`)
+	fontWeight     = regexp.MustCompile(`^(?i)(normal|bold|bolder|lighter|[1-9]00)$`)
+	fontStyle      = regexp.MustCompile(`^(?i)(normal|italic)$`)
+	textAlign      = regexp.MustCompile(`^(?i)(left|right|center|justify)$`)
+	textDecoration = regexp.MustCompile(`^(?i)(none|underline|line-through)$`)
+	smallInt       = regexp.MustCompile(`^[1-9][0-9]{0,3}$`)
+)
+
+var policy = newPolicy()
+
+func newPolicy() *bluemonday.Policy {
+	p := bluemonday.NewPolicy()
+	p.AllowElements("b", "i", "u", "s", "em", "strong", "small", "big", "sub", "sup", "font", "span", "div", "p",
+		"br", "hr", "center", "ul", "ol", "li", "table", "thead", "tbody", "tr", "td", "th",
+		"h1", "h2", "h3", "h4", "h5", "h6", "a", "img", "blockquote")
+	// bluemonday drops elements without attributes unless they are on its list, which lacks big.
+	p.AllowNoAttrs().OnElements("big")
+
+	p.AllowAttrs("color").Matching(fontColor).OnElements("font")
+	p.AllowAttrs("size").Matching(fontSize).OnElements("font")
+	p.AllowAttrs("colspan", "rowspan").Matching(smallInt).OnElements("td", "th")
+
+	p.AllowAttrs("href", "title").OnElements("a")
+	p.AllowAttrs("src", "alt", "title").OnElements("img")
+	p.AllowAttrs("width", "height").Matching(smallInt).OnElements("img")
+	p.AllowURLSchemes("http", "https")
+	p.RequireParseableURLs(true)
+	p.AllowRelativeURLs(false)
+
+	p.AllowStyles("color", "background-color").Matching(colorValue).Globally()
+	p.AllowStyles("font-size").Matching(cssFontSize).Globally()
+	p.AllowStyles("font-weight").Matching(fontWeight).Globally()
+	p.AllowStyles("font-style").Matching(fontStyle).Globally()
+	p.AllowStyles("text-align").Matching(textAlign).Globally()
+	p.AllowStyles("text-decoration").Matching(textDecoration).Globally()
+	return p
+}
+
+// Description returns description cleaned by the allowlist. Images keep their https addresses.
+func Description(description string) string {
+	return DescriptionWithImages(description, nil)
+}
+
+// DescriptionWithImages is Description with each image's https address passed to rewrite, which returns the
+// address to show instead, or false to drop the image. A nil rewrite keeps the addresses.
+func DescriptionWithImages(description string, rewrite func(src string) (string, bool)) string {
+	clean := policy.Sanitize(description)
+
+	// bluemonday has removed everything unsafe; this pass only adds fixed link attributes and checks images.
+	var b strings.Builder
+	z := html.NewTokenizer(strings.NewReader(clean))
+	for {
+		if z.Next() == html.ErrorToken {
+			return b.String()
+		}
+		t := z.Token()
+		if t.Type == html.StartTagToken || t.Type == html.SelfClosingTagToken {
+			switch t.Data {
+			case "a":
+				t.Attr = linkAttrs(t.Attr)
+			case "img":
+				var ok bool
+				if t.Attr, ok = imageAttrs(t.Attr, rewrite); !ok {
+					continue
+				}
+			}
+		}
+		b.WriteString(t.String())
+	}
+}
+
+func linkAttrs(attrs []html.Attribute) []html.Attribute {
+	for _, a := range attrs {
+		if a.Key == "href" {
+			return append(attrs,
+				html.Attribute{Key: "rel", Val: LinkRel},
+				html.Attribute{Key: "target", Val: "_blank"})
+		}
+	}
+	return attrs
+}
+
+func imageAttrs(attrs []html.Attribute, rewrite func(string) (string, bool)) ([]html.Attribute, bool) {
+	for i, a := range attrs {
+		if a.Key != "src" {
+			continue
+		}
+		if !isHTTPS(a.Val) {
+			return nil, false
+		}
+		if rewrite != nil {
+			src, ok := rewrite(a.Val)
+			if !ok || !isWebURL(src) {
+				return nil, false
+			}
+			attrs[i].Val = src
+		}
+		return attrs, true
+	}
+	return nil, false
+}
+
+func isHTTPS(s string) bool {
+	u, err := url.Parse(s)
+	return err == nil && u.Scheme == "https" && u.Host != ""
+}
+
+// isWebURL checks a rewritten image address: the image proxy can be http on the local machine.
+func isWebURL(s string) bool {
+	u, err := url.Parse(s)
+	return err == nil && (u.Scheme == "https" || u.Scheme == "http") && u.Host != "" &&
+		!strings.ContainsAny(s, " \t\r\n")
+}

safehtml/safehtml_test.go

@@ -0,0 +1,114 @@
+package safehtml
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestDescription(t *testing.T) {
+	const rel = ` rel="nofollow ugc noopener noreferrer" target="_blank"`
+	for _, c := range []struct{ in, want string }{
+		// Kept.
+		{`<font color="#ff0000" size="4"><b>Hi</b></font><br><center>Made by me</center>`,
+			`<font color="#ff0000" size="4"><b>Hi</b></font><br><center>Made by me</center>`},
+		{`<font color="ff0000" size="+2">x</font>`, `<font color="ff0000" size="+2">x</font>`},
+		{`<p style="color: red; text-align: center; font-weight: bold">x</p>`,
+			`<p style="color: red; text-align: center; font-weight: bold">x</p>`},
+		{`<span style="background-color: rgb(1, 2, 3); font-size: 12px; font-style: italic; text-decoration: underline">x</span>`,
+			`<span style="background-color: rgb(1, 2, 3); font-size: 12px; font-style: italic; text-decoration: underline">x</span>`},
+		{`<a href="https://example.com/a?b=c&amp;d=e">x</a>`, `<a href="https://example.com/a?b=c&amp;d=e"` + rel + `>x</a>`},
+		{`<a href="http://example.com/">x</a>`, `<a href="http://example.com/"` + rel + `>x</a>`},
+		{`<img src="https://example.com/a.png" alt="A" width="100">`, `<img src="https://example.com/a.png" alt="A" width="100">`},
+		{`<table><tbody><tr><td colspan="2">x</td></tr></tbody></table>`, `<table><tbody><tr><td colspan="2">x</td></tr></tbody></table>`},
+		{`<ul><li>a</li></ul><h1>b</h1><blockquote>c</blockquote><big>d</big><hr>`,
+			`<ul><li>a</li></ul><h1>b</h1><blockquote>c</blockquote><big>d</big><hr>`},
+		{`5 < 6 & "quotes"`, `5 &lt; 6 &amp; &#34;quotes&#34;`},
+
+		// Removed.
+		{`<script>alert(1)</script>x`, `x`},
+		{`<SCRIPT SRC=https://e.com/x.js></SCRIPT>x`, `x`},
+		{`<scr<script>ipt>alert(1)</script>`, `ipt&gt;alert(1)`},
+		{`<img src=x onerror=alert(1)>`, ``},
+		{`<img src="https://e.com/a.png" onerror="alert(1)">`, `<img src="https://e.com/a.png">`},
+		{`<b onclick="alert(1)" onmouseover=alert(1)>x</b>`, `<b>x</b>`},
+		{`<a href="javascript:alert(1)">x</a>`, `x`},
+		{`<a href="  javascript:alert(1)">x</a>`, `x`},
+		{`<a href="JaVaScRiPt:alert(1)">x</a>`, `x`},
+		{`<a href="&#106;avascript:alert(1)">x</a>`, `x`},
+		{`<a href="&#x6A;avascript&#58;alert(1)">x</a>`, `x`},
+		{`<a href="java&#x09;script:alert(1)">x</a>`, `x`},
+		{`<a href="vbscript:msgbox(1)">x</a>`, `x`},
+		{`<a href="data:text/html;base64,PHNjcmlwdD5hbGVydCgxKTwvc2NyaXB0Pg==">x</a>`, `x`},
+		{`<a href="chrome://browser/content/browser.xul">x</a>`, `x`},
+		{`<a href="/relative">x</a>`, `x`},
+		{`<a href="https://e.com/" target="_self" rel="opener">x</a>`, `<a href="https://e.com/"` + rel + `>x</a>`},
+		{`<img src="javascript:alert(1)">`, ``},
+		{`<img src="http://e.com/a.png">`, ``},
+		{`<img src="data:image/png;base64,iVBORw0KGgo=">`, ``},
+		{`<img alt="no src">`, ``},
+		{`<p style="background-color: url(https://e.com/x.png)">x</p>`, `<p>x</p>`},
+		{`<p style="background: url(https://e.com/x.png)">x</p>`, `<p>x</p>`},
+		{`<p style="color: expression(alert(1))">x</p>`, `<p>x</p>`},
+		{`<p style="color: red; position: fixed; top: 0; width: 100%">x</p>`, `<p style="color: red">x</p>`},
+		{`<p style="font-size: 99999px">x</p>`, `<p>x</p>`},
+		{`<font color="red" face="Arial" style="color: blue">x</font>`, `<font color="red" style="color: blue">x</font>`},
+		{`<style>body{display:none}</style>x`, `x`},
+		{`<link rel="stylesheet" href="https://e.com/x.css">x`, `x`},
+		{`<iframe src="https://e.com/"></iframe>x`, `x`},
+		{`<object data="x.swf"></object><embed src="x.swf">x`, `x`},
+		{`<form action="https://e.com/"><input name="p"></form>x`, `x`},
+		{`<!-- a comment -->x<!--[if IE]><script>alert(1)</script><![endif]-->`, `x`},
+		{`<svg onload="alert(1)"><script>alert(1)</script></svg>x`, `x`},
+		{`<math><mi xlink:href="javascript:alert(1)">x</mi></math>`, `x`},
+		{`<div><p><b>unclosed`, `<div><p><b>unclosed`},
+		{`<b><i>misnested</b></i>`, `<b><i>misnested</b></i>`},
+		{`<a href="https://e.com/"<script>alert(1)</script>>x</a>`, `<a href="https://e.com/"` + rel + `>alert(1)&gt;x</a>`},
+		{`<img src="https://e.com/a.png" width="100%" height="-1">`, `<img src="https://e.com/a.png">`},
+		{`<td rowspan="0">x</td>`, `<td>x</td>`},
+	} {
+		if got := Description(c.in); got != c.want {
+			t.Errorf("Description(%q)\n got %q\nwant %q", c.in, got, c.want)
+		}
+	}
+}
+
+func TestNoScriptAnywhere(t *testing.T) {
+	for _, in := range []string{
+		`<<script>script>alert(1)<</script>/script>`,
+		`<a href="https://e.com/" style="x:expression(alert(1))">x</a>`,
+		`<img src="https://e.com/a.png"/onerror=alert(1)>`,
+		`<IMG SRC="jav&#x0A;ascript:alert('XSS');">`,
+		`<a href="https://e.com/" onclick=alert(1)//>x</a>`,
+		`<base href="javascript:alert(1)//">`,
+		`<meta http-equiv="refresh" content="0;url=javascript:alert(1)">`,
+	} {
+		got := strings.ToLower(Description(in))
+		for _, bad := range []string{"<script", "javascript:", "onerror", "onclick", "expression", "<base", "<meta"} {
+			if strings.Contains(got, bad) {
+				t.Errorf("Description(%q) = %q, contains %q", in, got, bad)
+			}
+		}
+	}
+}
+
+func TestDescriptionWithImages(t *testing.T) {
+	rewrite := func(src string) (string, bool) {
+		if strings.Contains(src, "drop") {
+			return "", false
+		}
+		if strings.Contains(src, "evil") {
+			return "javascript:alert(1)", true
+		}
+		return "https://images.t5de.dev/?u=" + src, true
+	}
+	for _, c := range []struct{ in, want string }{
+		{`<img src="https://e.com/a.png">`, `<img src="https://images.t5de.dev/?u=https://e.com/a.png">`},
+		{`<img src="https://e.com/drop.png">x`, `x`},
+		{`<img src="https://e.com/evil.png">x`, `x`},
+		{`<img src="http://e.com/a.png">x`, `x`},
+	} {
+		if got := DescriptionWithImages(c.in, rewrite); got != c.want {
+			t.Errorf("DescriptionWithImages(%q)\n got %q\nwant %q", c.in, got, c.want)
+		}
+	}
+}