Add pack, cfl and safehtml

pack loads local product packs (format 1): t5de-product.json and the product's files, as product.chkn or files/, with index.xml's rating, type, gender and parent product. cfl writes and reads IMVU's product container, uncompressed. safehtml cleans product descriptions by an allowlist (bluemonday). Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-29 02:52:59 -07:00
commit 9bdc3ff455
@@ -0,0 +1,275 @@
+// Package pack loads local product packs (format 1): a folder with t5de-product.json and the product's files,
+// either zipped as product.chkn (IMVU Create Mode's project format) or in files/. See docs/pack.md.
+package pack
+
+import (
+	"archive/zip"
+	"bytes"
+	"crypto/sha256"
+	"encoding/binary"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+	"unicode/utf8"
+
+	"git.t5de.dev/t5de/t5de-api/cfl"
+)
+
+// Names in a pack folder.
+const (
+	ManifestName = "t5de-product.json"
+	ChknName     = "product.chkn"
+	FilesDir     = "files"
+	IndexName    = "index.xml"
+)
+
+// Limits.
+const (
+	MaxFiles     = 512
+	MaxFileSize  = 64 << 20
+	MaxTotalSize = 256 << 20
+	MaxNameBytes = 200
+	maxManifest  = 1 << 20
+)
+
+// Pack is a loaded product pack.
+type Pack struct {
+	Manifest Manifest
+	Files    []File // sorted by name
+	Index    Index  // from index.xml
+}
+
+// File is one of a product's files.
+type File struct {
+	Name string
+	Data []byte
+}
+
+// Load reads the pack in dir and checks it.
+func Load(dir string) (*Pack, error) {
+	data, err := readLimited(filepath.Join(dir, ManifestName), maxManifest)
+	if err != nil {
+		return nil, err
+	}
+	m, err := ParseManifest(data)
+	if err != nil {
+		return nil, err
+	}
+
+	chkn := filepath.Join(dir, ChknName)
+	filesDir := filepath.Join(dir, FilesDir)
+	_, chknErr := os.Stat(chkn)
+	_, dirErr := os.Stat(filesDir)
+	var files []File
+	switch {
+	case chknErr == nil && dirErr == nil:
+		return nil, fmt.Errorf("%s and %s/ both exist; keep one", ChknName, FilesDir)
+	case chknErr == nil:
+		files, err = readChkn(chkn)
+	case dirErr == nil:
+		files, err = readFilesDir(filesDir)
+	case errors.Is(chknErr, fs.ErrNotExist) && errors.Is(dirErr, fs.ErrNotExist):
+		return nil, fmt.Errorf("%s: no %s or %s/", dir, ChknName, FilesDir)
+	case !errors.Is(chknErr, fs.ErrNotExist):
+		return nil, chknErr
+	default:
+		return nil, dirErr
+	}
+	if err != nil {
+		return nil, err
+	}
+	slices.SortFunc(files, func(a, b File) int { return strings.Compare(a.Name, b.Name) })
+
+	p := &Pack{Manifest: *m, Files: files}
+	i, ok := slices.BinarySearchFunc(files, IndexName, func(f File, name string) int { return strings.Compare(f.Name, name) })
+	if !ok {
+		return nil, fmt.Errorf("no %s", IndexName)
+	}
+	index, err := ParseIndex(files[i].Data)
+	if err != nil {
+		return nil, fmt.Errorf("%s: %w", IndexName, err)
+	}
+	p.Index = *index
+	if want := ratings[m.Rating]; index.Rating != want {
+		return nil, fmt.Errorf("%s: ProductRating is %q, but %s says %q (%s)", IndexName, index.Rating, ManifestName, m.Rating, want)
+	}
+	return p, nil
+}
+
+// ratings maps manifest ratings to index.xml's ProductRating.
+var ratings = map[string]string{General: "GA", Adult: "AP"}
+
+func readLimited(path string, limit int64) ([]byte, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	data, err := io.ReadAll(io.LimitReader(f, limit+1))
+	if err != nil {
+		return nil, err
+	}
+	if int64(len(data)) > limit {
+		return nil, fmt.Errorf("%s: over %d bytes", filepath.Base(path), limit)
+	}
+	return data, nil
+}
+
+// collector checks files as they're added.
+type collector struct {
+	files []File
+	names map[string]bool
+	total int64
+}
+
+// add reads a file's data from r, at most the limits allow.
+func (c *collector) add(name string, r io.Reader) error {
+	if err := CheckName(name); err != nil {
+		return err
+	}
+	if c.names[name] {
+		return fmt.Errorf("%q: duplicate file", name)
+	}
+	if len(c.files) >= MaxFiles {
+		return fmt.Errorf("over %d files", MaxFiles)
+	}
+	data, err := io.ReadAll(io.LimitReader(r, MaxFileSize+1))
+	if err != nil {
+		return fmt.Errorf("%q: %w", name, err)
+	}
+	if len(data) > MaxFileSize {
+		return fmt.Errorf("%q: over %d MiB", name, MaxFileSize>>20)
+	}
+	c.total += int64(len(data))
+	if c.total > MaxTotalSize {
+		return fmt.Errorf("files over %d MiB in all", MaxTotalSize>>20)
+	}
+	if c.names == nil {
+		c.names = map[string]bool{}
+	}
+	c.names[name] = true
+	c.files = append(c.files, File{Name: name, Data: data})
+	return nil
+}
+
+func readChkn(path string) ([]File, error) {
+	z, err := zip.OpenReader(path)
+	if err != nil {
+		return nil, fmt.Errorf("%s: %w", ChknName, err)
+	}
+	defer z.Close()
+	var c collector
+	for _, f := range z.File {
+		if f.FileInfo().IsDir() {
+			continue
+		}
+		if strings.Contains(f.Name, "/") {
+			return nil, fmt.Errorf("%s: %q is in a folder; files must be at the top", ChknName, f.Name)
+		}
+		if err := addZipFile(&c, f); err != nil {
+			return nil, fmt.Errorf("%s: %w", ChknName, err)
+		}
+	}
+	return c.files, nil
+}
+
+func addZipFile(c *collector, f *zip.File) error {
+	rc, err := f.Open()
+	if err != nil {
+		return fmt.Errorf("%q: %w", f.Name, err)
+	}
+	defer rc.Close()
+	return c.add(f.Name, rc)
+}
+
+func readFilesDir(dir string) ([]File, error) {
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		return nil, err
+	}
+	var c collector
+	for _, e := range entries {
+		if e.IsDir() {
+			return nil, fmt.Errorf("%s/%s: folders aren't allowed in %s/", FilesDir, e.Name(), FilesDir)
+		}
+		if !e.Type().IsRegular() {
+			return nil, fmt.Errorf("%s/%s: not a regular file", FilesDir, e.Name())
+		}
+		if err := addDirFile(&c, filepath.Join(dir, e.Name()), e.Name()); err != nil {
+			return nil, fmt.Errorf("%s/: %w", FilesDir, err)
+		}
+	}
+	return c.files, nil
+}
+
+func addDirFile(c *collector, path, name string) error {
+	f, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	return c.add(name, f)
+}
+
+// CheckName checks a product file's name.
+func CheckName(name string) error {
+	switch {
+	case name == "":
+		return errors.New("empty file name")
+	case name == "." || name == "..":
+		return fmt.Errorf("%q isn't a file name", name)
+	case strings.ContainsAny(name, `/\`):
+		return fmt.Errorf("%q: file names can't have / or \\", name)
+	case !utf8.ValidString(name):
+		return fmt.Errorf("%q: file name isn't valid UTF-8", name)
+	case len(name) > MaxNameBytes:
+		return fmt.Errorf("%q: file name over %d bytes", name, MaxNameBytes)
+	case strings.EqualFold(name, "_contents.json"):
+		return fmt.Errorf("%q: packs can't have _contents.json", name)
+	}
+	for _, r := range name {
+		if r < 0x20 || (r >= 0x7f && r < 0xa0) {
+			return fmt.Errorf("%q: file name has control characters", name)
+		}
+		if r > 0xff {
+			return fmt.Errorf("%q: file name has characters outside Latin-1", name)
+		}
+	}
+	return nil
+}
+
+// Checksum returns a hex sha256 of the pack's files (names and contents), whatever their storage. It changes when
+// any file does.
+func (p *Pack) Checksum() string {
+	h := sha256.New()
+	h.Write([]byte("t5de-pack-files\n"))
+	var n [8]byte
+	for _, f := range p.Files {
+		binary.BigEndian.PutUint64(n[:], uint64(len(f.Name)))
+		h.Write(n[:])
+		h.Write([]byte(f.Name))
+		binary.BigEndian.PutUint64(n[:], uint64(len(f.Data)))
+		h.Write(n[:])
+		h.Write(f.Data)
+	}
+	return hex.EncodeToString(h.Sum(nil))
+}
+
+// CFL returns the pack's files as an uncompressed CFL, as the client downloads products.
+func (p *Pack) CFL() ([]byte, error) {
+	entries := make([]cfl.Entry, len(p.Files))
+	for i, f := range p.Files {
+		entries[i] = cfl.Entry{Name: f.Name, Data: f.Data}
+	}
+	var b bytes.Buffer
+	if err := cfl.Write(&b, entries); err != nil {
+		return nil, err
+	}
+	return b.Bytes(), nil
+}