t5de-api/pack/pack.go

// Package pack loads local product packs (format 1): a folder with t5de-product.json and the product's files,
// either zipped as product.chkn (IMVU Create Mode's project format) or in files/. See docs/pack.md.
package pack

import (
	"archive/zip"
	"bytes"
	"crypto/sha256"
	"encoding/binary"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"io/fs"
	"os"
	"path/filepath"
	"slices"
	"strings"
	"unicode/utf8"

	"git.t5de.dev/t5de/t5de-api/cfl"
)

// Names in a pack folder.
const (
	ManifestName = "t5de-product.json"
	ChknName     = "product.chkn"
	FilesDir     = "files"
	IndexName    = "index.xml"
)

// Limits.
const (
	MaxFiles     = 512
	MaxFileSize  = 64 << 20
	MaxTotalSize = 256 << 20
	MaxNameBytes = 200
	maxManifest  = 1 << 20
)

// Pack is a loaded product pack.
type Pack struct {
	Manifest Manifest
	Files    []File // sorted by name
	Index    Index  // from index.xml
}

// File is one of a product's files.
type File struct {
	Name string
	Data []byte
}

// Load reads the pack in dir and checks it.
func Load(dir string) (*Pack, error) {
	data, err := readLimited(filepath.Join(dir, ManifestName), maxManifest)
	if err != nil {
		return nil, err
	}
	m, err := ParseManifest(data)
	if err != nil {
		return nil, err
	}

	chkn := filepath.Join(dir, ChknName)
	filesDir := filepath.Join(dir, FilesDir)
	_, chknErr := os.Stat(chkn)
	_, dirErr := os.Stat(filesDir)
	var files []File
	switch {
	case chknErr == nil && dirErr == nil:
		return nil, fmt.Errorf("%s and %s/ both exist; keep one", ChknName, FilesDir)
	case chknErr == nil:
		files, err = readChkn(chkn)
	case dirErr == nil:
		files, err = readFilesDir(filesDir)
	case errors.Is(chknErr, fs.ErrNotExist) && errors.Is(dirErr, fs.ErrNotExist):
		return nil, fmt.Errorf("%s: no %s or %s/", dir, ChknName, FilesDir)
	case !errors.Is(chknErr, fs.ErrNotExist):
		return nil, chknErr
	default:
		return nil, dirErr
	}
	if err != nil {
		return nil, err
	}
	slices.SortFunc(files, func(a, b File) int { return strings.Compare(a.Name, b.Name) })

	p := &Pack{Manifest: *m, Files: files}
	i, ok := slices.BinarySearchFunc(files, IndexName, func(f File, name string) int { return strings.Compare(f.Name, name) })
	if !ok {
		return nil, fmt.Errorf("no %s", IndexName)
	}
	index, err := ParseIndex(files[i].Data)
	if err != nil {
		return nil, fmt.Errorf("%s: %w", IndexName, err)
	}
	p.Index = *index
	if want := ratings[m.Rating]; index.Rating != want {
		return nil, fmt.Errorf("%s: ProductRating is %q, but %s says %q (%s)", IndexName, index.Rating, ManifestName, m.Rating, want)
	}
	return p, nil
}

// ratings maps manifest ratings to index.xml's ProductRating.
var ratings = map[string]string{General: "GA", Adult: "AP"}

func readLimited(path string, limit int64) ([]byte, error) {
	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	data, err := io.ReadAll(io.LimitReader(f, limit+1))
	if err != nil {
		return nil, err
	}
	if int64(len(data)) > limit {
		return nil, fmt.Errorf("%s: over %d bytes", filepath.Base(path), limit)
	}
	return data, nil
}

// collector checks files as they're added.
type collector struct {
	files []File
	names map[string]bool
	total int64
}

// add reads a file's data from r, at most the limits allow.
func (c *collector) add(name string, r io.Reader) error {
	if err := CheckName(name); err != nil {
		return err
	}
	if c.names[name] {
		return fmt.Errorf("%q: duplicate file", name)
	}
	if len(c.files) >= MaxFiles {
		return fmt.Errorf("over %d files", MaxFiles)
	}
	data, err := io.ReadAll(io.LimitReader(r, MaxFileSize+1))
	if err != nil {
		return fmt.Errorf("%q: %w", name, err)
	}
	if len(data) > MaxFileSize {
		return fmt.Errorf("%q: over %d MiB", name, MaxFileSize>>20)
	}
	c.total += int64(len(data))
	if c.total > MaxTotalSize {
		return fmt.Errorf("files over %d MiB in all", MaxTotalSize>>20)
	}
	if c.names == nil {
		c.names = map[string]bool{}
	}
	c.names[name] = true
	c.files = append(c.files, File{Name: name, Data: data})
	return nil
}

func readChkn(path string) ([]File, error) {
	z, err := zip.OpenReader(path)
	if err != nil {
		return nil, fmt.Errorf("%s: %w", ChknName, err)
	}
	defer z.Close()
	var c collector
	for _, f := range z.File {
		if f.FileInfo().IsDir() {
			continue
		}
		if strings.Contains(f.Name, "/") {
			return nil, fmt.Errorf("%s: %q is in a folder; files must be at the top", ChknName, f.Name)
		}
		if err := addZipFile(&c, f); err != nil {
			return nil, fmt.Errorf("%s: %w", ChknName, err)
		}
	}
	return c.files, nil
}

func addZipFile(c *collector, f *zip.File) error {
	rc, err := f.Open()
	if err != nil {
		return fmt.Errorf("%q: %w", f.Name, err)
	}
	defer rc.Close()
	return c.add(f.Name, rc)
}

func readFilesDir(dir string) ([]File, error) {
	entries, err := os.ReadDir(dir)
	if err != nil {
		return nil, err
	}
	var c collector
	for _, e := range entries {
		if e.IsDir() {
			return nil, fmt.Errorf("%s/%s: folders aren't allowed in %s/", FilesDir, e.Name(), FilesDir)
		}
		if !e.Type().IsRegular() {
			return nil, fmt.Errorf("%s/%s: not a regular file", FilesDir, e.Name())
		}
		if err := addDirFile(&c, filepath.Join(dir, e.Name()), e.Name()); err != nil {
			return nil, fmt.Errorf("%s/: %w", FilesDir, err)
		}
	}
	return c.files, nil
}

func addDirFile(c *collector, path, name string) error {
	f, err := os.Open(path)
	if err != nil {
		return err
	}
	defer f.Close()
	return c.add(name, f)
}

// CheckName checks a product file's name.
func CheckName(name string) error {
	switch {
	case name == "":
		return errors.New("empty file name")
	case name == "." || name == "..":
		return fmt.Errorf("%q isn't a file name", name)
	case strings.ContainsAny(name, `/\`):
		return fmt.Errorf("%q: file names can't have / or \\", name)
	case !utf8.ValidString(name):
		return fmt.Errorf("%q: file name isn't valid UTF-8", name)
	case len(name) > MaxNameBytes:
		return fmt.Errorf("%q: file name over %d bytes", name, MaxNameBytes)
	case strings.EqualFold(name, "_contents.json"):
		return fmt.Errorf("%q: packs can't have _contents.json", name)
	}
	for _, r := range name {
		if r < 0x20 || (r >= 0x7f && r < 0xa0) {
			return fmt.Errorf("%q: file name has control characters", name)
		}
		if r > 0xff {
			return fmt.Errorf("%q: file name has characters outside Latin-1", name)
		}
	}
	return nil
}

// Checksum returns a hex sha256 of the pack's files (names and contents), whatever their storage. It changes when
// any file does.
func (p *Pack) Checksum() string {
	h := sha256.New()
	h.Write([]byte("t5de-pack-files\n"))
	var n [8]byte
	for _, f := range p.Files {
		binary.BigEndian.PutUint64(n[:], uint64(len(f.Name)))
		h.Write(n[:])
		h.Write([]byte(f.Name))
		binary.BigEndian.PutUint64(n[:], uint64(len(f.Data)))
		h.Write(n[:])
		h.Write(f.Data)
	}
	return hex.EncodeToString(h.Sum(nil))
}

// CFL returns the pack's files as an uncompressed CFL, as the client downloads products.
func (p *Pack) CFL() ([]byte, error) {
	entries := make([]cfl.Entry, len(p.Files))
	for i, f := range p.Files {
		entries[i] = cfl.Entry{Name: f.Name, Data: f.Data}
	}
	var b bytes.Buffer
	if err := cfl.Write(&b, entries); err != nil {
		return nil, err
	}
	return b.Bytes(), nil
}