// Package pack loads local product packs (format 1): a folder with t5de-product.json and the product's files, // either zipped as product.chkn (IMVU Create Mode's project format) or in files/. See docs/pack.md. package pack import ( "archive/zip" "bytes" "crypto/sha256" "encoding/binary" "encoding/hex" "errors" "fmt" "io" "io/fs" "os" "path/filepath" "slices" "strings" "unicode/utf8" "git.t5de.dev/t5de/t5de-api/cfl" ) // Names in a pack folder. const ( ManifestName = "t5de-product.json" ChknName = "product.chkn" FilesDir = "files" IndexName = "index.xml" ) // Limits. const ( MaxFiles = 512 MaxFileSize = 64 << 20 MaxTotalSize = 256 << 20 MaxNameBytes = 200 maxManifest = 1 << 20 ) // Pack is a loaded product pack. type Pack struct { Manifest Manifest Files []File // sorted by name Index Index // from index.xml } // File is one of a product's files. type File struct { Name string Data []byte } // Load reads the pack in dir and checks it. func Load(dir string) (*Pack, error) { data, err := readLimited(filepath.Join(dir, ManifestName), maxManifest) if err != nil { return nil, err } m, err := ParseManifest(data) if err != nil { return nil, err } chkn := filepath.Join(dir, ChknName) filesDir := filepath.Join(dir, FilesDir) _, chknErr := os.Stat(chkn) _, dirErr := os.Stat(filesDir) var files []File switch { case chknErr == nil && dirErr == nil: return nil, fmt.Errorf("%s and %s/ both exist; keep one", ChknName, FilesDir) case chknErr == nil: files, err = readChkn(chkn) case dirErr == nil: files, err = readFilesDir(filesDir) case errors.Is(chknErr, fs.ErrNotExist) && errors.Is(dirErr, fs.ErrNotExist): return nil, fmt.Errorf("%s: no %s or %s/", dir, ChknName, FilesDir) case !errors.Is(chknErr, fs.ErrNotExist): return nil, chknErr default: return nil, dirErr } if err != nil { return nil, err } slices.SortFunc(files, func(a, b File) int { return strings.Compare(a.Name, b.Name) }) p := &Pack{Manifest: *m, Files: files} i, ok := slices.BinarySearchFunc(files, IndexName, func(f File, name string) int { return strings.Compare(f.Name, name) }) if !ok { return nil, fmt.Errorf("no %s", IndexName) } index, err := ParseIndex(files[i].Data) if err != nil { return nil, fmt.Errorf("%s: %w", IndexName, err) } p.Index = *index if want := ratings[m.Rating]; index.Rating != want { return nil, fmt.Errorf("%s: ProductRating is %q, but %s says %q (%s)", IndexName, index.Rating, ManifestName, m.Rating, want) } return p, nil } // ratings maps manifest ratings to index.xml's ProductRating. var ratings = map[string]string{General: "GA", Adult: "AP"} func readLimited(path string, limit int64) ([]byte, error) { f, err := os.Open(path) if err != nil { return nil, err } defer f.Close() data, err := io.ReadAll(io.LimitReader(f, limit+1)) if err != nil { return nil, err } if int64(len(data)) > limit { return nil, fmt.Errorf("%s: over %d bytes", filepath.Base(path), limit) } return data, nil } // collector checks files as they're added. type collector struct { files []File names map[string]bool total int64 } // add reads a file's data from r, at most the limits allow. func (c *collector) add(name string, r io.Reader) error { if err := CheckName(name); err != nil { return err } if c.names[name] { return fmt.Errorf("%q: duplicate file", name) } if len(c.files) >= MaxFiles { return fmt.Errorf("over %d files", MaxFiles) } data, err := io.ReadAll(io.LimitReader(r, MaxFileSize+1)) if err != nil { return fmt.Errorf("%q: %w", name, err) } if len(data) > MaxFileSize { return fmt.Errorf("%q: over %d MiB", name, MaxFileSize>>20) } c.total += int64(len(data)) if c.total > MaxTotalSize { return fmt.Errorf("files over %d MiB in all", MaxTotalSize>>20) } if c.names == nil { c.names = map[string]bool{} } c.names[name] = true c.files = append(c.files, File{Name: name, Data: data}) return nil } func readChkn(path string) ([]File, error) { z, err := zip.OpenReader(path) if err != nil { return nil, fmt.Errorf("%s: %w", ChknName, err) } defer z.Close() var c collector for _, f := range z.File { if f.FileInfo().IsDir() { continue } if strings.Contains(f.Name, "/") { return nil, fmt.Errorf("%s: %q is in a folder; files must be at the top", ChknName, f.Name) } if err := addZipFile(&c, f); err != nil { return nil, fmt.Errorf("%s: %w", ChknName, err) } } return c.files, nil } func addZipFile(c *collector, f *zip.File) error { rc, err := f.Open() if err != nil { return fmt.Errorf("%q: %w", f.Name, err) } defer rc.Close() return c.add(f.Name, rc) } func readFilesDir(dir string) ([]File, error) { entries, err := os.ReadDir(dir) if err != nil { return nil, err } var c collector for _, e := range entries { if e.IsDir() { return nil, fmt.Errorf("%s/%s: folders aren't allowed in %s/", FilesDir, e.Name(), FilesDir) } if !e.Type().IsRegular() { return nil, fmt.Errorf("%s/%s: not a regular file", FilesDir, e.Name()) } if err := addDirFile(&c, filepath.Join(dir, e.Name()), e.Name()); err != nil { return nil, fmt.Errorf("%s/: %w", FilesDir, err) } } return c.files, nil } func addDirFile(c *collector, path, name string) error { f, err := os.Open(path) if err != nil { return err } defer f.Close() return c.add(name, f) } // CheckName checks a product file's name. func CheckName(name string) error { switch { case name == "": return errors.New("empty file name") case name == "." || name == "..": return fmt.Errorf("%q isn't a file name", name) case strings.ContainsAny(name, `/\`): return fmt.Errorf("%q: file names can't have / or \\", name) case !utf8.ValidString(name): return fmt.Errorf("%q: file name isn't valid UTF-8", name) case len(name) > MaxNameBytes: return fmt.Errorf("%q: file name over %d bytes", name, MaxNameBytes) case strings.EqualFold(name, "_contents.json"): return fmt.Errorf("%q: packs can't have _contents.json", name) } for _, r := range name { if r < 0x20 || (r >= 0x7f && r < 0xa0) { return fmt.Errorf("%q: file name has control characters", name) } if r > 0xff { return fmt.Errorf("%q: file name has characters outside Latin-1", name) } } return nil } // Checksum returns a hex sha256 of the pack's files (names and contents), whatever their storage. It changes when // any file does. func (p *Pack) Checksum() string { h := sha256.New() h.Write([]byte("t5de-pack-files\n")) var n [8]byte for _, f := range p.Files { binary.BigEndian.PutUint64(n[:], uint64(len(f.Name))) h.Write(n[:]) h.Write([]byte(f.Name)) binary.BigEndian.PutUint64(n[:], uint64(len(f.Data))) h.Write(n[:]) h.Write(f.Data) } return hex.EncodeToString(h.Sum(nil)) } // CFL returns the pack's files as an uncompressed CFL, as the client downloads products. func (p *Pack) CFL() ([]byte, error) { entries := make([]cfl.Entry, len(p.Files)) for i, f := range p.Files { entries[i] = cfl.Entry{Name: f.Name, Data: f.Data} } var b bytes.Buffer if err := cfl.Write(&b, entries); err != nil { return nil, err } return b.Bytes(), nil }