Add pack, cfl and safehtml
pack loads local product packs (format 1): t5de-product.json and the product's files, as product.chkn or files/, with index.xml's rating, type, gender and parent product. cfl writes and reads IMVU's product container, uncompressed. safehtml cleans product descriptions by an allowlist (bluemonday). Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
dhkatzandClaude Opus 5.5 committed
commit
9bdc3ff455
16 files changed
+1638
No files matched your search
+137
@@ -0,0 +1,137 @@
|
||||
package pack
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Index is what T5DE reads from a product's index.xml, IMVU's product template. Other elements are ignored.
|
||||
type Index struct {
|
||||
Rating string // ProductRating: "GA" (general) or "AP" (adult)
|
||||
Name string // ProductName
|
||||
Gender string // ProductGender
|
||||
TypeID int64 // ProductTypeID, 0 if absent
|
||||
BodyPattern int64 // BodyPatternProductId, 0 if absent
|
||||
Parent int64 // the product it derives from (__DATAIMPORT product://<id>/index.xml), 0 if none
|
||||
}
|
||||
|
||||
// ParseIndex parses index.xml.
|
||||
func ParseIndex(data []byte) (*Index, error) {
|
||||
d := xml.NewDecoder(bytes.NewReader(data))
|
||||
d.Strict = false
|
||||
d.Entity = xml.HTMLEntity
|
||||
d.CharsetReader = charsetReader
|
||||
|
||||
values := map[string]string{}
|
||||
depth := 0
|
||||
root := false
|
||||
var field string // the top-level element being read, if wanted
|
||||
var text strings.Builder
|
||||
for {
|
||||
tok, err := d.Token()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch t := tok.(type) {
|
||||
case xml.StartElement:
|
||||
depth++
|
||||
if depth == 1 && t.Name.Local != "Template" {
|
||||
return nil, fmt.Errorf("root element is <%s>, not <Template>", t.Name.Local)
|
||||
}
|
||||
root = true
|
||||
if depth == 2 && wanted[t.Name.Local] {
|
||||
if _, ok := values[t.Name.Local]; ok {
|
||||
return nil, fmt.Errorf("<%s> appears twice", t.Name.Local)
|
||||
}
|
||||
field = t.Name.Local
|
||||
text.Reset()
|
||||
}
|
||||
case xml.EndElement:
|
||||
if depth == 2 && field != "" {
|
||||
values[field] = strings.TrimSpace(text.String())
|
||||
field = ""
|
||||
}
|
||||
depth--
|
||||
case xml.CharData:
|
||||
if field != "" {
|
||||
text.Write(t)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !root {
|
||||
return nil, errors.New("no <Template>")
|
||||
}
|
||||
|
||||
var x Index
|
||||
x.Rating = values["ProductRating"]
|
||||
x.Name = values["ProductName"]
|
||||
x.Gender = values["ProductGender"]
|
||||
if x.Rating != "GA" && x.Rating != "AP" {
|
||||
return nil, fmt.Errorf("ProductRating: %q must be GA or AP", x.Rating)
|
||||
}
|
||||
var err error
|
||||
if x.TypeID, err = id(values, "ProductTypeID"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if x.BodyPattern, err = id(values, "BodyPatternProductId"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if imp := values["__DATAIMPORT"]; imp != "" {
|
||||
s, ok := strings.CutPrefix(imp, "product://")
|
||||
if ok {
|
||||
s, ok = strings.CutSuffix(s, "/index.xml")
|
||||
}
|
||||
n, err := strconv.ParseInt(s, 10, 64)
|
||||
if !ok || err != nil || n < 1 {
|
||||
return nil, fmt.Errorf("__DATAIMPORT: %q isn't product://<id>/index.xml", imp)
|
||||
}
|
||||
x.Parent = n
|
||||
}
|
||||
return &x, nil
|
||||
}
|
||||
|
||||
var wanted = map[string]bool{
|
||||
"ProductRating": true,
|
||||
"ProductName": true,
|
||||
"ProductGender": true,
|
||||
"ProductTypeID": true,
|
||||
"BodyPatternProductId": true,
|
||||
"__DATAIMPORT": true,
|
||||
}
|
||||
|
||||
func id(values map[string]string, name string) (int64, error) {
|
||||
s := values[name]
|
||||
if s == "" {
|
||||
return 0, nil
|
||||
}
|
||||
n, err := strconv.ParseInt(s, 10, 64)
|
||||
if err != nil || n < 0 {
|
||||
return 0, fmt.Errorf("%s: %q isn't a number", name, s)
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// charsetReader reads Latin-1 (and windows-1252, approximately) as well as UTF-8.
|
||||
func charsetReader(charset string, r io.Reader) (io.Reader, error) {
|
||||
switch strings.ToLower(charset) {
|
||||
case "iso-8859-1", "latin1", "latin-1", "windows-1252", "cp1252", "us-ascii", "ascii":
|
||||
data, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var b strings.Builder
|
||||
for _, c := range data {
|
||||
b.WriteRune(rune(c))
|
||||
}
|
||||
return strings.NewReader(b.String()), nil
|
||||
}
|
||||
return nil, fmt.Errorf("unsupported encoding %q", charset)
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
package pack
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// Format is the pack format this package reads.
|
||||
const Format = 1
|
||||
|
||||
// Ratings.
|
||||
const (
|
||||
General = "general"
|
||||
Adult = "adult"
|
||||
)
|
||||
|
||||
// Manifest limits.
|
||||
const (
|
||||
MaxIDLength = 64
|
||||
MaxNameLength = 64 // characters
|
||||
MaxCreatorLength = 64 // characters
|
||||
MaxDescriptionLength = 20000
|
||||
)
|
||||
|
||||
// Manifest is t5de-product.json.
|
||||
type Manifest struct {
|
||||
Format int `json:"format"`
|
||||
ID string `json:"id"`
|
||||
Revision int `json:"revision"`
|
||||
Name string `json:"name"`
|
||||
Creator string `json:"creator,omitempty"`
|
||||
Description string `json:"description,omitempty"` // HTML, cleaned by package safehtml when shown
|
||||
Rating string `json:"rating"`
|
||||
}
|
||||
|
||||
// ParseManifest parses and checks t5de-product.json. Name and Creator come back trimmed.
|
||||
func ParseManifest(data []byte) (*Manifest, error) {
|
||||
m, err := parseManifest(data)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", ManifestName, err)
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func parseManifest(data []byte) (*Manifest, error) {
|
||||
d := json.NewDecoder(bytes.NewReader(data))
|
||||
d.DisallowUnknownFields()
|
||||
var m Manifest
|
||||
if err := d.Decode(&m); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := d.Token(); err != io.EOF {
|
||||
return nil, errors.New("data after the object")
|
||||
}
|
||||
m.Name = strings.TrimSpace(m.Name)
|
||||
m.Creator = strings.TrimSpace(m.Creator)
|
||||
switch {
|
||||
case m.Format != Format:
|
||||
return nil, fmt.Errorf("format: is %d, must be %d", m.Format, Format)
|
||||
case !validID(m.ID):
|
||||
return nil, fmt.Errorf("id: %q must be 1-%d of a-z, 0-9 and -, not starting or ending with -", m.ID, MaxIDLength)
|
||||
case m.Revision < 1:
|
||||
return nil, fmt.Errorf("revision: is %d, must be at least 1", m.Revision)
|
||||
case m.Name == "":
|
||||
return nil, errors.New("name: missing")
|
||||
case utf8.RuneCountInString(m.Name) > MaxNameLength:
|
||||
return nil, fmt.Errorf("name: over %d characters", MaxNameLength)
|
||||
case hasControl(m.Name):
|
||||
return nil, errors.New("name: has control characters")
|
||||
case utf8.RuneCountInString(m.Creator) > MaxCreatorLength:
|
||||
return nil, fmt.Errorf("creator: over %d characters", MaxCreatorLength)
|
||||
case hasControl(m.Creator):
|
||||
return nil, errors.New("creator: has control characters")
|
||||
case len(m.Description) > MaxDescriptionLength:
|
||||
return nil, fmt.Errorf("description: over %d bytes", MaxDescriptionLength)
|
||||
case m.Rating != General && m.Rating != Adult:
|
||||
return nil, fmt.Errorf("rating: %q must be %q or %q", m.Rating, General, Adult)
|
||||
}
|
||||
return &m, nil
|
||||
}
|
||||
|
||||
func validID(id string) bool {
|
||||
if id == "" || len(id) > MaxIDLength || id[0] == '-' || id[len(id)-1] == '-' {
|
||||
return false
|
||||
}
|
||||
for _, c := range []byte(id) {
|
||||
if !(c >= 'a' && c <= 'z' || c >= '0' && c <= '9' || c == '-') {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func hasControl(s string) bool {
|
||||
return strings.ContainsFunc(s, unicode.IsControl)
|
||||
}
|
||||
+275
@@ -0,0 +1,275 @@
|
||||
// Package pack loads local product packs (format 1): a folder with t5de-product.json and the product's files,
|
||||
// either zipped as product.chkn (IMVU Create Mode's project format) or in files/. See docs/pack.md.
|
||||
package pack
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"git.t5de.dev/t5de/t5de-api/cfl"
|
||||
)
|
||||
|
||||
// Names in a pack folder.
|
||||
const (
|
||||
ManifestName = "t5de-product.json"
|
||||
ChknName = "product.chkn"
|
||||
FilesDir = "files"
|
||||
IndexName = "index.xml"
|
||||
)
|
||||
|
||||
// Limits.
|
||||
const (
|
||||
MaxFiles = 512
|
||||
MaxFileSize = 64 << 20
|
||||
MaxTotalSize = 256 << 20
|
||||
MaxNameBytes = 200
|
||||
maxManifest = 1 << 20
|
||||
)
|
||||
|
||||
// Pack is a loaded product pack.
|
||||
type Pack struct {
|
||||
Manifest Manifest
|
||||
Files []File // sorted by name
|
||||
Index Index // from index.xml
|
||||
}
|
||||
|
||||
// File is one of a product's files.
|
||||
type File struct {
|
||||
Name string
|
||||
Data []byte
|
||||
}
|
||||
|
||||
// Load reads the pack in dir and checks it.
|
||||
func Load(dir string) (*Pack, error) {
|
||||
data, err := readLimited(filepath.Join(dir, ManifestName), maxManifest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m, err := ParseManifest(data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
chkn := filepath.Join(dir, ChknName)
|
||||
filesDir := filepath.Join(dir, FilesDir)
|
||||
_, chknErr := os.Stat(chkn)
|
||||
_, dirErr := os.Stat(filesDir)
|
||||
var files []File
|
||||
switch {
|
||||
case chknErr == nil && dirErr == nil:
|
||||
return nil, fmt.Errorf("%s and %s/ both exist; keep one", ChknName, FilesDir)
|
||||
case chknErr == nil:
|
||||
files, err = readChkn(chkn)
|
||||
case dirErr == nil:
|
||||
files, err = readFilesDir(filesDir)
|
||||
case errors.Is(chknErr, fs.ErrNotExist) && errors.Is(dirErr, fs.ErrNotExist):
|
||||
return nil, fmt.Errorf("%s: no %s or %s/", dir, ChknName, FilesDir)
|
||||
case !errors.Is(chknErr, fs.ErrNotExist):
|
||||
return nil, chknErr
|
||||
default:
|
||||
return nil, dirErr
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
slices.SortFunc(files, func(a, b File) int { return strings.Compare(a.Name, b.Name) })
|
||||
|
||||
p := &Pack{Manifest: *m, Files: files}
|
||||
i, ok := slices.BinarySearchFunc(files, IndexName, func(f File, name string) int { return strings.Compare(f.Name, name) })
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("no %s", IndexName)
|
||||
}
|
||||
index, err := ParseIndex(files[i].Data)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", IndexName, err)
|
||||
}
|
||||
p.Index = *index
|
||||
if want := ratings[m.Rating]; index.Rating != want {
|
||||
return nil, fmt.Errorf("%s: ProductRating is %q, but %s says %q (%s)", IndexName, index.Rating, ManifestName, m.Rating, want)
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// ratings maps manifest ratings to index.xml's ProductRating.
|
||||
var ratings = map[string]string{General: "GA", Adult: "AP"}
|
||||
|
||||
func readLimited(path string, limit int64) ([]byte, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
data, err := io.ReadAll(io.LimitReader(f, limit+1))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if int64(len(data)) > limit {
|
||||
return nil, fmt.Errorf("%s: over %d bytes", filepath.Base(path), limit)
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// collector checks files as they're added.
|
||||
type collector struct {
|
||||
files []File
|
||||
names map[string]bool
|
||||
total int64
|
||||
}
|
||||
|
||||
// add reads a file's data from r, at most the limits allow.
|
||||
func (c *collector) add(name string, r io.Reader) error {
|
||||
if err := CheckName(name); err != nil {
|
||||
return err
|
||||
}
|
||||
if c.names[name] {
|
||||
return fmt.Errorf("%q: duplicate file", name)
|
||||
}
|
||||
if len(c.files) >= MaxFiles {
|
||||
return fmt.Errorf("over %d files", MaxFiles)
|
||||
}
|
||||
data, err := io.ReadAll(io.LimitReader(r, MaxFileSize+1))
|
||||
if err != nil {
|
||||
return fmt.Errorf("%q: %w", name, err)
|
||||
}
|
||||
if len(data) > MaxFileSize {
|
||||
return fmt.Errorf("%q: over %d MiB", name, MaxFileSize>>20)
|
||||
}
|
||||
c.total += int64(len(data))
|
||||
if c.total > MaxTotalSize {
|
||||
return fmt.Errorf("files over %d MiB in all", MaxTotalSize>>20)
|
||||
}
|
||||
if c.names == nil {
|
||||
c.names = map[string]bool{}
|
||||
}
|
||||
c.names[name] = true
|
||||
c.files = append(c.files, File{Name: name, Data: data})
|
||||
return nil
|
||||
}
|
||||
|
||||
func readChkn(path string) ([]File, error) {
|
||||
z, err := zip.OpenReader(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", ChknName, err)
|
||||
}
|
||||
defer z.Close()
|
||||
var c collector
|
||||
for _, f := range z.File {
|
||||
if f.FileInfo().IsDir() {
|
||||
continue
|
||||
}
|
||||
if strings.Contains(f.Name, "/") {
|
||||
return nil, fmt.Errorf("%s: %q is in a folder; files must be at the top", ChknName, f.Name)
|
||||
}
|
||||
if err := addZipFile(&c, f); err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", ChknName, err)
|
||||
}
|
||||
}
|
||||
return c.files, nil
|
||||
}
|
||||
|
||||
func addZipFile(c *collector, f *zip.File) error {
|
||||
rc, err := f.Open()
|
||||
if err != nil {
|
||||
return fmt.Errorf("%q: %w", f.Name, err)
|
||||
}
|
||||
defer rc.Close()
|
||||
return c.add(f.Name, rc)
|
||||
}
|
||||
|
||||
func readFilesDir(dir string) ([]File, error) {
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var c collector
|
||||
for _, e := range entries {
|
||||
if e.IsDir() {
|
||||
return nil, fmt.Errorf("%s/%s: folders aren't allowed in %s/", FilesDir, e.Name(), FilesDir)
|
||||
}
|
||||
if !e.Type().IsRegular() {
|
||||
return nil, fmt.Errorf("%s/%s: not a regular file", FilesDir, e.Name())
|
||||
}
|
||||
if err := addDirFile(&c, filepath.Join(dir, e.Name()), e.Name()); err != nil {
|
||||
return nil, fmt.Errorf("%s/: %w", FilesDir, err)
|
||||
}
|
||||
}
|
||||
return c.files, nil
|
||||
}
|
||||
|
||||
func addDirFile(c *collector, path, name string) error {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
return c.add(name, f)
|
||||
}
|
||||
|
||||
// CheckName checks a product file's name.
|
||||
func CheckName(name string) error {
|
||||
switch {
|
||||
case name == "":
|
||||
return errors.New("empty file name")
|
||||
case name == "." || name == "..":
|
||||
return fmt.Errorf("%q isn't a file name", name)
|
||||
case strings.ContainsAny(name, `/\`):
|
||||
return fmt.Errorf("%q: file names can't have / or \\", name)
|
||||
case !utf8.ValidString(name):
|
||||
return fmt.Errorf("%q: file name isn't valid UTF-8", name)
|
||||
case len(name) > MaxNameBytes:
|
||||
return fmt.Errorf("%q: file name over %d bytes", name, MaxNameBytes)
|
||||
case strings.EqualFold(name, "_contents.json"):
|
||||
return fmt.Errorf("%q: packs can't have _contents.json", name)
|
||||
}
|
||||
for _, r := range name {
|
||||
if r < 0x20 || (r >= 0x7f && r < 0xa0) {
|
||||
return fmt.Errorf("%q: file name has control characters", name)
|
||||
}
|
||||
if r > 0xff {
|
||||
return fmt.Errorf("%q: file name has characters outside Latin-1", name)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Checksum returns a hex sha256 of the pack's files (names and contents), whatever their storage. It changes when
|
||||
// any file does.
|
||||
func (p *Pack) Checksum() string {
|
||||
h := sha256.New()
|
||||
h.Write([]byte("t5de-pack-files\n"))
|
||||
var n [8]byte
|
||||
for _, f := range p.Files {
|
||||
binary.BigEndian.PutUint64(n[:], uint64(len(f.Name)))
|
||||
h.Write(n[:])
|
||||
h.Write([]byte(f.Name))
|
||||
binary.BigEndian.PutUint64(n[:], uint64(len(f.Data)))
|
||||
h.Write(n[:])
|
||||
h.Write(f.Data)
|
||||
}
|
||||
return hex.EncodeToString(h.Sum(nil))
|
||||
}
|
||||
|
||||
// CFL returns the pack's files as an uncompressed CFL, as the client downloads products.
|
||||
func (p *Pack) CFL() ([]byte, error) {
|
||||
entries := make([]cfl.Entry, len(p.Files))
|
||||
for i, f := range p.Files {
|
||||
entries[i] = cfl.Entry{Name: f.Name, Data: f.Data}
|
||||
}
|
||||
var b bytes.Buffer
|
||||
if err := cfl.Write(&b, entries); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return b.Bytes(), nil
|
||||
}
|
||||
@@ -0,0 +1,315 @@
|
||||
package pack
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.t5de.dev/t5de/t5de-api/cfl"
|
||||
)
|
||||
|
||||
// Synthetic test files: not IMVU's.
|
||||
const (
|
||||
testManifest = `{"format": 1, "id": "blue-coat", "revision": 2, "name": " Blue Coat ", "creator": "Tester",
|
||||
"description": "<p>A coat</p>", "rating": "general"}`
|
||||
testIndex = `<?xml version="1.0" encoding="UTF-8" ?><Template><ProductRating>GA</ProductRating>` +
|
||||
`<ProductTypeID>106</ProductTypeID><BodyPatternProductId>80</BodyPatternProductId>` +
|
||||
`<ProductGender>female</ProductGender><ParentCategoryIDs>1,2</ParentCategoryIDs>` +
|
||||
`<__DATAIMPORT>product://12345/index.xml</__DATAIMPORT><ProductName>Test Coat</ProductName>` +
|
||||
`<Other><ProductName>nested, ignored</ProductName></Other></Template>`
|
||||
)
|
||||
|
||||
var testPNG = []byte("\x89PNG\r\n\x1a\nfake")
|
||||
|
||||
func testFiles() map[string][]byte {
|
||||
return map[string][]byte{"index.xml": []byte(testIndex), "coat.png": testPNG, "coat.xmf": []byte("mesh")}
|
||||
}
|
||||
|
||||
func writeFile(t *testing.T, path string, data []byte) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, data, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// dirPack makes a pack with its files in files/.
|
||||
func dirPack(t *testing.T, manifest string, files map[string][]byte) string {
|
||||
dir := t.TempDir()
|
||||
writeFile(t, filepath.Join(dir, ManifestName), []byte(manifest))
|
||||
if err := os.Mkdir(filepath.Join(dir, FilesDir), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for name, data := range files {
|
||||
writeFile(t, filepath.Join(dir, FilesDir, name), data)
|
||||
}
|
||||
return dir
|
||||
}
|
||||
|
||||
type zipEntry struct {
|
||||
name string
|
||||
data []byte
|
||||
method uint16
|
||||
}
|
||||
|
||||
func makeZip(t *testing.T, entries []zipEntry) []byte {
|
||||
t.Helper()
|
||||
var b bytes.Buffer
|
||||
z := zip.NewWriter(&b)
|
||||
for _, e := range entries {
|
||||
w, err := z.CreateHeader(&zip.FileHeader{Name: e.name, Method: e.method})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w.Write(e.data)
|
||||
}
|
||||
if err := z.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b.Bytes()
|
||||
}
|
||||
|
||||
// chknPack makes a pack with its files in product.chkn.
|
||||
func chknPack(t *testing.T, manifest string, entries []zipEntry) string {
|
||||
dir := t.TempDir()
|
||||
writeFile(t, filepath.Join(dir, ManifestName), []byte(manifest))
|
||||
writeFile(t, filepath.Join(dir, ChknName), makeZip(t, entries))
|
||||
return dir
|
||||
}
|
||||
|
||||
func zipEntries(method uint16) []zipEntry {
|
||||
var entries []zipEntry
|
||||
for name, data := range testFiles() {
|
||||
entries = append(entries, zipEntry{name, data, method})
|
||||
}
|
||||
return entries
|
||||
}
|
||||
|
||||
func TestLoad(t *testing.T) {
|
||||
fromDir, err := Load(dirPack(t, testManifest, testFiles()))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := fromDir.Manifest
|
||||
if m.ID != "blue-coat" || m.Revision != 2 || m.Name != "Blue Coat" || m.Creator != "Tester" || m.Rating != General {
|
||||
t.Errorf("manifest: %+v", m)
|
||||
}
|
||||
want := Index{Rating: "GA", Name: "Test Coat", Gender: "female", TypeID: 106, BodyPattern: 80, Parent: 12345}
|
||||
if fromDir.Index != want {
|
||||
t.Errorf("index: got %+v, want %+v", fromDir.Index, want)
|
||||
}
|
||||
var names []string
|
||||
for _, f := range fromDir.Files {
|
||||
names = append(names, f.Name)
|
||||
}
|
||||
if got := strings.Join(names, " "); got != "coat.png coat.xmf index.xml" {
|
||||
t.Errorf("files: %s", got)
|
||||
}
|
||||
|
||||
// The same files zipped, stored or deflated, have the same checksum. Folder entries are ignored.
|
||||
for _, method := range []uint16{zip.Store, zip.Deflate} {
|
||||
entries := append(zipEntries(method), zipEntry{name: "folder/", method: method})
|
||||
fromZip, err := Load(chknPack(t, testManifest, entries))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if fromZip.Checksum() != fromDir.Checksum() {
|
||||
t.Errorf("method %d: checksum differs", method)
|
||||
}
|
||||
}
|
||||
|
||||
changed := testFiles()
|
||||
changed["coat.png"] = append(testPNG, 0)
|
||||
other, err := Load(dirPack(t, testManifest, changed))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if other.Checksum() == fromDir.Checksum() {
|
||||
t.Error("checksum didn't change")
|
||||
}
|
||||
|
||||
data, err := fromDir.CFL()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
entries, err := cfl.Read(data)
|
||||
if err != nil || len(entries) != 3 || entries[2].Name != "index.xml" {
|
||||
t.Errorf("CFL: %q, %v", entries, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadErrors(t *testing.T) {
|
||||
adult := strings.Replace(testManifest, `"general"`, `"adult"`, 1)
|
||||
with := func(name string, data []byte) map[string][]byte {
|
||||
f := testFiles()
|
||||
f[name] = data
|
||||
return f
|
||||
}
|
||||
without := func(name string) map[string][]byte {
|
||||
f := testFiles()
|
||||
delete(f, name)
|
||||
return f
|
||||
}
|
||||
extra := func(e ...zipEntry) []zipEntry { return append(zipEntries(zip.Store), e...) }
|
||||
|
||||
for _, c := range []struct {
|
||||
name, want string
|
||||
dir func(t *testing.T) string
|
||||
}{
|
||||
{"rating mismatch", "ProductRating", func(t *testing.T) string { return dirPack(t, adult, testFiles()) }},
|
||||
{"no index", "no index.xml", func(t *testing.T) string { return dirPack(t, testManifest, without("index.xml")) }},
|
||||
{"contents json", "_contents.json", func(t *testing.T) string {
|
||||
return dirPack(t, testManifest, with("_contents.json", []byte("{}")))
|
||||
}},
|
||||
{"bad index", "index.xml: ProductRating", func(t *testing.T) string {
|
||||
return dirPack(t, testManifest, with("index.xml", []byte("<Template/>")))
|
||||
}},
|
||||
{"subfolder", "folders", func(t *testing.T) string {
|
||||
dir := dirPack(t, testManifest, testFiles())
|
||||
writeFile(t, filepath.Join(dir, FilesDir, "sub", "a.png"), testPNG)
|
||||
return dir
|
||||
}},
|
||||
{"both", "both exist", func(t *testing.T) string {
|
||||
dir := dirPack(t, testManifest, testFiles())
|
||||
writeFile(t, filepath.Join(dir, ChknName), makeZip(t, zipEntries(zip.Store)))
|
||||
return dir
|
||||
}},
|
||||
{"neither", "no product.chkn", func(t *testing.T) string {
|
||||
dir := t.TempDir()
|
||||
writeFile(t, filepath.Join(dir, ManifestName), []byte(testManifest))
|
||||
return dir
|
||||
}},
|
||||
{"no manifest", ManifestName, func(t *testing.T) string { return t.TempDir() }},
|
||||
{"bad manifest", "revision", func(t *testing.T) string {
|
||||
return dirPack(t, strings.Replace(testManifest, `"revision": 2`, `"revision": 0`, 1), testFiles())
|
||||
}},
|
||||
{"nested zip entry", "in a folder", func(t *testing.T) string {
|
||||
return chknPack(t, testManifest, extra(zipEntry{"sub/a.png", testPNG, zip.Store}))
|
||||
}},
|
||||
{"backslash in zip", `can't have`, func(t *testing.T) string {
|
||||
return chknPack(t, testManifest, extra(zipEntry{`sub\a.png`, testPNG, zip.Store}))
|
||||
}},
|
||||
{"duplicate in zip", "duplicate", func(t *testing.T) string {
|
||||
return chknPack(t, testManifest, extra(zipEntry{"coat.png", testPNG, zip.Store}))
|
||||
}},
|
||||
{"not latin-1", "Latin-1", func(t *testing.T) string {
|
||||
return chknPack(t, testManifest, extra(zipEntry{"snow☃.png", testPNG, zip.Store}))
|
||||
}},
|
||||
{"too many files", "over 512 files", func(t *testing.T) string {
|
||||
var e []zipEntry
|
||||
for i := range MaxFiles {
|
||||
e = append(e, zipEntry{name: fmt.Sprintf("f%d.png", i), method: zip.Store})
|
||||
}
|
||||
return chknPack(t, testManifest, extra(e...))
|
||||
}},
|
||||
{"zip bomb", "over 64 MiB", func(t *testing.T) string {
|
||||
// Deflates to well under 1 MiB.
|
||||
return chknPack(t, testManifest, extra(zipEntry{"big.png", make([]byte, MaxFileSize+1), zip.Deflate}))
|
||||
}},
|
||||
{"too big in all", "in all", func(t *testing.T) string {
|
||||
var e []zipEntry
|
||||
for i := range 5 {
|
||||
e = append(e, zipEntry{fmt.Sprintf("big%d.png", i), make([]byte, MaxFileSize), zip.Deflate})
|
||||
}
|
||||
return chknPack(t, testManifest, extra(e...))
|
||||
}},
|
||||
{"not a zip", ChknName, func(t *testing.T) string {
|
||||
dir := t.TempDir()
|
||||
writeFile(t, filepath.Join(dir, ManifestName), []byte(testManifest))
|
||||
writeFile(t, filepath.Join(dir, ChknName), []byte("not a zip"))
|
||||
return dir
|
||||
}},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
_, err := Load(c.dir(t))
|
||||
if err == nil || !strings.Contains(err.Error(), c.want) {
|
||||
t.Errorf("got %v, want an error with %q", err, c.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseManifest(t *testing.T) {
|
||||
for _, c := range []struct{ json, want string }{
|
||||
{`{"format": 2, "id": "a", "revision": 1, "name": "A", "rating": "general"}`, "format"},
|
||||
{`{"id": "a", "revision": 1, "name": "A", "rating": "general"}`, "format"},
|
||||
{`{"format": 1, "id": "-a", "revision": 1, "name": "A", "rating": "general"}`, "id"},
|
||||
{`{"format": 1, "id": "a-", "revision": 1, "name": "A", "rating": "general"}`, "id"},
|
||||
{`{"format": 1, "id": "A", "revision": 1, "name": "A", "rating": "general"}`, "id"},
|
||||
{`{"format": 1, "id": "", "revision": 1, "name": "A", "rating": "general"}`, "id"},
|
||||
{`{"format": 1, "id": "` + strings.Repeat("a", 65) + `", "revision": 1, "name": "A", "rating": "general"}`, "id"},
|
||||
{`{"format": 1, "id": "a", "revision": 0, "name": "A", "rating": "general"}`, "revision"},
|
||||
{`{"format": 1, "id": "a", "revision": 1.5, "name": "A", "rating": "general"}`, "revision"},
|
||||
{`{"format": 1, "id": "a", "revision": 1, "name": " ", "rating": "general"}`, "name"},
|
||||
{`{"format": 1, "id": "a", "revision": 1, "name": "` + strings.Repeat("é", 65) + `", "rating": "general"}`, "name"},
|
||||
{`{"format": 1, "id": "a", "revision": 1, "name": "A\u0007", "rating": "general"}`, "name"},
|
||||
{`{"format": 1, "id": "a", "revision": 1, "name": "A", "creator": "` + strings.Repeat("c", 65) + `", "rating": "general"}`, "creator"},
|
||||
{`{"format": 1, "id": "a", "revision": 1, "name": "A", "description": "` + strings.Repeat("d", 20001) + `", "rating": "general"}`, "description"},
|
||||
{`{"format": 1, "id": "a", "revision": 1, "name": "A", "rating": "GA"}`, "rating"},
|
||||
{`{"format": 1, "id": "a", "revision": 1, "name": "A"}`, "rating"},
|
||||
{`{"format": 1, "id": "a", "revision": 1, "name": "A", "rating": "general", "price": 5}`, "price"},
|
||||
{`{"format": 1, "id": "a", "revision": 1, "name": "A", "rating": "general"} {}`, "after"},
|
||||
} {
|
||||
_, err := ParseManifest([]byte(c.json))
|
||||
if err == nil || !strings.Contains(err.Error(), c.want) || !strings.Contains(err.Error(), ManifestName) {
|
||||
t.Errorf("%s: got %v, want an error with %q", c.json, err, c.want)
|
||||
}
|
||||
}
|
||||
|
||||
m, err := ParseManifest([]byte(`{"format": 1, "id": "a-1", "revision": 1, "name": "` + strings.Repeat("é", 64) + `", "rating": "adult"}`))
|
||||
if err != nil || m.Rating != Adult {
|
||||
t.Errorf("valid manifest: %+v, %v", m, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseIndex(t *testing.T) {
|
||||
x, err := ParseIndex([]byte("<?xml version=\"1.0\" encoding=\"ISO-8859-1\"?><Template>" +
|
||||
"<ProductRating> AP </ProductRating><ProductName>Caf\xe9 & Bar</ProductName><Junk a=b>x<br></Junk></Template>"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if want := (Index{Rating: "AP", Name: "Café & Bar"}); *x != want {
|
||||
t.Errorf("got %+v, want %+v", *x, want)
|
||||
}
|
||||
|
||||
if x, err := ParseIndex([]byte("\ufeff<Template><ProductRating>GA</ProductRating></Template>")); err != nil || x.Rating != "GA" {
|
||||
t.Errorf("with a BOM: %+v, %v", x, err)
|
||||
}
|
||||
|
||||
for _, c := range []struct{ xml, want string }{
|
||||
{`<Product><ProductRating>GA</ProductRating></Product>`, "Template"},
|
||||
{``, "Template"},
|
||||
{`<Template></Template>`, "ProductRating"},
|
||||
{`<Template><ProductRating>PG</ProductRating></Template>`, "ProductRating"},
|
||||
{`<Template><ProductRating>GA</ProductRating><ProductTypeID>x</ProductTypeID></Template>`, "ProductTypeID"},
|
||||
{`<Template><ProductRating>GA</ProductRating><BodyPatternProductId>-1</BodyPatternProductId></Template>`, "BodyPatternProductId"},
|
||||
{`<Template><ProductRating>GA</ProductRating><__DATAIMPORT>http://e.com/index.xml</__DATAIMPORT></Template>`, "__DATAIMPORT"},
|
||||
{`<Template><ProductRating>GA</ProductRating><__DATAIMPORT>product://0/index.xml</__DATAIMPORT></Template>`, "__DATAIMPORT"},
|
||||
{`<Template><ProductRating>GA</ProductRating><ProductRating>AP</ProductRating></Template>`, "twice"},
|
||||
{`<?xml version="1.0" encoding="EBCDIC"?><Template/>`, "encoding"},
|
||||
} {
|
||||
_, err := ParseIndex([]byte(c.xml))
|
||||
if err == nil || !strings.Contains(err.Error(), c.want) {
|
||||
t.Errorf("%s: got %v, want an error with %q", c.xml, err, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckName(t *testing.T) {
|
||||
for _, name := range []string{"index.xml", "a b.png", "café.jpg", strings.Repeat("a", 200)} {
|
||||
if err := CheckName(name); err != nil {
|
||||
t.Errorf("%q: %v", name, err)
|
||||
}
|
||||
}
|
||||
for _, name := range []string{"", ".", "..", "a/b", `a\b`, "\xff", strings.Repeat("a", 201), "_CONTENTS.json", "a\x00b", "a\nb", "snow☃"} {
|
||||
if err := CheckName(name); err == nil {
|
||||
t.Errorf("%q: no error", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user