Add pack, cfl and safehtml
pack loads local product packs (format 1): t5de-product.json and the product's files, as product.chkn or files/, with index.xml's rating, type, gender and parent product. cfl writes and reads IMVU's product container, uncompressed. safehtml cleans product descriptions by an allowlist (bluemonday). Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
dhkatzandClaude Opus 5.5 committed
commit
9bdc3ff455
16 files changed
+1638
No files matched your search
+228
@@ -0,0 +1,228 @@
|
||||
// Package cfl reads and writes CFL files, IMVU's product container: a product's files in one archive, as the
|
||||
// client downloads and caches them.
|
||||
//
|
||||
// Layout (little-endian): "CFL3", u32 directory offset, u32 directory size (uncompressed). Each entry's data is a
|
||||
// u32 stored length and the stored bytes. At the directory offset: u32 compression flag, u32 stored length, the
|
||||
// directory. Each directory entry: i32 size, i32 offset (of the entry's length field), i32 compression flag, i16
|
||||
// name length, the name (Latin-1). "DFL3" files are the same, with each directory entry followed by an i32 length
|
||||
// and that many bytes of content hash.
|
||||
package cfl
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// Compression flags.
|
||||
const (
|
||||
None = 0
|
||||
LZMA = 4 // pylzma's format: 5 property bytes, then the raw stream
|
||||
)
|
||||
|
||||
// Entry is one file in a CFL.
|
||||
type Entry struct {
|
||||
Name string
|
||||
Data []byte
|
||||
}
|
||||
|
||||
// ErrLZMA is returned for CFLs with LZMA-compressed entries or directories.
|
||||
var ErrLZMA = errors.New("cfl: LZMA-compressed CFLs aren't supported")
|
||||
|
||||
const headerSize = 12
|
||||
|
||||
// Write writes entries as a CFL, in order and uncompressed. The output depends only on the entries.
|
||||
func Write(w io.Writer, entries []Entry) error {
|
||||
var body, dir bytes.Buffer
|
||||
offset := int64(headerSize)
|
||||
for _, e := range entries {
|
||||
name, err := Latin1(e.Name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(name) == 0 {
|
||||
return errors.New("cfl: empty entry name")
|
||||
}
|
||||
if len(name) > math.MaxInt16 {
|
||||
return fmt.Errorf("cfl: entry name %q is too long", e.Name)
|
||||
}
|
||||
if offset+4+int64(len(e.Data)) > math.MaxInt32 {
|
||||
return errors.New("cfl: entries over 2 GiB")
|
||||
}
|
||||
le32(&dir, uint32(len(e.Data)))
|
||||
le32(&dir, uint32(offset))
|
||||
le32(&dir, None)
|
||||
binary.Write(&dir, binary.LittleEndian, int16(len(name)))
|
||||
dir.Write(name)
|
||||
|
||||
le32(&body, uint32(len(e.Data)))
|
||||
body.Write(e.Data)
|
||||
offset += 4 + int64(len(e.Data))
|
||||
}
|
||||
if offset+8+int64(dir.Len()) > math.MaxInt32 {
|
||||
return errors.New("cfl: entries over 2 GiB")
|
||||
}
|
||||
|
||||
var head bytes.Buffer
|
||||
head.WriteString("CFL3")
|
||||
le32(&head, uint32(offset))
|
||||
le32(&head, uint32(dir.Len()))
|
||||
le32(&body, None)
|
||||
le32(&body, uint32(dir.Len()))
|
||||
for _, b := range [][]byte{head.Bytes(), body.Bytes(), dir.Bytes()} {
|
||||
if _, err := w.Write(b); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func le32(b *bytes.Buffer, v uint32) {
|
||||
b.Write(binary.LittleEndian.AppendUint32(nil, v))
|
||||
}
|
||||
|
||||
// Read reads a CFL (or DFL) whose entries and directory are uncompressed. Entries' Data share memory with data.
|
||||
func Read(data []byte) ([]Entry, error) {
|
||||
if len(data) < headerSize {
|
||||
return nil, errors.New("cfl: too short")
|
||||
}
|
||||
var hashes bool
|
||||
switch string(data[:4]) {
|
||||
case "CFL3":
|
||||
case "DFL3":
|
||||
hashes = true
|
||||
default:
|
||||
return nil, fmt.Errorf("cfl: unknown header %q", data[:4])
|
||||
}
|
||||
dirOffset := int64(binary.LittleEndian.Uint32(data[4:]))
|
||||
dirSize := int64(binary.LittleEndian.Uint32(data[8:]))
|
||||
|
||||
r := reader{data: data, pos: dirOffset}
|
||||
flag := r.u32()
|
||||
stored := int64(r.u32())
|
||||
if r.err != nil {
|
||||
return nil, fmt.Errorf("cfl: directory offset %d is past the end", dirOffset)
|
||||
}
|
||||
switch flag {
|
||||
case None:
|
||||
case LZMA:
|
||||
return nil, ErrLZMA
|
||||
default:
|
||||
return nil, fmt.Errorf("cfl: directory has unknown compression %d", flag)
|
||||
}
|
||||
if stored != dirSize {
|
||||
return nil, fmt.Errorf("cfl: directory is %d bytes, header says %d", stored, dirSize)
|
||||
}
|
||||
dir := r.bytes(stored)
|
||||
if r.err != nil {
|
||||
return nil, errors.New("cfl: directory runs past the end")
|
||||
}
|
||||
|
||||
var entries []Entry
|
||||
d := reader{data: dir}
|
||||
for d.pos < int64(len(dir)) {
|
||||
size := int64(int32(d.u32()))
|
||||
offset := int64(int32(d.u32()))
|
||||
flag := d.u32()
|
||||
nameLen := int64(int16(d.u16()))
|
||||
if nameLen < 0 {
|
||||
return nil, fmt.Errorf("cfl: directory entry %d has a negative name length", len(entries))
|
||||
}
|
||||
name := d.bytes(nameLen)
|
||||
if hashes {
|
||||
hashLen := int64(int32(d.u32()))
|
||||
if hashLen < 0 {
|
||||
return nil, fmt.Errorf("cfl: directory entry %d has a negative hash length", len(entries))
|
||||
}
|
||||
d.bytes(hashLen)
|
||||
}
|
||||
if d.err != nil {
|
||||
return nil, fmt.Errorf("cfl: directory entry %d is cut off", len(entries))
|
||||
}
|
||||
if len(name) == 0 {
|
||||
return nil, fmt.Errorf("cfl: directory entry %d has no name", len(entries))
|
||||
}
|
||||
n := latin1String(name)
|
||||
switch flag {
|
||||
case None:
|
||||
case LZMA:
|
||||
return nil, ErrLZMA
|
||||
default:
|
||||
return nil, fmt.Errorf("cfl: %q has unknown compression %d", n, flag)
|
||||
}
|
||||
if size < 0 || offset < 0 {
|
||||
return nil, fmt.Errorf("cfl: %q has a negative size or offset", n)
|
||||
}
|
||||
e := reader{data: data, pos: offset}
|
||||
stored := int64(e.u32())
|
||||
body := e.bytes(stored)
|
||||
if e.err != nil {
|
||||
return nil, fmt.Errorf("cfl: %q runs past the end", n)
|
||||
}
|
||||
if stored != size {
|
||||
return nil, fmt.Errorf("cfl: %q is %d bytes, directory says %d", n, stored, size)
|
||||
}
|
||||
entries = append(entries, Entry{Name: n, Data: body})
|
||||
}
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// reader reads little-endian values from data, setting err instead of reading past the end.
|
||||
type reader struct {
|
||||
data []byte
|
||||
pos int64
|
||||
err error
|
||||
}
|
||||
|
||||
func (r *reader) bytes(n int64) []byte {
|
||||
if r.err != nil || n < 0 || r.pos < 0 || r.pos > int64(len(r.data)) || n > int64(len(r.data))-r.pos {
|
||||
r.err = io.ErrUnexpectedEOF
|
||||
return nil
|
||||
}
|
||||
b := r.data[r.pos : r.pos+n : r.pos+n]
|
||||
r.pos += n
|
||||
return b
|
||||
}
|
||||
|
||||
func (r *reader) u32() uint32 {
|
||||
b := r.bytes(4)
|
||||
if b == nil {
|
||||
return 0
|
||||
}
|
||||
return binary.LittleEndian.Uint32(b)
|
||||
}
|
||||
|
||||
func (r *reader) u16() uint16 {
|
||||
b := r.bytes(2)
|
||||
if b == nil {
|
||||
return 0
|
||||
}
|
||||
return binary.LittleEndian.Uint16(b)
|
||||
}
|
||||
|
||||
// Latin1 encodes s as Latin-1, as CFLs store names.
|
||||
func Latin1(s string) ([]byte, error) {
|
||||
if !utf8.ValidString(s) {
|
||||
return nil, fmt.Errorf("cfl: name %q isn't valid UTF-8", s)
|
||||
}
|
||||
b := make([]byte, 0, len(s))
|
||||
for _, r := range s {
|
||||
if r > 0xff {
|
||||
return nil, fmt.Errorf("cfl: name %q has characters outside Latin-1", s)
|
||||
}
|
||||
b = append(b, byte(r))
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
|
||||
func latin1String(b []byte) string {
|
||||
r := make([]rune, len(b))
|
||||
for i, c := range b {
|
||||
r[i] = rune(c)
|
||||
}
|
||||
return string(r)
|
||||
}
|
||||
+153
@@ -0,0 +1,153 @@
|
||||
package cfl
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func write(t testing.TB, entries []Entry) []byte {
|
||||
t.Helper()
|
||||
var b bytes.Buffer
|
||||
if err := Write(&b, entries); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b.Bytes()
|
||||
}
|
||||
|
||||
func TestWriteLayout(t *testing.T) {
|
||||
got := write(t, []Entry{{Name: "a", Data: []byte("xy")}, {Name: "é", Data: nil}})
|
||||
want := "" +
|
||||
"43464c33" + "16000000" + "1e000000" + // CFL3, directory at 22, 30 bytes
|
||||
"02000000" + "7879" + // "a"
|
||||
"00000000" + // "é"
|
||||
"00000000" + "1e000000" + // directory: uncompressed, 30 bytes
|
||||
"02000000" + "0c000000" + "00000000" + "0100" + "61" +
|
||||
"00000000" + "12000000" + "00000000" + "0100" + "e9"
|
||||
if hex.EncodeToString(got) != want {
|
||||
t.Errorf("got %x\nwant %s", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoundTrip(t *testing.T) {
|
||||
entries := []Entry{
|
||||
{Name: "index.xml", Data: []byte("<Template/>")},
|
||||
{Name: "café.png", Data: []byte{0x89, 'P', 'N', 'G'}},
|
||||
{Name: "empty", Data: []byte{}},
|
||||
}
|
||||
got, err := Read(write(t, entries))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(got, entries) {
|
||||
t.Errorf("got %q, want %q", got, entries)
|
||||
}
|
||||
|
||||
if got, err := Read(write(t, nil)); err != nil || len(got) != 0 {
|
||||
t.Errorf("empty CFL: %v, %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteErrors(t *testing.T) {
|
||||
for _, name := range []string{"", "snow☃.png", "\xff"} {
|
||||
var b bytes.Buffer
|
||||
if err := Write(&b, []Entry{{Name: name}}); err == nil {
|
||||
t.Errorf("%q: no error", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestDFL builds a DFL3 by hand: one entry with a content hash after its directory entry.
|
||||
func TestDFL(t *testing.T) {
|
||||
var b []byte
|
||||
le := binary.LittleEndian
|
||||
b = append(b, "DFL3"...)
|
||||
b = le.AppendUint32(b, 18)
|
||||
dirSizeAt := len(b)
|
||||
b = le.AppendUint32(b, 0)
|
||||
b = le.AppendUint32(b, 2)
|
||||
b = append(b, "hi"...)
|
||||
var dir []byte
|
||||
dir = le.AppendUint32(dir, 2)
|
||||
dir = le.AppendUint32(dir, 12)
|
||||
dir = le.AppendUint32(dir, 0)
|
||||
dir = le.AppendUint16(dir, 1)
|
||||
dir = append(dir, 'x')
|
||||
dir = le.AppendUint32(dir, 3)
|
||||
dir = append(dir, "abc"...)
|
||||
le.PutUint32(b[dirSizeAt:], uint32(len(dir)))
|
||||
b = le.AppendUint32(b, 0)
|
||||
b = le.AppendUint32(b, uint32(len(dir)))
|
||||
b = append(b, dir...)
|
||||
|
||||
got, err := Read(b)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if want := []Entry{{Name: "x", Data: []byte("hi")}}; !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("got %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadLZMA(t *testing.T) {
|
||||
b := write(t, []Entry{{Name: "a", Data: []byte("xy")}})
|
||||
dir := binary.LittleEndian.Uint32(b[4:])
|
||||
|
||||
c := bytes.Clone(b)
|
||||
binary.LittleEndian.PutUint32(c[dir:], LZMA)
|
||||
if _, err := Read(c); !errors.Is(err, ErrLZMA) {
|
||||
t.Errorf("compressed directory: %v", err)
|
||||
}
|
||||
|
||||
c = bytes.Clone(b)
|
||||
binary.LittleEndian.PutUint32(c[dir+8+8:], LZMA) // the entry's flag
|
||||
if _, err := Read(c); !errors.Is(err, ErrLZMA) {
|
||||
t.Errorf("compressed entry: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadMalformed(t *testing.T) {
|
||||
good := write(t, []Entry{{Name: "a", Data: []byte("xy")}})
|
||||
for i := range len(good) {
|
||||
if _, err := Read(good[:i]); err == nil {
|
||||
t.Errorf("cut at %d: no error", i)
|
||||
}
|
||||
}
|
||||
bad := bytes.Clone(good)
|
||||
bad[0] = 'X'
|
||||
if _, err := Read(bad); err == nil {
|
||||
t.Error("bad header: no error")
|
||||
}
|
||||
bad = bytes.Clone(good)
|
||||
binary.LittleEndian.PutUint32(bad[12:], 1000) // the entry's stored length
|
||||
if _, err := Read(bad); err == nil {
|
||||
t.Error("bad entry length: no error")
|
||||
}
|
||||
}
|
||||
|
||||
func FuzzRead(f *testing.F) {
|
||||
f.Add(write(f, []Entry{{Name: "a", Data: []byte("xy")}, {Name: "b.png", Data: []byte{1, 2, 3}}}))
|
||||
f.Add(write(f, nil))
|
||||
f.Add([]byte("DFL3\x0c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"))
|
||||
f.Fuzz(func(t *testing.T, data []byte) {
|
||||
entries, err := Read(data)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
again, err := Read(write(t, entries))
|
||||
if err != nil {
|
||||
t.Fatalf("rewritten CFL: %v", err)
|
||||
}
|
||||
if len(again) != len(entries) {
|
||||
t.Fatalf("rewritten CFL has %d entries, want %d", len(again), len(entries))
|
||||
}
|
||||
for i := range entries {
|
||||
if again[i].Name != entries[i].Name || !bytes.Equal(again[i].Data, entries[i].Data) {
|
||||
t.Fatalf("entry %d changed", i)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
Reference in new issue
Block a user