A Go server that takes product uploads with hub upload tokens, checks them by t5de-api's pack rules, stores files by sha256, registers revisions with the hub (POST /api/v1/catalog/revisions), and serves files with immutable caching; files used only by adult revisions need an adult access token. Contract: t5de-hub's docs/products.md. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
dhkatzandClaude Opus 5.5 committed
commit
8742829e77
20 files changed
+2348
No files matched your search
@@ -0,0 +1,217 @@
|
||||
// Package store keeps the catalog's files and the revisions it hosts: files on disk named by their sha256
|
||||
// (files/<first 2 hex>/<sha256>), revisions in SQLite (catalog.db).
|
||||
package store
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
// Ratings.
|
||||
const (
|
||||
General = "general"
|
||||
Adult = "adult"
|
||||
)
|
||||
|
||||
// File is one entry of a revision's file list.
|
||||
type File struct {
|
||||
Name string `json:"name"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Size int64 `json:"size"`
|
||||
}
|
||||
|
||||
// Revision is a revision the catalog hosts.
|
||||
type Revision struct {
|
||||
Product string `json:"product"`
|
||||
Revision int `json:"revision"`
|
||||
Rating string `json:"rating"`
|
||||
Files []File `json:"files"` // sorted by name
|
||||
}
|
||||
|
||||
// ErrNotFound is returned for files and revisions the catalog doesn't host.
|
||||
var ErrNotFound = errors.New("not found")
|
||||
|
||||
var hashPattern = regexp.MustCompile(`^[0-9a-f]{64}$`)
|
||||
|
||||
// ValidHash reports whether s is a lowercase hex sha256.
|
||||
func ValidHash(s string) bool { return hashPattern.MatchString(s) }
|
||||
|
||||
// Store is the catalog's data folder.
|
||||
type Store struct {
|
||||
dir string
|
||||
db *sql.DB
|
||||
}
|
||||
|
||||
const schema = `
|
||||
CREATE TABLE IF NOT EXISTS revisions (
|
||||
product TEXT NOT NULL,
|
||||
revision INTEGER NOT NULL,
|
||||
rating TEXT NOT NULL CHECK (rating IN ('general', 'adult')),
|
||||
files TEXT NOT NULL, -- the file list, JSON
|
||||
created_at INTEGER NOT NULL DEFAULT (unixepoch()),
|
||||
PRIMARY KEY (product, revision)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS revision_files (
|
||||
product TEXT NOT NULL,
|
||||
revision INTEGER NOT NULL,
|
||||
sha256 TEXT NOT NULL,
|
||||
PRIMARY KEY (product, revision, sha256),
|
||||
FOREIGN KEY (product, revision) REFERENCES revisions (product, revision) ON DELETE CASCADE
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS revision_files_sha256 ON revision_files (sha256);
|
||||
`
|
||||
|
||||
// Open opens (or creates) the store in dir.
|
||||
func Open(dir string) (*Store, error) {
|
||||
if err := os.MkdirAll(filepath.Join(dir, "files"), 0o755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Leftovers of uploads interrupted by a crash.
|
||||
if err := os.RemoveAll(filepath.Join(dir, "tmp")); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(dir, "tmp"), 0o755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dsn := "file:" + filepath.ToSlash(filepath.Join(dir, "catalog.db")) +
|
||||
"?_pragma=busy_timeout(5000)&_pragma=journal_mode(WAL)&_pragma=foreign_keys(1)&_pragma=synchronous(NORMAL)"
|
||||
db, err := sql.Open("sqlite", dsn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
db.SetMaxOpenConns(1)
|
||||
if _, err := db.Exec(schema); err != nil {
|
||||
db.Close()
|
||||
return nil, fmt.Errorf("catalog.db: %w", err)
|
||||
}
|
||||
return &Store{dir: dir, db: db}, nil
|
||||
}
|
||||
|
||||
// Close closes the database.
|
||||
func (s *Store) Close() error { return s.db.Close() }
|
||||
|
||||
// TempDir is a folder on the store's file system for temporary files (emptied when the store opens).
|
||||
func (s *Store) TempDir() string { return filepath.Join(s.dir, "tmp") }
|
||||
|
||||
// Path returns where the file with this hash is (or would be) stored.
|
||||
func (s *Store) Path(hash string) string {
|
||||
return filepath.Join(s.dir, "files", hash[:2], hash)
|
||||
}
|
||||
|
||||
// PutFile stores data under its sha256 and returns the hash. A file already stored isn't written again.
|
||||
func (s *Store) PutFile(data []byte) (string, error) {
|
||||
sum := sha256.Sum256(data)
|
||||
hash := hex.EncodeToString(sum[:])
|
||||
path := s.Path(hash)
|
||||
if fi, err := os.Stat(path); err == nil && fi.Size() == int64(len(data)) {
|
||||
return hash, nil
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
return "", err
|
||||
}
|
||||
f, err := os.CreateTemp(filepath.Dir(path), ".tmp-*")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
tmp := f.Name()
|
||||
_, err = f.Write(data)
|
||||
if err == nil {
|
||||
err = f.Sync()
|
||||
}
|
||||
if cerr := f.Close(); err == nil {
|
||||
err = cerr
|
||||
}
|
||||
if err == nil {
|
||||
err = os.Rename(tmp, path)
|
||||
}
|
||||
if err != nil {
|
||||
os.Remove(tmp)
|
||||
return "", err
|
||||
}
|
||||
return hash, nil
|
||||
}
|
||||
|
||||
// Open opens a stored file.
|
||||
func (s *Store) Open(hash string) (*os.File, error) {
|
||||
if !ValidHash(hash) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
f, err := os.Open(s.Path(hash))
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
return f, err
|
||||
}
|
||||
|
||||
// AddRevision records a revision as hosted. Its files must be stored already. Recording the same revision again
|
||||
// replaces it.
|
||||
func (s *Store) AddRevision(r Revision) error {
|
||||
if r.Rating != General && r.Rating != Adult {
|
||||
return fmt.Errorf("rating %q", r.Rating)
|
||||
}
|
||||
files, err := json.Marshal(r.Files)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
if _, err := tx.Exec(`DELETE FROM revisions WHERE product = ? AND revision = ?`, r.Product, r.Revision); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(`INSERT INTO revisions (product, revision, rating, files) VALUES (?, ?, ?, ?)`,
|
||||
r.Product, r.Revision, r.Rating, string(files)); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, f := range r.Files {
|
||||
if _, err := tx.Exec(`INSERT OR IGNORE INTO revision_files (product, revision, sha256) VALUES (?, ?, ?)`,
|
||||
r.Product, r.Revision, f.SHA256); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// Revision returns a hosted revision.
|
||||
func (s *Store) Revision(product string, revision int) (*Revision, error) {
|
||||
r := Revision{Product: product, Revision: revision}
|
||||
var files string
|
||||
err := s.db.QueryRow(`SELECT rating, files FROM revisions WHERE product = ? AND revision = ?`, product, revision).
|
||||
Scan(&r.Rating, &files)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := json.Unmarshal([]byte(files), &r.Files); err != nil {
|
||||
return nil, fmt.Errorf("revision %s/%d: %w", product, revision, err)
|
||||
}
|
||||
return &r, nil
|
||||
}
|
||||
|
||||
// FileAccess tells whether a file belongs to a hosted revision (hosted) and whether every such revision is adult
|
||||
// (adultOnly). Files in no hosted revision (an upload the hub refused) aren't served.
|
||||
func (s *Store) FileAccess(hash string) (hosted, adultOnly bool, err error) {
|
||||
var n, general int
|
||||
err = s.db.QueryRow(`
|
||||
SELECT count(*), coalesce(sum(r.rating = 'general'), 0)
|
||||
FROM revision_files rf JOIN revisions r ON r.product = rf.product AND r.revision = rf.revision
|
||||
WHERE rf.sha256 = ?`, hash).Scan(&n, &general)
|
||||
if err != nil {
|
||||
return false, false, err
|
||||
}
|
||||
return n > 0, n > 0 && general == 0, nil
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func open(t *testing.T, dir string) *Store {
|
||||
t.Helper()
|
||||
s, err := Open(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { s.Close() })
|
||||
return s
|
||||
}
|
||||
|
||||
func hashOf(data string) string {
|
||||
sum := sha256.Sum256([]byte(data))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func TestFiles(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
s := open(t, dir)
|
||||
|
||||
hash, err := s.PutFile([]byte("hello"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if hash != hashOf("hello") {
|
||||
t.Fatalf("hash %s", hash)
|
||||
}
|
||||
want := filepath.Join(dir, "files", hash[:2], hash)
|
||||
if s.Path(hash) != want {
|
||||
t.Fatalf("path %s, want %s", s.Path(hash), want)
|
||||
}
|
||||
again, err := s.PutFile([]byte("hello"))
|
||||
if err != nil || again != hash {
|
||||
t.Fatalf("again: %s, %v", again, err)
|
||||
}
|
||||
f, err := s.Open(hash)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, _ := io.ReadAll(f)
|
||||
f.Close()
|
||||
if string(data) != "hello" {
|
||||
t.Fatalf("data %q", data)
|
||||
}
|
||||
// No temp files left beside it.
|
||||
entries, _ := os.ReadDir(filepath.Dir(want))
|
||||
if len(entries) != 1 {
|
||||
t.Fatalf("%d entries in %s", len(entries), filepath.Dir(want))
|
||||
}
|
||||
|
||||
if _, err := s.Open(hashOf("missing")); !errors.Is(err, ErrNotFound) {
|
||||
t.Fatalf("missing: %v", err)
|
||||
}
|
||||
for _, bad := range []string{"", "../catalog.db", hash[:63], "A" + hash[1:]} {
|
||||
if _, err := s.Open(bad); !errors.Is(err, ErrNotFound) {
|
||||
t.Fatalf("%q: %v", bad, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRevisions(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
s := open(t, dir)
|
||||
shared, general, adult := hashOf("shared"), hashOf("general"), hashOf("adult")
|
||||
|
||||
if hosted, _, err := s.FileAccess(shared); err != nil || hosted {
|
||||
t.Fatalf("before: hosted %v, %v", hosted, err)
|
||||
}
|
||||
|
||||
a := Revision{Product: "aaaaaaaaaa", Revision: 1, Rating: Adult, Files: []File{
|
||||
{Name: "a.png", SHA256: adult, Size: 5}, {Name: "index.xml", SHA256: shared, Size: 6},
|
||||
}}
|
||||
if err := s.AddRevision(a); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
check := func(hash string, wantHosted, wantAdult bool) {
|
||||
t.Helper()
|
||||
hosted, adultOnly, err := s.FileAccess(hash)
|
||||
if err != nil || hosted != wantHosted || adultOnly != wantAdult {
|
||||
t.Fatalf("%s: hosted %v adult %v (%v), want %v %v", hash[:8], hosted, adultOnly, err, wantHosted, wantAdult)
|
||||
}
|
||||
}
|
||||
check(shared, true, true)
|
||||
check(adult, true, true)
|
||||
check(general, false, false)
|
||||
|
||||
g := Revision{Product: "bbbbbbbbbb", Revision: 2, Rating: General, Files: []File{
|
||||
{Name: "g.png", SHA256: general, Size: 7}, {Name: "index.xml", SHA256: shared, Size: 6},
|
||||
}}
|
||||
if err := s.AddRevision(g); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
check(shared, true, false) // also in a general revision
|
||||
check(adult, true, true)
|
||||
check(general, true, false)
|
||||
|
||||
got, err := s.Revision("aaaaaaaaaa", 1)
|
||||
if err != nil || !reflect.DeepEqual(*got, a) {
|
||||
t.Fatalf("got %+v, %v", got, err)
|
||||
}
|
||||
if _, err := s.Revision("aaaaaaaaaa", 2); !errors.Is(err, ErrNotFound) {
|
||||
t.Fatalf("missing revision: %v", err)
|
||||
}
|
||||
if err := s.AddRevision(Revision{Product: "c", Revision: 1, Rating: "GA"}); err == nil {
|
||||
t.Fatal("bad rating accepted")
|
||||
}
|
||||
|
||||
// Recording a revision again replaces it, and the data survives reopening.
|
||||
a.Rating = General
|
||||
if err := s.AddRevision(a); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s.Close()
|
||||
s = open(t, dir)
|
||||
got, err = s.Revision("aaaaaaaaaa", 1)
|
||||
if err != nil || got.Rating != General {
|
||||
t.Fatalf("after reopening: %+v, %v", got, err)
|
||||
}
|
||||
if hosted, adultOnly, _ := s.FileAccess(adult); !hosted || adultOnly {
|
||||
t.Fatal("adult file still adult-only after the revision became general")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTempCleared(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
s := open(t, dir)
|
||||
if err := os.WriteFile(filepath.Join(s.TempDir(), "leftover"), []byte("x"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s.Close()
|
||||
s = open(t, dir)
|
||||
if entries, _ := os.ReadDir(s.TempDir()); len(entries) != 0 {
|
||||
t.Fatalf("temp folder not emptied: %v", entries)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user