A Go server that takes product uploads with hub upload tokens, checks them by t5de-api's pack rules, stores files by sha256, registers revisions with the hub (POST /api/v1/catalog/revisions), and serves files with immutable caching; files used only by adult revisions need an adult access token. Contract: t5de-hub's docs/products.md. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
dhkatzandClaude Opus 5.5 committed
commit
8742829e77
20 files changed
+2348
No files matched your search
@@ -0,0 +1,217 @@
|
||||
// Package store keeps the catalog's files and the revisions it hosts: files on disk named by their sha256
|
||||
// (files/<first 2 hex>/<sha256>), revisions in SQLite (catalog.db).
|
||||
package store
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
// Ratings.
|
||||
const (
|
||||
General = "general"
|
||||
Adult = "adult"
|
||||
)
|
||||
|
||||
// File is one entry of a revision's file list.
|
||||
type File struct {
|
||||
Name string `json:"name"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Size int64 `json:"size"`
|
||||
}
|
||||
|
||||
// Revision is a revision the catalog hosts.
|
||||
type Revision struct {
|
||||
Product string `json:"product"`
|
||||
Revision int `json:"revision"`
|
||||
Rating string `json:"rating"`
|
||||
Files []File `json:"files"` // sorted by name
|
||||
}
|
||||
|
||||
// ErrNotFound is returned for files and revisions the catalog doesn't host.
|
||||
var ErrNotFound = errors.New("not found")
|
||||
|
||||
var hashPattern = regexp.MustCompile(`^[0-9a-f]{64}$`)
|
||||
|
||||
// ValidHash reports whether s is a lowercase hex sha256.
|
||||
func ValidHash(s string) bool { return hashPattern.MatchString(s) }
|
||||
|
||||
// Store is the catalog's data folder.
|
||||
type Store struct {
|
||||
dir string
|
||||
db *sql.DB
|
||||
}
|
||||
|
||||
const schema = `
|
||||
CREATE TABLE IF NOT EXISTS revisions (
|
||||
product TEXT NOT NULL,
|
||||
revision INTEGER NOT NULL,
|
||||
rating TEXT NOT NULL CHECK (rating IN ('general', 'adult')),
|
||||
files TEXT NOT NULL, -- the file list, JSON
|
||||
created_at INTEGER NOT NULL DEFAULT (unixepoch()),
|
||||
PRIMARY KEY (product, revision)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS revision_files (
|
||||
product TEXT NOT NULL,
|
||||
revision INTEGER NOT NULL,
|
||||
sha256 TEXT NOT NULL,
|
||||
PRIMARY KEY (product, revision, sha256),
|
||||
FOREIGN KEY (product, revision) REFERENCES revisions (product, revision) ON DELETE CASCADE
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS revision_files_sha256 ON revision_files (sha256);
|
||||
`
|
||||
|
||||
// Open opens (or creates) the store in dir.
|
||||
func Open(dir string) (*Store, error) {
|
||||
if err := os.MkdirAll(filepath.Join(dir, "files"), 0o755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Leftovers of uploads interrupted by a crash.
|
||||
if err := os.RemoveAll(filepath.Join(dir, "tmp")); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(dir, "tmp"), 0o755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dsn := "file:" + filepath.ToSlash(filepath.Join(dir, "catalog.db")) +
|
||||
"?_pragma=busy_timeout(5000)&_pragma=journal_mode(WAL)&_pragma=foreign_keys(1)&_pragma=synchronous(NORMAL)"
|
||||
db, err := sql.Open("sqlite", dsn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
db.SetMaxOpenConns(1)
|
||||
if _, err := db.Exec(schema); err != nil {
|
||||
db.Close()
|
||||
return nil, fmt.Errorf("catalog.db: %w", err)
|
||||
}
|
||||
return &Store{dir: dir, db: db}, nil
|
||||
}
|
||||
|
||||
// Close closes the database.
|
||||
func (s *Store) Close() error { return s.db.Close() }
|
||||
|
||||
// TempDir is a folder on the store's file system for temporary files (emptied when the store opens).
|
||||
func (s *Store) TempDir() string { return filepath.Join(s.dir, "tmp") }
|
||||
|
||||
// Path returns where the file with this hash is (or would be) stored.
|
||||
func (s *Store) Path(hash string) string {
|
||||
return filepath.Join(s.dir, "files", hash[:2], hash)
|
||||
}
|
||||
|
||||
// PutFile stores data under its sha256 and returns the hash. A file already stored isn't written again.
|
||||
func (s *Store) PutFile(data []byte) (string, error) {
|
||||
sum := sha256.Sum256(data)
|
||||
hash := hex.EncodeToString(sum[:])
|
||||
path := s.Path(hash)
|
||||
if fi, err := os.Stat(path); err == nil && fi.Size() == int64(len(data)) {
|
||||
return hash, nil
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
return "", err
|
||||
}
|
||||
f, err := os.CreateTemp(filepath.Dir(path), ".tmp-*")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
tmp := f.Name()
|
||||
_, err = f.Write(data)
|
||||
if err == nil {
|
||||
err = f.Sync()
|
||||
}
|
||||
if cerr := f.Close(); err == nil {
|
||||
err = cerr
|
||||
}
|
||||
if err == nil {
|
||||
err = os.Rename(tmp, path)
|
||||
}
|
||||
if err != nil {
|
||||
os.Remove(tmp)
|
||||
return "", err
|
||||
}
|
||||
return hash, nil
|
||||
}
|
||||
|
||||
// Open opens a stored file.
|
||||
func (s *Store) Open(hash string) (*os.File, error) {
|
||||
if !ValidHash(hash) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
f, err := os.Open(s.Path(hash))
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
return f, err
|
||||
}
|
||||
|
||||
// AddRevision records a revision as hosted. Its files must be stored already. Recording the same revision again
|
||||
// replaces it.
|
||||
func (s *Store) AddRevision(r Revision) error {
|
||||
if r.Rating != General && r.Rating != Adult {
|
||||
return fmt.Errorf("rating %q", r.Rating)
|
||||
}
|
||||
files, err := json.Marshal(r.Files)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
if _, err := tx.Exec(`DELETE FROM revisions WHERE product = ? AND revision = ?`, r.Product, r.Revision); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(`INSERT INTO revisions (product, revision, rating, files) VALUES (?, ?, ?, ?)`,
|
||||
r.Product, r.Revision, r.Rating, string(files)); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, f := range r.Files {
|
||||
if _, err := tx.Exec(`INSERT OR IGNORE INTO revision_files (product, revision, sha256) VALUES (?, ?, ?)`,
|
||||
r.Product, r.Revision, f.SHA256); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// Revision returns a hosted revision.
|
||||
func (s *Store) Revision(product string, revision int) (*Revision, error) {
|
||||
r := Revision{Product: product, Revision: revision}
|
||||
var files string
|
||||
err := s.db.QueryRow(`SELECT rating, files FROM revisions WHERE product = ? AND revision = ?`, product, revision).
|
||||
Scan(&r.Rating, &files)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := json.Unmarshal([]byte(files), &r.Files); err != nil {
|
||||
return nil, fmt.Errorf("revision %s/%d: %w", product, revision, err)
|
||||
}
|
||||
return &r, nil
|
||||
}
|
||||
|
||||
// FileAccess tells whether a file belongs to a hosted revision (hosted) and whether every such revision is adult
|
||||
// (adultOnly). Files in no hosted revision (an upload the hub refused) aren't served.
|
||||
func (s *Store) FileAccess(hash string) (hosted, adultOnly bool, err error) {
|
||||
var n, general int
|
||||
err = s.db.QueryRow(`
|
||||
SELECT count(*), coalesce(sum(r.rating = 'general'), 0)
|
||||
FROM revision_files rf JOIN revisions r ON r.product = rf.product AND r.revision = rf.revision
|
||||
WHERE rf.sha256 = ?`, hash).Scan(&n, &general)
|
||||
if err != nil {
|
||||
return false, false, err
|
||||
}
|
||||
return n > 0, n > 0 && general == 0, nil
|
||||
}
|
||||
Reference in new issue
Block a user