// Package store keeps the catalog's files and the revisions it hosts: files on disk named by their sha256 // (files//), revisions in SQLite (catalog.db). package store import ( "crypto/sha256" "database/sql" "encoding/hex" "encoding/json" "errors" "fmt" "io/fs" "os" "path/filepath" "regexp" _ "modernc.org/sqlite" ) // Ratings. const ( General = "general" Adult = "adult" ) // File is one entry of a revision's file list. type File struct { Name string `json:"name"` SHA256 string `json:"sha256"` Size int64 `json:"size"` } // Revision is a revision the catalog hosts. type Revision struct { Product string `json:"product"` Revision int `json:"revision"` Rating string `json:"rating"` Files []File `json:"files"` // sorted by name } // ErrNotFound is returned for files and revisions the catalog doesn't host. var ErrNotFound = errors.New("not found") var hashPattern = regexp.MustCompile(`^[0-9a-f]{64}$`) // ValidHash reports whether s is a lowercase hex sha256. func ValidHash(s string) bool { return hashPattern.MatchString(s) } // Store is the catalog's data folder. type Store struct { dir string db *sql.DB } const schema = ` CREATE TABLE IF NOT EXISTS revisions ( product TEXT NOT NULL, revision INTEGER NOT NULL, rating TEXT NOT NULL CHECK (rating IN ('general', 'adult')), files TEXT NOT NULL, -- the file list, JSON created_at INTEGER NOT NULL DEFAULT (unixepoch()), PRIMARY KEY (product, revision) ); CREATE TABLE IF NOT EXISTS revision_files ( product TEXT NOT NULL, revision INTEGER NOT NULL, sha256 TEXT NOT NULL, PRIMARY KEY (product, revision, sha256), FOREIGN KEY (product, revision) REFERENCES revisions (product, revision) ON DELETE CASCADE ); CREATE INDEX IF NOT EXISTS revision_files_sha256 ON revision_files (sha256); ` // Open opens (or creates) the store in dir. func Open(dir string) (*Store, error) { if err := os.MkdirAll(filepath.Join(dir, "files"), 0o755); err != nil { return nil, err } // Leftovers of uploads interrupted by a crash. if err := os.RemoveAll(filepath.Join(dir, "tmp")); err != nil { return nil, err } if err := os.MkdirAll(filepath.Join(dir, "tmp"), 0o755); err != nil { return nil, err } dsn := "file:" + filepath.ToSlash(filepath.Join(dir, "catalog.db")) + "?_pragma=busy_timeout(5000)&_pragma=journal_mode(WAL)&_pragma=foreign_keys(1)&_pragma=synchronous(NORMAL)" db, err := sql.Open("sqlite", dsn) if err != nil { return nil, err } db.SetMaxOpenConns(1) if _, err := db.Exec(schema); err != nil { db.Close() return nil, fmt.Errorf("catalog.db: %w", err) } return &Store{dir: dir, db: db}, nil } // Close closes the database. func (s *Store) Close() error { return s.db.Close() } // TempDir is a folder on the store's file system for temporary files (emptied when the store opens). func (s *Store) TempDir() string { return filepath.Join(s.dir, "tmp") } // Path returns where the file with this hash is (or would be) stored. func (s *Store) Path(hash string) string { return filepath.Join(s.dir, "files", hash[:2], hash) } // PutFile stores data under its sha256 and returns the hash. A file already stored isn't written again. func (s *Store) PutFile(data []byte) (string, error) { sum := sha256.Sum256(data) hash := hex.EncodeToString(sum[:]) path := s.Path(hash) if fi, err := os.Stat(path); err == nil && fi.Size() == int64(len(data)) { return hash, nil } if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { return "", err } f, err := os.CreateTemp(filepath.Dir(path), ".tmp-*") if err != nil { return "", err } tmp := f.Name() _, err = f.Write(data) if err == nil { err = f.Sync() } if cerr := f.Close(); err == nil { err = cerr } if err == nil { err = os.Rename(tmp, path) } if err != nil { os.Remove(tmp) return "", err } return hash, nil } // Open opens a stored file. func (s *Store) Open(hash string) (*os.File, error) { if !ValidHash(hash) { return nil, ErrNotFound } f, err := os.Open(s.Path(hash)) if errors.Is(err, fs.ErrNotExist) { return nil, ErrNotFound } return f, err } // AddRevision records a revision as hosted. Its files must be stored already. Recording the same revision again // replaces it. func (s *Store) AddRevision(r Revision) error { if r.Rating != General && r.Rating != Adult { return fmt.Errorf("rating %q", r.Rating) } files, err := json.Marshal(r.Files) if err != nil { return err } tx, err := s.db.Begin() if err != nil { return err } defer tx.Rollback() if _, err := tx.Exec(`DELETE FROM revisions WHERE product = ? AND revision = ?`, r.Product, r.Revision); err != nil { return err } if _, err := tx.Exec(`INSERT INTO revisions (product, revision, rating, files) VALUES (?, ?, ?, ?)`, r.Product, r.Revision, r.Rating, string(files)); err != nil { return err } for _, f := range r.Files { if _, err := tx.Exec(`INSERT OR IGNORE INTO revision_files (product, revision, sha256) VALUES (?, ?, ?)`, r.Product, r.Revision, f.SHA256); err != nil { return err } } return tx.Commit() } // Revision returns a hosted revision. func (s *Store) Revision(product string, revision int) (*Revision, error) { r := Revision{Product: product, Revision: revision} var files string err := s.db.QueryRow(`SELECT rating, files FROM revisions WHERE product = ? AND revision = ?`, product, revision). Scan(&r.Rating, &files) if errors.Is(err, sql.ErrNoRows) { return nil, ErrNotFound } if err != nil { return nil, err } if err := json.Unmarshal([]byte(files), &r.Files); err != nil { return nil, fmt.Errorf("revision %s/%d: %w", product, revision, err) } return &r, nil } // FileAccess tells whether a file belongs to a hosted revision (hosted) and whether every such revision is adult // (adultOnly). Files in no hosted revision (an upload the hub refused) aren't served. func (s *Store) FileAccess(hash string) (hosted, adultOnly bool, err error) { var n, general int err = s.db.QueryRow(` SELECT count(*), coalesce(sum(r.rating = 'general'), 0) FROM revision_files rf JOIN revisions r ON r.product = rf.product AND r.revision = rf.revision WHERE rf.sha256 = ?`, hash).Scan(&n, &general) if err != nil { return false, false, err } return n > 0, n > 0 && general == 0, nil }