t5de-catalog: store and serve community product files

A Go server that takes product uploads with hub upload tokens, checks them by
t5de-api's pack rules, stores files by sha256, registers revisions with the hub
(POST /api/v1/catalog/revisions), and serves files with immutable caching;
files used only by adult revisions need an adult access token. Contract:
t5de-hub's docs/products.md.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

internal/server/server.go

@@ -0,0 +1,194 @@
+// Package server is the catalog's HTTP API (t5de-hub's docs/products.md, "Catalog API").
+package server
+
+import (
+	"encoding/json"
+	"errors"
+	"log"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"git.t5de.dev/t5de/t5de-catalog/internal/hub"
+	"git.t5de.dev/t5de/t5de-catalog/internal/store"
+	"git.t5de.dev/t5de/t5de-catalog/internal/token"
+)
+
+// Config is what the server needs.
+type Config struct {
+	Store     *store.Store
+	Verifier  *token.Verifier
+	Hub       *hub.Client
+	CatalogID string
+	WebOrigin string // the hub website's origin, allowed to upload from browsers (CORS); empty for none
+	Logger    *log.Logger
+	// MaxUploads is how many uploads are checked at once (each can hold up to 256 MiB of files in memory);
+	// default 2.
+	MaxUploads int
+}
+
+// Server serves the catalog's API.
+type Server struct {
+	cfg     Config
+	uploads chan struct{}
+	mux     *http.ServeMux
+}
+
+// New returns the catalog's API.
+func New(cfg Config) *Server {
+	if cfg.MaxUploads < 1 {
+		cfg.MaxUploads = 2
+	}
+	if cfg.Logger == nil {
+		cfg.Logger = log.Default()
+	}
+	cfg.WebOrigin = strings.TrimRight(cfg.WebOrigin, "/")
+	s := &Server{cfg: cfg, uploads: make(chan struct{}, cfg.MaxUploads), mux: http.NewServeMux()}
+	s.mux.HandleFunc("GET /health", s.health)
+	s.mux.HandleFunc("OPTIONS /api/v1/uploads", s.preflight)
+	s.mux.HandleFunc("POST /api/v1/uploads", s.upload)
+	s.mux.HandleFunc("GET /api/v1/files/{sha256}", s.file)
+	s.mux.HandleFunc("GET /api/v1/revisions/{product}/{revision}", s.revision)
+	return s
+}
+
+// ServeHTTP serves a request and logs it.
+func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	start := time.Now()
+	rec := &recorder{ResponseWriter: w, status: http.StatusOK}
+	s.mux.ServeHTTP(rec, r)
+	s.cfg.Logger.Printf("%s %s %d %dB %s", r.Method, r.URL.Path, rec.status, rec.bytes, time.Since(start).Round(time.Millisecond))
+}
+
+type recorder struct {
+	http.ResponseWriter
+	status int
+	bytes  int64
+	wrote  bool
+}
+
+func (r *recorder) WriteHeader(status int) {
+	if !r.wrote {
+		r.status, r.wrote = status, true
+	}
+	r.ResponseWriter.WriteHeader(status)
+}
+
+func (r *recorder) Write(b []byte) (int, error) {
+	r.wrote = true
+	n, err := r.ResponseWriter.Write(b)
+	r.bytes += int64(n)
+	return n, err
+}
+
+func (r *recorder) Unwrap() http.ResponseWriter { return r.ResponseWriter }
+
+func writeJSON(w http.ResponseWriter, status int, v any) {
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	w.WriteHeader(status)
+	json.NewEncoder(w).Encode(v)
+}
+
+func writeError(w http.ResponseWriter, status int, code, message string) {
+	if w.Header().Get("Cache-Control") == "" {
+		w.Header().Set("Cache-Control", "no-store")
+	}
+	writeJSON(w, status, map[string]string{"error": code, "message": message})
+}
+
+func (s *Server) health(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Cache-Control", "no-store")
+	writeJSON(w, http.StatusOK, map[string]bool{"ok": true})
+}
+
+// bearer returns the request's bearer token, if any.
+func bearer(r *http.Request) string {
+	scheme, tok, ok := strings.Cut(r.Header.Get("Authorization"), " ")
+	if !ok || !strings.EqualFold(scheme, "Bearer") {
+		return ""
+	}
+	return strings.TrimSpace(tok)
+}
+
+func (s *Server) file(w http.ResponseWriter, r *http.Request) {
+	hash := r.PathValue("sha256")
+	if !store.ValidHash(hash) {
+		writeError(w, http.StatusNotFound, "not_found", "No such file.")
+		return
+	}
+	hosted, adultOnly, err := s.cfg.Store.FileAccess(hash)
+	if err != nil {
+		s.cfg.Logger.Printf("files: %s: %v", hash, err)
+		writeError(w, http.StatusInternalServerError, "internal", "Something went wrong.")
+		return
+	}
+	if !hosted {
+		writeError(w, http.StatusNotFound, "not_found", "No such file.")
+		return
+	}
+	h := w.Header()
+	if adultOnly {
+		h.Set("Cache-Control", "private, no-store")
+		h.Set("Vary", "Authorization")
+		tok := bearer(r)
+		if tok == "" {
+			h.Set("WWW-Authenticate", `Bearer realm="t5de-catalog"`)
+			writeError(w, http.StatusUnauthorized, "token_required", "This file needs an access token.")
+			return
+		}
+		c, err := s.cfg.Verifier.Verify(r.Context(), tok, token.Access)
+		if err != nil {
+			h.Set("WWW-Authenticate", `Bearer realm="t5de-catalog", error="invalid_token"`)
+			writeError(w, http.StatusUnauthorized, "bad_token", "The access token isn't valid.")
+			return
+		}
+		if !c.Adult {
+			writeError(w, http.StatusForbidden, "adult_only", "This file is for adults only.")
+			return
+		}
+	}
+	f, err := s.cfg.Store.Open(hash)
+	if errors.Is(err, store.ErrNotFound) {
+		s.cfg.Logger.Printf("files: %s is in a hosted revision but missing on disk", hash)
+		writeError(w, http.StatusNotFound, "not_found", "No such file.")
+		return
+	}
+	if err != nil {
+		s.cfg.Logger.Printf("files: %s: %v", hash, err)
+		writeError(w, http.StatusInternalServerError, "internal", "Something went wrong.")
+		return
+	}
+	defer f.Close()
+	if adultOnly {
+		h.Set("Cache-Control", "private, max-age=31536000, immutable")
+	} else {
+		h.Set("Cache-Control", "public, max-age=31536000, immutable")
+	}
+	h.Set("ETag", `"`+hash+`"`)
+	h.Set("Content-Type", "application/octet-stream")
+	h.Set("X-Content-Type-Options", "nosniff")
+	http.ServeContent(w, r, "", time.Time{}, f)
+}
+
+func (s *Server) revision(w http.ResponseWriter, r *http.Request) {
+	product := r.PathValue("product")
+	n, err := strconv.Atoi(r.PathValue("revision"))
+	if !hub.ValidProduct(product) || err != nil || n < 1 {
+		writeError(w, http.StatusNotFound, "not_found", "No such revision.")
+		return
+	}
+	rev, err := s.cfg.Store.Revision(product, n)
+	if errors.Is(err, store.ErrNotFound) {
+		writeError(w, http.StatusNotFound, "not_found", "No such revision.")
+		return
+	}
+	if err != nil {
+		s.cfg.Logger.Printf("revisions: %s/%d: %v", product, n, err)
+		writeError(w, http.StatusInternalServerError, "internal", "Something went wrong.")
+		return
+	}
+	// Revisions never change, but moderation can remove one.
+	w.Header().Set("Cache-Control", "public, max-age=3600")
+	writeJSON(w, http.StatusOK, rev)
+}

internal/server/server_test.go

@@ -0,0 +1,514 @@
+package server_test
+
+import (
+	"archive/zip"
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"mime/multipart"
+	"net/http"
+	"net/http/httptest"
+	"slices"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"git.t5de.dev/t5de/t5de-catalog/internal/hub"
+	"git.t5de.dev/t5de/t5de-catalog/internal/server"
+	"git.t5de.dev/t5de/t5de-catalog/internal/store"
+	"git.t5de.dev/t5de/t5de-catalog/internal/token"
+	"git.t5de.dev/t5de/t5de-catalog/internal/token/tokentest"
+)
+
+const (
+	catalogID  = "cat1"
+	catalogKey = "secret-key"
+	webOrigin  = "https://t5de.example"
+)
+
+// fakeHub serves a JWKS, issues tokens and accepts revisions as docs/products.md says.
+type fakeHub struct {
+	*httptest.Server
+	t   *testing.T
+	key *tokentest.Key
+
+	mu       sync.Mutex
+	down     bool
+	products map[string][][]store.File // product -> file list of each revision
+	last     *hub.NewRevision
+	calls    int
+	next     int
+}
+
+func newFakeHub(t *testing.T) *fakeHub {
+	h := &fakeHub{t: t, key: tokentest.NewKey("hubkey"), products: map[string][][]store.File{}}
+	h.Server = httptest.NewServer(http.HandlerFunc(h.serve))
+	t.Cleanup(h.Close)
+	return h
+}
+
+func (h *fakeHub) serve(w http.ResponseWriter, r *http.Request) {
+	switch {
+	case r.Method == http.MethodGet && r.URL.Path == token.KeysPath:
+		w.Header().Set("Content-Type", "application/json")
+		w.Write(tokentest.JWKS(h.key))
+	case r.Method == http.MethodPost && r.URL.Path == "/api/v1/catalog/revisions":
+		h.revisions(w, r)
+	default:
+		http.NotFound(w, r)
+	}
+}
+
+func refuse(w http.ResponseWriter, status int, code, msg string) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(status)
+	json.NewEncoder(w).Encode(map[string]string{"error": code, "message": msg})
+}
+
+func (h *fakeHub) revisions(w http.ResponseWriter, r *http.Request) {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	h.calls++
+	if h.down {
+		http.Error(w, "down", http.StatusServiceUnavailable)
+		return
+	}
+	if r.Header.Get("Authorization") != "Catalog "+catalogID+":"+catalogKey {
+		refuse(w, http.StatusUnauthorized, "bad_catalog", "Unknown catalog.")
+		return
+	}
+	var req hub.NewRevision
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		refuse(w, http.StatusBadRequest, "invalid", err.Error())
+		return
+	}
+	h.last = &req
+	v := token.NewVerifier(h.URL, h.Client())
+	c, err := v.Verify(r.Context(), req.UploadToken, token.Upload)
+	if err != nil || c.Catalog != catalogID {
+		refuse(w, http.StatusForbidden, "bad_token", "Not for this catalog.")
+		return
+	}
+	product := c.Product
+	if product == "" {
+		h.next++
+		product = fmt.Sprintf("prod%06d", h.next)
+		product = strings.NewReplacer("0", "a", "1", "b", "8", "c", "9", "d").Replace(product)
+	} else {
+		revs := h.products[product]
+		if revs == nil {
+			refuse(w, http.StatusNotFound, "not_found", "No such product.")
+			return
+		}
+		if req.Notes == "" {
+			refuse(w, http.StatusBadRequest, "notes_required", "Notes are required.")
+			return
+		}
+		if slices.Equal(revs[len(revs)-1], req.Files) {
+			refuse(w, http.StatusConflict, "unchanged", "These files are the same as the latest revision's.")
+			return
+		}
+	}
+	h.products[product] = append(h.products[product], req.Files)
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(http.StatusCreated)
+	json.NewEncoder(w).Encode(hub.Registered{Product: product, Revision: len(h.products[product])})
+}
+
+func (h *fakeHub) token(typ string, claims map[string]any) string {
+	c := map[string]any{"iss": h.URL, "typ": typ, "sub": "person1", "iat": time.Now().Unix(), "exp": time.Now().Add(30 * time.Minute).Unix()}
+	for k, v := range claims {
+		c[k] = v
+	}
+	return h.key.Sign(c)
+}
+
+func (h *fakeHub) uploadToken(product string) string {
+	c := map[string]any{"catalog": catalogID}
+	if product != "" {
+		c["product"] = product
+	}
+	return h.token(token.Upload, c)
+}
+
+type env struct {
+	hub     *fakeHub
+	catalog *httptest.Server
+	store   *store.Store
+}
+
+func setup(t *testing.T) *env {
+	h := newFakeHub(t)
+	st, err := store.Open(t.TempDir())
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { st.Close() })
+	srv := server.New(server.Config{
+		Store:     st,
+		Verifier:  token.NewVerifier(h.URL, h.Client()),
+		Hub:       hub.New(h.URL, catalogID, catalogKey, h.Client()),
+		CatalogID: catalogID,
+		WebOrigin: webOrigin,
+		Logger:    log.New(io.Discard, "", 0),
+	})
+	c := httptest.NewServer(srv)
+	t.Cleanup(c.Close)
+	return &env{hub: h, catalog: c, store: st}
+}
+
+type file struct{ name, data string }
+
+func indexXML(rating, name string) string {
+	return `<?xml version="1.0" encoding="utf-8"?>
+<Template>
+  <ProductName>` + name + `</ProductName>
+  <ProductRating>` + rating + `</ProductRating>
+  <ProductGender>female</ProductGender>
+  <__DATAIMPORT>product://12345/index.xml</__DATAIMPORT>
+</Template>
+`
+}
+
+func makeZip(t *testing.T, files ...file) []byte {
+	var b bytes.Buffer
+	z := zip.NewWriter(&b)
+	for _, f := range files {
+		w, err := z.Create(f.name)
+		if err != nil {
+			t.Fatal(err)
+		}
+		w.Write([]byte(f.data))
+	}
+	if err := z.Close(); err != nil {
+		t.Fatal(err)
+	}
+	return b.Bytes()
+}
+
+// texture is a tiny synthetic "PNG": the catalog doesn't decode images.
+const texture = "\x89PNG\r\n\x1a\nnot really an image, just test bytes"
+
+type answer struct {
+	status int
+	header http.Header
+	body   map[string]any
+	raw    []byte
+}
+
+func (e *env) upload(t *testing.T, tok string, zipData []byte, notes *string) answer {
+	t.Helper()
+	var b bytes.Buffer
+	mw := multipart.NewWriter(&b)
+	if notes != nil {
+		mw.WriteField("notes", *notes)
+	}
+	if zipData != nil {
+		fw, _ := mw.CreateFormFile("file", "product.chkn")
+		fw.Write(zipData)
+	}
+	mw.Close()
+	req, _ := http.NewRequest(http.MethodPost, e.catalog.URL+"/api/v1/uploads", &b)
+	req.Header.Set("Content-Type", mw.FormDataContentType())
+	req.Header.Set("Origin", webOrigin)
+	if tok != "" {
+		req.Header.Set("Authorization", "Bearer "+tok)
+	}
+	return do(t, req)
+}
+
+func do(t *testing.T, req *http.Request) answer {
+	t.Helper()
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer resp.Body.Close()
+	raw, _ := io.ReadAll(resp.Body)
+	a := answer{status: resp.StatusCode, header: resp.Header, raw: raw}
+	json.Unmarshal(raw, &a.body)
+	return a
+}
+
+func (e *env) get(t *testing.T, path string, headers ...string) answer {
+	t.Helper()
+	req, _ := http.NewRequest(http.MethodGet, e.catalog.URL+path, nil)
+	for i := 0; i+1 < len(headers); i += 2 {
+		req.Header.Set(headers[i], headers[i+1])
+	}
+	return do(t, req)
+}
+
+func sum(s string) string {
+	h := sha256.Sum256([]byte(s))
+	return hex.EncodeToString(h[:])
+}
+
+func ptr(s string) *string { return &s }
+
+func TestUploadAndServe(t *testing.T) {
+	e := setup(t)
+	index := indexXML("GA", "Test Hat")
+	a := e.upload(t, e.hub.uploadToken(""), makeZip(t, file{"tex.png", texture}, file{"index.xml", index}), nil)
+	if a.status != http.StatusCreated {
+		t.Fatalf("upload: %d %s", a.status, a.raw)
+	}
+	if a.header.Get("Access-Control-Allow-Origin") != webOrigin {
+		t.Errorf("no CORS header on the upload's answer")
+	}
+	product, _ := a.body["product"].(string)
+	if !hub.ValidProduct(product) || a.body["revision"] != 1.0 {
+		t.Fatalf("answer %s", a.raw)
+	}
+
+	// What the hub was sent.
+	got := e.hub.last
+	wantFiles := []store.File{
+		{Name: "index.xml", SHA256: sum(index), Size: int64(len(index))},
+		{Name: "tex.png", SHA256: sum(texture), Size: int64(len(texture))},
+	}
+	if !slices.Equal(got.Files, wantFiles) || got.Rating != "general" || got.Parent != 12345 || got.Name != "Test Hat" || got.Notes != "" {
+		t.Fatalf("hub got %+v", got)
+	}
+
+	// Files come back, cacheable, with ranges and ETags.
+	f := e.get(t, "/api/v1/files/"+sum(texture))
+	if f.status != http.StatusOK || string(f.raw) != texture {
+		t.Fatalf("file: %d %q", f.status, f.raw)
+	}
+	if cc := f.header.Get("Cache-Control"); cc != "public, max-age=31536000, immutable" {
+		t.Errorf("Cache-Control %q", cc)
+	}
+	if et := f.header.Get("ETag"); et != `"`+sum(texture)+`"` {
+		t.Errorf("ETag %q", et)
+	}
+	if r := e.get(t, "/api/v1/files/"+sum(texture), "Range", "bytes=0-3"); r.status != http.StatusPartialContent || string(r.raw) != texture[:4] {
+		t.Errorf("range: %d %q", r.status, r.raw)
+	}
+	if r := e.get(t, "/api/v1/files/"+sum(texture), "If-None-Match", `"`+sum(texture)+`"`); r.status != http.StatusNotModified {
+		t.Errorf("If-None-Match: %d", r.status)
+	}
+	if r := e.get(t, "/api/v1/files/"+sum(index)); r.status != http.StatusOK || string(r.raw) != index {
+		t.Errorf("index.xml: %d", r.status)
+	}
+
+	// The revision.
+	rev := e.get(t, fmt.Sprintf("/api/v1/revisions/%s/1", product))
+	var r store.Revision
+	if err := json.Unmarshal(rev.raw, &r); rev.status != http.StatusOK || err != nil {
+		t.Fatalf("revision: %d %s", rev.status, rev.raw)
+	}
+	if r.Product != product || r.Revision != 1 || r.Rating != "general" || !slices.Equal(r.Files, wantFiles) {
+		t.Fatalf("revision %+v", r)
+	}
+	if rev := e.get(t, fmt.Sprintf("/api/v1/revisions/%s/2", product)); rev.status != http.StatusNotFound {
+		t.Errorf("missing revision: %d", rev.status)
+	}
+	if rev := e.get(t, "/api/v1/revisions/NOT-VALID/1"); rev.status != http.StatusNotFound {
+		t.Errorf("bad product: %d", rev.status)
+	}
+
+	// Not found.
+	if r := e.get(t, "/api/v1/files/"+sum("nothing")); r.status != http.StatusNotFound {
+		t.Errorf("unknown file: %d", r.status)
+	}
+	if r := e.get(t, "/api/v1/files/abc"); r.status != http.StatusNotFound {
+		t.Errorf("bad hash: %d", r.status)
+	}
+
+	// A new revision needs notes (checked before storing anything or calling the hub).
+	newIndex := indexXML("GA", "Test Hat")
+	tex2 := texture + " v2"
+	zip2 := makeZip(t, file{"index.xml", newIndex}, file{"tex.png", tex2})
+	calls := e.hub.calls
+	if a := e.upload(t, e.hub.uploadToken(product), zip2, nil); a.status != http.StatusBadRequest || a.body["error"] != "notes_required" {
+		t.Fatalf("no notes: %d %s", a.status, a.raw)
+	}
+	if e.hub.calls != calls {
+		t.Fatal("hub called without notes")
+	}
+	a = e.upload(t, e.hub.uploadToken(product), zip2, ptr("A brighter texture."))
+	if a.status != http.StatusCreated || a.body["product"] != product || a.body["revision"] != 2.0 {
+		t.Fatalf("revision 2: %d %s", a.status, a.raw)
+	}
+	if e.hub.last.Notes != "A brighter texture." {
+		t.Errorf("notes %q", e.hub.last.Notes)
+	}
+
+	// The hub refusing (the same files again) is a 502 with its error.
+	a = e.upload(t, e.hub.uploadToken(product), zip2, ptr("Again."))
+	if a.status != http.StatusBadGateway || a.body["error"] != "unchanged" {
+		t.Fatalf("unchanged: %d %s", a.status, a.raw)
+	}
+
+	// The hub unreachable: 502, and the stored files aren't served (no hosted revision has them).
+	e.hub.mu.Lock()
+	e.hub.down = true
+	e.hub.mu.Unlock()
+	tex3 := texture + " v3"
+	a = e.upload(t, e.hub.uploadToken(""), makeZip(t, file{"index.xml", index}, file{"tex.png", tex3}), nil)
+	if a.status != http.StatusBadGateway {
+		t.Fatalf("hub down: %d %s", a.status, a.raw)
+	}
+	if r := e.get(t, "/api/v1/files/"+sum(tex3)); r.status != http.StatusNotFound {
+		t.Errorf("file of a refused upload: %d", r.status)
+	}
+}
+
+func TestAdultFiles(t *testing.T) {
+	e := setup(t)
+	index := indexXML("AP", "Adult Thing")
+	adultTex := texture + " adult"
+	a := e.upload(t, e.hub.uploadToken(""), makeZip(t, file{"index.xml", index}, file{"a.png", adultTex}, file{"shared.png", texture}), nil)
+	if a.status != http.StatusCreated {
+		t.Fatalf("upload: %d %s", a.status, a.raw)
+	}
+	if e.hub.last.Rating != "adult" {
+		t.Fatalf("rating %q", e.hub.last.Rating)
+	}
+	path := "/api/v1/files/" + sum(adultTex)
+
+	r := e.get(t, path)
+	if r.status != http.StatusUnauthorized || !strings.HasPrefix(r.header.Get("Cache-Control"), "private") {
+		t.Fatalf("no token: %d %q", r.status, r.header.Get("Cache-Control"))
+	}
+	r = e.get(t, path, "Authorization", "Bearer garbage")
+	if r.status != http.StatusUnauthorized {
+		t.Fatalf("bad token: %d", r.status)
+	}
+	r = e.get(t, path, "Authorization", "Bearer "+e.hub.uploadToken(""))
+	if r.status != http.StatusUnauthorized {
+		t.Fatalf("upload token as access token: %d", r.status)
+	}
+	r = e.get(t, path, "Authorization", "Bearer "+e.hub.token(token.Access, map[string]any{"adult": false}))
+	if r.status != http.StatusForbidden || !strings.HasPrefix(r.header.Get("Cache-Control"), "private") {
+		t.Fatalf("not adult: %d %q", r.status, r.header.Get("Cache-Control"))
+	}
+	r = e.get(t, path, "Authorization", "Bearer "+e.hub.token(token.Access, map[string]any{"adult": true}))
+	if r.status != http.StatusOK || string(r.raw) != adultTex {
+		t.Fatalf("adult: %d %q", r.status, r.raw)
+	}
+	if cc := r.header.Get("Cache-Control"); !strings.HasPrefix(cc, "private") {
+		t.Errorf("adult file's Cache-Control %q", cc)
+	}
+
+	// shared.png also belongs to a general revision once one uses it: then it's public.
+	if r := e.get(t, "/api/v1/files/"+sum(texture)); r.status != http.StatusUnauthorized {
+		t.Fatalf("shared file before a general revision: %d", r.status)
+	}
+	a = e.upload(t, e.hub.uploadToken(""), makeZip(t, file{"index.xml", indexXML("GA", "Plain")}, file{"shared.png", texture}), nil)
+	if a.status != http.StatusCreated {
+		t.Fatalf("general upload: %d %s", a.status, a.raw)
+	}
+	if r := e.get(t, "/api/v1/files/"+sum(texture)); r.status != http.StatusOK {
+		t.Fatalf("shared file after a general revision: %d", r.status)
+	}
+}
+
+func TestBadUploads(t *testing.T) {
+	e := setup(t)
+	good := makeZip(t, file{"index.xml", indexXML("GA", "Hat")}, file{"tex.png", texture})
+
+	tokens := map[string]string{
+		"no token":      "",
+		"garbage":       "garbage",
+		"other catalog": e.hub.token(token.Upload, map[string]any{"catalog": "cat2"}),
+		"access token":  e.hub.token(token.Access, map[string]any{"adult": true}),
+		"expired":       e.hub.token(token.Upload, map[string]any{"catalog": catalogID, "exp": time.Now().Add(-time.Hour).Unix()}),
+		"wrong issuer":  e.hub.token(token.Upload, map[string]any{"catalog": catalogID, "iss": "https://other.example"}),
+		"other key":     tokentest.NewKey("hubkey").Sign(map[string]any{"iss": e.hub.URL, "typ": token.Upload, "catalog": catalogID, "exp": time.Now().Add(time.Hour).Unix()}),
+	}
+	for name, tok := range tokens {
+		if a := e.upload(t, tok, good, nil); a.status != http.StatusUnauthorized || a.body["error"] != "bad_token" {
+			t.Errorf("%s: %d %s", name, a.status, a.raw)
+		}
+	}
+
+	big := strings.Repeat("x", 100)
+	zips := map[string][]byte{
+		"not a zip":         []byte("this is not a zip file"),
+		"no index.xml":      makeZip(t, file{"tex.png", texture}),
+		"bad rating":        makeZip(t, file{"index.xml", indexXML("XX", "Hat")}),
+		"no rating":         makeZip(t, file{"index.xml", "<Template><ProductName>Hat</ProductName></Template>"}),
+		"not a template":    makeZip(t, file{"index.xml", "<Other/>"}),
+		"file in a folder":  makeZip(t, file{"index.xml", indexXML("GA", "Hat")}, file{"sub/tex.png", texture}),
+		"_contents.json":    makeZip(t, file{"index.xml", indexXML("GA", "Hat")}, file{"_contents.json", "{}"}),
+		"duplicate name":    makeZip(t, file{"index.xml", indexXML("GA", "Hat")}, file{"a.png", big}, file{"a.png", big}),
+		"non-Latin-1 name":  makeZip(t, file{"index.xml", indexXML("GA", "Hat")}, file{"一.png", big}),
+		"bad __DATAIMPORT":  makeZip(t, file{"index.xml", strings.Replace(indexXML("GA", "Hat"), "product://12345/index.xml", "http://x", 1)}),
+		"index.xml nested":  makeZip(t, file{"x/index.xml", indexXML("GA", "Hat")}),
+		"backslash in name": makeZip(t, file{"index.xml", indexXML("GA", "Hat")}, file{`a\b.png`, big}),
+	}
+	for name, z := range zips {
+		a := e.upload(t, e.hub.uploadToken(""), z, nil)
+		t.Logf("%s: %s", name, a.body["message"])
+		if a.status != http.StatusBadRequest || a.body["error"] != "bad_files" {
+			t.Errorf("%s: %d %s", name, a.status, a.raw)
+		}
+	}
+	if a := e.upload(t, e.hub.uploadToken(""), nil, ptr("notes but no file")); a.status != http.StatusBadRequest {
+		t.Errorf("no file: %d %s", a.status, a.raw)
+	}
+	req, _ := http.NewRequest(http.MethodPost, e.catalog.URL+"/api/v1/uploads", strings.NewReader("{}"))
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Authorization", "Bearer "+e.hub.uploadToken(""))
+	if a := do(t, req); a.status != http.StatusBadRequest {
+		t.Errorf("not multipart: %d %s", a.status, a.raw)
+	}
+	if e.hub.calls != 0 {
+		t.Errorf("hub called %d times for bad uploads", e.hub.calls)
+	}
+}
+
+// A zip whose files are larger than its headers say is cut off by the pack rules' counted reads.
+func TestUploadSizeCountedWhileReading(t *testing.T) {
+	e := setup(t)
+	// A file over the 64 MiB limit, compressed down to a small upload.
+	var b bytes.Buffer
+	z := zip.NewWriter(&b)
+	w, _ := z.Create("index.xml")
+	w.Write([]byte(indexXML("GA", "Hat")))
+	w, _ = z.Create("huge.png")
+	chunk := make([]byte, 1<<20)
+	for range 65 {
+		w.Write(chunk)
+	}
+	z.Close()
+	a := e.upload(t, e.hub.uploadToken(""), b.Bytes(), nil)
+	if a.status != http.StatusBadRequest || !strings.Contains(a.body["message"].(string), "64 MiB") {
+		t.Fatalf("huge file: %d %s", a.status, a.raw)
+	}
+}
+
+func TestCORSAndHealth(t *testing.T) {
+	e := setup(t)
+	preflight := func(origin string) http.Header {
+		req, _ := http.NewRequest(http.MethodOptions, e.catalog.URL+"/api/v1/uploads", nil)
+		req.Header.Set("Origin", origin)
+		req.Header.Set("Access-Control-Request-Method", "POST")
+		req.Header.Set("Access-Control-Request-Headers", "authorization")
+		a := do(t, req)
+		if a.status != http.StatusNoContent {
+			t.Fatalf("preflight: %d", a.status)
+		}
+		return a.header
+	}
+	h := preflight(webOrigin)
+	if h.Get("Access-Control-Allow-Origin") != webOrigin || !strings.Contains(h.Get("Access-Control-Allow-Headers"), "Authorization") ||
+		h.Get("Access-Control-Allow-Methods") != "POST" {
+		t.Errorf("preflight headers: %v", h)
+	}
+	if h := preflight("https://evil.example"); h.Get("Access-Control-Allow-Origin") != "" {
+		t.Errorf("other origin allowed: %v", h)
+	}
+
+	a := e.get(t, "/health")
+	if a.status != http.StatusOK || a.body["ok"] != true {
+		t.Errorf("health: %d %s", a.status, a.raw)
+	}
+}

internal/server/upload.go

@@ -0,0 +1,299 @@
+package server
+
+import (
+	"archive/zip"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"mime"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"unicode/utf8"
+
+	"git.t5de.dev/t5de/t5de-api/pack"
+
+	"git.t5de.dev/t5de/t5de-catalog/internal/hub"
+	"git.t5de.dev/t5de/t5de-catalog/internal/store"
+	"git.t5de.dev/t5de/t5de-catalog/internal/token"
+)
+
+// MaxBody caps an upload request: the zip (whose files may hold pack.MaxTotalSize uncompressed) and the notes.
+const MaxBody = 300 << 20
+
+// MaxNotes caps the revision notes.
+const MaxNotes = 64 << 10
+
+// corsHeaders adds CORS headers for the hub website's origin.
+func (s *Server) corsHeaders(w http.ResponseWriter, r *http.Request) bool {
+	h := w.Header()
+	h.Add("Vary", "Origin")
+	origin := r.Header.Get("Origin")
+	if origin == "" || s.cfg.WebOrigin == "" || origin != s.cfg.WebOrigin {
+		return false
+	}
+	h.Set("Access-Control-Allow-Origin", origin)
+	return true
+}
+
+func (s *Server) preflight(w http.ResponseWriter, r *http.Request) {
+	if s.corsHeaders(w, r) {
+		h := w.Header()
+		h.Add("Vary", "Access-Control-Request-Method")
+		h.Add("Vary", "Access-Control-Request-Headers")
+		h.Set("Access-Control-Allow-Methods", "POST")
+		h.Set("Access-Control-Allow-Headers", "Authorization, Content-Type")
+		h.Set("Access-Control-Max-Age", "600")
+	}
+	w.WriteHeader(http.StatusNoContent)
+}
+
+// upload takes a product (docs/products.md, POST /api/v1/uploads): checks the token and the files, stores the
+// files, registers the revision with the hub, and records it as hosted once the hub has accepted it.
+func (s *Server) upload(w http.ResponseWriter, r *http.Request) {
+	s.corsHeaders(w, r)
+	tok := bearer(r)
+	if tok == "" {
+		writeError(w, http.StatusUnauthorized, "bad_token", "An upload token is needed.")
+		return
+	}
+	claims, err := s.cfg.Verifier.Verify(r.Context(), tok, token.Upload)
+	if err != nil {
+		s.cfg.Logger.Printf("uploads: %v", err)
+		writeError(w, http.StatusUnauthorized, "bad_token", "The upload token isn't valid. Start the upload again.")
+		return
+	}
+	if claims.Catalog != s.cfg.CatalogID {
+		writeError(w, http.StatusUnauthorized, "bad_token", "The upload token is for another catalog.")
+		return
+	}
+	if claims.Product != "" && !hub.ValidProduct(claims.Product) {
+		writeError(w, http.StatusUnauthorized, "bad_token", "The upload token names no valid product.")
+		return
+	}
+
+	select {
+	case s.uploads <- struct{}{}:
+		defer func() { <-s.uploads }()
+	case <-r.Context().Done():
+		return
+	}
+
+	tmp, err := os.MkdirTemp(s.cfg.Store.TempDir(), "upload-*")
+	if err != nil {
+		s.cfg.Logger.Printf("uploads: %v", err)
+		writeError(w, http.StatusInternalServerError, "internal", "Something went wrong.")
+		return
+	}
+	defer os.RemoveAll(tmp)
+
+	r.Body = http.MaxBytesReader(w, r.Body, MaxBody)
+	notes, status, code, msg := readForm(r, filepath.Join(tmp, pack.ChknName))
+	if status != 0 {
+		if status == http.StatusInternalServerError {
+			s.cfg.Logger.Printf("uploads: %s", msg)
+			msg = "Something went wrong."
+		}
+		writeError(w, status, code, msg)
+		return
+	}
+	if claims.Product != "" && notes == "" {
+		writeError(w, http.StatusBadRequest, "notes_required", "Say what changed in this revision.")
+		return
+	}
+
+	p, rating, err := checkProduct(tmp)
+	if err != nil {
+		writeError(w, http.StatusBadRequest, "bad_files", "These product files can't be used: "+err.Error())
+		return
+	}
+
+	files := make([]store.File, len(p.Files))
+	for i, f := range p.Files {
+		hash, err := s.cfg.Store.PutFile(f.Data)
+		if err != nil {
+			s.cfg.Logger.Printf("uploads: storing %q: %v", f.Name, err)
+			writeError(w, http.StatusInternalServerError, "internal", "Something went wrong.")
+			return
+		}
+		files[i] = store.File{Name: f.Name, SHA256: hash, Size: int64(len(f.Data))}
+	}
+	index := p.Index
+	p = nil // the files' data can go before the hub answers
+
+	reg, err := s.cfg.Hub.RegisterRevision(r.Context(), hub.NewRevision{
+		UploadToken: tok, Files: files, Rating: rating, Parent: index.Parent, Name: strings.TrimSpace(index.Name), Notes: notes,
+	})
+	if err != nil {
+		var refused *hub.Error
+		if errors.As(err, &refused) {
+			s.cfg.Logger.Printf("uploads: %v", err)
+			code, msg := refused.Code, refused.Message
+			if code == "" {
+				code = "hub_refused"
+			}
+			if msg == "" {
+				msg = fmt.Sprintf("T5DE's hub refused the upload (%d).", refused.Status)
+			}
+			writeError(w, http.StatusBadGateway, code, msg)
+			return
+		}
+		s.cfg.Logger.Printf("uploads: registering with the hub: %v", err)
+		writeError(w, http.StatusBadGateway, "hub_unavailable", "T5DE's hub couldn't be reached. Try again later.")
+		return
+	}
+	if claims.Product != "" && reg.Product != claims.Product {
+		s.cfg.Logger.Printf("uploads: the hub registered product %s, but the token was for %s", reg.Product, claims.Product)
+		writeError(w, http.StatusBadGateway, "hub_failed", "T5DE's hub answered for another product.")
+		return
+	}
+	if err := s.cfg.Store.AddRevision(store.Revision{Product: reg.Product, Revision: reg.Revision, Rating: rating, Files: files}); err != nil {
+		s.cfg.Logger.Printf("uploads: recording %s/%d: %v", reg.Product, reg.Revision, err)
+		writeError(w, http.StatusInternalServerError, "internal", "Something went wrong.")
+		return
+	}
+	s.cfg.Logger.Printf("uploads: %s/%d (%s, %d files) from %s", reg.Product, reg.Revision, rating, len(files), claims.Subject)
+	writeJSON(w, http.StatusCreated, reg)
+}
+
+// readForm reads the upload's multipart form: the zip (part "file") into chkn, and the notes (part "notes").
+// Other parts are skipped. A non-zero status is an error to answer.
+func readForm(r *http.Request, chkn string) (notes string, status int, code, msg string) {
+	mt, _, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
+	if err != nil || mt != "multipart/form-data" {
+		return "", http.StatusBadRequest, "bad_request", "Send the product as multipart/form-data."
+	}
+	mr, err := r.MultipartReader()
+	if err != nil {
+		return "", http.StatusBadRequest, "bad_request", "Send the product as multipart/form-data."
+	}
+	tooLarge := func(err error) bool {
+		var mbe *http.MaxBytesError
+		return errors.As(err, &mbe)
+	}
+	haveFile, haveNotes := false, false
+	for {
+		part, err := mr.NextPart()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			if tooLarge(err) {
+				return "", http.StatusRequestEntityTooLarge, "too_large", fmt.Sprintf("Uploads can be at most %d MiB.", MaxBody>>20)
+			}
+			return "", http.StatusBadRequest, "bad_request", "The upload is cut short or malformed."
+		}
+		switch part.FormName() {
+		case "file":
+			if haveFile {
+				return "", http.StatusBadRequest, "bad_request", "Send one file."
+			}
+			haveFile = true
+			f, err := os.Create(chkn)
+			if err != nil {
+				return "", http.StatusInternalServerError, "internal", err.Error()
+			}
+			_, err = io.Copy(f, part)
+			if cerr := f.Close(); err == nil && cerr != nil {
+				return "", http.StatusInternalServerError, "internal", cerr.Error()
+			}
+			if err != nil {
+				if tooLarge(err) {
+					return "", http.StatusRequestEntityTooLarge, "too_large", fmt.Sprintf("Uploads can be at most %d MiB.", MaxBody>>20)
+				}
+				return "", http.StatusBadRequest, "bad_request", "The upload is cut short or malformed."
+			}
+		case "notes":
+			if haveNotes {
+				return "", http.StatusBadRequest, "bad_request", "Send the notes once."
+			}
+			haveNotes = true
+			data, err := io.ReadAll(io.LimitReader(part, MaxNotes+1))
+			if err != nil {
+				if tooLarge(err) {
+					return "", http.StatusRequestEntityTooLarge, "too_large", fmt.Sprintf("Uploads can be at most %d MiB.", MaxBody>>20)
+				}
+				return "", http.StatusBadRequest, "bad_request", "The upload is cut short or malformed."
+			}
+			if len(data) > MaxNotes {
+				return "", http.StatusBadRequest, "bad_request", fmt.Sprintf("The notes can be at most %d KiB.", MaxNotes>>10)
+			}
+			if !utf8.Valid(data) {
+				return "", http.StatusBadRequest, "bad_request", "The notes aren't valid UTF-8."
+			}
+			notes = strings.TrimSpace(string(data))
+		default:
+			if _, err := io.Copy(io.Discard, part); err != nil {
+				if tooLarge(err) {
+					return "", http.StatusRequestEntityTooLarge, "too_large", fmt.Sprintf("Uploads can be at most %d MiB.", MaxBody>>20)
+				}
+				return "", http.StatusBadRequest, "bad_request", "The upload is cut short or malformed."
+			}
+		}
+		part.Close()
+	}
+	if !haveFile {
+		return "", http.StatusBadRequest, "bad_request", "No file: send the product's .chkn as \"file\"."
+	}
+	return notes, 0, "", ""
+}
+
+// ratings maps index.xml's ProductRating to the hub's ratings.
+var ratings = map[string]string{"GA": store.General, "AP": store.Adult}
+
+// checkProduct checks the zip dir/product.chkn by the pack rules (t5de-api's pack package): it reads the rating
+// from its index.xml, writes a manifest with that rating beside it, and loads dir as a pack. It returns the pack
+// and its rating.
+func checkProduct(dir string) (*pack.Pack, string, error) {
+	index, err := readIndex(filepath.Join(dir, pack.ChknName))
+	if err != nil {
+		return nil, "", err
+	}
+	rating := ratings[index.Rating]
+	manifest, err := json.Marshal(pack.Manifest{Format: pack.Format, ID: "upload", Revision: 1, Name: "upload", Rating: rating})
+	if err != nil {
+		return nil, "", err
+	}
+	if err := os.WriteFile(filepath.Join(dir, pack.ManifestName), manifest, 0o644); err != nil {
+		return nil, "", err
+	}
+	p, err := pack.Load(dir)
+	if err != nil {
+		return nil, "", err
+	}
+	return p, rating, nil
+}
+
+// readIndex reads index.xml from the zip at path, its size counted while reading.
+func readIndex(path string) (*pack.Index, error) {
+	z, err := zip.OpenReader(path)
+	if err != nil {
+		return nil, errors.New("not a zip file")
+	}
+	defer z.Close()
+	for _, f := range z.File {
+		if f.Name != pack.IndexName {
+			continue
+		}
+		rc, err := f.Open()
+		if err != nil {
+			return nil, fmt.Errorf("%s: %v", pack.IndexName, err)
+		}
+		data, err := io.ReadAll(io.LimitReader(rc, pack.MaxFileSize+1))
+		rc.Close()
+		if err != nil {
+			return nil, fmt.Errorf("%s: %v", pack.IndexName, err)
+		}
+		if len(data) > pack.MaxFileSize {
+			return nil, fmt.Errorf("%s: over %d MiB", pack.IndexName, pack.MaxFileSize>>20)
+		}
+		index, err := pack.ParseIndex(data)
+		if err != nil {
+			return nil, fmt.Errorf("%s: %v", pack.IndexName, err)
+		}
+		return index, nil
+	}
+	return nil, fmt.Errorf("no %s at the top of the zip", pack.IndexName)
+}