commit 8742829e772b8aea55fcd8ccad946bb9d3f626cf Author: dhkatz Date: Thu Oct 29 18:59:16 2026 -0700 t5de-catalog: store and serve community product files A Go server that takes product uploads with hub upload tokens, checks them by t5de-api's pack rules, stores files by sha256, registers revisions with the hub (POST /api/v1/catalog/revisions), and serves files with immutable caching; files used only by adult revisions need an adult access token. Contract: t5de-hub's docs/products.md. Co-Authored-By: Claude Opus 5.5 diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..b3f4a08 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +/dist/ +/data/ +.git diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..48d029a --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +# gofmt requires LF; keep sources LF on every platform. +* text=auto eol=lf +*.exe binary diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml new file mode 100644 index 0000000..d5b5aee --- /dev/null +++ b/.gitea/workflows/ci.yml @@ -0,0 +1,37 @@ +# Tests the catalog and builds t5de-catalog for Linux (static) and its Docker image. +name: CI + +on: + push: + branches: [main] + tags: ["v*"] + pull_request: + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + env: + # T5DE's modules (t5de-api) come straight from its Gitea, not through Go's public module proxy. + GOPRIVATE: git.t5de.dev + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Test + run: | + unformatted=$(gofmt -l .) + if [ -n "$unformatted" ]; then echo "Not gofmt-formatted: $unformatted"; exit 1; fi + go vet ./... + go test ./... + + # Static and reproducible: no cgo (the SQLite driver is pure Go), no paths, no build ID. + - name: Build t5de-catalog + run: | + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -buildvcs=false \ + -ldflags "-s -w -buildid= -X git.t5de.dev/t5de/t5de-catalog/internal/buildinfo.Version=${GITHUB_SHA::12}" \ + -o dist/t5de-catalog ./cmd/t5de-catalog + sha256sum dist/t5de-catalog diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6559156 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/dist/ +/data/ diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..8133711 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,12 @@ +# t5de-catalog + +T5DE's catalog server (see README.md). Part of T5DE; the contract with the hub is t5de-hub's `docs/products.md` +(git.t5de.dev/t5de/t5de-hub, usually checked out beside this one), product file rules are t5de-api's `pack` +package (`docs/pack.md`). + +## Rules + +- Never commit IMVU's files: tests make their product files in code (a minimal `index.xml`, fake textures). +- Never commit secrets (catalog keys, tokens). +- Commits: author "dhkatz "; never the user's real name or email. +- Follow `docs/products.md` exactly; change it there (in t5de-hub) first if the contract needs to change. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..fd6e182 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,26 @@ +# t5de-catalog in a container: a static build on distroless. Data (catalog.db, files/) goes in the /data volume. +# +# docker build -t t5de-catalog . +# docker run -p 4102:4102 -v catalog-data:/data -e HUB_URL=... -e CATALOG_ID=... -e CATALOG_KEY=... \ +# -e CATALOG_PUBLIC_URL=... -e WEB_ORIGIN=... t5de-catalog +FROM golang:1.27-alpine AS build +RUN apk add --no-cache git +# T5DE's modules (t5de-api) come straight from its Gitea. +ENV GOPRIVATE=git.t5de.dev CGO_ENABLED=0 +ARG VERSION=dev +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY . . +RUN go build -trimpath -buildvcs=false \ + -ldflags "-s -w -buildid= -X git.t5de.dev/t5de/t5de-catalog/internal/buildinfo.Version=${VERSION}" \ + -o /out/t5de-catalog ./cmd/t5de-catalog \ + && mkdir /out/data + +FROM gcr.io/distroless/static-debian12:nonroot +COPY --from=build /out/t5de-catalog /t5de-catalog +COPY --from=build --chown=nonroot:nonroot /out/data /data +ENV CATALOG_ADDR=0.0.0.0:4102 CATALOG_DATA=/data +VOLUME /data +EXPOSE 4102 +ENTRYPOINT ["/t5de-catalog"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..7a41f76 --- /dev/null +++ b/README.md @@ -0,0 +1,75 @@ +# t5de-catalog + +T5DE's catalog server: it stores community products' files by sha256, serves them, and registers uploads with +T5DE's hub (git.t5de.dev/t5de/t5de-hub). Creators upload from the hub's website straight to a catalog, so large +files never pass through the hub; T5DE's proxy then downloads the files from the catalogs the hub lists. Anyone +can run a catalog and ask the hub to approve it. + +The contract (tokens, this API, the hub's API for catalogs) is t5de-hub's `docs/products.md`; the design and the +reasons are in its `docs/design.md` ("Products and catalogs", "Catalogs (`t5de-catalog`)"). + +Unofficial: not made by or affiliated with IMVU. + +## How it works + +- **Uploads** (`POST /api/v1/uploads`): the creator's browser sends the product as a zip (Create Mode's `.chkn`, + files at the root) with an upload token from the hub. The catalog checks the token (signed by the hub, for this + catalog), checks the files by t5de-api's pack rules (names, limits counted while reading, `index.xml` with + `ProductRating` GA or AP), stores each file under its sha256, and registers the revision with the hub + (`POST /api/v1/catalog/revisions`, `Authorization: Catalog :`). Only once the hub accepts it is the + revision recorded as hosted; files of a refused upload stay on disk but aren't served. +- **Files** (`GET /api/v1/files/`): immutable, so cached for a year (`ETag` is the hash; ranges work). A + file used only by adult revisions needs an access token from the hub with `adult: true`, and isn't cached by + shared caches. +- **Tokens** are EdDSA (Ed25519) JWTs; the hub's keys come from `HUB_URL/.well-known/t5de-keys.json`, cached and + fetched again when a token names an unknown key (at most every 30 s). +- **Storage**: `CATALOG_DATA/catalog.db` (SQLite: the revisions hosted and which files belong to them) and + `CATALOG_DATA/files//`, written whole (temp file, then rename). `CATALOG_DATA/tmp` holds + uploads being checked and is emptied at start. + +## API + +| | | +| --- | --- | +| `GET /health` | `{ok: true}` | +| `POST /api/v1/uploads` | `Authorization: Bearer `; multipart: `file` (the zip), `notes` (required for a new revision of an existing product). 201 `{product, revision}`. Errors are `{error, message}`: 400 bad files or form (`bad_files`, `notes_required`, `bad_request`), 401 bad token (`bad_token`), 413 over 300 MiB (`too_large`), 502 the hub refused (its error code, e.g. `unchanged`) or couldn't be reached (`hub_unavailable`). CORS for `WEB_ORIGIN`. | +| `GET /api/v1/files/` | the file; 401/403 for adult-only files without an adult access token; 404 if not hosted | +| `GET /api/v1/revisions//` | `{product, revision, rating, files}` for revisions hosted here | + +## Running + +Register the catalog on the hub's website (`POST /api/v1/catalogs`) to get its ID and key, and have a hub +moderator approve it. Then, for a hub running locally: + + HUB_URL=http://localhost:4101 \ + CATALOG_ID= CATALOG_KEY= \ + CATALOG_PUBLIC_URL=http://localhost:4102 \ + WEB_ORIGIN=http://localhost:4100 \ + CATALOG_DATA=./data \ + go run ./cmd/t5de-catalog + +| Variable | | +| --- | --- | +| `CATALOG_ADDR` | listen address (default `127.0.0.1:4102`) | +| `CATALOG_DATA` | data folder (default `./data`) | +| `CATALOG_PUBLIC_URL` | the catalog's public URL, as registered on the hub (logged) | +| `HUB_URL` | the hub's API; also the tokens' issuer (required) | +| `CATALOG_ID`, `CATALOG_KEY` | from registering on the hub (required) | +| `WEB_ORIGIN` | the hub website's origin, allowed to upload from browsers | + +It logs to standard error and stops cleanly on Ctrl+C or SIGTERM, finishing requests in progress. Put it behind +a TLS-terminating reverse proxy (or a CDN for `/api/v1/files/`) in production. + +With Docker: + + docker build -t t5de-catalog . + docker run -p 4102:4102 -v catalog-data:/data -e HUB_URL=... -e CATALOG_ID=... -e CATALOG_KEY=... \ + -e CATALOG_PUBLIC_URL=... -e WEB_ORIGIN=... t5de-catalog + +## Building + + go test ./... + CGO_ENABLED=0 go build -trimpath -o dist/t5de-catalog ./cmd/t5de-catalog + +The SQLite driver is pure Go (modernc.org/sqlite), so builds are static. t5de-api comes from git.t5de.dev +directly: set `GOPRIVATE=git.t5de.dev`. diff --git a/cmd/t5de-catalog/main.go b/cmd/t5de-catalog/main.go new file mode 100644 index 0000000..5bd26da --- /dev/null +++ b/cmd/t5de-catalog/main.go @@ -0,0 +1,130 @@ +// Command t5de-catalog is T5DE's catalog server: it stores community product files by sha256, serves them, and +// registers uploads with T5DE's hub (t5de-hub's docs/products.md). Configured by environment variables: +// +// CATALOG_ADDR listen address (default 127.0.0.1:4102) +// CATALOG_DATA data folder: catalog.db and files/ (default ./data) +// CATALOG_PUBLIC_URL the catalog's public URL, as registered on the hub +// HUB_URL the hub's API (required), e.g. https://api.t5de.dev +// CATALOG_ID the catalog's ID on the hub (required) +// CATALOG_KEY its key (required) +// WEB_ORIGIN the hub website's origin, allowed to upload from browsers (CORS) +// +// It logs to standard error and stops cleanly on SIGINT or SIGTERM. +package main + +import ( + "context" + "errors" + "fmt" + "log" + "net" + "net/http" + "net/url" + "os" + "os/signal" + "strings" + "syscall" + "time" + + "git.t5de.dev/t5de/t5de-catalog/internal/buildinfo" + "git.t5de.dev/t5de/t5de-catalog/internal/hub" + "git.t5de.dev/t5de/t5de-catalog/internal/server" + "git.t5de.dev/t5de/t5de-catalog/internal/store" + "git.t5de.dev/t5de/t5de-catalog/internal/token" +) + +func main() { + if len(os.Args) > 1 && (os.Args[1] == "-version" || os.Args[1] == "--version") { + fmt.Println(buildinfo.Version) + return + } + logger := log.New(os.Stderr, "", log.Ldate|log.Ltime|log.Lmicroseconds) + if err := run(logger); err != nil { + logger.Printf("t5de-catalog: %v", err) + os.Exit(1) + } +} + +func env(name, def string) string { + if v := strings.TrimSpace(os.Getenv(name)); v != "" { + return v + } + return def +} + +func run(logger *log.Logger) error { + addr := env("CATALOG_ADDR", "127.0.0.1:4102") + data := env("CATALOG_DATA", "data") + publicURL := env("CATALOG_PUBLIC_URL", "") + hubURL := strings.TrimRight(env("HUB_URL", ""), "/") + id := env("CATALOG_ID", "") + key := env("CATALOG_KEY", "") + webOrigin := strings.TrimRight(env("WEB_ORIGIN", ""), "/") + + var missing []string + for _, v := range [][2]string{{"HUB_URL", hubURL}, {"CATALOG_ID", id}, {"CATALOG_KEY", key}} { + if v[1] == "" { + missing = append(missing, v[0]) + } + } + if len(missing) > 0 { + return fmt.Errorf("set %s", strings.Join(missing, ", ")) + } + if u, err := url.Parse(hubURL); err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { + return fmt.Errorf("HUB_URL %q isn't an http(s) URL", hubURL) + } + if publicURL == "" { + logger.Printf("CATALOG_PUBLIC_URL isn't set") + } + if webOrigin == "" { + logger.Printf("WEB_ORIGIN isn't set: browsers can't upload") + } + + st, err := store.Open(data) + if err != nil { + return err + } + defer st.Close() + + srv := &http.Server{ + Addr: addr, + Handler: server.New(server.Config{ + Store: st, + Verifier: token.NewVerifier(hubURL, nil), + Hub: hub.New(hubURL, id, key, nil), + CatalogID: id, + WebOrigin: webOrigin, + Logger: logger, + }), + ReadHeaderTimeout: 10 * time.Second, + ReadTimeout: 15 * time.Minute, // large uploads on slow connections + WriteTimeout: 15 * time.Minute, + IdleTimeout: 2 * time.Minute, + ErrorLog: logger, + } + ln, err := net.Listen("tcp", addr) + if err != nil { + return err + } + logger.Printf("t5de-catalog %s: catalog %s on %s (public URL %s, hub %s, data %s)", buildinfo.Version, id, ln.Addr(), publicURL, hubURL, data) + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + served := make(chan error, 1) + go func() { served <- srv.Serve(ln) }() + select { + case err := <-served: + return err + case <-ctx.Done(): + } + logger.Printf("stopping") + shutdown, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + if err := srv.Shutdown(shutdown); err != nil && !errors.Is(err, context.DeadlineExceeded) { + return err + } + if err := <-served; !errors.Is(err, http.ErrServerClosed) { + return err + } + return nil +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..105fa59 --- /dev/null +++ b/go.mod @@ -0,0 +1,20 @@ +module git.t5de.dev/t5de/t5de-catalog + +go 1.27.0 + +require ( + git.t5de.dev/t5de/t5de-api v0.0.0-20261029101713-106b4fcc6064 + modernc.org/sqlite v1.61.0 +) + +require ( + github.com/dustin/go-humanize v1.1.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect + github.com/ncruces/go-strftime v1.1.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + golang.org/x/sys v0.49.0 // indirect + modernc.org/libc v1.79.0 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.12.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..449bab7 --- /dev/null +++ b/go.sum @@ -0,0 +1,52 @@ +git.t5de.dev/t5de/t5de-api v0.0.0-20261029101713-106b4fcc6064 h1:UdRdoSTTZHjUTW7wF9+EVJ7UFgBktXrls3FsMKiOp2Y= +git.t5de.dev/t5de/t5de-api v0.0.0-20261029101713-106b4fcc6064/go.mod h1:mRXJdl43TzJN3JlUeVhi6BSqle37+ALmqzxqTYchXLw= +github.com/dustin/go-humanize v1.1.0 h1:dbKTrvD0klcbBV/h4AWJdMuZogJACoMlvWIWZ5b2xWg= +github.com/dustin/go-humanize v1.1.0/go.mod h1:hc1CvRkJMsgxqjmjMQF3QNRAZBwY8AXBAzKYoSX9sFI= +github.com/google/pprof v0.0.0-20261016150608-1facf8f91b50 h1:PlNWJRxu5APSV0pWqfIsZ0qt9OOGHSWfZQ4zvx+WgWs= +github.com/google/pprof v0.0.0-20261016150608-1facf8f91b50/go.mod h1:yUo1AMMpsPIDFGuwDK8nLcf8+vvAd5l2O6JDZ6hpwrM= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= +github.com/ncruces/go-strftime v1.1.0 h1:vtPmPWMkb539RGV9WpWT9EwJGvrOCKvMNI4obWL47Hk= +github.com/ncruces/go-strftime v1.1.0/go.mod h1:yVaq7iwCN09vHRrKBrrflXT0M9tTMG2KmZ9Fj7xeG/g= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +golang.org/x/mod v0.42.0 h1:ICeL3t2L30ljSPHHxDnR4fqVcBt/zR2zJAb9juavmQg= +golang.org/x/mod v0.42.0/go.mod h1:u5fzIrRInkL8MaO1cbRkUHT64v303YIT5KEqpNr7jRs= +golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA= +golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= +golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/tools v0.52.0 h1:ryxg4U63oaUYk7uOLTF37n43f4qgtGvqV4Ioit1BaEs= +golang.org/x/tools v0.52.0/go.mod h1:ItcP5UNaVGG+Yc4eIYmaFLNJ4soEopkx5Y2d4lcEurs= +modernc.org/cc/v4 v4.30.3 h1:aDq8gIX1X7+VFgO/JYYji6pLWL2H4tv3g8WatT3KCys= +modernc.org/cc/v4 v4.30.3/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.37.0 h1:BAuJCUtIHIxFIYaFhqXIw5YaoMBlNCUlFovejHfMip4= +modernc.org/ccgo/v4 v4.37.0/go.mod h1:/8627VhwWtAbqUXmPORxFQU5SgLrAx2jl9seLw9GBhY= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.2.0 h1:/Y3LGDMQaoYf55PebP1qsZ5FDGgAZyG1RYKEHeQzh1o= +modernc.org/gc/v3 v3.2.0/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.79.0 h1:9Dvvv++bAMSkSuBzuOFAatbTV5OIoq6Y6FohmD3e7q8= +modernc.org/libc v1.79.0/go.mod h1:uT3kR61NsbeKJSLqq83GJwSnZMVLlSjyAz4DfTaBy1M= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g= +modernc.org/memory v1.12.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.61.0 h1:sQi63oLQ9uquAUA4CzOisH35Q3pEszkPBuHM0tFL8CY= +modernc.org/sqlite v1.61.0/go.mod h1:f4gr5tbpx2umklaT/QVtbi/50hNdM+4ng+2zMo8KXJ0= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/internal/buildinfo/buildinfo.go b/internal/buildinfo/buildinfo.go new file mode 100644 index 0000000..354b593 --- /dev/null +++ b/internal/buildinfo/buildinfo.go @@ -0,0 +1,5 @@ +// Package buildinfo holds what release builds set with -ldflags "-X ...". +package buildinfo + +// Version is the release's version, "X.Y.Z" or "X.Y.Z-nightly.N+commit". +var Version = "dev" diff --git a/internal/hub/hub.go b/internal/hub/hub.go new file mode 100644 index 0000000..ca181f5 --- /dev/null +++ b/internal/hub/hub.go @@ -0,0 +1,113 @@ +// Package hub is the catalog's client for T5DE's hub (server to server, docs/products.md "For catalogs"). +package hub + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "regexp" + "strings" + "time" + + "git.t5de.dev/t5de/t5de-catalog/internal/store" +) + +// Client calls the hub as one catalog. +type Client struct { + url string + auth string + client *http.Client +} + +// New returns a client for the hub at hubURL (its API), as catalog id with its key. client may be nil. +func New(hubURL, id, key string, client *http.Client) *Client { + if client == nil { + client = &http.Client{Timeout: 60 * time.Second} + } + return &Client{url: strings.TrimRight(hubURL, "/"), auth: "Catalog " + id + ":" + key, client: client} +} + +// NewRevision is what the catalog registers. +type NewRevision struct { + UploadToken string `json:"uploadToken"` + Files []store.File `json:"files"` + Rating string `json:"rating"` + Parent int64 `json:"parent"` + Name string `json:"name"` + Notes string `json:"notes"` +} + +// Registered is the hub's answer. +type Registered struct { + Product string `json:"product"` + Revision int `json:"revision"` +} + +// Error is the hub refusing a request: its status and, when it sent them, its error code and message. +type Error struct { + Status int + Code string + Message string +} + +func (e *Error) Error() string { + if e.Code != "" { + return fmt.Sprintf("hub: %d %s: %s", e.Status, e.Code, e.Message) + } + return fmt.Sprintf("hub: %d", e.Status) +} + +var productID = regexp.MustCompile(`^[a-z2-7]{10}$`) + +// ValidProduct reports whether s is a product ID (10 of a-z2-7). +func ValidProduct(s string) bool { return productID.MatchString(s) } + +// RegisterRevision posts a revision to the hub (POST /api/v1/catalog/revisions). A refusal is an *Error; other +// errors mean the hub couldn't be reached or answered nonsense. +func (c *Client) RegisterRevision(ctx context.Context, r NewRevision) (*Registered, error) { + if r.Files == nil { + r.Files = []store.File{} + } + body, err := json.Marshal(r) + if err != nil { + return nil, err + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.url+"/api/v1/catalog/revisions", bytes.NewReader(body)) + if err != nil { + return nil, err + } + req.Header.Set("Authorization", c.auth) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + resp, err := c.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + data, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK { + e := &Error{Status: resp.StatusCode} + var msg struct { + Error string `json:"error"` + Message string `json:"message"` + } + if json.Unmarshal(data, &msg) == nil { + e.Code, e.Message = msg.Error, msg.Message + } + return nil, e + } + var reg Registered + if err := json.Unmarshal(data, ®); err != nil { + return nil, fmt.Errorf("hub: bad answer: %v", err) + } + if !ValidProduct(reg.Product) || reg.Revision < 1 { + return nil, fmt.Errorf("hub: bad answer: product %q, revision %d", reg.Product, reg.Revision) + } + return ®, nil +} diff --git a/internal/server/server.go b/internal/server/server.go new file mode 100644 index 0000000..ed62f9b --- /dev/null +++ b/internal/server/server.go @@ -0,0 +1,194 @@ +// Package server is the catalog's HTTP API (t5de-hub's docs/products.md, "Catalog API"). +package server + +import ( + "encoding/json" + "errors" + "log" + "net/http" + "strconv" + "strings" + "time" + + "git.t5de.dev/t5de/t5de-catalog/internal/hub" + "git.t5de.dev/t5de/t5de-catalog/internal/store" + "git.t5de.dev/t5de/t5de-catalog/internal/token" +) + +// Config is what the server needs. +type Config struct { + Store *store.Store + Verifier *token.Verifier + Hub *hub.Client + CatalogID string + WebOrigin string // the hub website's origin, allowed to upload from browsers (CORS); empty for none + Logger *log.Logger + // MaxUploads is how many uploads are checked at once (each can hold up to 256 MiB of files in memory); + // default 2. + MaxUploads int +} + +// Server serves the catalog's API. +type Server struct { + cfg Config + uploads chan struct{} + mux *http.ServeMux +} + +// New returns the catalog's API. +func New(cfg Config) *Server { + if cfg.MaxUploads < 1 { + cfg.MaxUploads = 2 + } + if cfg.Logger == nil { + cfg.Logger = log.Default() + } + cfg.WebOrigin = strings.TrimRight(cfg.WebOrigin, "/") + s := &Server{cfg: cfg, uploads: make(chan struct{}, cfg.MaxUploads), mux: http.NewServeMux()} + s.mux.HandleFunc("GET /health", s.health) + s.mux.HandleFunc("OPTIONS /api/v1/uploads", s.preflight) + s.mux.HandleFunc("POST /api/v1/uploads", s.upload) + s.mux.HandleFunc("GET /api/v1/files/{sha256}", s.file) + s.mux.HandleFunc("GET /api/v1/revisions/{product}/{revision}", s.revision) + return s +} + +// ServeHTTP serves a request and logs it. +func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { + start := time.Now() + rec := &recorder{ResponseWriter: w, status: http.StatusOK} + s.mux.ServeHTTP(rec, r) + s.cfg.Logger.Printf("%s %s %d %dB %s", r.Method, r.URL.Path, rec.status, rec.bytes, time.Since(start).Round(time.Millisecond)) +} + +type recorder struct { + http.ResponseWriter + status int + bytes int64 + wrote bool +} + +func (r *recorder) WriteHeader(status int) { + if !r.wrote { + r.status, r.wrote = status, true + } + r.ResponseWriter.WriteHeader(status) +} + +func (r *recorder) Write(b []byte) (int, error) { + r.wrote = true + n, err := r.ResponseWriter.Write(b) + r.bytes += int64(n) + return n, err +} + +func (r *recorder) Unwrap() http.ResponseWriter { return r.ResponseWriter } + +func writeJSON(w http.ResponseWriter, status int, v any) { + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.WriteHeader(status) + json.NewEncoder(w).Encode(v) +} + +func writeError(w http.ResponseWriter, status int, code, message string) { + if w.Header().Get("Cache-Control") == "" { + w.Header().Set("Cache-Control", "no-store") + } + writeJSON(w, status, map[string]string{"error": code, "message": message}) +} + +func (s *Server) health(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Cache-Control", "no-store") + writeJSON(w, http.StatusOK, map[string]bool{"ok": true}) +} + +// bearer returns the request's bearer token, if any. +func bearer(r *http.Request) string { + scheme, tok, ok := strings.Cut(r.Header.Get("Authorization"), " ") + if !ok || !strings.EqualFold(scheme, "Bearer") { + return "" + } + return strings.TrimSpace(tok) +} + +func (s *Server) file(w http.ResponseWriter, r *http.Request) { + hash := r.PathValue("sha256") + if !store.ValidHash(hash) { + writeError(w, http.StatusNotFound, "not_found", "No such file.") + return + } + hosted, adultOnly, err := s.cfg.Store.FileAccess(hash) + if err != nil { + s.cfg.Logger.Printf("files: %s: %v", hash, err) + writeError(w, http.StatusInternalServerError, "internal", "Something went wrong.") + return + } + if !hosted { + writeError(w, http.StatusNotFound, "not_found", "No such file.") + return + } + h := w.Header() + if adultOnly { + h.Set("Cache-Control", "private, no-store") + h.Set("Vary", "Authorization") + tok := bearer(r) + if tok == "" { + h.Set("WWW-Authenticate", `Bearer realm="t5de-catalog"`) + writeError(w, http.StatusUnauthorized, "token_required", "This file needs an access token.") + return + } + c, err := s.cfg.Verifier.Verify(r.Context(), tok, token.Access) + if err != nil { + h.Set("WWW-Authenticate", `Bearer realm="t5de-catalog", error="invalid_token"`) + writeError(w, http.StatusUnauthorized, "bad_token", "The access token isn't valid.") + return + } + if !c.Adult { + writeError(w, http.StatusForbidden, "adult_only", "This file is for adults only.") + return + } + } + f, err := s.cfg.Store.Open(hash) + if errors.Is(err, store.ErrNotFound) { + s.cfg.Logger.Printf("files: %s is in a hosted revision but missing on disk", hash) + writeError(w, http.StatusNotFound, "not_found", "No such file.") + return + } + if err != nil { + s.cfg.Logger.Printf("files: %s: %v", hash, err) + writeError(w, http.StatusInternalServerError, "internal", "Something went wrong.") + return + } + defer f.Close() + if adultOnly { + h.Set("Cache-Control", "private, max-age=31536000, immutable") + } else { + h.Set("Cache-Control", "public, max-age=31536000, immutable") + } + h.Set("ETag", `"`+hash+`"`) + h.Set("Content-Type", "application/octet-stream") + h.Set("X-Content-Type-Options", "nosniff") + http.ServeContent(w, r, "", time.Time{}, f) +} + +func (s *Server) revision(w http.ResponseWriter, r *http.Request) { + product := r.PathValue("product") + n, err := strconv.Atoi(r.PathValue("revision")) + if !hub.ValidProduct(product) || err != nil || n < 1 { + writeError(w, http.StatusNotFound, "not_found", "No such revision.") + return + } + rev, err := s.cfg.Store.Revision(product, n) + if errors.Is(err, store.ErrNotFound) { + writeError(w, http.StatusNotFound, "not_found", "No such revision.") + return + } + if err != nil { + s.cfg.Logger.Printf("revisions: %s/%d: %v", product, n, err) + writeError(w, http.StatusInternalServerError, "internal", "Something went wrong.") + return + } + // Revisions never change, but moderation can remove one. + w.Header().Set("Cache-Control", "public, max-age=3600") + writeJSON(w, http.StatusOK, rev) +} diff --git a/internal/server/server_test.go b/internal/server/server_test.go new file mode 100644 index 0000000..173573c --- /dev/null +++ b/internal/server/server_test.go @@ -0,0 +1,514 @@ +package server_test + +import ( + "archive/zip" + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "log" + "mime/multipart" + "net/http" + "net/http/httptest" + "slices" + "strings" + "sync" + "testing" + "time" + + "git.t5de.dev/t5de/t5de-catalog/internal/hub" + "git.t5de.dev/t5de/t5de-catalog/internal/server" + "git.t5de.dev/t5de/t5de-catalog/internal/store" + "git.t5de.dev/t5de/t5de-catalog/internal/token" + "git.t5de.dev/t5de/t5de-catalog/internal/token/tokentest" +) + +const ( + catalogID = "cat1" + catalogKey = "secret-key" + webOrigin = "https://t5de.example" +) + +// fakeHub serves a JWKS, issues tokens and accepts revisions as docs/products.md says. +type fakeHub struct { + *httptest.Server + t *testing.T + key *tokentest.Key + + mu sync.Mutex + down bool + products map[string][][]store.File // product -> file list of each revision + last *hub.NewRevision + calls int + next int +} + +func newFakeHub(t *testing.T) *fakeHub { + h := &fakeHub{t: t, key: tokentest.NewKey("hubkey"), products: map[string][][]store.File{}} + h.Server = httptest.NewServer(http.HandlerFunc(h.serve)) + t.Cleanup(h.Close) + return h +} + +func (h *fakeHub) serve(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == token.KeysPath: + w.Header().Set("Content-Type", "application/json") + w.Write(tokentest.JWKS(h.key)) + case r.Method == http.MethodPost && r.URL.Path == "/api/v1/catalog/revisions": + h.revisions(w, r) + default: + http.NotFound(w, r) + } +} + +func refuse(w http.ResponseWriter, status int, code, msg string) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + json.NewEncoder(w).Encode(map[string]string{"error": code, "message": msg}) +} + +func (h *fakeHub) revisions(w http.ResponseWriter, r *http.Request) { + h.mu.Lock() + defer h.mu.Unlock() + h.calls++ + if h.down { + http.Error(w, "down", http.StatusServiceUnavailable) + return + } + if r.Header.Get("Authorization") != "Catalog "+catalogID+":"+catalogKey { + refuse(w, http.StatusUnauthorized, "bad_catalog", "Unknown catalog.") + return + } + var req hub.NewRevision + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + refuse(w, http.StatusBadRequest, "invalid", err.Error()) + return + } + h.last = &req + v := token.NewVerifier(h.URL, h.Client()) + c, err := v.Verify(r.Context(), req.UploadToken, token.Upload) + if err != nil || c.Catalog != catalogID { + refuse(w, http.StatusForbidden, "bad_token", "Not for this catalog.") + return + } + product := c.Product + if product == "" { + h.next++ + product = fmt.Sprintf("prod%06d", h.next) + product = strings.NewReplacer("0", "a", "1", "b", "8", "c", "9", "d").Replace(product) + } else { + revs := h.products[product] + if revs == nil { + refuse(w, http.StatusNotFound, "not_found", "No such product.") + return + } + if req.Notes == "" { + refuse(w, http.StatusBadRequest, "notes_required", "Notes are required.") + return + } + if slices.Equal(revs[len(revs)-1], req.Files) { + refuse(w, http.StatusConflict, "unchanged", "These files are the same as the latest revision's.") + return + } + } + h.products[product] = append(h.products[product], req.Files) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + json.NewEncoder(w).Encode(hub.Registered{Product: product, Revision: len(h.products[product])}) +} + +func (h *fakeHub) token(typ string, claims map[string]any) string { + c := map[string]any{"iss": h.URL, "typ": typ, "sub": "person1", "iat": time.Now().Unix(), "exp": time.Now().Add(30 * time.Minute).Unix()} + for k, v := range claims { + c[k] = v + } + return h.key.Sign(c) +} + +func (h *fakeHub) uploadToken(product string) string { + c := map[string]any{"catalog": catalogID} + if product != "" { + c["product"] = product + } + return h.token(token.Upload, c) +} + +type env struct { + hub *fakeHub + catalog *httptest.Server + store *store.Store +} + +func setup(t *testing.T) *env { + h := newFakeHub(t) + st, err := store.Open(t.TempDir()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + srv := server.New(server.Config{ + Store: st, + Verifier: token.NewVerifier(h.URL, h.Client()), + Hub: hub.New(h.URL, catalogID, catalogKey, h.Client()), + CatalogID: catalogID, + WebOrigin: webOrigin, + Logger: log.New(io.Discard, "", 0), + }) + c := httptest.NewServer(srv) + t.Cleanup(c.Close) + return &env{hub: h, catalog: c, store: st} +} + +type file struct{ name, data string } + +func indexXML(rating, name string) string { + return ` + +` +} + +func makeZip(t *testing.T, files ...file) []byte { + var b bytes.Buffer + z := zip.NewWriter(&b) + for _, f := range files { + w, err := z.Create(f.name) + if err != nil { + t.Fatal(err) + } + w.Write([]byte(f.data)) + } + if err := z.Close(); err != nil { + t.Fatal(err) + } + return b.Bytes() +} + +// texture is a tiny synthetic "PNG": the catalog doesn't decode images. +const texture = "\x89PNG\r\n\x1a\nnot really an image, just test bytes" + +type answer struct { + status int + header http.Header + body map[string]any + raw []byte +} + +func (e *env) upload(t *testing.T, tok string, zipData []byte, notes *string) answer { + t.Helper() + var b bytes.Buffer + mw := multipart.NewWriter(&b) + if notes != nil { + mw.WriteField("notes", *notes) + } + if zipData != nil { + fw, _ := mw.CreateFormFile("file", "product.chkn") + fw.Write(zipData) + } + mw.Close() + req, _ := http.NewRequest(http.MethodPost, e.catalog.URL+"/api/v1/uploads", &b) + req.Header.Set("Content-Type", mw.FormDataContentType()) + req.Header.Set("Origin", webOrigin) + if tok != "" { + req.Header.Set("Authorization", "Bearer "+tok) + } + return do(t, req) +} + +func do(t *testing.T, req *http.Request) answer { + t.Helper() + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + raw, _ := io.ReadAll(resp.Body) + a := answer{status: resp.StatusCode, header: resp.Header, raw: raw} + json.Unmarshal(raw, &a.body) + return a +} + +func (e *env) get(t *testing.T, path string, headers ...string) answer { + t.Helper() + req, _ := http.NewRequest(http.MethodGet, e.catalog.URL+path, nil) + for i := 0; i+1 < len(headers); i += 2 { + req.Header.Set(headers[i], headers[i+1]) + } + return do(t, req) +} + +func sum(s string) string { + h := sha256.Sum256([]byte(s)) + return hex.EncodeToString(h[:]) +} + +func ptr(s string) *string { return &s } + +func TestUploadAndServe(t *testing.T) { + e := setup(t) + index := indexXML("GA", "Test Hat") + a := e.upload(t, e.hub.uploadToken(""), makeZip(t, file{"tex.png", texture}, file{"index.xml", index}), nil) + if a.status != http.StatusCreated { + t.Fatalf("upload: %d %s", a.status, a.raw) + } + if a.header.Get("Access-Control-Allow-Origin") != webOrigin { + t.Errorf("no CORS header on the upload's answer") + } + product, _ := a.body["product"].(string) + if !hub.ValidProduct(product) || a.body["revision"] != 1.0 { + t.Fatalf("answer %s", a.raw) + } + + // What the hub was sent. + got := e.hub.last + wantFiles := []store.File{ + {Name: "index.xml", SHA256: sum(index), Size: int64(len(index))}, + {Name: "tex.png", SHA256: sum(texture), Size: int64(len(texture))}, + } + if !slices.Equal(got.Files, wantFiles) || got.Rating != "general" || got.Parent != 12345 || got.Name != "Test Hat" || got.Notes != "" { + t.Fatalf("hub got %+v", got) + } + + // Files come back, cacheable, with ranges and ETags. + f := e.get(t, "/api/v1/files/"+sum(texture)) + if f.status != http.StatusOK || string(f.raw) != texture { + t.Fatalf("file: %d %q", f.status, f.raw) + } + if cc := f.header.Get("Cache-Control"); cc != "public, max-age=31536000, immutable" { + t.Errorf("Cache-Control %q", cc) + } + if et := f.header.Get("ETag"); et != `"`+sum(texture)+`"` { + t.Errorf("ETag %q", et) + } + if r := e.get(t, "/api/v1/files/"+sum(texture), "Range", "bytes=0-3"); r.status != http.StatusPartialContent || string(r.raw) != texture[:4] { + t.Errorf("range: %d %q", r.status, r.raw) + } + if r := e.get(t, "/api/v1/files/"+sum(texture), "If-None-Match", `"`+sum(texture)+`"`); r.status != http.StatusNotModified { + t.Errorf("If-None-Match: %d", r.status) + } + if r := e.get(t, "/api/v1/files/"+sum(index)); r.status != http.StatusOK || string(r.raw) != index { + t.Errorf("index.xml: %d", r.status) + } + + // The revision. + rev := e.get(t, fmt.Sprintf("/api/v1/revisions/%s/1", product)) + var r store.Revision + if err := json.Unmarshal(rev.raw, &r); rev.status != http.StatusOK || err != nil { + t.Fatalf("revision: %d %s", rev.status, rev.raw) + } + if r.Product != product || r.Revision != 1 || r.Rating != "general" || !slices.Equal(r.Files, wantFiles) { + t.Fatalf("revision %+v", r) + } + if rev := e.get(t, fmt.Sprintf("/api/v1/revisions/%s/2", product)); rev.status != http.StatusNotFound { + t.Errorf("missing revision: %d", rev.status) + } + if rev := e.get(t, "/api/v1/revisions/NOT-VALID/1"); rev.status != http.StatusNotFound { + t.Errorf("bad product: %d", rev.status) + } + + // Not found. + if r := e.get(t, "/api/v1/files/"+sum("nothing")); r.status != http.StatusNotFound { + t.Errorf("unknown file: %d", r.status) + } + if r := e.get(t, "/api/v1/files/abc"); r.status != http.StatusNotFound { + t.Errorf("bad hash: %d", r.status) + } + + // A new revision needs notes (checked before storing anything or calling the hub). + newIndex := indexXML("GA", "Test Hat") + tex2 := texture + " v2" + zip2 := makeZip(t, file{"index.xml", newIndex}, file{"tex.png", tex2}) + calls := e.hub.calls + if a := e.upload(t, e.hub.uploadToken(product), zip2, nil); a.status != http.StatusBadRequest || a.body["error"] != "notes_required" { + t.Fatalf("no notes: %d %s", a.status, a.raw) + } + if e.hub.calls != calls { + t.Fatal("hub called without notes") + } + a = e.upload(t, e.hub.uploadToken(product), zip2, ptr("A brighter texture.")) + if a.status != http.StatusCreated || a.body["product"] != product || a.body["revision"] != 2.0 { + t.Fatalf("revision 2: %d %s", a.status, a.raw) + } + if e.hub.last.Notes != "A brighter texture." { + t.Errorf("notes %q", e.hub.last.Notes) + } + + // The hub refusing (the same files again) is a 502 with its error. + a = e.upload(t, e.hub.uploadToken(product), zip2, ptr("Again.")) + if a.status != http.StatusBadGateway || a.body["error"] != "unchanged" { + t.Fatalf("unchanged: %d %s", a.status, a.raw) + } + + // The hub unreachable: 502, and the stored files aren't served (no hosted revision has them). + e.hub.mu.Lock() + e.hub.down = true + e.hub.mu.Unlock() + tex3 := texture + " v3" + a = e.upload(t, e.hub.uploadToken(""), makeZip(t, file{"index.xml", index}, file{"tex.png", tex3}), nil) + if a.status != http.StatusBadGateway { + t.Fatalf("hub down: %d %s", a.status, a.raw) + } + if r := e.get(t, "/api/v1/files/"+sum(tex3)); r.status != http.StatusNotFound { + t.Errorf("file of a refused upload: %d", r.status) + } +} + +func TestAdultFiles(t *testing.T) { + e := setup(t) + index := indexXML("AP", "Adult Thing") + adultTex := texture + " adult" + a := e.upload(t, e.hub.uploadToken(""), makeZip(t, file{"index.xml", index}, file{"a.png", adultTex}, file{"shared.png", texture}), nil) + if a.status != http.StatusCreated { + t.Fatalf("upload: %d %s", a.status, a.raw) + } + if e.hub.last.Rating != "adult" { + t.Fatalf("rating %q", e.hub.last.Rating) + } + path := "/api/v1/files/" + sum(adultTex) + + r := e.get(t, path) + if r.status != http.StatusUnauthorized || !strings.HasPrefix(r.header.Get("Cache-Control"), "private") { + t.Fatalf("no token: %d %q", r.status, r.header.Get("Cache-Control")) + } + r = e.get(t, path, "Authorization", "Bearer garbage") + if r.status != http.StatusUnauthorized { + t.Fatalf("bad token: %d", r.status) + } + r = e.get(t, path, "Authorization", "Bearer "+e.hub.uploadToken("")) + if r.status != http.StatusUnauthorized { + t.Fatalf("upload token as access token: %d", r.status) + } + r = e.get(t, path, "Authorization", "Bearer "+e.hub.token(token.Access, map[string]any{"adult": false})) + if r.status != http.StatusForbidden || !strings.HasPrefix(r.header.Get("Cache-Control"), "private") { + t.Fatalf("not adult: %d %q", r.status, r.header.Get("Cache-Control")) + } + r = e.get(t, path, "Authorization", "Bearer "+e.hub.token(token.Access, map[string]any{"adult": true})) + if r.status != http.StatusOK || string(r.raw) != adultTex { + t.Fatalf("adult: %d %q", r.status, r.raw) + } + if cc := r.header.Get("Cache-Control"); !strings.HasPrefix(cc, "private") { + t.Errorf("adult file's Cache-Control %q", cc) + } + + // shared.png also belongs to a general revision once one uses it: then it's public. + if r := e.get(t, "/api/v1/files/"+sum(texture)); r.status != http.StatusUnauthorized { + t.Fatalf("shared file before a general revision: %d", r.status) + } + a = e.upload(t, e.hub.uploadToken(""), makeZip(t, file{"index.xml", indexXML("GA", "Plain")}, file{"shared.png", texture}), nil) + if a.status != http.StatusCreated { + t.Fatalf("general upload: %d %s", a.status, a.raw) + } + if r := e.get(t, "/api/v1/files/"+sum(texture)); r.status != http.StatusOK { + t.Fatalf("shared file after a general revision: %d", r.status) + } +} + +func TestBadUploads(t *testing.T) { + e := setup(t) + good := makeZip(t, file{"index.xml", indexXML("GA", "Hat")}, file{"tex.png", texture}) + + tokens := map[string]string{ + "no token": "", + "garbage": "garbage", + "other catalog": e.hub.token(token.Upload, map[string]any{"catalog": "cat2"}), + "access token": e.hub.token(token.Access, map[string]any{"adult": true}), + "expired": e.hub.token(token.Upload, map[string]any{"catalog": catalogID, "exp": time.Now().Add(-time.Hour).Unix()}), + "wrong issuer": e.hub.token(token.Upload, map[string]any{"catalog": catalogID, "iss": "https://other.example"}), + "other key": tokentest.NewKey("hubkey").Sign(map[string]any{"iss": e.hub.URL, "typ": token.Upload, "catalog": catalogID, "exp": time.Now().Add(time.Hour).Unix()}), + } + for name, tok := range tokens { + if a := e.upload(t, tok, good, nil); a.status != http.StatusUnauthorized || a.body["error"] != "bad_token" { + t.Errorf("%s: %d %s", name, a.status, a.raw) + } + } + + big := strings.Repeat("x", 100) + zips := map[string][]byte{ + "not a zip": []byte("this is not a zip file"), + "no index.xml": makeZip(t, file{"tex.png", texture}), + "bad rating": makeZip(t, file{"index.xml", indexXML("XX", "Hat")}), + "no rating": makeZip(t, file{"index.xml", ""}), + "not a template": makeZip(t, file{"index.xml", ""}), + "file in a folder": makeZip(t, file{"index.xml", indexXML("GA", "Hat")}, file{"sub/tex.png", texture}), + "_contents.json": makeZip(t, file{"index.xml", indexXML("GA", "Hat")}, file{"_contents.json", "{}"}), + "duplicate name": makeZip(t, file{"index.xml", indexXML("GA", "Hat")}, file{"a.png", big}, file{"a.png", big}), + "non-Latin-1 name": makeZip(t, file{"index.xml", indexXML("GA", "Hat")}, file{"δΈ€.png", big}), + "bad __DATAIMPORT": makeZip(t, file{"index.xml", strings.Replace(indexXML("GA", "Hat"), "product://12345/index.xml", "http://x", 1)}), + "index.xml nested": makeZip(t, file{"x/index.xml", indexXML("GA", "Hat")}), + "backslash in name": makeZip(t, file{"index.xml", indexXML("GA", "Hat")}, file{`a\b.png`, big}), + } + for name, z := range zips { + a := e.upload(t, e.hub.uploadToken(""), z, nil) + t.Logf("%s: %s", name, a.body["message"]) + if a.status != http.StatusBadRequest || a.body["error"] != "bad_files" { + t.Errorf("%s: %d %s", name, a.status, a.raw) + } + } + if a := e.upload(t, e.hub.uploadToken(""), nil, ptr("notes but no file")); a.status != http.StatusBadRequest { + t.Errorf("no file: %d %s", a.status, a.raw) + } + req, _ := http.NewRequest(http.MethodPost, e.catalog.URL+"/api/v1/uploads", strings.NewReader("{}")) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+e.hub.uploadToken("")) + if a := do(t, req); a.status != http.StatusBadRequest { + t.Errorf("not multipart: %d %s", a.status, a.raw) + } + if e.hub.calls != 0 { + t.Errorf("hub called %d times for bad uploads", e.hub.calls) + } +} + +// A zip whose files are larger than its headers say is cut off by the pack rules' counted reads. +func TestUploadSizeCountedWhileReading(t *testing.T) { + e := setup(t) + // A file over the 64 MiB limit, compressed down to a small upload. + var b bytes.Buffer + z := zip.NewWriter(&b) + w, _ := z.Create("index.xml") + w.Write([]byte(indexXML("GA", "Hat"))) + w, _ = z.Create("huge.png") + chunk := make([]byte, 1<<20) + for range 65 { + w.Write(chunk) + } + z.Close() + a := e.upload(t, e.hub.uploadToken(""), b.Bytes(), nil) + if a.status != http.StatusBadRequest || !strings.Contains(a.body["message"].(string), "64 MiB") { + t.Fatalf("huge file: %d %s", a.status, a.raw) + } +} + +func TestCORSAndHealth(t *testing.T) { + e := setup(t) + preflight := func(origin string) http.Header { + req, _ := http.NewRequest(http.MethodOptions, e.catalog.URL+"/api/v1/uploads", nil) + req.Header.Set("Origin", origin) + req.Header.Set("Access-Control-Request-Method", "POST") + req.Header.Set("Access-Control-Request-Headers", "authorization") + a := do(t, req) + if a.status != http.StatusNoContent { + t.Fatalf("preflight: %d", a.status) + } + return a.header + } + h := preflight(webOrigin) + if h.Get("Access-Control-Allow-Origin") != webOrigin || !strings.Contains(h.Get("Access-Control-Allow-Headers"), "Authorization") || + h.Get("Access-Control-Allow-Methods") != "POST" { + t.Errorf("preflight headers: %v", h) + } + if h := preflight("https://evil.example"); h.Get("Access-Control-Allow-Origin") != "" { + t.Errorf("other origin allowed: %v", h) + } + + a := e.get(t, "/health") + if a.status != http.StatusOK || a.body["ok"] != true { + t.Errorf("health: %d %s", a.status, a.raw) + } +} diff --git a/internal/server/upload.go b/internal/server/upload.go new file mode 100644 index 0000000..53c8423 --- /dev/null +++ b/internal/server/upload.go @@ -0,0 +1,299 @@ +package server + +import ( + "archive/zip" + "encoding/json" + "errors" + "fmt" + "io" + "mime" + "net/http" + "os" + "path/filepath" + "strings" + "unicode/utf8" + + "git.t5de.dev/t5de/t5de-api/pack" + + "git.t5de.dev/t5de/t5de-catalog/internal/hub" + "git.t5de.dev/t5de/t5de-catalog/internal/store" + "git.t5de.dev/t5de/t5de-catalog/internal/token" +) + +// MaxBody caps an upload request: the zip (whose files may hold pack.MaxTotalSize uncompressed) and the notes. +const MaxBody = 300 << 20 + +// MaxNotes caps the revision notes. +const MaxNotes = 64 << 10 + +// corsHeaders adds CORS headers for the hub website's origin. +func (s *Server) corsHeaders(w http.ResponseWriter, r *http.Request) bool { + h := w.Header() + h.Add("Vary", "Origin") + origin := r.Header.Get("Origin") + if origin == "" || s.cfg.WebOrigin == "" || origin != s.cfg.WebOrigin { + return false + } + h.Set("Access-Control-Allow-Origin", origin) + return true +} + +func (s *Server) preflight(w http.ResponseWriter, r *http.Request) { + if s.corsHeaders(w, r) { + h := w.Header() + h.Add("Vary", "Access-Control-Request-Method") + h.Add("Vary", "Access-Control-Request-Headers") + h.Set("Access-Control-Allow-Methods", "POST") + h.Set("Access-Control-Allow-Headers", "Authorization, Content-Type") + h.Set("Access-Control-Max-Age", "600") + } + w.WriteHeader(http.StatusNoContent) +} + +// upload takes a product (docs/products.md, POST /api/v1/uploads): checks the token and the files, stores the +// files, registers the revision with the hub, and records it as hosted once the hub has accepted it. +func (s *Server) upload(w http.ResponseWriter, r *http.Request) { + s.corsHeaders(w, r) + tok := bearer(r) + if tok == "" { + writeError(w, http.StatusUnauthorized, "bad_token", "An upload token is needed.") + return + } + claims, err := s.cfg.Verifier.Verify(r.Context(), tok, token.Upload) + if err != nil { + s.cfg.Logger.Printf("uploads: %v", err) + writeError(w, http.StatusUnauthorized, "bad_token", "The upload token isn't valid. Start the upload again.") + return + } + if claims.Catalog != s.cfg.CatalogID { + writeError(w, http.StatusUnauthorized, "bad_token", "The upload token is for another catalog.") + return + } + if claims.Product != "" && !hub.ValidProduct(claims.Product) { + writeError(w, http.StatusUnauthorized, "bad_token", "The upload token names no valid product.") + return + } + + select { + case s.uploads <- struct{}{}: + defer func() { <-s.uploads }() + case <-r.Context().Done(): + return + } + + tmp, err := os.MkdirTemp(s.cfg.Store.TempDir(), "upload-*") + if err != nil { + s.cfg.Logger.Printf("uploads: %v", err) + writeError(w, http.StatusInternalServerError, "internal", "Something went wrong.") + return + } + defer os.RemoveAll(tmp) + + r.Body = http.MaxBytesReader(w, r.Body, MaxBody) + notes, status, code, msg := readForm(r, filepath.Join(tmp, pack.ChknName)) + if status != 0 { + if status == http.StatusInternalServerError { + s.cfg.Logger.Printf("uploads: %s", msg) + msg = "Something went wrong." + } + writeError(w, status, code, msg) + return + } + if claims.Product != "" && notes == "" { + writeError(w, http.StatusBadRequest, "notes_required", "Say what changed in this revision.") + return + } + + p, rating, err := checkProduct(tmp) + if err != nil { + writeError(w, http.StatusBadRequest, "bad_files", "These product files can't be used: "+err.Error()) + return + } + + files := make([]store.File, len(p.Files)) + for i, f := range p.Files { + hash, err := s.cfg.Store.PutFile(f.Data) + if err != nil { + s.cfg.Logger.Printf("uploads: storing %q: %v", f.Name, err) + writeError(w, http.StatusInternalServerError, "internal", "Something went wrong.") + return + } + files[i] = store.File{Name: f.Name, SHA256: hash, Size: int64(len(f.Data))} + } + index := p.Index + p = nil // the files' data can go before the hub answers + + reg, err := s.cfg.Hub.RegisterRevision(r.Context(), hub.NewRevision{ + UploadToken: tok, Files: files, Rating: rating, Parent: index.Parent, Name: strings.TrimSpace(index.Name), Notes: notes, + }) + if err != nil { + var refused *hub.Error + if errors.As(err, &refused) { + s.cfg.Logger.Printf("uploads: %v", err) + code, msg := refused.Code, refused.Message + if code == "" { + code = "hub_refused" + } + if msg == "" { + msg = fmt.Sprintf("T5DE's hub refused the upload (%d).", refused.Status) + } + writeError(w, http.StatusBadGateway, code, msg) + return + } + s.cfg.Logger.Printf("uploads: registering with the hub: %v", err) + writeError(w, http.StatusBadGateway, "hub_unavailable", "T5DE's hub couldn't be reached. Try again later.") + return + } + if claims.Product != "" && reg.Product != claims.Product { + s.cfg.Logger.Printf("uploads: the hub registered product %s, but the token was for %s", reg.Product, claims.Product) + writeError(w, http.StatusBadGateway, "hub_failed", "T5DE's hub answered for another product.") + return + } + if err := s.cfg.Store.AddRevision(store.Revision{Product: reg.Product, Revision: reg.Revision, Rating: rating, Files: files}); err != nil { + s.cfg.Logger.Printf("uploads: recording %s/%d: %v", reg.Product, reg.Revision, err) + writeError(w, http.StatusInternalServerError, "internal", "Something went wrong.") + return + } + s.cfg.Logger.Printf("uploads: %s/%d (%s, %d files) from %s", reg.Product, reg.Revision, rating, len(files), claims.Subject) + writeJSON(w, http.StatusCreated, reg) +} + +// readForm reads the upload's multipart form: the zip (part "file") into chkn, and the notes (part "notes"). +// Other parts are skipped. A non-zero status is an error to answer. +func readForm(r *http.Request, chkn string) (notes string, status int, code, msg string) { + mt, _, err := mime.ParseMediaType(r.Header.Get("Content-Type")) + if err != nil || mt != "multipart/form-data" { + return "", http.StatusBadRequest, "bad_request", "Send the product as multipart/form-data." + } + mr, err := r.MultipartReader() + if err != nil { + return "", http.StatusBadRequest, "bad_request", "Send the product as multipart/form-data." + } + tooLarge := func(err error) bool { + var mbe *http.MaxBytesError + return errors.As(err, &mbe) + } + haveFile, haveNotes := false, false + for { + part, err := mr.NextPart() + if err == io.EOF { + break + } + if err != nil { + if tooLarge(err) { + return "", http.StatusRequestEntityTooLarge, "too_large", fmt.Sprintf("Uploads can be at most %d MiB.", MaxBody>>20) + } + return "", http.StatusBadRequest, "bad_request", "The upload is cut short or malformed." + } + switch part.FormName() { + case "file": + if haveFile { + return "", http.StatusBadRequest, "bad_request", "Send one file." + } + haveFile = true + f, err := os.Create(chkn) + if err != nil { + return "", http.StatusInternalServerError, "internal", err.Error() + } + _, err = io.Copy(f, part) + if cerr := f.Close(); err == nil && cerr != nil { + return "", http.StatusInternalServerError, "internal", cerr.Error() + } + if err != nil { + if tooLarge(err) { + return "", http.StatusRequestEntityTooLarge, "too_large", fmt.Sprintf("Uploads can be at most %d MiB.", MaxBody>>20) + } + return "", http.StatusBadRequest, "bad_request", "The upload is cut short or malformed." + } + case "notes": + if haveNotes { + return "", http.StatusBadRequest, "bad_request", "Send the notes once." + } + haveNotes = true + data, err := io.ReadAll(io.LimitReader(part, MaxNotes+1)) + if err != nil { + if tooLarge(err) { + return "", http.StatusRequestEntityTooLarge, "too_large", fmt.Sprintf("Uploads can be at most %d MiB.", MaxBody>>20) + } + return "", http.StatusBadRequest, "bad_request", "The upload is cut short or malformed." + } + if len(data) > MaxNotes { + return "", http.StatusBadRequest, "bad_request", fmt.Sprintf("The notes can be at most %d KiB.", MaxNotes>>10) + } + if !utf8.Valid(data) { + return "", http.StatusBadRequest, "bad_request", "The notes aren't valid UTF-8." + } + notes = strings.TrimSpace(string(data)) + default: + if _, err := io.Copy(io.Discard, part); err != nil { + if tooLarge(err) { + return "", http.StatusRequestEntityTooLarge, "too_large", fmt.Sprintf("Uploads can be at most %d MiB.", MaxBody>>20) + } + return "", http.StatusBadRequest, "bad_request", "The upload is cut short or malformed." + } + } + part.Close() + } + if !haveFile { + return "", http.StatusBadRequest, "bad_request", "No file: send the product's .chkn as \"file\"." + } + return notes, 0, "", "" +} + +// ratings maps index.xml's ProductRating to the hub's ratings. +var ratings = map[string]string{"GA": store.General, "AP": store.Adult} + +// checkProduct checks the zip dir/product.chkn by the pack rules (t5de-api's pack package): it reads the rating +// from its index.xml, writes a manifest with that rating beside it, and loads dir as a pack. It returns the pack +// and its rating. +func checkProduct(dir string) (*pack.Pack, string, error) { + index, err := readIndex(filepath.Join(dir, pack.ChknName)) + if err != nil { + return nil, "", err + } + rating := ratings[index.Rating] + manifest, err := json.Marshal(pack.Manifest{Format: pack.Format, ID: "upload", Revision: 1, Name: "upload", Rating: rating}) + if err != nil { + return nil, "", err + } + if err := os.WriteFile(filepath.Join(dir, pack.ManifestName), manifest, 0o644); err != nil { + return nil, "", err + } + p, err := pack.Load(dir) + if err != nil { + return nil, "", err + } + return p, rating, nil +} + +// readIndex reads index.xml from the zip at path, its size counted while reading. +func readIndex(path string) (*pack.Index, error) { + z, err := zip.OpenReader(path) + if err != nil { + return nil, errors.New("not a zip file") + } + defer z.Close() + for _, f := range z.File { + if f.Name != pack.IndexName { + continue + } + rc, err := f.Open() + if err != nil { + return nil, fmt.Errorf("%s: %v", pack.IndexName, err) + } + data, err := io.ReadAll(io.LimitReader(rc, pack.MaxFileSize+1)) + rc.Close() + if err != nil { + return nil, fmt.Errorf("%s: %v", pack.IndexName, err) + } + if len(data) > pack.MaxFileSize { + return nil, fmt.Errorf("%s: over %d MiB", pack.IndexName, pack.MaxFileSize>>20) + } + index, err := pack.ParseIndex(data) + if err != nil { + return nil, fmt.Errorf("%s: %v", pack.IndexName, err) + } + return index, nil + } + return nil, fmt.Errorf("no %s at the top of the zip", pack.IndexName) +} diff --git a/internal/store/store.go b/internal/store/store.go new file mode 100644 index 0000000..a07c51f --- /dev/null +++ b/internal/store/store.go @@ -0,0 +1,217 @@ +// Package store keeps the catalog's files and the revisions it hosts: files on disk named by their sha256 +// (files//), revisions in SQLite (catalog.db). +package store + +import ( + "crypto/sha256" + "database/sql" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "regexp" + + _ "modernc.org/sqlite" +) + +// Ratings. +const ( + General = "general" + Adult = "adult" +) + +// File is one entry of a revision's file list. +type File struct { + Name string `json:"name"` + SHA256 string `json:"sha256"` + Size int64 `json:"size"` +} + +// Revision is a revision the catalog hosts. +type Revision struct { + Product string `json:"product"` + Revision int `json:"revision"` + Rating string `json:"rating"` + Files []File `json:"files"` // sorted by name +} + +// ErrNotFound is returned for files and revisions the catalog doesn't host. +var ErrNotFound = errors.New("not found") + +var hashPattern = regexp.MustCompile(`^[0-9a-f]{64}$`) + +// ValidHash reports whether s is a lowercase hex sha256. +func ValidHash(s string) bool { return hashPattern.MatchString(s) } + +// Store is the catalog's data folder. +type Store struct { + dir string + db *sql.DB +} + +const schema = ` +CREATE TABLE IF NOT EXISTS revisions ( + product TEXT NOT NULL, + revision INTEGER NOT NULL, + rating TEXT NOT NULL CHECK (rating IN ('general', 'adult')), + files TEXT NOT NULL, -- the file list, JSON + created_at INTEGER NOT NULL DEFAULT (unixepoch()), + PRIMARY KEY (product, revision) +); +CREATE TABLE IF NOT EXISTS revision_files ( + product TEXT NOT NULL, + revision INTEGER NOT NULL, + sha256 TEXT NOT NULL, + PRIMARY KEY (product, revision, sha256), + FOREIGN KEY (product, revision) REFERENCES revisions (product, revision) ON DELETE CASCADE +); +CREATE INDEX IF NOT EXISTS revision_files_sha256 ON revision_files (sha256); +` + +// Open opens (or creates) the store in dir. +func Open(dir string) (*Store, error) { + if err := os.MkdirAll(filepath.Join(dir, "files"), 0o755); err != nil { + return nil, err + } + // Leftovers of uploads interrupted by a crash. + if err := os.RemoveAll(filepath.Join(dir, "tmp")); err != nil { + return nil, err + } + if err := os.MkdirAll(filepath.Join(dir, "tmp"), 0o755); err != nil { + return nil, err + } + dsn := "file:" + filepath.ToSlash(filepath.Join(dir, "catalog.db")) + + "?_pragma=busy_timeout(5000)&_pragma=journal_mode(WAL)&_pragma=foreign_keys(1)&_pragma=synchronous(NORMAL)" + db, err := sql.Open("sqlite", dsn) + if err != nil { + return nil, err + } + db.SetMaxOpenConns(1) + if _, err := db.Exec(schema); err != nil { + db.Close() + return nil, fmt.Errorf("catalog.db: %w", err) + } + return &Store{dir: dir, db: db}, nil +} + +// Close closes the database. +func (s *Store) Close() error { return s.db.Close() } + +// TempDir is a folder on the store's file system for temporary files (emptied when the store opens). +func (s *Store) TempDir() string { return filepath.Join(s.dir, "tmp") } + +// Path returns where the file with this hash is (or would be) stored. +func (s *Store) Path(hash string) string { + return filepath.Join(s.dir, "files", hash[:2], hash) +} + +// PutFile stores data under its sha256 and returns the hash. A file already stored isn't written again. +func (s *Store) PutFile(data []byte) (string, error) { + sum := sha256.Sum256(data) + hash := hex.EncodeToString(sum[:]) + path := s.Path(hash) + if fi, err := os.Stat(path); err == nil && fi.Size() == int64(len(data)) { + return hash, nil + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return "", err + } + f, err := os.CreateTemp(filepath.Dir(path), ".tmp-*") + if err != nil { + return "", err + } + tmp := f.Name() + _, err = f.Write(data) + if err == nil { + err = f.Sync() + } + if cerr := f.Close(); err == nil { + err = cerr + } + if err == nil { + err = os.Rename(tmp, path) + } + if err != nil { + os.Remove(tmp) + return "", err + } + return hash, nil +} + +// Open opens a stored file. +func (s *Store) Open(hash string) (*os.File, error) { + if !ValidHash(hash) { + return nil, ErrNotFound + } + f, err := os.Open(s.Path(hash)) + if errors.Is(err, fs.ErrNotExist) { + return nil, ErrNotFound + } + return f, err +} + +// AddRevision records a revision as hosted. Its files must be stored already. Recording the same revision again +// replaces it. +func (s *Store) AddRevision(r Revision) error { + if r.Rating != General && r.Rating != Adult { + return fmt.Errorf("rating %q", r.Rating) + } + files, err := json.Marshal(r.Files) + if err != nil { + return err + } + tx, err := s.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + if _, err := tx.Exec(`DELETE FROM revisions WHERE product = ? AND revision = ?`, r.Product, r.Revision); err != nil { + return err + } + if _, err := tx.Exec(`INSERT INTO revisions (product, revision, rating, files) VALUES (?, ?, ?, ?)`, + r.Product, r.Revision, r.Rating, string(files)); err != nil { + return err + } + for _, f := range r.Files { + if _, err := tx.Exec(`INSERT OR IGNORE INTO revision_files (product, revision, sha256) VALUES (?, ?, ?)`, + r.Product, r.Revision, f.SHA256); err != nil { + return err + } + } + return tx.Commit() +} + +// Revision returns a hosted revision. +func (s *Store) Revision(product string, revision int) (*Revision, error) { + r := Revision{Product: product, Revision: revision} + var files string + err := s.db.QueryRow(`SELECT rating, files FROM revisions WHERE product = ? AND revision = ?`, product, revision). + Scan(&r.Rating, &files) + if errors.Is(err, sql.ErrNoRows) { + return nil, ErrNotFound + } + if err != nil { + return nil, err + } + if err := json.Unmarshal([]byte(files), &r.Files); err != nil { + return nil, fmt.Errorf("revision %s/%d: %w", product, revision, err) + } + return &r, nil +} + +// FileAccess tells whether a file belongs to a hosted revision (hosted) and whether every such revision is adult +// (adultOnly). Files in no hosted revision (an upload the hub refused) aren't served. +func (s *Store) FileAccess(hash string) (hosted, adultOnly bool, err error) { + var n, general int + err = s.db.QueryRow(` + SELECT count(*), coalesce(sum(r.rating = 'general'), 0) + FROM revision_files rf JOIN revisions r ON r.product = rf.product AND r.revision = rf.revision + WHERE rf.sha256 = ?`, hash).Scan(&n, &general) + if err != nil { + return false, false, err + } + return n > 0, n > 0 && general == 0, nil +} diff --git a/internal/store/store_test.go b/internal/store/store_test.go new file mode 100644 index 0000000..379bc44 --- /dev/null +++ b/internal/store/store_test.go @@ -0,0 +1,147 @@ +package store + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "os" + "path/filepath" + "reflect" + "testing" +) + +func open(t *testing.T, dir string) *Store { + t.Helper() + s, err := Open(dir) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { s.Close() }) + return s +} + +func hashOf(data string) string { + sum := sha256.Sum256([]byte(data)) + return hex.EncodeToString(sum[:]) +} + +func TestFiles(t *testing.T) { + dir := t.TempDir() + s := open(t, dir) + + hash, err := s.PutFile([]byte("hello")) + if err != nil { + t.Fatal(err) + } + if hash != hashOf("hello") { + t.Fatalf("hash %s", hash) + } + want := filepath.Join(dir, "files", hash[:2], hash) + if s.Path(hash) != want { + t.Fatalf("path %s, want %s", s.Path(hash), want) + } + again, err := s.PutFile([]byte("hello")) + if err != nil || again != hash { + t.Fatalf("again: %s, %v", again, err) + } + f, err := s.Open(hash) + if err != nil { + t.Fatal(err) + } + data, _ := io.ReadAll(f) + f.Close() + if string(data) != "hello" { + t.Fatalf("data %q", data) + } + // No temp files left beside it. + entries, _ := os.ReadDir(filepath.Dir(want)) + if len(entries) != 1 { + t.Fatalf("%d entries in %s", len(entries), filepath.Dir(want)) + } + + if _, err := s.Open(hashOf("missing")); !errors.Is(err, ErrNotFound) { + t.Fatalf("missing: %v", err) + } + for _, bad := range []string{"", "../catalog.db", hash[:63], "A" + hash[1:]} { + if _, err := s.Open(bad); !errors.Is(err, ErrNotFound) { + t.Fatalf("%q: %v", bad, err) + } + } +} + +func TestRevisions(t *testing.T) { + dir := t.TempDir() + s := open(t, dir) + shared, general, adult := hashOf("shared"), hashOf("general"), hashOf("adult") + + if hosted, _, err := s.FileAccess(shared); err != nil || hosted { + t.Fatalf("before: hosted %v, %v", hosted, err) + } + + a := Revision{Product: "aaaaaaaaaa", Revision: 1, Rating: Adult, Files: []File{ + {Name: "a.png", SHA256: adult, Size: 5}, {Name: "index.xml", SHA256: shared, Size: 6}, + }} + if err := s.AddRevision(a); err != nil { + t.Fatal(err) + } + check := func(hash string, wantHosted, wantAdult bool) { + t.Helper() + hosted, adultOnly, err := s.FileAccess(hash) + if err != nil || hosted != wantHosted || adultOnly != wantAdult { + t.Fatalf("%s: hosted %v adult %v (%v), want %v %v", hash[:8], hosted, adultOnly, err, wantHosted, wantAdult) + } + } + check(shared, true, true) + check(adult, true, true) + check(general, false, false) + + g := Revision{Product: "bbbbbbbbbb", Revision: 2, Rating: General, Files: []File{ + {Name: "g.png", SHA256: general, Size: 7}, {Name: "index.xml", SHA256: shared, Size: 6}, + }} + if err := s.AddRevision(g); err != nil { + t.Fatal(err) + } + check(shared, true, false) // also in a general revision + check(adult, true, true) + check(general, true, false) + + got, err := s.Revision("aaaaaaaaaa", 1) + if err != nil || !reflect.DeepEqual(*got, a) { + t.Fatalf("got %+v, %v", got, err) + } + if _, err := s.Revision("aaaaaaaaaa", 2); !errors.Is(err, ErrNotFound) { + t.Fatalf("missing revision: %v", err) + } + if err := s.AddRevision(Revision{Product: "c", Revision: 1, Rating: "GA"}); err == nil { + t.Fatal("bad rating accepted") + } + + // Recording a revision again replaces it, and the data survives reopening. + a.Rating = General + if err := s.AddRevision(a); err != nil { + t.Fatal(err) + } + s.Close() + s = open(t, dir) + got, err = s.Revision("aaaaaaaaaa", 1) + if err != nil || got.Rating != General { + t.Fatalf("after reopening: %+v, %v", got, err) + } + if hosted, adultOnly, _ := s.FileAccess(adult); !hosted || adultOnly { + t.Fatal("adult file still adult-only after the revision became general") + } +} + +func TestTempCleared(t *testing.T) { + dir := t.TempDir() + s := open(t, dir) + if err := os.WriteFile(filepath.Join(s.TempDir(), "leftover"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + s.Close() + s = open(t, dir) + if entries, _ := os.ReadDir(s.TempDir()); len(entries) != 0 { + t.Fatalf("temp folder not emptied: %v", entries) + } +} diff --git a/internal/token/token.go b/internal/token/token.go new file mode 100644 index 0000000..8ac14d1 --- /dev/null +++ b/internal/token/token.go @@ -0,0 +1,257 @@ +// Package token checks the hub's tokens: EdDSA (Ed25519) JWTs in compact form, signed with a key from the hub's +// JWKS (GET /.well-known/t5de-keys.json). See t5de-hub's docs/products.md, "Tokens". +package token + +import ( + "context" + "crypto/ed25519" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strings" + "sync" + "time" +) + +// Token types. +const ( + Upload = "t5de-upload" + Access = "t5de-access" + Revision = "t5de-revision" +) + +// KeysPath is where the hub publishes its keys. +const KeysPath = "/.well-known/t5de-keys.json" + +// Leeway allows for clocks that disagree a little. +const Leeway = time.Minute + +// RefetchInterval is the least time between fetches of the hub's keys, however many unknown key IDs show up. +const RefetchInterval = 30 * time.Second + +const maxToken = 16 << 10 +const maxKeys = 1 << 20 + +// ErrInvalid wraps every reason a token is refused. +var ErrInvalid = errors.New("invalid token") + +// Claims are the claims T5DE's tokens carry. +type Claims struct { + Issuer string `json:"iss"` + Type string `json:"typ"` + Subject string `json:"sub"` + IssuedAt *int64 `json:"iat"` + Expires *int64 `json:"exp"` + NotYet *int64 `json:"nbf"` + Catalog string `json:"catalog"` // t5de-upload + Product string `json:"product"` // t5de-upload (optional), t5de-revision + Adult bool `json:"adult"` // t5de-access + Revision int `json:"revision"` // t5de-revision + Rating string `json:"rating"` // t5de-revision +} + +// Verifier checks tokens signed by one hub. +type Verifier struct { + issuer string + keyURL string + client *http.Client + now func() time.Time + + fetch sync.Mutex // held while fetching, so one fetch serves everyone waiting + mu sync.Mutex + keys map[string]ed25519.PublicKey + fetched time.Time // when the keys were last fetched (or tried), zero if never +} + +// NewVerifier returns a verifier for tokens from the hub at hubURL (its API's address, which is also the tokens' +// issuer). client may be nil. +func NewVerifier(hubURL string, client *http.Client) *Verifier { + hubURL = strings.TrimRight(hubURL, "/") + if client == nil { + client = &http.Client{Timeout: 10 * time.Second} + } + return &Verifier{issuer: hubURL, keyURL: hubURL + KeysPath, client: client, now: time.Now} +} + +// SetClock replaces the verifier's clock, for tests. +func (v *Verifier) SetClock(now func() time.Time) { v.now = now } + +type header struct { + Alg string `json:"alg"` + Kid string `json:"kid"` + Crit any `json:"crit"` +} + +// Verify checks a token of type typ: its signature, issuer, type and times. Tokens of other types than +// t5de-revision must have an expiry. +func (v *Verifier) Verify(ctx context.Context, token, typ string) (*Claims, error) { + c, err := v.verify(ctx, token, typ) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrInvalid, err) + } + return c, nil +} + +func (v *Verifier) verify(ctx context.Context, token, typ string) (*Claims, error) { + if len(token) > maxToken { + return nil, errors.New("too long") + } + parts := strings.Split(token, ".") + if len(parts) != 3 { + return nil, errors.New("not a compact JWT") + } + var h header + if err := decodePart(parts[0], &h); err != nil { + return nil, fmt.Errorf("header: %v", err) + } + if h.Alg != "EdDSA" { + return nil, fmt.Errorf("algorithm %q isn't EdDSA", h.Alg) + } + if h.Crit != nil { + return nil, errors.New("unsupported critical header") + } + sig, err := base64.RawURLEncoding.Strict().DecodeString(parts[2]) + if err != nil || len(sig) != ed25519.SignatureSize { + return nil, errors.New("bad signature encoding") + } + key, err := v.key(ctx, h.Kid) + if err != nil { + return nil, err + } + if !ed25519.Verify(key, []byte(parts[0]+"."+parts[1]), sig) { + return nil, errors.New("bad signature") + } + + var c Claims + if err := decodePart(parts[1], &c); err != nil { + return nil, fmt.Errorf("claims: %v", err) + } + if c.Issuer != v.issuer { + return nil, fmt.Errorf("issuer %q isn't the hub", c.Issuer) + } + if c.Type != typ { + return nil, fmt.Errorf("type %q isn't %q", c.Type, typ) + } + now := v.now() + if c.Expires == nil { + if typ != Revision { + return nil, errors.New("no expiry") + } + } else if !now.Before(time.Unix(*c.Expires, 0).Add(Leeway)) { + return nil, errors.New("expired") + } + if c.NotYet != nil && now.Add(Leeway).Before(time.Unix(*c.NotYet, 0)) { + return nil, errors.New("not valid yet") + } + if c.IssuedAt != nil && now.Add(Leeway).Before(time.Unix(*c.IssuedAt, 0)) { + return nil, errors.New("issued in the future") + } + return &c, nil +} + +func decodePart(s string, v any) error { + data, err := base64.RawURLEncoding.Strict().DecodeString(s) + if err != nil { + return err + } + return json.Unmarshal(data, v) +} + +// key returns the key with ID kid, fetching the hub's keys if it isn't known and they weren't fetched lately. +func (v *Verifier) key(ctx context.Context, kid string) (ed25519.PublicKey, error) { + if k, ok := v.cached(kid); ok { + return k, nil + } + v.fetch.Lock() + defer v.fetch.Unlock() + if k, ok := v.cached(kid); ok { // fetched while waiting + return k, nil + } + v.mu.Lock() + recent := !v.fetched.IsZero() && v.now().Sub(v.fetched) < RefetchInterval + v.mu.Unlock() + if recent { + return nil, fmt.Errorf("unknown key %q", kid) + } + keys, err := v.fetchKeys(ctx) + v.mu.Lock() + v.fetched = v.now() + if err == nil { + v.keys = keys + } + v.mu.Unlock() + if err != nil { + return nil, fmt.Errorf("fetching the hub's keys: %v", err) + } + if k, ok := keys[kid]; ok { + return k, nil + } + return nil, fmt.Errorf("unknown key %q", kid) +} + +func (v *Verifier) cached(kid string) (ed25519.PublicKey, bool) { + v.mu.Lock() + defer v.mu.Unlock() + k, ok := v.keys[kid] + return k, ok +} + +type jwk struct { + Kty string `json:"kty"` + Crv string `json:"crv"` + X string `json:"x"` + Kid string `json:"kid"` + Alg string `json:"alg"` + Use string `json:"use"` +} + +func (v *Verifier) fetchKeys(ctx context.Context) (map[string]ed25519.PublicKey, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, v.keyURL, nil) + if err != nil { + return nil, err + } + resp, err := v.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("%s: %s", v.keyURL, resp.Status) + } + data, err := io.ReadAll(io.LimitReader(resp.Body, maxKeys+1)) + if err != nil { + return nil, err + } + if len(data) > maxKeys { + return nil, errors.New("key set too large") + } + return ParseKeys(data) +} + +// ParseKeys reads the Ed25519 signing keys from a JWKS. Other keys are skipped. +func ParseKeys(data []byte) (map[string]ed25519.PublicKey, error) { + var set struct { + Keys []jwk `json:"keys"` + } + if err := json.Unmarshal(data, &set); err != nil { + return nil, err + } + keys := map[string]ed25519.PublicKey{} + for _, k := range set.Keys { + if k.Kty != "OKP" || k.Crv != "Ed25519" || k.Kid == "" { + continue + } + if (k.Alg != "" && k.Alg != "EdDSA") || (k.Use != "" && k.Use != "sig") { + continue + } + x, err := base64.RawURLEncoding.Strict().DecodeString(k.X) + if err != nil || len(x) != ed25519.PublicKeySize { + continue + } + keys[k.Kid] = ed25519.PublicKey(x) + } + return keys, nil +} diff --git a/internal/token/token_test.go b/internal/token/token_test.go new file mode 100644 index 0000000..e3ae9e5 --- /dev/null +++ b/internal/token/token_test.go @@ -0,0 +1,182 @@ +package token_test + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "git.t5de.dev/t5de/t5de-catalog/internal/token" + "git.t5de.dev/t5de/t5de-catalog/internal/token/tokentest" +) + +type hub struct { + *httptest.Server + mu sync.Mutex + keys []*tokentest.Key + fetches atomic.Int32 +} + +func newHub(t *testing.T, keys ...*tokentest.Key) *hub { + h := &hub{keys: keys} + h.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != token.KeysPath { + http.NotFound(w, r) + return + } + h.fetches.Add(1) + h.mu.Lock() + defer h.mu.Unlock() + w.Write(tokentest.JWKS(h.keys...)) + })) + t.Cleanup(h.Close) + return h +} + +func (h *hub) setKeys(keys ...*tokentest.Key) { + h.mu.Lock() + h.keys = keys + h.mu.Unlock() +} + +var now = time.Unix(1_800_000_000, 0) + +func claims(h *hub, typ string, extra map[string]any) map[string]any { + c := map[string]any{"iss": h.URL, "typ": typ, "sub": "person1", "iat": now.Unix(), "exp": now.Add(30 * time.Minute).Unix()} + for k, v := range extra { + if v == nil { + delete(c, k) + } else { + c[k] = v + } + } + return c +} + +func verifier(h *hub, url string) *token.Verifier { + v := token.NewVerifier(url, h.Client()) + v.SetClock(func() time.Time { return now }) + return v +} + +func TestVerify(t *testing.T) { + key := tokentest.NewKey("k1") + other := tokentest.NewKey("k1") // same ID, different key + h := newHub(t, key) + v := verifier(h, h.URL+"/") // a trailing slash in HUB_URL is fine + + c, err := v.Verify(context.Background(), key.Sign(claims(h, token.Upload, map[string]any{"catalog": "cat1", "product": "abcdefghij"})), token.Upload) + if err != nil { + t.Fatal(err) + } + if c.Subject != "person1" || c.Catalog != "cat1" || c.Product != "abcdefghij" { + t.Fatalf("claims: %+v", c) + } + c, err = v.Verify(context.Background(), key.Sign(claims(h, token.Access, map[string]any{"adult": true})), token.Access) + if err != nil || !c.Adult { + t.Fatalf("access: %+v, %v", c, err) + } + // Revision tokens don't expire. + if _, err := v.Verify(context.Background(), key.Sign(claims(h, token.Revision, map[string]any{"exp": nil})), token.Revision); err != nil { + t.Fatal(err) + } + + bad := map[string]string{ + "wrong type": key.Sign(claims(h, token.Access, nil)), + "wrong issuer": key.Sign(claims(h, token.Upload, map[string]any{"iss": "https://evil.example"})), + "expired": key.Sign(claims(h, token.Upload, map[string]any{"exp": now.Add(-2 * time.Minute).Unix()})), + "no expiry": key.Sign(claims(h, token.Upload, map[string]any{"exp": nil})), + "not yet": key.Sign(claims(h, token.Upload, map[string]any{"nbf": now.Add(time.Hour).Unix()})), + "future iat": key.Sign(claims(h, token.Upload, map[string]any{"iat": now.Add(time.Hour).Unix()})), + "other key": other.Sign(claims(h, token.Upload, nil)), + "alg none": key.SignWithHeader(map[string]any{"alg": "none", "kid": "k1"}, claims(h, token.Upload, nil)), + "alg HS256": key.SignWithHeader(map[string]any{"alg": "HS256", "kid": "k1"}, claims(h, token.Upload, nil)), + "crit": key.SignWithHeader(map[string]any{"alg": "EdDSA", "kid": "k1", "crit": []string{"x"}}, claims(h, token.Upload, nil)), + "two parts": "a.b", + "garbage": "not a token at all", + "empty signature": strings.Join(strings.Split(key.Sign(claims(h, token.Upload, nil)), ".")[:2], ".") + ".", + } + for name, tok := range bad { + if _, err := v.Verify(context.Background(), tok, token.Upload); !errors.Is(err, token.ErrInvalid) { + t.Errorf("%s: got %v, want ErrInvalid", name, err) + } + } + + // A tampered payload fails the signature. + parts := strings.Split(key.Sign(claims(h, token.Access, map[string]any{"adult": false})), ".") + forged := key.Sign(claims(h, token.Access, map[string]any{"adult": true})) + tampered := parts[0] + "." + strings.Split(forged, ".")[1] + "." + parts[2] + if _, err := v.Verify(context.Background(), tampered, token.Access); !errors.Is(err, token.ErrInvalid) { + t.Errorf("tampered: %v", err) + } +} + +func TestExpiryLeeway(t *testing.T) { + key := tokentest.NewKey("k1") + h := newHub(t, key) + v := verifier(h, h.URL) + tok := key.Sign(claims(h, token.Upload, map[string]any{"exp": now.Add(-30 * time.Second).Unix()})) + if _, err := v.Verify(context.Background(), tok, token.Upload); err != nil { + t.Fatalf("within leeway: %v", err) + } +} + +func TestKeyRotation(t *testing.T) { + k1, k2 := tokentest.NewKey("k1"), tokentest.NewKey("k2") + h := newHub(t, k1) + clock := now + v := token.NewVerifier(h.URL, h.Client()) + v.SetClock(func() time.Time { return clock }) + + for range 3 { + if _, err := v.Verify(context.Background(), k1.Sign(claims(h, token.Upload, nil)), token.Upload); err != nil { + t.Fatal(err) + } + } + if n := h.fetches.Load(); n != 1 { + t.Fatalf("keys fetched %d times, want 1 (cached)", n) + } + + // An unknown kid right after a fetch isn't fetched again (rate limit). + h.setKeys(k1, k2) + for range 5 { + if _, err := v.Verify(context.Background(), k2.Sign(claims(h, token.Upload, nil)), token.Upload); err == nil { + t.Fatal("unknown key accepted before refetch") + } + } + if n := h.fetches.Load(); n != 1 { + t.Fatalf("keys fetched %d times, want 1 (rate-limited)", n) + } + + // Later, an unknown kid fetches the keys again. + clock = clock.Add(token.RefetchInterval) + if _, err := v.Verify(context.Background(), k2.Sign(claims(h, token.Upload, nil)), token.Upload); err != nil { + t.Fatalf("rotated key: %v", err) + } + if n := h.fetches.Load(); n != 2 { + t.Fatalf("keys fetched %d times, want 2", n) + } +} + +func TestParseKeys(t *testing.T) { + k := tokentest.NewKey("good") + data := []byte(`{"keys":[ + {"kty":"RSA","kid":"rsa","n":"x","e":"AQAB"}, + {"kty":"OKP","crv":"X25519","kid":"x","x":"` + strings.Repeat("A", 43) + `"}, + {"kty":"OKP","crv":"Ed25519","kid":"short","x":"AAAA"}, + {"kty":"OKP","crv":"Ed25519","kid":"enc","use":"enc","x":"` + strings.Repeat("A", 43) + `"} + ]}`) + keys, err := token.ParseKeys(data) + if err != nil || len(keys) != 0 { + t.Fatalf("got %v, %v; want no keys", keys, err) + } + keys, err = token.ParseKeys(tokentest.JWKS(k)) + if err != nil || !keys["good"].Equal(k.Public) { + t.Fatalf("got %v, %v", keys, err) + } +} diff --git a/internal/token/tokentest/tokentest.go b/internal/token/tokentest/tokentest.go new file mode 100644 index 0000000..e9c25fe --- /dev/null +++ b/internal/token/tokentest/tokentest.go @@ -0,0 +1,60 @@ +// Package tokentest signs tokens the way the hub does, for tests. +package tokentest + +import ( + "crypto/ed25519" + "crypto/rand" + "encoding/base64" + "encoding/json" +) + +// Key is a test signing key. +type Key struct { + ID string + Private ed25519.PrivateKey + Public ed25519.PublicKey +} + +// NewKey makes a key with ID kid. +func NewKey(kid string) *Key { + pub, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + panic(err) + } + return &Key{ID: kid, Private: priv, Public: pub} +} + +// Sign returns a compact EdDSA JWT with these claims, with the header the hub sends. +func (k *Key) Sign(claims map[string]any) string { + return k.SignWithHeader(map[string]any{"alg": "EdDSA", "kid": k.ID, "typ": "JWT"}, claims) +} + +// SignWithHeader signs claims with any header. +func (k *Key) SignWithHeader(header, claims map[string]any) string { + input := part(header) + "." + part(claims) + return input + "." + base64.RawURLEncoding.EncodeToString(ed25519.Sign(k.Private, []byte(input))) +} + +func part(v any) string { + data, err := json.Marshal(v) + if err != nil { + panic(err) + } + return base64.RawURLEncoding.EncodeToString(data) +} + +// JWKS returns a key set holding these keys, as the hub serves it. +func JWKS(keys ...*Key) []byte { + var set struct { + Keys []map[string]string `json:"keys"` + } + set.Keys = []map[string]string{} + for _, k := range keys { + set.Keys = append(set.Keys, map[string]string{ + "kty": "OKP", "crv": "Ed25519", "x": base64.RawURLEncoding.EncodeToString(k.Public), + "kid": k.ID, "alg": "EdDSA", "use": "sig", + }) + } + data, _ := json.Marshal(set) + return data +}