A Go server that takes product uploads with hub upload tokens, checks them by t5de-api's pack rules, stores files by sha256, registers revisions with the hub (POST /api/v1/catalog/revisions), and serves files with immutable caching; files used only by adult revisions need an adult access token. Contract: t5de-hub's docs/products.md. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
dhkatzandClaude Opus 5.5 committed
commit
8742829e77
20 files changed
+2348
No files matched your search
@@ -0,0 +1,3 @@
|
|||||||
|
/dist/
|
||||||
|
/data/
|
||||||
|
.git
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
# gofmt requires LF; keep sources LF on every platform.
|
||||||
|
* text=auto eol=lf
|
||||||
|
*.exe binary
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
# Tests the catalog and builds t5de-catalog for Linux (static) and its Docker image.
|
||||||
|
name: CI
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [main]
|
||||||
|
tags: ["v*"]
|
||||||
|
pull_request:
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
build:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
env:
|
||||||
|
# T5DE's modules (t5de-api) come straight from its Gitea, not through Go's public module proxy.
|
||||||
|
GOPRIVATE: git.t5de.dev
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- uses: actions/setup-go@v5
|
||||||
|
with:
|
||||||
|
go-version-file: go.mod
|
||||||
|
|
||||||
|
- name: Test
|
||||||
|
run: |
|
||||||
|
unformatted=$(gofmt -l .)
|
||||||
|
if [ -n "$unformatted" ]; then echo "Not gofmt-formatted: $unformatted"; exit 1; fi
|
||||||
|
go vet ./...
|
||||||
|
go test ./...
|
||||||
|
|
||||||
|
# Static and reproducible: no cgo (the SQLite driver is pure Go), no paths, no build ID.
|
||||||
|
- name: Build t5de-catalog
|
||||||
|
run: |
|
||||||
|
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -buildvcs=false \
|
||||||
|
-ldflags "-s -w -buildid= -X git.t5de.dev/t5de/t5de-catalog/internal/buildinfo.Version=${GITHUB_SHA::12}" \
|
||||||
|
-o dist/t5de-catalog ./cmd/t5de-catalog
|
||||||
|
sha256sum dist/t5de-catalog
|
||||||
@@ -0,0 +1,2 @@
|
|||||||
|
/dist/
|
||||||
|
/data/
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
# t5de-catalog
|
||||||
|
|
||||||
|
T5DE's catalog server (see README.md). Part of T5DE; the contract with the hub is t5de-hub's `docs/products.md`
|
||||||
|
(git.t5de.dev/t5de/t5de-hub, usually checked out beside this one), product file rules are t5de-api's `pack`
|
||||||
|
package (`docs/pack.md`).
|
||||||
|
|
||||||
|
## Rules
|
||||||
|
|
||||||
|
- Never commit IMVU's files: tests make their product files in code (a minimal `index.xml`, fake textures).
|
||||||
|
- Never commit secrets (catalog keys, tokens).
|
||||||
|
- Commits: author "dhkatz <dev@t5de.dev>"; never the user's real name or email.
|
||||||
|
- Follow `docs/products.md` exactly; change it there (in t5de-hub) first if the contract needs to change.
|
||||||
+26
@@ -0,0 +1,26 @@
|
|||||||
|
# t5de-catalog in a container: a static build on distroless. Data (catalog.db, files/) goes in the /data volume.
|
||||||
|
#
|
||||||
|
# docker build -t t5de-catalog .
|
||||||
|
# docker run -p 4102:4102 -v catalog-data:/data -e HUB_URL=... -e CATALOG_ID=... -e CATALOG_KEY=... \
|
||||||
|
# -e CATALOG_PUBLIC_URL=... -e WEB_ORIGIN=... t5de-catalog
|
||||||
|
FROM golang:1.27-alpine AS build
|
||||||
|
RUN apk add --no-cache git
|
||||||
|
# T5DE's modules (t5de-api) come straight from its Gitea.
|
||||||
|
ENV GOPRIVATE=git.t5de.dev CGO_ENABLED=0
|
||||||
|
ARG VERSION=dev
|
||||||
|
WORKDIR /src
|
||||||
|
COPY go.mod go.sum ./
|
||||||
|
RUN go mod download
|
||||||
|
COPY . .
|
||||||
|
RUN go build -trimpath -buildvcs=false \
|
||||||
|
-ldflags "-s -w -buildid= -X git.t5de.dev/t5de/t5de-catalog/internal/buildinfo.Version=${VERSION}" \
|
||||||
|
-o /out/t5de-catalog ./cmd/t5de-catalog \
|
||||||
|
&& mkdir /out/data
|
||||||
|
|
||||||
|
FROM gcr.io/distroless/static-debian12:nonroot
|
||||||
|
COPY --from=build /out/t5de-catalog /t5de-catalog
|
||||||
|
COPY --from=build --chown=nonroot:nonroot /out/data /data
|
||||||
|
ENV CATALOG_ADDR=0.0.0.0:4102 CATALOG_DATA=/data
|
||||||
|
VOLUME /data
|
||||||
|
EXPOSE 4102
|
||||||
|
ENTRYPOINT ["/t5de-catalog"]
|
||||||
@@ -0,0 +1,75 @@
|
|||||||
|
# t5de-catalog
|
||||||
|
|
||||||
|
T5DE's catalog server: it stores community products' files by sha256, serves them, and registers uploads with
|
||||||
|
T5DE's hub (git.t5de.dev/t5de/t5de-hub). Creators upload from the hub's website straight to a catalog, so large
|
||||||
|
files never pass through the hub; T5DE's proxy then downloads the files from the catalogs the hub lists. Anyone
|
||||||
|
can run a catalog and ask the hub to approve it.
|
||||||
|
|
||||||
|
The contract (tokens, this API, the hub's API for catalogs) is t5de-hub's `docs/products.md`; the design and the
|
||||||
|
reasons are in its `docs/design.md` ("Products and catalogs", "Catalogs (`t5de-catalog`)").
|
||||||
|
|
||||||
|
Unofficial: not made by or affiliated with IMVU.
|
||||||
|
|
||||||
|
## How it works
|
||||||
|
|
||||||
|
- **Uploads** (`POST /api/v1/uploads`): the creator's browser sends the product as a zip (Create Mode's `.chkn`,
|
||||||
|
files at the root) with an upload token from the hub. The catalog checks the token (signed by the hub, for this
|
||||||
|
catalog), checks the files by t5de-api's pack rules (names, limits counted while reading, `index.xml` with
|
||||||
|
`ProductRating` GA or AP), stores each file under its sha256, and registers the revision with the hub
|
||||||
|
(`POST /api/v1/catalog/revisions`, `Authorization: Catalog <id>:<key>`). Only once the hub accepts it is the
|
||||||
|
revision recorded as hosted; files of a refused upload stay on disk but aren't served.
|
||||||
|
- **Files** (`GET /api/v1/files/<sha256>`): immutable, so cached for a year (`ETag` is the hash; ranges work). A
|
||||||
|
file used only by adult revisions needs an access token from the hub with `adult: true`, and isn't cached by
|
||||||
|
shared caches.
|
||||||
|
- **Tokens** are EdDSA (Ed25519) JWTs; the hub's keys come from `HUB_URL/.well-known/t5de-keys.json`, cached and
|
||||||
|
fetched again when a token names an unknown key (at most every 30 s).
|
||||||
|
- **Storage**: `CATALOG_DATA/catalog.db` (SQLite: the revisions hosted and which files belong to them) and
|
||||||
|
`CATALOG_DATA/files/<first 2 hex>/<sha256>`, written whole (temp file, then rename). `CATALOG_DATA/tmp` holds
|
||||||
|
uploads being checked and is emptied at start.
|
||||||
|
|
||||||
|
## API
|
||||||
|
|
||||||
|
| | |
|
||||||
|
| --- | --- |
|
||||||
|
| `GET /health` | `{ok: true}` |
|
||||||
|
| `POST /api/v1/uploads` | `Authorization: Bearer <t5de-upload token>`; multipart: `file` (the zip), `notes` (required for a new revision of an existing product). 201 `{product, revision}`. Errors are `{error, message}`: 400 bad files or form (`bad_files`, `notes_required`, `bad_request`), 401 bad token (`bad_token`), 413 over 300 MiB (`too_large`), 502 the hub refused (its error code, e.g. `unchanged`) or couldn't be reached (`hub_unavailable`). CORS for `WEB_ORIGIN`. |
|
||||||
|
| `GET /api/v1/files/<sha256>` | the file; 401/403 for adult-only files without an adult access token; 404 if not hosted |
|
||||||
|
| `GET /api/v1/revisions/<product>/<revision>` | `{product, revision, rating, files}` for revisions hosted here |
|
||||||
|
|
||||||
|
## Running
|
||||||
|
|
||||||
|
Register the catalog on the hub's website (`POST /api/v1/catalogs`) to get its ID and key, and have a hub
|
||||||
|
moderator approve it. Then, for a hub running locally:
|
||||||
|
|
||||||
|
HUB_URL=http://localhost:4101 \
|
||||||
|
CATALOG_ID=<id> CATALOG_KEY=<key> \
|
||||||
|
CATALOG_PUBLIC_URL=http://localhost:4102 \
|
||||||
|
WEB_ORIGIN=http://localhost:4100 \
|
||||||
|
CATALOG_DATA=./data \
|
||||||
|
go run ./cmd/t5de-catalog
|
||||||
|
|
||||||
|
| Variable | |
|
||||||
|
| --- | --- |
|
||||||
|
| `CATALOG_ADDR` | listen address (default `127.0.0.1:4102`) |
|
||||||
|
| `CATALOG_DATA` | data folder (default `./data`) |
|
||||||
|
| `CATALOG_PUBLIC_URL` | the catalog's public URL, as registered on the hub (logged) |
|
||||||
|
| `HUB_URL` | the hub's API; also the tokens' issuer (required) |
|
||||||
|
| `CATALOG_ID`, `CATALOG_KEY` | from registering on the hub (required) |
|
||||||
|
| `WEB_ORIGIN` | the hub website's origin, allowed to upload from browsers |
|
||||||
|
|
||||||
|
It logs to standard error and stops cleanly on Ctrl+C or SIGTERM, finishing requests in progress. Put it behind
|
||||||
|
a TLS-terminating reverse proxy (or a CDN for `/api/v1/files/`) in production.
|
||||||
|
|
||||||
|
With Docker:
|
||||||
|
|
||||||
|
docker build -t t5de-catalog .
|
||||||
|
docker run -p 4102:4102 -v catalog-data:/data -e HUB_URL=... -e CATALOG_ID=... -e CATALOG_KEY=... \
|
||||||
|
-e CATALOG_PUBLIC_URL=... -e WEB_ORIGIN=... t5de-catalog
|
||||||
|
|
||||||
|
## Building
|
||||||
|
|
||||||
|
go test ./...
|
||||||
|
CGO_ENABLED=0 go build -trimpath -o dist/t5de-catalog ./cmd/t5de-catalog
|
||||||
|
|
||||||
|
The SQLite driver is pure Go (modernc.org/sqlite), so builds are static. t5de-api comes from git.t5de.dev
|
||||||
|
directly: set `GOPRIVATE=git.t5de.dev`.
|
||||||
@@ -0,0 +1,130 @@
|
|||||||
|
// Command t5de-catalog is T5DE's catalog server: it stores community product files by sha256, serves them, and
|
||||||
|
// registers uploads with T5DE's hub (t5de-hub's docs/products.md). Configured by environment variables:
|
||||||
|
//
|
||||||
|
// CATALOG_ADDR listen address (default 127.0.0.1:4102)
|
||||||
|
// CATALOG_DATA data folder: catalog.db and files/ (default ./data)
|
||||||
|
// CATALOG_PUBLIC_URL the catalog's public URL, as registered on the hub
|
||||||
|
// HUB_URL the hub's API (required), e.g. https://api.t5de.dev
|
||||||
|
// CATALOG_ID the catalog's ID on the hub (required)
|
||||||
|
// CATALOG_KEY its key (required)
|
||||||
|
// WEB_ORIGIN the hub website's origin, allowed to upload from browsers (CORS)
|
||||||
|
//
|
||||||
|
// It logs to standard error and stops cleanly on SIGINT or SIGTERM.
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"os"
|
||||||
|
"os/signal"
|
||||||
|
"strings"
|
||||||
|
"syscall"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.t5de.dev/t5de/t5de-catalog/internal/buildinfo"
|
||||||
|
"git.t5de.dev/t5de/t5de-catalog/internal/hub"
|
||||||
|
"git.t5de.dev/t5de/t5de-catalog/internal/server"
|
||||||
|
"git.t5de.dev/t5de/t5de-catalog/internal/store"
|
||||||
|
"git.t5de.dev/t5de/t5de-catalog/internal/token"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
if len(os.Args) > 1 && (os.Args[1] == "-version" || os.Args[1] == "--version") {
|
||||||
|
fmt.Println(buildinfo.Version)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
logger := log.New(os.Stderr, "", log.Ldate|log.Ltime|log.Lmicroseconds)
|
||||||
|
if err := run(logger); err != nil {
|
||||||
|
logger.Printf("t5de-catalog: %v", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func env(name, def string) string {
|
||||||
|
if v := strings.TrimSpace(os.Getenv(name)); v != "" {
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
return def
|
||||||
|
}
|
||||||
|
|
||||||
|
func run(logger *log.Logger) error {
|
||||||
|
addr := env("CATALOG_ADDR", "127.0.0.1:4102")
|
||||||
|
data := env("CATALOG_DATA", "data")
|
||||||
|
publicURL := env("CATALOG_PUBLIC_URL", "")
|
||||||
|
hubURL := strings.TrimRight(env("HUB_URL", ""), "/")
|
||||||
|
id := env("CATALOG_ID", "")
|
||||||
|
key := env("CATALOG_KEY", "")
|
||||||
|
webOrigin := strings.TrimRight(env("WEB_ORIGIN", ""), "/")
|
||||||
|
|
||||||
|
var missing []string
|
||||||
|
for _, v := range [][2]string{{"HUB_URL", hubURL}, {"CATALOG_ID", id}, {"CATALOG_KEY", key}} {
|
||||||
|
if v[1] == "" {
|
||||||
|
missing = append(missing, v[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(missing) > 0 {
|
||||||
|
return fmt.Errorf("set %s", strings.Join(missing, ", "))
|
||||||
|
}
|
||||||
|
if u, err := url.Parse(hubURL); err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
|
||||||
|
return fmt.Errorf("HUB_URL %q isn't an http(s) URL", hubURL)
|
||||||
|
}
|
||||||
|
if publicURL == "" {
|
||||||
|
logger.Printf("CATALOG_PUBLIC_URL isn't set")
|
||||||
|
}
|
||||||
|
if webOrigin == "" {
|
||||||
|
logger.Printf("WEB_ORIGIN isn't set: browsers can't upload")
|
||||||
|
}
|
||||||
|
|
||||||
|
st, err := store.Open(data)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer st.Close()
|
||||||
|
|
||||||
|
srv := &http.Server{
|
||||||
|
Addr: addr,
|
||||||
|
Handler: server.New(server.Config{
|
||||||
|
Store: st,
|
||||||
|
Verifier: token.NewVerifier(hubURL, nil),
|
||||||
|
Hub: hub.New(hubURL, id, key, nil),
|
||||||
|
CatalogID: id,
|
||||||
|
WebOrigin: webOrigin,
|
||||||
|
Logger: logger,
|
||||||
|
}),
|
||||||
|
ReadHeaderTimeout: 10 * time.Second,
|
||||||
|
ReadTimeout: 15 * time.Minute, // large uploads on slow connections
|
||||||
|
WriteTimeout: 15 * time.Minute,
|
||||||
|
IdleTimeout: 2 * time.Minute,
|
||||||
|
ErrorLog: logger,
|
||||||
|
}
|
||||||
|
ln, err := net.Listen("tcp", addr)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
logger.Printf("t5de-catalog %s: catalog %s on %s (public URL %s, hub %s, data %s)", buildinfo.Version, id, ln.Addr(), publicURL, hubURL, data)
|
||||||
|
|
||||||
|
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||||
|
defer stop()
|
||||||
|
served := make(chan error, 1)
|
||||||
|
go func() { served <- srv.Serve(ln) }()
|
||||||
|
select {
|
||||||
|
case err := <-served:
|
||||||
|
return err
|
||||||
|
case <-ctx.Done():
|
||||||
|
}
|
||||||
|
logger.Printf("stopping")
|
||||||
|
shutdown, cancel := context.WithTimeout(context.Background(), time.Minute)
|
||||||
|
defer cancel()
|
||||||
|
if err := srv.Shutdown(shutdown); err != nil && !errors.Is(err, context.DeadlineExceeded) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := <-served; !errors.Is(err, http.ErrServerClosed) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
module git.t5de.dev/t5de/t5de-catalog
|
||||||
|
|
||||||
|
go 1.27.0
|
||||||
|
|
||||||
|
require (
|
||||||
|
git.t5de.dev/t5de/t5de-api v0.0.0-20261029101713-106b4fcc6064
|
||||||
|
modernc.org/sqlite v1.61.0
|
||||||
|
)
|
||||||
|
|
||||||
|
require (
|
||||||
|
github.com/dustin/go-humanize v1.1.0 // indirect
|
||||||
|
github.com/google/uuid v1.6.0 // indirect
|
||||||
|
github.com/mattn/go-isatty v0.0.24 // indirect
|
||||||
|
github.com/ncruces/go-strftime v1.1.0 // indirect
|
||||||
|
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||||
|
golang.org/x/sys v0.49.0 // indirect
|
||||||
|
modernc.org/libc v1.79.0 // indirect
|
||||||
|
modernc.org/mathutil v1.7.1 // indirect
|
||||||
|
modernc.org/memory v1.12.1 // indirect
|
||||||
|
)
|
||||||
@@ -0,0 +1,52 @@
|
|||||||
|
git.t5de.dev/t5de/t5de-api v0.0.0-20261029101713-106b4fcc6064 h1:UdRdoSTTZHjUTW7wF9+EVJ7UFgBktXrls3FsMKiOp2Y=
|
||||||
|
git.t5de.dev/t5de/t5de-api v0.0.0-20261029101713-106b4fcc6064/go.mod h1:mRXJdl43TzJN3JlUeVhi6BSqle37+ALmqzxqTYchXLw=
|
||||||
|
github.com/dustin/go-humanize v1.1.0 h1:dbKTrvD0klcbBV/h4AWJdMuZogJACoMlvWIWZ5b2xWg=
|
||||||
|
github.com/dustin/go-humanize v1.1.0/go.mod h1:hc1CvRkJMsgxqjmjMQF3QNRAZBwY8AXBAzKYoSX9sFI=
|
||||||
|
github.com/google/pprof v0.0.0-20261016150608-1facf8f91b50 h1:PlNWJRxu5APSV0pWqfIsZ0qt9OOGHSWfZQ4zvx+WgWs=
|
||||||
|
github.com/google/pprof v0.0.0-20261016150608-1facf8f91b50/go.mod h1:yUo1AMMpsPIDFGuwDK8nLcf8+vvAd5l2O6JDZ6hpwrM=
|
||||||
|
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||||
|
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||||
|
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
|
||||||
|
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
|
||||||
|
github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI=
|
||||||
|
github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
|
||||||
|
github.com/ncruces/go-strftime v1.1.0 h1:vtPmPWMkb539RGV9WpWT9EwJGvrOCKvMNI4obWL47Hk=
|
||||||
|
github.com/ncruces/go-strftime v1.1.0/go.mod h1:yVaq7iwCN09vHRrKBrrflXT0M9tTMG2KmZ9Fj7xeG/g=
|
||||||
|
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||||
|
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||||
|
golang.org/x/mod v0.42.0 h1:ICeL3t2L30ljSPHHxDnR4fqVcBt/zR2zJAb9juavmQg=
|
||||||
|
golang.org/x/mod v0.42.0/go.mod h1:u5fzIrRInkL8MaO1cbRkUHT64v303YIT5KEqpNr7jRs=
|
||||||
|
golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA=
|
||||||
|
golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
|
||||||
|
golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA=
|
||||||
|
golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||||
|
golang.org/x/tools v0.52.0 h1:ryxg4U63oaUYk7uOLTF37n43f4qgtGvqV4Ioit1BaEs=
|
||||||
|
golang.org/x/tools v0.52.0/go.mod h1:ItcP5UNaVGG+Yc4eIYmaFLNJ4soEopkx5Y2d4lcEurs=
|
||||||
|
modernc.org/cc/v4 v4.30.3 h1:aDq8gIX1X7+VFgO/JYYji6pLWL2H4tv3g8WatT3KCys=
|
||||||
|
modernc.org/cc/v4 v4.30.3/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
|
||||||
|
modernc.org/ccgo/v4 v4.37.0 h1:BAuJCUtIHIxFIYaFhqXIw5YaoMBlNCUlFovejHfMip4=
|
||||||
|
modernc.org/ccgo/v4 v4.37.0/go.mod h1:/8627VhwWtAbqUXmPORxFQU5SgLrAx2jl9seLw9GBhY=
|
||||||
|
modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM=
|
||||||
|
modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU=
|
||||||
|
modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
|
||||||
|
modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
|
||||||
|
modernc.org/gc/v3 v3.2.0 h1:/Y3LGDMQaoYf55PebP1qsZ5FDGgAZyG1RYKEHeQzh1o=
|
||||||
|
modernc.org/gc/v3 v3.2.0/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
|
||||||
|
modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
|
||||||
|
modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
|
||||||
|
modernc.org/libc v1.79.0 h1:9Dvvv++bAMSkSuBzuOFAatbTV5OIoq6Y6FohmD3e7q8=
|
||||||
|
modernc.org/libc v1.79.0/go.mod h1:uT3kR61NsbeKJSLqq83GJwSnZMVLlSjyAz4DfTaBy1M=
|
||||||
|
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
|
||||||
|
modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
|
||||||
|
modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g=
|
||||||
|
modernc.org/memory v1.12.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
|
||||||
|
modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg=
|
||||||
|
modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
|
||||||
|
modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
|
||||||
|
modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
|
||||||
|
modernc.org/sqlite v1.61.0 h1:sQi63oLQ9uquAUA4CzOisH35Q3pEszkPBuHM0tFL8CY=
|
||||||
|
modernc.org/sqlite v1.61.0/go.mod h1:f4gr5tbpx2umklaT/QVtbi/50hNdM+4ng+2zMo8KXJ0=
|
||||||
|
modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
|
||||||
|
modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
|
||||||
|
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
|
||||||
|
modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
// Package buildinfo holds what release builds set with -ldflags "-X ...".
|
||||||
|
package buildinfo
|
||||||
|
|
||||||
|
// Version is the release's version, "X.Y.Z" or "X.Y.Z-nightly.N+commit".
|
||||||
|
var Version = "dev"
|
||||||
@@ -0,0 +1,113 @@
|
|||||||
|
// Package hub is the catalog's client for T5DE's hub (server to server, docs/products.md "For catalogs").
|
||||||
|
package hub
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.t5de.dev/t5de/t5de-catalog/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Client calls the hub as one catalog.
|
||||||
|
type Client struct {
|
||||||
|
url string
|
||||||
|
auth string
|
||||||
|
client *http.Client
|
||||||
|
}
|
||||||
|
|
||||||
|
// New returns a client for the hub at hubURL (its API), as catalog id with its key. client may be nil.
|
||||||
|
func New(hubURL, id, key string, client *http.Client) *Client {
|
||||||
|
if client == nil {
|
||||||
|
client = &http.Client{Timeout: 60 * time.Second}
|
||||||
|
}
|
||||||
|
return &Client{url: strings.TrimRight(hubURL, "/"), auth: "Catalog " + id + ":" + key, client: client}
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewRevision is what the catalog registers.
|
||||||
|
type NewRevision struct {
|
||||||
|
UploadToken string `json:"uploadToken"`
|
||||||
|
Files []store.File `json:"files"`
|
||||||
|
Rating string `json:"rating"`
|
||||||
|
Parent int64 `json:"parent"`
|
||||||
|
Name string `json:"name"`
|
||||||
|
Notes string `json:"notes"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Registered is the hub's answer.
|
||||||
|
type Registered struct {
|
||||||
|
Product string `json:"product"`
|
||||||
|
Revision int `json:"revision"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Error is the hub refusing a request: its status and, when it sent them, its error code and message.
|
||||||
|
type Error struct {
|
||||||
|
Status int
|
||||||
|
Code string
|
||||||
|
Message string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Error) Error() string {
|
||||||
|
if e.Code != "" {
|
||||||
|
return fmt.Sprintf("hub: %d %s: %s", e.Status, e.Code, e.Message)
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("hub: %d", e.Status)
|
||||||
|
}
|
||||||
|
|
||||||
|
var productID = regexp.MustCompile(`^[a-z2-7]{10}$`)
|
||||||
|
|
||||||
|
// ValidProduct reports whether s is a product ID (10 of a-z2-7).
|
||||||
|
func ValidProduct(s string) bool { return productID.MatchString(s) }
|
||||||
|
|
||||||
|
// RegisterRevision posts a revision to the hub (POST /api/v1/catalog/revisions). A refusal is an *Error; other
|
||||||
|
// errors mean the hub couldn't be reached or answered nonsense.
|
||||||
|
func (c *Client) RegisterRevision(ctx context.Context, r NewRevision) (*Registered, error) {
|
||||||
|
if r.Files == nil {
|
||||||
|
r.Files = []store.File{}
|
||||||
|
}
|
||||||
|
body, err := json.Marshal(r)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.url+"/api/v1/catalog/revisions", bytes.NewReader(body))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
req.Header.Set("Authorization", c.auth)
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
req.Header.Set("Accept", "application/json")
|
||||||
|
resp, err := c.client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
data, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK {
|
||||||
|
e := &Error{Status: resp.StatusCode}
|
||||||
|
var msg struct {
|
||||||
|
Error string `json:"error"`
|
||||||
|
Message string `json:"message"`
|
||||||
|
}
|
||||||
|
if json.Unmarshal(data, &msg) == nil {
|
||||||
|
e.Code, e.Message = msg.Error, msg.Message
|
||||||
|
}
|
||||||
|
return nil, e
|
||||||
|
}
|
||||||
|
var reg Registered
|
||||||
|
if err := json.Unmarshal(data, ®); err != nil {
|
||||||
|
return nil, fmt.Errorf("hub: bad answer: %v", err)
|
||||||
|
}
|
||||||
|
if !ValidProduct(reg.Product) || reg.Revision < 1 {
|
||||||
|
return nil, fmt.Errorf("hub: bad answer: product %q, revision %d", reg.Product, reg.Revision)
|
||||||
|
}
|
||||||
|
return ®, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,194 @@
|
|||||||
|
// Package server is the catalog's HTTP API (t5de-hub's docs/products.md, "Catalog API").
|
||||||
|
package server
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"log"
|
||||||
|
"net/http"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.t5de.dev/t5de/t5de-catalog/internal/hub"
|
||||||
|
"git.t5de.dev/t5de/t5de-catalog/internal/store"
|
||||||
|
"git.t5de.dev/t5de/t5de-catalog/internal/token"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Config is what the server needs.
|
||||||
|
type Config struct {
|
||||||
|
Store *store.Store
|
||||||
|
Verifier *token.Verifier
|
||||||
|
Hub *hub.Client
|
||||||
|
CatalogID string
|
||||||
|
WebOrigin string // the hub website's origin, allowed to upload from browsers (CORS); empty for none
|
||||||
|
Logger *log.Logger
|
||||||
|
// MaxUploads is how many uploads are checked at once (each can hold up to 256 MiB of files in memory);
|
||||||
|
// default 2.
|
||||||
|
MaxUploads int
|
||||||
|
}
|
||||||
|
|
||||||
|
// Server serves the catalog's API.
|
||||||
|
type Server struct {
|
||||||
|
cfg Config
|
||||||
|
uploads chan struct{}
|
||||||
|
mux *http.ServeMux
|
||||||
|
}
|
||||||
|
|
||||||
|
// New returns the catalog's API.
|
||||||
|
func New(cfg Config) *Server {
|
||||||
|
if cfg.MaxUploads < 1 {
|
||||||
|
cfg.MaxUploads = 2
|
||||||
|
}
|
||||||
|
if cfg.Logger == nil {
|
||||||
|
cfg.Logger = log.Default()
|
||||||
|
}
|
||||||
|
cfg.WebOrigin = strings.TrimRight(cfg.WebOrigin, "/")
|
||||||
|
s := &Server{cfg: cfg, uploads: make(chan struct{}, cfg.MaxUploads), mux: http.NewServeMux()}
|
||||||
|
s.mux.HandleFunc("GET /health", s.health)
|
||||||
|
s.mux.HandleFunc("OPTIONS /api/v1/uploads", s.preflight)
|
||||||
|
s.mux.HandleFunc("POST /api/v1/uploads", s.upload)
|
||||||
|
s.mux.HandleFunc("GET /api/v1/files/{sha256}", s.file)
|
||||||
|
s.mux.HandleFunc("GET /api/v1/revisions/{product}/{revision}", s.revision)
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
// ServeHTTP serves a request and logs it.
|
||||||
|
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||||
|
start := time.Now()
|
||||||
|
rec := &recorder{ResponseWriter: w, status: http.StatusOK}
|
||||||
|
s.mux.ServeHTTP(rec, r)
|
||||||
|
s.cfg.Logger.Printf("%s %s %d %dB %s", r.Method, r.URL.Path, rec.status, rec.bytes, time.Since(start).Round(time.Millisecond))
|
||||||
|
}
|
||||||
|
|
||||||
|
type recorder struct {
|
||||||
|
http.ResponseWriter
|
||||||
|
status int
|
||||||
|
bytes int64
|
||||||
|
wrote bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *recorder) WriteHeader(status int) {
|
||||||
|
if !r.wrote {
|
||||||
|
r.status, r.wrote = status, true
|
||||||
|
}
|
||||||
|
r.ResponseWriter.WriteHeader(status)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *recorder) Write(b []byte) (int, error) {
|
||||||
|
r.wrote = true
|
||||||
|
n, err := r.ResponseWriter.Write(b)
|
||||||
|
r.bytes += int64(n)
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *recorder) Unwrap() http.ResponseWriter { return r.ResponseWriter }
|
||||||
|
|
||||||
|
func writeJSON(w http.ResponseWriter, status int, v any) {
|
||||||
|
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||||
|
w.WriteHeader(status)
|
||||||
|
json.NewEncoder(w).Encode(v)
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeError(w http.ResponseWriter, status int, code, message string) {
|
||||||
|
if w.Header().Get("Cache-Control") == "" {
|
||||||
|
w.Header().Set("Cache-Control", "no-store")
|
||||||
|
}
|
||||||
|
writeJSON(w, status, map[string]string{"error": code, "message": message})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) health(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.Header().Set("Cache-Control", "no-store")
|
||||||
|
writeJSON(w, http.StatusOK, map[string]bool{"ok": true})
|
||||||
|
}
|
||||||
|
|
||||||
|
// bearer returns the request's bearer token, if any.
|
||||||
|
func bearer(r *http.Request) string {
|
||||||
|
scheme, tok, ok := strings.Cut(r.Header.Get("Authorization"), " ")
|
||||||
|
if !ok || !strings.EqualFold(scheme, "Bearer") {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return strings.TrimSpace(tok)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) file(w http.ResponseWriter, r *http.Request) {
|
||||||
|
hash := r.PathValue("sha256")
|
||||||
|
if !store.ValidHash(hash) {
|
||||||
|
writeError(w, http.StatusNotFound, "not_found", "No such file.")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
hosted, adultOnly, err := s.cfg.Store.FileAccess(hash)
|
||||||
|
if err != nil {
|
||||||
|
s.cfg.Logger.Printf("files: %s: %v", hash, err)
|
||||||
|
writeError(w, http.StatusInternalServerError, "internal", "Something went wrong.")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !hosted {
|
||||||
|
writeError(w, http.StatusNotFound, "not_found", "No such file.")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
h := w.Header()
|
||||||
|
if adultOnly {
|
||||||
|
h.Set("Cache-Control", "private, no-store")
|
||||||
|
h.Set("Vary", "Authorization")
|
||||||
|
tok := bearer(r)
|
||||||
|
if tok == "" {
|
||||||
|
h.Set("WWW-Authenticate", `Bearer realm="t5de-catalog"`)
|
||||||
|
writeError(w, http.StatusUnauthorized, "token_required", "This file needs an access token.")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
c, err := s.cfg.Verifier.Verify(r.Context(), tok, token.Access)
|
||||||
|
if err != nil {
|
||||||
|
h.Set("WWW-Authenticate", `Bearer realm="t5de-catalog", error="invalid_token"`)
|
||||||
|
writeError(w, http.StatusUnauthorized, "bad_token", "The access token isn't valid.")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !c.Adult {
|
||||||
|
writeError(w, http.StatusForbidden, "adult_only", "This file is for adults only.")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
f, err := s.cfg.Store.Open(hash)
|
||||||
|
if errors.Is(err, store.ErrNotFound) {
|
||||||
|
s.cfg.Logger.Printf("files: %s is in a hosted revision but missing on disk", hash)
|
||||||
|
writeError(w, http.StatusNotFound, "not_found", "No such file.")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
s.cfg.Logger.Printf("files: %s: %v", hash, err)
|
||||||
|
writeError(w, http.StatusInternalServerError, "internal", "Something went wrong.")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
if adultOnly {
|
||||||
|
h.Set("Cache-Control", "private, max-age=31536000, immutable")
|
||||||
|
} else {
|
||||||
|
h.Set("Cache-Control", "public, max-age=31536000, immutable")
|
||||||
|
}
|
||||||
|
h.Set("ETag", `"`+hash+`"`)
|
||||||
|
h.Set("Content-Type", "application/octet-stream")
|
||||||
|
h.Set("X-Content-Type-Options", "nosniff")
|
||||||
|
http.ServeContent(w, r, "", time.Time{}, f)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) revision(w http.ResponseWriter, r *http.Request) {
|
||||||
|
product := r.PathValue("product")
|
||||||
|
n, err := strconv.Atoi(r.PathValue("revision"))
|
||||||
|
if !hub.ValidProduct(product) || err != nil || n < 1 {
|
||||||
|
writeError(w, http.StatusNotFound, "not_found", "No such revision.")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
rev, err := s.cfg.Store.Revision(product, n)
|
||||||
|
if errors.Is(err, store.ErrNotFound) {
|
||||||
|
writeError(w, http.StatusNotFound, "not_found", "No such revision.")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
s.cfg.Logger.Printf("revisions: %s/%d: %v", product, n, err)
|
||||||
|
writeError(w, http.StatusInternalServerError, "internal", "Something went wrong.")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// Revisions never change, but moderation can remove one.
|
||||||
|
w.Header().Set("Cache-Control", "public, max-age=3600")
|
||||||
|
writeJSON(w, http.StatusOK, rev)
|
||||||
|
}
|
||||||
@@ -0,0 +1,514 @@
|
|||||||
|
package server_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"archive/zip"
|
||||||
|
"bytes"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"log"
|
||||||
|
"mime/multipart"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.t5de.dev/t5de/t5de-catalog/internal/hub"
|
||||||
|
"git.t5de.dev/t5de/t5de-catalog/internal/server"
|
||||||
|
"git.t5de.dev/t5de/t5de-catalog/internal/store"
|
||||||
|
"git.t5de.dev/t5de/t5de-catalog/internal/token"
|
||||||
|
"git.t5de.dev/t5de/t5de-catalog/internal/token/tokentest"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
catalogID = "cat1"
|
||||||
|
catalogKey = "secret-key"
|
||||||
|
webOrigin = "https://t5de.example"
|
||||||
|
)
|
||||||
|
|
||||||
|
// fakeHub serves a JWKS, issues tokens and accepts revisions as docs/products.md says.
|
||||||
|
type fakeHub struct {
|
||||||
|
*httptest.Server
|
||||||
|
t *testing.T
|
||||||
|
key *tokentest.Key
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
down bool
|
||||||
|
products map[string][][]store.File // product -> file list of each revision
|
||||||
|
last *hub.NewRevision
|
||||||
|
calls int
|
||||||
|
next int
|
||||||
|
}
|
||||||
|
|
||||||
|
func newFakeHub(t *testing.T) *fakeHub {
|
||||||
|
h := &fakeHub{t: t, key: tokentest.NewKey("hubkey"), products: map[string][][]store.File{}}
|
||||||
|
h.Server = httptest.NewServer(http.HandlerFunc(h.serve))
|
||||||
|
t.Cleanup(h.Close)
|
||||||
|
return h
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *fakeHub) serve(w http.ResponseWriter, r *http.Request) {
|
||||||
|
switch {
|
||||||
|
case r.Method == http.MethodGet && r.URL.Path == token.KeysPath:
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
w.Write(tokentest.JWKS(h.key))
|
||||||
|
case r.Method == http.MethodPost && r.URL.Path == "/api/v1/catalog/revisions":
|
||||||
|
h.revisions(w, r)
|
||||||
|
default:
|
||||||
|
http.NotFound(w, r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func refuse(w http.ResponseWriter, status int, code, msg string) {
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
w.WriteHeader(status)
|
||||||
|
json.NewEncoder(w).Encode(map[string]string{"error": code, "message": msg})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *fakeHub) revisions(w http.ResponseWriter, r *http.Request) {
|
||||||
|
h.mu.Lock()
|
||||||
|
defer h.mu.Unlock()
|
||||||
|
h.calls++
|
||||||
|
if h.down {
|
||||||
|
http.Error(w, "down", http.StatusServiceUnavailable)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if r.Header.Get("Authorization") != "Catalog "+catalogID+":"+catalogKey {
|
||||||
|
refuse(w, http.StatusUnauthorized, "bad_catalog", "Unknown catalog.")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var req hub.NewRevision
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||||
|
refuse(w, http.StatusBadRequest, "invalid", err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
h.last = &req
|
||||||
|
v := token.NewVerifier(h.URL, h.Client())
|
||||||
|
c, err := v.Verify(r.Context(), req.UploadToken, token.Upload)
|
||||||
|
if err != nil || c.Catalog != catalogID {
|
||||||
|
refuse(w, http.StatusForbidden, "bad_token", "Not for this catalog.")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
product := c.Product
|
||||||
|
if product == "" {
|
||||||
|
h.next++
|
||||||
|
product = fmt.Sprintf("prod%06d", h.next)
|
||||||
|
product = strings.NewReplacer("0", "a", "1", "b", "8", "c", "9", "d").Replace(product)
|
||||||
|
} else {
|
||||||
|
revs := h.products[product]
|
||||||
|
if revs == nil {
|
||||||
|
refuse(w, http.StatusNotFound, "not_found", "No such product.")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if req.Notes == "" {
|
||||||
|
refuse(w, http.StatusBadRequest, "notes_required", "Notes are required.")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if slices.Equal(revs[len(revs)-1], req.Files) {
|
||||||
|
refuse(w, http.StatusConflict, "unchanged", "These files are the same as the latest revision's.")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
h.products[product] = append(h.products[product], req.Files)
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
w.WriteHeader(http.StatusCreated)
|
||||||
|
json.NewEncoder(w).Encode(hub.Registered{Product: product, Revision: len(h.products[product])})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *fakeHub) token(typ string, claims map[string]any) string {
|
||||||
|
c := map[string]any{"iss": h.URL, "typ": typ, "sub": "person1", "iat": time.Now().Unix(), "exp": time.Now().Add(30 * time.Minute).Unix()}
|
||||||
|
for k, v := range claims {
|
||||||
|
c[k] = v
|
||||||
|
}
|
||||||
|
return h.key.Sign(c)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *fakeHub) uploadToken(product string) string {
|
||||||
|
c := map[string]any{"catalog": catalogID}
|
||||||
|
if product != "" {
|
||||||
|
c["product"] = product
|
||||||
|
}
|
||||||
|
return h.token(token.Upload, c)
|
||||||
|
}
|
||||||
|
|
||||||
|
type env struct {
|
||||||
|
hub *fakeHub
|
||||||
|
catalog *httptest.Server
|
||||||
|
store *store.Store
|
||||||
|
}
|
||||||
|
|
||||||
|
func setup(t *testing.T) *env {
|
||||||
|
h := newFakeHub(t)
|
||||||
|
st, err := store.Open(t.TempDir())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { st.Close() })
|
||||||
|
srv := server.New(server.Config{
|
||||||
|
Store: st,
|
||||||
|
Verifier: token.NewVerifier(h.URL, h.Client()),
|
||||||
|
Hub: hub.New(h.URL, catalogID, catalogKey, h.Client()),
|
||||||
|
CatalogID: catalogID,
|
||||||
|
WebOrigin: webOrigin,
|
||||||
|
Logger: log.New(io.Discard, "", 0),
|
||||||
|
})
|
||||||
|
c := httptest.NewServer(srv)
|
||||||
|
t.Cleanup(c.Close)
|
||||||
|
return &env{hub: h, catalog: c, store: st}
|
||||||
|
}
|
||||||
|
|
||||||
|
type file struct{ name, data string }
|
||||||
|
|
||||||
|
func indexXML(rating, name string) string {
|
||||||
|
return `<?xml version="1.0" encoding="utf-8"?>
|
||||||
|
<Template>
|
||||||
|
<ProductName>` + name + `</ProductName>
|
||||||
|
<ProductRating>` + rating + `</ProductRating>
|
||||||
|
<ProductGender>female</ProductGender>
|
||||||
|
<__DATAIMPORT>product://12345/index.xml</__DATAIMPORT>
|
||||||
|
</Template>
|
||||||
|
`
|
||||||
|
}
|
||||||
|
|
||||||
|
func makeZip(t *testing.T, files ...file) []byte {
|
||||||
|
var b bytes.Buffer
|
||||||
|
z := zip.NewWriter(&b)
|
||||||
|
for _, f := range files {
|
||||||
|
w, err := z.Create(f.name)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
w.Write([]byte(f.data))
|
||||||
|
}
|
||||||
|
if err := z.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return b.Bytes()
|
||||||
|
}
|
||||||
|
|
||||||
|
// texture is a tiny synthetic "PNG": the catalog doesn't decode images.
|
||||||
|
const texture = "\x89PNG\r\n\x1a\nnot really an image, just test bytes"
|
||||||
|
|
||||||
|
type answer struct {
|
||||||
|
status int
|
||||||
|
header http.Header
|
||||||
|
body map[string]any
|
||||||
|
raw []byte
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *env) upload(t *testing.T, tok string, zipData []byte, notes *string) answer {
|
||||||
|
t.Helper()
|
||||||
|
var b bytes.Buffer
|
||||||
|
mw := multipart.NewWriter(&b)
|
||||||
|
if notes != nil {
|
||||||
|
mw.WriteField("notes", *notes)
|
||||||
|
}
|
||||||
|
if zipData != nil {
|
||||||
|
fw, _ := mw.CreateFormFile("file", "product.chkn")
|
||||||
|
fw.Write(zipData)
|
||||||
|
}
|
||||||
|
mw.Close()
|
||||||
|
req, _ := http.NewRequest(http.MethodPost, e.catalog.URL+"/api/v1/uploads", &b)
|
||||||
|
req.Header.Set("Content-Type", mw.FormDataContentType())
|
||||||
|
req.Header.Set("Origin", webOrigin)
|
||||||
|
if tok != "" {
|
||||||
|
req.Header.Set("Authorization", "Bearer "+tok)
|
||||||
|
}
|
||||||
|
return do(t, req)
|
||||||
|
}
|
||||||
|
|
||||||
|
func do(t *testing.T, req *http.Request) answer {
|
||||||
|
t.Helper()
|
||||||
|
resp, err := http.DefaultClient.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
raw, _ := io.ReadAll(resp.Body)
|
||||||
|
a := answer{status: resp.StatusCode, header: resp.Header, raw: raw}
|
||||||
|
json.Unmarshal(raw, &a.body)
|
||||||
|
return a
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *env) get(t *testing.T, path string, headers ...string) answer {
|
||||||
|
t.Helper()
|
||||||
|
req, _ := http.NewRequest(http.MethodGet, e.catalog.URL+path, nil)
|
||||||
|
for i := 0; i+1 < len(headers); i += 2 {
|
||||||
|
req.Header.Set(headers[i], headers[i+1])
|
||||||
|
}
|
||||||
|
return do(t, req)
|
||||||
|
}
|
||||||
|
|
||||||
|
func sum(s string) string {
|
||||||
|
h := sha256.Sum256([]byte(s))
|
||||||
|
return hex.EncodeToString(h[:])
|
||||||
|
}
|
||||||
|
|
||||||
|
func ptr(s string) *string { return &s }
|
||||||
|
|
||||||
|
func TestUploadAndServe(t *testing.T) {
|
||||||
|
e := setup(t)
|
||||||
|
index := indexXML("GA", "Test Hat")
|
||||||
|
a := e.upload(t, e.hub.uploadToken(""), makeZip(t, file{"tex.png", texture}, file{"index.xml", index}), nil)
|
||||||
|
if a.status != http.StatusCreated {
|
||||||
|
t.Fatalf("upload: %d %s", a.status, a.raw)
|
||||||
|
}
|
||||||
|
if a.header.Get("Access-Control-Allow-Origin") != webOrigin {
|
||||||
|
t.Errorf("no CORS header on the upload's answer")
|
||||||
|
}
|
||||||
|
product, _ := a.body["product"].(string)
|
||||||
|
if !hub.ValidProduct(product) || a.body["revision"] != 1.0 {
|
||||||
|
t.Fatalf("answer %s", a.raw)
|
||||||
|
}
|
||||||
|
|
||||||
|
// What the hub was sent.
|
||||||
|
got := e.hub.last
|
||||||
|
wantFiles := []store.File{
|
||||||
|
{Name: "index.xml", SHA256: sum(index), Size: int64(len(index))},
|
||||||
|
{Name: "tex.png", SHA256: sum(texture), Size: int64(len(texture))},
|
||||||
|
}
|
||||||
|
if !slices.Equal(got.Files, wantFiles) || got.Rating != "general" || got.Parent != 12345 || got.Name != "Test Hat" || got.Notes != "" {
|
||||||
|
t.Fatalf("hub got %+v", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Files come back, cacheable, with ranges and ETags.
|
||||||
|
f := e.get(t, "/api/v1/files/"+sum(texture))
|
||||||
|
if f.status != http.StatusOK || string(f.raw) != texture {
|
||||||
|
t.Fatalf("file: %d %q", f.status, f.raw)
|
||||||
|
}
|
||||||
|
if cc := f.header.Get("Cache-Control"); cc != "public, max-age=31536000, immutable" {
|
||||||
|
t.Errorf("Cache-Control %q", cc)
|
||||||
|
}
|
||||||
|
if et := f.header.Get("ETag"); et != `"`+sum(texture)+`"` {
|
||||||
|
t.Errorf("ETag %q", et)
|
||||||
|
}
|
||||||
|
if r := e.get(t, "/api/v1/files/"+sum(texture), "Range", "bytes=0-3"); r.status != http.StatusPartialContent || string(r.raw) != texture[:4] {
|
||||||
|
t.Errorf("range: %d %q", r.status, r.raw)
|
||||||
|
}
|
||||||
|
if r := e.get(t, "/api/v1/files/"+sum(texture), "If-None-Match", `"`+sum(texture)+`"`); r.status != http.StatusNotModified {
|
||||||
|
t.Errorf("If-None-Match: %d", r.status)
|
||||||
|
}
|
||||||
|
if r := e.get(t, "/api/v1/files/"+sum(index)); r.status != http.StatusOK || string(r.raw) != index {
|
||||||
|
t.Errorf("index.xml: %d", r.status)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The revision.
|
||||||
|
rev := e.get(t, fmt.Sprintf("/api/v1/revisions/%s/1", product))
|
||||||
|
var r store.Revision
|
||||||
|
if err := json.Unmarshal(rev.raw, &r); rev.status != http.StatusOK || err != nil {
|
||||||
|
t.Fatalf("revision: %d %s", rev.status, rev.raw)
|
||||||
|
}
|
||||||
|
if r.Product != product || r.Revision != 1 || r.Rating != "general" || !slices.Equal(r.Files, wantFiles) {
|
||||||
|
t.Fatalf("revision %+v", r)
|
||||||
|
}
|
||||||
|
if rev := e.get(t, fmt.Sprintf("/api/v1/revisions/%s/2", product)); rev.status != http.StatusNotFound {
|
||||||
|
t.Errorf("missing revision: %d", rev.status)
|
||||||
|
}
|
||||||
|
if rev := e.get(t, "/api/v1/revisions/NOT-VALID/1"); rev.status != http.StatusNotFound {
|
||||||
|
t.Errorf("bad product: %d", rev.status)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Not found.
|
||||||
|
if r := e.get(t, "/api/v1/files/"+sum("nothing")); r.status != http.StatusNotFound {
|
||||||
|
t.Errorf("unknown file: %d", r.status)
|
||||||
|
}
|
||||||
|
if r := e.get(t, "/api/v1/files/abc"); r.status != http.StatusNotFound {
|
||||||
|
t.Errorf("bad hash: %d", r.status)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A new revision needs notes (checked before storing anything or calling the hub).
|
||||||
|
newIndex := indexXML("GA", "Test Hat")
|
||||||
|
tex2 := texture + " v2"
|
||||||
|
zip2 := makeZip(t, file{"index.xml", newIndex}, file{"tex.png", tex2})
|
||||||
|
calls := e.hub.calls
|
||||||
|
if a := e.upload(t, e.hub.uploadToken(product), zip2, nil); a.status != http.StatusBadRequest || a.body["error"] != "notes_required" {
|
||||||
|
t.Fatalf("no notes: %d %s", a.status, a.raw)
|
||||||
|
}
|
||||||
|
if e.hub.calls != calls {
|
||||||
|
t.Fatal("hub called without notes")
|
||||||
|
}
|
||||||
|
a = e.upload(t, e.hub.uploadToken(product), zip2, ptr("A brighter texture."))
|
||||||
|
if a.status != http.StatusCreated || a.body["product"] != product || a.body["revision"] != 2.0 {
|
||||||
|
t.Fatalf("revision 2: %d %s", a.status, a.raw)
|
||||||
|
}
|
||||||
|
if e.hub.last.Notes != "A brighter texture." {
|
||||||
|
t.Errorf("notes %q", e.hub.last.Notes)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The hub refusing (the same files again) is a 502 with its error.
|
||||||
|
a = e.upload(t, e.hub.uploadToken(product), zip2, ptr("Again."))
|
||||||
|
if a.status != http.StatusBadGateway || a.body["error"] != "unchanged" {
|
||||||
|
t.Fatalf("unchanged: %d %s", a.status, a.raw)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The hub unreachable: 502, and the stored files aren't served (no hosted revision has them).
|
||||||
|
e.hub.mu.Lock()
|
||||||
|
e.hub.down = true
|
||||||
|
e.hub.mu.Unlock()
|
||||||
|
tex3 := texture + " v3"
|
||||||
|
a = e.upload(t, e.hub.uploadToken(""), makeZip(t, file{"index.xml", index}, file{"tex.png", tex3}), nil)
|
||||||
|
if a.status != http.StatusBadGateway {
|
||||||
|
t.Fatalf("hub down: %d %s", a.status, a.raw)
|
||||||
|
}
|
||||||
|
if r := e.get(t, "/api/v1/files/"+sum(tex3)); r.status != http.StatusNotFound {
|
||||||
|
t.Errorf("file of a refused upload: %d", r.status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAdultFiles(t *testing.T) {
|
||||||
|
e := setup(t)
|
||||||
|
index := indexXML("AP", "Adult Thing")
|
||||||
|
adultTex := texture + " adult"
|
||||||
|
a := e.upload(t, e.hub.uploadToken(""), makeZip(t, file{"index.xml", index}, file{"a.png", adultTex}, file{"shared.png", texture}), nil)
|
||||||
|
if a.status != http.StatusCreated {
|
||||||
|
t.Fatalf("upload: %d %s", a.status, a.raw)
|
||||||
|
}
|
||||||
|
if e.hub.last.Rating != "adult" {
|
||||||
|
t.Fatalf("rating %q", e.hub.last.Rating)
|
||||||
|
}
|
||||||
|
path := "/api/v1/files/" + sum(adultTex)
|
||||||
|
|
||||||
|
r := e.get(t, path)
|
||||||
|
if r.status != http.StatusUnauthorized || !strings.HasPrefix(r.header.Get("Cache-Control"), "private") {
|
||||||
|
t.Fatalf("no token: %d %q", r.status, r.header.Get("Cache-Control"))
|
||||||
|
}
|
||||||
|
r = e.get(t, path, "Authorization", "Bearer garbage")
|
||||||
|
if r.status != http.StatusUnauthorized {
|
||||||
|
t.Fatalf("bad token: %d", r.status)
|
||||||
|
}
|
||||||
|
r = e.get(t, path, "Authorization", "Bearer "+e.hub.uploadToken(""))
|
||||||
|
if r.status != http.StatusUnauthorized {
|
||||||
|
t.Fatalf("upload token as access token: %d", r.status)
|
||||||
|
}
|
||||||
|
r = e.get(t, path, "Authorization", "Bearer "+e.hub.token(token.Access, map[string]any{"adult": false}))
|
||||||
|
if r.status != http.StatusForbidden || !strings.HasPrefix(r.header.Get("Cache-Control"), "private") {
|
||||||
|
t.Fatalf("not adult: %d %q", r.status, r.header.Get("Cache-Control"))
|
||||||
|
}
|
||||||
|
r = e.get(t, path, "Authorization", "Bearer "+e.hub.token(token.Access, map[string]any{"adult": true}))
|
||||||
|
if r.status != http.StatusOK || string(r.raw) != adultTex {
|
||||||
|
t.Fatalf("adult: %d %q", r.status, r.raw)
|
||||||
|
}
|
||||||
|
if cc := r.header.Get("Cache-Control"); !strings.HasPrefix(cc, "private") {
|
||||||
|
t.Errorf("adult file's Cache-Control %q", cc)
|
||||||
|
}
|
||||||
|
|
||||||
|
// shared.png also belongs to a general revision once one uses it: then it's public.
|
||||||
|
if r := e.get(t, "/api/v1/files/"+sum(texture)); r.status != http.StatusUnauthorized {
|
||||||
|
t.Fatalf("shared file before a general revision: %d", r.status)
|
||||||
|
}
|
||||||
|
a = e.upload(t, e.hub.uploadToken(""), makeZip(t, file{"index.xml", indexXML("GA", "Plain")}, file{"shared.png", texture}), nil)
|
||||||
|
if a.status != http.StatusCreated {
|
||||||
|
t.Fatalf("general upload: %d %s", a.status, a.raw)
|
||||||
|
}
|
||||||
|
if r := e.get(t, "/api/v1/files/"+sum(texture)); r.status != http.StatusOK {
|
||||||
|
t.Fatalf("shared file after a general revision: %d", r.status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBadUploads(t *testing.T) {
|
||||||
|
e := setup(t)
|
||||||
|
good := makeZip(t, file{"index.xml", indexXML("GA", "Hat")}, file{"tex.png", texture})
|
||||||
|
|
||||||
|
tokens := map[string]string{
|
||||||
|
"no token": "",
|
||||||
|
"garbage": "garbage",
|
||||||
|
"other catalog": e.hub.token(token.Upload, map[string]any{"catalog": "cat2"}),
|
||||||
|
"access token": e.hub.token(token.Access, map[string]any{"adult": true}),
|
||||||
|
"expired": e.hub.token(token.Upload, map[string]any{"catalog": catalogID, "exp": time.Now().Add(-time.Hour).Unix()}),
|
||||||
|
"wrong issuer": e.hub.token(token.Upload, map[string]any{"catalog": catalogID, "iss": "https://other.example"}),
|
||||||
|
"other key": tokentest.NewKey("hubkey").Sign(map[string]any{"iss": e.hub.URL, "typ": token.Upload, "catalog": catalogID, "exp": time.Now().Add(time.Hour).Unix()}),
|
||||||
|
}
|
||||||
|
for name, tok := range tokens {
|
||||||
|
if a := e.upload(t, tok, good, nil); a.status != http.StatusUnauthorized || a.body["error"] != "bad_token" {
|
||||||
|
t.Errorf("%s: %d %s", name, a.status, a.raw)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
big := strings.Repeat("x", 100)
|
||||||
|
zips := map[string][]byte{
|
||||||
|
"not a zip": []byte("this is not a zip file"),
|
||||||
|
"no index.xml": makeZip(t, file{"tex.png", texture}),
|
||||||
|
"bad rating": makeZip(t, file{"index.xml", indexXML("XX", "Hat")}),
|
||||||
|
"no rating": makeZip(t, file{"index.xml", "<Template><ProductName>Hat</ProductName></Template>"}),
|
||||||
|
"not a template": makeZip(t, file{"index.xml", "<Other/>"}),
|
||||||
|
"file in a folder": makeZip(t, file{"index.xml", indexXML("GA", "Hat")}, file{"sub/tex.png", texture}),
|
||||||
|
"_contents.json": makeZip(t, file{"index.xml", indexXML("GA", "Hat")}, file{"_contents.json", "{}"}),
|
||||||
|
"duplicate name": makeZip(t, file{"index.xml", indexXML("GA", "Hat")}, file{"a.png", big}, file{"a.png", big}),
|
||||||
|
"non-Latin-1 name": makeZip(t, file{"index.xml", indexXML("GA", "Hat")}, file{"一.png", big}),
|
||||||
|
"bad __DATAIMPORT": makeZip(t, file{"index.xml", strings.Replace(indexXML("GA", "Hat"), "product://12345/index.xml", "http://x", 1)}),
|
||||||
|
"index.xml nested": makeZip(t, file{"x/index.xml", indexXML("GA", "Hat")}),
|
||||||
|
"backslash in name": makeZip(t, file{"index.xml", indexXML("GA", "Hat")}, file{`a\b.png`, big}),
|
||||||
|
}
|
||||||
|
for name, z := range zips {
|
||||||
|
a := e.upload(t, e.hub.uploadToken(""), z, nil)
|
||||||
|
t.Logf("%s: %s", name, a.body["message"])
|
||||||
|
if a.status != http.StatusBadRequest || a.body["error"] != "bad_files" {
|
||||||
|
t.Errorf("%s: %d %s", name, a.status, a.raw)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if a := e.upload(t, e.hub.uploadToken(""), nil, ptr("notes but no file")); a.status != http.StatusBadRequest {
|
||||||
|
t.Errorf("no file: %d %s", a.status, a.raw)
|
||||||
|
}
|
||||||
|
req, _ := http.NewRequest(http.MethodPost, e.catalog.URL+"/api/v1/uploads", strings.NewReader("{}"))
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
req.Header.Set("Authorization", "Bearer "+e.hub.uploadToken(""))
|
||||||
|
if a := do(t, req); a.status != http.StatusBadRequest {
|
||||||
|
t.Errorf("not multipart: %d %s", a.status, a.raw)
|
||||||
|
}
|
||||||
|
if e.hub.calls != 0 {
|
||||||
|
t.Errorf("hub called %d times for bad uploads", e.hub.calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A zip whose files are larger than its headers say is cut off by the pack rules' counted reads.
|
||||||
|
func TestUploadSizeCountedWhileReading(t *testing.T) {
|
||||||
|
e := setup(t)
|
||||||
|
// A file over the 64 MiB limit, compressed down to a small upload.
|
||||||
|
var b bytes.Buffer
|
||||||
|
z := zip.NewWriter(&b)
|
||||||
|
w, _ := z.Create("index.xml")
|
||||||
|
w.Write([]byte(indexXML("GA", "Hat")))
|
||||||
|
w, _ = z.Create("huge.png")
|
||||||
|
chunk := make([]byte, 1<<20)
|
||||||
|
for range 65 {
|
||||||
|
w.Write(chunk)
|
||||||
|
}
|
||||||
|
z.Close()
|
||||||
|
a := e.upload(t, e.hub.uploadToken(""), b.Bytes(), nil)
|
||||||
|
if a.status != http.StatusBadRequest || !strings.Contains(a.body["message"].(string), "64 MiB") {
|
||||||
|
t.Fatalf("huge file: %d %s", a.status, a.raw)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCORSAndHealth(t *testing.T) {
|
||||||
|
e := setup(t)
|
||||||
|
preflight := func(origin string) http.Header {
|
||||||
|
req, _ := http.NewRequest(http.MethodOptions, e.catalog.URL+"/api/v1/uploads", nil)
|
||||||
|
req.Header.Set("Origin", origin)
|
||||||
|
req.Header.Set("Access-Control-Request-Method", "POST")
|
||||||
|
req.Header.Set("Access-Control-Request-Headers", "authorization")
|
||||||
|
a := do(t, req)
|
||||||
|
if a.status != http.StatusNoContent {
|
||||||
|
t.Fatalf("preflight: %d", a.status)
|
||||||
|
}
|
||||||
|
return a.header
|
||||||
|
}
|
||||||
|
h := preflight(webOrigin)
|
||||||
|
if h.Get("Access-Control-Allow-Origin") != webOrigin || !strings.Contains(h.Get("Access-Control-Allow-Headers"), "Authorization") ||
|
||||||
|
h.Get("Access-Control-Allow-Methods") != "POST" {
|
||||||
|
t.Errorf("preflight headers: %v", h)
|
||||||
|
}
|
||||||
|
if h := preflight("https://evil.example"); h.Get("Access-Control-Allow-Origin") != "" {
|
||||||
|
t.Errorf("other origin allowed: %v", h)
|
||||||
|
}
|
||||||
|
|
||||||
|
a := e.get(t, "/health")
|
||||||
|
if a.status != http.StatusOK || a.body["ok"] != true {
|
||||||
|
t.Errorf("health: %d %s", a.status, a.raw)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,299 @@
|
|||||||
|
package server
|
||||||
|
|
||||||
|
import (
|
||||||
|
"archive/zip"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"mime"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"unicode/utf8"
|
||||||
|
|
||||||
|
"git.t5de.dev/t5de/t5de-api/pack"
|
||||||
|
|
||||||
|
"git.t5de.dev/t5de/t5de-catalog/internal/hub"
|
||||||
|
"git.t5de.dev/t5de/t5de-catalog/internal/store"
|
||||||
|
"git.t5de.dev/t5de/t5de-catalog/internal/token"
|
||||||
|
)
|
||||||
|
|
||||||
|
// MaxBody caps an upload request: the zip (whose files may hold pack.MaxTotalSize uncompressed) and the notes.
|
||||||
|
const MaxBody = 300 << 20
|
||||||
|
|
||||||
|
// MaxNotes caps the revision notes.
|
||||||
|
const MaxNotes = 64 << 10
|
||||||
|
|
||||||
|
// corsHeaders adds CORS headers for the hub website's origin.
|
||||||
|
func (s *Server) corsHeaders(w http.ResponseWriter, r *http.Request) bool {
|
||||||
|
h := w.Header()
|
||||||
|
h.Add("Vary", "Origin")
|
||||||
|
origin := r.Header.Get("Origin")
|
||||||
|
if origin == "" || s.cfg.WebOrigin == "" || origin != s.cfg.WebOrigin {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
h.Set("Access-Control-Allow-Origin", origin)
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) preflight(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if s.corsHeaders(w, r) {
|
||||||
|
h := w.Header()
|
||||||
|
h.Add("Vary", "Access-Control-Request-Method")
|
||||||
|
h.Add("Vary", "Access-Control-Request-Headers")
|
||||||
|
h.Set("Access-Control-Allow-Methods", "POST")
|
||||||
|
h.Set("Access-Control-Allow-Headers", "Authorization, Content-Type")
|
||||||
|
h.Set("Access-Control-Max-Age", "600")
|
||||||
|
}
|
||||||
|
w.WriteHeader(http.StatusNoContent)
|
||||||
|
}
|
||||||
|
|
||||||
|
// upload takes a product (docs/products.md, POST /api/v1/uploads): checks the token and the files, stores the
|
||||||
|
// files, registers the revision with the hub, and records it as hosted once the hub has accepted it.
|
||||||
|
func (s *Server) upload(w http.ResponseWriter, r *http.Request) {
|
||||||
|
s.corsHeaders(w, r)
|
||||||
|
tok := bearer(r)
|
||||||
|
if tok == "" {
|
||||||
|
writeError(w, http.StatusUnauthorized, "bad_token", "An upload token is needed.")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
claims, err := s.cfg.Verifier.Verify(r.Context(), tok, token.Upload)
|
||||||
|
if err != nil {
|
||||||
|
s.cfg.Logger.Printf("uploads: %v", err)
|
||||||
|
writeError(w, http.StatusUnauthorized, "bad_token", "The upload token isn't valid. Start the upload again.")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if claims.Catalog != s.cfg.CatalogID {
|
||||||
|
writeError(w, http.StatusUnauthorized, "bad_token", "The upload token is for another catalog.")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if claims.Product != "" && !hub.ValidProduct(claims.Product) {
|
||||||
|
writeError(w, http.StatusUnauthorized, "bad_token", "The upload token names no valid product.")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
select {
|
||||||
|
case s.uploads <- struct{}{}:
|
||||||
|
defer func() { <-s.uploads }()
|
||||||
|
case <-r.Context().Done():
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
tmp, err := os.MkdirTemp(s.cfg.Store.TempDir(), "upload-*")
|
||||||
|
if err != nil {
|
||||||
|
s.cfg.Logger.Printf("uploads: %v", err)
|
||||||
|
writeError(w, http.StatusInternalServerError, "internal", "Something went wrong.")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer os.RemoveAll(tmp)
|
||||||
|
|
||||||
|
r.Body = http.MaxBytesReader(w, r.Body, MaxBody)
|
||||||
|
notes, status, code, msg := readForm(r, filepath.Join(tmp, pack.ChknName))
|
||||||
|
if status != 0 {
|
||||||
|
if status == http.StatusInternalServerError {
|
||||||
|
s.cfg.Logger.Printf("uploads: %s", msg)
|
||||||
|
msg = "Something went wrong."
|
||||||
|
}
|
||||||
|
writeError(w, status, code, msg)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if claims.Product != "" && notes == "" {
|
||||||
|
writeError(w, http.StatusBadRequest, "notes_required", "Say what changed in this revision.")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
p, rating, err := checkProduct(tmp)
|
||||||
|
if err != nil {
|
||||||
|
writeError(w, http.StatusBadRequest, "bad_files", "These product files can't be used: "+err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
files := make([]store.File, len(p.Files))
|
||||||
|
for i, f := range p.Files {
|
||||||
|
hash, err := s.cfg.Store.PutFile(f.Data)
|
||||||
|
if err != nil {
|
||||||
|
s.cfg.Logger.Printf("uploads: storing %q: %v", f.Name, err)
|
||||||
|
writeError(w, http.StatusInternalServerError, "internal", "Something went wrong.")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
files[i] = store.File{Name: f.Name, SHA256: hash, Size: int64(len(f.Data))}
|
||||||
|
}
|
||||||
|
index := p.Index
|
||||||
|
p = nil // the files' data can go before the hub answers
|
||||||
|
|
||||||
|
reg, err := s.cfg.Hub.RegisterRevision(r.Context(), hub.NewRevision{
|
||||||
|
UploadToken: tok, Files: files, Rating: rating, Parent: index.Parent, Name: strings.TrimSpace(index.Name), Notes: notes,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
var refused *hub.Error
|
||||||
|
if errors.As(err, &refused) {
|
||||||
|
s.cfg.Logger.Printf("uploads: %v", err)
|
||||||
|
code, msg := refused.Code, refused.Message
|
||||||
|
if code == "" {
|
||||||
|
code = "hub_refused"
|
||||||
|
}
|
||||||
|
if msg == "" {
|
||||||
|
msg = fmt.Sprintf("T5DE's hub refused the upload (%d).", refused.Status)
|
||||||
|
}
|
||||||
|
writeError(w, http.StatusBadGateway, code, msg)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
s.cfg.Logger.Printf("uploads: registering with the hub: %v", err)
|
||||||
|
writeError(w, http.StatusBadGateway, "hub_unavailable", "T5DE's hub couldn't be reached. Try again later.")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if claims.Product != "" && reg.Product != claims.Product {
|
||||||
|
s.cfg.Logger.Printf("uploads: the hub registered product %s, but the token was for %s", reg.Product, claims.Product)
|
||||||
|
writeError(w, http.StatusBadGateway, "hub_failed", "T5DE's hub answered for another product.")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := s.cfg.Store.AddRevision(store.Revision{Product: reg.Product, Revision: reg.Revision, Rating: rating, Files: files}); err != nil {
|
||||||
|
s.cfg.Logger.Printf("uploads: recording %s/%d: %v", reg.Product, reg.Revision, err)
|
||||||
|
writeError(w, http.StatusInternalServerError, "internal", "Something went wrong.")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
s.cfg.Logger.Printf("uploads: %s/%d (%s, %d files) from %s", reg.Product, reg.Revision, rating, len(files), claims.Subject)
|
||||||
|
writeJSON(w, http.StatusCreated, reg)
|
||||||
|
}
|
||||||
|
|
||||||
|
// readForm reads the upload's multipart form: the zip (part "file") into chkn, and the notes (part "notes").
|
||||||
|
// Other parts are skipped. A non-zero status is an error to answer.
|
||||||
|
func readForm(r *http.Request, chkn string) (notes string, status int, code, msg string) {
|
||||||
|
mt, _, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
|
||||||
|
if err != nil || mt != "multipart/form-data" {
|
||||||
|
return "", http.StatusBadRequest, "bad_request", "Send the product as multipart/form-data."
|
||||||
|
}
|
||||||
|
mr, err := r.MultipartReader()
|
||||||
|
if err != nil {
|
||||||
|
return "", http.StatusBadRequest, "bad_request", "Send the product as multipart/form-data."
|
||||||
|
}
|
||||||
|
tooLarge := func(err error) bool {
|
||||||
|
var mbe *http.MaxBytesError
|
||||||
|
return errors.As(err, &mbe)
|
||||||
|
}
|
||||||
|
haveFile, haveNotes := false, false
|
||||||
|
for {
|
||||||
|
part, err := mr.NextPart()
|
||||||
|
if err == io.EOF {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
if tooLarge(err) {
|
||||||
|
return "", http.StatusRequestEntityTooLarge, "too_large", fmt.Sprintf("Uploads can be at most %d MiB.", MaxBody>>20)
|
||||||
|
}
|
||||||
|
return "", http.StatusBadRequest, "bad_request", "The upload is cut short or malformed."
|
||||||
|
}
|
||||||
|
switch part.FormName() {
|
||||||
|
case "file":
|
||||||
|
if haveFile {
|
||||||
|
return "", http.StatusBadRequest, "bad_request", "Send one file."
|
||||||
|
}
|
||||||
|
haveFile = true
|
||||||
|
f, err := os.Create(chkn)
|
||||||
|
if err != nil {
|
||||||
|
return "", http.StatusInternalServerError, "internal", err.Error()
|
||||||
|
}
|
||||||
|
_, err = io.Copy(f, part)
|
||||||
|
if cerr := f.Close(); err == nil && cerr != nil {
|
||||||
|
return "", http.StatusInternalServerError, "internal", cerr.Error()
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
if tooLarge(err) {
|
||||||
|
return "", http.StatusRequestEntityTooLarge, "too_large", fmt.Sprintf("Uploads can be at most %d MiB.", MaxBody>>20)
|
||||||
|
}
|
||||||
|
return "", http.StatusBadRequest, "bad_request", "The upload is cut short or malformed."
|
||||||
|
}
|
||||||
|
case "notes":
|
||||||
|
if haveNotes {
|
||||||
|
return "", http.StatusBadRequest, "bad_request", "Send the notes once."
|
||||||
|
}
|
||||||
|
haveNotes = true
|
||||||
|
data, err := io.ReadAll(io.LimitReader(part, MaxNotes+1))
|
||||||
|
if err != nil {
|
||||||
|
if tooLarge(err) {
|
||||||
|
return "", http.StatusRequestEntityTooLarge, "too_large", fmt.Sprintf("Uploads can be at most %d MiB.", MaxBody>>20)
|
||||||
|
}
|
||||||
|
return "", http.StatusBadRequest, "bad_request", "The upload is cut short or malformed."
|
||||||
|
}
|
||||||
|
if len(data) > MaxNotes {
|
||||||
|
return "", http.StatusBadRequest, "bad_request", fmt.Sprintf("The notes can be at most %d KiB.", MaxNotes>>10)
|
||||||
|
}
|
||||||
|
if !utf8.Valid(data) {
|
||||||
|
return "", http.StatusBadRequest, "bad_request", "The notes aren't valid UTF-8."
|
||||||
|
}
|
||||||
|
notes = strings.TrimSpace(string(data))
|
||||||
|
default:
|
||||||
|
if _, err := io.Copy(io.Discard, part); err != nil {
|
||||||
|
if tooLarge(err) {
|
||||||
|
return "", http.StatusRequestEntityTooLarge, "too_large", fmt.Sprintf("Uploads can be at most %d MiB.", MaxBody>>20)
|
||||||
|
}
|
||||||
|
return "", http.StatusBadRequest, "bad_request", "The upload is cut short or malformed."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
part.Close()
|
||||||
|
}
|
||||||
|
if !haveFile {
|
||||||
|
return "", http.StatusBadRequest, "bad_request", "No file: send the product's .chkn as \"file\"."
|
||||||
|
}
|
||||||
|
return notes, 0, "", ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// ratings maps index.xml's ProductRating to the hub's ratings.
|
||||||
|
var ratings = map[string]string{"GA": store.General, "AP": store.Adult}
|
||||||
|
|
||||||
|
// checkProduct checks the zip dir/product.chkn by the pack rules (t5de-api's pack package): it reads the rating
|
||||||
|
// from its index.xml, writes a manifest with that rating beside it, and loads dir as a pack. It returns the pack
|
||||||
|
// and its rating.
|
||||||
|
func checkProduct(dir string) (*pack.Pack, string, error) {
|
||||||
|
index, err := readIndex(filepath.Join(dir, pack.ChknName))
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
rating := ratings[index.Rating]
|
||||||
|
manifest, err := json.Marshal(pack.Manifest{Format: pack.Format, ID: "upload", Revision: 1, Name: "upload", Rating: rating})
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, pack.ManifestName), manifest, 0o644); err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
p, err := pack.Load(dir)
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
return p, rating, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// readIndex reads index.xml from the zip at path, its size counted while reading.
|
||||||
|
func readIndex(path string) (*pack.Index, error) {
|
||||||
|
z, err := zip.OpenReader(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.New("not a zip file")
|
||||||
|
}
|
||||||
|
defer z.Close()
|
||||||
|
for _, f := range z.File {
|
||||||
|
if f.Name != pack.IndexName {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
rc, err := f.Open()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%s: %v", pack.IndexName, err)
|
||||||
|
}
|
||||||
|
data, err := io.ReadAll(io.LimitReader(rc, pack.MaxFileSize+1))
|
||||||
|
rc.Close()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%s: %v", pack.IndexName, err)
|
||||||
|
}
|
||||||
|
if len(data) > pack.MaxFileSize {
|
||||||
|
return nil, fmt.Errorf("%s: over %d MiB", pack.IndexName, pack.MaxFileSize>>20)
|
||||||
|
}
|
||||||
|
index, err := pack.ParseIndex(data)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%s: %v", pack.IndexName, err)
|
||||||
|
}
|
||||||
|
return index, nil
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("no %s at the top of the zip", pack.IndexName)
|
||||||
|
}
|
||||||
@@ -0,0 +1,217 @@
|
|||||||
|
// Package store keeps the catalog's files and the revisions it hosts: files on disk named by their sha256
|
||||||
|
// (files/<first 2 hex>/<sha256>), revisions in SQLite (catalog.db).
|
||||||
|
package store
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/sha256"
|
||||||
|
"database/sql"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
|
|
||||||
|
_ "modernc.org/sqlite"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Ratings.
|
||||||
|
const (
|
||||||
|
General = "general"
|
||||||
|
Adult = "adult"
|
||||||
|
)
|
||||||
|
|
||||||
|
// File is one entry of a revision's file list.
|
||||||
|
type File struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
SHA256 string `json:"sha256"`
|
||||||
|
Size int64 `json:"size"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Revision is a revision the catalog hosts.
|
||||||
|
type Revision struct {
|
||||||
|
Product string `json:"product"`
|
||||||
|
Revision int `json:"revision"`
|
||||||
|
Rating string `json:"rating"`
|
||||||
|
Files []File `json:"files"` // sorted by name
|
||||||
|
}
|
||||||
|
|
||||||
|
// ErrNotFound is returned for files and revisions the catalog doesn't host.
|
||||||
|
var ErrNotFound = errors.New("not found")
|
||||||
|
|
||||||
|
var hashPattern = regexp.MustCompile(`^[0-9a-f]{64}$`)
|
||||||
|
|
||||||
|
// ValidHash reports whether s is a lowercase hex sha256.
|
||||||
|
func ValidHash(s string) bool { return hashPattern.MatchString(s) }
|
||||||
|
|
||||||
|
// Store is the catalog's data folder.
|
||||||
|
type Store struct {
|
||||||
|
dir string
|
||||||
|
db *sql.DB
|
||||||
|
}
|
||||||
|
|
||||||
|
const schema = `
|
||||||
|
CREATE TABLE IF NOT EXISTS revisions (
|
||||||
|
product TEXT NOT NULL,
|
||||||
|
revision INTEGER NOT NULL,
|
||||||
|
rating TEXT NOT NULL CHECK (rating IN ('general', 'adult')),
|
||||||
|
files TEXT NOT NULL, -- the file list, JSON
|
||||||
|
created_at INTEGER NOT NULL DEFAULT (unixepoch()),
|
||||||
|
PRIMARY KEY (product, revision)
|
||||||
|
);
|
||||||
|
CREATE TABLE IF NOT EXISTS revision_files (
|
||||||
|
product TEXT NOT NULL,
|
||||||
|
revision INTEGER NOT NULL,
|
||||||
|
sha256 TEXT NOT NULL,
|
||||||
|
PRIMARY KEY (product, revision, sha256),
|
||||||
|
FOREIGN KEY (product, revision) REFERENCES revisions (product, revision) ON DELETE CASCADE
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS revision_files_sha256 ON revision_files (sha256);
|
||||||
|
`
|
||||||
|
|
||||||
|
// Open opens (or creates) the store in dir.
|
||||||
|
func Open(dir string) (*Store, error) {
|
||||||
|
if err := os.MkdirAll(filepath.Join(dir, "files"), 0o755); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
// Leftovers of uploads interrupted by a crash.
|
||||||
|
if err := os.RemoveAll(filepath.Join(dir, "tmp")); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(filepath.Join(dir, "tmp"), 0o755); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
dsn := "file:" + filepath.ToSlash(filepath.Join(dir, "catalog.db")) +
|
||||||
|
"?_pragma=busy_timeout(5000)&_pragma=journal_mode(WAL)&_pragma=foreign_keys(1)&_pragma=synchronous(NORMAL)"
|
||||||
|
db, err := sql.Open("sqlite", dsn)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
db.SetMaxOpenConns(1)
|
||||||
|
if _, err := db.Exec(schema); err != nil {
|
||||||
|
db.Close()
|
||||||
|
return nil, fmt.Errorf("catalog.db: %w", err)
|
||||||
|
}
|
||||||
|
return &Store{dir: dir, db: db}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Close closes the database.
|
||||||
|
func (s *Store) Close() error { return s.db.Close() }
|
||||||
|
|
||||||
|
// TempDir is a folder on the store's file system for temporary files (emptied when the store opens).
|
||||||
|
func (s *Store) TempDir() string { return filepath.Join(s.dir, "tmp") }
|
||||||
|
|
||||||
|
// Path returns where the file with this hash is (or would be) stored.
|
||||||
|
func (s *Store) Path(hash string) string {
|
||||||
|
return filepath.Join(s.dir, "files", hash[:2], hash)
|
||||||
|
}
|
||||||
|
|
||||||
|
// PutFile stores data under its sha256 and returns the hash. A file already stored isn't written again.
|
||||||
|
func (s *Store) PutFile(data []byte) (string, error) {
|
||||||
|
sum := sha256.Sum256(data)
|
||||||
|
hash := hex.EncodeToString(sum[:])
|
||||||
|
path := s.Path(hash)
|
||||||
|
if fi, err := os.Stat(path); err == nil && fi.Size() == int64(len(data)) {
|
||||||
|
return hash, nil
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
f, err := os.CreateTemp(filepath.Dir(path), ".tmp-*")
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
tmp := f.Name()
|
||||||
|
_, err = f.Write(data)
|
||||||
|
if err == nil {
|
||||||
|
err = f.Sync()
|
||||||
|
}
|
||||||
|
if cerr := f.Close(); err == nil {
|
||||||
|
err = cerr
|
||||||
|
}
|
||||||
|
if err == nil {
|
||||||
|
err = os.Rename(tmp, path)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
os.Remove(tmp)
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return hash, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Open opens a stored file.
|
||||||
|
func (s *Store) Open(hash string) (*os.File, error) {
|
||||||
|
if !ValidHash(hash) {
|
||||||
|
return nil, ErrNotFound
|
||||||
|
}
|
||||||
|
f, err := os.Open(s.Path(hash))
|
||||||
|
if errors.Is(err, fs.ErrNotExist) {
|
||||||
|
return nil, ErrNotFound
|
||||||
|
}
|
||||||
|
return f, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// AddRevision records a revision as hosted. Its files must be stored already. Recording the same revision again
|
||||||
|
// replaces it.
|
||||||
|
func (s *Store) AddRevision(r Revision) error {
|
||||||
|
if r.Rating != General && r.Rating != Adult {
|
||||||
|
return fmt.Errorf("rating %q", r.Rating)
|
||||||
|
}
|
||||||
|
files, err := json.Marshal(r.Files)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
tx, err := s.db.Begin()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer tx.Rollback()
|
||||||
|
if _, err := tx.Exec(`DELETE FROM revisions WHERE product = ? AND revision = ?`, r.Product, r.Revision); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := tx.Exec(`INSERT INTO revisions (product, revision, rating, files) VALUES (?, ?, ?, ?)`,
|
||||||
|
r.Product, r.Revision, r.Rating, string(files)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, f := range r.Files {
|
||||||
|
if _, err := tx.Exec(`INSERT OR IGNORE INTO revision_files (product, revision, sha256) VALUES (?, ?, ?)`,
|
||||||
|
r.Product, r.Revision, f.SHA256); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return tx.Commit()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Revision returns a hosted revision.
|
||||||
|
func (s *Store) Revision(product string, revision int) (*Revision, error) {
|
||||||
|
r := Revision{Product: product, Revision: revision}
|
||||||
|
var files string
|
||||||
|
err := s.db.QueryRow(`SELECT rating, files FROM revisions WHERE product = ? AND revision = ?`, product, revision).
|
||||||
|
Scan(&r.Rating, &files)
|
||||||
|
if errors.Is(err, sql.ErrNoRows) {
|
||||||
|
return nil, ErrNotFound
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal([]byte(files), &r.Files); err != nil {
|
||||||
|
return nil, fmt.Errorf("revision %s/%d: %w", product, revision, err)
|
||||||
|
}
|
||||||
|
return &r, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// FileAccess tells whether a file belongs to a hosted revision (hosted) and whether every such revision is adult
|
||||||
|
// (adultOnly). Files in no hosted revision (an upload the hub refused) aren't served.
|
||||||
|
func (s *Store) FileAccess(hash string) (hosted, adultOnly bool, err error) {
|
||||||
|
var n, general int
|
||||||
|
err = s.db.QueryRow(`
|
||||||
|
SELECT count(*), coalesce(sum(r.rating = 'general'), 0)
|
||||||
|
FROM revision_files rf JOIN revisions r ON r.product = rf.product AND r.revision = rf.revision
|
||||||
|
WHERE rf.sha256 = ?`, hash).Scan(&n, &general)
|
||||||
|
if err != nil {
|
||||||
|
return false, false, err
|
||||||
|
}
|
||||||
|
return n > 0, n > 0 && general == 0, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,147 @@
|
|||||||
|
package store
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func open(t *testing.T, dir string) *Store {
|
||||||
|
t.Helper()
|
||||||
|
s, err := Open(dir)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { s.Close() })
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
func hashOf(data string) string {
|
||||||
|
sum := sha256.Sum256([]byte(data))
|
||||||
|
return hex.EncodeToString(sum[:])
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFiles(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
s := open(t, dir)
|
||||||
|
|
||||||
|
hash, err := s.PutFile([]byte("hello"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if hash != hashOf("hello") {
|
||||||
|
t.Fatalf("hash %s", hash)
|
||||||
|
}
|
||||||
|
want := filepath.Join(dir, "files", hash[:2], hash)
|
||||||
|
if s.Path(hash) != want {
|
||||||
|
t.Fatalf("path %s, want %s", s.Path(hash), want)
|
||||||
|
}
|
||||||
|
again, err := s.PutFile([]byte("hello"))
|
||||||
|
if err != nil || again != hash {
|
||||||
|
t.Fatalf("again: %s, %v", again, err)
|
||||||
|
}
|
||||||
|
f, err := s.Open(hash)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
data, _ := io.ReadAll(f)
|
||||||
|
f.Close()
|
||||||
|
if string(data) != "hello" {
|
||||||
|
t.Fatalf("data %q", data)
|
||||||
|
}
|
||||||
|
// No temp files left beside it.
|
||||||
|
entries, _ := os.ReadDir(filepath.Dir(want))
|
||||||
|
if len(entries) != 1 {
|
||||||
|
t.Fatalf("%d entries in %s", len(entries), filepath.Dir(want))
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := s.Open(hashOf("missing")); !errors.Is(err, ErrNotFound) {
|
||||||
|
t.Fatalf("missing: %v", err)
|
||||||
|
}
|
||||||
|
for _, bad := range []string{"", "../catalog.db", hash[:63], "A" + hash[1:]} {
|
||||||
|
if _, err := s.Open(bad); !errors.Is(err, ErrNotFound) {
|
||||||
|
t.Fatalf("%q: %v", bad, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRevisions(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
s := open(t, dir)
|
||||||
|
shared, general, adult := hashOf("shared"), hashOf("general"), hashOf("adult")
|
||||||
|
|
||||||
|
if hosted, _, err := s.FileAccess(shared); err != nil || hosted {
|
||||||
|
t.Fatalf("before: hosted %v, %v", hosted, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
a := Revision{Product: "aaaaaaaaaa", Revision: 1, Rating: Adult, Files: []File{
|
||||||
|
{Name: "a.png", SHA256: adult, Size: 5}, {Name: "index.xml", SHA256: shared, Size: 6},
|
||||||
|
}}
|
||||||
|
if err := s.AddRevision(a); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
check := func(hash string, wantHosted, wantAdult bool) {
|
||||||
|
t.Helper()
|
||||||
|
hosted, adultOnly, err := s.FileAccess(hash)
|
||||||
|
if err != nil || hosted != wantHosted || adultOnly != wantAdult {
|
||||||
|
t.Fatalf("%s: hosted %v adult %v (%v), want %v %v", hash[:8], hosted, adultOnly, err, wantHosted, wantAdult)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
check(shared, true, true)
|
||||||
|
check(adult, true, true)
|
||||||
|
check(general, false, false)
|
||||||
|
|
||||||
|
g := Revision{Product: "bbbbbbbbbb", Revision: 2, Rating: General, Files: []File{
|
||||||
|
{Name: "g.png", SHA256: general, Size: 7}, {Name: "index.xml", SHA256: shared, Size: 6},
|
||||||
|
}}
|
||||||
|
if err := s.AddRevision(g); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
check(shared, true, false) // also in a general revision
|
||||||
|
check(adult, true, true)
|
||||||
|
check(general, true, false)
|
||||||
|
|
||||||
|
got, err := s.Revision("aaaaaaaaaa", 1)
|
||||||
|
if err != nil || !reflect.DeepEqual(*got, a) {
|
||||||
|
t.Fatalf("got %+v, %v", got, err)
|
||||||
|
}
|
||||||
|
if _, err := s.Revision("aaaaaaaaaa", 2); !errors.Is(err, ErrNotFound) {
|
||||||
|
t.Fatalf("missing revision: %v", err)
|
||||||
|
}
|
||||||
|
if err := s.AddRevision(Revision{Product: "c", Revision: 1, Rating: "GA"}); err == nil {
|
||||||
|
t.Fatal("bad rating accepted")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Recording a revision again replaces it, and the data survives reopening.
|
||||||
|
a.Rating = General
|
||||||
|
if err := s.AddRevision(a); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
s.Close()
|
||||||
|
s = open(t, dir)
|
||||||
|
got, err = s.Revision("aaaaaaaaaa", 1)
|
||||||
|
if err != nil || got.Rating != General {
|
||||||
|
t.Fatalf("after reopening: %+v, %v", got, err)
|
||||||
|
}
|
||||||
|
if hosted, adultOnly, _ := s.FileAccess(adult); !hosted || adultOnly {
|
||||||
|
t.Fatal("adult file still adult-only after the revision became general")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTempCleared(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
s := open(t, dir)
|
||||||
|
if err := os.WriteFile(filepath.Join(s.TempDir(), "leftover"), []byte("x"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
s.Close()
|
||||||
|
s = open(t, dir)
|
||||||
|
if entries, _ := os.ReadDir(s.TempDir()); len(entries) != 0 {
|
||||||
|
t.Fatalf("temp folder not emptied: %v", entries)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,257 @@
|
|||||||
|
// Package token checks the hub's tokens: EdDSA (Ed25519) JWTs in compact form, signed with a key from the hub's
|
||||||
|
// JWKS (GET <hub>/.well-known/t5de-keys.json). See t5de-hub's docs/products.md, "Tokens".
|
||||||
|
package token
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/ed25519"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Token types.
|
||||||
|
const (
|
||||||
|
Upload = "t5de-upload"
|
||||||
|
Access = "t5de-access"
|
||||||
|
Revision = "t5de-revision"
|
||||||
|
)
|
||||||
|
|
||||||
|
// KeysPath is where the hub publishes its keys.
|
||||||
|
const KeysPath = "/.well-known/t5de-keys.json"
|
||||||
|
|
||||||
|
// Leeway allows for clocks that disagree a little.
|
||||||
|
const Leeway = time.Minute
|
||||||
|
|
||||||
|
// RefetchInterval is the least time between fetches of the hub's keys, however many unknown key IDs show up.
|
||||||
|
const RefetchInterval = 30 * time.Second
|
||||||
|
|
||||||
|
const maxToken = 16 << 10
|
||||||
|
const maxKeys = 1 << 20
|
||||||
|
|
||||||
|
// ErrInvalid wraps every reason a token is refused.
|
||||||
|
var ErrInvalid = errors.New("invalid token")
|
||||||
|
|
||||||
|
// Claims are the claims T5DE's tokens carry.
|
||||||
|
type Claims struct {
|
||||||
|
Issuer string `json:"iss"`
|
||||||
|
Type string `json:"typ"`
|
||||||
|
Subject string `json:"sub"`
|
||||||
|
IssuedAt *int64 `json:"iat"`
|
||||||
|
Expires *int64 `json:"exp"`
|
||||||
|
NotYet *int64 `json:"nbf"`
|
||||||
|
Catalog string `json:"catalog"` // t5de-upload
|
||||||
|
Product string `json:"product"` // t5de-upload (optional), t5de-revision
|
||||||
|
Adult bool `json:"adult"` // t5de-access
|
||||||
|
Revision int `json:"revision"` // t5de-revision
|
||||||
|
Rating string `json:"rating"` // t5de-revision
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verifier checks tokens signed by one hub.
|
||||||
|
type Verifier struct {
|
||||||
|
issuer string
|
||||||
|
keyURL string
|
||||||
|
client *http.Client
|
||||||
|
now func() time.Time
|
||||||
|
|
||||||
|
fetch sync.Mutex // held while fetching, so one fetch serves everyone waiting
|
||||||
|
mu sync.Mutex
|
||||||
|
keys map[string]ed25519.PublicKey
|
||||||
|
fetched time.Time // when the keys were last fetched (or tried), zero if never
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewVerifier returns a verifier for tokens from the hub at hubURL (its API's address, which is also the tokens'
|
||||||
|
// issuer). client may be nil.
|
||||||
|
func NewVerifier(hubURL string, client *http.Client) *Verifier {
|
||||||
|
hubURL = strings.TrimRight(hubURL, "/")
|
||||||
|
if client == nil {
|
||||||
|
client = &http.Client{Timeout: 10 * time.Second}
|
||||||
|
}
|
||||||
|
return &Verifier{issuer: hubURL, keyURL: hubURL + KeysPath, client: client, now: time.Now}
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetClock replaces the verifier's clock, for tests.
|
||||||
|
func (v *Verifier) SetClock(now func() time.Time) { v.now = now }
|
||||||
|
|
||||||
|
type header struct {
|
||||||
|
Alg string `json:"alg"`
|
||||||
|
Kid string `json:"kid"`
|
||||||
|
Crit any `json:"crit"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify checks a token of type typ: its signature, issuer, type and times. Tokens of other types than
|
||||||
|
// t5de-revision must have an expiry.
|
||||||
|
func (v *Verifier) Verify(ctx context.Context, token, typ string) (*Claims, error) {
|
||||||
|
c, err := v.verify(ctx, token, typ)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: %v", ErrInvalid, err)
|
||||||
|
}
|
||||||
|
return c, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *Verifier) verify(ctx context.Context, token, typ string) (*Claims, error) {
|
||||||
|
if len(token) > maxToken {
|
||||||
|
return nil, errors.New("too long")
|
||||||
|
}
|
||||||
|
parts := strings.Split(token, ".")
|
||||||
|
if len(parts) != 3 {
|
||||||
|
return nil, errors.New("not a compact JWT")
|
||||||
|
}
|
||||||
|
var h header
|
||||||
|
if err := decodePart(parts[0], &h); err != nil {
|
||||||
|
return nil, fmt.Errorf("header: %v", err)
|
||||||
|
}
|
||||||
|
if h.Alg != "EdDSA" {
|
||||||
|
return nil, fmt.Errorf("algorithm %q isn't EdDSA", h.Alg)
|
||||||
|
}
|
||||||
|
if h.Crit != nil {
|
||||||
|
return nil, errors.New("unsupported critical header")
|
||||||
|
}
|
||||||
|
sig, err := base64.RawURLEncoding.Strict().DecodeString(parts[2])
|
||||||
|
if err != nil || len(sig) != ed25519.SignatureSize {
|
||||||
|
return nil, errors.New("bad signature encoding")
|
||||||
|
}
|
||||||
|
key, err := v.key(ctx, h.Kid)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if !ed25519.Verify(key, []byte(parts[0]+"."+parts[1]), sig) {
|
||||||
|
return nil, errors.New("bad signature")
|
||||||
|
}
|
||||||
|
|
||||||
|
var c Claims
|
||||||
|
if err := decodePart(parts[1], &c); err != nil {
|
||||||
|
return nil, fmt.Errorf("claims: %v", err)
|
||||||
|
}
|
||||||
|
if c.Issuer != v.issuer {
|
||||||
|
return nil, fmt.Errorf("issuer %q isn't the hub", c.Issuer)
|
||||||
|
}
|
||||||
|
if c.Type != typ {
|
||||||
|
return nil, fmt.Errorf("type %q isn't %q", c.Type, typ)
|
||||||
|
}
|
||||||
|
now := v.now()
|
||||||
|
if c.Expires == nil {
|
||||||
|
if typ != Revision {
|
||||||
|
return nil, errors.New("no expiry")
|
||||||
|
}
|
||||||
|
} else if !now.Before(time.Unix(*c.Expires, 0).Add(Leeway)) {
|
||||||
|
return nil, errors.New("expired")
|
||||||
|
}
|
||||||
|
if c.NotYet != nil && now.Add(Leeway).Before(time.Unix(*c.NotYet, 0)) {
|
||||||
|
return nil, errors.New("not valid yet")
|
||||||
|
}
|
||||||
|
if c.IssuedAt != nil && now.Add(Leeway).Before(time.Unix(*c.IssuedAt, 0)) {
|
||||||
|
return nil, errors.New("issued in the future")
|
||||||
|
}
|
||||||
|
return &c, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func decodePart(s string, v any) error {
|
||||||
|
data, err := base64.RawURLEncoding.Strict().DecodeString(s)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return json.Unmarshal(data, v)
|
||||||
|
}
|
||||||
|
|
||||||
|
// key returns the key with ID kid, fetching the hub's keys if it isn't known and they weren't fetched lately.
|
||||||
|
func (v *Verifier) key(ctx context.Context, kid string) (ed25519.PublicKey, error) {
|
||||||
|
if k, ok := v.cached(kid); ok {
|
||||||
|
return k, nil
|
||||||
|
}
|
||||||
|
v.fetch.Lock()
|
||||||
|
defer v.fetch.Unlock()
|
||||||
|
if k, ok := v.cached(kid); ok { // fetched while waiting
|
||||||
|
return k, nil
|
||||||
|
}
|
||||||
|
v.mu.Lock()
|
||||||
|
recent := !v.fetched.IsZero() && v.now().Sub(v.fetched) < RefetchInterval
|
||||||
|
v.mu.Unlock()
|
||||||
|
if recent {
|
||||||
|
return nil, fmt.Errorf("unknown key %q", kid)
|
||||||
|
}
|
||||||
|
keys, err := v.fetchKeys(ctx)
|
||||||
|
v.mu.Lock()
|
||||||
|
v.fetched = v.now()
|
||||||
|
if err == nil {
|
||||||
|
v.keys = keys
|
||||||
|
}
|
||||||
|
v.mu.Unlock()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("fetching the hub's keys: %v", err)
|
||||||
|
}
|
||||||
|
if k, ok := keys[kid]; ok {
|
||||||
|
return k, nil
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("unknown key %q", kid)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *Verifier) cached(kid string) (ed25519.PublicKey, bool) {
|
||||||
|
v.mu.Lock()
|
||||||
|
defer v.mu.Unlock()
|
||||||
|
k, ok := v.keys[kid]
|
||||||
|
return k, ok
|
||||||
|
}
|
||||||
|
|
||||||
|
type jwk struct {
|
||||||
|
Kty string `json:"kty"`
|
||||||
|
Crv string `json:"crv"`
|
||||||
|
X string `json:"x"`
|
||||||
|
Kid string `json:"kid"`
|
||||||
|
Alg string `json:"alg"`
|
||||||
|
Use string `json:"use"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *Verifier) fetchKeys(ctx context.Context) (map[string]ed25519.PublicKey, error) {
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, v.keyURL, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
resp, err := v.client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return nil, fmt.Errorf("%s: %s", v.keyURL, resp.Status)
|
||||||
|
}
|
||||||
|
data, err := io.ReadAll(io.LimitReader(resp.Body, maxKeys+1))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if len(data) > maxKeys {
|
||||||
|
return nil, errors.New("key set too large")
|
||||||
|
}
|
||||||
|
return ParseKeys(data)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParseKeys reads the Ed25519 signing keys from a JWKS. Other keys are skipped.
|
||||||
|
func ParseKeys(data []byte) (map[string]ed25519.PublicKey, error) {
|
||||||
|
var set struct {
|
||||||
|
Keys []jwk `json:"keys"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(data, &set); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
keys := map[string]ed25519.PublicKey{}
|
||||||
|
for _, k := range set.Keys {
|
||||||
|
if k.Kty != "OKP" || k.Crv != "Ed25519" || k.Kid == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if (k.Alg != "" && k.Alg != "EdDSA") || (k.Use != "" && k.Use != "sig") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
x, err := base64.RawURLEncoding.Strict().DecodeString(k.X)
|
||||||
|
if err != nil || len(x) != ed25519.PublicKeySize {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
keys[k.Kid] = ed25519.PublicKey(x)
|
||||||
|
}
|
||||||
|
return keys, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,182 @@
|
|||||||
|
package token_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.t5de.dev/t5de/t5de-catalog/internal/token"
|
||||||
|
"git.t5de.dev/t5de/t5de-catalog/internal/token/tokentest"
|
||||||
|
)
|
||||||
|
|
||||||
|
type hub struct {
|
||||||
|
*httptest.Server
|
||||||
|
mu sync.Mutex
|
||||||
|
keys []*tokentest.Key
|
||||||
|
fetches atomic.Int32
|
||||||
|
}
|
||||||
|
|
||||||
|
func newHub(t *testing.T, keys ...*tokentest.Key) *hub {
|
||||||
|
h := &hub{keys: keys}
|
||||||
|
h.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.URL.Path != token.KeysPath {
|
||||||
|
http.NotFound(w, r)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
h.fetches.Add(1)
|
||||||
|
h.mu.Lock()
|
||||||
|
defer h.mu.Unlock()
|
||||||
|
w.Write(tokentest.JWKS(h.keys...))
|
||||||
|
}))
|
||||||
|
t.Cleanup(h.Close)
|
||||||
|
return h
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *hub) setKeys(keys ...*tokentest.Key) {
|
||||||
|
h.mu.Lock()
|
||||||
|
h.keys = keys
|
||||||
|
h.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
var now = time.Unix(1_800_000_000, 0)
|
||||||
|
|
||||||
|
func claims(h *hub, typ string, extra map[string]any) map[string]any {
|
||||||
|
c := map[string]any{"iss": h.URL, "typ": typ, "sub": "person1", "iat": now.Unix(), "exp": now.Add(30 * time.Minute).Unix()}
|
||||||
|
for k, v := range extra {
|
||||||
|
if v == nil {
|
||||||
|
delete(c, k)
|
||||||
|
} else {
|
||||||
|
c[k] = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
func verifier(h *hub, url string) *token.Verifier {
|
||||||
|
v := token.NewVerifier(url, h.Client())
|
||||||
|
v.SetClock(func() time.Time { return now })
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestVerify(t *testing.T) {
|
||||||
|
key := tokentest.NewKey("k1")
|
||||||
|
other := tokentest.NewKey("k1") // same ID, different key
|
||||||
|
h := newHub(t, key)
|
||||||
|
v := verifier(h, h.URL+"/") // a trailing slash in HUB_URL is fine
|
||||||
|
|
||||||
|
c, err := v.Verify(context.Background(), key.Sign(claims(h, token.Upload, map[string]any{"catalog": "cat1", "product": "abcdefghij"})), token.Upload)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if c.Subject != "person1" || c.Catalog != "cat1" || c.Product != "abcdefghij" {
|
||||||
|
t.Fatalf("claims: %+v", c)
|
||||||
|
}
|
||||||
|
c, err = v.Verify(context.Background(), key.Sign(claims(h, token.Access, map[string]any{"adult": true})), token.Access)
|
||||||
|
if err != nil || !c.Adult {
|
||||||
|
t.Fatalf("access: %+v, %v", c, err)
|
||||||
|
}
|
||||||
|
// Revision tokens don't expire.
|
||||||
|
if _, err := v.Verify(context.Background(), key.Sign(claims(h, token.Revision, map[string]any{"exp": nil})), token.Revision); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
bad := map[string]string{
|
||||||
|
"wrong type": key.Sign(claims(h, token.Access, nil)),
|
||||||
|
"wrong issuer": key.Sign(claims(h, token.Upload, map[string]any{"iss": "https://evil.example"})),
|
||||||
|
"expired": key.Sign(claims(h, token.Upload, map[string]any{"exp": now.Add(-2 * time.Minute).Unix()})),
|
||||||
|
"no expiry": key.Sign(claims(h, token.Upload, map[string]any{"exp": nil})),
|
||||||
|
"not yet": key.Sign(claims(h, token.Upload, map[string]any{"nbf": now.Add(time.Hour).Unix()})),
|
||||||
|
"future iat": key.Sign(claims(h, token.Upload, map[string]any{"iat": now.Add(time.Hour).Unix()})),
|
||||||
|
"other key": other.Sign(claims(h, token.Upload, nil)),
|
||||||
|
"alg none": key.SignWithHeader(map[string]any{"alg": "none", "kid": "k1"}, claims(h, token.Upload, nil)),
|
||||||
|
"alg HS256": key.SignWithHeader(map[string]any{"alg": "HS256", "kid": "k1"}, claims(h, token.Upload, nil)),
|
||||||
|
"crit": key.SignWithHeader(map[string]any{"alg": "EdDSA", "kid": "k1", "crit": []string{"x"}}, claims(h, token.Upload, nil)),
|
||||||
|
"two parts": "a.b",
|
||||||
|
"garbage": "not a token at all",
|
||||||
|
"empty signature": strings.Join(strings.Split(key.Sign(claims(h, token.Upload, nil)), ".")[:2], ".") + ".",
|
||||||
|
}
|
||||||
|
for name, tok := range bad {
|
||||||
|
if _, err := v.Verify(context.Background(), tok, token.Upload); !errors.Is(err, token.ErrInvalid) {
|
||||||
|
t.Errorf("%s: got %v, want ErrInvalid", name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A tampered payload fails the signature.
|
||||||
|
parts := strings.Split(key.Sign(claims(h, token.Access, map[string]any{"adult": false})), ".")
|
||||||
|
forged := key.Sign(claims(h, token.Access, map[string]any{"adult": true}))
|
||||||
|
tampered := parts[0] + "." + strings.Split(forged, ".")[1] + "." + parts[2]
|
||||||
|
if _, err := v.Verify(context.Background(), tampered, token.Access); !errors.Is(err, token.ErrInvalid) {
|
||||||
|
t.Errorf("tampered: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExpiryLeeway(t *testing.T) {
|
||||||
|
key := tokentest.NewKey("k1")
|
||||||
|
h := newHub(t, key)
|
||||||
|
v := verifier(h, h.URL)
|
||||||
|
tok := key.Sign(claims(h, token.Upload, map[string]any{"exp": now.Add(-30 * time.Second).Unix()}))
|
||||||
|
if _, err := v.Verify(context.Background(), tok, token.Upload); err != nil {
|
||||||
|
t.Fatalf("within leeway: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestKeyRotation(t *testing.T) {
|
||||||
|
k1, k2 := tokentest.NewKey("k1"), tokentest.NewKey("k2")
|
||||||
|
h := newHub(t, k1)
|
||||||
|
clock := now
|
||||||
|
v := token.NewVerifier(h.URL, h.Client())
|
||||||
|
v.SetClock(func() time.Time { return clock })
|
||||||
|
|
||||||
|
for range 3 {
|
||||||
|
if _, err := v.Verify(context.Background(), k1.Sign(claims(h, token.Upload, nil)), token.Upload); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if n := h.fetches.Load(); n != 1 {
|
||||||
|
t.Fatalf("keys fetched %d times, want 1 (cached)", n)
|
||||||
|
}
|
||||||
|
|
||||||
|
// An unknown kid right after a fetch isn't fetched again (rate limit).
|
||||||
|
h.setKeys(k1, k2)
|
||||||
|
for range 5 {
|
||||||
|
if _, err := v.Verify(context.Background(), k2.Sign(claims(h, token.Upload, nil)), token.Upload); err == nil {
|
||||||
|
t.Fatal("unknown key accepted before refetch")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if n := h.fetches.Load(); n != 1 {
|
||||||
|
t.Fatalf("keys fetched %d times, want 1 (rate-limited)", n)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Later, an unknown kid fetches the keys again.
|
||||||
|
clock = clock.Add(token.RefetchInterval)
|
||||||
|
if _, err := v.Verify(context.Background(), k2.Sign(claims(h, token.Upload, nil)), token.Upload); err != nil {
|
||||||
|
t.Fatalf("rotated key: %v", err)
|
||||||
|
}
|
||||||
|
if n := h.fetches.Load(); n != 2 {
|
||||||
|
t.Fatalf("keys fetched %d times, want 2", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseKeys(t *testing.T) {
|
||||||
|
k := tokentest.NewKey("good")
|
||||||
|
data := []byte(`{"keys":[
|
||||||
|
{"kty":"RSA","kid":"rsa","n":"x","e":"AQAB"},
|
||||||
|
{"kty":"OKP","crv":"X25519","kid":"x","x":"` + strings.Repeat("A", 43) + `"},
|
||||||
|
{"kty":"OKP","crv":"Ed25519","kid":"short","x":"AAAA"},
|
||||||
|
{"kty":"OKP","crv":"Ed25519","kid":"enc","use":"enc","x":"` + strings.Repeat("A", 43) + `"}
|
||||||
|
]}`)
|
||||||
|
keys, err := token.ParseKeys(data)
|
||||||
|
if err != nil || len(keys) != 0 {
|
||||||
|
t.Fatalf("got %v, %v; want no keys", keys, err)
|
||||||
|
}
|
||||||
|
keys, err = token.ParseKeys(tokentest.JWKS(k))
|
||||||
|
if err != nil || !keys["good"].Equal(k.Public) {
|
||||||
|
t.Fatalf("got %v, %v", keys, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,60 @@
|
|||||||
|
// Package tokentest signs tokens the way the hub does, for tests.
|
||||||
|
package tokentest
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/ed25519"
|
||||||
|
"crypto/rand"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/json"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Key is a test signing key.
|
||||||
|
type Key struct {
|
||||||
|
ID string
|
||||||
|
Private ed25519.PrivateKey
|
||||||
|
Public ed25519.PublicKey
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewKey makes a key with ID kid.
|
||||||
|
func NewKey(kid string) *Key {
|
||||||
|
pub, priv, err := ed25519.GenerateKey(rand.Reader)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
return &Key{ID: kid, Private: priv, Public: pub}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sign returns a compact EdDSA JWT with these claims, with the header the hub sends.
|
||||||
|
func (k *Key) Sign(claims map[string]any) string {
|
||||||
|
return k.SignWithHeader(map[string]any{"alg": "EdDSA", "kid": k.ID, "typ": "JWT"}, claims)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SignWithHeader signs claims with any header.
|
||||||
|
func (k *Key) SignWithHeader(header, claims map[string]any) string {
|
||||||
|
input := part(header) + "." + part(claims)
|
||||||
|
return input + "." + base64.RawURLEncoding.EncodeToString(ed25519.Sign(k.Private, []byte(input)))
|
||||||
|
}
|
||||||
|
|
||||||
|
func part(v any) string {
|
||||||
|
data, err := json.Marshal(v)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
return base64.RawURLEncoding.EncodeToString(data)
|
||||||
|
}
|
||||||
|
|
||||||
|
// JWKS returns a key set holding these keys, as the hub serves it.
|
||||||
|
func JWKS(keys ...*Key) []byte {
|
||||||
|
var set struct {
|
||||||
|
Keys []map[string]string `json:"keys"`
|
||||||
|
}
|
||||||
|
set.Keys = []map[string]string{}
|
||||||
|
for _, k := range keys {
|
||||||
|
set.Keys = append(set.Keys, map[string]string{
|
||||||
|
"kty": "OKP", "crv": "Ed25519", "x": base64.RawURLEncoding.EncodeToString(k.Public),
|
||||||
|
"kid": k.ID, "alg": "EdDSA", "use": "sig",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
data, _ := json.Marshal(set)
|
||||||
|
return data
|
||||||
|
}
|
||||||
Reference in new issue
Block a user