The proxy keeps the person's library in a cache folder (hub-products in its data folder), one pack per product, served like any other pack: the hub says which revision to load and which catalogs host it, each file comes from a catalog and is checked against the hub's sha256 (files are kept by hash, so a new revision only downloads what changed), and adult files are fetched with the hub's access token. Products someone wears in a room that aren't here yet are fetched too, and put on them when they arrive. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
dhkatzandClaude Opus 5.5 committed
1 parent
93fa2f5d90
commit
c281fb572c
6 files changed
+579
-3
No files matched your search
@@ -0,0 +1,293 @@
|
||||
// Package fetch gets T5 products from the hub's catalogs (t5de-hub's docs/products.md): the hub says which revision
|
||||
// to load and which catalogs host it; the files come from a catalog, each checked against the hub's sha256, and the
|
||||
// product is written as a pack in the cache folder, where the products package serves it like any other pack.
|
||||
package fetch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"git.t5de.dev/t5de/t5de-proxy/internal/hub"
|
||||
)
|
||||
|
||||
// Hub is what the fetcher asks the hub.
|
||||
type Hub interface {
|
||||
ProductFiles(ctx context.Context, id string) (*hub.Files, error)
|
||||
AccessToken(ctx context.Context) (string, error)
|
||||
Library(ctx context.Context) ([]hub.LibraryEntry, error)
|
||||
}
|
||||
|
||||
// IsHubID reports whether id is a hub product ID (10 of a-z2-7), as opposed to a local pack's.
|
||||
var IsHubID = regexp.MustCompile(`^[a-z2-7]{10}$`).MatchString
|
||||
|
||||
// Fetcher keeps hub products in a cache folder, one pack folder per product.
|
||||
type Fetcher struct {
|
||||
hub Hub
|
||||
dir string
|
||||
logf func(format string, args ...any)
|
||||
http *http.Client
|
||||
|
||||
// OnReady, if set, is called when a product was written (or updated) in the cache.
|
||||
OnReady func(id string)
|
||||
|
||||
mu sync.Mutex
|
||||
inflight map[string]bool
|
||||
failed map[string]time.Time // products that failed lately (not retried for a while)
|
||||
access string
|
||||
accessAt time.Time
|
||||
}
|
||||
|
||||
// New keeps products in dir.
|
||||
func New(h Hub, dir string, logf func(format string, args ...any)) *Fetcher {
|
||||
if logf == nil {
|
||||
logf = func(string, ...any) {}
|
||||
}
|
||||
return &Fetcher{hub: h, dir: dir, logf: logf, http: &http.Client{Timeout: 2 * time.Minute},
|
||||
inflight: map[string]bool{}, failed: map[string]time.Time{}}
|
||||
}
|
||||
|
||||
// Dir is the cache folder (a pack root).
|
||||
func (f *Fetcher) Dir() string { return f.dir }
|
||||
|
||||
// SyncLibrary makes sure every product in the person's library is in the cache, at the revision they use.
|
||||
func (f *Fetcher) SyncLibrary(ctx context.Context) {
|
||||
lib, err := f.hub.Library(ctx)
|
||||
if err != nil {
|
||||
f.logf("fetch: reading the library: %v", err)
|
||||
return
|
||||
}
|
||||
for _, e := range lib {
|
||||
if err := f.Ensure(ctx, e.Product.ID); err != nil {
|
||||
f.logf("fetch: %s (%s): %v", e.Product.ID, e.Product.Name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Want gets a product in the background, once at a time, and not again soon after a failure.
|
||||
func (f *Fetcher) Want(id string) {
|
||||
if !IsHubID(id) {
|
||||
return
|
||||
}
|
||||
f.mu.Lock()
|
||||
if f.inflight[id] || time.Since(f.failed[id]) < 5*time.Minute {
|
||||
f.mu.Unlock()
|
||||
return
|
||||
}
|
||||
f.inflight[id] = true
|
||||
f.mu.Unlock()
|
||||
go func() {
|
||||
err := f.Ensure(context.Background(), id)
|
||||
f.mu.Lock()
|
||||
delete(f.inflight, id)
|
||||
if err != nil {
|
||||
f.failed[id] = time.Now()
|
||||
}
|
||||
f.mu.Unlock()
|
||||
if err != nil {
|
||||
f.logf("fetch: %s: %v", id, err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// manifest is the pack manifest written for a product (t5de-api's pack format 1).
|
||||
type manifest struct {
|
||||
Format int `json:"format"`
|
||||
ID string `json:"id"`
|
||||
Revision int `json:"revision"`
|
||||
Name string `json:"name"`
|
||||
Creator string `json:"creator,omitempty"`
|
||||
Rating string `json:"rating"`
|
||||
}
|
||||
|
||||
// Ensure puts the product's current revision in the cache, unless it's there already.
|
||||
func (f *Fetcher) Ensure(ctx context.Context, id string) error {
|
||||
if !IsHubID(id) {
|
||||
return fmt.Errorf("not a hub product ID: %q", id)
|
||||
}
|
||||
info, err := f.hub.ProductFiles(ctx, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
final := filepath.Join(f.dir, id)
|
||||
if b, err := os.ReadFile(filepath.Join(final, "t5de-product.json")); err == nil {
|
||||
var m manifest
|
||||
if json.Unmarshal(b, &m) == nil && m.Revision == info.Revision {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
if len(info.Catalogs) == 0 {
|
||||
return errors.New("no catalog hosts it")
|
||||
}
|
||||
for _, fe := range info.Files {
|
||||
if fe.Name == "" || fe.Name == "." || fe.Name == ".." || filepath.Base(fe.Name) != fe.Name || strings.ContainsAny(fe.Name, `/\:`) {
|
||||
return fmt.Errorf("bad file name %q", fe.Name)
|
||||
}
|
||||
}
|
||||
token := ""
|
||||
if info.Rating == "adult" {
|
||||
if token, err = f.accessToken(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, fe := range info.Files {
|
||||
if err := f.blob(ctx, info.Catalogs, fe, token); err != nil {
|
||||
return fmt.Errorf("%s: %w", fe.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The pack, in a temporary folder, then in place.
|
||||
tmp, err := os.MkdirTemp(f.dir, "."+id+"-")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.RemoveAll(tmp)
|
||||
if err := os.Mkdir(filepath.Join(tmp, "files"), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, fe := range info.Files {
|
||||
if err := copyFile(f.blobPath(fe.SHA256), filepath.Join(tmp, "files", fe.Name)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
m := manifest{Format: 1, ID: id, Revision: info.Revision, Name: clip(info.Name, "Untitled"), Creator: clip(info.Creator, ""), Rating: info.Rating}
|
||||
b, _ := json.MarshalIndent(m, "", " ")
|
||||
if err := os.WriteFile(filepath.Join(tmp, "t5de-product.json"), b, 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
old := final + ".old"
|
||||
os.RemoveAll(old)
|
||||
if _, err := os.Stat(final); err == nil {
|
||||
if err := os.Rename(final, old); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := os.Rename(tmp, final); err != nil {
|
||||
os.Rename(old, final)
|
||||
return err
|
||||
}
|
||||
os.RemoveAll(old)
|
||||
f.logf("fetch: %s revision %d (%s) is ready", id, info.Revision, m.Name)
|
||||
if f.OnReady != nil {
|
||||
f.OnReady(id)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// clip makes a name fit the pack format (1-64 characters, no control characters).
|
||||
func clip(s, empty string) string {
|
||||
s = strings.Map(func(r rune) rune {
|
||||
if r < 0x20 || r == 0x7f {
|
||||
return -1
|
||||
}
|
||||
return r
|
||||
}, strings.TrimSpace(s))
|
||||
for utf8.RuneCountInString(s) > 64 {
|
||||
_, n := utf8.DecodeLastRuneInString(s)
|
||||
s = s[:len(s)-n]
|
||||
}
|
||||
if s == "" {
|
||||
return empty
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func (f *Fetcher) blobPath(sum string) string { return filepath.Join(f.dir, ".blobs", sum) }
|
||||
|
||||
// blob gets a file into the blob cache (by sha256), from the first catalog that has it.
|
||||
func (f *Fetcher) blob(ctx context.Context, catalogs []string, fe hub.FileEntry, token string) error {
|
||||
path := f.blobPath(fe.SHA256)
|
||||
if st, err := os.Stat(path); err == nil && st.Size() == fe.Size {
|
||||
return nil
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
var last error
|
||||
for _, c := range catalogs {
|
||||
if last = f.download(ctx, strings.TrimSuffix(c, "/")+"/api/v1/files/"+fe.SHA256, fe, token, path); last == nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return last
|
||||
}
|
||||
|
||||
func (f *Fetcher) download(ctx context.Context, url string, fe hub.FileEntry, token, path string) error {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("User-Agent", hub.UserAgent)
|
||||
if token != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
}
|
||||
resp, err := f.http.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("%s answered %d", url, resp.StatusCode)
|
||||
}
|
||||
tmp, err := os.CreateTemp(filepath.Dir(path), ".dl-")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.Remove(tmp.Name())
|
||||
h := sha256.New()
|
||||
n, err := io.Copy(io.MultiWriter(tmp, h), io.LimitReader(resp.Body, fe.Size+1))
|
||||
tmp.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n != fe.Size || hex.EncodeToString(h.Sum(nil)) != fe.SHA256 {
|
||||
return fmt.Errorf("%s sent a different file than the hub listed", url)
|
||||
}
|
||||
return os.Rename(tmp.Name(), path)
|
||||
}
|
||||
|
||||
func (f *Fetcher) accessToken(ctx context.Context) (string, error) {
|
||||
f.mu.Lock()
|
||||
tok, at := f.access, f.accessAt
|
||||
f.mu.Unlock()
|
||||
if tok != "" && time.Since(at) < 50*time.Minute {
|
||||
return tok, nil
|
||||
}
|
||||
tok, err := f.hub.AccessToken(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
f.mu.Lock()
|
||||
f.access, f.accessAt = tok, time.Now()
|
||||
f.mu.Unlock()
|
||||
return tok, nil
|
||||
}
|
||||
|
||||
func copyFile(src, dst string) error {
|
||||
in, err := os.Open(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer in.Close()
|
||||
out, err := os.Create(dst)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.Copy(out, in); err != nil {
|
||||
out.Close()
|
||||
return err
|
||||
}
|
||||
return out.Close()
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
package fetch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"git.t5de.dev/t5de/t5de-api/pack"
|
||||
"git.t5de.dev/t5de/t5de-proxy/internal/hub"
|
||||
)
|
||||
|
||||
const id = "abcdefgh23"
|
||||
|
||||
func indexXML(rating string) []byte {
|
||||
return []byte(`<?xml version="1.0" encoding="UTF-8" ?><Template><ProductRating>` + rating +
|
||||
`</ProductRating><__DATAIMPORT>product://500/index.xml</__DATAIMPORT><ProductName>x</ProductName></Template>`)
|
||||
}
|
||||
|
||||
func entry(name string, data []byte) hub.FileEntry {
|
||||
sum := sha256.Sum256(data)
|
||||
return hub.FileEntry{Name: name, SHA256: hex.EncodeToString(sum[:]), Size: int64(len(data))}
|
||||
}
|
||||
|
||||
// fakeHub answers for one product.
|
||||
type fakeHub struct {
|
||||
mu sync.Mutex
|
||||
files hub.Files
|
||||
}
|
||||
|
||||
func (h *fakeHub) ProductFiles(ctx context.Context, pid string) (*hub.Files, error) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
f := h.files
|
||||
return &f, nil
|
||||
}
|
||||
func (h *fakeHub) AccessToken(ctx context.Context) (string, error) { return "adult-token", nil }
|
||||
func (h *fakeHub) Library(ctx context.Context) ([]hub.LibraryEntry, error) {
|
||||
var e hub.LibraryEntry
|
||||
e.Product.ID = id
|
||||
return []hub.LibraryEntry{e}, nil
|
||||
}
|
||||
|
||||
// catalog serves blobs by sha256; adult ones need the token.
|
||||
func catalog(t *testing.T, blobs map[string][]byte, adult map[string]bool, served *int) *httptest.Server {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
sum := strings.TrimPrefix(r.URL.Path, "/api/v1/files/")
|
||||
b, ok := blobs[sum]
|
||||
if !ok {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
if adult[sum] && r.Header.Get("Authorization") != "Bearer adult-token" {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
*served++
|
||||
w.Write(b)
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
return srv
|
||||
}
|
||||
|
||||
func TestEnsureWritesAPack(t *testing.T) {
|
||||
index, tex := indexXML("GA"), []byte("texture one")
|
||||
tex2 := []byte("texture two")
|
||||
blobs := map[string][]byte{}
|
||||
for _, b := range [][]byte{index, tex, tex2} {
|
||||
blobs[entry("", b).SHA256] = b
|
||||
}
|
||||
served := 0
|
||||
cat := catalog(t, blobs, nil, &served)
|
||||
h := &fakeHub{files: hub.Files{Product: id, Revision: 1, Rating: "general", Name: "Blue Coat", Creator: "maker",
|
||||
Files: []hub.FileEntry{entry("index.xml", index), entry("tex.png", tex)}, Catalogs: []string{"http://127.0.0.1:1", cat.URL}}}
|
||||
dir := t.TempDir()
|
||||
f := New(h, dir, t.Logf)
|
||||
var ready []string
|
||||
f.OnReady = func(id string) { ready = append(ready, id) }
|
||||
|
||||
f.SyncLibrary(context.Background())
|
||||
p, err := pack.Load(filepath.Join(dir, id))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p.Manifest.Name != "Blue Coat" || p.Manifest.Revision != 1 || len(p.Files) != 2 || len(ready) != 1 {
|
||||
t.Fatalf("pack %+v, ready %v", p.Manifest, ready)
|
||||
}
|
||||
// Already there: nothing fetched again.
|
||||
if err := f.Ensure(context.Background(), id); err != nil || served != 2 || len(ready) != 1 {
|
||||
t.Fatalf("again: %v, served %d", err, served)
|
||||
}
|
||||
|
||||
// A new revision: only the changed file is downloaded; the pack is replaced.
|
||||
h.files.Revision, h.files.Files = 2, []hub.FileEntry{entry("index.xml", index), entry("tex.png", tex2)}
|
||||
if err := f.Ensure(context.Background(), id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if b, _ := os.ReadFile(filepath.Join(dir, id, "files", "tex.png")); string(b) != "texture two" || served != 3 {
|
||||
t.Fatalf("tex %q, served %d", b, served)
|
||||
}
|
||||
entries, _ := os.ReadDir(dir)
|
||||
for _, e := range entries {
|
||||
if e.Name() != id && e.Name() != ".blobs" {
|
||||
t.Fatalf("left behind %s", e.Name())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureChecksFilesAndAdultTokens(t *testing.T) {
|
||||
index := indexXML("AP")
|
||||
blobs := map[string][]byte{entry("", index).SHA256: index}
|
||||
adult := map[string]bool{entry("", index).SHA256: true}
|
||||
served := 0
|
||||
cat := catalog(t, blobs, adult, &served)
|
||||
h := &fakeHub{files: hub.Files{Product: id, Revision: 1, Rating: "adult", Name: "Lace",
|
||||
Files: []hub.FileEntry{entry("index.xml", index)}, Catalogs: []string{cat.URL}}}
|
||||
f := New(h, t.TempDir(), t.Logf)
|
||||
if err := f.Ensure(context.Background(), id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// A catalog sending something else than the hub listed is refused.
|
||||
h.files.Revision = 2
|
||||
h.files.Files = []hub.FileEntry{{Name: "index.xml", SHA256: strings.Repeat("0", 64), Size: int64(len(index))}}
|
||||
blobs[strings.Repeat("0", 64)] = []byte(strings.Repeat("x", len(index)))
|
||||
if err := f.Ensure(context.Background(), id); err == nil || !strings.Contains(err.Error(), "different file") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
// File names that would leave the pack's folder are refused.
|
||||
h.files.Files = []hub.FileEntry{{Name: `..\evil`, SHA256: strings.Repeat("0", 64), Size: 1}}
|
||||
if err := f.Ensure(context.Background(), id); err == nil || !strings.Contains(err.Error(), "bad file name") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
if err := f.Ensure(context.Background(), "../etc"); err == nil {
|
||||
t.Fatal("bad ID accepted")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user