The proxy keeps the person's library in a cache folder (hub-products in its data folder), one pack per product, served like any other pack: the hub says which revision to load and which catalogs host it, each file comes from a catalog and is checked against the hub's sha256 (files are kept by hash, so a new revision only downloads what changed), and adult files are fetched with the hub's access token. Products someone wears in a room that aren't here yet are fetched too, and put on them when they arrive. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
dhkatzandClaude Opus 5.5 committed
1 parent
93fa2f5d90
commit
c281fb572c
6 files changed
+579
-3
No files matched your search
@@ -0,0 +1,293 @@
|
||||
// Package fetch gets T5 products from the hub's catalogs (t5de-hub's docs/products.md): the hub says which revision
|
||||
// to load and which catalogs host it; the files come from a catalog, each checked against the hub's sha256, and the
|
||||
// product is written as a pack in the cache folder, where the products package serves it like any other pack.
|
||||
package fetch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"git.t5de.dev/t5de/t5de-proxy/internal/hub"
|
||||
)
|
||||
|
||||
// Hub is what the fetcher asks the hub.
|
||||
type Hub interface {
|
||||
ProductFiles(ctx context.Context, id string) (*hub.Files, error)
|
||||
AccessToken(ctx context.Context) (string, error)
|
||||
Library(ctx context.Context) ([]hub.LibraryEntry, error)
|
||||
}
|
||||
|
||||
// IsHubID reports whether id is a hub product ID (10 of a-z2-7), as opposed to a local pack's.
|
||||
var IsHubID = regexp.MustCompile(`^[a-z2-7]{10}$`).MatchString
|
||||
|
||||
// Fetcher keeps hub products in a cache folder, one pack folder per product.
|
||||
type Fetcher struct {
|
||||
hub Hub
|
||||
dir string
|
||||
logf func(format string, args ...any)
|
||||
http *http.Client
|
||||
|
||||
// OnReady, if set, is called when a product was written (or updated) in the cache.
|
||||
OnReady func(id string)
|
||||
|
||||
mu sync.Mutex
|
||||
inflight map[string]bool
|
||||
failed map[string]time.Time // products that failed lately (not retried for a while)
|
||||
access string
|
||||
accessAt time.Time
|
||||
}
|
||||
|
||||
// New keeps products in dir.
|
||||
func New(h Hub, dir string, logf func(format string, args ...any)) *Fetcher {
|
||||
if logf == nil {
|
||||
logf = func(string, ...any) {}
|
||||
}
|
||||
return &Fetcher{hub: h, dir: dir, logf: logf, http: &http.Client{Timeout: 2 * time.Minute},
|
||||
inflight: map[string]bool{}, failed: map[string]time.Time{}}
|
||||
}
|
||||
|
||||
// Dir is the cache folder (a pack root).
|
||||
func (f *Fetcher) Dir() string { return f.dir }
|
||||
|
||||
// SyncLibrary makes sure every product in the person's library is in the cache, at the revision they use.
|
||||
func (f *Fetcher) SyncLibrary(ctx context.Context) {
|
||||
lib, err := f.hub.Library(ctx)
|
||||
if err != nil {
|
||||
f.logf("fetch: reading the library: %v", err)
|
||||
return
|
||||
}
|
||||
for _, e := range lib {
|
||||
if err := f.Ensure(ctx, e.Product.ID); err != nil {
|
||||
f.logf("fetch: %s (%s): %v", e.Product.ID, e.Product.Name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Want gets a product in the background, once at a time, and not again soon after a failure.
|
||||
func (f *Fetcher) Want(id string) {
|
||||
if !IsHubID(id) {
|
||||
return
|
||||
}
|
||||
f.mu.Lock()
|
||||
if f.inflight[id] || time.Since(f.failed[id]) < 5*time.Minute {
|
||||
f.mu.Unlock()
|
||||
return
|
||||
}
|
||||
f.inflight[id] = true
|
||||
f.mu.Unlock()
|
||||
go func() {
|
||||
err := f.Ensure(context.Background(), id)
|
||||
f.mu.Lock()
|
||||
delete(f.inflight, id)
|
||||
if err != nil {
|
||||
f.failed[id] = time.Now()
|
||||
}
|
||||
f.mu.Unlock()
|
||||
if err != nil {
|
||||
f.logf("fetch: %s: %v", id, err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// manifest is the pack manifest written for a product (t5de-api's pack format 1).
|
||||
type manifest struct {
|
||||
Format int `json:"format"`
|
||||
ID string `json:"id"`
|
||||
Revision int `json:"revision"`
|
||||
Name string `json:"name"`
|
||||
Creator string `json:"creator,omitempty"`
|
||||
Rating string `json:"rating"`
|
||||
}
|
||||
|
||||
// Ensure puts the product's current revision in the cache, unless it's there already.
|
||||
func (f *Fetcher) Ensure(ctx context.Context, id string) error {
|
||||
if !IsHubID(id) {
|
||||
return fmt.Errorf("not a hub product ID: %q", id)
|
||||
}
|
||||
info, err := f.hub.ProductFiles(ctx, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
final := filepath.Join(f.dir, id)
|
||||
if b, err := os.ReadFile(filepath.Join(final, "t5de-product.json")); err == nil {
|
||||
var m manifest
|
||||
if json.Unmarshal(b, &m) == nil && m.Revision == info.Revision {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
if len(info.Catalogs) == 0 {
|
||||
return errors.New("no catalog hosts it")
|
||||
}
|
||||
for _, fe := range info.Files {
|
||||
if fe.Name == "" || fe.Name == "." || fe.Name == ".." || filepath.Base(fe.Name) != fe.Name || strings.ContainsAny(fe.Name, `/\:`) {
|
||||
return fmt.Errorf("bad file name %q", fe.Name)
|
||||
}
|
||||
}
|
||||
token := ""
|
||||
if info.Rating == "adult" {
|
||||
if token, err = f.accessToken(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, fe := range info.Files {
|
||||
if err := f.blob(ctx, info.Catalogs, fe, token); err != nil {
|
||||
return fmt.Errorf("%s: %w", fe.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The pack, in a temporary folder, then in place.
|
||||
tmp, err := os.MkdirTemp(f.dir, "."+id+"-")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.RemoveAll(tmp)
|
||||
if err := os.Mkdir(filepath.Join(tmp, "files"), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, fe := range info.Files {
|
||||
if err := copyFile(f.blobPath(fe.SHA256), filepath.Join(tmp, "files", fe.Name)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
m := manifest{Format: 1, ID: id, Revision: info.Revision, Name: clip(info.Name, "Untitled"), Creator: clip(info.Creator, ""), Rating: info.Rating}
|
||||
b, _ := json.MarshalIndent(m, "", " ")
|
||||
if err := os.WriteFile(filepath.Join(tmp, "t5de-product.json"), b, 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
old := final + ".old"
|
||||
os.RemoveAll(old)
|
||||
if _, err := os.Stat(final); err == nil {
|
||||
if err := os.Rename(final, old); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := os.Rename(tmp, final); err != nil {
|
||||
os.Rename(old, final)
|
||||
return err
|
||||
}
|
||||
os.RemoveAll(old)
|
||||
f.logf("fetch: %s revision %d (%s) is ready", id, info.Revision, m.Name)
|
||||
if f.OnReady != nil {
|
||||
f.OnReady(id)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// clip makes a name fit the pack format (1-64 characters, no control characters).
|
||||
func clip(s, empty string) string {
|
||||
s = strings.Map(func(r rune) rune {
|
||||
if r < 0x20 || r == 0x7f {
|
||||
return -1
|
||||
}
|
||||
return r
|
||||
}, strings.TrimSpace(s))
|
||||
for utf8.RuneCountInString(s) > 64 {
|
||||
_, n := utf8.DecodeLastRuneInString(s)
|
||||
s = s[:len(s)-n]
|
||||
}
|
||||
if s == "" {
|
||||
return empty
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func (f *Fetcher) blobPath(sum string) string { return filepath.Join(f.dir, ".blobs", sum) }
|
||||
|
||||
// blob gets a file into the blob cache (by sha256), from the first catalog that has it.
|
||||
func (f *Fetcher) blob(ctx context.Context, catalogs []string, fe hub.FileEntry, token string) error {
|
||||
path := f.blobPath(fe.SHA256)
|
||||
if st, err := os.Stat(path); err == nil && st.Size() == fe.Size {
|
||||
return nil
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
var last error
|
||||
for _, c := range catalogs {
|
||||
if last = f.download(ctx, strings.TrimSuffix(c, "/")+"/api/v1/files/"+fe.SHA256, fe, token, path); last == nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return last
|
||||
}
|
||||
|
||||
func (f *Fetcher) download(ctx context.Context, url string, fe hub.FileEntry, token, path string) error {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("User-Agent", hub.UserAgent)
|
||||
if token != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
}
|
||||
resp, err := f.http.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("%s answered %d", url, resp.StatusCode)
|
||||
}
|
||||
tmp, err := os.CreateTemp(filepath.Dir(path), ".dl-")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.Remove(tmp.Name())
|
||||
h := sha256.New()
|
||||
n, err := io.Copy(io.MultiWriter(tmp, h), io.LimitReader(resp.Body, fe.Size+1))
|
||||
tmp.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n != fe.Size || hex.EncodeToString(h.Sum(nil)) != fe.SHA256 {
|
||||
return fmt.Errorf("%s sent a different file than the hub listed", url)
|
||||
}
|
||||
return os.Rename(tmp.Name(), path)
|
||||
}
|
||||
|
||||
func (f *Fetcher) accessToken(ctx context.Context) (string, error) {
|
||||
f.mu.Lock()
|
||||
tok, at := f.access, f.accessAt
|
||||
f.mu.Unlock()
|
||||
if tok != "" && time.Since(at) < 50*time.Minute {
|
||||
return tok, nil
|
||||
}
|
||||
tok, err := f.hub.AccessToken(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
f.mu.Lock()
|
||||
f.access, f.accessAt = tok, time.Now()
|
||||
f.mu.Unlock()
|
||||
return tok, nil
|
||||
}
|
||||
|
||||
func copyFile(src, dst string) error {
|
||||
in, err := os.Open(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer in.Close()
|
||||
out, err := os.Create(dst)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.Copy(out, in); err != nil {
|
||||
out.Close()
|
||||
return err
|
||||
}
|
||||
return out.Close()
|
||||
}
|
||||
Reference in new issue
Block a user