// Package fetch gets T5 products from the hub's catalogs (t5de-hub's docs/products.md): the hub says which revision // to load and which catalogs host it; the files come from a catalog, each checked against the hub's sha256, and the // product is written as a pack in the cache folder, where the products package serves it like any other pack. package fetch import ( "context" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" "io" "net/http" "os" "path/filepath" "regexp" "strings" "sync" "time" "unicode/utf8" "git.t5de.dev/t5de/t5de-proxy/internal/hub" ) // Hub is what the fetcher asks the hub. type Hub interface { ProductFiles(ctx context.Context, id string) (*hub.Files, error) AccessToken(ctx context.Context) (string, error) Library(ctx context.Context) ([]hub.LibraryEntry, error) } // IsHubID reports whether id is a hub product ID (10 of a-z2-7), as opposed to a local pack's. var IsHubID = regexp.MustCompile(`^[a-z2-7]{10}$`).MatchString // Fetcher keeps hub products in a cache folder, one pack folder per product. type Fetcher struct { hub Hub dir string logf func(format string, args ...any) http *http.Client // OnReady, if set, is called when a product was written (or updated) in the cache. OnReady func(id string) mu sync.Mutex inflight map[string]bool failed map[string]time.Time // products that failed lately (not retried for a while) access string accessAt time.Time } // New keeps products in dir. func New(h Hub, dir string, logf func(format string, args ...any)) *Fetcher { if logf == nil { logf = func(string, ...any) {} } return &Fetcher{hub: h, dir: dir, logf: logf, http: &http.Client{Timeout: 2 * time.Minute}, inflight: map[string]bool{}, failed: map[string]time.Time{}} } // Dir is the cache folder (a pack root). func (f *Fetcher) Dir() string { return f.dir } // SyncLibrary makes sure every product in the person's library is in the cache, at the revision they use. func (f *Fetcher) SyncLibrary(ctx context.Context) { lib, err := f.hub.Library(ctx) if err != nil { f.logf("fetch: reading the library: %v", err) return } for _, e := range lib { if err := f.Ensure(ctx, e.Product.ID); err != nil { f.logf("fetch: %s (%s): %v", e.Product.ID, e.Product.Name, err) } } } // Want gets a product in the background, once at a time, and not again soon after a failure. func (f *Fetcher) Want(id string) { if !IsHubID(id) { return } f.mu.Lock() if f.inflight[id] || time.Since(f.failed[id]) < 5*time.Minute { f.mu.Unlock() return } f.inflight[id] = true f.mu.Unlock() go func() { err := f.Ensure(context.Background(), id) f.mu.Lock() delete(f.inflight, id) if err != nil { f.failed[id] = time.Now() } f.mu.Unlock() if err != nil { f.logf("fetch: %s: %v", id, err) } }() } // manifest is the pack manifest written for a product (t5de-api's pack format 1). type manifest struct { Format int `json:"format"` ID string `json:"id"` Revision int `json:"revision"` Name string `json:"name"` Creator string `json:"creator,omitempty"` Rating string `json:"rating"` } // Ensure puts the product's current revision in the cache, unless it's there already. func (f *Fetcher) Ensure(ctx context.Context, id string) error { if !IsHubID(id) { return fmt.Errorf("not a hub product ID: %q", id) } info, err := f.hub.ProductFiles(ctx, id) if err != nil { return err } final := filepath.Join(f.dir, id) if b, err := os.ReadFile(filepath.Join(final, "t5de-product.json")); err == nil { var m manifest if json.Unmarshal(b, &m) == nil && m.Revision == info.Revision { return nil } } if len(info.Catalogs) == 0 { return errors.New("no catalog hosts it") } for _, fe := range info.Files { if fe.Name == "" || fe.Name == "." || fe.Name == ".." || filepath.Base(fe.Name) != fe.Name || strings.ContainsAny(fe.Name, `/\:`) { return fmt.Errorf("bad file name %q", fe.Name) } } token := "" if info.Rating == "adult" { if token, err = f.accessToken(ctx); err != nil { return err } } for _, fe := range info.Files { if err := f.blob(ctx, info.Catalogs, fe, token); err != nil { return fmt.Errorf("%s: %w", fe.Name, err) } } // The pack, in a temporary folder, then in place. tmp, err := os.MkdirTemp(f.dir, "."+id+"-") if err != nil { return err } defer os.RemoveAll(tmp) if err := os.Mkdir(filepath.Join(tmp, "files"), 0o755); err != nil { return err } for _, fe := range info.Files { if err := copyFile(f.blobPath(fe.SHA256), filepath.Join(tmp, "files", fe.Name)); err != nil { return err } } m := manifest{Format: 1, ID: id, Revision: info.Revision, Name: clip(info.Name, "Untitled"), Creator: clip(info.Creator, ""), Rating: info.Rating} b, _ := json.MarshalIndent(m, "", " ") if err := os.WriteFile(filepath.Join(tmp, "t5de-product.json"), b, 0o644); err != nil { return err } old := final + ".old" os.RemoveAll(old) if _, err := os.Stat(final); err == nil { if err := os.Rename(final, old); err != nil { return err } } if err := os.Rename(tmp, final); err != nil { os.Rename(old, final) return err } os.RemoveAll(old) f.logf("fetch: %s revision %d (%s) is ready", id, info.Revision, m.Name) if f.OnReady != nil { f.OnReady(id) } return nil } // clip makes a name fit the pack format (1-64 characters, no control characters). func clip(s, empty string) string { s = strings.Map(func(r rune) rune { if r < 0x20 || r == 0x7f { return -1 } return r }, strings.TrimSpace(s)) for utf8.RuneCountInString(s) > 64 { _, n := utf8.DecodeLastRuneInString(s) s = s[:len(s)-n] } if s == "" { return empty } return s } func (f *Fetcher) blobPath(sum string) string { return filepath.Join(f.dir, ".blobs", sum) } // blob gets a file into the blob cache (by sha256), from the first catalog that has it. func (f *Fetcher) blob(ctx context.Context, catalogs []string, fe hub.FileEntry, token string) error { path := f.blobPath(fe.SHA256) if st, err := os.Stat(path); err == nil && st.Size() == fe.Size { return nil } if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { return err } var last error for _, c := range catalogs { if last = f.download(ctx, strings.TrimSuffix(c, "/")+"/api/v1/files/"+fe.SHA256, fe, token, path); last == nil { return nil } } return last } func (f *Fetcher) download(ctx context.Context, url string, fe hub.FileEntry, token, path string) error { req, err := http.NewRequestWithContext(ctx, "GET", url, nil) if err != nil { return err } req.Header.Set("User-Agent", hub.UserAgent) if token != "" { req.Header.Set("Authorization", "Bearer "+token) } resp, err := f.http.Do(req) if err != nil { return err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return fmt.Errorf("%s answered %d", url, resp.StatusCode) } tmp, err := os.CreateTemp(filepath.Dir(path), ".dl-") if err != nil { return err } defer os.Remove(tmp.Name()) h := sha256.New() n, err := io.Copy(io.MultiWriter(tmp, h), io.LimitReader(resp.Body, fe.Size+1)) tmp.Close() if err != nil { return err } if n != fe.Size || hex.EncodeToString(h.Sum(nil)) != fe.SHA256 { return fmt.Errorf("%s sent a different file than the hub listed", url) } return os.Rename(tmp.Name(), path) } func (f *Fetcher) accessToken(ctx context.Context) (string, error) { f.mu.Lock() tok, at := f.access, f.accessAt f.mu.Unlock() if tok != "" && time.Since(at) < 50*time.Minute { return tok, nil } tok, err := f.hub.AccessToken(ctx) if err != nil { return "", err } f.mu.Lock() f.access, f.accessAt = tok, time.Now() f.mu.Unlock() return tok, nil } func copyFile(src, dst string) error { in, err := os.Open(src) if err != nil { return err } defer in.Close() out, err := os.Create(dst) if err != nil { return err } if _, err := io.Copy(out, in); err != nil { out.Close() return err } return out.Close() }