t5de-proxy/internal/fetch/fetch.go

// Package fetch gets T5 products from the hub's catalogs (t5de-hub's docs/products.md): the hub says which revision
// to load and which catalogs host it; the files come from a catalog, each checked against the hub's sha256, and the
// product is written as a pack in the cache folder, where the products package serves it like any other pack.
package fetch

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"sync"
	"time"
	"unicode/utf8"

	"git.t5de.dev/t5de/t5de-proxy/internal/hub"
)

// Hub is what the fetcher asks the hub.
type Hub interface {
	ProductFiles(ctx context.Context, id string) (*hub.Files, error)
	AccessToken(ctx context.Context) (string, error)
	Library(ctx context.Context) ([]hub.LibraryEntry, error)
}

// IsHubID reports whether id is a hub product ID (10 of a-z2-7), as opposed to a local pack's.
var IsHubID = regexp.MustCompile(`^[a-z2-7]{10}$`).MatchString

// Fetcher keeps hub products in a cache folder, one pack folder per product.
type Fetcher struct {
	hub  Hub
	dir  string
	logf func(format string, args ...any)
	http *http.Client

	// OnReady, if set, is called when a product was written (or updated) in the cache.
	OnReady func(id string)
	// OnLibrary, if set, is called after the library was read and its products fetched.
	OnLibrary func()

	mu       sync.Mutex
	library  map[string]bool // the person's library, as last read
	entries  []hub.LibraryEntry
	inflight map[string]bool
	failed   map[string]time.Time // products that failed lately (not retried for a while)
	access   string
	accessAt time.Time
}

// New keeps products in dir.
func New(h Hub, dir string, logf func(format string, args ...any)) *Fetcher {
	if logf == nil {
		logf = func(string, ...any) {}
	}
	return &Fetcher{hub: h, dir: dir, logf: logf, http: &http.Client{Timeout: 2 * time.Minute},
		inflight: map[string]bool{}, failed: map[string]time.Time{}}
}

// Dir is the cache folder (a pack root).
func (f *Fetcher) Dir() string { return f.dir }

// SyncLibrary makes sure every product in the person's library is in the cache, at the revision they use.
func (f *Fetcher) SyncLibrary(ctx context.Context) {
	lib, err := f.hub.Library(ctx)
	if err != nil {
		f.logf("fetch: reading the library: %v", err)
		return
	}
	ids := map[string]bool{}
	for _, e := range lib {
		ids[e.Product.ID] = true
	}
	f.mu.Lock()
	f.library, f.entries = ids, lib
	f.mu.Unlock()
	defer func() {
		if f.OnLibrary != nil {
			f.OnLibrary()
		}
	}()
	for _, e := range lib {
		if err := f.Ensure(ctx, e.Product.ID); err != nil {
			f.logf("fetch: %s (%s): %v", e.Product.ID, e.Product.Name, err)
		}
	}
}

// Want gets a product in the background, once at a time, and not again soon after a failure.
func (f *Fetcher) Want(id string) {
	if !IsHubID(id) {
		return
	}
	f.mu.Lock()
	if f.inflight[id] || time.Since(f.failed[id]) < 5*time.Minute {
		f.mu.Unlock()
		return
	}
	f.inflight[id] = true
	f.mu.Unlock()
	go func() {
		err := f.Ensure(context.Background(), id)
		f.mu.Lock()
		delete(f.inflight, id)
		if err != nil {
			f.failed[id] = time.Now()
		}
		f.mu.Unlock()
		if err != nil {
			f.logf("fetch: %s: %v", id, err)
		}
	}()
}

// manifest is the pack manifest written for a product (t5de-api's pack format 1).
type manifest struct {
	Format   int    `json:"format"`
	ID       string `json:"id"`
	Revision int    `json:"revision"`
	Name     string `json:"name"`
	Creator  string `json:"creator,omitempty"`
	Rating   string `json:"rating"`
}

// Ensure puts the product's current revision in the cache, unless it's there already.
func (f *Fetcher) Ensure(ctx context.Context, id string) error {
	if !IsHubID(id) {
		return fmt.Errorf("not a hub product ID: %q", id)
	}
	info, err := f.hub.ProductFiles(ctx, id)
	if err != nil {
		return err
	}
	final := filepath.Join(f.dir, id)
	if b, err := os.ReadFile(filepath.Join(final, "t5de-product.json")); err == nil {
		var m manifest
		if json.Unmarshal(b, &m) == nil && m.Revision == info.Revision {
			return nil
		}
	}
	if len(info.Catalogs) == 0 {
		return errors.New("no catalog hosts it")
	}
	for _, fe := range info.Files {
		if fe.Name == "" || fe.Name == "." || fe.Name == ".." || filepath.Base(fe.Name) != fe.Name || strings.ContainsAny(fe.Name, `/\:`) {
			return fmt.Errorf("bad file name %q", fe.Name)
		}
	}
	token := ""
	if info.Rating == "adult" {
		if token, err = f.accessToken(ctx); err != nil {
			return err
		}
	}
	for _, fe := range info.Files {
		if err := f.blob(ctx, info.Catalogs, fe, token); err != nil {
			return fmt.Errorf("%s: %w", fe.Name, err)
		}
	}

	// The pack, in a temporary folder, then in place.
	tmp, err := os.MkdirTemp(f.dir, "."+id+"-")
	if err != nil {
		return err
	}
	defer os.RemoveAll(tmp)
	if err := os.Mkdir(filepath.Join(tmp, "files"), 0o755); err != nil {
		return err
	}
	for _, fe := range info.Files {
		if err := copyFile(f.blobPath(fe.SHA256), filepath.Join(tmp, "files", fe.Name)); err != nil {
			return err
		}
	}
	m := manifest{Format: 1, ID: id, Revision: info.Revision, Name: clip(info.Name, "Untitled"), Creator: clip(info.Creator, ""), Rating: info.Rating}
	b, _ := json.MarshalIndent(m, "", "  ")
	if err := os.WriteFile(filepath.Join(tmp, "t5de-product.json"), b, 0o644); err != nil {
		return err
	}
	old := final + ".old"
	os.RemoveAll(old)
	if _, err := os.Stat(final); err == nil {
		if err := os.Rename(final, old); err != nil {
			return err
		}
	}
	if err := os.Rename(tmp, final); err != nil {
		os.Rename(old, final)
		return err
	}
	os.RemoveAll(old)
	f.logf("fetch: %s revision %d (%s) is ready", id, info.Revision, m.Name)
	if f.OnReady != nil {
		f.OnReady(id)
	}
	return nil
}

// clip makes a name fit the pack format (1-64 characters, no control characters).
func clip(s, empty string) string {
	s = strings.Map(func(r rune) rune {
		if r < 0x20 || r == 0x7f {
			return -1
		}
		return r
	}, strings.TrimSpace(s))
	for utf8.RuneCountInString(s) > 64 {
		_, n := utf8.DecodeLastRuneInString(s)
		s = s[:len(s)-n]
	}
	if s == "" {
		return empty
	}
	return s
}

func (f *Fetcher) blobPath(sum string) string { return filepath.Join(f.dir, ".blobs", sum) }

// blob gets a file into the blob cache (by sha256), from the first catalog that has it.
func (f *Fetcher) blob(ctx context.Context, catalogs []string, fe hub.FileEntry, token string) error {
	path := f.blobPath(fe.SHA256)
	if st, err := os.Stat(path); err == nil && st.Size() == fe.Size {
		return nil
	}
	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
		return err
	}
	var last error
	for _, c := range catalogs {
		if last = f.download(ctx, strings.TrimSuffix(c, "/")+"/api/v1/files/"+fe.SHA256, fe, token, path); last == nil {
			return nil
		}
	}
	return last
}

func (f *Fetcher) download(ctx context.Context, url string, fe hub.FileEntry, token, path string) error {
	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
	if err != nil {
		return err
	}
	req.Header.Set("User-Agent", hub.UserAgent)
	if token != "" {
		req.Header.Set("Authorization", "Bearer "+token)
	}
	resp, err := f.http.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("%s answered %d", url, resp.StatusCode)
	}
	tmp, err := os.CreateTemp(filepath.Dir(path), ".dl-")
	if err != nil {
		return err
	}
	defer os.Remove(tmp.Name())
	h := sha256.New()
	n, err := io.Copy(io.MultiWriter(tmp, h), io.LimitReader(resp.Body, fe.Size+1))
	tmp.Close()
	if err != nil {
		return err
	}
	if n != fe.Size || hex.EncodeToString(h.Sum(nil)) != fe.SHA256 {
		return fmt.Errorf("%s sent a different file than the hub listed", url)
	}
	return os.Rename(tmp.Name(), path)
}

func (f *Fetcher) accessToken(ctx context.Context) (string, error) {
	f.mu.Lock()
	tok, at := f.access, f.accessAt
	f.mu.Unlock()
	if tok != "" && time.Since(at) < 50*time.Minute {
		return tok, nil
	}
	tok, err := f.hub.AccessToken(ctx)
	if err != nil {
		return "", err
	}
	f.mu.Lock()
	f.access, f.accessAt = tok, time.Now()
	f.mu.Unlock()
	return tok, nil
}

func copyFile(src, dst string) error {
	in, err := os.Open(src)
	if err != nil {
		return err
	}
	defer in.Close()
	out, err := os.Create(dst)
	if err != nil {
		return err
	}
	if _, err := io.Copy(out, in); err != nil {
		out.Close()
		return err
	}
	return out.Close()
}

// Owned reports whether a product is in the person's library (as last read).
func (f *Fetcher) Owned(id string) bool {
	f.mu.Lock()
	defer f.mu.Unlock()
	return f.library[id]
}

// Library is the person's library, as last read.
func (f *Fetcher) Library() []hub.LibraryEntry {
	f.mu.Lock()
	defer f.mu.Unlock()
	return append([]hub.LibraryEntry(nil), f.entries...)
}