t5de-proxy/internal/proxy/proxy.go

// Package proxy is T5DE's local proxy for IMVU Classic: the client sends it every HTTP request (Python's and its
// embedded browser's, which IMVU routes through Python) and its IMQ connection, and the proxy makes them with
// Go's networking: current TLS and root certificates, HTTP/2 and pooled connections, HTTPS for IMVU hosts the
// client reaches over plain HTTP, and reporting endpoints refused.
//
// Protocol (only the client's own requests are served: every one carries the token the proxy gave it):
//
//   - HTTP requests in absolute form, for http and https URLs alike, over plain HTTP on 127.0.0.1
//     ("GET https://api.imvu.com/... HTTP/1.1"), with "Proxy-Authorization: Basic base64(t5de:<token>)".
//     The proxy makes the request and returns the response as is: no redirects followed, no decompression.
//   - CONNECT host:port with the same header: the client then speaks plaintext, and the proxy connects to
//     host:port over TLS. The client uses this for IMQ.
package proxy

import (
	"bytes"
	"compress/gzip"
	"context"
	"crypto/subtle"
	"crypto/tls"
	"encoding/base64"
	"errors"
	"io"
	"log"
	"net"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"sync/atomic"
	"time"
)

// ErrorHeader is set on the proxy's own 502 responses, when it couldn't reach the server, to the error.
const ErrorHeader = "X-T5de-Proxy-Error"

// Server is the proxy, an http.Handler.
type Server struct {
	// Token authenticates the client's requests.
	Token string
	// Log receives a line per request and connection.
	Log *log.Logger
	// Trace adds full URLs (with queries) and every IMQ frame to the log.
	Trace bool
	// Transport makes HTTP requests; nil is NewTransport().
	Transport http.RoundTripper
	// TLSConfig is the base TLS configuration of CONNECT's upstream connections (tests set RootCAs); nil is the
	// default, with the system's roots.
	TLSConfig *tls.Config
	// Interceptor, if set, sees every HTTP request (with a body the proxy could hold in memory).
	Interceptor Interceptor
	// IMQFilter, if set, sees the client's outgoing IMQ messages (on CONNECT streams) and can change or drop them.
	IMQFilter IMQFilter
	// Dial makes CONNECT's upstream connections (tests point it at local servers); nil is a net.Dialer.
	Dial func(ctx context.Context, network, addr string) (net.Conn, error)

	requests atomic.Int64
}

// NewTransport is the HTTP transport the proxy uses by default: HTTP/2 where servers offer it, pooled
// connections, and responses passed through compressed.
func NewTransport() *http.Transport {
	return &http.Transport{
		DialContext:           (&net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second}).DialContext,
		ForceAttemptHTTP2:     true,
		MaxIdleConns:          100,
		MaxIdleConnsPerHost:   16,
		IdleConnTimeout:       90 * time.Second,
		TLSHandshakeTimeout:   15 * time.Second,
		ExpectContinueTimeout: time.Second,
		// The client asks for gzip itself and decompresses it; leave Accept-Encoding and bodies alone.
		DisableCompression: true,
	}
}

func (s *Server) transport() http.RoundTripper {
	if s.Transport == nil {
		s.Transport = NewTransport()
	}
	return s.Transport
}

func (s *Server) logf(format string, args ...any) {
	if s.Log != nil {
		s.Log.Printf(format, args...)
	}
}

// authorized checks the request's Proxy-Authorization against the token.
func (s *Server) authorized(r *http.Request) bool {
	auth := r.Header.Get("Proxy-Authorization")
	scheme, encoded, ok := strings.Cut(auth, " ")
	if !ok || !strings.EqualFold(scheme, "Basic") {
		return false
	}
	decoded, err := base64.StdEncoding.DecodeString(strings.TrimSpace(encoded))
	if err != nil {
		return false
	}
	user, password, _ := strings.Cut(string(decoded), ":")
	return s.Token != "" && user == "t5de" && subtle.ConstantTimeCompare([]byte(password), []byte(s.Token)) == 1
}

func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if !s.authorized(r) {
		w.Header().Set("Proxy-Authenticate", `Basic realm="t5de-proxy"`)
		http.Error(w, "t5de-proxy: not authorized", http.StatusProxyAuthRequired)
		return
	}
	if r.Method == http.MethodConnect {
		s.serveConnect(w, r)
		return
	}
	if !r.URL.IsAbs() || (r.URL.Scheme != "http" && r.URL.Scheme != "https") || r.URL.Host == "" {
		http.Error(w, "t5de-proxy: expected an absolute http or https URL", http.StatusBadRequest)
		return
	}
	s.forward(w, r)
}

// hopHeaders are a connection's own headers, not passed on (RFC 9110 7.6.1).
var hopHeaders = []string{
	"Connection", "Proxy-Connection", "Keep-Alive", "Proxy-Authenticate", "Proxy-Authorization",
	"Te", "Trailer", "Transfer-Encoding", "Upgrade",
}

func removeHopHeaders(h http.Header) {
	for _, field := range h.Values("Connection") {
		for _, name := range strings.Split(field, ",") {
			if name = strings.TrimSpace(name); name != "" {
				h.Del(name)
			}
		}
	}
	for _, name := range hopHeaders {
		h.Del(name)
	}
}

// logURL is u as logged: without its query (which can hold session values) unless tracing.
func (s *Server) logURL(u *url.URL) string {
	if s.Trace || u.RawQuery == "" {
		return u.String()
	}
	v := *u
	v.RawQuery = ""
	return v.String() + "?…"
}

func (s *Server) forward(w http.ResponseWriter, r *http.Request) {
	id := s.requests.Add(1)
	start := time.Now()
	target := *r.URL
	if Blocked(&target) {
		s.logf("#%d %s %s blocked", id, r.Method, s.logURL(&target))
		http.Error(w, "t5de: blocked request", http.StatusForbidden)
		return
	}
	upgraded := Upgrade(&target)
	if upgraded {
		target.Scheme = "https"
	}

	out := r.Clone(r.Context())
	out.RequestURI = ""
	out.URL = &target
	out.Host = ""
	removeHopHeaders(out.Header)
	if _, ok := out.Header["User-Agent"]; !ok {
		out.Header.Set("User-Agent", "") // send none, as the client didn't, rather than Go's
	}
	replayable, err := bufferBody(out, r)
	if err != nil {
		s.logf("#%d %s %s: reading the request: %v", id, r.Method, s.logURL(&target), err)
		http.Error(w, "t5de-proxy: "+err.Error(), http.StatusBadRequest)
		return
	}
	var modify func(*http.Response, []byte) ([]byte, error)
	if s.Interceptor != nil && replayable {
		x := &Exchange{URL: r.URL, Request: out}
		if out.GetBody != nil {
			body, _ := out.GetBody()
			x.Body, _ = io.ReadAll(body)
		}
		original := x.Body
		if err := s.Interceptor.Intercept(x); err != nil {
			s.logf("#%d %s %s: %v", id, r.Method, s.logURL(r.URL), err)
			w.Header().Set(ErrorHeader, err.Error())
			http.Error(w, "t5de-proxy: "+err.Error(), http.StatusBadGateway)
			return
		}
		if x.Response != nil {
			s.writeLocal(w, id, r, x.Response, start)
			return
		}
		if !bytes.Equal(x.Body, original) {
			setBody(out, x.Body)
		}
		modify = x.ModifyResponse
	}

	resp, err := s.transport().RoundTrip(out)
	if err != nil && upgraded && replayable && r.Context().Err() == nil {
		// The host didn't take HTTPS this time: fetch it as the client asked.
		s.logf("#%d %s %s over HTTPS failed (%v); retrying over HTTP", id, r.Method, s.logURL(&target), err)
		retry := out.Clone(r.Context())
		retry.URL = r.URL
		if out.GetBody != nil {
			retry.Body, _ = out.GetBody()
		}
		upgraded = false
		resp, err = s.transport().RoundTrip(retry)
	}
	if err != nil {
		s.logf("#%d %s %s failed after %s: %v", id, r.Method, s.logURL(&target), since(start), err)
		// The header tells the client this is the proxy's error, not the server's: it raises it as a network
		// error, as it would have connecting directly.
		w.Header().Set(ErrorHeader, err.Error())
		http.Error(w, "t5de-proxy: "+err.Error(), http.StatusBadGateway)
		return
	}
	defer resp.Body.Close()
	if modify != nil {
		if err := modifyResponse(resp, modify); err != nil {
			s.logf("#%d %s %s: changing the response: %v", id, r.Method, s.logURL(r.URL), err)
		}
	}

	removeHopHeaders(resp.Header)
	for name, values := range resp.Header {
		w.Header()[name] = values
	}
	w.WriteHeader(resp.StatusCode)
	n, err := copyFlushing(w, resp.Body)
	note := ""
	if upgraded {
		note = " (https)"
	}
	if err != nil && !errors.Is(err, context.Canceled) {
		note += " body error: " + err.Error()
	}
	s.logf("#%d %s %s %d %dB %s%s", id, r.Method, s.logURL(r.URL), resp.StatusCode, n, since(start), note)
}

// Interceptor can answer requests itself, change them before they're sent, and change their responses. The
// t5de-proxy's community products are served this way.
type Interceptor interface {
	Intercept(x *Exchange) error
}

// Exchange is a request on its way, as an Interceptor sees it.
type Exchange struct {
	// URL is the URL the client asked for (before any upgrade to HTTPS).
	URL *url.URL
	// Request is the request about to be sent; its body is Body.
	Request *http.Request
	// Body is the request's body; an Interceptor may replace it.
	Body []byte
	// Response, if an Interceptor sets it, answers the request: nothing is sent.
	Response *http.Response
	// ModifyResponse, if set, changes the server's response body (decompressed) before the client gets it.
	ModifyResponse func(resp *http.Response, body []byte) ([]byte, error)
}

// LocalResponse is a response made by the proxy.
func LocalResponse(status int, contentType string, body []byte) *http.Response {
	h := http.Header{}
	h.Set("Content-Type", contentType)
	h.Set("Content-Length", strconv.Itoa(len(body)))
	return &http.Response{StatusCode: status, Header: h, ContentLength: int64(len(body)), Body: io.NopCloser(bytes.NewReader(body))}
}

func (s *Server) writeLocal(w http.ResponseWriter, id int64, r *http.Request, resp *http.Response, start time.Time) {
	defer resp.Body.Close()
	for name, values := range resp.Header {
		w.Header()[name] = values
	}
	w.WriteHeader(resp.StatusCode)
	n, _ := io.Copy(w, resp.Body)
	s.logf("#%d %s %s %d %dB %s (answered by the proxy)", id, r.Method, s.logURL(r.URL), resp.StatusCode, n, since(start))
}

func setBody(out *http.Request, body []byte) {
	out.ContentLength = int64(len(body))
	out.Header.Set("Content-Length", strconv.Itoa(len(body)))
	out.GetBody = func() (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(body)), nil }
	out.Body, _ = out.GetBody()
	if len(body) == 0 {
		out.Body, out.GetBody = nil, nil
	}
}

// modifyResponse replaces resp's body with modify's version of it, uncompressed.
func modifyResponse(resp *http.Response, modify func(*http.Response, []byte) ([]byte, error)) error {
	raw, err := io.ReadAll(io.LimitReader(resp.Body, maxBuffered))
	resp.Body.Close()
	if err != nil {
		resp.Body = io.NopCloser(bytes.NewReader(raw))
		return err
	}
	body := raw
	if strings.EqualFold(resp.Header.Get("Content-Encoding"), "gzip") {
		zr, err := gzip.NewReader(bytes.NewReader(raw))
		if err == nil {
			body, err = io.ReadAll(io.LimitReader(zr, maxBuffered))
		}
		if err != nil {
			resp.Body = io.NopCloser(bytes.NewReader(raw))
			return err
		}
	}
	changed, err := modify(resp, body)
	if err != nil {
		changed = body
	}
	resp.Header.Del("Content-Encoding")
	resp.Header.Del("Content-Md5")
	resp.Header.Set("Content-Length", strconv.Itoa(len(changed)))
	resp.ContentLength = int64(len(changed))
	resp.Body = io.NopCloser(bytes.NewReader(changed))
	return err
}

// maxBuffered bounds the request bodies the proxy holds in memory; larger ones (uploads) are streamed.
const maxBuffered = 32 << 20

// bufferBody reads a request's body into memory, so it can be sent again: by the transport, when a pooled
// connection turns out to have been closed by the server, and over HTTP after a failed upgrade. It reports
// whether the request can be sent again.
func bufferBody(out, in *http.Request) (bool, error) {
	if in.Body == nil || in.ContentLength == 0 {
		out.Body, out.GetBody, out.ContentLength = nil, nil, 0
		return true, nil
	}
	head, err := io.ReadAll(io.LimitReader(in.Body, maxBuffered+1))
	if err != nil {
		return false, err
	}
	if len(head) > maxBuffered {
		out.Body = io.NopCloser(io.MultiReader(bytes.NewReader(head), in.Body))
		return false, nil
	}
	out.ContentLength = int64(len(head))
	out.GetBody = func() (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(head)), nil }
	out.Body, _ = out.GetBody()
	return true, nil
}

// copyFlushing copies a response body to the client, flushing as it arrives (long polls and streams included).
func copyFlushing(w http.ResponseWriter, body io.Reader) (int64, error) {
	rc := http.NewResponseController(w)
	buf := make([]byte, 32<<10)
	var total int64
	for {
		n, err := body.Read(buf)
		if n > 0 {
			if _, werr := w.Write(buf[:n]); werr != nil {
				return total, werr
			}
			total += int64(n)
			if err == nil {
				_ = rc.Flush()
			}
		}
		if err == io.EOF {
			return total, nil
		}
		if err != nil {
			return total, err
		}
	}
}

func since(t time.Time) string {
	return time.Since(t).Round(time.Millisecond).String()
}