From 2551888ed65b448f1da70d0960a86908f06975d1 Mon Sep 17 00:00:00 2001 From: dhkatz Date: Thu, 29 Oct 2026 00:03:40 -0700 Subject: [PATCH] T5DE's local proxy for IMVU Classic The client sends it every HTTP request and its IMQ connection; it makes them with current TLS and certificates, upgrades IMVU's plain-HTTP hosts to HTTPS, and refuses IMVU's reporting endpoints. CI tests it and publishes t5de-proxy.exe as nightlies and tagged releases. Co-Authored-By: Claude Opus 5.5 --- .gitattributes | 3 + .gitea/scripts/release.py | 153 +++++++++++++ .gitea/workflows/ci.yml | 57 +++++ .gitignore | 1 + CLAUDE.md | 26 +++ README.md | 41 ++++ cmd/t5de-proxy/main.go | 100 +++++++++ docs/hosts.md | 38 ++++ go.mod | 3 + internal/buildinfo/buildinfo.go | 5 + internal/imq/frame.go | 140 ++++++++++++ internal/imq/frame_test.go | 65 ++++++ internal/parent/parent.go | 7 + internal/parent/parent_other.go | 21 ++ internal/parent/parent_windows.go | 13 ++ internal/proxy/connect.go | 144 +++++++++++++ internal/proxy/policy.go | 76 +++++++ internal/proxy/proxy.go | 263 +++++++++++++++++++++++ internal/proxy/proxy_test.go | 342 ++++++++++++++++++++++++++++++ 19 files changed, 1498 insertions(+) create mode 100644 .gitattributes create mode 100644 .gitea/scripts/release.py create mode 100644 .gitea/workflows/ci.yml create mode 100644 .gitignore create mode 100644 CLAUDE.md create mode 100644 README.md create mode 100644 cmd/t5de-proxy/main.go create mode 100644 docs/hosts.md create mode 100644 go.mod create mode 100644 internal/buildinfo/buildinfo.go create mode 100644 internal/imq/frame.go create mode 100644 internal/imq/frame_test.go create mode 100644 internal/parent/parent.go create mode 100644 internal/parent/parent_other.go create mode 100644 internal/parent/parent_windows.go create mode 100644 internal/proxy/connect.go create mode 100644 internal/proxy/policy.go create mode 100644 internal/proxy/proxy.go create mode 100644 internal/proxy/proxy_test.go diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..48d029a --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +# gofmt requires LF; keep sources LF on every platform. +* text=auto eol=lf +*.exe binary diff --git a/.gitea/scripts/release.py b/.gitea/scripts/release.py new file mode 100644 index 0000000..b752ecd --- /dev/null +++ b/.gitea/scripts/release.py @@ -0,0 +1,153 @@ +"""Publish the proxy. Run by ci.yml. + + release.py version print version and channel + release.py publish publish the program in dir: a release for a vX.Y.Z tag, else the rolling nightly + +A vX.Y.Z tag is release X.Y.Z. Other builds are nightlies, "X.Y.Z-nightly.N+commit": N commits after the last +release tag (0.0.0 before the first). The nightly pre-release (tag `nightly`) is replaced on every push to main. +The same scheme as T5DE's own releases (t5de's .gitea/scripts/release.py). +""" + +import hashlib +import json +import os +import re +import subprocess +import sys +import urllib.error +import urllib.request +import uuid +from pathlib import Path + +ASSETS = ("t5de-proxy.exe",) +NIGHTLY_TAG = "nightly" +USER_AGENT = "t5de-release (https://git.t5de.dev/t5de/t5de-proxy, 1)" +RELEASE_TAG = re.compile(r"v(\d+)\.(\d+)\.(\d+)") + + +def git(*args: str) -> str: + return subprocess.run(["git", *args], capture_output=True, text=True, check=True).stdout.strip() + + +def version() -> dict[str, str]: + ref_name = os.environ.get("GITHUB_REF_NAME", "") + if os.environ.get("GITHUB_REF_TYPE") == "tag" and RELEASE_TAG.fullmatch(ref_name): + return {"version": ref_name[1:], "channel": "release", "tag": ref_name} + try: + last = git("describe", "--tags", "--abbrev=0", "--match", "v[0-9]*.[0-9]*.[0-9]*") + count = int(git("rev-list", "--count", f"{last}..HEAD")) + base = last[1:] + except subprocess.CalledProcessError: # no release yet + count, base = int(git("rev-list", "--count", "HEAD")), "0.0.0" + commit = git("rev-parse", "--short=7", "HEAD") + return {"version": f"{base}-nightly.{count}+{commit}", "channel": "nightly", "tag": NIGHTLY_TAG} + + +class Gitea: + def __init__(self): + self.api = f"{os.environ['GITHUB_SERVER_URL']}/api/v1/repos/{os.environ['GITHUB_REPOSITORY']}" + self.token = os.environ["GITEA_TOKEN"] + + def request(self, method: str, path: str, body: dict | None = None, data: bytes | None = None, + content_type: str = "application/json"): + if body is not None: + data = json.dumps(body).encode() + req = urllib.request.Request(self.api + path, method=method, data=data, + # Cloudflare in front of the server refuses Python's default User-Agent. + headers={"Authorization": f"token {self.token}", "Content-Type": content_type, + "User-Agent": USER_AGENT}) + try: + with urllib.request.urlopen(req, timeout=120) as response: + text = response.read() + return json.loads(text) if text else None + except urllib.error.HTTPError as e: + if e.code == 404 and method in ("GET", "DELETE"): + return None + raise SystemExit(f"{method} {path}: {e.code} {e.read().decode(errors='replace')}") + + def upload(self, release_id: int, name: str, content: bytes): + boundary = uuid.uuid4().hex + data = (f"--{boundary}\r\nContent-Disposition: form-data; name=\"attachment\"; filename=\"{name}\"\r\n" + f"Content-Type: application/octet-stream\r\n\r\n").encode() + content + f"\r\n--{boundary}--\r\n".encode() + return self.request("POST", f"/releases/{release_id}/assets?name={name}", data=data, + content_type=f"multipart/form-data; boundary={boundary}") + + +def notes(info: dict[str, str], files: dict[str, bytes]) -> str: + nightly = ("A nightly build of the latest commit, for testing; it's replaced by the next one.\n\n" + if info["channel"] == "nightly" else "") + hashes = "\n".join(f"`{name}` sha256: `{hashlib.sha256(content).hexdigest()}`" for name, content in files.items()) + return f"""{nightly}T5DE's proxy for IMVU Classic: the client connects to IMVU through it. T5DE's setup installs a +pinned release with its T5DE Proxy patch. To try this build instead, copy t5de-proxy.exe into a T5DE install's +folder (with that patch) while the client is closed. + +Unofficial: not made by or affiliated with IMVU. Use at your own risk. + +Version {info['version']} · commit {os.environ.get('GITHUB_SHA', git('rev-parse', 'HEAD'))} +{hashes} +""" + + +def publish(folder: str): + info = version() + files = {name: (Path(folder) / name).read_bytes() for name in ASSETS} + gitea = Gitea() + previous = None + if info["channel"] == "nightly": + old = gitea.request("GET", f"/releases/tags/{NIGHTLY_TAG}") + if old: + previous = old["target_commitish"] + gitea.request("DELETE", f"/releases/{old['id']}") + gitea.request("DELETE", f"/tags/{NIGHTLY_TAG}") + release = gitea.request("POST", "/releases", { + "tag_name": NIGHTLY_TAG, "target_commitish": os.environ["GITHUB_SHA"], + "name": f"Nightly {info['version']}", "body": notes(info, files), "prerelease": True}) + else: + release = gitea.request("GET", f"/releases/tags/{info['tag']}") + fields = {"name": f"T5DE Proxy {info['version']}", "body": notes(info, files), "prerelease": False} + if release: # a rerun: refresh it + release = gitea.request("PATCH", f"/releases/{release['id']}", fields) + for asset in release.get("assets") or []: + if asset["name"] in files: + gitea.request("DELETE", f"/releases/{release['id']}/assets/{asset['id']}") + else: + release = gitea.request("POST", "/releases", {"tag_name": info["tag"], **fields}) + for name, content in files.items(): + gitea.upload(release["id"], name, content) + print(f"Published {release['html_url']}") + if info["channel"] == "nightly" and os.environ.get("DISCORD_NIGHTLY_WEBHOOK"): + announce(os.environ["DISCORD_NIGHTLY_WEBHOOK"], info, release, previous) + + +def announce(webhook: str, info: dict[str, str], release: dict, previous: str | None): + """Post the release to a Discord channel. Failing to doesn't fail the release.""" + commits = [] + if previous: + try: + commits = git("log", "--format=%h %s", "--no-merges", f"{previous}..HEAD").splitlines() + except subprocess.CalledProcessError: # the previous nightly's commit is gone (history rewritten) + pass + shown = commits[:10] + ([f"and {len(commits) - 10} more"] if len(commits) > 10 else []) + description = f"[Download]({release['html_url']})" + if shown: + description += "\n\n" + "\n".join(f"• {line}" for line in shown) + body = {"embeds": [{"title": f"Proxy Nightly {info['version']}", "url": release["html_url"], + "description": description[:4000], "color": 0xFFD200}]} + req = urllib.request.Request(webhook, method="POST", data=json.dumps(body).encode(), + headers={"Content-Type": "application/json", + "User-Agent": USER_AGENT}) + try: + urllib.request.urlopen(req, timeout=30).close() + print("Announced on Discord") + except (urllib.error.URLError, TimeoutError) as e: + print(f"warning: couldn't post to Discord: {e}") + + +if __name__ == "__main__": + if sys.argv[1:] == ["version"]: + for key, value in version().items(): + print(f"{key}={value}") + elif len(sys.argv) == 3 and sys.argv[1] == "publish": + publish(sys.argv[2]) + else: + raise SystemExit(__doc__) diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml new file mode 100644 index 0000000..1aaa9ec --- /dev/null +++ b/.gitea/workflows/ci.yml @@ -0,0 +1,57 @@ +# Tests the proxy, builds t5de-proxy.exe for Windows, and publishes it: pushes to main replace the nightly +# pre-release, vX.Y.Z tags make release X.Y.Z (.gitea/scripts/release.py). T5DE pins a release in its t5de.toml. +name: CI + +on: + push: + branches: [main] + tags: ["v*"] + pull_request: + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # versions count commits since the last release tag + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Test + run: | + unformatted=$(gofmt -l .) + if [ -n "$unformatted" ]; then echo "Not gofmt-formatted: $unformatted"; exit 1; fi + go vet ./... + GOOS=windows go vet ./... + go test ./... + + - name: Version + id: version + run: python .gitea/scripts/release.py version | tee -a "$GITHUB_OUTPUT" + + # A GUI-subsystem program, so the client starting it opens no console window. Reproducible: no paths, no + # build ID. + - name: Build t5de-proxy.exe + env: + VERSION: ${{ steps.version.outputs.version }} + run: | + GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build -trimpath -buildvcs=false \ + -ldflags "-H=windowsgui -s -w -buildid= -X git.t5de.dev/t5de/t5de-proxy/internal/buildinfo.Version=$VERSION" \ + -o dist/t5de-proxy.exe ./cmd/t5de-proxy + sha256sum dist/t5de-proxy.exe + + - name: Publish the release + if: github.event_name != 'pull_request' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) + env: + GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Optional: a Discord webhook URL that nightlies are announced on. + DISCORD_NIGHTLY_WEBHOOK: ${{ secrets.DISCORD_NIGHTLY_WEBHOOK }} + run: python .gitea/scripts/release.py publish dist diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..178135c --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/dist/ diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..ea0098b --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,26 @@ +# t5de-proxy + +T5DE's local proxy for IMVU Classic (see README.md, docs/hosts.md). Part of T5DE; the patches and the patcher are +in the `t5de` repository (git.t5de.dev/t5de/t5de, usually checked out beside this one). The client side is t5de's +`t5de-proxy` patch (`main/client.py` `t5deProxy*`, `Network.urlopen`/`post_multipart`, +`IMQConnection.__buildStrategies`). A later goal is a backend of T5DE's own behind it. + +## Rules + +- Never commit IMVU's code: no decompiled source, no captured traffic with account data. Our own declarations of + IMVU's protocols (message type numbers, framing) and notes in our own words are fine. +- Test with a throwaway IMVU account, never a real one. +- Commits: author "dhkatz "; never the user's real name or email. +- The user rejects designs that delay what's shown: the proxy streams responses and adds no waiting. + +## Notes + +- The client's Python (2.7, OpenSSL 1.1.0f, TLS 1.2 only, `resources/cacert.pem` from IMVU) makes every HTTP + request in `Network.urlopen`, including the embedded browser's: Gecko 1.9.2's http/https protocol handlers are + replaced by IMVU's `PythonProtocolHandler` (imvugecko.dll), which calls `Network.performGeckoRequest`. No IMVU + native code makes HTTP requests (Flash and `IMVUQualityAgent.exe` aside). +- The log leaves out queries unless tracing (`-trace`, `T5DE_PROXY_TRACE=1` in the client's environment): they + can hold session values. Never log headers or bodies. +- t5de's `tools/drive.py` starts the built client, takes screenshots, clicks, and sends `imvu:` commands + (`showMode?mode=shop`, ...), so the proxy can be tested in the real client without a person; the test + account's saved login signs in by itself. The log is `%TEMP%\t5de-proxy.log`. diff --git a/README.md b/README.md new file mode 100644 index 0000000..20ed53d --- /dev/null +++ b/README.md @@ -0,0 +1,41 @@ +# t5de-proxy + +T5DE's local proxy for IMVU Classic. With T5DE's `t5de-proxy` patch, the client starts `t5de-proxy.exe` and +connects to IMVU through it: + +- **Every HTTP request**, the embedded browser's included (IMVU's browser hands its requests to the client's + Python). The proxy makes them with current TLS and root certificates (the client has only IMVU's bundled + certificates and TLS 1.2), pooled connections and HTTP/2. +- **HTTPS for IMVU hosts the client reaches over plain HTTP** (its XML-RPC API, images, CSS), so its account + headers don't cross the network in the clear ([docs/hosts.md](docs/hosts.md)). +- **IMQ**, IMVU's chat and presence connection: the client speaks plaintext to the proxy, which connects to the + gateway over TLS. +- **IMVU's reporting endpoints refused**, as T5DE's patches already do in the client. + +If the proxy isn't there or stops, the client connects directly, as before. + +Unofficial: not made by or affiliated with IMVU. + +## How it works + +The proxy listens on a free port of 127.0.0.1 and accepts only requests carrying the token it gave the client +(`Proxy-Authorization: Basic t5de:`), so other programs and web pages can't use it. + +- HTTP: requests in absolute form (`GET https://api.imvu.com/... HTTP/1.1`) over plain HTTP on loopback, for + http and https URLs alike. Responses come back as the server sent them: redirects aren't followed, bodies + aren't decompressed. When the proxy can't reach a server it answers 502 with `X-T5de-Proxy-Error`, which the + client raises as the network error it would have had connecting directly. +- `CONNECT host:port`: the client then speaks plaintext and the proxy connects to `host:port` over TLS. The + proxy reads IMQ's framing (message type and size, not contents) to log it. + +The client starts it as `t5de-proxy -parent -ready -log %TEMP%\t5de-proxy.log`; it writes its +address and token to the ready file once listening, and exits with the client. Setting `T5DE_PROXY_TRACE=1` +before starting the client adds full URLs (with queries) and every IMQ frame to the log. + +## Building + + go test ./... + GOOS=windows GOARCH=amd64 go build -trimpath -ldflags "-H=windowsgui" -o dist/t5de-proxy.exe ./cmd/t5de-proxy + +CI publishes `t5de-proxy.exe`: pushes to main replace the `nightly` pre-release, `vX.Y.Z` tags make releases. +To ship a release in T5DE, copy its url and sha256 into t5de's `t5de.toml` (`[[programs]]`, patch `t5de-proxy`). diff --git a/cmd/t5de-proxy/main.go b/cmd/t5de-proxy/main.go new file mode 100644 index 0000000..a8014cc --- /dev/null +++ b/cmd/t5de-proxy/main.go @@ -0,0 +1,100 @@ +// Command t5de-proxy is T5DE's local proxy for IMVU Classic (see internal/proxy). The client starts one as it +// starts, and it exits with the client: +// +// t5de-proxy -parent -ready [-log ] [-trace] +// +// It listens on a free port of 127.0.0.1 and, once it's listening, writes {"address", "token"} to the ready file +// (written whole, by renaming), which the client reads. +package main + +import ( + "crypto/rand" + "encoding/hex" + "encoding/json" + "flag" + "fmt" + "io" + "log" + "net" + "net/http" + "os" + "time" + + "git.t5de.dev/t5de/t5de-proxy/internal/buildinfo" + "git.t5de.dev/t5de/t5de-proxy/internal/parent" + "git.t5de.dev/t5de/t5de-proxy/internal/proxy" +) + +func main() { + listen := flag.String("listen", "127.0.0.1:0", "address to listen on") + ready := flag.String("ready", "", "file to write the address and token to once listening") + parentPID := flag.Int("parent", 0, "exit when this process exits") + logFile := flag.String("log", "", "log file (default: standard error)") + trace := flag.Bool("trace", false, "log full URLs and every IMQ frame") + version := flag.Bool("version", false, "print the version and exit") + flag.Parse() + if *version { + fmt.Println(buildinfo.Version) + return + } + + var out io.Writer = os.Stderr + if *logFile != "" { + f, err := os.Create(*logFile) + if err != nil { + os.Exit(1) + } + defer f.Close() + out = f + } + logger := log.New(out, "", log.Ldate|log.Ltime|log.Lmicroseconds) + if err := run(logger, *listen, *ready, *parentPID, *trace); err != nil { + logger.Printf("t5de-proxy: %v", err) + os.Exit(1) + } +} + +func run(logger *log.Logger, listen, ready string, parentPID int, trace bool) error { + secret := make([]byte, 24) + if _, err := rand.Read(secret); err != nil { + return err + } + token := hex.EncodeToString(secret) + + ln, err := net.Listen("tcp", listen) + if err != nil { + return err + } + logger.Printf("t5de-proxy %s listening on %s (client process %d)", buildinfo.Version, ln.Addr(), parentPID) + if ready != "" { + if err := writeReady(ready, ln.Addr().String(), token); err != nil { + return err + } + } + if parentPID != 0 { + go func() { + err := parent.Wait(parentPID) + logger.Printf("client process %d exited (%v); exiting", parentPID, err) + os.Exit(0) + }() + } + server := &http.Server{ + Handler: &proxy.Server{Token: token, Log: logger, Trace: trace}, + ReadHeaderTimeout: 30 * time.Second, + IdleTimeout: 2 * time.Minute, + ErrorLog: logger, + } + return server.Serve(ln) +} + +func writeReady(path, address, token string) error { + data, err := json.Marshal(map[string]string{"address": address, "token": token}) + if err != nil { + return err + } + tmp := path + ".tmp" + if err := os.WriteFile(tmp, data, 0o600); err != nil { + return err + } + return os.Rename(tmp, path) +} diff --git a/docs/hosts.md b/docs/hosts.md new file mode 100644 index 0000000..21b26e8 --- /dev/null +++ b/docs/hosts.md @@ -0,0 +1,38 @@ +# IMVU's hosts + +The hosts the client (554.8) reached through the proxy in a session of logging in, the home screen, the shop, +the chat room list and joining a room, and how the proxy treats them. + +## Upgraded to HTTPS + +The client asks for these over plain HTTP. Each served the same bytes over HTTPS for the paths the client used +(checked 2026-10-29: the same responses over both, and the client's session over HTTPS worked): + +| Host | What the client gets there | +| --- | --- | +| `client-dynamic.imvu.com` | XML-RPC API (`/api/xmlrpc/client.php`), avatar cards, actions, CSS for the UI | +| `chat.imvu.com` | XML-RPC chat API (`/api/xmlrpc/chat.php`) | +| `chatnow.imvu.com` | the same, for Chat Now | +| `www.imvu.com` | web pages the UI embeds | +| `userimages-akm.imvu.com`, `userimages01-akm.imvu.com` ... `05` | avatar pictures, room and product images | +| `webasset-akm.imvu.com` | resized images | +| `static-akm.imvu.com`, `static.imvu.com`, `userimages.imvu.com` | static files | + +If a request over HTTPS fails before reaching the server, the proxy retries it over HTTP. + +## Left alone + +- `api.imvu.com`, `secure.imvu.com`: the client already uses HTTPS. +- `product.virtual.imvu.com`, `program.virtual.imvu.com`: no HTTPS (the TLS connection fails). +- `imq.imvu.com:443`: IMQ, reached with CONNECT (TLS from the proxy). + +## Refused + +`bugs.imvu.com`, and on any host `/api/drv.php`, `/api/increment_counter.php`, `/api/record_gauge.php`, +`/catalog/web_submit_logs.php`, `/catalog/imvu_monitor_submit.php`, and `api.imvu.com/events`: crash reports, +logs and metrics, as T5DE's `reporting-blocklist` patch refuses them in the client. + +## Observed + +- `POST https://api.imvu.com/room/room--` answers 403 when joining someone else's room, with or + without the proxy (the client logs `HTTP Error 403` either way). diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..1f56b4c --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module git.t5de.dev/t5de/t5de-proxy + +go 1.27.0 diff --git a/internal/buildinfo/buildinfo.go b/internal/buildinfo/buildinfo.go new file mode 100644 index 0000000..354b593 --- /dev/null +++ b/internal/buildinfo/buildinfo.go @@ -0,0 +1,5 @@ +// Package buildinfo holds what release builds set with -ldflags "-X ...". +package buildinfo + +// Version is the release's version, "X.Y.Z" or "X.Y.Z-nightly.N+commit". +var Version = "dev" diff --git a/internal/imq/frame.go b/internal/imq/frame.go new file mode 100644 index 0000000..a851c59 --- /dev/null +++ b/internal/imq/frame.go @@ -0,0 +1,140 @@ +// Package imq reads the framing of IMQ, IMVU's real-time messaging protocol (chat, presence, room state). +// +// The stream is a sequence of frames, each a protobuf message with the message type in field 1 (varint) and the +// message itself in field 2 (bytes), with nothing between frames. Only the framing is decoded here: enough to log +// what goes by, by type and size, without the messages' contents. +package imq + +import ( + "errors" + "fmt" +) + +// Message types, as the client numbers them: C2g are sent by the client, G2c by the gateway. +var typeNames = map[uint64]string{ + 1301: "C2gPing", + 1302: "C2gConnect", + 1303: "C2gChallengeResponse", + 1304: "C2gSendMessage", + 1305: "C2gStateChange", + 1306: "C2gOpenFloodgates", + 1307: "C2gSuicide", + 1308: "C2gUnsubscribe", + 1309: "C2gSubscribe", + 3101: "G2cPong", + 3102: "G2cResult", + 3103: "G2cChallenge", + 3104: "G2cSendMessage", + 3105: "G2cStateChange", + 3106: "G2cJoinedQueue", + 3107: "G2cLeftQueue", + 3108: "G2cCreateMount", + 3109: "G2cConnectionClosed", +} + +// TypeName is a message type's name, or its number if it's unknown. +func TypeName(t uint64) string { + if name, ok := typeNames[t]; ok { + return name + } + return fmt.Sprintf("type %d", t) +} + +// Frame is one message: its type and the size of its data. +type Frame struct { + Type uint64 + Size int +} + +// ErrMalformed means the stream isn't IMQ framing; a Reader stops decoding after it. +var ErrMalformed = errors.New("imq: malformed frame") + +// Reader decodes frames from a stream given in pieces of any size. +type Reader struct { + buf []byte + err error +} + +// Write adds data from the stream and returns the frames it completed. +func (r *Reader) Write(data []byte) ([]Frame, error) { + if r.err != nil { + return nil, r.err + } + r.buf = append(r.buf, data...) + var frames []Frame + for { + f, n, err := parse(r.buf) + if err != nil { + r.err, r.buf = err, nil + return frames, err + } + if n == 0 { + return frames, nil + } + frames = append(frames, f) + r.buf = r.buf[n:] + } +} + +// maxFrame bounds a frame's size, so a stream that isn't IMQ isn't buffered without end. +const maxFrame = 16 << 20 + +// parse reads the frame at the start of b: the frame and its length, or a length of 0 if b holds only part of it. +func parse(b []byte) (Frame, int, error) { + var f Frame + i := 0 + for i < len(b) { + key, n := varint(b[i:]) + if n == 0 { + return f, 0, nil + } + if n < 0 { + return f, 0, ErrMalformed + } + i += n + switch key { + case 1<<3 | 0: // field 1, varint: the type + v, n := varint(b[i:]) + if n == 0 { + return f, 0, nil + } + if n < 0 { + return f, 0, ErrMalformed + } + f.Type = v + i += n + case 2<<3 | 2: // field 2, bytes: the message, the frame's last field + size, n := varint(b[i:]) + if n == 0 { + return f, 0, nil + } + if n < 0 || size > maxFrame { + return f, 0, ErrMalformed + } + i += n + if len(b)-i < int(size) { + return f, 0, nil + } + f.Size = int(size) + return f, i + int(size), nil + default: + return f, 0, ErrMalformed + } + } + return f, 0, nil +} + +// varint decodes a protobuf varint: its value and length, a length of 0 if b ends first, or -1 if it's too long. +func varint(b []byte) (uint64, int) { + var v uint64 + for i := 0; i < len(b); i++ { + if i == 10 { + return 0, -1 + } + v |= uint64(b[i]&0x7f) << (7 * i) + if b[i]&0x80 == 0 { + return v, i + 1 + } + } + return 0, 0 +} diff --git a/internal/imq/frame_test.go b/internal/imq/frame_test.go new file mode 100644 index 0000000..9f952cf --- /dev/null +++ b/internal/imq/frame_test.go @@ -0,0 +1,65 @@ +package imq + +import ( + "bytes" + "errors" + "testing" +) + +// frame encodes a frame as the client does: field 1 the type, field 2 the data. +func frame(t uint64, data []byte) []byte { + b := append([]byte{0x08}, uvarint(t)...) + b = append(b, 0x12) + b = append(b, uvarint(uint64(len(data)))...) + return append(b, data...) +} + +func uvarint(v uint64) []byte { + var b []byte + for v >= 0x80 { + b = append(b, byte(v)|0x80) + v >>= 7 + } + return append(b, byte(v)) +} + +func TestReaderSplitsFrames(t *testing.T) { + stream := append(frame(1302, []byte("hello")), frame(3103, bytes.Repeat([]byte{1}, 300))...) + stream = append(stream, frame(1301, nil)...) + // Fed a byte at a time, the frames still come out whole and in order. + var r Reader + var got []Frame + for i := range stream { + frames, err := r.Write(stream[i : i+1]) + if err != nil { + t.Fatal(err) + } + got = append(got, frames...) + } + want := []Frame{{1302, 5}, {3103, 300}, {1301, 0}} + if len(got) != len(want) { + t.Fatalf("got %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("got %v, want %v", got, want) + } + } +} + +func TestReaderRejectsOtherStreams(t *testing.T) { + var r Reader + // A TLS ClientHello isn't IMQ. + if _, err := r.Write([]byte{0x16, 0x03, 0x01, 0x00, 0xa5}); !errors.Is(err, ErrMalformed) { + t.Fatalf("got %v, want ErrMalformed", err) + } + if _, err := r.Write(frame(1301, nil)); !errors.Is(err, ErrMalformed) { + t.Fatal("a reader keeps failing after a malformed frame") + } +} + +func TestTypeName(t *testing.T) { + if TypeName(3104) != "G2cSendMessage" || TypeName(7) != "type 7" { + t.Fatal(TypeName(3104), TypeName(7)) + } +} diff --git a/internal/parent/parent.go b/internal/parent/parent.go new file mode 100644 index 0000000..6d4b85c --- /dev/null +++ b/internal/parent/parent.go @@ -0,0 +1,7 @@ +// Package parent watches the process that started the proxy, so the proxy exits with it. +package parent + +// Wait returns once the process pid has exited (at once if it already has, or can't be watched). +func Wait(pid int) error { + return wait(pid) +} diff --git a/internal/parent/parent_other.go b/internal/parent/parent_other.go new file mode 100644 index 0000000..afb9839 --- /dev/null +++ b/internal/parent/parent_other.go @@ -0,0 +1,21 @@ +//go:build !windows + +package parent + +import ( + "os" + "syscall" + "time" +) + +// wait polls: the client only runs on Windows; this is for running the proxy's tests and tools elsewhere. +func wait(pid int) error { + p, err := os.FindProcess(pid) + if err != nil { + return err + } + for p.Signal(syscall.Signal(0)) == nil { + time.Sleep(time.Second) + } + return nil +} diff --git a/internal/parent/parent_windows.go b/internal/parent/parent_windows.go new file mode 100644 index 0000000..b2c78e5 --- /dev/null +++ b/internal/parent/parent_windows.go @@ -0,0 +1,13 @@ +package parent + +import "syscall" + +func wait(pid int) error { + h, err := syscall.OpenProcess(syscall.SYNCHRONIZE, false, uint32(pid)) + if err != nil { + return err + } + defer syscall.CloseHandle(h) + _, err = syscall.WaitForSingleObject(h, syscall.INFINITE) + return err +} diff --git a/internal/proxy/connect.go b/internal/proxy/connect.go new file mode 100644 index 0000000..672c43e --- /dev/null +++ b/internal/proxy/connect.go @@ -0,0 +1,144 @@ +package proxy + +import ( + "bytes" + "crypto/tls" + "fmt" + "io" + "net" + "net/http" + "sync" + "time" + + "git.t5de.dev/t5de/t5de-proxy/internal/imq" +) + +// serveConnect connects the client's plaintext stream to host:port over TLS. +func (s *Server) serveConnect(w http.ResponseWriter, r *http.Request) { + id := s.requests.Add(1) + start := time.Now() + target := r.Host + host, _, err := net.SplitHostPort(target) + if err != nil { + http.Error(w, "t5de-proxy: expected CONNECT host:port", http.StatusBadRequest) + return + } + config := &tls.Config{} + if s.TLSConfig != nil { + config = s.TLSConfig.Clone() + } + config.ServerName = host + dial := s.Dial + if dial == nil { + dial = (&net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second}).DialContext + } + raw, err := dial(r.Context(), "tcp", target) + var upstream *tls.Conn + if err == nil { + upstream = tls.Client(raw, config) + if err = upstream.HandshakeContext(r.Context()); err != nil { + raw.Close() + } + } + if err != nil { + s.logf("#%d CONNECT %s failed after %s: %v", id, target, since(start), err) + http.Error(w, "t5de-proxy: "+err.Error(), http.StatusBadGateway) + return + } + defer upstream.Close() + + conn, buffered, err := http.NewResponseController(w).Hijack() + if err != nil { + s.logf("#%d CONNECT %s: %v", id, target, err) + return + } + defer conn.Close() + if _, err := io.WriteString(conn, "HTTP/1.1 200 Connection established\r\n\r\n"); err != nil { + return + } + s.logf("#%d CONNECT %s open (%s)", id, target, since(start)) + + // Whatever the client sent after its CONNECT and the server already read is the stream's start. + var client io.Reader = conn + if n := buffered.Reader.Buffered(); n > 0 { + head, _ := buffered.Reader.Peek(n) + client = io.MultiReader(bytes.NewReader(bytes.Clone(head)), conn) + } + + up := &stream{s: s, id: id, name: "c2g"} + down := &stream{s: s, id: id, name: "g2c"} + // IMQ is a session: when either side closes, it's over, so both connections close. + var once sync.Once + closeBoth := func() { + once.Do(func() { + conn.Close() + upstream.Close() + }) + } + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + defer closeBoth() + up.copy(upstream, client) + }() + go func() { + defer wg.Done() + defer closeBoth() + down.copy(conn, upstream) + }() + wg.Wait() + s.logf("#%d CONNECT %s closed after %s: %s, %s", id, target, since(start), up.summary(), down.summary()) +} + +// stream is one direction of a CONNECT, counting bytes and IMQ frames as they pass. +type stream struct { + s *Server + id int64 + name string + bytes int64 + frames int + other bool // not IMQ framing + reader imq.Reader +} + +func (st *stream) copy(dst io.Writer, src io.Reader) { + buf := make([]byte, 32<<10) + for { + n, err := src.Read(buf) + if n > 0 { + st.observe(buf[:n]) + if _, werr := dst.Write(buf[:n]); werr != nil { + return + } + } + if err != nil { + return + } + } +} + +func (st *stream) observe(data []byte) { + st.bytes += int64(len(data)) + if st.other { + return + } + frames, err := st.reader.Write(data) + st.frames += len(frames) + if st.s.Trace { + for _, f := range frames { + st.s.logf("#%d %s %s %dB", st.id, st.name, imq.TypeName(f.Type), f.Size) + } + } + if err != nil { + st.other = true + st.s.logf("#%d %s isn't IMQ framing; passing it through", st.id, st.name) + } +} + +func (st *stream) summary() string { + if st.other { + return fmt.Sprintf("%s %dB", st.name, st.bytes) + } + return fmt.Sprintf("%s %dB in %d frames", st.name, st.bytes, st.frames) +} diff --git a/internal/proxy/policy.go b/internal/proxy/policy.go new file mode 100644 index 0000000..c14be88 --- /dev/null +++ b/internal/proxy/policy.go @@ -0,0 +1,76 @@ +package proxy + +import ( + "net/url" + "regexp" + "strings" +) + +// upgradeHosts are IMVU hosts the client reaches over plain HTTP that serve the same over HTTPS: the proxy +// fetches them over HTTPS, so the client's account headers (X-imvu-auth, cookies) and what it browses don't cross +// the network in the clear. Checked against the client's real requests (docs/hosts.md). +var upgradeHosts = map[string]bool{ + "www.imvu.com": true, + "client-dynamic.imvu.com": true, + "secure.imvu.com": true, + "api.imvu.com": true, + "userimages.imvu.com": true, + "userimages-akm.imvu.com": true, + "static-akm.imvu.com": true, + "static.imvu.com": true, + "webasset-akm.imvu.com": true, + "chat.imvu.com": true, + "chatnow.imvu.com": true, +} + +// numberedUpgradeHosts are the numbered image hosts (userimages01-akm.imvu.com, ...), upgraded like the others. +var numberedUpgradeHosts = regexp.MustCompile(`^userimages[0-9]+-akm\.imvu\.com$`) + +// Upgrade reports whether an http URL is fetched over HTTPS instead. +func Upgrade(u *url.URL) bool { + if u.Scheme != "http" || u.Port() != "" { + return false + } + host := strings.ToLower(u.Hostname()) + return upgradeHosts[host] || numberedUpgradeHosts.MatchString(host) +} + +// blocked are IMVU's crash, error, log and metrics reporting endpoints: by host, or by path on hosts it shares +// with the API. T5DE's `reporting-blocklist` patch blocks the same in the client; the proxy blocks them again +// for anything that reaches it. +var ( + blockedHosts = []string{"bugs.imvu.com"} + blockedPaths = []string{ + "/api/drv.php", + "/api/increment_counter.php", + "/api/record_gauge.php", + "/catalog/web_submit_logs.php", + "/catalog/imvu_monitor_submit.php", + } + // blockedPrefixes are blocked on api.imvu.com only. + blockedAPIPrefixes = []string{"/events"} +) + +// Blocked reports whether a request to u is refused. +func Blocked(u *url.URL) bool { + host := strings.ToLower(u.Hostname()) + for _, h := range blockedHosts { + if host == h { + return true + } + } + path := strings.ToLower(u.EscapedPath()) + for _, p := range blockedPaths { + if path == p { + return true + } + } + if host == "api.imvu.com" { + for _, p := range blockedAPIPrefixes { + if path == p || strings.HasPrefix(path, p+"/") { + return true + } + } + } + return false +} diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go new file mode 100644 index 0000000..4757c4c --- /dev/null +++ b/internal/proxy/proxy.go @@ -0,0 +1,263 @@ +// Package proxy is T5DE's local proxy for IMVU Classic: the client sends it every HTTP request (Python's and its +// embedded browser's, which IMVU routes through Python) and its IMQ connection, and the proxy makes them with +// Go's networking: current TLS and root certificates, HTTP/2 and pooled connections, HTTPS for IMVU hosts the +// client reaches over plain HTTP, and reporting endpoints refused. +// +// Protocol (only the client's own requests are served: every one carries the token the proxy gave it): +// +// - HTTP requests in absolute form, for http and https URLs alike, over plain HTTP on 127.0.0.1 +// ("GET https://api.imvu.com/... HTTP/1.1"), with "Proxy-Authorization: Basic base64(t5de:)". +// The proxy makes the request and returns the response as is: no redirects followed, no decompression. +// - CONNECT host:port with the same header: the client then speaks plaintext, and the proxy connects to +// host:port over TLS. The client uses this for IMQ. +package proxy + +import ( + "bytes" + "context" + "crypto/subtle" + "crypto/tls" + "encoding/base64" + "errors" + "io" + "log" + "net" + "net/http" + "net/url" + "strings" + "sync/atomic" + "time" +) + +// ErrorHeader is set on the proxy's own 502 responses, when it couldn't reach the server, to the error. +const ErrorHeader = "X-T5de-Proxy-Error" + +// Server is the proxy, an http.Handler. +type Server struct { + // Token authenticates the client's requests. + Token string + // Log receives a line per request and connection. + Log *log.Logger + // Trace adds full URLs (with queries) and every IMQ frame to the log. + Trace bool + // Transport makes HTTP requests; nil is NewTransport(). + Transport http.RoundTripper + // TLSConfig is the base TLS configuration of CONNECT's upstream connections (tests set RootCAs); nil is the + // default, with the system's roots. + TLSConfig *tls.Config + // Dial makes CONNECT's upstream connections (tests point it at local servers); nil is a net.Dialer. + Dial func(ctx context.Context, network, addr string) (net.Conn, error) + + requests atomic.Int64 +} + +// NewTransport is the HTTP transport the proxy uses by default: HTTP/2 where servers offer it, pooled +// connections, and responses passed through compressed. +func NewTransport() *http.Transport { + return &http.Transport{ + DialContext: (&net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second}).DialContext, + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + MaxIdleConnsPerHost: 16, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 15 * time.Second, + ExpectContinueTimeout: time.Second, + // The client asks for gzip itself and decompresses it; leave Accept-Encoding and bodies alone. + DisableCompression: true, + } +} + +func (s *Server) transport() http.RoundTripper { + if s.Transport == nil { + s.Transport = NewTransport() + } + return s.Transport +} + +func (s *Server) logf(format string, args ...any) { + if s.Log != nil { + s.Log.Printf(format, args...) + } +} + +// authorized checks the request's Proxy-Authorization against the token. +func (s *Server) authorized(r *http.Request) bool { + auth := r.Header.Get("Proxy-Authorization") + scheme, encoded, ok := strings.Cut(auth, " ") + if !ok || !strings.EqualFold(scheme, "Basic") { + return false + } + decoded, err := base64.StdEncoding.DecodeString(strings.TrimSpace(encoded)) + if err != nil { + return false + } + user, password, _ := strings.Cut(string(decoded), ":") + return s.Token != "" && user == "t5de" && subtle.ConstantTimeCompare([]byte(password), []byte(s.Token)) == 1 +} + +func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if !s.authorized(r) { + w.Header().Set("Proxy-Authenticate", `Basic realm="t5de-proxy"`) + http.Error(w, "t5de-proxy: not authorized", http.StatusProxyAuthRequired) + return + } + if r.Method == http.MethodConnect { + s.serveConnect(w, r) + return + } + if !r.URL.IsAbs() || (r.URL.Scheme != "http" && r.URL.Scheme != "https") || r.URL.Host == "" { + http.Error(w, "t5de-proxy: expected an absolute http or https URL", http.StatusBadRequest) + return + } + s.forward(w, r) +} + +// hopHeaders are a connection's own headers, not passed on (RFC 9110 7.6.1). +var hopHeaders = []string{ + "Connection", "Proxy-Connection", "Keep-Alive", "Proxy-Authenticate", "Proxy-Authorization", + "Te", "Trailer", "Transfer-Encoding", "Upgrade", +} + +func removeHopHeaders(h http.Header) { + for _, field := range h.Values("Connection") { + for _, name := range strings.Split(field, ",") { + if name = strings.TrimSpace(name); name != "" { + h.Del(name) + } + } + } + for _, name := range hopHeaders { + h.Del(name) + } +} + +// logURL is u as logged: without its query (which can hold session values) unless tracing. +func (s *Server) logURL(u *url.URL) string { + if s.Trace || u.RawQuery == "" { + return u.String() + } + v := *u + v.RawQuery = "" + return v.String() + "?…" +} + +func (s *Server) forward(w http.ResponseWriter, r *http.Request) { + id := s.requests.Add(1) + start := time.Now() + target := *r.URL + if Blocked(&target) { + s.logf("#%d %s %s blocked", id, r.Method, s.logURL(&target)) + http.Error(w, "t5de: blocked request", http.StatusForbidden) + return + } + upgraded := Upgrade(&target) + if upgraded { + target.Scheme = "https" + } + + out := r.Clone(r.Context()) + out.RequestURI = "" + out.URL = &target + out.Host = "" + removeHopHeaders(out.Header) + if _, ok := out.Header["User-Agent"]; !ok { + out.Header.Set("User-Agent", "") // send none, as the client didn't, rather than Go's + } + replayable, err := bufferBody(out, r) + if err != nil { + s.logf("#%d %s %s: reading the request: %v", id, r.Method, s.logURL(&target), err) + http.Error(w, "t5de-proxy: "+err.Error(), http.StatusBadRequest) + return + } + + resp, err := s.transport().RoundTrip(out) + if err != nil && upgraded && replayable && r.Context().Err() == nil { + // The host didn't take HTTPS this time: fetch it as the client asked. + s.logf("#%d %s %s over HTTPS failed (%v); retrying over HTTP", id, r.Method, s.logURL(&target), err) + retry := out.Clone(r.Context()) + retry.URL = r.URL + if out.GetBody != nil { + retry.Body, _ = out.GetBody() + } + upgraded = false + resp, err = s.transport().RoundTrip(retry) + } + if err != nil { + s.logf("#%d %s %s failed after %s: %v", id, r.Method, s.logURL(&target), since(start), err) + // The header tells the client this is the proxy's error, not the server's: it raises it as a network + // error, as it would have connecting directly. + w.Header().Set(ErrorHeader, err.Error()) + http.Error(w, "t5de-proxy: "+err.Error(), http.StatusBadGateway) + return + } + defer resp.Body.Close() + + removeHopHeaders(resp.Header) + for name, values := range resp.Header { + w.Header()[name] = values + } + w.WriteHeader(resp.StatusCode) + n, err := copyFlushing(w, resp.Body) + note := "" + if upgraded { + note = " (https)" + } + if err != nil && !errors.Is(err, context.Canceled) { + note += " body error: " + err.Error() + } + s.logf("#%d %s %s %d %dB %s%s", id, r.Method, s.logURL(r.URL), resp.StatusCode, n, since(start), note) +} + +// maxBuffered bounds the request bodies the proxy holds in memory; larger ones (uploads) are streamed. +const maxBuffered = 32 << 20 + +// bufferBody reads a request's body into memory, so it can be sent again: by the transport, when a pooled +// connection turns out to have been closed by the server, and over HTTP after a failed upgrade. It reports +// whether the request can be sent again. +func bufferBody(out, in *http.Request) (bool, error) { + if in.Body == nil || in.ContentLength == 0 { + out.Body, out.GetBody, out.ContentLength = nil, nil, 0 + return true, nil + } + head, err := io.ReadAll(io.LimitReader(in.Body, maxBuffered+1)) + if err != nil { + return false, err + } + if len(head) > maxBuffered { + out.Body = io.NopCloser(io.MultiReader(bytes.NewReader(head), in.Body)) + return false, nil + } + out.ContentLength = int64(len(head)) + out.GetBody = func() (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(head)), nil } + out.Body, _ = out.GetBody() + return true, nil +} + +// copyFlushing copies a response body to the client, flushing as it arrives (long polls and streams included). +func copyFlushing(w http.ResponseWriter, body io.Reader) (int64, error) { + rc := http.NewResponseController(w) + buf := make([]byte, 32<<10) + var total int64 + for { + n, err := body.Read(buf) + if n > 0 { + if _, werr := w.Write(buf[:n]); werr != nil { + return total, werr + } + total += int64(n) + if err == nil { + _ = rc.Flush() + } + } + if err == io.EOF { + return total, nil + } + if err != nil { + return total, err + } + } +} + +func since(t time.Time) string { + return time.Since(t).Round(time.Millisecond).String() +} diff --git a/internal/proxy/proxy_test.go b/internal/proxy/proxy_test.go new file mode 100644 index 0000000..c9ae904 --- /dev/null +++ b/internal/proxy/proxy_test.go @@ -0,0 +1,342 @@ +package proxy + +import ( + "bufio" + "bytes" + "context" + "crypto/tls" + "encoding/base64" + "errors" + "fmt" + "io" + "log" + "net" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "sync" + "testing" + "time" +) + +const token = "secret" + +var auth = "Basic " + base64.StdEncoding.EncodeToString([]byte("t5de:"+token)) + +// send makes a raw request to the proxy, as the client does, and returns the response. +func send(t *testing.T, proxy *httptest.Server, raw string) *http.Response { + t.Helper() + conn, err := net.Dial("tcp", proxy.Listener.Addr().String()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { conn.Close() }) + if _, err := io.WriteString(conn, raw); err != nil { + t.Fatal(err) + } + resp, err := http.ReadResponse(bufio.NewReader(conn), nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { resp.Body.Close() }) + return resp +} + +func body(t *testing.T, resp *http.Response) string { + t.Helper() + b, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + return string(b) +} + +// logBuffer collects the proxy's log, written from its handlers' goroutines. +type logBuffer struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (b *logBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.Write(p) +} + +func (b *logBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.String() +} + +func newProxy(t *testing.T, s *Server) (*httptest.Server, *logBuffer) { + logs := &logBuffer{} + s.Token = token + s.Log = log.New(logs, "", 0) + p := httptest.NewServer(s) + t.Cleanup(p.Close) + return p, logs +} + +func TestRefusesRequestsWithoutTheToken(t *testing.T) { + origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Error("an unauthorized request reached the server") + })) + defer origin.Close() + p, _ := newProxy(t, &Server{}) + for _, header := range []string{"", "Proxy-Authorization: Basic " + base64.StdEncoding.EncodeToString([]byte("t5de:wrong")) + "\r\n"} { + resp := send(t, p, "GET "+origin.URL+"/ HTTP/1.1\r\nHost: x\r\n"+header+"\r\n") + if resp.StatusCode != http.StatusProxyAuthRequired { + t.Fatalf("got %d, want 407", resp.StatusCode) + } + } +} + +func TestForwardsAsTheClientAsked(t *testing.T) { + var got *http.Request + var gotBody string + origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + got = r + b, _ := io.ReadAll(r.Body) + gotBody = string(b) + w.Header().Set("Content-Encoding", "gzip") + w.Header().Set("Set-Cookie", "a=b") + w.Header().Set("Location", "https://example.invalid/next") + w.WriteHeader(http.StatusFound) + io.WriteString(w, "compressed bytes") + })) + defer origin.Close() + p, logs := newProxy(t, &Server{}) + resp := send(t, p, "POST "+origin.URL+"/api/xmlrpc/client.php?sid=1 HTTP/1.1\r\nHost: ignored\r\n"+ + "Proxy-Authorization: "+auth+"\r\nX-imvu-auth: abc\r\nAccept-Encoding: gzip\r\nConnection: close\r\n"+ + "Content-Length: 5\r\n\r\nhello") + // The response comes back as the server sent it: the redirect not followed, the body not decompressed. + if resp.StatusCode != http.StatusFound || resp.Header.Get("Location") != "https://example.invalid/next" || + resp.Header.Get("Content-Encoding") != "gzip" || resp.Header.Get("Set-Cookie") != "a=b" { + t.Fatalf("response: %d %v", resp.StatusCode, resp.Header) + } + if b := body(t, resp); b != "compressed bytes" { + t.Fatalf("body %q", b) + } + if got.URL.RequestURI() != "/api/xmlrpc/client.php?sid=1" || gotBody != "hello" || got.Header.Get("X-imvu-auth") != "abc" || + got.Header.Get("Accept-Encoding") != "gzip" { + t.Fatalf("request: %s %v %q", got.URL, got.Header, gotBody) + } + // Neither the proxy's token nor a user agent the client didn't send reach the server. + if got.Header.Get("Proxy-Authorization") != "" || got.Header.Get("User-Agent") != "" { + t.Fatalf("request headers: %v", got.Header) + } + if got.Host != strings.TrimPrefix(origin.URL, "http://") { + t.Fatalf("Host %q", got.Host) + } + // The log leaves out queries. + if !strings.Contains(logs.String(), "/api/xmlrpc/client.php?… 302") || strings.Contains(logs.String(), "sid=1") { + t.Fatalf("log: %s", logs) + } +} + +func TestForwardsHTTPS(t *testing.T) { + origin := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + io.WriteString(w, "secure "+r.URL.Path) + })) + defer origin.Close() + p, _ := newProxy(t, &Server{Transport: origin.Client().Transport}) + resp := send(t, p, "GET "+origin.URL+"/x HTTP/1.1\r\nHost: x\r\nProxy-Authorization: "+auth+"\r\n\r\n") + if b := body(t, resp); resp.StatusCode != 200 || b != "secure /x" { + t.Fatalf("%d %q", resp.StatusCode, b) + } +} + +// recorder is a transport that records requests and answers them itself. +type recorder struct { + mu sync.Mutex + urls []string + fail func(*http.Request) error +} + +func (r *recorder) RoundTrip(req *http.Request) (*http.Response, error) { + r.mu.Lock() + r.urls = append(r.urls, req.URL.String()) + r.mu.Unlock() + if r.fail != nil { + if err := r.fail(req); err != nil { + return nil, err + } + } + return &http.Response{StatusCode: 200, Header: http.Header{}, Body: io.NopCloser(strings.NewReader("ok")), Request: req}, nil +} + +func TestUpgradesIMVUHostsToHTTPS(t *testing.T) { + rec := &recorder{} + p, _ := newProxy(t, &Server{Transport: rec}) + for _, u := range []string{"http://client-dynamic.imvu.com/api/xmlrpc/client.php", "http://userimages03-akm.imvu.com/a.png", "http://example.com/a", "http://www.imvu.com:8080/a", "http://product.virtual.imvu.com/a"} { + body(t, send(t, p, "GET "+u+" HTTP/1.1\r\nHost: x\r\nProxy-Authorization: "+auth+"\r\n\r\n")) + } + want := []string{"https://client-dynamic.imvu.com/api/xmlrpc/client.php", "https://userimages03-akm.imvu.com/a.png", "http://example.com/a", "http://www.imvu.com:8080/a", "http://product.virtual.imvu.com/a"} + if fmt.Sprint(rec.urls) != fmt.Sprint(want) { + t.Fatalf("got %v, want %v", rec.urls, want) + } +} + +func TestFallsBackToHTTPWhenHTTPSFails(t *testing.T) { + rec := &recorder{fail: func(r *http.Request) error { + if r.URL.Scheme == "https" { + return errors.New("tls: handshake failure") + } + return nil + }} + p, _ := newProxy(t, &Server{Transport: rec}) + resp := send(t, p, "GET http://userimages.imvu.com/a.png HTTP/1.1\r\nHost: x\r\nProxy-Authorization: "+auth+"\r\n\r\n") + if b := body(t, resp); resp.StatusCode != 200 || b != "ok" { + t.Fatalf("%d %q", resp.StatusCode, b) + } + want := []string{"https://userimages.imvu.com/a.png", "http://userimages.imvu.com/a.png"} + if fmt.Sprint(rec.urls) != fmt.Sprint(want) { + t.Fatalf("got %v, want %v", rec.urls, want) + } +} + +func TestBlocksReportingEndpoints(t *testing.T) { + rec := &recorder{} + p, logs := newProxy(t, &Server{Transport: rec}) + for _, u := range []string{"http://bugs.imvu.com/x", "https://www.imvu.com/catalog/web_submit_logs.php", "https://api.imvu.com/events/1"} { + if resp := send(t, p, "POST "+u+" HTTP/1.1\r\nHost: x\r\nProxy-Authorization: "+auth+"\r\nContent-Length: 0\r\n\r\n"); resp.StatusCode != http.StatusForbidden { + t.Fatalf("%s: %d", u, resp.StatusCode) + } + } + if len(rec.urls) != 0 || strings.Count(logs.String(), "blocked") != 3 { + t.Fatalf("forwarded %v; log %s", rec.urls, logs) + } + for _, u := range []string{"https://api.imvu.com/eventsx", "https://api.imvu.com/user/1"} { + parsed, _ := url.Parse(u) + if Blocked(parsed) { + t.Fatalf("%s blocked", u) + } + } +} + +func TestRejectsOriginFormRequests(t *testing.T) { + p, _ := newProxy(t, &Server{Transport: &recorder{}}) + if resp := send(t, p, "GET /a HTTP/1.1\r\nHost: www.imvu.com\r\nProxy-Authorization: "+auth+"\r\n\r\n"); resp.StatusCode != http.StatusBadRequest { + t.Fatalf("got %d", resp.StatusCode) + } +} + +// imqFrame encodes an IMQ frame. +func imqFrame(typ uint64, data string) []byte { + b := []byte{0x08, byte(typ) | 0x80, byte(typ >> 7), 0x12, byte(len(data))} + return append(b, data...) +} + +func TestConnectSpeaksTLSUpstream(t *testing.T) { + // A TLS echo server stands in for IMQ's gateway. + gateway := httptest.NewUnstartedServer(nil) + gateway.StartTLS() + gateway.Close() // only for its certificate + cert := gateway.TLS.Certificates[0] + ln, err := tls.Listen("tcp", "127.0.0.1:0", &tls.Config{Certificates: []tls.Certificate{cert}}) + if err != nil { + t.Fatal(err) + } + defer ln.Close() + go func() { + c, err := ln.Accept() + if err != nil { + return + } + defer c.Close() + io.Copy(c, c) + }() + + roots := gateway.Client().Transport.(*http.Transport).TLSClientConfig.RootCAs + p, logs := newProxy(t, &Server{TLSConfig: &tls.Config{RootCAs: roots}, Trace: true, + Dial: func(ctx context.Context, network, addr string) (net.Conn, error) { + if addr != "example.com:443" { + return nil, fmt.Errorf("dialed %s", addr) + } + return net.Dial(network, ln.Addr().String()) + }}) + conn, err := net.Dial("tcp", p.Listener.Addr().String()) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + // The test certificate is for example.com. The client's first frame comes with its CONNECT. + hello := imqFrame(1302, "connect") + fmt.Fprintf(conn, "CONNECT example.com:443 HTTP/1.1\r\nProxy-authorization: %s\r\n\r\n%s", auth, hello) + r := bufio.NewReader(conn) + status, err := r.ReadString('\n') + if err != nil { + t.Fatalf("%v; log %s", err, logs) + } + if !strings.HasPrefix(status, "HTTP/1.1 200") { + t.Fatalf("status %q; log %s", status, logs) + } + for line, _ := r.ReadString('\n'); line != "\r\n"; line, _ = r.ReadString('\n') { + } + echo := make([]byte, len(hello)) + if _, err := io.ReadFull(r, echo); err != nil || !bytes.Equal(echo, hello) { + t.Fatalf("echo %q %v", echo, err) + } + conn.Close() + // The log names the frames each way. + for i := 0; i < 200 && !strings.Contains(logs.String(), "closed"); i++ { + time.Sleep(10 * time.Millisecond) + } + if !strings.Contains(logs.String(), "c2g C2gConnect 7B") || !strings.Contains(logs.String(), "g2c C2gConnect 7B") { + t.Fatalf("log: %s", logs) + } +} + +func TestReportsUnreachableServers(t *testing.T) { + rec := &recorder{fail: func(*http.Request) error { return errors.New("dial tcp: no such host") }} + p, _ := newProxy(t, &Server{Transport: rec}) + resp := send(t, p, "GET https://nowhere.imvu.com/ HTTP/1.1\r\nHost: x\r\nProxy-Authorization: "+auth+"\r\n\r\n") + if resp.StatusCode != http.StatusBadGateway || !strings.Contains(resp.Header.Get(ErrorHeader), "no such host") { + t.Fatalf("%d %v", resp.StatusCode, resp.Header) + } +} + +func TestBuffersBodiesToSendAgain(t *testing.T) { + // Go's transport sends a request again when the server closed a pooled connection as it went out, if it can + // get the body again (GetBody); without it, it fails with "cannot rewind body after connection loss". + in := httptest.NewRequest("POST", "http://x/", strings.NewReader("hello")) + out := in.Clone(in.Context()) + if ok, err := bufferBody(out, in); !ok || err != nil || out.GetBody == nil || out.ContentLength != 5 { + t.Fatalf("%v %v %d", ok, err, out.ContentLength) + } + for i := 0; i < 2; i++ { + body, _ := out.GetBody() + if b, _ := io.ReadAll(body); string(b) != "hello" { + t.Fatalf("body %q", b) + } + } + // Bodies over maxBuffered are streamed whole, and can't be sent again. + big := strings.Repeat("x", maxBuffered+10) + in = httptest.NewRequest("POST", "http://x/", strings.NewReader(big)) + out = in.Clone(in.Context()) + if ok, err := bufferBody(out, in); ok || err != nil || out.GetBody != nil { + t.Fatalf("%v %v", ok, err) + } + if b, _ := io.ReadAll(out.Body); len(b) != len(big) { + t.Fatalf("streamed %d bytes", len(b)) + } +} + +func TestFallsBackToHTTPWithTheBody(t *testing.T) { + var got []string + rec := &recorder{fail: func(r *http.Request) error { + b, _ := io.ReadAll(r.Body) + got = append(got, r.URL.Scheme+" "+string(b)) + if r.URL.Scheme == "https" { + return errors.New("tls: handshake failure") + } + return nil + }} + p, _ := newProxy(t, &Server{Transport: rec}) + resp := send(t, p, "POST http://client-dynamic.imvu.com/api/xmlrpc/client.php HTTP/1.1\r\nHost: x\r\nProxy-Authorization: "+auth+"\r\nContent-Length: 4\r\n\r\nbody") + if resp.StatusCode != 200 || fmt.Sprint(got) != "[https body http body]" { + t.Fatalf("%d %q", resp.StatusCode, got) + } +}