Add pack, cfl and safehtml
pack loads local product packs (format 1): t5de-product.json and the product's files, as product.chkn or files/, with index.xml's rating, type, gender and parent product. cfl writes and reads IMVU's product container, uncompressed. safehtml cleans product descriptions by an allowlist (bluemonday). Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
dhkatzandClaude Opus 5.5 committed
commit
9bdc3ff455
16 files changed
+1638
No files matched your search
@@ -0,0 +1,141 @@
|
||||
// Package safehtml makes product descriptions safe to show anywhere, including the client's privileged pages.
|
||||
//
|
||||
// Descriptions are cleaned by an allowlist: text formatting, lists, simple tables, headings, links and images,
|
||||
// and a few styles with checked values. Everything else is removed.
|
||||
package safehtml
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/microcosm-cc/bluemonday"
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
// MaxDescription is the most bytes a description may have. Description doesn't check it: callers validate
|
||||
// descriptions when they're saved.
|
||||
const MaxDescription = 20000
|
||||
|
||||
// LinkRel is the rel attribute every link gets.
|
||||
const LinkRel = "nofollow ugc noopener noreferrer"
|
||||
|
||||
var (
|
||||
color = `#[0-9a-f]{3,4}|#[0-9a-f]{6}|#[0-9a-f]{8}|[a-z]{1,30}|rgba?\(\s*\d{1,3}%?\s*,\s*\d{1,3}%?\s*,\s*\d{1,3}%?\s*(,\s*(0|1|0?\.\d{1,3})\s*)?\)`
|
||||
colorValue = regexp.MustCompile(`^(?i)(` + color + `)$`)
|
||||
// font's color attribute: CSS colors, and hex without '#', which browsers also take.
|
||||
fontColor = regexp.MustCompile(`^(?i)(` + color + `|[0-9a-f]{6})$`)
|
||||
fontSize = regexp.MustCompile(`^[+-]?[1-7]$`)
|
||||
cssFontSize = regexp.MustCompile(`^(?i)(\d{1,3}(\.\d{1,2})?(px|pt|em|%)|0|xx-small|x-small|small|medium|large|x-large|xx-large|smaller|larger)$`)
|
||||
fontWeight = regexp.MustCompile(`^(?i)(normal|bold|bolder|lighter|[1-9]00)$`)
|
||||
fontStyle = regexp.MustCompile(`^(?i)(normal|italic)$`)
|
||||
textAlign = regexp.MustCompile(`^(?i)(left|right|center|justify)$`)
|
||||
textDecoration = regexp.MustCompile(`^(?i)(none|underline|line-through)$`)
|
||||
smallInt = regexp.MustCompile(`^[1-9][0-9]{0,3}$`)
|
||||
)
|
||||
|
||||
var policy = newPolicy()
|
||||
|
||||
func newPolicy() *bluemonday.Policy {
|
||||
p := bluemonday.NewPolicy()
|
||||
p.AllowElements("b", "i", "u", "s", "em", "strong", "small", "big", "sub", "sup", "font", "span", "div", "p",
|
||||
"br", "hr", "center", "ul", "ol", "li", "table", "thead", "tbody", "tr", "td", "th",
|
||||
"h1", "h2", "h3", "h4", "h5", "h6", "a", "img", "blockquote")
|
||||
// bluemonday drops elements without attributes unless they are on its list, which lacks big.
|
||||
p.AllowNoAttrs().OnElements("big")
|
||||
|
||||
p.AllowAttrs("color").Matching(fontColor).OnElements("font")
|
||||
p.AllowAttrs("size").Matching(fontSize).OnElements("font")
|
||||
p.AllowAttrs("colspan", "rowspan").Matching(smallInt).OnElements("td", "th")
|
||||
|
||||
p.AllowAttrs("href", "title").OnElements("a")
|
||||
p.AllowAttrs("src", "alt", "title").OnElements("img")
|
||||
p.AllowAttrs("width", "height").Matching(smallInt).OnElements("img")
|
||||
p.AllowURLSchemes("http", "https")
|
||||
p.RequireParseableURLs(true)
|
||||
p.AllowRelativeURLs(false)
|
||||
|
||||
p.AllowStyles("color", "background-color").Matching(colorValue).Globally()
|
||||
p.AllowStyles("font-size").Matching(cssFontSize).Globally()
|
||||
p.AllowStyles("font-weight").Matching(fontWeight).Globally()
|
||||
p.AllowStyles("font-style").Matching(fontStyle).Globally()
|
||||
p.AllowStyles("text-align").Matching(textAlign).Globally()
|
||||
p.AllowStyles("text-decoration").Matching(textDecoration).Globally()
|
||||
return p
|
||||
}
|
||||
|
||||
// Description returns description cleaned by the allowlist. Images keep their https addresses.
|
||||
func Description(description string) string {
|
||||
return DescriptionWithImages(description, nil)
|
||||
}
|
||||
|
||||
// DescriptionWithImages is Description with each image's https address passed to rewrite, which returns the
|
||||
// address to show instead, or false to drop the image. A nil rewrite keeps the addresses.
|
||||
func DescriptionWithImages(description string, rewrite func(src string) (string, bool)) string {
|
||||
clean := policy.Sanitize(description)
|
||||
|
||||
// bluemonday has removed everything unsafe; this pass only adds fixed link attributes and checks images.
|
||||
var b strings.Builder
|
||||
z := html.NewTokenizer(strings.NewReader(clean))
|
||||
for {
|
||||
if z.Next() == html.ErrorToken {
|
||||
return b.String()
|
||||
}
|
||||
t := z.Token()
|
||||
if t.Type == html.StartTagToken || t.Type == html.SelfClosingTagToken {
|
||||
switch t.Data {
|
||||
case "a":
|
||||
t.Attr = linkAttrs(t.Attr)
|
||||
case "img":
|
||||
var ok bool
|
||||
if t.Attr, ok = imageAttrs(t.Attr, rewrite); !ok {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
b.WriteString(t.String())
|
||||
}
|
||||
}
|
||||
|
||||
func linkAttrs(attrs []html.Attribute) []html.Attribute {
|
||||
for _, a := range attrs {
|
||||
if a.Key == "href" {
|
||||
return append(attrs,
|
||||
html.Attribute{Key: "rel", Val: LinkRel},
|
||||
html.Attribute{Key: "target", Val: "_blank"})
|
||||
}
|
||||
}
|
||||
return attrs
|
||||
}
|
||||
|
||||
func imageAttrs(attrs []html.Attribute, rewrite func(string) (string, bool)) ([]html.Attribute, bool) {
|
||||
for i, a := range attrs {
|
||||
if a.Key != "src" {
|
||||
continue
|
||||
}
|
||||
if !isHTTPS(a.Val) {
|
||||
return nil, false
|
||||
}
|
||||
if rewrite != nil {
|
||||
src, ok := rewrite(a.Val)
|
||||
if !ok || !isWebURL(src) {
|
||||
return nil, false
|
||||
}
|
||||
attrs[i].Val = src
|
||||
}
|
||||
return attrs, true
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
func isHTTPS(s string) bool {
|
||||
u, err := url.Parse(s)
|
||||
return err == nil && u.Scheme == "https" && u.Host != ""
|
||||
}
|
||||
|
||||
// isWebURL checks a rewritten image address: the image proxy can be http on the local machine.
|
||||
func isWebURL(s string) bool {
|
||||
u, err := url.Parse(s)
|
||||
return err == nil && (u.Scheme == "https" || u.Scheme == "http") && u.Host != "" &&
|
||||
!strings.ContainsAny(s, " \t\r\n")
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
package safehtml
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDescription(t *testing.T) {
|
||||
const rel = ` rel="nofollow ugc noopener noreferrer" target="_blank"`
|
||||
for _, c := range []struct{ in, want string }{
|
||||
// Kept.
|
||||
{`<font color="#ff0000" size="4"><b>Hi</b></font><br><center>Made by me</center>`,
|
||||
`<font color="#ff0000" size="4"><b>Hi</b></font><br><center>Made by me</center>`},
|
||||
{`<font color="ff0000" size="+2">x</font>`, `<font color="ff0000" size="+2">x</font>`},
|
||||
{`<p style="color: red; text-align: center; font-weight: bold">x</p>`,
|
||||
`<p style="color: red; text-align: center; font-weight: bold">x</p>`},
|
||||
{`<span style="background-color: rgb(1, 2, 3); font-size: 12px; font-style: italic; text-decoration: underline">x</span>`,
|
||||
`<span style="background-color: rgb(1, 2, 3); font-size: 12px; font-style: italic; text-decoration: underline">x</span>`},
|
||||
{`<a href="https://example.com/a?b=c&d=e">x</a>`, `<a href="https://example.com/a?b=c&d=e"` + rel + `>x</a>`},
|
||||
{`<a href="http://example.com/">x</a>`, `<a href="http://example.com/"` + rel + `>x</a>`},
|
||||
{`<img src="https://example.com/a.png" alt="A" width="100">`, `<img src="https://example.com/a.png" alt="A" width="100">`},
|
||||
{`<table><tbody><tr><td colspan="2">x</td></tr></tbody></table>`, `<table><tbody><tr><td colspan="2">x</td></tr></tbody></table>`},
|
||||
{`<ul><li>a</li></ul><h1>b</h1><blockquote>c</blockquote><big>d</big><hr>`,
|
||||
`<ul><li>a</li></ul><h1>b</h1><blockquote>c</blockquote><big>d</big><hr>`},
|
||||
{`5 < 6 & "quotes"`, `5 < 6 & "quotes"`},
|
||||
|
||||
// Removed.
|
||||
{`<script>alert(1)</script>x`, `x`},
|
||||
{`<SCRIPT SRC=https://e.com/x.js></SCRIPT>x`, `x`},
|
||||
{`<scr<script>ipt>alert(1)</script>`, `ipt>alert(1)`},
|
||||
{`<img src=x onerror=alert(1)>`, ``},
|
||||
{`<img src="https://e.com/a.png" onerror="alert(1)">`, `<img src="https://e.com/a.png">`},
|
||||
{`<b onclick="alert(1)" onmouseover=alert(1)>x</b>`, `<b>x</b>`},
|
||||
{`<a href="javascript:alert(1)">x</a>`, `x`},
|
||||
{`<a href=" javascript:alert(1)">x</a>`, `x`},
|
||||
{`<a href="JaVaScRiPt:alert(1)">x</a>`, `x`},
|
||||
{`<a href="javascript:alert(1)">x</a>`, `x`},
|
||||
{`<a href="javascript:alert(1)">x</a>`, `x`},
|
||||
{`<a href="java	script:alert(1)">x</a>`, `x`},
|
||||
{`<a href="vbscript:msgbox(1)">x</a>`, `x`},
|
||||
{`<a href="data:text/html;base64,PHNjcmlwdD5hbGVydCgxKTwvc2NyaXB0Pg==">x</a>`, `x`},
|
||||
{`<a href="chrome://browser/content/browser.xul">x</a>`, `x`},
|
||||
{`<a href="/relative">x</a>`, `x`},
|
||||
{`<a href="https://e.com/" target="_self" rel="opener">x</a>`, `<a href="https://e.com/"` + rel + `>x</a>`},
|
||||
{`<img src="javascript:alert(1)">`, ``},
|
||||
{`<img src="http://e.com/a.png">`, ``},
|
||||
{`<img src="data:image/png;base64,iVBORw0KGgo=">`, ``},
|
||||
{`<img alt="no src">`, ``},
|
||||
{`<p style="background-color: url(https://e.com/x.png)">x</p>`, `<p>x</p>`},
|
||||
{`<p style="background: url(https://e.com/x.png)">x</p>`, `<p>x</p>`},
|
||||
{`<p style="color: expression(alert(1))">x</p>`, `<p>x</p>`},
|
||||
{`<p style="color: red; position: fixed; top: 0; width: 100%">x</p>`, `<p style="color: red">x</p>`},
|
||||
{`<p style="font-size: 99999px">x</p>`, `<p>x</p>`},
|
||||
{`<font color="red" face="Arial" style="color: blue">x</font>`, `<font color="red" style="color: blue">x</font>`},
|
||||
{`<style>body{display:none}</style>x`, `x`},
|
||||
{`<link rel="stylesheet" href="https://e.com/x.css">x`, `x`},
|
||||
{`<iframe src="https://e.com/"></iframe>x`, `x`},
|
||||
{`<object data="x.swf"></object><embed src="x.swf">x`, `x`},
|
||||
{`<form action="https://e.com/"><input name="p"></form>x`, `x`},
|
||||
{`<!-- a comment -->x<!--[if IE]><script>alert(1)</script><![endif]-->`, `x`},
|
||||
{`<svg onload="alert(1)"><script>alert(1)</script></svg>x`, `x`},
|
||||
{`<math><mi xlink:href="javascript:alert(1)">x</mi></math>`, `x`},
|
||||
{`<div><p><b>unclosed`, `<div><p><b>unclosed`},
|
||||
{`<b><i>misnested</b></i>`, `<b><i>misnested</b></i>`},
|
||||
{`<a href="https://e.com/"<script>alert(1)</script>>x</a>`, `<a href="https://e.com/"` + rel + `>alert(1)>x</a>`},
|
||||
{`<img src="https://e.com/a.png" width="100%" height="-1">`, `<img src="https://e.com/a.png">`},
|
||||
{`<td rowspan="0">x</td>`, `<td>x</td>`},
|
||||
} {
|
||||
if got := Description(c.in); got != c.want {
|
||||
t.Errorf("Description(%q)\n got %q\nwant %q", c.in, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoScriptAnywhere(t *testing.T) {
|
||||
for _, in := range []string{
|
||||
`<<script>script>alert(1)<</script>/script>`,
|
||||
`<a href="https://e.com/" style="x:expression(alert(1))">x</a>`,
|
||||
`<img src="https://e.com/a.png"/onerror=alert(1)>`,
|
||||
`<IMG SRC="jav
ascript:alert('XSS');">`,
|
||||
`<a href="https://e.com/" onclick=alert(1)//>x</a>`,
|
||||
`<base href="javascript:alert(1)//">`,
|
||||
`<meta http-equiv="refresh" content="0;url=javascript:alert(1)">`,
|
||||
} {
|
||||
got := strings.ToLower(Description(in))
|
||||
for _, bad := range []string{"<script", "javascript:", "onerror", "onclick", "expression", "<base", "<meta"} {
|
||||
if strings.Contains(got, bad) {
|
||||
t.Errorf("Description(%q) = %q, contains %q", in, got, bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDescriptionWithImages(t *testing.T) {
|
||||
rewrite := func(src string) (string, bool) {
|
||||
if strings.Contains(src, "drop") {
|
||||
return "", false
|
||||
}
|
||||
if strings.Contains(src, "evil") {
|
||||
return "javascript:alert(1)", true
|
||||
}
|
||||
return "https://images.t5de.dev/?u=" + src, true
|
||||
}
|
||||
for _, c := range []struct{ in, want string }{
|
||||
{`<img src="https://e.com/a.png">`, `<img src="https://images.t5de.dev/?u=https://e.com/a.png">`},
|
||||
{`<img src="https://e.com/drop.png">x`, `x`},
|
||||
{`<img src="https://e.com/evil.png">x`, `x`},
|
||||
{`<img src="http://e.com/a.png">x`, `x`},
|
||||
} {
|
||||
if got := DescriptionWithImages(c.in, rewrite); got != c.want {
|
||||
t.Errorf("DescriptionWithImages(%q)\n got %q\nwant %q", c.in, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user