From 79c719a48a387eed7023135695ff49f13e3e47c9 Mon Sep 17 00:00:00 2001 From: Stefan Ernst <24554551+stefan-ernst@users.noreply.github.com> Date: Thu, 12 Nov 2026 22:01:43 +0100 Subject: [PATCH] fix(email): preserve inbound body formatting (WI-2129) --- go.mod | 6 +- go.sum | 17 ++- internal/email/body_markdown.go | 199 ++++++++++++++++++++++++++++++++ internal/email/processor.go | 5 +- 4 files changed, 217 insertions(+), 10 deletions(-) create mode 100644 internal/email/body_markdown.go diff --git a/go.mod b/go.mod index b94081bf7..dbda97535 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( charm.land/lipgloss/v2 v2.0.5 charm.land/wish/v2 v2.0.1 github.com/BurntSushi/toml v1.3.2 + github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.2 github.com/SherClockHolmes/webpush-go v1.4.0 github.com/allegro/bigcache/v3 v3.1.0 github.com/aws/aws-sdk-go-v2 v1.47.7 @@ -52,6 +53,7 @@ require ( github.com/zitadel/oidc/v3 v3.45.3 golang.org/x/crypto v0.58.0 golang.org/x/image v0.45.0 + golang.org/x/net v0.60.0 golang.org/x/sync v0.24.0 golang.org/x/text v0.43.0 golang.org/x/time v0.15.0 @@ -65,6 +67,7 @@ require ( cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect cloud.google.com/go/compute/metadata v0.9.0 // indirect github.com/Azure/go-ntlmssp v0.1.1 // indirect + github.com/JohannesKaufmann/dom v0.3.1 // indirect github.com/KyleBanks/depth v1.2.1 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect @@ -150,7 +153,7 @@ require ( github.com/mailru/easyjson v0.7.7 // indirect github.com/mattermost/xml-roundtrip-validator v0.1.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-isatty v0.0.22 // indirect github.com/mattn/go-runewidth v0.0.27 // indirect github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/termenv v0.16.0 // indirect @@ -197,7 +200,6 @@ require ( go.yaml.in/yaml/v2 v2.4.4 // indirect golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 // indirect golang.org/x/mod v0.41.0 // indirect - golang.org/x/net v0.60.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/sys v0.49.0 // indirect golang.org/x/tools v0.51.0 // indirect diff --git a/go.sum b/go.sum index b66000577..14db82a68 100644 --- a/go.sum +++ b/go.sum @@ -23,6 +23,10 @@ github.com/Azure/go-ntlmssp v0.1.1/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/JohannesKaufmann/dom v0.3.1 h1:J16l9JAHWgkFPR3VIPbQ1gvS0cWab6laK1q7PFL3qh0= +github.com/JohannesKaufmann/dom v0.3.1/go.mod h1:BZPkf8ZeYrBgABjwJn9iiKt8aiCtkxpHkevms+Yp2DE= +github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.2 h1:XFJZFWESIWlUEHHjzBuv8RvrtCWnSGlimEX17ysSDb8= +github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.2/go.mod h1:BHWO8lJzttJLqwuV8Rb1B3OG2OSzLbssZDI1FRg2eAA= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= @@ -93,8 +97,8 @@ github.com/beevik/etree v1.8.1 h1:MchsAnqPGCGsfQezhwcouHPlAHlcAOqWpyCVZoyWfjU= github.com/beevik/etree v1.8.1/go.mod h1:bh4zJxiIr62SOf9pRzN7UUYaEDa9HEKafK25+sLc0Gc= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bmatcuk/doublestar/v4 v4.9.2 h1:b0mc6WyRSYLjzofB2v/0cuDUZ+MqoGyH3r0dVij35GI= -github.com/bmatcuk/doublestar/v4 v4.9.2/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= +github.com/bmatcuk/doublestar/v4 v4.10.0 h1:zU9WiOla1YA122oLM6i4EXvGW62DvKZVxIe6TYWexEs= +github.com/bmatcuk/doublestar/v4 v4.10.0/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -337,8 +341,8 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4= +github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= github.com/mattn/go-runewidth v0.0.27 h1:Feg/Oou5zI/wnpgDF6omIU0OokC9GxLC/WRknhVlIR0= github.com/mattn/go-runewidth v0.0.27/go.mod h1:3qAiGCV4Koz/yuveO58qUefmUTRm8r0IGEXZ9jeHp/8= github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= @@ -398,10 +402,14 @@ github.com/sahilm/fuzzy v0.1.3 h1:juByESSS32nVD81vr6tHmKmA/8zde7gE+x5CLxrzXPU= github.com/sahilm/fuzzy v0.1.3/go.mod h1:au6//VbVSqu6DFrkL2CfjlJ5iURpNCPeE+1GwY3XsT8= github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 h1:1EYB5IzjZawrrnELUi78f9fPu57HuXjmddZPjrls/28= github.com/santhosh-tekuri/jsonschema/v6 v6.0.3/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= +github.com/sebdah/goldie/v2 v2.8.0 h1:dZb9wR8q5++oplmEiJT+U/5KyotVD+HNGCAc5gNr8rc= +github.com/sebdah/goldie/v2 v2.8.0/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI= github.com/segmentio/asm v1.1.3 h1:WM03sfUOENvvKexOLp+pCqgb/WDjsi7EK8gIsICtzhc= github.com/segmentio/asm v1.1.3/go.mod h1:Ld3L4ZXGNcSLRg4JBsZ3//1+f/TjYl0Mzen/DQy1EJg= github.com/segmentio/encoding v0.5.4 h1:OW1VRern8Nw6ITAtwSZ7Idrl3MXCFwXHPgqESYfvNt0= github.com/segmentio/encoding v0.5.4/go.mod h1:HS1ZKa3kSN32ZHVZ7ZLPLXWvOVIiZtyJnO1gPH1sKt0= +github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw= +github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= @@ -538,7 +546,6 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= diff --git a/internal/email/body_markdown.go b/internal/email/body_markdown.go new file mode 100644 index 000000000..4b680a068 --- /dev/null +++ b/internal/email/body_markdown.go @@ -0,0 +1,199 @@ +package email + +import ( + "fmt" + "html" + "log/slog" + "regexp" + "strings" + "unicode/utf8" + + "github.com/JohannesKaufmann/html-to-markdown/v2/converter" + "github.com/JohannesKaufmann/html-to-markdown/v2/plugin/base" + "github.com/JohannesKaufmann/html-to-markdown/v2/plugin/commonmark" + "github.com/JohannesKaufmann/html-to-markdown/v2/plugin/table" + "github.com/microcosm-cc/bluemonday" + "github.com/yuin/goldmark" + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/text" + nethtml "golang.org/x/net/html" +) + +const ( + maxEmailHTMLBytes = 1024 * 1024 + minFidelityRunes = 100 + minHTMLFraction = 4 // Fall back below one quarter of the plain-text length. +) + +var ( + emailHTMLPolicy = newEmailHTMLPolicy() + plainMarkdownEscapes = strings.NewReplacer( + `\`, `\\`, "`", "\\`", "*", `\*`, "_", `\_`, + "[", `\[`, "]", `\]`, "<", `\<`, ">", `\>`, "#", `\#`, "|", `\|`, "~", `\~`, + ) + plainBlockMarker = regexp.MustCompile(`(?m)^([ \t]*)([-+])([ \t]+)`) + plainOrderedMarker = regexp.MustCompile(`(?m)^([ \t]*\d{1,9})([.)])([ \t]+)`) + plainDivider = regexp.MustCompile(`(?m)^([ \t]*)([-=]{2,})([ \t]*)$`) +) + +func newEmailHTMLPolicy() *bluemonday.Policy { + p := bluemonday.NewPolicy() + p.AllowElements("p", "div", "span", "br", "strong", "b", "em", "i", "s", "del", + "h1", "h2", "h3", "h4", "h5", "h6", "ul", "ol", "li", "blockquote", "pre", "code", "hr", + "table", "thead", "tbody", "tfoot", "tr", "th", "td", "caption", "a") + p.AllowAttrs("href").OnElements("a") + p.AllowAttrs("start").Matching(regexp.MustCompile(`^\d{1,9}$`)).OnElements("ol") + p.AllowAttrs("role").Matching(regexp.MustCompile(`^presentation$`)).OnElements("table") + p.RequireParseableURLs(true) + p.AllowURLSchemes("http", "https", "mailto", "tel") + return p +} + +// bodyMarkdown selects the MIME alternative before cleaning quotes or signatures. +func (e *ParsedEmail) bodyMarkdown(reply bool) string { + if e.HTMLBody != "" { + markdown, visible, err := emailHTMLMarkdown(e.HTMLBody) + plainLength := normalizedBodyLength(e.PlainBody) + htmlLength := normalizedBodyLength(visible) + switch { + case err != nil: + slog.Warn("email HTML conversion failed, using text fallback", "message_id", e.MessageID, "error", err) + case htmlLength == 0: + // Image-only and tracking-only alternatives have no message text. + case plainLength >= minFidelityRunes && htmlLength*minHTMLFraction < plainLength: + slog.Debug("email HTML alternative is incomplete, using plain text", "message_id", e.MessageID) + default: + return cleanEmailMarkdown(markdown, reply) + } + } + + body := e.PlainBody + if strings.TrimSpace(body) == "" && e.HTMLBody != "" { + // Keep an oversized or otherwise unconvertible HTML-only message readable. + body = html.UnescapeString(StripHTML(e.HTMLBody[:min(len(e.HTMLBody), maxEmailHTMLBytes)])) + } + if strings.TrimSpace(body) == "" { + return "" + } + body = strings.ReplaceAll(strings.ReplaceAll(body, "\r\n", "\n"), "\r", "\n") + if reply { + body = ExtractReplyContent(body) + } + body = StripSignature(body) + body = plainMarkdownEscapes.Replace(body) + body = plainBlockMarker.ReplaceAllString(body, "$1\\$2$3") + body = plainOrderedMarker.ReplaceAllString(body, "$1\\$2$3") + body = plainDivider.ReplaceAllString(body, "$1\\$2$3") + lines := strings.Split(body, "\n") + for i := 0; i+1 < len(lines); i++ { + if strings.TrimSpace(lines[i]) != "" && strings.TrimSpace(lines[i+1]) != "" { + lines[i] = strings.TrimRight(lines[i], " \t") + " " + } + } + return strings.Join(lines, "\n") +} + +func normalizedBodyLength(body string) int { + return utf8.RuneCountInString(strings.Join(strings.Fields(body), " ")) +} + +func emailHTMLMarkdown(body string) (markdown, visible string, err error) { + if len(body) > maxEmailHTMLBytes { + return "", "", fmt.Errorf("HTML body exceeds %d bytes", maxEmailHTMLBytes) + } + safeHTML := emailHTMLPolicy.Sanitize(body) + visible = html.UnescapeString(StripHTML(safeHTML)) + doc, err := nethtml.Parse(strings.NewReader(safeHTML)) + if err != nil { + return "", "", fmt.Errorf("parse email HTML: %w", err) + } + flattenEmailLayoutTables(doc) + conv := converter.NewConverter(converter.WithPlugins( + base.NewBasePlugin(), commonmark.NewCommonmarkPlugin(), table.NewTablePlugin(), + )) + converted, err := conv.ConvertNode(doc) + if err != nil { + return "", "", fmt.Errorf("convert email HTML: %w", err) + } + return string(converted), visible, nil +} + +// Only tables with explicit headers are treated as data tables. +func flattenEmailLayoutTables(node *nethtml.Node) { + if node.Type == nethtml.ElementNode && node.Data == "table" { + layout := !emailTableHasHeader(node) + for _, attr := range node.Attr { + if attr.Key == "role" && attr.Val == "presentation" { + layout = true + } + } + if layout { + flattenEmailTable(node, true) + } + } + for child := node.FirstChild; child != nil; child = child.NextSibling { + flattenEmailLayoutTables(child) + } +} + +func emailTableHasHeader(node *nethtml.Node) bool { + for child := node.FirstChild; child != nil; child = child.NextSibling { + if child.Type == nethtml.ElementNode && child.Data == "table" { + continue + } + if child.Data == "th" || emailTableHasHeader(child) { + return true + } + } + return false +} + +func flattenEmailTable(node *nethtml.Node, root bool) { + if !root && node.Type == nethtml.ElementNode && node.Data == "table" { + return + } + if node.Type == nethtml.ElementNode { + switch node.Data { + case "table", "thead", "tbody", "tfoot", "tr", "th", "td", "caption": + node.Data = "div" + node.DataAtom = 0 + } + } + for child := node.FirstChild; child != nil; child = child.NextSibling { + flattenEmailTable(child, false) + } +} + +// Hide code-block contents from the existing line-based email cleanup. +func cleanEmailMarkdown(body string, reply bool) string { + source := []byte(body) + doc := goldmark.DefaultParser().Parse(text.NewReader(source)) + type codeRange struct{ start, end int } + var ranges []codeRange + _ = ast.Walk(doc, func(node ast.Node, entering bool) (ast.WalkStatus, error) { + if entering && (node.Kind() == ast.KindFencedCodeBlock || node.Kind() == ast.KindCodeBlock) && node.Lines().Len() > 0 { + lines := node.Lines() + ranges = append(ranges, codeRange{lines.At(0).Start, lines.At(lines.Len() - 1).Stop}) + } + return ast.WalkContinue, nil + }) + prefix := "WINDSHIFT_EMAIL_CODE_" + for strings.Contains(body, prefix) { + prefix += "_" + } + restore := make([]string, 0, len(ranges)*2) + for i := len(ranges) - 1; i >= 0; i-- { + r := ranges[i] + placeholder := fmt.Sprintf("%s%d\n", prefix, i) + restore = append(restore, placeholder, body[r.start:r.end]) + body = body[:r.start] + placeholder + body[r.end:] + } + if reply { + body = ExtractReplyContent(body) + } + body = StripSignature(body) + if len(restore) > 0 { + body = strings.NewReplacer(restore...).Replace(body) + } + return body +} diff --git a/internal/email/processor.go b/internal/email/processor.go index a871f8f1a..67924b14d 100644 --- a/internal/email/processor.go +++ b/internal/email/processor.go @@ -633,7 +633,7 @@ func (p *Processor) createItemFromEmail( params := services.ItemCreationParams{ WorkspaceID: workspaceID, Title: sanitize.PlainTextField.Sanitize(email.GetSubjectForItem()), - Description: sanitize.Comment.Sanitize(StripSignature(email.GetBodyText())), + Description: sanitize.Comment.Sanitize(email.bodyMarkdown(false)), ItemTypeID: itemTypeID, PriorityID: config.EmailDefaultPriorityID, RequestTypeID: requestTypeID, @@ -716,8 +716,7 @@ func (p *Processor) addCommentFromReply( itemID int, customerID int, ) (*ProcessingResult, error) { - // Extract reply content (strip quoted text) - content := StripSignature(ExtractReplyContent(email.GetBodyText())) + content := email.bodyMarkdown(true) if strings.TrimSpace(content) == "" { // No new content - skip