Skip to content

fix(email): preserve inbound body formatting (WI-2129) #495

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
stefan-ernst merged 1 commit into from
Nov 12, 2026
+217 −10
Merged

fix(email): preserve inbound body formatting (WI-2129) #495

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ require (
charm.land/lipgloss/v2 v2.0.5
charm.land/wish/v2 v2.0.1
github.com/BurntSushi/toml v1.3.2
github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.2
github.com/SherClockHolmes/webpush-go v1.4.0
github.com/allegro/bigcache/v3 v3.1.0
github.com/aws/aws-sdk-go-v2 v1.47.7
Expand Down Expand Up @@ -52,6 +53,7 @@ require (
github.com/zitadel/oidc/v3 v3.45.3
golang.org/x/crypto v0.58.0
golang.org/x/image v0.45.0
golang.org/x/net v0.60.0
golang.org/x/sync v0.24.0
golang.org/x/text v0.43.0
golang.org/x/time v0.15.0
Expand All @@ -65,6 +67,7 @@ require (
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
cloud.google.com/go/compute/metadata v0.9.0 // indirect
github.com/Azure/go-ntlmssp v0.1.1 // indirect
github.com/JohannesKaufmann/dom v0.3.1 // indirect
github.com/KyleBanks/depth v1.2.1 // indirect
github.com/PuerkitoBio/purell v1.1.1 // indirect
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
Expand Down Expand Up @@ -150,7 +153,7 @@ require (
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattermost/xml-roundtrip-validator v0.1.0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-isatty v0.0.22 // indirect
github.com/mattn/go-runewidth v0.0.27 // indirect
github.com/muesli/cancelreader v0.2.2 // indirect
github.com/muesli/termenv v0.16.0 // indirect
Expand Down Expand Up @@ -197,7 +200,6 @@ require (
go.yaml.in/yaml/v2 v2.4.4 // indirect
golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 // indirect
golang.org/x/mod v0.41.0 // indirect
golang.org/x/net v0.60.0 // indirect
golang.org/x/oauth2 v0.36.0 // indirect
golang.org/x/sys v0.49.0 // indirect
golang.org/x/tools v0.51.0 // indirect
Expand Down
17 changes: 12 additions & 5 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,10 @@ github.com/Azure/go-ntlmssp v0.1.1/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8=
github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/JohannesKaufmann/dom v0.3.1 h1:J16l9JAHWgkFPR3VIPbQ1gvS0cWab6laK1q7PFL3qh0=
github.com/JohannesKaufmann/dom v0.3.1/go.mod h1:BZPkf8ZeYrBgABjwJn9iiKt8aiCtkxpHkevms+Yp2DE=
github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.2 h1:XFJZFWESIWlUEHHjzBuv8RvrtCWnSGlimEX17ysSDb8=
github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.2/go.mod h1:BHWO8lJzttJLqwuV8Rb1B3OG2OSzLbssZDI1FRg2eAA=
github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc=
github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE=
github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ=
Expand Down Expand Up @@ -93,8 +97,8 @@ github.com/beevik/etree v1.8.1 h1:MchsAnqPGCGsfQezhwcouHPlAHlcAOqWpyCVZoyWfjU=
github.com/beevik/etree v1.8.1/go.mod h1:bh4zJxiIr62SOf9pRzN7UUYaEDa9HEKafK25+sLc0Gc=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bmatcuk/doublestar/v4 v4.9.2 h1:b0mc6WyRSYLjzofB2v/0cuDUZ+MqoGyH3r0dVij35GI=
github.com/bmatcuk/doublestar/v4 v4.9.2/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
github.com/bmatcuk/doublestar/v4 v4.10.0 h1:zU9WiOla1YA122oLM6i4EXvGW62DvKZVxIe6TYWexEs=
github.com/bmatcuk/doublestar/v4 v4.10.0/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
Expand Down Expand Up @@ -337,8 +341,8 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4=
github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4=
github.com/mattn/go-runewidth v0.0.27 h1:Feg/Oou5zI/wnpgDF6omIU0OokC9GxLC/WRknhVlIR0=
github.com/mattn/go-runewidth v0.0.27/go.mod h1:3qAiGCV4Koz/yuveO58qUefmUTRm8r0IGEXZ9jeHp/8=
github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk=
Expand Down Expand Up @@ -398,10 +402,14 @@ github.com/sahilm/fuzzy v0.1.3 h1:juByESSS32nVD81vr6tHmKmA/8zde7gE+x5CLxrzXPU=
github.com/sahilm/fuzzy v0.1.3/go.mod h1:au6//VbVSqu6DFrkL2CfjlJ5iURpNCPeE+1GwY3XsT8=
github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 h1:1EYB5IzjZawrrnELUi78f9fPu57HuXjmddZPjrls/28=
github.com/santhosh-tekuri/jsonschema/v6 v6.0.3/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
github.com/sebdah/goldie/v2 v2.8.0 h1:dZb9wR8q5++oplmEiJT+U/5KyotVD+HNGCAc5gNr8rc=
github.com/sebdah/goldie/v2 v2.8.0/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI=
github.com/segmentio/asm v1.1.3 h1:WM03sfUOENvvKexOLp+pCqgb/WDjsi7EK8gIsICtzhc=
github.com/segmentio/asm v1.1.3/go.mod h1:Ld3L4ZXGNcSLRg4JBsZ3//1+f/TjYl0Mzen/DQy1EJg=
github.com/segmentio/encoding v0.5.4 h1:OW1VRern8Nw6ITAtwSZ7Idrl3MXCFwXHPgqESYfvNt0=
github.com/segmentio/encoding v0.5.4/go.mod h1:HS1ZKa3kSN32ZHVZ7ZLPLXWvOVIiZtyJnO1gPH1sKt0=
github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw=
github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
Expand Down Expand Up @@ -538,7 +546,6 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
Expand Down
199 changes: 199 additions & 0 deletions internal/email/body_markdown.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,199 @@
package email

import (
"fmt"
"html"
"log/slog"
"regexp"
"strings"
"unicode/utf8"

"github.com/JohannesKaufmann/html-to-markdown/v2/converter"
"github.com/JohannesKaufmann/html-to-markdown/v2/plugin/base"
"github.com/JohannesKaufmann/html-to-markdown/v2/plugin/commonmark"
"github.com/JohannesKaufmann/html-to-markdown/v2/plugin/table"
"github.com/microcosm-cc/bluemonday"
"github.com/yuin/goldmark"
"github.com/yuin/goldmark/ast"
"github.com/yuin/goldmark/text"
nethtml "golang.org/x/net/html"
)

const (
maxEmailHTMLBytes = 1024 * 1024
minFidelityRunes = 100
minHTMLFraction = 4 // Fall back below one quarter of the plain-text length.
)

var (
emailHTMLPolicy = newEmailHTMLPolicy()
plainMarkdownEscapes = strings.NewReplacer(
`\`, `\\`, "`", "\\`", "*", `\*`, "_", `\_`,
"[", `\[`, "]", `\]`, "<", `\<`, ">", `\>`, "#", `\#`, "|", `\|`, "~", `\~`,
)
plainBlockMarker = regexp.MustCompile(`(?m)^([ \t]*)([-+])([ \t]+)`)
plainOrderedMarker = regexp.MustCompile(`(?m)^([ \t]*\d{1,9})([.)])([ \t]+)`)
plainDivider = regexp.MustCompile(`(?m)^([ \t]*)([-=]{2,})([ \t]*)$`)
)

func newEmailHTMLPolicy() *bluemonday.Policy {
p := bluemonday.NewPolicy()
p.AllowElements("p", "div", "span", "br", "strong", "b", "em", "i", "s", "del",
"h1", "h2", "h3", "h4", "h5", "h6", "ul", "ol", "li", "blockquote", "pre", "code", "hr",
"table", "thead", "tbody", "tfoot", "tr", "th", "td", "caption", "a")
p.AllowAttrs("href").OnElements("a")
p.AllowAttrs("start").Matching(regexp.MustCompile(`^\d{1,9}$`)).OnElements("ol")
p.AllowAttrs("role").Matching(regexp.MustCompile(`^presentation$`)).OnElements("table")
p.RequireParseableURLs(true)
p.AllowURLSchemes("http", "https", "mailto", "tel")
return p
}

// bodyMarkdown selects the MIME alternative before cleaning quotes or signatures.
func (e *ParsedEmail) bodyMarkdown(reply bool) string {
if e.HTMLBody != "" {
markdown, visible, err := emailHTMLMarkdown(e.HTMLBody)
plainLength := normalizedBodyLength(e.PlainBody)
htmlLength := normalizedBodyLength(visible)
switch {
case err != nil:
slog.Warn("email HTML conversion failed, using text fallback", "message_id", e.MessageID, "error", err)
case htmlLength == 0:
// Image-only and tracking-only alternatives have no message text.
case plainLength >= minFidelityRunes && htmlLength*minHTMLFraction < plainLength:
slog.Debug("email HTML alternative is incomplete, using plain text", "message_id", e.MessageID)
default:
return cleanEmailMarkdown(markdown, reply)
}
}

body := e.PlainBody
if strings.TrimSpace(body) == "" && e.HTMLBody != "" {
// Keep an oversized or otherwise unconvertible HTML-only message readable.
body = html.UnescapeString(StripHTML(e.HTMLBody[:min(len(e.HTMLBody), maxEmailHTMLBytes)]))
}
if strings.TrimSpace(body) == "" {
return ""
}
body = strings.ReplaceAll(strings.ReplaceAll(body, "\r\n", "\n"), "\r", "\n")
if reply {
body = ExtractReplyContent(body)
}
body = StripSignature(body)
body = plainMarkdownEscapes.Replace(body)
body = plainBlockMarker.ReplaceAllString(body, "$1\\$2$3")
body = plainOrderedMarker.ReplaceAllString(body, "$1\\$2$3")
body = plainDivider.ReplaceAllString(body, "$1\\$2$3")
lines := strings.Split(body, "\n")
for i := 0; i+1 < len(lines); i++ {
if strings.TrimSpace(lines[i]) != "" && strings.TrimSpace(lines[i+1]) != "" {
lines[i] = strings.TrimRight(lines[i], " \t") + " "
}
}
return strings.Join(lines, "\n")
}

func normalizedBodyLength(body string) int {
return utf8.RuneCountInString(strings.Join(strings.Fields(body), " "))
}

func emailHTMLMarkdown(body string) (markdown, visible string, err error) {
if len(body) > maxEmailHTMLBytes {
return "", "", fmt.Errorf("HTML body exceeds %d bytes", maxEmailHTMLBytes)
}
safeHTML := emailHTMLPolicy.Sanitize(body)
visible = html.UnescapeString(StripHTML(safeHTML))
doc, err := nethtml.Parse(strings.NewReader(safeHTML))
if err != nil {
return "", "", fmt.Errorf("parse email HTML: %w", err)
}
flattenEmailLayoutTables(doc)
conv := converter.NewConverter(converter.WithPlugins(
base.NewBasePlugin(), commonmark.NewCommonmarkPlugin(), table.NewTablePlugin(),
))
converted, err := conv.ConvertNode(doc)
if err != nil {
return "", "", fmt.Errorf("convert email HTML: %w", err)
}
return string(converted), visible, nil
}

// Only tables with explicit headers are treated as data tables.
func flattenEmailLayoutTables(node *nethtml.Node) {
if node.Type == nethtml.ElementNode && node.Data == "table" {
layout := !emailTableHasHeader(node)
for _, attr := range node.Attr {
if attr.Key == "role" && attr.Val == "presentation" {
layout = true
}
}
if layout {
flattenEmailTable(node, true)
}
}
for child := node.FirstChild; child != nil; child = child.NextSibling {
flattenEmailLayoutTables(child)
}
}

func emailTableHasHeader(node *nethtml.Node) bool {
for child := node.FirstChild; child != nil; child = child.NextSibling {
if child.Type == nethtml.ElementNode && child.Data == "table" {
continue
}
if child.Data == "th" || emailTableHasHeader(child) {
return true
}
}
return false
}

func flattenEmailTable(node *nethtml.Node, root bool) {
if !root && node.Type == nethtml.ElementNode && node.Data == "table" {
return
}
if node.Type == nethtml.ElementNode {
switch node.Data {
case "table", "thead", "tbody", "tfoot", "tr", "th", "td", "caption":
node.Data = "div"
node.DataAtom = 0
}
}
for child := node.FirstChild; child != nil; child = child.NextSibling {
flattenEmailTable(child, false)
}
}

// Hide code-block contents from the existing line-based email cleanup.
func cleanEmailMarkdown(body string, reply bool) string {
source := []byte(body)
doc := goldmark.DefaultParser().Parse(text.NewReader(source))
type codeRange struct{ start, end int }
var ranges []codeRange
_ = ast.Walk(doc, func(node ast.Node, entering bool) (ast.WalkStatus, error) {
if entering && (node.Kind() == ast.KindFencedCodeBlock || node.Kind() == ast.KindCodeBlock) && node.Lines().Len() > 0 {
lines := node.Lines()
ranges = append(ranges, codeRange{lines.At(0).Start, lines.At(lines.Len() - 1).Stop})
}
return ast.WalkContinue, nil
})
prefix := "WINDSHIFT_EMAIL_CODE_"
for strings.Contains(body, prefix) {
prefix += "_"
}
restore := make([]string, 0, len(ranges)*2)
for i := len(ranges) - 1; i >= 0; i-- {
r := ranges[i]
placeholder := fmt.Sprintf("%s%d\n", prefix, i)
restore = append(restore, placeholder, body[r.start:r.end])
body = body[:r.start] + placeholder + body[r.end:]
}
if reply {
body = ExtractReplyContent(body)
}
body = StripSignature(body)
if len(restore) > 0 {
body = strings.NewReplacer(restore...).Replace(body)
}
return body
}
5 changes: 2 additions & 3 deletions internal/email/processor.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -633,7 +633,7 @@ func (p *Processor) createItemFromEmail(
params := services.ItemCreationParams{
WorkspaceID: workspaceID,
Title: sanitize.PlainTextField.Sanitize(email.GetSubjectForItem()),
Description: sanitize.Comment.Sanitize(StripSignature(email.GetBodyText())),
Description: sanitize.Comment.Sanitize(email.bodyMarkdown(false)),
ItemTypeID: itemTypeID,
PriorityID: config.EmailDefaultPriorityID,
RequestTypeID: requestTypeID,
Expand Down Expand Up @@ -716,8 +716,7 @@ func (p *Processor) addCommentFromReply(
itemID int,
customerID int,
) (*ProcessingResult, error) {
// Extract reply content (strip quoted text)
content := StripSignature(ExtractReplyContent(email.GetBodyText()))
content := email.bodyMarkdown(true)

if strings.TrimSpace(content) == "" {
// No new content - skip
Expand Down
Loading