diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index a2dd92b105..1d490e291f 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -1704,6 +1704,9 @@ LEVEL = Info ;; ;; convert \r\n to \n for Sendmail ;SENDMAIL_CONVERT_CRLF = true +;; +;; convert links of attached images to inline images +;BASE64_EMBED_IMAGES = true ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/models/fixtures/attachment.yml b/models/fixtures/attachment.yml index 7882d8bff2..b33276a391 100644 --- a/models/fixtures/attachment.yml +++ b/models/fixtures/attachment.yml @@ -153,3 +153,16 @@ download_count: 0 size: 0 created_unix: 946684800 + +- + id: 13 + uuid: 1b267670-1793-4cd0-abc1-449269b7cff9 + repo_id: 1 + issue_id: 2 + release_id: 0 + uploader_id: 2 + comment_id: 0 + name: gitea.png + download_count: 0 + size: 1458 + created_unix: 946684800 diff --git a/modules/setting/mailer.go b/modules/setting/mailer.go index d4db55dc7b..638c442884 100644 --- a/modules/setting/mailer.go +++ b/modules/setting/mailer.go @@ -28,6 +28,7 @@ type Mailer struct { SendAsPlainText bool `ini:"SEND_AS_PLAIN_TEXT"` SubjectPrefix string `ini:"SUBJECT_PREFIX"` OverrideHeader map[string][]string `ini:"-"` + Base64EmbedImages bool `ini:"BASE64_EMBED_IMAGES"` // SMTP sender Protocol string `ini:"PROTOCOL"` @@ -150,6 +151,7 @@ func loadMailerFrom(rootCfg ConfigProvider) { sec.Key("SENDMAIL_TIMEOUT").MustDuration(5 * time.Minute) sec.Key("SENDMAIL_CONVERT_CRLF").MustBool(true) sec.Key("FROM").MustString(sec.Key("USER").String()) + sec.Key("BASE64_EMBED_IMAGES").MustBool(false) // Now map the values on to the MailService MailService = &Mailer{} diff --git a/services/mailer/mail.go b/services/mailer/mail.go index 23c91595b7..040fb94a92 100644 --- a/services/mailer/mail.go +++ b/services/mailer/mail.go @@ -7,9 +7,12 @@ package mailer import ( "bytes" "context" + "encoding/base64" "fmt" "html/template" + "io" "mime" + "net/http" "regexp" "strconv" "strings" @@ -26,11 +29,13 @@ import ( "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/markup/markdown" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/translation" incoming_payload "code.gitea.io/gitea/services/mailer/incoming/payload" "code.gitea.io/gitea/services/mailer/token" + "golang.org/x/net/html" "gopkg.in/gomail.v2" ) @@ -195,7 +200,7 @@ func SendCollaboratorMail(u, doer *user_model.User, repo *repo_model.Repository) SendAsync(msg) } -func composeIssueCommentMessages(ctx *mailCommentContext, lang string, recipients []*user_model.User, fromMention bool, info string) ([]*Message, error) { +func composeIssueCommentMessages(ctx *MailCommentContext, lang string, recipients []*user_model.User, fromMention bool, info string) ([]*Message, error) { var ( subject string link string @@ -232,6 +237,15 @@ func composeIssueCommentMessages(ctx *mailCommentContext, lang string, recipient return nil, err } + if setting.MailService.Base64EmbedImages { + bodyStr := string(body) + bodyStr, err = Base64InlineImages(bodyStr, ctx) + if err != nil { + return nil, err + } + body = template.HTML(bodyStr) + } + actType, actName, tplName := actionToTemplate(ctx.Issue, ctx.ActionType, commentType, reviewType) if actName != "new" { @@ -363,6 +377,81 @@ func composeIssueCommentMessages(ctx *mailCommentContext, lang string, recipient return msgs, nil } +func Base64InlineImages(body string, ctx *MailCommentContext) (string, error) { + doc, err := html.Parse(strings.NewReader(body)) + if err != nil { + log.Error("Failed to parse HTML body: %v", err) + return "", err + } + + var processNode func(*html.Node) + processNode = func(n *html.Node) { + if n.Type == html.ElementNode { + if n.Data == "img" { + for i, attr := range n.Attr { + if attr.Key == "src" { + attachmentPath := attr.Val + dataURI, err := AttachmentSrcToBase64DataURI(attachmentPath, ctx) + if err != nil { + log.Trace("attachmentSrcToDataURI not possible: %v", err) // Not an error, just skip. This is probably an image from outside the gitea instance. + continue + } + log.Trace("Old value of src attribute: %s, new value (first 100 characters): %s", attr.Val, dataURI[:100]) + n.Attr[i].Val = dataURI + } + } + } + } + + for c := n.FirstChild; c != nil; c = c.NextSibling { + processNode(c) + } + } + + processNode(doc) + + var buf bytes.Buffer + err = html.Render(&buf, doc) + if err != nil { + log.Error("Failed to render modified HTML: %v", err) + return "", err + } + return buf.String(), nil +} + +func AttachmentSrcToBase64DataURI(attachmentPath string, ctx *MailCommentContext) (string, error) { + if !strings.HasPrefix(attachmentPath, setting.AppURL) { // external image + return "", fmt.Errorf("external image") + } + parts := strings.Split(attachmentPath, "/attachments/") + if len(parts) <= 1 { + return "", fmt.Errorf("invalid attachment path: %s", attachmentPath) + } + + attachmentUUID := parts[len(parts)-1] + attachment, err := repo_model.GetAttachmentByUUID(ctx, attachmentUUID) + if err != nil { + return "", err + } + + fr, err := storage.Attachments.Open(attachment.RelativePath()) + if err != nil { + return "", err + } + defer fr.Close() + + content, err := io.ReadAll(fr) + if err != nil { + return "", err + } + + mimeType := http.DetectContentType(content) + encoded := base64.StdEncoding.EncodeToString(content) + dataURI := fmt.Sprintf("data:%s;base64,%s", mimeType, encoded) + + return dataURI, nil +} + func generateMessageIDForIssue(issue *issues_model.Issue, comment *issues_model.Comment, actionType activities_model.ActionType) string { var path string if issue.IsPull { @@ -394,7 +483,7 @@ func generateMessageIDForRelease(release *repo_model.Release) string { return fmt.Sprintf("<%s/releases/%d@%s>", release.Repo.FullName(), release.ID, setting.Domain) } -func generateAdditionalHeaders(ctx *mailCommentContext, reason string, recipient *user_model.User) map[string]string { +func generateAdditionalHeaders(ctx *MailCommentContext, reason string, recipient *user_model.User) map[string]string { repo := ctx.Issue.Repo return map[string]string{ @@ -458,7 +547,7 @@ func SendIssueAssignedMail(ctx context.Context, issue *issues_model.Issue, doer } for lang, tos := range langMap { - msgs, err := composeIssueCommentMessages(&mailCommentContext{ + msgs, err := composeIssueCommentMessages(&MailCommentContext{ Context: ctx, Issue: issue, Doer: doer, diff --git a/services/mailer/mail_comment.go b/services/mailer/mail_comment.go index 1812441d5a..b9fe54ebc1 100644 --- a/services/mailer/mail_comment.go +++ b/services/mailer/mail_comment.go @@ -26,7 +26,7 @@ func MailParticipantsComment(ctx context.Context, c *issues_model.Comment, opTyp content = "" } if err := mailIssueCommentToParticipants( - &mailCommentContext{ + &MailCommentContext{ Context: ctx, Issue: issue, Doer: c.Poster, @@ -49,7 +49,7 @@ func MailMentionsComment(ctx context.Context, pr *issues_model.PullRequest, c *i visited := make(container.Set[int64], len(mentions)+1) visited.Add(c.Poster.ID) if err = mailIssueCommentBatch( - &mailCommentContext{ + &MailCommentContext{ Context: ctx, Issue: pr.Issue, Doer: c.Poster, diff --git a/services/mailer/mail_issue.go b/services/mailer/mail_issue.go index fab3315be2..e13770f165 100644 --- a/services/mailer/mail_issue.go +++ b/services/mailer/mail_issue.go @@ -22,7 +22,7 @@ func fallbackMailSubject(issue *issues_model.Issue) string { return fmt.Sprintf("[%s] %s (#%d)", issue.Repo.FullName(), issue.Title, issue.Index) } -type mailCommentContext struct { +type MailCommentContext struct { context.Context Issue *issues_model.Issue Doer *user_model.User @@ -41,7 +41,7 @@ const ( // This function sends two list of emails: // 1. Repository watchers (except for WIP pull requests) and users who are participated in comments. // 2. Users who are not in 1. but get mentioned in current issue/comment. -func mailIssueCommentToParticipants(ctx *mailCommentContext, mentions []*user_model.User) error { +func mailIssueCommentToParticipants(ctx *MailCommentContext, mentions []*user_model.User) error { // Required by the mail composer; make sure to load these before calling the async function if err := ctx.Issue.LoadRepo(ctx); err != nil { return fmt.Errorf("LoadRepo: %w", err) @@ -120,7 +120,7 @@ func mailIssueCommentToParticipants(ctx *mailCommentContext, mentions []*user_mo return nil } -func mailIssueCommentBatch(ctx *mailCommentContext, users []*user_model.User, visited container.Set[int64], fromMention bool) error { +func mailIssueCommentBatch(ctx *MailCommentContext, users []*user_model.User, visited container.Set[int64], fromMention bool) error { checkUnit := unit.TypeIssues if ctx.Issue.IsPull { checkUnit = unit.TypePullRequests @@ -186,7 +186,7 @@ func MailParticipants(ctx context.Context, issue *issues_model.Issue, doer *user } forceDoerNotification := opType == activities_model.ActionAutoMergePullRequest if err := mailIssueCommentToParticipants( - &mailCommentContext{ + &MailCommentContext{ Context: ctx, Issue: issue, Doer: doer, diff --git a/services/mailer/mail_test.go b/services/mailer/mail_test.go index 40fd21dea5..aa1ac990ab 100644 --- a/services/mailer/mail_test.go +++ b/services/mailer/mail_test.go @@ -83,7 +83,7 @@ func TestComposeIssueCommentMessage(t *testing.T) { bodyTemplates = template.Must(template.New("issue/comment").Parse(bodyTpl)) recipients := []*user_model.User{{Name: "Test", Email: "test@gitea.com"}, {Name: "Test2", Email: "test2@gitea.com"}} - msgs, err := composeIssueCommentMessages(&mailCommentContext{ + msgs, err := composeIssueCommentMessages(&MailCommentContext{ Context: context.TODO(), // TODO: use a correct context Issue: issue, Doer: doer, ActionType: activities_model.ActionCommentIssue, Content: fmt.Sprintf("test @%s %s#%d body", doer.Name, issue.Repo.FullName(), issue.Index), @@ -129,7 +129,7 @@ func TestComposeIssueMessage(t *testing.T) { bodyTemplates = template.Must(template.New("issue/new").Parse(bodyTpl)) recipients := []*user_model.User{{Name: "Test", Email: "test@gitea.com"}, {Name: "Test2", Email: "test2@gitea.com"}} - msgs, err := composeIssueCommentMessages(&mailCommentContext{ + msgs, err := composeIssueCommentMessages(&MailCommentContext{ Context: context.TODO(), // TODO: use a correct context Issue: issue, Doer: doer, ActionType: activities_model.ActionCreateIssue, Content: "test body", @@ -176,14 +176,14 @@ func TestTemplateSelection(t *testing.T) { assert.Contains(t, wholemsg, expBody) } - msg := testComposeIssueCommentMessage(t, &mailCommentContext{ + msg := testComposeIssueCommentMessage(t, &MailCommentContext{ Context: context.TODO(), // TODO: use a correct context Issue: issue, Doer: doer, ActionType: activities_model.ActionCreateIssue, Content: "test body", }, recipients, false, "TestTemplateSelection") expect(t, msg, "issue/new/subject", "issue/new/body") - msg = testComposeIssueCommentMessage(t, &mailCommentContext{ + msg = testComposeIssueCommentMessage(t, &MailCommentContext{ Context: context.TODO(), // TODO: use a correct context Issue: issue, Doer: doer, ActionType: activities_model.ActionCommentIssue, Content: "test body", Comment: comment, @@ -192,14 +192,14 @@ func TestTemplateSelection(t *testing.T) { pull := unittest.AssertExistsAndLoadBean(t, &issues_model.Issue{ID: 2, Repo: repo, Poster: doer}) comment = unittest.AssertExistsAndLoadBean(t, &issues_model.Comment{ID: 4, Issue: pull}) - msg = testComposeIssueCommentMessage(t, &mailCommentContext{ + msg = testComposeIssueCommentMessage(t, &MailCommentContext{ Context: context.TODO(), // TODO: use a correct context Issue: pull, Doer: doer, ActionType: activities_model.ActionCommentPull, Content: "test body", Comment: comment, }, recipients, false, "TestTemplateSelection") expect(t, msg, "pull/comment/subject", "pull/comment/body") - msg = testComposeIssueCommentMessage(t, &mailCommentContext{ + msg = testComposeIssueCommentMessage(t, &MailCommentContext{ Context: context.TODO(), // TODO: use a correct context Issue: issue, Doer: doer, ActionType: activities_model.ActionCloseIssue, Content: "test body", Comment: comment, @@ -218,7 +218,7 @@ func TestTemplateServices(t *testing.T) { bodyTemplates = template.Must(template.New("issue/default").Parse(tplBody)) recipients := []*user_model.User{{Name: "Test", Email: "test@gitea.com"}} - msg := testComposeIssueCommentMessage(t, &mailCommentContext{ + msg := testComposeIssueCommentMessage(t, &MailCommentContext{ Context: context.TODO(), // TODO: use a correct context Issue: issue, Doer: doer, ActionType: actionType, Content: "test body", Comment: comment, @@ -252,7 +252,7 @@ func TestTemplateServices(t *testing.T) { "//Re: //") } -func testComposeIssueCommentMessage(t *testing.T, ctx *mailCommentContext, recipients []*user_model.User, fromMention bool, info string) *Message { +func testComposeIssueCommentMessage(t *testing.T, ctx *MailCommentContext, recipients []*user_model.User, fromMention bool, info string) *Message { msgs, err := composeIssueCommentMessages(ctx, "en-US", recipients, fromMention, info) assert.NoError(t, err) assert.Len(t, msgs, 1) @@ -262,7 +262,7 @@ func testComposeIssueCommentMessage(t *testing.T, ctx *mailCommentContext, recip func TestGenerateAdditionalHeaders(t *testing.T) { doer, _, issue, _ := prepareMailerTest(t) - ctx := &mailCommentContext{Context: context.TODO() /* TODO: use a correct context */, Issue: issue, Doer: doer} + ctx := &MailCommentContext{Context: context.TODO() /* TODO: use a correct context */, Issue: issue, Doer: doer} recipient := &user_model.User{Name: "test", Email: "test@gitea.com"} headers := generateAdditionalHeaders(ctx, "dummy-reason", recipient) diff --git a/tests/integration/email_embed_b64_images_test.go b/tests/integration/email_embed_b64_images_test.go new file mode 100644 index 0000000000..52c8fd8e6d --- /dev/null +++ b/tests/integration/email_embed_b64_images_test.go @@ -0,0 +1,71 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package integration + +import ( + "context" + "testing" + + issues_model "code.gitea.io/gitea/models/issues" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/models/unittest" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/setting" + mail "code.gitea.io/gitea/services/mailer" + "code.gitea.io/gitea/tests" + + "github.com/stretchr/testify/assert" +) + +func TestEmailEmbedBase64Images(t *testing.T) { + defer tests.PrepareTestEnv(t)() + tests.PrepareAttachmentsStorage(t) + + user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1, Owner: user}) + issue := unittest.AssertExistsAndLoadBean(t, &issues_model.Issue{ID: 2, Repo: repo, Poster: user}) + + attachment := unittest.AssertExistsAndLoadBean(t, &repo_model.Attachment{ID: 13, IssueID: issue.ID, RepoID: repo.ID}) + ctx0 := context.Background() + + ctx := &mail.MailCommentContext{Context: ctx0 /* TODO: use a correct context */, Issue: issue, Doer: user} + + img1ExternalURL := "https://via.placeholder.com/10" + img1ExternalImg := "" + + img2InternalURL := setting.AppURL + repo.Owner.Name + "/" + repo.Name + "/attachments/" + attachment.UUID + img2InternalImg := "" + img2InternalBase64 := "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAFAAAAAxCAYAAABNuS5SAAAAAXNSR0IArs4c6QAABWxJREFUaEPVmm1sU1UYx/9n3RsuxmS+ECdC71YYbCjohmMDNxyJONBNxaIo29qNlZqYbTFhi0FkmZpgogH0w1g7t46ZmEA0KIl+MQsvwQSM8QOJSGDtFO0XFTQE0tG11/Rqa9fel3Pf7+6H7cN9zjnP8zv/53nOvbcElFe7n/mOsKimNAfAfpWTm3NgtCP4Df0Ya1q2DpXdl2OL7yYEbrC4O+FlwBMiif/cH6Grc4RZEotj2lphsddAyB9gSbSlqvvvZx/tqQCQ5x4pBVgUGelrAqIgQJePYY10Rslam1d7w841fSWJsW5/qZIpVI0hBM5sgCyIy8/EVc1s0OCmh3eFt9X0mwYwK4V3+h2LZtnYVYPiV73MUw91hV9c+4Z1AM6HtE2nvmllR/il2jetAZAGHgu8M+4J7U0PgmacaqkJTPDkSvdv22v3PmBWDUylsMtnPwiQHqFAky2b776ZADdWtv26o25gkQUACndcMXgJx80E+MSKV35pW//2YssClIJnNsCG5dt/dj3+7hJTAbp8zAyAfL70tDrA+vJt0+76/XazAfIemGngma3A9eXOYGf9e9wJ2oyDNNdEhGoYPUB7P0D269VpxeatW/r8VNeG98vmNUAzVVhb1nLF03jAYRZAkhurVK1AowDeteBeRGZvYSZ6MyXKmtJnLns3HlpqFkDuZYLaFDYC4FhXMCuTkzUvec+MGjgvAPLBS9IcmuzB+akTZpRfbk3tAPqZU2BRr0ckYgDTVWg5BSYJ00LR44lEDN7E2X2Y/HECvo6LyLMVpNw0EOTtgCdUQDzDJXfcJgX/V+Y0YrRHGb3qIK36+DZZb5BzXukLqYeNo2HcGzpNo0KtFSgELx3MBy+fRXHR/aLu6QWSCqCZacwPkIXbz52bU5eYStPtOkcciLPavWjPBHgSQIPQVtKmslYqFIJy5tIxjJ7un+NmY0UbWtcN0CQJZ6OVIrO+ykkFTwNRag7aKGlqX+ZcK0rq0LflE6ol1EJMZ5H6qNQ+zNwiBAvEPDACohi8q39ewlufN6F13SAaK3ZwroZ+v4DB4y2yUloXgPSdlBwKeIK9QqDVqpBGfXw2gTN7cOqnTzm3pOqibgDpIf6Lj0+RagCKBX4jcg3dE9XYvMoL52N9vPt3I3Id3RNVogC1hJdwIuu7sPMobEV/MbNUxWSu0UzAEyqUuwm0HTXz2VfIv4Rd76ZRrFq8gddEd4CJVZs+dBQsLIxFFEBUPIQm7WrKmuFtPCi4hhRktfD4sk70tzFq0lEuSaW1L30dvQHylSxRgGrSUQ5AmqcOx8Iq7Gk+Jjjtzo+XIRafFax/eqiPtwbq1V3FgNIAlOqueqtPqGlKKpAvcK1TWy3Aw5O9ODf1JQa3fo0Hi8uzXNZLfbIUKKzM0jUAe15OymbaqgVolvo0ASgGTo5ShSDG2RhyiI3qSCL26l/NBos9gSlKYVpn5ABMzil1nOHrumJj9Uxf3RWopovTgMyEkzlGC3gkl1071jF9Tkg0uipQDUAaRb52pAo3Z65zpmakryEK1AKi1BGGTx1aqO/OiK3wo+4rid8OCV66K5ADOGZfjSj5gbZ2itkNuS6gME/6x/haAKR5fWcIQK1UmA5233MnYL+nkqo7K9k4GniGpXAyACVdWSr4rdW78fQjr6bMvr18HP6Tr0sNk7xvSYB6KFGShAIDWniGKzCxYLuPaSbAFwriMmSIHHimAPxPhTEAOYYQkbGIPRyyDQxA1rdPw5pIZhwuH2MpiCzIC+Oe4GcyeHOmpgHklOhnjoKFU67TWtvns8X5vl3fR5XMaypAPbszLQy5NS9zXksA5NQ4vGw5SPQibeBq7dSCS67/D+Q9UQwzW88cAAAAAElFTkSuQmCC" + img2InternalBase64Img := "" + + // 1st Test: convert internal image to base64 + t.Run("replaceSpecifiedBase64ImagesInternal", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + resultImg1Internal, err := mail.AttachmentSrcToBase64DataURI(img2InternalURL, ctx) + assert.NoError(t, err) + assert.Equal(t, img2InternalBase64, resultImg1Internal) // replace cause internal image + }) + + // 2nd Test: convert external image to base64 -> abort cause external image + t.Run("replaceSpecifiedBase64ImagesExternal", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + resultImg1External, err := mail.AttachmentSrcToBase64DataURI(img1ExternalURL, ctx) + assert.Error(t, err) + assert.Equal(t, "", resultImg1External) // don't replace cause external image + }) + + // 3rd Test: generate email body with 1 internal and 1 external image, expect the result to have the internal image replaced with base64 data and the external not replaced + t.Run("generateEmailBody", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + mailBody := "

Test1

" + img1ExternalImg + "

Test2

" + img2InternalImg + "

Test3

" + expectedMailBody := "

Test1

" + img1ExternalImg + "

Test2

" + img2InternalBase64Img + "

Test3

" + resultMailBody, err := mail.Base64InlineImages(mailBody, ctx) + + assert.NoError(t, err) + assert.Equal(t, expectedMailBody, resultMailBody) + }) +} diff --git a/tests/testdata/data/attachments/1/b/1b267670-1793-4cd0-abc1-449269b7cff9 b/tests/testdata/data/attachments/1/b/1b267670-1793-4cd0-abc1-449269b7cff9 new file mode 100644 index 0000000000..52dd6a0f99 Binary files /dev/null and b/tests/testdata/data/attachments/1/b/1b267670-1793-4cd0-abc1-449269b7cff9 differ