From ed84adb679f3de70b2bffaead20a87711c38ee3a Mon Sep 17 00:00:00 2001 From: lunnyxiao Date: Wed, 17 Sep 2014 12:03:03 +0800 Subject: [PATCH] toutf8 improved & add max git diff lines --- conf/app.ini | 3 +++ models/git_diff.go | 36 +++++++++++++++++++++++------------- modules/base/template.go | 27 +++++++++++++++++++++++++++ modules/setting/setting.go | 3 +++ routers/repo/commit.go | 7 +++++-- routers/repo/view.go | 20 +------------------- templates/repo/diff.tmpl | 2 +- 7 files changed, 63 insertions(+), 35 deletions(-) diff --git a/conf/app.ini b/conf/app.ini index be49e06411..cb907b22de 100644 --- a/conf/app.ini +++ b/conf/app.ini @@ -255,3 +255,6 @@ CONN = [i18n] LANGS = en-US,zh-CN,de-DE NAMES = English,简体中文,Deutsch + +[git] +MAX_GITDIFF_LINES = 10000 \ No newline at end of file diff --git a/models/git_diff.go b/models/git_diff.go index bf7a9cd575..e093e7ab1b 100644 --- a/models/git_diff.go +++ b/models/git_diff.go @@ -70,7 +70,7 @@ func (diff *Diff) NumFiles() int { const DIFF_HEAD = "diff --git " -func ParsePatch(pid int64, cmd *exec.Cmd, reader io.Reader) (*Diff, error) { +func ParsePatch(pid int64, maxlines int, cmd *exec.Cmd, reader io.Reader) (*Diff, error) { scanner := bufio.NewScanner(reader) var ( curFile *DiffFile @@ -79,6 +79,7 @@ func ParsePatch(pid int64, cmd *exec.Cmd, reader io.Reader) (*Diff, error) { } leftLine, rightLine int + isTooLong bool ) diff := &Diff{Files: make([]*DiffFile, 0)} @@ -90,18 +91,19 @@ func ParsePatch(pid int64, cmd *exec.Cmd, reader io.Reader) (*Diff, error) { continue } - i = i + 1 - - // Diff data too large. - if i == 5000 { - log.Warn("Diff data too large") - return &Diff{}, nil - } - if line == "" { continue } + i = i + 1 + + // Diff data too large, we only show the first about maxlines lines + if i == maxlines { + isTooLong = true + log.Warn("Diff data too large") + //return &Diff{}, nil + } + switch { case line[0] == ' ': diffLine := &DiffLine{Type: DIFF_LINE_PLAIN, Content: line, LeftIdx: leftLine, RightIdx: rightLine} @@ -110,6 +112,10 @@ func ParsePatch(pid int64, cmd *exec.Cmd, reader io.Reader) (*Diff, error) { curSection.Lines = append(curSection.Lines, diffLine) continue case line[0] == '@': + if isTooLong { + return diff, nil + } + curSection = &DiffSection{} curFile.Sections = append(curFile.Sections, curSection) ss := strings.Split(line, "@@") @@ -143,6 +149,10 @@ func ParsePatch(pid int64, cmd *exec.Cmd, reader io.Reader) (*Diff, error) { // Get new file. if strings.HasPrefix(line, DIFF_HEAD) { + if isTooLong { + return diff, nil + } + fs := strings.Split(line[len(DIFF_HEAD):], " ") a := fs[0] @@ -174,7 +184,7 @@ func ParsePatch(pid int64, cmd *exec.Cmd, reader io.Reader) (*Diff, error) { return diff, nil } -func GetDiffRange(repoPath, beforeCommitId string, afterCommitId string) (*Diff, error) { +func GetDiffRange(repoPath, beforeCommitId string, afterCommitId string, maxlines int) (*Diff, error) { repo, err := git.OpenRepository(repoPath) if err != nil { return nil, err @@ -228,9 +238,9 @@ func GetDiffRange(repoPath, beforeCommitId string, afterCommitId string) (*Diff, } }() - return ParsePatch(pid, cmd, rd) + return ParsePatch(pid, maxlines, cmd, rd) } -func GetDiffCommit(repoPath, commitId string) (*Diff, error) { - return GetDiffRange(repoPath, "", commitId) +func GetDiffCommit(repoPath, commitId string, maxlines int) (*Diff, error) { + return GetDiffRange(repoPath, "", commitId, maxlines) } diff --git a/modules/base/template.go b/modules/base/template.go index f2ae00b915..6419572912 100644 --- a/modules/base/template.go +++ b/modules/base/template.go @@ -8,13 +8,16 @@ import ( "bytes" "container/list" "encoding/json" + "errors" "fmt" "html/template" "runtime" "strings" "time" + "code.google.com/p/mahonia" "github.com/gogits/gogs/modules/setting" + "github.com/saintfish/chardet" ) func Str2html(raw string) template.HTML { @@ -45,6 +48,29 @@ func ShortSha(sha1 string) string { return sha1 } +func ToUtf8WithErr(content []byte) (error, string) { + detector := chardet.NewTextDetector() + result, err := detector.DetectBest(content) + if err != nil { + return err, "" + } + + if result.Charset == "utf8" { + return nil, string(content) + } + + decoder := mahonia.NewDecoder(result.Charset) + if decoder != nil { + return nil, decoder.ConvertString(string(content)) + } + return errors.New("unknow char decoder"), string(content) +} + +func ToUtf8(content string) string { + _, res := ToUtf8WithErr([]byte(content)) + return res +} + var mailDomains = map[string]string{ "gmail.com": "gmail.com", } @@ -103,6 +129,7 @@ var TemplateFuncs template.FuncMap = map[string]interface{}{ "ActionContent2Commits": ActionContent2Commits, "Oauth2Icon": Oauth2Icon, "Oauth2Name": Oauth2Name, + "ToUtf8": ToUtf8, } type Actioner interface { diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 199b4f2c27..011c00c08c 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -64,6 +64,7 @@ var ( // Picture settings. PictureService string DisableGravatar bool + MaxGitDiffLines int // Log settings. LogRootPath string @@ -241,6 +242,8 @@ func NewConfigContext() { []string{"server"}) DisableGravatar = Cfg.MustBool("picture", "DISABLE_GRAVATAR") + MaxGitDiffLines = Cfg.MustInt("git", "MAX_GITDIFF_LINES", 5000) + Langs = Cfg.MustValueArray("i18n", "LANGS", ",") Names = Cfg.MustValueArray("i18n", "NAMES", ",") } diff --git a/routers/repo/commit.go b/routers/repo/commit.go index 54acc85b3e..f7feb4d95e 100644 --- a/routers/repo/commit.go +++ b/routers/repo/commit.go @@ -12,6 +12,7 @@ import ( "github.com/gogits/gogs/models" "github.com/gogits/gogs/modules/base" "github.com/gogits/gogs/modules/middleware" + "github.com/gogits/gogs/modules/setting" ) const ( @@ -114,7 +115,8 @@ func Diff(ctx *middleware.Context) { commit := ctx.Repo.Commit - diff, err := models.GetDiffCommit(models.RepoPath(userName, repoName), commitId) + diff, err := models.GetDiffCommit(models.RepoPath(userName, repoName), + commitId, setting.MaxGitDiffLines) if err != nil { ctx.Handle(404, "GetDiffCommit", err) return @@ -176,7 +178,8 @@ func CompareDiff(ctx *middleware.Context) { return } - diff, err := models.GetDiffRange(models.RepoPath(userName, repoName), beforeCommitId, afterCommitId) + diff, err := models.GetDiffRange(models.RepoPath(userName, repoName), beforeCommitId, + afterCommitId, setting.MaxGitDiffLines) if err != nil { ctx.Handle(404, "GetDiffRange", err) return diff --git a/routers/repo/view.go b/routers/repo/view.go index f98eee037d..e42894ae73 100644 --- a/routers/repo/view.go +++ b/routers/repo/view.go @@ -11,12 +11,9 @@ import ( "path/filepath" "strings" - "github.com/saintfish/chardet" - "github.com/gogits/gogs/modules/base" "github.com/gogits/gogs/modules/git" "github.com/gogits/gogs/modules/log" - "github.com/gogits/gogs/modules/mahonia" "github.com/gogits/gogs/modules/middleware" ) @@ -24,21 +21,6 @@ const ( HOME base.TplName = "repo/home" ) -func toUtf8(content []byte) (error, string) { - detector := chardet.NewTextDetector() - result, err := detector.DetectBest(content) - if err != nil { - return err, "" - } - - if result.Charset == "utf8" { - return nil, string(content) - } - - decoder := mahonia.NewDecoder(result.Charset) - return nil, decoder.ConvertString(string(content)) -} - func Home(ctx *middleware.Context) { ctx.Data["Title"] = ctx.Repo.Repository.Name @@ -117,7 +99,7 @@ func Home(ctx *middleware.Context) { if readmeExist { ctx.Data["FileContent"] = string(base.RenderMarkdown(buf, "")) } else { - if err, content := toUtf8(buf); err != nil { + if err, content := base.ToUtf8WithErr(buf); err != nil { if err != nil { log.Error(4, "Convert content encoding: %s", err) } diff --git a/templates/repo/diff.tmpl b/templates/repo/diff.tmpl index 7873345083..a2150f2849 100644 --- a/templates/repo/diff.tmpl +++ b/templates/repo/diff.tmpl @@ -105,7 +105,7 @@ {{if .RightIdx}}{{.RightIdx}}{{end}} - 
{{.Content}}
+ 
{{ToUtf8 .Content}}
{{end}}