From 372d24b84bc6f4c5562792009c6b6e6a4aeb85f8 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Wed, 12 Nov 2025 19:44:49 +0800 Subject: [PATCH] Limit reading bytes instead of ReadAll (#35928) --- modules/actions/workflows.go | 4 +-- modules/issue/template/unmarshal.go | 3 +- modules/packages/nuget/metadata.go | 2 +- modules/packages/pub/metadata.go | 2 +- modules/util/io.go | 2 +- routers/web/repo/wiki.go | 2 +- services/issue/template.go | 4 +-- services/repository/generate.go | 46 ++++++++++++++++------------ services/repository/generate_test.go | 26 ++++++++++++++++ services/webhook/deliver.go | 3 +- 10 files changed, 64 insertions(+), 30 deletions(-) diff --git a/modules/actions/workflows.go b/modules/actions/workflows.go index 69f71bf651..26a6ebc370 100644 --- a/modules/actions/workflows.go +++ b/modules/actions/workflows.go @@ -5,7 +5,6 @@ package actions import ( "bytes" - "io" "slices" "strings" @@ -13,6 +12,7 @@ import ( "code.gitea.io/gitea/modules/glob" "code.gitea.io/gitea/modules/log" api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/util" webhook_module "code.gitea.io/gitea/modules/webhook" "github.com/nektos/act/pkg/jobparser" @@ -77,7 +77,7 @@ func GetContentFromEntry(entry *git.TreeEntry) ([]byte, error) { if err != nil { return nil, err } - content, err := io.ReadAll(f) + content, err := util.ReadWithLimit(f, 1024*1024) _ = f.Close() if err != nil { return nil, err diff --git a/modules/issue/template/unmarshal.go b/modules/issue/template/unmarshal.go index 1d8e9dd02d..ceab6babf4 100644 --- a/modules/issue/template/unmarshal.go +++ b/modules/issue/template/unmarshal.go @@ -5,7 +5,6 @@ package template import ( "fmt" - "io" "path" "strconv" @@ -76,7 +75,7 @@ func unmarshalFromEntry(entry *git.TreeEntry, filename string) (*api.IssueTempla } defer r.Close() - content, err := io.ReadAll(r) + content, err := util.ReadWithLimit(r, 1024*1024) if err != nil { return nil, fmt.Errorf("read all: %w", err) } diff --git a/modules/packages/nuget/metadata.go b/modules/packages/nuget/metadata.go index 513b4dd2b9..5124627395 100644 --- a/modules/packages/nuget/metadata.go +++ b/modules/packages/nuget/metadata.go @@ -216,7 +216,7 @@ func ParseNuspecMetaData(archive *zip.Reader, r io.Reader) (*Package, error) { if p.Metadata.Readme != "" { f, err := archive.Open(p.Metadata.Readme) if err == nil { - buf, _ := io.ReadAll(f) + buf, _ := util.ReadWithLimit(f, 1024*1024) m.Readme = string(buf) _ = f.Close() } diff --git a/modules/packages/pub/metadata.go b/modules/packages/pub/metadata.go index 9b00472eb2..a2cf6b728a 100644 --- a/modules/packages/pub/metadata.go +++ b/modules/packages/pub/metadata.go @@ -89,7 +89,7 @@ func ParsePackage(r io.Reader) (*Package, error) { return nil, err } } else if strings.EqualFold(hd.Name, "readme.md") { - data, err := io.ReadAll(tr) + data, err := util.ReadWithLimit(tr, 1024*1024) if err != nil { return nil, err } diff --git a/modules/util/io.go b/modules/util/io.go index b3dde9d1f6..f5a3d320e5 100644 --- a/modules/util/io.go +++ b/modules/util/io.go @@ -29,7 +29,7 @@ func ReadAtMost(r io.Reader, buf []byte) (n int, err error) { // ReadWithLimit reads at most "limit" bytes from r into buf. // If EOF or ErrUnexpectedEOF occurs while reading, err will be nil. func ReadWithLimit(r io.Reader, n int) (buf []byte, err error) { - return readWithLimit(r, 1024, n) + return readWithLimit(r, 4*1024, n) } func readWithLimit(r io.Reader, batch, limit int) ([]byte, error) { diff --git a/routers/web/repo/wiki.go b/routers/web/repo/wiki.go index e7c34ba1d6..542ac9c731 100644 --- a/routers/web/repo/wiki.go +++ b/routers/web/repo/wiki.go @@ -133,7 +133,7 @@ func wikiContentsByEntry(ctx *context.Context, entry *git.TreeEntry) []byte { return nil } defer reader.Close() - content, err := io.ReadAll(reader) + content, err := util.ReadWithLimit(reader, 5*1024*1024) // 5MB should be enough for a wiki page if err != nil { ctx.ServerError("ReadAll", err) return nil diff --git a/services/issue/template.go b/services/issue/template.go index 4b0f1aa987..99977c67cf 100644 --- a/services/issue/template.go +++ b/services/issue/template.go @@ -5,7 +5,6 @@ package issue import ( "fmt" - "io" "net/url" "path" "strings" @@ -15,6 +14,7 @@ import ( "code.gitea.io/gitea/modules/issue/template" "code.gitea.io/gitea/modules/log" api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/util" "gopkg.in/yaml.v3" ) @@ -65,7 +65,7 @@ func GetTemplateConfig(gitRepo *git.Repository, path string, commit *git.Commit) defer reader.Close() - configContent, err := io.ReadAll(reader) + configContent, err := util.ReadWithLimit(reader, 1024*1024) if err != nil { return GetDefaultTemplateConfig(), err } diff --git a/services/repository/generate.go b/services/repository/generate.go index 062c6f4fb1..caf15265a0 100644 --- a/services/repository/generate.go +++ b/services/repository/generate.go @@ -7,7 +7,9 @@ import ( "bufio" "bytes" "context" + "errors" "fmt" + "io/fs" "os" "path/filepath" "regexp" @@ -138,31 +140,37 @@ func (gt *giteaTemplateFileMatcher) Match(s string) bool { return false } -func readGiteaTemplateFile(tmpDir string) (*giteaTemplateFileMatcher, error) { - localPath := filepath.Join(tmpDir, ".gitea", "template") - if _, err := os.Stat(localPath); os.IsNotExist(err) { - return nil, nil - } else if err != nil { +func readLocalTmpRepoFileContent(localPath string, limit int) ([]byte, error) { + ok, err := util.IsRegularFile(localPath) + if err != nil { return nil, err + } else if !ok { + return nil, fs.ErrNotExist } - content, err := os.ReadFile(localPath) + f, err := os.Open(localPath) if err != nil { return nil, err } + defer f.Close() + return util.ReadWithLimit(f, limit) +} + +func readGiteaTemplateFile(tmpDir string) (*giteaTemplateFileMatcher, error) { + localPath := filepath.Join(tmpDir, ".gitea", "template") + content, err := readLocalTmpRepoFileContent(localPath, 1024*1024) + if err != nil { + return nil, err + } return newGiteaTemplateFileMatcher(localPath, content), nil } func substGiteaTemplateFile(ctx context.Context, tmpDir, tmpDirSubPath string, templateRepo, generateRepo *repo_model.Repository) error { tmpFullPath := filepath.Join(tmpDir, tmpDirSubPath) - if ok, err := util.IsRegularFile(tmpFullPath); !ok { - return err - } - - content, err := os.ReadFile(tmpFullPath) + content, err := readLocalTmpRepoFileContent(tmpFullPath, 1024*1024) if err != nil { - return err + return util.Iif(errors.Is(err, fs.ErrNotExist), nil, err) } if err := util.Remove(tmpFullPath); err != nil { return err @@ -172,7 +180,7 @@ func substGiteaTemplateFile(ctx context.Context, tmpDir, tmpDirSubPath string, t substSubPath := filepath.Clean(filePathSanitize(generateExpansion(ctx, tmpDirSubPath, templateRepo, generateRepo))) newLocalPath := filepath.Join(tmpDir, substSubPath) regular, err := util.IsRegularFile(newLocalPath) - if canWrite := regular || os.IsNotExist(err); !canWrite { + if canWrite := regular || errors.Is(err, fs.ErrNotExist); !canWrite { return nil } if err := os.MkdirAll(filepath.Dir(newLocalPath), 0o755); err != nil { @@ -242,15 +250,15 @@ func generateRepoCommit(ctx context.Context, repo, templateRepo, generateRepo *r // Variable expansion fileMatcher, err := readGiteaTemplateFile(tmpDir) - if err != nil { - return fmt.Errorf("readGiteaTemplateFile: %w", err) - } - - if fileMatcher != nil { + if err == nil { err = processGiteaTemplateFile(ctx, tmpDir, templateRepo, generateRepo, fileMatcher) if err != nil { - return err + return fmt.Errorf("processGiteaTemplateFile: %w", err) } + } else if errors.Is(err, fs.ErrNotExist) { + log.Debug("skip processing repo template files: no available .gitea/template") + } else { + return fmt.Errorf("readGiteaTemplateFile: %w", err) } if err = git.InitRepository(ctx, tmpDir, false, templateRepo.ObjectFormatName); err != nil { diff --git a/services/repository/generate_test.go b/services/repository/generate_test.go index 19b84c7bde..9c01911ded 100644 --- a/services/repository/generate_test.go +++ b/services/repository/generate_test.go @@ -4,6 +4,7 @@ package repository import ( + "io/fs" "os" "path/filepath" "testing" @@ -175,6 +176,31 @@ func TestProcessGiteaTemplateFile(t *testing.T) { // subst from a link, skip, and the target is unchanged assertSymLink("subst-${TEMPLATE_NAME}-from-link", tmpDir+"/sub/link-target") } + + { + templateFilePath := tmpDir + "/.gitea/template" + + _ = os.Remove(templateFilePath) + _, err := os.Lstat(templateFilePath) + require.ErrorIs(t, err, fs.ErrNotExist) + _, err = readGiteaTemplateFile(tmpDir) // no template file + require.ErrorIs(t, err, fs.ErrNotExist) + + _ = os.WriteFile(templateFilePath+".target", []byte("test-data-target"), 0o644) + _ = os.Symlink(templateFilePath+".target", templateFilePath) + content, _ := os.ReadFile(templateFilePath) + require.Equal(t, "test-data-target", string(content)) + _, err = readGiteaTemplateFile(tmpDir) // symlinked template file + require.ErrorIs(t, err, fs.ErrNotExist) + + _ = os.Remove(templateFilePath) + _ = os.WriteFile(templateFilePath, []byte("test-data-regular"), 0o644) + content, _ = os.ReadFile(templateFilePath) + require.Equal(t, "test-data-regular", string(content)) + fm, err := readGiteaTemplateFile(tmpDir) // regular template file + require.NoError(t, err) + assert.Len(t, fm.globs, 1) + } } func TestTransformers(t *testing.T) { diff --git a/services/webhook/deliver.go b/services/webhook/deliver.go index b6611a3576..58fba9f68d 100644 --- a/services/webhook/deliver.go +++ b/services/webhook/deliver.go @@ -30,6 +30,7 @@ import ( "code.gitea.io/gitea/modules/queue" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/modules/util" webhook_module "code.gitea.io/gitea/modules/webhook" ) @@ -264,7 +265,7 @@ func Deliver(ctx context.Context, t *webhook_model.HookTask) error { t.ResponseInfo.Headers[k] = strings.Join(vals, ",") } - p, err := io.ReadAll(resp.Body) + p, err := util.ReadWithLimit(resp.Body, 1024*1024) if err != nil { t.ResponseInfo.Body = fmt.Sprintf("read body: %s", err) return fmt.Errorf("unable to deliver webhook task[%d] in %s as unable to read response body: %w", t.ID, w.URL, err)