diff --git a/MAINTAINERS b/MAINTAINERS index d383b8b164..74196d4bd8 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -44,8 +44,7 @@ Janis Estelmann (@KN4CK3R) Steven Kriegler (@justusbunsi) Jimmy Praet (@jpraet) Leon Hofmeister (@delvh) -silentcode (@silentcodeg) Wim (@42wim) -xinyu (@penlinux) +Xinyu Zhou (@xin-u) Jason Song (@wolfogre) Yarden Shoham (@yardenshoham) diff --git a/docs/content/doc/installation/from-source.en-us.md b/docs/content/doc/installation/from-source.en-us.md index 01a5e1eaee..4394d19203 100644 --- a/docs/content/doc/installation/from-source.en-us.md +++ b/docs/content/doc/installation/from-source.en-us.md @@ -145,7 +145,7 @@ launched manually from command line, it can be killed by pressing `Ctrl + C`. Gitea will search for a number of things from the _`CustomPath`_. By default this is the `custom/` directory in the current working directory when running Gitea. It will also -look for its configuration file _`CustomConf`_ in _`CustomPath`_/conf/app.ini`, and will use the +look for its configuration file _`CustomConf`_ in `$(CustomPath)/conf/app.ini`, and will use the current working directory as the relative base path _`AppWorkPath`_ for a number configurable values. Finally the static files will be served from _`StaticRootPath`_ which defaults to the _`AppWorkPath`_. diff --git a/go.mod b/go.mod index 6608029fae..3f096e1dd5 100644 --- a/go.mod +++ b/go.mod @@ -91,6 +91,7 @@ require ( github.com/unrolled/render v1.5.0 github.com/urfave/cli v1.22.10 github.com/xanzy/go-gitlab v0.73.1 + github.com/xeipuuv/gojsonschema v1.2.0 github.com/yohcop/openid-go v1.0.0 github.com/yuin/goldmark v1.5.2 github.com/yuin/goldmark-highlighting/v2 v2.0.0-20220924101305-151362477c87 @@ -277,6 +278,8 @@ require ( github.com/valyala/fastjson v1.6.3 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xanzy/ssh-agent v0.3.2 // indirect + github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect + github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect go.etcd.io/bbolt v1.3.6 // indirect diff --git a/go.sum b/go.sum index b031515017..3f37a8c52a 100644 --- a/go.sum +++ b/go.sum @@ -1487,6 +1487,12 @@ github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+ github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= diff --git a/models/git/lfs.go b/models/git/lfs.go index a86e84c050..8d418b928d 100644 --- a/models/git/lfs.go +++ b/models/git/lfs.go @@ -6,6 +6,7 @@ package git import ( "context" "fmt" + "time" "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/perm" @@ -14,6 +15,7 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/util" @@ -180,6 +182,12 @@ func GetLFSMetaObjectByOid(repoID int64, oid string) (*LFSMetaObject, error) { // RemoveLFSMetaObjectByOid removes a LFSMetaObject entry from database by its OID. // It may return ErrLFSObjectNotExist or a database error. func RemoveLFSMetaObjectByOid(repoID int64, oid string) (int64, error) { + return RemoveLFSMetaObjectByOidFn(repoID, oid, nil) +} + +// RemoveLFSMetaObjectByOidFn removes a LFSMetaObject entry from database by its OID. +// It may return ErrLFSObjectNotExist or a database error. It will run Fn with the current count within the transaction +func RemoveLFSMetaObjectByOidFn(repoID int64, oid string, fn func(count int64) error) (int64, error) { if len(oid) == 0 { return 0, ErrLFSObjectNotExist } @@ -200,6 +208,12 @@ func RemoveLFSMetaObjectByOid(repoID int64, oid string) (int64, error) { return count, err } + if fn != nil { + if err := fn(count); err != nil { + return count, err + } + } + return count, committer.Commit() } @@ -319,3 +333,43 @@ func GetRepoLFSSize(ctx context.Context, repoID int64) (int64, error) { } return lfsSize, nil } + +type IterateLFSMetaObjectsForRepoOptions struct { + OlderThan time.Time +} + +// IterateLFSMetaObjectsForRepo provides a iterator for LFSMetaObjects per Repo +func IterateLFSMetaObjectsForRepo(ctx context.Context, repoID int64, f func(context.Context, *LFSMetaObject, int64) error, opts *IterateLFSMetaObjectsForRepoOptions) error { + var start int + batchSize := setting.Database.IterateBufferSize + engine := db.GetEngine(ctx) + type CountLFSMetaObject struct { + Count int64 + LFSMetaObject + } + + for { + beans := make([]*CountLFSMetaObject, 0, batchSize) + // SELECT `lfs_meta_object`.*, COUNT(`l1`.id) as `count` FROM lfs_meta_object INNER JOIN lfs_meta_object AS l1 ON l1.oid = lfs_meta_object.oid WHERE lfs_meta_object.repository_id = ? GROUP BY lfs_meta_object.id + sess := engine.Select("`lfs_meta_object`.*, COUNT(`l1`.oid) AS `count`"). + Join("INNER", "`lfs_meta_object` AS l1", "`lfs_meta_object`.oid = `l1`.oid"). + Where("`lfs_meta_object`.repository_id = ?", repoID) + if !opts.OlderThan.IsZero() { + sess.And("`lfs_meta_object`.created_unix < ?", opts.OlderThan) + } + sess.GroupBy("`lfs_meta_object`.id") + if err := sess.Limit(batchSize, start).Find(&beans); err != nil { + return err + } + if len(beans) == 0 { + return nil + } + start += len(beans) + + for _, bean := range beans { + if err := f(ctx, &bean.LFSMetaObject, bean.Count); err != nil { + return err + } + } + } +} diff --git a/models/packages/package_version.go b/models/packages/package_version.go index a2a8a45d8f..928f9d47d6 100644 --- a/models/packages/package_version.go +++ b/models/packages/package_version.go @@ -302,9 +302,14 @@ func SearchLatestVersions(ctx context.Context, opts *PackageSearchOptions) ([]*P cond := opts.toConds(). And(builder.Expr("pv2.id IS NULL")) + joinCond := builder.Expr("package_version.package_id = pv2.package_id AND (package_version.created_unix < pv2.created_unix OR (package_version.created_unix = pv2.created_unix AND package_version.id < pv2.id))") + if !opts.IsInternal.IsNone() { + joinCond = joinCond.And(builder.Eq{"pv2.is_internal": opts.IsInternal.IsTrue()}) + } + sess := db.GetEngine(ctx). Table("package_version"). - Join("LEFT", "package_version pv2", "package_version.package_id = pv2.package_id AND (package_version.created_unix < pv2.created_unix OR (package_version.created_unix = pv2.created_unix AND package_version.id < pv2.id))"). + Join("LEFT", "package_version pv2", joinCond). Join("INNER", "package", "package.id = package_version.package_id"). Where(cond) diff --git a/modules/charset/escape.go b/modules/charset/escape.go index ce2eb1446d..3b1c206977 100644 --- a/modules/charset/escape.go +++ b/modules/charset/escape.go @@ -8,6 +8,7 @@ package charset import ( + "bufio" "io" "strings" @@ -31,7 +32,7 @@ func EscapeControlHTML(text string, locale translation.Locale, allowed ...rune) return streamer.escaped, sb.String() } -// EscapeControlReaders escapes the unicode control sequences in a provider reader and writer in a locale and returns the findings as an EscapeStatus and the escaped []byte +// EscapeControlReaders escapes the unicode control sequences in a provided reader of HTML content and writer in a locale and returns the findings as an EscapeStatus and the escaped []byte func EscapeControlReader(reader io.Reader, writer io.Writer, locale translation.Locale, allowed ...rune) (escaped *EscapeStatus, err error) { outputStream := &HTMLStreamerWriter{Writer: writer} streamer := NewEscapeStreamer(locale, outputStream, allowed...).(*escapeStreamer) @@ -43,6 +44,35 @@ func EscapeControlReader(reader io.Reader, writer io.Writer, locale translation. return streamer.escaped, err } +// EscapeControlStringReader escapes the unicode control sequences in a provided reader of string content and writer in a locale and returns the findings as an EscapeStatus and the escaped []byte +func EscapeControlStringReader(reader io.Reader, writer io.Writer, locale translation.Locale, allowed ...rune) (escaped *EscapeStatus, err error) { + bufRd := bufio.NewReader(reader) + outputStream := &HTMLStreamerWriter{Writer: writer} + streamer := NewEscapeStreamer(locale, outputStream, allowed...).(*escapeStreamer) + + for { + line, rdErr := bufRd.ReadString('\n') + if len(line) > 0 { + if err := streamer.Text(line); err != nil { + streamer.escaped.HasError = true + log.Error("Error whilst escaping: %v", err) + return streamer.escaped, err + } + } + if rdErr != nil { + if rdErr != io.EOF { + err = rdErr + } + break + } + if err := streamer.SelfClosingTag("br"); err != nil { + streamer.escaped.HasError = true + return streamer.escaped, err + } + } + return streamer.escaped, err +} + // EscapeControlString escapes the unicode control sequences in a provided string and returns the findings as an EscapeStatus and the escaped string func EscapeControlString(text string, locale translation.Locale, allowed ...rune) (escaped *EscapeStatus, output string) { sb := &strings.Builder{} diff --git a/modules/doctor/lfs.go b/modules/doctor/lfs.go new file mode 100644 index 0000000000..410ed5a9a5 --- /dev/null +++ b/modules/doctor/lfs.go @@ -0,0 +1,37 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package doctor + +import ( + "context" + "fmt" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/services/repository" +) + +func init() { + Register(&Check{ + Title: "Garbage collect LFS", + Name: "gc-lfs", + IsDefault: false, + Run: garbageCollectLFSCheck, + AbortIfFailed: false, + SkipDatabaseInitialization: false, + Priority: 1, + }) +} + +func garbageCollectLFSCheck(ctx context.Context, logger log.Logger, autofix bool) error { + if !setting.LFS.StartServer { + return fmt.Errorf("LFS support is disabled") + } + + if err := repository.GarbageCollectLFSMetaObjects(ctx, logger, autofix); err != nil { + return err + } + + return checkStorage(&checkStorageOptions{LFS: true})(ctx, logger, autofix) +} diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 1e402f81bd..a881d628bf 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -469,6 +469,13 @@ func getAppPath() (string, error) { appPath, err = exec.LookPath(os.Args[0]) } + if err != nil { + // FIXME: Once we switch to go 1.19 use !errors.Is(err, exec.ErrDot) + if !strings.Contains(err.Error(), "cannot run executable found relative to current directory") { + return "", err + } + appPath, err = filepath.Abs(os.Args[0]) + } if err != nil { return "", err } diff --git a/modules/storage/local.go b/modules/storage/local.go index a439a24592..ca51d26c9a 100644 --- a/modules/storage/local.go +++ b/modules/storage/local.go @@ -102,7 +102,8 @@ func (l *LocalStorage) Save(path string, r io.Reader, size int64) (int64, error) return 0, err } // Golang's tmp file (os.CreateTemp) always have 0o600 mode, so we need to change the file to follow the umask (as what Create/MkDir does) - if err := util.ApplyUmask(p, os.ModePerm); err != nil { + // but we don't want to make these files executable - so ensure that we mask out the executable bits + if err := util.ApplyUmask(p, os.ModePerm&0o666); err != nil { return 0, err } diff --git a/options/license/LOOP b/options/license/LOOP new file mode 100644 index 0000000000..434d2c45e2 --- /dev/null +++ b/options/license/LOOP @@ -0,0 +1,44 @@ +Portions of LOOP are Copyright (c) 1986 by the Massachusetts Institute of Technology. +All Rights Reserved. + +Permission to use, copy, modify and distribute this software and its +documentation for any purpose and without fee is hereby granted, +provided that the M.I.T. copyright notice appear in all copies and that +both that copyright notice and this permission notice appear in +supporting documentation. The names "M.I.T." and "Massachusetts +Institute of Technology" may not be used in advertising or publicity +pertaining to distribution of the software without specific, written +prior permission. Notice must be given in supporting documentation that +copying distribution is by permission of M.I.T. M.I.T. makes no +representations about the suitability of this software for any purpose. +It is provided "as is" without express or implied warranty. + +Massachusetts Institute of Technology +77 Massachusetts Avenue +Cambridge, Massachusetts 02139 +United States of America ++1-617-253-1000 + +Portions of LOOP are Copyright (c) 1989, 1990, 1991, 1992 by Symbolics, Inc. +All Rights Reserved. + +Permission to use, copy, modify and distribute this software and its +documentation for any purpose and without fee is hereby granted, +provided that the Symbolics copyright notice appear in all copies and +that both that copyright notice and this permission notice appear in +supporting documentation. The name "Symbolics" may not be used in +advertising or publicity pertaining to distribution of the software +without specific, written prior permission. Notice must be given in +supporting documentation that copying distribution is by permission of +Symbolics. Symbolics makes no representations about the suitability of +this software for any purpose. It is provided "as is" without express +or implied warranty. + +Symbolics, CLOE Runtime, and Minima are trademarks, and CLOE, Genera, +and Zetalisp are registered trademarks of Symbolics, Inc. + +Symbolics, Inc. +8 New England Executive Park, East +Burlington, Massachusetts 01803 +United States of America ++1-617-221-1000 diff --git a/routers/api/packages/npm/npm.go b/routers/api/packages/npm/npm.go index 4306d5ba36..1a09cb6f36 100644 --- a/routers/api/packages/npm/npm.go +++ b/routers/api/packages/npm/npm.go @@ -405,8 +405,9 @@ func setPackageTag(tag string, pv *packages_model.PackageVersion, deleteOnly boo func PackageSearch(ctx *context.Context) { pvs, total, err := packages_model.SearchLatestVersions(ctx, &packages_model.PackageSearchOptions{ - OwnerID: ctx.Package.Owner.ID, - Type: packages_model.TypeNpm, + OwnerID: ctx.Package.Owner.ID, + Type: packages_model.TypeNpm, + IsInternal: util.OptionalBoolFalse, Name: packages_model.SearchValue{ ExactMatch: false, Value: ctx.FormTrim("text"), diff --git a/routers/web/repo/view.go b/routers/web/repo/view.go index fa4eb6d61f..3e3a4efc31 100644 --- a/routers/web/repo/view.go +++ b/routers/web/repo/view.go @@ -9,7 +9,6 @@ import ( gocontext "context" "encoding/base64" "fmt" - gotemplate "html/template" "io" "net/http" "net/url" @@ -241,18 +240,19 @@ func findReadmeFile(ctx *context.Context, entries git.Entries, treeLink string) return readmeFile, readmeTreelink } -func renderReadmeFile(ctx *context.Context, readmeFile *namedBlob, readmeTreelink string) { - ctx.Data["RawFileLink"] = "" - ctx.Data["ReadmeInList"] = true - ctx.Data["ReadmeExist"] = true - ctx.Data["FileIsSymlink"] = readmeFile.isSymlink +type fileInfo struct { + isTextFile bool + isLFSFile bool + fileSize int64 + lfsMeta *lfs.Pointer + st typesniffer.SniffedType +} - dataRc, err := readmeFile.blob.DataAsync() +func getFileReader(repoID int64, blob *git.Blob) ([]byte, io.ReadCloser, *fileInfo, error) { + dataRc, err := blob.DataAsync() if err != nil { - ctx.ServerError("Data", err) - return + return nil, nil, nil, err } - defer dataRc.Close() buf := make([]byte, 1024) n, _ := util.ReadAtMost(dataRc, buf) @@ -261,67 +261,75 @@ func renderReadmeFile(ctx *context.Context, readmeFile *namedBlob, readmeTreelin st := typesniffer.DetectContentType(buf) isTextFile := st.IsText() - ctx.Data["FileIsText"] = isTextFile - ctx.Data["FileName"] = readmeFile.name - fileSize := int64(0) - isLFSFile := false - ctx.Data["IsLFSFile"] = false - // FIXME: what happens when README file is an image? - if isTextFile && setting.LFS.StartServer { - pointer, _ := lfs.ReadPointerFromBuffer(buf) - if pointer.IsValid() { - meta, err := git_model.GetLFSMetaObjectByOid(ctx.Repo.Repository.ID, pointer.Oid) - if err != nil && err != git_model.ErrLFSObjectNotExist { - ctx.ServerError("GetLFSMetaObject", err) - return - } - if meta != nil { - ctx.Data["IsLFSFile"] = true - isLFSFile = true - - // OK read the lfs object - var err error - dataRc, err = lfs.ReadMetaObject(pointer) - if err != nil { - ctx.ServerError("ReadMetaObject", err) - return - } - defer dataRc.Close() - - buf = make([]byte, 1024) - n, err = util.ReadAtMost(dataRc, buf) - if err != nil { - ctx.ServerError("Data", err) - return - } - buf = buf[:n] - - st = typesniffer.DetectContentType(buf) - isTextFile = st.IsText() - ctx.Data["IsTextFile"] = isTextFile - - fileSize = meta.Size - ctx.Data["FileSize"] = meta.Size - filenameBase64 := base64.RawURLEncoding.EncodeToString([]byte(readmeFile.name)) - ctx.Data["RawFileLink"] = fmt.Sprintf("%s.git/info/lfs/objects/%s/%s", ctx.Repo.Repository.HTMLURL(), url.PathEscape(meta.Oid), url.PathEscape(filenameBase64)) - } - } + if !isTextFile || !setting.LFS.StartServer { + return buf, dataRc, &fileInfo{isTextFile, false, blob.Size(), nil, st}, nil } - if !isTextFile { + pointer, _ := lfs.ReadPointerFromBuffer(buf) + if !pointer.IsValid() { // fallback to plain file + return buf, dataRc, &fileInfo{isTextFile, false, blob.Size(), nil, st}, nil + } + + meta, err := git_model.GetLFSMetaObjectByOid(repoID, pointer.Oid) + if err != git_model.ErrLFSObjectNotExist { // fallback to plain file + return buf, dataRc, &fileInfo{isTextFile, false, blob.Size(), nil, st}, nil + } + + dataRc.Close() + if err != nil { + return nil, nil, nil, err + } + + dataRc, err = lfs.ReadMetaObject(pointer) + if err != nil { + return nil, nil, nil, err + } + + buf = make([]byte, 1024) + n, err = util.ReadAtMost(dataRc, buf) + if err != nil { + dataRc.Close() + return nil, nil, nil, err + } + buf = buf[:n] + + st = typesniffer.DetectContentType(buf) + + return buf, dataRc, &fileInfo{st.IsText(), true, meta.Size, &meta.Pointer, st}, nil +} + +func renderReadmeFile(ctx *context.Context, readmeFile *namedBlob, readmeTreelink string) { + ctx.Data["RawFileLink"] = "" + ctx.Data["ReadmeInList"] = true + ctx.Data["ReadmeExist"] = true + ctx.Data["FileIsSymlink"] = readmeFile.isSymlink + + buf, dataRc, fInfo, err := getFileReader(ctx.Repo.Repository.ID, readmeFile.blob) + if err != nil { + ctx.ServerError("getFileReader", err) + return + } + defer dataRc.Close() + + ctx.Data["FileIsText"] = fInfo.isTextFile + ctx.Data["FileName"] = readmeFile.name + ctx.Data["IsLFSFile"] = fInfo.isLFSFile + + if fInfo.isLFSFile { + filenameBase64 := base64.RawURLEncoding.EncodeToString([]byte(readmeFile.name)) + ctx.Data["RawFileLink"] = fmt.Sprintf("%s.git/info/lfs/objects/%s/%s", ctx.Repo.Repository.HTMLURL(), url.PathEscape(fInfo.lfsMeta.Oid), url.PathEscape(filenameBase64)) + } + + if !fInfo.isTextFile { return } - if !isLFSFile { - fileSize = readmeFile.blob.Size() - } - - if fileSize >= setting.UI.MaxDisplayFileSize { + if fInfo.fileSize >= setting.UI.MaxDisplayFileSize { // Pretend that this is a normal text file to display 'This file is too large to be shown' ctx.Data["IsFileTooLarge"] = true ctx.Data["IsTextFile"] = true - ctx.Data["FileSize"] = fileSize + ctx.Data["FileSize"] = fInfo.fileSize return } @@ -341,15 +349,13 @@ func renderReadmeFile(ctx *context.Context, readmeFile *namedBlob, readmeTreelin if err != nil { log.Error("Render failed for %s in %-v: %v Falling back to rendering source", readmeFile.name, ctx.Repo.Repository, err) buf := &bytes.Buffer{} - ctx.Data["EscapeStatus"], _ = charset.EscapeControlReader(rd, buf, ctx.Locale) - ctx.Data["FileContent"] = strings.ReplaceAll( - gotemplate.HTMLEscapeString(buf.String()), "\n", `
`, - ) + ctx.Data["EscapeStatus"], _ = charset.EscapeControlStringReader(rd, buf, ctx.Locale) + ctx.Data["FileContent"] = buf.String() } } else { - ctx.Data["IsRenderedHTML"] = true + ctx.Data["IsPlainText"] = true buf := &bytes.Buffer{} - ctx.Data["EscapeStatus"], err = charset.EscapeControlReader(rd, &charset.BreakWriter{Writer: buf}, ctx.Locale, charset.RuneNBSP) + ctx.Data["EscapeStatus"], err = charset.EscapeControlStringReader(rd, buf, ctx.Locale) if err != nil { log.Error("Read failed: %v", err) } @@ -362,16 +368,14 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st ctx.Data["IsViewFile"] = true ctx.Data["HideRepoInfo"] = true blob := entry.Blob() - dataRc, err := blob.DataAsync() + buf, dataRc, fInfo, err := getFileReader(ctx.Repo.Repository.ID, blob) if err != nil { - ctx.ServerError("DataAsync", err) + ctx.ServerError("getFileReader", err) return } defer dataRc.Close() ctx.Data["Title"] = ctx.Tr("repo.file.title", ctx.Repo.Repository.Name+"/"+path.Base(ctx.Repo.TreePath), ctx.Repo.RefName) - - fileSize := blob.Size() ctx.Data["FileIsSymlink"] = entry.IsLink() ctx.Data["FileName"] = blob.Name() ctx.Data["RawFileLink"] = rawLink + "/" + util.PathEscapeSegments(ctx.Repo.TreePath) @@ -381,69 +385,27 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st ctx.Data["FileError"] = editorconfigErr } - buf := make([]byte, 1024) - n, _ := util.ReadAtMost(dataRc, buf) - buf = buf[:n] - - st := typesniffer.DetectContentType(buf) - isTextFile := st.IsText() - - isLFSFile := false isDisplayingSource := ctx.FormString("display") == "source" isDisplayingRendered := !isDisplayingSource - // Check for LFS meta file - if isTextFile && setting.LFS.StartServer { - pointer, _ := lfs.ReadPointerFromBuffer(buf) - if pointer.IsValid() { - meta, err := git_model.GetLFSMetaObjectByOid(ctx.Repo.Repository.ID, pointer.Oid) - if err != nil && err != git_model.ErrLFSObjectNotExist { - ctx.ServerError("GetLFSMetaObject", err) - return - } - if meta != nil { - isLFSFile = true - - // OK read the lfs object - var err error - dataRc, err = lfs.ReadMetaObject(pointer) - if err != nil { - ctx.ServerError("ReadMetaObject", err) - return - } - defer dataRc.Close() - - buf = make([]byte, 1024) - n, err = util.ReadAtMost(dataRc, buf) - if err != nil { - ctx.ServerError("Data", err) - return - } - buf = buf[:n] - - st = typesniffer.DetectContentType(buf) - isTextFile = st.IsText() - - fileSize = meta.Size - ctx.Data["RawFileLink"] = ctx.Repo.RepoLink + "/media/" + ctx.Repo.BranchNameSubURL() + "/" + util.PathEscapeSegments(ctx.Repo.TreePath) - } - } + if fInfo.isLFSFile { + ctx.Data["RawFileLink"] = ctx.Repo.RepoLink + "/media/" + ctx.Repo.BranchNameSubURL() + "/" + util.PathEscapeSegments(ctx.Repo.TreePath) } - isRepresentableAsText := st.IsRepresentableAsText() + isRepresentableAsText := fInfo.st.IsRepresentableAsText() if !isRepresentableAsText { // If we can't show plain text, always try to render. isDisplayingSource = false isDisplayingRendered = true } - ctx.Data["IsLFSFile"] = isLFSFile - ctx.Data["FileSize"] = fileSize - ctx.Data["IsTextFile"] = isTextFile + ctx.Data["IsLFSFile"] = fInfo.isLFSFile + ctx.Data["FileSize"] = fInfo.fileSize + ctx.Data["IsTextFile"] = fInfo.isTextFile ctx.Data["IsRepresentableAsText"] = isRepresentableAsText ctx.Data["IsDisplayingSource"] = isDisplayingSource ctx.Data["IsDisplayingRendered"] = isDisplayingRendered - isTextSource := isTextFile || isDisplayingSource + isTextSource := fInfo.isTextFile || isDisplayingSource ctx.Data["IsTextSource"] = isTextSource if isTextSource { ctx.Data["CanCopyContent"] = true @@ -468,7 +430,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st } // Assume file is not editable first. - if isLFSFile { + if fInfo.isLFSFile { ctx.Data["EditFileTooltip"] = ctx.Tr("repo.editor.cannot_edit_lfs_files") } else if !isRepresentableAsText { ctx.Data["EditFileTooltip"] = ctx.Tr("repo.editor.cannot_edit_non_text_files") @@ -476,13 +438,13 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st switch { case isRepresentableAsText: - if st.IsSvgImage() { + if fInfo.st.IsSvgImage() { ctx.Data["IsImageFile"] = true ctx.Data["CanCopyContent"] = true ctx.Data["HasSourceRenderedToggle"] = true } - if fileSize >= setting.UI.MaxDisplayFileSize { + if fInfo.fileSize >= setting.UI.MaxDisplayFileSize { ctx.Data["IsFileTooLarge"] = true break } @@ -527,15 +489,6 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st } // to prevent iframe load third-party url ctx.Resp.Header().Add("Content-Security-Policy", "frame-src 'self'") - } else if readmeExist && !shouldRenderSource { - buf := &bytes.Buffer{} - ctx.Data["IsRenderedHTML"] = true - - ctx.Data["EscapeStatus"], _ = charset.EscapeControlReader(rd, buf, ctx.Locale) - - ctx.Data["FileContent"] = strings.ReplaceAll( - gotemplate.HTMLEscapeString(buf.String()), "\n", `
`, - ) } else { buf, _ := io.ReadAll(rd) @@ -589,7 +542,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st ctx.Data["FileContent"] = fileContent ctx.Data["LineEscapeStatus"] = statuses } - if !isLFSFile { + if !fInfo.isLFSFile { if ctx.Repo.CanEnableEditor(ctx.Doer) { if lfsLock != nil && lfsLock.OwnerID != ctx.Doer.ID { ctx.Data["CanEditFile"] = false @@ -605,17 +558,17 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st } } - case st.IsPDF(): + case fInfo.st.IsPDF(): ctx.Data["IsPDFFile"] = true - case st.IsVideo(): + case fInfo.st.IsVideo(): ctx.Data["IsVideoFile"] = true - case st.IsAudio(): + case fInfo.st.IsAudio(): ctx.Data["IsAudioFile"] = true - case st.IsImage() && (setting.UI.SVG.Enabled || !st.IsSvgImage()): + case fInfo.st.IsImage() && (setting.UI.SVG.Enabled || !fInfo.st.IsSvgImage()): ctx.Data["IsImageFile"] = true ctx.Data["CanCopyContent"] = true default: - if fileSize >= setting.UI.MaxDisplayFileSize { + if fInfo.fileSize >= setting.UI.MaxDisplayFileSize { ctx.Data["IsFileTooLarge"] = true break } diff --git a/services/cron/tasks_basic.go b/services/cron/tasks_basic.go index acf3896b71..05aef6623d 100644 --- a/services/cron/tasks_basic.go +++ b/services/cron/tasks_basic.go @@ -63,7 +63,7 @@ func registerRepoHealthCheck() { for _, arg := range rhcConfig.Args { args = append(args, git.CmdArg(arg)) } - return repo_service.GitFsck(ctx, rhcConfig.Timeout, args) + return repo_service.GitFsckRepos(ctx, rhcConfig.Timeout, args) }) } diff --git a/services/pull/patch.go b/services/pull/patch.go index 809b75e6b4..e0da410c4d 100644 --- a/services/pull/patch.go +++ b/services/pull/patch.go @@ -53,6 +53,8 @@ var patchErrorSuffices = []string{ ": patch does not apply", ": already exists in working directory", "unrecognized input", + ": No such file or directory", + ": does not exist in index", } // TestPatch will test whether a simple patch will apply @@ -416,6 +418,7 @@ func checkConflicts(ctx context.Context, pr *issues_model.PullRequest, gitRepo * scanner := bufio.NewScanner(stderrReader) for scanner.Scan() { line := scanner.Text() + log.Trace("PullRequest[%d].testPatch: stderr: %s", pr.ID, line) if strings.HasPrefix(line, prefix) { conflict = true filepath := strings.TrimSpace(strings.Split(line[len(prefix):], ":")[0]) diff --git a/services/repository/adopt.go b/services/repository/adopt.go index 828068e8ec..93eeb56456 100644 --- a/services/repository/adopt.go +++ b/services/repository/adopt.go @@ -337,7 +337,7 @@ func ListUnadoptedRepositories(query string, opts *db.ListOptions) ([]string, in } repoNamesToCheck = append(repoNamesToCheck, name) - if len(repoNamesToCheck) > setting.Database.IterateBufferSize { + if len(repoNamesToCheck) >= setting.Database.IterateBufferSize { if err = checkUnadoptedRepositories(userName, repoNamesToCheck, unadopted); err != nil { return err } diff --git a/services/repository/check.go b/services/repository/check.go index 6e29dc93d1..293cb04d38 100644 --- a/services/repository/check.go +++ b/services/repository/check.go @@ -22,8 +22,8 @@ import ( "xorm.io/builder" ) -// GitFsck calls 'git fsck' to check repository health. -func GitFsck(ctx context.Context, timeout time.Duration, args []git.CmdArg) error { +// GitFsckRepos calls 'git fsck' to check repository health. +func GitFsckRepos(ctx context.Context, timeout time.Duration, args []git.CmdArg) error { log.Trace("Doing: GitFsck") if err := db.Iterate( @@ -35,15 +35,7 @@ func GitFsck(ctx context.Context, timeout time.Duration, args []git.CmdArg) erro return db.ErrCancelledf("before fsck of %s", repo.FullName()) default: } - log.Trace("Running health check on repository %v", repo) - repoPath := repo.RepoPath() - if err := git.Fsck(ctx, repoPath, timeout, args...); err != nil { - log.Warn("Failed to health check repository (%v): %v", repo, err) - if err = system_model.CreateRepositoryNotice("Failed to health check repository (%s): %v", repo.FullName(), err); err != nil { - log.Error("CreateRepositoryNotice: %v", err) - } - } - return nil + return GitFsckRepo(ctx, repo, timeout, args) }, ); err != nil { log.Trace("Error: GitFsck: %v", err) @@ -54,6 +46,19 @@ func GitFsck(ctx context.Context, timeout time.Duration, args []git.CmdArg) erro return nil } +// GitFsckRepo calls 'git fsck' to check an individual repository's health. +func GitFsckRepo(ctx context.Context, repo *repo_model.Repository, timeout time.Duration, args []git.CmdArg) error { + log.Trace("Running health check on repository %-v", repo) + repoPath := repo.RepoPath() + if err := git.Fsck(ctx, repoPath, timeout, args...); err != nil { + log.Warn("Failed to health check repository (%-v): %v", repo, err) + if err = system_model.CreateRepositoryNotice("Failed to health check repository (%s): %v", repo.FullName(), err); err != nil { + log.Error("CreateRepositoryNotice: %v", err) + } + } + return nil +} + // GitGcRepos calls 'git gc' to remove unnecessary files and optimize the local repository func GitGcRepos(ctx context.Context, timeout time.Duration, args ...git.CmdArg) error { log.Trace("Doing: GitGcRepos") @@ -68,33 +73,7 @@ func GitGcRepos(ctx context.Context, timeout time.Duration, args ...git.CmdArg) return db.ErrCancelledf("before GC of %s", repo.FullName()) default: } - log.Trace("Running git gc on %v", repo) - command := git.NewCommand(ctx, args...). - SetDescription(fmt.Sprintf("Repository Garbage Collection: %s", repo.FullName())) - var stdout string - var err error - stdout, _, err = command.RunStdString(&git.RunOpts{Timeout: timeout, Dir: repo.RepoPath()}) - - if err != nil { - log.Error("Repository garbage collection failed for %v. Stdout: %s\nError: %v", repo, stdout, err) - desc := fmt.Sprintf("Repository garbage collection failed for %s. Stdout: %s\nError: %v", repo.RepoPath(), stdout, err) - if err = system_model.CreateRepositoryNotice(desc); err != nil { - log.Error("CreateRepositoryNotice: %v", err) - } - return fmt.Errorf("Repository garbage collection failed in repo: %s: Error: %w", repo.FullName(), err) - } - - // Now update the size of the repository - if err := repo_module.UpdateRepoSize(ctx, repo); err != nil { - log.Error("Updating size as part of garbage collection failed for %v. Stdout: %s\nError: %v", repo, stdout, err) - desc := fmt.Sprintf("Updating size as part of garbage collection failed for %s. Stdout: %s\nError: %v", repo.RepoPath(), stdout, err) - if err = system_model.CreateRepositoryNotice(desc); err != nil { - log.Error("CreateRepositoryNotice: %v", err) - } - return fmt.Errorf("Updating size as part of garbage collection failed in repo: %s: Error: %w", repo.FullName(), err) - } - - return nil + return GitGcRepo(ctx, repo, timeout, args) }, ); err != nil { return err @@ -104,6 +83,37 @@ func GitGcRepos(ctx context.Context, timeout time.Duration, args ...git.CmdArg) return nil } +// GitGcRepo calls 'git gc' to remove unnecessary files and optimize the local repository +func GitGcRepo(ctx context.Context, repo *repo_model.Repository, timeout time.Duration, args []git.CmdArg) error { + log.Trace("Running git gc on %-v", repo) + command := git.NewCommand(ctx, args...). + SetDescription(fmt.Sprintf("Repository Garbage Collection: %s", repo.FullName())) + var stdout string + var err error + stdout, _, err = command.RunStdString(&git.RunOpts{Timeout: timeout, Dir: repo.RepoPath()}) + + if err != nil { + log.Error("Repository garbage collection failed for %v. Stdout: %s\nError: %v", repo, stdout, err) + desc := fmt.Sprintf("Repository garbage collection failed for %s. Stdout: %s\nError: %v", repo.RepoPath(), stdout, err) + if err = system_model.CreateRepositoryNotice(desc); err != nil { + log.Error("CreateRepositoryNotice: %v", err) + } + return fmt.Errorf("Repository garbage collection failed in repo: %s: Error: %w", repo.FullName(), err) + } + + // Now update the size of the repository + if err := repo_module.UpdateRepoSize(ctx, repo); err != nil { + log.Error("Updating size as part of garbage collection failed for %-v. Stdout: %s\nError: %v", repo, stdout, err) + desc := fmt.Sprintf("Updating size as part of garbage collection failed for %s. Stdout: %s\nError: %v", repo.RepoPath(), stdout, err) + if err = system_model.CreateRepositoryNotice(desc); err != nil { + log.Error("CreateRepositoryNotice: %v", err) + } + return fmt.Errorf("Updating size as part of garbage collection failed in repo: %s: Error: %w", repo.FullName(), err) + } + + return nil +} + func gatherMissingRepoRecords(ctx context.Context) ([]*repo_model.Repository, error) { repos := make([]*repo_model.Repository, 0, 10) if err := db.Iterate( diff --git a/services/repository/lfs.go b/services/repository/lfs.go new file mode 100644 index 0000000000..0e88d359a8 --- /dev/null +++ b/services/repository/lfs.go @@ -0,0 +1,105 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repository + +import ( + "context" + "fmt" + "time" + + "code.gitea.io/gitea/models/db" + git_model "code.gitea.io/gitea/models/git" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/lfs" + "code.gitea.io/gitea/modules/log" + + "xorm.io/builder" +) + +func GarbageCollectLFSMetaObjects(ctx context.Context, logger log.Logger, autofix bool) error { + log.Trace("Doing: GarbageCollectLFSMetaObjects") + + if err := db.Iterate( + ctx, + builder.And(builder.Gt{"id": 0}), + func(ctx context.Context, repo *repo_model.Repository) error { + return GarbageCollectLFSMetaObjectsForRepo(ctx, repo, logger, autofix) + }, + ); err != nil { + return err + } + + log.Trace("Finished: GarbageCollectLFSMetaObjects") + return nil +} + +func GarbageCollectLFSMetaObjectsForRepo(ctx context.Context, repo *repo_model.Repository, logger log.Logger, autofix bool) error { + if logger != nil { + logger.Info("Checking %-v", repo) + } + total, orphaned, collected, deleted := 0, 0, 0, 0 + if logger != nil { + defer func() { + if orphaned == 0 { + logger.Info("Found %d total LFSMetaObjects in %-v", total, repo) + } else if !autofix { + logger.Info("Found %d/%d orphaned LFSMetaObjects in %-v", orphaned, total, repo) + } else { + logger.Info("Collected %d/%d orphaned/%d total LFSMetaObjects in %-v. %d removed from storage.", collected, orphaned, total, repo, deleted) + } + }() + } + + gitRepo, err := git.OpenRepository(ctx, repo.RepoPath()) + if err != nil { + log.Error("Unable to open git repository %-v: %v", repo, err) + return err + } + defer gitRepo.Close() + + store := lfs.NewContentStore() + + return git_model.IterateLFSMetaObjectsForRepo(ctx, repo.ID, func(ctx context.Context, metaObject *git_model.LFSMetaObject, count int64) error { + total++ + pointerSha := git.ComputeBlobHash([]byte(metaObject.Pointer.StringContent())) + + if gitRepo.IsObjectExist(pointerSha.String()) { + return nil + } + orphaned++ + + if !autofix { + return nil + } + // Non-existent pointer file + _, err = git_model.RemoveLFSMetaObjectByOidFn(repo.ID, metaObject.Oid, func(count int64) error { + if count > 0 { + return nil + } + + if err := store.Delete(metaObject.RelativePath()); err != nil { + log.Error("Unable to remove lfs metaobject %s from store: %v", metaObject.Oid, err) + } + deleted++ + return nil + }) + if err != nil { + return fmt.Errorf("unable to remove meta-object %s in %s: %w", metaObject.Oid, repo.FullName(), err) + } + collected++ + + return nil + }, &git_model.IterateLFSMetaObjectsForRepoOptions{ + // Only attempt to garbage collect lfs meta objects older than a week as the order of git lfs upload + // and git object upload is not necessarily guaranteed. It's possible to imagine a situation whereby + // an LFS object is uploaded but the git branch is not uploaded immediately, or there are some rapid + // changes in new branches that might lead to lfs objects becoming temporarily unassociated with git + // objects. + // + // It is likely that a week is potentially excessive but it should definitely be enough that any + // unassociated LFS object is genuinely unassociated. + OlderThan: time.Now().Add(-24 * 7 * time.Hour), + }) +} diff --git a/templates/repo/settings/lfs_file.tmpl b/templates/repo/settings/lfs_file.tmpl index ce3c39eac2..6d20aaddef 100644 --- a/templates/repo/settings/lfs_file.tmpl +++ b/templates/repo/settings/lfs_file.tmpl @@ -17,11 +17,11 @@
{{template "repo/unicode_escape_prompt" dict "EscapeStatus" .EscapeStatus "root" $}} -
+
{{if .IsMarkup}} {{if .FileContent}}{{.FileContent | Safe}}{{end}} - {{else if .IsRenderedHTML}} - 
{{if .FileContent}}{{.FileContent | Str2html}}{{end}}
+ {{else if .IsPlainText}} + 
{{if .FileContent}}{{.FileContent | Safe}}{{end}}
{{else if not .IsTextFile}}
{{if .IsImageFile}} diff --git a/templates/repo/view_file.tmpl b/templates/repo/view_file.tmpl index 9d82cc018c..711a37461e 100644 --- a/templates/repo/view_file.tmpl +++ b/templates/repo/view_file.tmpl @@ -61,11 +61,11 @@ {{if not (or .IsMarkup .IsRenderedHTML)}} {{template "repo/unicode_escape_prompt" dict "EscapeStatus" .EscapeStatus "root" $}} {{end}} -
+
{{if .IsMarkup}} {{if .FileContent}}{{.FileContent | Safe}}{{end}} - {{else if .IsRenderedHTML}} - 
{{if .FileContent}}{{.FileContent | Str2html}}{{end}}
+ {{else if .IsPlainText}} + 
{{if .FileContent}}{{.FileContent | Safe}}{{end}}
{{else if not .IsTextSource}}
{{if .IsImageFile}} diff --git a/templates/user/dashboard/repolist.tmpl b/templates/user/dashboard/repolist.tmpl index 620cc322f0..9d0ea01ec5 100644 --- a/templates/user/dashboard/repolist.tmpl +++ b/templates/user/dashboard/repolist.tmpl @@ -128,9 +128,9 @@