mirror of
https://github.com/go-gitea/gitea.git
synced 2025-04-08 17:05:45 +02:00
Merge branch 'main' into feature/bots
This commit is contained in:
Jason Song
GitHub
commit
ab98c3ff43
MAINTAINERS
docs/content/doc/installation
go.modgo.summodels
modules
options/license
routers
services
templates
tests/integration
web_src/less
@ -44,8 +44,7 @@ Janis Estelmann <admin@oldschoolhack.me> (@KN4CK3R)
|
||||
Steven Kriegler <sk.bunsenbrenner@gmail.com> (@justusbunsi)
|
||||
Jimmy Praet <jimmy.praet@telenet.be> (@jpraet)
|
||||
Leon Hofmeister <dev.lh@web.de> (@delvh)
|
||||
silentcode <silentcode@senga.org> (@silentcodeg)
|
||||
Wim <wim@42.be> (@42wim)
|
||||
xinyu <xinyu@nerv.org.cn> (@penlinux)
|
||||
Xinyu Zhou <i@sourcehut.net> (@xin-u)
|
||||
Jason Song <i@wolfogre.com> (@wolfogre)
|
||||
Yarden Shoham <hrsi88@gmail.com> (@yardenshoham)
|
||||
|
||||
@ -145,7 +145,7 @@ launched manually from command line, it can be killed by pressing `Ctrl + C`.
|
||||
|
||||
Gitea will search for a number of things from the _`CustomPath`_. By default this is
|
||||
the `custom/` directory in the current working directory when running Gitea. It will also
|
||||
look for its configuration file _`CustomConf`_ in _`CustomPath`_/conf/app.ini`, and will use the
|
||||
look for its configuration file _`CustomConf`_ in `$(CustomPath)/conf/app.ini`, and will use the
|
||||
current working directory as the relative base path _`AppWorkPath`_ for a number configurable
|
||||
values. Finally the static files will be served from _`StaticRootPath`_ which defaults to the _`AppWorkPath`_.
|
||||
|
||||
|
||||
3
go.mod
3
go.mod
@ -91,6 +91,7 @@ require (
|
||||
github.com/unrolled/render v1.5.0
|
||||
github.com/urfave/cli v1.22.10
|
||||
github.com/xanzy/go-gitlab v0.73.1
|
||||
github.com/xeipuuv/gojsonschema v1.2.0
|
||||
github.com/yohcop/openid-go v1.0.0
|
||||
github.com/yuin/goldmark v1.5.2
|
||||
github.com/yuin/goldmark-highlighting/v2 v2.0.0-20220924101305-151362477c87
|
||||
@ -277,6 +278,8 @@ require (
|
||||
github.com/valyala/fastjson v1.6.3 // indirect
|
||||
github.com/x448/float16 v0.8.4 // indirect
|
||||
github.com/xanzy/ssh-agent v0.3.2 // indirect
|
||||
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
|
||||
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
|
||||
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
|
||||
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
|
||||
go.etcd.io/bbolt v1.3.6 // indirect
|
||||
|
||||
6
go.sum
6
go.sum
@ -1487,6 +1487,12 @@ github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+
|
||||
github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
|
||||
github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM=
|
||||
github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
|
||||
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c=
|
||||
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
|
||||
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
|
||||
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
|
||||
github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
|
||||
github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
|
||||
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo=
|
||||
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos=
|
||||
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8=
|
||||
|
||||
@ -6,6 +6,7 @@ package git
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"code.gitea.io/gitea/models/db"
|
||||
"code.gitea.io/gitea/models/perm"
|
||||
@ -14,6 +15,7 @@ import (
|
||||
user_model "code.gitea.io/gitea/models/user"
|
||||
"code.gitea.io/gitea/modules/lfs"
|
||||
"code.gitea.io/gitea/modules/log"
|
||||
"code.gitea.io/gitea/modules/setting"
|
||||
"code.gitea.io/gitea/modules/timeutil"
|
||||
"code.gitea.io/gitea/modules/util"
|
||||
|
||||
@ -180,6 +182,12 @@ func GetLFSMetaObjectByOid(repoID int64, oid string) (*LFSMetaObject, error) {
|
||||
// RemoveLFSMetaObjectByOid removes a LFSMetaObject entry from database by its OID.
|
||||
// It may return ErrLFSObjectNotExist or a database error.
|
||||
func RemoveLFSMetaObjectByOid(repoID int64, oid string) (int64, error) {
|
||||
return RemoveLFSMetaObjectByOidFn(repoID, oid, nil)
|
||||
}
|
||||
|
||||
// RemoveLFSMetaObjectByOidFn removes a LFSMetaObject entry from database by its OID.
|
||||
// It may return ErrLFSObjectNotExist or a database error. It will run Fn with the current count within the transaction
|
||||
func RemoveLFSMetaObjectByOidFn(repoID int64, oid string, fn func(count int64) error) (int64, error) {
|
||||
if len(oid) == 0 {
|
||||
return 0, ErrLFSObjectNotExist
|
||||
}
|
||||
@ -200,6 +208,12 @@ func RemoveLFSMetaObjectByOid(repoID int64, oid string) (int64, error) {
|
||||
return count, err
|
||||
}
|
||||
|
||||
if fn != nil {
|
||||
if err := fn(count); err != nil {
|
||||
return count, err
|
||||
}
|
||||
}
|
||||
|
||||
return count, committer.Commit()
|
||||
}
|
||||
|
||||
@ -319,3 +333,43 @@ func GetRepoLFSSize(ctx context.Context, repoID int64) (int64, error) {
|
||||
}
|
||||
return lfsSize, nil
|
||||
}
|
||||
|
||||
type IterateLFSMetaObjectsForRepoOptions struct {
|
||||
OlderThan time.Time
|
||||
}
|
||||
|
||||
// IterateLFSMetaObjectsForRepo provides a iterator for LFSMetaObjects per Repo
|
||||
func IterateLFSMetaObjectsForRepo(ctx context.Context, repoID int64, f func(context.Context, *LFSMetaObject, int64) error, opts *IterateLFSMetaObjectsForRepoOptions) error {
|
||||
var start int
|
||||
batchSize := setting.Database.IterateBufferSize
|
||||
engine := db.GetEngine(ctx)
|
||||
type CountLFSMetaObject struct {
|
||||
Count int64
|
||||
LFSMetaObject
|
||||
}
|
||||
|
||||
for {
|
||||
beans := make([]*CountLFSMetaObject, 0, batchSize)
|
||||
// SELECT `lfs_meta_object`.*, COUNT(`l1`.id) as `count` FROM lfs_meta_object INNER JOIN lfs_meta_object AS l1 ON l1.oid = lfs_meta_object.oid WHERE lfs_meta_object.repository_id = ? GROUP BY lfs_meta_object.id
|
||||
sess := engine.Select("`lfs_meta_object`.*, COUNT(`l1`.oid) AS `count`").
|
||||
Join("INNER", "`lfs_meta_object` AS l1", "`lfs_meta_object`.oid = `l1`.oid").
|
||||
Where("`lfs_meta_object`.repository_id = ?", repoID)
|
||||
if !opts.OlderThan.IsZero() {
|
||||
sess.And("`lfs_meta_object`.created_unix < ?", opts.OlderThan)
|
||||
}
|
||||
sess.GroupBy("`lfs_meta_object`.id")
|
||||
if err := sess.Limit(batchSize, start).Find(&beans); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(beans) == 0 {
|
||||
return nil
|
||||
}
|
||||
start += len(beans)
|
||||
|
||||
for _, bean := range beans {
|
||||
if err := f(ctx, &bean.LFSMetaObject, bean.Count); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -302,9 +302,14 @@ func SearchLatestVersions(ctx context.Context, opts *PackageSearchOptions) ([]*P
|
||||
cond := opts.toConds().
|
||||
And(builder.Expr("pv2.id IS NULL"))
|
||||
|
||||
joinCond := builder.Expr("package_version.package_id = pv2.package_id AND (package_version.created_unix < pv2.created_unix OR (package_version.created_unix = pv2.created_unix AND package_version.id < pv2.id))")
|
||||
if !opts.IsInternal.IsNone() {
|
||||
joinCond = joinCond.And(builder.Eq{"pv2.is_internal": opts.IsInternal.IsTrue()})
|
||||
}
|
||||
|
||||
sess := db.GetEngine(ctx).
|
||||
Table("package_version").
|
||||
Join("LEFT", "package_version pv2", "package_version.package_id = pv2.package_id AND (package_version.created_unix < pv2.created_unix OR (package_version.created_unix = pv2.created_unix AND package_version.id < pv2.id))").
|
||||
Join("LEFT", "package_version pv2", joinCond).
|
||||
Join("INNER", "package", "package.id = package_version.package_id").
|
||||
Where(cond)
|
||||
|
||||
|
||||
@ -8,6 +8,7 @@
|
||||
package charset
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
@ -31,7 +32,7 @@ func EscapeControlHTML(text string, locale translation.Locale, allowed ...rune)
|
||||
return streamer.escaped, sb.String()
|
||||
}
|
||||
|
||||
// EscapeControlReaders escapes the unicode control sequences in a provider reader and writer in a locale and returns the findings as an EscapeStatus and the escaped []byte
|
||||
// EscapeControlReaders escapes the unicode control sequences in a provided reader of HTML content and writer in a locale and returns the findings as an EscapeStatus and the escaped []byte
|
||||
func EscapeControlReader(reader io.Reader, writer io.Writer, locale translation.Locale, allowed ...rune) (escaped *EscapeStatus, err error) {
|
||||
outputStream := &HTMLStreamerWriter{Writer: writer}
|
||||
streamer := NewEscapeStreamer(locale, outputStream, allowed...).(*escapeStreamer)
|
||||
@ -43,6 +44,35 @@ func EscapeControlReader(reader io.Reader, writer io.Writer, locale translation.
|
||||
return streamer.escaped, err
|
||||
}
|
||||
|
||||
// EscapeControlStringReader escapes the unicode control sequences in a provided reader of string content and writer in a locale and returns the findings as an EscapeStatus and the escaped []byte
|
||||
func EscapeControlStringReader(reader io.Reader, writer io.Writer, locale translation.Locale, allowed ...rune) (escaped *EscapeStatus, err error) {
|
||||
bufRd := bufio.NewReader(reader)
|
||||
outputStream := &HTMLStreamerWriter{Writer: writer}
|
||||
streamer := NewEscapeStreamer(locale, outputStream, allowed...).(*escapeStreamer)
|
||||
|
||||
for {
|
||||
line, rdErr := bufRd.ReadString('\n')
|
||||
if len(line) > 0 {
|
||||
if err := streamer.Text(line); err != nil {
|
||||
streamer.escaped.HasError = true
|
||||
log.Error("Error whilst escaping: %v", err)
|
||||
return streamer.escaped, err
|
||||
}
|
||||
}
|
||||
if rdErr != nil {
|
||||
if rdErr != io.EOF {
|
||||
err = rdErr
|
||||
}
|
||||
break
|
||||
}
|
||||
if err := streamer.SelfClosingTag("br"); err != nil {
|
||||
streamer.escaped.HasError = true
|
||||
return streamer.escaped, err
|
||||
}
|
||||
}
|
||||
return streamer.escaped, err
|
||||
}
|
||||
|
||||
// EscapeControlString escapes the unicode control sequences in a provided string and returns the findings as an EscapeStatus and the escaped string
|
||||
func EscapeControlString(text string, locale translation.Locale, allowed ...rune) (escaped *EscapeStatus, output string) {
|
||||
sb := &strings.Builder{}
|
||||
|
||||
37
modules/doctor/lfs.go
Normal file
37
modules/doctor/lfs.go
Normal file
@ -0,0 +1,37 @@
|
||||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package doctor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"code.gitea.io/gitea/modules/log"
|
||||
"code.gitea.io/gitea/modules/setting"
|
||||
"code.gitea.io/gitea/services/repository"
|
||||
)
|
||||
|
||||
func init() {
|
||||
Register(&Check{
|
||||
Title: "Garbage collect LFS",
|
||||
Name: "gc-lfs",
|
||||
IsDefault: false,
|
||||
Run: garbageCollectLFSCheck,
|
||||
AbortIfFailed: false,
|
||||
SkipDatabaseInitialization: false,
|
||||
Priority: 1,
|
||||
})
|
||||
}
|
||||
|
||||
func garbageCollectLFSCheck(ctx context.Context, logger log.Logger, autofix bool) error {
|
||||
if !setting.LFS.StartServer {
|
||||
return fmt.Errorf("LFS support is disabled")
|
||||
}
|
||||
|
||||
if err := repository.GarbageCollectLFSMetaObjects(ctx, logger, autofix); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return checkStorage(&checkStorageOptions{LFS: true})(ctx, logger, autofix)
|
||||
}
|
||||
@ -469,6 +469,13 @@ func getAppPath() (string, error) {
|
||||
appPath, err = exec.LookPath(os.Args[0])
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
// FIXME: Once we switch to go 1.19 use !errors.Is(err, exec.ErrDot)
|
||||
if !strings.Contains(err.Error(), "cannot run executable found relative to current directory") {
|
||||
return "", err
|
||||
}
|
||||
appPath, err = filepath.Abs(os.Args[0])
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
@ -102,7 +102,8 @@ func (l *LocalStorage) Save(path string, r io.Reader, size int64) (int64, error)
|
||||
return 0, err
|
||||
}
|
||||
// Golang's tmp file (os.CreateTemp) always have 0o600 mode, so we need to change the file to follow the umask (as what Create/MkDir does)
|
||||
if err := util.ApplyUmask(p, os.ModePerm); err != nil {
|
||||
// but we don't want to make these files executable - so ensure that we mask out the executable bits
|
||||
if err := util.ApplyUmask(p, os.ModePerm&0o666); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
|
||||
44
options/license/LOOP
Normal file
44
options/license/LOOP
Normal file
@ -0,0 +1,44 @@
|
||||
Portions of LOOP are Copyright (c) 1986 by the Massachusetts Institute of Technology.
|
||||
All Rights Reserved.
|
||||
|
||||
Permission to use, copy, modify and distribute this software and its
|
||||
documentation for any purpose and without fee is hereby granted,
|
||||
provided that the M.I.T. copyright notice appear in all copies and that
|
||||
both that copyright notice and this permission notice appear in
|
||||
supporting documentation. The names "M.I.T." and "Massachusetts
|
||||
Institute of Technology" may not be used in advertising or publicity
|
||||
pertaining to distribution of the software without specific, written
|
||||
prior permission. Notice must be given in supporting documentation that
|
||||
copying distribution is by permission of M.I.T. M.I.T. makes no
|
||||
representations about the suitability of this software for any purpose.
|
||||
It is provided "as is" without express or implied warranty.
|
||||
|
||||
Massachusetts Institute of Technology
|
||||
77 Massachusetts Avenue
|
||||
Cambridge, Massachusetts 02139
|
||||
United States of America
|
||||
+1-617-253-1000
|
||||
|
||||
Portions of LOOP are Copyright (c) 1989, 1990, 1991, 1992 by Symbolics, Inc.
|
||||
All Rights Reserved.
|
||||
|
||||
Permission to use, copy, modify and distribute this software and its
|
||||
documentation for any purpose and without fee is hereby granted,
|
||||
provided that the Symbolics copyright notice appear in all copies and
|
||||
that both that copyright notice and this permission notice appear in
|
||||
supporting documentation. The name "Symbolics" may not be used in
|
||||
advertising or publicity pertaining to distribution of the software
|
||||
without specific, written prior permission. Notice must be given in
|
||||
supporting documentation that copying distribution is by permission of
|
||||
Symbolics. Symbolics makes no representations about the suitability of
|
||||
this software for any purpose. It is provided "as is" without express
|
||||
or implied warranty.
|
||||
|
||||
Symbolics, CLOE Runtime, and Minima are trademarks, and CLOE, Genera,
|
||||
and Zetalisp are registered trademarks of Symbolics, Inc.
|
||||
|
||||
Symbolics, Inc.
|
||||
8 New England Executive Park, East
|
||||
Burlington, Massachusetts 01803
|
||||
United States of America
|
||||
+1-617-221-1000
|
||||
@ -405,8 +405,9 @@ func setPackageTag(tag string, pv *packages_model.PackageVersion, deleteOnly boo
|
||||
|
||||
func PackageSearch(ctx *context.Context) {
|
||||
pvs, total, err := packages_model.SearchLatestVersions(ctx, &packages_model.PackageSearchOptions{
|
||||
OwnerID: ctx.Package.Owner.ID,
|
||||
Type: packages_model.TypeNpm,
|
||||
OwnerID: ctx.Package.Owner.ID,
|
||||
Type: packages_model.TypeNpm,
|
||||
IsInternal: util.OptionalBoolFalse,
|
||||
Name: packages_model.SearchValue{
|
||||
ExactMatch: false,
|
||||
Value: ctx.FormTrim("text"),
|
||||
|
||||
@ -9,7 +9,6 @@ import (
|
||||
gocontext "context"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
gotemplate "html/template"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
@ -241,18 +240,19 @@ func findReadmeFile(ctx *context.Context, entries git.Entries, treeLink string)
|
||||
return readmeFile, readmeTreelink
|
||||
}
|
||||
|
||||
func renderReadmeFile(ctx *context.Context, readmeFile *namedBlob, readmeTreelink string) {
|
||||
ctx.Data["RawFileLink"] = ""
|
||||
ctx.Data["ReadmeInList"] = true
|
||||
ctx.Data["ReadmeExist"] = true
|
||||
ctx.Data["FileIsSymlink"] = readmeFile.isSymlink
|
||||
type fileInfo struct {
|
||||
isTextFile bool
|
||||
isLFSFile bool
|
||||
fileSize int64
|
||||
lfsMeta *lfs.Pointer
|
||||
st typesniffer.SniffedType
|
||||
}
|
||||
|
||||
dataRc, err := readmeFile.blob.DataAsync()
|
||||
func getFileReader(repoID int64, blob *git.Blob) ([]byte, io.ReadCloser, *fileInfo, error) {
|
||||
dataRc, err := blob.DataAsync()
|
||||
if err != nil {
|
||||
ctx.ServerError("Data", err)
|
||||
return
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
defer dataRc.Close()
|
||||
|
||||
buf := make([]byte, 1024)
|
||||
n, _ := util.ReadAtMost(dataRc, buf)
|
||||
@ -261,67 +261,75 @@ func renderReadmeFile(ctx *context.Context, readmeFile *namedBlob, readmeTreelin
|
||||
st := typesniffer.DetectContentType(buf)
|
||||
isTextFile := st.IsText()
|
||||
|
||||
ctx.Data["FileIsText"] = isTextFile
|
||||
ctx.Data["FileName"] = readmeFile.name
|
||||
fileSize := int64(0)
|
||||
isLFSFile := false
|
||||
ctx.Data["IsLFSFile"] = false
|
||||
|
||||
// FIXME: what happens when README file is an image?
|
||||
if isTextFile && setting.LFS.StartServer {
|
||||
pointer, _ := lfs.ReadPointerFromBuffer(buf)
|
||||
if pointer.IsValid() {
|
||||
meta, err := git_model.GetLFSMetaObjectByOid(ctx.Repo.Repository.ID, pointer.Oid)
|
||||
if err != nil && err != git_model.ErrLFSObjectNotExist {
|
||||
ctx.ServerError("GetLFSMetaObject", err)
|
||||
return
|
||||
}
|
||||
if meta != nil {
|
||||
ctx.Data["IsLFSFile"] = true
|
||||
isLFSFile = true
|
||||
|
||||
// OK read the lfs object
|
||||
var err error
|
||||
dataRc, err = lfs.ReadMetaObject(pointer)
|
||||
if err != nil {
|
||||
ctx.ServerError("ReadMetaObject", err)
|
||||
return
|
||||
}
|
||||
defer dataRc.Close()
|
||||
|
||||
buf = make([]byte, 1024)
|
||||
n, err = util.ReadAtMost(dataRc, buf)
|
||||
if err != nil {
|
||||
ctx.ServerError("Data", err)
|
||||
return
|
||||
}
|
||||
buf = buf[:n]
|
||||
|
||||
st = typesniffer.DetectContentType(buf)
|
||||
isTextFile = st.IsText()
|
||||
ctx.Data["IsTextFile"] = isTextFile
|
||||
|
||||
fileSize = meta.Size
|
||||
ctx.Data["FileSize"] = meta.Size
|
||||
filenameBase64 := base64.RawURLEncoding.EncodeToString([]byte(readmeFile.name))
|
||||
ctx.Data["RawFileLink"] = fmt.Sprintf("%s.git/info/lfs/objects/%s/%s", ctx.Repo.Repository.HTMLURL(), url.PathEscape(meta.Oid), url.PathEscape(filenameBase64))
|
||||
}
|
||||
}
|
||||
if !isTextFile || !setting.LFS.StartServer {
|
||||
return buf, dataRc, &fileInfo{isTextFile, false, blob.Size(), nil, st}, nil
|
||||
}
|
||||
|
||||
if !isTextFile {
|
||||
pointer, _ := lfs.ReadPointerFromBuffer(buf)
|
||||
if !pointer.IsValid() { // fallback to plain file
|
||||
return buf, dataRc, &fileInfo{isTextFile, false, blob.Size(), nil, st}, nil
|
||||
}
|
||||
|
||||
meta, err := git_model.GetLFSMetaObjectByOid(repoID, pointer.Oid)
|
||||
if err != git_model.ErrLFSObjectNotExist { // fallback to plain file
|
||||
return buf, dataRc, &fileInfo{isTextFile, false, blob.Size(), nil, st}, nil
|
||||
}
|
||||
|
||||
dataRc.Close()
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
|
||||
dataRc, err = lfs.ReadMetaObject(pointer)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
|
||||
buf = make([]byte, 1024)
|
||||
n, err = util.ReadAtMost(dataRc, buf)
|
||||
if err != nil {
|
||||
dataRc.Close()
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
buf = buf[:n]
|
||||
|
||||
st = typesniffer.DetectContentType(buf)
|
||||
|
||||
return buf, dataRc, &fileInfo{st.IsText(), true, meta.Size, &meta.Pointer, st}, nil
|
||||
}
|
||||
|
||||
func renderReadmeFile(ctx *context.Context, readmeFile *namedBlob, readmeTreelink string) {
|
||||
ctx.Data["RawFileLink"] = ""
|
||||
ctx.Data["ReadmeInList"] = true
|
||||
ctx.Data["ReadmeExist"] = true
|
||||
ctx.Data["FileIsSymlink"] = readmeFile.isSymlink
|
||||
|
||||
buf, dataRc, fInfo, err := getFileReader(ctx.Repo.Repository.ID, readmeFile.blob)
|
||||
if err != nil {
|
||||
ctx.ServerError("getFileReader", err)
|
||||
return
|
||||
}
|
||||
defer dataRc.Close()
|
||||
|
||||
ctx.Data["FileIsText"] = fInfo.isTextFile
|
||||
ctx.Data["FileName"] = readmeFile.name
|
||||
ctx.Data["IsLFSFile"] = fInfo.isLFSFile
|
||||
|
||||
if fInfo.isLFSFile {
|
||||
filenameBase64 := base64.RawURLEncoding.EncodeToString([]byte(readmeFile.name))
|
||||
ctx.Data["RawFileLink"] = fmt.Sprintf("%s.git/info/lfs/objects/%s/%s", ctx.Repo.Repository.HTMLURL(), url.PathEscape(fInfo.lfsMeta.Oid), url.PathEscape(filenameBase64))
|
||||
}
|
||||
|
||||
if !fInfo.isTextFile {
|
||||
return
|
||||
}
|
||||
|
||||
if !isLFSFile {
|
||||
fileSize = readmeFile.blob.Size()
|
||||
}
|
||||
|
||||
if fileSize >= setting.UI.MaxDisplayFileSize {
|
||||
if fInfo.fileSize >= setting.UI.MaxDisplayFileSize {
|
||||
// Pretend that this is a normal text file to display 'This file is too large to be shown'
|
||||
ctx.Data["IsFileTooLarge"] = true
|
||||
ctx.Data["IsTextFile"] = true
|
||||
ctx.Data["FileSize"] = fileSize
|
||||
ctx.Data["FileSize"] = fInfo.fileSize
|
||||
return
|
||||
}
|
||||
|
||||
@ -341,15 +349,13 @@ func renderReadmeFile(ctx *context.Context, readmeFile *namedBlob, readmeTreelin
|
||||
if err != nil {
|
||||
log.Error("Render failed for %s in %-v: %v Falling back to rendering source", readmeFile.name, ctx.Repo.Repository, err)
|
||||
buf := &bytes.Buffer{}
|
||||
ctx.Data["EscapeStatus"], _ = charset.EscapeControlReader(rd, buf, ctx.Locale)
|
||||
ctx.Data["FileContent"] = strings.ReplaceAll(
|
||||
gotemplate.HTMLEscapeString(buf.String()), "\n", `<br>`,
|
||||
)
|
||||
ctx.Data["EscapeStatus"], _ = charset.EscapeControlStringReader(rd, buf, ctx.Locale)
|
||||
ctx.Data["FileContent"] = buf.String()
|
||||
}
|
||||
} else {
|
||||
ctx.Data["IsRenderedHTML"] = true
|
||||
ctx.Data["IsPlainText"] = true
|
||||
buf := &bytes.Buffer{}
|
||||
ctx.Data["EscapeStatus"], err = charset.EscapeControlReader(rd, &charset.BreakWriter{Writer: buf}, ctx.Locale, charset.RuneNBSP)
|
||||
ctx.Data["EscapeStatus"], err = charset.EscapeControlStringReader(rd, buf, ctx.Locale)
|
||||
if err != nil {
|
||||
log.Error("Read failed: %v", err)
|
||||
}
|
||||
@ -362,16 +368,14 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
|
||||
ctx.Data["IsViewFile"] = true
|
||||
ctx.Data["HideRepoInfo"] = true
|
||||
blob := entry.Blob()
|
||||
dataRc, err := blob.DataAsync()
|
||||
buf, dataRc, fInfo, err := getFileReader(ctx.Repo.Repository.ID, blob)
|
||||
if err != nil {
|
||||
ctx.ServerError("DataAsync", err)
|
||||
ctx.ServerError("getFileReader", err)
|
||||
return
|
||||
}
|
||||
defer dataRc.Close()
|
||||
|
||||
ctx.Data["Title"] = ctx.Tr("repo.file.title", ctx.Repo.Repository.Name+"/"+path.Base(ctx.Repo.TreePath), ctx.Repo.RefName)
|
||||
|
||||
fileSize := blob.Size()
|
||||
ctx.Data["FileIsSymlink"] = entry.IsLink()
|
||||
ctx.Data["FileName"] = blob.Name()
|
||||
ctx.Data["RawFileLink"] = rawLink + "/" + util.PathEscapeSegments(ctx.Repo.TreePath)
|
||||
@ -381,69 +385,27 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
|
||||
ctx.Data["FileError"] = editorconfigErr
|
||||
}
|
||||
|
||||
buf := make([]byte, 1024)
|
||||
n, _ := util.ReadAtMost(dataRc, buf)
|
||||
buf = buf[:n]
|
||||
|
||||
st := typesniffer.DetectContentType(buf)
|
||||
isTextFile := st.IsText()
|
||||
|
||||
isLFSFile := false
|
||||
isDisplayingSource := ctx.FormString("display") == "source"
|
||||
isDisplayingRendered := !isDisplayingSource
|
||||
|
||||
// Check for LFS meta file
|
||||
if isTextFile && setting.LFS.StartServer {
|
||||
pointer, _ := lfs.ReadPointerFromBuffer(buf)
|
||||
if pointer.IsValid() {
|
||||
meta, err := git_model.GetLFSMetaObjectByOid(ctx.Repo.Repository.ID, pointer.Oid)
|
||||
if err != nil && err != git_model.ErrLFSObjectNotExist {
|
||||
ctx.ServerError("GetLFSMetaObject", err)
|
||||
return
|
||||
}
|
||||
if meta != nil {
|
||||
isLFSFile = true
|
||||
|
||||
// OK read the lfs object
|
||||
var err error
|
||||
dataRc, err = lfs.ReadMetaObject(pointer)
|
||||
if err != nil {
|
||||
ctx.ServerError("ReadMetaObject", err)
|
||||
return
|
||||
}
|
||||
defer dataRc.Close()
|
||||
|
||||
buf = make([]byte, 1024)
|
||||
n, err = util.ReadAtMost(dataRc, buf)
|
||||
if err != nil {
|
||||
ctx.ServerError("Data", err)
|
||||
return
|
||||
}
|
||||
buf = buf[:n]
|
||||
|
||||
st = typesniffer.DetectContentType(buf)
|
||||
isTextFile = st.IsText()
|
||||
|
||||
fileSize = meta.Size
|
||||
ctx.Data["RawFileLink"] = ctx.Repo.RepoLink + "/media/" + ctx.Repo.BranchNameSubURL() + "/" + util.PathEscapeSegments(ctx.Repo.TreePath)
|
||||
}
|
||||
}
|
||||
if fInfo.isLFSFile {
|
||||
ctx.Data["RawFileLink"] = ctx.Repo.RepoLink + "/media/" + ctx.Repo.BranchNameSubURL() + "/" + util.PathEscapeSegments(ctx.Repo.TreePath)
|
||||
}
|
||||
|
||||
isRepresentableAsText := st.IsRepresentableAsText()
|
||||
isRepresentableAsText := fInfo.st.IsRepresentableAsText()
|
||||
if !isRepresentableAsText {
|
||||
// If we can't show plain text, always try to render.
|
||||
isDisplayingSource = false
|
||||
isDisplayingRendered = true
|
||||
}
|
||||
ctx.Data["IsLFSFile"] = isLFSFile
|
||||
ctx.Data["FileSize"] = fileSize
|
||||
ctx.Data["IsTextFile"] = isTextFile
|
||||
ctx.Data["IsLFSFile"] = fInfo.isLFSFile
|
||||
ctx.Data["FileSize"] = fInfo.fileSize
|
||||
ctx.Data["IsTextFile"] = fInfo.isTextFile
|
||||
ctx.Data["IsRepresentableAsText"] = isRepresentableAsText
|
||||
ctx.Data["IsDisplayingSource"] = isDisplayingSource
|
||||
ctx.Data["IsDisplayingRendered"] = isDisplayingRendered
|
||||
|
||||
isTextSource := isTextFile || isDisplayingSource
|
||||
isTextSource := fInfo.isTextFile || isDisplayingSource
|
||||
ctx.Data["IsTextSource"] = isTextSource
|
||||
if isTextSource {
|
||||
ctx.Data["CanCopyContent"] = true
|
||||
@ -468,7 +430,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
|
||||
}
|
||||
|
||||
// Assume file is not editable first.
|
||||
if isLFSFile {
|
||||
if fInfo.isLFSFile {
|
||||
ctx.Data["EditFileTooltip"] = ctx.Tr("repo.editor.cannot_edit_lfs_files")
|
||||
} else if !isRepresentableAsText {
|
||||
ctx.Data["EditFileTooltip"] = ctx.Tr("repo.editor.cannot_edit_non_text_files")
|
||||
@ -476,13 +438,13 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
|
||||
|
||||
switch {
|
||||
case isRepresentableAsText:
|
||||
if st.IsSvgImage() {
|
||||
if fInfo.st.IsSvgImage() {
|
||||
ctx.Data["IsImageFile"] = true
|
||||
ctx.Data["CanCopyContent"] = true
|
||||
ctx.Data["HasSourceRenderedToggle"] = true
|
||||
}
|
||||
|
||||
if fileSize >= setting.UI.MaxDisplayFileSize {
|
||||
if fInfo.fileSize >= setting.UI.MaxDisplayFileSize {
|
||||
ctx.Data["IsFileTooLarge"] = true
|
||||
break
|
||||
}
|
||||
@ -527,15 +489,6 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
|
||||
}
|
||||
// to prevent iframe load third-party url
|
||||
ctx.Resp.Header().Add("Content-Security-Policy", "frame-src 'self'")
|
||||
} else if readmeExist && !shouldRenderSource {
|
||||
buf := &bytes.Buffer{}
|
||||
ctx.Data["IsRenderedHTML"] = true
|
||||
|
||||
ctx.Data["EscapeStatus"], _ = charset.EscapeControlReader(rd, buf, ctx.Locale)
|
||||
|
||||
ctx.Data["FileContent"] = strings.ReplaceAll(
|
||||
gotemplate.HTMLEscapeString(buf.String()), "\n", `<br>`,
|
||||
)
|
||||
} else {
|
||||
buf, _ := io.ReadAll(rd)
|
||||
|
||||
@ -589,7 +542,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
|
||||
ctx.Data["FileContent"] = fileContent
|
||||
ctx.Data["LineEscapeStatus"] = statuses
|
||||
}
|
||||
if !isLFSFile {
|
||||
if !fInfo.isLFSFile {
|
||||
if ctx.Repo.CanEnableEditor(ctx.Doer) {
|
||||
if lfsLock != nil && lfsLock.OwnerID != ctx.Doer.ID {
|
||||
ctx.Data["CanEditFile"] = false
|
||||
@ -605,17 +558,17 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
|
||||
}
|
||||
}
|
||||
|
||||
case st.IsPDF():
|
||||
case fInfo.st.IsPDF():
|
||||
ctx.Data["IsPDFFile"] = true
|
||||
case st.IsVideo():
|
||||
case fInfo.st.IsVideo():
|
||||
ctx.Data["IsVideoFile"] = true
|
||||
case st.IsAudio():
|
||||
case fInfo.st.IsAudio():
|
||||
ctx.Data["IsAudioFile"] = true
|
||||
case st.IsImage() && (setting.UI.SVG.Enabled || !st.IsSvgImage()):
|
||||
case fInfo.st.IsImage() && (setting.UI.SVG.Enabled || !fInfo.st.IsSvgImage()):
|
||||
ctx.Data["IsImageFile"] = true
|
||||
ctx.Data["CanCopyContent"] = true
|
||||
default:
|
||||
if fileSize >= setting.UI.MaxDisplayFileSize {
|
||||
if fInfo.fileSize >= setting.UI.MaxDisplayFileSize {
|
||||
ctx.Data["IsFileTooLarge"] = true
|
||||
break
|
||||
}
|
||||
|
||||
@ -63,7 +63,7 @@ func registerRepoHealthCheck() {
|
||||
for _, arg := range rhcConfig.Args {
|
||||
args = append(args, git.CmdArg(arg))
|
||||
}
|
||||
return repo_service.GitFsck(ctx, rhcConfig.Timeout, args)
|
||||
return repo_service.GitFsckRepos(ctx, rhcConfig.Timeout, args)
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@ -53,6 +53,8 @@ var patchErrorSuffices = []string{
|
||||
": patch does not apply",
|
||||
": already exists in working directory",
|
||||
"unrecognized input",
|
||||
": No such file or directory",
|
||||
": does not exist in index",
|
||||
}
|
||||
|
||||
// TestPatch will test whether a simple patch will apply
|
||||
@ -416,6 +418,7 @@ func checkConflicts(ctx context.Context, pr *issues_model.PullRequest, gitRepo *
|
||||
scanner := bufio.NewScanner(stderrReader)
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
log.Trace("PullRequest[%d].testPatch: stderr: %s", pr.ID, line)
|
||||
if strings.HasPrefix(line, prefix) {
|
||||
conflict = true
|
||||
filepath := strings.TrimSpace(strings.Split(line[len(prefix):], ":")[0])
|
||||
|
||||
@ -337,7 +337,7 @@ func ListUnadoptedRepositories(query string, opts *db.ListOptions) ([]string, in
|
||||
}
|
||||
|
||||
repoNamesToCheck = append(repoNamesToCheck, name)
|
||||
if len(repoNamesToCheck) > setting.Database.IterateBufferSize {
|
||||
if len(repoNamesToCheck) >= setting.Database.IterateBufferSize {
|
||||
if err = checkUnadoptedRepositories(userName, repoNamesToCheck, unadopted); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@ -22,8 +22,8 @@ import (
|
||||
"xorm.io/builder"
|
||||
)
|
||||
|
||||
// GitFsck calls 'git fsck' to check repository health.
|
||||
func GitFsck(ctx context.Context, timeout time.Duration, args []git.CmdArg) error {
|
||||
// GitFsckRepos calls 'git fsck' to check repository health.
|
||||
func GitFsckRepos(ctx context.Context, timeout time.Duration, args []git.CmdArg) error {
|
||||
log.Trace("Doing: GitFsck")
|
||||
|
||||
if err := db.Iterate(
|
||||
@ -35,15 +35,7 @@ func GitFsck(ctx context.Context, timeout time.Duration, args []git.CmdArg) erro
|
||||
return db.ErrCancelledf("before fsck of %s", repo.FullName())
|
||||
default:
|
||||
}
|
||||
log.Trace("Running health check on repository %v", repo)
|
||||
repoPath := repo.RepoPath()
|
||||
if err := git.Fsck(ctx, repoPath, timeout, args...); err != nil {
|
||||
log.Warn("Failed to health check repository (%v): %v", repo, err)
|
||||
if err = system_model.CreateRepositoryNotice("Failed to health check repository (%s): %v", repo.FullName(), err); err != nil {
|
||||
log.Error("CreateRepositoryNotice: %v", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
return GitFsckRepo(ctx, repo, timeout, args)
|
||||
},
|
||||
); err != nil {
|
||||
log.Trace("Error: GitFsck: %v", err)
|
||||
@ -54,6 +46,19 @@ func GitFsck(ctx context.Context, timeout time.Duration, args []git.CmdArg) erro
|
||||
return nil
|
||||
}
|
||||
|
||||
// GitFsckRepo calls 'git fsck' to check an individual repository's health.
|
||||
func GitFsckRepo(ctx context.Context, repo *repo_model.Repository, timeout time.Duration, args []git.CmdArg) error {
|
||||
log.Trace("Running health check on repository %-v", repo)
|
||||
repoPath := repo.RepoPath()
|
||||
if err := git.Fsck(ctx, repoPath, timeout, args...); err != nil {
|
||||
log.Warn("Failed to health check repository (%-v): %v", repo, err)
|
||||
if err = system_model.CreateRepositoryNotice("Failed to health check repository (%s): %v", repo.FullName(), err); err != nil {
|
||||
log.Error("CreateRepositoryNotice: %v", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GitGcRepos calls 'git gc' to remove unnecessary files and optimize the local repository
|
||||
func GitGcRepos(ctx context.Context, timeout time.Duration, args ...git.CmdArg) error {
|
||||
log.Trace("Doing: GitGcRepos")
|
||||
@ -68,33 +73,7 @@ func GitGcRepos(ctx context.Context, timeout time.Duration, args ...git.CmdArg)
|
||||
return db.ErrCancelledf("before GC of %s", repo.FullName())
|
||||
default:
|
||||
}
|
||||
log.Trace("Running git gc on %v", repo)
|
||||
command := git.NewCommand(ctx, args...).
|
||||
SetDescription(fmt.Sprintf("Repository Garbage Collection: %s", repo.FullName()))
|
||||
var stdout string
|
||||
var err error
|
||||
stdout, _, err = command.RunStdString(&git.RunOpts{Timeout: timeout, Dir: repo.RepoPath()})
|
||||
|
||||
if err != nil {
|
||||
log.Error("Repository garbage collection failed for %v. Stdout: %s\nError: %v", repo, stdout, err)
|
||||
desc := fmt.Sprintf("Repository garbage collection failed for %s. Stdout: %s\nError: %v", repo.RepoPath(), stdout, err)
|
||||
if err = system_model.CreateRepositoryNotice(desc); err != nil {
|
||||
log.Error("CreateRepositoryNotice: %v", err)
|
||||
}
|
||||
return fmt.Errorf("Repository garbage collection failed in repo: %s: Error: %w", repo.FullName(), err)
|
||||
}
|
||||
|
||||
// Now update the size of the repository
|
||||
if err := repo_module.UpdateRepoSize(ctx, repo); err != nil {
|
||||
log.Error("Updating size as part of garbage collection failed for %v. Stdout: %s\nError: %v", repo, stdout, err)
|
||||
desc := fmt.Sprintf("Updating size as part of garbage collection failed for %s. Stdout: %s\nError: %v", repo.RepoPath(), stdout, err)
|
||||
if err = system_model.CreateRepositoryNotice(desc); err != nil {
|
||||
log.Error("CreateRepositoryNotice: %v", err)
|
||||
}
|
||||
return fmt.Errorf("Updating size as part of garbage collection failed in repo: %s: Error: %w", repo.FullName(), err)
|
||||
}
|
||||
|
||||
return nil
|
||||
return GitGcRepo(ctx, repo, timeout, args)
|
||||
},
|
||||
); err != nil {
|
||||
return err
|
||||
@ -104,6 +83,37 @@ func GitGcRepos(ctx context.Context, timeout time.Duration, args ...git.CmdArg)
|
||||
return nil
|
||||
}
|
||||
|
||||
// GitGcRepo calls 'git gc' to remove unnecessary files and optimize the local repository
|
||||
func GitGcRepo(ctx context.Context, repo *repo_model.Repository, timeout time.Duration, args []git.CmdArg) error {
|
||||
log.Trace("Running git gc on %-v", repo)
|
||||
command := git.NewCommand(ctx, args...).
|
||||
SetDescription(fmt.Sprintf("Repository Garbage Collection: %s", repo.FullName()))
|
||||
var stdout string
|
||||
var err error
|
||||
stdout, _, err = command.RunStdString(&git.RunOpts{Timeout: timeout, Dir: repo.RepoPath()})
|
||||
|
||||
if err != nil {
|
||||
log.Error("Repository garbage collection failed for %v. Stdout: %s\nError: %v", repo, stdout, err)
|
||||
desc := fmt.Sprintf("Repository garbage collection failed for %s. Stdout: %s\nError: %v", repo.RepoPath(), stdout, err)
|
||||
if err = system_model.CreateRepositoryNotice(desc); err != nil {
|
||||
log.Error("CreateRepositoryNotice: %v", err)
|
||||
}
|
||||
return fmt.Errorf("Repository garbage collection failed in repo: %s: Error: %w", repo.FullName(), err)
|
||||
}
|
||||
|
||||
// Now update the size of the repository
|
||||
if err := repo_module.UpdateRepoSize(ctx, repo); err != nil {
|
||||
log.Error("Updating size as part of garbage collection failed for %-v. Stdout: %s\nError: %v", repo, stdout, err)
|
||||
desc := fmt.Sprintf("Updating size as part of garbage collection failed for %s. Stdout: %s\nError: %v", repo.RepoPath(), stdout, err)
|
||||
if err = system_model.CreateRepositoryNotice(desc); err != nil {
|
||||
log.Error("CreateRepositoryNotice: %v", err)
|
||||
}
|
||||
return fmt.Errorf("Updating size as part of garbage collection failed in repo: %s: Error: %w", repo.FullName(), err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func gatherMissingRepoRecords(ctx context.Context) ([]*repo_model.Repository, error) {
|
||||
repos := make([]*repo_model.Repository, 0, 10)
|
||||
if err := db.Iterate(
|
||||
|
||||
105
services/repository/lfs.go
Normal file
105
services/repository/lfs.go
Normal file
@ -0,0 +1,105 @@
|
||||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"code.gitea.io/gitea/models/db"
|
||||
git_model "code.gitea.io/gitea/models/git"
|
||||
repo_model "code.gitea.io/gitea/models/repo"
|
||||
"code.gitea.io/gitea/modules/git"
|
||||
"code.gitea.io/gitea/modules/lfs"
|
||||
"code.gitea.io/gitea/modules/log"
|
||||
|
||||
"xorm.io/builder"
|
||||
)
|
||||
|
||||
func GarbageCollectLFSMetaObjects(ctx context.Context, logger log.Logger, autofix bool) error {
|
||||
log.Trace("Doing: GarbageCollectLFSMetaObjects")
|
||||
|
||||
if err := db.Iterate(
|
||||
ctx,
|
||||
builder.And(builder.Gt{"id": 0}),
|
||||
func(ctx context.Context, repo *repo_model.Repository) error {
|
||||
return GarbageCollectLFSMetaObjectsForRepo(ctx, repo, logger, autofix)
|
||||
},
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Trace("Finished: GarbageCollectLFSMetaObjects")
|
||||
return nil
|
||||
}
|
||||
|
||||
func GarbageCollectLFSMetaObjectsForRepo(ctx context.Context, repo *repo_model.Repository, logger log.Logger, autofix bool) error {
|
||||
if logger != nil {
|
||||
logger.Info("Checking %-v", repo)
|
||||
}
|
||||
total, orphaned, collected, deleted := 0, 0, 0, 0
|
||||
if logger != nil {
|
||||
defer func() {
|
||||
if orphaned == 0 {
|
||||
logger.Info("Found %d total LFSMetaObjects in %-v", total, repo)
|
||||
} else if !autofix {
|
||||
logger.Info("Found %d/%d orphaned LFSMetaObjects in %-v", orphaned, total, repo)
|
||||
} else {
|
||||
logger.Info("Collected %d/%d orphaned/%d total LFSMetaObjects in %-v. %d removed from storage.", collected, orphaned, total, repo, deleted)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
gitRepo, err := git.OpenRepository(ctx, repo.RepoPath())
|
||||
if err != nil {
|
||||
log.Error("Unable to open git repository %-v: %v", repo, err)
|
||||
return err
|
||||
}
|
||||
defer gitRepo.Close()
|
||||
|
||||
store := lfs.NewContentStore()
|
||||
|
||||
return git_model.IterateLFSMetaObjectsForRepo(ctx, repo.ID, func(ctx context.Context, metaObject *git_model.LFSMetaObject, count int64) error {
|
||||
total++
|
||||
pointerSha := git.ComputeBlobHash([]byte(metaObject.Pointer.StringContent()))
|
||||
|
||||
if gitRepo.IsObjectExist(pointerSha.String()) {
|
||||
return nil
|
||||
}
|
||||
orphaned++
|
||||
|
||||
if !autofix {
|
||||
return nil
|
||||
}
|
||||
// Non-existent pointer file
|
||||
_, err = git_model.RemoveLFSMetaObjectByOidFn(repo.ID, metaObject.Oid, func(count int64) error {
|
||||
if count > 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := store.Delete(metaObject.RelativePath()); err != nil {
|
||||
log.Error("Unable to remove lfs metaobject %s from store: %v", metaObject.Oid, err)
|
||||
}
|
||||
deleted++
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to remove meta-object %s in %s: %w", metaObject.Oid, repo.FullName(), err)
|
||||
}
|
||||
collected++
|
||||
|
||||
return nil
|
||||
}, &git_model.IterateLFSMetaObjectsForRepoOptions{
|
||||
// Only attempt to garbage collect lfs meta objects older than a week as the order of git lfs upload
|
||||
// and git object upload is not necessarily guaranteed. It's possible to imagine a situation whereby
|
||||
// an LFS object is uploaded but the git branch is not uploaded immediately, or there are some rapid
|
||||
// changes in new branches that might lead to lfs objects becoming temporarily unassociated with git
|
||||
// objects.
|
||||
//
|
||||
// It is likely that a week is potentially excessive but it should definitely be enough that any
|
||||
// unassociated LFS object is genuinely unassociated.
|
||||
OlderThan: time.Now().Add(-24 * 7 * time.Hour),
|
||||
})
|
||||
}
|
||||
@ -17,11 +17,11 @@
|
||||
</h4>
|
||||
<div class="ui attached table unstackable segment">
|
||||
{{template "repo/unicode_escape_prompt" dict "EscapeStatus" .EscapeStatus "root" $}}
|
||||
<div class="file-view{{if .IsMarkup}} markup {{.MarkupType}}{{else if .IsRenderedHTML}} plain-text{{else if .IsTextFile}} code-view{{end}}">
|
||||
<div class="file-view{{if .IsMarkup}} markup {{.MarkupType}}{{else if .IsPlainText}} plain-text{{else if .IsTextFile}} code-view{{end}}">
|
||||
{{if .IsMarkup}}
|
||||
{{if .FileContent}}{{.FileContent | Safe}}{{end}}
|
||||
{{else if .IsRenderedHTML}}
|
||||
<pre>{{if .FileContent}}{{.FileContent | Str2html}}{{end}}</pre>
|
||||
{{else if .IsPlainText}}
|
||||
<pre>{{if .FileContent}}{{.FileContent | Safe}}{{end}}</pre>
|
||||
{{else if not .IsTextFile}}
|
||||
<div class="view-raw ui center">
|
||||
{{if .IsImageFile}}
|
||||
|
||||
@ -61,11 +61,11 @@
|
||||
{{if not (or .IsMarkup .IsRenderedHTML)}}
|
||||
{{template "repo/unicode_escape_prompt" dict "EscapeStatus" .EscapeStatus "root" $}}
|
||||
{{end}}
|
||||
<div class="file-view{{if .IsMarkup}} markup {{.MarkupType}}{{else if .IsRenderedHTML}} plain-text{{else if .IsTextSource}} code-view{{end}}">
|
||||
<div class="file-view{{if .IsMarkup}} markup {{.MarkupType}}{{else if .IsPlainText}} plain-text{{else if .IsTextSource}} code-view{{end}}">
|
||||
{{if .IsMarkup}}
|
||||
{{if .FileContent}}{{.FileContent | Safe}}{{end}}
|
||||
{{else if .IsRenderedHTML}}
|
||||
<pre>{{if .FileContent}}{{.FileContent | Str2html}}{{end}}</pre>
|
||||
{{else if .IsPlainText}}
|
||||
<pre>{{if .FileContent}}{{.FileContent | Safe}}{{end}}</pre>
|
||||
{{else if not .IsTextSource}}
|
||||
<div class="view-raw ui center">
|
||||
{{if .IsImageFile}}
|
||||
|
||||
@ -128,9 +128,9 @@
|
||||
<ul class="repo-owner-name-list">
|
||||
<li v-for="repo in repos" :class="{'private': repo.private || repo.internal}">
|
||||
<a class="repo-list-link df ac sb" :href="repo.html_url">
|
||||
<div class="text truncate item-name f1">
|
||||
<component v-bind:is="repoIcon(repo)" size="16"></component>
|
||||
<strong>${repo.full_name}</strong>
|
||||
<div class="item-name df ac f1 mr-2">
|
||||
<component v-bind:is="repoIcon(repo)" size="16" class="mr-2"></component>
|
||||
<div class="text bold truncate ml-1">${repo.full_name}</div>
|
||||
<span v-if="repo.archived">
|
||||
{{svg "octicon-archive" 16 "ml-2"}}
|
||||
</span>
|
||||
|
||||
@ -27,6 +27,8 @@ func TestNodeinfo(t *testing.T) {
|
||||
onGiteaRun(t, func(*testing.T, *url.URL) {
|
||||
req := NewRequestf(t, "GET", "/api/v1/nodeinfo")
|
||||
resp := MakeRequest(t, req, http.StatusOK)
|
||||
VerifyJSONSchema(t, resp, "nodeinfo_2.1.json")
|
||||
|
||||
var nodeinfo api.NodeInfo
|
||||
DecodeJSON(t, resp, &nodeinfo)
|
||||
assert.True(t, nodeinfo.OpenRegistrations)
|
||||
|
||||
@ -33,6 +33,7 @@ import (
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/xeipuuv/gojsonschema"
|
||||
)
|
||||
|
||||
var c *web.Route
|
||||
@ -398,6 +399,25 @@ func DecodeJSON(t testing.TB, resp *httptest.ResponseRecorder, v interface{}) {
|
||||
assert.NoError(t, decoder.Decode(v))
|
||||
}
|
||||
|
||||
func VerifyJSONSchema(t testing.TB, resp *httptest.ResponseRecorder, schemaFile string) {
|
||||
t.Helper()
|
||||
|
||||
schemaFilePath := filepath.Join(filepath.Dir(setting.AppPath), "tests", "integration", "schemas", schemaFile)
|
||||
_, schemaFileErr := os.Stat(schemaFilePath)
|
||||
assert.Nil(t, schemaFileErr)
|
||||
|
||||
schema, schemaFileReadErr := os.ReadFile(schemaFilePath)
|
||||
assert.Nil(t, schemaFileReadErr)
|
||||
assert.True(t, len(schema) > 0)
|
||||
|
||||
nodeinfoSchema := gojsonschema.NewStringLoader(string(schema))
|
||||
nodeinfoString := gojsonschema.NewStringLoader(resp.Body.String())
|
||||
result, schemaValidationErr := gojsonschema.Validate(nodeinfoSchema, nodeinfoString)
|
||||
assert.Nil(t, schemaValidationErr)
|
||||
assert.Empty(t, result.Errors())
|
||||
assert.True(t, result.Valid())
|
||||
}
|
||||
|
||||
func GetCSRF(t testing.TB, session *TestSession, urlStr string) string {
|
||||
t.Helper()
|
||||
req := NewRequest(t, "GET", urlStr)
|
||||
|
||||
188
tests/integration/schemas/nodeinfo_2.1.json
Normal file
188
tests/integration/schemas/nodeinfo_2.1.json
Normal file
@ -0,0 +1,188 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
"id": "http://nodeinfo.diaspora.software/ns/schema/2.1#",
|
||||
"description": "NodeInfo schema version 2.1.",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"version",
|
||||
"software",
|
||||
"protocols",
|
||||
"services",
|
||||
"openRegistrations",
|
||||
"usage",
|
||||
"metadata"
|
||||
],
|
||||
"properties": {
|
||||
"version": {
|
||||
"description": "The schema version, must be 2.1.",
|
||||
"enum": [
|
||||
"2.1"
|
||||
]
|
||||
},
|
||||
"software": {
|
||||
"description": "Metadata about server software in use.",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"name",
|
||||
"version"
|
||||
],
|
||||
"properties": {
|
||||
"name": {
|
||||
"description": "The canonical name of this server software.",
|
||||
"type": "string",
|
||||
"pattern": "^[a-z0-9-]+$"
|
||||
},
|
||||
"version": {
|
||||
"description": "The version of this server software.",
|
||||
"type": "string"
|
||||
},
|
||||
"repository": {
|
||||
"description": "The url of the source code repository of this server software.",
|
||||
"type": "string"
|
||||
},
|
||||
"homepage": {
|
||||
"description": "The url of the homepage of this server software.",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"protocols": {
|
||||
"description": "The protocols supported on this server.",
|
||||
"type": "array",
|
||||
"minItems": 1,
|
||||
"items": {
|
||||
"enum": [
|
||||
"activitypub",
|
||||
"buddycloud",
|
||||
"dfrn",
|
||||
"diaspora",
|
||||
"libertree",
|
||||
"ostatus",
|
||||
"pumpio",
|
||||
"tent",
|
||||
"xmpp",
|
||||
"zot"
|
||||
]
|
||||
}
|
||||
},
|
||||
"services": {
|
||||
"description": "The third party sites this server can connect to via their application API.",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"inbound",
|
||||
"outbound"
|
||||
],
|
||||
"properties": {
|
||||
"inbound": {
|
||||
"description": "The third party sites this server can retrieve messages from for combined display with regular traffic.",
|
||||
"type": "array",
|
||||
"minItems": 0,
|
||||
"items": {
|
||||
"enum": [
|
||||
"atom1.0",
|
||||
"gnusocial",
|
||||
"imap",
|
||||
"pnut",
|
||||
"pop3",
|
||||
"pumpio",
|
||||
"rss2.0",
|
||||
"twitter"
|
||||
]
|
||||
}
|
||||
},
|
||||
"outbound": {
|
||||
"description": "The third party sites this server can publish messages to on the behalf of a user.",
|
||||
"type": "array",
|
||||
"minItems": 0,
|
||||
"items": {
|
||||
"enum": [
|
||||
"atom1.0",
|
||||
"blogger",
|
||||
"buddycloud",
|
||||
"diaspora",
|
||||
"dreamwidth",
|
||||
"drupal",
|
||||
"facebook",
|
||||
"friendica",
|
||||
"gnusocial",
|
||||
"google",
|
||||
"insanejournal",
|
||||
"libertree",
|
||||
"linkedin",
|
||||
"livejournal",
|
||||
"mediagoblin",
|
||||
"myspace",
|
||||
"pinterest",
|
||||
"pnut",
|
||||
"posterous",
|
||||
"pumpio",
|
||||
"redmatrix",
|
||||
"rss2.0",
|
||||
"smtp",
|
||||
"tent",
|
||||
"tumblr",
|
||||
"twitter",
|
||||
"wordpress",
|
||||
"xmpp"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"openRegistrations": {
|
||||
"description": "Whether this server allows open self-registration.",
|
||||
"type": "boolean"
|
||||
},
|
||||
"usage": {
|
||||
"description": "Usage statistics for this server.",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"users"
|
||||
],
|
||||
"properties": {
|
||||
"users": {
|
||||
"description": "statistics about the users of this server.",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"total": {
|
||||
"description": "The total amount of on this server registered users.",
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"activeHalfyear": {
|
||||
"description": "The amount of users that signed in at least once in the last 180 days.",
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"activeMonth": {
|
||||
"description": "The amount of users that signed in at least once in the last 30 days.",
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
"localPosts": {
|
||||
"description": "The amount of posts that were made by users that are registered on this server.",
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"localComments": {
|
||||
"description": "The amount of comments that were made by users that are registered on this server.",
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
"metadata": {
|
||||
"description": "Free form key value pairs for software specific values. Clients should not rely on any specific key present.",
|
||||
"type": "object",
|
||||
"minProperties": 0,
|
||||
"additionalProperties": true
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1354,6 +1354,12 @@ a.ui.card:hover,
|
||||
border-color: var(--color-error-border) !important;
|
||||
}
|
||||
|
||||
// A fix for text visibility issue in Chrome autofill in dark mode.
|
||||
// It's a problem from Formatic UI, and this rule overrides it.
|
||||
.ui.form .field.field input:-webkit-autofill {
|
||||
-webkit-text-fill-color: var(--color-black) !important;
|
||||
}
|
||||
|
||||
.ui.loading.loading.input > i.icon svg {
|
||||
visibility: hidden;
|
||||
}
|
||||
|
||||
@ -188,7 +188,11 @@
|
||||
}
|
||||
|
||||
.repo-owner-name-list .item-name {
|
||||
max-width: 70%;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.repo-owner-name-list .item-name svg {
|
||||
min-width: 16px;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -12,7 +12,8 @@
|
||||
margin: 0 .5rem !important;
|
||||
padding: .5rem !important;
|
||||
width: 320px;
|
||||
height: 60vh;
|
||||
height: calc(100vh - 450px);
|
||||
min-height: 60vh;
|
||||
overflow-y: scroll;
|
||||
flex: 0 0 auto;
|
||||
overflow: visible;
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user