feat: optimize log indexes

models/bots/task.go

@@ -5,8 +5,12 @@
 package bots
 
 import (
+	"bytes"
 	"context"
+	"encoding/binary"
+	"errors"
 	"fmt"
+	"io"
 
 	"code.gitea.io/gitea/core"
 	"code.gitea.io/gitea/models/db"
@@ -29,10 +33,10 @@ type Task struct {
 	Started  timeutil.TimeStamp
 	Stopped  timeutil.TimeStamp
 
-	LogURL     string  // dbfs:///a/b.log or s3://endpoint.com/a/b.log and etc.
-	LogLength  int64   // lines count
-	LogSize    int64   // blob size
-	LogIndexes []int64 `xorm:"JSON TEXT"` // line number to offset
+	LogURL     string      // dbfs:///a/b.log or s3://endpoint.com/a/b.log and etc.
+	LogLength  int64       // lines count
+	LogSize    int64       // blob size
+	LogIndexes *LogIndexes `xorm:"BLOB"` // line number to offset
 	LogExpired bool
 
 	Created timeutil.TimeStamp `xorm:"created"`
@@ -109,6 +113,30 @@ func (task *Task) FullSteps() []*TaskStep {
 	return steps
 }
 
+type LogIndexes []int64
+
+func (i *LogIndexes) FromDB(b []byte) error {
+	reader := bytes.NewReader(b)
+	for {
+		v, err := binary.ReadVarint(reader)
+		if err != nil {
+			if errors.Is(err, io.EOF) {
+				return nil
+			}
+			return fmt.Errorf("binary ReadVarint: %w", err)
+		}
+		*i = append(*i, v)
+	}
+}
+
+func (i *LogIndexes) ToDB() ([]byte, error) {
+	var buf []byte
+	for _, v := range *i {
+		buf = binary.AppendVarint(buf, v)
+	}
+	return buf, nil
+}
+
 // ErrTaskNotExist represents an error for bot task not exist
 type ErrTaskNotExist struct {
 	ID int64

models/migrations/v-dev.go

@@ -94,17 +94,20 @@ func addBotTables(x *xorm.Engine) error {
 	type BotsRunIndex db.ResourceIndex
 
 	type BotsTask struct {
-		ID        int64
-		JobID     int64
-		Attempt   int64
-		RunnerID  int64  `xorm:"index"`
-		LogToFile bool   // read log from database or from storage
-		LogURL    string // url of the log file in storage
-		Result    int32
-		Started   timeutil.TimeStamp
-		Stopped   timeutil.TimeStamp
-		Created   timeutil.TimeStamp `xorm:"created"`
-		Updated   timeutil.TimeStamp `xorm:"updated"`
+		ID         int64
+		JobID      int64
+		Attempt    int64
+		RunnerID   int64 `xorm:"index"`
+		Result     int32
+		Started    timeutil.TimeStamp
+		Stopped    timeutil.TimeStamp
+		LogURL     string   // dbfs:///a/b.log or s3://endpoint.com/a/b.log and etc.
+		LogLength  int64    // lines count
+		LogSize    int64    // blob size
+		LogIndexes *[]int64 `xorm:"BLOB"` // line number to offset
+		LogExpired bool
+		Created    timeutil.TimeStamp `xorm:"created"`
+		Updated    timeutil.TimeStamp `xorm:"updated"`
 	}
 
 	type BotsTaskStep struct {

models/repo/repo.go

@@ -142,9 +142,9 @@ type Repository struct {
 	NumProjects         int `xorm:"NOT NULL DEFAULT 0"`
 	NumClosedProjects   int `xorm:"NOT NULL DEFAULT 0"`
 	NumOpenProjects     int `xorm:"-"`
-	NumBuilds           int `xorm:"NOT NULL DEFAULT 0"`
-	NumClosedBuilds     int `xorm:"NOT NULL DEFAULT 0"`
-	NumOpenBuilds       int `xorm:"-"`
+	NumRuns             int `xorm:"NOT NULL DEFAULT 0"`
+	NumClosedRuns       int `xorm:"NOT NULL DEFAULT 0"`
+	NumOpenRuns         int `xorm:"-"`
 
 	IsPrivate  bool `xorm:"INDEX"`
 	IsEmpty    bool `xorm:"INDEX"`
@@ -237,7 +237,7 @@ func (repo *Repository) AfterLoad() {
 	repo.NumOpenPulls = repo.NumPulls - repo.NumClosedPulls
 	repo.NumOpenMilestones = repo.NumMilestones - repo.NumClosedMilestones
 	repo.NumOpenProjects = repo.NumProjects - repo.NumClosedProjects
-	repo.NumOpenBuilds = repo.NumBuilds - repo.NumClosedBuilds
+	repo.NumOpenRuns = repo.NumRuns - repo.NumClosedRuns
 }
 
 // LoadAttributes loads attributes of the repository.

routers/api/bots/runner/runner.go

@@ -186,8 +186,11 @@ func (s *Service) UpdateLog(
 		return nil, status.Errorf(codes.Internal, "write logs: %v", err)
 	}
 	task.LogLength += int64(len(rows))
+	if task.LogIndexes == nil {
+		task.LogIndexes = &bots_model.LogIndexes{}
+	}
 	for _, n := range ns {
-		task.LogIndexes = append(task.LogIndexes, task.LogSize)
+		*task.LogIndexes = append(*task.LogIndexes, task.LogSize)
 		task.LogSize += int64(n)
 	}
 	if err := bots_model.UpdateTask(ctx, task, "log_indexes", "log_length", "log_size"); err != nil {

routers/web/dev/buildview.go

@@ -179,7 +179,7 @@ func BuildViewPost(ctx *context.Context) {
 					if cursor.Cursor < step.LogLength || step.LogLength < 0 {
 						index := step.LogIndex + cursor.Cursor
 						length := step.LogLength - cursor.Cursor
-						offset := task.LogIndexes[index]
+						offset := (*task.LogIndexes)[index]
 						logRows, err = bots.ReadLogs(ctx, task.LogURL, offset, length)
 						if err != nil {
 							ctx.Error(http.StatusInternalServerError, err.Error())

templates/repo/builds/openclose.tmpl

@@ -1,10 +1,10 @@
 <div class="ui compact tiny menu">
 	<a class="{{if not .IsShowClosed}}active{{end}} item" href="{{$.Link}}?q={{$.Keyword}}&type={{$.ViewType}}&sort={{$.SortType}}&state=open&labels={{.SelectLabels}}&milestone={{.MilestoneID}}&assignee={{.AssigneeID}}">
 		{{svg "octicon-issue-opened" 16 "mr-3"}}
-		{{.locale.Tr "repo.issues.open_tab" .Repository.NumOpenBuilds}}
+		{{.locale.Tr "repo.issues.open_tab" .Repository.NumOpenRuns}}
 	</a>
 	<a class="{{if .IsShowClosed}}active{{end}} item" href="{{$.Link}}?q={{$.Keyword}}&type={{.ViewType}}&sort={{$.SortType}}&state=closed&labels={{.SelectLabels}}&milestone={{.MilestoneID}}&assignee={{.AssigneeID}}">
 		{{svg "octicon-issue-closed" 16 "mr-3"}}
-		{{.locale.Tr "repo.issues.close_tab" .Repository.NumClosedBuilds}}
+		{{.locale.Tr "repo.issues.close_tab" .Repository.NumClosedRuns}}
 	</a>
 </div>

templates/repo/header.tmpl

@@ -186,8 +186,8 @@
 				{{ if and (not .UnitBuildsGlobalDisabled) (.Permission.CanRead $.UnitTypeBuilds)}}
 					<a class="{{if .PageIsBuildList}}active{{end}} item" href="{{.RepoLink}}/builds">
 						{{svg "octicon-git-builds"}} {{.locale.Tr "repo.builds"}}
-						{{if .Repository.NumOpenBuilds}}
-							<span class="ui blue small label">{{CountFmt .Repository.NumOpenBuilds}}</span>
+						{{if .Repository.NumOpenRuns}}
+							<span class="ui blue small label">{{CountFmt .Repository.NumOpenRuns}}</span>
 						{{end}}
 					</a>
 				{{end}}