From 482832da7643205ce6c35153f600f25610b41105 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 16 Oct 2022 23:11:28 +0800 Subject: [PATCH] delete unused structs --- models/bots/build.go | 284 --------------------- models/bots/build_log.go | 42 --- models/bots/build_stage.go | 115 --------- models/bots/build_stage_list.go | 7 - models/bots/run.go | 37 +++ models/bots/run_job.go | 4 +- models/bots/{build_list.go => run_list.go} | 39 ++- models/bots/task.go | 2 + models/bots/task_log.go | 4 - models/migrations/v-dev.go | 117 +++++++-- modules/notification/bots/bots.go | 2 +- routers/api/bots/bots.go | 3 - routers/api/bots/runner/runner.go | 2 + routers/api/bots/scheduler/queue/queue.go | 5 +- routers/api/bots/socket.go | 207 --------------- routers/init.go | 2 - routers/web/repo/builds/builds.go | 75 +----- routers/web/web.go | 12 +- services/bots/bots.go | 13 +- 19 files changed, 190 insertions(+), 782 deletions(-) delete mode 100644 models/bots/build.go delete mode 100644 models/bots/build_log.go delete mode 100644 models/bots/build_stage.go delete mode 100644 models/bots/build_stage_list.go rename models/bots/{build_list.go => run_list.go} (51%) delete mode 100644 routers/api/bots/socket.go diff --git a/models/bots/build.go b/models/bots/build.go deleted file mode 100644 index e10f8725bf..0000000000 --- a/models/bots/build.go +++ /dev/null @@ -1,284 +0,0 @@ -// Copyright 2022 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package bots - -import ( - "context" - "errors" - "fmt" - "strconv" - - "code.gitea.io/gitea/core" - "code.gitea.io/gitea/models/db" - repo_model "code.gitea.io/gitea/models/repo" - user_model "code.gitea.io/gitea/models/user" - "code.gitea.io/gitea/models/webhook" - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/timeutil" - "code.gitea.io/gitea/modules/util" - - "github.com/google/uuid" - "xorm.io/builder" -) - -func init() { - db.RegisterModel(new(Build)) - db.RegisterModel(new(BuildIndex)) -} - -// Build represnets bot build task -type Build struct { - ID int64 - Name string - UUID string `xorm:"CHAR(36)"` - Index int64 `xorm:"index unique(repo_index)"` - RepoID int64 `xorm:"index unique(repo_index)"` - TriggerUserID int64 - TriggerUser *user_model.User `xorm:"-"` - Ref string - CommitSHA string - Event webhook.HookEventType - Token string // token for this task - Grant string // permissions for this task - EventPayload string `xorm:"LONGTEXT"` - RunnerID int64 `xorm:"index"` - Status core.BuildStatus `xorm:"index"` - Created timeutil.TimeStamp `xorm:"created"` - StartTime timeutil.TimeStamp - EndTime timeutil.TimeStamp - Updated timeutil.TimeStamp `xorm:"updated"` -} - -// TableName represents a bot build -func (Build) TableName() string { - return "bots_build" -} - -func (t *Build) HTMLURL() string { - return "" -} - -func updateRepoBuildsNumbers(ctx context.Context, repo *repo_model.Repository) error { - _, err := db.GetEngine(ctx).ID(repo.ID). - SetExpr("num_builds", - builder.Select("count(*)").From("bots_build"). - Where(builder.Eq{"repo_id": repo.ID}), - ). - SetExpr("num_closed_builds", - builder.Select("count(*)").From("bots_build"). - Where(builder.Eq{ - "repo_id": repo.ID, - }.And( - builder.In("status", core.StatusFailing, core.StatusKilled, core.StatusPassing), - ), - ), - ). - Update(repo) - return err -} - -// InsertBuild inserts a bot build task -func InsertBuild(t *Build, workflowsStatuses map[string]map[string]core.BuildStatus) error { - if t.UUID == "" { - t.UUID = uuid.New().String() - } - index, err := db.GetNextResourceIndex("bots_build_index", t.RepoID) - if err != nil { - return err - } - t.Index = index - - ctx, commiter, err := db.TxContext() - if err != nil { - return err - } - defer commiter.Close() - - if err := db.Insert(ctx, t); err != nil { - return err - } - - if err := updateRepoBuildsNumbers(ctx, &repo_model.Repository{ID: t.RepoID}); err != nil { - return err - } - - var buildStages []BuildStage - for filename, workflow := range workflowsStatuses { - for job, status := range workflow { - buildStages = append(buildStages, BuildStage{ - BuildID: t.ID, - Filename: filename, - Name: job, - Status: status, - }) - } - } - if err := db.Insert(ctx, buildStages); err != nil { - return err - } - - if err := commiter.Commit(); err != nil { - return err - } - - if err := CreateBuildLog(t.ID); err != nil { - log.Error("create build log for %d table failed, will try it again when received logs", t.ID) - } - return nil -} - -// UpdateBuild updates bot build -func UpdateBuild(t *Build, cols ...string) error { - _, err := db.GetEngine(db.DefaultContext).ID(t.ID).Cols(cols...).Update(t) - return err -} - -// ErrBuildNotExist represents an error for bot build not exist -type ErrBuildNotExist struct { - RepoID int64 - Index int64 - ID int64 - UUID string -} - -func (err ErrBuildNotExist) Error() string { - uuid := "" - if err.UUID != "" { - uuid = err.UUID - } - - if err.ID != 0 { - uuid = strconv.FormatInt(err.ID, 10) - } - return fmt.Sprintf("build [%s] is not exist", uuid) -} - -// GetBuildByUUID gets bot build by uuid -func GetBuildByUUID(buildUUID string) (*Build, error) { - var build Build - has, err := db.GetEngine(db.DefaultContext).Where("uuid=?", buildUUID).Get(&build) - if err != nil { - return nil, err - } else if !has { - return nil, ErrBuildNotExist{ - UUID: buildUUID, - } - } - return &build, nil -} - -func GetBuildByID(id int64) (*Build, error) { - var build Build - has, err := db.GetEngine(db.DefaultContext).Where("id=?", id).Get(&build) - if err != nil { - return nil, err - } else if !has { - return nil, ErrBuildNotExist{ - ID: id, - } - } - - return &build, nil -} - -// GetCurBuildByID return the build for the bot -func GetCurBuildByID(runnerID int64) (*Build, error) { - var builds []Build - err := db.GetEngine(db.DefaultContext). - Where("runner_id=?", runnerID). - And("status=?", core.StatusPending). - Asc("created"). - Find(&builds) - if err != nil { - return nil, err - } - if len(builds) == 0 { - return nil, nil - } - return &builds[0], err -} - -// GetCurBuildByUUID return the task for the bot -func GetCurBuildByUUID(runnerUUID string) (*Build, error) { - runner, err := GetRunnerByUUID(runnerUUID) - if err != nil { - return nil, err - } - return GetCurBuildByID(runner.ID) -} - -func GetBuildByRepoAndIndex(repoID, index int64) (*Build, error) { - var build Build - has, err := db.GetEngine(db.DefaultContext).Where("repo_id=?", repoID). - And("`index` = ?", index). - Get(&build) - if err != nil { - return nil, err - } else if !has { - return nil, ErrBuildNotExist{ - RepoID: repoID, - Index: index, - } - } - return &build, nil -} - -// AssignBuildToRunner assign a build to a runner -func AssignBuildToRunner(buildID, runnerID int64) error { - cnt, err := db.GetEngine(db.DefaultContext). - Where("runner_id=0"). - And("id=?", buildID). - Cols("runner_id"). - Update(&Build{ - RunnerID: runnerID, - }) - if err != nil { - return err - } - if cnt != 1 { - return errors.New("assign faild") - } - return nil -} - -type FindBuildOptions struct { - db.ListOptions - RepoID int64 - IsClosed util.OptionalBool -} - -func (opts FindBuildOptions) toConds() builder.Cond { - cond := builder.NewCond() - if opts.RepoID > 0 { - cond = cond.And(builder.Eq{"repo_id": opts.RepoID}) - } - if opts.IsClosed.IsTrue() { - cond = cond.And(builder.Expr("status IN (?,?,?,?)", core.StatusError, core.StatusFailing, core.StatusPassing)) - } else if opts.IsClosed.IsFalse() { - cond = cond.And(builder.Expr("status IN (?,?,?)", core.StatusWaiting, core.StatusPending, core.StatusRunning)) - } - return cond -} - -func FindBuilds(opts FindBuildOptions) (BuildList, error) { - sess := db.GetEngine(db.DefaultContext).Where(opts.toConds()) - if opts.ListOptions.PageSize > 0 { - skip, take := opts.GetSkipTake() - sess.Limit(take, skip) - } - var builds []*Build - return builds, sess.Find(&builds) -} - -func CountBuilds(opts FindBuildOptions) (int64, error) { - return db.GetEngine(db.DefaultContext).Table("bots_build").Where(opts.toConds()).Count() -} - -type BuildIndex db.ResourceIndex - -// TableName represents a bot build index -func (BuildIndex) TableName() string { - return "bots_build_index" -} diff --git a/models/bots/build_log.go b/models/bots/build_log.go deleted file mode 100644 index 7b1e22313f..0000000000 --- a/models/bots/build_log.go +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2022 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package bots - -import ( - "fmt" - - "code.gitea.io/gitea/models/db" - "code.gitea.io/gitea/modules/timeutil" -) - -// BuildLog represents a build's log, every build has a standalone table -type BuildLog struct { - ID int64 - StepID int64 `xorm:"index"` - Content string `xorm:"BINARY"` - Created timeutil.TimeStamp `xorm:"created"` -} - -func init() { - db.RegisterModel(new(BuildLog)) -} - -func GetBuildLogTableName(buildID int64) string { - return fmt.Sprintf("bots_build_log_%d", buildID) -} - -// CreateBuildLog table for a build -func CreateBuildLog(buildID int64) error { - return db.GetEngine(db.DefaultContext). - Table(GetBuildLogTableName(buildID)). - Sync2(new(BuildLog)) -} - -func GetBuildLogs(buildID, jobID int64) (logs []BuildLog, err error) { - err = db.GetEngine(db.DefaultContext).Table(GetBuildLogTableName(buildID)). - Where("build_step_id=?", jobID). - Find(&logs) - return -} diff --git a/models/bots/build_stage.go b/models/bots/build_stage.go deleted file mode 100644 index d0c5827436..0000000000 --- a/models/bots/build_stage.go +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright 2022 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package bots - -import ( - "context" - "fmt" - - "code.gitea.io/gitea/core" - "code.gitea.io/gitea/models/db" - "code.gitea.io/gitea/modules/timeutil" - "code.gitea.io/gitea/modules/util" - "xorm.io/builder" -) - -type BuildStage struct { - ID int64 - BuildID int64 `xorm:"index"` - Number int64 - Name string - Kind string - Type string - Machine string - OS string - Arch string - Filename string - Status core.BuildStatus - Started timeutil.TimeStamp - Stopped timeutil.TimeStamp - LogToFile bool // read log from database or from storage - Version int `xorm:"version"` - Created timeutil.TimeStamp `xorm:"created"` - Updated timeutil.TimeStamp `xorm:"updated"` -} - -func (bj BuildStage) TableName() string { - return "bots_build_stage" -} - -func init() { - db.RegisterModel(new(BuildStage)) -} - -type FindStageOptions struct { - db.ListOptions - BuildID int64 - IsClosed util.OptionalBool -} - -func (opts FindStageOptions) toConds() builder.Cond { - cond := builder.NewCond() - if opts.BuildID > 0 { - cond = cond.And(builder.Eq{"build_id": opts.BuildID}) - } - if opts.IsClosed.IsTrue() { - cond = cond.And(builder.Expr("status IN (?,?,?,?)", core.StatusError, core.StatusFailing, core.StatusPassing, core.StatusKilled)) - } else if opts.IsClosed.IsFalse() { - cond = cond.And(builder.Expr("status IN (?,?,?)", core.StatusPending, core.StatusRunning)) - } - return cond -} - -func FindStages(ctx context.Context, opts FindStageOptions) (BuildStageList, error) { - sess := db.GetEngine(ctx).Where(opts.toConds()) - if opts.ListOptions.PageSize > 0 { - skip, take := opts.GetSkipTake() - sess.Limit(take, skip) - } - var rows []*BuildStage - return rows, sess.Find(&rows) -} - -// GetStageByID gets build stage by id -func GetStageByID(id int64) (*BuildStage, error) { - var build BuildStage - has, err := db.GetEngine(db.DefaultContext).Where("id=?", id).Get(&build) - if err != nil { - return nil, err - } else if !has { - return nil, ErrBuildStageNotExist{ - ID: id, - } - } - return &build, nil -} - -// ErrBuildNotExist represents an error for bot build not exist -type ErrBuildStageNotExist struct { - ID int64 -} - -func (err ErrBuildStageNotExist) Error() string { - return fmt.Sprintf("build stage [%d] is not exist", err.ID) -} - -// UpdateBuildStage updates build stage -func UpdateBuildStage(t *BuildStage, cols ...string) (int64, error) { - return db.GetEngine(db.DefaultContext).ID(t.ID).Cols(cols...).Update(t) -} - -func GetBuildWorkflows(buildID int64) (map[string]map[string]*BuildStage, error) { - jobs := make(map[string]map[string]*BuildStage) - err := db.GetEngine(db.DefaultContext).Iterate(new(BuildStage), func(idx int, bean interface{}) error { - job := bean.(*BuildStage) - _, ok := jobs[job.Filename] - if !ok { - jobs[job.Filename] = make(map[string]*BuildStage) - } - jobs[job.Filename][job.Name] = job - return nil - }) - return jobs, err -} diff --git a/models/bots/build_stage_list.go b/models/bots/build_stage_list.go deleted file mode 100644 index 4137e63725..0000000000 --- a/models/bots/build_stage_list.go +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright 2022 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package bots - -type BuildStageList []*BuildStage diff --git a/models/bots/run.go b/models/bots/run.go index 2f689f12ae..bc607b35ba 100644 --- a/models/bots/run.go +++ b/models/bots/run.go @@ -15,6 +15,7 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/models/webhook" "code.gitea.io/gitea/modules/timeutil" + "xorm.io/builder" "github.com/nektos/act/pkg/jobparser" ) @@ -79,6 +80,30 @@ func (r *Run) LoadAttributes(ctx context.Context) error { return nil } +func updateRepoRunsNumbers(ctx context.Context, repo *repo_model.Repository) error { + _, err := db.GetEngine(ctx).ID(repo.ID). + SetExpr("num_runs", + builder.Select("count(*)").From("bots_run"). + Where(builder.Eq{"repo_id": repo.ID}), + ). + SetExpr("num_closed_runs", + builder.Select("count(*)").From("bots_run"). + Where(builder.Eq{ + "repo_id": repo.ID, + }.And( + builder.In("status", + core.StatusFailing, + core.StatusKilled, + core.StatusPassing, + core.StatusError, + ), + ), + ), + ). + Update(repo) + return err +} + // InsertRun inserts a bot run func InsertRun(run *Run, jobs []*jobparser.SingleWorkflow) error { var groupID int64 @@ -105,6 +130,18 @@ func InsertRun(run *Run, jobs []*jobparser.SingleWorkflow) error { return err } + if run.Repo == nil { + repo, err := repo_model.GetRepositoryByIDCtx(ctx, run.RepoID) + if err != nil { + return err + } + run.Repo = repo + } + + if err := updateRepoRunsNumbers(ctx, run.Repo); err != nil { + return err + } + runJobs := make([]*RunJob, 0, len(jobs)) for _, v := range jobs { id, job := v.Job() diff --git a/models/bots/run_job.go b/models/bots/run_job.go index 04de1f1a1b..d4b7d60914 100644 --- a/models/bots/run_job.go +++ b/models/bots/run_job.go @@ -16,8 +16,8 @@ import ( // RunJob represents a job of a run type RunJob struct { ID int64 - RunID int64 - Run *Run `xorm:"-"` + RunID int64 `xorm:"index"` + Run *Run `xorm:"-"` Name string Ready bool // ready to be executed Attempt int64 diff --git a/models/bots/build_list.go b/models/bots/run_list.go similarity index 51% rename from models/bots/build_list.go rename to models/bots/run_list.go index fdc6ef66f0..2f63261996 100644 --- a/models/bots/build_list.go +++ b/models/bots/run_list.go @@ -5,14 +5,18 @@ package bots import ( + "context" + "code.gitea.io/gitea/models/db" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/util" + "xorm.io/builder" ) -type BuildList []*Build +type RunList []*Run // GetUserIDs returns a slice of user's id -func (builds BuildList) GetUserIDs() []int64 { +func (builds RunList) GetUserIDs() []int64 { userIDsMap := make(map[int64]struct{}) for _, build := range builds { userIDsMap[build.TriggerUserID] = struct{}{} @@ -24,7 +28,7 @@ func (builds BuildList) GetUserIDs() []int64 { return userIDs } -func (builds BuildList) LoadTriggerUser() error { +func (builds RunList) LoadTriggerUser() error { userIDs := builds.GetUserIDs() users := make(map[int64]*user_model.User, len(userIDs)) if err := db.GetEngine(db.DefaultContext).In("id", userIDs).Find(&users); err != nil { @@ -35,3 +39,32 @@ func (builds BuildList) LoadTriggerUser() error { } return nil } + +type FindRunOptions struct { + db.ListOptions + RepoID int64 + IsClosed util.OptionalBool +} + +func (opts FindRunOptions) toConds() builder.Cond { + cond := builder.NewCond() + if opts.RepoID > 0 { + cond = cond.And(builder.Eq{"repo_id": opts.RepoID}) + } + if opts.IsClosed.IsFalse() { + + } else if opts.IsClosed.IsTrue() { + + } + return cond +} + +func FindRuns(ctx context.Context, opts FindRunOptions) (RunList, int64, error) { + e := db.GetEngine(ctx).Where(opts.toConds()) + if opts.PageSize>0&&opts.Page >=1 { + e.Limit(opts.PageSize, (opts.Page-1)*opts.PageSize) + } + var runs RunList + total, err := e.FindAndCount(&runs) + return runs, total, err +} diff --git a/models/bots/task.go b/models/bots/task.go index 58849d4086..7a883f3c13 100644 --- a/models/bots/task.go +++ b/models/bots/task.go @@ -10,6 +10,7 @@ import ( "code.gitea.io/gitea/core" "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/timeutil" runnerv1 "gitea.com/gitea/proto-go/runner/v1" @@ -141,6 +142,7 @@ func CreateTaskForRunner(runner *Runner) (*Task, bool, error) { // TODO: a more efficient way to filter labels var job *RunJob labels := append(runner.AgentLabels, runner.CustomLabels...) + log.Trace("runner labels: %v", labels) for _, v := range jobs { if isSubset(labels, v.RunsOn) { job = v diff --git a/models/bots/task_log.go b/models/bots/task_log.go index 5bdd11ec32..adfa17aa9c 100644 --- a/models/bots/task_log.go +++ b/models/bots/task_log.go @@ -19,10 +19,6 @@ type TaskLog struct { Content string `xorm:"LONGTEXT"` } -func init() { - db.RegisterModel(new(TaskLog)) -} - func GetTaskLogTableName(taskID int64) string { return fmt.Sprintf("bots_task_log_%d", taskID) } diff --git a/models/migrations/v-dev.go b/models/migrations/v-dev.go index 23695cf5e4..2facf0d0eb 100644 --- a/models/migrations/v-dev.go +++ b/models/migrations/v-dev.go @@ -16,46 +16,119 @@ func addBotTables(x *xorm.Engine) error { ID int64 UUID string `xorm:"CHAR(36) UNIQUE"` Name string `xorm:"VARCHAR(32) UNIQUE"` - OS string `xorm:"VARCHAR(16) index"` // the runner running os - Arch string `xorm:"VARCHAR(16) index"` // the runner running architecture - Type string `xorm:"VARCHAR(16)"` OwnerID int64 `xorm:"index"` // org level runner, 0 means system RepoID int64 `xorm:"index"` // repo level runner, if orgid also is zero, then it's a global Description string `xorm:"TEXT"` Base int // 0 native 1 docker 2 virtual machine RepoRange string // glob match which repositories could use this runner - Token string - LastOnline timeutil.TimeStamp `xorm:"index"` - Created timeutil.TimeStamp `xorm:"created"` + Token string `xorm:"CHAR(36) UNIQUE"` + + // instance status (idle, active, offline) + Status int32 + // Store OS and Artch. + AgentLabels []string + // Store custom labes use defined. + CustomLabels []string + + LastOnline timeutil.TimeStamp `xorm:"index"` + Created timeutil.TimeStamp `xorm:"created"` + Updated timeutil.TimeStamp `xorm:"updated"` + Deleted timeutil.TimeStamp `xorm:"deleted"` } - type BotsBuild struct { + type BotsRunnerToken struct { + ID int64 + Token string `xorm:"CHAR(36) UNIQUE"` + OwnerID int64 `xorm:"index"` // org level runner, 0 means system + RepoID int64 `xorm:"index"` // repo level runner, if orgid also is zero, then it's a global + IsActive bool + + Created timeutil.TimeStamp `xorm:"created"` + Updated timeutil.TimeStamp `xorm:"updated"` + Deleted timeutil.TimeStamp `xorm:"deleted"` + } + + type BotsRun struct { ID int64 - Title string - UUID string `xorm:"CHAR(36)"` - Index int64 `xorm:"index unique(repo_index)"` - RepoID int64 `xorm:"index unique(repo_index)"` + Name string + RepoID int64 `xorm:"index unique(repo_workflow_index)"` + WorkflowID string `xorm:"index unique(repo_workflow_index)"` // the name of workflow file + Index int64 `xorm:"index unique(repo_workflow_index)"` // a unique number for each run of a particular workflow in a repository TriggerUserID int64 Ref string CommitSHA string Event string - Token string // token for this task - Grant string // permissions for this task - EventPayload string `xorm:"LONGTEXT"` - RunnerID int64 `xorm:"index"` - Status int `xorm:"index"` - Created timeutil.TimeStamp `xorm:"created"` + Token string // token for this task + Grant string // permissions for this task + EventPayload string `xorm:"LONGTEXT"` + Status int32 `xorm:"index"` StartTime timeutil.TimeStamp EndTime timeutil.TimeStamp + Created timeutil.TimeStamp `xorm:"created"` Updated timeutil.TimeStamp `xorm:"updated"` } - type Repository struct { - NumBuilds int `xorm:"NOT NULL DEFAULT 0"` - NumClosedBuilds int `xorm:"NOT NULL DEFAULT 0"` + type BotsRunJob struct { + ID int64 + RunID int64 `xorm:"index"` + Name string + Ready bool // ready to be executed + Attempt int64 + WorkflowPayload []byte + JobID string // job id in workflow, not job's id + Needs []int64 `xorm:"JSON TEXT"` + RunsOn []string `xorm:"JSON TEXT"` + TaskID int64 // the latest task of the job + Status int32 `xorm:"index"` + Started timeutil.TimeStamp + Stopped timeutil.TimeStamp + Created timeutil.TimeStamp `xorm:"created"` + Updated timeutil.TimeStamp `xorm:"updated"` } - type BotsBuildIndex db.ResourceIndex + type Repository struct { + NumRuns int `xorm:"NOT NULL DEFAULT 0"` + NumClosedRuns int `xorm:"NOT NULL DEFAULT 0"` + } - return x.Sync2(new(BotsRunner), new(BotsBuild), new(Repository), new(BotsBuildIndex)) + type BotsRunIndex db.ResourceIndex + + type BotsTask struct { + ID int64 + JobID int64 + Attempt int64 + RunnerID int64 `xorm:"index"` + LogToFile bool // read log from database or from storage + LogURL string // url of the log file in storage + Result int32 + Started timeutil.TimeStamp + Stopped timeutil.TimeStamp + Created timeutil.TimeStamp `xorm:"created"` + Updated timeutil.TimeStamp `xorm:"updated"` + } + + type BotsTaskStep struct { + ID int64 + Name string + TaskID int64 `xorm:"index unique(task_number)"` + Number int64 `xorm:"index unique(task_number)"` + Result int32 + LogIndex int64 + LogLength int64 + Started timeutil.TimeStamp + Stopped timeutil.TimeStamp + Created timeutil.TimeStamp `xorm:"created"` + Updated timeutil.TimeStamp `xorm:"updated"` + } + + return x.Sync2( + new(BotsRunner), + new(BotsRunnerToken), + new(BotsRun), + new(BotsRunJob), + new(Repository), + new(BotsRunIndex), + new(BotsTask), + new(BotsTaskStep), + ) } diff --git a/modules/notification/bots/bots.go b/modules/notification/bots/bots.go index 0a1032648a..f877e72708 100644 --- a/modules/notification/bots/bots.go +++ b/modules/notification/bots/bots.go @@ -102,7 +102,7 @@ func notify(repo *repo_model.Repository, doer *user_model.User, payload, ref str continue } if err := bots_model.InsertRun(&run, jobs); err != nil { - log.Error("InsertBotTask: %v", err) + log.Error("InsertRun: %v", err) } } } diff --git a/routers/api/bots/bots.go b/routers/api/bots/bots.go index bddd4c92eb..2d28d0761c 100644 --- a/routers/api/bots/bots.go +++ b/routers/api/bots/bots.go @@ -25,9 +25,6 @@ func gRPCRouter(r *web.Route, fn grpc.RouteFn) { } func Routes(r *web.Route) *web.Route { - // socket connection - r.Get("/socket", socketServe) - gRPCRouter(r, grpc.V1Route) gRPCRouter(r, grpc.V1AlphaRoute) gRPCRouter(r, grpc.HealthRoute) diff --git a/routers/api/bots/runner/runner.go b/routers/api/bots/runner/runner.go index 46b6bc429c..d6274834b5 100644 --- a/routers/api/bots/runner/runner.go +++ b/routers/api/bots/runner/runner.go @@ -13,6 +13,7 @@ import ( "code.gitea.io/gitea/core" bots_model "code.gitea.io/gitea/models/bots" "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/timeutil" runnerv1 "gitea.com/gitea/proto-go/runner/v1" "gitea.com/gitea/proto-go/runner/v1/runnerv1connect" @@ -118,6 +119,7 @@ func (s *Service) FetchTask( var task *runnerv1.Task if t, ok, err := s.pickTask(ctx, runner); err != nil { + log.Error("pick task failed: %v", err) return nil, status.Errorf(codes.Internal, "pick task: %v", err) } else if ok { task = t diff --git a/routers/api/bots/scheduler/queue/queue.go b/routers/api/bots/scheduler/queue/queue.go index 95495be146..cb3bc916b4 100644 --- a/routers/api/bots/scheduler/queue/queue.go +++ b/routers/api/bots/scheduler/queue/queue.go @@ -10,7 +10,6 @@ import ( "time" "code.gitea.io/gitea/core" - "code.gitea.io/gitea/models/bots" runnerv1 "gitea.com/gitea/proto-go/runner/v1" ) @@ -92,7 +91,7 @@ func (q *queue) signal(ctx context.Context) error { if count == 0 { return nil } - items, err := bots.FindStages(ctx, bots.FindStageOptions{}) + /*items, err := bots.FindStages(ctx, bots.FindStageOptions{}) if err != nil { return err } @@ -138,7 +137,7 @@ func (q *queue) signal(ctx context.Context) error { delete(q.workers, w) break loop } - } + }*/ return nil } diff --git a/routers/api/bots/socket.go b/routers/api/bots/socket.go deleted file mode 100644 index 370b39fb1f..0000000000 --- a/routers/api/bots/socket.go +++ /dev/null @@ -1,207 +0,0 @@ -// Copyright 2022 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package bots - -import ( - "errors" - "fmt" - "net/http" - "strings" - "time" - - "code.gitea.io/gitea/core" - bots_model "code.gitea.io/gitea/models/bots" - "code.gitea.io/gitea/modules/json" - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/timeutil" - - "github.com/gorilla/websocket" -) - -var upgrader = websocket.Upgrader{ - ReadBufferSize: 4096, - WriteBufferSize: 4096, - EnableCompression: true, - CheckOrigin: func(r *http.Request) bool { - return true - }, -} - -var pongWait = 60 * time.Second - -type Message struct { - Version int // - Type int // message type, 1 register 2 error 3 task 4 no task - RunnerUUID string // runner uuid - BuildUUID string // build uuid - ErrCode int // error code - ErrContent string // errors message - EventName string - EventPayload string - JobID string // only run the special job, empty means run all the jobs -} - -const ( - version1 = 1 -) - -const ( - MsgTypeRegister = iota + 1 // register - MsgTypeError // error - MsgTypeRequestBuild // request build task - MsgTypeIdle // no task - MsgTypeBuildResult // build result - MsgTypeBuildJobResult // build job result -) - -func handleVersion1(r *http.Request, c *websocket.Conn, mt int, message []byte, msg *Message) error { - switch msg.Type { - case MsgTypeRegister: - log.Info("websocket[%s] registered", r.RemoteAddr) - runner, err := bots_model.GetRunnerByUUID(msg.RunnerUUID) - if err != nil { - if !errors.Is(err, bots_model.ErrRunnerNotExist{}) { - return fmt.Errorf("websocket[%s] get runner [%s] failed: %v", r.RemoteAddr, msg.RunnerUUID, err) - } - err = c.WriteMessage(mt, message) - if err != nil { - return fmt.Errorf("websocket[%s] sent message failed: %v", r.RemoteAddr, err) - } - } else { - fmt.Printf("-----%v\n", runner) - // TODO: handle read message - err = c.WriteMessage(mt, message) - if err != nil { - return fmt.Errorf("websocket[%s] sent message failed: %v", r.RemoteAddr, err) - } - } - case MsgTypeRequestBuild: - // TODO: find new task and send to client - build, err := bots_model.GetCurBuildByUUID(msg.RunnerUUID) - if err != nil { - return fmt.Errorf("websocket[%s] get task[%s] failed: %v", r.RemoteAddr, msg.RunnerUUID, err) - } - var returnMsg *Message - if build == nil { - time.Sleep(3 * time.Second) - returnMsg = &Message{ - Version: version1, - Type: MsgTypeIdle, - RunnerUUID: msg.RunnerUUID, - } - } else { - returnMsg = &Message{ - Version: version1, - Type: MsgTypeRequestBuild, - RunnerUUID: msg.RunnerUUID, - BuildUUID: build.UUID, - EventName: build.Event.Event(), - EventPayload: build.EventPayload, - } - } - bs, err := json.Marshal(&returnMsg) - if err != nil { - return fmt.Errorf("websocket[%s] marshal message failed: %v", r.RemoteAddr, err) - } - err = c.WriteMessage(mt, bs) - if err != nil { - return fmt.Errorf("websocket[%s] sent message failed: %v", r.RemoteAddr, err) - } - case MsgTypeBuildResult: - log.Info("websocket[%s] returned CI result: %v", r.RemoteAddr, msg) - build, err := bots_model.GetBuildByUUID(msg.BuildUUID) - if err != nil { - return fmt.Errorf("websocket[%s] get build by uuid failed: %v", r.RemoteAddr, err) - } - cols := []string{"status", "end_time"} - if msg.ErrCode == 0 { - build.Status = core.StatusPassing - } else { - build.Status = core.StatusFailing - } - build.EndTime = timeutil.TimeStampNow() - if err := bots_model.UpdateBuild(build, cols...); err != nil { - log.Error("websocket[%s] update build failed: %v", r.RemoteAddr, err) - } - default: - returnMsg := Message{ - Version: version1, - Type: MsgTypeError, - ErrCode: 1, - ErrContent: fmt.Sprintf("message type %d is not supported", msg.Type), - } - bs, err := json.Marshal(&returnMsg) - if err != nil { - return fmt.Errorf("websocket[%s] marshal message failed: %v", r.RemoteAddr, err) - } - err = c.WriteMessage(mt, bs) - if err != nil { - return fmt.Errorf("websocket[%s] sent message failed: %v", r.RemoteAddr, err) - } - } - return nil -} - -func socketServe(w http.ResponseWriter, r *http.Request) { - log.Trace("websocket init request begin from %s", r.RemoteAddr) - c, err := upgrader.Upgrade(w, r, nil) - if err != nil { - log.Error("websocket upgrade failed: %v", err) - return - } - defer c.Close() - log.Trace("websocket upgrade from %s successfully", r.RemoteAddr) - - _ = c.SetReadDeadline(time.Now().Add(pongWait)) - c.SetPongHandler(func(string) error { - return c.SetReadDeadline(time.Now().Add(pongWait)) - }) - - for { - // read message from client - mt, message, err := c.ReadMessage() - if err != nil { - if websocket.IsCloseError(err, websocket.CloseAbnormalClosure) || - websocket.IsCloseError(err, websocket.CloseNormalClosure) { - c.Close() - } else if !strings.Contains(err.Error(), "i/o timeout") { - log.Error("websocket[%s] read failed: %#v", r.RemoteAddr, err) - } - break - } - - log.Trace("websocket[%s] received message: %s", r.RemoteAddr, string(message)) - - // read message first - var msg Message - if err = json.Unmarshal(message, &msg); err != nil { - log.Error("websocket[%s] unmarshal failed: %#v", r.RemoteAddr, err) - break - } - - switch msg.Version { - case 1: - if err := handleVersion1(r, c, mt, message, &msg); err != nil { - log.Error("%v", err) - } - default: - returnMsg := Message{ - Version: 1, - Type: MsgTypeError, - ErrCode: 1, - ErrContent: "version is not supported", - } - bs, err := json.Marshal(&returnMsg) - if err != nil { - log.Error("websocket[%s] marshal message failed: %v", r.RemoteAddr, err) - } else { - err = c.WriteMessage(mt, bs) - if err != nil { - log.Error("websocket[%s] sent message failed: %v", r.RemoteAddr, err) - } - } - } - } -} diff --git a/routers/init.go b/routers/init.go index 65bf7b55d8..5d700792b4 100644 --- a/routers/init.go +++ b/routers/init.go @@ -40,7 +40,6 @@ import ( "code.gitea.io/gitea/services/auth" "code.gitea.io/gitea/services/auth/source/oauth2" "code.gitea.io/gitea/services/automerge" - bots_service "code.gitea.io/gitea/services/bots" "code.gitea.io/gitea/services/cron" "code.gitea.io/gitea/services/mailer" markup_service "code.gitea.io/gitea/services/markup" @@ -162,7 +161,6 @@ func GlobalInitInstalled(ctx context.Context) { mustInit(pull_service.Init) mustInit(automerge.Init) mustInit(task.Init) - mustInit(bots_service.Init) mustInit(repo_migrations.Init) eventsource.GetManager().Init() diff --git a/routers/web/repo/builds/builds.go b/routers/web/repo/builds/builds.go index 7058661dba..ce5d3904df 100644 --- a/routers/web/repo/builds/builds.go +++ b/routers/web/repo/builds/builds.go @@ -5,7 +5,6 @@ package builds import ( - "fmt" "net/http" bots_model "code.gitea.io/gitea/models/bots" @@ -46,7 +45,7 @@ func List(ctx *context.Context) { page = 1 } - opts := bots_model.FindBuildOptions{ + opts := bots_model.FindRunOptions{ ListOptions: db.ListOptions{ Page: page, PageSize: convert.ToCorrectPageSize(ctx.FormInt("limit")), @@ -58,7 +57,7 @@ func List(ctx *context.Context) { } else { opts.IsClosed = util.OptionalBoolFalse } - builds, err := bots_model.FindBuilds(opts) + builds, total, err := bots_model.FindRuns(ctx, opts) if err != nil { ctx.Error(http.StatusInternalServerError, err.Error()) return @@ -69,12 +68,6 @@ func List(ctx *context.Context) { return } - total, err := bots_model.CountBuilds(opts) - if err != nil { - ctx.Error(http.StatusInternalServerError, err.Error()) - return - } - ctx.Data["Builds"] = builds pager := context.NewPagination(int(total), opts.PageSize, opts.Page, 5) @@ -83,67 +76,3 @@ func List(ctx *context.Context) { ctx.HTML(http.StatusOK, tplListBuilds) } - -func ViewBuild(ctx *context.Context) { - index := ctx.ParamsInt64("index") - build, err := bots_model.GetBuildByRepoAndIndex(ctx.Repo.Repository.ID, index) - if err != nil { - ctx.Error(http.StatusInternalServerError, err.Error()) - return - } - - ctx.Data["Name"] = build.Name + " - " + ctx.Tr("repo.builds") - ctx.Data["PageIsBuildList"] = true - ctx.Data["Build"] = build - statuses, err := bots_model.GetBuildWorkflows(build.ID) - if err != nil { - ctx.Error(http.StatusInternalServerError, err.Error()) - return - } - - ctx.Data["WorkflowsStatuses"] = statuses - - ctx.HTML(http.StatusOK, tplViewBuild) -} - -func GetBuildJobLogs(ctx *context.Context) { - index := ctx.ParamsInt64("index") - build, err := bots_model.GetBuildByRepoAndIndex(ctx.Repo.Repository.ID, index) - if err != nil { - ctx.Error(http.StatusInternalServerError, err.Error()) - return - } - workflows, err := bots_model.GetBuildWorkflows(build.ID) - if err != nil { - ctx.Error(http.StatusInternalServerError, err.Error()) - return - } - var buildJob *bots_model.BuildStage - wf := ctx.Params("workflow") - jobname := ctx.Params("jobname") -LOOP_WORKFLOWS: - for workflow, jobs := range workflows { - if workflow == wf { - for _, job := range jobs { - if jobname == job.Name { - buildJob = job - break LOOP_WORKFLOWS - } - } - } - } - if buildJob == nil { - ctx.Error(http.StatusNotFound, fmt.Sprintf("workflow %s job %s not exist", wf, jobname)) - return - } - - // TODO: if buildJob.LogToFile is true, read the logs from the file - - logs, err := bots_model.GetBuildLogs(build.ID, buildJob.ID) - if err != nil { - ctx.Error(http.StatusInternalServerError, err.Error()) - return - } - - ctx.JSON(http.StatusOK, logs) -} diff --git a/routers/web/web.go b/routers/web/web.go index 9ed94099e2..b8e72d14eb 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -1189,10 +1189,14 @@ func RegisterRoutes(m *web.Route) { m.Group("/builds", func() { m.Get("", builds.List) - m.Group("/{index}", func() { - m.Get("", builds.ViewBuild) - m.Get("/{workflow}/job/{jobname}/logs", builds.GetBuildJobLogs) - }) + + m.Combo("/run/{runid}"). + Get(dev.BuildView). + Post(bindIgnErr(dev.BuildViewRequest{}), dev.BuildViewPost) + + m.Combo("/run/{runid}/jobs/{jobid}"). + Get(dev.BuildView). + Post(bindIgnErr(dev.BuildViewRequest{}), dev.BuildViewPost) }, reqRepoBuildsReader, builds.MustEnableBuilds) m.Group("/wiki", func() { diff --git a/services/bots/bots.go b/services/bots/bots.go index f894a678ea..64d151ea19 100644 --- a/services/bots/bots.go +++ b/services/bots/bots.go @@ -4,16 +4,9 @@ package bots -import ( - "fmt" - - bots_model "code.gitea.io/gitea/models/bots" - "code.gitea.io/gitea/modules/graceful" - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/queue" - //"code.gitea.io/gitea/modules/json" -) +//"code.gitea.io/gitea/modules/json" +/* // buildQueue is a global queue of bot build var buildQueue queue.Queue @@ -59,4 +52,4 @@ func handle(data ...queue.Data) []queue.Data { } } return unhandled -} +}*/