From feab4b160123cedca81bb0bc734c3c0160ebed11 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 20 Oct 2022 18:27:27 +0800 Subject: [PATCH] feat: update task status --- assets/go-licenses.json | 5 +++ go.mod | 4 +- go.sum | 2 + models/bots/run.go | 26 +++++++---- models/bots/run_job.go | 71 ++++++++++++++++++++++++++++--- models/bots/status.go | 38 +++++++++++++++++ models/bots/task.go | 50 ++++++++++++++++++---- models/bots/task_step.go | 1 + models/migrations/v-dev.go | 11 ++--- modules/notification/bots/bots.go | 3 +- routers/web/dev/buildview.go | 17 ++++---- 11 files changed, 186 insertions(+), 42 deletions(-) create mode 100644 models/bots/status.go diff --git a/assets/go-licenses.json b/assets/go-licenses.json index 3b8e84ecc4..9e6f128ca5 100644 --- a/assets/go-licenses.json +++ b/assets/go-licenses.json @@ -974,6 +974,11 @@ "path": "golang.org/x/crypto/LICENSE", "licenseText": "Copyright (c) 2009 The Go Authors. All rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are\nmet:\n\n * Redistributions of source code must retain the above copyright\nnotice, this list of conditions and the following disclaimer.\n * Redistributions in binary form must reproduce the above\ncopyright notice, this list of conditions and the following disclaimer\nin the documentation and/or other materials provided with the\ndistribution.\n * Neither the name of Google Inc. nor the names of its\ncontributors may be used to endorse or promote products derived from\nthis software without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n\"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\nLIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\nA PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\nOWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\nSPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\nLIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\nDATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\nTHEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" }, + { + "name": "golang.org/x/exp", + "path": "golang.org/x/exp/LICENSE", + "licenseText": "Copyright (c) 2009 The Go Authors. All rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are\nmet:\n\n * Redistributions of source code must retain the above copyright\nnotice, this list of conditions and the following disclaimer.\n * Redistributions in binary form must reproduce the above\ncopyright notice, this list of conditions and the following disclaimer\nin the documentation and/or other materials provided with the\ndistribution.\n * Neither the name of Google Inc. nor the names of its\ncontributors may be used to endorse or promote products derived from\nthis software without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n\"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\nLIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\nA PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\nOWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\nSPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\nLIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\nDATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\nTHEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" + }, { "name": "golang.org/x/mod/semver", "path": "golang.org/x/mod/semver/LICENSE", diff --git a/go.mod b/go.mod index aeedd6d172..ee111c32ad 100644 --- a/go.mod +++ b/go.mod @@ -100,8 +100,8 @@ require ( github.com/yuin/goldmark-meta v1.1.0 go.jolheiser.com/hcaptcha v0.0.4 go.jolheiser.com/pwn v0.0.3 - golang.org/x/crypto v0.2.1-0.20221112162523-6fad3dfc1891 - golang.org/x/net v0.2.0 + golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be + golang.org/x/net v0.0.0-20220927171203-f486391704dc golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1 golang.org/x/sys v0.2.0 golang.org/x/text v0.4.0 diff --git a/go.sum b/go.sum index b341c59a53..bba6b4231a 100644 --- a/go.sum +++ b/go.sum @@ -1643,6 +1643,8 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= +golang.org/x/exp v0.0.0-20221019170559-20944726eadf h1:nFVjjKDgNY37+ZSYCJmtYf7tOlfQswHqplG2eosjOMg= +golang.org/x/exp v0.0.0-20221019170559-20944726eadf/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= diff --git a/models/bots/run.go b/models/bots/run.go index af6e87a7cc..99a9cee4b5 100644 --- a/models/bots/run.go +++ b/models/bots/run.go @@ -33,12 +33,12 @@ type Run struct { Ref string CommitSHA string Event webhook.HookEventType - Token string // token for this task - Grant string // permissions for this task - EventPayload string `xorm:"LONGTEXT"` - Status core.BuildStatus `xorm:"index"` - StartTime timeutil.TimeStamp - EndTime timeutil.TimeStamp + Token string // token for this task + Grant string // permissions for this task + EventPayload string `xorm:"LONGTEXT"` + Status Status `xorm:"index"` + Started timeutil.TimeStamp + Stopped timeutil.TimeStamp Created timeutil.TimeStamp `xorm:"created"` Updated timeutil.TimeStamp `xorm:"updated"` } @@ -85,7 +85,7 @@ func (r *Run) LoadAttributes(ctx context.Context) error { } func (run *Run) TakeTime() time.Duration { - return run.EndTime.AsTime().Sub(run.StartTime.AsTime()) + return run.Started.AsTime().Sub(run.Stopped.AsTime()) } func updateRepoRunsNumbers(ctx context.Context, repo *repo_model.Repository) error { @@ -154,8 +154,7 @@ func InsertRun(run *Run, jobs []*jobparser.SingleWorkflow) error { JobID: id, Needs: nil, // TODO: analyse needs RunsOn: job.RunsOn(), - TaskID: 0, - Status: core.StatusPending, + Status: StatusWaiting, }) } if err := db.Insert(ctx, runJobs); err != nil { @@ -188,6 +187,15 @@ func GetRunByID(ctx context.Context, id int64) (*Run, error) { return &run, nil } +func UpdateRun(ctx context.Context, run *Run, cols ...string) error { + sess := db.GetEngine(ctx).ID(run.ID) + if len(cols) > 0 { + sess.Cols(cols...) + } + _, err := sess.Update(run) + return err +} + type RunIndex db.ResourceIndex func (RunIndex) TableName() string { diff --git a/models/bots/run_job.go b/models/bots/run_job.go index d4b7d60914..103b9f53ce 100644 --- a/models/bots/run_job.go +++ b/models/bots/run_job.go @@ -8,9 +8,10 @@ import ( "context" "fmt" - "code.gitea.io/gitea/core" "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/modules/timeutil" + + "golang.org/x/exp/slices" ) // RunJob represents a job of a run @@ -22,11 +23,11 @@ type RunJob struct { Ready bool // ready to be executed Attempt int64 WorkflowPayload []byte - JobID string // job id in workflow, not job's id - Needs []int64 `xorm:"JSON TEXT"` - RunsOn []string `xorm:"JSON TEXT"` - TaskID int64 // the latest task of the job - Status core.BuildStatus `xorm:"index"` + JobID string // job id in workflow, not job's id + Needs []int64 `xorm:"JSON TEXT"` + RunsOn []string `xorm:"JSON TEXT"` + TaskID int64 // the latest task of the job + Status Status `xorm:"index"` Started timeutil.TimeStamp Stopped timeutil.TimeStamp Created timeutil.TimeStamp `xorm:"created"` @@ -88,3 +89,61 @@ func GetRunJobsByRunID(ctx context.Context, runID int64) ([]*RunJob, error) { } return jobs, nil } + +func UpdateRunJob(ctx context.Context, job *RunJob, cols ...string) error { + e := db.GetEngine(ctx) + + sess := e.ID(job.ID) + if len(cols) > 0 { + sess.Cols(cols...) + } + if _, err := sess.Update(job); err != nil { + return err + } + + if !(slices.Contains(cols, "status") || job.Status != 0) { + return nil + } + + jobs, err := GetRunJobsByRunID(ctx, job.RunID) + if err != nil { + return err + } + + runStatus := aggregateJobStatus(jobs) + run := &Run{ + ID: job.RunID, + Status: runStatus, + } + if runStatus.IsDone() { + run.Stopped = timeutil.TimeStampNow() + } + return UpdateRun(ctx, run) +} + +func aggregateJobStatus(jobs []*RunJob) Status { + allDone := true + allWaiting := true + hasFailure := false + for _, job := range jobs { + if !job.Status.IsDone() { + allDone = false + } + if job.Status != StatusWaiting { + allWaiting = false + } + if job.Status == StatusFailure { + hasFailure = true + } + } + if allDone { + if hasFailure { + return StatusFailure + } + return StatusSuccess + } + if allWaiting { + return StatusWaiting + } + return StatusRunning +} diff --git a/models/bots/status.go b/models/bots/status.go new file mode 100644 index 0000000000..68cdc2e6b5 --- /dev/null +++ b/models/bots/status.go @@ -0,0 +1,38 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package bots + +// Status represents the status of Run, RunJob, Task, or TaskStep +type Status int + +const ( + StatusUnknown Status = iota // 0, consistent with runnerv1.Result_RESULT_UNSPECIFIED + StatusSuccess // 1, consistent with runnerv1.Result_RESULT_SUCCESS + StatusFailure // 2, consistent with runnerv1.Result_RESULT_FAILURE + StatusCancelled // 3, consistent with runnerv1.Result_RESULT_CANCELLED + StatusSkipped // 4, consistent with runnerv1.Result_RESULT_SKIPPED + StatusWaiting // 5 + StatusRunning // 6 +) + +// String returns the string name of the Status +func (s Status) String() string { + return statusNames[s] +} + +// IsDone returns whether the Status is final +func (s Status) IsDone() bool { + return s > StatusUnknown && s <= StatusSkipped +} + +var statusNames = map[Status]string{ + StatusUnknown: "unknown", + StatusWaiting: "waiting", + StatusRunning: "running", + StatusSuccess: "success", + StatusFailure: "failure", + StatusCancelled: "cancelled", + StatusSkipped: "skipped", +} diff --git a/models/bots/task.go b/models/bots/task.go index 10b5f03e86..5a9054d162 100644 --- a/models/bots/task.go +++ b/models/bots/task.go @@ -12,7 +12,6 @@ import ( "fmt" "io" - "code.gitea.io/gitea/core" "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/timeutil" @@ -30,6 +29,7 @@ type Task struct { Attempt int64 RunnerID int64 `xorm:"index"` Result runnerv1.Result + Status Status `xorm:"index"` Started timeutil.TimeStamp Stopped timeutil.TimeStamp @@ -82,6 +82,9 @@ func (task *Task) LoadAttributes(ctx context.Context) error { // FullSteps returns steps with "Set up job" and "Complete job" func (task *Task) FullSteps() []*TaskStep { + + // TODO: The logic here is too complex and tricky, may need to be rewritten + var firstStep, lastStep *TaskStep if l := len(task.Steps); l > 0 { firstStep = task.Steps[0] @@ -93,10 +96,15 @@ func (task *Task) FullSteps() []*TaskStep { Name: "Set up job", LogLength: task.LogLength, Started: task.Started, + Status: StatusWaiting, + } + if task.LogLength > 0 { + headStep.Status = StatusRunning } if firstStep != nil && firstStep.LogLength > 0 { headStep.LogLength = firstStep.LogIndex headStep.Stopped = firstStep.Started + headStep.Status = StatusSuccess } index += headStep.LogLength @@ -107,11 +115,13 @@ func (task *Task) FullSteps() []*TaskStep { tailStep := &TaskStep{ Name: "Complete job", Stopped: task.Stopped, + Status: StatusWaiting, } if lastStep != nil && lastStep.Result != runnerv1.Result_RESULT_UNSPECIFIED { tailStep.LogIndex = index tailStep.LogLength = task.LogLength - tailStep.LogIndex tailStep.Started = lastStep.Stopped + tailStep.Status = StatusSuccess } steps := make([]*TaskStep, 0, len(task.Steps)+2) steps = append(steps, headStep) @@ -202,13 +212,14 @@ func CreateTaskForRunner(runner *Runner) (*Task, bool, error) { now := timeutil.TimeStampNow() job.Attempt++ job.Started = now - job.Status = core.StatusRunning + job.Status = StatusRunning task := &Task{ JobID: job.ID, Attempt: job.Attempt, RunnerID: runner.ID, Started: now, + Status: StatusRunning, } var wolkflowJob *jobparser.Job @@ -235,6 +246,7 @@ func CreateTaskForRunner(runner *Runner) (*Task, bool, error) { Name: v.String(), TaskID: task.ID, Number: int64(i), + Status: StatusWaiting, } } if _, err := e.Insert(steps); err != nil { @@ -243,7 +255,7 @@ func CreateTaskForRunner(runner *Runner) (*Task, bool, error) { task.Steps = steps job.TaskID = task.ID - if _, err := e.ID(job.ID).Update(job); err != nil { + if err := UpdateRunJob(ctx, job); err != nil { return nil, false, err } @@ -277,15 +289,27 @@ func UpdateTaskByState(state *runnerv1.TaskState) error { } defer commiter.Close() + e := db.GetEngine(ctx) + task := &Task{} - if _, err := db.GetEngine(ctx).ID(state.Id).Get(task); err != nil { + if _, err := e.ID(state.Id).Get(task); err != nil { return err } task.Result = state.Result - task.Stopped = timeutil.TimeStamp(state.StoppedAt.AsTime().Unix()) + if task.Result != runnerv1.Result_RESULT_UNSPECIFIED { + task.Status = Status(task.Result) + task.Stopped = timeutil.TimeStamp(state.StoppedAt.AsTime().Unix()) + if err := UpdateRunJob(ctx, &RunJob{ + ID: task.JobID, + Status: task.Status, + Stopped: task.Stopped, + }); err != nil { + return err + } + } - if _, err := db.GetEngine(ctx).ID(task.ID).Update(task); err != nil { + if _, err := e.ID(task.ID).Update(task); err != nil { return err } @@ -293,14 +317,22 @@ func UpdateTaskByState(state *runnerv1.TaskState) error { return err } + prevStepDone := true for _, step := range task.Steps { if v, ok := stepStates[step.Number]; ok { step.Result = v.Result step.LogIndex = v.LogIndex step.LogLength = v.LogLength - if _, err := db.GetEngine(ctx).ID(step.ID).Update(step); err != nil { - return err - } + } + if step.Result != runnerv1.Result_RESULT_UNSPECIFIED { + step.Status = Status(step.Result) + prevStepDone = true + } else if prevStepDone { + step.Status = StatusRunning + prevStepDone = false + } + if _, err := e.ID(step.ID).Update(step); err != nil { + return err } } diff --git a/models/bots/task_step.go b/models/bots/task_step.go index 3cd7d1973f..1f529ae364 100644 --- a/models/bots/task_step.go +++ b/models/bots/task_step.go @@ -19,6 +19,7 @@ type TaskStep struct { TaskID int64 `xorm:"index unique(task_number)"` Number int64 `xorm:"index unique(task_number)"` Result runnerv1.Result + Status Status `xorm:"index"` LogIndex int64 LogLength int64 Started timeutil.TimeStamp diff --git a/models/migrations/v-dev.go b/models/migrations/v-dev.go index 27e94fdffe..b513bf7952 100644 --- a/models/migrations/v-dev.go +++ b/models/migrations/v-dev.go @@ -7,7 +7,6 @@ package migrations import ( "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/modules/timeutil" - "xorm.io/xorm" ) @@ -61,9 +60,9 @@ func addBotTables(x *xorm.Engine) error { Token string // token for this task Grant string // permissions for this task EventPayload string `xorm:"LONGTEXT"` - Status int32 `xorm:"index"` - StartTime timeutil.TimeStamp - EndTime timeutil.TimeStamp + Status int `xorm:"index"` + Started timeutil.TimeStamp + Stopped timeutil.TimeStamp Created timeutil.TimeStamp `xorm:"created"` Updated timeutil.TimeStamp `xorm:"updated"` } @@ -79,7 +78,7 @@ func addBotTables(x *xorm.Engine) error { Needs []int64 `xorm:"JSON TEXT"` RunsOn []string `xorm:"JSON TEXT"` TaskID int64 // the latest task of the job - Status int32 `xorm:"index"` + Status int `xorm:"index"` Started timeutil.TimeStamp Stopped timeutil.TimeStamp Created timeutil.TimeStamp `xorm:"created"` @@ -99,6 +98,7 @@ func addBotTables(x *xorm.Engine) error { Attempt int64 RunnerID int64 `xorm:"index"` Result int32 + Status int `xorm:"index"` Started timeutil.TimeStamp Stopped timeutil.TimeStamp LogFilename string // file name of log @@ -117,6 +117,7 @@ func addBotTables(x *xorm.Engine) error { TaskID int64 `xorm:"index unique(task_number)"` Number int64 `xorm:"index unique(task_number)"` Result int32 + Status int `xorm:"index"` LogIndex int64 LogLength int64 Started timeutil.TimeStamp diff --git a/modules/notification/bots/bots.go b/modules/notification/bots/bots.go index 45db27ca66..ef07f2323f 100644 --- a/modules/notification/bots/bots.go +++ b/modules/notification/bots/bots.go @@ -9,7 +9,6 @@ import ( "encoding/json" "fmt" - "code.gitea.io/gitea/core" bots_model "code.gitea.io/gitea/models/bots" "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/perm" @@ -94,7 +93,7 @@ func notify(repo *repo_model.Repository, doer *user_model.User, payload, ref str CommitSHA: commit.ID.String(), Event: evt, EventPayload: payload, - Status: core.StatusPending, + Status: bots_model.StatusWaiting, } if len(run.Title) > 255 { run.Title = run.Title[:255] diff --git a/routers/web/dev/buildview.go b/routers/web/dev/buildview.go index 63f212da49..270125e027 100644 --- a/routers/web/dev/buildview.go +++ b/routers/web/dev/buildview.go @@ -5,7 +5,6 @@ import ( "net/http" "time" - "code.gitea.io/gitea/core" bots_model "code.gitea.io/gitea/models/bots" "code.gitea.io/gitea/modules/bots" "code.gitea.io/gitea/modules/context" @@ -63,15 +62,15 @@ type BuildViewGroup struct { } type BuildViewJob struct { - Id int64 `json:"id"` - Name string `json:"name"` - Status core.BuildStatus `json:"status"` + Id int64 `json:"id"` + Name string `json:"name"` + Status string `json:"status"` } type BuildViewJobStep struct { - Summary string `json:"summary"` - Duration float64 `json:"duration"` - Status core.BuildStatus `json:"status"` + Summary string `json:"summary"` + Duration float64 `json:"duration"` + Status string `json:"status"` } type BuildViewStepLog struct { @@ -128,7 +127,7 @@ func BuildViewPost(ctx *context.Context) { respJobs[i] = &BuildViewJob{ Id: v.ID, Name: v.Name, - Status: v.Status, + Status: v.Status.String(), } } @@ -168,7 +167,7 @@ func BuildViewPost(ctx *context.Context) { resp.StateData.CurrentJobSteps[i] = BuildViewJobStep{ Summary: v.Name, Duration: float64(v.Stopped - v.Started), - Status: core.StatusRunning, // TODO: add status to step, + Status: v.Status.String(), } }