diff --git a/models/bots/runner.go b/models/bots/runner.go index dd97b92866..24ac953492 100644 --- a/models/bots/runner.go +++ b/models/bots/runner.go @@ -8,6 +8,8 @@ import ( "fmt" "code.gitea.io/gitea/models/db" + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/timeutil" "xorm.io/builder" @@ -25,16 +27,18 @@ func (err ErrRunnerNotExist) Error() string { // Runner represents runner machines type Runner struct { ID int64 - UUID string `xorm:"CHAR(36) UNIQUE"` - Name string `xorm:"VARCHAR(32) UNIQUE"` - OS string `xorm:"VARCHAR(16) index"` // the runner running os - Arch string `xorm:"VARCHAR(16) index"` // the runner running architecture - Type string `xorm:"VARCHAR(16)"` - OwnerID int64 `xorm:"index"` // org level runner, 0 means system - RepoID int64 `xorm:"index"` // repo level runner, if orgid also is zero, then it's a global - Description string `xorm:"TEXT"` - Base int // 0 native 1 docker 2 virtual machine - RepoRange string // glob match which repositories could use this runner + UUID string `xorm:"CHAR(36) UNIQUE"` + Name string `xorm:"VARCHAR(32) UNIQUE"` + OS string `xorm:"VARCHAR(16) index"` // the runner running os + Arch string `xorm:"VARCHAR(16) index"` // the runner running architecture + Type string `xorm:"VARCHAR(16)"` + OwnerID int64 `xorm:"index"` // org level runner, 0 means system + Owner *user_model.User `xorm:"-"` + RepoID int64 `xorm:"index"` // repo level runner, if orgid also is zero, then it's a global + Repo *repo_model.Repository `xorm:"-"` + Description string `xorm:"TEXT"` + Base int // 0 native 1 docker 2 virtual machine + RepoRange string // glob match which repositories could use this runner Token string LastOnline timeutil.TimeStamp `xorm:"index"` Created timeutil.TimeStamp `xorm:"created"` @@ -44,16 +48,28 @@ func (Runner) TableName() string { return "bots_runner" } +func (r *Runner) OwnType() string { + if r.OwnerID == 0 { + return "Global Type" + } + if r.RepoID == 0 { + return r.Owner.Name + } + + return r.Repo.FullName() +} + func init() { db.RegisterModel(&Runner{}) } -type GetRunnerOptions struct { +type FindRunnerOptions struct { + db.ListOptions RepoID int64 OwnerID int64 } -func (opts GetRunnerOptions) toCond() builder.Cond { +func (opts FindRunnerOptions) toCond() builder.Cond { cond := builder.NewCond() if opts.RepoID > 0 { cond = cond.And(builder.Eq{"repo_id": opts.RepoID}) @@ -65,8 +81,24 @@ func (opts GetRunnerOptions) toCond() builder.Cond { return cond } +func CountRunners(opts FindRunnerOptions) (int64, error) { + return db.GetEngine(db.DefaultContext). + Table("bots_runner"). + Where(opts.toCond()). + Count() +} + +func FindRunners(opts FindRunnerOptions) (runners RunnerList, err error) { + sess := db.GetEngine(db.DefaultContext). + Where(opts.toCond()) + if opts.Page > 0 { + sess.Limit(opts.PageSize, (opts.Page-1)*opts.PageSize) + } + return runners, sess.Find(&runners) +} + // GetUsableRunner returns the usable runner -func GetUsableRunner(opts GetRunnerOptions) (*Runner, error) { +func GetUsableRunner(opts FindRunnerOptions) (*Runner, error) { var runner Runner has, err := db.GetEngine(db.DefaultContext). Where(opts.toCond()). diff --git a/models/bots/runner_list.go b/models/bots/runner_list.go new file mode 100644 index 0000000000..197865d8e2 --- /dev/null +++ b/models/bots/runner_list.go @@ -0,0 +1,84 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package bots + +import ( + "context" + + "code.gitea.io/gitea/models/db" + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/container" +) + +type RunnerList []*Runner + +// GetUserIDs returns a slice of user's id +func (runners RunnerList) GetUserIDs() []int64 { + userIDsMap := make(map[int64]struct{}) + for _, runner := range runners { + if runner.OwnerID == 0 { + continue + } + userIDsMap[runner.OwnerID] = struct{}{} + } + userIDs := make([]int64, 0, len(userIDsMap)) + for userID := range userIDsMap { + userIDs = append(userIDs, userID) + } + return userIDs +} + +func (runners RunnerList) LoadOwners(ctx context.Context) error { + userIDs := runners.GetUserIDs() + users := make(map[int64]*user_model.User, len(userIDs)) + if err := db.GetEngine(ctx).In("id", userIDs).Find(&users); err != nil { + return err + } + for _, runner := range runners { + if runner.OwnerID > 0 && runner.Owner == nil { + runner.Owner = users[runner.OwnerID] + } + } + return nil +} + +func (runners RunnerList) getRepoIDs() []int64 { + repoIDs := make(map[int64]struct{}, len(runners)) + for _, runner := range runners { + if runner.RepoID == 0 { + continue + } + if _, ok := repoIDs[runner.RepoID]; !ok { + repoIDs[runner.RepoID] = struct{}{} + } + } + return container.KeysInt64(repoIDs) +} + +func (runners RunnerList) LoadRepos(ctx context.Context) error { + repoIDs := runners.getRepoIDs() + repos := make(map[int64]*repo_model.Repository, len(repoIDs)) + if err := db.GetEngine(ctx).In("id", repoIDs).Find(&repos); err != nil { + return err + } + + for _, runner := range runners { + if runner.RepoID > 0 && runner.Repo == nil { + runner.Repo = repos[runner.RepoID] + } + } + return nil +} + +func (runners RunnerList) LoadAttributes(ctx context.Context) error { + if err := runners.LoadOwners(ctx); err != nil { + return err + } + if err := runners.LoadRepos(ctx); err != nil { + return err + } + return nil +} diff --git a/routers/api/bots/bots.go b/routers/api/bots/bots.go index 5513caa801..89d445c6f1 100644 --- a/routers/api/bots/bots.go +++ b/routers/api/bots/bots.go @@ -14,6 +14,7 @@ import ( bots_model "code.gitea.io/gitea/models/bots" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/web" "github.com/gorilla/websocket" @@ -40,10 +41,113 @@ type Message struct { Version int // Type int // message type, 1 register 2 error 3 task 4 no task RunnerUUID string // runner uuid + BuildUUID string // build uuid ErrCode int // error code ErrContent string // errors message EventName string EventPayload string + JobID string // only run the special job, empty means run all the jobs +} + +const ( + version1 = 1 +) + +const ( + MsgTypeRegister = iota + 1 // register + MsgTypeError // error + MsgTypeRequestBuild // request build task + MsgTypeIdle // no task + MsgTypeBuildResult // build result + MsgTypeBuildJobResult // build job result +) + +func handleVersion1(r *http.Request, c *websocket.Conn, mt int, message []byte, msg *Message) error { + switch msg.Type { + case MsgTypeRegister: + log.Info("websocket[%s] registered", r.RemoteAddr) + runner, err := bots_model.GetRunnerByUUID(msg.RunnerUUID) + if err != nil { + if !errors.Is(err, bots_model.ErrRunnerNotExist{}) { + return fmt.Errorf("websocket[%s] get runner [%s] failed: %v", r.RemoteAddr, msg.RunnerUUID, err) + } + err = c.WriteMessage(mt, message) + if err != nil { + return fmt.Errorf("websocket[%s] sent message failed: %v", r.RemoteAddr, err) + } + } else { + fmt.Printf("-----%v\n", runner) + // TODO: handle read message + err = c.WriteMessage(mt, message) + if err != nil { + return fmt.Errorf("websocket[%s] sent message failed: %v", r.RemoteAddr, err) + } + } + case MsgTypeRequestBuild: + // TODO: find new task and send to client + build, err := bots_model.GetCurBuildByUUID(msg.RunnerUUID) + if err != nil { + return fmt.Errorf("websocket[%s] get task[%s] failed: %v", r.RemoteAddr, msg.RunnerUUID, err) + } + var returnMsg *Message + if build == nil { + time.Sleep(3 * time.Second) + returnMsg = &Message{ + Version: version1, + Type: MsgTypeIdle, + RunnerUUID: msg.RunnerUUID, + } + } else { + returnMsg = &Message{ + Version: version1, + Type: MsgTypeRequestBuild, + RunnerUUID: msg.RunnerUUID, + BuildUUID: build.UUID, + EventName: build.Event.Event(), + EventPayload: build.EventPayload, + } + } + bs, err := json.Marshal(&returnMsg) + if err != nil { + return fmt.Errorf("websocket[%s] marshal message failed: %v", r.RemoteAddr, err) + } + err = c.WriteMessage(mt, bs) + if err != nil { + return fmt.Errorf("websocket[%s] sent message failed: %v", r.RemoteAddr, err) + } + case MsgTypeBuildResult: + log.Info("websocket[%s] returned CI result: %v", r.RemoteAddr, msg) + build, err := bots_model.GetBuildByUUID(msg.BuildUUID) + if err != nil { + return fmt.Errorf("websocket[%s] get build by uuid failed: %v", r.RemoteAddr, err) + } + cols := []string{"status", "end_time"} + if msg.ErrCode == 0 { + build.Status = bots_model.BuildFinished + } else { + build.Status = bots_model.BuildFailed + } + build.EndTime = timeutil.TimeStampNow() + if err := bots_model.UpdateBuild(build, cols...); err != nil { + log.Error("websocket[%s] update build failed: %v", r.RemoteAddr, err) + } + default: + returnMsg := Message{ + Version: version1, + Type: MsgTypeError, + ErrCode: 1, + ErrContent: fmt.Sprintf("message type %d is not supported", msg.Type), + } + bs, err := json.Marshal(&returnMsg) + if err != nil { + return fmt.Errorf("websocket[%s] marshal message failed: %v", r.RemoteAddr, err) + } + err = c.WriteMessage(mt, bs) + if err != nil { + return fmt.Errorf("websocket[%s] sent message failed: %v", r.RemoteAddr, err) + } + } + return nil } func Serve(w http.ResponseWriter, r *http.Request) { @@ -59,24 +163,21 @@ func Serve(w http.ResponseWriter, r *http.Request) { c.SetReadDeadline(time.Now().Add(pongWait)) c.SetPongHandler(func(string) error { c.SetReadDeadline(time.Now().Add(pongWait)); return nil }) -MESSAGE_BUMP: for { - // read log from client + // read message from client mt, message, err := c.ReadMessage() if err != nil { if websocket.IsCloseError(err, websocket.CloseAbnormalClosure) || websocket.IsCloseError(err, websocket.CloseNormalClosure) { c.Close() - break - } - if !strings.Contains(err.Error(), "i/o timeout") { + } else if !strings.Contains(err.Error(), "i/o timeout") { log.Error("websocket[%s] read failed: %#v", r.RemoteAddr, err) } break - } else { - log.Trace("websocket[%s] received message: %s", r.RemoteAddr, message) } + log.Trace("websocket[%s] received message: %s", r.RemoteAddr, string(message)) + // read message first var msg Message if err = json.Unmarshal(message, &msg); err != nil { @@ -86,103 +187,25 @@ MESSAGE_BUMP: switch msg.Version { case 1: - switch msg.Type { - case 1: - log.Info("websocket[%s] registered", r.RemoteAddr) - runner, err := bots_model.GetRunnerByUUID(msg.RunnerUUID) - if err != nil { - if !errors.Is(err, bots_model.ErrRunnerNotExist{}) { - log.Error("websocket[%s] get runner [%s] failed: %v", r.RemoteAddr, msg.RunnerUUID, err) - break - } - err = c.WriteMessage(mt, message) - if err != nil { - log.Error("websocket[%s] sent message failed: %v", r.RemoteAddr, err) - break - } - } else { - fmt.Printf("-----%v\n", runner) - // TODO: handle read message - err = c.WriteMessage(mt, message) - if err != nil { - log.Error("websocket[%s] sent message failed: %v", r.RemoteAddr, err) - break - } - } - default: - returnMsg := Message{ - Version: 1, - Type: 2, - ErrCode: 1, - ErrContent: fmt.Sprintf("message type %d is not supported", msg.Type), - } - bs, err := json.Marshal(&returnMsg) - if err != nil { - log.Error("websocket[%s] marshal message failed: %v", r.RemoteAddr, err) - break MESSAGE_BUMP - } - err = c.WriteMessage(mt, bs) - if err != nil { - log.Error("websocket[%s] sent message failed: %v", r.RemoteAddr, err) - } - break MESSAGE_BUMP + if err := handleVersion1(r, c, mt, message, &msg); err != nil { + log.Error("%v", err) } default: returnMsg := Message{ Version: 1, - Type: 2, + Type: MsgTypeError, ErrCode: 1, ErrContent: "version is not supported", } bs, err := json.Marshal(&returnMsg) if err != nil { log.Error("websocket[%s] marshal message failed: %v", r.RemoteAddr, err) - break MESSAGE_BUMP - } - err = c.WriteMessage(mt, bs) - if err != nil { - log.Error("websocket[%s] sent message failed: %v", r.RemoteAddr, err) - } - break MESSAGE_BUMP - } - - // TODO: find new task and send to client - task, err := bots_model.GetCurBuildByUUID(msg.RunnerUUID) - if err != nil { - log.Error("websocket[%s] get task failed: %v", r.RemoteAddr, err) - break - } - if task == nil { - returnMsg := Message{ - Version: 1, - Type: 4, - } - bs, err := json.Marshal(&returnMsg) - if err != nil { - log.Error("websocket[%s] marshal message failed: %v", r.RemoteAddr, err) - break MESSAGE_BUMP - } - err = c.WriteMessage(mt, bs) - if err != nil { - log.Error("websocket[%s] sent message failed: %v", r.RemoteAddr, err) - } - } else { - returnMsg := Message{ - Version: 1, - Type: 3, - EventName: task.Event.Event(), - EventPayload: task.EventPayload, - } - bs, err := json.Marshal(&returnMsg) - if err != nil { - log.Error("websocket[%s] marshal message failed: %v", r.RemoteAddr, err) - break - } - err = c.WriteMessage(mt, bs) - if err != nil { - log.Error("websocket[%s] sent message failed: %v", r.RemoteAddr, err) + } else { + err = c.WriteMessage(mt, bs) + if err != nil { + log.Error("websocket[%s] sent message failed: %v", r.RemoteAddr, err) + } } } - } } diff --git a/routers/web/admin/runners.go b/routers/web/admin/runners.go new file mode 100644 index 0000000000..952c71466b --- /dev/null +++ b/routers/web/admin/runners.go @@ -0,0 +1,129 @@ +// Copyright 2014 The Gogs Authors. All rights reserved. +// Copyright 2020 The Gitea Authors. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package admin + +import ( + "net/http" + "net/url" + + bots_model "code.gitea.io/gitea/models/bots" + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/modules/base" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/setting" +) + +const ( + tplRunners base.TplName = "admin/runner/list" + tplRunnerNew base.TplName = "admin/runner/new" + tplRunnerEdit base.TplName = "admin/runner/edit" +) + +// Runners show all the runners +func Runners(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("admin.runners") + ctx.Data["PageIsAdmin"] = true + ctx.Data["PageIsAdminRunners"] = true + + page := ctx.FormInt("page") + if page <= 1 { + page = 1 + } + + opts := bots_model.FindRunnerOptions{ + ListOptions: db.ListOptions{ + Page: page, + PageSize: 100, + }, + } + + count, err := bots_model.CountRunners(opts) + if err != nil { + ctx.ServerError("SearchUsers", err) + return + } + + runners, err := bots_model.FindRunners(opts) + if err != nil { + ctx.ServerError("SearchUsers", err) + return + } + if err := runners.LoadAttributes(ctx); err != nil { + ctx.ServerError("LoadAttributes", err) + return + } + + ctx.Data["Runners"] = runners + ctx.Data["Total"] = count + + pager := context.NewPagination(int(count), opts.PageSize, opts.Page, 5) + ctx.Data["Page"] = pager + + ctx.HTML(http.StatusOK, tplRunners) +} + +// NewRunner render adding a new runner page +func NewRunner(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("admin.runners.new") + ctx.Data["PageIsAdmin"] = true + ctx.Data["PageIsAdminRunners"] = true + + ctx.HTML(http.StatusOK, tplRunnerNew) +} + +// NewRunnerPost response for adding a new runner +func NewRunnerPost(ctx *context.Context) { + // form := web.GetForm(ctx).(*forms.AdminCreateRunnerForm) + ctx.Data["Title"] = ctx.Tr("admin.runners.new") + ctx.Data["PageIsAdmin"] = true + ctx.Data["PageIsAdminRunners"] = true + + if ctx.HasError() { + ctx.HTML(http.StatusOK, tplRunnerNew) + return + } + + // ctx.Flash.Success(ctx.Tr("admin.runners.new_success", u.Name)) + // ctx.Redirect(setting.AppSubURL + "/admin/users/" + strconv.FormatInt(u.ID, 10)) +} + +// EditRunner show editing runner page +func EditRunner(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("admin.runners.edit") + ctx.Data["PageIsAdmin"] = true + ctx.Data["PageIsAdminRunners"] = true + + prepareUserInfo(ctx) + if ctx.Written() { + return + } + + ctx.HTML(http.StatusOK, tplUserEdit) +} + +// EditRunnerPost response for editing runner +func EditRunnerPost(ctx *context.Context) { + // form := web.GetForm(ctx).(*forms.AdminEditRunnerForm) + ctx.Data["Title"] = ctx.Tr("admin.runners.edit") + ctx.Data["PageIsAdmin"] = true + ctx.Data["PageIsAdminRunners"] = true + + if ctx.HasError() { + ctx.HTML(http.StatusOK, tplUserEdit) + return + } + + ctx.Flash.Success(ctx.Tr("admin.users.update_profile_success")) + ctx.Redirect(setting.AppSubURL + "/admin/users/" + url.PathEscape(ctx.Params(":userid"))) +} + +// DeleteRunner response for deleting a runner +func DeleteRunner(ctx *context.Context) { + ctx.Flash.Success(ctx.Tr("admin.runners.deletion_success")) + ctx.JSON(http.StatusOK, map[string]interface{}{ + "redirect": setting.AppSubURL + "/admin/runners", + }) +} diff --git a/routers/web/web.go b/routers/web/web.go index 09b2c3f812..a1f601b9c4 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -624,6 +624,13 @@ func RegisterRoutes(m *web.Route) { return } }) + + m.Group("/runners", func() { + m.Get("", admin.Runners) + m.Combo("/new").Get(admin.NewRunner).Post(bindIgnErr(forms.AdminCreateRunnerForm{}), admin.NewRunnerPost) + m.Combo("/{runnerid}").Get(admin.EditRunner).Post(bindIgnErr(forms.AdminEditRunnerForm{}), admin.EditRunnerPost) + m.Post("/{runnerid}/delete", admin.DeleteRunner) + }) }, func(ctx *context.Context) { ctx.Data["EnableOAuth2"] = setting.OAuth2.Enable ctx.Data["EnablePackages"] = setting.Packages.Enabled diff --git a/services/bots/bots.go b/services/bots/bots.go index 98b9476535..f894a678ea 100644 --- a/services/bots/bots.go +++ b/services/bots/bots.go @@ -24,7 +24,7 @@ func PushToQueue(task *bots_model.Build) { // Dispatch assign a task to a runner func Dispatch(task *bots_model.Build) (*bots_model.Runner, error) { - runner, err := bots_model.GetUsableRunner(bots_model.GetRunnerOptions{ + runner, err := bots_model.GetUsableRunner(bots_model.FindRunnerOptions{ RepoID: task.RepoID, }) if err != nil { diff --git a/services/forms/admin.go b/services/forms/admin.go index 537b9f982c..8c1ba0589e 100644 --- a/services/forms/admin.go +++ b/services/forms/admin.go @@ -71,3 +71,27 @@ func (f *AdminDashboardForm) Validate(req *http.Request, errs binding.Errors) bi ctx := context.GetContext(req) return middleware.Validate(errs, ctx.Data, f, ctx.Locale) } + +// AdminCreateRunnerForm form for admin to create runner +type AdminCreateRunnerForm struct { + Name string `binding:"Required"` + Type string +} + +// Validate validates form fields +func (f *AdminCreateRunnerForm) Validate(req *http.Request, errs binding.Errors) binding.Errors { + ctx := context.GetContext(req) + return middleware.Validate(errs, ctx.Data, f, ctx.Locale) +} + +// AdminEditRunnerForm form for admin to create runner +type AdminEditRunnerForm struct { + Name string `binding:"Required"` + Type string +} + +// Validate validates form fields +func (f *AdminEditRunnerForm) Validate(req *http.Request, errs binding.Errors) binding.Errors { + ctx := context.GetContext(req) + return middleware.Validate(errs, ctx.Data, f, ctx.Locale) +} diff --git a/templates/admin/navbar.tmpl b/templates/admin/navbar.tmpl index 1c8b12fc2f..d40814978a 100644 --- a/templates/admin/navbar.tmpl +++ b/templates/admin/navbar.tmpl @@ -33,6 +33,9 @@ {{.locale.Tr "settings.applications"}} {{end}} + + {{.locale.Tr "admin.runners"}} + {{.locale.Tr "admin.config"}} diff --git a/templates/admin/runner/edit.tmpl b/templates/admin/runner/edit.tmpl new file mode 100644 index 0000000000..1ee46f3077 --- /dev/null +++ b/templates/admin/runner/edit.tmpl @@ -0,0 +1,209 @@ +{{template "base/head" .}} +
{{.i18n.Tr "settings.delete_account_desc"}}
+ID | ++ {{.i18n.Tr "admin.runners.name"}} + | +{{.i18n.Tr "admin.runners.own_type"}} | +{{.i18n.Tr "admin.runners.uuid"}} | +{{.i18n.Tr "admin.runners.created"}} | +{{.i18n.Tr "admin.runners.edit"}} | +
---|---|---|---|---|---|
{{.ID}} | +{{.Name}} | +{{.OwnType}} | +{{.UUID}} | +{{.Created}} | +{{svg "octicon-pencil"}} | +