feat: parse workflows

This commit is contained in:
Jason Song 2022-10-08 18:20:17 +08:00
parent 91bea969d4
commit f355188e12
7 changed files with 173 additions and 55 deletions

File diff suppressed because one or more lines are too long

7
go.mod
View File

@ -170,6 +170,7 @@ require (
github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect
github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1 // indirect
github.com/envoyproxy/protoc-gen-validate v0.6.2 // indirect
github.com/fatih/color v1.13.0 // indirect
github.com/felixge/httpsnoop v1.0.3 // indirect
github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect
github.com/fullstorydev/grpcurl v1.8.1 // indirect
@ -227,6 +228,7 @@ require (
github.com/magiconair/properties v1.8.6 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/markbates/going v1.0.0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-runewidth v0.0.14 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
github.com/mholt/acmez v1.0.4 // indirect
@ -250,7 +252,9 @@ require (
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.37.0 // indirect
github.com/prometheus/procfs v0.8.0 // indirect
github.com/rhysd/actionlint v1.6.17 // indirect
github.com/rivo/uniseg v0.4.2 // indirect
github.com/robfig/cron v1.2.0 // indirect
github.com/rogpeppe/go-internal v1.9.0 // indirect
github.com/rs/xid v1.4.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
@ -291,6 +295,7 @@ require (
go.uber.org/multierr v1.8.0 // indirect
go.uber.org/zap v1.23.0 // indirect
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde // indirect
golang.org/x/time v0.0.0-20220922220347-f3bd1da661af // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90 // indirect
@ -314,4 +319,4 @@ exclude github.com/gofrs/uuid v4.0.0+incompatible
exclude github.com/goccy/go-json v0.4.11
replace github.com/nektos/act => gitea.com/gitea/act v0.0.0-20220922135643-52a5bba9e7fa
replace github.com/nektos/act => gitea.com/gitea/act v0.0.0-20221008102131-d89ab14fb580

13
go.sum
View File

@ -81,8 +81,8 @@ contrib.go.opencensus.io/resource v0.1.1/go.mod h1:F361eGI91LCmW1I/Saf+rX0+OFcig
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
git.sr.ht/~mariusor/go-xsd-duration v0.0.0-20220703122237-02e73435a078 h1:cliQ4HHsCo6xi2oWZYKWW4bly/Ory9FuTpFPRxj/mAg=
git.sr.ht/~mariusor/go-xsd-duration v0.0.0-20220703122237-02e73435a078/go.mod h1:g/V2Hjas6Z1UHUp4yIx6bATpNzJ7DYtD0FG3+xARWxs=
gitea.com/gitea/act v0.0.0-20220922135643-52a5bba9e7fa h1:HHqlvfIvqFlny3sgJgAM1BYeLNr1uM4yXtvF7aAoYK8=
gitea.com/gitea/act v0.0.0-20220922135643-52a5bba9e7fa/go.mod h1:9W/Nz16tjfnWp7O5DUo3EjZBnZFBI/5rlWstX4o7+hU=
gitea.com/gitea/act v0.0.0-20221008102131-d89ab14fb580 h1:F/VSl4oP5Gqe0FQ2i6BX/ODIhBc/cxfp5p5yA8pOSb4=
gitea.com/gitea/act v0.0.0-20221008102131-d89ab14fb580/go.mod h1:lpzib6X73FHLSaTqTakan1xcsCAVhlZvPSpLns7jkRo=
gitea.com/gitea/proto-go v0.0.0-20220921035813-6a5980d5b2a2 h1:SKAZ2Fnay8Yx1Yw1BSU/ATEk3/WAFL/lMXhvBBVzyu4=
gitea.com/gitea/proto-go v0.0.0-20220921035813-6a5980d5b2a2/go.mod h1:hD8YwSHusjwjEEgubW6XFvnZuNhMZTHz6lwjfltEt/Y=
gitea.com/go-chi/binding v0.0.0-20220309004920-114340dabecb h1:Yy0Bxzc8R2wxiwXoG/rECGplJUSpXqCsog9PuJFgiHs=
@ -449,6 +449,7 @@ github.com/ethantkoenig/rupture v1.0.1/go.mod h1:Sjqo/nbffZp1pVVXNGhpugIjsWmuS9K
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g=
github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw=
github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
@ -1057,7 +1058,9 @@ github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcncea
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc=
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
@ -1067,6 +1070,7 @@ github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd
github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
@ -1291,11 +1295,15 @@ github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqn
github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk=
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/rhysd/actionlint v1.6.17 h1:U6Idx7mLvWFBnaL8tm8oye0fdBUH9ehgoxG25bBy9p8=
github.com/rhysd/actionlint v1.6.17/go.mod h1:fCaKErUfJqiyFdeUp3858hULCCupHWlPRB3RhIHS6pY=
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.4.2 h1:YwD0ulJSJytLpiaWua0sBDusfsCZohxjxzVTYjwxfV8=
github.com/rivo/uniseg v0.4.2/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481/go.mod h1:C9WhFzY47SzYBIvzFqSvHIR6ROgDo4TtdTuRaOMjF/s=
github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ=
github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
@ -1775,6 +1783,7 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde h1:ejfdSekXMDxDLbRrJMwUk6KnSLZ2McaUCVcIKM+N6jc=
golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

View File

@ -5,20 +5,25 @@
package bots
import (
"fmt"
"hash/fnv"
"code.gitea.io/gitea/core"
"code.gitea.io/gitea/models/db"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/models/webhook"
"code.gitea.io/gitea/modules/timeutil"
"github.com/nektos/act/pkg/jobparser"
)
// Run represents a run of a workflow file
type Run struct {
ID int64
Name string
RepoID int64 `xorm:"index unique(repo_workflow_number)"`
WorkflowID string `xorm:"index unique(repo_workflow_number)"` // the name of workflow file
Number int64 `xorm:"index unique(repo_workflow_number)"` // a unique number for each run of a particular workflow in a repository
RepoID int64 `xorm:"index unique(repo_workflow_index)"`
WorkflowID string `xorm:"index unique(repo_workflow_index)"` // the name of workflow file
Index int64 `xorm:"index unique(repo_workflow_index)"` // a unique number for each run of a particular workflow in a repository
TriggerUserID int64
TriggerUser *user_model.User `xorm:"-"`
Ref string
@ -36,8 +41,66 @@ type Run struct {
func init() {
db.RegisterModel(new(Run))
db.RegisterModel(new(RunIndex))
}
func (Run) TableName() string {
return "bots_run"
}
// InsertRun inserts a bot run
func InsertRun(run *Run, jobs []*jobparser.SingleWorkflow) error {
var groupId int64
{
// tricky way to get resource group id
h := fnv.New64()
_, _ = h.Write([]byte(fmt.Sprintf("%d_%s", run.RepoID, run.WorkflowID)))
groupId = int64(h.Sum64())
}
index, err := db.GetNextResourceIndex("bots_run_index", groupId)
if err != nil {
return err
}
run.Index = index
ctx, commiter, err := db.TxContext()
if err != nil {
return err
}
defer commiter.Close()
if err := db.Insert(ctx, run); err != nil {
return err
}
runJobs := make([]*RunJob, 0, len(jobs))
for _, v := range jobs {
_, job := v.Job()
payload, _ := v.Marshal()
runJobs = append(runJobs, &RunJob{
RunID: run.ID,
Name: job.Name,
WorkflowPayload: payload,
Needs: nil, // TODO: analyse needs
RunsOn: job.RunsOn(),
TaskID: 0,
Status: core.StatusPending,
})
}
if err := db.Insert(ctx, runJobs); err != nil {
return err
}
if err := commiter.Commit(); err != nil {
return err
}
return nil
}
type RunIndex db.ResourceIndex
func (RunIndex) TableName() string {
return "bots_run_index"
}

View File

@ -15,12 +15,13 @@ type RunJob struct {
ID int64
RunID int64
Name string
WorkflowPayload string `xorm:"LONGTEXT"`
WorkflowPayload []byte
Needs []int64 `xorm:"JSON TEXT"`
RunsOn []string `xorm:"JSON TEXT"`
TaskID int64 // the latest task of the job
Status core.BuildStatus `xorm:"index"`
StartTime timeutil.TimeStamp
EndTime timeutil.TimeStamp
Started timeutil.TimeStamp
Stopped timeutil.TimeStamp
Created timeutil.TimeStamp `xorm:"created"`
Updated timeutil.TimeStamp `xorm:"updated"`
}

View File

@ -5,57 +5,75 @@
package bots
import (
"bytes"
"io"
"strings"
"code.gitea.io/gitea/models/webhook"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
"github.com/nektos/act/pkg/model"
)
func DetectWorkflows(commit *git.Commit, event webhook.HookEventType) (git.Entries, []map[string]*model.Job, error) {
tree, err := commit.SubTree(".github/workflows")
func ListWorkflows(commit *git.Commit) (git.Entries, error) {
tree, err := commit.SubTree(".gitea/workflows")
if _, ok := err.(git.ErrNotExist); ok {
tree, err = commit.SubTree(".gitea/workflows")
tree, err = commit.SubTree(".github/workflows")
}
if _, ok := err.(git.ErrNotExist); ok {
return nil, nil, nil
return nil, nil
}
if err != nil {
return nil, nil, err
return nil, err
}
entries, err := tree.ListEntriesRecursiveFast()
if err != nil {
return nil, nil, err
return nil, err
}
matchedEntries := make(git.Entries, 0, len(entries))
jobs := make([]map[string]*model.Job, 0, len(entries))
idx := 0
for _, entry := range entries {
if !strings.HasSuffix(entry.Name(), ".yml") && !strings.HasSuffix(entry.Name(), ".yaml") {
continue
}
entries[idx] = entry
idx++
}
return entries[:idx], nil
}
func DetectWorkflows(commit *git.Commit, event webhook.HookEventType) (map[string][]byte, error) {
entries, err := ListWorkflows(commit)
if err != nil {
return nil, err
}
workflows := make(map[string][]byte, len(entries))
for _, entry := range entries {
f, err := entry.Blob().DataAsync()
if err != nil {
return nil, nil, err
return nil, err
}
workflow, err := model.ReadWorkflow(f)
content, err := io.ReadAll(f)
_ = f.Close()
if err != nil {
f.Close()
return nil, nil, err
return nil, err
}
workflow, err := model.ReadWorkflow(bytes.NewReader(content))
if err != nil {
log.Warn("ignore invalid workflow %q: %v", entry.Name(), err)
continue
}
for _, e := range workflow.On() {
if e == event.Event() {
matchedEntries = append(matchedEntries, entry)
jobs = append(jobs, workflow.Jobs)
workflows[entry.Name()] = content
break
}
}
f.Close()
}
return matchedEntries, jobs, nil
return workflows, nil
}

View File

@ -26,7 +26,8 @@ import (
"code.gitea.io/gitea/modules/repository"
"code.gitea.io/gitea/modules/setting"
api "code.gitea.io/gitea/modules/structs"
bots_service "code.gitea.io/gitea/services/bots"
"github.com/nektos/act/pkg/jobparser"
)
type botsNotifier struct {
@ -72,41 +73,37 @@ func notify(repo *repo_model.Repository, doer *user_model.User, payload, ref str
return
}
matchedEntries, jobs, err := bots_module.DetectWorkflows(commit, evt)
workflows, err := bots_module.DetectWorkflows(commit, evt)
if err != nil {
log.Error("detectWorkflows: %v", err)
log.Error("DetectWorkflows: %v", err)
return
}
log.Trace("detected %s has %d entries", commit.ID, len(matchedEntries))
if len(matchedEntries) == 0 {
if len(workflows) == 0 {
log.Trace("repo %s with commit %s couldn't find workflows", repo.RepoPath(), commit.ID)
return
}
workflowsStatuses := make(map[string]map[string]core.BuildStatus)
for i, entry := range matchedEntries {
taskStatuses := make(map[string]core.BuildStatus)
for k := range jobs[i] {
taskStatuses[k] = core.StatusPending
for id, content := range workflows {
run := bots_model.Run{
Name: commit.Message(),
RepoID: repo.ID,
WorkflowID: id,
TriggerUserID: doer.ID,
Ref: ref,
CommitSHA: commit.ID.String(),
Event: evt,
EventPayload: payload,
Status: core.StatusPending,
}
jobs, err := jobparser.Parse(content) // TODO: parse with options
if err != nil {
log.Error("jobparser.Parse: %v", err)
continue
}
if err := bots_model.InsertRun(&run, jobs); err != nil {
log.Error("InsertBotTask: %v", err)
}
workflowsStatuses[entry.Name()] = taskStatuses
}
build := bots_model.Build{
Name: commit.Message(),
RepoID: repo.ID,
TriggerUserID: doer.ID,
Event: evt,
EventPayload: payload,
Status: core.StatusPending,
Ref: ref,
CommitSHA: commit.ID.String(),
}
if err := bots_model.InsertBuild(&build, workflowsStatuses); err != nil {
log.Error("InsertBotTask: %v", err)
} else {
bots_service.PushToQueue(&build)
}
}