diff --git a/go.mod b/go.mod index ca8c79c689..5e6529e0e4 100644 --- a/go.mod +++ b/go.mod @@ -233,6 +233,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mrjones/oauth v0.0.0-20190623134757-126b35219450 // indirect github.com/mschoch/smat v0.2.0 // indirect + github.com/nektos/act v0.2.26 // indirect github.com/nwaples/rardecode v1.1.3 // indirect github.com/oklog/ulid v1.3.1 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect diff --git a/models/bots/build.go b/models/bots/build.go new file mode 100644 index 0000000000..cd24205370 --- /dev/null +++ b/models/bots/build.go @@ -0,0 +1,289 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package bots + +import ( + "context" + "errors" + "fmt" + + "code.gitea.io/gitea/models/db" + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/models/webhook" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/modules/util" + + "github.com/google/uuid" + "xorm.io/builder" +) + +func init() { + db.RegisterModel(new(Build)) + db.RegisterModel(new(BuildIndex)) +} + +// BuildStatus represents a build status +type BuildStatus int + +// enumerate all the statuses of bot build +const ( + BuildPending BuildStatus = iota // wait for assign + BuildAssigned // assigned to a runner + BuildRunning // running + BuildFailed + BuildFinished + BuildCanceled + BuildTimeout +) + +func (status BuildStatus) IsPending() bool { + return status == BuildPending || status == BuildAssigned +} + +func (status BuildStatus) IsRunning() bool { + return status == BuildRunning +} + +func (status BuildStatus) IsFailed() bool { + return status == BuildFailed || status == BuildCanceled || status == BuildTimeout +} + +func (status BuildStatus) IsSuccess() bool { + return status == BuildFinished +} + +// Build represnets bot build task +type Build struct { + ID int64 + Title string + UUID string `xorm:"CHAR(36)"` + Index int64 `xorm:"index unique(repo_index)"` + RepoID int64 `xorm:"index unique(repo_index)"` + TriggerUserID int64 + TriggerUser *user_model.User `xorm:"-"` + Ref string + CommitSHA string + Event webhook.HookEventType + Token string // token for this task + Grant string // permissions for this task + EventPayload string `xorm:"LONGTEXT"` + RunnerID int64 `xorm:"index"` + Status BuildStatus `xorm:"index"` + Created timeutil.TimeStamp `xorm:"created"` + StartTime timeutil.TimeStamp + EndTime timeutil.TimeStamp + Updated timeutil.TimeStamp `xorm:"updated"` +} + +// TableName represents a bot build +func (Build) TableName() string { + return "bots_build" +} + +func (t *Build) HTMLURL() string { + return fmt.Sprintf("") +} + +func updateRepoBuildsNumbers(ctx context.Context, repo *repo_model.Repository) error { + _, err := db.GetEngine(ctx).ID(repo.ID). + SetExpr("num_builds", + builder.Select("count(*)").From("bots_build"). + Where(builder.Eq{"repo_id": repo.ID}), + ). + SetExpr("num_closed_builds", + builder.Select("count(*)").From("bots_build"). + Where(builder.Eq{ + "repo_id": repo.ID, + }.And( + builder.In("status", BuildFailed, BuildCanceled, BuildTimeout, BuildFinished), + ), + ), + ). + Update(repo) + return err +} + +// InsertBuild inserts a bot build task +func InsertBuild(t *Build, workflowsStatuses map[string]map[string]BuildStatus) error { + if t.UUID == "" { + t.UUID = uuid.New().String() + } + index, err := db.GetNextResourceIndex("bots_build_index", t.RepoID) + if err != nil { + return err + } + t.Index = index + + ctx, commiter, err := db.TxContext() + if err != nil { + return err + } + defer commiter.Close() + + if err := db.Insert(ctx, t); err != nil { + return err + } + + if err := updateRepoBuildsNumbers(ctx, &repo_model.Repository{ID: t.RepoID}); err != nil { + return err + } + + var buildJobs []BuildJob + for filename, workflow := range workflowsStatuses { + for job, status := range workflow { + buildJobs = append(buildJobs, BuildJob{ + BuildID: t.ID, + Filename: filename, + Jobname: job, + Status: status, + }) + } + } + if err := db.Insert(ctx, buildJobs); err != nil { + return err + } + + if err := commiter.Commit(); err != nil { + return err + } + + if err := CreateBuildLog(t.ID); err != nil { + log.Error("create build log for %d table failed, will try it again when received logs", t.ID) + } + return nil +} + +// UpdateBuild updates bot build +func UpdateBuild(t *Build, cols ...string) error { + _, err := db.GetEngine(db.DefaultContext).ID(t.ID).Cols(cols...).Update(t) + return err +} + +// ErrBuildNotExist represents an error for bot build not exist +type ErrBuildNotExist struct { + RepoID int64 + Index int64 + UUID string +} + +func (err ErrBuildNotExist) Error() string { + return fmt.Sprintf("Bot build [%s] is not exist", err.UUID) +} + +// GetBuildByUUID gets bot build by uuid +func GetBuildByUUID(buildUUID string) (*Build, error) { + var build Build + has, err := db.GetEngine(db.DefaultContext).Where("uuid=?", buildUUID).Get(&build) + if err != nil { + return nil, err + } else if !has { + return nil, ErrBuildNotExist{ + UUID: buildUUID, + } + } + return &build, nil +} + +// GetCurBuildByID return the build for the bot +func GetCurBuildByID(runnerID int64) (*Build, error) { + var builds []Build + err := db.GetEngine(db.DefaultContext). + Where("runner_id=?", runnerID). + And("status=?", BuildPending). + Asc("created"). + Find(&builds) + if err != nil { + return nil, err + } + if len(builds) == 0 { + return nil, nil + } + return &builds[0], err +} + +// GetCurBuildByUUID return the task for the bot +func GetCurBuildByUUID(runnerUUID string) (*Build, error) { + runner, err := GetRunnerByUUID(runnerUUID) + if err != nil { + return nil, err + } + return GetCurBuildByID(runner.ID) +} + +func GetBuildByRepoAndIndex(repoID, index int64) (*Build, error) { + var build Build + has, err := db.GetEngine(db.DefaultContext).Where("repo_id=?", repoID). + And("`index` = ?", index). + Get(&build) + if err != nil { + return nil, err + } else if !has { + return nil, ErrBuildNotExist{ + RepoID: repoID, + Index: index, + } + } + return &build, nil +} + +// AssignBuildToRunner assign a build to a runner +func AssignBuildToRunner(buildID int64, runnerID int64) error { + cnt, err := db.GetEngine(db.DefaultContext). + Where("runner_id=0"). + And("id=?", buildID). + Cols("runner_id"). + Update(&Build{ + RunnerID: runnerID, + }) + if err != nil { + return err + } + if cnt != 1 { + return errors.New("assign faild") + } + return nil +} + +type FindBuildOptions struct { + db.ListOptions + RepoID int64 + IsClosed util.OptionalBool +} + +func (opts FindBuildOptions) toConds() builder.Cond { + cond := builder.NewCond() + if opts.RepoID > 0 { + cond = cond.And(builder.Eq{"repo_id": opts.RepoID}) + } + if opts.IsClosed.IsTrue() { + cond = cond.And(builder.Expr("status IN (?,?,?,?)", BuildCanceled, BuildFailed, BuildTimeout, BuildFinished)) + } else if opts.IsClosed.IsFalse() { + cond = cond.And(builder.Expr("status IN (?,?,?)", BuildPending, BuildAssigned, BuildRunning)) + } + return cond +} + +func FindBuilds(opts FindBuildOptions) (BuildList, error) { + sess := db.GetEngine(db.DefaultContext).Where(opts.toConds()) + if opts.ListOptions.PageSize > 0 { + skip, take := opts.GetSkipTake() + sess.Limit(take, skip) + } + var builds []*Build + return builds, sess.Find(&builds) +} + +func CountBuilds(opts FindBuildOptions) (int64, error) { + return db.GetEngine(db.DefaultContext).Table("bots_build").Where(opts.toConds()).Count() +} + +type BuildIndex db.ResourceIndex + +// TableName represents a bot build index +func (BuildIndex) TableName() string { + return "bots_build_index" +} diff --git a/models/bots/build_job.go b/models/bots/build_job.go new file mode 100644 index 0000000000..62ec3b0fd0 --- /dev/null +++ b/models/bots/build_job.go @@ -0,0 +1,42 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package bots + +import ( + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/modules/timeutil" +) + +type BuildJob struct { + ID int64 + BuildID int64 `xorm:"index"` + Filename string + Jobname string + Status BuildStatus + LogToFile bool // read log from database or from storage + Created timeutil.TimeStamp `xorm:"created"` +} + +func (bj BuildJob) TableName() string { + return "bots_build_job" +} + +func init() { + db.RegisterModel(new(BuildJob)) +} + +func GetBuildWorkflows(buildID int64) (map[string]map[string]*BuildJob, error) { + jobs := make(map[string]map[string]*BuildJob) + err := db.GetEngine(db.DefaultContext).Iterate(new(BuildJob), func(idx int, bean interface{}) error { + job := bean.(*BuildJob) + _, ok := jobs[job.Filename] + if !ok { + jobs[job.Filename] = make(map[string]*BuildJob) + } + jobs[job.Filename][job.Jobname] = job + return nil + }) + return jobs, err +} diff --git a/models/bots/task_list.go b/models/bots/build_list.go similarity index 73% rename from models/bots/task_list.go rename to models/bots/build_list.go index 351e334f7d..fdc6ef66f0 100644 --- a/models/bots/task_list.go +++ b/models/bots/build_list.go @@ -9,13 +9,13 @@ import ( user_model "code.gitea.io/gitea/models/user" ) -type TaskList []*Task +type BuildList []*Build // GetUserIDs returns a slice of user's id -func (tasks TaskList) GetUserIDs() []int64 { +func (builds BuildList) GetUserIDs() []int64 { userIDsMap := make(map[int64]struct{}) - for _, task := range tasks { - userIDsMap[task.TriggerUserID] = struct{}{} + for _, build := range builds { + userIDsMap[build.TriggerUserID] = struct{}{} } userIDs := make([]int64, 0, len(userIDsMap)) for userID := range userIDsMap { @@ -24,13 +24,13 @@ func (tasks TaskList) GetUserIDs() []int64 { return userIDs } -func (tasks TaskList) LoadTriggerUser() error { - userIDs := tasks.GetUserIDs() +func (builds BuildList) LoadTriggerUser() error { + userIDs := builds.GetUserIDs() users := make(map[int64]*user_model.User, len(userIDs)) if err := db.GetEngine(db.DefaultContext).In("id", userIDs).Find(&users); err != nil { return err } - for _, task := range tasks { + for _, task := range builds { task.TriggerUser = users[task.TriggerUserID] } return nil diff --git a/models/bots/build_log.go b/models/bots/build_log.go new file mode 100644 index 0000000000..93d3b695e0 --- /dev/null +++ b/models/bots/build_log.go @@ -0,0 +1,43 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package bots + +import ( + "fmt" + + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/modules/timeutil" +) + +// BuildLog represents a build's log, every build has a standalone table +type BuildLog struct { + ID int64 + BuildJobID int64 `xorm:"index"` + LineNumber int + Content string `xorm:"LONGTEXT"` + Created timeutil.TimeStamp `xorm:"created"` +} + +func init() { + db.RegisterModel(new(BuildLog)) +} + +func GetBuildLogTableName(buildID int64) string { + return fmt.Sprintf("bots_build_log_%d", buildID) +} + +// CreateBuildLog table for a build +func CreateBuildLog(buildID int64) error { + return db.GetEngine(db.DefaultContext). + Table(GetBuildLogTableName(buildID)). + Sync2(new(BuildLog)) +} + +func GetBuildLogs(buildID, jobID int64) (logs []BuildLog, err error) { + err = db.GetEngine(db.DefaultContext).Table(GetBuildLogTableName(buildID)). + Where("build_job_id=?", jobID). + Find(&logs) + return +} diff --git a/models/bots/task.go b/models/bots/task.go deleted file mode 100644 index 73a9da867c..0000000000 --- a/models/bots/task.go +++ /dev/null @@ -1,266 +0,0 @@ -// Copyright 2022 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package bots - -import ( - "context" - "errors" - "fmt" - - "code.gitea.io/gitea/models/db" - repo_model "code.gitea.io/gitea/models/repo" - user_model "code.gitea.io/gitea/models/user" - "code.gitea.io/gitea/models/webhook" - "code.gitea.io/gitea/modules/timeutil" - "code.gitea.io/gitea/modules/util" - - "github.com/google/uuid" - "xorm.io/builder" -) - -func init() { - db.RegisterModel(new(Task)) - db.RegisterModel(new(BuildIndex)) -} - -// TaskStatus represents a task status -type TaskStatus int - -// enumerate all the statuses of bot task -const ( - TaskPending TaskStatus = iota // wait for assign - TaskAssigned // assigned to a runner - TaskRunning // running - TaskFailed - TaskFinished - TaskCanceled - TaskTimeout -) - -// Task represnets bot tasks -type Task struct { - ID int64 - Title string - UUID string `xorm:"CHAR(36)"` - Index int64 `xorm:"index unique(repo_index)"` - RepoID int64 `xorm:"index unique(repo_index)"` - TriggerUserID int64 - TriggerUser *user_model.User `xorm:"-"` - Ref string - CommitSHA string - Event webhook.HookEventType - Token string // token for this task - Grant string // permissions for this task - EventPayload string `xorm:"LONGTEXT"` - RunnerID int64 `xorm:"index"` - Status TaskStatus `xorm:"index"` - WorkflowsStatuses map[string]map[string]TaskStatus `xorm:"LONGTEXT"` - Created timeutil.TimeStamp `xorm:"created"` - StartTime timeutil.TimeStamp - EndTime timeutil.TimeStamp - Updated timeutil.TimeStamp `xorm:"updated"` -} - -func (t *Task) IsPending() bool { - return t.Status == TaskPending || t.Status == TaskAssigned -} - -func (t *Task) IsRunning() bool { - return t.Status == TaskRunning -} - -func (t *Task) IsFailed() bool { - return t.Status == TaskFailed || t.Status == TaskCanceled || t.Status == TaskTimeout -} - -func (t *Task) IsSuccess() bool { - return t.Status == TaskFinished -} - -// TableName represents a bot task -func (Task) TableName() string { - return "bots_task" -} - -func (t *Task) HTMLURL() string { - return fmt.Sprintf("") -} - -func updateRepoBuildsNumbers(ctx context.Context, repo *repo_model.Repository) error { - _, err := db.GetEngine(ctx).ID(repo.ID). - SetExpr("num_builds", - builder.Select("count(*)").From("bots_task"). - Where(builder.Eq{"repo_id": repo.ID}), - ). - SetExpr("num_closed_builds", - builder.Select("count(*)").From("bots_task"). - Where(builder.Eq{ - "repo_id": repo.ID, - }.And( - builder.In("status", TaskFailed, TaskCanceled, TaskTimeout, TaskFinished), - ), - ), - ). - Update(repo) - return err -} - -// InsertTask inserts a bot task -func InsertTask(t *Task) error { - if t.UUID == "" { - t.UUID = uuid.New().String() - } - index, err := db.GetNextResourceIndex("build_index", t.RepoID) - if err != nil { - return err - } - t.Index = index - - ctx, commiter, err := db.TxContext() - if err != nil { - return err - } - defer commiter.Close() - - if err := db.Insert(ctx, t); err != nil { - return err - } - - if err := updateRepoBuildsNumbers(ctx, &repo_model.Repository{ID: t.RepoID}); err != nil { - return err - } - - return commiter.Commit() -} - -// UpdateTask updates bot task -func UpdateTask(t *Task, cols ...string) error { - _, err := db.GetEngine(db.DefaultContext).ID(t.ID).Cols(cols...).Update(t) - return err -} - -// ErrTaskNotExist represents an error for bot task not exist -type ErrTaskNotExist struct { - RepoID int64 - Index int64 - UUID string -} - -func (err ErrTaskNotExist) Error() string { - return fmt.Sprintf("Bot task [%s] is not exist", err.UUID) -} - -// GetTaskByUUID gets bot task by uuid -func GetTaskByUUID(taskUUID string) (*Task, error) { - var task Task - has, err := db.GetEngine(db.DefaultContext).Where("uuid=?", taskUUID).Get(&task) - if err != nil { - return nil, err - } else if !has { - return nil, ErrTaskNotExist{ - UUID: taskUUID, - } - } - return &task, nil -} - -// GetCurTaskByID return the task for the bot -func GetCurTaskByID(runnerID int64) (*Task, error) { - var tasks []Task - // FIXME: for test, just return all tasks - err := db.GetEngine(db.DefaultContext).Where("status=?", TaskPending).Find(&tasks) - // err := x.Where("runner_id = ?", botID). - // And("status=?", BotTaskPending). - // Find(&tasks) - if err != nil { - return nil, err - } - if len(tasks) == 0 { - return nil, nil - } - return &tasks[0], err -} - -// GetCurTaskByUUID return the task for the bot -func GetCurTaskByUUID(runnerUUID string) (*Task, error) { - runner, err := GetRunnerByUUID(runnerUUID) - if err != nil { - return nil, err - } - return GetCurTaskByID(runner.ID) -} - -func GetTaskByRepoAndIndex(repoID, index int64) (*Task, error) { - var task Task - has, err := db.GetEngine(db.DefaultContext).Where("repo_id=?", repoID). - And("`index` = ?", index). - Get(&task) - if err != nil { - return nil, err - } else if !has { - return nil, ErrTaskNotExist{ - RepoID: repoID, - Index: index, - } - } - return &task, nil -} - -// AssignTaskToRunner assign a task to a runner -func AssignTaskToRunner(taskID int64, runnerID int64) error { - cnt, err := db.GetEngine(db.DefaultContext). - Where("runner_id=0"). - And("id=?", taskID). - Cols("runner_id"). - Update(&Task{ - RunnerID: runnerID, - }) - if err != nil { - return err - } - if cnt != 1 { - return errors.New("assign faild") - } - return nil -} - -type FindTaskOptions struct { - db.ListOptions - RepoID int64 - IsClosed util.OptionalBool -} - -func (opts FindTaskOptions) toConds() builder.Cond { - cond := builder.NewCond() - if opts.RepoID > 0 { - cond = cond.And(builder.Eq{"repo_id": opts.RepoID}) - } - if opts.IsClosed.IsTrue() { - cond = cond.And(builder.Expr("status IN (?,?,?,?)", TaskCanceled, TaskFailed, TaskTimeout, TaskFinished)) - } else if opts.IsClosed.IsFalse() { - cond = cond.And(builder.Expr("status IN (?,?,?)", TaskPending, TaskAssigned, TaskRunning)) - } - return cond -} - -func FindTasks(opts FindTaskOptions) (TaskList, error) { - sess := db.GetEngine(db.DefaultContext).Where(opts.toConds()) - if opts.ListOptions.PageSize > 0 { - skip, take := opts.GetSkipTake() - sess.Limit(take, skip) - } - var tasks []*Task - return tasks, sess.Find(&tasks) -} - -func CountTasks(opts FindTaskOptions) (int64, error) { - return db.GetEngine(db.DefaultContext).Table("bots_task").Where(opts.toConds()).Count() -} - -type TaskStage struct{} - -type StageStep struct{} - -type BuildIndex db.ResourceIndex diff --git a/models/migrations/v216.go b/models/migrations/v216.go index fd0d700574..a8e1110d3f 100644 --- a/models/migrations/v216.go +++ b/models/migrations/v216.go @@ -5,9 +5,6 @@ package migrations import ( - "code.gitea.io/gitea/models/db" - "code.gitea.io/gitea/modules/timeutil" - "xorm.io/xorm" ) @@ -29,7 +26,7 @@ func addBotTables(x *xorm.Engine) error { Created timeutil.TimeStamp `xorm:"created"` } - type BotsTask struct { + type BotsBuild struct { ID int64 Title string UUID string `xorm:"CHAR(36)"` @@ -55,7 +52,7 @@ func addBotTables(x *xorm.Engine) error { NumClosedBuilds int `xorm:"NOT NULL DEFAULT 0"` } - type BuildIndex db.ResourceIndex + type BotsBuildIndex db.ResourceIndex - return x.Sync2(new(BotsRunner), new(BotsTask), new(Repository), new(BuildIndex)) + return x.Sync2(new(BotsRunner), new(BotsBuild), new(Repository), new(BotsBuildIndex)) } diff --git a/modules/actions/gitea/action.go b/modules/actions/gitea/action.go deleted file mode 100644 index ef4a4a41dd..0000000000 --- a/modules/actions/gitea/action.go +++ /dev/null @@ -1,79 +0,0 @@ -package gitea - -import ( - "fmt" - "io" - "strings" - - "gopkg.in/yaml.v3" -) - -// ActionRunsUsing is the type of runner for the action -type ActionRunsUsing string - -func (a *ActionRunsUsing) UnmarshalYAML(unmarshal func(interface{}) error) error { - var using string - if err := unmarshal(&using); err != nil { - return err - } - - // Force input to lowercase for case insensitive comparison - format := ActionRunsUsing(strings.ToLower(using)) - switch format { - case ActionRunsUsingNode12, ActionRunsUsingDocker: - *a = format - default: - return fmt.Errorf(fmt.Sprintf("The runs.using key in action.yml must be one of: %v, got %s", []string{ - ActionRunsUsingDocker, - ActionRunsUsingNode12, - }, format)) - } - return nil -} - -const ( - // ActionRunsUsingNode12 for running with node12 - ActionRunsUsingNode12 = "node12" - // ActionRunsUsingDocker for running with docker - ActionRunsUsingDocker = "docker" -) - -// Action describes a metadata file for GitHub actions. The metadata filename must be either action.yml or action.yaml. The data in the metadata file defines the inputs, outputs and main entrypoint for your action. -type Action struct { - Name string `yaml:"name"` - Author string `yaml:"author"` - Description string `yaml:"description"` - Inputs map[string]Input `yaml:"inputs"` - Outputs map[string]Output `yaml:"outputs"` - Runs struct { - Using ActionRunsUsing `yaml:"using"` - Env map[string]string `yaml:"env"` - Main string `yaml:"main"` - Image string `yaml:"image"` - Entrypoint []string `yaml:"entrypoint"` - Args []string `yaml:"args"` - } `yaml:"runs"` - Branding struct { - Color string `yaml:"color"` - Icon string `yaml:"icon"` - } `yaml:"branding"` -} - -// Input parameters allow you to specify data that the action expects to use during runtime. GitHub stores input parameters as environment variables. Input ids with uppercase letters are converted to lowercase during runtime. We recommended using lowercase input ids. -type Input struct { - Description string `yaml:"description"` - Required bool `yaml:"required"` - Default string `yaml:"default"` -} - -// Output parameters allow you to declare data that an action sets. Actions that run later in a workflow can use the output data set in previously run actions. For example, if you had an action that performed the addition of two inputs (x + y = z), the action could output the sum (z) for other actions to use as an input. -type Output struct { - Description string `yaml:"description"` -} - -// ReadAction reads an action from a reader -func ReadAction(in io.Reader) (*Action, error) { - a := new(Action) - err := yaml.NewDecoder(in).Decode(a) - return a, err -} diff --git a/modules/actions/gitea/gitea.go b/modules/actions/gitea/gitea.go deleted file mode 100644 index 8f7c0835e5..0000000000 --- a/modules/actions/gitea/gitea.go +++ /dev/null @@ -1,60 +0,0 @@ -package gitea - -import ( - "code.gitea.io/gitea/models/webhook" - "code.gitea.io/gitea/modules/bot/runner" - "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/util" - "code.gitea.io/gitea/modules/json" -) - -func init() { - runner.RegisterRunnerType(new(GiteaRunner)) -} - -type GiteaRunner struct { -} - -func (gw *GiteaRunner) Name() string { - return "gitea" -} - -func (gw *GiteaRunner) Detect(commit *git.Commit, event webhook.HookEventType, ref string) (bool, string, error) { - tree, err := commit.SubTree(".gitea/workflow") - if err != nil { - return false, "", err - } - entries, err := tree.ListEntries() - if err != nil { - return false, "", err - } - - var wfs []*Workflow - for _, entry := range entries { - blob := entry.Blob() - rd, err := blob.DataAsync() - if err != nil { - return false, "", err - } - defer rd.Close() - wf, err := ReadWorkflow(rd) - if err != nil { - log.Error("ReadWorkflow file %s failed: %v", entry.Name(), err) - continue - } - - // FIXME: we have to convert the event type to github known name - if !util.IsStringInSlice(string(event), wf.On()) { - continue - } - - wfs = append(wfs, wf) - } - - wfBs, err := json.Marshal(wfs) - if err != nil { - return false, "", err - } - return true, string(wfBs), nil -} diff --git a/modules/actions/gitea/planner.go b/modules/actions/gitea/planner.go deleted file mode 100644 index 6d80e79d49..0000000000 --- a/modules/actions/gitea/planner.go +++ /dev/null @@ -1,265 +0,0 @@ -package gitea - -import ( - "io" - "io/ioutil" - "math" - "os" - "path/filepath" - "sort" - - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" -) - -// WorkflowPlanner contains methods for creating plans -type WorkflowPlanner interface { - PlanEvent(eventName string) *Plan - PlanJob(jobName string) *Plan - GetEvents() []string -} - -// Plan contains a list of stages to run in series -type Plan struct { - Stages []*Stage -} - -// Stage contains a list of runs to execute in parallel -type Stage struct { - Runs []*Run -} - -// Run represents a job from a workflow that needs to be run -type Run struct { - Workflow *Workflow - JobID string -} - -func (r *Run) String() string { - jobName := r.Job().Name - if jobName == "" { - jobName = r.JobID - } - return jobName -} - -// Job returns the job for this Run -func (r *Run) Job() *Job { - return r.Workflow.GetJob(r.JobID) -} - -// NewWorkflowPlanner will load a specific workflow or all workflows from a directory -func NewWorkflowPlanner(path string) (WorkflowPlanner, error) { - fi, err := os.Stat(path) - if err != nil { - return nil, err - } - - var files []os.FileInfo - var dirname string - - if fi.IsDir() { - log.Debugf("Loading workflows from '%s'", path) - dirname = path - files, err = ioutil.ReadDir(path) - } else { - log.Debugf("Loading workflow '%s'", path) - dirname, err = filepath.Abs(filepath.Dir(path)) - files = []os.FileInfo{fi} - } - if err != nil { - return nil, err - } - - wp := new(workflowPlanner) - for _, file := range files { - ext := filepath.Ext(file.Name()) - if ext == ".yml" || ext == ".yaml" { - f, err := os.Open(filepath.Join(dirname, file.Name())) - if err != nil { - return nil, err - } - - log.Debugf("Reading workflow '%s'", f.Name()) - workflow, err := ReadWorkflow(f) - if err != nil { - f.Close() - if err == io.EOF { - return nil, errors.WithMessagef(err, "unable to read workflow, %s file is empty", file.Name()) - } - return nil, err - } - if workflow.Name == "" { - workflow.Name = file.Name() - } - wp.workflows = append(wp.workflows, workflow) - f.Close() - } - } - - return wp, nil -} - -type workflowPlanner struct { - workflows []*Workflow -} - -// PlanEvent builds a new list of runs to execute in parallel for an event name -func (wp *workflowPlanner) PlanEvent(eventName string) *Plan { - plan := new(Plan) - if len(wp.workflows) == 0 { - log.Debugf("no events found for workflow: %s", eventName) - } - - for _, w := range wp.workflows { - for _, e := range w.When().Events { - if e.Type == eventName { - plan.mergeStages(createStages(w, w.GetJobIDs()...)) - } - } - } - return plan -} - -// PlanJob builds a new run to execute in parallel for a job name -func (wp *workflowPlanner) PlanJob(jobName string) *Plan { - plan := new(Plan) - if len(wp.workflows) == 0 { - log.Debugf("no jobs found for workflow: %s", jobName) - } - - for _, w := range wp.workflows { - plan.mergeStages(createStages(w, jobName)) - } - return plan -} - -// GetEvents gets all the events in the workflows file -func (wp *workflowPlanner) GetEvents() []string { - events := make([]string, 0) - for _, w := range wp.workflows { - found := false - for _, e := range events { - for _, we := range w.When().Events { - if e == we.Type { - found = true - break - } - } - if found { - break - } - } - - if !found { - for _, evt := range w.When().Events { - events = append(events, evt.Type) - } - } - } - - // sort the list based on depth of dependencies - sort.Slice(events, func(i, j int) bool { - return events[i] < events[j] - }) - - return events -} - -// MaxRunNameLen determines the max name length of all jobs -func (p *Plan) MaxRunNameLen() int { - maxRunNameLen := 0 - for _, stage := range p.Stages { - for _, run := range stage.Runs { - runNameLen := len(run.String()) - if runNameLen > maxRunNameLen { - maxRunNameLen = runNameLen - } - } - } - return maxRunNameLen -} - -// GetJobIDs will get all the job names in the stage -func (s *Stage) GetJobIDs() []string { - names := make([]string, 0) - for _, r := range s.Runs { - names = append(names, r.JobID) - } - return names -} - -// Merge stages with existing stages in plan -func (p *Plan) mergeStages(stages []*Stage) { - newStages := make([]*Stage, int(math.Max(float64(len(p.Stages)), float64(len(stages))))) - for i := 0; i < len(newStages); i++ { - newStages[i] = new(Stage) - if i >= len(p.Stages) { - newStages[i].Runs = append(newStages[i].Runs, stages[i].Runs...) - } else if i >= len(stages) { - newStages[i].Runs = append(newStages[i].Runs, p.Stages[i].Runs...) - } else { - newStages[i].Runs = append(newStages[i].Runs, p.Stages[i].Runs...) - newStages[i].Runs = append(newStages[i].Runs, stages[i].Runs...) - } - } - p.Stages = newStages -} - -func createStages(w *Workflow, jobIDs ...string) []*Stage { - // first, build a list of all the necessary jobs to run, and their dependencies - jobDependencies := make(map[string][]string) - for len(jobIDs) > 0 { - newJobIDs := make([]string, 0) - for _, jID := range jobIDs { - // make sure we haven't visited this job yet - if _, ok := jobDependencies[jID]; !ok { - if job := w.GetJob(jID); job != nil { - jobDependencies[jID] = job.Needs() - newJobIDs = append(newJobIDs, job.Needs()...) - } - } - } - jobIDs = newJobIDs - } - - // next, build an execution graph - stages := make([]*Stage, 0) - for len(jobDependencies) > 0 { - stage := new(Stage) - for jID, jDeps := range jobDependencies { - // make sure all deps are in the graph already - if listInStages(jDeps, stages...) { - stage.Runs = append(stage.Runs, &Run{ - Workflow: w, - JobID: jID, - }) - delete(jobDependencies, jID) - } - } - if len(stage.Runs) == 0 { - log.Fatalf("Unable to build dependency graph!") - } - stages = append(stages, stage) - } - - return stages -} - -// return true iff all strings in srcList exist in at least one of the stages -func listInStages(srcList []string, stages ...*Stage) bool { - for _, src := range srcList { - found := false - for _, stage := range stages { - for _, search := range stage.GetJobIDs() { - if src == search { - found = true - } - } - } - if !found { - return false - } - } - return true -} diff --git a/modules/actions/gitea/workflow.go b/modules/actions/gitea/workflow.go deleted file mode 100644 index 2a8a5d04f0..0000000000 --- a/modules/actions/gitea/workflow.go +++ /dev/null @@ -1,377 +0,0 @@ -package gitea - -import ( - "fmt" - "io" - "reflect" - "regexp" - "strings" - - log "github.com/sirupsen/logrus" - "gopkg.in/yaml.v3" -) - -// Workflow is the structure of the files in .github/workflows -type Workflow struct { - Name string `yaml:"name"` - RawWhen yaml.Node `yaml:"when"` - Env map[string]string `yaml:"env"` - Jobs map[string]*Job `yaml:"jobs"` - Defaults Defaults `yaml:"defaults"` -} - -type Event struct { - Type string - Ref string -} - -type When struct { - Events []Event -} - -func (w *When) Match(tp string) bool { - for _, evt := range w.Events { - if strings.EqualFold(tp, evt.Type) { - return true - } - } - return false -} - -// When events for the workflow -func (w *Workflow) When() *When { - switch w.RawWhen.Kind { - case yaml.ScalarNode: - var val string - err := w.RawWhen.Decode(&val) - if err != nil { - log.Fatal(err) - } - return &When{ - Events: []Event{ - { - Type: val, - }, - }, - } - case yaml.SequenceNode: - var vals []string - err := w.RawWhen.Decode(&vals) - if err != nil { - log.Fatal(err) - } - var when When - for _, val := range vals { - when.Events = append(when.Events, Event{ - Type: val, - }) - } - return &when - case yaml.MappingNode: - var val map[string]interface{} - err := w.RawWhen.Decode(&val) - if err != nil { - log.Fatal(err) - } - var keys []string - for k := range val { - keys = append(keys, k) - } - var when When - for _, val := range keys { - when.Events = append(when.Events, Event{ - Type: val, - }) - } - return &when - } - return nil -} - -// Job is the structure of one job in a workflow -type Job struct { - Name string `yaml:"name"` - RawNeeds yaml.Node `yaml:"needs"` - RawRunsOn yaml.Node `yaml:"runs-on"` - Env map[string]string `yaml:"env"` - If string `yaml:"if"` - Steps []*Step `yaml:"steps"` - TimeoutMinutes int64 `yaml:"timeout-minutes"` - Services map[string]*ContainerSpec `yaml:"services"` - Strategy *Strategy `yaml:"strategy"` - RawContainer yaml.Node `yaml:"container"` - Defaults Defaults `yaml:"defaults"` -} - -// Strategy for the job -type Strategy struct { - FailFast bool `yaml:"fail-fast"` - MaxParallel int `yaml:"max-parallel"` - Matrix map[string][]interface{} `yaml:"matrix"` -} - -// Default settings that will apply to all steps in the job or workflow -type Defaults struct { - Run RunDefaults `yaml:"run"` -} - -// Defaults for all run steps in the job or workflow -type RunDefaults struct { - Shell string `yaml:"shell"` - WorkingDirectory string `yaml:"working-directory"` -} - -// Container details for the job -func (j *Job) Container() *ContainerSpec { - var val *ContainerSpec - switch j.RawContainer.Kind { - case yaml.ScalarNode: - val = new(ContainerSpec) - err := j.RawContainer.Decode(&val.Image) - if err != nil { - log.Fatal(err) - } - case yaml.MappingNode: - val = new(ContainerSpec) - err := j.RawContainer.Decode(val) - if err != nil { - log.Fatal(err) - } - } - return val -} - -// Needs list for Job -func (j *Job) Needs() []string { - - switch j.RawNeeds.Kind { - case yaml.ScalarNode: - var val string - err := j.RawNeeds.Decode(&val) - if err != nil { - log.Fatal(err) - } - return []string{val} - case yaml.SequenceNode: - var val []string - err := j.RawNeeds.Decode(&val) - if err != nil { - log.Fatal(err) - } - return val - } - return nil -} - -// RunsOn list for Job -func (j *Job) RunsOn() []string { - - switch j.RawRunsOn.Kind { - case yaml.ScalarNode: - var val string - err := j.RawRunsOn.Decode(&val) - if err != nil { - log.Fatal(err) - } - return []string{val} - case yaml.SequenceNode: - var val []string - err := j.RawRunsOn.Decode(&val) - if err != nil { - log.Fatal(err) - } - return val - } - return nil -} - -// GetMatrixes returns the matrix cross product -func (j *Job) GetMatrixes() []map[string]interface{} { - matrixes := make([]map[string]interface{}, 0) - /*if j.Strategy != nil { - includes := make([]map[string]interface{}, 0) - for _, v := range j.Strategy.Matrix["include"] { - includes = append(includes, v.(map[string]interface{})) - } - delete(j.Strategy.Matrix, "include") - - excludes := make([]map[string]interface{}, 0) - for _, v := range j.Strategy.Matrix["exclude"] { - excludes = append(excludes, v.(map[string]interface{})) - } - delete(j.Strategy.Matrix, "exclude") - - matrixProduct := common.CartesianProduct(j.Strategy.Matrix) - - MATRIX: - for _, matrix := range matrixProduct { - for _, exclude := range excludes { - if commonKeysMatch(matrix, exclude) { - log.Debugf("Skipping matrix '%v' due to exclude '%v'", matrix, exclude) - continue MATRIX - } - } - matrixes = append(matrixes, matrix) - } - for _, include := range includes { - log.Debugf("Adding include '%v'", include) - matrixes = append(matrixes, include) - } - - } else { - matrixes = append(matrixes, make(map[string]interface{})) - }*/ - return matrixes -} - -func commonKeysMatch(a map[string]interface{}, b map[string]interface{}) bool { - for aKey, aVal := range a { - if bVal, ok := b[aKey]; ok && !reflect.DeepEqual(aVal, bVal) { - return false - } - } - return true -} - -// ContainerSpec is the specification of the container to use for the job -type ContainerSpec struct { - Image string `yaml:"image"` - Env map[string]string `yaml:"env"` - Ports []string `yaml:"ports"` - Volumes []string `yaml:"volumes"` - Options string `yaml:"options"` - Entrypoint string - Args string - Name string - Reuse bool -} - -// Step is the structure of one step in a job -type Step struct { - ID string `yaml:"id"` - If string `yaml:"if"` - Name string `yaml:"name"` - Uses string `yaml:"uses"` - Run string `yaml:"run"` - WorkingDirectory string `yaml:"working-directory"` - Shell string `yaml:"shell"` - Env map[string]string `yaml:"env"` - With map[string]string `yaml:"with"` - ContinueOnError bool `yaml:"continue-on-error"` - TimeoutMinutes int64 `yaml:"timeout-minutes"` -} - -// String gets the name of step -func (s *Step) String() string { - if s.Name != "" { - return s.Name - } else if s.Uses != "" { - return s.Uses - } else if s.Run != "" { - return s.Run - } - return s.ID -} - -// GetEnv gets the env for a step -func (s *Step) GetEnv() map[string]string { - rtnEnv := make(map[string]string) - for k, v := range s.Env { - rtnEnv[k] = v - } - for k, v := range s.With { - envKey := regexp.MustCompile("[^A-Z0-9-]").ReplaceAllString(strings.ToUpper(k), "_") - envKey = fmt.Sprintf("INPUT_%s", strings.ToUpper(envKey)) - rtnEnv[envKey] = v - } - return rtnEnv -} - -// ShellCommand returns the command for the shell -func (s *Step) ShellCommand() string { - shellCommand := "" - - switch s.Shell { - case "", "bash": - shellCommand = "bash --noprofile --norc -eo pipefail {0}" - case "pwsh": - shellCommand = "pwsh -command \"& '{0}'\"" - case "python": - shellCommand = "python {0}" - case "sh": - shellCommand = "sh -e -c {0}" - case "cmd": - shellCommand = "%ComSpec% /D /E:ON /V:OFF /S /C \"CALL \"{0}\"\"" - case "powershell": - shellCommand = "powershell -command \"& '{0}'\"" - default: - shellCommand = s.Shell - } - return shellCommand -} - -// StepType describes what type of step we are about to run -type StepType int - -const ( - // StepTypeRun is all steps that have a `run` attribute - StepTypeRun StepType = iota - - //StepTypeUsesDockerURL is all steps that have a `uses` that is of the form `docker://...` - StepTypeUsesDockerURL - - //StepTypeUsesActionLocal is all steps that have a `uses` that is a local action in a subdirectory - StepTypeUsesActionLocal - - //StepTypeUsesActionRemote is all steps that have a `uses` that is a reference to a github repo - StepTypeUsesActionRemote -) - -// Type returns the type of the step -func (s *Step) Type() StepType { - if s.Run != "" { - return StepTypeRun - } else if strings.HasPrefix(s.Uses, "docker://") { - return StepTypeUsesDockerURL - } else if strings.HasPrefix(s.Uses, "./") { - return StepTypeUsesActionLocal - } - return StepTypeUsesActionRemote -} - -// ReadWorkflow returns a list of jobs for a given workflow file reader -func ReadWorkflow(in io.Reader) (*Workflow, error) { - w := new(Workflow) - err := yaml.NewDecoder(in).Decode(w) - return w, err -} - -// GetJob will get a job by name in the workflow -func (w *Workflow) GetJob(jobID string) *Job { - for id, j := range w.Jobs { - if jobID == id { - if j.Name == "" { - j.Name = id - } - return j - } - } - return nil -} - -// GetJobIDs will get all the job names in the workflow -func (w *Workflow) GetJobIDs() []string { - ids := make([]string, 0) - for id := range w.Jobs { - ids = append(ids, id) - } - return ids -} - -func (w *Workflow) On() []string { - var evts []string - for _, job := range w.Jobs { - evts = append(evts, job.RunsOn()...) - } - return evts -} \ No newline at end of file diff --git a/modules/actions/gitea/workflow_test.go b/modules/actions/gitea/workflow_test.go deleted file mode 100644 index 6df4a15e75..0000000000 --- a/modules/actions/gitea/workflow_test.go +++ /dev/null @@ -1,100 +0,0 @@ -package gitea - -import ( - "strings" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestReadWorkflow_StringEvent(t *testing.T) { - yaml := ` -name: local-action-docker-url -on: push - -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: ./actions/docker-url -` - - workflow, err := ReadWorkflow(strings.NewReader(yaml)) - assert.NoError(t, err, "read workflow should succeed") - - assert.Len(t, workflow.On(), 1) - assert.Contains(t, workflow.On(), "push") -} - -func TestReadWorkflow_ListEvent(t *testing.T) { - yaml := ` -name: local-action-docker-url -on: [push, pull_request] - -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: ./actions/docker-url -` - - workflow, err := ReadWorkflow(strings.NewReader(yaml)) - assert.NoError(t, err, "read workflow should succeed") - - assert.Len(t, workflow.On(), 2) - assert.Contains(t, workflow.On(), "push") - assert.Contains(t, workflow.On(), "pull_request") -} - -func TestReadWorkflow_MapEvent(t *testing.T) { - yaml := ` -name: local-action-docker-url -on: - push: - branches: - - master - pull_request: - branches: - - master - -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: ./actions/docker-url -` - - workflow, err := ReadWorkflow(strings.NewReader(yaml)) - assert.NoError(t, err, "read workflow should succeed") - assert.Len(t, workflow.On(), 2) - assert.Contains(t, workflow.On(), "push") - assert.Contains(t, workflow.On(), "pull_request") -} - -func TestReadWorkflow_StringContainer(t *testing.T) { - yaml := ` -name: local-action-docker-url - -jobs: - test: - container: nginx:latest - runs-on: ubuntu-latest - steps: - - uses: ./actions/docker-url - test2: - container: - image: nginx:latest - env: - foo: bar - runs-on: ubuntu-latest - steps: - - uses: ./actions/docker-url -` - - workflow, err := ReadWorkflow(strings.NewReader(yaml)) - assert.NoError(t, err, "read workflow should succeed") - assert.Len(t, workflow.Jobs, 2) - assert.Contains(t, workflow.Jobs["test"].Container().Image, "nginx:latest") - assert.Contains(t, workflow.Jobs["test2"].Container().Image, "nginx:latest") - assert.Contains(t, workflow.Jobs["test2"].Container().Env["foo"], "bar") -} diff --git a/modules/actions/github/github.go b/modules/actions/github/github.go deleted file mode 100644 index b39539fa0d..0000000000 --- a/modules/actions/github/github.go +++ /dev/null @@ -1,165 +0,0 @@ -package github - -import ( - "context" - "encoding/json" - "fmt" - "io" - "os" - "path/filepath" - - bot_model "code.gitea.io/gitea/models/bot" - repo_model "code.gitea.io/gitea/models/repo" - user_model "code.gitea.io/gitea/models/user" - "code.gitea.io/gitea/models/webhook" - "code.gitea.io/gitea/modules/bot/runner" - "code.gitea.io/gitea/modules/git" - - //"code.gitea.io/gitea/modules/log" - //"code.gitea.io/gitea/modules/util" - - "github.com/nektos/act/pkg/model" - act_runner "github.com/nektos/act/pkg/runner" -) - -func init() { - runner.RegisterRunnerType(new(GithubRunner)) -} - -type GithubRunner struct { -} - -func (gw *GithubRunner) Name() string { - return "github" -} - -func (gw *GithubRunner) Detect(commit *git.Commit, event webhook.HookEventType, ref string) (bool, string, error) { - tree, err := commit.SubTree(".github/workflow") - if err != nil { - return false, "", err - } - entries, err := tree.ListEntries() - if err != nil { - return false, "", err - } - - var content = make(map[string]string) - for _, entry := range entries { - blob := entry.Blob() - rd, err := blob.DataAsync() - if err != nil { - return false, "", err - } - - bs, err := io.ReadAll(rd) - rd.Close() - if err != nil { - return false, "", err - } - content[entry.Name()] = string(bs) - } - - res, err := json.Marshal(content) - if err != nil { - return false, "", err - } - return true, string(res), nil -} - -func (gw *GithubRunner) Run(task *bot_model.Task) error { - tmpDir, err := os.MkdirTemp("", fmt.Sprintf("%d", task.ID)) - if err != nil { - return err - } - - var files = make(map[string]string) - if err := json.Unmarshal([]byte(task.Content), &files); err != nil { - return err - } - for name, content := range files { - f, err := os.Create(filepath.Join(tmpDir, name)) - if err != nil { - return err - } - if _, err := f.WriteString(content); err != nil { - f.Close() - return err - } - f.Close() - } - - repo, err := repo_model.GetRepositoryByID(task.RepoID) - if err != nil { - return err - } - - evtFilePath := filepath.Join(tmpDir, "event.json") - evtFile, err := os.Create(evtFilePath) - if err != nil { - return err - } - - if _, err := evtFile.WriteString(task.EventPayload); err != nil { - evtFile.Close() - return err - } - evtFile.Close() - - planner, err := model.NewWorkflowPlanner(tmpDir, false) - if err != nil { - return err - } - plan := planner.PlanEvent(task.Event) - - actor, err := user_model.GetUserByID(task.TriggerUserID) - if err != nil { - return err - } - - // run the plan - config := &act_runner.Config{ - Actor: actor.LoginName, - EventName: task.Event, - EventPath: evtFilePath, - DefaultBranch: repo.DefaultBranch, - /*ForcePull: input.forcePull, - ForceRebuild: input.forceRebuild, - ReuseContainers: input.reuseContainers, - Workdir: input.Workdir(), - BindWorkdir: input.bindWorkdir, - LogOutput: !input.noOutput,*/ - //Env: envs, - Secrets: map[string]string{ - "token": "614e597274a527b6fcf6ddfe45def79430126f08", - }, - //InsecureSecrets: input.insecureSecrets,*/ - Platforms: map[string]string{ - "ubuntu-latest": "node:12-buster-slim", - "ubuntu-20.04": "node:12-buster-slim", - "ubuntu-18.04": "node:12-buster-slim", - }, - /*Privileged: input.privileged, - UsernsMode: input.usernsMode, - ContainerArchitecture: input.containerArchitecture, - ContainerDaemonSocket: input.containerDaemonSocket, - UseGitIgnore: input.useGitIgnore,*/ - GitHubInstance: "gitea.com", - /*ContainerCapAdd: input.containerCapAdd, - ContainerCapDrop: input.containerCapDrop, - AutoRemove: input.autoRemove, - ArtifactServerPath: input.artifactServerPath, - ArtifactServerPort: input.artifactServerPort,*/ - } - r, err := act_runner.New(config) - if err != nil { - return err - } - - //ctx, cancel := context.WithTimeout(context.Background(), ) - - executor := r.NewPlanExecutor(plan).Finally(func(ctx context.Context) error { - //cancel() - return nil - }) - return executor(context.Background()) -} diff --git a/modules/actions/runner/runner.go b/modules/actions/runner/runner.go deleted file mode 100644 index 2a9540ad76..0000000000 --- a/modules/actions/runner/runner.go +++ /dev/null @@ -1,27 +0,0 @@ -package runner - -import ( - bots_model "code.gitea.io/gitea/models/bots" - "code.gitea.io/gitea/models/webhook" - "code.gitea.io/gitea/modules/git" -) - -var runnerTypes = make(map[string]RunnerType) - -type RunnerType interface { - Name() string - Detect(commit *git.Commit, event webhook.HookEventType, ref string) (bool, string, error) - Run(task *bots_model.Task) error -} - -func RegisterRunnerType(runnerType RunnerType) { - runnerTypes[runnerType.Name()] = runnerType -} - -func GetRunnerType(name string) RunnerType { - return runnerTypes[name] -} - -func GetRunnerTypes() map[string]RunnerType { - return runnerTypes -} diff --git a/modules/bots/bots.go b/modules/bots/bots.go new file mode 100644 index 0000000000..a83732e06e --- /dev/null +++ b/modules/bots/bots.go @@ -0,0 +1,61 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package bots + +import ( + "strings" + + "code.gitea.io/gitea/models/webhook" + "code.gitea.io/gitea/modules/git" + + "github.com/nektos/act/pkg/model" +) + +func DetectWorkflows(commit *git.Commit, event webhook.HookEventType) (git.Entries, []map[string]*model.Job, error) { + tree, err := commit.SubTree(".github/workflows") + if _, ok := err.(git.ErrNotExist); ok { + tree, err = commit.SubTree(".gitea/workflows") + } + if _, ok := err.(git.ErrNotExist); ok { + return nil, nil, nil + } + if err != nil { + return nil, nil, err + } + + entries, err := tree.ListEntriesRecursive() + if err != nil { + return nil, nil, err + } + + matchedEntries := make(git.Entries, 0, len(entries)) + jobs := make([]map[string]*model.Job, 0, len(entries)) + + for _, entry := range entries { + if !strings.HasSuffix(entry.Name(), ".yml") && !strings.HasSuffix(entry.Name(), ".yaml") { + continue + } + f, err := entry.Blob().DataAsync() + if err != nil { + return nil, nil, err + } + workflow, err := model.ReadWorkflow(f) + if err != nil { + f.Close() + return nil, nil, err + } + + for _, e := range workflow.On() { + if e == event.Event() { + matchedEntries = append(matchedEntries, entry) + jobs = append(jobs, workflow.Jobs) + break + } + } + f.Close() + } + + return matchedEntries, jobs, nil +} diff --git a/modules/notification/bots/bots.go b/modules/notification/bots/bots.go index a6dec9f449..f013e795f1 100644 --- a/modules/notification/bots/bots.go +++ b/modules/notification/bots/bots.go @@ -16,6 +16,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/models/webhook" + bots_module "code.gitea.io/gitea/modules/bots" "code.gitea.io/gitea/modules/convert" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/graceful" @@ -39,28 +40,6 @@ func NewNotifier() base.Notifier { return &botsNotifier{} } -func detectWorkflows(commit *git.Commit, event webhook.HookEventType, ref string) (bool, error) { - tree, err := commit.SubTree(".github/workflows") - if _, ok := err.(git.ErrNotExist); ok { - tree, err = commit.SubTree(".gitea/workflows") - } - if _, ok := err.(git.ErrNotExist); ok { - return false, nil - } - if err != nil { - return false, err - } - - entries, err := tree.ListEntries() - if err != nil { - return false, err - } - - log.Trace("detected %s has %d entries", commit.ID, len(entries)) - - return len(entries) > 0, nil -} - func notifyIssue(issue *models.Issue, doer *user_model.User, evt webhook.HookEventType, payload string) { err := issue.LoadRepo(db.DefaultContext) if err != nil { @@ -75,8 +54,11 @@ func notifyIssue(issue *models.Issue, doer *user_model.User, evt webhook.HookEve if ref == "" { ref = issue.Repo.DefaultBranch } + notify(issue.Repo, doer, payload, ref, evt) +} - gitRepo, err := git.OpenRepository(context.Background(), issue.Repo.RepoPath()) +func notify(repo *repo_model.Repository, doer *user_model.User, payload, ref string, evt webhook.HookEventType) { + gitRepo, err := git.OpenRepository(context.Background(), repo.RepoPath()) if err != nil { log.Error("issue.LoadRepo: %v", err) return @@ -90,31 +72,41 @@ func notifyIssue(issue *models.Issue, doer *user_model.User, evt webhook.HookEve return } - hasWorkflows, err := detectWorkflows(commit, evt, ref) + matchedEntries, jobs, err := bots_module.DetectWorkflows(commit, evt) if err != nil { log.Error("detectWorkflows: %v", err) return } - if !hasWorkflows { - log.Trace("repo %s with commit %s couldn't find workflows", issue.Repo.RepoPath(), commit.ID) + log.Trace("detected %s has %d entries", commit.ID, len(matchedEntries)) + if len(matchedEntries) == 0 { + log.Trace("repo %s with commit %s couldn't find workflows", repo.RepoPath(), commit.ID) return } - task := bots_model.Task{ - Title: commit.CommitMessage, - RepoID: issue.RepoID, + workflowsStatuses := make(map[string]map[string]bots_model.BuildStatus) + for i, entry := range matchedEntries { + taskStatuses := make(map[string]bots_model.BuildStatus) + for k := range jobs[i] { + taskStatuses[k] = bots_model.BuildPending + } + workflowsStatuses[entry.Name()] = taskStatuses + } + + build := bots_model.Build{ + Title: commit.Message(), + RepoID: repo.ID, TriggerUserID: doer.ID, Event: evt, EventPayload: payload, - Status: bots_model.TaskPending, + Status: bots_model.BuildPending, Ref: ref, CommitSHA: commit.ID.String(), } - if err := bots_model.InsertTask(&task); err != nil { + if err := bots_model.InsertBuild(&build, workflowsStatuses); err != nil { log.Error("InsertBotTask: %v", err) } else { - bots_service.PushToQueue(&task) + bots_service.PushToQueue(&build) } } @@ -190,29 +182,6 @@ func (a *botsNotifier) NotifyPushCommits(pusher *user_model.User, repo *repo_mod return } - gitRepo, err := git.OpenRepository(ctx, repo.RepoPath()) - if err != nil { - log.Error("commits.ToAPIPayloadCommits failed: %v", err) - return - } - defer gitRepo.Close() - - commit, err := gitRepo.GetCommit(commits.HeadCommit.Sha1) - if err != nil { - log.Error("commits.ToAPIPayloadCommits failed: %v", err) - return - } - - hasWorkflows, err := detectWorkflows(commit, webhook.HookEventPush, opts.RefFullName) - if err != nil { - log.Error("detectWorkflows: %v", err) - return - } - if !hasWorkflows { - log.Trace("repo %s with commit %s couldn't find workflows", repo.RepoPath(), commit.ID) - return - } - payload := &api.PushPayload{ Ref: opts.RefFullName, Before: opts.OldCommitID, @@ -231,20 +200,7 @@ func (a *botsNotifier) NotifyPushCommits(pusher *user_model.User, repo *repo_mod return } - task := bots_model.Task{ - Title: commit.Message(), - RepoID: repo.ID, - TriggerUserID: pusher.ID, - Event: webhook.HookEventPush, - EventPayload: string(bs), - Status: bots_model.TaskPending, - } - - if err := bots_model.InsertTask(&task); err != nil { - log.Error("InsertBotTask: %v", err) - } else { - bots_service.PushToQueue(&task) - } + notify(repo, pusher, string(bs), opts.RefFullName, webhook.HookEventPush) } func (a *botsNotifier) NotifyCreateRef(doer *user_model.User, repo *repo_model.Repository, refType, refFullName, refID string) { diff --git a/routers/api/bots/bots.go b/routers/api/bots/bots.go index 32fdc75673..5513caa801 100644 --- a/routers/api/bots/bots.go +++ b/routers/api/bots/bots.go @@ -147,7 +147,7 @@ MESSAGE_BUMP: } // TODO: find new task and send to client - task, err := bots_model.GetCurTaskByUUID(msg.RunnerUUID) + task, err := bots_model.GetCurBuildByUUID(msg.RunnerUUID) if err != nil { log.Error("websocket[%s] get task failed: %v", r.RemoteAddr, err) break diff --git a/routers/web/repo/builds/builds.go b/routers/web/repo/builds/builds.go index f7d8be4a58..7194347b25 100644 --- a/routers/web/repo/builds/builds.go +++ b/routers/web/repo/builds/builds.go @@ -5,6 +5,7 @@ package builds import ( + "fmt" "net/http" bots_model "code.gitea.io/gitea/models/bots" @@ -45,7 +46,7 @@ func List(ctx *context.Context) { page = 1 } - opts := bots_model.FindTaskOptions{ + opts := bots_model.FindBuildOptions{ ListOptions: db.ListOptions{ Page: page, PageSize: convert.ToCorrectPageSize(ctx.FormInt("limit")), @@ -57,24 +58,24 @@ func List(ctx *context.Context) { } else { opts.IsClosed = util.OptionalBoolFalse } - tasks, err := bots_model.FindTasks(opts) + builds, err := bots_model.FindBuilds(opts) if err != nil { ctx.Error(http.StatusInternalServerError, err.Error()) return } - if err := tasks.LoadTriggerUser(); err != nil { + if err := builds.LoadTriggerUser(); err != nil { ctx.Error(http.StatusInternalServerError, err.Error()) return } - total, err := bots_model.CountTasks(opts) + total, err := bots_model.CountBuilds(opts) if err != nil { ctx.Error(http.StatusInternalServerError, err.Error()) return } - ctx.Data["Tasks"] = tasks + ctx.Data["Builds"] = builds pager := context.NewPagination(int(total), opts.PageSize, opts.Page, 5) pager.SetDefaultParams(ctx) @@ -85,15 +86,64 @@ func List(ctx *context.Context) { func ViewBuild(ctx *context.Context) { index := ctx.ParamsInt64("index") - task, err := bots_model.GetTaskByRepoAndIndex(ctx.Repo.Repository.ID, index) + build, err := bots_model.GetBuildByRepoAndIndex(ctx.Repo.Repository.ID, index) if err != nil { ctx.Error(http.StatusInternalServerError, err.Error()) return } - ctx.Data["Title"] = task.Title + " - " + ctx.Tr("repo.builds") + ctx.Data["Title"] = build.Title + " - " + ctx.Tr("repo.builds") ctx.Data["PageIsBuildList"] = true - ctx.Data["Build"] = task + ctx.Data["Build"] = build + statuses, err := bots_model.GetBuildWorkflows(build.ID) + if err != nil { + ctx.Error(http.StatusInternalServerError, err.Error()) + return + } + + ctx.Data["WorkflowsStatuses"] = statuses ctx.HTML(http.StatusOK, tplViewBuild) } + +func GetBuildJobLogs(ctx *context.Context) { + index := ctx.ParamsInt64("index") + build, err := bots_model.GetBuildByRepoAndIndex(ctx.Repo.Repository.ID, index) + if err != nil { + ctx.Error(http.StatusInternalServerError, err.Error()) + return + } + workflows, err := bots_model.GetBuildWorkflows(build.ID) + if err != nil { + ctx.Error(http.StatusInternalServerError, err.Error()) + return + } + var buildJob *bots_model.BuildJob + wf := ctx.Params("workflow") + jobname := ctx.Params("jobname") +LOOP_WORKFLOWS: + for workflow, jobs := range workflows { + if workflow == wf { + for _, job := range jobs { + if jobname == job.Jobname { + buildJob = job + break LOOP_WORKFLOWS + } + } + } + } + if buildJob == nil { + ctx.Error(http.StatusNotFound, fmt.Sprintf("workflow %s job %s not exist", wf, jobname)) + return + } + + // TODO: if buildJob.LogToFile is true, read the logs from the file + + logs, err := bots_model.GetBuildLogs(build.ID, buildJob.ID) + if err != nil { + ctx.Error(http.StatusInternalServerError, err.Error()) + return + } + + ctx.JSON(http.StatusOK, logs) +} diff --git a/routers/web/web.go b/routers/web/web.go index 2dff1d3544..09b2c3f812 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -1175,6 +1175,7 @@ func RegisterRoutes(m *web.Route) { m.Get("", builds.List) m.Group("/{index}", func() { m.Get("", builds.ViewBuild) + m.Get("/{workflow}/job/{jobname}/logs", builds.GetBuildJobLogs) }) }, reqRepoBuildsReader, builds.MustEnableBuilds) diff --git a/services/bots/bots.go b/services/bots/bots.go index 3970110565..98b9476535 100644 --- a/services/bots/bots.go +++ b/services/bots/bots.go @@ -14,16 +14,16 @@ import ( //"code.gitea.io/gitea/modules/json" ) -// taskQueue is a global queue of tasks -var taskQueue queue.Queue +// buildQueue is a global queue of bot build +var buildQueue queue.Queue // PushToQueue -func PushToQueue(task *bots_model.Task) { - taskQueue.Push(task) +func PushToQueue(task *bots_model.Build) { + buildQueue.Push(task) } // Dispatch assign a task to a runner -func Dispatch(task *bots_model.Task) (*bots_model.Runner, error) { +func Dispatch(task *bots_model.Build) (*bots_model.Runner, error) { runner, err := bots_model.GetUsableRunner(bots_model.GetRunnerOptions{ RepoID: task.RepoID, }) @@ -31,17 +31,17 @@ func Dispatch(task *bots_model.Task) (*bots_model.Runner, error) { return nil, err } - return runner, bots_model.AssignTaskToRunner(task.ID, runner.ID) + return runner, bots_model.AssignBuildToRunner(task.ID, runner.ID) } // Init will start the service to get all unfinished tasks and run them func Init() error { - taskQueue = queue.CreateQueue("actions_task", handle, &bots_model.Task{}) - if taskQueue == nil { + buildQueue = queue.CreateQueue("actions_task", handle, &bots_model.Build{}) + if buildQueue == nil { return fmt.Errorf("Unable to create Task Queue") } - go graceful.GetManager().RunWithShutdownFns(taskQueue.Run) + go graceful.GetManager().RunWithShutdownFns(buildQueue.Run) return nil } @@ -49,13 +49,13 @@ func Init() error { func handle(data ...queue.Data) []queue.Data { var unhandled []queue.Data for _, datum := range data { - task := datum.(*bots_model.Task) - runner, err := Dispatch(task) + build := datum.(*bots_model.Build) + runner, err := Dispatch(build) if err != nil { - log.Error("Run task failed: %v", err) - unhandled = append(unhandled, task) + log.Error("Run build failed: %v", err) + unhandled = append(unhandled, build) } else { - log.Trace("task %v assigned to %s", task.UUID, runner.UUID) + log.Trace("build %v assigned to %s", build.UUID, runner.UUID) } } return unhandled diff --git a/templates/repo/builds/build_list.tmpl b/templates/repo/builds/build_list.tmpl index 55d177cbcb..84d3a785e6 100644 --- a/templates/repo/builds/build_list.tmpl +++ b/templates/repo/builds/build_list.tmpl @@ -1,5 +1,5 @@
- {{range .Tasks}} + {{range .Builds}}
  • {{if $.CanWriteIssuesOrPulls}} @@ -8,20 +8,7 @@
    {{end}} -
    - {{if .IsPending}} - - {{end}} - {{if .IsRunning}} - - {{end}} - {{if .IsSuccess}} - - {{end}} - {{if .IsFailed}} - - {{end}} -
    + {{template "repo/builds/status" .Status}}
    • diff --git a/templates/repo/builds/status.tmpl b/templates/repo/builds/status.tmpl new file mode 100644 index 0000000000..a48a895dd6 --- /dev/null +++ b/templates/repo/builds/status.tmpl @@ -0,0 +1,14 @@ +
    + {{if .IsPending}} + + {{end}} + {{if .IsRunning}} + + {{end}} + {{if .IsSuccess}} + + {{end}} + {{if .IsFailed}} + + {{end}} +
    diff --git a/templates/repo/builds/view.tmpl b/templates/repo/builds/view.tmpl index 8b01e586dc..512b03e295 100644 --- a/templates/repo/builds/view.tmpl +++ b/templates/repo/builds/view.tmpl @@ -10,6 +10,7 @@
    + {{template "repo/builds/view_left" .}} {{template "repo/builds/view_content" .}} diff --git a/templates/repo/builds/view_content.tmpl b/templates/repo/builds/view_content.tmpl index 7dc3ca9ac8..c7015dbcff 100644 --- a/templates/repo/builds/view_content.tmpl +++ b/templates/repo/builds/view_content.tmpl @@ -1,8 +1,19 @@ -
    - {{if .Flash}} -
    - {{template "base/alert" .}} -
    - {{end}} - +
    +
    +
    +
    +
    +

    Console Logs

    +
    +
    + 
    
+						
    1Initialized empty Git repository in /drone/src/.git/
+0s
    2+ git fetch origin +refs/heads/main:
+0s
    3From https://github.com/go-gitea/gitea
+20s
    4 * branch                main       -> FETCH_HEAD
+20s
    5 * [new branch]          main       -> origin/main
+20s
    6+ git checkout c8ec2261a99590f15699e9147a28e4b61c1c2ea5 -b main
+20s
    7Switched to a new branch 'main'
+20s
    Exit Code 0
    diff --git a/templates/repo/builds/view_left.tmpl b/templates/repo/builds/view_left.tmpl new file mode 100644 index 0000000000..4dfee9ec37 --- /dev/null +++ b/templates/repo/builds/view_left.tmpl @@ -0,0 +1,18 @@ +
    +
    +

    Menu

    +
    + {{range $file, $jobs := .WorkflowsStatuses}} +
    + {{ $file }} +
    + {{range $jobname, $job := $jobs}} + {{template "repo/builds/status" $job.Status}} + {{ $jobname }} + {{end}} +
    +
    + {{end}} +
    +
    +