From 3a1503138b7360ff143d7e48b09effc5ef45f3ab Mon Sep 17 00:00:00 2001 From: "Bo-Yi.Wu" Date: Sun, 2 Oct 2022 12:33:17 +0800 Subject: [PATCH] chore(runner): refactor register flow Signed-off-by: Bo-Yi.Wu --- client/client.go | 8 ++-- cmd/config.go | 74 ---------------------------------- cmd/daemon.go | 14 +++++-- cmd/root.go | 6 ++- config/config.go | 101 +++++++++++++++++++++++++++++++++++++++++++++++ go.mod | 3 +- go.sum | 8 ++-- poller/poller.go | 48 ++++++++++++++++++---- 8 files changed, 165 insertions(+), 97 deletions(-) delete mode 100644 cmd/config.go create mode 100644 config/config.go diff --git a/client/client.go b/client/client.go index 15aec62..1b3dcfb 100644 --- a/client/client.go +++ b/client/client.go @@ -6,11 +6,9 @@ import ( ) type Filter struct { - Kind string `json:"kind"` - Type string `json:"type"` - OS string `json:"os"` - Arch string `json:"arch"` - Capacity int `json:"capacity"` + OS string `json:"os"` + Arch string `json:"arch"` + Labels []string `json:"labels"` } // A Client manages communication with the runner. diff --git a/cmd/config.go b/cmd/config.go deleted file mode 100644 index 4de3e48..0000000 --- a/cmd/config.go +++ /dev/null @@ -1,74 +0,0 @@ -package cmd - -import ( - "fmt" - "os" - - "github.com/joho/godotenv" - "github.com/kelseyhightower/envconfig" -) - -type ( - // Config provides the system configuration. - Config struct { - Debug bool `envconfig:"GITEA_DEBUG"` - Trace bool `envconfig:"GITEA_TRACE"` - - Client struct { - Address string `ignored:"true"` - Proto string `envconfig:"GITEA_RPC_PROTO" default:"http"` - Host string `envconfig:"GITEA_RPC_HOST" required:"true"` - Secret string `envconfig:"GITEA_RPC_SECRET" required:"true"` - SkipVerify bool `envconfig:"GITEA_RPC_SKIP_VERIFY"` - GRPC bool `envconfig:"GITEA_RPC_GRPC" default:"true"` - GRPCWeb bool `envconfig:"GITEA_RPC_GRPC_WEB"` - } - - Runner struct { - Name string `envconfig:"GITEA_RUNNER_NAME"` - Capacity int `envconfig:"GITEA_RUNNER_CAPACITY" default:"2"` - Procs int64 `envconfig:"GITEA_RUNNER_MAX_PROCS"` - Environ map[string]string `envconfig:"GITEA_RUNNER_ENVIRON"` - EnvFile string `envconfig:"GITEA_RUNNER_ENV_FILE"` - } - - Platform struct { - OS string `envconfig:"GITEA_PLATFORM_OS" default:"linux"` - Arch string `envconfig:"GITEA_PLATFORM_ARCH" default:"amd64"` - Kernel string `envconfig:"GITEA_PLATFORM_KERNEL"` - Variant string `envconfig:"GITEA_PLATFORM_VARIANT"` - } - } -) - -// fromEnviron returns the settings from the environment. -func fromEnviron() (Config, error) { - cfg := Config{} - err := envconfig.Process("", &cfg) - - // runner config - if cfg.Runner.Environ == nil { - cfg.Runner.Environ = map[string]string{} - } - if cfg.Runner.Name == "" { - cfg.Runner.Name, _ = os.Hostname() - } - - cfg.Client.Address = fmt.Sprintf( - "%s://%s", - cfg.Client.Proto, - cfg.Client.Host, - ) - - if file := cfg.Runner.EnvFile; file != "" { - envs, err := godotenv.Read(file) - if err != nil { - return cfg, err - } - for k, v := range envs { - cfg.Runner.Environ[k] = v - } - } - - return cfg, err -} diff --git a/cmd/daemon.go b/cmd/daemon.go index bd64041..cf00918 100644 --- a/cmd/daemon.go +++ b/cmd/daemon.go @@ -5,6 +5,7 @@ import ( "time" "gitea.com/gitea/act_runner/client" + "gitea.com/gitea/act_runner/config" "gitea.com/gitea/act_runner/engine" "gitea.com/gitea/act_runner/poller" "gitea.com/gitea/act_runner/runtime" @@ -22,7 +23,7 @@ func runDaemon(ctx context.Context, task *runtime.Task) func(cmd *cobra.Command, log.Infoln("Starting runner daemon") _ = godotenv.Load(task.Input.EnvFile) - cfg, err := fromEnviron() + cfg, err := config.FromEnviron() if err != nil { log.WithError(err). Fatalln("invalid configuration") @@ -79,9 +80,9 @@ func runDaemon(ctx context.Context, task *runtime.Task) func(cmd *cobra.Command, cli, runner.Run, &client.Filter{ - OS: cfg.Platform.OS, - Arch: cfg.Platform.Arch, - Capacity: cfg.Runner.Capacity, + OS: cfg.Platform.OS, + Arch: cfg.Platform.Arch, + Labels: cfg.Runner.Labels, }, ) @@ -92,6 +93,11 @@ func runDaemon(ctx context.Context, task *runtime.Task) func(cmd *cobra.Command, WithField("arch", cfg.Platform.Arch). Infoln("polling the remote server") + // register new runner + if err := poller.Register(ctx, cfg.Runner); err != nil { + return err + } + return poller.Poll(ctx, cfg.Runner.Capacity) }) diff --git a/cmd/root.go b/cmd/root.go index 8f567a2..09d0312 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -5,8 +5,10 @@ import ( "os" "strconv" + "gitea.com/gitea/act_runner/config" "gitea.com/gitea/act_runner/engine" "gitea.com/gitea/act_runner/runtime" + "github.com/mattn/go-isatty" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -15,7 +17,7 @@ import ( const version = "0.1" // initLogging setup the global logrus logger. -func initLogging(cfg Config) { +func initLogging(cfg config.Config) { isTerm := isatty.IsTerminal(os.Stdout.Fd()) log.SetFormatter(&log.TextFormatter{ DisableColors: !isTerm, @@ -76,7 +78,7 @@ func runRoot(ctx context.Context, task *runtime.Task) func(cmd *cobra.Command, a } task.BuildID, _ = strconv.ParseInt(jobID, 10, 64) - task.Run(ctx, nil) + _ = task.Run(ctx, nil) return nil } } diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..96881ce --- /dev/null +++ b/config/config.go @@ -0,0 +1,101 @@ +package config + +import ( + "fmt" + "net/url" + "os" + "runtime" + + "github.com/joho/godotenv" + "github.com/kelseyhightower/envconfig" +) + +type ( + // Config provides the system configuration. + Config struct { + Debug bool `envconfig:"GITEA_DEBUG"` + Trace bool `envconfig:"GITEA_TRACE"` + Client Client + Runner Runner + Platform Platform + } + + Client struct { + Address string `ignored:"true"` + Proto string `envconfig:"GITEA_RPC_PROTO" default:"http"` + Host string `envconfig:"GITEA_RPC_HOST"` + Secret string `envconfig:"GITEA_RPC_SECRET"` + SkipVerify bool `envconfig:"GITEA_RPC_SKIP_VERIFY"` + GRPC bool `envconfig:"GITEA_RPC_GRPC" default:"true"` + GRPCWeb bool `envconfig:"GITEA_RPC_GRPC_WEB"` + } + + Runner struct { + Name string `envconfig:"GITEA_RUNNER_NAME"` + URL string `envconfig:"GITEA_URL" required:"true"` + Token string `envconfig:"GITEA_TOKEN" required:"true"` + Capacity int `envconfig:"GITEA_RUNNER_CAPACITY" default:"1"` + Environ map[string]string `envconfig:"GITEA_RUNNER_ENVIRON"` + EnvFile string `envconfig:"GITEA_RUNNER_ENV_FILE"` + Labels []string `envconfig:"GITEA_RUNNER_LABELS"` + } + + Platform struct { + OS string `envconfig:"GITEA_PLATFORM_OS"` + Arch string `envconfig:"GITEA_PLATFORM_ARCH"` + Kernel string `envconfig:"GITEA_PLATFORM_KERNEL"` + Variant string `envconfig:"GITEA_PLATFORM_VARIANT"` + } +) + +// FromEnviron returns the settings from the environment. +func FromEnviron() (Config, error) { + cfg := Config{} + if err := envconfig.Process("", &cfg); err != nil { + return cfg, err + } + + // check runner remote url + u, err := url.Parse(cfg.Runner.URL) + if err != nil { + return cfg, err + } + + cfg.Client.Proto = u.Scheme + cfg.Client.Host = u.Host + cfg.Client.Secret = cfg.Runner.Token + + // runner config + if cfg.Runner.Environ == nil { + cfg.Runner.Environ = map[string]string{} + } + if cfg.Runner.Name == "" { + cfg.Runner.Name, _ = os.Hostname() + } + + // platform config + if cfg.Platform.OS == "" { + cfg.Platform.OS = runtime.GOOS + } + if cfg.Platform.Arch == "" { + cfg.Platform.Arch = runtime.GOARCH + } + + cfg.Client.Address = fmt.Sprintf( + "%s://%s", + cfg.Client.Proto, + cfg.Client.Host, + ) + + if file := cfg.Runner.EnvFile; file != "" { + envs, err := godotenv.Read(file) + if err != nil { + return cfg, err + } + for k, v := range envs { + cfg.Runner.Environ[k] = v + } + } + + return cfg, err +} diff --git a/go.mod b/go.mod index ffdf51e..2847917 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,8 @@ module gitea.com/gitea/act_runner go 1.18 require ( - gitea.com/gitea/proto-go v0.0.0-20220929140437-812ae50fdce4 + gitea.com/gitea/proto-go v0.0.0-20221002020351-750a3b99a850 + github.com/appleboy/com v0.1.6 github.com/avast/retry-go/v4 v4.1.0 github.com/bufbuild/connect-go v0.5.0 github.com/docker/docker v20.10.17+incompatible diff --git a/go.sum b/go.sum index 55309d5..cf4c50a 100644 --- a/go.sum +++ b/go.sum @@ -25,10 +25,8 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= gitea.com/gitea/act v0.0.0-20220922135643-52a5bba9e7fa h1:HHqlvfIvqFlny3sgJgAM1BYeLNr1uM4yXtvF7aAoYK8= gitea.com/gitea/act v0.0.0-20220922135643-52a5bba9e7fa/go.mod h1:9W/Nz16tjfnWp7O5DUo3EjZBnZFBI/5rlWstX4o7+hU= -gitea.com/gitea/proto-go v0.0.0-20220925101213-1ac8a05257e1 h1:JGApntYc07jawNxrxv1WhU6IHX0i73nqhloZlaUR5Nc= -gitea.com/gitea/proto-go v0.0.0-20220925101213-1ac8a05257e1/go.mod h1:hD8YwSHusjwjEEgubW6XFvnZuNhMZTHz6lwjfltEt/Y= -gitea.com/gitea/proto-go v0.0.0-20220929140437-812ae50fdce4 h1:HW38qGi3yd/7eUk8ihkz+opF6YGb1uLn8d1ZCUaxNg8= -gitea.com/gitea/proto-go v0.0.0-20220929140437-812ae50fdce4/go.mod h1:hD8YwSHusjwjEEgubW6XFvnZuNhMZTHz6lwjfltEt/Y= +gitea.com/gitea/proto-go v0.0.0-20221002020351-750a3b99a850 h1:BDnr9A52zCPb5BH64yTm8cIfhhjTvql0u6QvWjJViGo= +gitea.com/gitea/proto-go v0.0.0-20221002020351-750a3b99a850/go.mod h1:hD8YwSHusjwjEEgubW6XFvnZuNhMZTHz6lwjfltEt/Y= github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= @@ -89,6 +87,8 @@ github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:C github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/appleboy/com v0.1.6 h1:vP9ryTIbSFaXSrZcFTU7RRcgPbrpGJ0Oy5wpgEkQ2m8= +github.com/appleboy/com v0.1.6/go.mod h1:jnufjIC3opMlReyPPPye+8JqNvUzLm25o7h6SOy8nv0= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= diff --git a/poller/poller.go b/poller/poller.go index 1a4770d..8e5e480 100644 --- a/poller/poller.go +++ b/poller/poller.go @@ -2,12 +2,16 @@ package poller import ( "context" + "encoding/json" "errors" + "os" "time" "gitea.com/gitea/act_runner/client" + "gitea.com/gitea/act_runner/config" runnerv1 "gitea.com/gitea/proto-go/runner/v1" + "github.com/appleboy/com/file" "github.com/bufbuild/connect-go" log "github.com/sirupsen/logrus" ) @@ -31,18 +35,50 @@ type Poller struct { routineGroup *routineGroup } -func (p *Poller) Poll(ctx context.Context, n int) error { +type runner struct { + id int64 + uuid string + name string + token string +} + +func (p *Poller) Register(ctx context.Context, cfg config.Runner) error { + // check .runner config exist + if file.IsFile(".runner") { + return nil + } + // register new runner. - _, err := p.Client.Register(ctx, connect.NewRequest(&runnerv1.RegisterRequest{ - Os: p.Filter.OS, - Arch: p.Filter.Arch, - Capacity: int64(p.Filter.Capacity), + resp, err := p.Client.Register(ctx, connect.NewRequest(&runnerv1.RegisterRequest{ + Os: p.Filter.OS, + Arch: p.Filter.Arch, + Labels: p.Filter.Labels, + Name: cfg.Name, + Token: cfg.Token, })) if err != nil { log.WithError(err).Error("poller: cannot register new runner") return err } + data := &runner{ + id: resp.Msg.Runner.Id, + uuid: resp.Msg.Runner.Uuid, + name: resp.Msg.Runner.Name, + token: resp.Msg.Runner.Token, + } + + file, err := json.MarshalIndent(data, "", " ") + if err != nil { + log.WithError(err).Error("poller: cannot marshal the json input") + return err + } + + // store runner config in .runner file + return os.WriteFile(".runner", file, 0o644) +} + +func (p *Poller) Poll(ctx context.Context, n int) error { for i := 0; i < n; i++ { func(i int) { p.routineGroup.Run(func() { @@ -79,10 +115,8 @@ func (p *Poller) poll(ctx context.Context, thread int) error { // request a new build stage for execution from the central // build server. resp, err := p.Client.FetchTask(ctx, connect.NewRequest(&runnerv1.FetchTaskRequest{ - Kind: p.Filter.Kind, Os: p.Filter.OS, Arch: p.Filter.Arch, - Type: p.Filter.Type, })) if err == context.Canceled || err == context.DeadlineExceeded { l.WithError(err).Trace("poller: no stage returned")