From 6b1266b6b32c3a1c4a02c7e913a2c5fdfe0d2b4a Mon Sep 17 00:00:00 2001 From: zeripath Date: Tue, 13 Oct 2020 04:58:34 +0100 Subject: [PATCH] Provide self-registering storage system (#12978) * Provide self-registering storage system Signed-off-by: Andrew Thornton * More simplification Signed-off-by: Andrew Thornton * Remove old strings from setting Signed-off-by: Andrew Thornton * oops attachments not attachment Signed-off-by: Andrew Thornton Co-authored-by: Lunny Xiao --- cmd/migrate_storage.go | 37 +++++++----- models/unit_tests.go | 2 - modules/setting/attachment.go | 39 +----------- modules/setting/lfs.go | 37 ++---------- modules/setting/setting.go | 1 - modules/setting/storage.go | 108 +++++++++++++++++++--------------- modules/storage/helper.go | 65 ++++++++++++++++++++ modules/storage/local.go | 32 +++++++++- modules/storage/minio.go | 47 +++++++++++---- modules/storage/storage.go | 70 +++++++++++++--------- 10 files changed, 264 insertions(+), 174 deletions(-) create mode 100644 modules/storage/helper.go diff --git a/cmd/migrate_storage.go b/cmd/migrate_storage.go index aed81ddb0180..5f19556d87b2 100644 --- a/cmd/migrate_storage.go +++ b/cmd/migrate_storage.go @@ -32,8 +32,8 @@ var CmdMigrateStorage = cli.Command{ }, cli.StringFlag{ Name: "storage, s", - Value: setting.LocalStorageType, - Usage: "New storage type, local or minio", + Value: "", + Usage: "New storage type: local (default) or minio", }, cli.StringFlag{ Name: "path, p", @@ -107,6 +107,8 @@ func runMigrateStorage(ctx *cli.Context) error { return err } + goCtx := context.Background() + if err := storage.Init(); err != nil { return err } @@ -114,24 +116,31 @@ func runMigrateStorage(ctx *cli.Context) error { var dstStorage storage.ObjectStorage var err error switch strings.ToLower(ctx.String("storage")) { - case setting.LocalStorageType: + case "": + fallthrough + case string(storage.LocalStorageType): p := ctx.String("path") if p == "" { log.Fatal("Path must be given when storage is loal") return nil } - dstStorage, err = storage.NewLocalStorage(p) - case setting.MinioStorageType: + dstStorage, err = storage.NewLocalStorage( + goCtx, + storage.LocalStorageConfig{ + Path: p, + }) + case string(storage.MinioStorageType): dstStorage, err = storage.NewMinioStorage( - context.Background(), - ctx.String("minio-endpoint"), - ctx.String("minio-access-key-id"), - ctx.String("minio-secret-access-key"), - ctx.String("minio-bucket"), - ctx.String("minio-location"), - ctx.String("minio-base-path"), - ctx.Bool("minio-use-ssl"), - ) + goCtx, + storage.MinioStorageConfig{ + Endpoint: ctx.String("minio-endpoint"), + AccessKeyID: ctx.String("minio-access-key-id"), + SecretAccessKey: ctx.String("minio-secret-access-key"), + Bucket: ctx.String("minio-bucket"), + Location: ctx.String("minio-location"), + BasePath: ctx.String("minio-base-path"), + UseSSL: ctx.Bool("minio-use-ssl"), + }) default: return fmt.Errorf("Unsupported attachments storage type: %s", ctx.String("storage")) } diff --git a/models/unit_tests.go b/models/unit_tests.go index c4f9091a4b89..031744629c72 100644 --- a/models/unit_tests.go +++ b/models/unit_tests.go @@ -67,10 +67,8 @@ func MainTest(m *testing.M, pathToGiteaRoot string) { if err != nil { fatalTestError("url.Parse: %v\n", err) } - setting.Attachment.Storage.Type = setting.LocalStorageType setting.Attachment.Storage.Path = filepath.Join(setting.AppDataPath, "attachments") - setting.LFS.Storage.Type = setting.LocalStorageType setting.LFS.Storage.Path = filepath.Join(setting.AppDataPath, "lfs") if err = storage.Init(); err != nil { fatalTestError("storage.Init: %v\n", err) diff --git a/modules/setting/attachment.go b/modules/setting/attachment.go index a51b23913a21..98c4be94e9cd 100644 --- a/modules/setting/attachment.go +++ b/modules/setting/attachment.go @@ -4,12 +4,6 @@ package setting -import ( - "path/filepath" - - "code.gitea.io/gitea/modules/log" -) - var ( // Attachment settings Attachment = struct { @@ -20,7 +14,6 @@ var ( Enabled bool }{ Storage: Storage{ - Type: LocalStorageType, ServeDirect: false, }, AllowedTypes: "image/jpeg,image/png,application/zip,application/gzip", @@ -32,37 +25,9 @@ var ( func newAttachmentService() { sec := Cfg.Section("attachment") - Attachment.Storage.Type = sec.Key("STORAGE_TYPE").MustString("") - if Attachment.Storage.Type == "" { - Attachment.Storage.Type = "default" - } + storageType := sec.Key("STORAGE_TYPE").MustString("") - if Attachment.Storage.Type != LocalStorageType && Attachment.Storage.Type != MinioStorageType { - storage, ok := storages[Attachment.Storage.Type] - if !ok { - log.Fatal("Failed to get attachment storage type: %s", Attachment.Storage.Type) - } - Attachment.Storage = storage - } - - // Override - Attachment.ServeDirect = sec.Key("SERVE_DIRECT").MustBool(Attachment.ServeDirect) - - switch Attachment.Storage.Type { - case LocalStorageType: - Attachment.Path = sec.Key("PATH").MustString(filepath.Join(AppDataPath, "attachments")) - if !filepath.IsAbs(Attachment.Path) { - Attachment.Path = filepath.Join(AppWorkPath, Attachment.Path) - } - case MinioStorageType: - Attachment.Minio.Endpoint = sec.Key("MINIO_ENDPOINT").MustString(Attachment.Minio.Endpoint) - Attachment.Minio.AccessKeyID = sec.Key("MINIO_ACCESS_KEY_ID").MustString(Attachment.Minio.AccessKeyID) - Attachment.Minio.SecretAccessKey = sec.Key("MINIO_SECRET_ACCESS_KEY").MustString(Attachment.Minio.SecretAccessKey) - Attachment.Minio.Bucket = sec.Key("MINIO_BUCKET").MustString(Attachment.Minio.Bucket) - Attachment.Minio.Location = sec.Key("MINIO_LOCATION").MustString(Attachment.Minio.Location) - Attachment.Minio.UseSSL = sec.Key("MINIO_USE_SSL").MustBool(Attachment.Minio.UseSSL) - Attachment.Minio.BasePath = sec.Key("MINIO_BASE_PATH").MustString("attachments/") - } + Attachment.Storage = getStorage("attachments", storageType, sec) Attachment.AllowedTypes = sec.Key("ALLOWED_TYPES").MustString(".docx,.gif,.gz,.jpeg,.jpg,.log,.pdf,.png,.pptx,.txt,.xlsx,.zip") Attachment.MaxSize = sec.Key("MAX_SIZE").MustInt64(4) diff --git a/modules/setting/lfs.go b/modules/setting/lfs.go index da34d3a5ff22..8ba8b008516a 100644 --- a/modules/setting/lfs.go +++ b/modules/setting/lfs.go @@ -37,40 +37,15 @@ func newLFSService() { } lfsSec := Cfg.Section("lfs") - LFS.Storage.Type = lfsSec.Key("STORAGE_TYPE").MustString("") - if LFS.Storage.Type == "" { - LFS.Storage.Type = "default" - } + storageType := lfsSec.Key("STORAGE_TYPE").MustString("") - if LFS.Storage.Type != LocalStorageType && LFS.Storage.Type != MinioStorageType { - storage, ok := storages[LFS.Storage.Type] - if !ok { - log.Fatal("Failed to get lfs storage type: %s", LFS.Storage.Type) - } - LFS.Storage = storage - } + // Specifically default PATH to LFS_CONTENT_PATH + lfsSec.Key("PATH").MustString( + sec.Key("LFS_CONTENT_PATH").String()) - // Override - LFS.ServeDirect = lfsSec.Key("SERVE_DIRECT").MustBool(LFS.ServeDirect) - switch LFS.Storage.Type { - case LocalStorageType: - // keep compatible - LFS.Path = sec.Key("LFS_CONTENT_PATH").MustString(filepath.Join(AppDataPath, "lfs")) - LFS.Path = lfsSec.Key("PATH").MustString(LFS.Path) - if !filepath.IsAbs(LFS.Path) { - LFS.Path = filepath.Join(AppWorkPath, LFS.Path) - } - - case MinioStorageType: - LFS.Minio.Endpoint = lfsSec.Key("MINIO_ENDPOINT").MustString(LFS.Minio.Endpoint) - LFS.Minio.AccessKeyID = lfsSec.Key("MINIO_ACCESS_KEY_ID").MustString(LFS.Minio.AccessKeyID) - LFS.Minio.SecretAccessKey = lfsSec.Key("MINIO_SECRET_ACCESS_KEY").MustString(LFS.Minio.SecretAccessKey) - LFS.Minio.Bucket = lfsSec.Key("MINIO_BUCKET").MustString(LFS.Minio.Bucket) - LFS.Minio.Location = lfsSec.Key("MINIO_LOCATION").MustString(LFS.Minio.Location) - LFS.Minio.UseSSL = lfsSec.Key("MINIO_USE_SSL").MustBool(LFS.Minio.UseSSL) - LFS.Minio.BasePath = lfsSec.Key("MINIO_BASE_PATH").MustString("lfs/") - } + LFS.Storage = getStorage("lfs", storageType, lfsSec) + // Rest of LFS service settings if LFS.LocksPagingNum == 0 { LFS.LocksPagingNum = 50 } diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 69f0080f640a..4d8e02b9b08a 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -804,7 +804,6 @@ func NewContext() { } } - newStorageService() newAttachmentService() newLFSService() diff --git a/modules/setting/storage.go b/modules/setting/storage.go index c678a08f5b9c..ab0598ccf8d4 100644 --- a/modules/setting/storage.go +++ b/modules/setting/storage.go @@ -5,65 +5,77 @@ package setting import ( - "strings" + "path/filepath" + "reflect" - "code.gitea.io/gitea/modules/log" ini "gopkg.in/ini.v1" ) -// enumerate all storage types -const ( - LocalStorageType = "local" - MinioStorageType = "minio" -) - // Storage represents configuration of storages type Storage struct { Type string Path string + Section *ini.Section ServeDirect bool - Minio struct { - Endpoint string - AccessKeyID string - SecretAccessKey string - UseSSL bool - Bucket string - Location string - BasePath string - } } -var ( - storages = make(map[string]Storage) -) - -func getStorage(sec *ini.Section) Storage { - var storage Storage - storage.Type = sec.Key("STORAGE_TYPE").MustString(LocalStorageType) - storage.ServeDirect = sec.Key("SERVE_DIRECT").MustBool(false) - switch storage.Type { - case LocalStorageType: - case MinioStorageType: - storage.Minio.Endpoint = sec.Key("MINIO_ENDPOINT").MustString("localhost:9000") - storage.Minio.AccessKeyID = sec.Key("MINIO_ACCESS_KEY_ID").MustString("") - storage.Minio.SecretAccessKey = sec.Key("MINIO_SECRET_ACCESS_KEY").MustString("") - storage.Minio.Bucket = sec.Key("MINIO_BUCKET").MustString("gitea") - storage.Minio.Location = sec.Key("MINIO_LOCATION").MustString("us-east-1") - storage.Minio.UseSSL = sec.Key("MINIO_USE_SSL").MustBool(false) +// MapTo implements the Mappable interface +func (s *Storage) MapTo(v interface{}) error { + pathValue := reflect.ValueOf(v).FieldByName("Path") + if pathValue.IsValid() && pathValue.Kind() == reflect.String { + pathValue.SetString(s.Path) } + if s.Section != nil { + return s.Section.MapTo(v) + } + return nil +} + +func getStorage(name, typ string, overrides ...*ini.Section) Storage { + sectionName := "storage" + if len(name) > 0 { + sectionName = sectionName + "." + typ + } + sec := Cfg.Section(sectionName) + + if len(overrides) == 0 { + overrides = []*ini.Section{ + Cfg.Section(sectionName + "." + name), + } + } + + var storage Storage + + storage.Type = sec.Key("STORAGE_TYPE").MustString("") + storage.ServeDirect = sec.Key("SERVE_DIRECT").MustBool(false) + + // Global Defaults + sec.Key("MINIO_ENDPOINT").MustString("localhost:9000") + sec.Key("MINIO_ACCESS_KEY_ID").MustString("") + sec.Key("MINIO_SECRET_ACCESS_KEY").MustString("") + sec.Key("MINIO_BUCKET").MustString("gitea") + sec.Key("MINIO_LOCATION").MustString("us-east-1") + sec.Key("MINIO_USE_SSL").MustBool(false) + + storage.Section = sec + + for _, override := range overrides { + for _, key := range storage.Section.Keys() { + if !override.HasKey(key.Name()) { + _, _ = override.NewKey(key.Name(), key.Value()) + } + } + storage.ServeDirect = override.Key("SERVE_DIRECT").MustBool(false) + storage.Section = override + } + + // Specific defaults + storage.Path = storage.Section.Key("PATH").MustString(filepath.Join(AppDataPath, name)) + if !filepath.IsAbs(storage.Path) { + storage.Path = filepath.Join(AppWorkPath, storage.Path) + storage.Section.Key("PATH").SetValue(storage.Path) + } + storage.Section.Key("MINIO_BASE_PATH").MustString(name + "/") + return storage } - -func newStorageService() { - sec := Cfg.Section("storage") - storages["default"] = getStorage(sec) - - for _, sec := range Cfg.Section("storage").ChildSections() { - name := strings.TrimPrefix(sec.Name(), "storage.") - if name == "default" || name == LocalStorageType || name == MinioStorageType { - log.Error("storage name %s is system reserved!", name) - continue - } - storages[name] = getStorage(sec) - } -} diff --git a/modules/storage/helper.go b/modules/storage/helper.go new file mode 100644 index 000000000000..93f22734e57f --- /dev/null +++ b/modules/storage/helper.go @@ -0,0 +1,65 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package storage + +import ( + "encoding/json" + "reflect" +) + +// Mappable represents an interface that can MapTo another interface +type Mappable interface { + MapTo(v interface{}) error +} + +// toConfig will attempt to convert a given configuration cfg into the provided exemplar type. +// +// It will tolerate the cfg being passed as a []byte or string of a json representation of the +// exemplar or the correct type of the exemplar itself +func toConfig(exemplar, cfg interface{}) (interface{}, error) { + + // First of all check if we've got the same type as the exemplar - if so it's all fine. + if reflect.TypeOf(cfg).AssignableTo(reflect.TypeOf(exemplar)) { + return cfg, nil + } + + // Now if not - does it provide a MapTo function we can try? + if mappable, ok := cfg.(Mappable); ok { + newVal := reflect.New(reflect.TypeOf(exemplar)) + if err := mappable.MapTo(newVal.Interface()); err == nil { + return newVal.Elem().Interface(), nil + } + // MapTo has failed us ... let's try the json route ... + } + + // OK we've been passed a byte array right? + configBytes, ok := cfg.([]byte) + if !ok { + // oh ... it's a string then? + var configStr string + + configStr, ok = cfg.(string) + configBytes = []byte(configStr) + } + if !ok { + // hmm ... can we marshal it to json? + var err error + + configBytes, err = json.Marshal(cfg) + ok = (err == nil) + } + if !ok { + // no ... we've tried hard enough at this point - throw an error! + return nil, ErrInvalidConfiguration{cfg: cfg} + } + + // OK unmarshal the byte array into a new copy of the exemplar + newVal := reflect.New(reflect.TypeOf(exemplar)) + if err := json.Unmarshal(configBytes, newVal.Interface()); err != nil { + // If we can't unmarshal it then return an error! + return nil, ErrInvalidConfiguration{cfg: cfg, err: err} + } + return newVal.Elem().Interface(), nil +} diff --git a/modules/storage/local.go b/modules/storage/local.go index 41859816643c..e270a40b77f8 100644 --- a/modules/storage/local.go +++ b/modules/storage/local.go @@ -5,6 +5,7 @@ package storage import ( + "context" "io" "net/url" "os" @@ -17,19 +18,35 @@ var ( _ ObjectStorage = &LocalStorage{} ) +// LocalStorageType is the type descriptor for local storage +const LocalStorageType Type = "local" + +// LocalStorageConfig represents the configuration for a local storage +type LocalStorageConfig struct { + Path string `ini:"PATH"` +} + // LocalStorage represents a local files storage type LocalStorage struct { + ctx context.Context dir string } // NewLocalStorage returns a local files -func NewLocalStorage(bucket string) (*LocalStorage, error) { - if err := os.MkdirAll(bucket, os.ModePerm); err != nil { +func NewLocalStorage(ctx context.Context, cfg interface{}) (ObjectStorage, error) { + configInterface, err := toConfig(LocalStorageConfig{}, cfg) + if err != nil { + return nil, err + } + config := configInterface.(LocalStorageConfig) + + if err := os.MkdirAll(config.Path, os.ModePerm); err != nil { return nil, err } return &LocalStorage{ - dir: bucket, + ctx: ctx, + dir: config.Path, }, nil } @@ -80,6 +97,11 @@ func (l *LocalStorage) IterateObjects(fn func(path string, obj Object) error) er if err != nil { return err } + select { + case <-l.ctx.Done(): + return l.ctx.Err() + default: + } if path == l.dir { return nil } @@ -98,3 +120,7 @@ func (l *LocalStorage) IterateObjects(fn func(path string, obj Object) error) er return fn(relPath, obj) }) } + +func init() { + RegisterStorageType(LocalStorageType, NewLocalStorage) +} diff --git a/modules/storage/minio.go b/modules/storage/minio.go index d205eff7fd26..eb43fa96ee04 100644 --- a/modules/storage/minio.go +++ b/modules/storage/minio.go @@ -18,8 +18,9 @@ import ( ) var ( - _ ObjectStorage = &MinioStorage{} - quoteEscaper = strings.NewReplacer("\\", "\\\\", `"`, "\\\"") + _ ObjectStorage = &MinioStorage{} + + quoteEscaper = strings.NewReplacer("\\", "\\\\", `"`, "\\\"") ) type minioObject struct { @@ -35,6 +36,20 @@ func (m *minioObject) Stat() (os.FileInfo, error) { return &minioFileInfo{oi}, nil } +// MinioStorageType is the type descriptor for minio storage +const MinioStorageType Type = "minio" + +// MinioStorageConfig represents the configuration for a minio storage +type MinioStorageConfig struct { + Endpoint string `ini:"MINIO_ENDPOINT"` + AccessKeyID string `ini:"MINIO_ACCESS_KEY_ID"` + SecretAccessKey string `ini:"MINIO_SECRET_ACCESS_KEY"` + Bucket string `ini:"MINIO_BUCKET"` + Location string `ini:"MINIO_LOCATION"` + BasePath string `ini:"MINIO_BASE_PATH"` + UseSSL bool `ini:"MINIO_USE_SSL"` +} + // MinioStorage returns a minio bucket storage type MinioStorage struct { ctx context.Context @@ -44,20 +59,26 @@ type MinioStorage struct { } // NewMinioStorage returns a minio storage -func NewMinioStorage(ctx context.Context, endpoint, accessKeyID, secretAccessKey, bucket, location, basePath string, useSSL bool) (*MinioStorage, error) { - minioClient, err := minio.New(endpoint, &minio.Options{ - Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), - Secure: useSSL, +func NewMinioStorage(ctx context.Context, cfg interface{}) (ObjectStorage, error) { + configInterface, err := toConfig(MinioStorageConfig{}, cfg) + if err != nil { + return nil, err + } + config := configInterface.(MinioStorageConfig) + + minioClient, err := minio.New(config.Endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(config.AccessKeyID, config.SecretAccessKey, ""), + Secure: config.UseSSL, }) if err != nil { return nil, err } - if err := minioClient.MakeBucket(ctx, bucket, minio.MakeBucketOptions{ - Region: location, + if err := minioClient.MakeBucket(ctx, config.Bucket, minio.MakeBucketOptions{ + Region: config.Location, }); err != nil { // Check to see if we already own this bucket (which happens if you run this twice) - exists, errBucketExists := minioClient.BucketExists(ctx, bucket) + exists, errBucketExists := minioClient.BucketExists(ctx, config.Bucket) if !exists || errBucketExists != nil { return nil, err } @@ -66,8 +87,8 @@ func NewMinioStorage(ctx context.Context, endpoint, accessKeyID, secretAccessKey return &MinioStorage{ ctx: ctx, client: minioClient, - bucket: bucket, - basePath: basePath, + bucket: config.Bucket, + basePath: config.BasePath, }, nil } @@ -183,3 +204,7 @@ func (m *MinioStorage) IterateObjects(fn func(path string, obj Object) error) er } return nil } + +func init() { + RegisterStorageType(MinioStorageType, NewMinioStorage) +} diff --git a/modules/storage/storage.go b/modules/storage/storage.go index 2cf7b17b4982..8b1c336ae699 100644 --- a/modules/storage/storage.go +++ b/modules/storage/storage.go @@ -22,6 +22,38 @@ var ( ErrIterateObjectsNotSupported = errors.New("iterateObjects method not supported") ) +// ErrInvalidConfiguration is called when there is invalid configuration for a storage +type ErrInvalidConfiguration struct { + cfg interface{} + err error +} + +func (err ErrInvalidConfiguration) Error() string { + if err.err != nil { + return fmt.Sprintf("Invalid Configuration Argument: %v: Error: %v", err.cfg, err.err) + } + return fmt.Sprintf("Invalid Configuration Argument: %v", err.cfg) +} + +// IsErrInvalidConfiguration checks if an error is an ErrInvalidConfiguration +func IsErrInvalidConfiguration(err error) bool { + _, ok := err.(ErrInvalidConfiguration) + return ok +} + +// Type is a type of Storage +type Type string + +// NewStorageFunc is a function that creates a storage +type NewStorageFunc func(ctx context.Context, cfg interface{}) (ObjectStorage, error) + +var storageMap = map[Type]NewStorageFunc{} + +// RegisterStorageType registers a provided storage type with a function to create it +func RegisterStorageType(typ Type, fn func(ctx context.Context, cfg interface{}) (ObjectStorage, error)) { + storageMap[typ] = fn +} + // Object represents the object on the storage type Object interface { io.ReadCloser @@ -67,41 +99,25 @@ func Init() error { return initLFS() } -func initStorage(storageCfg setting.Storage) (ObjectStorage, error) { - var err error - var s ObjectStorage - switch storageCfg.Type { - case setting.LocalStorageType: - s, err = NewLocalStorage(storageCfg.Path) - case setting.MinioStorageType: - minio := storageCfg.Minio - s, err = NewMinioStorage( - context.Background(), - minio.Endpoint, - minio.AccessKeyID, - minio.SecretAccessKey, - minio.Bucket, - minio.Location, - minio.BasePath, - minio.UseSSL, - ) - default: - return nil, fmt.Errorf("Unsupported attachment store type: %s", storageCfg.Type) +// NewStorage takes a storage type and some config and returns an ObjectStorage or an error +func NewStorage(typStr string, cfg interface{}) (ObjectStorage, error) { + if len(typStr) == 0 { + typStr = string(LocalStorageType) + } + fn, ok := storageMap[Type(typStr)] + if !ok { + return nil, fmt.Errorf("Unsupported storage type: %s", typStr) } - if err != nil { - return nil, err - } - - return s, nil + return fn(context.Background(), cfg) } func initAttachments() (err error) { - Attachments, err = initStorage(setting.Attachment.Storage) + Attachments, err = NewStorage(setting.Attachment.Storage.Type, setting.Attachment.Storage) return } func initLFS() (err error) { - LFS, err = initStorage(setting.LFS.Storage) + LFS, err = NewStorage(setting.LFS.Storage.Type, setting.LFS.Storage) return }