Bindata is optional and over-writable on restart (#354)

* Moved conf assets into options folder

* Dropped old bindata

* Started to integrate options bindata and accessors

* Do not enforce a builtin app.ini

* Replaced bindata calls with options

* Dropped bindata task from makefile, it's the generate task now

* Always embedd app.ini to provide sane config defaults

* Use sane defaults for the configuration

* Defined default value for SSH_KEYGEN_PATH

* Dropped "NEVER EVER MODIFY THIS FILE" header from app.ini

* Fixed new paths in latest test additions

* Drop bindata with make clean task

* Set more proper default values

.gitignore

@@ -28,6 +28,7 @@ _testmain.go
 
 coverage.out
 
+/modules/options/bindata.go
 /modules/public/bindata.go
 /modules/templates/bindata.go
 

Makefile

@@ -2,10 +2,7 @@ DIST := dist
 EXECUTABLE := gitea
 IMPORT := code.gitea.io/gitea
 
-SHA := $(shell git rev-parse --short HEAD)
+BINDATA := modules/{options,public,templates}/bindata.go
-DATE := $(shell date -u '+%Y-%m-%d %I:%M:%S %Z')
-
-BINDATA := $(shell find conf | sed 's/ /\\ /g')
 STYLESHEETS := $(wildcard public/less/index.less public/less/_*.less)
 JAVASCRIPTS :=
 
@@ -33,7 +30,7 @@ all: build
 .PHONY: clean
 clean:
 	go clean -i ./...
-	rm -rf $(EXECUTABLE) $(DIST)
+	rm -rf $(EXECUTABLE) $(DIST) $(BINDATA)
 
 .PHONY: fmt
 fmt:
@@ -119,19 +116,6 @@ release-copy:
 release-check:
 	cd $(DIST)/release; $(foreach file,$(wildcard $(DIST)/release/$(EXECUTABLE)-*),sha256sum $(notdir $(file)) > $(notdir $(file)).sha256;)
 
-.PHONY: bindata
-bindata: modules/bindata/bindata.go
-
-.IGNORE: modules/bindata/bindata.go
-modules/bindata/bindata.go: $(BINDATA)
-	@which go-bindata > /dev/null; if [ $$? -ne 0 ]; then \
-		go get -u github.com/jteeuwen/go-bindata/...; \
-	fi
-	go-bindata -o=$@ -ignore="\\.go|README.md|TRANSLATORS" -pkg=bindata conf/...
-	go fmt $@
-	sed -i.bak 's/confLocaleLocale_/confLocaleLocale/' $@
-	rm $@.bak
-
 .PHONY: javascripts
 javascripts: public/js/index.js
 
@@ -147,4 +131,4 @@ public/css/index.css: $(STYLESHEETS)
 	lessc $< $@
 
 .PHONY: assets
-assets: bindata javascripts stylesheets
+assets: javascripts stylesheets

cmd/web.go

@@ -16,9 +16,9 @@ import (
 
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/auth"
-	"code.gitea.io/gitea/modules/bindata"
 	"code.gitea.io/gitea/modules/context"
 	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/options"
 	"code.gitea.io/gitea/modules/public"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/templates"
@@ -99,22 +99,29 @@ func newMacaron() *macaron.Macaron {
 	m.Use(templates.Renderer())
 	models.InitMailRender(templates.Mailer())
 
-	localeNames, err := bindata.AssetDir("conf/locale")
+	localeNames, err := options.Dir("locale")
+
 	if err != nil {
 		log.Fatal(4, "Fail to list locale files: %v", err)
 	}
+
 	localFiles := make(map[string][]byte)
+
 	for _, name := range localeNames {
-		localFiles[name] = bindata.MustAsset("conf/locale/" + name)
+		localFiles[name], err = options.Locale(name)
+
+		if err != nil {
+			log.Fatal(4, "Failed to load %s locale file. %v", name, err)
+		}
 	}
+
 	m.Use(i18n.I18n(i18n.Options{
-		SubURL:          setting.AppSubURL,
+		SubURL:      setting.AppSubURL,
-		Files:           localFiles,
+		Files:       localFiles,
-		CustomDirectory: path.Join(setting.CustomPath, "conf/locale"),
+		Langs:       setting.Langs,
-		Langs:           setting.Langs,
+		Names:       setting.Names,
-		Names:           setting.Names,
+		DefaultLang: "en-US",
-		DefaultLang:     "en-US",
+		Redirect:    true,
-		Redirect:        true,
 	}))
 	m.Use(cache.Cacher(cache.Options{
 		Adapter:       setting.CacheAdapter,

conf/README.md

@@ -1,3 +0,0 @@
-Execute following command in ROOT directory when anything is changed:
-
-$ make bindata

conf/app.ini

@@ -1,6 +1,3 @@
-# NEVER EVER MODIFY THIS FILE
-# PLEASE MAKE CHANGES ON CORRESPONDING CUSTOM CONFIG FILE
-
 ; App name that shows on every page title
 APP_NAME = Gitea: Git with a cup of tea
 ; Change it if you run locally

models/repo.go

@@ -20,9 +20,9 @@ import (
 	"time"
 
 	"code.gitea.io/git"
-	"code.gitea.io/gitea/modules/bindata"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/markdown"
+	"code.gitea.io/gitea/modules/options"
 	"code.gitea.io/gitea/modules/process"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/sync"
@@ -80,11 +80,11 @@ func LoadRepoConfig() {
 	types := []string{"gitignore", "license", "readme", "label"}
 	typeFiles := make([][]string, 4)
 	for i, t := range types {
-		files, err := bindata.AssetDir("conf/" + t)
+		files, err := options.Dir(t)
 		if err != nil {
 			log.Fatal(4, "Fail to get %s files: %v", t, err)
 		}
-		customPath := path.Join(setting.CustomPath, "conf", t)
+		customPath := path.Join(setting.CustomPath, "options", t)
 		if com.IsDir(customPath) {
 			customFiles, err := com.StatDir(customPath)
 			if err != nil {
@@ -827,14 +827,25 @@ type CreateRepoOptions struct {
 }
 
 func getRepoInitFile(tp, name string) ([]byte, error) {
-	relPath := path.Join("conf", tp, strings.TrimLeft(name, "./"))
+	cleanedName := strings.TrimLeft(name, "./")
+	relPath := path.Join("options", tp, cleanedName)
 
 	// Use custom file when available.
 	customPath := path.Join(setting.CustomPath, relPath)
 	if com.IsFile(customPath) {
 		return ioutil.ReadFile(customPath)
 	}
-	return bindata.Asset(relPath)
+
+	switch tp {
+	case "readme":
+		return options.Readme(cleanedName)
+	case "gitignore":
+		return options.Gitignore(cleanedName)
+	case "license":
+		return options.License(cleanedName)
+	default:
+		return []byte{}, fmt.Errorf("Invalid init file type")
+	}
 }
 
 func prepareRepoCommit(repo *Repository, tmpDir, repoPath string, opts CreateRepoOptions) error {

modules/base/tool_test.go

@@ -25,7 +25,7 @@ const (
 func TestMain(m *testing.M) {
 	// setup
 	macaroni18n.I18n(macaroni18n.Options{
-		Directory:   "../../conf/locale/",
+		Directory:   "../../options/locale/",
 		DefaultLang: "en-US",
 		Langs:       []string{"en-US"},
 		Names:       []string{"english"},

modules/options/dynamic.go

@@ -0,0 +1,94 @@
+// +build !bindata
+
+// Copyright 2016 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package options
+
+import (
+	"fmt"
+	"io/ioutil"
+	"path"
+
+	"code.gitea.io/gitea/modules/setting"
+	"github.com/Unknwon/com"
+)
+
+var (
+	directories = make(directorySet)
+)
+
+// Dir returns all files from static or custom directory.
+func Dir(name string) ([]string, error) {
+	if directories.Filled(name) {
+		return directories.Get(name), nil
+	}
+
+	var (
+		result []string
+	)
+
+	customDir := path.Join(setting.CustomPath, "options", name)
+
+	if com.IsDir(customDir) {
+		files, err := com.StatDir(customDir, true)
+
+		if err != nil {
+			return []string{}, fmt.Errorf("Failed to read custom directory. %v", err)
+		}
+
+		result = append(result, files...)
+	}
+
+	staticDir := path.Join(setting.StaticRootPath, "options", name)
+
+	if com.IsDir(staticDir) {
+		files, err := com.StatDir(staticDir, true)
+
+		if err != nil {
+			return []string{}, fmt.Errorf("Failed to read static directory. %v", err)
+		}
+
+		result = append(result, files...)
+	}
+
+	return directories.AddAndGet(name, result), nil
+}
+
+// Locale reads the content of a specific locale from static or custom path.
+func Locale(name string) ([]byte, error) {
+	return fileFromDir(path.Join("locale", name))
+}
+
+// Readme reads the content of a specific readme from static or custom path.
+func Readme(name string) ([]byte, error) {
+	return fileFromDir(path.Join("readme", name))
+}
+
+// Gitignore reads the content of a specific gitignore from static or custom path.
+func Gitignore(name string) ([]byte, error) {
+	return fileFromDir(path.Join("gitignore", name))
+}
+
+// License reads the content of a specific license from static or custom path.
+func License(name string) ([]byte, error) {
+	return fileFromDir(path.Join("license", name))
+}
+
+// fileFromDir is a helper to read files from static or custom path.
+func fileFromDir(name string) ([]byte, error) {
+	customPath := path.Join(setting.CustomPath, "options", name)
+
+	if com.IsFile(customPath) {
+		return ioutil.ReadFile(customPath)
+	}
+
+	staticPath := path.Join(setting.StaticRootPath, "options", name)
+
+	if com.IsFile(staticPath) {
+		return ioutil.ReadFile(staticPath)
+	}
+
+	return []byte{}, fmt.Errorf("Asset file does not exist: %s", name)
+}

modules/options/options.go

@@ -0,0 +1,51 @@
+// Copyright 2016 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package options
+
+//go:generate go-bindata -tags "bindata" -ignore "TRANSLATORS" -pkg "options" -o "bindata.go" ../../options/...
+//go:generate go fmt bindata.go
+//go:generate sed -i.bak s/..\/..\/options\/// bindata.go
+//go:generate rm -f bindata.go.bak
+
+type directorySet map[string][]string
+
+func (s directorySet) Add(key string, value []string) {
+	_, ok := s[key]
+
+	if !ok {
+		s[key] = make([]string, 0, len(value))
+	}
+
+	s[key] = append(s[key], value...)
+}
+
+func (s directorySet) Get(key string) []string {
+	_, ok := s[key]
+
+	if ok {
+		result := []string{}
+		seen := map[string]string{}
+
+		for _, val := range s[key] {
+			if _, ok := seen[val]; !ok {
+				result = append(result, val)
+				seen[val] = val
+			}
+		}
+
+		return result
+	}
+
+	return []string{}
+}
+
+func (s directorySet) AddAndGet(key string, value []string) []string {
+	s.Add(key, value)
+	return s.Get(key)
+}
+
+func (s directorySet) Filled(key string) bool {
+	return len(s[key]) > 0
+}

modules/options/static.go

@@ -0,0 +1,84 @@
+// +build bindata
+
+// Copyright 2016 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package options
+
+import (
+	"fmt"
+	"io/ioutil"
+	"path"
+
+	"code.gitea.io/gitea/modules/setting"
+	"github.com/Unknwon/com"
+)
+
+var (
+	directories = make(directorySet)
+)
+
+// Dir returns all files from bindata or custom directory.
+func Dir(name string) ([]string, error) {
+	if directories.Filled(name) {
+		return directories.Get(name), nil
+	}
+
+	var (
+		result []string
+	)
+
+	customDir := path.Join(setting.CustomPath, "options", name)
+
+	if com.IsDir(customDir) {
+		files, err := com.StatDir(customDir, true)
+
+		if err != nil {
+			return []string{}, fmt.Errorf("Failed to read custom directory. %v", err)
+		}
+
+		result = append(result, files...)
+	}
+
+	files, err := AssetDir(path.Join("..", "..", "options", name))
+
+	if err != nil {
+		return []string{}, fmt.Errorf("Failed to read embedded directory. %v", err)
+	}
+
+	result = append(result, files...)
+
+	return directories.AddAndGet(name, result), nil
+}
+
+// Locale reads the content of a specific locale from bindata or custom path.
+func Locale(name string) ([]byte, error) {
+	return fileFromDir(path.Join("locale", name))
+}
+
+// Readme reads the content of a specific readme from bindata or custom path.
+func Readme(name string) ([]byte, error) {
+	return fileFromDir(path.Join("readme", name))
+}
+
+// Gitignore reads the content of a gitignore locale from bindata or custom path.
+func Gitignore(name string) ([]byte, error) {
+	return fileFromDir(path.Join("gitignore", name))
+}
+
+// License reads the content of a specific license from bindata or custom path.
+func License(name string) ([]byte, error) {
+	return fileFromDir(path.Join("license", name))
+}
+
+// fileFromDir is a helper to read files from bindata or custom path.
+func fileFromDir(name string) ([]byte, error) {
+	customPath := path.Join(setting.CustomPath, "options", name)
+
+	if com.IsFile(customPath) {
+		return ioutil.ReadFile(customPath)
+	}
+
+	return Asset(name)
+}

modules/setting/setting.go

@@ -17,6 +17,8 @@ import (
 	"strings"
 	"time"
 
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/user"
 	"github.com/Unknwon/com"
 	_ "github.com/go-macaron/cache/memcache" // memcache plugin for cache
 	_ "github.com/go-macaron/cache/redis"
@@ -25,10 +27,6 @@ import (
 	_ "github.com/kardianos/minwinsvc"      // import minwinsvc for windows services
 	"gopkg.in/ini.v1"
 	"strk.kbt.io/projects/go/libravatar"
-
-	"code.gitea.io/gitea/modules/bindata"
-	"code.gitea.io/gitea/modules/log"
-	"code.gitea.io/gitea/modules/user"
 )
 
 // Scheme describes protocol types
@@ -349,9 +347,10 @@ func NewContext() {
 		log.Fatal(4, "Fail to get work directory: %v", err)
 	}
 
-	Cfg, err = ini.Load(bindata.MustAsset("conf/app.ini"))
+	Cfg = ini.Empty()
+
 	if err != nil {
-		log.Fatal(4, "Fail to parse 'conf/app.ini': %v", err)
+		log.Fatal(4, "Fail to parse 'app.ini': %v", err)
 	}
 
 	CustomPath = os.Getenv("GITEA_CUSTOM")
@@ -444,6 +443,10 @@ please consider changing to GITEA_CUSTOM`)
 	if err = Cfg.Section("server").MapTo(&SSH); err != nil {
 		log.Fatal(4, "Fail to map SSH settings: %v", err)
 	}
+
+	SSH.KeygenPath = sec.Key("SSH_KEYGEN_PATH").MustString("ssh-keygen")
+	SSH.Port = sec.Key("SSH_PORT").MustInt(22)
+
 	// When disable SSH, start builtin server value is ignored.
 	if SSH.Disabled {
 		SSH.StartBuiltinServer = false
@@ -502,7 +505,7 @@ please consider changing to GITEA_CUSTOM`)
 		"StampNano":   time.StampNano,
 	}[Cfg.Section("time").Key("FORMAT").MustString("RFC1123")]
 
-	RunUser = Cfg.Section("").Key("RUN_USER").String()
+	RunUser = Cfg.Section("").Key("RUN_USER").MustString(user.CurrentUsername())
 	// Does not check run user when the install lock is off.
 	if InstallLock {
 		currentUser, match := IsRunUserMatchCurrentUser(RunUser)
@@ -593,7 +596,17 @@ please consider changing to GITEA_CUSTOM`)
 	}
 
 	Langs = Cfg.Section("i18n").Key("LANGS").Strings(",")
+	if len(Langs) == 0 {
+		Langs = []string{
+			"en-US",
+		}
+	}
 	Names = Cfg.Section("i18n").Key("NAMES").Strings(",")
+	if len(Names) == 0 {
+		Names = []string{
+			"English",
+		}
+	}
 	dateLangs = Cfg.Section("i18n.datelang").KeysHash()
 
 	ShowFooterBranding = Cfg.Section("other").Key("SHOW_FOOTER_BRANDING").MustBool()
@@ -639,16 +652,18 @@ var logLevels = map[string]string{
 }
 
 func newLogService() {
-	log.Info("%s %s", AppName, AppVer)
+	log.Info("Gitea v%s", AppVer)
 
-	// Get and check log mode.
 	LogModes = strings.Split(Cfg.Section("log").Key("MODE").MustString("console"), ",")
 	LogConfigs = make([]string, len(LogModes))
+
 	for i, mode := range LogModes {
 		mode = strings.TrimSpace(mode)
+
 		sec, err := Cfg.GetSection("log." + mode)
+
 		if err != nil {
-			log.Fatal(4, "Unknown log mode: %s", mode)
+			sec, _ = Cfg.NewSection("log." + mode)
 		}
 
 		validLevels := []string{"Trace", "Debug", "Info", "Warn", "Error", "Critical"}