test1234
/
gitea
forked from gitea/gitea
1
0
Fork 0
Code Pull Requests Activity

Remove legacy `unknwon/com` package (#19298) 

Follows: #19284
* The `CopyDir` is only used inside test code
* Rewrite `ToSnakeCase` with more test cases
* The `RedisCacher` only put strings into cache, here we use internal `toStr` to replace the legacy `ToStr`
* The `UniqueQueue` can use string as ID directly, no need to call `ToStr`
This commit is contained in:
wxiaoguang 2022-04-02 00:34:57 +08:00 committed by GitHub
parent 4c5cb1e2f2
commit 4f27c28947
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 353 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cmd/manager_logging.go
View File

@ -11,6 +11,7 @@ import (
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/private"
"github.com/urfave/cli"
)

2
contrib/pr/checkout.go
View File

@ -112,7 +112,7 @@ func runPR() {
unittest.LoadFixtures()
util.RemoveAll(setting.RepoRootPath)
util.RemoveAll(models.LocalCopyPath())
util.CopyDir(path.Join(curDir, "integrations/gitea-repositories-meta"), setting.RepoRootPath)
unittest.CopyDir(path.Join(curDir, "integrations/gitea-repositories-meta"), setting.RepoRootPath)
log.Printf("[PR] Setting up router\n")
// routers.GlobalInit()

2
go.mod
View File

@ -78,7 +78,6 @@ require (
github.com/stretchr/testify v1.7.0
github.com/syndtr/goleveldb v1.0.0
github.com/tstranex/u2f v1.0.0
github.com/unknwon/com v1.0.1
github.com/unknwon/i18n v0.0.0-20210904045753-ff3a8617e361
github.com/unknwon/paginater v0.0.0-20200328080006-042474bd0eae
github.com/unrolled/render v1.4.1
@ -251,6 +250,7 @@ require (
github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect
github.com/toqueteos/webbrowser v1.2.0 // indirect
github.com/ulikunitz/xz v0.5.10 // indirect
github.com/unknwon/com v1.0.1 // indirect
github.com/x448/float16 v0.8.4 // indirect
github.com/xanzy/ssh-agent v0.3.1 // indirect
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect

4
integrations/integration_test.go
View File

@ -254,7 +254,7 @@ func prepareTestEnv(t testing.TB, skip ...int) func() {
assert.NoError(t, unittest.LoadFixtures())
assert.NoError(t, util.RemoveAll(setting.RepoRootPath))
assert.NoError(t, util.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), setting.RepoRootPath))
assert.NoError(t, unittest.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), setting.RepoRootPath))
ownerDirs, err := os.ReadDir(setting.RepoRootPath)
if err != nil {
assert.NoError(t, err, "unable to read the new repo root: %v\n", err)
@ -550,7 +550,7 @@ func resetFixtures(t *testing.T) {
assert.NoError(t, queue.GetManager().FlushAll(context.Background(), -1))
assert.NoError(t, unittest.LoadFixtures())
assert.NoError(t, util.RemoveAll(setting.RepoRootPath))
assert.NoError(t, util.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), setting.RepoRootPath))
assert.NoError(t, unittest.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), setting.RepoRootPath))
ownerDirs, err := os.ReadDir(setting.RepoRootPath)
if err != nil {
assert.NoError(t, err, "unable to read the new repo root: %v\n", err)

3
integrations/migration-test/migration_test.go
View File

@ -21,6 +21,7 @@ import (
"code.gitea.io/gitea/integrations"
"code.gitea.io/gitea/models/db"
"code.gitea.io/gitea/models/migrations"
"code.gitea.io/gitea/models/unittest"
"code.gitea.io/gitea/modules/base"
"code.gitea.io/gitea/modules/charset"
"code.gitea.io/gitea/modules/git"
@ -60,7 +61,7 @@ func initMigrationTest(t *testing.T) func() {
assert.True(t, len(setting.RepoRootPath) != 0)
assert.NoError(t, util.RemoveAll(setting.RepoRootPath))
assert.NoError(t, util.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), setting.RepoRootPath))
assert.NoError(t, unittest.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), setting.RepoRootPath))
ownerDirs, err := os.ReadDir(setting.RepoRootPath)
if err != nil {
assert.NoError(t, err, "unable to read the new repo root: %v\n", err)

2
models/migrations/migrations_test.go
View File

@ -203,7 +203,7 @@ func prepareTestEnv(t *testing.T, skip int, syncModels ...interface{}) (*xorm.En
deferFn := PrintCurrentTest(t, ourSkip)
assert.NoError(t, os.RemoveAll(setting.RepoRootPath))
assert.NoError(t, util.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"),
assert.NoError(t, unittest.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"),
setting.RepoRootPath))
ownerDirs, err := os.ReadDir(setting.RepoRootPath)
if err != nil {

103
models/unittest/fscopy.go Normal file
View File

@ -0,0 +1,103 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package unittest
import (
"errors"
"io"
"os"
"path"
"strings"
"code.gitea.io/gitea/modules/util"
)
// Copy copies file from source to target path.
func Copy(src, dest string) error {
// Gather file information to set back later.
si, err := os.Lstat(src)
if err != nil {
return err
}
// Handle symbolic link.
if si.Mode()&os.ModeSymlink != 0 {
target, err := os.Readlink(src)
if err != nil {
return err
}
// NOTE: os.Chmod and os.Chtimes don't recognize symbolic link,
// which will lead "no such file or directory" error.
return os.Symlink(target, dest)
}
sr, err := os.Open(src)
if err != nil {
return err
}
defer sr.Close()
dw, err := os.Create(dest)
if err != nil {
return err
}
defer dw.Close()
if _, err = io.Copy(dw, sr); err != nil {
return err
}
// Set back file information.
if err = os.Chtimes(dest, si.ModTime(), si.ModTime()); err != nil {
return err
}
return os.Chmod(dest, si.Mode())
}
// CopyDir copy files recursively from source to target directory.
//
// The filter accepts a function that process the path info.
// and should return true for need to filter.
//
// It returns error when error occurs in underlying functions.
func CopyDir(srcPath, destPath string, filters ...func(filePath string) bool) error {
// Check if target directory exists.
if _, err := os.Stat(destPath); !errors.Is(err, os.ErrNotExist) {
return errors.New("file or directory already exists: " + destPath)
}
err := os.MkdirAll(destPath, os.ModePerm)
if err != nil {
return err
}
// Gather directory info.
infos, err := util.StatDir(srcPath, true)
if err != nil {
return err
}
var filter func(filePath string) bool
if len(filters) > 0 {
filter = filters[0]
}
for _, info := range infos {
if filter != nil && filter(info) {
continue
}
curPath := path.Join(destPath, info)
if strings.HasSuffix(info, "/") {
err = os.MkdirAll(curPath, os.ModePerm)
} else {
err = Copy(path.Join(srcPath, info), curPath)
}
if err != nil {
return err
}
}
return nil
}

4
models/unittest/testdb.go
View File

@ -104,7 +104,7 @@ func MainTest(m *testing.M, pathToGiteaRoot string, fixtureFiles ...string) {
if err = util.RemoveAll(repoRootPath); err != nil {
fatalTestError("util.RemoveAll: %v\n", err)
}
if err = util.CopyDir(filepath.Join(pathToGiteaRoot, "integrations", "gitea-repositories-meta"), setting.RepoRootPath); err != nil {
if err = CopyDir(filepath.Join(pathToGiteaRoot, "integrations", "gitea-repositories-meta"), setting.RepoRootPath); err != nil {
fatalTestError("util.CopyDir: %v\n", err)
}
@ -175,7 +175,7 @@ func PrepareTestEnv(t testing.TB) {
assert.NoError(t, PrepareTestDatabase())
assert.NoError(t, util.RemoveAll(setting.RepoRootPath))
metaPath := filepath.Join(giteaRoot, "integrations", "gitea-repositories-meta")
assert.NoError(t, util.CopyDir(metaPath, setting.RepoRootPath))
assert.NoError(t, CopyDir(metaPath, setting.RepoRootPath))
ownerDirs, err := os.ReadDir(setting.RepoRootPath)
assert.NoError(t, err)

33
modules/cache/cache_redis.go vendored
View File

@ -6,11 +6,11 @@ package cache
import (
"fmt"
"strconv"
"time"
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/nosql"
"code.gitea.io/gitea/modules/util"
"gitea.com/go-chi/cache"
"github.com/go-redis/redis/v8"
@ -24,20 +24,37 @@ type RedisCacher struct {
occupyMode bool
}
// Put puts value into cache with key and expire time.
// toStr convert string/int/int64 interface to string. it's only used by the RedisCacher.Put internally
func toStr(v interface{}) string {
if v == nil {
return ""
}
switch v := v.(type) {
case string:
return v
case []byte:
return string(v)
case int:
return strconv.FormatInt(int64(v), 10)
case int64:
return strconv.FormatInt(v, 10)
default:
return fmt.Sprint(v) // as what the old com.ToStr does in most cases
}
}
// Put puts value (string type) into cache with key and expire time.
// If expired is 0, it lives forever.
func (c *RedisCacher) Put(key string, val interface{}, expire int64) error {
// this function is not well-designed, it only puts string values into cache
key = c.prefix + key
if expire == 0 {
if err := c.c.Set(graceful.GetManager().HammerContext(), key, util.ToStr(val), 0).Err(); err != nil {
if err := c.c.Set(graceful.GetManager().HammerContext(), key, toStr(val), 0).Err(); err != nil {
return err
}
} else {
dur, err := time.ParseDuration(util.ToStr(expire) + "s")
if err != nil {
return err
}
if err = c.c.Set(graceful.GetManager().HammerContext(), key, util.ToStr(val), dur).Err(); err != nil {
dur := time.Duration(expire) * time.Second
if err := c.c.Set(graceful.GetManager().HammerContext(), key, toStr(val), dur).Err(); err != nil {
return err
}
}

15
modules/context/csrf.go
View File

@ -22,8 +22,10 @@ import (
"encoding/base32"
"fmt"
"net/http"
"strconv"
"time"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"
"code.gitea.io/gitea/modules/web/middleware"
@ -215,9 +217,16 @@ func Csrfer(opt CsrfOptions, ctx *Context) CSRF {
}
x.ID = "0"
uid := ctx.Session.Get(opt.SessionKey)
if uid != nil {
x.ID = util.ToStr(uid)
uidAny := ctx.Session.Get(opt.SessionKey)
if uidAny != nil {
switch uidVal := uidAny.(type) {
case string:
x.ID = uidVal
case int64:
x.ID = strconv.FormatInt(uidVal, 10)
default:
log.Error("invalid uid type in session: %T", uidAny)
}
}
needsNew := false

1
modules/nosql/manager.go
View File

@ -11,6 +11,7 @@ import (
"time"
"code.gitea.io/gitea/modules/process"
"github.com/go-redis/redis/v8"
"github.com/syndtr/goleveldb/leveldb"
)

25
modules/sync/unique_queue.go
View File

@ -5,8 +5,6 @@
package sync
import "code.gitea.io/gitea/modules/util"
// UniqueQueue is a queue which guarantees only one instance of same
// identity is in the line. Instances with same identity will be
// discarded if there is already one in the line.
@ -53,10 +51,10 @@ func (q *UniqueQueue) IsClosed() <-chan struct{} {
}
// IDs returns the current ids in the pool
func (q *UniqueQueue) IDs() []interface{} {
func (q *UniqueQueue) IDs() []string {
q.table.lock.Lock()
defer q.table.lock.Unlock()
ids := make([]interface{}, 0, len(q.table.pool))
ids := make([]string, 0, len(q.table.pool))
for id := range q.table.pool {
ids = append(ids, id)
}
@ -70,20 +68,19 @@ func (q *UniqueQueue) Queue() <-chan string {
// Exist returns true if there is an instance with given identity
// exists in the queue.
func (q *UniqueQueue) Exist(id interface{}) bool {
return q.table.IsRunning(util.ToStr(id))
func (q *UniqueQueue) Exist(id string) bool {
return q.table.IsRunning(id)
}
// AddFunc adds new instance to the queue with a custom runnable function,
// the queue is blocked until the function exits.
func (q *UniqueQueue) AddFunc(id interface{}, fn func()) {
idStr := util.ToStr(id)
func (q *UniqueQueue) AddFunc(id string, fn func()) {
q.table.lock.Lock()
if _, ok := q.table.pool[idStr]; ok {
if _, ok := q.table.pool[id]; ok {
q.table.lock.Unlock()
return
}
q.table.pool[idStr] = struct{}{}
q.table.pool[id] = struct{}{}
if fn != nil {
fn()
}
@ -91,17 +88,17 @@ func (q *UniqueQueue) AddFunc(id interface{}, fn func()) {
select {
case <-q.closed:
return
case q.queue <- idStr:
case q.queue <- id:
return
}
}
// Add adds new instance to the queue.
func (q *UniqueQueue) Add(id interface{}) {
func (q *UniqueQueue) Add(id string) {
q.AddFunc(id, nil)
}
// Remove removes instance from the queue.
func (q *UniqueQueue) Remove(id interface{}) {
q.table.Stop(util.ToStr(id))
func (q *UniqueQueue) Remove(id string) {
q.table.Stop(id)
}

40
modules/util/legacy.go
View File

@ -9,29 +9,37 @@ import (
"crypto/cipher"
"crypto/rand"
"errors"
"github.com/unknwon/com" //nolint:depguard
"io"
"os"
)
// CopyFile copies file from source to target path.
func CopyFile(src, dest string) error {
return com.Copy(src, dest)
}
si, err := os.Lstat(src)
if err != nil {
return err
}
// CopyDir copy files recursively from source to target directory.
// It returns error when error occurs in underlying functions.
func CopyDir(srcPath, destPath string) error {
return com.CopyDir(srcPath, destPath)
}
sr, err := os.Open(src)
if err != nil {
return err
}
defer sr.Close()
// ToStr converts any interface to string. should be replaced.
func ToStr(value interface{}, args ...int) string {
return com.ToStr(value, args...)
}
dw, err := os.Create(dest)
if err != nil {
return err
}
defer dw.Close()
// ToSnakeCase converts a string to snake_case. should be replaced.
func ToSnakeCase(str string) string {
return com.ToSnakeCase(str)
if _, err = io.Copy(dw, sr); err != nil {
return err
}
if err = os.Chtimes(dest, si.ModTime(), si.ModTime()); err != nil {
return err
}
return os.Chmod(dest, si.Mode())
}
// AESGCMEncrypt (from legacy package): encrypts plaintext with the given key using AES in GCM mode. should be replaced.

33
modules/util/legacy_test.go
View File

@ -7,12 +7,38 @@ package util
import (
"crypto/aes"
"crypto/rand"
"fmt"
"os"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/unknwon/com" //nolint:depguard
)
func TestCopyFile(t *testing.T) {
testContent := []byte("hello")
tmpDir := os.TempDir()
now := time.Now()
srcFile := fmt.Sprintf("%s/copy-test-%d-src.txt", tmpDir, now.UnixMicro())
dstFile := fmt.Sprintf("%s/copy-test-%d-dst.txt", tmpDir, now.UnixMicro())
_ = os.Remove(srcFile)
_ = os.Remove(dstFile)
defer func() {
_ = os.Remove(srcFile)
_ = os.Remove(dstFile)
}()
err := os.WriteFile(srcFile, testContent, 0o777)
assert.NoError(t, err)
err = CopyFile(srcFile, dstFile)
assert.NoError(t, err)
dstContent, err := os.ReadFile(dstFile)
assert.NoError(t, err)
assert.Equal(t, testContent, dstContent)
}
func TestAESGCM(t *testing.T) {
t.Parallel()
@ -29,9 +55,4 @@ func TestAESGCM(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, plaintext, decrypted)
// at the moment, we make sure the result is the same as the legacy package, this assertion can be removed in next round refactoring
legacy, err := com.AESGCMDecrypt(key, ciphertext)
assert.NoError(t, err)
assert.Equal(t, legacy, plaintext)
}

88
modules/util/string.go Normal file
View File

@ -0,0 +1,88 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package util
import "github.com/yuin/goldmark/util"
func isSnakeCaseUpper(c byte) bool {
return 'A' <= c && c <= 'Z'
}
func isSnakeCaseLowerOrNumber(c byte) bool {
return 'a' <= c && c <= 'z' || '0' <= c && c <= '9'
}
// ToSnakeCase convert the input string to snake_case format.
//
// Some samples.
// "FirstName" => "first_name"
// "HTTPServer" => "http_server"
// "NoHTTPS" => "no_https"
// "GO_PATH" => "go_path"
// "GO PATH" => "go_path" // space is converted to underscore.
// "GO-PATH" => "go_path" // hyphen is converted to underscore.
//
func ToSnakeCase(input string) string {
if len(input) == 0 {
return ""
}
var res []byte
if len(input) == 1 {
c := input[0]
if isSnakeCaseUpper(c) {
res = []byte{c + 'a' - 'A'}
} else if isSnakeCaseLowerOrNumber(c) {
res = []byte{c}
} else {
res = []byte{'_'}
}
} else {
res = make([]byte, 0, len(input)*4/3)
pos := 0
needSep := false
for pos < len(input) {
c := input[pos]
if c >= 0x80 {
res = append(res, c)
pos++
continue
}
isUpper := isSnakeCaseUpper(c)
if isUpper || isSnakeCaseLowerOrNumber(c) {
end := pos + 1
if isUpper {
// skip the following upper letters
for end < len(input) && isSnakeCaseUpper(input[end]) {
end++
}
if end-pos > 1 && end < len(input) && isSnakeCaseLowerOrNumber(input[end]) {
end--
}
}
// skip the following lower or number letters
for end < len(input) && (isSnakeCaseLowerOrNumber(input[end]) || input[end] >= 0x80) {
end++
}
if needSep {
res = append(res, '_')
}
res = append(res, input[pos:end]...)
pos = end
needSep = true
} else {
res = append(res, '_')
pos++
needSep = false
}
}
for i := 0; i < len(res); i++ {
if isSnakeCaseUpper(res[i]) {
res[i] += 'a' - 'A'
}
}
}
return util.BytesToReadOnlyString(res)
}

48
modules/util/string_test.go Normal file
View File

@ -0,0 +1,48 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package util
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestToSnakeCase(t *testing.T) {
cases := map[string]string{
// all old cases from the legacy package
"HTTPServer": "http_server",
"_camelCase": "_camel_case",
"NoHTTPS": "no_https",
"Wi_thF": "wi_th_f",
"_AnotherTES_TCaseP": "_another_tes_t_case_p",
"ALL": "all",
"_HELLO_WORLD_": "_hello_world_",
"HELLO_WORLD": "hello_world",
"HELLO____WORLD": "hello____world",
"TW": "tw",
"_C": "_c",
" sentence case ": "__sentence_case__",
" Mixed-hyphen case _and SENTENCE_case and UPPER-case": "_mixed_hyphen_case__and_sentence_case_and_upper_case",
// new cases
" ": "_",
"A": "a",
"A0": "a0",
"a0": "a0",
"Aa0": "aa0",
"啊": "啊",
"A啊": "a啊",
"Aa啊b": "aa啊b",
"A啊B": "a啊_b",
"Aa啊B": "aa啊_b",
"TheCase2": "the_case2",
"ObjIDs": "obj_i_ds", // the strange database column name which already exists
}
for input, expected := range cases {
assert.Equal(t, expected, ToSnakeCase(input))
}
}

7
services/webhook/webhook.go
View File

@ -6,6 +6,7 @@ package webhook
import (
"fmt"
"strconv"
"strings"
repo_model "code.gitea.io/gitea/models/repo"
@ -106,7 +107,7 @@ func PrepareWebhook(w *webhook_model.Webhook, repo *repo_model.Repository, event
return err
}
go hookQueue.Add(repo.ID)
go hookQueue.Add(strconv.FormatInt(repo.ID, 10))
return nil
}
@ -187,7 +188,7 @@ func PrepareWebhooks(repo *repo_model.Repository, event webhook_model.HookEventT
return err
}
go hookQueue.Add(repo.ID)
go hookQueue.Add(strconv.FormatInt(repo.ID, 10))
return nil
}
@ -239,7 +240,7 @@ func ReplayHookTask(w *webhook_model.Webhook, uuid string) error {
return err
}
go hookQueue.Add(t.RepoID)
go hookQueue.Add(strconv.FormatInt(t.RepoID, 10))
return nil
}