forked from gitea/gitea
1
0
Fork 0

[Feature] add precise search type for Elastic Search (#12869)

* feat: add type query parameters for specifying precise search

* feat: add select dropdown in search box

Co-authored-by: Lauris BH <lauris@nix.lv>
Co-authored-by: techknowlogick <techknowlogick@gitea.io>
This commit is contained in:
Jui-Nan Lin 2021-01-27 18:00:35 +08:00 committed by GitHub
parent b2c20b68a0
commit c10503afec
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 77 additions and 25 deletions

View File

@ -53,4 +53,5 @@ func (p *Pagination) SetDefaultParams(ctx *Context) {
p.AddParam(ctx, "sort", "SortType") p.AddParam(ctx, "sort", "SortType")
p.AddParam(ctx, "q", "Keyword") p.AddParam(ctx, "q", "Keyword")
p.AddParam(ctx, "tab", "TabName") p.AddParam(ctx, "tab", "TabName")
p.AddParam(ctx, "t", "queryType")
} }

View File

@ -280,12 +280,23 @@ func (b *BleveIndexer) Delete(repoID int64) error {
// Search searches for files in the specified repo. // Search searches for files in the specified repo.
// Returns the matching file-paths // Returns the matching file-paths
func (b *BleveIndexer) Search(repoIDs []int64, language, keyword string, page, pageSize int) (int64, []*SearchResult, []*SearchResultLanguages, error) { func (b *BleveIndexer) Search(repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error) {
phraseQuery := bleve.NewMatchPhraseQuery(keyword) var (
phraseQuery.FieldVal = "Content" indexerQuery query.Query
phraseQuery.Analyzer = repoIndexerAnalyzer keywordQuery query.Query
)
if isMatch {
prefixQuery := bleve.NewPrefixQuery(keyword)
prefixQuery.FieldVal = "Content"
keywordQuery = prefixQuery
} else {
phraseQuery := bleve.NewMatchPhraseQuery(keyword)
phraseQuery.FieldVal = "Content"
phraseQuery.Analyzer = repoIndexerAnalyzer
keywordQuery = phraseQuery
}
var indexerQuery query.Query
if len(repoIDs) > 0 { if len(repoIDs) > 0 {
var repoQueries = make([]query.Query, 0, len(repoIDs)) var repoQueries = make([]query.Query, 0, len(repoIDs))
for _, repoID := range repoIDs { for _, repoID := range repoIDs {
@ -294,10 +305,10 @@ func (b *BleveIndexer) Search(repoIDs []int64, language, keyword string, page, p
indexerQuery = bleve.NewConjunctionQuery( indexerQuery = bleve.NewConjunctionQuery(
bleve.NewDisjunctionQuery(repoQueries...), bleve.NewDisjunctionQuery(repoQueries...),
phraseQuery, keywordQuery,
) )
} else { } else {
indexerQuery = phraseQuery indexerQuery = keywordQuery
} }
// Save for reuse without language filter // Save for reuse without language filter

View File

@ -27,6 +27,10 @@ import (
const ( const (
esRepoIndexerLatestVersion = 1 esRepoIndexerLatestVersion = 1
// multi-match-types, currently only 2 types are used
// Reference: https://www.elastic.co/guide/en/elasticsearch/reference/7.0/query-dsl-multi-match-query.html#multi-match-types
esMultiMatchTypeBestFields = "best_fields"
esMultiMatchTypePhrasePrefix = "phrase_prefix"
) )
var ( var (
@ -330,8 +334,13 @@ func extractAggs(searchResult *elastic.SearchResult) []*SearchResultLanguages {
} }
// Search searches for codes and language stats by given conditions. // Search searches for codes and language stats by given conditions.
func (b *ElasticSearchIndexer) Search(repoIDs []int64, language, keyword string, page, pageSize int) (int64, []*SearchResult, []*SearchResultLanguages, error) { func (b *ElasticSearchIndexer) Search(repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error) {
kwQuery := elastic.NewMultiMatchQuery(keyword, "content") searchType := esMultiMatchTypeBestFields
if isMatch {
searchType = esMultiMatchTypePhrasePrefix
}
kwQuery := elastic.NewMultiMatchQuery(keyword, "content").Type(searchType)
query := elastic.NewBoolQuery() query := elastic.NewBoolQuery()
query = query.Must(kwQuery) query = query.Must(kwQuery)
if len(repoIDs) > 0 { if len(repoIDs) > 0 {

View File

@ -43,7 +43,7 @@ type SearchResultLanguages struct {
type Indexer interface { type Indexer interface {
Index(repo *models.Repository, sha string, changes *repoChanges) error Index(repo *models.Repository, sha string, changes *repoChanges) error
Delete(repoID int64) error Delete(repoID int64) error
Search(repoIDs []int64, language, keyword string, page, pageSize int) (int64, []*SearchResult, []*SearchResultLanguages, error) Search(repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error)
Close() Close()
} }

View File

@ -64,7 +64,7 @@ func testIndexer(name string, t *testing.T, indexer Indexer) {
for _, kw := range keywords { for _, kw := range keywords {
t.Run(kw.Keyword, func(t *testing.T) { t.Run(kw.Keyword, func(t *testing.T) {
total, res, langs, err := indexer.Search(kw.RepoIDs, "", kw.Keyword, 1, 10) total, res, langs, err := indexer.Search(kw.RepoIDs, "", kw.Keyword, 1, 10, false)
assert.NoError(t, err) assert.NoError(t, err)
assert.EqualValues(t, len(kw.IDs), total) assert.EqualValues(t, len(kw.IDs), total)
assert.EqualValues(t, kw.Langs, len(langs)) assert.EqualValues(t, kw.Langs, len(langs))

View File

@ -106,12 +106,12 @@ func searchResult(result *SearchResult, startIndex, endIndex int) (*Result, erro
} }
// PerformSearch perform a search on a repository // PerformSearch perform a search on a repository
func PerformSearch(repoIDs []int64, language, keyword string, page, pageSize int) (int, []*Result, []*SearchResultLanguages, error) { func PerformSearch(repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int, []*Result, []*SearchResultLanguages, error) {
if len(keyword) == 0 { if len(keyword) == 0 {
return 0, nil, nil, nil return 0, nil, nil, nil
} }
total, results, resultLanguages, err := indexer.Search(repoIDs, language, keyword, page, pageSize) total, results, resultLanguages, err := indexer.Search(repoIDs, language, keyword, page, pageSize, isMatch)
if err != nil { if err != nil {
return 0, nil, nil, err return 0, nil, nil, err
} }

View File

@ -73,12 +73,12 @@ func (w *wrappedIndexer) Delete(repoID int64) error {
return indexer.Delete(repoID) return indexer.Delete(repoID)
} }
func (w *wrappedIndexer) Search(repoIDs []int64, language, keyword string, page, pageSize int) (int64, []*SearchResult, []*SearchResultLanguages, error) { func (w *wrappedIndexer) Search(repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error) {
indexer, err := w.get() indexer, err := w.get()
if err != nil { if err != nil {
return 0, nil, nil, err return 0, nil, nil, err
} }
return indexer.Search(repoIDs, language, keyword, page, pageSize) return indexer.Search(repoIDs, language, keyword, page, pageSize, isMatch)
} }

View File

@ -237,6 +237,8 @@ users = Users
organizations = Organizations organizations = Organizations
search = Search search = Search
code = Code code = Code
search.fuzzy = Fuzzy
search.match = Match
repo_no_results = No matching repositories found. repo_no_results = No matching repositories found.
user_no_results = No matching users found. user_no_results = No matching users found.
org_no_results = No matching organizations found. org_no_results = No matching organizations found.
@ -1462,6 +1464,8 @@ activity.git_stats_deletion_n = %d deletions
search = Search search = Search
search.search_repo = Search repository search.search_repo = Search repository
search.fuzzy = Fuzzy
search.match = Match
search.results = Search results for "%s" in <a href="%s">%s</a> search.results = Search results for "%s" in <a href="%s">%s</a>
settings = Settings settings = Settings

View File

@ -299,6 +299,9 @@ func ExploreCode(ctx *context.Context) {
page = 1 page = 1
} }
queryType := strings.TrimSpace(ctx.Query("t"))
isMatch := queryType == "match"
var ( var (
repoIDs []int64 repoIDs []int64
err error err error
@ -342,14 +345,14 @@ func ExploreCode(ctx *context.Context) {
ctx.Data["RepoMaps"] = rightRepoMap ctx.Data["RepoMaps"] = rightRepoMap
total, searchResults, searchResultLanguages, err = code_indexer.PerformSearch(repoIDs, language, keyword, page, setting.UI.RepoSearchPagingNum) total, searchResults, searchResultLanguages, err = code_indexer.PerformSearch(repoIDs, language, keyword, page, setting.UI.RepoSearchPagingNum, isMatch)
if err != nil { if err != nil {
ctx.ServerError("SearchResults", err) ctx.ServerError("SearchResults", err)
return return
} }
// if non-login user or isAdmin, no need to check UnitTypeCode // if non-login user or isAdmin, no need to check UnitTypeCode
} else if (ctx.User == nil && len(repoIDs) > 0) || isAdmin { } else if (ctx.User == nil && len(repoIDs) > 0) || isAdmin {
total, searchResults, searchResultLanguages, err = code_indexer.PerformSearch(repoIDs, language, keyword, page, setting.UI.RepoSearchPagingNum) total, searchResults, searchResultLanguages, err = code_indexer.PerformSearch(repoIDs, language, keyword, page, setting.UI.RepoSearchPagingNum, isMatch)
if err != nil { if err != nil {
ctx.ServerError("SearchResults", err) ctx.ServerError("SearchResults", err)
return return
@ -380,6 +383,7 @@ func ExploreCode(ctx *context.Context) {
ctx.Data["Keyword"] = keyword ctx.Data["Keyword"] = keyword
ctx.Data["Language"] = language ctx.Data["Language"] = language
ctx.Data["queryType"] = queryType
ctx.Data["SearchResults"] = searchResults ctx.Data["SearchResults"] = searchResults
ctx.Data["SearchResultLanguages"] = searchResultLanguages ctx.Data["SearchResultLanguages"] = searchResultLanguages
ctx.Data["RequireHighlightJS"] = true ctx.Data["RequireHighlightJS"] = true

View File

@ -28,14 +28,18 @@ func Search(ctx *context.Context) {
if page <= 0 { if page <= 0 {
page = 1 page = 1
} }
queryType := strings.TrimSpace(ctx.Query("t"))
isMatch := queryType == "match"
total, searchResults, searchResultLanguages, err := code_indexer.PerformSearch([]int64{ctx.Repo.Repository.ID}, total, searchResults, searchResultLanguages, err := code_indexer.PerformSearch([]int64{ctx.Repo.Repository.ID},
language, keyword, page, setting.UI.RepoSearchPagingNum) language, keyword, page, setting.UI.RepoSearchPagingNum, isMatch)
if err != nil { if err != nil {
ctx.ServerError("SearchResults", err) ctx.ServerError("SearchResults", err)
return return
} }
ctx.Data["Keyword"] = keyword ctx.Data["Keyword"] = keyword
ctx.Data["Language"] = language ctx.Data["Language"] = language
ctx.Data["queryType"] = queryType
ctx.Data["SourcePath"] = setting.AppSubURL + "/" + ctx.Data["SourcePath"] = setting.AppSubURL + "/" +
path.Join(ctx.Repo.Repository.Owner.Name, ctx.Repo.Repository.Name) path.Join(ctx.Repo.Repository.Owner.Name, ctx.Repo.Repository.Name)
ctx.Data["SearchResults"] = searchResults ctx.Data["SearchResults"] = searchResults

View File

@ -5,9 +5,19 @@
<form class="ui form ignore-dirty" style="max-width: 100%"> <form class="ui form ignore-dirty" style="max-width: 100%">
<input type="hidden" name="tab" value="{{$.TabName}}"> <input type="hidden" name="tab" value="{{$.TabName}}">
<div class="ui fluid action input"> <div class="ui fluid action input">
<div class="twelve wide field">
<input name="q" value="{{.Keyword}}" placeholder="{{.i18n.Tr "explore.search"}}..." autofocus> <input name="q" value="{{.Keyword}}" placeholder="{{.i18n.Tr "explore.search"}}..." autofocus>
</div>
<div class="two wide field">
<select name="t">
<option value="">{{.i18n.Tr "explore.search.fuzzy"}}</option>
<option value="match" {{if eq .queryType "match"}}selected{{end}}>{{.i18n.Tr "explore.search.match"}}</option>
</select>
</div>
<div class="three field">
<button class="ui blue button">{{.i18n.Tr "explore.search"}}</button> <button class="ui blue button">{{.i18n.Tr "explore.search"}}</button>
</div> </div>
</div>
</form> </form>
<div class="ui divider"></div> <div class="ui divider"></div>
@ -18,7 +28,7 @@
</h3> </h3>
<div class="df ac fw"> <div class="df ac fw">
{{range $term := .SearchResultLanguages}} {{range $term := .SearchResultLanguages}}
<a class="ui text-label df ac mr-1 my-1 {{if eq $.Language $term.Language}}primary {{end}}basic label" href="{{AppSubUrl}}/explore/code?q={{$.Keyword}}{{if ne $.Language $term.Language}}&l={{$term.Language}}{{end}}"> <a class="ui text-label df ac mr-1 my-1 {{if eq $.Language $term.Language}}primary {{end}}basic label" href="{{AppSubUrl}}/explore/code?q={{$.Keyword}}{{if ne $.Language $term.Language}}&l={{$term.Language}}{{end}}{{if ne $.queryType ""}}&t={{$.queryType}}{{end}}">
<i class="color-icon mr-3" style="background-color: {{$term.Color}}"></i> <i class="color-icon mr-3" style="background-color: {{$term.Color}}"></i>
{{$term.Language}} {{$term.Language}}
<div class="detail">{{$term.Count}}</div> <div class="detail">{{$term.Count}}</div>
@ -62,4 +72,3 @@
</div> </div>
</div> </div>
{{template "base/footer" .}} {{template "base/footer" .}}

View File

@ -5,10 +5,20 @@
<div class="ui repo-search"> <div class="ui repo-search">
<form class="ui form ignore-dirty" method="get"> <form class="ui form ignore-dirty" method="get">
<div class="ui fluid action input"> <div class="ui fluid action input">
<input name="q" value="{{.Keyword}}" placeholder="{{.i18n.Tr "repo.search.search_repo"}}"> <div class="twelve wide field">
<button class="ui button" type="submit"> <input name="q" value="{{.Keyword}}" placeholder="{{.i18n.Tr "repo.search.search_repo"}}">
<i class="icon df ac jc">{{svg "octicon-search" 16}}</i> </div>
</button> <div class="two wide field">
<select name="t">
<option value="">{{.i18n.Tr "repo.search.fuzzy"}}</option>
<option value="match" {{if eq .queryType "match"}}selected{{end}}>{{.i18n.Tr "repo.search.match"}}</option>
</select>
</div>
<div class="three field">
<button class="ui button" type="submit">
<i class="icon df ac jc">{{svg "octicon-search" 16}}</i>
</button>
</div>
</div> </div>
</form> </form>
</div> </div>
@ -18,7 +28,7 @@
</h3> </h3>
<div class="df ac fw"> <div class="df ac fw">
{{range $term := .SearchResultLanguages}} {{range $term := .SearchResultLanguages}}
<a class="ui text-label df ac mr-1 my-1 {{if eq $.Language $term.Language}}primary {{end}}basic label" href="{{EscapePound $.SourcePath}}/search?q={{$.Keyword}}{{if ne $.Language $term.Language}}&l={{$term.Language}}{{end}}"> <a class="ui text-label df ac mr-1 my-1 {{if eq $.Language $term.Language}}primary {{end}}basic label" href="{{EscapePound $.SourcePath}}/search?q={{$.Keyword}}{{if ne $.Language $term.Language}}&l={{$term.Language}}{{end}}{{if ne $.queryType ""}}&t={{$.queryType}}{{end}}">
<i class="color-icon mr-3" style="background-color: {{$term.Color}}"></i> <i class="color-icon mr-3" style="background-color: {{$term.Color}}"></i>
{{$term.Language}} {{$term.Language}}
<div class="detail">{{$term.Count}}</div> <div class="detail">{{$term.Count}}</div>