Support Elasticsearch v8 #7

Merged
maxim.yanchuk merged 3 commits from es8 into master 2023-11-01 21:30:55 +00:00
13 changed files with 433 additions and 4 deletions
Showing only changes of commit bc3cec49c2 - Show all commits

View File

@ -158,6 +158,7 @@ The following aggregations are currently supported:
| `"string_stats"` | `StringStats()` | | `"string_stats"` | `StringStats()` |
| `"top_hits"` | `TopHits()` | | `"top_hits"` | `TopHits()` |
| `"terms"` | `TermsAgg()` | | `"terms"` | `TermsAgg()` |
| `"date_histogram"` | `DateHistogramAgg()` |
### Supported Top Level Options ### Supported Top Level Options

View File

@ -62,7 +62,14 @@ func TestAggregations(t *testing.T) {
"a complex, multi-aggregation, nested", "a complex, multi-aggregation, nested",
Aggregate( Aggregate(
NestedAgg("categories", "categories"). NestedAgg("categories", "categories").
Aggs(TermsAgg("type", "outdoors")), Aggs(
TermsAgg("type", "outdoors").Aggs(
DateHistogramAgg("time", "timestamp").
Fixedinterval("3m").MinDocCount(0).Aggs(
Sum("sumPeople", "people"),
),
),
),
FilterAgg("filtered", FilterAgg("filtered",
Term("type", "t-shirt")), Term("type", "t-shirt")),
), ),
@ -77,6 +84,22 @@ func TestAggregations(t *testing.T) {
"terms": map[string]interface{}{ "terms": map[string]interface{}{
"field": "outdoors", "field": "outdoors",
}, },
"aggs": map[string]interface{}{
"time": map[string]interface{}{
"date_histogram": map[string]interface{}{
"field": "timestamp",
"fixed_interval": "3m",
"min_doc_count": 0,
},
"aggs": map[string]interface{}{
"sumPeople": map[string]interface{}{
"sum": map[string]interface{}{
"field": "people",
},
},
},
},
},
}, },
}, },
}, },

View File

@ -110,3 +110,144 @@ func (agg *TermsAggregation) Map() map[string]interface{} {
return outerMap return outerMap
} }
//----------------------------------------------------------------------------//
// DateHistogramAggregation represents an aggregation of type "date_histogram", as described in
// https://www.elastic.co/guide/en/elasticsearch/reference/current/
// search-aggregations-bucket-datehistogram-aggregation.html
type DateHistogramAggregation struct {
name string
field string
calendarInterval string
fixedInterval string
format string
offset string
keyed *bool
minDocCount *uint64
missing string
order map[string]string
aggs []Aggregation
}
// DateHistogramAgg creates a new aggregation of type "date_histogram".
func DateHistogramAgg(name, field string) *DateHistogramAggregation {
return &DateHistogramAggregation{
name: name,
field: field,
}
}
// Name returns the name of the aggregation.
func (agg *DateHistogramAggregation) Name() string {
return agg.name
}
// Aggs sets sub-aggregations for the aggregation.
func (agg *DateHistogramAggregation) Aggs(aggs ...Aggregation) *DateHistogramAggregation {
agg.aggs = aggs
return agg
}
// CalendarInterval sets calendarInterval
func (agg *DateHistogramAggregation) CalendarInterval(interval string) *DateHistogramAggregation {
agg.calendarInterval = interval
return agg
}
// Fixedinterval sets fixedInterval
func (agg *DateHistogramAggregation) Fixedinterval(interval string) *DateHistogramAggregation {
agg.fixedInterval = interval
return agg
}
// Format sets format
func (agg *DateHistogramAggregation) Format(format string) *DateHistogramAggregation {
agg.format = format
return agg
}
// Offset sets offset
func (agg *DateHistogramAggregation) Offset(offset string) *DateHistogramAggregation {
agg.offset = offset
return agg
}
// Order sets the sort for terms agg
func (agg *DateHistogramAggregation) Order(order map[string]string) *DateHistogramAggregation {
agg.order = order
return agg
}
// Keyed sets keyed is true or false
func (agg *DateHistogramAggregation) Keyed(keyed bool) *DateHistogramAggregation {
agg.keyed = &keyed
return agg
}
// Missing sets missing value
func (agg *DateHistogramAggregation) Missing(missing string) *DateHistogramAggregation {
agg.missing = missing
return agg
}
// MinDocCount sets min doc count
func (agg *DateHistogramAggregation) MinDocCount(minDocCount uint64) *DateHistogramAggregation {
agg.minDocCount = &minDocCount
return agg
}
// Map returns a map representation of the aggregation, thus implementing the
// Mappable interface.
func (agg *DateHistogramAggregation) Map() map[string]interface{} {
innerMap := map[string]interface{}{
"field": agg.field,
}
if agg.calendarInterval != "" {
innerMap["calendar_interval"] = agg.calendarInterval
}
if agg.fixedInterval != "" {
innerMap["fixed_interval"] = agg.fixedInterval
}
if agg.format != "" {
innerMap["format"] = agg.format
}
if agg.offset != "" {
innerMap["offset"] = agg.offset
}
if agg.missing != "" {
innerMap["missing"] = agg.missing
}
if agg.minDocCount != nil {
innerMap["min_doc_count"] = agg.minDocCount
}
if agg.keyed != nil {
innerMap["keyed"] = *agg.keyed
}
if agg.order != nil {
innerMap["order"] = agg.order
}
outerMap := map[string]interface{}{
"date_histogram": innerMap,
}
if len(agg.aggs) > 0 {
subAggs := make(map[string]map[string]interface{})
for _, sub := range agg.aggs {
subAggs[sub.Name()] = sub.Map()
}
outerMap["aggs"] = subAggs
}
return outerMap
}

2
go.mod
View File

@ -3,7 +3,7 @@ module github.com/aquasecurity/esquery/v8
go 1.13 go 1.13
require ( require (
github.com/elastic/go-elasticsearch/v8 v8.4.0 github.com/elastic/go-elasticsearch/v8 v8.10.1
github.com/fatih/structs v1.1.0 github.com/fatih/structs v1.1.0
github.com/jgroeneveld/schema v1.0.0 // indirect github.com/jgroeneveld/schema v1.0.0 // indirect
github.com/jgroeneveld/trial v2.0.0+incompatible github.com/jgroeneveld/trial v2.0.0+incompatible

86
query_combined_fields.go Normal file
View File

@ -0,0 +1,86 @@
package esquery
import "github.com/fatih/structs"
type CombinedFieldsQuery struct {
params combinedFieldsParams
}
// Map returns a map representation of the query; implementing the
// Mappable interface.
func (q *CombinedFieldsQuery) Map() map[string]interface{} {
return map[string]interface{}{
"combined_fields": structs.Map(q.params),
}
}
type combinedFieldsParams struct {
Qry interface{} `structs:"query"`
Fields []string `structs:"fields"`
Boost float32 `structs:"boost,omitempty"`
AutoGenerate *bool `structs:"auto_generate_synonyms_phrase_query,omitempty"`
Op MatchOperator `structs:"operator,string,omitempty"`
MinMatch string `structs:"minimum_should_match,omitempty"`
ZeroTerms ZeroTerms `structs:"zero_terms_query,string,omitempty"`
}
// CombinedFields creates a new query of type "combined_fields"
func CombinedFields(simpleQuery interface{}) *CombinedFieldsQuery {
return newCombinedFields(simpleQuery)
}
func newCombinedFields(simpleQuery interface{}) *CombinedFieldsQuery {
return &CombinedFieldsQuery{
params: combinedFieldsParams{
Qry: simpleQuery,
},
}
}
// Query sets the data to find in the query's field (it is the "query" component
// of the query).
func (q *CombinedFieldsQuery) Query(data interface{}) *CombinedFieldsQuery {
q.params.Qry = data
return q
}
// Fields sets the fields used in the query
func (q *CombinedFieldsQuery) Fields(a ...string) *CombinedFieldsQuery {
q.params.Fields = append(q.params.Fields, a...)
return q
}
// AutoGenerateSynonymsPhraseQuery sets the "auto_generate_synonyms_phrase_query"
// boolean.
func (q *CombinedFieldsQuery) AutoGenerateSynonymsPhraseQuery(b bool) *CombinedFieldsQuery {
q.params.AutoGenerate = &b
return q
}
// Boost
func (q *CombinedFieldsQuery) Boost(l float32) *CombinedFieldsQuery {
q.params.Boost = l
return q
}
// Operator sets the boolean logic used to interpret text in the query value.
func (q *CombinedFieldsQuery) Operator(op MatchOperator) *CombinedFieldsQuery {
q.params.Op = op
return q
}
// MinimumShouldMatch sets the minimum number of clauses that must match for a
// document to be returned.
func (q *CombinedFieldsQuery) MinimumShouldMatch(s string) *CombinedFieldsQuery {
q.params.MinMatch = s
return q
}
// ZeroTermsQuery sets the "zero_terms_query" option to use. This indicates
// whether no documents are returned if the analyzer removes all tokens, such as
// when using a stop filter.
func (q *CombinedFieldsQuery) ZeroTermsQuery(s ZeroTerms) *CombinedFieldsQuery {
q.params.ZeroTerms = s
return q
}

View File

@ -0,0 +1,43 @@
package esquery
import (
"testing"
)
func TestCombinedFields(t *testing.T) {
runMapTests(t, []mapTest{
{
"simple combined_fields",
CombinedFields("value1").Fields("title"),
map[string]interface{}{
"combined_fields": map[string]interface{}{
"fields": []string{"title"},
"query": "value1",
},
},
},
{
"combined_fields all params",
CombinedFields("original").
Query("test").
Fields("title", "body").
AutoGenerateSynonymsPhraseQuery(true).
Boost(6.4).
Operator(OperatorAnd).
MinimumShouldMatch("3<90%").
ZeroTermsQuery(ZeroTermsAll),
map[string]interface{}{
"combined_fields": map[string]interface{}{
"auto_generate_synonyms_phrase_query": true,
"boost": 6.4,
"minimum_should_match": "3<90%",
"operator": "AND",
"zero_terms_query": "all",
"query": "test",
"fields": []string{"title", "body"},
},
},
},
})
}

40
query_joining.go Normal file
View File

@ -0,0 +1,40 @@
package esquery
// NestedQuery represents a query of type nested as described in:
// https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-nested-query.html
type NestedQuery struct {
path string
query Mappable
scoreMode string
ignoreUnmapped bool
}
func Nested(path string, query Mappable) *NestedQuery {
return &NestedQuery{
path: path,
query: query,
}
}
func (n *NestedQuery) ScoreMode(mode string) *NestedQuery {
n.scoreMode = mode
return n
}
func (n *NestedQuery) IgnoreUnmapped(val bool) *NestedQuery {
n.ignoreUnmapped = val
return n
}
// Map returns a map representation of the query, thus implementing the
// Mappable interface.
func (n *NestedQuery) Map() map[string]interface{} {
innerMap := map[string]interface{}{"path": n.path, "query": n.query.Map()}
if n.scoreMode != "" {
innerMap["score_mode"] = n.scoreMode
}
if n.ignoreUnmapped == true {
innerMap["ignore_unmapped"] = n.ignoreUnmapped
}
return map[string]interface{}{"nested": innerMap}
}

22
query_joining_test.go Normal file
View File

@ -0,0 +1,22 @@
package esquery
import (
"testing"
)
func TestNested(t *testing.T) {
runMapTests(t, []mapTest{
{
"Nested Query",
Nested("dns_values", Term("dns_values.type", "A")).ScoreMode("max").IgnoreUnmapped(true),
map[string]interface{}{
"nested": map[string]interface{}{
"path": "dns_values",
"query": Term("dns_values.type", "A").Map(),
"score_mode": "max",
"ignore_unmapped": true,
},
},
},
})
}

View File

@ -71,6 +71,7 @@ type matchParams struct {
MinMatch string `structs:"minimum_should_match,omitempty"` MinMatch string `structs:"minimum_should_match,omitempty"`
ZeroTerms ZeroTerms `structs:"zero_terms_query,string,omitempty"` ZeroTerms ZeroTerms `structs:"zero_terms_query,string,omitempty"`
Slp uint16 `structs:"slop,omitempty"` // only relevant for match_phrase query Slp uint16 `structs:"slop,omitempty"` // only relevant for match_phrase query
Boost float32 `structs:"boost,omitempty"`
} }
// Match creates a new query of type "match" with the provided field name. // Match creates a new query of type "match" with the provided field name.
@ -202,6 +203,12 @@ func (q *MatchQuery) ZeroTermsQuery(s ZeroTerms) *MatchQuery {
return q return q
} }
// Boost sets the boost value of the query.
func (a *MatchQuery) Boost(b float32) *MatchQuery {
a.params.Boost = b
return a
}
// MatchOperator is an enumeration type representing supported values for a // MatchOperator is an enumeration type representing supported values for a
// match query's "operator" parameter. // match query's "operator" parameter.
type MatchOperator uint8 type MatchOperator uint8

View File

@ -17,6 +17,18 @@ func TestMatch(t *testing.T) {
}, },
}, },
}, },
{
"simple match",
Match("title", "sample text").Boost(50),
map[string]interface{}{
"match": map[string]interface{}{
"title": map[string]interface{}{
"query": "sample text",
"boost": 50,
},
},
},
},
{ {
"match with more params", "match with more params",
Match("issue_number").Query(16).Transpositions(false).MaxExpansions(32).Operator(OperatorAnd), Match("issue_number").Query(16).Transpositions(false).MaxExpansions(32).Operator(OperatorAnd),

View File

@ -18,7 +18,7 @@ func (q *MultiMatchQuery) Map() map[string]interface{} {
type multiMatchParams struct { type multiMatchParams struct {
Qry interface{} `structs:"query"` Qry interface{} `structs:"query"`
Fields []string `structs:"fields"` Fields []string `structs:"fields,omitempty"`
Type MultiMatchType `structs:"type,string,omitempty"` Type MultiMatchType `structs:"type,string,omitempty"`
TieBrk float32 `structs:"tie_breaker,omitempty"` TieBrk float32 `structs:"tie_breaker,omitempty"`
Boost float32 `structs:"boost,omitempty"` Boost float32 `structs:"boost,omitempty"`

View File

@ -518,3 +518,45 @@ func (q TermsSetQuery) Map() map[string]interface{} {
}, },
} }
} }
// geoFilterParams represents a query of type "geo_distance", as described in:
// https://www.elastic.co/guide/en/elasticsearch/reference/7.17/query-dsl-geo-distance-query.html
type GeoFilter struct {
params geoFilterParams
filed string
}
func GeoFilterFunc(distance string, MiddleCentroid []float64, filed string) *GeoFilter {
return &GeoFilter{
params: geoFilterParams{
Distance: distance,
MiddleCentroid: MiddleCentroid,
},
filed: filed,
}
}
type geoFilterParams struct {
Distance string `structs:"distance,omitempty"`
MiddleCentroid []float64 `structs:"location,omitempty"`
}
func (g *GeoFilter) Distance(distance string) *GeoFilter {
g.params.Distance = distance
return g
}
func (g *GeoFilter) MiddleCentroid(middleCentroid []float64) *GeoFilter {
g.params.MiddleCentroid = middleCentroid
return g
}
func (g *GeoFilter) Map() map[string]interface{} {
m := structs.Map(g.params)
m[g.filed] = m["location"]
delete(m, "location")
response := map[string]interface{}{
"geo_distance": m}
return response
}

View File

@ -6,7 +6,6 @@ import (
"fmt" "fmt"
"time" "time"
"github.com/elastic/go-elasticsearch/v8"
"github.com/elastic/go-elasticsearch/v8/esapi" "github.com/elastic/go-elasticsearch/v8/esapi"
) )
@ -25,6 +24,7 @@ type SearchRequest struct {
size *uint64 size *uint64
sort Sort sort Sort
source Source source Source
collapse map[string]interface{}
timeout *time.Duration timeout *time.Duration
} }
@ -39,6 +39,14 @@ func (req *SearchRequest) Query(q Mappable) *SearchRequest {
return req return req
} }
// Collapse sets one field to collapse for the request.
func (req *SearchRequest) Collapse(field string) *SearchRequest {
req.collapse = map[string]interface{}{
"field": field,
}
return req
}
// Aggs sets one or more aggregations for the request. // Aggs sets one or more aggregations for the request.
func (req *SearchRequest) Aggs(aggs ...Aggregation) *SearchRequest { func (req *SearchRequest) Aggs(aggs ...Aggregation) *SearchRequest {
req.aggs = append(req.aggs, aggs...) req.aggs = append(req.aggs, aggs...)
@ -152,6 +160,10 @@ func (req *SearchRequest) Map() map[string]interface{} {
m["search_after"] = req.searchAfter m["search_after"] = req.searchAfter
} }
if req.collapse != nil {
m["collapse"] = req.collapse
}
source := req.source.Map() source := req.source.Map()
if len(source) > 0 { if len(source) > 0 {
m["_source"] = source m["_source"] = source