diff --git a/services/search/pkg/bleve/batch.go b/services/search/pkg/bleve/batch.go index a6966a8a02..5def54347a 100644 --- a/services/search/pkg/bleve/batch.go +++ b/services/search/pkg/bleve/batch.go @@ -45,10 +45,12 @@ func (b *Batch) Upsert(id string, r search.Resource) error { // type-specific adaptations via the mapping package) and appends it to the // batch under id. func (b *Batch) indexResource(id string, r search.Resource) error { - doc, err := mapping.PrepareForIndex(r, r.SearchFieldOverrides()) + overrides := r.SearchFieldOverrides() + doc, err := mapping.PrepareForIndex(r, overrides) if err != nil { return err } + addGeohashValues(doc, overrides) return b.batch.Index(id, doc) } diff --git a/services/search/pkg/bleve/geo_verify_test.go b/services/search/pkg/bleve/geo_verify_test.go index 2bfb0d6a69..d1f465c750 100644 --- a/services/search/pkg/bleve/geo_verify_test.go +++ b/services/search/pkg/bleve/geo_verify_test.go @@ -7,12 +7,44 @@ import ( . "github.com/onsi/gomega" libregraph "github.com/opencloud-eu/libre-graph-api-go" + "github.com/opencloud-eu/opencloud/pkg/log" "github.com/opencloud-eu/opencloud/services/search/pkg/bleve" "github.com/opencloud-eu/opencloud/services/search/pkg/content" "github.com/opencloud-eu/opencloud/services/search/pkg/mapping" + bleveQuery "github.com/opencloud-eu/opencloud/services/search/pkg/query/bleve" "github.com/opencloud-eu/opencloud/services/search/pkg/search" ) +var _ = Describe("Location geohash sibling", func() { + It("indexes one depth-tagged term per precision", func() { + idxMapping, err := bleve.NewMapping() + Expect(err).ToNot(HaveOccurred()) + idx, err := bleveSearch.NewMemOnly(idxMapping) + Expect(err).ToNot(HaveOccurred()) + eng := bleve.NewBackend(idx, bleveQuery.DefaultCreator, log.Logger{}) + + // the geohash is written by the batch, not by PrepareForIndex + lon, lat := 10.40744, 57.64911 + r := search.Resource{ + ID: "x", + Document: content.Document{Name: "team.jpg", Location: &libregraph.GeoCoordinates{Longitude: &lon, Latitude: &lat}}, + } + Expect(eng.Upsert(r.ID, r)).To(Succeed()) + + req := bleveSearch.NewSearchRequest(bleveSearch.NewMatchAllQuery()) + req.Size = 0 + fr := bleveSearch.NewFacetRequest("location_geohash", 10) + fr.TermPrefix = "5/" + req.AddFacet("cells", fr) + res, err := idx.Search(req) + Expect(err).ToNot(HaveOccurred()) + terms := res.Facets["cells"].Terms.Terms() + Expect(terms).To(HaveLen(1)) + Expect(terms[0].Term).To(Equal("5/u4pru")) + Expect(terms[0].Count).To(Equal(1)) + }) +}) + // geoFixture builds an in-memory bleve index with a single resource carrying // the given lon/lat/alt, indexed through the full bleve pipeline. func geoFixture(lon, lat, alt float64) bleveSearch.Index { diff --git a/services/search/pkg/bleve/geohash.go b/services/search/pkg/bleve/geohash.go new file mode 100644 index 0000000000..b6770165d7 --- /dev/null +++ b/services/search/pkg/bleve/geohash.go @@ -0,0 +1,113 @@ +package bleve + +import ( + "strings" + + "github.com/blevesearch/bleve/v2" + "github.com/blevesearch/bleve/v2/mapping" + + searchmapping "github.com/opencloud-eu/opencloud/services/search/pkg/mapping" +) + +// A geohash sibling of every geopoint field, bleve only: OpenSearch can +// bucket a geohash_grid aggregation on the geo_point itself, bleve cannot. +// The geohash is indexed with the geohash analyzer, one depth-tagged term per +// precision ("1/u", "2/u4", ...), so the terms with the prefix "/" +// are the cells at that precision. Nothing queries it yet (#3272). +const ( + geohashAnalyzer = "geohash" + geohashSuffix = "_geohash" + geohashPrecision = 12 + geohashBase32 = "0123456789bcdefghjkmnpqrstuvwxyz" +) + +// encodeGeohash matches Lucene/OpenSearch so both engines bucket into the +// same cells +func encodeGeohash(lat, lon float64, precision int) string { + latMin, latMax := -90.0, 90.0 + lonMin, lonMax := -180.0, 180.0 + var b strings.Builder + even := true + bit, ch := 0, 0 + for b.Len() < precision { + if even { + mid := (lonMin + lonMax) / 2 + if lon >= mid { + ch |= 1 << (4 - bit) + lonMin = mid + } else { + lonMax = mid + } + } else { + mid := (latMin + latMax) / 2 + if lat >= mid { + ch |= 1 << (4 - bit) + latMin = mid + } else { + latMax = mid + } + } + even = !even + if bit < 4 { + bit++ + } else { + b.WriteByte(geohashBase32[ch]) + bit, ch = 0, 0 + } + } + return b.String() +} + +// geopointFields yields the parent path and leaf name of every TypeGeopoint +// override, the same fields addGeopointSiblings gives a geopoint sibling +func geopointFields(overrides map[string]searchmapping.FieldOpts, fn func(parents []string, leaf string)) { + for key, opts := range overrides { + if opts.Type == searchmapping.TypeGeopoint { + parts := strings.Split(key, ".") + fn(parts[:len(parts)-1], parts[len(parts)-1]) + } + } +} + +// addGeohashFields maps a _geohash field next to the _geopoint +// field of every geopoint override +func addGeohashFields(dm *mapping.DocumentMapping, overrides map[string]searchmapping.FieldOpts) { + geopointFields(overrides, func(parents []string, leaf string) { + parent := dm + for _, p := range parents { + if parent = parent.Properties[p]; parent == nil { + return + } + } + fm := bleve.NewTextFieldMapping() + fm.Analyzer = geohashAnalyzer + fm.Store = false + fm.IncludeInAll = false + fm.IncludeTermVectors = false + parent.AddFieldMappingsAt(leaf+geohashSuffix, fm) + }) +} + +// addGeohashValues writes the geohash next to the {lat, lon} sibling of every +// geopoint override in a prepared document +func addGeohashValues(doc map[string]any, overrides map[string]searchmapping.FieldOpts) { + geopointFields(overrides, func(parents []string, leaf string) { + parent := doc + for _, p := range parents { + next, ok := parent[p].(map[string]any) + if !ok { + return + } + parent = next + } + obj, ok := parent[leaf+searchmapping.GeopointSuffix].(map[string]any) + if !ok { + return + } + lat, hasLat := obj["lat"].(float64) + lon, hasLon := obj["lon"].(float64) + if hasLat && hasLon { + parent[leaf+geohashSuffix] = encodeGeohash(lat, lon, geohashPrecision) + } + }) +} diff --git a/services/search/pkg/bleve/geohash_test.go b/services/search/pkg/bleve/geohash_test.go new file mode 100644 index 0000000000..22530b45ee --- /dev/null +++ b/services/search/pkg/bleve/geohash_test.go @@ -0,0 +1,45 @@ +package bleve + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + searchmapping "github.com/opencloud-eu/opencloud/services/search/pkg/mapping" +) + +var _ = Describe("geohash", func() { + // reference vector from the geohash spec, Lucene and OpenSearch agree + const refLat, refLon = 57.64911, 10.40744 + + It("matches the canonical vector", func() { + Expect(encodeGeohash(refLat, refLon, 11)).To(Equal("u4pruydqqvj")) + }) + + It("is a prefix code", func() { + full := encodeGeohash(refLat, refLon, 12) + for p := 1; p <= 12; p++ { + Expect(encodeGeohash(refLat, refLon, p)).To(Equal(full[:p])) + } + }) + + It("writes the geohash next to the geopoint sibling of every geopoint override", func() { + // journey.start is the dotted-path example of addGeopointSibling, broken + // has no usable lat/lon and gets no geohash, like it gets no sibling + overrides := map[string]searchmapping.FieldOpts{ + "location": {Type: searchmapping.TypeGeopoint}, + "journey.start": {Type: searchmapping.TypeGeopoint}, + "broken": {Type: searchmapping.TypeGeopoint}, + } + doc := map[string]any{ + "location_geopoint": map[string]any{"lat": refLat, "lon": refLon}, + "journey": map[string]any{ + "start_geopoint": map[string]any{"lat": 0.0, "lon": 0.0}, + }, + "broken_geopoint": map[string]any{"lat": "x"}, + } + addGeohashValues(doc, overrides) + Expect(doc["location_geohash"]).To(Equal(encodeGeohash(refLat, refLon, 12))) + Expect(doc["journey"].(map[string]any)["start_geohash"]).To(Equal("s00000000000")) + Expect(doc).ToNot(HaveKey("broken_geohash")) + }) +}) diff --git a/services/search/pkg/bleve/index.go b/services/search/pkg/bleve/index.go index e72754b814..2bd7074d95 100644 --- a/services/search/pkg/bleve/index.go +++ b/services/search/pkg/bleve/index.go @@ -184,6 +184,7 @@ func NewMapping() (mapping.IndexMapping, error) { if err != nil { return nil, err } + addGeohashFields(docMapping, overrides) indexMapping := bleve.NewIndexMapping() indexMapping.DefaultAnalyzer = keyword.Name @@ -225,12 +226,8 @@ func NewMapping() (mapping.IndexMapping, error) { if err != nil { return nil, err } - // geohash: every prefix is a depth-tagged term (1/u, 2/u4, ...), so a terms - // facet with TermPrefix "/" is a geohash grid at that precision. - // No field uses it yet. It is part of the v5 schema so that #3272 can add - // its geohash field additively: new fields reconcile at startup, a changed - // analysis block does not (classifyStoredMapping), so the names and the - // config below must not change. + // geohash: every prefix is a depth-tagged term (1/u, 2/u4, ...), one per + // precision (see geohash.go) err = indexMapping.AddCustomTokenizer("geohash_hierarchy", map[string]any{ "type": hierarchy.Name, "tag_depth": true, @@ -238,7 +235,7 @@ func NewMapping() (mapping.IndexMapping, error) { if err != nil { return nil, err } - err = indexMapping.AddCustomAnalyzer("geohash", map[string]any{ + err = indexMapping.AddCustomAnalyzer(geohashAnalyzer, map[string]any{ "type": custom.Name, "tokenizer": "geohash_hierarchy", }) diff --git a/services/search/pkg/bleve/testdata/mapping.golden.json b/services/search/pkg/bleve/testdata/mapping.golden.json index 73f6baa748..f9c9838a16 100644 --- a/services/search/pkg/bleve/testdata/mapping.golden.json +++ b/services/search/pkg/bleve/testdata/mapping.golden.json @@ -820,6 +820,18 @@ } } }, + "location_geohash": { + "enabled": true, + "dynamic": true, + "fields": [ + { + "type": "text", + "analyzer": "geohash", + "index": true, + "docvalues": true + } + ] + }, "location_geopoint": { "enabled": true, "dynamic": true,