feat(search): geohash support in bleve

services/search/pkg/bleve/batch.go

@@ -45,10 +45,12 @@ func (b *Batch) Upsert(id string, r search.Resource) error {
 // type-specific adaptations via the mapping package) and appends it to the
 // batch under id.
 func (b *Batch) indexResource(id string, r search.Resource) error {
-	doc, err := mapping.PrepareForIndex(r, r.SearchFieldOverrides())
+	overrides := r.SearchFieldOverrides()
+	doc, err := mapping.PrepareForIndex(r, overrides)
 	if err != nil {
 		return err
 	}
+	addGeohashValues(doc, overrides)
 	return b.batch.Index(id, doc)
 }
 

services/search/pkg/bleve/geo_verify_test.go

@@ -7,12 +7,44 @@ import (
 	. "github.com/onsi/gomega"
 	libregraph "github.com/opencloud-eu/libre-graph-api-go"
 
+	"github.com/opencloud-eu/opencloud/pkg/log"
 	"github.com/opencloud-eu/opencloud/services/search/pkg/bleve"
 	"github.com/opencloud-eu/opencloud/services/search/pkg/content"
 	"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
+	bleveQuery "github.com/opencloud-eu/opencloud/services/search/pkg/query/bleve"
 	"github.com/opencloud-eu/opencloud/services/search/pkg/search"
 )
 
+var _ = Describe("Location geohash sibling", func() {
+	It("indexes one depth-tagged term per precision", func() {
+		idxMapping, err := bleve.NewMapping()
+		Expect(err).ToNot(HaveOccurred())
+		idx, err := bleveSearch.NewMemOnly(idxMapping)
+		Expect(err).ToNot(HaveOccurred())
+		eng := bleve.NewBackend(idx, bleveQuery.DefaultCreator, log.Logger{})
+
+		// the geohash is written by the batch, not by PrepareForIndex
+		lon, lat := 10.40744, 57.64911
+		r := search.Resource{
+			ID:       "x",
+			Document: content.Document{Name: "team.jpg", Location: &libregraph.GeoCoordinates{Longitude: &lon, Latitude: &lat}},
+		}
+		Expect(eng.Upsert(r.ID, r)).To(Succeed())
+
+		req := bleveSearch.NewSearchRequest(bleveSearch.NewMatchAllQuery())
+		req.Size = 0
+		fr := bleveSearch.NewFacetRequest("location_geohash", 10)
+		fr.TermPrefix = "5/"
+		req.AddFacet("cells", fr)
+		res, err := idx.Search(req)
+		Expect(err).ToNot(HaveOccurred())
+		terms := res.Facets["cells"].Terms.Terms()
+		Expect(terms).To(HaveLen(1))
+		Expect(terms[0].Term).To(Equal("5/u4pru"))
+		Expect(terms[0].Count).To(Equal(1))
+	})
+})
+
 // geoFixture builds an in-memory bleve index with a single resource carrying
 // the given lon/lat/alt, indexed through the full bleve pipeline.
 func geoFixture(lon, lat, alt float64) bleveSearch.Index {

services/search/pkg/bleve/geohash.go

@@ -0,0 +1,113 @@
+package bleve
+
+import (
+	"strings"
+
+	"github.com/blevesearch/bleve/v2"
+	"github.com/blevesearch/bleve/v2/mapping"
+
+	searchmapping "github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
+)
+
+// A geohash sibling of every geopoint field, bleve only: OpenSearch can
+// bucket a geohash_grid aggregation on the geo_point itself, bleve cannot.
+// The geohash is indexed with the geohash analyzer, one depth-tagged term per
+// precision ("1/u", "2/u4", ...), so the terms with the prefix "<precision>/"
+// are the cells at that precision. Nothing queries it yet (#3272).
+const (
+	geohashAnalyzer  = "geohash"
+	geohashSuffix    = "_geohash"
+	geohashPrecision = 12
+	geohashBase32    = "0123456789bcdefghjkmnpqrstuvwxyz"
+)
+
+// encodeGeohash matches Lucene/OpenSearch so both engines bucket into the
+// same cells
+func encodeGeohash(lat, lon float64, precision int) string {
+	latMin, latMax := -90.0, 90.0
+	lonMin, lonMax := -180.0, 180.0
+	var b strings.Builder
+	even := true
+	bit, ch := 0, 0
+	for b.Len() < precision {
+		if even {
+			mid := (lonMin + lonMax) / 2
+			if lon >= mid {
+				ch |= 1 << (4 - bit)
+				lonMin = mid
+			} else {
+				lonMax = mid
+			}
+		} else {
+			mid := (latMin + latMax) / 2
+			if lat >= mid {
+				ch |= 1 << (4 - bit)
+				latMin = mid
+			} else {
+				latMax = mid
+			}
+		}
+		even = !even
+		if bit < 4 {
+			bit++
+		} else {
+			b.WriteByte(geohashBase32[ch])
+			bit, ch = 0, 0
+		}
+	}
+	return b.String()
+}
+
+// geopointFields yields the parent path and leaf name of every TypeGeopoint
+// override, the same fields addGeopointSiblings gives a geopoint sibling
+func geopointFields(overrides map[string]searchmapping.FieldOpts, fn func(parents []string, leaf string)) {
+	for key, opts := range overrides {
+		if opts.Type == searchmapping.TypeGeopoint {
+			parts := strings.Split(key, ".")
+			fn(parts[:len(parts)-1], parts[len(parts)-1])
+		}
+	}
+}
+
+// addGeohashFields maps a <name>_geohash field next to the <name>_geopoint
+// field of every geopoint override
+func addGeohashFields(dm *mapping.DocumentMapping, overrides map[string]searchmapping.FieldOpts) {
+	geopointFields(overrides, func(parents []string, leaf string) {
+		parent := dm
+		for _, p := range parents {
+			if parent = parent.Properties[p]; parent == nil {
+				return
+			}
+		}
+		fm := bleve.NewTextFieldMapping()
+		fm.Analyzer = geohashAnalyzer
+		fm.Store = false
+		fm.IncludeInAll = false
+		fm.IncludeTermVectors = false
+		parent.AddFieldMappingsAt(leaf+geohashSuffix, fm)
+	})
+}
+
+// addGeohashValues writes the geohash next to the {lat, lon} sibling of every
+// geopoint override in a prepared document
+func addGeohashValues(doc map[string]any, overrides map[string]searchmapping.FieldOpts) {
+	geopointFields(overrides, func(parents []string, leaf string) {
+		parent := doc
+		for _, p := range parents {
+			next, ok := parent[p].(map[string]any)
+			if !ok {
+				return
+			}
+			parent = next
+		}
+		obj, ok := parent[leaf+searchmapping.GeopointSuffix].(map[string]any)
+		if !ok {
+			return
+		}
+		lat, hasLat := obj["lat"].(float64)
+		lon, hasLon := obj["lon"].(float64)
+		if hasLat && hasLon {
+			parent[leaf+geohashSuffix] = encodeGeohash(lat, lon, geohashPrecision)
+		}
+	})
+}

services/search/pkg/bleve/geohash_test.go

@@ -0,0 +1,45 @@
+package bleve
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	searchmapping "github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
+)
+
+var _ = Describe("geohash", func() {
+	// reference vector from the geohash spec, Lucene and OpenSearch agree
+	const refLat, refLon = 57.64911, 10.40744
+
+	It("matches the canonical vector", func() {
+		Expect(encodeGeohash(refLat, refLon, 11)).To(Equal("u4pruydqqvj"))
+	})
+
+	It("is a prefix code", func() {
+		full := encodeGeohash(refLat, refLon, 12)
+		for p := 1; p <= 12; p++ {
+			Expect(encodeGeohash(refLat, refLon, p)).To(Equal(full[:p]))
+		}
+	})
+
+	It("writes the geohash next to the geopoint sibling of every geopoint override", func() {
+		// journey.start is the dotted-path example of addGeopointSibling, broken
+		// has no usable lat/lon and gets no geohash, like it gets no sibling
+		overrides := map[string]searchmapping.FieldOpts{
+			"location":      {Type: searchmapping.TypeGeopoint},
+			"journey.start": {Type: searchmapping.TypeGeopoint},
+			"broken":        {Type: searchmapping.TypeGeopoint},
+		}
+		doc := map[string]any{
+			"location_geopoint": map[string]any{"lat": refLat, "lon": refLon},
+			"journey": map[string]any{
+				"start_geopoint": map[string]any{"lat": 0.0, "lon": 0.0},
+			},
+			"broken_geopoint": map[string]any{"lat": "x"},
+		}
+		addGeohashValues(doc, overrides)
+		Expect(doc["location_geohash"]).To(Equal(encodeGeohash(refLat, refLon, 12)))
+		Expect(doc["journey"].(map[string]any)["start_geohash"]).To(Equal("s00000000000"))
+		Expect(doc).ToNot(HaveKey("broken_geohash"))
+	})
+})

services/search/pkg/bleve/index.go

@@ -184,6 +184,7 @@ func NewMapping() (mapping.IndexMapping, error) {
 	if err != nil {
 		return nil, err
 	}
+	addGeohashFields(docMapping, overrides)
 
 	indexMapping := bleve.NewIndexMapping()
 	indexMapping.DefaultAnalyzer = keyword.Name
@@ -225,20 +226,16 @@ func NewMapping() (mapping.IndexMapping, error) {
 	if err != nil {
 		return nil, err
 	}
-	// geohash: every prefix is a depth-tagged term (1/u, 2/u4, ...), so a terms
-	// facet with TermPrefix "<precision>/" is a geohash grid at that precision.
-	// No field uses it yet. It is part of the v5 schema so that #3272 can add
-	// its geohash field additively: new fields reconcile at startup, a changed
-	// analysis block does not (classifyStoredMapping), so the names and the
-	// config below must not change.
+	// geohash: every prefix is a depth-tagged term (1/u, 2/u4, ...), one per
+	// precision (see geohash.go)
 	err = indexMapping.AddCustomTokenizer("geohash_hierarchy", map[string]any{
 		"type":      hierarchy.Name,
 		"tag_depth": true,
 	})
 	if err != nil {
 		return nil, err
 	}
-	err = indexMapping.AddCustomAnalyzer("geohash", map[string]any{
+	err = indexMapping.AddCustomAnalyzer(geohashAnalyzer, map[string]any{
 		"type":      custom.Name,
 		"tokenizer": "geohash_hierarchy",
 	})

services/search/pkg/bleve/testdata/mapping.golden.json

@@ -820,6 +820,18 @@
           }
         }
       },
+      "location_geohash": {
+        "enabled": true,
+        "dynamic": true,
+        "fields": [
+          {
+            "type": "text",
+            "analyzer": "geohash",
+            "index": true,
+            "docvalues": true
+          }
+        ]
+      },
       "location_geopoint": {
         "enabled": true,
         "dynamic": true,