fix(search): hierarchy tokenizer for bleve path fields

services/search/pkg/bleve/backend.go

@@ -75,14 +75,10 @@ func (b *Backend) Search(_ context.Context, sir *searchService.SearchIndexReques
 			},
 		)
 		// Scope below the space root: restrict at query level so totals and
-		// paging respect the path too. Path is a case-preserving keyword
-		// (paths act as references, /Foo and /foo are distinct), so the exact
-		// folder or the folder prefix matches all of, and only, the scope.
+		// paging respect the path too. The folder term matches the folder and
+		// its descendants (see PathAnalyzer).
 		if requestedPath := utils.MakeRelativePath(sir.Ref.Path); requestedPath != "." {
-			q.Conjuncts = append(q.Conjuncts, query.NewDisjunctionQuery([]query.Query{
-				&query.TermQuery{FieldVal: "Path", Term: requestedPath},
-				&query.PrefixQuery{FieldVal: "Path", Prefix: requestedPath + "/"},
-			}))
+			q.Conjuncts = append(q.Conjuncts, &query.TermQuery{FieldVal: "Path", Term: requestedPath})
 		}
 	}
 

services/search/pkg/bleve/bleve.go

@@ -1,8 +1,6 @@
 package bleve
 
 import (
-	"regexp"
-
 	bleveSearch "github.com/blevesearch/bleve/v2/search"
 	storageProvider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
 
@@ -11,8 +9,6 @@ import (
 	"github.com/opencloud-eu/opencloud/services/search/pkg/search"
 )
 
-var queryEscape = regexp.MustCompile(`([` + regexp.QuoteMeta(`+=&|><!(){}[]^\"~*?:\/`) + `\-\s])`)
-
 func getFieldValue[T any](m map[string]any, key string) (out T) {
 	val, ok := m[key]
 	if !ok {
@@ -84,7 +80,3 @@ func hitToFacet[T any](fields map[string]any, prefix string) *T {
 func matchToResource(match *bleveSearch.DocumentMatch) *search.Resource {
 	return mapping.Deserialize[search.Resource](match.Fields)
 }
-
-func escapeQuery(s string) string {
-	return queryEscape.ReplaceAllString(s, "\\$1")
-}

services/search/pkg/bleve/hierarchy/hierarchy.go

@@ -0,0 +1,82 @@
+package hierarchy
+
+import (
+	"bytes"
+	"strconv"
+
+	"github.com/blevesearch/bleve/v2/analysis"
+	"github.com/blevesearch/bleve/v2/registry"
+)
+
+// emits every prefix up to a level: "./a/b" -> ".", "./a", "./a/b" with
+// delimiter "/", one level per byte without. tag_depth prepends "<depth>/".
+const Name = "hierarchy"
+
+type Tokenizer struct {
+	delimiter []byte
+	tagDepth  bool
+}
+
+func (t *Tokenizer) Tokenize(input []byte) analysis.TokenStream {
+	if len(input) == 0 {
+		return nil
+	}
+	var out analysis.TokenStream
+	emit := func(depth, end int) {
+		term := input[:end]
+		if t.tagDepth {
+			term = strconv.AppendInt(make([]byte, 0, end+4), int64(depth), 10)
+			term = append(term, '/')
+			term = append(term, input[:end]...)
+		}
+		out = append(out, &analysis.Token{
+			Term:     term,
+			Position: depth,
+			Start:    0,
+			End:      end,
+			Type:     analysis.AlphaNumeric,
+		})
+	}
+
+	if len(t.delimiter) == 0 {
+		for i := range input {
+			emit(i+1, i+1)
+		}
+		return out
+	}
+
+	depth := 0
+	for start := 0; start <= len(input); {
+		i := bytes.Index(input[start:], t.delimiter)
+		if i < 0 {
+			if start < len(input) {
+				depth++
+				emit(depth, len(input))
+			}
+			break
+		}
+		if i > 0 {
+			depth++
+			emit(depth, start+i)
+		}
+		start += i + len(t.delimiter)
+	}
+	return out
+}
+
+func Constructor(config map[string]interface{}, _ *registry.Cache) (analysis.Tokenizer, error) {
+	t := &Tokenizer{}
+	if d, ok := config["delimiter"].(string); ok {
+		t.delimiter = []byte(d)
+	}
+	if v, ok := config["tag_depth"].(bool); ok {
+		t.tagDepth = v
+	}
+	return t, nil
+}
+
+func init() {
+	if err := registry.RegisterTokenizer(Name, Constructor); err != nil {
+		panic(err)
+	}
+}

services/search/pkg/bleve/hierarchy/hierarchy_suite_test.go

@@ -0,0 +1,13 @@
+package hierarchy_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestHierarchy(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "hierarchy tokenizer")
+}

services/search/pkg/bleve/hierarchy/hierarchy_test.go

@@ -0,0 +1,56 @@
+package hierarchy_test
+
+import (
+	"github.com/blevesearch/bleve/v2/analysis"
+	"github.com/blevesearch/bleve/v2/registry"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/opencloud-eu/opencloud/services/search/pkg/bleve/hierarchy"
+)
+
+func terms(ts analysis.TokenStream) []string {
+	out := make([]string, 0, len(ts))
+	for _, t := range ts {
+		out = append(out, string(t.Term))
+	}
+	return out
+}
+
+func tokenize(config map[string]any, input string) []string {
+	tok, err := hierarchy.Constructor(config, registry.NewCache())
+	Expect(err).ToNot(HaveOccurred())
+	return terms(tok.Tokenize([]byte(input)))
+}
+
+var _ = Describe("hierarchy tokenizer", func() {
+	path := map[string]any{"delimiter": "/"}
+	geohash := map[string]any{"tag_depth": true}
+
+	DescribeTable("emits every prefix up to a level boundary",
+		func(config map[string]any, input string, want []string) {
+			Expect(tokenize(config, input)).To(Equal(want))
+		},
+		Entry("relative path", path, "./a/b.txt", []string{".", "./a", "./a/b.txt"}),
+		Entry("space root", path, ".", []string{"."}),
+		Entry("trailing delimiter is not a level", path, "./a/", []string{".", "./a"}),
+		Entry("delimiter only", path, "/", []string{}),
+		Entry("leading delimiter", path, "/abs/x", []string{"/abs", "/abs/x"}),
+		Entry("double delimiter", path, "./a//b", []string{".", "./a", "./a//b"}),
+		Entry("spaces and special characters stay literal", path, "./odd name*[1]/f:x?.txt",
+			[]string{".", "./odd name*[1]", "./odd name*[1]/f:x?.txt"}),
+		Entry("empty input", path, "", []string{}),
+		Entry("geohash, one level per byte, depth tagged", geohash, "u4pru",
+			[]string{"1/u", "2/u4", "3/u4p", "4/u4pr", "5/u4pru"}),
+	)
+
+	It("keeps byte offsets on the source value", func() {
+		tok, err := hierarchy.Constructor(path, registry.NewCache())
+		Expect(err).ToNot(HaveOccurred())
+		ts := tok.Tokenize([]byte("./a/b"))
+		Expect(ts).To(HaveLen(3))
+		Expect(ts[2].Start).To(Equal(0))
+		Expect(ts[2].End).To(Equal(5))
+		Expect(ts[2].Position).To(Equal(3))
+	})
+})

services/search/pkg/bleve/index.go

@@ -19,6 +19,7 @@ import (
 	storageProvider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
 
 	"github.com/opencloud-eu/opencloud/pkg/log"
+	"github.com/opencloud-eu/opencloud/services/search/pkg/bleve/hierarchy"
 	searchmapping "github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
 	"github.com/opencloud-eu/opencloud/services/search/pkg/search"
 )
@@ -208,6 +209,42 @@ func NewMapping() (mapping.IndexMapping, error) {
 	if err != nil {
 		return nil, err
 	}
+	// path: every ancestor prefix is a term, so one term query matches a folder
+	// and all of its descendants
+	err = indexMapping.AddCustomTokenizer("path_hierarchy", map[string]any{
+		"type":      hierarchy.Name,
+		"delimiter": "/",
+	})
+	if err != nil {
+		return nil, err
+	}
+	err = indexMapping.AddCustomAnalyzer(searchmapping.PathAnalyzer, map[string]any{
+		"type":      custom.Name,
+		"tokenizer": "path_hierarchy",
+	})
+	if err != nil {
+		return nil, err
+	}
+	// geohash: every prefix is a depth-tagged term (1/u, 2/u4, ...), so a terms
+	// facet with TermPrefix "<precision>/" is a geohash grid at that precision.
+	// No field uses it yet. It is part of the v5 schema so that #3272 can add
+	// its geohash field additively: new fields reconcile at startup, a changed
+	// analysis block does not (classifyStoredMapping), so the names and the
+	// config below must not change.
+	err = indexMapping.AddCustomTokenizer("geohash_hierarchy", map[string]any{
+		"type":      hierarchy.Name,
+		"tag_depth": true,
+	})
+	if err != nil {
+		return nil, err
+	}
+	err = indexMapping.AddCustomAnalyzer("geohash", map[string]any{
+		"type":      custom.Name,
+		"tokenizer": "geohash_hierarchy",
+	})
+	if err != nil {
+		return nil, err
+	}
 
 	return indexMapping, nil
 }
@@ -226,11 +263,15 @@ func searchResourceByID(id string, index bleve.Index) (*search.Resource, error)
 	return matchToResource(res.Hits[0]), nil
 }
 
+// searchResourcesByPath returns the descendants of the folder at lookupPath.
+// The folder term matches the folder and everything below it in one term
+// query (see PathAnalyzer); the folder itself is dropped from the result.
 func searchResourcesByPath(rootID string, lookupPath string, index bleve.Index) ([]*search.Resource, error) {
-	q := bleve.NewConjunctionQuery(
-		bleve.NewQueryStringQuery("RootID:"+rootID),
-		bleve.NewQueryStringQuery("Path:"+escapeQuery(lookupPath+"/*")),
-	)
+	rootQuery := bleve.NewTermQuery(rootID)
+	rootQuery.SetField("RootID")
+	pathQuery := bleve.NewTermQuery(lookupPath)
+	pathQuery.SetField("Path")
+	q := bleve.NewConjunctionQuery(rootQuery, pathQuery)
 	bleveReq := bleve.NewSearchRequest(q)
 	bleveReq.Size = math.MaxInt
 	bleveReq.Fields = []string{"*"}
@@ -241,7 +282,11 @@ func searchResourcesByPath(rootID string, lookupPath string, index bleve.Index)
 
 	resources := make([]*search.Resource, 0, res.Hits.Len())
 	for _, match := range res.Hits {
-		resources = append(resources, matchToResource(match))
+		resource := matchToResource(match)
+		if resource.Path == lookupPath {
+			continue
+		}
+		resources = append(resources, resource)
 	}
 
 	return resources, nil

services/search/pkg/bleve/testdata/mapping.golden.json

@@ -160,7 +160,7 @@
         "fields": [
           {
             "type": "text",
-            "analyzer": "keyword",
+            "analyzer": "path_hierarchy",
             "store": true,
             "index": true,
             "include_term_vectors": true,
@@ -1259,7 +1259,25 @@
         "type": "regexp"
       }
     },
+    "tokenizers": {
+      "geohash_hierarchy": {
+        "tag_depth": true,
+        "type": "hierarchy"
+      },
+      "path_hierarchy": {
+        "delimiter": "/",
+        "type": "hierarchy"
+      }
+    },
     "analyzers": {
+      "geohash": {
+        "tokenizer": "geohash_hierarchy",
+        "type": "custom"
+      },
+      "path_hierarchy": {
+        "tokenizer": "path_hierarchy",
+        "type": "custom"
+      },
       "words": {
         "char_filters": [
           "dot_to_space"

services/search/pkg/mapping/bleve.go

@@ -60,7 +60,6 @@ func buildBleveDocMapping(t reflect.Type, overrides map[string]FieldOpts, prefix
 		}
 
 		if fieldType == TypeKeyword || fieldType == TypePath {
-			// bleve has no path tokenizer, so a path is a plain keyword here.
 			base := bleveKeywordMapping(fieldType, opts)
 			doc.AddFieldMappingsAt(fi.Name, base)
 			if opts.caseInsensitive() {
@@ -84,8 +83,9 @@ func buildBleveDocMapping(t reflect.Type, overrides map[string]FieldOpts, prefix
 	return doc, err
 }
 
-// bleveKeywordMapping is a case-preserving keyword field; path fields stay out
-// of _all by default.
+// bleveKeywordMapping is a case-preserving keyword field; path fields are
+// analyzed into their ancestor prefixes (see PathAnalyzer) and stay out of
+// _all by default.
 func bleveKeywordMapping(fieldType string, opts FieldOpts) *bleveMapping.FieldMapping {
 	fm := bleve.NewKeywordFieldMapping()
 	switch {
@@ -94,6 +94,9 @@ func bleveKeywordMapping(fieldType string, opts FieldOpts) *bleveMapping.FieldMa
 	case fieldType == TypePath:
 		fm.IncludeInAll = false
 	}
+	if fieldType == TypePath {
+		fm.Analyzer = PathAnalyzer
+	}
 	return fm
 }
 

services/search/pkg/mapping/opensearch.go

@@ -61,7 +61,7 @@ func buildOpenSearchProperties(t reflect.Type, overrides map[string]FieldOpts, p
 			// path_hierarchy is case-preserving here; casing lives in the value.
 			m := map[string]any{"type": "keyword"}
 			if fieldType == TypePath {
-				m = map[string]any{"type": "text", "analyzer": "path_hierarchy"}
+				m = map[string]any{"type": "text", "analyzer": PathAnalyzer}
 			}
 			props[fi.Name] = m
 			if opts.caseInsensitive() {

services/search/pkg/mapping/opensearch_test.go

@@ -100,8 +100,8 @@ var _ = Describe("OpenSearchBuildMapping", func() {
 		Expect(content["term_vector"]).To(Equal("with_positions_offsets"), "Content: %#v", content)
 		Expect(content["analyzer"]).To(Equal(WordsAnalyzer), "Content uses the words analyzer, like bleve")
 		// Path: path_hierarchy base + lowercased sibling, both case-preserving.
-		Expect(props["Path"]).To(Equal(map[string]any{"type": "text", "analyzer": "path_hierarchy"}))
-		Expect(props["Path_lowercase"]).To(Equal(map[string]any{"type": "text", "analyzer": "path_hierarchy"}))
+		Expect(props["Path"]).To(Equal(map[string]any{"type": "text", "analyzer": PathAnalyzer}))
+		Expect(props["Path_lowercase"]).To(Equal(map[string]any{"type": "text", "analyzer": PathAnalyzer}))
 		mime := props["MimeType"].(map[string]any)
 		Expect(mime["type"]).To(Equal("wildcard"), "MimeType: %#v", mime)
 	})

services/search/pkg/mapping/opts.go

@@ -26,6 +26,11 @@ const WordsSuffix = "_words"
 // WordsAnalyzer names the analyzer both engines register for the words sibling.
 const WordsAnalyzer = "words"
 
+// PathAnalyzer names the analyzer both engines register for TypePath fields:
+// every ancestor prefix of a path is a term, so one term query matches a
+// folder and its descendants.
+const PathAnalyzer = "path_hierarchy"
+
 // FieldOpts overrides the default type inference for a struct field. Keys in
 // the override map are json-tag names (e.g. "Name", "location", "audio.artist"),
 // not Go field names.

services/search/pkg/opensearch/index.go

@@ -82,7 +82,7 @@ func buildResourceMapping() ([]byte, error) {
 			"analysis": map[string]any{
 				// path_hierarchy is case-preserving; casing lives in the value.
 				"analyzer": map[string]any{
-					"path_hierarchy": map[string]any{
+					searchmapping.PathAnalyzer: map[string]any{
 						"type":      "custom",
 						"tokenizer": "path_hierarchy",
 					},