From 5287b6508034452f8ace1f79d8a54ce72f268c42 Mon Sep 17 00:00:00 2001 From: Mitchell Kotler Date: Fri, 30 Oct 2026 15:01:46 -0400 Subject: [PATCH 01/14] Keep search boosts and fuzzy degrees out of exponent notation luqum normalizes boost and fuzzy values with Decimal.normalize(), so "taser"^50 was re-serialized as "taser"^5E+1, which Solr reads as a boost of 5 plus a stray term E+1 in both the query and the highlighter. Format these values in plain notation after every parse whose tree is turned back into a query string. Fixes #418 Co-Authored-By: Claude Opus 5.5 --- documentcloud/documents/search.py | 23 ++++++++++++++++++-- documentcloud/documents/tests/test_search.py | 16 ++++++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/documentcloud/documents/search.py b/documentcloud/documents/search.py index e8faf3a2..b86c0433 100644 --- a/documentcloud/documents/search.py +++ b/documentcloud/documents/search.py @@ -406,6 +406,20 @@ def visit_search_field(self, _node, _parents): return None +class NumberFormatter(LuceneTreeTransformer): + """Write boosts and fuzzy degrees in plain notation, as luqum normalizes them + to exponents (^50 becomes ^5E+1), which Solr reads as ^5 plus the term E+1 + """ + + def visit_boost(self, node, _parents): + node.force = format(node.force, "f") + return node + + def visit_fuzzy(self, node, _parents): + node.degree = format(node.degree, "f") + return node + + class AnonymousTransformer(LuceneTreeTransformer): """Remove computational expesnive searches from anonymous queries""" @@ -420,6 +434,11 @@ def visit_term(self, node, _parents): return node +def _parse_tree(text_query): + """Parse a text query into a tree that serializes back to valid Solr syntax""" + return NumberFormatter().visit(parser.parse(text_query)) + + def _parse(text_query, query_params, user): """Parse the text query and pull out filters and sorts @@ -432,7 +451,7 @@ def _parse(text_query, query_params, user): """ if text_query.strip(): try: - tree = parser.parse(text_query) + tree = _parse_tree(text_query) escaped = False except (ParseError, TypeError): tree = parser.parse(escape(text_query)) @@ -865,7 +884,7 @@ def _expand_list(results, key, queryset, serializer): def _extract_note_query(text_query): """Filter out all search fields""" - tree = parser.parse(text_query) + tree = _parse_tree(text_query) note_extractor = NoteExtractor() tree = note_extractor.visit(tree) if tree is None: diff --git a/documentcloud/documents/tests/test_search.py b/documentcloud/documents/tests/test_search.py index 166bbadf..8e681d6a 100644 --- a/documentcloud/documents/tests/test_search.py +++ b/documentcloud/documents/tests/test_search.py @@ -22,6 +22,7 @@ DateValidator, FilterExtractor, NoteExtractor, + _extract_note_query, _parse, search, ) @@ -643,6 +644,18 @@ def test_extract_note(self, query, new_query): assert new_query == (str(tree) if tree else "") +class TestExtractNoteQuery: + @pytest.mark.parametrize( + "query,note_query", + [ + ('"a" AND "b"^50', '"a" AND "b"^50'), + ("foo~10", "foo~10"), + ], + ) + def test_extract_note_query(self, query, note_query): + assert _extract_note_query(query) == note_query + + class TestParse: @pytest.mark.parametrize( "query,query_params,new_query,filters,sort,escaped,use_hl", @@ -677,6 +690,9 @@ class TestParse: ("foo~1", "", "foo~1", "", None, False, False), ("foo hl:true", "", "foo", "", None, False, True), ("foo~1 hl:true", "", "foo~1", "", None, False, False), + ("foo^50", "", "foo^50", "", None, False, False), + ('"a" AND "b"^100', "", '"a" AND "b"^100', "", None, False, False), + ("foo~10", "", "foo~10", "", None, False, False), ], ) def test_auth_parse( From a00583e57850585199eab3f8ca8bc680cc4b2553 Mon Sep 17 00:00:00 2001 From: Mitchell Kotler Date: Fri, 30 Oct 2026 15:06:21 -0400 Subject: [PATCH 02/14] Skip note highlighting when a query has no text outside search fields NoteExtractor only dropped emptied operations and groups below the root, so a boolean query made only of search fields (user:1 AND +tag:"x") became an empty AndOperation rather than None. The highlight query then became "title: description:", which Solr rejects, failing the whole search with a 503 for pro users with highlighting on. Fixes #515 Co-Authored-By: Claude Opus 5.5 --- documentcloud/documents/search.py | 11 ++++++----- documentcloud/documents/tests/test_search.py | 11 +++++++++++ 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/documentcloud/documents/search.py b/documentcloud/documents/search.py index b86c0433..27ed0f5a 100644 --- a/documentcloud/documents/search.py +++ b/documentcloud/documents/search.py @@ -393,12 +393,13 @@ class NoteExtractor(LuceneTreeTransformer): def visit(self, node, parents=None): new_node = super().visit(node, parents) - if ( - isinstance(new_node, (Group, BaseOperation, Unary)) - and not any(node.children) - and parents + if isinstance(new_node, (Group, BaseOperation, Unary)) and not any( + node.children ): - self.replace_node(new_node, None, parents[-1]) + # an emptied root must also be dropped, or it serializes to an + # empty or "(None)" query which Solr cannot parse + if parents: + self.replace_node(new_node, None, parents[-1]) return None return new_node diff --git a/documentcloud/documents/tests/test_search.py b/documentcloud/documents/tests/test_search.py index 8e681d6a..8b73609d 100644 --- a/documentcloud/documents/tests/test_search.py +++ b/documentcloud/documents/tests/test_search.py @@ -466,6 +466,13 @@ def test_search_notes_escape_quotes(self): response = self.search('q="delta"', user=1) assert len(response["results"]) == 1 + def test_search_notes_highlight_only_fields(self): + """A boolean query of only search fields leaves no text to search notes for""" + + response = self.search("q=id:1 AND access:public&hl=true", user=2) + assert [d["id"] for d in response["results"]] == ["1"] + assert response["results"][0]["notes"] + class TestBooleanDetector: @pytest.mark.parametrize( @@ -650,6 +657,10 @@ class TestExtractNoteQuery: [ ('"a" AND "b"^50', '"a" AND "b"^50'), ("foo~10", "foo~10"), + ("user:1 AND tag:a", None), + ('user:1 AND +data__tag:"a"', None), + ("(user:1 AND tag:a)", None), + ("(user:1) OR (tag:a)", None), ], ) def test_extract_note_query(self, query, note_query): From 4b13e624e135943e68a9a4d203a483d86fb14644 Mon Sep 17 00:00:00 2001 From: Mitchell Kotler Date: Fri, 30 Oct 2026 15:30:03 -0400 Subject: [PATCH 03/14] Drop boosts left empty when extracting the note query A boosted search field such as user:1^2 left Boost(None) behind once NoteExtractor removed the field, so the note query searched for the literal term "None" (e.g. "foo OR None^2"), adding documents whose notes contain "none" and breaking note highlights. Co-Authored-By: Claude Opus 5.5 --- documentcloud/documents/search.py | 2 +- documentcloud/documents/tests/test_search.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/documentcloud/documents/search.py b/documentcloud/documents/search.py index 27ed0f5a..a74f47b0 100644 --- a/documentcloud/documents/search.py +++ b/documentcloud/documents/search.py @@ -393,7 +393,7 @@ class NoteExtractor(LuceneTreeTransformer): def visit(self, node, parents=None): new_node = super().visit(node, parents) - if isinstance(new_node, (Group, BaseOperation, Unary)) and not any( + if isinstance(new_node, (Boost, Group, BaseOperation, Unary)) and not any( node.children ): # an emptied root must also be dropped, or it serializes to an diff --git a/documentcloud/documents/tests/test_search.py b/documentcloud/documents/tests/test_search.py index 8b73609d..31cbe3d7 100644 --- a/documentcloud/documents/tests/test_search.py +++ b/documentcloud/documents/tests/test_search.py @@ -661,6 +661,9 @@ class TestExtractNoteQuery: ('user:1 AND +data__tag:"a"', None), ("(user:1 AND tag:a)", None), ("(user:1) OR (tag:a)", None), + ("user:1^2 AND tag:a", None), + ("foo OR (user:1)^2", "foo"), + ("(user:1 AND tag:a)^2 foo", "foo"), ], ) def test_extract_note_query(self, query, note_query): From 1c4dd727fd572773ef49a028842365f26d029c19 Mon Sep 17 00:00:00 2001 From: Mitchell Kotler Date: Fri, 30 Oct 2026 15:32:24 -0400 Subject: [PATCH 04/14] Fall back to an escaped query on malformed boost and fuzzy numbers luqum builds boosts and fuzzy degrees with Decimal(), which raises InvalidOperation for input such as foo^1.2.3 or foo^. rather than a ParseError, so the search returned a 500 for any user. Treat it like any other unparseable query and escape it. Co-Authored-By: Claude Opus 5.5 --- documentcloud/documents/search.py | 4 +++- documentcloud/documents/tests/test_search.py | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/documentcloud/documents/search.py b/documentcloud/documents/search.py index a74f47b0..92b910ff 100644 --- a/documentcloud/documents/search.py +++ b/documentcloud/documents/search.py @@ -7,6 +7,7 @@ import re from collections import defaultdict from datetime import datetime +from decimal import InvalidOperation # Third Party import pysolr @@ -454,7 +455,8 @@ def _parse(text_query, query_params, user): try: tree = _parse_tree(text_query) escaped = False - except (ParseError, TypeError): + except (ParseError, TypeError, InvalidOperation): + # InvalidOperation is luqum failing to read a boost or fuzzy number tree = parser.parse(escape(text_query)) escaped = True diff --git a/documentcloud/documents/tests/test_search.py b/documentcloud/documents/tests/test_search.py index 31cbe3d7..ceb5fad0 100644 --- a/documentcloud/documents/tests/test_search.py +++ b/documentcloud/documents/tests/test_search.py @@ -707,6 +707,9 @@ class TestParse: ("foo^50", "", "foo^50", "", None, False, False), ('"a" AND "b"^100', "", '"a" AND "b"^100', "", None, False, False), ("foo~10", "", "foo~10", "", None, False, False), + ("foo^1.2.3", "", "foo\\^1.2.3", "", None, True, False), + ("foo^.", "", "foo\\^.", "", None, True, False), + ("foo~1.2.3", "", "foo\\~1.2.3", "", None, True, False), ], ) def test_auth_parse( From 38eb1a47404f6e4377d016253d85779dd9bd2714 Mon Sep 17 00:00:00 2001 From: Mitchell Kotler Date: Fri, 30 Oct 2026 15:51:21 -0400 Subject: [PATCH 05/14] Pass note subqueries to Solr as dereferenced params The note query was inlined in single-quoted local params (v='...') with only double quotes escaped, so any apostrophe ended the value early. Solr's lenient parser swallowed the error and the note joins matched nothing, silently dropping note results for queries like "don't". Escaping the apostrophe is not enough, as the value passes through two layers of unescaping and a user's own backslashes are still mangled, so send each subquery as its own request param and reference it with v=$param instead. Co-Authored-By: Claude Opus 5.5 --- documentcloud/documents/search.py | 26 +++++++++++++------- documentcloud/documents/tests/test_search.py | 9 +++++++ 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/documentcloud/documents/search.py b/documentcloud/documents/search.py index 92b910ff..ebc7005f 100644 --- a/documentcloud/documents/search.py +++ b/documentcloud/documents/search.py @@ -142,7 +142,8 @@ def search(user, query_params): ): # turn note queries on for all pro users # *:* returns all documents, do not enable note queries - text_query = _add_note_query(text_query, user) + text_query, note_params = _add_note_query(text_query, user) + kwargs.update(note_params) kwargs["uf"] = "* _query_ -projects_edit_access" kwargs["hl.q"] = original_text_query query_notes = True @@ -896,6 +897,9 @@ def _extract_note_query(text_query): def _add_note_query(text_query, user): + """Add note searches to the text query, returning the new query and the + request params it references + """ organizations = " ".join(str(o.pk) for o in user.organizations.all()) projects = " ".join( str(p.pk) @@ -909,8 +913,15 @@ def _add_note_query(text_query, user): note_query = _extract_note_query(text_query) if note_query is None: - return text_query - escaped_note_query = note_query.replace('"', '\\"') + return text_query, {} + # the note queries are sent as separate params and dereferenced, as any + # quoting needed to inline them in local params is mangled by the user's + # own quotes and backslashes + note_match = f"+(title:({note_query}) description:({note_query}))" + note_params = { + "note_q_public": f"+type:note +(access:public OR user:{user.pk}) {note_match}", + "note_q_organization": f"+type:note +(access:organization) {note_match}", + } return_query = ( # the original query to search for in documents @@ -919,9 +930,7 @@ def _add_note_query(text_query, user): # on all documents you can view f""" _query_:"{{!join from=document_s fromIndex=notes to=id score=total - v='+type:note +(access:public OR user:{user.pk}) - +(title:({escaped_note_query}) description:({escaped_note_query}))' - }}" + v=$note_q_public}}" """ # search through notes which are organization access # on all documents you can edit @@ -936,11 +945,10 @@ def _add_note_query(text_query, user): (access:(public organization) AND organization:({organizations})) ) +{{!join from=document_s fromIndex=notes to=id score=total - v='+type:note +(access:organization) - +(title:({escaped_note_query}) description:({escaped_note_query}))'}} + v=$note_q_organization}} " """ ) # replace runs of white space with a single space - keeps the template readable # without incurring overhead of sending extra white space to solr - return re.sub(r"\s+", " ", return_query) + return re.sub(r"\s+", " ", return_query), note_params diff --git a/documentcloud/documents/tests/test_search.py b/documentcloud/documents/tests/test_search.py index ceb5fad0..b2a6fece 100644 --- a/documentcloud/documents/tests/test_search.py +++ b/documentcloud/documents/tests/test_search.py @@ -466,6 +466,15 @@ def test_search_notes_escape_quotes(self): response = self.search('q="delta"', user=1) assert len(response["results"]) == 1 + @pytest.mark.parametrize( + "query", ["alice OR don't", '"alice" OR "don\'t"', "alice OR don\\'t"] + ) + def test_search_notes_apostrophe(self, query): + """Apostrophes must not break out of the note subqueries""" + + response = self.search(f"q={query}", user=2) + assert [d["id"] for d in response["results"]] == ["1"] + def test_search_notes_highlight_only_fields(self): """A boolean query of only search fields leaves no text to search notes for""" From c01dd91ed146b05a735f1542a7ab2121172c50c3 Mon Sep 17 00:00:00 2001 From: Mitchell Kotler Date: Fri, 30 Oct 2026 16:01:23 -0400 Subject: [PATCH 06/14] Scope every term of the note highlight query to the note fields The highlight query was built as "title: description:", so for a multi-term query only the first term was searched in title or description; the rest went to the default field, and notes matched by the main search came back without highlights. Co-Authored-By: Claude Opus 5.5 --- documentcloud/documents/search.py | 2 +- documentcloud/documents/tests/test_search.py | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/documentcloud/documents/search.py b/documentcloud/documents/search.py index ebc7005f..25f82982 100644 --- a/documentcloud/documents/search.py +++ b/documentcloud/documents/search.py @@ -777,7 +777,7 @@ def _highlight_notes(response, text_query): note_query = _extract_note_query(text_query) if note_query is None: return response - query = f"title:{note_query} description:{note_query}" + query = f"title:({note_query}) description:({note_query})" kwargs = { "fq": fq_, diff --git a/documentcloud/documents/tests/test_search.py b/documentcloud/documents/tests/test_search.py index b2a6fece..fb38c282 100644 --- a/documentcloud/documents/tests/test_search.py +++ b/documentcloud/documents/tests/test_search.py @@ -475,6 +475,17 @@ def test_search_notes_apostrophe(self, query): response = self.search(f"q={query}", user=2) assert [d["id"] for d in response["results"]] == ["1"] + @pytest.mark.parametrize( + "term,field", [("alice", "description"), ("note", "title")] + ) + def test_search_notes_highlight_multiple_terms(self, term, field): + """Every term of the note query is highlighted in each note field""" + + response = self.search(f"q=zzz OR {term}&hl=true", user=2) + document = next(d for d in response["results"] if d["id"] == "1") + highlight = str(document["note_highlights"]["1"][field]).lower() + assert f"{term}" in highlight + def test_search_notes_highlight_only_fields(self): """A boolean query of only search fields leaves no text to search notes for""" From dce80a2423b929615c88d57b62d4e20d3a924a55 Mon Sep 17 00:00:00 2001 From: Mitchell Kotler Date: Fri, 30 Oct 2026 16:06:46 -0400 Subject: [PATCH 07/14] Escape text params that would not parse as a query title=, description=, source=, text= and page_no_N= values were appended to the query unchecked. Solr tolerates something like title:(bar)), but for pro users the note query re-parses the whole query with luqum, which raised an uncaught ParseError and returned a 500. Escape any value luqum cannot parse before appending it. Co-Authored-By: Claude Opus 5.5 --- documentcloud/documents/search.py | 28 ++++++++++++++++---- documentcloud/documents/tests/test_search.py | 10 +++++++ 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/documentcloud/documents/search.py b/documentcloud/documents/search.py index 25f82982..04f82ce7 100644 --- a/documentcloud/documents/search.py +++ b/documentcloud/documents/search.py @@ -62,6 +62,9 @@ } DYNAMIC_TEXT_FIELDS = [r"page_no_[0-9]+"] +# InvalidOperation is luqum failing to read a boost or fuzzy number +PARSE_ERRORS = (ParseError, TypeError, InvalidOperation) + SORT_MAP = { "score": "score desc, created_at desc, id desc", "created_at": "created_at desc, id desc", @@ -456,8 +459,7 @@ def _parse(text_query, query_params, user): try: tree = _parse_tree(text_query) escaped = False - except (ParseError, TypeError, InvalidOperation): - # InvalidOperation is luqum failing to read a boost or fuzzy number + except PARSE_ERRORS: tree = parser.parse(escape(text_query)) escaped = True @@ -487,7 +489,9 @@ def _parse(text_query, query_params, user): use_hl = False # pull text queries from the parameters into the text query - additional_text = _handle_params(query_params, TEXT_FIELDS, DYNAMIC_TEXT_FIELDS) + additional_text = _handle_params( + query_params, TEXT_FIELDS, DYNAMIC_TEXT_FIELDS, escape_invalid=True + ) if additional_text: # pylint: disable=consider-using-f-string new_query = "{} {}".format(new_query, " ".join(additional_text)) @@ -520,7 +524,16 @@ def _validate_range(value, validator): return all(validator.visit(node)) and isinstance(node, (Word, Range)) -def _handle_params(query_params, fields, dynamic_fields): +def _escape_invalid(value): + """Escape a value which would not parse, as the note query re-parses it""" + try: + parser.parse(value) + return value + except PARSE_ERRORS: + return escape(value) + + +def _handle_params(query_params, fields, dynamic_fields, escape_invalid=False): """Convert query params to a list of Solr field queries""" return_list = [] items = list(fields.items()) @@ -530,6 +543,8 @@ def _handle_params(query_params, fields, dynamic_fields): if param in query_params: # joining with whitespace will default to OR values = query_params.getlist(param) + if escape_invalid: + values = [_escape_invalid(v) for v in values] if field in NUMERIC_FIELDS: values = [v for v in values if _validate_range(v, NumberValidator())] if field in DATE_FIELDS: @@ -547,7 +562,10 @@ def _handle_params(query_params, fields, dynamic_fields): # allow for negated dynamic fields dynamic_params = [p for p in query_params if re.match(rf"^-?{pattern}$", p)] for param in dynamic_params: - values = " ".join(query_params.getlist(param)) + values = query_params.getlist(param) + if escape_invalid: + values = [_escape_invalid(v) for v in values] + values = " ".join(values) return_list.append(f"{param}:({values})") return return_list diff --git a/documentcloud/documents/tests/test_search.py b/documentcloud/documents/tests/test_search.py index fb38c282..af0f2d9a 100644 --- a/documentcloud/documents/tests/test_search.py +++ b/documentcloud/documents/tests/test_search.py @@ -486,6 +486,12 @@ def test_search_notes_highlight_multiple_terms(self, term, field): highlight = str(document["note_highlights"]["1"][field]).lower() assert f"{term}" in highlight + def test_search_notes_invalid_text_param(self): + """An unparseable text param must not break the note query""" + + response = self.search("q=alice&title=bar)", user=2) + assert [d["id"] for d in response["results"]] == ["1"] + def test_search_notes_highlight_only_fields(self): """A boolean query of only search fields leaves no text to search notes for""" @@ -730,6 +736,10 @@ class TestParse: ("foo^1.2.3", "", "foo\\^1.2.3", "", None, True, False), ("foo^.", "", "foo\\^.", "", None, True, False), ("foo~1.2.3", "", "foo\\~1.2.3", "", None, True, False), + ("foo", "title=bar)", "foo title:(bar\\))", "", None, False, False), + ("foo", "-title=(bar", "foo -title:(\\(bar)", "", None, False, False), + ("foo", "text=x^1.2.3", "foo doctext:(x\\^1.2.3)", "", None, False, False), + ("foo", "page_no_1=bar)", "foo page_no_1:(bar\\))", "", None, False, False), ], ) def test_auth_parse( From 3a5669b0ed7c28509dcca25643ae3e0fb5028947 Mon Sep 17 00:00:00 2001 From: Mitchell Kotler Date: Fri, 30 Oct 2026 16:41:17 -0400 Subject: [PATCH 08/14] Fall back to an escaped query on fractional proximity slop luqum reads a proximity slop with int(), so "a b"~1.5 or "a b"~. raised ValueError rather than a ParseError and the search returned a 500 for any user. Treat it like the other unparseable queries and escape it. Co-Authored-By: Claude Opus 5.5 --- documentcloud/documents/search.py | 5 +++-- documentcloud/documents/tests/test_search.py | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/documentcloud/documents/search.py b/documentcloud/documents/search.py index 04f82ce7..72d99587 100644 --- a/documentcloud/documents/search.py +++ b/documentcloud/documents/search.py @@ -62,8 +62,9 @@ } DYNAMIC_TEXT_FIELDS = [r"page_no_[0-9]+"] -# InvalidOperation is luqum failing to read a boost or fuzzy number -PARSE_ERRORS = (ParseError, TypeError, InvalidOperation) +# luqum raises InvalidOperation or ValueError when it fails to read the number in +# a boost, fuzzy or proximity search +PARSE_ERRORS = (ParseError, TypeError, InvalidOperation, ValueError) SORT_MAP = { "score": "score desc, created_at desc, id desc", diff --git a/documentcloud/documents/tests/test_search.py b/documentcloud/documents/tests/test_search.py index af0f2d9a..36742dbf 100644 --- a/documentcloud/documents/tests/test_search.py +++ b/documentcloud/documents/tests/test_search.py @@ -736,6 +736,8 @@ class TestParse: ("foo^1.2.3", "", "foo\\^1.2.3", "", None, True, False), ("foo^.", "", "foo\\^.", "", None, True, False), ("foo~1.2.3", "", "foo\\~1.2.3", "", None, True, False), + ('"a b"~1.5', "", '"a b" \\~1.5', "", None, True, False), + ('"a b"~.', "", '"a b" \\~.', "", None, True, False), ("foo", "title=bar)", "foo title:(bar\\))", "", None, False, False), ("foo", "-title=(bar", "foo -title:(\\(bar)", "", None, False, False), ("foo", "text=x^1.2.3", "foo doctext:(x\\^1.2.3)", "", None, False, False), From 73314dcfdb1b3c7ff45f0c70c6ae861adb0a7a3e Mon Sep 17 00:00:00 2001 From: Mitchell Kotler Date: Fri, 30 Oct 2026 16:42:26 -0400 Subject: [PATCH 09/14] Treat unparseable numeric and date filter params as invalid _validate_range let luqum errors escape, so a filter param such as page_count=1^, id=1^1.2.3 or created_at="1 2"~1.5 returned a 500 for any user. Fail validation instead, which drops the value like any other invalid filter. Co-Authored-By: Claude Opus 5.5 --- documentcloud/documents/search.py | 5 ++++- documentcloud/documents/tests/test_search.py | 16 ++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/documentcloud/documents/search.py b/documentcloud/documents/search.py index 72d99587..9237904f 100644 --- a/documentcloud/documents/search.py +++ b/documentcloud/documents/search.py @@ -521,7 +521,10 @@ def _validate_range(value, validator): """Validate using the validator as well as ensuring the top level node is either a word or range """ - node = parser.parse(value) + try: + node = parser.parse(value) + except PARSE_ERRORS: + return False return all(validator.visit(node)) and isinstance(node, (Word, Range)) diff --git a/documentcloud/documents/tests/test_search.py b/documentcloud/documents/tests/test_search.py index 36742dbf..04cbd54b 100644 --- a/documentcloud/documents/tests/test_search.py +++ b/documentcloud/documents/tests/test_search.py @@ -22,8 +22,10 @@ DateValidator, FilterExtractor, NoteExtractor, + NumberValidator, _extract_note_query, _parse, + _validate_range, search, ) from documentcloud.documents.search_escape import escape @@ -553,6 +555,20 @@ def test_validator(self, query, valid): assert all(DateValidator().visit(tree)) is valid +class TestValidateRange: + @pytest.mark.parametrize( + "value,validator", + [ + ("1^", NumberValidator()), + ("1^1.2.3", NumberValidator()), + ('"1 2"~1.5', NumberValidator()), + ('"NOW 2"~1.5', DateValidator()), + ], + ) + def test_unparseable(self, value, validator): + assert _validate_range(value, validator) is False + + class TestFilterExtractor: @pytest.mark.parametrize( "query,new_query,filters,sort_only,sort", From 307fdc9dd12a7cf413f6e446eff4c02067cf16b1 Mon Sep 17 00:00:00 2001 From: Mitchell Kotler Date: Fri, 30 Oct 2026 16:47:43 -0400 Subject: [PATCH 10/14] Build the note query from q instead of escaping text params Text params are always field searches, which the note extractor drops, so they never contribute to the note query. They only broke it because _add_note_query and _highlight_notes re-parsed the combined query, and escaping each param missed cases such as title=foo\ and page_no_1=. Extract the note query once in _parse from the parsed q, before params are appended, and pass it to both, which also avoids parsing it twice. Text params go to Solr unescaped again, as before. Co-Authored-By: Claude Opus 5.5 --- documentcloud/documents/search.py | 51 ++++++++------------ documentcloud/documents/tests/test_search.py | 45 ++++++++++++----- 2 files changed, 53 insertions(+), 43 deletions(-) diff --git a/documentcloud/documents/search.py b/documentcloud/documents/search.py index 9237904f..8148bcac 100644 --- a/documentcloud/documents/search.py +++ b/documentcloud/documents/search.py @@ -6,6 +6,7 @@ import math import re from collections import defaultdict +from copy import deepcopy from datetime import datetime from decimal import InvalidOperation @@ -103,7 +104,7 @@ def search(user, query_params): # pylint: disable=too-many-locals text_query = query_params.get("q", "") - text_query, filter_params, sort_order, escaped, use_hl = _parse( + text_query, filter_params, sort_order, escaped, use_hl, note_query = _parse( text_query, query_params, user ) @@ -146,7 +147,7 @@ def search(user, query_params): ): # turn note queries on for all pro users # *:* returns all documents, do not enable note queries - text_query, note_params = _add_note_query(text_query, user) + text_query, note_params = _add_note_query(text_query, note_query, user) kwargs.update(note_params) kwargs["uf"] = "* _query_ -projects_edit_access" kwargs["hl.q"] = original_text_query @@ -177,7 +178,7 @@ def search(user, query_params): results, query_params, user, escaped, page_response_data ) if use_hl and query_notes: - response = _highlight_notes(response, original_text_query) + response = _highlight_notes(response, note_query) if settings.DEBUG or user.is_staff: response["debug"] = {"text_query": text_query, "qtime": results.qtime, **kwargs} @@ -450,11 +451,12 @@ def _parse(text_query, query_params, user): """Parse the text query and pull out filters and sorts Accepts a text query - Returns a tuple of (text_query, filters, sort) + Returns a tuple of (text_query, filters, sort, escaped, use_hl, note_query) text_query - new text query with filters and sorts removed filters - a list of filters to be passed in to solr as field queries (`fq` field) sort - a string from the SORT_MAP to sort on + note_query - the text query to search notes for, or None """ if text_query.strip(): try: @@ -476,6 +478,7 @@ def _parse(text_query, query_params, user): tree = AnonymousTransformer().visit(tree) new_query = str(tree) if tree is not None else "" + note_query = _extract_note_query(tree) filters = filter_extractor.filters sort = filter_extractor.sort # only use highilighting for queries with no fuzzy searches and @@ -488,11 +491,10 @@ def _parse(text_query, query_params, user): sort = None escaped = False use_hl = False + note_query = None # pull text queries from the parameters into the text query - additional_text = _handle_params( - query_params, TEXT_FIELDS, DYNAMIC_TEXT_FIELDS, escape_invalid=True - ) + additional_text = _handle_params(query_params, TEXT_FIELDS, DYNAMIC_TEXT_FIELDS) if additional_text: # pylint: disable=consider-using-f-string new_query = "{} {}".format(new_query, " ".join(additional_text)) @@ -502,7 +504,7 @@ def _parse(text_query, query_params, user): if not new_query: new_query = "*:*" - return new_query, filters, sort, escaped, use_hl + return new_query, filters, sort, escaped, use_hl, note_query def _filter_queries(user, query_params): @@ -528,16 +530,7 @@ def _validate_range(value, validator): return all(validator.visit(node)) and isinstance(node, (Word, Range)) -def _escape_invalid(value): - """Escape a value which would not parse, as the note query re-parses it""" - try: - parser.parse(value) - return value - except PARSE_ERRORS: - return escape(value) - - -def _handle_params(query_params, fields, dynamic_fields, escape_invalid=False): +def _handle_params(query_params, fields, dynamic_fields): """Convert query params to a list of Solr field queries""" return_list = [] items = list(fields.items()) @@ -547,8 +540,6 @@ def _handle_params(query_params, fields, dynamic_fields, escape_invalid=False): if param in query_params: # joining with whitespace will default to OR values = query_params.getlist(param) - if escape_invalid: - values = [_escape_invalid(v) for v in values] if field in NUMERIC_FIELDS: values = [v for v in values if _validate_range(v, NumberValidator())] if field in DATE_FIELDS: @@ -566,10 +557,7 @@ def _handle_params(query_params, fields, dynamic_fields, escape_invalid=False): # allow for negated dynamic fields dynamic_params = [p for p in query_params if re.match(rf"^-?{pattern}$", p)] for param in dynamic_params: - values = query_params.getlist(param) - if escape_invalid: - values = [_escape_invalid(v) for v in values] - values = " ".join(values) + values = " ".join(query_params.getlist(param)) return_list.append(f"{param}:({values})") return return_list @@ -787,7 +775,7 @@ def format_note(result): return [format_note(r) for r in results] -def _highlight_notes(response, text_query): +def _highlight_notes(response, note_query): """Do a second query to highlight notes if necessary""" notes = [n for d in response["results"] for n in d["notes"]] note_ids = [n["id"] for n in notes] @@ -796,7 +784,6 @@ def _highlight_notes(response, text_query): parents = {n["id"]: d["id"] for d in response["results"] for n in d["notes"]} fq_ = [f"id:({' '.join(note_ids)})"] - note_query = _extract_note_query(text_query) if note_query is None: return response query = f"title:({note_query}) description:({note_query})" @@ -908,17 +895,18 @@ def _expand_list(results, key, queryset, serializer): return results -def _extract_note_query(text_query): +def _extract_note_query(tree): """Filter out all search fields""" - tree = _parse_tree(text_query) - note_extractor = NoteExtractor() - tree = note_extractor.visit(tree) + if tree is None: + return None + # copy, as the extractor removes nodes in place + tree = NoteExtractor().visit(deepcopy(tree)) if tree is None: return None return str(tree) -def _add_note_query(text_query, user): +def _add_note_query(text_query, note_query, user): """Add note searches to the text query, returning the new query and the request params it references """ @@ -933,7 +921,6 @@ def _add_note_query(text_query, user): ) ) - note_query = _extract_note_query(text_query) if note_query is None: return text_query, {} # the note queries are sent as separate params and dereferenced, as any diff --git a/documentcloud/documents/tests/test_search.py b/documentcloud/documents/tests/test_search.py index 04cbd54b..d29120af 100644 --- a/documentcloud/documents/tests/test_search.py +++ b/documentcloud/documents/tests/test_search.py @@ -25,6 +25,7 @@ NumberValidator, _extract_note_query, _parse, + _parse_tree, _validate_range, search, ) @@ -488,10 +489,11 @@ def test_search_notes_highlight_multiple_terms(self, term, field): highlight = str(document["note_highlights"]["1"][field]).lower() assert f"{term}" in highlight - def test_search_notes_invalid_text_param(self): + @pytest.mark.parametrize("param", ["title=bar)", "title=foo\\", "page_no_1="]) + def test_search_notes_invalid_text_param(self, param): """An unparseable text param must not break the note query""" - response = self.search("q=alice&title=bar)", user=2) + response = self.search(f"q=alice&{param}", user=2) assert [d["id"] for d in response["results"]] == ["1"] def test_search_notes_highlight_only_fields(self): @@ -709,7 +711,12 @@ class TestExtractNoteQuery: ], ) def test_extract_note_query(self, query, note_query): - assert _extract_note_query(query) == note_query + assert _extract_note_query(_parse_tree(query)) == note_query + + def test_tree_unchanged(self): + tree = _parse_tree("a AND user:1") + _extract_note_query(tree) + assert str(tree) == "a AND user:1" class TestParse: @@ -754,10 +761,6 @@ class TestParse: ("foo~1.2.3", "", "foo\\~1.2.3", "", None, True, False), ('"a b"~1.5', "", '"a b" \\~1.5', "", None, True, False), ('"a b"~.', "", '"a b" \\~.', "", None, True, False), - ("foo", "title=bar)", "foo title:(bar\\))", "", None, False, False), - ("foo", "-title=(bar", "foo -title:(\\(bar)", "", None, False, False), - ("foo", "text=x^1.2.3", "foo doctext:(x\\^1.2.3)", "", None, False, False), - ("foo", "page_no_1=bar)", "foo page_no_1:(bar\\))", "", None, False, False), ], ) def test_auth_parse( @@ -766,7 +769,7 @@ def test_auth_parse( # pylint: disable=too-many-arguments user = Mock() user.is_authenticated = True - assert _parse(query, QueryDict(query_params, mutable=True), user) == ( + assert _parse(query, QueryDict(query_params, mutable=True), user)[:5] == ( new_query, QueryDict(filters), sort, @@ -787,9 +790,29 @@ def test_anon_parse( self, query, query_params, new_query, filters, sort, escaped, use_hl ): # pylint: disable=too-many-arguments - assert _parse( - query, QueryDict(query_params, mutable=True), AnonymousUser() - ) == (new_query, QueryDict(filters), sort, escaped, use_hl) + assert _parse(query, QueryDict(query_params, mutable=True), AnonymousUser())[ + :5 + ] == (new_query, QueryDict(filters), sort, escaped, use_hl) + + @pytest.mark.parametrize( + "query,query_params,note_query", + [ + ("", "", None), + ("", "title=bar", None), + ("user:1 AND tag:a", "", None), + ("a user:1", "", "a"), + ("a", "title=bar)", "a"), + ("a", "title=foo\\", "a"), + ("a", "page_no_1=", "a"), + ('"a" AND "b"^50', "", '"a" AND "b"^50'), + ], + ) + def test_note_query(self, query, query_params, note_query): + user = Mock() + user.is_authenticated = True + assert ( + _parse(query, QueryDict(query_params, mutable=True), user)[5] == note_query + ) class TestEscape: From 8c0631c5c0584626a20ff6d98126aeaae92e6ea9 Mon Sep 17 00:00:00 2001 From: Mitchell Kotler Date: Thu, 5 Nov 2026 09:55:41 -0500 Subject: [PATCH 11/14] Truncate fractional phrase slop instead of escaping the query luqum cannot parse a fractional proximity slop, so "test pdf"~1.5 fell back to the escaped query and searched for and highlighted "1.5" as a term. Solr itself truncates the slop ("~1.5" is "~1", "~." is none), so do the same before parsing. Co-Authored-By: Claude Opus 5.5 --- documentcloud/documents/search.py | 14 ++++++++++++++ documentcloud/documents/tests/test_search.py | 8 ++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/documentcloud/documents/search.py b/documentcloud/documents/search.py index 8148bcac..6f24cdfe 100644 --- a/documentcloud/documents/search.py +++ b/documentcloud/documents/search.py @@ -67,6 +67,10 @@ # a boost, fuzzy or proximity search PARSE_ERRORS = (ParseError, TypeError, InvalidOperation, ValueError) +# matches escapes and phrases, so a quote inside a phrase is never mistaken for one +# which ends it, capturing any fractional proximity slop after a phrase +PHRASE_SLOP_RE = re.compile(r'\\.|("(?:[^"\\]|\\.)*")(?:~(\d*)\.[\d.]*)?') + SORT_MAP = { "score": "score desc, created_at desc, id desc", "created_at": "created_at desc, id desc", @@ -442,8 +446,18 @@ def visit_term(self, node, _parents): return node +def _truncate_slop(match): + phrase, slop = match.groups() + if slop is None: + return match.group(0) + return f"{phrase}~{slop}" if slop else phrase + + def _parse_tree(text_query): """Parse a text query into a tree that serializes back to valid Solr syntax""" + # Solr truncates a fractional phrase slop to a whole number, but luqum fails + # to parse one + text_query = PHRASE_SLOP_RE.sub(_truncate_slop, text_query) return NumberFormatter().visit(parser.parse(text_query)) diff --git a/documentcloud/documents/tests/test_search.py b/documentcloud/documents/tests/test_search.py index d29120af..457ed444 100644 --- a/documentcloud/documents/tests/test_search.py +++ b/documentcloud/documents/tests/test_search.py @@ -759,8 +759,12 @@ class TestParse: ("foo^1.2.3", "", "foo\\^1.2.3", "", None, True, False), ("foo^.", "", "foo\\^.", "", None, True, False), ("foo~1.2.3", "", "foo\\~1.2.3", "", None, True, False), - ('"a b"~1.5', "", '"a b" \\~1.5', "", None, True, False), - ('"a b"~.', "", '"a b" \\~.', "", None, True, False), + ('"a b"~1.5', "", '"a b"~1', "", None, False, False), + ('"a b"~.', "", '"a b"', "", None, False, False), + ('"a b"~0.5 c', "", '"a b"~0 c', "", None, False, False), + ('"x" c "~2.5"~3.5', "", '"x" c "~2.5"~3', "", None, False, False), + ('"x \\" y"~1.5', "", '"x \\" y"~1', "", None, False, False), + ('\\" "a b"~1.5', "", '\\" "a b"~1', "", None, False, False), ], ) def test_auth_parse( From 9a28f110fd3141ea2858bd54d04e3ba3f6b31a92 Mon Sep 17 00:00:00 2001 From: Mitchell Kotler Date: Thu, 5 Nov 2026 10:07:53 -0500 Subject: [PATCH 12/14] Highlight only the text of a query, not its filter values In a boolean query the filter fields stay in q, and both q and the pro user hl.q were used for highlighting, so foo AND (user:1 OR tag:email) also highlighted "email" in page text. Build a separate highlight query with the filter fields removed (text fields such as title: are kept), use it as hl.q for every user, and turn highlighting off when only filters are left. Co-Authored-By: Claude Opus 5.5 --- documentcloud/documents/search.py | 39 +++++++++++---- documentcloud/documents/tests/search_data.py | 1 + documentcloud/documents/tests/test_search.py | 52 ++++++++++++++++++-- 3 files changed, 77 insertions(+), 15 deletions(-) diff --git a/documentcloud/documents/search.py b/documentcloud/documents/search.py index 6f24cdfe..6e2ceec3 100644 --- a/documentcloud/documents/search.py +++ b/documentcloud/documents/search.py @@ -108,8 +108,8 @@ def search(user, query_params): # pylint: disable=too-many-locals text_query = query_params.get("q", "") - text_query, filter_params, sort_order, escaped, use_hl, note_query = _parse( - text_query, query_params, user + text_query, filter_params, sort_order, escaped, use_hl, note_query, hl_query = ( + _parse(text_query, query_params, user) ) filter_params.update(query_params) @@ -133,6 +133,9 @@ def search(user, query_params): # allow explicit enabling of highlighting if query_params.get("hl", "").lower() == "true" and settings.SOLR_USE_HL: use_hl = True + if hl_query is None: + # nothing but filters, which should not be highlighted + use_hl = False kwargs = { "fq": filter_queries, @@ -142,7 +145,8 @@ def search(user, query_params): "hl.weightMatches": settings.SOLR_HL_WEIGHT_MATCHES, **page_query_data, } - original_text_query = text_query + if use_hl: + kwargs["hl.q"] = hl_query if ( settings.SOLR_QUERY_NOTES and user.is_authenticated @@ -154,7 +158,6 @@ def search(user, query_params): text_query, note_params = _add_note_query(text_query, note_query, user) kwargs.update(note_params) kwargs["uf"] = "* _query_ -projects_edit_access" - kwargs["hl.q"] = original_text_query query_notes = True else: query_notes = False @@ -432,6 +435,17 @@ def visit_fuzzy(self, node, _parents): return node +class HighlightExtractor(NoteExtractor): + """Remove filter fields from a query, leaving the text to highlight""" + + def visit_search_field(self, node, _parents): + if node.name in FILTER_FIELDS.values() or any( + re.match(rf"^{p}$", node.name) for p in DYNAMIC_FILTER_FIELDS + ): + return None + return node + + class AnonymousTransformer(LuceneTreeTransformer): """Remove computational expesnive searches from anonymous queries""" @@ -465,12 +479,14 @@ def _parse(text_query, query_params, user): """Parse the text query and pull out filters and sorts Accepts a text query - Returns a tuple of (text_query, filters, sort, escaped, use_hl, note_query) + Returns a tuple of + (text_query, filters, sort, escaped, use_hl, note_query, hl_query) text_query - new text query with filters and sorts removed filters - a list of filters to be passed in to solr as field queries (`fq` field) sort - a string from the SORT_MAP to sort on note_query - the text query to search notes for, or None + hl_query - the text query to highlight, or None """ if text_query.strip(): try: @@ -492,7 +508,8 @@ def _parse(text_query, query_params, user): tree = AnonymousTransformer().visit(tree) new_query = str(tree) if tree is not None else "" - note_query = _extract_note_query(tree) + note_query = _extract_query(tree, NoteExtractor) + hl_query = _extract_query(tree, HighlightExtractor) filters = filter_extractor.filters sort = filter_extractor.sort # only use highilighting for queries with no fuzzy searches and @@ -506,19 +523,21 @@ def _parse(text_query, query_params, user): escaped = False use_hl = False note_query = None + hl_query = None # pull text queries from the parameters into the text query additional_text = _handle_params(query_params, TEXT_FIELDS, DYNAMIC_TEXT_FIELDS) if additional_text: # pylint: disable=consider-using-f-string new_query = "{} {}".format(new_query, " ".join(additional_text)) + hl_query = " ".join(([hl_query] if hl_query else []) + additional_text) # if nothing is left in the query after pulling out filters, default to *:* # which matches everything, otherwise convert the parse tree back to a text query if not new_query: new_query = "*:*" - return new_query, filters, sort, escaped, use_hl, note_query + return new_query, filters, sort, escaped, use_hl, note_query, hl_query def _filter_queries(user, query_params): @@ -909,12 +928,12 @@ def _expand_list(results, key, queryset, serializer): return results -def _extract_note_query(tree): - """Filter out all search fields""" +def _extract_query(tree, extractor): + """Remove nodes from a copy of the tree, returning what is left as a query""" if tree is None: return None # copy, as the extractor removes nodes in place - tree = NoteExtractor().visit(deepcopy(tree)) + tree = extractor().visit(deepcopy(tree)) if tree is None: return None return str(tree) diff --git a/documentcloud/documents/tests/search_data.py b/documentcloud/documents/tests/search_data.py index e2b9d71d..592868d9 100644 --- a/documentcloud/documents/tests/search_data.py +++ b/documentcloud/documents/tests/search_data.py @@ -101,6 +101,7 @@ "status": Status.success, "created_at": datetime(2011, 6, 1, tzinfo=pytz.utc), "data": {"edition": "first"}, + "pages": ["A quorx arrived with the first edition, and none since"], }, { "id": 9, diff --git a/documentcloud/documents/tests/test_search.py b/documentcloud/documents/tests/test_search.py index 457ed444..b4d80f34 100644 --- a/documentcloud/documents/tests/test_search.py +++ b/documentcloud/documents/tests/test_search.py @@ -23,7 +23,7 @@ FilterExtractor, NoteExtractor, NumberValidator, - _extract_note_query, + _extract_query, _parse, _parse_tree, _validate_range, @@ -57,7 +57,7 @@ def setup_solr(django_db_setup, django_db_blocker): `django_db_setup` causes pytest to initiate the test database """ - # pylint: disable=unused-argument + # pylint: disable=unused-argument, too-many-locals solr = pysolr.Solr(settings.SOLR_URL, auth=settings.SOLR_AUTH) solr_notes = pysolr.Solr(settings.SOLR_NOTES_URL, auth=settings.SOLR_AUTH) with django_db_blocker.unblock(): @@ -65,6 +65,7 @@ def setup_solr(django_db_setup, django_db_blocker): organizations = {} users = {} documents = {} + pages = {} notes = {} # this enables searching through notes for users in that org entitlement = OrganizationEntitlementFactory() @@ -82,6 +83,7 @@ def setup_solr(django_db_setup, django_db_blocker): doc = doc.copy() user = doc.pop("user") org = doc.pop("organization") + pages[doc["id"]] = doc.pop("pages", []) documents[doc["id"]] = DocumentFactory( user=users[user], organization=organizations[org], **doc ) @@ -106,7 +108,8 @@ def setup_solr(django_db_setup, django_db_blocker): edit_collaborators=[users[a] for a in proj["edit_collaborators"]], ) for doc in documents.values(): - solr.add([doc.solr()]) + page_text = {"pages": [{"contents": c} for c in pages[doc.pk]]} + solr.add([doc.solr(index_text=page_text)]) for note in notes.values(): solr_notes.add([note.solr()]) solr.commit() @@ -496,6 +499,26 @@ def test_search_notes_invalid_text_param(self, param): response = self.search(f"q=alice&{param}", user=2) assert [d["id"] for d in response["results"]] == ["1"] + @pytest.mark.parametrize("user", [None, 2]) + def test_search_highlight_skips_filters(self, user): + """Values of filter fields are not highlighted in the page text""" + + response = self.search( + "q=quorx AND (data_edition:first OR user:99)&hl=true", user=user + ) + assert [d["id"] for d in response["results"]] == ["8"] + highlights = str(response["results"][0]["highlights"]) + assert "quorx" in highlights + assert "first" not in highlights + + @pytest.mark.parametrize("user", [None, 2]) + def test_search_highlight_only_filters(self, user): + """A query of only filters has nothing to highlight""" + + response = self.search("q=user:3 AND data_edition:first&hl=true", user=user) + assert [d["id"] for d in response["results"]] == ["8"] + assert "" not in str(response["results"][0]["highlights"]) + def test_search_notes_highlight_only_fields(self): """A boolean query of only search fields leaves no text to search notes for""" @@ -711,11 +734,11 @@ class TestExtractNoteQuery: ], ) def test_extract_note_query(self, query, note_query): - assert _extract_note_query(_parse_tree(query)) == note_query + assert _extract_query(_parse_tree(query), NoteExtractor) == note_query def test_tree_unchanged(self): tree = _parse_tree("a AND user:1") - _extract_note_query(tree) + _extract_query(tree, NoteExtractor) assert str(tree) == "a AND user:1" @@ -798,6 +821,25 @@ def test_anon_parse( :5 ] == (new_query, QueryDict(filters), sort, escaped, use_hl) + @pytest.mark.parametrize( + "query,query_params,hl_query", + [ + ("", "", None), + ("user:1 AND tag:a", "", None), + ("foo AND (user:1 OR tag:email)", "", "foo"), + ("foo AND data_edition:first", "", "foo"), + ("foo AND user:bar", "", "foo"), + ("foo AND title:bar", "", "foo AND title:bar"), + ("text:foo AND user:1", "", "doctext:foo"), + ("", "title=bar", "title:(bar)"), + ("foo AND user:1", "title=bar", "foo title:(bar)"), + ], + ) + def test_highlight_query(self, query, query_params, hl_query): + user = Mock() + user.is_authenticated = True + assert _parse(query, QueryDict(query_params, mutable=True), user)[6] == hl_query + @pytest.mark.parametrize( "query,query_params,note_query", [ From 4553415ea4b533aa48ce9cc89901abbced71cd26 Mon Sep 17 00:00:00 2001 From: Mitchell Kotler Date: Thu, 5 Nov 2026 12:08:00 -0500 Subject: [PATCH 13/14] Send the text query to Solr as a dereferenced param for note searches For pro users the text query was inlined in parens ahead of the note join subqueries, so an unbalanced paren from a text param such as title=bar) closed the group early. Solr then reparsed the query leniently and flattened the required clauses of the organization note subquery into the top level, returning no results at all. Pass the text query as text_q and reference it with {!edismax v=$text_q}, as the note queries already are, so nothing a user types can change the structure around it. The test now also runs as a user with an edit project, where the organization note subquery is valid. Co-Authored-By: Claude Opus 5.5 --- documentcloud/documents/search.py | 8 ++++---- documentcloud/documents/tests/test_search.py | 5 +++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/documentcloud/documents/search.py b/documentcloud/documents/search.py index 6e2ceec3..d3a9ed40 100644 --- a/documentcloud/documents/search.py +++ b/documentcloud/documents/search.py @@ -956,18 +956,18 @@ def _add_note_query(text_query, note_query, user): if note_query is None: return text_query, {} - # the note queries are sent as separate params and dereferenced, as any - # quoting needed to inline them in local params is mangled by the user's - # own quotes and backslashes + # dereferenced params keep user input from mangling local params quoting, or + # unbalancing the parens of the clauses around it note_match = f"+(title:({note_query}) description:({note_query}))" note_params = { + "text_q": text_query, "note_q_public": f"+type:note +(access:public OR user:{user.pk}) {note_match}", "note_q_organization": f"+type:note +(access:organization) {note_match}", } return_query = ( # the original query to search for in documents - f"({text_query}) " + '_query_:"{!edismax v=$text_q}" ' # search through notes which are public or that you own # on all documents you can view f""" diff --git a/documentcloud/documents/tests/test_search.py b/documentcloud/documents/tests/test_search.py index b4d80f34..3fddd2df 100644 --- a/documentcloud/documents/tests/test_search.py +++ b/documentcloud/documents/tests/test_search.py @@ -493,10 +493,11 @@ def test_search_notes_highlight_multiple_terms(self, term, field): assert f"{term}" in highlight @pytest.mark.parametrize("param", ["title=bar)", "title=foo\\", "page_no_1="]) - def test_search_notes_invalid_text_param(self, param): + @pytest.mark.parametrize("user", [1, 2]) + def test_search_notes_invalid_text_param(self, param, user): """An unparseable text param must not break the note query""" - response = self.search(f"q=alice&{param}", user=2) + response = self.search(f"q=alice&{param}", user=user) assert [d["id"] for d in response["results"]] == ["1"] @pytest.mark.parametrize("user", [None, 2]) From 3a848f84a0c325f1613d9ef1bf8e4c0970e2dbc4 Mon Sep 17 00:00:00 2001 From: Mitchell Kotler Date: Thu, 5 Nov 2026 12:13:32 -0500 Subject: [PATCH 14/14] Leave out projects_edit_access when a user has no edit projects A pro user with no projects they can edit got projects_edit_access:() in the organization note subquery. That is invalid syntax, and Solr's lenient parsing silently dropped the whole subquery, so those users never matched organization access notes on documents they can edit. Only add the projects clause when there are projects. A new seeded organization note, written by another member on a document user 2 can edit, tests the case. Co-Authored-By: Claude Opus 5.5 --- documentcloud/documents/search.py | 6 +++++- documentcloud/documents/tests/search_data.py | 9 +++++++++ documentcloud/documents/tests/test_search.py | 3 +++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/documentcloud/documents/search.py b/documentcloud/documents/search.py index d3a9ed40..28496fb6 100644 --- a/documentcloud/documents/search.py +++ b/documentcloud/documents/search.py @@ -953,6 +953,10 @@ def _add_note_query(text_query, note_query, user): ) ) ) + # an empty projects_edit_access:() is invalid and drops the org note search + edit_access = f"user:{user.pk}" + if projects: + edit_access += f" OR projects_edit_access:({projects})" if note_query is None: return text_query, {} @@ -980,7 +984,7 @@ def _add_note_query(text_query, note_query, user): _query_:" +( ( - (user:{user.pk} OR projects_edit_access:({projects})) + ({edit_access}) AND access:(private organization) ) OR diff --git a/documentcloud/documents/tests/search_data.py b/documentcloud/documents/tests/search_data.py index 592868d9..5672c807 100644 --- a/documentcloud/documents/tests/search_data.py +++ b/documentcloud/documents/tests/search_data.py @@ -234,6 +234,15 @@ "content": "foxtrot", "access": Access.organization, }, + { + "id": 7, + "user": 1, + "organization": 1, + "document": 4, + "page_number": 0, + "content": "golf", + "access": Access.organization, + }, ] diff --git a/documentcloud/documents/tests/test_search.py b/documentcloud/documents/tests/test_search.py index 3fddd2df..df28425a 100644 --- a/documentcloud/documents/tests/test_search.py +++ b/documentcloud/documents/tests/test_search.py @@ -450,6 +450,9 @@ def test_search_notes_returned(self, user, doc, notes): (7, 3, True), (7, 4, False), (7, 5, True), + # org member without edit projects may search org notes + (2, 6, True), + (3, 6, False), ], ) def test_search_notes_content(self, user, note, viewable):