diff --git a/documentcloud/documents/search.py b/documentcloud/documents/search.py index e8faf3a2..28496fb6 100644 --- a/documentcloud/documents/search.py +++ b/documentcloud/documents/search.py @@ -6,7 +6,9 @@ import math import re from collections import defaultdict +from copy import deepcopy from datetime import datetime +from decimal import InvalidOperation # Third Party import pysolr @@ -61,6 +63,14 @@ } DYNAMIC_TEXT_FIELDS = [r"page_no_[0-9]+"] +# luqum raises InvalidOperation or ValueError when it fails to read the number in +# a boost, fuzzy or proximity search +PARSE_ERRORS = (ParseError, TypeError, InvalidOperation, ValueError) + +# matches escapes and phrases, so a quote inside a phrase is never mistaken for one +# which ends it, capturing any fractional proximity slop after a phrase +PHRASE_SLOP_RE = re.compile(r'\\.|("(?:[^"\\]|\\.)*")(?:~(\d*)\.[\d.]*)?') + SORT_MAP = { "score": "score desc, created_at desc, id desc", "created_at": "created_at desc, id desc", @@ -98,8 +108,8 @@ def search(user, query_params): # pylint: disable=too-many-locals text_query = query_params.get("q", "") - text_query, filter_params, sort_order, escaped, use_hl = _parse( - text_query, query_params, user + text_query, filter_params, sort_order, escaped, use_hl, note_query, hl_query = ( + _parse(text_query, query_params, user) ) filter_params.update(query_params) @@ -123,6 +133,9 @@ def search(user, query_params): # allow explicit enabling of highlighting if query_params.get("hl", "").lower() == "true" and settings.SOLR_USE_HL: use_hl = True + if hl_query is None: + # nothing but filters, which should not be highlighted + use_hl = False kwargs = { "fq": filter_queries, @@ -132,7 +145,8 @@ def search(user, query_params): "hl.weightMatches": settings.SOLR_HL_WEIGHT_MATCHES, **page_query_data, } - original_text_query = text_query + if use_hl: + kwargs["hl.q"] = hl_query if ( settings.SOLR_QUERY_NOTES and user.is_authenticated @@ -141,9 +155,9 @@ def search(user, query_params): ): # turn note queries on for all pro users # *:* returns all documents, do not enable note queries - text_query = _add_note_query(text_query, user) + text_query, note_params = _add_note_query(text_query, note_query, user) + kwargs.update(note_params) kwargs["uf"] = "* _query_ -projects_edit_access" - kwargs["hl.q"] = original_text_query query_notes = True else: query_notes = False @@ -171,7 +185,7 @@ def search(user, query_params): results, query_params, user, escaped, page_response_data ) if use_hl and query_notes: - response = _highlight_notes(response, original_text_query) + response = _highlight_notes(response, note_query) if settings.DEBUG or user.is_staff: response["debug"] = {"text_query": text_query, "qtime": results.qtime, **kwargs} @@ -393,12 +407,13 @@ class NoteExtractor(LuceneTreeTransformer): def visit(self, node, parents=None): new_node = super().visit(node, parents) - if ( - isinstance(new_node, (Group, BaseOperation, Unary)) - and not any(node.children) - and parents + if isinstance(new_node, (Boost, Group, BaseOperation, Unary)) and not any( + node.children ): - self.replace_node(new_node, None, parents[-1]) + # an emptied root must also be dropped, or it serializes to an + # empty or "(None)" query which Solr cannot parse + if parents: + self.replace_node(new_node, None, parents[-1]) return None return new_node @@ -406,6 +421,31 @@ def visit_search_field(self, _node, _parents): return None +class NumberFormatter(LuceneTreeTransformer): + """Write boosts and fuzzy degrees in plain notation, as luqum normalizes them + to exponents (^50 becomes ^5E+1), which Solr reads as ^5 plus the term E+1 + """ + + def visit_boost(self, node, _parents): + node.force = format(node.force, "f") + return node + + def visit_fuzzy(self, node, _parents): + node.degree = format(node.degree, "f") + return node + + +class HighlightExtractor(NoteExtractor): + """Remove filter fields from a query, leaving the text to highlight""" + + def visit_search_field(self, node, _parents): + if node.name in FILTER_FIELDS.values() or any( + re.match(rf"^{p}$", node.name) for p in DYNAMIC_FILTER_FIELDS + ): + return None + return node + + class AnonymousTransformer(LuceneTreeTransformer): """Remove computational expesnive searches from anonymous queries""" @@ -420,21 +460,39 @@ def visit_term(self, node, _parents): return node +def _truncate_slop(match): + phrase, slop = match.groups() + if slop is None: + return match.group(0) + return f"{phrase}~{slop}" if slop else phrase + + +def _parse_tree(text_query): + """Parse a text query into a tree that serializes back to valid Solr syntax""" + # Solr truncates a fractional phrase slop to a whole number, but luqum fails + # to parse one + text_query = PHRASE_SLOP_RE.sub(_truncate_slop, text_query) + return NumberFormatter().visit(parser.parse(text_query)) + + def _parse(text_query, query_params, user): """Parse the text query and pull out filters and sorts Accepts a text query - Returns a tuple of (text_query, filters, sort) + Returns a tuple of + (text_query, filters, sort, escaped, use_hl, note_query, hl_query) text_query - new text query with filters and sorts removed filters - a list of filters to be passed in to solr as field queries (`fq` field) sort - a string from the SORT_MAP to sort on + note_query - the text query to search notes for, or None + hl_query - the text query to highlight, or None """ if text_query.strip(): try: - tree = parser.parse(text_query) + tree = _parse_tree(text_query) escaped = False - except (ParseError, TypeError): + except PARSE_ERRORS: tree = parser.parse(escape(text_query)) escaped = True @@ -450,6 +508,8 @@ def _parse(text_query, query_params, user): tree = AnonymousTransformer().visit(tree) new_query = str(tree) if tree is not None else "" + note_query = _extract_query(tree, NoteExtractor) + hl_query = _extract_query(tree, HighlightExtractor) filters = filter_extractor.filters sort = filter_extractor.sort # only use highilighting for queries with no fuzzy searches and @@ -462,19 +522,22 @@ def _parse(text_query, query_params, user): sort = None escaped = False use_hl = False + note_query = None + hl_query = None # pull text queries from the parameters into the text query additional_text = _handle_params(query_params, TEXT_FIELDS, DYNAMIC_TEXT_FIELDS) if additional_text: # pylint: disable=consider-using-f-string new_query = "{} {}".format(new_query, " ".join(additional_text)) + hl_query = " ".join(([hl_query] if hl_query else []) + additional_text) # if nothing is left in the query after pulling out filters, default to *:* # which matches everything, otherwise convert the parse tree back to a text query if not new_query: new_query = "*:*" - return new_query, filters, sort, escaped, use_hl + return new_query, filters, sort, escaped, use_hl, note_query, hl_query def _filter_queries(user, query_params): @@ -493,7 +556,10 @@ def _validate_range(value, validator): """Validate using the validator as well as ensuring the top level node is either a word or range """ - node = parser.parse(value) + try: + node = parser.parse(value) + except PARSE_ERRORS: + return False return all(validator.visit(node)) and isinstance(node, (Word, Range)) @@ -742,7 +808,7 @@ def format_note(result): return [format_note(r) for r in results] -def _highlight_notes(response, text_query): +def _highlight_notes(response, note_query): """Do a second query to highlight notes if necessary""" notes = [n for d in response["results"] for n in d["notes"]] note_ids = [n["id"] for n in notes] @@ -751,10 +817,9 @@ def _highlight_notes(response, text_query): parents = {n["id"]: d["id"] for d in response["results"] for n in d["notes"]} fq_ = [f"id:({' '.join(note_ids)})"] - note_query = _extract_note_query(text_query) if note_query is None: return response - query = f"title:{note_query} description:{note_query}" + query = f"title:({note_query}) description:({note_query})" kwargs = { "fq": fq_, @@ -863,17 +928,21 @@ def _expand_list(results, key, queryset, serializer): return results -def _extract_note_query(text_query): - """Filter out all search fields""" - tree = parser.parse(text_query) - note_extractor = NoteExtractor() - tree = note_extractor.visit(tree) +def _extract_query(tree, extractor): + """Remove nodes from a copy of the tree, returning what is left as a query""" + if tree is None: + return None + # copy, as the extractor removes nodes in place + tree = extractor().visit(deepcopy(tree)) if tree is None: return None return str(tree) -def _add_note_query(text_query, user): +def _add_note_query(text_query, note_query, user): + """Add note searches to the text query, returning the new query and the + request params it references + """ organizations = " ".join(str(o.pk) for o in user.organizations.all()) projects = " ".join( str(p.pk) @@ -884,22 +953,30 @@ def _add_note_query(text_query, user): ) ) ) + # an empty projects_edit_access:() is invalid and drops the org note search + edit_access = f"user:{user.pk}" + if projects: + edit_access += f" OR projects_edit_access:({projects})" - note_query = _extract_note_query(text_query) if note_query is None: - return text_query - escaped_note_query = note_query.replace('"', '\\"') + return text_query, {} + # dereferenced params keep user input from mangling local params quoting, or + # unbalancing the parens of the clauses around it + note_match = f"+(title:({note_query}) description:({note_query}))" + note_params = { + "text_q": text_query, + "note_q_public": f"+type:note +(access:public OR user:{user.pk}) {note_match}", + "note_q_organization": f"+type:note +(access:organization) {note_match}", + } return_query = ( # the original query to search for in documents - f"({text_query}) " + '_query_:"{!edismax v=$text_q}" ' # search through notes which are public or that you own # on all documents you can view f""" _query_:"{{!join from=document_s fromIndex=notes to=id score=total - v='+type:note +(access:public OR user:{user.pk}) - +(title:({escaped_note_query}) description:({escaped_note_query}))' - }}" + v=$note_q_public}}" """ # search through notes which are organization access # on all documents you can edit @@ -907,18 +984,17 @@ def _add_note_query(text_query, user): _query_:" +( ( - (user:{user.pk} OR projects_edit_access:({projects})) + ({edit_access}) AND access:(private organization) ) OR (access:(public organization) AND organization:({organizations})) ) +{{!join from=document_s fromIndex=notes to=id score=total - v='+type:note +(access:organization) - +(title:({escaped_note_query}) description:({escaped_note_query}))'}} + v=$note_q_organization}} " """ ) # replace runs of white space with a single space - keeps the template readable # without incurring overhead of sending extra white space to solr - return re.sub(r"\s+", " ", return_query) + return re.sub(r"\s+", " ", return_query), note_params diff --git a/documentcloud/documents/tests/search_data.py b/documentcloud/documents/tests/search_data.py index e2b9d71d..5672c807 100644 --- a/documentcloud/documents/tests/search_data.py +++ b/documentcloud/documents/tests/search_data.py @@ -101,6 +101,7 @@ "status": Status.success, "created_at": datetime(2011, 6, 1, tzinfo=pytz.utc), "data": {"edition": "first"}, + "pages": ["A quorx arrived with the first edition, and none since"], }, { "id": 9, @@ -233,6 +234,15 @@ "content": "foxtrot", "access": Access.organization, }, + { + "id": 7, + "user": 1, + "organization": 1, + "document": 4, + "page_number": 0, + "content": "golf", + "access": Access.organization, + }, ] diff --git a/documentcloud/documents/tests/test_search.py b/documentcloud/documents/tests/test_search.py index 166bbadf..df28425a 100644 --- a/documentcloud/documents/tests/test_search.py +++ b/documentcloud/documents/tests/test_search.py @@ -22,7 +22,11 @@ DateValidator, FilterExtractor, NoteExtractor, + NumberValidator, + _extract_query, _parse, + _parse_tree, + _validate_range, search, ) from documentcloud.documents.search_escape import escape @@ -53,7 +57,7 @@ def setup_solr(django_db_setup, django_db_blocker): `django_db_setup` causes pytest to initiate the test database """ - # pylint: disable=unused-argument + # pylint: disable=unused-argument, too-many-locals solr = pysolr.Solr(settings.SOLR_URL, auth=settings.SOLR_AUTH) solr_notes = pysolr.Solr(settings.SOLR_NOTES_URL, auth=settings.SOLR_AUTH) with django_db_blocker.unblock(): @@ -61,6 +65,7 @@ def setup_solr(django_db_setup, django_db_blocker): organizations = {} users = {} documents = {} + pages = {} notes = {} # this enables searching through notes for users in that org entitlement = OrganizationEntitlementFactory() @@ -78,6 +83,7 @@ def setup_solr(django_db_setup, django_db_blocker): doc = doc.copy() user = doc.pop("user") org = doc.pop("organization") + pages[doc["id"]] = doc.pop("pages", []) documents[doc["id"]] = DocumentFactory( user=users[user], organization=organizations[org], **doc ) @@ -102,7 +108,8 @@ def setup_solr(django_db_setup, django_db_blocker): edit_collaborators=[users[a] for a in proj["edit_collaborators"]], ) for doc in documents.values(): - solr.add([doc.solr()]) + page_text = {"pages": [{"contents": c} for c in pages[doc.pk]]} + solr.add([doc.solr(index_text=page_text)]) for note in notes.values(): solr_notes.add([note.solr()]) solr.commit() @@ -443,6 +450,9 @@ def test_search_notes_returned(self, user, doc, notes): (7, 3, True), (7, 4, False), (7, 5, True), + # org member without edit projects may search org notes + (2, 6, True), + (3, 6, False), ], ) def test_search_notes_content(self, user, note, viewable): @@ -465,6 +475,61 @@ def test_search_notes_escape_quotes(self): response = self.search('q="delta"', user=1) assert len(response["results"]) == 1 + @pytest.mark.parametrize( + "query", ["alice OR don't", '"alice" OR "don\'t"', "alice OR don\\'t"] + ) + def test_search_notes_apostrophe(self, query): + """Apostrophes must not break out of the note subqueries""" + + response = self.search(f"q={query}", user=2) + assert [d["id"] for d in response["results"]] == ["1"] + + @pytest.mark.parametrize( + "term,field", [("alice", "description"), ("note", "title")] + ) + def test_search_notes_highlight_multiple_terms(self, term, field): + """Every term of the note query is highlighted in each note field""" + + response = self.search(f"q=zzz OR {term}&hl=true", user=2) + document = next(d for d in response["results"] if d["id"] == "1") + highlight = str(document["note_highlights"]["1"][field]).lower() + assert f"{term}" in highlight + + @pytest.mark.parametrize("param", ["title=bar)", "title=foo\\", "page_no_1="]) + @pytest.mark.parametrize("user", [1, 2]) + def test_search_notes_invalid_text_param(self, param, user): + """An unparseable text param must not break the note query""" + + response = self.search(f"q=alice&{param}", user=user) + assert [d["id"] for d in response["results"]] == ["1"] + + @pytest.mark.parametrize("user", [None, 2]) + def test_search_highlight_skips_filters(self, user): + """Values of filter fields are not highlighted in the page text""" + + response = self.search( + "q=quorx AND (data_edition:first OR user:99)&hl=true", user=user + ) + assert [d["id"] for d in response["results"]] == ["8"] + highlights = str(response["results"][0]["highlights"]) + assert "quorx" in highlights + assert "first" not in highlights + + @pytest.mark.parametrize("user", [None, 2]) + def test_search_highlight_only_filters(self, user): + """A query of only filters has nothing to highlight""" + + response = self.search("q=user:3 AND data_edition:first&hl=true", user=user) + assert [d["id"] for d in response["results"]] == ["8"] + assert "" not in str(response["results"][0]["highlights"]) + + def test_search_notes_highlight_only_fields(self): + """A boolean query of only search fields leaves no text to search notes for""" + + response = self.search("q=id:1 AND access:public&hl=true", user=2) + assert [d["id"] for d in response["results"]] == ["1"] + assert response["results"][0]["notes"] + class TestBooleanDetector: @pytest.mark.parametrize( @@ -519,6 +584,20 @@ def test_validator(self, query, valid): assert all(DateValidator().visit(tree)) is valid +class TestValidateRange: + @pytest.mark.parametrize( + "value,validator", + [ + ("1^", NumberValidator()), + ("1^1.2.3", NumberValidator()), + ('"1 2"~1.5', NumberValidator()), + ('"NOW 2"~1.5', DateValidator()), + ], + ) + def test_unparseable(self, value, validator): + assert _validate_range(value, validator) is False + + class TestFilterExtractor: @pytest.mark.parametrize( "query,new_query,filters,sort_only,sort", @@ -643,6 +722,30 @@ def test_extract_note(self, query, new_query): assert new_query == (str(tree) if tree else "") +class TestExtractNoteQuery: + @pytest.mark.parametrize( + "query,note_query", + [ + ('"a" AND "b"^50', '"a" AND "b"^50'), + ("foo~10", "foo~10"), + ("user:1 AND tag:a", None), + ('user:1 AND +data__tag:"a"', None), + ("(user:1 AND tag:a)", None), + ("(user:1) OR (tag:a)", None), + ("user:1^2 AND tag:a", None), + ("foo OR (user:1)^2", "foo"), + ("(user:1 AND tag:a)^2 foo", "foo"), + ], + ) + def test_extract_note_query(self, query, note_query): + assert _extract_query(_parse_tree(query), NoteExtractor) == note_query + + def test_tree_unchanged(self): + tree = _parse_tree("a AND user:1") + _extract_query(tree, NoteExtractor) + assert str(tree) == "a AND user:1" + + class TestParse: @pytest.mark.parametrize( "query,query_params,new_query,filters,sort,escaped,use_hl", @@ -677,6 +780,18 @@ class TestParse: ("foo~1", "", "foo~1", "", None, False, False), ("foo hl:true", "", "foo", "", None, False, True), ("foo~1 hl:true", "", "foo~1", "", None, False, False), + ("foo^50", "", "foo^50", "", None, False, False), + ('"a" AND "b"^100', "", '"a" AND "b"^100', "", None, False, False), + ("foo~10", "", "foo~10", "", None, False, False), + ("foo^1.2.3", "", "foo\\^1.2.3", "", None, True, False), + ("foo^.", "", "foo\\^.", "", None, True, False), + ("foo~1.2.3", "", "foo\\~1.2.3", "", None, True, False), + ('"a b"~1.5', "", '"a b"~1', "", None, False, False), + ('"a b"~.', "", '"a b"', "", None, False, False), + ('"a b"~0.5 c', "", '"a b"~0 c', "", None, False, False), + ('"x" c "~2.5"~3.5', "", '"x" c "~2.5"~3', "", None, False, False), + ('"x \\" y"~1.5', "", '"x \\" y"~1', "", None, False, False), + ('\\" "a b"~1.5', "", '\\" "a b"~1', "", None, False, False), ], ) def test_auth_parse( @@ -685,7 +800,7 @@ def test_auth_parse( # pylint: disable=too-many-arguments user = Mock() user.is_authenticated = True - assert _parse(query, QueryDict(query_params, mutable=True), user) == ( + assert _parse(query, QueryDict(query_params, mutable=True), user)[:5] == ( new_query, QueryDict(filters), sort, @@ -706,9 +821,48 @@ def test_anon_parse( self, query, query_params, new_query, filters, sort, escaped, use_hl ): # pylint: disable=too-many-arguments - assert _parse( - query, QueryDict(query_params, mutable=True), AnonymousUser() - ) == (new_query, QueryDict(filters), sort, escaped, use_hl) + assert _parse(query, QueryDict(query_params, mutable=True), AnonymousUser())[ + :5 + ] == (new_query, QueryDict(filters), sort, escaped, use_hl) + + @pytest.mark.parametrize( + "query,query_params,hl_query", + [ + ("", "", None), + ("user:1 AND tag:a", "", None), + ("foo AND (user:1 OR tag:email)", "", "foo"), + ("foo AND data_edition:first", "", "foo"), + ("foo AND user:bar", "", "foo"), + ("foo AND title:bar", "", "foo AND title:bar"), + ("text:foo AND user:1", "", "doctext:foo"), + ("", "title=bar", "title:(bar)"), + ("foo AND user:1", "title=bar", "foo title:(bar)"), + ], + ) + def test_highlight_query(self, query, query_params, hl_query): + user = Mock() + user.is_authenticated = True + assert _parse(query, QueryDict(query_params, mutable=True), user)[6] == hl_query + + @pytest.mark.parametrize( + "query,query_params,note_query", + [ + ("", "", None), + ("", "title=bar", None), + ("user:1 AND tag:a", "", None), + ("a user:1", "", "a"), + ("a", "title=bar)", "a"), + ("a", "title=foo\\", "a"), + ("a", "page_no_1=", "a"), + ('"a" AND "b"^50', "", '"a" AND "b"^50'), + ], + ) + def test_note_query(self, query, query_params, note_query): + user = Mock() + user.is_authenticated = True + assert ( + _parse(query, QueryDict(query_params, mutable=True), user)[5] == note_query + ) class TestEscape: