diff --git a/documentcloud/documents/search.py b/documentcloud/documents/search.py
index e8faf3a2..28496fb6 100644
--- a/documentcloud/documents/search.py
+++ b/documentcloud/documents/search.py
@@ -6,7 +6,9 @@
import math
import re
from collections import defaultdict
+from copy import deepcopy
from datetime import datetime
+from decimal import InvalidOperation
# Third Party
import pysolr
@@ -61,6 +63,14 @@
}
DYNAMIC_TEXT_FIELDS = [r"page_no_[0-9]+"]
+# luqum raises InvalidOperation or ValueError when it fails to read the number in
+# a boost, fuzzy or proximity search
+PARSE_ERRORS = (ParseError, TypeError, InvalidOperation, ValueError)
+
+# matches escapes and phrases, so a quote inside a phrase is never mistaken for one
+# which ends it, capturing any fractional proximity slop after a phrase
+PHRASE_SLOP_RE = re.compile(r'\\.|("(?:[^"\\]|\\.)*")(?:~(\d*)\.[\d.]*)?')
+
SORT_MAP = {
"score": "score desc, created_at desc, id desc",
"created_at": "created_at desc, id desc",
@@ -98,8 +108,8 @@ def search(user, query_params):
# pylint: disable=too-many-locals
text_query = query_params.get("q", "")
- text_query, filter_params, sort_order, escaped, use_hl = _parse(
- text_query, query_params, user
+ text_query, filter_params, sort_order, escaped, use_hl, note_query, hl_query = (
+ _parse(text_query, query_params, user)
)
filter_params.update(query_params)
@@ -123,6 +133,9 @@ def search(user, query_params):
# allow explicit enabling of highlighting
if query_params.get("hl", "").lower() == "true" and settings.SOLR_USE_HL:
use_hl = True
+ if hl_query is None:
+ # nothing but filters, which should not be highlighted
+ use_hl = False
kwargs = {
"fq": filter_queries,
@@ -132,7 +145,8 @@ def search(user, query_params):
"hl.weightMatches": settings.SOLR_HL_WEIGHT_MATCHES,
**page_query_data,
}
- original_text_query = text_query
+ if use_hl:
+ kwargs["hl.q"] = hl_query
if (
settings.SOLR_QUERY_NOTES
and user.is_authenticated
@@ -141,9 +155,9 @@ def search(user, query_params):
):
# turn note queries on for all pro users
# *:* returns all documents, do not enable note queries
- text_query = _add_note_query(text_query, user)
+ text_query, note_params = _add_note_query(text_query, note_query, user)
+ kwargs.update(note_params)
kwargs["uf"] = "* _query_ -projects_edit_access"
- kwargs["hl.q"] = original_text_query
query_notes = True
else:
query_notes = False
@@ -171,7 +185,7 @@ def search(user, query_params):
results, query_params, user, escaped, page_response_data
)
if use_hl and query_notes:
- response = _highlight_notes(response, original_text_query)
+ response = _highlight_notes(response, note_query)
if settings.DEBUG or user.is_staff:
response["debug"] = {"text_query": text_query, "qtime": results.qtime, **kwargs}
@@ -393,12 +407,13 @@ class NoteExtractor(LuceneTreeTransformer):
def visit(self, node, parents=None):
new_node = super().visit(node, parents)
- if (
- isinstance(new_node, (Group, BaseOperation, Unary))
- and not any(node.children)
- and parents
+ if isinstance(new_node, (Boost, Group, BaseOperation, Unary)) and not any(
+ node.children
):
- self.replace_node(new_node, None, parents[-1])
+ # an emptied root must also be dropped, or it serializes to an
+ # empty or "(None)" query which Solr cannot parse
+ if parents:
+ self.replace_node(new_node, None, parents[-1])
return None
return new_node
@@ -406,6 +421,31 @@ def visit_search_field(self, _node, _parents):
return None
+class NumberFormatter(LuceneTreeTransformer):
+ """Write boosts and fuzzy degrees in plain notation, as luqum normalizes them
+ to exponents (^50 becomes ^5E+1), which Solr reads as ^5 plus the term E+1
+ """
+
+ def visit_boost(self, node, _parents):
+ node.force = format(node.force, "f")
+ return node
+
+ def visit_fuzzy(self, node, _parents):
+ node.degree = format(node.degree, "f")
+ return node
+
+
+class HighlightExtractor(NoteExtractor):
+ """Remove filter fields from a query, leaving the text to highlight"""
+
+ def visit_search_field(self, node, _parents):
+ if node.name in FILTER_FIELDS.values() or any(
+ re.match(rf"^{p}$", node.name) for p in DYNAMIC_FILTER_FIELDS
+ ):
+ return None
+ return node
+
+
class AnonymousTransformer(LuceneTreeTransformer):
"""Remove computational expesnive searches from anonymous queries"""
@@ -420,21 +460,39 @@ def visit_term(self, node, _parents):
return node
+def _truncate_slop(match):
+ phrase, slop = match.groups()
+ if slop is None:
+ return match.group(0)
+ return f"{phrase}~{slop}" if slop else phrase
+
+
+def _parse_tree(text_query):
+ """Parse a text query into a tree that serializes back to valid Solr syntax"""
+ # Solr truncates a fractional phrase slop to a whole number, but luqum fails
+ # to parse one
+ text_query = PHRASE_SLOP_RE.sub(_truncate_slop, text_query)
+ return NumberFormatter().visit(parser.parse(text_query))
+
+
def _parse(text_query, query_params, user):
"""Parse the text query and pull out filters and sorts
Accepts a text query
- Returns a tuple of (text_query, filters, sort)
+ Returns a tuple of
+ (text_query, filters, sort, escaped, use_hl, note_query, hl_query)
text_query - new text query with filters and sorts removed
filters - a list of filters to be passed in to solr as field queries
(`fq` field)
sort - a string from the SORT_MAP to sort on
+ note_query - the text query to search notes for, or None
+ hl_query - the text query to highlight, or None
"""
if text_query.strip():
try:
- tree = parser.parse(text_query)
+ tree = _parse_tree(text_query)
escaped = False
- except (ParseError, TypeError):
+ except PARSE_ERRORS:
tree = parser.parse(escape(text_query))
escaped = True
@@ -450,6 +508,8 @@ def _parse(text_query, query_params, user):
tree = AnonymousTransformer().visit(tree)
new_query = str(tree) if tree is not None else ""
+ note_query = _extract_query(tree, NoteExtractor)
+ hl_query = _extract_query(tree, HighlightExtractor)
filters = filter_extractor.filters
sort = filter_extractor.sort
# only use highilighting for queries with no fuzzy searches and
@@ -462,19 +522,22 @@ def _parse(text_query, query_params, user):
sort = None
escaped = False
use_hl = False
+ note_query = None
+ hl_query = None
# pull text queries from the parameters into the text query
additional_text = _handle_params(query_params, TEXT_FIELDS, DYNAMIC_TEXT_FIELDS)
if additional_text:
# pylint: disable=consider-using-f-string
new_query = "{} {}".format(new_query, " ".join(additional_text))
+ hl_query = " ".join(([hl_query] if hl_query else []) + additional_text)
# if nothing is left in the query after pulling out filters, default to *:*
# which matches everything, otherwise convert the parse tree back to a text query
if not new_query:
new_query = "*:*"
- return new_query, filters, sort, escaped, use_hl
+ return new_query, filters, sort, escaped, use_hl, note_query, hl_query
def _filter_queries(user, query_params):
@@ -493,7 +556,10 @@ def _validate_range(value, validator):
"""Validate using the validator as well as ensuring the top level node
is either a word or range
"""
- node = parser.parse(value)
+ try:
+ node = parser.parse(value)
+ except PARSE_ERRORS:
+ return False
return all(validator.visit(node)) and isinstance(node, (Word, Range))
@@ -742,7 +808,7 @@ def format_note(result):
return [format_note(r) for r in results]
-def _highlight_notes(response, text_query):
+def _highlight_notes(response, note_query):
"""Do a second query to highlight notes if necessary"""
notes = [n for d in response["results"] for n in d["notes"]]
note_ids = [n["id"] for n in notes]
@@ -751,10 +817,9 @@ def _highlight_notes(response, text_query):
parents = {n["id"]: d["id"] for d in response["results"] for n in d["notes"]}
fq_ = [f"id:({' '.join(note_ids)})"]
- note_query = _extract_note_query(text_query)
if note_query is None:
return response
- query = f"title:{note_query} description:{note_query}"
+ query = f"title:({note_query}) description:({note_query})"
kwargs = {
"fq": fq_,
@@ -863,17 +928,21 @@ def _expand_list(results, key, queryset, serializer):
return results
-def _extract_note_query(text_query):
- """Filter out all search fields"""
- tree = parser.parse(text_query)
- note_extractor = NoteExtractor()
- tree = note_extractor.visit(tree)
+def _extract_query(tree, extractor):
+ """Remove nodes from a copy of the tree, returning what is left as a query"""
+ if tree is None:
+ return None
+ # copy, as the extractor removes nodes in place
+ tree = extractor().visit(deepcopy(tree))
if tree is None:
return None
return str(tree)
-def _add_note_query(text_query, user):
+def _add_note_query(text_query, note_query, user):
+ """Add note searches to the text query, returning the new query and the
+ request params it references
+ """
organizations = " ".join(str(o.pk) for o in user.organizations.all())
projects = " ".join(
str(p.pk)
@@ -884,22 +953,30 @@ def _add_note_query(text_query, user):
)
)
)
+ # an empty projects_edit_access:() is invalid and drops the org note search
+ edit_access = f"user:{user.pk}"
+ if projects:
+ edit_access += f" OR projects_edit_access:({projects})"
- note_query = _extract_note_query(text_query)
if note_query is None:
- return text_query
- escaped_note_query = note_query.replace('"', '\\"')
+ return text_query, {}
+ # dereferenced params keep user input from mangling local params quoting, or
+ # unbalancing the parens of the clauses around it
+ note_match = f"+(title:({note_query}) description:({note_query}))"
+ note_params = {
+ "text_q": text_query,
+ "note_q_public": f"+type:note +(access:public OR user:{user.pk}) {note_match}",
+ "note_q_organization": f"+type:note +(access:organization) {note_match}",
+ }
return_query = (
# the original query to search for in documents
- f"({text_query}) "
+ '_query_:"{!edismax v=$text_q}" '
# search through notes which are public or that you own
# on all documents you can view
f"""
_query_:"{{!join from=document_s fromIndex=notes to=id score=total
- v='+type:note +(access:public OR user:{user.pk})
- +(title:({escaped_note_query}) description:({escaped_note_query}))'
- }}"
+ v=$note_q_public}}"
"""
# search through notes which are organization access
# on all documents you can edit
@@ -907,18 +984,17 @@ def _add_note_query(text_query, user):
_query_:"
+(
(
- (user:{user.pk} OR projects_edit_access:({projects}))
+ ({edit_access})
AND access:(private organization)
)
OR
(access:(public organization) AND organization:({organizations}))
)
+{{!join from=document_s fromIndex=notes to=id score=total
- v='+type:note +(access:organization)
- +(title:({escaped_note_query}) description:({escaped_note_query}))'}}
+ v=$note_q_organization}}
"
"""
)
# replace runs of white space with a single space - keeps the template readable
# without incurring overhead of sending extra white space to solr
- return re.sub(r"\s+", " ", return_query)
+ return re.sub(r"\s+", " ", return_query), note_params
diff --git a/documentcloud/documents/tests/search_data.py b/documentcloud/documents/tests/search_data.py
index e2b9d71d..5672c807 100644
--- a/documentcloud/documents/tests/search_data.py
+++ b/documentcloud/documents/tests/search_data.py
@@ -101,6 +101,7 @@
"status": Status.success,
"created_at": datetime(2011, 6, 1, tzinfo=pytz.utc),
"data": {"edition": "first"},
+ "pages": ["A quorx arrived with the first edition, and none since"],
},
{
"id": 9,
@@ -233,6 +234,15 @@
"content": "foxtrot",
"access": Access.organization,
},
+ {
+ "id": 7,
+ "user": 1,
+ "organization": 1,
+ "document": 4,
+ "page_number": 0,
+ "content": "golf",
+ "access": Access.organization,
+ },
]
diff --git a/documentcloud/documents/tests/test_search.py b/documentcloud/documents/tests/test_search.py
index 166bbadf..df28425a 100644
--- a/documentcloud/documents/tests/test_search.py
+++ b/documentcloud/documents/tests/test_search.py
@@ -22,7 +22,11 @@
DateValidator,
FilterExtractor,
NoteExtractor,
+ NumberValidator,
+ _extract_query,
_parse,
+ _parse_tree,
+ _validate_range,
search,
)
from documentcloud.documents.search_escape import escape
@@ -53,7 +57,7 @@ def setup_solr(django_db_setup, django_db_blocker):
`django_db_setup` causes pytest to initiate the test database
"""
- # pylint: disable=unused-argument
+ # pylint: disable=unused-argument, too-many-locals
solr = pysolr.Solr(settings.SOLR_URL, auth=settings.SOLR_AUTH)
solr_notes = pysolr.Solr(settings.SOLR_NOTES_URL, auth=settings.SOLR_AUTH)
with django_db_blocker.unblock():
@@ -61,6 +65,7 @@ def setup_solr(django_db_setup, django_db_blocker):
organizations = {}
users = {}
documents = {}
+ pages = {}
notes = {}
# this enables searching through notes for users in that org
entitlement = OrganizationEntitlementFactory()
@@ -78,6 +83,7 @@ def setup_solr(django_db_setup, django_db_blocker):
doc = doc.copy()
user = doc.pop("user")
org = doc.pop("organization")
+ pages[doc["id"]] = doc.pop("pages", [])
documents[doc["id"]] = DocumentFactory(
user=users[user], organization=organizations[org], **doc
)
@@ -102,7 +108,8 @@ def setup_solr(django_db_setup, django_db_blocker):
edit_collaborators=[users[a] for a in proj["edit_collaborators"]],
)
for doc in documents.values():
- solr.add([doc.solr()])
+ page_text = {"pages": [{"contents": c} for c in pages[doc.pk]]}
+ solr.add([doc.solr(index_text=page_text)])
for note in notes.values():
solr_notes.add([note.solr()])
solr.commit()
@@ -443,6 +450,9 @@ def test_search_notes_returned(self, user, doc, notes):
(7, 3, True),
(7, 4, False),
(7, 5, True),
+ # org member without edit projects may search org notes
+ (2, 6, True),
+ (3, 6, False),
],
)
def test_search_notes_content(self, user, note, viewable):
@@ -465,6 +475,61 @@ def test_search_notes_escape_quotes(self):
response = self.search('q="delta"', user=1)
assert len(response["results"]) == 1
+ @pytest.mark.parametrize(
+ "query", ["alice OR don't", '"alice" OR "don\'t"', "alice OR don\\'t"]
+ )
+ def test_search_notes_apostrophe(self, query):
+ """Apostrophes must not break out of the note subqueries"""
+
+ response = self.search(f"q={query}", user=2)
+ assert [d["id"] for d in response["results"]] == ["1"]
+
+ @pytest.mark.parametrize(
+ "term,field", [("alice", "description"), ("note", "title")]
+ )
+ def test_search_notes_highlight_multiple_terms(self, term, field):
+ """Every term of the note query is highlighted in each note field"""
+
+ response = self.search(f"q=zzz OR {term}&hl=true", user=2)
+ document = next(d for d in response["results"] if d["id"] == "1")
+ highlight = str(document["note_highlights"]["1"][field]).lower()
+ assert f"{term}" in highlight
+
+ @pytest.mark.parametrize("param", ["title=bar)", "title=foo\\", "page_no_1="])
+ @pytest.mark.parametrize("user", [1, 2])
+ def test_search_notes_invalid_text_param(self, param, user):
+ """An unparseable text param must not break the note query"""
+
+ response = self.search(f"q=alice&{param}", user=user)
+ assert [d["id"] for d in response["results"]] == ["1"]
+
+ @pytest.mark.parametrize("user", [None, 2])
+ def test_search_highlight_skips_filters(self, user):
+ """Values of filter fields are not highlighted in the page text"""
+
+ response = self.search(
+ "q=quorx AND (data_edition:first OR user:99)&hl=true", user=user
+ )
+ assert [d["id"] for d in response["results"]] == ["8"]
+ highlights = str(response["results"][0]["highlights"])
+ assert "quorx" in highlights
+ assert "first" not in highlights
+
+ @pytest.mark.parametrize("user", [None, 2])
+ def test_search_highlight_only_filters(self, user):
+ """A query of only filters has nothing to highlight"""
+
+ response = self.search("q=user:3 AND data_edition:first&hl=true", user=user)
+ assert [d["id"] for d in response["results"]] == ["8"]
+ assert "" not in str(response["results"][0]["highlights"])
+
+ def test_search_notes_highlight_only_fields(self):
+ """A boolean query of only search fields leaves no text to search notes for"""
+
+ response = self.search("q=id:1 AND access:public&hl=true", user=2)
+ assert [d["id"] for d in response["results"]] == ["1"]
+ assert response["results"][0]["notes"]
+
class TestBooleanDetector:
@pytest.mark.parametrize(
@@ -519,6 +584,20 @@ def test_validator(self, query, valid):
assert all(DateValidator().visit(tree)) is valid
+class TestValidateRange:
+ @pytest.mark.parametrize(
+ "value,validator",
+ [
+ ("1^", NumberValidator()),
+ ("1^1.2.3", NumberValidator()),
+ ('"1 2"~1.5', NumberValidator()),
+ ('"NOW 2"~1.5', DateValidator()),
+ ],
+ )
+ def test_unparseable(self, value, validator):
+ assert _validate_range(value, validator) is False
+
+
class TestFilterExtractor:
@pytest.mark.parametrize(
"query,new_query,filters,sort_only,sort",
@@ -643,6 +722,30 @@ def test_extract_note(self, query, new_query):
assert new_query == (str(tree) if tree else "")
+class TestExtractNoteQuery:
+ @pytest.mark.parametrize(
+ "query,note_query",
+ [
+ ('"a" AND "b"^50', '"a" AND "b"^50'),
+ ("foo~10", "foo~10"),
+ ("user:1 AND tag:a", None),
+ ('user:1 AND +data__tag:"a"', None),
+ ("(user:1 AND tag:a)", None),
+ ("(user:1) OR (tag:a)", None),
+ ("user:1^2 AND tag:a", None),
+ ("foo OR (user:1)^2", "foo"),
+ ("(user:1 AND tag:a)^2 foo", "foo"),
+ ],
+ )
+ def test_extract_note_query(self, query, note_query):
+ assert _extract_query(_parse_tree(query), NoteExtractor) == note_query
+
+ def test_tree_unchanged(self):
+ tree = _parse_tree("a AND user:1")
+ _extract_query(tree, NoteExtractor)
+ assert str(tree) == "a AND user:1"
+
+
class TestParse:
@pytest.mark.parametrize(
"query,query_params,new_query,filters,sort,escaped,use_hl",
@@ -677,6 +780,18 @@ class TestParse:
("foo~1", "", "foo~1", "", None, False, False),
("foo hl:true", "", "foo", "", None, False, True),
("foo~1 hl:true", "", "foo~1", "", None, False, False),
+ ("foo^50", "", "foo^50", "", None, False, False),
+ ('"a" AND "b"^100', "", '"a" AND "b"^100', "", None, False, False),
+ ("foo~10", "", "foo~10", "", None, False, False),
+ ("foo^1.2.3", "", "foo\\^1.2.3", "", None, True, False),
+ ("foo^.", "", "foo\\^.", "", None, True, False),
+ ("foo~1.2.3", "", "foo\\~1.2.3", "", None, True, False),
+ ('"a b"~1.5', "", '"a b"~1', "", None, False, False),
+ ('"a b"~.', "", '"a b"', "", None, False, False),
+ ('"a b"~0.5 c', "", '"a b"~0 c', "", None, False, False),
+ ('"x" c "~2.5"~3.5', "", '"x" c "~2.5"~3', "", None, False, False),
+ ('"x \\" y"~1.5', "", '"x \\" y"~1', "", None, False, False),
+ ('\\" "a b"~1.5', "", '\\" "a b"~1', "", None, False, False),
],
)
def test_auth_parse(
@@ -685,7 +800,7 @@ def test_auth_parse(
# pylint: disable=too-many-arguments
user = Mock()
user.is_authenticated = True
- assert _parse(query, QueryDict(query_params, mutable=True), user) == (
+ assert _parse(query, QueryDict(query_params, mutable=True), user)[:5] == (
new_query,
QueryDict(filters),
sort,
@@ -706,9 +821,48 @@ def test_anon_parse(
self, query, query_params, new_query, filters, sort, escaped, use_hl
):
# pylint: disable=too-many-arguments
- assert _parse(
- query, QueryDict(query_params, mutable=True), AnonymousUser()
- ) == (new_query, QueryDict(filters), sort, escaped, use_hl)
+ assert _parse(query, QueryDict(query_params, mutable=True), AnonymousUser())[
+ :5
+ ] == (new_query, QueryDict(filters), sort, escaped, use_hl)
+
+ @pytest.mark.parametrize(
+ "query,query_params,hl_query",
+ [
+ ("", "", None),
+ ("user:1 AND tag:a", "", None),
+ ("foo AND (user:1 OR tag:email)", "", "foo"),
+ ("foo AND data_edition:first", "", "foo"),
+ ("foo AND user:bar", "", "foo"),
+ ("foo AND title:bar", "", "foo AND title:bar"),
+ ("text:foo AND user:1", "", "doctext:foo"),
+ ("", "title=bar", "title:(bar)"),
+ ("foo AND user:1", "title=bar", "foo title:(bar)"),
+ ],
+ )
+ def test_highlight_query(self, query, query_params, hl_query):
+ user = Mock()
+ user.is_authenticated = True
+ assert _parse(query, QueryDict(query_params, mutable=True), user)[6] == hl_query
+
+ @pytest.mark.parametrize(
+ "query,query_params,note_query",
+ [
+ ("", "", None),
+ ("", "title=bar", None),
+ ("user:1 AND tag:a", "", None),
+ ("a user:1", "", "a"),
+ ("a", "title=bar)", "a"),
+ ("a", "title=foo\\", "a"),
+ ("a", "page_no_1=", "a"),
+ ('"a" AND "b"^50', "", '"a" AND "b"^50'),
+ ],
+ )
+ def test_note_query(self, query, query_params, note_query):
+ user = Mock()
+ user.is_authenticated = True
+ assert (
+ _parse(query, QueryDict(query_params, mutable=True), user)[5] == note_query
+ )
class TestEscape: