Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
148 changes: 112 additions & 36 deletions documentcloud/documents/search.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,9 @@
import math
import re
from collections import defaultdict
from copy import deepcopy
from datetime import datetime
from decimal import InvalidOperation

# Third Party
import pysolr
Expand Down Expand Up @@ -61,6 +63,14 @@
}
DYNAMIC_TEXT_FIELDS = [r"page_no_[0-9]+"]

# luqum raises InvalidOperation or ValueError when it fails to read the number in
# a boost, fuzzy or proximity search
PARSE_ERRORS = (ParseError, TypeError, InvalidOperation, ValueError)

# matches escapes and phrases, so a quote inside a phrase is never mistaken for one
# which ends it, capturing any fractional proximity slop after a phrase
PHRASE_SLOP_RE = re.compile(r'\\.|("(?:[^"\\]|\\.)*")(?:~(\d*)\.[\d.]*)?')

SORT_MAP = {
"score": "score desc, created_at desc, id desc",
"created_at": "created_at desc, id desc",
Expand Down Expand Up @@ -98,8 +108,8 @@ def search(user, query_params):
# pylint: disable=too-many-locals
text_query = query_params.get("q", "")

text_query, filter_params, sort_order, escaped, use_hl = _parse(
text_query, query_params, user
text_query, filter_params, sort_order, escaped, use_hl, note_query, hl_query = (
_parse(text_query, query_params, user)
)

filter_params.update(query_params)
Expand All @@ -123,6 +133,9 @@ def search(user, query_params):
# allow explicit enabling of highlighting
if query_params.get("hl", "").lower() == "true" and settings.SOLR_USE_HL:
use_hl = True
if hl_query is None:
# nothing but filters, which should not be highlighted
use_hl = False

kwargs = {
"fq": filter_queries,
Expand All @@ -132,7 +145,8 @@ def search(user, query_params):
"hl.weightMatches": settings.SOLR_HL_WEIGHT_MATCHES,
**page_query_data,
}
original_text_query = text_query
if use_hl:
kwargs["hl.q"] = hl_query
if (
settings.SOLR_QUERY_NOTES
and user.is_authenticated
Expand All @@ -141,9 +155,9 @@ def search(user, query_params):
):
# turn note queries on for all pro users
# *:* returns all documents, do not enable note queries
text_query = _add_note_query(text_query, user)
text_query, note_params = _add_note_query(text_query, note_query, user)
kwargs.update(note_params)
kwargs["uf"] = "* _query_ -projects_edit_access"
kwargs["hl.q"] = original_text_query
query_notes = True
else:
query_notes = False
Expand Down Expand Up @@ -171,7 +185,7 @@ def search(user, query_params):
results, query_params, user, escaped, page_response_data
)
if use_hl and query_notes:
response = _highlight_notes(response, original_text_query)
response = _highlight_notes(response, note_query)

if settings.DEBUG or user.is_staff:
response["debug"] = {"text_query": text_query, "qtime": results.qtime, **kwargs}
Expand Down Expand Up @@ -393,19 +407,45 @@ class NoteExtractor(LuceneTreeTransformer):

def visit(self, node, parents=None):
new_node = super().visit(node, parents)
if (
isinstance(new_node, (Group, BaseOperation, Unary))
and not any(node.children)
and parents
if isinstance(new_node, (Boost, Group, BaseOperation, Unary)) and not any(
node.children
):
self.replace_node(new_node, None, parents[-1])
# an emptied root must also be dropped, or it serializes to an
# empty or "(None)" query which Solr cannot parse
if parents:
self.replace_node(new_node, None, parents[-1])
return None
return new_node

def visit_search_field(self, _node, _parents):
return None


class NumberFormatter(LuceneTreeTransformer):
"""Write boosts and fuzzy degrees in plain notation, as luqum normalizes them
to exponents (^50 becomes ^5E+1), which Solr reads as ^5 plus the term E+1
"""

def visit_boost(self, node, _parents):
node.force = format(node.force, "f")
return node

def visit_fuzzy(self, node, _parents):
node.degree = format(node.degree, "f")
return node


class HighlightExtractor(NoteExtractor):
"""Remove filter fields from a query, leaving the text to highlight"""

def visit_search_field(self, node, _parents):
if node.name in FILTER_FIELDS.values() or any(
re.match(rf"^{p}$", node.name) for p in DYNAMIC_FILTER_FIELDS
):
return None
return node


class AnonymousTransformer(LuceneTreeTransformer):
"""Remove computational expesnive searches from anonymous queries"""

Expand All @@ -420,21 +460,39 @@ def visit_term(self, node, _parents):
return node


def _truncate_slop(match):
phrase, slop = match.groups()
if slop is None:
return match.group(0)
return f"{phrase}~{slop}" if slop else phrase


def _parse_tree(text_query):
"""Parse a text query into a tree that serializes back to valid Solr syntax"""
# Solr truncates a fractional phrase slop to a whole number, but luqum fails
# to parse one
text_query = PHRASE_SLOP_RE.sub(_truncate_slop, text_query)
return NumberFormatter().visit(parser.parse(text_query))


def _parse(text_query, query_params, user):
"""Parse the text query and pull out filters and sorts

Accepts a text query
Returns a tuple of (text_query, filters, sort)
Returns a tuple of
(text_query, filters, sort, escaped, use_hl, note_query, hl_query)
text_query - new text query with filters and sorts removed
filters - a list of filters to be passed in to solr as field queries
(`fq` field)
sort - a string from the SORT_MAP to sort on
note_query - the text query to search notes for, or None
hl_query - the text query to highlight, or None
"""
if text_query.strip():
try:
tree = parser.parse(text_query)
tree = _parse_tree(text_query)
escaped = False
except (ParseError, TypeError):
except PARSE_ERRORS:
tree = parser.parse(escape(text_query))
escaped = True

Expand All @@ -450,6 +508,8 @@ def _parse(text_query, query_params, user):
tree = AnonymousTransformer().visit(tree)

new_query = str(tree) if tree is not None else ""
note_query = _extract_query(tree, NoteExtractor)
hl_query = _extract_query(tree, HighlightExtractor)
filters = filter_extractor.filters
sort = filter_extractor.sort
# only use highilighting for queries with no fuzzy searches and
Expand All @@ -462,19 +522,22 @@ def _parse(text_query, query_params, user):
sort = None
escaped = False
use_hl = False
note_query = None
hl_query = None

# pull text queries from the parameters into the text query
additional_text = _handle_params(query_params, TEXT_FIELDS, DYNAMIC_TEXT_FIELDS)
if additional_text:
# pylint: disable=consider-using-f-string
new_query = "{} {}".format(new_query, " ".join(additional_text))
hl_query = " ".join(([hl_query] if hl_query else []) + additional_text)

# if nothing is left in the query after pulling out filters, default to *:*
# which matches everything, otherwise convert the parse tree back to a text query
if not new_query:
new_query = "*:*"

return new_query, filters, sort, escaped, use_hl
return new_query, filters, sort, escaped, use_hl, note_query, hl_query


def _filter_queries(user, query_params):
Expand All @@ -493,7 +556,10 @@ def _validate_range(value, validator):
"""Validate using the validator as well as ensuring the top level node
is either a word or range
"""
node = parser.parse(value)
try:
node = parser.parse(value)
except PARSE_ERRORS:
return False
return all(validator.visit(node)) and isinstance(node, (Word, Range))


Expand Down Expand Up @@ -742,7 +808,7 @@ def format_note(result):
return [format_note(r) for r in results]


def _highlight_notes(response, text_query):
def _highlight_notes(response, note_query):
"""Do a second query to highlight notes if necessary"""
notes = [n for d in response["results"] for n in d["notes"]]
note_ids = [n["id"] for n in notes]
Expand All @@ -751,10 +817,9 @@ def _highlight_notes(response, text_query):
parents = {n["id"]: d["id"] for d in response["results"] for n in d["notes"]}
fq_ = [f"id:({' '.join(note_ids)})"]

note_query = _extract_note_query(text_query)
if note_query is None:
return response
query = f"title:{note_query} description:{note_query}"
query = f"title:({note_query}) description:({note_query})"

kwargs = {
"fq": fq_,
Expand Down Expand Up @@ -863,17 +928,21 @@ def _expand_list(results, key, queryset, serializer):
return results


def _extract_note_query(text_query):
"""Filter out all search fields"""
tree = parser.parse(text_query)
note_extractor = NoteExtractor()
tree = note_extractor.visit(tree)
def _extract_query(tree, extractor):
"""Remove nodes from a copy of the tree, returning what is left as a query"""
if tree is None:
return None
# copy, as the extractor removes nodes in place
tree = extractor().visit(deepcopy(tree))
if tree is None:
return None
return str(tree)


def _add_note_query(text_query, user):
def _add_note_query(text_query, note_query, user):
"""Add note searches to the text query, returning the new query and the
request params it references
"""
organizations = " ".join(str(o.pk) for o in user.organizations.all())
projects = " ".join(
str(p.pk)
Expand All @@ -884,41 +953,48 @@ def _add_note_query(text_query, user):
)
)
)
# an empty projects_edit_access:() is invalid and drops the org note search
edit_access = f"user:{user.pk}"
if projects:
edit_access += f" OR projects_edit_access:({projects})"

note_query = _extract_note_query(text_query)
if note_query is None:
return text_query
escaped_note_query = note_query.replace('"', '\\"')
return text_query, {}
# dereferenced params keep user input from mangling local params quoting, or
# unbalancing the parens of the clauses around it
note_match = f"+(title:({note_query}) description:({note_query}))"
note_params = {
"text_q": text_query,
"note_q_public": f"+type:note +(access:public OR user:{user.pk}) {note_match}",
"note_q_organization": f"+type:note +(access:organization) {note_match}",
}

return_query = (
# the original query to search for in documents
f"({text_query}) "
'_query_:"{!edismax v=$text_q}" '
# search through notes which are public or that you own
# on all documents you can view
f"""
_query_:"{{!join from=document_s fromIndex=notes to=id score=total
v='+type:note +(access:public OR user:{user.pk})
+(title:({escaped_note_query}) description:({escaped_note_query}))'
}}"
v=$note_q_public}}"
"""
# search through notes which are organization access
# on all documents you can edit
f"""
_query_:"
+(
(
(user:{user.pk} OR projects_edit_access:({projects}))
({edit_access})
AND access:(private organization)
)
OR
(access:(public organization) AND organization:({organizations}))
)
+{{!join from=document_s fromIndex=notes to=id score=total
v='+type:note +(access:organization)
+(title:({escaped_note_query}) description:({escaped_note_query}))'}}
v=$note_q_organization}}
"
"""
)
# replace runs of white space with a single space - keeps the template readable
# without incurring overhead of sending extra white space to solr
return re.sub(r"\s+", " ", return_query)
return re.sub(r"\s+", " ", return_query), note_params
10 changes: 10 additions & 0 deletions documentcloud/documents/tests/search_data.py
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@
"status": Status.success,
"created_at": datetime(2011, 6, 1, tzinfo=pytz.utc),
"data": {"edition": "first"},
"pages": ["A quorx arrived with the first edition, and none since"],
},
{
"id": 9,
Expand Down Expand Up @@ -233,6 +234,15 @@
"content": "foxtrot",
"access": Access.organization,
},
{
"id": 7,
"user": 1,
"organization": 1,
"document": 4,
"page_number": 0,
"content": "golf",
"access": Access.organization,
},
]


Expand Down
Loading
Loading