diff --git a/changelog.d/5801-clone-lazy-snapshot.fixed.md b/changelog.d/5801-clone-lazy-snapshot.fixed.md new file mode 100644 index 0000000000..dcac3b7fa0 --- /dev/null +++ b/changelog.d/5801-clone-lazy-snapshot.fixed.md @@ -0,0 +1,5 @@ +- **A copy-on-write copy no longer raises while another thread reads the shared message.** When a + `Message` and its `copy()` were read on two threads, the first write copied the shared tree with + `copy.deepcopy`. A lazy segment split on the other thread could change the map that copy was + walking, and the write raised `RuntimeError`. The copy now takes that map in one call, so the + split lands before or after it. (`BACKLOG #5801`) diff --git a/changelog.d/5802-batch-header-delimiters.fixed.md b/changelog.d/5802-batch-header-delimiters.fixed.md new file mode 100644 index 0000000000..1c75c9bf43 --- /dev/null +++ b/changelog.d/5802-batch-header-delimiters.fixed.md @@ -0,0 +1,8 @@ +- **The outbound delimiter override now writes a `BHS` or `FHS` header correctly.** It wrote the + header's field separator as an extra field and changed only the escape character of its old + encoding characters, so every batch or file header field moved one place. With a new delimiter + set, field 2 of a `BHS` or `FHS` header now advertises the target encoding characters, as + `MSH-2` does. A `BHS` or `FHS` line whose fields 1 and 2 the rewrite cannot carry is refused, + which fails the delivery; the docstring of `_header_encoding` says which. With the message's own + delimiter set, `BHS` and `FHS` lines are written as they came. `MSH` lines are rewritten as + before. (`BACKLOG #5802`) diff --git a/changelog.d/5917-group-set.added.md b/changelog.d/5917-group-set.added.md new file mode 100644 index 0000000000..42776ac58c --- /dev/null +++ b/changelog.d/5917-group-set.added.md @@ -0,0 +1,4 @@ +- **`SegmentGroup.set` and `SegmentGroup.set_data` write a field within one order group.** Their + `occurrence` counts within the group, as `SegmentGroup.field` does. A group's `ordinal` counts a fieldless bare boundary line + and `Message.set` does not, so `Message.set(..., occurrence=group.ordinal)` after a bare `OBR` + could write to another order. (`BACKLOG #5917`) diff --git a/changelog.d/5920-continuation-and-ack-no-exemption.security.md b/changelog.d/5920-continuation-and-ack-no-exemption.security.md new file mode 100644 index 0000000000..c01a923e7e --- /dev/null +++ b/changelog.d/5920-continuation-and-ack-no-exemption.security.md @@ -0,0 +1,4 @@ +- **Strict validation refuses six more dropped segments in an "any HL7 segment" structure.** In a + structure such as `MFN_M01`, an `MSH`, `DSC`, `MSA`, `ERR`, `QAK` or `MFA` line that `hl7apy` + could not place was passed and listed in `unvalidated_segments`. It is now refused. Other defined + segments keep the exemption. (`BACKLOG #5920`) diff --git a/docs/HL7-VALIDATION.md b/docs/HL7-VALIDATION.md index 5b1b73691d..da3f53f354 100644 --- a/docs/HL7-VALIDATION.md +++ b/docs/HL7-VALIDATION.md @@ -85,7 +85,9 @@ with its position, as it lists a Z-segment, though only when the comparison runs `hl7apy` parsed but an earlier check refused lists its Z-segments alone, and a size, segment-cap or parse refusal lists nothing. A dropped segment ID the version does not define is still refused, and so is a dropped bare defined ID with no field separator (a lone `ORC` line in -`MFN_M01`); a bare line `hl7apy` placed is not. Every dropped segment is refused where MSH-9.3 +`MFN_M01`); a bare line `hl7apy` placed is not. A dropped `MSH`, `DSC`, `MSA`, `ERR`, `QAK` or +`MFA` line is refused too. A batch or file envelope line (`BHS`, `BTS`, `FHS`, `FTS`) is exempted like any other +defined segment, because batch splitting can leave one on a message. Every dropped segment is refused where MSH-9.3 borrows a slot-bearing structure under another message code (`ADT^A01^MFN_M01`); a structure borrowed under the same message code (`MFN^M02^MFN_M01`) still gets the exemption, since MSH-9.2 is not compared. Where the slot is required and its group is present, `hl7apy`'s own check diff --git a/messagefoundry/parsing/_builtin_hl7.py b/messagefoundry/parsing/_builtin_hl7.py index bbf39bb187..27cfb4682b 100644 --- a/messagefoundry/parsing/_builtin_hl7.py +++ b/messagefoundry/parsing/_builtin_hl7.py @@ -14,7 +14,7 @@ Data model (a plain built-in structure):: ParsedMessage = { - "_lazy": {index: raw line}, # first, so a deepcopy reads it before segments (_ensure_split) + "_lazy": {index: raw line}, # first, read before segments (_ensure_split, clone) "segments": [ {"id": str, "fields": list[str | dict]}, ... ], "raw": str | None, # cached encode; None once dirtied "seps": (field, component, repetition, subcomponent, escape), @@ -45,6 +45,7 @@ from __future__ import annotations +import copy import operator import re from collections.abc import Callable @@ -59,6 +60,7 @@ "HL7ParseError", "DelimiterRewriteRefused", "parse", + "clone", "extract_field", "extract_part", "raw_field", @@ -124,7 +126,8 @@ class Segment(TypedDict): class ParsedMessage(TypedDict): """The whole parsed message — see the module docstring for the shape. Build it with ``_lazy`` - as the first key, as :func:`parse` does: :func:`_ensure_split` relies on that order.""" + as the first key, as :func:`parse` does, and copy it with :func:`clone`, never a walk of the + dict (:func:`_ensure_split` says why).""" # Per-segment raw text awaiting a lazy split (index in ``segments`` → raw line). MSH is never # here (parsed eagerly). A segment leaves this map the first time a field path touches it. @@ -243,8 +246,8 @@ def parse(norm: str) -> ParsedMessage: segments.append({"id": line.split(field_sep, 1)[0] if line else "", "fields": []}) lazy[index] = line - # ``_lazy`` goes in first on purpose: ``copy.deepcopy`` (``Message._ensure_owned``) copies a dict - # in insertion order, so it reads ``_lazy`` before ``segments``, the order _ensure_split needs. + # ``_lazy`` goes in first so a reader walking the dict in insertion order meets it before + # ``segments``, the order _ensure_split needs; clone() takes that order explicitly. return {"_lazy": lazy, "segments": segments, "raw": strmsg + "\r", "seps": seps} @@ -258,14 +261,12 @@ def _ensure_split(msg: ParsedMessage, seg_index: int) -> None: The fields are stored **before** the raw line leaves ``_lazy``. A :meth:`Message.copy` clone shares this tree until one side writes, so another thread can read the segment mid-split. That can happen on a GIL build too, not only a free-threaded one: the GIL can switch threads between - storing the fields and the ``pop``, or inside a deepcopy's loop. A reader that looks at - ``_lazy`` before ``fields`` then sees either the raw line or the full field list, never an - empty segment (BACKLOG #5801). That holds for at least the readers in this module and the - copy-on-write ``copy.deepcopy``, because :func:`parse` puts ``_lazy`` first in the dict; a new - reader must keep the same order. It does not make a deepcopy safe against the ``pop`` itself: - one that is iterating ``_lazy`` at that moment can still raise ``RuntimeError`` for a dict that - changed size. The ``pop`` keeps its default because a racing split of the same segment may have - removed it. + storing the fields and the ``pop``. A reader that looks at ``_lazy`` before ``fields`` then sees + either the raw line or the full field list, never an empty segment (BACKLOG #5801). That holds + for at least the readers in this module and :func:`clone`; a new reader must keep the same + order. The ``pop`` changes the size of ``_lazy``, so nothing may iterate a shared ``_lazy`` in + Python code: :func:`clone` copies it in one ``dict.copy`` call instead. The ``pop`` keeps its + default because a racing split of the same segment may have removed it. """ lazy = msg["_lazy"] raw = lazy.get(seg_index) @@ -279,6 +280,25 @@ def _ensure_split(msg: ParsedMessage, seg_index: int) -> None: lazy.pop(seg_index, None) +def clone(msg: ParsedMessage) -> ParsedMessage: + """A private deep copy of ``msg``, safe while another thread lazily splits the same tree. + + The copy-on-write copy behind :meth:`Message._ensure_owned`. ``_lazy`` is copied first and in + one ``dict.copy`` call, which runs no Python code, so a concurrent :func:`_ensure_split` lands + wholly before or after it and its ``pop`` cannot change the dict's size mid-iteration, the + ``RuntimeError`` a ``copy.deepcopy`` of the tree could raise (BACKLOG #5801). ``segments`` is + deep-copied after it, so a split landing in between leaves the copy holding the raw line, the + fields, or both, never neither. The raw lines are ``str`` and shared, as ``copy.deepcopy`` + shares them. + """ + return { + "_lazy": msg["_lazy"].copy(), # first: a dict display evaluates in order + "segments": copy.deepcopy(msg["segments"]), + "raw": msg["raw"], + "seps": msg["seps"], + } + + def segment_index(msg: ParsedMessage, seg_id: str, occurrence: int) -> int | None: """List index of the ``occurrence``-th (1-based) segment with ``seg_id``, or None if absent. @@ -1185,17 +1205,22 @@ def encode_with_separators(msg: ParsedMessage, target: tuple[str, str, str, str, * any other escape sequence (``H``, ``N``, ``Xhh``, ``.br`` and the rest) keeps its text under the target escape character. - Two values the target set cannot represent raise :class:`DelimiterRewriteRefused`: an escape - sequence other than the five separator escapes whose text holds a target delimiter, and a segment - id holding the target field separator. A field holding no source escape character takes a single - ``str.translate`` that also escapes its data characters, and every field takes the plain one when - the target set equals the source set. + At least these values the target set cannot represent raise :class:`DelimiterRewriteRefused`: + an escape sequence other than the five separator escapes whose text holds a target delimiter, a + segment id holding the target field separator, and, with a target set unlike the source set, a + ``BHS`` or ``FHS`` line whose field 1 or 2 holds text :func:`_header_encoding` cannot carry. A + field holding no source escape character takes a single ``str.translate`` that also escapes its + data characters, and every field takes the plain one when the target set equals the source set. Where two source separators are the same character, the one split first wins: repetition, then - component, then subcomponent, the order python-hl7 split them. The segment id is kept verbatim. On - an ``FHS`` or ``BHS`` header the raw MSH-1/MSH-2 leaves change only their escape character. Both - match the python-hl7 tree walk this replaced (``transports/mllp.py`` ``reencode_delimiters``). Only - an ``MSH`` header has its MSH-2 rewritten. Blank segment lines are kept. + component, then subcomponent, the order python-hl7 split them, as the python-hl7 tree walk this + replaced did (``transports/mllp.py`` ``reencode_delimiters``). The segment id is kept verbatim. + Every ``MSH`` line has its MSH-2 written as the four target encoding characters, whatever it + held. When the target set differs, a ``BHS`` or ``FHS`` header has its field 2 rewritten the + same way, so a batch or file header advertises the delimiters its message is written in + (BACKLOG #5802); :func:`_header_encoding` says which field 2 it refuses. When the target set + equals the source set, a ``BHS`` or ``FHS`` header's fields 1 and 2 are written as they came. + Blank segment lines are kept. Raises ``ValueError`` when the message's own field, component, repetition and subcomponent characters are not four distinct characters. Such a header cannot say which structure a character @@ -1215,9 +1240,10 @@ def encode_with_separators(msg: ParsedMessage, target: tuple[str, str, str, str, table.setdefault(ord(s_esc), t_esc) # The same set on both sides changes no leaf, so one translate is exact. Any difference can make # a literal character a target delimiter, or give a separator escape's letter a new meaning. + same_set = tuple(msg["seps"]) == tuple(target) rewrite: Callable[[str], str] = ( operator.methodcaller("translate", table) - if tuple(msg["seps"]) == tuple(target) + if same_set else _LeafRewrite(table, structure, msg["seps"], target) ) lines: list[str] = [] @@ -1228,10 +1254,17 @@ def encode_with_separators(msg: ParsedMessage, target: tuple[str, str, str, str, lines.append(seg["id"]) continue if seg["id"] == "MSH" and len(texts) >= 3: + # MSH keeps the handling it had before BACKLOG #5802: MSH-2 always becomes the four + # target characters, with no refusal. parts = ["MSH", t_comp + t_rep + t_esc + t_sub] tail = texts[3:] elif seg["id"] in ("BHS", "FHS") and len(texts) >= 3: - parts = [texts[0], texts[1].replace(s_esc, t_esc), texts[2].replace(s_esc, t_esc)] + if same_set: + # Nothing to translate in fields 1 and 2, so the header keeps its own bytes. + lines.append(_join_header([*texts[:3], *map(rewrite, texts[3:])], t_field)) + continue + # Field 1 is the field separator the join writes, so only field 2 is a part. + parts = [seg["id"], _header_encoding(texts[1], texts[2], msg["seps"], target)] tail = texts[3:] else: if t_field in texts[0]: @@ -1245,6 +1278,42 @@ def encode_with_separators(msg: ParsedMessage, target: tuple[str, str, str, str, return "\r".join(lines) + "\r" +def _header_encoding( + field_1: str, + field_2: str, + source: tuple[str, str, str, str, str], + target: tuple[str, str, str, str, str], +) -> str: + """Field 2 of a ``BHS`` or ``FHS`` line rewritten for :func:`encode_with_separators`. + + A field 2 that is the source encoding characters, a leading part of them, or empty becomes the + four target characters. One character after the four is kept as the v2.7 truncation character + when it is printable, not a space, and not a source or target delimiter. Any other field 2, or + a field 1 that is not the source field separator, raises :class:`DelimiterRewriteRefused` + (BACKLOG #5802): rewriting it would lose text, or write a character the receiver would read as + structure.""" + s_field, s_comp, s_rep, s_sub, s_esc = source + t_field, t_comp, t_rep, t_sub, t_esc = target + s_enc = s_comp + s_rep + s_esc + s_sub + t_enc = t_comp + t_rep + t_esc + t_sub + if field_1 == s_field: + if s_enc.startswith(field_2): + return t_enc + truncation = field_2[len(s_enc) :] + if ( + field_2.startswith(s_enc) + and len(truncation) == 1 + and truncation.isprintable() + and not truncation.isspace() + and truncation not in s_field + s_enc + t_field + t_enc + ): + return t_enc + truncation + raise DelimiterRewriteRefused( + "a batch or file header's field separator or encoding characters are not ones the " + "rewrite can carry, so the rewrite would lose text" + ) + + #: How many distinct escape sequences one :class:`_LeafRewrite` remembers. A real message repeats a #: handful; the cap keeps a field of many distinct ones from building a table as large as itself. _SEQUENCE_CACHE_MAX = 64 diff --git a/messagefoundry/parsing/groups.py b/messagefoundry/parsing/groups.py index c8d571330c..1464171813 100644 --- a/messagefoundry/parsing/groups.py +++ b/messagefoundry/parsing/groups.py @@ -106,7 +106,8 @@ def ordinal(self) -> int: Positional, so a fieldless bare ``OBR`` line counts here though :meth:`Message.field` and :meth:`Message.count_segments` skip it: after one, ``ordinal`` is not the occurrence - ``Message.field`` reads. Read the boundary through :meth:`field` instead (BACKLOG #5917).""" + ``Message.field`` reads. Read and write the boundary through :meth:`field` and :meth:`set` + instead (BACKLOG #5917).""" return self._ordinal def segment_ids(self) -> list[str]: @@ -139,16 +140,38 @@ def field(self, path: str, *, occurrence: int = 1, repetition: int | None = None separator/escaping/repetition behavior is identical. Returns None if absent, which includes every boundary field of a group opened by a fieldless bare boundary line, and a fieldless member line is no occurrence (BACKLOG #5917).""" - if occurrence < 1: - raise ValueError("occurrence is 1-based (>= 1)") - seg = self._segment_of_path(path) - global_occurrence = self._to_global_occurrence(seg, occurrence) + _seg, global_occurrence = self._resolve(path, occurrence) if global_occurrence is None: return None return self._m.field(path, occurrence=global_occurrence, repetition=repetition) # --- mutate -------------------------------------------------------------- + def set( + self, path: str, value: str, *, occurrence: int = 1, repetition: int | None = None + ) -> None: + """Write ``value`` at ``path`` within this group, with ``occurrence`` scoped to the group as + :meth:`field` scopes it. + + Use this, not ``Message.set(..., occurrence=group.ordinal)``: after a fieldless bare + boundary line the two numberings differ, and that write would land on another order + (BACKLOG #5917). Raises ``KeyError`` first when the group holds fewer than ``occurrence`` + addressable segments of that id, before any check on ``value``; a group opened by a + fieldless boundary line has no boundary to write. Otherwise it is :meth:`Message.set` at + the translated occurrence, with its escaping and refusals.""" + self._m.set( + path, value, occurrence=self._write_occurrence(path, occurrence), repetition=repetition + ) + + def set_data( + self, path: str, value: str, *, occurrence: int = 1, repetition: int | None = None + ) -> None: + """:meth:`Message.set_data` within this group, with ``occurrence`` scoped to the group as + :meth:`set` scopes it, and the same ``KeyError`` first.""" + self._m.set_data( + path, value, occurrence=self._write_occurrence(path, occurrence), repetition=repetition + ) + def append_segment(self, line: str) -> None: """Add a raw segment ``line`` (``"OBX|3|NM|K^Potassium||4.1"``) at the **end of this group**. @@ -201,6 +224,22 @@ def rebuild(self, lines: list[str]) -> None: # --- internals ----------------------------------------------------------- + def _resolve(self, path: str, occurrence: int) -> tuple[str, int | None]: + """``path``'s segment id and the message-wide occurrence of its ``occurrence``-th segment + in this group (None if the group holds fewer), the translation :meth:`field` and + :meth:`set` share.""" + if occurrence < 1: + raise ValueError("occurrence is 1-based (>= 1)") + seg = self._segment_of_path(path) + return seg, self._to_global_occurrence(seg, occurrence) + + def _write_occurrence(self, path: str, occurrence: int) -> int: + """The message-wide occurrence a group-scoped write lands on, or ``KeyError``.""" + seg, global_occurrence = self._resolve(path, occurrence) + if global_occurrence is None: + raise KeyError(f"group {self._ordinal} holds no {seg} occurrence {occurrence}") + return global_occurrence + def _segment_of_path(self, path: str) -> str: """The segment id of an HL7 field ``path`` (e.g. ``"OBX-5"`` -> ``"OBX"``), validated.""" from messagefoundry.parsing.peek import parse_path diff --git a/messagefoundry/parsing/message.py b/messagefoundry/parsing/message.py index 3fad91de70..b520b1da7f 100644 --- a/messagefoundry/parsing/message.py +++ b/messagefoundry/parsing/message.py @@ -562,12 +562,36 @@ def copy(self) -> Message: clone._shared = True return clone + def __copy__(self) -> Message: + # copy.copy shares the tree, so both sides take the copy-on-write flag, or a write would + # reach both. Built on type(self) from the instance __dict__ (slots are not copied). + new = object.__new__(type(self)) + new.__dict__.update(self.__dict__) + self._shared = new._shared = True + return new + + def __deepcopy__(self, memo: dict[int, object]) -> Message: + # A deep copy shares nothing and leaves the source's flag alone. The tree goes through + # clone(): copy.deepcopy would walk a _lazy a reader of a clone may be splitting (#5801). + new = object.__new__(type(self)) + memo[id(self)] = new + for name, value in self.__dict__.items(): + if name == "_m": + value = _builtin_hl7.clone(value) + elif name == "_shared": + value = False + else: + value = copy.deepcopy(value, memo) + setattr(new, name, value) + return new + def _ensure_owned(self) -> None: """Copy-on-write hook — every mutator calls this before touching ``self._m``. If the backing tree is still shared with a :meth:`copy` snapshot/clone, take a private deepcopy now (and clear the flag) so the mutation lands only on this message; if already owned it is a cheap no-op.""" if self._shared: - self._m = copy.deepcopy(self._m) + # Not copy.deepcopy: a reader of the other side may be splitting this tree (#5801). + self._m = _builtin_hl7.clone(self._m) self._shared = False def __str__(self) -> str: diff --git a/messagefoundry/parsing/validate.py b/messagefoundry/parsing/validate.py index c0607eb1c8..85dd1c58f8 100644 --- a/messagefoundry/parsing/validate.py +++ b/messagefoundry/parsing/validate.py @@ -37,7 +37,8 @@ which is when no earlier check refused the message (a Z-segment is listed whenever hl7apy parsed the message, and nothing is listed for a size, segment-cap or parse refusal). A dropped ID the version does not define, and a dropped bare defined ID with no field separator, are still -refused. The exemption is per structure, not per slot: it does not count against the slot's +refused, and so is a dropped MSH, DSC, MSA, ERR, QAK or MFA line (``_NEVER_EXEMPT`` says why). +The exemption is per structure, not per slot: it does not count against the slot's cardinality or position. It holds only where MSH-9.1 names the structure's own message code, so a structure borrowed through MSH-9.3 under another message code (``ADT^A01^MFN_M01``) gets none; one borrowed under the same code (``MFN^M02^MFN_M01``) still gets it. Where the slot sits inside a @@ -364,6 +365,15 @@ def _exempts_defined_segments(message: Any) -> bool: ) +# Six IDs a dropped line of which is refused even in a structure with the "any HL7 segment" slot +# (BACKLOG #5920): MSH opens a message, DSC points at a continuation, and MSA, ERR, QAK and MFA +# answer another message, so none of the six is the data that slot carries. The list is not every +# header, continuation or acknowledgment segment the standard defines: at least ADD, SFT, UAC, +# QPD, QRD, QRF and RCP are not on it and keep the exemption. The batch and file envelope (BHS, +# BTS, FHS, FTS) is not here either: split_batch cuts only at MSH, so a member can carry its +# batch's trailers or the next batch's header. +_NEVER_EXEMPT = frozenset({"MSH", "DSC", "MSA", "ERR", "QAK", "MFA"}) + # Structures whose tree the dropped-segment check could not read, so each is logged once. _compare_failures: set[tuple[str, str | None]] = set() @@ -377,7 +387,8 @@ def _dropped_lines(lines: _Lines, message: Any) -> tuple[list[str], list[tuple[s ``unknown segment``, the rule ``docs/PHI.md`` sets for the anonymization report. Which occurrence of a repeated ID was dropped is not knowable from the tree, so the later ones are named; the count per ID is exact. In a structure with an "any HL7 segment" slot, a defined ID - followed by a field separator is not refused, as the module docstring says. + followed by a field separator is not refused, as the module docstring says, unless it is one + of :data:`_NEVER_EXEMPT`. """ import hl7apy @@ -401,7 +412,7 @@ def _dropped_lines(lines: _Lines, message: Any) -> tuple[list[str], list[tuple[s elif name in defined: if exempts is None: exempts = _exempts_defined_segments(message) - if exempts and number not in lines.bare: + if exempts and number not in lines.bare and name not in _NEVER_EXEMPT: exempted.append((name, number)) else: refusals.append( diff --git a/tests/test_builtin_hl7_header_no_trailing_separator.py b/tests/test_builtin_hl7_header_no_trailing_separator.py index a989f79f0f..b722aa53c5 100644 --- a/tests/test_builtin_hl7_header_no_trailing_separator.py +++ b/tests/test_builtin_hl7_header_no_trailing_separator.py @@ -91,16 +91,115 @@ def test_a_leading_header_with_no_encoding_character_is_the_contract_error() -> Message.parse("\r".join(["BHS|", _MSH, _PID])) -@pytest.mark.parametrize("line", ["MSH|^~\\&", "MSH|^~\\&|"]) +@pytest.mark.parametrize( + "line", ["MSH|^~\\&", "MSH|^~\\&|", "BHS|^~\\&", "BHS|^~\\&|", "FHS|^~\\&", "FHS|^~\\&|"] +) def test_the_delimiter_rewrite_keeps_a_later_header_line_in_shape(line: str) -> None: """The third header re-join, the outbound delimiter override, follows the same three-or-four - entry split for a later MSH line: an unchanged delimiter set gives the line back. A BHS or FHS - line is not covered here: that path writes it with its field separator doubled, on ``main`` - before this change too.""" + entry split for a later header line: an unchanged delimiter set gives the line back. A BHS or + FHS line once came back with its field separator doubled (BACKLOG #5802).""" parsed = _builtin_hl7.parse("\r".join([_MSH, _PID, line])) assert line in _builtin_hl7.encode_with_separators(parsed, parsed["seps"]).split("\r") +_TARGET = ("#", "@", "$", "%", "!") # field, component, repetition, subcomponent, escape + + +@pytest.mark.parametrize( + ("line", "rewritten"), + [ + ("BHS|^~\\&|A", "BHS#@$!%#A"), + ("BHS|^~\\&*|A", "BHS#@$!%*#A"), # a v2.7 truncation character is kept + ("FHS|^~|A", "FHS#@$!%#A"), # a leading part of the four + ], +) +def test_a_later_header_line_takes_the_target_encoding_characters( + line: str, rewritten: str +) -> None: + parsed = _builtin_hl7.parse("\r".join([_MSH, _PID, line])) + assert _builtin_hl7.encode_with_separators(parsed, _TARGET).split("\r")[2] == rewritten + + +@pytest.mark.parametrize( + "line", + [ + "BHS|SYNTHTEXT", + "BHS|SYNTHTEXT|A", + "BHSX|A|B", + "BHS|^~\\&EXTRA|B", + "BHS|^~\\&#*|B", + "BHS|^~\\&#|B", # a fifth character that is a target delimiter + "BHS|^~\\&^|B", # or a source delimiter + "BHS|^~\\&\x0b|B", # or a control character + "BHS|^~\\& |B", # or a space + ], +) +def test_a_header_line_not_in_the_message_delimiters_is_refused(line: str) -> None: + """Rewriting field 2 of these would drop its text or write a character the receiver would read + as structure, so a new delimiter set refuses (BACKLOG #5802).""" + parsed = _builtin_hl7.parse("\r".join([_MSH, _PID, line])) + with pytest.raises(_builtin_hl7.DelimiterRewriteRefused, match="would lose text"): + _builtin_hl7.encode_with_separators(parsed, _TARGET) + + +@pytest.mark.parametrize( + "line", ["BHS|SYNTHTEXT", "BHS|", "BHS|^~\\&EXTRA|B", "BHSX|A|B", "FHS|^~\\&|A"] +) +def test_a_same_set_override_writes_a_batch_header_line_as_it_came(line: str) -> None: + """With the target set equal to the message's own, nothing is refused or rewritten.""" + parsed = _builtin_hl7.parse("\r".join([_MSH, _PID, line])) + assert _builtin_hl7.encode_with_separators(parsed, parsed["seps"]).split("\r")[2] == line + + +#: MSH lines and what ``encode_with_separators`` wrote for them on ``origin/main`` before BACKLOG +#: #5802, as (message, same-set output, ``_TARGET`` output). MSH-2 always becomes the four target +#: characters, and nothing here is refused. +_MSH_ON_MAIN = [ + ( + "MSH|^~\\&#|A|B\r" + _PID, + "MSH|^~\\&|A|B\r" + _PID + "\r", + "MSH#@$!%#A#B\rPID#1##100@@@H@MR##DOE@JANE\r", + ), + ( + "MSH|^~\\&EXTRA|A\r" + _PID, + "MSH|^~\\&|A\r" + _PID + "\r", + "MSH#@$!%#A\rPID#1##100@@@H@MR##DOE@JANE\r", + ), + ( + "MSH|^~|A\r" + _PID, + "MSH|^~\\&|A\r" + _PID + "\r", + "MSH#@$!%#A\rPID#1##100@@@H@MR##DOE@JANE\r", + ), + ( + "MSH|^~\\&\x0b|A\r" + _PID, + "MSH|^~\\&|A\r" + _PID + "\r", + "MSH#@$!%#A\rPID#1##100@@@H@MR##DOE@JANE\r", + ), + ( + "FHS|^~\\&|A\rMSH!^~\\&!A\r" + _PID, + "MSH|^~\\&|A\r" + _PID + "\r", + "MSH#@$!%#A\rPID#1##100@@@H@MR##DOE@JANE\r", + ), +] + + +@pytest.mark.parametrize(("message", "same", "target"), _MSH_ON_MAIN) +def test_an_msh_line_is_rewritten_as_main_rewrote_it(message: str, same: str, target: str) -> None: + """BACKLOG #5802 changed only BHS and FHS. An MSH-2 that is long, truncated, or carries a fifth + character becomes the four target characters, under the same set too, and is never refused. + The comparison starts at the MSH line, since a leading FHS line is #5802's to change.""" + parsed = _builtin_hl7.parse(message) + for seps, expected in ((parsed["seps"], same), (_TARGET, target)): + encoded = _builtin_hl7.encode_with_separators(parsed, seps) + assert encoded[encoded.index("MSH") :] == expected + + +def test_a_v27_header_with_a_truncation_character_still_rewrites() -> None: + """MSH-2 ``^~\\&#`` with a target field separator of ``#`` is delivered, not refused.""" + parsed = _builtin_hl7.parse("MSH|^~\\&#|SND|FAC|RCV|FAC|20260101||ADT^A01|1|P|2.8\r" + _PID) + assert _builtin_hl7.encode_with_separators(parsed, _TARGET).startswith("MSH#@$!%#SND#") + + #: Bodies whose first line ``strip`` would change: padded single lines, whitespace past the encoding #: characters, a trailing ``\r`` run, and a normal multi-line message as the control. The last two #: hold whitespace inside the head and data after it on the last line, pinning that the head is diff --git a/tests/test_groups.py b/tests/test_groups.py index 87bac91b89..2a33aad494 100644 --- a/tests/test_groups.py +++ b/tests/test_groups.py @@ -265,3 +265,67 @@ def test_appending_to_earlier_group_does_not_stale_later_group_view() -> None: assert g2.field("OBR-4.1") == "LIPID" g1.clear() # shrink the first order; g2 shifts up, ordinal 2 still exists assert g2.field("OBR-4.1") == "LIPID" # g2 still resolves correctly + + +# A fieldless bare OBR line before the first real order: groups stay positional, so it opens +# group 1, while Message occurrences skip it (BACKLOG #5917). +BARE_OBR_FIRST = ( + "MSH|^~\\&|LAB|LABFAC|EHR|HOSP|20260101||ORU^R01^ORU_R01|MSG1|P|2.5.1\r" + "PID|1||100^^^HOSP^MR||DOE^JANE^Q||19800101|F\r" + "OBR\r" + "OBR|1|PLAC1|FILL1|CBC^Complete Blood Count\r" + "OBX|1|NM|WBC^White Cells||7.5|10*9/L\r" + "OBR|2|PLAC2|FILL2|LIPID^Lipid Panel\r" + "OBX|1|NM|CHOL^Cholesterol||190|mg/dL\r" +) + + +def test_group_set_after_a_bare_boundary_writes_its_own_order() -> None: + m = Message.parse(BARE_OBR_FIRST) + _bare, cbc, lipid = m.groups() + assert cbc.ordinal == 2 and cbc.field("OBR-4.1") == "CBC" + + cbc.set("OBR-4.1", "CBC2") + cbc.set("OBX-5", "8.0") + lipid.set("OBX-3.1", "LDL", occurrence=1) + + out = Message.parse(m.encode()) + assert [g.field("OBR-4.1") for g in out.groups()] == [None, "CBC2", "LIPID"] + assert out.groups()[1].field("OBX-5") == "8.0" + assert out.groups()[2].field("OBX-3.1") == "LDL" + assert out.groups()[2].field("OBX-5") == "190" + assert "\rOBR\r" in m.encode() # the bare line is untouched + + +def test_message_set_at_a_group_ordinal_hits_another_order() -> None: + # The hazard group.set exists for: Message occurrence 2 is the LIPID order, not group 2. + m = Message.parse(BARE_OBR_FIRST) + cbc = m.groups()[1] + m.set("OBR-4.1", "CBC2", occurrence=cbc.ordinal) + assert cbc.field("OBR-4.1") == "CBC" + assert m.groups()[2].field("OBR-4.1") == "CBC2" + + +def test_group_set_refuses_what_the_group_does_not_hold() -> None: + m = Message.parse(BARE_OBR_FIRST) + bare, cbc, _lipid = m.groups() + before = m.encode() + with pytest.raises(KeyError, match="group 1 holds no OBR occurrence 1"): + bare.set("OBR-4", "X") # a fieldless boundary has nothing to write + with pytest.raises(KeyError, match="group 2 holds no OBX occurrence 2"): + cbc.set("OBX-5", "X", occurrence=2) # never reaches into the next order's OBX + with pytest.raises(ValueError, match="1-based"): + cbc.set("OBX-5", "X", occurrence=0) + with pytest.raises(KeyError, match="group 1 holds no OBR occurrence 1"): + bare.set_data("OBR-4", "X") + assert m.encode() == before + + +def test_group_set_data_writes_a_leaf_in_its_own_order() -> None: + m = Message.parse(BARE_OBR_FIRST) + cbc = m.groups()[1] + cbc.set_data("OBR-4", "A^B") # separators carried as data, not structure + out = Message.parse(m.encode()).groups() + assert out[1].field("OBR-4") == "A\\S\\B" # the whole field, escaped + assert out[1].field("OBR-4.1") == "A^B" # one leaf, read back as written + assert out[2].field("OBR-4.1") == "LIPID" diff --git a/tests/test_message.py b/tests/test_message.py index 8358fe3599..fc83f24b24 100644 --- a/tests/test_message.py +++ b/tests/test_message.py @@ -5,7 +5,7 @@ from __future__ import annotations import copy -from collections.abc import Callable +from collections.abc import Callable, ItemsView import pytest @@ -572,7 +572,7 @@ def __deepcopy__(self, memo: dict[int, object]) -> list[_builtin_hl7.Segment]: def test_a_split_during_the_copy_on_write_deepcopy_keeps_the_segment() -> None: - # copy.deepcopy reads the tree's keys in insertion order. Were segments read before _lazy, a + # The copy-on-write copy (_builtin_hl7.clone) reads _lazy first. Were segments read before it, a # split landing between the two would leave the private copy with neither the raw line nor the # fields, and it would encode PID as its bare id (BACKLOG #5801). m = Message.parse(ADT) @@ -589,3 +589,107 @@ def test_a_split_during_the_copy_on_write_deepcopy_keeps_the_segment() -> None: assert clone.encode() == ADT.replace("19800101|F", "19800101|M") tree["raw"] = None # rebuild the original from its split fields, not parse's cached text assert m.encode() == ADT + + +class _SplitWhenCopied(str): + """A raw line whose deepcopy runs a lazy split of another segment: a reader on another thread + popping ``_lazy`` while a copy-on-write copy is walking it. clone() never deep-copies a raw + line, so on the fixed code the hook does not fire; the test guards against a return to + ``copy.deepcopy`` of the tree, and the never-walks test below covers any other walk.""" + + msg: _builtin_hl7.ParsedMessage + other: int + + def __deepcopy__(self, memo: dict[int, object]) -> str: + _builtin_hl7._ensure_split(self.msg, self.other) + return str(self) + + +def test_a_split_while_the_copy_walks_lazy_does_not_raise() -> None: + # Iterating _lazy while another thread's split pops an entry raised RuntimeError ("dictionary + # changed size during iteration") from copy.deepcopy (BACKLOG #5801, added item 1). + m = Message.parse(ADT) + clone = m.copy() + tree = m._m + pid = _builtin_hl7.segment_index(tree, "PID", 1) + assert pid is not None and pid in tree["_lazy"] + evn = pid - 1 + assert evn in tree["_lazy"] + hook = _SplitWhenCopied(tree["_lazy"][evn]) + hook.msg, hook.other = tree, pid + tree["_lazy"][evn] = hook + + clone.set("PID-8", "M") # the write takes the private copy + + assert clone["PID-5.1"] == "DOE" + assert clone.encode() == ADT.replace("19800101|F", "19800101|M") + tree["raw"] = None + assert m.encode() == ADT + + +class _SplitOnWalkLazy(dict[int, str]): + """A ``_lazy`` map that splits another segment the moment Python code walks its ``items()`` (as + ``copy.deepcopy`` and a comprehension over it do), as a reader on another thread could. + ``dict.copy`` runs in C and does not call it. ``__iter__`` is left alone on purpose: overriding + it would send ``dict.copy`` down its generic path, which a real ``_lazy`` never takes.""" + + def __init__(self, msg: _builtin_hl7.ParsedMessage, other: int) -> None: + super().__init__(msg["_lazy"]) + self._msg = msg + self._other = other + + def _split(self) -> None: + _builtin_hl7._ensure_split(self._msg, self._other) + + def items(self) -> ItemsView[int, str]: # type: ignore[override] + self._split() + return super().items() + + +def test_the_copy_on_write_copy_never_walks_the_shared_lazy_map() -> None: + # Any Python-level walk of a shared _lazy (a deepcopy, a comprehension) can meet a concurrent + # split's pop; the copy must take it in one C call instead (BACKLOG #5801, added item 1). + m = Message.parse(ADT) + clone = m.copy() + tree = m._m + pid = _builtin_hl7.segment_index(tree, "PID", 1) + assert pid is not None and pid in tree["_lazy"] + tree["_lazy"] = _SplitOnWalkLazy(tree, pid) + + clone.set("PID-8", "M") # the write takes the private copy + + assert pid in tree["_lazy"] # no walk ran, so no split ran + assert clone.encode() == ADT.replace("19800101|F", "19800101|M") + + +@pytest.mark.parametrize("copier", [copy.copy, copy.deepcopy], ids=["copy", "deepcopy"]) +def test_the_copy_module_takes_the_copy_on_write_path(copier: Callable[[Message], Message]) -> None: + m = Message.parse(ADT) + tree = m._m + pid = _builtin_hl7.segment_index(tree, "PID", 1) + assert pid is not None + tree["_lazy"] = _SplitOnWalkLazy(tree, pid) + + other = copier(m) + other.set("PID-8", "M") + + assert pid in tree["_lazy"] # the shared map was never walked + assert m["PID-8"] == "F" # the write did not reach the original + assert other["PID-8"] == "M" + + +class _TaggedMessage(Message): + tag: str + + +def test_a_deep_copy_shares_nothing_and_leaves_the_source_unflagged() -> None: + m = _TaggedMessage(Message.parse(ADT)._m) + m.tag = "t" + deep = copy.deepcopy(m) + assert m._shared is False and deep._shared is False + assert deep._m is not m._m and deep._m["segments"] is not m._m["segments"] + assert type(deep) is _TaggedMessage and deep.tag == "t" + m.set("PID-8", "M") # the source writes in place, with nothing shared + assert deep["PID-8"] == "F" + shallow = copy.copy(m) + assert type(shallow) is _TaggedMessage and m._shared and shallow._shared diff --git a/tests/test_mllp_encoding_override.py b/tests/test_mllp_encoding_override.py index d072600e2d..75aaffbd00 100644 --- a/tests/test_mllp_encoding_override.py +++ b/tests/test_mllp_encoding_override.py @@ -155,22 +155,27 @@ def test_reencode_rejects_unparseable_payload(garbage: str) -> None: # The re-encoder walks the built-in parse; before python-hl7 left, it was compared with python-hl7's # tree walk over 1,870 input and delimiter pairs and none differed. These pin the shapes that -# comparison covered and the corpus tests do not: an FHS/BHS header, whose raw MSH-1/MSH-2 leaves -# change only their escape character, and a blank segment line, which survives the rewrite. +# comparison covered and the corpus tests do not: an FHS/BHS header, whose field 2 now advertises +# the target set as MSH-2 does (python-hl7 wrote its field separator twice, BACKLOG #5802), and a +# blank segment line, which survives the rewrite. @pytest.mark.parametrize( ("payload", "expected"), [ ( "BHS|^~\\&|A|B\\E\\x\rMSH|^~\\&|A|B|C|D|1||ADT^A01|1|P|2.5.1\r" "PID|1|a\\E\\b^c~d&e\rBTS|1\r", - "BHS#|#^~^&#A#B\\x\rMSH#@$^%#A#B#C#D#1##ADT@A01#1#P#2.5.1\rPID#1#a\\b@c$d%e\rBTS#1\r", + "BHS#@$^%#A#B\\x\rMSH#@$^%#A#B#C#D#1##ADT@A01#1#P#2.5.1\rPID#1#a\\b@c$d%e\rBTS#1\r", + ), + ( + "FHS|^~\\&|A\rBHS|^~\\&|B\rMSH|^~\\&|A|B|C|D|1||ADT^A01|1|P|2.5.1\rBTS|1\rFTS|1\r", + "FHS#@$^%#A\rBHS#@$^%#B\rMSH#@$^%#A#B#C#D#1##ADT@A01#1#P#2.5.1\rBTS#1\rFTS#1\r", ), ( "MSH|^~\\&|A|B|C|D|1||ADT^A01|1|P|2.5.1\r\rPID|1||a^b~c&d\\F\\x\r", "MSH#@$^%#A#B#C#D#1##ADT@A01#1#P#2.5.1\r\rPID#1##a@b$c%d|x\r", ), ], - ids=["batch-header", "blank-line"], + ids=["batch-header", "file-and-batch-headers", "blank-line"], ) def test_reencode_batch_header_and_blank_line(payload: str, expected: str) -> None: assert reencode_delimiters(payload, parse_encoding_characters("#@$^%")) == expected diff --git a/tests/test_validate_dropped_segments.py b/tests/test_validate_dropped_segments.py index 8904b83beb..8c72bd7f33 100644 --- a/tests/test_validate_dropped_segments.py +++ b/tests/test_validate_dropped_segments.py @@ -171,6 +171,30 @@ def test_a_bare_defined_id_in_an_any_segment_message_is_refused() -> None: assert result.errors == ["segment ORC at line 4 has no place in this message's structure"] +@pytest.mark.parametrize( + "extra", + ["DSC|CONT1|F", "MSA|AA|1", "ERR|1", "QAK|Q1|OK", "MFA|MAD|1|20260101|S^OK"], + ids=lambda line: line[:3], +) +def test_a_continuation_or_acknowledgment_segment_gets_no_exemption(extra: str) -> None: + """The slot carries master-file or query data; a continuation pointer or an acknowledgment + segment dropped from such a message is refused (BACKLOG #5920 step 2).""" + result = validate(_MFN_M01 + "\r" + extra) + assert not result.ok + assert result.errors == [ + f"segment {extra[:3]} at line 4 has no place in this message's structure" + ] + assert result.unvalidated_segments == () + + +def test_batch_envelope_lines_on_a_split_member_keep_the_exemption() -> None: + """split_batch cuts only at MSH, so a member can end with its batch's trailers or the next + batch's header; those are not refused here.""" + result = validate(_MFN_M01 + "\rBTS|2\rBHS|^~\\&\rFTS|1\rFHS|^~\\&") + assert result.ok, result.errors + assert result.unvalidated_segments == (("BTS", 4), ("BHS", 5), ("FTS", 6), ("FHS", 7)) + + def test_a_structure_borrowed_under_the_same_message_code_keeps_the_exemption() -> None: """MSH-9.2 is not compared, so ``MFN^M02^MFN_M01`` is exempted as ``MFN^M01`` is.