diff --git a/config/config.json b/config/config.json index bd3ae2f2..dac20f26 100644 --- a/config/config.json +++ b/config/config.json @@ -41,7 +41,7 @@ ], "console_logging_level": "ERROR", "enable_logging": true, - "file_logging_level": "WARN", + "file_logging_level": "WARNING", "log_directory": "/Users/dwolfson/localGit/egeria-v5-3/egeria-python/logs", "logging_console_format": "{time:YYYY-MM-DD HH:mm:ss.SSS} | {level: <8} | {name}:{line} - {message}", "logging_file_format": "{time:YYYY-MM-DD HH:mm:ss} | {level} | {module}-{function}:{line}-{message}-{extra}" diff --git a/docs/dr_egeria_manual.md b/docs/dr_egeria_manual.md index ca8210bb..c3591632 100644 --- a/docs/dr_egeria_manual.md +++ b/docs/dr_egeria_manual.md @@ -248,7 +248,7 @@ Dr.Egeria organizes its commands into "families," each corresponding to a specif - **Governance Officer**: Manage governance definitions, policies, and responsibilities. - **Action Author**: Define governance action process flows — reusable single-step action types and multi-step processes — and wire them to the engines and elements that execute them, without writing code (e.g., `Create Governance Action Process`, `Create Governance Action Process Step`, `Link First Process Step`, `Link Next Process Step`, `Link Action to Action Executor`, `Link Action to Target`). - **Curation**: Apply classifications and relationships to *existing* Referenceable elements after the fact — this family creates no elements of its own, it curates ones created elsewhere. Every command names a `Target Element` (Reference Name) and either a classification level/status or a second element to relate to. - - **Classifications** (`Classify`/`Reclassify`/`Declassify`, or `Classify`/`Update`/`Declassify` for the three multi-value ones): Impact, Confidence, Confidentiality, Criticality, Retention, Ownership, Digital Resource Origin, Zone Membership, Security Tags, Data Scope, Governance Expectations, Governance Measurements, Known Duplicate, Consolidated Duplicate, and the 0438 naming-standards markers Class Word, Modifier, Prime Word (added 2026-08-09 once Egeria PR #9166 shipped the backing `glossary-manager` endpoints — routed through `GlossaryManager`, not `ClassificationExplorer`, unlike every other classification here). Also, added 2026-08-21 once an `omvs_audit.py` gap-closure pass shipped their backing REST endpoints (all confirmed against a live 6.2-SNAPSHOT server): the 10 governance-point classifications (0435: Control/Verification/Enforcement/Execution Point, Policy Administration/Decision/Enforcement/Information/Management/Retrieval Point — routed through `GovernanceOfficer`), 6 metamodel markers (0463: Incomplete, ObjectIdentifier, ReferenceData, MobileResource, InstanceMetadata, MetamodelInstance — routed through `ClassificationExplorer`), ProjectKind (routed through `ProjectManager`), CollectionKind (routed through `CollectionManager`), and a Data Sharing Agreement retrofit pair for classifying/declassifying an already-existing Agreement (routed through `DigitalBusiness` — `Create Data Sharing Agreement`, Digital Products family, already sets this at creation time via `initialClassifications`). + - **Classifications** (`Classify`/`Reclassify`/`Declassify`, or `Classify`/`Update`/`Declassify` for the three multi-value ones): Impact, Confidence, Confidentiality, Criticality, Retention, Ownership, Digital Resource Origin, Zone Membership, Security Tags, Data Scope, Governance Expectations, Governance Measurements, Known Duplicate, Consolidated Duplicate, and the 0438 naming-standards markers Class Word, Modifier, Prime Word (added 2026-08-09 once Egeria PR #9166 shipped the backing `glossary-manager` endpoints — routed through `GlossaryManager`, not `ClassificationExplorer`, unlike every other classification here). Also, added 2026-08-21 once an `omvs_audit.py` gap-closure pass shipped their backing REST endpoints (all confirmed against a live 6.2-SNAPSHOT server): the 10 governance-point classifications (0435: Control/Verification/Enforcement/Execution Point, Policy Administration/Decision/Enforcement/Information/Management/Retrieval Point — routed through `GovernanceOfficer`), 6 metamodel markers (0463: Incomplete, ObjectIdentifier, ReferenceData, MobileResource, InstanceMetadata, MetamodelInstance — routed through `ClassificationExplorer`), ProjectKind (routed through `ProjectManager`), CollectionKind (routed through `CollectionManager`), and a Data Sharing Agreement retrofit pair for classifying/declassifying an already-existing Agreement (routed through `DigitalBusiness` — `Create Data Sharing Agreement`, Digital Products family, already sets this at creation time via `initialClassifications`). Also Promise (0010, added 2026-09-30 — `Classify`/`Declassify Promise`, routed through `ClassificationExplorer`): marks an element as a placeholder for a real-world resource that has not yet been delivered, with Deployment Status, Start/Due/Last Review/Completion Time and Additional Properties. A promised element is only visible to lineage requests, so `Declassify Promise` looks up its Target Element and sends the clear request with `forLineage=true` automatically. - **Relationships** (`Link`/`Unlink`, or `Attach`/`Update`/`Detach` for Search Keyword): Search Keyword, Semantic Assignment, Semantic Definition, Link Element To Scope (`ScopedBy`), Link Resource To Element (`ResourceList`), Link More Information, Peer Duplicate, Consolidated Duplicate Link. - Investigation (0130 ProjectKind, routed through `ProjectManager`) and NamingStandardsVocabulary (0438, routed through `GlossaryManager`) — new Data Standards types from Egeria PR #9300, added 2026-09-08. **Not yet verified against a live server**: the type isn't deployed there yet (`validate_compact_specs` fails its OM_TYPE-against-live-server check for both until the server picks up the new archive), and this checkout's `.http` ground truth files haven't been updated for this PR either — re-verify (including `validate_compact_specs`) once both are current. - No classification in this family remains genuinely unimplemented as of 2026-08-21 — Policy Management Point (previously blocked on a missing pyegeria SDK method) is now registered along with the 9 sibling governance-point classifications above. diff --git a/md_processing/data/compact_commands/commands_curation_compact.json b/md_processing/data/compact_commands/commands_curation_compact.json index ee9128eb..e2fcca4d 100644 --- a/md_processing/data/compact_commands/commands_curation_compact.json +++ b/md_processing/data/compact_commands/commands_curation_compact.json @@ -587,7 +587,7 @@ "default_value": "", "valid_values": [], "existing_element": "", - "description": "Due date/time for a Meeting, ToDo, or Review person action.", + "description": "Due date/time - for a Meeting, ToDo, or Review person action, or for delivery of the resource described by a Promise.", "generated": false, "style": "Simple", "user_specified": true, @@ -2842,6 +2842,112 @@ "max_cardinality": 1, "level": "Domain", "Journal Entry": "" + }, + "Deployment Status": { + "variable_name": "deployment_status", + "inUpdate": true, + "attr_labels": "", + "examples": "", + "default_value": "", + "valid_values": [ + "PROPOSED", + "UNDER_DEVELOPMENT", + "DEVELOPMENT_COMPLETE", + "APPROVED_FOR_DEPLOYMENT", + "REJECTED", + "STANDBY", + "ACTIVE", + "DISABLED", + "FAILED", + "OTHER" + ], + "existing_element": "", + "description": "Progress toward delivering the promised real-world digital resource/artifact - one of an enumerated set of values (DeploymentStatus).", + "generated": false, + "style": "Valid Value", + "user_specified": true, + "unique": false, + "input_required": false, + "min_cardinality": 0, + "max_cardinality": 1, + "level": "Basic", + "Journal Entry": "Added 2026-09-30 -- Promise classification (0010), Classification Explorer setElementAsPromise." + }, + "User Defined Deployment Status": { + "variable_name": "user_defined_deployment_status", + "inUpdate": true, + "attr_labels": "", + "examples": "", + "default_value": "", + "valid_values": [], + "existing_element": "", + "description": "Locally defined deployment status - used when Deployment Status is OTHER.", + "generated": false, + "style": "Simple", + "user_specified": true, + "unique": false, + "input_required": false, + "min_cardinality": 0, + "max_cardinality": 1, + "level": "Domain", + "Journal Entry": "Added 2026-09-30 -- Promise classification (0010), Classification Explorer setElementAsPromise." + }, + "Start Time": { + "variable_name": "start_time", + "inUpdate": true, + "attr_labels": "", + "examples": "", + "default_value": "", + "valid_values": [], + "existing_element": "", + "description": "Date/time (ISO 8601) that work started on delivering the promised resource.", + "generated": false, + "style": "Simple", + "user_specified": true, + "unique": false, + "input_required": false, + "min_cardinality": 0, + "max_cardinality": 1, + "level": "Domain", + "Journal Entry": "Added 2026-09-30 -- Promise classification (0010), Classification Explorer setElementAsPromise." + }, + "Last Review Time": { + "variable_name": "last_review_time", + "inUpdate": true, + "attr_labels": "", + "examples": "", + "default_value": "", + "valid_values": [], + "existing_element": "", + "description": "Date/time (ISO 8601) that the promise was last reviewed.", + "generated": false, + "style": "Simple", + "user_specified": true, + "unique": false, + "input_required": false, + "min_cardinality": 0, + "max_cardinality": 1, + "level": "Domain", + "Journal Entry": "Added 2026-09-30 -- Promise classification (0010), Classification Explorer setElementAsPromise." + }, + "Completion Time": { + "variable_name": "completion_time", + "inUpdate": true, + "attr_labels": "", + "examples": "", + "default_value": "", + "valid_values": [], + "existing_element": "", + "description": "Date/time (ISO 8601) that the promised resource was delivered.", + "generated": false, + "style": "Simple", + "user_specified": true, + "unique": false, + "input_required": false, + "min_cardinality": 0, + "max_cardinality": 1, + "level": "Domain", + "Journal Entry": "Added 2026-09-30 -- Promise classification (0010), Classification Explorer setElementAsPromise." } }, "bundles": { @@ -3316,6 +3422,19 @@ "Target Element", "Additional Properties" ] + }, + "Promise Base": { + "inherits": "Request Base", + "own_attributes": [ + "Target Element", + "Deployment Status", + "User Defined Deployment Status", + "Start Time", + "Due Time", + "Last Review Time", + "Completion Time", + "Additional Properties" + ] } }, "commands": { @@ -5370,6 +5489,44 @@ "bundle": "Declassify Base", "custom_attributes": [], "Journal Entry": "" + }, + "Classify Promise": { + "display_name": "promise", + "qn_prefix": "Promise", + "alternate_names": [], + "family": "Curation", + "description": "Classify an existing element as a Promise - a placeholder for a real-world digital resource/artifact that has not yet been delivered. Once classified, the element is only returned to lineage requests (For Lineage = true).", + "verb": "Classify", + "upsert": false, + "attach": false, + "level": "Basic", + "find_method": "", + "find_constraints": "", + "extra_find": "", + "extra_constraints": "", + "OM_TYPE": "Promise", + "bundle": "Promise Base", + "custom_attributes": [], + "Journal Entry": "Added 2026-09-30 -- Promise classification (0010), Classification Explorer setElementAsPromise." + }, + "Declassify Promise": { + "display_name": "promise", + "qn_prefix": "Promise", + "alternate_names": [], + "family": "Curation", + "description": "Remove the Promise classification from an element - typically because the promised resource has now been delivered. The Target Element is looked up with For Lineage = true, since a promised element is otherwise invisible.", + "verb": "Declassify", + "upsert": false, + "attach": false, + "level": "Basic", + "find_method": "", + "find_constraints": "", + "extra_find": "", + "extra_constraints": "", + "OM_TYPE": "Promise", + "bundle": "Declassify Base", + "custom_attributes": [], + "Journal Entry": "Added 2026-09-30 -- Promise classification (0010), Classification Explorer setElementAsPromise." } } } diff --git a/md_processing/v2/curation.py b/md_processing/v2/curation.py index 4aa7a983..6a0b9b46 100644 --- a/md_processing/v2/curation.py +++ b/md_processing/v2/curation.py @@ -13,8 +13,11 @@ Policy Administration/Policy Decision/Policy Enforcement/Policy Information/ Policy Management/Policy Retrieval Point), 6 metamodel/classification- explorer markers (0463: Incomplete, ObjectIdentifier, ReferenceData, -MobileResource, InstanceMetadata, MetamodelInstance), and ProjectKind (0130)/ -CollectionKind (0021). +MobileResource, InstanceMetadata, MetamodelInstance), ProjectKind (0130)/ +CollectionKind (0021), and Promise (0010 -- a placeholder for a not-yet-delivered +resource; the promised element is only visible to forLineage=true requests, so +Declassify Promise resolves its Target Element and sends its clear request with +forLineage=true, see ClassificationSpec.declassify_for_lineage). CurationLinkProcessor -- Link/Unlink/Attach/Detach relationship commands for: Semantic Assignment, Semantic Definition, Scoped By, Peer Duplicate, @@ -88,6 +91,10 @@ class ClassificationSpec: update_method: Optional[str] = None level_field: Optional[str] = None # e.g. "severityLevel" -- reads "Level Identifier" fields: Dict[str, str] = dc_field(default_factory=dict) # attribute name -> property name + # True when the classified element is hidden from normal queries (e.g. Promise), so + # Declassify must both resolve its Target Element and send the clear request with + # forLineage=true -- Egeria rejects/cannot see it otherwise. + declassify_for_lineage: bool = False # Shared governance-classification fields (Impact/Confidence/Confidentiality/Criticality/Retention) @@ -236,6 +243,17 @@ class ClassificationSpec: "NamingStandardsVocabulary": ClassificationSpec( "_async_set_glossary_as_naming_standards_vocabulary", "_async_clear_glossary_as_naming_standards_vocabulary", "NamingStandardsVocabularyProperties"), + # Promise (0010) -- added 2026-09-30 once ClassificationExplorer's setElementAsPromise/ + # clearElementAsPromise shipped upstream. A promised element is only returned to + # lineage requests, hence declassify_for_lineage. Not yet verified against a live server. + "Promise": ClassificationSpec( + "_async_set_element_as_promise", "_async_clear_element_as_promise", "PromiseProperties", + fields={"Deployment Status": "deploymentStatus", + "User Defined Deployment Status": "userDefinedDeploymentStatus", + "Start Time": "startTime", "Due Time": "dueTime", + "Last Review Time": "lastReviewTime", "Completion Time": "completionTime", + "Additional Properties": "additionalProperties"}, + declassify_for_lineage=True), } # OM_TYPEs in CLASSIFICATION_METHODS whose set/clear methods live on a client other than @@ -301,6 +319,10 @@ def supports_target_element_lookup(self) -> bool: async def fetch_as_is(self) -> Optional[Dict[str, Any]]: return None + def lookup_for_lineage(self) -> bool: + class_spec = CLASSIFICATION_METHODS.get(self.get_command_spec().get("OM_TYPE")) + return bool(class_spec and class_spec.declassify_for_lineage and self.command.verb == "Declassify") + async def apply_changes(self) -> str: verb = self.command.verb object_type = self.canonical_object_type or self.command.object_type @@ -319,6 +341,8 @@ async def apply_changes(self) -> str: if verb == "Declassify": body = {"class": "DeleteClassificationRequestBody", **_audit_fields(attributes)} + if class_spec.declassify_for_lineage: + body["forLineage"] = True await getattr(client, class_spec.clear_method)(element_guid, body) logger.success(f"Declassified {object_type} on {element_guid}") return f"\n\n## {verb} {object_type}\n\nRemoved {om_type} classification from {element_guid}." diff --git a/md_processing/v2/processors.py b/md_processing/v2/processors.py index bd038154..23dffeec 100644 --- a/md_processing/v2/processors.py +++ b/md_processing/v2/processors.py @@ -1389,6 +1389,12 @@ async def fetch_element(self, guid: str, _max_timeout_retries: int = 2) -> Optio return None + def lookup_for_lineage(self) -> bool: + """Whether reference lookups for this command should include elements only visible to + lineage requests (forLineage=true) -- e.g. Declassify Promise, whose target is hidden + from normal queries once promised. Overridden per processor; default False.""" + return False + async def resolve_element_guid(self, name_or_guid: str, tech_type: Optional[str] = None) -> Optional[str]: """ Resolves a name or GUID to a GUID using various strategies. @@ -1478,7 +1484,11 @@ def _cache_if_not_found(key: tuple, value: Any) -> None: unsupported_type_warnings = self.context.setdefault("_unsupported_lookup_types_warned", set()) # Pass 1: Try WITH type constraint (fastest, avoids ambiguity) - pass1_key = (name_or_guid, tech_type or None) + for_lineage = self.lookup_for_lineage() + # Lineage lookups see a different element set, so they get their own cache keys. + lineage_key = ("forLineage",) if for_lineage else () + lineage_kwargs = {"for_lineage": True} if for_lineage else {} + pass1_key = (name_or_guid, tech_type or None, *lineage_key) cached1 = query_cache.get(pass1_key, _NOT_CACHED) if cached1 is _AMBIGUOUS: logger.debug(f"resolve_element_guid: cache hit (ambiguous, Pass 1) for {pass1_key!r}") @@ -1489,7 +1499,7 @@ def _cache_if_not_found(key: tuple, value: Any) -> None: res = cached1 else: try: - res = await self.client.__async_get_guid__(qualified_name=name_or_guid, display_name=name_or_guid, property_name="displayName", tech_type=tech_type or None) + res = await self.client.__async_get_guid__(qualified_name=name_or_guid, display_name=name_or_guid, property_name="displayName", tech_type=tech_type or None, **lineage_kwargs) _cache_if_not_found(pass1_key, res) except PyegeriaException as e: # Catch multiple matches error @@ -1517,7 +1527,7 @@ def _cache_if_not_found(key: tuple, value: Any) -> None: # Pass 2: If no result (or if type was invalid), try WITHOUT type constraint is_not_found = not res or (isinstance(res, str) and (res.startswith("No ") or " found" in res)) if is_not_found and tech_type: - pass2_key = (name_or_guid, None) + pass2_key = (name_or_guid, None, *lineage_key) cached2 = query_cache.get(pass2_key, _NOT_CACHED) if cached2 is _AMBIGUOUS: logger.debug(f"resolve_element_guid: cache hit (ambiguous, Pass 2) for {pass2_key!r}") @@ -1528,7 +1538,7 @@ def _cache_if_not_found(key: tuple, value: Any) -> None: res = cached2 else: try: - res = await self.client.__async_get_guid__(qualified_name=name_or_guid, display_name=name_or_guid, property_name="displayName") + res = await self.client.__async_get_guid__(qualified_name=name_or_guid, display_name=name_or_guid, property_name="displayName", **lineage_kwargs) _cache_if_not_found(pass2_key, res) except PyegeriaException as e: if "Multiple elements found" in str(e): diff --git a/md_processing/v2/utils.py b/md_processing/v2/utils.py index 3b38eea8..a100379d 100644 --- a/md_processing/v2/utils.py +++ b/md_processing/v2/utils.py @@ -1,6 +1,7 @@ """ General utility functions for Dr.Egeria v2 processing. """ +import json import re from typing import Any, Optional, Type from enum import Enum @@ -11,10 +12,27 @@ def parse_key_value(text: str) -> dict[str, Any]: - Markdown tables: | Key | Value | - Lists: * Key: Value or - Key = Value - Simple blocks: Key: Value (one per line) + - A JSON object: {"Key": "Value", ...} + + Values are always returned as strings (callers treat the result as a + Map). A non-string JSON value is JSON-encoded, so a caller + that needs its real type back (report.py's _coerce_analytic_value) can + json.loads it. """ if not text: return {} - + + # JSON object -- previously split on its first ':' like a "Key: Value" line, + # silently producing keys like '{"owner"'. + stripped = text.strip() + if stripped.startswith("{") and stripped.endswith("}"): + try: + parsed = json.loads(stripped) + except ValueError: + parsed = None + if isinstance(parsed, dict): + return {str(k): v if isinstance(v, str) else json.dumps(v) for k, v in parsed.items()} + results = {} # Check for Markdown Table (| Key | Value |) diff --git a/pyegeria/core/_server_client.py b/pyegeria/core/_server_client.py index 93d72c93..ca2a7520 100644 --- a/pyegeria/core/_server_client.py +++ b/pyegeria/core/_server_client.py @@ -229,6 +229,7 @@ async def __async_get_guid__( qualified_name: Optional[str] = None, tech_type: Optional[str] = None, organization_name: Optional[str] = None, + for_lineage: bool = False, ) -> str: """Helper function to return a server_guid - one of server_guid, qualified_name or display_name should contain information. If all are None, an exception will be thrown. If all contain @@ -241,6 +242,9 @@ async def __async_get_guid__( An InvalidParameter Exception is thrown if multiple matches are found for the given property name. If this occurs, use a qualified name for the property name. + + Set for_lineage=True to also find elements only visible to lineage requests (e.g. ones classified + as Promise or Memento). Async version. """ try: @@ -256,7 +260,7 @@ async def __async_get_guid__( "class": "FindPropertyNameProperties", "propertyValue": qualified_name, "propertyName": "qualifiedName", - "forLineage": False, + "forLineage": for_lineage, "forDuplicateProcessing": False, "effectiveTime": None, } @@ -285,7 +289,7 @@ async def __async_get_guid__( "class": "FindPropertyNameProperties", "propertyValue": name, "propertyName": property_name, - "forLineage": False, + "forLineage": for_lineage, "forDuplicateProcessing": False, "effectiveTime": None, } @@ -301,7 +305,7 @@ async def __async_get_guid__( "class": "FindPropertyNameProperties", "propertyValue": display_name, "propertyName": property_name, - "forLineage": False, + "forLineage": for_lineage, "forDuplicateProcessing": False, "effectiveTime": None, } diff --git a/pyegeria/omvs/asset_catalog.py b/pyegeria/omvs/asset_catalog.py index e901e558..161a7861 100644 --- a/pyegeria/omvs/asset_catalog.py +++ b/pyegeria/omvs/asset_catalog.py @@ -451,6 +451,7 @@ async def _async_get_asset_lineage_graph_by_guid( page_size: int = 0, graph_query_depth: int = 5, max_mermaid_node_count: int = 10, + for_lineage: Optional[bool] = None, output_format: str = "DICT", report_spec: str | dict = "Common-Mermaid", **kwargs, @@ -483,6 +484,9 @@ async def _async_get_asset_lineage_graph_by_guid( How many hops of the lineage graph to traverse. max_mermaid_node_count : int, default 5 Maximum number of nodes to include in the generated mermaid graph. + for_lineage : bool, optional + Set True to include elements only visible to lineage requests (e.g. ones classified as + Promise or Memento). If not specified, Egeria's default (False) applies. output_format : str, default "DICT" The desired output format. report_spec : str | dict, default "Common-Mermaid" @@ -529,6 +533,7 @@ async def _async_get_asset_lineage_graph_by_guid( "pageSize": page_size, "graphQueryDepth": graph_query_depth, "maxMermaidNodeCount": max_mermaid_node_count, + "forLineage": for_lineage, } response = await self._async_make_request("POST", url, body_slimmer(body)) @@ -559,6 +564,7 @@ def get_asset_lineage_graph_by_guid( page_size: int = 0, graph_query_depth: int = 5, max_mermaid_node_count: int = 10, + for_lineage: Optional[bool] = None, output_format: str = "DICT", report_spec: str | dict = "Common-Mermaid", **kwargs, @@ -589,6 +595,9 @@ def get_asset_lineage_graph_by_guid( How many hops of the lineage graph to traverse. max_mermaid_node_count : int, default 5 Maximum number of nodes to include in the generated mermaid graph. + for_lineage : bool, optional + Set True to include elements only visible to lineage requests (e.g. ones classified as + Promise or Memento). If not specified, Egeria's default (False) applies. output_format : str, default "DICT" The desired output format. report_spec : str | dict, default "Common-Mermaid" @@ -621,6 +630,7 @@ def get_asset_lineage_graph_by_guid( page_size=page_size, graph_query_depth=graph_query_depth, max_mermaid_node_count=max_mermaid_node_count, + for_lineage=for_lineage, output_format=output_format, report_spec=report_spec, **kwargs, @@ -641,6 +651,7 @@ def get_asset_lineage_mermaid_graph( page_size: int = 0, graph_query_depth: int = 5, max_mermaid_node_count: int = 10, + for_lineage: Optional[bool] = None, ) -> str: """Return the asset lineage including a mermaid markdown string. @@ -668,6 +679,9 @@ def get_asset_lineage_mermaid_graph( How many hops of the lineage graph to traverse. max_mermaid_node_count : int, default 5 Maximum number of nodes to include in the generated mermaid graph. + for_lineage : bool, optional + Set True to include elements only visible to lineage requests (e.g. ones classified as + Promise or Memento). If not specified, Egeria's default (False) applies. Returns ------- @@ -694,6 +708,7 @@ def get_asset_lineage_mermaid_graph( page_size=page_size, graph_query_depth=graph_query_depth, max_mermaid_node_count=max_mermaid_node_count, + for_lineage=for_lineage, output_format="JSON", ) return asset_graph.get("mermaidGraph") diff --git a/pyegeria/omvs/classification_explorer.py b/pyegeria/omvs/classification_explorer.py index d643c590..b7b222ce 100644 --- a/pyegeria/omvs/classification_explorer.py +++ b/pyegeria/omvs/classification_explorer.py @@ -13466,6 +13466,73 @@ def clear_element_as_metamodel_instance(self, element_guid: str, loop = asyncio.get_event_loop() loop.run_until_complete(self._async_clear_element_as_metamodel_instance(element_guid, body)) + @dynamic_catch + async def _async_set_element_as_promise(self, element_guid: str, + body: Optional[dict | NewClassificationRequestBody] = None) -> None: + """Classify an element as a Promise - a placeholder for a real-world digital resource/artifact that has + not yet been delivered. Once classified, the element is only returned to lineage requests + (forLineage=true). Async version. + + https://egeria-project.org/types/0/0010-Base-Model/#promise-classification + + Sample body: + + { + "class" : "NewClassificationRequestBody", + "properties": { + "class": "PromiseProperties", + "deploymentStatus": "UNDER_DEVELOPMENT", + "userDefinedDeploymentStatus": "add status here", + "startTime": "isoTimestamp", + "dueTime": "isoTimestamp", + "lastReviewTime": "isoTimestamp", + "completionTime": "isoTimestamp", + "additionalProperties": {"property1Name" : "property1Value"}, + "effectiveFrom": "isoTimestamp", + "effectiveTo": "isoTimestamp" + }, + "externalSourceGUID": "add guid here", + "externalSourceName": "add qualified name here", + "effectiveTime" : "isoTimestamp", + "forLineage" : false, + "forDuplicateProcessing" : false + } + """ + url = f"{self.classification_command_root}/elements/{element_guid}/promise" + if body is None: + body = {"class": "NewClassificationRequestBody", "properties": {"class": "PromiseProperties"}} + await self._async_new_classification_request(url, prop=["PromiseProperties"], body=body) + logger.info(f"Added Promise classification to {element_guid}") + + @dynamic_catch + def set_element_as_promise(self, element_guid: str, + body: Optional[dict | NewClassificationRequestBody] = None) -> None: + """Classify an element as a Promise - a placeholder for a real-world digital resource/artifact that has + not yet been delivered. Once classified, the element is only returned to lineage requests (forLineage=true). + See _async_set_element_as_promise for a sample body.""" + loop = asyncio.get_event_loop() + loop.run_until_complete(self._async_set_element_as_promise(element_guid, body)) + + @dynamic_catch + async def _async_clear_element_as_promise(self, element_guid: str, + body: Optional[dict | DeleteClassificationRequestBody] = None) -> None: + """Remove the Promise classification from an element - typically because the promised resource has now + been delivered. Egeria requires forLineage=true on this request (the promised element is otherwise + invisible), so the default body sets it; a caller-supplied body must set it too. Async version.""" + url = f"{self.classification_command_root}/elements/{element_guid}/promise/remove" + if body is None: + body = {"class": "DeleteClassificationRequestBody", "forLineage": True} + await self._async_delete_classification_request(url, body) + logger.info(f"Removed Promise classification from {element_guid}") + + @dynamic_catch + def clear_element_as_promise(self, element_guid: str, + body: Optional[dict | DeleteClassificationRequestBody] = None) -> None: + """Remove the Promise classification from an element - typically because the promised resource has now + been delivered. The default body sets forLineage=true, which Egeria requires for this request.""" + loop = asyncio.get_event_loop() + loop.run_until_complete(self._async_clear_element_as_promise(element_guid, body)) + if __name__ == "__main__": print("Main-Classification Manager") diff --git a/sample-data/templates/advanced/Curation/Classify_Promise.md b/sample-data/templates/advanced/Curation/Classify_Promise.md new file mode 100644 index 00000000..81fc8753 --- /dev/null +++ b/sample-data/templates/advanced/Curation/Classify_Promise.md @@ -0,0 +1,148 @@ +___ + +## Classify Promise +> Classify an existing element as a Promise - a placeholder for a real-world digital resource/artifact that has not yet been delivered. Once classified, the element is only returned to lineage requests (For Lineage = true). + +### Target Element +> **Input Required**: True + +> **Attribute Type**: Reference Name + +> **Description**: Qualified name of the existing element being classified or linked. + + +### User Defined Deployment Status +> **Input Required**: False + +> **Attribute Type**: Simple + +> **Description**: Locally defined deployment status - used when Deployment Status is OTHER. + + +### Start Time +> **Input Required**: False + +> **Attribute Type**: Simple + +> **Description**: Date/time (ISO 8601) that work started on delivering the promised resource. + + +### Due Time +> **Input Required**: False + +> **Attribute Type**: Simple + +> **Description**: Due date/time - for a Meeting, ToDo, or Review person action, or for delivery of the resource described by a Promise. + + +### Last Review Time +> **Input Required**: False + +> **Attribute Type**: Simple + +> **Description**: Date/time (ISO 8601) that the promise was last reviewed. + + +### Completion Time +> **Input Required**: False + +> **Attribute Type**: Simple + +> **Description**: Date/time (ISO 8601) that the promised resource was delivered. + + +### Journal Entry +> **Input Required**: False + +> **Attribute Type**: Simple + +> **Description**: A text entry into a journal. + + +### Deployment Status +> **Input Required**: False + +> **Attribute Type**: Valid Value + +> **Description**: Progress toward delivering the promised real-world digital resource/artifact - one of an enumerated set of values (DeploymentStatus). + +> **Valid Values**: PROPOSED,UNDER_DEVELOPMENT,DEVELOPMENT_COMPLETE,APPROVED_FOR_DEPLOYMENT,REJECTED,STANDBY,ACTIVE,DISABLED,FAILED,OTHER + + +### Effective From +> **Input Required**: False + +> **Attribute Type**: Simple + +> **Description**: The beginning of when an element is viewable. + + +### Effective Time +> **Input Required**: False + +> **Attribute Type**: Simple + +> **Description**: The time at which an element must be effective in order to be returned by the request. + + +### Effective To +> **Input Required**: False + +> **Attribute Type**: Simple + +> **Description**: The ending time at which an element is visible. + + +### External Source GUID +> **Input Required**: False + +> **Attribute Type**: GUID + +> **Description**: The unique identifier of an external source. + + +### External Source Name +> **Input Required**: False + +> **Attribute Type**: Simple + +> **Description**: The name of an external source + + +### For Duplicate Processing +> **Input Required**: False + +> **Attribute Type**: Bool + +> **Description**: Flag indicating if the request is to support duplicate processing. + + +### For Lineage +> **Input Required**: False + +> **Attribute Type**: Bool + +> **Description**: Flag indicating if the request is to support lineage. + + +### Request ID +> **Input Required**: False + +> **Attribute Type**: Simple + +> **Description**: A user provided or system generated request id for a conversation. + + +### Additional Properties +> **Input Required**: False + +> **Attribute Type**: Dictionary + +> **Description**: Additional Properties allow arbitrary properties not defined in the type definitions to be added to any referenceable element. + +> | Parameter Name | Parameter Value | +> |---|---| +> | example_key | example_value | + + +___ diff --git a/sample-data/templates/advanced/Curation/Declassify_Promise.md b/sample-data/templates/advanced/Curation/Declassify_Promise.md new file mode 100644 index 00000000..b11b9d87 --- /dev/null +++ b/sample-data/templates/advanced/Curation/Declassify_Promise.md @@ -0,0 +1,86 @@ +___ + +## Declassify Promise +> Remove the Promise classification from an element - typically because the promised resource has now been delivered. The Target Element is looked up with For Lineage = true, since a promised element is otherwise invisible. + +### Target Element +> **Input Required**: True + +> **Attribute Type**: Reference Name + +> **Description**: Qualified name of the existing element being classified or linked. + + +### Journal Entry +> **Input Required**: False + +> **Attribute Type**: Simple + +> **Description**: A text entry into a journal. + + +### Effective From +> **Input Required**: False + +> **Attribute Type**: Simple + +> **Description**: The beginning of when an element is viewable. + + +### Effective Time +> **Input Required**: False + +> **Attribute Type**: Simple + +> **Description**: The time at which an element must be effective in order to be returned by the request. + + +### Effective To +> **Input Required**: False + +> **Attribute Type**: Simple + +> **Description**: The ending time at which an element is visible. + + +### External Source GUID +> **Input Required**: False + +> **Attribute Type**: GUID + +> **Description**: The unique identifier of an external source. + + +### External Source Name +> **Input Required**: False + +> **Attribute Type**: Simple + +> **Description**: The name of an external source + + +### For Duplicate Processing +> **Input Required**: False + +> **Attribute Type**: Bool + +> **Description**: Flag indicating if the request is to support duplicate processing. + + +### For Lineage +> **Input Required**: False + +> **Attribute Type**: Bool + +> **Description**: Flag indicating if the request is to support lineage. + + +### Request ID +> **Input Required**: False + +> **Attribute Type**: Simple + +> **Description**: A user provided or system generated request id for a conversation. + + +___ diff --git a/sample-data/templates/basic/Curation/Classify_Promise.md b/sample-data/templates/basic/Curation/Classify_Promise.md new file mode 100644 index 00000000..66edbac1 --- /dev/null +++ b/sample-data/templates/basic/Curation/Classify_Promise.md @@ -0,0 +1,72 @@ +___ + +## Classify Promise +> Classify an existing element as a Promise - a placeholder for a real-world digital resource/artifact that has not yet been delivered. Once classified, the element is only returned to lineage requests (For Lineage = true). + +### Target Element +> **Input Required**: True + +> **Attribute Type**: Reference Name + +> **Description**: Qualified name of the existing element being classified or linked. + + +### User Defined Deployment Status +> **Input Required**: False + +> **Attribute Type**: Simple + +> **Description**: Locally defined deployment status - used when Deployment Status is OTHER. + + +### Start Time +> **Input Required**: False + +> **Attribute Type**: Simple + +> **Description**: Date/time (ISO 8601) that work started on delivering the promised resource. + + +### Due Time +> **Input Required**: False + +> **Attribute Type**: Simple + +> **Description**: Due date/time - for a Meeting, ToDo, or Review person action, or for delivery of the resource described by a Promise. + + +### Last Review Time +> **Input Required**: False + +> **Attribute Type**: Simple + +> **Description**: Date/time (ISO 8601) that the promise was last reviewed. + + +### Completion Time +> **Input Required**: False + +> **Attribute Type**: Simple + +> **Description**: Date/time (ISO 8601) that the promised resource was delivered. + + +### Journal Entry +> **Input Required**: False + +> **Attribute Type**: Simple + +> **Description**: A text entry into a journal. + + +### Deployment Status +> **Input Required**: False + +> **Attribute Type**: Valid Value + +> **Description**: Progress toward delivering the promised real-world digital resource/artifact - one of an enumerated set of values (DeploymentStatus). + +> **Valid Values**: PROPOSED,UNDER_DEVELOPMENT,DEVELOPMENT_COMPLETE,APPROVED_FOR_DEPLOYMENT,REJECTED,STANDBY,ACTIVE,DISABLED,FAILED,OTHER + + +___ diff --git a/sample-data/templates/basic/Curation/Declassify_Promise.md b/sample-data/templates/basic/Curation/Declassify_Promise.md new file mode 100644 index 00000000..fa857650 --- /dev/null +++ b/sample-data/templates/basic/Curation/Declassify_Promise.md @@ -0,0 +1,22 @@ +___ + +## Declassify Promise +> Remove the Promise classification from an element - typically because the promised resource has now been delivered. The Target Element is looked up with For Lineage = true, since a promised element is otherwise invisible. + +### Target Element +> **Input Required**: True + +> **Attribute Type**: Reference Name + +> **Description**: Qualified name of the existing element being classified or linked. + + +### Journal Entry +> **Input Required**: False + +> **Attribute Type**: Simple + +> **Description**: A text entry into a journal. + + +___ diff --git a/tests/dr-egeria-command-tests/dr_test_curation.md b/tests/dr-egeria-command-tests/dr_test_curation.md index 71113b4e..01c7c4d7 100644 --- a/tests/dr-egeria-command-tests/dr_test_curation.md +++ b/tests/dr-egeria-command-tests/dr_test_curation.md @@ -27,6 +27,10 @@ > None of the 2026-08-21 additions have dedicated regression coverage in > this file yet -- add cases here if you're touching that area. > +> Promise (0010, added 2026-09-30) is covered by CT-63/CT-64. A promised +> element is only visible to forLineage=true requests; CT-64 exercises the +> lineage-aware Target Element lookup that Declassify Promise depends on. +> > `--process` cleanup: this file creates two Data Dictionaries, one Glossary, > one Glossary Term, one Meeting, one ToDo, and one Review as persistent > elements. Delete them from the target server after the run if you don't @@ -965,3 +969,34 @@ REQUESTED ### Description Created by dr_test_curation.md regression coverage. +___ + +# CT-63: Classify Promise + +## Classify Promise + +### Target Element +DataDictionary::CurationTest::Target::1.0 + +### Deployment Status +UNDER_DEVELOPMENT + +### Start Time +2026-09-30T00:00:00Z + +### Due Time +2026-12-31T00:00:00Z + +### Additional Properties +tracking_ticket: PROMISE-1 + +___ + +# CT-64: Declassify Promise + +## Declassify Promise + +### Target Element +DataDictionary::CurationTest::Target::1.0 + +___ diff --git a/tests/micro-tests/test_asset_lineage_for_lineage.py b/tests/micro-tests/test_asset_lineage_for_lineage.py new file mode 100644 index 00000000..21e1e276 --- /dev/null +++ b/tests/micro-tests/test_asset_lineage_for_lineage.py @@ -0,0 +1,39 @@ +""" +AssetCatalog.get_asset_lineage_graph_by_guid's for_lineage parameter: omitted +from the request body by default (leaving Egeria's own default in force), sent +as "forLineage": true when set -- so callers no longer need to inject it. +""" +import json + +import pytest + +from pyegeria.omvs.asset_catalog import AssetCatalog + + +class _Response: + def json(self): + return {"element": {"mermaidGraph": "graph TD"}} + + +def _client_capturing(bodies: list) -> AssetCatalog: + client = AssetCatalog("view-server", "https://localhost:9443", "user", "pwd") + + async def fake_make_request(method, url, body=None, *args, **kwargs): + bodies.append(json.loads(body) if isinstance(body, str) else body) + return _Response() + + client._async_make_request = fake_make_request + return client + + +@pytest.mark.asyncio +@pytest.mark.parametrize("kwargs, expected", [({}, None), ({"for_lineage": True}, True)]) +async def test_for_lineage_in_lineage_graph_body(kwargs, expected): + bodies: list = [] + client = _client_capturing(bodies) + + await client._async_get_asset_lineage_graph_by_guid("guid-1", output_format="JSON", **kwargs) + + assert bodies[0].get("forLineage") == expected + if expected is None: + assert "forLineage" not in bodies[0] diff --git a/tests/micro-tests/test_curation_promise.py b/tests/micro-tests/test_curation_promise.py new file mode 100644 index 00000000..19f2aaf3 --- /dev/null +++ b/tests/micro-tests/test_curation_promise.py @@ -0,0 +1,99 @@ +""" +Classify/Declassify Promise (0010) through the real dispatcher and +CurationClassifyProcessor, with a fake client. + +A Promise-classified element is only visible to forLineage=true requests, so +Declassify Promise must (a) resolve its Target Element with for_lineage=True +and (b) send its clear request with forLineage=true. Classify Promise, whose +target is not yet hidden, must keep using a normal lookup. +""" +from typing import Any, Dict, List + +import pytest + +from pyegeria import NO_ELEMENTS_FOUND + +from md_processing.dr_egeria import register_curation_processors +from md_processing.v2.dispatcher import V2Dispatcher +from md_processing.v2.extraction import DrECommand + +_TARGET = "DataSet::Promised Sales Feed" +_TARGET_GUID = "11111111-2222-3333-4444-555555555555" + + +class _FakeClassificationManager: + def __init__(self): + self.set_calls: List[tuple] = [] + self.clear_calls: List[tuple] = [] + + async def _async_set_element_as_promise(self, element_guid, body): + self.set_calls.append((element_guid, body)) + + async def _async_clear_element_as_promise(self, element_guid, body): + self.clear_calls.append((element_guid, body)) + + +class _FakeClient: + """Target is found by a normal lookup until it has been promised; after + that only a for_lineage=True lookup finds it (as on a real server).""" + + def __init__(self, promised: bool): + self.promised = promised + self.lookups: List[bool] = [] + self.classification_manager = _FakeClassificationManager() + + async def __async_get_guid__(self, qualified_name=None, display_name=None, property_name="qualifiedName", + tech_type=None, for_lineage=False, **kwargs): + name = qualified_name or display_name + if name != _TARGET: + return NO_ELEMENTS_FOUND + self.lookups.append(for_lineage) + if self.promised and not for_lineage: + return NO_ELEMENTS_FOUND + return _TARGET_GUID + + +def _dispatcher(client) -> V2Dispatcher: + dispatcher = V2Dispatcher(client) + register_curation_processors(dispatcher.register) + return dispatcher + + +def _cmd(verb: str, attributes: Dict[str, Any]) -> DrECommand: + raw = f"## {verb} Promise\n" + "".join(f"### {k}\n{v}\n" for k, v in attributes.items()) + return DrECommand(verb=verb, object_type="Promise", attributes=attributes, raw_block=raw) + + +@pytest.mark.asyncio +async def test_classify_promise_sends_promise_properties(): + client = _FakeClient(promised=False) + results = await _dispatcher(client).dispatch_batch( + [_cmd("Classify", {"Target Element": _TARGET, "Deployment Status": "UNDER_DEVELOPMENT", + "Due Time": "2026-12-31T00:00:00"})], + {"directive": "process"}, + ) + + assert results[0]["status"] == "success", results[0] + [(guid, body)] = client.classification_manager.set_calls + assert guid == _TARGET_GUID + assert body["class"] == "NewClassificationRequestBody" + assert body["properties"]["class"] == "PromiseProperties" + assert body["properties"]["deploymentStatus"] == "UNDER_DEVELOPMENT" + assert body["properties"]["dueTime"] == "2026-12-31T00:00:00" + assert True not in client.lookups # target not hidden yet: normal lookup only + + +@pytest.mark.asyncio +async def test_declassify_promise_uses_lineage_lookup_and_body(): + client = _FakeClient(promised=True) + results = await _dispatcher(client).dispatch_batch( + [_cmd("Declassify", {"Target Element": _TARGET})], + {"directive": "process"}, + ) + + assert results[0]["status"] == "success", results[0] + [(guid, body)] = client.classification_manager.clear_calls + assert guid == _TARGET_GUID + assert body["class"] == "DeleteClassificationRequestBody" + assert body["forLineage"] is True + assert client.lookups and all(client.lookups) diff --git a/tests/micro-tests/test_dr_egeria_extraction.py b/tests/micro-tests/test_dr_egeria_extraction.py index 74acbddb..4166c12e 100644 --- a/tests/micro-tests/test_dr_egeria_extraction.py +++ b/tests/micro-tests/test_dr_egeria_extraction.py @@ -114,6 +114,11 @@ def test_parse_key_value(): kv = parse_key_value(text_list) assert kv == {"x": "y", "z": "10"} + # JSON object: parsed as JSON, not split on its first ':' (which used to + # yield the key '{"owner"'). Non-string values come back JSON-encoded. + kv = parse_key_value('{"owner": "promise-test", "points": 12, "type_map": [["Terms", "GlossaryTerm"]]}') + assert kv == {"owner": "promise-test", "points": "12", "type_map": '[["Terms", "GlossaryTerm"]]'} + @pytest.mark.asyncio async def test_extract_and_parse_integration(): text = """