Merge pull request #226 from dwolfson/Dr.E-Refactoring

Dr.e refactoring

Author: dwolfson

md_processing/md_processing_utils/common_md_utils.py

@@ -61,9 +61,18 @@
 global COMMAND_DEFINITIONS
 
 def split_tb_string(input: str)-> [Any]:
-    """Split the string and trim the items"""
-    l = [item.strip() for item in re.split(r'[;,\n]+',input)] if input is not None else None
-    return l
+    """Split the string and trim the items, supporting markdown bullets"""
+    if input is None:
+        return None
+    items = re.split(r'[;,\n]+', input)
+    results = []
+    for item in items:
+        item = item.strip()
+        if item:
+            # Strip leading markdown bullet if present
+            item = re.sub(r'^[\-\*\+]\s*', '', item)
+            results.append(item)
+    return results
 
 def str_to_bool(value: str) -> bool:
     """Converts a string to a boolean value."""
@@ -371,10 +380,13 @@ def set_element_prop_body(object_type: str, qualified_name: str, attributes: dic
         "class": prop_name + "Properties",
         "typeName": prop_name,
         "displayName": attributes.get('Display Name', {}).get('value', None),
+        "name": attributes.get('Display Name', {}).get('value', None) if ("Project" in object_type or "Campaign" in object_type or "Task" in object_type or "StudyProject" in object_type) else None,
         "qualifiedName" : qualified_name,
         "description": attributes.get('Description', {}).get('value', None),
         "category": attributes.get('Category', {}).get('value', None),
         "identifier": attributes.get('Identifier', {}).get('value', None),
+        "url": attributes.get('URL', {}).get('value', None),
+        "authors": attributes.get('Authors', {}).get('value', []),
         "contentStatus": attributes.get('Content Status', {}).get('value', None) or attributes.get('Status', {}).get('value', None),
         "userDefinedContentStatus": attributes.get('User Defined Content Status', {}).get('value', None),
         "versionIdentifier": attributes.get('Version Identifier', {}).get('value', None),
@@ -409,6 +421,9 @@ def set_collection_manager_body(object_type: str, qualified_name: str, attribute
         prop_bod.update({
             "agreementType": attributes.get('Agreement Type', {}).get('value', None),
             "agreementStatus": attributes.get('Agreement Status', {}).get('value', None),
+            "agreementIdentifier": attributes.get('Agreement Identifier', {}).get('value', None),
+            "agreementStatusIdentifier": attributes.get('Agreement Status Identifier', {}).get('value', None),
+            "agreementVersionIdentifier": attributes.get('Agreement Version Identifier', {}).get('value', None),
         })
         if "Subscription" in object_type:
             prop_bod.update({
@@ -485,9 +500,13 @@ def set_gov_prop_body(object_type: str, qualified_name: str, attributes: dict)->
 
     prop_bod["scope"] = attributes.get('Scope', {}).get('value', None)
     prop_bod["importance"] = attributes.get('Importance', {}).get('value', None)
-    prop_bod["implications"] = attributes.get('Implication', {}).get('value', [])
+    prop_bod["implications"] = attributes.get('Implications', {}).get('value', [])
     prop_bod["outcomes"] = attributes.get('Outcomes', {}).get('value', [])
     prop_bod["results"] = attributes.get('Results', {}).get('value', []) or []
+    prop_bod["entitlements"] = attributes.get('Entitlements', {}).get('value', [])
+    prop_bod["obligations"] = attributes.get('Obligations', {}).get('value', [])
+    prop_bod["restrictions"] = attributes.get('Restrictions', {}).get('value', [])
+    prop_bod["usage"] = attributes.get('Usage', {}).get('value', None)
     prop_bod["effectiveFrom"] = attributes.get('Effective From', {}).get('value', None)
     prop_bod["effectiveTo"] = attributes.get('Effective To', {}).get('value', None)
     prop_bod["additionalProperties"] = attributes.get('Additional Properties', {}).get('value', None)

md_processing/v2/glossary.py

@@ -140,6 +140,13 @@ async def apply_changes(self) -> str:
         prop_body["abbreviation"] = attributes.get('Abbreviation', {}).get('value', None)
         prop_body["usage"] = attributes.get('Usage', {}).get('value', None)
         prop_body["user_defined_status"] = attributes.get('UserDefinedStatus', {}).get('value', None)
+        prop_body["contextDescription"] = attributes.get('Context Description', {}).get('value', None)
+        prop_body["contextScope"] = attributes.get('Context Scope', {}).get('value', None)
+        prop_body["isAbstractConcept"] = attributes.get('Is Abstract Concept', {}).get('value', False)
+        prop_body["isActivityDescription"] = attributes.get('Is Activity Description', {}).get('value', False)
+        prop_body["isContext"] = attributes.get('Is Context', {}).get('value', False)
+        prop_body["isDataValue"] = attributes.get('Is Data Value', {}).get('value', False)
+        prop_body["termActivityType"] = attributes.get('Term Activity Type', {}).get('value', None)
 
         # 2. Extract collection GUIDs
         # We may have one or more collections listed (Glossary Name, Folders)

md_processing/v2/parsing.py

@@ -34,7 +34,13 @@ async def parse(self) -> Dict[str, Any]:
         if not self.spec:
             logger.error(f"No specification found for command: {self.command.verb} {self.command.object_type}")
             self.errors.append(f"No specification found for command: {self.command.verb} {self.command.object_type}")
-            return {}
+            return {
+                "attributes": {},
+                "valid": False,
+                "errors": self.errors,
+                "warnings": self.warnings,
+                "qualified_name": None
+            }
 
         # Spec can use "Attributes" or "attributes"
         spec_attrs = self.spec.get("Attributes", self.spec.get("attributes", []))
@@ -364,7 +370,15 @@ async def _process_attribute_value(self, value: str, details: dict) -> Any:
             return v
 
         elif style in {"List", "NameList", "Simple List", "Reference Name List"}:
-            return [v.strip() for v in re.split(r'[,\n]', value) if v.strip()]
+            items = re.split(r'[;,\n]+', value)
+            results = []
+            for item in items:
+                item = item.strip()
+                if item:
+                    # Strip leading markdown bullet if present
+                    item = re.sub(r'^[\-\*\+]\s*', '', item)
+                    results.append(item)
+            return results
 
         return value
 

md_processing/v2/processors.py

@@ -12,7 +12,7 @@
 from pyegeria import EgeriaTech, PyegeriaException, NO_ELEMENTS_FOUND, print_basic_exception
 from pyegeria.core.utils import make_format_set_name_from_type
 from pyegeria.view.base_report_formats import select_report_spec
-from pyegeria.view.output_formatter import generate_output, format_for_markdown_table
+from pyegeria.view.output_formatter import generate_output, format_for_markdown_table, populate_columns_from_properties
 
 from md_processing.v2.extraction import DrECommand
 from md_processing.v2.parsing import AttributeFirstParser
@@ -723,7 +723,8 @@ async def render_result_markdown(self, guid: str) -> str:
                 search_string=self.parsed_output.get("qualified_name", "Created/Updated Element"),
                 entity_type=self.command.object_type,
                 output_format="MD",
-                columns_struct=columns_struct
+                columns_struct=columns_struct,
+                extract_properties_func=populate_columns_from_properties
             )
             return markdown
         except Exception as e:

md_processing/v2/project.py

@@ -44,14 +44,8 @@ async def apply_changes(self) -> str:
         project_types = ["Campaign", "Task", "Personal Project", "Study Project"]
 
         # 1. Properties
-        # ProjectManager expects 'name' instead of 'displayName' and 'plannedEndDate' instead of 'endDate'
-        prop_body = {
-            "class": f"{base_type}Properties",
-            # "typeName": self.command.object_type.replace(" ", ""),
-            "qualifiedName": qualified_name,
-            "displayName": display_name,
-            "description": attributes.get('Description', {}).get('value'),
-            "identifier": attributes.get('Identifier', {}).get('value'),
+        prop_body = set_element_prop_body(base_type, qualified_name, attributes)
+        prop_body.update({
             "projectStatus": attributes.get('Project Status', {}).get('value'),
             "projectPhase": attributes.get('Project Phase', {}).get('value'),
             "projectHealth": attributes.get('Project Health', {}).get('value'),
@@ -60,11 +54,16 @@ async def apply_changes(self) -> str:
             "actualStartDate": attributes.get('Actual Start Date', {}).get('value'),
             "actualCompletionDate": attributes.get('Actual Completion Date', {}).get('value'),
             "mission": attributes.get('Mission', {}).get('value'),
-            "purposes": attributes.get('Purposes', {}).get('value'),
+            "purposes": attributes.get('Purposes', {}).get('value', []),
             "priority": attributes.get('Priority', {}).get('value'),
             "successCriteria": attributes.get('Success Criteria', {}).get('value'),
-            "contentStatus": attributes.get('Content Status', {}).get('value'),
-        }
+            "projectIdentifier": attributes.get('Project Identifier', {}).get('value'),
+            "projectType": attributes.get('Project Type', {}).get('value'),
+            "projectScope": attributes.get('Project Scope', {}).get('value'),
+            "projectApproach": attributes.get('Project Approach', {}).get('value'),
+            "projectManagementStyle": attributes.get('Project Management Style', {}).get('value'),
+            "projectResultsUsage": attributes.get('Project Results Usage', {}).get('value'),
+        })
 
         if verb == "Update":
             guid = self.parsed_output.get("guid") or (self.as_is_element['elementHeader']['guid'] if self.as_is_element else None)

pyegeria/models/models.py

@@ -141,6 +141,10 @@ class ReferenceableProperties(OpenMetadataRootProperties):
     version_identifier: str | None = None
     description: str | None = None
     category: str | None = None
+    authors: list[str] | None = None
+    usage: str | None = None
+    url: str | None = None
+    content_status: str | None = None
     additional_properties: dict | None = None
 
 

pyegeria/omvs/glossary_manager.py

@@ -42,6 +42,13 @@ class GlossaryTermProperties(ReferenceableProperties):
     usage: str = None
     user_defined_status: str = None
     publishVersionIdentifier: str = None
+    contextDescription: str = None
+    contextScope: str = None
+    isAbstractConcept: bool = False
+    isActivityDescription: bool = False
+    isContext: bool = False
+    isDataValue: bool = False
+    termActivityType: str = None
 
 
 class GlossaryManager(CollectionManager):

pyegeria/omvs/governance_officer.py

@@ -69,6 +69,9 @@ class GovernanceDefinitionProperties(ReferenceableProperties):
     implications: list[str] | None = None
     outcomes: list[str] | None = None
     results: list[str] | None = None
+    entitlements: list[str] | None = None
+    obligations: list[str] | None = None
+    restrictions: list[str] | None = None
     userDefinedStatus: str | None = None
 
 

pyegeria/omvs/security_officer.py

@@ -69,7 +69,26 @@ async def __get_platform_guid__(self, platform_name: str) -> str:
     async def _async_set_user_account(
         self, platform_name: str, body: Union[dict, Any], platform_guid: str = None
     ) -> None:
-        """Create or update a user account in the platform metadata security connector. Async version."""
+        """Create or update a user account in the platform metadata security connector. Async version.
+        {
+            "class": "UserAccountRequestBody",
+            "userAccount": {
+                "class": "OpenMetadataUserAccount",
+                "userId": "{{accountUserId}}",
+                "userName": "Freddie Mercury",
+                "userAccountType": "EXTERNAL",
+                "employeeNumber": "",
+                "employeeType": "",
+                "givenName": "Freddie",
+                "surname": "Mercury",
+                "email": "freddiemercury@queen.com",
+                "securityRoles": ["serverOperator", "serverInvestigator"],
+                "userAccountStatus": "CREDENTIALS_EXPIRED",
+                "secrets": {
+                    "clearPassword": "itsakindofmagic"
+                }
+            }
+        """
         if not platform_guid:
             platform_guid = await self.__get_platform_guid__(platform_name)
         url = f"{self.security_officer_base_url}/platforms/{platform_guid}/user-accounts"