Merge pull request #227 from dwolfson/Dr.E-Refactoring

Dr.e refactoring

Author: dwolfson

commands/cat/Dr-Egeria_md-orig.py

@@ -1,5 +1,5 @@
 """
-This is an ongoing experiment in parsing and playing with Freddie docs
+This is an ongoing experiment in parsing and playing with Dr.Egeria docs
 """
 import json
 import os

commands/cat/dr_egeria.py

@@ -1,5 +1,5 @@
 """
-This is an ongoing experiment in parsing and playing with Freddie docs
+This is an ongoing experiment in parsing and playing with Dr.Egeria docs
 """
 import os
 import sys
@@ -35,8 +35,7 @@
 console = Console(width=EGERIA_WIDTH)
 
 @click.command("process_markdown_file", help="Process a markdown file and return the output as a string.")
-@click.option("--input-file", help="Markdown file to process.", default="dr_egeria_intro_part1.md",
-              prompt="Markdown file to process")
+@click.argument("input_file", default="dr_egeria_intro_part1.md", required=False)
 @click.option("--output-folder", help="Output folder.", default="", required=False)
 @click.option("--directive", default="validate", help="How to process the file (display/validate/process). "
               "Overridden by --validate or --process flags.",
@@ -109,14 +108,4 @@ def _running_in_pycharm_debugger() -> bool:
     return sys.gettrace() is not None or os.environ.get("PYCHARM_HOSTED") is not None
 
 if __name__ == "__main__":
-    # if _running_in_pycharm_debugger():
-    input_file = Prompt.ask("Markdown File name to process:", default="dr_egeria_intro_part1.md")
-    directive = Prompt.ask("Directive (display/validate/process):", default="validate")
-    advanced = Prompt.ask("Use advanced usage level? [y/N]:", default="N").strip().lower() in ("y", "yes")
-    process_markdown_file.callback(input_file, "", directive,
-                            do_validate=False, do_process=False,
-                            server=EGERIA_VIEW_SERVER, url=EGERIA_VIEW_SERVER_URL, userid=EGERIA_USER,
-                            user_pass=EGERIA_USER_PASSWORD, parse_summary="all", attribute_logs="debug",
-                            advanced=advanced, summary_only=False, debug=False)
-    # else:
-    #     process_markdown_file()
+    process_markdown_file()

commands/cat/dr_egeria_jupyter.py

@@ -1,5 +1,5 @@
 """
-This is an ongoing experiment in parsing and playing with Freddie docs
+This is an ongoing experiment in parsing and playing with Dr.Egeria docs
 """
 
 import nbformat
@@ -45,8 +45,7 @@
 
 
 @click.command("process-jupyter")
-@click.option("--file-path", help="File path to notebook",
-              default="glossary_creation_experiment.ipynb")
+@click.argument("input_file", default="glossary_creation_experiment.ipynb", required=False)
 @click.option("--directive", default="display-only", help="How to process the file")
 @click.option("--server", default=EGERIA_VIEW_SERVER, help="Egeria view server to use.")
 @click.option(
@@ -55,7 +54,7 @@
 @click.option("--userid", default=EGERIA_USER, help="Egeria user")
 @click.option("--user_pass", default=EGERIA_USER_PASSWORD, help="Egeria user password")
 def process_jupyter_notebook(
-        file_path: str,
+        input_file: str,
         directive: str,
         server: str,
         url: str,
@@ -69,7 +68,7 @@ def process_jupyter_notebook(
 
     try:
         updated = False
-        full_file_path = os.path.join(EGERIA_ROOT_PATH, EGERIA_INBOX_PATH, file_path)
+        full_file_path = os.path.join(EGERIA_ROOT_PATH, EGERIA_INBOX_PATH, input_file)
         click.echo(f"Processing notebook: {full_file_path}")
         with open(full_file_path, 'r') as f:
             nb = nbformat.read(f, as_version=4)
@@ -101,7 +100,7 @@ def process_jupyter_notebook(
 
 
         if updated:
-            path, filename = os.path.split(file_path)  # Get both parts
+            path, filename = os.path.split(input_file)  # Get both parts
             new_filename = f"processed-{get_current_datetime_string()}-{filename}"  # Create the new filename
             new_file_path = os.path.join(EGERIA_ROOT_PATH, EGERIA_OUTBOX_PATH, new_filename)  # Construct the new path
             os.makedirs(os.path.dirname(new_file_path), exist_ok=True)

commands/tech/gen_report_specs.py

@@ -275,12 +275,26 @@ def _load_command_specs(path: Path | str) -> dict:
     path_str = str(path)
     try:
         from md_processing.md_processing_utils.compact_loader import load_compact_specs_from_dir
-        specs = load_compact_specs_from_dir(path_str)
-        if specs:
-            logger.info(f"Loaded compact specs from {path_str} with {len(specs)} commands")
-            return specs
+        if os.path.isdir(path_str):
+            specs = load_compact_specs_from_dir(path_str)
+            if specs:
+                logger.info(f"Loaded compact specs from {path_str} with {len(specs)} commands")
+                return specs
+        elif os.path.isfile(path_str):
+            # Fallback for single file (used in tests)
+            from md_processing.md_processing_utils.parse_compact_export import load_compact_json, expand_command
+            from md_processing.md_processing_utils.compact_loader import _command_to_spec
+            data = load_compact_json(path_str)
+            all_attr_defs = data.get("attribute_definitions", {})
+            all_bundles = data.get("bundles", {})
+            specs = {}
+            for cname, cdef in data.get("commands", {}).items():
+                expanded = expand_command(cdef, all_bundles, all_attr_defs)
+                specs[cname] = _command_to_spec(cname, cdef, expanded)
+            if specs:
+                return specs
     except Exception as e:
-        logger.error(f"Failed to use compact_loader on dir {path_str}: {e}")
+        logger.error(f"Failed to load specs from {path_str}: {e}")
     return {}
 
 def build_format_sets_from_commands(

commands/tech/generate_dr_help.py

@@ -180,10 +180,10 @@ def create_help_terms(client: Optional[ServerClient] = None, advanced: bool = Fa
                                         "attributes": columns}
                       }
 
-    term_entry = f"""# Create Glossary\n
-## Display Name\n\n{glossary_name}\n
-## Qualified Name\n\n{glossary_qn}\n
-## Description\n\nDr.Egeria command help glossary.\n\n___\n\n"""
+    term_entry = f"""## Create Glossary\n
+### Display Name\n\n{glossary_name}\n
+### Qualified Name\n\n{glossary_qn}\n
+### Description\n\nDr.Egeria command help glossary.\n\n___\n\n"""
 
     families = set()
     for command, values in commands.items():
@@ -194,7 +194,7 @@ def create_help_terms(client: Optional[ServerClient] = None, advanced: bool = Fa
 
     for family in families:
         family_qn = f"CollectionFolder::dr-egeria:{family.lower().replace(' ', '-')}"
-        term_entry += f"# Create Collection Folder\n## Display Name\n\n{family}\n\n## Qualified Name\n\n{family_qn}\n\n## Purpose\n\nDr-Egeria Definitions\n\n## Parent ID\n\n{glossary_qn}\n\n## Parent Relationship Type Name\n\nCollectionMembership\n\n___\n\n"
+        term_entry += f"## Create Collection Folder\n### Display Name\n\n{family}\n\n### Qualified Name\n\n{family_qn}\n\n### Purpose\n\nDr-Egeria Definitions\n\n### Parent ID\n\n{glossary_qn}\n\n### Parent Relationship Type Name\n\nCollectionMembership\n\n___\n\n"
 
     for command, values in commands.items():
         if command == "exported":
@@ -206,17 +206,17 @@ def create_help_terms(client: Optional[ServerClient] = None, advanced: bool = Fa
         family = commands[command].get("family", "General")
         family_qn = f"CollectionFolder::dr-egeria:{family.lower().replace(' ', '-')}"
 
-        term_entry+= "# Create Term\n"
-        term_entry+= f"## Display Name\n\n{command}\n\n"
-        term_entry+= f"## Description\n\n{command_description}\n\n"
-        term_entry+= f"## Glossary Name\n\n{glossary_qn}\n\n"
-        term_entry+= f"## Folders\n\n{family_qn}\n\n"
+        term_entry+= "## Create Term\n"
+        term_entry+= f"### Display Name\n\n{command}\n\n"
+        term_entry+= f"### Description\n\n{command_description}\n\n"
+        term_entry+= f"### Glossary Name\n\n{glossary_qn}\n\n"
+        term_entry+= f"### Folders\n\n{family_qn}\n\n"
 
 
         du = _extract_help_fields(command, client=client, advanced=advanced)
         output = generate_entity_md_table(du, "", "", _extract_help_function, columns_struct, None, "help" )
 
-        term_entry+= f"## Usage\n\n{output}\n\n___\n\n"
+        term_entry+= f"### Usage\n\n{output}\n\n___\n\n"
 
     print(term_entry)
     # Generate filename with current date and time in ISO 8601 format

commands/tech/generate_md_cmd_templates.py

@@ -77,8 +77,8 @@ def _level_visible(attr_level: str, usage_level: str = "Advanced") -> bool:
 # ---------------------------------------------------------------------------
 
 def _write_attr_block(out: io.StringIO, key: str, value: dict, client: Optional[ServerClient] = None) -> None:
-    """Write a single ## attribute block into the StringIO buffer."""
-    out.write(f"\n## {key}\n")
+    """Write a single ### attribute block into the StringIO buffer."""
+    out.write(f"\n### {key}\n")
     out.write(f">\t**Input Required**: {value.get('input_required', 'false')}\n\n")
 
     attr_type = value.get("data_type") or value.get("style") or ""
@@ -124,10 +124,15 @@ def _write_attr_block(out: io.StringIO, key: str, value: dict, client: Optional[
     if str(default_value).strip():
         out.write(f">\t**Default Value**: {default_value}\n\n")
 
+    if attr_type and attr_type.strip().lower() in ("dictionary", "keyvalue", "named dict"):
+        out.write(">\t| Parameter Name | Parameter Value |\n")
+        out.write(">\t|---|---|\n")
+        out.write(">\t| example_key | example_value |\n\n")
+
 
 def _print_attr(key: str, value: dict, client: Optional[ServerClient] = None) -> None:
     """Mirror of _write_attr_block for console output."""
-    print(f"\n## {key}")
+    print(f"\n### {key}")
     print(f">\tInput Required: {value.get('input_required', 'false')}")
 
     attr_type = value.get("data_type") or value.get("style") or ""
@@ -172,30 +177,35 @@ def _print_attr(key: str, value: dict, client: Optional[ServerClient] = None) ->
     if str(default_value).strip():
         print(f">\tDefault Value: {default_value}")
 
+    if attr_type and attr_type.strip().lower() in ("dictionary", "keyvalue", "named dict"):
+        print(">\t| Parameter Name | Parameter Value |")
+        print(">\t|---|---|")
+        print(">\t| example_key | example_value |\n")
+
 
 @logger.catch
 def main():
     """Generate markdown command templates organised by family.
 
     Each command file is structured as:
 
-        # <Command Name>
+        ## <Command Name>
         > <description>
 
-        # Required
-        ## <attr>  ...   (input_required=true, any visible level)
+        ## Required
+        ### <attr>  ...   (input_required=true, any visible level)
 
-        # Common Properties
-        ## <attr>  ...   (level=Common, optional)
+        ## Common Properties
+        ### <attr>  ...   (level=Common, optional)
 
-        # <DisplayName> Properties
-        ## <attr>  ...   (level=Domain, optional; heading uses command display_name)
+        ## <DisplayName> Properties
+        ### <attr>  ...   (level=Domain, optional; heading uses command display_name)
 
-        # Additional Properties
-        ## <attr>  ...   (level=Basic, optional)
+        ## Additional Properties
+        ### <attr>  ...   (level=Basic, optional)
 
-        # Advanced Properties
-        ## <attr>  ...   (level=Advanced, optional)
+        ## Advanced Properties
+        ### <attr>  ...   (level=Advanced, optional)
 
     Sections with no attributes are omitted entirely.
     Link commands only ever produce Required and Advanced sections from defaults;
@@ -258,7 +268,7 @@ def main():
 
         family_dir = os.path.join(base_output_dir, family)
         os.makedirs(family_dir, exist_ok=True)
-        print(f"\n# Family: {family}\n")
+        print(f"\n## Family: {family}\n")
 
         for command in sorted(command_list):
             values = commands[command]
@@ -326,13 +336,14 @@ def main():
             # Build the output buffer
             # -----------------------------------------------------------
             command_output = io.StringIO()
-            command_output.write(f"# {command}\n")
+            command_output.write("___\n\n")
+            command_output.write(f"## {command}\n")
             if command_description:
                 command_output.write(f"> {command_description}\n")
             if alternate_names:
                 command_output.write(f">\n>\t**Alternative Names**: {alternate_names}\n")
 
-            print(f"\n# {command}")
+            print(f"\n## {command}")
             if command_description:
                 print(f"> {command_description}")
             if alternate_names:
@@ -352,13 +363,15 @@ def main():
                 if not attr_list:
                     continue   # omit empty sections entirely
 
-                # command_output.write(f"\n# {section_title}\n")
-                # print(f"\n# {section_title}")
+                # command_output.write(f"\n## {section_title}\n")
+                # print(f"\n## {section_title}")
 
                 for key, value in attr_list:
                     _write_attr_block(command_output, key, value, client=client)
                     _print_attr(key, value, client=client)
 
+            command_output.write("\n___\n")
+
             # -----------------------------------------------------------
             # Save to file
             # -----------------------------------------------------------

docs/design/dr-egeria-execution-engine-walkthrough.md

@@ -42,7 +42,7 @@ I verified the v2 engine's compatibility with legacy DrE markdown samples using
 - **Infrastructure Fixes**: Resolved bugs in `processors.py` regarding parser initialization and `DrECommand.raw_block` attribute naming.
 
 ```bash
-PYTHONPATH=. EGERIA_ROOT_PATH=sample-data python commands/cat/dr_egeria_md.py --input-file Simple-Data-Lens.md --directive validate
+PYTHONPATH=. EGERIA_ROOT_PATH=sample-data python commands/cat/dr_egeria.py Simple-Data-Lens.md --directive validate
 ```
 
 ## Phase Status Summary
@@ -78,22 +78,22 @@ PYTHONPATH=. EGERIA_ROOT_PATH=sample-data python commands/cat/dr_egeria_md.py --
 
 ```bash
 # Sequential execution of multiple commands in feedback.md
-PYTHONPATH=. EGERIA_ROOT_PATH=sample-data python commands/cat/dr_egeria_md.py --input-file feedback.md --directive validate
+PYTHONPATH=. EGERIA_ROOT_PATH=sample-data python commands/cat/dr_egeria.py feedback.md --directive validate
 ```
 
 ### 5. Invocation & Dependency Fixes
 
 - **Unified Entry Point**: Renamed the CLI script to `commands/cat/dr_egeria.py` and added a `dr_egeria` script alias in `pyproject.toml`.
 - **Direct Execution**: Restored the `if __name__ == "__main__"` block in `md_processing/dr_egeria.py` for manual invocation.
 - **Dependency Management**: Updated the `v2/README.md` with explicit installation instructions (`pip install -e .`) to ensure all dependencies like `loguru` are correctly resolved.
-- **Consistency**: Standardized command-line arguments to use `--input-file` across all entry points.
+- **Consistency**: Standardized command-line arguments to use a positional input file across all entry points.
 
 ```bash
 # Correct usage with installed script
-EGERIA_ROOT_PATH=sample-data dr_egeria --input-file project.md --directive validate
+EGERIA_ROOT_PATH=sample-data dr_egeria project.md --directive validate
 
 # Direct execution of the command in /commands/cat
-EGERIA_ROOT_PATH=sample-data python commands/cat/dr_egeria.py --input-file project.md --directive validate
+EGERIA_ROOT_PATH=sample-data python commands/cat/dr_egeria.py project.md --directive validate
 ```
 
 ### 6. Configuration & Path Robustness
@@ -104,7 +104,7 @@ EGERIA_ROOT_PATH=sample-data python commands/cat/dr_egeria.py --input-file proje
 
 ```bash
 # Works out of the box from the project root
-dr_egeria --input-file project.md --directive validate
+dr_egeria project.md --directive validate
 ```
 
 ## Final Status Summary

docs/design/dr-egeria-module-structure.md

@@ -79,7 +79,7 @@ sequenceDiagram
     participant SDK  as EgeriaTech / OMVS
     participant EGERIA as Egeria Server
 
-    User->>CLI: hey_egeria cat process-markdown-file --input-file X.md [--validate|--process] [--advanced] [--debug]
+    User->>CLI: dr_egeria X.md [--validate|--process] [--advanced] [--debug]
     CLI->>ORCH: asyncio.run(process_md_file_v2(..., debug=...))
 
     opt debug=True
@@ -244,7 +244,7 @@ flowchart LR
 
 | File | Role |
 |------|------|
-| `commands/cat/dr_egeria.py` | Click CLI entry-point; `--validate` / `--process` shortcut flags, `--advanced`, `--debug` |
+| `commands/cat/dr_egeria.py` | Click CLI entry-point: `dr_egeria <file.md>`. Shortcuts: `--validate` / `--process`, `--advanced`, `--debug` |
 | `md_processing/dr_egeria.py` | Async orchestrator: extraction → dispatch → output; owns debug patch/restore |
 | `md_processing/v2/extraction.py` | `UniversalExtractor` — produces `DrECommand` list from raw Markdown |
 | `md_processing/v2/parsing.py` | `AttributeFirstParser` — async, spec-driven, with server-side validation cache |

docs/design/dr-egeria-parser-change.md

@@ -0,0 +1,44 @@
+# Proposal: Shift Dr.Egeria Markdown Syntax Hierarchy
+
+This plan outlines the changes required to adjust the Dr.Egeria parsing and generation syntax so that:
+- **`#` (Single Hash)**: Ignored by the parser; treated as standard Markdown document headers or comments.
+- **`##` (Double Hash)**: Defines a Dr.Egeria Command.
+- **`###` (Triple Hash)**: Defines a Command Attribute.
+
+## Motivation
+This change will allow users to comfortably use single-hash `#` headers to title their overarching Markdown documents or sections, reserving the `##` and `###` headers strictly for Dr.Egeria execution contexts.
+
+## Proposed Changes
+
+### 1. Update the Extraction Logic (`md_processing/v2/extraction.py`)
+The `UniversalExtractor` determines how markdown text is split into command blocks and attributes.
+- **Command Header Matching**: 
+  - Update `self.cmd_header_rx` to match `## <Verb> <Object>`. 
+  - Update `_split_into_blocks()` so that it looks for `line.startswith("## ")` instead of `line.startswith("# ")` when identifying block boundaries.
+- **Attribute Header Matching**:
+  - Update `_extract_attributes_from_block()` to use `^###\s+` in its regular expression instead of `^##\s+`.
+
+### 2. Update Template Generation (`commands/tech/generate_md_cmd_templates.py`)
+The generator needs to produce templates in the new syntax.
+- **Command Titles**: Update `_generate_md_file_content()` and the console output equivalent to prefix commands with `## ` instead of `# `.
+- **Attribute Titles**: Update `_write_attr_block()` and `_print_attr()` to prefix attributes with `### ` instead of `## `.
+
+### 3. Documentation & Sample Data Migration
+Existing Markdown files (smoke tests, templates, inbox documentation) will no longer parse correctly.
+- Run `uv run gen_md_cmd_templates --usage-level Basic` and `--usage-level Advanced` to forcefully regenerate all built-in template files in the new syntax.
+- Manually (or via script) update `sample-data/smoke_test_gov_officer.md` and `sample-data/smoke_test_terms_and_conditions.md` to shift all hashes down one level.
+- Any Markdown files provided to users in the inbox (e.g. `dr_egeria_intro_*.md`) may need an update to reflect the new expected format.
+
+## Verification Plan
+
+### Automated Verification
+- Re-run Pytest unit tests. Many tests under `tests/micro-tests/` that feed string literals to `UniversalExtractor` will fail and need to be updated to the new `##` and `###` syntax.
+- Specifically, the parser tests in `tests/micro-tests/md_processing/v2/test_extraction.py` will serve as our primary validation that the single hash `#` is ignored and `##`/`###` parse correctly.
+
+### Manual Verification
+- Re-run validation on the smoke tests: `dr_egeria sample-data/smoke_test_gov_officer.md --validate` to ensure it parses successfully.
+
+> [!WARNING] 
+> **Breaking Change**: This is a breaking change for any existing user Markdown scripts. Users who have created their own `dr_egeria` markdown documents will need to run a quick search-and-replace to shift their headers down one level. 
+
+Does this approach and scope of changes align with what you had in mind?