From 80afeaeed2b4bf398076e64b3025b39630250a50 Mon Sep 17 00:00:00 2001 From: Dan Wolfson Date: Mon, 5 Oct 2026 13:00:31 -0500 Subject: [PATCH 1/2] feat(overview): recent activity from curated content types Replaces the "Not measured" Recent Activity panels with live data: the metadata change stream (last 48 h), activity by type (7 d) and most active stewards, read per curated content type (glossary terms, collections and products, governance definitions, projects) from each type's most-recently-updated elements. A plain "latest changes" scan is useless here: the 10,000 most recent updates span 20 minutes and 98% are one connector refreshing APIParameter elements. The panels name the types they cover, count people and service accounts separately, return "Not measured" if any query fails, and label a page-limited read as a lower bound. Signed-off-by: Dan Wolfson --- .../PyegeriaWebHandler/egeria-overview.html | 60 +++++++-- .../PyegeriaWebHandler/overview_handler.py | 110 ++++++++++++++++ .../PyegeriaWebHandler/egeria-overview.html | 60 +++++++-- .../PyegeriaWebHandler/overview_handler.py | 110 ++++++++++++++++ .../tests/test_recent_activity.py | 117 ++++++++++++++++++ 5 files changed, 439 insertions(+), 18 deletions(-) create mode 100644 compose-configs/egeria-quickstart/PyegeriaWebHandler/tests/test_recent_activity.py diff --git a/compose-configs/egeria-freshstart/PyegeriaWebHandler/egeria-overview.html b/compose-configs/egeria-freshstart/PyegeriaWebHandler/egeria-overview.html index 2aeba632..21708705 100644 --- a/compose-configs/egeria-freshstart/PyegeriaWebHandler/egeria-overview.html +++ b/compose-configs/egeria-freshstart/PyegeriaWebHandler/egeria-overview.html @@ -690,19 +690,19 @@

❓ Questions & Answers Activity

Metadata Change Stream

-
Recent context events & catalog changes (last 48h)
-
-
Not measured: there is no metadata change feed yet
+
Recent changes to curated content
+
+
No data yet

Activity by Type (7d)

-
What's driving change
+
What's driving change
Most active stewards
-
Not measured: steward activity is not aggregated yet
+
No data yet
@@ -1372,15 +1372,15 @@

—

{ name: 'Most-Engaged Assets', status: 'illustrative' }, ], activity: [ - { name: 'Metadata Change Stream', status: 'illustrative' }, // no /api/overview/activity endpoint exists - { name: 'Activity by Type', status: 'illustrative' }, - { name: 'Most Active Stewards', status: 'illustrative' }, + { name: 'Metadata Change Stream', status: 'live' }, // /api/overview/activity: most recent changes to curated content types + { name: 'Activity by Type', status: 'live' }, + { name: 'Most Active Stewards', status: 'live' }, // people only (service accounts counted separately as automation) ], }; // Which backend source feeds each section's "live" items. A section whose source failed to load // shows "no data" -- the constants above only say a live computation EXISTS, not that it worked. const SECTION_SOURCE = { value:'summary', growth:'growth', composition:'summary', usage:'usage-context', - quality:'summary', ai:'ai-context', people:'people' }; + quality:'summary', ai:'ai-context', people:'people', activity:'activity' }; const FAILED_SOURCES = new Set(); function computeSectionBadge(sectionKey){ const items = SECTION_PROVENANCE[sectionKey]; @@ -1723,6 +1723,47 @@

—

if (d.technologyUnrecorded) rows.push(['No technology recorded', d.technologyUnrecorded, '#33415c']); hbars('byTech', rows, null, 'No assets catalogued'); } + // Recent activity: changes to curated content types (the backend names them). null = could not be read. + function escHtml(s){ return String(s).replace(/&/g, '&').replace(//g, '>').replace(/"/g, '"'); } + function ageText(iso){ + var t = new Date(iso + 'Z'); // the backend sends naive UTC + if (isNaN(t.getTime())) return ''; + var mins = Math.max(0, Math.round((Date.now() - t.getTime()) / 60000)); + return mins < 60 ? mins + ' min ago' : mins < 2880 ? Math.round(mins / 60) + ' h ago' : Math.round(mins / 1440) + ' days ago'; + } + function renderActivity(a){ + var stream = document.getElementById('activityStream'), sub = document.getElementById('act-stream-sub'), + tsub = document.getElementById('act-type-sub'), st = document.getElementById('activityStewards'); + if (!a) { + var msg = 'Not measured: the change query did not return'; + if (stream) stream.innerHTML = '
' + msg + '
'; + if (typeof hbars === 'function') hbars('activity', [], null, msg); + if (st) st.textContent = msg; + return; + } + var types = (a.types || []).join(', '), atLeast = a.capped ? 'at least ' : ''; + if (sub) sub.textContent = 'Changes to ' + types + ' in the last ' + a.recentHours + ' h'; + if (stream) stream.innerHTML = a.recent && a.recent.length ? a.recent.map(function(c){ + return '
' + + '
' + escHtml(c.name) + ' \u00b7 ' + escHtml(kindLabel(c.type)) + '
' + + '
' + escHtml(c.action) + ' by ' + escHtml(c.actor || 'unknown') + (c.automated ? ' (automation)' : '') + + ' \u00b7 ' + escHtml(ageText(c.time)) + '
'; + }).join('') : '
No changes to ' + escHtml(types) + ' in the last ' + a.recentHours + ' hours
'; + var rows = Object.keys(a.byType || {}).map(function(k){ return [kindLabel(k), a.byType[k], 'var(--c3)']; }) + .sort(function(x, y){ return y[1] - x[1]; }); + if (tsub) tsub.textContent = atLeast + a.total.toLocaleString() + ' changes in the last ' + a.windowDays + ' days to ' + types + + ' (' + a.people.toLocaleString() + ' by people, ' + a.automated.toLocaleString() + ' by automation)' + + (a.capped ? ' \u00b7 a page limit was reached, so counts are a lower bound' : ''); + if (typeof hbars === 'function') hbars('activity', rows, null, 'No changes to ' + types + ' in the last ' + a.windowDays + ' days'); + if (st) st.innerHTML = a.stewards && a.stewards.length ? a.stewards.map(function(s){ + return '
' + escHtml(s[0]) + '' + s[1].toLocaleString() + '
'; + }).join('') : 'No changes by people in the last ' + a.windowDays + ' days'; + } + function loadActivity(){ + return getJSON('/api/overview/activity', null) + .then(function(d){ if (!d || d.activity == null) throw 0; FAILED_SOURCES.delete('activity'); renderSectionBadges(); renderActivity(d.activity); }) + .catch(function(){ FAILED_SOURCES.add('activity'); renderSectionBadges(); renderActivity(null); }); + } function renderConfidentiality(d){ if (typeof hbars !== 'function') return; if (d.confidentialityLevels == null) { hbars('byConf', [], null, 'Not measured: the confidentiality query did not return'); return; } @@ -2409,6 +2450,7 @@

—

loadTileInfo(); var rs = document.getElementById('range-sel'); loadGrowth(rs ? rs.value : '6mo'); + loadActivity(); stamp(); } function boot(){ diff --git a/compose-configs/egeria-freshstart/PyegeriaWebHandler/overview_handler.py b/compose-configs/egeria-freshstart/PyegeriaWebHandler/overview_handler.py index dcac5199..cc2445bb 100644 --- a/compose-configs/egeria-freshstart/PyegeriaWebHandler/overview_handler.py +++ b/compose-configs/egeria-freshstart/PyegeriaWebHandler/overview_handler.py @@ -651,6 +651,116 @@ def _schema_coverage(mgr, as_of: Optional[str] = None, analysed=None) -> dict: return out +# ── Recent activity (changes to curated content) ──────────────────────────── +# Egeria's most-recently-updated elements are dominated by connector churn here (one catalog refresh +# rewrote ~10,000 elements in 20 minutes), so a plain "latest changes" scan says nothing about what +# people curate. Activity is therefore read per CURATED content type, and the payload names the types. +_ACTIVITY_TYPES = ["GlossaryTerm", "Collection", "GovernanceDefinition", "Project"] +_ACTIVITY_WINDOW_D = 7 +_ACTIVITY_RECENT_H = 48 +_ACTIVITY_PAGE = 200 +_ACTIVITY_MAX_PAGES = 5 # per type: 1000 most recent; a type still inside the window after that is a lower bound +_ACTIVITY_STREAM = 12 +_ACTIVITY_STEWARDS = 5 + + +def _looks_automated(user) -> bool: + """Heuristic: Egeria's own service accounts end in "npa" (non-person account) or are engines.""" + u = str(user or "").lower() + return u.endswith("npa") or "engine" in u + + +def _element_label(e: dict) -> str: + props = e.get("elementProperties") or {} + props = props.get("propertiesAsStrings") or props + return str(props.get("displayName") or props.get("name") or props.get("qualifiedName") or "(unnamed)") + + +def _recent_changes(mgr, now=None) -> Optional[dict]: + """Changes to curated content in the last _ACTIVITY_WINDOW_D days, from each type's most-recently-updated + elements (read in pages until one is older than the window). None if any query failed (a half-read + list must not pass for the whole). `capped` is True when some type was still inside the window after + _ACTIVITY_MAX_PAGES pages, which makes the counts a lower bound.""" + import datetime as _dt + now = now or _dt.datetime.utcnow() + cutoff = now - _dt.timedelta(days=_ACTIVITY_WINDOW_D) + recent_cut = now - _dt.timedelta(hours=_ACTIVITY_RECENT_H) + changes: dict = {} + capped = False + try: + for tname in _ACTIVITY_TYPES: + reached = False + for page in range(_ACTIVITY_MAX_PAGES): + chunk = mgr.find_metadata_elements({ + "class": "FindRequestBody", "metadataElementTypeName": tname, + "limitResultsByStatus": ["ACTIVE"], "graphQueryDepth": 0, + "sequencingOrder": "LAST_UPDATE_RECENT", + "startFrom": page * _ACTIVITY_PAGE, "pageSize": _ACTIVITY_PAGE}) + chunk = [e for e in (chunk if isinstance(chunk, list) else []) if isinstance(e, dict)] + if not chunk: + reached = True + break + for e in chunk: + v = e.get("versions") or {} + when = _parse_time(v.get("updateTime")) + if when is None: + continue + if when < cutoff: + reached = True + break + guid = _guid_of(e) + if guid and guid not in changes: + changes[guid] = {"time": when, "actor": v.get("updatedBy") or "", "version": v.get("version"), + "type": (e.get("type") or {}).get("typeName") or tname, + "name": _element_label(e), "guid": guid} + if reached: + break + if not reached: + capped = True + except Exception as exc: # noqa: BLE001 + logger.debug(f"overview recent activity failed: {exc}") + return None + rows = sorted(changes.values(), key=lambda c: c["time"], reverse=True) + by_type: dict = {} + stewards: dict = {} + automated = 0 + for c in rows: + by_type[c["type"]] = by_type.get(c["type"], 0) + 1 + if _looks_automated(c["actor"]): + automated += 1 + elif c["actor"]: + stewards[c["actor"]] = stewards.get(c["actor"], 0) + 1 + recent = [{"time": c["time"].isoformat(), "actor": c["actor"], "type": c["type"], "name": c["name"], + "guid": c["guid"], "action": "created" if c["version"] == 1 else "updated", + "automated": _looks_automated(c["actor"])} + for c in rows if c["time"] >= recent_cut][:_ACTIVITY_STREAM] + return {"types": list(_ACTIVITY_TYPES), "windowDays": _ACTIVITY_WINDOW_D, "recentHours": _ACTIVITY_RECENT_H, + "total": len(rows), "automated": automated, "people": len(rows) - automated, "capped": capped, + "byType": by_type, + "stewards": sorted(stewards.items(), key=lambda kv: (-kv[1], kv[0]))[:_ACTIVITY_STEWARDS], + "recent": recent} + + +@router.get("/api/overview/activity", summary="Recent changes to curated content") +def get_activity( + url: Optional[str] = Query(None), server: Optional[str] = Query(None), + user_id: Optional[str] = Query(None), user_pwd: Optional[str] = Query(None), +): + """Change stream, activity by type and most active stewards for curated content types (glossary terms, + collections/products, governance definitions, projects). Always "now": Egeria's asOfTime cannot + say who changed what. `activity` is None when a query failed -- the page then shows "Not measured".""" + ckey = f"activity|{url}|{server}|{user_id}" + cached = _cache_get(ckey) + if cached is not None: + return JSONResponse(cached) + activity = None + try: + activity = _recent_changes(_expert(url, server, user_id, user_pwd)) + except Exception as exc: # noqa: BLE001 + logger.debug(f"overview activity: client build failed: {exc}") + return JSONResponse(_cache_put(ckey, {"activity": activity, "source": "live:activity"})) + + _SURVEY_LINK_CAP = 1000 # one relationship page / one find page; a result at the cap is a lower bound, reported as such diff --git a/compose-configs/egeria-quickstart/PyegeriaWebHandler/egeria-overview.html b/compose-configs/egeria-quickstart/PyegeriaWebHandler/egeria-overview.html index 8aca3503..e6df9b85 100644 --- a/compose-configs/egeria-quickstart/PyegeriaWebHandler/egeria-overview.html +++ b/compose-configs/egeria-quickstart/PyegeriaWebHandler/egeria-overview.html @@ -690,19 +690,19 @@

❓ Questions & Answers Activity

Metadata Change Stream

-
Recent context events & catalog changes (last 48h)
-
-
Not measured: there is no metadata change feed yet
+
Recent changes to curated content
+
+
No data yet

Activity by Type (7d)

-
What's driving change
+
What's driving change
Most active stewards
-
Not measured: steward activity is not aggregated yet
+
No data yet
@@ -1369,15 +1369,15 @@

—

{ name: 'Most-Engaged Assets', status: 'illustrative' }, ], activity: [ - { name: 'Metadata Change Stream', status: 'illustrative' }, // no /api/overview/activity endpoint exists - { name: 'Activity by Type', status: 'illustrative' }, - { name: 'Most Active Stewards', status: 'illustrative' }, + { name: 'Metadata Change Stream', status: 'live' }, // /api/overview/activity: most recent changes to curated content types + { name: 'Activity by Type', status: 'live' }, + { name: 'Most Active Stewards', status: 'live' }, // people only (service accounts counted separately as automation) ], }; // Which backend source feeds each section's "live" items. A section whose source failed to load // shows "no data" -- the constants above only say a live computation EXISTS, not that it worked. const SECTION_SOURCE = { value:'summary', growth:'growth', composition:'summary', usage:'usage-context', - quality:'summary', ai:'ai-context', people:'people' }; + quality:'summary', ai:'ai-context', people:'people', activity:'activity' }; const FAILED_SOURCES = new Set(); function computeSectionBadge(sectionKey){ const items = SECTION_PROVENANCE[sectionKey]; @@ -1711,6 +1711,47 @@

—

if (d.technologyUnrecorded) rows.push(['No technology recorded', d.technologyUnrecorded, '#33415c']); hbars('byTech', rows, null, 'No assets catalogued'); } + // Recent activity: changes to curated content types (the backend names them). null = could not be read. + function escHtml(s){ return String(s).replace(/&/g, '&').replace(//g, '>').replace(/"/g, '"'); } + function ageText(iso){ + var t = new Date(iso + 'Z'); // the backend sends naive UTC + if (isNaN(t.getTime())) return ''; + var mins = Math.max(0, Math.round((Date.now() - t.getTime()) / 60000)); + return mins < 60 ? mins + ' min ago' : mins < 2880 ? Math.round(mins / 60) + ' h ago' : Math.round(mins / 1440) + ' days ago'; + } + function renderActivity(a){ + var stream = document.getElementById('activityStream'), sub = document.getElementById('act-stream-sub'), + tsub = document.getElementById('act-type-sub'), st = document.getElementById('activityStewards'); + if (!a) { + var msg = 'Not measured: the change query did not return'; + if (stream) stream.innerHTML = '
' + msg + '
'; + if (typeof hbars === 'function') hbars('activity', [], null, msg); + if (st) st.textContent = msg; + return; + } + var types = (a.types || []).join(', '), atLeast = a.capped ? 'at least ' : ''; + if (sub) sub.textContent = 'Changes to ' + types + ' in the last ' + a.recentHours + ' h'; + if (stream) stream.innerHTML = a.recent && a.recent.length ? a.recent.map(function(c){ + return '
' + + '
' + escHtml(c.name) + ' \u00b7 ' + escHtml(kindLabel(c.type)) + '
' + + '
' + escHtml(c.action) + ' by ' + escHtml(c.actor || 'unknown') + (c.automated ? ' (automation)' : '') + + ' \u00b7 ' + escHtml(ageText(c.time)) + '
'; + }).join('') : '
No changes to ' + escHtml(types) + ' in the last ' + a.recentHours + ' hours
'; + var rows = Object.keys(a.byType || {}).map(function(k){ return [kindLabel(k), a.byType[k], 'var(--c3)']; }) + .sort(function(x, y){ return y[1] - x[1]; }); + if (tsub) tsub.textContent = atLeast + a.total.toLocaleString() + ' changes in the last ' + a.windowDays + ' days to ' + types + + ' (' + a.people.toLocaleString() + ' by people, ' + a.automated.toLocaleString() + ' by automation)' + + (a.capped ? ' \u00b7 a page limit was reached, so counts are a lower bound' : ''); + if (typeof hbars === 'function') hbars('activity', rows, null, 'No changes to ' + types + ' in the last ' + a.windowDays + ' days'); + if (st) st.innerHTML = a.stewards && a.stewards.length ? a.stewards.map(function(s){ + return '
' + escHtml(s[0]) + '' + s[1].toLocaleString() + '
'; + }).join('') : 'No changes by people in the last ' + a.windowDays + ' days'; + } + function loadActivity(){ + return getJSON('/api/overview/activity', null) + .then(function(d){ if (!d || d.activity == null) throw 0; FAILED_SOURCES.delete('activity'); renderSectionBadges(); renderActivity(d.activity); }) + .catch(function(){ FAILED_SOURCES.add('activity'); renderSectionBadges(); renderActivity(null); }); + } function renderConfidentiality(d){ if (typeof hbars !== 'function') return; if (d.confidentialityLevels == null) { hbars('byConf', [], null, 'Not measured: the confidentiality query did not return'); return; } @@ -2388,6 +2429,7 @@

—

loadTileInfo(); var rs = document.getElementById('range-sel'); loadGrowth(rs ? rs.value : '6mo'); + loadActivity(); stamp(); } function boot(){ diff --git a/compose-configs/egeria-quickstart/PyegeriaWebHandler/overview_handler.py b/compose-configs/egeria-quickstart/PyegeriaWebHandler/overview_handler.py index 55cfe871..198e73ee 100644 --- a/compose-configs/egeria-quickstart/PyegeriaWebHandler/overview_handler.py +++ b/compose-configs/egeria-quickstart/PyegeriaWebHandler/overview_handler.py @@ -661,6 +661,116 @@ def _schema_coverage(mgr, as_of: Optional[str] = None, analysed=None) -> dict: return out +# ── Recent activity (changes to curated content) ──────────────────────────── +# Egeria's most-recently-updated elements are dominated by connector churn here (one catalog refresh +# rewrote ~10,000 elements in 20 minutes), so a plain "latest changes" scan says nothing about what +# people curate. Activity is therefore read per CURATED content type, and the payload names the types. +_ACTIVITY_TYPES = ["GlossaryTerm", "Collection", "GovernanceDefinition", "Project"] +_ACTIVITY_WINDOW_D = 7 +_ACTIVITY_RECENT_H = 48 +_ACTIVITY_PAGE = 200 +_ACTIVITY_MAX_PAGES = 5 # per type: 1000 most recent; a type still inside the window after that is a lower bound +_ACTIVITY_STREAM = 12 +_ACTIVITY_STEWARDS = 5 + + +def _looks_automated(user) -> bool: + """Heuristic: Egeria's own service accounts end in "npa" (non-person account) or are engines.""" + u = str(user or "").lower() + return u.endswith("npa") or "engine" in u + + +def _element_label(e: dict) -> str: + props = e.get("elementProperties") or {} + props = props.get("propertiesAsStrings") or props + return str(props.get("displayName") or props.get("name") or props.get("qualifiedName") or "(unnamed)") + + +def _recent_changes(mgr, now=None) -> Optional[dict]: + """Changes to curated content in the last _ACTIVITY_WINDOW_D days, from each type's most-recently-updated + elements (read in pages until one is older than the window). None if any query failed (a half-read + list must not pass for the whole). `capped` is True when some type was still inside the window after + _ACTIVITY_MAX_PAGES pages, which makes the counts a lower bound.""" + import datetime as _dt + now = now or _dt.datetime.utcnow() + cutoff = now - _dt.timedelta(days=_ACTIVITY_WINDOW_D) + recent_cut = now - _dt.timedelta(hours=_ACTIVITY_RECENT_H) + changes: dict = {} + capped = False + try: + for tname in _ACTIVITY_TYPES: + reached = False + for page in range(_ACTIVITY_MAX_PAGES): + chunk = mgr.find_metadata_elements({ + "class": "FindRequestBody", "metadataElementTypeName": tname, + "limitResultsByStatus": ["ACTIVE"], "graphQueryDepth": 0, + "sequencingOrder": "LAST_UPDATE_RECENT", + "startFrom": page * _ACTIVITY_PAGE, "pageSize": _ACTIVITY_PAGE}) + chunk = [e for e in (chunk if isinstance(chunk, list) else []) if isinstance(e, dict)] + if not chunk: + reached = True + break + for e in chunk: + v = e.get("versions") or {} + when = _parse_time(v.get("updateTime")) + if when is None: + continue + if when < cutoff: + reached = True + break + guid = _guid_of(e) + if guid and guid not in changes: + changes[guid] = {"time": when, "actor": v.get("updatedBy") or "", "version": v.get("version"), + "type": (e.get("type") or {}).get("typeName") or tname, + "name": _element_label(e), "guid": guid} + if reached: + break + if not reached: + capped = True + except Exception as exc: # noqa: BLE001 + logger.debug(f"overview recent activity failed: {exc}") + return None + rows = sorted(changes.values(), key=lambda c: c["time"], reverse=True) + by_type: dict = {} + stewards: dict = {} + automated = 0 + for c in rows: + by_type[c["type"]] = by_type.get(c["type"], 0) + 1 + if _looks_automated(c["actor"]): + automated += 1 + elif c["actor"]: + stewards[c["actor"]] = stewards.get(c["actor"], 0) + 1 + recent = [{"time": c["time"].isoformat(), "actor": c["actor"], "type": c["type"], "name": c["name"], + "guid": c["guid"], "action": "created" if c["version"] == 1 else "updated", + "automated": _looks_automated(c["actor"])} + for c in rows if c["time"] >= recent_cut][:_ACTIVITY_STREAM] + return {"types": list(_ACTIVITY_TYPES), "windowDays": _ACTIVITY_WINDOW_D, "recentHours": _ACTIVITY_RECENT_H, + "total": len(rows), "automated": automated, "people": len(rows) - automated, "capped": capped, + "byType": by_type, + "stewards": sorted(stewards.items(), key=lambda kv: (-kv[1], kv[0]))[:_ACTIVITY_STEWARDS], + "recent": recent} + + +@router.get("/api/overview/activity", summary="Recent changes to curated content") +def get_activity( + url: Optional[str] = Query(None), server: Optional[str] = Query(None), + user_id: Optional[str] = Query(None), user_pwd: Optional[str] = Query(None), +): + """Change stream, activity by type and most active stewards for curated content types (glossary terms, + collections/products, governance definitions, projects). Always "now": Egeria's asOfTime cannot + say who changed what. `activity` is None when a query failed -- the page then shows "Not measured".""" + ckey = f"activity|{url}|{server}|{user_id}" + cached = _cache_get(ckey) + if cached is not None: + return JSONResponse(cached) + activity = None + try: + activity = _recent_changes(_expert(url, server, user_id, user_pwd)) + except Exception as exc: # noqa: BLE001 + logger.debug(f"overview activity: client build failed: {exc}") + return JSONResponse(_cache_put(ckey, {"activity": activity, "source": "live:activity"})) + + _SURVEY_LINK_CAP = 1000 # one relationship page / one find page; a result at the cap is a lower bound, reported as such diff --git a/compose-configs/egeria-quickstart/PyegeriaWebHandler/tests/test_recent_activity.py b/compose-configs/egeria-quickstart/PyegeriaWebHandler/tests/test_recent_activity.py new file mode 100644 index 00000000..dbe0bde1 --- /dev/null +++ b/compose-configs/egeria-quickstart/PyegeriaWebHandler/tests/test_recent_activity.py @@ -0,0 +1,117 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright Contributors to the ODPi Egeria project. +"""_recent_changes() reads changes to curated content types, and must stay honest about its window. + +Egeria's most-recently-updated elements are dominated by connector churn, so activity is read per curated +type. The function pages each type (newest first) until an element is older than the window, splits people +from automation, and returns None -- never a half-read list -- if any query failed. A type still inside the +window after the page limit makes the counts a lower bound (`capped`). + +The functions are extracted from each handler file (quickstart and freshstart carry identical copies) and +run against a fake manager, without importing pyegeria or FastAPI. +""" +import ast +import datetime as dt +from pathlib import Path +from typing import Optional + +import pytest + +_HERE = Path(__file__).resolve().parent +_HANDLERS = { + "quickstart": _HERE.parent / "overview_handler.py", + "freshstart": _HERE.parents[2] / "egeria-freshstart" / "PyegeriaWebHandler" / "overview_handler.py", +} +_NOW = dt.datetime(2026, 10, 5, 12, 0, 0) +_NEEDED = {"_parse_time", "_guid_of", "_looks_automated", "_element_label", "_recent_changes"} + + +def _load(path: Path): + if not path.exists(): + pytest.skip(f"{path} not present in this checkout") + tree = ast.parse(path.read_text(encoding="utf-8")) + body = [n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name in _NEEDED] + assert {n.name for n in body} == _NEEDED + consts = [n for n in tree.body if isinstance(n, ast.Assign) and n.targets[0].id.startswith("_ACTIVITY_")] + ns = {"Optional": Optional, "logger": type("L", (), {"debug": staticmethod(lambda *a, **k: None)})()} + exec(compile(ast.Module(body=consts + body, type_ignores=[]), str(path), "exec"), ns) + return ns + + +@pytest.fixture(params=sorted(_HANDLERS)) +def ns(request): + return _load(_HANDLERS[request.param]) + + +def _el(guid, typ, hours_ago, actor="steward1", version=2, name="n"): + t = (_NOW - dt.timedelta(hours=hours_ago)).strftime("%Y-%m-%dT%H:%M:%S.000+00:00") + return {"elementGUID": guid, "type": {"typeName": typ}, + "versions": {"updateTime": t, "updatedBy": actor, "version": version}, + "elementProperties": {"propertiesAsStrings": {"displayName": name}}} + + +class _Mgr: + """Serves each type's elements newest-first, honouring startFrom/pageSize.""" + def __init__(self, by_type, fail_type=None): + self.by_type, self.fail_type, self.calls = by_type, fail_type, [] + + def find_metadata_elements(self, body): + t = body["metadataElementTypeName"] + self.calls.append((t, body["startFrom"], body.get("sequencingOrder"))) + if t == self.fail_type: + raise RuntimeError("boom") + lo = body["startFrom"] + return self.by_type.get(t, [])[lo:lo + body["pageSize"]] + + +def test_splits_people_from_automation_and_counts_by_type(ns): + m = _Mgr({"GlossaryTerm": [_el("a", "GlossaryTerm", 1, "erinoverview"), _el("b", "GlossaryTerm", 5, "apicatnpa"), + _el("c", "GlossaryTerm", 30, "erinoverview", version=1)], + "Collection": [_el("d", "DigitalProduct", 2, "peterprofile")]}) + out = ns["_recent_changes"](m, _NOW) + assert out["total"] == 4 and out["automated"] == 1 and out["people"] == 3 and out["capped"] is False + assert out["byType"] == {"GlossaryTerm": 3, "DigitalProduct": 1} + assert out["stewards"] == [("erinoverview", 2), ("peterprofile", 1)] + assert all(c[2] == "LAST_UPDATE_RECENT" for c in m.calls) + + +def test_the_stream_is_the_last_48_hours_newest_first_with_created_vs_updated(ns): + m = _Mgr({"GlossaryTerm": [_el("a", "GlossaryTerm", 1, version=1), _el("b", "GlossaryTerm", 47), + _el("c", "GlossaryTerm", 49), _el("d", "GlossaryTerm", 100)]}) + out = ns["_recent_changes"](m, _NOW) + assert [r["guid"] for r in out["recent"]] == ["a", "b"] + assert [r["action"] for r in out["recent"]] == ["created", "updated"] + assert out["total"] == 4 # 7-day window still counts the older two + + +def test_nothing_older_than_the_window_is_counted_and_paging_stops_there(ns): + m = _Mgr({"GlossaryTerm": [_el("a", "GlossaryTerm", 1)] + [_el(f"o{i}", "GlossaryTerm", 24 * 8 + i) for i in range(500)]}) + out = ns["_recent_changes"](m, _NOW) + assert out["total"] == 1 and out["capped"] is False + assert [c for c in m.calls if c[0] == "GlossaryTerm"] == [("GlossaryTerm", 0, "LAST_UPDATE_RECENT")] # one page + + +def test_a_type_still_inside_the_window_after_the_page_limit_is_a_lower_bound(ns): + busy = [_el(f"g{i}", "GlossaryTerm", 1) for i in range(1200)] # 6 pages of 200, all inside the window + out = ns["_recent_changes"](_Mgr({"GlossaryTerm": busy}), _NOW) + assert out["capped"] is True and out["total"] == 1000 + + +def test_an_element_in_two_queried_types_is_counted_once(ns): + m = _Mgr({"GlossaryTerm": [_el("x", "GlossaryTerm", 1)], "Collection": [_el("x", "GlossaryTerm", 1)]}) + assert ns["_recent_changes"](m, _NOW)["total"] == 1 + + +def test_no_changes_is_a_real_zero(ns): + out = ns["_recent_changes"](_Mgr({}), _NOW) + assert out["total"] == 0 and out["people"] == 0 and out["recent"] == [] and out["stewards"] == [] and out["byType"] == {} + + +def test_a_failed_query_returns_none_not_a_partial_result(ns): + m = _Mgr({"GlossaryTerm": [_el("a", "GlossaryTerm", 1)]}, fail_type="Project") + assert ns["_recent_changes"](m, _NOW) is None + + +def test_automation_heuristic(ns): + f = ns["_looks_automated"] + assert f("apicatnpa") and f("EgeriaGovernanceEngine") and not f("erinoverview") and not f("") and not f(None) From b28543b01213f8962b7d7a1d054b69bd62f8fa00 Mon Sep 17 00:00:00 2001 From: Dan Wolfson Date: Mon, 5 Oct 2026 13:03:30 -0500 Subject: [PATCH 2/2] fix(overview): list only people's changes in the change stream Checked in the browser: the 12 newest curated-type changes were all the subscription manager rewriting notification types, hiding everything a person did. The stream now lists people's changes only and says how many automation changes it left out; activity by type and the 7-day totals still count both. Signed-off-by: Dan Wolfson --- .../PyegeriaWebHandler/egeria-overview.html | 5 +++-- .../PyegeriaWebHandler/overview_handler.py | 11 +++++++---- .../PyegeriaWebHandler/egeria-overview.html | 5 +++-- .../PyegeriaWebHandler/overview_handler.py | 11 +++++++---- .../PyegeriaWebHandler/tests/test_recent_activity.py | 8 ++++++++ 5 files changed, 28 insertions(+), 12 deletions(-) diff --git a/compose-configs/egeria-freshstart/PyegeriaWebHandler/egeria-overview.html b/compose-configs/egeria-freshstart/PyegeriaWebHandler/egeria-overview.html index 21708705..42e24812 100644 --- a/compose-configs/egeria-freshstart/PyegeriaWebHandler/egeria-overview.html +++ b/compose-configs/egeria-freshstart/PyegeriaWebHandler/egeria-overview.html @@ -1742,13 +1742,14 @@

—

return; } var types = (a.types || []).join(', '), atLeast = a.capped ? 'at least ' : ''; - if (sub) sub.textContent = 'Changes to ' + types + ' in the last ' + a.recentHours + ' h'; + var autoNote = a.recentAutomated ? ' (' + a.recentAutomated.toLocaleString() + ' more by automation not listed)' : ''; + if (sub) sub.textContent = 'Changes by people to ' + types + ' in the last ' + a.recentHours + ' h' + autoNote; if (stream) stream.innerHTML = a.recent && a.recent.length ? a.recent.map(function(c){ return '
' + '
' + escHtml(c.name) + ' \u00b7 ' + escHtml(kindLabel(c.type)) + '
' + '
' + escHtml(c.action) + ' by ' + escHtml(c.actor || 'unknown') + (c.automated ? ' (automation)' : '') + ' \u00b7 ' + escHtml(ageText(c.time)) + '
'; - }).join('') : '
No changes to ' + escHtml(types) + ' in the last ' + a.recentHours + ' hours
'; + }).join('') : '
No changes by people to ' + escHtml(types) + ' in the last ' + a.recentHours + ' hours' + escHtml(autoNote) + '
'; var rows = Object.keys(a.byType || {}).map(function(k){ return [kindLabel(k), a.byType[k], 'var(--c3)']; }) .sort(function(x, y){ return y[1] - x[1]; }); if (tsub) tsub.textContent = atLeast + a.total.toLocaleString() + ' changes in the last ' + a.windowDays + ' days to ' + types diff --git a/compose-configs/egeria-freshstart/PyegeriaWebHandler/overview_handler.py b/compose-configs/egeria-freshstart/PyegeriaWebHandler/overview_handler.py index cc2445bb..7eec6c27 100644 --- a/compose-configs/egeria-freshstart/PyegeriaWebHandler/overview_handler.py +++ b/compose-configs/egeria-freshstart/PyegeriaWebHandler/overview_handler.py @@ -730,15 +730,18 @@ def _recent_changes(mgr, now=None) -> Optional[dict]: automated += 1 elif c["actor"]: stewards[c["actor"]] = stewards.get(c["actor"], 0) + 1 + # The stream is people's changes: automation (a subscription manager rewriting notification types every + # few minutes) would otherwise fill it and hide everything a person did. It is counted, not listed. + in_recent = [c for c in rows if c["time"] >= recent_cut] recent = [{"time": c["time"].isoformat(), "actor": c["actor"], "type": c["type"], "name": c["name"], - "guid": c["guid"], "action": "created" if c["version"] == 1 else "updated", - "automated": _looks_automated(c["actor"])} - for c in rows if c["time"] >= recent_cut][:_ACTIVITY_STREAM] + "guid": c["guid"], "action": "created" if c["version"] == 1 else "updated", "automated": False} + for c in in_recent if not _looks_automated(c["actor"])][:_ACTIVITY_STREAM] + recent_automated = sum(1 for c in in_recent if _looks_automated(c["actor"])) return {"types": list(_ACTIVITY_TYPES), "windowDays": _ACTIVITY_WINDOW_D, "recentHours": _ACTIVITY_RECENT_H, "total": len(rows), "automated": automated, "people": len(rows) - automated, "capped": capped, "byType": by_type, "stewards": sorted(stewards.items(), key=lambda kv: (-kv[1], kv[0]))[:_ACTIVITY_STEWARDS], - "recent": recent} + "recent": recent, "recentAutomated": recent_automated} @router.get("/api/overview/activity", summary="Recent changes to curated content") diff --git a/compose-configs/egeria-quickstart/PyegeriaWebHandler/egeria-overview.html b/compose-configs/egeria-quickstart/PyegeriaWebHandler/egeria-overview.html index e6df9b85..dd020200 100644 --- a/compose-configs/egeria-quickstart/PyegeriaWebHandler/egeria-overview.html +++ b/compose-configs/egeria-quickstart/PyegeriaWebHandler/egeria-overview.html @@ -1730,13 +1730,14 @@

—

return; } var types = (a.types || []).join(', '), atLeast = a.capped ? 'at least ' : ''; - if (sub) sub.textContent = 'Changes to ' + types + ' in the last ' + a.recentHours + ' h'; + var autoNote = a.recentAutomated ? ' (' + a.recentAutomated.toLocaleString() + ' more by automation not listed)' : ''; + if (sub) sub.textContent = 'Changes by people to ' + types + ' in the last ' + a.recentHours + ' h' + autoNote; if (stream) stream.innerHTML = a.recent && a.recent.length ? a.recent.map(function(c){ return '
' + '
' + escHtml(c.name) + ' \u00b7 ' + escHtml(kindLabel(c.type)) + '
' + '
' + escHtml(c.action) + ' by ' + escHtml(c.actor || 'unknown') + (c.automated ? ' (automation)' : '') + ' \u00b7 ' + escHtml(ageText(c.time)) + '
'; - }).join('') : '
No changes to ' + escHtml(types) + ' in the last ' + a.recentHours + ' hours
'; + }).join('') : '
No changes by people to ' + escHtml(types) + ' in the last ' + a.recentHours + ' hours' + escHtml(autoNote) + '
'; var rows = Object.keys(a.byType || {}).map(function(k){ return [kindLabel(k), a.byType[k], 'var(--c3)']; }) .sort(function(x, y){ return y[1] - x[1]; }); if (tsub) tsub.textContent = atLeast + a.total.toLocaleString() + ' changes in the last ' + a.windowDays + ' days to ' + types diff --git a/compose-configs/egeria-quickstart/PyegeriaWebHandler/overview_handler.py b/compose-configs/egeria-quickstart/PyegeriaWebHandler/overview_handler.py index 198e73ee..b4f9af09 100644 --- a/compose-configs/egeria-quickstart/PyegeriaWebHandler/overview_handler.py +++ b/compose-configs/egeria-quickstart/PyegeriaWebHandler/overview_handler.py @@ -740,15 +740,18 @@ def _recent_changes(mgr, now=None) -> Optional[dict]: automated += 1 elif c["actor"]: stewards[c["actor"]] = stewards.get(c["actor"], 0) + 1 + # The stream is people's changes: automation (a subscription manager rewriting notification types every + # few minutes) would otherwise fill it and hide everything a person did. It is counted, not listed. + in_recent = [c for c in rows if c["time"] >= recent_cut] recent = [{"time": c["time"].isoformat(), "actor": c["actor"], "type": c["type"], "name": c["name"], - "guid": c["guid"], "action": "created" if c["version"] == 1 else "updated", - "automated": _looks_automated(c["actor"])} - for c in rows if c["time"] >= recent_cut][:_ACTIVITY_STREAM] + "guid": c["guid"], "action": "created" if c["version"] == 1 else "updated", "automated": False} + for c in in_recent if not _looks_automated(c["actor"])][:_ACTIVITY_STREAM] + recent_automated = sum(1 for c in in_recent if _looks_automated(c["actor"])) return {"types": list(_ACTIVITY_TYPES), "windowDays": _ACTIVITY_WINDOW_D, "recentHours": _ACTIVITY_RECENT_H, "total": len(rows), "automated": automated, "people": len(rows) - automated, "capped": capped, "byType": by_type, "stewards": sorted(stewards.items(), key=lambda kv: (-kv[1], kv[0]))[:_ACTIVITY_STEWARDS], - "recent": recent} + "recent": recent, "recentAutomated": recent_automated} @router.get("/api/overview/activity", summary="Recent changes to curated content") diff --git a/compose-configs/egeria-quickstart/PyegeriaWebHandler/tests/test_recent_activity.py b/compose-configs/egeria-quickstart/PyegeriaWebHandler/tests/test_recent_activity.py index dbe0bde1..38975859 100644 --- a/compose-configs/egeria-quickstart/PyegeriaWebHandler/tests/test_recent_activity.py +++ b/compose-configs/egeria-quickstart/PyegeriaWebHandler/tests/test_recent_activity.py @@ -84,6 +84,14 @@ def test_the_stream_is_the_last_48_hours_newest_first_with_created_vs_updated(ns assert out["total"] == 4 # 7-day window still counts the older two +def test_the_stream_lists_people_only_and_counts_automation_separately(ns): + m = _Mgr({"GlossaryTerm": [_el("a", "GlossaryTerm", 1, "baudotnpa"), _el("b", "GlossaryTerm", 2, "baudotnpa"), + _el("c", "GlossaryTerm", 3, "erinoverview"), _el("d", "GlossaryTerm", 60, "baudotnpa")]}) + out = ns["_recent_changes"](m, _NOW) + assert [r["guid"] for r in out["recent"]] == ["c"] and out["recentAutomated"] == 2 # "d" is older than 48 h + assert all(r["automated"] is False for r in out["recent"]) + + def test_nothing_older_than_the_window_is_counted_and_paging_stops_there(ns): m = _Mgr({"GlossaryTerm": [_el("a", "GlossaryTerm", 1)] + [_el(f"o{i}", "GlossaryTerm", 24 * 8 + i) for i in range(500)]}) out = ns["_recent_changes"](m, _NOW)