feat(overview): recent activity from curated content types

compose-configs/egeria-freshstart/PyegeriaWebHandler/egeria-overview.html

@@ -690,19 +690,19 @@ <h3>❓ Questions &amp; Answers Activity</h3>
   <div class="grid g-12">
     <div class="panel">
       <h3>Metadata Change Stream</h3>
-      <div class="ph-sub">Recent context events &amp; catalog changes (last 48h)</div>
-      <div style="margin-top:6px">
-        <div style="color:var(--muted);font-size:.8rem;padding:6px 0">Not measured: there is no metadata change feed yet</div>
+      <div class="ph-sub" id="act-stream-sub">Recent changes to curated content</div>
+      <div style="margin-top:6px" id="activityStream">
+        <div style="color:var(--muted);font-size:.8rem;padding:6px 0">No data yet</div>
       </div>
     </div>
 
     <div class="panel">
       <h3>Activity by Type (7d)</h3>
-      <div class="ph-sub">What's driving change</div>
+      <div class="ph-sub" id="act-type-sub">What's driving change</div>
       <div id="activity"></div>
       <div style="margin-top:14px; padding-top:12px; border-top:1px solid var(--border)">
         <div style="font-size:.74rem;color:var(--muted);margin-bottom:8px">Most active stewards</div>
-        <div style="color:var(--muted);font-size:.8rem">Not measured: steward activity is not aggregated yet</div>
+        <div style="color:var(--muted);font-size:.8rem" id="activityStewards">No data yet</div>
 </div>
     </div>
   </div>
@@ -1372,15 +1372,15 @@ <h2 id="d-title">—</h2>
     { name: 'Most-Engaged Assets',                 status: 'illustrative' },
   ],
   activity: [
-    { name: 'Metadata Change Stream', status: 'illustrative' }, // no /api/overview/activity endpoint exists
-    { name: 'Activity by Type',       status: 'illustrative' },
-    { name: 'Most Active Stewards',   status: 'illustrative' },
+    { name: 'Metadata Change Stream', status: 'live' },  // /api/overview/activity: most recent changes to curated content types
+    { name: 'Activity by Type',       status: 'live' },
+    { name: 'Most Active Stewards',   status: 'live' },  // people only (service accounts counted separately as automation)
   ],
 };
 // Which backend source feeds each section's "live" items. A section whose source failed to load
 // shows "no data" -- the constants above only say a live computation EXISTS, not that it worked.
 const SECTION_SOURCE = { value:'summary', growth:'growth', composition:'summary', usage:'usage-context',
-                         quality:'summary', ai:'ai-context', people:'people' };
+                         quality:'summary', ai:'ai-context', people:'people', activity:'activity' };
 const FAILED_SOURCES = new Set();
 function computeSectionBadge(sectionKey){
   const items = SECTION_PROVENANCE[sectionKey];
@@ -1723,6 +1723,48 @@ <h2 id="d-title">—</h2>
     if (d.technologyUnrecorded) rows.push(['No technology recorded', d.technologyUnrecorded, '#33415c']);
     hbars('byTech', rows, null, 'No assets catalogued');
   }
+  // Recent activity: changes to curated content types (the backend names them). null = could not be read.
+  function escHtml(s){ return String(s).replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;').replace(/"/g, '&quot;'); }
+  function ageText(iso){
+    var t = new Date(iso + 'Z');                       // the backend sends naive UTC
+    if (isNaN(t.getTime())) return '';
+    var mins = Math.max(0, Math.round((Date.now() - t.getTime()) / 60000));
+    return mins < 60 ? mins + ' min ago' : mins < 2880 ? Math.round(mins / 60) + ' h ago' : Math.round(mins / 1440) + ' days ago';
+  }
+  function renderActivity(a){
+    var stream = document.getElementById('activityStream'), sub = document.getElementById('act-stream-sub'),
+        tsub = document.getElementById('act-type-sub'), st = document.getElementById('activityStewards');
+    if (!a) {
+      var msg = 'Not measured: the change query did not return';
+      if (stream) stream.innerHTML = '<div style="color:var(--muted);font-size:.8rem;padding:6px 0">' + msg + '</div>';
+      if (typeof hbars === 'function') hbars('activity', [], null, msg);
+      if (st) st.textContent = msg;
+      return;
+    }
+    var types = (a.types || []).join(', '), atLeast = a.capped ? 'at least ' : '';
+    var autoNote = a.recentAutomated ? ' (' + a.recentAutomated.toLocaleString() + ' more by automation not listed)' : '';
+    if (sub) sub.textContent = 'Changes by people to ' + types + ' in the last ' + a.recentHours + ' h' + autoNote;
+    if (stream) stream.innerHTML = a.recent && a.recent.length ? a.recent.map(function(c){
+      return '<div class="feed-item"><span class="feed-dot" style="background:' + (c.automated ? '#33415c' : 'var(--c3)') + '"></span>'
+        + '<div class="feed-body"><div class="fb-main"><b>' + escHtml(c.name) + '</b> \u00b7 ' + escHtml(kindLabel(c.type)) + '</div>'
+        + '<div class="fb-meta">' + escHtml(c.action) + ' by ' + escHtml(c.actor || 'unknown') + (c.automated ? ' (automation)' : '')
+        + ' \u00b7 ' + escHtml(ageText(c.time)) + '</div></div></div>';
+    }).join('') : '<div style="color:var(--muted);font-size:.8rem;padding:6px 0">No changes by people to ' + escHtml(types) + ' in the last ' + a.recentHours + ' hours' + escHtml(autoNote) + '</div>';
+    var rows = Object.keys(a.byType || {}).map(function(k){ return [kindLabel(k), a.byType[k], 'var(--c3)']; })
+                     .sort(function(x, y){ return y[1] - x[1]; });
+    if (tsub) tsub.textContent = atLeast + a.total.toLocaleString() + ' changes in the last ' + a.windowDays + ' days to ' + types
+      + ' (' + a.people.toLocaleString() + ' by people, ' + a.automated.toLocaleString() + ' by automation)'
+      + (a.capped ? ' \u00b7 a page limit was reached, so counts are a lower bound' : '');
+    if (typeof hbars === 'function') hbars('activity', rows, null, 'No changes to ' + types + ' in the last ' + a.windowDays + ' days');
+    if (st) st.innerHTML = a.stewards && a.stewards.length ? a.stewards.map(function(s){
+      return '<div style="display:flex;justify-content:space-between;padding:2px 0"><span>' + escHtml(s[0]) + '</span><b>' + s[1].toLocaleString() + '</b></div>';
+    }).join('') : 'No changes by people in the last ' + a.windowDays + ' days';
+  }
+  function loadActivity(){
+    return getJSON('/api/overview/activity', null)
+      .then(function(d){ if (!d || d.activity == null) throw 0; FAILED_SOURCES.delete('activity'); renderSectionBadges(); renderActivity(d.activity); })
+      .catch(function(){ FAILED_SOURCES.add('activity'); renderSectionBadges(); renderActivity(null); });
+  }
   function renderConfidentiality(d){
     if (typeof hbars !== 'function') return;
     if (d.confidentialityLevels == null) { hbars('byConf', [], null, 'Not measured: the confidentiality query did not return'); return; }
@@ -2409,6 +2451,7 @@ <h2 id="d-title">—</h2>
     loadTileInfo();
     var rs = document.getElementById('range-sel');
     loadGrowth(rs ? rs.value : '6mo');
+    loadActivity();
     stamp();
   }
   function boot(){

compose-configs/egeria-freshstart/PyegeriaWebHandler/overview_handler.py

@@ -651,6 +651,119 @@ def _schema_coverage(mgr, as_of: Optional[str] = None, analysed=None) -> dict:
     return out
 
 
+# ── Recent activity (changes to curated content) ────────────────────────────
+# Egeria's most-recently-updated elements are dominated by connector churn here (one catalog refresh
+# rewrote ~10,000 elements in 20 minutes), so a plain "latest changes" scan says nothing about what
+# people curate. Activity is therefore read per CURATED content type, and the payload names the types.
+_ACTIVITY_TYPES = ["GlossaryTerm", "Collection", "GovernanceDefinition", "Project"]
+_ACTIVITY_WINDOW_D = 7
+_ACTIVITY_RECENT_H = 48
+_ACTIVITY_PAGE = 200
+_ACTIVITY_MAX_PAGES = 5      # per type: 1000 most recent; a type still inside the window after that is a lower bound
+_ACTIVITY_STREAM = 12
+_ACTIVITY_STEWARDS = 5
+
+
+def _looks_automated(user) -> bool:
+    """Heuristic: Egeria's own service accounts end in "npa" (non-person account) or are engines."""
+    u = str(user or "").lower()
+    return u.endswith("npa") or "engine" in u
+
+
+def _element_label(e: dict) -> str:
+    props = e.get("elementProperties") or {}
+    props = props.get("propertiesAsStrings") or props
+    return str(props.get("displayName") or props.get("name") or props.get("qualifiedName") or "(unnamed)")
+
+
+def _recent_changes(mgr, now=None) -> Optional[dict]:
+    """Changes to curated content in the last _ACTIVITY_WINDOW_D days, from each type's most-recently-updated
+    elements (read in pages until one is older than the window). None if any query failed (a half-read
+    list must not pass for the whole). `capped` is True when some type was still inside the window after
+    _ACTIVITY_MAX_PAGES pages, which makes the counts a lower bound."""
+    import datetime as _dt
+    now = now or _dt.datetime.utcnow()
+    cutoff = now - _dt.timedelta(days=_ACTIVITY_WINDOW_D)
+    recent_cut = now - _dt.timedelta(hours=_ACTIVITY_RECENT_H)
+    changes: dict = {}
+    capped = False
+    try:
+        for tname in _ACTIVITY_TYPES:
+            reached = False
+            for page in range(_ACTIVITY_MAX_PAGES):
+                chunk = mgr.find_metadata_elements({
+                    "class": "FindRequestBody", "metadataElementTypeName": tname,
+                    "limitResultsByStatus": ["ACTIVE"], "graphQueryDepth": 0,
+                    "sequencingOrder": "LAST_UPDATE_RECENT",
+                    "startFrom": page * _ACTIVITY_PAGE, "pageSize": _ACTIVITY_PAGE})
+                chunk = [e for e in (chunk if isinstance(chunk, list) else []) if isinstance(e, dict)]
+                if not chunk:
+                    reached = True
+                    break
+                for e in chunk:
+                    v = e.get("versions") or {}
+                    when = _parse_time(v.get("updateTime"))
+                    if when is None:
+                        continue
+                    if when < cutoff:
+                        reached = True
+                        break
+                    guid = _guid_of(e)
+                    if guid and guid not in changes:
+                        changes[guid] = {"time": when, "actor": v.get("updatedBy") or "", "version": v.get("version"),
+                                         "type": (e.get("type") or {}).get("typeName") or tname,
+                                         "name": _element_label(e), "guid": guid}
+                if reached:
+                    break
+            if not reached:
+                capped = True
+    except Exception as exc:  # noqa: BLE001
+        logger.debug(f"overview recent activity failed: {exc}")
+        return None
+    rows = sorted(changes.values(), key=lambda c: c["time"], reverse=True)
+    by_type: dict = {}
+    stewards: dict = {}
+    automated = 0
+    for c in rows:
+        by_type[c["type"]] = by_type.get(c["type"], 0) + 1
+        if _looks_automated(c["actor"]):
+            automated += 1
+        elif c["actor"]:
+            stewards[c["actor"]] = stewards.get(c["actor"], 0) + 1
+    # The stream is people's changes: automation (a subscription manager rewriting notification types every
+    # few minutes) would otherwise fill it and hide everything a person did. It is counted, not listed.
+    in_recent = [c for c in rows if c["time"] >= recent_cut]
+    recent = [{"time": c["time"].isoformat(), "actor": c["actor"], "type": c["type"], "name": c["name"],
+               "guid": c["guid"], "action": "created" if c["version"] == 1 else "updated", "automated": False}
+              for c in in_recent if not _looks_automated(c["actor"])][:_ACTIVITY_STREAM]
+    recent_automated = sum(1 for c in in_recent if _looks_automated(c["actor"]))
+    return {"types": list(_ACTIVITY_TYPES), "windowDays": _ACTIVITY_WINDOW_D, "recentHours": _ACTIVITY_RECENT_H,
+            "total": len(rows), "automated": automated, "people": len(rows) - automated, "capped": capped,
+            "byType": by_type,
+            "stewards": sorted(stewards.items(), key=lambda kv: (-kv[1], kv[0]))[:_ACTIVITY_STEWARDS],
+            "recent": recent, "recentAutomated": recent_automated}
+
+
+@router.get("/api/overview/activity", summary="Recent changes to curated content")
+def get_activity(
+    url: Optional[str] = Query(None), server: Optional[str] = Query(None),
+    user_id: Optional[str] = Query(None), user_pwd: Optional[str] = Query(None),
+):
+    """Change stream, activity by type and most active stewards for curated content types (glossary terms,
+    collections/products, governance definitions, projects). Always "now": Egeria's asOfTime cannot
+    say who changed what. `activity` is None when a query failed -- the page then shows "Not measured"."""
+    ckey = f"activity|{url}|{server}|{user_id}"
+    cached = _cache_get(ckey)
+    if cached is not None:
+        return JSONResponse(cached)
+    activity = None
+    try:
+        activity = _recent_changes(_expert(url, server, user_id, user_pwd))
+    except Exception as exc:  # noqa: BLE001
+        logger.debug(f"overview activity: client build failed: {exc}")
+    return JSONResponse(_cache_put(ckey, {"activity": activity, "source": "live:activity"}))
+
+
 _SURVEY_LINK_CAP = 1000   # one relationship page / one find page; a result at the cap is a lower bound, reported as such
 
 

compose-configs/egeria-quickstart/PyegeriaWebHandler/egeria-overview.html

@@ -690,19 +690,19 @@ <h3>❓ Questions &amp; Answers Activity</h3>
   <div class="grid g-12">
     <div class="panel">
       <h3>Metadata Change Stream</h3>
-      <div class="ph-sub">Recent context events &amp; catalog changes (last 48h)</div>
-      <div style="margin-top:6px">
-        <div style="color:var(--muted);font-size:.8rem;padding:6px 0">Not measured: there is no metadata change feed yet</div>
+      <div class="ph-sub" id="act-stream-sub">Recent changes to curated content</div>
+      <div style="margin-top:6px" id="activityStream">
+        <div style="color:var(--muted);font-size:.8rem;padding:6px 0">No data yet</div>
       </div>
     </div>
 
     <div class="panel">
       <h3>Activity by Type (7d)</h3>
-      <div class="ph-sub">What's driving change</div>
+      <div class="ph-sub" id="act-type-sub">What's driving change</div>
       <div id="activity"></div>
       <div style="margin-top:14px; padding-top:12px; border-top:1px solid var(--border)">
         <div style="font-size:.74rem;color:var(--muted);margin-bottom:8px">Most active stewards</div>
-        <div style="color:var(--muted);font-size:.8rem">Not measured: steward activity is not aggregated yet</div>
+        <div style="color:var(--muted);font-size:.8rem" id="activityStewards">No data yet</div>
 </div>
     </div>
   </div>
@@ -1369,15 +1369,15 @@ <h2 id="d-title">—</h2>
     { name: 'Most-Engaged Assets',                 status: 'illustrative' },
   ],
   activity: [
-    { name: 'Metadata Change Stream', status: 'illustrative' }, // no /api/overview/activity endpoint exists
-    { name: 'Activity by Type',       status: 'illustrative' },
-    { name: 'Most Active Stewards',   status: 'illustrative' },
+    { name: 'Metadata Change Stream', status: 'live' },  // /api/overview/activity: most recent changes to curated content types
+    { name: 'Activity by Type',       status: 'live' },
+    { name: 'Most Active Stewards',   status: 'live' },  // people only (service accounts counted separately as automation)
   ],
 };
 // Which backend source feeds each section's "live" items. A section whose source failed to load
 // shows "no data" -- the constants above only say a live computation EXISTS, not that it worked.
 const SECTION_SOURCE = { value:'summary', growth:'growth', composition:'summary', usage:'usage-context',
-                         quality:'summary', ai:'ai-context', people:'people' };
+                         quality:'summary', ai:'ai-context', people:'people', activity:'activity' };
 const FAILED_SOURCES = new Set();
 function computeSectionBadge(sectionKey){
   const items = SECTION_PROVENANCE[sectionKey];
@@ -1711,6 +1711,48 @@ <h2 id="d-title">—</h2>
     if (d.technologyUnrecorded) rows.push(['No technology recorded', d.technologyUnrecorded, '#33415c']);
     hbars('byTech', rows, null, 'No assets catalogued');
   }
+  // Recent activity: changes to curated content types (the backend names them). null = could not be read.
+  function escHtml(s){ return String(s).replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;').replace(/"/g, '&quot;'); }
+  function ageText(iso){
+    var t = new Date(iso + 'Z');                       // the backend sends naive UTC
+    if (isNaN(t.getTime())) return '';
+    var mins = Math.max(0, Math.round((Date.now() - t.getTime()) / 60000));
+    return mins < 60 ? mins + ' min ago' : mins < 2880 ? Math.round(mins / 60) + ' h ago' : Math.round(mins / 1440) + ' days ago';
+  }
+  function renderActivity(a){
+    var stream = document.getElementById('activityStream'), sub = document.getElementById('act-stream-sub'),
+        tsub = document.getElementById('act-type-sub'), st = document.getElementById('activityStewards');
+    if (!a) {
+      var msg = 'Not measured: the change query did not return';
+      if (stream) stream.innerHTML = '<div style="color:var(--muted);font-size:.8rem;padding:6px 0">' + msg + '</div>';
+      if (typeof hbars === 'function') hbars('activity', [], null, msg);
+      if (st) st.textContent = msg;
+      return;
+    }
+    var types = (a.types || []).join(', '), atLeast = a.capped ? 'at least ' : '';
+    var autoNote = a.recentAutomated ? ' (' + a.recentAutomated.toLocaleString() + ' more by automation not listed)' : '';
+    if (sub) sub.textContent = 'Changes by people to ' + types + ' in the last ' + a.recentHours + ' h' + autoNote;
+    if (stream) stream.innerHTML = a.recent && a.recent.length ? a.recent.map(function(c){
+      return '<div class="feed-item"><span class="feed-dot" style="background:' + (c.automated ? '#33415c' : 'var(--c3)') + '"></span>'
+        + '<div class="feed-body"><div class="fb-main"><b>' + escHtml(c.name) + '</b> \u00b7 ' + escHtml(kindLabel(c.type)) + '</div>'
+        + '<div class="fb-meta">' + escHtml(c.action) + ' by ' + escHtml(c.actor || 'unknown') + (c.automated ? ' (automation)' : '')
+        + ' \u00b7 ' + escHtml(ageText(c.time)) + '</div></div></div>';
+    }).join('') : '<div style="color:var(--muted);font-size:.8rem;padding:6px 0">No changes by people to ' + escHtml(types) + ' in the last ' + a.recentHours + ' hours' + escHtml(autoNote) + '</div>';
+    var rows = Object.keys(a.byType || {}).map(function(k){ return [kindLabel(k), a.byType[k], 'var(--c3)']; })
+                     .sort(function(x, y){ return y[1] - x[1]; });
+    if (tsub) tsub.textContent = atLeast + a.total.toLocaleString() + ' changes in the last ' + a.windowDays + ' days to ' + types
+      + ' (' + a.people.toLocaleString() + ' by people, ' + a.automated.toLocaleString() + ' by automation)'
+      + (a.capped ? ' \u00b7 a page limit was reached, so counts are a lower bound' : '');
+    if (typeof hbars === 'function') hbars('activity', rows, null, 'No changes to ' + types + ' in the last ' + a.windowDays + ' days');
+    if (st) st.innerHTML = a.stewards && a.stewards.length ? a.stewards.map(function(s){
+      return '<div style="display:flex;justify-content:space-between;padding:2px 0"><span>' + escHtml(s[0]) + '</span><b>' + s[1].toLocaleString() + '</b></div>';
+    }).join('') : 'No changes by people in the last ' + a.windowDays + ' days';
+  }
+  function loadActivity(){
+    return getJSON('/api/overview/activity', null)
+      .then(function(d){ if (!d || d.activity == null) throw 0; FAILED_SOURCES.delete('activity'); renderSectionBadges(); renderActivity(d.activity); })
+      .catch(function(){ FAILED_SOURCES.add('activity'); renderSectionBadges(); renderActivity(null); });
+  }
   function renderConfidentiality(d){
     if (typeof hbars !== 'function') return;
     if (d.confidentialityLevels == null) { hbars('byConf', [], null, 'Not measured: the confidentiality query did not return'); return; }
@@ -2388,6 +2430,7 @@ <h2 id="d-title">—</h2>
     loadTileInfo();
     var rs = document.getElementById('range-sel');
     loadGrowth(rs ? rs.value : '6mo');
+    loadActivity();
     stamp();
   }
   function boot(){