Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -463,6 +463,11 @@ <h3>Data Stores by Kind</h3>
<div class="ph-sub">What the catalogued data stores are, by element type</div>
<div id="byKind"></div>
</div>
<div class="panel">
<h3>Assets by Technology</h3>
<div class="ph-sub">The technology each asset records (<code style="font-size:.7rem">deployedImplementationType</code>); most assets record none</div>
<div id="byTech"></div>
</div>
</div>

<!-- ── USAGE CONTEXT (ISC + BLUEPRINTS) ── -->
Expand Down Expand Up @@ -988,7 +993,7 @@ <h4>Share your feedback</h4>
hbars('byType',[],null,'No data yet');
hbars('byZone',[],null,'No data yet');
hbars('byConf',[],null,'No data yet');
hbars('byKind',[],null,'No data yet');
hbars('byKind',[],null,'No data yet'); hbars('byTech',[],null,'No data yet');
hbars('activity',[],null,'Not measured: no activity feed yet');
hbars('fbType',[],null,'No data yet');

Expand Down Expand Up @@ -1444,6 +1449,7 @@ <h4>Share your feedback</h4>
{ name: 'Assets by Type', status: 'live' }, // counts_by_type via /api/overview/summary
{ name: 'Elements by Governance Zone', status: 'live' }, // governed_coverage's topZones
{ name: 'Data Stores by Kind', status: 'live' }, // element types of the DataStore find the survey coverage already reads (no extra call)
{ name: 'Assets by Technology', status: 'live' }, // paged find of Assets grouped by deployedImplementationType
{ name: 'Confidentiality Distribution', status: 'live' }, // one bounded find of elements carrying a Confidentiality classification, grouped by level
],
usage: [
Expand Down Expand Up @@ -1826,6 +1832,20 @@ <h4>Share your feedback</h4>
if (rest.length) rows.push(['Other kinds (' + rest.length + ')', rest.reduce(function(a, k){ return a + k[1]; }, 0), 'var(--c6)']);
hbars('byKind', rows);
}
// Assets by Technology: exclusive groups that add up to the asset total (each technology, assets that
// record none, template placeholders). null = the asset list could not be read in full (failed or capped).
function renderAssetsByTechnology(d){
if (typeof hbars !== 'function') return;
if (d.assetsByTechnology == null) { hbars('byTech', [], null, 'Not measured: the asset list could not be read in full'); return; }
var ts = Object.keys(d.assetsByTechnology).map(function(k){ return [k, d.assetsByTechnology[k]]; })
.sort(function(a, b){ return b[1] - a[1] || (a[0] < b[0] ? -1 : 1); });
var rows = ts.slice(0, 8).map(function(k){ return [k[0], k[1], 'var(--c3)']; });
var rest = ts.slice(8);
if (rest.length) rows.push(['Other technologies (' + rest.length + ')', rest.reduce(function(a, k){ return a + k[1]; }, 0), 'var(--c6)']);
if (d.technologyPlaceholders) rows.push(['Template placeholders', d.technologyPlaceholders, '#33415c']);
if (d.technologyUnrecorded) rows.push(['No technology recorded', d.technologyUnrecorded, '#33415c']);
hbars('byTech', rows, null, 'No assets catalogued');
}
function renderConfidentiality(d){
if (typeof hbars !== 'function') return;
if (d.confidentialityLevels == null) { hbars('byConf', [], null, 'Not measured: the confidentiality query did not return'); return; }
Expand Down Expand Up @@ -2025,7 +2045,7 @@ <h4>Share your feedback</h4>
store.latestSurveyTime = d.latestSurveyTime; store.surveysCompleted7d = d.surveysCompleted7d;
store.engineActionHealth = d.engineActionHealth;
// render only after every value above is stored (this used to run before the survey values were set)
if (primary) { renderCertsSoon(d); renderSurveyCoverage(store); renderConfidentiality(d); renderDataStoreKinds(d); }
if (primary) { renderCertsSoon(d); renderSurveyCoverage(store); renderConfidentiality(d); renderDataStoreKinds(d); renderAssetsByTechnology(d); }
store.governedCount = d.governedCount;
if (d.assetTotal && d.governedCount != null) {
store.governedPctNum = Math.round(100 * d.governedCount / Math.max(d.assetTotal, d.governedCount));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -463,6 +463,7 @@ def get_summary(
schema = _schema_coverage(mgr, as_of_time, survey.get("schemaAnalysedGuids"))
engine_health = _engine_action_health(url, server, user_id, user_pwd, as_of_time)
confidentiality = _confidentiality_levels(mgr, as_of_time)
technology = _assets_by_technology(mgr, as_of_time)

payload = {
"asOfTime": as_of_time,
Expand Down Expand Up @@ -496,6 +497,10 @@ def get_summary(
"latestSurveyTime": survey["latestSurveyTime"],
"surveysCompleted7d": survey["surveysCompleted7d"],
"dataStoresByKind": survey["dataStoresByKind"],
"assetsByTechnology": technology["technologies"],
"technologyUnrecorded": technology["unrecorded"],
"technologyPlaceholders": technology["placeholders"],
"technologyAssetTotal": technology["total"],
"engineActionHealth": engine_health,
"surveysCompleted": survey["surveysCompleted"],
"surveysFailed": survey["surveysFailed"],
Expand Down Expand Up @@ -569,6 +574,46 @@ def _confidentiality_levels(mgr, as_of: Optional[str] = None) -> dict:
return out


def _assets_by_technology(mgr, as_of: Optional[str] = None) -> dict:
"""Assets grouped by the technology they record in `deployedImplementationType`. Most assets do not
record one, so the figures are exclusive and add up to the asset total: each technology, the assets
that record none (`unrecorded`), and template placeholders such as ``~{deployedImplementationType}~``
(`placeholders`: template elements carry the unfilled name, which is not a technology). Read in pages
until an empty one (Egeria's paging contract); every field is None when a page failed or
_REL_MAX_PAGES full pages were read without reaching the end (a capped read can only under-count)."""
out = {"technologies": None, "unrecorded": None, "placeholders": None, "total": None}
try:
counts: dict = {}
unrecorded = placeholders = total = 0
done = False
for page in range(_REL_MAX_PAGES):
body = {"class": "FindRequestBody", "metadataElementTypeName": "Asset",
"limitResultsByStatus": ["ACTIVE"], "graphQueryDepth": 0,
"startFrom": page * _SURVEY_LINK_CAP, "pageSize": _SURVEY_LINK_CAP}
if as_of:
body["asOfTime"] = as_of
chunk = mgr.find_metadata_elements(body)
chunk = [e for e in (chunk if isinstance(chunk, list) else []) if isinstance(e, dict)]
if not chunk:
done = True
break
for e in chunk:
total += 1
props = e.get("elementProperties") or {}
tech = str((props.get("propertiesAsStrings") or props).get("deployedImplementationType") or "").strip()
if not tech:
unrecorded += 1
elif tech.startswith("~{") and tech.endswith("}~"):
placeholders += 1
else:
counts[tech] = counts.get(tech, 0) + 1
if done:
out.update(technologies=counts, unrecorded=unrecorded, placeholders=placeholders, total=total)
except Exception as exc: # noqa: BLE001
logger.debug(f"overview assets by technology failed: {exc}")
return out


def _schema_coverage(mgr, as_of: Optional[str] = None, analysed=None) -> dict:
"""How many elements have a schema: the distinct anchors of SchemaType elements (a schema's anchor is
the asset it describes) UNION the assets a survey analysed the schema of (SchemaAnalysisAnnotation --
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -463,6 +463,11 @@ <h3>Data Stores by Kind</h3>
<div class="ph-sub">What the catalogued data stores are, by element type</div>
<div id="byKind"></div>
</div>
<div class="panel">
<h3>Assets by Technology</h3>
<div class="ph-sub">The technology each asset records (<code style="font-size:.7rem">deployedImplementationType</code>); most assets record none</div>
<div id="byTech"></div>
</div>
</div>

<!-- ── USAGE CONTEXT (ISC + BLUEPRINTS) ── -->
Expand Down Expand Up @@ -985,7 +990,7 @@ <h4>Share your feedback</h4>
hbars('byType',[],null,'No data yet');
hbars('byZone',[],null,'No data yet');
hbars('byConf',[],null,'No data yet');
hbars('byKind',[],null,'No data yet');
hbars('byKind',[],null,'No data yet'); hbars('byTech',[],null,'No data yet');
hbars('activity',[],null,'Not measured: no activity feed yet');
hbars('fbType',[],null,'No data yet');

Expand Down Expand Up @@ -1441,6 +1446,7 @@ <h4>Share your feedback</h4>
{ name: 'Assets by Type', status: 'live' }, // counts_by_type via /api/overview/summary
{ name: 'Elements by Governance Zone', status: 'live' }, // governed_coverage's topZones
{ name: 'Data Stores by Kind', status: 'live' }, // element types of the DataStore find the survey coverage already reads (no extra call)
{ name: 'Assets by Technology', status: 'live' }, // paged find of Assets grouped by deployedImplementationType
{ name: 'Confidentiality Distribution', status: 'live' }, // one bounded find of elements carrying a Confidentiality classification, grouped by level
],
usage: [
Expand Down Expand Up @@ -1814,6 +1820,20 @@ <h4>Share your feedback</h4>
if (rest.length) rows.push(['Other kinds (' + rest.length + ')', rest.reduce(function(a, k){ return a + k[1]; }, 0), 'var(--c6)']);
hbars('byKind', rows);
}
// Assets by Technology: exclusive groups that add up to the asset total (each technology, assets that
// record none, template placeholders). null = the asset list could not be read in full (failed or capped).
function renderAssetsByTechnology(d){
if (typeof hbars !== 'function') return;
if (d.assetsByTechnology == null) { hbars('byTech', [], null, 'Not measured: the asset list could not be read in full'); return; }
var ts = Object.keys(d.assetsByTechnology).map(function(k){ return [k, d.assetsByTechnology[k]]; })
.sort(function(a, b){ return b[1] - a[1] || (a[0] < b[0] ? -1 : 1); });
var rows = ts.slice(0, 8).map(function(k){ return [k[0], k[1], 'var(--c3)']; });
var rest = ts.slice(8);
if (rest.length) rows.push(['Other technologies (' + rest.length + ')', rest.reduce(function(a, k){ return a + k[1]; }, 0), 'var(--c6)']);
if (d.technologyPlaceholders) rows.push(['Template placeholders', d.technologyPlaceholders, '#33415c']);
if (d.technologyUnrecorded) rows.push(['No technology recorded', d.technologyUnrecorded, '#33415c']);
hbars('byTech', rows, null, 'No assets catalogued');
}
function renderConfidentiality(d){
if (typeof hbars !== 'function') return;
if (d.confidentialityLevels == null) { hbars('byConf', [], null, 'Not measured: the confidentiality query did not return'); return; }
Expand Down Expand Up @@ -2013,7 +2033,7 @@ <h4>Share your feedback</h4>
store.latestSurveyTime = d.latestSurveyTime; store.surveysCompleted7d = d.surveysCompleted7d;
store.engineActionHealth = d.engineActionHealth;
// render only after every value above is stored (this used to run before the survey values were set)
if (primary) { renderCertsSoon(d); renderSurveyCoverage(store); renderConfidentiality(d); renderDataStoreKinds(d); }
if (primary) { renderCertsSoon(d); renderSurveyCoverage(store); renderConfidentiality(d); renderDataStoreKinds(d); renderAssetsByTechnology(d); }
store.governedCount = d.governedCount;
if (d.assetTotal && d.governedCount != null) {
store.governedPctNum = Math.round(100 * d.governedCount / Math.max(d.assetTotal, d.governedCount));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -473,6 +473,7 @@ def get_summary(
schema = _schema_coverage(mgr, as_of_time, survey.get("schemaAnalysedGuids"))
engine_health = _engine_action_health(url, server, user_id, user_pwd, as_of_time)
confidentiality = _confidentiality_levels(mgr, as_of_time)
technology = _assets_by_technology(mgr, as_of_time)

payload = {
"asOfTime": as_of_time,
Expand Down Expand Up @@ -506,6 +507,10 @@ def get_summary(
"latestSurveyTime": survey["latestSurveyTime"],
"surveysCompleted7d": survey["surveysCompleted7d"],
"dataStoresByKind": survey["dataStoresByKind"],
"assetsByTechnology": technology["technologies"],
"technologyUnrecorded": technology["unrecorded"],
"technologyPlaceholders": technology["placeholders"],
"technologyAssetTotal": technology["total"],
"engineActionHealth": engine_health,
"surveysCompleted": survey["surveysCompleted"],
"surveysFailed": survey["surveysFailed"],
Expand Down Expand Up @@ -579,6 +584,46 @@ def _confidentiality_levels(mgr, as_of: Optional[str] = None) -> dict:
return out


def _assets_by_technology(mgr, as_of: Optional[str] = None) -> dict:
"""Assets grouped by the technology they record in `deployedImplementationType`. Most assets do not
record one, so the figures are exclusive and add up to the asset total: each technology, the assets
that record none (`unrecorded`), and template placeholders such as ``~{deployedImplementationType}~``
(`placeholders`: template elements carry the unfilled name, which is not a technology). Read in pages
until an empty one (Egeria's paging contract); every field is None when a page failed or
_REL_MAX_PAGES full pages were read without reaching the end (a capped read can only under-count)."""
out = {"technologies": None, "unrecorded": None, "placeholders": None, "total": None}
try:
counts: dict = {}
unrecorded = placeholders = total = 0
done = False
for page in range(_REL_MAX_PAGES):
body = {"class": "FindRequestBody", "metadataElementTypeName": "Asset",
"limitResultsByStatus": ["ACTIVE"], "graphQueryDepth": 0,
"startFrom": page * _SURVEY_LINK_CAP, "pageSize": _SURVEY_LINK_CAP}
if as_of:
body["asOfTime"] = as_of
chunk = mgr.find_metadata_elements(body)
chunk = [e for e in (chunk if isinstance(chunk, list) else []) if isinstance(e, dict)]
if not chunk:
done = True
break
for e in chunk:
total += 1
props = e.get("elementProperties") or {}
tech = str((props.get("propertiesAsStrings") or props).get("deployedImplementationType") or "").strip()
if not tech:
unrecorded += 1
elif tech.startswith("~{") and tech.endswith("}~"):
placeholders += 1
else:
counts[tech] = counts.get(tech, 0) + 1
if done:
out.update(technologies=counts, unrecorded=unrecorded, placeholders=placeholders, total=total)
except Exception as exc: # noqa: BLE001
logger.debug(f"overview assets by technology failed: {exc}")
return out


def _schema_coverage(mgr, as_of: Optional[str] = None, analysed=None) -> dict:
"""How many elements have a schema: the distinct anchors of SchemaType elements (a schema's anchor is
the asset it describes) UNION the assets a survey analysed the schema of (SchemaAnalysisAnnotation --
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright Contributors to the ODPi Egeria project.
"""_assets_by_technology() groups assets by deployedImplementationType into exclusive buckets.

Most catalogued assets record no technology, so the result must add up to the asset total (each technology
+ assets that record none + template placeholders) and must never present a partial read as a total: it
reads pages until an empty one and returns None everywhere if a page failed or the page limit was hit.

The function is extracted from each handler file (quickstart and freshstart carry identical copies) and run
against a fake manager, without importing pyegeria or FastAPI.
"""
import ast
from pathlib import Path
from typing import Optional

import pytest

_HERE = Path(__file__).resolve().parent
_HANDLERS = {
"quickstart": _HERE.parent / "overview_handler.py",
"freshstart": _HERE.parents[2] / "egeria-freshstart" / "PyegeriaWebHandler" / "overview_handler.py",
}
_PAGE = 1000
_NONE = {"technologies": None, "unrecorded": None, "placeholders": None, "total": None}


def _load(path: Path):
if not path.exists():
pytest.skip(f"{path} not present in this checkout")
tree = ast.parse(path.read_text(encoding="utf-8"))
body = [n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == "_assets_by_technology"]
assert len(body) == 1
ns = {"Optional": Optional, "_SURVEY_LINK_CAP": _PAGE, "_REL_MAX_PAGES": 10,
"logger": type("L", (), {"debug": staticmethod(lambda *a, **k: None)})()}
exec(compile(ast.Module(body=body, type_ignores=[]), str(path), "exec"), ns)
return ns["_assets_by_technology"]


def _asset(tech=None):
props = {} if tech is None else {"deployedImplementationType": tech}
return {"elementGUID": "g", "elementProperties": {"propertiesAsStrings": props}}


class _Mgr:
def __init__(self, assets, fail_at=None):
self.assets, self.fail_at, self.bodies = assets, fail_at, []

def find_metadata_elements(self, body):
self.bodies.append(body)
if self.fail_at is not None and body["startFrom"] >= self.fail_at:
raise RuntimeError("boom")
lo = body["startFrom"]
return self.assets[lo:lo + body["pageSize"]]


@pytest.fixture(params=sorted(_HANDLERS))
def by_tech(request):
return _load(_HANDLERS[request.param])


def test_buckets_are_exclusive_and_add_up_to_the_total(by_tech):
assets = [_asset("CSV Data File")] * 3 + [_asset("Integration Connector")] + [_asset()] * 5 \
+ [_asset("~{deployedImplementationType}~")] * 2 + [_asset(" ")]
out = by_tech(_Mgr(assets))
assert out["technologies"] == {"CSV Data File": 3, "Integration Connector": 1}
assert out["unrecorded"] == 6 and out["placeholders"] == 2 and out["total"] == 12
assert sum(out["technologies"].values()) + out["unrecorded"] + out["placeholders"] == out["total"]


def test_reads_every_page_and_ends_on_an_empty_one(by_tech):
m = _Mgr([_asset("A")] * 2500)
out = by_tech(m)
assert out["technologies"] == {"A": 2500} and out["total"] == 2500
assert [b["startFrom"] for b in m.bodies] == [0, 1000, 2000, 3000]


def test_as_of_time_is_sent_and_only_active_assets_are_read(by_tech):
m = _Mgr([_asset("A")])
by_tech(m, "2026-10-01T00:00:00")
assert m.bodies[0]["asOfTime"] == "2026-10-01T00:00:00"
assert m.bodies[0]["limitResultsByStatus"] == ["ACTIVE"] and m.bodies[0]["metadataElementTypeName"] == "Asset"


def test_no_assets_is_a_real_zero(by_tech):
assert by_tech(_Mgr([])) == {"technologies": {}, "unrecorded": 0, "placeholders": 0, "total": 0}


def test_hitting_the_page_limit_withholds_everything(by_tech):
assert by_tech(_Mgr([_asset("A")] * (10 * _PAGE + 1))) == _NONE


def test_a_failed_page_discards_everything(by_tech):
assert by_tech(_Mgr([_asset("A")] * 5000, fail_at=2000)) == _NONE
Loading