feat(llma): add typed offline evaluations SDK (#1005)

* feat(llma): add typed offline evaluations sdk

* fix(llma): reject integer scores that lose precision

Author: Radu-Raicea

.github/scripts/check_public_api.py

@@ -59,6 +59,15 @@ def _is_public_alias(obj: object) -> bool:
     return _is_public_posthog_path(target_path)
 
 
+def _is_private_sdk_alias(obj: object) -> bool:
+    target = _alias_target_path(obj)
+    return (
+        target.startswith("posthog.")
+        and not _is_excluded_path(target)
+        and not _is_public_posthog_path(target)
+    )
+
+
 def _module_exports(module: object) -> set[str] | None:
     exports = getattr(module, "exports", None)
     if exports is None:
@@ -98,7 +107,9 @@ def _is_public_member(parent: object, name: str, member: object) -> bool:
         return False
 
     if _is_alias(member):
-        return _is_public_alias(member)
+        return _is_public_alias(member) or (
+            exports is not None and _is_private_sdk_alias(member)
+        )
 
     if _is_module(member):
         return _is_public_posthog_path(_object_path(member))
@@ -119,7 +130,7 @@ def _signature(obj: object) -> str | None:
 
 def _signature_for_path(obj: object) -> str | None:
     signature = _signature(obj)
-    if signature is None:
+    if signature is None or (not signature and _is_alias(obj)):
         return None
 
     name = str(getattr(obj, "name", ""))
@@ -144,7 +155,7 @@ def _attribute_details(obj: object) -> str:
 
 
 def _record(obj: object) -> str:
-    if _is_alias(obj):
+    if _is_alias(obj) and not _is_private_sdk_alias(obj):
         return f"alias {_object_path(obj)} -> {_alias_target_path(obj)}"
 
     if _is_module(obj):
@@ -169,15 +180,21 @@ def _record(obj: object) -> str:
     return f"{getattr(obj, 'kind', 'object')} {_object_path(obj)}"
 
 
-def _iter_class_members(cls: object) -> Iterable[object]:
+def _iter_class_members(
+    cls: object, *, exported_alias: bool = False
+) -> Iterable[object]:
     members = getattr(cls, "members", {})
     for name, member in sorted(members.items()):
-        if not _is_public_member(cls, str(name), member):
+        if not (
+            _is_public_name(str(name))
+            if exported_alias
+            else _is_public_member(cls, str(name), member)
+        ):
             continue
 
         yield member
-        if not _is_alias(member) and _is_class(member):
-            yield from _iter_class_members(member)
+        if _is_class(member) and (exported_alias or not _is_alias(member)):
+            yield from _iter_class_members(member, exported_alias=exported_alias)
 
 
 def _iter_module_members(module: object) -> Iterable[object]:
@@ -188,6 +205,8 @@ def _iter_module_members(module: object) -> Iterable[object]:
         if _is_alias(member):
             if _is_public_member(module, str(name), member):
                 yield member
+                if _is_private_sdk_alias(member) and _is_class(member):
+                    yield from _iter_class_members(member, exported_alias=True)
             continue
 
         if _is_module(member):

.github/scripts/check_strict_types.sh

@@ -15,6 +15,14 @@ import atexit
 
 import posthog
 from posthog import FeatureFlagEvaluations, FlagValue, Posthog
+from posthog.ai.evaluations import (
+    AsyncOfflineEvaluations,
+    BooleanScorerConfig,
+    EvaluationItem,
+    NumericPassingRule,
+    NumericScorerConfig,
+    OfflineEvaluations,
+)
 
 client = Posthog("phc_test")
 atexit.register(client.shutdown)
@@ -31,6 +39,40 @@ active: posthog.Span | None = posthog.get_active_span()
 span.end()
 
 _ = (flag_value, all_flags, enabled, payload, evaluations, active)
+
+
+async def offline_evaluation_types(
+    sync_client: OfflineEvaluations, async_client: AsyncOfflineEvaluations
+) -> None:
+    numeric = sync_client.scorers.create(
+        name="Relevance",
+        kind="numeric",
+        config=NumericScorerConfig(
+            min=0, max=1,
+            passing_rule=NumericPassingRule(operator="gte", threshold=0.8),
+        ),
+    )
+    config: NumericScorerConfig = numeric.config
+    experiment = sync_client.create_experiment(name="baseline", run_source="ci")
+    item = EvaluationItem(input="question", output="answer")
+    experiment.upload_result(item=item, scorer_version_id=numeric.current_version_id, value=0.9)
+    boolean = await async_client.scorers.create(
+        name="Toxicity", kind="boolean", config=BooleanScorerConfig(true_is_failure=True)
+    )
+    async_experiment = await async_client.create_experiment(name="async run")
+    await async_experiment.upload_result(item=item, scorer_version_id=boolean.current_version_id, value=False)
+    _ = config
+PY
+
+cat > "$tmp/invalid_offline_types.py" <<'PY'
+# pyright: strict
+from posthog.ai.evaluations import BooleanScorerConfig, CategoricalScorerConfig, NumericPassingRule, NumericScorerConfig, OfflineEvaluations
+
+client = OfflineEvaluations(project_id=123, secret_key="phx_example")
+NumericPassingRule(operator="gt", threshold=0.8)  # expected-type-error
+NumericScorerConfig(unknown_setting=True)  # expected-type-error
+CategoricalScorerConfig(selection_mode="single")  # expected-type-error
+client.scorers.create(name="Bad config", kind="numeric", config=BooleanScorerConfig(true_is_failure=True))  # expected-type-error
 PY
 
 "$tmp/.venv/bin/python" - <<'PY' > "$tmp/public_api_access.py"
@@ -73,3 +115,21 @@ JSON
 
 cd "$tmp"
 "$tmp/.venv/bin/python" -m pyright strict_posthog_types.py public_api_access.py
+
+if "$tmp/.venv/bin/python" -m pyright --outputjson invalid_offline_types.py > invalid_types.json; then
+    echo "Invalid offline evaluation configurations unexpectedly passed type checking."
+    exit 1
+fi
+"$tmp/.venv/bin/python" - <<'PY'
+import json
+from pathlib import Path
+
+expected = {
+    index for index, line in enumerate(Path("invalid_offline_types.py").read_text().splitlines())
+    if "expected-type-error" in line
+}
+diagnostics = json.loads(Path("invalid_types.json").read_text())["generalDiagnostics"]
+actual = {item["range"]["start"]["line"] for item in diagnostics if item["severity"] == "error"}
+assert actual == expected, diagnostics
+print("Offline evaluation configuration types reject invalid inputs.")
+PY

.github/scripts/test_check_public_api.py

@@ -6,6 +6,7 @@
 import importlib.util
 from pathlib import Path
 from types import SimpleNamespace
+from tempfile import TemporaryDirectory
 
 
 SCRIPT_PATH = Path(__file__).with_name("check_public_api.py")
@@ -42,8 +43,41 @@ def test_attribute_details_uses_placeholder_values() -> None:
     )
 
 
+def test_explicit_exports_from_private_modules_include_signatures() -> None:
+    import griffe
+
+    check_public_api = load_check_public_api()
+    with TemporaryDirectory() as directory:
+        package = Path(directory) / "posthog"
+        package.mkdir()
+        (package / "__init__.py").write_text(
+            'from ._implementation import Client, Hidden\n__all__ = ["Client"]\n'
+        )
+        (package / "_implementation.py").write_text(
+            "class Client:\n"
+            "    def __init__(self, *, key: str): pass\n"
+            "    def upload(self, value: float) -> bool: return True\n"
+            "    def _internal(self): pass\n"
+            "class Hidden: pass\n"
+        )
+        module = griffe.load(
+            "posthog",
+            search_paths=[directory],
+            allow_inspection=False,
+            try_relative_path=False,
+        )
+        records = [
+            check_public_api._record(obj)
+            for obj in check_public_api._iter_module_members(module)
+        ]
+    assert "class posthog.Client(*, key: str)" in records
+    assert "method posthog.Client.upload(value: float)" in records
+    assert not any("Hidden" in record or "_internal" in record for record in records)
+
+
 def main() -> int:
     test_attribute_details_uses_placeholder_values()
+    test_explicit_exports_from_private_modules_include_signatures()
     print("check_public_api tests passed.")
     return 0
 

.sampo/changesets/offline-evaluations.md

@@ -0,0 +1,5 @@
+---
+pypi/posthog: minor
+---
+
+Add typed synchronous and asynchronous offline evaluation clients with reusable items, acknowledged bulk uploads, resumable identities, and explicit experiment completion. Manage scorers and immutable versions through the SDK, including boolean, numeric, and categorical polarity configuration.