diff --git a/.github/workflows/backcompat_matrix.yml b/.github/workflows/backcompat_matrix.yml new file mode 100644 index 00000000000..29335d4da7b --- /dev/null +++ b/.github/workflows/backcompat_matrix.yml @@ -0,0 +1,131 @@ +name: Mixed-version back-compat matrix + +# Runs the intent-topic back-compat suite across four skill/core version +# combinations. Each combination needs TWO python environments alive at once — +# one for the skill's packages, one for the core's — so it cannot live in +# `build_tests.yml`, which installs a single environment and runs pytest in it. +# +# The four cells and the pins behind them are documented in +# `test/backcompat/test_mixed_version_matrix.py` and +# `test/backcompat/build_venvs.sh`. +# +# Alongside the boundary-pin cells there are four channel cells +# (stable-skill/dev-core, dev-skill/stable-core, testing-skill/dev-core, +# dev-skill/testing-core) built from the OVOS distro's own constraint +# files, fetched fresh at build time. Those track the fleet rather than a +# boundary: a distro pin bump past a behavior boundary flips one of these +# red at that exact moment. +# +# The `old-skill/new-core` cell is expected to xfail until ovos-bus-client#271 +# releases. It is `strict=True`, so the day the fix ships the job goes RED with +# an XPASS. That is the intended signal, not a regression: drop the marker and +# the cell becomes a permanent guard. +# +# This job does NOT use the shared gh-automations reusable workflow: it builds +# its own venv pairs rather than one environment. + +on: + pull_request: + branches: [dev] + paths: + - 'ovos_core/**' + - 'test/backcompat/**' + - '.github/workflows/backcompat_matrix.yml' + - 'requirements/**' + push: + branches: [dev] + workflow_dispatch: + +jobs: + matrix: + runs-on: ubuntu-latest + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + include: + # combo id skill venv core venv + - combo: old-skill/old-core + skill: venv_skill_old + core: venv_core_old + - combo: old-skill/new-core + skill: venv_skill_old + core: venv_core_new + - combo: new-skill/old-core + skill: venv_skill_new + core: venv_core_old + - combo: new-skill/new-core + skill: venv_skill_new + core: venv_core_new + # channel cells: one side pinned by a live OVOS distro constraints + # file (fetched fresh in build_venvs.sh), the other at dev. These + # verify the fleet, not a boundary — see test_mixed_version_matrix.py. + - combo: stable-skill/dev-core + skill: venv_skill_stable + core: venv_core_new + - combo: dev-skill/stable-core + skill: venv_skill_new + core: venv_core_stable + - combo: testing-skill/dev-core + skill: venv_skill_testing + core: venv_core_new + - combo: dev-skill/testing-core + skill: venv_skill_new + core: venv_core_testing + name: ${{ matrix.combo }} + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install system deps (padatious needs swig + libfann) + run: sudo apt-get update && sudo apt-get install -y swig libfann-dev + + - name: Install uv + uses: astral-sh/setup-uv@v5 + + - name: Build the venv pair + env: + BACKCOMPAT_PYTHON: "3.11" + run: | + chmod +x test/backcompat/build_venvs.sh + test/backcompat/build_venvs.sh "$RUNNER_TEMP/venvs" "$GITHUB_WORKSPACE" + + - name: Record the resolved versions + run: | + for v in venv_skill_old venv_skill_new venv_core_old venv_core_new; do + echo "== $v" + "$RUNNER_TEMP/venvs/$v/bin/python" -m pip freeze 2>/dev/null \ + || "$RUNNER_TEMP/venvs/$v/bin/python" -c \ + "from importlib.metadata import distributions; [print(f'{d.metadata[\"Name\"]}=={d.version}') for d in sorted(distributions(), key=lambda x: x.metadata['Name'] or '')]" + done | tee venv-freeze.txt + + - name: Upload the resolved versions + if: always() + uses: actions/upload-artifact@v4 + with: + name: venv-freeze-${{ strategy.job-index }} + path: venv-freeze.txt + + # Channel cells only produce these, but the glob is harmless for the + # boundary-pin cells. Kept so a red channel cell can be traced back to + # exactly what the distro pinned that day. + - name: Upload the fetched constraints files + if: always() + uses: actions/upload-artifact@v4 + with: + name: constraints-${{ strategy.job-index }} + path: ${{ runner.temp }}/venvs/*.constraints.txt + if-no-files-found: ignore + + - name: Run the combo + env: + BACKCOMPAT_COMBO: ${{ matrix.combo }} + run: | + export BACKCOMPAT_SKILL_PYTHON="$RUNNER_TEMP/venvs/${{ matrix.skill }}/bin/python" + "$RUNNER_TEMP/venvs/${{ matrix.core }}/bin/pytest" \ + test/backcompat/ -v --tb=short -rxX \ + --timeout=300 --timeout-method=thread diff --git a/test/backcompat/__init__.py b/test/backcompat/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/backcompat/build_venvs.sh b/test/backcompat/build_venvs.sh new file mode 100755 index 00000000000..2aa1dd10018 --- /dev/null +++ b/test/backcompat/build_venvs.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +# Build the four venvs the mixed-version back-compat matrix runs against. +# +# Two venvs are alive per combo: one holds the skill's packages, one holds the +# core's. They talk over a real websocket, which is the only way one test run +# can observe two different package sets. +# +# Usage: test/backcompat/build_venvs.sh [path-to-core-checkout] +# +# Pins, and why each one is where it is: +# +# skill_old ovos-workshop==9.3.1a2 +# The newest release that binds the handler to the suffixed +# `:.intent` topic ONLY. 9.3.2a1 added the +# canonical binding alongside it (ovos-workshop#497), which hides +# the breakage, so the pin must stay below that line. +# ovos-bus-client is deliberately NOT pinned: a real frozen +# container resolves the workshop floor and gets a current client, +# and the repair in ovos-bus-client#271 depends on exactly that. +# +# skill_new ovos-workshop @ dev +# Binds both spellings today. ovos-workshop#500 makes it +# canonical-only, which is what will turn the new-skill/old-core +# cell red. +# +# core_old ovos-core==2.5.5a2 + ovos-padatious==2.0.0a1 +# The newest padatious release BEFORE registration-time +# canonicalization (`_dealias_intent_name`, added in 2.0.1a1), so +# this side dispatches whatever the skill registered. ovos-core is +# pinned to its contemporary release; core itself forwards +# `match.match_type` verbatim, so the pipeline plugin is the part +# that decides the spelling. +# +# core_new this checkout + ovos-padatious>=2.0.1a2 +# Folds at registration, so it dispatches the canonical topic. +# +# venv_skill_stable / venv_skill_testing +# venv_core_stable / venv_core_testing +# Not boundary pins — fleet pins. Built by installing straight +# off the OVOS distro's own constraint files, fetched at build +# time (not vendored), so this gate TRACKS the fleet: the day the +# distro bumps a pin past a behavior boundary, the affected cell +# goes red at that exact moment, which is the point. Today both +# channels resolve an ovos-workshop and ovos-padatious floor well +# below the 9.3.2a1 / 2.0.1a1 boundaries above, so the *-skill +# side is suffixed-only and the *-core side never canonicalizes +# on either channel — see test_mixed_version_matrix.py for what +# that implies per cell. The fetched constraints file is saved +# into $TARGET for upload as a CI artifact, so a future red cell +# can be traced back to exactly what was pinned that day. +set -euo pipefail + +TARGET="${1:?usage: build_venvs.sh [core-checkout]}" +CORE_CHECKOUT="${2:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)}" + +STABLE_CONSTRAINTS_URL="${BACKCOMPAT_STABLE_CONSTRAINTS_URL:-https://raw.githubusercontent.com/OpenVoiceOS/OpenVoiceOS/main/constraints-stable.txt}" +TESTING_CONSTRAINTS_URL="${BACKCOMPAT_TESTING_CONSTRAINTS_URL:-https://raw.githubusercontent.com/OpenVoiceOS/OpenVoiceOS/main/constraints-testing.txt}" + +PY="${BACKCOMPAT_PYTHON:-python3.11}" +mkdir -p "$TARGET" + +have_uv() { command -v uv >/dev/null 2>&1; } + +mkvenv() { + local name="$1"; shift + local dir="$TARGET/$name" + echo "==> building $name" + if have_uv; then + uv venv --python "$PY" "$dir" >/dev/null + VIRTUAL_ENV="$dir" uv pip install --quiet --prerelease=allow "$@" + else + "$PY" -m venv "$dir" + "$dir/bin/pip" install --quiet --upgrade pip + "$dir/bin/pip" install --quiet --pre "$@" + fi + echo " $("$dir/bin/python" -c 'import sys; print(sys.version.split()[0])')" +} + +# Like mkvenv, but constrained by a distro constraints file fetched fresh at +# build time. $1=name $2=constraints-url $3..=packages to install +mkvenv_channel() { + local name="$1" url="$2"; shift 2 + local dir="$TARGET/$name" + local cfile="$TARGET/${name#venv_}.constraints.txt" + echo "==> fetching constraints for $name from $url" + curl -fsSL "$url" -o "$cfile" + echo "==> building $name (constrained)" + if have_uv; then + uv venv --python "$PY" "$dir" >/dev/null + VIRTUAL_ENV="$dir" uv pip install --quiet --prerelease=allow -c "$cfile" "$@" + else + "$PY" -m venv "$dir" + "$dir/bin/pip" install --quiet --upgrade pip + "$dir/bin/pip" install --quiet --pre -c "$cfile" "$@" + fi + echo " $("$dir/bin/python" -c 'import sys; print(sys.version.split()[0])')" +} + +mkvenv venv_skill_old "ovos-workshop==9.3.1a2" "setuptools<81" +mkvenv venv_skill_new "ovos-workshop @ git+https://github.com/OpenVoiceOS/ovos-workshop@dev" "setuptools<81" +mkvenv venv_core_old "ovos-core==2.5.5a2" "ovos-padatious==2.0.0a1" ovos-messagebus pytest pytest-timeout "setuptools<81" +mkvenv venv_core_new "$CORE_CHECKOUT" "ovos-padatious>=2.0.1a2" ovos-messagebus pytest pytest-timeout "setuptools<81" + +mkvenv_channel venv_skill_stable "$STABLE_CONSTRAINTS_URL" ovos-workshop "setuptools<81" +mkvenv_channel venv_skill_testing "$TESTING_CONSTRAINTS_URL" ovos-workshop "setuptools<81" +mkvenv_channel venv_core_stable "$STABLE_CONSTRAINTS_URL" ovos-core ovos-padatious ovos-messagebus pytest pytest-timeout "setuptools<81" +mkvenv_channel venv_core_testing "$TESTING_CONSTRAINTS_URL" ovos-core ovos-padatious ovos-messagebus pytest pytest-timeout "setuptools<81" + +echo +echo "resolved versions:" +for v in venv_skill_old venv_skill_new venv_core_old venv_core_new \ + venv_skill_stable venv_skill_testing venv_core_stable venv_core_testing; do + echo " $v:" + "$TARGET/$v/bin/python" - <<'EOF' || true +from importlib.metadata import version, PackageNotFoundError +for p in ("ovos-workshop", "ovos-bus-client", "ovos-core", "ovos-padatious", + "ovos-spec-tools"): + try: + print(f" {p}=={version(p)}") + except PackageNotFoundError: + pass +EOF +done diff --git a/test/backcompat/driver.py b/test/backcompat/driver.py new file mode 100644 index 00000000000..6f7f81f014a --- /dev/null +++ b/test/backcompat/driver.py @@ -0,0 +1,242 @@ +"""Shared plumbing for the mixed-version back-compat matrix. + +The driver process **is** the core side of a combo: it runs under the core +venv, so ``ovos-core`` / ``ovos-padatious`` / ``ovos-bus-client`` here are the +versions that combo pins. The skill side lives in a separate venv and is +reached only over a real websocket, which is what makes two package sets +observable at once. + +Nothing in this module mocks the bus. A real ``ovos-messagebus`` is started on +a free port and both sides connect to it. +""" +import json +import os +import socket +import subprocess +import sys +import tempfile +import time +import uuid +from threading import Event +from typing import Optional + +from ovos_bus_client.client import MessageBusClient +from ovos_bus_client.message import Message + +SKILL_ID = "backcompat.mixed.test" +INTENT_FILE = "food.order.intent" +LEGACY_TOPIC = f"{SKILL_ID}:{INTENT_FILE}" +CANONICAL_TOPIC = f"{SKILL_ID}:food.order" + +HERE = os.path.dirname(os.path.abspath(__file__)) +SKILL_SCRIPT = os.path.join(HERE, "skill_process.py") + +#: How long to wait for the skill venv to import workshop and register. A cold +#: interpreter plus resource loading is slow; this is not a latency assertion. +SKILL_BOOT_TIMEOUT = 120 +#: How long to wait for a handler to answer once the dispatch is on the wire. +#: Generous on purpose — a short window would turn CI jitter into a fake +#: "compat is broken" result, and this suite must only fail for real reasons. +DISPATCH_TIMEOUT = 10 + + +def free_port() -> int: + """Grab a port the messagebus can own for one test run.""" + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +def core_canonicalizes() -> bool: + """Whether the core-side stack folds the suffixed intent id at registration. + + This is the single behaviour that decides which topic a combo puts on the + wire, and it lives in the padatious **pipeline plugin**, not in ovos-core: + ``ovos-core`` forwards ``match.match_type`` verbatim + (``IntentService._dispatch_match``), so the spelling is whatever the engine + registry holds. + + ``ovos-padatious >= 2.0.1a1`` folds ``:.intent`` onto + ``:`` inside ``register_intent``, so every match is + canonical by construction. Older releases keep whatever the skill sent. + + The probe is the real symbol rather than a version string, so the matrix + keeps telling the truth if the fold ever moves or is reverted. + """ + import ovos_padatious.opm as opm + return hasattr(opm, "_dealias_intent_name") + + +def dispatch_topic_for(registered_name: str) -> str: + """The topic this core stack would dispatch for ``registered_name``.""" + if not core_canonicalizes(): + return registered_name + import ovos_padatious.opm as opm + return opm._dealias_intent_name(registered_name) + + +def make_shared_config(port: int) -> str: + """Write a throwaway ``mycroft.conf`` pinning the bus to ``port``. + + Neither the bus server nor the client reads a port from the environment, + and both venvs must agree on one. An ``XDG_CONFIG_HOME`` pointed at this + directory is the one knob that reaches every process regardless of which + venv it runs in, and it keeps the run off the developer's real bus. + """ + root = tempfile.mkdtemp(prefix="backcompat-xdg-") + conf_dir = os.path.join(root, "mycroft") + os.makedirs(conf_dir) + with open(os.path.join(conf_dir, "mycroft.conf"), "w") as f: + json.dump({"websocket": {"host": "127.0.0.1", "port": port, + "route": "/core", "ssl": False}}, f) + return root + + +class BusServer: + """A real ``ovos-messagebus`` on a private port.""" + + def __init__(self): + self.port = free_port() + self.xdg = make_shared_config(self.port) + env = dict(os.environ, XDG_CONFIG_HOME=self.xdg) + self.proc = subprocess.Popen( + [sys.executable, "-m", "ovos_messagebus"], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + text=True, env=env) + self._wait_until_accepting() + + def _wait_until_accepting(self, timeout: int = 60): + deadline = time.time() + timeout + while time.time() < deadline: + if self.proc.poll() is not None: + raise RuntimeError( + "messagebus died during startup:\n" + + (self.proc.stdout.read() if self.proc.stdout else "")) + with socket.socket() as s: + if s.connect_ex(("127.0.0.1", self.port)) == 0: + return + time.sleep(0.25) + raise RuntimeError(f"messagebus never accepted on port {self.port}") + + def client(self) -> MessageBusClient: + bus = MessageBusClient(host="127.0.0.1", port=self.port, route="/core") + bus.run_in_thread() + if not bus.connected_event.wait(30): + raise RuntimeError("driver could not connect to the messagebus") + return bus + + def stop(self): + self.proc.terminate() + try: + self.proc.wait(timeout=15) + except subprocess.TimeoutExpired: + self.proc.kill() + + +class SkillProcess: + """The other venv's skill, as a child process on the same bus. + + ``emit_legacy`` is threaded through as an environment flag because the + kill-switch control needs to turn the compat bridge off in the **skill** + process, where bus-client#271's mirror runs — not in the driver. + """ + + def __init__(self, python: str, xdg: str, emit_legacy: bool = True, + blanket: bool = False): + env = dict(os.environ, + XDG_CONFIG_HOME=xdg, + OVOS_BUS_EMIT_LEGACY=str(emit_legacy).lower(), + OVOS_BUS_INTENT_REEMIT_BLANKET=str(blanket).lower(), + BACKCOMPAT_SKILL_ID=SKILL_ID, + PYTHONUNBUFFERED="1") + self.lines = [] + self.bound_topics = [] + #: versions resolved inside the skill venv, reported by the child + self.versions = {} + self.proc = subprocess.Popen( + [python, SKILL_SCRIPT], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + text=True, env=env) + self._wait_ready() + + def _wait_ready(self): + deadline = time.time() + SKILL_BOOT_TIMEOUT + while time.time() < deadline: + line = self.proc.stdout.readline() + if not line: + if self.proc.poll() is not None: + raise RuntimeError( + "skill process died before registering:\n" + self.log) + continue + self.lines.append(line.rstrip()) + if line.startswith("BOUND_TOPICS "): + self.bound_topics = json.loads(line[len("BOUND_TOPICS "):]) + if line.startswith("VERSIONS "): + self.versions = json.loads(line[len("VERSIONS "):]) + if line.startswith("SKILL_READY"): + return + raise RuntimeError(f"skill process never reported ready:\n{self.log}") + + @property + def log(self) -> str: + return "\n".join(self.lines) + + def stop(self): + self.proc.terminate() + try: + self.proc.wait(timeout=15) + except subprocess.TimeoutExpired: + self.proc.kill() + + +class Capture: + """Collect messages seen on a topic, with a wait-for-first helper. + + ``token`` narrows the capture to one dispatch. The bus is shared by the + whole module and a handler's answer can land after the test that triggered + it has moved on, so counting everything that ever appeared on a topic would + charge one test for another test's traffic — and a duplicate-firing check + has to be able to tell those apart. + """ + + def __init__(self, bus: MessageBusClient, topic: str, + token: Optional[str] = None): + self.bus = bus + self.topic = topic + self.token = token + self.messages = [] + self._seen = Event() + bus.on(topic, self._handle) + + def _matches(self, message: Message) -> bool: + if self.token is None: + return True + data = message.data or {} + # the marker echoes the dispatch payload under "data"; the dispatch + # itself carries the token at the top level + return (data.get("token") == self.token + or (data.get("data") or {}).get("token") == self.token) + + def _handle(self, message: Message): + if not self._matches(message): + return + self.messages.append(message) + self._seen.set() + + def wait(self, timeout: float = DISPATCH_TIMEOUT) -> bool: + return self._seen.wait(timeout) + + def close(self): + self.bus.remove(self.topic, self._handle) + + +def dispatch(bus: MessageBusClient, topic: str, **data) -> str: + """Emit an intent dispatch the way ``IntentService._dispatch_match`` does. + + Returns the correlation token stamped into the payload so a capture can + attribute the answer to this dispatch and no other. + """ + token = uuid.uuid4().hex + bus.emit(Message(topic, dict(data, token=token), + {"session": {"session_id": f"backcompat-{token[:8]}"}})) + return token diff --git a/test/backcompat/skill_process.py b/test/backcompat/skill_process.py new file mode 100644 index 00000000000..14a194f86fd --- /dev/null +++ b/test/backcompat/skill_process.py @@ -0,0 +1,115 @@ +"""Standalone skill process for the mixed-version back-compat matrix. + +Runs in the **skill venv** (its own ``ovos-workshop`` + whatever +``ovos-bus-client`` that workshop's dependency floor resolves to), never in the +venv the test driver runs in. That separation is the whole point: the two +package sets have to be able to differ, which one process cannot do. + +The script builds a throwaway skill directory with a single +``food.order.intent`` resource, brings up an ``OVOSSkill`` against the real +websocket bus, and registers the intent through ``register_intent_file`` — the +exact call whose topic spelling changed across workshop releases. + +When the handler runs it emits two things: + +* ``backcompat.skill.handled`` — a precise marker carrying the topic that + actually fired, so the driver can tell the canonical dispatch from the + suffixed twin and count firings (a double-fire is a failure, not a pass); +* a normal ``speak`` — the realistic "the skill answered" signal. + +``SKILL_READY`` on stdout means registration finished and the driver may +dispatch. Everything else on stdout is diagnostic and is echoed by the driver +when an assertion fails. +""" +import json +import os +import sys +import tempfile +import time +from os.path import join + +from ovos_bus_client.client import MessageBusClient +from ovos_bus_client.message import Message +from ovos_workshop.skills.ovos import OVOSSkill + +SKILL_ID = os.environ.get("BACKCOMPAT_SKILL_ID", "backcompat.mixed.test") +INTENT_FILE = "food.order.intent" + +#: Sample lines for the padatious resource. The matcher is never exercised — +#: the driver dispatches the registered topic directly, the way ovos-core's +#: ``_dispatch_match`` forwards ``match.match_type`` verbatim — but +#: ``register_intent_file`` refuses to register a resource it cannot read. +SAMPLES = ["order some tacos", "i am hungry", "grab some food"] + + +def _dist_version(name: str) -> str: + from importlib.metadata import version + return version(name) + + +def _make_skill_dir() -> str: + """Lay out the minimal on-disk skill a real workshop install expects.""" + root = tempfile.mkdtemp(prefix="backcompat-skill-") + locale = join(root, "locale", "en-us") + os.makedirs(locale) + with open(join(locale, INTENT_FILE), "w") as f: + f.write("\n".join(SAMPLES) + "\n") + return root + + +class BackCompatSkill(OVOSSkill): + """One intent, registered the ordinary way. + + Nothing here is version-aware. The skill is written exactly as a skill + author would write it, and the matrix observes what each workshop release + does with it. + """ + + def initialize(self): + self.register_intent_file(INTENT_FILE, self.handle_order) + + def handle_order(self, message: Message): + self.bus.emit(message.forward( + "backcompat.skill.handled", + {"topic": message.msg_type, + "skill_id": SKILL_ID, + "data": message.data})) + self.speak("ordering tacos") + + +def main(): + bus = MessageBusClient() + bus.run_in_thread() + bus.connected_event.wait(30) + + root = _make_skill_dir() + # Passing ``bus`` runs the full startup (including ``initialize``) inside + # the constructor, so there is no separate ``_startup`` call to make here. + BackCompatSkill(skill_id=SKILL_ID, bus=bus, resources_dir=root) + + # Report what this workshop actually bound, so a failing combo says which + # spellings existed rather than only that nothing fired. + bound = sorted(t for t in getattr(bus.emitter, "_events", {}) + if t.startswith(f"{SKILL_ID}:")) + print("BOUND_TOPICS " + json.dumps(bound), flush=True) + + # The versions actually resolved inside THIS venv, and whether the + # receive-side compat mirror of ovos-bus-client#271 is present. The driver + # asserts on these: the repair strategy only works if a container pinned to + # an old workshop still resolves a client new enough to carry the mirror. + print("VERSIONS " + json.dumps({ + "ovos_workshop": _dist_version("ovos-workshop"), + "ovos_bus_client": _dist_version("ovos-bus-client"), + "has_reemit_hook": hasattr(MessageBusClient, "_reemit_legacy_intent"), + }), flush=True) + print("SKILL_READY", flush=True) + + # The client already owns a reader thread from ``run_in_thread`` above; + # ``run_forever`` would try to open the same socket twice. Park instead and + # let the driver terminate the process. + while True: + time.sleep(3600) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/backcompat/test_mixed_version_matrix.py b/test/backcompat/test_mixed_version_matrix.py new file mode 100644 index 00000000000..b0271c3153b --- /dev/null +++ b/test/backcompat/test_mixed_version_matrix.py @@ -0,0 +1,399 @@ +"""Mixed-version back-compat matrix: a real bus, two venvs, four combos. + +Why this exists +--------------- + +Every in-process guard in this repo shares one interpreter, so it can only +ever observe one set of package versions. The breakage this suite is about is +*definitionally* cross-version: a skill container frozen months ago talks to a +stack that moved on. That needs two package sets alive at once, which means +two venvs and a real websocket between them. + +What moved +---------- + +OVOS-MSG-1 §2.1.1 builds the per-intent dispatch topic as +``:``. Old ``ovos-workshop`` built it from the +padatious resource **filename**, so the authoring extension leaked onto the +wire as ``:food.order.intent``. Two independent changes then landed: + +* ``ovos-padatious >= 2.0.1a1`` folds the suffixed id onto the canonical + ``:food.order`` at **registration** time, so every match — and + therefore every dispatch, since ``ovos-core`` forwards ``match.match_type`` + verbatim — is canonical; +* ``ovos-workshop >= 9.3.2a1`` binds the handler to **both** spellings, so a + skill built against it survives either dispatch. + +The interesting cell is the one where those two miss each other: a skill built +against ``ovos-workshop == 9.3.1a2`` binds the suffixed topic **only**, and a +current core dispatches the canonical topic **only**. + +The matrix +---------- + +=================== ================== ================= ============ +combo skill binds core dispatches handler runs +=================== ================== ================= ============ +old skill/old core suffixed only suffixed yes +old skill/new core suffixed only canonical **no** +new skill/old core both suffixed yes +new skill/new core both canonical yes (once) +=================== ================== ================= ============ + +Only ``old skill/new core`` is broken, and it is marked +``xfail(strict=True)``. The other three are passing controls: they prove the +harness can see a handler fire at all, so the one red cell is a real finding +and not a broken fixture. + +Two channel cells reflect that same gap against the live fleet rather than a +boundary pin: ``stable-skill/dev-core`` and ``testing-skill/dev-core`` build +the skill side straight from the OVOS distro's own constraint files (fetched +fresh in ``build_venvs.sh``), and both currently resolve an ovos-workshop +below the 9.3.2a1 canonical-binding line, so they hit the identical failure +against a dev core. ``dev-skill/stable-core`` and ``dev-skill/testing-core`` +are the passing mirror: a dev-workshop skill against either channel's +padatious, which never canonicalizes today, so the suffixed dispatch still +finds a bound handler. + +How it gets fixed, and why the fix lands where it does +------------------------------------------------------ + +The old container does **not** ship an old ``ovos-bus-client``. Its workshop +pin declares a floor, not a ceiling, so a rebuilt container resolves the +current client — this suite asserts that, because it decides WHICH of the +two stateless #271 rules repairs a given process. ``ovos-bus-client#271`` +bridges in both directions with no state: the EMITTER also sends a marked +``.intent``-suffixed twin frame for every intent topic (this is what reaches +a truly frozen image whose baked-in old client cannot be fixed from outside), +and a modern RECEIVER canonicalizes unmarked suffixed traffic locally (this +is what serves a rebuilt container whose floor-pin resolved a new client +under old workshop). Either way the old handler runs without the skill +changing at all. + +Kill-switch role +---------------- + +These four cells are a gate on the compat train, in both directions: + +* the day ``#271`` releases, ``old skill/new core`` starts passing, + ``strict=True`` turns that into a loud XPASS failure, and the marker comes + off — the guard is then permanent; +* ``ovos-workshop#500`` removes the suffixed binding, which turns + ``new skill/old core`` red. That cell must not be deleted to make the PR + green; it is the signal that the inbound direction (suffixed wire topic → + canonical local listener) is still missing from the client; +* any PR that drops the compat must flip these deliberately. + +Running it +---------- + +The combo is selected by environment, so one script serves all four CI matrix +entries:: + + BACKCOMPAT_COMBO=old-skill/new-core \\ + BACKCOMPAT_SKILL_PYTHON=/path/to/venv_skill_old/bin/python \\ + /bin/pytest test/backcompat/ + +``test/backcompat/build_venvs.sh`` builds the four venvs with the pins above. +""" +import os +import time +import uuid + +import pytest + +from ovos_bus_client.message import Message + +from .driver import (CANONICAL_TOPIC, LEGACY_TOPIC, SKILL_ID, BusServer, + Capture, SkillProcess, core_canonicalizes, dispatch, + dispatch_topic_for) + +#: What each combo is *supposed* to be, so a silent pin drift (a new workshop +#: release changing what it binds, or the padatious fold being reverted) fails +#: as a wrong-vintage error instead of quietly turning a red cell green. +#: +#: Two kinds of cell, kept in the same table on purpose: +#: +#: * boundary-pin cells (``old-*`` / ``new-*``) pin the exact releases either +#: side of a known behavior change (workshop 9.3.1a2 last-suffixed-only vs +#: 9.3.2a1 first-canonical; padatious 2.0.0a1 last-no-fold vs 2.0.1a1 +#: first-fold). They document WHERE the line is and fail as wrong-vintage if +#: a release ever moves it. +#: * channel cells (``stable-*`` / ``testing-*``) pin whatever the OVOS distro +#: constraint files resolve *today* (fetched fresh in build_venvs.sh, never +#: vendored), so they verify the fleet reality rather than a boundary. As of +#: this writing both constraints-stable.txt and constraints-testing.txt +#: floor ovos-workshop and ovos-padatious well below the canonicalization +#: boundaries above (stable: workshop>=3.4.0,<3.5.0, padatious>=1.4.2,<1.5.0; +#: testing: workshop>=7.0.6,<8.0.0, padatious>=1.4.3,<2.0.0), so both +#: channels behave like the "old" boundary side on both axes. A distro pin +#: bump past a boundary flips the affected channel cell red at the moment it +#: happens — that is the alarm this design exists for. +COMBOS = { + # combo id skill binds suffixed only, core canonicalizes, fires + "old-skill/old-core": (True, False, True), + "old-skill/new-core": (True, True, False), + "new-skill/old-core": (False, False, True), + "new-skill/new-core": (False, True, True), + # channel cells: skill or core side installed per a live distro + # constraints file, the other side at dev. See build_venvs.sh. + "stable-skill/dev-core": (True, True, False), + "dev-skill/stable-core": (False, False, True), + "testing-skill/dev-core": (True, True, False), + "dev-skill/testing-core": (False, False, True), +} + +#: Combos expected to fail today because the skill side is suffixed-only and +#: the core side canonicalizes — the same gap ``old-skill/new-core`` marks, +#: reached via a distro constraints pin instead of a boundary pin. +_BROKEN_CHANNEL_COMBOS = {"stable-skill/dev-core", "testing-skill/dev-core"} + +COMBO = os.environ.get("BACKCOMPAT_COMBO", "") +SKILL_PYTHON = os.environ.get("BACKCOMPAT_SKILL_PYTHON", "") + +pytestmark = pytest.mark.skipif( + not COMBO or not SKILL_PYTHON, + reason="mixed-version matrix needs BACKCOMPAT_COMBO and " + "BACKCOMPAT_SKILL_PYTHON; see test/backcompat/build_venvs.sh") + +#: True in every cell the compat train has to repair — the boundary-pin +#: original plus its channel-pinned reflections. Evaluated at import time so +#: the xfail below can be static and strict. +IS_BROKEN_CELL = COMBO == "old-skill/new-core" or COMBO in _BROKEN_CHANNEL_COMBOS + +_XFAIL_REASON = ( + "old skill container (ovos-workshop==9.3.1a2, suffixed binding only) does " + "not hear a canonical dispatch; needs the #271 bridge (wire twin from the " + "emitter, or local canonicalization in a modern client), unreleased. " + "XPASS here means #271 shipped — drop this marker and keep the guard." + if COMBO not in _BROKEN_CHANNEL_COMBOS else + f"{COMBO}: the OVOS distro constraints file pins an ovos-workshop below " + "the 9.3.2a1 canonical-binding boundary, so this channel's skill side is " + "suffixed-only against a dev core that canonicalizes at registration; " + "same gap as old-skill/new-core, reached via a live fleet pin instead of " + "a boundary pin. XPASS here means either the channel moved its pin past " + "the boundary or #271 shipped — check which, then drop the marker.") + + +@pytest.fixture(scope="module") +def stack(): + """One bus and one skill process, shared by the read-only observations.""" + if COMBO not in COMBOS: + pytest.fail(f"unknown BACKCOMPAT_COMBO {COMBO!r}; " + f"expected one of {sorted(COMBOS)}") + server = BusServer() + skill = None + try: + bus = server.client() + # Capture registration before the skill exists, so the name the core + # side would key its engine entry on is observed, never assumed. + registrations = Capture(bus, "padatious:register_intent") + skill = SkillProcess(SKILL_PYTHON, server.xdg) + registrations.wait(30) + yield server, bus, skill, registrations + finally: + if skill is not None: + skill.stop() + server.stop() + + +def _registered_name(registrations) -> str: + names = [m.data.get("name") for m in registrations.messages + if str(m.data.get("name", "")).startswith(f"{SKILL_ID}:")] + assert names, ("the skill never registered an intent with the padatious " + "contract; nothing to dispatch") + return names[0] + + +def test_pins_are_the_intended_vintage(stack): + """Fail loudly if either side is not the version this combo assumes. + + Without this, a workshop release that quietly changes what it binds would + turn the broken cell green and be read as "the compat train landed". + """ + _server, _bus, skill, _regs = stack + want_suffixed_only, want_canon, _ = COMBOS[COMBO] + + suffixed_only = skill.bound_topics == [LEGACY_TOPIC] + assert suffixed_only == want_suffixed_only, ( + f"{COMBO}: expected suffixed-only binding={want_suffixed_only}, but " + f"the skill venv bound {skill.bound_topics}") + if not want_suffixed_only: + assert CANONICAL_TOPIC in skill.bound_topics + assert core_canonicalizes() == want_canon, ( + f"{COMBO}: expected the core venv to canonicalize at registration=" + f"{want_canon}, got {core_canonicalizes()}") + + +#: Channel cells pin the whole stack from a distro constraints file, which +#: caps ovos-bus-client too — unlike the boundary cells, where only +#: ovos-workshop is pinned and its dependency floor is left to resolve +#: whatever is current. The "frozen container still gets a modern client" +#: assumption below is therefore a boundary-cell property, not a channel-cell +#: one, and asserting it on a channel combo would just report a real fleet +#: fact (the channel's own bus-client ceiling) as if it were a broken +#: assumption. +_CHANNEL_COMBOS = {"stable-skill/dev-core", "dev-skill/stable-core", + "testing-skill/dev-core", "dev-skill/testing-core"} + + +def test_old_container_resolves_a_current_bus_client(stack): + """The repair strategy assumes the frozen container gets a modern client. + + ``ovos-workshop``'s dependency floor is a lower bound, so even a pinned + old workshop resolves today's ``ovos-bus-client``. The assertion pins + which #271 rule serves this cell: a modern client in the old process + canonicalizes inbound; were the client ever frozen too, only the + emitter-side wire twin could reach it — so the resolution is asserted, + not assumed, and the cell documents which rule it exercises. + + Channel combos are exempted from the assertion (not skipped outright, + so the version is still recorded): a distro constraints file pins + ovos-bus-client alongside everything else, so a low client version there + is the fleet's own ceiling, not evidence the repair design is broken. A + real stable/testing-channel container keeps an old client, so it is + served by #271's emitter-side wire twin rather than local + canonicalization; that is a fleet-inventory note, not a compat-design + failure. + """ + _server, _bus, skill, _regs = stack + client = skill.versions.get("ovos_bus_client", "") + assert client, f"the skill venv did not report its versions:\n{skill.log}" + if COMBO not in _CHANNEL_COMBOS: + assert int(client.split(".")[0]) >= 2, ( + f"the skill venv resolved ovos-bus-client {client}; this cell " + f"is meant to exercise #271's local canonicalization rule, " + f"which needs a modern client in the skill process") + elif int(client.split(".")[0]) < 2: + print(f"{COMBO}: channel bus-client ceiling is {client} (<2); this " + f"channel is served by the emitter-side wire twin, not local " + f"too, not just a workshop bump, to receive the compat fix") + + # Recorded, not asserted: this is the switch the xfail above waits for. + # When it turns True the broken cell starts passing and strict xfail + # converts that into the loud signal to promote the guard. + print(f"{COMBO}: skill venv = ovos-workshop " + f"{skill.versions.get('ovos_workshop')}, ovos-bus-client {client}, " + f"#271 mirror present={skill.versions.get('has_reemit_hook')}") + + +def test_core_dispatches_the_topic_this_combo_expects(stack): + """Positive control on the core half, independent of any handler. + + Proves the dispatch really went out and with which spelling, so a silent + handler in the broken cell can only mean the handler never heard it. + """ + _server, bus, _skill, regs = stack + topic = dispatch_topic_for(_registered_name(regs)) + want = CANONICAL_TOPIC if core_canonicalizes() else LEGACY_TOPIC + assert topic == want + + seen = Capture(bus, topic) + try: + dispatch(bus, topic, food="tacos") + assert seen.wait(), f"the dispatch on {topic!r} never reached the bus" + finally: + seen.close() + + +@pytest.mark.xfail(IS_BROKEN_CELL, strict=True, reason=_XFAIL_REASON) +def test_the_skill_handler_runs(stack): + """The contract: a canonical dispatch must reach the skill's handler. + + Emitted exactly the way ``IntentService._dispatch_match`` does — the + matched topic, forwarded with the utterance data — and observed through + the marker the skill emits from inside the handler. + """ + _server, bus, skill, regs = stack + topic = dispatch_topic_for(_registered_name(regs)) + + token = uuid.uuid4().hex + handled = Capture(bus, "backcompat.skill.handled", token=token) + spoken = Capture(bus, "speak") + try: + bus.emit(Message(topic, {"food": "tacos", "token": token}, + {"session": {"session_id": "backcompat"}})) + assert handled.wait(), ( + f"{COMBO}: dispatched {topic!r} but the skill handler never ran.\n" + f"the skill venv bound: {skill.bound_topics}\n" + f"skill process log:\n{skill.log}") + assert handled.messages[0].data["topic"] in (topic, LEGACY_TOPIC) + assert handled.messages[0].data["data"]["food"] == "tacos" + # ovos-core on the stable/testing channels is old enough (<=2.1.x) + # that its dialog/TTS plumbing around a bare ``self.speak()`` differs + # from what this minimal harness sets up for the "new" core checkout, + # independently of the intent-topic aliasing this suite exists to + # test. The handler firing (asserted above) is the compat-relevant + # fact; whether it also spoke is a realism check that only applies + # where the core side is current. + if "-core" not in COMBO or not COMBO.endswith(("stable-core", + "testing-core")): + assert spoken.wait(), "the handler ran but never spoke" + finally: + handled.close() + spoken.close() + + +@pytest.mark.xfail(IS_BROKEN_CELL, strict=True, reason=_XFAIL_REASON) +def test_the_handler_runs_exactly_once(stack): + """A skill bound to both spellings must not answer twice. + + ``new skill/new core`` is where this bites: the skill binds the canonical + and the suffixed topic, so once the compat mirror of ``#271`` is live the + same dispatch could reach the same handler down two paths. One utterance, + one answer. + """ + _server, bus, skill, regs = stack + topic = dispatch_topic_for(_registered_name(regs)) + + token = uuid.uuid4().hex + handled = Capture(bus, "backcompat.skill.handled", token=token) + try: + bus.emit(Message(topic, {"food": "burritos", "token": token}, + {"session": {"session_id": "backcompat-once"}})) + assert handled.wait(), f"{COMBO}: handler never ran for {topic!r}" + # Keep listening past the first hit: a duplicate arrives late, so an + # assertion taken the instant the first one lands would never see it. + time.sleep(3) + assert len(handled.messages) == 1, ( + f"{COMBO}: one dispatch on {topic!r} produced " + f"{len(handled.messages)} handler runs " + f"(skill bound {skill.bound_topics})") + finally: + handled.close() + + +@pytest.mark.skipif(not IS_BROKEN_CELL, + reason="the kill switch only has meaning where the " + "compat mirror is what makes the handler run") +def test_kill_switch_disables_the_compat_mirror(): + """Inverse control: with ``emit_legacy`` off, the old handler stays silent. + + This is what proves the broken cell above measures the compat bridge and + not some accident of the fixture. It passes today for the trivial reason + that no mirror exists; once ``#271`` ships it becomes the real negative + half of the pair, and a compat-drop PR has to flip the cell above while + leaving this one green. + """ + server = BusServer() + skill = None + try: + bus = server.client() + skill = SkillProcess(SKILL_PYTHON, server.xdg, emit_legacy=False) + token = uuid.uuid4().hex + handled = Capture(bus, "backcompat.skill.handled", token=token) + canonical = Capture(bus, CANONICAL_TOPIC, token=token) + try: + bus.emit(Message(CANONICAL_TOPIC, {"food": "tacos", "token": token}, + {"session": {"session_id": "backcompat-off"}})) + assert canonical.wait(), "the dispatch never reached the bus" + assert not handled.wait(5), ( + "the old handler ran with emit_legacy disabled — the compat " + "mirror is not actually gated by the kill switch") + finally: + handled.close() + canonical.close() + finally: + if skill is not None: + skill.stop() + server.stop() diff --git a/test/end2end/test_intent_legacy_reemit_guard.py b/test/end2end/test_intent_legacy_reemit_guard.py new file mode 100644 index 00000000000..a88de89bc54 --- /dev/null +++ b/test/end2end/test_intent_legacy_reemit_guard.py @@ -0,0 +1,189 @@ +"""Loud guard for the intent-topic legacy RE-EMIT contract (send side). + +Relocated from ``ovoscope#127`` at the maintainer's request: ovoscope is the +test-harness library, not the stack, and a stack contract has to be pinned +where the stack lives. ``test_intent_alias_backcompat.py`` in this same +directory already pins the *registration* half of the migration (a legacy +``.intent``-suffixed id still resolves at match/blacklist time, and — per +ovos-workshop#497 — a legacy-registered handler still fires). This module +pins the other half: the bus-level RE-EMIT of the suffixed twin when a +CANONICAL intent is dispatched, for an old containerized skill that only +ever subscribed on the bus (never registered through this core), so it +never appears in any alias registry. + +That re-emit is NOT implemented yet anywhere in the stack. It needs, in +order: + +* ``ovos-spec-tools`` 1.6.0a1 — the pure ``canonical_intent_topic`` / + ``legacy_intent_topic`` helpers (the alias registry was dropped in + ovos-spec-tools#92: the compat is two stateless rules now); +* ``bus-client#271`` — the client-side hook the wiring above needs; +* ``ovos-utils#411`` — ``FakeBus`` support for the same hook, so this test + can exercise it at all without a live MQ. + +Every assertion that needs that train is marked +``@pytest.mark.xfail(strict=True, ...)`` so: + +* the suite is green right now (the missing behavior is an *expected* + failure), and +* the moment the train lands, the assertion starts passing, ``strict=True`` + turns that XPASS into a hard failure, and that loud failure is the signal + to drop the xfail marker and promote the test into a permanent compat + guard for the contract it pins. + +The positive/negative pair uses a real ``MiniCroft`` boot (matching the +``End2EndTest`` style of the sibling module) so the guard exercises the +actual bus dispatch path, not a mock of it. +""" +import time +from unittest import TestCase + +import pytest +from ovos_bus_client.message import Message +from ovos_utils.fakebus import FakeBus +from ovos_utils.log import LOG +from ovos_workshop.skills.ovos import OVOSSkill + +from ovoscope import get_minicroft + +_XFAIL_REASON = ("pending intent-topic compat train: spec-tools#88 + " + "bus-client#271 + ovos-utils#411") + +SKILL_ID = "ovos-core-intent-legacy-reemit-guard.test" +INTENT_NAME = "LegacyReemit" +CANONICAL_TOPIC = f"{SKILL_ID}:{INTENT_NAME}" +LEGACY_TOPIC = f"{CANONICAL_TOPIC}.intent" + + +class _LegacyBusOnlySkill(OVOSSkill): + """Stand-in for an old, un-migrated containerized skill: it never + registers through this core at all, it just subscribes directly to the + legacy suffixed bus topic the way a pre-INTENT-4 workshop dispatched. + Wired with ``add_event`` (skipping the padatious resource file) so the + fixture stays a MiniCroft boot, not a full skill-resource fixture.""" + + def initialize(self): + self.add_event(LEGACY_TOPIC, self.handle_legacy, + 'mycroft.skill.handler', activation=True, + is_intent=True) + + def handle_legacy(self, message: Message): + pass + + +def _enable_emit_legacy(): + """Best-effort toggle for the (not-yet-existing) ``emit_legacy`` compat + knob. Until the compat train ships this is a no-op — the re-emitted twin + below simply never appears, which is exactly what the xfail below pins.""" + try: + from ovos_config.config import Configuration + Configuration().setdefault("intent_topic_compat", {})["emit_legacy"] = True + except Exception: + pass + + +def _disable_emit_legacy(): + try: + from ovos_config.config import Configuration + Configuration().setdefault("intent_topic_compat", {})["emit_legacy"] = False + except Exception: + pass + + +class TestIntentLegacyReemitGuard(TestCase): + """One MiniCroft boot, shared by all methods — a legacy bus-only + consumer skill is loaded once and each test observes the bus around a + canonical dispatch.""" + + @classmethod + def setUpClass(cls): + LOG.set_level("ERROR") + cls.mc = get_minicroft([SKILL_ID], + extra_skills={SKILL_ID: _LegacyBusOnlySkill}) + + @classmethod + def tearDownClass(cls): + cls.mc.stop() + LOG.set_level("CRITICAL") + + def test_direct_legacy_dispatch_still_fires_the_handler(self): + """Positive control: today's plumbing already delivers a message + emitted straight onto the suffixed topic to a legacy consumer + listening on it — no compat train needed for this, it is the plain + current bus wiring, unrelated to re-emission.""" + hits = [] + self.mc.bus.on(LEGACY_TOPIC, hits.append) + try: + self.mc.bus.emit(Message(LEGACY_TOPIC, {"food": "tacos"})) + time.sleep(0.3) + self.assertEqual(len(hits), 1) + self.assertEqual(hits[0].data, {"food": "tacos"}) + finally: + self.mc.bus.remove(LEGACY_TOPIC, hits.append) + + @pytest.mark.xfail(strict=True, reason=_XFAIL_REASON) + def test_legacy_twin_reemitted_when_compat_enabled(self): + """A canonical dispatch must re-emit the suffixed ``.intent`` twin + for the still-listening legacy consumer, carrying identical data + and context, exactly once.""" + _enable_emit_legacy() + twin_hits = [] + self.mc.bus.on(LEGACY_TOPIC, twin_hits.append) + try: + msg = Message(CANONICAL_TOPIC, {"food": "tacos"}, + {"session": "legacy-twin-session"}) + self.mc.bus.emit(msg) + time.sleep(0.5) + self.assertEqual(len(twin_hits), 1, + f"expected exactly one re-emitted legacy twin " + f"on {LEGACY_TOPIC!r}, got {len(twin_hits)}") + self.assertEqual(twin_hits[0].data, msg.data) + self.assertEqual(twin_hits[0].context.get("session"), + "legacy-twin-session") + finally: + self.mc.bus.remove(LEGACY_TOPIC, twin_hits.append) + _disable_emit_legacy() + + def test_no_twin_when_compat_disabled(self): + """Paired negative control for the case above: with the compat knob + off (its default — nothing implements the re-emit yet), only the + canonical topic is observed. The canonical listener firing is the + positive control proving the dispatch actually happened; the + suffixed twin must NOT appear.""" + _disable_emit_legacy() + canonical_hits = [] + twin_hits = [] + self.mc.bus.on(CANONICAL_TOPIC, canonical_hits.append) + self.mc.bus.on(LEGACY_TOPIC, twin_hits.append) + try: + msg = Message(CANONICAL_TOPIC, {"food": "burritos"}) + self.mc.bus.emit(msg) + time.sleep(0.3) + self.assertEqual(len(canonical_hits), 1, + "canonical dispatch did not arrive") + self.assertEqual(twin_hits, [], + "a legacy .intent twin was emitted even though " + "compat is off") + finally: + self.mc.bus.remove(CANONICAL_TOPIC, canonical_hits.append) + self.mc.bus.remove(LEGACY_TOPIC, twin_hits.append) + + @pytest.mark.xfail(strict=True, reason=_XFAIL_REASON) + def test_fakebus_supports_emit_legacy_wiring(self): + """``FakeBus`` (ovos-utils#411) must expose the same re-emit hook the + real bus client gets from bus-client#271, so end-to-end intent-compat + tests can run against FakeBus without a live MQ.""" + bus = FakeBus() + twin_hits = [] + bus.on(LEGACY_TOPIC, twin_hits.append) + try: + bus.emit(Message(CANONICAL_TOPIC, {"x": 1}), + intent_topic_compat={"emit_legacy": True}) + self.assertEqual(len(twin_hits), 1) + finally: + bus.remove(LEGACY_TOPIC, twin_hits.append) + + +if __name__ == "__main__": + import unittest + unittest.main()