diff --git a/.agents/skills/add-ros2-package/assets/package_template/setup.py b/.agents/skills/add-ros2-package/assets/package_template/setup.py index 4056e5d5f..3982cc356 100644 --- a/.agents/skills/add-ros2-package/assets/package_template/setup.py +++ b/.agents/skills/add-ros2-package/assets/package_template/setup.py @@ -26,7 +26,9 @@ maintainer_email='your.email@example.com', # TODO: Update description='Brief description of your module', # TODO: Update license='Apache-2.0', - tests_require=['pytest'], + extras_require={ + 'test': ['pytest'], + }, entry_points={ 'console_scripts': [ # TODO: Add your node executables here diff --git a/.agents/skills/add-unit-tests/SKILL.md b/.agents/skills/add-unit-tests/SKILL.md new file mode 100644 index 000000000..27aeb3a87 --- /dev/null +++ b/.agents/skills/add-unit-tests/SKILL.md @@ -0,0 +1,304 @@ +--- +name: add-unit-tests +description: Add Python or C++ unit tests to an AirStack ROS 2 package. Covers the co-location pattern (test source in package/test/), registering the package in colcon_unit_test_packages.yaml so pytest tests/ and airstack test -m unit collect it, and how to extend to sim components. +license: MIT +metadata: + author: AirLab CMU + repository: AirStack +--- + +# Skill: Add Unit Tests to an AirStack Module + +## When to Use + +Use this skill when: + +- Adding Python unit tests for a ROS 2 package (perception, sensors, local, global, behavior, interface) +- Adding C++ unit tests (`gtest`) to a package already using `ament_cmake` +- Extending unit tests to sim-side Python (`simulation/**//test/`) +- Verifying that `airstack test -m unit` picks up your new tests + +For system tests (full Docker stack, sim, sensors, takeoff/hover/land) see the +`run-system-tests` skill instead. + +## Architecture Overview + +Unit test **source lives co-located with its package** (ROS 2 / colcon convention). +`tests/colcon_unit_test_packages.yaml` lists which packages have unit tests, and the root +harness collects them from there — you only edit files under the package itself. + +``` +robot/ros_ws/src/// +├── src/ # production source (Python or C++) +├── test/ +│ ├── test_.py # ← unit test SOURCE (collected directly) +│ ├── test_.cpp # ← C++ gtest SOURCE (optional) +│ └── fake_.hpp # ← C++ test doubles (optional) +└── CMakeLists.txt # wires ament_add_gtest under BUILD_TESTING + +tests/colcon_unit_test_packages.yaml # ← list the package here (single source of truth) +``` + +`tests/conftest.py` reads the YAML, resolves each listed package to its `test/` dir, +and injects the non-linter `test_*.py` files into collection under +`--import-mode=importlib` (set in `tests/pytest.ini`). Each collected item is +auto-tagged `@pytest.mark.unit` by path, so `-m unit` selects it. ament lint files +(`test_copyright.py`, etc.) are excluded — they run under `colcon test`. This means: + +| Invocation | What runs | +|---|---| +| `airstack test -m unit` | Package `test/test_*.py`, collected directly from source | +| `cd tests && pytest -m unit` | Same path — the containerless equivalent | +| `pytest tests/ -m unit` | Same path — what CI runs | +| `colcon test --packages-select ` | C++ gtests and linters; Python only for `ament_python` packages (see below) | + +**Two runners, split by language — because C++ needs a build and Python does not.** A +gtest is a binary: it must be compiled against the package's headers and rclcpp, so it can +only run where the ROS toolchain is. That is `colcon test` inside the robot container, +which CI reaches via the **`build_packages`** mark +(`tests/system/test_build_packages.py::test_colcon_test_robot`, which builds with +`-DBUILD_TESTING=ON` first). Python unit tests are deliberately hermetic — they stub ROS +at the import boundary and touch no ROS runtime — so they need no build and no container, +which is what lets the root harness run all of them in about a second. + +Preserve that property when adding tests: a Python test that needs a live ROS node belongs +in `tests/integration/` or `tests/system/`, not here. + +Whether `colcon test` *also* picks up a package's Python tests depends on its build type: + +| Package | Build type | Python tests under `colcon test` | +|---|---|---| +| `natnet_ros2` | `ament_cmake` | **No** — `CMakeLists.txt` registers `ament_add_gtest` but no `ament_add_pytest_test` | +| `lidar_point_cloud_filter` | `ament_python` | **Yes** — `setup.cfg` sets `testpaths = test`, so colcon's pytest runner finds them | + +So a Python test in an `ament_cmake` package runs *only* via the root harness — which is +fine, since that is what CI invokes. + +Naming a path *below* `tests/` narrows the run and skips the injection, so +`pytest tests/system/test_x.py` stays fast and does not drag in unit tests. The rule lives +in `harness.discovery.collection_is_broad` and is pinned by +`tests/meta/test_collection_contract.py`. + +## Step-by-Step: Adding a Python Unit Test + +### 1. Identify pure-Python logic to test + +Good candidates are functions/classes with **no ROS or hardware dependencies**: +- Pure math / geometry helpers +- Protocol parsers +- Data-structure converters +- Any function that takes plain Python types and returns plain Python types + +If the code imports ROS types, stub them out at the import boundary +(see `test_natnet_ros2.py` for the `sys.modules` stub pattern). + +### 2. Write the test source in the package + +Create `robot/ros_ws/src///test/test_.py`: + +```python +# Copyright (c) 2024 Carnegie Mellon University +# MIT License - see LICENSE in the repository root for full text. +"""Unit tests for — .""" + +import sys +from pathlib import Path +import pytest + +# Add the package src/ dir so the production module is importable +# without colcon installing the package first. +_src = Path(__file__).resolve().parent.parent / "src" +if str(_src) not in sys.path: + sys.path.insert(0, str(_src)) + +from my_module import my_function # noqa: E402 + + +def test_my_function_basic(): + assert my_function(1, 2) == 3 +``` + +**Key points:** +- **Do not write `@pytest.mark.unit`.** `pytest_itemcollected` in `tests/conftest.py` + applies it by file location to everything under a registered package's `test/` dir. + Writing it by hand is redundant, and it warns (`PytestUnknownMarkWarning`) under any + invocation where `tests/pytest.ini` is not the configfile — e.g. `colcon test`. +- Import `pytest` only if you need its API (`approx`, `raises`, `parametrize`, + `importorskip`). +- Compute paths relative to `__file__` (`parent.parent / "src"`) — never hardcode + absolute paths. +- For packages with a Python module directory (`//`), add the package + root (`parent.parent`) to `sys.path` and import as + `from . import ...`. +- If the code uses ROS types, stub `sys.modules` before importing: + +```python +import sys +from unittest.mock import MagicMock + +sys.modules.setdefault("rclpy", MagicMock()) +sys.modules.setdefault("rclpy.node", MagicMock()) +sys.modules.setdefault("geometry_msgs", MagicMock()) +sys.modules.setdefault("geometry_msgs.msg", MagicMock()) +# ... then import your module +``` + +For `rclpy.node.Node` subclasses use a real dummy base class instead of a +`MagicMock()` to ensure `__init_subclass__` fires and method bodies are defined +(see `test_natnet_ros2.py` for the full pattern). + +### 3. Register the package in colcon_unit_test_packages.yaml + +If the package isn't already listed, add it under the `robot` workspace in +[`tests/colcon_unit_test_packages.yaml`](../../../tests/colcon_unit_test_packages.yaml): + +```yaml +robot: + packages: + - natnet_ros2 + - lidar_point_cloud_filter + - # ← add here + pytest_args: [] +``` + +Leave `pytest_args` empty. It is forwarded to `colcon test` via `PYTEST_ADDOPTS`, and +ament's pytest runner ignores `-m` there — a marker expression in this field silently +does nothing. + +That's the whole registration. `conftest.py` globs +`robot/ros_ws/src/**//test`, collects its non-linter `test_*.py`, and marks +them `unit`. The test file must be self-contained: if it imports package code, set up +`sys.path` at the top of the test file (see `test_validation_core.py`, which inserts its +package root). Same YAML, different workspace key (`sim:`), for Isaac-extension unit tests. + +### 4. Run locally to verify + +```bash +airstack test -m unit -v +# or, containerless: +AIRSTACK_ROOT=$(pwd) pytest tests/ -m unit -v +``` + +All 155 existing tests plus your new ones should pass. Collected items point straight +at the co-located source: +``` +../robot/ros_ws/src///test/test_.py::test_my_function_basic PASSED +``` + +### 5. Running in CI + +`unit-tests.yml` invokes `pytest tests/ -m unit` on GitHub-hosted `ubuntu-latest` +whenever a PR targeting `main` or `develop` is opened, synchronized, or reopened. +It does not consume an OSMO GPU. +C++ gtests still run through the OSMO `build_packages` mark because they require the +ROS workspace and toolchain inside the robot container. + +--- + +## Step-by-Step: Adding a C++ gtest + +C++ tests live entirely within the package and run exclusively via `colcon test`. + +### 1. Write the test in `package/test/` + +```cpp +// Copyright (c) 2024 Carnegie Mellon University +// MIT License - see LICENSE in the repository root for full text. +#include +#include "my_package/my_header.hpp" + +TEST(MyGroup, BasicCase) { + EXPECT_EQ(my_function(1, 2), 3); +} +``` + +### 2. Wire `ament_add_gtest` in `CMakeLists.txt` + +```cmake +if(BUILD_TESTING) + find_package(ament_lint_auto REQUIRED) + ament_lint_auto_find_test_dependencies() + + find_package(ament_cmake_gtest REQUIRED) + ament_add_gtest(test_my_name test/test_my_name.cpp) + target_include_directories(test_my_name PRIVATE + $ + $) + # Link any production library targets here if needed: + # target_link_libraries(test_my_name my_lib) +endif() +``` + +### 3. Add test depend in `package.xml` + +```xml +ament_cmake_gtest +``` + +### 4. Build and run + +```bash +# Inside the robot container: +docker exec airstack-robot-desktop-1 bash -c \ + "bws --cmake-args '-DBUILD_TESTING=ON' --packages-select " +docker exec airstack-robot-desktop-1 bash -c \ + "colcon test --packages-select --event-handlers console_direct+" +docker exec airstack-robot-desktop-1 bash -c \ + "colcon test-result --all" +``` + +The `build_packages` system test in CI (`tests/system/test_build_packages.py`) also +runs `colcon test` with `BUILD_TESTING=ON` for the robot container. Packages gated +there are listed in [`tests/colcon_unit_test_packages.yaml`](../../../tests/colcon_unit_test_packages.yaml) +— add your package under `robot.packages` when it has gtests or pytest tests in +`package/test/`. + +--- + +## Extending to sim and GCS + +The same mechanism applies — add the package under a workspace key in the YAML. The +workspace→source glob is defined in `tests/harness/discovery.py` (`_WORKSPACE_PKG_TEST_GLOBS`): `robot` → +`robot/ros_ws/src/**//test`, `sim` → `simulation/**//test`. Add a new workspace +key there (e.g. `gcs`) if you extend to a new tree. + +```yaml +# tests/colcon_unit_test_packages.yaml +sim: + packages: + - # → simulation/**//test collected directly +``` + +--- + +## Pattern Summary + +| Concern | Answer | +|---|---| +| Where does test source live? | `/…//test/` (co-located with the package) | +| Where does pytest discover tests? | From the package `test/` dir listed in `colcon_unit_test_packages.yaml` | +| How are duplicate basenames handled? | `--import-mode=importlib` (set in `pytest.ini`) | +| What mark do all unit tests use? | `@pytest.mark.unit` — auto-applied by path in `conftest.py`; do not write it yourself | +| How do I run them? | `airstack test -m unit`, `cd tests && pytest -m unit`, or `pytest tests/ -m unit` | +| What CI workflow runs them? | Python: `unit-tests.yml`; C++: the `build_packages` path in `system-tests.yml` — see §5 | +| Do system tests (`liveliness`, etc.) run too? | No — `-m unit` filters to hermetic tests only | +| Does `colcon test` also run these? | Only if the package registers them. `ament_add_gtest` covers C++; a Python test needs `ament_add_pytest_test`, which `natnet_ros2` does **not** have — its Python tests run only under the root harness | +| Can I add pure C++ gtests? | Yes — `ament_add_gtest` in CMakeLists.txt | + +## Reference Implementations + +| Package | Python test | What it covers | +|---|---|---| +| `natnet_ros2` | `robot/ros_ws/src/perception/natnet_ros2/test/test_natnet_ros2.py` | `VisionPoseConverterNode._canonical_quaternion` (ROS-stubbed) | +| `natnet_ros2` (C++) | `robot/ros_ws/src/perception/natnet_ros2/test/test_natnet_logic.cpp` | `build_covariance_6x6`, `negotiate()`, `INatNetClient` seam | +| `lidar_point_cloud_filter` | `robot/ros_ws/src/sensors/lidar_point_cloud_filter/test/test_validation_core.py` | Pure-numpy range validation rules | + +Both are collected from their package `test/` dir. + +## Files to Know + +- `.airstack/modules/dev.sh` — what `airstack test` runs (bare `pytest` with `working_dir` `tests/`) +- `tests/pytest.ini` — mark registration + `--import-mode=importlib` + `testpaths` +- `tests/colcon_unit_test_packages.yaml` — the package list driving unit-test collection +- `tests/conftest.py` — `unit_test_files()` / `pytest_configure` inject package tests; `pytest_itemcollected` auto-marks `unit` +- `tests/README.md` — full test harness reference diff --git a/.agents/skills/bump-version-and-release/SKILL.md b/.agents/skills/bump-version-and-release/SKILL.md index 3c056b774..a4a74c77e 100644 --- a/.agents/skills/bump-version-and-release/SKILL.md +++ b/.agents/skills/bump-version-and-release/SKILL.md @@ -64,11 +64,13 @@ Three workflows in `.github/workflows/` interact with `VERSION`: - **Trigger:** push to `main` or `develop` whose changed paths include `.env`, **and** the `VERSION=` line in `.env` differs from the previous commit. Also runs on manual `workflow_dispatch`. - **Behavior on tag change:** 1. Runs on a self-hosted ephemeral GPU runner (`[self-hosted, airstack-ephemeral]`). - 2. `docker compose build` for profiles `desktop,isaac-sim,ms-airsim`. - 3. `docker compose push` to `${PROJECT_DOCKER_REGISTRY}` (set in `.env` — currently `airlab-docker.andrew.cmu.edu/airstack`). - 4. Keyless `cosign sign` of every pushed image digest via GitHub OIDC. - 5. `cosign verify` against the workflow's certificate identity. + 2. Plans per service via `.github/workflows/scripts/docker_image_plan.py` (content fingerprint vs previous versioned image label). + 3. **Unchanged image inputs** → registry retag of the previous `v${PREV}_…` digest to `v${VERSION}_…` and `cache_*` (no rebuild). + 4. **Changed inputs** (or missing/unlabeled previous image, or `force_rebuild=true`) → `docker compose build` / `push` for those services only, labeling the new digest with `org.airstack.content-fingerprint`. + 5. Keyless `cosign sign` of every published image digest via GitHub OIDC. + 6. `cosign verify` against the workflow's certificate identity. - **Skip behavior:** if the merge commit on `main`/`develop` does not actually change `VERSION=`, the build job is skipped (the check-changes job sets `tag-changed=false`). +- **Docs-only VERSION bumps:** still required by `check-version-increment`, but publish should retag rather than rebuild once fingerprints are on the previous images. First publish after this feature lands (or `force_rebuild=true`) must rebuild to write the labels. ### 3. `deploy_docs_from_release.yaml` — versioned docs @@ -76,7 +78,7 @@ Three workflows in `.github/workflows/` interact with `VERSION`: - **Behavior:** runs `mike deploy --push --update-aliases latest`, publishing the docs site under the release tag and pointing the `latest` alias at it. - Companion workflows publish unversioned docs from `main` (default alias `main`) and `develop` (alias `develop`). -So the full release path is: bump `VERSION` → PR → merge to `main`/`develop` (rebuild + push + sign) → cut a GitHub Release matching that VERSION (versioned docs go live). +So the full release path is: bump `VERSION` → PR → merge to `main`/`develop` (retag unchanged images and/or rebuild changed ones + push + sign) → cut a GitHub Release matching that VERSION (versioned docs go live). ## Choosing the Bump Type @@ -258,5 +260,5 @@ For a true release (dropping the pre-release suffix): ## Related Skills -- [`run-system-tests`](../run-system-tests) — what fires on every PR alongside the version check +- [`run-system-tests`](../run-system-tests) — automatic unit/package gates and how to request simulation campaigns - [`update-documentation`](../update-documentation) — for docs-only PRs that may still need a VERSION bump to clear the gate diff --git a/.agents/skills/configure-multi-robot/SKILL.md b/.agents/skills/configure-multi-robot/SKILL.md index 4fc977472..a6ffacef1 100644 --- a/.agents/skills/configure-multi-robot/SKILL.md +++ b/.agents/skills/configure-multi-robot/SKILL.md @@ -42,6 +42,9 @@ docker-compose.yaml (ROBOT_NAME_SOURCE=container_name | hostname, │ ▼ robot/docker/.bashrc (runs on container shell start) + │ + ├─ ROBOT_NAME already set in env? → KEEP IT, skip resolution entirely + │ (guard: `if [ -z "${ROBOT_NAME:-}" ]`; lets an override/compose pin the name) │ ├─ ROBOT_NAME_SOURCE=container_name → resolve `hostname` back to docker container name │ (e.g. `airstack-robot-desktop-1`) @@ -67,7 +70,7 @@ The default mapping rule in [`robot/docker/robot_name_map/default_robot_name_map robot: 'robot_{1}' domain_id: '{1}' - pattern: '.*' # catch-all - robot: 'unknown-robot' + robot: 'unknown_robot' # must be a valid ROS token (no hyphen) or launch fails domain_id: '0' ``` @@ -97,9 +100,38 @@ docker exec airstack-robot-desktop-1 bash -c 'echo $ROBOT_NAME $ROS_DOMAIN_ID' # robot_1 1 ``` -If you need a non-default name (custom hostname scheme on a physical robot, or you want `drone_alpha` instead of `robot_1`), write a new mapping YAML in `robot/docker/robot_name_map/` and point `ROBOT_NAME_MAP_CONFIG_FILE` at it. Do **not** hardcode `ROBOT_NAME=...` in compose unless you know what you are doing — it bypasses the resolver and you lose `ROS_DOMAIN_ID` co-assignment. +If you need a non-default name (custom hostname scheme on a physical robot, or you want `drone_alpha` instead of `robot_1`), you have two options: + +1. **Write a mapping YAML** in `robot/docker/robot_name_map/` and point `ROBOT_NAME_MAP_CONFIG_FILE` at it. Preferred when the name should be derived from the machine (hostname/container) — keeps the resolver in charge of `ROS_DOMAIN_ID` co-assignment. +2. **Rename the device** so the default map resolves it. On real hardware + (`ROBOT_NAME_SOURCE=hostname`) the OS hostname *is* the identity, so + `hostnamectl set-hostname robot-1` is a complete, one-time fix — and it scales to a + fleet, since `robot-2` and `robot-3` then resolve on their own. + +!!! danger "Setting `ROBOT_NAME` in an env file does nothing" + No compose service declares `ROBOT_NAME` or `ROS_DOMAIN_ID` in its `environment:` + block, and Docker Compose only injects a variable into a container if some service + names it there. Putting `ROBOT_NAME=robot_1` in an override `.env` sets it for + **compose's own interpolation**, not for the container — `.bashrc` sees it unset, + the map lookup runs anyway, and there is no error. The robot simply comes up under + the resolved name instead of yours. + + `overrides/l4t-px4-realrobot.env` used to ship `ROBOT_NAME` / `ROS_DOMAIN_ID` on + this basis; they never had any effect and have been removed. Use a hostname or a + map file instead. -For a one-off override (e.g. ad hoc debugging): + The general lesson applies to **any** deployment knob: it needs a declaration in + the service's `environment:` *and* a consumer that reads it. Always + [verify](#verification-commands) rather than assuming. + +**Never hardcode `ROBOT_NAME` on a service in compose either.** `robot-desktop` and +friends are reused for every replica, so a pinned name there would collapse all robots +onto one name and domain and silently break multi-robot. Identity must come from +something that differs per container — the container name in sim, the device hostname on +real hardware — or from a map rule that derives it. + +For a one-off override (e.g. ad hoc debugging), pass it to the shell directly, which +does work because `docker exec -e` sets it in the process environment: ```bash docker exec -e ROBOT_NAME=robot_5 -e ROS_DOMAIN_ID=5 -it airstack-robot-desktop-1 bash @@ -119,7 +151,7 @@ robot-desktop: So `NUM_ROBOTS=3 airstack up` produces **three** robot containers (`airstack-robot-desktop-1`, `-2`, `-3`), each with its own `ROBOT_NAME` and its own `ROS_DOMAIN_ID`. Each container runs the full autonomy stack independently. Cross-robot communication, when needed, goes through the DDS router (see [`onboard_all/config/dds_router.yaml`](../../../robot/ros_ws/src/autonomy_bringup/onboard_all/config/dds_router.yaml)) which bridges allowlisted topics from each per-robot domain into a shared GCS domain. ```bash -NUM_ROBOTS=3 airstack up +airstack up --sim isaac --robots 3 # sets NUM_ROBOTS and the multi-drone Isaac script together docker ps --format '{{.Names}}' | grep robot-desktop # airstack-robot-desktop-1 # airstack-robot-desktop-2 @@ -223,13 +255,13 @@ for i in range(1, NUM_ROBOTS + 1): spawn_drone(i) ``` -To use the multi-drone launcher, set in `.env`: +To use the multi-drone launcher, either launch with `airstack up --sim isaac --robots N` (which selects it automatically) or set in `.env`: ``` ISAAC_SIM_SCRIPT_NAME="example_multi_px4_pegasus_launch_script.py" ``` -(The default `example_one_px4_pegasus_launch_script.py` only spawns one.) +(The default `example_one_px4_pegasus_launch_script.py` only spawns one; `airstack up` preflight rejects `NUM_ROBOTS>1` with a single-drone script.) ### Test harness @@ -242,7 +274,7 @@ env_overrides = { } ``` -Tests that act on robots iterate `n=1..num_robots` and address them as `/robot_{n}/...` directly (see `_takeoff_one_robot` in `tests/test_takeoff_hover_land.py`). The test sets `ROS_DOMAIN_ID=n` for each per-robot subprocess (`domain_id=n` in `ros2_exec(...)`), matching what the resolver assigned inside the container. **If you write a new test that talks to a robot, follow this same `domain_id=n` + `/robot_{n}/...` pattern.** +Tests that act on robots iterate `n=1..num_robots` and address them as `/robot_{n}/...` directly (see `_takeoff_one_robot` in `tests/system/test_takeoff_hover_land.py`). The test sets `ROS_DOMAIN_ID=n` for each per-robot subprocess (`domain_id=n` in `ros2_exec(...)`), matching what the resolver assigned inside the container. **If you write a new test that talks to a robot, follow this same `domain_id=n` + `/robot_{n}/...` pattern.** CLI passthrough: @@ -286,7 +318,7 @@ Without `allow_substs="true"`, the substitution string is loaded literally and t If two robots share a domain, every topic collides — both `/robot_1/odometry` publishers will be visible to both subscribers, and DDS will sometimes deliver crossed data. The default `robot_name_map` derives the domain from the robot index, so this only happens if you: - Hardcode `ROS_DOMAIN_ID` in compose to the same value for two replicas -- Use a hostname that doesn't match any rule and falls through to the catch-all (both robots get `unknown-robot`, domain `0`) +- Use a hostname that doesn't match any rule and falls through to the catch-all (both robots get `unknown_robot`, domain `0`) Always verify after starting: @@ -329,9 +361,30 @@ This is a common foot-gun: Either keep the remap relative (`to="odometry"`) so it joins the namespace, or write the full path explicitly (`to="/$(env ROBOT_NAME)/odometry"`). -### 9. Hostname doesn't match any rule on real robots +### 9. Real robots and the `unknown_robot` fallback + +On VOXL/Jetson the service uses `ROBOT_NAME_SOURCE=hostname`, so the **OS hostname** is what gets mapped — not a compose replica index. The stock `default_robot_name_map.yaml` only matches `robot-`, so a device named `airlab-jetson-42` falls through to the catch-all and comes up as **`ROBOT_NAME=unknown_robot`, domain `0`** (with a map that has *no* catch-all, the resolver instead exits non-zero and `ROBOT_NAME` is left unset — same confusing "empty namespace" symptom). This is the usual "why is my real robot `unknown_robot`?" report. + +Pick whichever fix matches your topology (see [Configuring a Single Robot](#configuring-a-single-robot)): + +- **Quickest, no config:** rename the device — `hostnamectl set-hostname robot-1`. The default map resolves it to `robot_1` on domain 1, and a fleet named `robot-2`, `robot-3`, … resolves the same way with nothing further to maintain. +- **Hostnames you can't change:** ship a mapping YAML matching them and point `ROBOT_NAME_MAP_CONFIG_FILE` at it. Needs no code change — the variable is already forwarded and `robot_name_map/` is bind-mounted into the container — and keeps the resolver co-assigning `ROS_DOMAIN_ID`. + +Setting `ROBOT_NAME` in an override env file is **not** an option: nothing declares it in +compose, so it never reaches the container. See the danger note under +[Configuring a Single Robot](#configuring-a-single-robot). + +Verify on the device — do this every time, especially after pinning `ROBOT_NAME`, since +a pin that never reached the container fails silently: + +```bash +docker exec bash -c 'echo "$(hostname) -> ROBOT_NAME=$ROBOT_NAME ROS_DOMAIN_ID=$ROS_DOMAIN_ID"' +``` -On VOXL/Jetson with `ROBOT_NAME_SOURCE=hostname`, the device hostname must match a rule in the mapping YAML. If `hostname` returns `airlab-jetson-42` and your config only matches `robot-N`, the resolver exits non-zero and `ROBOT_NAME` is unset — the autonomy stack will then launch with empty namespaces and break in confusing ways. Either rename the device or extend the mapping config. +If it still reports `unknown_robot` after you set `ROBOT_NAME`, the variable did not +reach the container. Check that the service (or the base compose file it extends) +declares it in `environment:` — see the warning under +[Configuring a Single Robot](#configuring-a-single-robot). ## Pre-Merge Checklist diff --git a/.agents/skills/docker-build-profiles/SKILL.md b/.agents/skills/docker-build-profiles/SKILL.md new file mode 100644 index 000000000..3cff9e38c --- /dev/null +++ b/.agents/skills/docker-build-profiles/SKILL.md @@ -0,0 +1,137 @@ +# docker-build-profiles SKILL + +Summary +- Purpose: Provide actionable build-time validation snippets and YAML guidance for AirStack Docker builds. Designed for Claude/GPT-style agents that automate repo changes, CI checks, or PR review suggestions. +- Location: .agents/skills/docker-build-profiles/SKILL.md + +When to use +- When adding or updating a `docker-compose` profile that passes `PYTHON_VERSION`, `ROS_DISTRO`, or other numeric-like build args. +- When an automated agent needs to verify a new profile will produce a correct `PYTHONPATH` and avoid YAML float-parsing bugs. + +Actions the agent can perform +1. Validate `docker-compose.yaml` args are quoted when numeric-like (e.g. `PYTHON_VERSION: "3.10"`). +2. Insert a build-time validation `RUN` into `robot/docker/Dockerfile.robot` to fail early when the ROS Python path does not exist. +3. Add or update a short test in documentation showing how to build the `builder` stage and check `ament_package` import. +4. Suggest `network: host` under `build:` for L4T/Jetson profiles only when necessary (kernel iptables workarounds). + +Snippets (copyable) + +- YAML-check rule (agent pseudocode): + + - If a `build.args` key named `PYTHON_VERSION` exists and the value matches `/^\d+\.\d+$/`, ensure it's a quoted string in YAML; otherwise update to `""`. + +- Dockerfile validation snippet (recommended, place before using `PYTHON_VERSION` to compose `PYTHONPATH`): + +```dockerfile +RUN test -d /opt/ros/${ROS_DISTRO}/lib/python${PYTHON_VERSION} \ + || (echo "Invalid PYTHON_VERSION=${PYTHON_VERSION} or missing ROS python path" && exit 1) +``` + +- Quick builder-stage test commands (agent can run or instruct user to run): + +```bash +DOCKER_BUILDKIT=1 docker build --target builder \ + -f robot/docker/Dockerfile.robot \ + --build-arg BASE_IMAGE= \ + --build-arg ROS_DISTRO= \ + --build-arg PYTHON_VERSION="" \ + -t airstack-builder-test:local robot/docker + +docker run --rm -it airstack-builder-test:local bash -c "python3 -c 'import ament_package; print(ament_package.__file__)'" +``` + +Guidance for agents when editing the repo +- Prefer making minimal, reversible changes: add the `RUN test -d ...` check early in the Dockerfile and gate it with informative message text. +- When updating `docker-compose.yaml`, only quote the numeric-like values; do not change unrelated fields. +- If creating PRs, include a short note in the PR description instructing maintainers to run the builder-stage sanity build on both an amd64 desktop profile and an arm64 L4T profile. + +Troubleshooting notes +- YAML quirk: unquoted `3.10` may be parsed as float `3.1` — this changes path strings and breaks imports (e.g., `python3.1` instead of `python3.10`). +- Jetson/L4T builds may require `network: host` during the build to avoid kernel iptables/raw table missing-module errors. +- Jetson **`robot-l4t`** builds from **`robot-l4t-stack-base`** (`robot/docker/Dockerfile.l4t-stack-base`), not raw dustynv, so **`Dockerfile.robot` stays Ubuntu-shaped.** `airstack image-build --profile l4t robot-l4t` triggers **`robot-l4t-stack-base`** first (`airstack.sh`); bare `compose build robot-l4t` can still parallelize badly, so list stack-base explicitly if not using AirStack CLI. +- **dustynv `/ros_entrypoint.sh` shadows the apt Jazzy runtime (mavros symbol-lookup crash).** The dustynv base sources a prebuilt *source* ROS at `$ROS_ROOT/install` from PID 1, prepending its older libs (e.g. `fastcdr` 2.2.5) ahead of the apt Jazzy (2.2.7) that `Dockerfile.robot` layers on top — apt-built nodes like mavros then die with symbol-lookup errors under tmux autolaunch. `Dockerfile.l4t-stack-base` neutralizes it by overwriting `/ros_entrypoint.sh` with a `exec "$@"` passthrough; shells get ROS from `/opt/ros/jazzy/setup.bash` via `.bashrc`. If a Jetson node suddenly can't resolve symbols after a base-image bump, check whether the entrypoint passthrough is still in place. +- **ZED SDK version is pinned across `zed/Dockerfile.zed-l4t`** — the `ZED_SDK_URL` (e.g. `.../zedsdk/5.2/...`) and the ROS dep args (`ZED_MSGS_VERSION`, `POINTCLOUD_TRANSPORT*_VERSION`, `BACKWARD_ROS_VERSION`) must move together; a mismatched `zed_msgs` vs SDK breaks the driver build. Bumping the SDK is camera-firmware-coupled, so confirm the target camera runs that SDK line before merging. +- **`pytest` is pinned to `7.4.*` in `Dockerfile.robot` — do not remove or bump it.** The builder-stage `pip3 install` pulls `pytest` transitively into `/usr/local` (copied into the runtime image), which shadows Jazzy's apt `python3-pytest` 7.4. `pytest` 8 removed the `path` argument from the `pytest_pycollect_makemodule` hook, which apt's `launch_pytest` plugin still declares — so an unpinned (>=8) pytest aborts **every** pytest run in the container at plugin registration. That breaks `colcon test` for `ament_python` packages (e.g. `lidar_point_cloud_filter` in `test_colcon_test_robot`), while `ament_cmake` gtest packages are unaffected. Keeping the pin at Jazzy's version keeps `launch_testing` / `launch_pytest` usable for launch-based tests. The `tests/docker` runner is a separate interpreter and is free to use a newer pytest. + +Examples of agent prompts +- "Check `robot/docker/docker-compose.yaml` for `PYTHON_VERSION` entries and quote any unquoted numeric values; open a PR with the fixes and include a test log from a builder-stage build." +- "Insert a build-time validation `RUN` in `robot/docker/Dockerfile.robot` that ensures `/opt/ros/${ROS_DISTRO}/lib/python${PYTHON_VERSION}` exists; push as a separate small commit." + +Notes +- This SKILL is intended for agent workflows (automated PRs, repo fixes, review suggestions). Keep changes explicit and reversible. +- For human-facing docs, maintain a high-level page in `docs/` that links to this SKILL for actionable snippets and agent tasks. + +SKILL vs human docs + +- Keep SKILLs low-level and exact: this file contains raw `docker` commands and copyable build-time snippets intended for agents and automation. +- Keep human-facing docs (`docs/`) showing the `airstack` CLI equivalents and higher-level workflows. This reduces cognitive load for maintainers while preserving exact commands in SKILLs for automation and debugging. +- For the robot profile, human docs should prefer `airstack image-build --target builder --progress=plain ` when showing how to inspect build output. + +Creating a new profile (step-by-step) + +This section shows the minimal, recommended steps an agent or maintainer should perform to add a new `docker-compose` profile that builds from `Dockerfile.robot`. + +1. Pick a sensible service name and base image + + - Choose a service name that clearly indicates the platform, e.g. `robot-desktop`, `robot-l4t`, or `robot-myboard`. + - Select an appropriate `BASE_IMAGE` (amd64 desktop base or `nvcr.io/nvidia/l4t-jetpack:...` for Jetson). + +2. Add the profile with quoted numeric args + + - Add a service block in `robot/docker/docker-compose.yaml` (or an override file) and set `build.args` for the profile. + - Always quote `PYTHON_VERSION` values (e.g. `"3.10"`) so YAML does not convert them to floats. + + Example snippet to add: + + ```yaml + robot-myboard: + build: + context: ./robot/docker + dockerfile: ./Dockerfile.robot + args: + BASE_IMAGE: nvcr.io/nvidia/l4t-jetpack:r36.4.0 + ROS_DISTRO: humble + PYTHON_VERSION: "3.10" + REAL_ROBOT: true + SKIP_MACVO: true + # for L4T builds only when necessary + # network: host + ``` + +3. Add an optional validate-early check (recommended) + + - Insert the `RUN test -d /opt/ros/${ROS_DISTRO}/lib/python${PYTHON_VERSION}` check near the top of `Dockerfile.robot` (before `ENV PYTHONPATH` or any Python-dependent operations). This ensures the build fails fast with a clear message. + +4. Run the builder-stage sanity build + + - Run the builder-target build locally (or in CI) to verify the image picks up the correct Python/ROS paths and that `ament_package` imports: + + ```bash + DOCKER_BUILDKIT=1 docker build --target builder \ + -f robot/docker/Dockerfile.robot \ + --build-arg BASE_IMAGE=nvcr.io/nvidia/l4t-jetpack:r36.4.0 \ + --build-arg ROS_DISTRO=humble \ + --build-arg PYTHON_VERSION="3.10" \ + -t airstack-builder-test:local robot/docker + + docker run --rm airstack-builder-test:local python3 -c "import ament_package; print('ok', ament_package.__file__)" + ``` + +5. Smoke-run the full compose build (optional but recommended) + + - Use `docker compose -f robot/docker/docker-compose.yaml build robot-myboard` to ensure compose passes the args correctly. + +6. Prepare the PR with clear validation notes + + - Make the code change small and focused (one commit to `docker-compose.yaml`, one optional commit for the `Dockerfile` validation line). + - In the PR description include the builder-stage test command output and request a reviewer to run the builder-stage test on both an amd64 and arm64 profile if possible. + +7. Merge and monitor + + - After merge, ensure CI (if configured) runs the sanity build or that maintainers run the checks on the target hardware. + +Agent implementation tips + +- When automating the change, produce a single commit that updates only the new service block and, if needed, a second commit that adds the `RUN` check to `Dockerfile.robot`. +- If the target is Jetson/L4T, add `network: host` under `build:` only when prior builds show iptables/kernel errors; do not enable it by default. +- If you detect a pre-existing unquoted `PYTHON_VERSION` in the repo, prefer to update that entry in-place and include an explanatory commit message about YAML float parsing. diff --git a/.agents/skills/integrate-module-into-layer/SKILL.md b/.agents/skills/integrate-module-into-layer/SKILL.md index c4f7bfb50..46166877b 100644 --- a/.agents/skills/integrate-module-into-layer/SKILL.md +++ b/.agents/skills/integrate-module-into-layer/SKILL.md @@ -244,7 +244,7 @@ Launch the complete autonomy stack to test integration: ```bash # Stop any running containers -airstack stop +airstack down # Launch with full autonomy AUTOLAUNCH=true airstack up robot-desktop diff --git a/.agents/skills/optitrack-development/SKILL.md b/.agents/skills/optitrack-development/SKILL.md new file mode 100644 index 000000000..d5e656c9a --- /dev/null +++ b/.agents/skills/optitrack-development/SKILL.md @@ -0,0 +1,221 @@ +--- +name: optitrack-development +description: Develop and integrate OptiTrack NatNet in AirStack — robot client (natnet_ros2), Isaac Sim Motive emulator, wire-protocol handshake, and libNatNet 4.4 unicast behavior. Use when working on natnet_ros2, optitrack.natnet.emulator, LAUNCH_NATNET, or NatNet UDP protocol compatibility. +license: Apache-2.0 +metadata: + author: AirLab CMU + repository: AirStack +--- + +# Skill: OptiTrack / NatNet Development + +## When to Use + +- Implementing or debugging the **Motive emulator** in Isaac Sim + (`simulation/isaac-sim/extensions/optitrack.natnet.emulator/`) +- Integrating or testing **`natnet_ros2`** on the robot stack +- Understanding **NatNet wire protocol** (connect, model def, frame streaming) +- Capturing what **`libNatNet.so`** actually sends on the network +- Enabling OptiTrack in sim: `LAUNCH_NATNET=true`, `natnet_config.yaml`, Docker IPs +- Sim testing with mocap: bring the stack up with `overrides/isaac-optitrack-simulation.env`, which starts Isaac + the emulator and switches PX4 EKF2 to external-vision fusion (GPS/baro/range aiding off, so mocap is the only position source) + +## Architecture in AirStack + +```mermaid +flowchart LR + subgraph sim ["Isaac Sim (172.31.0.200)"] + Emulator["optitrack.natnet.emulator\n(NatNet UDP server)"] + end + subgraph robot ["Robot container"] + Node["natnet_ros2_node"] + SDK["libNatNet.so client"] + Node --> SDK + end + SDK -->|"UDP 1510 (unicast: cmd + frames)"| Emulator + Node --> Topics["/{ROBOT_NAME}/perception/optitrack/..."] +``` + +| Component | Path | Role | +|-----------|------|------| +| Robot client | [`robot/ros_ws/src/perception/natnet_ros2/`](../../../robot/ros_ws/src/perception/natnet_ros2/) | ROS 2 node; uses **official NatNet SDK** (`NatNetClient::Connect`) | +| SDK install | `natnet_ros2/lib/libNatNet.so`, `include/natnet/` | Download via `airstack setup --natnet` (proprietary, not in git) | +| Emulator (WIP) | [`simulation/isaac-sim/extensions/optitrack.natnet.emulator/`](../../../simulation/isaac-sim/extensions/optitrack.natnet.emulator/) | Python NatNet **server** for sim / integration tests | +| Integration tests | [`tests/integration/natnet/README.md`](../../../tests/integration/natnet/README.md) | End-to-end UDP tests against real SDK parser (mark: `integration`) | + +**Enable on robot:** `LAUNCH_NATNET=true` in `.env` → [`perception.launch.xml`](../../../robot/ros_ws/src/perception/perception_bringup/launch/perception.launch.xml) includes `natnet_ros2.launch.py`. + +**Enable in sim:** set ``ISAAC_SIM_SCRIPT_NAME`` to a NatNet Pegasus launch script (no env gate in the script — NatNet always starts): + +| Script | Use | +|--------|-----| +| [`example_one_px4_pegasus_natnet_launch_script.py`](../../../simulation/isaac-sim/launch_scripts/example_one_px4_pegasus_natnet_launch_script.py) | Single drone + static ``Target`` | +| [`example_multi_px4_pegasus_natnet_launch_script.py`](../../../simulation/isaac-sim/launch_scripts/example_multi_px4_pegasus_natnet_launch_script.py) | ``NUM_ROBOTS`` drones + shared ``Target`` (pair with 3-profile ``natnet_config.yaml``) | + +Helpers: [`isaac/scene_setup.py`](../../../simulation/isaac-sim/extensions/optitrack.natnet.emulator/optitrack/natnet/emulator/isaac/scene_setup.py) (`start_drone_natnet_server`, `author_static_target`). Drone body: single = ``Drone`` (id 1); multi = ``Drone`` (id ``i``); target = ``Target`` (id 100). In the single-drone script these are overridable via ``NATNET_BODY_NAME``/``NATNET_TARGET_NAME`` env vars; in the multi script they are constants. Either way they must match the ``natnet_config.yaml`` profile — change both together. The client filters frames by numeric id, so a mismatch is silent: it connects and never publishes. Baseline Pegasus scripts (no NatNet) remain ``example_one_px4_pegasus_launch_script.py`` / ``example_multi_px4_pegasus_launch_script.py``. + +**Default client config:** unicast, `server_ip` → Motive/emulator (use `172.31.0.200` for Isaac container), ports 1510/1511. The config is per-robot: each `robots[$ROBOT_NAME]` profile lists the bodies it tracks (each a `rigid_body_name` + `id` mapped to a relative `topic`, with `pose`/`pose_cov` toggles and per-body covariance) and an optional `vision_pose` block that drives the MAVROS bridge. See [`natnet_config.yaml`](../../../robot/ros_ws/src/perception/natnet_ros2/config/natnet_config.yaml). + +## NatNet: Two UDP Channels + +| Port (server default) | Channel | Direction | +|----------------------|---------|-----------| +| **1510** | Command | Client → server: `NAT_CONNECT`, `NAT_REQUEST_MODELDEF`, keepalives. Server → client: `NAT_SERVERINFO`, `NAT_MODELDEF`, `NAT_RESPONSE` | +| **1511** | Data | Server → client: `NAT_FRAMEOFDATA` (mocap frames). Multicast group `239.255.42.99` when using multicast. **The server must send frames from a socket bound to the data port** (source port == `data_port`); see below. | + +**Critical rules (verified against the real `libNatNet.so` 4.4 unicast + `NatNet_SetLogCallback`):** + +- Command **responses** go to the client's endpoint from `recvfrom` on the server command listener (`1510`), sent via the **command** socket. +- **Frames must be sent from the server's DATA socket** (bound to `data_port`, e.g. `1511`) so the datagram **source port == `data_port`**. libNatNet routes inbound unicast datagrams by source port: frames from the **command** port are treated as command traffic and **silently dropped** (no error, no callback). This was the single biggest gotcha. +- **libNatNet 4.4 unicast uses one client UDP socket** (one ephemeral local port for command send/recv and frame recv). The client receives frames there regardless of the server's source port — but libNatNet only **dispatches** them to the frame callback when they came from the server's data port. Do **not** assume `data_port = cmd_port + 1`. +- **Every `NAT_FRAMEOFDATA` must end with a 4-byte end-of-data tag** (after the frame `params`). libNatNet's unpacker reads it; without it the unpacked length mismatches `nDataBytes` and the SDK drops the whole frame. (The lenient Python `NatNetClient` does not require it — always validate against the C SDK.) +- The **269-byte `NAT_CONNECT` payload does not include** the client port; the port is learned from the datagram **source address** on `NAT_CONNECT`. +- Do **not** trust `/proc`/`ss` alone for the client port — extra bound sockets may appear that do not match wire traffic. **`NAT_CONNECT` source `(ip, port)` is ground truth.** +- Do **not** parse connect payloads with in-memory `sNatNetClientConnectParams` (contains pointers). Use on-wire layouts below. + +## libNatNet 4.4 `NAT_CONNECT` (verified 2025-06) + +Observed against `127.0.0.1:1510` with the same unicast params as [`natnet_client_adapter.cpp`](../../../robot/ros_ws/src/perception/natnet_ros2/src/natnet_client_adapter.cpp). + +### What the client sends + +| Field | Observed value | +|-------|----------------| +| Message | `NAT_CONNECT` (0), `nDataBytes = 269`, total datagram 273 bytes | +| Payload layout | `sSender` (264 B) + `sConnectionOptions` (5 B) | +| `sSender.szName` | `"NatNetLib"` | +| `sSender.Version` | `[4, 4, 0, 0]` | +| `sSender.NatNetVersion` | `[4, 4, 0, 0]` | +| `subscribedDataOnly` | `0` | +| `BitstreamVersion` | `[0, 0, 0, 0]` → client defers to server version | +| Trailing port bytes | **None** (exactly 269 bytes; not PacketClient's optional +4) | +| UDP source port | Ephemeral (e.g. `41449`) — **client command + data port (same socket)** | + +Example hex (payload only, after 4-byte header): + +``` +NatNetLib\0 ... (256-byte name field) +04 04 00 00 (Version) +04 04 00 00 (NatNetVersion) +00 (subscribedDataOnly) +00 00 00 00 (BitstreamVersion) +``` + +## libNatNet 4.4 unicast: single client socket (verified 2025-06) + +Confirmed with wire capture on server `:1510`/`:1511`, `strace` on a minimal `NatNetClient::Connect()` binary, and `/proc//net/udp` cross-checks against the same `libNatNet.so` used by `natnet_ros2`. + +### What we observed + +| Signal | Result | +|--------|--------| +| Wire capture on server `:1510` | All client packets (`NAT_CONNECT`, `NAT_KEEPALIVE`, `NAT_REQUEST_MODELDEF`) from **one** source port | +| Wire capture on server `:1511` | **No** inbound packets from the client | +| strace on minimal client | **One** `bind()`, **one** fd for all `sendto` → server `:1510` and `recvfrom` ← server `:1510` | +| `NAT_CONNECT` payload | **No** trailing client port bytes (269 B total) | + +### Emulator rule (unicast + `natnet_ros2`) + +For libNatNet 4.4 unicast, treat the client as **single-endpoint**: + +```text +On NAT_CONNECT → store client_endpoint = (ip, port) from recvfrom +NAT_SERVERINFO → sendto(command_socket, client_endpoint) # source port = command_port +NAT_MODELDEF → sendto(command_socket, client_endpoint) # source port = command_port +NAT_FRAMEOFDATA → sendto(data_socket, client_endpoint) # source port = data_port (REQUIRED) +NAT_KEEPALIVE → no reply (client -> server only) +``` + +The client always learns its endpoint from the **`NAT_CONNECT` source address** (the +client uses a single socket), so the **destination** of frames is that endpoint. The +**source** of frames, however, must be the server's data port — bind a dedicated +`data_socket` to `('', data_port)` and `sendto` frames from it. + +`ConnectionDataPort = 1511` in `NAT_SERVERINFO` is required (the SDK uses it to +recognize the data channel — i.e. which source port valid frames arrive from). + +### When two client ports may still apply + +- **Multicast** clients (separate multicast data listener on `239.255.42.99:1511`) +- **PacketClient-style** samples that open explicit command + data sockets (optional +4 port bytes in connect) +- Other NatNet client implementations — always verify with protocol capture before assuming a two-socket model + +Do **not** assume `data_port = cmd_port + 1` for any client without capture. + +### What the server must reply (for `Connect()` + `GetServerDescription()`) + +1. **`NAT_SERVERINFO` (1)** on the **command port** to the connect datagram source. +2. Payload: packed **`sSender_Server`** (279 B), **not** `sServerDescription`. libNatNet + parses the `NAT_SERVERINFO` payload as `sSender_Server`; sending the larger + `sServerDescription` makes it misread the version/host. Fields: + - `Common.szName = "Motive"` (256-byte field) + - `Common.Version = {3, 1, 0, 0}` (Motive app), `Common.NatNetVersion = {4, 4, 0, 0}` + - `HighResClockFrequency`, `DataPort = 1511`, `IsMulticast = 0` (unicast) + +Pre-built in emulator: [`NatNetServer._build_connect_response_payload()`](../../../simulation/isaac-sim/extensions/optitrack.natnet.emulator/optitrack/natnet/emulator/server/natnet_server.py). + +### After connect (required for `natnet_ros2` topics) + +| SDK call | Server must handle | +|----------|-------------------| +| `GetDataDescriptionList()` | `NAT_REQUEST_MODELDEF` → `NAT_MODELDEF` with rigid body name/ID (e.g. `"Drone"`) | +| Frame callback | Stream `NAT_FRAMEOFDATA` to **`NAT_CONNECT` source `(ip, port)`** from the server **data socket** (source port = `data_port`); end each frame with the 4-byte EOD tag; set `rb.params & 0x01` (tracking valid) | +| Unicast keepalive | Accept `NAT_KEEPALIVE` on command port; **send no reply** | + +Verified end-to-end against the real `libNatNet.so` with a C probe that registers +`SetFrameReceivedCallback` + `NatNet_SetLogCallback`: with the data-port source, +EOD tag, `sSender_Server` reply, and no keepalive reply, the probe reports +`Server: Motive 3.1.0.0 NatNet 4.4.0.0`, `data descriptions: 1`, and ~74 Hz callbacks. + +## Wire format reference (do not confuse) + +| Client type | Connect payload | +|-------------|-----------------| +| **`libNatNet` / `natnet_ros2`** | `sSender` + `sConnectionOptions` (269 B observed) | +| **PacketClient sample** | Same + optional 4 trailing bytes (often zero in sample) | +| **Python NatNetClient sample** | Legacy 270-byte `"Ping"` blob — **not** used by `natnet_ros2` | + +API struct `sNatNetClientConnectParams` ([`NatNetTypes.h`](../../../simulation/isaac-sim/extensions/optitrack.natnet.emulator/NatNetClientSDK/NatNetSDK/include/NatNetTypes.h)) is for `Connect()` in process memory only — **not** the on-wire layout. + +## Protocol capture (optional, for debugging) + +Not part of the repo. If you need to re-verify wire behavior or debug a new client/server pairing, build a **minimal out-of-band harness**: + +1. **Minimal C++ client** — tiny binary linking `libNatNet.so` from `natnet_ros2`; call `NatNetClient::Connect()` with the same params as [`natnet_client_adapter.cpp`](../../../robot/ros_ws/src/perception/natnet_ros2/src/natnet_client_adapter.cpp). Optional: `GetDataDescriptionList()`, frame callback, `--hold-seconds` sleep. +2. **Python UDP stub server** — bind `:1510` (and optionally `:1511`); reply to `NAT_CONNECT` with canned `NAT_SERVERINFO`, to `NAT_REQUEST_MODELDEF` with `NAT_MODELDEF`, to `NAT_KEEPALIVE` with ack; log every `(ip, port)` and message id. +3. **Connect capture** — run the client against the stub; hex-dump the first datagram; confirm 269-byte `sSender` + `sConnectionOptions` payload and ephemeral source port. +4. **Endpoint discovery** — during a full connect + model-def fetch: + - `tcpdump -i any udp and host ` or the stub's packet log + - `strace -e trace=bind,sendto,recvfrom` on the client binary + - `/proc//net/udp` or `ss -uapn` (treat **`NAT_CONNECT` source port** as ground truth if they disagree) +5. **Frame delivery check** — confirm the client's frame callback fires. Register both `SetFrameReceivedCallback` **and** `NatNet_SetLogCallback` (the log callback surfaces silent drops). Frames must be sent from the server **data socket** (source port = `data_port`) and end with the 4-byte EOD tag, or the SDK drops them with no callback. + +Use the SDK's `NatNetTypes.h` and `PacketClient.cpp` for on-wire layouts — not in-memory `sNatNetClientConnectParams`. + +## Emulator implementation checklist + +1. **Command listener** on `0.0.0.0:1510` +2. **`NAT_CONNECT`** → register `client_endpoint` from `recvfrom`; reply `NAT_SERVERINFO` +3. **`NAT_REQUEST_MODELDEF`** → reply `NAT_MODELDEF` (match `body_name` in config) +4. **Frame loop** → `NAT_FRAMEOFDATA` to `client_endpoint` **from the data socket** (source port = `data_port`); end each frame with the 4-byte EOD tag +5. **Isaac integration** → sample drone pose → `sFrameOfMocapData` → `enqueue_mocap_data()` +6. **Docker** → emulator on `172.31.0.200`; robot `server_ip` points there + +## Testing levels + +| Level | Approach | Validates | +|-------|----------|-----------| +| Unit (no network) | `test_natnet_logic.cpp`, `FakeNatNetClient` | Negotiation logic, topic names | +| Protocol capture | Minimal client + UDP stub (see above) | Wire-format `NAT_CONNECT`, client endpoint model | +| Integration | `tests/integration/natnet/` | Full SDK parser + `natnet_ros2_node` (mark: `integration`) | +| System (future) | `airstack test -m sensors` | Topic Hz on `/perception/optitrack/...` | + +```bash +# Unit tests (robot container) +docker exec airstack-robot-desktop-1 bash -c "sws && colcon test --packages-select natnet_ros2 --event-handlers console_direct+" +``` + +## References + +- OptiTrack NatNet docs: https://docs.optitrack.com/developer-tools/natnet-sdk/natnet-4.0 +- SDK samples (wire format): `NatNet_SDK_*/Samples/PacketClient/`, `PythonClient/` (legacy connect in Python only) +- Integration test: [`tests/integration/natnet/README.md`](../../../tests/integration/natnet/README.md) diff --git a/.agents/skills/run-system-tests/SKILL.md b/.agents/skills/run-system-tests/SKILL.md index 868d8c695..c2260d29e 100644 --- a/.agents/skills/run-system-tests/SKILL.md +++ b/.agents/skills/run-system-tests/SKILL.md @@ -1,6 +1,6 @@ --- name: run-system-tests -description: Run, interpret, and extend AirStack's pytest system test suite (build_packages, build_docker, liveliness, sensors, takeoff_hover_land), trigger runs via /pytest PR comments, and read metrics.json regression reports. Use for invoking tests, debugging failures from results.xml/metrics.json, or adding a new system test. +description: Run, interpret, and extend AirStack's pytest system test suite (build_packages, build_docker, liveliness, sensors, takeoff_hover_land, autonomy), trigger runs via /pytest PR comments, and read run_meta.json/metrics.json reports. Use for invoking tests, distinguishing infrastructure failures from policy regressions, or adding a new system test. license: Apache-2.0 metadata: author: AirLab CMU @@ -14,7 +14,7 @@ metadata: Use this skill when you need to: - Invoke the pytest system tests locally (via `airstack test`) or on CI (via `/pytest` PR comment or `workflow_dispatch`) -- Diagnose a failing system test — interpret `results.xml`, per-test logs, and `metrics.json` from `tests/results//` +- Diagnose a failing system test — interpret `summary.txt`, `results.xml`, `run_meta.json`, and `metrics.json` from `tests/results//` - Compare metrics against a baseline run (`parse_metrics.py --baseline`) to confirm a regression or improvement - Add a new system test to `tests/`: pick the right mark, wire up `airstack_env` parametrization, and record metrics with `MetricsRecorder` @@ -22,15 +22,41 @@ This skill is about the **test harness itself** — pytest marks, fixtures, the ## Test Suite Overview -The suite lives at `tests/` (repo root) and is fully pytest-based. Configuration is in `tests/pytest.ini` and shared infrastructure in `tests/conftest.py`. Marks include `build_docker`, `build_packages`, `liveliness`, `sensors`, and `takeoff_hover_land`: +The suite lives at `tests/` (repo root) and is fully pytest-based. Configuration is in `tests/pytest.ini` and shared infrastructure in `tests/conftest.py`. + +- **`tests/system/`** — Docker stack integration tests. Marks: `build_docker`, `build_packages`, `liveliness`, `sensors`, `takeoff_hover_land`, `autonomy`. +- **`tests/integration/`** — Cross-component tests (`integration` mark): robot container + a host-side component, no sim/GPU. +- **Unit tests** (`@pytest.mark.unit`) — Hermetic. Source is **co-located** with each ROS 2 package in its own `test/` dir (ROS 2 / colcon convention). `tests/colcon_unit_test_packages.yaml` lists which packages have unit tests; `conftest.py` resolves each to its `test/` dir and collects the non-linter `test_*.py` under `--import-mode=importlib`. + +### Unit tests vs system tests + +| Concern | Unit (`-m unit`) | System (`-m liveliness` etc.) | +|---|---|---| +| Hardware required | None — pure Python | Docker daemon, NVIDIA GPU, sim license | +| CI workflow | `unit-tests.yml` (`ubuntu-latest`) | `system-tests.yml` (ephemeral OSMO GPU pod) | +| Trigger | PR opened, synchronized, or reopened | Automatic `build_packages` on PR open/update/reopen; simulation via `/pytest` or `workflow_dispatch` | +| Source location | `/test/test_*.py` (collected directly, listed in `colcon_unit_test_packages.yaml`) | `tests/system/` | +| How to add | See `add-unit-tests` skill | See *Adding a New System Test* below | + +Run unit tests without any Docker stack: + +```bash +airstack test -m unit -v +# or directly: +AIRSTACK_ROOT=$(pwd) pytest tests/ -m unit -v +``` + +For details on the co-located layout and adding new unit tests, see the +`add-unit-tests` skill. | File | Mark | What it tests | Hardware required | |------|------|---------------|-------------------| -| `tests/test_build_docker.py` | `build_docker` | `airstack image-build` for `robot-desktop`, `gcs`, `isaac-sim`, `ms-airsim`; records image size to `metrics.json` | Docker daemon | -| `tests/test_build_packages.py` | `build_packages` | `colcon build` (`bws`) inside the robot, GCS, and ms-airsim ROS workspaces — brought up with `AUTOLAUNCH=false` | Docker daemon | -| `tests/test_liveliness.py` | `liveliness` | Stack bring-up: containers Running, `/clock` readiness, tmux panes, sentinel ROS 2 nodes, compute, infra-only `test_stable` | Docker daemon, NVIDIA GPU + `nvidia-container-toolkit`, sim license / Omniverse creds | -| `tests/test_sensors.py` | `sensors` | Topic Hz (Isaac: batched on sim + robot; LiDAR `echo-once` + cloud sanity), RTF, `test_sensor_streams_stable` | Docker daemon, NVIDIA GPU + `nvidia-container-toolkit`, sim license / Omniverse creds | -| `tests/test_takeoff_hover_land.py` | `takeoff_hover_land` | 4-phase flight chain per `(sim, num_robots, iteration, velocity)`: `test_px4_ready` → `test_takeoff` → `test_hover` → `test_landing`. Records altitude error, overshoot, hover stability, landing accuracy, odometry drift | Docker daemon, NVIDIA GPU, sim license | +| `tests/system/test_build_docker.py` | `build_docker` | `airstack image-build` for `robot-desktop`, `gcs`, `isaac-sim`, `ms-airsim`; records image size to `metrics.json` | Docker daemon | +| `tests/system/test_build_packages.py` | `build_packages` | `colcon build` (`bws`) inside the robot, GCS, and ms-airsim ROS workspaces — brought up with `AUTOLAUNCH=false` | Docker daemon | +| `tests/system/test_liveliness.py` | `liveliness` | Stack bring-up: containers Running, `/clock` readiness, tmux panes, sentinel ROS 2 nodes, compute, infra-only `test_stable` | Docker daemon, NVIDIA GPU + `nvidia-container-toolkit`, sim license / Omniverse creds | +| `tests/system/test_sensors.py` | `sensors` | Topic Hz (Isaac: batched on sim + robot; LiDAR `echo-once` + cloud sanity), RTF, `test_sensor_streams_stable` | Docker daemon, NVIDIA GPU + `nvidia-container-toolkit`, sim license / Omniverse creds | +| `tests/system/test_takeoff_hover_land.py` | `takeoff_hover_land` | 4-phase flight chain per `(sim, num_robots, iteration, velocity)`: `test_px4_ready` → `test_takeoff` → `test_hover` → `test_landing`. Records altitude error, overshoot, hover stability, landing accuracy, odometry drift | Docker daemon, NVIDIA GPU, sim license | +| `tests/system/test_fixed_trajectory.py` | `autonomy` | 4-phase flight chain per `(sim, num_robots, iteration, trajectory_type)`: `test_px4_ready` → `test_takeoff` → `test_fixed_trajectory` → `test_landing`. Records cross-track error, path RMSE, trajectory success/time for Circle/Figure8/Racetrack/Line | Docker daemon, NVIDIA GPU, sim license | The marks are declared in `tests/pytest.ini`. **Do not invent new marks ad-hoc** — register any new mark there or pytest will warn about unknown marks. @@ -39,10 +65,10 @@ The marks are declared in `tests/pytest.ini`. **Do not invent new marks ad-hoc** `conftest.py` enforces a deterministic global order so cheap-and-fast-failing tests surface first: ``` -test_build_docker → test_build_packages → test_liveliness → test_sensors → test_takeoff_hover_land +system.test_build_docker → system.test_build_packages → system.test_liveliness → system.test_sensors → system.test_takeoff_hover_land → system.test_fixed_trajectory ``` -Within `test_takeoff_hover_land`, items are re-sorted to `(airstack_env, velocity, phase)` so each `(sim, robots, iter)` env brings the stack up once and the drone goes ground → air → ground per velocity before pytest moves to the next velocity. +Within `system.test_takeoff_hover_land`, items are re-sorted to `(airstack_env, velocity, phase)` so each `(sim, robots, iter)` env brings the stack up once and the drone goes ground → air → ground per velocity before pytest moves to the next velocity. `system.test_fixed_trajectory` is re-sorted the same way by `(airstack_env, trajectory_type, phase)`. ### Isaac Sim (`sensors`): why Hz is batched and LiDAR uses `echo --once` @@ -55,7 +81,7 @@ rates if too many `ros2 topic hz` processes run concurrently. - **Robot-side (Isaac):** two passes — both stereo images, then both depths. **ms-airsim** keeps a single four-topic parallel batch on the robot container. - **Filtered LiDAR** (`PointCloud2`): uses `ros2 topic echo --once` per robot - (see `parallel_echo_once_robot_topics` in `conftest.py`), not `topic hz`. + (see `parallel_echo_once_robot_topics` in `tests/harness/sim.py`), not `topic hz`. - **Multi-drone Pegasus script:** pytest sets `ENABLE_LIDAR=true` in `conftest.py` `SIM_CONFIG["isaacsim"]["extra_env"]` so LiDAR matches the single-drone example (which always enables RTX LiDAR). @@ -72,6 +98,7 @@ The `system-tests.yml` workflow's `Parse pytest args` step automatically prepend - `/pytest -m takeoff_hover_land` → effectively runs `-m "build_packages or takeoff_hover_land"` - `/pytest` (no marks) → pytest defaults (everything) - `/pytest -m build_docker` → unchanged (the build_docker tests rebuild from scratch anyway) +- `/pytest -m build_packages` → **pull-only** (retag `cache_*`, no `image-build`, no Isaac). Add `--no-image-build` on other marks to skip the bake. This guarantees that ROS 2 workspaces are built inside the containers before any launch/liveliness test tries to source them. If you intentionally want to skip `build_packages` (e.g. you trust the prebuilt images), include it explicitly: `-m "liveliness and not build_packages"` would work, but the simpler path is to run locally where the prepend logic doesn't apply. @@ -131,13 +158,13 @@ The `airstack_env` fixture is parametrized over `(sim, num_robots, iteration)` t | Flag | Default | Affects | Becomes | |------|---------|---------|---------| -| `--sim` | `msairsim,isaacsim` | `airstack_env` | One env-tuple per sim | +| `--sim` | `isaacsim` | `airstack_env` | One env-tuple per sim (`msairsim` opt-in) | | `--num-robots` | `1,3` | `airstack_env` | Cross-product with sim | | `--stress-iterations` | `1` | `airstack_env` | Up/down cycles per `(sim, num_robots)` | -| `--stable-duration` | `120` | `test_liveliness::test_stable` and `test_sensors::test_sensor_streams_stable` | Total seconds polled | -| `--stable-interval` | `10` | `test_liveliness::test_stable` and `test_sensors::test_sensor_streams_stable` | Seconds between polls | +| `--stable-duration` | `120` | `system.test_liveliness::test_stable` and `system.test_sensors::test_sensor_streams_stable` | Total seconds polled | +| `--stable-interval` | `10` | `system.test_liveliness::test_stable` and `system.test_sensors::test_sensor_streams_stable` | Seconds between polls | | `--gui` | off (headless) | `airstack_env` | Sets `QT_QPA_PLATFORM=offscreen` when off | -| `--takeoff-velocities` | `0.5` (current default) | `test_takeoff_hover_land` | One full 4-phase chain per velocity | +| `--takeoff-velocities` | `0.5` (current default) | `system.test_takeoff_hover_land` | One full 4-phase chain per velocity | Total parametrize cardinality for sim tests = `len(sims) × len(num_robots) × stress_iterations × len(velocities for takeoff)`. Keep this small locally — a 2×2×3×3 sweep on a workstation is several hours. @@ -153,7 +180,10 @@ Total parametrize cardinality for sim tests = `len(sims) × len(num_robots) × s The `system-tests.yml` workflow accepts three trigger paths: -1. **PR opened** (same-repo only) — auto-runs pytest with conftest defaults. Fork PRs are skipped to keep arbitrary code off the self-hosted runner. +1. **PR opened, synchronized, or reopened** (same-repo only) — auto-runs the + `build_packages` mark. Fork PRs are skipped to keep arbitrary code off the + privileged self-hosted runner. Python unit tests run separately in + `unit-tests.yml`, including for fork PRs. 2. **`/pytest` issue comment** on a PR — only honored from users with `OWNER`, `MEMBER`, or `COLLABORATOR` author association. Fork PRs are explicitly rejected by the `Resolve PR head` step (the PR's head repo must equal `${context.repo.owner}/${context.repo.repo}`). 3. **`workflow_dispatch`** — manual run from the Actions tab with form inputs (`marks`, `sim`, `num_robots`, `stress_iterations`, `stable_duration`, `baseline_run_id`). @@ -178,14 +208,14 @@ notes: testing the new altitude controller The workflow: 1. Posts an acknowledgment PR comment showing the resolved `pytest tests/ ` command and a link to the run 2. Opens an in-progress GitHub Check Run on the PR's head SHA so the run shows up in the **Checks** tab (issue_comment events otherwise associate runs with the default branch) -3. Runs pytest on a freshly-spawned ephemeral OpenStack runner (`runs-on: [self-hosted, airstack-ephemeral]`) +3. Runs pytest on a freshly-spawned ephemeral OSMO GPU pod (`runs-on: [self-hosted, airstack-ephemeral]`) 4. Uploads `tests/results/` as artifact `test-results--` (90-day retention) -5. The downstream `report` job runs `parse_metrics.py` against the latest baseline artifact from the PR's base branch and posts a markdown table back as a PR comment + job summary +5. The downstream `report` job runs `parse_metrics.py`, compares only a matching complete simulation baseline, posts the result, and finalizes the PR-head Check Run 6. Closes the Check Run with the final conclusion ### Why fork PRs are blocked -The runner is GPU-equipped, has Docker root access, and is reused (briefly) across the lifetime of one job. Running arbitrary fork code on it would let a contributor exfiltrate registry creds, mine crypto, or pivot into the OpenStack tenant. The same-repo guard is the only line of defense and **must not be removed**. If you need to test a fork PR, mirror the branch into the upstream repo first. +The runner is GPU-equipped, has Docker root access, and is reused (briefly) across the lifetime of one job. Running arbitrary fork code on it would let a contributor exfiltrate registry creds, mine crypto, or abuse the privileged OSMO CI pool. The same-repo guard is the only line of defense and **must not be removed**. If you need to test a fork PR, mirror the branch into the upstream repo first. ## Interpreting Results and Metrics @@ -195,23 +225,23 @@ Every run (local or CI) produces a fresh timestamped directory under `tests/resu ``` tests/results/2025-04-21_14-30-00/ +├── summary.txt # Human-readable key metrics — open this first ├── results.xml # JUnit XML — durations + pass/fail per test -├── metrics.json # Custom metrics keyed by test_node_id → metric_key -└── logs/ - ├── test_build_docker.TestDockerBuilds.test_build_robot_desktop.log - ├── test_sensors.TestSensors.test_sensor_streams_stable[msairsim-rob#1-iter0].log - ├── test_liveliness.TestLiveliness.test_stable[msairsim-rob#1-iter0].log - ├── airstack_env.test_liveliness.TestLiveliness.test_robot_containers_running[...].log - └── ... +└── metrics.json # Custom metrics keyed by test_node_id → metric_key ``` -**One log file per test execution**, plus separate `airstack_env.*.log` files for fixture narration (the `up`/`down` of each parametrize tuple). The fixture log file is named to track the rewritten test ID so it lands next to the triggering test. +There is **no `logs/` subdirectory**. Live output streams to the terminal during +the run (pytest `log_cli`), and each subprocess's combined stdout/stderr is held +in memory so a failed assertion can include the tail of the last command's output +inline. `summary.txt` is written once at session end by +`run_summary.write_summary()`, so the key metrics land in one place without +digging through raw output. ### `metrics.json` structure ```json { - "test_liveliness.TestLiveliness.test_stable[msairsim-rob#1-iter0]": { + "system.test_liveliness.TestLiveliness.test_stable[msairsim-rob#1-iter0]": { "airstack_up_duration_s": {"value": 42.7, "unit": "s", "direction": "lower_is_better"}, "robot.sensors.front_stereo.left.image_rect.hz_samples": { "samples": [{"t": 10, "value": 19.27}, {"t": 20, "value": 19.31}, ...] @@ -246,7 +276,7 @@ The report has three sections per test module: - **Sim publishing rates** — pivoted Hz aggregates per topic (`mean`, `start_mean`, `end_mean`, `min`, `max`) from the `sensors` mark (sim + robot streams) - **Compute usage** — pivoted CPU/mem/GPU per container -Regressions exceeding `--threshold` (default 20%) are flagged `:red_circle:`; improvements beyond threshold get `:green_circle:`. CI fails the job on any regression. +Regressions exceeding `--threshold` (default 20%) are flagged `:red_circle:`; improvements beyond threshold get `:green_circle:`. CI fails only when both artifacts are complete and have the same simulation campaign fingerprint. When local-debugging a CI regression, download both artifacts (`test-results--` from the PR run and from the base branch's most recent run), unzip them under `tests/results/`, and run `parse_metrics.py` locally to see the same table the bot posted. @@ -266,20 +296,20 @@ If your test... ### 2. File location and naming -- File: `tests/test_.py` — matches pytest's default test discovery (`test_*.py`) +- File: `tests/system/test_.py` — matches pytest's default test discovery (`test_*.py`) under the system suite - Class: `Test` with the mark applied at the class level: `@pytest.mark.` - Add a class-level `@pytest.mark.timeout()` — long-running sim tests need it -- Imports: pull helpers from `conftest` directly (`from conftest import ...`); `tests/` is on `sys.path` because `testpaths = .` in pytest.ini +- Imports: pull helpers from `conftest` directly (`from conftest import ...`); they physically live in the `tests/harness/` package but are re-exported through `conftest`, so either `from conftest import ...` or `from harness import ...` works. `tests/` is on `sys.path` because `testpaths = .` in pytest.ini ### 3. Decide if you need `airstack_env` - **Need full stack up (sim + robot + GCS)?** Take `airstack_env` as a fixture argument. You'll automatically be parametrized over `(sim, num_robots, iteration)` from CLI flags — `pytest_generate_tests` in conftest activates this only for tests that name the fixture. -- **Just need one container or no containers?** Don't take `airstack_env` — bring up only what you need with `airstack_cmd("up", "", env_overrides={"AUTOLAUNCH": "false"})` and tear down in a `try/finally`, the way `test_build_packages.py` does. +- **Just need one container or no containers?** Don't take `airstack_env` — bring up only what you need with `airstack_cmd("up", "", env_overrides={"AUTOLAUNCH": "false"})` and tear down in a `try/finally`, the way `tests/system/test_build_packages.py` does. - **Need extra parametrization** (e.g. velocity for `takeoff_hover_land`)? Add a module-level `pytest_generate_tests(metafunc)` in your test file. Don't put it in `conftest.py` unless it applies broadly. ### 4. Use the existing helpers -`conftest.py` exports a deliberate API. Prefer these over rolling your own: +The `tests/harness/` package exports a deliberate API (re-exported through `conftest`). Prefer these over rolling your own: | Helper | Purpose | |--------|---------| @@ -321,7 +351,7 @@ Conventions: ### 6. Fixture extension -If multiple tests need the same setup, add a fixture in `conftest.py` (not in your test file) so it's available repo-wide. Mirror the `airstack_env` pattern: yield a dict, narrate via `logger_to(log)`, record any setup/teardown timing as metrics. +If multiple tests need the same setup, add a fixture in `conftest.py` (not in your test file) so it's available repo-wide. Mirror the `airstack_env` pattern: yield a dict, log progress via the shared `logger` (output streams to the terminal via `log_cli`), record any setup/teardown timing as metrics. ## Common Pitfalls @@ -330,10 +360,10 @@ If multiple tests need the same setup, add a fixture in `conftest.py` (not in yo - **Running on insufficient hardware**. `liveliness`, `sensors`, and `takeoff_hover_land` require an NVIDIA GPU plus nvidia-container-toolkit; without them the sim container won't get GPU access and topic Hz checks will time out. If you only have a CPU, scope to `-m "build_docker or build_packages"`. - **Expecting interactive sim feedback**. `airstack_env` runs headless by default (`MS_AIRSIM_HEADLESS=true`, `ISAAC_SIM_HEADLESS=true`, `QT_QPA_PLATFORM=offscreen`). Don't add stdin prompts, GUI dialogs, or `input()` calls to test code — they will hang in CI. For local visual debugging only, pass `--gui`. - **Not capturing metrics in a new test**. If a test fails silently (no metric recorded) the regression report has nothing to compare. Always record at least one scalar via `MetricsRecorder` so the test shows up in `metrics.json`. -- **Letting parametrize cardinality explode**. Defaults `--sim msairsim,isaacsim --num-robots 1,3` with `--stress-iterations 3` multiply stack bring-ups for each selected mark (`liveliness`, `sensors`, `takeoff_hover_land`, …) — expensive. Override locally to a single tuple while iterating. +- **Letting parametrize cardinality explode**. Default `--num-robots 1,3` (and `--sim msairsim` if you opt in) multiplies stack bring-ups for each selected mark (`liveliness`, `sensors`, `takeoff_hover_land`, …) — expensive. Override locally to a single tuple while iterating. `--sim` defaults to `isaacsim` only. - **Hardcoded container names**. Always use `find_container`, `get_robot_containers`, or `wait_for_container` — replica suffixes (`-1`, `-2`, `-3`) and compose project prefixes change. -- **Asserting on stdout instead of using `read_log_tail`**. The conftest tees subprocess output to per-test log files; assertions should reference those logs (`f"airstack up failed:\n{read_log_tail()}"`) so failures attach the relevant context to the JUnit XML. -- **Trying to SSH into a CI runner mid-job**. Workers are ephemeral OpenStack VMs destroyed within ~30s of job completion. Re-running the job creates a fresh VM. For genuine debugging on the runner, see `.github/orchestrator/README.md` (also exposed at `tests/ci-cd-orchestrator.md`) — but in 99% of cases, reproduce locally with `airstack test`. +- **Asserting on stdout instead of using `read_log_tail`**. The conftest captures each subprocess's combined stdout/stderr in memory; assertions should reference it via `read_log_tail()` (`f"airstack up failed:\n{read_log_tail()}"`) so failures attach the relevant context to the JUnit XML. +- **Trying to SSH into a CI runner mid-job**. Workers are ephemeral OSMO pods destroyed after job completion. Re-running creates a fresh pod. For genuine runner debugging, see `.github/orchestrator/README.md` (also exposed at `tests/ci-cd-orchestrator.md`) — but in most cases, reproduce locally with `airstack test`. - **Forgetting to register a new mark**. Adding `@pytest.mark.my_new_mark` without updating `tests/pytest.ini` produces "PytestUnknownMarkWarning" and makes `-m my_new_mark` fail to filter as expected. ## Quick Reference @@ -396,12 +426,13 @@ python tests/parse_metrics.py \ ### Files to know -- `tests/conftest.py` — fixtures, helpers, `MetricsRecorder`, ordering hooks +- `tests/conftest.py` — pytest hooks + the `airstack_env` / `robot_autonomy_stack` fixtures (re-exports the harness API) +- `tests/harness/` — helpers split by concern: `session`, `discovery`, `commands`, `containers`, `metrics` (`MetricsRecorder`), `run_meta`, `test_ids`, `sim`, `collection` (ordering) - `tests/pytest.ini` — mark registration, log format - `tests/parse_metrics.py` — markdown reporter, regression diff - `tests/README.md` — user-facing docs (CLI options, output layout, CI/CD orchestrator) - `.github/workflows/system-tests.yml` — CI workflow with `/pytest` comment trigger -- `.github/orchestrator/README.md` — ephemeral OpenStack runner setup and SSH-debug procedure +- `.github/orchestrator/README.md` — ephemeral OSMO runner setup and worker-debug procedure ## References diff --git a/.agents/skills/test-in-simulation/SKILL.md b/.agents/skills/test-in-simulation/SKILL.md index c9f84b8fd..c5882a312 100644 --- a/.agents/skills/test-in-simulation/SKILL.md +++ b/.agents/skills/test-in-simulation/SKILL.md @@ -395,8 +395,9 @@ Don't just test the happy path: If module supports multi-robot: ```bash -# Launch multi-robot simulation -NUM_ROBOTS=2 airstack up isaac-sim robot +# Launch multi-robot simulation (--robots also selects the multi-drone Isaac script; +# a bare NUM_ROBOTS=2 with the single-drone default script is rejected by preflight) +airstack up --sim isaac --robots 2 # Verify each robot runs independently docker exec airstack-robot-desktop-1 bash -c "ros2 node list | grep robot" diff --git a/.agents/skills/use-airstack-cli/SKILL.md b/.agents/skills/use-airstack-cli/SKILL.md index 7a3a6ead2..564ca5324 100644 --- a/.agents/skills/use-airstack-cli/SKILL.md +++ b/.agents/skills/use-airstack-cli/SKILL.md @@ -355,9 +355,14 @@ airstack config:git-hooks # Install git pre-commit hooks airstack install # Install Docker + nvidia-container-toolkit (one time) airstack setup # Add airstack to PATH (one time per shell) airstack up # Start default profile from .env +airstack up --sim isaac|airsim # Pick the simulator (profile + URDF + Isaac script derived) +airstack up --sim isaac --robots 2 # Multi-robot (keeps NUM_ROBOTS and the sim script consistent) +airstack up --play --wait # Auto-play sim, block until flight-ready +airstack up --dry-run --sim airsim # Print + validate resolved config; start nothing +airstack ready # Wait until flight-ready (--json for scripts) airstack up robot-desktop # Start one service -AUTOLAUNCH=false airstack up robot-desktop # Start idle (for development) — IMPORTANT -NUM_ROBOTS=2 AUTOLAUNCH=false airstack up # Multi-robot, idle +airstack up --no-autolaunch robot-desktop # Start idle (for development) — IMPORTANT +airstack up --no-autolaunch --robots 2 --sim isaac # Multi-robot, idle airstack status # List running containers airstack down # Stop and remove containers airstack clean # Stop, remove containers, prune volumes/networks diff --git a/.agents/skills/use-feature-notebook/SKILL.md b/.agents/skills/use-feature-notebook/SKILL.md new file mode 100644 index 000000000..eb465b69e --- /dev/null +++ b/.agents/skills/use-feature-notebook/SKILL.md @@ -0,0 +1,97 @@ +--- +name: use-feature-notebook +description: Maintain a local, gitignored notebook/ directory that records the design spec and test results for every feature an agent implements. Trigger at the START of any feature-implementation task (create notebook/NNN-feature-slug/design_spec.md before writing code), while implementing (keep the spec's per-section status labels DESIGN/TODO / WIP / DONE current), whenever tests for that feature produce output worth keeping (store under results/
/), and when opening the feature's PR (populate the PR body from results/results_summary.md). +license: Apache-2.0 +metadata: + author: AirLab CMU + repository: AirStack +--- + +# Skill: Use the Feature Notebook + +## Purpose + +Every feature implemented by a coding agent gets a **notebook entry**: a numbered folder under `notebook/` at the repo root that holds the design spec written *before* implementation and the test results produced *during* validation. The notebook is the agent's lab journal — it captures the session context that would otherwise be lost when the conversation ends, and it is the source material for the feature's PR description. + +`notebook/` is **gitignored and local-only**. It never lands in a commit. Each developer's machine has its own copy. What *does* leave the machine is the distilled content: the PR body is populated from `results/results_summary.md`, and figures/tables from `results/` are attached to the PR. + +## Directory Layout + +``` +notebook/ +├── 001-add-new-planner/ +│ ├── design_spec.md # Written BEFORE implementation +│ └── results/ +│ ├── results_summary.md # Written AFTER tests; feeds the PR +│ ├── a-planner-core/ # Raw artifacts for test section (a) +│ │ ├── run1_metrics.json +│ │ └── trajectory_plot.png +│ └── b-planner-hyperparameters/ # Raw artifacts for test section (b) +│ └── sweep_table.csv +├── 002-fix-lidar-filter/ +│ └── ... +``` + +Naming rules: + +- **Feature folder:** `NNN-short-kebab-slug`, where `NNN` is zero-padded three digits. Pick the next number by listing `notebook/` and incrementing the highest existing prefix (start at `001` if empty or missing — create `notebook/` yourself, it is not committed). +- **Results subfolders:** one per lettered test section in `design_spec.md`, named `-` (e.g. section "(a) Planner core" → `results/a-planner-core/`). The letters MUST match the test-plan section letters in the spec so a reader can navigate spec ↔ results directly. + +**Date and timestamp everything.** The notebook is a lab journal, and a journal entry without a date is unusable later. Every design doc, experiment, and results file records when it happened: + +- `design_spec.md` header: `Date started` and `Last updated` (update the latter whenever you revise the spec), as `YYYY-MM-DD`. +- Each test run stored under `results/-/`: record the run timestamp (`YYYY-MM-DD HH:MM` local time) — keep the harness's timestamped directory name when copying from `tests/results//`, or prefix artifact filenames / note the timestamp in the section of `results_summary.md`. +- `results_summary.md` header: the date written; each per-section **Setup** line: when that run was executed. + +## Workflow + +### 1. On starting a feature — write `design_spec.md` + +Before writing any implementation code, create `notebook/NNN-feature-slug/design_spec.md` from [assets/design_spec_template.md](assets/design_spec_template.md). It must capture: + +- **Problem context** — what the developer is trying to solve, in the developer's own framing from the session: motivation, constraints, prior attempts, and any decisions already made in the conversation. This is the section that preserves context which exists nowhere else. +- **Proposed implementation** — the design: affected packages, new/changed nodes and topics, algorithms, data flow. Diagrams (mermaid) welcome. Split into subsections if the implementation has multiple parts. +- **Test plan** — lettered sections `(a)`, `(b)`, `(c)`… each describing one validation axis: what is run (unit test, system test mark, sim scenario), what is measured, and what outcome counts as pass. These letters define the `results/` subfolder names. + +If the design changes materially mid-implementation, update the spec — it should describe what was actually built, with a short note on what changed and why. + +### 2. While implementing — keep the spec's status labels current + +`design_spec.md` carries an implementation status at two levels, using the values **`DESIGN/TODO`**, **`WIP`**, or **`DONE`**: + +- **Overall status** in the header block — the least-advanced status of any implementation section (all sections `DONE` → overall `DONE`; anything in progress → `WIP`; nothing started → `DESIGN/TODO`). +- **Per-section status** on each Proposed Implementation subsection heading (e.g. `### 2.1 Cost-map integration — \`WIP\``) — so when the implementation has multiple parts, a reader can see exactly which parts are designed, in progress, or finished. + +Update the labels **as you work**, not retroactively: mark a section `WIP` when you start writing its code and `DONE` when it is implemented and building. A spec whose statuses lag reality misleads the next agent that picks up the feature. + +### 3. During validation — store raw results + +Every test run that validates the feature drops its artifacts into the matching section folder, e.g. `notebook/001-add-new-planner/results/a-planner-core/`: + +- Metrics files (`metrics.json`, CSVs), copied from `tests/results//` when using the system test harness +- Plots and screenshots (cross-track error curves, Foxglove/RViz captures, sim screenshots) +- Relevant log excerpts — excerpts, not full container logs + +Keep raw artifacts as-produced; interpretation belongs in the summary. Preserve the run's timestamp with the artifacts (keep the `tests/results//` directory name, or timestamp-prefix the copied files) so repeated runs of the same section stay distinguishable and ordered. + +### 4. After validation — write `results/results_summary.md` + +Create `results/results_summary.md` from [assets/results_summary_template.md](assets/results_summary_template.md). One section per test-plan letter, mirroring the spec. The summary must be **self-contained**: embed the quantitative tables and qualitative figures directly in the document (markdown tables; images via relative paths like `![xte](a-planner-core/trajectory_plot.png)`) so a developer can understand the results all at once without opening the raw artifact folders. End with an overall verdict: which spec sections passed, which didn't, known limitations. + +### 5. On opening the PR — populate it from the notebook + +The PR body for the feature is built from the notebook, since reviewers cannot see `notebook/` itself: + +- **Motivation / context** ← `design_spec.md` problem context +- **What changed** ← proposed implementation (as-built) +- **Validation** ← `results_summary.md`: paste the summary tables, upload the key figures as PR attachments, and state the per-section verdicts + +## Pitfalls + +- ❌ Writing the spec after the code — the spec exists to record intent and session context before they're lost. +- ❌ Stale status labels — a spec still marked `DESIGN/TODO` (or a section marked `WIP`) after the work shipped misleads the next reader; update statuses as you go. +- ❌ Committing `notebook/` or referencing `notebook/...` paths from committed code, docs, or tests — it doesn't exist on other machines or in CI. +- ❌ Results subfolder letters that don't match the spec's test-plan letters. +- ❌ Undated documents or results — a spec without `Date started`/`Last updated`, or test artifacts with no run timestamp, can't be sequenced against other runs or the code they tested. +- ❌ A `results_summary.md` that just links to raw files — embed the tables and figures. +- ❌ Confusing this with [capture-discovered-knowledge](../capture-discovered-knowledge): the notebook records *per-feature* design and evidence locally; durable repo-wide knowledge still goes to AGENTS.md/skills, and module documentation still follows [update-documentation](../update-documentation). diff --git a/.agents/skills/use-feature-notebook/assets/design_spec_template.md b/.agents/skills/use-feature-notebook/assets/design_spec_template.md new file mode 100644 index 000000000..a50c2c92f --- /dev/null +++ b/.agents/skills/use-feature-notebook/assets/design_spec_template.md @@ -0,0 +1,58 @@ +# Design Spec: + +> Notebook entry: `notebook/NNN-feature-slug/` · Date started: YYYY-MM-DD · Last updated: YYYY-MM-DD · Branch: `` +> +> **Status: `DESIGN/TODO`** + +## 1. Problem Context + + + +## 2. Proposed Implementation + + + +### 2.1 — `DESIGN/TODO` + + + +### 2.2 — `DESIGN/TODO` + + + +### Affected packages + +| Package | Change | +|---------|--------| +| `path/to/package` | ... | + +### Interfaces + +| Topic / Service / Param | Type | Direction | Purpose | +|-------------------------|------|-----------|---------| +| | | | | + +## 3. Test Plan + + + +### (a)
+ +- **What is run:** +- **What is measured:** +- **Pass criteria:** + +### (b)
+ +- **What is run:** +- **What is measured:** +- **Pass criteria:** diff --git a/.agents/skills/use-feature-notebook/assets/results_summary_template.md b/.agents/skills/use-feature-notebook/assets/results_summary_template.md new file mode 100644 index 000000000..7a5929a0f --- /dev/null +++ b/.agents/skills/use-feature-notebook/assets/results_summary_template.md @@ -0,0 +1,33 @@ +# Results Summary: + +> Spec: [`../design_spec.md`](../design_spec.md) · Date: YYYY-MM-DD · Commit tested: `` + + + +## (a)
+ +**Setup:** +**Run at:** YYYY-MM-DD HH:MM + +| Metric | Value | Pass criterion | Pass? | +|--------|-------|----------------|-------| +| | | | | + +![description](a-section-slug/figure.png) + +**Interpretation:** + +## (b)
+ +... + +## Overall Verdict + +| Spec section | Verdict | +|--------------|---------| +| (a) ... | ✅ / ❌ | +| (b) ... | ✅ / ❌ | + +**Known limitations:** diff --git a/.agents/skills/write-isaac-sim-scene/SKILL.md b/.agents/skills/write-isaac-sim-scene/SKILL.md index 2f46c75d3..c1de34ff9 100644 --- a/.agents/skills/write-isaac-sim-scene/SKILL.md +++ b/.agents/skills/write-isaac-sim-scene/SKILL.md @@ -1,674 +1,131 @@ --- name: write-isaac-sim-scene -description: Create custom simulation environments in Isaac Sim using standalone Python scripts with Pegasus extension. Use when creating test scenarios, multi-robot simulations, or custom environments for testing autonomy modules. +description: Create custom simulation scenarios in Isaac Sim by declaring them on top of the shared pegasus_app.PegasusApp base class. Use when creating test scenarios, multi-robot simulations, or custom environments for testing autonomy modules. license: Apache-2.0 metadata: author: AirLab CMU repository: AirStack --- -# Skill: Write Isaac Sim Scene in Standalone Python Mode +# Skill: Write an Isaac Sim Scene (Standalone Launch Script) ## When to Use Creating custom simulation environments for testing autonomy modules, multi-robot scenarios, or specific environmental conditions. -## Prerequisites - -- Isaac Sim container running or accessible -- Understanding of Pegasus Simulator extension for drones -- Knowledge of required sensors and vehicle configuration -- Familiarity with Python and basic Isaac Sim concepts +## The One Rule That Matters -## Isaac Sim Integration Overview +**Do NOT copy-paste an existing launch script wholesale.** All shared boilerplate (SimulationApp creation, extension enabling, Pegasus world + environment loading, stage prep, drone/sensor spawning, the run loop) lives once in `simulation/isaac-sim/launch_scripts/pegasus_app.py`. A launch script is a *scenario declaration*: an environment URL, a list of drone configs, sensor toggles, and (only if needed) hook overrides. If you find yourself copying more than ~50 lines, you are re-creating the duplication this base class removed. -AirStack uses NVIDIA Isaac Sim with the **Pegasus Simulator extension** for high-fidelity drone simulation. There are two ways to define scenes: +## Prerequisites -1. **USD Files:** Static scene description files (`.usd` format) -2. **Standalone Python Scripts:** Dynamic scene creation with full programmatic control (recommended for complex scenarios) +- Isaac Sim container image present (`airstack image-pull`) +- The scenario you want: which environment, how many drones, which sensors -This skill covers **standalone Python mode**. +## How a Scene Reaches the Simulator -## Script Structure Overview +`airstack up --sim isaac` starts the isaac-sim service, which (with `.env`'s default `ISAAC_SIM_USE_STANDALONE=true`) runs the Python file named by `ISAAC_SIM_SCRIPT_NAME` from `simulation/isaac-sim/launch_scripts/`. Scripts must live in that directory; set the variable to the filename only. -Standalone Python scripts follow this pattern: +Env vars every script honors automatically (via the base class — do not re-implement): -``` -1. Start SimulationApp (BEFORE any omni imports) -2. Import required modules -3. Enable necessary extensions -4. Create PegasusApp class - - Initialize Pegasus interface - - Load environment - - Spawn vehicles with sensors - - Setup physics and backends -5. Run simulation loop -6. Clean up -``` +| Env var | Effect | +|---|---| +| `ISAAC_SIM_HEADLESS` | run without a window | +| `ISAAC_SIM_LIVESTREAM` (+`_UDP_PORT`) | headless + WebRTC livestream | +| `PLAY_SIM_ON_START` | auto-play the timeline after setup (`airstack up --play`) | ## Steps -### 1. Create Script File +### 1. Create the script from the minimal template -**Location:** `simulation/isaac-sim/launch_scripts/.py` - -```bash -cd simulation/isaac-sim/launch_scripts/ -touch your_scene_name.py -chmod +x your_scene_name.py -``` - -### 2. Script Header and SimulationApp Initialization - -**Critical:** SimulationApp MUST be started before importing any `omni` modules. +Copy `barebones_pegasus_launch.py` (an environment, no drones) or start from this skeleton. The **import-order contract** is the only fragile part: Kit requires the `SimulationApp` to exist before any `omni.*`/`pegasus.*` import. ```python #!/usr/bin/env python -""" -Description: Brief description of your simulation scene -Author: Your Name -Date: YYYY-MM-DD - -This script creates a simulation environment for testing . -- Number of drones: X -- Sensors: Camera, LiDAR, etc. -- Environment: Description -""" - -import carb -from isaacsim import SimulationApp - -# MUST start SimulationApp before importing omni modules -# Set headless=False for GUI, headless=True for automated testing -simulation_app = SimulationApp({"headless": False}) - -# Now safe to import omni and other modules -import rclpy -print(f"[Launcher] SUCCESS: rclpy imported from {rclpy.__file__}") -``` - -### 3. Import Required Modules - -```python -import omni.kit.app -import omni.timeline -import omni.ui -from omni.isaac.core.world import World -from datetime import datetime -from pxr import UsdLux, Gf, UsdGeom - -# Pegasus imports -from pegasus.simulator.params import SIMULATION_ENVIRONMENTS, ROBOTS -from pegasus.simulator.logic.interface.pegasus_interface import PegasusInterface -from pegasus.simulator.ogn.api.spawn_multirotor import spawn_px4_multirotor_node -from pegasus.simulator.ogn.api.spawn_zed_camera import add_zed_stereo_camera_subgraph -from pegasus.simulator.ogn.api.spawn_rtx_lidar import add_rtx_lidar_subgraph -from pegasus.simulator.logic.vehicles.multirotor import Multirotor, MultirotorConfig -from pegasus.simulator.logic.state import State -from pegasus.simulator.logic.backends.px4_mavlink_backend import ( - PX4MavlinkBackend, - PX4MavlinkBackendConfig -) -from pegasus.simulator.logic.backends.ros2_backend import ROS2Backend -from scipy.spatial.transform import Rotation -import numpy as np +"""One-line description of the scenario.""" import os -import subprocess -import threading -import signal -import atexit -import time - -# Scene preparation utilities (scaling, collision, lighting, export) -# NOTE: importlib is used instead of a normal import because Isaac Sim's -# script runner does not reliably set __file__, making sys.path manipulation -# fragile. Loading the module by absolute file path is the robust approach. -import importlib.util as _ilu, os as _os -_scene_prep_path = _os.path.join(_os.path.dirname(_os.path.abspath(__file__)), "..", "utils", "scene_prep.py") -_spec = _ilu.spec_from_file_location("scene_prep", _os.path.normpath(_scene_prep_path)) -_scene_prep = _ilu.module_from_spec(_spec); _spec.loader.exec_module(_scene_prep) -scale_stage_prim = _scene_prep.scale_stage_prim -add_colliders = _scene_prep.add_colliders -add_dome_light = _scene_prep.add_dome_light -save_scene_as_contained_usd = _scene_prep.save_scene_as_contained_usd -``` - -### 4. Enable Required Extensions +import sys -```python -# Explicitly enable required extensions -ext_manager = omni.kit.app.get_app().get_extension_manager() - -# Required extensions for Pegasus and OmniGraph -required_extensions = [ - "omni.graph.core", # Core runtime for OmniGraph engine - "omni.graph.action", # Action Graph framework - "omni.graph.action_nodes", # Built-in Action Graph node library - "isaacsim.core.nodes", # Core helper nodes for OmniGraph - "omni.graph.ui", # UI scaffolding for graph tools - "omni.graph.visualization.nodes", # Visualization helper nodes - "omni.graph.scriptnode", # Python script node support - "omni.graph.window.action", # Action Graph editor window - "omni.graph.window.generic", # Generic graph UI tools - "omni.graph.ui_nodes", # UI node building helpers - "pegasus.simulator", # Pegasus Simulator extension -] - -for ext in required_extensions: - if not ext_manager.is_extension_enabled(ext): - print(f"[Launcher] Enabling extension: {ext}") - ext_manager.set_extension_enabled_immediate(ext, True) - print(f"[Launcher] Successfully enabled extension: {ext}") - else: - print(f"[Launcher] Extension already enabled: {ext}") -``` +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from pegasus_app import create_simulation_app -### 5. Create PegasusApp Class +simulation_app = create_simulation_app() # FIRST — before any omni/pegasus import -```python -class YourSceneApp: - """ - Simulation application for your specific scenario. - """ - - def __init__(self): - print("[YourScene] Initializing simulation...") - - # Start Pegasus interface - self.pg = PegasusInterface() - - # Create Isaac Sim world - self.world = World(**self.pg.world_settings) - self.pg.world = self.world - - # Dictionary to store vehicle instances - self.vehicles = {} - - # PX4 process handles (if using PX4 SITL) - self.px4_processes = [] - - # Load environment - self.load_environment() - - # Prepare environment (scale, colliders, lighting) - stage = omni.usd.get_context().get_stage() - self._prepare_environment(stage) - - # Spawn vehicles - self.spawn_vehicles() - - # Setup simulation - self.world.reset() - - print("[YourScene] Simulation initialized successfully") - - def load_environment(self): - """Load or create the simulation environment.""" - print("[YourScene] Loading environment...") - - # Option 1: Load pre-defined environment - # Available: "Grid", "Outdoor", "Office", etc. - # See SIMULATION_ENVIRONMENTS in Pegasus for options - stage = self.pg.load_environment(SIMULATION_ENVIRONMENTS["Grid"]["usd"]) - - # Option 2: Add ground plane only - # self.world.scene.add_default_ground_plane() - - # Option 3: Load custom USD environment - # stage = self.pg.load_environment("/path/to/your/environment.usd") - - # Add obstacles or other static objects - self._add_environment_objects() - - def _prepare_environment(self, stage): - """Scale, add collisions, and light the environment.""" - stage_prim = stage.GetPrimAtPath("/World/stage") - if stage_prim.IsValid(): - # STAGE_SCALE: use 0.01 for Nucleus assets authored in cm, 1.0 if already in meters - scale_stage_prim(stage, "/World/stage", STAGE_SCALE) - add_colliders(stage_prim) - # Allow physics to settle after adding colliders - for _ in range(10): - omni.kit.app.get_app().update() - # add_dome_light defaults: intensity=3500, exposure=-3 - # Override via kwargs, e.g. add_dome_light(stage, intensity=5000, exposure=-2) - add_dome_light(stage) - - def _add_environment_objects(self): - """Add obstacles or other objects to the environment.""" - # Example: Add a cube obstacle - stage = omni.usd.get_context().get_stage() - - # cube_prim = stage.DefinePrim("/World/Obstacle1", "Cube") - # UsdGeom.Xformable(cube_prim).AddTranslateOp().Set(Gf.Vec3d(5.0, 0.0, 0.5)) - # UsdGeom.Xformable(cube_prim).AddScaleOp().Set(Gf.Vec3d(1.0, 1.0, 1.0)) - - pass - - def spawn_vehicles(self): - """Spawn drone vehicles with sensors and backends.""" - print("[YourScene] Spawning vehicles...") - - # Vehicle 1: Primary drone - self._spawn_vehicle( - vehicle_id=0, - vehicle_name="drone1", - position=[0.0, 0.0, 1.0], # [x, y, z] - orientation=[0.0, 0.0, 0.0, 1.0], # quaternion [x, y, z, w] - px4_autostart_id=4001, # PX4 vehicle type (4001 = quadrotor) - mavlink_tcp_port=4560, # PX4 MAVLink port - px4_instance=0, - sensors={ - "camera": True, - "lidar": False - } - ) - - # Vehicle 2: Second drone (optional, for multi-robot) - # self._spawn_vehicle( - # vehicle_id=1, - # vehicle_name="drone2", - # position=[5.0, 0.0, 1.0], - # orientation=[0.0, 0.0, 0.0, 1.0], - # px4_autostart_id=4001, - # mavlink_tcp_port=4561, - # px4_instance=1, - # sensors={"camera": True, "lidar": True} - # ) - - def _spawn_vehicle(self, vehicle_id, vehicle_name, position, orientation, - px4_autostart_id, mavlink_tcp_port, px4_instance, - sensors=None): - """ - Spawn a single vehicle with specified configuration. - - Args: - vehicle_id: Unique vehicle ID - vehicle_name: Name for the vehicle - position: [x, y, z] spawn position - orientation: [x, y, z, w] quaternion orientation - px4_autostart_id: PX4 vehicle type ID - mavlink_tcp_port: MAVLink TCP port for PX4 communication - px4_instance: PX4 instance number - sensors: Dict of sensors to add {"camera": bool, "lidar": bool} - """ - if sensors is None: - sensors = {"camera": True, "lidar": False} - - # Configure multirotor - config = MultirotorConfig() - - # PX4 MAVLink backend configuration - px4_backend_config = PX4MavlinkBackendConfig({ - "vehicle_id": vehicle_id, - "px4_autostart": px4_autostart_id, - "px4_dir": os.environ.get("PX4_DIR", "/PX4-Autopilot"), - "px4_instance": px4_instance, - "mavlink_tcp_port": mavlink_tcp_port, - "enable_lockstep": True, - "update_rate": 250.0 # Hz - }) - - # Add ROS 2 backend for ROS communication - ros2_backend = ROS2Backend( - vehicle_id=vehicle_id, - config={ - "namespace": vehicle_name, - "pub_sensors": True, - "pub_state": True - } - ) - - # Attach backends - config.backends = [ - PX4MavlinkBackend(px4_backend_config), - ros2_backend - ] - - # Create vehicle - vehicle = Multirotor( - stage_prefix="/World", - prim_path=f"/World/{vehicle_name}", - name=vehicle_name, - usd_model=ROBOTS["Iris"]["usd"], # or other model - init_pos=position, - init_orientation=orientation, - config=config - ) - - # Add sensors - if sensors.get("camera", False): - self._add_camera_sensor(vehicle) - - # RTX LiDAR uses OmniGraph: spawn_px4_multirotor_node() returns graph_handle, - # then call self._add_lidar_sensor(vehicle, graph_handle). See - # example_one_px4_pegasus_launch_script.py for the full pattern. - - # Initialize vehicle in world - self.world.scene.add(vehicle) - self.vehicles[vehicle_name] = vehicle - - print(f"[YourScene] Spawned vehicle: {vehicle_name}") - - def _add_camera_sensor(self, vehicle): - """Add stereo camera to vehicle.""" - add_zed_stereo_camera_subgraph( - camera_prim_path=vehicle.prim_path + "/ZedCamera", - parent_prim_path=vehicle.prim_path, - config={ - "graph_evaluator": "execution", # or "push" - "resolution": (1280, 720), - "position": (0.3, 0.0, -0.1), # Relative to vehicle - "orientation": (0.0, 0.0, 0.0, 1.0), - } - ) - - def _add_lidar_sensor(self, vehicle, graph_handle): - """Add RTX LiDAR (OmniGraph subgraph) to vehicle.""" - add_rtx_lidar_subgraph( - parent_graph_handle=graph_handle, - drone_prim=vehicle.prim_path, - robot_name="robot_1", - lidar_config="ouster_os1", - lidar_offset=[0.0, 0.0, 0.025], - lidar_rotation_offset=[0.0, 0.0, 0.0], - min_range=0.75, - ) - - def run(self): - """Main simulation loop.""" - print("[YourScene] Starting simulation loop...") - - # Optionally auto-start timeline - # omni.timeline.get_timeline_interface().play() - - step_count = 0 - while simulation_app.is_running(): - # Step the simulation - self.world.step(render=True) - - # Optional: Add periodic logic - if step_count % 100 == 0: - # print(f"[YourScene] Simulation step: {step_count}") - pass - - step_count += 1 - - print("[YourScene] Simulation loop ended") - - def cleanup(self): - """Clean up resources.""" - print("[YourScene] Cleaning up...") - - # Stop PX4 processes - for process in self.px4_processes: - if process.poll() is None: # Process still running - process.terminate() - process.wait() - - self.px4_processes.clear() -``` +from pegasus.simulator.params import SIMULATION_ENVIRONMENTS # noqa: E402 +from pegasus_app import PegasusApp, row_spawn_configs # noqa: E402 -### 6. Main Entry Point -```python def main(): - """Main entry point for the simulation.""" - try: - # Create and run simulation - app = YourSceneApp() - app.run() - except Exception as e: - print(f"[YourScene] Error: {e}") - import traceback - traceback.print_exc() - finally: - # Clean up - if 'app' in locals(): - app.cleanup() - simulation_app.close() + PegasusApp( + env_url=SIMULATION_ENVIRONMENTS["Default Environment"], + drone_configs=row_spawn_configs(int(os.environ.get("NUM_ROBOTS", "1"))), + enable_lidar=os.environ.get("ENABLE_LIDAR", "false").lower() == "true", + ).run() + if __name__ == "__main__": main() ``` -### 7. Configure in .env - -Update the main `.env` file to use your script: +### 2. Declare the scenario via constructor kwargs -```bash -# Set to standalone script mode -ISAAC_SIM_USE_STANDALONE="true" +The full list with defaults is in `PegasusApp.__init__`'s signature and docstring; the ones you'll set: -# Specify your script name -ISAAC_SIM_SCRIPT_NAME="your_scene_name.py" -``` +| Kwarg | Purpose | +|---|---| +| `env_url` | A `SIMULATION_ENVIRONMENTS[...]` entry or any `omniverse://` / file USD URL | +| `drone_configs` | Per-drone dicts (below); `row_spawn_configs(n, spacing_m, z_m)` for the standard row | +| `stage_scale` | `0.01` for cm-authored Nucleus assets, `1.0` for metric scenes | +| `enable_camera`, `camera_offset` | ZED stereo subgraph per drone (default on, offset `[0.2, 0, -0.05]`) | +| `enable_lidar`, `lidar_min_range`, ... | RTX Ouster lidar subgraph per drone | +| `dome_light` | `True` (defaults), `False`, or `{"prim_path":…, "intensity":…, "exposure":…}` | +| `world_gps_origin` | `(lat, lon, alt)` — writes per-drone PX4 GPS homes before SITL boots (see [spawning_drones.md](../../../docs/simulation/isaac_sim/spawning_drones.md)) | +| `scale_spawn_positions` | `True` when spawn meters must be converted into non-metric stage units | +| `save_scene_to` | Directory to export a self-contained USD of the prepared scene | +| `extra_extensions` | Additional Kit extensions to enable | -Alternatively, override from command line: -```bash -ISAAC_SIM_USE_STANDALONE=true ISAAC_SIM_SCRIPT_NAME="your_scene_name.py" airstack up isaac-sim -``` +Per-drone config dict keys: `domain_id` (required — ROS domain and default vehicle id; MAVLink port `14540 + vehicle_id`), `x_m`/`y_m`/`z_m`, `orient` (quaternion `[x,y,z,w]`), and optional overrides `prim`, `node_name`, `lidar`, `lidar_min_range`, `camera_offset`. -### 8. Test the Scene +### 3. Custom behavior goes in hooks, not copied blocks -Launch Isaac Sim with your script: - -```bash -# Start Isaac Sim container with your scene -airstack up isaac-sim - -# Check logs for errors -airstack logs isaac-sim - -# If errors occur, connect to container for debugging -airstack connect isaac-sim -``` +Subclass `PegasusApp` and override (each receives the loaded USD stage): -### 9. Document the Scene +- `pre_scene_prep(stage)` — right after the environment loads (e.g. `dedupe_physics_scenes`, `reference_root_prims_under_world` for imported scenes) +- `post_scene_prep(stage)` — after scale/colliders/dome light, before drones (e.g. overhead map camera) +- `post_spawn(stage)` — after all drones exist (e.g. author the NatNet mocap interface) -Create a README.md next to your script: +Reference subclasses to study (not copy): `example_multi_drone_scene_import.py` (Nucleus scene import, explicit poses, overhead camera, GPS origins) and `example_multi_px4_pegasus_natnet_launch_script.py` (`post_spawn` mocap authoring). -**File:** `simulation/isaac-sim/launch_scripts/your_scene_name.md` +Stage-prep helpers live in `simulation/isaac-sim/utils/scene_prep.py` (`add_colliders`, `scale_stage_prim`, `add_dome_light`, `add_orthographic_camera`, …) — documented in [spawning_drones.md](../../../docs/simulation/isaac_sim/spawning_drones.md). -```markdown -# Your Scene Name +Scene-level things that need to happen **before Pegasus imports** (e.g. overriding the Nucleus asset root via `carb.settings`) go at script top level right after `create_simulation_app()` — see the top of `example_multi_drone_scene_import.py`. -## Overview -Brief description of the simulation scene. - -## Purpose -Why this scene was created and what it tests. - -## Configuration - -### Vehicles -- Number of drones: X -- Vehicle types: Quadrotor, fixed-wing, etc. -- Initial positions: List positions - -### Sensors -- Cameras: Resolution, FoV -- LiDAR: Model, range -- Other sensors - -### Environment -Description of the environment, obstacles, lighting. - -## Usage +### 4. Run it ```bash -# Launch scene -ISAAC_SIM_SCRIPT_NAME="your_scene_name.py" airstack up isaac-sim - -# With robot autonomy -airstack up isaac-sim robot -``` - -## Parameters -Any configurable parameters in the script. - -## Known Issues -Any limitations or known problems. +ISAAC_SIM_SCRIPT_NAME=my_scenario.py airstack up --sim isaac --play --wait ``` -## Advanced Topics +`--wait` (or `airstack ready`) blocks until the sim publishes `/clock`, the autonomy nodes are up, and PX4 is armable — so a hang here tells you which layer is broken. Watch script output from the host with `airstack logs isaac-sim` (tmux panes are mirrored to docker logs) or attach with `airstack connect isaac-sim`. -### Scene Preparation Utilities - -**File:** `simulation/isaac-sim/utils/scene_prep.py` - -Four reusable helpers that cover the most common environment setup tasks. Import them as shown in Step 3. - -| Function | When to use | -|----------|-------------| -| `scale_stage_prim(stage, prim_path, scale)` | Nucleus assets authored in centimeters need `STAGE_SCALE=0.01`; assets already in meters use `1.0`. | -| `add_colliders(stage_prim)` | **Must** be called for physics to interact with environment meshes. Without it drones fall through the floor. Call after scaling. | -| `add_dome_light(stage, **kwargs)` | Adds uniform hemisphere lighting. Defaults: `intensity=3500`, `exposure=-3`. Pass kwargs to override, e.g. `add_dome_light(stage, intensity=5000)`. | -| `save_scene_as_contained_usd(src_url, output_dir)` | Copies a Nucleus-hosted stage (and all its textures/MDLs) to a local directory using `omni.kit.usd.collect.Collector`. Useful for archiving or offline replay. | - -**Two-step save pattern** used internally by `save_scene_as_contained_usd`: -1. `export_as_stage_async` — writes a flat `.usd` of the live stage -2. `Collector` — resolves and copies all referenced Nucleus assets locally - -Set `SAVE_SCENE_TO = None` in your script to skip saving entirely. - ---- - -### Multi-Robot Scenarios - -For multiple robots, spawn additional vehicles with unique IDs and ports: - -```python -def spawn_vehicles(self): - for i in range(num_robots): - self._spawn_vehicle( - vehicle_id=i, - vehicle_name=f"drone{i}", - position=[i * 5.0, 0.0, 1.0], # Space them out - orientation=[0.0, 0.0, 0.0, 1.0], - px4_autostart_id=4001, - mavlink_tcp_port=4560 + i, # Unique port per vehicle - px4_instance=i, - sensors={"camera": True, "lidar": False} - ) -``` - -### Custom Sensor Configuration - -Create custom sensor configurations: - -```python -def _add_custom_camera(self, vehicle, config): - """Add camera with custom parameters.""" - add_zed_stereo_camera_subgraph( - camera_prim_path=vehicle.prim_path + "/CustomCamera", - parent_prim_path=vehicle.prim_path, - config={ - "resolution": config.get("resolution", (1920, 1080)), - "horizontal_fov": config.get("fov", 90.0), - "position": config.get("position", (0.3, 0.0, 0.0)), - "orientation": config.get("orientation", (0.0, 0.0, 0.0, 1.0)), - } - ) -``` - -### Dynamic Obstacles - -Add moving obstacles: - -```python -def _add_dynamic_obstacle(self): - """Add a moving obstacle to the scene.""" - from omni.isaac.core.objects import DynamicCuboid - - obstacle = DynamicCuboid( - prim_path="/World/DynamicObstacle", - position=[10.0, 0.0, 1.0], - scale=[1.0, 1.0, 1.0], - color=[1.0, 0.0, 0.0] # Red - ) - self.world.scene.add(obstacle) - - # In simulation loop, update position - # obstacle.set_world_pose(position=[x, y, z]) -``` - -## Common Pitfalls - -### SimulationApp Import Order -- ❌ **Importing omni modules before SimulationApp** - - ✅ ALWAYS create SimulationApp first, then import omni modules - -### Extension Loading -- ❌ **Missing required extensions** - - ✅ Enable all required extensions before using their features - - ✅ Check extension status with `ext_manager.is_extension_enabled()` - -### PX4 Port Conflicts -- ❌ **Using same MAVLink port for multiple vehicles** - - ✅ Each vehicle needs unique mavlink_tcp_port - - ✅ Increment port number for each vehicle: 4560, 4561, 4562, ... - -### Sensor Configuration -- ❌ **Incorrect sensor placement (inside vehicle mesh)** - - ✅ Position sensors outside vehicle collision geometry - - ✅ Typical camera position: forward of vehicle center - -### Missing Colliders on Environment Meshes -- ❌ Loading a Nucleus environment without calling `add_colliders()` - - ✅ Call `add_colliders(stage_prim)` after scaling — drones will fall through the floor otherwise - -### World Reset -- ❌ **Not calling world.reset()** - - ✅ Call world.reset() after adding all objects before stepping - -## Debugging - -### View Scene in GUI - -Run with headless=False to see the scene: -```python -simulation_app = SimulationApp({"headless": False}) -``` - -### Print Vehicle Info - -```python -def run(self): - while simulation_app.is_running(): - self.world.step(render=True) - - # Print vehicle state - for name, vehicle in self.vehicles.items(): - pos, ori = vehicle.get_world_pose() - print(f"{name}: pos={pos}, ori={ori}") -``` - -### Check ROS 2 Topics - -```bash -# From another terminal, check topics are publishing -docker exec airstack-isaac-sim-1 bash -c "ros2 topic list" -docker exec airstack-isaac-sim-1 bash -c "ros2 topic hz /drone1/sensors/camera/image" -``` +For multi-drone scenarios, `airstack up --sim isaac --robots N` keeps `NUM_ROBOTS` (robot containers) and the launch script consistent; if your custom script reads `NUM_ROBOTS`, say so in its docstring — preflight warns when `--robots > 1` is used with a custom script name. -## References +### 5. Verify -- **Pegasus Simulator:** - - [Pegasus GitHub](https://github.com/PegasusSimulator/PegasusSimulator) - - [Pegasus Documentation](https://pegasussimulator.github.io/PegasusSimulator/) +1. `python3 -m py_compile simulation/isaac-sim/launch_scripts/my_scenario.py` +2. `airstack up --dry-run --sim isaac` with your `ISAAC_SIM_SCRIPT_NAME` — preflight validates the config +3. Full bring-up with `--wait`; then `ros2 topic hz` the sensor topics per drone (see the debug-module skill) +4. For scenarios meant to gate CI: run the relevant system-test marks (`airstack test -m liveliness --sim isaacsim ...`) -- **Isaac Sim:** - - [Isaac Sim Documentation](https://docs.omniverse.nvidia.com/isaacsim/latest/index.html) - - [USD Introduction](https://docs.omniverse.nvidia.com/py/isaacsim/source/extensions/omni.isaac.core/docs/index.html) +## Pitfalls -- **AirStack Examples:** - - Single drone: `simulation/isaac-sim/launch_scripts/example_one_px4_pegasus_launch_script.py` - - Multiple drones: `simulation/isaac-sim/launch_scripts/example_multi_px4_pegasus_launch_script.py` +- ❌ Importing anything `omni.*`/`pegasus.*` before `create_simulation_app()` — Kit crashes or hangs +- ❌ Copying the extension-enable loop / run loop / stage-prep blocks into your script — they're in the base class +- ❌ Duplicate `domain_id`s in `drone_configs` — port and domain collisions, silent MAVROS failures +- ❌ Hardcoding a drone count while robot containers scale with `NUM_ROBOTS` — extra robots will wait forever for a PX4 that doesn't exist +- ❌ Forgetting `scale_spawn_positions=True` for cm-authored scenes — drones spawn 100× too far apart +- ❌ Re-reading `PLAY_SIM_ON_START`/`ISAAC_SIM_HEADLESS` yourself — the base class already does -- **Scene Preparation Utilities:** - - `simulation/isaac-sim/utils/scene_prep.py` +## Documentation -- **Related Skills:** - - [test-in-simulation](../test-in-simulation) - Testing modules in Isaac Sim - - [debug-module](../debug-module) - Debugging simulation issues +Follow [update-documentation](../update-documentation): a scenario intended for others should be mentioned in `docs/simulation/isaac_sim/index.md` and, if it introduces new patterns, documented alongside [spawning_drones.md](../../../docs/simulation/isaac_sim/spawning_drones.md). diff --git a/.airstack/modules/osmo.sh b/.airstack/modules/osmo.sh new file mode 100755 index 000000000..053decbee --- /dev/null +++ b/.airstack/modules/osmo.sh @@ -0,0 +1,719 @@ +#!/usr/bin/env bash + +# osmo.sh — AirStack-on-OSMO convenience commands. +# +# Wraps `osmo workflow submit/port-forward/logs/cancel` for the +# osmo/workflows/airstack-dev.yaml workflow so a Mac/Windows student doesn't +# have to memorize the WebRTC port range or the entry-script path. +# +# This module is pure bash + the cross-platform `osmo` CLI — no Docker +# dependency. Safe to run on a laptop with no AirStack runtime. +# +# Most commands need a workflow id. `osmo:up` saves the id to +# $OSMO_STATE_FILE; the other commands read it from there. You can also +# override it for a single invocation by exporting AIRSTACK_OSMO_WF. + +# State directory and file: ~/.airstack/osmo-state stores the most recent +# workflow id submitted with `airstack osmo:up`. +OSMO_STATE_DIR="${HOME}/.airstack" +OSMO_STATE_FILE="${OSMO_STATE_DIR}/osmo-state" + +# WebRTC livestream ports — must match the ports published by the +# isaac-sim-livestream service in +# simulation/isaac-sim/docker/docker-compose.yaml AND the +# app.livestream.fixedHostPort setting pinned in the Pegasus launch script +# (simulation/isaac-sim/launch_scripts/example_one_px4_pegasus_launch_script.py). +# +# Two ports total: +# TCP 49100 — omni.kit.livestream.webrtc WebSocket signaling +# UDP 49099 — SRTP media (pinned; Kit 107 otherwise picks dynamically and +# escapes both the compose-published and CLI-forwarded ranges) +OSMO_WEBRTC_TCP="49100" +OSMO_WEBRTC_UDP="49099" + +# GCS Foxglove websocket: container 8765 → host 8766 (per +# gcs/docker/docker-compose.yaml). +OSMO_FOXGLOVE_PORT="8766:8766" + +# SSH port-forward: local 2200 → pod 22. +OSMO_SSH_PORT="2200:22" + +# Default `osmo workflow port-forward` connect-timeout (24h). +OSMO_PF_TIMEOUT="${OSMO_PF_TIMEOUT:-86400}" + +# Helper: ensure the osmo CLI is on PATH. +function _osmo_check_cli { + if ! command -v osmo >/dev/null 2>&1; then + log_error "osmo CLI not found on PATH. Install from https://github.com/NVIDIA/OSMO and run 'osmo login'." + return 1 + fi +} + +# Helper: strip leading/trailing whitespace + CR/NUL bytes from the +# variable named in $1. +# +# Why this exists: bracket-paste mode and cross-OS clipboards (RDP, VNC, +# Windows-side note apps) routinely smuggle invisible bytes around long +# pastes — Nucleus API tokens (JWT, ~1 KB) and SSH keys are the usual +# victims. Nucleus's auth endpoint silently `DENIES` a token that has +# one extra trailing byte, with no actionable error from the client side. +# Stripping defensively at prompt time saves an entire round-trip of +# "regenerate token → still denied → check auth-service log" debugging. +function _osmo_trim { + local var_name="$1" + local val="${!var_name}" + local original_len="${#val}" + val="${val//$'\r'/}" + val="${val//$'\0'/}" + val="${val#"${val%%[![:space:]]*}"}" + val="${val%"${val##*[![:space:]]}"}" + if [ "${#val}" -ne "$original_len" ]; then + log_warn "Stripped $((original_len - ${#val})) whitespace/control byte(s) from ${var_name}." + fi + printf -v "$var_name" '%s' "$val" +} + +# Helper: read a value with prompt; supports -s for silent (passwords). +# +# Visible prompts switch the TTY out of canonical mode for the duration of +# the read. Without this, macOS caps each input line at MAX_CANON = 1024 +# bytes (per ) and rings the terminal bell on Enter when +# the buffer overflows. Nucleus API tokens are JWTs ~950 bytes long, so +# `Nucleus API token: ` lands right at the cap. `stty -icanon` makes +# the kernel deliver bytes to bash as they're typed, with no line-buffer +# limit; bash's `read` still terminates on newline normally. +# +# We use a trap to guarantee the saved stty is restored if the user Ctrl-Cs +# mid-paste — otherwise the shell would be left in raw mode. +# +# After reading we always run _osmo_trim — see comment there. +function _osmo_prompt { + local var_name="$1" + local prompt_text="$2" + local silent="${3:-false}" + local saved_stty="" + + if [ "$silent" = "true" ]; then + # Passwords are short — canonical-mode cap is fine here. + read -r -s -p "${prompt_text}: " "$var_name" + printf "\n" >&2 + else + if [ -t 0 ]; then + saved_stty="$(stty -g 2>/dev/null || true)" + if [ -n "$saved_stty" ]; then + trap 'stty "$saved_stty" 2>/dev/null; trap - INT' INT + stty -icanon 2>/dev/null + fi + fi + read -r -p "${prompt_text}: " "$var_name" + if [ -n "$saved_stty" ]; then + stty "$saved_stty" 2>/dev/null + trap - INT + fi + fi + + _osmo_trim "$var_name" + + if [ -z "${!var_name}" ]; then + log_error "Empty input for ${var_name}; aborting." + return 1 + fi +} + +# osmo:setup — interactively register the three OSMO credentials AirStack +# needs (airlab-docker-registry, airlab-docker-login, airlab-nucleus). +# Idempotent — re-running rotates the credentials. +function cmd_osmo_setup { + _osmo_check_cli || return 1 + + cat >&2 <<'EOF' + +This sets up the three per-user OSMO credentials AirStack-on-OSMO needs: + + 1. airlab-docker-registry (REGISTRY) — for OSMO to pull the workspace image + 2. airlab-docker-login (GENERIC) — for the inner dockerd to pull AirStack images + 3. airlab-nucleus (GENERIC) — for Isaac Sim Nucleus access + +You'll be asked for: + + - your Andrew ID (no @andrew.cmu.edu suffix) + - your AirLab Docker password (same as your Andrew password) + - your Nucleus API token (https://airlab-nucleus.andrew.cmu.edu/omni/web3/ + → right-click cloud → API Tokens). NOT your Andrew password. + +Values go directly to OSMO; nothing is written to disk locally. + +EOF + + local andrew_id andrew_password nucleus_token + _osmo_prompt andrew_id "Andrew ID" false || return 1 + _osmo_prompt andrew_password "AirLab Docker password (hidden)" true || return 1 + _osmo_prompt nucleus_token "Nucleus API token" false || return 1 + + # Sanity-check the Nucleus token shape. Nucleus issues RS256 JWTs: + # base64url(header).base64url(payload).base64url(signature), with the + # header always starting `eyJ` (base64url of `{"`). Catching a wrong + # paste here (e.g. Andrew password, or token without the trailing + # signature segment) saves the user from a silent `InternalCredentials + # .auth: DENIED` round-trip later on. We do not validate the signature. + case "$nucleus_token" in + eyJ*.*.*) ;; # looks like a 3-segment JWT + *) + log_error "That doesn't look like a Nucleus API token." + log_error " - Expected: a JWT of the form eyJ……… (~1 KB long)" + log_error " - Got: ${#nucleus_token} chars, prefix '$(printf '%s' "$nucleus_token" | head -c 8)…'" + log_error " Generate one at https://airlab-nucleus.andrew.cmu.edu/omni/web3/" + log_error " → right-click cloud icon → API Tokens → Create." + return 1 + ;; + esac + + local omni_server="${OMNI_SERVER:-omniverse://airlab-nucleus.andrew.cmu.edu/NVIDIA/Assets/Isaac/5.1}" + local airlab_registry="${AIRLAB_REGISTRY:-airlab-docker.andrew.cmu.edu}" + + # `osmo credential set` is NOT an upsert for GENERIC credentials — re-setting + # one that already exists fails with `400 duplicate key value violates unique + # constraint "credential_pkey"`. Delete first so re-running osmo:setup + # (e.g. to rotate a Nucleus token) is idempotent. The `|| true` swallows the + # "credential not found" case on a first-time run. + log_info "Refreshing airlab-docker-registry (REGISTRY)..." + osmo credential delete airlab-docker-registry >/dev/null 2>&1 || true + osmo credential set airlab-docker-registry \ + --type REGISTRY \ + --payload "registry=${airlab_registry}" \ + "username=${andrew_id}" \ + "auth=${andrew_password}" \ + || { log_error "osmo credential set airlab-docker-registry failed"; return 1; } + + log_info "Refreshing airlab-docker-login (GENERIC)..." + osmo credential delete airlab-docker-login >/dev/null 2>&1 || true + osmo credential set airlab-docker-login \ + --type GENERIC \ + --payload "username=${andrew_id}" \ + "password=${andrew_password}" \ + || { log_error "osmo credential set airlab-docker-login failed"; return 1; } + + log_info "Refreshing airlab-nucleus (GENERIC)..." + osmo credential delete airlab-nucleus >/dev/null 2>&1 || true + osmo credential set airlab-nucleus \ + --type GENERIC \ + --payload "omni_user=${andrew_id}" \ + "omni_pass=${nucleus_token}" \ + "omni_server=${omni_server}" \ + || { log_error "osmo credential set airlab-nucleus failed"; return 1; } + + log_info "All three credentials registered. List them with: osmo credential list" + log_info "Next: airstack osmo:up [--pool POOL]" +} + +# Helper: pick the first existing SSH public key on the host. +function _osmo_pick_pubkey { + local candidates=( + "${HOME}/.ssh/id_ed25519.pub" + "${HOME}/.ssh/id_ecdsa.pub" + "${HOME}/.ssh/id_rsa.pub" + ) + for k in "${candidates[@]}"; do + if [ -f "$k" ]; then + echo "$k" + return 0 + fi + done + return 1 +} + +# Helper: get the active workflow id (env override first, then state file). +# +# The state file persists across shell sessions, so it can easily go stale +# (e.g. a previous airstack-dev-N is now FAILED/CANCELED). To avoid the +# confusing "Workflow airstack-dev-10 is not running!" 410 error from the +# downstream osmo command, this helper verifies the saved id is still in a +# live state (PENDING / RUNNING) before returning it. +function _osmo_wf_id { + local wf + if [ -n "${AIRSTACK_OSMO_WF:-}" ]; then + wf="${AIRSTACK_OSMO_WF}" + elif [ -f "${OSMO_STATE_FILE}" ]; then + wf="$(cat "${OSMO_STATE_FILE}")" + else + log_error "No workflow id found. Run 'airstack osmo:up' first, or export AIRSTACK_OSMO_WF=." + return 1 + fi + + # Validate the workflow is still alive (only when osmo CLI is available). + if command -v osmo >/dev/null 2>&1; then + local status + status="$(osmo workflow query "${wf}" 2>/dev/null | awk -F': +' '/^Status/ {print $2; exit}' | tr -d ' \r\n')" + case "${status}" in + PENDING|RUNNING|"") + # "" means we couldn't reach osmo; let the downstream + # command surface the real error rather than failing here. + ;; + *) + log_error "Saved workflow '${wf}' is ${status}, not running." + log_warn "Run 'airstack osmo:up' to launch a fresh one, or:" + log_warn " rm ${OSMO_STATE_FILE}" + log_warn " export AIRSTACK_OSMO_WF=" + return 1 + ;; + esac + fi + + echo "${wf}" + return 0 +} + +# Helper: persist the workflow id. +function _osmo_save_wf_id { + mkdir -p "${OSMO_STATE_DIR}" + echo "$1" > "${OSMO_STATE_FILE}" + log_info "Saved workflow id '$1' to ${OSMO_STATE_FILE}" +} + +# Helper: best-effort detection of the user's current AirStack branch so +# `airstack osmo:up` can default --branch to whatever the user is editing +# locally. Returns the branch name on stdout, or empty if we shouldn't +# auto-pin (detached HEAD, not a git repo, etc.). +# +# Why default to the local branch: the pod's entrypoint clones AirStack +# fresh from GitHub on every workflow start (the pod fs is ephemeral, so +# nothing else makes sense). If we don't tell it which branch, it +# defaults to `main` — and any developer testing branch-only OSMO +# changes (compose services, entrypoint tweaks, workflow yaml edits) +# silently runs against stale `main` code instead of their work. +# Defaulting to the local branch makes "edit on laptop, push, osmo:up" +# the natural workflow. +function _osmo_local_branch { + if ! command -v git >/dev/null 2>&1; then + return 0 + fi + local b + b="$(git -C "${PROJECT_ROOT}" rev-parse --abbrev-ref HEAD 2>/dev/null)" || return 0 + case "$b" in + ""|HEAD) return 0 ;; # detached HEAD or empty + esac + echo "$b" +} + +# Helper: warn if the about-to-submit branch isn't safely pushed. The +# pod clones from GitHub, so unpushed commits / dirty working tree don't +# make it into the pod even if the user thinks they did. Catching this +# before submit avoids a 60-90s "wait for pod, then realize" round trip. +function _osmo_check_branch_pushed { + local branch="$1" + command -v git >/dev/null 2>&1 || return 0 + local repo="${PROJECT_ROOT}" + [ -d "${repo}/.git" ] || return 0 + + local local_sha upstream_sha + local_sha="$(git -C "$repo" rev-parse "${branch}" 2>/dev/null)" || return 0 + + # Look for a remote-tracking branch first (the explicit upstream + # set by `git push -u`); fall back to origin/. + upstream_sha="$(git -C "$repo" rev-parse "${branch}@{upstream}" 2>/dev/null)" + if [ -z "$upstream_sha" ]; then + upstream_sha="$(git -C "$repo" rev-parse "origin/${branch}" 2>/dev/null)" + fi + + if [ -z "$upstream_sha" ]; then + log_warn "Branch '${branch}' has no upstream on origin — the pod's clone will fail. Run: git push -u origin ${branch}" + return 0 + fi + + if [ "$local_sha" != "$upstream_sha" ]; then + local ahead behind + ahead="$(git -C "$repo" rev-list --count "${upstream_sha}..${local_sha}" 2>/dev/null)" + behind="$(git -C "$repo" rev-list --count "${local_sha}..${upstream_sha}" 2>/dev/null)" + if [ "${ahead:-0}" -gt 0 ]; then + log_warn "Local '${branch}' is ${ahead} commit(s) ahead of origin/${branch} — the pod will clone the older origin tip. Run: git push" + fi + if [ "${behind:-0}" -gt 0 ]; then + log_info "Local '${branch}' is ${behind} commit(s) behind origin/${branch} (pod will clone the newer origin tip)." + fi + fi + + if [ -n "$(git -C "$repo" status --porcelain 2>/dev/null)" ]; then + log_warn "Working tree has uncommitted changes — the pod will not see them. Commit + push first if you want the pod to pick them up." + fi +} + +# osmo:up — submit airstack-dev.yaml with the local pubkey injected. +# +# Usage: airstack osmo:up [--pool POOL] [--key PATH] [--branch BRANCH] +# +# --branch defaults to the local repo's current branch (or `main` if we +# can't detect one), and is passed through as AIRSTACK_BRANCH so the +# pod's entrypoint clones the matching code. Pass `--branch main` +# explicitly to override. +function cmd_osmo_up { + _osmo_check_cli || return 1 + + local pool="${OSMO_POOL:-}" + local pubkey_file="" + local branch="" + local branch_explicit=false + local extra_args=() + + while [ $# -gt 0 ]; do + case "$1" in + --pool) pool="$2"; shift 2 ;; + --key) pubkey_file="$2"; shift 2 ;; + --branch) branch="$2"; branch_explicit=true; shift 2 ;; + *) extra_args+=("$1"); shift ;; + esac + done + + if [ -z "$pubkey_file" ]; then + if ! pubkey_file="$(_osmo_pick_pubkey)"; then + log_error "No SSH public key found in ~/.ssh. Generate one with: ssh-keygen -t ed25519" + return 1 + fi + fi + log_info "Using SSH public key: ${pubkey_file}" + + local workflow_yaml="${PROJECT_ROOT}/osmo/workflows/airstack-dev.yaml" + if [ ! -f "$workflow_yaml" ]; then + log_error "Workflow file not found: ${workflow_yaml}" + return 1 + fi + + # Auto-pin --branch to the local checkout if the user didn't pass one. + if [ "$branch_explicit" = false ] && [ -z "$branch" ]; then + branch="$(_osmo_local_branch)" + if [ -n "$branch" ]; then + log_info "Auto-detected local branch '${branch}'; pod will clone from origin/${branch} (override with --branch main)." + else + log_info "Could not detect local branch (detached HEAD?); pod will clone from origin/main." + fi + fi + if [ -n "$branch" ]; then + _osmo_check_branch_pushed "$branch" + fi + + local cmd=(osmo workflow submit "$workflow_yaml") + if [ -n "$pool" ]; then + cmd+=(--pool "$pool") + else + log_warn "No --pool provided and OSMO_POOL is unset; using your osmo profile's default pool." + fi + # IMPORTANT: `osmo workflow submit --set-env` is variadic. Passing two + # separate `--set-env A=1 --set-env B=2` silently drops the first one + # (only the last `--set-env` flag's values are kept). We collect all + # K=V pairs and pass them under a single `--set-env`. + local env_kvs=("SSH_PUB_KEY=$(cat "$pubkey_file")") + if [ -n "$branch" ]; then + env_kvs+=("AIRSTACK_BRANCH=${branch}") + fi + cmd+=(--set-env "${env_kvs[@]}") + if [ ${#extra_args[@]} -gt 0 ]; then + cmd+=("${extra_args[@]}") + fi + + log_info "Submitting: ${cmd[*]}" + local output + if ! output="$("${cmd[@]}" 2>&1)"; then + echo "$output" >&2 + log_error "osmo workflow submit failed." + return 1 + fi + echo "$output" + + # Parse the workflow id out of the submit output. The cookbook examples + # show "Workflow ID - " formatted output (see OSMO + # submission.rst). Match that line. + local wf_id + wf_id="$(echo "$output" | awk -F'- ' '/^Workflow ID/ {print $2; exit}' | tr -d ' \r\n')" + if [ -z "$wf_id" ]; then + log_warn "Could not parse workflow id from submit output. Set it manually:" + log_warn " echo > ${OSMO_STATE_FILE}" + return 0 + fi + _osmo_save_wf_id "$wf_id" + + log_info "Next steps:" + log_info " airstack osmo:logs # follow startup until 'sshd listening'" + log_info " airstack osmo:ide # port-forward sshd + open VS Code" + log_info " airstack osmo:webrtc # forward Isaac Sim WebRTC ports" + log_info " airstack osmo:foxglove # forward GCS Foxglove websocket" + log_info " airstack osmo:down # cancel the workflow" +} + +# osmo:logs — follow the workspace task logs. +# +# Despite the `osmo workflow logs --help` output advertising only `-n +# LAST_N_LINES` (no `--follow`), the CLI in fact streams the tail and keeps +# the connection open as new lines arrive — i.e. it already behaves like +# `tail -f`. We just exec it in the foreground so the user sees output +# immediately and can Ctrl+C to stop. (An earlier implementation wrapped +# this in `out=$(osmo workflow logs ...)`; command substitution waits for +# the process to exit, which never happened, so nothing was ever printed.) +function cmd_osmo_logs { + _osmo_check_cli || return 1 + local wf; wf="$(_osmo_wf_id)" || return 1 + + local task="${OSMO_LOGS_TASK:-workspace}" + local lines="${OSMO_LOGS_TAIL:-500}" + + log_info "Following ${task} logs for ${wf} (last ${lines} lines, then live; Ctrl+C to stop)" + + # Filter stderr for the same OSMOUserError-when-workflow-dies case + # the port-forward path hits — same noisy asyncio Traceback + + # "Task exception was never retrieved" header. _osmo_pf_filter + # collapses it into one clean log line. + osmo workflow logs "${wf}" -t "${task}" -n "${lines}" \ + 2> >(_osmo_pf_filter "${wf}") +} + +# osmo:ide — port-forward sshd + (optionally) launch VS Code/Cursor on the +# `airstack-osmo` host. Runs the port-forward in the foreground so closing +# the terminal closes the tunnel. +# +# Usage: airstack osmo:ide [--no-open] [code|cursor] +function cmd_osmo_ide { + _osmo_check_cli || return 1 + local wf; wf="$(_osmo_wf_id)" || return 1 + + local open_ide=true + local ide_cmd="" + while [ $# -gt 0 ]; do + case "$1" in + --no-open) open_ide=false; shift ;; + code|cursor) ide_cmd="$1"; shift ;; + *) log_warn "Ignoring unknown osmo:ide arg: $1"; shift ;; + esac + done + + if [ -z "$ide_cmd" ]; then + if command -v cursor >/dev/null 2>&1; then + ide_cmd="cursor" + elif command -v code >/dev/null 2>&1; then + ide_cmd="code" + else + log_warn "Neither 'cursor' nor 'code' found on PATH; will only port-forward (open the IDE manually and Connect to Host airstack-osmo)." + open_ide=false + fi + fi + + log_info "Make sure ~/.ssh/config has a 'Host airstack-osmo' entry pointing at localhost:2200, User root." + + # Local TCP port the user's IDE will connect to (the local side of the + # `--port LOCAL:REMOTE` mapping). + local local_port="${OSMO_SSH_PORT%%:*}" + + # Every fresh OSMO pod ships a new sshd host key. If the user's + # ~/.ssh/known_hosts still has an entry for [localhost]:${local_port} + # from a previous workflow, ssh aborts with "Host key for [localhost] + # :${local_port} has changed and you have requested strict checking", + # which the IDE surfaces as a generic "could not connect" error. + # + # The recommended ~/.ssh/config block for `airstack-osmo` uses + # `UserKnownHostsFile /dev/null`, which sidesteps this entirely — but + # users who set up before that change still have a stale entry on + # disk. Scrub it defensively on every osmo:ide invocation. ssh-keygen + # -R is idempotent: a no-op if the entry doesn't exist. + if command -v ssh-keygen >/dev/null 2>&1; then + ssh-keygen -R "[localhost]:${local_port}" >/dev/null 2>&1 || true + fi + + # Reuse an existing forward if one is already listening (the user might + # have run this from a second terminal, or osmo:foxglove already opened + # a multi-port forward). Otherwise spawn one in the background and wait + # for it to bind before launching the IDE — this avoids the race where + # Cursor/VS Code tries to SSH before the tunnel exists and dies with + # "connect to host localhost port 2200: Connection refused". + local pf_pid="" + if nc -z localhost "$local_port" 2>/dev/null; then + log_info "Port ${local_port} is already listening; reusing existing port-forward." + else + log_info "osmo workflow port-forward ${wf} workspace --port ${OSMO_SSH_PORT} --connect-timeout ${OSMO_PF_TIMEOUT}" + osmo workflow port-forward "$wf" workspace --port "$OSMO_SSH_PORT" --connect-timeout "$OSMO_PF_TIMEOUT" \ + > "${OSMO_STATE_DIR}/ssh-pf.log" 2>&1 & + pf_pid=$! + # Wait up to 30s for the tunnel to start accepting connections. + local waited=0 + until nc -z localhost "$local_port" 2>/dev/null; do + sleep 1; waited=$((waited+1)) + if [ "$waited" -ge 30 ]; then + log_error "Timed out waiting for port-forward on :${local_port} after ${waited}s." + log_error " port-forward log: ${OSMO_STATE_DIR}/ssh-pf.log" + kill "$pf_pid" 2>/dev/null + return 1 + fi + if ! kill -0 "$pf_pid" 2>/dev/null; then + log_error "port-forward exited early. Tail:" + tail -10 "${OSMO_STATE_DIR}/ssh-pf.log" >&2 + return 1 + fi + done + log_info "Port-forward established on localhost:${local_port} (pid ${pf_pid})." + fi + + if [ "$open_ide" = true ]; then + # vscode-remote URI launches the IDE pre-attached to the remote host. + local uri="vscode-remote://ssh-remote+airstack-osmo/root/AirStack" + log_info "Launching ${ide_cmd} → ${uri}" + ( "$ide_cmd" --folder-uri "$uri" >/dev/null 2>&1 || \ + "$ide_cmd" "$uri" >/dev/null 2>&1 || \ + log_warn "Could not launch ${ide_cmd} automatically; open it and pick airstack-osmo from Remote-SSH manually." ) & + fi + + if [ -n "$pf_pid" ]; then + log_info "Leave this terminal running for the length of your session (Ctrl+C to disconnect)." + # Forward Ctrl+C to the port-forward and clean up. + trap 'kill "$pf_pid" 2>/dev/null; exit 0' INT TERM + wait "$pf_pid" + else + log_info "Existing port-forward owns the tunnel; this command will exit immediately." + log_info "Stop the tunnel with: pkill -f 'osmo workflow port-forward' or airstack osmo:down" + fi +} + +# Helper: filter `osmo workflow port-forward` stderr through awk to +# suppress the asyncio traceback that erupts whenever the workflow gets +# canceled mid-flight (e.g. via osmo:down in another shell, or because +# OSMO timed it out). The CLI raises OSMOUserError("Workflow X is not +# running!") from inside an asyncio Task, which then prints "Task +# exception was never retrieved" + a multi-line Traceback that obscures +# the actual one-line cause. We translate that into a single clean log +# line and drop everything else. +function _osmo_pf_filter { + local wf="$1" + awk -v WF="$wf" ' + /^Task exception was never retrieved/ { skipping=1; next } + /^future:/ { skipping=1; next } + /^Traceback \(most recent call last\):/ { skipping=1; next } + /^ File "/ { next } + /^src\.lib\.utils\.osmo_errors\.OSMOUserError/ { + sub(/^src\.lib\.utils\.osmo_errors\.OSMOUserError: */, "") + printf "\033[0;31m[ERROR]\033[0m %s (run `airstack osmo:up` to start a new workflow)\n", $0 + next + } + /OSMOUserError: Workflow .* is not running!/ { + printf "\033[0;31m[ERROR]\033[0m Workflow %s is no longer running (run `airstack osmo:up` to start a new one).\n", WF + next + } + skipping && /^$/ { skipping=0; next } + skipping { next } + { print } + ' >&2 +} + +# Helper: run `osmo workflow port-forward` with the noise filter +# attached. Returns the underlying exit code so callers can decide +# whether to retry / fail. Args after the helper name are passed to +# `osmo workflow port-forward` verbatim. +function _osmo_run_port_forward { + osmo workflow port-forward "$@" 2> >(_osmo_pf_filter "$1") +} + +# osmo:webrtc — forward both Isaac Sim WebRTC port ranges (TCP in this +# terminal, spawn UDP in the background). Cleans up the UDP child on +# exit (Ctrl+C, foreground TCP failure, or the workflow disappearing +# mid-stream) so we don't leak a port-forward into the user's process +# table. +function cmd_osmo_webrtc { + _osmo_check_cli || return 1 + local wf; wf="$(_osmo_wf_id)" || return 1 + + log_info "Spawning UDP port-forward in background: ${OSMO_WEBRTC_UDP}" + nohup osmo workflow port-forward "$wf" workspace \ + --port "$OSMO_WEBRTC_UDP" --udp \ + --connect-timeout "$OSMO_PF_TIMEOUT" \ + > "${OSMO_STATE_DIR}/webrtc-udp.log" 2>&1 & + local udp_pid=$! + log_info " UDP log: ${OSMO_STATE_DIR}/webrtc-udp.log (pid ${udp_pid})" + + # Tear the UDP fork down when this function exits, by any path. + # Without this, hitting Ctrl+C on the TCP foreground (or the + # workflow being canceled, which surfaces as the foreground exiting + # non-zero) leaves the UDP `osmo workflow port-forward` running + # against a dead workflow until the user notices and pkill's it. + trap ' + if kill -0 "'"${udp_pid}"'" 2>/dev/null; then + kill "'"${udp_pid}"'" 2>/dev/null + wait "'"${udp_pid}"'" 2>/dev/null + fi + trap - EXIT INT TERM + ' EXIT INT TERM + + log_info "Foreground TCP port-forward: ${OSMO_WEBRTC_TCP}" + log_info "Open the Omniverse Streaming Client / WebRTC client at http://localhost" + _osmo_run_port_forward "$wf" workspace \ + --port "$OSMO_WEBRTC_TCP" \ + --connect-timeout "$OSMO_PF_TIMEOUT" +} + +# osmo:foxglove — install the AirStack Foxglove extensions into the local +# Foxglove Desktop user-extensions dir, then forward the GCS Foxglove +# websocket. +# +# The extension install is the same script the GCS container runs on +# startup — gcs/foxglove_extensions/install.py — invoked with env-var +# overrides that point at the local laptop dirs. Default destination on +# Linux/macOS is ~/.foxglove-studio/extensions (Foxglove's canonical user +# extensions path; the macOS rebrand still reads from here). Override +# with OSMO_FOXGLOVE_EXT_DIR, or skip the install entirely with +# OSMO_FOXGLOVE_SKIP_EXTENSIONS=1 (e.g. when using app.foxglove.dev +# which doesn't load local extensions anyway). +function cmd_osmo_foxglove { + _osmo_check_cli || return 1 + local wf; wf="$(_osmo_wf_id)" || return 1 + + local ext_src="${PROJECT_ROOT}/gcs/foxglove_extensions" + local ext_dst="${OSMO_FOXGLOVE_EXT_DIR:-${HOME}/.foxglove-studio/extensions}" + + if [ "${OSMO_FOXGLOVE_SKIP_EXTENSIONS:-0}" != "1" ] && [ -d "${ext_src}" ]; then + if command -v python3 >/dev/null 2>&1; then + log_info "Installing Foxglove extensions to ${ext_dst}" + FOXGLOVE_EXT_SRC="${ext_src}" FOXGLOVE_EXT_DST="${ext_dst}" \ + python3 "${ext_src}/install.py" \ + || log_warn "Foxglove extension install failed; panels like 'Robot Tasks' may show as 'Unknown panel type' in Foxglove" + else + log_warn "python3 not found on PATH — skipping Foxglove extension install." + log_warn " Custom panels (Robot Tasks, Waypoint Editor, Polygon Editor) will show as 'Unknown panel type'." + log_warn " Install python3 (e.g. 'brew install python') or copy ${ext_src}/* manually to ${ext_dst}." + fi + elif [ "${OSMO_FOXGLOVE_SKIP_EXTENSIONS:-0}" = "1" ]; then + log_info "Skipping Foxglove extension install (OSMO_FOXGLOVE_SKIP_EXTENSIONS=1)." + fi + + log_info "osmo workflow port-forward ${wf} workspace --port ${OSMO_FOXGLOVE_PORT} --connect-timeout ${OSMO_PF_TIMEOUT}" + log_info "Then in Foxglove Desktop: Open connection → ws://localhost:8766" + log_info " Layouts → Import from file → ${ext_src}/airstack_default.json" + log_info " (Restart Foxglove Desktop once if newly-installed panels still show as 'Unknown panel type'.)" + _osmo_run_port_forward "$wf" workspace \ + --port "$OSMO_FOXGLOVE_PORT" \ + --connect-timeout "$OSMO_PF_TIMEOUT" +} + +# osmo:down — cancel the active workflow. Reminds you to push first. +function cmd_osmo_down { + _osmo_check_cli || return 1 + local wf; wf="$(_osmo_wf_id)" || return 1 + + log_warn "About to cancel workflow '${wf}'." + log_warn "Anything not pushed to git in /root/AirStack inside the pod will be LOST." + log_warn "Hit Ctrl-C in the next 5 seconds to abort." + sleep 5 + osmo workflow cancel "$wf" + rm -f "${OSMO_STATE_FILE}" +} + +# Register commands from this module. +function register_osmo_commands { + COMMANDS["osmo:setup"]="cmd_osmo_setup" + COMMANDS["osmo:up"]="cmd_osmo_up" + COMMANDS["osmo:logs"]="cmd_osmo_logs" + COMMANDS["osmo:ide"]="cmd_osmo_ide" + COMMANDS["osmo:webrtc"]="cmd_osmo_webrtc" + COMMANDS["osmo:foxglove"]="cmd_osmo_foxglove" + COMMANDS["osmo:down"]="cmd_osmo_down" + + COMMAND_HELP["osmo:setup"]="One-time per-user OSMO credential setup (airlab-docker-registry, airlab-docker-login, airlab-nucleus)" + COMMAND_HELP["osmo:up"]="Submit osmo/workflows/airstack-dev.yaml with your SSH pubkey injected (--pool POOL, --key PATH, --branch BRANCH)" + COMMAND_HELP["osmo:logs"]="Follow the workspace task logs (osmo workflow logs -t workspace -n 500; OSMO_LOGS_TASK / OSMO_LOGS_TAIL override)" + COMMAND_HELP["osmo:ide"]="Port-forward sshd (2200:22) and open VS Code/Cursor on Host airstack-osmo" + COMMAND_HELP["osmo:webrtc"]="Port-forward Isaac Sim WebRTC ranges (TCP foreground + UDP background)" + COMMAND_HELP["osmo:foxglove"]="Install AirStack Foxglove extensions locally, then port-forward GCS Foxglove websocket (8766:8766). Override target dir with OSMO_FOXGLOVE_EXT_DIR; skip install with OSMO_FOXGLOVE_SKIP_EXTENSIONS=1." + COMMAND_HELP["osmo:down"]="Cancel the active workflow (push to git before running this)" +} diff --git a/.airstack/modules/ready.sh b/.airstack/modules/ready.sh new file mode 100644 index 000000000..12ceb2897 --- /dev/null +++ b/.airstack/modules/ready.sh @@ -0,0 +1,245 @@ +#!/bin/bash +# Readiness gates for a running AirStack stack. +# +# `airstack up` reports success the moment `docker compose up -d` returns — +# before workspaces build, the sim loads, or PX4 boots. `airstack ready` +# answers the question users otherwise guess at: "can I press Takeoff yet?" +# +# Gates and budgets mirror the system-test suite (the source of truth for +# real-world timings — tests/system/test_liveliness.py and +# tests/system/test_takeoff_hover_land.py): +# 1. containers Running (120 s) +# 2. sim publishing /clock (600 s — Isaac scene loads are slow) +# 3. sentinel ROS 2 nodes per robot (300 s — includes the colcon build in dev mode) +# 4. PX4 ready per robot: MAVROS connected (300 s) +# then local_position/odom streaming (EKF converged = armable; connected +# alone fires ~25 s too early and takeoff returns "failed to arm") + +# Defaults match the system-test budgets; overridable from the environment +# (e.g. READY_CLOCK_TIMEOUT=60 airstack ready). +: "${READY_CONTAINERS_TIMEOUT:=120}" +: "${READY_CLOCK_TIMEOUT:=600}" +: "${READY_NODES_TIMEOUT:=300}" +: "${READY_PX4_TIMEOUT:=300}" +: "${READY_POLL_INTERVAL:=5}" + +# Sentinel nodes expected per robot domain (matches tests/system/test_liveliness.py). +READY_SENTINEL_TEMPLATES=( + "/robot_%d/interface/mavros/mavros" + "/robot_%d/robot_state_publisher" + "/robot_%d/trajectory_controller/trajectory_control_node" +) + +function _ready_now { date +%s; } + +function _ready_elapsed { + echo "$(( $(_ready_now) - $1 ))" +} + +# Run a ros2 command inside a robot container on a given domain, sourcing the +# workspace if it is built yet (mavros msgs need it). +function _ready_ros2_exec { + local container="$1" domain="$2" cmd="$3" timeout_s="${4:-10}" + docker exec "$container" bash -c " + source /opt/ros/jazzy/setup.bash >/dev/null 2>&1 + [ -f /root/AirStack/robot/ros_ws/install/setup.bash ] && source /root/AirStack/robot/ros_ws/install/setup.bash >/dev/null 2>&1 + export ROS_DOMAIN_ID=$domain + timeout $timeout_s $cmd" 2>/dev/null +} + +# List running robot containers (compose replicas), one per line. +function _ready_robot_containers { + docker ps --format '{{.Names}}' | grep -E -- '-robot-' | sort +} + +# domain for robot container (via the same .bashrc resolution airstack status uses) +function _ready_domain_of { + local container="$1" vars + vars=$(docker exec "$container" bash --login -c \ + 'printf "AIRSTACK_VARS:%s:%s\n" "$ROBOT_NAME" "$ROS_DOMAIN_ID"' 2>/dev/null \ + | grep "^AIRSTACK_VARS:" | tail -1) + [ -z "$vars" ] && return 1 + echo "${vars##*:}" +} + +function _ready_robot_name_of { + local container="$1" vars + vars=$(docker exec "$container" bash --login -c \ + 'printf "AIRSTACK_VARS:%s:%s\n" "$ROBOT_NAME" "$ROS_DOMAIN_ID"' 2>/dev/null \ + | grep "^AIRSTACK_VARS:" | tail -1) + [ -z "$vars" ] && return 1 + vars="${vars#AIRSTACK_VARS:}" + echo "${vars%%:*}" +} + +# Poll a predicate function until it returns 0 or the timeout expires. +# Usage: _ready_poll