diff --git a/.ansible-lint b/.ansible-lint new file mode 100644 index 0000000..397e174 --- /dev/null +++ b/.ansible-lint @@ -0,0 +1,12 @@ +--- +profile: production +exclude_paths: + - .tmp/ +mock_modules: + - zuul_return +mock_roles: + - collect-container-logs + - ensure-output-dirs + - ensure-podman + - run-buildset-registry + - use-buildset-registry diff --git a/.github/workflows/build-and-push.yml b/.github/workflows/build-and-push.yml index 068674f..f7f375d 100644 --- a/.github/workflows/build-and-push.yml +++ b/.github/workflows/build-and-push.yml @@ -34,5 +34,5 @@ jobs: # I want to test we are setting the namespace and login properly before pushing # uncomment later - #- name: Push images - # run: tox -e custom -- push all + # - name: Push images + # run: tox -e custom -- push all diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a5fc396 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/.tmp/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9b86197..080e0a0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,16 +1,31 @@ --- -# Containerfile and repo hygiene hooks. +default_language_version: + python: python3 + repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v6.0.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer - - id: check-merge-conflict - - id: check-yaml - files: \.(yaml|yml)$ - id: mixed-line-ending args: [--fix=lf] + - id: fix-byte-order-marker + - id: check-executables-have-shebangs + - id: check-shebang-scripts-are-executable + - id: check-yaml + files: .*\.(yaml|yml)$ + args: [--allow-multiple-documents] + - id: check-json + files: .*\.json$ + - id: check-ast + - id: check-added-large-files + - id: check-case-conflict + - id: check-symlinks + - id: detect-private-key + - id: check-merge-conflict + - id: debug-statements + - id: check-docstring-first - repo: https://github.com/openstack/bashate rev: 2.1.1 @@ -36,3 +51,24 @@ repos: name: hadolint (Containerfiles) # identify tags Containerfile as type dockerfile types: [dockerfile] + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.12.1 + hooks: + - id: ruff-check + args: [--fix, --unsafe-fixes] + files: ^tests/.*\.py$ + - id: ruff-format + files: ^tests/.*\.py$ + + - repo: https://opendev.org/openstack/hacking + rev: 8.0.0 + hooks: + - id: hacking + additional_dependencies: [] + files: ^tests/.*\.py$ + + - repo: https://github.com/ansible/ansible-lint + rev: v25.9.0 + hooks: + - id: ansible-lint diff --git a/.zuul.yaml b/.zuul.yaml new file mode 100644 index 0000000..7976aa9 --- /dev/null +++ b/.zuul.yaml @@ -0,0 +1,57 @@ +--- +- job: + name: s2i-openstack-containers-unit + parent: tox + description: Run the stdlib unit test suite. + vars: + tox_envlist: unit + +- job: + name: s2i-openstack-containers-linters + parent: tox-linters + description: Run repository formatting and style checks. + +- job: + name: s2i-openstack-containers-molecule + parent: tox + description: Run provider registry and cleanup contract scenarios. + vars: + tox_envlist: molecule + +- nodeset: + name: s2i-openstack-containers-image-builder + nodes: + - name: builder + label: cloud-centos-10-stream + +- job: + name: s2i-openstack-container-content-provider + parent: base + abstract: true + description: | + Build and publish a maintained subset of OpenStack service container + images from committed source pins for dependent jobs. + + The job uses the upstream container repository by default. A child job + can add another container repository to ``required-projects`` and select + it with ``s2i_ci_container_project``. + timeout: 7200 + nodeset: s2i-openstack-containers-image-builder + required-projects: + - github.com/openstack-k8s-operators/s2i-openstack-containers + - opendev.org/zuul/zuul-jobs + pre-run: + - playbooks/container-ci/zuul/reset-static-node.yaml + - playbooks/container-ci/zuul/pre.yaml + run: playbooks/container-ci/zuul/run.yaml + post-run: playbooks/container-ci/zuul/post.yaml + vars: + s2i_ci_container_project: >- + github.com/openstack-k8s-operators/s2i-openstack-containers + s2i_ci_images: + - watcher/watcher-base + - cyborg/cyborg + - cyborg/cyborg-agent + s2i_ci_stream: master + s2i_ci_content_provider: true + s2i_ci_reset_static_node: false diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..d19f74c --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,21 @@ +# Agent Guidance + +Read `developer-guide.md` for repository architecture, contributor workflows, +and validation guidance. + +## Python imports + +Follow the OpenStack Hacking import conventions: + +- Do not use relative imports. +- Import modules rather than objects. +- Use `import package.module` or `from package import module` style. +- Keep imports grouped and alphabetized by full module path. + +## Quality tooling + +- Treat `pyproject.toml`, `.pre-commit-config.yaml`, and `tox.ini` as the + portable tooling configuration. +- Use stdlib `unittest`, never pytest or Bash, for non-Ansible tests. +- Do not add new shell test harnesses. +- Run tests through tox and keep generated state below `.tmp/`. diff --git a/README.md b/README.md index abf4e28..ed39772 100644 --- a/README.md +++ b/README.md @@ -1,363 +1,148 @@ # S2I OpenStack Containers -Source-to-image container builds for OpenStack services on UBI 10 (ubi-minimal). +Source-to-image container builds for OpenStack services on UBI 10 +(`ubi-minimal`). Service code is built from pinned upstream Git sources with +Python dependencies constrained by generated lock files. Runtime RPM +requirements are installed with `microdnf`. -Services are built from pinned upstream source using multi-stage Containerfiles. -All Python dependencies are installed via pip from wheels compiled in a build -stage, constrained by a `pip-compile`-generated lockfile. System (RPM) -dependencies are installed via `microdnf`. +The repository currently builds: -## Repository structure +- `openstack-base`; +- `openstack-cyborg`; +- `openstack-cyborg-agent`; +- `openstack-watcher-base`. -``` -build.sh # Build orchestrator -containers/ - base/ # Base image (openstack-base) - Containerfile - bindeps.txt # System packages for base image - pythondeps.txt # Python packages for base image - rpms.repo # DNF repo config for RPM lockfile - sources.txt # Pinned upstream sources (upper-constraints) - scripts/ # Kolla helper scripts (uid_gid_manage, kolla_start, ...) - / # e.g., watcher - sources.txt # Pinned sources for this project (service repo + upper-constraints) - src/ # Cloned sources (auto-managed, .gitkeep only in git) - rpms.in.yaml # [generated] RPM packages for rpm-lockfile-prototype - requirements.lock. # [generated] pip-compile lockfile - buildrequirements.lock. # [generated] pybuild-deps build lockfile - upper-constraints.txt.# [generated] upstream constraints snapshot - / # e.g., watcher, watcher-api - Containerfile - bindeps.txt # Runtime system packages - builddeps.txt # Build-stage system packages - pythondeps.txt # Extra Python packages (oslo.db[mysql], etc.) - pythonbuilddeps.txt # Build-stage Python packages (pbr, etc.) - src/ # Image-specific sources (if any) -``` +See [`developer-guide.md`](developer-guide.md) for the complete contributor +reference, including source ownership, generated files, build architecture, +testing, maintenance, and troubleshooting. -## Source management +## Quick start -Source code for each service is cloned into `src/` directories and made -available to the Containerfile build context. `build.sh` supports two -levels of `sources.txt` and `src/` directories: +Install the host container tools: -- **Project level** (`containers//sources.txt` and - `containers//src/`) -- Sources shared by all images in the - project. This is where the main service repo lives (e.g., `watcher`). -- **Image level** (`containers///sources.txt` and - `containers///src/`) -- Sources specific to a single - image. Use this when one image needs an extra dependency that other - images in the same project don't need. - -During a build, the Containerfile merges both levels into `/src/` inside -the container: - -```dockerfile -COPY src/ /src/ # project-level sources -COPY /src/ /src/ # image-level sources (merged on top) +```console +./build.sh install-deps ``` -The build context is set to `containers//`, so both directories -are reachable. - -### Automatic cloning and cleanup - -When running `build` or `update-sources`, `build.sh` reads `sources.txt` -at both levels and clones any repo that doesn't already exist in `src/`. -These auto-cloned repos are tracked and **removed automatically on exit** -(via an EXIT trap), so `src/` directories stay clean in the repo (only a -`.gitkeep` is committed). - -If a checkout already exists in `src/` (e.g., a local development clone), -`build.sh` uses it as-is and does **not** remove it on exit. This lets you -work on a local branch without `build.sh` overwriting it. - -### Source overrides - -To patch or replace a transitive dependency, place the modified source in -`containers//src/overrides//`. The build stage picks up -everything under `src/overrides/` automatically -- no `sources.txt` entry -is needed. The filtered constraints file excludes source-built packages so -the overridden version takes precedence over PyPI. - -## Manually maintained files - -These files are created and updated by hand. `build.sh` reads them but -never overwrites them. +List the available image targets: -| File | Location | Purpose | -|------|----------|---------| -| `Containerfile` | `containers/base/`, `containers///` | Multi-stage build definition | -| `sources.txt` | `containers/base/`, `containers//` | Pinned source repos and branches per stream | -| `bindeps.txt` | base and each image | Runtime RPM packages (installed via `microdnf`) | -| `builddeps.txt` | each image | Build-stage RPM packages (compilers, `-devel` headers) | -| `pythondeps.txt` | base and each image | Extra pip packages beyond the service's `requirements.txt` | -| `pythonbuilddeps.txt` | each image | Build-stage pip packages (e.g., `pbr`) | -| `rpms.repo` | `containers/base/` | DNF repo configuration for RPM lockfile | -| `scripts/*` | `containers/base/scripts/` | Kolla helper scripts (`kolla_start`, `uid_gid_manage`, etc.) | -| `config/*` | `containers/service/` | Config files manually maintained out of upstream repo | - -## Streams - -A **stream** is a coherent set of source repos at specific commits. Typical -streams are `master` (tracking upstream HEAD) and `stable` (tracking a -stable branch like `stable/2026.1`). Different projects in the same stream -may follow different branches. - -Each stream gets its own set of generated files (lockfile, constraints). -Multiple streams can coexist in the same repo -- they are distinguished -by the `.` suffix on generated files. - -### sources.txt format - -Each line defines a source repo pinned to a specific commit, grouped by stream: - -``` - +```console +./build.sh list ``` -Example: +Build all images from the maintained `master` source pins: +```console +STREAM=master tox -e build ``` -master upper-constraints https://opendev.org/openstack/requirements.git master 4bb8ff9ad664e832d78139e23f5933cca6054d35 -master watcher https://opendev.org/openstack/watcher.git master 4abcf29a3ec323a6df3f567d7485b320354af4f4 -stable upper-constraints https://opendev.org/openstack/requirements.git stable/2026.1 c4c55d5279d824dc261a43ac51b56146ccc4dd4f -stable watcher https://opendev.org/openstack/watcher.git stable/2026.1 ba7b161dc24a6f2f1f7b7a2a529b8d93c65fee6c -``` - -The special name `upper-constraints` tells `build.sh` to fetch -`upper-constraints.txt` from the repo instead of cloning the full repo -into `src/`. The upper-constraints.txt file will be used as constraints -file via a lock file automatically created using pip-compile. - -### Dependency files - -Each image directory has four dependency files, all plain text with one -entry per line (blank lines and `#` comments are ignored): - -- **`builddeps.txt`** -- System packages needed during the build stage only - (compilers, header files). Not present in the final image. -- **`pythonbuilddeps.txt`** -- Python packages needed during the build stage. -- **`bindeps.txt`** -- System packages installed in the final runtime image. -- **`pythondeps.txt`** -- Extra Python packages installed via pip in the - final image (database drivers, caching backends, CLI clients). -The base image (`containers/base/`) also has `bindeps.txt` and `pythondeps.txt` -for packages shared across all service images. - -## Auto-generated files - -These files are created and updated by `build.sh update-sources`. They -should be committed to the repository but never edited by hand. - -| File | Location | Generated from | -|------|----------|----------------| -| `upper-constraints.txt.` | `containers//`, `containers/base/` | Fetched from the `upper-constraints` entry in `sources.txt` | -| `requirements.lock.` | `containers//`, `containers/base/` | `pip-compile` against all `requirements.txt` + `pythondeps.txt` + `pythonbuilddeps.txt`, constrained by `upper-constraints.txt.` | -| `buildrequirements.lock.` | `containers//`, `containers/base/` | `pybuild-deps compile` against `requirements.lock.` | -| `rpms.in.yaml` | `containers//`, `containers/base/` | Union of all `bindeps.txt` + `builddeps.txt` across images in the project | - -When the stream being updated matches `DEFAULT_STREAM` (default: `master`), -un-suffixed symlinks are also created: +Run the unit suite and repository checks: +```console +tox -e unit +tox -e linters ``` -requirements.lock -> requirements.lock.master -buildrequirements.lock -> buildrequirements.lock.master -upper-constraints.txt -> upper-constraints.txt.master -``` - -These symlinks allow Containerfiles to use `ARG CONSTRAINTS_FILE=requirements.lock` -without needing to know which stream is active. - -**Important:** Whenever you modify `sources.txt`, `pythondeps.txt`, -`pythonbuilddeps.txt`, `bindeps.txt`, or `builddeps.txt`, you must re-run -`build.sh update-sources` (or `tox -e update-sources`) to regenerate the -lockfile, constraints, and `rpms.in.yaml`. Failing to do so will cause -builds to use stale dependency pins. -## Prerequisites - -### System packages - -- `git` -- cloning source repos -- `buildah` -- building container images -- `podman` -- running and inspecting built images - -Install all at once: +## Repository structure -```bash -./build.sh install-deps +```text +build.sh build and source-maintenance implementation +containers/base/ common openstack-base image +containers/cyborg/ Cyborg sources and image contexts +containers/watcher/ Watcher sources and image context +tests/ stdlib unittest suite +tox.ini contributor command environments +developer-guide.md architecture and workflow reference +``` + +A service project normally contains shared source and generated dependency +data plus one or more image directories: + +```text +containers// + sources.txt + requirements.lock. + buildrequirements.lock. + upper-constraints.txt. + rpms.in.yaml + src/ + / + Containerfile + bindeps.txt + builddeps.txt + pythondeps.txt + pythonbuilddeps.txt ``` -This runs `sudo dnf install` (or the appropriate package manager) for the -system packages and `pip install pip-tools` for the Python dependencies. - -### Python packages +## Source and generated files -- `pip-tools` -- provides `pip-compile`, used by `update-sources` to - generate lockfiles -- `pybuild-deps` -- used by `update-sources` to generate build-requirements - lockfiles +A `sources.txt` record identifies a stream, project name, repository, branch to +follow during maintenance, and exact commit used by builds: -If using tox (recommended), Python dependencies are installed automatically -in the tox virtualenv. - -## Workflow +```text + +``` -### Using tox +`build.sh` clones missing sources into the applicable `src/` directory and +removes checkouts it created when the command exits. A checkout that already +exists is used as-is and is not removed, allowing intentional local source +experiments. -Tox manages a virtualenv with the required Python dependencies and passes -through all relevant environment variables (`STREAM`, `REGISTRY`, `TAG`, etc.): +Containerfiles, `sources.txt`, dependency lists, base scripts, and maintained +configuration are edited by contributors. These tracked files are generated by +`build.sh update-sources` and must not be edited by hand: -```bash -# Update sources for all projects -STREAM=master tox -eupdate-sources +- `upper-constraints.txt.`; +- `requirements.lock.`; +- `buildrequirements.lock.`; +- `rpms.in.yaml`; +- default-stream symlinks to the corresponding generated files. -# Build all images -STREAM=master tox -ebuild +After changing a source or dependency input, regenerate the derived files: -# Run any build.sh command via the generic 'custom' target -STREAM=master tox -ecustom -- update-sources watcher -STREAM=stable tox -ecustom -- build watcher/watcher-api -tox -ecustom -- list +```console +STREAM=master tox -e update-sources ``` -### Initial setup - -1. Create `containers//sources.txt` with entries for each stream. -2. Create image directories under `containers///` with - a `Containerfile` and the four dependency files. -3. Run `update-sources` to generate lockfiles and constraints. - -### Updating sources (pinning to latest upstream) +To regenerate from the existing commits without advancing source pins: -```bash -STREAM=master ./build.sh update-sources +```console +STREAM=master SKIP_HASH_UPDATE=1 tox -e update-sources ``` -This will: -1. Clone each source repo at the branch tip to resolve the latest commit hash. -2. Update `sources.txt` with the new pinned hashes. -3. Fetch `upper-constraints.txt` from the requirements repo. -4. Generate `rpms.in.yaml` from all `bindeps.txt` + `builddeps.txt` files. -5. Run `pip-compile` to generate `requirements.lock.`. -6. Run `pybuild-deps compile` to generate `buildrequirements.lock.`. -7. Create default-stream symlinks if `STREAM == DEFAULT_STREAM`. +## Build commands -Auto-cloned repos in `src/` are cleaned up automatically on exit. -Pre-existing checkouts in `src/` are used as-is and not removed. +Build a project or one image with the shell interface: -To regenerate lockfiles without updating pinned hashes (steps 1--3 are -skipped; repos are cloned at the existing pinned hashes instead): - -```bash -STREAM=master SKIP_HASH_UPDATE=1 ./build.sh update-sources +```console +STREAM=master ./build.sh build watcher +STREAM=master ./build.sh build cyborg/cyborg-agent ``` -### Building images - -```bash -# Build all images for a stream -STREAM=master ./build.sh build all +The `custom` tox environment exposes the same interface when a tox-managed +Python environment is useful: -# Build a single project (all its images) -STREAM=master ./build.sh build watcher - -# Build a specific image -STREAM=master ./build.sh build watcher/watcher-api +```console +STREAM=master tox -e custom -- build watcher +STREAM=master tox -e custom -- update-sources watcher ``` -Build order: the base image is always built first when targeting `all`. -Service images use the base image via `--build-arg BASE_IMAGE`. - -### Pushing images +Push verifies that every requested local tag exists before publication: -```bash +```console STREAM=master REGISTRY=quay.io NAMESPACE=myorg ./build.sh push all ``` -All image tags are verified to exist locally before any push begins. +See the developer guide before changing source records, generated files, +Containerfiles, or image dependencies. -### Listing images +## Publication -```bash -./build.sh list -``` +Konflux is authoritative for hermetic production provenance and publication. +The GitHub workflows provide development build and validation coverage. The +`build-and-push` workflow logs in to Quay, but its push step is commented out +and therefore does not publish images. + +## License -## Build architecture - -### Base image (`openstack-base`) - -Single-stage build on `ubi10/ubi-minimal`. Installs system packages, Python, -pip, kolla helper scripts, and common Python dependencies. All service -images inherit from this. - -### Service images - -Two-stage build: - -1. **Build stage** (FROM base AS build): - - Installs build-time system and Python dependencies. - - Copies source repos from the build context (`src/`). - - Builds wheels from source with `pip wheel --no-deps`. - - Generates a filtered constraints file (excludes source-built packages). - - Records a build manifest (`source-built-packages.txt`) with - package name, commit hash, and version. - - Runs oslo-config-generator for config files (service-specific). - -2. **Runtime stage** (FROM base): - - Creates the service user via `uid_gid_manage`. - - Installs runtime system packages from `bindeps.txt`. - - Installs wheels from the build stage plus extra Python deps from - `pythondeps.txt`, constrained by the filtered constraints file. - - Sets up directories, config files, and permissions. - -## Environment variables - -| Variable | Default | Description | -|----------|---------|-------------| -| `STREAM` | `master` | Stream name (selects which `sources.txt` entries to use) | -| `REGISTRY` | `localhost` | Container registry | -| `NAMESPACE` | `openstack` | Registry namespace | -| `TAG` | `${STREAM}-latest` | Image tag(s), comma-separated for multiple | -| `IMAGE_PREFIX` | `openstack` | Prefix for image names (e.g., `openstack-watcher`) | -| `BASE_IMAGE` | `${REGISTRY}/${NAMESPACE}/${IMAGE_PREFIX}-base:${TAG}` | Base image for service builds | -| `CONSTRAINTS_FILE` | `requirements.lock` | Lockfile base name used during builds | -| `BUILD_CONSTRAINTS_FILE` | `buildrequirements.lock` | Build-requirements lockfile base name | -| `DEFAULT_STREAM` | `master` | Stream for which un-suffixed symlinks are created | -| `PARALLEL` | `nproc` | Max concurrent builds for `build-parallel` | -| `BUILD_LOGS_DIR` | *(tmpdir, deleted)* | Directory to persist `build-parallel` logs | -| `SKIP_HASH_UPDATE` | *(unset)* | If set, `update-sources` skips updating pinned hashes and clones repos at existing pins; lockfiles are still regenerated | -| `PIP_NO_BINARY` | *(unset)* | If set, passed as `--build-arg` to the container build so pip builds packages from source (e.g., `:all:`) | - -## Adding a new service - -1. Create the project directory structure: - - ``` - containers// - sources.txt - src/.gitkeep - / - Containerfile - bindeps.txt - builddeps.txt - pythondeps.txt - pythonbuilddeps.txt - src/.gitkeep - ``` - -2. Populate `sources.txt` with `upper-constraints` and service repo entries - for each stream. - -3. Write the `Containerfile` following the multi-stage pattern (see - `containers/watcher/watcher/Containerfile` as an example). - -4. Fill in the dependency files for the image. - -5. Run update-sources and build: - - ```bash - STREAM=master ./build.sh update-sources - STREAM=master ./build.sh build - ``` +See [`LICENSE.txt`](LICENSE.txt). diff --git a/TESTING.md b/TESTING.md new file mode 100644 index 0000000..c1d491f --- /dev/null +++ b/TESTING.md @@ -0,0 +1,23 @@ +# Testing + +The canonical testing and validation guidance is in the +[developer guide](developer-guide.md#part-v---testing-and-validation). + +For the common checks, run: + +```console +tox -e unit +tox -e linters +``` + +The compatibility environments `test` and `py3` run the same stdlib +`unittest` suite as `unit`: + +```console +tox -e test +tox -e py3 +``` + +Use the narrowest applicable check first. Tests must keep generated state below +`.tmp/` and must not modify maintained source files or unrelated container +resources. diff --git a/build.sh b/build.sh index 8df932b..6fbff90 100755 --- a/build.sh +++ b/build.sh @@ -83,6 +83,8 @@ # PIP_NO_BINARY If set, passed as --build-arg to buildah so Containerfiles # can set ENV PIP_NO_BINARY. Use ":all:" to force pip to # build all packages from source instead of using wheels. +# REGISTRY_AUTH_FILE Authentication file passed explicitly to buildah push. +# REGISTRY_CERT_DIR TLS certificate directory passed explicitly to buildah push. set -euo pipefail @@ -102,6 +104,8 @@ UPSTREAM_CONSTRAINTS="upper-constraints.txt" DEFAULT_STREAM="${DEFAULT_STREAM:-master}" SKIP_HASH_UPDATE="${SKIP_HASH_UPDATE:-}" PIP_NO_BINARY="${PIP_NO_BINARY:-}" +REGISTRY_AUTH_FILE="${REGISTRY_AUTH_FILE:-}" +REGISTRY_CERT_DIR="${REGISTRY_CERT_DIR:-}" PARALLEL="${PARALLEL:-$(nproc)}" # Discover all buildable images from the directory structure. @@ -370,10 +374,33 @@ push_image() { name="$(image_name "${dir_name}")" IFS=',' read -ra tags <<< "${TAG}" + local registry_args=() + [[ -n "${REGISTRY_AUTH_FILE}" ]] && \ + registry_args+=(--authfile "${REGISTRY_AUTH_FILE}") + [[ -n "${REGISTRY_CERT_DIR}" ]] && \ + registry_args+=(--cert-dir "${REGISTRY_CERT_DIR}") + for t in "${tags[@]}"; do local full_tag="${REGISTRY}/${NAMESPACE}/${name}:${t}" echo "=== Pushing ${full_tag} ===" - buildah push "${full_tag}" + buildah push "${registry_args[@]}" "${full_tag}" + done +} + +# Print the exact image references produced for a target. +target_refs() { + local target="$1" + local dir_name + local name + local tag + local -a tags + resolve_targets_array "${target}" || return 1 + for dir_name in "${_RESOLVED_TARGETS[@]}"; do + name="$(image_name "${dir_name}")" + IFS=',' read -ra tags <<< "${TAG}" + for tag in "${tags[@]}"; do + echo "${REGISTRY}/${NAMESPACE}/${name}:${tag}" + done done } @@ -397,12 +424,52 @@ list_images() { fi } -# Resolve which images to process +# Resolve which images to process. A comma-separated expression is an explicit +# ordered union and includes the base image when it selects a service image. +# Existing single-image, project, and all expressions retain their behavior. resolve_targets() { local target="$1" local all_images all_images=($(discover_images)) + if [[ "${target}" == *,* ]]; then + local -a requested + local -a resolved=("base") + local item + local image + local item_images + local seen_base=0 + declare -A seen=() + + IFS=',' read -ra requested <<< "${target}" + for item in "${requested[@]}"; do + if [[ -z "${item}" || "${item}" != "${item//[[:space:]]/}" ]]; then + echo "ERROR: Invalid empty or whitespace-containing target '${item}'" >&2 + return 1 + fi + if ! item_images=$(resolve_targets "${item}"); then + return 1 + fi + for image in ${item_images}; do + if [[ "${image}" == "base" ]]; then + seen_base=1 + continue + fi + if [[ -z "${seen[${image}]:-}" ]]; then + resolved+=("${image}") + seen["${image}"]=1 + fi + done + done + + if [[ ${#resolved[@]} -eq 1 && ${seen_base} -eq 0 ]]; then + echo "ERROR: Explicit target selection is empty" >&2 + return 1 + fi + echo "${resolved[@]}" + return + fi + if [[ "${target}" == "all" ]]; then echo "${all_images[@]}" return @@ -437,6 +504,17 @@ resolve_targets() { return 1 } +# Resolve an expression without losing failures in command substitutions. +declare -a _RESOLVED_TARGETS=() +resolve_targets_array() { + local target="$1" + local output + if ! output=$(resolve_targets "${target}"); then + return 1 + fi + _RESOLVED_TARGETS=(${output}) +} + # Clone a repo at a branch tip (or tag) and store the resolved commit hash # in _CLONE_RESULT. Must NOT be called via command substitution ($(...)) # because _AUTO_CLONED assignments would be lost in the subshell. @@ -699,7 +777,8 @@ generate_locks_for_targets() { fi local targets - targets=($(resolve_targets "${target}")) + resolve_targets_array "${target}" || return 1 + targets=("${_RESOLVED_TARGETS[@]}") declare -A _lock_projects_seen for img in "${targets[@]}"; do @@ -748,7 +827,8 @@ generate_buildlocks_for_targets() { fi local targets - targets=($(resolve_targets "${target}")) + resolve_targets_array "${target}" || return 1 + targets=("${_RESOLVED_TARGETS[@]}") declare -A _buildlock_projects_seen for img in "${targets[@]}"; do @@ -841,7 +921,8 @@ generate_rpms_in_for_targets() { local target="$1" local targets - targets=($(resolve_targets "${target}")) + resolve_targets_array "${target}" || return 1 + targets=("${_RESOLVED_TARGETS[@]}") declare -A _rpms_projects_seen for img in "${targets[@]}"; do @@ -871,7 +952,8 @@ ensure_sources_for_targets() { fi local targets - targets=($(resolve_targets "${target}")) + resolve_targets_array "${target}" || return 1 + targets=("${_RESOLVED_TARGETS[@]}") declare -A _ensure_projects_seen @@ -910,7 +992,8 @@ update_sources() { fi local targets - targets=($(resolve_targets "${target}")) + resolve_targets_array "${target}" || return 1 + targets=("${_RESOLVED_TARGETS[@]}") declare -A projects_seen @@ -971,17 +1054,32 @@ TARGET="${2:-all}" case "${ACTION}" in build) - for img in $(resolve_targets "${TARGET}"); do + resolve_targets_array "${TARGET}" || exit 1 + for img in "${_RESOLVED_TARGETS[@]}"; do build_image "${img}" done ;; build-parallel) - _bp_targets=($(resolve_targets "${TARGET}")) + resolve_targets_array "${TARGET}" || exit 1 + _bp_targets=("${_RESOLVED_TARGETS[@]}") + if [[ ! "${PARALLEL}" =~ ^[1-9][0-9]*$ ]]; then + echo "ERROR: PARALLEL must be a positive integer" >&2 + exit 1 + fi + if [[ -n "${BUILD_LOGS_DIR:-}" ]]; then + _bp_logdir="${BUILD_LOGS_DIR}" + mkdir -p "${_bp_logdir}" + else + _bp_logdir=$(mktemp -d) + fi # Build base first (all service images depend on it) for _bp_img in "${_bp_targets[@]}"; do [[ -n "$(project_name "${_bp_img}")" ]] && continue - build_image "${_bp_img}" + set -o pipefail + build_image "${_bp_img}" 2>&1 | + sed -u "s|^|[${_bp_img}] |" | + tee "${_bp_logdir}/${_bp_img//\//_}.log" done # Pre-clone sources so parallel builds don't race on the same directories @@ -991,12 +1089,6 @@ case "${ACTION}" in done # Build service images in parallel (max PARALLEL at a time) - if [[ -n "${BUILD_LOGS_DIR:-}" ]]; then - _bp_logdir="${BUILD_LOGS_DIR}" - mkdir -p "${_bp_logdir}" - else - _bp_logdir=$(mktemp -d) - fi _bp_service_imgs=() for _bp_img in "${_bp_targets[@]}"; do [[ -z "$(project_name "${_bp_img}")" ]] && continue @@ -1010,47 +1102,50 @@ case "${ACTION}" in _bp_running=0 for _bp_img in "${_bp_service_imgs[@]}"; do - # Wait for a slot if at the limit while [[ ${_bp_running} -ge ${PARALLEL} ]]; do - if ! wait -n; then + _bp_finished="" + if wait -n -p _bp_finished "${!_bp_pids[@]}"; then + unset '_bp_pids['"${_bp_finished}"']' + ((_bp_running--)) || true + else _bp_fail=1 + [[ -n "${_bp_finished}" ]] && unset '_bp_pids['"${_bp_finished}"']' break 2 fi - ((_bp_running--)) || true done _bp_log="${_bp_logdir}/${_bp_img//\//_}.log" - build_image "${_bp_img}" > "${_bp_log}" 2>&1 & + ( + set -o pipefail + build_image "${_bp_img}" 2>&1 | + sed -u "s|^|[${_bp_img}] |" | + tee "${_bp_log}" + ) & _bp_pids[$!]="${_bp_img}" ((_bp_running++)) || true done - # Wait for remaining builds if [[ ${_bp_fail} -eq 0 ]]; then while [[ ${_bp_running} -gt 0 ]]; do - if ! wait -n; then + _bp_finished="" + if wait -n -p _bp_finished "${!_bp_pids[@]}"; then + unset '_bp_pids['"${_bp_finished}"']' + ((_bp_running--)) || true + else _bp_fail=1 + [[ -n "${_bp_finished}" ]] && unset '_bp_pids['"${_bp_finished}"']' break fi - ((_bp_running--)) || true done fi - # Show logs for all builds - for _bp_log in "${_bp_logdir}"/*.log; do - _bp_name=$(basename "${_bp_log}" .log) - echo "=== ${_bp_name} ===" - cat "${_bp_log}" - echo "" - done - if [[ ${_bp_fail} -eq 1 ]]; then - echo "ERROR: A build failed, killing remaining builds" >&2 + echo "ERROR: A build failed; stopping remaining builds" >&2 for _bp_pid in "${!_bp_pids[@]}"; do kill "${_bp_pid}" 2>/dev/null || true done wait 2>/dev/null || true - [[ -z "${BUILD_LOGS_DIR:-}" ]] && rm -rf "${_bp_logdir}" + echo "Build logs are available in ${_bp_logdir}" >&2 exit 1 fi @@ -1059,7 +1154,8 @@ case "${ACTION}" in fi ;; push) - _push_targets=($(resolve_targets "${TARGET}")) + resolve_targets_array "${TARGET}" || exit 1 + _push_targets=("${_RESOLVED_TARGETS[@]}") # Verify all images and tags exist before pushing any echo "--- Verifying all images exist locally ---" @@ -1072,6 +1168,9 @@ case "${ACTION}" in push_image "${img}" done ;; + refs) + target_refs "${TARGET}" + ;; update-sources) if [[ -n "${SKIP_HASH_UPDATE}" ]]; then echo "=== Skipping hash update (SKIP_HASH_UPDATE is set) ===" @@ -1097,7 +1196,8 @@ case "${ACTION}" in if [[ "${STREAM}" == "${DEFAULT_STREAM}" ]]; then echo "" echo "=== Creating default stream symlinks (${DEFAULT_STREAM}) ===" - _symlink_targets=($(resolve_targets "${TARGET}")) + resolve_targets_array "${TARGET}" || exit 1 + _symlink_targets=("${_RESOLVED_TARGETS[@]}") declare -A _symlink_seen for _s_img in "${_symlink_targets[@]}"; do _s_project="$(project_name "${_s_img}")" @@ -1141,7 +1241,7 @@ case "${ACTION}" in list_images ;; *) - echo "Usage: STREAM= $0 {build|build-parallel|push|update-sources|install-deps|list} [image-name|all]" + echo "Usage: STREAM= $0 {build|build-parallel|push|refs|update-sources|install-deps|list} [image-name|all]" echo "" echo "Images (discovered from containers/):" for dir_name in $(discover_images); do @@ -1165,6 +1265,8 @@ case "${ACTION}" in echo " BUILD_LOGS_DIR Persist build-parallel logs to this directory" echo " SKIP_HASH_UPDATE Skip updating pinned hashes; regenerate locks only" echo " PIP_NO_BINARY Pass PIP_NO_BINARY to container build (e.g., ':all:')" + echo " REGISTRY_AUTH_FILE Registry authentication file for pushes" + echo " REGISTRY_CERT_DIR Registry TLS certificate directory for pushes" echo "" echo "Source directories: containers//src//" echo "Overrides: containers//src/overrides//" diff --git a/containers/base/image.yaml b/containers/base/image.yaml new file mode 100644 index 0000000..c8ad192 --- /dev/null +++ b/containers/base/image.yaml @@ -0,0 +1,3 @@ +--- +openstack_version: + custom_container_images: [] diff --git a/containers/cyborg/cyborg-agent/image.yaml b/containers/cyborg/cyborg-agent/image.yaml new file mode 100644 index 0000000..c8ad192 --- /dev/null +++ b/containers/cyborg/cyborg-agent/image.yaml @@ -0,0 +1,3 @@ +--- +openstack_version: + custom_container_images: [] diff --git a/containers/cyborg/cyborg/image.yaml b/containers/cyborg/cyborg/image.yaml new file mode 100644 index 0000000..c8ad192 --- /dev/null +++ b/containers/cyborg/cyborg/image.yaml @@ -0,0 +1,3 @@ +--- +openstack_version: + custom_container_images: [] diff --git a/containers/watcher/rpms.in.yaml b/containers/watcher/rpms.in.yaml index 1ad985c..9a0608d 100644 --- a/containers/watcher/rpms.in.yaml +++ b/containers/watcher/rpms.in.yaml @@ -19,11 +19,15 @@ packages: - gcc-c++ - git-core - httpd + - libffi - libffi-devel + - libxml2 - libxml2-devel + - libxslt - libxslt-devel - mod_ssl - openssl-devel + - openssl-libs - python3 - python3-cryptography - python3-devel diff --git a/containers/watcher/watcher-base/Containerfile b/containers/watcher/watcher-base/Containerfile index 5d979e9..323d4c1 100644 --- a/containers/watcher/watcher-base/Containerfile +++ b/containers/watcher/watcher-base/Containerfile @@ -72,8 +72,8 @@ RUN mkdir -p /configfiles/etc/watcher && \ # --- Runtime stage --- FROM ${BASE_IMAGE} -LABEL summary="OpenStack Watcher API" \ - io.k8s.description="Watcher API service (WSGI via httpd) built from source with kolla interface" +LABEL summary="OpenStack Watcher services" \ + io.k8s.description="Consolidated Watcher API, applier, and decision-engine image built from source with the Kolla interface" RUN uid_gid_manage watcher @@ -103,7 +103,7 @@ COPY --from=build /configfiles/etc/watcher/watcher.conf /etc/watcher/watcher.con RUN chmod 640 /etc/watcher/watcher.conf && \ chown watcher:watcher /etc/watcher/watcher.conf -# Apache httpd setup for WSGI +# Apache httpd setup for the API process in the consolidated image RUN sed -i -r 's,^(Listen 80),#\1,' /etc/httpd/conf/httpd.conf && \ sed -i -r 's,^(Listen 443),#\1,' /etc/httpd/conf.d/ssl.conf 2>/dev/null || true && \ usermod --append --groups apache watcher diff --git a/containers/watcher/watcher-base/bindeps.txt b/containers/watcher/watcher-base/bindeps.txt index 6696112..35693e7 100644 --- a/containers/watcher/watcher-base/bindeps.txt +++ b/containers/watcher/watcher-base/bindeps.txt @@ -1,7 +1,12 @@ -# Apache WSGI server for watcher-api +# API WSGI runtime httpd mod_ssl python3-mod_wsgi +# Shared Watcher API, applier, and decision-engine runtime python3 python3-pip python3-cryptography +libffi +libxml2 +libxslt +openssl-libs diff --git a/containers/watcher/watcher-base/image.yaml b/containers/watcher/watcher-base/image.yaml new file mode 100644 index 0000000..6897dee --- /dev/null +++ b/containers/watcher/watcher-base/image.yaml @@ -0,0 +1,6 @@ +--- +openstack_version: + custom_container_images: + - watcherAPIImage + - watcherApplierImage + - watcherDecisionEngineImage diff --git a/developer-guide.md b/developer-guide.md new file mode 100644 index 0000000..6670f81 --- /dev/null +++ b/developer-guide.md @@ -0,0 +1,682 @@ +# S2I OpenStack Containers Developer Guide + +This guide explains how this repository builds OpenStack service containers, +how source pins and generated dependency files fit together, and how +contributors can validate changes. It describes only workflows available in +this repository today. + +Konflux is authoritative for hermetic production provenance and publication. +The shell and GitHub workflows described here are contributor and development +interfaces; they do not replace that production authority. + +## How to read this guide + +New contributors should read Parts I through III before changing a +Containerfile or dependency file. Parts IV through VI are task-oriented +references. + +```text +Part I Purpose, repository map, and terminology +Part II Host preparation and quick starts +Part III Sources, generated files, and build architecture +Part IV Build and source-maintenance workflows +Part V Testing and validation +Part VI Maintenance and troubleshooting +``` + +Headings are intentionally not manually numbered below the part level. Use the +Markdown outline or search for a heading name when following a reference. + +# Part I - Orientation + +## Repository purpose + +The repository produces UBI 10 based OpenStack service images from upstream Git +sources. It keeps service revisions, Python constraints, build requirements, +and RPM input lists in version control so a review can show the inputs expected +by a build. + +The current image set is: + +```text +base openstack-base +cyborg/cyborg openstack-cyborg +cyborg/cyborg-agent openstack-cyborg-agent +watcher/watcher-base openstack-watcher-base +``` + +`build.sh` is the build and source-maintenance interface. Buildah performs +image builds and pushes. Podman is useful for inspecting and running the +resulting images. + +## Publication authority + +Konflux owns hermetic production provenance and publication. Treat its recorded +inputs and outputs as authoritative for production images. + +The GitHub and Zuul workflows have narrower development roles. The abstract +Zuul content-provider job publishes selected speculative images only to its +ephemeral buildset registry; it is not a production publication lane. + +The GitHub workflows have these roles: + +- `build.yml` builds images for pull requests and manual runs. +- `test.yml` runs tests and verifies source regeneration. +- `linter.yml` runs repository checks. +- `update-sources.yml` proposes source and generated-file updates. +- `build-and-push.yml` builds on pushes to `main` and can be started manually. + +The last workflow logs in to Quay, but its push step is commented out. It does +not currently publish images. Do not describe successful GitHub login or image +builds as a production publication path. + +## Repository map + +```text +README.md short project entry point + +developer-guide.md contributor architecture and workflow reference +TESTING.md concise compatibility testing entry point +build.sh build, push, and source-maintenance implementation +tox.ini dependency-managed contributor commands +pyproject.toml Python lint and format configuration +.pre-commit-config.yaml repository-wide quality checks +.ansible-lint Ansible lint policy +.zuul.yaml reusable test and content-provider jobs +.github/workflows/ GitHub development automation +playbooks/container-ci/ shared builder and Zuul adapter playbooks +containers/base/ common UBI-based runtime image +containers/cyborg/ Cyborg source and image contexts +containers/watcher/ Watcher source and image context +tests/ stdlib unittest suite +.tmp/ ignored generated test and tool state +``` + +A service project directory contains shared source and dependency data. Each +image below it contains its Containerfile and image-specific dependency lists. + +```text +containers// + sources.txt + requirements.lock. + buildrequirements.lock. + upper-constraints.txt. + rpms.in.yaml + src/ + / + Containerfile + image.yaml + bindeps.txt + builddeps.txt + pythondeps.txt + pythonbuilddeps.txt +``` + +## Terminology + +**Image target** +: A buildable directory known to `build.sh`, such as `base`, + `cyborg/cyborg-agent`, or `watcher/watcher-base`. + +**Project** +: A directory below `containers/` that can share sources and generated + dependency data across one or more images. + +**Stream** +: A named set of source revisions and generated dependency files. `master` is + the maintained default stream. + +**Source pin** +: The exact Git commit in a `sources.txt` record. The adjacent branch name + records which branch source maintenance follows; builds use the commit. + +**Build context** +: The project directory passed to Buildah. It contains project-level sources + and the selected image subdirectory. + +**Maintained input** +: A file edited by contributors, such as a Containerfile, `sources.txt`, or a + dependency list. + +**Generated file** +: A tracked file produced by `build.sh update-sources`, such as a lock file, + constraints snapshot, RPM input file, or default-stream symlink. + +**Content provider** +: A paused Zuul job that builds and publishes selected images to the buildset's + ephemeral registry, then returns exact references to dependent jobs. + +**Deployment key** +: An OpenStackVersion custom-container-image field listed in an image's local + `image.yaml`. The provider expands that field to the image's exact successful + buildset reference. Empty key lists are valid. + +# Part II - Preparing and Building + +## Host prerequisites + +Building requires Linux, Git, Buildah, Podman, and enough storage for UBI base +layers and Python wheels. Source maintenance also needs the Python tools pinned +by tox. + +Install the container tools supported by the host package manager with: + +```console +./build.sh install-deps +``` + +The command uses `sudo` and supports `dnf`, `microdnf`, and `apt-get`. It does +not configure registry credentials or alter source pins. + +Tox manages Python test and generation dependencies. Use the repository's tox +environments rather than installing those packages into the system Python. + +## Quick start: inspect targets + +```console +./build.sh list +``` + +This discovers targets from the `containers/` directory and prints their final +image names. + +## Quick start: build everything + +```console +STREAM=master tox -e build +``` + +The build uses maintained `master` pins. It builds the base before service +images and tags images under the default local naming scheme. + +For a narrower build, call the shell interface directly or through `custom`: + +```console +STREAM=master ./build.sh build watcher +STREAM=master tox -e custom -- build cyborg/cyborg-agent +``` + +A project target builds all images in that project. An image target builds only +that image and its required base. + +## Quick start: run checks + +```console +tox -e unit +tox -e linters +``` + +The `test` and `py3` environments are aliases for the same unit suite: + +```console +tox -e test +tox -e py3 +``` + +# Part III - Sources and Build Architecture + +## Source records + +A non-comment `sources.txt` line has five fields: + +```text + +``` + +For example: + +```text +master watcher https://opendev.org/openstack/watcher.git master +``` + +The branch field is maintenance input. A normal build checks out the exact +commit. Updating branch tips is an explicit source-maintenance action. + +Source records can appear at global, project, or image level. Project sources +are shared by every image in the project. Image sources add dependencies needed +by only one image. `upper-constraints` is special: its file is fetched from the +pinned OpenStack requirements revision instead of being treated as a service +checkout. + +## Source checkout ownership + +Before a build, `build.sh` places missing repositories below the applicable +`src/` directory. Checkouts created by the script are removed by its exit trap. +A checkout already present before the command is used as-is and is not removed. +This permits an intentional developer checkout to replace a maintained pin for +a local experiment. + +That override is powerful and visible only in the local filesystem. Remove it +before claiming that a result came from maintained pins. + +A transitive source override can be placed at: + +```text +containers//src/overrides// +``` + +Containerfiles build directories found there as source packages. No additional +`sources.txt` entry is required for this explicit developer override. + +## Maintained and generated files + +Edit these inputs directly: + +- `Containerfile`; +- `sources.txt`; +- `bindeps.txt` and `builddeps.txt`; +- `pythondeps.txt` and `pythonbuilddeps.txt`; +- base scripts and repository-maintained service configuration. + +Do not hand-edit these generated outputs: + +- `upper-constraints.txt.`; +- `requirements.lock.`; +- `buildrequirements.lock.`; +- `rpms.in.yaml`; +- unsuffixed default-stream symlinks. + +When a source record or dependency input changes, regenerate the related files +in the same change. Review both the maintained input and generated diff. + +Tool caches, test state, and temporary checkouts belong below ignored `.tmp/`. +Tests must not leave generated state elsewhere in the repository. + +## Base image + +`containers/base/Containerfile` starts from UBI 10 minimal. It installs common +RPM and Python dependencies, Kolla helper scripts, service user/group support, +and the common entry point. Service images refer to its resulting tag through +the `BASE_IMAGE` build argument. + +## Service images + +Service Containerfiles use two stages. + +The build stage: + +1. copies project and image source directories; +2. installs build-only RPM and Python dependencies; +3. removes source-built package names from effective constraints; +4. builds service and override wheels; +5. builds remaining dependency wheels; +6. records source package versions and commits; and +7. generates service configuration where required. + +The runtime stage: + +1. starts from `openstack-base`; +2. creates the service user; +3. installs runtime RPM dependencies; +4. installs the built wheels under the filtered constraints; +5. installs configuration and Kolla integration; and +6. switches to the deployed service user. + +`/source-built-packages.txt` in a service image records package, commit, and +version information collected during the build. + +## Image names and tags + +The default naming inputs are: + +```text +REGISTRY localhost +NAMESPACE openstack +IMAGE_PREFIX openstack +TAG -latest +``` + +A target such as `watcher/watcher-base` therefore becomes +`localhost/openstack/openstack-watcher-base:master-latest` for the `master` +stream. `TAG` accepts a comma-separated list when multiple tags are needed. + +# Part IV - Contributor Workflows + +## Build commands + +Build one image serially: + +```console +STREAM=master ./build.sh build watcher/watcher-base +``` + +Build all service images with bounded parallelism: + +```console +STREAM=master PARALLEL=4 ./build.sh build-parallel all +``` + +The parallel command builds the base first, prepares shared sources, then runs +service builds concurrently. Set `BUILD_LOGS_DIR` to retain per-image logs. + +Useful overrides include: + +```text +REGISTRY +NAMESPACE +TAG +IMAGE_PREFIX +BASE_IMAGE +CONSTRAINTS_FILE +BUILD_CONSTRAINTS_FILE +PARALLEL +BUILD_LOGS_DIR +PIP_NO_BINARY +``` + +Use `PIP_NO_BINARY=:all:` when specifically testing source distribution builds. +It increases build time and is not needed for ordinary validation. + +## Push commands + +Authenticate to the destination registry before pushing. Then run: + +```console +STREAM=master REGISTRY=quay.io NAMESPACE= \ + ./build.sh push all +``` + +The push operation verifies that every expected local tag exists before it +pushes any target. This command is an explicit contributor action. It does not +change Konflux's production authority. + +## Selective Zuul content provider + +The abstract `s2i-openstack-container-content-provider` job runs on the +CentOS Stream 10 nodeset host named `builder`. All host preparation, registry +validation, builds, publication, result generation, and cleanup target that +host explicitly. The Zuul executor controls Ansible but does not perform those +mutations. + +The provider defaults to this explicit set: + +```text +watcher/watcher-base +cyborg/cyborg +cyborg/cyborg-agent +``` + +`build.sh` automatically places `base` first for this comma-separated explicit +selection. Existing single-image, project, and `all` shell targets keep their +standalone behavior. The provider uses normal `build.sh` source handling, so it +builds service content from the exact maintained `sources.txt` pins. It does +not consume speculative service checkouts at this stage. + +### Image deployment metadata + +Every buildable Containerfile has a sibling `image.yaml` with this local +schema: + +```yaml +openstack_version: + custom_container_images: [] +``` + +The list may be empty or contain multiple deployment keys. The consolidated +`watcher/watcher-base` image declares: + +```yaml +openstack_version: + custom_container_images: + - watcherAPIImage + - watcherApplierImage + - watcherDecisionEngineImage +``` + +All three keys resolve to the same exact `openstack-watcher-base` reference. +The image contains the API, applier, and decision-engine entry points and the +union of their runtime dependencies. Watcher is intentionally not split into +process-specific images. + +Both Cyborg images build and publish, but their tracked key lists are empty. +Their exact references therefore appear in provider diagnostics without adding +fields to the default deployment map. + +A child job may provide `s2i_ci_image_mappings` as a mapping from a selected +image target to a replacement list of keys. Replacement is per image rather +than additive. The provider records whether each effective list came from +tracked or inventory metadata and rejects malformed values, unknown or unbuilt +image targets, empty key strings, duplicate keys, and a key assigned to more +than one image. + +### Registry and returned data + +The provider starts or inherits a Zuul buildset registry, validates push and +pull with a dedicated UBI tag, builds and pushes the selected image set, and +pulls every exact result back. Credentials and certificate data remain in +Zuul secret data. Returned public diagnostics use the buildset registry's +reachable host or IP and port, never the builder-local registry alias. + +`s2i_ci_content.images` contains every exact successful reference, including +base and both unmapped Cyborg images. The partial +`content_provider_os_custom_container_images` map contains only effective +keys joined to exact successful references. The legacy global OS registry URL +remains the neutral `null` sentinel, while its namespace/tag and gating-repo +fields remain empty or false because this selective provider does not publish a +complete OpenStack image namespace. `cifmw_build_images_output` +remains an empty mapping and is not repurposed for service images. + +Intended references are written before build mutation. Post-run cleanup +removes only those exact Podman pullback and Buildah build tags, verifies exact +absence, and removes a buildset registry only when its ownership marker is +valid. +Per-image parallel logs and registry/result manifests are retained under +`zuul-output/logs/container-build/`. + +The provider pauses while dependent jobs run. Private onboarding may attach a +trivial child that prints the returned registry paths and maps. That debug job +does not pull images, patch an OpenStackVersion resource, deploy OpenStack, or +invoke a downstream repository's playbooks. Downstream consumption is separate +work. + +## Updating source pins and locks + +Advance source records and regenerate dependent files with: + +```console +STREAM=master tox -e update-sources +``` + +To restrict work to a project: + +```console +STREAM=master TARGET=watcher tox -e update-sources +``` + +This resolves configured branch tips, updates exact source commits, refreshes +constraints, regenerates RPM inputs and both lock-file classes, and recreates +default-stream symlinks. + +To regenerate from the existing commits without advancing them: + +```console +STREAM=master SKIP_HASH_UPDATE=1 tox -e update-sources +``` + +Always inspect `git diff -- containers/`. A clean command exit does not replace +review of source changes and generated dependency movement. + +## Adding an image + +1. Create `containers///Containerfile`. +2. Add its runtime and build dependency files. +3. Add project-level or image-level source records as appropriate. +4. Add an image `src/.gitkeep` only when an image-level source directory is + needed. +5. Run source generation for the project. +6. Build the image target. +7. Run unit and linter validation. + +Follow an existing service Containerfile with similar runtime behavior. Keep +build-only packages out of `bindeps.txt`, and keep runtime packages out of +`builddeps.txt` unless they are genuinely needed in both stages. + +# Part V - Testing and Validation + +## Validation strategy + +Start with the narrowest check that proves a change, then broaden validation +before publishing it. Tests use stdlib `unittest`; shell test harnesses are not +part of the supported non-Ansible test model. + +A practical progression is: + +1. run the affected unit test module; +2. run `tox -e unit`; +3. run `tox -e linters`; +4. regenerate files when maintained inputs changed; and +5. build the affected image or project. + +## Unit tests + +Run the complete suite with: + +```console +tox -e unit +``` + +For focused iteration, pass a unittest name pattern through the environment: + +```console +tox -e unit -- -k test_name_pattern +``` + +Tests use temporary directories and local bare Git remotes so they do not +change a contributor's source checkouts or container storage. + +## Formatting and static analysis + +Run: + +```console +tox -e linters +``` + +Pre-commit checks whitespace, YAML, JSON, Python syntax, shell style, +Containerfiles, Ruff formatting and lint rules, OpenStack Hacking conventions, +and Ansible lint policy. Configuration is committed so local and automation +interfaces use the same rules. + +## Automation coverage + +GitHub runs build, unit, linter, and source-update workflows as described in +`Publication authority`. The repository also defines reusable unit, linter, +Molecule, and abstract selective content-provider jobs in `.zuul.yaml`; project +attachment is intentionally managed outside those job definitions. + +The provider's Molecule scenario exercises normalized registry validation, +tracked and inventory deployment metadata, exact-reference expansion, and +matching Podman/Buildah cleanup with self-contained fake container clients. + +Neither development automation path changes Konflux's authority for hermetic +production provenance and publication. + +## Generated-state checks + +After tests or source generation, run: + +```console +git status --short +git diff --check +``` + +Unexpected tracked changes indicate a test isolation bug or stale generated +files. Tool-created untracked state should remain below `.tmp/`. + +# Part VI - Maintenance and Troubleshooting + +## Change-to-validation map + +| Change | Minimum focused validation | +| --- | --- | +| Python test/helper | affected unittest, `tox -e unit`, `tox -e linters` | +| Containerfile | `tox -e linters`, affected image build | +| Python dependency input | regenerate locks, unit, linter, affected build | +| RPM dependency input | regenerate `rpms.in.yaml`, linter, affected build | +| `sources.txt` | regenerate project, inspect pins and locks, affected build | +| base image or script | unit, linter, build all service images | +| GitHub workflow | YAML/linter checks and workflow review | + +## Build uses an unexpected source revision + +Check for a pre-existing checkout below the project or image `src/` directory. +Such a checkout intentionally overrides `sources.txt`. Compare its `HEAD` with +the maintained pin, then remove or relocate it when a pinned build is required. + +## A generated file changes unexpectedly + +Confirm `STREAM`, `TARGET`, `DEFAULT_STREAM`, and `SKIP_HASH_UPDATE`. Verify +that all source records contain the expected exact commits. Re-run generation +from a clean tree and compare the complete `containers/` diff. + +Moving branch tips can legitimately change pins when `SKIP_HASH_UPDATE` is not +set. Use keep-pins regeneration when testing determinism against committed +sources. + +## A service package still comes from an index + +Verify the service repository exists in the applicable `sources.txt`, that the +checkout appears under `src/`, and that its package metadata can be recognized +by the Containerfile. Inspect `/source-built-packages.txt` in a successful +image. + +## A build cannot find constraints or locks + +Check the stream-suffixed files in the project directory and the unsuffixed +symlinks used by Containerfiles. Regenerate the project for the selected stream +rather than manually repairing a generated symlink or lock file. + +## A parallel build fails + +Set `BUILD_LOGS_DIR` and inspect the retained per-image files: + +```console +STREAM=master BUILD_LOGS_DIR=.tmp/build-logs \ + ./build.sh build-parallel all +``` + +Parallel service output is prefixed by image and streamed while each build +runs. The command preserves the failing exit status, stops outstanding builds, +and reports the log directory without replaying every successful log. + +## Provider output or cleanup is incomplete + +Read provider artifacts in this order: + +```text +intended-images.json +registry-state.json +build.log and per-image logs +push.log +published-images.json +``` + +`intended-images.json` exists before builds start and is the cleanup authority +after partial failure. `published-images.json` contains the completed exact +references, effective mappings, and mapping-source diagnostics. Cleanup failure +is a job failure; compare the recorded references with both Podman and Buildah +image listings. + +Parallel image lines are prefixed by target and emitted while builds run. The +same prefixed lines remain in per-image logs, without a second successful-log +replay at the end. + +## Registry push fails + +Confirm login state, registry certificate trust, the `REGISTRY` and `NAMESPACE` +values, and every expected local tag. The push command checks tags before +starting, but authentication and network errors can still interrupt +publication. + +## Contributor checklist + +Before proposing a change: + +1. confirm maintained and generated files are separated correctly; +2. run the narrow unit test and the complete unit suite; +3. run linters; +4. regenerate dependencies when an input changed; +5. build the affected image set; +6. inspect `git status`, `git diff`, and `git diff --check`; and +7. describe development publication accurately and keep Konflux production + authority explicit. diff --git a/molecule/provider-contract/cleanup.yml b/molecule/provider-contract/cleanup.yml new file mode 100644 index 0000000..9b147fe --- /dev/null +++ b/molecule/provider-contract/cleanup.yml @@ -0,0 +1,13 @@ +--- +- name: Remove provider contract fixtures + hosts: builder + gather_facts: false + vars: + s2i_test_root: >- + {{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}/.tmp/molecule/provider-contract + + tasks: + - name: Remove provider contract fixture state + ansible.builtin.file: + path: "{{ s2i_test_root }}" + state: absent diff --git a/molecule/provider-contract/converge.yml b/molecule/provider-contract/converge.yml new file mode 100644 index 0000000..dccad80 --- /dev/null +++ b/molecule/provider-contract/converge.yml @@ -0,0 +1,122 @@ +--- +- name: Validate the fake normalized registry + ansible.builtin.import_playbook: >- + ../../playbooks/container-ci/shared/validate-registry.yaml + vars: + s2i_ci_output_dir: >- + {{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}/.tmp/molecule/provider-contract/zuul-output + +- name: Prepare exact cleanup fixtures + hosts: builder + gather_facts: false + vars: + s2i_test_repo_root: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}" + s2i_test_root: "{{ s2i_test_repo_root }}/.tmp/molecule/provider-contract" + s2i_test_images: + - registry.test:5000/openstack/openstack-base:test + - registry.test:5000/openstack/openstack-watcher-base:test + s2i_ci_container_repo: "{{ s2i_test_repo_root }}" + s2i_ci_selected_images: + - base + - cyborg/cyborg + - watcher/watcher-base + s2i_ci_image_mappings: + cyborg/cyborg: + - futureCyborgImage + + tasks: + - name: Initialize metadata expansion fixtures + ansible.builtin.set_fact: + s2i_ci_effective_image_mappings: {} + s2i_ci_mapping_sources: {} + + - name: Expand tracked and inventory image metadata + ansible.builtin.include_tasks: >- + {{ s2i_test_repo_root }}/playbooks/container-ci/shared/image-metadata-item.yaml + loop: "{{ s2i_ci_selected_images }}" + loop_control: + loop_var: s2i_ci_image + + - name: Validate effective deployment key ownership + ansible.builtin.include_tasks: >- + {{ s2i_test_repo_root }}/playbooks/container-ci/shared/validate-deployment-keys.yaml + + - name: Require tracked and replacement metadata expansion + ansible.builtin.assert: + that: + - s2i_ci_effective_image_mappings['base'] == [] + - s2i_ci_effective_image_mappings['cyborg/cyborg'] == + ['futureCyborgImage'] + - s2i_ci_effective_image_mappings['watcher/watcher-base'] | length == 3 + - s2i_ci_mapping_sources['cyborg/cyborg'] == 'inventory' + - s2i_ci_mapping_sources['watcher/watcher-base'] == 'tracked' + + - name: Initialize exact reference expansion fixtures + ansible.builtin.set_fact: + s2i_ci_target_references: + base: registry.test:5000/openstack/openstack-base:test + cyborg/cyborg: registry.test:5000/openstack/openstack-cyborg:test + watcher/watcher-base: >- + registry.test:5000/openstack/openstack-watcher-base:test + s2i_ci_custom_container_images: {} + + - name: Expand exact metadata references + ansible.builtin.include_tasks: >- + {{ s2i_test_repo_root }}/playbooks/container-ci/shared/deployment-map-item.yaml + loop: "{{ s2i_ci_selected_images }}" + loop_control: + loop_var: s2i_ci_image + + - name: Require consolidated Watcher reference expansion + ansible.builtin.assert: + that: + - s2i_ci_custom_container_images.watcherAPIImage == + s2i_ci_target_references['watcher/watcher-base'] + - s2i_ci_custom_container_images.watcherApplierImage == + s2i_ci_target_references['watcher/watcher-base'] + - s2i_ci_custom_container_images.watcherDecisionEngineImage == + s2i_ci_target_references['watcher/watcher-base'] + - s2i_ci_custom_container_images.futureCyborgImage == + s2i_ci_target_references['cyborg/cyborg'] + + - name: Read fake container storage after registry validation + ansible.builtin.slurp: + src: "{{ s2i_test_root }}/state.json" + register: s2i_test_state_data + + - name: Decode fake container storage + ansible.builtin.set_fact: + s2i_test_state: >- + {{ s2i_test_state_data.content | b64decode | from_json }} + + - name: Add exact workflow tags to both storage clients + ansible.builtin.copy: + dest: "{{ s2i_test_root }}/state.json" + mode: "0600" + content: | + {{ s2i_test_state | + combine({ + 'podman': s2i_test_state.podman + s2i_test_images, + 'buildah': s2i_test_images + }) | to_nice_json }} + + - name: Write intended exact cleanup metadata + ansible.builtin.copy: + dest: >- + {{ s2i_test_root }}/zuul-output/logs/container-build/intended-images.json + mode: "0644" + content: | + {{ { + 'registry': 'registry.test:5000', + 'namespace': 'openstack', + 'images': s2i_test_images, + 'selected_images': ['base', 'watcher/watcher-base'], + 'target': 'watcher/watcher-base' + } | to_nice_json }} + +- name: Exercise exact shared cleanup + ansible.builtin.import_playbook: >- + ../../playbooks/container-ci/shared/cleanup-images.yaml + vars: + s2i_ci_output_dir: >- + {{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}/.tmp/molecule/provider-contract/zuul-output diff --git a/molecule/provider-contract/duplicate-metadata.yml b/molecule/provider-contract/duplicate-metadata.yml new file mode 100644 index 0000000..faedb61 --- /dev/null +++ b/molecule/provider-contract/duplicate-metadata.yml @@ -0,0 +1,16 @@ +--- +- name: Reject duplicate deployment keys + hosts: builder + gather_facts: false + vars: + s2i_test_repo_root: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}" + s2i_ci_effective_image_mappings: + base: + - duplicateImage + watcher/watcher-base: + - duplicateImage + + tasks: + - name: Validate duplicate fixture + ansible.builtin.include_tasks: >- + {{ s2i_test_repo_root }}/playbooks/container-ci/shared/validate-deployment-keys.yaml diff --git a/molecule/provider-contract/files/fake-container-client.py b/molecule/provider-contract/files/fake-container-client.py new file mode 100755 index 0000000..ade2a2c --- /dev/null +++ b/molecule/provider-contract/files/fake-container-client.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 + +import json +import os +import pathlib +import sys + + +state_path = pathlib.Path(os.environ["S2I_FAKE_STATE"]) +client = pathlib.Path(sys.argv[0]).name +arguments = sys.argv[1:] +state = json.loads(state_path.read_text(encoding="utf-8")) +state["commands"].append([client, *arguments]) +images = state[client] +return_code = 0 + +if client == "podman": + if arguments[:1] == ["pull"]: + images.append(arguments[-1]) + elif arguments[:1] == ["tag"]: + images.append(arguments[-1]) + elif arguments[:1] == ["push"]: + pass + elif arguments[:1] == ["untag"]: + if arguments[-1] in images: + images.remove(arguments[-1]) + elif arguments[:2] in (["image", "inspect"], ["image", "exists"]): + return_code = 0 if arguments[-1] in images else 1 + elif arguments[:2] == ["image", "rm"]: + if arguments[-1] in images: + images.remove(arguments[-1]) + else: + return_code = 1 + elif arguments[:1] == ["images"]: + print("\n".join(images)) + else: + return_code = 2 +elif client == "buildah": + if arguments[:1] == ["inspect"]: + return_code = 0 if arguments[-1] in images else 125 + elif arguments[:1] == ["rmi"]: + if arguments[-1] in images: + images.remove(arguments[-1]) + else: + return_code = 125 + elif arguments[:1] == ["images"]: + print("\n".join(images)) + else: + return_code = 2 +else: + return_code = 2 + +state[client] = list(dict.fromkeys(images)) +state_path.write_text(json.dumps(state, indent=2) + "\n", encoding="utf-8") +sys.exit(return_code) diff --git a/molecule/provider-contract/malformed-mapping.yml b/molecule/provider-contract/malformed-mapping.yml new file mode 100644 index 0000000..60c2033 --- /dev/null +++ b/molecule/provider-contract/malformed-mapping.yml @@ -0,0 +1,18 @@ +--- +- name: Reject malformed mapping value + hosts: builder + gather_facts: false + vars: + s2i_test_repo_root: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}" + s2i_ci_container_repo: "{{ s2i_test_repo_root }}" + s2i_ci_image_mappings: + base: not-a-list + s2i_ci_effective_image_mappings: {} + s2i_ci_mapping_sources: {} + + tasks: + - name: Validate malformed mapping fixture + ansible.builtin.include_tasks: >- + {{ s2i_test_repo_root }}/playbooks/container-ci/shared/image-metadata-item.yaml + vars: + s2i_ci_image: base diff --git a/molecule/provider-contract/molecule.yml b/molecule/provider-contract/molecule.yml new file mode 100644 index 0000000..d3eed45 --- /dev/null +++ b/molecule/provider-contract/molecule.yml @@ -0,0 +1,34 @@ +--- +driver: + name: default + options: + managed: false + ansible_connection_options: + ansible_connection: local + +platforms: + - name: localhost + groups: + - builder + +provisioner: + name: ansible + env: + ANSIBLE_LOCAL_TEMP: "${MOLECULE_PROJECT_DIRECTORY}/.tmp/ansible/local-tmp" + PATH: "${MOLECULE_PROJECT_DIRECTORY}/.tmp/molecule/provider-contract/bin:${PATH}" + S2I_FAKE_STATE: "${MOLECULE_PROJECT_DIRECTORY}/.tmp/molecule/provider-contract/state.json" + inventory: + host_vars: + localhost: + ansible_connection: local + +verifier: + name: ansible + +scenario: + test_sequence: + - syntax + - prepare + - converge + - verify + - cleanup diff --git a/molecule/provider-contract/prepare.yml b/molecule/provider-contract/prepare.yml new file mode 100644 index 0000000..3874ab9 --- /dev/null +++ b/molecule/provider-contract/prepare.yml @@ -0,0 +1,57 @@ +--- +- name: Prepare provider contract fixtures + hosts: builder + gather_facts: false + vars: + s2i_test_repo_root: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}" + s2i_test_root: "{{ s2i_test_repo_root }}/.tmp/molecule/provider-contract" + + tasks: + - name: Reset provider contract fixture state + ansible.builtin.file: + path: "{{ s2i_test_root }}" + state: absent + + - name: Create provider contract fixture directories + ansible.builtin.file: + path: "{{ item }}" + state: directory + mode: "0755" + loop: + - "{{ s2i_test_root }}/bin" + - "{{ s2i_test_root }}/zuul-output/logs/container-build" + + - name: Install the fake container client + ansible.builtin.copy: + src: fake-container-client.py + dest: "{{ s2i_test_root }}/bin/fake-container-client.py" + mode: "0755" + + - name: Link fake Podman and Buildah clients + ansible.builtin.file: + src: "{{ s2i_test_root }}/bin/fake-container-client.py" + dest: "{{ s2i_test_root }}/bin/{{ item }}" + state: link + loop: + - podman + - buildah + + - name: Initialize fake container storage + ansible.builtin.copy: + dest: "{{ s2i_test_root }}/state.json" + mode: "0600" + content: | + {{ {'podman': [], 'buildah': [], 'commands': []} | to_nice_json }} + + - name: Write normalized registry connection + ansible.builtin.copy: + dest: "{{ s2i_test_root }}/zuul-output/.s2i-ci-registry-connection.json" + mode: "0600" + content: | + {{ { + 'endpoint': 'registry.test:5000', + 'public_host': '192.0.2.10', + 'public_port': 5000, + 'auth_file': '', + 'cert_dir': '' + } | to_nice_json }} diff --git a/molecule/provider-contract/unknown-mapping.yml b/molecule/provider-contract/unknown-mapping.yml new file mode 100644 index 0000000..73dec15 --- /dev/null +++ b/molecule/provider-contract/unknown-mapping.yml @@ -0,0 +1,17 @@ +--- +- name: Reject unknown mapping target + hosts: builder + gather_facts: false + vars: + s2i_test_repo_root: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}" + s2i_ci_selected_images: + - base + - watcher/watcher-base + s2i_ci_image_mappings: + cyborg/cyborg: + - unknownImage + + tasks: + - name: Validate unknown mapping fixture + ansible.builtin.include_tasks: >- + {{ s2i_test_repo_root }}/playbooks/container-ci/shared/validate-mapping-overrides.yaml diff --git a/molecule/provider-contract/verify.yml b/molecule/provider-contract/verify.yml new file mode 100644 index 0000000..0243d40 --- /dev/null +++ b/molecule/provider-contract/verify.yml @@ -0,0 +1,89 @@ +--- +- name: Verify provider registry and cleanup contracts + hosts: builder + gather_facts: false + vars: + s2i_test_repo_root: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}" + s2i_test_root: "{{ s2i_test_repo_root }}/.tmp/molecule/provider-contract" + s2i_test_validation_image: >- + registry.test:5000/s2i-validation/ubi10-minimal:latest + s2i_test_images: + - registry.test:5000/openstack/openstack-base:test + - registry.test:5000/openstack/openstack-watcher-base:test + + tasks: + - name: Exercise invalid metadata contracts + ansible.builtin.command: + argv: + - ansible-playbook + - -i + - builder, + - -c + - local + - "{{ lookup('env', 'MOLECULE_SCENARIO_DIRECTORY') }}/{{ item }}" + loop: + - duplicate-metadata.yml + - unknown-mapping.yml + - malformed-mapping.yml + register: s2i_test_invalid_metadata + changed_when: false + failed_when: false + + - name: Require invalid metadata to fail deterministically + ansible.builtin.assert: + that: + - s2i_test_invalid_metadata.results | + map(attribute='rc') | reject('equalto', 0) | list | length == 3 + - >- + 'A deployment key may be assigned to only one image' in + (s2i_test_invalid_metadata.results[0].stdout + + s2i_test_invalid_metadata.results[0].stderr) + - >- + 'Image mapping overrides must name selected image targets' in + (s2i_test_invalid_metadata.results[1].stdout + + s2i_test_invalid_metadata.results[1].stderr) + - >- + 'Invalid effective image mapping for base' in + (s2i_test_invalid_metadata.results[2].stdout + + s2i_test_invalid_metadata.results[2].stderr) + + - name: Read final fake container storage + ansible.builtin.slurp: + src: "{{ s2i_test_root }}/state.json" + register: s2i_test_state_data + + - name: Decode final fake container storage + ansible.builtin.set_fact: + s2i_test_state: >- + {{ s2i_test_state_data.content | b64decode | from_json }} + + - name: Validate exact cleanup and retained source image + ansible.builtin.assert: + that: + - s2i_test_images | intersect(s2i_test_state.podman) | length == 0 + - s2i_test_images | intersect(s2i_test_state.buildah) | length == 0 + - s2i_test_validation_image not in s2i_test_state.podman + - >- + 'registry.access.redhat.com/ubi10/ubi-minimal:latest' in + s2i_test_state.podman + - >- + ['podman', 'push', '--remove-signatures', + 'registry.test:5000/s2i-validation/ubi10-minimal:latest'] in + s2i_test_state.commands + - >- + ['podman', 'image', 'rm', '--force', s2i_test_images[0]] in + s2i_test_state.commands + - >- + ['buildah', 'rmi', s2i_test_images[0]] in s2i_test_state.commands + + - name: Check public registry state artifact + ansible.builtin.stat: + path: >- + {{ s2i_test_root }}/zuul-output/logs/container-build/registry-state.json + register: s2i_test_registry_state + + - name: Require registry state artifact + ansible.builtin.assert: + that: + - s2i_test_registry_state.stat.exists + - s2i_test_registry_state.stat.mode == '0644' diff --git a/playbooks/container-ci/shared/cleanup-images.yaml b/playbooks/container-ci/shared/cleanup-images.yaml new file mode 100644 index 0000000..2100cba --- /dev/null +++ b/playbooks/container-ci/shared/cleanup-images.yaml @@ -0,0 +1,208 @@ +--- +- name: Remove exact images recorded by the workflow + hosts: builder + gather_facts: false + vars: + s2i_ci_output_dir: "{{ ansible_user_dir }}/zuul-output" + s2i_ci_log_dir: "{{ s2i_ci_output_dir }}/logs/container-build" + s2i_ci_cleanup_images: true + + tasks: + - name: Check intended image cleanup metadata + when: s2i_ci_cleanup_images | bool + ansible.builtin.stat: + path: "{{ s2i_ci_log_dir }}/intended-images.json" + register: s2i_ci_intended_images_metadata + + - name: Load intended image cleanup metadata + when: + - s2i_ci_cleanup_images | bool + - s2i_ci_intended_images_metadata.stat.exists + ansible.builtin.slurp: + src: "{{ s2i_ci_log_dir }}/intended-images.json" + register: s2i_ci_intended_images_data + + - name: Decode intended image cleanup metadata + when: + - s2i_ci_cleanup_images | bool + - s2i_ci_intended_images_metadata.stat.exists + ansible.builtin.set_fact: + s2i_ci_cleanup_metadata: >- + {{ s2i_ci_intended_images_data.content | b64decode | from_json }} + + - name: Validate intended image cleanup metadata + when: + - s2i_ci_cleanup_images | bool + - s2i_ci_intended_images_metadata.stat.exists + ansible.builtin.assert: + that: + - s2i_ci_cleanup_metadata is mapping + - s2i_ci_cleanup_metadata.registry is string + - s2i_ci_cleanup_metadata.registry is match('^[A-Za-z0-9.:-]+$') + - s2i_ci_cleanup_metadata.namespace is string + - s2i_ci_cleanup_metadata.namespace is match('^[A-Za-z0-9_.-]+$') + - s2i_ci_cleanup_metadata.images is sequence + - s2i_ci_cleanup_metadata.images is not string + - s2i_ci_cleanup_metadata.images | unique | list | length == + s2i_ci_cleanup_metadata.images | length + - s2i_ci_cleanup_metadata.images | + select('match', '^' + s2i_ci_cleanup_metadata.registry | + regex_escape + '/' + s2i_ci_cleanup_metadata.namespace | + regex_escape + '/[A-Za-z0-9_.-]+:[A-Za-z0-9_.-]+$') | + list | length == s2i_ci_cleanup_metadata.images | length + fail_msg: Refusing unsafe or malformed intended image cleanup metadata + + - name: Record exact intended images for cleanup + ansible.builtin.set_fact: + s2i_ci_cleanup_images_exact: >- + {{ s2i_ci_cleanup_metadata.images | reverse | list }} + when: + - s2i_ci_cleanup_images | bool + - s2i_ci_intended_images_metadata.stat.exists + + - name: Default exact intended image list + ansible.builtin.set_fact: + s2i_ci_cleanup_images_exact: [] + when: + - not s2i_ci_cleanup_images | bool or + not s2i_ci_intended_images_metadata.stat.exists + + - name: Check Podman pullback tags + ansible.builtin.command: + argv: + - podman + - image + - exists + - "{{ item }}" + loop: "{{ s2i_ci_cleanup_images_exact }}" + register: s2i_ci_podman_image_checks + changed_when: false + failed_when: s2i_ci_podman_image_checks.rc not in [0, 1] + + - name: Remove exact Podman pullback tags + ansible.builtin.command: + argv: + - podman + - image + - rm + - --force + - "{{ item.item }}" + loop: "{{ s2i_ci_podman_image_checks.results }}" + when: item.rc == 0 + changed_when: true + + - name: Check Buildah-created image tags + ansible.builtin.command: + argv: + - buildah + - inspect + - --type + - image + - "{{ item }}" + loop: "{{ s2i_ci_cleanup_images_exact }}" + register: s2i_ci_buildah_image_checks + changed_when: false + failed_when: s2i_ci_buildah_image_checks.rc not in [0, 125] + + - name: Remove exact Buildah-created image tags + ansible.builtin.command: + argv: + - buildah + - rmi + - "{{ item.item }}" + loop: "{{ s2i_ci_buildah_image_checks.results }}" + when: item.rc == 0 + changed_when: true + + - name: List remaining Podman image references + ansible.builtin.command: + argv: + - podman + - images + - --format + - "{{ '{{.Repository}}:{{.Tag}}' }}" + register: s2i_ci_remaining_podman_images + changed_when: false + + - name: List remaining Buildah image references + ansible.builtin.command: + argv: + - buildah + - images + - --format + - "{{ '{{.Name}}:{{.Tag}}' }}" + register: s2i_ci_remaining_buildah_images + changed_when: false + + - name: Require exact workflow references to be absent + ansible.builtin.assert: + that: + - item not in s2i_ci_remaining_podman_images.stdout_lines + - item not in s2i_ci_remaining_buildah_images.stdout_lines + fail_msg: "Image cleanup left exact workflow reference {{ item }}" + loop: "{{ s2i_ci_cleanup_images_exact }}" + + - name: Check registry validation metadata + ansible.builtin.stat: + path: "{{ s2i_ci_log_dir }}/registry-state.json" + register: s2i_ci_registry_state_metadata + + - name: Load registry validation metadata + when: s2i_ci_registry_state_metadata.stat.exists + ansible.builtin.slurp: + src: "{{ s2i_ci_log_dir }}/registry-state.json" + register: s2i_ci_registry_state_data + + - name: Decode registry validation metadata + when: s2i_ci_registry_state_metadata.stat.exists + ansible.builtin.set_fact: + s2i_ci_registry_state: >- + {{ s2i_ci_registry_state_data.content | b64decode | from_json }} + + - name: Validate registry validation metadata + when: s2i_ci_registry_state_metadata.stat.exists + ansible.builtin.assert: + that: + - s2i_ci_registry_state is mapping + - s2i_ci_registry_state.endpoint is string + - s2i_ci_registry_state.endpoint is match('^[A-Za-z0-9.:-]+$') + - s2i_ci_registry_state.validation_image is string + - s2i_ci_registry_state.validation_image.startswith( + s2i_ci_registry_state.endpoint + '/') + + - name: Check the Podman registry validation tag + when: s2i_ci_registry_state_metadata.stat.exists + ansible.builtin.command: + argv: + - podman + - image + - exists + - "{{ s2i_ci_registry_state.validation_image }}" + register: s2i_ci_validation_image_check + changed_when: false + failed_when: s2i_ci_validation_image_check.rc not in [0, 1] + + - name: Remove the exact Podman registry validation tag + when: + - s2i_ci_registry_state_metadata.stat.exists + - s2i_ci_validation_image_check.rc == 0 + ansible.builtin.command: + argv: + - podman + - image + - rm + - --force + - "{{ s2i_ci_registry_state.validation_image }}" + changed_when: true + + - name: Require registry validation tag to be absent + when: s2i_ci_registry_state_metadata.stat.exists + ansible.builtin.command: + argv: + - podman + - image + - exists + - "{{ s2i_ci_registry_state.validation_image }}" + register: s2i_ci_validation_image_absent + changed_when: false + failed_when: s2i_ci_validation_image_absent.rc != 1 diff --git a/playbooks/container-ci/shared/deployment-map-item.yaml b/playbooks/container-ci/shared/deployment-map-item.yaml new file mode 100644 index 0000000..c38a4e0 --- /dev/null +++ b/playbooks/container-ci/shared/deployment-map-item.yaml @@ -0,0 +1,7 @@ +--- +- name: Expand image reference for each deployment key + ansible.builtin.set_fact: + s2i_ci_custom_container_images: >- + {{ s2i_ci_custom_container_images | + combine({item: s2i_ci_target_references[s2i_ci_image]}) }} + loop: "{{ s2i_ci_effective_image_mappings[s2i_ci_image] }}" diff --git a/playbooks/container-ci/shared/image-metadata-item.yaml b/playbooks/container-ci/shared/image-metadata-item.yaml new file mode 100644 index 0000000..1e3f97e --- /dev/null +++ b/playbooks/container-ci/shared/image-metadata-item.yaml @@ -0,0 +1,68 @@ +--- +- name: Check selected image files + ansible.builtin.stat: + path: "{{ item }}" + loop: + - "{{ s2i_ci_container_repo }}/containers/{{ s2i_ci_image }}/Containerfile" + - "{{ s2i_ci_container_repo }}/containers/{{ s2i_ci_image }}/image.yaml" + register: s2i_ci_image_files + +- name: Require selected image files + ansible.builtin.assert: + that: + - s2i_ci_image_files.results | map(attribute='stat.exists') | + select('equalto', false) | list | length == 0 + - s2i_ci_image_files.results | map(attribute='stat.isreg') | + select('equalto', false) | list | length == 0 + fail_msg: >- + Selected image {{ s2i_ci_image }} requires Containerfile and image.yaml + +- name: Read tracked image metadata + ansible.builtin.slurp: + src: "{{ s2i_ci_container_repo }}/containers/{{ s2i_ci_image }}/image.yaml" + register: s2i_ci_image_metadata_data + +- name: Decode tracked image metadata + ansible.builtin.set_fact: + s2i_ci_image_metadata: >- + {{ s2i_ci_image_metadata_data.content | b64decode | from_yaml }} + +- name: Validate tracked image metadata schema + ansible.builtin.assert: + that: + - s2i_ci_image_metadata is mapping + - s2i_ci_image_metadata.openstack_version is mapping + - s2i_ci_image_metadata.openstack_version.custom_container_images is sequence + - s2i_ci_image_metadata.openstack_version.custom_container_images is not string + fail_msg: "Malformed image metadata for {{ s2i_ci_image }}" + +- name: Select effective image deployment keys + ansible.builtin.set_fact: + s2i_ci_effective_image_keys: >- + {{ s2i_ci_image_mappings[s2i_ci_image] + if s2i_ci_image in s2i_ci_image_mappings + else s2i_ci_image_metadata.openstack_version.custom_container_images }} + s2i_ci_effective_mapping_source: >- + {{ 'inventory' if s2i_ci_image in s2i_ci_image_mappings else 'tracked' }} + +- name: Validate effective image deployment keys + ansible.builtin.assert: + that: + - s2i_ci_effective_image_keys is sequence + - s2i_ci_effective_image_keys is not string + - s2i_ci_effective_image_keys | select('string') | list | length == + s2i_ci_effective_image_keys | length + - s2i_ci_effective_image_keys | + reject('match', '^[A-Za-z][A-Za-z0-9]*$') | list | length == 0 + - s2i_ci_effective_image_keys | unique | list | length == + s2i_ci_effective_image_keys | length + fail_msg: "Invalid effective image mapping for {{ s2i_ci_image }}" + +- name: Record effective image metadata + ansible.builtin.set_fact: + s2i_ci_effective_image_mappings: >- + {{ s2i_ci_effective_image_mappings | + combine({s2i_ci_image: s2i_ci_effective_image_keys | list}) }} + s2i_ci_mapping_sources: >- + {{ s2i_ci_mapping_sources | + combine({s2i_ci_image: s2i_ci_effective_mapping_source}) }} diff --git a/playbooks/container-ci/shared/prepare-host.yaml b/playbooks/container-ci/shared/prepare-host.yaml new file mode 100644 index 0000000..64104d4 --- /dev/null +++ b/playbooks/container-ci/shared/prepare-host.yaml @@ -0,0 +1,46 @@ +--- +- name: Prepare the shared container image build tools + hosts: builder + vars: + s2i_ci_output_dir: "{{ ansible_user_dir }}/zuul-output" + s2i_ci_log_dir: "{{ s2i_ci_output_dir }}/logs/container-build" + + tasks: + - name: Ensure the container build log directory exists + ansible.builtin.file: + path: "{{ s2i_ci_log_dir }}" + state: directory + mode: "0755" + + - name: Update the apt package cache + when: + - s2i_ci_install_host_packages | default(true) | bool + - ansible_facts.os_family == "Debian" + become: true + ansible.builtin.apt: + update_cache: true + cache_valid_time: 3600 + + - name: Install container build host packages + when: s2i_ci_install_host_packages | default(true) | bool + become: true + ansible.builtin.package: + name: + - buildah + - ca-certificates + - git + - openssl + state: present + + - name: Ensure Podman is installed and usable + when: s2i_ci_install_host_packages | default(true) | bool + ansible.builtin.include_role: + name: ensure-podman + vars: + ensure_podman_rootless: false + ensure_podman_validate: true + + - name: Validate Buildah + when: s2i_ci_install_host_packages | default(true) | bool + ansible.builtin.command: buildah version + changed_when: false diff --git a/playbooks/container-ci/shared/published-image-item.yaml b/playbooks/container-ci/shared/published-image-item.yaml new file mode 100644 index 0000000..1b30ee8 --- /dev/null +++ b/playbooks/container-ci/shared/published-image-item.yaml @@ -0,0 +1,10 @@ +--- +- name: Record exact reference for selected image + ansible.builtin.set_fact: + s2i_ci_target_references: >- + {{ s2i_ci_target_references | + combine({s2i_ci_image: s2i_ci_published_images[s2i_ci_image_index]}) }} + s2i_ci_local_target_references: >- + {{ s2i_ci_local_target_references | + combine({s2i_ci_image: + s2i_ci_local_published_images[s2i_ci_image_index]}) }} diff --git a/playbooks/container-ci/shared/run.yaml b/playbooks/container-ci/shared/run.yaml new file mode 100644 index 0000000..2d44b80 --- /dev/null +++ b/playbooks/container-ci/shared/run.yaml @@ -0,0 +1,340 @@ +--- +- name: Build and publish selected container images + hosts: builder + vars: + s2i_ci_output_dir: "{{ ansible_user_dir }}/zuul-output" + s2i_ci_log_dir: "{{ s2i_ci_output_dir }}/logs/container-build" + s2i_ci_registry_connection_file: >- + {{ s2i_ci_output_dir }}/.s2i-ci-registry-connection.json + s2i_ci_workspace_root: "{{ zuul_user_dir | default(ansible_user_dir) }}" + + tasks: + - name: Default optional selective provider inputs + ansible.builtin.set_fact: + s2i_ci_namespace: "{{ s2i_ci_namespace | default('openstack') }}" + s2i_ci_tag: >- + {{ s2i_ci_tag | default( + (zuul.change | default('build') | string) ~ '-' ~ + (zuul.patchset | default('latest') | string) + ) }} + s2i_ci_stream: "{{ s2i_ci_stream | default('master') }}" + s2i_ci_parallel: >- + {{ s2i_ci_parallel | + default(ansible_facts.processor_vcpus | default(1)) }} + s2i_ci_images: "{{ s2i_ci_images | default([]) }}" + s2i_ci_image_mappings: "{{ s2i_ci_image_mappings | default({}) }}" + + - name: Validate the selected container project + ansible.builtin.assert: + that: + - s2i_ci_container_project in zuul.projects + fail_msg: The selected container project is not in the Zuul workspace + + - name: Resolve the inventory-described container repository + ansible.builtin.set_fact: + s2i_ci_container_repo: >- + {{ s2i_ci_workspace_root }}/{{ + zuul.projects[s2i_ci_container_project].src_dir }} + + - name: Inspect the container build entry point on the builder + ansible.builtin.stat: + path: "{{ s2i_ci_container_repo }}/build.sh" + follow: true + register: s2i_ci_build_entry_point + + - name: Canonicalize the builder workspace root + ansible.builtin.command: + argv: [realpath, --canonicalize-existing, "{{ s2i_ci_workspace_root }}"] + register: s2i_ci_workspace_root_realpath + changed_when: false + failed_when: false + + - name: Canonicalize the container repository on the builder + ansible.builtin.command: + argv: [realpath, --canonicalize-existing, "{{ s2i_ci_container_repo }}"] + register: s2i_ci_container_repo_realpath + changed_when: false + failed_when: false + + - name: Validate selective provider inputs + ansible.builtin.assert: + that: + - s2i_ci_build_entry_point.stat.isreg | default(false) + - s2i_ci_workspace_root_realpath.rc == 0 + - s2i_ci_container_repo_realpath.rc == 0 + - s2i_ci_container_repo_realpath.stdout.startswith( + s2i_ci_workspace_root_realpath.stdout + '/') + - s2i_ci_images is sequence + - s2i_ci_images is not string + - s2i_ci_images | length > 0 + - s2i_ci_images | select('string') | list | length == + s2i_ci_images | length + - s2i_ci_images | reject('match', '^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$') | + list | length == 0 + - s2i_ci_images | unique | list | length == s2i_ci_images | length + - s2i_ci_image_mappings is mapping + - s2i_ci_tag is string + - "',' not in s2i_ci_tag" + fail_msg: Selective provider inputs are invalid + + - name: Record normalized selective targets + ansible.builtin.set_fact: + s2i_ci_selected_images: "{{ (['base'] + s2i_ci_images) | unique | list }}" + s2i_ci_target_expression: "{{ s2i_ci_images | join(',') }}" + s2i_ci_effective_image_mappings: {} + s2i_ci_mapping_sources: {} + + - name: Validate optional mapping overrides + ansible.builtin.include_tasks: validate-mapping-overrides.yaml + + - name: Load and validate selected image metadata + ansible.builtin.include_tasks: image-metadata-item.yaml + loop: "{{ s2i_ci_selected_images }}" + loop_control: + loop_var: s2i_ci_image + label: "{{ s2i_ci_image }}" + + - name: Validate effective deployment key ownership + ansible.builtin.include_tasks: validate-deployment-keys.yaml + + - name: Load the normalized registry connection + ansible.builtin.slurp: + src: "{{ s2i_ci_registry_connection_file }}" + register: s2i_ci_registry_connection_data + no_log: true + + - name: Decode the normalized registry connection + ansible.builtin.set_fact: + s2i_ci_registry: >- + {{ s2i_ci_registry_connection_data.content | b64decode | from_json }} + no_log: true + + - name: Validate normalized registry connection + ansible.builtin.assert: + that: + - s2i_ci_registry is mapping + - s2i_ci_registry.endpoint is string + - s2i_ci_registry.endpoint | length > 0 + - s2i_ci_registry.public_host is string + - s2i_ci_registry.public_host | length > 0 + - s2i_ci_registry.public_port | int > 0 + - s2i_ci_registry.auth_file | default('') is string + - s2i_ci_registry.cert_dir | default('') is string + + - name: Resolve intended exact image references before mutation + ansible.builtin.command: + argv: + - "{{ s2i_ci_container_repo }}/build.sh" + - refs + - "{{ s2i_ci_target_expression }}" + chdir: "{{ s2i_ci_container_repo }}" + environment: + STREAM: "{{ s2i_ci_stream }}" + REGISTRY: "{{ s2i_ci_registry.endpoint }}" + NAMESPACE: "{{ s2i_ci_namespace }}" + TAG: "{{ s2i_ci_tag }}" + register: s2i_ci_intended_refs + changed_when: false + + - name: Record intended exact image references + ansible.builtin.set_fact: + s2i_ci_intended_images: >- + {{ s2i_ci_intended_refs.stdout_lines | reject('equalto', '') | list }} + + - name: Validate intended exact image references + ansible.builtin.assert: + that: + - s2i_ci_intended_images | length == s2i_ci_selected_images | length + - s2i_ci_intended_images | unique | list | length == + s2i_ci_intended_images | length + - s2i_ci_intended_images | + select('match', '^' + s2i_ci_registry.endpoint | regex_escape + + '/' + s2i_ci_namespace + '/[A-Za-z0-9_.-]+:[A-Za-z0-9_.-]+$') | + list | length == s2i_ci_intended_images | length + fail_msg: Intended image references are incomplete or unsafe + + - name: Persist intended references for partial-failure cleanup + ansible.builtin.copy: + dest: "{{ s2i_ci_log_dir }}/intended-images.json" + mode: "0644" + content: | + {{ { + 'registry': s2i_ci_registry.endpoint, + 'namespace': s2i_ci_namespace, + 'images': s2i_ci_intended_images, + 'selected_images': s2i_ci_selected_images, + 'target': s2i_ci_target_expression + } | to_nice_json }} + + - name: Build selected images from committed source pins + ansible.builtin.command: + argv: + - "{{ s2i_ci_container_repo }}/build.sh" + - build-parallel + - "{{ s2i_ci_target_expression }}" + chdir: "{{ s2i_ci_container_repo }}" + environment: + STREAM: "{{ s2i_ci_stream }}" + REGISTRY: "{{ s2i_ci_registry.endpoint }}" + NAMESPACE: "{{ s2i_ci_namespace }}" + TAG: "{{ s2i_ci_tag }}" + PARALLEL: "{{ s2i_ci_parallel | string }}" + BUILD_LOGS_DIR: "{{ s2i_ci_log_dir }}" + register: s2i_ci_build + changed_when: s2i_ci_build.rc == 0 + + - name: Save build command output + ansible.builtin.copy: + dest: "{{ s2i_ci_log_dir }}/build.log" + mode: "0644" + content: | + {{ s2i_ci_build.stdout }} + {{ s2i_ci_build.stderr }} + + - name: Push selected images to the buildset registry + ansible.builtin.command: + argv: + - "{{ s2i_ci_container_repo }}/build.sh" + - push + - "{{ s2i_ci_target_expression }}" + chdir: "{{ s2i_ci_container_repo }}" + environment: + STREAM: "{{ s2i_ci_stream }}" + REGISTRY: "{{ s2i_ci_registry.endpoint }}" + NAMESPACE: "{{ s2i_ci_namespace }}" + TAG: "{{ s2i_ci_tag }}" + REGISTRY_AUTH_FILE: "{{ s2i_ci_registry.auth_file | default('') }}" + REGISTRY_CERT_DIR: "{{ s2i_ci_registry.cert_dir | default('') }}" + register: s2i_ci_push + changed_when: s2i_ci_push.rc == 0 + + - name: Save push command output + ansible.builtin.copy: + dest: "{{ s2i_ci_log_dir }}/push.log" + mode: "0644" + content: | + {{ s2i_ci_push.stdout }} + {{ s2i_ci_push.stderr }} + + - name: Resolve successfully published image references + ansible.builtin.command: + argv: + - "{{ s2i_ci_container_repo }}/build.sh" + - refs + - "{{ s2i_ci_target_expression }}" + chdir: "{{ s2i_ci_container_repo }}" + environment: + STREAM: "{{ s2i_ci_stream }}" + REGISTRY: "{{ s2i_ci_registry.endpoint }}" + NAMESPACE: "{{ s2i_ci_namespace }}" + TAG: "{{ s2i_ci_tag }}" + register: s2i_ci_refs + changed_when: false + + - name: Record exact builder-local published image references + ansible.builtin.set_fact: + s2i_ci_local_published_images: >- + {{ s2i_ci_refs.stdout_lines | reject('equalto', '') | list }} + + - name: Require intended and published references to match + ansible.builtin.assert: + that: + - s2i_ci_local_published_images == s2i_ci_intended_images + fail_msg: Published references differ from the pre-recorded intent + + - name: Record the public buildset registry endpoint + ansible.builtin.set_fact: + s2i_ci_public_registry_endpoint: >- + {{ s2i_ci_registry.public_host }}:{{ s2i_ci_registry.public_port }} + + - name: Convert published references to the public buildset endpoint + ansible.builtin.set_fact: + s2i_ci_published_images: >- + {{ s2i_ci_local_published_images | + map('regex_replace', + '^' + (s2i_ci_registry.endpoint | regex_escape), + s2i_ci_public_registry_endpoint) | list }} + + - name: Validate public exact published references + ansible.builtin.assert: + that: + - s2i_ci_published_images | length == + s2i_ci_local_published_images | length + - s2i_ci_published_images | + select('match', '^' + + (s2i_ci_public_registry_endpoint | regex_escape) + '/') | + list | length == s2i_ci_published_images | length + - s2i_ci_published_images | + select('search', s2i_ci_registry.endpoint | regex_escape) | + list | length == 0 + fail_msg: Public references must not expose the builder-local alias + + - name: Pull published images back from the registry + ansible.builtin.command: + argv: >- + {{ ['podman', 'pull'] + + (['--authfile', s2i_ci_registry.auth_file] + if s2i_ci_registry.auth_file | default('') | length > 0 else []) + + (['--cert-dir', s2i_ci_registry.cert_dir] + if s2i_ci_registry.cert_dir | default('') | length > 0 else []) + + [item] }} + loop: "{{ s2i_ci_local_published_images }}" + changed_when: true + + - name: Initialize exact reference expansion + ansible.builtin.set_fact: + s2i_ci_target_references: {} + s2i_ci_local_target_references: {} + s2i_ci_custom_container_images: {} + + - name: Map selected targets to exact references + ansible.builtin.include_tasks: published-image-item.yaml + loop: "{{ s2i_ci_selected_images }}" + loop_control: + loop_var: s2i_ci_image + index_var: s2i_ci_image_index + label: "{{ s2i_ci_image }}" + + - name: Expand exact references into deployment keys + ansible.builtin.include_tasks: deployment-map-item.yaml + loop: "{{ s2i_ci_selected_images }}" + loop_control: + loop_var: s2i_ci_image + label: "{{ s2i_ci_image }}" + + - name: Validate consolidated Watcher process entry points + ansible.builtin.command: + argv: + - podman + - run + - --rm + - --user + - watcher + - --entrypoint + - /bin/sh + - "{{ s2i_ci_local_target_references['watcher/watcher-base'] }}" + - -ec + - >- + for command in watcher-api watcher-applier watcher-decision-engine; + do command -v "$command"; "$command" --help >/dev/null; done + when: "'watcher/watcher-base' in s2i_ci_selected_images" + changed_when: false + + - name: Save published image metadata + ansible.builtin.copy: + dest: "{{ s2i_ci_log_dir }}/published-images.json" + mode: "0644" + content: | + {{ { + 'registry': s2i_ci_public_registry_endpoint, + 'namespace': s2i_ci_namespace, + 'tag': s2i_ci_tag, + 'target': s2i_ci_target_expression, + 'stream': s2i_ci_stream, + 'selected_images': s2i_ci_selected_images, + 'images': s2i_ci_published_images, + 'image_mappings': s2i_ci_effective_image_mappings, + 'mapping_sources': s2i_ci_mapping_sources, + 'custom_container_images': + s2i_ci_custom_container_images + } | to_nice_json }} diff --git a/playbooks/container-ci/shared/validate-deployment-keys.yaml b/playbooks/container-ci/shared/validate-deployment-keys.yaml new file mode 100644 index 0000000..f975070 --- /dev/null +++ b/playbooks/container-ci/shared/validate-deployment-keys.yaml @@ -0,0 +1,12 @@ +--- +- name: Record all effective deployment keys + ansible.builtin.set_fact: + s2i_ci_all_deployment_keys: >- + {{ s2i_ci_effective_image_mappings.values() | flatten | list }} + +- name: Reject duplicate deployment keys + ansible.builtin.assert: + that: + - s2i_ci_all_deployment_keys | unique | list | length == + s2i_ci_all_deployment_keys | length + fail_msg: A deployment key may be assigned to only one image diff --git a/playbooks/container-ci/shared/validate-mapping-overrides.yaml b/playbooks/container-ci/shared/validate-mapping-overrides.yaml new file mode 100644 index 0000000..081c990 --- /dev/null +++ b/playbooks/container-ci/shared/validate-mapping-overrides.yaml @@ -0,0 +1,8 @@ +--- +- name: Reject mapping overrides for unselected images + ansible.builtin.assert: + that: + - s2i_ci_image_mappings is mapping + - s2i_ci_image_mappings.keys() | + difference(s2i_ci_selected_images) | length == 0 + fail_msg: Image mapping overrides must name selected image targets diff --git a/playbooks/container-ci/shared/validate-registry.yaml b/playbooks/container-ci/shared/validate-registry.yaml new file mode 100644 index 0000000..cad25f0 --- /dev/null +++ b/playbooks/container-ci/shared/validate-registry.yaml @@ -0,0 +1,109 @@ +--- +- name: Validate the prepared image registry connection + hosts: builder + vars: + s2i_ci_output_dir: "{{ ansible_user_dir }}/zuul-output" + s2i_ci_log_dir: "{{ s2i_ci_output_dir }}/logs/container-build" + s2i_ci_registry_connection_file: >- + {{ s2i_ci_output_dir }}/.s2i-ci-registry-connection.json + s2i_ci_validation_source_image: registry.access.redhat.com/ubi10/ubi-minimal:latest + s2i_ci_validation_repository: s2i-validation/ubi10-minimal:latest + + tasks: + - name: Load the normalized registry connection + ansible.builtin.slurp: + src: "{{ s2i_ci_registry_connection_file }}" + register: s2i_ci_registry_connection_data + + - name: Decode the normalized registry connection + ansible.builtin.set_fact: + s2i_ci_registry_connection: >- + {{ s2i_ci_registry_connection_data.content | b64decode | from_json }} + + - name: Require normalized registry connection data + ansible.builtin.assert: + that: + - s2i_ci_registry_connection is mapping + - s2i_ci_registry_connection.endpoint is string + - s2i_ci_registry_connection.endpoint | length > 0 + - s2i_ci_registry_connection.public_host is string + - s2i_ci_registry_connection.public_host | length > 0 + - s2i_ci_registry_connection.public_port | int > 0 + - s2i_ci_registry_connection.auth_file | default('') is string + - s2i_ci_registry_connection.cert_dir | default('') is string + + - name: Set normalized registry command parameters + ansible.builtin.set_fact: + s2i_ci_registry_endpoint: "{{ s2i_ci_registry_connection.endpoint }}" + s2i_ci_registry_auth_file: >- + {{ s2i_ci_registry_connection.auth_file | default('') }} + s2i_ci_registry_cert_dir: >- + {{ s2i_ci_registry_connection.cert_dir | default('') }} + + - name: Pull the UBI 10 base image + ansible.builtin.command: + argv: + - podman + - pull + - "{{ s2i_ci_validation_source_image }}" + register: s2i_ci_ubi_pull + changed_when: "'Copying blob' in s2i_ci_ubi_pull.stdout" + + - name: Tag the UBI 10 image for registry validation + ansible.builtin.command: + argv: + - podman + - tag + - "{{ s2i_ci_validation_source_image }}" + - "{{ s2i_ci_registry_endpoint }}/{{ s2i_ci_validation_repository }}" + changed_when: true + + - name: Push the UBI 10 validation image + ansible.builtin.command: + argv: >- + {{ ['podman', 'push', '--remove-signatures'] + + (['--authfile', s2i_ci_registry_auth_file] + if s2i_ci_registry_auth_file | length > 0 else []) + + (['--cert-dir', s2i_ci_registry_cert_dir] + if s2i_ci_registry_cert_dir | length > 0 else []) + + [s2i_ci_registry_endpoint + '/' + s2i_ci_validation_repository] }} + changed_when: true + + - name: Remove the validation tag before pulling it back + ansible.builtin.command: + argv: + - podman + - untag + - "{{ s2i_ci_registry_endpoint }}/{{ s2i_ci_validation_repository }}" + changed_when: true + + - name: Pull the UBI 10 validation image back from the registry + ansible.builtin.command: + argv: >- + {{ ['podman', 'pull'] + + (['--authfile', s2i_ci_registry_auth_file] + if s2i_ci_registry_auth_file | length > 0 else []) + + (['--cert-dir', s2i_ci_registry_cert_dir] + if s2i_ci_registry_cert_dir | length > 0 else []) + + [s2i_ci_registry_endpoint + '/' + s2i_ci_validation_repository] }} + changed_when: false + + - name: Inspect the UBI 10 validation image from the registry + ansible.builtin.command: + argv: + - podman + - image + - inspect + - "{{ s2i_ci_registry_endpoint }}/{{ s2i_ci_validation_repository }}" + changed_when: false + + - name: Record non-secret registry state + ansible.builtin.copy: + dest: "{{ s2i_ci_log_dir }}/registry-state.json" + mode: "0644" + content: | + {{ { + 'endpoint': s2i_ci_registry_endpoint, + 'validation_source': s2i_ci_validation_source_image, + 'validation_image': s2i_ci_registry_endpoint + '/' + s2i_ci_validation_repository + } | to_nice_json }} diff --git a/playbooks/container-ci/zuul/content-provider-return.yaml b/playbooks/container-ci/zuul/content-provider-return.yaml new file mode 100644 index 0000000..f4e1abe --- /dev/null +++ b/playbooks/container-ci/zuul/content-provider-return.yaml @@ -0,0 +1,32 @@ +--- +- name: Return selective image metadata and pause the provider + zuul_return: + data: + zuul: + pause: true + s2i_ci_content: + registry: "{{ s2i_ci_public_registry_endpoint }}" + registry_host: "{{ s2i_ci_registry.public_host }}" + registry_port: "{{ s2i_ci_registry.public_port }}" + namespace: "{{ s2i_ci_namespace }}" + tag: "{{ s2i_ci_tag }}" + target: "{{ s2i_ci_target_expression }}" + stream: "{{ s2i_ci_stream }}" + selected_images: "{{ s2i_ci_selected_images }}" + images: "{{ s2i_ci_published_images }}" + image_mappings: "{{ s2i_ci_effective_image_mappings }}" + mapping_sources: "{{ s2i_ci_mapping_sources }}" + custom_container_images: "{{ s2i_ci_custom_container_images }}" + content_provider_os_custom_container_images: >- + {{ s2i_ci_custom_container_images }} + content_provider_os_registry_url: "null" + content_provider_os_registry_namespace: "" + content_provider_os_registry_tag: "" + content_provider_dlrn_md5_hash: "" + content_provider_gating_repo_available: false + content_provider_gating_repo_url: "" + content_provider_registry_available: true + content_provider_registry_ip: "{{ s2i_ci_registry.public_host }}" + content_provider_registry_ip_port: >- + {{ s2i_ci_public_registry_endpoint }} + cifmw_build_images_output: {} diff --git a/playbooks/container-ci/zuul/post.yaml b/playbooks/container-ci/zuul/post.yaml new file mode 100644 index 0000000..6b63478 --- /dev/null +++ b/playbooks/container-ci/zuul/post.yaml @@ -0,0 +1,129 @@ +--- +- name: Collect Zuul container logs + hosts: builder + gather_facts: false + vars: + s2i_ci_output_dir: "{{ ansible_user_dir }}/zuul-output" + s2i_ci_log_dir: "{{ s2i_ci_output_dir }}/logs/container-build" + + tasks: + - name: Ensure the container build log directory exists + ansible.builtin.file: + path: "{{ s2i_ci_log_dir }}" + state: directory + mode: "0755" + failed_when: false + + - name: Collect buildset registry logs + block: + - name: Run the trusted container log collection role + ansible.builtin.include_role: + name: collect-container-logs + vars: + container_command: podman + rescue: + - name: Report non-fatal Zuul container log collection failure + ansible.builtin.debug: + msg: Container log collection failed; registry cleanup will continue + +- name: Remove exact workflow images + ansible.builtin.import_playbook: ../shared/cleanup-images.yaml + +- name: Remove the project-owned Zuul buildset registry + hosts: builder + gather_facts: false + vars: + s2i_ci_output_dir: "{{ ansible_user_dir }}/zuul-output" + s2i_ci_registry_root: "{{ ansible_user_dir }}/buildset_registry" + s2i_ci_registry_connection_file: >- + {{ s2i_ci_output_dir }}/.s2i-ci-registry-connection.json + s2i_ci_registry_port: 5000 + + tasks: + - name: Check buildset registry ownership + ansible.builtin.stat: + path: "{{ s2i_ci_registry_root }}/.s2i-ci-registry-owner.json" + register: s2i_ci_registry_owner_marker + + - name: Default buildset registry ownership to unloaded + ansible.builtin.set_fact: + s2i_ci_registry_owner_loaded: false + s2i_ci_registry_owner_is_valid: false + + - name: Load buildset registry ownership + when: s2i_ci_registry_owner_marker.stat.exists | default(false) + block: + - name: Read the buildset registry ownership marker + ansible.builtin.slurp: + src: "{{ s2i_ci_registry_root }}/.s2i-ci-registry-owner.json" + register: s2i_ci_registry_owner_data + + - name: Decode the buildset registry ownership marker + ansible.builtin.set_fact: + s2i_ci_registry_owner: >- + {{ s2i_ci_registry_owner_data.content | b64decode | from_json }} + + - name: Record successful ownership loading + ansible.builtin.set_fact: + s2i_ci_registry_owner_loaded: true + rescue: + - name: Refuse unreadable registry ownership metadata + ansible.builtin.fail: + msg: Refusing to clean a registry whose ownership marker is invalid + + - name: Validate buildset registry ownership + when: s2i_ci_registry_owner_loaded | bool + ansible.builtin.assert: + that: + - s2i_ci_registry_owner.port | default(-1) | int == + s2i_ci_registry_port | int + - s2i_ci_registry_owner.container | default('') == + ('buildset_registry' if s2i_ci_registry_port | int == 5000 + else 'buildset_registry_' + s2i_ci_registry_port | string) + fail_msg: Refusing to clean a registry with an invalid ownership marker + + - name: Record valid buildset registry ownership + when: s2i_ci_registry_owner_loaded | bool + ansible.builtin.set_fact: + s2i_ci_registry_owner_is_valid: true + + - name: Remove the project-owned buildset registry container + when: + - s2i_ci_registry_owner_loaded | bool + - s2i_ci_registry_owner_is_valid | bool + ansible.builtin.command: + argv: + - podman + - rm + - --force + - "{{ s2i_ci_registry_owner.container }}" + register: s2i_ci_removed_buildset_registry + changed_when: s2i_ci_removed_buildset_registry.stdout | length > 0 + + - name: Stop the project-owned buildset registry tunnel + when: + - s2i_ci_registry_owner_loaded | bool + - s2i_ci_registry_owner_is_valid | bool + ansible.builtin.shell: | + pids=$(pgrep -f '^socat -d -d TCP6-LISTEN:{{ s2i_ci_registry_owner.port }},fork TCP:127.0.0.1:1{{ s2i_ci_registry_owner.port }}$' || true) + if [ -n "$pids" ]; then + printf '%s\n' "$pids" + kill $pids + fi + args: + executable: /bin/bash + register: s2i_ci_stopped_registry_tunnel + changed_when: s2i_ci_stopped_registry_tunnel.stdout | length > 0 + + - name: Remove project-owned buildset registry state + when: + - s2i_ci_registry_owner_loaded | bool + - s2i_ci_registry_owner_is_valid | bool + ansible.builtin.file: + path: "{{ s2i_ci_registry_root }}" + state: absent + + - name: Remove persisted buildset registry connection data + ansible.builtin.file: + path: "{{ s2i_ci_registry_connection_file }}" + state: absent diff --git a/playbooks/container-ci/zuul/pre.yaml b/playbooks/container-ci/zuul/pre.yaml new file mode 100644 index 0000000..49f52ba --- /dev/null +++ b/playbooks/container-ci/zuul/pre.yaml @@ -0,0 +1,127 @@ +--- +- name: Prepare shared build tools + ansible.builtin.import_playbook: ../shared/prepare-host.yaml + +- name: Prepare the Zuul buildset registry + hosts: builder + vars: + s2i_ci_output_dir: "{{ ansible_user_dir }}/zuul-output" + s2i_ci_registry_root: "{{ ansible_user_dir }}/buildset_registry" + s2i_ci_registry_connection_file: >- + {{ s2i_ci_output_dir }}/.s2i-ci-registry-connection.json + s2i_ci_registry_port: 5000 + s2i_ci_registry_image: quay.io/zuul-ci/zuul-registry@sha256:0bc02a2eed546daa570ebb18829579057cefceefd9ee15660dcf5a1a752d8f78 + + tasks: + - name: Record whether this job owns the buildset registry + ansible.builtin.set_fact: + s2i_ci_owns_buildset_registry: "{{ buildset_registry is not defined }}" + + - name: Discard a stale ownership marker for an inherited registry + when: not s2i_ci_owns_buildset_registry | bool + ansible.builtin.file: + path: "{{ s2i_ci_registry_root }}/.s2i-ci-registry-owner.json" + state: absent + + - name: Remove a stale project-owned buildset registry container + when: s2i_ci_owns_buildset_registry | bool + ansible.builtin.command: + argv: [podman, rm, --force, buildset_registry] + register: s2i_ci_stale_buildset_registry + changed_when: s2i_ci_stale_buildset_registry.stdout | length > 0 + failed_when: s2i_ci_stale_buildset_registry.rc not in [0, 1] + + - name: Start the Zuul buildset registry + when: s2i_ci_owns_buildset_registry | bool + ansible.builtin.include_role: + name: run-buildset-registry + vars: + buildset_registry_root: "{{ s2i_ci_registry_root }}" + buildset_registry_port: "{{ s2i_ci_registry_port }}" + buildset_registry_image: "{{ s2i_ci_registry_image }}" + container_command: podman + + - name: Record ownership of the started buildset registry + when: s2i_ci_owns_buildset_registry | bool + ansible.builtin.copy: + dest: "{{ s2i_ci_registry_root }}/.s2i-ci-registry-owner.json" + mode: "0600" + content: | + {{ { + 'container': ('buildset_registry' if s2i_ci_registry_port | int == 5000 else 'buildset_registry_' + s2i_ci_registry_port | string), + 'port': s2i_ci_registry_port + } | to_nice_json }} + + - name: Remove the unlabeled buildset registry container + when: + - s2i_ci_owns_buildset_registry | bool + - ansible_selinux.status | default("disabled") == "enabled" + ansible.builtin.command: + argv: + - podman + - rm + - --force + - "{{ 'buildset_registry' if s2i_ci_registry_port | int == 5000 else 'buildset_registry_' + s2i_ci_registry_port | string }}" + changed_when: true + + - name: Start the buildset registry with SELinux-labeled volumes + when: + - s2i_ci_owns_buildset_registry | bool + - ansible_selinux.status | default("disabled") == "enabled" + ansible.builtin.command: + argv: + - podman + - run + - --detach + - "--name={{ 'buildset_registry' if s2i_ci_registry_port | int == 5000 else 'buildset_registry_' + s2i_ci_registry_port | string }}" + - --restart=always + - "--publish=1{{ s2i_ci_registry_port }}:5000" + - "--volume={{ s2i_ci_registry_root }}/tls:/tls:Z" + - "--volume={{ s2i_ci_registry_root }}/conf:/conf:Z" + - "{{ s2i_ci_registry_image }}" + - zuul-registry + - -d + changed_when: true + + - name: Wait for the buildset registry authentication endpoint + when: s2i_ci_owns_buildset_registry | bool + ansible.builtin.uri: + url: "https://127.0.0.1:{{ s2i_ci_registry_port }}/v2/" + validate_certs: false + status_code: 401 + register: s2i_ci_buildset_registry_api + retries: 30 + delay: 1 + until: s2i_ci_buildset_registry_api.status == 401 + + - name: Configure the host to use the Zuul buildset registry + ansible.builtin.include_role: + name: use-buildset-registry + vars: + buildset_registry_namespaces: + - [docker.io, https://registry-1.docker.io] + - [quay.io, https://quay.io] + - [gcr.io, https://gcr.io] + - [registry.k8s.io, https://registry.k8s.io] + - [registry.access.redhat.com, https://registry.access.redhat.com] + + - name: Set the Zuul buildset registry endpoint + ansible.builtin.set_fact: + s2i_ci_registry_endpoint: >- + {{ buildset_registry_alias }}:{{ buildset_registry.port }} + + - name: Write the normalized registry connection + ansible.builtin.copy: + dest: "{{ s2i_ci_registry_connection_file }}" + mode: "0600" + content: | + {{ { + 'endpoint': s2i_ci_registry_endpoint, + 'public_host': buildset_registry.host, + 'public_port': buildset_registry.port, + 'auth_file': '', + 'cert_dir': '' + } | to_nice_json }} + +- name: Validate the normalized buildset registry + ansible.builtin.import_playbook: ../shared/validate-registry.yaml diff --git a/playbooks/container-ci/zuul/reset-static-node.yaml b/playbooks/container-ci/zuul/reset-static-node.yaml new file mode 100644 index 0000000..da8a034 --- /dev/null +++ b/playbooks/container-ci/zuul/reset-static-node.yaml @@ -0,0 +1,86 @@ +--- +- name: Reset temporary state on a reused static build node + hosts: builder + gather_facts: false + vars: + s2i_ci_reset_static_node: false + s2i_ci_workspace_root: "{{ zuul_user_dir | default(ansible_user_dir) }}" + s2i_ci_registry_root: "{{ ansible_user_dir }}/buildset_registry" + s2i_ci_output_dir: "{{ ansible_user_dir }}/zuul-output" + s2i_ci_registry_port: 5000 + s2i_ci_validation_repository: s2i-validation/ubi10-minimal:latest + + tasks: + - name: Resolve the inventory-described container repository + when: s2i_ci_reset_static_node | bool + ansible.builtin.set_fact: + s2i_ci_container_repo: >- + {{ s2i_ci_workspace_root }}/{{ zuul.projects[s2i_ci_container_project].src_dir }} + + - name: Require safe static-node cleanup boundaries + when: s2i_ci_reset_static_node | bool + ansible.builtin.assert: + that: + - s2i_ci_registry_root == ansible_user_dir + '/buildset_registry' + - s2i_ci_output_dir == ansible_user_dir + '/zuul-output' + - (s2i_ci_container_repo | realpath).startswith((s2i_ci_workspace_root | realpath) + '/') + - s2i_ci_container_repo != s2i_ci_workspace_root + fail_msg: Static-node cleanup is restricted to job-owned paths + + - name: Remove a stale buildset registry container + when: s2i_ci_reset_static_node | bool + ansible.builtin.command: + argv: + - podman + - rm + - --force + - "{{ 'buildset_registry' if s2i_ci_registry_port | int == 5000 else 'buildset_registry_' + s2i_ci_registry_port | string }}" + register: s2i_ci_stale_registry + changed_when: s2i_ci_stale_registry.rc == 0 + failed_when: s2i_ci_stale_registry.rc not in [0, 1] + + - name: Stop a stale buildset registry tunnel + when: s2i_ci_reset_static_node | bool + ansible.builtin.shell: | + pids=$(pgrep -f '^socat -d -d TCP6-LISTEN:{{ s2i_ci_registry_port }},fork TCP:127.0.0.1:1{{ s2i_ci_registry_port }}$' || true) + if [ -n "$pids" ]; then + printf '%s\n' "$pids" + kill $pids + fi + args: + executable: /bin/bash + register: s2i_ci_stale_registry_tunnel + changed_when: s2i_ci_stale_registry_tunnel.stdout | length > 0 + + - name: Remove the stale registry validation tag + when: s2i_ci_reset_static_node | bool + ansible.builtin.command: + argv: + - podman + - image + - rm + - --force + - "zuul-jobs.buildset-registry:{{ s2i_ci_registry_port }}/{{ s2i_ci_validation_repository }}" + register: s2i_ci_stale_validation_image + changed_when: s2i_ci_stale_validation_image.rc == 0 + failed_when: s2i_ci_stale_validation_image.rc not in [0, 1] + + - name: Remove interrupted Buildah working containers + when: s2i_ci_reset_static_node | bool + ansible.builtin.command: + argv: + - buildah + - rm + - --all + register: s2i_ci_stale_buildah_containers + changed_when: s2i_ci_stale_buildah_containers.stdout | length > 0 + + - name: Remove stale job-owned directories + when: s2i_ci_reset_static_node | bool + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - "{{ s2i_ci_registry_root }}" + - "{{ s2i_ci_output_dir }}" + - "{{ s2i_ci_container_repo }}/.tmp" diff --git a/playbooks/container-ci/zuul/run.yaml b/playbooks/container-ci/zuul/run.yaml new file mode 100644 index 0000000..e8afdfd --- /dev/null +++ b/playbooks/container-ci/zuul/run.yaml @@ -0,0 +1,12 @@ +--- +- name: Run shared selective image publication + ansible.builtin.import_playbook: ../shared/run.yaml + +- name: Return published image metadata to Zuul + hosts: builder + gather_facts: false + + tasks: + - name: Return content-provider metadata to Zuul + when: s2i_ci_content_provider | default(false) | bool + ansible.builtin.include_tasks: content-provider-return.yaml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..bcdffb5 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,29 @@ +[tool.ruff] +line-length = 79 +target-version = "py311" + +[tool.ruff.lint] +select = ["E4", "E7", "E9", "F", "G", "I", "LOG", "S", "UP", "W", "C90"] +external = ["H"] +ignore = [ + "S101", + "S105", + "S106", +] + +[tool.ruff.lint.per-file-ignores] +"tests/*" = ["S"] + +[tool.ruff.lint.isort] +force-single-line = true +lines-after-imports = 2 +lines-between-types = 1 +section-order = ["future", "standard-library", "third-party", "first-party"] +force-sort-within-sections = false + +[tool.ruff.lint.mccabe] +max-complexity = 20 + +[tool.ruff.format] +quote-style = "preserve" +docstring-code-format = true diff --git a/tests/test_provider_architecture.py b/tests/test_provider_architecture.py new file mode 100644 index 0000000..192da46 --- /dev/null +++ b/tests/test_provider_architecture.py @@ -0,0 +1,220 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import pathlib +import re +import unittest + + +class ProviderArchitectureTest(unittest.TestCase): + def setUp(self): + self.repo_root = pathlib.Path(__file__).resolve().parents[1] + self.container_ci = self.repo_root / "playbooks" / "container-ci" + + def _read(self, path): + return (self.repo_root / path).read_text(encoding="utf-8") + + def _metadata_keys(self, path): + keys = [] + for line in path.read_text(encoding="utf-8").splitlines(): + match = re.match(r"^ - (\S+)$", line) + if match: + keys.append(match.group(1)) + return keys + + def test_every_containerfile_has_local_image_metadata(self): + containerfiles = sorted( + (self.repo_root / "containers").glob("**/Containerfile") + ) + + self.assertEqual(4, len(containerfiles)) + for containerfile in containerfiles: + metadata = containerfile.with_name("image.yaml") + self.assertTrue(metadata.is_file(), metadata) + self.assertIn( + "openstack_version:\n custom_container_images:", + metadata.read_text(encoding="utf-8"), + ) + + def test_tracked_deployment_mappings_are_intentional(self): + expected = { + "containers/base/image.yaml": [], + "containers/cyborg/cyborg/image.yaml": [], + "containers/cyborg/cyborg-agent/image.yaml": [], + "containers/watcher/watcher-base/image.yaml": [ + "watcherAPIImage", + "watcherApplierImage", + "watcherDecisionEngineImage", + ], + } + + for relative_path, keys in expected.items(): + self.assertEqual( + keys, + self._metadata_keys(self.repo_root / relative_path), + ) + + def test_container_ci_mutations_target_only_builder(self): + playbooks = sorted(self.container_ci.glob("**/*.yaml")) + + self.assertTrue(playbooks) + for playbook in playbooks: + content = playbook.read_text(encoding="utf-8") + self.assertNotIn("hosts: all", content, playbook) + self.assertNotIn("hosts: localhost", content, playbook) + self.assertNotIn("delegate_to: localhost", content, playbook) + self.assertNotIn("local_action:", content, playbook) + for host_pattern in re.findall(r"^ hosts: (.+)$", content, re.M): + self.assertEqual("builder", host_pattern, playbook) + + def test_exact_cleanup_does_not_hide_failures(self): + cleanup = self._read( + "playbooks/container-ci/shared/cleanup-images.yaml" + ) + + self.assertNotIn("failed_when: false", cleanup) + self.assertIn("podman", cleanup) + self.assertIn("buildah", cleanup) + self.assertIn( + "Require exact workflow references to be absent", cleanup + ) + + def test_shared_and_zuul_ownership_is_separated(self): + shared = { + path.name for path in (self.container_ci / "shared").glob("*") + } + zuul = {path.name for path in (self.container_ci / "zuul").glob("*")} + + self.assertTrue( + { + "prepare-host.yaml", + "validate-registry.yaml", + "run.yaml", + "cleanup-images.yaml", + }.issubset(shared) + ) + self.assertTrue( + { + "pre.yaml", + "run.yaml", + "post.yaml", + "reset-static-node.yaml", + "content-provider-return.yaml", + }.issubset(zuul) + ) + self.assertNotIn("content-provider-return.yaml", shared) + + def test_provider_job_keeps_explicit_builder_contract(self): + zuul = self._read(".zuul.yaml") + + self.assertIn("name: builder", zuul) + self.assertIn("nodeset: s2i-openstack-containers-image-builder", zuul) + self.assertIn("- watcher/watcher-base", zuul) + self.assertIn("- cyborg/cyborg", zuul) + self.assertIn("- cyborg/cyborg-agent", zuul) + self.assertNotIn("- project:", zuul) + + def test_provider_validates_repository_on_builder(self): + run = self._read("playbooks/container-ci/shared/run.yaml") + + self.assertIn("ansible.builtin.stat:", run) + self.assertIn("s2i_ci_build_entry_point.stat.isreg", run) + self.assertIn("argv: [realpath, --canonicalize-existing", run) + self.assertNotIn(" is file", run) + self.assertNotIn(" | realpath", run) + + def test_inventory_provider_inputs_are_not_masked_by_play_vars(self): + shared_run = self._read("playbooks/container-ci/shared/run.yaml") + zuul_run = self._read("playbooks/container-ci/zuul/run.yaml") + prepare_host = self._read( + "playbooks/container-ci/shared/prepare-host.yaml" + ) + + shared_play_vars = shared_run.split(" tasks:", 1)[0] + for variable in ( + "s2i_ci_namespace:", + "s2i_ci_tag:", + "s2i_ci_stream:", + "s2i_ci_parallel:", + "s2i_ci_images:", + "s2i_ci_image_mappings:", + ): + self.assertNotIn(variable, shared_play_vars) + self.assertNotIn("s2i_ci_content_provider: false", zuul_run) + self.assertNotIn("s2i_ci_install_host_packages: true", prepare_host) + self.assertIn("s2i_ci_images | default([])", shared_run) + self.assertIn( + "s2i_ci_content_provider | default(false) | bool", zuul_run + ) + + def test_return_contract_is_selective_and_secret_free(self): + returned = self._read( + "playbooks/container-ci/zuul/content-provider-return.yaml" + ) + + for field in ( + "s2i_ci_content:", + "content_provider_os_custom_container_images:", + 'content_provider_os_registry_url: "null"', + "content_provider_dlrn_md5_hash:", + "content_provider_gating_repo_available: false", + "content_provider_gating_repo_url:", + "content_provider_registry_ip:", + "content_provider_registry_ip_port:", + "cifmw_build_images_output: {}", + "pause: true", + ): + self.assertIn(field, returned) + for secret in ("password", "username", "auth_file", "cert_dir"): + self.assertNotIn(secret, returned) + self.assertIn("s2i_ci_public_registry_endpoint", returned) + self.assertNotIn("s2i_ci_registry.endpoint", returned) + + def test_watcher_image_is_process_neutral(self): + containerfile = self._read( + "containers/watcher/watcher-base/Containerfile" + ) + bindeps = set( + line + for line in self._read( + "containers/watcher/watcher-base/bindeps.txt" + ).splitlines() + if line and not line.startswith("#") + ) + + self.assertIn( + "Consolidated Watcher API, applier, and decision-engine", + containerfile, + ) + self.assertTrue( + { + "httpd", + "python3-mod_wsgi", + "libffi", + "libxml2", + "libxslt", + }.issubset(bindeps) + ) + + def test_c2_has_no_oib_or_local_adapter(self): + self.assertFalse((self.repo_root / "openstack_image_builder").exists()) + self.assertFalse((self.container_ci / "local").exists()) + all_content = "\n".join( + path.read_text(encoding="utf-8") + for path in self.container_ci.glob("**/*.yaml") + ) + for forbidden in ("S2I_CONTEXTS_ROOT", "ERROR_ON_CLONE", "oib"): + self.assertNotIn(forbidden, all_content) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_provider_shell.py b/tests/test_provider_shell.py new file mode 100644 index 0000000..69110eb --- /dev/null +++ b/tests/test_provider_shell.py @@ -0,0 +1,192 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import pathlib +import subprocess +import tempfile +import time +import unittest + + +class ProviderShellTest(unittest.TestCase): + def setUp(self): + self.repo_root = pathlib.Path(__file__).resolve().parents[1] + temporary_root = self.repo_root / ".tmp" + temporary_root.mkdir(exist_ok=True) + self.temporary_directory = tempfile.TemporaryDirectory( + dir=temporary_root, prefix="provider-shell." + ) + self.addCleanup(self.temporary_directory.cleanup) + self.root = pathlib.Path(self.temporary_directory.name) + (self.root / "build.sh").symlink_to(self.repo_root / "build.sh") + self.bin_dir = self.root / "bin" + self.bin_dir.mkdir() + self.logs_dir = self.root / "logs" + self._create_images() + self._create_fake_buildah() + + def _create_images(self): + for target in ("base", "alpha/one", "beta/two"): + image_root = self.root / "containers" / target + image_root.mkdir(parents=True) + (image_root / "Containerfile").write_text( + "FROM scratch\n", encoding="utf-8" + ) + project = target.split("/", maxsplit=1)[0] + project_root = self.root / "containers" / project + (project_root / "requirements.lock.master").touch() + if "/" in target: + (project_root / "src" / project).mkdir( + parents=True, exist_ok=True + ) + + def _create_fake_buildah(self): + fake = self.bin_dir / "buildah" + fake.write_text( + """#!/usr/bin/env python3 +import os +import pathlib +import sys +import time + +args = sys.argv[1:] +if args[0] == "bud": + containerfile = pathlib.Path(args[args.index("-f") + 1]) + image = containerfile.parent.name + if containerfile.parent.parent.name != "containers": + image = f"{containerfile.parent.parent.name}/{image}" + if image != "base": + print(f"LIVE {image}", flush=True) + time.sleep(0.75) + if os.environ.get("FAIL_IMAGE") and image.endswith(os.environ["FAIL_IMAGE"]): + print(f"FAIL {image}", flush=True) + sys.exit(9) + print(f"DONE {image}", flush=True) + sys.exit(0) +if args[0] == "inspect": + sys.exit(0) +if args[0] == "push": + sys.exit(0) +sys.exit(2) +""", + encoding="utf-8", + ) + fake.chmod(0o755) + + def _environment(self): + environment = os.environ.copy() + environment.update( + { + "PATH": f"{self.bin_dir}:{environment['PATH']}", + "STREAM": "master", + "REGISTRY": "registry.test:5000", + "NAMESPACE": "openstack", + "TAG": "test", + "PARALLEL": "2", + "BUILD_LOGS_DIR": str(self.logs_dir), + } + ) + return environment + + def _run(self, *arguments, environment=None): + return subprocess.run( + [str(self.root / "build.sh"), *arguments], + cwd=self.root, + env=environment or self._environment(), + check=False, + capture_output=True, + text=True, + ) + + def test_explicit_union_is_ordered_and_includes_base(self): + result = self._run("refs", "beta/two,alpha/one,beta/two") + + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual( + [ + "registry.test:5000/openstack/openstack-base:test", + "registry.test:5000/openstack/openstack-two:test", + "registry.test:5000/openstack/openstack-one:test", + ], + result.stdout.splitlines(), + ) + + def test_single_target_semantics_are_unchanged(self): + result = self._run("refs", "alpha/one") + + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual( + ["registry.test:5000/openstack/openstack-one:test"], + result.stdout.splitlines(), + ) + + def test_explicit_union_rejects_unknown_target(self): + result = self._run("refs", "alpha/one,unknown/image") + + self.assertNotEqual(0, result.returncode) + self.assertIn("Unknown image or project", result.stderr) + + def test_parallel_output_is_live_and_logs_are_retained(self): + process = subprocess.Popen( + [ + str(self.root / "build.sh"), + "build-parallel", + "alpha/one,beta/two", + ], + cwd=self.root, + env=self._environment(), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + self.addCleanup( + lambda: process.kill() if process.poll() is None else None + ) + self.addCleanup(process.stdout.close) + output = [] + deadline = time.monotonic() + 5 + while time.monotonic() < deadline: + line = process.stdout.readline() + output.append(line) + if "LIVE" in line: + break + + self.assertIn("LIVE", "".join(output)) + self.assertIsNone( + process.poll(), "build exited before live output arrived" + ) + output.append(process.stdout.read()) + process.wait(timeout=10) + self.assertEqual(0, process.returncode, "".join(output)) + self.assertIn("[alpha/one] LIVE alpha/one", "".join(output)) + self.assertIn("[beta/two] LIVE beta/two", "".join(output)) + self.assertTrue((self.logs_dir / "base.log").is_file()) + self.assertTrue((self.logs_dir / "alpha_one.log").is_file()) + self.assertTrue((self.logs_dir / "beta_two.log").is_file()) + self.assertEqual(1, "".join(output).count("LIVE alpha/one")) + + def test_parallel_failure_propagates(self): + environment = self._environment() + environment["FAIL_IMAGE"] = "two" + + result = self._run( + "build-parallel", "alpha/one,beta/two", environment=environment + ) + + self.assertNotEqual(0, result.returncode) + self.assertIn("FAIL beta/two", result.stdout + result.stderr) + self.assertIn("stopping remaining builds", result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_update_sources.py b/tests/test_update_sources.py new file mode 100644 index 0000000..8c0c1f4 --- /dev/null +++ b/tests/test_update_sources.py @@ -0,0 +1,296 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Tests for build.sh update-sources behavior.""" + +import os +import pathlib +import subprocess +import tempfile +import unittest + + +class UpdateSourcesTest(unittest.TestCase): + def setUp(self): + self.repo_root = pathlib.Path(__file__).resolve().parents[1] + tmp_root = self.repo_root / ".tmp" + tmp_root.mkdir(exist_ok=True) + self.temporary_directory = tempfile.TemporaryDirectory( + dir=tmp_root, prefix="update-sources-test." + ) + self.addCleanup(self.temporary_directory.cleanup) + self.test_root = pathlib.Path(self.temporary_directory.name) + upstream_root = self.test_root / "upstream" + upstream_root.mkdir() + + self.upstream_requirements = upstream_root / "requirements.git" + self.requirements_old, self.requirements_new = self.create_remote( + self.upstream_requirements, + [ + {"upper-constraints.txt": "six==1.17.0\n"}, + {"upper-constraints.txt": ("six==1.17.0\npbr==7.0.3\n")}, + ], + ) + self.upstream_service = upstream_root / "test-svc.git" + self.service_old, self.service_new = self.create_remote( + self.upstream_service, + [ + {"requirements.txt": "six\npbr\n"}, + {"requirements.txt": "six\npbr\n# v2\n"}, + ], + ) + (self.test_root / "build.sh").symlink_to(self.repo_root / "build.sh") + self.project_root = self.test_root / "containers" / "test-svc" + image_root = self.project_root / "test-svc" + (self.project_root / "src").mkdir(parents=True) + (image_root / "src").mkdir(parents=True) + self.write_sources( + "master upper-constraints " + f"{self.upstream_requirements} master {self.requirements_old}\n" + "master test-svc " + f"{self.upstream_service} master {self.service_old}\n" + ) + (image_root / "Containerfile").write_text( + "FROM scratch\n", encoding="utf-8" + ) + (image_root / "bindeps.txt").write_text("python3\n", encoding="utf-8") + (image_root / "builddeps.txt").write_text("gcc\n", encoding="utf-8") + (image_root / "pythondeps.txt").touch() + (image_root / "pythonbuilddeps.txt").touch() + + def run_command(self, command, cwd=None): + result = subprocess.run( + command, + cwd=cwd, + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + self.fail( + f"command failed with {result.returncode}: {command}\n" + f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}" + ) + return result.stdout.strip() + + def create_remote(self, destination, commits): + work = self.test_root / f"work-{destination.stem}" + self.run_command(["git", "init", "-b", "master", str(work)]) + self.run_command( + ["git", "config", "user.email", "test@example.com"], work + ) + self.run_command(["git", "config", "user.name", "Test"], work) + self.run_command(["git", "config", "commit.gpgsign", "false"], work) + hashes = [] + for index, files in enumerate(commits, start=1): + for name, content in files.items(): + (work / name).write_text(content, encoding="utf-8") + self.run_command(["git", "add", "-A"], work) + self.run_command(["git", "commit", "-m", f"v{index}"], work) + hashes.append(self.run_command(["git", "rev-parse", "HEAD"], work)) + self.run_command( + ["git", "clone", "--bare", str(work), str(destination)] + ) + return hashes[0], hashes[-1] + + def write_sources(self, content): + (self.project_root / "sources.txt").write_text( + content, encoding="utf-8" + ) + + def run_update(self, **environment): + command_environment = os.environ.copy() + command_environment.update({"STREAM": "master"}) + command_environment.update(environment) + result = subprocess.run( + ["bash", "./build.sh", "update-sources", "test-svc"], + cwd=self.test_root, + check=False, + capture_output=True, + text=True, + env=command_environment, + ) + (self.test_root / "build.log").write_text( + result.stdout + result.stderr, encoding="utf-8" + ) + if result.returncode != 0: + self.fail( + f"update-sources failed with {result.returncode}:\n" + f"{result.stdout}{result.stderr}" + ) + return result.stdout + result.stderr + + def source_field(self, name, field): + for line in ( + (self.project_root / "sources.txt") + .read_text(encoding="utf-8") + .splitlines() + ): + columns = line.split() + if columns[:2] == ["master", name]: + return columns[field - 1] + self.fail(f"source entry was not found: {name}") + + def test_updates_hashes_to_branch_tip(self): + self.run_update() + + self.assertEqual( + self.requirements_new, + self.source_field("upper-constraints", 5), + ) + self.assertEqual(self.service_new, self.source_field("test-svc", 5)) + + def test_fetches_upper_constraints(self): + self.run_update() + + constraints = ( + self.project_root / "upper-constraints.txt.master" + ).read_text(encoding="utf-8") + self.assertIn("six==1.17.0", constraints) + self.assertIn("pbr==7.0.3", constraints) + + def test_generates_rpms_in_yaml(self): + self.run_update() + + rpms = (self.project_root / "rpms.in.yaml").read_text(encoding="utf-8") + self.assertIn("python3", rpms) + self.assertIn("gcc", rpms) + + def test_generates_requirements_lock(self): + self.run_update() + + lock = (self.project_root / "requirements.lock.master").read_text( + encoding="utf-8" + ) + self.assertIn("six", lock) + + def test_generates_buildrequirements_lock(self): + self.run_update() + + self.assertTrue( + (self.project_root / "buildrequirements.lock.master").is_file() + ) + + def test_creates_default_stream_symlinks(self): + self.run_update(DEFAULT_STREAM="master") + + expected = { + "upper-constraints.txt": "upper-constraints.txt.master", + "requirements.lock": "requirements.lock.master", + "buildrequirements.lock": "buildrequirements.lock.master", + } + for name, target in expected.items(): + link = self.project_root / name + self.assertTrue(link.is_symlink()) + self.assertEqual(target, os.readlink(link)) + + def test_skips_symlinks_for_non_default_stream(self): + self.run_update(DEFAULT_STREAM="other") + + for name in ( + "upper-constraints.txt", + "requirements.lock", + "buildrequirements.lock", + ): + self.assertFalse((self.project_root / name).is_symlink()) + + def test_skip_hash_update_preserves_hashes(self): + self.run_update(SKIP_HASH_UPDATE="1") + + self.assertEqual( + self.requirements_old, + self.source_field("upper-constraints", 5), + ) + self.assertEqual(self.service_old, self.source_field("test-svc", 5)) + self.assertTrue( + (self.project_root / "requirements.lock.master").is_file() + ) + + def test_skip_hash_update_uses_pinned_constraints(self): + self.run_update(SKIP_HASH_UPDATE="1") + + constraints = ( + self.project_root / "upper-constraints.txt.master" + ).read_text(encoding="utf-8") + self.assertIn("six==1.17.0", constraints) + self.assertNotIn("pbr", constraints) + + def test_hash_in_branch_field_selects_constraints_commit(self): + self.write_sources( + "master upper-constraints " + f"{self.upstream_requirements} {self.requirements_old} " + f"{self.requirements_new}\n" + "master test-svc " + f"{self.upstream_service} master {self.service_old}\n" + ) + + self.run_update() + + self.assertEqual( + self.requirements_old, + self.source_field("upper-constraints", 5), + ) + constraints = ( + self.project_root / "upper-constraints.txt.master" + ).read_text(encoding="utf-8") + self.assertIn("six==1.17.0", constraints) + self.assertNotIn("pbr", constraints) + + def test_hash_in_branch_field_selects_regular_repo_commit(self): + self.write_sources( + "master upper-constraints " + f"{self.upstream_requirements} master {self.requirements_old}\n" + "master test-svc " + f"{self.upstream_service} {self.service_old} " + f"{self.service_new}\n" + ) + + self.run_update() + + self.assertEqual(self.service_old, self.source_field("test-svc", 5)) + self.assertTrue( + (self.project_root / "requirements.lock.master").is_file() + ) + + def test_lockfile_excludes_rpm_python_packages(self): + image_root = self.project_root / "test-svc" + (image_root / "bindeps.txt").write_text( + "python3\npython3-six\n", encoding="utf-8" + ) + + output = self.run_update() + + lock = (self.project_root / "requirements.lock.master").read_text( + encoding="utf-8" + ) + self.assertNotIn("six==", lock) + self.assertIn("Filtering RPM-provided packages", output) + + def test_preexisting_checkout_is_preserved(self): + source = self.project_root / "src" / "test-svc" + source.mkdir() + (source / "MARKER").write_text("local-dev\n", encoding="utf-8") + (source / "requirements.txt").write_text( + "six\npbr\n", encoding="utf-8" + ) + + self.run_update() + + self.assertEqual( + "local-dev\n", + (source / "MARKER").read_text(encoding="utf-8"), + ) + self.assertEqual(self.service_old, self.source_field("test-svc", 5)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_update_sources.sh b/tests/test_update_sources.sh deleted file mode 100755 index 81dbd5c..0000000 --- a/tests/test_update_sources.sh +++ /dev/null @@ -1,354 +0,0 @@ -#!/usr/bin/env bash -# Tests for build.sh update-sources functionality. -# -# Uses local bare git repos as fake remotes so tests run offline and fast. -# pip-compile and pybuild-deps must be on PATH for lockfile tests. -# -# Usage: -# PATH=".tox/update-sources/bin:$PATH" bash tests/test_update_sources.sh -# tox -e test -# -set -uo pipefail - -# ── Test runner ────────────────────────────────────────────────────────── - -_PASS=0 -_FAIL=0 -_SKIP=0 - -assert() { - local desc="$1" - shift - if "$@"; then - return 0 - fi - echo " ASSERTION FAILED: ${desc}" - echo " command: $*" - return 1 -} - -assert_file_exists() { assert "file exists: $1" test -f "$1"; } -assert_symlink() { assert "symlink exists: $1" test -L "$1"; } -assert_no_symlink() { assert "no symlink: $1" test ! -L "$1"; } -assert_grep() { assert "grep '$1' in $2" grep -q "$1" "$2"; } -assert_no_grep() { - if grep -q "$1" "$2" 2>/dev/null; then - echo " ASSERTION FAILED: '$1' should not appear in $2" - return 1 - fi -} - -assert_link_target() { - local link="$1" expected="$2" - local actual - actual="$(readlink "$1")" - assert "symlink $1 -> $2 (actual: ${actual})" test "${actual}" = "${expected}" -} - -assert_field() { - local file="$1" stream="$2" name="$3" field="$4" expected="$5" - local actual - actual=$(awk -v s="${stream}" -v n="${name}" '$1==s && $2==n {print $'${field}'}' "${file}") - assert "sources.txt ${name} field ${field} == ${expected} (actual: ${actual})" \ - test "${actual}" = "${expected}" -} - -run_test() { - local name="$1" - - _setup_fixture - - local rc=0 - # Run in subshell with set -e so first failed assertion stops the test - ( set -e; "${name}" ) || rc=$? - - if [[ ${rc} -eq 0 ]]; then - echo " PASS ${name}" - ((_PASS++)) - elif [[ ${rc} -eq 99 ]]; then - echo " SKIP ${name}" - ((_SKIP++)) - else - echo " FAIL ${name}" - ((_FAIL++)) - if [[ -f "${TEST_DIR}/build.log" ]]; then - echo " --- build.log (last 20 lines) ---" - tail -20 "${TEST_DIR}/build.log" | sed 's/^/ /' - echo " ---" - fi - fi - - _teardown_fixture -} - -skip_test() { - echo " skipping: $1" - return 99 -} - -# ── Fixture ────────────────────────────────────────────────────────────── - -SCRIPT_DIR="$(cd "$(dirname "$0")/.." && pwd)" -TEST_DIR="" -UPSTREAM_REQ="" -UPSTREAM_SVC="" -REQ_HASH_OLD="" -REQ_HASH_NEW="" -SVC_HASH_OLD="" -SVC_HASH_NEW="" - -_init_work_repo() { - local dir="$1" - git init -b master "${dir}" >/dev/null 2>&1 - git -C "${dir}" config user.email "test@test.com" - git -C "${dir}" config user.name "Test" -} - -_setup_fixture() { - TEST_DIR="$(mktemp -d)" - local work - - # ── Upstream requirements repo (2 commits) ── - UPSTREAM_REQ="${TEST_DIR}/upstream/requirements.git" - mkdir -p "${TEST_DIR}/upstream" - work="$(mktemp -d)" - _init_work_repo "${work}" - - echo "six==1.17.0" > "${work}/upper-constraints.txt" - git -C "${work}" add -A >/dev/null && git -C "${work}" commit -m "v1" >/dev/null 2>&1 - - printf 'six==1.17.0\npbr==7.0.3\n' > "${work}/upper-constraints.txt" - git -C "${work}" add -A >/dev/null && git -C "${work}" commit -m "v2" >/dev/null 2>&1 - - git clone --bare "${work}" "${UPSTREAM_REQ}" >/dev/null 2>&1 - rm -rf "${work}" - - REQ_HASH_OLD="$(git -C "${UPSTREAM_REQ}" rev-parse master~1)" - REQ_HASH_NEW="$(git -C "${UPSTREAM_REQ}" rev-parse master)" - - # ── Upstream service repo (2 commits) ── - UPSTREAM_SVC="${TEST_DIR}/upstream/test-svc.git" - work="$(mktemp -d)" - _init_work_repo "${work}" - - echo "six" > "${work}/requirements.txt" - git -C "${work}" add -A >/dev/null && git -C "${work}" commit -m "v1" >/dev/null 2>&1 - - printf 'six\npbr\n' > "${work}/requirements.txt" - git -C "${work}" add -A >/dev/null && git -C "${work}" commit -m "v2" >/dev/null 2>&1 - - git clone --bare "${work}" "${UPSTREAM_SVC}" >/dev/null 2>&1 - rm -rf "${work}" - - SVC_HASH_OLD="$(git -C "${UPSTREAM_SVC}" rev-parse master~1)" - SVC_HASH_NEW="$(git -C "${UPSTREAM_SVC}" rev-parse master)" - - # ── Symlink build.sh ── - ln -s "${SCRIPT_DIR}/build.sh" "${TEST_DIR}/build.sh" - - # ── Containers tree ── - mkdir -p "${TEST_DIR}/containers/test-svc/src" - mkdir -p "${TEST_DIR}/containers/test-svc/test-svc/src" - - cat > "${TEST_DIR}/containers/test-svc/sources.txt" < "${TEST_DIR}/containers/test-svc/test-svc/Containerfile" - echo "python3" > "${TEST_DIR}/containers/test-svc/test-svc/bindeps.txt" - echo "gcc" > "${TEST_DIR}/containers/test-svc/test-svc/builddeps.txt" - touch "${TEST_DIR}/containers/test-svc/test-svc/pythondeps.txt" - touch "${TEST_DIR}/containers/test-svc/test-svc/pythonbuilddeps.txt" -} - -_teardown_fixture() { - [[ -n "${TEST_DIR}" ]] && rm -rf "${TEST_DIR}" -} - -# Helper: run build.sh inside TEST_DIR with env vars passed as arguments. -# Usage: _run_build STREAM=master [SKIP_HASH_UPDATE=1 ...] -_run_build() { - (cd "${TEST_DIR}" && env "$@" ./build.sh update-sources test-svc) >"${TEST_DIR}/build.log" 2>&1 -} - -# ── Tests ──────────────────────────────────────────────────────────────── - -test_updates_hashes_to_branch_tip() { - _run_build STREAM=master - - local src="${TEST_DIR}/containers/test-svc/sources.txt" - assert_field "${src}" master upper-constraints 5 "${REQ_HASH_NEW}" - assert_field "${src}" master test-svc 5 "${SVC_HASH_NEW}" -} - -test_fetches_upper_constraints() { - _run_build STREAM=master - - local uc="${TEST_DIR}/containers/test-svc/upper-constraints.txt.master" - assert_file_exists "${uc}" - assert_grep "six==1.17.0" "${uc}" - assert_grep "pbr==7.0.3" "${uc}" -} - -test_generates_rpms_in_yaml() { - _run_build STREAM=master - - local rpms="${TEST_DIR}/containers/test-svc/rpms.in.yaml" - assert_file_exists "${rpms}" - assert_grep "python3" "${rpms}" - assert_grep "gcc" "${rpms}" -} - -test_generates_requirements_lock() { - command -v pip-compile >/dev/null 2>&1 || skip_test "pip-compile not on PATH" - - _run_build STREAM=master - - local lock="${TEST_DIR}/containers/test-svc/requirements.lock.master" - assert_file_exists "${lock}" - assert_grep "six" "${lock}" -} - -test_generates_buildrequirements_lock() { - command -v pip-compile >/dev/null 2>&1 || skip_test "pip-compile not on PATH" - command -v pybuild-deps >/dev/null 2>&1 || skip_test "pybuild-deps not on PATH" - - _run_build STREAM=master - - assert_file_exists "${TEST_DIR}/containers/test-svc/buildrequirements.lock.master" -} - -test_creates_default_stream_symlinks() { - command -v pip-compile >/dev/null 2>&1 || skip_test "pip-compile not on PATH" - command -v pybuild-deps >/dev/null 2>&1 || skip_test "pybuild-deps not on PATH" - - _run_build STREAM=master DEFAULT_STREAM=master - - local d="${TEST_DIR}/containers/test-svc" - assert_symlink "${d}/upper-constraints.txt" - assert_symlink "${d}/requirements.lock" - assert_symlink "${d}/buildrequirements.lock" - assert_link_target "${d}/requirements.lock" "requirements.lock.master" - assert_link_target "${d}/buildrequirements.lock" "buildrequirements.lock.master" - assert_link_target "${d}/upper-constraints.txt" "upper-constraints.txt.master" -} - -test_skips_symlinks_for_non_default_stream() { - command -v pip-compile >/dev/null 2>&1 || skip_test "pip-compile not on PATH" - command -v pybuild-deps >/dev/null 2>&1 || skip_test "pybuild-deps not on PATH" - - _run_build STREAM=master DEFAULT_STREAM=other - - local d="${TEST_DIR}/containers/test-svc" - assert_no_symlink "${d}/requirements.lock" - assert_no_symlink "${d}/buildrequirements.lock" - assert_no_symlink "${d}/upper-constraints.txt" -} - -test_skip_hash_update_preserves_hashes() { - command -v pip-compile >/dev/null 2>&1 || skip_test "pip-compile not on PATH" - - _run_build STREAM=master SKIP_HASH_UPDATE=1 - - local src="${TEST_DIR}/containers/test-svc/sources.txt" - assert_field "${src}" master upper-constraints 5 "${REQ_HASH_OLD}" - assert_field "${src}" master test-svc 5 "${SVC_HASH_OLD}" - - assert_file_exists "${TEST_DIR}/containers/test-svc/requirements.lock.master" -} - -test_skip_hash_update_fetches_constraints_at_pinned_hash() { - _run_build STREAM=master SKIP_HASH_UPDATE=1 - - local uc="${TEST_DIR}/containers/test-svc/upper-constraints.txt.master" - assert_file_exists "${uc}" - assert_grep "six==1.17.0" "${uc}" - assert_no_grep "pbr" "${uc}" -} - -test_hash_in_branch_field_upper_constraints() { - cat > "${TEST_DIR}/containers/test-svc/sources.txt" </dev/null 2>&1 || skip_test "pip-compile not on PATH" - - cat > "${TEST_DIR}/containers/test-svc/sources.txt" </dev/null 2>&1 || skip_test "pip-compile not on PATH" - - printf 'python3\npython3-six\n' > "${TEST_DIR}/containers/test-svc/test-svc/bindeps.txt" - - _run_build STREAM=master - - local lock="${TEST_DIR}/containers/test-svc/requirements.lock.master" - assert_file_exists "${lock}" - assert_no_grep "^six==" "${lock}" - assert_grep "Filtering RPM-provided packages" "${TEST_DIR}/build.log" -} - -test_preexisting_checkout_is_preserved() { - local src_dir="${TEST_DIR}/containers/test-svc/src/test-svc" - mkdir -p "${src_dir}" - echo "local-dev" > "${src_dir}/MARKER" - echo "six" > "${src_dir}/requirements.txt" - - _run_build STREAM=master - - assert_file_exists "${src_dir}/MARKER" - assert_grep "local-dev" "${src_dir}/MARKER" - assert_field "${TEST_DIR}/containers/test-svc/sources.txt" master test-svc 5 "${SVC_HASH_OLD}" -} - -# ── Run all tests ──────────────────────────────────────────────────────── - -echo "=== update-sources tests ===" -echo "" - -TESTS=( - test_updates_hashes_to_branch_tip - test_fetches_upper_constraints - test_generates_rpms_in_yaml - test_generates_requirements_lock - test_generates_buildrequirements_lock - test_creates_default_stream_symlinks - test_skips_symlinks_for_non_default_stream - test_skip_hash_update_preserves_hashes - test_skip_hash_update_fetches_constraints_at_pinned_hash - test_hash_in_branch_field_upper_constraints - test_hash_in_branch_field_regular_repo - test_lockfile_excludes_rpm_python_packages - test_preexisting_checkout_is_preserved -) - -for t in "${TESTS[@]}"; do - run_test "${t}" -done - -echo "" -echo "=== ${_PASS} passed, ${_FAIL} failed, ${_SKIP} skipped ===" - -[[ ${_FAIL} -eq 0 ]] diff --git a/tox.ini b/tox.ini index cfb586d..a0245c7 100644 --- a/tox.ini +++ b/tox.ini @@ -1,10 +1,14 @@ [tox] -envlist = linters,update-sources,build,test +envlist = unit,linters skipsdist = true +toxworkdir = {toxinidir}/.tmp/tox [testenv] +setenv = + PYTHONPYCACHEPREFIX = {toxinidir}/.tmp/python-cache passenv = STREAM + TARGET REGISTRY NAMESPACE TAG @@ -20,31 +24,64 @@ passenv = BUILD_LOGS_DIR # pin to pip==26.1.2 as 26.2 has broken pip-tools deps = - pip-tools - pybuild-deps + pip-tools==7.6.0 + pybuild-deps==0.5.0 pip==26.1.2 allowlist_externals = bash +[testenv:{unit,test,py3}] +description = Run all non-Ansible tests through stdlib unittest +commands = + python -m unittest discover -s {toxinidir}/tests -p 'test_*.py' -v {posargs} + [testenv:linters] -description = Lint all Containerfiles with pre-commit (hadolint) +description = Run formatting and style checks through pre-commit deps = pre-commit>=3.0.0 +skip_install = true +setenv = + PYTHONPYCACHEPREFIX = {toxinidir}/.tmp/python-cache + PRE_COMMIT_HOME = {toxinidir}/.tmp/pre-commit + ANSIBLE_HOME = {toxinidir}/.tmp/ansible/home + ANSIBLE_COLLECTIONS_PATH = {toxinidir}/.tmp/ansible/collections + ANSIBLE_LOCAL_TEMP = {toxinidir}/.tmp/ansible/local-tmp +commands = + pre-commit run -a --show-diff-on-failure +commands_post = + python -c "import pathlib, shutil; shutil.rmtree(pathlib.Path(r'{toxinidir}') / '.ansible', ignore_errors=True)" + +[testenv:molecule] +description = Exercise provider registry and cleanup contracts with Molecule +deps = + ansible-core==2.18.7 + molecule==26.6.0 +setenv = + PYTHONPYCACHEPREFIX = {toxinidir}/.tmp/python-cache + MOLECULE_EPHEMERAL_DIRECTORY = {toxinidir}/.tmp/molecule/ephemeral/{envname} + ANSIBLE_HOME = {toxinidir}/.tmp/ansible/home + ANSIBLE_COLLECTIONS_PATH = {toxinidir}/.tmp/ansible/collections + ANSIBLE_LOCAL_TEMP = {toxinidir}/.tmp/ansible/local-tmp + ANSIBLE_ROLES_PATH = {toxinidir}/.tmp/ansible/roles commands = - pre-commit run --all-files --show-diff-on-failure {posargs} + molecule reset -s provider-contract + molecule test -s provider-contract [testenv:update-sources] +description = Refresh source pins and generated lockfiles commands = - bash {toxinidir}/build.sh update-sources all + bash {toxinidir}/build.sh update-sources {env:TARGET:all} [testenv:build] commands = bash {toxinidir}/build.sh build all -[testenv:test] -commands = - bash {toxinidir}/tests/test_update_sources.sh - [testenv:custom] commands = bash {toxinidir}/build.sh {posargs} + +[flake8] +select = H +ignore = H404,H405 +show-source = true +exclude = .git,.tmp,.tox,.venv