rough demo layout for uv workspace setup

.github/workflows/deploy_server.yaml

@@ -77,7 +77,7 @@ jobs:
           --build-arg DRIVER_IMAGE=$DRIVER_IMAGE:latest \
           --tag $SERVER_IMAGE:latest \
           --tag $SERVER_IMAGE:$IMAGE_TAG \
-          server
+          packages/server
 
     - name: "push server image"
       run: |
@@ -96,7 +96,7 @@ jobs:
         URL=$(gcloud run services describe server-test --region australia-southeast1 --platform managed --format "value(status.url)")
         TOKEN=$(gcloud auth print-identity-token --impersonate-service-account="server-deploy@analysis-runner.iam.gserviceaccount.com" --audiences="$URL" --include-email)
 
-        req_obj='{"output": "ci-test-batch/v42", "dataset": "fewgenomes", "accessLevel": "test", "repo": "analysis-runner", "commit": "${{ github.sha }}", "cwd": "test", "script": ["./main.py"], "description": "analysis-runner-test", "wait": true}'
+        req_obj='{"output": "ci-test-batch/v42", "dataset": "fewgenomes", "accessLevel": "test", "repo": "analysis-runner", "commit": "${{ github.sha }}", "cwd": "packages/analysis-runner/tests", "script": ["./main.py"], "description": "analysis-runner-test", "wait": true}'
 
         # sometimes the container takes about 3-4 minutes to load the first time
         # this causes a timeout because we're waiting, so we retry a few times

.github/workflows/deploy_web_server.yaml

@@ -36,7 +36,7 @@ jobs:
 
     - name: "build image"
       run: |
-        docker build --tag $DOCKER_IMAGE web
+        docker build --tag $DOCKER_IMAGE packages/web
 
     - name: "push image"
       run: |

.github/workflows/lint.yaml

@@ -11,18 +11,15 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
-    - uses: actions/setup-python@v5
-      with:
-        python-version: '3.10'
-        cache: 'pip'
+    - name: Install uv
+      uses: astral-sh/setup-uv@v5
 
-    - name: Install packages
-      run: |
-        pip install .
-        pip install -r requirements-dev.txt
+    - name: Install dev tooling
+      run: uv sync
 
     - name: pre-commit
-      run: pre-commit run --all-files
+      run: uv run pre-commit run --all-files
 
-    - name: Run-tests
-      run: python -m unittest test/test_analysis_runner.py
+    - name: Run tests
+      working-directory: packages/analysis-runner
+      run: uv run python -m unittest tests/test_analysis_runner.py

.github/workflows/package.yaml

@@ -14,30 +14,23 @@ jobs:
     defaults:
       run:
         shell: bash -l {0}
+        working-directory: packages/analysis-runner
     steps:
       - uses: actions/checkout@v4
 
-      - uses: actions/setup-python@v5
-        with:
-          python-version: '3.10'
-
-      # Needed until setup-python's image contains a PEP625-compliant setuptools
-      - name: Upgrade setuptools
-        run: pip install --upgrade 'setuptools>=69.3.0'
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
 
       - name: Build
-        run: python setup.py sdist
-
-      - name: Test install
-        run: pip install dist/*
+        run: uv build
 
       - name: Run tests
-        run: python -m unittest test/test_analysis_runner.py
+        run: uv run python -m unittest tests/test_analysis_runner.py
 
       - uses: actions/upload-artifact@v4
         with:
-          name: sdist
-          path: dist/*
+          name: dist
+          path: packages/analysis-runner/dist/*
           retention-days: 2
 
   upload_pypi:
@@ -52,7 +45,7 @@ jobs:
     steps:
       - uses: actions/download-artifact@v4
         with:
-          name: sdist
+          name: dist
           path: dist
 
       # `skip_existing: true` makes sure that the package will be published

.github/workflows/security.yaml

@@ -5,53 +5,28 @@ on:
   push:
 
 jobs:
-  pip-audit-package:
+  pip-audit:
     runs-on: ubuntu-latest
-    name: Pip Audit (Package)
+    strategy:
+      fail-fast: false
+      matrix:
+        package:
+          - analysis-runner
+          - server
+          - web
+          - metamist-consumer
+    name: Pip Audit (${{ matrix.package }})
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
-        with:
-          python-version: '3.10'
-          cache: 'pip'
-          cache-dependency-path: setup.py
-      - name: Install from setup.py
-       # match setuptools version in package workflow
-        run: |
-          pip install --upgrade 'setuptools>=69.3.0'
-          pip install .
-
-      - uses: pypa/gh-action-pip-audit@v1.1.0
-        with:
-          summary: true
 
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
 
-  pip-audit-dev:
-    runs-on: ubuntu-latest
-    name: Pip Audit (Dev)
-    steps:
-      - uses: actions/checkout@v4
-      - uses: pypa/gh-action-pip-audit@v1.1.0
-        with:
-          inputs: requirements-dev.txt
-          summary: true
+      - name: Install locked dependencies
+        working-directory: packages/${{ matrix.package }}
+        run: uv sync --frozen
 
-  pip-audit-web:
-    runs-on: ubuntu-latest
-    name: Pip Audit (Web)
-    steps:
-      - uses: actions/checkout@v4
-      - uses: pypa/gh-action-pip-audit@v1.1.0
-        with:
-          inputs: web/requirements.txt
-          summary: true
-
-  pip-audit-server:
-    runs-on: ubuntu-latest
-    name: Pip Audit (Server)
-    steps:
-      - uses: actions/checkout@v4
       - uses: pypa/gh-action-pip-audit@v1.1.0
         with:
-          inputs: server/requirements.txt
+          virtual-environment: packages/${{ matrix.package }}/.venv
           summary: true

Makefile

@@ -1,18 +1,23 @@
-.PHONY: compile-requirements lint install-dev
+.PHONY: lock install-dev lint
 
-compile-requirements:
-	docker run --platform linux/amd64 -v $$(pwd):/opt/deps python:3.10 /bin/bash -c '\
-		cd /opt/deps; \
-		pip install pip-tools; \
-		pip-compile requirements-dev.in;\
-		pip-compile web/requirements.in;\
-	'
+# Re-resolve every package lockfile. Each package is an independent uv project,
+# so lock them individually; the metamist Cloud Function additionally needs a
+# requirements.txt exported from its lock (Cloud Functions build from pip).
+lock:
+	uv lock
+	uv lock --project packages/analysis-runner
+	uv lock --project packages/server
+	uv lock --project packages/web
+	uv lock --project packages/metamist-consumer
+	uv export --project packages/metamist-consumer --no-dev --no-hashes --no-emit-project \
+		-o packages/metamist-consumer/requirements.txt
 
 
-lint:
-	ruff check
-	ruff format
+# Install the repo-wide dev tooling (ruff, pre-commit, pylint).
+install-dev:
+	uv sync
 
 
-install-dev:
-	pip install --no-deps -r requirements-dev.txt
+lint:
+	uv run ruff check
+	uv run ruff format

README.md

@@ -148,16 +148,43 @@ cluster.add_job('examples/dataproc/query.py', job_name='example')
 
 You can ignore this section if you just want to run the tool.
 
-To set up a development environment for the analysis runner using pip, run
-the following:
+This repository is a monorepo of independent [`uv`](https://docs.astral.sh/uv/)
+projects under [`packages/`](packages):
+
+- [`packages/analysis-runner`](packages/analysis-runner) — the published CLI (the
+  only distributed package).
+- [`packages/server`](packages/server) — the analysis-runner backend (Cloud Run).
+- [`packages/web`](packages/web) — the web-bucket proxy server (Cloud Run).
+- [`packages/metamist-consumer`](packages/metamist-consumer) — the Pub/Sub
+  Cloud Function that records submissions in metamist.
+
+Each package has its own `pyproject.toml`, `uv.lock` and virtual environment, and
+is resolved independently — this is **not** a `uv` workspace, so run `uv` commands
+from within the relevant package directory rather than using `uv run --package`.
+
+Install the repo-wide dev tooling (ruff, pre-commit, pylint) at the root:
+
+```bash
+uv sync            # or: make install-dev
+```
+
+To work on a specific package, sync it from its own directory, e.g.:
+
+```bash
+cd packages/analysis-runner
+uv sync
+uv run python -m unittest tests/test_analysis_runner.py
+```
+
+Re-resolve every package's lockfile (and regenerate the metamist Cloud Function's
+`requirements.txt`) with:
 
 ```bash
-pip install -r requirements-dev.txt
-pip install --editable .
+make lock
 ```
 
 ### Deployment
 
 The server can be deployed by manually running the [`deploy_server.yaml`](https://github.com/populationgenomics/analysis-runner/actions/workflows/deploy_server.yaml) GitHub action. This will also deploy the driver image.
 
-The CLI tool is shipped as a pip package, this happens automatically on pushes to `main.py`. To build a new version, you should add a [bump2version](https://pypi.org/project/bump2version/) commit to your branch.
+The CLI tool is shipped as a pip package, this happens automatically on pushes to `main`. To build a new version, bump the version in [`packages/analysis-runner/pyproject.toml`](packages/analysis-runner/pyproject.toml) with `uv version` (e.g. `uv version --bump patch` run from that directory) and commit the change.

packages/analysis-runner/pyproject.toml

@@ -0,0 +1,41 @@
+[project]
+name = "analysis-runner"
+# Bumped with `uv version` (e.g. `uv version --bump patch`).
+version = "3.2.6"
+description = "Analysis runner to help make analysis results reproducible"
+readme = "src/analysis_runner/README.md"
+requires-python = ">=3.10"
+license = "MIT"
+keywords = ["bioinformatics"]
+dependencies = [
+    "cloudpathlib[all]",
+    "cpg-utils>=5.0.0",
+    # Avoid dependency resolution backtracking caused by hail pinning an
+    # old version of protobuf and recent versions of grpcio-status requiring
+    # a much newer version
+    "grpcio-status>=1.48,<1.50",
+    "hail>=0.2.134",
+    "requests",
+    "tabulate",
+]
+classifiers = [
+    "Environment :: Console",
+    "Intended Audience :: Science/Research",
+    "Natural Language :: English",
+    "Operating System :: MacOS :: MacOS X",
+    "Operating System :: POSIX",
+    "Operating System :: Unix",
+    "Programming Language :: Python",
+    "Topic :: Scientific/Engineering",
+    "Topic :: Scientific/Engineering :: Bio-Informatics",
+]
+
+[project.urls]
+Homepage = "https://github.com/populationgenomics/analysis-runner"
+
+[project.scripts]
+analysis-runner = "analysis_runner.cli:main_from_args"
+
+[build-system]
+requires = ["uv_build>=0.11,<0.12"]
+build-backend = "uv_build"

packages/analysis-runner/src/analysis_runner/_version.py

@@ -0,0 +1,7 @@
+"""Can be used to access the version from the code"""
+
+from importlib.metadata import version
+
+# The canonical version lives in pyproject.toml ([project].version) and is bumped
+# with `uv version`. Read it back from the installed package metadata.
+__version__ = version('analysis-runner')