Migrate to uv from poetry

.github/workflows/ldn-lint-build-test-push.yml

@@ -23,7 +23,7 @@ on:
       # Limit what triggers the workflow to prevent unnecessary runs on unrelated changes.
       - "ldn/**"
       - "pyproject.toml"
-      - "poetry.lock"
+      - "uv.lock"
       - "Dockerfile"
       - ".github/workflows/ldn-lint-build-test-push.yml"
   push:
@@ -33,7 +33,7 @@ on:
       # Limit what triggers the workflow to prevent unnecessary runs on unrelated changes.
       - "ldn/**"
       - "pyproject.toml"
-      - "poetry.lock"
+      - "uv.lock"
       - "Dockerfile"
       - ".github/workflows/ldn-lint-build-test-push.yml"
   release:
@@ -56,7 +56,7 @@ jobs:
         with:
           src: "ldn"
 
-  build-and-test:
+  build-test-push:
     runs-on: ubuntu-latest
     needs: [lint]
     permissions:
@@ -84,30 +84,44 @@ jobs:
           git fetch --all --tags
           echo "VERSION=$(git describe --tags --always)" >> $GITHUB_OUTPUT
 
-      - name: Build image
+      - name: Build test image
         uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0
         with:
           context: .
+          target: test
           load: true
           tags: ldn-lulc:test
           cache-from: type=gha
           cache-to: type=gha,mode=max
 
       - name: Run tests
-        run: docker run --rm ldn-lulc:test pytest ldn/tests/ -v
+        run: docker run --rm ldn-lulc:test
+
+      - name: Build production image
+        uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0
+        with:
+          context: .
+          target: final
+          load: true
+          tags: ldn-lulc:latest
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Smoketest production image
+        run: docker run --rm ldn-lulc:latest ldn --help
 
       - name: Tag and push
         if: github.ref == 'refs/heads/main'
         run: |
-          docker tag ldn-lulc:test ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.get_version.outputs.VERSION }}
-          docker tag ldn-lulc:test ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+          docker tag ldn-lulc:latest ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.get_version.outputs.VERSION }}
+          docker tag ldn-lulc:latest ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
           docker push ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.get_version.outputs.VERSION }}
           docker push ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
 
       - name: Tag and push release
         if: github.event_name == 'release'
         run: |
-          docker tag ldn-lulc:test ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.event.release.tag_name }}
-          docker tag ldn-lulc:test ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+          docker tag ldn-lulc:latest ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.event.release.tag_name }}
+          docker tag ldn-lulc:latest ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
           docker push ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.event.release.tag_name }}
           docker push ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest

.vscode/settings.json

@@ -4,7 +4,6 @@
     ],
     "python.testing.unittestEnabled": false,
     "python.testing.pytestEnabled": true,
-    "python-envs.defaultEnvManager": "ms-python.python:poetry",
-    "python-envs.defaultPackageManager": "ms-python.python:poetry",
-    "python-envs.pythonProjects": []
+    "python-envs.defaultEnvManager": "ms-python.python:venv",
+    "python-envs.defaultPackageManager": "ms-python.python:pip"
 }

Dockerfile

@@ -1,37 +1,30 @@
-FROM ghcr.io/osgeo/gdal:ubuntu-small-3.12.3
+FROM ghcr.io/osgeo/gdal:ubuntu-small-3.13.1 AS base
 
 RUN apt-get update && apt-get install -y \
-    python3-pip \
-    python3-venv \
-    git \
-    curl \
-    && apt-get clean \
-    && rm -rf /var/lib/{apt,dpkg,cache,log}
+    python3-pip python3-venv git curl build-essential pkg-config \
+    && apt-get clean && rm -rf /var/lib/{apt,dpkg,cache,log}
 
-# Install Rust via rustup (needed to build datacube-compute)
 ENV CARGO_HOME="/usr/local/cargo" RUSTUP_HOME="/usr/local/rustup"
 ENV PATH="$CARGO_HOME/bin:$PATH"
 RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable --profile minimal
 
-RUN pip install --break-system-packages poetry
-
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /usr/local/bin/
+ENV UV_PROJECT_ENVIRONMENT=/code/.venv UV_LINK_MODE=copy
 WORKDIR /code
 
-COPY pyproject.toml poetry.lock ./
-# Install dependencies first to leverage Docker caching.
-# Keep layer separate from installing the package itself to avoid re-building dependencies when our code changes.
-# Make venv in-project (in container's working directory).
-RUN poetry config virtualenvs.in-project true && \
-    poetry install --no-root
-
-# Rust is no longer needed after dependencies are built
+COPY pyproject.toml uv.lock ./
+RUN uv sync --frozen --no-install-project --no-dev
 RUN rustup self uninstall -y && rm -rf /usr/local/cargo /usr/local/rustup
 
 COPY . .
-# Install the package itself. Keep separate from dependencies to avoid re-building dependencies when our code changes.
-RUN poetry install --only-root
-
+RUN uv sync --frozen --no-dev
 ENV PATH="/code/.venv/bin:$PATH"
 
-# Smoketest
+# ---- test stage: adds dev deps on top of the production layer ----
+FROM base AS test
+RUN uv sync --frozen --group dev
+CMD ["uv", "run", "pytest", "ldn/tests/", "-v"]
+
+# ---- final stage = base, i.e. production image with no dev deps ----
+FROM base AS final
 RUN ldn --help

README.md

@@ -21,39 +21,36 @@ rustup-init
 export PATH="$HOME/.cargo/bin:$PATH"
 ```
 
+3. Install uv if you don't have it already:
 ```bash
-poetry lock --no-update
-poetry export -f requirements.txt --output requirements.txt --with dev --with visualisation --without-hashes
-uv pip install -r requirements.txt
-uv run ldn --help
+  brew install uv
 ```
-
-3. Install Poetry if you don't have it already:
+or
 ```bash
-  pip install poetry
+  curl -LsSf https://astral.sh/uv/install.sh | sh
 ```
 
-4. Create and activate a Poetry virtual environment pointing at Homebrew's Python 3.12 and install dependencies:
-This installs main group. Deps like `cogeo-mosaic` and `boto3` are in both main and visualisation group.
+4. Sync dependencies. uv will create a `.venv` automatically using the Python version pinned by `requires-python` in `pyproject.toml` (installing it if needed).
+
+This installs the main dependency group plus the `dev` group (synced by default). Deps like `cogeo-mosaic` and `boto3` are in both the main group and the `visualisation` group.
 ```bash
-  poetry env use $(brew --prefix python@3.12)/bin/python3.12
-  poetry install
+  uv sync
 ```
 
-  For development dependencies:
+  For main dependencies only (no dev tools):
 ```bash
-   poetry install --with dev
+   uv sync --no-dev
 ```
 
-  For visualisation dependencies:
+  For visualisation dependencies only (no project, no dev group):
 ```bash
-   poetry install --no-root --only visualisation
+   uv sync --only-group visualisation
 ```
 
 5. Run the CLI tool:
 ```bash
-   poetry run ldn --help
-   poetry run make {command from Makefile}
+   uv run ldn --help
+   uv run make {command from Makefile}
 ```
 
 ## AWS
@@ -66,15 +63,17 @@ Docs on AWS SSO here: https://github.com/digitalearthpacific/internal-documentat
 
 ### To add a dependency
 
-Run: `poetry add --dev pytest`
+Run: `uv add --group dev pytest`
 
 Others:
-poetry add "dep-tools@git+https://github.com/digitalearthpacific/dep-tools.git"
-poetry add "datacube-compute@git+https://github.com/auspatious/datacube-compute.git"
+```bash
+uv add "dep-tools @ git+https://github.com/digitalearthpacific/dep-tools.git"
+uv add "datacube-compute @ git+https://github.com/auspatious/datacube-compute.git"
+```
 
 ### To run tests
 
-Simply run: `poetry run pytest` or for a specific file: `poetry run pytest ldn/tests/test_mosaic.py`
+Simply run: `uv run pytest` or for a specific file: `uv run pytest ldn/tests/test_mosaic.py`
 
 
 ### Pre-commit hooks
@@ -84,18 +83,18 @@ Formats Python, YAML, and JSON.
 To use pre-commit to automatically run ruff, and other checks on each commit, make sure the development dependencies are installed and then run:
 
 ```bash
-poetry run pre-commit install
+uv run pre-commit install
 ```
 
-Note that you will need to run `poetry run pre-commit run --all-files` if any of the hooks in `.pre-commit-config.yaml` change.
+Note that you will need to run `uv run pre-commit run --all-files` if any of the hooks in `.pre-commit-config.yaml` change.
 
 
 ## Running Commands
 
 You can run these:
-- `poetry run ldn --help`
-- `poetry run ldn version`
-- `poetry run ldn grid list-countries` or `make grid-list-countries`
+- `uv run ldn --help`
+- `uv run ldn version`
+- `uv run ldn grid list-countries` or `make grid-list-countries`
 
 Future commands could look like:
 - Get a class: `ldn grid <class_name>` e.g. forest or grassland
@@ -140,16 +139,16 @@ A tile server for viewing GeoMedian/GeoMAD and predicted LULC mosaics, built wit
 ### Run locally
 
 ```bash
-poetry install --with visualisation
-poetry run uvicorn visualisation.app:app --host 0.0.0.0 --port 8081 --reload
+uv sync --group visualisation
+uv run uvicorn visualisation.app:app --host 0.0.0.0 --port 8081 --reload
 ```
 
 ### Deploy
 
 From the project root:
 ```bash
-poetry install --with visualisation # Needed for ldn make-mosaics command.
-poetry run bash visualisation/deploy.sh
+uv sync --group visualisation # Needed for ldn make-mosaics command.
+uv run bash visualisation/deploy.sh
 ```
 
 This will:

ldn/training_data.py

@@ -57,8 +57,8 @@
     is_bucket_source_coop,
     owner_for_region,
     parse_tile_id,
+    standardise_class,
 )
-from notebooks.src.Compare_LULC_func import standardise_class
 
 logger = logging.getLogger(__name__)
 

ldn/utils.py

@@ -4,9 +4,11 @@
 from typing import Literal
 
 from dep_tools.grids import COUNTRIES_AND_CODES as DEP_COUNTRIES_AND_CODES
+from numpy import nan
 from pystac import ItemCollection
 from rustac import read_sync
 from rustac import store as rustac_store
+from xarray import full_like
 
 from ldn.aws import aws_session, get_credential_provider
 
@@ -337,3 +339,14 @@ def build_prefix(
     if is_bucket_source_coop(bucket):
         prefix = f"{source_coop_prefix(dataset)}/{prefix}"
     return prefix
+
+
+# project the current land cover classes to UNCCD, based on the given mapping directory
+def standardise_class(DataArray, mapping):
+    # Create a copy to preserve original metadata
+    remapped = full_like(DataArray, fill_value=nan)
+
+    for original, new in mapping.items():
+        remapped = remapped.where(DataArray != original, new)
+
+    return remapped

notebooks/Compare_LULC_per_class.ipynb

@@ -24,7 +24,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "id": "b749fc44",
    "metadata": {},
    "outputs": [],
@@ -49,9 +49,10 @@
     "    heterogeneity_func,\n",
     "    load_sankey_params,\n",
     "    standard_legend,\n",
-    "    standardise_class,\n",
     ")\n",
-    "from src.spatial import get_country_boundary, list_countries"
+    "from src.spatial import get_country_boundary, list_countries\n",
+    "\n",
+    "from ldn.utils import standardise_class"
    ]
   },
   {
@@ -2673,7 +2674,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "ldn-TK5rT0MB-py3.12",
+   "display_name": "ldn (3.12.12)",
    "language": "python",
    "name": "python3"
   },

notebooks/Test_LoadOne.ipynb

@@ -792,7 +792,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "ldn-TK5rT0MB-py3.12",
+   "display_name": "ldn (3.12.12)",
    "language": "python",
    "name": "python3"
   },

notebooks/src/Compare_LULC_func.py

@@ -1,6 +1,5 @@
 # package for comparing LULC products
 import numpy as np
-import xarray as xr
 from matplotlib.colors import BoundaryNorm, ListedColormap
 
 from ldn.typology import classes_flipped as standard_legend
@@ -33,17 +32,6 @@ def get_standard_norm():
     return BoundaryNorm(ids + [max(ids) + 1], len(ids))
 
 
-# project the current land cover classes to UNCCD, based on the given mapping directory
-def standardise_class(DataArray, mapping):
-    # Create a copy to preserve original metadata
-    remapped = xr.full_like(DataArray, fill_value=np.nan)
-
-    for original, new in mapping.items():
-        remapped = remapped.where(DataArray != original, new)
-
-    return remapped
-
-
 # Given the source and target data, generate the parameters for sankey diagrams
 def load_sankey_params(s_data, t_data, mask, count_limit=10):
     pairs, counts = np.unique(np.vstack([s_data[mask], t_data[mask]]).T, axis=0, return_counts=True)