Skip to content

alt build #760

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
MattWellie wants to merge 2 commits into from
Closed

alt build #760

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0873dfc
alt build
MattWellie Jul 15, 2026
78a8ed1
fix linting
MattWellie Jul 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions .github/workflows/lint.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,7 @@ jobs:

- name: Install packages
run: |
pip install .
pip install -r requirements-dev.txt
pip install ".[dev,test]"
Comment on lines 20 to +21

@Alexander-Stuckey Alexander-Stuckey Jul 15, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@violetbrina is this where we could use uv sync --extra dev --extra test?

@MattWellie MattWellie Jul 15, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I believe so. The current uv.lock (not committed) looks like

...
[package.optional-dependencies]
dev = [
    { name = "bump2version" },
    { name = "mypy" },
    { name = "pre-commit" },
    { name = "pylint" },
    { name = "ruff" },
    { name = "toml" },
]
test = [
    { name = "pytest" },
    { name = "pytest-mock" },
]
...

and that uv sync command works as expected


- name: pre-commit
run: pre-commit run --all-files
Expand Down
2 changes: 1 addition & 1 deletion analysis_runner/cli.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

import argparse
import sys
from typing import Callable, Sequence
from collections.abc import Callable, Sequence

from analysis_runner._version import __version__
from analysis_runner.cli_analysisrunner import (
Expand Down
2 changes: 1 addition & 1 deletion analysis_runner/cli_cromwell.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -336,7 +336,7 @@ def visualise_cromwell_metadata(

def try_parse_value(value: str | None):
"""Try parse value from command line string"""
if value is None or value == 'None' or value == 'null':
if value in {None, 'None', 'null'}:
return value

if isinstance(value, list):
Expand Down
2 changes: 1 addition & 1 deletion analysis_runner/cromwell.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,6 @@

@deprecated('Use cpg_utils.cromwell.run_cromwell_workflow_from_repo_and_get_outputs')
def run_cromwell_workflow_from_repo_and_get_outputs(*args, **kwargs):
from cpg_utils.cromwell import run_cromwell_workflow_from_repo_and_get_outputs as f
from cpg_utils.cromwell import run_cromwell_workflow_from_repo_and_get_outputs as f # noqa: PLC0415, I001

return f(*args, **kwargs)
6 changes: 3 additions & 3 deletions analysis_runner/dataproc.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,20 +5,20 @@

@deprecated(reason='Use cpg_utils.dataproc.setup_dataproc instead')
def setup_dataproc(*args, **kwargs): # noqa: ANN002, ANN003
from cpg_utils.dataproc import setup_dataproc as _setup_dataproc
from cpg_utils.dataproc import setup_dataproc as _setup_dataproc # noqa: PLC0415

@MattWellie MattWellie Jul 15, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would be very ok with removing all these deprecated methods - we don't use them, so we don't need to soft fail


return _setup_dataproc(*args, **kwargs)


@deprecated(reason='Use cpg_utils.dataproc.hail_dataproc_job instead')
def hail_dataproc_job(*args, **kwargs): # noqa: ANN002, ANN003
from cpg_utils.dataproc import hail_dataproc_job as _hail_dataproc_job
from cpg_utils.dataproc import hail_dataproc_job as _hail_dataproc_job # noqa: PLC0415, I001

return _hail_dataproc_job(*args, **kwargs)


@deprecated(reason='Use cpg_utils.dataproc._add_submit_job instead')
def _add_submit_job(*args, **kwargs): # noqa: ANN002, ANN003, ANN202
from cpg_utils.dataproc import _add_submit_job as __add_submit_job
from cpg_utils.dataproc import _add_submit_job as __add_submit_job # noqa: PLC0415

return __add_submit_job(*args, **kwargs)
2 changes: 1 addition & 1 deletion analysis_runner/util.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ def get_project_id_from_service_account_email(service_account_email: str) -> str
'tob-wgs'
"""
# quick and dirty
return service_account_email.split('@')[-1].split('.')[0]
return service_account_email.rsplit('@', maxsplit=1)[-1].split('.', maxsplit=1)[0]


def add_general_args(parser: argparse.ArgumentParser):
Expand Down
1 change: 0 additions & 1 deletion pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,6 @@ test = [
"pytest-mock>=3.15.1",
]
dev = [
"analysis-runner[test]",
"bump2version",
"mypy~=2.1.0",
"pre-commit",
Expand Down
65 changes: 24 additions & 41 deletions requirements-dev.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,5 @@
#
# This file is autogenerated by pip-compile with Python 3.10
# by the following command:
#
# pip-compile --extra=dev --output-file=requirements-dev.txt pyproject.toml
#
# This file was autogenerated by uv via the following command:
# uv pip compile --extra=dev --extra=test --output-file=requirements-dev.txt pyproject.toml
aiodns==2.0.0
# via hail
aiohappyeyeballs==2.7.1
Expand All @@ -12,14 +8,12 @@ aiohttp==3.14.1
# via hail
aiosignal==1.4.0
# via aiohttp
analysis-runner[test] @ file:///opt/deps
# via analysis-runner (pyproject.toml)
annotated-doc==0.0.4
# via typer
ast-serialize==0.6.0
# via mypy
astroid==3.3.9
# via pylint
async-timeout==5.0.1
# via aiohttp
attrs==26.1.0
# via aiohttp
avro==1.11.5
Expand Down Expand Up @@ -77,19 +71,16 @@ charset-normalizer==3.4.1
# via requests
click==8.4.2
# via typer
cloudpathlib[all,azure,gs,s3]==0.21.0
cloudpathlib==0.21.0
# via
# analysis-runner
# analysis-runner (pyproject.toml)
# cpg-utils
commonmark==0.9.2
# via rich
contourpy==1.3.2
# via bokeh
cpg-utils==5.2.0
# via
# analysis-runner
# analysis-runner (pyproject.toml)
# via analysis-runner (pyproject.toml)
cryptography==44.0.2
# via
# azure-identity
Expand All @@ -108,8 +99,6 @@ dill==0.4.1
# pylint
distlib==0.3.9
# via virtualenv
exceptiongroup==1.3.1
# via pytest
filelock==3.18.0
# via virtualenv
frozendict==2.4.6
Expand All @@ -119,7 +108,7 @@ frozenlist==1.8.0
# aiohttp
# aiosignal
# hail
google-api-core[grpc]==2.24.2
google-api-core==2.24.2
# via
# google-cloud-core
# google-cloud-secret-manager
Expand Down Expand Up @@ -147,7 +136,7 @@ google-crc32c==1.7.0
# google-resumable-media
google-resumable-media==2.7.2
# via google-cloud-storage
googleapis-common-protos[grpc]==1.69.2
googleapis-common-protos==1.69.2
# via
# google-api-core
# grpc-google-iam-v1
Expand All @@ -162,13 +151,10 @@ grpcio==1.71.0
# grpcio-status
grpcio-status==1.49.1
# via
# analysis-runner
# analysis-runner (pyproject.toml)
# google-api-core
hail==0.2.138
# via
# analysis-runner
# analysis-runner (pyproject.toml)
# via analysis-runner (pyproject.toml)
humanize==4.16.0
# via hail
identify==2.6.9
Expand Down Expand Up @@ -196,6 +182,8 @@ jmespath==1.0.1
# botocore
jproperties==2.1.2
# via hail
librt==0.13.0
# via mypy
markupsafe==3.0.3
# via jinja2
mccabe==0.7.0
Expand All @@ -212,6 +200,10 @@ multidict==6.7.1
# via
# aiohttp
# yarl
mypy==2.1.0
# via analysis-runner (pyproject.toml)
mypy-extensions==1.1.0
# via mypy
nest-asyncio==1.6.0
# via hail
nodeenv==1.9.1
Expand All @@ -238,6 +230,8 @@ pandas==2.3.3
# hail
parsimonious==0.11.0
# via hail
pathspec==1.1.1
# via mypy
pillow==12.3.0
# via bokeh
platformdirs==4.3.7
Expand Down Expand Up @@ -282,18 +276,18 @@ pygments==2.20.0
# via
# pytest
# rich
pyjwt[crypto]==2.13.0
pyjwt==2.13.0
# via msal
pylint==3.3.6
# via analysis-runner (pyproject.toml)
pyspark==3.5.8
# via hail
pytest==9.1.1
# via
# analysis-runner
# analysis-runner (pyproject.toml)
# pytest-mock
pytest-mock==3.15.1
# via analysis-runner
# via analysis-runner (pyproject.toml)
python-dateutil==2.9.0.post0
# via
# botocore
Expand All @@ -311,7 +305,6 @@ regex==2026.6.28
# via parsimonious
requests==2.34.2
# via
# analysis-runner
# analysis-runner (pyproject.toml)
# azure-core
# cpg-utils
Expand All @@ -331,12 +324,14 @@ rich==12.6.0
# typer
rsa==4.9
# via google-auth
ruff==0.11.2
ruff==0.15.21
# via analysis-runner (pyproject.toml)
s3transfer==0.11.4
# via boto3
scipy==1.15.3
# via hail
setuptools==81.0.0
# via hail
shellingham==1.5.4
# via typer
six==1.17.0
Expand All @@ -348,7 +343,6 @@ sortedcontainers==2.4.0
# via hail
tabulate==0.9.0
# via
# analysis-runner
# analysis-runner (pyproject.toml)
# cpg-utils
# hail
Expand All @@ -358,10 +352,6 @@ toml==0.10.2
# via
# analysis-runner (pyproject.toml)
# cpg-utils
tomli==2.2.1
# via
# pylint
# pytest
tomlkit==0.13.2
# via pylint
tornado==6.5.7
Expand All @@ -372,16 +362,12 @@ typing-extensions==4.12.2
# via
# aiohttp
# aiosignal
# astroid
# azure-core
# azure-identity
# azure-storage-blob
# azure-storage-file-datalake
# cloudpathlib
# exceptiongroup
# janus
# multidict
# pyjwt
# mypy
tzdata==2026.2
# via pandas
urllib3==2.3.0
Expand All @@ -398,6 +384,3 @@ xyzservices==2026.3.0
# via bokeh
yarl==1.24.2
# via aiohttp

# The following packages are considered to be unsafe in a requirements file:
# setuptools
6 changes: 2 additions & 4 deletions scripts/copy_files_to_release.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
client = storage.Client()


def check_paths_exist(paths: list[str]):
def check_paths_exist(paths: list[str]) -> bool:
"""
Checks a list of gs:// paths to see if they point to an existing blob
Logs the invalid paths if any are found
Expand All @@ -40,9 +40,7 @@ def check_paths_exist(paths: list[str]):
logging.info(f'Invalid path: {path}')
invalid_paths = True

if invalid_paths:
return False
return True
return not invalid_paths


def copy_to_release(project: str, billing_project: str, paths: list[str]):
Expand Down
8 changes: 3 additions & 5 deletions scripts/copy_sample_cram_to_release.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,12 @@
from metamist.models import AnalysisType


def check_paths_exist(paths: list[str]):
def check_paths_exist(paths: list[str]) -> bool:
"""
Checks a list of gs:// paths to see if they point to an existing blob
Logs the invalid paths if any are found
"""
invalid_paths = False
invalid_paths: bool = False
for path in paths:
# gsutil ls <path> returns '<path>\n' if path exists
result = subprocess.run(
Expand All @@ -38,9 +38,7 @@ def check_paths_exist(paths: list[str]):
logging.info(f'Invalid path: {path}')
invalid_paths = True

if invalid_paths:
return False
return True
return not invalid_paths


def copy_to_release(project: str, billing_project: str, paths: list[str]):
Expand Down
2 changes: 1 addition & 1 deletion scripts/cromwell_status_parser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ def parse_subworkflow_status_and_outputs(
"""
status: dict = {}
outputs: dict[str, dict] = {}
subworkflow_name = subworkflow.split('.')[-1]
subworkflow_name = subworkflow.rsplit('.', maxsplit=1)[-1]
if not attempts:
status[subworkflow_name] = 'Not Started'
return status, outputs
Expand Down
6 changes: 3 additions & 3 deletions scripts/subset_hail_table.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,9 +151,9 @@ def clean_locus(contig: str, pos: str) -> hl.IntervalExpression | None:
end = 'end'

elif '-' in pos:
assert (
pos.count('-') == 1
), f'Positions must be one value, or a range between two values: {pos}'
assert pos.count('-') == 1, (
f'Positions must be one value, or a range between two values: {pos}'
)
start, end = pos.split('-')
if start != 'start':
assert int(start), f'start value could not be converted to an int: {start}'
Expand Down
12 changes: 6 additions & 6 deletions scripts/subset_matrix_table.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,9 +176,9 @@ def clean_locus(contig: str, pos: str) -> hl.IntervalExpression | None:
end = 'end'

elif '-' in pos:
assert (
pos.count('-') == 1
), f'Positions must be one value, or a range between two values: {pos}'
assert pos.count('-') == 1, (
f'Positions must be one value, or a range between two values: {pos}'
)
start, end = pos.split('-')
if start != 'start':
assert int(start), f'start value could not be converted to an int: {start}'
Expand Down Expand Up @@ -269,9 +269,9 @@ def clean_locus(contig: str, pos: str) -> hl.IntervalExpression | None:
if unknown:
raise ValueError(f'Unknown args, could not parse: {unknown!r}')

assert not (
args.biallelic and args.keep_all_ref
), 'choose one of --biallelic and --keep_all_ref'
assert not (args.biallelic and args.keep_all_ref), (
'choose one of --biallelic and --keep_all_ref'
)

init_batch()
locus_interval = clean_locus(args.chr, args.pos)
Expand Down
1 change: 0 additions & 1 deletion scripts/subset_matrix_table_to_genes.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
#!/usr/bin/env python3
# ruff: noqa: PLR2004

"""
Takes a path to a MatrixTable and a name prefix
Expand Down
2 changes: 1 addition & 1 deletion scripts/unzip_wrapper.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
def get_commit_hash():
return (
subprocess.check_output(
['git', 'describe', '--always'], # noqa: S603
['git', 'describe', '--always'],
)
.strip()
.decode()
Expand Down
Loading
Loading