pyproject.toml

[build-system]
# PEP 639 (`license = "MIT"` as an SPDX expression plus `license-files`) needs setuptools>=77,
# and no setuptools that supports Python 3.8 is anywhere near that -- 75.3.4 is the last one
# published for it. `pip install -e .` builds from source on every leg, so requiring 77 made the
# package uninstallable on a Python this project still declares support for; the three-OS CI
# matrix caught it on its first run (the earlier "BUILD-time only, does not narrow requires-python"
# reasoning was simply wrong). The deprecated `license` table is kept until either 3.8 support is
# dropped or setuptools' 2027-Feb-18 removal date approaches, whichever comes first.
# `wheel` is deliberately not listed: setuptools.build_meta declares it itself via
# get_requires_for_build_wheel(), so an explicit entry is the legacy `setup.py bdist_wheel`
# pattern and only pins an extra package into every isolated build env.
requires = ["setuptools>=61.0"]
build-backend = "setuptools.build_meta"

[project]
name = "pyutilz"
version = "1.1.0"
description = "Modular utilities for dataframes, databases, web scraping, cloud storage, system introspection, parallelism, and a unified async LLM-provider interface (Anthropic, OpenAI, Gemini, and more) — few hard dependencies, install only what you need beyond the core"
authors = [{name = "Anatoly Alexeev"}]
license = {text = "MIT"}
readme = "README.md"
requires-python = ">=3.8"
keywords = [
    "utilities", "pandas", "polars", "numpy", "numba", "data-science", "dataframe",
    "database", "web-scraping", "cloud-storage", "s3", "gcs",
    "llm", "anthropic", "openai", "gemini", "async",
    "gpu", "cuda", "cupy", "parallelism", "system-monitoring",
]
classifiers = [
    "Development Status :: 4 - Beta",
    "Intended Audience :: Developers",
    "Intended Audience :: Science/Research",
    "License :: OSI Approved :: MIT License",
    "Operating System :: OS Independent",
    "Programming Language :: Python :: 3",
    "Programming Language :: Python :: 3 :: Only",
    "Programming Language :: Python :: 3.8",
    "Programming Language :: Python :: 3.9",
    "Programming Language :: Python :: 3.10",
    "Programming Language :: Python :: 3.11",
    "Programming Language :: Python :: 3.12",
    "Programming Language :: Python :: 3.13",
    "Programming Language :: Python :: 3.14",
    "Framework :: AsyncIO",
    "Topic :: Software Development :: Libraries",
    "Topic :: Software Development :: Libraries :: Python Modules",
    "Topic :: Scientific/Engineering",
    "Topic :: Database",
    "Topic :: Internet :: WWW/HTTP",
    "Topic :: System :: Monitoring",
]

dependencies = [
    # core/pythonlib.py -- imported by nearly every other subpackage (filemaker, data/*,
    # database/db, dev/logginglib, text/strings, system/*, performance/kernel_tuning/cache) --
    # unconditionally imports numba (njit-decorated helper), joblib (default constructor args
    # on ObjectsDumper/ObjectsLoader, evaluated at import time), and portalocker (open_safe_shelve).
    # Found 2026-07-21 audit: these were undeclared, breaking a bare `pip install pyutilz` and
    # several extras groups (`[polars]`, `[database]`, `[nlp]`) that transitively hit pythonlib.py
    # without installing `[system]`. None of the three is guarded by try/except like the module's
    # other optional imports (catboost, IPython, nvidia) -- promoting them to real core deps is
    # the honest fix (the alternative, a lazy-import refactor of the library's most widely-shared
    # helper module, has a much larger blast radius for three lightweight, pure-pip-installable
    # packages with no heavy system/CUDA prerequisites of their own).
    # orjson is the one JSON backend for every module (stdlib json is banned in src/, see core/jsonlib.py).
    "orjson>=3",
    "numba>=0.56",
    # numba itself hard-requires numpy at its own import time (not lazily) -- found 2026-07-21
    # audit follow-up: masking numpy alone (leaving numba unmasked, since numba is core) still
    # broke `import pyutilz.core.pythonlib`, because `from numba import njit` transitively fails
    # without numpy. Declaring it directly here (rather than leaving it "core in practice, but
    # only via an undeclared transitive pull through numba") matches this project's own stated
    # preference elsewhere for explicit declaration over relying on transitive luck.
    "numpy>=1.20",
    "joblib>=1.2",
    "portalocker>=2.0",
    # pyutilz.system.system (the subpackage core/serialization.py's `ensure_dir_exists()` and
    # data/polarslib.py's `clean_ram()` both reach into) eagerly imports ALL SIX of its own
    # submodules in its __init__.py (_common, probing, memory, fsutils, misc, sysinfo) --
    # EVERY one of them unconditionally imports psutil at module level, found 2026-07-21 audit.
    # Since that eager-import-everything design means psutil is needed just to import the
    # subpackage AT ALL regardless of which single function a caller wants, and psutil is a
    # small, universally-wheel-available package with no heavy/compiled dependency chain of its
    # own (unlike scipy/Pillow), declaring it core (rather than refactoring six submodules'
    # worth of eager imports into a lazy, per-symbol-aware __getattr__) is the honest fix.
    "psutil>=5.9",
    # Same eager-import-all-six-submodules design as psutil above: system/system/memory.py
    # (one of the six) unconditionally imports pandas at module level too, and
    # system/parallel.py's applyfunc_parallel (return_dataframe=True default path) does the
    # same -- found 2026-08-03, downstream of the psutil fix but missed at the time. Any extras
    # combination that doesn't happen to also pull in a "pandas" extras group (e.g. a bare
    # install, or `[llm]` alone) broke on `import pyutilz.system.system` /
    # `import pyutilz.system.parallel` with ModuleNotFoundError. Promoting to core, not
    # redeclaring in yet another extras group, for the identical reason psutil was.
    "pandas>=1.5",
    # Same eager-import-all-six-submodules design once more: system/system/misc.py (one of the
    # six) unconditionally imports BOTH tqdm and pympler at module level -- found 2026-08-03
    # alongside the pandas fix above, same "any extras combination missing the right group
    # breaks `import pyutilz.system.system`" failure mode (this time via `pyutilz[llm]`, no
    # "system" extra pulled in). Promoting both to core for the identical reason.
    "tqdm>=4.65",
    "pympler>=1.0",
    # system/config.py unconditionally imports tomllib (stdlib only since Python 3.11, PEP 680)
    # with no guard -- found 2026-07-21 audit: broke the declared Python 3.8/3.9/3.10 support
    # floor outright, on every extras combination, not just a packaging gap. `tomli` is the
    # widely-used backport; only installed pre-3.11, where the code now falls back to it.
    "tomli>=2.0 ; python_version < '3.11'",
]

[project.optional-dependencies]
# Data science packages
pandas = [
    "pandas>=1.5",
    # numpy moved to core [project.dependencies] (transitively required by numba, itself core).
    "pyarrow>=10.0",
    # data/pandaslib/_common.py (the shared-imports hub every pandaslib submodule pulls in)
    # unconditionally imports polars too, purely to re-export it for sibling submodules --
    # found 2026-07-21 audit: `pip install pyutilz[pandas]` alone could not import
    # `pyutilz.data.pandaslib`. Declaring it here (rather than a lazy-import refactor of the
    # hub) matches how this project already resolves this bug class elsewhere in this file.
    "polars>=1.0",
    # pandas itself hard-requires python-dateutil at import time (its own real pip dependency) --
    # declared explicitly rather than relying on that transitive pull, matching the project's
    # existing convention for pandas' other hard transitive deps (see numpy's note above).
    "python-dateutil>=2.8",
    # pandas/tqdm/pympler are NOT redeclared here: all three are core [project.dependencies]
    # (see their rationale there), so every install of this extra already has them. Why this
    # group needs them at all is still worth recording: data/pandaslib/_common.py does
    # `from pyutilz.system.system import tqdmu, ensure_dir_exists`, and pyutilz.system.system's
    # own __init__.py eagerly imports ALL SIX of its submodules, so reaching it for any single
    # symbol pulls misc.py's tqdm+pympler and memory.py's pandas. Redeclaring a core dep inside
    # an extra buys nothing and creates up-to-six-place floor drift on the next bump
    # (2026-09-02 packaging audit).
]
polars = [
    # Floor raised to 1.0 (2026-10-15): the polars floor matrix showed that the code calls LazyFrame.collect_schema
    # (added in polars 1.0) and pl.len, and that polars 1.0.0 is the lowest release where the polars-facing tests pass
    # (checked on the full set: 189 passed). Earlier releases on the 0.19 and 0.20 lines raise AttributeError on these calls.
    "polars>=1.0",
    # numpy moved to core [project.dependencies] (transitively required by numba, itself core) --
    # data/polarslib.py unconditionally imports numpy too; modern polars treats numpy as an
    # optional extra of its own, so this would otherwise have been an undeclared dependency
    # of this extras group specifically.
    #
    # data/polarslib.py also does `from pyutilz.system.system import clean_ram`, which -- for
    # the same "eager-imports-all-six-submodules" reason documented on the "pandas" extras
    # group above -- transitively needs misc.py's tqdm+pympler AND memory.py's own pandas
    # (+ pandas' own dateutil requirement), not just psutil (now core). tqdm/pympler/pandas are
    # themselves core dependencies now, so only pandas' own dateutil is declared here.
    "python-dateutil>=2.8",
]

# Combined dataframe stack (self-referential to avoid duplication/drift).
# pandas/polars stay as individual extras too for callers who only need one.
dataframes = [
    "pyutilz[pandas,polars]",
]

# Database packages
database = [
    "sqlalchemy>=1.4",
    # Capped on 3.8: 2.9.9 is the last release with a cp38 wheel, and every later one makes pip
    # build from source, which fails on the Windows runner (no libpq, link.exe exit 1181).
    "psycopg2-binary>=2.9,<2.9.10 ; python_version < '3.9'",
    "psycopg2-binary>=2.9 ; python_version >= '3.9'",
    "pymysql>=1.0",
    # database/deltalakes.py takes its lock from filelock (audit finding A-05): declared here, so the lock works out of the box.
    "filelock>=3.0",
    # database/db/__init__.py imports dateutil.relativedelta unconditionally at module level
    # (database/__init__.py itself is a 4-line re-export shim with no imports of its own --
    # the actual unconditional import lives one directory deeper, in the db/ subpackage);
    # redislib.py imports redis unconditionally at module level. Neither is guarded by
    # try/except (unlike e.g. orjson elsewhere), so they're real requirements of this
    # extra, not correctly-optional add-ons.
    "python-dateutil>=2.8",
    "redis>=4.0",
    # database/db/__init__.py also unconditionally imports pandas (pd.read_sql calls) -- found
    # 2026-07-21 audit, same undeclared-import class as dateutil/redis above; pandas is a core
    # dependency now, so it is not redeclared here.
]

# Web scraping packages
web = [
    "selenium>=4.0",
    "undetected-chromedriver>=3.0",
    "requests>=2.28",
    "grequests>=0.7",
    "fake-useragent>=1.0",
    "curl-cffi>=0.7",
]

# Cloud storage packages
cloud = [
    "google-cloud-storage>=2.0",
    "boto3>=1.26"
]

# dev/dashlib.py imports flask/dash/dash_bootstrap_components unconditionally at module
# level (unlike flask_login a few lines down in the same file, which is try/except-guarded)
# -- a real, currently-undeclared requirement for anyone importing this niche notebook-
# dashboard helper module.
dash = [
    "flask>=2.0",
    "dash>=2.17",  # dcc.Loading(delay_show=, overlay_style=) used by dev/dashlib.py landed in 2.17.0
    "dash-bootstrap-components>=1.0",
]

# system/scheduling/prefect.py imports prefect unconditionally at module level.
prefect = [
    "prefect>=2.0",
    # `pyutilz.system.scheduling.prefect` imports `pyutilz.web.graphql` at module scope, which
    # needs requests -- so `pip install pyutilz[prefect]` alone could not import the module it
    # is named for. Declared here rather than lazy-importing the hub, matching how this file
    # already resolves this bug class for the pandas extra above.
    "requests>=2.0",
]

# system/parallel.py's set_tf_gpu() imports tensorflow.
tensorflow = [
    "tensorflow>=2.0",
]

# NLP packages
nlp = [
    # spacy pulls thinc>=8.3.12, whose own Requires-Python floor is >=3.10 -- without this
    # marker `pip install pyutilz[nlp]` is simply unresolvable on the 3.8/3.9 interpreters this
    # package still declares support for (requires-python = ">=3.8"), and CI had to hand-drop
    # [nlp] from those matrix legs. Expressing the constraint in metadata (matching the
    # black/google-genai markers elsewhere in this file) makes the extra installable everywhere:
    # every spacy import site in the codebase is already try/except-guarded (text/tokenizers.py),
    # so 3.8/3.9 users get the rest of the group and a clean skip of the spacy-backed paths.
    "spacy>=3.0 ; python_version >= '3.10'",
    "nltk>=3.8",
    "jellyfish>=0.9",
    "tiktoken>=0.5",
    # text/strings/webtext.py lazily imports these three (function-local, on first call --
    # corrected 2026-07-21 audit, this comment previously and incorrectly said "unconditionally
    # ... at module level", which the actual code does not do: each is guarded by a
    # `global X; if X is None: import Y` cache pattern, same shape as this file's own `nlp`
    # spaCy-model lazy-load). Being lazy doesn't make them optional, though: there is no
    # try/except fallback (unlike orjson elsewhere in the codebase) -- calling
    # get_ascii_emojies/get_unicode_emojies/suffixize without the package installed raises
    # ModuleNotFoundError at call time -- so they must still be real declared dependencies of
    # this extra rather than relying on some other extra's transitive pull. deptry DEP003
    # flagged all three as used-but-undeclared regardless of the eager-vs-lazy distinction.
    "beautifulsoup4>=4.0",
    "inflect>=6.0",
    "emoji-data-python>=1.6",
    # text/strings/__init__.py, webtext.py, basics.py, and textentropy.py all unconditionally
    # import pandas (numpy moved to core [project.dependencies], transitively required by numba)
    # -- found 2026-07-21 audit: no dedicated "text"/"strings" extras group exists, and this is
    # the closest-matching domain extra for the text/strings subpackage. pandas is core now, so
    # only its own hard transitive dependency python-dateutil is declared here, for the same
    # reason noted on the "pandas" extras group above.
    "python-dateutil>=2.8",
]

# LLM providers
llm = [
    "anthropic>=0.40",
    # The Gemini provider sends ThinkingConfig.thinking_level, which google-genai added in 1.51.0 (older
    # versions reject the field as an extra input); 1.51 declares Requires-Python >=3.10. An unmarked entry
    # would make `pip install pyutilz[llm]` fail on older interpreters, so the marker follows the library.
    # The gemini-specific tests guard with pytest.importorskip("google.genai"), and factory.py imports the
    # provider lazily, so a 3.8/3.9 install just loses the Gemini provider rather than the group.
    "google-genai>=1.51 ; python_version >= '3.10'",
    "httpx>=0.25",
    "tenacity>=8.0",
    "pydantic>=2.0",
    "pydantic-settings>=2.0",
    "tiktoken>=0.5",
]

# System utilities
system = [
    # numba, joblib, portalocker, psutil, tqdm, pympler and pandas all moved to the core
    # [project.dependencies] list above -- required unconditionally by core/pythonlib.py /
    # pyutilz.system.system (whose __init__.py eagerly imports all six submodules, including
    # misc.py's tqdm+pympler and memory.py's pandas), not specific to this "system" domain, and
    # deliberately not redeclared here (2026-09-02 packaging audit: a core dep re-listed in an
    # extra is pure floor-drift surface -- the extra can never be installed without it).
    # python-dateutil is pandas' own hard transitive dependency, declared explicitly for the
    # same reason noted on the "pandas" extras group above (numpy itself is core, see above).
    "python-dateutil>=2.8",
    # core/image.py and core/matrix.py import Pillow/scipy unconditionally at module level;
    # neither maps to an existing domain extra, so they'd otherwise be genuinely undeclared
    # requirements for anyone importing those two core submodules directly.
    "Pillow>=9.0",
    "scipy>=1.9",
    # Hardware detection (migrated from ml_perf_test)
    "py-cpuinfo>=9.0",
    "GPUtil>=1.4",
    "xmltodict>=0.13",
    # text/similarity.py unconditionally imports jellyfish at module level (found by
    # code_audit's undeclared_import scanner, 2026-07-21 audit follow-up); jellyfish is
    # otherwise only declared under the "nlp" extras group.
    "jellyfish>=0.9",
]

# Statistics (normality tests). pyutilz.stats is a standalone, mypy-strict-mode subpackage (see
# [[tool.mypy.overrides]] below) with its own minimal numpy-only requirement, distinct from the
# heavier pandas/system stacks. Found 2026-07-21 audit: stats/normality.py unconditionally
# imports numpy with no matching extras group of its own.
# Its sole member, numpy, is a core [project.dependencies] entry now, so this group is an empty,
# documented alias: `pip install pyutilz[stats]` keeps working (and keeps the subpackage
# discoverable) without a second copy of a floor that would then have to be bumped twice.
stats = []

# GPU acceleration (cupy). Every cupy import site in the codebase is already
# try/except-guarded (kernel_tuning device probing, gpu_dispatch.py) -- this extra
# exists purely for discoverability (`pip install pyutilz[gpu]`) rather than to make
# cupy required. Deliberately excluded from `all` (mirrors mlframe's [gpu] extra):
# cupy's cudaXX-specific wheel resolution doesn't belong in a default "everything" install.
# Installability caveat (2026-09-02 audit): the `cupy` project on PyPI publishes an SDIST ONLY --
# the prebuilt binaries live under the CUDA-version-specific names (cupy-cuda12x, cupy-cuda11x,
# cupy-rocm-*). `pip install pyutilz[gpu]` therefore triggers a full local NVCC source build,
# needing a complete CUDA toolkit rather than just a driver. Callers who want a wheel should
# install the matching binary package themselves (`pip install cupy-cuda12x`) and skip this extra
# entirely; nothing breaks without it, since every cupy import site in the codebase is
# try/except-guarded.
gpu = [
    "cupy>=12.0",
]

# All optional dependencies (self-referential to avoid duplication/drift).
#
# Deliberately EXCLUDED, and this comment is the single source of truth for why (previously it was
# recorded only in .github/workflows/mypy-full.yml): `gpu` (source-only cupy, see its own note
# above), `dash` (flask + dash + dash-bootstrap-components -- a whole web-server stack for one
# niche notebook-dashboard helper, dev/dashlib.py), `prefect` (a full workflow-orchestration
# runtime with a large transitive tree, for system/scheduling/prefect.py) and `tensorflow` (a
# multi-hundred-MB ML framework used by exactly one function, system/parallel.py's set_tf_gpu()).
# Drop-in accelerators: nothing imports these unconditionally, every use site falls back to the
# stdlib when they are absent, so this group only ever makes an existing code path faster.
# orjson is now a core dependency (see [project.dependencies]); the speedups group is kept empty for install compatibility.
speedups = []

# Consequence to keep in mind: `pip install pyutilz[all]` canNOT import pyutilz.dev.dashlib or
# pyutilz.system.scheduling.prefect -- add the matching extra explicitly, e.g.
# `pip install "pyutilz[all,dash]"`.
all = [
    "pyutilz[pandas,polars,database,web,cloud,nlp,llm,system,stats,speedups]",
]

# Development dependencies
dev = [
    # dev/benchmarking.py unconditionally imports numpy -- found 2026-07-21 audit: no extras
    # group covering the dev domain declared it. numpy is a core dependency now (numba needs it
    # at import time), so `pip install .[dev]` alone gets it without a redundant redeclaration.
    # dev/signature_models.py builds pydantic models from signatures and imports pydantic at module level.
    "pydantic>=2.0",
    "pytest>=7.0",
    "pytest-cov>=4.0",
    "pytest-benchmark>=4.0",
    "pytest-asyncio>=0.21",
    "pytest-instafail>=0.5",
    "pytest-progress>=1.3",
    "pytest-timeout>=2.1",
    # tests/test_similarity.py importorskips hypothesis; undeclared, it skipped the whole similarity suite on the 3.8 leg
    # and left its coverage (~0.4 points of the 85% gate) out. pip picks the newest hypothesis each interpreter accepts.
    "hypothesis>=6.0",
    # ci.yml runs the suite with `-n auto --dist=loadgroup` (4 vCPU runners); loadgroup honours any
    # @pytest.mark.xdist_group pin. pytest-cov merges worker data itself.
    "pytest-xdist>=3.5",
    # pytest-randomly randomises test ORDER each run, which is what makes order-dependence
    # (shared module state such as pyutilz.llm.factory._provider_cache, or a sys.modules stub
    # leaking between files) actually surface instead of hiding behind a fixed alphabetical
    # order. tests/conftest.py ships a thinc seed-overflow compat shim that only has a reason
    # to exist when the plugin is installed. It lived in requirements-dev.txt alone until the
    # 2026-09-03 packaging audit found that NO workflow installs that file, so the
    # randomisation the suite is designed around never ran on any CI leg; unlike py-ci-shared
    # it is an ordinary PyPI package with no direct reference, so it belongs here, where every
    # workflow's `[dev]` install picks it up.
    "pytest-randomly>=3.12",
    # black==26.10.0 requires Python >=3.10 (found 2026-07-19: installing dev-extras on the py3.8/
    # py3.9 CI matrix cells hard-failed the whole `pip install -e ".[dev]"` step, since pip can't
    # satisfy an unconditional exact pin against an interpreter the release doesn't support). The
    # marker keeps `pip install .[dev]` installable there -- those cells don't need black to run
    # pytest, matching the identical py-ci-shared marker below.
    "black==26.10.1 ; python_version >= '3.10'",   # pinned exact, matching CI's uvx black==26.10.1 (black-filtered.yml)
    "ruff==0.16.10",                 # pinned exact, matching RUFF_VERSION in py-ci-shared's tool_versions.py, which its ruff-blocking.yml runs --
                                    # an open range here is exactly how mlframe's prior v0.8.6 pre-commit-
                                    # vs-CI drift went unnoticed for a long time (2026-07-18 architecture review).
    "mypy==2.4.0 ; python_version >= '3.10'",  # no mypy 2.x is published for 3.8/3.9, and the gate runs on 3.11; pinned exact, matching CI's mypy-full.yml `mypy-version` and .pre-commit-config.yaml's
                                    # mirrors-mypy `rev: v2.3.1`. Was an open `mypy>=1.0`, which spanned a MAJOR version
                                    # boundary (1.x vs the 2.x this config is written against, see [tool.mypy] below) on the
                                    # one whole-project blocking gate -- the same drift class the ruff pin above documents.
    "bandit>=1.7",
    # sqlglot parses the SQL the query builders emit (tests/test_meta/test_generated_sql_property.py),
    # so the "no UPDATE without a WHERE" invariant is asserted against a real parse tree rather than a
    # substring search -- the builders emit multi-CTE, multi-statement Postgres where a regex over the
    # text cannot tell which WHERE belongs to which statement. requires-python >=3.9, matching the
    # markers above; the test importorskips it, so the py3.8 CI cell skips rather than errors.
    "sqlglot>=25 ; python_version >= '3.9'",
    # py-ci-shared (needed by .pre-commit-config.yaml's local hooks) is deliberately NOT declared
    # here: PyPI rejects any distribution whose metadata carries a PEP 440 direct reference
    # (`400 ... Can't have direct dependency`), and `twine check` does not catch it -- so a
    # `git+https://` requirement anywhere in [project] (even in a dev-only extra, since extras
    # still land in the wheel's Requires-Dist) makes the whole package unpublishable, failing only
    # at the final upload step of publish.yml. It lives in requirements-dev.txt instead; install
    # contributor tooling with `pip install -e ".[dev]" -r requirements-dev.txt`. publish.yml has
    # a metadata gate that fails the release build if a direct reference ever reappears here.
]

# Documentation site (MkDocs Material). Build with `mkdocs build` / serve with `mkdocs serve`.
docs = [
    "mkdocs-material>=9.5",
]

[project.urls]
Homepage = "https://github.com/fingoldo/pyutilz"
Documentation = "https://fingoldo.github.io/pyutilz/"
Repository = "https://github.com/fingoldo/pyutilz.git"
Issues = "https://github.com/fingoldo/pyutilz/issues"
Changelog = "https://github.com/fingoldo/pyutilz/blob/master/CHANGELOG.md"

[tool.setuptools.packages.find]
where = ["src"]
include = ["pyutilz*"]
exclude = ["tests*", "scripts*", "docs*"]

[tool.setuptools.package-data]
pyutilz = ["py.typed"]

# Pytest configuration
[tool.pytest.ini_options]
testpaths = ["tests"]
pythonpath = ["src"]
python_files = ["test_*.py"]
python_classes = ["Test*"]
python_functions = ["test_*"]
addopts = "-v --tb=short --strict-markers -ra --timeout=120"
markers = [
    "slow: marks tests as slow (deselect with '-m \"not slow\"')",
    "xdist_group: keeps the marked tests on one xdist worker, so the code-audit scan memo they share is hit",
    "gpu: marks tests requiring GPU (deselected in default CI; no GPU runner)",
    "live: marks tests requiring live provider API keys (skipped by default; pass --run-live to execute)"
]

# Coverage configuration
[tool.coverage.run]
source = ["src/pyutilz"]
omit = [
    "tests/*",
    "*/scheduling/*",
    "*/cloud/*",
    "*/web/browser.py",
    "*/dev/dashlib.py",
    "*/dev/notebook_init.py",
    "*/text/tokenizers.py",
]

[tool.coverage.report]
exclude_lines = [
    "pragma: no cover",
    "def __repr__",
    "def __str__",
    "raise AssertionError",
    "raise NotImplementedError",
    "if __name__ == .__main__.:",
    "if TYPE_CHECKING:",
    "except:",
    "if 0:",
    "if False:"
]
precision = 2
show_missing = true

# Coverage ratchet. Measured 87.11% on 2026-09-03 over the whole suite (Windows dev box,
# `python -m coverage run -m pytest --no-cov -m "not gpu"`), up from 84.87% the day before as two
# audit waves added regression tests. Set a couple of points under that: the matrix now runs three
# OSes, and each leaves the other platforms' probes in system/ uncovered, so the floor has to hold
# on the least-covered one rather than on this box. RAISE this when coverage rises; never lower it
# to make a run pass. ci.yml passes its own --cov-fail-under, and a meta-test keeps the two equal.
fail_under = 85

# Black code formatter
[tool.black]
line-length = 160
target-version = ['py38', 'py39', 'py310', 'py311', 'py312', 'py313', 'py314']
exclude = '''
/(
    \.git
  | \.venv
  | \.pytest_cache
  | build
  | dist
  | __pycache__
)/
'''

# Ruff linter. Base select/ignore (E,W,F,I,N,UP,B,DTZ,A,T20,RUF,G,ISC,PERF,C90 + the generic
# ignores) is pulled in from $PY_CI_SHARED_DIR/configs/ruff-base.toml via `extend` -- a real merge,
# not copy-paste; ruff expands env vars in `extend` paths (docs.astral.sh/ruff/settings), so
# PY_CI_SHARED_DIR just needs to point at ANY real py-ci-shared checkout, not a sibling directory
# specifically. See that repo's README "Using the shared ruff config" section for the full
# rationale. Everything below is pyutilz-specific: its own dir excludes, target-version floor
# (py38, older than mlframe's py39), and its own MEASURED mccabe threshold (complexity
# distributions differ per codebase; never copy another repo's number -- see the comment below).
[tool.ruff]
extend = "$PY_CI_SHARED_DIR/configs/ruff-base.toml"
target-version = "py38"
# _benchmarks added 2026-07-09 while wiring the blocking gate: its scripts print timing/identity
# results to stdout by design (that IS the benchmark's interface, same exemption class as
# py-ci-shared's own CLI scripts) -- T201/PERF401 findings there are noise, not bugs. Mirrors
# mlframe's benchmarks/_benchmarks/profiling exclusion (found via the identical gap there).
# `tests` and `scripts` removed from this list 2026-09-02: both were entirely unlinted, hiding 307
# and 30 findings respectively (F401 redefinitions, a blind `pytest.raises(Exception)`, a `==`
# type comparison, and 19 `# noqa` directives naming rules that were never enabled, i.e. dead
# suppressions their authors believed were load-bearing). They are now in scope, with the
# genuinely test-/script-idiomatic codes exempted per-file below rather than by hiding the dirs.
# `_benchmarks` removed from this list 2026-09-03 (audit 06/F09): it was the last directory
# with zero static analysis. Measured at removal time: 76 findings, ALL in codes already
# exempted per-file for tests/scripts as genuinely idiomatic (66 T201, 8 NPY002, 1 PERF203,
# 1 PERF401) -- `F`, `B` and `E9` were clean, so this is a zero-finding change that closes the
# structural gap: a directory nobody lints will never report the F821 it eventually acquires.
exclude = [".git", "__pycache__", "build", "dist"]

# B006 (mutable default argument) is in the shared base's `ignore` list ("audited case-by-case per
# call site"). Measured 2026-09-02: `ruff check src tests scripts --select B006 --isolated` reports
# `All checks passed!`, i.e. the exposure is zero everywhere, so honoring the rule costs nothing
# today and the case-by-case audit can be re-litigated per finding if one ever appears. The repo
# already treats this as a real bug class (pyutilz.dev.code_audit ships a mutable_defaults scanner
# for it), but that scanner is wired into no blocking hook -- this puts the class under a gate.
[tool.ruff.lint]
# RET503 (implicit `return None` on a path that can also return a value) added 2026-09-03
# (audit 06/F03,F04,F08): nine functions fell off the end returning None where their own
# docstring promised a value, one of them (`get_table_fields`) feeding string concatenation
# into SQL. All are fixed, so the class is at zero and this keeps it there. Only RET503 --
# the rest of the RET5xx family (RET501/502/504) is stylistic and stays out.
# PLE2510/2512/2513/2514/2515 (invalid unescaped control / invisible characters) added at the
# same time: RUF001-003 are in the shared base's `ignore` list for good reason (this codebase
# has intentional non-ASCII prose), and that exemption also covered genuinely INVISIBLE
# characters, which are never legitimate -- a U+200B inside a test docstring went unseen by
# every gate. These five are the false-positive-free subset of that problem.
extend-select = ["B006", "RET503", "PLE2510", "PLE2512", "PLE2513", "PLE2514", "PLE2515"]

# Measured 2026-07-09 when C90 was newly added via the shared base (pyutilz had never selected
# it before): default threshold=10 flags 68 functions, median complexity among those 15, mean
# 18.8, max 61. threshold=20 keeps 17 findings -- the genuine outliers (complexity 20-61) --
# without flagging routine 10-20 dispatch/branching functions. Lower than mlframe's 40 because
# pyutilz's overall complexity distribution is genuinely lower; this is pyutilz's own measured
# number, not copied from mlframe.
[tool.ruff.lint.mccabe]
# Owner decision 2026-10-12: 10. Two njit kernels are exempt by per-file-ignores (splitting measured slower / needs njit helpers).
max-complexity = 10

[tool.ruff.lint.per-file-ignores]
"__init__.py" = ["F401"]  # unused imports OK in __init__
"src/pyutilz/__init__.py" = ["N807"]  # module-level __getattr__ (PEP 562) is a real dunder, not a naming violation
"src/pyutilz/stats/normality.py" = ["C901"]  # @njit kernel; splitting it needs new njit helpers
"src/pyutilz/text/similarity/_numba_kernels.py" = ["C901"]  # njit kernels; a split measured ~1.7x slower on 300-word inputs
"src/pyutilz/llm/claude_code_provider.py" = ["N814"]  # local import aliases (_TRS, _TSS) follow the project's underscore-prefixed-short-alias convention, not a "constant" naming
"src/pyutilz/dev/notebook_init.py" = ["T201"]  # entire module's job is printing interactive Jupyter-notebook-init status messages to stdout for the user to see; already excluded from coverage as a notebook-only utility
"src/pyutilz/web/proxy/decodo.py" = ["T201"]  # all findings are in DecodoProvider.print_usage(), explicitly documented as "Print human-readable subscription + traffic summary to stdout"

# tests/ entered ruff's scope 2026-09-02 (see the `exclude` note above). Every code below is
# exempted for a stated reason; anything NOT listed here is enforced on tests exactly as on src.
"tests/**" = [
    "C901",     # test bodies are flat fixtures and case tables; splitting them into helpers adds indirection, not clarity
    "T201",     # tests print diagnostics for a failing run to be readable in CI logs; stdout is not a test's product but it is a legitimate debugging channel
    "RUF012",   # test helper/fixture classes hold mutable class attributes on purpose; ClassVar[...] on throwaway stubs is annotation noise, not safety
    "RUF059",   # an unused unpacked variable documents the shape of the tuple under test (`_, count = f()` style); renaming every one to `_` loses that
    "PERF401",  # an explicit append loop in a test is deliberately readable step-by-step; a comprehension rewrite optimises the thing least worth optimising
    "PERF203",  # try/except inside a loop IS the mechanism under test in the retry/backoff suites
    "DTZ001", "DTZ003", "DTZ011",  # naive datetimes / utcnow() / date.today() are the FIXTURE VALUES these tests feed in; forcing tz-aware ones changes what is being asserted
    "NPY002",   # legacy global-seed RNG (np.random.seed/rand/choice) is deliberate: these tests assert against values generated from the legacy global stream, and pytest-randomly reseeds it per test, so the state is not actually order-dependent here
    "B008",     # function calls in argument defaults are the exact signature shape the introspection tests exist to inspect
    "B018",     # a bare attribute access inside `with pytest.raises(...)` IS the assertion
    "RUF005",   # list concatenation in fixture construction; the iterable-unpacking rewrite is not clearer
    "UP028",    # `yield` in a for loop, kept explicit in generator-protocol tests
    "RUF017",   # quadratic list summation over fixture-sized (single-digit) lists
]

# _benchmarks/ entered ruff's scope 2026-09-03 (audit 06/F09), same treatment as tests/ and
# scripts/ before it: the directory is linted, and only the codes that are genuinely idiomatic
# for a benchmark script are exempted, one line per code.
"_benchmarks/**" = [
    "T201",     # printing the timing/identity result to stdout IS a benchmark's interface, the same exemption class scripts/ gets
    "NPY002",   # legacy global-seed RNG is deliberate: a benchmark reproduces a specific input stream, and np.random.seed is how these scripts pin it
    "PERF203",  # try/except inside the timing loop keeps one failing size from aborting the whole sweep
    "PERF401",  # an explicit append loop inside a benchmark is deliberately not the thing being optimised
]

# scripts/ entered ruff's scope 2026-09-02 alongside tests/.
"scripts/vulture_whitelist.py" = ["F821", "B018"]  # the file's ENTIRE content is bare name references, so vulture counts those names as used -- undefined names and useless expressions are the file working exactly as designed
"scripts/auto_refactor.py" = ["T201"]  # a CLI script whose output to stdout is its user interface
"scripts/prove_meta_checks.py" = ["T201"]  # a CLI script whose CAUGHT/MISSED report to stdout is its user interface

# Deptry: unused / missing / transitive dependency scan. Made blocking 2026-07-09 after a real
# triage pass (run with the [all,dev] extras actually installed -- deptry's DEP001/DEP002 are
# both meaningless in a bare environment, see py-ci-shared/README). 70 raw findings reduced to 0
# by [tool.deptry.package_module_name_map] (PyPI-name vs. import-name mismatches deptry can't
# infer on its own) and the per_rule_ignores below (genuinely optional third-party libs behind
# try/except ImportError guards, verified individually -- never a hard runtime requirement).
[tool.deptry]
# .claude added 2026-07-22: nested Claude Code agent worktrees live under .claude/worktrees/<name>/
# (each a full checkout with its own tests/, src/, etc.) -- deptry's file-tree scan isn't git-aware
# and was reading their duplicate source trees as part of THIS project, misattributing imports from
# worktree-local branches (transitive-dep, since-removed, or not-yet-merged) to the real project.
exclude = ["tests", "scripts", ".venv", "build", "dist", ".claude"]

[tool.deptry.package_module_name_map]
"psycopg2-binary" = "psycopg2"
beautifulsoup4 = "bs4"
"google-genai" = "google"
Pillow = "PIL"
"py-cpuinfo" = "cpuinfo"
"python-dateutil" = "dateutil"
GPUtil = "GPUtil"

[tool.deptry.per_rule_ignores]
# selenium/undetected-chromedriver/boto3/google-cloud-storage: real runtime deps, but
# imported lazily inside deeply-nested functions (see cloud/cloud.py, web/browser.py,
# performance/kernel_tuning/remote.py) -- deptry's AST usage scan misses some of these.
# pytest*/black/ruff/mypy/bandit/mkdocs-material: dev-tool CLI packages, invoked as
# commands (pytest, black, ruff, mypy, bandit, mkdocs) -- never `import`-ed by name, so
# deptry's static-usage scan can never see them as "used" no matter how the code changes.
# fake-useragent: referenced only via the dynamic `ensure_installed("... fake_useragent")`
# runtime-install helper (web/web.py) and a commented-out future `from fake_useragent
# import UserAgent`, not a static `import` deptry's scanner can detect.
DEP002 = [
    "sqlglot",  # parses the query builders' output in tests/test_meta/test_generated_sql_property.py; deptry scans src/ only, so a test-only dep can never look used
    "hypothesis",  # property-based tests in tests/test_similarity.py import it through importorskip; deptry scans src/ only
    "selenium",
    "undetected-chromedriver",
    "boto3",
    "google-cloud-storage",
    "pytest",
    "pytest-cov",
    "pytest-benchmark",
    "pytest-asyncio",
    "pytest-instafail",
    "pytest-progress",
    "pytest-timeout",
    "pytest-randomly",  # a pytest PLUGIN: loaded by pytest through its entry point, never `import`-ed by the suite
    "pytest-xdist",  # a pytest PLUGIN (-n N), loaded through its entry point, never `import`-ed
    "black",
    "ruff",
    "mypy",
    "bandit",
    "mkdocs-material",
    "fake-useragent",
    "py-ci-shared",  # invoked as `python -m py_ci_shared.*` CLI/pre-commit-hook commands, never `import py_ci_shared`
]

# DEP001 (import missing from declared deps): 35 findings from the 2026-07-09 full triage (run
# with [all,dev] actually installed), each individually verified to be a genuinely optional
# third-party import behind a try/except ImportError guard -- never a hard runtime requirement.
# One real bug found and fixed along the way, not ignored: core/pythonlib.py's load_file() had
# an UNGUARDED module-level `from catboost import CatBoostClassifier`, making catboost a hard
# dependency for callers loading .joblib/.pckl files too; moved to a lazy import inside the
# .bin-only branch that actually needs it (still unguarded there, since that branch always did
# require catboost -- the bug was the import's SCOPE, not the absence of a try/except).
DEP001 = [
    "catboost",  # core/pythonlib.py load_file() -- lazy-imported only for the .bin/CatBoost branch
    "IPython",  # core/pythonlib.py is_jupyter_notebook(), data/pandaslib/_common.py, dev/notebook_init.py -- notebook-only
    "nvidia",  # core/pythonlib.py -- CUDA_HOME discovery from the nvidia pip package, guarded
    "polars_ds",  # data/polarslib.py -- gated on check_cpu_flag("avx2")
    "filelock",  # database/deltalakes.py -- guarded, FileLock/Timeout set to None on ImportError
    "flask_login",  # dev/dashlib.py -- guarded, Dash-only feature
    "concurrent_log_handler",  # dev/logginglib.py -- guarded, falls back to stdlib RotatingFileHandler
    "matplotlib",  # dev/notebook_init.py -- guarded, notebook-only
    "seaborn",  # dev/notebook_init.py -- guarded, notebook-only
    "claude_code_sdk",  # llm/claude_code_provider.py -- guarded SDK monkey-patch block
    "py_ci_shared",  # dev/code_audit aliases and the deferred-entry counter import it; a development tool from requirements-dev.txt, absent on the 3.8 legs
    "anyio",  # llm/claude_code_provider.py -- inside the same guarded claude_code_sdk block
    "botocore",  # performance/kernel_tuning/remote.py -- guarded, S3 remote-cache is opt-in
    "wmi",  # system/system/probing.py -- guarded, Windows-only
    "pgvector",  # database/psycopg2_pool.py -- guarded, only used if the caller's DB has the pgvector extension
]

# DEP003 (import used but declared as transitive, not direct): same soft-import names as DEP001
# above, minus concurrent_log_handler/claude_code_sdk/wmi (which stayed genuinely absent in this
# environment). Whether deptry reports DEP001 or DEP003 for a given name depends on whether
# something else's dependency resolution happens to pull it in transitively -- not something
# this repo controls or wants to depend on staying stable, so both codes are ignored
# for the same verified-optional names rather than re-triaging every time a transitive pin shifts.
DEP003 = [
    "catboost", "IPython", "nvidia", "polars_ds", "filelock", "matplotlib", "seaborn", "anyio", "botocore",
    "flask_login",  # dev/dashlib.py -- guarded, Dash-only feature; now pulled in transitively in some envs (previously genuinely absent)
    "pgvector",  # database/psycopg2_pool.py -- guarded, only used if the caller's DB has the pgvector extension
    "claude_code_sdk",  # llm/claude_code_provider.py -- guarded SDK monkey-patch block; present in dev envs with the CLI SDK installed
    "wmi",  # system/system/probing.py -- guarded, Windows-only; present on Windows dev boxes, absent in Linux CI
    "py_ci_shared",  # dev/code_audit wrappers import it lazily; a development tool installed from requirements-dev.txt, never a runtime dependency
]

# Interrogate: docstring coverage. Blocking at --fail-under=100 since 2026-07-09 (see the CI
# lint-blocking job and the .pre-commit-config.yaml interrogate-blocking hook) -- pyutilz was
# already at 100% coverage (917/917) before this config was wired up to gate on it. `_benchmarks`
# needs no explicit exclude here (unlike mlframe's) since it lives at the repo root, outside the
# `src/pyutilz` path this scan is scoped to.
[tool.interrogate]
ignore-init-method = true
ignore-init-module = true
ignore-magic = true
ignore-private = true
exclude = ["tests", "scripts", "docs", "build", "dist"]
verbose = 1

# Codespell. THE single source of truth for the ignore list: CI runs `codespell --toml pyproject.toml`
# and the pre-commit hook now passes that same `--toml` instead of repeating the list on its command
# line. It used to carry its own, longer `--ignore-words-list=...`, and the two drifted -- words only
# the hook knew about ("re-declared", "ser", "utilizaton", "statics") passed the local gate and failed
# CI (2026-09-03). Never re-add an --ignore-words-list flag anywhere; extend this list instead.
# "coo" = coordinate sparse-matrix format (core/matrix.py); "notin" = the ast.NotIn node name, which
# codespell reads as a misspelling of "not in"; "re-declare"/"re-declared" are the deliberate hyphenated
# prose spelling; "utilizaton" is a long-standing public attribute name on UtilizationMonitor; "ser" is
# the conventional pandas Series abbreviation; "statics" is used as a noun for static assets.
[tool.codespell]
ignore-words-list = "coo,caf,ser,statics,utilizaton,re-declare,re-declared,re-uses,re-using,unparseable,notin,pre-empt,pre-empts,nd"

# MyPy type checker
[tool.mypy]
# The project's minimum RUNTIME support may still be older; this is only the syntax-checking
# target mypy 2.x type-checks against, and mypy 2.x dropped support for targets below 3.10
# (raises "Python 3.8 is not supported" otherwise -- it silently fell back and kept checking,
# but the declared target should match what mypy actually enforces).
python_version = "3.10"
warn_return_any = true
warn_unused_configs = true
disallow_untyped_defs = false  # Too strict for now, enable gradually
ignore_missing_imports = true
check_untyped_defs = true
# Enabled 2026-09-02 after clearing all 25 hits it reported: 11 were unannotated module globals
# (`browser = None` etc.) that mypy typed as exactly `None`, which silently disabled ALL checking
# of the busiest objects in web/browser.py, database/redislib.py and cloud/cloud.py; the rest were
# annotation-vs-body contradictions (a `dict` param whose body handles list/str/None) and one
# genuinely dead fallback in llm/openai_compat.py. `mypy src/pyutilz` is clean WITH this flag on,
# verified over two consecutive --no-incremental runs.
warn_unreachable = true
# `warn_unused_ignores` is deliberately NOT set. Measured 2026-09-02: it reports 33 unused ignores
# on this Windows box with the [all,dev] extras installed, but essentially all of them are
# platform- or optional-dependency-conditional and are REQUIRED elsewhere -- system/system/memory.py
# and probing.py's `ctypes.windll` / `winsound` ignores are needed on CI's Linux runner, and the
# pgvector / filelock / IPython / numba / cupy import ignores are needed in any install
# that lacks those extras. Turning the flag on would make the blocking gate fail differently per
# platform and per extras set, i.e. exactly the nondeterminism the gate exists to prevent. The
# environment-INDEPENDENT stale ignores were removed by hand instead (2026-09-02).

# `tests.*` had an `ignore_errors = true` override here until 2026-09-02. It was dead config --
# every mypy entry point (CI's mypy-full.yml, both mypy-full-* hooks, the beachhead hook) targets
# `src/pyutilz` only, so the section never matched a module mypy loads and `warn_unused_configs`
# reported it as an unused section on every run. Removed rather than kept, because "tests are
# checked with errors suppressed" and "tests are not checked at all" are materially different
# postures and the config claimed the wrong one.

# Strict-mode beachhead (mirrors mlframe's approach): small, cohesive, stable-surface subpackages
# verified clean under strict mode.
# Established 2026-07-06:
# - dev.code_audit (11 files): moved to py-ci-shared 2026-10-22, where it is typed and gated;
#   the pyutilz copy and its mypy override were removed.
# - stats (2 files): numba-jitted normality tests; fixed 2 real errors in the njit-fallback
#   stub (missing annotations on the *args/**kwargs decorator pattern).
# Added 2026-09-02, each independently verified clean under the strict flags before being listed:
# - text.secrets_scrub and system.single_flight_cache: already fully annotated, zero changes needed.
#   (performance.kernel_tuning.cache was measured as a candidate at the same time and NOT added:
#   it still reports 33 [no-untyped-def] errors across 6 files, so it needs its own typing pass
#   first -- listing it here would have broken the whole-project blocking gate.)
# - system.monitoring: required 13 added annotations (decorator factories and their wrappers).
#   Prioritized because its unannotated `func` param was making `job_completed`'s wrong `data`
#   type and a genuine `str | None` arg-type invisible to the whole-project run.
# Added 2026-09-03 (audit 06/F07), each measured clean under the strict flags before listing:
# - web.proxy (5 modules): 2 missing return annotations, both on @contextmanager session
#   factories in session.py (-> Iterator[Any]).
# - llm.openrouter_provider (4 modules): 3 functions -- _catalogue._pkg, _health._best and
#   _provider._to_float.
# The performance.kernel_tuning.cache measurement recorded above was RE-TAKEN 2026-09-03 and has
# not moved in a month: still exactly 33 [no-untyped-def] across the same 6 files (_common.py 8,
# cache_hooks.py 9, cache_tuning.py 9, cache_sweeping.py 3, cache_base.py 2, cache_class.py 2).
# It is a note, not a queue; the two subpackages above were the cheap admissions.
# Expansion to other subpackages should follow only after each is independently verified
# clean, same policy as mlframe.
[[tool.mypy.overrides]]
module = [
    "pyutilz.stats.*",
    "pyutilz.text.secrets_scrub",
    "pyutilz.system.single_flight_cache",
    "pyutilz.system.monitoring",
    "pyutilz.web.proxy.*",
    "pyutilz.llm.openrouter_provider.*",
]
disallow_untyped_defs = true
disallow_incomplete_defs = true
warn_return_any = true
no_implicit_optional = true

# Secret shapes for py_ci_shared.tracked_secret_shapes (pre-commit hook and tests/test_meta). detect-secrets has no bearer-header
# detector, and the repo handles bearer tokens (see the 2026-09-02 security audit, LoginAndGetCookies). A deliberate fake carries
# `pragma: allowlist secret` on its line.
[tool.py_ci_shared.secret_shapes]
bearer_token = { pattern = "[Bb]earer [A-Za-z0-9._~+/-]{32,}=*", min_length = 40, note = "a bearer token literal: revoke it, git history is not rewritten" }

# pydoclint (docstring-vs-signature consistency, run as `pydoclint --style=google src/pyutilz` by .github/workflows/pydoclint.yml).
# Mirrors mlframe's [tool.pydoclint]: the stock defaults flag every terse docstring that omits a repeated type hint or a Raises section,
# which is this codebase's documented style (no restated types, Raises only where it adds information).
[tool.pydoclint]
arg-type-hints-in-docstring = false  # types already live in the signature.
check-return-types = false           # ditto for return types.
skip-checking-raises = true          # a Raises section is not required for every raise statement.
check-class-attributes = false       # ditto for dataclass/NamedTuple attribute docs.