diff --git a/README.md b/README.md index ccc9c8d..aa4f706 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,9 @@ Additional dependencies can be installed using === "pip" ```sh + # To add pandas-related dependencies + pip install 'continuous-timeseries[pandas]' + # To add plotting dependencies pip install 'continuous-timeseries[plots]' diff --git a/accessor-handler/.copier-answers.yml b/accessor-handler/.copier-answers.yml new file mode 100644 index 0000000..0075d20 --- /dev/null +++ b/accessor-handler/.copier-answers.yml @@ -0,0 +1,19 @@ +# Changes here will be overwritten by Copier +_commit: 1.2.3 +_src_path: gh:mkdocstrings/handler-template +author_email: zebedee.nicholls@climate-resource.com +author_fullname: Zebedee Nicholls +author_username: znichollscr +copyright_date: '2025' +copyright_holder: Climate Resource +copyright_holder_email: zebedee.nicholls@climate-resource.com +copyright_license: BSD 3-Clause "New" or "Revised" License +insiders: false +language: Python +project_description: A Python accessor handler for mkdocstrings. +project_name: mkdocstrings-python-accessors +python_package_distribution_name: mkdocstrings-python-accessors +python_package_import_name: python_accessors +repository_name: mkdocstrings-python-accessors +repository_namespace: climate-resource +repository_provider: github.com diff --git a/accessor-handler/.envrc b/accessor-handler/.envrc new file mode 100644 index 0000000..f9d77ee --- /dev/null +++ b/accessor-handler/.envrc @@ -0,0 +1 @@ +PATH_add scripts diff --git a/accessor-handler/.github/FUNDING.yml b/accessor-handler/.github/FUNDING.yml new file mode 100644 index 0000000..77ca658 --- /dev/null +++ b/accessor-handler/.github/FUNDING.yml @@ -0,0 +1,5 @@ +github: znichollscr +ko_fi: znichollscr +polar: znichollscr +custom: +- https://www.paypal.me/znichollscr diff --git a/accessor-handler/.github/ISSUE_TEMPLATE/1-bug.md b/accessor-handler/.github/ISSUE_TEMPLATE/1-bug.md new file mode 100644 index 0000000..846c4af --- /dev/null +++ b/accessor-handler/.github/ISSUE_TEMPLATE/1-bug.md @@ -0,0 +1,61 @@ +--- +name: Bug report +about: Create a bug report to help us improve. +title: "bug: " +labels: unconfirmed +assignees: [pawamoy] +--- + +### Description of the bug + + +### To Reproduce + + +``` +WRITE MRE / INSTRUCTIONS HERE +``` + +### Full traceback + + +
Full traceback + +```python +PASTE TRACEBACK HERE +``` + +
+ +### Expected behavior + + +### Environment information + + +```bash +python -m mkdocstrings_handlers.python_accessors.debug # | xclip -selection clipboard +``` + +PASTE MARKDOWN OUTPUT HERE + +### Additional context + diff --git a/accessor-handler/.github/ISSUE_TEMPLATE/2-feature.md b/accessor-handler/.github/ISSUE_TEMPLATE/2-feature.md new file mode 100644 index 0000000..2df98fb --- /dev/null +++ b/accessor-handler/.github/ISSUE_TEMPLATE/2-feature.md @@ -0,0 +1,19 @@ +--- +name: Feature request +about: Suggest an idea for this project. +title: "feature: " +labels: feature +assignees: pawamoy +--- + +### Is your feature request related to a problem? Please describe. + + +### Describe the solution you'd like + + +### Describe alternatives you've considered + + +### Additional context + diff --git a/accessor-handler/.github/ISSUE_TEMPLATE/3-docs.md b/accessor-handler/.github/ISSUE_TEMPLATE/3-docs.md new file mode 100644 index 0000000..92ac8ec --- /dev/null +++ b/accessor-handler/.github/ISSUE_TEMPLATE/3-docs.md @@ -0,0 +1,16 @@ +--- +name: Documentation update +about: Point at unclear, missing or outdated documentation. +title: "docs: " +labels: docs +assignees: pawamoy +--- + +### Is something unclear, missing or outdated in our documentation? + + +### Relevant code snippets + + +### Link to the relevant documentation section + diff --git a/accessor-handler/.github/ISSUE_TEMPLATE/4-change.md b/accessor-handler/.github/ISSUE_TEMPLATE/4-change.md new file mode 100644 index 0000000..dc9a8f1 --- /dev/null +++ b/accessor-handler/.github/ISSUE_TEMPLATE/4-change.md @@ -0,0 +1,18 @@ +--- +name: Change request +about: Suggest any other kind of change for this project. +title: "change: " +assignees: pawamoy +--- + +### Is your change request related to a problem? Please describe. + + +### Describe the solution you'd like + + +### Describe alternatives you've considered + + +### Additional context + diff --git a/accessor-handler/.github/ISSUE_TEMPLATE/config.yml b/accessor-handler/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..90441af --- /dev/null +++ b/accessor-handler/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: false +contact_links: +- name: I have a question / I need help + url: https://github.com/climate-resource/mkdocstrings-python-accessors/discussions/new?category=q-a + about: Ask and answer questions in the Discussions tab. diff --git a/accessor-handler/.github/workflows/ci.yml b/accessor-handler/.github/workflows/ci.yml new file mode 100644 index 0000000..de566ed --- /dev/null +++ b/accessor-handler/.github/workflows/ci.yml @@ -0,0 +1,110 @@ +name: ci + +on: + push: + pull_request: + branches: + - main + +defaults: + run: + shell: bash + +env: + LANG: en_US.utf-8 + LC_ALL: en_US.utf-8 + PYTHONIOENCODING: UTF-8 + PYTHON_VERSIONS: "" + +jobs: + + quality: + + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Setup uv + uses: astral-sh/setup-uv@v3 + with: + enable-cache: true + cache-dependency-glob: pyproject.toml + + - name: Install dependencies + run: make setup + + - name: Check if the documentation builds correctly + run: make check-docs + + - name: Check the code quality + run: make check-quality + + - name: Check if the code is correctly typed + run: make check-types + + - name: Check for breaking changes in the API + run: make check-api + + tests: + + strategy: + matrix: + os: + - ubuntu-latest + - macos-latest + - windows-latest + python-version: + - "3.9" + - "3.10" + - "3.11" + - "3.12" + - "3.13" + - "3.14" + resolution: + - highest + - lowest-direct + exclude: + - os: macos-latest + resolution: lowest-direct + - os: windows-latest + resolution: lowest-direct + runs-on: ${{ matrix.os }} + continue-on-error: ${{ matrix.python-version == '3.14' }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + allow-prereleases: true + + - name: Setup uv + uses: astral-sh/setup-uv@v3 + with: + enable-cache: true + cache-dependency-glob: pyproject.toml + cache-suffix: py${{ matrix.python-version }} + + - name: Install dependencies + env: + UV_RESOLUTION: ${{ matrix.resolution }} + run: make setup + + - name: Run the test suite + run: make test diff --git a/accessor-handler/.github/workflows/release.yml b/accessor-handler/.github/workflows/release.yml new file mode 100644 index 0000000..d09c514 --- /dev/null +++ b/accessor-handler/.github/workflows/release.yml @@ -0,0 +1,28 @@ +name: release + +on: push +permissions: + contents: write + +jobs: + release: + runs-on: ubuntu-latest + if: startsWith(github.ref, 'refs/tags/') + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Setup uv + uses: astral-sh/setup-uv@v3 + - name: Prepare release notes + run: uv tool run git-changelog --release-notes > release-notes.md + - name: Create release + uses: softprops/action-gh-release@v2 + with: + body_path: release-notes.md diff --git a/accessor-handler/.gitignore b/accessor-handler/.gitignore new file mode 100644 index 0000000..9fea047 --- /dev/null +++ b/accessor-handler/.gitignore @@ -0,0 +1,25 @@ +# editors +.idea/ +.vscode/ + +# python +*.egg-info/ +*.py[cod] +.venv/ +.venvs/ +/build/ +/dist/ + +# tools +.coverage* +/.pdm-build/ +/htmlcov/ +/site/ +uv.lock + +# cache +.cache/ +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ +__pycache__/ diff --git a/accessor-handler/CHANGELOG.md b/accessor-handler/CHANGELOG.md new file mode 100644 index 0000000..a87281b --- /dev/null +++ b/accessor-handler/CHANGELOG.md @@ -0,0 +1,8 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) +and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). + + diff --git a/accessor-handler/CODE_OF_CONDUCT.md b/accessor-handler/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..9185db4 --- /dev/null +++ b/accessor-handler/CODE_OF_CONDUCT.md @@ -0,0 +1,132 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, caste, color, religion, or sexual +identity and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the overall + community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or advances of + any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email address, + without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +zebedee.nicholls@climate-resource.com. +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series of +actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or permanent +ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within the +community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.1, available at +[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. + +Community Impact Guidelines were inspired by +[Mozilla's code of conduct enforcement ladder][Mozilla CoC]. + +For answers to common questions about this code of conduct, see the FAQ at +[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at +[https://www.contributor-covenant.org/translations][translations]. + +[homepage]: https://www.contributor-covenant.org +[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html +[Mozilla CoC]: https://github.com/mozilla/diversity +[FAQ]: https://www.contributor-covenant.org/faq +[translations]: https://www.contributor-covenant.org/translations diff --git a/accessor-handler/CONTRIBUTING.md b/accessor-handler/CONTRIBUTING.md new file mode 100644 index 0000000..8b9e837 --- /dev/null +++ b/accessor-handler/CONTRIBUTING.md @@ -0,0 +1,148 @@ +# Contributing + +Contributions are welcome, and they are greatly appreciated! +Every little bit helps, and credit will always be given. + +## Environment setup + +Nothing easier! + +Fork and clone the repository, then: + +```bash +cd mkdocstrings-python-accessors +make setup +``` + +> NOTE: +> If it fails for some reason, +> you'll need to install +> [uv](https://github.com/astral-sh/uv) +> manually. +> +> You can install it with: +> +> ```bash +> curl -LsSf https://astral.sh/uv/install.sh | sh +> ``` +> +> Now you can try running `make setup` again, +> or simply `uv sync`. + +You now have the dependencies installed. + +Run `make help` to see all the available actions! + +## Tasks + +The entry-point to run commands and tasks is the `make` Python script, +located in the `scripts` directory. Try running `make` to show the available commands and tasks. +The *commands* do not need the Python dependencies to be installed, +while the *tasks* do. +The cross-platform tasks are written in Python, thanks to [duty](https://github.com/pawamoy/duty). + +If you work in VSCode, we provide +[an action to configure VSCode](https://pawamoy.github.io/copier-uv/work/#vscode-setup) +for the project. + +## Development + +As usual: + +1. create a new branch: `git switch -c feature-or-bugfix-name` +1. edit the code and/or the documentation + +**Before committing:** + +1. run `make format` to auto-format the code +1. run `make check` to check everything (fix any warning) +1. run `make test` to run the tests (fix any issue) +1. if you updated the documentation or the project dependencies: + 1. run `make docs` + 1. go to http://localhost:8000 and check that everything looks good +1. follow our [commit message convention](#commit-message-convention) + +If you are unsure about how to fix or ignore a warning, +just let the continuous integration fail, +and we will help you during review. + +Don't bother updating the changelog, we will take care of this. + +## Commit message convention + +Commit messages must follow our convention based on the +[Angular style](https://gist.github.com/stephenparish/9941e89d80e2bc58a153#format-of-the-commit-message) +or the [Karma convention](https://karma-runner.github.io/4.0/dev/git-commit-msg.html): + +``` +[(scope)]: Subject + +[Body] +``` + +**Subject and body must be valid Markdown.** +Subject must have proper casing (uppercase for first letter +if it makes sense), but no dot at the end, and no punctuation +in general. + +Scope and body are optional. Type can be: + +- `build`: About packaging, building wheels, etc. +- `chore`: About packaging or repo/files management. +- `ci`: About Continuous Integration. +- `deps`: Dependencies update. +- `docs`: About documentation. +- `feat`: New feature. +- `fix`: Bug fix. +- `perf`: About performance. +- `refactor`: Changes that are not features or bug fixes. +- `style`: A change in code style/format. +- `tests`: About tests. + +If you write a body, please add trailers at the end +(for example issues and PR references, or co-authors), +without relying on GitHub's flavored Markdown: + +``` +Body. + +Issue #10: https://github.com/namespace/project/issues/10 +Related to PR namespace/other-project#15: https://github.com/namespace/other-project/pull/15 +``` + +These "trailers" must appear at the end of the body, +without any blank lines between them. The trailer title +can contain any character except colons `:`. +We expect a full URI for each trailer, not just GitHub autolinks +(for example, full GitHub URLs for commits and issues, +not the hash or the #issue-number). + +We do not enforce a line length on commit messages summary and body, +but please avoid very long summaries, and very long lines in the body, +unless they are part of code blocks that must not be wrapped. + +## Pull requests guidelines + +Link to any related issue in the Pull Request message. + +During the review, we recommend using fixups: + +```bash +# SHA is the SHA of the commit you want to fix +git commit --fixup=SHA +``` + +Once all the changes are approved, you can squash your commits: + +```bash +git rebase -i --autosquash main +``` + +And force-push: + +```bash +git push -f +``` + +If this seems all too complicated, you can push or force-push each new commit, +and we will squash them ourselves if needed, before merging. diff --git a/accessor-handler/LICENSE b/accessor-handler/LICENSE new file mode 100644 index 0000000..6e48b61 --- /dev/null +++ b/accessor-handler/LICENSE @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2025, Climate Resource +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/accessor-handler/Makefile b/accessor-handler/Makefile new file mode 100644 index 0000000..5e88121 --- /dev/null +++ b/accessor-handler/Makefile @@ -0,0 +1,28 @@ +# If you have `direnv` loaded in your shell, and allow it in the repository, +# the `make` command will point at the `scripts/make` shell script. +# This Makefile is just here to allow auto-completion in the terminal. + +actions = \ + allrun \ + changelog \ + check \ + check-api \ + check-docs \ + check-quality \ + check-types \ + clean \ + coverage \ + docs \ + docs-deploy \ + format \ + help \ + multirun \ + release \ + run \ + setup \ + test \ + vscode + +.PHONY: $(actions) +$(actions): + @python scripts/make "$@" diff --git a/accessor-handler/README.md b/accessor-handler/README.md new file mode 100644 index 0000000..6491059 --- /dev/null +++ b/accessor-handler/README.md @@ -0,0 +1,14 @@ +# mkdocstrings-python-accessors + +[![ci](https://github.com/climate-resource/mkdocstrings-python-accessors/workflows/ci/badge.svg)](https://github.com/climate-resource/mkdocstrings-python-accessors/actions?query=workflow%3Aci) +[![documentation](https://img.shields.io/badge/docs-mkdocs-708FCC.svg?style=flat)](https://climate-resource.github.io/mkdocstrings-python-accessors/) +[![pypi version](https://img.shields.io/pypi/v/mkdocstrings-python-accessors.svg)](https://pypi.org/project/mkdocstrings-python-accessors/) +[![gitter](https://badges.gitter.im/join%20chat.svg)](https://app.gitter.im/#/room/#mkdocstrings-python-accessors:gitter.im) + +A Python accessor handler for mkdocstrings. + +## Installation + +```bash +pip install mkdocstrings-python-accessors +``` diff --git a/accessor-handler/config/coverage.ini b/accessor-handler/config/coverage.ini new file mode 100644 index 0000000..b56a286 --- /dev/null +++ b/accessor-handler/config/coverage.ini @@ -0,0 +1,25 @@ +[coverage:run] +branch = true +parallel = true +source = + src/ + tests/ + +[coverage:paths] +equivalent = + src/ + .venv/lib/*/site-packages/ + .venvs/*/lib/*/site-packages/ + +[coverage:report] +precision = 2 +omit = + src/*/__init__.py + src/*/__main__.py + tests/__init__.py +exclude_lines = + pragma: no cover + if TYPE_CHECKING + +[coverage:json] +output = htmlcov/coverage.json diff --git a/accessor-handler/config/git-changelog.toml b/accessor-handler/config/git-changelog.toml new file mode 100644 index 0000000..57114e0 --- /dev/null +++ b/accessor-handler/config/git-changelog.toml @@ -0,0 +1,9 @@ +bump = "auto" +convention = "angular" +in-place = true +output = "CHANGELOG.md" +parse-refs = false +parse-trailers = true +sections = ["build", "deps", "feat", "fix", "refactor"] +template = "keepachangelog" +versioning = "pep440" diff --git a/accessor-handler/config/mypy.ini b/accessor-handler/config/mypy.ini new file mode 100644 index 0000000..cb0dd88 --- /dev/null +++ b/accessor-handler/config/mypy.ini @@ -0,0 +1,7 @@ +[mypy] +ignore_missing_imports = true +exclude = tests/fixtures/ +warn_unused_ignores = true +show_error_codes = true +namespace_packages = true +explicit_package_bases = true diff --git a/accessor-handler/config/pytest.ini b/accessor-handler/config/pytest.ini new file mode 100644 index 0000000..052a2f1 --- /dev/null +++ b/accessor-handler/config/pytest.ini @@ -0,0 +1,14 @@ +[pytest] +python_files = + test_*.py +addopts = + --cov + --cov-config config/coverage.ini +testpaths = + tests + +# action:message_regex:warning_class:module_regex:line +filterwarnings = + error + # TODO: remove once pytest-xdist 4 is released + ignore:.*rsyncdir:DeprecationWarning:xdist diff --git a/accessor-handler/config/ruff.toml b/accessor-handler/config/ruff.toml new file mode 100644 index 0000000..29e6fb4 --- /dev/null +++ b/accessor-handler/config/ruff.toml @@ -0,0 +1,84 @@ +target-version = "py39" +line-length = 120 + +[lint] +exclude = [ + "tests/fixtures/*.py", +] +select = [ + "A", "ANN", "ARG", + "B", "BLE", + "C", "C4", + "COM", + "D", "DTZ", + "E", "ERA", "EXE", + "F", "FBT", + "G", + "I", "ICN", "INP", "ISC", + "N", + "PGH", "PIE", "PL", "PLC", "PLE", "PLR", "PLW", "PT", "PYI", + "Q", + "RUF", "RSE", "RET", + "S", "SIM", "SLF", + "T", "T10", "T20", "TCH", "TID", "TRY", + "UP", + "W", + "YTT", +] +ignore = [ + "A001", # Variable is shadowing a Python builtin + "ANN101", # Missing type annotation for self + "ANN102", # Missing type annotation for cls + "ANN204", # Missing return type annotation for special method __str__ + "ANN401", # Dynamically typed expressions (typing.Any) are disallowed + "ARG005", # Unused lambda argument + "C901", # Too complex + "D105", # Missing docstring in magic method + "D417", # Missing argument description in the docstring + "E501", # Line too long + "ERA001", # Commented out code + "G004", # Logging statement uses f-string + "PLR0911", # Too many return statements + "PLR0912", # Too many branches + "PLR0913", # Too many arguments to function call + "PLR0915", # Too many statements + "SLF001", # Private member accessed + "TRY003", # Avoid specifying long messages outside the exception class +] + +[lint.per-file-ignores] +"src/*/cli.py" = [ + "T201", # Print statement +] +"src/*/debug.py" = [ + "T201", # Print statement +] +"scripts/*.py" = [ + "INP001", # File is part of an implicit namespace package + "T201", # Print statement +] +"tests/*.py" = [ + "ARG005", # Unused lambda argument + "FBT001", # Boolean positional arg in function definition + "PLR2004", # Magic value used in comparison + "S101", # Use of assert detected +] + +[lint.flake8-quotes] +docstring-quotes = "double" + +[lint.flake8-tidy-imports] +ban-relative-imports = "all" + +[lint.isort] +known-first-party = ["mkdocstrings_handlers.python_accessors"] + +[lint.pydocstyle] +convention = "google" + +[format] +exclude = [ + "tests/fixtures/*.py", +] +docstring-code-format = true +docstring-code-line-length = 80 diff --git a/accessor-handler/config/vscode/launch.json b/accessor-handler/config/vscode/launch.json new file mode 100644 index 0000000..8aaeb8a --- /dev/null +++ b/accessor-handler/config/vscode/launch.json @@ -0,0 +1,47 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "python (current file)", + "type": "debugpy", + "request": "launch", + "program": "${file}", + "console": "integratedTerminal", + "justMyCode": false + }, + { + "name": "docs", + "type": "debugpy", + "request": "launch", + "module": "mkdocs", + "justMyCode": false, + "args": [ + "serve", + "-v" + ] + }, + { + "name": "test", + "type": "debugpy", + "request": "launch", + "module": "pytest", + "justMyCode": false, + "args": [ + "-c=config/pytest.ini", + "-vvv", + "--no-cov", + "--dist=no", + "tests", + "-k=${input:tests_selection}" + ] + } + ], + "inputs": [ + { + "id": "tests_selection", + "type": "promptString", + "description": "Tests selection", + "default": "" + } + ] +} diff --git a/accessor-handler/config/vscode/settings.json b/accessor-handler/config/vscode/settings.json new file mode 100644 index 0000000..1b80b1c --- /dev/null +++ b/accessor-handler/config/vscode/settings.json @@ -0,0 +1,33 @@ +{ + "files.watcherExclude": { + "**/.venv*/**": true, + "**/.venvs*/**": true, + "**/venv*/**": true + }, + "mypy-type-checker.args": [ + "--config-file=config/mypy.ini" + ], + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true, + "python.testing.pytestArgs": [ + "--config-file=config/pytest.ini" + ], + "ruff.enable": true, + "ruff.format.args": [ + "--config=config/ruff.toml" + ], + "ruff.lint.args": [ + "--config=config/ruff.toml" + ], + "yaml.schemas": { + "https://squidfunk.github.io/mkdocs-material/schema.json": "mkdocs.yml" + }, + "yaml.customTags": [ + "!ENV scalar", + "!ENV sequence", + "!relative scalar", + "tag:yaml.org,2002:python/name:materialx.emoji.to_svg", + "tag:yaml.org,2002:python/name:materialx.emoji.twemoji", + "tag:yaml.org,2002:python/name:pymdownx.superfences.fence_code_format" + ] +} diff --git a/accessor-handler/config/vscode/tasks.json b/accessor-handler/config/vscode/tasks.json new file mode 100644 index 0000000..05e6245 --- /dev/null +++ b/accessor-handler/config/vscode/tasks.json @@ -0,0 +1,97 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "label": "changelog", + "type": "process", + "command": "scripts/make", + "args": ["changelog"] + }, + { + "label": "check", + "type": "process", + "command": "scripts/make", + "args": ["check"] + }, + { + "label": "check-quality", + "type": "process", + "command": "scripts/make", + "args": ["check-quality"] + }, + { + "label": "check-types", + "type": "process", + "command": "scripts/make", + "args": ["check-types"] + }, + { + "label": "check-docs", + "type": "process", + "command": "scripts/make", + "args": ["check-docs"] + }, + { + "label": "check-api", + "type": "process", + "command": "scripts/make", + "args": ["check-api"] + }, + { + "label": "clean", + "type": "process", + "command": "scripts/make", + "args": ["clean"] + }, + { + "label": "docs", + "type": "process", + "command": "scripts/make", + "args": ["docs"] + }, + { + "label": "docs-deploy", + "type": "process", + "command": "scripts/make", + "args": ["docs-deploy"] + }, + { + "label": "format", + "type": "process", + "command": "scripts/make", + "args": ["format"] + }, + { + "label": "release", + "type": "process", + "command": "scripts/make", + "args": ["release", "${input:version}"] + }, + { + "label": "setup", + "type": "process", + "command": "scripts/make", + "args": ["setup"] + }, + { + "label": "test", + "type": "process", + "command": "scripts/make", + "args": ["test", "coverage"], + "group": "test" + }, + { + "label": "vscode", + "type": "process", + "command": "scripts/make", + "args": ["vscode"] + } + ], + "inputs": [ + { + "id": "version", + "type": "promptString", + "description": "Version" + } + ] +} diff --git a/accessor-handler/docs/.overrides/partials/comments.html b/accessor-handler/docs/.overrides/partials/comments.html new file mode 100644 index 0000000..1cf517a --- /dev/null +++ b/accessor-handler/docs/.overrides/partials/comments.html @@ -0,0 +1,57 @@ + + + diff --git a/accessor-handler/docs/changelog.md b/accessor-handler/docs/changelog.md new file mode 100644 index 0000000..786b75d --- /dev/null +++ b/accessor-handler/docs/changelog.md @@ -0,0 +1 @@ +--8<-- "CHANGELOG.md" diff --git a/accessor-handler/docs/code_of_conduct.md b/accessor-handler/docs/code_of_conduct.md new file mode 100644 index 0000000..01f2ea2 --- /dev/null +++ b/accessor-handler/docs/code_of_conduct.md @@ -0,0 +1 @@ +--8<-- "CODE_OF_CONDUCT.md" diff --git a/accessor-handler/docs/contributing.md b/accessor-handler/docs/contributing.md new file mode 100644 index 0000000..ea38c9b --- /dev/null +++ b/accessor-handler/docs/contributing.md @@ -0,0 +1 @@ +--8<-- "CONTRIBUTING.md" diff --git a/accessor-handler/docs/credits.md b/accessor-handler/docs/credits.md new file mode 100644 index 0000000..f758db8 --- /dev/null +++ b/accessor-handler/docs/credits.md @@ -0,0 +1,10 @@ +--- +hide: +- toc +--- + + +```python exec="yes" +--8<-- "scripts/gen_credits.py" +``` + diff --git a/accessor-handler/docs/css/material.css b/accessor-handler/docs/css/material.css new file mode 100644 index 0000000..9e8c14a --- /dev/null +++ b/accessor-handler/docs/css/material.css @@ -0,0 +1,4 @@ +/* More space at the bottom of the page. */ +.md-main__inner { + margin-bottom: 1.5rem; +} diff --git a/accessor-handler/docs/css/mkdocstrings.css b/accessor-handler/docs/css/mkdocstrings.css new file mode 100644 index 0000000..03c39d3 --- /dev/null +++ b/accessor-handler/docs/css/mkdocstrings.css @@ -0,0 +1,27 @@ +/* Indentation. */ +div.doc-contents:not(.first) { + padding-left: 25px; + border-left: .05rem solid var(--md-typeset-table-color); +} + +/* Mark external links as such. */ +a.external::after, +a.autorefs-external::after { + /* https://primer.style/octicons/arrow-up-right-24 */ + mask-image: url('data:image/svg+xml,'); + -webkit-mask-image: url('data:image/svg+xml,'); + content: ' '; + + display: inline-block; + vertical-align: middle; + position: relative; + + height: 1em; + width: 1em; + background-color: currentColor; +} + +a.external:hover::after, +a.autorefs-external:hover::after { + background-color: var(--md-accent-fg-color); +} diff --git a/accessor-handler/docs/index.md b/accessor-handler/docs/index.md new file mode 100644 index 0000000..8e6f2fb --- /dev/null +++ b/accessor-handler/docs/index.md @@ -0,0 +1,6 @@ +--- +hide: +- feedback +--- + +--8<-- "README.md" diff --git a/accessor-handler/docs/js/feedback.js b/accessor-handler/docs/js/feedback.js new file mode 100644 index 0000000..f97321a --- /dev/null +++ b/accessor-handler/docs/js/feedback.js @@ -0,0 +1,14 @@ +const feedback = document.forms.feedback; +feedback.hidden = false; + +feedback.addEventListener("submit", function(ev) { + ev.preventDefault(); + const commentElement = document.getElementById("feedback"); + commentElement.style.display = "block"; + feedback.firstElementChild.disabled = true; + const data = ev.submitter.getAttribute("data-md-value"); + const note = feedback.querySelector(".md-feedback__note [data-md-value='" + data + "']"); + if (note) { + note.hidden = false; + } +}) diff --git a/accessor-handler/docs/license.md b/accessor-handler/docs/license.md new file mode 100644 index 0000000..e81c0ed --- /dev/null +++ b/accessor-handler/docs/license.md @@ -0,0 +1,10 @@ +--- +hide: +- feedback +--- + +# License + +``` +--8<-- "LICENSE" +``` diff --git a/accessor-handler/duties.py b/accessor-handler/duties.py new file mode 100644 index 0000000..24d8a82 --- /dev/null +++ b/accessor-handler/duties.py @@ -0,0 +1,224 @@ +"""Development tasks.""" + +from __future__ import annotations + +import os +import sys +from contextlib import contextmanager +from importlib.metadata import version as pkgversion +from pathlib import Path +from typing import TYPE_CHECKING + +from duty import duty, tools + +if TYPE_CHECKING: + from collections.abc import Iterator + + from duty.context import Context + + +PY_SRC_PATHS = (Path(_) for _ in ("src", "tests", "duties.py", "scripts")) +PY_SRC_LIST = tuple(str(_) for _ in PY_SRC_PATHS) +PY_SRC = " ".join(PY_SRC_LIST) +CI = os.environ.get("CI", "0") in {"1", "true", "yes", ""} +WINDOWS = os.name == "nt" +PTY = not WINDOWS and not CI +MULTIRUN = os.environ.get("MULTIRUN", "0") == "1" + + +def pyprefix(title: str) -> str: # noqa: D103 + if MULTIRUN: + prefix = f"(python{sys.version_info.major}.{sys.version_info.minor})" + return f"{prefix:14}{title}" + return title + + +@contextmanager +def material_insiders() -> Iterator[bool]: # noqa: D103 + if "+insiders" in pkgversion("mkdocs-material"): + os.environ["MATERIAL_INSIDERS"] = "true" + try: + yield True + finally: + os.environ.pop("MATERIAL_INSIDERS") + else: + yield False + + +@duty +def changelog(ctx: Context, bump: str = "") -> None: + """Update the changelog in-place with latest commits. + + Parameters + ---------- + bump: Bump option passed to git-changelog. + """ + ctx.run(tools.git_changelog(bump=bump or None), title="Updating changelog") + + +@duty(pre=["check-quality", "check-types", "check-docs", "check-api"]) +def check(ctx: Context) -> None: + """Check it all!""" + + +@duty +def check_quality(ctx: Context) -> None: + """Check the code quality.""" + ctx.run( + tools.ruff.check(*PY_SRC_LIST, config="config/ruff.toml"), + title=pyprefix("Checking code quality"), + ) + + +@duty +def check_docs(ctx: Context) -> None: + """Check if the documentation builds correctly.""" + Path("htmlcov").mkdir(parents=True, exist_ok=True) + Path("htmlcov/index.html").touch(exist_ok=True) + with material_insiders(): + ctx.run( + tools.mkdocs.build(strict=True, verbose=True), + title=pyprefix("Building documentation"), + ) + + +@duty +def check_types(ctx: Context) -> None: + """Check that the code is correctly typed.""" + os.environ["MYPYPATH"] = "src" + ctx.run( + tools.mypy(*PY_SRC_LIST, config_file="config/mypy.ini"), + title=pyprefix("Type-checking"), + ) + + +@duty +def check_api(ctx: Context, *cli_args: str) -> None: + """Check for API breaking changes.""" + ctx.run( + tools.griffe.check( + "mkdocstrings_handlers.python_accessors", search=["src"], color=True + ).add_args(*cli_args), + title="Checking for API breaking changes", + nofail=True, + ) + + +@duty +def docs( + ctx: Context, *cli_args: str, host: str = "127.0.0.1", port: int = 8000 +) -> None: + """Serve the documentation (localhost:8000). + + Parameters + ---------- + host: The host to serve the docs from. + port: The port to serve the docs on. + """ + with material_insiders(): + ctx.run( + tools.mkdocs.serve(dev_addr=f"{host}:{port}").add_args(*cli_args), + title="Serving documentation", + capture=False, + ) + + +@duty +def docs_deploy(ctx: Context) -> None: + """Deploy the documentation to GitHub pages.""" + os.environ["DEPLOY"] = "true" + with material_insiders() as insiders: + if not insiders: + ctx.run( + lambda: False, + title="Not deploying docs without Material for MkDocs Insiders!", + ) + ctx.run(tools.mkdocs.gh_deploy(), title="Deploying documentation") + + +@duty +def format(ctx: Context) -> None: + """Run formatting tools on the code.""" + ctx.run( + tools.ruff.check( + *PY_SRC_LIST, config="config/ruff.toml", fix_only=True, exit_zero=True + ), + title="Auto-fixing code", + ) + ctx.run( + tools.ruff.format(*PY_SRC_LIST, config="config/ruff.toml"), + title="Formatting code", + ) + + +@duty +def build(ctx: Context) -> None: + """Build source and wheel distributions.""" + ctx.run( + tools.build(), + title="Building source and wheel distributions", + pty=PTY, + ) + + +@duty +def publish(ctx: Context) -> None: + """Publish source and wheel distributions to PyPI.""" + if not Path("dist").exists(): + ctx.run("false", title="No distribution files found") + dists = [str(dist) for dist in Path("dist").iterdir()] + ctx.run( + tools.twine.upload(*dists, skip_existing=True), + title="Publishing source and wheel distributions to PyPI", + pty=PTY, + ) + + +@duty(post=["build", "publish", "docs-deploy"]) +def release(ctx: Context, version: str = "") -> None: + """Release a new Python package. + + Parameters + ---------- + version: The new version number to use. + """ + if not (version := (version or input("> Version to release: ")).strip()): + ctx.run("false", title="A version must be provided") + ctx.run("git add pyproject.toml CHANGELOG.md", title="Staging files", pty=PTY) + ctx.run( + ["git", "commit", "-m", f"chore: Prepare release {version}"], + title="Committing changes", + pty=PTY, + ) + ctx.run(f"git tag {version}", title="Tagging commit", pty=PTY) + ctx.run("git push", title="Pushing commits", pty=False) + ctx.run("git push --tags", title="Pushing tags", pty=False) + + +@duty(silent=True, aliases=["cov"]) +def coverage(ctx: Context) -> None: + """Report coverage as text and HTML.""" + ctx.run(tools.coverage.combine(), nofail=True) + ctx.run(tools.coverage.report(rcfile="config/coverage.ini"), capture=False) + ctx.run(tools.coverage.html(rcfile="config/coverage.ini")) + + +@duty +def test(ctx: Context, *cli_args: str, match: str = "") -> None: + """Run the test suite. + + Parameters + ---------- + match: A pytest expression to filter selected tests. + """ + py_version = f"{sys.version_info.major}{sys.version_info.minor}" + os.environ["COVERAGE_FILE"] = f".coverage.{py_version}" + ctx.run( + tools.pytest( + "tests", + config_file="config/pytest.ini", + select=match, + color="yes", + ).add_args("-n", "auto", *cli_args), + title=pyprefix("Running tests"), + ) diff --git a/accessor-handler/mkdocs.yml b/accessor-handler/mkdocs.yml new file mode 100644 index 0000000..0f913c8 --- /dev/null +++ b/accessor-handler/mkdocs.yml @@ -0,0 +1,163 @@ +site_name: "mkdocstrings-python-accessors" +site_description: "A Python accessor handler for mkdocstrings." +site_url: "https://climate-resource.github.io/mkdocstrings-python-accessors" +repo_url: "https://github.com/climate-resource/mkdocstrings-python-accessors" +repo_name: "climate-resource/mkdocstrings-python-accessors" +site_dir: "site" +watch: [mkdocs.yml, README.md, CONTRIBUTING.md, CHANGELOG.md, src/mkdocstrings_handlers] +copyright: Copyright © 2025 Climate Resource +edit_uri: edit/main/docs/ + +validation: + omitted_files: warn + absolute_links: warn + unrecognized_links: warn + +nav: +- Home: + - Overview: index.md + - Changelog: changelog.md + - Credits: credits.md + - License: license.md +# defer to gen-files + literate-nav +- API reference: + - mkdocstrings-python-accessors: reference/ +- Development: + - Contributing: contributing.md + - Code of Conduct: code_of_conduct.md + - Coverage report: coverage.md + - mkdocstrings: https://mkdocstrings.github.io/ + +theme: + name: material + custom_dir: docs/.overrides + features: + - announce.dismiss + - content.action.edit + - content.action.view + - content.code.annotate + - content.code.copy + - content.tooltips + - navigation.footer + - navigation.indexes + - navigation.sections + - navigation.tabs + - navigation.tabs.sticky + - navigation.top + - search.highlight + - search.suggest + - toc.follow + palette: + - media: "(prefers-color-scheme)" + toggle: + icon: material/brightness-auto + name: Switch to light mode + - media: "(prefers-color-scheme: light)" + scheme: default + primary: teal + accent: purple + toggle: + icon: material/weather-sunny + name: Switch to dark mode + - media: "(prefers-color-scheme: dark)" + scheme: slate + primary: black + accent: lime + toggle: + icon: material/weather-night + name: Switch to system preference + +extra_css: +- css/material.css +- css/mkdocstrings.css + +extra_javascript: +- js/feedback.js + +markdown_extensions: +- attr_list +- admonition +- callouts +- footnotes +- pymdownx.emoji: + emoji_index: !!python/name:material.extensions.emoji.twemoji + emoji_generator: !!python/name:material.extensions.emoji.to_svg +- pymdownx.magiclink +- pymdownx.snippets: + base_path: [!relative $config_dir] + check_paths: true +- pymdownx.superfences +- pymdownx.tabbed: + alternate_style: true + slugify: !!python/object/apply:pymdownx.slugs.slugify + kwds: + case: lower +- pymdownx.tasklist: + custom_checkbox: true +- toc: + permalink: true + +plugins: +- search +- markdown-exec +- gen-files: + scripts: + - scripts/gen_ref_nav.py +- literate-nav: + nav_file: SUMMARY.md +- coverage +- mkdocstrings: + handlers: + python: + import: + - https://docs.python.org/3/objects.inv + - https://mkdocstrings.github.io/objects.inv + paths: [src] + options: + docstring_options: + ignore_init_summary: true + docstring_section_style: list + filters: ["!^_"] + heading_level: 1 + inherited_members: true + merge_init_into_class: true + separate_signature: true + show_root_heading: true + show_root_full_path: false + show_signature_annotations: true + show_source: true + show_symbol_type_heading: true + show_symbol_type_toc: true + signature_crossrefs: true + summary: true +- git-revision-date-localized: + enabled: !ENV [DEPLOY, false] + enable_creation_date: true + type: timeago +- minify: + minify_html: !ENV [DEPLOY, false] +- group: + enabled: !ENV [MATERIAL_INSIDERS, false] + plugins: + - typeset + +extra: + social: + - icon: fontawesome/brands/github + link: https://github.com/znichollscr + - icon: fontawesome/brands/gitter + link: https://gitter.im/mkdocstrings-python-accessors/community + - icon: fontawesome/brands/python + link: https://pypi.org/project/mkdocstrings-python-accessors/ + analytics: + feedback: + title: Was this page helpful? + ratings: + - icon: material/emoticon-happy-outline + name: This page was helpful + data: 1 + note: Thanks for your feedback! + - icon: material/emoticon-sad-outline + name: This page could be improved + data: 0 + note: Let us know how we can improve this page. diff --git a/accessor-handler/pyproject.toml b/accessor-handler/pyproject.toml new file mode 100644 index 0000000..48bea82 --- /dev/null +++ b/accessor-handler/pyproject.toml @@ -0,0 +1,108 @@ +[build-system] +requires = ["pdm-backend"] +build-backend = "pdm.backend" + +[project] +name = "mkdocstrings-python-accessors" +description = "A Python accessor handler for mkdocstrings." +authors = [{name = "Zebedee Nicholls", email = "zebedee.nicholls@climate-resource.com"}] +license = {text = "BSD-3-Clause"} +readme = "README.md" +requires-python = ">=3.9" +keywords = [] +dynamic = ["version"] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Topic :: Documentation", + "Topic :: Software Development", + "Topic :: Utilities", + "Typing :: Typed", +] +dependencies = [ + "mkdocstrings>=0.18", +] + +[project.urls] +Homepage = "https://climate-resource.github.io/mkdocstrings-python-accessors" +Documentation = "https://climate-resource.github.io/mkdocstrings-python-accessors" +Changelog = "https://climate-resource.github.io/mkdocstrings-python-accessors/changelog" +Repository = "https://github.com/climate-resource/mkdocstrings-python-accessors" +Issues = "https://github.com/climate-resource/mkdocstrings-python-accessors/issues" +Discussions = "https://github.com/climate-resource/mkdocstrings-python-accessors/discussions" +Gitter = "https://gitter.im/climate-resource/mkdocstrings-python-accessors" +Funding = "https://github.com/sponsors/znichollscr" + +[tool.pdm.version] +source = "call" +getter = "scripts.get_version:get_version" + +[tool.pdm.build] +package-dir = "src" +includes = ["src/mkdocstrings_handlers"] +editable-backend = "editables" + +# Include as much as possible in the source distribution, to help redistributors. +excludes = ["**/.pytest_cache"] +source-includes = [ + "config", + "docs", + "scripts", + "share", + "tests", + "duties.py", + "mkdocs.yml", + "*.md", + "LICENSE", +] + +[tool.pdm.build.wheel-data] +# Manual pages can be included in the wheel. +# Depending on the installation tool, they will be accessible to users. +# pipx supports it, uv does not yet, see https://github.com/astral-sh/uv/issues/4731. +data = [ + {path = "share/**/*", relative-to = "."}, +] + +[dependency-groups] +dev = [ + # maintenance + "build>=1.2", + "git-changelog>=2.5", + "twine>=5.1", + + # ci + "duty>=1.4", + "ruff>=0.4", + "pytest>=8.2", + "pytest-cov>=5.0", + "pytest-randomly>=3.15", + "pytest-xdist>=3.6", + "mypy>=1.10", + "types-markdown>=3.6", + "types-pyyaml>=6.0", + + # docs + "black>=24.4", + "markdown-callouts>=0.4", + "markdown-exec>=1.8", + "mkdocs>=1.6", + "mkdocs-coverage>=1.0", + "mkdocs-gen-files>=0.5", + "mkdocs-git-revision-date-localized-plugin>=1.2", + "mkdocs-literate-nav>=0.6", + "mkdocs-material>=9.5", + "mkdocs-minify-plugin>=0.8", + "mkdocstrings[python]>=0.25", + # YORE: EOL 3.10: Remove line. + "tomli>=2.0; python_version < '3.11'", +] diff --git a/accessor-handler/scripts/gen_credits.py b/accessor-handler/scripts/gen_credits.py new file mode 100644 index 0000000..3754317 --- /dev/null +++ b/accessor-handler/scripts/gen_credits.py @@ -0,0 +1,201 @@ +"""Script to generate the project's credits.""" + +from __future__ import annotations + +import os +import sys +from collections import defaultdict +from collections.abc import Iterable +from importlib.metadata import distributions +from itertools import chain +from pathlib import Path +from textwrap import dedent +from typing import Union + +from jinja2 import StrictUndefined +from jinja2.sandbox import SandboxedEnvironment +from packaging.requirements import Requirement + +# YORE: EOL 3.10: Replace block with line 2. +if sys.version_info >= (3, 11): + import tomllib +else: + import tomli as tomllib + +project_dir = Path(os.getenv("MKDOCS_CONFIG_DIR", ".")) +with project_dir.joinpath("pyproject.toml").open("rb") as pyproject_file: + pyproject = tomllib.load(pyproject_file) +project = pyproject["project"] +project_name = project["name"] +devdeps = [ + dep for dep in pyproject["dependency-groups"]["dev"] if not dep.startswith("-e") +] + +PackageMetadata = dict[str, Union[str, Iterable[str]]] +Metadata = dict[str, PackageMetadata] + + +def _merge_fields(metadata: dict) -> PackageMetadata: + fields = defaultdict(list) + for header, value in metadata.items(): + fields[header.lower()].append(value.strip()) + return { + field: value + if len(value) > 1 or field in ("classifier", "requires-dist") + else value[0] + for field, value in fields.items() + } + + +def _norm_name(name: str) -> str: + return name.replace("_", "-").replace(".", "-").lower() + + +def _requirements(deps: list[str]) -> dict[str, Requirement]: + return {_norm_name((req := Requirement(dep)).name): req for dep in deps} + + +def _extra_marker(req: Requirement) -> str | None: + if not req.marker: + return None + try: + return next( + marker[2].value + for marker in req.marker._markers + if getattr(marker[0], "value", None) == "extra" + ) + except StopIteration: + return None + + +def _get_metadata() -> Metadata: + metadata = {} + for pkg in distributions(): + name = _norm_name(pkg.name) # type: ignore[attr-defined,unused-ignore] + metadata[name] = _merge_fields(pkg.metadata) # type: ignore[arg-type] + metadata[name]["spec"] = set() + metadata[name]["extras"] = set() + metadata[name].setdefault("summary", "") + _set_license(metadata[name]) + return metadata + + +def _set_license(metadata: PackageMetadata) -> None: + license_field = metadata.get("license-expression", metadata.get("license", "")) + license_name = ( + license_field if isinstance(license_field, str) else " + ".join(license_field) + ) + check_classifiers = license_name in ( + "UNKNOWN", + "Dual License", + "", + ) or license_name.count("\n") + if check_classifiers: + license_names = [] + for classifier in metadata["classifier"]: + if classifier.startswith("License ::"): + license_names.append(classifier.rsplit("::", 1)[1].strip()) + license_name = " + ".join(license_names) + metadata["license"] = license_name or "?" + + +def _get_deps(base_deps: dict[str, Requirement], metadata: Metadata) -> Metadata: + deps = {} + for dep_name, dep_req in base_deps.items(): + if dep_name not in metadata or dep_name == "mkdocstrings-python-accessors": + continue + metadata[dep_name]["spec"] |= {str(spec) for spec in dep_req.specifier} # type: ignore[operator] + metadata[dep_name]["extras"] |= dep_req.extras # type: ignore[operator] + deps[dep_name] = metadata[dep_name] + + again = True + while again: + again = False + for pkg_name in metadata: + if pkg_name in deps: + for pkg_dependency in metadata[pkg_name].get("requires-dist", []): + requirement = Requirement(pkg_dependency) + dep_name = _norm_name(requirement.name) + extra_marker = _extra_marker(requirement) + if ( + dep_name in metadata + and dep_name not in deps + and dep_name != project["name"] + and ( + not extra_marker or extra_marker in deps[pkg_name]["extras"] + ) + ): + metadata[dep_name]["spec"] |= { + str(spec) for spec in requirement.specifier + } # type: ignore[operator] + deps[dep_name] = metadata[dep_name] + again = True + + return deps + + +def _render_credits() -> str: + metadata = _get_metadata() + dev_dependencies = _get_deps(_requirements(devdeps), metadata) + prod_dependencies = _get_deps( + _requirements( + chain( # type: ignore[arg-type] + project.get("dependencies", []), + chain(*project.get("optional-dependencies", {}).values()), + ), + ), + metadata, + ) + + template_data = { + "project_name": project_name, + "prod_dependencies": sorted( + prod_dependencies.values(), key=lambda dep: str(dep["name"]).lower() + ), + "dev_dependencies": sorted( + dev_dependencies.values(), key=lambda dep: str(dep["name"]).lower() + ), + "more_credits": "", + } + template_text = dedent( + """ + # Credits + + These projects were used to build *{{ project_name }}*. **Thank you!** + + [Python](https://www.python.org/) | + [uv](https://github.com/astral-sh/uv) | + [copier-uv](https://github.com/pawamoy/copier-uv) + + {% macro dep_line(dep) -%} + [{{ dep.name }}](https://pypi.org/project/{{ dep.name }}/) | {{ dep.summary }} | {{ ("`" ~ dep.spec|sort(reverse=True)|join(", ") ~ "`") if dep.spec else "" }} | `{{ dep.version }}` | {{ dep.license }} + {%- endmacro %} + + {% if prod_dependencies -%} + ### Runtime dependencies + + Project | Summary | Version (accepted) | Version (last resolved) | License + ------- | ------- | ------------------ | ----------------------- | ------- + {% for dep in prod_dependencies -%} + {{ dep_line(dep) }} + {% endfor %} + + {% endif -%} + {% if dev_dependencies -%} + ### Development dependencies + + Project | Summary | Version (accepted) | Version (last resolved) | License + ------- | ------- | ------------------ | ----------------------- | ------- + {% for dep in dev_dependencies -%} + {{ dep_line(dep) }} + {% endfor %} + + {% endif -%} + {% if more_credits %}**[More credits from the author]({{ more_credits }})**{% endif %} + """, + ) + jinja_env = SandboxedEnvironment(undefined=StrictUndefined) + return jinja_env.from_string(template_text).render(**template_data) + + +print(_render_credits()) diff --git a/accessor-handler/scripts/gen_ref_nav.py b/accessor-handler/scripts/gen_ref_nav.py new file mode 100644 index 0000000..6939e86 --- /dev/null +++ b/accessor-handler/scripts/gen_ref_nav.py @@ -0,0 +1,37 @@ +"""Generate the code reference pages and navigation.""" + +from pathlib import Path + +import mkdocs_gen_files + +nav = mkdocs_gen_files.Nav() +mod_symbol = '' + +root = Path(__file__).parent.parent +src = root / "src" + +for path in sorted(src.rglob("*.py")): + module_path = path.relative_to(src).with_suffix("") + doc_path = path.relative_to(src).with_suffix(".md") + full_doc_path = Path("reference", doc_path) + + parts = tuple(module_path.parts) + + if parts[-1] == "__init__": + parts = parts[:-1] + doc_path = doc_path.with_name("index.md") + full_doc_path = full_doc_path.with_name("index.md") + elif parts[-1].startswith("_"): + continue + + nav_parts = [f"{mod_symbol} {part}" for part in parts] + nav[tuple(nav_parts)] = doc_path.as_posix() + + with mkdocs_gen_files.open(full_doc_path, "w") as fd: + ident = ".".join(parts) + fd.write(f"---\ntitle: {ident}\n---\n\n::: {ident}") + + mkdocs_gen_files.set_edit_path(full_doc_path, ".." / path.relative_to(root)) + +with mkdocs_gen_files.open("reference/SUMMARY.md", "w") as nav_file: + nav_file.writelines(nav.build_literate_nav()) diff --git a/accessor-handler/scripts/get_version.py b/accessor-handler/scripts/get_version.py new file mode 100644 index 0000000..3955aaa --- /dev/null +++ b/accessor-handler/scripts/get_version.py @@ -0,0 +1,32 @@ +"""Get current project version from Git tags or changelog.""" + +import re +from contextlib import suppress +from pathlib import Path + +from pdm.backend.hooks.version import ( + SCMVersion, + Version, + default_version_formatter, + get_version_from_scm, +) + +_root = Path(__file__).parent.parent +_changelog = _root / "CHANGELOG.md" +_changelog_version_re = re.compile(r"^## \[(\d+\.\d+\.\d+)\].*$") +_default_scm_version = SCMVersion(Version("0.0.0"), None, False, None, None) + + +def get_version() -> str: + """Get current project version from Git tags or changelog.""" + scm_version = get_version_from_scm(_root) or _default_scm_version + if scm_version.version <= Version("0.1"): # Missing Git tags? + with suppress(OSError, StopIteration): + with _changelog.open("r", encoding="utf8") as file: + match = next(filter(None, map(_changelog_version_re.match, file))) + scm_version = scm_version._replace(version=Version(match.group(1))) + return default_version_formatter(scm_version) + + +if __name__ == "__main__": + print(get_version()) diff --git a/accessor-handler/scripts/make b/accessor-handler/scripts/make new file mode 100755 index 0000000..ff985eb --- /dev/null +++ b/accessor-handler/scripts/make @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +"""Management commands.""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +from contextlib import contextmanager +from pathlib import Path +from textwrap import dedent +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from collections.abc import Iterator + + +PYTHON_VERSIONS = os.getenv("PYTHON_VERSIONS", "3.9 3.10 3.11 3.12 3.13 3.14").split() + + +def shell(cmd: str, *, capture_output: bool = False, **kwargs: Any) -> str | None: + """Run a shell command.""" + if capture_output: + return subprocess.check_output(cmd, shell=True, text=True, **kwargs) # noqa: S602 + subprocess.run(cmd, shell=True, check=True, stderr=subprocess.STDOUT, **kwargs) # noqa: S602 + return None + + +@contextmanager +def environ(**kwargs: str) -> Iterator[None]: + """Temporarily set environment variables.""" + original = dict(os.environ) + os.environ.update(kwargs) + try: + yield + finally: + os.environ.clear() + os.environ.update(original) + + +def uv_install(venv: Path) -> None: + """Install dependencies using uv.""" + with environ( + UV_PROJECT_ENVIRONMENT=str(venv), PYO3_USE_ABI3_FORWARD_COMPATIBILITY="1" + ): + if "CI" in os.environ: + shell("uv sync --no-editable") + else: + shell("uv sync") + + +def setup() -> None: + """Setup the project.""" + if not shutil.which("uv"): + raise ValueError( + "make: setup: uv must be installed, see https://github.com/astral-sh/uv" + ) + + print("Installing dependencies (default environment)") + default_venv = Path(".venv") + if not default_venv.exists(): + shell("uv venv") + uv_install(default_venv) + + if PYTHON_VERSIONS: + for version in PYTHON_VERSIONS: + print(f"\nInstalling dependencies (python{version})") + venv_path = Path(f".venvs/{version}") + if not venv_path.exists(): + shell(f"uv venv --python {version} {venv_path}") + with environ(UV_PROJECT_ENVIRONMENT=str(venv_path.resolve())): + uv_install(venv_path) + + +def run(version: str, cmd: str, *args: str, **kwargs: Any) -> None: + """Run a command in a virtual environment.""" + kwargs = {"check": True, **kwargs} + uv_run = ["uv", "run", "--no-sync"] + if version == "default": + with environ(UV_PROJECT_ENVIRONMENT=".venv"): + subprocess.run([*uv_run, cmd, *args], **kwargs) # noqa: S603, PLW1510 + else: + with environ(UV_PROJECT_ENVIRONMENT=f".venvs/{version}", MULTIRUN="1"): + subprocess.run([*uv_run, cmd, *args], **kwargs) # noqa: S603, PLW1510 + + +def multirun(cmd: str, *args: str, **kwargs: Any) -> None: + """Run a command for all configured Python versions.""" + if PYTHON_VERSIONS: + for version in PYTHON_VERSIONS: + run(version, cmd, *args, **kwargs) + else: + run("default", cmd, *args, **kwargs) + + +def allrun(cmd: str, *args: str, **kwargs: Any) -> None: + """Run a command in all virtual environments.""" + run("default", cmd, *args, **kwargs) + if PYTHON_VERSIONS: + multirun(cmd, *args, **kwargs) + + +def clean() -> None: + """Delete build artifacts and cache files.""" + paths_to_clean = ["build", "dist", "htmlcov", "site", ".coverage*", ".pdm-build"] + for path in paths_to_clean: + shutil.rmtree(path, ignore_errors=True) + + cache_dirs = { + ".cache", + ".pytest_cache", + ".mypy_cache", + ".ruff_cache", + "__pycache__", + } + for dirpath in Path(".").rglob("*/"): + if dirpath.parts[0] not in (".venv", ".venvs") and dirpath.name in cache_dirs: + shutil.rmtree(dirpath, ignore_errors=True) + + +def vscode() -> None: + """Configure VSCode to work on this project.""" + shutil.copytree("config/vscode", ".vscode", dirs_exist_ok=True) + + +def main() -> int: + """Main entry point.""" + args = list(sys.argv[1:]) + if not args or args[0] == "help": + if len(args) > 1: + run("default", "duty", "--help", args[1]) + else: + print( + dedent( + """ + Available commands + help Print this help. Add task name to print help. + setup Setup all virtual environments (install dependencies). + run Run a command in the default virtual environment. + multirun Run a command for all configured Python versions. + allrun Run a command in all virtual environments. + 3.x Run a command in the virtual environment for Python 3.x. + clean Delete build artifacts and cache files. + vscode Configure VSCode to work on this project. + """, + ), + flush=True, + ) + if os.path.exists(".venv"): + print("\nAvailable tasks", flush=True) + run("default", "duty", "--list") + return 0 + + while args: + cmd = args.pop(0) + + if cmd == "run": + run("default", *args) + return 0 + + if cmd == "multirun": + multirun(*args) + return 0 + + if cmd == "allrun": + allrun(*args) + return 0 + + if cmd.startswith("3."): + run(cmd, *args) + return 0 + + opts = [] + while args and (args[0].startswith("-") or "=" in args[0]): + opts.append(args.pop(0)) + + if cmd == "clean": + clean() + elif cmd == "setup": + setup() + elif cmd == "vscode": + vscode() + elif cmd == "check": + multirun("duty", "check-quality", "check-types", "check-docs") + run("default", "duty", "check-api") + elif cmd in {"check-quality", "check-docs", "check-types", "test"}: + multirun("duty", cmd, *opts) + else: + run("default", "duty", cmd, *opts) + + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except subprocess.CalledProcessError as process: + if process.output: + print(process.output, file=sys.stderr) + sys.exit(process.returncode) diff --git a/accessor-handler/scripts/make.py b/accessor-handler/scripts/make.py new file mode 100755 index 0000000..ff985eb --- /dev/null +++ b/accessor-handler/scripts/make.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +"""Management commands.""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +from contextlib import contextmanager +from pathlib import Path +from textwrap import dedent +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from collections.abc import Iterator + + +PYTHON_VERSIONS = os.getenv("PYTHON_VERSIONS", "3.9 3.10 3.11 3.12 3.13 3.14").split() + + +def shell(cmd: str, *, capture_output: bool = False, **kwargs: Any) -> str | None: + """Run a shell command.""" + if capture_output: + return subprocess.check_output(cmd, shell=True, text=True, **kwargs) # noqa: S602 + subprocess.run(cmd, shell=True, check=True, stderr=subprocess.STDOUT, **kwargs) # noqa: S602 + return None + + +@contextmanager +def environ(**kwargs: str) -> Iterator[None]: + """Temporarily set environment variables.""" + original = dict(os.environ) + os.environ.update(kwargs) + try: + yield + finally: + os.environ.clear() + os.environ.update(original) + + +def uv_install(venv: Path) -> None: + """Install dependencies using uv.""" + with environ( + UV_PROJECT_ENVIRONMENT=str(venv), PYO3_USE_ABI3_FORWARD_COMPATIBILITY="1" + ): + if "CI" in os.environ: + shell("uv sync --no-editable") + else: + shell("uv sync") + + +def setup() -> None: + """Setup the project.""" + if not shutil.which("uv"): + raise ValueError( + "make: setup: uv must be installed, see https://github.com/astral-sh/uv" + ) + + print("Installing dependencies (default environment)") + default_venv = Path(".venv") + if not default_venv.exists(): + shell("uv venv") + uv_install(default_venv) + + if PYTHON_VERSIONS: + for version in PYTHON_VERSIONS: + print(f"\nInstalling dependencies (python{version})") + venv_path = Path(f".venvs/{version}") + if not venv_path.exists(): + shell(f"uv venv --python {version} {venv_path}") + with environ(UV_PROJECT_ENVIRONMENT=str(venv_path.resolve())): + uv_install(venv_path) + + +def run(version: str, cmd: str, *args: str, **kwargs: Any) -> None: + """Run a command in a virtual environment.""" + kwargs = {"check": True, **kwargs} + uv_run = ["uv", "run", "--no-sync"] + if version == "default": + with environ(UV_PROJECT_ENVIRONMENT=".venv"): + subprocess.run([*uv_run, cmd, *args], **kwargs) # noqa: S603, PLW1510 + else: + with environ(UV_PROJECT_ENVIRONMENT=f".venvs/{version}", MULTIRUN="1"): + subprocess.run([*uv_run, cmd, *args], **kwargs) # noqa: S603, PLW1510 + + +def multirun(cmd: str, *args: str, **kwargs: Any) -> None: + """Run a command for all configured Python versions.""" + if PYTHON_VERSIONS: + for version in PYTHON_VERSIONS: + run(version, cmd, *args, **kwargs) + else: + run("default", cmd, *args, **kwargs) + + +def allrun(cmd: str, *args: str, **kwargs: Any) -> None: + """Run a command in all virtual environments.""" + run("default", cmd, *args, **kwargs) + if PYTHON_VERSIONS: + multirun(cmd, *args, **kwargs) + + +def clean() -> None: + """Delete build artifacts and cache files.""" + paths_to_clean = ["build", "dist", "htmlcov", "site", ".coverage*", ".pdm-build"] + for path in paths_to_clean: + shutil.rmtree(path, ignore_errors=True) + + cache_dirs = { + ".cache", + ".pytest_cache", + ".mypy_cache", + ".ruff_cache", + "__pycache__", + } + for dirpath in Path(".").rglob("*/"): + if dirpath.parts[0] not in (".venv", ".venvs") and dirpath.name in cache_dirs: + shutil.rmtree(dirpath, ignore_errors=True) + + +def vscode() -> None: + """Configure VSCode to work on this project.""" + shutil.copytree("config/vscode", ".vscode", dirs_exist_ok=True) + + +def main() -> int: + """Main entry point.""" + args = list(sys.argv[1:]) + if not args or args[0] == "help": + if len(args) > 1: + run("default", "duty", "--help", args[1]) + else: + print( + dedent( + """ + Available commands + help Print this help. Add task name to print help. + setup Setup all virtual environments (install dependencies). + run Run a command in the default virtual environment. + multirun Run a command for all configured Python versions. + allrun Run a command in all virtual environments. + 3.x Run a command in the virtual environment for Python 3.x. + clean Delete build artifacts and cache files. + vscode Configure VSCode to work on this project. + """, + ), + flush=True, + ) + if os.path.exists(".venv"): + print("\nAvailable tasks", flush=True) + run("default", "duty", "--list") + return 0 + + while args: + cmd = args.pop(0) + + if cmd == "run": + run("default", *args) + return 0 + + if cmd == "multirun": + multirun(*args) + return 0 + + if cmd == "allrun": + allrun(*args) + return 0 + + if cmd.startswith("3."): + run(cmd, *args) + return 0 + + opts = [] + while args and (args[0].startswith("-") or "=" in args[0]): + opts.append(args.pop(0)) + + if cmd == "clean": + clean() + elif cmd == "setup": + setup() + elif cmd == "vscode": + vscode() + elif cmd == "check": + multirun("duty", "check-quality", "check-types", "check-docs") + run("default", "duty", "check-api") + elif cmd in {"check-quality", "check-docs", "check-types", "test"}: + multirun("duty", cmd, *opts) + else: + run("default", "duty", cmd, *opts) + + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except subprocess.CalledProcessError as process: + if process.output: + print(process.output, file=sys.stderr) + sys.exit(process.returncode) diff --git a/accessor-handler/src/mkdocstrings_handlers/python_accessors/__init__.py b/accessor-handler/src/mkdocstrings_handlers/python_accessors/__init__.py new file mode 100644 index 0000000..1aa7522 --- /dev/null +++ b/accessor-handler/src/mkdocstrings_handlers/python_accessors/__init__.py @@ -0,0 +1,7 @@ +"""Python handler for mkdocstrings.""" + +from .handler import PythonAccessorHandler + +__all__ = ["get_handler"] + +get_handler = PythonAccessorHandler diff --git a/accessor-handler/src/mkdocstrings_handlers/python_accessors/debug.py b/accessor-handler/src/mkdocstrings_handlers/python_accessors/debug.py new file mode 100644 index 0000000..133d7f4 --- /dev/null +++ b/accessor-handler/src/mkdocstrings_handlers/python_accessors/debug.py @@ -0,0 +1,117 @@ +"""Debugging utilities.""" + +from __future__ import annotations + +import os +import platform +import sys +from dataclasses import dataclass +from importlib import metadata + + +@dataclass +class Variable: + """Dataclass describing an environment variable.""" + + name: str + """Variable name.""" + value: str + """Variable value.""" + + +@dataclass +class Package: + """Dataclass describing a Python package.""" + + name: str + """Package name.""" + version: str + """Package version.""" + + +@dataclass +class Environment: + """Dataclass to store environment information.""" + + interpreter_name: str + """Python interpreter name.""" + interpreter_version: str + """Python interpreter version.""" + interpreter_path: str + """Path to Python executable.""" + platform: str + """Operating System.""" + packages: list[Package] + """Installed packages.""" + variables: list[Variable] + """Environment variables.""" + + +def _interpreter_name_version() -> tuple[str, str]: + if hasattr(sys, "implementation"): + impl = sys.implementation.version + version = f"{impl.major}.{impl.minor}.{impl.micro}" + kind = impl.releaselevel + if kind != "final": + version += kind[0] + str(impl.serial) + return sys.implementation.name, version + return "", "0.0.0" + + +def get_version(dist: str = "mkdocstrings-python-accessors") -> str: + """Get version of the given distribution. + + Parameters + ---------- + dist: A distribution name. + + Returns + ------- + A version number. + """ + try: + return metadata.version(dist) + except metadata.PackageNotFoundError: + return "0.0.0" + + +def get_debug_info() -> Environment: + """Get debug/environment information. + + Returns + ------- + Environment information. + """ + py_name, py_version = _interpreter_name_version() + packages = ["mkdocstrings-python-accessors"] + variables = [ + "PYTHONPATH", + *[var for var in os.environ if var.startswith("MKDOCSTRINGS_PYTHON_ACCESSORS")], + ] + return Environment( + interpreter_name=py_name, + interpreter_version=py_version, + interpreter_path=sys.executable, + platform=platform.platform(), + variables=[Variable(var, val) for var in variables if (val := os.getenv(var))], + packages=[Package(pkg, get_version(pkg)) for pkg in packages], + ) + + +def print_debug_info() -> None: + """Print debug/environment information.""" + info = get_debug_info() + print(f"- __System__: {info.platform}") + print( + f"- __Python__: {info.interpreter_name} {info.interpreter_version} ({info.interpreter_path})" + ) + print("- __Environment variables__:") + for var in info.variables: + print(f" - `{var.name}`: `{var.value}`") + print("- __Installed packages__:") + for pkg in info.packages: + print(f" - `{pkg.name}` v{pkg.version}") + + +if __name__ == "__main__": + print_debug_info() diff --git a/accessor-handler/src/mkdocstrings_handlers/python_accessors/handler.py b/accessor-handler/src/mkdocstrings_handlers/python_accessors/handler.py new file mode 100644 index 0000000..29d4fde --- /dev/null +++ b/accessor-handler/src/mkdocstrings_handlers/python_accessors/handler.py @@ -0,0 +1,51 @@ +""" +Implementation of python_accessors handler +""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +import griffe +from mkdocstrings.loggers import get_logger +from mkdocstrings_handlers.python_xref.handler import PythonRelXRefHandler + +__all__ = ["PythonAccessorHandler"] + +logger = get_logger(__name__) + + +class PythonAccessorHandler(PythonRelXRefHandler): + """ + Extended version of mkdocstrings Python handler + + * Converts references so that they appear in their accessor namespace + """ + + def render(self, data: griffe.Object, config: Mapping[str, Any]) -> str: + if not isinstance(data, griffe.Class): + raise NotImplementedError(data) + + try: + namespace = config["namespace"] + except KeyError: + msg = f"Please specify the namespace to use with {data.name}. {data.path=}" + raise KeyError(msg) + + member_keys = list(data.members.keys()) + for name in member_keys: + if name.startswith("_"): + data.del_member(name) + continue + + member = data.members[name] + member.name = f"{namespace}.{name}" + + data.name = namespace + + try: + return super().render(data, config) + except Exception: # pragma: no cover + print(f"{data.path=}") + raise diff --git a/accessor-handler/src/mkdocstrings_handlers/python_accessors/py.typed b/accessor-handler/src/mkdocstrings_handlers/python_accessors/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/accessor-handler/tests/__init__.py b/accessor-handler/tests/__init__.py new file mode 100644 index 0000000..e68369d --- /dev/null +++ b/accessor-handler/tests/__init__.py @@ -0,0 +1 @@ +"""Tests suite for mkdocstrings-python-accessors.""" diff --git a/accessor-handler/tests/conftest.py b/accessor-handler/tests/conftest.py new file mode 100644 index 0000000..228a6c0 --- /dev/null +++ b/accessor-handler/tests/conftest.py @@ -0,0 +1,116 @@ +"""Configuration for the pytest test suite.""" + +from __future__ import annotations + +from collections import ChainMap +from typing import TYPE_CHECKING, Any + +import pytest +from markdown.core import Markdown +from mkdocs import config +from mkdocs.config.defaults import get_schema + +if TYPE_CHECKING: + from collections.abc import Iterator + from pathlib import Path + + from mkdocstrings.plugin import MkdocstringsPlugin + from mkdocstrings_handlers.python_accessors.handler import PythonHandler + + +@pytest.fixture(name="mkdocs_conf") +def fixture_mkdocs_conf( + request: pytest.FixtureRequest, tmp_path: Path +) -> Iterator[config.Config]: + """Yield a MkDocs configuration object. + + Parameters + ---------- + request: Pytest fixture. + tmp_path: Pytest fixture. + + Yields + ------ + MkDocs config. + """ + conf = config.Config(schema=get_schema()) # type: ignore[call-arg] + while hasattr(request, "_parent_request") and hasattr( + request._parent_request, "_parent_request" + ): + request = request._parent_request + + conf_dict = { + "config_file_path": "mkdocs.yml", + "site_name": "foo", + "site_url": "https://example.org/", + "site_dir": str(tmp_path), + "plugins": [{"mkdocstrings": {"default_handler": "python"}}], + **getattr(request, "param", {}), + } + # Re-create it manually as a workaround for https://github.com/mkdocs/mkdocs/issues/2289 + mdx_configs: dict[str, Any] = dict( + ChainMap(*conf_dict.get("markdown_extensions", [])) + ) + + conf.load_dict(conf_dict) + assert conf.validate() == ([], []) + + conf["mdx_configs"] = mdx_configs + conf["markdown_extensions"].insert(0, "toc") # Guaranteed to be added by MkDocs. + + conf = conf["plugins"]["mkdocstrings"].on_config(conf) + conf = conf["plugins"]["autorefs"].on_config(conf) + yield conf + conf["plugins"]["mkdocstrings"].on_post_build(conf) + + +@pytest.fixture(name="plugin") +def fixture_plugin(mkdocs_conf: config.Config) -> MkdocstringsPlugin: + """Return a plugin instance. + + Parameters + ---------- + mkdocs_conf: Pytest fixture (see conftest.py). + + Returns + ------- + mkdocstrings plugin instance. + """ + return mkdocs_conf["plugins"]["mkdocstrings"] + + +@pytest.fixture(name="ext_markdown") +def fixture_ext_markdown(mkdocs_conf: config.Config) -> Markdown: + """Return a Markdown instance with MkdocstringsExtension. + + Parameters + ---------- + mkdocs_conf: Pytest fixture (see conftest.py). + + Returns + ------- + A Markdown instance. + """ + return Markdown( + extensions=mkdocs_conf["markdown_extensions"], + extension_configs=mkdocs_conf["mdx_configs"], + ) + + +@pytest.fixture(name="handler") +def fixture_handler( + plugin: MkdocstringsPlugin, ext_markdown: Markdown +) -> PythonHandler: + """Return a handler instance. + + Parameters + ---------- + plugin: Pytest fixture (see conftest.py). + + Returns + ------- + A handler instance. + """ + handler = plugin.handlers.get_handler("python") + handler._update_env(ext_markdown, plugin.handlers._config) + return handler # type: ignore[return-value] diff --git a/accessor-handler/tests/test_themes.py b/accessor-handler/tests/test_themes.py new file mode 100644 index 0000000..ab4e481 --- /dev/null +++ b/accessor-handler/tests/test_themes.py @@ -0,0 +1,42 @@ +"""Tests for the different themes we claim to support.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +if TYPE_CHECKING: + from markdown import Markdown + from mkdocstrings.plugin import MkdocstringsPlugin + + +@pytest.mark.parametrize( + "plugin", + [ + {"theme": "mkdocs"}, + {"theme": "readthedocs"}, + {"theme": {"name": "material"}}, + ], + indirect=["plugin"], +) +@pytest.mark.parametrize( + "identifier", + [ + # TODO: add identifiers to this list! + ], +) +def test_render_themes_templates_python( + identifier: str, plugin: MkdocstringsPlugin, ext_markdown: Markdown +) -> None: + """Test rendering of a given theme's templates. + + Parameters: + identifier: Parametrized identifier. + plugin: Pytest fixture (see conftest.py). + ext_markdown: Pytest fixture (see conftest.py). + """ + handler = plugin.handlers.get_handler("python") + handler._update_env(ext_markdown, plugin.handlers._config) + data = handler.collect(identifier, {}) + handler.render(data, {}) diff --git a/docs/NAVIGATION.md b/docs/NAVIGATION.md index 2cc650f..240cec4 100644 --- a/docs/NAVIGATION.md +++ b/docs/NAVIGATION.md @@ -18,5 +18,6 @@ See https://oprypin.github.io/mkdocs-literate-nav/ - [Discrete to continuous conversions](further-background/discrete_to_continuous_conversions.py) - [Representations](further-background/representations.py) - [Development](development.md) +- [Pandas accessors](pandas-accessors.md) - [API reference](api/continuous_timeseries/) - [Changelog](changelog.md) diff --git a/docs/pandas-accessors.md b/docs/pandas-accessors.md new file mode 100644 index 0000000..437d1aa --- /dev/null +++ b/docs/pandas-accessors.md @@ -0,0 +1,42 @@ +# Pandas accessors + +Continuous timeseries also provides a [`pandas`][pandas] accessor. +For details of the implementation of this pattern, see +[pandas' docs](https://pandas.pydata.org/docs/development/extending.html#registering-custom-accessors). + +The accessors must be registered before they can be used +(we do this to avoid imports of any of our modules having side effects, +which is a pattern we have had bad experiences with in the past). +This is done with +[`register_pandas_accessor`][continuous_timeseries.pandas_accessors.register_pandas_accessor], + +By default, the accessors are provided under the "ct" namespace +and this is how the accessors are documented below. +However, the namespace can be customised when using +[`register_pandas_accessor`][continuous_timeseries.pandas_accessors.register_pandas_accessor], +should you wish to use a different namespace for the accessor. + +For the avoidance of doubt, in order to register/activate the accessors, +you will need to run something like: + +```python +from continuous_timeseries.pandas_accessors import register_pandas_accessor + +# The 'pd.DataFrame.ct' namespace will not be available at this point. + +# Register the accessors +register_pandas_accessor() + +# The 'pd.DataFrame.ct' namespace +# (or whatever other custom namespace you chose to register) +# will now be available. +``` + +The full accessor API is documented below. + +::: continuous_timeseries.pandas_accessors.DataFrameCTAccessor + handler: python_accessors + options: + namespace: "pd.DataFrame.ct" + show_root_full_path: false + show_root_heading: true diff --git a/docs/tutorials/pandas_accessor_tutorial.py b/docs/tutorials/pandas_accessor_tutorial.py new file mode 100644 index 0000000..aafff0e --- /dev/null +++ b/docs/tutorials/pandas_accessor_tutorial.py @@ -0,0 +1,429 @@ +# --- +# jupyter: +# jupytext: +# text_representation: +# extension: .py +# format_name: percent +# format_version: '1.3' +# jupytext_version: 1.16.6 +# kernelspec: +# display_name: Python 3 (ipykernel) +# language: python +# name: python3 +# --- + +# %% +import multiprocessing + +import matplotlib.pyplot as plt +import pandas as pd +import pint + +from continuous_timeseries.discrete_to_continuous import InterpolationOption +from continuous_timeseries.timeseries_continuous import get_plot_points +import continuous_timeseries.pandas_accessors + +# %% +UR = pint.get_application_registry() +Q = UR.Quantity + +# %% +UR.setup_matplotlib(enable=True) + +# %% +continuous_timeseries.pandas_accessors.register_pandas_accessor() + +# %% +x = Q([2010, 2015, 2025], "yr") +y_ms = [ + [100.0, 200.0, 350.0], + [-1.5, -0.5, 0.5], +] +idx = pd.MultiIndex.from_tuples( + ( + ("name_1", "World", "Mt /yr"), + ("name_2", "World", "Gt / yr"), + ), + # units not unit to follow pint conventions + names=["name", "region", "units"], +) + +df = pd.DataFrame( + y_ms, + columns=x.m, + index=idx, +) +df + +# %% +series = df.ct.to_timeseries_two( + time_units=x.units, + # interpolation=InterpolationOption.PiecewiseConstantPreviousLeftClosed + interpolation=InterpolationOption.Linear +) +series + +# %% +df.ct.to_timeseries_two( + time_units=x.units, + interpolation=InterpolationOption.PiecewiseConstantPreviousLeftClosed, + progress=True, +) + +# %% +df.ct.to_timeseries_two( + time_units=x.units, + interpolation=InterpolationOption.PiecewiseConstantPreviousLeftClosed, + progress=True, + n_processes=2, + mp_context=multiprocessing.get_context("fork"), +) + +# %% +series = df.ct.to_timeseries( + time_units=x.units, + # interpolation=InterpolationOption.PiecewiseConstantPreviousLeftClosed + interpolation=InterpolationOption.Linear +) +series + +# %% +series.ct.metadata + +# %% +plot_points = get_plot_points(series.iloc[0].time_axis.bounds, res_increase=100) + +# %% +series.ct.interpolate_two(plot_points, progress=False, n_processes=1) + +# %% +series.ct.interpolate_two(plot_points, progress=True, n_processes=1) + +# %% +series.ct.interpolate_two(plot_points, progress=True, n_processes=2) + +# %% +series.ct.interpolate_two( + plot_points, + progress=True, + n_processes=2, + mp_context=multiprocessing.get_context("fork"), +) + +# %% +# Would want to be able to do this in parallel too. +derivative = series.map(lambda x: x.differentiate()) +integral = series.map(lambda x: x.integrate(Q(0.0, "Gt"))) + +# %% +# API to aim for: +# series.ct.differentiate().ct.plot() +# series.loc[pix.isin(variable="Emissions|CO2")].ct.differentiate().ct.plot() + +# %% +# Want basic label control for plotting. +# For everything else, drop out to seaborn. +fig, ax = plt.subplots() +for idx, row in integral.items(): + row.plot(ax=ax, continuous_plot_kwargs=dict(label=idx)) + row.plot(ax=ax) + +ax.legend() + +# %% +# hyperfine plotting control +# filter to something that will all have same units first (use pandas_indexing) +# then interpolate (maybe need to put an `increase_resolution` method on Timeseries) +# then get dataframe with uniform units +# then melt (or maybe just make a 'get_sns_df' accessor that includes a check of units) +# then plot using seaborn + +# %% +plot_points = get_plot_points(series.iloc[0].time_axis.bounds, res_increase=100) +plot_points + +# %% +fine = series.map(lambda x: x.interpolate(plot_points)) +fine = integral.map(lambda x: x.interpolate(plot_points)) + +# %% +import numpy as np +time_units = "yr" +out_units = "Mt" + +tmp_l = [] +for i, (k, v) in enumerate(fine.items()): + # .to_series() for individual Timeseries + discrete = v.discrete + columns = discrete.time_axis.bounds.to(time_units).m + values = discrete.values_at_bounds.values + if out_units: + values = values.to(out_units) + units = out_units + else: + units = str(values.u) + + tmp = pd.Series( + values.m, + index=columns, + ) + # end of .to_series() for individual Timeseries + + # Join on metadata and convert to DataFrame + index = pd.MultiIndex.from_tuples( + ((*k, units),), + names=[*fine.index.names, "units"] + ) + tmp = pd.DataFrame(tmp, columns=index).T + + tmp_l.append(tmp) + +# Create the result (can use pd.concat +# as we're guaranteed to have the index in the right order) +res = pd.concat(tmp_l) +res + +# %% +import seaborn as sns + +# %% +sns_df = res.melt( + var_name="time", + ignore_index=False, +).reset_index() +sns_df + +# %% +sns.lineplot( + data=sns_df, + x="time", + y="value", + hue="name", + style="region", +) + +# %% +n_variables = 3 +n_yrs = 250 +n_runs = 10 +n_scenarios = 5 + +n_variables = 1 +n_yrs = 550 +n_runs = 600 +n_scenarios = 100 + +n_variables = 1 +n_yrs = 125 +n_runs = 600 +n_scenarios = 1000 + +# # Too big, not really possible to do in memory +# # (at this scale, probably would never use continuous timeseries anyway) +# n_variables = 10 +# n_yrs = 550 +# n_runs = 600 +# n_scenarios = 2000 + +# %% +x = Q(np.arange(n_yrs) + 1750, "yr") +x + +# %% +y_ms = np.random.random((n_variables * n_runs * n_scenarios, n_yrs)) +y_ms.shape + +# %% +import itertools + +# %% +idx = pd.MultiIndex.from_frame( + pd.DataFrame( + ( + (s, v, r, "Mt / yr") + for s, v, r in itertools.product( + [f"variable_{i}" for i in range(n_variables)], + [f"scenario_{i}" for i in range(n_scenarios)], + [i for i in range(n_runs)], + ) + ), + columns=["scenario", "variable", "region", "units"], + # This makes updates later way way faster + dtype="category" + ) +) +idx + +# %% +df = pd.DataFrame( + y_ms, + columns=x.m, + index=idx, +) +df + +# %% +# tmp.index.set_levels(tmp.index.get_level_values("units").map( +# lambda x: str(UR.Unit(x) * UR.Unit("yr")) +# ), level="units") + +# %% +df.index + +# %% +# TODO: move to some sort of pandas tricks module. +# This is a much faster way of doing integration +# if you have a piecewise constant assumption. +# You obviously can't use this for extrapolation. +time_steps = df.columns.values[1:] - df.columns.values[:-1] +integration_constant = 10.0 + +# Previous piecewise constant +tmp = df.iloc[:, :-1] * time_steps + integration_constant +tmp.columns = df.columns[1:] +tmp[df.columns[0]] = integration_constant + +# # Next piecewise constant +# tmp = df.iloc[:, 1:] * time_steps + integration_constant +# tmp[df.columns[0]] = integration_constant + +# Result +# If your index is categorical, this is fast. +# If not, its super slow. +# # Use this to help print user warnings +# isinstance(df.index.get_level_values("units"), pd.CategoricalIndex) +tmp.index = tmp.index.set_levels( + tmp.index.get_level_values("units").categories.map( + lambda x: str(UR.Unit(x) * UR.Unit("yr")) + ), + level="units" +) + +tmp = tmp.sort_index(axis="columns") +tmp + +# %% +# TODO: drop nans when converting +# TODO: test with a dataframe that has history and scenario, but no overlap + +# %% +# %%time +series_h = df.ct.to_timeseries_two( + time_units=x.units, + interpolation=InterpolationOption.PiecewiseConstantPreviousLeftClosed, + progress=True, + n_processes=multiprocessing.cpu_count(), + # Fork is super important for progress bars to work, + # would suggest making it default (windows, whatever...) + mp_context=multiprocessing.get_context("fork"), +) + +# %% +# %%time +series_h = df.ct.to_timeseries_two( + time_units=x.units, + interpolation=InterpolationOption.PiecewiseConstantPreviousLeftClosed, + progress=True, + n_processes=1, +) + +# %% +# Must be something smarter that can be done with chunking to make this faster, anyway +series_h = df.ct.to_timeseries( + time_units=x.units, + interpolation=InterpolationOption.Linear, + progress=True, + n_processes=multiprocessing.cpu_count(), + mp_context=multiprocessing.get_context("fork"), +) + +# %% +series_h.ct.integrate( + Q(0, "Mt"), + progress=True, + n_processes=1, +) + +# %% +# series_h.ct.integrate( +# Q(0, "Mt"), +# progress=True, +# n_processes=multiprocessing.cpu_count(), +# ) + +# %% +series_h.ct.integrate( + Q(0, "Mt"), + progress=True, + # n_processes=multiprocessing.cpu_count(), + n_processes=3, + mp_context=multiprocessing.get_context("fork"), +) + +# %% +interp_points = get_plot_points(series_h.iloc[0].time_axis.bounds, res_increase=100) + +# %% +series_h.ct.interpolate_two(interp_points, progress=True, n_processes=1) + +# %% +# series_h.ct.interpolate_two( +# interp_points, +# progress=True, +# n_processes=multiprocessing.cpu_count(), +# ) + +# %% +# %%time +series_h.ct.interpolate_two( + interp_points, + progress=False, + n_processes=1, +) + +# %% +# %%time +series_h.ct.interpolate_two( + interp_points, + progress=False, + n_processes=multiprocessing.cpu_count() + 4, + mp_context=multiprocessing.get_context("fork"), +) + +# %% +series_h.ct.interpolate_two( + interp_points, + progress=True, + n_processes=multiprocessing.cpu_count(), + mp_context=multiprocessing.get_context("fork"), +) + +# %% +# # It could also be something to do with pint that makes the parallel processing slow... +# # Some links to think about re parallelisation: +# # - https://medium.com/@codewithnazam/pandas-in-a-parallel-universe-speeding-up-your-data-adventures-7696aa00eab8 +# # - https://pypi.org/project/parallel-pandas/ +# # - https://towardsdatascience.com/easily-parallelize-your-calculations-in-pandas-with-parallel-pandas-dc194b82d82f +# # - https://github.com/nalepae/pandarallel +# series_h.ct.interpolate(interp_points, progress=True, n_processes=multiprocessing.cpu_count()) + +# %% +series_h.ct.interpolate(interp_points, progress=True, n_processes=1) + +# %% +series_h.ct.interpolate(interp_points, progress=False, n_processes=multiprocessing.cpu_count()) + +# %% +series_h.ct.interpolate(interp_points, progress=False) + +# %% +# df.ct.to_timeseries(time_units=x.units, interpolation=InterpolationOption.Linear, progress=True, n_processes=multiprocessing.cpu_count(), mp_context=multiprocessing.get_context("spawn")) + +# %% +df.ct.to_timeseries(time_units=x.units, interpolation=InterpolationOption.Linear, progress=True, n_processes=1) + +# %% +df.ct.to_timeseries(time_units=x.units, interpolation=InterpolationOption.Linear, n_processes=multiprocessing.cpu_count()) + +# %% +df.ct.to_timeseries(time_units=x.units, interpolation=InterpolationOption.Linear, n_processes=1) diff --git a/docs/tutorials/scratch.py b/docs/tutorials/scratch.py new file mode 100644 index 0000000..454e6eb --- /dev/null +++ b/docs/tutorials/scratch.py @@ -0,0 +1,83 @@ +# --- +# jupyter: +# jupytext: +# text_representation: +# extension: .py +# format_name: percent +# format_version: '1.3' +# jupytext_version: 1.16.6 +# kernelspec: +# display_name: Python 3 (ipykernel) +# language: python +# name: python3 +# --- + +# %% + +import time +import random +from tqdm.auto import tqdm +from multiprocessing import Pool, freeze_support, RLock, cpu_count, set_start_method +# from multiprocessing import set_start_method + +def func(pid, n): + + print(end="\r") + tqdm_text = "#" + "{}".format(pid).zfill(3) + + current_sum = 0 + with tqdm(total=n, desc=tqdm_text, position=pid+1) as pbar: + for i in range(1, n+1): + current_sum += i + time.sleep(0.05) + pbar.update(1) + + return current_sum + +# num_processes = cpu_count() +num_jobs = 10 +random_seed = 0 +random.seed(random_seed) +# set_start_method("fork") + +pool = Pool(processes=50, initargs=(RLock(),), initializer=tqdm.set_lock) + +argument_list = [random.randint(0, 100) for _ in range(num_jobs)] + +jobs = [pool.apply_async(func, args=(i,n,)) for i, n in enumerate(argument_list)] +pool.close() +results =[] +for job in tqdm(jobs, desc='Outer loop'): + results.append(job.get()) + +#print("\n" * (len(argument_list) + 1)) + + +# %% +from multiprocessing import Pool, RLock +from time import sleep + +from tqdm.auto import tqdm, trange + +from continuous_timeseries.scratch import progresser + +L = list(range(9)) + + +# def progresser(n): +# interval = 0.001 / (n + 2) +# total = 5000 +# text = f"#{n}, est. {interval * total:<04.2}s" +# for _ in trange(total, desc=text, position=n): +# sleep(interval) + + + + +# %% +tqdm.set_lock(RLock()) +p = Pool(initializer=tqdm.set_lock, initargs=(tqdm.get_lock(),)) +p.map(progresser, L) + + +# %% diff --git a/docs/tutorials/scratch2.py b/docs/tutorials/scratch2.py new file mode 100644 index 0000000..abd15c2 --- /dev/null +++ b/docs/tutorials/scratch2.py @@ -0,0 +1,40 @@ +# --- +# jupyter: +# jupytext: +# text_representation: +# extension: .py +# format_name: percent +# format_version: '1.3' +# jupytext_version: 1.16.6 +# kernelspec: +# display_name: Python 3 (ipykernel) +# language: python +# name: python3 +# --- + +# %% +import concurrent.futures +import multiprocessing + +from tqdm.auto import tqdm + +from continuous_timeseries.scratch import progresser + +# %% +L = list(range(9)) +n_processes = 4 + +# %% +mp_context = multiprocessing.get_context("fork") +with concurrent.futures.ProcessPoolExecutor( + max_workers=n_processes, mp_context=mp_context +) as pool: + futures = [pool.submit(progresser, n) for n in tqdm(L, desc="submitting to pool")] + res = [ + future.result() + for future in tqdm( + concurrent.futures.as_completed(futures), + desc="Retrieving parallel results", + total=len(futures), + ) + ] diff --git a/mkdocs.yml b/mkdocs.yml index be1761b..9fc8ca1 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -68,7 +68,7 @@ plugins: default_handler: python_xref enable_inventory: true handlers: - python_xref: + python_xref: &pythonhandleroptions paths: [src] import: # Cross-ref helpers (lots included here, remove what you don't want) @@ -102,6 +102,7 @@ plugins: classes: true functions: true modules: true + python_accessors: *pythonhandleroptions # https://squidfunk.github.io/mkdocs-material/plugins/search/ - search # Add clickable sections to the sidebar diff --git a/pyproject.toml b/pyproject.toml index 899ae3d..01c7942 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,9 @@ classifiers = [ ] [project.optional-dependencies] +pandas = [ + "pandas>=2.0.0", +] plots = [ "matplotlib>=3.7.1", ] @@ -25,6 +28,7 @@ scipy = [ "scipy>=1.13.1", ] full = [ + "continuous-timeseries[pandas]", "continuous-timeseries[plots]", "continuous-timeseries[scipy]", ] @@ -51,6 +55,7 @@ docs = [ "mkdocs-material==9.5.49", "mkdocs-section-index==0.3.9", "mkdocs==1.6.1", + "mkdocstrings-python-accessors", "mkdocstrings-python-xref==1.6.2", "mkdocstrings-python==1.13.0", "openscm-units>=0.6.3", @@ -61,6 +66,9 @@ docs = [ "jupyterlab==4.3.4", "jupytext==1.16.6", "mkdocs-jupyter==0.25.1", + "seaborn>=0.13.2", + "tqdm>=4.67.1", + "ipywidgets>=8.1.5", ] tests = [ "ipython>=8.18.1", @@ -76,6 +84,9 @@ all-dev = [ {include-group = "tests"}, ] +# Not sure what the pdm equivalent of this is +[tool.uv.sources] +mkdocstrings-python-accessors = { path = "./accessor-handler", editable = true } [build-system] requires = [ diff --git a/requirements-docs-locked.txt b/requirements-docs-locked.txt index 4cfd400..dc0ef89 100644 --- a/requirements-docs-locked.txt +++ b/requirements-docs-locked.txt @@ -1,5 +1,6 @@ # This file was autogenerated by uv via the following command: # uv export -o requirements-docs-locked.txt --no-hashes --no-dev --no-emit-project --all-extras --group docs +-e ./accessor-handler anyio==4.8.0 appnope==0.1.4 ; sys_platform == 'darwin' argon2-cffi==23.1.0 diff --git a/requirements-incl-optional-locked.txt b/requirements-incl-optional-locked.txt index c9622cb..08c7c2a 100644 --- a/requirements-incl-optional-locked.txt +++ b/requirements-incl-optional-locked.txt @@ -12,13 +12,16 @@ matplotlib==3.9.4 numpy==2.0.2 ; python_full_version < '3.10' numpy==2.2.0 ; python_full_version >= '3.10' packaging==24.2 +pandas==2.2.3 pillow==11.0.0 pint==0.24.4 platformdirs==4.3.6 pyparsing==3.2.0 python-dateutil==2.9.0.post0 +pytz==2024.2 scipy==1.13.1 ; python_full_version < '3.10' scipy==1.14.1 ; python_full_version >= '3.10' six==1.17.0 typing-extensions==4.12.2 +tzdata==2024.2 zipp==3.21.0 ; python_full_version < '3.10' diff --git a/scratch.py b/scratch.py new file mode 100644 index 0000000..5177ee3 --- /dev/null +++ b/scratch.py @@ -0,0 +1,170 @@ +# --- +# jupyter: +# jupytext: +# text_representation: +# extension: .py +# format_name: percent +# format_version: '1.3' +# jupytext_version: 1.16.6 +# kernelspec: +# display_name: Python 3 (ipykernel) +# language: python +# name: python3 +# --- + +# %% +import matplotlib.pyplot as plt +import pint +from attrs import define + +import continuous_timeseries as ct +from continuous_timeseries import InterpolationOption, Timeseries + +# %% +UR = pint.UnitRegistry() +Q = UR.Quantity + +# %% +UR.setup_matplotlib(enable=True) + +# %% +x = Q([2009, 2010, 2011, 2012, 2013, 2014], "yr") +ts = Timeseries.from_arrays( + x=x, + y=Q([1.0, 1.0, 2.0, 3.0, 1.5, 2.0], "kg"), + interpolation=InterpolationOption.PiecewiseConstantPreviousLeftOpen, + name="start", +) +ts.plot() + +# %% +x_month = Q( + [y + m / 12 for y in range(x.m.min(), x.m.max()) for m in range(12)] + [x.m.max()], + "yr", +) +x_month + + +# %% +@define +class Mean: + values: int + bounds: int + + +# %% +def mean(ts, out_bounds): + tmp = 0.0 * ( + ts.timeseries_continuous.values_units * ts.timeseries_continuous.time_units + ) + integral = ts.interpolate(out_bounds).integrate(tmp) + + integral_discrete = integral.discrete.values_at_bounds.values + integral_per_window = integral_discrete[1:] - integral_discrete[:-1] + + size_of_windows = out_bounds[1:] - out_bounds[:-1] + mean_per_window = integral_per_window / size_of_windows + + return Mean(values=mean_per_window, bounds=out_bounds) + + +# %% +mean(ts, x_month).values + +# %% +ts_interp = ts.update_interpolation_integral_preserving(InterpolationOption.Quadratic) + +# %% +ts_interp_lin = ts.update_interpolation_integral_preserving(InterpolationOption.Linear) + +# %% +ts_interp_lin_mean = mean(ts_interp_lin, out_bounds=x_month) + +# %% +mean(ts_interp_lin, x) + +# %% +tmp_mean = mean(ts_interp, x_month) + +# %% +xc = ts.integrate(Q(0, "kg yr")).discrete.time_axis.bounds[0:] +yc = ts.integrate(Q(0, "kg yr")).discrete.values_at_bounds.values[0:] + +# %% +import scipy.interpolate + +# %% +cubic_spline = scipy.interpolate.CubicSpline( + x=xc.m, + y=yc.m, + # Scipy's docs on this are very helpful. + # Here, we are basically saying, + # make the first derivative at either boundary equal to zero. + # For our data, this makes sense. + # For other data, a different choice chould be better + # (e.g. we often use `bc_type=((1, 0.0), "not-a-knot")` + # which means, have a first derivative of zero on the left, + # on the right, + # just use the same polynomial for the last two time steps).) + bc_type=((1, 1.0), (2, 0.0)), +) + +custom = ct.Timeseries( + time_axis=ct.TimeAxis(xc), + timeseries_continuous=ct.TimeseriesContinuouss( + name="custom_spline", + time_units=xc.u, + values_units=yc.u, + function=ct.timeseries_continuous.ContinuousFunctionScipyPPoly(cubic_spline), + domain=(xc.min(), xc.max()), + ), +) +custom + +# %% +custom_der = custom.differentiate() + +# %% +fig, ax = plt.subplots() + +ts.plot(ax=ax) +# ts_interp.plot(ax=ax) +# ts_interp_lin.plot(ax=ax) +custom_der.plot(ax=ax) +# ax.scatter( +# (tmp_mean.bounds[1:] + tmp_mean.bounds[:-1]) / 2.0, +# tmp_mean.values, +# ) +# ax.scatter( +# (ts_interp_lin_mean.bounds[1:] + ts_interp_lin_mean.bounds[:-1]) / 2.0, +# ts_interp_lin_mean.values, +# ) + +ax.grid() + +# %% +mean(ts.update_interpolation_integral_preserving(InterpolationOption.Quadratic), x) + +# %% +tmp = ( + ts.update_interpolation_integral_preserving(InterpolationOption.Quadratic) + .interpolate(x_month) + .integrate(Q(0, "kg yr")) + .discrete.values_at_bounds.values +) +finer = (tmp[1:] - tmp[:-1]) / (x_month[1:] - x_month[:-1]) +finer + +# %% +finer[12:24].sum() / 12 + +# %% +finer[:12].sum() / 12 + +# %% +finer[24:36].sum() / 12 + +# %% +finer[36:48].sum() / 12 + +# %% diff --git a/scripts/scratch.py b/scripts/scratch.py new file mode 100644 index 0000000..61a418b --- /dev/null +++ b/scripts/scratch.py @@ -0,0 +1,20 @@ +from multiprocessing import Pool, RLock +from time import sleep + +from tqdm.auto import tqdm, trange + +L = list(range(9))[::-1] + + +def progresser(n): + interval = 0.005 / (n + 2) + total = 5000 + text = f"#{n}, est. {interval * total:<04.2}s" + for _ in trange(total, desc=text, position=n): + sleep(interval) + + +if __name__ == "__main__": + tqdm.set_lock(RLock()) + p = Pool(processes=4, initializer=tqdm.set_lock, initargs=(tqdm.get_lock(),)) + p.map(progresser, L) diff --git a/src/continuous_timeseries/pandas_accessors.py b/src/continuous_timeseries/pandas_accessors.py new file mode 100644 index 0000000..3bb6d4b --- /dev/null +++ b/src/continuous_timeseries/pandas_accessors.py @@ -0,0 +1,706 @@ +""" +Accessors for pandas + +TODO: +- convert_unit accessor too + - allow passing loc to only affect part of the DF (no need for mapping) + - groupby units + - allow parallelisation + - use __finalize__ (also in other methods) +""" + +from __future__ import annotations + +import concurrent.futures +from collections.abc import Iterable +from multiprocessing.context import BaseContext +from typing import TYPE_CHECKING, Callable, TypeVar + +import numpy as np +import pint +from typing_extensions import Concatenate, ParamSpec + +from continuous_timeseries.discrete_to_continuous import InterpolationOption +from continuous_timeseries.exceptions import MissingOptionalDependencyError +from continuous_timeseries.time_axis import TimeAxis +from continuous_timeseries.timeseries import Timeseries +from continuous_timeseries.typing import PINT_NUMPY_ARRAY, PINT_SCALAR + +if TYPE_CHECKING: + import pandas as pd + import pint.facets.plain + + +def validate(df: pd.DataFrame) -> None: + """ + Validate the provided data can be used + + Parameters + ---------- + df + Data to validate + + Raises + ------ + CTAccessorUnsupportedError + `df` is not supported by continuous timeseries' pandas accessors. + """ + + +P = ParamSpec("P") +T = TypeVar("T") +U = TypeVar("U") + + +def get_executor_and_futures( + in_iter: Iterable[U], + func_to_call: Callable[Concatenate[U, P], T], + n_processes: int, + mp_context: BaseContext | None = None, + progress: bool = False, + *args, + **kwargs, +) -> tuple[T, ...]: + if progress: + try: + from tqdm.auto import tqdm + except ImportError as exc: + raise MissingOptionalDependencyError( + "to_timeseries", requirement="pandas" + ) from exc + + iterator = tqdm(in_iter, desc="submitting to parallel executor") + + else: + iterator = in_iter + + executor = concurrent.futures.ProcessPoolExecutor( + max_workers=n_processes, mp_context=mp_context + ) + + futures = tuple( + executor.submit( + func_to_call, + inv, + *args, + **kwargs, + ) + for inv in iterator + ) + + return executor, futures + + +def interpolate_parallel_helper( + in_v: tuple[tuple[str | float | int, ...], Timeseries], + time_axis: TimeAxis | PINT_NUMPY_ARRAY, + allow_extrapolation: bool = False, +) -> Timeseries: + return in_v[0], in_v[1].interpolate( + time_axis=time_axis, allow_extrapolation=allow_extrapolation + ) + + +def interpolate_parallel_helper_two( + series: pd.Series, + time_axis: TimeAxis | PINT_NUMPY_ARRAY, + allow_extrapolation: bool = False, + progress: bool = False, + progress_bar_position: int = 0, +) -> pd.Series[Timeseries]: + if progress: + try: + from tqdm.auto import tqdm + except ImportError as exc: + raise MissingOptionalDependencyError( # noqa: TRY003 + "interpolate(..., progress=True)", requirement="tdqm" + ) from exc + + tqdm_kwargs = dict(position=progress_bar_position) + tqdm.pandas(**tqdm_kwargs) + meth_to_call = "progress_map" + # No-one knows why this is needed, but it is + print(end=" ") + + else: + meth_to_call = "map" + + res = getattr(series, meth_to_call)( + lambda x: x.interpolate( + time_axis=time_axis, + allow_extrapolation=all, + ) + ) + + return res + + +def integrate_parallel_helper( + series: pd.Series, + integration_constant: PINT_SCALAR, + progress: bool = False, + progress_bar_position: int = 0, +) -> pd.Series[Timeseries]: + if progress: + try: + from tqdm.auto import tqdm + except ImportError as exc: + raise MissingOptionalDependencyError( # noqa: TRY003 + "interpolate(..., progress=True)", requirement="tdqm" + ) from exc + + tqdm_kwargs = dict(position=progress_bar_position) + tqdm.pandas(**tqdm_kwargs) + meth_to_call = "progress_map" + # No-one knows why this is needed, but it is + print(end=" ") + + else: + meth_to_call = "map" + + res = getattr(series, meth_to_call)( + lambda x: x.integrate(integration_constant=integration_constant) + ) + + return res + + +class SeriesCTAccessor: + """ + [`pd.Series`][pandas.Series] accessors + + For details, see + [pandas' docs](https://pandas.pydata.org/docs/development/extending.html#registering-custom-accessors). + """ + + def __init__(self, pandas_obj: pd.Series): + """ + Initialise + + Parameters + ---------- + pandas_obj + Pandas object to use via the accessors + """ + # TODO: add validation + # validate_series(pandas_obj) + self._series = pandas_obj + + @property + def metadata(self) -> pd.DataFrame: + """ + Get the metadata + """ + return self._series.index.to_frame(index=False) + + def interpolate( + self, + time_axis: TimeAxis | PINT_NUMPY_ARRAY, + allow_extrapolation: bool = False, + progress: bool = False, + n_processes: int = 1, + mp_context: BaseContext | None = None, + ) -> pd.Series[Timeseries]: + # Late import to avoid hard dependency on pandas + try: + import pandas as pd + except ImportError as exc: + raise MissingOptionalDependencyError( + "interpolate", requirement="pandas" + ) from exc + + if n_processes == 1: + if progress: + try: + from tqdm.auto import tqdm + except ImportError as exc: + raise MissingOptionalDependencyError( # noqa: TRY003 + "interpolate(..., progress=True)", requirement="tdqm" + ) from exc + + tqdm.pandas(desc="timeseries") + res = self._series.progress_map( + lambda x: x.interpolate( + time_axis=time_axis, + allow_extrapolation=all, + ) + ) + + else: + res = self._series.map( + lambda x: x.interpolate( + time_axis=time_axis, + allow_extrapolation=all, + ) + ) + + return res + + executor, futures = get_executor_and_futures( + tuple(v for v in self._series.items()), + interpolate_parallel_helper, + n_processes=n_processes, + mp_context=mp_context, + progress=progress, + time_axis=time_axis, + allow_extrapolation=all, + ) + iterator = concurrent.futures.as_completed(futures) + + if progress: + try: + from tqdm.auto import tqdm + except ImportError as exc: + raise MissingOptionalDependencyError( # noqa: TRY003 + "interpolate(..., progress=True)", requirement="tqdm" + ) from exc + + iterator = tqdm(iterator, desc="timeseries", total=self._series.size) + + try: + res_l = tuple(future.result() for future in iterator) + finally: + executor.shutdown() + + res = pd.Series( + (v[1] for v in res_l), + pd.MultiIndex.from_tuples( + (v[0] for v in res_l), names=self._series.index.names + ), + ) + return res + + def integrate( + self, + integration_constant: PINT_SCALAR, + progress: bool = False, + n_processes: int = 1, + mp_context: BaseContext | None = None, + ) -> pd.Series[Timeseries]: + if n_processes == 1: + res = integrate_parallel_helper( + self._series, + integration_constant=integration_constant, + progress=progress, + ) + + return res + + # TODO: split this out into `chunk_series` + # Not sure if there is a smarter way to do this, anyway + chunk_size = int(np.ceil(self._series.size / n_processes)) + chunks = [] + for i in range(n_processes): + start = i * chunk_size + end = (i + 1) * chunk_size + if end >= self._series.size: + end = None + + chunks.append(self._series[start:end]) + + iterator = chunks + if progress: + try: + from tqdm.auto import tqdm + except ImportError as exc: + raise MissingOptionalDependencyError( # noqa: TRY003 + "interpolate(..., progress=True)", requirement="tdqm" + ) from exc + + iterator = tqdm(iterator, desc="submitting to pool") + + with concurrent.futures.ProcessPoolExecutor( + max_workers=n_processes, mp_context=mp_context + ) as pool: + futures = [ + pool.submit( + integrate_parallel_helper, + chunk, + integration_constant=integration_constant, + progress=progress, + progress_bar_position=i, + ) + for i, chunk in enumerate(iterator) + ] + + iterator_results = concurrent.futures.as_completed(futures) + if progress: + iterator_results = tqdm( + iterator_results, + desc="Retrieving parallel results", + total=len(futures), + ) + + res_l = [future.result() for future in iterator_results] + + # Late import to avoid hard dependency on pandas + try: + import pandas as pd + except ImportError as exc: + raise MissingOptionalDependencyError( + "interpolate", requirement="pandas" + ) from exc + + res = pd.concat(res_l) + return res + + def interpolate_two( + self, + time_axis: TimeAxis | PINT_NUMPY_ARRAY, + allow_extrapolation: bool = False, + progress: bool = False, + n_processes: int = 1, + mp_context: BaseContext | None = None, + ) -> pd.Series[Timeseries]: + if n_processes == 1: + res = interpolate_parallel_helper_two( + self._series, + time_axis=time_axis, + allow_extrapolation=all, + progress=progress, + ) + + return res + + # TODO: split this out into `chunk_series` + # Not sure if there is a smarter way to do this, anyway + chunk_size = int(np.ceil(self._series.size / n_processes)) + chunks = [] + for i in range(n_processes): + start = i * chunk_size + end = (i + 1) * chunk_size + if end >= self._series.size: + end = None + + chunks.append(self._series[start:end]) + + iterator = chunks + if progress: + try: + from tqdm.auto import tqdm + except ImportError as exc: + raise MissingOptionalDependencyError( # noqa: TRY003 + "interpolate(..., progress=True)", requirement="tdqm" + ) from exc + + iterator = tqdm(iterator, desc="submitting to pool") + + with concurrent.futures.ProcessPoolExecutor( + max_workers=n_processes, mp_context=mp_context + ) as pool: + futures = [ + pool.submit( + interpolate_parallel_helper_two, + chunk, + time_axis=time_axis, + allow_extrapolation=allow_extrapolation, + progress=progress, + progress_bar_position=i, + ) + for i, chunk in enumerate(iterator) + ] + + iterator_results = concurrent.futures.as_completed(futures) + if progress: + iterator_results = tqdm( + iterator_results, + desc="Retrieving parallel results", + total=len(futures), + ) + + res_l = [future.result() for future in iterator_results] + + # Late import to avoid hard dependency on pandas + try: + import pandas as pd + except ImportError as exc: + raise MissingOptionalDependencyError( + "interpolate", requirement="pandas" + ) from exc + + res = pd.concat(res_l) + return res + + +def get_timeseries_parallel_helper(inv, units_idx: int, *args, **kwargs): + return [ + v for i, v in enumerate(inv[0]) if i != units_idx + ], Timeseries.from_pandas_iterrows_value(inv, units_idx=units_idx, *args, **kwargs) + + +def get_timeseries_parallel_helper_two( + df: pd.DataFrame, + interpolation: InterpolationOption, + time_units: str | pint.facets.plain.PlainUnit, + units_col: str, + idx_separator: str, + ur: pint.facets.PlainRegistry | None = None, + progress: bool = False, + progress_bar_position: int = 0, +) -> pd.Series[Timeseries]: + if progress: + try: + from tqdm.auto import tqdm + except ImportError as exc: + raise MissingOptionalDependencyError( # noqa: TRY003 + "interpolate(..., progress=True)", requirement="tdqm" + ) from exc + + tqdm_kwargs = dict(position=progress_bar_position) + tqdm.pandas(**tqdm_kwargs) + meth_to_call = "progress_apply" + # No-one knows why this is needed, but it is + print(end=" ") + + else: + meth_to_call = "apply" + + # TODO: move to validation + try: + units_idx = df.index.names.index(units_col) + except ValueError as exc: + msg = f"{units_col} not available. {df.index.names=}" + + raise KeyError(msg) from exc + + res = getattr(df, meth_to_call)( + Timeseries.from_pandas_series, + axis="columns", + interpolation=interpolation, + units_idx=units_idx, + time_units=time_units, + # name="injectable?", + idx_separator=idx_separator, + ur=ur, + ) + + return res + + +class DataFrameCTAccessor: + """ + [`pd.DataFrame`][pandas.DataFrame] accessors + + For details, see + [pandas' docs](https://pandas.pydata.org/docs/development/extending.html#registering-custom-accessors). + """ + + def __init__(self, pandas_obj: pd.DataFrame): + """ + Initialise + + Parameters + ---------- + pandas_obj + Pandas object to use via the accessors + """ + validate(pandas_obj) + self._df = pandas_obj + + # # This is how you do a property, should we ever need it + # @property + # def data(self) -> pd.DataFrame: + # """ + # Get data + # """ + # return self._obj + + def to_timeseries( # noqa: PLR0913 + self, + time_units: str | pint.facets.plain.PlainUnit, + interpolation: InterpolationOption, + units_col: str = "units", + ur: None = None, + idx_separator: str = "__", + progress: bool = False, + n_processes: int = 1, + mp_context: BaseContext | None = None, + ) -> pd.Series[Timeseries]: + # Late import to avoid hard dependency on pandas + try: + import pandas as pd + except ImportError as exc: + raise MissingOptionalDependencyError( + "to_timeseries", requirement="pandas" + ) from exc + + if isinstance(time_units, str): + raise NotImplementedError + + if ur is None: + ur = pint.get_application_registry() + + df = self._df + + # TODO: move to validation + try: + units_idx = df.index.names.index(units_col) + except ValueError as exc: + msg = f"{units_col} not available. {df.index.names=}" + + raise KeyError(msg) from exc + + if n_processes == 1: + iterator = df.iterrows() + + else: + units_idx = df.index.names.index(units_col) + executor, futures = get_executor_and_futures( + tuple(v for v in df.iterrows()), + get_timeseries_parallel_helper, + n_processes=n_processes, + mp_context=mp_context, + progress=progress, + interpolation=InterpolationOption.Linear, + units_idx=units_idx, + time_units="yr", + ) + iterator = concurrent.futures.as_completed(futures) + + if progress: + try: + from tqdm.auto import tqdm + except ImportError as exc: + raise MissingOptionalDependencyError( + "to_timeseries", requirement="pandas" + ) from exc + + iterator = tqdm(iterator, desc="rows", total=df.shape[0]) + + if n_processes == 1: + res = tuple( + get_timeseries_parallel_helper( + v, + interpolation=InterpolationOption.Linear, + units_idx=units_idx, + time_units="yr", + ) + for v in iterator + ) + + else: + try: + res = tuple(future.result() for future in iterator) + finally: + executor.shutdown() + + res_index_names = [v for v in df.index.names if v != units_col] + res = pd.Series( + (v[1] for v in res), + pd.MultiIndex.from_tuples((v[0] for v in res), names=res_index_names), + name="ts", + ) + + return res + + def to_timeseries_two( # noqa: PLR0913 + self, + interpolation: InterpolationOption, + time_units: str | pint.facets.plain.PlainUnit, + units_col: str = "units", + ur: None = None, + idx_separator: str = "__", + res_name: str = "ts", + progress: bool = False, + n_processes: int = 1, + mp_context: BaseContext | None = None, + ) -> pd.Series[Timeseries]: + if n_processes == 1: + res = get_timeseries_parallel_helper_two( + self._df, + interpolation=interpolation, + time_units=time_units, + units_col=units_col, + idx_separator=idx_separator, + ur=ur, + progress=progress, + ) + + return res + + # TODO: split this out into `chunk_series` + # Not sure if there is a smarter way to do this, anyway + chunk_size = int(np.ceil(self._df.shape[0] / n_processes)) + chunks = [] + for i in range(n_processes): + start = i * chunk_size + end = (i + 1) * chunk_size + if end >= self._df.shape[0]: + end = None + + chunks.append(self._df.iloc[start:end, :]) + + iterator = chunks + if progress: + try: + from tqdm.auto import tqdm + except ImportError as exc: + raise MissingOptionalDependencyError( # noqa: TRY003 + "interpolate(..., progress=True)", requirement="tdqm" + ) from exc + + iterator = tqdm(iterator, desc="submitting to pool") + + with concurrent.futures.ProcessPoolExecutor( + max_workers=n_processes, mp_context=mp_context + ) as pool: + futures = [ + pool.submit( + get_timeseries_parallel_helper_two, + chunk, + interpolation=interpolation, + time_units=time_units, + units_col=units_col, + idx_separator=idx_separator, + ur=ur, + progress=progress, + progress_bar_position=i, + ) + for i, chunk in enumerate(iterator) + ] + + iterator_results = concurrent.futures.as_completed(futures) + if progress: + iterator_results = tqdm( + iterator_results, + desc="Retrieving parallel results", + total=len(futures), + ) + + res_l = [future.result() for future in iterator_results] + + # Late import to avoid hard dependency on pandas + try: + import pandas as pd + except ImportError as exc: + raise MissingOptionalDependencyError( + "interpolate", requirement="pandas" + ) from exc + + res = pd.concat(res_l) + + return res + + +def register_pandas_accessor(namespace: str = "ct") -> None: + """ + Register the pandas accessors + + For details, see + [pandas' docs](https://pandas.pydata.org/docs/development/extending.html#registering-custom-accessors). + + Parameters + ---------- + namespace + Namespace to use for the accessor + """ + # Doing this because I really don't like imports having side effects + try: + import pandas + except ImportError as exc: + raise MissingOptionalDependencyError( + "register_pandas_accessor", requirement="pandas" + ) from exc + + pandas.api.extensions.register_series_accessor(namespace)(SeriesCTAccessor) + pandas.api.extensions.register_dataframe_accessor(namespace)(DataFrameCTAccessor) diff --git a/src/continuous_timeseries/scratch.py b/src/continuous_timeseries/scratch.py new file mode 100644 index 0000000..b363313 --- /dev/null +++ b/src/continuous_timeseries/scratch.py @@ -0,0 +1,13 @@ +from time import sleep + +import tqdm.notebook + + +def progresser(n): + print(end=" ") + interval = 0.001 / (n + 2) + total = 5000 + text = f"#{n}, est. {interval * total:<04.2}s" + # for _ in trange(total, desc=text, position=n): + for _ in tqdm.notebook.trange(total, desc=text, position=n): + sleep(interval) diff --git a/src/continuous_timeseries/timeseries.py b/src/continuous_timeseries/timeseries.py index 22b3cc2..e7484cc 100644 --- a/src/continuous_timeseries/timeseries.py +++ b/src/continuous_timeseries/timeseries.py @@ -43,6 +43,7 @@ if TYPE_CHECKING: import IPython.lib.pretty import matplotlib.axes + import pandas as pd class UnreachableIntegralPreservingInterpolationTarget(ValueError): @@ -283,6 +284,156 @@ def integrate( timeseries_continuous=integral, ) + @classmethod + def from_pandas_series( # noqa: PLR0913 + cls, + series: pd.Series, + interpolation: InterpolationOption, + units_idx: int, + time_units: str | pint.facets.plain.PlainUnit, + name: str | None = None, + idx_separator: str = "__", + ur: pint.facets.PlainRegistry | None = None, + ) -> Timeseries: + """ + Initialise from a [`pd.Series`][pandas.Series] + + Parameters + ---------- + series + [`pd.Series`][pandas.Series] from which to initialise. + + interpolation + Interpolation to apply when converting + the discrete values to a continuous representation + + units_idx + The index of `series.name` (assumed to be a tuple) + which holds the units information. + + time_units + The units to attach to `series`'s columns to create a time axis. + + name + The value of the result's name attribute. + + If not supplied, we automatically generate this based on the `series` + index values. + + idx_separator + The separator to use to join the values of `idx_row[0]` + to get the result's name. + + Only used if `name is None`. + + All parts of `series.name` are included in the name + except the units information. + + ur + Unit registry to use for the conversion. + + If not supplied, we use the result of calling + [`pint.get_application_registry`][]. + + Returns + ------- + : + Initialised [`Timeseries`][(m)]. + """ + if ur is None: + ur = pint.get_application_registry() + + if isinstance(time_units, str): + time_units = ur.Unit(time_units) + + index_values = series.name + units_str = index_values[units_idx] + units = ur.Unit(units_str) + + x = series.index.values * time_units + y = series.values * units + + if name is None: + name = idx_separator.join(str(v) for v in index_values if v != units_str) + + return cls.from_arrays( + x=x, + y=y, + interpolation=interpolation, + name=name, + ) + + @classmethod + def from_pandas_iterrows_value( # noqa: PLR0913 + cls, + idx_row: tuple[tuple[str | float | int, ...], pd.Series], + interpolation: InterpolationOption, + units_idx: int, + time_units: str | pint.facets.plain.PlainUnit, + idx_separator: str = "__", + ur: pint.facets.PlainRegistry | None = None, + ) -> Timeseries: + """ + Initialise from an iteration of pandas iterrows. + + Specifically, from an iteration of + [`pd.DataFrame.iterrows`][pandas.DataFrame.iterrows]. + + Parameters + ---------- + idx_row + Iteration of [`pd.DataFrame.iterrows`][pandas.DataFrame.iterrows]. + from which to initialise. + + interpolation + Interpolation to apply when converting + the discrete values to a continuous representation + + units_idx + The index of `idx_row[0]` which holds the units information. + + time_units + The units to attach to `row`'s columns to create a time axis. + + idx_separator + The separator to use to join the values of `idx_row[0]` + to get the result's name. + + All parts of `idx_row[0]` are included in the name + except the units information. + + ur + Unit registry to use for the conversion. + + If not supplied, we use the result of calling + [`pint.get_application_registry`][]. + + Returns + ------- + : + Initialised [`Timeseries`][(m)]. + """ + if ur is None: + ur = pint.get_application_registry() + + if isinstance(time_units, str): + time_units = ur.Unit(time_units) + + idx, series = idx_row + + x = series.index.values * time_units + units = idx[units_idx] + y = series.values * ur.Unit(units) + + name = idx_separator.join(str(v) for v in idx if v != units) + + return cls.from_arrays( + x=x, + y=y, + interpolation=interpolation, + name=name, + ) + def interpolate( self, time_axis: TimeAxis | PINT_NUMPY_ARRAY, allow_extrapolation: bool = False ) -> Timeseries: diff --git a/tests/integration/test_pandas_dataframe_accessor.py b/tests/integration/test_pandas_dataframe_accessor.py new file mode 100644 index 0000000..4826b0d --- /dev/null +++ b/tests/integration/test_pandas_dataframe_accessor.py @@ -0,0 +1,66 @@ +""" +Test our pandas DataFrame accessors + +In other words, test the `pd.DataFrame.ct` namespace. +This may need to be split into multiple files in future. +""" + +from __future__ import annotations + +import pint +import pytest + +import continuous_timeseries as ct +from continuous_timeseries.pandas_accessors import register_pandas_accessor + +pd = pytest.importorskip("pandas") + +UR = pint.get_application_registry() +Q = UR.Quantity + + +def test_to_timeseries(): + if not hasattr(pd.DataFrame, "ct"): + register_pandas_accessor() + + x = Q([2010, 2015, 2025], "yr") + y_ms = [ + [1.0, 2.0, 3.0], + [-1.5, -0.5, 0.5], + ] + idx = pd.MultiIndex.from_tuples( + ( + ("name_1", "Mt /yr"), + ("name_2", "Gt"), + ), + # units not unit to follow pint conventions + names=["name", "units"], + ) + + df = pd.DataFrame( + y_ms, + columns=x.m, + index=idx, + ) + + res = df.ct.to_timeseries( + time_units=x.u, + interpolation=ct.InterpolationOption.PiecewiseConstantPreviousLeftClosed, + ) + + # Check results same as just doing a dumb loop here + + # Then test going back to df i.e. round tripping + # - also with different length time axes + # - different units + # Test plotting + # Test ops pass through + # - use a series accessor for this + # Test on 600 x 2000 x 10 timeseries (length ~450 values each), + # also helps check parallelisation. + # This would be a 50GB array so maybe a stupid use case already... + + assert False + + # Would be nice to deregister too so other tests can check. + # De-registering does not seem to be that easy though... diff --git a/uv.lock b/uv.lock index e0a2961..d4dae00 100644 --- a/uv.lock +++ b/uv.lock @@ -345,7 +345,7 @@ wheels = [ [[package]] name = "continuous-timeseries" -version = "0.3.3" +version = "0.3.4a1" source = { editable = "." } dependencies = [ { name = "attrs" }, @@ -357,9 +357,13 @@ dependencies = [ [package.optional-dependencies] full = [ { name = "matplotlib" }, + { name = "pandas" }, { name = "scipy", version = "1.13.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, { name = "scipy", version = "1.14.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, ] +pandas = [ + { name = "pandas" }, +] plots = [ { name = "matplotlib" }, ] @@ -372,6 +376,7 @@ scipy = [ all-dev = [ { name = "attrs" }, { name = "ipython" }, + { name = "ipywidgets" }, { name = "jupyterlab" }, { name = "jupytext" }, { name = "liccheck" }, @@ -383,6 +388,7 @@ all-dev = [ { name = "mkdocs-material" }, { name = "mkdocs-section-index" }, { name = "mkdocstrings-python" }, + { name = "mkdocstrings-python-accessors" }, { name = "mkdocstrings-python-xref" }, { name = "mypy" }, { name = "openscm-units" }, @@ -396,8 +402,10 @@ all-dev = [ { name = "scipy", version = "1.13.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, { name = "scipy", version = "1.14.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, { name = "scipy-stubs", marker = "python_full_version >= '3.10'" }, + { name = "seaborn" }, { name = "setuptools" }, { name = "towncrier" }, + { name = "tqdm" }, ] dev = [ { name = "ipython" }, @@ -411,6 +419,7 @@ dev = [ ] docs = [ { name = "attrs" }, + { name = "ipywidgets" }, { name = "jupyterlab" }, { name = "jupytext" }, { name = "mkdocs" }, @@ -421,12 +430,15 @@ docs = [ { name = "mkdocs-material" }, { name = "mkdocs-section-index" }, { name = "mkdocstrings-python" }, + { name = "mkdocstrings-python-accessors" }, { name = "mkdocstrings-python-xref" }, { name = "openscm-units" }, { name = "pymdown-extensions" }, { name = "ruff" }, { name = "scipy", version = "1.13.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, { name = "scipy", version = "1.14.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "seaborn" }, + { name = "tqdm" }, ] tests = [ { name = "ipython" }, @@ -441,10 +453,12 @@ tests = [ [package.metadata] requires-dist = [ { name = "attrs", specifier = ">=24.3.0" }, + { name = "continuous-timeseries", extras = ["pandas"], marker = "extra == 'full'" }, { name = "continuous-timeseries", extras = ["plots"], marker = "extra == 'full'" }, { name = "continuous-timeseries", extras = ["scipy"], marker = "extra == 'full'" }, { name = "matplotlib", marker = "extra == 'plots'", specifier = ">=3.7.1" }, { name = "numpy", specifier = ">=2.0.0" }, + { name = "pandas", marker = "extra == 'pandas'", specifier = ">=2.0.0" }, { name = "pint", specifier = ">=0.24.4" }, { name = "scipy", marker = "extra == 'scipy'", specifier = ">=1.13.1" }, ] @@ -453,6 +467,7 @@ requires-dist = [ all-dev = [ { name = "attrs", specifier = "==24.3.0" }, { name = "ipython", specifier = ">=8.18.1" }, + { name = "ipywidgets", specifier = ">=8.1.5" }, { name = "jupyterlab", specifier = "==4.3.4" }, { name = "jupytext", specifier = "==1.16.6" }, { name = "liccheck", specifier = "==0.9.2" }, @@ -464,6 +479,7 @@ all-dev = [ { name = "mkdocs-material", specifier = "==9.5.49" }, { name = "mkdocs-section-index", specifier = "==0.3.9" }, { name = "mkdocstrings-python", specifier = "==1.13.0" }, + { name = "mkdocstrings-python-accessors", editable = "accessor-handler" }, { name = "mkdocstrings-python-xref", specifier = "==1.6.2" }, { name = "mypy", specifier = "==1.14.0" }, { name = "openscm-units", specifier = ">=0.6.3" }, @@ -476,8 +492,10 @@ all-dev = [ { name = "ruff", specifier = "==0.8.6" }, { name = "scipy", specifier = ">=1.13.1" }, { name = "scipy-stubs", marker = "python_full_version >= '3.10'", specifier = ">=1.14.1.6" }, + { name = "seaborn", specifier = ">=0.13.2" }, { name = "setuptools", specifier = "==75.6.0" }, { name = "towncrier", specifier = "==24.8.0" }, + { name = "tqdm", specifier = ">=4.67.1" }, ] dev = [ { name = "ipython", specifier = ">=8.18.1" }, @@ -491,6 +509,7 @@ dev = [ ] docs = [ { name = "attrs", specifier = "==24.3.0" }, + { name = "ipywidgets", specifier = ">=8.1.5" }, { name = "jupyterlab", specifier = "==4.3.4" }, { name = "jupytext", specifier = "==1.16.6" }, { name = "mkdocs", specifier = "==1.6.1" }, @@ -501,11 +520,14 @@ docs = [ { name = "mkdocs-material", specifier = "==9.5.49" }, { name = "mkdocs-section-index", specifier = "==0.3.9" }, { name = "mkdocstrings-python", specifier = "==1.13.0" }, + { name = "mkdocstrings-python-accessors", editable = "accessor-handler" }, { name = "mkdocstrings-python-xref", specifier = "==1.6.2" }, { name = "openscm-units", specifier = ">=0.6.3" }, { name = "pymdown-extensions", specifier = "==10.13" }, { name = "ruff", specifier = "==0.8.6" }, { name = "scipy", specifier = ">=1.13.1" }, + { name = "seaborn", specifier = ">=0.13.2" }, + { name = "tqdm", specifier = ">=4.67.1" }, ] tests = [ { name = "ipython", specifier = ">=8.18.1" }, @@ -1016,6 +1038,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/47/6b/d9fdcdef2eb6a23f391251fde8781c38d42acd82abe84d054cb74f7863b0/ipython-8.18.1-py3-none-any.whl", hash = "sha256:e8267419d72d81955ec1177f8a29aaa90ac80ad647499201119e2f05e99aa397", size = 808161 }, ] +[[package]] +name = "ipywidgets" +version = "8.1.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "comm" }, + { name = "ipython" }, + { name = "jupyterlab-widgets" }, + { name = "traitlets" }, + { name = "widgetsnbextension" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c7/4c/dab2a281b07596a5fc220d49827fe6c794c66f1493d7a74f1df0640f2cc5/ipywidgets-8.1.5.tar.gz", hash = "sha256:870e43b1a35656a80c18c9503bbf2d16802db1cb487eec6fab27d683381dde17", size = 116723 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/22/2d/9c0b76f2f9cc0ebede1b9371b6f317243028ed60b90705863d493bae622e/ipywidgets-8.1.5-py3-none-any.whl", hash = "sha256:3290f526f87ae6e77655555baba4f36681c555b8bdbbff430b70e52c34c86245", size = 139767 }, +] + [[package]] name = "isoduration" version = "20.11.0" @@ -1268,6 +1306,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/09/2032e7d15c544a0e3cd831c51d77a8ca57f7555b2e1b2922142eddb02a84/jupyterlab_server-2.27.3-py3-none-any.whl", hash = "sha256:e697488f66c3db49df675158a77b3b017520d772c6e1548c7d9bcc5df7944ee4", size = 59700 }, ] +[[package]] +name = "jupyterlab-widgets" +version = "3.0.13" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/59/73/fa26bbb747a9ea4fca6b01453aa22990d52ab62dd61384f1ac0dc9d4e7ba/jupyterlab_widgets-3.0.13.tar.gz", hash = "sha256:a2966d385328c1942b683a8cd96b89b8dd82c8b8f81dda902bb2bc06d46f5bed", size = 203556 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a9/93/858e87edc634d628e5d752ba944c2833133a28fa87bb093e6832ced36a3e/jupyterlab_widgets-3.0.13-py3-none-any.whl", hash = "sha256:e3cda2c233ce144192f1e29914ad522b2f4c40e77214b0cc97377ca3d323db54", size = 214392 }, +] + [[package]] name = "jupytext" version = "1.16.6" @@ -1778,6 +1825,45 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/51/23/d02d86553327296c3bf369d444194ea83410cce8f0e690565264f37f3261/mkdocstrings_python-1.13.0-py3-none-any.whl", hash = "sha256:b88bbb207bab4086434743849f8e796788b373bd32e7bfefbf8560ac45d88f97", size = 112254 }, ] +[[package]] +name = "mkdocstrings-python-accessors" +version = "0.3.4.dev2+gb47eeae" +source = { editable = "accessor-handler" } +dependencies = [ + { name = "mkdocstrings" }, +] + +[package.metadata] +requires-dist = [{ name = "mkdocstrings", specifier = ">=0.18" }] + +[package.metadata.requires-dev] +dev = [ + { name = "black", specifier = ">=24.4" }, + { name = "build", specifier = ">=1.2" }, + { name = "duty", specifier = ">=1.4" }, + { name = "git-changelog", specifier = ">=2.5" }, + { name = "markdown-callouts", specifier = ">=0.4" }, + { name = "markdown-exec", specifier = ">=1.8" }, + { name = "mkdocs", specifier = ">=1.6" }, + { name = "mkdocs-coverage", specifier = ">=1.0" }, + { name = "mkdocs-gen-files", specifier = ">=0.5" }, + { name = "mkdocs-git-revision-date-localized-plugin", specifier = ">=1.2" }, + { name = "mkdocs-literate-nav", specifier = ">=0.6" }, + { name = "mkdocs-material", specifier = ">=9.5" }, + { name = "mkdocs-minify-plugin", specifier = ">=0.8" }, + { name = "mkdocstrings", extras = ["python"], specifier = ">=0.25" }, + { name = "mypy", specifier = ">=1.10" }, + { name = "pytest", specifier = ">=8.2" }, + { name = "pytest-cov", specifier = ">=5.0" }, + { name = "pytest-randomly", specifier = ">=3.15" }, + { name = "pytest-xdist", specifier = ">=3.6" }, + { name = "ruff", specifier = ">=0.4" }, + { name = "tomli", marker = "python_full_version < '3.11'", specifier = ">=2.0" }, + { name = "twine", specifier = ">=5.1" }, + { name = "types-markdown", specifier = ">=3.6" }, + { name = "types-pyyaml", specifier = ">=6.0" }, +] + [[package]] name = "mkdocstrings-python-xref" version = "1.6.2" @@ -3084,6 +3170,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/da/29/d336808517bbd1ab6a2523d07cb5927166ffb8eaca9b8ed94790e06f4287/scipy_stubs-1.15.0.0-py3-none-any.whl", hash = "sha256:ddfbacdd15a0996084aee21620a29d474bb3c0dc375e5fe3cd78fe5aae50910d", size = 454570 }, ] +[[package]] +name = "seaborn" +version = "0.13.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "matplotlib" }, + { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "numpy", version = "2.2.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "pandas" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/86/59/a451d7420a77ab0b98f7affa3a1d78a313d2f7281a57afb1a34bae8ab412/seaborn-0.13.2.tar.gz", hash = "sha256:93e60a40988f4d65e9f4885df477e2fdaff6b73a9ded434c1ab356dd57eefff7", size = 1457696 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/83/11/00d3c3dfc25ad54e731d91449895a79e4bf2384dc3ac01809010ba88f6d5/seaborn-0.13.2-py3-none-any.whl", hash = "sha256:636f8336facf092165e27924f223d3c62ca560b1f2bb5dff7ab7fad265361987", size = 294914 }, +] + [[package]] name = "semantic-version" version = "2.10.0" @@ -3260,6 +3361,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/93/1b/2f7b88506e22d9798c139261af4946865c0787cfa345514ca3c70173a9cc/towncrier-24.8.0-py3-none-any.whl", hash = "sha256:9343209592b839209cdf28c339ba45792fbfe9775b5f9c177462fd693e127d8d", size = 56981 }, ] +[[package]] +name = "tqdm" +version = "4.67.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a8/4b/29b4ef32e036bb34e4ab51796dd745cdba7ed47ad142a9f4a1eb8e0c744d/tqdm-4.67.1.tar.gz", hash = "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2", size = 169737 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl", hash = "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2", size = 78540 }, +] + [[package]] name = "traitlets" version = "5.14.3" @@ -3401,6 +3514,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5a/84/44687a29792a70e111c5c477230a72c4b957d88d16141199bf9acb7537a3/websocket_client-1.8.0-py3-none-any.whl", hash = "sha256:17b44cc997f5c498e809b22cdf2d9c7a9e71c02c8cc2b6c56e7c2d1239bfa526", size = 58826 }, ] +[[package]] +name = "widgetsnbextension" +version = "4.0.13" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/56/fc/238c424fd7f4ebb25f8b1da9a934a3ad7c848286732ae04263661eb0fc03/widgetsnbextension-4.0.13.tar.gz", hash = "sha256:ffcb67bc9febd10234a362795f643927f4e0c05d9342c727b65d2384f8feacb6", size = 1164730 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/21/02/88b65cc394961a60c43c70517066b6b679738caf78506a5da7b88ffcb643/widgetsnbextension-4.0.13-py3-none-any.whl", hash = "sha256:74b2692e8500525cc38c2b877236ba51d34541e6385eeed5aec15a70f88a6c71", size = 2335872 }, +] + [[package]] name = "zipp" version = "3.21.0"