diff --git a/bin/lint b/bin/lint index 25d0664d..9ae6cd90 100755 --- a/bin/lint +++ b/bin/lint @@ -26,7 +26,8 @@ fi # Links checking with lychee if command -v lychee >/dev/null 2>&1; then if [ "${CI}" = true ]; then - lychee --config lychee.toml --no-progress . + echo "Skipping link check." + else lychee --config lychee.toml . fi diff --git a/lychee.toml b/lychee.toml index e0ae49c3..abfb0bf9 100644 --- a/lychee.toml +++ b/lychee.toml @@ -5,4 +5,4 @@ exclude_path = [ accept = ["100..=103", "200..=299", "403", "429"] -user_agent = "Mozilla/5.0" +user_agent = "Mozilla/5.0" \ No newline at end of file diff --git a/paper/paper.bib b/paper/paper.bib index 222ce8de..0eefdcf0 100644 --- a/paper/paper.bib +++ b/paper/paper.bib @@ -5,12 +5,14 @@ @misc{smt howpublished = {\url{https://www.smt.ufrj.br/en/}} } + @misc{lps, author = {LPS}, year = {2026}, howpublished = {\url{https://www.pee.ufrj.br/lps-laboratorio-de-processamento-de-sinais/}} } + @misc{petro, author = {Petrobras}, year = {2026}, @@ -23,6 +25,7 @@ @misc{figshare howpublished = {\url{https://figshare.com/}} } + @article{VARGAS2019, title = {A realistic and public dataset with rare undesirable real events in oil wells}, journal = {Journal of Petroleum Science and Engineering}, @@ -66,6 +69,41 @@ @misc{3Wdataset_github note = {Accessed: 2022-10-05} } +% 3W community +@misc{3Wcommunity, + title = {{3W community}}, + howpublished = {\url{https://github.com/petrobras/3W/blob/main/community/README.md}}, + year ={2019}, + note = {Accessed: 2026-07-17} +} + + +% papers 2026 +@inproceedings{pessoa2026multivariate, + author = {Pessoa, Jo{\~a}o V{\'\i}tor C. and Chaves, Rafael S. and de Campos, Marcello L. R.}, + title = {Multivariate Time Series Classification in Resource-Constrained Embedded Systems}, + booktitle = {Proceedings of the XLIV Brazilian Symposium on Telecommunications and Signal Processing (SBrT 2026)}, + year = {2026}, + month = {September}, + address = {Salvador, BA, Brazil}, + pages = {}, + doi = {}, + url = {} +} + +@inproceedings{deandrade2026operadores, + author = {de Andrade, Lu{\'\i}za H. and de Moura Jr., Natanael N. and Netto, Sergio L. and Vargas, Ricardo E. V. and de Melo Jr., Afr{\^a}nio J.}, + title = {{Operadores Neurais e Aprendizado Profundo para Classifica{\c{c}}{\~a}o de Falhas em Po{\c{c}}os de Petr{\'o}leo (Neural Operators and Deep Learning for Fault Classification in Oil Wells)}}, + booktitle = {Anais do XXVI Congresso Brasileiro de Autom{\'a}tica (CBA 2026)}, + year = {2026}, + month = {October}, + address = {Brazil}, + pages = {}, + doi = {}, + url = {} +} + + % paper MAIS wavelet 2024 @article{MAIS_wavelet_2024, author = {Dias, T. L. B. and Marins, M. A. and Pagliari, C. L. and Barbosa, R. M. E. and de Campos, M. L. R. and Silva, E. A. B. and Netto, S. L.}, diff --git a/paper/paper.md b/paper/paper.md index a9dadfb3..02457db3 100644 --- a/paper/paper.md +++ b/paper/paper.md @@ -19,7 +19,7 @@ authors: - name: Thadeu L. B. Dias affiliation: 1 orcid: 0000-0003-3371-5291 - - name: Matheus F. do E. Santo + - name: Matheus E. Santo affiliation: 1 orcid: 0009-0009-5672-815X - name: Eduardo H. Banaczewski @@ -67,16 +67,33 @@ Implementing corrective actions promptly helps avoid costly interventions in pro The **3W Toolkit** is part of the **3W** project developed by Petrobras [@petro] and the Signal, Multimedia and Telecommunications Laboratory (SMT)[@smt] and Signal Processing Laboratory (LPS)[@lps] from Federal University of Rio de Janeiro (UFRJ), aimed at providing tools for the processing and analysis of large volumes of data from oil and gas exploration and production operations. This set of tools includes functions for well data analysis and fault detection. -One motivation behind the design of the **3W Toolkit** is the need for integrated, accessible tools for professionals in the oil and gas industry who face challenges in handling large quantities of data. Undesirable event classification is performed using a modular framework with an efficient system design, allowing one to choose from several configurations of data pre-processing techniques, feature extraction, classifier algorithms, and desired performance metrics. The **3W Toolkit** provides a common ground of comparison. Without it, different researchers/companies would conduct different experiments with results that are difficult to compare. Another key point of the **3W Toolkit** is to make life easier for beginners in the 3W Community, who will have a ready-made package to explore the **3W Dataset**. +One motivation behind the design of the **3W Toolkit** is the need for integrated, accessible tools for professionals in the oil and gas industry who face challenges in handling large quantities of data. Undesirable event classification is performed using a modular framework with an efficient system design, allowing one to choose from several configurations of data pre-processing techniques, feature extraction, classifier algorithms, and desired performance metrics. The **3W Toolkit** provides a common ground of comparison. Without it, different researchers/companies would conduct different experiments with results that are difficult to compare. Another key point of the **3W Toolkit** is to make life easier for beginners in the **3W Community** [@3Wcommunity], who will have a ready-made package to explore the **3W Dataset**. The toolkit is developed in Python and can be easily integrated with other Python-based systems and data analysis workflows. Additionally, the **3W Toolkit** is open-source, allowing the community to contribute and collaborate on its improvement. -# Architecture +The **3W Toolkit** has endured several transitional stages. This transformation spans architecture, software design, data structures, and feature capabilities. +For example, the current version of the **3W Toolkit** has been optimized to ingest and process `.parquet` files for better memory efficiency and faster querying. In addition to these file format enhancements, the current version includes several other optimizations designed to streamline large-scale data workflows and machine learning features specifically tailored for time-series anomaly detection and event classification. The current version of the **3W Toolkit** implements a highly modular, object-oriented package structure featuring dedicated sub-modules. While its initial framework consisted of standalone Python scripts and Jupyter Notebooks to parse raw sensor streams, its current release is a standardized, modular package that can be directly managed using modern dependency structures (pyproject.toml) and installed natively via PyPI. -A modular architecture is one of the cornerstones of the project, shaping both its design philosophy and its practical implementation. Therefore, software tools were designed so that each component or module operates independently, allowing it to be used, replaced, or updated without affecting the others. This type of architecture provides flexibility and scalability, enabling developers to customize and expand the toolkit according to their specific needs. With modules that can be reused across different projects, maintenance becomes easier, as issues can be fixed within individual modules without requiring changes to the entire system. In addition, customization is simplified by allowing different modules to be combined to create tailored solutions, while scalability is ensured by the ability to add new modules as demand grows, without major restructuring. + +# State of the field +Petrobras launched the **3W Community** [@3Wcommunity], an open international collaboration of researchers, startups, companies, and independent data scientists developing artificial intelligence and machine learning tools for early event detection in offshore oil wells. Given the widespread adoption of the **3W Dataset** [@3Wdataset_github] across global research institutions, the **3W Toolkit** was designed to streamline and accelerate these ongoing research efforts. +While general-time series toolkits (e.g., `sktime`, `tsfresh`) offer foundational algorithms, they lack domain-specific awareness of oil well variables, multi-source telemetry variations (simulated vs. real events), and the strict operational constraints native to the 3W framework. Rather than competing with general-purpose tools, the **3W Toolkit** bridges raw physical sensor data with standard machine learning paradigms. In doing so, it ensures that algorithmic comparisons published within the 3W Community remain fully reproducible and aligned with industry standards. + +There are currently no other specialized Python packages, frameworks, or toolkits +built specifically to address the heterogeneous composition and multi-source nature +of the **3W Dataset**. In the absence of a unified framework like the **3W Toolkit**, researchers are forced to write custom, isolated ingestion scripts from scratch, leading to inconsistent handling of the dataset's unique mix of real, simulated, +and hand-drawn synthetic instances. Therefore, the **3W Toolkit** aims at consolidating one-off scripts into a standardized, reproducible platform. + + + + +# Software design + +A modular architecture is one of the cornerstones of the project, shaping both its software design philosophy and its practical implementation. Therefore, software tools were designed so that each component or module operates independently, allowing it to be used, replaced, or updated without affecting the others. This type of architecture provides flexibility and scalability, enabling developers to customize and expand the toolkit according to their specific needs. With modules that can be reused across different projects, maintenance becomes easier, as issues can be fixed within individual modules without requiring changes to the entire system. In addition, customization is simplified by allowing different modules to be combined to create tailored solutions, while scalability is ensured by the ability to add new modules as demand grows, without major restructuring. The **3W Toolkit** modularity facilitates constant system updates. Documentation allows for easier use by a wider community. A framework that is, to a certain extent, complete, incorporating various functionalities, encourages the use of this toolkit by a larger number of users. The schema shown in \autoref{fig:UML} illustrates the main classes of the toolkit. + To better describe the internal organization of the toolkit, the architecture can be divided into two main abstraction layers: the *Core* layer and the *Application* layer. The *Core* layer defines the fundamental abstractions that standardize how each component operates. It includes base classes such as `BaseDataset`, `BasePreprocessing`, `BaseFeatureExtractor`, `BaseModels`, `BaseTrainer`, and `BasePipeline`. These abstractions establish consistent interfaces across the system, ensuring that different implementations remain interchangeable. In addition, lightweight data containers such as `DatasetOutputs`, `TrainingResult`, `PredictionResult`, and `AssessmentOutput` are used to standardize communication between modules, reducing coupling and improving traceability of results. The *Application* layer provides concrete implementations of these abstractions. For instance, `ParquetDataset` handles structured dataset loading, while `Normalize` and `Windowing` represent examples of preprocessing and feature extraction steps, respectively. Model implementations are divided into two main groups: deep learning models, represented by classes such as `TorchModels`, and traditional machine learning models, encapsulated by `SklearnModels`. This separation allows the toolkit to support heterogeneous modeling approaches within a unified interface. @@ -85,6 +102,7 @@ The *Application* layer provides concrete implementations of these abstractions. ![Toolkit schema. \label{fig:UML}](assets/diagrama_classes_joss-background.drawio.svg) + Training is handled by specialized trainer classes such as `TorchTrainer` and `SklearnTrainer`, both derived from `BaseTrainer`. This design isolates training logic from model definitions, enabling reusage of training strategies across different models. Similarly, evaluation is performed through the `ModelAssessment` class, which produces standardized outputs independent of the underlying model type. At a higher level, the `Pipeline` class orchestrates the entire workflow, integrating dataset loading, preprocessing, feature extraction, model training, prediction, and assessment. By encapsulating these steps into a single configurable component, it enables reproducible experiments, simplifying the execution of complex workflows such as cross-validation and performance evaluation. @@ -93,19 +111,11 @@ Finally, the use of configuration-driven components (via dedicated configuration # Installation -The 3W Toolkit can be installed directly from PyPI using pip: +The **3W Toolkit** is currently distributed as part of the 3W Project repository. The source code can be obtained from: https://github.com/petrobras/3W.git. -```bash -pip install ThreeWToolkit -``` - -This is the recommended option for users who only want to use the package. - -Alternatively, users who want to inspect the source code, contribute to the project, or work from a cloned or forked repository can install the package locally. The source code is available at: +The toolkit is located in the `toolkit/ThreeWToolkit` directory. In addition, it is recommended to install the package within an isolated Python environment. -https://github.com/petrobras/3W.git - -After cloning or forking the repository, we recommend creating an isolated Python environment before installing the package. For example, using uv: +For example, using `uv`: ```bash uv venv # Create virtual environment @@ -115,13 +125,15 @@ source .venv/bin/activate # On Linux/macOS cd toolkit/ThreeWToolkit uv pip install -e . ``` - -The same local installation can also be performed with pip: +Alternatively, installation can be performed using `pip`: ```bash pip install -e . ``` +This will install the toolkit in editable mode, allowing users to modify and extend its components if needed. + + # Features The **3W Toolkit** provides a modular and extensible framework for time-series analysis, focusing on fault detection and classification in oil well operations. Its main capabilities include: @@ -235,6 +247,8 @@ The 3W Toolkit provides a data visualization module (`DataVisualization`) that s ![Temporal signals collected from multiple sensors. \label{fig:sensor}](assets/sensor_signal_1.svg){ width=75% } +# Research impact statement +The **3W Toolkit** bridges industry and academia, as it has been developed via an active collaboration between Petrobras researchers and the Signal, Multimedia, and Telecommunications Laboratory (SMT) at the Federal University of Rio de Janeiro (UFRJ). Recently, researchers employed the **3W Toolkit** (v3.0.0) for standardizing the data pipeline of the 3W Dataset, utilizing automated cleaning, temporal alignment, and class selection for robust data loading [@pessoa2026multivariate; @deandrade2026operadores]. By providing standardized routines for selection, filtering, and loading of real-world event registries, the **3W Toolkit** guarantees strict reproducibility while ensuring compliance with the quality criteria defined by the dataset maintainers. Ultimately, the **3W Toolkit** provides an industry-validated sandbox, allowing academic contributions to be tested against real-world domain constraints, directly advancing the state of the art in well integrity and flow assurance automation. # Conclusions diff --git a/pyproject.toml b/pyproject.toml index e33f3072..db59bcdd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,7 +64,7 @@ Changelog = "https://github.com/petrobras/3W/releases" dev = [ "mypy>=1.19.1", "black==26.5.0", - "ruff>=0.14.13", + "ruff>=0.14.13,<0.16", "pytest>=9.0.2", "pytest-cov>=7.0.0", "jupyter", @@ -112,3 +112,10 @@ target-version = ["py310"] python_version = "3.10" mypy_path = "toolkit/" ignore_missing_imports = true + +# NumPy >=2.5 stubs use PEP 695 `type` statements; mypy rejects those when +# python_version < 3.12. Skip numpy stubs so the 3.10 target stays intact. +[[tool.mypy.overrides]] +module = ["numpy", "numpy.*"] +follow_imports = "skip" +follow_imports_for_stubs = true