diff --git a/.gitignore b/.gitignore index 795ddfa..1a2ea5d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,222 @@ -__pycache__ FBs/.vscode/tasks.json + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[codz] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py.cover +*.lcov +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +# Pipfile.lock + +# UV +# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# uv.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +# poetry.lock +# poetry.toml + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python. +# https://pdm-project.org/en/latest/usage/project/#working-with-version-control +# pdm.lock +# pdm.toml +.pdm-python +.pdm-build/ + +# pixi +# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control. +# pixi.lock +# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one +# in the .venv directory. It is recommended not to include this directory in version control. +.pixi/* +!.pixi/config.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule* +celerybeat.pid + +# Redis +*.rdb +*.aof +*.pid + +# RabbitMQ +mnesia/ +rabbitmq/ +rabbitmq-data/ + +# ActiveMQ +activemq-data/ + +# SageMath parsed files +*.sage.py + +# Environments +.env +.envrc +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +# .idea/ + +# Abstra +# Abstra is an AI-powered process automation framework. +# Ignore directories containing user credentials, local state, and settings. +# Learn more at https://abstra.io/docs +.abstra/ + +# Visual Studio Code +# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore +# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore +# and can be added to the global gitignore or merged into this file. However, if you prefer, +# you could uncomment the following to ignore the entire vscode folder +# .vscode/ +# Temporary file for partial code execution +tempCodeRunnerFile.py + +# Ruff stuff: +.ruff_cache/ + +# PyPI configuration file +.pypirc + +# Marimo +marimo/_static/ +marimo/_lsp/ +__marimo__/ + +# Streamlit +.streamlit/secrets.toml diff --git a/README.md b/README.md index 7e1e9a6..d838a43 100644 --- a/README.md +++ b/README.md @@ -2,47 +2,49 @@ This repository is dedicated to all Function Blocks that might be used in DINASORE and 4DIAC already with a folder organization suitable to several needs. The main idea is to add new or update Functions Blocks according to new developments. +To sync the function blocks with your DINASORE instances, see [sync](#sync). + ## FBs The "FBs" folder contains the set of previously mentioned function blocks. What follows is a brief description of the contents of each sub-folder. -### ARITHMETIC +#### ARITHMETIC In this folder you will find function blocks that perform relatively simple mathematical calculations. -### CLUSTERING +#### CLUSTERING This folder contains function blocks that perform clustering methods. -### DATA_HANDLING +#### DATA_HANDLING This folder contains function blocks that handle the data that is given to them without changing its value. -### DATA_TRANSFORMATIONS +#### DATA_TRANSFORMATIONS In this folder you will find function blocks that perform certain operations on data, changing its value. -### DB +#### DB This folder contains function blocks for access and usage of databases. -### FILES_AND_PLOTS +#### FILES_AND_PLOTS This folder contains function blocks that read data from files and plot the respective graphs and other representations. -### MISCELLANEOUS +#### MISCELLANEOUS In this folder you will find function blocks that do not fit in any of the other categories. -### MQTT +#### MQTT In this folder you will find function blocks that make use of the MQTT protocol for communication. -### OPC-UA +#### OPC-UA This folder contains function blocks that make use of the OPC-UA protocol for information display/fetch. -#### INFO_WRAPPER +##### INFO_WRAPPER
@@ -67,7 +69,7 @@ You can decide what the inputs are for the method by filling out the FB's INPUT **Note:** When you deploy a pipeline that is wrapped by INFO_WRAPPER to DINASORE from 4DIAC, DINASORE checks to see which inputs already have a value assigned to them. Consequently, if you specify the value of an input in 4DIAC, you will not have to specify its value again when calling the method. -#### METHOD_CALLER +##### METHOD_CALLER
@@ -83,25 +85,31 @@ The METHOD_CALLER function block calls the OPC-UA that is specified in its prope The OUTPUT will be a list in the same format as INPUT_VARS. Its values are the ones from the pipeline that the method has finished calling. -### OPTIMIZATION +#### OPTIMIZATION This folder contains function blocks that optimize the inputs given. -### SENSORS_AND_SIMULATORS +#### SENSORS_AND_SIMULATORS This folder contains function blocks that are either sensors, simulators or tools for either of these types of function blocks. -### TEST +#### TEST This folder contains function blocks that are used by DINASORE's unit tests. -### USE_CASES +#### USE_CASES This folder further contains more folders, each representing a specific use case (pipeline) for which the function blocks that they contain were used. ## sync -The "sync" folder is a package that allows you to synchronize your "FBs" folder with other local or remote DINASOREs. Synchronization with remote DINASORES requires an SSH connection with the device running the DINASORE in question. Instructions are as follows. +The "sync" folder is a package that allows you to synchronize your "FBs" folder with other local or remote DINASOREs. + +If the the DINASORE is running local (and not in a Docker), set the address to `localhost` and the port to `-1`. + +Synchronization with remote DINASORES requires an SSH connection with the device running the DINASORE in question. Setup a SSH connection in the device running DINASORE using "openssh-server" for example. In your machine you should be able to SSH into the device like `ssh root@x.x.x.x -p 22`. + +If you are using Docker locally, don't forget to map the SSH port (by default 22) to the host. The address to be used in Docker locally should be `localhost`. So you would run `ssh root@localhost -p 22`. ### config.json @@ -113,6 +121,7 @@ In this package you will find a configuration file called "config.json". This fi "dinasores": [ # array of dinasores to be synchronized { "address": # DINASORE address , + "port": # DINASORE port , "usename": # SSH username , "password": # SSH password , "dinasore-path": # path to the DINASORE root folder in the device running the DINASORE in question @@ -129,10 +138,12 @@ In this package you will find a configuration file called "config.json". This fi "dinasores": [ { "address": "localhost", + "port": -1, "dinasore-path": ".../dinasore" }, { "address": "192.168.1.14", + "port": 22, "username": "pi", "password": "raspberry" "dinasore-path": ".../dinasore" diff --git a/sync/base_sync.py b/sync/base_sync.py index e69704f..7fc070a 100644 --- a/sync/base_sync.py +++ b/sync/base_sync.py @@ -1,7 +1,7 @@ from abc import ABC, abstractmethod class BaseSync(ABC): - + @abstractmethod def synchronize(self): pass diff --git a/sync/config.json b/sync/config.json index c6f7826..26e9071 100644 --- a/sync/config.json +++ b/sync/config.json @@ -1,10 +1,11 @@ { - "master-fbs-path": "C:/Users/Rafael/Documents/dinasore_function_blocks/FBs", - "dinasores": [ + "master-fbs-path": "./FBs", + "dinasores": [ { "address": "localhost", - "dinasore-path": "C:/Users/Rafael/Documents/dinasore-cluster/dinasore-2/dinasore-ua" + "port": 22, + "dinasore-path": "/dinasore" } ], - "strategy": "" -} \ No newline at end of file + "strategy": "wipe" +} diff --git a/sync/fb_sync.py b/sync/fb_sync.py index fc1fd69..8b87b4e 100644 --- a/sync/fb_sync.py +++ b/sync/fb_sync.py @@ -14,25 +14,26 @@ def synchronize(self, dinasore): 'dinasore-path' not in dinasore: logging.warning('Can\'t synchronize without dinasore path.') return - + address = dinasore.get('address') + + port = 22 + if "port" in dinasore: + port = int(dinasore.get('port')) + dinasore_path = dinasore.get('dinasore-path') + '/resources/function_blocks' sync_obj = None - if address == 'localhost' or\ - address == '127.0.0.1': + if (address == 'localhost' or address == '127.0.0.1') and port == -1: sync_obj = local_sync.LocalSync(self.master_fbs_path, dinasore_path) else: username = getpass.getuser() if 'username' not in dinasore else dinasore.get('username') password = None if 'password' not in dinasore else dinasore.get('password') - sync_obj = remote_sync.RemoteSync(self.master_fbs_path, address, dinasore_path, username, password) + sync_obj = remote_sync.RemoteSync(self.master_fbs_path, address, port, dinasore_path, username, password) strategies = self.strategy.split(' ') if 'wipe' in strategies: sync_obj.wipe() - - sync_obj.synchronize() - - + sync_obj.synchronize() diff --git a/sync/local_sync.py b/sync/local_sync.py index ab79923..070f8eb 100644 --- a/sync/local_sync.py +++ b/sync/local_sync.py @@ -20,4 +20,3 @@ def synchronize(self): copier = syncer.Syncer(self.master_fbs_path, self.path, 'sync') copier.do_work() return set(copier._changed).union(copier._added).union(copier._deleted) - diff --git a/sync/remote_sync.py b/sync/remote_sync.py index 4598a7e..3b9910b 100644 --- a/sync/remote_sync.py +++ b/sync/remote_sync.py @@ -5,9 +5,10 @@ class RemoteSync(base_sync.BaseSync): - def __init__(self, master_fbs_path, address, path, username, password): + def __init__(self, master_fbs_path, address, port, path, username, password): self.master_fbs_path = master_fbs_path self.address = address + self.port = port self.path = path self.username = username self.password = password @@ -17,7 +18,7 @@ def wipe(self): client = paramiko.SSHClient() client.load_system_host_keys() client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - client.connect(self.address, 22, self.username, self.password) # use port 22 for ssh + client.connect(self.address, self.port, self.username, self.password) # use port 22 for ssh # delete fbs folder command = 'rm -rf ' + self.path client.exec_command(command) @@ -39,19 +40,19 @@ def synchronize(self): client = paramiko.SSHClient() client.load_system_host_keys() client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - client.connect(self.address, 22, self.username, self.password) # use port 22 for ssh - - t = paramiko.Transport((self.address, 22)) + client.connect(self.address, self.port, self.username, self.password) + + t = paramiko.Transport((self.address, self.port)) t.connect( username=self.username, password=self.password ) - + sftp = paramiko.SFTPClient.from_transport(t) # get list of local fbs scanned_fbs = self.scantree(self.master_fbs_path) - + for fb in scanned_fbs: correct_path = fb.path.replace('\\', '/') dest_path = self.path + correct_path.replace(self.master_fbs_path, '') @@ -60,7 +61,7 @@ def synchronize(self): except FileNotFoundError: # file does not exist on remote self.force_put(sftp, correct_path, dest_path, self.path) - else: + else: if fb.stat().st_mtime > dest_fb_stats.st_mtime: # local file is updated sftp.put(correct_path, dest_path) @@ -106,8 +107,5 @@ def force_put(self, sftp, src, trgt, origin_path): except IOError: # directory already exists pass - - sftp.put(src, trgt) - - \ No newline at end of file + sftp.put(src, trgt) diff --git a/sync/synchronize.py b/sync/synchronize.py index f4500b3..b2887b4 100644 --- a/sync/synchronize.py +++ b/sync/synchronize.py @@ -10,7 +10,7 @@ f = open(f_path) except FileNotFoundError: logging.error('Could not find configuration file.') - exit() + exit() data = json.load(f) f.close() @@ -32,5 +32,3 @@ synchronizer.synchronize(dinasore) print('Copying process concluded') print('Synchronized {} DINASOREs'.format(len(dinasores))) - -