From 6e78ac8f8a18499b81779627fe93478e89c752ec Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Thu, 8 Jun 2023 01:06:50 +0100 Subject: [PATCH] feat/DatabaseApi (#30) --- .github/workflows/license_tests.yml | 27 +- .gitignore | 88 +++- MANIFEST.in | 4 + ovos_backend_client/api.py | 382 +++++++++++--- ovos_backend_client/backends/__init__.py | 14 +- ovos_backend_client/backends/base.py | 457 ++++++++++++++--- ovos_backend_client/backends/offline.py | 623 +++++++++++++++++++---- ovos_backend_client/backends/personal.py | 357 ++++++++++++- ovos_backend_client/database.py | 471 ++++++++++++++--- ovos_backend_client/settings.py | 43 +- ovos_backend_client/version.py | 6 +- test/license_tests.py | 53 -- test/unittests/test_skill_settings.py | 127 +++++ 13 files changed, 2228 insertions(+), 424 deletions(-) create mode 100644 MANIFEST.in delete mode 100644 test/license_tests.py create mode 100644 test/unittests/test_skill_settings.py diff --git a/.github/workflows/license_tests.yml b/.github/workflows/license_tests.yml index 2ea5333..29f4063 100644 --- a/.github/workflows/license_tests.yml +++ b/.github/workflows/license_tests.yml @@ -6,9 +6,6 @@ on: pull_request: branches: - dev - paths: - - 'requirements/**' - - 'setup.py' workflow_dispatch: jobs: @@ -16,8 +13,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - with: - ref: ${{ github.head_ref }} - name: Setup Python uses: actions/setup-python@v1 with: @@ -32,12 +27,18 @@ jobs: - name: Install core repo run: | pip install . - - name: Install licheck - run: | - pip install git+https://github.com/NeonJarbas/lichecker - - name: Install test dependencies + - name: Get explicit and transitive dependencies run: | - pip install pytest pytest-timeout pytest-cov - - name: Test Licenses - run: | - pytest test/license_tests.py \ No newline at end of file + pip freeze > requirements-all.txt + - name: Check python + id: license_check_report + uses: pilosus/action-pip-license-checker@v0.5.0 + with: + requirements: 'requirements-all.txt' + fail: 'Copyleft,Other,Error' + fails-only: true + exclude: '^(tqdm).*' + exclude-license: '^(Mozilla).*$' + - name: Print report + if: ${{ always() }} + run: echo "${{ steps.license_check_report.outputs.report }}" \ No newline at end of file diff --git a/.gitignore b/.gitignore index 9595be5..2677076 100644 --- a/.gitignore +++ b/.gitignore @@ -1,20 +1,72 @@ -dev.env -.dev_opts.json -.idea -*.code-workspace -*.pyc -*.swp -*~ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ *.egg-info/ -build -dist +.installed.cfg +*.egg +pip-wheel-metadata/ + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ .coverage -/htmlcov -.installed -.mypy_cache -.vscode -.theia -.venv/ - -# Created by unit tests -.pytest_cache/ +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Ipython Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# poetry +poetry.lock + diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..3fa08ea --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,4 @@ +include README.md +include requirements.txt +include CHANGELOG.md +include LICENSE \ No newline at end of file diff --git a/ovos_backend_client/api.py b/ovos_backend_client/api.py index 6a9b933..85e8322 100644 --- a/ovos_backend_client/api.py +++ b/ovos_backend_client/api.py @@ -1,8 +1,16 @@ from ovos_utils import timed_lru_cache from ovos_utils.log import LOG -from ovos_backend_client.backends import OfflineBackend, PersonalBackend, \ - BackendType, get_backend_config, API_REGISTRY +import json +import os +from os import makedirs +from os.path import isfile +from ovos_config.config import Configuration +from ovos_utils.configuration import get_xdg_config_save_path +from ovos_backend_client.backends import OfflineBackend, \ + PersonalBackend, BackendType, get_backend_config, API_REGISTRY +from ovos_backend_client.database import SkillSettingsModel +from ovos_backend_client.settings import get_local_settings class BaseApi: @@ -12,9 +20,9 @@ def __init__(self, url=None, version="v1", identity_file=None, backend_type=None self.url = url self.credentials = credentials or {} if backend_type == BackendType.PERSONAL: - self.backend = PersonalBackend(url, version, identity_file) + self.backend = PersonalBackend(url, version, identity_file, credentials=credentials) else: # if backend_type == BackendType.OFFLINE: - self.backend = OfflineBackend(url, version, identity_file) + self.backend = OfflineBackend(url, version, identity_file, credentials=credentials) self.validate_backend_type() def validate_backend_type(self): @@ -150,14 +158,6 @@ def get(self, url=None, *args, **kwargs): """ Retrieve all device information from the web backend """ return self.backend.device_get() - def get_skill_settings_v1(self): - """ old style deprecated bidirectional skill settings api, still available! """ - return self.backend.device_get_skill_settings_v1() - - def put_skill_settings_v1(self, data): - """ old style deprecated bidirectional skill settings api, still available! """ - return self.backend.device_put_skill_settings_v1(data) - def get_settings(self): """ Retrieve device settings information from the web backend @@ -184,9 +184,6 @@ def update_version(self, enclosure_version="unknown"): return self.backend.device_update_version(core_version, platform, platform_build, enclosure_version) - def report_metric(self, name, data): - return self.backend.device_report_metric(name, data) - def get_location(self): """ Retrieve device location information from the web backend @@ -195,47 +192,6 @@ def get_location(self): """ return self.backend.device_get_location() - def get_subscription(self): - """ - Get information about type of subscription this unit is connected - to. - - Returns: dictionary with subscription information - """ - return self.backend.device_get_subscription() - - @property - def is_subscriber(self): - """ - status of subscription. True if device is connected to a paying - subscriber. - """ - return self.backend.is_subscriber - - def get_subscriber_voice_url(self, voice=None, arch=None): - return self.backend.device_get_subscriber_voice_url(voice, arch) - - def get_oauth_token(self, dev_cred): - """ - Get Oauth token for dev_credential dev_cred. - - Argument: - dev_cred: development credentials identifier - - Returns: - json string containing token and additional information - """ - return self.backend.device_get_oauth_token(dev_cred) - - # cached for 30 seconds because often 1 call per skill is done in quick succession - @timed_lru_cache(seconds=30) - def get_skill_settings(self): - """Get the remote skill settings for all skills on this device.""" - return self.backend.device_get_skill_settings() - - def send_email(self, title, body, sender): - return self.backend.device_send_email(title, body, sender) - def upload_skill_metadata(self, settings_meta): """Upload skill metadata. @@ -281,12 +237,87 @@ def upload_skills_data(self, data): return self.backend.device_upload_skills_data(to_send) + ## DEPRECATED APIS below, use dedicated classes instead + def get_oauth_token(self, dev_cred): + """ + Get Oauth token for dev_credential dev_cred. + + Argument: + dev_cred: development credentials identifier + + Returns: + json string containing token and additional information + """ + ## DEPRECATED - compat only for old devices + LOG.warning("DEPRECATED: use OAuthApi class instead") + return self.backend.device_get_oauth_token(dev_cred) + + def report_metric(self, name, data): + ## DEPRECATED - compat only for old devices + LOG.warning("DEPRECATED: use MetricsApi class instead") + return self.backend.device_report_metric(name, data) + + def get_subscription(self): + """ + Get information about type of subscription this unit is connected + to. + + Returns: dictionary with subscription information + """ + ## DEPRECATED - compat only for old devices + LOG.warning("DEPRECATED: there are no subscriptions") + return self.backend.device_get_subscription() + + @property + def is_subscriber(self): + """ + status of subscription. True if device is connected to a paying + subscriber. + """ + ## DEPRECATED - compat only for old devices + LOG.warning("DEPRECATED: there are no subscriptions") + return self.backend.is_subscriber + + def get_subscriber_voice_url(self, voice=None, arch=None): + ## DEPRECATED - compat only for old devices + LOG.warning("DEPRECATED: there are no subscriptions") + return self.backend.device_get_subscriber_voice_url(voice, arch) + + def get_skill_settings_v1(self): + """ old style deprecated bidirectional skill settings api, still available! """ + ## DEPRECATED - compat only for old devices + LOG.warning("DEPRECATED: use SkillSettingsApi class instead") + return self.backend.device_get_skill_settings_v1() + + def put_skill_settings_v1(self, data): + """ old style deprecated bidirectional skill settings api, still available! """ + ## DEPRECATED - compat only for old devices + LOG.warning("DEPRECATED: use SkillSettingsApi class instead") + return self.backend.device_put_skill_settings_v1(data) + + # cached for 30 seconds because often 1 call per skill is done in quick succession + @timed_lru_cache(seconds=30) + def get_skill_settings(self): + """Get the remote skill settings for all skills on this device.""" + ## DEPRECATED - compat only for old devices + LOG.warning("DEPRECATED: use SkillSettingsApi class instead") + return self.backend.device_get_skill_settings() + + def send_email(self, title, body, sender): + ## DEPRECATED - compat only for old devices + LOG.warning("DEPRECATED: use EmailApi class instead") + return self.backend.device_send_email(title, body, sender) + def upload_wake_word_v1(self, audio, params): """ upload precise wake word V1 endpoint - DEPRECATED""" + ## DEPRECATED - compat only for old devices + LOG.warning("DEPRECATED: use DatasetApi class instead") return self.backend.device_upload_wake_word_v1(audio, params) def upload_wake_word(self, audio, params): """ upload precise wake word V2 endpoint """ + ## DEPRECATED - compat only for old devices + LOG.warning("DEPRECATED: use DatasetApi class instead") return self.backend.device_upload_wake_word(audio, params) @@ -491,6 +522,33 @@ def send_email(self, title, body, sender): return self.backend.email_send(title, body, sender) +class SkillSettingsApi(BaseApi): + """Web API wrapper for skill settings""" + + def __init__(self, url=None, version="v1", identity_file=None, backend_type=None): + super().__init__(url, version, identity_file, backend_type) + + def validate_backend_type(self): + if not API_REGISTRY[self.backend_type]["skill_settings"]: + raise ValueError(f"{self.__class__.__name__} not available for {self.backend_type}") + + def upload_skill_settings(self): + """ upload skill settings from XDG path""" + return self.backend.skill_settings_upload(get_local_settings()) + + def download_skill_settings(self): + """ write downloaded settings to XDG path""" + settings = self.backend.skill_settings_download() + for s in settings: # list of SkillSettingsModel or dicts + settings_path = f"{get_xdg_config_save_path()}/skills/{s.skill_id}" + makedirs(settings_path, exist_ok=True) + with open( f"{settings_path}/settingsmeta.json", "w") as f: + json.dump(s.meta, f, indent=4, ensure_ascii=False) + with open(f"{settings_path}/settings.json", "w") as f: + json.dump(s.skill_settings, f, indent=4, ensure_ascii=False) + return settings + + class DatasetApi(BaseApi): """Web API wrapper for dataset collection""" @@ -504,6 +562,9 @@ def validate_backend_type(self): def upload_wake_word(self, audio, params, upload_url=None): return self.backend.dataset_upload_wake_word(audio, params, upload_url) + def upload_stt_recording(self, audio, params, upload_url=None): + return self.backend.dataset_upload_stt_recording(audio, params, upload_url) + class MetricsApi(BaseApi): """Web API wrapper for netrics collection""" @@ -542,11 +603,218 @@ def get_oauth_token(self, dev_cred): return self.backend.oauth_get_token(dev_cred) +class DatabaseApi(BaseApi): + """Web API wrapper for oauth api""" + + def __init__(self, admin_key=None, url=None, version="v1", identity_file=None, backend_type=None): + super().__init__(url, version, identity_file, backend_type, credentials={"admin": admin_key}) + self.url = f"{self.backend_url}/{self.backend_version}/admin" + + def validate_backend_type(self): + if not API_REGISTRY[self.backend_type]["database"]: + raise ValueError(f"{self.__class__.__name__} not available for {self.backend_type}") + if self.backend_type in [BackendType.PERSONAL] and not self.credentials.get("admin"): + raise ValueError(f"Admin key not set, can not access remote database") + + def list_devices(self): + return self.backend.db_list_devices() + + def get_device(self, uuid): + return self.backend.db_get_device(uuid) + + def update_device(self, uuid, name=None, + device_location=None, opt_in=False, + location=None, lang=None, date_format=None, + system_unit=None, time_format=None, email=None, + isolated_skills=False, ww_id=None, voice_id=None): + return self.backend.db_update_device(uuid, name, device_location, opt_in, + location, lang, date_format, system_unit, time_format, + email, isolated_skills, ww_id, voice_id) + + def delete_device(self, uuid): + return self.backend.db_delete_device(uuid) + + def add_device(self, uuid, token, name=None, + device_location="somewhere", + opt_in=Configuration().get("opt_in", False), + location=Configuration().get("location"), + lang=Configuration().get("lang"), + date_format=Configuration().get("date_format", "DMY"), + system_unit=Configuration().get("system_unit", "metric"), + time_format=Configuration().get("date_format", "full"), + email=None, + isolated_skills=False, + ww_id=None, + voice_id=None): + return self.backend.db_post_device(uuid, token, name, device_location, opt_in, + location, lang, date_format, system_unit, time_format, + email, isolated_skills, ww_id, voice_id) + + def list_shared_skill_settings(self): + return self.backend.db_list_shared_skill_settings() + + def get_shared_skill_settings(self, skill_id): + return self.backend.db_get_shared_skill_settings(skill_id) + + def update_shared_skill_settings(self, skill_id, + display_name=None, + settings_json=None, + metadata_json=None): + return self.backend.db_update_shared_skill_settings(skill_id, display_name, + settings_json, metadata_json) + + def delete_shared_skill_settings(self, skill_id): + return self.backend.db_delete_shared_skill_settings(skill_id) + + def add_shared_skill_settings(self, skill_id, + display_name, + settings_json, + metadata_json): + return self.backend.db_post_shared_skill_settings(skill_id, display_name, + settings_json, metadata_json) + + def list_skill_settings(self, uuid): + return self.backend.db_list_skill_settings(uuid) + + def get_skill_settings(self, uuid, skill_id): + return self.backend.db_get_skill_settings(uuid, skill_id) + + def update_skill_settings(self, uuid, skill_id, + display_name=None, + settings_json=None, + metadata_json=None): + return self.backend.db_update_skill_settings(uuid, skill_id, display_name, + settings_json, metadata_json) + + def delete_skill_settings(self, uuid, skill_id): + return self.backend.db_delete_skill_settings(uuid, skill_id) + + def add_skill_settings(self, uuid, skill_id, + display_name, + settings_json, + metadata_json): + return self.backend.db_post_skill_settings(uuid, skill_id, display_name, + settings_json, metadata_json) + + def list_oauth_apps(self): + return self.backend.db_list_oauth_apps() + + def get_oauth_app(self, token_id): + return self.backend.db_get_oauth_app(token_id) + + def update_oauth_app(self, token_id, client_id=None, client_secret=None, + auth_endpoint=None, token_endpoint=None, refresh_endpoint=None, + callback_endpoint=None, scope=None, shell_integration=None): + return self.backend.db_update_oauth_app(token_id, client_id, client_secret, auth_endpoint, token_endpoint, + refresh_endpoint, callback_endpoint, scope, shell_integration) + + def delete_oauth_app(self, token_id): + return self.backend.db_delete_oauth_app(token_id) + + def add_oauth_app(self, token_id, client_id, client_secret, + auth_endpoint, token_endpoint, refresh_endpoint, + callback_endpoint, scope, shell_integration=True): + return self.backend.db_post_oauth_app(token_id, client_id, client_secret, auth_endpoint, token_endpoint, + refresh_endpoint, callback_endpoint, scope, shell_integration) + + def list_oauth_tokens(self): + return self.backend.db_list_oauth_tokens() + + def get_oauth_token(self, token_id): + return self.backend.db_get_oauth_token(token_id) + + def update_oauth_token(self, token_id, token_data): + return self.backend.db_update_oauth_token(token_id, token_data) + + def delete_oauth_token(self, token_id): + return self.backend.db_delete_oauth_token(token_id) + + def add_oauth_token(self, token_id, token_data): + return self.backend.db_post_oauth_token(token_id, token_data) + + def list_stt_recordings(self): + return self.backend.db_list_stt_recordings() + + def get_stt_recording(self, rec_id): + return self.backend.db_get_stt_recording(rec_id) + + def update_stt_recording(self, rec_id, transcription=None, metadata=None): + return self.backend.db_update_stt_recording(rec_id, transcription, metadata) + + def delete_stt_recording(self, rec_id): + return self.backend.db_delete_stt_recording(rec_id) + + def add_stt_recording(self, byte_data, transcription, metadata=None): + return self.backend.db_post_stt_recording(byte_data, transcription, metadata) + + def list_ww_recordings(self): + return self.backend.db_list_ww_recordings() + + def get_ww_recording(self, rec_id): + return self.backend.db_get_ww_recording(rec_id) + + def update_ww_recording(self, rec_id, transcription=None, metadata=None): + return self.backend.db_update_ww_recording(rec_id, transcription, metadata) + + def delete_ww_recording(self, rec_id): + return self.backend.db_delete_ww_recording(rec_id) + + def add_ww_recording(self, byte_data, transcription, metadata=None): + return self.backend.db_post_ww_recording(byte_data, transcription, metadata) + + def list_metrics(self): + return self.backend.db_list_metrics() + + def get_metric(self, metric_id): + return self.backend.db_get_metric(metric_id) + + def update_metric(self, metric_id, metadata): + return self.backend.db_update_metric(metric_id, metadata) + + def delete_metric(self, metric_id): + return self.backend.db_delete_metric(metric_id) + + def add_metric(self, metric_type, metadata): + return self.backend.db_post_metric(metric_type, metadata) + + def list_ww_definitions(self): + return self.backend.db_list_ww_definitions() + + def get_ww_definition(self, ww_id): + return self.backend.db_get_ww_definition(ww_id) + + def update_ww_definition(self, ww_id, name, lang, ww_config, plugin): + return self.backend.db_update_ww_definition(ww_id, name, lang, ww_config, plugin) + + def delete_ww_definition(self, ww_id): + return self.backend.db_delete_ww_definition(ww_id) + + def add_ww_definition(self, name, lang, ww_config, plugin): + return self.backend.db_post_ww_definition(name, lang, ww_config, plugin) + + def list_voice_definitions(self): + return self.backend.db_list_voice_definitions() + + def get_voice_definition(self, voice_id): + return self.backend.db_get_voice_definition(voice_id) + + def update_voice_definition(self, voice_id, name=None, lang=None, plugin=None, + tts_config=None, offline=None, gender=None): + return self.backend.db_update_voice_definition(voice_id, name, lang, plugin, tts_config, offline, gender) + + def delete_voice_definition(self, voice_id): + return self.backend.db_delete_voice_definition(voice_id) + + def add_voice_definition(self, name, lang, plugin, + tts_config, offline, gender=None): + return self.backend.db_post_voice_definition(name, lang, plugin, tts_config, offline, gender) + + if __name__ == "__main__": # d = DeviceApi(FAKE_BACKEND_URL) # TODO turn these into unittests - # ident = load_identity() + # voice_id = load_identity() # paired = is_paired() geo = GeolocationApi(backend_type=BackendType.OFFLINE) data = geo.get_geolocation("Missouri Kansas") diff --git a/ovos_backend_client/backends/__init__.py b/ovos_backend_client/backends/__init__.py index 12d9378..376dd5b 100644 --- a/ovos_backend_client/backends/__init__.py +++ b/ovos_backend_client/backends/__init__.py @@ -7,19 +7,23 @@ API_REGISTRY = { BackendType.OFFLINE: { "admin": True, # updates mycroft.conf if used - "device": True, # shared database with local backend for UI compat - "dataset": True, # shared database with local backend for ww tagger UI compat - "metrics": True, # shared database with local backend for metrics UI compat + "database": True, # manages local files only + "device": True, # manages local files only + "skill_settings": True, # manages local files only + "dataset": True, # manages local files only + "metrics": True, # manages local files only "wolfram": True, # key needs to be set "geolocate": True, # nominatim - no key needed "stt": True, # uses OPM and reads from mycroft.conf "owm": True, # key needs to be set "email": True, # smtp config needs to be set - "oauth": True # use local backend UI on same device to register apps + "oauth": True # oauth PHAL plugin to register apps }, BackendType.PERSONAL: { "admin": True, + "database": True, # requires ovos-personal-backend>=0.2.0 "device": True, + "skill_settings": True, "dataset": True, "metrics": True, "wolfram": True, @@ -27,7 +31,7 @@ "stt": True, "owm": True, "email": True, - "oauth": True # can use local backend UI to register apps + "oauth": True # can use personal-backend-manager to register apps } } diff --git a/ovos_backend_client/backends/base.py b/ovos_backend_client/backends/base.py index 774df8a..2c77f69 100644 --- a/ovos_backend_client/backends/base.py +++ b/ovos_backend_client/backends/base.py @@ -1,9 +1,12 @@ import abc +import json from enum import Enum +from io import BytesIO, StringIO import requests from ovos_config.config import Configuration +from ovos_backend_client.database import SkillSettingsModel from ovos_backend_client.identity import IdentityManager try: @@ -98,6 +101,16 @@ def patch(self, url=None, *args, **kwargs): self.check_token() return requests.patch(url, headers=headers, timeout=(3.05, 15), *args, **kwargs) + def delete(self, url=None, *args, **kwargs): + url = url or self.url + if not url.startswith("http"): + url = f"http://{url}" + headers = self.headers + if "headers" in kwargs: + headers.update(kwargs.pop("headers")) + self.check_token() + return requests.delete(url, headers=headers, timeout=(3.05, 15), *args, **kwargs) + # OWM Api @staticmethod def _get_lat_lon(**kwargs): @@ -121,9 +134,8 @@ def owm_language(lang: str): """ OPEN_WEATHER_MAP_LANGUAGES = ( "af", "al", "ar", "bg", "ca", "cz", "da", "de", "el", "en", "es", "eu", "fa", "fi", "fr", "gl", "he", "hi", - "hr", "hu", "id", "it", "ja", "kr", "la", "lt", "mk", "nl", "no", "pl", "pt", "pt_br", "ro", "ru", "se", - "sk", - "sl", "sp", "sr", "sv", "th", "tr", "ua", "uk", "vi", "zh_cn", "zh_tw", "zu" + "hr", "hu", "id", "it", "ja", "kr", "la", "lt", "mk", "nl", "no", "pl", "pt", "pt_br", "ro", "ru", + "se", "sk", "sl", "sp", "sr", "sv", "th", "tr", "ua", "uk", "vi", "zh_cn", "zh_tw", "zu" ) special_cases = {"cs": "cz", "ko": "kr", "lv": "la"} lang_primary, lang_subtag = lang.split('-') @@ -263,18 +275,6 @@ def stt_get(self, audio, language="en-us", limit=1): raise NotImplementedError() # Device Api - @property - def is_subscriber(self): - """ - status of subscription. True if device is connected to a paying - subscriber. - """ - try: - return self.device_get_subscription().get('@type') != 'free' - except Exception: - # If can't retrieve, assume not paired and not a subscriber yet - return False - def device_get(self): """ Retrieve all device information from the web backend """ return {"uuid": IdentityManager.get().uuid, @@ -295,16 +295,6 @@ def device_get_settings(self): """ raise NotImplementedError() - @abc.abstractmethod - def device_get_skill_settings_v1(self): - """ old style bidirectional skill settings api, still available!""" - raise NotImplementedError() - - @abc.abstractmethod - def device_put_skill_settings_v1(self, data=None): - """ old style bidirectional skill settings api, still available!""" - raise NotImplementedError() - @abc.abstractmethod def device_get_code(self, state=None): raise NotImplementedError() @@ -325,10 +315,6 @@ def device_update_version(self, enclosure_version="unknown"): raise NotImplementedError() - @abc.abstractmethod - def device_report_metric(self, name, data): - raise NotImplementedError() - @abc.abstractmethod def device_get_location(self): """ Retrieve device location information from the web backend @@ -338,48 +324,6 @@ def device_get_location(self): """ raise NotImplementedError() - def device_get_subscription(self): - """ - Get information about type of subscription this unit is connected - to. - - Returns: dictionary with subscription information - """ - return {"@type": "free"} - - def device_get_subscriber_voice_url(self, voice=None, arch=None): - return None - - @abc.abstractmethod - def device_get_oauth_token(self, dev_cred): - """ - Get Oauth token for dev_credential dev_cred. - - Argument: - dev_cred: development credentials identifier - - Returns: - json string containing token and additional information - """ - raise NotImplementedError() - - @abc.abstractmethod - def device_get_skill_settings(self): - """Get the remote skill settings for all skills on this device.""" - raise NotImplementedError() - - def device_send_email(self, title, body, sender): - return self.email_send(title, body, sender) - - @abc.abstractmethod - def device_upload_skill_metadata(self, settings_meta): - """Upload skill metadata. - - Args: - settings_meta (dict): skill info and settings in JSON format - """ - raise NotImplementedError() - @abc.abstractmethod def device_upload_skills_data(self, data): """ Upload skills.json file. This file contains a manifest of installed @@ -390,16 +334,6 @@ def device_upload_skills_data(self, data): """ raise NotImplementedError() - @abc.abstractmethod - def device_upload_wake_word_v1(self, audio, params): - """ upload precise wake word V1 endpoint - url can be external to backend""" - raise NotImplementedError() - - @abc.abstractmethod - def device_upload_wake_word(self, audio, params): - """ upload precise wake word V2 endpoint - integrated with device api""" - raise NotImplementedError() - # Metrics API @abc.abstractmethod def metrics_upload(self, name, data): @@ -421,16 +355,48 @@ def oauth_get_token(self, dev_cred): raise NotImplementedError() # Dataset API - @abc.abstractmethod - def dataset_upload_wake_word(self, audio, params): + def dataset_upload_wake_word(self, audio, params, upload_url=None): """ upload wake word sample - url can be external to backend""" - raise NotImplementedError() + byte_data = audio.get_wav_data() + upload_url = upload_url or Configuration().get("listener", {}).get("wake_word_upload", {}).get("url") + if upload_url: + # upload to arbitrary server + ww_files = { + 'audio': BytesIO(byte_data), + 'metadata': StringIO(json.dumps(params)) + } + return self.post(upload_url, files=ww_files) + return {} + + def dataset_upload_stt_recording(self, audio, params, upload_url=None): + """ upload stt sample - url can be external to backend""" + byte_data = audio.get_wav_data() + upload_url = upload_url or Configuration().get("listener", {}).get("utterance_upload", {}).get("url") + if upload_url: + # upload to arbitrary server + ww_files = { + 'audio': BytesIO(byte_data), + 'metadata': StringIO(json.dumps(params)) + } + return self.post(upload_url, files=ww_files) + return {} # Email API @abc.abstractmethod def email_send(self, title, body, sender): raise NotImplementedError() + # Skill settings api + @abc.abstractmethod + def skill_settings_upload(self, skill_settings): + # list of SkillSettingsModel or dicts + raise NotImplementedError() + + @abc.abstractmethod + def skill_settings_download(self): + # return list of SkillSettingsModel + raise NotImplementedError() + # Admin Api @abc.abstractmethod def admin_pair(self, uuid=None): @@ -493,3 +459,326 @@ def admin_set_device_info(self, uuid, info): "lang": "en-us"} """ raise NotImplementedError() + + # Database api + @abc.abstractmethod + def db_list_devices(self): + raise NotImplementedError() + + @abc.abstractmethod + def db_get_device(self, uuid): + raise NotImplementedError() + + @abc.abstractmethod + def db_update_device(self, uuid, name=None, + device_location=None, opt_in=False, + location=None, lang=None, date_format=None, + system_unit=None, time_format=None, email=None, + isolated_skills=False, ww_id=None, voice_id=None): + raise NotImplementedError() + + @abc.abstractmethod + def db_delete_device(self, uuid): + raise NotImplementedError() + + @abc.abstractmethod + def db_post_device(self, uuid, token, name=None, + device_location="somewhere", + opt_in=Configuration().get("opt_in", False), + location=Configuration().get("location"), + lang=Configuration().get("lang"), + date_format=Configuration().get("date_format", "DMY"), + system_unit=Configuration().get("system_unit", "metric"), + time_format=Configuration().get("date_format", "full"), + email=None, + isolated_skills=False, + ww_id=None, + voice_id=None): + raise NotImplementedError() + + @abc.abstractmethod + def db_list_shared_skill_settings(self): + raise NotImplementedError() + + @abc.abstractmethod + def db_get_shared_skill_settings(self, skill_id): + raise NotImplementedError() + + @abc.abstractmethod + def db_update_shared_skill_settings(self, skill_id, + display_name=None, + settings_json=None, + metadata_json=None): + raise NotImplementedError() + + @abc.abstractmethod + def db_delete_shared_skill_settings(self, skill_id): + raise NotImplementedError() + + @abc.abstractmethod + def db_post_shared_skill_settings(self, skill_id, + display_name, + settings_json, + metadata_json): + raise NotImplementedError() + + @abc.abstractmethod + def db_list_skill_settings(self, uuid): + raise NotImplementedError() + + @abc.abstractmethod + def db_get_skill_settings(self, uuid, skill_id): + raise NotImplementedError() + + @abc.abstractmethod + def db_update_skill_settings(self, uuid, skill_id, + display_name=None, + settings_json=None, + metadata_json=None): + raise NotImplementedError() + + @abc.abstractmethod + def db_delete_skill_settings(self, uuid, skill_id): + raise NotImplementedError() + + @abc.abstractmethod + def db_post_skill_settings(self, uuid, skill_id, + display_name, + settings_json, + metadata_json): + raise NotImplementedError() + + @abc.abstractmethod + def db_list_oauth_apps(self): + raise NotImplementedError() + + @abc.abstractmethod + def db_get_oauth_app(self, token_id): + raise NotImplementedError() + + @abc.abstractmethod + def db_update_oauth_app(self, token_id, client_id=None, client_secret=None, + auth_endpoint=None, token_endpoint=None, refresh_endpoint=None, + callback_endpoint=None, scope=None, shell_integration=None): + raise NotImplementedError() + + @abc.abstractmethod + def db_delete_oauth_app(self, token_id): + raise NotImplementedError() + + @abc.abstractmethod + def db_post_oauth_app(self, token_id, client_id, client_secret, + auth_endpoint, token_endpoint, refresh_endpoint, + callback_endpoint, scope, shell_integration=True): + raise NotImplementedError() + + @abc.abstractmethod + def db_list_oauth_tokens(self): + raise NotImplementedError() + + @abc.abstractmethod + def db_get_oauth_token(self, token_id): + raise NotImplementedError() + + @abc.abstractmethod + def db_update_oauth_token(self, token_id, token_data): + raise NotImplementedError() + + @abc.abstractmethod + def db_delete_oauth_token(self, token_id): + raise NotImplementedError() + + @abc.abstractmethod + def db_post_oauth_token(self, token_id, token_data): + raise NotImplementedError() + + @abc.abstractmethod + def db_list_stt_recordings(self): + raise NotImplementedError() + + @abc.abstractmethod + def db_get_stt_recording(self, rec_id): + raise NotImplementedError() + + @abc.abstractmethod + def db_update_stt_recording(self, rec_id, transcription=None, metadata=None): + raise NotImplementedError() + + @abc.abstractmethod + def db_delete_stt_recording(self, rec_id): + raise NotImplementedError() + + @abc.abstractmethod + def db_post_stt_recording(self, byte_data, transcription, metadata=None): + raise NotImplementedError() + + @abc.abstractmethod + def db_list_ww_recordings(self): + raise NotImplementedError() + + @abc.abstractmethod + def db_get_ww_recording(self, rec_id): + raise NotImplementedError() + + @abc.abstractmethod + def db_update_ww_recording(self, rec_id, transcription=None, metadata=None): + raise NotImplementedError() + + @abc.abstractmethod + def db_delete_ww_recording(self, rec_id): + raise NotImplementedError() + + @abc.abstractmethod + def db_post_ww_recording(self, byte_data, transcription, metadata=None): + raise NotImplementedError() + + @abc.abstractmethod + def db_list_metrics(self): + raise NotImplementedError() + + @abc.abstractmethod + def db_get_metric(self, metric_id): + raise NotImplementedError() + + @abc.abstractmethod + def db_update_metric(self, metric_id, metadata): + raise NotImplementedError() + + @abc.abstractmethod + def db_delete_metric(self, metric_id): + raise NotImplementedError() + + @abc.abstractmethod + def db_post_metric(self, metric_type, metadata): + raise NotImplementedError() + + @abc.abstractmethod + def db_list_ww_definitions(self): + raise NotImplementedError() + + @abc.abstractmethod + def db_get_ww_definition(self, ww_id): + raise NotImplementedError() + + @abc.abstractmethod + def db_update_ww_definition(self, ww_id, name, lang, ww_config, plugin): + raise NotImplementedError() + + @abc.abstractmethod + def db_delete_ww_definition(self, ww_id): + raise NotImplementedError() + + @abc.abstractmethod + def db_post_ww_definition(self, name, lang, ww_config, plugin): + raise NotImplementedError() + + @abc.abstractmethod + def db_list_voice_definitions(self): + raise NotImplementedError() + + @abc.abstractmethod + def db_get_voice_definition(self, voice_id): + raise NotImplementedError() + + @abc.abstractmethod + def db_update_voice_definition(self, voice_id, name=None, lang=None, plugin=None, + tts_config=None, offline=None, gender=None): + raise NotImplementedError() + + @abc.abstractmethod + def db_delete_voice_definition(self, voice_id): + raise NotImplementedError() + + @abc.abstractmethod + def db_post_voice_definition(self, name, lang, plugin, + tts_config, offline, gender=None): + raise NotImplementedError() + + # DEPRECATED APIS + @property + def is_subscriber(self): + """ + status of subscription. True if device is connected to a paying + subscriber. + """ + ## DEPRECATED - compat only for old devices + try: + return self.device_get_subscription().get('@type') != 'free' + except Exception: + # If can't retrieve, assume not paired and not a subscriber yet + return False + + def device_report_metric(self, name, data): + ## DEPRECATED - compat only for old devices + return self.metrics_upload(name, data) + + def device_get_oauth_token(self, dev_cred): + """ + Get Oauth token for dev_credential dev_cred. + + Argument: + dev_cred: development credentials identifier + + Returns: + json string containing token and additional information + """ + ## DEPRECATED - compat only for old devices + raise self.oauth_get_token(dev_cred) + + def device_get_skill_settings_v1(self): + """ old style bidirectional skill settings api, still available!""" + ## DEPRECATED - compat only for old devices + return [s.serialize() for s in self.skill_settings_download()] + + def device_put_skill_settings_v1(self, data=None): + """ old style bidirectional skill settings api, still available!""" + ## DEPRECATED - compat only for old devices + s = SkillSettingsModel.deserialize(data) + settings = [s] + self.skill_settings_upload(settings) + return {} + + def device_get_skill_settings(self): + """Get the remote skill settings for all skills on this device.""" + ## DEPRECATED - compat only for old devices + return {s.skill_id: s.skill_settings for s in self.skill_settings_download()} + + def device_upload_skill_metadata(self, settings_meta): + """Upload skill metadata. + + Args: + settings_meta (dict): skill info and settings in JSON format + """ + ## DEPRECATED - compat only for old devices + s = SkillSettingsModel.deserialize(settings_meta) + settings = self.device_get_skill_settings() + old_s = settings.get(s.skill_id) + if old_s: # keep old settings value, update meta values only + s.skill_settings = old_s.skill_settings + self.device_put_skill_settings_v1(s.serialize()) + + def device_send_email(self, title, body, sender): + ## DEPRECATED - compat only for old devices + return self.email_send(title, body, sender) + + def device_upload_wake_word_v1(self, audio, params, upload_url=None): + """ upload precise wake word V1 endpoint - url can be external to backend""" + return self.dataset_upload_wake_word(audio, params, upload_url) + + def device_upload_wake_word(self, audio, params): + """ upload precise wake word V2 endpoint - integrated with device api""" + return self.dataset_upload_wake_word(audio, params) + + def device_get_subscription(self): + """ + Get information about type of subscription this unit is connected + to. + + Returns: dictionary with subscription information + """ + ## DEPRECATED - compat only for old devices + return {"@type": "free"} + + def device_get_subscriber_voice_url(self, voice=None, arch=None): + ## DEPRECATED - compat only for old devices + return None diff --git a/ovos_backend_client/backends/offline.py b/ovos_backend_client/backends/offline.py index c91931f..63158f2 100644 --- a/ovos_backend_client/backends/offline.py +++ b/ovos_backend_client/backends/offline.py @@ -1,22 +1,50 @@ import json +import os import time -from io import BytesIO, StringIO +from os import listdir, makedirs, remove +from os.path import isfile, join from tempfile import NamedTemporaryFile from uuid import uuid4 import requests -from json_database import JsonStorageXDG from ovos_config.config import Configuration from ovos_config.config import update_mycroft_config -from ovos_plugin_manager.stt import OVOSSTTFactory, get_stt_config +from ovos_config.locations import USER_CONFIG from ovos_utils import timed_lru_cache -from ovos_utils.log import LOG +from ovos_utils.configuration import get_xdg_config_save_path, get_xdg_data_save_path from ovos_utils.network_utils import get_external_ip from ovos_utils.smtp_utils import send_smtp +from ovos_utils.xdg_utils import xdg_data_home from ovos_backend_client.backends.base import AbstractBackend, BackendType -from ovos_backend_client.database import BackendDatabase +from ovos_backend_client.database import JsonMetricDatabase, JsonWakeWordDatabase, \ + SkillSettingsModel, OAuthTokenDatabase, OAuthApplicationDatabase, DeviceModel, JsonUtteranceDatabase from ovos_backend_client.identity import IdentityManager +from ovos_backend_client.settings import get_local_settings + +try: + from ovos_plugin_manager.tts import get_voices, get_voice_id + from ovos_plugin_manager.wakewords import get_ww_id, get_wws +except ImportError: + from hashlib import md5 + + + def get_ww_id(plugin_name, ww_name, ww_config): + ww_hash = md5(json.dumps(ww_config, sort_keys=True).encode("utf-8")).hexdigest() + return f"{plugin_name}_{ww_name}_{ww_hash}" + + + def get_voice_id(plugin_name, lang, tts_config): + tts_hash = md5(json.dumps(tts_config, sort_keys=True).encode("utf-8")).hexdigest() + return f"{plugin_name}_{lang}_{tts_hash}" + + + def get_voices(): + return [] + + + def get_wws(): + return [] class OfflineBackend(AbstractBackend): @@ -296,32 +324,18 @@ def ip_geolocation_get(self, ip): # Device Api def device_get(self): - """ Retrieve all device information from the web backend """ - data = JsonStorageXDG("ovos_device_info.json", subfolder="OpenVoiceOS") - for k, v in super().device_get().items(): - if k not in data: - data[k] = v - return data + """ Retrieve all device information from the json db""" + device = DeviceModel() + return device.selene_device def device_get_settings(self): - """ Retrieve device settings information from the web backend + """ Retrieve device settings information from the json db Returns: str: JSON string with user configuration information. """ - LOG.warning("Offline Backend, you may want to reference " - "`ovos_config.Configuration()` to get full config") - return Configuration.remote - - def device_get_skill_settings_v1(self): - """ old style bidirectional skill settings api, still available!""" - # TODO scan skill xdg paths - return [] - - def device_put_skill_settings_v1(self, data=None): - """ old style bidirectional skill settings api, still available!""" - # do nothing, skills manage their own settings lifecycle - return {} + device = DeviceModel() + return device.selene_settings def device_get_code(self, state=None): return "ABCDEF" # dummy data @@ -331,19 +345,7 @@ def device_activate(self, state, token, platform="unknown", platform_build="unknown", enclosure_version="unknown"): - data = {"state": state, - "token": token, - "coreVersion": core_version, - "platform": platform, - "platform_build": platform_build, - "enclosureVersion": enclosure_version} identity = self.admin_pair(state) - data["uuid"] = data.pop("state") - data["token"] = self.access_token - BackendDatabase(self.uuid).update_device_db(data) - db = JsonStorageXDG("ovos_device_info.json", subfolder="OpenVoiceOS") - db.update(data) - db.store() return identity def device_update_version(self, @@ -351,53 +353,16 @@ def device_update_version(self, platform="unknown", platform_build="unknown", enclosure_version="unknown"): - data = {"coreVersion": core_version, - "platform": platform, - "platform_build": platform_build, - "enclosureVersion": enclosure_version, - "token": self.access_token} - BackendDatabase(self.uuid).update_device_db(data) - db = JsonStorageXDG("ovos_device_info.json", subfolder="OpenVoiceOS") - db.update(data) - db.store() - - def device_report_metric(self, name, data): - return self.metrics_upload(name, data) + pass # irrelevant info def device_get_location(self): - """ Retrieve device location information from the web backend + """ Retrieve device location information from Configuration Returns: str: JSON string with user location. """ return Configuration().get("location") or {} - def device_get_oauth_token(self, dev_cred): - """ - Get Oauth token for dev_credential dev_cred. - - Argument: - dev_cred: development credentials identifier - - Returns: - json string containing token and additional information - """ - raise self.oauth_get_token(dev_cred) - - def device_get_skill_settings(self): - """Get the remote skill settings for all skills on this device.""" - # TODO - scan xdg paths ? - return {} - - def device_upload_skill_metadata(self, settings_meta): - """Upload skill metadata. - - Args: - settings_meta (dict): skill info and settings in JSON format - """ - # Do nothing, skills manage their own settingsmeta.json files - return - def device_upload_skills_data(self, data): """ Upload skills.json file. This file contains a manifest of installed and failed installations for use with the Marketplace. @@ -405,38 +370,37 @@ def device_upload_skills_data(self, data): Args: data: dictionary with skills data from msm """ - with JsonStorageXDG("ovos_skills_meta.json", subfolder="OpenVoiceOS") as db: - db.update(data) - - def device_upload_wake_word_v1(self, audio, params, upload_url=None): - """ upload precise wake word V1 endpoint - url can be external to backend""" - return self.dataset_upload_wake_word(audio, params, upload_url) - - def device_upload_wake_word(self, audio, params): - """ upload precise wake word V2 endpoint - integrated with device api""" - return self.dataset_upload_wake_word(audio, params) + pass # Metrics API def metrics_upload(self, name, data): """ upload metrics""" - BackendDatabase(self.uuid).update_metrics_db(name, data) - return {} + return self.db_post_metric(name, data) + + # Skill settings api + def skill_settings_upload(self, skill_settings): + # update on disk, settings already local + for s in skill_settings: + s.store() + + def skill_settings_download(self): + # settings already local + return get_local_settings() # Dataset API def dataset_upload_wake_word(self, audio, params, upload_url=None): """ upload wake word sample - url can be external to backend""" + byte_data = audio.get_wav_data() if Configuration().get("listener", {}).get('record_wake_words'): - BackendDatabase(self.uuid).update_ww_db(params) # update metadata db for ww tagging UI - - upload_url = upload_url or Configuration().get("listener", {}).get("wake_word_upload", {}).get("url") - if upload_url: - # upload to arbitrary server - ww_files = { - 'audio': BytesIO(audio.get_wav_data()), - 'metadata': StringIO(json.dumps(params)) - } - return self.post(upload_url, files=ww_files) - return {} + self.db_post_ww_recording(byte_data, params["name"], params) + return super().dataset_upload_stt_recording(audio, params, upload_url) + + def dataset_upload_stt_recording(self, audio, params, upload_url=None): + """ upload stt sample - url can be external to backend""" + byte_data = audio.get_wav_data() + if Configuration().get("listener", {}).get('record_utterances'): + self.db_post_ww_recording(byte_data, params["transcription"], params) + return super().dataset_upload_wake_word(audio, params, upload_url) # Email API def email_send(self, title, body, sender): @@ -469,7 +433,7 @@ def oauth_get_token(self, dev_cred): Returns: json string containing token and additional information """ - return JsonStorageXDG("ovos_oauth").get(dev_cred) or {} + return self.db_get_oauth_token(dev_cred) # Admin API def admin_pair(self, uuid=None): @@ -525,8 +489,6 @@ def admin_set_device_prefs(self, uuid, prefs): "tts_module": "ovos-tts-plugin-mimic", "tts_config": {"voice": "ap"}} """ - with JsonStorageXDG("ovos_device_info.json", subfolder="OpenVoiceOS") as db: - db.update(prefs) cfg = dict(prefs) cfg["listener"] = {} cfg["hotwords"] = {} @@ -562,12 +524,14 @@ def admin_set_device_info(self, uuid, info): "isolated_skills": False, "lang": "en-us"} """ - update_mycroft_config({"opt_in": info["opt_in"], "lang": info["lang"]}) - with JsonStorageXDG("ovos_device_info.json", subfolder="OpenVoiceOS") as db: - db.update(info) + update_mycroft_config({"opt_in": info["opt_in"], + "email": {"recipient": info.get("email")}, + "lang": info["lang"]}) # STT Api def load_stt_plugin(self, config=None, lang=None): + from ovos_plugin_manager.stt import OVOSSTTFactory, get_stt_config + config = config or get_stt_config(config) if lang: config["lang"] = lang @@ -594,6 +558,455 @@ def stt_get(self, audio, language="en-us", limit=1): tx = [tx] return tx + # Database API + def db_list_devices(self): + _mail_cfg = self.credentials.get("email", {}) + + tts_plug = Configuration().get("tts").get("module") + tts_config = Configuration().get("tts")[tts_plug] + + default_ww = Configuration().get("listener").get("wake_word", "hey_mycroft") + ww_config = Configuration().get("hotwords")[default_ww] + + device = { + "uuid": self.uuid, + "token": "DUMMYTOKEN123", + "isolated_skills": True, + "opt_in": Configuration().get("opt_in", False), + "name": f"Device-{self.uuid}", + "device_location": "somewhere", + "email": _mail_cfg.get("recipient") or + _mail_cfg.get("smtp", {}).get("username"), + "time_format": Configuration().get("time_format", "full"), + "date_format": Configuration().get("date_format", "DMY"), + "system_unit": Configuration().get("system_unit", "metric"), + "lang": Configuration().get("lang") or "en-us", + "location": Configuration().get("location"), + "default_tts": tts_plug, + "default_tts_cfg": tts_config, + "default_ww": default_ww, + "default_ww_cfg": ww_config + } + return [device] + + def db_get_device(self, uuid): + if uuid != self.uuid: + return None + return self.db_list_devices()[0] + + def db_update_device(self, uuid, name=None, + device_location=None, opt_in=None, + location=None, lang=None, date_format=None, + system_unit=None, time_format=None, email=None, + isolated_skills=False, ww_id=None, voice_id=None): + if uuid != self.uuid: + identity = self.admin_pair(uuid) + new_config = { + + } + if opt_in is not None: + new_config["opt_in"] = opt_in + if location is not None: + new_config["location"] = location + if lang is not None: + new_config["lang"] = lang + if time_format is not None: + new_config["time_format"] = time_format + if date_format is not None: + new_config["date_format"] = date_format + if system_unit is not None: + new_config["system_unit"] = system_unit + if email is not None: + new_config["email"]["recipient"] = email + if device_location is not None: + pass # not tracked locally, reserved for future usage + if ww_id is not None: + ww_def = self.db_get_ww_definition(ww_id) + if ww_def: + name = ww_def["name"] + cfg = ww_def["ww_config"] + if "module" not in cfg: + cfg["module"] = ww_def["plugin"] + new_config["listener"]["wake_word"] = name + new_config["hotwords"][name] = cfg + + if voice_id is not None: + ww_def = self.db_get_voice_definition(voice_id) + if ww_def: + plugin = ww_def["plugin"] + cfg = ww_def["tts_config"] + # TODO - gender -> persona + if ww_def.get("lang"): + cfg["lang"] = ww_def["lang"] + new_config["tts"]["module"] = plugin + new_config["tts"][plugin] = cfg + update_mycroft_config(new_config) + + def db_delete_device(self, uuid): + # delete identity file/user config/skill settings + + settings_path = f"{get_xdg_config_save_path()}/skills" + + skill_ids = listdir(settings_path) + + # delete skill settings + for skill_id in skill_ids: + s = f"{settings_path}/{skill_id}/settings.json" + if isfile(s): + remove(s) + + if isfile(IdentityManager.IDENTITY_FILE): + remove(IdentityManager.IDENTITY_FILE) + + if isfile(USER_CONFIG): + remove(USER_CONFIG) + + def db_post_device(self, uuid, token, *args, **kwargs): + return self.db_update_device(uuid, *args, **kwargs) + + def db_list_shared_skill_settings(self): + return [s.serialize() for s in get_local_settings()] + + def db_get_shared_skill_settings(self, skill_id): + settings = get_local_settings() + skill_settings = [] + for s in settings: + if s.skill_id == skill_id: + skill_settings.append(s.serialize()) + return skill_settings + + def db_update_shared_skill_settings(self, skill_id, + display_name=None, + settings_json=None, + metadata_json=None): + settings_path = f"{get_xdg_config_save_path()}/skills/{skill_id}" + makedirs(settings_path, exist_ok=True) + if metadata_json: + s = f"{settings_path}/settingsmeta.json" + with open(s, "w") as f: + json.dump(metadata_json, f) + if settings_json: + s = f"{settings_path}/settings.json" + with open(s, "w") as f: + json.dump(metadata_json, f) + return SkillSettingsModel(skill_id=skill_id, + skill_settings=settings_path, + meta=metadata_json, + display_name=display_name).serialize() + + def db_delete_shared_skill_settings(self, skill_id): + settings_path = f"{get_xdg_config_save_path()}/skills/{skill_id}" + deleted = False + s = f"{settings_path}/settingsmeta.json" + if isfile(s): + remove(s) + deleted = True + s = f"{settings_path}/settings.json" + if isfile(s): + remove(s) + deleted = True + return deleted + + def db_post_shared_skill_settings(self, skill_id, + display_name, + settings_json, + metadata_json): + return self.db_update_shared_skill_settings(skill_id, + display_name=display_name, + settings_json=settings_json, + metadata_json=metadata_json) + + def db_list_skill_settings(self, uuid): + return self.db_list_shared_skill_settings() + + def db_get_skill_settings(self, uuid, skill_id): + return self.db_get_shared_skill_settings(skill_id) + + def db_update_skill_settings(self, uuid, skill_id, + display_name=None, + settings_json=None, + metadata_json=None): + return self.db_update_shared_skill_settings(skill_id, display_name=display_name, + settings_json=settings_json, metadata_json=metadata_json) + + def db_delete_skill_settings(self, uuid, skill_id): + return self.db_delete_shared_skill_settings(skill_id) + + def db_post_skill_settings(self, uuid, skill_id, + display_name, + settings_json, + metadata_json): + return self.db_post_shared_skill_settings(skill_id, display_name=display_name, + settings_json=settings_json, metadata_json=metadata_json) + + def db_list_oauth_apps(self): + return OAuthApplicationDatabase().values() + + def db_get_oauth_app(self, token_id): + return OAuthApplicationDatabase().get_application(token_id) + + def db_update_oauth_app(self, token_id, client_id=None, client_secret=None, + auth_endpoint=None, token_endpoint=None, refresh_endpoint=None, + callback_endpoint=None, scope=None, shell_integration=None): + with OAuthApplicationDatabase() as db: + return db.add_token(token_id, client_id, client_secret, + auth_endpoint, token_endpoint, refresh_endpoint, + callback_endpoint, scope, shell_integration) + + def db_delete_oauth_app(self, token_id): + with OAuthApplicationDatabase() as db: + return db.delete_application(token_id) + + def db_post_oauth_app(self, token_id, client_id, client_secret, + auth_endpoint, token_endpoint, refresh_endpoint, + callback_endpoint, scope, shell_integration=True): + with OAuthApplicationDatabase() as db: + return db.add_application(token_id, client_id, client_secret, + auth_endpoint, token_endpoint, + refresh_endpoint, callback_endpoint, + scope, shell_integration) + + def db_list_oauth_tokens(self): + return OAuthTokenDatabase().values() + + def db_get_oauth_token(self, token_id): + """ + Get Oauth token for dev_credential dev_cred. + + Argument: + dev_cred: development credentials identifier + + Returns: + json string containing token and additional information + """ + return OAuthTokenDatabase().get_token(token_id) + + def db_update_oauth_token(self, token_id, token_data): + with OAuthTokenDatabase() as db: + return db.add_token(token_id, token_data) + + def db_delete_oauth_token(self, token_id): + with OAuthTokenDatabase() as db: + return db.delete_token(token_id) + + def db_post_oauth_token(self, token_id, token_data): + with OAuthTokenDatabase() as db: + return db.add_token(token_id, token_data) + + def db_list_stt_recordings(self): + return JsonUtteranceDatabase().values() + + def db_get_stt_recording(self, rec_id): + return JsonUtteranceDatabase().get_utterance(rec_id).serialize() + + def db_update_stt_recording(self, rec_id, transcription=None, metadata=None): + # TODO - metadata unused, extend db + return JsonUtteranceDatabase().update_utterance(rec_id, transcription) + + def db_delete_stt_recording(self, rec_id): + return JsonUtteranceDatabase().delete_utterance(rec_id) + + def db_post_stt_recording(self, byte_data, transcription, metadata=None): + # TODO - metadata unused, extend db + save_path = Configuration().get("listener", {}).get('save_path') or \ + f"{get_xdg_data_save_path()}/listener/utterances" + os.makedirs(save_path, exist_ok=True) + + with JsonUtteranceDatabase() as db: + n = f"{transcription.lower().replace('/', '_').replace(' ', '_')}_{db.total_utterances() + 1}" + with open(f"{save_path}/{n}.wav", "wb") as f: + f.write(byte_data) + return db.add_utterance(transcription, f"{save_path}/{n}.wav", self.uuid) + + def db_list_ww_recordings(self): + return JsonWakeWordDatabase().values() + + def db_get_ww_recording(self, rec_id): + return JsonWakeWordDatabase().get_wakeword(rec_id).serialize() + + def db_update_ww_recording(self, rec_id, transcription=None, metadata=None): + with JsonWakeWordDatabase() as db: + db.update_wakeword(rec_id, transcription=transcription, meta=metadata) + + def db_delete_ww_recording(self, rec_id): + with JsonWakeWordDatabase() as db: + db.delete_wakeword(rec_id) + + def db_post_ww_recording(self, byte_data, transcription, metadata=None): + listener_config = Configuration().get("listener", {}) + save_path = listener_config.get('save_path', f"{get_xdg_data_save_path()}/listener/wake_words") + filename = join(save_path, '_'.join(str(metadata[k]) for k in sorted(metadata)) + '.wav') + os.makedirs(save_path, exist_ok=True) + metadata = metadata or {} + with open(save_path, "wb") as f: + f.write(byte_data) + with JsonWakeWordDatabase() as db: + db.add_wakeword(metadata["name"], filename, metadata, self.uuid) + + def db_list_metrics(self): + return JsonMetricDatabase().values() + + def db_get_metric(self, metric_id): + return JsonMetricDatabase().get(metric_id) + + def db_update_metric(self, metric_id, metadata): + m = self.db_get_metric(metric_id) + m.meta = metadata + with JsonMetricDatabase() as db: + db[metric_id] = m + + def db_delete_metric(self, metric_id): + with JsonMetricDatabase() as db: + if metric_id in db: + db.pop(metric_id) + return True + return False + + def db_post_metric(self, metric_type, metadata): + with JsonMetricDatabase() as db: + m = db.add_metric(metric_type, metadata, self.uuid) + return m.serialize() + + def db_list_ww_definitions(self): + ww_defs = [] + for ww_id, ww_cfg in get_wws().items(): # TODO scan=True once implemented + plugin, name, _ = ww_id.split("_", 3) + ww_defs.append({ + "ww_id": ww_id, + "name": ww_cfg.get("display_name") or name, + "lang": ww_cfg.get("stt_lang") or + ww_cfg.get("lang") or + Configuration().get("lang", "en-us"), + "plugin": ww_cfg.get("module") or plugin, + "ww_config": ww_cfg + }) + return ww_defs + + def db_get_ww_definition(self, ww_id): + ww_defs = self.db_list_ww_definitions() + for ww in ww_defs: + if ww["ww_id"] == ww_id: + return ww + + def db_update_ww_definition(self, ww_id, name=None, lang=None, ww_config=None, plugin=None): + ww_folders = f"{xdg_data_home()}/OPM/ww_configs" + path = "" + if not lang: + for l in listdir(ww_folders): + if isfile(f"{ww_folders}/{l}/{ww_id}.json"): + path = f"{ww_folders}/{l}/{ww_id}.json" + break + else: + lang = Configuration().get("lang") + + path = path or f"{ww_folders}/{lang}/{ww_id}.json" + + ww_config = ww_config or {} + if isfile(path): + with open(path) as f: + old_cfg = json.load(f) + if ww_config: + old_cfg.update(ww_config) + if plugin: + old_cfg["module"] = plugin + if name: + old_cfg["display_name"] = name + if lang: + old_cfg["stt_lang"] = lang + with open(path, "w") as f: + json.dump(ww_config, f, indent=4, ensure_ascii=False) + + def db_delete_ww_definition(self, ww_id): + for lang in listdir(f"{xdg_data_home()}/OPM/ww_configs"): + if f"_{lang}_" in ww_id: + path = f"{xdg_data_home()}/OPM/ww_configs/{lang}/{ww_id}.json" + if isfile(path): + remove(path) + return True + return False + + def db_post_ww_definition(self, name, lang, ww_config, plugin): + ww_id = get_ww_id(plugin_name=plugin, ww_name=name, ww_config=ww_config) + path = f"{xdg_data_home()}/OPM/ww_configs/{lang}/{ww_id}.json" + ww_config["stt_lang"] = lang # tag language in STT step + with open(path, "w") as f: + json.dump(ww_config, f, indent=4, ensure_ascii=False) + + def db_list_voice_definitions(self): + return [{ + "voice_id": voice_id, + "lang": voice_data["meta"].get("lang"), + "plugin": voice_data["module"], + "tts_config": voice_data, + "offline": voice_data["meta"].get("offline"), + "gender": voice_data["meta"].get("gender"), + } for voice_id, voice_data in get_voices(scan=True).items()] + + def db_get_voice_definition(self, voice_id): + voices = get_voices(scan=True) + if voice_id in voices: + voice_data = voices[voice_id] + return { + "voice_id": voice_id, + "lang": voice_data["meta"].get("lang"), + "plugin": voice_data["module"], + "tts_config": voice_data, + "offline": voice_data["meta"].get("offline"), + "gender": voice_data["meta"].get("gender"), + } + return {} + + def db_update_voice_definition(self, voice_id, name=None, lang=None, plugin=None, + tts_config=None, offline=None, gender=None): + VOICES_FOLDER = f"{xdg_data_home()}/OPM/voice_configs" + if not lang: + for l in listdir(VOICES_FOLDER): + if f"_{l}_" in voice_id: + lang = l + break + + path = f"{VOICES_FOLDER}/{lang}/{voice_id}.json" + voicedef = {"meta": {}} + if isfile(path): + with open(path) as f: + voicedef = json.load(f) + if tts_config: + voicedef.update(tts_config) + if name: + voicedef["meta"]["name"] = name + if name: + voicedef["meta"]["offline"] = offline + if gender: + voicedef["meta"]["gender"] = gender + if lang: + voicedef["lang"] = lang + voicedef["meta"]["lang"] = lang + if plugin: + voicedef["module"] = plugin + + with open(path, "w") as f: + json.dump(voicedef, f, indent=4, ensure_ascii=False) + + def db_delete_voice_definition(self, voice_id): + VOICES_FOLDER = f"{xdg_data_home()}/OPM/voice_configs" + for lang in listdir(VOICES_FOLDER): + if f"_{lang}_" in voice_id: + path = f"{VOICES_FOLDER}/{lang}/{voice_id}.json" + if isfile(path): + remove(path) + return True + return False + + def db_post_voice_definition(self, name, lang, plugin, + tts_config, offline, gender=None): + voice_id = get_voice_id(plugin_name=plugin, lang=lang, tts_config=tts_config) + path = f"{xdg_data_home()}/OPM/voice_configs/{lang}/{voice_id}.json" + with open(path, "w") as f: + tts_config["lang"] = lang + tts_config["meta"] = {"offline": offline, "gender": gender, + "name": name, "lang": lang} + json.dump(tts_config, f, indent=4, ensure_ascii=False) + class AbstractPartialBackend(OfflineBackend): """ helper class that internally delegates unimplemented methods to offline backend implementation diff --git a/ovos_backend_client/backends/personal.py b/ovos_backend_client/backends/personal.py index 63e93ed..d4c324d 100644 --- a/ovos_backend_client/backends/personal.py +++ b/ovos_backend_client/backends/personal.py @@ -1,15 +1,17 @@ +import base64 import json import os import time from io import BytesIO, StringIO +import requests from ovos_config.config import Configuration from ovos_utils.log import LOG from requests.exceptions import HTTPError from ovos_backend_client.backends.offline import AbstractPartialBackend, BackendType +from ovos_backend_client.database import SkillSettingsModel from ovos_backend_client.identity import IdentityManager, identity_lock -import requests class PersonalBackend(AbstractPartialBackend): @@ -19,7 +21,7 @@ def __init__(self, url="http://0.0.0.0:6712", version="v1", identity_file=None, def refresh_token(self): try: - self.identity.get() # Ensure loading so identity property doesn't cause deadlock + self.identity.get() # Ensure loading so identity property doesn't cause deadlock identity_lock.acquire(blocking=False) # NOTE: requests needs to be used instead of self.get due to self.get calling this data = requests.get(f"{self.backend_url}/{self.backend_version}/auth/token", headers=self.headers).json() @@ -332,6 +334,18 @@ def device_upload_wake_word(self, audio, params): } return self.post(url, files=ww_files) + # Skill settings api + def skill_settings_upload(self, skill_settings): + # serialize and upload + for s in skill_settings: + assert isinstance(s, SkillSettingsModel) + self.device_put_skill_settings_v1(s.serialize()) + + def skill_settings_download(self): + # download and deserialize + return [SkillSettingsModel.deserialize(s) + for s in self.device_get_skill_settings_v1()] + # Metrics API def metrics_upload(self, name, data): """ upload metrics""" @@ -344,6 +358,12 @@ def dataset_upload_wake_word(self, audio, params, upload_url=None): return self.device_upload_wake_word_v1(audio, params, upload_url) return self.device_upload_wake_word(audio, params) + def dataset_upload_stt_recording(self, audio, params, upload_url=None): + """ upload stt sample - url can be external to backend""" + if upload_url: + return super().dataset_upload_stt_recording(audio, params, upload_url) + raise NotImplementedError() # TODO - add to backend, currently needs external url + # OAuth API def oauth_get_token(self, dev_cred): """ @@ -426,3 +446,336 @@ def admin_set_device_info(self, uuid, info): """ return self.put(f"{self.backend_url}/{self.backend_version}/admin/{uuid}/device", json=info, headers={"Authorization": f"Bearer {self.credentials['admin']}"}) + + # Database api + def db_list_devices(self): + return self.get(url=f"{self.backend_url}/{self.backend_version}/admin/devices/list", + headers={"Authorization": f"Bearer {self.credentials['admin']}"}) + + def db_get_device(self, uuid): + return self.get(url=f"{self.backend_url}/{self.backend_version}/admin/devices/{uuid}", + headers={"Authorization": f"Bearer {self.credentials['admin']}"}) + + def db_update_device(self, uuid, name=None, + device_location=None, opt_in=False, + location=None, lang=None, date_format=None, + system_unit=None, time_format=None, email=None, + isolated_skills=False, ww_id=None, voice_id=None): + payload = {"name": name, + "device_location": device_location, + "opt_in": opt_in, + "location": location, + "lang": lang, + "date_format": date_format, + "system_unit": system_unit, + "time_format": time_format, + "email": email, + "isolated_skills": isolated_skills, + "ww_id": ww_id, + "voice_id": voice_id} + return self.put(url=f"{self.backend_url}/{self.backend_version}/admin/devices/{uuid}", + json={k: v for k, v in payload.items() if v is not None}, + headers={"Authorization": f"Bearer {self.credentials['admin']}"}) + + def db_delete_device(self, uuid): + return self.delete(url=f"{self.backend_url}/{self.backend_version}/admin/devices/{uuid}", + headers={"Authorization": f"Bearer {self.credentials['admin']}"}) + + def db_post_device(self, uuid, token, name=None, + device_location="somewhere", + opt_in=Configuration().get("opt_in", False), + location=Configuration().get("location"), + lang=Configuration().get("lang"), + date_format=Configuration().get("date_format", "DMY"), + system_unit=Configuration().get("system_unit", "metric"), + time_format=Configuration().get("time_format", "full"), + email=None, + isolated_skills=False, + ww_id=None, + voice_id=None): + payload = {"name": name, + "device_location": device_location, + "opt_in": opt_in, + "location": location, + "lang": lang, + "date_format": date_format, + "system_unit": system_unit, + "time_format": time_format, + "email": email, + "isolated_skills": isolated_skills, + "ww_id": ww_id, + "voice_id": voice_id, + "uuid": uuid} + return self.post(url=f"{self.backend_url}/{self.backend_version}/admin/devices", + json=payload, + headers={"Authorization": f"Bearer {self.credentials['admin']}"}) + + def db_list_shared_skill_settings(self): + return self.get(url=f"{self.backend_url}/{self.backend_version}/admin/skill_settings/list", + headers={"Authorization": f"Bearer {self.credentials['admin']}"}) + + def db_get_shared_skill_settings(self, skill_id): + + return self.get(url=f"{self.backend_url}/{self.backend_version}/admin/skill_settings/{skill_id}", + headers={"Authorization": f"Bearer {self.credentials['admin']}"}) + + def db_update_shared_skill_settings(self, skill_id, + display_name=None, + settings_json=None, + metadata_json=None): + payload = {"display_name": display_name, + "settings_json": settings_json, + "metadata_json": metadata_json} + return self.put(url=f"{self.backend_url}/{self.backend_version}/admin/skill_settings/{skill_id}", + json=payload, + headers={"Authorization": f"Bearer {self.credentials['admin']}"}) + + def db_delete_shared_skill_settings(self, skill_id): + return self.delete(url=f"{self.backend_url}/{self.backend_version}/admin/skill_settings/{skill_id}", + headers={"Authorization": f"Bearer {self.credentials['admin']}"}) + + def db_post_shared_skill_settings(self, skill_id, + display_name, + settings_json, + metadata_json): + payload = {"display_name": display_name, + "skill_id": skill_id, + "settings_json": settings_json, + "metadata_json": metadata_json} + return self.post(url=f"{self.backend_url}/{self.backend_version}/admin/skill_settings", + json=payload, + headers={"Authorization": f"Bearer {self.credentials['admin']}"}) + + def db_list_skill_settings(self, uuid): + return self.get(url=f"{self.backend_url}/{self.backend_version}/admin/{uuid}/skill_settings/list", + headers={"Authorization": f"Bearer {self.credentials['admin']}"}) + + def db_get_skill_settings(self, uuid, skill_id): + return self.get(url=f"{self.backend_url}/{self.backend_version}/admin/{uuid}/skill_settings/{skill_id}", + headers={"Authorization": f"Bearer {self.credentials['admin']}"}) + + def db_update_skill_settings(self, uuid, skill_id, + display_name=None, + settings_json=None, + metadata_json=None): + payload = {"display_name": display_name, + "settings_json": settings_json, + "metadata_json": metadata_json} + return self.put(url=f"{self.backend_url}/{self.backend_version}/admin/{uuid}/skill_settings/{skill_id}", + json=payload, + headers={"Authorization": f"Bearer {self.credentials['admin']}"}) + + def db_delete_skill_settings(self, uuid, skill_id): + return self.delete(url=f"{self.backend_url}/{self.backend_version}/admin/{uuid}/skill_settings/{skill_id}", + headers={"Authorization": f"Bearer {self.credentials['admin']}"}) + + def db_post_skill_settings(self, uuid, skill_id, + display_name, + settings_json, + metadata_json): + payload = {"display_name": display_name, + "skill_id": skill_id, + "settings_json": settings_json, + "metadata_json": metadata_json} + return self.post(url=f"{self.backend_url}/{self.backend_version}/admin/{uuid}/skill_settings", + json=payload, + headers={"Authorization": f"Bearer {self.credentials['admin']}"}) + + def db_list_oauth_apps(self): + return self.get(url=f"{self.backend_url}/{self.backend_version}/admin/oauth_apps/list", + headers={"Authorization": f"Bearer {self.credentials['admin']}"}) + + def db_get_oauth_app(self, token_id): + return self.get(url=f"{self.backend_url}/{self.backend_version}/admin/oauth_apps/{token_id}", + headers={"Authorization": f"Bearer {self.credentials['admin']}"}) + + def db_update_oauth_app(self, token_id, client_id=None, client_secret=None, + auth_endpoint=None, token_endpoint=None, refresh_endpoint=None, + callback_endpoint=None, scope=None, shell_integration=None): + payload = {"client_id": client_id, "client_secret": client_secret, + "auth_endpoint": auth_endpoint, + "token_endpoint": token_endpoint, + "refresh_endpoint": refresh_endpoint, + "callback_endpoint": callback_endpoint, + "scope": scope, "shell_integration": True} + return self.put(url=f"{self.backend_url}/{self.backend_version}/admin/oauth_apps/{token_id}", + json=payload, + headers={"Authorization": f"Bearer {self.credentials['admin']}"}) + + def db_delete_oauth_app(self, token_id): + return self.delete(url=f"{self.backend_url}/{self.backend_version}/admin/oauth_apps/{token_id}", + headers={"Authorization": f"Bearer {self.credentials['admin']}"}) + + def db_post_oauth_app(self, token_id, client_id, client_secret, + auth_endpoint, token_endpoint, refresh_endpoint, + callback_endpoint, scope, shell_integration=True): + payload = {"token_id": token_id, + "client_id": client_id, + "client_secret": client_secret, + "auth_endpoint": auth_endpoint, + "token_endpoint": token_endpoint, + "refresh_endpoint": refresh_endpoint, + "callback_endpoint": callback_endpoint, + "scope": scope, "shell_integration": True} + return self.post(url=f"{self.backend_url}/{self.backend_version}/admin/oauth_apps", + json=payload, + headers={"Authorization": f"Bearer {self.credentials['admin']}"}) + + def db_list_oauth_tokens(self): + return self.get(url=f"{self.backend_url}/{self.backend_version}/admin/oauth_toks/list", + headers={"Authorization": f"Bearer {self.credentials['admin']}"}) + + def db_get_oauth_token(self, token_id): + return self.get(url=f"{self.backend_url}/{self.backend_version}/admin/oauth_toks/{token_id}", + headers={"Authorization": f"Bearer {self.credentials['admin']}"}) + + def db_update_oauth_token(self, token_id, token_data): + payload = {"token_data": token_data} + return self.put(url=f"{self.backend_url}/{self.backend_version}/admin/oauth_toks/{token_id}", + headers={"Authorization": f"Bearer {self.credentials['admin']}"}, + json=payload) + + def db_delete_oauth_token(self, token_id): + return self.delete(url=f"{self.backend_url}/{self.backend_version}/admin/oauth_toks/{token_id}", + headers={"Authorization": f"Bearer {self.credentials['admin']}"}) + + def db_post_oauth_token(self, token_id, token_data): + payload = {"token_data": token_data, "token_id": token_id} + return self.put(url=f"{self.backend_url}/{self.backend_version}/admin/oauth_toks", + json=payload, + headers={"Authorization": f"Bearer {self.credentials['admin']}"}) + + def db_list_stt_recordings(self): + return self.get(url=f"{self.backend_url}/{self.backend_version}/admin/voice_recs/list", + headers={"Authorization": f"Bearer {self.credentials['admin']}"}) + + def db_get_stt_recording(self, rec_id): + return self.get(url=f"{self.backend_url}/{self.backend_version}/admin/voice_recs/{rec_id}", + headers={"Authorization": f"Bearer {self.credentials['admin']}"}) + + def db_update_stt_recording(self, rec_id, transcription=None, metadata=None): + payload = {"transcription": transcription, + "metadata": metadata} + return self.put(url=f"{self.backend_url}/{self.backend_version}/admin/voice_recs/{rec_id}", + json=payload, + headers={"Authorization": f"Bearer {self.credentials['admin']}"}) + + def db_delete_stt_recording(self, rec_id): + return self.delete(url=f"{self.backend_url}/{self.backend_version}/admin/voice_recs/{rec_id}", + headers={"Authorization": f"Bearer {self.credentials['admin']}"}) + + def db_post_stt_recording(self, byte_data, transcription, metadata=None): + payload = {"transcription": transcription, + "metadata": metadata, + "audio_b64": base64.encodebytes(byte_data)} + return self.put(url=f"{self.backend_url}/{self.backend_version}/admin/voice_recs", + json=payload, + headers={"Authorization": f"Bearer {self.credentials['admin']}"}) + + def db_list_ww_recordings(self): + return self.get(url=f"{self.backend_url}/{self.backend_version}/admin/ww_recs/list", + headers={"Authorization": f"Bearer {self.credentials['admin']}"}) + + def db_get_ww_recording(self, rec_id): + return self.get(url=f"{self.backend_url}/{self.backend_version}/admin/ww_recs/{rec_id}", + headers={"Authorization": f"Bearer {self.credentials['admin']}"}) + + def db_update_ww_recording(self, rec_id, transcription=None, metadata=None): + payload = {"transcription": transcription, + "metadata": metadata} + return self.put(url=f"{self.backend_url}/{self.backend_version}/admin/ww_recs/{rec_id}", + json=payload, + headers={"Authorization": f"Bearer {self.credentials['admin']}"}) + + def db_delete_ww_recording(self, rec_id): + return self.delete(url=f"{self.backend_url}/{self.backend_version}/admin/ww_recs/{rec_id}", + headers={"Authorization": f"Bearer {self.credentials['admin']}"}) + + def db_post_ww_recording(self, byte_data, transcription, metadata=None): + payload = {"transcription": transcription, + "metadata": metadata, + "audio_b64": base64.encodebytes(byte_data)} + return self.post(url=f"{self.backend_url}/{self.backend_version}/admin/ww_recs", + json=payload, + headers={"Authorization": f"Bearer {self.credentials['admin']}"}) + + def db_list_metrics(self): + return self.get(url=f"{self.backend_url}/{self.backend_version}/admin/metrics/list", + headers={"Authorization": f"Bearer {self.credentials['admin']}"}) + + def db_get_metric(self, metric_id): + return self.get(url=f"{self.backend_url}/{self.backend_version}/admin/metrics/{metric_id}", + headers={"Authorization": f"Bearer {self.credentials['admin']}"}) + + def db_update_metric(self, metric_id, metadata): + payload = {"metadata": metadata} + return self.put(url=f"{self.backend_url}/{self.backend_version}/admin/metrics/{metric_id}", + json=payload, + headers={"Authorization": f"Bearer {self.credentials['admin']}"}) + + def db_delete_metric(self, metric_id): + return self.delete(url=f"{self.backend_url}/{self.backend_version}/admin/metrics/{metric_id}", + headers={"Authorization": f"Bearer {self.credentials['admin']}"}) + + def db_post_metric(self, metric_type, metadata): + payload = {"metadata": metadata, "metric_type": metric_type} + return self.put(url=f"{self.backend_url}/{self.backend_version}/admin/metrics", + json=payload, + headers={"Authorization": f"Bearer {self.credentials['admin']}"}) + + def db_list_ww_definitions(self): + return self.get(url=f"{self.backend_url}/{self.backend_version}/admin/ww_defs/list", + headers={"Authorization": f"Bearer {self.credentials['admin']}"}) + + def db_get_ww_definition(self, ww_id): + return self.get(url=f"{self.backend_url}/{self.backend_version}/admin/ww_defs/{ww_id}", + headers={"Authorization": f"Bearer {self.credentials['admin']}"}) + + def db_update_ww_definition(self, ww_id, name=None, lang=None, ww_config=None, plugin=None): + payload = {"name": name, "lang": lang, + "plugin": plugin, "ww_config": ww_config} + return self.put(url=f"{self.backend_url}/{self.backend_version}/admin/ww_defs/{ww_id}", + json=payload, + headers={"Authorization": f"Bearer {self.credentials['admin']}"}) + + def db_delete_ww_definition(self, ww_id): + return self.delete(url=f"{self.backend_url}/{self.backend_version}/admin/ww_defs/{ww_id}", + headers={"Authorization": f"Bearer {self.credentials['admin']}"}) + + def db_post_ww_definition(self, name, lang, ww_config, plugin): + payload = {"name": name, "lang": lang, + "plugin": plugin, "ww_config": ww_config} + return self.post(url=f"{self.backend_url}/{self.backend_version}/admin/ww_defs", + json=payload, + headers={"Authorization": f"Bearer {self.credentials['admin']}"}) + + def db_list_voice_definitions(self): + + return self.get(url=f"{self.backend_url}/{self.backend_version}/admin/voice_defs/list", + headers={"Authorization": f"Bearer {self.credentials['admin']}"}) + + def db_get_voice_definition(self, voice_id): + return self.get(url=f"{self.backend_url}/{self.backend_version}/admin/voice_defs/{voice_id}", + headers={"Authorization": f"Bearer {self.credentials['admin']}"}) + + def db_update_voice_definition(self, voice_id, name=None, lang=None, plugin=None, + tts_config=None, offline=None, gender=None): + payload = {"name": name, "lang": lang, + "plugin": plugin, "tts_config": tts_config, + "offline": offline, "gender": gender} + return self.put(url=f"{self.backend_url}/{self.backend_version}/admin/voice_defs/{voice_id}", + json=payload, + headers={"Authorization": f"Bearer {self.credentials['admin']}"}) + + def db_delete_voice_definition(self, voice_id): + return self.delete(url=f"{self.backend_url}/{self.backend_version}/admin/voice_defs/{voice_id}", + headers={"Authorization": f"Bearer {self.credentials['admin']}"}) + + def db_post_voice_definition(self, name, lang, plugin, + tts_config, offline, gender=None): + payload = {"name": name, "lang": lang, + "plugin": plugin, "tts_config": tts_config, + "offline": offline, "gender": gender} + return self.post(url=f"{self.backend_url}/{self.backend_version}/admin/voice_defs", + json=payload, + headers={"Authorization": f"Bearer {self.credentials['admin']}"}) diff --git a/ovos_backend_client/database.py b/ovos_backend_client/database.py index fdf9bd2..a89693f 100644 --- a/ovos_backend_client/database.py +++ b/ovos_backend_client/database.py @@ -1,19 +1,390 @@ -import os -from os.path import join +import enum +import json +from copy import deepcopy from json_database import JsonStorageXDG, JsonDatabaseXDG from ovos_config.config import Configuration -from ovos_utils.configuration import get_xdg_data_save_path +from ovos_utils.configuration import get_xdg_config_save_path, get_xdg_base + +from ovos_backend_client.identity import IdentityManager + + +class AudioTag(str, enum.Enum): + UNTAGGED = "untagged" + WAKE_WORD = "wake_word" + SPEECH = "speech" + NOISE = "noise" + SILENCE = "silence" + + +class SpeakerTag(str, enum.Enum): + UNTAGGED = "untagged" + MALE = "male" + FEMALE = "female" + CHILDREN = "children" + + +class DatabaseModel: + def __init__(self, **kwargs): + for k, v in kwargs.items(): + self.__setattr__(k, v) + + def serialize(self): + return self.__dict__ + + @classmethod + def deserialize(cls, kwargs): + return cls(**kwargs) + + +class MetricModel(DatabaseModel): + def __init__(self, metric_id, metric_type, meta=None, uuid="AnonDevice"): + if isinstance(meta, str): + meta = json.loads(meta) + super().__init__(metric_id=metric_id, metric_type=metric_type, meta=meta, uuid=uuid) + + +class WakeWordRecordingModel(DatabaseModel): + def __init__(self, wakeword_id, transcription, path, meta=None, + uuid="AnonDevice", tag=AudioTag.UNTAGGED, speaker_type=SpeakerTag.UNTAGGED): + if isinstance(meta, str): + meta = json.loads(meta) + super().__init__(wakeword_id=wakeword_id, transcription=transcription, + path=path, meta=meta or [], uuid=uuid, + tag=tag, speaker_type=speaker_type) + + +class UtteranceRecordingModel(DatabaseModel): + def __init__(self, utterance_id, transcription, path, uuid="AnonDevice"): + super().__init__(utterance_id=utterance_id, transcription=transcription, path=path, uuid=uuid) + + +class SkillSettingsModel(DatabaseModel): + """ represents skill settings for a individual skill""" + + def __init__(self, skill_id, skill_settings=None, + meta=None, display_name=None, remote_id=None): + remote_id = remote_id or skill_id + if not remote_id.startswith("@"): + remote_id = f"@|{remote_id}" + if isinstance(meta, str): + meta = json.loads(meta) + super().__init__(skill_id=skill_id, skill_settings=skill_settings or {}, + meta=meta or {}, display_name=display_name or skill_id, remote_id=remote_id) + + def store(self): + with open(f"{get_xdg_config_save_path()}/skills/{self.skill_id}/settings.json" "w") as f: + json.dump(self.skill_settings, f, indent=4, ensure_ascii=False) + # TODO - autogen meta if needed (?) + with open(f"{get_xdg_config_save_path()}/skills/{self.skill_id}/settingsmeta.json" "w") as f: + json.dump(self.meta, f, indent=4, ensure_ascii=False) + + def serialize(self): + # settings meta with updated placeholder values from settings + # old style selene db stored skill settings this way + meta = deepcopy(self.meta) + for idx, section in enumerate(meta.get('sections', [])): + for idx2, field in enumerate(section["fields"]): + if "value" not in field: + continue + if field["name"] in self.skill_settings: + meta['sections'][idx]["fields"][idx2]["value"] = self.skill_settings[field["name"]] + return {'skillMetadata': meta, + "skill_gid": self.remote_id, + "display_name": self.display_name} + + @staticmethod + def deserialize(data): + if isinstance(data, str): + data = json.loads(data) + + skill_json = {} + skill_meta = data.get("skillMetadata") or {} + for s in skill_meta.get("sections", []): + for f in s.get("fields", []): + if "name" in f and "value" in f: + val = f["value"] + if isinstance(val, str): + t = f.get("type", "") + if t == "checkbox": + if val.lower() == "true" or val == "1": + val = True + else: + val = False + elif t == "number": + if val == "False": + val = 0 + elif val == "True": + val = 1 + else: + val = float(val) + elif val.lower() in ["none", "null", "nan"]: + val = None + elif val == "[]": + val = [] + elif val == "{}": + val = {} + skill_json[f["name"]] = val + + remote_id = data.get("skill_gid") or data.get("identifier") + # this is a mess, possible keys seen by logging data + # - @|XXX + # - @{uuid}|XXX + # - XXX + + # where XXX has been observed to be + # - {skill_id} <- ovos-core + # - {msm_name} <- mycroft-core + # - {mycroft_marketplace_name} <- all default skills + # - {MycroftSkill.name} <- sometimes sent to msm (very uncommon) + # - {skill_id.split(".")[0]} <- fallback msm name + # - XXX|{branch} <- append by msm (?) + # - {whatever we feel like uploading} <- SeleneCloud utils + fields = remote_id.split("|") + skill_id = fields[0] + if len(fields) > 1 and fields[0].startswith("@"): + skill_id = fields[1] + + display_name = data.get("display_name") or \ + skill_id.split(".")[0].replace("-", " ").replace("_", " ").title() + + return SkillSettingsModel(skill_id, skill_json, skill_meta, display_name, + remote_id=remote_id) + + +class DeviceModel(DatabaseModel): + """ global device settings + represent some fields from mycroft.conf but also contain some extra fields + """ + + def __init__(self): + identity = IdentityManager.get() + + default_ww = Configuration().get("listener", {}).get("wake_word", "hey_mycroft") + default_tts = Configuration().get("tts", {}).get("module", "ovos-tts-plugin-mimic3-server") + mail_cfg = Configuration().get("email", {}) + + uuid = identity["uuid"] + super().__init__(uuid=uuid, token=identity["access"], + isolated_skills=True, + name=f"Device-{uuid}", + device_location="somewhere", # indoor location + email=mail_cfg.get("recipient") or \ + mail_cfg.get("smtp", {}).get("username"), + date_format=Configuration().get("date_format") or "DMY", + system_unit=Configuration().get("system_unit") or "metric", + time_format=Configuration().get("time_format") or "full", + opt_in=Configuration().get("opt_in") or False, + lang=Configuration().get("lang") or "en-us", + location=Configuration["location"], + default_tts=default_tts, + default_tts_cfg=Configuration().get("tts", {}).get(default_tts, {}), + default_ww=default_ww.replace(" ", "_"), + default_ww_cfg=Configuration().get("hotwords", {}).get(default_ww, {}) + ) + + @property + def selene_device(self): + return { + "description": self.device_location, + "uuid": self.uuid, + "name": self.name, + + # not tracked / meaningless + # just for api compliance with selene + 'coreVersion': "unknown", + 'platform': 'unknown', + 'enclosureVersion': "", + "user": {"uuid": self.uuid} # users not tracked + } + + @property + def selene_settings(self): + # this endpoint corresponds to a mycroft.conf + # location is usually grabbed in a separate endpoint + # in here we return it in case downstream is + # aware of this and wants to save 1 http call + + # NOTE - selene returns the full listener config + # this SHOULD NOT be done, since backend has no clue of hardware downstream + # we return only wake word config + if self.default_ww and self.default_ww_cfg: + ww_cfg = {self.default_ww: self.default_ww_cfg} + listener = {"wakeWord": self.default_ww.replace(" ", "_")} + else: + ww_cfg = {} + listener = {} + + tts_config = dict(self.default_tts_cfg) + if "module" in tts_config: + tts = tts_config.pop("module") + tts_settings = {"module": tts, tts: tts_config} + else: + tts_settings = {} + return { + "dateFormat": self.date_format, + "optIn": self.opt_in, + "systemUnit": self.system_unit, + "timeFormat": self.time_format, + "uuid": self.uuid, + "lang": self.lang, + "location": self.location, + "listenerSetting": listener, + "hotwordsSetting": ww_cfg, # not present in selene, parsed correctly by core + 'ttsSettings': tts_settings + } + + +class JsonMetricDatabase(JsonDatabaseXDG): + def __init__(self): + super().__init__("ovos_metrics", xdg_folder=get_xdg_base()) + + def add_metric(self, metric_type=None, meta=None, uuid="AnonDevice"): + metric_id = self.total_metrics() + 1 + metric = MetricModel(metric_id, metric_type, meta, uuid) + self.add_item(metric) + return metric + + def total_metrics(self): + return len(self) + + def __enter__(self): + """ Context handler """ + return self + + def __exit__(self, _type, value, traceback): + """ Commits changes and Closes the session """ + try: + self.commit() + except Exception as e: + print(e) + + +class JsonWakeWordDatabase(JsonDatabaseXDG): + def __init__(self): + super().__init__("ovos_wakewords", xdg_folder=get_xdg_base()) + + def add_wakeword(self, transcription, path, meta=None, + uuid="AnonDevice", tag=AudioTag.UNTAGGED, + speaker_type=SpeakerTag.UNTAGGED): + wakeword_id = self.total_wakewords() + 1 + wakeword = WakeWordRecordingModel(wakeword_id, + transcription, + path, meta, uuid, + tag, speaker_type) + self.add_item(wakeword) + return wakeword + + def get_wakeword(self, rec_id): + ww = self.get(rec_id) + if ww: + return WakeWordRecordingModel.deserialize(ww) + return None + + def update_wakeword(self, rec_id, transcription=None, path=None, + meta=None, tag=AudioTag.UNTAGGED, + speaker_type=SpeakerTag.UNTAGGED): + ww = self.get_wakeword(rec_id) + if not ww: + return None + if transcription: + ww.transcription = transcription + if path: + ww.path = path + if tag: + ww.tag = tag + if speaker_type: + ww.speaker_type = speaker_type + self[rec_id] = ww.serialize() + return ww + + def delete_wakeword(self, rec_id): + if self.get(rec_id): + self.pop(rec_id) + return True + return False + + def total_wakewords(self): + return len(self) + + def __enter__(self): + """ Context handler """ + return self + + def __exit__(self, _type, value, traceback): + """ Commits changes and Closes the session """ + try: + self.commit() + except Exception as e: + print(e) + + +class JsonUtteranceDatabase(JsonDatabaseXDG): + def __init__(self): + super().__init__("ovos_utterances", xdg_folder=get_xdg_base()) + + def add_utterance(self, transcription, path, uuid="AnonDevice"): + utterance_id = self.total_utterances() + 1 + utterance = UtteranceRecordingModel(utterance_id, transcription, + path, uuid) + self.add_item(utterance) + + def get_utterance(self, rec_id): + ww = self.get(rec_id) + if ww: + return UtteranceRecordingModel.deserialize(ww) + return None + + def update_utterance(self, rec_id, transcription): + ww = self.get_utterance(rec_id) + if not ww: + return None + ww.transcription = transcription + self[rec_id] = ww.serialize() + return ww + + def delete_utterance(self, rec_id): + if self.get(rec_id): + self.pop(rec_id) + return True + return False + + def total_utterances(self): + return len(self) + + def __enter__(self): + """ Context handler """ + return self + + def __exit__(self, _type, value, traceback): + """ Commits changes and Closes the session """ + try: + self.commit() + except Exception as e: + print(e) class OAuthTokenDatabase(JsonStorageXDG): """ This helper class creates ovos-config-assistant/ovos-backend-manager compatible json databases This allows users to use oauth even when not using a backend""" + def __init__(self): - super().__init__("ovos_oauth") + super().__init__("ovos_oauth", xdg_folder=get_xdg_base()) + + def add_token(self, token_id, token_data): + self[token_id] = token_data - def add_token(self, oauth_service, token_data): - self[oauth_service] = token_data + def update_token(self, token_id, token_data): + self.add_token(token_id, token_data) + + def get_token(self, token_id): + return self.get(token_id) + + def delete_token(self, token_id): + if token_id in self: + self.pop(token_id) + return True + return False def total_tokens(self): return len(self) @@ -22,8 +393,9 @@ def total_tokens(self): class OAuthApplicationDatabase(JsonStorageXDG): """ This helper class creates ovos-config-assistant/ovos-backend-manager compatible json databases This allows users to use oauth even when not using a backend""" + def __init__(self): - super().__init__("ovos_oauth_apps") + super().__init__("ovos_oauth_apps", xdg_folder=get_xdg_base()) def add_application(self, oauth_service, client_id, client_secret, @@ -39,72 +411,23 @@ def add_application(self, oauth_service, "scope": scope, "shell_integration": shell_integration} - def total_apps(self): - return len(self) + def get_application(self, oauth_service): + return self.get(oauth_service) + def update_application(self, oauth_service, + client_id, client_secret, + auth_endpoint, token_endpoint, refresh_endpoint, + callback_endpoint, scope, shell_integration=True): + self.update_application(oauth_service, + client_id, client_secret, + auth_endpoint, token_endpoint, refresh_endpoint, + callback_endpoint, scope, shell_integration) -class BackendDatabase: - """ This helper class creates ovos-config-assistant/ovos-backend-manager compatible json databases - This allows users to visualize metrics, tag wake words and configure devices - even when not using a backend""" - - def __init__(self, uuid): - self.uuid = uuid - - def update_device_db(self, data): - with JsonStorageXDG("ovos_preferences", subfolder="OpenVoiceOS") as db: - db.update(data) - cfg = Configuration() - tts = cfg.get("tts", {}).get("module") - ww = cfg.get("listener", {}).get("wake_word", "hey_mycroft") - - with JsonStorageXDG("ovos_devices") as db: - skips = ["state", "coreVersion", "platform", "platform_build", "enclosureVersion"] - default = { - "uuid": self.uuid, - "isolated_skills": True, - "name": "LocalDevice", - "device_location": "127.0.0.1", - "email": "", - "date_format": cfg.get("date_format") or "DMY", - "time_format": cfg.get("time_format") or "full", - "system_unit": cfg.get("system_unit") or "metric", - "opt_in": cfg.get("opt_in", False), - "lang": cfg.get("lang", "en-us"), - "location": cfg.get("location", {}), - "default_tts": tts, - "default_tts_cfg": cfg.get("tts", {}).get(tts, {}), - "default_ww": ww, - "default_ww_cfg": cfg.get("hotwords", {}).get(ww, {}) - } - data = {k: v if k not in data else data[k] - for k, v in default.items() if k not in skips} - db[self.uuid] = data - - def update_metrics_db(self, name, data): - # shared with personal backend for UI compat - with JsonDatabaseXDG("ovos_metrics") as db: - db.add_item({ - "metric_id": len(db) + 1, - "uuid": self.uuid, - "metric_type": name, - "meta": data - }) - - def update_ww_db(self, params): - listener_config = Configuration().get("listener", {}) - save_path = listener_config.get('save_path', f"{get_xdg_data_save_path()}/listener") - saved_wake_words_dir = join(save_path, 'wake_words') - filename = join(saved_wake_words_dir, - '_'.join(str(params[k]) for k in sorted(params)) + - '.wav') - if os.path.isfile(filename): - with JsonDatabaseXDG("ovos_wakewords") as db: - db.add_item({ - "wakeword_id": len(db) + 1, - "uuid": self.uuid, - "meta": params, - "path": filename, - "transcription": params["name"] - }) - return filename + def delete_application(self, oauth_service): + if oauth_service in self: + self.pop(oauth_service) + return True + return False + + def total_apps(self): + return len(self) diff --git a/ovos_backend_client/settings.py b/ovos_backend_client/settings.py index 79475e1..4d63669 100644 --- a/ovos_backend_client/settings.py +++ b/ovos_backend_client/settings.py @@ -9,8 +9,8 @@ from ovos_config import Configuration from ovos_utils import camel_case_split from ovos_utils.configuration import get_xdg_config_save_path, get_xdg_data_save_path, get_xdg_data_dirs - -from ovos_backend_client.api import DeviceApi +from ovos_backend_client.database import SkillSettingsModel +import ovos_backend_client.api as _api def get_display_name(skill_name: str): @@ -20,6 +20,29 @@ def get_display_name(skill_name: str): return camel_case_split(skill_name).title().strip() +def get_local_settings(): + settings_path = f"{get_xdg_config_save_path()}/skills" + if not isdir(settings_path): + return [] + settings = {} + meta = {} + all_settings = [] + for skill_id in os.listdir(settings_path): + s = f"{settings_path}/{skill_id}/settings.json" + if isfile(s): + with open(s) as f: + settings = json.load(f) + s = f"{settings_path}/{skill_id}/settingsmeta.json" + if isfile(s): + meta = json.load(f) + + display_name = skill_id.split(".")[-1].replace("_", "").replace("-", "").title() + s = SkillSettingsModel(skill_id=skill_id, meta=meta, + skill_settings=settings, display_name=display_name) + all_settings.append(s) + return all_settings + + class RemoteSkillSettings: """ WARNING: selene backend does not use proper skill_id, if you have skills with same name but different author settings will overwrite each @@ -37,7 +60,7 @@ class RemoteSkillSettings: """ def __init__(self, skill_id, settings=None, meta=None, url=None, version="v1", remote_id=None): - self.api = DeviceApi(url, version) + self.api = _api.DeviceApi(url, version) self.skill_id = skill_id self.identifier = remote_id or \ self.selene_gid if not skill_id.startswith("@") else skill_id @@ -158,7 +181,7 @@ def match_settings(x, against): if s: self.meta = s.meta - self.settings = s.settings + self.settings = s.skill_settings # update actual identifier from selene self.identifier = s.identifier @@ -272,7 +295,7 @@ def __init__(self, api=None): if "skills" not in self: self["skills"] = [] self.store() - self.api = api or DeviceApi() + self.api = api or _api.DeviceApi() @staticmethod def _get_default_skills_directory(conf=None): @@ -411,15 +434,15 @@ def scan_skills(self, skill_dirs=None): print(s.api.get_skill_settings_v1()) s.download() print(s) - s.settings["not"] = "yes" # ignored, not in meta - s.settings["show_time"] = True + s.skill_settings["not"] = "yes" # ignored, not in meta + s.skill_settings["show_time"] = True s.upload() s.download() print(s) - s.settings["not"] = "yes" + s.skill_settings["not"] = "yes" s.generate_meta() # now in meta - s.settings["not"] = "no" - s.settings["show_time"] = False + s.skill_settings["not"] = "no" + s.skill_settings["show_time"] = False s.upload() s.download() print(s) diff --git a/ovos_backend_client/version.py b/ovos_backend_client/version.py index 4c24a00..62313c8 100644 --- a/ovos_backend_client/version.py +++ b/ovos_backend_client/version.py @@ -1,7 +1,7 @@ # The following lines are replaced during the release process. # START_VERSION_BLOCK VERSION_MAJOR = 0 -VERSION_MINOR = 0 -VERSION_BUILD = 7 -VERSION_ALPHA = 9 +VERSION_MINOR = 1 +VERSION_BUILD = 0 +VERSION_ALPHA = 1 # END_VERSION_BLOCK diff --git a/test/license_tests.py b/test/license_tests.py deleted file mode 100644 index 5920eca..0000000 --- a/test/license_tests.py +++ /dev/null @@ -1,53 +0,0 @@ -import unittest -from pprint import pprint - -from lichecker import LicenseChecker - -# these packages dont define license in setup.py -# manually verified and injected -license_overrides = { - "kthread": "MIT", - 'yt-dlp': "Unlicense", - 'pyxdg': 'GPL-2.0', - 'ptyprocess': 'ISC license', - 'psutil': 'BSD3' -} -# explicitly allow these packages that would fail otherwise -whitelist = ["python-dateutil"] - -# validation flags -allow_nonfree = False -allow_viral = False -allow_unknown = False -allow_unlicense = True -allow_ambiguous = False - -pkg_name = "ovos_backend_client" - - -class TestLicensing(unittest.TestCase): - @classmethod - def setUpClass(self): - licheck = LicenseChecker(pkg_name, - license_overrides=license_overrides, - whitelisted_packages=whitelist, - allow_ambiguous=allow_ambiguous, - allow_unlicense=allow_unlicense, - allow_unknown=allow_unknown, - allow_viral=allow_viral, - allow_nonfree=allow_nonfree) - print("Package", pkg_name) - print("Version", licheck.version) - print("License", licheck.license) - print("Transient Requirements (dependencies of dependencies)") - pprint(licheck.transient_dependencies) - self.licheck = licheck - - def test_license_compliance(self): - print("Package Versions") - pprint(self.licheck.versions) - - print("Dependency Licenses") - pprint(self.licheck.licenses) - - self.licheck.validate() diff --git a/test/unittests/test_skill_settings.py b/test/unittests/test_skill_settings.py new file mode 100644 index 0000000..cb43019 --- /dev/null +++ b/test/unittests/test_skill_settings.py @@ -0,0 +1,127 @@ +import unittest + +from ovos_backend_client.database import SkillSettingsModel + + +class TestSkillSettings(unittest.TestCase): + + def test_deserialize(self): + meta = {"sections": [ + { + "fields": [ + {"name": "test", "value": True} + ] + } + ]} + + data = { + "skillMetadata": meta, + "skill_gid": "@|test_skill" + } + s = SkillSettingsModel.deserialize(data) + self.assertEqual(s.display_name, "Test Skill") + self.assertEqual(s.skill_id, "test_skill") + self.assertEqual(s.remote_id, "@|test_skill") + self.assertEqual(s.skill_settings, {"test": True}) + self.assertEqual(s.meta, meta) + + old_data = { + "skillMetadata": meta, + "display_name": "Test Skill", + "identifier": "@|test_skill" + } + s = SkillSettingsModel.deserialize(old_data) + self.assertEqual(s.display_name, "Test Skill") + self.assertEqual(s.skill_id, "test_skill") + self.assertEqual(s.remote_id, "@|test_skill") + self.assertEqual(s.skill_settings, {"test": True}) + self.assertEqual(s.meta, meta) + + def test_serialize(self): + meta = {"sections": [ + { + "fields": [ + {"name": "test", "value": False} + ] + } + ]} + settings = {"test": True} + updated_meta = {"sections": [ + { + "fields": [ + {"name": "test", "value": True} + ] + } + ]} + + s = SkillSettingsModel("test_skill", settings, meta, "Test Skill") + self.assertEqual(s.skill_settings, settings) + self.assertEqual(s.meta, meta) + self.assertEqual(s.display_name, "Test Skill") + self.assertEqual(s.skill_id, "test_skill") + + s2 = s.serialize() + self.assertEqual(s.meta, meta) + self.assertEqual(s2["display_name"], "Test Skill") + self.assertEqual(s2["skill_gid"], "@|test_skill") + self.assertEqual(s2['skillMetadata'], updated_meta) + + def test_skill_id(self): + uuid = "jbgblnkl-dgsg-sgsdg-sgags" + meta = {"sections": [ + { + "fields": [ + {"name": "test", "value": True} + ] + } + ]} + + data = { + "skillMetadata": meta, + "skill_gid": "@|test_skill" + } + s = SkillSettingsModel.deserialize(data) + self.assertEqual(s.skill_id, "test_skill") + + data = { + "skillMetadata": meta, + "skill_gid": f"@{uuid}|test_skill" + } + s = SkillSettingsModel.deserialize(data) + self.assertEqual(s.skill_id, "test_skill") + self.assertEqual(s.display_name, "Test Skill") + + data = { + "skillMetadata": meta, + "skill_gid": "@|test_skill|20.02" + } + s = SkillSettingsModel.deserialize(data) + self.assertEqual(s.skill_id, "test_skill") + self.assertEqual(s.display_name, "Test Skill") + + data = { + "skillMetadata": meta, + "skill_gid": f"@{uuid}|test_skill|20.02" + } + s = SkillSettingsModel.deserialize(data) + self.assertEqual(s.skill_id, "test_skill") + self.assertEqual(s.display_name, "Test Skill") + + data = { + "skillMetadata": meta, + "skill_gid": "test_skill" + } + s = SkillSettingsModel.deserialize(data) + self.assertEqual(s.skill_id, "test_skill") + self.assertEqual(s.remote_id, "@|test_skill") + self.assertEqual(s.display_name, "Test Skill") + + data = { + "skillMetadata": meta, + "skill_gid": "test_skill.author" + } + s = SkillSettingsModel.deserialize(data) + self.assertEqual(s.skill_id, "test_skill.author") + self.assertEqual(s.remote_id, "@|test_skill.author") + self.assertEqual(s.display_name, "Test Skill") +