rename again to ovos-backend-client + improve personal backend UI compat

cleanup docstrs and rename stuff

add database helper + compat with device db personal backend UI

wakeword metadata compat - UI tagger

comparison table in readme

.github/workflows/build_tests.yml

@@ -7,7 +7,7 @@ on:
     branches:
       - dev
     paths-ignore:
-      - 'ovos_backend_api/version.py'
+      - 'ovos_backend_client/version.py'
       - 'test/**'
       - 'examples/**'
       - '.github/**'

.github/workflows/notify_matrix.yml

@@ -20,4 +20,4 @@ jobs:
           token: ${{ secrets.MATRIX_TOKEN }}
           channel: '!WjxEKjjINpyBRPFgxl:krbel.duckdns.org'
           message: |
-            new ovos_backend_api PR merged! https://github.com/OpenVoiceOS/ovos-backend-api/pull/${{ github.event.number }}
+            new ovos_backend_client PR merged! https://github.com/OpenVoiceOS/ovos-backend-client/pull/${{ github.event.number }}

.github/workflows/publish_alpha.yml

@@ -6,7 +6,7 @@ on:
     branches:
       - dev
     paths-ignore:
-      - 'ovos_backend_api/version.py'
+      - 'ovos_backend_client/version.py'
       - 'test/**'
       - 'examples/**'
       - '.github/**'

.github/workflows/unit_tests.yml

@@ -6,7 +6,7 @@ on:
     branches:
       - dev
     paths-ignore:
-      - 'ovos_backend_api/version.py'
+      - 'ovos_backend_client/version.py'
       - 'requirements/**'
       - 'examples/**'
       - '.github/**'
@@ -20,7 +20,7 @@ on:
     branches:
       - master
     paths-ignore:
-      - 'ovos_backend_api/version.py'
+      - 'ovos_backend_client/version.py'
       - 'requirements/**'
       - 'examples/**'
       - '.github/**'
@@ -58,7 +58,7 @@ jobs:
           pip install pytest pytest-timeout pytest-cov
       - name: Run unittests
         run: |
-          pytest --cov=ovos_backend_api --cov-report xml test/unittests
+          pytest --cov=ovos_backend_client --cov-report xml test/unittests
           # NOTE: additional pytest invocations should also add the --cov-append flag
           #       or they will overwrite previous invocations' coverage reports
           #       (for an example, see OVOS Skill Manager's workflow)

README.md

@@ -8,14 +8,32 @@ Python client library for interaction with several supported backends under a si
 - Neon MQ Service - https://api.neon.ai
 - Offline - support for setting your own api keys and query services directly
 
+## Backend Overview
+
+| API       | Offline | Personal | Selene | OVOS   | Neon        |
+|-----------|---------|----------|--------|--------|-------------|
+| Admin     | yes [2] | yes      | no     | no [1] | no [1]      |
+| Device    | yes [3] | yes      | yes    | no [1] | no [1]      |
+| Wolfram   | yes [5] | yes      | yes    | yes    | yes         |
+| Geolocate | yes     | yes      | yes    | yes    | yes         |
+| STT       | no      | yes      | yes    | yes    | no          |
+| Weather   | yes [5] | yes      | yes    | yes    | partial [4] |
+
+    [1] will be cast to Offline type for compatibility
+    [2] will update user level mycroft.conf
+    [3] shared metrics db with personal backend for UI compat
+    [4] hourly/daily forecast endpoints not available
+    [5] needs api key key
+
+
 ## STT
 
 a companion stt plugin is available - [ovos-stt-plugin-selene](https://github.com/OpenVoiceOS/ovos-stt-plugin-selene)
 
 ## Geolocation
 
 ```python
-from ovos_backend_api.api import GeolocationApi
+from ovos_backend_client.api import GeolocationApi
 
 geo = GeolocationApi()
 data = geo.get_geolocation("Lisbon Portugal")
@@ -29,7 +47,7 @@ data = geo.get_geolocation("Lisbon Portugal")
 ## OpenWeatherMap Proxy
 
 ```python
-from ovos_backend_api.api import OpenWeatherMapApi
+from ovos_backend_client.api import OpenWeatherMapApi
 
 owm = OpenWeatherMapApi()
 data = owm.get_weather()
@@ -39,7 +57,7 @@ data = owm.get_weather()
 ## Wolfram Alpha proxy
 
 ```python
-from ovos_backend_api.api import WolframAlphaApi
+from ovos_backend_client.api import WolframAlphaApi
 
 wolf = WolframAlphaApi()
 answer = wolf.spoken("what is the speed of light")
@@ -54,7 +72,7 @@ data = wolf.full_results("2+2")
 To interact with skill settings on selene
 
 ```python
-from ovos_backend_api.settings import RemoteSkillSettings
+from ovos_backend_client.settings import RemoteSkillSettings
 
 # in ovos-core skill_id is deterministic and safe
 s = RemoteSkillSettings("skill.author")
@@ -81,7 +99,7 @@ s.upload()
 by hijacking skill settings we allows storing arbitrary data in selene and use it across devices and skills
 
 ```python
-from ovos_backend_api.cloud import SeleneCloud
+from ovos_backend_client.cloud import SeleneCloud
 
 cloud = SeleneCloud()
 cloud.add_entry("test", {"secret": "NOT ENCRYPTED MAN"})
@@ -91,7 +109,7 @@ data = cloud.get_entry("test")
 an encrypted version is also supported if you dont trust selene!
 
 ```python
-from ovos_backend_api.cloud import SecretSeleneCloud
+from ovos_backend_client.cloud import SecretSeleneCloud
 
 k = "D8fmXEP5VqzVw2HE"  # you need this to read back the data
 cloud = SecretSeleneCloud(k)
@@ -107,7 +125,7 @@ since local backend does not provide a web ui a [admin api](https://github.com/O
 can be used to manage your devices
 
 ```python
-from ovos_backend_api.api import AdminApi
+from ovos_backend_client.api import AdminApi
 
 admin = AdminApi("secret_admin_key")
 uuid = "..."  # check identity2.json in the device you want to manage

ovos_backend_client/__init__.py

@@ -0,0 +1,5 @@
+from ovos_backend_client.api import DeviceApi, WolframAlphaApi, OpenWeatherMapApi, STTApi, GeolocationApi
+from ovos_backend_client.cloud import SeleneCloud, SecretSeleneCloud
+from ovos_backend_client.settings import RemoteSkillSettings
+from ovos_backend_client.pairing import is_paired, has_been_paired, check_remote_pairing, PairingManager
+from ovos_backend_client.config import RemoteConfigManager

ovos_backend_api/api.py → ovos_backend_client/api.py

@@ -4,15 +4,17 @@
 from enum import Enum
 from io import BytesIO, StringIO
 from uuid import uuid4
-from json_database import JsonStorageXDG, JsonDatabaseXDG
+
 import requests
+from json_database import JsonStorageXDG
 from ovos_config.config import update_mycroft_config, Configuration
 from ovos_utils import timed_lru_cache
 from ovos_utils.log import LOG
 from ovos_utils.ovos_service_api import OVOSApiService
 from requests.exceptions import HTTPError
 
-from ovos_backend_api.identity import IdentityManager, identity_lock
+from ovos_backend_client.database import BackendDatabase
+from ovos_backend_client.identity import IdentityManager, identity_lock
 
 try:
     from neon_api_proxy.client import NeonAPIProxyClient
@@ -401,7 +403,6 @@ def get_skill_settings_v1(self):
 
     def put_skill_settings_v1(self, data):
         """ old style deprecated bidirectional skill settings api, still available! """
-
         if self.backend_type == BackendType.OFFLINE:
             # do nothing, skills manage their own settings lifecycle
             return {}
@@ -437,8 +438,9 @@ def activate(self, state, token,
                 "enclosureVersion": enclosure_version}
         if self.backend_type == BackendType.OFFLINE:
             identity = AdminApi("", backend_type=BackendType.OFFLINE).pair(state)
-            with JsonStorageXDG("ovos_device_info.json", subfolder="OpenVoiceOS") as db:
-                db.update(data)
+            data["uuid"] = data.pop("state")
+            data["token"] = self.access_token
+            BackendDatabase(self.uuid).update_device_db(data)
         else:
             return self.post(self.url + "/activate", json=data).json()
 
@@ -454,21 +456,14 @@ def update_version(self,
                 "enclosureVersion": enclosure_version}
 
         if self.backend_type == BackendType.OFFLINE:
-            with JsonStorageXDG("ovos_device_info.json", subfolder="OpenVoiceOS") as db:
-                db.update(data)
+            data["token"] = self.access_token
+            BackendDatabase(self.uuid).update_device_db(data)
         else:
             return self.patch(self.url + "/" + self.uuid, json=data)
 
     def report_metric(self, name, data):
         if self.backend_type == BackendType.OFFLINE:
-            # shared with personal backend for UI compat
-            with JsonDatabaseXDG("ovos_metrics") as db:
-                db.add_item({
-                    "metric_id": len(db) + 1,
-                    "uuid": self.uuid,
-                    "metric_type": name,
-                    "meta": data
-                })
+            BackendDatabase(self.uuid).update_metrics_db(name, data)
             return {}
         elif self.backend_type == BackendType.NEON_MQ:
             raise NotImplemented("TODO - implement neon metrics report")
@@ -609,7 +604,9 @@ def upload_wake_word_v1(self, audio, params):
             'metadata': StringIO(json.dumps(params))
         }
         if self.backend_type == BackendType.OFFLINE:
-            return {}  # Do nothing, core has its own internal save flag
+            if Configuration().get("listener", {}).get('record_wake_words'):
+                BackendDatabase(self.uuid).update_ww_db(params)  # update metadata db for ww tagging UI
+            return {}
         return self.post(self.precise_url_v1, files=ww_files)
 
     def upload_wake_word(self, audio, params):
@@ -626,7 +623,9 @@ def upload_wake_word(self, audio, params):
             'metadata': StringIO(json.dumps(request_data))
         }
         if self.backend_type == BackendType.OFFLINE:
-            return {}  # Do nothing, core has its own internal save flag
+            if Configuration().get("listener", {}).get('record_wake_words'):
+                BackendDatabase(self.uuid).update_ww_db(params)  # update metadata db for ww tagging UI
+            return {}
         return self.post(url, files=ww_files)
 
 

ovos_backend_api/cloud.py → ovos_backend_client/cloud.py

@@ -3,7 +3,7 @@
 
 from ovos_utils.security import encrypt, decrypt
 
-from ovos_backend_api.api import DeviceApi
+from ovos_backend_client.api import DeviceApi
 
 
 class SeleneCloud:

ovos_backend_api/config.py → ovos_backend_client/config.py

@@ -2,8 +2,8 @@
 
 from ovos_utils import camel_case_split
 from ovos_utils.log import LOG
-from ovos_backend_api.pairing import is_paired
-from ovos_backend_api.api import DeviceApi
+from ovos_backend_client.pairing import is_paired
+from ovos_backend_client.api import DeviceApi
 from pprint import pformat
 
 

ovos_backend_client/database.py

@@ -0,0 +1,72 @@
+import os
+from os.path import join
+
+from json_database import JsonStorageXDG, JsonDatabaseXDG
+from ovos_config.config import Configuration
+from ovos_utils.configuration import get_xdg_data_save_path
+
+
+class BackendDatabase:
+    """ This helper class creates ovos-backend-ui compatible json databases
+    This allows users to visualize metrics, tag wake words and configure devices
+    even when not using a backend"""
+    def __init__(self, uuid):
+        self.uuid = uuid
+
+    def update_device_db(self, data):
+        with JsonStorageXDG("ovos_preferences", subfolder="OpenVoiceOS") as db:
+            db.update(data)
+        cfg = Configuration()
+        tts = cfg.get("tts", {}).get("module")
+        ww = cfg.get("listener", {}).get("wake_word", "hey_mycroft")
+
+        with JsonStorageXDG("ovos_devices") as db:
+            skips = ["state", "coreVersion", "platform", "platform_build", "enclosureVersion"]
+            default = {
+                "uuid": self.uuid,
+                "isolated_skills": True,
+                "name": "LocalDevice",
+                "device_location": "127.0.0.1",
+                "email": "",
+                "date_format": cfg.get("date_format") or "DMY",
+                "time_format": cfg.get("time_format") or "full",
+                "system_unit": cfg.get("system_unit") or "metric",
+                "opt_in": cfg.get("opt_in", False),
+                "lang": cfg.get("lang", "en-us"),
+                "location": cfg.get("location", {}),
+                "default_tts": tts,
+                "default_tts_cfg": cfg.get("tts", {}).get(tts, {}),
+                "default_ww": ww,
+                "default_ww_cfg": cfg.get("hotwords", {}).get(ww, {})
+            }
+            data = {k: v if k not in data else data[k]
+                    for k, v in default.items() if k not in skips}
+            db[self.uuid] = data
+
+    def update_metrics_db(self, name, data):
+        # shared with personal backend for UI compat
+        with JsonDatabaseXDG("ovos_metrics") as db:
+            db.add_item({
+                "metric_id": len(db) + 1,
+                "uuid": self.uuid,
+                "metric_type": name,
+                "meta": data
+            })
+
+    def update_ww_db(self, params):
+        listener_config = Configuration().get("listener", {})
+        save_path = listener_config.get('save_path', f"{get_xdg_data_save_path()}/listener")
+        saved_wake_words_dir = join(save_path, 'wake_words')
+        filename = join(saved_wake_words_dir,
+                        '_'.join(str(params[k]) for k in sorted(params)) +
+                        '.wav')
+        if os.path.isfile(filename):
+            with JsonDatabaseXDG("ovos_wakewords") as db:
+                db.add_item({
+                    "wakeword_id": len(db) + 1,
+                    "uuid": self.uuid,
+                    "meta": params,
+                    "path": filename,
+                    "transcription": params["name"]
+                })
+        return filename

ovos_backend_api/pairing.py → ovos_backend_client/pairing.py

@@ -2,9 +2,9 @@
 from ovos_utils.network_utils import is_connected
 from ovos_utils.enclosure.api import EnclosureAPI
 from ovos_utils.messagebus import Message, FakeBus
-from ovos_backend_api.exceptions import BackendDown, InternetDown, HTTPError
-from ovos_backend_api.identity import IdentityManager
-from ovos_backend_api.api import DeviceApi
+from ovos_backend_client.exceptions import BackendDown, InternetDown, HTTPError
+from ovos_backend_client.identity import IdentityManager
+from ovos_backend_client.api import DeviceApi
 import time
 from threading import Timer, Lock
 from uuid import uuid4

ovos_backend_api/settings.py → ovos_backend_client/settings.py

@@ -10,7 +10,7 @@
 from ovos_utils import camel_case_split
 from ovos_utils.configuration import get_xdg_config_save_path, get_xdg_data_save_path, get_xdg_data_dirs
 from ovos_config import Configuration
-from ovos_backend_api.api import DeviceApi
+from ovos_backend_client.api import DeviceApi
 
 
 def get_display_name(skill_name: str):

scripts/bump_alpha.py

@@ -2,7 +2,7 @@
 from os.path import join, dirname
 
 
-version_file = join(dirname(dirname(__file__)), "ovos_backend_api", "version.py")
+version_file = join(dirname(dirname(__file__)), "ovos_backend_client", "version.py")
 version_var_name = "VERSION_ALPHA"
 
 with open(version_file, "r", encoding="utf-8") as v:

scripts/bump_build.py

@@ -2,7 +2,7 @@
 from os.path import join, dirname
 
 
-version_file = join(dirname(dirname(__file__)), "ovos_backend_api", "version.py")
+version_file = join(dirname(dirname(__file__)), "ovos_backend_client", "version.py")
 version_var_name = "VERSION_BUILD"
 alpha_var_name = "VERSION_ALPHA"
 

scripts/bump_major.py

@@ -2,7 +2,7 @@
 from os.path import join, dirname
 
 
-version_file = join(dirname(dirname(__file__)), "ovos_backend_api", "version.py")
+version_file = join(dirname(dirname(__file__)), "ovos_backend_client", "version.py")
 version_var_name = "VERSION_MAJOR"
 minor_var_name = "VERSION_MINOR"
 build_var_name = "VERSION_BUILD"

scripts/bump_minor.py

@@ -2,7 +2,7 @@
 from os.path import join, dirname
 
 
-version_file = join(dirname(dirname(__file__)), "ovos_backend_api", "version.py")
+version_file = join(dirname(dirname(__file__)), "ovos_backend_client", "version.py")
 version_var_name = "VERSION_MINOR"
 build_var_name = "VERSION_BUILD"
 alpha_var_name = "VERSION_ALPHA"

scripts/remove_alpha.py

@@ -2,7 +2,7 @@
 from os.path import join, dirname
 
 
-version_file = join(dirname(dirname(__file__)), "ovos_backend_api", "version.py")
+version_file = join(dirname(dirname(__file__)), "ovos_backend_client", "version.py")
 
 alpha_var_name = "VERSION_ALPHA"