From 4c8f57f3b9346d474d38d5f15073bfead75000fb Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Tue, 2 Nov 2021 14:23:57 +0100 Subject: [PATCH 001/129] This is RC2 --- core/commons/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/commons/constants.py b/core/commons/constants.py index a6bc517c4..38078afac 100644 --- a/core/commons/constants.py +++ b/core/commons/constants.py @@ -17,7 +17,7 @@ # # Last modified: 2021.04.13 at 12:56:46 CEST -VERSION = '1.0.0-rc1' +VERSION = '1.0.0-rc2' # System ALL = 'all' From d1f8de2cc19720e83621ca7f661314b642c530ac Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Tue, 2 Nov 2021 14:47:29 +0100 Subject: [PATCH 002/129] Marking webui 1.0.0-rc2 as default branch --- .gitmodules | 1 + core/webui/public | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index b9fbd9f50..e9eae2d28 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,4 @@ [submodule "core/webui/public"] path = core/webui/public url = https://github.com/project-alice-assistant/webui.git + branch = 1.0.0-rc2 \ No newline at end of file diff --git a/core/webui/public b/core/webui/public index ff3d60f11..ca4c57e2e 160000 --- a/core/webui/public +++ b/core/webui/public @@ -1 +1 @@ -Subproject commit ff3d60f1138e9081cc6af00d1c81fd55bbec6de8 +Subproject commit ca4c57e2e088fbc9dee55c6a3f16180f37a15bc6 From d5c882b8aca14a9df458f626ba6706adaeec7432 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Tue, 2 Nov 2021 14:59:57 +0100 Subject: [PATCH 003/129] For readability --- core/commons/constants.py | 279 +++++++++++++++++++------------------- 1 file changed, 140 insertions(+), 139 deletions(-) diff --git a/core/commons/constants.py b/core/commons/constants.py index 38078afac..9c76fac86 100644 --- a/core/commons/constants.py +++ b/core/commons/constants.py @@ -17,152 +17,153 @@ # # Last modified: 2021.04.13 at 12:56:46 CEST -VERSION = '1.0.0-rc2' +VERSION = '1.0.0-rc2' # System -ALL = 'all' -DATABASE_FILE = 'system/database/data.db' -DEFAULT = 'default' -DUMMY = 'dummy' -EVERYWHERE = 'everywhere' -GITHUB_API_URL = 'https://api.github.com/repos/project-alice-assistant' -GITHUB_RAW_URL = 'https://raw.githubusercontent.com/project-alice-assistant' -GITHUB_REPOSITORY_ID = 193512918 -GITHUB_URL = 'https://github.com/project-alice-assistant' -JSON_EXT = '.json' -PNG_EXT = '.png' -RANDOM = 'random' -SKILL_INSTALL_TICKET_PATH = 'system/skillInstallTickets' -SKILL_REDIRECT_URL = 'https://skills.projectalice.ch' -SKILLS_SAMPLES_STORE_ASSETS = 'https://skills.projectalice.io/assets/store/skills.samples' -SKILLS_STORE_ASSETS = 'https://skills.projectalice.io/assets/store/skills.json' -UNKNOWN = 'unknown' -UNKNOWN_MANAGER = 'unknownManager' -UNKNOWN_USER = 'unknownUser' -UNKNOWN_WORD = 'unknownword' +ALL = 'all' +DATABASE_FILE = 'system/database/data.db' +DEFAULT = 'default' +DUMMY = 'dummy' +EVERYWHERE = 'everywhere' +GITHUB_API_URL = 'https://api.github.com/repos/project-alice-assistant' +GITHUB_RAW_URL = 'https://raw.githubusercontent.com/project-alice-assistant' +GITHUB_REPOSITORY_ID = 193512918 +GITHUB_URL = 'https://github.com/project-alice-assistant' +JSON_EXT = '.json' +PNG_EXT = '.png' +RANDOM = 'random' +SKILL_INSTALL_TICKET_PATH = 'system/skillInstallTickets' +SKILL_REDIRECT_URL = 'https://skills.projectalice.ch' +SKILLS_SAMPLES_STORE_ASSETS = 'https://skills.projectalice.io/assets/store/skills.samples' +SKILLS_STORE_ASSETS = 'https://skills.projectalice.io/assets/store/skills.json' +UNKNOWN = 'unknown' +UNKNOWN_MANAGER = 'unknownManager' +UNKNOWN_USER = 'unknownUser' +UNKNOWN_WORD = 'unknownword' # Hermes -TOPIC_ASR_START_LISTENING = 'hermes/asr/startListening' -TOPIC_ASR_STOP_LISTENING = 'hermes/asr/stopListening' -TOPIC_ASR_TOGGLE_OFF = 'hermes/asr/toggleOff' -TOPIC_ASR_TOGGLE_ON = 'hermes/asr/toggleOn' -TOPIC_AUDIO_FRAME = 'hermes/audioServer/{}/audioFrame' -TOPIC_CONTINUE_SESSION = 'hermes/dialogueManager/continueSession' -TOPIC_DIALOGUE_MANAGER_CONFIGURE = 'hermes/dialogueManager/configure' -TOPIC_END_SESSION = 'hermes/dialogueManager/endSession' -TOPIC_HOTWORD_DETECTED = 'hermes/hotword/default/detected' -TOPIC_HOTWORD_TOGGLE_OFF = 'hermes/hotword/toggleOff' -TOPIC_HOTWORD_TOGGLE_ON = 'hermes/hotword/toggleOn' -TOPIC_INTENT_NOT_RECOGNIZED = 'hermes/dialogueManager/intentNotRecognized' -TOPIC_INTENT_PARSED = 'hermes/nlu/intentParsed' -TOPIC_NLU_ERROR = 'hermes/error/nlu' -TOPIC_NLU_INTENT_NOT_RECOGNIZED = 'hermes/nlu/intentNotRecognized' -TOPIC_NLU_QUERY = 'hermes/nlu/query' -TOPIC_PARTIAL_TEXT_CAPTURED = 'hermes/asr/partialTextCaptured' -TOPIC_PLAY_BYTES = 'hermes/audioServer/{}/playBytes/#' # hermes/audioServer//playBytes/ -TOPIC_PLAY_BYTES_FINISHED = 'hermes/audioServer/{}/playFinished' -TOPIC_SESSION_ENDED = 'hermes/dialogueManager/sessionEnded' -TOPIC_SESSION_QUEUED = 'hermes/dialogueManager/sessionQueued' -TOPIC_SESSION_STARTED = 'hermes/dialogueManager/sessionStarted' -TOPIC_START_SESSION = 'hermes/dialogueManager/startSession' -TOPIC_SYSTEM_UPDATE = 'hermes/leds/systemUpdate' -TOPIC_TEXT_CAPTURED = 'hermes/asr/textCaptured' -TOPIC_TOGGLE_FEEDBACK = 'hermes/feedback/sound/toggle{}' -TOPIC_TOGGLE_FEEDBACK_OFF = 'hermes/feedback/sound/toggleOff' -TOPIC_TOGGLE_FEEDBACK_ON = 'hermes/feedback/sound/toggleOn' -TOPIC_TTS_FINISHED = 'hermes/tts/sayFinished' -TOPIC_TTS_SAY = 'hermes/tts/say' -TOPIC_VAD_DOWN = 'hermes/voiceActivity/{}/vadDown' -TOPIC_VAD_UP = 'hermes/voiceActivity/{}/vadUp' -TOPIC_WAKEWORD_DETECTED = 'hermes/hotword/{}/detected' +TOPIC_ASR_START_LISTENING = 'hermes/asr/startListening' +TOPIC_ASR_STOP_LISTENING = 'hermes/asr/stopListening' +TOPIC_ASR_TOGGLE_OFF = 'hermes/asr/toggleOff' +TOPIC_ASR_TOGGLE_ON = 'hermes/asr/toggleOn' +TOPIC_AUDIO_FRAME = 'hermes/audioServer/{}/audioFrame' +TOPIC_CONTINUE_SESSION = 'hermes/dialogueManager/continueSession' +TOPIC_DIALOGUE_MANAGER_CONFIGURE = 'hermes/dialogueManager/configure' +TOPIC_END_SESSION = 'hermes/dialogueManager/endSession' +TOPIC_HOTWORD_DETECTED = 'hermes/hotword/default/detected' +TOPIC_HOTWORD_TOGGLE_OFF = 'hermes/hotword/toggleOff' +TOPIC_HOTWORD_TOGGLE_ON = 'hermes/hotword/toggleOn' +TOPIC_INTENT_NOT_RECOGNIZED = 'hermes/dialogueManager/intentNotRecognized' +TOPIC_INTENT_PARSED = 'hermes/nlu/intentParsed' +TOPIC_NLU_ERROR = 'hermes/error/nlu' +TOPIC_NLU_INTENT_NOT_RECOGNIZED = 'hermes/nlu/intentNotRecognized' +TOPIC_NLU_QUERY = 'hermes/nlu/query' +TOPIC_PARTIAL_TEXT_CAPTURED = 'hermes/asr/partialTextCaptured' +TOPIC_PLAY_BYTES = 'hermes/audioServer/{}/playBytes/#' # hermes/audioServer//playBytes/ +TOPIC_PLAY_BYTES_FINISHED = 'hermes/audioServer/{}/playFinished' +TOPIC_SESSION_ENDED = 'hermes/dialogueManager/sessionEnded' +TOPIC_SESSION_QUEUED = 'hermes/dialogueManager/sessionQueued' +TOPIC_SESSION_STARTED = 'hermes/dialogueManager/sessionStarted' +TOPIC_START_SESSION = 'hermes/dialogueManager/startSession' +TOPIC_SYSTEM_UPDATE = 'hermes/leds/systemUpdate' +TOPIC_TEXT_CAPTURED = 'hermes/asr/textCaptured' +TOPIC_TOGGLE_FEEDBACK = 'hermes/feedback/sound/toggle{}' +TOPIC_TOGGLE_FEEDBACK_OFF = 'hermes/feedback/sound/toggleOff' +TOPIC_TOGGLE_FEEDBACK_ON = 'hermes/feedback/sound/toggleOn' +TOPIC_TTS_FINISHED = 'hermes/tts/sayFinished' +TOPIC_TTS_SAY = 'hermes/tts/say' +TOPIC_VAD_DOWN = 'hermes/voiceActivity/{}/vadDown' +TOPIC_VAD_UP = 'hermes/voiceActivity/{}/vadUp' +TOPIC_WAKEWORD_DETECTED = 'hermes/hotword/{}/detected' # Alice -TOPIC_CORE_DISCONNECTION = 'projectalice/devices/coreDisconnection' -TOPIC_CORE_HEARTBEAT = 'projectalice/devices/coreHeartbeat' -TOPIC_CORE_RECONNECTION = 'projectalice/devices/coreReconnection' -TOPIC_DEVICE_ACCEPTED = 'projectalice/devices/connectionAccepted' -TOPIC_DEVICE_DELETED = 'projectalice/devices/deleted' -TOPIC_DEVICE_HEARTBEAT = 'projectalice/devices/heartbeat' -TOPIC_DEVICE_REFUSED = 'projectalice/devices/connectionRefused' -TOPIC_DEVICE_STATUS = 'projectalice/devices/status' -TOPIC_DEVICE_UPDATED = 'projectalice/devices/updated' -TOPIC_DND = 'projectalice/devices/stopListen' -TOPIC_NEW_HOTWORD = 'projectalice/devices/alice/newHotword' -TOPIC_NLU_TRAINING_STATUS = 'projectalice/nlu/trainingStatus' -TOPIC_RESOURCE_USAGE = 'projectalice/devices/resourceUsage' -TOPIC_SKILL_DELETED = 'projectalice/skills/deleted' -TOPIC_SKILL_INSTALLED = 'projectalice/skills/installed' -TOPIC_SKILL_INSTRUCTIONS = 'projectalice/skills/instructions' +TOPIC_CORE_DISCONNECTION = 'projectalice/devices/coreDisconnection' +TOPIC_CORE_HEARTBEAT = 'projectalice/devices/coreHeartbeat' +TOPIC_CORE_RECONNECTION = 'projectalice/devices/coreReconnection' +TOPIC_DEVICE_ACCEPTED = 'projectalice/devices/connectionAccepted' +TOPIC_DEVICE_DELETED = 'projectalice/devices/deleted' +TOPIC_DEVICE_HEARTBEAT = 'projectalice/devices/heartbeat' +TOPIC_DEVICE_REFUSED = 'projectalice/devices/connectionRefused' +TOPIC_DEVICE_STATUS = 'projectalice/devices/status' +TOPIC_DEVICE_UPDATED = 'projectalice/devices/updated' +TOPIC_DND = 'projectalice/devices/stopListen' +TOPIC_NEW_HOTWORD = 'projectalice/devices/alice/newHotword' +TOPIC_NLU_TRAINING_STATUS = 'projectalice/nlu/trainingStatus' +TOPIC_RESOURCE_USAGE = 'projectalice/devices/resourceUsage' +TOPIC_SKILL_DELETED = 'projectalice/skills/deleted' +TOPIC_SKILL_INSTALLED = 'projectalice/skills/installed' +TOPIC_SKILL_INSTRUCTIONS = 'projectalice/skills/instructions' TOPIC_SKILL_UPDATE_CORE_CONFIG_WARNING = 'projectalice/skills/coreConfigUpdateWarning' -TOPIC_SKILL_UPDATED = 'projectalice/skills/updated' -TOPIC_SKILL_UPDATING = 'projectalice/skills/updating' -TOPIC_STOP_DND = 'projectalice/devices/startListen' -TOPIC_SYSLOG = 'projectalice/logging/syslog' -TOPIC_TOGGLE_DND = 'projectalice/devices/toggleListen' -TOPIC_UI_NOTIFICATION = 'projectalice/notifications/ui/notification' +TOPIC_SKILL_UPDATED = 'projectalice/skills/updated' +TOPIC_SKILL_UPDATING = 'projectalice/skills/updating' +TOPIC_STOP_DND = 'projectalice/devices/startListen' +TOPIC_SYSLOG = 'projectalice/logging/syslog' +TOPIC_TOGGLE_DND = 'projectalice/devices/toggleListen' +TOPIC_UI_NOTIFICATION = 'projectalice/notifications/ui/notification' # Events -EVENT_ASR_TOGGLE_OFF = 'asrToggleOff' -EVENT_ASR_TOGGLE_ON = 'asrToggleOn' -EVENT_AUDIO_FRAME = 'audioFrame' -EVENT_BOOTED = 'booted' -EVENT_BROADCASTING_FOR_NEW_DEVICE = 'broadcastingForNewDeviceStart' -EVENT_CAPTURED = 'captured' -EVENT_CONFIGURE_INTENT = 'configureIntent' -EVENT_CONTEXT_SENSITIVE_DELETE = 'contextSensitiveDelete' -EVENT_CONTEXT_SENSITIVE_EDIT = 'contextSensitiveEdit' -EVENT_CONTINUE_SESSION = 'continueSession' -EVENT_DEVICE_ADDED = 'deviceAdded' -EVENT_DEVICE_CONNECTING = 'deviceConnecting' -EVENT_DEVICE_DISCONNECTED = 'deviceDisconnected' -EVENT_DEVICE_DISCONNECTING = 'deviceDisconnecting' -EVENT_DEVICE_DISCOVERED = 'deviceDiscovered' -EVENT_DEVICE_HEARTBEAT = 'deviceHeartbeat' -EVENT_DEVICE_REMOVED = 'deviceRemoved' -EVENT_END_SESSION = 'endSession' -EVENT_FIVE_MINUTE = 'fiveMinute' -EVENT_FULL_HOUR = 'fullHour' -EVENT_FULL_MINUTE = 'fullMinute' -EVENT_HOTWORD = 'hotword' -EVENT_HOTWORD_TOGGLE_OFF = 'hotwordToggleOff' -EVENT_HOTWORD_TOGGLE_ON = 'hotwordToggleOn' -EVENT_INTENT = 'intent' -EVENT_INTENT_NOT_RECOGNIZED = 'intentNotRecognized' -EVENT_INTENT_PARSED = 'intentParsed' -EVENT_INTERNET_CONNECTED = 'internetConnected' -EVENT_INTERNET_LOST = 'internetLost' -EVENT_MESSAGE = 'message' -EVENT_NLU_ERROR = 'nluError' -EVENT_NLU_INTENT_NOT_RECOGNIZED = 'nluIntentNotRecognized' -EVENT_NLU_QUERY = 'nluQuery' -EVENT_NLU_TRAINED = 'nluTrained' -EVENT_PARTIAL_TEXT_CAPTURED = 'partialTextCaptured' -EVENT_PLAY_BYTES = 'playBytes' -EVENT_PLAY_BYTES_FINISHED = 'playBytesFinished' -EVENT_QUARTER_HOUR = 'quarterHour' -EVENT_SAY = 'say' -EVENT_SAY_FINISHED = 'sayFinished' -EVENT_SESSION_ENDED = 'sessionEnded' -EVENT_SESSION_ERROR = 'sessionError' -EVENT_SESSION_QUEUED = 'sessionQueued' -EVENT_SESSION_STARTED = 'sessionStarted' -EVENT_SESSION_TIMEOUT = 'sessionTimeout' -EVENT_SKILL_DELETED = 'skillDeleted' -EVENT_SKILL_INSTALL_FAILED = 'skillInstallFailed' -EVENT_SKILL_INSTALLED = 'skillInstalled' -EVENT_SKILL_STARTED = 'skillStarted' -EVENT_SKILL_STOPPED = 'skillStopped' -EVENT_SKILL_UPDATED = 'skillUpdated' -EVENT_SLEEP = 'sleep' -EVENT_START_LISTENING = 'startListening' -EVENT_START_SESSION = 'startSession' +EVENT_ASR_TOGGLE_OFF = 'asrToggleOff' +EVENT_ASR_TOGGLE_ON = 'asrToggleOn' +EVENT_AUDIO_FRAME = 'audioFrame' +EVENT_BOOTED = 'booted' +EVENT_BROADCASTING_FOR_NEW_DEVICE = 'broadcastingForNewDeviceStart' +EVENT_CAPTURED = 'captured' +EVENT_CONFIGURE_INTENT = 'configureIntent' +EVENT_CONTEXT_SENSITIVE_DELETE = 'contextSensitiveDelete' +EVENT_CONTEXT_SENSITIVE_EDIT = 'contextSensitiveEdit' +EVENT_CONTINUE_SESSION = 'continueSession' +EVENT_DEVICE_ADDED = 'deviceAdded' +EVENT_DEVICE_CONNECTING = 'deviceConnecting' +EVENT_DEVICE_DISCONNECTED = 'deviceDisconnected' +EVENT_DEVICE_DISCONNECTING = 'deviceDisconnecting' +EVENT_DEVICE_DISCOVERED = 'deviceDiscovered' +EVENT_DEVICE_HEARTBEAT = 'deviceHeartbeat' +EVENT_DEVICE_REMOVED = 'deviceRemoved' +EVENT_END_SESSION = 'endSession' +EVENT_FIVE_MINUTE = 'fiveMinute' +EVENT_FULL_HOUR = 'fullHour' +EVENT_FULL_MINUTE = 'fullMinute' +EVENT_HOTWORD = 'hotword' +EVENT_HOTWORD_TOGGLE_OFF = 'hotwordToggleOff' +EVENT_HOTWORD_TOGGLE_ON = 'hotwordToggleOn' +EVENT_INTENT = 'intent' +EVENT_INTENT_NOT_RECOGNIZED = 'intentNotRecognized' +EVENT_INTENT_PARSED = 'intentParsed' +EVENT_INTERNET_CONNECTED = 'internetConnected' +EVENT_INTERNET_LOST = 'internetLost' +EVENT_MESSAGE = 'message' +EVENT_NLU_ERROR = 'nluError' +EVENT_NLU_INTENT_NOT_RECOGNIZED = 'nluIntentNotRecognized' +EVENT_NLU_QUERY = 'nluQuery' +EVENT_NLU_TRAINED = 'nluTrained' +EVENT_PARTIAL_TEXT_CAPTURED = 'partialTextCaptured' +EVENT_PLAY_BYTES = 'playBytes' +EVENT_PLAY_BYTES_FINISHED = 'playBytesFinished' +EVENT_QUARTER_HOUR = 'quarterHour' +EVENT_SAY = 'say' +EVENT_SAY_FINISHED = 'sayFinished' +EVENT_SESSION_ENDED = 'sessionEnded' +EVENT_SESSION_ERROR = 'sessionError' +EVENT_SESSION_QUEUED = 'sessionQueued' +EVENT_SESSION_STARTED = 'sessionStarted' +EVENT_SESSION_TIMEOUT = 'sessionTimeout' +EVENT_SKILL_DELETED = 'skillDeleted' +EVENT_SKILL_INSTALL_FAILED = 'skillInstallFailed' +EVENT_SKILL_INSTALLED = 'skillInstalled' +EVENT_SKILL_STARTED = 'skillStarted' +EVENT_SKILL_STOPPED = 'skillStopped' +EVENT_SKILL_UPDATED = 'skillUpdated' +EVENT_SLEEP = 'sleep' +EVENT_START_LISTENING = 'startListening' +EVENT_START_SESSION = 'startSession' EVENT_STOP_BROADCASTING_FOR_NEW_DEVICE = 'broadcastingForNewDeviceStop' -EVENT_STOP_LISTENING = 'stopListening' -EVENT_SYSLOG = 'sysLog' -EVENT_TOGGLE_FEEDBACK_OFF = 'toggleFeedbackOff' -EVENT_TOGGLE_FEEDBACK_ON = 'toggleFeedbackOn' -EVENT_USER_CANCEL = 'userCancel' -EVENT_VAD_DOWN = 'vadDown' -EVENT_VAD_UP = 'vadUp' -EVENT_WAKEUP = 'wakeup' -EVENT_WAKEWORD = 'wakeword' +EVENT_STOP_LISTENING = 'stopListening' +EVENT_SYSLOG = 'sysLog' +EVENT_TOGGLE_FEEDBACK_OFF = 'toggleFeedbackOff' +EVENT_TOGGLE_FEEDBACK_ON = 'toggleFeedbackOn' +EVENT_USER_CANCEL = 'userCancel' +EVENT_VAD_DOWN = 'vadDown' +EVENT_VAD_UP = 'vadUp' +EVENT_WAKEUP = 'wakeup' +EVENT_WAKEWORD = 'wakeword' + From 2635df5d9b92ad410b4f87485398302c3b26472c Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Thu, 4 Nov 2021 09:24:27 +0100 Subject: [PATCH 004/129] Updating google cloud speech to use latest 2021 tech instead of 2019.... --- core/asr/model/GoogleAsr.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/core/asr/model/GoogleAsr.py b/core/asr/model/GoogleAsr.py index 9e3f8d356..286af3240 100644 --- a/core/asr/model/GoogleAsr.py +++ b/core/asr/model/GoogleAsr.py @@ -31,8 +31,8 @@ try: - # noinspection PyUnresolvedReferences,PyPackageRequirements - from google.cloud.speech import SpeechClient, enums, types + # noinspection PyPackageRequirements + from google.cloud import speech except: pass # Auto installed @@ -43,7 +43,7 @@ class GoogleAsr(Asr): DEPENDENCIES = { 'system': [], 'pip' : { - 'google-cloud-speech==1.3.1' + 'google-cloud-speech==2.11.1' } } @@ -54,8 +54,8 @@ def __init__(self): self._capableOfArbitraryCapture = True self._isOnlineASR = True - self._client: Optional[SpeechClient] = None - self._streamingConfig: Optional[types.StreamingRecognitionConfig] = None + self._client: Optional[speech.SpeechClient] = None + self._streamingConfig: Optional[speech.RecognitionConfig] = None if self._credentialsFile.exists() and not self.ConfigManager.getAliceConfigByName('googleASRCredentials'): self.ConfigManager.updateAliceConfiguration(key='googleASRCredentials', value=self._credentialsFile.read_text(), doPreAndPostProcessing=False) @@ -70,15 +70,17 @@ def onStart(self): super().onStart() os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = str(self._credentialsFile) - self._client = SpeechClient() + self._client = speech.SpeechClient() # noinspection PyUnresolvedReferences - config = types.RecognitionConfig( - encoding=enums.RecognitionConfig.AudioEncoding.LINEAR16, + config = speech.RecognitionConfig( + encoding=speech.RecognitionConfig.AudioEncoding.LINEAR16, sample_rate_hertz=self.AudioServer.SAMPLERATE, - language_code=self.LanguageManager.getLanguageAndCountryCode() + language_code=self.LanguageManager.getLanguageAndCountryCode(), + max_alternatives=1, + model='command_and_search' ) - self._streamingConfig = types.StreamingRecognitionConfig(config=config, interim_results=True) + self._streamingConfig = speech.StreamingRecognitionConfig(config=config, interim_results=True) def decodeStream(self, session: DialogSession) -> Optional[ASRResult]: @@ -93,7 +95,7 @@ def decodeStream(self, session: DialogSession) -> Optional[ASRResult]: audioStream = stream.audioStream() # noinspection PyUnresolvedReferences try: - requests = (types.StreamingRecognizeRequest(audio_content=content) for content in audioStream) + requests = (speech.StreamingRecognizeRequest(audio_content=content) for content in audioStream) responses = self._client.streaming_recognize(self._streamingConfig, requests) result = self._checkResponses(session, responses) except Exception as e: From 6b5f111a91e3875d08cc437ec339ae68d3701a2a Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Thu, 4 Nov 2021 16:54:44 +0100 Subject: [PATCH 005/129] Upgrade venv --- core/Initializer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/Initializer.py b/core/Initializer.py index 54f5c0bac..380d9ca20 100644 --- a/core/Initializer.py +++ b/core/Initializer.py @@ -331,7 +331,7 @@ def checkVenv(self) -> bool: def updateVenv(self): subprocess.run([self.PIP, 'uninstall', '-y', '-r', str(Path(self.rootDir, 'pipuninstalls.txt'))]) subprocess.run([self.PIP, 'install', 'wheel']) - subprocess.run([self.PIP, 'install', '-r', str(Path(self.rootDir, 'requirements.txt'))]) + subprocess.run([self.PIP, 'install', '-r', str(Path(self.rootDir, 'requirements.txt')), '--upgrade', '--no-cache-dir']) @staticmethod From 477d3c35286ceab004469a50e20e56b7031316f7 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Thu, 4 Nov 2021 18:54:18 +0100 Subject: [PATCH 006/129] I want to remove Githubcloner and have the git functions included in AliceSkill --- core/asr/model/GoogleAsr.py | 12 +++--- core/base/SkillManager.py | 58 ++++++++++++++-------------- core/base/model/AliceSkill.py | 6 ++- core/base/model/GithubCloner.py | 68 +++++++++++++++++---------------- 4 files changed, 77 insertions(+), 67 deletions(-) diff --git a/core/asr/model/GoogleAsr.py b/core/asr/model/GoogleAsr.py index 286af3240..a9e249c69 100644 --- a/core/asr/model/GoogleAsr.py +++ b/core/asr/model/GoogleAsr.py @@ -32,7 +32,7 @@ try: # noinspection PyPackageRequirements - from google.cloud import speech + from google.cloud.speech import SpeechClient, RecognitionConfig, StreamingRecognitionConfig, StreamingRecognizeRequest except: pass # Auto installed @@ -54,8 +54,8 @@ def __init__(self): self._capableOfArbitraryCapture = True self._isOnlineASR = True - self._client: Optional[speech.SpeechClient] = None - self._streamingConfig: Optional[speech.RecognitionConfig] = None + self._client: Optional[SpeechClient] = None + self._streamingConfig: Optional[RecognitionConfig] = None if self._credentialsFile.exists() and not self.ConfigManager.getAliceConfigByName('googleASRCredentials'): self.ConfigManager.updateAliceConfiguration(key='googleASRCredentials', value=self._credentialsFile.read_text(), doPreAndPostProcessing=False) @@ -70,7 +70,7 @@ def onStart(self): super().onStart() os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = str(self._credentialsFile) - self._client = speech.SpeechClient() + self._client = SpeechClient() # noinspection PyUnresolvedReferences config = speech.RecognitionConfig( encoding=speech.RecognitionConfig.AudioEncoding.LINEAR16, @@ -80,7 +80,7 @@ def onStart(self): model='command_and_search' ) - self._streamingConfig = speech.StreamingRecognitionConfig(config=config, interim_results=True) + self._streamingConfig = StreamingRecognitionConfig(config=config, interim_results=True) def decodeStream(self, session: DialogSession) -> Optional[ASRResult]: @@ -95,7 +95,7 @@ def decodeStream(self, session: DialogSession) -> Optional[ASRResult]: audioStream = stream.audioStream() # noinspection PyUnresolvedReferences try: - requests = (speech.StreamingRecognizeRequest(audio_content=content) for content in audioStream) + requests = (StreamingRecognizeRequest(audio_content=content) for content in audioStream) responses = self._client.streaming_recognize(self._streamingConfig, requests) result = self._checkResponses(session, responses) except Exception as e: diff --git a/core/base/SkillManager.py b/core/base/SkillManager.py index 32e360e6a..dd946e503 100644 --- a/core/base/SkillManager.py +++ b/core/base/SkillManager.py @@ -719,34 +719,36 @@ def _installSkills(self, skills: list) -> dict: self.logError(f'Error stopping "{skillName}" for update: {e}') raise - gitCloner = GithubCloner(baseUrl=f'{constants.GITHUB_URL}/skill_{skillName}.git', dest=directory, skillName=skillName) - - try: - gitCloner.clone(skillName=skillName) - self.logInfo('Skill successfully downloaded') - self._installSkill(res) - skillsToBoot[skillName] = { - 'update': updating - } - except (GithubTokenFailed, GithubRateLimit): - self.logError('Failed cloning skill') - raise - except GithubNotFound: - if self.ConfigManager.getAliceConfigByName('devMode'): - if not Path(f'{self.Commons.rootDir}/skills/{skillName}').exists() or not \ - Path(f'{self.Commons.rootDir}/skills/{skillName}/{skillName.py}').exists() or not \ - Path(f'{self.Commons.rootDir}/skills/{skillName}/dialogTemplate').exists() or not \ - Path(f'{self.Commons.rootDir}/skills/{skillName}/talks').exists(): - self.logWarning(f'Skill "{skillName}" cannot be installed in dev mode due to missing base files') - else: - self._installSkill(res) - skillsToBoot[skillName] = { - 'update': updating - } - continue - else: - self.logWarning(f'Skill "{skillName}" is not available on Github, cannot install') - raise + skill = AliceSkill() + + # gitCloner = GithubCloner(baseUrl=f'{constants.GITHUB_URL}/skill_{skillName}.git', dest=directory, skillName=skillName) + # + # try: + # gitCloner.cloneSkill() + # self.logInfo('Skill successfully downloaded') + # self._installSkill(res) + # skillsToBoot[skillName] = { + # 'update': updating + # } + # except (GithubTokenFailed, GithubRateLimit): + # self.logError('Failed cloning skill') + # raise + # except GithubNotFound: + # if self.ConfigManager.getAliceConfigByName('devMode'): + # if not Path(f'{self.Commons.rootDir}/skills/{skillName}').exists() or not \ + # Path(f'{self.Commons.rootDir}/skills/{skillName}/{skillName.py}').exists() or not \ + # Path(f'{self.Commons.rootDir}/skills/{skillName}/dialogTemplate').exists() or not \ + # Path(f'{self.Commons.rootDir}/skills/{skillName}/talks').exists(): + # self.logWarning(f'Skill "{skillName}" cannot be installed in dev mode due to missing base files') + # else: + # self._installSkill(res) + # skillsToBoot[skillName] = { + # 'update': updating + # } + # continue + # else: + # self.logWarning(f'Skill "{skillName}" is not available on Github, cannot install') + # raise except SkillNotConditionCompliant as e: self.logInfo(f'Skill "{skillName}" does not comply to "{e.condition}" condition, required "{e.conditionValue}"') diff --git a/core/base/model/AliceSkill.py b/core/base/model/AliceSkill.py index 32724e172..43c539e2c 100644 --- a/core/base/model/AliceSkill.py +++ b/core/base/model/AliceSkill.py @@ -43,8 +43,12 @@ class AliceSkill(ProjectAliceObject): - def __init__(self, supportedIntents: Iterable = None, databaseSchema: dict = None, **kwargs): + def __init__(self, supportedIntents: Iterable = None, databaseSchema: dict = None, isNew: bool = False, **kwargs): super().__init__(**kwargs) + + if isNew: + return + try: self._skillPath = Path(inspect.getfile(self.__class__)).parent self._installFile = Path(inspect.getfile(self.__class__)).with_suffix('.install') diff --git a/core/base/model/GithubCloner.py b/core/base/model/GithubCloner.py index 06dd4e820..a299dee20 100644 --- a/core/base/model/GithubCloner.py +++ b/core/base/model/GithubCloner.py @@ -23,7 +23,7 @@ import requests from dulwich.errors import NotGitRepository -from dulwich.porcelain import RemoteExists, clone, commit, fetch, pull, push, remote_add, status +from dulwich import porcelain as git from dulwich.repo import Repo from core.base.SuperManager import SuperManager @@ -39,9 +39,14 @@ def __init__(self, baseUrl: str, dest: Path, skillName: str = None): self._baseUrl = baseUrl self._dest = dest self._skillName = skillName - self._repo = None - if skillName and skillName in self.SkillManager.allSkills: - self._modified = self.SkillManager.allSkills[skillName]['modified'] + + if dest.exists(): + self._repo = Repo(dest) + else: + self._repo = Repo.init(dest, mkdir=True) + + if skillName: + self._modified = self.SkillManager.allSkills.get(skillName, dict()).get('modified', False) else: self._modified = False @@ -67,24 +72,20 @@ def hasAuth(cls) -> bool: return cls.getGithubAuth() is not None - def clone(self, skillName: str) -> bool: + def cloneSkill(self): """ - Clone a skill from github to the skills folder + Clones a skill from github to the skills folder This will stash and clean all changes that have been made locally - :param skillName: :return: """ - if not self._dest.exists(): - self._dest.mkdir(parents=True) - else: - if Path(self._dest / '.git').exists(): - self.Commons.runSystemCommand(['git', '-C', str(self._dest), 'stash']) - self.Commons.runSystemCommand(['git', '-C', str(self._dest), 'clean', '-df']) - else: - shutil.rmtree(str(self._dest)) - self._dest.mkdir(parents=True) + try: + self._repo = Repo.init(self._dest, mkdir=True) - return self._doClone(skillName) + git.stash_push(self._repo) + git.clean(self._repo) + git.clone(source=self._baseUrl, target=f'skills/{self._skillName}') + except Exception: + raise def _doClone(self, skillName: str) -> bool: @@ -171,7 +172,10 @@ def checkoutOwnFork(self) -> bool: def getRemote(self, AliceSK: bool = False, origin: bool = False, noToken: bool = False): # NOSONAR - tokenPrefix = f'{self.ConfigManager.getAliceConfigByName("githubUsername")}:{self.ConfigManager.getAliceConfigByName("githubToken")}@' + tokenPrefix = '' + if self.ConfigManager.getAliceConfigByName('githubUsername') and self.ConfigManager.getAliceConfigByName('githubToken'): + tokenPrefix = f'{self.ConfigManager.getAliceConfigByName("githubUsername")}:{self.ConfigManager.getAliceConfigByName("githubToken")}@' + if self._skillName: if AliceSK: return f'https://{"" if noToken else tokenPrefix}github.com/{self.ConfigManager.getAliceConfigByName("githubUsername")}/skill_{self._skillName}.git' @@ -229,11 +233,11 @@ def repo(self, repo: Repo): def pull(self, refSpecs: str = b'master'): - pull(repo=self.repo, remote_location=self.getRemote(), refspecs=refSpecs) + git.pull(repo=self.repo, remote_location=self.getRemote(), refspecs=refSpecs) def fetch(self): - remoteRefs = fetch(repo=self.repo, remote_location=self.getRemote()) + remoteRefs = git.fetch(repo=self.repo, remote_location=self.getRemote()) for key, value in remoteRefs.items(): self.repo.refs[key] = value @@ -247,10 +251,10 @@ def init(self) -> bool: create a repository online and clone it to the skills folder - only afterwards fill it with files by AliceSK :return: """ - clone(source=self.getRemote(), target=f'skills/{self._skillName}') + git.clone(source=self.getRemote(), target=f'skills/{self._skillName}') self.repo = Repo.init(f'skills/{self._skillName}') - remote_add(repo=self.repo, name=f'AliceSK', url=self.getRemote()) + git.remote_add(repo=self.repo, name=f'AliceSK', url=self.getRemote()) return True @@ -260,7 +264,7 @@ def add(self) -> bool: add all changes to the current tree :return: """ - stat = status(self.repo) + stat = git.status(self.repo) self.repo.stage(stat.unstaged + stat.untracked) return True @@ -271,7 +275,7 @@ def commit(self, message: str = 'pushed by AliceSK') -> bool: :param message: :return: """ - commit(repo=self.repo, message=message) + git.commit(repo=self.repo, message=message) return True @@ -280,7 +284,7 @@ def push(self) -> bool: push the skills changes to AliceSK upstream :return: """ - push(repo=self.repo, remote_location=self.getRemote(), refspecs=b'master') + git.push(repo=self.repo, remote_location=self.getRemote(), refspecs=b'master') return True @@ -310,13 +314,13 @@ def createRepo(self, aliceSK: bool = False) -> bool: except Exception: return False try: - remote_add(repo=self.repo, name=b'origin', url=self.getRemote(origin=True)) - except RemoteExists: + git.remote_add(repo=self.repo, name=b'origin', url=self.getRemote(origin=True)) + except git.RemoteExists: pass try: if aliceSK: - remote_add(repo=self.repo, name=b'AliceSK', url=self.getRemote(AliceSK=True)) - except RemoteExists: + git.remote_add(repo=self.repo, name=b'AliceSK', url=self.getRemote(AliceSK=True)) + except git.RemoteExists: pass return True @@ -345,10 +349,10 @@ def gitDoMyTest(self): rep: Repo = Repo(f'skills/{skillName}') self.logInfo(f'got {rep} in {rep.path}') - stat = status(rep) + stat = git.status(rep) self.logInfo(f'statstaged {stat.staged}') self.logInfo(f'statuntrack {stat.untracked}') self.logInfo(f'statunstag {stat.unstaged}') rep.stage(stat.unstaged + stat.untracked) - self.logInfo(f'commit {commit(repo=rep, message="pushed by AliceSK")}') - self.logInfo(f'push {push(repo=rep, remote_location=self.getRemote(), refspecs=b"master")}') + self.logInfo(f'commit {git.commit(repo=rep, message="pushed by AliceSK")}') + self.logInfo(f'push {git.push(repo=rep, remote_location=self.getRemote(), refspecs=b"master")}') From 549281e97104e6ec885cf30db8e6b793f4ef8d8a Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Thu, 4 Nov 2021 19:11:53 +0100 Subject: [PATCH 007/129] out of time for now --- core/base/SkillManager.py | 22 +++++++++++++--------- core/base/model/AliceSkill.py | 18 +++++++++++++++++- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/core/base/SkillManager.py b/core/base/SkillManager.py index dd946e503..ba12d85e2 100644 --- a/core/base/SkillManager.py +++ b/core/base/SkillManager.py @@ -30,7 +30,7 @@ import requests -from core.ProjectAliceExceptions import AccessLevelTooLow, GithubNotFound, GithubRateLimit, GithubTokenFailed, SkillNotConditionCompliant, SkillStartDelayed, SkillStartingFailed +from core.ProjectAliceExceptions import AccessLevelTooLow, GithubNotFound, SkillNotConditionCompliant, SkillStartDelayed, SkillStartingFailed from core.base.SuperManager import SuperManager from core.base.model import Intent from core.base.model.AliceSkill import AliceSkill @@ -712,14 +712,18 @@ def _installSkills(self, skills: list) -> dict: self.checkSkillConditions(installFile) - if skillName in self._activeSkills: - try: - self._activeSkills[skillName].onStop() - except Exception as e: - self.logError(f'Error stopping "{skillName}" for update: {e}') - raise - - skill = AliceSkill() + if not updating: + skill = AliceSkill(isNew=True) + else: + skill = self._skillList[skillName] + if skillName in self._activeSkills: + try: + self._activeSkills[skillName].onStop() + except Exception as e: + self.logError(f'Error stopping "{skillName}" for update: {e}') + raise + + skill.clone(f'{constants.GITHUB_URL}/skill_{skillName}.git', skillName=skillName) # gitCloner = GithubCloner(baseUrl=f'{constants.GITHUB_URL}/skill_{skillName}.git', dest=directory, skillName=skillName) # diff --git a/core/base/model/AliceSkill.py b/core/base/model/AliceSkill.py index 43c539e2c..ba3d28d59 100644 --- a/core/base/model/AliceSkill.py +++ b/core/base/model/AliceSkill.py @@ -28,6 +28,8 @@ from typing import Any, Dict, Iterable, List, Optional, Union import flask +from dulwich import porcelain as git +from dulwich.repo import Repo from markdown import markdown from paho.mqtt import client as MQTTClient @@ -46,11 +48,14 @@ class AliceSkill(ProjectAliceObject): def __init__(self, supportedIntents: Iterable = None, databaseSchema: dict = None, isNew: bool = False, **kwargs): super().__init__(**kwargs) + self._remote = '' + self._repo = None + self._skillPath = Path(inspect.getfile(self.__class__)).parent + if isNew: return try: - self._skillPath = Path(inspect.getfile(self.__class__)).parent self._installFile = Path(inspect.getfile(self.__class__)).with_suffix('.install') self._installer = json.loads(self._installFile.read_text()) except FileNotFoundError: @@ -110,6 +115,17 @@ def failedStarting(self, value: bool): self._failedStarting = value + def clone(self, remote: str, skillName: str): + self._remote = remote + self._name = skillName + if self.getResource().exists(): + self._repo = Repo(self.getResource()) + git.stash_push(self._repo) + git.pull(self._repo) + else: + self._repo = git.clone(source=remote, target=self.getResource()) + + def registerDeviceInstance(self, device: Device): if device.paired: self._myDevices[device.uid] = device From 34a61a1390c85f3fadfefe153c164e3bc9eb3328 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Fri, 5 Nov 2021 06:07:21 +0100 Subject: [PATCH 008/129] tbc --- core/base/SkillManager.py | 21 +++++++++++++++++---- core/base/model/AliceSkill.py | 12 ++---------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/core/base/SkillManager.py b/core/base/SkillManager.py index ba12d85e2..e727e5369 100644 --- a/core/base/SkillManager.py +++ b/core/base/SkillManager.py @@ -29,6 +29,8 @@ from typing import Dict, List, Optional, Union import requests +from dulwich import errors as gitErrors +from dulwich import porcelain as git from core.ProjectAliceExceptions import AccessLevelTooLow, GithubNotFound, SkillNotConditionCompliant, SkillStartDelayed, SkillStartingFailed from core.base.SuperManager import SuperManager @@ -712,9 +714,7 @@ def _installSkills(self, skills: list) -> dict: self.checkSkillConditions(installFile) - if not updating: - skill = AliceSkill(isNew=True) - else: + if updating: skill = self._skillList[skillName] if skillName in self._activeSkills: try: @@ -723,9 +723,22 @@ def _installSkills(self, skills: list) -> dict: self.logError(f'Error stopping "{skillName}" for update: {e}') raise + try: + repository = git.Repo(directory) + git.stash_push(repository) + git.clean(repository) + git.update_head(repo=repository, target=self.SkillStoreManager.getSkillUpdateTag(skillName)) + git.pull(repository) + except gitErrors.NotGitRepository: + repository = git.clone( + source=f'{constants.GITHUB_URL}/skill_{skillName}.git', + target='skills/AliceCore', + checkout=self.SkillStoreManager.getSkillUpdateTag(skillName) + ) + skill.clone(f'{constants.GITHUB_URL}/skill_{skillName}.git', skillName=skillName) - # gitCloner = GithubCloner(baseUrl=f'{constants.GITHUB_URL}/skill_{skillName}.git', dest=directory, skillName=skillName) + gitCloner = GithubCloner(baseUrl=f'{constants.GITHUB_URL}/skill_{skillName}.git', dest=directory, skillName=skillName) # # try: # gitCloner.cloneSkill() diff --git a/core/base/model/AliceSkill.py b/core/base/model/AliceSkill.py index ba3d28d59..52071ec08 100644 --- a/core/base/model/AliceSkill.py +++ b/core/base/model/AliceSkill.py @@ -28,8 +28,6 @@ from typing import Any, Dict, Iterable, List, Optional, Union import flask -from dulwich import porcelain as git -from dulwich.repo import Repo from markdown import markdown from paho.mqtt import client as MQTTClient @@ -45,17 +43,11 @@ class AliceSkill(ProjectAliceObject): - def __init__(self, supportedIntents: Iterable = None, databaseSchema: dict = None, isNew: bool = False, **kwargs): + def __init__(self, supportedIntents: Iterable = None, databaseSchema: dict = None, **kwargs): super().__init__(**kwargs) - self._remote = '' - self._repo = None - self._skillPath = Path(inspect.getfile(self.__class__)).parent - - if isNew: - return - try: + self._skillPath = Path(inspect.getfile(self.__class__)).parent self._installFile = Path(inspect.getfile(self.__class__)).with_suffix('.install') self._installer = json.loads(self._installFile.read_text()) except FileNotFoundError: From 1ac55db228f00ab2ca3678a5063a2693c2660c1c Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Sun, 7 Nov 2021 13:51:42 +0100 Subject: [PATCH 009/129] Round one, do not use yet, doesn't use git status yet and will crush your skill local changes!! --- core/ProjectAliceExceptions.py | 18 +- core/base/SkillManager.py | 380 +++++++++++++++++++++++---------- core/base/SkillStoreManager.py | 9 +- core/base/model/AliceSkill.py | 27 +-- core/commons/CommonsManager.py | 25 ++- 5 files changed, 309 insertions(+), 150 deletions(-) diff --git a/core/ProjectAliceExceptions.py b/core/ProjectAliceExceptions.py index 318c6aa8c..5e2445b39 100755 --- a/core/ProjectAliceExceptions.py +++ b/core/ProjectAliceExceptions.py @@ -56,11 +56,25 @@ def __init__(self, clazz: str, funcName: str): class SkillStartingFailed(ProjectAliceException): - def __init__(self, skillName: str = '', error: str = ''): + def __init__(self, skillName: str, error: str = ''): super().__init__(message=error) self._logger.logWarning(f'[{skillName}] Error starting skill: {error}') - if skillName: + if skillName in SuperManager.getInstance().skillManager.NEEDED_SKILLS: + self._logger.logFatal(f'Skill **{skillName}** is required to continue, sorry') + else: + SuperManager.getInstance().skillManager.deactivateSkill(skillName) + + +class SkillInstanceFailed(ProjectAliceException): + + def __init__(self, skillName: str, error: str = ''): + super().__init__(message=error) + self._logger.logWarning(f'[{skillName}] Error creating skill instance: {error}') + + if skillName in SuperManager.getInstance().skillManager.NEEDED_SKILLS: + self._logger.logFatal(f'Skill **{skillName}** is required to continue, sorry') + else: SuperManager.getInstance().skillManager.deactivateSkill(skillName) diff --git a/core/base/SkillManager.py b/core/base/SkillManager.py index e727e5369..e584bfe41 100644 --- a/core/base/SkillManager.py +++ b/core/base/SkillManager.py @@ -19,20 +19,20 @@ import getpass +import traceback + import importlib import json import os +import requests import shutil import threading -import traceback +from dulwich import errors as gitErrors, porcelain as git +from dulwich.repo import Repo from pathlib import Path -from typing import Dict, List, Optional, Union +from typing import Any, Dict, List, Optional, Union -import requests -from dulwich import errors as gitErrors -from dulwich import porcelain as git - -from core.ProjectAliceExceptions import AccessLevelTooLow, GithubNotFound, SkillNotConditionCompliant, SkillStartDelayed, SkillStartingFailed +from core.ProjectAliceExceptions import AccessLevelTooLow, GithubNotFound, SkillInstanceFailed, SkillNotConditionCompliant, SkillStartDelayed, SkillStartingFailed from core.base.SuperManager import SuperManager from core.base.model import Intent from core.base.model.AliceSkill import AliceSkill @@ -49,12 +49,6 @@ class SkillManager(Manager): DBTAB_SKILLS = 'skills' - NEEDED_SKILLS = [ - 'AliceCore', - 'ContextSensitive', - 'RedQueen' - ] - DATABASE = { DBTAB_SKILLS: [ 'skillName TEXT NOT NULL UNIQUE', @@ -64,6 +58,20 @@ class SkillManager(Manager): ] } + BASE_SKILLS = [ + 'AliceCore', + 'ContextSensitive', + 'DateDayTimeYear', + 'RedQueen', + 'Telemetry' + ] + + NEEDED_SKILLS = [ + 'AliceCore', + 'ContextSensitive', + 'RedQueen' + ] + def __init__(self): super().__init__(databaseSchema=self.DATABASE) @@ -88,30 +96,40 @@ def onStart(self): super().onStart() self._busyInstalling = self.ThreadManager.newEvent('skillInstallation') - self._skillList = self._loadSkills() # If it's the first time we start, don't delay skill install and do it on main thread + # if not self._skillList: + # self.logInfo('Looks like a fresh install or skills were nuked. Let\'s install the basic skills!') + # self.wipeSkills(True) + # self._checkForSkillInstall() + # elif self.checkForSkillUpdates(): + # self._checkForSkillInstall() + #self._skillInstallThread = self.ThreadManager.newThread(name='SkillInstallThread', target=self._checkForSkillInstall, autostart=False) + if not self._skillList: self.logInfo('Looks like a fresh install or skills were nuked. Let\'s install the basic skills!') - self.wipeSkills(True) - self._checkForSkillInstall() - elif self.checkForSkillUpdates(): - self._checkForSkillInstall() + self.downloadSkills(skills=self.BASE_SKILLS) + elif sorted(list(self._skillList.keys())) != sorted(self.BASE_SKILLS): + self.logInfo('Some required skills are missing, let\'s download them!') + self.downloadSkills(skills=list(set(self.NEEDED_SKILLS) - set(list(self._skillList.keys())))) - self._skillInstallThread = self.ThreadManager.newThread(name='SkillInstallThread', target=self._checkForSkillInstall, autostart=False) self._initSkills() for skillName in self._deactivatedSkills: self.configureSkillIntents(skillName=skillName, state=False) + updates = self.checkForSkillUpdates() + if updates: + self.updateSkills(skills=updates) + self.ConfigManager.loadCheckAndUpdateSkillConfigurations() self.startAllSkills() # noinspection SqlResolve - def _loadSkills(self) -> dict: + def _loadSkills(self) -> Dict[str, Dict[str, Any]]: skills = self.loadSkillsFromDB() skills = [skill['skillName'] for skill in skills] @@ -152,6 +170,72 @@ def _loadSkills(self) -> dict: return dict(sorted(data.items())) + def updateSkills(self, skills: Union[str, List[str]]): + """ + Updates skills to latest available version for this Alice version + :param skills: + :return: + """ + if isinstance(skills, str): + skills = [skills] + + self.MqttManager.mqttBroadcast(topic=constants.TOPIC_SYSTEM_UPDATE, payload={'sticky': True}) + for skillName in skills: + try: + repository = self.getSkillRepository(skillName=skillName) + git.stash_push(repository) + git.reset(repository, mode='hard') + git.pull(repo=repository, refspecs=self.SkillStoreManager.getSkillUpdateTag(skillName=skillName)) + except Exception as e: + self.logError(f'Error updating skill **{skillName}** : {e}') + continue + + + def downloadSkills(self, skills: Union[str, List[str]]): + """ + Clones skills + :param skills: + :return: + """ + if isinstance(skills, str): + skills = [skills] + + self.MqttManager.mqttBroadcast(topic=constants.TOPIC_SYSTEM_UPDATE, payload={'sticky': True}) + for skillName in skills: + try: + source = self.getGitRemoteSourceUrl(skillName=skillName, doAuth=False) + repository = self.getSkillRepository(skillName=skillName) + if not repository: + git.clone(source=source, target=self.getSkillDirectory(skillName=skillName), checkout=True) + + git.pull(repo=repository, refspecs=self.SkillStoreManager.getSkillUpdateTag(skillName=skillName)) + + self._skillList[skillName] = { + 'active' : 1, + 'modified' : 0, + 'installer': self.getSkillDirectory(skillName=skillName) / f'{skillName}/{skillName}.install' + } + except GithubNotFound: + if skillName in self.NEEDED_SKILLS: + self.MqttManager.mqttBroadcast(topic='hermes/leds/clear') + self.logFatal(f"Skill **{skillName}** is required but wasn't found in released skills, cannot continue") + return + else: + self.logError(f'Skill "{skillName}" not found in released skills') + continue + except Exception as e: + if skillName in self.NEEDED_SKILLS: + self.MqttManager.mqttBroadcast(topic='hermes/leds/clear') + self.logFatal(f'Error downloading skill **{skillName}** but skill is required, cannot continue: {e}') + return + else: + self.logError(f'Error downloading skill "{skillName}": {e}') + continue + + + self.MqttManager.mqttBroadcast(topic='hermes/leds/clear') + + def loadSkillsFromDB(self) -> List: return self.databaseFetch(tableName='skills') @@ -289,9 +373,16 @@ def dispatchMessage(self, session: DialogSession) -> bool: return False - def _initSkills(self, loadOnly: str = '', reload: bool = False): + def _initSkills(self, onlyInit: str = '', reload: bool = False): + """ + Loops over the available skills, creates their instances + :param onlyInit: If specified, will only init the given skill name + :param reload: If the skill is already instanciated, performs a module reload, after an update per example. + :return: + """ + for skillName, data in self._skillList.items(): - if loadOnly and skillName != loadOnly: + if onlyInit and skillName != onlyInit: continue self._activeSkills.pop(skillName, None) @@ -301,17 +392,14 @@ def _initSkills(self, loadOnly: str = '', reload: bool = False): try: if not data['active']: if skillName in self.NEEDED_SKILLS: - self.logInfo(f"Skill {skillName} marked as disabled but it shouldn't be") - self.ProjectAlice.onStop() - break - - self.logInfo(f'Skill {skillName} is disabled') - - if data['active']: + self.logFatal(f"Skill {skillName} marked as disabled but it shouldn't be") + return + else: + self.logInfo(f'Skill {skillName} is disabled') + else: self.checkSkillConditions(self._skillList[skillName]['installer']) skillInstance = self.instanciateSkill(skillName=skillName, reload=reload) - skillInstance.modified = data.get('modified', False) if skillInstance: if skillName in self.NEEDED_SKILLS: skillInstance.required = True @@ -320,17 +408,22 @@ def _initSkills(self, loadOnly: str = '', reload: bool = False): self._activeSkills[skillInstance.name] = skillInstance else: self._deactivatedSkills[skillName] = skillInstance - else: - self._failedSkills[skillName] = FailedAliceSkill(data['installer']) - except SkillStartingFailed as e: - self.logWarning(f'Failed loading skill: {e}') - self._failedSkills[skillName] = FailedAliceSkill(data['installer']) - continue + skillInstance.modified = data.get('modified', False) + else: + if skillName in self.NEEDED_SKILLS: + self.logFatal(f'The skill is required to continue...') + return + else: + self._failedSkills[skillName] = FailedAliceSkill(data['installer']) except SkillNotConditionCompliant as e: - self.logInfo(f'Skill {skillName} does not comply to "{e.condition}" condition, offers only "{e.conditionValue}"') - self._failedSkills[skillName] = FailedAliceSkill(data['installer']) - continue + if skillName in self.NEEDED_SKILLS: + self.logFatal(f'Skill {skillName} does not comply to "{e.condition}" condition, offers only "{e.conditionValue}". The skill is required to continue') + return + else: + self.logInfo(f'Skill {skillName} does not comply to "{e.condition}" condition, offers only "{e.conditionValue}"') + self._failedSkills[skillName] = FailedAliceSkill(data['installer']) + continue except Exception as e: self.logWarning(f'Something went wrong loading skill {skillName}: {e}') self._failedSkills[skillName] = FailedAliceSkill(data['installer']) @@ -338,8 +431,8 @@ def _initSkills(self, loadOnly: str = '', reload: bool = False): # noinspection PyTypeChecker - def instanciateSkill(self, skillName: str, skillResource: str = '', reload: bool = False) -> AliceSkill: - instance: AliceSkill = None + def instanciateSkill(self, skillName: str, skillResource: str = '', reload: bool = False) -> Optional[AliceSkill]: + instance: Optional[AliceSkill] = None skillResource = skillResource or skillName try: @@ -355,8 +448,10 @@ def instanciateSkill(self, skillName: str, skillResource: str = '', reload: bool traceback.print_exc() except AttributeError as e: self.logError(f"Couldn't find main class for skill {skillName}.{skillResource}: {e}") + except SkillInstanceFailed as e: + self.logError(f"Couldn't instanciate skill {skillName}.{skillResource}") except Exception as e: - self.logError(f"Couldn't instanciate skill {skillName}.{skillResource}: {e} {traceback.print_exc()}") + self.logError(f"Unknown error instanciating {skillName}.{skillResource}: {e} {traceback.print_exc()}") return instance @@ -531,7 +626,7 @@ def toggleSkillState(self, skillName: str, persistent: bool = False): @Online(catchOnly=True) @IfSetting(settingName='stayCompletelyOffline', settingValue=False) - def checkForSkillUpdates(self, skillToCheck: str = None) -> bool: + def checkForSkillUpdates(self, skillToCheck: str = None) -> List[str]: """ Check all installed skills for availability of updates. Includes failed skills but not inactive. @@ -539,7 +634,7 @@ def checkForSkillUpdates(self, skillToCheck: str = None) -> bool: :return: """ self.logInfo('Checking for skill updates') - updateCount = 0 + skillsToUpdate = list() for skillName, data in self._skillList.items(): if not data['active']: @@ -552,7 +647,7 @@ def checkForSkillUpdates(self, skillToCheck: str = None) -> bool: remoteVersion = self.SkillStoreManager.getSkillUpdateVersion(skillName) localVersion = Version.fromString(self._skillList[skillName]['installer']['version']) if localVersion < remoteVersion: - updateCount += 1 + skillsToUpdate.append(skillName) self.WebUINotificationManager.newNotification( typ=UINotificationType.INFO, @@ -563,7 +658,7 @@ def checkForSkillUpdates(self, skillToCheck: str = None) -> bool: if data.get('modified', False): self.allSkills[skillName].updateAvailable = True - self.logInfo(f'![blue]({skillName}) - Version {self._skillList[skillName]["installer"]["version"]} < {str(remoteVersion)} in {self.ConfigManager.getAliceConfigByName("skillsUpdateChannel")} - ![blue](LOCKED) for local changes!') + self.logInfo(f'![blue]({skillName}) - Version {self._skillList[skillName]["installer"]["version"]} < {str(remoteVersion)} in {self.ConfigManager.getAliceConfigByName("skillsUpdateChannel")} - Locked for local changes!') continue self.logInfo(f'![yellow]({skillName}) - Version {self._skillList[skillName]["installer"]["version"]} < {str(remoteVersion)} in {self.ConfigManager.getAliceConfigByName("skillsUpdateChannel")}') @@ -576,7 +671,7 @@ def checkForSkillUpdates(self, skillToCheck: str = None) -> bool: raise Exception else: if data.get('modified', False): - self.logInfo(f'![blue]({skillName}) - Version {self._skillList[skillName]["installer"]["version"]} in {self.ConfigManager.getAliceConfigByName("skillsUpdateChannel")} - ![blue](LOCKED) for local changes!') + self.logInfo(f'![blue]({skillName}) - Version {self._skillList[skillName]["installer"]["version"]} in {self.ConfigManager.getAliceConfigByName("skillsUpdateChannel")} - Locked for local changes!') else: self.logInfo(f'![green]({skillName}) - Version {self._skillList[skillName]["installer"]["version"]} in {self.ConfigManager.getAliceConfigByName("skillsUpdateChannel")}') @@ -586,8 +681,8 @@ def checkForSkillUpdates(self, skillToCheck: str = None) -> bool: except Exception as e: self.logError(f'Error checking updates for skill **{skillName}**: {e}') - self.logInfo(f'Found {updateCount} skill update', plural='update') - return updateCount > 0 + self.logInfo(f'Found {len(skillsToUpdate)} skill update', plural='update') + return skillsToUpdate @Online(catchOnly=True) @@ -607,7 +702,7 @@ def _checkForSkillInstall(self): skillsToBoot = dict() try: - skillsToBoot = self._installSkills(files) + skillsToBoot = self._installSkillTickets(files) except Exception as e: self._logger.logError(f'Error installing skill: {e}') finally: @@ -631,7 +726,7 @@ def _finishInstall(self, skills: dict = None, startSkill: bool = False): for skillName, info in skills.items(): if startSkill: - self._initSkills(loadOnly=skillName, reload=info['update']) + self._initSkills(onlyInit=skillName, reload=info['update']) self.ConfigManager.loadCheckAndUpdateSkillConfigurations(skillToLoad=skillName) try: @@ -670,7 +765,12 @@ def _finishInstall(self, skills: dict = None, startSkill: bool = False): self.AssistantManager.checkAssistant() - def _installSkills(self, skills: list) -> dict: + def _installSkillTickets(self, skills: list) -> dict: + """ + Installs the skills from found install tickets + :param skills: list of tickets + :return: + """ root = Path(self.Commons.rootDir(), constants.SKILL_INSTALL_TICKET_PATH) skillsToBoot = dict() self.MqttManager.mqttBroadcast(topic=constants.TOPIC_SYSTEM_UPDATE, payload={'sticky': True}) @@ -689,18 +789,16 @@ def _installSkills(self, skills: list) -> dict: self.logError('Skill name to install not found, aborting to avoid casualties!') continue - directory = Path(self.Commons.rootDir()) / 'skills' / skillName - if skillName in self._skillList: installedVersion = Version.fromString(self._skillList[skillName]['installer']['version']) remoteVersion = Version.fromString(installFile['version']) if installedVersion >= remoteVersion: - self.logWarning(f'Skill "{skillName}" is already installed, skipping') + self.logWarning(f'Skill "{skillName}" is already installed and up to date, skipping') self.Commons.runRootSystemCommand(['rm', res]) continue else: - self.logWarning(f'Skill "{skillName}" needs updating') + self.logWarning(f'Skill "{skillName}" installed but needs updating') updating = True self.MqttManager.mqttBroadcast( @@ -714,58 +812,68 @@ def _installSkills(self, skills: list) -> dict: self.checkSkillConditions(installFile) + try: + skillRepository = self.getSkillRepository(skillName=skillName) + except GithubNotFound: + if self.ConfigManager.getAliceConfigByName('devMode'): + if not Path(f'{self.Commons.rootDir}/skills/{skillName}').exists() or not \ + Path(f'{self.Commons.rootDir}/skills/{skillName}/{skillName.py}').exists() or not \ + Path(f'{self.Commons.rootDir}/skills/{skillName}/dialogTemplate').exists() or not \ + Path(f'{self.Commons.rootDir}/skills/{skillName}/talks').exists(): + self.logWarning(f'Skill "{skillName}" cannot be installed in dev mode due to missing base files') + else: + self._installSkill(res) + skillsToBoot[skillName] = { + 'update': updating + } + continue + else: + self.logWarning(f'Skill "{skillName}" is not available on Github, cannot install') + raise + + if updating: - skill = self._skillList[skillName] - if skillName in self._activeSkills: - try: - self._activeSkills[skillName].onStop() - except Exception as e: - self.logError(f'Error stopping "{skillName}" for update: {e}') - raise + try: + self._skillList[skillName].onStop() + except Exception as e: + self.logError(f'Error stopping "{skillName}" for update: {e}') + raise + + git.stash_push(skillRepository) + git.reset(skillRepository, mode='hard') try: - repository = git.Repo(directory) - git.stash_push(repository) - git.clean(repository) - git.update_head(repo=repository, target=self.SkillStoreManager.getSkillUpdateTag(skillName)) - git.pull(repository) - except gitErrors.NotGitRepository: - repository = git.clone( - source=f'{constants.GITHUB_URL}/skill_{skillName}.git', - target='skills/AliceCore', - checkout=self.SkillStoreManager.getSkillUpdateTag(skillName) - ) + git.pull(skillRepository, refspecs=self.SkillStoreManager.getSkillUpdateTag(skillName)) + except: + pass - skill.clone(f'{constants.GITHUB_URL}/skill_{skillName}.git', skillName=skillName) - - gitCloner = GithubCloner(baseUrl=f'{constants.GITHUB_URL}/skill_{skillName}.git', dest=directory, skillName=skillName) - # - # try: - # gitCloner.cloneSkill() - # self.logInfo('Skill successfully downloaded') - # self._installSkill(res) - # skillsToBoot[skillName] = { - # 'update': updating - # } - # except (GithubTokenFailed, GithubRateLimit): - # self.logError('Failed cloning skill') - # raise - # except GithubNotFound: - # if self.ConfigManager.getAliceConfigByName('devMode'): - # if not Path(f'{self.Commons.rootDir}/skills/{skillName}').exists() or not \ - # Path(f'{self.Commons.rootDir}/skills/{skillName}/{skillName.py}').exists() or not \ - # Path(f'{self.Commons.rootDir}/skills/{skillName}/dialogTemplate').exists() or not \ - # Path(f'{self.Commons.rootDir}/skills/{skillName}/talks').exists(): - # self.logWarning(f'Skill "{skillName}" cannot be installed in dev mode due to missing base files') - # else: - # self._installSkill(res) - # skillsToBoot[skillName] = { - # 'update': updating - # } - # continue - # else: - # self.logWarning(f'Skill "{skillName}" is not available on Github, cannot install') - # raise + # + # try: + # gitCloner.cloneSkill() + # self.logInfo('Skill successfully downloaded') + # self._installSkill(res) + # skillsToBoot[skillName] = { + # 'update': updating + # } + # except (GithubTokenFailed, GithubRateLimit): + # self.logError('Failed cloning skill') + # raise + # except GithubNotFound: + # if self.ConfigManager.getAliceConfigByName('devMode'): + # if not Path(f'{self.Commons.rootDir}/skills/{skillName}').exists() or not \ + # Path(f'{self.Commons.rootDir}/skills/{skillName}/{skillName.py}').exists() or not \ + # Path(f'{self.Commons.rootDir}/skills/{skillName}/dialogTemplate').exists() or not \ + # Path(f'{self.Commons.rootDir}/skills/{skillName}/talks').exists(): + # self.logWarning(f'Skill "{skillName}" cannot be installed in dev mode due to missing base files') + # else: + # self._installSkill(res) + # skillsToBoot[skillName] = { + # 'update': updating + # } + # continue + # else: + # self.logWarning(f'Skill "{skillName}" is not available on Github, cannot install') + # raise except SkillNotConditionCompliant as e: self.logInfo(f'Skill "{skillName}" does not comply to "{e.condition}" condition, required "{e.conditionValue}"') @@ -793,6 +901,52 @@ def _installSkills(self, skills: list) -> dict: return skillsToBoot + def getSkillDirectory(self, skillName: str) -> Path: + return Path(self.Commons.rootDir()) / 'skills' / skillName + + + def getSkillRepository(self, skillName: str, directory: str = None) -> Optional[Repo]: + """ + Returns a dulwich repository object for the given skill + :param skillName: + :param directory: where to clone that skill, if not standard directory + :return: + """ + + if not directory: + directory = self.getSkillDirectory(skillName=skillName) + + try: + return Repo(directory) + except gitErrors.NotGitRepository: + return None + + + def getGitRemoteSourceUrl(self, skillName: str, doAuth: bool = True) -> str: + """ + Returns the url for the skillname, taking into account if the user provided github auth + This does check if the remote exists and raises an exception in case it does not + :param skillName: + :param doAuth: Pull, clone, fetch, non oauth requests, aren't concerned by rate limit + :return: + """ + tokenPrefix = '' + if doAuth: + auth = self.Commons.getGithubAuth() + if auth: + tokenPrefix = f'{auth[0]}:{auth[1]}@' + + url = f'{constants.GITHUB_URL}/skill_{skillName}.git' + if tokenPrefix: + url = url.replace('://', f'://{tokenPrefix}') + + response = requests.get(url=url) + if response.status_code != '200': + raise GithubNotFound + + return url + + def _installSkill(self, res: Path): try: installFile = json.loads(res.read_text()) @@ -941,7 +1095,7 @@ def reloadSkill(self, skillName: str): self._activeSkills[skillName].onStop() self.broadcast(method=constants.EVENT_SKILL_STOPPED, exceptions=[self.name], propagateToSkills=True, skill=self) - self._initSkills(loadOnly=skillName, reload=True) + self._initSkills(onlyInit=skillName, reload=True) self.AssistantManager.checkAssistant() @@ -979,25 +1133,17 @@ def getSkillScenarioVersion(self, skillName: str) -> Version: return Version.fromString(data[0]['scenarioVersion']) - def wipeSkills(self, addDefaults: bool = True): + def wipeSkills(self): shutil.rmtree(Path(self.Commons.rootDir(), 'skills')) Path(self.Commons.rootDir(), 'skills').mkdir() - if addDefaults: - tickets = [ - 'https://skills.projectalice.ch/AliceCore', - 'https://skills.projectalice.ch/ContextSensitive', - 'https://skills.projectalice.ch/RedQueen', - 'https://skills.projectalice.ch/Telemetry', - 'https://skills.projectalice.ch/DateDayTimeYear' - ] - for link in tickets: - self.downloadInstallTicket(link.rsplit('/')[-1]) + for skillName in self._skillList: + self.removeSkillFromDB(skillName=skillName) self._activeSkills = dict() self._deactivatedSkills = dict() self._failedSkills = dict() - self._loadSkills() + self._skillList = dict() def createNewSkill(self, skillDefinition: dict) -> bool: diff --git a/core/base/SkillStoreManager.py b/core/base/SkillStoreManager.py index 31e8b6dff..b1d19d9c0 100644 --- a/core/base/SkillStoreManager.py +++ b/core/base/SkillStoreManager.py @@ -18,10 +18,9 @@ # Last modified: 2021.04.13 at 12:56:46 CEST import difflib -from random import shuffle -from typing import Optional - import requests +from random import shuffle +from typing import Optional, Tuple from core.ProjectAliceExceptions import GithubNotFound from core.base.model.Manager import Manager @@ -92,7 +91,7 @@ def prepareSamplesData(self, data: dict): self._skillSamplesData.setdefault(skillName, skill.get(self.LanguageManager.activeLanguage, list())) - def _getSkillUpdateVersion(self, skillName: str) -> Optional[tuple]: + def _getSkillUpdateVersion(self, skillName: str) -> Optional[Tuple[Version, str]]: """ Get the highest skill version number a user can install. This is based on the user preferences, depending on the current Alice version @@ -100,7 +99,7 @@ def _getSkillUpdateVersion(self, skillName: str) -> Optional[tuple]: In case nothing is found, DO NOT FALLBACK TO MASTER :param skillName: The skill to look for - :return: tuple + :return: tuple (Version object, tag string) """ versionMapping = self._skillStoreData.get(skillName, dict()).get('versionMapping', dict()) diff --git a/core/base/model/AliceSkill.py b/core/base/model/AliceSkill.py index 52071ec08..171160573 100644 --- a/core/base/model/AliceSkill.py +++ b/core/base/model/AliceSkill.py @@ -19,19 +19,19 @@ from __future__ import annotations +from copy import copy + +import flask import importlib import inspect import json import re -from copy import copy -from pathlib import Path -from typing import Any, Dict, Iterable, List, Optional, Union - -import flask from markdown import markdown from paho.mqtt import client as MQTTClient +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional, Union -from core.ProjectAliceExceptions import AccessLevelTooLow, SkillStartingFailed +from core.ProjectAliceExceptions import AccessLevelTooLow, SkillInstanceFailed from core.base.model.Intent import Intent from core.base.model.ProjectAliceObject import ProjectAliceObject from core.base.model.Version import Version @@ -51,9 +51,9 @@ def __init__(self, supportedIntents: Iterable = None, databaseSchema: dict = Non self._installFile = Path(inspect.getfile(self.__class__)).with_suffix('.install') self._installer = json.loads(self._installFile.read_text()) except FileNotFoundError: - raise SkillStartingFailed(skillName=constants.UNKNOWN, error=f'[{type(self).__name__}] Cannot find install file') + raise SkillInstanceFailed(skillName=constants.UNKNOWN, error=f'[{type(self).__name__}] Cannot find install file') except Exception as e: - raise SkillStartingFailed(skillName=constants.UNKNOWN, error=f'[{type(self).__name__}] Failed loading skill: {e}') + raise SkillInstanceFailed(skillName=constants.UNKNOWN, error=f'[{type(self).__name__}] Failed loading skill: {e}') instructionsFile = self.getResource(f'instructions/{self.LanguageManager.activeLanguage}.md') if not instructionsFile.exists(): @@ -107,17 +107,6 @@ def failedStarting(self, value: bool): self._failedStarting = value - def clone(self, remote: str, skillName: str): - self._remote = remote - self._name = skillName - if self.getResource().exists(): - self._repo = Repo(self.getResource()) - git.stash_push(self._repo) - git.pull(self._repo) - else: - self._repo = git.clone(source=remote, target=self.getResource()) - - def registerDeviceInstance(self, device: Device): if device.paired: self._myDevices[device.uid] = device diff --git a/core/commons/CommonsManager.py b/core/commons/CommonsManager.py index f1d1f1217..d4c348e8b 100644 --- a/core/commons/CommonsManager.py +++ b/core/commons/CommonsManager.py @@ -17,10 +17,15 @@ # # Last modified: 2021.04.13 at 12:56:46 CEST +from collections import defaultdict +from ctypes import * + import hashlib import inspect +import jinja2 import json import random +import requests import socket import sqlite3 import string @@ -28,19 +33,14 @@ import tempfile import time import uuid -from collections import defaultdict from contextlib import contextmanager, suppress -from ctypes import * from datetime import datetime +from googletrans import Translator +from paho.mqtt.client import MQTTMessage from pathlib import Path from typing import Any, Union from uuid import UUID -import jinja2 -import requests -from googletrans import Translator -from paho.mqtt.client import MQTTMessage - import core.base.SuperManager as SuperManager import core.commons.model.Slot as slotModel from core.base.model.Manager import Manager @@ -421,6 +421,17 @@ def dictFromRow(row: sqlite3.Row) -> dict: return dict(zip(row.keys(), row)) + def getGithubAuth(self) -> tuple: + """ + Returns the users configured username and token for github as a tuple + When one of the values is not supplied None is returned. + :return: + """ + username = self.ConfigManager.getAliceConfigByName('githubUsername') + token = self.ConfigManager.getAliceConfigByName('githubToken') + return (username, token) if (username and token) else None + + # noinspection PyUnusedLocal def py_error_handler(filename, line, function, err, fmt): # NOSONAR # Errors are handled by our loggers From 28c5d76c153b6251b0eb360b3ea83068a93791ae Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Sun, 7 Nov 2021 15:43:56 +0100 Subject: [PATCH 010/129] Fix skill install process, from nothing to basic skills, without tickets --- core/base/SkillManager.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/core/base/SkillManager.py b/core/base/SkillManager.py index e584bfe41..f088a2657 100644 --- a/core/base/SkillManager.py +++ b/core/base/SkillManager.py @@ -19,19 +19,19 @@ import getpass -import traceback - import importlib import json import os -import requests import shutil import threading -from dulwich import errors as gitErrors, porcelain as git -from dulwich.repo import Repo +import traceback from pathlib import Path from typing import Any, Dict, List, Optional, Union +import requests +from dulwich import errors as gitErrors, porcelain as git +from dulwich.repo import Repo + from core.ProjectAliceExceptions import AccessLevelTooLow, GithubNotFound, SkillInstanceFailed, SkillNotConditionCompliant, SkillStartDelayed, SkillStartingFailed from core.base.SuperManager import SuperManager from core.base.model import Intent @@ -81,7 +81,7 @@ def __init__(self): self._skillInstallThread: Optional[threading.Thread] = None self._supportedIntents = list() - # This is only a dict of the skills, with name: dict(status, install file, modified) + # This is a dict of the skills, with name: dict(status, install file, modified) self._skillList = dict() # These are dict of the skills, with name: skill instance @@ -206,14 +206,14 @@ def downloadSkills(self, skills: Union[str, List[str]]): source = self.getGitRemoteSourceUrl(skillName=skillName, doAuth=False) repository = self.getSkillRepository(skillName=skillName) if not repository: - git.clone(source=source, target=self.getSkillDirectory(skillName=skillName), checkout=True) + repository = git.clone(source=source, target=self.getSkillDirectory(skillName=skillName), checkout=True) git.pull(repo=repository, refspecs=self.SkillStoreManager.getSkillUpdateTag(skillName=skillName)) self._skillList[skillName] = { 'active' : 1, 'modified' : 0, - 'installer': self.getSkillDirectory(skillName=skillName) / f'{skillName}/{skillName}.install' + 'installer': json.loads(Path(self.getSkillDirectory(skillName=skillName), f'{skillName}.install').read_text()) } except GithubNotFound: if skillName in self.NEEDED_SKILLS: @@ -377,7 +377,7 @@ def _initSkills(self, onlyInit: str = '', reload: bool = False): """ Loops over the available skills, creates their instances :param onlyInit: If specified, will only init the given skill name - :param reload: If the skill is already instanciated, performs a module reload, after an update per example. + :param reload: If the skill is already instantiated, performs a module reload, after an update per example. :return: """ @@ -397,7 +397,7 @@ def _initSkills(self, onlyInit: str = '', reload: bool = False): else: self.logInfo(f'Skill {skillName} is disabled') else: - self.checkSkillConditions(self._skillList[skillName]['installer']) + self.checkSkillConditions(data['installer']) skillInstance = self.instanciateSkill(skillName=skillName, reload=reload) if skillInstance: @@ -941,7 +941,7 @@ def getGitRemoteSourceUrl(self, skillName: str, doAuth: bool = True) -> str: url = url.replace('://', f'://{tokenPrefix}') response = requests.get(url=url) - if response.status_code != '200': + if response.status_code != 200: raise GithubNotFound return url From 837aaca8a441c7fa740c81636976ef34433d83a4 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Sun, 7 Nov 2021 16:07:12 +0100 Subject: [PATCH 011/129] Further work needed --- core/base/SkillManager.py | 67 +++++++++++++++++------------------ core/util/ThreadManager.py | 4 +-- core/util/model/AliceEvent.py | 33 ++++++++++------- 3 files changed, 55 insertions(+), 49 deletions(-) diff --git a/core/base/SkillManager.py b/core/base/SkillManager.py index f088a2657..34ea334ac 100644 --- a/core/base/SkillManager.py +++ b/core/base/SkillManager.py @@ -42,7 +42,7 @@ from core.base.model.Version import Version from core.commons import constants from core.dialog.model.DialogSession import DialogSession -from core.util.Decorators import IfSetting, Online +from core.util.Decorators import IfSetting, Online, deprecated from core.webui.model.UINotificationType import UINotificationType @@ -95,18 +95,9 @@ def __init__(self): def onStart(self): super().onStart() - self._busyInstalling = self.ThreadManager.newEvent('skillInstallation') + self._busyInstalling = self.ThreadManager.newEvent(name='skillInstallation', onSetCallback=self.notifyInstalling, onClearCallback=self.notifyFinishedInstalling) self._skillList = self._loadSkills() - # If it's the first time we start, don't delay skill install and do it on main thread - # if not self._skillList: - # self.logInfo('Looks like a fresh install or skills were nuked. Let\'s install the basic skills!') - # self.wipeSkills(True) - # self._checkForSkillInstall() - # elif self.checkForSkillUpdates(): - # self._checkForSkillInstall() - #self._skillInstallThread = self.ThreadManager.newThread(name='SkillInstallThread', target=self._checkForSkillInstall, autostart=False) - if not self._skillList: self.logInfo('Looks like a fresh install or skills were nuked. Let\'s install the basic skills!') self.downloadSkills(skills=self.BASE_SKILLS) @@ -128,6 +119,14 @@ def onStart(self): self.startAllSkills() + def notifyInstalling(self): + self.MqttManager.mqttBroadcast(topic=constants.TOPIC_SYSTEM_UPDATE, payload={'sticky': True}) + + + def notifyFinishedInstalling(self): + self.MqttManager.mqttBroadcast(topic='hermes/leds/clear') + + # noinspection SqlResolve def _loadSkills(self) -> Dict[str, Dict[str, Any]]: skills = self.loadSkillsFromDB() @@ -176,10 +175,11 @@ def updateSkills(self, skills: Union[str, List[str]]): :param skills: :return: """ + self._busyInstalling.set() + if isinstance(skills, str): skills = [skills] - self.MqttManager.mqttBroadcast(topic=constants.TOPIC_SYSTEM_UPDATE, payload={'sticky': True}) for skillName in skills: try: repository = self.getSkillRepository(skillName=skillName) @@ -189,6 +189,7 @@ def updateSkills(self, skills: Union[str, List[str]]): except Exception as e: self.logError(f'Error updating skill **{skillName}** : {e}') continue + self._busyInstalling.clear() def downloadSkills(self, skills: Union[str, List[str]]): @@ -197,10 +198,11 @@ def downloadSkills(self, skills: Union[str, List[str]]): :param skills: :return: """ + self._busyInstalling.set() + if isinstance(skills, str): skills = [skills] - self.MqttManager.mqttBroadcast(topic=constants.TOPIC_SYSTEM_UPDATE, payload={'sticky': True}) for skillName in skills: try: source = self.getGitRemoteSourceUrl(skillName=skillName, doAuth=False) @@ -217,7 +219,7 @@ def downloadSkills(self, skills: Union[str, List[str]]): } except GithubNotFound: if skillName in self.NEEDED_SKILLS: - self.MqttManager.mqttBroadcast(topic='hermes/leds/clear') + self._busyInstalling.clear() self.logFatal(f"Skill **{skillName}** is required but wasn't found in released skills, cannot continue") return else: @@ -225,15 +227,14 @@ def downloadSkills(self, skills: Union[str, List[str]]): continue except Exception as e: if skillName in self.NEEDED_SKILLS: - self.MqttManager.mqttBroadcast(topic='hermes/leds/clear') + self._busyInstalling.clear() self.logFatal(f'Error downloading skill **{skillName}** but skill is required, cannot continue: {e}') return else: self.logError(f'Error downloading skill "{skillName}": {e}') continue - - self.MqttManager.mqttBroadcast(topic='hermes/leds/clear') + self._busyInstalling.clear() def loadSkillsFromDB(self) -> List: @@ -275,8 +276,6 @@ def removeSkillFromDB(self, skillName: str): def onAssistantInstalled(self, **kwargs): - self.MqttManager.mqttBroadcast(topic='hermes/leds/clear') - argv = kwargs.get('skillsInfos', dict()) if not argv: return @@ -399,7 +398,7 @@ def _initSkills(self, onlyInit: str = '', reload: bool = False): else: self.checkSkillConditions(data['installer']) - skillInstance = self.instanciateSkill(skillName=skillName, reload=reload) + skillInstance = self.instantiateSkill(skillName=skillName, reload=reload) if skillInstance: if skillName in self.NEEDED_SKILLS: skillInstance.required = True @@ -431,7 +430,7 @@ def _initSkills(self, onlyInit: str = '', reload: bool = False): # noinspection PyTypeChecker - def instanciateSkill(self, skillName: str, skillResource: str = '', reload: bool = False) -> Optional[AliceSkill]: + def instantiateSkill(self, skillName: str, skillResource: str = '', reload: bool = False) -> Optional[AliceSkill]: instance: Optional[AliceSkill] = None skillResource = skillResource or skillName @@ -448,10 +447,10 @@ def instanciateSkill(self, skillName: str, skillResource: str = '', reload: bool traceback.print_exc() except AttributeError as e: self.logError(f"Couldn't find main class for skill {skillName}.{skillResource}: {e}") - except SkillInstanceFailed as e: + except SkillInstanceFailed: self.logError(f"Couldn't instanciate skill {skillName}.{skillResource}") except Exception as e: - self.logError(f"Unknown error instanciating {skillName}.{skillResource}: {e} {traceback.print_exc()}") + self.logError(f"Unknown error instantiating {skillName}.{skillResource}: {e} {traceback.print_exc()}") return instance @@ -465,14 +464,18 @@ def onStop(self): def onQuarterHour(self): - self.checkForSkillUpdates() + if self._busyInstalling.isSet() or self.ProjectAlice.restart or self.ProjectAlice.updating or self.NluManager.training: + return + + updates = self.checkForSkillUpdates() + if updates: + self.updateSkills(skills=updates) def startAllSkills(self): supportedIntents = list() - tmp = self._activeSkills.copy() - for skillName in tmp: + for skillName in self._activeSkills.copy(): try: supportedIntents += self._startSkill(skillName) except SkillStartingFailed: @@ -481,7 +484,6 @@ def startAllSkills(self): self.logInfo(f'Skill {skillName} start is delayed') supportedIntents = list(set(supportedIntents)) - self._supportedIntents = supportedIntents self.logInfo(f'Skills started. {len(supportedIntents)} intents supported') @@ -492,13 +494,13 @@ def _startSkill(self, skillName: str) -> dict: skillInstance = self._activeSkills[skillName] elif skillName in self._deactivatedSkills: self._deactivatedSkills.pop(skillName, None) - skillInstance = self.instanciateSkill(skillName=skillName) + skillInstance = self.instantiateSkill(skillName=skillName) if skillInstance: self.activeSkills[skillName] = skillInstance else: return dict() elif skillName in self._failedSkills: - skillInstance = self.instanciateSkill(skillName=skillName) + skillInstance = self.instantiateSkill(skillName=skillName) if skillInstance: self.activeSkills[skillName] = skillInstance else: @@ -647,7 +649,6 @@ def checkForSkillUpdates(self, skillToCheck: str = None) -> List[str]: remoteVersion = self.SkillStoreManager.getSkillUpdateVersion(skillName) localVersion = Version.fromString(self._skillList[skillName]['installer']['version']) if localVersion < remoteVersion: - skillsToUpdate.append(skillName) self.WebUINotificationManager.newNotification( typ=UINotificationType.INFO, @@ -667,8 +668,7 @@ def checkForSkillUpdates(self, skillToCheck: str = None) -> List[str]: if skillName in self.allSkills: self.allSkills[skillName].updateAvailable = True else: - if not self.downloadInstallTicket(skillName, isUpdate=True): - raise Exception + skillsToUpdate.append(skillName) else: if data.get('modified', False): self.logInfo(f'![blue]({skillName}) - Version {self._skillList[skillName]["installer"]["version"]} in {self.ConfigManager.getAliceConfigByName("skillsUpdateChannel")} - Locked for local changes!') @@ -706,8 +706,6 @@ def _checkForSkillInstall(self): except Exception as e: self._logger.logError(f'Error installing skill: {e}') finally: - self.MqttManager.mqttBroadcast(topic='hermes/leds/clear') - if skillsToBoot and self.ProjectAlice.isBooted: self._finishInstall(skillsToBoot, True) else: @@ -765,6 +763,7 @@ def _finishInstall(self, skills: dict = None, startSkill: bool = False): self.AssistantManager.checkAssistant() + @deprecated def _installSkillTickets(self, skills: list) -> dict: """ Installs the skills from found install tickets diff --git a/core/util/ThreadManager.py b/core/util/ThreadManager.py index e4dcd590c..5e62a17a5 100644 --- a/core/util/ThreadManager.py +++ b/core/util/ThreadManager.py @@ -18,7 +18,7 @@ # Last modified: 2021.04.13 at 12:56:48 CEST import threading -from typing import Callable +from typing import Callable, Union from core.base.model.Manager import Manager from core.util.Decorators import IfSetting @@ -169,7 +169,7 @@ def isThreadAlive(self, name: str) -> bool: return self._threads[name].isAlive() - def newEvent(self, name: str, onSetCallback: str = None, onClearCallback: str = None) -> AliceEvent: + def newEvent(self, name: str, onSetCallback: Union[str, Callable] = None, onClearCallback: Union[str, Callable] = None) -> AliceEvent: if name in self._events: self._events[name].clear() diff --git a/core/util/model/AliceEvent.py b/core/util/model/AliceEvent.py index 7c899106e..a8028a640 100644 --- a/core/util/model/AliceEvent.py +++ b/core/util/model/AliceEvent.py @@ -18,6 +18,7 @@ # Last modified: 2021.04.13 at 12:56:47 CEST from threading import Event +from typing import Callable, Union from core.base.model.ProjectAliceObject import ProjectAliceObject from core.commons import constants @@ -25,7 +26,7 @@ class AliceEvent(Event, ProjectAliceObject): - def __init__(self, name: str, onSet: str = None, onClear: str = None): + def __init__(self, name: str, onSet: Union[str, Callable] = None, onClear: Union[str, Callable] = None): super().__init__() self._name = name self._onSet = onSet @@ -44,12 +45,15 @@ def set(self, **kwargs) -> None: if not self._onSet: self.doBroadcast(state='set', **kwargs) else: - self.broadcast( - method=self._onSet, - exceptions=[constants.DUMMY], - propagateToSkills=True, - **kwargs - ) + if isinstance(self._onSet, str): + self.broadcast( + method=self._onSet, + exceptions=[constants.DUMMY], + propagateToSkills=True, + **kwargs + ) + else: + self._onSet() def clear(self, **kwargs) -> None: @@ -66,12 +70,15 @@ def clear(self, **kwargs) -> None: if not self._onClear: self.doBroadcast(state='clear', **self._kwargs) else: - self.broadcast( - method=self._onClear, - exceptions=[constants.DUMMY], - propagateToSkills=True, - **self._kwargs - ) + if isinstance(self._onClear, str): + self.broadcast( + method=self._onClear, + exceptions=[constants.DUMMY], + propagateToSkills=True, + **self._kwargs + ) + else: + self._onClear() def cancel(self) -> None: From 7d261a77a31bd6929b7b0b811ed8424f1e7b2819 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Mon, 8 Nov 2021 05:16:37 +0100 Subject: [PATCH 012/129] Saving state to continue at work --- core/base/SkillManager.py | 113 ++++++++++++++++++++------------- core/webApi/model/SkillsApi.py | 6 +- 2 files changed, 72 insertions(+), 47 deletions(-) diff --git a/core/base/SkillManager.py b/core/base/SkillManager.py index 34ea334ac..3dcf0cffe 100644 --- a/core/base/SkillManager.py +++ b/core/base/SkillManager.py @@ -21,7 +21,6 @@ import getpass import importlib import json -import os import shutil import threading import traceback @@ -89,8 +88,6 @@ def __init__(self): self._deactivatedSkills: Dict[str, AliceSkill] = dict() self._failedSkills: Dict[str, Union[AliceSkill, FailedAliceSkill]] = dict() - self._postBootSkillActions = dict() - def onStart(self): super().onStart() @@ -100,12 +97,12 @@ def onStart(self): if not self._skillList: self.logInfo('Looks like a fresh install or skills were nuked. Let\'s install the basic skills!') - self.downloadSkills(skills=self.BASE_SKILLS) + self.installSkill(skills=self.BASE_SKILLS) elif sorted(list(self._skillList.keys())) != sorted(self.BASE_SKILLS): self.logInfo('Some required skills are missing, let\'s download them!') - self.downloadSkills(skills=list(set(self.NEEDED_SKILLS) - set(list(self._skillList.keys())))) + self.installSkill(skills=list(set(self.NEEDED_SKILLS) - set(list(self._skillList.keys())))) - self._initSkills() + self.initSkills() for skillName in self._deactivatedSkills: self.configureSkillIntents(skillName=skillName, state=False) @@ -169,10 +166,11 @@ def _loadSkills(self) -> Dict[str, Dict[str, Any]]: return dict(sorted(data.items())) - def updateSkills(self, skills: Union[str, List[str]]): + def updateSkills(self, skills: Union[str, List[str]], withSkillRestart: bool = True): """ Updates skills to latest available version for this Alice version :param skills: + :param withSkillRestart: Whether or not to start the skill after updating it :return: """ self._busyInstalling.set() @@ -181,6 +179,10 @@ def updateSkills(self, skills: Union[str, List[str]]): skills = [skills] for skillName in skills: + if skillName in self._activeSkills: + self._activeSkills[skillName].onStop() + self._activeSkills.pop(skillName, None) + try: repository = self.getSkillRepository(skillName=skillName) git.stash_push(repository) @@ -189,6 +191,25 @@ def updateSkills(self, skills: Union[str, List[str]]): except Exception as e: self.logError(f'Error updating skill **{skillName}** : {e}') continue + + self.allSkills[skillName].onSkillUpdated(skill=skillName) + self.MqttManager.mqttBroadcast( + topic=constants.TOPIC_SKILL_UPDATED, + payload={ + 'skillName': skillName + } + ) + + self.WebUINotificationManager.newNotification( + typ=UINotificationType.INFO, + notification='skillUpdated', + key=f'skillUpdate_{skillName}', + replaceBody=[skillName, self._skillList[skillName]['installer']['version']] + ) + + if withSkillRestart: + self._startSkill(skillName=skillName) + self._busyInstalling.clear() @@ -372,7 +393,7 @@ def dispatchMessage(self, session: DialogSession) -> bool: return False - def _initSkills(self, onlyInit: str = '', reload: bool = False): + def initSkills(self, onlyInit: str = '', reload: bool = False): """ Loops over the available skills, creates their instances :param onlyInit: If specified, will only init the given skill name @@ -724,7 +745,7 @@ def _finishInstall(self, skills: dict = None, startSkill: bool = False): for skillName, info in skills.items(): if startSkill: - self._initSkills(onlyInit=skillName, reload=info['update']) + self.initSkills(onlyInit=skillName, reload=info['update']) self.ConfigManager.loadCheckAndUpdateSkillConfigurations(skillToLoad=skillName) try: @@ -946,42 +967,48 @@ def getGitRemoteSourceUrl(self, skillName: str, doAuth: bool = True) -> str: return url - def _installSkill(self, res: Path): - try: - installFile = json.loads(res.read_text()) - pipReqs = installFile.get('pipRequirements', list()) - sysReqs = installFile.get('systemRequirements', list()) - scriptReq = installFile.get('script') - directory = Path(self.Commons.rootDir()) / 'skills' / installFile['name'] - - for requirement in pipReqs: - self.logInfo(f'Installing pip requirement: {requirement}') - self.Commons.runSystemCommand(['./venv/bin/pip3', 'install', requirement]) - - for requirement in sysReqs: - self.logInfo(f'Installing system requirement: {requirement}') - self.Commons.runRootSystemCommand(['apt-get', 'install', '-y', requirement]) - - if scriptReq: - self.logInfo('Running post install script') - self.Commons.runRootSystemCommand(['chmod', '+x', str(directory / scriptReq)]) - self.Commons.runRootSystemCommand([str(directory / scriptReq)]) - - self.addSkillToDB(installFile['name']) - self._skillList[installFile['name']] = { - 'active' : 1, - 'installer': installFile, - 'modified' : False - } + def installSkill(self, skills: Union[str, List[str]]): + self._busyInstalling.set() + if isinstance(skills, str): + skills = [skills] + + for skillName in skills: + try: + repository = self.getSkillRepository(skillName=skillName) + if not repository: + self.downloadSkills(skills=skillName) - os.unlink(str(res)) + directory = repository.path + installFile = json.loads(Path(directory, f'{skillName}.install').read_text()) + pipReqs = installFile.get('pipRequirements', list()) + sysReqs = installFile.get('systemRequirements', list()) + scriptReq = installFile.get('script') - if installFile.get('rebootAfterInstall', False): - self.Commons.runRootSystemCommand('sudo shutdown -r now'.split()) - return + for requirement in pipReqs: + self.logInfo(f'Installing pip requirement: {requirement}') + self.Commons.runSystemCommand(['./venv/bin/pip3', 'install', requirement]) - except Exception: - raise + for requirement in sysReqs: + self.logInfo(f'Installing system requirement: {requirement}') + self.Commons.runRootSystemCommand(['apt-get', 'install', '-y', requirement]) + + if scriptReq: + self.logInfo('Running post install script') + self.Commons.runRootSystemCommand(['chmod', '+x', str(directory / scriptReq)]) + self.Commons.runRootSystemCommand([str(directory / scriptReq)]) + + self.addSkillToDB(installFile['name']) + self._skillList[skillName] = { + 'active' : 1, + 'installer': installFile + } + + if installFile.get('rebootAfterInstall', False): + self.Commons.runRootSystemCommand('sudo shutdown -r now'.split()) + except: + raise + else: + self._busyInstalling.clear() def checkSkillConditions(self, installer: dict = None) -> bool: @@ -1094,7 +1121,7 @@ def reloadSkill(self, skillName: str): self._activeSkills[skillName].onStop() self.broadcast(method=constants.EVENT_SKILL_STOPPED, exceptions=[self.name], propagateToSkills=True, skill=self) - self._initSkills(onlyInit=skillName, reload=True) + self.initSkills(onlyInit=skillName, reload=True) self.AssistantManager.checkAssistant() diff --git a/core/webApi/model/SkillsApi.py b/core/webApi/model/SkillsApi.py index fc4b81b17..a71125638 100644 --- a/core/webApi/model/SkillsApi.py +++ b/core/webApi/model/SkillsApi.py @@ -122,10 +122,8 @@ def installSkills(self) -> Response: status = dict() for skill in skills: - if self.SkillManager.downloadInstallTicket(skill): - status[skill] = 'ok' - else: - status[skill] = 'ko' + self.SkillManager.installSkill(skills=skill) + status[skill] = 'ok' return jsonify(success=True, status=status) except Exception as e: From 65948bc08dde6b93fb007a0a81ab94ce5737796b Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Mon, 8 Nov 2021 07:24:07 +0100 Subject: [PATCH 013/129] Save before firing clean -dfx, just in case... --- core/base/SkillManager.py | 147 +++++++++++++++++++-------------- core/webApi/model/SkillsApi.py | 5 +- 2 files changed, 86 insertions(+), 66 deletions(-) diff --git a/core/base/SkillManager.py b/core/base/SkillManager.py index 3dcf0cffe..910d1b5cc 100644 --- a/core/base/SkillManager.py +++ b/core/base/SkillManager.py @@ -19,17 +19,17 @@ import getpass +import traceback + import importlib import json +import requests import shutil import threading -import traceback -from pathlib import Path -from typing import Any, Dict, List, Optional, Union - -import requests from dulwich import errors as gitErrors, porcelain as git from dulwich.repo import Repo +from pathlib import Path +from typing import Dict, List, Optional, Union from core.ProjectAliceExceptions import AccessLevelTooLow, GithubNotFound, SkillInstanceFailed, SkillNotConditionCompliant, SkillStartDelayed, SkillStartingFailed from core.base.SuperManager import SuperManager @@ -52,8 +52,7 @@ class SkillManager(Manager): DBTAB_SKILLS: [ 'skillName TEXT NOT NULL UNIQUE', 'active INTEGER NOT NULL DEFAULT 1', - 'scenarioVersion TEXT NOT NULL DEFAULT "0.0.0"', - 'modified INTEGER NOT NULL DEFAULT 0' + 'scenarioVersion TEXT NOT NULL DEFAULT "0.0.0"' ] } @@ -80,8 +79,8 @@ def __init__(self): self._skillInstallThread: Optional[threading.Thread] = None self._supportedIntents = list() - # This is a dict of the skills, with name: dict(status, install file, modified) - self._skillList = dict() + # This is a list of the skill names installed + self._skillList = list() # These are dict of the skills, with name: skill instance self._activeSkills: Dict[str, AliceSkill] = dict() @@ -97,10 +96,10 @@ def onStart(self): if not self._skillList: self.logInfo('Looks like a fresh install or skills were nuked. Let\'s install the basic skills!') - self.installSkill(skills=self.BASE_SKILLS) - elif sorted(list(self._skillList.keys())) != sorted(self.BASE_SKILLS): + self.installSkills(skills=self.BASE_SKILLS) + elif sorted(self._skillList) != sorted(self.BASE_SKILLS): self.logInfo('Some required skills are missing, let\'s download them!') - self.installSkill(skills=list(set(self.NEEDED_SKILLS) - set(list(self._skillList.keys())))) + self.installSkills(skills=list(set(self.NEEDED_SKILLS) - set(self._skillList))) self.initSkills() @@ -125,7 +124,7 @@ def notifyFinishedInstalling(self): # noinspection SqlResolve - def _loadSkills(self) -> Dict[str, Dict[str, Any]]: + def _loadSkills(self) -> List[str]: skills = self.loadSkillsFromDB() skills = [skill['skillName'] for skill in skills] @@ -151,19 +150,15 @@ def _loadSkills(self) -> Dict[str, Dict[str, Any]]: # Those represent the skills we have skills = self.loadSkillsFromDB() - data = dict() - for skill in skills.copy(): + data = list() + for skill in skills: try: - installer = json.loads(Path(self.Commons.rootDir(), f'skills/{skill["skillName"]}/{skill["skillName"]}.install').read_text()) - data[skill['skillName']] = { - 'active' : skill['active'], - 'modified' : skill['modified'] == 1, - 'installer': installer - } + Path(self.Commons.rootDir(), f'skills/{skill["skillName"]}').read_text() # Just test is the directory exists + data.append(skill['skillName']) except Exception as e: self.logError(f'Error loading skill **{skill["skillName"]}**: {e}') - return dict(sorted(data.items())) + return sorted(data) def updateSkills(self, skills: Union[str, List[str]], withSkillRestart: bool = True): @@ -187,6 +182,7 @@ def updateSkills(self, skills: Union[str, List[str]], withSkillRestart: bool = T repository = self.getSkillRepository(skillName=skillName) git.stash_push(repository) git.reset(repository, mode='hard') + self.Commons.runSystemCommand(f'git -C {str(repository.path)} clean -dfx') git.pull(repo=repository, refspecs=self.SkillStoreManager.getSkillUpdateTag(skillName=skillName)) except Exception as e: self.logError(f'Error updating skill **{skillName}** : {e}') @@ -204,7 +200,7 @@ def updateSkills(self, skills: Union[str, List[str]], withSkillRestart: bool = T typ=UINotificationType.INFO, notification='skillUpdated', key=f'skillUpdate_{skillName}', - replaceBody=[skillName, self._skillList[skillName]['installer']['version']] + replaceBody=[skillName, json.loads(self.getSkillInstallFile(skillName=skillName).read_text())['version']] ) if withSkillRestart: @@ -219,8 +215,6 @@ def downloadSkills(self, skills: Union[str, List[str]]): :param skills: :return: """ - self._busyInstalling.set() - if isinstance(skills, str): skills = [skills] @@ -232,12 +226,6 @@ def downloadSkills(self, skills: Union[str, List[str]]): repository = git.clone(source=source, target=self.getSkillDirectory(skillName=skillName), checkout=True) git.pull(repo=repository, refspecs=self.SkillStoreManager.getSkillUpdateTag(skillName=skillName)) - - self._skillList[skillName] = { - 'active' : 1, - 'modified' : 0, - 'installer': json.loads(Path(self.getSkillDirectory(skillName=skillName), f'{skillName}.install').read_text()) - } except GithubNotFound: if skillName in self.NEEDED_SKILLS: self._busyInstalling.clear() @@ -255,8 +243,6 @@ def downloadSkills(self, skills: Union[str, List[str]]): self.logError(f'Error downloading skill "{skillName}": {e}') continue - self._busyInstalling.clear() - def loadSkillsFromDB(self) -> List: return self.databaseFetch(tableName='skills') @@ -401,7 +387,7 @@ def initSkills(self, onlyInit: str = '', reload: bool = False): :return: """ - for skillName, data in self._skillList.items(): + for skillName in self._skillList: if onlyInit and skillName != onlyInit: continue @@ -409,47 +395,60 @@ def initSkills(self, onlyInit: str = '', reload: bool = False): self._failedSkills.pop(skillName, None) self._deactivatedSkills.pop(skillName, None) + installFile = self.getSkillInstallFile(skillName=skillName) + if not installFile.exists(): + if skillName in self.NEEDED_SKILLS: + self.logFatal(f'Cannot find skill install file for skill **{skillName}**. The skill is required to continue') + return + else: + self.logWarning(f'Cannot find skill install file for skill **{skillName}**, skipping.') + else: + installFile = json.loads(installFile.read_text()) + try: - if not data['active']: + skillActiveState = self.isSkillActive(skillName=skillName) + if skillActiveState: if skillName in self.NEEDED_SKILLS: self.logFatal(f"Skill {skillName} marked as disabled but it shouldn't be") return else: self.logInfo(f'Skill {skillName} is disabled') else: - self.checkSkillConditions(data['installer']) + self.checkSkillConditions(installFile) skillInstance = self.instantiateSkill(skillName=skillName, reload=reload) if skillInstance: if skillName in self.NEEDED_SKILLS: skillInstance.required = True - if data['active']: + if skillActiveState: self._activeSkills[skillInstance.name] = skillInstance else: self._deactivatedSkills[skillName] = skillInstance - - skillInstance.modified = data.get('modified', False) else: if skillName in self.NEEDED_SKILLS: self.logFatal(f'The skill is required to continue...') return else: - self._failedSkills[skillName] = FailedAliceSkill(data['installer']) + self._failedSkills[skillName] = FailedAliceSkill(installFile) except SkillNotConditionCompliant as e: if skillName in self.NEEDED_SKILLS: self.logFatal(f'Skill {skillName} does not comply to "{e.condition}" condition, offers only "{e.conditionValue}". The skill is required to continue') return else: self.logInfo(f'Skill {skillName} does not comply to "{e.condition}" condition, offers only "{e.conditionValue}"') - self._failedSkills[skillName] = FailedAliceSkill(data['installer']) + self._failedSkills[skillName] = FailedAliceSkill(installFile) continue except Exception as e: self.logWarning(f'Something went wrong loading skill {skillName}: {e}') - self._failedSkills[skillName] = FailedAliceSkill(data['installer']) + self._failedSkills[skillName] = FailedAliceSkill(installFile) continue + def getSkillInstallFile(self, skillName: str) -> Path: + return Path(self.Commons.rootDir(), f'skills/{skillName}/{skillName}.install') + + # noinspection PyTypeChecker def instantiateSkill(self, skillName: str, skillResource: str = '', reload: bool = False) -> Optional[AliceSkill]: instance: Optional[AliceSkill] = None @@ -560,6 +559,11 @@ def _startSkill(self, skillName: str) -> dict: def isSkillActive(self, skillName: str) -> bool: if skillName in self._activeSkills: return self._activeSkills[skillName].active + elif skillName in self._skillList: + row = self.databaseFetch(tableName=self.DBTAB_SKILLS, values={'skillName': skillName}) + if not row: + return False + return row[0]['active'] == 1 return False @@ -659,16 +663,18 @@ def checkForSkillUpdates(self, skillToCheck: str = None) -> List[str]: self.logInfo('Checking for skill updates') skillsToUpdate = list() - for skillName, data in self._skillList.items(): - if not data['active']: + for skillName in self._skillList: + if not self.isSkillActive(skillName=skillName): continue try: if skillToCheck and skillName != skillToCheck: continue + installer = json.loads(self.getSkillInstallFile(skillName=skillName).read_text()) + remoteVersion = self.SkillStoreManager.getSkillUpdateVersion(skillName) - localVersion = Version.fromString(self._skillList[skillName]['installer']['version']) + localVersion = Version.fromString(installer['version']) if localVersion < remoteVersion: self.WebUINotificationManager.newNotification( @@ -680,10 +686,10 @@ def checkForSkillUpdates(self, skillToCheck: str = None) -> List[str]: if data.get('modified', False): self.allSkills[skillName].updateAvailable = True - self.logInfo(f'![blue]({skillName}) - Version {self._skillList[skillName]["installer"]["version"]} < {str(remoteVersion)} in {self.ConfigManager.getAliceConfigByName("skillsUpdateChannel")} - Locked for local changes!') + self.logInfo(f'![blue]({skillName}) - Version {installer["version"]} < {str(remoteVersion)} in {self.ConfigManager.getAliceConfigByName("skillsUpdateChannel")} - Locked for local changes!') continue - self.logInfo(f'![yellow]({skillName}) - Version {self._skillList[skillName]["installer"]["version"]} < {str(remoteVersion)} in {self.ConfigManager.getAliceConfigByName("skillsUpdateChannel")}') + self.logInfo(f'![yellow]({skillName}) - Version {installer["version"]} < {str(remoteVersion)} in {self.ConfigManager.getAliceConfigByName("skillsUpdateChannel")}') if not self.ConfigManager.getAliceConfigByName('skillAutoUpdate'): if skillName in self.allSkills: @@ -692,9 +698,9 @@ def checkForSkillUpdates(self, skillToCheck: str = None) -> List[str]: skillsToUpdate.append(skillName) else: if data.get('modified', False): - self.logInfo(f'![blue]({skillName}) - Version {self._skillList[skillName]["installer"]["version"]} in {self.ConfigManager.getAliceConfigByName("skillsUpdateChannel")} - Locked for local changes!') + self.logInfo(f'![blue]({skillName}) - Version {installer["version"]} in {self.ConfigManager.getAliceConfigByName("skillsUpdateChannel")} - Locked for local changes!') else: - self.logInfo(f'![green]({skillName}) - Version {self._skillList[skillName]["installer"]["version"]} in {self.ConfigManager.getAliceConfigByName("skillsUpdateChannel")}') + self.logInfo(f'![green]({skillName}) - Version {installer["installer"]["version"]} in {self.ConfigManager.getAliceConfigByName("skillsUpdateChannel")}') except GithubNotFound: self.logInfo(f'![red](Skill **{skillName}**) is not available on Github. Deprecated or is it a dev skill?') @@ -929,7 +935,7 @@ def getSkillRepository(self, skillName: str, directory: str = None) -> Optional[ """ Returns a dulwich repository object for the given skill :param skillName: - :param directory: where to clone that skill, if not standard directory + :param directory: where to look for that skill, if not standard directory :return: """ @@ -967,7 +973,12 @@ def getGitRemoteSourceUrl(self, skillName: str, doAuth: bool = True) -> str: return url - def installSkill(self, skills: Union[str, List[str]]): + def installSkills(self, skills: Union[str, List[str]]): + """ + Installs the given skills + :param skills: Either a list of skillnames to install or a single skill name + :return: + """ self._busyInstalling.set() if isinstance(skills, str): skills = [skills] @@ -997,18 +1008,16 @@ def installSkill(self, skills: Union[str, List[str]]): self.Commons.runRootSystemCommand(['chmod', '+x', str(directory / scriptReq)]) self.Commons.runRootSystemCommand([str(directory / scriptReq)]) - self.addSkillToDB(installFile['name']) - self._skillList[skillName] = { - 'active' : 1, - 'installer': installFile - } + self.addSkillToDB(skillName) + self._skillList.append(skillName) if installFile.get('rebootAfterInstall', False): self.Commons.runRootSystemCommand('sudo shutdown -r now'.split()) - except: - raise - else: - self._busyInstalling.clear() + break + except Exception as e: + self.logError(f'Error installing skill **{skillName}**: {e}') + + self._busyInstalling.clear() def checkSkillConditions(self, installer: dict = None) -> bool: @@ -1034,7 +1043,7 @@ def checkSkillConditions(self, installer: dict = None) -> bool: elif conditionName == 'skill': for requiredSkill in conditionValue: - if requiredSkill in self._skillList and not self._skillList[requiredSkill]['active']: + if requiredSkill in self._skillList and not self.isSkillActive(skillName=installer['name']): raise SkillNotConditionCompliant(message=notCompliant, skillName=installer['name'], condition=conditionName, conditionValue=conditionValue) elif requiredSkill not in self._skillList: self.logInfo(f'Skill {installer["name"]} has another skill as dependency, adding download') @@ -1044,7 +1053,7 @@ def checkSkillConditions(self, installer: dict = None) -> bool: elif conditionName == 'notSkill': for excludedSkill in conditionValue: author, name = excludedSkill.split('/') - if name in self._skillList and self._skillList[name]['active']: + if name in self._skillList and self.isSkillActive(skillName=installer['name']): raise SkillNotConditionCompliant(message=notCompliant, skillName=installer['name'], condition=conditionName, conditionValue=conditionValue) elif conditionName == 'asrArbitraryCapture': @@ -1103,7 +1112,7 @@ def removeSkill(self, skillName: str): } ) - self._skillList.pop(skillName, None) + self._skillList.remove(skillName) self._activeSkills.pop(skillName, None) self._deactivatedSkills.pop(skillName, None) self._failedSkills.pop(skillName, None) @@ -1340,3 +1349,15 @@ def downloadInstallTicket(self, skillName: str, isUpdate: bool = False) -> bool: def setSkillModified(self, skillName: str, modified: bool): self._skillList[skillName][modified] = modified + + + def isSkillUserModified(self, skillName: str) -> bool: + """ + Checks git status to see if the skill was modified from original online + :param skillName: + :return: + """ + try: + status = git.status(self.getSkillRepository(skillName=skillName)) + except gitErrors.NotGitRepository: + return False diff --git a/core/webApi/model/SkillsApi.py b/core/webApi/model/SkillsApi.py index a71125638..a6a87becd 100644 --- a/core/webApi/model/SkillsApi.py +++ b/core/webApi/model/SkillsApi.py @@ -19,10 +19,9 @@ import json -from pathlib import Path - from flask import Response, jsonify, request from flask_classful import route +from pathlib import Path from core.base.model.GithubCloner import GithubCloner from core.commons import constants @@ -122,7 +121,7 @@ def installSkills(self) -> Response: status = dict() for skill in skills: - self.SkillManager.installSkill(skills=skill) + self.SkillManager.installSkills(skills=skill) status[skill] = 'ok' return jsonify(success=True, status=status) From d3c199a04b91d00c62280bc2d4b15828ecca5e7e Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Tue, 9 Nov 2021 10:49:18 +0100 Subject: [PATCH 014/129] save --- core/base/SkillManager.py | 11 ++++++----- core/base/model/AliceSkill.py | 5 +++++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/core/base/SkillManager.py b/core/base/SkillManager.py index 910d1b5cc..12fe9f389 100644 --- a/core/base/SkillManager.py +++ b/core/base/SkillManager.py @@ -520,6 +520,7 @@ def _startSkill(self, skillName: str) -> dict: else: return dict() elif skillName in self._failedSkills: + self._failedSkills.pop(skillName, None) skillInstance = self.instantiateSkill(skillName=skillName) if skillInstance: self.activeSkills[skillName] = skillInstance @@ -538,7 +539,7 @@ def _startSkill(self, skillName: str) -> dict: try: skillInstance.failedStarting = True except: - self._failedSkills[skillName] = FailedAliceSkill(self._skillList[skillName]['installer']) + self._failedSkills[skillName] = FailedAliceSkill(skillInstance.installer) except SkillStartDelayed: raise except Exception as e: @@ -551,7 +552,7 @@ def _startSkill(self, skillName: str) -> dict: self._activeSkills.pop(skillName, None) self._deactivatedSkills.pop(skillName, None) - self._failedSkills[skillName] = FailedAliceSkill(self._skillList[skillName]['installer']) + self._failedSkills[skillName] = FailedAliceSkill(skillInstance.installer) return skillInstance.supportedIntents @@ -684,7 +685,7 @@ def checkForSkillUpdates(self, skillToCheck: str = None) -> List[str]: replaceBody=[skillName, str(remoteVersion)] ) - if data.get('modified', False): + if self.isSkillUserModified(skillName=skillName): self.allSkills[skillName].updateAvailable = True self.logInfo(f'![blue]({skillName}) - Version {installer["version"]} < {str(remoteVersion)} in {self.ConfigManager.getAliceConfigByName("skillsUpdateChannel")} - Locked for local changes!') continue @@ -697,7 +698,7 @@ def checkForSkillUpdates(self, skillToCheck: str = None) -> List[str]: else: skillsToUpdate.append(skillName) else: - if data.get('modified', False): + if self.isSkillUserModified(skillName=skillName): self.logInfo(f'![blue]({skillName}) - Version {installer["version"]} in {self.ConfigManager.getAliceConfigByName("skillsUpdateChannel")} - Locked for local changes!') else: self.logInfo(f'![green]({skillName}) - Version {installer["installer"]["version"]} in {self.ConfigManager.getAliceConfigByName("skillsUpdateChannel")}') @@ -773,7 +774,7 @@ def _finishInstall(self, skills: dict = None, startSkill: bool = False): typ=UINotificationType.INFO, notification='skillUpdated', key='skillUpdate_{}'.format(skillName), - replaceBody=[skillName, self._skillList[skillName]['installer']['version']] + replaceBody=[skillName, json.loads(self.getSkillInstallFile(skillName=skillName).read_text())['version']] ) else: self.allSkills[skillName].onSkillInstalled(skill=skillName) diff --git a/core/base/model/AliceSkill.py b/core/base/model/AliceSkill.py index 171160573..e4bbb7f28 100644 --- a/core/base/model/AliceSkill.py +++ b/core/base/model/AliceSkill.py @@ -480,6 +480,11 @@ def installFile(self) -> Path: return self._installFile + @property + def installer(self) -> Dict: + return self._installer + + @property def skillPath(self) -> Path: return self._skillPath From d534bd7d6030ed6ff6315ea5e1aeb60b845c5945 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Wed, 10 Nov 2021 17:55:49 +0100 Subject: [PATCH 015/129] Some basic stuff --- core/base/model/Git.py | 140 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100644 core/base/model/Git.py diff --git a/core/base/model/Git.py b/core/base/model/Git.py new file mode 100644 index 000000000..41f900c49 --- /dev/null +++ b/core/base/model/Git.py @@ -0,0 +1,140 @@ +# Copyright (c) 2021 +# +# This file, Git.py, is part of Project Alice. +# +# Project Alice is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see +# +# Last modified: 2021.11.10 at 14:35:51 CET +from __future__ import annotations + +import shutil +import subprocess +from pathlib import Path +from typing import Union + +import requests + + +class PathNotFoundException(Exception): + def __init__(self, path: Path): + super().__init__(f'Path "{path}" does not exist') + +class NotGitRepository(Exception): + def __init__(self, path: Path): + super().__init__(f'Directory "{path}" is not a git repository') + +class AlreadyGitRepository(Exception): + def __init__(self, path: Path): + super().__init__(f'Directory "{path}" is already a git repository') + +class InvalidUrl(Exception): + def __init__(self, url: str): + super().__init__(f'The provided url "{url}" is not valid') + + +class Git: + + def __init__(self, directory: Union[str, Path], makeDir: bool = False, init: bool = False, url: str = '', quiet: bool = True): + if directory and isinstance(directory, str): + directory = Path(directory) + + if not directory.exists() and not makeDir: + raise PathNotFoundException(directory) + + if directory.exists() and not Path(directory, '.git').exists() and not init: + raise NotGitRepository(directory) + + directory.mkdir(parents=True, exist_ok=True) + + if not Path(directory, '.git').exists() and not init: + raise NotGitRepository(directory) + + self.path = directory + self._quiet = quiet + self._url = url + self._tags = set() + self._branches = set() + + if not Path(directory, '.git').exists(): + self.execute(f'git -C {str(directory)} init') + + + tags = self.execute('git tag') + self._tags = set(tags.split('\n')) + branches = self.execute('git branch') + self._branches = set(branches.split('\n')) + + + @classmethod + def clone(cls, url: str, directory: Union[str, Path], branch: str = 'master', makeDir: bool = False, force: bool = False, quiet: bool = True) -> Git: + if directory and isinstance(directory, str): + directory = Path(directory) + + response = requests.get(url) + if response.status_code != 200: + raise InvalidUrl(url) + + if not directory.exists() and not makeDir: + raise PathNotFoundException(directory) + + if Path(directory, '.git').exists(): + if not force: + raise AlreadyGitRepository(directory) + else: + shutil.rmtree(str(directory), ignore_errors=True) + + directory.mkdir(parents=True, exist_ok=True) + cmd = f'git clone {url} {str(directory)} --branch {branch} --recurse-submodules' + if quiet: + cmd = f'{cmd} --quiet' + subprocess.run(cmd) + return Git(directory=directory, url=url, quiet=quiet) + + + def checkout(self, branch: str = 'master', tag: str = '', force: bool = False): + if tag: + target = f'tags/{tag} -B Branch_{tag}' + else: + target = branch + + self.execute(f'git -C {str(self.path)} checkout {target} --recurse-submodules') + + + def execute(self, command: str) -> str: + if self._quiet: + command = f'{command} --quiet' + result = subprocess.run(command.split(), capture_output=True, text=True) + return result.stdout.strip() + + + def status(self) -> Status: + return Status(directory=self.path) + + + def isDirty(self) -> bool: + status = self.status() + return status.isDirty() + + +class Status: + + def __init__(self, directory: Union[str, Path]): + if directory and isinstance(directory, str): + directory = Path(directory) + + self._status = subprocess.run(f'git -C {str(directory)} status'.split(), capture_output=True, text=True).stdout.strip() + + + def isDirty(self): + return 'working tree clean' not in self._status From 651dec0f37b4d264738adcf85fe20984d715172e Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Thu, 11 Nov 2021 11:12:11 +0100 Subject: [PATCH 016/129] git stash, stash list, pull, destroy --- core/base/model/Git.py | 86 ++++++++++++++++++++++++++++++++---------- 1 file changed, 67 insertions(+), 19 deletions(-) diff --git a/core/base/model/Git.py b/core/base/model/Git.py index 41f900c49..e147fb8be 100644 --- a/core/base/model/Git.py +++ b/core/base/model/Git.py @@ -21,7 +21,7 @@ import shutil import subprocess from pathlib import Path -from typing import Union +from typing import List, Union import requests @@ -30,19 +30,27 @@ class PathNotFoundException(Exception): def __init__(self, path: Path): super().__init__(f'Path "{path}" does not exist') + class NotGitRepository(Exception): def __init__(self, path: Path): super().__init__(f'Directory "{path}" is not a git repository') + class AlreadyGitRepository(Exception): def __init__(self, path: Path): super().__init__(f'Directory "{path}" is already a git repository') + class InvalidUrl(Exception): def __init__(self, url: str): super().__init__(f'The provided url "{url}" is not valid') +class DirtyRepository(Exception): + def __init__(self): + super().__init__(f'The repository is dirty. Either use the force option or stash your changes before trying again') + + class Git: def __init__(self, directory: Union[str, Path], makeDir: bool = False, init: bool = False, url: str = '', quiet: bool = True): @@ -60,20 +68,17 @@ def __init__(self, directory: Union[str, Path], makeDir: bool = False, init: boo if not Path(directory, '.git').exists() and not init: raise NotGitRepository(directory) - self.path = directory - self._quiet = quiet - self._url = url - self._tags = set() - self._branches = set() + self.path = directory + self._quiet = quiet + self._url = url if not Path(directory, '.git').exists(): - self.execute(f'git -C {str(directory)} init') + self.execute(f'git init') - - tags = self.execute('git tag') - self._tags = set(tags.split('\n')) - branches = self.execute('git branch') - self._branches = set(branches.split('\n')) + tags = self.execute('git tag') + self.tags = set(tags.split('\n')) + branches = self.execute('git branch') + self.branches = set(branches.split('\n')) @classmethod @@ -108,14 +113,13 @@ def checkout(self, branch: str = 'master', tag: str = '', force: bool = False): else: target = branch - self.execute(f'git -C {str(self.path)} checkout {target} --recurse-submodules') + if self.isDirty(): + if not force: + raise DirtyRepository() + else: + self.stash() - - def execute(self, command: str) -> str: - if self._quiet: - command = f'{command} --quiet' - result = subprocess.run(command.split(), capture_output=True, text=True) - return result.stdout.strip() + self.execute(f'git checkout {target} --recurse-submodules') def status(self) -> Status: @@ -127,6 +131,50 @@ def isDirty(self) -> bool: return status.isDirty() + def listStash(self) -> List[str]: + result = self.execute(f'git stash list') + return result.split('\n') + + + def stash(self) -> int: + self.execute(f'git stash push') + return len(self.listStash()) - 1 + + + def dropStash(self, index: Union[int, str] = -1) -> List[str]: + if index == 'all': + self.execute(f'git stash clear') + return list() + else: + self.execute(f'git stash drop {index}') + return self.listStash() + + + def pull(self, force: bool = False): + if self.isDirty(): + if not force: + raise DirtyRepository() + else: + self.stash() + + self.execute(f'git pull') + + + def destroy(self): + shutil.rmtree(self.path, ignore_errors=True) + + + def execute(self, command: str) -> str: + if not command.startswith('git -C'): + command = command.replace('git', f'git -C {str(self.path)}', 1) + + if self._quiet: + command = f'{command} --quiet' + + result = subprocess.run(command.split(), capture_output=True, text=True) + return result.stdout.strip() + + class Status: def __init__(self, directory: Union[str, Path]): From 8cd908f62ebd58c8364ba0c2f8521e01d56ab955 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Thu, 11 Nov 2021 11:45:56 +0100 Subject: [PATCH 017/129] git clean and restore --- core/base/model/Git.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/core/base/model/Git.py b/core/base/model/Git.py index e147fb8be..be8ebce97 100644 --- a/core/base/model/Git.py +++ b/core/base/model/Git.py @@ -117,7 +117,8 @@ def checkout(self, branch: str = 'master', tag: str = '', force: bool = False): if not force: raise DirtyRepository() else: - self.stash() + self.restore() + self.clean() self.execute(f'git checkout {target} --recurse-submodules') @@ -137,7 +138,7 @@ def listStash(self) -> List[str]: def stash(self) -> int: - self.execute(f'git stash push') + self.execute(f'git stash push {str(self.path)}/') return len(self.listStash()) - 1 @@ -155,11 +156,28 @@ def pull(self, force: bool = False): if not force: raise DirtyRepository() else: - self.stash() + self.restore() + self.clean() self.execute(f'git pull') + def clean(self, removeUntrackedFiles: bool = True, removeUntrackedDirectory: bool = True): + options = '' + if removeUntrackedFiles: + options += 'f' + if removeUntrackedDirectory: + options += 'd' + if options: + options = f'-{options}' + + self.execute(f'git clean {options}') + + + def restore(self): + self.execute(f'git restore {str(self.path)}') + + def destroy(self): shutil.rmtree(self.path, ignore_errors=True) From 124e3339633e6870f5a7f78e0a7c082a4dbe851a Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Thu, 11 Nov 2021 14:59:05 +0100 Subject: [PATCH 018/129] isRepository, revert and fix potential permission issues on git dir --- core/base/model/Git.py | 79 +++++++++++++++++++++++++++++++++--------- 1 file changed, 63 insertions(+), 16 deletions(-) diff --git a/core/base/model/Git.py b/core/base/model/Git.py index be8ebce97..97c5c6d1d 100644 --- a/core/base/model/Git.py +++ b/core/base/model/Git.py @@ -18,10 +18,12 @@ # Last modified: 2021.11.10 at 14:35:51 CET from __future__ import annotations +import os import shutil +import stat import subprocess from pathlib import Path -from typing import List, Union +from typing import Callable, List, Union import requests @@ -54,7 +56,7 @@ def __init__(self): class Git: def __init__(self, directory: Union[str, Path], makeDir: bool = False, init: bool = False, url: str = '', quiet: bool = True): - if directory and isinstance(directory, str): + if isinstance(directory, str): directory = Path(directory) if not directory.exists() and not makeDir: @@ -65,16 +67,20 @@ def __init__(self, directory: Union[str, Path], makeDir: bool = False, init: boo directory.mkdir(parents=True, exist_ok=True) - if not Path(directory, '.git').exists() and not init: - raise NotGitRepository(directory) + isRepository = self.isRepository(directory=directory) + if init: + if not isRepository: + self.execute(f'git init') + else: + raise AlreadyGitRepository + else: + if not isRepository: + raise NotGitRepository self.path = directory self._quiet = quiet self._url = url - if not Path(directory, '.git').exists(): - self.execute(f'git init') - tags = self.execute('git tag') self.tags = set(tags.split('\n')) branches = self.execute('git branch') @@ -83,7 +89,7 @@ def __init__(self, directory: Union[str, Path], makeDir: bool = False, init: boo @classmethod def clone(cls, url: str, directory: Union[str, Path], branch: str = 'master', makeDir: bool = False, force: bool = False, quiet: bool = True) -> Git: - if directory and isinstance(directory, str): + if isinstance(directory, str): directory = Path(directory) response = requests.get(url) @@ -93,11 +99,11 @@ def clone(cls, url: str, directory: Union[str, Path], branch: str = 'master', ma if not directory.exists() and not makeDir: raise PathNotFoundException(directory) - if Path(directory, '.git').exists(): + if cls.isRepository(directory=directory): if not force: raise AlreadyGitRepository(directory) else: - shutil.rmtree(str(directory), ignore_errors=True) + shutil.rmtree(str(directory), onerror=cls.fixPermissions) directory.mkdir(parents=True, exist_ok=True) cmd = f'git clone {url} {str(directory)} --branch {branch} --recurse-submodules' @@ -107,6 +113,34 @@ def clone(cls, url: str, directory: Union[str, Path], branch: str = 'master', ma return Git(directory=directory, url=url, quiet=quiet) + @staticmethod + def isRepository(directory: Union[str, Path]) -> bool: + if directory and isinstance(directory, str): + directory = Path(directory) + + gitDir = directory / '.git' + if not gitDir.exists(): + return False + + expected = [ + 'hooks', + 'info', + 'logs', + 'objects', + 'refs', + 'config', + 'description', + 'HEAD', + 'index', + 'packed-refs' + ] + + for item in expected: + if not Path(gitDir, item).exists(): + return False + return True + + def checkout(self, branch: str = 'master', tag: str = '', force: bool = False): if tag: target = f'tags/{tag} -B Branch_{tag}' @@ -117,8 +151,7 @@ def checkout(self, branch: str = 'master', tag: str = '', force: bool = False): if not force: raise DirtyRepository() else: - self.restore() - self.clean() + self.revert() self.execute(f'git checkout {target} --recurse-submodules') @@ -132,6 +165,12 @@ def isDirty(self) -> bool: return status.isDirty() + def revert(self): + self.restore() + self.clean() + self.execute('git checkout HEAD') + + def listStash(self) -> List[str]: result = self.execute(f'git stash list') return result.split('\n') @@ -156,8 +195,7 @@ def pull(self, force: bool = False): if not force: raise DirtyRepository() else: - self.restore() - self.clean() + self.revert() self.execute(f'git pull') @@ -179,7 +217,16 @@ def restore(self): def destroy(self): - shutil.rmtree(self.path, ignore_errors=True) + shutil.rmtree(self.path, onerror=self.fixPermissions) + + + @staticmethod + def fixPermissions(func: Callable, path: Path, *_args): + if not os.access(path, os.W_OK): + os.chmod(path, stat.S_IWUSR) + func(path) + else: + raise # NOSONAR def execute(self, command: str) -> str: @@ -196,7 +243,7 @@ def execute(self, command: str) -> str: class Status: def __init__(self, directory: Union[str, Path]): - if directory and isinstance(directory, str): + if isinstance(directory, str): directory = Path(directory) self._status = subprocess.run(f'git -C {str(directory)} status'.split(), capture_output=True, text=True).stdout.strip() From fbcbaf0164142b3850063c4383cd06f602f65816 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Thu, 11 Nov 2021 15:29:50 +0100 Subject: [PATCH 019/129] git add, commit, push and repository.file that returns a given filepath --- core/base/model/Git.py | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/core/base/model/Git.py b/core/base/model/Git.py index 97c5c6d1d..dce7017ea 100644 --- a/core/base/model/Git.py +++ b/core/base/model/Git.py @@ -79,7 +79,7 @@ def __init__(self, directory: Union[str, Path], makeDir: bool = False, init: boo self.path = directory self._quiet = quiet - self._url = url + self.url = url tags = self.execute('git tag') self.tags = set(tags.split('\n')) @@ -240,6 +240,37 @@ def execute(self, command: str) -> str: return result.stdout.strip() + def add(self): + self.execute('git add --all') + + + def commit(self, message: str = 'Commit by ProjectAliceBot'): + self.execute(f'git commit -m "{message}"') + + + def push(self, repository: str = None): + if not repository: + repository = self.url + self.execute(f'git push --repo={repository} origin') + + + def file(self, filePath: Union[str, Path]) -> Path: + if isinstance(filePath, str): + filePath = Path(filePath) + + return self.path / filePath + + + @property + def quiet(self) -> bool: + return self._quiet + + + @quiet.setter + def quiet(self, value: bool): + self._quiet = value + + class Status: def __init__(self, directory: Union[str, Path]): From a0d10e5694cb7e749a280ac58f366a63702381ed Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Thu, 11 Nov 2021 15:33:29 +0100 Subject: [PATCH 020/129] dfx option :-) --- core/base/model/Git.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/base/model/Git.py b/core/base/model/Git.py index dce7017ea..200089aba 100644 --- a/core/base/model/Git.py +++ b/core/base/model/Git.py @@ -200,12 +200,14 @@ def pull(self, force: bool = False): self.execute(f'git pull') - def clean(self, removeUntrackedFiles: bool = True, removeUntrackedDirectory: bool = True): + def clean(self, removeUntrackedFiles: bool = True, removeUntrackedDirectory: bool = True, removeIgnored: bool = False): options = '' if removeUntrackedFiles: options += 'f' if removeUntrackedDirectory: options += 'd' + if removeIgnored: + options += 'x' if options: options = f'-{options}' From 8337ecb6af744e2eaf032e4598f925405461f35c Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Thu, 11 Nov 2021 15:37:43 +0100 Subject: [PATCH 021/129] autoadd --- core/base/model/Git.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/core/base/model/Git.py b/core/base/model/Git.py index 200089aba..fffb31841 100644 --- a/core/base/model/Git.py +++ b/core/base/model/Git.py @@ -246,8 +246,11 @@ def add(self): self.execute('git add --all') - def commit(self, message: str = 'Commit by ProjectAliceBot'): - self.execute(f'git commit -m "{message}"') + def commit(self, message: str = 'Commit by ProjectAliceBot', autoAdd: bool = False): + cmd = f'git commit -m "{message}"' + if autoAdd: + cmd += ' --all' + self.execute(cmd) def push(self, repository: str = None): From ba9c3cacdb1d7ef254fd1e7bb5b1e94b669500e1 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Thu, 11 Nov 2021 18:31:43 +0100 Subject: [PATCH 022/129] fix skill manager already done work, replaced dulwich with Git --- core/base/SkillManager.py | 60 +++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/core/base/SkillManager.py b/core/base/SkillManager.py index 12fe9f389..85cbbc2ea 100644 --- a/core/base/SkillManager.py +++ b/core/base/SkillManager.py @@ -19,23 +19,18 @@ import getpass -import traceback - import importlib import json -import requests -import shutil import threading -from dulwich import errors as gitErrors, porcelain as git -from dulwich.repo import Repo -from pathlib import Path -from typing import Dict, List, Optional, Union +import traceback +from typing import Dict, Optional from core.ProjectAliceExceptions import AccessLevelTooLow, GithubNotFound, SkillInstanceFailed, SkillNotConditionCompliant, SkillStartDelayed, SkillStartingFailed from core.base.SuperManager import SuperManager from core.base.model import Intent from core.base.model.AliceSkill import AliceSkill from core.base.model.FailedAliceSkill import FailedAliceSkill +from core.base.model.Git import * from core.base.model.GithubCloner import GithubCloner from core.base.model.Manager import Manager from core.base.model.Version import Version @@ -153,7 +148,8 @@ def _loadSkills(self) -> List[str]: data = list() for skill in skills: try: - Path(self.Commons.rootDir(), f'skills/{skill["skillName"]}').read_text() # Just test is the directory exists + if not Path(self.Commons.rootDir(), f'skills/{skill["skillName"]}').exists(): + raise Exception('Skill directory not existing') data.append(skill['skillName']) except Exception as e: self.logError(f'Error loading skill **{skill["skillName"]}**: {e}') @@ -180,10 +176,7 @@ def updateSkills(self, skills: Union[str, List[str]], withSkillRestart: bool = T try: repository = self.getSkillRepository(skillName=skillName) - git.stash_push(repository) - git.reset(repository, mode='hard') - self.Commons.runSystemCommand(f'git -C {str(repository.path)} clean -dfx') - git.pull(repo=repository, refspecs=self.SkillStoreManager.getSkillUpdateTag(skillName=skillName)) + repository.checkout(tag=self.SkillStoreManager.getSkillUpdateTag(skillName=skillName), force=True) except Exception as e: self.logError(f'Error updating skill **{skillName}** : {e}') continue @@ -223,9 +216,9 @@ def downloadSkills(self, skills: Union[str, List[str]]): source = self.getGitRemoteSourceUrl(skillName=skillName, doAuth=False) repository = self.getSkillRepository(skillName=skillName) if not repository: - repository = git.clone(source=source, target=self.getSkillDirectory(skillName=skillName), checkout=True) + repository = Git.clone(url=source, directory=self.getSkillDirectory(skillName=skillName), makeDir=True) - git.pull(repo=repository, refspecs=self.SkillStoreManager.getSkillUpdateTag(skillName=skillName)) + repository.checkout(tag=self.SkillStoreManager.getSkillUpdateTag(skillName=skillName)) except GithubNotFound: if skillName in self.NEEDED_SKILLS: self._busyInstalling.clear() @@ -407,7 +400,7 @@ def initSkills(self, onlyInit: str = '', reload: bool = False): try: skillActiveState = self.isSkillActive(skillName=skillName) - if skillActiveState: + if not skillActiveState: if skillName in self.NEEDED_SKILLS: self.logFatal(f"Skill {skillName} marked as disabled but it shouldn't be") return @@ -561,10 +554,11 @@ def isSkillActive(self, skillName: str) -> bool: if skillName in self._activeSkills: return self._activeSkills[skillName].active elif skillName in self._skillList: - row = self.databaseFetch(tableName=self.DBTAB_SKILLS, values={'skillName': skillName}) + # noinspection SqlResolve + row = self.databaseFetch(tableName=self.DBTAB_SKILLS, query='SELECT active FROM :__table__ WHERE skillName = :skillName LIMIT 1', values={'skillName': skillName}) if not row: return False - return row[0]['active'] == 1 + return int(row[0]['active']) == 1 return False @@ -701,7 +695,7 @@ def checkForSkillUpdates(self, skillToCheck: str = None) -> List[str]: if self.isSkillUserModified(skillName=skillName): self.logInfo(f'![blue]({skillName}) - Version {installer["version"]} in {self.ConfigManager.getAliceConfigByName("skillsUpdateChannel")} - Locked for local changes!') else: - self.logInfo(f'![green]({skillName}) - Version {installer["installer"]["version"]} in {self.ConfigManager.getAliceConfigByName("skillsUpdateChannel")}') + self.logInfo(f'![green]({skillName}) - Version {installer["version"]} in {self.ConfigManager.getAliceConfigByName("skillsUpdateChannel")}') except GithubNotFound: self.logInfo(f'![red](Skill **{skillName}**) is not available on Github. Deprecated or is it a dev skill?') @@ -932,9 +926,9 @@ def getSkillDirectory(self, skillName: str) -> Path: return Path(self.Commons.rootDir()) / 'skills' / skillName - def getSkillRepository(self, skillName: str, directory: str = None) -> Optional[Repo]: + def getSkillRepository(self, skillName: str, directory: str = None) -> Optional[Git]: """ - Returns a dulwich repository object for the given skill + Returns a Git object for the given skill :param skillName: :param directory: where to look for that skill, if not standard directory :return: @@ -944,8 +938,8 @@ def getSkillRepository(self, skillName: str, directory: str = None) -> Optional[ directory = self.getSkillDirectory(skillName=skillName) try: - return Repo(directory) - except gitErrors.NotGitRepository: + return Git(directory=directory) + except: return None @@ -990,8 +984,7 @@ def installSkills(self, skills: Union[str, List[str]]): if not repository: self.downloadSkills(skills=skillName) - directory = repository.path - installFile = json.loads(Path(directory, f'{skillName}.install').read_text()) + installFile = json.loads(repository.file(f'{skillName}.install').read_text()) pipReqs = installFile.get('pipRequirements', list()) sysReqs = installFile.get('systemRequirements', list()) scriptReq = installFile.get('script') @@ -1006,8 +999,12 @@ def installSkills(self, skills: Union[str, List[str]]): if scriptReq: self.logInfo('Running post install script') - self.Commons.runRootSystemCommand(['chmod', '+x', str(directory / scriptReq)]) - self.Commons.runRootSystemCommand([str(directory / scriptReq)]) + req = repository.file(scriptReq) + if not req: + self.logWarning(f'Missing post install script **{str(req)}** as declared in install file') + continue + self.Commons.runRootSystemCommand(['chmod', '+x', str(req)]) + self.Commons.runRootSystemCommand([str(req)]) self.addSkillToDB(skillName) self._skillList.append(skillName) @@ -1329,6 +1326,7 @@ def uploadSkillToGithub(self, skillName: str, skillDesc: str) -> bool: return False + @deprecated def downloadInstallTicket(self, skillName: str, isUpdate: bool = False) -> bool: try: tmpFile = Path(self.Commons.rootDir(), f'system/skillInstallTickets/{skillName}.install') @@ -1348,8 +1346,9 @@ def downloadInstallTicket(self, skillName: str, isUpdate: bool = False) -> bool: return False + @deprecated def setSkillModified(self, skillName: str, modified: bool): - self._skillList[skillName][modified] = modified + return def isSkillUserModified(self, skillName: str) -> bool: @@ -1359,6 +1358,7 @@ def isSkillUserModified(self, skillName: str) -> bool: :return: """ try: - status = git.status(self.getSkillRepository(skillName=skillName)) - except gitErrors.NotGitRepository: + repository = self.getSkillRepository(skillName=skillName) + return repository.isDirty() + except: return False From f9699a7f718671b26b11c909858f6a738b15d2b8 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Fri, 12 Nov 2021 05:10:03 +0100 Subject: [PATCH 023/129] can't finish now --- core/base/SkillManager.py | 56 +++++++++------------------------------ 1 file changed, 13 insertions(+), 43 deletions(-) diff --git a/core/base/SkillManager.py b/core/base/SkillManager.py index 85cbbc2ea..a269b5696 100644 --- a/core/base/SkillManager.py +++ b/core/base/SkillManager.py @@ -853,49 +853,6 @@ def _installSkillTickets(self, skills: list) -> dict: raise - if updating: - try: - self._skillList[skillName].onStop() - except Exception as e: - self.logError(f'Error stopping "{skillName}" for update: {e}') - raise - - git.stash_push(skillRepository) - git.reset(skillRepository, mode='hard') - - try: - git.pull(skillRepository, refspecs=self.SkillStoreManager.getSkillUpdateTag(skillName)) - except: - pass - - # - # try: - # gitCloner.cloneSkill() - # self.logInfo('Skill successfully downloaded') - # self._installSkill(res) - # skillsToBoot[skillName] = { - # 'update': updating - # } - # except (GithubTokenFailed, GithubRateLimit): - # self.logError('Failed cloning skill') - # raise - # except GithubNotFound: - # if self.ConfigManager.getAliceConfigByName('devMode'): - # if not Path(f'{self.Commons.rootDir}/skills/{skillName}').exists() or not \ - # Path(f'{self.Commons.rootDir}/skills/{skillName}/{skillName.py}').exists() or not \ - # Path(f'{self.Commons.rootDir}/skills/{skillName}/dialogTemplate').exists() or not \ - # Path(f'{self.Commons.rootDir}/skills/{skillName}/talks').exists(): - # self.logWarning(f'Skill "{skillName}" cannot be installed in dev mode due to missing base files') - # else: - # self._installSkill(res) - # skillsToBoot[skillName] = { - # 'update': updating - # } - # continue - # else: - # self.logWarning(f'Skill "{skillName}" is not available on Github, cannot install') - # raise - except SkillNotConditionCompliant as e: self.logInfo(f'Skill "{skillName}" does not comply to "{e.condition}" condition, required "{e.conditionValue}"') if res.exists(): @@ -989,6 +946,8 @@ def installSkills(self, skills: Union[str, List[str]]): sysReqs = installFile.get('systemRequirements', list()) scriptReq = installFile.get('script') + self.checkSkillConditions(installFile) + for requirement in pipReqs: self.logInfo(f'Installing pip requirement: {requirement}') self.Commons.runSystemCommand(['./venv/bin/pip3', 'install', requirement]) @@ -1012,8 +971,19 @@ def installSkills(self, skills: Union[str, List[str]]): if installFile.get('rebootAfterInstall', False): self.Commons.runRootSystemCommand('sudo shutdown -r now'.split()) break + except SkillNotConditionCompliant as e: + self.broadcast( + method=constants.EVENT_SKILL_INSTALL_FAILED, + exceptions=self._name, + skill=skillName + ) except Exception as e: self.logError(f'Error installing skill **{skillName}**: {e}') + self.broadcast( + method=constants.EVENT_SKILL_INSTALL_FAILED, + exceptions=self._name, + skill=skillName + ) self._busyInstalling.clear() From 38cf55457c87bc4e55026e736e33ab8f9c492b6c Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Fri, 12 Nov 2021 19:11:30 +0100 Subject: [PATCH 024/129] Get installer information from raw.github before downloading, to be able to check the conditions before anything happens --- core/base/SkillManager.py | 51 ++++++++++++++++++++++++++++++--------- 1 file changed, 39 insertions(+), 12 deletions(-) diff --git a/core/base/SkillManager.py b/core/base/SkillManager.py index a269b5696..7545f990c 100644 --- a/core/base/SkillManager.py +++ b/core/base/SkillManager.py @@ -23,6 +23,7 @@ import json import threading import traceback +from contextlib import suppress from typing import Dict, Optional from core.ProjectAliceExceptions import AccessLevelTooLow, GithubNotFound, SkillInstanceFailed, SkillNotConditionCompliant, SkillStartDelayed, SkillStartingFailed @@ -212,13 +213,23 @@ def downloadSkills(self, skills: Union[str, List[str]]): skills = [skills] for skillName in skills: + installFile = dict() try: + tag = self.SkillStoreManager.getSkillUpdateTag(skillName=skillName) + + response = requests.get(f'{constants.GITHUB_RAW_URL}/skill_{skillName}/{tag}/{skillName}.install') + if response.status_code != 200: + raise GithubNotFound + + installFile = json.loads(response.json()) + self.checkSkillConditions(installer=installFile) + source = self.getGitRemoteSourceUrl(skillName=skillName, doAuth=False) repository = self.getSkillRepository(skillName=skillName) if not repository: repository = Git.clone(url=source, directory=self.getSkillDirectory(skillName=skillName), makeDir=True) - repository.checkout(tag=self.SkillStoreManager.getSkillUpdateTag(skillName=skillName)) + repository.checkout(tag=tag) except GithubNotFound: if skillName in self.NEEDED_SKILLS: self._busyInstalling.clear() @@ -227,16 +238,37 @@ def downloadSkills(self, skills: Union[str, List[str]]): else: self.logError(f'Skill "{skillName}" not found in released skills') continue + except SkillNotConditionCompliant as e: + if self.notCompliantSkill(skillName=skillName, exception=e): + self._failedSkills[skillName] = FailedAliceSkill(installFile) + continue + else: + return except Exception as e: if skillName in self.NEEDED_SKILLS: self._busyInstalling.clear() - self.logFatal(f'Error downloading skill **{skillName}** but skill is required, cannot continue: {e}') + self.logFatal(f'Error downloading skill **{skillName}** and it is required, cannot continue: {e}') return else: self.logError(f'Error downloading skill "{skillName}": {e}') continue + def notCompliantSkill(self, skillName: str, exception: SkillNotConditionCompliant) -> bool: + """ + Print out the fact a skill is not compliant and return false if Alice cannot continue as it's a needed skill + :param skillName + :param exception: + :return: + """ + if skillName in self.NEEDED_SKILLS: + self.logFatal(f'Skill {skillName} does not comply to "{exception.condition}" condition, offers only "{exception.conditionValue}". The skill is required to continue') + return False + else: + self.logInfo(f'Skill {skillName} does not comply to "{exception.condition}" condition, offers only "{exception.conditionValue}"') + return True + + def loadSkillsFromDB(self) -> List: return self.databaseFetch(tableName='skills') @@ -425,13 +457,11 @@ def initSkills(self, onlyInit: str = '', reload: bool = False): else: self._failedSkills[skillName] = FailedAliceSkill(installFile) except SkillNotConditionCompliant as e: - if skillName in self.NEEDED_SKILLS: - self.logFatal(f'Skill {skillName} does not comply to "{e.condition}" condition, offers only "{e.conditionValue}". The skill is required to continue') - return - else: - self.logInfo(f'Skill {skillName} does not comply to "{e.condition}" condition, offers only "{e.conditionValue}"') + if self.notCompliantSkill(skillName=skillName, exception=e): self._failedSkills[skillName] = FailedAliceSkill(installFile) continue + else: + return except Exception as e: self.logWarning(f'Something went wrong loading skill {skillName}: {e}') self._failedSkills[skillName] = FailedAliceSkill(installFile) @@ -749,11 +779,8 @@ def _finishInstall(self, skills: dict = None, startSkill: bool = False): self.initSkills(onlyInit=skillName, reload=info['update']) self.ConfigManager.loadCheckAndUpdateSkillConfigurations(skillToLoad=skillName) - try: + with suppress(SkillStartDelayed): self._startSkill(skillName) - except SkillStartDelayed: - # The skill start was delayed - pass if info['update']: self.allSkills[skillName].onSkillUpdated(skill=skillName) @@ -971,7 +998,7 @@ def installSkills(self, skills: Union[str, List[str]]): if installFile.get('rebootAfterInstall', False): self.Commons.runRootSystemCommand('sudo shutdown -r now'.split()) break - except SkillNotConditionCompliant as e: + except SkillNotConditionCompliant: self.broadcast( method=constants.EVENT_SKILL_INSTALL_FAILED, exceptions=self._name, From 0857e0d22d6de86c48fe0992eb19aa0ff6f4534a Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Fri, 12 Nov 2021 19:17:58 +0100 Subject: [PATCH 025/129] Init the repo after downloading the skill --- core/base/SkillManager.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/core/base/SkillManager.py b/core/base/SkillManager.py index 7545f990c..538958367 100644 --- a/core/base/SkillManager.py +++ b/core/base/SkillManager.py @@ -203,15 +203,17 @@ def updateSkills(self, skills: Union[str, List[str]], withSkillRestart: bool = T self._busyInstalling.clear() - def downloadSkills(self, skills: Union[str, List[str]]): + def downloadSkills(self, skills: Union[str, List[str]]) -> Optional[Dict]: """ Clones skills :param skills: - :return: + :return: Dict: a dict of created repositories """ if isinstance(skills, str): skills = [skills] + repositories = dict() + for skillName in skills: installFile = dict() try: @@ -230,28 +232,29 @@ def downloadSkills(self, skills: Union[str, List[str]]): repository = Git.clone(url=source, directory=self.getSkillDirectory(skillName=skillName), makeDir=True) repository.checkout(tag=tag) + repositories[skillName] = repository except GithubNotFound: if skillName in self.NEEDED_SKILLS: self._busyInstalling.clear() self.logFatal(f"Skill **{skillName}** is required but wasn't found in released skills, cannot continue") - return + return None else: self.logError(f'Skill "{skillName}" not found in released skills') continue except SkillNotConditionCompliant as e: if self.notCompliantSkill(skillName=skillName, exception=e): - self._failedSkills[skillName] = FailedAliceSkill(installFile) continue else: - return + return None except Exception as e: if skillName in self.NEEDED_SKILLS: self._busyInstalling.clear() self.logFatal(f'Error downloading skill **{skillName}** and it is required, cannot continue: {e}') - return + return None else: self.logError(f'Error downloading skill "{skillName}": {e}') continue + return repositories def notCompliantSkill(self, skillName: str, exception: SkillNotConditionCompliant) -> bool: @@ -955,7 +958,7 @@ def getGitRemoteSourceUrl(self, skillName: str, doAuth: bool = True) -> str: def installSkills(self, skills: Union[str, List[str]]): """ Installs the given skills - :param skills: Either a list of skillnames to install or a single skill name + :param skills: Either a list of skill names to install or a single skill name :return: """ self._busyInstalling.set() @@ -966,7 +969,11 @@ def installSkills(self, skills: Union[str, List[str]]): try: repository = self.getSkillRepository(skillName=skillName) if not repository: - self.downloadSkills(skills=skillName) + repositories = self.downloadSkills(skills=skillName) + repository = repositories.get(skillName, None) + + if not repository: + raise Exception(f'Failed downloading skill **{skillName}** for some unknown reason') installFile = json.loads(repository.file(f'{skillName}.install').read_text()) pipReqs = installFile.get('pipRequirements', list()) From 38d5c1957d295f3cbc8d21e302948effe8ecc5d7 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Fri, 12 Nov 2021 19:27:23 +0100 Subject: [PATCH 026/129] better flow baby --- core/base/SkillManager.py | 34 ++++++++++++++----- .../LanguageManager/notifications.json | 12 +++++++ 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/core/base/SkillManager.py b/core/base/SkillManager.py index 538958367..5f1552c90 100644 --- a/core/base/SkillManager.py +++ b/core/base/SkillManager.py @@ -182,7 +182,8 @@ def updateSkills(self, skills: Union[str, List[str]], withSkillRestart: bool = T self.logError(f'Error updating skill **{skillName}** : {e}') continue - self.allSkills[skillName].onSkillUpdated(skill=skillName) + self.broadcast(constants.EVENT_SKILL_UPDATED, propagateToSkills=True, skill=skillName) + #self.allSkills[skillName].onSkillUpdated(skill=skillName) # Not sure why this was here self.MqttManager.mqttBroadcast( topic=constants.TOPIC_SKILL_UPDATED, payload={ @@ -190,13 +191,6 @@ def updateSkills(self, skills: Union[str, List[str]], withSkillRestart: bool = T } ) - self.WebUINotificationManager.newNotification( - typ=UINotificationType.INFO, - notification='skillUpdated', - key=f'skillUpdate_{skillName}', - replaceBody=[skillName, json.loads(self.getSkillInstallFile(skillName=skillName).read_text())['version']] - ) - if withSkillRestart: self._startSkill(skillName=skillName) @@ -1018,10 +1012,34 @@ def installSkills(self, skills: Union[str, List[str]]): exceptions=self._name, skill=skillName ) + else: + self.broadcast( + method=constants.EVENT_SKILL_INSTALLED, + exceptions=[constants.DUMMY], + skill=skillName + ) self._busyInstalling.clear() + def onSkillInstalled(self, skill: str): + self.WebUINotificationManager.newNotification( + typ=UINotificationType.INFO, + notification='skillInstalled', + key=f'skillUpdate_{skill}', + replaceBody=[skill] + ) + + + def onSkillUpdated(self, skill: str): + self.WebUINotificationManager.newNotification( + typ=UINotificationType.INFO, + notification='skillUpdated', + key=f'skillUpdate_{skill}', + replaceBody=[skill, json.loads(self.getSkillInstallFile(skillName=skill).read_text())['version']] + ) + + def checkSkillConditions(self, installer: dict = None) -> bool: conditions = { 'aliceMinVersion': installer['aliceMinVersion'], diff --git a/system/manager/LanguageManager/notifications.json b/system/manager/LanguageManager/notifications.json index d50f24994..663d5a6ff 100644 --- a/system/manager/LanguageManager/notifications.json +++ b/system/manager/LanguageManager/notifications.json @@ -59,6 +59,18 @@ "de": "Der Skill \"{}\" wurde auf Version {} aktualisiert." } }, + "skillInstalled": { + "title": { + "en": "New skill", + "fr": "Nouveau skill", + "de": "Neue Skill" + }, + "body": { + "en": "The skill \"{}\" was just installed", + "fr": "Le skill \"{}\" vient d'être installé", + "de": "Der Skill \"{}\" wurde gerade installiert." + } + }, "aliceUpdated": { "title": { "en": "Project Alice update", From 0fa4798863c0783850e7a9e2acfe89db93c2ad52 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Sat, 13 Nov 2021 06:06:44 +0100 Subject: [PATCH 027/129] will continue at work --- core/base/SkillManager.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/core/base/SkillManager.py b/core/base/SkillManager.py index 5f1552c90..6a0cf6893 100644 --- a/core/base/SkillManager.py +++ b/core/base/SkillManager.py @@ -209,7 +209,6 @@ def downloadSkills(self, skills: Union[str, List[str]]) -> Optional[Dict]: repositories = dict() for skillName in skills: - installFile = dict() try: tag = self.SkillStoreManager.getSkillUpdateTag(skillName=skillName) @@ -239,6 +238,7 @@ def downloadSkills(self, skills: Union[str, List[str]]) -> Optional[Dict]: if self.notCompliantSkill(skillName=skillName, exception=e): continue else: + self._busyInstalling.clear() return None except Exception as e: if skillName in self.NEEDED_SKILLS: @@ -320,12 +320,6 @@ def onAssistantInstalled(self, **kwargs): self._activeSkills[skillName].onBooted() - self.broadcast( - method=constants.EVENT_SKILL_UPDATED if skill['update'] else constants.EVENT_SKILL_INSTALLED, - exceptions=[constants.DUMMY], - skill=skillName - ) - @property def supportedIntents(self) -> list: @@ -443,6 +437,8 @@ def initSkills(self, onlyInit: str = '', reload: bool = False): if skillName in self.NEEDED_SKILLS: skillInstance.required = True + self.ConfigManager.loadCheckAndUpdateSkillConfigurations(skillToLoad=skillName) + if skillActiveState: self._activeSkills[skillInstance.name] = skillInstance else: From f37e01019a8cae41ad831d0090869f22b5cdd3f8 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Sun, 14 Nov 2021 05:22:08 +0100 Subject: [PATCH 028/129] Will need some more rethinking --- core/base/SkillManager.py | 769 ++++++++++++++++++++------------------ core/commons/constants.py | 1 + 2 files changed, 409 insertions(+), 361 deletions(-) diff --git a/core/base/SkillManager.py b/core/base/SkillManager.py index 6a0cf6893..cff408308 100644 --- a/core/base/SkillManager.py +++ b/core/base/SkillManager.py @@ -84,6 +84,41 @@ def __init__(self): self._failedSkills: Dict[str, Union[AliceSkill, FailedAliceSkill]] = dict() + @property + def supportedIntents(self) -> list: + return self._supportedIntents + + + @property + def neededSkills(self) -> list: + return self.NEEDED_SKILLS + + + @property + def activeSkills(self) -> Dict[str, AliceSkill]: + return self._activeSkills + + + @property + def deactivatedSkills(self) -> Dict[str, AliceSkill]: + return self._deactivatedSkills + + + @property + def failedSkills(self) -> dict: + return self._failedSkills + + + @property + def allSkills(self) -> dict: + return {**self._activeSkills, **self._deactivatedSkills, **self._failedSkills} + + + @property + def allWorkingSkills(self) -> dict: + return {**self._activeSkills, **self._deactivatedSkills} + + def onStart(self): super().onStart() @@ -116,7 +151,7 @@ def notifyInstalling(self): def notifyFinishedInstalling(self): - self.MqttManager.mqttBroadcast(topic='hermes/leds/clear') + self.MqttManager.mqttBroadcast(topic=constants.TOPIC_HLC_CLEAR_LEDS) # noinspection SqlResolve @@ -149,7 +184,7 @@ def _loadSkills(self) -> List[str]: data = list() for skill in skills: try: - if not Path(self.Commons.rootDir(), f'skills/{skill["skillName"]}').exists(): + if not self.getSkillDirectory(skill['skillName']).exists(): raise Exception('Skill directory not existing') data.append(skill['skillName']) except Exception as e: @@ -158,45 +193,117 @@ def _loadSkills(self) -> List[str]: return sorted(data) - def updateSkills(self, skills: Union[str, List[str]], withSkillRestart: bool = True): + def loadSkillsFromDB(self) -> List: + return self.databaseFetch(tableName='skills') + + + def addSkillToDB(self, skillName: str, active: int = 1): + self.DatabaseManager.replace( + tableName='skills', + values={'skillName': skillName, 'active': active} + ) + + + # noinspection SqlResolve + def removeSkillFromDB(self, skillName: str): + self.DatabaseManager.delete( + tableName='skills', + callerName=self.name, + query='DELETE FROM :__table__ WHERE skillName = :skill', + values={'skill': skillName} + ) + + + def installSkills(self, skills: Union[str, List[str]]): """ - Updates skills to latest available version for this Alice version - :param skills: - :param withSkillRestart: Whether or not to start the skill after updating it + Installs the given skills + :param skills: Either a list of skill names to install or a single skill name :return: """ self._busyInstalling.set() - if isinstance(skills, str): skills = [skills] for skillName in skills: - if skillName in self._activeSkills: - self._activeSkills[skillName].onStop() - self._activeSkills.pop(skillName, None) - try: repository = self.getSkillRepository(skillName=skillName) - repository.checkout(tag=self.SkillStoreManager.getSkillUpdateTag(skillName=skillName), force=True) - except Exception as e: - self.logError(f'Error updating skill **{skillName}** : {e}') - continue + if not repository: + repositories = self.downloadSkills(skills=skillName) + repository = repositories.get(skillName, None) - self.broadcast(constants.EVENT_SKILL_UPDATED, propagateToSkills=True, skill=skillName) - #self.allSkills[skillName].onSkillUpdated(skill=skillName) # Not sure why this was here - self.MqttManager.mqttBroadcast( - topic=constants.TOPIC_SKILL_UPDATED, - payload={ - 'skillName': skillName - } - ) + if not repository: + raise Exception(f'Failed downloading skill **{skillName}** for some unknown reason') - if withSkillRestart: - self._startSkill(skillName=skillName) + installFile = json.loads(repository.file(f'{skillName}.install').read_text()) + pipReqs = installFile.get('pipRequirements', list()) + sysReqs = installFile.get('systemRequirements', list()) + scriptReq = installFile.get('script') + + self.checkSkillConditions(installFile) + + for requirement in pipReqs: + self.logInfo(f'Installing pip requirement: {requirement}') + self.Commons.runSystemCommand(['./venv/bin/pip3', 'install', requirement]) + + for requirement in sysReqs: + self.logInfo(f'Installing system requirement: {requirement}') + self.Commons.runRootSystemCommand(['apt-get', 'install', '-y', requirement]) + + if scriptReq: + self.logInfo('Running post install script') + req = repository.file(scriptReq) + if not req: + self.logWarning(f'Missing post install script **{str(req)}** as declared in install file') + continue + self.Commons.runRootSystemCommand(['chmod', '+x', str(req)]) + self.Commons.runRootSystemCommand([str(req)]) + + self.addSkillToDB(skillName) + self._skillList.append(skillName) + + if installFile.get('rebootAfterInstall', False): + self.Commons.runRootSystemCommand('sudo shutdown -r now'.split()) + break + except SkillNotConditionCompliant: + self.broadcast( + method=constants.EVENT_SKILL_INSTALL_FAILED, + exceptions=self._name, + skill=skillName + ) + except Exception as e: + self.logError(f'Error installing skill **{skillName}**: {e}') + self.broadcast( + method=constants.EVENT_SKILL_INSTALL_FAILED, + exceptions=self._name, + skill=skillName + ) + else: + self.broadcast( + method=constants.EVENT_SKILL_INSTALLED, + exceptions=[constants.DUMMY], + skill=skillName + ) self._busyInstalling.clear() + def getSkillRepository(self, skillName: str, directory: str = None) -> Optional[Git]: + """ + Returns a Git object for the given skill + :param skillName: + :param directory: where to look for that skill, if not standard directory + :return: + """ + + if not directory: + directory = self.getSkillDirectory(skillName=skillName) + + try: + return Git(directory=directory) + except: + return None + + def downloadSkills(self, skills: Union[str, List[str]]) -> Optional[Dict]: """ Clones skills @@ -266,229 +373,340 @@ def notCompliantSkill(self, skillName: str, exception: SkillNotConditionComplia return True - def loadSkillsFromDB(self) -> List: - return self.databaseFetch(tableName='skills') + def getSkillDirectory(self, skillName: str) -> Path: + return Path(self.Commons.rootDir()) / 'skills' / skillName - def changeSkillStateInDB(self, skillName: str, newState: bool): - # Changes the state of a skill in db and also deactivates widgets - # and device types if state is False - self.DatabaseManager.update( - tableName='skills', - callerName=self.name, - values={ - 'active': 1 if newState else 0 - }, - row=('skillName', skillName) - ) + def getGitRemoteSourceUrl(self, skillName: str, doAuth: bool = True) -> str: + """ + Returns the url for the skill name, taking into account if the user provided github auth + This does check if the remote exists and raises an exception in case it does not + :param skillName: + :param doAuth: Pull, clone, fetch, non oauth requests, aren't concerned by rate limit + :return: + """ + tokenPrefix = '' + if doAuth: + auth = self.Commons.getGithubAuth() + if auth: + tokenPrefix = f'{auth[0]}:{auth[1]}@' - if not newState: - self.WidgetManager.skillDeactivated(skillName=skillName) - self.DeviceManager.skillDeactivated(skillName=skillName) + url = f'{constants.GITHUB_URL}/skill_{skillName}.git' + if tokenPrefix: + url = url.replace('://', f'://{tokenPrefix}') + response = requests.get(url=url) + if response.status_code != 200: + raise GithubNotFound - def addSkillToDB(self, skillName: str, active: int = 1): - self.DatabaseManager.replace( - tableName='skills', - values={'skillName': skillName, 'active': active} - ) + return url - # noinspection SqlResolve - def removeSkillFromDB(self, skillName: str): - self.DatabaseManager.delete( - tableName='skills', - callerName=self.name, - query='DELETE FROM :__table__ WHERE skillName = :skill', - values={'skill': skillName} - ) + def initSkills(self, onlyInit: str = '', reload: bool = False): + """ + Loops over the available skills, creates their instances + :param onlyInit: If specified, will only init the given skill name + :param reload: If the skill is already instantiated, performs a module reload, after an update per example. + :return: + """ + for skillName in self._skillList: + if onlyInit and skillName != onlyInit: + continue - def onAssistantInstalled(self, **kwargs): - argv = kwargs.get('skillsInfos', dict()) - if not argv: - return + self._activeSkills.pop(skillName, None) + self._failedSkills.pop(skillName, None) + self._deactivatedSkills.pop(skillName, None) + + installFile = self.getSkillInstallFile(skillName=skillName) + if not installFile.exists(): + if skillName in self.NEEDED_SKILLS: + self.logFatal(f'Cannot find skill install file for skill **{skillName}**. The skill is required to continue') + return + else: + self.logWarning(f'Cannot find skill install file for skill **{skillName}**, skipping.') + else: + installFile = json.loads(installFile.read_text()) - for skillName, skill in argv.items(): try: - self._startSkill(skillName=skillName) - except SkillStartDelayed: - self.logInfo(f'Skill "{skillName}" start is delayed') - except KeyError as e: - self.logError(f'Skill "{skillName} not found, skipping: {e}') + skillActiveState = self.isSkillActive(skillName=skillName) + if not skillActiveState: + if skillName in self.NEEDED_SKILLS: + self.logFatal(f"Skill {skillName} marked as disabled but it shouldn't be") + return + else: + self.logInfo(f'Skill {skillName} is disabled') + else: + self.checkSkillConditions(installFile) + + skillInstance = self.instantiateSkill(skillName=skillName, reload=reload) + if skillInstance: + if skillName in self.NEEDED_SKILLS: + skillInstance.required = True + + self.ConfigManager.loadCheckAndUpdateSkillConfigurations(skillToLoad=skillName) + + if skillActiveState: + self._activeSkills[skillInstance.name] = skillInstance + else: + self._deactivatedSkills[skillName] = skillInstance + else: + if skillName in self.NEEDED_SKILLS: + self.logFatal(f'The skill is required to continue...') + return + else: + self._failedSkills[skillName] = FailedAliceSkill(installFile) + except SkillNotConditionCompliant as e: + if self.notCompliantSkill(skillName=skillName, exception=e): + self._failedSkills[skillName] = FailedAliceSkill(installFile) + continue + else: + return + except Exception as e: + self.logWarning(f'Something went wrong loading skill {skillName}: {e}') + self._failedSkills[skillName] = FailedAliceSkill(installFile) continue - self._activeSkills[skillName].onBooted() + def getSkillInstallFile(self, skillName: str) -> Path: + return Path(self.Commons.rootDir(), f'skills/{skillName}/{skillName}.install') - @property - def supportedIntents(self) -> list: - return self._supportedIntents + # noinspection PyTypeChecker + def instantiateSkill(self, skillName: str, skillResource: str = '', reload: bool = False) -> Optional[AliceSkill]: + instance: Optional[AliceSkill] = None + skillResource = skillResource or skillName - @property - def neededSkills(self) -> list: - return self.NEEDED_SKILLS + try: + skillImport = importlib.import_module(f'skills.{skillName}.{skillResource}') + if reload: + skillImport = importlib.reload(skillImport) - @property - def activeSkills(self) -> Dict[str, AliceSkill]: - return self._activeSkills + klass = getattr(skillImport, skillName) + instance: AliceSkill = klass() + except ImportError as e: + self.logError(f"Couldn't import skill {skillName}.{skillResource}: {e}") + traceback.print_exc() + except AttributeError as e: + self.logError(f"Couldn't find main class for skill {skillName}.{skillResource}: {e}") + except SkillInstanceFailed: + self.logError(f"Couldn't instanciate skill {skillName}.{skillResource}") + except Exception as e: + self.logError(f"Unknown error instantiating {skillName}.{skillResource}: {e} {traceback.print_exc()}") + return instance - @property - def deactivatedSkills(self) -> Dict[str, AliceSkill]: - return self._deactivatedSkills + def isSkillActive(self, skillName: str) -> bool: + if skillName in self._activeSkills: + return self._activeSkills[skillName].active + elif skillName in self._skillList: + # noinspection SqlResolve + row = self.databaseFetch(tableName=self.DBTAB_SKILLS, query='SELECT active FROM :__table__ WHERE skillName = :skillName LIMIT 1', values={'skillName': skillName}) + if not row: + return False + return int(row[0]['active']) == 1 + return False - @property - def failedSkills(self) -> dict: - return self._failedSkills + def checkSkillConditions(self, installer: dict = None) -> bool: + conditions = { + 'aliceMinVersion': installer['aliceMinVersion'], + **installer.get('conditions', dict()) + } - @property - def allSkills(self) -> dict: - return {**self._activeSkills, **self._deactivatedSkills, **self._failedSkills} + notCompliant = 'Skill is not compliant' + if 'aliceMinVersion' in conditions and \ + Version.fromString(conditions['aliceMinVersion']) > Version.fromString(constants.VERSION): + raise SkillNotConditionCompliant(message=notCompliant, skillName=installer['name'], condition='Alice minimum version', conditionValue=conditions['aliceMinVersion']) - @property - def allWorkingSkills(self) -> dict: - return {**self._activeSkills, **self._deactivatedSkills} + for conditionName, conditionValue in conditions.items(): + if conditionName == 'lang' and self.LanguageManager.activeLanguage not in conditionValue: + raise SkillNotConditionCompliant(message=notCompliant, skillName=installer['name'], condition=conditionName, conditionValue=conditionValue) + elif conditionName == 'online': + if conditionValue and self.ConfigManager.getAliceConfigByName('stayCompletelyOffline') \ + or not conditionValue and not self.ConfigManager.getAliceConfigByName('stayCompletelyOffline'): + raise SkillNotConditionCompliant(message=notCompliant, skillName=installer['name'], condition=conditionName, conditionValue=conditionValue) - def onBooted(self): - self.skillBroadcast(constants.EVENT_BOOTED) - self._finishInstall() + elif conditionName == 'skill': + for requiredSkill in conditionValue: + if requiredSkill in self._skillList and not self.isSkillActive(skillName=installer['name']): + raise SkillNotConditionCompliant(message=notCompliant, skillName=installer['name'], condition=conditionName, conditionValue=conditionValue) + elif requiredSkill not in self._skillList: + self.logInfo(f'Skill {installer["name"]} has another skill as dependency, adding download') + if not self.downloadInstallTicket(requiredSkill): + raise SkillNotConditionCompliant(message=notCompliant, skillName=installer['name'], condition=conditionName, conditionValue=conditionValue) - if self._skillInstallThread: - self._skillInstallThread.start() + elif conditionName == 'notSkill': + for excludedSkill in conditionValue: + author, name = excludedSkill.split('/') + if name in self._skillList and self.isSkillActive(skillName=installer['name']): + raise SkillNotConditionCompliant(message=notCompliant, skillName=installer['name'], condition=conditionName, conditionValue=conditionValue) + elif conditionName == 'asrArbitraryCapture': + if conditionValue and not self.ASRManager.asr.capableOfArbitraryCapture: + raise SkillNotConditionCompliant(message=notCompliant, skillName=installer['name'], condition=conditionName, conditionValue=conditionValue) - def dispatchMessage(self, session: DialogSession) -> bool: - for skillName, skillInstance in self._activeSkills.items(): - try: - consumed = skillInstance.onMessageDispatch(session) - except AccessLevelTooLow: - # The command was recognized but required higher access level - return True - except Exception as e: - self.logError(f'Error dispatching message "{session.intentName.split("/")[-1]}" to {skillInstance.name}: {e}') - self.MqttManager.endDialog( - sessionId=session.sessionId, - text=self.TalkManager.randomTalk(talk='error', skill='system') - ) - traceback.print_exc() - return True + elif conditionName == 'activeManager': + for manager in conditionValue: + if not manager: + continue - if consumed: - self.logDebug(f'The intent "{session.intentName.split("/")[-1]}" was consumed by {skillName}') + man = SuperManager.getInstance().getManager(manager) + if not man or not man.isActive: + raise SkillNotConditionCompliant(message=notCompliant, skillName=installer['name'], condition=conditionName, conditionValue=conditionValue) - if self.MultiIntentManager.isProcessing(session.sessionId): - self.MultiIntentManager.processNextIntent(session=session) + return True - return True - if self.MultiIntentManager.isProcessing(session.sessionId): - self.MultiIntentManager.processNextIntent(session=session) - return True - return False - def initSkills(self, onlyInit: str = '', reload: bool = False): + + + + + + + + + + + + def updateSkills(self, skills: Union[str, List[str]], withSkillRestart: bool = True): """ - Loops over the available skills, creates their instances - :param onlyInit: If specified, will only init the given skill name - :param reload: If the skill is already instantiated, performs a module reload, after an update per example. + Updates skills to latest available version for this Alice version + :param skills: + :param withSkillRestart: Whether or not to start the skill after updating it :return: """ + self._busyInstalling.set() - for skillName in self._skillList: - if onlyInit and skillName != onlyInit: + if isinstance(skills, str): + skills = [skills] + + for skillName in skills: + if skillName in self._activeSkills: + self._activeSkills[skillName].onStop() + self._activeSkills.pop(skillName, None) + + try: + repository = self.getSkillRepository(skillName=skillName) + repository.checkout(tag=self.SkillStoreManager.getSkillUpdateTag(skillName=skillName), force=True) + except Exception as e: + self.logError(f'Error updating skill **{skillName}** : {e}') continue - self._activeSkills.pop(skillName, None) - self._failedSkills.pop(skillName, None) - self._deactivatedSkills.pop(skillName, None) + self.broadcast(constants.EVENT_SKILL_UPDATED, propagateToSkills=True, skill=skillName) + #self.allSkills[skillName].onSkillUpdated(skill=skillName) # Not sure why this was here + self.MqttManager.mqttBroadcast( + topic=constants.TOPIC_SKILL_UPDATED, + payload={ + 'skillName': skillName + } + ) - installFile = self.getSkillInstallFile(skillName=skillName) - if not installFile.exists(): - if skillName in self.NEEDED_SKILLS: - self.logFatal(f'Cannot find skill install file for skill **{skillName}**. The skill is required to continue') - return - else: - self.logWarning(f'Cannot find skill install file for skill **{skillName}**, skipping.') - else: - installFile = json.loads(installFile.read_text()) + if withSkillRestart: + self._startSkill(skillName=skillName) + + self._busyInstalling.clear() + + + + + + + + + def changeSkillStateInDB(self, skillName: str, newState: bool): + # Changes the state of a skill in db and also deactivates widgets + # and device types if state is False + self.DatabaseManager.update( + tableName='skills', + callerName=self.name, + values={ + 'active': 1 if newState else 0 + }, + row=('skillName', skillName) + ) + + if not newState: + self.WidgetManager.skillDeactivated(skillName=skillName) + self.DeviceManager.skillDeactivated(skillName=skillName) + + + + + + + + + def onAssistantInstalled(self, **kwargs): + argv = kwargs.get('skillsInfos', dict()) + if not argv: + return + + for skillName, skill in argv.items(): + try: + self._startSkill(skillName=skillName) + except SkillStartDelayed: + self.logInfo(f'Skill "{skillName}" start is delayed') + except KeyError as e: + self.logError(f'Skill "{skillName} not found, skipping: {e}') + continue + + self._activeSkills[skillName].onBooted() + + + + + + def onBooted(self): + self.skillBroadcast(constants.EVENT_BOOTED) + self._finishInstall() + + if self._skillInstallThread: + self._skillInstallThread.start() + + def dispatchMessage(self, session: DialogSession) -> bool: + for skillName, skillInstance in self._activeSkills.items(): try: - skillActiveState = self.isSkillActive(skillName=skillName) - if not skillActiveState: - if skillName in self.NEEDED_SKILLS: - self.logFatal(f"Skill {skillName} marked as disabled but it shouldn't be") - return - else: - self.logInfo(f'Skill {skillName} is disabled') - else: - self.checkSkillConditions(installFile) + consumed = skillInstance.onMessageDispatch(session) + except AccessLevelTooLow: + # The command was recognized but required higher access level + return True + except Exception as e: + self.logError(f'Error dispatching message "{session.intentName.split("/")[-1]}" to {skillInstance.name}: {e}') + self.MqttManager.endDialog( + sessionId=session.sessionId, + text=self.TalkManager.randomTalk(talk='error', skill='system') + ) + traceback.print_exc() + return True - skillInstance = self.instantiateSkill(skillName=skillName, reload=reload) - if skillInstance: - if skillName in self.NEEDED_SKILLS: - skillInstance.required = True + if consumed: + self.logDebug(f'The intent "{session.intentName.split("/")[-1]}" was consumed by {skillName}') - self.ConfigManager.loadCheckAndUpdateSkillConfigurations(skillToLoad=skillName) + if self.MultiIntentManager.isProcessing(session.sessionId): + self.MultiIntentManager.processNextIntent(session=session) - if skillActiveState: - self._activeSkills[skillInstance.name] = skillInstance - else: - self._deactivatedSkills[skillName] = skillInstance - else: - if skillName in self.NEEDED_SKILLS: - self.logFatal(f'The skill is required to continue...') - return - else: - self._failedSkills[skillName] = FailedAliceSkill(installFile) - except SkillNotConditionCompliant as e: - if self.notCompliantSkill(skillName=skillName, exception=e): - self._failedSkills[skillName] = FailedAliceSkill(installFile) - continue - else: - return - except Exception as e: - self.logWarning(f'Something went wrong loading skill {skillName}: {e}') - self._failedSkills[skillName] = FailedAliceSkill(installFile) - continue + return True + if self.MultiIntentManager.isProcessing(session.sessionId): + self.MultiIntentManager.processNextIntent(session=session) + return True - def getSkillInstallFile(self, skillName: str) -> Path: - return Path(self.Commons.rootDir(), f'skills/{skillName}/{skillName}.install') + return False - # noinspection PyTypeChecker - def instantiateSkill(self, skillName: str, skillResource: str = '', reload: bool = False) -> Optional[AliceSkill]: - instance: Optional[AliceSkill] = None - skillResource = skillResource or skillName - try: - skillImport = importlib.import_module(f'skills.{skillName}.{skillResource}') - if reload: - skillImport = importlib.reload(skillImport) - klass = getattr(skillImport, skillName) - instance: AliceSkill = klass() - except ImportError as e: - self.logError(f"Couldn't import skill {skillName}.{skillResource}: {e}") - traceback.print_exc() - except AttributeError as e: - self.logError(f"Couldn't find main class for skill {skillName}.{skillResource}: {e}") - except SkillInstanceFailed: - self.logError(f"Couldn't instanciate skill {skillName}.{skillResource}") - except Exception as e: - self.logError(f"Unknown error instantiating {skillName}.{skillResource}: {e} {traceback.print_exc()}") - return instance def onStop(self): @@ -573,16 +791,7 @@ def _startSkill(self, skillName: str) -> dict: return skillInstance.supportedIntents - def isSkillActive(self, skillName: str) -> bool: - if skillName in self._activeSkills: - return self._activeSkills[skillName].active - elif skillName in self._skillList: - # noinspection SqlResolve - row = self.databaseFetch(tableName=self.DBTAB_SKILLS, query='SELECT active FROM :__table__ WHERE skillName = :skillName LIMIT 1', values={'skillName': skillName}) - if not row: - return False - return int(row[0]['active']) == 1 - return False + def getSkillInstance(self, skillName: str, silent: bool = False) -> Optional[AliceSkill]: @@ -899,123 +1108,10 @@ def _installSkillTickets(self, skills: list) -> dict: return skillsToBoot - def getSkillDirectory(self, skillName: str) -> Path: - return Path(self.Commons.rootDir()) / 'skills' / skillName - - - def getSkillRepository(self, skillName: str, directory: str = None) -> Optional[Git]: - """ - Returns a Git object for the given skill - :param skillName: - :param directory: where to look for that skill, if not standard directory - :return: - """ - - if not directory: - directory = self.getSkillDirectory(skillName=skillName) - - try: - return Git(directory=directory) - except: - return None - - - def getGitRemoteSourceUrl(self, skillName: str, doAuth: bool = True) -> str: - """ - Returns the url for the skillname, taking into account if the user provided github auth - This does check if the remote exists and raises an exception in case it does not - :param skillName: - :param doAuth: Pull, clone, fetch, non oauth requests, aren't concerned by rate limit - :return: - """ - tokenPrefix = '' - if doAuth: - auth = self.Commons.getGithubAuth() - if auth: - tokenPrefix = f'{auth[0]}:{auth[1]}@' - - url = f'{constants.GITHUB_URL}/skill_{skillName}.git' - if tokenPrefix: - url = url.replace('://', f'://{tokenPrefix}') - - response = requests.get(url=url) - if response.status_code != 200: - raise GithubNotFound - - return url - - - def installSkills(self, skills: Union[str, List[str]]): - """ - Installs the given skills - :param skills: Either a list of skill names to install or a single skill name - :return: - """ - self._busyInstalling.set() - if isinstance(skills, str): - skills = [skills] - - for skillName in skills: - try: - repository = self.getSkillRepository(skillName=skillName) - if not repository: - repositories = self.downloadSkills(skills=skillName) - repository = repositories.get(skillName, None) - - if not repository: - raise Exception(f'Failed downloading skill **{skillName}** for some unknown reason') - - installFile = json.loads(repository.file(f'{skillName}.install').read_text()) - pipReqs = installFile.get('pipRequirements', list()) - sysReqs = installFile.get('systemRequirements', list()) - scriptReq = installFile.get('script') - - self.checkSkillConditions(installFile) - - for requirement in pipReqs: - self.logInfo(f'Installing pip requirement: {requirement}') - self.Commons.runSystemCommand(['./venv/bin/pip3', 'install', requirement]) - - for requirement in sysReqs: - self.logInfo(f'Installing system requirement: {requirement}') - self.Commons.runRootSystemCommand(['apt-get', 'install', '-y', requirement]) - if scriptReq: - self.logInfo('Running post install script') - req = repository.file(scriptReq) - if not req: - self.logWarning(f'Missing post install script **{str(req)}** as declared in install file') - continue - self.Commons.runRootSystemCommand(['chmod', '+x', str(req)]) - self.Commons.runRootSystemCommand([str(req)]) - self.addSkillToDB(skillName) - self._skillList.append(skillName) - if installFile.get('rebootAfterInstall', False): - self.Commons.runRootSystemCommand('sudo shutdown -r now'.split()) - break - except SkillNotConditionCompliant: - self.broadcast( - method=constants.EVENT_SKILL_INSTALL_FAILED, - exceptions=self._name, - skill=skillName - ) - except Exception as e: - self.logError(f'Error installing skill **{skillName}**: {e}') - self.broadcast( - method=constants.EVENT_SKILL_INSTALL_FAILED, - exceptions=self._name, - skill=skillName - ) - else: - self.broadcast( - method=constants.EVENT_SKILL_INSTALLED, - exceptions=[constants.DUMMY], - skill=skillName - ) - self._busyInstalling.clear() def onSkillInstalled(self, skill: str): @@ -1036,56 +1132,7 @@ def onSkillUpdated(self, skill: str): ) - def checkSkillConditions(self, installer: dict = None) -> bool: - conditions = { - 'aliceMinVersion': installer['aliceMinVersion'], - **installer.get('conditions', dict()) - } - - notCompliant = 'Skill is not compliant' - - if 'aliceMinVersion' in conditions and \ - Version.fromString(conditions['aliceMinVersion']) > Version.fromString(constants.VERSION): - raise SkillNotConditionCompliant(message=notCompliant, skillName=installer['name'], condition='Alice minimum version', conditionValue=conditions['aliceMinVersion']) - - for conditionName, conditionValue in conditions.items(): - if conditionName == 'lang' and self.LanguageManager.activeLanguage not in conditionValue: - raise SkillNotConditionCompliant(message=notCompliant, skillName=installer['name'], condition=conditionName, conditionValue=conditionValue) - - elif conditionName == 'online': - if conditionValue and self.ConfigManager.getAliceConfigByName('stayCompletelyOffline') \ - or not conditionValue and not self.ConfigManager.getAliceConfigByName('stayCompletelyOffline'): - raise SkillNotConditionCompliant(message=notCompliant, skillName=installer['name'], condition=conditionName, conditionValue=conditionValue) - - elif conditionName == 'skill': - for requiredSkill in conditionValue: - if requiredSkill in self._skillList and not self.isSkillActive(skillName=installer['name']): - raise SkillNotConditionCompliant(message=notCompliant, skillName=installer['name'], condition=conditionName, conditionValue=conditionValue) - elif requiredSkill not in self._skillList: - self.logInfo(f'Skill {installer["name"]} has another skill as dependency, adding download') - if not self.downloadInstallTicket(requiredSkill): - raise SkillNotConditionCompliant(message=notCompliant, skillName=installer['name'], condition=conditionName, conditionValue=conditionValue) - - elif conditionName == 'notSkill': - for excludedSkill in conditionValue: - author, name = excludedSkill.split('/') - if name in self._skillList and self.isSkillActive(skillName=installer['name']): - raise SkillNotConditionCompliant(message=notCompliant, skillName=installer['name'], condition=conditionName, conditionValue=conditionValue) - - elif conditionName == 'asrArbitraryCapture': - if conditionValue and not self.ASRManager.asr.capableOfArbitraryCapture: - raise SkillNotConditionCompliant(message=notCompliant, skillName=installer['name'], condition=conditionName, conditionValue=conditionValue) - - elif conditionName == 'activeManager': - for manager in conditionValue: - if not manager: - continue - - man = SuperManager.getInstance().getManager(manager) - if not man or not man.isActive: - raise SkillNotConditionCompliant(message=notCompliant, skillName=installer['name'], condition=conditionName, conditionValue=conditionValue) - return True def configureSkillIntents(self, skillName: str, state: bool): diff --git a/core/commons/constants.py b/core/commons/constants.py index 9c76fac86..60897f65e 100644 --- a/core/commons/constants.py +++ b/core/commons/constants.py @@ -66,6 +66,7 @@ TOPIC_SESSION_STARTED = 'hermes/dialogueManager/sessionStarted' TOPIC_START_SESSION = 'hermes/dialogueManager/startSession' TOPIC_SYSTEM_UPDATE = 'hermes/leds/systemUpdate' +TOPIC_HLC_CLEAR_LEDS = 'hermes/leds/clear' TOPIC_TEXT_CAPTURED = 'hermes/asr/textCaptured' TOPIC_TOGGLE_FEEDBACK = 'hermes/feedback/sound/toggle{}' TOPIC_TOGGLE_FEEDBACK_OFF = 'hermes/feedback/sound/toggleOff' From d43ebddef416d35c31e45304b9971b12d234e4e3 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Sun, 14 Nov 2021 13:41:25 +0100 Subject: [PATCH 029/129] SkillManager cleanup almost done. UNTESTED --- core/base/SkillManager.py | 728 ++++++++++--------------- core/base/model/AliceSkill.py | 6 +- core/base/model/ProjectAliceObject.py | 24 +- core/commons/constants.py | 7 + core/device/DeviceManager.py | 9 +- core/server/MqttManager.py | 89 ++- core/webApi/model/SkillsApi.py | 4 +- core/webui/WebUINotificationManager.py | 18 + core/webui/WidgetManager.py | 4 +- 9 files changed, 417 insertions(+), 472 deletions(-) diff --git a/core/base/SkillManager.py b/core/base/SkillManager.py index cff408308..957001224 100644 --- a/core/base/SkillManager.py +++ b/core/base/SkillManager.py @@ -19,10 +19,10 @@ import getpass +import traceback + import importlib import json -import threading -import traceback from contextlib import suppress from typing import Dict, Optional @@ -71,8 +71,6 @@ def __init__(self): super().__init__(databaseSchema=self.DATABASE) self._busyInstalling = None - - self._skillInstallThread: Optional[threading.Thread] = None self._supportedIntents = list() # This is a list of the skill names installed @@ -132,20 +130,40 @@ def onStart(self): self.logInfo('Some required skills are missing, let\'s download them!') self.installSkills(skills=list(set(self.NEEDED_SKILLS) - set(self._skillList))) + updates = self.checkForSkillUpdates() + if updates: + self.updateSkills(skills=updates, withSkillRestart=False) + self.initSkills() for skillName in self._deactivatedSkills: self.configureSkillIntents(skillName=skillName, state=False) - updates = self.checkForSkillUpdates() - if updates: - self.updateSkills(skills=updates) - self.ConfigManager.loadCheckAndUpdateSkillConfigurations() self.startAllSkills() + def onBooted(self): + self.skillBroadcast(constants.EVENT_BOOTED) + + + def onStop(self): + super().onStop() + + for skillName in self._activeSkills: + self.stopSkill(skillName=skillName) + + + def onQuarterHour(self): + if self._busyInstalling.isSet() or self.ProjectAlice.restart or self.ProjectAlice.updating or self.NluManager.training: + return + + updates = self.checkForSkillUpdates() + if updates: + self.updateSkills(skills=updates) + + def notifyInstalling(self): self.MqttManager.mqttBroadcast(topic=constants.TOPIC_SYSTEM_UPDATE, payload={'sticky': True}) @@ -268,19 +286,22 @@ def installSkills(self, skills: Union[str, List[str]]): self.broadcast( method=constants.EVENT_SKILL_INSTALL_FAILED, exceptions=self._name, + propagateToSkills=True, skill=skillName ) except Exception as e: self.logError(f'Error installing skill **{skillName}**: {e}') self.broadcast( method=constants.EVENT_SKILL_INSTALL_FAILED, - exceptions=self._name, + exceptions=[constants.DUMMY], + propagateToSkills=True, skill=skillName ) else: self.broadcast( method=constants.EVENT_SKILL_INSTALLED, exceptions=[constants.DUMMY], + propagateToSkills=True, skill=skillName ) @@ -323,6 +344,9 @@ def downloadSkills(self, skills: Union[str, List[str]]) -> Optional[Dict]: if response.status_code != 200: raise GithubNotFound + with suppress: # Increment download counter + requests.get(f'https://skills.projectalice.ch/{skillName}') + installFile = json.loads(response.json()) self.checkSkillConditions(installer=installFile) @@ -404,7 +428,6 @@ def getGitRemoteSourceUrl(self, skillName: str, doAuth: bool = True) -> str: def initSkills(self, onlyInit: str = '', reload: bool = False): """ - Loops over the available skills, creates their instances :param onlyInit: If specified, will only init the given skill name :param reload: If the skill is already instantiated, performs a module reload, after an update per example. :return: @@ -432,7 +455,7 @@ def initSkills(self, onlyInit: str = '', reload: bool = False): skillActiveState = self.isSkillActive(skillName=skillName) if not skillActiveState: if skillName in self.NEEDED_SKILLS: - self.logFatal(f"Skill {skillName} marked as disabled but it shouldn't be") + self.logFatal(f"Skill {skillName} marked as disabled but it cannot be") return else: self.logInfo(f'Skill {skillName} is disabled') @@ -464,8 +487,12 @@ def initSkills(self, onlyInit: str = '', reload: bool = False): return except Exception as e: self.logWarning(f'Something went wrong loading skill {skillName}: {e}') - self._failedSkills[skillName] = FailedAliceSkill(installFile) - continue + if skillName in self.NEEDED_SKILLS: + self.logFatal(f'The skill is required to continue...') + return + else: + self._failedSkills[skillName] = FailedAliceSkill(installFile) + continue def getSkillInstallFile(self, skillName: str) -> Path: @@ -537,7 +564,9 @@ def checkSkillConditions(self, installer: dict = None) -> bool: raise SkillNotConditionCompliant(message=notCompliant, skillName=installer['name'], condition=conditionName, conditionValue=conditionValue) elif requiredSkill not in self._skillList: self.logInfo(f'Skill {installer["name"]} has another skill as dependency, adding download') - if not self.downloadInstallTicket(requiredSkill): + try: + self.downloadSkills(skills=requiredSkill) + except: raise SkillNotConditionCompliant(message=notCompliant, skillName=installer['name'], condition=conditionName, conditionValue=conditionValue) elif conditionName == 'notSkill': @@ -562,20 +591,6 @@ def checkSkillConditions(self, installer: dict = None) -> bool: return True - - - - - - - - - - - - - - def updateSkills(self, skills: Union[str, List[str]], withSkillRestart: bool = True): """ Updates skills to latest available version for this Alice version @@ -589,9 +604,9 @@ def updateSkills(self, skills: Union[str, List[str]], withSkillRestart: bool = T skills = [skills] for skillName in skills: - if skillName in self._activeSkills: - self._activeSkills[skillName].onStop() - self._activeSkills.pop(skillName, None) + self.logInfo(f'Now updating skill **{skillName}**') + self.stopSkill(skillName=skillName) + self._failedSkills.pop(skillName, None) try: repository = self.getSkillRepository(skillName=skillName) @@ -600,130 +615,54 @@ def updateSkills(self, skills: Union[str, List[str]], withSkillRestart: bool = T self.logError(f'Error updating skill **{skillName}** : {e}') continue - self.broadcast(constants.EVENT_SKILL_UPDATED, propagateToSkills=True, skill=skillName) - #self.allSkills[skillName].onSkillUpdated(skill=skillName) # Not sure why this was here - self.MqttManager.mqttBroadcast( - topic=constants.TOPIC_SKILL_UPDATED, - payload={ - 'skillName': skillName - } + self.broadcast( + method=constants.EVENT_SKILL_UPDATED, + exceptions=[constants.DUMMY], + propagateToSkills=True, + skill=skillName ) + self.initSkills(onlyInit=skillName, reload=True) if withSkillRestart: - self._startSkill(skillName=skillName) + self.startSkill(skillName=skillName) self._busyInstalling.clear() + def stopSkill(self, skillName: str) -> Optional[AliceSkill]: + skill = None + if skillName in self._activeSkills: + skill = self._activeSkills.pop(skillName, None) + self.broadcast( + method=constants.EVENT_SKILL_STOPPED, + exceptions=[constants.DUMMY], + propagateToSkills=True, + skill=skillName + ) + return skill + def configureSkillIntents(self, skillName: str, state: bool): + try: + skills = self.allWorkingSkills + confs = [{ + 'intentId': intent.justTopic if isinstance(intent, Intent.Intent) else intent.split('/')[-1], + 'enable' : state + } for intent in skills[skillName].supportedIntents if not self.isIntentInUse(intent=intent, filtered=[skillName])] + self.MqttManager.configureIntents(confs) + if state: + skills[skillName].subscribeIntents() + else: + skills[skillName].unsubscribeIntents() + except Exception as e: + self.logWarning(f'Intent configuration failed: {e}') - def changeSkillStateInDB(self, skillName: str, newState: bool): - # Changes the state of a skill in db and also deactivates widgets - # and device types if state is False - self.DatabaseManager.update( - tableName='skills', - callerName=self.name, - values={ - 'active': 1 if newState else 0 - }, - row=('skillName', skillName) - ) - - if not newState: - self.WidgetManager.skillDeactivated(skillName=skillName) - self.DeviceManager.skillDeactivated(skillName=skillName) - - - - - - - - - def onAssistantInstalled(self, **kwargs): - argv = kwargs.get('skillsInfos', dict()) - if not argv: - return - - for skillName, skill in argv.items(): - try: - self._startSkill(skillName=skillName) - except SkillStartDelayed: - self.logInfo(f'Skill "{skillName}" start is delayed') - except KeyError as e: - self.logError(f'Skill "{skillName} not found, skipping: {e}') - continue - - self._activeSkills[skillName].onBooted() - - - - - - def onBooted(self): - self.skillBroadcast(constants.EVENT_BOOTED) - self._finishInstall() - - if self._skillInstallThread: - self._skillInstallThread.start() - - - def dispatchMessage(self, session: DialogSession) -> bool: - for skillName, skillInstance in self._activeSkills.items(): - try: - consumed = skillInstance.onMessageDispatch(session) - except AccessLevelTooLow: - # The command was recognized but required higher access level - return True - except Exception as e: - self.logError(f'Error dispatching message "{session.intentName.split("/")[-1]}" to {skillInstance.name}: {e}') - self.MqttManager.endDialog( - sessionId=session.sessionId, - text=self.TalkManager.randomTalk(talk='error', skill='system') - ) - traceback.print_exc() - return True - - if consumed: - self.logDebug(f'The intent "{session.intentName.split("/")[-1]}" was consumed by {skillName}') - - if self.MultiIntentManager.isProcessing(session.sessionId): - self.MultiIntentManager.processNextIntent(session=session) - - return True - - if self.MultiIntentManager.isProcessing(session.sessionId): - self.MultiIntentManager.processNextIntent(session=session) - return True - - return False - - - - - - - - - def onStop(self): - super().onStop() - - for skillItem in self._activeSkills.values(): - skillItem.onStop() - self.broadcast(method=constants.EVENT_SKILL_STOPPED, exceptions=[self.name], propagateToSkills=True, skill=self) - - - def onQuarterHour(self): - if self._busyInstalling.isSet() or self.ProjectAlice.restart or self.ProjectAlice.updating or self.NluManager.training: - return - - updates = self.checkForSkillUpdates() - if updates: - self.updateSkills(skills=updates) + def isIntentInUse(self, intent: Intent, filtered: list) -> bool: + skills = self.allWorkingSkills + return any(intent in skill.supportedIntents for name, skill in skills.items() if name not in filtered) def startAllSkills(self): @@ -731,7 +670,7 @@ def startAllSkills(self): for skillName in self._activeSkills.copy(): try: - supportedIntents += self._startSkill(skillName) + supportedIntents += self.startSkill(skillName) except SkillStartingFailed: continue except SkillStartDelayed: @@ -743,7 +682,7 @@ def startAllSkills(self): self.logInfo(f'Skills started. {len(supportedIntents)} intents supported') - def _startSkill(self, skillName: str) -> dict: + def startSkill(self, skillName: str) -> dict: if skillName in self._activeSkills: skillInstance = self._activeSkills[skillName] elif skillName in self._deactivatedSkills: @@ -768,7 +707,13 @@ def _startSkill(self, skillName: str) -> dict: skillInstance.onStart() if self.ProjectAlice.isBooted: skillInstance.onBooted() - self.broadcast(method=constants.EVENT_SKILL_STARTED, exceptions=[self.name], propagateToSkills=True, skill=self) + + self.broadcast( + method=constants.EVENT_SKILL_STARTED, + exceptions=[constants.DUMMY], + propagateToSkills=True, + skill=skillName + ) except SkillStartingFailed: try: skillInstance.failedStarting = True @@ -791,58 +736,17 @@ def _startSkill(self, skillName: str) -> dict: return skillInstance.supportedIntents - - - - def getSkillInstance(self, skillName: str, silent: bool = False) -> Optional[AliceSkill]: - if skillName in self._activeSkills: - return self._activeSkills[skillName] - elif skillName in self._deactivatedSkills: - return self._deactivatedSkills[skillName] - elif skillName in self._failedSkills: - return self._failedSkills[skillName] - else: - if not silent: - self.logWarning(f'Skill "{skillName}" does not exist in skills manager') - - return None - - - def skillBroadcast(self, method: str, filterOut: list = None, **kwargs): - """ - Broadcasts a call to the given method on every skill - :param filterOut: array, skills not to broadcast to - :param method: str, the method name to call on every skill - :return: - """ - - if not method.startswith('on'): - method = f'on{method[0].capitalize() + method[1:]}' - - for skillName, skillInstance in self._activeSkills.items(): - - if filterOut and skillName in filterOut: - continue - - try: - func = getattr(skillInstance, method, None) - if func: - func(**kwargs) - - func = getattr(skillInstance, 'onEvent', None) - if func: - func(event=method, **kwargs) - - except TypeError as e: - self.logWarning(f'Failed to broadcast event {method} to {skillName}: {e}') - - def deactivateSkill(self, skillName: str, persistent: bool = False): if skillName in self._activeSkills: - skillInstance = self._activeSkills.pop(skillName) - self._deactivatedSkills[skillName] = skillInstance - skillInstance.onStop() - self.broadcast(method=constants.EVENT_SKILL_STOPPED, exceptions=[self.name], propagateToSkills=True, skill=self) + skillInstance = self.stopSkill(skillName=skillName) + if skillInstance: + self._deactivatedSkills[skillName] = skillInstance + self.broadcast( + method=constants.EVENT_SKILL_DEACTIVATED, + exceptions=[constants.DUMMY], + propagateToSkills=True, + skill=skillInstance + ) if persistent: self.changeSkillStateInDB(skillName=skillName, newState=False) @@ -859,13 +763,20 @@ def activateSkill(self, skillName: str, persistent: bool = False): return try: - self._startSkill(skillName) + self.startSkill(skillName) if persistent: self.changeSkillStateInDB(skillName=skillName, newState=True) self.logInfo(f'Activated skill "{skillName}" with persistence') else: self.logInfo(f'Activated skill "{skillName}" without persistence') + + self.broadcast( + method=constants.EVENT_SKILL_ACTIVATED, + exceptions=[constants.DUMMY], + propagateToSkills=True, + skill=self.activeSkills[skillName] + ) except: self.logError(f'Failed activating skill "{skillName}"') return @@ -878,6 +789,49 @@ def toggleSkillState(self, skillName: str, persistent: bool = False): self.activateSkill(skillName=skillName, persistent=persistent) + def changeSkillStateInDB(self, skillName: str, newState: bool): + # Changes the state of a skill in db + self.DatabaseManager.update( + tableName='skills', + callerName=self.name, + values={ + 'active': 1 if newState else 0 + }, + row=('skillName', skillName) + ) + + + def dispatchMessage(self, session: DialogSession) -> bool: + for skillName, skillInstance in self._activeSkills.items(): + try: + consumed = skillInstance.onMessageDispatch(session) + except AccessLevelTooLow: + # The command was recognized but required higher access level + return True + except Exception as e: + self.logError(f'Error dispatching message "{session.intentName.split("/")[-1]}" to {skillInstance.name}: {e}') + self.MqttManager.endDialog( + sessionId=session.sessionId, + text=self.TalkManager.randomTalk(talk='error', skill='system') + ) + traceback.print_exc() + return True + + if consumed: + self.logDebug(f'The intent "{session.intentName.split("/")[-1]}" was consumed by {skillName}') + + if self.MultiIntentManager.isProcessing(session.sessionId): + self.MultiIntentManager.processNextIntent(session=session) + + return True + + if self.MultiIntentManager.isProcessing(session.sessionId): + self.MultiIntentManager.processNextIntent(session=session) + return True + + return False + + @Online(catchOnly=True) @IfSetting(settingName='stayCompletelyOffline', settingValue=False) def checkForSkillUpdates(self, skillToCheck: str = None) -> List[str]: @@ -891,9 +845,6 @@ def checkForSkillUpdates(self, skillToCheck: str = None) -> List[str]: skillsToUpdate = list() for skillName in self._skillList: - if not self.isSkillActive(skillName=skillName): - continue - try: if skillToCheck and skillName != skillToCheck: continue @@ -939,240 +890,59 @@ def checkForSkillUpdates(self, skillToCheck: str = None) -> List[str]: return skillsToUpdate - @Online(catchOnly=True) - def _checkForSkillInstall(self): - # Don't start the install timer from the main thread in case it's the first start - if self._skillInstallThread: - self.ThreadManager.newTimer(interval=10, func=self._checkForSkillInstall, autoStart=True) - - root = Path(self.Commons.rootDir(), constants.SKILL_INSTALL_TICKET_PATH) - files = [f for f in root.iterdir() if f.suffix == '.install'] - - if self._busyInstalling.isSet() or not files or self.ProjectAlice.restart or self.ProjectAlice.updating or self.NluManager.training: - return - - self.logInfo(f'Found {len(files)} install ticket', plural='ticket') - self._busyInstalling.set() - - skillsToBoot = dict() - try: - skillsToBoot = self._installSkillTickets(files) - except Exception as e: - self._logger.logError(f'Error installing skill: {e}') - finally: - if skillsToBoot and self.ProjectAlice.isBooted: - self._finishInstall(skillsToBoot, True) - else: - self._postBootSkillActions = skillsToBoot.copy() - - self._busyInstalling.clear() - - - def _finishInstall(self, skills: dict = None, startSkill: bool = False): - if not skills and not self._postBootSkillActions: - return - - if not skills and self._postBootSkillActions: - skills = self._postBootSkillActions - - for skillName, info in skills.items(): - - if startSkill: - self.initSkills(onlyInit=skillName, reload=info['update']) - self.ConfigManager.loadCheckAndUpdateSkillConfigurations(skillToLoad=skillName) - - with suppress(SkillStartDelayed): - self._startSkill(skillName) - - if info['update']: - self.allSkills[skillName].onSkillUpdated(skill=skillName) - self.MqttManager.mqttBroadcast( - topic=constants.TOPIC_SKILL_UPDATED, - payload={ - 'skillName': skillName - } - ) - - self.WebUINotificationManager.newNotification( - typ=UINotificationType.INFO, - notification='skillUpdated', - key='skillUpdate_{}'.format(skillName), - replaceBody=[skillName, json.loads(self.getSkillInstallFile(skillName=skillName).read_text())['version']] - ) - else: - self.allSkills[skillName].onSkillInstalled(skill=skillName) - self.MqttManager.mqttBroadcast( - topic=constants.TOPIC_SKILL_INSTALLED, - payload={ - 'skillName': skillName - } - ) - - self.allSkills[skillName].onBooted() + def getSkillInstance(self, skillName: str, silent: bool = False) -> Optional[AliceSkill]: + if skillName in self._activeSkills: + return self._activeSkills[skillName] + elif skillName in self._deactivatedSkills: + return self._deactivatedSkills[skillName] + elif skillName in self._failedSkills: + return self._failedSkills[skillName] + else: + if not silent: + self.logWarning(f'Skill "{skillName}" does not exist in skills manager') - self._postBootSkillActions = dict() - self.AssistantManager.checkAssistant() + return None - @deprecated - def _installSkillTickets(self, skills: list) -> dict: + def skillBroadcast(self, method: str, filterOut: list = None, **kwargs): """ - Installs the skills from found install tickets - :param skills: list of tickets + Broadcasts a call to the given method on every skill + :param filterOut: array, skills not to broadcast to + :param method: str, the method name to call on every skill :return: """ - root = Path(self.Commons.rootDir(), constants.SKILL_INSTALL_TICKET_PATH) - skillsToBoot = dict() - self.MqttManager.mqttBroadcast(topic=constants.TOPIC_SYSTEM_UPDATE, payload={'sticky': True}) - for file in skills: - skillName = Path(file).stem - - self.logInfo(f'Now taking care of skill {skillName}') - res = root / file - - try: - installFile = json.loads(res.read_text()) - - skillName = installFile['name'] - - if not skillName: - self.logError('Skill name to install not found, aborting to avoid casualties!') - continue - - if skillName in self._skillList: - installedVersion = Version.fromString(self._skillList[skillName]['installer']['version']) - remoteVersion = Version.fromString(installFile['version']) - - if installedVersion >= remoteVersion: - self.logWarning(f'Skill "{skillName}" is already installed and up to date, skipping') - self.Commons.runRootSystemCommand(['rm', res]) - continue - else: - self.logWarning(f'Skill "{skillName}" installed but needs updating') - updating = True - - self.MqttManager.mqttBroadcast( - topic=constants.TOPIC_SKILL_UPDATING, - payload={ - 'skillName': skillName - } - ) - else: - updating = False - - self.checkSkillConditions(installFile) - - try: - skillRepository = self.getSkillRepository(skillName=skillName) - except GithubNotFound: - if self.ConfigManager.getAliceConfigByName('devMode'): - if not Path(f'{self.Commons.rootDir}/skills/{skillName}').exists() or not \ - Path(f'{self.Commons.rootDir}/skills/{skillName}/{skillName.py}').exists() or not \ - Path(f'{self.Commons.rootDir}/skills/{skillName}/dialogTemplate').exists() or not \ - Path(f'{self.Commons.rootDir}/skills/{skillName}/talks').exists(): - self.logWarning(f'Skill "{skillName}" cannot be installed in dev mode due to missing base files') - else: - self._installSkill(res) - skillsToBoot[skillName] = { - 'update': updating - } - continue - else: - self.logWarning(f'Skill "{skillName}" is not available on Github, cannot install') - raise - - - except SkillNotConditionCompliant as e: - self.logInfo(f'Skill "{skillName}" does not comply to "{e.condition}" condition, required "{e.conditionValue}"') - if res.exists(): - res.unlink() - - self.broadcast( - method=constants.EVENT_SKILL_INSTALL_FAILED, - exceptions=self._name, - skill=skillName - ) - - except Exception: - self.logError(f'Failed installing skill "{skillName}"') - if res.exists(): - res.unlink() - - self.broadcast( - method=constants.EVENT_SKILL_INSTALL_FAILED, - exceptions=self.name, - skill=skillName - ) - raise - - return skillsToBoot - - - - - - - - - def onSkillInstalled(self, skill: str): - self.WebUINotificationManager.newNotification( - typ=UINotificationType.INFO, - notification='skillInstalled', - key=f'skillUpdate_{skill}', - replaceBody=[skill] - ) - - - def onSkillUpdated(self, skill: str): - self.WebUINotificationManager.newNotification( - typ=UINotificationType.INFO, - notification='skillUpdated', - key=f'skillUpdate_{skill}', - replaceBody=[skill, json.loads(self.getSkillInstallFile(skillName=skill).read_text())['version']] - ) - - - + if not method.startswith('on'): + method = f'on{method[0].capitalize() + method[1:]}' - def configureSkillIntents(self, skillName: str, state: bool): - try: - skills = self.allWorkingSkills - confs = [{ - 'intentId': intent.justTopic if isinstance(intent, Intent.Intent) else intent.split('/')[-1], - 'enable' : state - } for intent in skills[skillName].supportedIntents if not self.isIntentInUse(intent=intent, filtered=[skillName])] + for skillName, skillInstance in self._activeSkills.items(): - self.MqttManager.configureIntents(confs) + if filterOut and skillName in filterOut: + continue - if state: - skills[skillName].subscribeIntents() - else: - skills[skillName].unsubscribeIntents() - except Exception as e: - self.logWarning(f'Intent configuration failed: {e}') + try: + func = getattr(skillInstance, method, None) + if func: + func(**kwargs) + func = getattr(skillInstance, 'onEvent', None) + if func: + func(event=method, **kwargs) - def isIntentInUse(self, intent: Intent, filtered: list) -> bool: - skills = self.allWorkingSkills - return any(intent in skill.supportedIntents for name, skill in skills.items() if name not in filtered) + except TypeError as e: + self.logWarning(f'Failed to broadcast event {method} to {skillName}: {e}') def removeSkill(self, skillName: str): if skillName not in self.allSkills: return - if skillName in self._activeSkills: - self._activeSkills[skillName].onStop() - self.broadcast(method=constants.EVENT_SKILL_STOPPED, exceptions=[self.name], propagateToSkills=True, skill=self) - - self.broadcast(method=constants.EVENT_SKILL_DELETED, exceptions=[self.name], propagateToSkills=True, skill=skillName) - - self.MqttManager.mqttBroadcast( - topic=constants.TOPIC_SKILL_DELETED, - payload={ - 'skillName': skillName - } + self.deactivateSkill(skillName=skillName, persistent=False) + self.broadcast( + method=constants.EVENT_SKILL_DELETED, + exceptions=[self.name], + propagateToSkills=True, + skill=skillName ) self._skillList.remove(skillName) @@ -1181,7 +951,9 @@ def removeSkill(self, skillName: str): self._failedSkills.pop(skillName, None) self.removeSkillFromDB(skillName=skillName) - shutil.rmtree(Path(self.Commons.rootDir(), 'skills', skillName)) + repo = self.getSkillRepository(skillName=skillName) + if repo: + repo.destroy() self.AssistantManager.checkAssistant() @@ -1189,15 +961,10 @@ def removeSkill(self, skillName: str): def reloadSkill(self, skillName: str): self.logInfo(f'Reloading skill "{skillName}"') - if skillName in self._activeSkills: - self._activeSkills[skillName].onStop() - self.broadcast(method=constants.EVENT_SKILL_STOPPED, exceptions=[self.name], propagateToSkills=True, skill=self) - + self.stopSkill(skillName=skillName) self.initSkills(onlyInit=skillName, reload=True) - self.AssistantManager.checkAssistant() - - self._startSkill(skillName=skillName) + self.startSkill(skillName=skillName) def allScenarioNodes(self) -> Dict[str, tuple]: @@ -1232,6 +999,10 @@ def getSkillScenarioVersion(self, skillName: str) -> Version: def wipeSkills(self): + """ + Lazy version to delete all skill, remove the entire directory and recreate it + :return: + """ shutil.rmtree(Path(self.Commons.rootDir(), 'skills')) Path(self.Commons.rootDir(), 'skills').mkdir() @@ -1242,6 +1013,84 @@ def wipeSkills(self): self._deactivatedSkills = dict() self._failedSkills = dict() self._skillList = dict() + + + def isSkillUserModified(self, skillName: str) -> bool: + """ + Checks git status to see if the skill was modified from original online + :param skillName: + :return: + """ + try: + repository = self.getSkillRepository(skillName=skillName) + return repository.isDirty() + except: + return False + + + + + + + + + + + + # def onAssistantInstalled(self, **kwargs): + # argv = kwargs.get('skillsInfos', dict()) + # if not argv: + # return + # + # for skillName, skill in argv.items(): + # try: + # self._startSkill(skillName=skillName) + # except SkillStartDelayed: + # self.logInfo(f'Skill "{skillName}" start is delayed') + # except KeyError as e: + # self.logError(f'Skill "{skillName} not found, skipping: {e}') + # continue + # + # self._activeSkills[skillName].onBooted() + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + def createNewSkill(self, skillDefinition: dict) -> bool: @@ -1338,7 +1187,6 @@ def createNewSkill(self, skillDefinition: dict) -> bool: self._failedSkills[skillName] = FailedAliceSkill(data) self._skillList[skillName] = { 'active' : False, - 'modified' : True, 'installer': data } self._failedSkills[skillName].modified = True @@ -1391,39 +1239,9 @@ def uploadSkillToGithub(self, skillName: str, skillDesc: str) -> bool: return False - @deprecated - def downloadInstallTicket(self, skillName: str, isUpdate: bool = False) -> bool: - try: - tmpFile = Path(self.Commons.rootDir(), f'system/skillInstallTickets/{skillName}.install') - if not self.Commons.downloadFile( - url=f'{constants.GITHUB_RAW_URL}/skill_{skillName}/{self.SkillStoreManager.getSkillUpdateTag(skillName)}/{skillName}.install', - dest=str(tmpFile.with_suffix('.tmp')) - ): - raise Exception - - if not isUpdate: - requests.get(f'https://skills.projectalice.ch/{skillName}') - - shutil.move(tmpFile.with_suffix('.tmp'), tmpFile) - return True - except Exception as e: - self.logError(f'Error downloading install ticket for skill "{skillName}": {e}') - return False - - @deprecated def setSkillModified(self, skillName: str, modified: bool): return - def isSkillUserModified(self, skillName: str) -> bool: - """ - Checks git status to see if the skill was modified from original online - :param skillName: - :return: - """ - try: - repository = self.getSkillRepository(skillName=skillName) - return repository.isDirty() - except: - return False + diff --git a/core/base/model/AliceSkill.py b/core/base/model/AliceSkill.py index e4bbb7f28..de8b44ae3 100644 --- a/core/base/model/AliceSkill.py +++ b/core/base/model/AliceSkill.py @@ -599,7 +599,6 @@ def onStop(self): self._active = False self.SkillManager.configureSkillIntents(self._name, False) self.logInfo(f'![green](Stopped)') - self.broadcast(method=constants.EVENT_SKILL_STOPPED, exceptions=[self.name], propagateToSkills=True, skill=self) def onBooted(self) -> bool: @@ -610,6 +609,11 @@ def onBooted(self) -> bool: return True + def onSkillStopped(self, skill): + if skill == self._name: + self.onStop() + + def onSkillInstalled(self, **kwargs): self.onSkillUpdated(**kwargs) diff --git a/core/base/model/ProjectAliceObject.py b/core/base/model/ProjectAliceObject.py index d4837bdb9..82bbcf710 100644 --- a/core/base/model/ProjectAliceObject.py +++ b/core/base/model/ProjectAliceObject.py @@ -19,14 +19,14 @@ from __future__ import annotations +from copy import copy + import json import re -from copy import copy +from importlib_metadata import PackageNotFoundError, version as packageVersion from pathlib import Path from typing import TYPE_CHECKING, Union -from importlib_metadata import PackageNotFoundError, version as packageVersion - import core.base.SuperManager as SM from core.base.model.Version import Version from core.commons import constants @@ -314,6 +314,10 @@ def onSkillUpdated(self, skill: str): pass # Super object function is overridden only if needed + def onSkillUpdating(self, skill: str): + pass # Super object function is overridden only if needed + + def onInternetConnected(self): pass # Super object function is overridden only if needed @@ -673,6 +677,20 @@ def onSkillStopped(self, skill): pass # Super object function is overridden only if needed + def onSkillActivated(self, skill): + """ + :param skill: AliceSkill instance + """ + pass # Super object function is overridden only if needed + + + def onSkillDeactivated(self, skill): + """ + :param skill: AliceSkill instance + """ + pass # Super object function is overridden only if needed + + @property def ProjectAlice(self) -> ProjectAlice: # NOSONAR return SM.SuperManager.getInstance().projectAlice diff --git a/core/commons/constants.py b/core/commons/constants.py index 60897f65e..22ad40863 100644 --- a/core/commons/constants.py +++ b/core/commons/constants.py @@ -91,12 +91,17 @@ TOPIC_NEW_HOTWORD = 'projectalice/devices/alice/newHotword' TOPIC_NLU_TRAINING_STATUS = 'projectalice/nlu/trainingStatus' TOPIC_RESOURCE_USAGE = 'projectalice/devices/resourceUsage' +TOPIC_SKILL_STARTED = 'projectalice/skills/started' +TOPIC_SKILL_STOPPED = 'projectalice/skills/stopped' TOPIC_SKILL_DELETED = 'projectalice/skills/deleted' TOPIC_SKILL_INSTALLED = 'projectalice/skills/installed' +TOPIC_SKILL_INSTALL_FAILED = 'projectalice/skills/installFalied' TOPIC_SKILL_INSTRUCTIONS = 'projectalice/skills/instructions' TOPIC_SKILL_UPDATE_CORE_CONFIG_WARNING = 'projectalice/skills/coreConfigUpdateWarning' TOPIC_SKILL_UPDATED = 'projectalice/skills/updated' TOPIC_SKILL_UPDATING = 'projectalice/skills/updating' +TOPIC_SKILL_ACTIVATED = 'projectalice/skills/activated' +TOPIC_SKILL_DEACTIVATED = 'projectalice/skills/deactivated' TOPIC_STOP_DND = 'projectalice/devices/startListen' TOPIC_SYSLOG = 'projectalice/logging/syslog' TOPIC_TOGGLE_DND = 'projectalice/devices/toggleListen' @@ -154,6 +159,8 @@ EVENT_SKILL_STARTED = 'skillStarted' EVENT_SKILL_STOPPED = 'skillStopped' EVENT_SKILL_UPDATED = 'skillUpdated' +EVENT_SKILL_DEACTIVATED = 'skillDeactivated' +EVENT_SKILL_ACTIVATED = 'skillActivated' EVENT_SLEEP = 'sleep' EVENT_START_LISTENING = 'startListening' EVENT_START_SESSION = 'startSession' diff --git a/core/device/DeviceManager.py b/core/device/DeviceManager.py index e968aa20a..5fe16036f 100644 --- a/core/device/DeviceManager.py +++ b/core/device/DeviceManager.py @@ -22,10 +22,9 @@ import threading import time import uuid -from typing import Dict, List, Optional, Union - from paho.mqtt.client import MQTTMessage from serial.tools import list_ports +from typing import Dict, List, Optional, Union from core.base.model.Manager import Manager from core.commons import constants @@ -160,11 +159,11 @@ def onSkillDeleted(self, skill: str): ) - def skillDeactivated(self, skillName: str): - self.removeDeviceTypesForSkill(skillName=skillName) + def onSkillDeactivated(self, skill): + self.removeDeviceTypesForSkill(skillName=skill.name) tmp = self._devices.copy() for deviceUid, device in tmp.items(): - if device.skillName == skillName: + if device.skillName == skill.name: self._devices.pop(deviceUid, None) diff --git a/core/server/MqttManager.py b/core/server/MqttManager.py index b2ee75d3b..a06137bfa 100644 --- a/core/server/MqttManager.py +++ b/core/server/MqttManager.py @@ -17,17 +17,17 @@ # # Last modified: 2021.07.28 at 16:07:59 CEST +import traceback + import json +import paho.mqtt.client as mqtt +import paho.mqtt.publish as publish import random import re -import traceback import uuid from pathlib import Path from typing import List, Union -import paho.mqtt.client as mqtt -import paho.mqtt.publish as publish - from core.base.model.Intent import Intent from core.base.model.Manager import Manager from core.commons import constants @@ -953,3 +953,84 @@ def toggleFeedbackSounds(self, state='On'): for deviceUid in deviceList: publish.single(constants.TOPIC_TOGGLE_FEEDBACK.format(state.title()), payload=json.dumps({'siteId': deviceUid}), hostname=self.ConfigManager.getAliceConfigByName('mqttHost')) + + + def onSkillInstalled(self, skill: str): + self.mqttBroadcast( + topic=constants.TOPIC_SKILL_INSTALLED, + payload={ + 'skillName': skill + } + ) + + + def onSkillUpdated(self, skill: str): + self.mqttBroadcast( + topic=constants.TOPIC_SKILL_UPDATED, + payload={ + 'skillName': skill + } + ) + + + def onSkillUpdating(self, skill: str): + self.mqttBroadcast( + topic=constants.TOPIC_SKILL_UPDATING, + payload={ + 'skillName': skill + } + ) + + + def onSkillDeleted(self, skill: str): + self.mqttBroadcast( + topic=constants.TOPIC_SKILL_DELETED, + payload={ + 'skillName': skill + } + ) + + + def onSkillInstallFailed(self, skill: str): + self.mqttBroadcast( + topic=constants.TOPIC_SKILL_INSTALL_FAILED, + payload={ + 'skillName': skill + } + ) + + + def onSkillDeactivated(self, skill): + self.mqttBroadcast( + topic=constants.TOPIC_SKILL_DEACTIVATED, + payload={ + 'skillName': skill + } + ) + + + def onSkillActivated(self, skill): + self.mqttBroadcast( + topic=constants.TOPIC_SKILL_ACTIVATED, + payload={ + 'skillName': skill + } + ) + + + def onSkillStopped(self, skill): + self.mqttBroadcast( + topic=constants.TOPIC_SKILL_STOPPED, + payload={ + 'skillName': skill + } + ) + + + def onSkillStarted(self, skill): + self.mqttBroadcast( + topic=constants.TOPIC_SKILL_STARTED, + payload={ + 'skillName': skill + } + ) diff --git a/core/webApi/model/SkillsApi.py b/core/webApi/model/SkillsApi.py index a6a87becd..a37462d85 100644 --- a/core/webApi/model/SkillsApi.py +++ b/core/webApi/model/SkillsApi.py @@ -239,8 +239,8 @@ def put(self, skillName: str) -> Response: return jsonify(success=False, reason='skill already installed') try: - if not self.SkillManager.downloadInstallTicket(skillName): - return jsonify(success=False, reason='skill not found') + self.SkillManager.downloadSkills(skills=skillName) + self.SkillManager.startSkill(skillName=skillName) except Exception as e: self.logWarning(f'Failed installing skill: {e}', printStack=True) return jsonify(success=False, message=str(e)) diff --git a/core/webui/WebUINotificationManager.py b/core/webui/WebUINotificationManager.py index 34c559336..a6d164e0c 100644 --- a/core/webui/WebUINotificationManager.py +++ b/core/webui/WebUINotificationManager.py @@ -169,3 +169,21 @@ def markAsRead(self, notificationId: int): notification = self._notifications.pop(notificationId, None) if notification: self._keysToIds.pop(notification.get('key', 'dummy'), None) + + + def onSkillUpdated(self, skill: str): + self.newNotification( + typ=UINotificationType.INFO, + notification='skillUpdated', + key='skillUpdate_{}'.format(skill), + replaceBody=[skill, self.SkillManager.getSkillInstance(skillName=skill).version] + ) + + + def onSkillInstalled(self, skill: str): + self.newNotification( + typ=UINotificationType.INFO, + notification='skillInstalled', + key=f'skillUpdate_{skill}', + replaceBody=[skill] + ) diff --git a/core/webui/WidgetManager.py b/core/webui/WidgetManager.py index 937c7bc53..99d124fb8 100644 --- a/core/webui/WidgetManager.py +++ b/core/webui/WidgetManager.py @@ -277,10 +277,10 @@ def onSkillDeleted(self, skill: str): ) - def skillDeactivated(self, skillName: str): + def onSkillDeactivated(self, skill): tmp = self._widgets.copy() for wid, widget in tmp.items(): - if widget.skill == skillName: + if widget.skill == skill.name: self._widgets.pop(wid, None) From ef2be0dffd45f28467bd7e426acfd5a882403037 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Sun, 14 Nov 2021 16:09:12 +0100 Subject: [PATCH 030/129] Fixing flow, skills are now loading, updating and all correctly. Now onto re-adding new skill stuff --- core/base/SkillManager.py | 18 +++++++++++------- core/base/model/Git.py | 32 ++++++++++++++++++-------------- 2 files changed, 29 insertions(+), 21 deletions(-) diff --git a/core/base/SkillManager.py b/core/base/SkillManager.py index 957001224..c4504eb7f 100644 --- a/core/base/SkillManager.py +++ b/core/base/SkillManager.py @@ -19,10 +19,9 @@ import getpass -import traceback - import importlib import json +import traceback from contextlib import suppress from typing import Dict, Optional @@ -467,12 +466,12 @@ def initSkills(self, onlyInit: str = '', reload: bool = False): if skillName in self.NEEDED_SKILLS: skillInstance.required = True - self.ConfigManager.loadCheckAndUpdateSkillConfigurations(skillToLoad=skillName) - if skillActiveState: self._activeSkills[skillInstance.name] = skillInstance else: self._deactivatedSkills[skillName] = skillInstance + + self.ConfigManager.loadCheckAndUpdateSkillConfigurations(skillToLoad=skillName) else: if skillName in self.NEEDED_SKILLS: self.logFatal(f'The skill is required to continue...') @@ -623,6 +622,9 @@ def updateSkills(self, skills: Union[str, List[str]], withSkillRestart: bool = T ) self.initSkills(onlyInit=skillName, reload=True) + if skillName in self.activeSkills: + self.logInfo(f'Updated skill **{skillName}** to version **{self.activeSkills[skillName].version}**') + if withSkillRestart: self.startSkill(skillName=skillName) @@ -862,8 +864,10 @@ def checkForSkillUpdates(self, skillToCheck: str = None) -> List[str]: replaceBody=[skillName, str(remoteVersion)] ) - if self.isSkillUserModified(skillName=skillName): - self.allSkills[skillName].updateAvailable = True + if self.isSkillUserModified(skillName=skillName) and self.ConfigManager.getAliceConfigByName('devMode'): + if skillName in self.allSkills: + self.allSkills[skillName].updateAvailable = True + self.logInfo(f'![blue]({skillName}) - Version {installer["version"]} < {str(remoteVersion)} in {self.ConfigManager.getAliceConfigByName("skillsUpdateChannel")} - Locked for local changes!') continue @@ -875,7 +879,7 @@ def checkForSkillUpdates(self, skillToCheck: str = None) -> List[str]: else: skillsToUpdate.append(skillName) else: - if self.isSkillUserModified(skillName=skillName): + if self.isSkillUserModified(skillName=skillName) and self.ConfigManager.getAliceConfigByName('devMode'): self.logInfo(f'![blue]({skillName}) - Version {installer["version"]} in {self.ConfigManager.getAliceConfigByName("skillsUpdateChannel")} - Locked for local changes!') else: self.logInfo(f'![green]({skillName}) - Version {installer["version"]} in {self.ConfigManager.getAliceConfigByName("skillsUpdateChannel")}') diff --git a/core/base/model/Git.py b/core/base/model/Git.py index fffb31841..7cd0ccac3 100644 --- a/core/base/model/Git.py +++ b/core/base/model/Git.py @@ -23,7 +23,7 @@ import stat import subprocess from pathlib import Path -from typing import Callable, List, Union +from typing import Callable, List, Tuple, Union import requests @@ -81,9 +81,9 @@ def __init__(self, directory: Union[str, Path], makeDir: bool = False, init: boo self._quiet = quiet self.url = url - tags = self.execute('git tag') + tags = self.execute('git tag')[0] self.tags = set(tags.split('\n')) - branches = self.execute('git branch') + branches = self.execute('git branch')[0] self.branches = set(branches.split('\n')) @@ -125,14 +125,11 @@ def isRepository(directory: Union[str, Path]) -> bool: expected = [ 'hooks', 'info', - 'logs', 'objects', 'refs', 'config', 'description', - 'HEAD', - 'index', - 'packed-refs' + 'HEAD' ] for item in expected: @@ -147,6 +144,9 @@ def checkout(self, branch: str = 'master', tag: str = '', force: bool = False): else: target = branch + if not target: + raise Exception('Checkout target cannot be empty§') + if self.isDirty(): if not force: raise DirtyRepository() @@ -166,13 +166,13 @@ def isDirty(self) -> bool: def revert(self): - self.restore() + self.reset() self.clean() self.execute('git checkout HEAD') def listStash(self) -> List[str]: - result = self.execute(f'git stash list') + result = self.execute(f'git stash list')[0] return result.split('\n') @@ -197,7 +197,11 @@ def pull(self, force: bool = False): else: self.revert() - self.execute(f'git pull') + self.execute('git pull') + + + def reset(self): + self.execute('git reset --hard') def clean(self, removeUntrackedFiles: bool = True, removeUntrackedDirectory: bool = True, removeIgnored: bool = False): @@ -215,7 +219,7 @@ def clean(self, removeUntrackedFiles: bool = True, removeUntrackedDirectory: boo def restore(self): - self.execute(f'git restore {str(self.path)}') + self.execute(f'git restore {str(self.path)}', noDashCOption=True) def destroy(self): @@ -231,15 +235,15 @@ def fixPermissions(func: Callable, path: Path, *_args): raise # NOSONAR - def execute(self, command: str) -> str: - if not command.startswith('git -C'): + def execute(self, command: str, noDashCOption: bool = False) -> Tuple[str, str]: + if not command.startswith('git -C') and not noDashCOption: command = command.replace('git', f'git -C {str(self.path)}', 1) if self._quiet: command = f'{command} --quiet' result = subprocess.run(command.split(), capture_output=True, text=True) - return result.stdout.strip() + return result.stdout.strip(), result.stderr.strip() def add(self): From e0cba243297d334c480a2264a47338bae9e413fb Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Sun, 14 Nov 2021 16:18:24 +0100 Subject: [PATCH 031/129] Fix wrong column name for widgets and finally made the "changed size during iteration" error when stopping disappear --- core/base/SuperManager.py | 2 ++ core/webui/WidgetManager.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/core/base/SuperManager.py b/core/base/SuperManager.py index 95524f518..6c455c1a2 100644 --- a/core/base/SuperManager.py +++ b/core/base/SuperManager.py @@ -269,6 +269,8 @@ def onStop(self): mqttManager.onStop() except KeyError as e: Logger().logWarning(f'Manager **{managerName}** was not running: {e}') + except RuntimeError: + pass # We know, the dict changed size during iteration blabla except Exception as e: Logger().logError(f'Error while shutting down manager **{managerName}**: {e}') traceback.print_exc() diff --git a/core/webui/WidgetManager.py b/core/webui/WidgetManager.py index 99d124fb8..6ffdd10f7 100644 --- a/core/webui/WidgetManager.py +++ b/core/webui/WidgetManager.py @@ -93,7 +93,7 @@ def loadWidgets(self): tableName=self.WIDGETS_TABLE, callerName=self.name, values={ - 'parent': widget['skill'], + 'skill' : widget['skill'], 'name' : widget['name'] } ) From b980c6500c45db1a78958068c65b8fc52cbd15d1 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Sun, 14 Nov 2021 16:45:53 +0100 Subject: [PATCH 032/129] Fix all known issues when shutting down Alice --- core/base/SkillManager.py | 8 +++++--- core/base/SuperManager.py | 8 +++++--- core/device/DeviceManager.py | 10 +++++----- core/util/SubprocessManager.py | 3 ++- 4 files changed, 17 insertions(+), 12 deletions(-) diff --git a/core/base/SkillManager.py b/core/base/SkillManager.py index c4504eb7f..f73bb4ad2 100644 --- a/core/base/SkillManager.py +++ b/core/base/SkillManager.py @@ -150,7 +150,7 @@ def onBooted(self): def onStop(self): super().onStop() - for skillName in self._activeSkills: + for skillName in list(self._activeSkills.keys()): self.stopSkill(skillName=skillName) @@ -635,6 +635,7 @@ def stopSkill(self, skillName: str) -> Optional[AliceSkill]: skill = None if skillName in self._activeSkills: skill = self._activeSkills.pop(skillName, None) + skill.onStop() self.broadcast( method=constants.EVENT_SKILL_STOPPED, exceptions=[constants.DUMMY], @@ -659,7 +660,8 @@ def configureSkillIntents(self, skillName: str, state: bool): else: skills[skillName].unsubscribeIntents() except Exception as e: - self.logWarning(f'Intent configuration failed: {e}') + if not self.ProjectAlice.shuttingDown: + self.logWarning(f'Intent configuration failed: {e}') def isIntentInUse(self, intent: Intent, filtered: list) -> bool: @@ -902,7 +904,7 @@ def getSkillInstance(self, skillName: str, silent: bool = False) -> Optional[Ali elif skillName in self._failedSkills: return self._failedSkills[skillName] else: - if not silent: + if not silent and not self.ProjectAlice.shuttingDown: self.logWarning(f'Skill "{skillName}" does not exist in skills manager') return None diff --git a/core/base/SuperManager.py b/core/base/SuperManager.py index 6c455c1a2..53edc8eb0 100644 --- a/core/base/SuperManager.py +++ b/core/base/SuperManager.py @@ -260,7 +260,10 @@ def initManagers(self): def onStop(self): managerName = constants.UNKNOWN_MANAGER try: - mqttManager = self._managers.pop('MqttManager') + mqttManager = self._managers.pop('MqttManager') # Mqtt goes down as last + + skillManager = self._managers.pop('SkillManager') # Skill manager goes down first, to tell the skills + skillManager.onStop() for managerName, manager in self._managers.items(): manager.onStop() @@ -269,8 +272,7 @@ def onStop(self): mqttManager.onStop() except KeyError as e: Logger().logWarning(f'Manager **{managerName}** was not running: {e}') - except RuntimeError: - pass # We know, the dict changed size during iteration blabla + except Exception as e: Logger().logError(f'Error while shutting down manager **{managerName}**: {e}') traceback.print_exc() diff --git a/core/device/DeviceManager.py b/core/device/DeviceManager.py index 5fe16036f..fcfa5e956 100644 --- a/core/device/DeviceManager.py +++ b/core/device/DeviceManager.py @@ -22,9 +22,10 @@ import threading import time import uuid +from typing import Dict, List, Optional, Union + from paho.mqtt.client import MQTTMessage from serial.tools import list_ports -from typing import Dict, List, Optional, Union from core.base.model.Manager import Manager from core.commons import constants @@ -148,7 +149,6 @@ def onStop(self): def onSkillDeleted(self, skill: str): - self.skillDeactivated(skillName=skill) # noinspection SqlResolve self.DatabaseManager.delete( tableName=self.DB_DEVICE, @@ -178,8 +178,8 @@ def loadDevices(self): klass = getattr(skillImport, data.get('typeName')) device = klass(data) self._devices[device.id] = device - except Exception as e: - self.logError(f"Couldn't create device instance: {e}") + except Exception: + self.logError("Couldn't create device instance") def loadLinks(self): @@ -263,7 +263,7 @@ def registerDeviceType(self, skillName: str, data: dict): self.logError('Cannot register new device type without a type name and a skill name') return - # Try to create the device type, if overriden by the user, else fallback to default generic type + # Try to create the device type, if overridden by the user, else fallback to default generic type try: skillImport = importlib.import_module(f'skills.{skillName}.device.type.{data.get("deviceTypeName")}') klass = getattr(skillImport, data.get('deviceTypeName')) diff --git a/core/util/SubprocessManager.py b/core/util/SubprocessManager.py index 39312b319..c57bb663f 100644 --- a/core/util/SubprocessManager.py +++ b/core/util/SubprocessManager.py @@ -47,11 +47,12 @@ def onStart(self): def onStop(self): super().onStop() + self._flag.clear() + for subproc in self._subproc.values(): if subproc.process is not None and subproc.process.poll() is None: subproc.process.terminate() - self._flag.clear() if self._thread and self._thread.is_alive(): self.ThreadManager.terminateThread(name='subprocessManager') From 859d24426494e299a498ee5229193ae83e0de350 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Sun, 14 Nov 2021 16:51:11 +0100 Subject: [PATCH 033/129] this is dup code --- core/base/SkillManager.py | 1 + 1 file changed, 1 insertion(+) diff --git a/core/base/SkillManager.py b/core/base/SkillManager.py index f73bb4ad2..64cb30355 100644 --- a/core/base/SkillManager.py +++ b/core/base/SkillManager.py @@ -1203,6 +1203,7 @@ def createNewSkill(self, skillDefinition: dict) -> bool: return False + @deprecated # this is available on AliceSK def uploadSkillToGithub(self, skillName: str, skillDesc: str) -> bool: try: self.logInfo(f'Uploading {skillName} to Github') From afa4dfad4a610ea87eda74fa751e8ab7dca8b359 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Mon, 15 Nov 2021 12:42:46 +0100 Subject: [PATCH 034/129] Use AliceGit deployed on pypi --- core/base/SkillManager.py | 16 ++- core/base/model/Git.py | 293 -------------------------------------- requirements.txt | 3 +- 3 files changed, 12 insertions(+), 300 deletions(-) delete mode 100644 core/base/model/Git.py diff --git a/core/base/SkillManager.py b/core/base/SkillManager.py index 64cb30355..89fcfb0a0 100644 --- a/core/base/SkillManager.py +++ b/core/base/SkillManager.py @@ -19,18 +19,22 @@ import getpass +import traceback + import importlib import json -import traceback +import requests +import shutil from contextlib import suppress -from typing import Dict, Optional +from pathlib import Path +from typing import Dict, List, Optional, Union +from AliceGit import AliceGit from core.ProjectAliceExceptions import AccessLevelTooLow, GithubNotFound, SkillInstanceFailed, SkillNotConditionCompliant, SkillStartDelayed, SkillStartingFailed from core.base.SuperManager import SuperManager from core.base.model import Intent from core.base.model.AliceSkill import AliceSkill from core.base.model.FailedAliceSkill import FailedAliceSkill -from core.base.model.Git import * from core.base.model.GithubCloner import GithubCloner from core.base.model.Manager import Manager from core.base.model.Version import Version @@ -307,7 +311,7 @@ def installSkills(self, skills: Union[str, List[str]]): self._busyInstalling.clear() - def getSkillRepository(self, skillName: str, directory: str = None) -> Optional[Git]: + def getSkillRepository(self, skillName: str, directory: str = None) -> Optional[AliceGit]: """ Returns a Git object for the given skill :param skillName: @@ -319,7 +323,7 @@ def getSkillRepository(self, skillName: str, directory: str = None) -> Optional[ directory = self.getSkillDirectory(skillName=skillName) try: - return Git(directory=directory) + return AliceGit(directory=directory) except: return None @@ -352,7 +356,7 @@ def downloadSkills(self, skills: Union[str, List[str]]) -> Optional[Dict]: source = self.getGitRemoteSourceUrl(skillName=skillName, doAuth=False) repository = self.getSkillRepository(skillName=skillName) if not repository: - repository = Git.clone(url=source, directory=self.getSkillDirectory(skillName=skillName), makeDir=True) + repository = AliceGit.clone(url=source, directory=self.getSkillDirectory(skillName=skillName), makeDir=True) repository.checkout(tag=tag) repositories[skillName] = repository diff --git a/core/base/model/Git.py b/core/base/model/Git.py deleted file mode 100644 index 7cd0ccac3..000000000 --- a/core/base/model/Git.py +++ /dev/null @@ -1,293 +0,0 @@ -# Copyright (c) 2021 -# -# This file, Git.py, is part of Project Alice. -# -# Project Alice is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see -# -# Last modified: 2021.11.10 at 14:35:51 CET -from __future__ import annotations - -import os -import shutil -import stat -import subprocess -from pathlib import Path -from typing import Callable, List, Tuple, Union - -import requests - - -class PathNotFoundException(Exception): - def __init__(self, path: Path): - super().__init__(f'Path "{path}" does not exist') - - -class NotGitRepository(Exception): - def __init__(self, path: Path): - super().__init__(f'Directory "{path}" is not a git repository') - - -class AlreadyGitRepository(Exception): - def __init__(self, path: Path): - super().__init__(f'Directory "{path}" is already a git repository') - - -class InvalidUrl(Exception): - def __init__(self, url: str): - super().__init__(f'The provided url "{url}" is not valid') - - -class DirtyRepository(Exception): - def __init__(self): - super().__init__(f'The repository is dirty. Either use the force option or stash your changes before trying again') - - -class Git: - - def __init__(self, directory: Union[str, Path], makeDir: bool = False, init: bool = False, url: str = '', quiet: bool = True): - if isinstance(directory, str): - directory = Path(directory) - - if not directory.exists() and not makeDir: - raise PathNotFoundException(directory) - - if directory.exists() and not Path(directory, '.git').exists() and not init: - raise NotGitRepository(directory) - - directory.mkdir(parents=True, exist_ok=True) - - isRepository = self.isRepository(directory=directory) - if init: - if not isRepository: - self.execute(f'git init') - else: - raise AlreadyGitRepository - else: - if not isRepository: - raise NotGitRepository - - self.path = directory - self._quiet = quiet - self.url = url - - tags = self.execute('git tag')[0] - self.tags = set(tags.split('\n')) - branches = self.execute('git branch')[0] - self.branches = set(branches.split('\n')) - - - @classmethod - def clone(cls, url: str, directory: Union[str, Path], branch: str = 'master', makeDir: bool = False, force: bool = False, quiet: bool = True) -> Git: - if isinstance(directory, str): - directory = Path(directory) - - response = requests.get(url) - if response.status_code != 200: - raise InvalidUrl(url) - - if not directory.exists() and not makeDir: - raise PathNotFoundException(directory) - - if cls.isRepository(directory=directory): - if not force: - raise AlreadyGitRepository(directory) - else: - shutil.rmtree(str(directory), onerror=cls.fixPermissions) - - directory.mkdir(parents=True, exist_ok=True) - cmd = f'git clone {url} {str(directory)} --branch {branch} --recurse-submodules' - if quiet: - cmd = f'{cmd} --quiet' - subprocess.run(cmd) - return Git(directory=directory, url=url, quiet=quiet) - - - @staticmethod - def isRepository(directory: Union[str, Path]) -> bool: - if directory and isinstance(directory, str): - directory = Path(directory) - - gitDir = directory / '.git' - if not gitDir.exists(): - return False - - expected = [ - 'hooks', - 'info', - 'objects', - 'refs', - 'config', - 'description', - 'HEAD' - ] - - for item in expected: - if not Path(gitDir, item).exists(): - return False - return True - - - def checkout(self, branch: str = 'master', tag: str = '', force: bool = False): - if tag: - target = f'tags/{tag} -B Branch_{tag}' - else: - target = branch - - if not target: - raise Exception('Checkout target cannot be empty§') - - if self.isDirty(): - if not force: - raise DirtyRepository() - else: - self.revert() - - self.execute(f'git checkout {target} --recurse-submodules') - - - def status(self) -> Status: - return Status(directory=self.path) - - - def isDirty(self) -> bool: - status = self.status() - return status.isDirty() - - - def revert(self): - self.reset() - self.clean() - self.execute('git checkout HEAD') - - - def listStash(self) -> List[str]: - result = self.execute(f'git stash list')[0] - return result.split('\n') - - - def stash(self) -> int: - self.execute(f'git stash push {str(self.path)}/') - return len(self.listStash()) - 1 - - - def dropStash(self, index: Union[int, str] = -1) -> List[str]: - if index == 'all': - self.execute(f'git stash clear') - return list() - else: - self.execute(f'git stash drop {index}') - return self.listStash() - - - def pull(self, force: bool = False): - if self.isDirty(): - if not force: - raise DirtyRepository() - else: - self.revert() - - self.execute('git pull') - - - def reset(self): - self.execute('git reset --hard') - - - def clean(self, removeUntrackedFiles: bool = True, removeUntrackedDirectory: bool = True, removeIgnored: bool = False): - options = '' - if removeUntrackedFiles: - options += 'f' - if removeUntrackedDirectory: - options += 'd' - if removeIgnored: - options += 'x' - if options: - options = f'-{options}' - - self.execute(f'git clean {options}') - - - def restore(self): - self.execute(f'git restore {str(self.path)}', noDashCOption=True) - - - def destroy(self): - shutil.rmtree(self.path, onerror=self.fixPermissions) - - - @staticmethod - def fixPermissions(func: Callable, path: Path, *_args): - if not os.access(path, os.W_OK): - os.chmod(path, stat.S_IWUSR) - func(path) - else: - raise # NOSONAR - - - def execute(self, command: str, noDashCOption: bool = False) -> Tuple[str, str]: - if not command.startswith('git -C') and not noDashCOption: - command = command.replace('git', f'git -C {str(self.path)}', 1) - - if self._quiet: - command = f'{command} --quiet' - - result = subprocess.run(command.split(), capture_output=True, text=True) - return result.stdout.strip(), result.stderr.strip() - - - def add(self): - self.execute('git add --all') - - - def commit(self, message: str = 'Commit by ProjectAliceBot', autoAdd: bool = False): - cmd = f'git commit -m "{message}"' - if autoAdd: - cmd += ' --all' - self.execute(cmd) - - - def push(self, repository: str = None): - if not repository: - repository = self.url - self.execute(f'git push --repo={repository} origin') - - - def file(self, filePath: Union[str, Path]) -> Path: - if isinstance(filePath, str): - filePath = Path(filePath) - - return self.path / filePath - - - @property - def quiet(self) -> bool: - return self._quiet - - - @quiet.setter - def quiet(self, value: bool): - self._quiet = value - - -class Status: - - def __init__(self, directory: Union[str, Path]): - if isinstance(directory, str): - directory = Path(directory) - - self._status = subprocess.run(f'git -C {str(directory)} status'.split(), capture_output=True, text=True).stdout.strip() - - - def isDirty(self): - return 'working tree clean' not in self._status diff --git a/requirements.txt b/requirements.txt index 1d93b286b..a83e72b6a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,4 +30,5 @@ dulwich~=0.20.25 botocore~=1.20.85 scipy~=1.7.1 Werkzeug~=2.0.1 -Jinja2~=3.0.1 \ No newline at end of file +Jinja2~=3.0.1 +AliceGit From a33b431017c61cda22d51f35dba3894ebc5f2a99 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Mon, 15 Nov 2021 13:57:13 +0100 Subject: [PATCH 035/129] merge conflicts --- core/webApi/model/SkillsApi.py | 3 +++ requirements.txt | 4 ++-- requirements_test.txt | 1 + 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/core/webApi/model/SkillsApi.py b/core/webApi/model/SkillsApi.py index a37462d85..e9fe942a2 100644 --- a/core/webApi/model/SkillsApi.py +++ b/core/webApi/model/SkillsApi.py @@ -19,6 +19,8 @@ import json + +import ProjectAliceSK.ProjectAliceSkillKit from flask import Response, jsonify, request from flask_classful import route from pathlib import Path @@ -104,6 +106,7 @@ def uploadToGithub(self) -> Response: if not skillName: raise Exception('Missing skill name') + if self.SkillManager.uploadSkillToGithub(skillName, skillDesc): return jsonify(success=True, url=f'https://github.com/{self.ConfigManager.getAliceConfigByName("githubUsername")}/skill_{skillName}.git') diff --git a/requirements.txt b/requirements.txt index a83e72b6a..e891682c2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,5 +30,5 @@ dulwich~=0.20.25 botocore~=1.20.85 scipy~=1.7.1 Werkzeug~=2.0.1 -Jinja2~=3.0.1 -AliceGit +projectalice-sk +AliceGit \ No newline at end of file diff --git a/requirements_test.txt b/requirements_test.txt index 922304006..da85ad310 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -25,3 +25,4 @@ webrtcvad~=2.0.10 pyjwt~=2.1.0 markdown~=3.3.4 sounddevice==0.4.2 +projectalice-sk \ No newline at end of file From d7299ceaa355141765b977feb4e1493ceffc2d11 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Mon, 15 Nov 2021 14:26:36 +0100 Subject: [PATCH 036/129] Alignment --- core/Initializer.py | 16 +++++++++++----- core/util/model/Logger.py | 2 +- main.py | 8 ++++---- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/core/Initializer.py b/core/Initializer.py index 380d9ca20..970559e01 100644 --- a/core/Initializer.py +++ b/core/Initializer.py @@ -58,27 +58,33 @@ def __getitem__(self, item): class SimpleLogger: def __init__(self, prepend: str = None): - self._prepend = f'[{prepend}]\t ' + self._prepend = f'[{prepend}]' self._logger = logging.getLogger('ProjectAlice') def logInfo(self, text: str): - self._logger.info(f'{self._prepend} {text}') + self._logger.info(f'{self.spacer(text)}') def logWarning(self, text: str): - self._logger.warning(f'{self._prepend} {text}') + self._logger.warning(f'{self.spacer(text)}') def logError(self, text: str): - self._logger.error(f'{self._prepend} {text}') + self._logger.error(f'{self.spacer(text)}') def logFatal(self, text: str): - self._logger.fatal(f'{self._prepend} {text}') + self._logger.fatal(f'{self.spacer(text)}') exit(1) + def spacer(self, msg: str) -> str: + space = ''.join([' ' for _ in range(35 - len(self._prepend) + 1)]) + msg = f'{self._prepend}{space}{msg}' + return msg + + class PreInit: """ Pre init checks and makes sure vital stuff is installed and running. Not much, but internet, venv and so on diff --git a/core/util/model/Logger.py b/core/util/model/Logger.py index b7866abb3..42ebe4449 100644 --- a/core/util/model/Logger.py +++ b/core/util/model/Logger.py @@ -78,7 +78,7 @@ def doLog(self, function: callable, msg: str, printStack=True, plural: Union[lis match = re.match(r'^(\[[\w ]+])(.*)$', msg) if match: tag, log = match.groups() - space = ''.join([' ' for _ in range(25 - len(tag))]) + space = ''.join([' ' for _ in range(35 - len(tag))]) msg = f'{tag}{space}{log}' func = getattr(self._logger, function) diff --git a/main.py b/main.py index adf0996fb..db69549c8 100644 --- a/main.py +++ b/main.py @@ -92,7 +92,7 @@ def exceptionListener(*exc_info): # NOSONAR global _logger - _logger.error('[Project Alice] An unhandled exception occured') + _logger.error('[Project Alice] An unhandled exception occurred') text = ''.join(traceback.format_exception(*exc_info)) _logger.error(f'- Traceback: {text}') @@ -125,7 +125,7 @@ def restartProcess(): os.close(handler.fd) except Exception as e: - print(f'Failed restarting Project Alice: {e}') + print(f'[Project Alice] Failed restarting Project Alice: {e}') python = sys.executable os.execl(python, python, *sys.argv) @@ -143,13 +143,13 @@ def main(): while RUNNING: time.sleep(0.1) except KeyboardInterrupt: - _logger.info('[Project Alice] Interruption detected, preparing shutdown') + _logger.info('[Project Alice] Interruption detected, preparing shutdown') finally: if projectAlice.isBooted: projectAlice.onStop() - _logger.info('[Project Alice] Shutdown completed, see you soon!') + _logger.info('[Project Alice] Shutdown completed, see you soon!') if projectAlice.restart: time.sleep(3) restartProcess() From 5d11a634a56e7ad963fa400bb0fa8b207001e3b9 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Mon, 15 Nov 2021 15:52:10 +0100 Subject: [PATCH 037/129] Pushing all this to merge this branch --- core/ProjectAlice.py | 2 +- core/asr/ASRManager.py | 2 +- core/base/ConfigManager.py | 87 ++++++++------- core/base/SkillManager.py | 190 +++++++++++---------------------- core/webApi/model/SkillsApi.py | 7 +- 5 files changed, 114 insertions(+), 174 deletions(-) diff --git a/core/ProjectAlice.py b/core/ProjectAlice.py index ea3a6f2cf..ba7ecbbec 100644 --- a/core/ProjectAlice.py +++ b/core/ProjectAlice.py @@ -173,7 +173,7 @@ def updateProjectAlice(self): self._superManager.stateManager.setState(STATE, newState=StateType.RUNNING) self._isUpdating = True - req = requests.get(url=f'{constants.GITHUB_API_URL}/ProjectAlice/branches', auth=SuperManager.getInstance().configManager.getGithubAuth()) + req = requests.get(url=f'{constants.GITHUB_API_URL}/ProjectAlice/branches', auth=SuperManager.getInstance().configManager.githubAuth) if req.status_code != 200: self._logger.logWarning('Failed checking for updates') self._superManager.stateManager.setState(STATE, newState=StateType.ERROR) diff --git a/core/asr/ASRManager.py b/core/asr/ASRManager.py index c89d2fb89..8a9fb9937 100644 --- a/core/asr/ASRManager.py +++ b/core/asr/ASRManager.py @@ -116,7 +116,7 @@ def _startASREngine(self, forceAsr=None): self.logFatal("Couldn't start any ASR, going down") return - self.logWarning(f'Something went wrong starting user ASR, falling back to **{fallback}**: {e}') + self.logWarning(f'Something went wrong starting user ASR, falling back to **{fallback}**: {e}', printStack=True) self._startASREngine(forceAsr=fallback) diff --git a/core/base/ConfigManager.py b/core/base/ConfigManager.py index 54c0f1757..e690010d0 100644 --- a/core/base/ConfigManager.py +++ b/core/base/ConfigManager.py @@ -20,8 +20,8 @@ import json import logging import re -import typing from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple, Union import sounddevice as sd @@ -48,13 +48,13 @@ def __init__(self): self._vitalConfigs = list() self._aliceConfigurationCategories = list() - self._aliceTemplateConfigurations: typing.Dict[str, dict] = self.loadJsonFromFile(self.TEMPLATE_FILE) - self._aliceConfigurations: typing.Dict[str, typing.Any] = dict() + self._aliceTemplateConfigurations: Dict[str, dict] = self.loadJsonFromFile(self.TEMPLATE_FILE) + self._aliceConfigurations: Dict[str, Any] = dict() self._loadCheckAndUpdateAliceConfigFile() self._skillsConfigurations = dict() - self._skillsTemplateConfigurations: typing.Dict[str, dict] = dict() + self._skillsTemplateConfigurations: Dict[str, dict] = dict() self._pendingAliceConfUpdates = dict() @@ -101,7 +101,7 @@ def _loadCheckAndUpdateAliceConfigFile(self): if not aliceConfigs.get('uuid', None): import uuid - ##uuid4: no collission expected until extinction of all life (only on earth though!) + ##uuid4: no collision expected until extinction of all life (only on earth though!) aliceConfigs['uuid'] = str(uuid.uuid4()) changes = True @@ -124,17 +124,17 @@ def _loadCheckAndUpdateAliceConfigFile(self): try: # First try to cast the setting we have to the new type aliceConfigs[setting] = type(definition['defaultValue'])(aliceConfigs[setting]) - self.logWarning(f'Existing configuration type missmatch: **{setting}**, cast variable to template configuration type') + self.logWarning(f'Existing configuration type mismatch: **{setting}**, cast variable to template configuration type') except Exception: # If casting failed let's fall back to the new default value - self.logWarning(f'Existing configuration type missmatch: **{setting}**, replaced with template configuration') + self.logWarning(f'Existing configuration type mismatch: **{setting}**, replaced with template configuration') aliceConfigs[setting] = definition['defaultValue'] elif definition['dataType'] == 'list' and 'onInit' not in definition: values = definition['values'].values() if isinstance(definition['values'], dict) else definition['values'] if aliceConfigs[setting] and aliceConfigs[setting] not in values: changes = True - self.logWarning(f'Selected value **{aliceConfigs[setting]}** for setting **{setting}** doesn\'t exist, reverted to default value --{definition["defaultValue"]}--') + self.logWarning(f"Selected value **{aliceConfigs[setting]}** for setting **{setting}** doesn't exist, reverted to default value --{definition['defaultValue']}--") aliceConfigs[setting] = definition['defaultValue'] function = definition.get('onInit', None) @@ -172,9 +172,9 @@ def _loadCheckAndUpdateAliceConfigFile(self): self._aliceConfigurations = aliceConfigs - def updateAliceConfigDefinitionValues(self, setting: str, value: typing.Any): + def updateAliceConfigDefinitionValues(self, setting: str, value: Any): if setting not in self._aliceTemplateConfigurations: - self.logWarning(f'Was asked to update **{setting}** from config templates, but setting doesn\'t exist') + self.logWarning(f"Was asked to update **{setting}** from config templates, but setting doesn't exist") return self._aliceTemplateConfigurations[setting]['values'] = value @@ -189,7 +189,7 @@ def loadJsonFromFile(jsonFile: Path) -> dict: raise - def updateMainDeviceName(self, value: typing.Any): + def updateMainDeviceName(self, value: Any): device = self.DeviceManager.getMainDevice() if not device.displayName: @@ -198,7 +198,7 @@ def updateMainDeviceName(self, value: typing.Any): device.updateConfigs(configs={'displayName': value}) - def updateAliceConfiguration(self, key: str, value: typing.Any, dump: bool = True, + def updateAliceConfiguration(self, key: str, value: Any, dump: bool = True, doPreAndPostProcessing: bool = True): """ Updating a core config is sensitive, if the request comes from a skill. @@ -238,7 +238,7 @@ def updateAliceConfiguration(self, key: str, value: typing.Any, dump: bool = Tru # return if key not in self._aliceConfigurations: - self.logWarning(f'Was asked to update **{key}** but key doesn\'t exist') + self.logWarning(f"Was asked to update **{key}** but key doesn't exist") raise ConfigurationUpdateFailed() pre = self.getAliceConfUpdatePreProcessing(key) @@ -264,7 +264,7 @@ def bulkUpdateAliceConfigurations(self): for key, value in self._pendingAliceConfUpdates.items(): if key not in self._aliceConfigurations: - self.logWarning(f'Was asked to update **{key}** but key doesn\'t exist') + self.logWarning(f"Was asked to update **{key}** but key doesn't exist") continue self.updateAliceConfiguration(key, value, False) @@ -276,13 +276,13 @@ def deletePendingAliceConfigurationUpdates(self): self._pendingAliceConfUpdates = dict() - def updateSkillConfigurationFile(self, skillName: str, key: str, value: typing.Any): + def updateSkillConfigurationFile(self, skillName: str, key: str, value: Any): if skillName not in self._skillsConfigurations: - self.logWarning(f'Was asked to update **{key}** in skill **{skillName}** but skill doesn\'t exist') + self.logWarning(f"Was asked to update **{key}** in skill **{skillName}** but skill doesn't exist") return if key not in self._skillsConfigurations[skillName]: - self.logWarning(f'Was asked to update **{key}** in skill **{skillName}** but key doesn\'t exist') + self.logWarning(f"Was asked to update **{key}** in skill **{skillName}** but key doesn't exist") return # Cast value to template defined type @@ -297,7 +297,7 @@ def updateSkillConfigurationFile(self, skillName: str, key: str, value: typing.A try: value = int(value) except: - self.logWarning(f'Value missmatch for config **{key}** in skill **{skillName}**') + self.logWarning(f'Value mismatch for config **{key}** in skill **{skillName}**') value = 0 elif vartype == 'float' or vartype == 'range': try: @@ -363,6 +363,7 @@ def writeToAliceConfigurationFile(self, confs: dict = None): """ confs = confs if confs else self._aliceConfigurations + # noinspection PyTypeChecker sort = dict(sorted(confs.items())) self._aliceConfigurations = sort @@ -395,7 +396,7 @@ def configSkillExists(self, configName: str, skillName: str) -> bool: return skillName in self._skillsConfigurations and configName in self._skillsConfigurations[skillName] - def getAliceConfigByName(self, configName: str) -> typing.Any: + def getAliceConfigByName(self, configName: str) -> Any: if configName in self._aliceConfigurations: return self._aliceConfigurations[configName] else: @@ -403,7 +404,7 @@ def getAliceConfigByName(self, configName: str) -> typing.Any: return '' - def getAliceConfigTemplateByName(self, configName: str) -> typing.Any: + def getAliceConfigTemplateByName(self, configName: str) -> Any: if configName in self._aliceTemplateConfigurations: return self._aliceTemplateConfigurations[configName] else: @@ -411,7 +412,7 @@ def getAliceConfigTemplateByName(self, configName: str) -> typing.Any: return '' - def getSkillConfigByName(self, skillName: str, configName: str) -> typing.Any: + def getSkillConfigByName(self, skillName: str, configName: str) -> Any: if not self._loadingDone: raise Exception(f'Loading skill configs is not yet done! Don\'t load configs in __init__, but only after onStart is called') return self._skillsConfigurations.get(skillName, dict()).get(configName, None) @@ -421,7 +422,7 @@ def getSkillConfigs(self, skillName: str) -> dict: return self._skillsConfigurations.get(skillName, dict()) - def getSkillConfigsTemplateByName(self, skillName: str, configName: str) -> typing.Any: + def getSkillConfigsTemplateByName(self, skillName: str, configName: str) -> Any: return self._skillsTemplateConfigurations.get(skillName, dict()).get(configName, None) @@ -471,10 +472,10 @@ def loadCheckAndUpdateSkillConfigurations(self, skillToLoad: str = None): try: # First try to cast the setting we have to the new type config[setting] = type(definition['defaultValue'])(config[setting]) - self.logInfo(f'- Existing configuration type missmatch for skill **{skillName}**: {setting}, cast variable to template configuration type') + self.logInfo(f'- Existing configuration type mismatch for skill **{skillName}**: {setting}, cast variable to template configuration type') except Exception: # If casting failed let's fall back to the new default value - self.logInfo(f'- Existing configuration type missmatch for skill **{skillName}**: {setting}, replaced with template configuration') + self.logInfo(f'- Existing configuration type mismatch for skill **{skillName}**: {setting}, replaced with template configuration') config[setting] = definition['defaultValue'] temp = config.copy() @@ -526,7 +527,7 @@ def changeActiveLanguage(self, toLang: str): return False - def getAliceConfigType(self, confName: str) -> typing.Optional[str]: + def getAliceConfigType(self, confName: str) -> Optional[str]: # noinspection PyTypeChecker return self._aliceConfigurations.get(confName['dataType']) @@ -539,17 +540,17 @@ def isAliceConfSensitive(self, confName: str) -> bool: return self._aliceTemplateConfigurations.get(confName, dict()).get('isSensitive', False) - def getAliceConfUpdatePreProcessing(self, confName: str) -> typing.Optional[str]: + def getAliceConfUpdatePreProcessing(self, confName: str) -> Optional[str]: # Some config need some pre processing to run some checks before saving return self._aliceTemplateConfigurations.get(confName, dict()).get('beforeUpdate', None) - def getAliceConfUpdatePostProcessing(self, confName: str) -> typing.Optional[str]: + def getAliceConfUpdatePostProcessing(self, confName: str) -> Optional[str]: # Some config need some post processing if updated while Alice is running return self._aliceTemplateConfigurations.get(confName, dict()).get('onUpdate', None) - def doConfigUpdatePreProcessing(self, function: str, value: typing.Any) -> bool: + def doConfigUpdatePreProcessing(self, function: str, value: Any) -> bool: # Call alice config pre processing functions. try: mngr = self @@ -583,7 +584,7 @@ def doConfigUpdatePreProcessing(self, function: str, value: typing.Any) -> bool: return False - def doConfigUpdatePostProcessing(self, functions: typing.Union[str, set]): + def doConfigUpdatePostProcessing(self, functions: Union[str, set]): # Call alice config post processing functions. This will call methods that are needed after a certain setting was # updated while Project Alice was running @@ -711,12 +712,6 @@ def toggleDebugLogs(self): logging.getLogger('ProjectAlice').setLevel(logging.WARN) - def getGithubAuth(self) -> tuple: - username = self.getAliceConfigByName('githubUsername') - token = self.getAliceConfigByName('githubToken') - return (username, token) if (username and token) else None - - def populateAudioInputConfig(self): try: devices = self._listAudioDevices() @@ -736,7 +731,7 @@ def populateAudioOutputConfig(self): @staticmethod - def _listAudioDevices() -> list: + def _listAudioDevices() -> List: try: devices = [device['name'] for device in sd.query_devices()] if not devices: @@ -748,20 +743,32 @@ def _listAudioDevices() -> list: @property - def aliceConfigurations(self) -> dict: + def aliceConfigurations(self) -> Dict: return self._aliceConfigurations @property - def aliceConfigurationCategories(self) -> list: + def aliceConfigurationCategories(self) -> List: return sorted(self._aliceConfigurationCategories) @property - def vitalConfigs(self) -> list: + def vitalConfigs(self) -> List: return self._vitalConfigs @property - def aliceTemplateConfigurations(self) -> dict: + def aliceTemplateConfigurations(self) -> Dict: return self._aliceTemplateConfigurations + + + @property + def githubAuth(self) -> Tuple[str, str]: + """ + Returns the users configured username and token for github as a tuple + When one of the values is not supplied None is returned. + :return: + """ + username = self.getAliceConfigByName('githubUsername') + token = self.getAliceConfigByName('githubToken') + return (username, token) if (username and token) else None diff --git a/core/base/SkillManager.py b/core/base/SkillManager.py index 89fcfb0a0..9ee4b97e9 100644 --- a/core/base/SkillManager.py +++ b/core/base/SkillManager.py @@ -18,24 +18,22 @@ # Last modified: 2021.08.02 at 06:12:17 CEST -import getpass -import traceback - import importlib import json -import requests import shutil +import traceback from contextlib import suppress from pathlib import Path from typing import Dict, List, Optional, Union -from AliceGit import AliceGit +import requests + +from AliceGit import AliceGit, Exceptions as GitErrors from core.ProjectAliceExceptions import AccessLevelTooLow, GithubNotFound, SkillInstanceFailed, SkillNotConditionCompliant, SkillStartDelayed, SkillStartingFailed from core.base.SuperManager import SuperManager from core.base.model import Intent from core.base.model.AliceSkill import AliceSkill from core.base.model.FailedAliceSkill import FailedAliceSkill -from core.base.model.GithubCloner import GithubCloner from core.base.model.Manager import Manager from core.base.model.Version import Version from core.commons import constants @@ -247,8 +245,9 @@ def installSkills(self, skills: Union[str, List[str]]): for skillName in skills: try: - repository = self.getSkillRepository(skillName=skillName) - if not repository: + try: + repository = self.getSkillRepository(skillName=skillName) + except: repositories = self.downloadSkills(skills=skillName) repository = repositories.get(skillName, None) @@ -325,7 +324,7 @@ def getSkillRepository(self, skillName: str, directory: str = None) -> Optional[ try: return AliceGit(directory=directory) except: - return None + raise def downloadSkills(self, skills: Union[str, List[str]]) -> Optional[Dict]: @@ -354,8 +353,10 @@ def downloadSkills(self, skills: Union[str, List[str]]) -> Optional[Dict]: self.checkSkillConditions(installer=installFile) source = self.getGitRemoteSourceUrl(skillName=skillName, doAuth=False) - repository = self.getSkillRepository(skillName=skillName) - if not repository: + + try: + repository = self.getSkillRepository(skillName=skillName) + except: repository = AliceGit.clone(url=source, directory=self.getSkillDirectory(skillName=skillName), makeDir=True) repository.checkout(tag=tag) @@ -444,15 +445,16 @@ def initSkills(self, onlyInit: str = '', reload: bool = False): self._failedSkills.pop(skillName, None) self._deactivatedSkills.pop(skillName, None) - installFile = self.getSkillInstallFile(skillName=skillName) - if not installFile.exists(): + installFilePath = self.getSkillInstallFilePath(skillName=skillName) + if not installFilePath.exists(): if skillName in self.NEEDED_SKILLS: self.logFatal(f'Cannot find skill install file for skill **{skillName}**. The skill is required to continue') return else: self.logWarning(f'Cannot find skill install file for skill **{skillName}**, skipping.') + continue else: - installFile = json.loads(installFile.read_text()) + installFile = json.loads(installFilePath.read_text()) try: skillActiveState = self.isSkillActive(skillName=skillName) @@ -498,8 +500,8 @@ def initSkills(self, onlyInit: str = '', reload: bool = False): continue - def getSkillInstallFile(self, skillName: str) -> Path: - return Path(self.Commons.rootDir(), f'skills/{skillName}/{skillName}.install') + def getSkillInstallFilePath(self, skillName: str) -> Path: + return self.getSkillDirectory(skillName=skillName) / f'{skillName}.install' # noinspection PyTypeChecker @@ -857,7 +859,7 @@ def checkForSkillUpdates(self, skillToCheck: str = None) -> List[str]: if skillToCheck and skillName != skillToCheck: continue - installer = json.loads(self.getSkillInstallFile(skillName=skillName).read_text()) + installer = json.loads(self.getSkillInstallFilePath(skillName=skillName).read_text()) remoteVersion = self.SkillStoreManager.getSkillUpdateVersion(skillName) localVersion = Version.fromString(installer['version']) @@ -961,8 +963,9 @@ def removeSkill(self, skillName: str): self._failedSkills.pop(skillName, None) self.removeSkillFromDB(skillName=skillName) - repo = self.getSkillRepository(skillName=skillName) - if repo: + + with suppress: + repo = self.getSkillRepository(skillName=skillName) repo.destroy() self.AssistantManager.checkAssistant() @@ -1036,71 +1039,6 @@ def isSkillUserModified(self, skillName: str) -> bool: return repository.isDirty() except: return False - - - - - - - - - - - - # def onAssistantInstalled(self, **kwargs): - # argv = kwargs.get('skillsInfos', dict()) - # if not argv: - # return - # - # for skillName, skill in argv.items(): - # try: - # self._startSkill(skillName=skillName) - # except SkillStartDelayed: - # self.logInfo(f'Skill "{skillName}" start is delayed') - # except KeyError as e: - # self.logError(f'Skill "{skillName} not found, skipping: {e}') - # continue - # - # self._activeSkills[skillName].onBooted() - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - def createNewSkill(self, skillDefinition: dict) -> bool: @@ -1109,9 +1047,9 @@ def createNewSkill(self, skillDefinition: dict) -> bool: skillName = skillDefinition['name'].capitalize() - localDirectory = Path('/home', getpass.getuser(), f'ProjectAlice/skills/{skillName}') + localDirectory = self.getSkillDirectory(skillName=skillName) if localDirectory.exists(): - raise Exception("Skill name exists locally") + raise Exception('Skill name exists locally') supportedLanguages = [ 'en' @@ -1171,35 +1109,25 @@ def createNewSkill(self, skillDefinition: dict) -> bool: 'speakableName' : skillDefinition['speakableName'], 'langs' : supportedLanguages, 'createInstructions': skillDefinition.get('instructions', False), - 'pipreq' : [req.strip() for req in skillDefinition.get('pipreq', "").split(',')], - 'sysreq' : [req.strip() for req in skillDefinition.get('sysreq', "").split(',')], + 'pipreq' : [req.strip() for req in skillDefinition.get('pipreq', '').split(',')], + 'sysreq' : [req.strip() for req in skillDefinition.get('sysreq', '').split(',')], 'widgets' : widgets, 'scenarioNodes' : scenarioNodes, 'devices' : devices, - 'outputDestination' : str(Path(self.Commons.rootDir()) / 'skills' / skillName), + 'outputDestination' : str(localDirectory), 'conditions' : conditions } dump = Path(f'/tmp/{skillName}.json') dump.write_text(json.dumps(data, ensure_ascii=False)) - self.Commons.runSystemCommand(['./venv/bin/pip', 'install', '--upgrade', 'projectalice-sk']) self.Commons.runSystemCommand(['./venv/bin/projectalice-sk', 'create', '--file', f'{str(dump)}']) self.logInfo(f'Created **{skillName}** skill') - # todo: ugly.. - data['name'] = data['skillName'] - data['author'] = data['username'] - data['desc'] = data['description'] - # ok, version never filled in frontend and every skill should be created in initial version - data['version'] = '0.0.1' - - self._failedSkills[skillName] = FailedAliceSkill(data) self._skillList[skillName] = { 'active' : False, - 'installer': data + 'installer': json.loads(self.getSkillInstallFilePath(skillName=skillName).read_text()) } - self._failedSkills[skillName].modified = True return True except Exception as e: @@ -1207,42 +1135,23 @@ def createNewSkill(self, skillDefinition: dict) -> bool: return False - @deprecated # this is available on AliceSK def uploadSkillToGithub(self, skillName: str, skillDesc: str) -> bool: try: self.logInfo(f'Uploading {skillName} to Github') skillName = skillName[0].upper() + skillName[1:] - localDirectory = Path('/home', getpass.getuser(), f'ProjectAlice/skills/{skillName}') - if not localDirectory.exists(): - raise Exception("Local skill doesn't exist") - - data = { - 'name' : f'skill_{skillName}', - 'description': skillDesc, - 'has-issues' : True, - 'has-wiki' : False - } - req = requests.post('https://api.github.com/user/repos', data=json.dumps(data), auth=GithubCloner.getGithubAuth()) - - if req.status_code != 201: - raise Exception("Couldn't create the repository on Github") - - self.Commons.runSystemCommand(['rm', '-rf', f'{str(localDirectory)}/.git']) - self.Commons.runSystemCommand(['git', '-C', str(localDirectory), 'init']) - - self.Commons.runSystemCommand(['git', 'config', '--global', 'user.email', 'githubbot@projectalice.io']) - self.Commons.runSystemCommand(['git', 'config', '--global', 'user.name', 'githubbot@projectalice.io']) - - remote = f'https://{self.ConfigManager.getAliceConfigByName("githubUsername")}:{self.ConfigManager.getAliceConfigByName("githubToken")}@github.com/{self.ConfigManager.getAliceConfigByName("githubUsername")}/skill_{skillName}.git' - self.Commons.runSystemCommand(['git', '-C', str(localDirectory), 'remote', 'add', 'origin', remote]) + try: + repository = self.getSkillRepository(skillName=skillName) + except GitErrors.PathNotFoundException: + raise Exception(f"Local skill **{skillName}** doesn't exist") + except GitErrors.NotGitRepository: + raise Exception(f'Skill **{skillName}** found but is not a git repository') - self.Commons.runSystemCommand(['git', '-C', str(localDirectory), 'add', '--all']) - self.Commons.runSystemCommand(['git', '-C', str(localDirectory), 'commit', '-m', '"Initial upload by Project Alice Skill Kit"']) - self.Commons.runSystemCommand(['git', '-C', str(localDirectory), 'push', '--set-upstream', 'origin', 'master']) + auth = self.ConfigManager.githubAuth + self.Commons.runSystemCommand(f'./venv/bin/projectalice-sk uploadToGithub --token {auth[1]} --author {auth[0]} --path {str(repository.path)} --desc {skillDesc}') - url = f'https://github.com/{self.ConfigManager.getAliceConfigByName("githubUsername")}/skill_{skillName}.git' + url = f'https://github.com/{auth[0]}/skill_{skillName}.git' self.logInfo(f'Skill uploaded! You can find it on {url}') return True except Exception as e: @@ -1250,6 +1159,31 @@ def uploadSkillToGithub(self, skillName: str, skillDesc: str) -> bool: return False + + + + + + + + # def onAssistantInstalled(self, **kwargs): + # argv = kwargs.get('skillsInfos', dict()) + # if not argv: + # return + # + # for skillName, skill in argv.items(): + # try: + # self._startSkill(skillName=skillName) + # except SkillStartDelayed: + # self.logInfo(f'Skill "{skillName}" start is delayed') + # except KeyError as e: + # self.logError(f'Skill "{skillName} not found, skipping: {e}') + # continue + # + # self._activeSkills[skillName].onBooted() + + + @deprecated def setSkillModified(self, skillName: str, modified: bool): return diff --git a/core/webApi/model/SkillsApi.py b/core/webApi/model/SkillsApi.py index e9fe942a2..ba3dcbf8d 100644 --- a/core/webApi/model/SkillsApi.py +++ b/core/webApi/model/SkillsApi.py @@ -19,11 +19,10 @@ import json +from pathlib import Path -import ProjectAliceSK.ProjectAliceSkillKit from flask import Response, jsonify, request from flask_classful import route -from pathlib import Path from core.base.model.GithubCloner import GithubCloner from core.commons import constants @@ -87,7 +86,8 @@ def createSkill(self) -> Response: if not self.SkillManager.createNewSkill(newSkill): raise Exception - skillName = request.form.get('name', '').capitalize() + + skillName = newSkill['name'] skill = self.SkillManager.getSkillInstance(skillName=skillName, silent=False) return jsonify(success=True, skill=skill.toDict() if skill else dict()) @@ -106,7 +106,6 @@ def uploadToGithub(self) -> Response: if not skillName: raise Exception('Missing skill name') - if self.SkillManager.uploadSkillToGithub(skillName, skillDesc): return jsonify(success=True, url=f'https://github.com/{self.ConfigManager.getAliceConfigByName("githubUsername")}/skill_{skillName}.git') From c3524e69b3c571b05f36ee3429ed7d80c2b1dad3 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Mon, 15 Nov 2021 19:30:00 +0100 Subject: [PATCH 038/129] Run the bot on rc2 --- .github/dependabot.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 30d0be667..2691ea21c 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -6,7 +6,7 @@ updates: interval: "daily" # Include a list of updated dependencies # with a prefix determined by the dependency group - target-branch: "1.0.0-rc1" + target-branch: "1.0.0-rc2" commit-message: prefix: "pip prod" prefix-development: "pip dev" From 0b3d3b9c2b72fecb7d4c4083ad3c9b0fa3224daa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 16 Nov 2021 05:53:56 +0000 Subject: [PATCH 039/129] pip prod(deps): update coveralls requirement from ~=3.2.0 to ~=3.3.1 Updates the requirements on [coveralls](https://github.com/TheKevJames/coveralls-python) to permit the latest version. - [Release notes](https://github.com/TheKevJames/coveralls-python/releases) - [Changelog](https://github.com/TheKevJames/coveralls-python/blob/master/CHANGELOG.md) - [Commits](https://github.com/TheKevJames/coveralls-python/compare/3.2.0...3.3.1) --- updated-dependencies: - dependency-name: coveralls dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements_test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_test.txt b/requirements_test.txt index da85ad310..a16e6d928 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -1,7 +1,7 @@ pytest~=6.2.5 coverage~=5.5 pytest-cov~=2.12.1 -coveralls~=3.2.0 +coveralls~=3.3.1 wheel~=0.37.0 python-dateutil~=2.8.2 paho-mqtt~=1.5.1 From 68bf76d46364cb299b1edd6e39f1cfc753493a33 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 16 Nov 2021 05:53:58 +0000 Subject: [PATCH 040/129] pip prod(deps): bump sounddevice from 0.4.2 to 0.4.3 Bumps [sounddevice](https://github.com/spatialaudio/python-sounddevice) from 0.4.2 to 0.4.3. - [Release notes](https://github.com/spatialaudio/python-sounddevice/releases) - [Changelog](https://github.com/spatialaudio/python-sounddevice/blob/master/NEWS.rst) - [Commits](https://github.com/spatialaudio/python-sounddevice/compare/0.4.2...0.4.3) --- updated-dependencies: - dependency-name: sounddevice dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- requirements_test.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index e891682c2..abcd01b66 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,7 +21,7 @@ PyAudio==0.2.11 pyjwt==2.1.0 markdown==3.3.4 ProjectAlice-sk -sounddevice==0.4.2 +sounddevice==0.4.3 htmlmin==0.1.12 jsmin==3.0.0 cssmin==0.2.0 diff --git a/requirements_test.txt b/requirements_test.txt index da85ad310..39c2043c8 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -24,5 +24,5 @@ langdetect~=1.0.9 webrtcvad~=2.0.10 pyjwt~=2.1.0 markdown~=3.3.4 -sounddevice==0.4.2 +sounddevice==0.4.3 projectalice-sk \ No newline at end of file From 292d8d3af7c5d55d646d241470739d9d8faf14a5 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Tue, 16 Nov 2021 11:03:24 +0100 Subject: [PATCH 041/129] requirements part 1 --- requirements.txt | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/requirements.txt b/requirements.txt index e891682c2..da81f3768 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,34 +1,34 @@ wheel==0.37.0 python-dateutil==2.8.2 -paho-mqtt==1.5.1 +paho-mqtt==1.6.1 requests==2.26.0 -esptool==3.1 +esptool==3.2 pyserial==3.5 pydub==0.25.1 -PyYAML==5.4.1 -flask==2.0.1 +PyYAML==6.0 +flask==2.0.2 flask-classful==0.14.2 pympler==0.9 Flask-Cors==3.0.10 googletrans==3.0.0 bcrypt==3.2.0 psutil==5.8.0 -numpy==1.21.2 -importlib_metadata==4.8.1 +numpy==1.21.4 +importlib_metadata==4.8.2 langdetect==1.0.9 webrtcvad==2.0.10 PyAudio==0.2.11 -pyjwt==2.1.0 +pyjwt==2.3.0 markdown==3.3.4 ProjectAlice-sk -sounddevice==0.4.2 +sounddevice==0.4.3 htmlmin==0.1.12 jsmin==3.0.0 cssmin==0.2.0 -boto3~=1.17.85 +boto3==1.20.6 dulwich~=0.20.25 -botocore~=1.20.85 +botocore==1.23.6 scipy~=1.7.1 Werkzeug~=2.0.1 projectalice-sk -AliceGit \ No newline at end of file +AliceGit From 4d5a167134d2159367fce7dbe389519a6e677989 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Tue, 16 Nov 2021 11:49:22 +0100 Subject: [PATCH 042/129] Use AliceGit new format --- core/base/SkillManager.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/core/base/SkillManager.py b/core/base/SkillManager.py index 9ee4b97e9..9301b60b5 100644 --- a/core/base/SkillManager.py +++ b/core/base/SkillManager.py @@ -28,7 +28,8 @@ import requests -from AliceGit import AliceGit, Exceptions as GitErrors +from AliceGit import Exceptions as GitErrors +from AliceGit.Git import Repository from core.ProjectAliceExceptions import AccessLevelTooLow, GithubNotFound, SkillInstanceFailed, SkillNotConditionCompliant, SkillStartDelayed, SkillStartingFailed from core.base.SuperManager import SuperManager from core.base.model import Intent @@ -310,7 +311,7 @@ def installSkills(self, skills: Union[str, List[str]]): self._busyInstalling.clear() - def getSkillRepository(self, skillName: str, directory: str = None) -> Optional[AliceGit]: + def getSkillRepository(self, skillName: str, directory: str = None) -> Optional[Repository]: """ Returns a Git object for the given skill :param skillName: @@ -322,7 +323,7 @@ def getSkillRepository(self, skillName: str, directory: str = None) -> Optional[ directory = self.getSkillDirectory(skillName=skillName) try: - return AliceGit(directory=directory) + return Repository(directory=directory) except: raise @@ -357,7 +358,7 @@ def downloadSkills(self, skills: Union[str, List[str]]) -> Optional[Dict]: try: repository = self.getSkillRepository(skillName=skillName) except: - repository = AliceGit.clone(url=source, directory=self.getSkillDirectory(skillName=skillName), makeDir=True) + repository = Repository.clone(url=source, directory=self.getSkillDirectory(skillName=skillName), makeDir=True) repository.checkout(tag=tag) repositories[skillName] = repository From 3515608990448d7e0a943bf8a3dcb66b4b24e56d Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Thu, 18 Nov 2021 13:19:43 +0100 Subject: [PATCH 043/129] After creation, the skill is not yet instanciated, changing how install file geetting works --- core/base/SkillManager.py | 11 ++++++----- core/webApi/model/SkillsApi.py | 5 +---- requirements.txt | 7 ++----- 3 files changed, 9 insertions(+), 14 deletions(-) diff --git a/core/base/SkillManager.py b/core/base/SkillManager.py index 9301b60b5..10b9e5f8a 100644 --- a/core/base/SkillManager.py +++ b/core/base/SkillManager.py @@ -1122,13 +1122,13 @@ def createNewSkill(self, skillDefinition: dict) -> bool: dump = Path(f'/tmp/{skillName}.json') dump.write_text(json.dumps(data, ensure_ascii=False)) - self.Commons.runSystemCommand(['./venv/bin/projectalice-sk', 'create', '--file', f'{str(dump)}']) + result = self.Commons.runSystemCommand(['./venv/bin/projectalice-sk', 'create', '--file', f'{str(dump)}']) + if result.stderr: + raise Exception('SK create failed') + self.logInfo(f'Created **{skillName}** skill') - self._skillList[skillName] = { - 'active' : False, - 'installer': json.loads(self.getSkillInstallFilePath(skillName=skillName).read_text()) - } + self._skillList.append(skillName) return True except Exception as e: @@ -1149,6 +1149,7 @@ def uploadSkillToGithub(self, skillName: str, skillDesc: str) -> bool: except GitErrors.NotGitRepository: raise Exception(f'Skill **{skillName}** found but is not a git repository') + auth = self.ConfigManager.githubAuth self.Commons.runSystemCommand(f'./venv/bin/projectalice-sk uploadToGithub --token {auth[1]} --author {auth[0]} --path {str(repository.path)} --desc {skillDesc}') diff --git a/core/webApi/model/SkillsApi.py b/core/webApi/model/SkillsApi.py index ba3dcbf8d..2aa55a039 100644 --- a/core/webApi/model/SkillsApi.py +++ b/core/webApi/model/SkillsApi.py @@ -537,10 +537,7 @@ def getInstallFile(self, skillName: str) -> Response: if skillName not in self.SkillManager.allSkills: return self.skillNotFound() - skill = self.SkillManager.getSkillInstance(skillName=skillName) - - installFile = skill.getResource(f'{skillName}.install') - + installFile = self.SkillManager.getSkillInstallFilePath(skillName=skillName) return jsonify(success=True, installFile=json.loads(installFile.read_text())) diff --git a/requirements.txt b/requirements.txt index da81f3768..338943af7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,15 +20,12 @@ webrtcvad==2.0.10 PyAudio==0.2.11 pyjwt==2.3.0 markdown==3.3.4 -ProjectAlice-sk sounddevice==0.4.3 htmlmin==0.1.12 jsmin==3.0.0 cssmin==0.2.0 -boto3==1.20.6 dulwich~=0.20.25 -botocore==1.23.6 scipy~=1.7.1 Werkzeug~=2.0.1 -projectalice-sk -AliceGit +projectalice-sk~=2.2.1 +AliceGit~=0.0.4 From 3f9eaa301646a3fce6c672783064ff35fc22a998 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Thu, 18 Nov 2021 13:27:42 +0100 Subject: [PATCH 044/129] should be debug --- core/webApi/model/SkillsApi.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/core/webApi/model/SkillsApi.py b/core/webApi/model/SkillsApi.py index 2aa55a039..907fe61ee 100644 --- a/core/webApi/model/SkillsApi.py +++ b/core/webApi/model/SkillsApi.py @@ -442,7 +442,7 @@ def setTemplate(self, skillName: str) -> Response: :param skillName: :return: """ - self.logInfo(f'DialogTemplate API access for skill {skillName}') + self.logDebug(f'DialogTemplate API access for skill {skillName}') if skillName not in self.SkillManager.allSkills: self.logError(f'Skill {skillName} not found') return self.skillNotFound() @@ -511,7 +511,7 @@ def setTalkFile(self, skillName: str) -> Response: :param skillName: :return: """ - self.logInfo(f'writing talkFile API access for skill {skillName}') + self.logDebug(f'Writing talkFile API access for skill {skillName}') if skillName not in self.SkillManager.allSkills: return self.skillNotFound() @@ -534,13 +534,15 @@ def getInstallFile(self, skillName: str) -> Response: :param skillName: :return: """ - if skillName not in self.SkillManager.allSkills: + print(skillName) + try: + installFile = self.SkillManager.getSkillInstallFilePath(skillName=skillName) + if not installFile.exists: + raise Exception + return jsonify(success=True, installFile=json.loads(installFile.read_text())) + except: return self.skillNotFound() - installFile = self.SkillManager.getSkillInstallFilePath(skillName=skillName) - return jsonify(success=True, installFile=json.loads(installFile.read_text())) - - @route('//setInstallFile/', methods=['PATCH']) @ApiAuthenticated def setInstallFile(self, skillName: str) -> Response: @@ -549,7 +551,7 @@ def setInstallFile(self, skillName: str) -> Response: :param skillName: :return: """ - self.logInfo(f'installFile API access for skill {skillName}') + self.logDebug(f'InstallFile API access for skill {skillName}') if skillName not in self.SkillManager.allSkills: return self.skillNotFound() @@ -573,7 +575,7 @@ def createWidget(self, skillName: str, widgetName: str) -> Response: :param skillName: :return: """ - self.logInfo(f'Creating new widget {widgetName} for skill {skillName}') + self.logDebug(f'Creating new widget {widgetName} for skill {skillName}') try: dest = self.getSkillDest(skillName=skillName) self.Commons.runSystemCommand(['./venv/bin/pip', 'install', '--upgrade', 'projectalice-sk']) @@ -592,7 +594,7 @@ def createDeviceType(self, skillName: str, deviceName: str) -> Response: :param skillName: :return: """ - self.logInfo(f'Creating new device type {deviceName} for skill {skillName}') + self.logDebug(f'Creating new device type {deviceName} for skill {skillName}') try: dest = self.getSkillDest(skillName=skillName) self.Commons.runSystemCommand(['./venv/bin/pip', 'install', '--upgrade', 'projectalice-sk']) From e27913d13b300253ba87a9a9a48d38497f061473 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Thu, 18 Nov 2021 13:30:10 +0100 Subject: [PATCH 045/129] start a skill after creation --- core/webApi/model/SkillsApi.py | 1 + 1 file changed, 1 insertion(+) diff --git a/core/webApi/model/SkillsApi.py b/core/webApi/model/SkillsApi.py index 907fe61ee..c32e6229b 100644 --- a/core/webApi/model/SkillsApi.py +++ b/core/webApi/model/SkillsApi.py @@ -88,6 +88,7 @@ def createSkill(self) -> Response: raise Exception skillName = newSkill['name'] + self.SkillManager.startSkill(skillName=skillName) skill = self.SkillManager.getSkillInstance(skillName=skillName, silent=False) return jsonify(success=True, skill=skill.toDict() if skill else dict()) From 42e8952efba60770f44ee83c609278759689490b Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Thu, 18 Nov 2021 13:42:25 +0100 Subject: [PATCH 046/129] This fixes skill creation --- core/base/SkillManager.py | 5 +++-- core/webApi/model/SkillsApi.py | 6 ++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/core/base/SkillManager.py b/core/base/SkillManager.py index 10b9e5f8a..276ca6778 100644 --- a/core/base/SkillManager.py +++ b/core/base/SkillManager.py @@ -1126,9 +1126,10 @@ def createNewSkill(self, skillDefinition: dict) -> bool: if result.stderr: raise Exception('SK create failed') - self.logInfo(f'Created **{skillName}** skill') - self._skillList.append(skillName) + self.addSkillToDB(skillName=skillName) + + self.logInfo(f'Created **{skillName}** skill') return True except Exception as e: diff --git a/core/webApi/model/SkillsApi.py b/core/webApi/model/SkillsApi.py index c32e6229b..72fa25272 100644 --- a/core/webApi/model/SkillsApi.py +++ b/core/webApi/model/SkillsApi.py @@ -87,9 +87,8 @@ def createSkill(self) -> Response: if not self.SkillManager.createNewSkill(newSkill): raise Exception - skillName = newSkill['name'] - self.SkillManager.startSkill(skillName=skillName) - skill = self.SkillManager.getSkillInstance(skillName=skillName, silent=False) + self.SkillManager.initSkills(onlyInit=newSkill['name']) + skill = self.SkillManager.getSkillInstance(skillName=newSkill['name']) return jsonify(success=True, skill=skill.toDict() if skill else dict()) except Exception as e: @@ -535,7 +534,6 @@ def getInstallFile(self, skillName: str) -> Response: :param skillName: :return: """ - print(skillName) try: installFile = self.SkillManager.getSkillInstallFilePath(skillName=skillName) if not installFile.exists: From dae0d023b693e14d5f85ad449690f4e1000344ac Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Thu, 18 Nov 2021 16:48:36 +0100 Subject: [PATCH 047/129] Some prep to skill upload to github --- core/webApi/model/SkillsApi.py | 35 ++++++++++++++++++++-------------- requirements.txt | 4 ++-- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/core/webApi/model/SkillsApi.py b/core/webApi/model/SkillsApi.py index 72fa25272..7838fad7a 100644 --- a/core/webApi/model/SkillsApi.py +++ b/core/webApi/model/SkillsApi.py @@ -19,11 +19,15 @@ import json +from contextlib import suppress from pathlib import Path from flask import Response, jsonify, request from flask_classful import route +from AliceGit.Exceptions import AlreadyGitRepository, GithubRepoNotFound, GithubUserNotFound +from AliceGit.Git import Repository +from AliceGit.Github import Github from core.base.model.GithubCloner import GithubCloner from core.commons import constants from core.util.Decorators import ApiAuthenticated @@ -318,21 +322,23 @@ def upload(self, skillName: str) -> Response: """ if skillName not in self.SkillManager.allSkills: return self.skillNotFound() - self.SkillManager.getSkillInstance(skillName=skillName).modified = True - gitCloner = GithubCloner(baseUrl=f'{constants.GITHUB_URL}/skill_{skillName}.git', - dest=Path(self.Commons.rootDir()) / 'skills' / skillName, - skillName=skillName) - if not gitCloner.isRepo() and not gitCloner.createRepo(): - return jsonify(success=False, message="Failed creating repository") - elif not gitCloner.add(): - return jsonify(success=False, message="Failed adding to git") - elif not gitCloner.commit(message="pushed via API"): - return jsonify(success=False, message="Failed creating commit") - elif not gitCloner.push(): - return jsonify(success=False, message="Failed pushing to git") - else: - return jsonify(success=True) + installFilePath = self.SkillManager.getSkillInstallFilePath(skillName=skillName) + try: + with suppress(AlreadyGitRepository): + repository = Repository(directory=self.SkillManager.getSkillDirectory(skillName=skillName), init=True) + + auth = self.ConfigManager.githubAuth + github = Github(username=auth[0], token=auth[1], repositoryName=f'skill_{skillName}') + repository.remoteAdd(url=github.url, name='master') + repository.commit(message='Save through Alice web UI', autoAdd=True) + repository.push() + except GithubUserNotFound: + return jsonify(success=False, message='The provided Github user is not existing') + except GithubRepoNotFound: + if not self.SkillManager.uploadSkillToGithub(skillName=skillName, skillDesc=json.loads(installFilePath.read_text())['desc']): + return jsonify(success=False, message='Failed uploading to Github') + return jsonify(success=True) @route('//gitStatus/') @@ -542,6 +548,7 @@ def getInstallFile(self, skillName: str) -> Response: except: return self.skillNotFound() + @route('//setInstallFile/', methods=['PATCH']) @ApiAuthenticated def setInstallFile(self, skillName: str) -> Response: diff --git a/requirements.txt b/requirements.txt index 338943af7..b6814f5a6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,5 +27,5 @@ cssmin==0.2.0 dulwich~=0.20.25 scipy~=1.7.1 Werkzeug~=2.0.1 -projectalice-sk~=2.2.1 -AliceGit~=0.0.4 +projectalice-sk~=2.1.2 +AliceGit~=0.0.5 From edc96d8b5694e2d719127eea6c9c7f5391c75f68 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Thu, 18 Nov 2021 18:54:40 +0100 Subject: [PATCH 048/129] git status --- core/base/model/AliceSkill.py | 50 +++++++---------- core/base/model/FailedAliceSkill.py | 32 ++++------- core/webApi/model/SkillsApi.py | 84 ++++++++++++++++------------- requirements.txt | 2 +- 4 files changed, 78 insertions(+), 90 deletions(-) diff --git a/core/base/model/AliceSkill.py b/core/base/model/AliceSkill.py index de8b44ae3..8e5f4aa4d 100644 --- a/core/base/model/AliceSkill.py +++ b/core/base/model/AliceSkill.py @@ -19,18 +19,19 @@ from __future__ import annotations -from copy import copy - -import flask import importlib import inspect import json import re -from markdown import markdown -from paho.mqtt import client as MQTTClient +from copy import copy from pathlib import Path from typing import Any, Dict, Iterable, List, Optional, Union +import flask +from markdown import markdown +from paho.mqtt import client as MQTTClient + +from AliceGit.Git import Repository from core.ProjectAliceExceptions import AccessLevelTooLow, SkillInstanceFailed from core.base.model.Intent import Intent from core.base.model.ProjectAliceObject import ProjectAliceObject @@ -71,7 +72,6 @@ def __init__(self, supportedIntents: Iterable = None, databaseSchema: dict = Non self._category = self._installer.get('category', constants.UNKNOWN) self._conditions = self._installer.get('conditions', dict()) self._updateAvailable = False - self._modified = False self._active = False self._delayed = False self._required = False @@ -83,8 +83,8 @@ def __init__(self, supportedIntents: Iterable = None, databaseSchema: dict = Non self._intentsDefinitions = dict() self._scenarioPackageName = '' self._scenarioPackageVersion = Version(mainVersion=0, updateVersion=0, hotfix=0) - self._supportedIntents: Dict[str, Intent] = self.buildIntentList(supportedIntents) + self._repository = Repository(directory=self._skillPath, init=True, raiseIfExisting=False) self.loadIntentsDefinition() self._utteranceSlotCleaner = re.compile('{(.+?):=>.+?}') @@ -92,6 +92,16 @@ def __init__(self, supportedIntents: Iterable = None, databaseSchema: dict = Non self._myDevices: Dict[str, Device] = dict() + @property + def modified(self) -> bool: + return self._repository.isDirty() + + + @property + def repository(self) -> Repository: + return self._repository + + @property def failedStarting(self) -> bool: return self._failedStarting @@ -438,28 +448,6 @@ def delayed(self, value: bool): self._delayed = value - @property - def modified(self) -> bool: - return self._modified - - - @modified.setter - def modified(self, value: bool): - """ - As the skill has no writeToDB method and this is the only value that has to be saved right away - a update of the value on the DB is performed. This should only occure manually triggered when the user starts to make local changes - :param value: - :return: - """ - self._modified = value - self.SkillManager.setSkillModified(skillName=self.name, modified=self._modified) - dbVal = 1 if value else 0 - self.DatabaseManager.update(tableName=self.SkillManager.DBTAB_SKILLS, - callerName=self.SkillManager.name, - row=('skillname', self.name), - values={'modified': dbVal}) - - @property def scenarioNodeName(self) -> str: return self._scenarioPackageName @@ -520,7 +508,7 @@ def authenticateIntent(self, session: DialogSession): text=self.TalkManager.randomTalk(talk='unknownUser', skill='system') ) raise AccessLevelTooLow() - # Return if intent is for auth users only and the user doesn't have the accesslevel for it + # Return if intent is for auth users only and the user doesn't have the access level for it if not self.UserManager.hasAccessLevel(session.user, intent.authLevel): self.endDialog( sessionId=session.sessionId, @@ -750,7 +738,7 @@ def toDict(self) -> dict: 'name' : self._name, 'author' : self._author, 'version' : self._version, - 'modified' : self._modified, + 'modified' : self.modified, 'updateAvailable' : self._updateAvailable, 'active' : self._active, 'delayed' : self._delayed, diff --git a/core/base/model/FailedAliceSkill.py b/core/base/model/FailedAliceSkill.py index 1a17e746c..2faa97189 100644 --- a/core/base/model/FailedAliceSkill.py +++ b/core/base/model/FailedAliceSkill.py @@ -22,6 +22,7 @@ import json from pathlib import Path +from AliceGit.Git import Repository from core.base.model.ProjectAliceObject import ProjectAliceObject from core.base.model.Version import Version from core.commons import constants @@ -34,7 +35,6 @@ def __init__(self, installer: dict): self._installer = installer self._updateAvailable = False self._name = installer['name'] - self._modified = False self._icon = self._installer.get('icon', 'fas fa-biohazard') self._aliceMinVersion = Version.fromString(self._installer.get('aliceMinVersion', '1.0.0-b4')) self._maintainers = self._installer.get('maintainers', list()) @@ -42,6 +42,7 @@ def __init__(self, installer: dict): self._category = self._installer.get('category', constants.UNKNOWN) self._conditions = self._installer.get('conditions', dict()) self._skillPath = Path('skills') / self._name + self._repository = Repository(directory=self._skillPath, init=True, raiseIfExisting=False) super().__init__() @@ -55,7 +56,7 @@ def onStart(self): def onStop(self): - pass # Is always handeled by the sibling + pass # Is always handled by the sibling def onBooted(self) -> bool: @@ -80,25 +81,12 @@ def __str__(self) -> str: @property def modified(self) -> bool: - return self._modified - - - @modified.setter - def modified(self, value: bool): - """ - As the skill has no writeToDB method and this is the only value that has to be saved right away - a update of the value on the DB is performed. This should only occure manually triggered when the user starts to make local changes - :param value: - :return: - """ - self._modified = value - self.SkillManager.setSkillModified(skillName=self._name, modified=self._modified) - dbVal = 1 if value else 0 - self.logInfo(f'Wrote dbval {dbVal}') - self.DatabaseManager.update(tableName=self.SkillManager.DBTAB_SKILLS, - callerName=self.SkillManager.name, - row=('skillname', self._name), - values={'modified': dbVal}) + return self._repository.isDirty() + + + @property + def repository(self) -> Repository: + return self._repository @property @@ -115,7 +103,7 @@ def toDict(self) -> dict: 'name' : self._name, 'author' : self._installer['author'], 'version' : self._installer['version'], - 'modified' : self._modified, + 'modified' : self.modified, 'updateAvailable': self._updateAvailable, 'maintainers' : self._maintainers, 'settings' : self.ConfigManager.getSkillConfigs(self._name), diff --git a/core/webApi/model/SkillsApi.py b/core/webApi/model/SkillsApi.py index 7838fad7a..f44bb0feb 100644 --- a/core/webApi/model/SkillsApi.py +++ b/core/webApi/model/SkillsApi.py @@ -25,7 +25,7 @@ from flask import Response, jsonify, request from flask_classful import route -from AliceGit.Exceptions import AlreadyGitRepository, GithubRepoNotFound, GithubUserNotFound +from AliceGit.Exceptions import AlreadyGitRepository, GithubRepoNotFound, GithubUserNotFound, NotGitRepository from AliceGit.Git import Repository from AliceGit.Github import Github from core.base.model.GithubCloner import GithubCloner @@ -100,25 +100,6 @@ def createSkill(self) -> Response: return jsonify(success=False, message=str(e)) - @ApiAuthenticated - @route('/uploadSkill/', methods=['POST']) - def uploadToGithub(self) -> Response: - try: - skillName = request.form.get('skillName', '') - skillDesc = request.form.get('skillDesc', '') - - if not skillName: - raise Exception('Missing skill name') - - if self.SkillManager.uploadSkillToGithub(skillName, skillDesc): - return jsonify(success=True, url=f'https://github.com/{self.ConfigManager.getAliceConfigByName("githubUsername")}/skill_{skillName}.git') - - return jsonify(success=False, message=f'Error while uploading to github!') - except Exception as e: - self.logError(f'Failed uploading to github: {e}') - return jsonify(success=False, message=str(e)) - - @ApiAuthenticated @route('/installSkills/', methods=['PUT']) def installSkills(self) -> Response: @@ -268,6 +249,19 @@ def checkUpdate(self, skillName: str) -> Response: return jsonify(success=self.SkillManager.checkForSkillUpdates(skillToCheck=skillName)) + @route('//isDirty/', methods=['GET']) + @ApiAuthenticated + def isDirty(self, skillName: str) -> Response: + try: + repo = self.SkillManager.getSkillRepository(skillName=skillName) + if repo.isDirty(): + return jsonify(success=True, message='dirty') + else: + return jsonify(success=True, message='clean') + except NotGitRepository: + return jsonify(success=True, message='dirty') + + @route('//setModified/') @ApiAuthenticated def setModified(self, skillName: str) -> Response: @@ -283,7 +277,6 @@ def setModified(self, skillName: str) -> Response: if not GithubCloner.hasAuth(): return self.githubMissing() - self.SkillManager.getSkillInstance(skillName=skillName).modified = True gitCloner = GithubCloner(baseUrl=f'{constants.GITHUB_URL}/skill_{skillName}.git', dest=Path(self.Commons.rootDir()) / 'skills' / skillName, skillName=skillName) @@ -303,11 +296,9 @@ def revert(self, skillName: str) -> Response: """ if skillName not in self.SkillManager.allSkills: return self.skillNotFound() - self.SkillManager.getSkillInstance(skillName=skillName).modified = False - gitCloner = GithubCloner(baseUrl=f'{constants.GITHUB_URL}/skill_{skillName}.git', - dest=Path(self.Commons.rootDir()) / 'skills' / skillName, - skillName=skillName) - gitCloner.checkoutMaster() + + skill = self.SkillManager.getSkillInstance(skillName=skillName) + skill.repository.revert() return self.checkUpdate(skillName) @@ -352,20 +343,41 @@ def getGitStatus(self, skillName: str) -> Response: :param skillName: :return: """ - if skillName not in self.SkillManager.allSkills: + + skill = self.SkillManager.getSkillInstance(skillName=skillName) + if not skill: return self.skillNotFound() - gitCloner = GithubCloner(baseUrl=f'{constants.GITHUB_URL}/skill_{skillName}.git', - dest=Path(self.Commons.rootDir()) / 'skills' / skillName, - skillName=skillName) + try: + Github(useUrlInstead=skill.repository.url) + privateStatus = True + except: + privateStatus = False + + + explode = skill.repository.url.split('/') + explode[len(explode) - 2] = 'project-alice-assistant' + url = '/'.join(explode) + try: + Github(useUrlInstead=url) + publicStatus = True + except: + publicStatus = False return jsonify(success=True, - result={'Public' : {'name' : 'Public', - 'url' : gitCloner.getRemote(origin=True, noToken=True), - 'status': gitCloner.checkRemote(origin=True)}, - 'Private': {'name' : 'Private', - 'url' : gitCloner.getRemote(AliceSK=True, noToken=True), - 'status': gitCloner.checkRemote(AliceSK=True)}}) + result={ + 'Public' : { + 'name' : 'Public', + 'url' : url, + 'status': publicStatus + }, + 'Private': { + 'name' : 'Private', + 'url' : skill.repository.url, + 'status': privateStatus + } + } + ) @route('//getInstructions/', methods=['GET', 'POST']) diff --git a/requirements.txt b/requirements.txt index b6814f5a6..73d9fcce8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,4 +28,4 @@ dulwich~=0.20.25 scipy~=1.7.1 Werkzeug~=2.0.1 projectalice-sk~=2.1.2 -AliceGit~=0.0.5 +AliceGit~=0.0.6 From 919a0c0fc7db17610ca8e8266ff73273edd77776 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Thu, 18 Nov 2021 19:05:51 +0100 Subject: [PATCH 049/129] Cleanup requirements.txt --- requirements.txt | 54 +++++++++++++++++++++--------------------------- 1 file changed, 24 insertions(+), 30 deletions(-) diff --git a/requirements.txt b/requirements.txt index 73d9fcce8..7f4c66357 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,31 +1,25 @@ -wheel==0.37.0 -python-dateutil==2.8.2 -paho-mqtt==1.6.1 -requests==2.26.0 -esptool==3.2 -pyserial==3.5 -pydub==0.25.1 -PyYAML==6.0 -flask==2.0.2 -flask-classful==0.14.2 -pympler==0.9 -Flask-Cors==3.0.10 -googletrans==3.0.0 -bcrypt==3.2.0 -psutil==5.8.0 -numpy==1.21.4 -importlib_metadata==4.8.2 -langdetect==1.0.9 -webrtcvad==2.0.10 -PyAudio==0.2.11 -pyjwt==2.3.0 -markdown==3.3.4 -sounddevice==0.4.3 -htmlmin==0.1.12 -jsmin==3.0.0 -cssmin==0.2.0 -dulwich~=0.20.25 -scipy~=1.7.1 -Werkzeug~=2.0.1 -projectalice-sk~=2.1.2 +numpy~=1.21.4 +paho-mqtt~=1.6.1 +googletrans~=3.0.0 +langdetect~=1.0.9 +Flask~=2.0.2 +Markdown~=3.3.6 AliceGit~=0.0.6 +requests~=2.26.0 +dulwich~=0.20.26 +sounddevice~=0.4.3 +bcrypt~=3.2.0 +PyJWT~=2.3.0 +Pympler~=0.9 +pydub~=0.25.1 +PyAudio~=0.2.11 +htmlmin~=0.1.12 +cssmin~=0.2.0 +jsmin~=3.0.0 +psutil~=5.8.0 +pyserial~=3.5 +pyyaml~=6.0 +scipy~=1.7.2 +webrtcvad~=2.0.10 +Werkzeug~=2.0.2 +Jinja2~=3.0.3 \ No newline at end of file From 5888c16ddb7368c43e62e664dba70dc7355fc3d4 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Thu, 18 Nov 2021 19:22:19 +0100 Subject: [PATCH 050/129] SSL for API. You need to configure --insecure for now, will buy and share a certificate when 1.0.0 is out --- configTemplate.json | 7 +++++++ core/webApi/ApiManager.py | 17 +++++++++++------ requirements.txt | 3 ++- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/configTemplate.json b/configTemplate.json index 8c986e906..a80319753 100644 --- a/configTemplate.json +++ b/configTemplate.json @@ -19,6 +19,13 @@ "description" : "Displays the current system usage on the interface", "category" : "system" }, + "enableSSL" : { + "defaultValue": false, + "dataType" : "boolean", + "isSensitive" : false, + "description" : "Enables SSL for both the UI and API", + "category" : "system" + }, "delegateNluTraining" : { "defaultValue": false, "dataType" : "boolean", diff --git a/core/webApi/ApiManager.py b/core/webApi/ApiManager.py index 4b242d23b..c2b2404c0 100644 --- a/core/webApi/ApiManager.py +++ b/core/webApi/ApiManager.py @@ -77,13 +77,18 @@ def startThread(self): if not self.isActive: return + options = { + 'debug' : self.ConfigManager.getAliceConfigByName('debug'), + 'port' : int(self.ConfigManager.getAliceConfigByName('apiPort')), + 'host' : '0.0.0.0', + 'use_reloader': False + } + + if self.ConfigManager.getAliceConfigByName('enableSSL'): + options['ssl_context'] = 'adhoc' + self.ThreadManager.newThread( name='API', target=self.app.run, - kwargs={ - 'debug' : self.ConfigManager.getAliceConfigByName('debug'), - 'port' : int(self.ConfigManager.getAliceConfigByName('apiPort')), - 'host' : '0.0.0.0', - 'use_reloader': False - } + kwargs=options ) diff --git a/requirements.txt b/requirements.txt index 7f4c66357..7024afce6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,4 +22,5 @@ pyyaml~=6.0 scipy~=1.7.2 webrtcvad~=2.0.10 Werkzeug~=2.0.2 -Jinja2~=3.0.3 \ No newline at end of file +Jinja2~=3.0.3 +pyopenssl~=21.0.0 \ No newline at end of file From 41f4e05a4868c54a14fee9933958eb841078bb9a Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Sat, 20 Nov 2021 05:57:54 +0100 Subject: [PATCH 051/129] missing requirements --- requirements.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7024afce6..e6c4d3bff 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,8 @@ paho-mqtt~=1.6.1 googletrans~=3.0.0 langdetect~=1.0.9 Flask~=2.0.2 +Flask-Cors~=3.0.10 +Flask-Classful~=0.14.2 Markdown~=3.3.6 AliceGit~=0.0.6 requests~=2.26.0 @@ -23,4 +25,4 @@ scipy~=1.7.2 webrtcvad~=2.0.10 Werkzeug~=2.0.2 Jinja2~=3.0.3 -pyopenssl~=21.0.0 \ No newline at end of file +pyopenssl~=21.0.0 From c4f361eee3f3dd7ac3b80c48a018041f39f169ba Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Sat, 20 Nov 2021 19:49:03 +0100 Subject: [PATCH 052/129] Runs smoother, but there's still an issue downloading skills after fresh install on rc2 --- core/base/SkillManager.py | 12 ++++++------ requirements.txt | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/core/base/SkillManager.py b/core/base/SkillManager.py index 276ca6778..4384b2f3c 100644 --- a/core/base/SkillManager.py +++ b/core/base/SkillManager.py @@ -347,10 +347,10 @@ def downloadSkills(self, skills: Union[str, List[str]]) -> Optional[Dict]: if response.status_code != 200: raise GithubNotFound - with suppress: # Increment download counter + with suppress(): # Increment download counter requests.get(f'https://skills.projectalice.ch/{skillName}') - installFile = json.loads(response.json()) + installFile = response.json() self.checkSkillConditions(installer=installFile) source = self.getGitRemoteSourceUrl(skillName=skillName, doAuth=False) @@ -366,7 +366,7 @@ def downloadSkills(self, skills: Union[str, List[str]]) -> Optional[Dict]: if skillName in self.NEEDED_SKILLS: self._busyInstalling.clear() self.logFatal(f"Skill **{skillName}** is required but wasn't found in released skills, cannot continue") - return None + return repositories else: self.logError(f'Skill "{skillName}" not found in released skills') continue @@ -375,12 +375,12 @@ def downloadSkills(self, skills: Union[str, List[str]]) -> Optional[Dict]: continue else: self._busyInstalling.clear() - return None + return repositories except Exception as e: if skillName in self.NEEDED_SKILLS: self._busyInstalling.clear() self.logFatal(f'Error downloading skill **{skillName}** and it is required, cannot continue: {e}') - return None + return repositories else: self.logError(f'Error downloading skill "{skillName}": {e}') continue @@ -965,7 +965,7 @@ def removeSkill(self, skillName: str): self.removeSkillFromDB(skillName=skillName) - with suppress: + with suppress(): repo = self.getSkillRepository(skillName=skillName) repo.destroy() diff --git a/requirements.txt b/requirements.txt index e6c4d3bff..dbe70300a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ Flask~=2.0.2 Flask-Cors~=3.0.10 Flask-Classful~=0.14.2 Markdown~=3.3.6 -AliceGit~=0.0.6 +AliceGit~=0.0.7 requests~=2.26.0 dulwich~=0.20.26 sounddevice~=0.4.3 From 71fb295e80973180406150056cda29b861aa35fc Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Sun, 21 Nov 2021 08:39:03 +0100 Subject: [PATCH 053/129] error handling --- core/base/SkillManager.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/core/base/SkillManager.py b/core/base/SkillManager.py index 4384b2f3c..dbd2f8b32 100644 --- a/core/base/SkillManager.py +++ b/core/base/SkillManager.py @@ -29,6 +29,7 @@ import requests from AliceGit import Exceptions as GitErrors +from AliceGit.Exceptions import NotGitRepository, PathNotFoundException from AliceGit.Git import Repository from core.ProjectAliceExceptions import AccessLevelTooLow, GithubNotFound, SkillInstanceFailed, SkillNotConditionCompliant, SkillStartDelayed, SkillStartingFailed from core.base.SuperManager import SuperManager @@ -357,8 +358,13 @@ def downloadSkills(self, skills: Union[str, List[str]]) -> Optional[Dict]: try: repository = self.getSkillRepository(skillName=skillName) - except: + except PathNotFoundException: + repository = Repository.clone(url=source, directory=self.getSkillDirectory(skillName=skillName), makeDir=True) + except NotGitRepository: + shutil.rmtree(self.getSkillDirectory(skillName=skillName), ignore_errors=True) repository = Repository.clone(url=source, directory=self.getSkillDirectory(skillName=skillName), makeDir=True) + except: + raise repository.checkout(tag=tag) repositories[skillName] = repository From 50a6a4fdd2ddfb85625100ad6c2a2633f6cc6994 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Sun, 21 Nov 2021 08:49:02 +0100 Subject: [PATCH 054/129] fix missing deps --- .github/workflows/pytest.yml | 4 ++++ requirements_test.txt | 26 +------------------------- 2 files changed, 5 insertions(+), 25 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index b980817df..3709f7e23 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -21,6 +21,10 @@ jobs: with: python-version: 3.7 - name: Install dependencies + uses: py-actions/py-dependency-install@v2 + with: + path: requirements.txt + - name: Install test dependencies uses: py-actions/py-dependency-install@v2 with: path: requirements_test.txt diff --git a/requirements_test.txt b/requirements_test.txt index 98cf997e1..d479e43a4 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -1,28 +1,4 @@ pytest~=6.2.5 coverage~=5.5 -pytest-cov~=2.12.1 +pytest-cov~=3.0.0 coveralls~=3.3.1 -wheel~=0.37.0 -python-dateutil~=2.8.2 -paho-mqtt~=1.5.1 -requests~=2.26.0 -esptool~=3.1 -pyserial~=3.5 -pydub~=0.25.1 -terminaltables~=3.1.0 -PyYAML~=5.4.1 -flask~=2.0.1 -flask-classful~=0.14.2 -pympler~=0.9 -Flask-Cors~=3.0.10 -googletrans~=3.0.0 -bcrypt~=3.2.0 -psutil~=5.8.0 -numpy~=1.21.2 -importlib_metadata~=4.8.1 -langdetect~=1.0.9 -webrtcvad~=2.0.10 -pyjwt~=2.1.0 -markdown~=3.3.4 -sounddevice==0.4.3 -projectalice-sk \ No newline at end of file From 5b5c56c54fe1acfe4642978e2b0a7df5b699998d Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Sun, 21 Nov 2021 08:52:06 +0100 Subject: [PATCH 055/129] maybe this way --- .github/workflows/pytest.yml | 4 ---- requirements_test.txt | 27 +++++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 3709f7e23..b980817df 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -21,10 +21,6 @@ jobs: with: python-version: 3.7 - name: Install dependencies - uses: py-actions/py-dependency-install@v2 - with: - path: requirements.txt - - name: Install test dependencies uses: py-actions/py-dependency-install@v2 with: path: requirements_test.txt diff --git a/requirements_test.txt b/requirements_test.txt index d479e43a4..dee001065 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -2,3 +2,30 @@ pytest~=6.2.5 coverage~=5.5 pytest-cov~=3.0.0 coveralls~=3.3.1 +numpy~=1.21.4 +paho-mqtt~=1.6.1 +googletrans~=3.0.0 +langdetect~=1.0.9 +Flask~=2.0.2 +Flask-Cors~=3.0.10 +Flask-Classful~=0.14.2 +Markdown~=3.3.6 +AliceGit~=0.0.7 +requests~=2.26.0 +dulwich~=0.20.26 +sounddevice~=0.4.3 +bcrypt~=3.2.0 +PyJWT~=2.3.0 +Pympler~=0.9 +pydub~=0.25.1 +htmlmin~=0.1.12 +cssmin~=0.2.0 +jsmin~=3.0.0 +psutil~=5.8.0 +pyserial~=3.5 +pyyaml~=6.0 +scipy~=1.7.2 +webrtcvad~=2.0.10 +Werkzeug~=2.0.2 +Jinja2~=3.0.3 +pyopenssl~=21.0.0 From b44a14a44259564d4b7d954b39afab2c59b7daea Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Sun, 21 Nov 2021 08:56:53 +0100 Subject: [PATCH 056/129] resolve conflicts --- requirements.txt | 2 +- requirements_test.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index dbe70300a..e203201f4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ Flask~=2.0.2 Flask-Cors~=3.0.10 Flask-Classful~=0.14.2 Markdown~=3.3.6 -AliceGit~=0.0.7 +AliceGit~=0.0.8 requests~=2.26.0 dulwich~=0.20.26 sounddevice~=0.4.3 diff --git a/requirements_test.txt b/requirements_test.txt index dee001065..7b9a94e7d 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -10,7 +10,7 @@ Flask~=2.0.2 Flask-Cors~=3.0.10 Flask-Classful~=0.14.2 Markdown~=3.3.6 -AliceGit~=0.0.7 +AliceGit~=0.0.8 requests~=2.26.0 dulwich~=0.20.26 sounddevice~=0.4.3 From fc9a9413dc4315c16569f95012169122a3c36da8 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Sun, 21 Nov 2021 09:47:25 +0100 Subject: [PATCH 057/129] require latest AliceGit --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e203201f4..9a29f0f2f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ Flask~=2.0.2 Flask-Cors~=3.0.10 Flask-Classful~=0.14.2 Markdown~=3.3.6 -AliceGit~=0.0.8 +AliceGit~=0.0.9 requests~=2.26.0 dulwich~=0.20.26 sounddevice~=0.4.3 From 1b4fe40198303913f78d97f52ea791affb62b662 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Sun, 21 Nov 2021 09:47:53 +0100 Subject: [PATCH 058/129] require latest AliceGit --- requirements_test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_test.txt b/requirements_test.txt index 7b9a94e7d..b81cb9055 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -10,7 +10,7 @@ Flask~=2.0.2 Flask-Cors~=3.0.10 Flask-Classful~=0.14.2 Markdown~=3.3.6 -AliceGit~=0.0.8 +AliceGit~=0.0.9 requests~=2.26.0 dulwich~=0.20.26 sounddevice~=0.4.3 From b65b227a7acde35055cebc1dc2c49a67b20689e9 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Sun, 21 Nov 2021 16:44:55 +0100 Subject: [PATCH 059/129] use AliceGit 0.0.10 to fix skill download --- requirements.txt | 2 +- requirements_test.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 9a29f0f2f..de8b9cd76 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ Flask~=2.0.2 Flask-Cors~=3.0.10 Flask-Classful~=0.14.2 Markdown~=3.3.6 -AliceGit~=0.0.9 +AliceGit~=0.0.10 requests~=2.26.0 dulwich~=0.20.26 sounddevice~=0.4.3 diff --git a/requirements_test.txt b/requirements_test.txt index b81cb9055..64605bb81 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -10,7 +10,7 @@ Flask~=2.0.2 Flask-Cors~=3.0.10 Flask-Classful~=0.14.2 Markdown~=3.3.6 -AliceGit~=0.0.9 +AliceGit~=0.0.10 requests~=2.26.0 dulwich~=0.20.26 sounddevice~=0.4.3 From bdbfeab149df303f013379888bbcb4709b555fa5 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Sun, 21 Nov 2021 17:09:26 +0100 Subject: [PATCH 060/129] fix needed skills check, fix double skill config check, added info about skill download --- core/base/SkillManager.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/core/base/SkillManager.py b/core/base/SkillManager.py index dbd2f8b32..557b7ae4d 100644 --- a/core/base/SkillManager.py +++ b/core/base/SkillManager.py @@ -129,9 +129,12 @@ def onStart(self): if not self._skillList: self.logInfo('Looks like a fresh install or skills were nuked. Let\'s install the basic skills!') self.installSkills(skills=self.BASE_SKILLS) - elif sorted(self._skillList) != sorted(self.BASE_SKILLS): - self.logInfo('Some required skills are missing, let\'s download them!') - self.installSkills(skills=list(set(self.NEEDED_SKILLS) - set(self._skillList))) + + for skill in self.NEEDED_SKILLS: + if skill not in self._skillList: + self.logInfo('Some required skills are missing, let\'s download them!') + self.installSkills(skills=list(set(self.NEEDED_SKILLS) - set(self._skillList))) + break updates = self.checkForSkillUpdates() if updates: @@ -142,8 +145,6 @@ def onStart(self): for skillName in self._deactivatedSkills: self.configureSkillIntents(skillName=skillName, state=False) - self.ConfigManager.loadCheckAndUpdateSkillConfigurations() - self.startAllSkills() @@ -348,6 +349,8 @@ def downloadSkills(self, skills: Union[str, List[str]]) -> Optional[Dict]: if response.status_code != 200: raise GithubNotFound + self.logInfo(f'Now downloading **{skillName}** version **{tag}**') + with suppress(): # Increment download counter requests.get(f'https://skills.projectalice.ch/{skillName}') From 950816e60166dc0f2d419629664ef92637b6ef83 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Mon, 22 Nov 2021 05:48:10 +0100 Subject: [PATCH 061/129] We need to git fetch before updating a skill, to update the tags --- core/asr/model/GoogleAsr.py | 34 +++++++++++++++++++--------------- core/base/SkillManager.py | 1 + core/webui/public | 2 +- requirements.txt | 2 +- 4 files changed, 22 insertions(+), 17 deletions(-) diff --git a/core/asr/model/GoogleAsr.py b/core/asr/model/GoogleAsr.py index a9e249c69..f6df6e6e5 100644 --- a/core/asr/model/GoogleAsr.py +++ b/core/asr/model/GoogleAsr.py @@ -18,10 +18,11 @@ # Last modified: 2021.04.13 at 12:56:45 CEST import os +from contextlib import suppress from pathlib import Path from threading import Event from time import time -from typing import Generator, Optional +from typing import Iterable, Optional from core.asr.model.ASRResult import ASRResult from core.asr.model.Asr import Asr @@ -30,11 +31,11 @@ from core.util.Stopwatch import Stopwatch -try: +with suppress(ModuleNotFoundError): # noinspection PyPackageRequirements - from google.cloud.speech import SpeechClient, RecognitionConfig, StreamingRecognitionConfig, StreamingRecognizeRequest -except: - pass # Auto installed + from google.cloud.speech import SpeechClient + # noinspection PyPackageRequirements + from google.cloud.speech_v1 import RecognitionConfig, StreamingRecognitionConfig, StreamingRecognizeRequest # noinspection PyAbstractClass @@ -72,15 +73,18 @@ def onStart(self): self._client = SpeechClient() # noinspection PyUnresolvedReferences - config = speech.RecognitionConfig( - encoding=speech.RecognitionConfig.AudioEncoding.LINEAR16, - sample_rate_hertz=self.AudioServer.SAMPLERATE, - language_code=self.LanguageManager.getLanguageAndCountryCode(), - max_alternatives=1, - model='command_and_search' - ) + config = RecognitionConfig({ + 'encoding': RecognitionConfig.AudioEncoding.LINEAR16, + 'sample_rate_hertz': self.AudioServer.SAMPLERATE, + 'language_code': self.LanguageManager.getLanguageAndCountryCode(), + 'max_alternatives': 1, + 'model': 'command_and_search' + }) - self._streamingConfig = StreamingRecognitionConfig(config=config, interim_results=True) + self._streamingConfig = StreamingRecognitionConfig({ + 'config': config, + 'interim_results': True + }) def decodeStream(self, session: DialogSession) -> Optional[ASRResult]: @@ -96,7 +100,7 @@ def decodeStream(self, session: DialogSession) -> Optional[ASRResult]: # noinspection PyUnresolvedReferences try: requests = (StreamingRecognizeRequest(audio_content=content) for content in audioStream) - responses = self._client.streaming_recognize(self._streamingConfig, requests) + responses = self._client.streaming_recognize(config=self._streamingConfig, requests=requests) result = self._checkResponses(session, responses) except Exception as e: self._internetLostFlag.clear() @@ -116,7 +120,7 @@ def onInternetLost(self): self._internetLostFlag.set() - def _checkResponses(self, session: DialogSession, responses: Generator) -> Optional[tuple]: + def _checkResponses(self, session: DialogSession, responses: Iterable) -> Optional[tuple]: if responses is None: return None diff --git a/core/base/SkillManager.py b/core/base/SkillManager.py index 557b7ae4d..8feb955d2 100644 --- a/core/base/SkillManager.py +++ b/core/base/SkillManager.py @@ -625,6 +625,7 @@ def updateSkills(self, skills: Union[str, List[str]], withSkillRestart: bool = T try: repository = self.getSkillRepository(skillName=skillName) + repository.fetch(force=True) repository.checkout(tag=self.SkillStoreManager.getSkillUpdateTag(skillName=skillName), force=True) except Exception as e: self.logError(f'Error updating skill **{skillName}** : {e}') diff --git a/core/webui/public b/core/webui/public index ca4c57e2e..3e2ddd251 160000 --- a/core/webui/public +++ b/core/webui/public @@ -1 +1 @@ -Subproject commit ca4c57e2e088fbc9dee55c6a3f16180f37a15bc6 +Subproject commit 3e2ddd25186d4e6d61ad3551290453f12e569ccb diff --git a/requirements.txt b/requirements.txt index de8b9cd76..8f9aaefc3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ Flask~=2.0.2 Flask-Cors~=3.0.10 Flask-Classful~=0.14.2 Markdown~=3.3.6 -AliceGit~=0.0.10 +AliceGit~=0.0.11 requests~=2.26.0 dulwich~=0.20.26 sounddevice~=0.4.3 From 52d2528bc86c3235c73373b83d1363109aaeb696 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Mon, 22 Nov 2021 06:04:11 +0100 Subject: [PATCH 062/129] Load asound file from system when alice config is empty --- core/base/ConfigManager.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/core/base/ConfigManager.py b/core/base/ConfigManager.py index e690010d0..4cd440448 100644 --- a/core/base/ConfigManager.py +++ b/core/base/ConfigManager.py @@ -159,6 +159,13 @@ def _loadCheckAndUpdateAliceConfigFile(self): if aliceConfigs['debug']: logging.getLogger('ProjectAlice').setLevel(logging.DEBUG) + # Load asound if needed + if not aliceConfigs['asoundConfig']: + asound = Path('/etc/asound.conf') + if asound.exists(): + changes = True + aliceConfigs['asoundConfig'] = asound.read_text() + temp = aliceConfigs.copy() for key in temp: if key not in self._aliceTemplateConfigurations: @@ -690,7 +697,7 @@ def refreshStoreData(self): def injectAsound(self, newSettings: str): newSettings = newSettings.replace('\r\n', '\n') - if self.getAliceConfigByName('asoundConfig') != newSettings: + if self.getAliceConfigByName('asoundConfig') and self.getAliceConfigByName('asoundConfig') != newSettings: tmp = Path('/tmp/asound') tmp.write_text(newSettings) self.Commons.runRootSystemCommand(['sudo', 'mv', tmp, '/etc/asound.conf']) From e1a2e68d6e5d7848a6e395868671b336426aff2b Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Mon, 22 Nov 2021 07:00:54 +0100 Subject: [PATCH 063/129] Pydocs for SkillManager --- core/base/SkillManager.py | 243 ++++++++++++++++++++++++++++++------- core/base/model/Manager.py | 4 +- core/webui/public | 2 +- 3 files changed, 203 insertions(+), 46 deletions(-) diff --git a/core/base/SkillManager.py b/core/base/SkillManager.py index 8feb955d2..32f190466 100644 --- a/core/base/SkillManager.py +++ b/core/base/SkillManager.py @@ -18,15 +18,15 @@ # Last modified: 2021.08.02 at 06:12:17 CEST +import traceback + import importlib import json +import requests import shutil -import traceback from contextlib import suppress from pathlib import Path -from typing import Dict, List, Optional, Union - -import requests +from typing import Any, Dict, List, Optional, Union from AliceGit import Exceptions as GitErrors from AliceGit.Exceptions import NotGitRepository, PathNotFoundException @@ -41,6 +41,7 @@ from core.commons import constants from core.dialog.model.DialogSession import DialogSession from core.util.Decorators import IfSetting, Online, deprecated +from core.util.model.AliceEvent import AliceEvent from core.webui.model.UINotificationType import UINotificationType @@ -73,11 +74,11 @@ class SkillManager(Manager): def __init__(self): super().__init__(databaseSchema=self.DATABASE) - self._busyInstalling = None - self._supportedIntents = list() + self._busyInstalling: Optional[AliceEvent] = None + self._supportedIntents: List[Dict[str, Intent]] = list() # This is a list of the skill names installed - self._skillList = list() + self._skillList: List[str] = list() # These are dict of the skills, with name: skill instance self._activeSkills: Dict[str, AliceSkill] = dict() @@ -86,37 +87,65 @@ def __init__(self): @property - def supportedIntents(self) -> list: + def supportedIntents(self) -> List[Dict[str, Intent]]: + """ + Returns a list of all supported intents + :return: + """ return self._supportedIntents @property - def neededSkills(self) -> list: + def neededSkills(self) -> List[str]: + """ + List of skills that are needed for Alice to start + :return: + """ return self.NEEDED_SKILLS @property def activeSkills(self) -> Dict[str, AliceSkill]: + """ + Returns skills that inited and are active + :return: + """ return self._activeSkills @property def deactivatedSkills(self) -> Dict[str, AliceSkill]: + """ + Returns skills that inited but are disabled by user + :return: + """ return self._deactivatedSkills @property - def failedSkills(self) -> dict: + def failedSkills(self) -> Dict[str, FailedAliceSkill]: + """ + Returns skills that failed init + :return: + """ return self._failedSkills @property - def allSkills(self) -> dict: + def allSkills(self) -> Dict[str, Union[AliceSkill, FailedAliceSkill]]: + """ + Returns all skill that inited. This might not contain skills that are physically present but weren't inited + :return: + """ return {**self._activeSkills, **self._deactivatedSkills, **self._failedSkills} @property - def allWorkingSkills(self) -> dict: + def allWorkingSkills(self) -> Dict[str, AliceSkill]: + """ + Returns a list of skills that are functional, but might be activated or deactivated. These skills passed init + :return: + """ return {**self._activeSkills, **self._deactivatedSkills} @@ -160,7 +189,7 @@ def onStop(self): def onQuarterHour(self): - if self._busyInstalling.isSet() or self.ProjectAlice.restart or self.ProjectAlice.updating or self.NluManager.training: + if self._busyInstalling.is_set() or self.ProjectAlice.restart or self.ProjectAlice.updating or self.NluManager.training: return updates = self.checkForSkillUpdates() @@ -169,15 +198,29 @@ def onQuarterHour(self): def notifyInstalling(self): + """ + Sends a MQTT message to notify that Alice is updating + :return: + """ self.MqttManager.mqttBroadcast(topic=constants.TOPIC_SYSTEM_UPDATE, payload={'sticky': True}) def notifyFinishedInstalling(self): + """ + Sends a MQTT message to notify that Alice finished installing something + :return: + """ self.MqttManager.mqttBroadcast(topic=constants.TOPIC_HLC_CLEAR_LEDS) # noinspection SqlResolve def _loadSkills(self) -> List[str]: + """ + Loads skills present on the disk and checks if they are declared in DB and cleans offenders + Loads skills that are in DB and check if they are on the disk and cleans offenders + Returns a sorted list of potentially usable skill names + :return: + """ skills = self.loadSkillsFromDB() skills = [skill['skillName'] for skill in skills] @@ -215,11 +258,21 @@ def _loadSkills(self) -> List[str]: return sorted(data) - def loadSkillsFromDB(self) -> List: + def loadSkillsFromDB(self) -> List[Dict[str, Any]]: + """ + Loads skills from the database + :return: + """ return self.databaseFetch(tableName='skills') def addSkillToDB(self, skillName: str, active: int = 1): + """ + Adds given skill to database + :param skillName: + :param active: + :return: + """ self.DatabaseManager.replace( tableName='skills', values={'skillName': skillName, 'active': active} @@ -228,6 +281,11 @@ def addSkillToDB(self, skillName: str, active: int = 1): # noinspection SqlResolve def removeSkillFromDB(self, skillName: str): + """ + Removes given skill from database + :param skillName: + :return: + """ self.DatabaseManager.delete( tableName='skills', callerName=self.name, @@ -332,7 +390,7 @@ def getSkillRepository(self, skillName: str, directory: str = None) -> Optional[ def downloadSkills(self, skills: Union[str, List[str]]) -> Optional[Dict]: """ - Clones skills + Clones skills. Existance of the skill on line is checked :param skills: :return: Dict: a dict of created repositories """ @@ -412,6 +470,11 @@ def notCompliantSkill(self, skillName: str, exception: SkillNotConditionComplia def getSkillDirectory(self, skillName: str) -> Path: + """ + Returns the full path to a skill + :param skillName: + :return: + """ return Path(self.Commons.rootDir()) / 'skills' / skillName @@ -442,6 +505,8 @@ def getGitRemoteSourceUrl(self, skillName: str, doAuth: bool = True) -> str: def initSkills(self, onlyInit: str = '', reload: bool = False): """ + Initializing skills by checking their condition compliance and instantiating them. + Does check if a skill fails and is required :param onlyInit: If specified, will only init the given skill name :param reload: If the skill is already instantiated, performs a module reload, after an update per example. :return: @@ -511,11 +576,23 @@ def initSkills(self, onlyInit: str = '', reload: bool = False): def getSkillInstallFilePath(self, skillName: str) -> Path: + """ + Returns the full path to a skill's install file + :param skillName: + :return: + """ return self.getSkillDirectory(skillName=skillName) / f'{skillName}.install' # noinspection PyTypeChecker def instantiateSkill(self, skillName: str, skillResource: str = '', reload: bool = False) -> Optional[AliceSkill]: + """ + Instantiates the given skill at the gien path + :param skillName: + :param skillResource: + :param reload: + :return: + """ instance: Optional[AliceSkill] = None skillResource = skillResource or skillName @@ -541,6 +618,11 @@ def instantiateSkill(self, skillName: str, skillResource: str = '', reload: bool def isSkillActive(self, skillName: str) -> bool: + """ + Returns true or false depending if the skill is declared as active + :param skillName: + :return: + """ if skillName in self._activeSkills: return self._activeSkills[skillName].active elif skillName in self._skillList: @@ -553,6 +635,11 @@ def isSkillActive(self, skillName: str) -> bool: def checkSkillConditions(self, installer: dict = None) -> bool: + """ + Checks if the given skill is compliant to it's conditions + :param installer: + :return: + """ conditions = { 'aliceMinVersion': installer['aliceMinVersion'], **installer.get('conditions', dict()) @@ -649,6 +736,11 @@ def updateSkills(self, skills: Union[str, List[str]], withSkillRestart: bool = T def stopSkill(self, skillName: str) -> Optional[AliceSkill]: + """ + Stops the given skill + :param skillName: + :return: + """ skill = None if skillName in self._activeSkills: skill = self._activeSkills.pop(skillName, None) @@ -663,6 +755,12 @@ def stopSkill(self, skillName: str) -> Optional[AliceSkill]: def configureSkillIntents(self, skillName: str, state: bool): + """ + Subs or unsubs the skill intents. Alice only recognizes subscribed intents + :param skillName: + :param state: + :return: + """ try: skills = self.allWorkingSkills confs = [{ @@ -682,11 +780,21 @@ def configureSkillIntents(self, skillName: str, state: bool): def isIntentInUse(self, intent: Intent, filtered: list) -> bool: + """ + Returns whether an intent is used by any WORKING skill + :param intent: + :param filtered: + :return: + """ skills = self.allWorkingSkills return any(intent in skill.supportedIntents for name, skill in skills.items() if name not in filtered) def startAllSkills(self): + """ + Starts all the discoverd skills + :return: + """ supportedIntents = list() for skillName in self._activeSkills.copy(): @@ -703,7 +811,12 @@ def startAllSkills(self): self.logInfo(f'Skills started. {len(supportedIntents)} intents supported') - def startSkill(self, skillName: str) -> dict: + def startSkill(self, skillName: str) -> Dict[str, Intent]: + """ + Starts a skill + :param skillName: + :return: + """ if skillName in self._activeSkills: skillInstance = self._activeSkills[skillName] elif skillName in self._deactivatedSkills: @@ -758,6 +871,12 @@ def startSkill(self, skillName: str) -> dict: def deactivateSkill(self, skillName: str, persistent: bool = False): + """ + Deactivates a skill and broadcasts it + :param skillName: + :param persistent: + :return: + """ if skillName in self._activeSkills: skillInstance = self.stopSkill(skillName=skillName) if skillInstance: @@ -779,6 +898,12 @@ def deactivateSkill(self, skillName: str, persistent: bool = False): def activateSkill(self, skillName: str, persistent: bool = False): + """ + Activates a skill and broadcasts it + :param skillName: + :param persistent: + :return: + """ if skillName not in self._deactivatedSkills and skillName not in self._failedSkills: self.logWarning(f'Skill "{skillName} is not deactivated or failed') return @@ -804,6 +929,12 @@ def activateSkill(self, skillName: str, persistent: bool = False): def toggleSkillState(self, skillName: str, persistent: bool = False): + """ + Activate the given skill if deactivated or deactivates the given skill if actived + :param skillName: + :param persistent: + :return: + """ if self.isSkillActive(skillName): self.deactivateSkill(skillName=skillName, persistent=persistent) else: @@ -811,7 +942,12 @@ def toggleSkillState(self, skillName: str, persistent: bool = False): def changeSkillStateInDB(self, skillName: str, newState: bool): - # Changes the state of a skill in db + """ + Updates the given skill DB entry state + :param skillName: + :param newState: + :return: + """ self.DatabaseManager.update( tableName='skills', callerName=self.name, @@ -823,6 +959,11 @@ def changeSkillStateInDB(self, skillName: str, newState: bool): def dispatchMessage(self, session: DialogSession) -> bool: + """ + Dispatches a MQTT message to skills until one accepts it and returns True. If the intent wasn't consumed, return False + :param session: + :return: + """ for skillName, skillInstance in self._activeSkills.items(): try: consumed = skillInstance.onMessageDispatch(session) @@ -857,7 +998,7 @@ def dispatchMessage(self, session: DialogSession) -> bool: @IfSetting(settingName='stayCompletelyOffline', settingValue=False) def checkForSkillUpdates(self, skillToCheck: str = None) -> List[str]: """ - Check all installed skills for availability of updates. + Checks all installed skills for availability of updates. Includes failed skills but not inactive. :param skillToCheck: :return: @@ -914,6 +1055,12 @@ def checkForSkillUpdates(self, skillToCheck: str = None) -> List[str]: def getSkillInstance(self, skillName: str, silent: bool = False) -> Optional[AliceSkill]: + """ + Retuns a skill instance, if available + :param skillName: + :param silent: + :return: + """ if skillName in self._activeSkills: return self._activeSkills[skillName] elif skillName in self._deactivatedSkills: @@ -957,6 +1104,11 @@ def skillBroadcast(self, method: str, filterOut: list = None, **kwargs): def removeSkill(self, skillName: str): + """ + Deletes a skill completely + :param skillName: + :return: + """ if skillName not in self.allSkills: return @@ -983,6 +1135,11 @@ def removeSkill(self, skillName: str): def reloadSkill(self, skillName: str): + """ + Reloads a skill + :param skillName: + :return: + """ self.logInfo(f'Reloading skill "{skillName}"') self.stopSkill(skillName=skillName) @@ -992,6 +1149,10 @@ def reloadSkill(self, skillName: str): def allScenarioNodes(self) -> Dict[str, tuple]: + """ + Retunrs list of Node-Red nodes added by skills + :return: + """ ret = dict() for skillName, skillInstance in self._activeSkills.items(): if not skillInstance.hasScenarioNodes(): @@ -1003,6 +1164,11 @@ def allScenarioNodes(self) -> Dict[str, tuple]: def skillScenarioNode(self, skillName: str) -> Optional[Path]: + """ + Returns list of Node-Red nodes for the given skill + :param skillName: + :return: + """ if skillName not in self.allWorkingSkills: return None @@ -1010,6 +1176,11 @@ def skillScenarioNode(self, skillName: str) -> Optional[Path]: def getSkillScenarioVersion(self, skillName: str) -> Version: + """ + Return Node Red scenario node version for the given skill + :param skillName: + :return: + """ if skillName not in self._skillList: return Version.fromString('0.0.0') else: @@ -1053,6 +1224,11 @@ def isSkillUserModified(self, skillName: str) -> bool: def createNewSkill(self, skillDefinition: dict) -> bool: + """ + Used to create a new skill by the usr + :param skillDefinition: + :return: + """ try: self.logInfo(f'Creating new skill "{skillDefinition["name"]}"') @@ -1148,6 +1324,12 @@ def createNewSkill(self, skillDefinition: dict) -> bool: def uploadSkillToGithub(self, skillName: str, skillDesc: str) -> bool: + """ + sends skill name oin Github + :param skillName: + :param skillDesc: + :return: + """ try: self.logInfo(f'Uploading {skillName} to Github') @@ -1172,31 +1354,6 @@ def uploadSkillToGithub(self, skillName: str, skillDesc: str) -> bool: return False - - - - - - - - # def onAssistantInstalled(self, **kwargs): - # argv = kwargs.get('skillsInfos', dict()) - # if not argv: - # return - # - # for skillName, skill in argv.items(): - # try: - # self._startSkill(skillName=skillName) - # except SkillStartDelayed: - # self.logInfo(f'Skill "{skillName}" start is delayed') - # except KeyError as e: - # self.logError(f'Skill "{skillName} not found, skipping: {e}') - # continue - # - # self._activeSkills[skillName].onBooted() - - - @deprecated def setSkillModified(self, skillName: str, modified: bool): return diff --git a/core/base/model/Manager.py b/core/base/model/Manager.py index b9912ba08..01ccd02a6 100644 --- a/core/base/model/Manager.py +++ b/core/base/model/Manager.py @@ -17,7 +17,7 @@ # # Last modified: 2021.04.13 at 12:56:46 CEST -from typing import List, Optional +from typing import Any, Dict, List, Optional from core.base.SuperManager import SuperManager from core.base.model.ProjectAliceObject import ProjectAliceObject @@ -99,7 +99,7 @@ def _initDB(self): # HELPERS - def databaseFetch(self, tableName: str, query: str = None, values: dict = None) -> List: + def databaseFetch(self, tableName: str, query: str = None, values: dict = None) -> List[Dict[str, Any]]: if not query: query = 'SELECT * FROM :__table__' diff --git a/core/webui/public b/core/webui/public index 3e2ddd251..ca4c57e2e 160000 --- a/core/webui/public +++ b/core/webui/public @@ -1 +1 @@ -Subproject commit 3e2ddd25186d4e6d61ad3551290453f12e569ccb +Subproject commit ca4c57e2e088fbc9dee55c6a3f16180f37a15bc6 From e2a2226fab0cf073f559d0ec02050bd828c0e4bd Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Tue, 23 Nov 2021 12:03:22 +0100 Subject: [PATCH 064/129] Skill store preloads skill conditions --- core/base/SkillManager.py | 60 ++++++++++++++++++++++++---------- core/base/SkillStoreManager.py | 8 +++-- 2 files changed, 48 insertions(+), 20 deletions(-) diff --git a/core/base/SkillManager.py b/core/base/SkillManager.py index 32f190466..8d7740ec4 100644 --- a/core/base/SkillManager.py +++ b/core/base/SkillManager.py @@ -634,10 +634,11 @@ def isSkillActive(self, skillName: str) -> bool: return False - def checkSkillConditions(self, installer: dict = None) -> bool: + def checkSkillConditions(self, installer: dict = None, checkOnly=False) -> Union[bool, Dict[str, str]]: """ Checks if the given skill is compliant to it's conditions :param installer: + :param checkOnly: Do not perform any other action (download other skill, etc) but checking conditions :return: """ conditions = { @@ -646,40 +647,60 @@ def checkSkillConditions(self, installer: dict = None) -> bool: } notCompliant = 'Skill is not compliant' + notCompliantRules = list() - if 'aliceMinVersion' in conditions and \ - Version.fromString(conditions['aliceMinVersion']) > Version.fromString(constants.VERSION): - raise SkillNotConditionCompliant(message=notCompliant, skillName=installer['name'], condition='Alice minimum version', conditionValue=conditions['aliceMinVersion']) + if 'aliceMinVersion' in conditions and Version.fromString(conditions['aliceMinVersion']) > Version.fromString(constants.VERSION): + if not checkOnly: + raise SkillNotConditionCompliant(message=notCompliant, skillName=installer['name'], condition='Alice minimum version', conditionValue=conditions['aliceMinVersion']) + else: + notCompliantRules.append({'Alice version': conditions['aliceMinVersion']}) for conditionName, conditionValue in conditions.items(): if conditionName == 'lang' and self.LanguageManager.activeLanguage not in conditionValue: - raise SkillNotConditionCompliant(message=notCompliant, skillName=installer['name'], condition=conditionName, conditionValue=conditionValue) + if not checkOnly: + raise SkillNotConditionCompliant(message=notCompliant, skillName=installer['name'], condition=conditionName, conditionValue=conditionValue) + else: + notCompliantRules.append({conditionName: conditionValue}) elif conditionName == 'online': - if conditionValue and self.ConfigManager.getAliceConfigByName('stayCompletelyOffline') \ - or not conditionValue and not self.ConfigManager.getAliceConfigByName('stayCompletelyOffline'): - raise SkillNotConditionCompliant(message=notCompliant, skillName=installer['name'], condition=conditionName, conditionValue=conditionValue) + if conditionValue and self.ConfigManager.getAliceConfigByName('stayCompletelyOffline') or not conditionValue and not self.ConfigManager.getAliceConfigByName('stayCompletelyOffline'): + if not checkOnly: + raise SkillNotConditionCompliant(message=notCompliant, skillName=installer['name'], condition=conditionName, conditionValue=conditionValue) + else: + notCompliantRules.append({conditionName: conditionValue}) elif conditionName == 'skill': for requiredSkill in conditionValue: if requiredSkill in self._skillList and not self.isSkillActive(skillName=installer['name']): - raise SkillNotConditionCompliant(message=notCompliant, skillName=installer['name'], condition=conditionName, conditionValue=conditionValue) - elif requiredSkill not in self._skillList: - self.logInfo(f'Skill {installer["name"]} has another skill as dependency, adding download') - try: - self.downloadSkills(skills=requiredSkill) - except: + if not checkOnly: raise SkillNotConditionCompliant(message=notCompliant, skillName=installer['name'], condition=conditionName, conditionValue=conditionValue) + else: + notCompliantRules.append({conditionName: conditionValue}) + elif requiredSkill not in self._skillList: + if not checkOnly: + self.logInfo(f'Skill {installer["name"]} has another skill as dependency, adding download') + try: + self.downloadSkills(skills=requiredSkill) + except: + raise SkillNotConditionCompliant(message=notCompliant, skillName=installer['name'], condition=conditionName, conditionValue=conditionValue) + else: + notCompliantRules.append({conditionName: conditionValue}) elif conditionName == 'notSkill': for excludedSkill in conditionValue: author, name = excludedSkill.split('/') if name in self._skillList and self.isSkillActive(skillName=installer['name']): - raise SkillNotConditionCompliant(message=notCompliant, skillName=installer['name'], condition=conditionName, conditionValue=conditionValue) + if not checkOnly: + raise SkillNotConditionCompliant(message=notCompliant, skillName=installer['name'], condition=conditionName, conditionValue=conditionValue) + else: + notCompliantRules.append({conditionName: conditionValue}) elif conditionName == 'asrArbitraryCapture': if conditionValue and not self.ASRManager.asr.capableOfArbitraryCapture: - raise SkillNotConditionCompliant(message=notCompliant, skillName=installer['name'], condition=conditionName, conditionValue=conditionValue) + if not checkOnly: + raise SkillNotConditionCompliant(message=notCompliant, skillName=installer['name'], condition=conditionName, conditionValue=conditionValue) + else: + notCompliantRules.append({conditionName: conditionValue}) elif conditionName == 'activeManager': for manager in conditionValue: @@ -688,9 +709,12 @@ def checkSkillConditions(self, installer: dict = None) -> bool: man = SuperManager.getInstance().getManager(manager) if not man or not man.isActive: - raise SkillNotConditionCompliant(message=notCompliant, skillName=installer['name'], condition=conditionName, conditionValue=conditionValue) + if not checkOnly: + raise SkillNotConditionCompliant(message=notCompliant, skillName=installer['name'], condition=conditionName, conditionValue=conditionValue) + else: + notCompliantRules.append({conditionName: conditionValue}) - return True + return True if checkOnly else notCompliantRules def updateSkills(self, skills: Union[str, List[str]], withSkillRestart: bool = True): diff --git a/core/base/SkillStoreManager.py b/core/base/SkillStoreManager.py index b1d19d9c0..3401bd721 100644 --- a/core/base/SkillStoreManager.py +++ b/core/base/SkillStoreManager.py @@ -66,7 +66,7 @@ def refreshStoreData(self): return self._skillStoreData = req.json() - self.markInstalledSkills() + self.checkConditions() if not self.ConfigManager.getAliceConfigByName('suggestSkillsToInstall'): return @@ -78,10 +78,14 @@ def refreshStoreData(self): self.prepareSamplesData(req.json()) - def markInstalledSkills(self): + def checkConditions(self): for skillName, skillData in self._skillStoreData.items(): skillData['installed'] = skillName in self.SkillManager.allSkills.keys() + offendingConditions = self.SkillManager.checkSkillConditions(installer=skillData, checkOnly=True) + skillData['offendingConditions'] = offendingConditions + skillData['compatible'] = False if offendingConditions else True + def prepareSamplesData(self, data: dict): if not data: From af5bf52367a8755de484109b339afa04062e597b Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Tue, 23 Nov 2021 12:23:41 +0100 Subject: [PATCH 065/129] Adding sonar project version for new code coverage --- sonar-project.properties | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sonar-project.properties b/sonar-project.properties index e1abec4f4..d333b0fd5 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -20,6 +20,7 @@ # # suppress inspection "UnusedProperty" for whole file +sonar.projectVersion=1.0.0-rc2 sonar.projectKey=project-alice-assistant_ProjectAlice sonar.organization=project-alice-assistant sonar.python.coverage.reportPaths=./coverage.xml @@ -44,4 +45,4 @@ sonar.coverage.exclusions=core/interface/static/js/jquery* # Encoding of the source code. Default is default system encoding #sonar.sourceEncoding=UTF-8 -sonar.python.version=3.7 \ No newline at end of file +sonar.python.version=3.7 From 0abb4cfa0a2b3f9f4c26af8e07447435cd81945f Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Tue, 23 Nov 2021 13:58:45 +0100 Subject: [PATCH 066/129] Fixing old class definition to new style --- core/Initializer.py | 6 +++--- core/asr/model/ASRResult.py | 2 +- core/base/SuperManager.py | 2 +- core/base/model/Intent.py | 2 +- core/base/model/ProjectAliceObject.py | 2 +- core/base/model/State.py | 2 +- core/base/model/Version.py | 2 +- core/commons/model/Singleton.py | 2 +- core/commons/model/Slot.py | 2 +- core/dialog/model/DialogSession.py | 2 +- core/dialog/model/DialogState.py | 2 +- core/dialog/model/DialogTemplate.py | 2 +- core/dialog/model/DialogTemplateIntent.py | 2 +- core/dialog/model/DialogTemplateSlotType.py | 2 +- core/dialog/model/MultiIntent.py | 2 +- core/util/Stopwatch.py | 2 +- core/util/model/AliceSubprocess.py | 2 +- core/util/model/Logger.py | 2 +- core/util/model/ThreadTimer.py | 2 +- core/webui/model/WidgetPage.py | 2 +- core/webui/public | 2 +- tests/base/model/test_Intent.py | 2 +- tests/commons/test_CommonsManager.py | 20 ++++++++++---------- tests/util/test_Decorators.py | 8 ++++---- 24 files changed, 38 insertions(+), 38 deletions(-) diff --git a/core/Initializer.py b/core/Initializer.py index 970559e01..4b97054af 100644 --- a/core/Initializer.py +++ b/core/Initializer.py @@ -55,7 +55,7 @@ def __getitem__(self, item): return '' -class SimpleLogger: +class SimpleLogger(object): def __init__(self, prepend: str = None): self._prepend = f'[{prepend}]' @@ -85,7 +85,7 @@ def spacer(self, msg: str) -> str: return msg -class PreInit: +class PreInit(object): """ Pre init checks and makes sure vital stuff is installed and running. Not much, but internet, venv and so on Pre init is meant to run on the system python and not on the venv @@ -361,7 +361,7 @@ def setServiceFileTo(pointer: str): time.sleep(1) -class Initializer: +class Initializer(object): PIP = './venv/bin/pip' diff --git a/core/asr/model/ASRResult.py b/core/asr/model/ASRResult.py index 2759153f4..a67272225 100644 --- a/core/asr/model/ASRResult.py +++ b/core/asr/model/ASRResult.py @@ -23,7 +23,7 @@ @dataclass -class ASRResult: +class ASRResult(object): text: str session: DialogSession likelihood: float diff --git a/core/base/SuperManager.py b/core/base/SuperManager.py index 53edc8eb0..cb7d4bb75 100644 --- a/core/base/SuperManager.py +++ b/core/base/SuperManager.py @@ -26,7 +26,7 @@ from core.util.model.Logger import Logger -class SuperManager: +class SuperManager(object): NAME = 'SuperManager' _INSTANCE = None diff --git a/core/base/model/Intent.py b/core/base/model/Intent.py index ce0746acb..4433abf74 100644 --- a/core/base/model/Intent.py +++ b/core/base/model/Intent.py @@ -25,7 +25,7 @@ @dataclass -class Intent: +class Intent(object): topic: str = field(init=False) action: str = field(repr=False) userIntent: bool = True diff --git a/core/base/model/ProjectAliceObject.py b/core/base/model/ProjectAliceObject.py index 82bbcf710..1cdf58370 100644 --- a/core/base/model/ProjectAliceObject.py +++ b/core/base/model/ProjectAliceObject.py @@ -68,7 +68,7 @@ from core.webui.WebUINotificationManager import WebUINotificationManager -class ProjectAliceObject: +class ProjectAliceObject(object): DEPENDENCIES = { 'internal': {}, 'external': {}, diff --git a/core/base/model/State.py b/core/base/model/State.py index 88f4661cd..b34adeec6 100644 --- a/core/base/model/State.py +++ b/core/base/model/State.py @@ -25,7 +25,7 @@ @dataclass -class State: +class State(object): name: str currentState: StateType = StateType.BORN logger: Logger = Logger(prepend='[State]') diff --git a/core/base/model/Version.py b/core/base/model/Version.py index 5de3ebcb6..4c02f67b1 100644 --- a/core/base/model/Version.py +++ b/core/base/model/Version.py @@ -24,7 +24,7 @@ @dataclass(order=True) -class Version: +class Version(object): mainVersion: int = 0 updateVersion: int = 0 hotfix: int = 0 diff --git a/core/commons/model/Singleton.py b/core/commons/model/Singleton.py index f6ba66ba0..a0e46e0c8 100644 --- a/core/commons/model/Singleton.py +++ b/core/commons/model/Singleton.py @@ -20,7 +20,7 @@ from core.util.model.Logger import Logger -class Singleton: +class Singleton(object): INSTANCE = None diff --git a/core/commons/model/Slot.py b/core/commons/model/Slot.py index 6cc03b843..8c2fee58f 100644 --- a/core/commons/model/Slot.py +++ b/core/commons/model/Slot.py @@ -22,7 +22,7 @@ @dataclass -class Slot: +class Slot(object): slotName: str entity: str rawValue: str diff --git a/core/dialog/model/DialogSession.py b/core/dialog/model/DialogSession.py index 4eff95bc3..fe807fe61 100644 --- a/core/dialog/model/DialogSession.py +++ b/core/dialog/model/DialogSession.py @@ -30,7 +30,7 @@ @dataclass -class DialogSession: +class DialogSession(object): deviceUid: str sessionId: str = '' increaseTimeout: int = 0 diff --git a/core/dialog/model/DialogState.py b/core/dialog/model/DialogState.py index 1254c41b2..b67fa5d89 100644 --- a/core/dialog/model/DialogState.py +++ b/core/dialog/model/DialogState.py @@ -20,7 +20,7 @@ from core.base.SuperManager import SuperManager -class DialogState: +class DialogState(object): def __init__(self, state: str): if ':' not in state: diff --git a/core/dialog/model/DialogTemplate.py b/core/dialog/model/DialogTemplate.py index 8fd3b2d8f..2f38b7364 100644 --- a/core/dialog/model/DialogTemplate.py +++ b/core/dialog/model/DialogTemplate.py @@ -29,7 +29,7 @@ @dataclass -class DialogTemplate: +class DialogTemplate(object): skill: str slotTypes: list intents: list diff --git a/core/dialog/model/DialogTemplateIntent.py b/core/dialog/model/DialogTemplateIntent.py index 51885664b..7c8c14b61 100644 --- a/core/dialog/model/DialogTemplateIntent.py +++ b/core/dialog/model/DialogTemplateIntent.py @@ -21,7 +21,7 @@ @dataclass -class DialogTemplateIntent: +class DialogTemplateIntent(object): name: str enabledByDefault: bool utterances: list = field(default_factory=list) diff --git a/core/dialog/model/DialogTemplateSlotType.py b/core/dialog/model/DialogTemplateSlotType.py index ea01985a9..4fa24aa57 100644 --- a/core/dialog/model/DialogTemplateSlotType.py +++ b/core/dialog/model/DialogTemplateSlotType.py @@ -21,7 +21,7 @@ @dataclass -class DialogTemplateSlotType: +class DialogTemplateSlotType(object): name: str automaticallyExtensible: bool useSynonyms: bool diff --git a/core/dialog/model/MultiIntent.py b/core/dialog/model/MultiIntent.py index 90dc943d1..ca72b5001 100644 --- a/core/dialog/model/MultiIntent.py +++ b/core/dialog/model/MultiIntent.py @@ -25,7 +25,7 @@ @dataclass -class MultiIntent: +class MultiIntent(object): session: DialogSession processedString: str intents: Deque[str] = field(default_factory=deque) diff --git a/core/util/Stopwatch.py b/core/util/Stopwatch.py index e594608c1..c68c4e195 100644 --- a/core/util/Stopwatch.py +++ b/core/util/Stopwatch.py @@ -20,7 +20,7 @@ import time -class Stopwatch: +class Stopwatch(object): def __init__(self, precision: int = 2): self._startTime = None diff --git a/core/util/model/AliceSubprocess.py b/core/util/model/AliceSubprocess.py index 251e85df4..f921b4fa2 100644 --- a/core/util/model/AliceSubprocess.py +++ b/core/util/model/AliceSubprocess.py @@ -21,7 +21,7 @@ from typing import Callable -class AliceSubprocess: +class AliceSubprocess(object): def __init__(self, name: str, cmd: str, stoppedCallback: Callable, autoRestart: bool): self.name = name diff --git a/core/util/model/Logger.py b/core/util/model/Logger.py index 42ebe4449..8b51846d8 100644 --- a/core/util/model/Logger.py +++ b/core/util/model/Logger.py @@ -23,7 +23,7 @@ from typing import Match, Union -class Logger: +class Logger(object): def __init__(self, prepend: str = None, **kwargs): self._prepend = prepend diff --git a/core/util/model/ThreadTimer.py b/core/util/model/ThreadTimer.py index e03da11d1..56483961a 100644 --- a/core/util/model/ThreadTimer.py +++ b/core/util/model/ThreadTimer.py @@ -23,7 +23,7 @@ @dataclass -class ThreadTimer: +class ThreadTimer(object): callback: Callable args: list = field(default_factory=list) kwargs: dict = field(default_factory=dict) diff --git a/core/webui/model/WidgetPage.py b/core/webui/model/WidgetPage.py index 20483a427..543a62d1f 100644 --- a/core/webui/model/WidgetPage.py +++ b/core/webui/model/WidgetPage.py @@ -21,7 +21,7 @@ @dataclass -class WidgetPage: +class WidgetPage(object): data: dict id: int = 0 icon: str = '' diff --git a/core/webui/public b/core/webui/public index ca4c57e2e..3e2ddd251 160000 --- a/core/webui/public +++ b/core/webui/public @@ -1 +1 @@ -Subproject commit ca4c57e2e088fbc9dee55c6a3f16180f37a15bc6 +Subproject commit 3e2ddd25186d4e6d61ad3551290453f12e569ccb diff --git a/tests/base/model/test_Intent.py b/tests/base/model/test_Intent.py index faed28d0e..acd688297 100644 --- a/tests/base/model/test_Intent.py +++ b/tests/base/model/test_Intent.py @@ -98,7 +98,7 @@ def dummyCallable(): @patch('core.base.SuperManager.SuperManager') def test_get_mapping(self, mock_superManager): @dataclass - class Session: + class Session(object): currentState: str diff --git a/tests/commons/test_CommonsManager.py b/tests/commons/test_CommonsManager.py index ee42dd645..1055c7762 100644 --- a/tests/commons/test_CommonsManager.py +++ b/tests/commons/test_CommonsManager.py @@ -49,7 +49,7 @@ def test_dictMaxValue(self): def test_payload(self): - class MQTTMessage: + class MQTTMessage(object): def __init__(self, payload): self.payload = payload @@ -87,7 +87,7 @@ def __init__(self, payload): def test_parseSlotsToObjects(self): - class MQTTMessage: + class MQTTMessage(object): def __init__(self, payload): self.payload = payload @@ -101,7 +101,7 @@ def __init__(self, payload): def test_parseSlots(self): - class MQTTMessage: + class MQTTMessage(object): def __init__(self, payload): self.payload = payload @@ -122,7 +122,7 @@ def __init__(self, payload): def test_parseSessionId(self): - class MQTTMessage: + class MQTTMessage(object): def __init__(self, payload): self.payload = payload @@ -136,7 +136,7 @@ def __init__(self, payload): def test_parseCustomData(self): - class MQTTMessage: + class MQTTMessage(object): def __init__(self, payload): self.payload = payload @@ -163,7 +163,7 @@ def __init__(self, payload): @mock.patch('core.base.SuperManager.SuperManager') def test_parseDeviceUid(self, mock_superManager): - class MQTTMessage: + class MQTTMessage(object): def __init__(self, payload): self.payload = payload @@ -195,13 +195,13 @@ def test_getDuration(self): """Test getDuration method""" - class DialogSession: + class DialogSession(object): def __init__(self, slotsAsObjects: dict): self.slotsAsObjects = slotsAsObjects - class TimeObject: + class TimeObject(object): def __init__(self, value: dict, entity: str = 'snips/duration'): self.value = value @@ -267,13 +267,13 @@ def test_angleToCardinal(self): def test_isYes(self): - class DialogSession: + class DialogSession(object): def __init__(self, slotsAsObjects: dict): self.slotsAsObjects = slotsAsObjects - class Slot: + class Slot(object): def __init__(self, value): self.value = {'value': value} diff --git a/tests/util/test_Decorators.py b/tests/util/test_Decorators.py index 630250bfc..654f4f30f 100644 --- a/tests/util/test_Decorators.py +++ b/tests/util/test_Decorators.py @@ -38,7 +38,7 @@ def legacy_function(): @mock.patch('core.util.Decorators.SuperManager') def test_online(self, mock_superManager): - class AliceSkill: + class AliceSkill(object): @property def name(self): return 'AliceSkill' @@ -63,7 +63,7 @@ def catch_offlineHandler(self, *args, **kwargs): def catch_staticMethod(*args, **kwargs): raise Exception - class InternetManager: + class InternetManager(object): def __init__(self, online: bool, keepOffline: bool = False): self.online = online self.keepOffline = keepOffline @@ -161,7 +161,7 @@ def checkOnlineState(self) -> bool: @mock.patch('core.util.Decorators.Logger') @mock.patch('core.util.Decorators.SuperManager') def test_anyExcept(self, mock_superManager, mock_logger): - class AliceSkill: + class AliceSkill(object): @property def name(self): return 'AliceSkill' @@ -257,7 +257,7 @@ def catch_staticMethod(*args, **kwargs): @mock.patch('core.util.Decorators.Intent') def test_IntentHandler(self, mock_intent): - class Example: + class Example(object): @IntentHandler('intent1') def single_decorator(self, *args, **kwargs): return self, args, kwargs From b6ebd78252f898d5a4c7e2f65ce8408e03986472 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Tue, 23 Nov 2021 14:19:19 +0100 Subject: [PATCH 067/129] Cleanup and Sonar fixes --- core/asr/model/PocketSphinxAsr.py | 2 +- core/asr/model/Recorder.py | 4 +- core/myHome/LocationManager.py | 2 +- core/myHome/model/Construction.py | 76 +++------------------- core/myHome/model/Furniture.py | 72 ++------------------- core/myHome/model/Location.py | 2 +- core/myHome/model/MyHomeObject.py | 93 +++++++++++++++++++++++++++ core/util/ThreadManager.py | 2 +- core/util/model/ThreadTimer.py | 4 +- core/voice/model/PorcupineWakeword.py | 6 +- 10 files changed, 119 insertions(+), 144 deletions(-) create mode 100644 core/myHome/model/MyHomeObject.py diff --git a/core/asr/model/PocketSphinxAsr.py b/core/asr/model/PocketSphinxAsr.py index 3a403908f..38b9ff5af 100644 --- a/core/asr/model/PocketSphinxAsr.py +++ b/core/asr/model/PocketSphinxAsr.py @@ -137,7 +137,7 @@ def decodeStream(self, session: DialogSession) -> Optional[ASRResult]: self._decoder.start_utt() inSpeech = False for chunk in recorder: - if self._timeout.isSet(): + if self._timeout.is_set(): break self._decoder.process_raw(chunk, False, False) diff --git a/core/asr/model/Recorder.py b/core/asr/model/Recorder.py index 82af4e26d..c41d883f7 100644 --- a/core/asr/model/Recorder.py +++ b/core/asr/model/Recorder.py @@ -20,7 +20,7 @@ import io import queue import wave -from typing import Optional +from typing import Generator import paho.mqtt.client as mqtt @@ -109,7 +109,7 @@ def __iter__(self): yield b''.join(data) - def audioStream(self) -> Optional[bytes]: + def audioStream(self) -> Generator: while not self._buffer.empty() or self._recording: if self._timeoutFlag.is_set(): return diff --git a/core/myHome/LocationManager.py b/core/myHome/LocationManager.py index 92ed95ba5..3b377feea 100644 --- a/core/myHome/LocationManager.py +++ b/core/myHome/LocationManager.py @@ -306,7 +306,7 @@ def updateLocation(self, locId: int, data: dict) -> Location: location.parentLocation = data['parentLocation'] if 'synonyms' in data: - location.updatesynonyms(set(data['synonyms'])) + location.updateSynonyms(set(data['synonyms'])) if 'settings' in data: location.updateSettings(data['settings']) diff --git a/core/myHome/model/Construction.py b/core/myHome/model/Construction.py index 90334c959..0f93df7f6 100644 --- a/core/myHome/model/Construction.py +++ b/core/myHome/model/Construction.py @@ -1,6 +1,6 @@ # Copyright (c) 2021 # -# This file, Construction.py, is part of Project Alice. +# This file, Furniture.py, is part of Project Alice. # # Project Alice is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -17,76 +17,16 @@ # # Last modified: 2021.04.13 at 12:56:47 CEST -import json -from dataclasses import dataclass, field +from dataclasses import dataclass +from typing import Dict -from core.base.model.ProjectAliceObject import ProjectAliceObject +from core.myHome.model.MyHomeObject import MyHomeObject @dataclass -class Construction(ProjectAliceObject): +class Construction(MyHomeObject): data: dict - id: int = field(init=False) - parentLocation: int = field(init=False) - settings: dict = field(init=False) - - - def __post_init__(self): - self.id = self.data.get('id', -1) - self.parentLocation = self.data['parentLocation'] - self.settings = json.loads(self.data.get('settings', '{}')) if isinstance(self.data.get('settings', '{}'), str) else self.data.get('settings', dict) - - settings = { - 'x': 0, - 'y': 0, - 'z': 0, - 'w': 10, - 'h': 50, - 'r': 0, - 'c': '', - 'b': '' - } - - self.settings = {**settings, **self.settings} - - if self.id == -1: - self.saveToDB() - - - # noinspection SqlResolve - def saveToDB(self): - if self.id != -1: - self.DatabaseManager.replace( - tableName=self.LocationManager.CONSTRUCTIONS_TABLE, - query='REPLACE INTO :__table__ (id, parentLocation, settings) VALUES (:id, :parentLocation, :settings)', - callerName=self.LocationManager.name, - values={ - 'id' : self.id, - 'parentLocation': self.parentLocation, - 'settings' : json.dumps(self.settings) - } - ) - else: - constructionId = self.DatabaseManager.insert( - tableName=self.LocationManager.CONSTRUCTIONS_TABLE, - callerName=self.LocationManager.name, - values={ - 'parentLocation': self.parentLocation, - 'settings' : json.dumps(self.settings) - } - ) - - self.id = constructionId - - - def updateSettings(self, settings: dict): - self.settings = {**self.settings, **settings} - - - def toDict(self) -> dict: - return { - 'id' : self.id, - 'parentLocation': self.parentLocation, - 'settings' : self.settings - } + def __init__(self, data: Dict): + self.myDatabase = self.LocationManager.CONSTRUCTIONS_TABLE + super().__init__(data) diff --git a/core/myHome/model/Furniture.py b/core/myHome/model/Furniture.py index 6385d86b0..25e9cffb2 100644 --- a/core/myHome/model/Furniture.py +++ b/core/myHome/model/Furniture.py @@ -17,74 +17,16 @@ # # Last modified: 2021.04.13 at 12:56:47 CEST -import json -from dataclasses import dataclass, field +from dataclasses import dataclass +from typing import Dict -from core.base.model.ProjectAliceObject import ProjectAliceObject +from core.myHome.model.MyHomeObject import MyHomeObject @dataclass -class Furniture(ProjectAliceObject): +class Furniture(MyHomeObject): data: dict - id: int = field(init=False) - parentLocation: int = field(init=False) - settings: dict = field(init=False) - - - def __post_init__(self): - self.id = self.data.get('id', -1) - self.parentLocation = self.data['parentLocation'] - self.settings = json.loads(self.data.get('settings', '{}')) if isinstance(self.data.get('settings', '{}'), str) else self.data.get('settings', dict) - - settings = { - 'x': 0, - 'y': 0, - 'z': 0, - 'w': 25, - 'h': 25, - 'r': 0, - 't': '' - } - self.settings = {**settings, **self.settings} - - if self.id == -1: - self.saveToDB() - - - # noinspection SqlResolve - def saveToDB(self): - if self.id != -1: - self.DatabaseManager.replace( - tableName=self.LocationManager.FURNITURE_TABLE, - query='REPLACE INTO :__table__ (id, parentLocation, settings) VALUES (:id, :parentLocation, :settings)', - callerName=self.LocationManager.name, - values={ - 'id' : self.id, - 'parentLocation': self.parentLocation, - 'settings' : json.dumps(self.settings) - } - ) - else: - constructionId = self.DatabaseManager.insert( - tableName=self.LocationManager.FURNITURE_TABLE, - callerName=self.LocationManager.name, - values={ - 'parentLocation': self.parentLocation, - 'settings' : json.dumps(self.settings) - } - ) - - self.id = constructionId - - - def updateSettings(self, settings: dict): - self.settings = {**self.settings, **settings} - - - def toDict(self) -> dict: - return { - 'id' : self.id, - 'parentLocation': self.parentLocation, - 'settings' : self.settings - } + def __init__(self, data: Dict): + self.myDatabase = self.LocationManager.FURNITURE_TABLE + super().__init__(data) diff --git a/core/myHome/model/Location.py b/core/myHome/model/Location.py index 9a52c6b78..ed9fbe7af 100644 --- a/core/myHome/model/Location.py +++ b/core/myHome/model/Location.py @@ -93,7 +93,7 @@ def updateSettings(self, settings: dict): self.settings = {**self.settings, **settings} - def updatesynonyms(self, synonyms): + def updateSynonyms(self, synonyms): self.synonyms = synonyms diff --git a/core/myHome/model/MyHomeObject.py b/core/myHome/model/MyHomeObject.py new file mode 100644 index 000000000..1ae0d8c81 --- /dev/null +++ b/core/myHome/model/MyHomeObject.py @@ -0,0 +1,93 @@ +# Copyright (c) 2021 +# +# This file, Construction.py, is part of Project Alice. +# +# Project Alice is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see +# +# Last modified: 2021.04.13 at 12:56:47 CEST + +import json +from dataclasses import dataclass, field + +from core.base.model.ProjectAliceObject import ProjectAliceObject + + +@dataclass +class MyHomeObject(ProjectAliceObject): + data: dict + + id: int = field(init=False) + parentLocation: int = field(init=False) + settings: dict = field(init=False) + myDatabase: str = field(init=False) + + + def __post_init__(self): + self.id = self.data.get('id', -1) + self.parentLocation = self.data['parentLocation'] + self.settings = json.loads(self.data.get('settings', '{}')) if isinstance(self.data.get('settings', '{}'), str) else self.data.get('settings', dict) + + settings = { + 'x': 0, + 'y': 0, + 'z': 0, + 'w': 10, + 'h': 50, + 'r': 0, + 'c': '', + 'b': '' + } + + self.settings = {**settings, **self.settings} + + if self.id == -1: + self.saveToDB() + + + # noinspection SqlResolve + def saveToDB(self): + if self.id != -1: + self.DatabaseManager.replace( + tableName=self.myDatabase, + query='REPLACE INTO :__table__ (id, parentLocation, settings) VALUES (:id, :parentLocation, :settings)', + callerName=self.LocationManager.name, + values={ + 'id' : self.id, + 'parentLocation': self.parentLocation, + 'settings' : json.dumps(self.settings) + } + ) + else: + constructionId = self.DatabaseManager.insert( + tableName=self.myDatabase, + callerName=self.LocationManager.name, + values={ + 'parentLocation': self.parentLocation, + 'settings' : json.dumps(self.settings) + } + ) + + self.id = constructionId + + + def updateSettings(self, settings: dict): + self.settings = {**self.settings, **settings} + + + def toDict(self) -> dict: + return { + 'id' : self.id, + 'parentLocation': self.parentLocation, + 'settings' : self.settings + } diff --git a/core/util/ThreadManager.py b/core/util/ThreadManager.py index 5e62a17a5..0a96aa42a 100644 --- a/core/util/ThreadManager.py +++ b/core/util/ThreadManager.py @@ -49,7 +49,7 @@ def onStop(self): thread.join(timeout=1) for event in self._events.values(): - if event.isSet(): + if event.is_set(): event.clear() diff --git a/core/util/model/ThreadTimer.py b/core/util/model/ThreadTimer.py index 56483961a..1bc202fce 100644 --- a/core/util/model/ThreadTimer.py +++ b/core/util/model/ThreadTimer.py @@ -19,7 +19,7 @@ from dataclasses import dataclass, field from threading import Timer -from typing import Callable +from typing import Callable, Optional @dataclass @@ -27,4 +27,4 @@ class ThreadTimer(object): callback: Callable args: list = field(default_factory=list) kwargs: dict = field(default_factory=dict) - timer: Timer = None + timer: Optional[Timer] = None diff --git a/core/voice/model/PorcupineWakeword.py b/core/voice/model/PorcupineWakeword.py index 3aaa78864..161ee33d1 100644 --- a/core/voice/model/PorcupineWakeword.py +++ b/core/voice/model/PorcupineWakeword.py @@ -21,7 +21,7 @@ import queue import struct import wave -from typing import Optional +from typing import Generator import pyaudio from paho.mqtt.client import MQTTMessage @@ -34,7 +34,7 @@ try: import pvporcupine except ModuleNotFoundError: - pass # Will autoinstall + pass # Will auto install class PorcupineWakeword(WakewordEngine): @@ -128,7 +128,7 @@ def worker(self): return - def audioStream(self) -> Optional[bytes]: + def audioStream(self) -> Generator: while self._working.is_set(): chunk = self._buffer.get() if not chunk: From 35a904c0db47982652fff8cfa846275e3fa8af97 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Wed, 24 Nov 2021 10:48:51 +0100 Subject: [PATCH 068/129] Well, just make it work again for now --- core/base/SkillManager.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/core/base/SkillManager.py b/core/base/SkillManager.py index 8d7740ec4..237f1531a 100644 --- a/core/base/SkillManager.py +++ b/core/base/SkillManager.py @@ -18,16 +18,16 @@ # Last modified: 2021.08.02 at 06:12:17 CEST -import traceback - import importlib import json -import requests import shutil +import traceback from contextlib import suppress from pathlib import Path from typing import Any, Dict, List, Optional, Union +import requests + from AliceGit import Exceptions as GitErrors from AliceGit.Exceptions import NotGitRepository, PathNotFoundException from AliceGit.Git import Repository @@ -87,7 +87,7 @@ def __init__(self): @property - def supportedIntents(self) -> List[Dict[str, Intent]]: + def supportedIntents(self) -> List[Dict]: """ Returns a list of all supported intents :return: @@ -835,7 +835,7 @@ def startAllSkills(self): self.logInfo(f'Skills started. {len(supportedIntents)} intents supported') - def startSkill(self, skillName: str) -> Dict[str, Intent]: + def startSkill(self, skillName: str) -> Dict: """ Starts a skill :param skillName: From bb8dba2e15b3f0291118abbf5879c05af566a969 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Wed, 24 Nov 2021 11:39:21 +0100 Subject: [PATCH 069/129] Fix skill compatibility check --- core/base/SkillManager.py | 2 +- core/base/SkillStoreManager.py | 5 +++-- system/manager/WebUIManager/de.json | 4 +++- system/manager/WebUIManager/en.json | 7 +++++-- system/manager/WebUIManager/fr.json | 4 +++- system/manager/WebUIManager/it.json | 4 +++- 6 files changed, 18 insertions(+), 8 deletions(-) diff --git a/core/base/SkillManager.py b/core/base/SkillManager.py index 237f1531a..83e36c3d5 100644 --- a/core/base/SkillManager.py +++ b/core/base/SkillManager.py @@ -714,7 +714,7 @@ def checkSkillConditions(self, installer: dict = None, checkOnly=False) -> Union else: notCompliantRules.append({conditionName: conditionValue}) - return True if checkOnly else notCompliantRules + return True if not checkOnly else notCompliantRules def updateSkills(self, skills: Union[str, List[str]], withSkillRestart: bool = True): diff --git a/core/base/SkillStoreManager.py b/core/base/SkillStoreManager.py index 3401bd721..de7eb05d5 100644 --- a/core/base/SkillStoreManager.py +++ b/core/base/SkillStoreManager.py @@ -18,10 +18,11 @@ # Last modified: 2021.04.13 at 12:56:46 CEST import difflib -import requests from random import shuffle from typing import Optional, Tuple +import requests + from core.ProjectAliceExceptions import GithubNotFound from core.base.model.Manager import Manager from core.base.model.Version import Version @@ -84,7 +85,7 @@ def checkConditions(self): offendingConditions = self.SkillManager.checkSkillConditions(installer=skillData, checkOnly=True) skillData['offendingConditions'] = offendingConditions - skillData['compatible'] = False if offendingConditions else True + skillData['compatible'] = False if len(offendingConditions) > 0 else True def prepareSamplesData(self, data: dict): diff --git a/system/manager/WebUIManager/de.json b/system/manager/WebUIManager/de.json index 08c34a02e..788706d04 100644 --- a/system/manager/WebUIManager/de.json +++ b/system/manager/WebUIManager/de.json @@ -72,7 +72,9 @@ "addDevice" : "Neues Gerät hinzufügen", "removeDevice" : "Gerät entfernen", "linkDevice" : "Gerät mit Lokationen verknüpfen", - "unlinkDevice" : "Gerätelink entfernen" + "unlinkDevice" : "Gerätelink entfernen", + "compatible" : "Compatible", + "notCompatible" : "Not compatible" }, "notifications": { "errors" : { diff --git a/system/manager/WebUIManager/en.json b/system/manager/WebUIManager/en.json index abdf94476..21128fcd9 100644 --- a/system/manager/WebUIManager/en.json +++ b/system/manager/WebUIManager/en.json @@ -80,7 +80,9 @@ "addDevice" : "Add new device", "removeDevice" : "Remove devices", "linkDevice" : "Link devices with multiple locations", - "unlinkDevice" : "Unlink devices" + "unlinkDevice" : "Unlink devices", + "compatible" : "Compatible", + "notCompatible" : "Not compatible" }, "notifications": { "errors" : { @@ -95,7 +97,8 @@ "skillDeleteFailed": "Failed deleting the skill", "noLocationEmptyName": "The location name cannot be empty", "widgetSavingFailed": "Failed saving widget", - "newTileFailed": "Failed uploading new tile" + "newTileFailed": "Failed uploading new tile", + "skillIncompatible" : "Unfortunately this skill is not compatible with your current Alice. If you still want to install it, please turn on dev mode." }, "successes": { "deviceUnlinked": "Device successfully unlinked", diff --git a/system/manager/WebUIManager/fr.json b/system/manager/WebUIManager/fr.json index 5a68c2e06..2a517f183 100644 --- a/system/manager/WebUIManager/fr.json +++ b/system/manager/WebUIManager/fr.json @@ -67,7 +67,9 @@ "addDevice" : "Ajouter nouvel appareil", "removeDevice" : "Supprimer appareil", "linkDevice" : "Lier un appareil à plusieures endroits", - "unlinkDevice" : "Délier appareil" + "unlinkDevice" : "Délier appareil", + "compatible" : "Compatible", + "notCompatible" : "Pas compatible" }, "notifications": { "errors" : { diff --git a/system/manager/WebUIManager/it.json b/system/manager/WebUIManager/it.json index f31cf83b7..a7eb7d4ab 100644 --- a/system/manager/WebUIManager/it.json +++ b/system/manager/WebUIManager/it.json @@ -67,7 +67,9 @@ "addDevice" : "Aggiungi nuovi dispositivo", "removeDevice" : "Rimuovi dispositivo", "linkDevice" : "Collega dispositivi con posizioni multiple", - "unlinkDevice" : "Sblocca dispositivi" + "unlinkDevice" : "Sblocca dispositivi", + "compatible" : "Compatible", + "notCompatible" : "Not compatible" }, "notifications": { "errors" : { From 4ca75eb4ada5e313840779b385f94c1f45e01b5b Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Wed, 24 Nov 2021 19:10:58 +0100 Subject: [PATCH 070/129] Allow skill download in dev mode if not compliant --- core/base/SkillManager.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/core/base/SkillManager.py b/core/base/SkillManager.py index 83e36c3d5..bbf5e30f1 100644 --- a/core/base/SkillManager.py +++ b/core/base/SkillManager.py @@ -309,8 +309,11 @@ def installSkills(self, skills: Union[str, List[str]]): try: repository = self.getSkillRepository(skillName=skillName) except: - repositories = self.downloadSkills(skills=skillName) - repository = repositories.get(skillName, None) + try: + repositories = self.downloadSkills(skills=skillName) + repository = repositories.get(skillName, None) + except SkillNotConditionCompliant: + continue if not repository: raise Exception(f'Failed downloading skill **{skillName}** for some unknown reason') @@ -390,7 +393,7 @@ def getSkillRepository(self, skillName: str, directory: str = None) -> Optional[ def downloadSkills(self, skills: Union[str, List[str]]) -> Optional[Dict]: """ - Clones skills. Existance of the skill on line is checked + Clones skills. Existence of the skill on line is checked :param skills: :return: Dict: a dict of created repositories """ @@ -413,7 +416,9 @@ def downloadSkills(self, skills: Union[str, List[str]]) -> Optional[Dict]: requests.get(f'https://skills.projectalice.ch/{skillName}') installFile = response.json() - self.checkSkillConditions(installer=installFile) + + if not self.ConfigManager.getAliceConfigByName('devMode'): + self.checkSkillConditions(installer=installFile) source = self.getGitRemoteSourceUrl(skillName=skillName, doAuth=False) From f0eb1addc9afa41366773e227ffa42e1b67c3884 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 Nov 2021 00:07:08 +0000 Subject: [PATCH 071/129] pip prod(deps): update scipy requirement from ~=1.7.2 to ~=1.7.3 Updates the requirements on [scipy](https://github.com/scipy/scipy) to permit the latest version. - [Release notes](https://github.com/scipy/scipy/releases) - [Commits](https://github.com/scipy/scipy/compare/v1.7.2...v1.7.3) --- updated-dependencies: - dependency-name: scipy dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- requirements_test.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 8f9aaefc3..db279a025 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,7 +21,7 @@ jsmin~=3.0.0 psutil~=5.8.0 pyserial~=3.5 pyyaml~=6.0 -scipy~=1.7.2 +scipy~=1.7.3 webrtcvad~=2.0.10 Werkzeug~=2.0.2 Jinja2~=3.0.3 diff --git a/requirements_test.txt b/requirements_test.txt index 64605bb81..6e891cc55 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -24,7 +24,7 @@ jsmin~=3.0.0 psutil~=5.8.0 pyserial~=3.5 pyyaml~=6.0 -scipy~=1.7.2 +scipy~=1.7.3 webrtcvad~=2.0.10 Werkzeug~=2.0.2 Jinja2~=3.0.3 From 7954559f701f6c372a10e6e0eaeff810dc132c1a Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Thu, 25 Nov 2021 11:08:30 +0100 Subject: [PATCH 072/129] A skill dependency is not an offending condition --- core/base/SkillManager.py | 9 ++++----- core/base/SkillStoreManager.py | 10 +++++++++- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/core/base/SkillManager.py b/core/base/SkillManager.py index bbf5e30f1..bca72785f 100644 --- a/core/base/SkillManager.py +++ b/core/base/SkillManager.py @@ -405,7 +405,6 @@ def downloadSkills(self, skills: Union[str, List[str]]) -> Optional[Dict]: for skillName in skills: try: tag = self.SkillStoreManager.getSkillUpdateTag(skillName=skillName) - response = requests.get(f'{constants.GITHUB_RAW_URL}/skill_{skillName}/{tag}/{skillName}.install') if response.status_code != 200: raise GithubNotFound @@ -639,7 +638,7 @@ def isSkillActive(self, skillName: str) -> bool: return False - def checkSkillConditions(self, installer: dict = None, checkOnly=False) -> Union[bool, Dict[str, str]]: + def checkSkillConditions(self, installer: dict = None, checkOnly=False) -> Union[bool, List[Dict[str, str]]]: """ Checks if the given skill is compliant to it's conditions :param installer: @@ -821,7 +820,7 @@ def isIntentInUse(self, intent: Intent, filtered: list) -> bool: def startAllSkills(self): """ - Starts all the discoverd skills + Starts all the discovered skills :return: """ supportedIntents = list() @@ -1085,7 +1084,7 @@ def checkForSkillUpdates(self, skillToCheck: str = None) -> List[str]: def getSkillInstance(self, skillName: str, silent: bool = False) -> Optional[AliceSkill]: """ - Retuns a skill instance, if available + Returns a skill instance, if available :param skillName: :param silent: :return: @@ -1179,7 +1178,7 @@ def reloadSkill(self, skillName: str): def allScenarioNodes(self) -> Dict[str, tuple]: """ - Retunrs list of Node-Red nodes added by skills + Returns list of Node-Red nodes added by skills :return: """ ret = dict() diff --git a/core/base/SkillStoreManager.py b/core/base/SkillStoreManager.py index de7eb05d5..41d599fc9 100644 --- a/core/base/SkillStoreManager.py +++ b/core/base/SkillStoreManager.py @@ -84,6 +84,11 @@ def checkConditions(self): skillData['installed'] = skillName in self.SkillManager.allSkills.keys() offendingConditions = self.SkillManager.checkSkillConditions(installer=skillData, checkOnly=True) + + for offender in offendingConditions.copy(): + if offender.get('skill', False): + offendingConditions.remove(offender) + skillData['offendingConditions'] = offendingConditions skillData['compatible'] = False if len(offendingConditions) > 0 else True @@ -132,7 +137,10 @@ def _getSkillUpdateVersion(self, skillName: str) -> Optional[Tuple[Version, str] skillUpdateVersion = (repoVersion, f'{str(repoVersion)}_{str(aliceMinVersion)}') if not skillUpdateVersion[0].isVersionNumber: - raise GithubNotFound + if self.ConfigManager.getAliceConfigByName('devMode'): + return Version.fromString('master'), 'master' + else: + raise GithubNotFound return skillUpdateVersion From 8aa91e8e4130f3476606dfcfe49207ee93657264 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Thu, 25 Nov 2021 14:49:55 +0100 Subject: [PATCH 073/129] Not used anymore --- system/skillInstallTickets/.gitkeep | 0 system/skillInstallTickets/AliceCore.install | 28 ------------------- .../ContextSensitive.install | 24 ---------------- .../DateDayTimeYear.install | 27 ------------------ system/skillInstallTickets/RedQueen.install | 24 ---------------- system/skillInstallTickets/Telemetry.install | 27 ------------------ 6 files changed, 130 deletions(-) delete mode 100644 system/skillInstallTickets/.gitkeep delete mode 100644 system/skillInstallTickets/AliceCore.install delete mode 100644 system/skillInstallTickets/ContextSensitive.install delete mode 100644 system/skillInstallTickets/DateDayTimeYear.install delete mode 100644 system/skillInstallTickets/RedQueen.install delete mode 100644 system/skillInstallTickets/Telemetry.install diff --git a/system/skillInstallTickets/.gitkeep b/system/skillInstallTickets/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/system/skillInstallTickets/AliceCore.install b/system/skillInstallTickets/AliceCore.install deleted file mode 100644 index 12b2faded..000000000 --- a/system/skillInstallTickets/AliceCore.install +++ /dev/null @@ -1,28 +0,0 @@ -{ - "name": "AliceCore", - "version": "1.5.11", - "icon": "fab fa-creative-commons-by", - "category": "alice", - "author": "ProjectAlice", - "maintainers": [ - "Psycho", - "Jierka", - "maxbachmann", - "philipp2310", - "ChrisB85", - "lazza" - ], - "desc": "AliceCore is the official skill that handles all core intents.", - "aliceMinVersion": "1.0.0-b5", - "systemRequirements": [], - "pipRequirements": [], - "conditions": { - "lang": [ - "en", - "fr", - "de", - "it", - "pl" - ] - } -} diff --git a/system/skillInstallTickets/ContextSensitive.install b/system/skillInstallTickets/ContextSensitive.install deleted file mode 100644 index 671dd2ab0..000000000 --- a/system/skillInstallTickets/ContextSensitive.install +++ /dev/null @@ -1,24 +0,0 @@ -{ - "name": "ContextSensitive", - "version": "1.0.31", - "icon": "fas fa-brain", - "category": "alice", - "author": "ProjectAlice", - "maintainers": [ - "Psycho", - "maxbachmann" - ], - "desc": "ContextSensitive is the official context sensitive skill. It handles intents like 'Delete this', 'Change that', 'Please repeat", - "aliceMinVersion": "1.0.0-b5", - "systemRequirements": [], - "pipRequirements": [], - "conditions": { - "lang": [ - "en", - "fr", - "de", - "it", - "pl" - ] - } -} diff --git a/system/skillInstallTickets/DateDayTimeYear.install b/system/skillInstallTickets/DateDayTimeYear.install deleted file mode 100644 index 79f004789..000000000 --- a/system/skillInstallTickets/DateDayTimeYear.install +++ /dev/null @@ -1,27 +0,0 @@ -{ - "name": "DateDayTimeYear", - "speakableName": "date time functions", - "version": "1.1.0", - "icon": "far fa-clock", - "category": "assistance", - "author": "Psychokiller1888", - "maintainers": [ - "maxbachmann", - "SkyHyperV" - ], - "desc": "Ask for the time, date, day and year", - "aliceMinVersion": "1.0.0-b5", - "systemRequirements": [], - "pipRequirements": [ - "Babel" - ], - "conditions": { - "lang": [ - "en", - "fr", - "de", - "it", - "pl" - ] - } -} diff --git a/system/skillInstallTickets/RedQueen.install b/system/skillInstallTickets/RedQueen.install deleted file mode 100644 index 16a818ab6..000000000 --- a/system/skillInstallTickets/RedQueen.install +++ /dev/null @@ -1,24 +0,0 @@ -{ - "name": "RedQueen", - "version": "1.0.37", - "icon": "fas fa-biohazard", - "category": "alice", - "author": "ProjectAlice", - "maintainers": [ - "Psycho", - "maxbachmann" - ], - "desc": "Red Queen is the official Project Alice personality skill", - "aliceMinVersion": "1.0.0-b5", - "systemRequirements": [], - "pipRequirements": [], - "conditions": { - "lang": [ - "en", - "fr", - "de", - "it", - "pl" - ] - } -} diff --git a/system/skillInstallTickets/Telemetry.install b/system/skillInstallTickets/Telemetry.install deleted file mode 100644 index abdf88617..000000000 --- a/system/skillInstallTickets/Telemetry.install +++ /dev/null @@ -1,27 +0,0 @@ -{ - "name": "Telemetry", - "version": "1.3.2", - "icon": "fas fa-database", - "category": "alice", - "author": "ProjectAlice", - "maintainers": [ - "Psychokiller1888", - "maxbachmann", - "lazza" - ], - "desc": "Access your stored telemetry data", - "aliceMinVersion": "1.0.0-b5", - "pipRequirements": [], - "systemRequirements": [], - "conditions": { - "lang": [ - "en", - "de", - "it", - "pl" - ], - "activeManager": [ - "TelemetryManager" - ] - } -} From 5c3bc50ec8062c47cad777c937c07e71f600ed81 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Thu, 25 Nov 2021 14:55:03 +0100 Subject: [PATCH 074/129] We need the skill list --- core/base/SkillManager.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/core/base/SkillManager.py b/core/base/SkillManager.py index bca72785f..1a3531383 100644 --- a/core/base/SkillManager.py +++ b/core/base/SkillManager.py @@ -18,16 +18,16 @@ # Last modified: 2021.08.02 at 06:12:17 CEST +import traceback + import importlib import json +import requests import shutil -import traceback from contextlib import suppress from pathlib import Path from typing import Any, Dict, List, Optional, Union -import requests - from AliceGit import Exceptions as GitErrors from AliceGit.Exceptions import NotGitRepository, PathNotFoundException from AliceGit.Git import Repository @@ -140,6 +140,15 @@ def allSkills(self) -> Dict[str, Union[AliceSkill, FailedAliceSkill]]: return {**self._activeSkills, **self._deactivatedSkills, **self._failedSkills} + @property + def skillList(self) -> List: + """ + Returns all skills present in the skill directory. These might not be inited, might have failed etc etc + :return: + """ + return self._skillList + + @property def allWorkingSkills(self) -> Dict[str, AliceSkill]: """ From d464e0ccaf9a346088b4ad37d960fc50f5f34a90 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Sat, 27 Nov 2021 06:01:49 +0100 Subject: [PATCH 075/129] Try except for downloads --- core/webApi/model/SkillsApi.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/core/webApi/model/SkillsApi.py b/core/webApi/model/SkillsApi.py index f44bb0feb..eaa1ac702 100644 --- a/core/webApi/model/SkillsApi.py +++ b/core/webApi/model/SkillsApi.py @@ -20,10 +20,9 @@ import json from contextlib import suppress -from pathlib import Path - from flask import Response, jsonify, request from flask_classful import route +from pathlib import Path from AliceGit.Exceptions import AlreadyGitRepository, GithubRepoNotFound, GithubUserNotFound, NotGitRepository from AliceGit.Git import Repository @@ -108,8 +107,11 @@ def installSkills(self) -> Response: status = dict() for skill in skills: - self.SkillManager.installSkills(skills=skill) - status[skill] = 'ok' + try: + self.SkillManager.installSkills(skills=skill) + status[skill] = 'ok' + except: + status[skill] = 'nok' return jsonify(success=True, status=status) except Exception as e: From d929879fd0e252c14adbb900df5c4fac44eb982e Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Sat, 27 Nov 2021 06:11:11 +0100 Subject: [PATCH 076/129] API install skill should also start it --- core/commons/constants.py | 2 +- core/webApi/model/SkillsApi.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/core/commons/constants.py b/core/commons/constants.py index 22ad40863..cf4322752 100644 --- a/core/commons/constants.py +++ b/core/commons/constants.py @@ -95,7 +95,7 @@ TOPIC_SKILL_STOPPED = 'projectalice/skills/stopped' TOPIC_SKILL_DELETED = 'projectalice/skills/deleted' TOPIC_SKILL_INSTALLED = 'projectalice/skills/installed' -TOPIC_SKILL_INSTALL_FAILED = 'projectalice/skills/installFalied' +TOPIC_SKILL_INSTALL_FAILED = 'projectalice/skills/installFailed' TOPIC_SKILL_INSTRUCTIONS = 'projectalice/skills/instructions' TOPIC_SKILL_UPDATE_CORE_CONFIG_WARNING = 'projectalice/skills/coreConfigUpdateWarning' TOPIC_SKILL_UPDATED = 'projectalice/skills/updated' diff --git a/core/webApi/model/SkillsApi.py b/core/webApi/model/SkillsApi.py index eaa1ac702..c9b102b72 100644 --- a/core/webApi/model/SkillsApi.py +++ b/core/webApi/model/SkillsApi.py @@ -113,6 +113,24 @@ def installSkills(self) -> Response: except: status[skill] = 'nok' + for skillName, status in status.copy().items(): + if status != 'ok': + continue + try: + self.SkillManager.initSkills(onlyInit=skillName) + status[skillName] = 'ok' + except: + status[skillName] = 'nok' + + for skillName, status in status.copy().items(): + if status != 'ok': + continue + try: + self.SkillManager.startSkill(skillName=skillName) + status[skillName] = 'ok' + except: + status[skillName] = 'nok' + return jsonify(success=True, status=status) except Exception as e: self.logWarning(f'Failed installing skill: {e}', printStack=True) From f1e21c657d5f4c12793b0892b754bea81d308a85 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Sat, 27 Nov 2021 15:26:19 +0100 Subject: [PATCH 077/129] Completely fix skill install --- core/base/SkillManager.py | 6 +++- core/webApi/model/SkillsApi.py | 50 ++++++++++------------------------ core/webui/public | 2 +- 3 files changed, 20 insertions(+), 38 deletions(-) diff --git a/core/base/SkillManager.py b/core/base/SkillManager.py index 1a3531383..b4d6d4e52 100644 --- a/core/base/SkillManager.py +++ b/core/base/SkillManager.py @@ -303,10 +303,11 @@ def removeSkillFromDB(self, skillName: str): ) - def installSkills(self, skills: Union[str, List[str]]): + def installSkills(self, skills: Union[str, List[str]], startSkill: bool = False): """ Installs the given skills :param skills: Either a list of skill names to install or a single skill name + :param startSkill: If the skill should be immediately started :return: """ self._busyInstalling.set() @@ -357,6 +358,9 @@ def installSkills(self, skills: Union[str, List[str]]): if installFile.get('rebootAfterInstall', False): self.Commons.runRootSystemCommand('sudo shutdown -r now'.split()) break + elif startSkill: + self.initSkills(onlyInit=skillName) + self.startSkill(skillName=skillName) except SkillNotConditionCompliant: self.broadcast( method=constants.EVENT_SKILL_INSTALL_FAILED, diff --git a/core/webApi/model/SkillsApi.py b/core/webApi/model/SkillsApi.py index c9b102b72..1f51a19e0 100644 --- a/core/webApi/model/SkillsApi.py +++ b/core/webApi/model/SkillsApi.py @@ -108,34 +108,29 @@ def installSkills(self) -> Response: status = dict() for skill in skills: try: - self.SkillManager.installSkills(skills=skill) + self.SkillManager.installSkills(skills=skill, startSkill=True) status[skill] = 'ok' except: status[skill] = 'nok' - for skillName, status in status.copy().items(): - if status != 'ok': - continue - try: - self.SkillManager.initSkills(onlyInit=skillName) - status[skillName] = 'ok' - except: - status[skillName] = 'nok' + self.AssistantManager.checkAssistant() + return jsonify(success=True, status=status) + except Exception as e: + self.logWarning(f'Failed installing skill: {e}', printStack=True) + return jsonify(success=False, message=str(e)) - for skillName, status in status.copy().items(): - if status != 'ok': - continue - try: - self.SkillManager.startSkill(skillName=skillName) - status[skillName] = 'ok' - except: - status[skillName] = 'nok' - return jsonify(success=True, status=status) + @ApiAuthenticated + def put(self, skillName: str) -> Response: + try: + self.SkillManager.installSkills(skills=skillName, startSkill=True) + self.AssistantManager.checkAssistant() except Exception as e: self.logWarning(f'Failed installing skill: {e}', printStack=True) return jsonify(success=False, message=str(e)) + return jsonify(success=True) + @route('//', methods=['PATCH']) @ApiAuthenticated @@ -164,7 +159,7 @@ def get(self, skillName: str) -> Response: if isinstance(skill, dict): raise Exception('Skill not found') - return jsonify(success=True, skill=skill) + return jsonify(success=True, skill=skill.toDict()) except Exception as e: self.logWarning(f'Failed fetching skill: {e}', printStack=True) return jsonify(success=False, message=str(e)) @@ -238,23 +233,6 @@ def reload(self, skillName: str) -> Response: return jsonify(success=False, message=str(e)) - @ApiAuthenticated - def put(self, skillName: str) -> Response: - if not self.SkillStoreManager.skillExists(skillName): - return self.skillNotFound() - elif self.SkillManager.getSkillInstance(skillName, True) is not None: - return jsonify(success=False, reason='skill already installed') - - try: - self.SkillManager.downloadSkills(skills=skillName) - self.SkillManager.startSkill(skillName=skillName) - except Exception as e: - self.logWarning(f'Failed installing skill: {e}', printStack=True) - return jsonify(success=False, message=str(e)) - - return jsonify(success=True) - - @route('//checkUpdate/') @ApiAuthenticated def checkUpdate(self, skillName: str) -> Response: diff --git a/core/webui/public b/core/webui/public index 3e2ddd251..85cc3667a 160000 --- a/core/webui/public +++ b/core/webui/public @@ -1 +1 @@ -Subproject commit 3e2ddd25186d4e6d61ad3551290453f12e569ccb +Subproject commit 85cc3667a6409b3155b37ec82b524bc838dc4181 From 55361407aef2aac0caa6291cb36cec3a48cb6678 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Sat, 27 Nov 2021 16:09:27 +0100 Subject: [PATCH 078/129] Fix skill state toggle, added condition checks, init, etc --- core/base/SkillManager.py | 40 +++++++++++++++++++---------- core/webApi/model/SkillsApi.py | 6 ++--- system/manager/WebUIManager/en.json | 3 ++- 3 files changed, 32 insertions(+), 17 deletions(-) diff --git a/core/base/SkillManager.py b/core/base/SkillManager.py index b4d6d4e52..7c3f502c6 100644 --- a/core/base/SkillManager.py +++ b/core/base/SkillManager.py @@ -579,6 +579,7 @@ def initSkills(self, onlyInit: str = '', reload: bool = False): except SkillNotConditionCompliant as e: if self.notCompliantSkill(skillName=skillName, exception=e): self._failedSkills[skillName] = FailedAliceSkill(installFile) + self.changeSkillStateInDB(skillName=skillName, newState=False) continue else: return @@ -589,6 +590,7 @@ def initSkills(self, onlyInit: str = '', reload: bool = False): return else: self._failedSkills[skillName] = FailedAliceSkill(installFile) + self.changeSkillStateInDB(skillName=skillName, newState=False) continue @@ -604,7 +606,7 @@ def getSkillInstallFilePath(self, skillName: str) -> Path: # noinspection PyTypeChecker def instantiateSkill(self, skillName: str, skillResource: str = '', reload: bool = False) -> Optional[AliceSkill]: """ - Instantiates the given skill at the gien path + Instantiates the given skill at the given path :param skillName: :param skillResource: :param reload: @@ -642,6 +644,8 @@ def isSkillActive(self, skillName: str) -> bool: """ if skillName in self._activeSkills: return self._activeSkills[skillName].active + elif skillName in self._failedSkills or skillName in self._deactivatedSkills: + return False elif skillName in self._skillList: # noinspection SqlResolve row = self.databaseFetch(tableName=self.DBTAB_SKILLS, query='SELECT active FROM :__table__ WHERE skillName = :skillName LIMIT 1', values={'skillName': skillName}) @@ -860,16 +864,19 @@ def startSkill(self, skillName: str) -> Dict: """ if skillName in self._activeSkills: skillInstance = self._activeSkills[skillName] - elif skillName in self._deactivatedSkills: - self._deactivatedSkills.pop(skillName, None) - skillInstance = self.instantiateSkill(skillName=skillName) - if skillInstance: - self.activeSkills[skillName] = skillInstance + elif skillName in self._deactivatedSkills or skillName in self._failedSkills: + if skillName in self._failedSkills: + skill = self._failedSkills.pop(skillName, None) else: + skill = self._deactivatedSkills.pop(skillName, None) + + try: + skillInstance = self.instantiateSkill(skillName=skillName) + self.checkSkillConditions(installer=json.loads(self.getSkillInstallFilePath(skillName=skillName).read_text())) + except: + self._failedSkills[skillName] = FailedAliceSkill(json.loads(skill.getResource(f'{skillName}.install').read_text())) return dict() - elif skillName in self._failedSkills: - self._failedSkills.pop(skillName, None) - skillInstance = self.instantiateSkill(skillName=skillName) + if skillInstance: self.activeSkills[skillName] = skillInstance else: @@ -911,7 +918,7 @@ def startSkill(self, skillName: str) -> Dict: return skillInstance.supportedIntents - def deactivateSkill(self, skillName: str, persistent: bool = False): + def deactivateSkill(self, skillName: str, persistent: bool = False) -> bool: """ Deactivates a skill and broadcasts it :param skillName: @@ -934,11 +941,14 @@ def deactivateSkill(self, skillName: str, persistent: bool = False): self.logInfo(f'Deactivated skill "{skillName}" with persistence') else: self.logInfo(f'Deactivated skill "{skillName}" without persistence') + + return True else: self.logWarning(f'Skill "{skillName} is not active') + return False - def activateSkill(self, skillName: str, persistent: bool = False): + def activateSkill(self, skillName: str, persistent: bool = False) -> bool: """ Activates a skill and broadcasts it :param skillName: @@ -947,11 +957,14 @@ def activateSkill(self, skillName: str, persistent: bool = False): """ if skillName not in self._deactivatedSkills and skillName not in self._failedSkills: self.logWarning(f'Skill "{skillName} is not deactivated or failed') - return + return False try: self.startSkill(skillName) + if skillName not in self._activeSkills: + return False + if persistent: self.changeSkillStateInDB(skillName=skillName, newState=True) self.logInfo(f'Activated skill "{skillName}" with persistence') @@ -964,9 +977,10 @@ def activateSkill(self, skillName: str, persistent: bool = False): propagateToSkills=True, skill=self.activeSkills[skillName] ) + return True except: self.logError(f'Failed activating skill "{skillName}"') - return + return False def toggleSkillState(self, skillName: str, persistent: bool = False): diff --git a/core/webApi/model/SkillsApi.py b/core/webApi/model/SkillsApi.py index 1f51a19e0..9748d10d0 100644 --- a/core/webApi/model/SkillsApi.py +++ b/core/webApi/model/SkillsApi.py @@ -176,11 +176,11 @@ def toggleActiveState(self, skillName: str) -> Response: if skillName in self.SkillManager.neededSkills: return jsonify(success=False, message='Required skill cannot be deactivated!') - self.SkillManager.deactivateSkill(skillName=skillName, persistent=True) + result = self.SkillManager.deactivateSkill(skillName=skillName, persistent=True) else: - self.SkillManager.activateSkill(skillName=skillName, persistent=True) + result = self.SkillManager.activateSkill(skillName=skillName, persistent=True) - return jsonify(success=True) + return jsonify(success=result) except Exception as e: self.logWarning(f'Failed toggling skill: {e}', printStack=True) return jsonify(success=False, message=str(e)) diff --git a/system/manager/WebUIManager/en.json b/system/manager/WebUIManager/en.json index 21128fcd9..d8260b5fd 100644 --- a/system/manager/WebUIManager/en.json +++ b/system/manager/WebUIManager/en.json @@ -98,7 +98,8 @@ "noLocationEmptyName": "The location name cannot be empty", "widgetSavingFailed": "Failed saving widget", "newTileFailed": "Failed uploading new tile", - "skillIncompatible" : "Unfortunately this skill is not compatible with your current Alice. If you still want to install it, please turn on dev mode." + "skillIncompatible": "Unfortunately this skill is not compatible with your current Alice. If you still want to install it, please turn on dev mode.", + "skillNotActivated": "Skill activation failed" }, "successes": { "deviceUnlinked": "Device successfully unlinked", From 3d7d4abda06bf266290f3586436773032753d2f7 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Sat, 27 Nov 2021 16:36:42 +0100 Subject: [PATCH 079/129] Why capture ability to play a sound --- core/base/SuperManager.py | 2 +- core/server/MqttManager.py | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/core/base/SuperManager.py b/core/base/SuperManager.py index cb7d4bb75..539d65b42 100644 --- a/core/base/SuperManager.py +++ b/core/base/SuperManager.py @@ -175,7 +175,7 @@ def onBooted(self): except Exception as e: Logger().logError(f'Error while sending onBooted to manager **{manager.name}**: {e}') - deviceList = self.deviceManager.getDevicesWithAbilities([DeviceAbility.IS_SATELITTE, DeviceAbility.IS_CORE]) + deviceList = self.deviceManager.getDevicesWithAbilities([DeviceAbility.IS_SATELITTE, DeviceAbility.IS_CORE], connectedOnly=False) self.mqttManager.playSound(soundFilename='boot', deviceUid=deviceList) diff --git a/core/server/MqttManager.py b/core/server/MqttManager.py index a06137bfa..1192f7f9c 100644 --- a/core/server/MqttManager.py +++ b/core/server/MqttManager.py @@ -17,17 +17,17 @@ # # Last modified: 2021.07.28 at 16:07:59 CEST -import traceback - import json -import paho.mqtt.client as mqtt -import paho.mqtt.publish as publish import random import re +import traceback import uuid from pathlib import Path from typing import List, Union +import paho.mqtt.client as mqtt +import paho.mqtt.publish as publish + from core.base.model.Intent import Intent from core.base.model.Manager import Manager from core.commons import constants @@ -870,9 +870,8 @@ def playSound(self, soundFilename: str, location: Path = None, sessionId: str = location = Path(self.Commons.rootDir()) / location if deviceUid == constants.ALL or isinstance(deviceUid, list): - if not isinstance(deviceUid, list): - deviceList = [device.uid for device in self.DeviceManager.getDevicesWithAbilities(abilities=[DeviceAbility.PLAY_SOUND, DeviceAbility.CAPTURE_SOUND])] + deviceList = [device.uid for device in self.DeviceManager.getDevicesWithAbilities(abilities=[DeviceAbility.PLAY_SOUND])] else: deviceList = [uid if isinstance(uid, str) else uid.uid for uid in deviceUid] From eac626657b2e0ef5f526ac0f6c4319f65ce09aff Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Sat, 27 Nov 2021 16:44:57 +0100 Subject: [PATCH 080/129] Fix skill state toggle event propagation --- core/base/SkillManager.py | 12 +++---- core/base/model/ProjectAliceObject.py | 50 ++++++++++----------------- core/device/DeviceManager.py | 6 ++-- core/server/MqttManager.py | 8 ++--- core/webui/WidgetManager.py | 2 +- 5 files changed, 33 insertions(+), 45 deletions(-) diff --git a/core/base/SkillManager.py b/core/base/SkillManager.py index 7c3f502c6..41b8b2547 100644 --- a/core/base/SkillManager.py +++ b/core/base/SkillManager.py @@ -18,16 +18,16 @@ # Last modified: 2021.08.02 at 06:12:17 CEST -import traceback - import importlib import json -import requests import shutil +import traceback from contextlib import suppress from pathlib import Path from typing import Any, Dict, List, Optional, Union +import requests + from AliceGit import Exceptions as GitErrors from AliceGit.Exceptions import NotGitRepository, PathNotFoundException from AliceGit.Git import Repository @@ -933,7 +933,7 @@ def deactivateSkill(self, skillName: str, persistent: bool = False) -> bool: method=constants.EVENT_SKILL_DEACTIVATED, exceptions=[constants.DUMMY], propagateToSkills=True, - skill=skillInstance + skill=skillInstance.name ) if persistent: @@ -975,7 +975,7 @@ def activateSkill(self, skillName: str, persistent: bool = False) -> bool: method=constants.EVENT_SKILL_ACTIVATED, exceptions=[constants.DUMMY], propagateToSkills=True, - skill=self.activeSkills[skillName] + skill=skillName ) return True except: @@ -985,7 +985,7 @@ def activateSkill(self, skillName: str, persistent: bool = False) -> bool: def toggleSkillState(self, skillName: str, persistent: bool = False): """ - Activate the given skill if deactivated or deactivates the given skill if actived + Activate the given skill if deactivated or deactivates the given skill if activated :param skillName: :param persistent: :return: diff --git a/core/base/model/ProjectAliceObject.py b/core/base/model/ProjectAliceObject.py index 1cdf58370..da2714ad3 100644 --- a/core/base/model/ProjectAliceObject.py +++ b/core/base/model/ProjectAliceObject.py @@ -19,14 +19,14 @@ from __future__ import annotations -from copy import copy - import json import re -from importlib_metadata import PackageNotFoundError, version as packageVersion +from copy import copy from pathlib import Path from typing import TYPE_CHECKING, Union +from importlib_metadata import PackageNotFoundError, version as packageVersion + import core.base.SuperManager as SM from core.base.model.Version import Version from core.commons import constants @@ -318,6 +318,22 @@ def onSkillUpdating(self, skill: str): pass # Super object function is overridden only if needed + def onSkillStarted(self, skill: str): + pass # Super object function is overridden only if needed + + + def onSkillStopped(self, skill: str): + pass # Super object function is overridden only if needed + + + def onSkillActivated(self, skill: str): + pass # Super object function is overridden only if needed + + + def onSkillDeactivated(self, skill: str): + pass # Super object function is overridden only if needed + + def onInternetConnected(self): pass # Super object function is overridden only if needed @@ -663,34 +679,6 @@ def onDeviceStatus(self, session): pass # Super object function is overridden only if needed - def onSkillStarted(self, skill): - """ - param skill: AliceSkill instance - """ - pass # Super object function is overridden only if needed - - - def onSkillStopped(self, skill): - """ - :param skill: AliceSkill instance - """ - pass # Super object function is overridden only if needed - - - def onSkillActivated(self, skill): - """ - :param skill: AliceSkill instance - """ - pass # Super object function is overridden only if needed - - - def onSkillDeactivated(self, skill): - """ - :param skill: AliceSkill instance - """ - pass # Super object function is overridden only if needed - - @property def ProjectAlice(self) -> ProjectAlice: # NOSONAR return SM.SuperManager.getInstance().projectAlice diff --git a/core/device/DeviceManager.py b/core/device/DeviceManager.py index fcfa5e956..ffe120a27 100644 --- a/core/device/DeviceManager.py +++ b/core/device/DeviceManager.py @@ -159,11 +159,11 @@ def onSkillDeleted(self, skill: str): ) - def onSkillDeactivated(self, skill): - self.removeDeviceTypesForSkill(skillName=skill.name) + def onSkillDeactivated(self, skill: str): + self.removeDeviceTypesForSkill(skillName=skill) tmp = self._devices.copy() for deviceUid, device in tmp.items(): - if device.skillName == skill.name: + if device.skillName == skill: self._devices.pop(deviceUid, None) diff --git a/core/server/MqttManager.py b/core/server/MqttManager.py index 1192f7f9c..21bcd7dc7 100644 --- a/core/server/MqttManager.py +++ b/core/server/MqttManager.py @@ -999,7 +999,7 @@ def onSkillInstallFailed(self, skill: str): ) - def onSkillDeactivated(self, skill): + def onSkillDeactivated(self, skill: str): self.mqttBroadcast( topic=constants.TOPIC_SKILL_DEACTIVATED, payload={ @@ -1008,7 +1008,7 @@ def onSkillDeactivated(self, skill): ) - def onSkillActivated(self, skill): + def onSkillActivated(self, skill: str): self.mqttBroadcast( topic=constants.TOPIC_SKILL_ACTIVATED, payload={ @@ -1017,7 +1017,7 @@ def onSkillActivated(self, skill): ) - def onSkillStopped(self, skill): + def onSkillStopped(self, skill: str): self.mqttBroadcast( topic=constants.TOPIC_SKILL_STOPPED, payload={ @@ -1026,7 +1026,7 @@ def onSkillStopped(self, skill): ) - def onSkillStarted(self, skill): + def onSkillStarted(self, skill: str): self.mqttBroadcast( topic=constants.TOPIC_SKILL_STARTED, payload={ diff --git a/core/webui/WidgetManager.py b/core/webui/WidgetManager.py index 6ffdd10f7..f81b09ba3 100644 --- a/core/webui/WidgetManager.py +++ b/core/webui/WidgetManager.py @@ -280,7 +280,7 @@ def onSkillDeleted(self, skill: str): def onSkillDeactivated(self, skill): tmp = self._widgets.copy() for wid, widget in tmp.items(): - if widget.skill == skill.name: + if widget.skill == skill: self._widgets.pop(wid, None) From ddd7bd0e0bb799659749241f5ed27e55bc3dcaea Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Sat, 27 Nov 2021 16:56:46 +0100 Subject: [PATCH 081/129] starting and stopping display usage needs to start/stop the thread core side --- configTemplate.json | 3 ++- core/webui/WebUIManager.py | 36 ++++++++++++++++++++++++++++++++---- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/configTemplate.json b/configTemplate.json index a80319753..a92f7f34d 100644 --- a/configTemplate.json +++ b/configTemplate.json @@ -17,7 +17,8 @@ "dataType" : "boolean", "isSensitive" : false, "description" : "Displays the current system usage on the interface", - "category" : "system" + "category" : "system", + "onUpdate" : "WebUIManager.toggleSystemUsage" }, "enableSSL" : { "defaultValue": false, diff --git a/core/webui/WebUIManager.py b/core/webui/WebUIManager.py index 86728011b..064be2711 100644 --- a/core/webui/WebUIManager.py +++ b/core/webui/WebUIManager.py @@ -18,6 +18,7 @@ # Last modified: 2021.04.13 at 12:56:49 CEST from pathlib import Path +from threading import Event import psutil as psutil @@ -27,6 +28,11 @@ class WebUIManager(Manager): + + def __init__(self): + self._systemUsageThreadFlag = Event() + super().__init__() + def onStart(self): super().onStart() @@ -37,16 +43,38 @@ def onStart(self): try: self.startWebserver() if self.ConfigManager.getAliceConfigByName('displaySystemUsage'): - self.ThreadManager.newThread( - name='DisplayResourceUsage', - target=self.publishResourceUsage - ) + self.startSystemUsagePublisher() except Exception as e: self.logWarning(f'WebUI starting failed: {e}') self.onStop() + def toggleSystemUsage(self): + if self.ConfigManager.getAliceConfigByName('displaySystemUsage'): + self._systemUsageThreadFlag.clear() + self.ThreadManager.terminateThread(name='DisplayResourceUsage') + self.ThreadManager.doLater(interval=3, func=self.startSystemUsagePublisher) + else: + self.ThreadManager.terminateThread(name='DisplayResourceUsage') + self._systemUsageThreadFlag.clear() + + + def startSystemUsagePublisher(self): + """ + Starts publishing system resource usage over mqtt + :return: + """ + self._systemUsageThreadFlag.set() + self.ThreadManager.newThread( + name='DisplayResourceUsage', + target=self.publishResourceUsage + ) + + def publishResourceUsage(self): + if not self._systemUsageThreadFlag.is_set(): + return + self.MqttManager.publish( topic=constants.TOPIC_RESOURCE_USAGE, payload={ From 97884ae0974d4c4e3e5fcff15f99230f157e7fcd Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Sat, 27 Nov 2021 17:05:29 +0100 Subject: [PATCH 082/129] Fix Alice stop when fatal error --- core/base/SuperManager.py | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/core/base/SuperManager.py b/core/base/SuperManager.py index 539d65b42..2dde5ae87 100644 --- a/core/base/SuperManager.py +++ b/core/base/SuperManager.py @@ -19,8 +19,6 @@ from __future__ import annotations -import traceback - from core.commons import constants from core.device.model.DeviceAbility import DeviceAbility from core.util.model.Logger import Logger @@ -259,23 +257,26 @@ def initManagers(self): def onStop(self): managerName = constants.UNKNOWN_MANAGER - try: - mqttManager = self._managers.pop('MqttManager') # Mqtt goes down as last + mqttManager = self._managers.pop('MqttManager', None) # Mqtt goes down as last - skillManager = self._managers.pop('SkillManager') # Skill manager goes down first, to tell the skills - skillManager.onStop() + skillManager = self._managers.pop('SkillManager', None) # Skill manager goes down first, to tell the skills + if skillManager: + try: + skillManager.onStop() + except Exception as e: + Logger().logError(f'Error stopping SkillManager: {e}') - for managerName, manager in self._managers.items(): + for managerName, manager in self._managers.items(): + try: manager.onStop() - - managerName = mqttManager.name - mqttManager.onStop() - except KeyError as e: - Logger().logWarning(f'Manager **{managerName}** was not running: {e}') - - except Exception as e: - Logger().logError(f'Error while shutting down manager **{managerName}**: {e}') - traceback.print_exc() + except Exception as e: + Logger().logError(f'Error while shutting down manager **{managerName}**: {e}') + + if mqttManager: + try: + mqttManager.onStop() + except Exception as e: + Logger().logError(f'Error stopping MqttManager: {e}') def getManager(self, managerName: str): From 9fc467cf5d3980147056f78acf2a37fbebad304f Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Sat, 27 Nov 2021 17:06:59 +0100 Subject: [PATCH 083/129] Fix arbitrary condition check if boot failed and ASR is none --- core/base/SkillManager.py | 2 +- core/base/SuperManager.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/core/base/SkillManager.py b/core/base/SkillManager.py index 41b8b2547..8765171a4 100644 --- a/core/base/SkillManager.py +++ b/core/base/SkillManager.py @@ -717,7 +717,7 @@ def checkSkillConditions(self, installer: dict = None, checkOnly=False) -> Union notCompliantRules.append({conditionName: conditionValue}) elif conditionName == 'asrArbitraryCapture': - if conditionValue and not self.ASRManager.asr.capableOfArbitraryCapture: + if conditionValue and self.ASRManager.asr and not self.ASRManager.asr.capableOfArbitraryCapture: if not checkOnly: raise SkillNotConditionCompliant(message=notCompliant, skillName=installer['name'], condition=conditionName, conditionValue=conditionValue) else: diff --git a/core/base/SuperManager.py b/core/base/SuperManager.py index 2dde5ae87..3187a3c2d 100644 --- a/core/base/SuperManager.py +++ b/core/base/SuperManager.py @@ -19,7 +19,6 @@ from __future__ import annotations -from core.commons import constants from core.device.model.DeviceAbility import DeviceAbility from core.util.model.Logger import Logger @@ -256,7 +255,6 @@ def initManagers(self): def onStop(self): - managerName = constants.UNKNOWN_MANAGER mqttManager = self._managers.pop('MqttManager', None) # Mqtt goes down as last skillManager = self._managers.pop('SkillManager', None) # Skill manager goes down first, to tell the skills From ef14bd1ecb330ccfabac9ca3bcf4b6eee6d88f22 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Sat, 27 Nov 2021 17:09:40 +0100 Subject: [PATCH 084/129] Don't wait for device manager if we are shutting down --- core/device/DeviceManager.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/device/DeviceManager.py b/core/device/DeviceManager.py index ffe120a27..e597904aa 100644 --- a/core/device/DeviceManager.py +++ b/core/device/DeviceManager.py @@ -237,12 +237,14 @@ def getDevice(self, deviceId: int = None, uid: [str, uuid.UUID] = None) -> Optio raise Exception('Cannot get a device without id or uid') if ret is None and not self.loadingDone: + if self.ProjectAlice.shuttingDown: + return None + self._loopCounter += 1 - self.logInfo(f'waiting for deviceManager ({self._loopCounter})') + self.logInfo(f'Waiting for {self.name} ({self._loopCounter})') if self._loopCounter > 20: self.logWarning(f'Impossible to find device instance for id/uid {deviceId}/{uid}, skipping') self._loopCounter = 0 - return None time.sleep(1) From 61caeda5d1b34155a1d03ea8e7f08fe9f3c3fa61 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Sat, 27 Nov 2021 17:32:26 +0100 Subject: [PATCH 085/129] revert google asr --- core/asr/model/GoogleAsr.py | 36 +++++++++++++++--------------------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/core/asr/model/GoogleAsr.py b/core/asr/model/GoogleAsr.py index f6df6e6e5..563a475d3 100644 --- a/core/asr/model/GoogleAsr.py +++ b/core/asr/model/GoogleAsr.py @@ -18,7 +18,6 @@ # Last modified: 2021.04.13 at 12:56:45 CEST import os -from contextlib import suppress from pathlib import Path from threading import Event from time import time @@ -31,11 +30,11 @@ from core.util.Stopwatch import Stopwatch -with suppress(ModuleNotFoundError): - # noinspection PyPackageRequirements - from google.cloud.speech import SpeechClient - # noinspection PyPackageRequirements - from google.cloud.speech_v1 import RecognitionConfig, StreamingRecognitionConfig, StreamingRecognizeRequest +try: + # noinspection PyUnresolvedReferences,PyPackageRequirements + from google.cloud.speech import SpeechClient, enums, types +except: + pass # Auto installed # noinspection PyAbstractClass @@ -44,7 +43,7 @@ class GoogleAsr(Asr): DEPENDENCIES = { 'system': [], 'pip' : { - 'google-cloud-speech==2.11.1' + 'google-cloud-speech==1.3.1' } } @@ -56,7 +55,7 @@ def __init__(self): self._isOnlineASR = True self._client: Optional[SpeechClient] = None - self._streamingConfig: Optional[RecognitionConfig] = None + self._streamingConfig: Optional[types.StreamingRecognitionConfig] = None if self._credentialsFile.exists() and not self.ConfigManager.getAliceConfigByName('googleASRCredentials'): self.ConfigManager.updateAliceConfiguration(key='googleASRCredentials', value=self._credentialsFile.read_text(), doPreAndPostProcessing=False) @@ -73,18 +72,13 @@ def onStart(self): self._client = SpeechClient() # noinspection PyUnresolvedReferences - config = RecognitionConfig({ - 'encoding': RecognitionConfig.AudioEncoding.LINEAR16, - 'sample_rate_hertz': self.AudioServer.SAMPLERATE, - 'language_code': self.LanguageManager.getLanguageAndCountryCode(), - 'max_alternatives': 1, - 'model': 'command_and_search' - }) + config = types.RecognitionConfig( + encoding=enums.RecognitionConfig.AudioEncoding.LINEAR16, + sample_rate_hertz=self.AudioServer.SAMPLERATE, + language_code=self.LanguageManager.getLanguageAndCountryCode() + ) - self._streamingConfig = StreamingRecognitionConfig({ - 'config': config, - 'interim_results': True - }) + self._streamingConfig = types.StreamingRecognitionConfig(config=config, interim_results=True) def decodeStream(self, session: DialogSession) -> Optional[ASRResult]: @@ -99,8 +93,8 @@ def decodeStream(self, session: DialogSession) -> Optional[ASRResult]: audioStream = stream.audioStream() # noinspection PyUnresolvedReferences try: - requests = (StreamingRecognizeRequest(audio_content=content) for content in audioStream) - responses = self._client.streaming_recognize(config=self._streamingConfig, requests=requests) + requests = (types.StreamingRecognizeRequest(audio_content=content) for content in audioStream) + responses = self._client.streaming_recognize(self._streamingConfig, requests) result = self._checkResponses(session, responses) except Exception as e: self._internetLostFlag.clear() From 625bcccc6f29b94f8f87d8ac5722bb94659be38d Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Sun, 28 Nov 2021 05:48:21 +0100 Subject: [PATCH 086/129] Fix shut down error when broadcast --- core/base/model/ProjectAliceObject.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/base/model/ProjectAliceObject.py b/core/base/model/ProjectAliceObject.py index da2714ad3..f29c97f73 100644 --- a/core/base/model/ProjectAliceObject.py +++ b/core/base/model/ProjectAliceObject.py @@ -122,7 +122,7 @@ def broadcast(self, method: str, exceptions: list = None, manager=None, propagat self.logWarning(f'Failed to broadcast event **{method}** to **DialogManager**: {e}') deadManagers = list() - for name, man in SM.SuperManager.getInstance().managers.items(): + for name, man in SM.SuperManager.getInstance().managers.copy().items(): if not man: deadManagers.append(name) continue From bbacf02b788cb0d01977473af05b8cb540c0edfa Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Tue, 30 Nov 2021 16:21:09 +0100 Subject: [PATCH 087/129] adding missing languages --- core/asr/model/CoquiAsr.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/core/asr/model/CoquiAsr.py b/core/asr/model/CoquiAsr.py index 457f8b8b4..eb089ee95 100644 --- a/core/asr/model/CoquiAsr.py +++ b/core/asr/model/CoquiAsr.py @@ -101,13 +101,29 @@ def downloadLanguage(self) -> bool: # NOSONAR self.Commons.downloadFile('https://github.com/coqui-ai/STT-models/releases/download/german/AASHISHAG/v0.9.0/model.tflite', str(self.tFlite)) self.Commons.downloadFile('https://github.com/philipp2310/Coqui-models/releases/download/de_v093/lm.scorer', str(self._langPath / 'lm.scorer')) return True - if self.LanguageManager.activeLanguage == 'en': - self.Commons.downloadFile('https://github.com/coqui-ai/STT/releases/download/v0.9.3/coqui-stt-0.9.3-models.tflite', str(self.tFlite)) - self.Commons.downloadFile('https://github.com/coqui-ai/STT/releases/download/v0.9.3/coqui-stt-0.9.3-models.scorer', str(self._langPath / 'lm.scorer')) + elif self.LanguageManager.activeLanguage == 'en': + self.Commons.downloadFile('https://github.com/coqui-ai/STT-models/releases/download/english%2Fcoqui%2Fv1.0.0-large-vocab/model.tflite', str(self.tFlite)) + self.Commons.downloadFile('https://github.com/coqui-ai/STT-models/releases/download/english%2Fcoqui%2Fv1.0.0-large-vocab/large_vocabulary.scorer', str(self._langPath / 'lm.scorer')) return True - - url = 'error' # TODO adjust path in respect to language - self.logError(f'WIP! Only de/en supported for now - Please install language manually into PA/trained/asr/Coqui//!') + elif self.LanguageManager.activeLanguage == 'fr': + self.Commons.downloadFile('https://github.com/coqui-ai/STT-models/releases/download/french/commonvoice-fr/v0.6/model.tflite', str(self.tFlite)) + self.Commons.downloadFile('https://github.com/coqui-ai/STT-models/releases/download/french/commonvoice-fr/v0.6/fr-cvfr-2-prune-kenlm.scorer', str(self._langPath / 'lm.scorer')) + return True + elif self.LanguageManager.activeLanguage == 'it': + self.Commons.downloadFile('https://github.com/coqui-ai/STT-models/releases/download/italian/mozillaitalia/2020.8.7/model.tflite', str(self.tFlite)) + self.Commons.downloadFile('https://github.com/coqui-ai/STT-models/releases/download/italian/mozillaitalia/2020.8.7/it-mzit-1-prune-kenlm.scorer', str(self._langPath / 'lm.scorer')) + return True + elif self.LanguageManager.activeLanguage == 'pl': + self.Commons.downloadFile('https://github.com/coqui-ai/STT-models/releases/download/polish/jaco-assistant/v0.0.1/model.tflite', str(self.tFlite)) + self.Commons.downloadFile('https://github.com/coqui-ai/STT-models/releases/download/polish/jaco-assistant/v0.0.1/kenlm_pl.scorer', str(self._langPath / 'lm.scorer')) + return True + elif self.LanguageManager.activeLanguage == 'pt': + self.Commons.downloadFile('https://github.com/coqui-ai/STT-models/releases/download/portuguese/itml/v0.1.0/model.tflite', str(self.tFlite)) + self.Commons.downloadFile('https://github.com/coqui-ai/STT-models/releases/download/portuguese/itml/v0.1.0/pt-itml-0-prune-kenlm.scorer', str(self._langPath / 'lm.scorer')) + return True + else: + self.logError(f'WIP! Only de/en supported for now - Please install language manually into PA/trained/asr/Coqui//!') + return False downloadPath = (self._langPath / url.rsplit('/')[-1]) try: From 6849421fce8f9c90fab761c829534ac97aa47ed4 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Tue, 30 Nov 2021 16:51:32 +0100 Subject: [PATCH 088/129] When tts fallback, save it to the config file --- core/voice/model/Tts.py | 3 ++ .../LanguageManager/notifications.json | 34 +++++++++++++++++-- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/core/voice/model/Tts.py b/core/voice/model/Tts.py index de615ef8c..ec9c5a81b 100644 --- a/core/voice/model/Tts.py +++ b/core/voice/model/Tts.py @@ -90,16 +90,19 @@ def onStart(self): if self._lang not in self._supportedLangAndVoices: self.logWarning(f'Language **{self._lang}** not found, falling back to **en-US**') self._lang = 'en-US' + self.ConfigManager.updateAliceConfiguration(key='ttsLanguage', value=self._lang) if self._type not in self._supportedLangAndVoices[self._lang]: ttsType = self._type self._type = next(iter(self._supportedLangAndVoices[self._lang])) + self.ConfigManager.updateAliceConfiguration(key='ttsType', value={self._type}) self.logWarning(f'Type **{ttsType}** not found for the language, falling back to **{self._type}**') if self._voice not in self._supportedLangAndVoices[self._lang][self._type]: voice = self._voice self._voice = next(iter(self._supportedLangAndVoices[self._lang][self._type])) self._neuralVoice = self._supportedLangAndVoices[self._lang][self._type][self._voice]['neural'] + self.ConfigManager.updateAliceConfiguration(key='ttsVoice', value=self._voice) self.logWarning(f'Voice **{voice}** not found for the language and type, falling back to **{self._voice}**') else: self._neuralVoice = self._supportedLangAndVoices[self._lang][self._type][self._voice]['neural'] diff --git a/system/manager/LanguageManager/notifications.json b/system/manager/LanguageManager/notifications.json index 663d5a6ff..874ccca0b 100644 --- a/system/manager/LanguageManager/notifications.json +++ b/system/manager/LanguageManager/notifications.json @@ -89,8 +89,38 @@ "fr": "Mise à jours configuration système" }, "body": { - "en": "The skill\"{}\" wants to modify the setting \"{}\" to the new value \"{}\"", - "fr": "The skill\"{}\" veut modifier le parametre \"{}\" à la valeure \"{}\"" + "en": "The skill \"{}\" wants to modify the setting \"{}\" to the new value \"{}\"", + "fr": "The skill \"{}\" veux modifier le paramètre \"{}\" à la valeure \"{}\"" + } + }, + "startedDownload": { + "title": { + "en": "Download", + "fr": "Téléchargement" + }, + "body": { + "en": "Downloading \"{}\"", + "fr": "Téléchargement de \"{}\"" + } + }, + "doneDownload": { + "title": { + "en": "Download", + "fr": "Téléchargement" + }, + "body": { + "en": "Finished downloading \"{}\"", + "fr": "Téléchargement de \"{}\" terminé" + } + }, + "failedDownload": { + "title": { + "en": "Download", + "fr": "Téléchargement" + }, + "body": { + "en": "Failed downloading \"{}\"", + "fr": "Téléchargement de \"{}\" échoué" } } } From bf5c7805bf0266220e6c98aa009d38fb1b8ca61c Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Tue, 30 Nov 2021 16:54:39 +0100 Subject: [PATCH 089/129] Createe a notification when Alice downloads something --- core/commons/CommonsManager.py | 19 ++++++++++++------- core/webui/public | 2 +- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/core/commons/CommonsManager.py b/core/commons/CommonsManager.py index d4c348e8b..25aac4771 100644 --- a/core/commons/CommonsManager.py +++ b/core/commons/CommonsManager.py @@ -17,15 +17,10 @@ # # Last modified: 2021.04.13 at 12:56:46 CEST -from collections import defaultdict -from ctypes import * - import hashlib import inspect -import jinja2 import json import random -import requests import socket import sqlite3 import string @@ -33,20 +28,26 @@ import tempfile import time import uuid +from collections import defaultdict from contextlib import contextmanager, suppress +from ctypes import * from datetime import datetime -from googletrans import Translator -from paho.mqtt.client import MQTTMessage from pathlib import Path from typing import Any, Union from uuid import UUID +import jinja2 +import requests +from googletrans import Translator +from paho.mqtt.client import MQTTMessage + import core.base.SuperManager as SuperManager import core.commons.model.Slot as slotModel from core.base.model.Manager import Manager from core.commons import constants from core.commons.model.PartOfDay import PartOfDay from core.dialog.model.DialogSession import DialogSession +from core.webui.model.UINotificationType import UINotificationType class CommonsManager(Manager): @@ -360,16 +361,20 @@ def downloadFile(self, url: str, dest: str) -> bool: if not self.Commons.rootDir() in dest: dest = f'{self.Commons.rootDir()}/{dest}' + key = f'download_{time.time()}' try: + self.WebUINotificationManager.newNotification(typ=UINotificationType.INFO, notification='startedDownload', replaceBody=[Path(dest).stem], key=key) with requests.get(url, stream=True) as r: r.raise_for_status() with Path(dest).open('wb') as fp: for chunk in r.iter_content(chunk_size=8192): if chunk: fp.write(chunk) + self.WebUINotificationManager.newNotification(typ=UINotificationType.INFO, notification='doneDownload', replaceBody=[Path(dest).stem], key=key) return True except Exception as e: self.logWarning(f'Failed downloading file: {e}') + self.WebUINotificationManager.newNotification(typ=UINotificationType.ALERT, notification='failedDownload', replaceBody=[Path(dest).stem], key=key) return False diff --git a/core/webui/public b/core/webui/public index 85cc3667a..862ce6cab 160000 --- a/core/webui/public +++ b/core/webui/public @@ -1 +1 @@ -Subproject commit 85cc3667a6409b3155b37ec82b524bc838dc4181 +Subproject commit 862ce6cab5aef577405fae5f1acb985ac3b5cfc8 From 3ec96b47d1ca8df162750b5a1037d577b70c409f Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Tue, 30 Nov 2021 17:16:40 +0100 Subject: [PATCH 090/129] Do not allow to install an alreeady installed skill.... --- core/base/SkillManager.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/base/SkillManager.py b/core/base/SkillManager.py index 8765171a4..f98189a15 100644 --- a/core/base/SkillManager.py +++ b/core/base/SkillManager.py @@ -315,6 +315,9 @@ def installSkills(self, skills: Union[str, List[str]], startSkill: bool = False) skills = [skills] for skillName in skills: + if skillName in self._skillList: + continue + try: try: repository = self.getSkillRepository(skillName=skillName) From 384ed4125acc8060ec02abb4d5a924e9206a2f61 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Tue, 30 Nov 2021 17:20:54 +0100 Subject: [PATCH 091/129] Don't suggest to download an already available skill Alice.... --- core/base/SkillManager.py | 1 + core/base/SkillStoreManager.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/core/base/SkillManager.py b/core/base/SkillManager.py index f98189a15..d59a7f90e 100644 --- a/core/base/SkillManager.py +++ b/core/base/SkillManager.py @@ -316,6 +316,7 @@ def installSkills(self, skills: Union[str, List[str]], startSkill: bool = False) for skillName in skills: if skillName in self._skillList: + self.logDebug(f'Skill **{skillName}** already installed, skipping') continue try: diff --git a/core/base/SkillStoreManager.py b/core/base/SkillStoreManager.py index 41d599fc9..9e7f19e3c 100644 --- a/core/base/SkillStoreManager.py +++ b/core/base/SkillStoreManager.py @@ -182,6 +182,9 @@ def findSkillSuggestion(self, session: DialogSession, string: str = None) -> set ret = set() for suggestedSkillName in suggestions: + if suggestedSkillName in self.SkillManager.allSkills.keys(): + continue + speakableName = self._skillStoreData.get(suggestedSkillName, dict()).get('speakableName', '') if not speakableName: From dad19a9196675704832010f68128bc4735b9c655 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Sun, 5 Dec 2021 16:40:56 +0100 Subject: [PATCH 092/129] Reintegrate HLC into Alice, tested working but initializer not tested --- core/Initializer.py | 28 ++++++++++++++-------------- core/ProjectAlice.py | 2 +- core/webui/public | 2 +- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/core/Initializer.py b/core/Initializer.py index 4b97054af..98a893523 100644 --- a/core/Initializer.py +++ b/core/Initializer.py @@ -564,14 +564,15 @@ def initProjectAlice(self) -> bool: # NOSONAR confs['disableCapture'] = False hlcServiceFilePath = Path('/etc/systemd/system/hermesledcontrol.service') - hlcConfigFilePath = Path(f'/home/{getpass.getuser()}/hermesLedControl/configuration.yml') + hlcDistributedServiceFilePath = Path(f'/home/{getpass.getuser()}/HermesLedControl/hermesledcontrol.service') + hlcConfigFilePath = Path(f'/home/{getpass.getuser()}/HermesLedControl/configuration.yml') hlcConfig = dict() if initConfs['useHLC']: - hlcDir = Path('/home', getpass.getuser(), 'hermesLedControl') + hlcDir = Path('/home', getpass.getuser(), 'HermesLedControl') if not hlcDir.exists(): - subprocess.run(['git', 'clone', 'https://github.com/project-alice-assistant/hermesLedControl.git', str(Path('/home', getpass.getuser(), 'hermesLedControl'))]) + subprocess.run(['git', 'clone', 'https://github.com/project-alice-assistant/hermesLedControl.git', str(Path('/home', getpass.getuser(), 'HermesLedControl'))]) else: subprocess.run(['git', '-C', hlcDir, 'stash']) subprocess.run(['git', '-C', hlcDir, 'pull']) @@ -579,20 +580,11 @@ def initProjectAlice(self) -> bool: # NOSONAR if hlcServiceFilePath.exists(): subprocess.run(['sudo', 'systemctl', 'stop', 'hermesledcontrol']) + subprocess.run(['sudo', 'systemctl', 'disable', 'hermesledcontrol']) subprocess.run(['sudo', 'rm', hlcServiceFilePath]) subprocess.run(['python3.7', '-m', 'venv', f'/home/{getpass.getuser()}/hermesLedControl/venv']) - - reqs = [ - 'RPi.GPIO', - 'spidev', - 'gpiozero', - 'paho-mqtt', - 'numpy', - 'pyyaml' - ] - for req in reqs: - subprocess.run([f'/home/{getpass.getuser()}/hermesLedControl/venv/bin/pip', 'install', req]) + subprocess.run([f'{str(hlcDir)}/venv/bin/pip', 'install', '-r', f'{str(hlcDir / "requirements.txt")}', '--no-cache-dir']) import yaml @@ -606,6 +598,14 @@ def initProjectAlice(self) -> bool: # NOSONAR hlcConfig['pattern'] = 'projectalice' hlcConfig['enableDoA'] = False + serviceFile = hlcDistributedServiceFilePath.read_text() + serviceFile = serviceFile.replace('%WORKING_DIR%', f'WorkingDirectory=/home/{getpass.getuser()}/HermesLedControl') + serviceFile = serviceFile.replace('%EXECSTART%', f'WorkingDirectory=/home/{getpass.getuser()}/HermesLedControl/venv/bin/python main.py --hermesLedControlConfig=/home/{getpass.getuser()}/.config/hermesLedControl/configuration.yml') + serviceFile = serviceFile.replace('%USER%', f'User={getpass.getuser()}') + hlcDistributedServiceFilePath.write_text(serviceFile) + subprocess.run(['sudo', 'cp', hlcDistributedServiceFilePath, hlcServiceFilePath]) + + useFallbackHLC = False if initConfs['installSound']: if audioHardware in {'respeaker2', 'respeaker4', 'respeaker6MicArray'}: diff --git a/core/ProjectAlice.py b/core/ProjectAlice.py index ba7ecbbec..043f780af 100644 --- a/core/ProjectAlice.py +++ b/core/ProjectAlice.py @@ -68,7 +68,7 @@ def __init__(self, restartHandler: callable): def checkDependencies(self) -> bool: """ - Compares .hash files against requirements.txt and sysrrequirement.txt. Updates dependencies if necessary + Compares .hash files against requirements.txt and sysrequirement.txt. Updates dependencies if necessary :return: boolean False if the check failed, new deps were installed (reboot maybe? :) ) """ HASH_SUFFIX = '.hash' diff --git a/core/webui/public b/core/webui/public index 862ce6cab..58931f08e 160000 --- a/core/webui/public +++ b/core/webui/public @@ -1 +1 @@ -Subproject commit 862ce6cab5aef577405fae5f1acb985ac3b5cfc8 +Subproject commit 58931f08e37c7d61524ba150f96eb02ce9faaabd From 3701ca89406b24e90e98dfbaaebf13373baa32a9 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Wed, 8 Dec 2021 17:53:13 +0100 Subject: [PATCH 093/129] Cleaned Logger, and added Alice auto bug report --- core/base/ConfigManager.py | 2 +- core/base/SuperManager.py | 93 +++++++++++++++++++++-------------- core/util/BugReportManager.py | 91 ++++++++++++++++++++++++++++++++++ core/util/model/Logger.py | 17 +++++-- 4 files changed, 161 insertions(+), 42 deletions(-) create mode 100644 core/util/BugReportManager.py diff --git a/core/base/ConfigManager.py b/core/base/ConfigManager.py index 4cd440448..9265fdfd1 100644 --- a/core/base/ConfigManager.py +++ b/core/base/ConfigManager.py @@ -772,7 +772,7 @@ def aliceTemplateConfigurations(self) -> Dict: @property def githubAuth(self) -> Tuple[str, str]: """ - Returns the users configured username and token for github as a tuple + Returns the users configured username and token for Github as a tuple When one of the values is not supplied None is returned. :return: """ diff --git a/core/base/SuperManager.py b/core/base/SuperManager.py index 3187a3c2d..f3589d778 100644 --- a/core/base/SuperManager.py +++ b/core/base/SuperManager.py @@ -39,46 +39,52 @@ def __init__(self, mainClass): SuperManager._INSTANCE = self self._managers = dict() - self.projectAlice = mainClass - self.commons = None - self.commonsManager = None - self.configManager = None - self.databaseManager = None - self.languageManager = None - self.asrManager = None - self.ttsManager = None - self.threadManager = None - self.mqttManager = None - self.timeManager = None - self.multiIntentManager = None - self.telemetryManager = None - self.skillManager = None - self.widgetManager = None - self.deviceManager = None - self.locationManager = None - self.internetManager = None - self.wakewordRecorder = None - self.userManager = None - self.talkManager = None - self.webUiManager = None - self.apiManager = None - self.nodeRedManager = None - self.skillStoreManager = None - self.nluManager = None - self.dialogTemplateManager = None - self.aliceWatchManager = None - self.audioManager = None - self.dialogManager = None - self.locationManager = None - self.wakewordManager = None - self.assistantManager = None - self.stateManager = None - self.subprocessManager = None + self.projectAlice = mainClass + self.aliceWatchManager = None + self.apiManager = None + self.asrManager = None + self.assistantManager = None + self.audioManager = None + self.bugReportManager = None + self.commons = None + self.commonsManager = None + self.configManager = None + self.databaseManager = None + self.deviceManager = None + self.dialogManager = None + self.dialogTemplateManager = None + self.internetManager = None + self.languageManager = None + self.locationManager = None + self.locationManager = None + self.mqttManager = None + self.multiIntentManager = None + self.nluManager = None + self.nodeRedManager = None + self.skillManager = None + self.skillStoreManager = None + self.stateManager = None + self.subprocessManager = None + self.talkManager = None + self.telemetryManager = None + self.threadManager = None + self.timeManager = None + self.ttsManager = None + self.userManager = None + self.wakewordManager = None + self.wakewordRecorder = None + self.webUiManager = None self.webUINotificationManager = None + self.widgetManager = None + def onStart(self): try: + bugReportManager = self._managers.pop('BugReportManager') + bugReportManager.onStart() + self._managers[bugReportManager.name] = bugReportManager + commons = self._managers.pop('CommonsManager') commons.onStart() @@ -125,7 +131,7 @@ def onStart(self): nodeRedManager = self._managers.pop('NodeRedManager') for manager in self._managers.values(): - if manager: + if manager and manager.name != self.bugReportManager.name: manager.onStart() talkManager.onStart() @@ -156,6 +162,7 @@ def onStart(self): self._managers[nodeRedManager.name] = nodeRedManager self._managers[stateManager.name] = stateManager self._managers[subprocessManager.name] = subprocessManager + self._managers[bugReportManager.name] = bugReportManager except Exception as e: import traceback @@ -215,7 +222,9 @@ def initManagers(self): from core.base.StateManager import StateManager from core.util.SubprocessManager import SubprocessManager from core.webui.WebUINotificationManager import WebUINotificationManager + from core.util.BugReportManager import BugReportManager + self.bugReportManager = BugReportManager() self.commonsManager = CommonsManager() self.commons = self.commonsManager self.stateManager = StateManager() @@ -255,7 +264,8 @@ def initManagers(self): def onStop(self): - mqttManager = self._managers.pop('MqttManager', None) # Mqtt goes down as last + mqttManager = self._managers.pop('MqttManager', None) # Mqtt goes down last with bug reporter + bugReporterManager = self._managers.pop('BugReporterManager', None) # bug reporter goes down as last skillManager = self._managers.pop('SkillManager', None) # Skill manager goes down first, to tell the skills if skillManager: @@ -266,7 +276,8 @@ def onStop(self): for managerName, manager in self._managers.items(): try: - manager.onStop() + if manager.isActive: + manager.onStop() except Exception as e: Logger().logError(f'Error while shutting down manager **{managerName}**: {e}') @@ -276,6 +287,12 @@ def onStop(self): except Exception as e: Logger().logError(f'Error stopping MqttManager: {e}') + if bugReporterManager: + try: + bugReporterManager.onStop() + except Exception as e: + Logger().logError(f'Error stopping BugReporterManager: {e}') + def getManager(self, managerName: str): return self._managers.get(managerName, None) diff --git a/core/util/BugReportManager.py b/core/util/BugReportManager.py new file mode 100644 index 000000000..04c894634 --- /dev/null +++ b/core/util/BugReportManager.py @@ -0,0 +1,91 @@ +# Copyright (c) 2021 +# +# This file, AliceWatchManager.py, is part of Project Alice. +# +# Project Alice is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see +# +# Last modified: 2021.04.24 at 12:56:47 CEST +import json +import os +import traceback +from pathlib import Path + +import requests + +from core.base.model.Manager import Manager +from core.commons import constants + + +class BugReportManager(Manager): + + def __init__(self): + self._flagFile = Path('alice.bugreport') + if self._flagFile.exists(): + self._recording = True + else: + self._recording = False + self._history = list() + self._title = '' + + super().__init__(name='BugReportManager') + + + def onStart(self): + super().onStart() + if self._recording: + self.logInfo('Flag file detected, recording errors for this run') + + + @property + def isRecording(self) -> bool: + return self._recording + + + def addToHistory(self, log: str): + if not self._recording: + return + + if len(self._history) > 2500: + del self._history[0] + + self._history.append(log) + + if not self._title and traceback.format_exc().strip() != 'NoneType: None': + self._title = traceback.format_exc().strip().split('\n').pop() + + + def onStop(self): + super().onStop() + if not self._recording: + return + + if not self._history or not self._title: + self.logInfo('Nothing to report') + elif not self.ConfigManager.githubAuth: + self.logWarning('Cannot report bugs if Github user and token are not set in configs') + else: + title = f'[AUTO BUG REPORT] {self._title}' + body = '\n'.join(self._history) + data = { + 'title': title, + 'body': f'```{body}\n```' + } + + request = requests.post(url=f'{constants.GITHUB_API_URL}/ProjectAlice/issues', data=json.dumps(data), auth=self.ConfigManager.githubAuth) + if request.status_code != 201: + self.logError(f'Something went wrong reporting a bug, status: {request.status_code}, error: {request.json()}') + else: + self.logInfo(f'Created new issue: {request.json()["url"]}') + + os.remove(self._flagFile) diff --git a/core/util/model/Logger.py b/core/util/model/Logger.py index 8b51846d8..8dd04f583 100644 --- a/core/util/model/Logger.py +++ b/core/util/model/Logger.py @@ -25,7 +25,7 @@ class Logger(object): - def __init__(self, prepend: str = None, **kwargs): + def __init__(self, prepend: str = None, **_kwargs): self._prepend = prepend self._logger = logging.getLogger('ProjectAlice') @@ -74,6 +74,8 @@ def doLog(self, function: callable, msg: str, printStack=True, plural: Union[lis if self._prepend: msg = f'{self._prepend} {msg}' + elif not msg.startswith('['): + msg = f'[Project Alice Logger] {msg}' match = re.match(r'^(\[[\w ]+])(.*)$', msg) if match: @@ -82,9 +84,18 @@ def doLog(self, function: callable, msg: str, printStack=True, plural: Union[lis msg = f'{tag}{space}{log}' func = getattr(self._logger, function) - func(msg, exc_info=printStack) + func(msg) if printStack: - traceback.print_exc() + for line in traceback.format_exc().split('\n'): + if not line.strip(): + continue + self.doLog(function=function, msg=f'[Traceback] {line}', printStack=False) + + try: + from core.base.SuperManager import SuperManager + SuperManager.getInstance().bugReportManager.addToHistory(msg) + except: + pass # We really can't do anything here @staticmethod From 77eb4bbf7f834d5a0849d2b740af565d37dbee86 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Thu, 9 Dec 2021 13:59:57 +0100 Subject: [PATCH 094/129] Tell about bug reporting earlier in logs --- core/util/BugReportManager.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/core/util/BugReportManager.py b/core/util/BugReportManager.py index 04c894634..f64989110 100644 --- a/core/util/BugReportManager.py +++ b/core/util/BugReportManager.py @@ -30,22 +30,17 @@ class BugReportManager(Manager): def __init__(self): + super().__init__(name='BugReportManager') + self._flagFile = Path('alice.bugreport') if self._flagFile.exists(): self._recording = True + self.logInfo('Flag file detected, recording errors for this run') else: self._recording = False self._history = list() self._title = '' - super().__init__(name='BugReportManager') - - - def onStart(self): - super().onStart() - if self._recording: - self.logInfo('Flag file detected, recording errors for this run') - @property def isRecording(self) -> bool: From b888871fa55ca4c893783cb4ac0ad1f898403815 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Thu, 9 Dec 2021 14:23:49 +0100 Subject: [PATCH 095/129] Display git commit id in the bug report --- core/util/BugReportManager.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/core/util/BugReportManager.py b/core/util/BugReportManager.py index f64989110..16d3c0aef 100644 --- a/core/util/BugReportManager.py +++ b/core/util/BugReportManager.py @@ -18,6 +18,7 @@ # Last modified: 2021.04.24 at 12:56:47 CEST import json import os +import subprocess import traceback from pathlib import Path @@ -51,8 +52,13 @@ def addToHistory(self, log: str): if not self._recording: return + if len(self._history) <= 0: + self._history.append('Project Alice logs') + version = subprocess.run(f'git rev-parse HEAD', capture_output=True, text=True, shell=True).stdout.strip() + self.logInfo(f'Git commit id: {version}') + if len(self._history) > 2500: - del self._history[0] + del self._history[1] # Don't delete first line, it's the git commit id self._history.append(log) @@ -74,13 +80,13 @@ def onStop(self): body = '\n'.join(self._history) data = { 'title': title, - 'body': f'```{body}\n```' + 'body': f'```\n{body}\n```' } request = requests.post(url=f'{constants.GITHUB_API_URL}/ProjectAlice/issues', data=json.dumps(data), auth=self.ConfigManager.githubAuth) if request.status_code != 201: self.logError(f'Something went wrong reporting a bug, status: {request.status_code}, error: {request.json()}') else: - self.logInfo(f'Created new issue: {request.json()["url"]}') + self.logInfo(f'Created new issue: {request.json()["html_url"]}') os.remove(self._flagFile) From c26f5a8d11b9b7d383614466c73b63ef9b1b2575 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Thu, 9 Dec 2021 14:28:30 +0100 Subject: [PATCH 096/129] Should be restarted by system reboot, but it doesn't seem to always be the case --- core/Initializer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/core/Initializer.py b/core/Initializer.py index 98a893523..b598030c7 100644 --- a/core/Initializer.py +++ b/core/Initializer.py @@ -433,6 +433,7 @@ def initProjectAlice(self) -> bool: # NOSONAR subprocess.run(['sudo', 'systemctl', 'stop', 'mosquitto']) subprocess.run('sudo sed -i -e \'s/persistence true/persistence false/\' /etc/mosquitto/mosquitto.conf'.split()) subprocess.run(['sudo', 'rm', '/var/lib/mosquitto/mosquitto.db']) + subprocess.run(['sudo', 'systemctl', 'start', 'mosquitto']) # Now let's dump some values to their respective places # First those that need some checks and self filling in case unspecified From b7fe97bdb17e8cb278946e0517fc42347636d06e Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Thu, 9 Dec 2021 17:16:26 +0100 Subject: [PATCH 097/129] Few oops --- core/base/SuperManager.py | 8 ++++---- core/util/BugReportManager.py | 10 ++++------ 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/core/base/SuperManager.py b/core/base/SuperManager.py index f3589d778..43418b97b 100644 --- a/core/base/SuperManager.py +++ b/core/base/SuperManager.py @@ -265,7 +265,7 @@ def initManagers(self): def onStop(self): mqttManager = self._managers.pop('MqttManager', None) # Mqtt goes down last with bug reporter - bugReporterManager = self._managers.pop('BugReporterManager', None) # bug reporter goes down as last + bugReportManager = self._managers.pop('BugReportManager', None) # bug reporter goes down as last skillManager = self._managers.pop('SkillManager', None) # Skill manager goes down first, to tell the skills if skillManager: @@ -287,11 +287,11 @@ def onStop(self): except Exception as e: Logger().logError(f'Error stopping MqttManager: {e}') - if bugReporterManager: + if bugReportManager: try: - bugReporterManager.onStop() + bugReportManager.onStop() except Exception as e: - Logger().logError(f'Error stopping BugReporterManager: {e}') + Logger().logError(f'Error stopping BugReportManager: {e}') def getManager(self, managerName: str): diff --git a/core/util/BugReportManager.py b/core/util/BugReportManager.py index 16d3c0aef..90fbdeac7 100644 --- a/core/util/BugReportManager.py +++ b/core/util/BugReportManager.py @@ -37,6 +37,9 @@ def __init__(self): if self._flagFile.exists(): self._recording = True self.logInfo('Flag file detected, recording errors for this run') + version = subprocess.run(f'git rev-parse HEAD', capture_output=True, text=True, shell=True).stdout.strip() + self.logInfo(f'Project Alice logs') + self.logInfo(f'Git commit id: {version}') else: self._recording = False self._history = list() @@ -52,11 +55,6 @@ def addToHistory(self, log: str): if not self._recording: return - if len(self._history) <= 0: - self._history.append('Project Alice logs') - version = subprocess.run(f'git rev-parse HEAD', capture_output=True, text=True, shell=True).stdout.strip() - self.logInfo(f'Git commit id: {version}') - if len(self._history) > 2500: del self._history[1] # Don't delete first line, it's the git commit id @@ -89,4 +87,4 @@ def onStop(self): else: self.logInfo(f'Created new issue: {request.json()["html_url"]}') - os.remove(self._flagFile) + os.remove(self._flagFile) From 449609d4d045c2ef849b6d1a52b2597ed489e558 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Fri, 10 Dec 2021 05:25:59 +0100 Subject: [PATCH 098/129] Stop nginx and disable it in initialyzer --- core/Initializer.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/Initializer.py b/core/Initializer.py index b598030c7..b85c7e1e7 100644 --- a/core/Initializer.py +++ b/core/Initializer.py @@ -435,6 +435,9 @@ def initProjectAlice(self) -> bool: # NOSONAR subprocess.run(['sudo', 'rm', '/var/lib/mosquitto/mosquitto.db']) subprocess.run(['sudo', 'systemctl', 'start', 'mosquitto']) + subprocess.run(['sudo', 'systemctl', 'stop', 'nginx']) + subprocess.run(['sudo', 'systemctl', 'disable', 'nginx']) + # Now let's dump some values to their respective places # First those that need some checks and self filling in case unspecified confs['mqttHost'] = str(initConfs['mqttHost']) or 'localhost' From abfbd9813d15fd45651eed28c6901bb6d479f469 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Sun, 12 Dec 2021 12:44:39 +0100 Subject: [PATCH 099/129] Force debug mod if asking for auto report --- core/base/ConfigManager.py | 7 +++++++ core/base/model/ProjectAliceObject.py | 5 +++++ 2 files changed, 12 insertions(+) diff --git a/core/base/ConfigManager.py b/core/base/ConfigManager.py index 9265fdfd1..74f54e086 100644 --- a/core/base/ConfigManager.py +++ b/core/base/ConfigManager.py @@ -61,6 +61,10 @@ def __init__(self): def onStart(self): super().onStart() + + if self.BugReportManager.isRecording: + self._aliceConfigurations['debug'] = True + for conf in self._vitalConfigs: if conf not in self._aliceConfigurations or self._aliceConfigurations[conf] == '': raise VitalConfigMissing(conf) @@ -178,6 +182,9 @@ def _loadCheckAndUpdateAliceConfigFile(self): else: self._aliceConfigurations = aliceConfigs + if self.BugReportManager.isRecording: + self._aliceConfigurations['debug'] = True + def updateAliceConfigDefinitionValues(self, setting: str, value: Any): if setting not in self._aliceTemplateConfigurations: diff --git a/core/base/model/ProjectAliceObject.py b/core/base/model/ProjectAliceObject.py index f29c97f73..da4665e05 100644 --- a/core/base/model/ProjectAliceObject.py +++ b/core/base/model/ProjectAliceObject.py @@ -56,6 +56,7 @@ from core.util.TelemetryManager import TelemetryManager from core.util.ThreadManager import ThreadManager from core.util.TimeManager import TimeManager + from core.util.BugReportManager import BugReportManager from core.util.SubprocessManager import SubprocessManager from core.voice.LanguageManager import LanguageManager from core.voice.TTSManager import TTSManager @@ -847,3 +848,7 @@ def SubprocessManager(self) -> SubprocessManager: # NOSONAR @property def WebUINotificationManager(self) -> WebUINotificationManager: # NOSONAR return SM.SuperManager.getInstance().webUINotificationManager + + @property + def BugReportManager(self) -> BugReportManager: # NOSONAR + return SM.SuperManager.getInstance().bugReportManager From baa379d027de4733c5b7df4cd61f9c074fb0555c Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Sun, 12 Dec 2021 13:02:53 +0100 Subject: [PATCH 100/129] Will add later --- core/base/ConfigManager.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/core/base/ConfigManager.py b/core/base/ConfigManager.py index 74f54e086..955947e42 100644 --- a/core/base/ConfigManager.py +++ b/core/base/ConfigManager.py @@ -62,9 +62,6 @@ def __init__(self): def onStart(self): super().onStart() - if self.BugReportManager.isRecording: - self._aliceConfigurations['debug'] = True - for conf in self._vitalConfigs: if conf not in self._aliceConfigurations or self._aliceConfigurations[conf] == '': raise VitalConfigMissing(conf) @@ -182,9 +179,6 @@ def _loadCheckAndUpdateAliceConfigFile(self): else: self._aliceConfigurations = aliceConfigs - if self.BugReportManager.isRecording: - self._aliceConfigurations['debug'] = True - def updateAliceConfigDefinitionValues(self, setting: str, value: Any): if setting not in self._aliceTemplateConfigurations: From b247cddd51a39f2c382536e93ab1a082b569a985 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Sun, 12 Dec 2021 13:04:18 +0100 Subject: [PATCH 101/129] Cleanup --- core/base/ConfigManager.py | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/core/base/ConfigManager.py b/core/base/ConfigManager.py index 955947e42..656471a90 100644 --- a/core/base/ConfigManager.py +++ b/core/base/ConfigManager.py @@ -206,8 +206,7 @@ def updateMainDeviceName(self, value: Any): device.updateConfigs(configs={'displayName': value}) - def updateAliceConfiguration(self, key: str, value: Any, dump: bool = True, - doPreAndPostProcessing: bool = True): + def updateAliceConfiguration(self, key: str, value: Any, dump: bool = True, doPreAndPostProcessing: bool = True): """ Updating a core config is sensitive, if the request comes from a skill. First check if the request came from a skill at anytime and if so ask permission @@ -219,7 +218,6 @@ def updateAliceConfiguration(self, key: str, value: Any, dump: bool = True, :return: None """ - # TODO reimplement UI side rootSkills = [name.lower() for name in self.SkillManager.NEEDED_SKILLS] callers = [inspect.getmodulename(frame[1]).lower() for frame in inspect.stack()] if 'aliceskill' in callers: @@ -231,20 +229,6 @@ def updateAliceConfiguration(self, key: str, value: Any, dump: bool = True, self.WebUINotificationManager.newNotification(typ=UINotificationType.ALERT, notification='coreConfigUpdateWarning', replaceBody=[skillName, key, value]) return - # self.ThreadManager.doLater( - # interval=2, - # func=self.MqttManager.publish, - # kwargs={ - # 'topic': constants.TOPIC_SKILL_UPDATE_CORE_CONFIG_WARNING, - # 'payload': { - # 'skill': skillName, - # 'key' : key, - # 'value': value - # } - # } - # ) - # return - if key not in self._aliceConfigurations: self.logWarning(f"Was asked to update **{key}** but key doesn't exist") raise ConfigurationUpdateFailed() From cb3015c2f110726de8000b71bccbed10220bc5f3 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Sun, 12 Dec 2021 13:07:43 +0100 Subject: [PATCH 102/129] Should fix error if something goes bad onStart --- core/base/SuperManager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/base/SuperManager.py b/core/base/SuperManager.py index 43418b97b..11d6afbbf 100644 --- a/core/base/SuperManager.py +++ b/core/base/SuperManager.py @@ -130,7 +130,7 @@ def onStart(self): nluManager = self._managers.pop('NluManager') nodeRedManager = self._managers.pop('NodeRedManager') - for manager in self._managers.values(): + for manager in self._managers.copy().values(): if manager and manager.name != self.bugReportManager.name: manager.onStart() From 56fca4693dcce33aa47c6cfda6f7ca1e809774e5 Mon Sep 17 00:00:00 2001 From: philipp2310 <41761223+philipp2310@users.noreply.github.com> Date: Thu, 16 Dec 2021 02:37:05 +0100 Subject: [PATCH 103/129] hermes path in init --- core/Initializer.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/core/Initializer.py b/core/Initializer.py index b85c7e1e7..5991442ca 100644 --- a/core/Initializer.py +++ b/core/Initializer.py @@ -567,16 +567,16 @@ def initProjectAlice(self) -> bool: # NOSONAR confs['disableSound'] = False confs['disableCapture'] = False + hlcDir = Path('/home', getpass.getuser(), 'HermesLedControl') hlcServiceFilePath = Path('/etc/systemd/system/hermesledcontrol.service') - hlcDistributedServiceFilePath = Path(f'/home/{getpass.getuser()}/HermesLedControl/hermesledcontrol.service') - hlcConfigFilePath = Path(f'/home/{getpass.getuser()}/HermesLedControl/configuration.yml') + hlcDistributedServiceFilePath = hlcDir / 'hermesledcontrol.service' + hlcConfigTemplatePath = hlcDir / 'configuration.yml' hlcConfig = dict() if initConfs['useHLC']: - hlcDir = Path('/home', getpass.getuser(), 'HermesLedControl') - if not hlcDir.exists(): - subprocess.run(['git', 'clone', 'https://github.com/project-alice-assistant/hermesLedControl.git', str(Path('/home', getpass.getuser(), 'HermesLedControl'))]) + #todo: replace with AliceGit maybe? + subprocess.run(['git', 'clone', 'https://github.com/project-alice-assistant/hermesLedControl.git', str(hlcDir)]) else: subprocess.run(['git', '-C', hlcDir, 'stash']) subprocess.run(['git', '-C', hlcDir, 'pull']) @@ -587,13 +587,13 @@ def initProjectAlice(self) -> bool: # NOSONAR subprocess.run(['sudo', 'systemctl', 'disable', 'hermesledcontrol']) subprocess.run(['sudo', 'rm', hlcServiceFilePath]) - subprocess.run(['python3.7', '-m', 'venv', f'/home/{getpass.getuser()}/hermesLedControl/venv']) - subprocess.run([f'{str(hlcDir)}/venv/bin/pip', 'install', '-r', f'{str(hlcDir / "requirements.txt")}', '--no-cache-dir']) + subprocess.run(['python3.7', '-m', 'venv', f'{str(hlcDir)}/venv']) + subprocess.run([f'{str(hlcDir)}/venv/bin/pip', 'install', '-r', f'{str(hlcDir)}/requirements.txt', '--no-cache-dir']) import yaml try: - hlcConfig = yaml.safe_load(hlcConfigFilePath.read_text()) + hlcConfig = yaml.safe_load(hlcConfigTemplatePath.read_text()) except yaml.YAMLError as e: self._logger.logFatal(f'Failed loading HLC configurations: {e}') else: @@ -603,8 +603,8 @@ def initProjectAlice(self) -> bool: # NOSONAR hlcConfig['enableDoA'] = False serviceFile = hlcDistributedServiceFilePath.read_text() - serviceFile = serviceFile.replace('%WORKING_DIR%', f'WorkingDirectory=/home/{getpass.getuser()}/HermesLedControl') - serviceFile = serviceFile.replace('%EXECSTART%', f'WorkingDirectory=/home/{getpass.getuser()}/HermesLedControl/venv/bin/python main.py --hermesLedControlConfig=/home/{getpass.getuser()}/.config/hermesLedControl/configuration.yml') + serviceFile = serviceFile.replace('%WORKING_DIR%', f'{str(hlcDir)}') + serviceFile = serviceFile.replace('%EXECSTART%', f'WorkingDirectory={str(hlcDir)}/venv/bin/python main.py --hermesLedControlConfig=/home/{getpass.getuser()}/.config/HermesLedControl/configuration.yml') serviceFile = serviceFile.replace('%USER%', f'User={getpass.getuser()}') hlcDistributedServiceFilePath.write_text(serviceFile) subprocess.run(['sudo', 'cp', hlcDistributedServiceFilePath, hlcServiceFilePath]) From 66b419cb067b49a87c37d7853ea0a32ce6356044 Mon Sep 17 00:00:00 2001 From: philipp2310 <41761223+philipp2310@users.noreply.github.com> Date: Fri, 17 Dec 2021 02:07:12 +0100 Subject: [PATCH 104/129] no .dumps without .loads required --- core/Initializer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/Initializer.py b/core/Initializer.py index 5991442ca..1eac137fd 100644 --- a/core/Initializer.py +++ b/core/Initializer.py @@ -402,7 +402,7 @@ def initProjectAlice(self) -> bool: # NOSONAR self._logger.logFatal('Unfortunately it won\'t be possible, config sample is not existing') return False - self._confsFile.write_text(json.dumps(self._confsSample.read_text(), indent='\t')) + self._confsFile.write_text(self._confsSample.read_text()) try: confs = json.loads(self._confsFile.read_text()) From 59ff7ad175d65b948fded7d375494d8136b39109 Mon Sep 17 00:00:00 2001 From: philipp2310 <41761223+philipp2310@users.noreply.github.com> Date: Sat, 18 Dec 2021 00:33:38 +0100 Subject: [PATCH 105/129] nasty falsy values --- core/Initializer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/Initializer.py b/core/Initializer.py index 1eac137fd..e18c5f4f8 100644 --- a/core/Initializer.py +++ b/core/Initializer.py @@ -46,10 +46,10 @@ def __init__(self, default: dict): def __getitem__(self, item): try: - item = super().__getitem__(item) - if not item: + value = super().__getitem__(item) + if value is None: raise Exception - return item + return value except: print(f'Missing key **{item}** in provided yaml file.') return '' From 9265e899db1c017a1705af95e9fae162515c4feb Mon Sep 17 00:00:00 2001 From: philipp2310 <41761223+philipp2310@users.noreply.github.com> Date: Sat, 18 Dec 2021 01:37:03 +0100 Subject: [PATCH 106/129] allow leading zeros in admin pin --- core/Initializer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/Initializer.py b/core/Initializer.py index e18c5f4f8..6b80df3d1 100644 --- a/core/Initializer.py +++ b/core/Initializer.py @@ -445,9 +445,9 @@ def initProjectAlice(self) -> bool: # NOSONAR pinCode = initConfs['adminPinCode'] try: - pin = int(pinCode) - if len(str(pin)) != 4: + if len(str(pinCode)) != 4: raise Exception + pin = int(pinCode).zfill(4) except: self._logger.logFatal('Pin code must be 4 digits') From 79842e0219f5f5aca51099caeb7098b4f2954b45 Mon Sep 17 00:00:00 2001 From: philipp2310 <41761223+philipp2310@users.noreply.github.com> Date: Sat, 18 Dec 2021 01:46:00 +0100 Subject: [PATCH 107/129] wrong line for filling zeros --- core/Initializer.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/Initializer.py b/core/Initializer.py index 6b80df3d1..149c2271c 100644 --- a/core/Initializer.py +++ b/core/Initializer.py @@ -447,11 +447,11 @@ def initProjectAlice(self) -> bool: # NOSONAR try: if len(str(pinCode)) != 4: raise Exception - pin = int(pinCode).zfill(4) + pin = int(pinCode) except: self._logger.logFatal('Pin code must be 4 digits') - confs['adminPinCode'] = int(pinCode) + confs['adminPinCode'] = int(pinCode).zfill(4) confs['stayCompletelyOffline'] = bool(initConfs['stayCompletelyOffline'] or False) if confs['stayCompletelyOffline']: @@ -529,6 +529,7 @@ def initProjectAlice(self) -> bool: # NOSONAR try: import pkg_resources + self._logger.logInfo("*** Trying to load SNIPS-NLU.") pkg_resources.require('snips-nlu') subprocess.run(['./venv/bin/snips-nlu', 'download', confs['activeLanguage']]) except: From 5d15c535977c572dee4bbc38f862ef3fbefe3fe7 Mon Sep 17 00:00:00 2001 From: philipp2310 <41761223+philipp2310@users.noreply.github.com> Date: Sat, 18 Dec 2021 02:18:33 +0100 Subject: [PATCH 108/129] just take it as it was entered - was checked before... --- core/Initializer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/Initializer.py b/core/Initializer.py index 149c2271c..459600c53 100644 --- a/core/Initializer.py +++ b/core/Initializer.py @@ -451,7 +451,7 @@ def initProjectAlice(self) -> bool: # NOSONAR except: self._logger.logFatal('Pin code must be 4 digits') - confs['adminPinCode'] = int(pinCode).zfill(4) + confs['adminPinCode'] = pinCode confs['stayCompletelyOffline'] = bool(initConfs['stayCompletelyOffline'] or False) if confs['stayCompletelyOffline']: From 0a28c1f713f6e3b623000c5ec06e4d1a5b16c10d Mon Sep 17 00:00:00 2001 From: philipp2310 <41761223+philipp2310@users.noreply.github.com> Date: Sat, 18 Dec 2021 03:02:08 +0100 Subject: [PATCH 109/129] continue without hlc config --- core/Initializer.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/core/Initializer.py b/core/Initializer.py index 459600c53..f3c9ca772 100644 --- a/core/Initializer.py +++ b/core/Initializer.py @@ -574,6 +574,7 @@ def initProjectAlice(self) -> bool: # NOSONAR hlcConfigTemplatePath = hlcDir / 'configuration.yml' hlcConfig = dict() if initConfs['useHLC']: + self._logger.logInfo("*** Taking care of HLC.") if not hlcDir.exists(): #todo: replace with AliceGit maybe? @@ -596,12 +597,13 @@ def initProjectAlice(self) -> bool: # NOSONAR try: hlcConfig = yaml.safe_load(hlcConfigTemplatePath.read_text()) except yaml.YAMLError as e: - self._logger.logFatal(f'Failed loading HLC configurations: {e}') - else: - hlcConfig['engine'] = 'projectalice' - hlcConfig['pathToConfig'] = f'/home/{getpass.getuser()}/ProjectAlice/config.json' - hlcConfig['pattern'] = 'projectalice' - hlcConfig['enableDoA'] = False + self._logger.logWarning(f'Failed loading HLC configurations - creating new: {e}') + hlcConfig = dict() + + hlcConfig['engine'] = 'projectalice' + hlcConfig['pathToConfig'] = f'/home/{getpass.getuser()}/ProjectAlice/config.json' + hlcConfig['pattern'] = 'projectalice' + hlcConfig['enableDoA'] = False serviceFile = hlcDistributedServiceFilePath.read_text() serviceFile = serviceFile.replace('%WORKING_DIR%', f'{str(hlcDir)}') From 20bb740b15ddcac2756636bc24b25b375a89e532 Mon Sep 17 00:00:00 2001 From: philipp2310 <41761223+philipp2310@users.noreply.github.com> Date: Sat, 18 Dec 2021 03:50:51 +0100 Subject: [PATCH 110/129] changed fallback ASR from deepspeech to coqui --- core/Initializer.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/core/Initializer.py b/core/Initializer.py index f3c9ca772..cde22ed38 100644 --- a/core/Initializer.py +++ b/core/Initializer.py @@ -36,6 +36,7 @@ ASOUND = '/etc/asound.conf' TEMP = Path('/tmp/service') ALLOWED_LANGUAGES = {'en', 'de', 'fr', 'it', 'pt', 'pl'} +FALLBACK_ASR = 'coqui' class InitDict(dict): @@ -458,7 +459,7 @@ def initProjectAlice(self) -> bool: # NOSONAR confs['keepASROffline'] = True confs['keepTTSOffline'] = True confs['skillAutoUpdate'] = False - confs['asr'] = 'deepspeech' + confs['asr'] = FALLBACK_ASR confs['tts'] = 'pico' confs['awsRegion'] = '' confs['awsAccessKey'] = '' @@ -472,14 +473,14 @@ def initProjectAlice(self) -> bool: # NOSONAR confs['awsAccessKey'] = initConfs['awsAccessKey'] confs['awsSecretKey'] = initConfs['awsSecretKey'] - confs['asr'] = initConfs['asr'] if initConfs['asr'] in {'pocketsphinx', 'google', 'deepspeech', 'snips', 'coqui'} else 'deepspeech' + confs['asr'] = initConfs['asr'] if initConfs['asr'] in {'pocketsphinx', 'google', 'deepspeech', 'snips', 'coqui'} else FALLBACK_ASR if confs['asr'] == 'google' and not initConfs['googleServiceFile']: - self._logger.logInfo('You cannot use Google Asr without a google service file, falling back to Deepspeech') - confs['asr'] = 'deepspeech' + self._logger.logInfo(f'You cannot use Google Asr without a google service file, falling back to {FALLBACK_ASR}') + confs['asr'] = FALLBACK_ASR if confs['asr'] == 'snips' and confs['activeLanguage'] != 'en': - self._logger.logInfo('You can only use Snips Asr for english, falling back to Deepspeech') - confs['asr'] = 'deepspeech' + self._logger.logInfo(f'You can only use Snips Asr for english, falling back to {FALLBACK_ASR}') + confs['asr'] = FALLBACK_ASR if initConfs['googleServiceFile']: googleCreds = Path(self._rootDir, 'credentials/googlecredentials.json') From 828d98820b46c475d587ede526f3af74e0b95b9e Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Sat, 18 Dec 2021 08:20:11 +0100 Subject: [PATCH 111/129] Actually, don't delete anything --- core/util/BugReportManager.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/core/util/BugReportManager.py b/core/util/BugReportManager.py index 90fbdeac7..e3a946ab3 100644 --- a/core/util/BugReportManager.py +++ b/core/util/BugReportManager.py @@ -55,9 +55,6 @@ def addToHistory(self, log: str): if not self._recording: return - if len(self._history) > 2500: - del self._history[1] # Don't delete first line, it's the git commit id - self._history.append(log) if not self._title and traceback.format_exc().strip() != 'NoneType: None': From 72c58b0ce3c65af9421f69fe0825c55e9d13d984 Mon Sep 17 00:00:00 2001 From: philipp2310 <41761223+philipp2310@users.noreply.github.com> Date: Sat, 18 Dec 2021 14:24:28 +0100 Subject: [PATCH 112/129] first show log in case update fails --- core/voice/model/Tts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/voice/model/Tts.py b/core/voice/model/Tts.py index ec9c5a81b..079e19ae2 100644 --- a/core/voice/model/Tts.py +++ b/core/voice/model/Tts.py @@ -95,8 +95,8 @@ def onStart(self): if self._type not in self._supportedLangAndVoices[self._lang]: ttsType = self._type self._type = next(iter(self._supportedLangAndVoices[self._lang])) - self.ConfigManager.updateAliceConfiguration(key='ttsType', value={self._type}) self.logWarning(f'Type **{ttsType}** not found for the language, falling back to **{self._type}**') + self.ConfigManager.updateAliceConfiguration(key='ttsType', value={self._type}) if self._voice not in self._supportedLangAndVoices[self._lang][self._type]: voice = self._voice From 19a47ae2e8419bcafba42d27febc8d72ac5b26b1 Mon Sep 17 00:00:00 2001 From: philipp2310 <41761223+philipp2310@users.noreply.github.com> Date: Sat, 18 Dec 2021 14:27:52 +0100 Subject: [PATCH 113/129] no list required --- core/voice/model/Tts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/voice/model/Tts.py b/core/voice/model/Tts.py index 079e19ae2..88ebae292 100644 --- a/core/voice/model/Tts.py +++ b/core/voice/model/Tts.py @@ -96,7 +96,7 @@ def onStart(self): ttsType = self._type self._type = next(iter(self._supportedLangAndVoices[self._lang])) self.logWarning(f'Type **{ttsType}** not found for the language, falling back to **{self._type}**') - self.ConfigManager.updateAliceConfiguration(key='ttsType', value={self._type}) + self.ConfigManager.updateAliceConfiguration(key='ttsType', value=self._type) if self._voice not in self._supportedLangAndVoices[self._lang][self._type]: voice = self._voice From e97e1757af4b22cda74da45336d3f6863e6bfdbb Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Sat, 18 Dec 2021 15:26:19 +0100 Subject: [PATCH 114/129] No log reports if offline --- core/util/BugReportManager.py | 7 +++++-- requirements.txt | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/core/util/BugReportManager.py b/core/util/BugReportManager.py index e3a946ab3..5f2eb3930 100644 --- a/core/util/BugReportManager.py +++ b/core/util/BugReportManager.py @@ -18,12 +18,11 @@ # Last modified: 2021.04.24 at 12:56:47 CEST import json import os +import requests import subprocess import traceback from pathlib import Path -import requests - from core.base.model.Manager import Manager from core.commons import constants @@ -66,6 +65,10 @@ def onStop(self): if not self._recording: return + if not self.InternetManager.online: + self.logInfo('We are currently offline, cannot send log reports') + return + if not self._history or not self._title: self.logInfo('Nothing to report') elif not self.ConfigManager.githubAuth: diff --git a/requirements.txt b/requirements.txt index db279a025..5e3a86935 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ dulwich~=0.20.26 sounddevice~=0.4.3 bcrypt~=3.2.0 PyJWT~=2.3.0 -Pympler~=0.9 +Pympler~=1.0 pydub~=0.25.1 PyAudio~=0.2.11 htmlmin~=0.1.12 From 6d395b1fe7de57b6dffa54860c81614ff3c5738c Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Sat, 18 Dec 2021 19:53:54 +0100 Subject: [PATCH 115/129] A warning is not an error --- core/util/BugReportManager.py | 10 ++++++++-- core/util/model/Logger.py | 4 ++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/core/util/BugReportManager.py b/core/util/BugReportManager.py index 5f2eb3930..58540a2b5 100644 --- a/core/util/BugReportManager.py +++ b/core/util/BugReportManager.py @@ -29,6 +29,12 @@ class BugReportManager(Manager): + ERROR_LOGS = [ + 'fatal', + 'error', + 'critical' + ] + def __init__(self): super().__init__(name='BugReportManager') @@ -50,13 +56,13 @@ def isRecording(self) -> bool: return self._recording - def addToHistory(self, log: str): + def addToHistory(self, function: str, log: str): if not self._recording: return self._history.append(log) - if not self._title and traceback.format_exc().strip() != 'NoneType: None': + if function in self.ERROR_LOGS and not self._title and traceback.format_exc().strip() != 'NoneType: None': self._title = traceback.format_exc().strip().split('\n').pop() diff --git a/core/util/model/Logger.py b/core/util/model/Logger.py index 8dd04f583..b501163d6 100644 --- a/core/util/model/Logger.py +++ b/core/util/model/Logger.py @@ -65,7 +65,7 @@ def logCritical(self, msg: str, plural: Union[list, str] = None): self.doLog(function='critical', msg=msg, plural=plural) - def doLog(self, function: callable, msg: str, printStack=True, plural: Union[list, str] = None): + def doLog(self, function: str, msg: str, printStack=True, plural: Union[list, str] = None): if not msg: return @@ -93,7 +93,7 @@ def doLog(self, function: callable, msg: str, printStack=True, plural: Union[lis try: from core.base.SuperManager import SuperManager - SuperManager.getInstance().bugReportManager.addToHistory(msg) + SuperManager.getInstance().bugReportManager.addToHistory(function, msg) except: pass # We really can't do anything here From 21009acee2261783399214ddd4689ef94d348992 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Sun, 19 Dec 2021 10:54:34 +0100 Subject: [PATCH 116/129] Mitigation for #608 --- core/device/DeviceManager.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/core/device/DeviceManager.py b/core/device/DeviceManager.py index e597904aa..8498e6e43 100644 --- a/core/device/DeviceManager.py +++ b/core/device/DeviceManager.py @@ -22,10 +22,9 @@ import threading import time import uuid -from typing import Dict, List, Optional, Union - from paho.mqtt.client import MQTTMessage from serial.tools import list_ports +from typing import Dict, List, Optional, Union from core.base.model.Manager import Manager from core.commons import constants @@ -173,6 +172,10 @@ def loadDevices(self): :return: None """ for data in self.databaseFetch(tableName=self.DB_DEVICE): + if data.get('skillName') not in self.SkillManager.allWorkingSkills: + self.logInfo(f'Device of type **{data.get("typeName")}** skipped because skill **{data.get("skillName")}** is not working') + continue + try: skillImport = importlib.import_module(f'skills.{data.get("skillName")}.devices.{data.get("typeName")}') klass = getattr(skillImport, data.get('typeName')) From 2076cb3da7f3eda52e9d09fc242030572043dfab Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Sun, 19 Dec 2021 11:20:55 +0100 Subject: [PATCH 117/129] Require to be uptodate with AliceGit 0.0.12 --- core/util/BugReportManager.py | 6 ++++++ requirements.txt | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/core/util/BugReportManager.py b/core/util/BugReportManager.py index 58540a2b5..b11de03f6 100644 --- a/core/util/BugReportManager.py +++ b/core/util/BugReportManager.py @@ -23,6 +23,7 @@ import traceback from pathlib import Path +from AliceGit import Git from core.base.model.Manager import Manager from core.commons import constants @@ -75,6 +76,11 @@ def onStop(self): self.logInfo('We are currently offline, cannot send log reports') return + repo = Git(directory=self.Commons.rootDir()) + if not repo.isUpToDate(): + self.logInfo('You are currently no up to date with you Alice install. Please first update to latest version and retry before trying to submit a bug report again.') + return + if not self._history or not self._title: self.logInfo('Nothing to report') elif not self.ConfigManager.githubAuth: diff --git a/requirements.txt b/requirements.txt index 5e3a86935..4021533cb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ Flask~=2.0.2 Flask-Cors~=3.0.10 Flask-Classful~=0.14.2 Markdown~=3.3.6 -AliceGit~=0.0.11 +AliceGit~=0.0.12 requests~=2.26.0 dulwich~=0.20.26 sounddevice~=0.4.3 From 6091049499566344ce87c0e1206e9b0e10a6353f Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Sun, 19 Dec 2021 11:38:38 +0100 Subject: [PATCH 118/129] Fix online check and make sure Alice is up to date for bug reporting --- core/util/BugReportManager.py | 13 +++++++++---- requirements.txt | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/core/util/BugReportManager.py b/core/util/BugReportManager.py index b11de03f6..e013889fe 100644 --- a/core/util/BugReportManager.py +++ b/core/util/BugReportManager.py @@ -23,7 +23,7 @@ import traceback from pathlib import Path -from AliceGit import Git +from AliceGit.Git import Repository from core.base.model.Manager import Manager from core.commons import constants @@ -72,13 +72,18 @@ def onStop(self): if not self._recording: return - if not self.InternetManager.online: + try: + online = requests.get('https://clients3.google.com/generate_204').status_code == 204 + except: + online = False + + if not online: self.logInfo('We are currently offline, cannot send log reports') return - repo = Git(directory=self.Commons.rootDir()) + repo = Repository(directory=self.Commons.rootDir()) if not repo.isUpToDate(): - self.logInfo('You are currently no up to date with you Alice install. Please first update to latest version and retry before trying to submit a bug report again.') + self.logInfo('Alice is not up to date. Please first update to latest version and retry before trying to submit a bug report again.') return if not self._history or not self._title: diff --git a/requirements.txt b/requirements.txt index 4021533cb..71d184ec3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ Flask~=2.0.2 Flask-Cors~=3.0.10 Flask-Classful~=0.14.2 Markdown~=3.3.6 -AliceGit~=0.0.12 +AliceGit~=0.0.13 requests~=2.26.0 dulwich~=0.20.26 sounddevice~=0.4.3 From 0fcaae0bee17da0c06bb6285b71664bff5492c09 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Sun, 19 Dec 2021 11:40:49 +0100 Subject: [PATCH 119/129] Delete flag file --- core/util/BugReportManager.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/util/BugReportManager.py b/core/util/BugReportManager.py index e013889fe..b7eda279a 100644 --- a/core/util/BugReportManager.py +++ b/core/util/BugReportManager.py @@ -79,11 +79,13 @@ def onStop(self): if not online: self.logInfo('We are currently offline, cannot send log reports') + os.remove(self._flagFile) return repo = Repository(directory=self.Commons.rootDir()) if not repo.isUpToDate(): self.logInfo('Alice is not up to date. Please first update to latest version and retry before trying to submit a bug report again.') + os.remove(self._flagFile) return if not self._history or not self._title: From c45c8558687cd948a1e458a53819cffd2d911cf2 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Sun, 19 Dec 2021 11:52:44 +0100 Subject: [PATCH 120/129] Checking for online condition towards our server, not google anymore --- core/util/BugReportManager.py | 2 +- core/util/InternetManager.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/util/BugReportManager.py b/core/util/BugReportManager.py index b7eda279a..0ad854523 100644 --- a/core/util/BugReportManager.py +++ b/core/util/BugReportManager.py @@ -73,7 +73,7 @@ def onStop(self): return try: - online = requests.get('https://clients3.google.com/generate_204').status_code == 204 + online = requests.get('https://api.projectalice.io/generate_204').status_code == 204 except: online = False diff --git a/core/util/InternetManager.py b/core/util/InternetManager.py index 21aa224bb..8cb27cc7d 100644 --- a/core/util/InternetManager.py +++ b/core/util/InternetManager.py @@ -59,7 +59,7 @@ def checkInternet(self): self.ThreadManager.doLater(interval=self._checkFrequency, func=self.checkInternet) - def checkOnlineState(self, addr: str = 'https://clients3.google.com/generate_204', silent: bool = False) -> bool: + def checkOnlineState(self, addr: str = 'https://api.projectalice.io/generate_204', silent: bool = False) -> bool: if self.ConfigManager.getAliceConfigByName('stayCompletelyOffline'): return False From ee7193d723c13be4d9605d3d1e3893ce9164ba94 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Sun, 19 Dec 2021 19:22:00 +0100 Subject: [PATCH 121/129] Cleanup --- core/base/model/GithubCloner.py | 358 -------------------------------- core/webApi/model/SkillsApi.py | 21 +- requirements.txt | 1 - 3 files changed, 12 insertions(+), 368 deletions(-) delete mode 100644 core/base/model/GithubCloner.py diff --git a/core/base/model/GithubCloner.py b/core/base/model/GithubCloner.py deleted file mode 100644 index a299dee20..000000000 --- a/core/base/model/GithubCloner.py +++ /dev/null @@ -1,358 +0,0 @@ -# Copyright (c) 2021 -# -# This file, GithubCloner.py, is part of Project Alice. -# -# Project Alice is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see -# -# Last modified: 2021.08.21 at 12:56:45 CEST - -import json -import shutil -from pathlib import Path - -import requests -from dulwich.errors import NotGitRepository -from dulwich import porcelain as git -from dulwich.repo import Repo - -from core.base.SuperManager import SuperManager -from core.base.model.ProjectAliceObject import ProjectAliceObject - - -class GithubCloner(ProjectAliceObject): - NAME = 'GithubCloner' - - - def __init__(self, baseUrl: str, dest: Path, skillName: str = None): - super().__init__() - self._baseUrl = baseUrl - self._dest = dest - self._skillName = skillName - - if dest.exists(): - self._repo = Repo(dest) - else: - self._repo = Repo.init(dest, mkdir=True) - - if skillName: - self._modified = self.SkillManager.allSkills.get(skillName, dict()).get('modified', False) - else: - self._modified = False - - - @classmethod - def getGithubAuth(cls) -> tuple: - """ - Returns the users configured username and token for github as a tuple - When one of the values is not supplied None is returned. - :return: - """ - username = SuperManager.getInstance().configManager.getAliceConfigByName('githubUsername') - token = SuperManager.getInstance().configManager.getAliceConfigByName('githubToken') - return (username, token) if (username and token) else None - - - @classmethod - def hasAuth(cls) -> bool: - """ - Returns if the user has entered the github data for authentication - :return: - """ - return cls.getGithubAuth() is not None - - - def cloneSkill(self): - """ - Clones a skill from github to the skills folder - This will stash and clean all changes that have been made locally - :return: - """ - try: - self._repo = Repo.init(self._dest, mkdir=True) - - git.stash_push(self._repo) - git.clean(self._repo) - git.clone(source=self._baseUrl, target=f'skills/{self._skillName}') - except Exception: - raise - - - def _doClone(self, skillName: str) -> bool: - """ - internal method to perform the clone of a skill - assumes there are no pending changes - :param skillName: - :return: - """ - try: - updateTag = self.SkillStoreManager.getSkillUpdateTag(skillName) - try: - if self.repo: - pass # It is already a repo, so continue - except NotGitRepository: - self.init() - - self.fetch() - self.pull(refSpecs=updateTag) - - return True - except FileExistsError: - pass # The repo was just created - except Exception as e: - self.logWarning(f'Something went wrong cloning github repo cln: {e}') - return False - - - def checkOwnRepoAvailable(self, skillName: str) -> bool: - """ - check if a repository for the given skill name exists in the users github - :param skillName: - :return: - """ - req = requests.get(f'https://api.github.com/repos/{self.getGithubAuth()[0]}/skill_{skillName}', auth=GithubCloner.getGithubAuth()) - if req.status_code != 200: - self.logInfo("Couldn't find repository on github") - return False - return True - - - def createForkForSkill(self, skillName: str) -> bool: - """ - create a fork of the skill from alice official github to the users github. - :param skillName: - :return: - """ - data = { - 'owner': self.ConfigManager.getAliceConfigByName("githubUsername"), - 'repo' : f'skill_{skillName}' - } - req = requests.post(f'https://api.github.com/repos/project-alice-assistant/skill_{skillName}/forks', data=json.dumps(data), auth=GithubCloner.getGithubAuth()) - if req.status_code == 404: - self.createRepo(aliceSK=True) - if req.status_code != 202: - self.logError(f'Couldn\'t create fork for repository! {req.status_code}') - return False - - return True - - - def checkoutOwnFork(self) -> bool: - """ - Assumes there is already a fork for the current skill on the users repository. - Clone that repository, set upstream to the original repository. - Will only work on master! - :return: - """ - try: - if not Path(self._dest / '.git').exists(): - self.Commons.runSystemCommand(['git', '-C', str(self._dest), 'init']) - self.Commons.runSystemCommand(['git', '-C', str(self._dest), 'remote', 'add', 'origin', self._baseUrl]) - self.Commons.runSystemCommand(['git', '-C', str(self._dest), 'pull']) - - remote = self.getRemote() - self.Commons.runSystemCommand(['git', '-C', str(self._dest), 'remote', 'add', 'AliceSK', remote]) - self.Commons.runSystemCommand(['git', '-C', str(self._dest), 'checkout', 'master']) - self.Commons.runSystemCommand(['git', '-C', str(self._dest), 'branch', '--set-upstream-to=AliceSK/master']) - self.Commons.runSystemCommand(['git', '-C', str(self._dest), 'pull']) - - return True - except Exception as e: - self.logWarning(f'Something went wrong cloning github repo frk: {e}') - return False - - - def getRemote(self, AliceSK: bool = False, origin: bool = False, noToken: bool = False): # NOSONAR - tokenPrefix = '' - if self.ConfigManager.getAliceConfigByName('githubUsername') and self.ConfigManager.getAliceConfigByName('githubToken'): - tokenPrefix = f'{self.ConfigManager.getAliceConfigByName("githubUsername")}:{self.ConfigManager.getAliceConfigByName("githubToken")}@' - - if self._skillName: - if AliceSK: - return f'https://{"" if noToken else tokenPrefix}github.com/{self.ConfigManager.getAliceConfigByName("githubUsername")}/skill_{self._skillName}.git' - elif origin: - return f'https://{"" if noToken else tokenPrefix}github.com/project-alice-assistant/skill_{self._skillName}.git' - elif self._modified: - return self.getRemote(AliceSK=True) - else: - return self.getRemote(origin=True) - else: - raise Exception("Skillname not set. Can't find git remote!") - - - def checkRemote(self, AliceSK: bool = False, origin: bool = False): # NOSONAR - req = requests.get(self.getRemote(AliceSK=AliceSK, origin=origin), auth=GithubCloner.getGithubAuth()) - if req.status_code != 200: - return False - return True - - - def checkoutMaster(self) -> bool: - """ - set upstream to origin/master - :return: - """ - try: - self.Commons.runSystemCommand(['git', '-C', str(self._dest), 'branch', '--set-upstream-to=origin/master']) - return True - except Exception as e: - self.logWarning(f'Something went wrong cloning github repo chk: {e}') - return False - - - def gitDefaults(self) -> bool: - try: - self.Commons.runSystemCommand(['git', '-C', str(self._dest), 'config', 'user.email', self.ConfigManager.getAliceConfigByName('githubMail') or 'githubbot@projectalice.io']) - self.Commons.runSystemCommand(['git', '-C', str(self._dest), 'config', 'user.name', self.ConfigManager.getAliceConfigByName('githubUsername') or 'ProjectAliceBot']) - return True - except Exception as e: - self.logWarning(f'Something went wrong cloning github repo def: {e}') - return False - - - @property - def repo(self) -> Repo: - if self._repo: - return self._repo - self._repo = Repo(f'skills/{self._skillName}') - return self._repo - - - @repo.setter - def repo(self, repo: Repo): - self._repo = repo - - - def pull(self, refSpecs: str = b'master'): - git.pull(repo=self.repo, remote_location=self.getRemote(), refspecs=refSpecs) - - - def fetch(self): - remoteRefs = git.fetch(repo=self.repo, remote_location=self.getRemote()) - for key, value in remoteRefs.items(): - self.repo.refs[key] = value - - - def getPath(self) -> Path: - return Path(f'skills/{self._skillName}') - - - def init(self) -> bool: - """ - create a repository online and clone it to the skills folder - only afterwards fill it with files by AliceSK - :return: - """ - git.clone(source=self.getRemote(), target=f'skills/{self._skillName}') - - self.repo = Repo.init(f'skills/{self._skillName}') - git.remote_add(repo=self.repo, name=f'AliceSK', url=self.getRemote()) - - return True - - - def add(self) -> bool: - """ - add all changes to the current tree - :return: - """ - stat = git.status(self.repo) - self.repo.stage(stat.unstaged + stat.untracked) - return True - - - def commit(self, message: str = 'pushed by AliceSK') -> bool: - """ - commit the current changes for that skill - :param message: - :return: - """ - git.commit(repo=self.repo, message=message) - return True - - - def push(self) -> bool: - """ - push the skills changes to AliceSK upstream - :return: - """ - git.push(repo=self.repo, remote_location=self.getRemote(), refspecs=b'master') - return True - - - def isRepo(self) -> bool: - """ - check if the skills folder is already a repository - :return: - """ - try: - # noinspection PyStatementEffect - self.repo # NOSONAR - return True - except NotGitRepository as e: - self.logInfo(f'Error repository: {e}') - return False - - - def createRepo(self, aliceSK: bool = False) -> bool: - """ - create a repository and set the remotes for origin and AliceSK - :return: - """ - if not self.isRepo(): - self.repo = Repo.init(self.getPath()) - try: - self.createRemote() - except Exception: - return False - try: - git.remote_add(repo=self.repo, name=b'origin', url=self.getRemote(origin=True)) - except git.RemoteExists: - pass - try: - if aliceSK: - git.remote_add(repo=self.repo, name=b'AliceSK', url=self.getRemote(AliceSK=True)) - except git.RemoteExists: - pass - return True - - - def createRemote(self) -> bool: - """ - create the remote repository for the current user - :return: - """ - data = { - 'name' : f'skill_{self._skillName}', - 'description': 'test', - 'has-issues' : True, - 'has-wiki' : False - } - req = requests.post('https://api.github.com/user/repos', data=json.dumps(data), auth=GithubCloner.getGithubAuth()) - - if req.status_code != 201: - raise Exception("Couldn't create the repository on Github") - - return True - - - def gitDoMyTest(self): - skillName = 'FritzBox' - rep: Repo = Repo(f'skills/{skillName}') - self.logInfo(f'got {rep} in {rep.path}') - - stat = git.status(rep) - self.logInfo(f'statstaged {stat.staged}') - self.logInfo(f'statuntrack {stat.untracked}') - self.logInfo(f'statunstag {stat.unstaged}') - rep.stage(stat.unstaged + stat.untracked) - self.logInfo(f'commit {git.commit(repo=rep, message="pushed by AliceSK")}') - self.logInfo(f'push {git.push(repo=rep, remote_location=self.getRemote(), refspecs=b"master")}') diff --git a/core/webApi/model/SkillsApi.py b/core/webApi/model/SkillsApi.py index 9748d10d0..ba1e47f92 100644 --- a/core/webApi/model/SkillsApi.py +++ b/core/webApi/model/SkillsApi.py @@ -27,8 +27,6 @@ from AliceGit.Exceptions import AlreadyGitRepository, GithubRepoNotFound, GithubUserNotFound, NotGitRepository from AliceGit.Git import Repository from AliceGit.Github import Github -from core.base.model.GithubCloner import GithubCloner -from core.commons import constants from core.util.Decorators import ApiAuthenticated from core.webApi.model.Api import Api @@ -272,15 +270,20 @@ def setModified(self, skillName: str) -> Response: if skillName not in self.SkillManager.allSkills: return self.skillNotFound() - if not GithubCloner.hasAuth(): + if not self.ConfigManager.githubAuth: return self.githubMissing() - gitCloner = GithubCloner(baseUrl=f'{constants.GITHUB_URL}/skill_{skillName}.git', - dest=Path(self.Commons.rootDir()) / 'skills' / skillName, - skillName=skillName) - if not gitCloner.checkOwnRepoAvailable(skillName=skillName): - gitCloner.createForkForSkill(skillName=skillName) - gitCloner.checkoutOwnFork() + try: + auth = self.ConfigManager.githubAuth + Github( + username=auth[0], + token=auth[1], + repositoryName=f'skill_{skillName}', + createRepository=True + ) + except GithubUserNotFound: + return jsonify(success=False, reason='Github user not existing') + return jsonify(success=True) diff --git a/requirements.txt b/requirements.txt index 71d184ec3..8813816ee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,6 @@ Flask-Classful~=0.14.2 Markdown~=3.3.6 AliceGit~=0.0.13 requests~=2.26.0 -dulwich~=0.20.26 sounddevice~=0.4.3 bcrypt~=3.2.0 PyJWT~=2.3.0 From 1d3415a3bdc873b9873e67eb3623c6ce23ea0985 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Sun, 19 Dec 2021 19:29:14 +0100 Subject: [PATCH 122/129] Fix unittests --- tests/util/test_InternetManager.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/util/test_InternetManager.py b/tests/util/test_InternetManager.py index 9a3a7ce06..c18eee332 100644 --- a/tests/util/test_InternetManager.py +++ b/tests/util/test_InternetManager.py @@ -17,11 +17,11 @@ # # Last modified: 2021.04.13 at 12:56:52 CEST -import unittest from unittest import mock -from unittest.mock import MagicMock, PropertyMock +import unittest from requests.exceptions import RequestException +from unittest.mock import MagicMock, PropertyMock from core.util.InternetManager import InternetManager @@ -33,6 +33,8 @@ class TestInternetManager(unittest.TestCase): @mock.patch('core.util.InternetManager.InternetManager.Commons', new_callable=PropertyMock) @mock.patch('core.base.SuperManager.SuperManager') def test_checkOnlineState(self, mock_superManager, mock_commons, mock_requests, mock_broadcast): + address = 'https://api.projectalice.io/generate_204' + common_mock = MagicMock() common_mock.getFunctionCaller.return_value = 'InternetManager' mock_commons.return_value = common_mock @@ -50,7 +52,7 @@ def test_checkOnlineState(self, mock_superManager, mock_commons, mock_requests, type(mock_requestsResult).status_code = mock_statusCode mock_requests.get.return_value = mock_requestsResult - # Not called if stay completly offline + # Not called if stay completely offline mock_instance.configManager.getAliceConfigByName.return_value = True internetManager.checkOnlineState() mock_requests.get.asset_not_called() @@ -58,7 +60,7 @@ def test_checkOnlineState(self, mock_superManager, mock_commons, mock_requests, mock_instance.configManager.getAliceConfigByName.return_value = False internetManager.checkOnlineState() - mock_requests.get.assert_called_once_with('https://clients3.google.com/generate_204') + mock_requests.get.assert_called_once_with(address) mock_broadcast.assert_called_once_with(method='internetConnected', exceptions=['InternetManager'], propagateToSkills=True) self.assertEqual(internetManager.online, True) mock_broadcast.reset_mock() @@ -66,7 +68,7 @@ def test_checkOnlineState(self, mock_superManager, mock_commons, mock_requests, # when calling check online state a second time it does not broadcast again internetManager.checkOnlineState() - mock_requests.get.assert_called_once_with('https://clients3.google.com/generate_204') + mock_requests.get.assert_called_once_with(address) mock_broadcast.assert_not_called() self.assertEqual(internetManager.online, True) mock_broadcast.reset_mock() @@ -80,7 +82,7 @@ def test_checkOnlineState(self, mock_superManager, mock_commons, mock_requests, # when wrong status code is returned (and currently online) internetManager.checkOnlineState() - mock_requests.get.assert_called_once_with('https://clients3.google.com/generate_204') + mock_requests.get.assert_called_once_with(address) mock_broadcast.assert_called_once_with(method='internetLost', exceptions=['InternetManager'], propagateToSkills=True) self.assertEqual(internetManager.online, False) mock_broadcast.reset_mock() @@ -88,7 +90,7 @@ def test_checkOnlineState(self, mock_superManager, mock_commons, mock_requests, # when calling check online state a second time it does not broadcast again internetManager.checkOnlineState() - mock_requests.get.assert_called_once_with('https://clients3.google.com/generate_204') + mock_requests.get.assert_called_once_with(address) mock_broadcast.assert_not_called() self.assertEqual(internetManager.online, False) mock_broadcast.reset_mock() @@ -106,7 +108,7 @@ def test_checkOnlineState(self, mock_superManager, mock_commons, mock_requests, # request raises exception is the same as non 204 status code mock_requests.get.side_effect = RequestException internetManager.checkOnlineState() - mock_requests.get.assert_called_once_with('https://clients3.google.com/generate_204') + mock_requests.get.assert_called_once_with(address) mock_broadcast.assert_called_once_with(method='internetLost', exceptions=['InternetManager'], propagateToSkills=True) self.assertEqual(internetManager.online, False) From 1d53ff88b6a53f3cc949bb673b854a81030b3914 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Sun, 19 Dec 2021 19:36:25 +0100 Subject: [PATCH 123/129] Cleanup --- core/Initializer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/Initializer.py b/core/Initializer.py index cde22ed38..61f149bc5 100644 --- a/core/Initializer.py +++ b/core/Initializer.py @@ -448,7 +448,7 @@ def initProjectAlice(self) -> bool: # NOSONAR try: if len(str(pinCode)) != 4: raise Exception - pin = int(pinCode) + int(pinCode) except: self._logger.logFatal('Pin code must be 4 digits') From 66b031dc86baa8232c2dcd07bcad811f11fd1239 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Sun, 19 Dec 2021 19:39:11 +0100 Subject: [PATCH 124/129] Cleanup CoquiAsr.py --- core/asr/model/CoquiAsr.py | 32 +++++++++----------------------- 1 file changed, 9 insertions(+), 23 deletions(-) diff --git a/core/asr/model/CoquiAsr.py b/core/asr/model/CoquiAsr.py index eb089ee95..6b63a08cd 100644 --- a/core/asr/model/CoquiAsr.py +++ b/core/asr/model/CoquiAsr.py @@ -17,11 +17,10 @@ # # Last modified: 2021.04.13 at 12:56:45 CEST +import numpy as np from pathlib import Path from typing import Generator, Optional -import numpy as np - from core.asr.model.ASRResult import ASRResult from core.asr.model.Asr import Asr from core.asr.model.Recorder import Recorder @@ -97,46 +96,33 @@ def tFlite(self) -> Path: def downloadLanguage(self) -> bool: # NOSONAR self.logInfo(f'Downloading language model for "{self.LanguageManager.activeLanguage}", hold on, this is going to take some time!') # TODO TEMP! until real model zoo exists + target = str(self._langPath / 'lm.scorer') if self.LanguageManager.activeLanguage == 'de': self.Commons.downloadFile('https://github.com/coqui-ai/STT-models/releases/download/german/AASHISHAG/v0.9.0/model.tflite', str(self.tFlite)) - self.Commons.downloadFile('https://github.com/philipp2310/Coqui-models/releases/download/de_v093/lm.scorer', str(self._langPath / 'lm.scorer')) + self.Commons.downloadFile('https://github.com/philipp2310/Coqui-models/releases/download/de_v093/lm.scorer', target) return True elif self.LanguageManager.activeLanguage == 'en': self.Commons.downloadFile('https://github.com/coqui-ai/STT-models/releases/download/english%2Fcoqui%2Fv1.0.0-large-vocab/model.tflite', str(self.tFlite)) - self.Commons.downloadFile('https://github.com/coqui-ai/STT-models/releases/download/english%2Fcoqui%2Fv1.0.0-large-vocab/large_vocabulary.scorer', str(self._langPath / 'lm.scorer')) + self.Commons.downloadFile('https://github.com/coqui-ai/STT-models/releases/download/english%2Fcoqui%2Fv1.0.0-large-vocab/large_vocabulary.scorer', target) return True elif self.LanguageManager.activeLanguage == 'fr': self.Commons.downloadFile('https://github.com/coqui-ai/STT-models/releases/download/french/commonvoice-fr/v0.6/model.tflite', str(self.tFlite)) - self.Commons.downloadFile('https://github.com/coqui-ai/STT-models/releases/download/french/commonvoice-fr/v0.6/fr-cvfr-2-prune-kenlm.scorer', str(self._langPath / 'lm.scorer')) + self.Commons.downloadFile('https://github.com/coqui-ai/STT-models/releases/download/french/commonvoice-fr/v0.6/fr-cvfr-2-prune-kenlm.scorer', target) return True elif self.LanguageManager.activeLanguage == 'it': self.Commons.downloadFile('https://github.com/coqui-ai/STT-models/releases/download/italian/mozillaitalia/2020.8.7/model.tflite', str(self.tFlite)) - self.Commons.downloadFile('https://github.com/coqui-ai/STT-models/releases/download/italian/mozillaitalia/2020.8.7/it-mzit-1-prune-kenlm.scorer', str(self._langPath / 'lm.scorer')) + self.Commons.downloadFile('https://github.com/coqui-ai/STT-models/releases/download/italian/mozillaitalia/2020.8.7/it-mzit-1-prune-kenlm.scorer', target) return True elif self.LanguageManager.activeLanguage == 'pl': self.Commons.downloadFile('https://github.com/coqui-ai/STT-models/releases/download/polish/jaco-assistant/v0.0.1/model.tflite', str(self.tFlite)) - self.Commons.downloadFile('https://github.com/coqui-ai/STT-models/releases/download/polish/jaco-assistant/v0.0.1/kenlm_pl.scorer', str(self._langPath / 'lm.scorer')) + self.Commons.downloadFile('https://github.com/coqui-ai/STT-models/releases/download/polish/jaco-assistant/v0.0.1/kenlm_pl.scorer', target) return True elif self.LanguageManager.activeLanguage == 'pt': self.Commons.downloadFile('https://github.com/coqui-ai/STT-models/releases/download/portuguese/itml/v0.1.0/model.tflite', str(self.tFlite)) - self.Commons.downloadFile('https://github.com/coqui-ai/STT-models/releases/download/portuguese/itml/v0.1.0/pt-itml-0-prune-kenlm.scorer', str(self._langPath / 'lm.scorer')) + self.Commons.downloadFile('https://github.com/coqui-ai/STT-models/releases/download/portuguese/itml/v0.1.0/pt-itml-0-prune-kenlm.scorer', target) return True else: - self.logError(f'WIP! Only de/en supported for now - Please install language manually into PA/trained/asr/Coqui//!') - return False - - downloadPath = (self._langPath / url.rsplit('/')[-1]) - try: - self.Commons.downloadFile(url, str(downloadPath)) - - self.logInfo(f'Language model for "{self.LanguageManager.activeLanguage}" downloaded, now extracting...') - self.Commons.runSystemCommand(['tar', '-C', f'{str(downloadPath.parent)}', '-zxvf', str(downloadPath)]) # TODO adjust to new dl requirements - - downloadPath.unlink() - return True - except Exception as e: - self.logError(f'Error installing language model: {e}') - downloadPath.unlink() + self.logError(f'WIP! Only de/en supported for now - Please install language manually into PA/trained/asr/Coqui//!') return False From f89e128e450c9df9e7b2cceba3bc60771618e1b3 Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Mon, 20 Dec 2021 11:24:36 +0100 Subject: [PATCH 125/129] Updated node-red settings --- core/webui/NodeRedManager.py | 8 +- .../LanguageManager/notifications.json | 30 + system/node-red/settings.js | 658 +++++++++++------- 3 files changed, 459 insertions(+), 237 deletions(-) diff --git a/core/webui/NodeRedManager.py b/core/webui/NodeRedManager.py index df12243ac..6fb5af4b3 100644 --- a/core/webui/NodeRedManager.py +++ b/core/webui/NodeRedManager.py @@ -25,6 +25,7 @@ from subprocess import PIPE, Popen from core.base.model.Manager import Manager +from core.webui.model.UINotificationType import UINotificationType class NodeRedManager(Manager): @@ -70,6 +71,7 @@ def onStart(self): def install(self): self.logInfo('Node-RED not found, installing, this might take a while...') + self.WebUINotificationManager.newNotification(typ=UINotificationType.INFO, notification='installNodeRed', key='nodered') self.Commons.downloadFile( url='https://raw.githubusercontent.com/node-red/linux-installers/master/deb/update-nodejs-and-nodered', dest='var/cache/node-red.sh' @@ -82,6 +84,7 @@ def install(self): process.stdin.write(b'n\n') except IOError: self.logError('Failed installing Node-RED') + self.WebUINotificationManager.newNotification(typ=UINotificationType.ERROR, notification='failedInstallNodeRed', key='nodered') self.onStop() return @@ -90,6 +93,7 @@ def install(self): if returnCode: self.logError('Failed installing Node-red') + self.WebUINotificationManager.newNotification(typ=UINotificationType.ERROR, notification='failedInstallNodeRed', key='nodered') self.onStop() else: self.logInfo('Successfully installed Node-red') @@ -104,7 +108,6 @@ def configureNewNodeRed(self): time.sleep(5) self.Commons.runRootSystemCommand(['systemctl', 'stop', 'nodered']) time.sleep(3) - config = Path(self.PACKAGE_PATH.parent, '.config.nodes.json') data = json.loads(config.read_text()) for package in data.values(): @@ -118,9 +121,10 @@ def configureNewNodeRed(self): self.logInfo('Nodes configured') self.logInfo('Applying Project Alice settings') - self.Commons.runSystemCommand('npm install --prefix ~/.node-red @node-red-contrib-themes/midnight-red'.split()) + self.Commons.runSystemCommand('npm install --prefix ~/.node-red @node-red-contrib-themes/midnight-red'.split(), shell=True) shutil.copy(Path('system/node-red/settings.js'), Path(os.path.expanduser('~/.node-red'), 'settings.js')) self.logInfo("All done, let's start all this") + self.WebUINotificationManager.newNotification(typ=UINotificationType.INFO, notification='installedNodeRed', key='nodered') def onStop(self): diff --git a/system/manager/LanguageManager/notifications.json b/system/manager/LanguageManager/notifications.json index 874ccca0b..c79c70812 100644 --- a/system/manager/LanguageManager/notifications.json +++ b/system/manager/LanguageManager/notifications.json @@ -122,5 +122,35 @@ "en": "Failed downloading \"{}\"", "fr": "Téléchargement de \"{}\" échoué" } + }, + "installNodeRed": { + "title": { + "en": "Node-Red", + "fr": "Node-Red" + }, + "body": { + "en": "Installing...", + "fr": "Installation..." + } + }, + "failedInstallNodeRed": { + "title": { + "en": "Node-Red", + "fr": "Node-Red" + }, + "body": { + "en": "Install failed", + "fr": "L'installation a échoué." + } + }, + "installedNodeRed": { + "title": { + "en": "Node-Red", + "fr": "Node-Red" + }, + "body": { + "en": "Install successful", + "fr": "Installation terminée" + } } } diff --git a/system/node-red/settings.js b/system/node-red/settings.js index 091767c76..7d7720830 100644 --- a/system/node-red/settings.js +++ b/system/node-red/settings.js @@ -1,121 +1,78 @@ /** - * Copyright JS Foundation and other contributors, http://js.foundation + * This is the default settings file provided by Node-RED. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * It can contain any valid JavaScript code that will get run when Node-RED + * is started. * - * http://www.apache.org/licenses/LICENSE-2.0 + * Lines that start with // are commented out. + * Each entry should be separated from the entries above and below by a comma ',' + * + * For more information about individual settings, refer to the documentation: + * https://nodered.org/docs/user-guide/runtime/configuration + * + * The settings are split into the following sections: + * - Flow File and User Directory Settings + * - Security + * - Server Settings + * - Runtime Settings + * - Editor Settings + * - Node Settings * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. **/ module.exports = { - // the tcp port that the Node-RED web server is listening on - uiPort: process.env.PORT || 1880, - - // By default, the Node-RED UI accepts connections on all IPv4 interfaces. - // To listen on all IPv6 addresses, set uiHost to "::", - // The following property can be used to listen on a specific interface. For - // example, the following would only allow connections from the local machine. - //uiHost: "127.0.0.1", - - // Retry time in milliseconds for MQTT connections - mqttReconnectTime: 15000, - - // Retry time in milliseconds for Serial port connections - serialReconnectTime: 15000, - - // Retry time in milliseconds for TCP socket connections - //socketReconnectTime: 10000, - - // Timeout in milliseconds for TCP server socket connections - // defaults to no timeout - //socketTimeout: 120000, - - // Maximum number of messages to wait in queue while attempting to connect to TCP socket - // defaults to 1000 - //tcpMsgQueueSize: 2000, - - // Timeout in milliseconds for HTTP request connections - // defaults to 120 seconds - //httpRequestTimeout: 120000, - - // The maximum length, in characters, of any message sent to the debug sidebar tab - debugMaxLength: 1000, - - // The maximum number of messages nodes will buffer internally as part of their - // operation. This applies across a range of nodes that operate on message sequences. - // defaults to no limit. A value of 0 also means no limit is applied. - //nodeMessageBufferMaxLength: 0, - - // To disable the option for using local files for storing keys and certificates in the TLS configuration - // node, set this to true - //tlsConfigDisableLocalFiles: true, - - // Colourise the console output of the debug node - //debugUseColors: true, - - // The file containing the flows. If not set, it defaults to flows_.json - //flowFile: 'flows.json', - // To enabled pretty-printing of the flow within the flow file, set the following - // property to true: - //flowFilePretty: true, - - // By default, credentials are encrypted in storage using a generated key. To - // specify your own secret, set the following property. - // If you want to disable encryption of credentials, set this property to false. - // Note: once you set this property, do not change it - doing so will prevent - // node-red from being able to decrypt your existing credentials and they will be - // lost. + /******************************************************************************* + * Flow File and User Directory Settings + * - flowFile + * - credentialSecret + * - flowFilePretty + * - userDir + * - nodesDir + ******************************************************************************/ + + /** The file containing the flows. If not set, defaults to flows_.json **/ + flowFile: 'flows.json', + + /** By default, credentials are encrypted in storage using a generated key. To + * specify your own secret, set the following property. + * If you want to disable encryption of credentials, set this property to false. + * Note: once you set this property, do not change it - doing so will prevent + * node-red from being able to decrypt your existing credentials and they will be + * lost. + */ //credentialSecret: "a-secret-key", - // By default, all user data is stored in a directory called `.node-red` under - // the user's home directory. To use a different location, the following - // property can be used + /** By default, the flow JSON will be formatted over multiple lines making + * it easier to compare changes when using version control. + * To disable pretty-printing of the JSON set the following property to false. + */ + flowFilePretty: true, + + /** By default, all user data is stored in a directory called `.node-red` under + * the user's home directory. To use a different location, the following + * property can be used + */ //userDir: '/home/nol/.node-red/', - // Node-RED scans the `nodes` directory in the userDir to find local node files. - // The following property can be used to specify an additional directory to scan. + /** Node-RED scans the `nodes` directory in the userDir to find local node files. + * The following property can be used to specify an additional directory to scan. + */ //nodesDir: '/home/nol/.node-red/nodes', - // By default, the Node-RED UI is available at http://localhost:1880/ - // The following property can be used to specify a different root path. - // If set to false, this is disabled. - //httpAdminRoot: '/admin', - - // Some nodes, such as HTTP In, can be used to listen for incoming http requests. - // By default, these are served relative to '/'. The following property - // can be used to specifiy a different root path. If set to false, this is - // disabled. - //httpNodeRoot: '/red-nodes', - - // The following property can be used in place of 'httpAdminRoot' and 'httpNodeRoot', - // to apply the same root to both parts. - //httpRoot: '/red', - - // When httpAdminRoot is used to move the UI to a different root path, the - // following property can be used to identify a directory of static content - // that should be served at http://localhost:1880/. - //httpStatic: '/home/nol/node-red-static/', - - // The maximum size of HTTP request that will be accepted by the runtime api. - // Default: 5mb - //apiMaxLength: '5mb', - - // If you installed the optional node-red-dashboard you can set it's path - // relative to httpRoot - //ui: { path: "ui" }, - - // Securing Node-RED - // ----------------- - // To password protect the Node-RED editor and admin API, the following - // property can be used. See http://nodered.org/docs/security.html for details. + /******************************************************************************* + * Security + * - adminAuth + * - https + * - httpsRefreshInterval + * - requireHttps + * - httpNodeAuth + * - httpStaticAuth + ******************************************************************************/ + + /** To password protect the Node-RED editor and admin API, the following + * property can be used. See http://nodered.org/docs/security.html for details. + */ //adminAuth: { // type: "credentials", // users: [{ @@ -125,24 +82,20 @@ module.exports = { // }] //}, - // To password protect the node-defined HTTP endpoints (httpNodeRoot), or - // the static content (httpStatic), the following properties can be used. - // The pass field is a bcrypt hash of the password. - // See http://nodered.org/docs/security.html#generating-the-password-hash - //httpNodeAuth: {user:"user",pass:"$2a$08$zZWtXTja0fB1pzD4sHCMyOCMYz2Z6dNbM6tl8sJogENOMcxWV9DN."}, - //httpStaticAuth: {user:"user",pass:"$2a$08$zZWtXTja0fB1pzD4sHCMyOCMYz2Z6dNbM6tl8sJogENOMcxWV9DN."}, + /** The following property can be used to enable HTTPS + * This property can be either an object, containing both a (private) key + * and a (public) certificate, or a function that returns such an object. + * See http://nodejs.org/api/https.html#https_https_createserver_options_requestlistener + * for details of its contents. + */ - // The following property can be used to enable HTTPS - // See http://nodejs.org/api/https.html#https_https_createserver_options_requestlistener - // for details on its contents. - // This property can be either an object, containing both a (private) key and a (public) certificate, - // or a function that returns such an object: - //// https object: + /** Option 1: static object */ //https: { // key: require("fs").readFileSync('privkey.pem'), // cert: require("fs").readFileSync('cert.pem') //}, - ////https function: + + /** Option 2: function that returns the HTTP configuration object */ // https: function() { // // This function should return the options object, or a Promise // // that resolves to the options object @@ -152,53 +105,71 @@ module.exports = { // } // }, - // The following property can be used to refresh the https settings at a - // regular time interval in hours. - // This requires: - // - the `https` setting to be a function that can be called to get - // the refreshed settings. - // - Node.js 11 or later. + /** If the `https` setting is a function, the following setting can be used + * to set how often, in hours, the function will be called. That can be used + * to refresh any certificates. + */ //httpsRefreshInterval : 12, - // The following property can be used to cause insecure HTTP connections to - // be redirected to HTTPS. + /** The following property can be used to cause insecure HTTP connections to + * be redirected to HTTPS. + */ //requireHttps: true, - // The following property can be used to disable the editor. The admin API - // is not affected by this option. To disable both the editor and the admin - // API, use either the httpRoot or httpAdminRoot properties - //disableEditor: false, + /** To password protect the node-defined HTTP endpoints (httpNodeRoot), + * including node-red-dashboard, or the static content (httpStatic), the + * following properties can be used. + * The `pass` field is a bcrypt hash of the password. + * See http://nodered.org/docs/security.html#generating-the-password-hash + */ + //httpNodeAuth: {user:"user",pass:"$2a$08$zZWtXTja0fB1pzD4sHCMyOCMYz2Z6dNbM6tl8sJogENOMcxWV9DN."}, + //httpStaticAuth: {user:"user",pass:"$2a$08$zZWtXTja0fB1pzD4sHCMyOCMYz2Z6dNbM6tl8sJogENOMcxWV9DN."}, - // The following property can be used to configure cross-origin resource sharing - // in the HTTP nodes. - // See https://github.com/troygoode/node-cors#configuration-options for - // details on its contents. The following is a basic permissive set of options: - //httpNodeCors: { - // origin: "*", - // methods: "GET,PUT,POST,DELETE" - //}, + /******************************************************************************* + * Server Settings + * - uiPort + * - uiHost + * - apiMaxLength + * - httpServerOptions + * - httpAdminRoot + * - httpAdminMiddleware + * - httpNodeRoot + * - httpNodeCors + * - httpNodeMiddleware + * - httpStatic + ******************************************************************************/ + + /** the tcp port that the Node-RED web server is listening on */ + uiPort: process.env.PORT || 1880, - // If you need to set an http proxy please set an environment variable - // called http_proxy (or HTTP_PROXY) outside of Node-RED in the operating system. - // For example - http_proxy=http://myproxy.com:8080 - // (Setting it here will have no effect) - // You may also specify no_proxy (or NO_PROXY) to supply a comma separated - // list of domains to not proxy, eg - no_proxy=.acme.co,.acme.co.uk + /** By default, the Node-RED UI accepts connections on all IPv4 interfaces. + * To listen on all IPv6 addresses, set uiHost to "::", + * The following property can be used to listen on a specific interface. For + * example, the following would only allow connections from the local machine. + */ + //uiHost: "127.0.0.1", - // The following property can be used to add a custom middleware function - // in front of all http in nodes. This allows custom authentication to be - // applied to all http in nodes, or any other sort of common request processing. - //httpNodeMiddleware: function(req,res,next) { - // // Handle/reject the request, or pass it on to the http in node by calling next(); - // // Optionally skip our rawBodyParser by setting this to true; - // //req.skipRawBodyParser = true; - // next(); - //}, + /** The maximum size of HTTP request that will be accepted by the runtime api. + * Default: 5mb + */ + //apiMaxLength: '5mb', + /** The following property can be used to pass custom options to the Express.js + * server used by Node-RED. For a full list of available options, refer + * to http://expressjs.com/en/api.html#app.settings.table + */ + //httpServerOptions: { }, - // The following property can be used to add a custom middleware function - // in front of all admin http routes. For example, to set custom http - // headers + /** By default, the Node-RED UI is available at http://localhost:1880/ + * The following property can be used to specify a different root path. + * If set to false, this is disabled. + */ + //httpAdminRoot: '/admin', + + /** The following property can be used to add a custom middleware function + * in front of all admin http routes. For example, to set custom http + * headers. It can be a single function or an array of middleware functions. + */ // httpAdminMiddleware: function(req,res,next) { // // Set the X-Frame-Options header to limit where the editor // // can be embedded @@ -206,101 +177,318 @@ module.exports = { // next(); // }, - // The following property can be used to pass custom options to the Express.js - // server used by Node-RED. For a full list of available options, refer - // to http://expressjs.com/en/api.html#app.settings.table - //httpServerOptions: { }, - // The following property can be used to verify websocket connection attempts. - // This allows, for example, the HTTP request headers to be checked to ensure - // they include valid authentication information. - //webSocketNodeVerifyClient: function(info) { - // // 'info' has three properties: - // // - origin : the value in the Origin header - // // - req : the HTTP request - // // - secure : true if req.connection.authorized or req.connection.encrypted is set - // // - // // The function should return true if the connection should be accepted, false otherwise. - // // - // // Alternatively, if this function is defined to accept a second argument, callback, - // // it can be used to verify the client asynchronously. - // // The callback takes three arguments: - // // - result : boolean, whether to accept the connection or not - // // - code : if result is false, the HTTP error status to return - // // - reason: if result is false, the HTTP reason string to return + /** Some nodes, such as HTTP In, can be used to listen for incoming http requests. + * By default, these are served relative to '/'. The following property + * can be used to specifiy a different root path. If set to false, this is + * disabled. + */ + //httpNodeRoot: '/red-nodes', + + /** The following property can be used to configure cross-origin resource sharing + * in the HTTP nodes. + * See https://github.com/troygoode/node-cors#configuration-options for + * details on its contents. The following is a basic permissive set of options: + */ + //httpNodeCors: { + // origin: "*", + // methods: "GET,PUT,POST,DELETE" //}, - // The following property can be used to seed Global Context with predefined - // values. This allows extra node modules to be made available with the - // Function node. - // For example, - // functionGlobalContext: { os:require('os') } - // can be accessed in a function block as: - // global.get("os") - functionGlobalContext : { - // os:require('os'), - // jfive:require("johnny-five"), - // j5board:require("johnny-five").Board({repl:false}) - }, - // `global.keys()` returns a list of all properties set in global context. - // This allows them to be displayed in the Context Sidebar within the editor. - // In some circumstances it is not desirable to expose them to the editor. The - // following property can be used to hide any property set in `functionGlobalContext` - // from being list by `global.keys()`. - // By default, the property is set to false to avoid accidental exposure of - // their values. Setting this to true will cause the keys to be listed. - exportGlobalContextKeys: false, + /** If you need to set an http proxy please set an environment variable + * called http_proxy (or HTTP_PROXY) outside of Node-RED in the operating system. + * For example - http_proxy=http://myproxy.com:8080 + * (Setting it here will have no effect) + * You may also specify no_proxy (or NO_PROXY) to supply a comma separated + * list of domains to not proxy, eg - no_proxy=.acme.co,.acme.co.uk + */ + + /** The following property can be used to add a custom middleware function + * in front of all http in nodes. This allows custom authentication to be + * applied to all http in nodes, or any other sort of common request processing. + * It can be a single function or an array of middleware functions. + */ + //httpNodeMiddleware: function(req,res,next) { + // // Handle/reject the request, or pass it on to the http in node by calling next(); + // // Optionally skip our rawBodyParser by setting this to true; + // //req.skipRawBodyParser = true; + // next(); + //}, + + /** When httpAdminRoot is used to move the UI to a different root path, the + * following property can be used to identify a directory of static content + * that should be served at http://localhost:1880/. + */ + //httpStatic: '/home/nol/node-red-static/', + /******************************************************************************* + * Runtime Settings + * - lang + * - logging + * - contextStorage + * - exportGlobalContextKeys + * - externalModules + ******************************************************************************/ + + /** Uncomment the following to run node-red in your preferred language. + * Available languages include: en-US (default), ja, de, zh-CN, zh-TW, ru, ko + * Some languages are more complete than others. + */ + // lang: "de", + + /** Configure the logging output */ + logging: { + /** Only console logging is currently supported */ + console: { + /** Level of logging to be recorded. Options are: + * fatal - only those errors which make the application unusable should be recorded + * error - record errors which are deemed fatal for a particular request + fatal errors + * warn - record problems which are non fatal + errors + fatal errors + * info - record information about the general running of the application + warn + error + fatal errors + * debug - record information which is more verbose than info + info + warn + error + fatal errors + * trace - record very detailed logging + debug + info + warn + error + fatal errors + * off - turn off all logging (doesn't affect metrics or audit) + */ + level: "info", + /** Whether or not to include metric events in the log output */ + metrics: false, + /** Whether or not to include audit events in the log output */ + audit: false + } + }, - // Context Storage - // The following property can be used to enable context storage. The configuration - // provided here will enable file-based context that flushes to disk every 30 seconds. - // Refer to the documentation for further options: https://nodered.org/docs/api/context/ - // + /** Context Storage + * The following property can be used to enable context storage. The configuration + * provided here will enable file-based context that flushes to disk every 30 seconds. + * Refer to the documentation for further options: https://nodered.org/docs/api/context/ + */ //contextStorage: { // default: { // module:"localfilesystem" // }, //}, - // The following property can be used to order the categories in the editor - // palette. If a node's category is not in the list, the category will get - // added to the end of the palette. - // If not set, the following default order is used: - //paletteCategories: ['subflows', 'common', 'function', 'network', 'sequence', 'parser', 'storage'], + /** `global.keys()` returns a list of all properties set in global context. + * This allows them to be displayed in the Context Sidebar within the editor. + * In some circumstances it is not desirable to expose them to the editor. The + * following property can be used to hide any property set in `functionGlobalContext` + * from being list by `global.keys()`. + * By default, the property is set to false to avoid accidental exposure of + * their values. Setting this to true will cause the keys to be listed. + */ + exportGlobalContextKeys: false, - // Configure the logging output - logging: { - // Only console logging is currently supported - console: { - // Level of logging to be recorded. Options are: - // fatal - only those errors which make the application unusable should be recorded - // error - record errors which are deemed fatal for a particular request + fatal errors - // warn - record problems which are non fatal + errors + fatal errors - // info - record information about the general running of the application + warn + error + fatal errors - // debug - record information which is more verbose than info + info + warn + error + fatal errors - // trace - record very detailed logging + debug + info + warn + error + fatal errors - // off - turn off all logging (doesn't affect metrics or audit) - level : 'info', - // Whether or not to include metric events in the log output - metrics: false, - // Whether or not to include audit events in the log output - audit : false - } + /** Configure how the runtime will handle external npm modules. + * This covers: + * - whether the editor will allow new node modules to be installed + * - whether nodes, such as the Function node are allowed to have their + * own dynamically configured dependencies. + * The allow/denyList options can be used to limit what modules the runtime + * will install/load. It can use '*' as a wildcard that matches anything. + */ + externalModules: { + // autoInstall: false, /** Whether the runtime will attempt to automatically install missing modules */ + // autoInstallRetry: 30, /** Interval, in seconds, between reinstall attempts */ + // palette: { /** Configuration for the Palette Manager */ + // allowInstall: true, /** Enable the Palette Manager in the editor */ + // allowUpdate: true, /** Allow modules to be updated in the Palette Manager */ + // allowUpload: true, /** Allow module tgz files to be uploaded and installed */ + // allowList: ['*'], + // denyList: [], + // allowUpdateList: ['*'], + // denyUpdateList: [] + // }, + // modules: { /** Configuration for node-specified modules */ + // allowInstall: true, + // allowList: [], + // denyList: [] + // } }, - // Customising the editor + + /******************************************************************************* + * Editor Settings + * - disableEditor + * - editorTheme + ******************************************************************************/ + + /** The following property can be used to disable the editor. The admin API + * is not affected by this option. To disable both the editor and the admin + * API, use either the httpRoot or httpAdminRoot properties + */ + //disableEditor: false, + + /** Customising the editor + * See https://nodered.org/docs/user-guide/runtime/configuration#editor-themes + * for all available options. + */ editorTheme: { - page : { - title : 'Project Alice Node-RED', - favicon: '~/ProjectAlice/core/interface/static/favicon.ico', - css : [ - '/home/pi/.node-red/node_modules/@node-red-contrib-themes/midnight-red/theme.css', - '/home/pi/.node-red/node_modules/@node-red-contrib-themes/midnight-red/scrollbars.css' - ] + /** The following property can be used to set a custom theme for the editor. + * See https://github.com/node-red-contrib-themes/theme-collection for + * a collection of themes to chose from. + */ + //theme: "", + theme: "midnight-red", + page: { + title: 'Project Alice Node-RED', + favicon: '~/ProjectAlice/core/interface/static/favicon.ico' + }, + palette: { + /** The following property can be used to order the categories in the editor + * palette. If a node's category is not in the list, the category will get + * added to the end of the palette. + * If not set, the following default order is used: + */ + //categories: ['subflows', 'common', 'function', 'network', 'sequence', 'parser', 'storage'], }, projects: { - enabled: false + /** To enable the Projects feature, set this value to true */ + enabled: false, + workflow: { + /** Set the default projects workflow mode. + * - manual - you must manually commit changes + * - auto - changes are automatically committed + * This can be overridden per-user from the 'Git config' + * section of 'User Settings' within the editor + */ + mode: "manual" + } + }, + codeEditor: { + /** Select the text editor component used by the editor. + * Defaults to "ace", but can be set to "ace" or "monaco" + */ + lib: "ace", + options: { + /** The follow options only apply if the editor is set to "monaco" + * + * theme - must match the file name of a theme in + * packages/node_modules/@node-red/editor-client/src/vendor/monaco/dist/theme + * e.g. "tomorrow-night", "upstream-sunburst", "github", "my-theme" + */ + theme: "vs", + /** other overrides can be set e.g. fontSize, fontFamily, fontLigatures etc. + * for the full list, see https://microsoft.github.io/monaco-editor/api/interfaces/monaco.editor.istandaloneeditorconstructionoptions.html + */ + //fontSize: 14, + //fontFamily: "Cascadia Code, Fira Code, Consolas, 'Courier New', monospace", + //fontLigatures: true, + } } - } -}; + }, + + /******************************************************************************* + * Node Settings + * - fileWorkingDirectory + * - functionGlobalContext + * - functionExternalModules + * - nodeMessageBufferMaxLength + * - ui (for use with Node-RED Dashboard) + * - debugUseColors + * - debugMaxLength + * - execMaxBufferSize + * - httpRequestTimeout + * - mqttReconnectTime + * - serialReconnectTime + * - socketReconnectTime + * - socketTimeout + * - tcpMsgQueueSize + * - inboundWebSocketTimeout + * - tlsConfigDisableLocalFiles + * - webSocketNodeVerifyClient + ******************************************************************************/ + + /** The working directory to handle relative file paths from within the File nodes + * defaults to the working directory of the Node-RED process. + */ + //fileWorkingDirectory: "", + + /** Allow the Function node to load additional npm modules directly */ + functionExternalModules: true, + + /** The following property can be used to set predefined values in Global Context. + * This allows extra node modules to be made available with in Function node. + * For example, the following: + * functionGlobalContext: { os:require('os') } + * will allow the `os` module to be accessed in a Function node using: + * global.get("os") + */ + functionGlobalContext: { + // os:require('os'), + }, + + /** The maximum number of messages nodes will buffer internally as part of their + * operation. This applies across a range of nodes that operate on message sequences. + * defaults to no limit. A value of 0 also means no limit is applied. + */ + //nodeMessageBufferMaxLength: 0, + + /** If you installed the optional node-red-dashboard you can set it's path + * relative to httpNodeRoot + * Other optional properties include + * readOnly:{boolean}, + * middleware:{function or array}, (req,res,next) - http middleware + * ioMiddleware:{function or array}, (socket,next) - socket.io middleware + */ + //ui: { path: "ui" }, + + /** Colourise the console output of the debug node */ + //debugUseColors: true, + + /** The maximum length, in characters, of any message sent to the debug sidebar tab */ + debugMaxLength: 1000, + + /** Maximum buffer size for the exec node. Defaults to 10Mb */ + //execMaxBufferSize: 10000000, + + /** Timeout in milliseconds for HTTP request connections. Defaults to 120s */ + //httpRequestTimeout: 120000, + + /** Retry time in milliseconds for MQTT connections */ + mqttReconnectTime: 15000, + + /** Retry time in milliseconds for Serial port connections */ + serialReconnectTime: 15000, + + /** Retry time in milliseconds for TCP socket connections */ + //socketReconnectTime: 10000, + + /** Timeout in milliseconds for TCP server socket connections. Defaults to no timeout */ + //socketTimeout: 120000, + + /** Maximum number of messages to wait in queue while attempting to connect to TCP socket + * defaults to 1000 + */ + //tcpMsgQueueSize: 2000, + + /** Timeout in milliseconds for inbound WebSocket connections that do not + * match any configured node. Defaults to 5000 + */ + //inboundWebSocketTimeout: 5000, + + /** To disable the option for using local files for storing keys and + * certificates in the TLS configuration node, set this to true. + */ + //tlsConfigDisableLocalFiles: true, + + /** The following property can be used to verify websocket connection attempts. + * This allows, for example, the HTTP request headers to be checked to ensure + * they include valid authentication information. + */ + //webSocketNodeVerifyClient: function(info) { + // /** 'info' has three properties: + // * - origin : the value in the Origin header + // * - req : the HTTP request + // * - secure : true if req.connection.authorized or req.connection.encrypted is set + // * + // * The function should return true if the connection should be accepted, false otherwise. + // * + // * Alternatively, if this function is defined to accept a second argument, callback, + // * it can be used to verify the client asynchronously. + // * The callback takes three arguments: + // * - result : boolean, whether to accept the connection or not + // * - code : if result is false, the HTTP error status to return + // * - reason: if result is false, the HTTP reason string to return + // */ + //}, +} From 500dd03be323b2ec7f63580b9b25893e49f1ed2e Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Mon, 20 Dec 2021 13:59:48 +0100 Subject: [PATCH 126/129] Fixed skill update flow --- core/base/SkillManager.py | 20 ++++++++++---------- core/device/DeviceManager.py | 4 ++-- core/webApi/model/SkillsApi.py | 12 ++++++++---- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/core/base/SkillManager.py b/core/base/SkillManager.py index d59a7f90e..47fbff7dc 100644 --- a/core/base/SkillManager.py +++ b/core/base/SkillManager.py @@ -20,17 +20,16 @@ import importlib import json +import requests import shutil import traceback +from AliceGit import Exceptions as GitErrors +from AliceGit.Exceptions import NotGitRepository, PathNotFoundException +from AliceGit.Git import Repository from contextlib import suppress from pathlib import Path from typing import Any, Dict, List, Optional, Union -import requests - -from AliceGit import Exceptions as GitErrors -from AliceGit.Exceptions import NotGitRepository, PathNotFoundException -from AliceGit.Git import Repository from core.ProjectAliceExceptions import AccessLevelTooLow, GithubNotFound, SkillInstanceFailed, SkillNotConditionCompliant, SkillStartDelayed, SkillStartingFailed from core.base.SuperManager import SuperManager from core.base.model import Intent @@ -744,9 +743,9 @@ def checkSkillConditions(self, installer: dict = None, checkOnly=False) -> Union def updateSkills(self, skills: Union[str, List[str]], withSkillRestart: bool = True): """ - Updates skills to latest available version for this Alice version + Updates skills to the latest available version for this Alice version :param skills: - :param withSkillRestart: Whether or not to start the skill after updating it + :param withSkillRestart: Whether to start the skill after updating it :return: """ self._busyInstalling.set() @@ -775,12 +774,12 @@ def updateSkills(self, skills: Union[str, List[str]], withSkillRestart: bool = T ) self.initSkills(onlyInit=skillName, reload=True) - if skillName in self.activeSkills: - self.logInfo(f'Updated skill **{skillName}** to version **{self.activeSkills[skillName].version}**') - if withSkillRestart: self.startSkill(skillName=skillName) + if skillName in self.activeSkills: + self.logInfo(f'Updated skill **{skillName}** to version **{self.activeSkills[skillName].version}**') + self._busyInstalling.clear() @@ -793,6 +792,7 @@ def stopSkill(self, skillName: str) -> Optional[AliceSkill]: skill = None if skillName in self._activeSkills: skill = self._activeSkills.pop(skillName, None) + self.deactivatedSkills[skillName] = skill skill.onStop() self.broadcast( method=constants.EVENT_SKILL_STOPPED, diff --git a/core/device/DeviceManager.py b/core/device/DeviceManager.py index 8498e6e43..d4d7103bb 100644 --- a/core/device/DeviceManager.py +++ b/core/device/DeviceManager.py @@ -618,7 +618,7 @@ def deviceConnecting(self, uid: str) -> Optional[Device]: device.connected = True self.broadcast(method=constants.EVENT_DEVICE_CONNECTING, exceptions=[self.name], propagateToSkills=True) self.MqttManager.publish(constants.TOPIC_DEVICE_UPDATED, payload={'device': device.toDict()}) - self.logInfo(f'Device named **{device.displayName}** in the {self.LocationManager.getLocation(locId=device.parentLocation).name} connected') + self.logInfo(f'Device named **{device.displayName}** ({device.uid}) in the {self.LocationManager.getLocation(locId=device.parentLocation).name} connected') self._heartbeats[uid] = round(time.time()) if not self._heartbeatsCheckTimer: @@ -641,7 +641,7 @@ def deviceDisconnecting(self, uid: str): return if device.connected: - self.logInfo(f'Device with uid **{uid}** disconnected') + self.logInfo(f'Device named **{device.displayName}** ({device.uid}) in the {self.LocationManager.getLocation(locId=device.parentLocation).name} disconnected') device.connected = False self.broadcast(method=constants.EVENT_DEVICE_DISCONNECTING, exceptions=[self.name], propagateToSkills=True) self.MqttManager.publish(constants.TOPIC_DEVICE_UPDATED, payload={'device': device.toDict()}) diff --git a/core/webApi/model/SkillsApi.py b/core/webApi/model/SkillsApi.py index ba1e47f92..bd5aa4ddf 100644 --- a/core/webApi/model/SkillsApi.py +++ b/core/webApi/model/SkillsApi.py @@ -19,14 +19,14 @@ import json +from AliceGit.Exceptions import AlreadyGitRepository, GithubRepoNotFound, GithubUserNotFound, NotGitRepository +from AliceGit.Git import Repository +from AliceGit.Github import Github from contextlib import suppress from flask import Response, jsonify, request from flask_classful import route from pathlib import Path -from AliceGit.Exceptions import AlreadyGitRepository, GithubRepoNotFound, GithubUserNotFound, NotGitRepository -from AliceGit.Git import Repository -from AliceGit.Github import Github from core.util.Decorators import ApiAuthenticated from core.webApi.model.Api import Api @@ -242,7 +242,11 @@ def checkUpdate(self, skillName: str) -> Response: if skillName not in self.SkillManager.allSkills: return self.skillNotFound() - return jsonify(success=self.SkillManager.checkForSkillUpdates(skillToCheck=skillName)) + update = self.SkillManager.checkForSkillUpdates(skillToCheck=skillName) + if update: + self.SkillManager.updateSkills(skills=update) + + return jsonify(success=True) @route('//isDirty/', methods=['GET']) From 95d8dc350c8d29ffae190b6b8381d6dd5217447f Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Tue, 21 Dec 2021 11:36:17 +0100 Subject: [PATCH 127/129] Fix reloadTTS not existing --- configTemplate.json | 10 +++++----- core/device/DeviceManager.py | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/configTemplate.json b/configTemplate.json index a92f7f34d..b865657b3 100644 --- a/configTemplate.json +++ b/configTemplate.json @@ -443,7 +443,7 @@ "IBM Watson" : "watson" }, "description" : "The Tts to use. Can't use an online Tts if you have set keepTTSOffline!", - "onUpdate" : "reloadTTS", + "onUpdate" : "reloadTTSManager", "category" : "tts" }, "ttsFallback" : { @@ -458,7 +458,7 @@ "IBM Watson" : "watson" }, "description" : "The Tts to use in case the default Tts becomes unavailable.", - "onUpdate" : "reloadTTS", + "onUpdate" : "reloadTTSManager", "category" : "tts" }, "ttsLanguage" : { @@ -466,7 +466,7 @@ "dataType" : "string", "isSensitive" : false, "description" : "Language for the Tts to use", - "onUpdate" : "reloadTTS", + "onUpdate" : "reloadTTSManager", "category" : "tts" }, "ttsType" : { @@ -478,7 +478,7 @@ "female" ], "description" : "Choose the voice gender you want", - "onUpdate" : "reloadTTS", + "onUpdate" : "reloadTTSManager", "category" : "tts" }, "ttsNeural" : { @@ -493,7 +493,7 @@ "dataType" : "string", "isSensitive" : false, "description" : "The voice the Tts should use", - "onUpdate" : "reloadTTS", + "onUpdate" : "reloadTTSManager", "category" : "tts", "display" : "hidden" }, diff --git a/core/device/DeviceManager.py b/core/device/DeviceManager.py index d4d7103bb..3a289aceb 100644 --- a/core/device/DeviceManager.py +++ b/core/device/DeviceManager.py @@ -618,7 +618,7 @@ def deviceConnecting(self, uid: str) -> Optional[Device]: device.connected = True self.broadcast(method=constants.EVENT_DEVICE_CONNECTING, exceptions=[self.name], propagateToSkills=True) self.MqttManager.publish(constants.TOPIC_DEVICE_UPDATED, payload={'device': device.toDict()}) - self.logInfo(f'Device named **{device.displayName}** ({device.uid}) in the {self.LocationManager.getLocation(locId=device.parentLocation).name} connected') + self.logInfo(f'Device named **{device.displayName}** ({device.uid}) in {self.LocationManager.getLocation(locId=device.parentLocation).name} connected') self._heartbeats[uid] = round(time.time()) if not self._heartbeatsCheckTimer: @@ -641,7 +641,7 @@ def deviceDisconnecting(self, uid: str): return if device.connected: - self.logInfo(f'Device named **{device.displayName}** ({device.uid}) in the {self.LocationManager.getLocation(locId=device.parentLocation).name} disconnected') + self.logInfo(f'Device named **{device.displayName}** ({device.uid}) in {self.LocationManager.getLocation(locId=device.parentLocation).name} disconnected') device.connected = False self.broadcast(method=constants.EVENT_DEVICE_DISCONNECTING, exceptions=[self.name], propagateToSkills=True) self.MqttManager.publish(constants.TOPIC_DEVICE_UPDATED, payload={'device': device.toDict()}) From bdb8f7a8357463a68d92bd25f4f007d2a73e5e0c Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Tue, 21 Dec 2021 12:07:02 +0100 Subject: [PATCH 128/129] Grammar and spelling cleanup --- core/base/ConfigManager.py | 11 +++++------ core/base/SkillManager.py | 18 +++++++++--------- core/base/SkillStoreManager.py | 5 ++--- core/commons/CommonsManager.py | 20 ++++++++++---------- core/device/DeviceManager.py | 16 ++++++++-------- core/device/model/Device.py | 6 +++--- core/nlu/model/SnipsNlu.py | 4 ++-- core/server/MqttManager.py | 11 +++++------ core/user/UserManager.py | 9 ++++----- core/util/Decorators.py | 17 ++++++++--------- core/webApi/model/SkillsApi.py | 8 ++++---- 11 files changed, 60 insertions(+), 65 deletions(-) diff --git a/core/base/ConfigManager.py b/core/base/ConfigManager.py index 656471a90..c24f6da4d 100644 --- a/core/base/ConfigManager.py +++ b/core/base/ConfigManager.py @@ -20,11 +20,10 @@ import json import logging import re +import sounddevice as sd from pathlib import Path from typing import Any, Dict, List, Optional, Tuple, Union -import sounddevice as sd - from core.ProjectAliceExceptions import ConfigurationUpdateFailed, VitalConfigMissing from core.base.SuperManager import SuperManager from core.base.model.Manager import Manager @@ -211,7 +210,7 @@ def updateAliceConfiguration(self, key: str, value: Any, dump: bool = True, doPr Updating a core config is sensitive, if the request comes from a skill. First check if the request came from a skill at anytime and if so ask permission to the user - :param doPreAndPostProcessing: If set to false, all pre and post processing won't be called + :param doPreAndPostProcessing: If set to false, all pre- and post-processing won't be called :param key: str :param value: str :param dump: bool If set to False, the configs won't be dumped to the json file @@ -538,7 +537,7 @@ def getAliceConfUpdatePreProcessing(self, confName: str) -> Optional[str]: def getAliceConfUpdatePostProcessing(self, confName: str) -> Optional[str]: - # Some config need some post processing if updated while Alice is running + # Some config need some post-processing if updated while Alice is running return self._aliceTemplateConfigurations.get(confName, dict()).get('onUpdate', None) @@ -577,7 +576,7 @@ def doConfigUpdatePreProcessing(self, function: str, value: Any) -> bool: def doConfigUpdatePostProcessing(self, functions: Union[str, set]): - # Call alice config post processing functions. This will call methods that are needed after a certain setting was + # Call alice config post-processing functions. This will call methods that are needed after a certain setting was # updated while Project Alice was running if isinstance(functions, str): @@ -757,7 +756,7 @@ def aliceTemplateConfigurations(self) -> Dict: @property def githubAuth(self) -> Tuple[str, str]: """ - Returns the users configured username and token for Github as a tuple + Returns the users configured username and token for GitHub as a tuple When one of the values is not supplied None is returned. :return: """ diff --git a/core/base/SkillManager.py b/core/base/SkillManager.py index 47fbff7dc..484f88584 100644 --- a/core/base/SkillManager.py +++ b/core/base/SkillManager.py @@ -23,13 +23,13 @@ import requests import shutil import traceback -from AliceGit import Exceptions as GitErrors -from AliceGit.Exceptions import NotGitRepository, PathNotFoundException -from AliceGit.Git import Repository from contextlib import suppress from pathlib import Path from typing import Any, Dict, List, Optional, Union +from AliceGit import Exceptions as GitErrors +from AliceGit.Exceptions import NotGitRepository, PathNotFoundException +from AliceGit.Git import Repository from core.ProjectAliceExceptions import AccessLevelTooLow, GithubNotFound, SkillInstanceFailed, SkillNotConditionCompliant, SkillStartDelayed, SkillStartingFailed from core.base.SuperManager import SuperManager from core.base.model import Intent @@ -142,7 +142,7 @@ def allSkills(self) -> Dict[str, Union[AliceSkill, FailedAliceSkill]]: @property def skillList(self) -> List: """ - Returns all skills present in the skill directory. These might not be inited, might have failed etc etc + Returns all skills present in the skill directory. These might not be inited, might have failed etc. :return: """ return self._skillList @@ -409,7 +409,7 @@ def getSkillRepository(self, skillName: str, directory: str = None) -> Optional[ def downloadSkills(self, skills: Union[str, List[str]]) -> Optional[Dict]: """ - Clones skills. Existence of the skill on line is checked + Clones skills. Existence of the skill online is checked :param skills: :return: Dict: a dict of created repositories """ @@ -500,7 +500,7 @@ def getSkillDirectory(self, skillName: str) -> Path: def getGitRemoteSourceUrl(self, skillName: str, doAuth: bool = True) -> str: """ - Returns the url for the skill name, taking into account if the user provided github auth + Returns the url for the skill name, taking into account if the user provided GitHub auth This does check if the remote exists and raises an exception in case it does not :param skillName: :param doAuth: Pull, clone, fetch, non oauth requests, aren't concerned by rate limit @@ -641,7 +641,7 @@ def instantiateSkill(self, skillName: str, skillResource: str = '', reload: bool def isSkillActive(self, skillName: str) -> bool: """ - Returns true or false depending if the skill is declared as active + Returns true or false depending on if the skill is declared as active :param skillName: :return: """ @@ -660,9 +660,9 @@ def isSkillActive(self, skillName: str) -> bool: def checkSkillConditions(self, installer: dict = None, checkOnly=False) -> Union[bool, List[Dict[str, str]]]: """ - Checks if the given skill is compliant to it's conditions + Checks if the given skill is compliant to its conditions :param installer: - :param checkOnly: Do not perform any other action (download other skill, etc) but checking conditions + :param checkOnly: Do not perform any other action (download other skill, etc.) but checking conditions :return: """ conditions = { diff --git a/core/base/SkillStoreManager.py b/core/base/SkillStoreManager.py index 9e7f19e3c..d47d3d856 100644 --- a/core/base/SkillStoreManager.py +++ b/core/base/SkillStoreManager.py @@ -18,11 +18,10 @@ # Last modified: 2021.04.13 at 12:56:46 CEST import difflib +import requests from random import shuffle from typing import Optional, Tuple -import requests - from core.ProjectAliceExceptions import GithubNotFound from core.base.model.Manager import Manager from core.base.model.Version import Version @@ -106,7 +105,7 @@ def _getSkillUpdateVersion(self, skillName: str) -> Optional[Tuple[Version, str] Get the highest skill version number a user can install. This is based on the user preferences, depending on the current Alice version and the user's selected update channel for skills - In case nothing is found, DO NOT FALLBACK TO MASTER + In case nothing is found, DO NOT FALL BACK TO MASTER :param skillName: The skill to look for :return: tuple (Version object, tag string) diff --git a/core/commons/CommonsManager.py b/core/commons/CommonsManager.py index 25aac4771..542dae4c5 100644 --- a/core/commons/CommonsManager.py +++ b/core/commons/CommonsManager.py @@ -17,10 +17,15 @@ # # Last modified: 2021.04.13 at 12:56:46 CEST +from collections import defaultdict +from ctypes import * + import hashlib import inspect +import jinja2 import json import random +import requests import socket import sqlite3 import string @@ -28,19 +33,14 @@ import tempfile import time import uuid -from collections import defaultdict from contextlib import contextmanager, suppress -from ctypes import * from datetime import datetime +from googletrans import Translator +from paho.mqtt.client import MQTTMessage from pathlib import Path from typing import Any, Union from uuid import UUID -import jinja2 -import requests -from googletrans import Translator -from paho.mqtt.client import MQTTMessage - import core.base.SuperManager as SuperManager import core.commons.model.Slot as slotModel from core.base.model.Manager import Manager @@ -317,8 +317,8 @@ def isWritable(path: Path): def translate(self, text: Union[str, list], destLang: str = None, srcLang: str = None) -> Union[str, list]: """ Translates a string or a list of strings into a different language using - google translator. Especially helpful when a api is only available in one - language, but the skill should support other languages aswell. + google translator. Especially helpful when an api is only available in one + language, but the skill should support other languages as well. :param text: string or list of strings to translate :param destLang: language to translate to (ISO639-1 code) @@ -428,7 +428,7 @@ def dictFromRow(row: sqlite3.Row) -> dict: def getGithubAuth(self) -> tuple: """ - Returns the users configured username and token for github as a tuple + Returns the users configured username and token for GitHub as a tuple When one of the values is not supplied None is returned. :return: """ diff --git a/core/device/DeviceManager.py b/core/device/DeviceManager.py index 3a289aceb..edf968196 100644 --- a/core/device/DeviceManager.py +++ b/core/device/DeviceManager.py @@ -288,7 +288,7 @@ def getDevicesWithAbilities(self, abilities: List[DeviceAbility], connectedOnly: """ Returns a list of Device instances having AT LEAST the provided abilities :param abilities: A list of DeviceAbility the device must have - :param connectedOnly: Whether or not to return non connected devices + :param connectedOnly: Whether to return non-connected devices :return: A list of Device instances """ ret = list() @@ -305,7 +305,7 @@ def getDevicesWithAbilities(self, abilities: List[DeviceAbility], connectedOnly: def getDevicesByType(self, deviceType: DeviceType, connectedOnly: bool = True) -> List[Device]: """ Returns a list of devices that are of the given type from the given skill - :param connectedOnly: Whether or not to return non connected devices + :param connectedOnly: Whether to return non-connected devices :param deviceType: DeviceType :return: list of Device instances """ @@ -324,10 +324,10 @@ def getDevicesByType(self, deviceType: DeviceType, connectedOnly: bool = True) - def getDevicesByLocation(self, locationId: int, deviceType: DeviceType = None, abilities: List[DeviceAbility] = None, connectedOnly: bool = True) -> List[Device]: """ Returns a list of devices fitting the locationId and the optional arguments - :param locationId: the location Id, only mandatory argument + :param locationId: the location id, only mandatory argument :param deviceType: The device type that it must be :param abilities: The abilities the device has to have - :param connectedOnly: Whether or not to return non connected devices + :param connectedOnly: Whether to return non-connected devices :return: list of Device instances """ return self._filterDevices(locationId=locationId, deviceType=deviceType, abilities=abilities, connectedOnly=connectedOnly) @@ -336,10 +336,10 @@ def getDevicesByLocation(self, locationId: int, deviceType: DeviceType = None, a def getDevicesBySkill(self, skillName: str, deviceType: DeviceType = None, abilities: List[DeviceAbility] = None, connectedOnly: bool = True) -> List[Device]: """ Returns a list of devices fitting the skill name and the optional arguments - :param skillName: the location Id, only mandatory argument + :param skillName: the location id, only mandatory argument :param deviceType: The device type that it must be :param abilities: The abilities the device has to have - :param connectedOnly: Whether or not to return non connected devices + :param connectedOnly: Whether to return non-connected devices :return: list of Device instances """ return self._filterDevices(skillName=skillName, deviceType=deviceType, abilities=abilities, connectedOnly=connectedOnly) @@ -348,11 +348,11 @@ def getDevicesBySkill(self, skillName: str, deviceType: DeviceType = None, abili def _filterDevices(self, locationId: int = None, skillName: str = None, deviceType: DeviceType = None, abilities: List[DeviceAbility] = None, connectedOnly: bool = True) -> List[Device]: """ Returns a list of devices fitting the optional arguments - :param locationId: the location Id, only mandatory argument + :param locationId: the location id, only mandatory argument :param skillName: the skill the device belongs to :param deviceType: The device type that it must be :param abilities: The abilities the device has to have - :param connectedOnly: Whether or not to return non connected devices + :param connectedOnly: Whether to return non-connected devices :return: list of Device instances """ diff --git a/core/device/model/Device.py b/core/device/model/Device.py index 577648dcc..85b286c1a 100644 --- a/core/device/model/Device.py +++ b/core/device/model/Device.py @@ -38,7 +38,7 @@ class Device(ProjectAliceObject): def __init__(self, data: Union[sqlite3.Row, Dict]): # settings: Holds the device display settings, such as x and y position, size and that stuff - # deviceParams: Holds the device non declared params, such as sound muted and so on. These are not controlled values that can be completely random + # deviceParams: Holds the device non-declared params, such as sound muted and so on. These are not controlled values that can be completely random # deviceConfigs: Holds the device configurations, provided by the device's .config.template. These configs and values are controlled and cannot be random at all! super().__init__() @@ -332,7 +332,7 @@ def pairingDone(self, uid: str): def onDeviceUIReply(self, data: dict): """ - When a device is clicked in the UI, it receive an optional reply directive that calls this method + When a device is clicked in the UI, it receives an optional reply directive that calls this method :param data: :return: """ @@ -347,7 +347,7 @@ def settings(self) -> dict: @property def connected(self) -> bool: """ - Returns whether or not this device is connected + Returns whether this device is connected :return: """ return self._connected diff --git a/core/nlu/model/SnipsNlu.py b/core/nlu/model/SnipsNlu.py index 14fbcd1b1..1870badf0 100644 --- a/core/nlu/model/SnipsNlu.py +++ b/core/nlu/model/SnipsNlu.py @@ -250,8 +250,8 @@ def trainingFailed(self): def getLanguage(self) -> str: """ - get the language that should be used for the training. - Currently only portuguese needs a special handling + Get the language that should be used for the training. + Currently, only portuguese needs a special handling :return: """ lang = self.LanguageManager.activeLanguage diff --git a/core/server/MqttManager.py b/core/server/MqttManager.py index 21bcd7dc7..4af1f174e 100644 --- a/core/server/MqttManager.py +++ b/core/server/MqttManager.py @@ -18,6 +18,8 @@ # Last modified: 2021.07.28 at 16:07:59 CEST import json +import paho.mqtt.client as mqtt +import paho.mqtt.publish as publish import random import re import traceback @@ -25,9 +27,6 @@ from pathlib import Path from typing import List, Union -import paho.mqtt.client as mqtt -import paho.mqtt.publish as publish - from core.base.model.Intent import Intent from core.base.model.Manager import Manager from core.commons import constants @@ -680,7 +679,7 @@ def ask(self, text: str, deviceUid: str = None, intentFilter: list = None, custo Initiates a new session by asking something and waiting on user answer :param probabilityThreshold: The override threshold for the user's answer to this question :param currentDialogState: a str representing a state in the dialog, useful for multi-turn dialogs - :param canBeEnqueued: whether or not this can be played later if the dialog manager is busy + :param canBeEnqueued: whether this can be played later if the dialog manager is busy :param text: str The text to speak :param deviceUid: str Where to ask :param intentFilter: array Filter to force user intents @@ -754,12 +753,12 @@ def continueDialog(self, sessionId: str, text: str, customData: dict = None, int """ Continues a dialog :param probabilityThreshold: The probability threshold override for the user's answer to this coming conversation round - :param currentDialogState: a str representing a state in the dialog, usefull for multiturn dialogs + :param currentDialogState: a str representing a state in the dialog, useful for multi-turn dialogs :param sessionId: int session id to continue :param customData: json str :param text: str text spoken :param intentFilter: array intent filter for user randomTalk - :param slot: Optional String, requires intentFilter to contain a single value - If set, the dialogue engine will not run the the intent classification on the user response and go straight to slot filling, assuming the intent is the one passed in the intentFilter, and searching the value of the given slot + :param slot: Optional String, requires intentFilter to contain a single value - If set, the dialogue engine will not run the intent classification on the user response and go straight to slot filling, assuming the intent is the one passed in the intentFilter, and searching the value of the given slot """ jsonDict = { diff --git a/core/user/UserManager.py b/core/user/UserManager.py index c0baa6fbd..f26c44a8d 100644 --- a/core/user/UserManager.py +++ b/core/user/UserManager.py @@ -17,12 +17,11 @@ # # Last modified: 2021.04.13 at 12:56:47 CEST -from pathlib import Path -from time import time -from typing import Any, Dict, Optional - import bcrypt import jwt +from pathlib import Path +from time import time +from typing import Any, Dict, Optional, Union from core.base.model.Manager import Manager from core.commons import constants @@ -260,7 +259,7 @@ def updateUserState(self, name, state): self._loadUsers() - def hasAccessLevel(self, user: str, requiredAccessLevel: int) -> bool: + def hasAccessLevel(self, user: str, requiredAccessLevel: Union[AccessLevel, int]) -> bool: if isinstance(requiredAccessLevel, AccessLevel): requiredAccessLevel = requiredAccessLevel.value diff --git a/core/util/Decorators.py b/core/util/Decorators.py index 9ea91eb97..c77b18ea3 100644 --- a/core/util/Decorators.py +++ b/core/util/Decorators.py @@ -21,9 +21,8 @@ import functools import warnings -from typing import Any, Callable, Optional, Tuple, Union - from flask import jsonify, request +from typing import Any, Callable, Optional, Tuple, Union from core.base.SuperManager import SuperManager from core.base.model.Intent import Intent @@ -118,19 +117,19 @@ def Online(func: Callable = None, text: str = 'offline', offlineHandler: Callabl """ (return a) decorator to mark a function that requires ethernet. - This decorator can be used (with or or without parameters) to define + This decorator can be used (with or without parameters) to define a function that requires ethernet. In the Default mode without arguments shown - in the example it will either execute whats in the function or when alice is + in the example it will either execute what's in the function or when alice is offline ends the dialog with a random offline answer. Using the parameters: @online(text=) - a own text can be used when being offline aswell and using the parameters: + An own text can be used when being offline as well and using the parameters: @online(offlineHandler=) - a own offline handler can be called, which is helpful when not only endDialog has to be called, - but some other cleanup is required aswell + An own offline handler can be called, which is helpful when not only endDialog has to be called, + but some other cleanup is required as well When there is no named argument 'session' of type DialogSession in the arguments of the decorated function, - the decorator will return the text instead. This behaviour can be enforced aswell using: + the decorator will return the text instead. This behaviour can be enforced as well using: @online(returnText=True) :param catchOnly: If catch only, do not raise anything @@ -207,7 +206,7 @@ def wrapper(*args, **kwargs): def KnownUser(func: Callable = None): # NOSONAR """ Checks if the session is started by a know user or not. This is important for skills that are security - sensitive and you need to make sure Alice is not talking to someone unknown + sensitive, and you need to make sure Alice is not talking to someone unknown :param func: :return: """ diff --git a/core/webApi/model/SkillsApi.py b/core/webApi/model/SkillsApi.py index bd5aa4ddf..00165ae5e 100644 --- a/core/webApi/model/SkillsApi.py +++ b/core/webApi/model/SkillsApi.py @@ -19,14 +19,14 @@ import json -from AliceGit.Exceptions import AlreadyGitRepository, GithubRepoNotFound, GithubUserNotFound, NotGitRepository -from AliceGit.Git import Repository -from AliceGit.Github import Github from contextlib import suppress from flask import Response, jsonify, request from flask_classful import route from pathlib import Path +from AliceGit.Exceptions import AlreadyGitRepository, GithubRepoNotFound, GithubUserNotFound, NotGitRepository +from AliceGit.Git import Repository +from AliceGit.Github import Github from core.util.Decorators import ApiAuthenticated from core.webApi.model.Api import Api @@ -341,7 +341,7 @@ def upload(self, skillName: str) -> Response: @ApiAuthenticated def getGitStatus(self, skillName: str) -> Response: """ - returns a list containing the public and private github URL of that skill. + returns a list containing the public and private GitHub URL of that skill. The repository does not have to exist yet! The current status of the repository is included as well Currently possible status: True/False From aa2811ade3ca27abb24719e22b1eaaa800d643bc Mon Sep 17 00:00:00 2001 From: Psychokiller1888 Date: Tue, 21 Dec 2021 13:36:22 +0100 Subject: [PATCH 129/129] update refs --- core/webui/public | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/webui/public b/core/webui/public index 58931f08e..0e01f0a88 160000 --- a/core/webui/public +++ b/core/webui/public @@ -1 +1 @@ -Subproject commit 58931f08e37c7d61524ba150f96eb02ce9faaabd +Subproject commit 0e01f0a888ad9b1d155e9bbc69aa335da7e0b616