From 97146204c6a9e1170f86df9286095d61edfcf419 Mon Sep 17 00:00:00 2001 From: Dzmitry Aliaksandrau Date: Wed, 9 Aug 2023 18:56:22 +0300 Subject: [PATCH 1/2] Add upload encrypted files functionality --- .../human_protocol_sdk/storage.py | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/packages/sdk/python/human-protocol-sdk/human_protocol_sdk/storage.py b/packages/sdk/python/human-protocol-sdk/human_protocol_sdk/storage.py index 363e751c6a..7fddeb981d 100644 --- a/packages/sdk/python/human-protocol-sdk/human_protocol_sdk/storage.py +++ b/packages/sdk/python/human-protocol-sdk/human_protocol_sdk/storage.py @@ -175,31 +175,36 @@ def download_files(self, files: List[str], bucket: str) -> List[bytes]: raise StorageClientError(str(e)) return result_files - def upload_files(self, files: List[object], bucket: str) -> List[str]: + def upload_files(self, files: List[dict], bucket: str) -> List[dict]: """ Uploads a list of files to the specified S3-compatible bucket. Args: - files (list[object]): A list of files to upload. + files (list[dict]): A list of files to upload. bucket (str): The name of the S3-compatible bucket to upload to. Returns: - list: A list of keys assigned to the uploaded files. + list: List of dict with key, url, hash fields Raises: StorageClientError: If an error occurs while uploading the files. """ result_files = [] for file in files: - try: - artifact = json.dumps(file, sort_keys=True) - except Exception as e: - LOG.error("Can't extract the json from the object") - raise e + if "file" in file and "key" in file and "hash" in file: + data = file["file"] + hash = file["hash"] + key = file["key"] + else: + try: + artifact = json.dumps(file, sort_keys=True) + except Exception as e: + LOG.error("Can't extract the json from the object") + raise e + data = artifact.encode("utf-8") + hash = hashlib.sha1(data).hexdigest() + key = f"s3{hash}.json" - data = artifact.encode("utf-8") - hash = hashlib.sha1(data).hexdigest() - key = f"s3{hash}.json" url = ( f"{'https' if self.secure else 'http'}://{self.endpoint}/{bucket}/{key}" ) From 503d0710d1ab6e0ac5ca477fb1a4d670b6d869a3 Mon Sep 17 00:00:00 2001 From: Dzmitry Aliaksandrau Date: Mon, 21 Aug 2023 16:42:32 +0300 Subject: [PATCH 2/2] EscrowConfig can skip manifest URL validation --- .../sdk/python/human-protocol-sdk/Pipfile | 2 +- .../human_protocol_sdk/escrow.py | 6 ++-- .../test/human_protocol_sdk/test_escrow.py | 29 +++++++++++++++++++ .../test/human_protocol_sdk/test_storage.py | 29 +++++++++++++++++++ 4 files changed, 63 insertions(+), 3 deletions(-) diff --git a/packages/sdk/python/human-protocol-sdk/Pipfile b/packages/sdk/python/human-protocol-sdk/Pipfile index 89d46af1f9..fe05827cc3 100644 --- a/packages/sdk/python/human-protocol-sdk/Pipfile +++ b/packages/sdk/python/human-protocol-sdk/Pipfile @@ -12,7 +12,7 @@ setuptools-pipfile = "*" [packages] cryptography = "*" minio = "*" -validators = "*" +validators = "==0.20.0" web3 = "==6.8.*" aiohttp = "<4.0.0" # broken freeze in one of dependencies pgpy = "*" diff --git a/packages/sdk/python/human-protocol-sdk/human_protocol_sdk/escrow.py b/packages/sdk/python/human-protocol-sdk/human_protocol_sdk/escrow.py index 5b5bc64888..45017fb996 100644 --- a/packages/sdk/python/human-protocol-sdk/human_protocol_sdk/escrow.py +++ b/packages/sdk/python/human-protocol-sdk/human_protocol_sdk/escrow.py @@ -48,6 +48,7 @@ def __init__( reputation_oracle_fee: Decimal, manifest_url: str, hash: str, + skip_manifest_url_validation: bool = False, ): """ Initializes a Escrow instance. @@ -59,6 +60,7 @@ def __init__( reputation_oracle_fee (Decimal): Fee percentage of the Reputation Oracle manifest_url (str): Manifest file url hash (str): Manifest file hash + skip_manifest_url_validation (bool): Identify wether validate manifest_url """ if not Web3.is_address(recording_oracle_address): raise EscrowClientError( @@ -74,7 +76,7 @@ def __init__( raise EscrowClientError("Fee must be between 0 and 100") if recording_oracle_fee + reputation_oracle_fee > 100: raise EscrowClientError("Total fee must be less than 100") - if not URL(manifest_url): + if not URL(manifest_url) and not skip_manifest_url_validation: raise EscrowClientError(f"Invalid manifest URL: {manifest_url}") if not hash: raise EscrowClientError("Invalid empty manifest hash") @@ -187,7 +189,7 @@ def create_escrow(self, token_address: str, trusted_handlers: List[str]) -> str: ) return next( ( - self.factory_contract.events.Launched().processLog(log) + self.factory_contract.events.Launched().process_log(log) for log in transaction_receipt["logs"] if log["address"] == self.network["factory_address"] ), diff --git a/packages/sdk/python/human-protocol-sdk/test/human_protocol_sdk/test_escrow.py b/packages/sdk/python/human-protocol-sdk/test/human_protocol_sdk/test_escrow.py index 916a1578f1..5f3e00a73f 100644 --- a/packages/sdk/python/human-protocol-sdk/test/human_protocol_sdk/test_escrow.py +++ b/packages/sdk/python/human-protocol-sdk/test/human_protocol_sdk/test_escrow.py @@ -90,6 +90,35 @@ def test_escrow_config_valid_params(self): self.assertEqual(escrow_config.manifest_url, manifest_url) self.assertEqual(escrow_config.hash, hash) + def test_escrow_config_valid_params(self): + recording_oracle_address = "0x1234567890123456789012345678901234567890" + reputation_oracle_address = "0x1234567890123456789012345678901234567890" + recording_oracle_fee = 10 + reputation_oracle_fee = 10 + manifest_url = "http://test:6000" + hash = "test" + + escrow_config = EscrowConfig( + recording_oracle_address, + reputation_oracle_address, + recording_oracle_fee, + reputation_oracle_fee, + manifest_url, + hash, + True, + ) + + self.assertEqual( + escrow_config.recording_oracle_address, recording_oracle_address + ) + self.assertEqual( + escrow_config.reputation_oracle_address, reputation_oracle_address + ) + self.assertEqual(escrow_config.recording_oracle_fee, recording_oracle_fee) + self.assertEqual(escrow_config.reputation_oracle_fee, reputation_oracle_fee) + self.assertEqual(escrow_config.manifest_url, manifest_url) + self.assertEqual(escrow_config.hash, hash) + def test_escrow_config_invalid_address(self): invalid_address = "invalid_address" valid_address = "0x1234567890123456789012345678901234567890" diff --git a/packages/sdk/python/human-protocol-sdk/test/human_protocol_sdk/test_storage.py b/packages/sdk/python/human-protocol-sdk/test/human_protocol_sdk/test_storage.py index 7eb6ec2a75..2556ed673e 100644 --- a/packages/sdk/python/human-protocol-sdk/test/human_protocol_sdk/test_storage.py +++ b/packages/sdk/python/human-protocol-sdk/test/human_protocol_sdk/test_storage.py @@ -176,6 +176,35 @@ def test_upload_files(self): ) self.assertEqual(result[0]["hash"], hash) + def test_upload_encrypted_files(self): + encrypted_file = "encrypted file content" + hash = hashlib.sha1(json.dumps(encrypted_file).encode("utf-8")).hexdigest() + encrypted_file_key = f"s3{hash}" + file = { + "file": encrypted_file.encode("utf-8"), + "hash": hash, + "key": encrypted_file_key, + } + + self.client.client.stat_object = MagicMock( + side_effect=S3Error( + code="NoSuchKey", + message="Object does not exist", + resource="", + request_id="", + host_id="", + response="", + ) + ) + self.client.client.put_object = MagicMock() + result = self.client.upload_files(files=[file], bucket=self.bucket) + self.assertEqual(result[0]["key"], encrypted_file_key) + self.assertEqual( + result[0]["url"], + f"https://s3.us-west-2.amazonaws.com/my-bucket/{encrypted_file_key}", + ) + self.assertEqual(result[0]["hash"], hash) + def test_upload_files_exist(self): file3 = "file3 content" hash = hashlib.sha1(json.dumps("file3 content").encode("utf-8")).hexdigest()