Skip to content

Commit

Permalink
clean up testing code
Browse files Browse the repository at this point in the history
  • Loading branch information
Ken Yocum committed Apr 23, 2024
1 parent 02fcebe commit 44be800
Showing 1 changed file with 0 additions and 50 deletions.
50 changes: 0 additions & 50 deletions disdat/utility/asyncio_aws_s3.py
Original file line number Diff line number Diff line change
Expand Up @@ -606,53 +606,3 @@ def split_s3_url(s3_url):
if len(key) == 0:
key = None
return bucket, key


from os import path
from time import time

if __name__ == "__main__":
push_test_url = "ai-advisory/kentest"
pull_test_url = "disdat-cdo-prd/context/ai-advisory/objects"
# test_url = "ai-advisory/"
s3_url = f"s3://{pull_test_url}"

# test each of the concurrent calls

start = time()
# def ls_s3_url_keys(s3_url, is_object_directory=False):
ls_result = ls_s3_url_keys(s3_url, is_object_directory=True)
end = time()
print(f"ls_s3_url_keys: {ls_result}")
print(f"Elapsed: {end-start}")

ls_result = ls_result[:10]

if True:
start = time()
# bucket_key_file_tuples (tuple): (bucket:str, key:str, filename=None)
# gets = [("ai-advisory", f, f"./CRAP/{path.basename(f)}") for f in ls_result]
gets = [("disdat-cdo-prd", f, f"./CRAP/{path.basename(f)}") for f in ls_result]
get_result = get_s3_key_many(gets)
end = time()
print(f"get_s3_key_many: {get_result}")
print(f"Elapsed: {end-start}")

if True:
start = time()
# bucket_key_file_tuples (list[tuple]): (filename, s3_path)
puts = [
(f, f"s3://{push_test_url}/junk/{path.basename(f)}") for f in get_result
]
put_results = put_s3_key_many(puts)
end = time()
print(f"put_s3_key_many: {put_results}")
print(f"Elapsed: {end-start}")

start = time()
# s3_urls(list(str)): list of s3_urls we will remove
deletes = [f"s3://{push_test_url}/junk/{path.basename(f)}" for f in put_results]
del_results = delete_s3_dir_many(deletes)
end = time()
print(f"delete_s3_key_many: {deletes} {del_results}")
print(f"Elapsed: {end-start}")

0 comments on commit 44be800

Please sign in to comment.