diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 6b7e9f987..4f257ae6d 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -11,20 +11,20 @@ on: jobs: changelog: - uses: obervinov/_templates/.github/workflows/changelog.yaml@v1.2.5 + uses: obervinov/_templates/.github/workflows/changelog.yaml@v1.2.6 pylint: - uses: obervinov/_templates/.github/workflows/pylint.yaml@v1.2.5 + uses: obervinov/_templates/.github/workflows/pylint.yaml@v1.2.6 pytest: - uses: obervinov/_templates/.github/workflows/pytest-with-vault.yaml@v1.2.5 + uses: obervinov/_templates/.github/workflows/pytest-with-vault.yaml@v1.2.6 pyproject: - uses: obervinov/_templates/.github/workflows/pyproject.yaml@v1.2.5 + uses: obervinov/_templates/.github/workflows/pyproject.yaml@v1.2.6 pr: - uses: obervinov/_templates/.github/workflows/pr.yaml@v1.2.5 + uses: obervinov/_templates/.github/workflows/pr.yaml@v1.2.6 build-pr-image: - uses: obervinov/_templates/.github/workflows/docker.yaml@v1.2.5 + uses: obervinov/_templates/.github/workflows/docker.yaml@v1.2.6 needs: [changelog, pylint, pytest, pyproject] diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 8add5c523..ea47241a7 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -10,8 +10,8 @@ on: jobs: create-release: - uses: obervinov/_templates/.github/workflows/release.yaml@v1.2.5 + uses: obervinov/_templates/.github/workflows/release.yaml@v1.2.6 # milestone: - # uses: obervinov/_templates/.github/workflows/milestone.yaml@v1.2.5 + # uses: obervinov/_templates/.github/workflows/milestone.yaml@v1.2.6 # needs: [create-release] diff --git a/CHANGELOG.md b/CHANGELOG.md index 57016a8a0..cf7e5e694 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,27 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## v2.1.6 - 2024-06-22 +### What's Changed +**Full Changelog**: https://github.com/obervinov/pyinstabot-downloader/compare/v2.1.5...v2.1.6 by @obervinov in https://github.com/obervinov/pyinstabot-downloader/pull/70 +#### 💥 Breaking Changes +* remove unused database `environment` attribute (permanent path in the Vault: `configurations/database`) +* remove unused environment variable `PROJECT_ENVIRONMENT` +* the automatic queue verification mechanism has been removed. Instead of this method, added functionality to update the queue processing time via a message to the bot +* change the structure of the table `messages`: add a new column `state` and `updated_at`, rename column `timestamp` to `created_at` +#### 🐛 Bug Fixes +* [Bug: Add a limit on the number of items in the queue to be displayed in the `Your last activity` message](https://github.com/obervinov/pyinstabot-downloader/issues/69) +* [Bug: Bot can't update status message](https://github.com/obervinov/pyinstabot-downloader/issues/62) +* [Bug: Crashes the queue processing thread when a post from the queue no longer exists in the content sources](https://github.com/obervinov/pyinstabot-downloader/issues/67) +* [Bug: queue rescheduler does not always work correctly](https://github.com/obervinov/pyinstabot-downloader/issues/64) +* [Bug: For some reason the bot tried to edit a message with the same content in the message](https://github.com/obervinov/pyinstabot-downloader/issues/65) +* Removed duplicates in rights checking +* Small refactoring code +#### 🚀 Features +* Bump dependency versions for modules and workflows +* Add button for rescheduling the queue + + ## v2.1.5 - 2024-05-29 ### What's Changed **Full Changelog**: https://github.com/obervinov/pyinstabot-downloader/compare/v2.1.4...v2.1.5 by @obervinov in https://github.com/obervinov/pyinstabot-downloader/pull/61 diff --git a/README.md b/README.md index fb4282e12..db8cf8134 100644 --- a/README.md +++ b/README.md @@ -30,9 +30,9 @@ This project is a telegram bot that allows you to create backups of content from

**Main functions** -- a backup copy of a `specific post` by link -- a backup copy of `list of posts` by links -- the ability to backup to the `Mega` or `Dropbox` clouds +- a backup copy of a __specific post__ by link +- a backup copy of __list of posts__ by links +- the ability to backup to the __Mega__ or __Dropbox__ clouds **Preview of the bot in action**

@@ -60,7 +60,6 @@ This project is a telegram bot that allows you to create backups of content from ## Environment variables | Variable | Description | Default value | | ------------- | ------------- | ------------- | -| `PROJECT_ENVIRONMENT` | The environment in which the project is running (`dev`, `prod`) | `dev` | | `LOGGER_LEVEL` | [The logging level of the logging module](https://docs.python.org/3/library/logging.html#logging-levels) | `INFO` | | `BOT_NAME` | The name of the bot, used to determine the unique mount point in the vault | `pyinstabot-downloader` | | `MESSAGES_CONFIG` | The path to the message template file | `src/configs/messages.json` | @@ -84,8 +83,8 @@ This project is a telegram bot that allows you to create backups of content from ### Bot configuration source and supported parameters All bot configuration is stored in the `Vault Secrets`
-_except for the part of the configuration that configures the connection to `Vault` and external modules_
-- `configuration/database-`: database connection parameters (depends on the environment variable) +_except for the part of the configuration that configures the connection to `Vault`_
+- `configuration/database`: database connection parameters ```json { "database": "pyinstabot-downloader", @@ -224,4 +223,4 @@ docker compose -f docker-compose.yml up -d ## GitHub Actions | Name | Version | | ------------------------ | ----------- | -| GitHub Actions Templates | [v1.2.2](https://github.com/obervinov/_templates/tree/v1.2.2) | +| GitHub Actions Templates | [v1.2.6](https://github.com/obervinov/_templates/tree/v1.2.6) | diff --git a/docker-compose.yml b/docker-compose.yml index bb8b2624b..6c9fe821e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -50,7 +50,6 @@ services: container_name: pyinstabot-downloader restart: always environment: - - PROJECT_ENVIRONMENT=dev - TELEGRAM_BOT_NAME=pyinstabot-downloader - VAULT_APPROLE_ID=${VAULT_APPROLE_ID} - VAULT_APPROLE_SECRETID=${VAULT_APPROLE_SECRETID} diff --git a/poetry.lock b/poetry.lock index c2ed044a0..a31b4358d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2,13 +2,13 @@ [[package]] name = "certifi" -version = "2024.2.2" +version = "2024.6.2" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" files = [ - {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, - {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, + {file = "certifi-2024.6.2-py3-none-any.whl", hash = "sha256:ddc6c8ce995e6987e7faf5e3f1b02b302836a0e5d98ece18392cb1a36c72ad56"}, + {file = "certifi-2024.6.2.tar.gz", hash = "sha256:3cd43f1c6fa7dedc5899d69d3ad0398fd018ad1a17fba83ddaf78aa46c747516"}, ] [[package]] @@ -187,43 +187,43 @@ files = [ [[package]] name = "cryptography" -version = "42.0.7" +version = "42.0.8" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false python-versions = ">=3.7" files = [ - {file = "cryptography-42.0.7-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:a987f840718078212fdf4504d0fd4c6effe34a7e4740378e59d47696e8dfb477"}, - {file = "cryptography-42.0.7-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:bd13b5e9b543532453de08bcdc3cc7cebec6f9883e886fd20a92f26940fd3e7a"}, - {file = "cryptography-42.0.7-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a79165431551042cc9d1d90e6145d5d0d3ab0f2d66326c201d9b0e7f5bf43604"}, - {file = "cryptography-42.0.7-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a47787a5e3649008a1102d3df55424e86606c9bae6fb77ac59afe06d234605f8"}, - {file = "cryptography-42.0.7-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:02c0eee2d7133bdbbc5e24441258d5d2244beb31da5ed19fbb80315f4bbbff55"}, - {file = "cryptography-42.0.7-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:5e44507bf8d14b36b8389b226665d597bc0f18ea035d75b4e53c7b1ea84583cc"}, - {file = "cryptography-42.0.7-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:7f8b25fa616d8b846aef64b15c606bb0828dbc35faf90566eb139aa9cff67af2"}, - {file = "cryptography-42.0.7-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:93a3209f6bb2b33e725ed08ee0991b92976dfdcf4e8b38646540674fc7508e13"}, - {file = "cryptography-42.0.7-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:e6b8f1881dac458c34778d0a424ae5769de30544fc678eac51c1c8bb2183e9da"}, - {file = "cryptography-42.0.7-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:3de9a45d3b2b7d8088c3fbf1ed4395dfeff79d07842217b38df14ef09ce1d8d7"}, - {file = "cryptography-42.0.7-cp37-abi3-win32.whl", hash = "sha256:789caea816c6704f63f6241a519bfa347f72fbd67ba28d04636b7c6b7da94b0b"}, - {file = "cryptography-42.0.7-cp37-abi3-win_amd64.whl", hash = "sha256:8cb8ce7c3347fcf9446f201dc30e2d5a3c898d009126010cbd1f443f28b52678"}, - {file = "cryptography-42.0.7-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:a3a5ac8b56fe37f3125e5b72b61dcde43283e5370827f5233893d461b7360cd4"}, - {file = "cryptography-42.0.7-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:779245e13b9a6638df14641d029add5dc17edbef6ec915688f3acb9e720a5858"}, - {file = "cryptography-42.0.7-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0d563795db98b4cd57742a78a288cdbdc9daedac29f2239793071fe114f13785"}, - {file = "cryptography-42.0.7-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:31adb7d06fe4383226c3e963471f6837742889b3c4caa55aac20ad951bc8ffda"}, - {file = "cryptography-42.0.7-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:efd0bf5205240182e0f13bcaea41be4fdf5c22c5129fc7ced4a0282ac86998c9"}, - {file = "cryptography-42.0.7-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:a9bc127cdc4ecf87a5ea22a2556cab6c7eda2923f84e4f3cc588e8470ce4e42e"}, - {file = "cryptography-42.0.7-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:3577d029bc3f4827dd5bf8bf7710cac13527b470bbf1820a3f394adb38ed7d5f"}, - {file = "cryptography-42.0.7-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2e47577f9b18723fa294b0ea9a17d5e53a227867a0a4904a1a076d1646d45ca1"}, - {file = "cryptography-42.0.7-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:1a58839984d9cb34c855197043eaae2c187d930ca6d644612843b4fe8513c886"}, - {file = "cryptography-42.0.7-cp39-abi3-win32.whl", hash = "sha256:e6b79d0adb01aae87e8a44c2b64bc3f3fe59515280e00fb6d57a7267a2583cda"}, - {file = "cryptography-42.0.7-cp39-abi3-win_amd64.whl", hash = "sha256:16268d46086bb8ad5bf0a2b5544d8a9ed87a0e33f5e77dd3c3301e63d941a83b"}, - {file = "cryptography-42.0.7-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:2954fccea107026512b15afb4aa664a5640cd0af630e2ee3962f2602693f0c82"}, - {file = "cryptography-42.0.7-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:362e7197754c231797ec45ee081f3088a27a47c6c01eff2ac83f60f85a50fe60"}, - {file = "cryptography-42.0.7-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:4f698edacf9c9e0371112792558d2f705b5645076cc0aaae02f816a0171770fd"}, - {file = "cryptography-42.0.7-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:5482e789294854c28237bba77c4c83be698be740e31a3ae5e879ee5444166582"}, - {file = "cryptography-42.0.7-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e9b2a6309f14c0497f348d08a065d52f3020656f675819fc405fb63bbcd26562"}, - {file = "cryptography-42.0.7-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:d8e3098721b84392ee45af2dd554c947c32cc52f862b6a3ae982dbb90f577f14"}, - {file = "cryptography-42.0.7-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c65f96dad14f8528a447414125e1fc8feb2ad5a272b8f68477abbcc1ea7d94b9"}, - {file = "cryptography-42.0.7-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:36017400817987670037fbb0324d71489b6ead6231c9604f8fc1f7d008087c68"}, - {file = "cryptography-42.0.7.tar.gz", hash = "sha256:ecbfbc00bf55888edda9868a4cf927205de8499e7fabe6c050322298382953f2"}, + {file = "cryptography-42.0.8-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:81d8a521705787afe7a18d5bfb47ea9d9cc068206270aad0b96a725022e18d2e"}, + {file = "cryptography-42.0.8-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:961e61cefdcb06e0c6d7e3a1b22ebe8b996eb2bf50614e89384be54c48c6b63d"}, + {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e3ec3672626e1b9e55afd0df6d774ff0e953452886e06e0f1eb7eb0c832e8902"}, + {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e599b53fd95357d92304510fb7bda8523ed1f79ca98dce2f43c115950aa78801"}, + {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:5226d5d21ab681f432a9c1cf8b658c0cb02533eece706b155e5fbd8a0cdd3949"}, + {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:6b7c4f03ce01afd3b76cf69a5455caa9cfa3de8c8f493e0d3ab7d20611c8dae9"}, + {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:2346b911eb349ab547076f47f2e035fc8ff2c02380a7cbbf8d87114fa0f1c583"}, + {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:ad803773e9df0b92e0a817d22fd8a3675493f690b96130a5e24f1b8fabbea9c7"}, + {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2f66d9cd9147ee495a8374a45ca445819f8929a3efcd2e3df6428e46c3cbb10b"}, + {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:d45b940883a03e19e944456a558b67a41160e367a719833c53de6911cabba2b7"}, + {file = "cryptography-42.0.8-cp37-abi3-win32.whl", hash = "sha256:a0c5b2b0585b6af82d7e385f55a8bc568abff8923af147ee3c07bd8b42cda8b2"}, + {file = "cryptography-42.0.8-cp37-abi3-win_amd64.whl", hash = "sha256:57080dee41209e556a9a4ce60d229244f7a66ef52750f813bfbe18959770cfba"}, + {file = "cryptography-42.0.8-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:dea567d1b0e8bc5764b9443858b673b734100c2871dc93163f58c46a97a83d28"}, + {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c4783183f7cb757b73b2ae9aed6599b96338eb957233c58ca8f49a49cc32fd5e"}, + {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0608251135d0e03111152e41f0cc2392d1e74e35703960d4190b2e0f4ca9c70"}, + {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:dc0fdf6787f37b1c6b08e6dfc892d9d068b5bdb671198c72072828b80bd5fe4c"}, + {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:9c0c1716c8447ee7dbf08d6db2e5c41c688544c61074b54fc4564196f55c25a7"}, + {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:fff12c88a672ab9c9c1cf7b0c80e3ad9e2ebd9d828d955c126be4fd3e5578c9e"}, + {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:cafb92b2bc622cd1aa6a1dce4b93307792633f4c5fe1f46c6b97cf67073ec961"}, + {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:31f721658a29331f895a5a54e7e82075554ccfb8b163a18719d342f5ffe5ecb1"}, + {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b297f90c5723d04bcc8265fc2a0f86d4ea2e0f7ab4b6994459548d3a6b992a14"}, + {file = "cryptography-42.0.8-cp39-abi3-win32.whl", hash = "sha256:2f88d197e66c65be5e42cd72e5c18afbfae3f741742070e3019ac8f4ac57262c"}, + {file = "cryptography-42.0.8-cp39-abi3-win_amd64.whl", hash = "sha256:fa76fbb7596cc5839320000cdd5d0955313696d9511debab7ee7278fc8b5c84a"}, + {file = "cryptography-42.0.8-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:ba4f0a211697362e89ad822e667d8d340b4d8d55fae72cdd619389fb5912eefe"}, + {file = "cryptography-42.0.8-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:81884c4d096c272f00aeb1f11cf62ccd39763581645b0812e99a91505fa48e0c"}, + {file = "cryptography-42.0.8-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c9bb2ae11bfbab395bdd072985abde58ea9860ed84e59dbc0463a5d0159f5b71"}, + {file = "cryptography-42.0.8-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:7016f837e15b0a1c119d27ecd89b3515f01f90a8615ed5e9427e30d9cdbfed3d"}, + {file = "cryptography-42.0.8-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5a94eccb2a81a309806027e1670a358b99b8fe8bfe9f8d329f27d72c094dde8c"}, + {file = "cryptography-42.0.8-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:dec9b018df185f08483f294cae6ccac29e7a6e0678996587363dc352dc65c842"}, + {file = "cryptography-42.0.8-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:343728aac38decfdeecf55ecab3264b015be68fc2816ca800db649607aeee648"}, + {file = "cryptography-42.0.8-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:013629ae70b40af70c9a7a5db40abe5d9054e6f4380e50ce769947b73bf3caad"}, + {file = "cryptography-42.0.8.tar.gz", hash = "sha256:8d09d05439ce7baa8e9e95b07ec5b6c886f548deb7e0f69ef25f64b3bce842f2"}, ] [package.dependencies] @@ -241,21 +241,20 @@ test-randomorder = ["pytest-randomly"] [[package]] name = "dropbox" -version = "12.0.0" +version = "12.0.2" description = "Official Dropbox API Client" optional = false python-versions = "*" files = [ - {file = "dropbox-12.0.0-py2-none-any.whl", hash = "sha256:ebe32dd98f39c3e3361a517deb4242a812f738f7f2f397c18fa5e0cee8bcde37"}, - {file = "dropbox-12.0.0-py3-none-any.whl", hash = "sha256:65e264bf2fe76fc779a3254fd90d4f9a7b4d8725f08f09c538cffd0f3ef6c3d5"}, - {file = "dropbox-12.0.0.tar.gz", hash = "sha256:facd1af160c246fcceff4a9d2b9732c46db03a95618937a2d8fae9412ebfe60b"}, + {file = "dropbox-12.0.2-py2-none-any.whl", hash = "sha256:4b8207a9f4afd33726ec886c0d223f4bbc42fe649b87718690a24704f5e24c0c"}, + {file = "dropbox-12.0.2-py3-none-any.whl", hash = "sha256:c5b7e9c2668adb6b12dcecd84342565dc50f7d35ab6a748d155cb79040979d1c"}, + {file = "dropbox-12.0.2.tar.gz", hash = "sha256:50057fd5ad5fcf047f542dfc6747a896e7ef982f1b5f8500daf51f3abd609962"}, ] [package.dependencies] -requests = "<2.30" +requests = ">=2.16.2" six = ">=1.12.0" stone = ">=2,<3.3.3" -urllib3 = "<2" [[package]] name = "emoji" @@ -486,24 +485,24 @@ test = ["pytest", "pytest-cov"] [[package]] name = "more-itertools" -version = "10.2.0" +version = "10.3.0" description = "More routines for operating on iterables, beyond itertools" optional = false python-versions = ">=3.8" files = [ - {file = "more-itertools-10.2.0.tar.gz", hash = "sha256:8fccb480c43d3e99a00087634c06dd02b0d50fbf088b380de5a41a015ec239e1"}, - {file = "more_itertools-10.2.0-py3-none-any.whl", hash = "sha256:686b06abe565edfab151cb8fd385a05651e1fdf8f0a14191e4439283421f8684"}, + {file = "more-itertools-10.3.0.tar.gz", hash = "sha256:e5d93ef411224fbcef366a6e8ddc4c5781bc6359d43412a65dd5964e46111463"}, + {file = "more_itertools-10.3.0-py3-none-any.whl", hash = "sha256:ea6a02e24a9161e51faad17a8782b92a0df82c12c1c8886fec7f0c3fa1a1b320"}, ] [[package]] name = "packaging" -version = "24.0" +version = "24.1" description = "Core utilities for Python packages" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "packaging-24.0-py3-none-any.whl", hash = "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5"}, - {file = "packaging-24.0.tar.gz", hash = "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9"}, + {file = "packaging-24.1-py3-none-any.whl", hash = "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"}, + {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"}, ] [[package]] @@ -678,13 +677,13 @@ files = [ [[package]] name = "pytelegrambotapi" -version = "4.18.1" +version = "4.19.1" description = "Python Telegram bot api." optional = false python-versions = ">=3.8" files = [ - {file = "pytelegrambotapi-4.18.1-py3-none-any.whl", hash = "sha256:07951383c5831b1f810edaf01e06ee95a40253486d725780cf88de15aa0893ce"}, - {file = "pytelegrambotapi-4.18.1.tar.gz", hash = "sha256:6bf79a726624441e84724d933312edb3138ad22906ffea2fee09ee6846236ac0"}, + {file = "pytelegrambotapi-4.19.1-py3-none-any.whl", hash = "sha256:22b0835f06a79eea93cc2c29079e4bb10f706b282ae10a3e537557b4679b12f2"}, + {file = "pytelegrambotapi-4.19.1.tar.gz", hash = "sha256:cd5e5188a49f50a5c1fb1d8d195a852a05ae3525d21f26fbe8365cb408ee1342"}, ] [package.dependencies] @@ -704,13 +703,13 @@ watchdog = ["watchdog"] [[package]] name = "pytest" -version = "8.2.1" +version = "8.2.2" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.8" files = [ - {file = "pytest-8.2.1-py3-none-any.whl", hash = "sha256:faccc5d332b8c3719f40283d0d44aa5cf101cec36f88cde9ed8f2bc0538612b1"}, - {file = "pytest-8.2.1.tar.gz", hash = "sha256:5046e5b46d8e4cac199c373041f26be56fdb81eb4e67dc11d4e10811fc3408fd"}, + {file = "pytest-8.2.2-py3-none-any.whl", hash = "sha256:c434598117762e2bd304e526244f67bf66bbd7b5d6cf22138be51ff661980343"}, + {file = "pytest-8.2.2.tar.gz", hash = "sha256:de4bb8104e201939ccdc688b27a89a7be2079b22e2bd2b07f806b6ba71117977"}, ] [package.dependencies] @@ -751,20 +750,20 @@ files = [ [[package]] name = "requests" -version = "2.29.0" +version = "2.32.3" description = "Python HTTP for Humans." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "requests-2.29.0-py3-none-any.whl", hash = "sha256:e8f3c9be120d3333921d213eef078af392fba3933ab7ed2d1cba3b56f2568c3b"}, - {file = "requests-2.29.0.tar.gz", hash = "sha256:f2e34a75f4749019bb0e3effb66683630e4ffeaf75819fb51bebef1bf5aef059"}, + {file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"}, + {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"}, ] [package.dependencies] certifi = ">=2017.4.17" charset-normalizer = ">=2,<4" idna = ">=2.5,<4" -urllib3 = ">=1.21.1,<1.27" +urllib3 = ">=1.21.1,<3" [package.extras] socks = ["PySocks (>=1.5.6,!=1.5.7)"] @@ -864,30 +863,31 @@ files = [ [[package]] name = "typing-extensions" -version = "4.12.0" +version = "4.12.2" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" files = [ - {file = "typing_extensions-4.12.0-py3-none-any.whl", hash = "sha256:b349c66bea9016ac22978d800cfff206d5f9816951f12a7d0ec5578b0a819594"}, - {file = "typing_extensions-4.12.0.tar.gz", hash = "sha256:8cbcdc8606ebcb0d95453ad7dc5065e6237b6aa230a31e81d0f440c30fed5fd8"}, + {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, + {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, ] [[package]] name = "urllib3" -version = "1.26.18" +version = "2.2.1" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +python-versions = ">=3.8" files = [ - {file = "urllib3-1.26.18-py2.py3-none-any.whl", hash = "sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07"}, - {file = "urllib3-1.26.18.tar.gz", hash = "sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0"}, + {file = "urllib3-2.2.1-py3-none-any.whl", hash = "sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d"}, + {file = "urllib3-2.2.1.tar.gz", hash = "sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19"}, ] [package.extras] -brotli = ["brotli (==1.0.9)", "brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] -secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] -socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] [[package]] name = "users" @@ -932,18 +932,18 @@ resolved_reference = "54a312b747ad84c391a837c5bddaed7a021c9d76" [[package]] name = "zipp" -version = "3.19.0" +version = "3.19.2" description = "Backport of pathlib-compatible object wrapper for zip files" optional = false python-versions = ">=3.8" files = [ - {file = "zipp-3.19.0-py3-none-any.whl", hash = "sha256:96dc6ad62f1441bcaccef23b274ec471518daf4fbbc580341204936a5a3dddec"}, - {file = "zipp-3.19.0.tar.gz", hash = "sha256:952df858fb3164426c976d9338d3961e8e8b3758e2e059e0f754b8c4262625ee"}, + {file = "zipp-3.19.2-py3-none-any.whl", hash = "sha256:f091755f667055f2d02b32c53771a7a6c8b47e1fdbc4b72a8b9072b3eef8015c"}, + {file = "zipp-3.19.2.tar.gz", hash = "sha256:bf1dcf6450f873a13e952a29504887c89e6de7506209e5b1bcc3460135d4de19"}, ] [package.extras] -docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -testing = ["big-O", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy", "pytest-ruff (>=0.2.1)"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy", "pytest-ruff (>=0.2.1)"] [metadata] lock-version = "2.0" diff --git a/pyproject.toml b/pyproject.toml index b3bb60e87..5e0d2d411 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pyinstabot-downloader" -version = "2.1.5" +version = "2.1.6" description = "This project is a Telegram bot that allows you to backup post content from your Instagram profile to Dropbox or Mega clouds." authors = ["Bervinov Oleg "] maintainers = ["Bervinov Oleg "] diff --git a/src/bot.py b/src/bot.py index 6267b0ffc..0411c032b 100644 --- a/src/bot.py +++ b/src/bot.py @@ -11,11 +11,10 @@ from mock import MagicMock from logger import log -from telegram import TelegramBot -from telegram import exceptions as TelegramExceptions +from telegram import TelegramBot, exceptions as TelegramExceptions from users import Users from vault import VaultClient -from configs.constants import (PROJECT_ENVIRONMENT, TELEGRAM_BOT_NAME, ROLES_MAP, QUEUE_FREQUENCY, STATUSES_MESSAGE_FREQUENCY) +from configs.constants import (TELEGRAM_BOT_NAME, ROLES_MAP, QUEUE_FREQUENCY, STATUSES_MESSAGE_FREQUENCY) from modules.database import DatabaseClient from modules.exceptions import FailedMessagesStatusUpdater from modules.tools import get_hash @@ -62,7 +61,7 @@ uploader.run_transfers.return_value = 'completed' # Client for communication with the database -database = DatabaseClient(vault=vault, environment=PROJECT_ENVIRONMENT) +database = DatabaseClient(vault=vault) # START HANDLERS BLOCK ############################################################################################################## @@ -79,10 +78,11 @@ def start_command(message: telegram.telegram_types.Message = None) -> None: None """ if users.user_access_check(message.chat.id).get('access', None) == users.user_status_allow: - log.info('[Bot]: Processing `start` command for user %s...', message.chat.id) + log.info('[Bot]: Processing "start" command for user %s...', message.chat.id) # Add user to the database - _ = database.add_user(user_id=message.chat.id, chat_id=message.chat.id) + response = database.add_user(user_id=message.chat.id, chat_id=message.chat.id) + log.info('[Bot]: user %s added to the database: %s', message.chat.id, response) # Main message reply_markup = telegram.create_inline_markup(ROLES_MAP.keys()) @@ -120,14 +120,32 @@ def bot_callback_query_handler(call: telegram.callback_query = None) -> None: Returns: None """ - log.info('[Bot]: Processing button `%s` for user %s...', call.data, call.message.chat.id) + log.info('[Bot]: Processing button "%s" for user %s...', call.data, call.message.chat.id) if users.user_access_check(call.message.chat.id, ROLES_MAP[call.data]).get('permissions', None) == users.user_status_allow: if call.data == "Post": - button_post(call=call) + help_message = telegram.send_styled_message( + chat_id=call.message.chat.id, + messages_template={'alias': 'help_for_post'} + ) + bot.register_next_step_handler(call.message, process_one_post, help_message) + elif call.data == "Posts List": - button_posts_list(call=call) + help_message = telegram.send_styled_message( + chat_id=call.message.chat.id, + messages_template={'alias': 'help_for_posts_list'} + ) + bot.register_next_step_handler(call.message, process_list_posts, help_message) + + elif call.data == "Reschedule Queue": + help_message = telegram.send_styled_message( + chat_id=call.message.chat.id, + messages_template={'alias': 'help_for_reschedule_queue'} + ) + bot.register_next_step_handler(call.message, reschedule_queue, help_message) + else: - log.error('[Bot]: Handler for button %s not found', call.data) + log.error('[Bot]: Handler for button "%s" not found', call.data) + else: telegram.send_styled_message( chat_id=call.message.chat.id, @@ -151,11 +169,8 @@ def unknown_command(message: telegram.telegram_types.Message = None) -> None: None """ if users.user_access_check(message.chat.id).get('access', None) == users.user_status_allow: - log.error('[Bot]: Invalid command `%s` from user %s', message.text, message.chat.id) - telegram.send_styled_message( - chat_id=message.chat.id, - messages_template={'alias': 'unknown_command'} - ) + log.error('[Bot]: Invalid command "%s" from user %s', message.text, message.chat.id) + telegram.send_styled_message(chat_id=message.chat.id, messages_template={'alias': 'unknown_command'}) else: telegram.send_styled_message( chat_id=message.chat.id, @@ -167,71 +182,6 @@ def unknown_command(message: telegram.telegram_types.Message = None) -> None: # END HANDLERS BLOCK ############################################################################################################## -# START BUTTONS BLOCK ############################################################################################################# -# Inline button handler for Post -def button_post(call: telegram.callback_query = None) -> None: - """ - The handler for the Post button. - - Args: - call (telegram.callback_query): The callback query object. - - Returns: - None - """ - user = users.user_access_check(call.message.chat.id, ROLES_MAP['Post']) - if user.get('permissions', None) == users.user_status_allow: - help_message = telegram.send_styled_message( - chat_id=call.message.chat.id, - messages_template={'alias': 'help_for_post'} - ) - bot.register_next_step_handler( - call.message, - process_one_post, - help_message - ) - else: - telegram.send_styled_message( - chat_id=call.message.chat.id, - messages_template={ - 'alias': 'permission_denied_message', - 'kwargs': {'username': call.message.chat.username, 'userid': call.message.chat.id} - } - ) - - -# Inline button handler for Posts List -def button_posts_list(call: telegram.callback_query = None) -> None: - """ - The handler for the Posts List button. - - Args: - call (telegram.callback_query): The callback query object. - - Returns: - None - """ - user = users.user_access_check(call.message.chat.id, ROLES_MAP['Posts List']) - if user.get('permissions', None) == users.user_status_allow: - help_message = telegram.send_styled_message( - chat_id=call.message.chat.id, - messages_template={'alias': 'help_for_posts_list'} - ) - bot.register_next_step_handler( - call.message, - process_list_posts, - help_message - ) - else: - telegram.send_styled_message( - chat_id=call.message.chat.id, - messages_template={ - 'alias': 'permission_denied_message', - 'kwargs': {'username': call.message.chat.username, 'userid': call.message.chat.id} - } - ) - - # START BLOCK ADDITIONAL FUNCTIONS ###################################################################################################### def update_status_message(user_id: str = None) -> None: """ @@ -244,79 +194,88 @@ def update_status_message(user_id: str = None) -> None: None """ try: - chat_id = user_id - exist_status_message = database.get_considered_message(message_type='status_message', chat_id=chat_id) + exist_status_message = database.get_considered_message(message_type='status_message', chat_id=user_id) message_statuses = get_user_messages(user_id=user_id) diff_between_messages = False + if exist_status_message: - # check difference between messages content - if exist_status_message[3] != get_hash(message_statuses): - diff_between_messages = True + # checking competition of status_message update by another thread + if exist_status_message[5] == 'updating': + while exist_status_message[5] == 'updating': + time.sleep(1) + exist_status_message = database.get_considered_message(message_type='status_message', chat_id=user_id) + else: + database.keep_message( + message_id=exist_status_message[0], + chat_id=exist_status_message[1], + message_type='status_message', + message_content=message_statuses, + state='updating' + ) + + diff_between_messages = exist_status_message[4] != get_hash(message_statuses) # if message already sended and expiring (because bot can edit message only first 48 hours) - # automatic renew message every 23 hours - if exist_status_message[2] < datetime.now() - timedelta(hours=23): - if exist_status_message[2] < datetime.now() - timedelta(hours=48): - log.warning('[Bot]: `status_message` for user %s old more than 48 hours, can not delete them', user_id) - else: + # automatic renew message every 24 hours + if exist_status_message[2] < datetime.now() - timedelta(hours=24): + if exist_status_message[2] > datetime.now() - timedelta(hours=48): _ = bot.delete_message( - chat_id=chat_id, + chat_id=user_id, message_id=exist_status_message[0] ) status_message = telegram.send_styled_message( - chat_id=chat_id, - messages_template={ - 'alias': 'message_statuses', - 'kwargs': message_statuses - } + chat_id=user_id, + messages_template={'alias': 'message_statuses', 'kwargs': message_statuses} ) - bot.pin_chat_message(status_message.chat.id, status_message.id) database.keep_message( message_id=status_message.message_id, chat_id=status_message.chat.id, message_type='status_message', - message_content=message_statuses + message_content=message_statuses, + state='updated', + recreated=True ) log.info('[Bot]: `status_message` for user %s has been renewed', user_id) + elif message_statuses is not None and diff_between_messages: log.info( '[Bot]: `status_message` for user %s is outdated, updating %s -> %s...', user_id, exist_status_message[3], get_hash(message_statuses) ) editable_message = telegram.send_styled_message( - chat_id=chat_id, - messages_template={ - 'alias': 'message_statuses', - 'kwargs': message_statuses - }, + chat_id=user_id, + messages_template={'alias': 'message_statuses', 'kwargs': message_statuses}, editable_message_id=exist_status_message[0] ) database.keep_message( message_id=editable_message.message_id, chat_id=editable_message.chat.id, message_type='status_message', - message_content=message_statuses + message_content=message_statuses, + state='updated' ) log.info('[Bot]: `status_message` for user %s has been updated', user_id) + elif not diff_between_messages: log.info('[Bot]: `status_message` for user %s is actual', user_id) + database.keep_message( + message_id=exist_status_message[0], + chat_id=exist_status_message[1], + message_type='status_message', + message_content=message_statuses, + state='updated' + ) + else: status_message = telegram.send_styled_message( - chat_id=chat_id, - messages_template={ - 'alias': 'message_statuses', - 'kwargs': message_statuses - } - ) - bot.pin_chat_message( - chat_id=status_message.chat.id, - message_id=status_message.id + chat_id=user_id, + messages_template={'alias': 'message_statuses', 'kwargs': message_statuses} ) database.keep_message( message_id=status_message.message_id, chat_id=status_message.chat.id, message_type='status_message', - message_content=message_statuses + message_content=message_statuses, ) log.info('[Bot]: `status_message` for user %s has been created', user_id) except TypeError as exception: @@ -351,7 +310,7 @@ def get_user_messages(user_id: str = None) -> dict: if queue_dict is not None: sorted_data = sorted(queue_dict[user_id], key=lambda x: x['scheduled_time'], reverse=False) for item in sorted_data: - queue_string = queue_string + f"+ {item['post_id']}: will be started {item['scheduled_time']}\n" + queue_string = queue_string + f"+ {item['post_id']}: scheduled for {item['scheduled_time']}\n" else: queue_string = 'queue is empty' @@ -434,25 +393,17 @@ def process_one_post( # Check if the message is unique if database.check_message_uniqueness(data['post_id'], data['user_id']): - _ = database.add_message_to_queue(data) + status = database.add_message_to_queue(data) update_status_message(user_id=message.chat.id) - log.info('[Bot]: post %s from user %s has been added to the queue', message.text, message.chat.id) + log.info('[Bot]: %s from user %s', status, message.chat.id) else: - log.info('[Bot]: post %s from user %s already in queue or processed', data['post_id'], message.chat.id) + log.info('[Bot]: post %s from user %s already exist in the database', data['post_id'], message.chat.id) # If it is not a list of posts - delete users message if mode == 'single': telegram.delete_message(message.chat.id, message.id) if help_message is not None: telegram.delete_message(message.chat.id, help_message.id) - else: - telegram.send_styled_message( - chat_id=message.chat.id, - messages_template={ - 'alias': 'reject_message', - 'kwargs': {'username': message.chat.username, 'userid': message.chat.id} - } - ) def process_list_posts( @@ -481,14 +432,46 @@ def process_list_posts( telegram.delete_message(message.chat.id, message.id) if help_message is not None: telegram.delete_message(message.chat.id, help_message.id) - else: - telegram.send_styled_message( - chat_id=message.chat.id, - messages_template={ - 'alias': 'reject_message', - 'kwargs': {'username': message.chat.username, 'userid': message.chat.id} - } - ) + + +def reschedule_queue( + message: telegram.telegram_types.Message = None, + help_message: telegram.telegram_types.Message = None +) -> None: + """ + Manually reschedules the queue for the user. + + Args: + message (telegram.telegram_types.Message, optional): The message containing the list of post links. Defaults to None. + help_message (telegram.telegram_types.Message, optional): The help message to be deleted. Defaults to None. + + Returns: + None + """ + user = users.user_access_check(message.chat.id, ROLES_MAP['Reschedule Queue']) + if user.get('permissions', None) == users.user_status_allow: + for item in message.text.split('\n'): + item = item.split('=') + post_id = item[0].strip() + new_scheduled_time = datetime.strptime(item[1].strip(), '%Y-%m-%d %H:%M:%S.%f') + if ( + isinstance(post_id, str) and len(post_id) == 11 and + isinstance(new_scheduled_time, datetime) and new_scheduled_time > datetime.now() + ): + database.update_schedule_time_in_queue( + post_id=post_id, + user_id=message.chat.id, + scheduled_time=new_scheduled_time + ) + else: + telegram.send_styled_message( + chat_id=message.chat.id, + messages_template={'alias': 'wrong_reschedule_queue'} + ) + telegram.delete_message(message.chat.id, message.id) + if help_message is not None: + telegram.delete_message(message.chat.id, help_message.id) + update_status_message(user_id=message.chat.id) # END BLOCK PROCESSING FUNCTIONS #################################################################################################### @@ -503,10 +486,10 @@ def status_message_updater_thread() -> None: Returns: None """ - log.info('[Message-updater-thread]: started thread for `status_message` updater') + log.info('[Message-updater-thread]: started thread for "status_message" updater') while True: + time.sleep(STATUSES_MESSAGE_FREQUENCY) try: - time.sleep(STATUSES_MESSAGE_FREQUENCY) if database.get_users(): for user in database.get_users(): user_id = user[0] @@ -515,7 +498,7 @@ def status_message_updater_thread() -> None: except Exception as exception: exception_context = { 'call': threading.current_thread().name, - 'message': 'Failed to update the message with the status of received messages ', + 'message': 'Failed to update the message with the status of received messages', 'users': database.get_users(), 'user': user, 'exception': exception @@ -534,8 +517,6 @@ def queue_handler_thread() -> None: None """ log.info('[Queue-handler-thread]: started thread for queue handler') - # Verify scheduled timestamps in the users queue for cases when bot was down - database.verify_users_queue() while True: time.sleep(QUEUE_FREQUENCY) @@ -547,9 +528,9 @@ def queue_handler_thread() -> None: post_id = message[2] owner_id = message[4] - log.info('[Queue-handler-thread] starting handler for post url %s...', message[3]) + log.info('[Queue-handler-thread] starting handler for post %s...', message[2]) # download the contents of an instagram post to a temporary folder - if download_status != 'completed': + if download_status not in ['completed', 'not_found']: download_metadata = downloader.get_post_content(shortcode=post_id) owner_id = download_metadata['owner'] download_status = download_metadata['status'] @@ -560,8 +541,17 @@ def queue_handler_thread() -> None: upload_status=upload_status, post_owner=owner_id ) + # downloader couldn't find the post for some reason + if download_status == 'not_found': + database.update_message_state_in_queue( + post_id=post_id, + state='processed', + download_status=download_status, + upload_status=download_status, + post_owner=owner_id + ) # upload the received content to the destination storage - if upload_status != 'completed': + if upload_status != 'completed' and download_status == 'completed': upload_status = uploader.run_transfers(sub_directory=owner_id) database.update_message_state_in_queue( post_id=post_id, @@ -580,6 +570,8 @@ def queue_handler_thread() -> None: post_owner=owner_id ) log.info('[Queue-handler-thread] the post %s has been processed successfully', post_id) + elif download_status == 'not_found' and upload_status == 'not_found': + log.warning('[Queue-handler-thread] the post %s not found, message was marked as processed', post_id) else: log.warning( '[Queue-handler-thread] the post %s has not been processed yet (download: %s, uploader: %s)', diff --git a/src/configs/constants.py b/src/configs/constants.py index 090abc2ae..e8fd6e243 100644 --- a/src/configs/constants.py +++ b/src/configs/constants.py @@ -4,14 +4,14 @@ import os # environment variables -PROJECT_ENVIRONMENT = os.environ.get("PROJECT_ENVIRONMENT", "dev") TELEGRAM_BOT_NAME = os.environ.get('TELEGRAM_BOT_NAME', 'pyinstabot-downloader') # permissions roles and buttons mapping # 'button_title': 'role' ROLES_MAP = { 'Post': 'post', - 'Posts List': 'posts_list' + 'Posts List': 'posts_list', + 'Reschedule Queue': 'reschedule_queue', } # Queue handler diff --git a/src/configs/databases.json b/src/configs/databases.json index fbdefc789..4ec57a45c 100644 --- a/src/configs/databases.json +++ b/src/configs/databases.json @@ -54,10 +54,12 @@ "id SERIAL PRIMARY KEY, ", "message_id VARCHAR(255) NOT NULL, ", "chat_id VARCHAR(255) NOT NULL, ", - "timestamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, ", + "created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, ", + "updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, ", "message_type VARCHAR(255) NOT NULL , ", "producer VARCHAR(255) NOT NULL , ", - "message_content_hash VARCHAR(64) NOT NULL" + "message_content_hash VARCHAR(64) NOT NULL , ", + "state VARCHAR(255) NOT NULL DEFAULT 'added'" ] }, { diff --git a/src/configs/messages.json b/src/configs/messages.json index 22127be8c..22883efe8 100644 --- a/src/configs/messages.json +++ b/src/configs/messages.json @@ -5,7 +5,7 @@ "args": ["username", "userid", ":raised_hand:", ":unlocked:"] }, "message_statuses": { - "text": "{0} Your last activity:\n\n{1} processed (last 10)\n{2}\n{3} in queue\n{4}", + "text": "{0} Your last activity (last 10):\n\n{1} processed\n{2}\n{3} in queue\n{4}", "args": [":bar_chart:", ":check_mark_button:", "processed", ":shopping_cart:", "queue"] }, "reject_message": { @@ -24,6 +24,13 @@ "text": "{0} To get a backup copy of the list of posts, send links to posts in a list (each new link with a new message line). This message will be automatically split into a number of messages equal to the number of links and processed in the order of the queue.\n {1} Example:\nhttps://www.instagram.com/p/QwEr_tY1234\nhttps://www.instagram.com/p/QwEr_tY1235\nhttps://www.instagram.com/p/QwEr_tY1236", "args": [":information:", ":link:"] }, + "help_for_reschedule_queue": { + "text": "{0} To reschedule the processing of messages in the queue, simply send a list of messages with a modified processing time.\nfor example:\nq1wRty12345 = 2021-01-01 12:00:00\nq1wRty12346 = 2021-01-01 12:00:00\nq1wRty12347 = 2021-01-01 12:00:00", + "args": [":information:"] + }, + "wrong_reschedule_queue": { + "text": "{0} Incorrect format for rescheduling messages in the queue. Please check this conditions:\n1. Post-id is a string and its length is equal to 11 characters.\n2. Date-time format is correct and the date is in the future.\n3. The message is separated by a equal sign(=) and a space.\n4. Each new message is on a new line." + }, "unknown_command": { "text": "{0} Invalid button command. Please use inline keyboard.", "args": [":warning:"] diff --git a/src/migrations/0001_vault_historical_data.py b/src/migrations/0001_vault_historical_data.py index a15e41103..afb65a8de 100644 --- a/src/migrations/0001_vault_historical_data.py +++ b/src/migrations/0001_vault_historical_data.py @@ -67,5 +67,8 @@ def execute(obj): # Will be fixed after the issue https://github.com/obervinov/vault-package/issues/46 is resolved # pylint: disable=broad-exception-caught except Exception as migration_error: - print(f"{NAME}: Migration cannot be completed due to an error: {migration_error}") - print(f"{NAME}: Perhaps the history is empty or the Vault secrets path does not exist. It's not critical for the bot.") + print( + f"{NAME}: Migration cannot be completed due to an error: {migration_error}. " + "Perhaps the history is empty or the Vault secrets path does not exist and migration isn't unnecessary." + "It's not a critical error, so the migration will be skipped." + ) diff --git a/src/migrations/0002_messages_table.py b/src/migrations/0002_messages_table.py new file mode 100644 index 000000000..cd49bcf86 --- /dev/null +++ b/src/migrations/0002_messages_table.py @@ -0,0 +1,66 @@ +# pylint: disable=C0103,R0914 +""" +Add additional column 'created_at' and replace column 'timestamp' with 'updated_at' in the messages table. +https://github.com/obervinov/pyinstabot-downloader/issues/62 +""" +VERSION = '1.0' +NAME = '0002_messages_table' + + +def execute(obj): + """ + Add additional column 'created_at' and replace column 'timestamp' with 'updated_at' in the messages table. + + Args: + obj: An obj containing the database connection and cursor, as well as the Vault instance. + + Returns: + None + """ + # database settings + table_name = 'messages' + rename_columns = [('timestamp', 'updated_at')] + add_columns = [('created_at', 'TIMESTAMP', 'CURRENT_TIMESTAMP'), ('state', 'VARCHAR(255)', "'added'")] + print(f"{NAME}: Start migration for the {table_name} table: Rename columns {rename_columns}, Add columns {add_columns}...") + + # check if the table exists and has the necessary schema for execute the migration + # check table + obj.cursor.execute("SELECT * FROM information_schema.tables WHERE table_schema = 'public' AND table_name = %s;", (table_name,)) + table = obj.cursor.fetchone() + + # check columns in the table + obj.cursor.execute("SELECT column_name FROM information_schema.columns WHERE table_schema = 'public' AND table_name = %s;", (table_name,)) + columns = [row[0] for row in obj.cursor.fetchall()] + + if not table: + print(f"{NAME}: The {table_name} table does not exist. Skip the migration.") + + elif len(columns) < 1: + print(f"{NAME}: The {table_name} table does not have the necessary columns to execute the migration. Skip the migration.") + + else: + for column in rename_columns: + try: + print(f"{NAME}: Rename column {column[0]} to {column[1]} in the {table_name} table...") + obj.cursor.execute(f"ALTER TABLE {table_name} RENAME COLUMN {column[0]} TO {column[1]}") + obj.database_connection.commit() + print(f"{NAME}: Column {column[0]} has been renamed to {column[1]} in the {table_name} table.") + except obj.errors.DuplicateColumn as error: + print(f"{NAME}: Columns in the {table_name} table have already been renamed. Skip renaming: {error}") + obj.database_connection.rollback() + except obj.errors.UndefinedColumn as error: + print(f"{NAME}: Columns in the {table_name} table have not been renamed. Skip renaming: {error}") + obj.database_connection.rollback() + + for column in add_columns: + try: + print(f"{NAME}: Add column {column[0]} to the {table_name} table...") + obj.cursor.execute(f"ALTER TABLE {table_name} ADD COLUMN {column[0]} {column[1]} DEFAULT {column[2]}") + obj.database_connection.commit() + print(f"{NAME}: Column {column[0]} has been added to the {table_name} table.") + except obj.errors.DuplicateColumn as error: + print(f"{NAME}: Columns in the {table_name} table have already been added. Skip adding: {error}") + obj.database_connection.rollback() + except obj.errors.FeatureNotSupported as error: + print(f"{NAME}: Columns in the {table_name} table have not been added. Skip adding: {error}") + obj.database_connection.rollback() diff --git a/src/modules/database.py b/src/modules/database.py index ddcd52cfa..ffb45c81c 100644 --- a/src/modules/database.py +++ b/src/modules/database.py @@ -4,7 +4,6 @@ import importlib import json from typing import Union -from datetime import datetime, timedelta import psycopg2 from logger import log from .tools import get_hash @@ -16,15 +15,13 @@ class DatabaseClient: """ def __init__( self, - vault: object = None, - environment: str = None + vault: object = None ) -> None: """ Initializes a new instance of the Database client. Args: vault (object): An object representing a HashiCorp Vault client for retrieving secrets with the database configuration. - environment (str): The environment to use for the database connection. Attributes: database_connection (psycopg2.extensions.connection): A connection to the PostgreSQL database. @@ -49,10 +46,7 @@ def __init__( >>> vault = Vault() >>> db = Database(vault=vault) """ - if environment: - db_configuration = vault.read_secret(path=f"configuration/database-{environment}") - else: - db_configuration = vault.read_secret(path='configuration/database') + db_configuration = vault.read_secret(path='configuration/database') self.database_connection = psycopg2.connect( host=db_configuration['host'], @@ -66,6 +60,7 @@ def __init__( __class__.__name__, db_configuration['host'], db_configuration['port'], db_configuration['database'] ) + self.errors = psycopg2.errors self.cursor = self.database_connection.cursor() self.vault = vault @@ -503,79 +498,33 @@ def update_message_state_in_queue( return response - def verify_users_queue(self) -> None: + def update_schedule_time_in_queue( + self, + post_id: str = None, + user_id: str = None, + scheduled_time: str = None + ) -> str: """ - Verify the queue for all users and reschedule messages if necessary. - If the message is not processed in time (for example, the bot was down), reschedule the time of the message processing. + Update the scheduled time of a message in the queue table. Args: - None + post_id (str): The ID of the post. + user_id (str): The ID of the user. + scheduled_time (str): The new scheduled time for the message. Returns: - None + str: A response message indicating the status of the update. Examples: - >>> verify_users_queue() + >>> update_schedule_time_in_queue(post_id='123', user_id='12345', scheduled_time='2022-01-01 12:00:00') + '123: scheduled time updated' """ - log.info("[class.%s] Database: verifying the message of users in queue...", __class__.__name__) - users = self.get_users() - - for user in users: - user_id = user[0] - need_reschedule = False - full_queue = self._select( - table_name='queue', - columns=("id", "scheduled_time"), - condition=f"user_id = '{user_id}'", - order_by='scheduled_time ASC', - limit=1000 - ) - - for message in full_queue: - if message[1] < datetime.now() - timedelta(minutes=10): - need_reschedule = True - log.warning( - "[class.%s] Database: found a message in the queue that was not processed in time for user %s", - __class__.__name__, user_id - ) - break - - if need_reschedule: - log.warning("[class.%s] Database: rescheduling messages in the queue for user %s", __class__.__name__, user_id) - # The lag between the current time and the scheduled time of the message in the seconds - lag = None - # The difference in minutes between the current message and the previous message in the seconds - diff = None - # The new scheduled time for the message after rescheduling - new_schedule_time = None - # The previous scheduled time of the message for calculate the skew between the messages. For keep rate limit. - previous_schedule_time = None - # Reschedule the all messages in the queue - for message in full_queue: - schedule_time = message[1] - lag = (datetime.now() - schedule_time).total_seconds() - - # If haven't previous message value for compare difference between the messages - if not previous_schedule_time: - new_schedule_time = datetime.now() - self._update( - table_name='queue', - values=f"scheduled_time = '{new_schedule_time}'", - condition=f"id = '{message[0]}'" - ) - else: - diff = (schedule_time - previous_schedule_time).total_seconds() - # Add the difference in minutes between the current message and the previous message to the lag - skew = diff + lag - new_schedule_time = datetime.now() + timedelta(seconds=skew) - self._update( - table_name='queue', - values=f"scheduled_time = '{new_schedule_time}'", - condition=f"id = '{message[0]}'" - ) - previous_schedule_time = schedule_time - log.info("[class.%s] Database: rescheduled message %s: %s -> %s", __class__.__name__, message[0], message[1], new_schedule_time) - log.info("[class.%s] Database: users queue verification completed", __class__.__name__) + self._update( + table_name='queue', + values=f"scheduled_time = '{scheduled_time}'", + condition=f"post_id = '{post_id}' AND user_id = '{user_id}'" + ) + return f"{post_id}: scheduled time updated" def get_user_queue( self, @@ -599,7 +548,8 @@ def get_user_queue( table_name='queue', columns=("post_id", "scheduled_time"), condition=f"user_id = '{user_id}'", - limit=1000 + order_by='scheduled_time ASC', + limit=10 ) for message in queue: if user_id not in result: @@ -678,8 +628,8 @@ def keep_message( self, message_id: str = None, chat_id: str = None, - message_type: str = None, - message_content: Union[str, dict] = None + message_content: Union[str, dict] = None, + **kwargs ) -> str: """ Add a message to the messages table in the database. @@ -688,29 +638,53 @@ def keep_message( Args: message_id (str): The ID of the message. chat_id (str): The ID of the chat. - message_type (str): The type of the message. message_content (Union[str, dict]): The content of the message. + Keyword Args: + message_type (str): The type of the message. + state (str): The state of the message. + recreated (bool): A flag indicating whether the message was recreated. + Returns: str: A message indicating that the message was added to the messages table. Examples: - >>> keep_message('12345', '67890', 'status_message', 'Hello, username\n...') + >>> keep_message('12345', '67890', 'Hello, World!', message_type='status_message', state='updated') '12345 kept' or '12345 updated' """ + message_type = kwargs.get('message_type', None) + state = kwargs.get('state', 'updated') + recreated = kwargs.get('recreated', False) message_content_hash = get_hash(message_content) check_exist_message_type = self._select( table_name='messages', columns=("id", "message_id"), condition=f"message_type = '{message_type}' AND chat_id = '{chat_id}'", ) - if check_exist_message_type: + response = None + + if check_exist_message_type and recreated: self._update( table_name='messages', values=( f"message_content_hash = '{message_content_hash}', " f"message_id = '{message_id}', " - f"timestamp = CURRENT_TIMESTAMP" + f"state = '{state}', " + "updated_at = CURRENT_TIMESTAMP, " + "created_at = CURRENT_TIMESTAMP" + ), + condition=f"id = '{check_exist_message_type[0][0]}'" + ) + response = f"{message_id} recreated" + + elif check_exist_message_type and not recreated: + self._update( + table_name='messages', + values=( + f"message_content_hash = '{message_content_hash}', " + f"message_id = '{message_id}', " + f"state = '{state}', " + f"updated_at = CURRENT_TIMESTAMP" ), condition=f"id = '{check_exist_message_type[0][0]}'" ) @@ -747,7 +721,7 @@ def add_user( '12345 already exists' """ exist_user = self._select(table_name='users', columns=("user_id",), condition=f"user_id = '{user_id}'") - if exist_user and user_id in exist_user[0]: + if exist_user: result = f"{user_id} already exists" else: self._insert( @@ -784,7 +758,7 @@ def get_considered_message( self, message_type: str = None, chat_id: str = None - ) -> str: + ) -> tuple: """ Get a message with specified type and chat ID from the messages table in the database. @@ -797,12 +771,12 @@ def get_considered_message( Examples: >>> current_message_id(message_type='status_message', chat_id='12345') - # ('message_id', 'chat_id', 'timestamp', 'message_content_hash') - ('123456789', '12345', datetime.datetime(2023, 11, 14, 21, 14, 26, 680024), '2ef7bde608ce5404e97d5f042f95f89f1c232871d3d7') + # ('message_id', 'chat_id', 'created_at', 'updated_at', 'message_content_hash', 'state') + ('123456789', '12345', datetime.datetime, datetime.datetime, 'hash', 'updated') """ message = self._select( table_name='messages', - columns=("message_id", "chat_id", "timestamp", "message_content_hash",), + columns=("message_id", "chat_id", "created_at", "updated_at", "message_content_hash", "state"), condition=f"message_type = '{message_type}' AND chat_id = '{chat_id}'", limit=1 ) diff --git a/src/modules/downloader.py b/src/modules/downloader.py index 5a372d902..f9a9f148a 100644 --- a/src/modules/downloader.py +++ b/src/modules/downloader.py @@ -69,7 +69,7 @@ def __init__( "Failed to initialize the Downloader instance." "Please check the configuration in class argument or the secret with the configuration in the Vault." ) - log.info('[class.%s] Downloader: creating a new instance of the Downloader...', __class__.__name__) + log.info('[Downloader]: creating a new instance of the Downloader...') self.instaloader = instaloader.Instaloader( quiet=True, user_agent=self.configuration.get('user-agent', None), @@ -90,8 +90,8 @@ def __init__( ) auth_status = self._login() log.info( - '[class.%s] Downloader: downloader instance created successfully: %s in %s', - __class__.__name__, auth_status, self.configuration['username'] + '[Downloader]: downloader instance created successfully: %s in %s', + auth_status, self.configuration['username'] ) def _login(self) -> Union[str, None]: @@ -115,7 +115,7 @@ def _login(self) -> Union[str, None]: self.configuration['username'], self.configuration['session-file'] ) - log.info('[class.%s] Downloader: session file %s was load success', __class__.__name__, self.configuration['session-file']) + log.info('[Downloader]: session file %s was load success', self.configuration['session-file']) return 'logged_in' if self.configuration['login-method'] == 'password': @@ -125,13 +125,13 @@ def _login(self) -> Union[str, None]: ) self.instaloader.save_session_to_file(self.configuration['session-file']) log.info( - '[class.%s] Downloader: login with password was successful. Save session in %s', - __class__.__name__, self.configuration['sessionfile'] + '[Downloader]: login with password was successful. Save session in %s', + self.configuration['sessionfile'] ) return 'logged_in' if self.configuration['login-method'] == 'anonymous': - log.warning('[class.%s] Downloader: initialization without authentication into an account (anonymous)', __class__.__name__) + log.warning('[Downloader]: initialization without authentication into an account (anonymous)') return None raise FailedAuthInstaloader( @@ -151,19 +151,31 @@ def get_post_content( Returns: (dict) { 'post': shortcode, - 'owner': post.owner_username, - 'type': post.typename, + 'owner': owner, + 'type': typename, 'status': 'completed' } """ - log.info('[class.%s] Downloader: downloading the contents of the post %s...', __class__.__name__, shortcode) - post = instaloader.Post.from_shortcode(self.instaloader.context, shortcode) - self.instaloader.download_post(post, '') - log.info('[class.%s] Downloader: the contents of the post %s have been successfully downloaded', __class__.__name__, shortcode) - metadata = { + log.info('[Downloader]: downloading the contents of the post %s...', shortcode) + try: + post = instaloader.Post.from_shortcode(self.instaloader.context, shortcode) + self.instaloader.download_post(post, '') + log.info('[Downloader]: the contents of the post %s have been successfully downloaded', shortcode) + status = 'completed' + owner = post.owner_username + typename = post.typename + except instaloader.exceptions.BadResponseException as error: + log.error('[Downloader]: error downloading post content: %s', error) + if "Fetching Post metadata failed" in str(error): + status = 'not_found' + log.warning('[Downloader]: post %s not found, perhaps it was deleted. Message will be marked as processed.', shortcode) + else: + status = 'failed' + owner = 'undefined' + typename = 'undefined' + return { 'post': shortcode, - 'owner': post.owner_username, - 'type': post.typename, - 'status': 'completed' + 'owner': owner, + 'type': typename, + 'status': status } - return metadata