From d2d8f7d370a34b8ddb9755f578c7359a8c8ee0a7 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 27 Jul 2023 01:21:22 +0400 Subject: [PATCH 01/14] feat: add support for batch updates --- composer.lock | 273 +++++++++++++++++-------- src/Database/Adapter.php | 10 + src/Database/Adapter/MariaDB.php | 88 +++++++- src/Database/Database.php | 51 +++++ tests/Database/Adapter/MariaDBTest.php | 60 ++++++ 5 files changed, 390 insertions(+), 92 deletions(-) diff --git a/composer.lock b/composer.lock index 24ba6740d..c5e5c7744 100644 --- a/composer.lock +++ b/composer.lock @@ -336,23 +336,24 @@ }, { "name": "utopia-php/framework", - "version": "0.27.0", + "version": "0.28.4", "source": { "type": "git", "url": "https://github.com/utopia-php/framework.git", - "reference": "b8d0447f5c98291d7759db05460ecced29a0f9ee" + "reference": "98c5469efe195aeecc63745dbf8e2f357f8cedac" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/framework/zipball/b8d0447f5c98291d7759db05460ecced29a0f9ee", - "reference": "b8d0447f5c98291d7759db05460ecced29a0f9ee", + "url": "https://api.github.com/repos/utopia-php/framework/zipball/98c5469efe195aeecc63745dbf8e2f357f8cedac", + "reference": "98c5469efe195aeecc63745dbf8e2f357f8cedac", "shasum": "" }, "require": { - "php": ">=8.0.0" + "php": ">=8.0" }, "require-dev": { "laravel/pint": "^1.2", + "phpstan/phpstan": "1.9.x-dev", "phpunit/phpunit": "^9.5.25", "vimeo/psalm": "4.27.0" }, @@ -374,9 +375,9 @@ ], "support": { "issues": "https://github.com/utopia-php/framework/issues", - "source": "https://github.com/utopia-php/framework/tree/0.27.0" + "source": "https://github.com/utopia-php/framework/tree/0.28.4" }, - "time": "2023-01-29T05:36:17+00:00" + "time": "2023-06-03T14:09:22+00:00" }, { "name": "utopia-php/mongo", @@ -788,6 +789,53 @@ }, "time": "2019-12-04T15:06:13+00:00" }, + { + "name": "doctrine/deprecations", + "version": "v1.1.1", + "source": { + "type": "git", + "url": "https://github.com/doctrine/deprecations.git", + "reference": "612a3ee5ab0d5dd97b7cf3874a6efe24325efac3" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/doctrine/deprecations/zipball/612a3ee5ab0d5dd97b7cf3874a6efe24325efac3", + "reference": "612a3ee5ab0d5dd97b7cf3874a6efe24325efac3", + "shasum": "" + }, + "require": { + "php": "^7.1 || ^8.0" + }, + "require-dev": { + "doctrine/coding-standard": "^9", + "phpstan/phpstan": "1.4.10 || 1.10.15", + "phpstan/phpstan-phpunit": "^1.0", + "phpunit/phpunit": "^7.5 || ^8.5 || ^9.5", + "psalm/plugin-phpunit": "0.18.4", + "psr/log": "^1 || ^2 || ^3", + "vimeo/psalm": "4.30.0 || 5.12.0" + }, + "suggest": { + "psr/log": "Allows logging deprecations via PSR-3 logger implementation" + }, + "type": "library", + "autoload": { + "psr-4": { + "Doctrine\\Deprecations\\": "lib/Doctrine/Deprecations" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "description": "A small layer on top of trigger_error(E_USER_DEPRECATED) or PSR-3 logging with options to disable all deprecations or selectively for packages.", + "homepage": "https://www.doctrine-project.org/", + "support": { + "issues": "https://github.com/doctrine/deprecations/issues", + "source": "https://github.com/doctrine/deprecations/tree/v1.1.1" + }, + "time": "2023-06-03T09:27:29+00:00" + }, { "name": "doctrine/instantiator", "version": "2.0.0", @@ -860,16 +908,16 @@ }, { "name": "fakerphp/faker", - "version": "v1.21.0", + "version": "v1.23.0", "source": { "type": "git", "url": "https://github.com/FakerPHP/Faker.git", - "reference": "92efad6a967f0b79c499705c69b662f738cc9e4d" + "reference": "e3daa170d00fde61ea7719ef47bb09bb8f1d9b01" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/FakerPHP/Faker/zipball/92efad6a967f0b79c499705c69b662f738cc9e4d", - "reference": "92efad6a967f0b79c499705c69b662f738cc9e4d", + "url": "https://api.github.com/repos/FakerPHP/Faker/zipball/e3daa170d00fde61ea7719ef47bb09bb8f1d9b01", + "reference": "e3daa170d00fde61ea7719ef47bb09bb8f1d9b01", "shasum": "" }, "require": { @@ -922,9 +970,9 @@ ], "support": { "issues": "https://github.com/FakerPHP/Faker/issues", - "source": "https://github.com/FakerPHP/Faker/tree/v1.21.0" + "source": "https://github.com/FakerPHP/Faker/tree/v1.23.0" }, - "time": "2022-12-13T13:54:32+00:00" + "time": "2023-06-12T08:44:38+00:00" }, { "name": "felixfbecker/advanced-json-rpc", @@ -1029,16 +1077,16 @@ }, { "name": "myclabs/deep-copy", - "version": "1.11.0", + "version": "1.11.1", "source": { "type": "git", "url": "https://github.com/myclabs/DeepCopy.git", - "reference": "14daed4296fae74d9e3201d2c4925d1acb7aa614" + "reference": "7284c22080590fb39f2ffa3e9057f10a4ddd0e0c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/myclabs/DeepCopy/zipball/14daed4296fae74d9e3201d2c4925d1acb7aa614", - "reference": "14daed4296fae74d9e3201d2c4925d1acb7aa614", + "url": "https://api.github.com/repos/myclabs/DeepCopy/zipball/7284c22080590fb39f2ffa3e9057f10a4ddd0e0c", + "reference": "7284c22080590fb39f2ffa3e9057f10a4ddd0e0c", "shasum": "" }, "require": { @@ -1076,7 +1124,7 @@ ], "support": { "issues": "https://github.com/myclabs/DeepCopy/issues", - "source": "https://github.com/myclabs/DeepCopy/tree/1.11.0" + "source": "https://github.com/myclabs/DeepCopy/tree/1.11.1" }, "funding": [ { @@ -1084,7 +1132,7 @@ "type": "tidelift" } ], - "time": "2022-03-03T13:19:32+00:00" + "time": "2023-03-08T13:26:56+00:00" }, { "name": "netresearch/jsonmapper", @@ -1139,16 +1187,16 @@ }, { "name": "nikic/php-parser", - "version": "v4.15.3", + "version": "v4.16.0", "source": { "type": "git", "url": "https://github.com/nikic/PHP-Parser.git", - "reference": "570e980a201d8ed0236b0a62ddf2c9cbb2034039" + "reference": "19526a33fb561ef417e822e85f08a00db4059c17" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/570e980a201d8ed0236b0a62ddf2c9cbb2034039", - "reference": "570e980a201d8ed0236b0a62ddf2c9cbb2034039", + "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/19526a33fb561ef417e822e85f08a00db4059c17", + "reference": "19526a33fb561ef417e822e85f08a00db4059c17", "shasum": "" }, "require": { @@ -1189,9 +1237,9 @@ ], "support": { "issues": "https://github.com/nikic/PHP-Parser/issues", - "source": "https://github.com/nikic/PHP-Parser/tree/v4.15.3" + "source": "https://github.com/nikic/PHP-Parser/tree/v4.16.0" }, - "time": "2023-01-16T22:05:37+00:00" + "time": "2023-06-25T14:52:30+00:00" }, { "name": "openlss/lib-array2xml", @@ -1469,24 +1517,27 @@ }, { "name": "phpdocumentor/type-resolver", - "version": "1.6.2", + "version": "1.7.2", "source": { "type": "git", "url": "https://github.com/phpDocumentor/TypeResolver.git", - "reference": "48f445a408c131e38cab1c235aa6d2bb7a0bb20d" + "reference": "b2fe4d22a5426f38e014855322200b97b5362c0d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/48f445a408c131e38cab1c235aa6d2bb7a0bb20d", - "reference": "48f445a408c131e38cab1c235aa6d2bb7a0bb20d", + "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/b2fe4d22a5426f38e014855322200b97b5362c0d", + "reference": "b2fe4d22a5426f38e014855322200b97b5362c0d", "shasum": "" }, "require": { + "doctrine/deprecations": "^1.0", "php": "^7.4 || ^8.0", - "phpdocumentor/reflection-common": "^2.0" + "phpdocumentor/reflection-common": "^2.0", + "phpstan/phpdoc-parser": "^1.13" }, "require-dev": { "ext-tokenizer": "*", + "phpbench/phpbench": "^1.2", "phpstan/extension-installer": "^1.1", "phpstan/phpstan": "^1.8", "phpstan/phpstan-phpunit": "^1.1", @@ -1518,29 +1569,76 @@ "description": "A PSR-5 based resolver of Class names, Types and Structural Element Names", "support": { "issues": "https://github.com/phpDocumentor/TypeResolver/issues", - "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.6.2" + "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.7.2" }, - "time": "2022-10-14T12:47:21+00:00" + "time": "2023-05-30T18:13:47+00:00" + }, + { + "name": "phpstan/phpdoc-parser", + "version": "1.23.0", + "source": { + "type": "git", + "url": "https://github.com/phpstan/phpdoc-parser.git", + "reference": "a2b24135c35852b348894320d47b3902a94bc494" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/a2b24135c35852b348894320d47b3902a94bc494", + "reference": "a2b24135c35852b348894320d47b3902a94bc494", + "shasum": "" + }, + "require": { + "php": "^7.2 || ^8.0" + }, + "require-dev": { + "doctrine/annotations": "^2.0", + "nikic/php-parser": "^4.15", + "php-parallel-lint/php-parallel-lint": "^1.2", + "phpstan/extension-installer": "^1.0", + "phpstan/phpstan": "^1.5", + "phpstan/phpstan-phpunit": "^1.1", + "phpstan/phpstan-strict-rules": "^1.0", + "phpunit/phpunit": "^9.5", + "symfony/process": "^5.2" + }, + "type": "library", + "autoload": { + "psr-4": { + "PHPStan\\PhpDocParser\\": [ + "src/" + ] + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "description": "PHPDoc parser with support for nullable, intersection and generic types", + "support": { + "issues": "https://github.com/phpstan/phpdoc-parser/issues", + "source": "https://github.com/phpstan/phpdoc-parser/tree/1.23.0" + }, + "time": "2023-07-23T22:17:56+00:00" }, { "name": "phpunit/php-code-coverage", - "version": "9.2.24", + "version": "9.2.27", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-code-coverage.git", - "reference": "2cf940ebc6355a9d430462811b5aaa308b174bed" + "reference": "b0a88255cb70d52653d80c890bd7f38740ea50d1" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/2cf940ebc6355a9d430462811b5aaa308b174bed", - "reference": "2cf940ebc6355a9d430462811b5aaa308b174bed", + "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/b0a88255cb70d52653d80c890bd7f38740ea50d1", + "reference": "b0a88255cb70d52653d80c890bd7f38740ea50d1", "shasum": "" }, "require": { "ext-dom": "*", "ext-libxml": "*", "ext-xmlwriter": "*", - "nikic/php-parser": "^4.14", + "nikic/php-parser": "^4.15", "php": ">=7.3", "phpunit/php-file-iterator": "^3.0.3", "phpunit/php-text-template": "^2.0.2", @@ -1555,8 +1653,8 @@ "phpunit/phpunit": "^9.3" }, "suggest": { - "ext-pcov": "*", - "ext-xdebug": "*" + "ext-pcov": "PHP extension that provides line coverage", + "ext-xdebug": "PHP extension that provides line coverage as well as branch and path coverage" }, "type": "library", "extra": { @@ -1589,7 +1687,8 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/php-code-coverage/issues", - "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.24" + "security": "https://github.com/sebastianbergmann/php-code-coverage/security/policy", + "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.27" }, "funding": [ { @@ -1597,7 +1696,7 @@ "type": "github" } ], - "time": "2023-01-26T08:26:55+00:00" + "time": "2023-07-26T13:44:30+00:00" }, { "name": "phpunit/php-file-iterator", @@ -1842,16 +1941,16 @@ }, { "name": "phpunit/phpunit", - "version": "9.6.3", + "version": "9.6.10", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "e7b1615e3e887d6c719121c6d4a44b0ab9645555" + "reference": "a6d351645c3fe5a30f5e86be6577d946af65a328" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/e7b1615e3e887d6c719121c6d4a44b0ab9645555", - "reference": "e7b1615e3e887d6c719121c6d4a44b0ab9645555", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/a6d351645c3fe5a30f5e86be6577d946af65a328", + "reference": "a6d351645c3fe5a30f5e86be6577d946af65a328", "shasum": "" }, "require": { @@ -1884,8 +1983,8 @@ "sebastian/version": "^3.0.2" }, "suggest": { - "ext-soap": "*", - "ext-xdebug": "*" + "ext-soap": "To be able to generate mocks based on WSDL files", + "ext-xdebug": "PHP extension that provides line coverage as well as branch and path coverage" }, "bin": [ "phpunit" @@ -1924,7 +2023,8 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/phpunit/issues", - "source": "https://github.com/sebastianbergmann/phpunit/tree/9.6.3" + "security": "https://github.com/sebastianbergmann/phpunit/security/policy", + "source": "https://github.com/sebastianbergmann/phpunit/tree/9.6.10" }, "funding": [ { @@ -1940,7 +2040,7 @@ "type": "tidelift" } ], - "time": "2023-02-04T13:37:15+00:00" + "time": "2023-07-10T04:04:23+00:00" }, { "name": "psr/container", @@ -2345,16 +2445,16 @@ }, { "name": "sebastian/diff", - "version": "4.0.4", + "version": "4.0.5", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/diff.git", - "reference": "3461e3fccc7cfdfc2720be910d3bd73c69be590d" + "reference": "74be17022044ebaaecfdf0c5cd504fc9cd5a7131" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/diff/zipball/3461e3fccc7cfdfc2720be910d3bd73c69be590d", - "reference": "3461e3fccc7cfdfc2720be910d3bd73c69be590d", + "url": "https://api.github.com/repos/sebastianbergmann/diff/zipball/74be17022044ebaaecfdf0c5cd504fc9cd5a7131", + "reference": "74be17022044ebaaecfdf0c5cd504fc9cd5a7131", "shasum": "" }, "require": { @@ -2399,7 +2499,7 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/diff/issues", - "source": "https://github.com/sebastianbergmann/diff/tree/4.0.4" + "source": "https://github.com/sebastianbergmann/diff/tree/4.0.5" }, "funding": [ { @@ -2407,7 +2507,7 @@ "type": "github" } ], - "time": "2020-10-26T13:10:38+00:00" + "time": "2023-05-07T05:35:17+00:00" }, { "name": "sebastian/environment", @@ -3053,16 +3153,16 @@ }, { "name": "symfony/console", - "version": "v5.4.19", + "version": "v5.4.24", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "dccb8d251a9017d5994c988b034d3e18aaabf740" + "reference": "560fc3ed7a43e6d30ea94a07d77f9a60b8ed0fb8" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/dccb8d251a9017d5994c988b034d3e18aaabf740", - "reference": "dccb8d251a9017d5994c988b034d3e18aaabf740", + "url": "https://api.github.com/repos/symfony/console/zipball/560fc3ed7a43e6d30ea94a07d77f9a60b8ed0fb8", + "reference": "560fc3ed7a43e6d30ea94a07d77f9a60b8ed0fb8", "shasum": "" }, "require": { @@ -3127,12 +3227,12 @@ "homepage": "https://symfony.com", "keywords": [ "cli", - "command line", + "command-line", "console", "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v5.4.19" + "source": "https://github.com/symfony/console/tree/v5.4.24" }, "funding": [ { @@ -3148,20 +3248,20 @@ "type": "tidelift" } ], - "time": "2023-01-01T08:32:19+00:00" + "time": "2023-05-26T05:13:16+00:00" }, { "name": "symfony/deprecation-contracts", - "version": "v3.2.0", + "version": "v3.3.0", "source": { "type": "git", "url": "https://github.com/symfony/deprecation-contracts.git", - "reference": "1ee04c65529dea5d8744774d474e7cbd2f1206d3" + "reference": "7c3aff79d10325257a001fcf92d991f24fc967cf" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/deprecation-contracts/zipball/1ee04c65529dea5d8744774d474e7cbd2f1206d3", - "reference": "1ee04c65529dea5d8744774d474e7cbd2f1206d3", + "url": "https://api.github.com/repos/symfony/deprecation-contracts/zipball/7c3aff79d10325257a001fcf92d991f24fc967cf", + "reference": "7c3aff79d10325257a001fcf92d991f24fc967cf", "shasum": "" }, "require": { @@ -3170,7 +3270,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-main": "3.3-dev" + "dev-main": "3.4-dev" }, "thanks": { "name": "symfony/contracts", @@ -3199,7 +3299,7 @@ "description": "A generic function and convention to trigger deprecation notices", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/deprecation-contracts/tree/v3.2.0" + "source": "https://github.com/symfony/deprecation-contracts/tree/v3.3.0" }, "funding": [ { @@ -3215,7 +3315,7 @@ "type": "tidelift" } ], - "time": "2022-11-25T10:21:52+00:00" + "time": "2023-05-23T14:45:45+00:00" }, { "name": "symfony/polyfill-ctype", @@ -3628,16 +3728,16 @@ }, { "name": "symfony/service-contracts", - "version": "v3.2.0", + "version": "v3.3.0", "source": { "type": "git", "url": "https://github.com/symfony/service-contracts.git", - "reference": "aac98028c69df04ee77eb69b96b86ee51fbf4b75" + "reference": "40da9cc13ec349d9e4966ce18b5fbcd724ab10a4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/service-contracts/zipball/aac98028c69df04ee77eb69b96b86ee51fbf4b75", - "reference": "aac98028c69df04ee77eb69b96b86ee51fbf4b75", + "url": "https://api.github.com/repos/symfony/service-contracts/zipball/40da9cc13ec349d9e4966ce18b5fbcd724ab10a4", + "reference": "40da9cc13ec349d9e4966ce18b5fbcd724ab10a4", "shasum": "" }, "require": { @@ -3647,13 +3747,10 @@ "conflict": { "ext-psr": "<1.1|>=2" }, - "suggest": { - "symfony/service-implementation": "" - }, "type": "library", "extra": { "branch-alias": { - "dev-main": "3.3-dev" + "dev-main": "3.4-dev" }, "thanks": { "name": "symfony/contracts", @@ -3693,7 +3790,7 @@ "standards" ], "support": { - "source": "https://github.com/symfony/service-contracts/tree/v3.2.0" + "source": "https://github.com/symfony/service-contracts/tree/v3.3.0" }, "funding": [ { @@ -3709,20 +3806,20 @@ "type": "tidelift" } ], - "time": "2022-11-25T10:21:52+00:00" + "time": "2023-05-23T14:45:45+00:00" }, { "name": "symfony/string", - "version": "v6.2.5", + "version": "v6.3.0", "source": { "type": "git", "url": "https://github.com/symfony/string.git", - "reference": "b2dac0fa27b1ac0f9c0c0b23b43977f12308d0b0" + "reference": "f2e190ee75ff0f5eced645ec0be5c66fac81f51f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/string/zipball/b2dac0fa27b1ac0f9c0c0b23b43977f12308d0b0", - "reference": "b2dac0fa27b1ac0f9c0c0b23b43977f12308d0b0", + "url": "https://api.github.com/repos/symfony/string/zipball/f2e190ee75ff0f5eced645ec0be5c66fac81f51f", + "reference": "f2e190ee75ff0f5eced645ec0be5c66fac81f51f", "shasum": "" }, "require": { @@ -3733,13 +3830,13 @@ "symfony/polyfill-mbstring": "~1.0" }, "conflict": { - "symfony/translation-contracts": "<2.0" + "symfony/translation-contracts": "<2.5" }, "require-dev": { "symfony/error-handler": "^5.4|^6.0", "symfony/http-client": "^5.4|^6.0", "symfony/intl": "^6.2", - "symfony/translation-contracts": "^2.0|^3.0", + "symfony/translation-contracts": "^2.5|^3.0", "symfony/var-exporter": "^5.4|^6.0" }, "type": "library", @@ -3779,7 +3876,7 @@ "utf8" ], "support": { - "source": "https://github.com/symfony/string/tree/v6.2.5" + "source": "https://github.com/symfony/string/tree/v6.3.0" }, "funding": [ { @@ -3795,7 +3892,7 @@ "type": "tidelift" } ], - "time": "2023-01-01T08:38:09+00:00" + "time": "2023-03-21T21:06:29+00:00" }, { "name": "theseer/tokenizer", @@ -4171,5 +4268,5 @@ "ext-redis": "*", "ext-mongodb": "*" }, - "plugin-api-version": "2.2.0" + "plugin-api-version": "2.3.0" } diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index 153784927..60f71e8e5 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -295,6 +295,16 @@ abstract public function getDocument(string $collection, string $id, array $quer */ abstract public function createDocument(string $collection, Document $document): Document; + /** + * Create Documents + * + * @param string $collection + * @param Document[] $documents + * + * @return Document[] + */ + abstract public function createDocuments(string $collection, array $documents): array; + /** * Update Document * diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index c1cef9de4..7cb5abfcf 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -428,6 +428,86 @@ public function createDocument(string $collection, Document $document): Document return $document; } + public function createDocuments(string $collection, array $documents): array + { + // Start transaction + $this->getPDO()->beginTransaction(); + + try { + // Prepare SQL for multiple inserts + $sql = "INSERT INTO {$this->getSQLTable($collection)} "; + + $insertQueryParts = []; + $insertValues = []; + + foreach ($documents as $index => $document) { + $attributes = $document->getAttributes(); + $attributes['_createdAt'] = $document->getCreatedAt(); + $attributes['_updatedAt'] = $document->getUpdatedAt(); + $attributes['_permissions'] = json_encode($document->getPermissions()); + + // Parse statement + $columnNames = ''; + $bindValues = ''; + foreach ($attributes as $attribute => $value) { + $column = $this->filter($attribute); + $columnNames .= "`{$column}`, "; + $bindValues .= $this->getPDO()->quote($value) . ", "; + } + + $columnNames = rtrim($columnNames, ', '); + $bindValues = rtrim($bindValues, ', '); + + $insertQueryParts[] = "({$bindValues})"; + $insertValues[] = $bindValues; + + $permissions = []; + foreach (Database::PERMISSIONS as $type) { + foreach ($document->getPermissionsByType($type) as $permission) { + $permission = \str_replace('"', '', $permission); + $permissions[] = "('{$type}', '{$permission}', '{$document->getId()}')"; + } + } + + if (!empty($permissions)) { + $queryPermissions = "INSERT INTO {$this->getSQLTable($collection . '_perms')} (_type, _permission, _document) VALUES " . implode(', ', $permissions); + $this->getPDO()->exec($queryPermissions); + } + } + + $sql .= "({$columnNames}) VALUES " . implode(", ", $insertQueryParts); + + var_dump($sql); + exit(); + + // Execute SQL + $this->getPDO()->exec($sql); + + // Commit the transaction + $this->getPDO()->commit(); + + // Update internal ID of documents + foreach ($documents as $document) { + $document['$internalId'] = $this->getDocument($collection, $document->getId())->getInternalId(); + } + + return $documents; + } catch (PDOException $e) { + // Rollback the transaction on error + $this->getPDO()->rollBack(); + + switch ($e->getCode()) { + case 1062: + case 23000: + throw new Duplicate('Duplicated document: ' . $e->getMessage()); + + default: + throw $e; + } + } + } + + /** * Update Document * @@ -1003,10 +1083,10 @@ protected function getSQLCondition(Query $query): string default => $query->getAttribute() }); - $attribute = "`{$query->getAttribute()}`" ; + $attribute = "`{$query->getAttribute()}`"; $placeholder = $this->getSQLPlaceholder($query); - switch ($query->getMethod()){ + switch ($query->getMethod()) { case Query::TYPE_SEARCH: /** * Replace reserved chars with space. @@ -1016,7 +1096,7 @@ protected function getSQLCondition(Query $query): string * Prepend wildcard by default on the back. */ $value = $this->getSQLValue($query->getMethod(), $value); - return 'MATCH('.$attribute.') AGAINST ('.$this->getPDO()->quote($value).' IN BOOLEAN MODE)'; + return 'MATCH(' . $attribute . ') AGAINST (' . $this->getPDO()->quote($value) . ' IN BOOLEAN MODE)'; case Query::TYPE_BETWEEN: return "table_main.{$attribute} BETWEEN :{$placeholder}_0 AND :{$placeholder}_1"; @@ -1028,7 +1108,7 @@ protected function getSQLCondition(Query $query): string default: $conditions = []; foreach ($query->getValues() as $key => $value) { - $conditions[] = $attribute.' '.$this->getSQLOperator($query->getMethod()).' '.':'.$placeholder.'_'.$key; + $conditions[] = $attribute . ' ' . $this->getSQLOperator($query->getMethod()) . ' ' . ':' . $placeholder . '_' . $key; } $condition = implode(' OR ', $conditions); return empty($condition) ? '' : '(' . $condition . ')'; diff --git a/src/Database/Database.php b/src/Database/Database.php index dfb78e486..734fe523d 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -82,8 +82,10 @@ class Database const EVENT_DOCUMENT_FIND = 'document_find'; const EVENT_DOCUMENT_CREATE = 'document_create'; + const EVENT_DOCUMENTS_CREATE = 'documents_create'; const EVENT_DOCUMENT_READ = 'document_read'; const EVENT_DOCUMENT_UPDATE = 'document_update'; + const EVENT_DOCUMENTS_UPDATE = 'documents_update'; const EVENT_DOCUMENT_DELETE = 'document_delete'; const EVENT_DOCUMENT_COUNT = 'document_count'; const EVENT_DOCUMENT_SUM = 'document_sum'; @@ -1446,6 +1448,55 @@ public function createDocument(string $collection, Document $document): Document return $document; } + /** + * Create Document + * + * @param string $collection + * @param Document $document + * + * @return Document + * + * @throws AuthorizationException + * @throws StructureException + * @throws Exception|Throwable + */ + public function createDocuments(string $collection, array $documents): array + { + $collection = $this->silent(fn() => $this->getCollection($collection)); + + $time = DateTime::now(); + + $documents = array_map(function($document) use ($collection, $time){ + $document + ->setAttribute('$id', empty($document->getId()) ? ID::unique() : $document->getId()) + ->setAttribute('$collection', $collection->getId()) + ->setAttribute('$createdAt', $time) + ->setAttribute('$updatedAt', $time); + + $document = $this->encode($collection, $document); + + $validator = new Structure($collection); + + if (!$validator->isValid($document)) { + throw new StructureException($validator->getDescription()); + } + + return $document; + }, $documents); + + $documents = $this->adapter->createDocuments($collection->getId(), $documents); + + $documents = array_map(function($document) use ($collection){ + $document = $this->decode($collection, $document); + + $this->trigger(self::EVENT_DOCUMENT_CREATE, $document); + + return $document; + }, $documents); + + return $documents; + } + /** * Update Document * diff --git a/tests/Database/Adapter/MariaDBTest.php b/tests/Database/Adapter/MariaDBTest.php index 9aa1c225d..c683c3daa 100644 --- a/tests/Database/Adapter/MariaDBTest.php +++ b/tests/Database/Adapter/MariaDBTest.php @@ -8,6 +8,9 @@ use Utopia\Database\Adapter\MariaDB; use Utopia\Cache\Cache; use Utopia\Cache\Adapter\Redis as RedisAdapter; +use Utopia\Database\Document; +use Utopia\Database\Helpers\Permission; +use Utopia\Database\Helpers\Role; use Utopia\Tests\Base; class MariaDBTest extends Base @@ -55,4 +58,61 @@ static function getDatabase(): Database return self::$database = $database; } + + public function testCreateDocuments() + { + static::getDatabase()->createCollection('documents'); + + // $this->assertEquals(true, static::getDatabase()->createAttribute('documents', 'string', Database::VAR_STRING, 128, true)); + // $this->assertEquals(true, static::getDatabase()->createAttribute('documents', 'integer', Database::VAR_INTEGER, 0, true)); + // $this->assertEquals(true, static::getDatabase()->createAttribute('documents', 'bigint', Database::VAR_INTEGER, 8, true)); + // $this->assertEquals(true, static::getDatabase()->createAttribute('documents', 'float', Database::VAR_FLOAT, 0, true)); + // $this->assertEquals(true, static::getDatabase()->createAttribute('documents', 'boolean', Database::VAR_BOOLEAN, 0, true)); + // $this->assertEquals(true, static::getDatabase()->createAttribute('documents', 'colors', Database::VAR_STRING, 32, true, null, true, true)); + // $this->assertEquals(true, static::getDatabase()->createAttribute('documents', 'empty', Database::VAR_STRING, 32, false, null, true, true)); + // $this->assertEquals(true, static::getDatabase()->createAttribute('documents', 'with-dash', Database::VAR_STRING, 128, false, null)); + + + // // Create an array of documents with random attributes. Dont use the createDocument function + // $documents = []; + + // for ($i = 0; $i < 5; $i++) { + // $documents[] = new Document([ + // '$permissions' => [ + // Permission::read(Role::any()), + // Permission::create(Role::any()), + // Permission::update(Role::any()), + // Permission::delete(Role::any()), + // ], + // 'string' => 'textđź“ť', + // 'integer' => 5, + // 'bigint' => 8589934592, // 2^33 + // 'float' => 5.55, + // 'boolean' => true, + // 'colors' => ['pink', 'green', 'blue'], + // 'empty' => [], + // 'with-dash' => 'Works', + // ]); + // } + + // static::getDatabase()->createDocuments('documents', $documents); + + // $document = $documents[0]; + + // $this->assertNotEmpty(true, $document->getId()); + // $this->assertIsString($document->getAttribute('string')); + // $this->assertEquals('textđź“ť', $document->getAttribute('string')); // Also makes sure an emoji is working + // $this->assertIsInt($document->getAttribute('integer')); + // $this->assertEquals(5, $document->getAttribute('integer')); + // $this->assertIsInt($document->getAttribute('bigint')); + // $this->assertEquals(8589934592, $document->getAttribute('bigint')); + // $this->assertIsFloat($document->getAttribute('float')); + // $this->assertEquals(5.55, $document->getAttribute('float')); + // $this->assertIsBool($document->getAttribute('boolean')); + // $this->assertEquals(true, $document->getAttribute('boolean')); + // $this->assertIsArray($document->getAttribute('colors')); + // $this->assertEquals(['pink', 'green', 'blue'], $document->getAttribute('colors')); + // $this->assertEquals([], $document->getAttribute('empty')); + // $this->assertEquals('Works', $document->getAttribute('with-dash')); + } } \ No newline at end of file From 5cb99f914959b4096b701251d5de75b2dd3fefb4 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Tue, 1 Aug 2023 01:23:22 +0400 Subject: [PATCH 02/14] feat: add support for batch updates --- src/Database/Adapter.php | 10 ++ src/Database/Adapter/MariaDB.php | 131 ++++++++++---- src/Database/Database.php | 36 ++++ tests/Database/Adapter/MariaDBTest.php | 232 +++++++++++++++++++------ 4 files changed, 318 insertions(+), 91 deletions(-) diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index 60f71e8e5..f9e4c4e57 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -315,6 +315,16 @@ abstract public function createDocuments(string $collection, array $documents): */ abstract public function updateDocument(string $collection, Document $document): Document; + /** + * Update Documents + * + * @param string $collection + * @param Document[] $documents + * + * @return Document[] + */ + abstract public function updateDocuments(string $collection, array $documents): array; + /** * Delete Document * diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index 7cb5abfcf..8977ed93b 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -430,36 +430,52 @@ public function createDocument(string $collection, Document $document): Document public function createDocuments(string $collection, array $documents): array { - // Start transaction + if (empty($documents)) { + return $documents; + } + $this->getPDO()->beginTransaction(); - + try { - // Prepare SQL for multiple inserts - $sql = "INSERT INTO {$this->getSQLTable($collection)} "; + $name = $this->filter($collection); - $insertQueryParts = []; - $insertValues = []; - - foreach ($documents as $index => $document) { + foreach ($documents as $document) { $attributes = $document->getAttributes(); $attributes['_createdAt'] = $document->getCreatedAt(); $attributes['_updatedAt'] = $document->getUpdatedAt(); $attributes['_permissions'] = json_encode($document->getPermissions()); - // Parse statement + $columns = ''; $columnNames = ''; - $bindValues = ''; + $bindIndex = 0; + $bindValues = []; foreach ($attributes as $attribute => $value) { $column = $this->filter($attribute); - $columnNames .= "`{$column}`, "; - $bindValues .= $this->getPDO()->quote($value) . ", "; + $bindKey = 'key_' . $bindIndex; + $columns .= "`{$column}`, "; + + if (is_array($value)) { // arrays & objects should be saved as strings + $value = json_encode($value); + } + $value = (is_bool($value)) ? (int)$value : $value; + + $columnNames .= ':' . $bindKey . ', '; + $bindValues[$bindKey] = $value; + $bindIndex++; + } + + $columnNames .= ':_uid'; + $bindValues['_uid'] = $document->getId(); + + $stmt = $this->getPDO() + ->prepare("INSERT INTO {$this->getSQLTable($name)} + ({$columns}_uid) VALUES ({$columnNames})"); + + foreach ($bindValues as $key => $value) { + $stmt->bindValue(':' . $key, $value, $this->getPDOType($value)); } - $columnNames = rtrim($columnNames, ', '); - $bindValues = rtrim($bindValues, ', '); - - $insertQueryParts[] = "({$bindValues})"; - $insertValues[] = $bindValues; + $stmt->execute(); $permissions = []; foreach (Database::PERMISSIONS as $type) { @@ -468,44 +484,34 @@ public function createDocuments(string $collection, array $documents): array $permissions[] = "('{$type}', '{$permission}', '{$document->getId()}')"; } } - + if (!empty($permissions)) { - $queryPermissions = "INSERT INTO {$this->getSQLTable($collection . '_perms')} (_type, _permission, _document) VALUES " . implode(', ', $permissions); - $this->getPDO()->exec($queryPermissions); + $queryPermissions = "INSERT INTO {$this->getSQLTable($name . '_perms')} (_type, _permission, _document) VALUES " . implode(', ', $permissions); + $stmtPermissions = $this->getPDO()->prepare($queryPermissions); + $stmtPermissions->execute(); + $permissions = []; } } - - $sql .= "({$columnNames}) VALUES " . implode(", ", $insertQueryParts); - - var_dump($sql); - exit(); - - // Execute SQL - $this->getPDO()->exec($sql); - - // Commit the transaction - $this->getPDO()->commit(); - - // Update internal ID of documents - foreach ($documents as $document) { - $document['$internalId'] = $this->getDocument($collection, $document->getId())->getInternalId(); + + if (!$this->getPDO()->commit()) { + throw new Exception('Failed to commit transaction'); } return $documents; + } catch (PDOException $e) { - // Rollback the transaction on error $this->getPDO()->rollBack(); - switch ($e->getCode()) { case 1062: case 23000: throw new Duplicate('Duplicated document: ' . $e->getMessage()); - + default: throw $e; } } } + /** @@ -696,6 +702,55 @@ public function updateDocument(string $collection, Document $document): Document return $document; } + + public function updateDocuments(string $collection, array $documents): array + { + if (empty($documents)) { + return []; + } + + $ids = []; + $binds = []; + $sets = []; + + foreach ($documents as $document) { + $attributes = $document->getAttributes(); + $attributes['_createdAt'] = $document->getCreatedAt(); + $attributes['_updatedAt'] = $document->getUpdatedAt(); + $attributes['_permissions'] = json_encode($document->getPermissions()); + + $id = $document->getId(); + $ids[] = $id; + + foreach ($attributes as $attribute => $value) { + $bindKey = ':' . $attribute . '_' . $id; + $binds[$bindKey] = $value; + $sets[$attribute][] = 'WHEN ' . $id . ' THEN ' . $bindKey; + } + } + + $setClauses = []; + foreach ($sets as $attribute => $whenThens) { + $setClauses[] = '`' . $attribute . '` = CASE _uid ' . implode(' ', $whenThens) . ' END'; + } + + $sql = "UPDATE {$this->getSQLTable($collection)} + SET " . implode(', ', $setClauses) . " + WHERE _uid IN (" . implode(',', $ids) . ")"; + + $stmt = $this->getPDO()->prepare($sql); + + foreach ($binds as $bindKey => $value) { + $stmt->bindValue($bindKey, $value, $this->getPDOType($value)); + } + + $stmt->execute(); + + return $documents; + } + + + /** * Increase or decrease an attribute value * diff --git a/src/Database/Database.php b/src/Database/Database.php index 734fe523d..a9191360d 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -1544,6 +1544,42 @@ public function updateDocument(string $collection, string $id, Document $documen return $document; } + public function updateDocuments(string $collection, array $documents): array + { + if (empty($documents)) { + return []; + } + + foreach ($documents as $document) { + if (!$document->getId()) { + throw new Exception('Must define $id attribute for each document'); + } + } + + $time = DateTime::now(); + $collection = $this->silent(fn() => $this->getCollection($collection)); + + foreach ($documents as $document) { + $document->setAttribute('$updatedAt', $time); + $document = $this->encode($collection, $document); + + $validator = new Structure($collection); + if (!$validator->isValid($document)) { + throw new StructureException($validator->getDescription()); + } + } + + $documents = $this->adapter->updateDocuments($collection->getId(), $documents); + + foreach ($documents as $document) { + $document = $this->decode($collection, $document); + $this->trigger(self::EVENT_DOCUMENT_UPDATE, $document); + } + + return $documents; +} + + /** * Increase a document attribute by a value * diff --git a/tests/Database/Adapter/MariaDBTest.php b/tests/Database/Adapter/MariaDBTest.php index c683c3daa..eb3885123 100644 --- a/tests/Database/Adapter/MariaDBTest.php +++ b/tests/Database/Adapter/MariaDBTest.php @@ -56,63 +56,189 @@ static function getDatabase(): Database $database->setDefaultDatabase('utopiaTests'); $database->setNamespace('myapp_'.uniqid()); + $database->create(); + return self::$database = $database; } public function testCreateDocuments() { - static::getDatabase()->createCollection('documents'); - - // $this->assertEquals(true, static::getDatabase()->createAttribute('documents', 'string', Database::VAR_STRING, 128, true)); - // $this->assertEquals(true, static::getDatabase()->createAttribute('documents', 'integer', Database::VAR_INTEGER, 0, true)); - // $this->assertEquals(true, static::getDatabase()->createAttribute('documents', 'bigint', Database::VAR_INTEGER, 8, true)); - // $this->assertEquals(true, static::getDatabase()->createAttribute('documents', 'float', Database::VAR_FLOAT, 0, true)); - // $this->assertEquals(true, static::getDatabase()->createAttribute('documents', 'boolean', Database::VAR_BOOLEAN, 0, true)); - // $this->assertEquals(true, static::getDatabase()->createAttribute('documents', 'colors', Database::VAR_STRING, 32, true, null, true, true)); - // $this->assertEquals(true, static::getDatabase()->createAttribute('documents', 'empty', Database::VAR_STRING, 32, false, null, true, true)); - // $this->assertEquals(true, static::getDatabase()->createAttribute('documents', 'with-dash', Database::VAR_STRING, 128, false, null)); - - - // // Create an array of documents with random attributes. Dont use the createDocument function - // $documents = []; - - // for ($i = 0; $i < 5; $i++) { - // $documents[] = new Document([ - // '$permissions' => [ - // Permission::read(Role::any()), - // Permission::create(Role::any()), - // Permission::update(Role::any()), - // Permission::delete(Role::any()), - // ], - // 'string' => 'textđź“ť', - // 'integer' => 5, - // 'bigint' => 8589934592, // 2^33 - // 'float' => 5.55, - // 'boolean' => true, - // 'colors' => ['pink', 'green', 'blue'], - // 'empty' => [], - // 'with-dash' => 'Works', - // ]); - // } - - // static::getDatabase()->createDocuments('documents', $documents); - - // $document = $documents[0]; - - // $this->assertNotEmpty(true, $document->getId()); - // $this->assertIsString($document->getAttribute('string')); - // $this->assertEquals('textđź“ť', $document->getAttribute('string')); // Also makes sure an emoji is working - // $this->assertIsInt($document->getAttribute('integer')); - // $this->assertEquals(5, $document->getAttribute('integer')); - // $this->assertIsInt($document->getAttribute('bigint')); - // $this->assertEquals(8589934592, $document->getAttribute('bigint')); - // $this->assertIsFloat($document->getAttribute('float')); - // $this->assertEquals(5.55, $document->getAttribute('float')); - // $this->assertIsBool($document->getAttribute('boolean')); - // $this->assertEquals(true, $document->getAttribute('boolean')); - // $this->assertIsArray($document->getAttribute('colors')); - // $this->assertEquals(['pink', 'green', 'blue'], $document->getAttribute('colors')); - // $this->assertEquals([], $document->getAttribute('empty')); - // $this->assertEquals('Works', $document->getAttribute('with-dash')); + $count = 100; + $collection = 'testCreateDocuments'; + + static::getDatabase()->createCollection($collection); + + $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'string', Database::VAR_STRING, 128, true)); + $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'integer', Database::VAR_INTEGER, 0, true)); + $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'bigint', Database::VAR_INTEGER, 8, true)); + $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'float', Database::VAR_FLOAT, 0, true)); + $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'boolean', Database::VAR_BOOLEAN, 0, true)); + $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'colors', Database::VAR_STRING, 32, true, null, true, true)); + $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'empty', Database::VAR_STRING, 32, false, null, true, true)); + $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'with-dash', Database::VAR_STRING, 128, false, null)); + + + // Create an array of documents with random attributes. Dont use the createDocument function + $documents = []; + + for ($i = 0; $i < $count; $i++) { + $documents[] = new Document([ + '$permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + 'string' => 'textđź“ť', + 'integer' => 5, + 'bigint' => 8589934592, // 2^33 + 'float' => 5.55, + 'boolean' => true, + 'colors' => ['pink', 'green', 'blue'], + 'empty' => [], + 'with-dash' => 'Works', + ]); + } + $start = microtime(true); + $res = static::getDatabase()->createDocuments($collection, $documents); + $end = microtime(true); + var_dump('createDocuments: '.($end - $start).'s'); + + $this->assertEquals($count, count($res)); + + foreach ($res as $document) { + $this->assertNotEmpty(true, $document->getId()); + $this->assertIsString($document->getAttribute('string')); + $this->assertEquals('textđź“ť', $document->getAttribute('string')); // Also makes sure an emoji is working + $this->assertIsInt($document->getAttribute('integer')); + $this->assertEquals(5, $document->getAttribute('integer')); + $this->assertIsInt($document->getAttribute('bigint')); + $this->assertEquals(8589934592, $document->getAttribute('bigint')); + $this->assertIsFloat($document->getAttribute('float')); + $this->assertEquals(5.55, $document->getAttribute('float')); + $this->assertIsBool($document->getAttribute('boolean')); + $this->assertEquals(true, $document->getAttribute('boolean')); + $this->assertIsArray($document->getAttribute('colors')); + $this->assertEquals(['pink', 'green', 'blue'], $document->getAttribute('colors')); + $this->assertEquals([], $document->getAttribute('empty')); + $this->assertEquals('Works', $document->getAttribute('with-dash')); + } } + + public function testCreateDocumentsSequential() + { + $count = 100; + $collection = 'testCreateDocumentsSequential'; + + static::getDatabase()->createCollection($collection); + $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'string', Database::VAR_STRING, 128, true)); + $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'integer', Database::VAR_INTEGER, 0, true)); + $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'bigint', Database::VAR_INTEGER, 8, true)); + $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'float', Database::VAR_FLOAT, 0, true)); + $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'boolean', Database::VAR_BOOLEAN, 0, true)); + $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'colors', Database::VAR_STRING, 32, true, null, true, true)); + $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'empty', Database::VAR_STRING, 32, false, null, true, true)); + $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'with-dash', Database::VAR_STRING, 128, false, null)); + + + $start = microtime(true); + $res = []; + for ($i = 0; $i < $count; $i++) { + $res[] = static::getDatabase()->createDocument($collection, new Document([ + '$permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + 'string' => 'textđź“ť', + 'integer' => 5, + 'bigint' => 8589934592, // 2^33 + 'float' => 5.55, + 'boolean' => true, + 'colors' => ['pink', 'green', 'blue'], + 'empty' => [], + 'with-dash' => 'Works', + ])); + } + $end = microtime(true); + var_dump('createDocumentsSequential: '.($end - $start).'s'); + + $this->assertEquals($count, count($res)); + foreach ($res as $document) { + $this->assertNotEmpty(true, $document->getId()); + $this->assertIsString($document->getAttribute('string')); + $this->assertEquals('textđź“ť', $document->getAttribute('string')); // Also makes sure an emoji is working + $this->assertIsInt($document->getAttribute('integer')); + $this->assertEquals(5, $document->getAttribute('integer')); + $this->assertIsInt($document->getAttribute('bigint')); + $this->assertEquals(8589934592, $document->getAttribute('bigint')); + $this->assertIsFloat($document->getAttribute('float')); + $this->assertEquals(5.55, $document->getAttribute('float')); + $this->assertIsBool($document->getAttribute('boolean')); + $this->assertEquals(true, $document->getAttribute('boolean')); + $this->assertIsArray($document->getAttribute('colors')); + $this->assertEquals(['pink', 'green', 'blue'], $document->getAttribute('colors')); + $this->assertEquals([], $document->getAttribute('empty')); + $this->assertEquals('Works', $document->getAttribute('with-dash')); + } + } + + // public function testUpdateDocuments(Document $document) + // { + // $document + // ->setAttribute('string', 'textđź“ť updated') + // ->setAttribute('integer', 6) + // ->setAttribute('float', 5.56) + // ->setAttribute('boolean', false) + // ->setAttribute('colors', 'red', Document::SET_TYPE_APPEND) + // ->setAttribute('with-dash', 'Works'); + + // $new = $this->getDatabase()->updateDocument($document->getCollection(), $document->getId(), $document); + + // $this->assertNotEmpty(true, $new->getId()); + // $this->assertIsString($new->getAttribute('string')); + // $this->assertEquals('textđź“ť updated', $new->getAttribute('string')); + // $this->assertIsInt($new->getAttribute('integer')); + // $this->assertEquals(6, $new->getAttribute('integer')); + // $this->assertIsFloat($new->getAttribute('float')); + // $this->assertEquals(5.56, $new->getAttribute('float')); + // $this->assertIsBool($new->getAttribute('boolean')); + // $this->assertEquals(false, $new->getAttribute('boolean')); + // $this->assertIsArray($new->getAttribute('colors')); + // $this->assertEquals(['pink', 'green', 'blue', 'red'], $new->getAttribute('colors')); + // $this->assertEquals('Works', $new->getAttribute('with-dash')); + + // $oldPermissions = $document->getPermissions(); + + // $new + // ->setAttribute('$permissions', Permission::read(Role::guests()), Document::SET_TYPE_APPEND) + // ->setAttribute('$permissions', Permission::create(Role::guests()), Document::SET_TYPE_APPEND) + // ->setAttribute('$permissions', Permission::update(Role::guests()), Document::SET_TYPE_APPEND) + // ->setAttribute('$permissions', Permission::delete(Role::guests()), Document::SET_TYPE_APPEND); + + // $this->getDatabase()->updateDocument($new->getCollection(), $new->getId(), $new); + + // $new = $this->getDatabase()->getDocument($new->getCollection(), $new->getId()); + + // $this->assertContains('guests', $new->getRead()); + // $this->assertContains('guests', $new->getWrite()); + // $this->assertContains('guests', $new->getCreate()); + // $this->assertContains('guests', $new->getUpdate()); + // $this->assertContains('guests', $new->getDelete()); + + // $new->setAttribute('$permissions', $oldPermissions); + + // $this->getDatabase()->updateDocument($new->getCollection(), $new->getId(), $new); + + // $new = $this->getDatabase()->getDocument($new->getCollection(), $new->getId()); + + // $this->assertNotContains('guests', $new->getRead()); + // $this->assertNotContains('guests', $new->getWrite()); + // $this->assertNotContains('guests', $new->getCreate()); + // $this->assertNotContains('guests', $new->getUpdate()); + // $this->assertNotContains('guests', $new->getDelete()); + + // return $document; + // } } \ No newline at end of file From 24fde911ae3c5251ca54fa86bfc9fadc14d89fd7 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 2 Aug 2023 14:21:35 +0400 Subject: [PATCH 03/14] feat: add support for batch updates --- src/Database/Adapter.php | 10 +++ src/Database/Adapter/MariaDB.php | 91 ++++++++++++++++++++++++++ src/Database/Database.php | 49 ++++++++++++++ tests/Database/Adapter/MariaDBTest.php | 69 ++++++++++++++++++- 4 files changed, 217 insertions(+), 2 deletions(-) diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index f9e4c4e57..cbf07709e 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -305,6 +305,16 @@ abstract public function createDocument(string $collection, Document $document): */ abstract public function createDocuments(string $collection, array $documents): array; + /** + * Create Documents + * + * @param string $collection + * @param Document[] $documents + * + * @return Document[] + */ + abstract public function createDocumentsBatch(string $collection, array $documents, int $size): array; + /** * Update Document * diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index 8977ed93b..c092b6e0f 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -512,6 +512,97 @@ public function createDocuments(string $collection, array $documents): array } } + public function createDocumentsBatch(string $collection, array $documents, int $size): array + { + if (empty($documents)) { + return $documents; + } + + $this->getPDO()->beginTransaction(); + + try { + $name = $this->filter($collection); + $batches = array_chunk($documents, $size); + + foreach ($batches as $batch) { + $bindIndex = 0; + $batchKeys = []; + $batchValues = []; + $permissions = []; + foreach ($batch as $document) { + $attributes = $document->getAttributes(); + $attributes['_uid'] = $document->getId(); + $attributes['_createdAt'] = $document->getCreatedAt(); + $attributes['_updatedAt'] = $document->getUpdatedAt(); + $attributes['_permissions'] = json_encode($document->getPermissions()); + + + $columns = array_map(function ($attribute) { + $column = $this->filter($attribute); + return "`$column`"; + }, array_keys($attributes)); + $columns = '(' . implode(', ', $columns) . ')'; + + $bindKeys = []; + + foreach ($attributes as $attribute => $value) { + if (is_array($value)) { // arrays & objects should be saved as strings + $value = json_encode($value); + } + $value = (is_bool($value)) ? (int)$value : $value; + $bindKey = 'key_' . $bindIndex; + $bindKeys[] = ':' . $bindKey; + $batchValues[$bindKey] = $value; + $bindIndex++; + } + + $batchKeys[] = '(' . implode(', ', $bindKeys) . ')'; + foreach (Database::PERMISSIONS as $type) { + foreach ($document->getPermissionsByType($type) as $permission) { + $permission = \str_replace('"', '', $permission); + $permissions[] = "('{$type}', '{$permission}', '{$document->getId()}')"; + } + } + } + + $stmt = $this->getPDO() + ->prepare("INSERT INTO {$this->getSQLTable($name)} + $columns VALUES " . implode(', ', $batchKeys)); + + foreach ($batchValues as $key => $value) { + $stmt->bindValue($key, $value, $this->getPDOType($value)); + } + + $stmt->execute(); + + if (!empty($permissions)) { + $queryPermissions = "INSERT INTO {$this->getSQLTable($name . '_perms')} (_type, _permission, _document) VALUES " . implode(', ', $permissions); + $stmtPermissions = $this->getPDO()->prepare($queryPermissions); + if (isset($stmtPermissions)) { + $stmtPermissions->execute(); + } + } + } + + if (!$this->getPDO()->commit()) { + throw new Exception('Failed to commit transaction'); + } + + return $documents; + + } catch (PDOException $e) { + $this->getPDO()->rollBack(); + switch ($e->getCode()) { + case 1062: + case 23000: + throw new Duplicate('Duplicated document: ' . $e->getMessage()); + + default: + throw $e; + } + } + } + /** diff --git a/src/Database/Database.php b/src/Database/Database.php index a9191360d..e930d7ad8 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -1497,6 +1497,55 @@ public function createDocuments(string $collection, array $documents): array return $documents; } + /** + * Create Documents in a batch + * + * @param string $collection + * @param Document $document + * + * @return Document + * + * @throws AuthorizationException + * @throws StructureException + * @throws Exception|Throwable + */ + public function createDocumentsBatch(string $collection, array $documents, int $size): array + { + $collection = $this->silent(fn() => $this->getCollection($collection)); + + $time = DateTime::now(); + + $documents = array_map(function($document) use ($collection, $time){ + $document + ->setAttribute('$id', empty($document->getId()) ? ID::unique() : $document->getId()) + ->setAttribute('$collection', $collection->getId()) + ->setAttribute('$createdAt', $time) + ->setAttribute('$updatedAt', $time); + + $document = $this->encode($collection, $document); + + $validator = new Structure($collection); + + if (!$validator->isValid($document)) { + throw new StructureException($validator->getDescription()); + } + + return $document; + }, $documents); + + $documents = $this->adapter->createDocumentsBatch($collection->getId(), $documents, $size); + + $documents = array_map(function($document) use ($collection){ + $document = $this->decode($collection, $document); + + $this->trigger(self::EVENT_DOCUMENT_CREATE, $document); + + return $document; + }, $documents); + + return $documents; + } + /** * Update Document * diff --git a/tests/Database/Adapter/MariaDBTest.php b/tests/Database/Adapter/MariaDBTest.php index eb3885123..38e31a283 100644 --- a/tests/Database/Adapter/MariaDBTest.php +++ b/tests/Database/Adapter/MariaDBTest.php @@ -63,7 +63,7 @@ static function getDatabase(): Database public function testCreateDocuments() { - $count = 100; + $count = 100000; $collection = 'testCreateDocuments'; static::getDatabase()->createCollection($collection); @@ -125,9 +125,74 @@ public function testCreateDocuments() } } + + public function testCreateDocumentsBatch() + { + $count = 100000; + $collection = 'testCreateDocumentsBatch'; + + static::getDatabase()->createCollection($collection); + + $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'string', Database::VAR_STRING, 128, true)); + $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'integer', Database::VAR_INTEGER, 0, true)); + $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'bigint', Database::VAR_INTEGER, 8, true)); + $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'float', Database::VAR_FLOAT, 0, true)); + $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'boolean', Database::VAR_BOOLEAN, 0, true)); + $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'colors', Database::VAR_STRING, 32, true, null, true, true)); + $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'empty', Database::VAR_STRING, 32, false, null, true, true)); + $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'with-dash', Database::VAR_STRING, 128, false, null)); + + + // Create an array of documents with random attributes. Dont use the createDocument function + $documents = []; + + for ($i = 0; $i < $count; $i++) { + $documents[] = new Document([ + '$permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + 'string' => 'textđź“ť', + 'integer' => 5, + 'bigint' => 8589934592, // 2^33 + 'float' => 5.55, + 'boolean' => true, + 'colors' => ['pink', 'green', 'blue'], + 'empty' => [], + 'with-dash' => 'Works', + ]); + } + $start = microtime(true); + $res = static::getDatabase()->createDocumentsBatch($collection, $documents, 10000); + $end = microtime(true); + var_dump('createDocumentsBatch: '.($end - $start).'s'); + + $this->assertEquals($count, count($res)); + + foreach ($res as $document) { + $this->assertNotEmpty(true, $document->getId()); + $this->assertIsString($document->getAttribute('string')); + $this->assertEquals('textđź“ť', $document->getAttribute('string')); // Also makes sure an emoji is working + $this->assertIsInt($document->getAttribute('integer')); + $this->assertEquals(5, $document->getAttribute('integer')); + $this->assertIsInt($document->getAttribute('bigint')); + $this->assertEquals(8589934592, $document->getAttribute('bigint')); + $this->assertIsFloat($document->getAttribute('float')); + $this->assertEquals(5.55, $document->getAttribute('float')); + $this->assertIsBool($document->getAttribute('boolean')); + $this->assertEquals(true, $document->getAttribute('boolean')); + $this->assertIsArray($document->getAttribute('colors')); + $this->assertEquals(['pink', 'green', 'blue'], $document->getAttribute('colors')); + $this->assertEquals([], $document->getAttribute('empty')); + $this->assertEquals('Works', $document->getAttribute('with-dash')); + } + } + public function testCreateDocumentsSequential() { - $count = 100; + $count = 100000; $collection = 'testCreateDocumentsSequential'; static::getDatabase()->createCollection($collection); From 690b21d53e035923096a3e208535ebabf5905db5 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 2 Aug 2023 19:43:09 +0400 Subject: [PATCH 04/14] feat: add support for batch creates --- src/Database/Adapter.php | 15 +-- src/Database/Adapter/MariaDB.php | 98 ++----------------- src/Database/Database.php | 55 +---------- tests/Database/Adapter/MariaDBTest.php | 129 +------------------------ 4 files changed, 17 insertions(+), 280 deletions(-) diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index cbf07709e..9382c13d7 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -296,24 +296,15 @@ abstract public function getDocument(string $collection, string $id, array $quer abstract public function createDocument(string $collection, Document $document): Document; /** - * Create Documents + * Create Documents in batches * * @param string $collection * @param Document[] $documents + * @param int $batchSize * * @return Document[] */ - abstract public function createDocuments(string $collection, array $documents): array; - - /** - * Create Documents - * - * @param string $collection - * @param Document[] $documents - * - * @return Document[] - */ - abstract public function createDocumentsBatch(string $collection, array $documents, int $size): array; + abstract public function createDocuments(string $collection, array $documents, int $batchSize): array; /** * Update Document diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index c092b6e0f..f303357c6 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -427,92 +427,8 @@ public function createDocument(string $collection, Document $document): Document return $document; } - - public function createDocuments(string $collection, array $documents): array - { - if (empty($documents)) { - return $documents; - } - - $this->getPDO()->beginTransaction(); - - try { - $name = $this->filter($collection); - - foreach ($documents as $document) { - $attributes = $document->getAttributes(); - $attributes['_createdAt'] = $document->getCreatedAt(); - $attributes['_updatedAt'] = $document->getUpdatedAt(); - $attributes['_permissions'] = json_encode($document->getPermissions()); - - $columns = ''; - $columnNames = ''; - $bindIndex = 0; - $bindValues = []; - foreach ($attributes as $attribute => $value) { - $column = $this->filter($attribute); - $bindKey = 'key_' . $bindIndex; - $columns .= "`{$column}`, "; - - if (is_array($value)) { // arrays & objects should be saved as strings - $value = json_encode($value); - } - $value = (is_bool($value)) ? (int)$value : $value; - - $columnNames .= ':' . $bindKey . ', '; - $bindValues[$bindKey] = $value; - $bindIndex++; - } - - $columnNames .= ':_uid'; - $bindValues['_uid'] = $document->getId(); - $stmt = $this->getPDO() - ->prepare("INSERT INTO {$this->getSQLTable($name)} - ({$columns}_uid) VALUES ({$columnNames})"); - - foreach ($bindValues as $key => $value) { - $stmt->bindValue(':' . $key, $value, $this->getPDOType($value)); - } - - $stmt->execute(); - - $permissions = []; - foreach (Database::PERMISSIONS as $type) { - foreach ($document->getPermissionsByType($type) as $permission) { - $permission = \str_replace('"', '', $permission); - $permissions[] = "('{$type}', '{$permission}', '{$document->getId()}')"; - } - } - - if (!empty($permissions)) { - $queryPermissions = "INSERT INTO {$this->getSQLTable($name . '_perms')} (_type, _permission, _document) VALUES " . implode(', ', $permissions); - $stmtPermissions = $this->getPDO()->prepare($queryPermissions); - $stmtPermissions->execute(); - $permissions = []; - } - } - - if (!$this->getPDO()->commit()) { - throw new Exception('Failed to commit transaction'); - } - - return $documents; - - } catch (PDOException $e) { - $this->getPDO()->rollBack(); - switch ($e->getCode()) { - case 1062: - case 23000: - throw new Duplicate('Duplicated document: ' . $e->getMessage()); - - default: - throw $e; - } - } - } - - public function createDocumentsBatch(string $collection, array $documents, int $size): array + public function createDocuments(string $collection, array $documents, int $batchSize = Database::INSERT_BATCH_SIZE): array { if (empty($documents)) { return $documents; @@ -522,13 +438,14 @@ public function createDocumentsBatch(string $collection, array $documents, int $ try { $name = $this->filter($collection); - $batches = array_chunk($documents, $size); + $batches = array_chunk($documents, $batchSize); foreach ($batches as $batch) { $bindIndex = 0; $batchKeys = []; - $batchValues = []; + $bindValues = []; $permissions = []; + foreach ($batch as $document) { $attributes = $document->getAttributes(); $attributes['_uid'] = $document->getId(); @@ -536,7 +453,6 @@ public function createDocumentsBatch(string $collection, array $documents, int $ $attributes['_updatedAt'] = $document->getUpdatedAt(); $attributes['_permissions'] = json_encode($document->getPermissions()); - $columns = array_map(function ($attribute) { $column = $this->filter($attribute); return "`$column`"; @@ -552,7 +468,7 @@ public function createDocumentsBatch(string $collection, array $documents, int $ $value = (is_bool($value)) ? (int)$value : $value; $bindKey = 'key_' . $bindIndex; $bindKeys[] = ':' . $bindKey; - $batchValues[$bindKey] = $value; + $bindValues[$bindKey] = $value; $bindIndex++; } @@ -569,7 +485,7 @@ public function createDocumentsBatch(string $collection, array $documents, int $ ->prepare("INSERT INTO {$this->getSQLTable($name)} $columns VALUES " . implode(', ', $batchKeys)); - foreach ($batchValues as $key => $value) { + foreach ($bindValues as $key => $value) { $stmt->bindValue($key, $value, $this->getPDOType($value)); } @@ -592,6 +508,8 @@ public function createDocumentsBatch(string $collection, array $documents, int $ } catch (PDOException $e) { $this->getPDO()->rollBack(); + var_dump($e->getMessage()); + var_dump($e->getCode()); switch ($e->getCode()) { case 1062: case 23000: diff --git a/src/Database/Database.php b/src/Database/Database.php index e930d7ad8..0a1c8c3d4 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -100,6 +100,8 @@ class Database const EVENT_INDEX_CREATE = 'index_create'; const EVENT_INDEX_DELETE = 'index_delete'; + // Insert Batch Size + const INSERT_BATCH_SIZE = 100; /** * @var Adapter @@ -1448,55 +1450,6 @@ public function createDocument(string $collection, Document $document): Document return $document; } - /** - * Create Document - * - * @param string $collection - * @param Document $document - * - * @return Document - * - * @throws AuthorizationException - * @throws StructureException - * @throws Exception|Throwable - */ - public function createDocuments(string $collection, array $documents): array - { - $collection = $this->silent(fn() => $this->getCollection($collection)); - - $time = DateTime::now(); - - $documents = array_map(function($document) use ($collection, $time){ - $document - ->setAttribute('$id', empty($document->getId()) ? ID::unique() : $document->getId()) - ->setAttribute('$collection', $collection->getId()) - ->setAttribute('$createdAt', $time) - ->setAttribute('$updatedAt', $time); - - $document = $this->encode($collection, $document); - - $validator = new Structure($collection); - - if (!$validator->isValid($document)) { - throw new StructureException($validator->getDescription()); - } - - return $document; - }, $documents); - - $documents = $this->adapter->createDocuments($collection->getId(), $documents); - - $documents = array_map(function($document) use ($collection){ - $document = $this->decode($collection, $document); - - $this->trigger(self::EVENT_DOCUMENT_CREATE, $document); - - return $document; - }, $documents); - - return $documents; - } - /** * Create Documents in a batch * @@ -1509,7 +1462,7 @@ public function createDocuments(string $collection, array $documents): array * @throws StructureException * @throws Exception|Throwable */ - public function createDocumentsBatch(string $collection, array $documents, int $size): array + public function createDocuments(string $collection, array $documents, int $batchSize = self::INSERT_BATCH_SIZE): array { $collection = $this->silent(fn() => $this->getCollection($collection)); @@ -1533,7 +1486,7 @@ public function createDocumentsBatch(string $collection, array $documents, int $ return $document; }, $documents); - $documents = $this->adapter->createDocumentsBatch($collection->getId(), $documents, $size); + $documents = $this->adapter->createDocuments($collection->getId(), $documents, $batchSize); $documents = array_map(function($document) use ($collection){ $document = $this->decode($collection, $document); diff --git a/tests/Database/Adapter/MariaDBTest.php b/tests/Database/Adapter/MariaDBTest.php index 38e31a283..5e29004a2 100644 --- a/tests/Database/Adapter/MariaDBTest.php +++ b/tests/Database/Adapter/MariaDBTest.php @@ -61,6 +61,7 @@ static function getDatabase(): Database return self::$database = $database; } + public function testCreateDocuments() { $count = 100000; @@ -99,75 +100,8 @@ public function testCreateDocuments() 'with-dash' => 'Works', ]); } - $start = microtime(true); - $res = static::getDatabase()->createDocuments($collection, $documents); - $end = microtime(true); - var_dump('createDocuments: '.($end - $start).'s'); - - $this->assertEquals($count, count($res)); - - foreach ($res as $document) { - $this->assertNotEmpty(true, $document->getId()); - $this->assertIsString($document->getAttribute('string')); - $this->assertEquals('textđź“ť', $document->getAttribute('string')); // Also makes sure an emoji is working - $this->assertIsInt($document->getAttribute('integer')); - $this->assertEquals(5, $document->getAttribute('integer')); - $this->assertIsInt($document->getAttribute('bigint')); - $this->assertEquals(8589934592, $document->getAttribute('bigint')); - $this->assertIsFloat($document->getAttribute('float')); - $this->assertEquals(5.55, $document->getAttribute('float')); - $this->assertIsBool($document->getAttribute('boolean')); - $this->assertEquals(true, $document->getAttribute('boolean')); - $this->assertIsArray($document->getAttribute('colors')); - $this->assertEquals(['pink', 'green', 'blue'], $document->getAttribute('colors')); - $this->assertEquals([], $document->getAttribute('empty')); - $this->assertEquals('Works', $document->getAttribute('with-dash')); - } - } - - - public function testCreateDocumentsBatch() - { - $count = 100000; - $collection = 'testCreateDocumentsBatch'; - - static::getDatabase()->createCollection($collection); - $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'string', Database::VAR_STRING, 128, true)); - $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'integer', Database::VAR_INTEGER, 0, true)); - $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'bigint', Database::VAR_INTEGER, 8, true)); - $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'float', Database::VAR_FLOAT, 0, true)); - $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'boolean', Database::VAR_BOOLEAN, 0, true)); - $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'colors', Database::VAR_STRING, 32, true, null, true, true)); - $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'empty', Database::VAR_STRING, 32, false, null, true, true)); - $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'with-dash', Database::VAR_STRING, 128, false, null)); - - - // Create an array of documents with random attributes. Dont use the createDocument function - $documents = []; - - for ($i = 0; $i < $count; $i++) { - $documents[] = new Document([ - '$permissions' => [ - Permission::read(Role::any()), - Permission::create(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()), - ], - 'string' => 'textđź“ť', - 'integer' => 5, - 'bigint' => 8589934592, // 2^33 - 'float' => 5.55, - 'boolean' => true, - 'colors' => ['pink', 'green', 'blue'], - 'empty' => [], - 'with-dash' => 'Works', - ]); - } - $start = microtime(true); - $res = static::getDatabase()->createDocumentsBatch($collection, $documents, 10000); - $end = microtime(true); - var_dump('createDocumentsBatch: '.($end - $start).'s'); + $res = static::getDatabase()->createDocuments($collection, $documents, 14000); $this->assertEquals($count, count($res)); @@ -190,65 +124,6 @@ public function testCreateDocumentsBatch() } } - public function testCreateDocumentsSequential() - { - $count = 100000; - $collection = 'testCreateDocumentsSequential'; - - static::getDatabase()->createCollection($collection); - $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'string', Database::VAR_STRING, 128, true)); - $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'integer', Database::VAR_INTEGER, 0, true)); - $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'bigint', Database::VAR_INTEGER, 8, true)); - $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'float', Database::VAR_FLOAT, 0, true)); - $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'boolean', Database::VAR_BOOLEAN, 0, true)); - $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'colors', Database::VAR_STRING, 32, true, null, true, true)); - $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'empty', Database::VAR_STRING, 32, false, null, true, true)); - $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'with-dash', Database::VAR_STRING, 128, false, null)); - - - $start = microtime(true); - $res = []; - for ($i = 0; $i < $count; $i++) { - $res[] = static::getDatabase()->createDocument($collection, new Document([ - '$permissions' => [ - Permission::read(Role::any()), - Permission::create(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()), - ], - 'string' => 'textđź“ť', - 'integer' => 5, - 'bigint' => 8589934592, // 2^33 - 'float' => 5.55, - 'boolean' => true, - 'colors' => ['pink', 'green', 'blue'], - 'empty' => [], - 'with-dash' => 'Works', - ])); - } - $end = microtime(true); - var_dump('createDocumentsSequential: '.($end - $start).'s'); - - $this->assertEquals($count, count($res)); - foreach ($res as $document) { - $this->assertNotEmpty(true, $document->getId()); - $this->assertIsString($document->getAttribute('string')); - $this->assertEquals('textđź“ť', $document->getAttribute('string')); // Also makes sure an emoji is working - $this->assertIsInt($document->getAttribute('integer')); - $this->assertEquals(5, $document->getAttribute('integer')); - $this->assertIsInt($document->getAttribute('bigint')); - $this->assertEquals(8589934592, $document->getAttribute('bigint')); - $this->assertIsFloat($document->getAttribute('float')); - $this->assertEquals(5.55, $document->getAttribute('float')); - $this->assertIsBool($document->getAttribute('boolean')); - $this->assertEquals(true, $document->getAttribute('boolean')); - $this->assertIsArray($document->getAttribute('colors')); - $this->assertEquals(['pink', 'green', 'blue'], $document->getAttribute('colors')); - $this->assertEquals([], $document->getAttribute('empty')); - $this->assertEquals('Works', $document->getAttribute('with-dash')); - } - } - // public function testUpdateDocuments(Document $document) // { // $document From d43e521f20651caf9c34787b3f5c9ad7e18c439b Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 10 Aug 2023 19:37:31 +0400 Subject: [PATCH 05/14] feat: batch updates using insert --- src/Database/Adapter.php | 5 +- src/Database/Adapter/MariaDB.php | 235 +++++++++++++++++++------ src/Database/Database.php | 20 +-- tests/Database/Adapter/MariaDBTest.php | 105 +++++++---- 4 files changed, 264 insertions(+), 101 deletions(-) diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index 9382c13d7..13c6a903e 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -317,14 +317,15 @@ abstract public function createDocuments(string $collection, array $documents, i abstract public function updateDocument(string $collection, Document $document): Document; /** - * Update Documents + * Update Documents in batches * * @param string $collection * @param Document[] $documents + * @param int $batchSize * * @return Document[] */ - abstract public function updateDocuments(string $collection, array $documents): array; + abstract public function updateDocuments(string $collection, array $documents, int $batchSize): array; /** * Delete Document diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index f303357c6..259adfd4d 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -427,25 +427,25 @@ public function createDocument(string $collection, Document $document): Document return $document; } - + public function createDocuments(string $collection, array $documents, int $batchSize = Database::INSERT_BATCH_SIZE): array { if (empty($documents)) { return $documents; } - + $this->getPDO()->beginTransaction(); - + try { $name = $this->filter($collection); $batches = array_chunk($documents, $batchSize); - + foreach ($batches as $batch) { $bindIndex = 0; $batchKeys = []; $bindValues = []; $permissions = []; - + foreach ($batch as $document) { $attributes = $document->getAttributes(); $attributes['_uid'] = $document->getId(); @@ -471,7 +471,7 @@ public function createDocuments(string $collection, array $documents, int $batch $bindValues[$bindKey] = $value; $bindIndex++; } - + $batchKeys[] = '(' . implode(', ', $bindKeys) . ')'; foreach (Database::PERMISSIONS as $type) { foreach ($document->getPermissionsByType($type) as $permission) { @@ -484,13 +484,13 @@ public function createDocuments(string $collection, array $documents, int $batch $stmt = $this->getPDO() ->prepare("INSERT INTO {$this->getSQLTable($name)} $columns VALUES " . implode(', ', $batchKeys)); - + foreach ($bindValues as $key => $value) { $stmt->bindValue($key, $value, $this->getPDOType($value)); } - + $stmt->execute(); - + if (!empty($permissions)) { $queryPermissions = "INSERT INTO {$this->getSQLTable($name . '_perms')} (_type, _permission, _document) VALUES " . implode(', ', $permissions); $stmtPermissions = $this->getPDO()->prepare($queryPermissions); @@ -499,28 +499,25 @@ public function createDocuments(string $collection, array $documents, int $batch } } } - + if (!$this->getPDO()->commit()) { throw new Exception('Failed to commit transaction'); } - + return $documents; - } catch (PDOException $e) { $this->getPDO()->rollBack(); - var_dump($e->getMessage()); - var_dump($e->getCode()); switch ($e->getCode()) { case 1062: case 23000: throw new Duplicate('Duplicated document: ' . $e->getMessage()); - + default: throw $e; } } } - + /** @@ -712,52 +709,188 @@ public function updateDocument(string $collection, Document $document): Document } - public function updateDocuments(string $collection, array $documents): array + public function updateDocuments(string $collection, array $documents, int $batchSize = Database::INSERT_BATCH_SIZE): array { if (empty($documents)) { - return []; + return $documents; } - $ids = []; - $binds = []; - $sets = []; - - foreach ($documents as $document) { - $attributes = $document->getAttributes(); - $attributes['_createdAt'] = $document->getCreatedAt(); - $attributes['_updatedAt'] = $document->getUpdatedAt(); - $attributes['_permissions'] = json_encode($document->getPermissions()); - - $id = $document->getId(); - $ids[] = $id; - - foreach ($attributes as $attribute => $value) { - $bindKey = ':' . $attribute . '_' . $id; - $binds[$bindKey] = $value; - $sets[$attribute][] = 'WHEN ' . $id . ' THEN ' . $bindKey; - } - } + $this->getPDO()->beginTransaction(); - $setClauses = []; - foreach ($sets as $attribute => $whenThens) { - $setClauses[] = '`' . $attribute . '` = CASE _uid ' . implode(' ', $whenThens) . ' END'; - } + try { + $name = $this->filter($collection); + $batches = array_chunk($documents, $batchSize); - $sql = "UPDATE {$this->getSQLTable($collection)} - SET " . implode(', ', $setClauses) . " - WHERE _uid IN (" . implode(',', $ids) . ")"; + foreach ($batches as $batch) { + $bindIndex = 0; + $batchKeys = []; + $bindValues = []; - $stmt = $this->getPDO()->prepare($sql); + foreach ($batch as $document) { + $attributes = $document->getAttributes(); + $attributes['_uid'] = $document->getId(); + $attributes['_createdAt'] = $document->getCreatedAt(); + $attributes['_updatedAt'] = $document->getUpdatedAt(); + $attributes['_permissions'] = json_encode($document->getPermissions()); - foreach ($binds as $bindKey => $value) { - $stmt->bindValue($bindKey, $value, $this->getPDOType($value)); - } + $columns = array_map(function ($attribute) { + return "`" . $this->filter($attribute) . "`"; + }, array_keys($attributes)); - $stmt->execute(); + $bindKeys = []; - return $documents; - } + foreach ($attributes as $attribute => $value) { + if (is_array($value)) { + $value = json_encode($value); + } + $value = (is_bool($value)) ? (int)$value : $value; + $bindKey = 'key_' . $bindIndex; + $bindKeys[] = ':' . $bindKey; + $bindValues[$bindKey] = $value; + $bindIndex++; + } + + $batchKeys[] = '(' . implode(', ', $bindKeys) . ')'; + + // Permissions logic + $permissionsStmt = $this->getPDO()->prepare(" + SELECT _type, _permission + FROM {$this->getSQLTable($name . '_perms')} p + WHERE p._document = :_uid + "); + $permissionsStmt->bindValue(':_uid', $document->getId()); + $permissionsStmt->execute(); + $permissions = $permissionsStmt->fetchAll(PDO::FETCH_ASSOC); + + $initial = []; + foreach (Database::PERMISSIONS as $type) { + $initial[$type] = []; + } + + $permissions = array_reduce($permissions, function (array $carry, array $item) { + $carry[$item['_type']][] = $item['_permission']; + return $carry; + }, $initial); + // Get removed Permissions + $removals = []; + foreach (Database::PERMISSIONS as $type) { + $diff = array_diff($permissions[$type], $document->getPermissionsByType($type)); + if (!empty($diff)) { + $removals[$type] = $diff; + } + } + + // Query to remove permissions + $removeQuery = ''; + if (!empty($removals)) { + $removeQuery = 'AND ('; + foreach ($removals as $type => $permissions) { + $removeQuery .= "( + _type = '{$type}' + AND _permission IN (" . implode(', ', array_map(fn (string $i) => ":_remove_{$type}_{$i}", array_keys($permissions))) . ") + )"; + if ($type !== array_key_last($removals)) { + $removeQuery .= ' OR '; + } + } + } + if (!empty($removeQuery)) { + $removeQuery .= ')'; + $stmtRemovePermissions = $this->getPDO() + ->prepare(" + DELETE + FROM {$this->getSQLTable($name . '_perms')} + WHERE + _document = :_uid + {$removeQuery} + "); + $stmtRemovePermissions->bindValue(':_uid', $document->getId()); + + foreach ($removals as $type => $permissions) { + foreach ($permissions as $i => $permission) { + $stmtRemovePermissions->bindValue(":_remove_{$type}_{$i}", $permission); + } + } + } + + // Get added Permissions + $additions = []; + foreach (Database::PERMISSIONS as $type) { + $diff = array_diff($document->getPermissionsByType($type), $permissions[$type]); + if (!empty($diff)) { + $additions[$type] = $diff; + } + } + + // Query to add permissions + if (!empty($additions)) { + $values = []; + foreach ($additions as $type => $permissions) { + foreach ($permissions as $i => $_) { + $values[] = "( :_uid, '{$type}', :_add_{$type}_{$i} )"; + } + } + + $stmtAddPermissions = $this->getPDO() + ->prepare( + " + INSERT INTO {$this->getSQLTable($name . '_perms')} + (_document, _type, _permission) VALUES " . implode(', ', $values) + ); + + $stmtAddPermissions->bindValue(":_uid", $document->getId()); + foreach ($additions as $type => $permissions) { + foreach ($permissions as $i => $permission) { + $stmtAddPermissions->bindValue(":_add_{$type}_{$i}", $permission); + } + } + } + } + + $updateClause = implode(', ', array_map(function ($column) { + return "$column=VALUES($column)"; + }, array_slice($columns, 1))); // Exclude UID + + $stmt = $this->getPDO() + ->prepare("INSERT INTO {$this->getSQLTable($name)} + (" . implode(", ", $columns) . ") VALUES " . implode(', ', $batchKeys) . " ON DUPLICATE KEY UPDATE $updateClause"); + + foreach ($bindValues as $key => $value) { + $stmt->bindValue($key, $value, $this->getPDOType($value)); + } + + $stmt->execute(); + + // Execute permissions statements for each document in the batch + if (!empty($removals)) { + var_dump($stmtRemovePermissions->queryString); + $stmtRemovePermissions->execute(); + } + + if (!empty($additions)) { + var_dump($stmtAddPermissions->queryString); + $stmtAddPermissions->execute(); + } + } + + if (!$this->getPDO()->commit()) { + throw new Exception('Failed to commit transaction'); + } + + return $documents; + } catch (PDOException $e) { + $this->getPDO()->rollBack(); + switch ($e->getCode()) { + case 1062: + case 23000: + throw new Duplicate('Duplicated document: ' . $e->getMessage()); + + default: + throw $e; + } + } + } /** diff --git a/src/Database/Database.php b/src/Database/Database.php index 0a1c8c3d4..c4db56e3c 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -1546,22 +1546,20 @@ public function updateDocument(string $collection, string $id, Document $documen return $document; } - public function updateDocuments(string $collection, array $documents): array + public function updateDocuments(string $collection, array $documents, int $batchSize): array { if (empty($documents)) { return []; } + $time = DateTime::now(); + $collection = $this->silent(fn() => $this->getCollection($collection)); + foreach ($documents as $document) { if (!$document->getId()) { throw new Exception('Must define $id attribute for each document'); } - } - $time = DateTime::now(); - $collection = $this->silent(fn() => $this->getCollection($collection)); - - foreach ($documents as $document) { $document->setAttribute('$updatedAt', $time); $document = $this->encode($collection, $document); @@ -1571,12 +1569,12 @@ public function updateDocuments(string $collection, array $documents): array } } - $documents = $this->adapter->updateDocuments($collection->getId(), $documents); + $documents = $this->adapter->updateDocuments($collection->getId(), $documents, $batchSize); - foreach ($documents as $document) { - $document = $this->decode($collection, $document); - $this->trigger(self::EVENT_DOCUMENT_UPDATE, $document); - } + // foreach ($documents as $document) { + // $document = $this->decode($collection, $document); + // $this->trigger(self::EVENT_DOCUMENT_UPDATE, $document); + // } return $documents; } diff --git a/tests/Database/Adapter/MariaDBTest.php b/tests/Database/Adapter/MariaDBTest.php index 5e29004a2..5e56001aa 100644 --- a/tests/Database/Adapter/MariaDBTest.php +++ b/tests/Database/Adapter/MariaDBTest.php @@ -37,7 +37,7 @@ static function getAdapterName(): string */ static function getDatabase(): Database { - if(!is_null(self::$database)) { + if (!is_null(self::$database)) { return self::$database; } @@ -54,7 +54,7 @@ static function getDatabase(): Database $database = new Database(new MariaDB($pdo), $cache); $database->setDefaultDatabase('utopiaTests'); - $database->setNamespace('myapp_'.uniqid()); + $database->setNamespace('myapp_' . uniqid()); $database->create(); @@ -64,7 +64,7 @@ static function getDatabase(): Database public function testCreateDocuments() { - $count = 100000; + $count = 1000; $collection = 'testCreateDocuments'; static::getDatabase()->createCollection($collection); @@ -101,7 +101,7 @@ public function testCreateDocuments() ]); } - $res = static::getDatabase()->createDocuments($collection, $documents, 14000); + $res = static::getDatabase()->createDocuments($collection, $documents, $count); $this->assertEquals($count, count($res)); @@ -122,19 +122,30 @@ public function testCreateDocuments() $this->assertEquals([], $document->getAttribute('empty')); $this->assertEquals('Works', $document->getAttribute('with-dash')); } + + return $res; } - // public function testUpdateDocuments(Document $document) + // /** + // * @depends testCreateDocuments + // */ + // public function testUpdateDocumentsSequential(array $documents) // { - // $document - // ->setAttribute('string', 'textđź“ť updated') - // ->setAttribute('integer', 6) - // ->setAttribute('float', 5.56) - // ->setAttribute('boolean', false) - // ->setAttribute('colors', 'red', Document::SET_TYPE_APPEND) - // ->setAttribute('with-dash', 'Works'); - // $new = $this->getDatabase()->updateDocument($document->getCollection(), $document->getId(), $document); + // $start = microtime(true); + // foreach ($documents as $document) { + // $document + // ->setAttribute('string', 'textđź“ť updated') + // ->setAttribute('integer', 6) + // ->setAttribute('float', 5.56) + // ->setAttribute('boolean', false) + // ->setAttribute('colors', 'red', Document::SET_TYPE_APPEND) + // ->setAttribute('with-dash', 'Works'); + + // $new = static::getDatabase()->updateDocument($document->getCollection(), $document->getId(), $document); + // } + // $end = microtime(true); + // var_dump('Sequential update time for ' . count($documents) . ' documents : ' . ($end - $start)); // $this->assertNotEmpty(true, $new->getId()); // $this->assertIsString($new->getAttribute('string')); @@ -149,36 +160,56 @@ public function testCreateDocuments() // $this->assertEquals(['pink', 'green', 'blue', 'red'], $new->getAttribute('colors')); // $this->assertEquals('Works', $new->getAttribute('with-dash')); - // $oldPermissions = $document->getPermissions(); - - // $new - // ->setAttribute('$permissions', Permission::read(Role::guests()), Document::SET_TYPE_APPEND) - // ->setAttribute('$permissions', Permission::create(Role::guests()), Document::SET_TYPE_APPEND) - // ->setAttribute('$permissions', Permission::update(Role::guests()), Document::SET_TYPE_APPEND) - // ->setAttribute('$permissions', Permission::delete(Role::guests()), Document::SET_TYPE_APPEND); + // // $oldPermissions = $document->getPermissions(); - // $this->getDatabase()->updateDocument($new->getCollection(), $new->getId(), $new); + // // $new + // // ->setAttribute('$permissions', Permission::read(Role::guests()), Document::SET_TYPE_APPEND) + // // ->setAttribute('$permissions', Permission::create(Role::guests()), Document::SET_TYPE_APPEND) + // // ->setAttribute('$permissions', Permission::update(Role::guests()), Document::SET_TYPE_APPEND) + // // ->setAttribute('$permissions', Permission::delete(Role::guests()), Document::SET_TYPE_APPEND); - // $new = $this->getDatabase()->getDocument($new->getCollection(), $new->getId()); + // // $this->getDatabase()->updateDocument($new->getCollection(), $new->getId(), $new); - // $this->assertContains('guests', $new->getRead()); - // $this->assertContains('guests', $new->getWrite()); - // $this->assertContains('guests', $new->getCreate()); - // $this->assertContains('guests', $new->getUpdate()); - // $this->assertContains('guests', $new->getDelete()); + // // $new = $this->getDatabase()->getDocument($new->getCollection(), $new->getId()); - // $new->setAttribute('$permissions', $oldPermissions); + // // $this->assertContains('guests', $new->getRead()); + // // $this->assertContains('guests', $new->getWrite()); + // // $this->assertContains('guests', $new->getCreate()); + // // $this->assertContains('guests', $new->getUpdate()); + // // $this->assertContains('guests', $new->getDelete()); - // $this->getDatabase()->updateDocument($new->getCollection(), $new->getId(), $new); + // // $new->setAttribute('$permissions', $oldPermissions); - // $new = $this->getDatabase()->getDocument($new->getCollection(), $new->getId()); + // // $this->getDatabase()->updateDocument($new->getCollection(), $new->getId(), $new); - // $this->assertNotContains('guests', $new->getRead()); - // $this->assertNotContains('guests', $new->getWrite()); - // $this->assertNotContains('guests', $new->getCreate()); - // $this->assertNotContains('guests', $new->getUpdate()); - // $this->assertNotContains('guests', $new->getDelete()); + // // $new = $this->getDatabase()->getDocument($new->getCollection(), $new->getId()); - // return $document; + // // $this->assertNotContains('guests', $new->getRead()); + // // $this->assertNotContains('guests', $new->getWrite()); + // // $this->assertNotContains('guests', $new->getCreate()); + // // $this->assertNotContains('guests', $new->getUpdate()); + // // $this->assertNotContains('guests', $new->getDelete()); // } -} \ No newline at end of file + + /** + * @depends testCreateDocuments + */ + public function testUpdateDocumentsBatch(array $documents) + { + + foreach ($documents as $document) { + $document + ->setAttribute('string', 'textđź“ť updated') + ->setAttribute('integer', 6) + ->setAttribute('float', 5.56) + ->setAttribute('boolean', false) + ->setAttribute('colors', 'red', Document::SET_TYPE_APPEND) + ->setAttribute('with-dash', 'Works'); + } + + $start = microtime(true); + $updatedDocuments = static::getDatabase()->updateDocuments($document->getCollection(), $documents, count($documents)); + $end = microtime(true); + var_dump('Batch update time for ' . count($documents) . ' documents : ' . ($end - $start)); + } +} From 3565f269c240b8a5bd426755a363ae4154c12eb6 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 11 Aug 2023 18:59:12 -0400 Subject: [PATCH 06/14] Fix permission batch updates --- src/Database/Adapter/MariaDB.php | 233 ++++++++++++++----------- src/Database/Database.php | 52 ++++-- tests/Database/Adapter/MariaDBTest.php | 153 +--------------- tests/Database/Adapter/MySQLTest.php | 2 + tests/Database/Base.php | 98 +++++++++++ 5 files changed, 267 insertions(+), 271 deletions(-) diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index 259adfd4d..ff35634a9 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -428,6 +428,9 @@ public function createDocument(string $collection, Document $document): Document return $document; } + /** + * @throws Duplicate + */ public function createDocuments(string $collection, array $documents, int $batchSize = Database::INSERT_BATCH_SIZE): array { if (empty($documents)) { @@ -438,7 +441,7 @@ public function createDocuments(string $collection, array $documents, int $batch try { $name = $this->filter($collection); - $batches = array_chunk($documents, $batchSize); + $batches = \array_chunk($documents, $batchSize); foreach ($batches as $batch) { $bindIndex = 0; @@ -451,28 +454,29 @@ public function createDocuments(string $collection, array $documents, int $batch $attributes['_uid'] = $document->getId(); $attributes['_createdAt'] = $document->getCreatedAt(); $attributes['_updatedAt'] = $document->getUpdatedAt(); - $attributes['_permissions'] = json_encode($document->getPermissions()); + $attributes['_permissions'] = \json_encode($document->getPermissions()); - $columns = array_map(function ($attribute) { - $column = $this->filter($attribute); - return "`$column`"; - }, array_keys($attributes)); - $columns = '(' . implode(', ', $columns) . ')'; + $columns = []; + foreach (\array_keys($attributes) as $key => $attribute) { + $columns[$key] = "`{$this->filter($attribute)}`"; + } + + $columns = '(' . \implode(', ', $columns) . ')'; $bindKeys = []; - foreach ($attributes as $attribute => $value) { - if (is_array($value)) { // arrays & objects should be saved as strings - $value = json_encode($value); + foreach ($attributes as $value) { + if (\is_array($value)) { + $value = \json_encode($value); } - $value = (is_bool($value)) ? (int)$value : $value; + $value = (\is_bool($value)) ? (int)$value : $value; $bindKey = 'key_' . $bindIndex; $bindKeys[] = ':' . $bindKey; $bindValues[$bindKey] = $value; $bindIndex++; } - $batchKeys[] = '(' . implode(', ', $bindKeys) . ')'; + $batchKeys[] = '(' . \implode(', ', $bindKeys) . ')'; foreach (Database::PERMISSIONS as $type) { foreach ($document->getPermissionsByType($type) as $permission) { $permission = \str_replace('"', '', $permission); @@ -481,9 +485,10 @@ public function createDocuments(string $collection, array $documents, int $batch } } - $stmt = $this->getPDO() - ->prepare("INSERT INTO {$this->getSQLTable($name)} - $columns VALUES " . implode(', ', $batchKeys)); + $stmt = $this->getPDO()->prepare(" + INSERT INTO {$this->getSQLTable($name)} {$columns} + VALUES " . \implode(', ', $batchKeys) + ); foreach ($bindValues as $key => $value) { $stmt->bindValue($key, $value, $this->getPDOType($value)); @@ -492,11 +497,11 @@ public function createDocuments(string $collection, array $documents, int $batch $stmt->execute(); if (!empty($permissions)) { - $queryPermissions = "INSERT INTO {$this->getSQLTable($name . '_perms')} (_type, _permission, _document) VALUES " . implode(', ', $permissions); - $stmtPermissions = $this->getPDO()->prepare($queryPermissions); - if (isset($stmtPermissions)) { - $stmtPermissions->execute(); - } + $stmtPermissions = $this->getPDO()->prepare(" + INSERT INTO {$this->getSQLTable($name . '_perms')} (_type, _permission, _document) + VALUES " . \implode(', ', $permissions) + ); + $stmtPermissions?->execute(); } } @@ -505,21 +510,17 @@ public function createDocuments(string $collection, array $documents, int $batch } return $documents; + } catch (PDOException $e) { $this->getPDO()->rollBack(); - switch ($e->getCode()) { - case 1062: - case 23000: - throw new Duplicate('Duplicated document: ' . $e->getMessage()); - default: - throw $e; - } + throw match ($e->getCode()) { + 1062, 23000 => new Duplicate('Duplicated document: ' . $e->getMessage()), + default => $e, + }; } } - - /** * Update Document * @@ -544,12 +545,14 @@ public function updateDocument(string $collection, Document $document): Document * Get current permissions from the database */ $permissionsStmt = $this->getPDO()->prepare(" - SELECT _type, _permission - FROM {$this->getSQLTable($name . '_perms')} p - WHERE p._document = :_uid + SELECT _type, _permission + FROM {$this->getSQLTable($name . '_perms')} + WHERE _document = :_uid "); + $permissionsStmt->bindValue(':_uid', $document->getId()); $permissionsStmt->execute(); + $permissions = $permissionsStmt->fetchAll(); $initial = []; @@ -708,7 +711,9 @@ public function updateDocument(string $collection, Document $document): Document return $document; } - + /** + * @throws Duplicate + */ public function updateDocuments(string $collection, array $documents, int $batchSize = Database::INSERT_BATCH_SIZE): array { if (empty($documents)) { @@ -719,28 +724,34 @@ public function updateDocuments(string $collection, array $documents, int $batch try { $name = $this->filter($collection); - $batches = array_chunk($documents, $batchSize); + $batches = \array_chunk($documents, $batchSize); foreach ($batches as $batch) { $bindIndex = 0; $batchKeys = []; $bindValues = []; - foreach ($batch as $document) { + $removeQuery = ''; + $removeBindValues = []; + + $addQuery = ''; + $addBindValues = []; + + foreach ($batch as $index => $document) { $attributes = $document->getAttributes(); $attributes['_uid'] = $document->getId(); $attributes['_createdAt'] = $document->getCreatedAt(); $attributes['_updatedAt'] = $document->getUpdatedAt(); $attributes['_permissions'] = json_encode($document->getPermissions()); - $columns = array_map(function ($attribute) { + $columns = \array_map(function ($attribute) { return "`" . $this->filter($attribute) . "`"; - }, array_keys($attributes)); + }, \array_keys($attributes)); $bindKeys = []; - foreach ($attributes as $attribute => $value) { - if (is_array($value)) { + foreach ($attributes as $value) { + if (\is_array($value)) { $value = json_encode($value); } $value = (is_bool($value)) ? (int)$value : $value; @@ -755,8 +766,8 @@ public function updateDocuments(string $collection, array $documents, int $batch // Permissions logic $permissionsStmt = $this->getPDO()->prepare(" SELECT _type, _permission - FROM {$this->getSQLTable($name . '_perms')} p - WHERE p._document = :_uid + FROM {$this->getSQLTable($name . '_perms')} + WHERE _document = :_uid "); $permissionsStmt->bindValue(':_uid', $document->getId()); $permissionsStmt->execute(); @@ -767,7 +778,7 @@ public function updateDocuments(string $collection, array $documents, int $batch $initial[$type] = []; } - $permissions = array_reduce($permissions, function (array $carry, array $item) { + $permissions = \array_reduce($permissions, function (array $carry, array $item) { $carry[$item['_type']][] = $item['_permission']; return $carry; }, $initial); @@ -781,80 +792,82 @@ public function updateDocuments(string $collection, array $documents, int $batch } } - // Query to remove permissions - $removeQuery = ''; + // Build inner query to remove permissions if (!empty($removals)) { - $removeQuery = 'AND ('; - foreach ($removals as $type => $permissions) { + foreach ($removals as $type => $permissionsToRemove) { + $bindKey = 'uid_' . $index; + $removeBindKeys[] = ':uid_' . $index; + $removeBindValues[$bindKey] = $document->getId(); + $removeQuery .= "( - _type = '{$type}' - AND _permission IN (" . implode(', ', array_map(fn (string $i) => ":_remove_{$type}_{$i}", array_keys($permissions))) . ") + _document = :uid_{$index} + AND _type = '{$type}' + AND _permission IN (" . implode(', ', \array_map(function (string $i) use ($permissionsToRemove, $index, $type, &$removeBindKeys, &$removeBindValues) { + $bindKey = 'remove_' . $type . '_' . $index . '_' . $i; + $removeBindKeys[] = ':' . $bindKey; + $removeBindValues[$bindKey] = $permissionsToRemove[$i]; + + return ':' . $bindKey; + }, \array_keys($permissionsToRemove))) . + ") )"; - if ($type !== array_key_last($removals)) { + + if ($type !== \array_key_last($removals)) { $removeQuery .= ' OR '; } } - } - if (!empty($removeQuery)) { - $removeQuery .= ')'; - $stmtRemovePermissions = $this->getPDO() - ->prepare(" - DELETE - FROM {$this->getSQLTable($name . '_perms')} - WHERE - _document = :_uid - {$removeQuery} - "); - $stmtRemovePermissions->bindValue(':_uid', $document->getId()); - - foreach ($removals as $type => $permissions) { - foreach ($permissions as $i => $permission) { - $stmtRemovePermissions->bindValue(":_remove_{$type}_{$i}", $permission); - } + + if ($index !== \array_key_last($batch)) { + $removeQuery .= ' OR '; } } // Get added Permissions $additions = []; foreach (Database::PERMISSIONS as $type) { - $diff = array_diff($document->getPermissionsByType($type), $permissions[$type]); + $diff = \array_diff($document->getPermissionsByType($type), $permissions[$type]); if (!empty($diff)) { $additions[$type] = $diff; } } - // Query to add permissions + // Build inner query to add permissions if (!empty($additions)) { - $values = []; - foreach ($additions as $type => $permissions) { - foreach ($permissions as $i => $_) { - $values[] = "( :_uid, '{$type}', :_add_{$type}_{$i} )"; - } - } + foreach ($additions as $type => $permissionsToAdd) { + foreach ($permissionsToAdd as $i => $permission) { + $bindKey = 'uid_' . $index; + $addBindValues[$bindKey] = $document->getId(); - $stmtAddPermissions = $this->getPDO() - ->prepare( - " - INSERT INTO {$this->getSQLTable($name . '_perms')} - (_document, _type, _permission) VALUES " . implode(', ', $values) - ); - - $stmtAddPermissions->bindValue(":_uid", $document->getId()); - foreach ($additions as $type => $permissions) { - foreach ($permissions as $i => $permission) { - $stmtAddPermissions->bindValue(":_add_{$type}_{$i}", $permission); + $bindKey = 'add_' . $type . '_' . $index . '_' . $i; + $addBindValues[$bindKey] = $permission; + + $addQuery .= "(:uid_{$index}, '{$type}', :{$bindKey})"; + + if ($i !== \array_key_last($permissionsToAdd) || $type !== \array_key_last($additions)) { + $addQuery .= ', '; + } } } + if ($index !== \array_key_last($batch)) { + $addQuery .= ', '; + } } } - $updateClause = implode(', ', array_map(function ($column) { - return "$column=VALUES($column)"; - }, array_slice($columns, 1))); // Exclude UID + $updateClause = ''; + for ($i = 0; $i < \count($columns); $i++) { + $column = $columns[$i]; + if (!empty($updateClause)) { + $updateClause .= ', '; + } + $updateClause .= "{$column} = VALUES({$column})"; + } - $stmt = $this->getPDO() - ->prepare("INSERT INTO {$this->getSQLTable($name)} - (" . implode(", ", $columns) . ") VALUES " . implode(', ', $batchKeys) . " ON DUPLICATE KEY UPDATE $updateClause"); + $stmt = $this->getPDO()->prepare(" + INSERT INTO {$this->getSQLTable($name)} (" . \implode(", ", $columns) . ") + VALUES " . \implode(', ', $batchKeys) . " + ON DUPLICATE KEY UPDATE $updateClause + "); foreach ($bindValues as $key => $value) { $stmt->bindValue($key, $value, $this->getPDOType($value)); @@ -862,14 +875,30 @@ public function updateDocuments(string $collection, array $documents, int $batch $stmt->execute(); - // Execute permissions statements for each document in the batch - if (!empty($removals)) { - var_dump($stmtRemovePermissions->queryString); + if (!empty($removeQuery)) { + $stmtRemovePermissions = $this->getPDO()->prepare(" + DELETE + FROM {$this->getSQLTable($name . '_perms')} + WHERE ({$removeQuery}) + "); + + foreach ($removeBindValues as $key => $value) { + $stmtRemovePermissions->bindValue($key, $value, $this->getPDOType($value)); + } + $stmtRemovePermissions->execute(); } - if (!empty($additions)) { - var_dump($stmtAddPermissions->queryString); + if (!empty($addQuery)) { + $stmtAddPermissions = $this->getPDO()->prepare(" + INSERT INTO {$this->getSQLTable($name . '_perms')} (`_document`, `_type`, `_permission`) + VALUES {$addQuery} + "); + + foreach ($addBindValues as $key => $value) { + $stmtAddPermissions->bindValue($key, $value, $this->getPDOType($value)); + } + $stmtAddPermissions->execute(); } } @@ -881,18 +910,14 @@ public function updateDocuments(string $collection, array $documents, int $batch return $documents; } catch (PDOException $e) { $this->getPDO()->rollBack(); - switch ($e->getCode()) { - case 1062: - case 23000: - throw new Duplicate('Duplicated document: ' . $e->getMessage()); - default: - throw $e; - } + throw match ($e->getCode()) { + 1062, 23000 => new Duplicate('Duplicated document: ' . $e->getMessage()), + default => $e, + }; } } - /** * Increase or decrease an attribute value * diff --git a/src/Database/Database.php b/src/Database/Database.php index c4db56e3c..41c49ea33 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -1464,11 +1464,17 @@ public function createDocument(string $collection, Document $document): Document */ public function createDocuments(string $collection, array $documents, int $batchSize = self::INSERT_BATCH_SIZE): array { + if (empty($documents)) { + return []; + } + $collection = $this->silent(fn() => $this->getCollection($collection)); + + $time = DateTime::now(); - $documents = array_map(function($document) use ($collection, $time){ + foreach ($documents as $key => $document) { $document ->setAttribute('$id', empty($document->getId()) ? ID::unique() : $document->getId()) ->setAttribute('$collection', $collection->getId()) @@ -1478,23 +1484,20 @@ public function createDocuments(string $collection, array $documents, int $batch $document = $this->encode($collection, $document); $validator = new Structure($collection); - if (!$validator->isValid($document)) { throw new StructureException($validator->getDescription()); } - return $document; - }, $documents); + $documents[$key] = $document; + } $documents = $this->adapter->createDocuments($collection->getId(), $documents, $batchSize); - $documents = array_map(function($document) use ($collection){ - $document = $this->decode($collection, $document); - - $this->trigger(self::EVENT_DOCUMENT_CREATE, $document); + foreach ($documents as $key => $document) { + $documents[$key] = $this->decode($collection, $document); + } - return $document; - }, $documents); + $this->trigger(self::EVENT_DOCUMENTS_CREATE, $documents); return $documents; } @@ -1546,6 +1549,11 @@ public function updateDocument(string $collection, string $id, Document $documen return $document; } + /** + * @throws AuthorizationException + * @throws Throwable + * @throws StructureException + */ public function updateDocuments(string $collection, array $documents, int $batchSize): array { if (empty($documents)) { @@ -1563,6 +1571,17 @@ public function updateDocuments(string $collection, array $documents, int $batch $document->setAttribute('$updatedAt', $time); $document = $this->encode($collection, $document); + $old = Authorization::skip(fn() => $this->silent(fn() => $this->getDocument( + $collection->getId(), + $document->getId()) + )); + + $validator = new Authorization(self::PERMISSION_UPDATE); + if ($collection->getId() !== self::METADATA + && !$validator->isValid($old->getUpdate())) { + throw new AuthorizationException($validator->getDescription()); + } + $validator = new Structure($collection); if (!$validator->isValid($document)) { throw new StructureException($validator->getDescription()); @@ -1570,11 +1589,14 @@ public function updateDocuments(string $collection, array $documents, int $batch } $documents = $this->adapter->updateDocuments($collection->getId(), $documents, $batchSize); - - // foreach ($documents as $document) { - // $document = $this->decode($collection, $document); - // $this->trigger(self::EVENT_DOCUMENT_UPDATE, $document); - // } + + foreach ($documents as $key => $document) { + $documents[$key] = $this->decode($collection, $document); + + $this->cache->purge('cache-' . $this->getNamespace() . ':' . $collection->getId() . ':' . $document->getId() . ':*'); + } + + $this->trigger(self::EVENT_DOCUMENTS_UPDATE, $documents); return $documents; } diff --git a/tests/Database/Adapter/MariaDBTest.php b/tests/Database/Adapter/MariaDBTest.php index 5e56001aa..32b9375fb 100644 --- a/tests/Database/Adapter/MariaDBTest.php +++ b/tests/Database/Adapter/MariaDBTest.php @@ -11,6 +11,7 @@ use Utopia\Database\Document; use Utopia\Database\Helpers\Permission; use Utopia\Database\Helpers\Role; +use Utopia\Database\Query; use Utopia\Tests\Base; class MariaDBTest extends Base @@ -60,156 +61,4 @@ static function getDatabase(): Database return self::$database = $database; } - - - public function testCreateDocuments() - { - $count = 1000; - $collection = 'testCreateDocuments'; - - static::getDatabase()->createCollection($collection); - - $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'string', Database::VAR_STRING, 128, true)); - $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'integer', Database::VAR_INTEGER, 0, true)); - $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'bigint', Database::VAR_INTEGER, 8, true)); - $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'float', Database::VAR_FLOAT, 0, true)); - $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'boolean', Database::VAR_BOOLEAN, 0, true)); - $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'colors', Database::VAR_STRING, 32, true, null, true, true)); - $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'empty', Database::VAR_STRING, 32, false, null, true, true)); - $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'with-dash', Database::VAR_STRING, 128, false, null)); - - - // Create an array of documents with random attributes. Dont use the createDocument function - $documents = []; - - for ($i = 0; $i < $count; $i++) { - $documents[] = new Document([ - '$permissions' => [ - Permission::read(Role::any()), - Permission::create(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()), - ], - 'string' => 'textđź“ť', - 'integer' => 5, - 'bigint' => 8589934592, // 2^33 - 'float' => 5.55, - 'boolean' => true, - 'colors' => ['pink', 'green', 'blue'], - 'empty' => [], - 'with-dash' => 'Works', - ]); - } - - $res = static::getDatabase()->createDocuments($collection, $documents, $count); - - $this->assertEquals($count, count($res)); - - foreach ($res as $document) { - $this->assertNotEmpty(true, $document->getId()); - $this->assertIsString($document->getAttribute('string')); - $this->assertEquals('textđź“ť', $document->getAttribute('string')); // Also makes sure an emoji is working - $this->assertIsInt($document->getAttribute('integer')); - $this->assertEquals(5, $document->getAttribute('integer')); - $this->assertIsInt($document->getAttribute('bigint')); - $this->assertEquals(8589934592, $document->getAttribute('bigint')); - $this->assertIsFloat($document->getAttribute('float')); - $this->assertEquals(5.55, $document->getAttribute('float')); - $this->assertIsBool($document->getAttribute('boolean')); - $this->assertEquals(true, $document->getAttribute('boolean')); - $this->assertIsArray($document->getAttribute('colors')); - $this->assertEquals(['pink', 'green', 'blue'], $document->getAttribute('colors')); - $this->assertEquals([], $document->getAttribute('empty')); - $this->assertEquals('Works', $document->getAttribute('with-dash')); - } - - return $res; - } - - // /** - // * @depends testCreateDocuments - // */ - // public function testUpdateDocumentsSequential(array $documents) - // { - - // $start = microtime(true); - // foreach ($documents as $document) { - // $document - // ->setAttribute('string', 'textđź“ť updated') - // ->setAttribute('integer', 6) - // ->setAttribute('float', 5.56) - // ->setAttribute('boolean', false) - // ->setAttribute('colors', 'red', Document::SET_TYPE_APPEND) - // ->setAttribute('with-dash', 'Works'); - - // $new = static::getDatabase()->updateDocument($document->getCollection(), $document->getId(), $document); - // } - // $end = microtime(true); - // var_dump('Sequential update time for ' . count($documents) . ' documents : ' . ($end - $start)); - - // $this->assertNotEmpty(true, $new->getId()); - // $this->assertIsString($new->getAttribute('string')); - // $this->assertEquals('textđź“ť updated', $new->getAttribute('string')); - // $this->assertIsInt($new->getAttribute('integer')); - // $this->assertEquals(6, $new->getAttribute('integer')); - // $this->assertIsFloat($new->getAttribute('float')); - // $this->assertEquals(5.56, $new->getAttribute('float')); - // $this->assertIsBool($new->getAttribute('boolean')); - // $this->assertEquals(false, $new->getAttribute('boolean')); - // $this->assertIsArray($new->getAttribute('colors')); - // $this->assertEquals(['pink', 'green', 'blue', 'red'], $new->getAttribute('colors')); - // $this->assertEquals('Works', $new->getAttribute('with-dash')); - - // // $oldPermissions = $document->getPermissions(); - - // // $new - // // ->setAttribute('$permissions', Permission::read(Role::guests()), Document::SET_TYPE_APPEND) - // // ->setAttribute('$permissions', Permission::create(Role::guests()), Document::SET_TYPE_APPEND) - // // ->setAttribute('$permissions', Permission::update(Role::guests()), Document::SET_TYPE_APPEND) - // // ->setAttribute('$permissions', Permission::delete(Role::guests()), Document::SET_TYPE_APPEND); - - // // $this->getDatabase()->updateDocument($new->getCollection(), $new->getId(), $new); - - // // $new = $this->getDatabase()->getDocument($new->getCollection(), $new->getId()); - - // // $this->assertContains('guests', $new->getRead()); - // // $this->assertContains('guests', $new->getWrite()); - // // $this->assertContains('guests', $new->getCreate()); - // // $this->assertContains('guests', $new->getUpdate()); - // // $this->assertContains('guests', $new->getDelete()); - - // // $new->setAttribute('$permissions', $oldPermissions); - - // // $this->getDatabase()->updateDocument($new->getCollection(), $new->getId(), $new); - - // // $new = $this->getDatabase()->getDocument($new->getCollection(), $new->getId()); - - // // $this->assertNotContains('guests', $new->getRead()); - // // $this->assertNotContains('guests', $new->getWrite()); - // // $this->assertNotContains('guests', $new->getCreate()); - // // $this->assertNotContains('guests', $new->getUpdate()); - // // $this->assertNotContains('guests', $new->getDelete()); - // } - - /** - * @depends testCreateDocuments - */ - public function testUpdateDocumentsBatch(array $documents) - { - - foreach ($documents as $document) { - $document - ->setAttribute('string', 'textđź“ť updated') - ->setAttribute('integer', 6) - ->setAttribute('float', 5.56) - ->setAttribute('boolean', false) - ->setAttribute('colors', 'red', Document::SET_TYPE_APPEND) - ->setAttribute('with-dash', 'Works'); - } - - $start = microtime(true); - $updatedDocuments = static::getDatabase()->updateDocuments($document->getCollection(), $documents, count($documents)); - $end = microtime(true); - var_dump('Batch update time for ' . count($documents) . ' documents : ' . ($end - $start)); - } } diff --git a/tests/Database/Adapter/MySQLTest.php b/tests/Database/Adapter/MySQLTest.php index b06176a2b..ac41d6d6a 100644 --- a/tests/Database/Adapter/MySQLTest.php +++ b/tests/Database/Adapter/MySQLTest.php @@ -64,6 +64,8 @@ static function getDatabase(): Database $database->setDefaultDatabase('utopiaTests'); $database->setNamespace('myapp_'.uniqid()); + $database->create(); + return self::$database = $database; } } \ No newline at end of file diff --git a/tests/Database/Base.php b/tests/Database/Base.php index a27bb9a25..15b69e489 100644 --- a/tests/Database/Base.php +++ b/tests/Database/Base.php @@ -605,6 +605,51 @@ public function testCreateDocument() return $document; } + public function testCreateDocuments() + { + $count = 3; + $collection = 'testCreateDocuments'; + + static::getDatabase()->createCollection($collection); + + $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'string', Database::VAR_STRING, 128, true)); + $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'integer', Database::VAR_INTEGER, 0, true)); + $this->assertEquals(true, static::getDatabase()->createAttribute($collection, 'bigint', Database::VAR_INTEGER, 8, true)); + + // Create an array of documents with random attributes. Don't use the createDocument function + $documents = []; + + for ($i = 0; $i < $count; $i++) { + $documents[] = new Document([ + '$permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + 'string' => 'textđź“ť', + 'integer' => 5, + 'bigint' => 8589934592, // 2^33 + ]); + } + + $documents = static::getDatabase()->createDocuments($collection, $documents, 3); + + $this->assertEquals($count, count($documents)); + + foreach ($documents as $document) { + $this->assertNotEmpty(true, $document->getId()); + $this->assertIsString($document->getAttribute('string')); + $this->assertEquals('textđź“ť', $document->getAttribute('string')); // Also makes sure an emoji is working + $this->assertIsInt($document->getAttribute('integer')); + $this->assertEquals(5, $document->getAttribute('integer')); + $this->assertIsInt($document->getAttribute('bigint')); + $this->assertEquals(8589934592, $document->getAttribute('bigint')); + } + + return $documents; + } + public function testRespectNulls() { static::getDatabase()->createCollection('documents_nulls'); @@ -921,6 +966,58 @@ public function testUpdateDocument(Document $document) return $document; } + /** + * @depends testCreateDocuments + */ + public function testUpdateDocuments(array $documents) + { + $collection = 'testCreateDocuments'; + + foreach ($documents as $document) { + $document + ->setAttribute('string', 'textđź“ť updated') + ->setAttribute('integer', 6) + ->setAttribute('$permissions', [ + Permission::read(Role::users()), + Permission::create(Role::users()), + Permission::update(Role::users()), + Permission::delete(Role::users()), + ]); + } + + $start = microtime(true); + + $documents = static::getDatabase()->updateDocuments( + $collection, + $documents, + \count($documents) + ); + + $end = microtime(true); + + \var_dump('Batch update time for ' . \count($documents) . ' documents : ' . ($end - $start)); + + foreach ($documents as $document) { + $this->assertEquals('textđź“ť updated', $document->getAttribute('string')); + $this->assertEquals(6, $document->getAttribute('integer')); + } + + $documents = static::getDatabase()->find($collection, [ + Query::limit(\count($documents)) + ]); + + foreach ($documents as $document) { + $this->assertEquals('textđź“ť updated', $document->getAttribute('string')); + $this->assertEquals(6, $document->getAttribute('integer')); + $this->assertEquals([ + Permission::read(Role::users()), + Permission::create(Role::users()), + Permission::update(Role::users()), + Permission::delete(Role::users()), + ], $document->getAttribute('$permissions')); + } + } + /** * @depends testGetDocument */ @@ -3506,6 +3603,7 @@ public function testWritePermissions() $this->assertEquals('newCat', $docs[0]['type']); } + public function testEvents() { Authorization::skip(function () { From 1662d195a76aaf845aca8e1aeb2175992cd1520a Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 11 Aug 2023 18:59:28 -0400 Subject: [PATCH 07/14] Add postgres batch create/update --- src/Database/Adapter/Postgres.php | 302 +++++++++++++++++++++++- tests/Database/Adapter/PostgresTest.php | 2 + 2 files changed, 303 insertions(+), 1 deletion(-) diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index c0c77c700..27152131b 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -98,7 +98,7 @@ public function createCollection(string $name, array $attributes = [], array $in //INDEX (\"_createdAt\"), //INDEX (\"_updatedAt\") $stmtIndex = $this->getPDO() - ->prepare("CREATE UNIQUE INDEX \"index_{$namespace}_{$id}_uid\" on {$this->getSQLTable($id)} (LOWER(_uid));"); + ->prepare("CREATE UNIQUE INDEX \"index_{$namespace}_{$id}_uid\" ON {$this->getSQLTable($id)} (_uid);"); try { $stmt->execute(); $stmtIndex->execute(); @@ -437,6 +437,99 @@ public function createDocument(string $collection, Document $document): Document return $document; } + /** + * @throws Duplicate + */ + public function createDocuments(string $collection, array $documents, int $batchSize = Database::INSERT_BATCH_SIZE): array + { + if (empty($documents)) { + return $documents; + } + + $this->getPDO()->beginTransaction(); + + try { + $name = $this->filter($collection); + $batches = \array_chunk($documents, $batchSize); + + foreach ($batches as $batch) { + $bindIndex = 0; + $batchKeys = []; + $bindValues = []; + $permissions = []; + + foreach ($batch as $document) { + $attributes = $document->getAttributes(); + $attributes['_uid'] = $document->getId(); + $attributes['_createdAt'] = $document->getCreatedAt(); + $attributes['_updatedAt'] = $document->getUpdatedAt(); + $attributes['_permissions'] = \json_encode($document->getPermissions()); + + $columns = []; + foreach (\array_keys($attributes) as $key => $attribute) { + $columns[$key] = "\"{$this->filter($attribute)}\""; + } + + $columns = '(' . \implode(', ', $columns) . ')'; + + $bindKeys = []; + + foreach ($attributes as $value) { + if (\is_array($value)) { + $value = \json_encode($value); + } + $value = (\is_bool($value)) ? (int)$value : $value; + $bindKey = 'key_' . $bindIndex; + $bindKeys[] = ':' . $bindKey; + $bindValues[$bindKey] = $value; + $bindIndex++; + } + + $batchKeys[] = '(' . \implode(', ', $bindKeys) . ')'; + foreach (Database::PERMISSIONS as $type) { + foreach ($document->getPermissionsByType($type) as $permission) { + $permission = \str_replace('"', '', $permission); + $permissions[] = "('{$type}', '{$permission}', '{$document->getId()}')"; + } + } + } + + $stmt = $this->getPDO()->prepare(" + INSERT INTO {$this->getSQLTable($name)} {$columns} + VALUES " . \implode(', ', $batchKeys) + ); + + foreach ($bindValues as $key => $value) { + $stmt->bindValue($key, $value, $this->getPDOType($value)); + } + + $stmt->execute(); + + if (!empty($permissions)) { + $stmtPermissions = $this->getPDO()->prepare(" + INSERT INTO {$this->getSQLTable($name . '_perms')} (_type, _permission, _document) + VALUES " . \implode(', ', $permissions) + ); + $stmtPermissions?->execute(); + } + } + + if (!$this->getPDO()->commit()) { + throw new Exception('Failed to commit transaction'); + } + + return $documents; + + } catch (PDOException $e) { + $this->getPDO()->rollBack(); + + throw match ($e->getCode()) { + 1062, 23000 => new Duplicate('Duplicated document: ' . $e->getMessage()), + default => $e, + }; + } + } + /** * Update Document * @@ -622,6 +715,213 @@ public function updateDocument(string $collection, Document $document): Document return $document; } + /** + * @throws Duplicate + */ + public function updateDocuments(string $collection, array $documents, int $batchSize = Database::INSERT_BATCH_SIZE): array + { + if (empty($documents)) { + return $documents; + } + + $this->getPDO()->beginTransaction(); + + try { + $name = $this->filter($collection); + $batches = \array_chunk($documents, $batchSize); + + foreach ($batches as $batch) { + $bindIndex = 0; + $batchKeys = []; + $bindValues = []; + + $removeQuery = ''; + $removeBindValues = []; + + $addQuery = ''; + $addBindValues = []; + + foreach ($batch as $index => $document) { + $attributes = $document->getAttributes(); + $attributes['_uid'] = $document->getId(); + $attributes['_createdAt'] = $document->getCreatedAt(); + $attributes['_updatedAt'] = $document->getUpdatedAt(); + $attributes['_permissions'] = json_encode($document->getPermissions()); + + $columns = \array_map(function ($attribute) { + return '"' . $this->filter($attribute) . '"'; + }, \array_keys($attributes)); + + $bindKeys = []; + + foreach ($attributes as $value) { + if (\is_array($value)) { + $value = json_encode($value); + } + $value = (is_bool($value)) ? (int)$value : $value; + $bindKey = 'key_' . $bindIndex; + $bindKeys[] = ':' . $bindKey; + $bindValues[$bindKey] = $value; + $bindIndex++; + } + + $batchKeys[] = '(' . implode(', ', $bindKeys) . ')'; + + // Permissions logic + $permissionsStmt = $this->getPDO()->prepare(" + SELECT _type, _permission + FROM {$this->getSQLTable($name . '_perms')} + WHERE _document = :_uid + "); + $permissionsStmt->bindValue(':_uid', $document->getId()); + $permissionsStmt->execute(); + $permissions = $permissionsStmt->fetchAll(PDO::FETCH_ASSOC); + + $initial = []; + foreach (Database::PERMISSIONS as $type) { + $initial[$type] = []; + } + + $permissions = \array_reduce($permissions, function (array $carry, array $item) { + $carry[$item['_type']][] = $item['_permission']; + return $carry; + }, $initial); + + // Get removed Permissions + $removals = []; + foreach (Database::PERMISSIONS as $type) { + $diff = array_diff($permissions[$type], $document->getPermissionsByType($type)); + if (!empty($diff)) { + $removals[$type] = $diff; + } + } + + // Build inner query to remove permissions + if (!empty($removals)) { + foreach ($removals as $type => $permissionsToRemove) { + $bindKey = 'uid_' . $index; + $removeBindKeys[] = ':uid_' . $index; + $removeBindValues[$bindKey] = $document->getId(); + + $removeQuery .= "( + _document = :uid_{$index} + AND _type = '{$type}' + AND _permission IN (" . implode(', ', \array_map(function (string $i) use ($permissionsToRemove, $index, $type, &$removeBindKeys, &$removeBindValues) { + $bindKey = 'remove_' . $type . '_' . $index . '_' . $i; + $removeBindKeys[] = ':' . $bindKey; + $removeBindValues[$bindKey] = $permissionsToRemove[$i]; + + return ':' . $bindKey; + }, \array_keys($permissionsToRemove))) . + ") + )"; + + if ($type !== \array_key_last($removals)) { + $removeQuery .= ' OR '; + } + } + + if ($index !== \array_key_last($batch)) { + $removeQuery .= ' OR '; + } + } + + // Get added Permissions + $additions = []; + foreach (Database::PERMISSIONS as $type) { + $diff = \array_diff($document->getPermissionsByType($type), $permissions[$type]); + if (!empty($diff)) { + $additions[$type] = $diff; + } + } + + // Build inner query to add permissions + if (!empty($additions)) { + foreach ($additions as $type => $permissionsToAdd) { + foreach ($permissionsToAdd as $i => $permission) { + $bindKey = 'uid_' . $index; + $addBindValues[$bindKey] = $document->getId(); + + $bindKey = 'add_' . $type . '_' . $index . '_' . $i; + $addBindValues[$bindKey] = $permission; + + $addQuery .= "(:uid_{$index}, '{$type}', :{$bindKey})"; + + if ($i !== \array_key_last($permissionsToAdd) || $type !== \array_key_last($additions)) { + $addQuery .= ', '; + } + } + } + if ($index !== \array_key_last($batch)) { + $addQuery .= ', '; + } + } + } + + $updateClause = ''; + for ($i = 0; $i < \count($columns); $i++) { + $column = $columns[$i]; + if (!empty($updateClause)) { + $updateClause .= ', '; + } + $updateClause .= "{$column} = excluded.{$column}"; + } + + $stmt = $this->getPDO()->prepare(" + INSERT INTO {$this->getSQLTable($name)} (" . \implode(", ", $columns) . ") + VALUES " . \implode(', ', $batchKeys) . " + ON CONFLICT (\"_uid\") DO UPDATE SET $updateClause + "); + + foreach ($bindValues as $key => $value) { + $stmt->bindValue($key, $value, $this->getPDOType($value)); + } + + $stmt->execute(); + + if (!empty($removeQuery)) { + $stmtRemovePermissions = $this->getPDO()->prepare(" + DELETE + FROM {$this->getSQLTable($name . '_perms')} + WHERE ({$removeQuery}) + "); + + foreach ($removeBindValues as $key => $value) { + $stmtRemovePermissions->bindValue($key, $value, $this->getPDOType($value)); + } + + $stmtRemovePermissions->execute(); + } + + if (!empty($addQuery)) { + $stmtAddPermissions = $this->getPDO()->prepare(" + INSERT INTO {$this->getSQLTable($name . '_perms')} (_document, _type, _permission) + VALUES {$addQuery} + "); + + foreach ($addBindValues as $key => $value) { + $stmtAddPermissions->bindValue($key, $value, $this->getPDOType($value)); + } + + $stmtAddPermissions->execute(); + } + } + + if (!$this->getPDO()->commit()) { + throw new Exception('Failed to commit transaction'); + } + + return $documents; + } catch (PDOException $e) { + $this->getPDO()->rollBack(); + + throw match ($e->getCode()) { + 1062, 23000 => new Duplicate('Duplicated document: ' . $e->getMessage()), + default => $e, + }; + } + } + /** * Increase or decrease an attribute value * diff --git a/tests/Database/Adapter/PostgresTest.php b/tests/Database/Adapter/PostgresTest.php index a797bfa38..4eddea981 100644 --- a/tests/Database/Adapter/PostgresTest.php +++ b/tests/Database/Adapter/PostgresTest.php @@ -51,6 +51,8 @@ static function getDatabase(): Database $database->setDefaultDatabase('utopiaTests'); $database->setNamespace('myapp_'.uniqid()); + $database->create(); + return self::$database = $database; } } \ No newline at end of file From 9c0bd440d22e1d403efe63f40d030687fa606b7c Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 11 Aug 2023 19:19:08 -0400 Subject: [PATCH 08/14] Add mongo batch create/update --- composer.json | 2 +- composer.lock | 18 +++++++------- src/Database/Adapter/Mongo.php | 43 ++++++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 9 deletions(-) diff --git a/composer.json b/composer.json index 90f4011ec..661b6842c 100755 --- a/composer.json +++ b/composer.json @@ -15,7 +15,7 @@ "php": ">=8.0", "utopia-php/framework": "0.*.*", "utopia-php/cache": "0.8.*", - "utopia-php/mongo": "0.0.2" + "utopia-php/mongo": "dev-feat-insert-many" }, "require-dev": { "ext-redis": "*", diff --git a/composer.lock b/composer.lock index c5e5c7744..0f638ab3d 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "21a731a53a90dda637bc28e7d7b771c0", + "content-hash": "dc6582e79b47c667aa0db1e3c68dabea", "packages": [ { "name": "composer/package-versions-deprecated", @@ -381,16 +381,16 @@ }, { "name": "utopia-php/mongo", - "version": "0.0.2", + "version": "dev-feat-insert-many", "source": { "type": "git", "url": "https://github.com/utopia-php/mongo.git", - "reference": "62f9a9c0201af91b6d0dd4f0aa8a335ec9b56a1e" + "reference": "4ab1a0c2a4bbf178658e9aa21b219c4cd80dce90" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/mongo/zipball/62f9a9c0201af91b6d0dd4f0aa8a335ec9b56a1e", - "reference": "62f9a9c0201af91b6d0dd4f0aa8a335ec9b56a1e", + "url": "https://api.github.com/repos/utopia-php/mongo/zipball/4ab1a0c2a4bbf178658e9aa21b219c4cd80dce90", + "reference": "4ab1a0c2a4bbf178658e9aa21b219c4cd80dce90", "shasum": "" }, "require": { @@ -435,9 +435,9 @@ ], "support": { "issues": "https://github.com/utopia-php/mongo/issues", - "source": "https://github.com/utopia-php/mongo/tree/0.0.2" + "source": "https://github.com/utopia-php/mongo/tree/feat-insert-many" }, - "time": "2022-11-08T11:58:46+00:00" + "time": "2023-08-11T22:55:31+00:00" } ], "packages-dev": [ @@ -4258,7 +4258,9 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": { + "utopia-php/mongo": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index 6025d2baf..daa69ff18 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -439,6 +439,32 @@ public function createDocument(string $collection, Document $document): Document return new Document($result); } + public function createDocuments(string $collection, array $documents, int $batchSize): array + { + $name = $this->getNamespace() . '_' . $this->filter($collection); + + $records = []; + foreach ($documents as $document) { + $document->removeAttribute('$internalId'); + + $record = $this->replaceChars('$', '_', (array)$document); + $record = $this->timeToMongo($record); + + $records[] = $this->removeNullKeys($record); + } + + $documents = $this->client->insertMany($name, $records); + + foreach ($documents as $index => $document) { + $documents[$index] = $this->replaceChars('_', '$', $this->client->toArray($document)); + $documents[$index] = $this->timeToDocument($documents[$index]); + + $documents[$index] = new Document($documents[$index]); + } + + return $documents; + } + /** * @throws Duplicate */ @@ -486,6 +512,23 @@ public function updateDocument(string $collection, Document $document): Document return $document; } + public function updateDocuments(string $collection, array $documents, int $batchSize): array + { + $name = $this->getNamespace() . '_' . $this->filter($collection); + + foreach ($documents as $index => $document) { + $document = $document->getArrayCopy(); + $document = $this->replaceChars('$', '_', $document); + $document = $this->timeToMongo($document); + + $this->client->update($name, ['_uid' => $document['_uid']], $document); + + $documents[$index] = new Document($document); + } + + return $documents; + } + /** * Increase or decrease an attribute value * From 920b0f2ce85ce4bb325584b6281bac8537226a95 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 14 Aug 2023 22:41:02 -0400 Subject: [PATCH 09/14] Update version --- composer.json | 2 +- composer.lock | 18 ++++++++---------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/composer.json b/composer.json index 661b6842c..bae8d628c 100755 --- a/composer.json +++ b/composer.json @@ -15,7 +15,7 @@ "php": ">=8.0", "utopia-php/framework": "0.*.*", "utopia-php/cache": "0.8.*", - "utopia-php/mongo": "dev-feat-insert-many" + "utopia-php/mongo": "0.3.*" }, "require-dev": { "ext-redis": "*", diff --git a/composer.lock b/composer.lock index 0f638ab3d..ab0abe94e 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "dc6582e79b47c667aa0db1e3c68dabea", + "content-hash": "88a3434eb0ff1981519116680e700cab", "packages": [ { "name": "composer/package-versions-deprecated", @@ -381,16 +381,16 @@ }, { "name": "utopia-php/mongo", - "version": "dev-feat-insert-many", + "version": "0.3.0", "source": { "type": "git", "url": "https://github.com/utopia-php/mongo.git", - "reference": "4ab1a0c2a4bbf178658e9aa21b219c4cd80dce90" + "reference": "d94717db8489f6c235ec7d1f05bc9f56db179277" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/mongo/zipball/4ab1a0c2a4bbf178658e9aa21b219c4cd80dce90", - "reference": "4ab1a0c2a4bbf178658e9aa21b219c4cd80dce90", + "url": "https://api.github.com/repos/utopia-php/mongo/zipball/d94717db8489f6c235ec7d1f05bc9f56db179277", + "reference": "d94717db8489f6c235ec7d1f05bc9f56db179277", "shasum": "" }, "require": { @@ -435,9 +435,9 @@ ], "support": { "issues": "https://github.com/utopia-php/mongo/issues", - "source": "https://github.com/utopia-php/mongo/tree/feat-insert-many" + "source": "https://github.com/utopia-php/mongo/tree/0.3.0" }, - "time": "2023-08-11T22:55:31+00:00" + "time": "2023-08-14T17:20:27+00:00" } ], "packages-dev": [ @@ -4258,9 +4258,7 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "utopia-php/mongo": 20 - }, + "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, "platform": { From 5875e8d4df486bd544da2d3aec3564a8fadb4008 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 15 Aug 2023 00:42:38 -0400 Subject: [PATCH 10/14] Fix postgres --- src/Database/Adapter/Postgres.php | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index 27152131b..9113136c9 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -98,7 +98,7 @@ public function createCollection(string $name, array $attributes = [], array $in //INDEX (\"_createdAt\"), //INDEX (\"_updatedAt\") $stmtIndex = $this->getPDO() - ->prepare("CREATE UNIQUE INDEX \"index_{$namespace}_{$id}_uid\" ON {$this->getSQLTable($id)} (_uid);"); + ->prepare("CREATE UNIQUE INDEX \"index_{$namespace}_{$id}_uid\" ON {$this->getSQLTable($id)} (LOWER(_uid));"); try { $stmt->execute(); $stmtIndex->execute(); @@ -422,11 +422,9 @@ public function createDocument(string $collection, Document $document): Document case 23505: $this->getPDO()->rollBack(); throw new Duplicate('Duplicated document: ' . $e->getMessage()); - break; default: throw $e; - break; } } @@ -870,7 +868,7 @@ public function updateDocuments(string $collection, array $documents, int $batch $stmt = $this->getPDO()->prepare(" INSERT INTO {$this->getSQLTable($name)} (" . \implode(", ", $columns) . ") VALUES " . \implode(', ', $batchKeys) . " - ON CONFLICT (\"_uid\") DO UPDATE SET $updateClause + ON CONFLICT (LOWER(_uid)) DO UPDATE SET $updateClause "); foreach ($bindValues as $key => $value) { From 3067c71d62c6651115cb399c14b2153b93e0f998 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 15 Aug 2023 01:15:49 -0400 Subject: [PATCH 11/14] Fix SQLite --- src/Database/Adapter/SQLite.php | 207 ++++++++++++++++++++++++++++++++ 1 file changed, 207 insertions(+) diff --git a/src/Database/Adapter/SQLite.php b/src/Database/Adapter/SQLite.php index 9dad4a543..34d959497 100644 --- a/src/Database/Adapter/SQLite.php +++ b/src/Database/Adapter/SQLite.php @@ -569,6 +569,213 @@ public function updateDocument(string $collection, Document $document): Document return $document; } + /** + * @throws Duplicate + */ + public function updateDocuments(string $collection, array $documents, int $batchSize = Database::INSERT_BATCH_SIZE): array + { + if (empty($documents)) { + return $documents; + } + + $this->getPDO()->beginTransaction(); + + try { + $name = $this->filter($collection); + $batches = \array_chunk($documents, $batchSize); + + foreach ($batches as $batch) { + $bindIndex = 0; + $batchKeys = []; + $bindValues = []; + + $removeQuery = ''; + $removeBindValues = []; + + $addQuery = ''; + $addBindValues = []; + + foreach ($batch as $index => $document) { + $attributes = $document->getAttributes(); + $attributes['_uid'] = $document->getId(); + $attributes['_createdAt'] = $document->getCreatedAt(); + $attributes['_updatedAt'] = $document->getUpdatedAt(); + $attributes['_permissions'] = json_encode($document->getPermissions()); + + $columns = \array_map(function ($attribute) { + return "`" . $this->filter($attribute) . "`"; + }, \array_keys($attributes)); + + $bindKeys = []; + + foreach ($attributes as $value) { + if (\is_array($value)) { + $value = json_encode($value); + } + $value = (is_bool($value)) ? (int)$value : $value; + $bindKey = 'key_' . $bindIndex; + $bindKeys[] = ':' . $bindKey; + $bindValues[$bindKey] = $value; + $bindIndex++; + } + + $batchKeys[] = '(' . implode(', ', $bindKeys) . ')'; + + // Permissions logic + $permissionsStmt = $this->getPDO()->prepare(" + SELECT _type, _permission + FROM {$this->getSQLTable($name . '_perms')} + WHERE _document = :_uid + "); + $permissionsStmt->bindValue(':_uid', $document->getId()); + $permissionsStmt->execute(); + $permissions = $permissionsStmt->fetchAll(PDO::FETCH_ASSOC); + + $initial = []; + foreach (Database::PERMISSIONS as $type) { + $initial[$type] = []; + } + + $permissions = \array_reduce($permissions, function (array $carry, array $item) { + $carry[$item['_type']][] = $item['_permission']; + return $carry; + }, $initial); + + // Get removed Permissions + $removals = []; + foreach (Database::PERMISSIONS as $type) { + $diff = array_diff($permissions[$type], $document->getPermissionsByType($type)); + if (!empty($diff)) { + $removals[$type] = $diff; + } + } + + // Build inner query to remove permissions + if (!empty($removals)) { + foreach ($removals as $type => $permissionsToRemove) { + $bindKey = 'uid_' . $index; + $removeBindKeys[] = ':uid_' . $index; + $removeBindValues[$bindKey] = $document->getId(); + + $removeQuery .= "( + _document = :uid_{$index} + AND _type = '{$type}' + AND _permission IN (" . implode(', ', \array_map(function (string $i) use ($permissionsToRemove, $index, $type, &$removeBindKeys, &$removeBindValues) { + $bindKey = 'remove_' . $type . '_' . $index . '_' . $i; + $removeBindKeys[] = ':' . $bindKey; + $removeBindValues[$bindKey] = $permissionsToRemove[$i]; + + return ':' . $bindKey; + }, \array_keys($permissionsToRemove))) . + ") + )"; + + if ($type !== \array_key_last($removals)) { + $removeQuery .= ' OR '; + } + } + + if ($index !== \array_key_last($batch)) { + $removeQuery .= ' OR '; + } + } + + // Get added Permissions + $additions = []; + foreach (Database::PERMISSIONS as $type) { + $diff = \array_diff($document->getPermissionsByType($type), $permissions[$type]); + if (!empty($diff)) { + $additions[$type] = $diff; + } + } + + // Build inner query to add permissions + if (!empty($additions)) { + foreach ($additions as $type => $permissionsToAdd) { + foreach ($permissionsToAdd as $i => $permission) { + $bindKey = 'uid_' . $index; + $addBindValues[$bindKey] = $document->getId(); + + $bindKey = 'add_' . $type . '_' . $index . '_' . $i; + $addBindValues[$bindKey] = $permission; + + $addQuery .= "(:uid_{$index}, '{$type}', :{$bindKey})"; + + if ($i !== \array_key_last($permissionsToAdd) || $type !== \array_key_last($additions)) { + $addQuery .= ', '; + } + } + } + if ($index !== \array_key_last($batch)) { + $addQuery .= ', '; + } + } + } + + $updateClause = ''; + for ($i = 0; $i < \count($columns); $i++) { + $column = $columns[$i]; + if (!empty($updateClause)) { + $updateClause .= ', '; + } + $updateClause .= "{$column} = excluded.{$column}"; + } + + $stmt = $this->getPDO()->prepare(" + INSERT INTO {$this->getSQLTable($name)} (" . \implode(", ", $columns) . ") + VALUES " . \implode(', ', $batchKeys) . " + ON CONFLICT(_uid) DO UPDATE SET $updateClause + "); + + foreach ($bindValues as $key => $value) { + $stmt->bindValue($key, $value, $this->getPDOType($value)); + } + + $stmt->execute(); + + if (!empty($removeQuery)) { + $stmtRemovePermissions = $this->getPDO()->prepare(" + DELETE + FROM {$this->getSQLTable($name . '_perms')} + WHERE ({$removeQuery}) + "); + + foreach ($removeBindValues as $key => $value) { + $stmtRemovePermissions->bindValue($key, $value, $this->getPDOType($value)); + } + + $stmtRemovePermissions->execute(); + } + + if (!empty($addQuery)) { + $stmtAddPermissions = $this->getPDO()->prepare(" + INSERT INTO {$this->getSQLTable($name . '_perms')} (`_document`, `_type`, `_permission`) + VALUES {$addQuery} + "); + + foreach ($addBindValues as $key => $value) { + $stmtAddPermissions->bindValue($key, $value, $this->getPDOType($value)); + } + + $stmtAddPermissions->execute(); + } + } + + if (!$this->getPDO()->commit()) { + throw new Exception('Failed to commit transaction'); + } + + return $documents; + } catch (PDOException $e) { + $this->getPDO()->rollBack(); + + throw match ($e->getCode()) { + 1062, 23000 => new Duplicate('Duplicated document: ' . $e->getMessage()), + default => $e, + }; + } + } + /** * Is schemas supported? * From 8c7b3949b1c07d819ba5e020ac96977ded7a1d4a Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 15 Aug 2023 01:34:54 -0400 Subject: [PATCH 12/14] Fix test --- tests/Database/Validator/StructureTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Database/Validator/StructureTest.php b/tests/Database/Validator/StructureTest.php index 293af3a33..bb6e5e4ca 100644 --- a/tests/Database/Validator/StructureTest.php +++ b/tests/Database/Validator/StructureTest.php @@ -251,7 +251,7 @@ public function testStringValidation() 'feedback' => 'team@appwrite.io', ]))); - $this->assertEquals('Invalid document structure: Attribute "title" has invalid type. Value must be a valid string and no longer than 256 chars', $validator->getDescription()); + $this->assertEquals('Invalid document structure: Attribute "title" has invalid type. Value must be a valid string and at least 1 chars and no longer than 256 chars', $validator->getDescription()); } public function testArrayOfStringsValidation() From 794f5f2c9b73b1e6d9ebd56dca026df697c88c82 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 15 Aug 2023 11:40:04 -0400 Subject: [PATCH 13/14] Fix tests --- tests/Database/Base.php | 6 ------ tests/Database/Validator/StructureTest.php | 6 +++--- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/tests/Database/Base.php b/tests/Database/Base.php index 15b69e489..5886ed75c 100644 --- a/tests/Database/Base.php +++ b/tests/Database/Base.php @@ -985,18 +985,12 @@ public function testUpdateDocuments(array $documents) ]); } - $start = microtime(true); - $documents = static::getDatabase()->updateDocuments( $collection, $documents, \count($documents) ); - $end = microtime(true); - - \var_dump('Batch update time for ' . \count($documents) . ' documents : ' . ($end - $start)); - foreach ($documents as $document) { $this->assertEquals('textđź“ť updated', $document->getAttribute('string')); $this->assertEquals(6, $document->getAttribute('integer')); diff --git a/tests/Database/Validator/StructureTest.php b/tests/Database/Validator/StructureTest.php index bb6e5e4ca..594935fef 100644 --- a/tests/Database/Validator/StructureTest.php +++ b/tests/Database/Validator/StructureTest.php @@ -269,7 +269,7 @@ public function testArrayOfStringsValidation() 'feedback' => 'team@appwrite.io', ]))); - $this->assertEquals('Invalid document structure: Attribute "tags[\'0\']" has invalid type. Value must be a valid string and no longer than 55 chars', $validator->getDescription()); + $this->assertEquals('Invalid document structure: Attribute "tags[\'0\']" has invalid type. Value must be a valid string and at least 1 chars and no longer than 55 chars', $validator->getDescription()); $this->assertEquals(false, $validator->isValid(new Document([ '$collection' => ID::custom('posts'), @@ -282,7 +282,7 @@ public function testArrayOfStringsValidation() 'feedback' => 'team@appwrite.io', ]))); - $this->assertEquals('Invalid document structure: Attribute "tags[\'0\']" has invalid type. Value must be a valid string and no longer than 55 chars', $validator->getDescription()); + $this->assertEquals('Invalid document structure: Attribute "tags[\'0\']" has invalid type. Value must be a valid string and at least 1 chars and no longer than 55 chars', $validator->getDescription()); $this->assertEquals(true, $validator->isValid(new Document([ '$collection' => ID::custom('posts'), @@ -307,7 +307,7 @@ public function testArrayOfStringsValidation() 'feedback' => 'team@appwrite.io', ]))); - $this->assertEquals('Invalid document structure: Attribute "tags[\'0\']" has invalid type. Value must be a valid string and no longer than 55 chars', $validator->getDescription()); + $this->assertEquals('Invalid document structure: Attribute "tags[\'0\']" has invalid type. Value must be a valid string and at least 1 chars and no longer than 55 chars', $validator->getDescription()); } public function testIntegerValidation() From 75391eab7cca70cd408c3ccfbde8c1dde62a01ba Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 16 Aug 2023 13:36:53 -0400 Subject: [PATCH 14/14] Remove extra space --- src/Database/Database.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index 41c49ea33..f293599a3 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -1470,8 +1470,6 @@ public function createDocuments(string $collection, array $documents, int $batch $collection = $this->silent(fn() => $this->getCollection($collection)); - - $time = DateTime::now(); foreach ($documents as $key => $document) {