diff --git a/composer.lock b/composer.lock index d8cfb9cd..e597babb 100644 --- a/composer.lock +++ b/composer.lock @@ -74,16 +74,16 @@ }, { "name": "utopia-php/framework", - "version": "0.30.0", + "version": "0.31.0", "source": { "type": "git", "url": "https://github.com/utopia-php/framework.git", - "reference": "0969d429997996ceade53e477fce31221b415651" + "reference": "207f77378965fca9a9bc3783ea379d3549f86bc0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/framework/zipball/0969d429997996ceade53e477fce31221b415651", - "reference": "0969d429997996ceade53e477fce31221b415651", + "url": "https://api.github.com/repos/utopia-php/framework/zipball/207f77378965fca9a9bc3783ea379d3549f86bc0", + "reference": "207f77378965fca9a9bc3783ea379d3549f86bc0", "shasum": "" }, "require": { @@ -113,22 +113,22 @@ ], "support": { "issues": "https://github.com/utopia-php/framework/issues", - "source": "https://github.com/utopia-php/framework/tree/0.30.0" + "source": "https://github.com/utopia-php/framework/tree/0.31.0" }, - "time": "2023-08-07T07:55:48+00:00" + "time": "2023-08-30T16:10:04+00:00" }, { "name": "utopia-php/system", - "version": "0.7.0", + "version": "0.7.1", "source": { "type": "git", "url": "https://github.com/utopia-php/system.git", - "reference": "d385e9521ca3f68aa007ec1cadd11c4dbd871b90" + "reference": "01bf0d283aded0ee0a7a6e5ff540acf64270ab27" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/system/zipball/d385e9521ca3f68aa007ec1cadd11c4dbd871b90", - "reference": "d385e9521ca3f68aa007ec1cadd11c4dbd871b90", + "url": "https://api.github.com/repos/utopia-php/system/zipball/01bf0d283aded0ee0a7a6e5ff540acf64270ab27", + "reference": "01bf0d283aded0ee0a7a6e5ff540acf64270ab27", "shasum": "" }, "require": { @@ -170,9 +170,9 @@ ], "support": { "issues": "https://github.com/utopia-php/system/issues", - "source": "https://github.com/utopia-php/system/tree/0.7.0" + "source": "https://github.com/utopia-php/system/tree/0.7.1" }, - "time": "2023-08-07T09:34:26+00:00" + "time": "2023-08-30T09:14:37+00:00" } ], "packages-dev": [ @@ -422,12 +422,12 @@ "source": { "type": "git", "url": "https://github.com/composer/semver.git", - "reference": "fa1ec24f0ab1efe642671ec15c51a3ab879f59bf" + "reference": "1d09200268e7d1052ded8e5da9c73c96a63d18f5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/composer/semver/zipball/fa1ec24f0ab1efe642671ec15c51a3ab879f59bf", - "reference": "fa1ec24f0ab1efe642671ec15c51a3ab879f59bf", + "url": "https://api.github.com/repos/composer/semver/zipball/1d09200268e7d1052ded8e5da9c73c96a63d18f5", + "reference": "1d09200268e7d1052ded8e5da9c73c96a63d18f5", "shasum": "" }, "require": { @@ -496,7 +496,7 @@ "type": "tidelift" } ], - "time": "2023-01-13T15:47:53+00:00" + "time": "2023-08-31T12:20:31+00:00" }, { "name": "composer/xdebug-handler", @@ -605,12 +605,12 @@ "source": { "type": "git", "url": "https://github.com/doctrine/deprecations.git", - "reference": "bdaa697ed9c7f5ee2f7d3b5f9c2a6f2519523cd9" + "reference": "4f2d4f2836e7ec4e7a8625e75c6aa916004db931" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/doctrine/deprecations/zipball/bdaa697ed9c7f5ee2f7d3b5f9c2a6f2519523cd9", - "reference": "bdaa697ed9c7f5ee2f7d3b5f9c2a6f2519523cd9", + "url": "https://api.github.com/repos/doctrine/deprecations/zipball/4f2d4f2836e7ec4e7a8625e75c6aa916004db931", + "reference": "4f2d4f2836e7ec4e7a8625e75c6aa916004db931", "shasum": "" }, "require": { @@ -643,9 +643,9 @@ "homepage": "https://www.doctrine-project.org/", "support": { "issues": "https://github.com/doctrine/deprecations/issues", - "source": "https://github.com/doctrine/deprecations/tree/1.1.x" + "source": "https://github.com/doctrine/deprecations/tree/1.1.2" }, - "time": "2023-07-29T16:12:19+00:00" + "time": "2023-09-27T20:04:15+00:00" }, { "name": "doctrine/instantiator", @@ -938,12 +938,12 @@ "source": { "type": "git", "url": "https://github.com/nikic/PHP-Parser.git", - "reference": "a6303e50c90c355c7eeee2c4a8b27fe8dc8fef1d" + "reference": "54103d838734be0499172026525e38cbf6af2711" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/a6303e50c90c355c7eeee2c4a8b27fe8dc8fef1d", - "reference": "a6303e50c90c355c7eeee2c4a8b27fe8dc8fef1d", + "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/54103d838734be0499172026525e38cbf6af2711", + "reference": "54103d838734be0499172026525e38cbf6af2711", "shasum": "" }, "require": { @@ -954,7 +954,6 @@ "ircmaxell/php-yacc": "^0.0.7", "phpunit/phpunit": "^6.5 || ^7.0 || ^8.0 || ^9.0" }, - "default-branch": true, "bin": [ "bin/php-parse" ], @@ -985,9 +984,9 @@ ], "support": { "issues": "https://github.com/nikic/PHP-Parser/issues", - "source": "https://github.com/nikic/PHP-Parser/tree/v4.17.1" + "source": "https://github.com/nikic/PHP-Parser/tree/4.x" }, - "time": "2023-08-13T19:53:39+00:00" + "time": "2023-10-03T21:00:18+00:00" }, { "name": "openlss/lib-array2xml", @@ -1339,16 +1338,16 @@ }, { "name": "phpstan/phpdoc-parser", - "version": "1.23.x-dev", + "version": "1.24.2", "source": { "type": "git", "url": "https://github.com/phpstan/phpdoc-parser.git", - "reference": "e540adcd37d57d1f46e2b6e172d6140d005226cb" + "reference": "bcad8d995980440892759db0c32acae7c8e79442" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/e540adcd37d57d1f46e2b6e172d6140d005226cb", - "reference": "e540adcd37d57d1f46e2b6e172d6140d005226cb", + "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/bcad8d995980440892759db0c32acae7c8e79442", + "reference": "bcad8d995980440892759db0c32acae7c8e79442", "shasum": "" }, "require": { @@ -1365,7 +1364,6 @@ "phpunit/phpunit": "^9.5", "symfony/process": "^5.2" }, - "default-branch": true, "type": "library", "autoload": { "psr-4": { @@ -1381,9 +1379,9 @@ "description": "PHPDoc parser with support for nullable, intersection and generic types", "support": { "issues": "https://github.com/phpstan/phpdoc-parser/issues", - "source": "https://github.com/phpstan/phpdoc-parser/tree/1.23.x" + "source": "https://github.com/phpstan/phpdoc-parser/tree/1.24.2" }, - "time": "2023-08-16T11:40:53+00:00" + "time": "2023-09-26T12:28:12+00:00" }, { "name": "phpunit/php-code-coverage", @@ -1391,12 +1389,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-code-coverage.git", - "reference": "a74e6341296fe533cb69c8c94193afc7537443ec" + "reference": "0b6cb84a359426d34e439ce6d1173b41e07cb672" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/a74e6341296fe533cb69c8c94193afc7537443ec", - "reference": "a74e6341296fe533cb69c8c94193afc7537443ec", + "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/0b6cb84a359426d34e439ce6d1173b41e07cb672", + "reference": "0b6cb84a359426d34e439ce6d1173b41e07cb672", "shasum": "" }, "require": { @@ -1461,7 +1459,7 @@ "type": "github" } ], - "time": "2023-08-21T05:57:35+00:00" + "time": "2023-10-06T09:53:04+00:00" }, { "name": "phpunit/php-file-iterator", @@ -1710,12 +1708,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "1e5fec0e7eb197de7e50d5eb5012722dcc38a692" + "reference": "18a76cf7293527f275342720eeb063dff346c97e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/1e5fec0e7eb197de7e50d5eb5012722dcc38a692", - "reference": "1e5fec0e7eb197de7e50d5eb5012722dcc38a692", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/18a76cf7293527f275342720eeb063dff346c97e", + "reference": "18a76cf7293527f275342720eeb063dff346c97e", "shasum": "" }, "require": { @@ -1730,7 +1728,7 @@ "phar-io/manifest": "^2.0.3", "phar-io/version": "^3.0.2", "php": ">=7.3", - "phpunit/php-code-coverage": "^9.2.13", + "phpunit/php-code-coverage": "^9.2.28", "phpunit/php-file-iterator": "^3.0.5", "phpunit/php-invoker": "^3.1.1", "phpunit/php-text-template": "^2.0.3", @@ -1805,7 +1803,7 @@ "type": "tidelift" } ], - "time": "2023-08-26T05:31:07+00:00" + "time": "2023-10-06T09:52:37+00:00" }, { "name": "psr/container", @@ -1813,12 +1811,12 @@ "source": { "type": "git", "url": "https://github.com/php-fig/container.git", - "reference": "90db7b9ac2a2c5b849fcb69dde58f3ae182c68f5" + "reference": "707984727bd5b2b670e59559d3ed2500240cf875" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/php-fig/container/zipball/90db7b9ac2a2c5b849fcb69dde58f3ae182c68f5", - "reference": "90db7b9ac2a2c5b849fcb69dde58f3ae182c68f5", + "url": "https://api.github.com/repos/php-fig/container/zipball/707984727bd5b2b670e59559d3ed2500240cf875", + "reference": "707984727bd5b2b670e59559d3ed2500240cf875", "shasum": "" }, "require": { @@ -1857,9 +1855,9 @@ ], "support": { "issues": "https://github.com/php-fig/container/issues", - "source": "https://github.com/php-fig/container/tree/master" + "source": "https://github.com/php-fig/container" }, - "time": "2022-07-19T17:36:59+00:00" + "time": "2023-09-22T11:11:30+00:00" }, { "name": "psr/log", @@ -3107,7 +3105,7 @@ "portable" ], "support": { - "source": "https://github.com/symfony/polyfill-ctype/tree/1.x" + "source": "https://github.com/symfony/polyfill-ctype/tree/v1.28.0" }, "funding": [ { @@ -3189,7 +3187,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-intl-grapheme/tree/1.x" + "source": "https://github.com/symfony/polyfill-intl-grapheme/tree/v1.28.0" }, "funding": [ { @@ -3274,7 +3272,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-intl-normalizer/tree/1.x" + "source": "https://github.com/symfony/polyfill-intl-normalizer/tree/v1.28.0" }, "funding": [ { @@ -3358,7 +3356,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-mbstring/tree/1.x" + "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.28.0" }, "funding": [ { @@ -3438,7 +3436,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-php73/tree/1.x" + "source": "https://github.com/symfony/polyfill-php73/tree/v1.28.0" }, "funding": [ { @@ -3522,7 +3520,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-php80/tree/1.x" + "source": "https://github.com/symfony/polyfill-php80/tree/v1.28.0" }, "funding": [ { @@ -3629,12 +3627,12 @@ "source": { "type": "git", "url": "https://github.com/symfony/string.git", - "reference": "fe9228ba417441e16f31d36ddad2b3076135f453" + "reference": "742351f7542c9b9799873e399a716cb5af6f70f2" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/string/zipball/fe9228ba417441e16f31d36ddad2b3076135f453", - "reference": "fe9228ba417441e16f31d36ddad2b3076135f453", + "url": "https://api.github.com/repos/symfony/string/zipball/742351f7542c9b9799873e399a716cb5af6f70f2", + "reference": "742351f7542c9b9799873e399a716cb5af6f70f2", "shasum": "" }, "require": { @@ -3707,7 +3705,7 @@ "type": "tidelift" } ], - "time": "2023-07-27T06:52:43+00:00" + "time": "2023-09-18T10:40:25+00:00" }, { "name": "theseer/tokenizer", diff --git a/src/Storage/Device.php b/src/Storage/Device.php index 2882b9ff..ac742146 100644 --- a/src/Storage/Device.php +++ b/src/Storage/Device.php @@ -7,10 +7,15 @@ abstract class Device { /** - * Max chunk size while transfering file from one device to another + * Max chunk size while transferring file from one device to another */ protected int $transferChunkSize = 20000000; //20 MB + /** + * Sets the maximum number of keys returned to the response. By default, the action returns up to 1,000 key names. + */ + protected const MAX_PAGE_SIZE = PHP_INT_MAX; + /** * Set Transfer Chunk Size * @@ -277,9 +282,11 @@ abstract public function getPartitionTotalSpace(): float; * Get all files and directories inside a directory. * * @param string $dir Directory to scan - * @return string[] + * @param int $max + * @param string $continuationToken + * @return array */ - abstract public function getFiles(string $dir): array; + abstract public function getFiles(string $dir, int $max = self::MAX_PAGE_SIZE, string $continuationToken = ''): array; /** * Get the absolute path by resolving strings like ../, .., //, /\ and so on. diff --git a/src/Storage/Device/Local.php b/src/Storage/Device/Local.php index 66cf317b..65149917 100644 --- a/src/Storage/Device/Local.php +++ b/src/Storage/Device/Local.php @@ -510,10 +510,12 @@ public function getPartitionTotalSpace(): float /** * Get all files and directories inside a directory. * - * @param string $dir Directory to scan + * @param string $dir + * @param int $max + * @param string $continuationToken * @return string[] */ - public function getFiles(string $dir): array + public function getFiles(string $dir, int $max = self::MAX_PAGE_SIZE, string $continuationToken = ''): array { if (! (\str_ends_with($dir, DIRECTORY_SEPARATOR))) { $dir .= DIRECTORY_SEPARATOR; diff --git a/src/Storage/Device/S3.php b/src/Storage/Device/S3.php index 625056b5..79aadeff 100644 --- a/src/Storage/Device/S3.php +++ b/src/Storage/Device/S3.php @@ -92,6 +92,8 @@ class S3 extends Device const ACL_AUTHENTICATED_READ = 'authenticated-read'; + protected const MAX_PAGE_SIZE = 1000; + /** * @var string */ @@ -474,13 +476,19 @@ public function delete(string $path, bool $recursive = false): bool /** * Get list of objects in the given path. * - * @param string $path + * @param string $prefix + * @param int $maxKeys + * @param string $continuationToken * @return array * - * @throws \Exception + * @throws Exception */ - private function listObjects($prefix = '', $maxKeys = 1000, $continuationToken = '') + private function listObjects(string $prefix = '', int $maxKeys = self::MAX_PAGE_SIZE, string $continuationToken = ''): array { + if ($maxKeys > self::MAX_PAGE_SIZE) { + throw new Exception('Cannot list more than '.self::MAX_PAGE_SIZE.' objects'); + } + $uri = '/'; $prefix = ltrim($prefix, '/'); /** S3 specific requirement that prefix should never contain a leading slash */ $this->headers['content-type'] = 'text/plain'; @@ -491,9 +499,11 @@ private function listObjects($prefix = '', $maxKeys = 1000, $continuationToken = 'prefix' => $prefix, 'max-keys' => $maxKeys, ]; + if (! empty($continuationToken)) { $parameters['continuation-token'] = $continuationToken; } + $response = $this->call(self::METHOD_GET, $uri, '', $parameters); return $response->body; @@ -657,17 +667,35 @@ public function getPartitionTotalSpace(): float * Get all files and directories inside a directory. * * @param string $dir Directory to scan - * @return string[] + * @param int $max + * @param string $continuationToken + * @return array + * + * @throws Exception */ - public function getFiles(string $dir): array + public function getFiles(string $dir, int $max = self::MAX_PAGE_SIZE, string $continuationToken = ''): array { - throw new Exception('Not implemented.'); + $data = $this->listObjects($dir, $max, $continuationToken); + + // Set to false if all the results were returned. Set to true if more keys are available to return. + $data['IsTruncated'] = $data['IsTruncated'] === 'true'; + + // KeyCount is the number of keys returned with this request. + $data['KeyCount'] = intval($data['KeyCount']); + + // Sets the maximum number of keys returned to the response. By default, the action returns up to 1,000 key names. + $data['MaxKeys'] = intval($data['MaxKeys']); + + return $data; } /** * Get file info * + * @param string $path * @return array + * + * @throws Exception */ private function getInfo(string $path): array { diff --git a/tests/Storage/S3Base.php b/tests/Storage/S3Base.php index 70eea4b1..00a60f52 100644 --- a/tests/Storage/S3Base.php +++ b/tests/Storage/S3Base.php @@ -57,6 +57,42 @@ public function tearDown(): void $this->removeTestFiles(); } + public function testGetFiles() + { + $path = $this->object->getPath('testing/'); + $files = $this->object->getFiles($path); + $this->assertEquals(4, $files['KeyCount']); + $this->assertEquals(false, $files['IsTruncated']); + $this->assertIsArray($files['Contents']); + + $file = $files['Contents'][0]; + + $this->assertArrayHasKey('Key', $file); + $this->assertArrayHasKey('LastModified', $file); + $this->assertArrayHasKey('ETag', $file); + $this->assertArrayHasKey('StorageClass', $file); + $this->assertArrayHasKey('Size', $file); + } + + public function testGetFilesPagination() + { + $path = $this->object->getPath('testing/'); + + $files = $this->object->getFiles($path, 3); + $this->assertEquals(3, $files['KeyCount']); + $this->assertEquals(3, $files['MaxKeys']); + $this->assertEquals(true, $files['IsTruncated']); + $this->assertIsArray($files['Contents']); + $this->assertArrayHasKey('NextContinuationToken', $files); + + $files = $this->object->getFiles($path, 1000, $files['NextContinuationToken']); + $this->assertEquals(1, $files['KeyCount']); + $this->assertEquals(1000, $files['MaxKeys']); + $this->assertEquals(false, $files['IsTruncated']); + $this->assertIsArray($files['Contents']); + $this->assertArrayNotHasKey('NextContinuationToken', $files); + } + public function testName() { $this->assertEquals($this->getAdapterName(), $this->object->getName());