diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 00000000..43d7ca7e --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,82 @@ +name: "Tests" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: [pull_request] +jobs: + build: + name: Build & Unit + runs-on: ubuntu-latest + + steps: + - name: Checkout Repo + uses: actions/checkout@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Build image + uses: docker/build-push-action@v3 + with: + context: . + push: false + load: true + tags: storage-dev + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Start storage + env: + DO_ACCESS_KEY: ${{ secrets.DO_ACCESS_KEY }} + DO_SECRET: ${{ secrets.DO_SECRET }} + LINODE_ACCESS_KEY: ${{ secrets.LINODE_ACCESS_KEY }} + LINODE_SECRET: ${{ secrets.LINODE_SECRET }} + S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }} + S3_SECRET: ${{ secrets.S3_SECRET }} + WASABI_ACCESS_KEY: ${{ secrets.WASABI_ACCESS_KEY }} + WASABI_SECRET: ${{ secrets.WASABI_SECRET }} + BACKBLAZE_ACCESS_KEY: ${{ secrets.BACKBLAZE_ACCESS_KEY }} + BACKBLAZE_SECRET: ${{ secrets.BACKBLAZE_SECRET }} + run: | + docker compose up -d + sleep 10 + + - name: Doctor + run: | + docker compose logs tests + docker ps + + - name: Unit Tests + run: docker compose exec -T tests vendor/bin/phpunit --configuration phpunit.xml --debug --testsuite unit + + e2e_test: + name: E2E Test + runs-on: ubuntu-latest + needs: build + strategy: + fail-fast: false + matrix: + devices: [BackblazeTest, DOSpacesTest, LinodeTest, LocalTest, S3Test, WasabiTest] + + steps: + - name: checkout + uses: actions/checkout@v3 + - name: Start storage + env: + DO_ACCESS_KEY: ${{ secrets.DO_ACCESS_KEY }} + DO_SECRET: ${{ secrets.DO_SECRET }} + LINODE_ACCESS_KEY: ${{ secrets.LINODE_ACCESS_KEY }} + LINODE_SECRET: ${{ secrets.LINODE_SECRET }} + S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }} + S3_SECRET: ${{ secrets.S3_SECRET }} + WASABI_ACCESS_KEY: ${{ secrets.WASABI_ACCESS_KEY }} + WASABI_SECRET: ${{ secrets.WASABI_SECRET }} + BACKBLAZE_ACCESS_KEY: ${{ secrets.BACKBLAZE_ACCESS_KEY }} + BACKBLAZE_SECRET: ${{ secrets.BACKBLAZE_SECRET }} + run: | + docker compose up -d + sleep 10 + - name: Run ${{matrix.devices}} + run: docker compose exec -T tests vendor/bin/phpunit tests/Storage/Device/${{matrix.devices}}.php diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index c7438e66..00000000 --- a/.travis.yml +++ /dev/null @@ -1,36 +0,0 @@ -dist: focal - -arch: - - amd64 - -os: linux - -language: shell - -notifications: - email: - - team@appwrite.io - -before_script: docker run --rm --interactive --tty --volume "$(pwd)":/app composer update --ignore-platform-reqs --optimize-autoloader --no-plugins --no-scripts --prefer-dist - -before_install: - - curl -fsSL https://get.docker.com | sh - - echo '{"experimental":"enabled"}' | sudo tee /etc/docker/daemon.json - - mkdir -p $HOME/.docker - - echo '{"experimental":"enabled"}' | sudo tee $HOME/.docker/config.json - - sudo service docker start - - > - if [ ! -z "${DOCKERHUB_PULL_USERNAME:-}" ]; then - echo "${DOCKERHUB_PULL_PASSWORD}" | docker login --username "${DOCKERHUB_PULL_USERNAME}" --password-stdin - fi - - docker --version - -install: - - docker-compose up -d - - sleep 10 - -script: - - docker ps - - docker-compose exec tests vendor/bin/phpunit --configuration phpunit.xml --debug --testsuite unit - - docker-compose exec tests vendor/bin/phpunit --configuration phpunit.xml --debug --testsuite e2e - - docker-compose exec tests vendor/bin/psalm --show-info=true \ No newline at end of file diff --git a/composer.json b/composer.json index 46d12e03..3dde33f0 100644 --- a/composer.json +++ b/composer.json @@ -24,7 +24,8 @@ "ext-lz4": "*", "ext-snappy": "*", "php": ">=8.0", - "utopia-php/framework": "0.*.*" + "utopia-php/framework": "0.*.*", + "utopia-php/system": "0.*.*" }, "require-dev": { "phpunit/phpunit": "^9.3", diff --git a/composer.lock b/composer.lock index 56f1adf2..54da72ab 100644 --- a/composer.lock +++ b/composer.lock @@ -4,29 +4,96 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "0e15ac0ae89a77c7420f825af10e33fc", + "content-hash": "024174312546d59cde20dd0dd4e6186d", "packages": [ + { + "name": "laravel/pint", + "version": "v1.2.1", + "source": { + "type": "git", + "url": "https://github.com/laravel/pint.git", + "reference": "e60e2112ee779ce60f253695b273d1646a17d6f1" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/laravel/pint/zipball/e60e2112ee779ce60f253695b273d1646a17d6f1", + "reference": "e60e2112ee779ce60f253695b273d1646a17d6f1", + "shasum": "" + }, + "require": { + "ext-json": "*", + "ext-mbstring": "*", + "ext-tokenizer": "*", + "ext-xml": "*", + "php": "^8.0" + }, + "require-dev": { + "friendsofphp/php-cs-fixer": "^3.11.0", + "illuminate/view": "^9.32.0", + "laravel-zero/framework": "^9.2.0", + "mockery/mockery": "^1.5.1", + "nunomaduro/larastan": "^2.2.0", + "nunomaduro/termwind": "^1.14.0", + "pestphp/pest": "^1.22.1" + }, + "bin": [ + "builds/pint" + ], + "type": "project", + "autoload": { + "psr-4": { + "App\\": "app/", + "Database\\Seeders\\": "database/seeders/", + "Database\\Factories\\": "database/factories/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nuno Maduro", + "email": "enunomaduro@gmail.com" + } + ], + "description": "An opinionated code formatter for PHP.", + "homepage": "https://laravel.com", + "keywords": [ + "format", + "formatter", + "lint", + "linter", + "php" + ], + "support": { + "issues": "https://github.com/laravel/pint/issues", + "source": "https://github.com/laravel/pint" + }, + "time": "2022-11-29T16:25:20+00:00" + }, { "name": "utopia-php/framework", - "version": "0.27.0", + "version": "0.30.0", "source": { "type": "git", "url": "https://github.com/utopia-php/framework.git", - "reference": "b8d0447f5c98291d7759db05460ecced29a0f9ee" + "reference": "0969d429997996ceade53e477fce31221b415651" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/framework/zipball/b8d0447f5c98291d7759db05460ecced29a0f9ee", - "reference": "b8d0447f5c98291d7759db05460ecced29a0f9ee", + "url": "https://api.github.com/repos/utopia-php/framework/zipball/0969d429997996ceade53e477fce31221b415651", + "reference": "0969d429997996ceade53e477fce31221b415651", "shasum": "" }, "require": { - "php": ">=8.0.0" + "php": ">=8.0" }, "require-dev": { "laravel/pint": "^1.2", - "phpunit/phpunit": "^9.5.25", - "vimeo/psalm": "4.27.0" + "phpbench/phpbench": "^1.2", + "phpstan/phpstan": "^1.10", + "phpunit/phpunit": "^9.5.25" }, "type": "library", "autoload": { @@ -46,9 +113,66 @@ ], "support": { "issues": "https://github.com/utopia-php/framework/issues", - "source": "https://github.com/utopia-php/framework/tree/0.27.0" + "source": "https://github.com/utopia-php/framework/tree/0.30.0" + }, + "time": "2023-08-07T07:55:48+00:00" + }, + { + "name": "utopia-php/system", + "version": "0.7.0", + "source": { + "type": "git", + "url": "https://github.com/utopia-php/system.git", + "reference": "d385e9521ca3f68aa007ec1cadd11c4dbd871b90" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/system/zipball/d385e9521ca3f68aa007ec1cadd11c4dbd871b90", + "reference": "d385e9521ca3f68aa007ec1cadd11c4dbd871b90", + "shasum": "" + }, + "require": { + "laravel/pint": "1.2.*", + "php": ">=8.0.0" + }, + "require-dev": { + "phpunit/phpunit": "^9.3", + "squizlabs/php_codesniffer": "^3.6", + "vimeo/psalm": "4.0.1" + }, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\System\\": "src/System" + } }, - "time": "2023-01-29T05:36:17+00:00" + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Eldad Fux", + "email": "eldad@appwrite.io" + }, + { + "name": "Torsten Dittmann", + "email": "torsten@appwrite.io" + } + ], + "description": "A simple library for obtaining information about the host's system.", + "keywords": [ + "framework", + "php", + "system", + "upf", + "utopia" + ], + "support": { + "issues": "https://github.com/utopia-php/system/issues", + "source": "https://github.com/utopia-php/system/tree/0.7.0" + }, + "time": "2023-08-07T09:34:26+00:00" } ], "packages-dev": [ @@ -475,18 +599,66 @@ }, "time": "2019-12-04T15:06:13+00:00" }, + { + "name": "doctrine/deprecations", + "version": "1.1.x-dev", + "source": { + "type": "git", + "url": "https://github.com/doctrine/deprecations.git", + "reference": "bdaa697ed9c7f5ee2f7d3b5f9c2a6f2519523cd9" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/doctrine/deprecations/zipball/bdaa697ed9c7f5ee2f7d3b5f9c2a6f2519523cd9", + "reference": "bdaa697ed9c7f5ee2f7d3b5f9c2a6f2519523cd9", + "shasum": "" + }, + "require": { + "php": "^7.1 || ^8.0" + }, + "require-dev": { + "doctrine/coding-standard": "^9", + "phpstan/phpstan": "1.4.10 || 1.10.15", + "phpstan/phpstan-phpunit": "^1.0", + "phpunit/phpunit": "^7.5 || ^8.5 || ^9.5", + "psalm/plugin-phpunit": "0.18.4", + "psr/log": "^1 || ^2 || ^3", + "vimeo/psalm": "4.30.0 || 5.12.0" + }, + "suggest": { + "psr/log": "Allows logging deprecations via PSR-3 logger implementation" + }, + "default-branch": true, + "type": "library", + "autoload": { + "psr-4": { + "Doctrine\\Deprecations\\": "lib/Doctrine/Deprecations" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "description": "A small layer on top of trigger_error(E_USER_DEPRECATED) or PSR-3 logging with options to disable all deprecations or selectively for packages.", + "homepage": "https://www.doctrine-project.org/", + "support": { + "issues": "https://github.com/doctrine/deprecations/issues", + "source": "https://github.com/doctrine/deprecations/tree/1.1.x" + }, + "time": "2023-07-29T16:12:19+00:00" + }, { "name": "doctrine/instantiator", "version": "2.0.x-dev", "source": { "type": "git", "url": "https://github.com/doctrine/instantiator.git", - "reference": "d6eef505a6c46e963e54bf73bb9de43cdea70821" + "reference": "40fc8fcd149f6ccff851e767c3fc5ded4e6a96c0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/doctrine/instantiator/zipball/d6eef505a6c46e963e54bf73bb9de43cdea70821", - "reference": "d6eef505a6c46e963e54bf73bb9de43cdea70821", + "url": "https://api.github.com/repos/doctrine/instantiator/zipball/40fc8fcd149f6ccff851e767c3fc5ded4e6a96c0", + "reference": "40fc8fcd149f6ccff851e767c3fc5ded4e6a96c0", "shasum": "" }, "require": { @@ -499,7 +671,7 @@ "phpbench/phpbench": "^1.2", "phpstan/phpstan": "^1.9.4", "phpstan/phpstan-phpunit": "^1.3", - "phpunit/phpunit": "^9.5.27", + "phpunit/phpunit": "^10.1.0", "vimeo/psalm": "^5.4" }, "default-branch": true, @@ -544,7 +716,7 @@ "type": "tidelift" } ], - "time": "2023-01-04T15:42:40+00:00" + "time": "2023-07-30T10:00:15+00:00" }, { "name": "felixfbecker/advanced-json-rpc", @@ -648,84 +820,18 @@ }, "time": "2022-06-19T17:15:06+00:00" }, - { - "name": "laravel/pint", - "version": "v1.2.1", - "source": { - "type": "git", - "url": "https://github.com/laravel/pint.git", - "reference": "e60e2112ee779ce60f253695b273d1646a17d6f1" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/laravel/pint/zipball/e60e2112ee779ce60f253695b273d1646a17d6f1", - "reference": "e60e2112ee779ce60f253695b273d1646a17d6f1", - "shasum": "" - }, - "require": { - "ext-json": "*", - "ext-mbstring": "*", - "ext-tokenizer": "*", - "ext-xml": "*", - "php": "^8.0" - }, - "require-dev": { - "friendsofphp/php-cs-fixer": "^3.11.0", - "illuminate/view": "^9.32.0", - "laravel-zero/framework": "^9.2.0", - "mockery/mockery": "^1.5.1", - "nunomaduro/larastan": "^2.2.0", - "nunomaduro/termwind": "^1.14.0", - "pestphp/pest": "^1.22.1" - }, - "bin": [ - "builds/pint" - ], - "type": "project", - "autoload": { - "psr-4": { - "App\\": "app/", - "Database\\Seeders\\": "database/seeders/", - "Database\\Factories\\": "database/factories/" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Nuno Maduro", - "email": "enunomaduro@gmail.com" - } - ], - "description": "An opinionated code formatter for PHP.", - "homepage": "https://laravel.com", - "keywords": [ - "format", - "formatter", - "lint", - "linter", - "php" - ], - "support": { - "issues": "https://github.com/laravel/pint/issues", - "source": "https://github.com/laravel/pint" - }, - "time": "2022-11-29T16:25:20+00:00" - }, { "name": "myclabs/deep-copy", "version": "1.x-dev", "source": { "type": "git", "url": "https://github.com/myclabs/DeepCopy.git", - "reference": "14daed4296fae74d9e3201d2c4925d1acb7aa614" + "reference": "f6f48cfecf52ab791fe18cc1b11d6345512dc4b8" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/myclabs/DeepCopy/zipball/14daed4296fae74d9e3201d2c4925d1acb7aa614", - "reference": "14daed4296fae74d9e3201d2c4925d1acb7aa614", + "url": "https://api.github.com/repos/myclabs/DeepCopy/zipball/f6f48cfecf52ab791fe18cc1b11d6345512dc4b8", + "reference": "f6f48cfecf52ab791fe18cc1b11d6345512dc4b8", "shasum": "" }, "require": { @@ -733,11 +839,12 @@ }, "conflict": { "doctrine/collections": "<1.6.8", - "doctrine/common": "<2.13.3 || >=3,<3.2.2" + "doctrine/common": "<2.13.3 || >=3 <3.2.2" }, "require-dev": { "doctrine/collections": "^1.6.8", "doctrine/common": "^2.13.3 || ^3.2.2", + "phpspec/prophecy": "^1.10", "phpunit/phpunit": "^7.5.20 || ^8.5.23 || ^9.5.13" }, "default-branch": true, @@ -764,7 +871,7 @@ ], "support": { "issues": "https://github.com/myclabs/DeepCopy/issues", - "source": "https://github.com/myclabs/DeepCopy/tree/1.11.0" + "source": "https://github.com/myclabs/DeepCopy/tree/1.x" }, "funding": [ { @@ -772,7 +879,7 @@ "type": "tidelift" } ], - "time": "2022-03-03T13:19:32+00:00" + "time": "2023-07-30T10:01:33+00:00" }, { "name": "netresearch/jsonmapper", @@ -831,12 +938,12 @@ "source": { "type": "git", "url": "https://github.com/nikic/PHP-Parser.git", - "reference": "570e980a201d8ed0236b0a62ddf2c9cbb2034039" + "reference": "cfc54e30a4f5e5af5f9d9ce86697cfcc5f7e7c24" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/570e980a201d8ed0236b0a62ddf2c9cbb2034039", - "reference": "570e980a201d8ed0236b0a62ddf2c9cbb2034039", + "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/cfc54e30a4f5e5af5f9d9ce86697cfcc5f7e7c24", + "reference": "cfc54e30a4f5e5af5f9d9ce86697cfcc5f7e7c24", "shasum": "" }, "require": { @@ -878,9 +985,9 @@ ], "support": { "issues": "https://github.com/nikic/PHP-Parser/issues", - "source": "https://github.com/nikic/PHP-Parser/tree/v4.15.3" + "source": "https://github.com/nikic/PHP-Parser/tree/4.x" }, - "time": "2023-01-16T22:05:37+00:00" + "time": "2023-07-30T21:38:32+00:00" }, { "name": "openlss/lib-array2xml", @@ -941,12 +1048,12 @@ "source": { "type": "git", "url": "https://github.com/phar-io/manifest.git", - "reference": "36d8a21e851a9512db2b086dc5ac2c61308f0138" + "reference": "67729272c564ab9f953c81f48db44e8b1cb1e1c3" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phar-io/manifest/zipball/36d8a21e851a9512db2b086dc5ac2c61308f0138", - "reference": "36d8a21e851a9512db2b086dc5ac2c61308f0138", + "url": "https://api.github.com/repos/phar-io/manifest/zipball/67729272c564ab9f953c81f48db44e8b1cb1e1c3", + "reference": "67729272c564ab9f953c81f48db44e8b1cb1e1c3", "shasum": "" }, "require": { @@ -955,7 +1062,7 @@ "ext-phar": "*", "ext-xmlwriter": "*", "phar-io/version": "^3.0.1", - "php": "^7.2 || ^8.0" + "php": "^7.3 || ^8.0" }, "default-branch": true, "type": "library", @@ -1001,7 +1108,7 @@ "type": "github" } ], - "time": "2022-02-21T19:55:33+00:00" + "time": "2023-06-01T14:19:47+00:00" }, { "name": "phar-io/version", @@ -1113,12 +1220,12 @@ "source": { "type": "git", "url": "https://github.com/phpDocumentor/ReflectionDocBlock.git", - "reference": "566af9fb94c556de91562fcfcbc392f66680111b" + "reference": "7b217217725dc991a0ae7b995041cee1d5019561" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpDocumentor/ReflectionDocBlock/zipball/566af9fb94c556de91562fcfcbc392f66680111b", - "reference": "566af9fb94c556de91562fcfcbc392f66680111b", + "url": "https://api.github.com/repos/phpDocumentor/ReflectionDocBlock/zipball/7b217217725dc991a0ae7b995041cee1d5019561", + "reference": "7b217217725dc991a0ae7b995041cee1d5019561", "shasum": "" }, "require": { @@ -1169,7 +1276,7 @@ "issues": "https://github.com/phpDocumentor/ReflectionDocBlock/issues", "source": "https://github.com/phpDocumentor/ReflectionDocBlock/tree/master" }, - "time": "2022-11-19T20:28:46+00:00" + "time": "2023-03-12T10:50:44+00:00" }, { "name": "phpdocumentor/type-resolver", @@ -1177,15 +1284,16 @@ "source": { "type": "git", "url": "https://github.com/phpDocumentor/TypeResolver.git", - "reference": "06f36c92b434ac686de06b6563e88046943bccbe" + "reference": "07100e65d09fd50608d649fc656cae1c921a2495" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/06f36c92b434ac686de06b6563e88046943bccbe", - "reference": "06f36c92b434ac686de06b6563e88046943bccbe", + "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/07100e65d09fd50608d649fc656cae1c921a2495", + "reference": "07100e65d09fd50608d649fc656cae1c921a2495", "shasum": "" }, "require": { + "doctrine/deprecations": "^1.0", "php": "^7.4 || ^8.0", "phpdocumentor/reflection-common": "^2.0", "phpstan/phpdoc-parser": "^1.13" @@ -1227,26 +1335,28 @@ "issues": "https://github.com/phpDocumentor/TypeResolver/issues", "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.x" }, - "time": "2022-12-16T10:25:14+00:00" + "time": "2023-07-20T19:57:33+00:00" }, { "name": "phpstan/phpdoc-parser", - "version": "1.16.1", + "version": "1.23.x-dev", "source": { "type": "git", "url": "https://github.com/phpstan/phpdoc-parser.git", - "reference": "e27e92d939e2e3636f0a1f0afaba59692c0bf571" + "reference": "846ae76eef31c6d7790fac9bc399ecee45160b26" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/e27e92d939e2e3636f0a1f0afaba59692c0bf571", - "reference": "e27e92d939e2e3636f0a1f0afaba59692c0bf571", + "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/846ae76eef31c6d7790fac9bc399ecee45160b26", + "reference": "846ae76eef31c6d7790fac9bc399ecee45160b26", "shasum": "" }, "require": { "php": "^7.2 || ^8.0" }, "require-dev": { + "doctrine/annotations": "^2.0", + "nikic/php-parser": "^4.15", "php-parallel-lint/php-parallel-lint": "^1.2", "phpstan/extension-installer": "^1.0", "phpstan/phpstan": "^1.5", @@ -1255,6 +1365,7 @@ "phpunit/phpunit": "^9.5", "symfony/process": "^5.2" }, + "default-branch": true, "type": "library", "autoload": { "psr-4": { @@ -1270,9 +1381,9 @@ "description": "PHPDoc parser with support for nullable, intersection and generic types", "support": { "issues": "https://github.com/phpstan/phpdoc-parser/issues", - "source": "https://github.com/phpstan/phpdoc-parser/tree/1.16.1" + "source": "https://github.com/phpstan/phpdoc-parser/tree/1.23.1" }, - "time": "2023-02-07T18:11:17+00:00" + "time": "2023-08-03T16:32:59+00:00" }, { "name": "phpunit/php-code-coverage", @@ -1280,19 +1391,19 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-code-coverage.git", - "reference": "b5f3416da036fb951a557518e8304b3595ff966a" + "reference": "9e1baee3a7525530f5600035759ce754a8b3750b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/b5f3416da036fb951a557518e8304b3595ff966a", - "reference": "b5f3416da036fb951a557518e8304b3595ff966a", + "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/9e1baee3a7525530f5600035759ce754a8b3750b", + "reference": "9e1baee3a7525530f5600035759ce754a8b3750b", "shasum": "" }, "require": { "ext-dom": "*", "ext-libxml": "*", "ext-xmlwriter": "*", - "nikic/php-parser": "^4.14", + "nikic/php-parser": "^4.15", "php": ">=7.3", "phpunit/php-file-iterator": "^3.0.3", "phpunit/php-text-template": "^2.0.2", @@ -1307,8 +1418,8 @@ "phpunit/phpunit": "^9.3" }, "suggest": { - "ext-pcov": "*", - "ext-xdebug": "*" + "ext-pcov": "PHP extension that provides line coverage", + "ext-xdebug": "PHP extension that provides line coverage as well as branch and path coverage" }, "type": "library", "extra": { @@ -1341,6 +1452,7 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/php-code-coverage/issues", + "security": "https://github.com/sebastianbergmann/php-code-coverage/security/policy", "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2" }, "funding": [ @@ -1349,7 +1461,7 @@ "type": "github" } ], - "time": "2023-02-18T16:27:54+00:00" + "time": "2023-08-02T09:00:29+00:00" }, { "name": "phpunit/php-file-iterator", @@ -1598,12 +1710,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "c1bf2dd252dec994ee27d804c038a98fd0d1d940" + "reference": "6904cfa5f4a6211e759de8ef36fb51fc939a83fa" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/c1bf2dd252dec994ee27d804c038a98fd0d1d940", - "reference": "c1bf2dd252dec994ee27d804c038a98fd0d1d940", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/6904cfa5f4a6211e759de8ef36fb51fc939a83fa", + "reference": "6904cfa5f4a6211e759de8ef36fb51fc939a83fa", "shasum": "" }, "require": { @@ -1636,8 +1748,8 @@ "sebastian/version": "^3.0.2" }, "suggest": { - "ext-soap": "*", - "ext-xdebug": "*" + "ext-soap": "To be able to generate mocks based on WSDL files", + "ext-xdebug": "PHP extension that provides line coverage as well as branch and path coverage" }, "bin": [ "phpunit" @@ -1676,6 +1788,7 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/phpunit/issues", + "security": "https://github.com/sebastianbergmann/phpunit/security/policy", "source": "https://github.com/sebastianbergmann/phpunit/tree/9.6" }, "funding": [ @@ -1692,7 +1805,7 @@ "type": "tidelift" } ], - "time": "2023-02-18T16:26:49+00:00" + "time": "2023-08-02T08:56:58+00:00" }, { "name": "psr/container", @@ -2098,16 +2211,16 @@ }, { "name": "sebastian/diff", - "version": "4.0.4", + "version": "4.0.x-dev", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/diff.git", - "reference": "3461e3fccc7cfdfc2720be910d3bd73c69be590d" + "reference": "74be17022044ebaaecfdf0c5cd504fc9cd5a7131" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/diff/zipball/3461e3fccc7cfdfc2720be910d3bd73c69be590d", - "reference": "3461e3fccc7cfdfc2720be910d3bd73c69be590d", + "url": "https://api.github.com/repos/sebastianbergmann/diff/zipball/74be17022044ebaaecfdf0c5cd504fc9cd5a7131", + "reference": "74be17022044ebaaecfdf0c5cd504fc9cd5a7131", "shasum": "" }, "require": { @@ -2152,7 +2265,7 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/diff/issues", - "source": "https://github.com/sebastianbergmann/diff/tree/4.0.4" + "source": "https://github.com/sebastianbergmann/diff/tree/4.0" }, "funding": [ { @@ -2160,7 +2273,7 @@ "type": "github" } ], - "time": "2020-10-26T13:10:38+00:00" + "time": "2023-05-07T05:35:17+00:00" }, { "name": "sebastian/environment", @@ -2308,12 +2421,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/global-state.git", - "reference": "0ca8db5a5fc9c8646244e629625ac486fa286bf2" + "reference": "bde739e7565280bda77be70044ac1047bc007e34" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/global-state/zipball/0ca8db5a5fc9c8646244e629625ac486fa286bf2", - "reference": "0ca8db5a5fc9c8646244e629625ac486fa286bf2", + "url": "https://api.github.com/repos/sebastianbergmann/global-state/zipball/bde739e7565280bda77be70044ac1047bc007e34", + "reference": "bde739e7565280bda77be70044ac1047bc007e34", "shasum": "" }, "require": { @@ -2356,7 +2469,7 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/global-state/issues", - "source": "https://github.com/sebastianbergmann/global-state/tree/5.0.5" + "source": "https://github.com/sebastianbergmann/global-state/tree/5.0" }, "funding": [ { @@ -2364,7 +2477,7 @@ "type": "github" } ], - "time": "2022-02-14T08:28:10+00:00" + "time": "2023-08-02T09:26:13+00:00" }, { "name": "sebastian/lines-of-code", @@ -2604,12 +2717,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/resource-operations.git", - "reference": "e1157eac767e4dc4ae40dd9aab7fb4de6e56bd32" + "reference": "20bdda85c7c585ab265c0c37ec052a019bae29c4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/resource-operations/zipball/e1157eac767e4dc4ae40dd9aab7fb4de6e56bd32", - "reference": "e1157eac767e4dc4ae40dd9aab7fb4de6e56bd32", + "url": "https://api.github.com/repos/sebastianbergmann/resource-operations/zipball/20bdda85c7c585ab265c0c37ec052a019bae29c4", + "reference": "20bdda85c7c585ab265c0c37ec052a019bae29c4", "shasum": "" }, "require": { @@ -2651,7 +2764,7 @@ "type": "github" } ], - "time": "2023-02-08T06:53:39+00:00" + "time": "2023-03-25T08:11:39+00:00" }, { "name": "sebastian/type", @@ -2768,12 +2881,12 @@ "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "bc04b7a3c23fd36e4b165afe7947b3d00989677b" + "reference": "b504a3d266ad2bb632f196c0936ef2af5ff6e273" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/bc04b7a3c23fd36e4b165afe7947b3d00989677b", - "reference": "bc04b7a3c23fd36e4b165afe7947b3d00989677b", + "url": "https://api.github.com/repos/symfony/console/zipball/b504a3d266ad2bb632f196c0936ef2af5ff6e273", + "reference": "b504a3d266ad2bb632f196c0936ef2af5ff6e273", "shasum": "" }, "require": { @@ -2838,7 +2951,7 @@ "homepage": "https://symfony.com", "keywords": [ "cli", - "command line", + "command-line", "console", "terminal" ], @@ -2859,7 +2972,7 @@ "type": "tidelift" } ], - "time": "2023-02-14T14:10:40+00:00" + "time": "2023-07-19T20:11:33+00:00" }, { "name": "symfony/deprecation-contracts", @@ -2867,12 +2980,12 @@ "source": { "type": "git", "url": "https://github.com/symfony/deprecation-contracts.git", - "reference": "1ee04c65529dea5d8744774d474e7cbd2f1206d3" + "reference": "7c3aff79d10325257a001fcf92d991f24fc967cf" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/deprecation-contracts/zipball/1ee04c65529dea5d8744774d474e7cbd2f1206d3", - "reference": "1ee04c65529dea5d8744774d474e7cbd2f1206d3", + "url": "https://api.github.com/repos/symfony/deprecation-contracts/zipball/7c3aff79d10325257a001fcf92d991f24fc967cf", + "reference": "7c3aff79d10325257a001fcf92d991f24fc967cf", "shasum": "" }, "require": { @@ -2882,7 +2995,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-main": "3.3-dev" + "dev-main": "3.4-dev" }, "thanks": { "name": "symfony/contracts", @@ -2911,7 +3024,7 @@ "description": "A generic function and convention to trigger deprecation notices", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/deprecation-contracts/tree/v3.2.0" + "source": "https://github.com/symfony/deprecation-contracts/tree/v3.3.0" }, "funding": [ { @@ -2927,11 +3040,11 @@ "type": "tidelift" } ], - "time": "2022-11-25T10:21:52+00:00" + "time": "2023-05-23T14:45:45+00:00" }, { "name": "symfony/polyfill-ctype", - "version": "dev-main", + "version": "1.x-dev", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-ctype.git", @@ -2994,7 +3107,7 @@ "portable" ], "support": { - "source": "https://github.com/symfony/polyfill-ctype/tree/main" + "source": "https://github.com/symfony/polyfill-ctype/tree/1.x" }, "funding": [ { @@ -3014,7 +3127,7 @@ }, { "name": "symfony/polyfill-intl-grapheme", - "version": "dev-main", + "version": "1.x-dev", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-intl-grapheme.git", @@ -3076,7 +3189,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-intl-grapheme/tree/main" + "source": "https://github.com/symfony/polyfill-intl-grapheme/tree/1.x" }, "funding": [ { @@ -3096,7 +3209,7 @@ }, { "name": "symfony/polyfill-intl-normalizer", - "version": "dev-main", + "version": "1.x-dev", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-intl-normalizer.git", @@ -3161,7 +3274,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-intl-normalizer/tree/main" + "source": "https://github.com/symfony/polyfill-intl-normalizer/tree/1.x" }, "funding": [ { @@ -3181,16 +3294,16 @@ }, { "name": "symfony/polyfill-mbstring", - "version": "dev-main", + "version": "1.x-dev", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-mbstring.git", - "reference": "f9c7affe77a00ae32ca127ca6833d034e6d33f25" + "reference": "42292d99c55abe617799667f454222c54c60e229" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/f9c7affe77a00ae32ca127ca6833d034e6d33f25", - "reference": "f9c7affe77a00ae32ca127ca6833d034e6d33f25", + "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/42292d99c55abe617799667f454222c54c60e229", + "reference": "42292d99c55abe617799667f454222c54c60e229", "shasum": "" }, "require": { @@ -3245,7 +3358,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-mbstring/tree/main" + "source": "https://github.com/symfony/polyfill-mbstring/tree/1.x" }, "funding": [ { @@ -3261,11 +3374,11 @@ "type": "tidelift" } ], - "time": "2023-01-30T17:25:47+00:00" + "time": "2023-07-28T09:04:16+00:00" }, { "name": "symfony/polyfill-php73", - "version": "dev-main", + "version": "1.x-dev", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-php73.git", @@ -3325,7 +3438,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-php73/tree/main" + "source": "https://github.com/symfony/polyfill-php73/tree/1.x" }, "funding": [ { @@ -3345,7 +3458,7 @@ }, { "name": "symfony/polyfill-php80", - "version": "dev-main", + "version": "1.x-dev", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-php80.git", @@ -3409,7 +3522,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-php80/tree/main" + "source": "https://github.com/symfony/polyfill-php80/tree/1.x" }, "funding": [ { @@ -3433,12 +3546,12 @@ "source": { "type": "git", "url": "https://github.com/symfony/service-contracts.git", - "reference": "aac98028c69df04ee77eb69b96b86ee51fbf4b75" + "reference": "a4025a1c812c231d88ed0780e866b0cc644f4a84" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/service-contracts/zipball/aac98028c69df04ee77eb69b96b86ee51fbf4b75", - "reference": "aac98028c69df04ee77eb69b96b86ee51fbf4b75", + "url": "https://api.github.com/repos/symfony/service-contracts/zipball/a4025a1c812c231d88ed0780e866b0cc644f4a84", + "reference": "a4025a1c812c231d88ed0780e866b0cc644f4a84", "shasum": "" }, "require": { @@ -3448,14 +3561,11 @@ "conflict": { "ext-psr": "<1.1|>=2" }, - "suggest": { - "symfony/service-implementation": "" - }, "default-branch": true, "type": "library", "extra": { "branch-alias": { - "dev-main": "3.3-dev" + "dev-main": "3.4-dev" }, "thanks": { "name": "symfony/contracts", @@ -3495,7 +3605,7 @@ "standards" ], "support": { - "source": "https://github.com/symfony/service-contracts/tree/v3.2.0" + "source": "https://github.com/symfony/service-contracts/tree/main" }, "funding": [ { @@ -3511,20 +3621,20 @@ "type": "tidelift" } ], - "time": "2022-11-25T10:21:52+00:00" + "time": "2023-07-29T13:12:44+00:00" }, { "name": "symfony/string", - "version": "6.3.x-dev", + "version": "6.4.x-dev", "source": { "type": "git", "url": "https://github.com/symfony/string.git", - "reference": "6515bacc98212164f10db4371fa70ee7b8c70b80" + "reference": "fe9228ba417441e16f31d36ddad2b3076135f453" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/string/zipball/6515bacc98212164f10db4371fa70ee7b8c70b80", - "reference": "6515bacc98212164f10db4371fa70ee7b8c70b80", + "url": "https://api.github.com/repos/symfony/string/zipball/fe9228ba417441e16f31d36ddad2b3076135f453", + "reference": "fe9228ba417441e16f31d36ddad2b3076135f453", "shasum": "" }, "require": { @@ -3538,11 +3648,11 @@ "symfony/translation-contracts": "<2.5" }, "require-dev": { - "symfony/error-handler": "^5.4|^6.0", - "symfony/http-client": "^5.4|^6.0", - "symfony/intl": "^6.2", + "symfony/error-handler": "^5.4|^6.0|^7.0", + "symfony/http-client": "^5.4|^6.0|^7.0", + "symfony/intl": "^6.2|^7.0", "symfony/translation-contracts": "^2.5|^3.0", - "symfony/var-exporter": "^5.4|^6.0" + "symfony/var-exporter": "^5.4|^6.0|^7.0" }, "type": "library", "autoload": { @@ -3581,7 +3691,7 @@ "utf8" ], "support": { - "source": "https://github.com/symfony/string/tree/6.3" + "source": "https://github.com/symfony/string/tree/6.4" }, "funding": [ { @@ -3597,7 +3707,7 @@ "type": "tidelift" } ], - "time": "2023-02-14T09:04:20+00:00" + "time": "2023-07-27T06:52:43+00:00" }, { "name": "theseer/tokenizer", @@ -3919,6 +4029,7 @@ "ext-fileinfo": "*", "ext-zlib": "*", "ext-zstd": "*", + "ext-xz": "*", "ext-brotli": "*", "ext-lz4": "*", "ext-snappy": "*", diff --git a/docker-compose.yml b/docker-compose.yml index e6af4bb2..4cd5e9f7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,10 +3,13 @@ version: '3.1' services: tests: container_name: tests + image: storage-dev build: context: . volumes: - - ./:/usr/src/code + - ./src:/usr/src/code/src + - ./tests:/usr/src/code/tests + - ./phpunit.xml:/usr/src/code/phpunit.xml environment: - S3_ACCESS_KEY - S3_SECRET diff --git a/src/Storage/Compression/Algorithms/XZ.php b/src/Storage/Compression/Algorithms/XZ.php index 3bd2c386..aaa79fae 100644 --- a/src/Storage/Compression/Algorithms/XZ.php +++ b/src/Storage/Compression/Algorithms/XZ.php @@ -17,8 +17,7 @@ public function getName(): string /** * Compress. * - * @param string $data - * + * @param string $data * @return string */ public function compress(string $data): string @@ -29,8 +28,7 @@ public function compress(string $data): string /** * Decompress. * - * @param string $data - * + * @param string $data * @return string */ public function decompress(string $data): string diff --git a/src/Storage/Device.php b/src/Storage/Device.php index 3756e97e..23027d94 100644 --- a/src/Storage/Device.php +++ b/src/Storage/Device.php @@ -4,6 +4,32 @@ abstract class Device { + /** + * Max chunk size while transfering file from one device to another + */ + protected int $transferChunkSize = 20000000; //20 MB + + /** + * Set Transfer Chunk Size + * + * @param int $chunkSize + * @return void + */ + public function setTransferChunkSize(int $chunkSize): void + { + $this->transferChunkSize = $chunkSize; + } + + /** + * Get Transfer Chunk Size + * + * @return int + */ + public function getTransferChunkSize(): int + { + return $this->transferChunkSize; + } + /** * Get Name. * @@ -68,6 +94,24 @@ abstract public function getPath(string $filename, string $prefix = null): strin */ abstract public function upload(string $source, string $path, int $chunk = 1, int $chunks = 1, array &$metadata = []): int; + /** + * Upload Data. + * + * Upload file contents to desired destination in the selected disk. + * return number of chunks uploaded or 0 if it fails. + * + * @param string $data + * @param string $path + * @param string $contentType + * @param int chunk + * @param int chunks + * @param array $metadata + * @return int + * + * @throws \Exception + */ + abstract public function uploadData(string $data, string $path, string $contentType, int $chunk = 1, int $chunks = 1, array &$metadata = []): int; + /** * Abort Chunked Upload * @@ -87,6 +131,17 @@ abstract public function abort(string $path, string $extra = ''): bool; */ abstract public function read(string $path, int $offset = 0, int $length = null): string; + /** + * Transfer + * Transfer a file from current device to destination device. + * + * @param string $path + * @param string $destination + * @param Device $device + * @return bool + */ + abstract public function transfer(string $path, string $destination, Device $device): bool; + /** * Write file by given path. * @@ -105,7 +160,14 @@ abstract public function write(string $path, string $data, string $contentType): * @param string $target * @return bool */ - abstract public function move(string $source, string $target): bool; + public function move(string $source, string $target): bool + { + if ($this->transfer($source, $target, $this)) { + return $this->delete($source); + } + + return false; + } /** * Delete file in given path return true on success and false on failure. diff --git a/src/Storage/Device/Local.php b/src/Storage/Device/Local.php index 8367158e..6d5c0f13 100644 --- a/src/Storage/Device/Local.php +++ b/src/Storage/Device/Local.php @@ -82,11 +82,7 @@ public function getPath(string $filename, string $prefix = null): string */ public function upload(string $source, string $path, int $chunk = 1, int $chunks = 1, array &$metadata = []): int { - if (! \file_exists(\dirname($path))) { // Checks if directory path to file exists - if (! @\mkdir(\dirname($path), 0755, true)) { - throw new Exception('Can\'t create directory: '.\dirname($path)); - } - } + $this->createDirectory(\dirname($path)); //move_uploaded_file() verifies the file is not tampered with if ($chunks === 1) { @@ -98,11 +94,7 @@ public function upload(string $source, string $path, int $chunk = 1, int $chunks } $tmp = \dirname($path).DIRECTORY_SEPARATOR.'tmp_'.\basename($path).DIRECTORY_SEPARATOR.\basename($path).'_chunks.log'; - if (! \file_exists(\dirname($tmp))) { // Checks if directory path to file exists - if (! @\mkdir(\dirname($tmp), 0755, true)) { - throw new Exception('Can\'t create directory: '.\dirname($tmp)); - } - } + $this->createDirectory(\dirname($tmp)); if (! file_put_contents($tmp, "$chunk\n", FILE_APPEND)) { throw new Exception('Can\'t write chunk log '.$tmp); } @@ -119,19 +111,61 @@ public function upload(string $source, string $path, int $chunk = 1, int $chunks } if ($chunks === $chunksReceived) { - for ($i = 1; $i <= $chunks; $i++) { - $part = dirname($tmp).DIRECTORY_SEPARATOR.pathinfo($path, PATHINFO_FILENAME).'.part.'.$i; - $data = file_get_contents($part); - if (! $data) { - throw new Exception('Failed to read chunk '.$part); - } - - if (! file_put_contents($path, $data, FILE_APPEND)) { - throw new Exception('Failed to append chunk '.$part); - } - \unlink($part); + $this->joinChunks($path, $chunks); + + return $chunksReceived; + } + + return $chunksReceived; + } + + /** + * Upload Data. + * + * Upload file contents to desired destination in the selected disk. + * return number of chunks uploaded or 0 if it fails. + * + * @param string $source + * @param string $path + * @param string $contentType + * @param int chunk + * @param int chunks + * @param array $metadata + * @return int + * + * @throws \Exception + */ + public function uploadData(string $data, string $path, string $contentType, int $chunk = 1, int $chunks = 1, array &$metadata = []): int + { + $this->createDirectory(\dirname($path)); + + if ($chunks === 1) { + if (! \file_put_contents($path, $data)) { + throw new Exception('Can\'t write file '.$path); } - \unlink($tmp); + + return $chunks; + } + $tmp = \dirname($path).DIRECTORY_SEPARATOR.'tmp_'.\basename($path).DIRECTORY_SEPARATOR.\basename($path).'_chunks.log'; + + $this->createDirectory(\dirname($tmp)); + if (! file_put_contents($tmp, "$chunk\n", FILE_APPEND)) { + throw new Exception('Can\'t write chunk log '.$tmp); + } + + $chunkLogs = file($tmp); + if (! $chunkLogs) { + throw new Exception('Unable to read chunk log '.$tmp); + } + + $chunksReceived = count(file($tmp)); + + if (! \file_put_contents(dirname($tmp).DIRECTORY_SEPARATOR.pathinfo($path, PATHINFO_FILENAME).'.part.'.$chunk, $data)) { + throw new Exception('Failed to write chunk '.$chunk); + } + + if ($chunks === $chunksReceived) { + $this->joinChunks($path, $chunks); return $chunksReceived; } @@ -139,6 +173,58 @@ public function upload(string $source, string $path, int $chunk = 1, int $chunks return $chunksReceived; } + private function joinChunks(string $path, int $chunks) + { + $tmp = \dirname($path).DIRECTORY_SEPARATOR.'tmp_'.\basename($path).DIRECTORY_SEPARATOR.\basename($path).'_chunks.log'; + for ($i = 1; $i <= $chunks; $i++) { + $part = dirname($tmp).DIRECTORY_SEPARATOR.pathinfo($path, PATHINFO_FILENAME).'.part.'.$i; + $data = file_get_contents($part); + if (! $data) { + throw new Exception('Failed to read chunk '.$part); + } + + if (! file_put_contents($path, $data, FILE_APPEND)) { + throw new Exception('Failed to append chunk '.$part); + } + \unlink($part); + } + \unlink($tmp); + \rmdir(dirname($tmp)); + } + + /** + * Transfer + * + * @param string $path + * @param string $destination + * @param Device $device + * @return string + */ + public function transfer(string $path, string $destination, Device $device): bool + { + if (! $this->exists($path)) { + throw new Exception('File Not Found'); + } + $size = $this->getFileSize($path); + $contentType = $this->getFileMimeType($path); + + if ($size <= $this->transferChunkSize) { + $source = $this->read($path); + + return $device->write($destination, $source, $contentType); + } + + $totalChunks = \ceil($size / $this->transferChunkSize); + $metadata = ['content_type' => $contentType]; + for ($counter = 0; $counter < $totalChunks; $counter++) { + $start = $counter * $this->transferChunkSize; + $data = $this->read($path, $start, $this->transferChunkSize); + $device->uploadData($data, $destination, $contentType, $counter + 1, $totalChunks, $metadata); + } + + return true; + } + /** * Abort Chunked Upload * diff --git a/src/Storage/Device/S3.php b/src/Storage/Device/S3.php index a8447b98..2c771719 100644 --- a/src/Storage/Device/S3.php +++ b/src/Storage/Device/S3.php @@ -222,17 +222,38 @@ public function getPath(string $filename, string $prefix = null): string * @throws \Exception */ public function upload(string $source, string $path, int $chunk = 1, int $chunks = 1, array &$metadata = []): int + { + return $this->uploadData(\file_get_contents($source), $path, \mime_content_type($source), $chunk, $chunks, $metadata); + } + + /** + * Upload Data. + * + * Upload file contents to desired destination in the selected disk. + * return number of chunks uploaded or 0 if it fails. + * + * @param string $source + * @param string $path + * @param string $contentType + * @param int chunk + * @param int chunks + * @param array $metadata + * @return int + * + * @throws \Exception + */ + public function uploadData(string $data, string $path, string $contentType, int $chunk = 1, int $chunks = 1, array &$metadata = []): int { if ($chunk == 1 && $chunks == 1) { - return $this->write($path, \file_get_contents($source), \mime_content_type($source)); + return $this->write($path, $data, $contentType); } $uploadId = $metadata['uploadId'] ?? null; if (empty($uploadId)) { - $uploadId = $this->createMultipartUpload($path, $metadata['content_type']); + $uploadId = $this->createMultipartUpload($path, $contentType); $metadata['uploadId'] = $uploadId; } - $etag = $this->uploadPart($source, $path, $chunk, $uploadId); + $etag = $this->uploadPart($data, $path, $contentType, $chunk, $uploadId); $metadata['parts'] ??= []; $metadata['parts'][] = ['partNumber' => $chunk, 'etag' => $etag]; $metadata['chunks'] ??= 0; @@ -244,6 +265,42 @@ public function upload(string $source, string $path, int $chunk = 1, int $chunks return $metadata['chunks']; } + /** + * Transfer + * + * @param string $path + * @param string $destination + * @param Device $device + * @return string + */ + public function transfer(string $path, string $destination, Device $device): bool + { + $response = []; + try { + $response = $this->getInfo($path); + } catch (\Throwable $e) { + throw new Exception('File not found'); + } + $size = (int) ($response['content-length'] ?? 0); + $contentType = $response['content-type'] ?? ''; + + if ($size <= $this->transferChunkSize) { + $source = $this->read($path); + + return $device->write($destination, $source, $contentType); + } + + $totalChunks = \ceil($size / $this->transferChunkSize); + $metadata = ['content_type' => $contentType]; + for ($counter = 0; $counter < $totalChunks; $counter++) { + $start = $counter * $this->transferChunkSize; + $data = $this->read($path, $start, $this->transferChunkSize); + $device->uploadData($data, $destination, $contentType, $counter + 1, $totalChunks, $metadata); + } + + return true; + } + /** * Start Multipart Upload * @@ -279,12 +336,11 @@ protected function createMultipartUpload(string $path, string $contentType): str * * @throws \Exception */ - protected function uploadPart(string $source, string $path, int $chunk, string $uploadId): string + protected function uploadPart(string $data, string $path, string $contentType, int $chunk, string $uploadId): string { $uri = $path !== '' ? '/'.\str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)) : '/'; - $data = \file_get_contents($source); - $this->headers['content-type'] = \mime_content_type($source); + $this->headers['content-type'] = $contentType; $this->headers['content-md5'] = \base64_encode(md5($data, true)); $this->amzHeaders['x-amz-content-sha256'] = \hash('sha256', $data); unset($this->amzHeaders['x-amz-acl']); // ACL header is not allowed in parts, only createMultipartUpload accepts this header. @@ -392,29 +448,6 @@ public function write(string $path, string $data, string $contentType = ''): boo return true; } - /** - * Move file from given source to given path, Return true on success and false on failure. - * - * @see http://php.net/manual/en/function.filesize.php - * - * @param string $source - * @param string $target - * - * @throw \Exception - * - * @return bool - */ - public function move(string $source, string $target): bool - { - $type = $this->getFileMimeType($source); - - if ($this->write($target, $this->read($source), $type)) { - $this->delete($source); - } - - return true; - } - /** * Delete file in given path, Return true on success and false on failure. * @@ -704,7 +737,7 @@ private function getSignatureV4(string $method, string $uri, array $parameters = $kService = \hash_hmac('sha256', $service, $kRegion, true); $kSigning = \hash_hmac('sha256', 'aws4_request', $kService, true); - $signature = \hash_hmac('sha256', \utf8_encode($stringToSignStr), $kSigning); + $signature = \hash_hmac('sha256', \mb_convert_encoding($stringToSignStr, 'utf-8'), $kSigning); return $algorithm.' '.\implode(',', [ 'Credential='.$this->accessKey.'/'.\implode('/', $credentialScope), diff --git a/tests/Storage/Compression/Algorithms/XZTest.php b/tests/Storage/Compression/Algorithms/XZTest.php index 6fec876e..2b55ee01 100644 --- a/tests/Storage/Compression/Algorithms/XZTest.php +++ b/tests/Storage/Compression/Algorithms/XZTest.php @@ -2,8 +2,8 @@ namespace Utopia\Tests\Storage\Compression\Algorithms; -use Utopia\Storage\Compression\Algorithms\XZ; use PHPUnit\Framework\TestCase; +use Utopia\Storage\Compression\Algorithms\XZ; class XZTest extends TestCase { @@ -22,7 +22,7 @@ public function testName() { $this->assertEquals($this->object->getName(), 'xz'); } - + public function testCompressDecompressWithText() { $demo = 'This is a demo string'; @@ -33,13 +33,13 @@ public function testCompressDecompressWithText() $this->assertEquals($demoSize, 21); $this->assertEquals($dataSize, 80); - + $this->assertEquals($this->object->decompress($data), $demo); } - + public function testCompressDecompressWithJPGImage() { - $demo = \file_get_contents(__DIR__ . '/../../../resources/disk-a/kitten-1.jpg'); + $demo = \file_get_contents(__DIR__.'/../../../resources/disk-a/kitten-1.jpg'); $demoSize = mb_strlen($demo, '8bit'); $data = $this->object->compress($demo); @@ -47,18 +47,18 @@ public function testCompressDecompressWithJPGImage() $this->assertEquals($demoSize, 599639); $this->assertEquals($dataSize, 599432); - + $this->assertGreaterThan($dataSize, $demoSize); - + $data = $this->object->decompress($data); $dataSize = mb_strlen($data, '8bit'); - + $this->assertEquals($dataSize, 599639); } - + public function testCompressDecompressWithPNGImage() { - $demo = \file_get_contents(__DIR__ . '/../../../resources/disk-b/kitten-1.png'); + $demo = \file_get_contents(__DIR__.'/../../../resources/disk-b/kitten-1.png'); $demoSize = mb_strlen($demo, '8bit'); $data = $this->object->compress($demo); @@ -66,12 +66,12 @@ public function testCompressDecompressWithPNGImage() $this->assertEquals($demoSize, 3038056); $this->assertEquals($dataSize, 2981000); - + $this->assertGreaterThan($dataSize, $demoSize); - + $data = $this->object->decompress($data); $dataSize = mb_strlen($data, '8bit'); - + $this->assertEquals($dataSize, 3038056); } } diff --git a/tests/Storage/Device/BackblazeTest.php b/tests/Storage/Device/BackblazeTest.php index 159bcb0f..91ce2f8f 100644 --- a/tests/Storage/Device/BackblazeTest.php +++ b/tests/Storage/Device/BackblazeTest.php @@ -12,7 +12,7 @@ protected function init(): void $this->root = '/root'; $key = $_SERVER['BACKBLAZE_ACCESS_KEY'] ?? ''; $secret = $_SERVER['BACKBLAZE_SECRET'] ?? ''; - $bucket = 'backblaze-demo'; + $bucket = 'utopia-storage-test'; $this->object = new Backblaze($this->root, $key, $secret, $bucket, Backblaze::US_WEST_004, Backblaze::ACL_PRIVATE); } diff --git a/tests/Storage/Device/LinodeTest.php b/tests/Storage/Device/LinodeTest.php index 397e62bc..95118739 100644 --- a/tests/Storage/Device/LinodeTest.php +++ b/tests/Storage/Device/LinodeTest.php @@ -12,9 +12,9 @@ protected function init(): void $this->root = '/root'; $key = $_SERVER['LINODE_ACCESS_KEY'] ?? ''; $secret = $_SERVER['LINODE_SECRET'] ?? ''; - $bucket = 'everly-test'; + $bucket = 'storage-test'; - $this->object = new Linode($this->root, $key, $secret, $bucket, Linode::EU_CENTRAL_1, Linode::ACL_PRIVATE); + $this->object = new Linode($this->root, $key, $secret, $bucket, Linode::AP_SOUTH_1, Linode::ACL_PRIVATE); } protected function getAdapterName(): string diff --git a/tests/Storage/Device/LocalTest.php b/tests/Storage/Device/LocalTest.php index 1cc3ea11..9db0510c 100644 --- a/tests/Storage/Device/LocalTest.php +++ b/tests/Storage/Device/LocalTest.php @@ -4,6 +4,7 @@ use PHPUnit\Framework\TestCase; use Utopia\Storage\Device\Local; +use Utopia\Storage\Device\S3; class LocalTest extends TestCase { @@ -234,7 +235,6 @@ public function testPartRead($path) $chunk = file_get_contents($source, false, null, 0, 500); $readChunk = $this->object->read($path, 0, 500); $this->assertEquals($chunk, $readChunk); - $this->object->delete($path); } public function testPartitionFreeSpace() @@ -247,6 +247,51 @@ public function testPartitionTotalSpace() $this->assertGreaterThan(0, $this->object->getPartitionTotalSpace()); } + /** + * @depends testPartUpload + */ + public function testTransferLarge($path) + { + // chunked file + $this->object->setTransferChunkSize(10000000); //10 mb + + $key = $_SERVER['S3_ACCESS_KEY'] ?? ''; + $secret = $_SERVER['S3_SECRET'] ?? ''; + $bucket = 'utopia-storage-test'; + + $device = new S3('/root', $key, $secret, $bucket, S3::EU_CENTRAL_1, S3::ACL_PRIVATE); + $destination = $device->getPath('largefile.mp4'); + + $this->assertTrue($this->object->transfer($path, $destination, $device)); + $this->assertTrue($device->exists($destination)); + $this->assertEquals($device->getFileMimeType($destination), 'video/mp4'); + + $device->delete($destination); + $this->object->delete($path); + } + + public function testTransferSmall() + { + $this->object->setTransferChunkSize(10000000); //10 mb + + $key = $_SERVER['S3_ACCESS_KEY'] ?? ''; + $secret = $_SERVER['S3_SECRET'] ?? ''; + $bucket = 'utopia-storage-test'; + + $device = new S3('/root', $key, $secret, $bucket, S3::EU_CENTRAL_1, S3::ACL_PRIVATE); + + $path = $this->object->getPath('text-for-read.txt'); + $this->object->write($path, 'Hello World'); + + $destination = $device->getPath('hello.txt'); + $this->assertTrue($this->object->transfer($path, $destination, $device)); + $this->assertTrue($device->exists($destination)); + $this->assertEquals($device->read($destination), 'Hello World'); + + $this->object->delete($path); + $device->delete($destination); + } + public function testDeletePath() { // Test Single Object diff --git a/tests/Storage/Device/S3Test.php b/tests/Storage/Device/S3Test.php index d8e05c3b..2b0255c8 100644 --- a/tests/Storage/Device/S3Test.php +++ b/tests/Storage/Device/S3Test.php @@ -12,9 +12,9 @@ protected function init(): void $this->root = '/root'; $key = $_SERVER['S3_ACCESS_KEY'] ?? ''; $secret = $_SERVER['S3_SECRET'] ?? ''; - $bucket = 'appwrite-test-bucket'; + $bucket = 'utopia-storage-test'; - $this->object = new S3($this->root, $key, $secret, $bucket, S3::EU_WEST_1, S3::ACL_PRIVATE); + $this->object = new S3($this->root, $key, $secret, $bucket, S3::EU_CENTRAL_1, S3::ACL_PRIVATE); } /** diff --git a/tests/Storage/S3Base.php b/tests/Storage/S3Base.php index cf17d915..a06c8aa9 100644 --- a/tests/Storage/S3Base.php +++ b/tests/Storage/S3Base.php @@ -3,6 +3,7 @@ namespace Utopia\Tests\Storage; use PHPUnit\Framework\TestCase; +use Utopia\Storage\Device\Local; use Utopia\Storage\Device\S3; abstract class S3Base extends TestCase @@ -261,6 +262,42 @@ public function testPartRead($path) $chunk = file_get_contents($source, false, null, 0, 500); $readChunk = $this->object->read($path, 0, 500); $this->assertEquals($chunk, $readChunk); + } + + /** + * @depends testPartUpload + */ + public function testTransferLarge($path) + { + // chunked file + $this->object->setTransferChunkSize(10000000); //10 mb + + $device = new Local(__DIR__.'/../resources/disk-a'); + $destination = $device->getPath('largefile.mp4'); + + $this->assertTrue($this->object->transfer($path, $destination, $device)); + $this->assertTrue($device->exists($destination)); + $this->assertEquals($device->getFileMimeType($destination), 'video/mp4'); + + $device->delete($destination); + $this->object->delete($path); + } + + public function testTransferSmall() + { + $this->object->setTransferChunkSize(10000000); //10 mb + + $device = new Local(__DIR__.'/../resources/disk-a'); + + $path = $this->object->getPath('text-for-read.txt'); + $this->object->write($path, 'Hello World', 'text/plain'); + + $destination = $device->getPath('hello.txt'); + $this->assertTrue($this->object->transfer($path, $destination, $device)); + $this->assertTrue($device->exists($destination)); + $this->assertEquals($device->read($destination), 'Hello World'); + $this->object->delete($path); + $device->delete($destination); } }