Skip to content

Commit

Permalink
[Task]: Verify we are using Flysystem everywhere (#254)
Browse files Browse the repository at this point in the history
* Use Flysystem storage for zip upload

* use storage for the zip and csv download

* Apply php-cs-fixer changes

* Refactor Download and Upload to respect flysystem (#252)

* Refactor CSV Download to jobrunId based

* Apply php-cs-fixer changes

* Updating storage

* Apply php-cs-fixer changes

* Try to use fly system for zip

* Apply php-cs-fixer changes

* fix: STAN

* make zip download sequential

* Apply php-cs-fixer changes

* remove unused storage service

* make csv export sequential

* Apply php-cs-fixer changes

* fixes

* add csv export changes

* Apply php-cs-fixer changes

* add zip upload changes

* Apply php-cs-fixer changes

* fix: qodana

* fix: qodana

* Apply php-cs-fixer changes

* fix: STAN

* update download service

* Apply php-cs-fixer changes

* fix: qodana and remove unused ENUMs

* add review changes

---------

Co-authored-by: mattamon <mattamon@users.noreply.github.com>
Co-authored-by: lukmzig <lukas.mzigot@pimcore.com>
Co-authored-by: lukmzig <lukmzig@users.noreply.github.com>

---------

Co-authored-by: lukmzig <lukmzig@users.noreply.github.com>
Co-authored-by: Matthias Schuhmayer <38959016+mattamon@users.noreply.github.com>
Co-authored-by: mattamon <mattamon@users.noreply.github.com>
  • Loading branch information
4 people authored Jul 24, 2024
1 parent e1b04df commit b9bae93
Show file tree
Hide file tree
Showing 65 changed files with 1,430 additions and 706 deletions.
14 changes: 7 additions & 7 deletions config/assets.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -89,24 +89,24 @@ services:
# Handler
#

Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Handler\CollectionHandler: ~
Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Handler\ZipCreationHandler: ~
Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Handler\CsvCreationHandler: ~
Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Handler\AssetCloneHandler: ~
Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Handler\AssetDeleteHandler: ~
Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Handler\AssetUploadHandler: ~
Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Handler\CsvCreationHandler: ~
Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Handler\CsvDataCollectionHandler: ~
Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Handler\ZipDownloadHandler: ~
Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Handler\ZipUploadHandler: ~

#
# Event Subscriber
#

Pimcore\Bundle\StudioBackendBundle\Asset\EventSubscriber\ZipDownloadSubscriber: ~
Pimcore\Bundle\StudioBackendBundle\Asset\EventSubscriber\ZipUploadSubscriber: ~
Pimcore\Bundle\StudioBackendBundle\Asset\EventSubscriber\CsvDownloadSubscriber: ~
Pimcore\Bundle\StudioBackendBundle\Asset\EventSubscriber\DeletionSubscriber: ~
Pimcore\Bundle\StudioBackendBundle\Asset\EventSubscriber\CloneSubscriber: ~
Pimcore\Bundle\StudioBackendBundle\Asset\EventSubscriber\CsvCreationSubscriber: ~
Pimcore\Bundle\StudioBackendBundle\Asset\EventSubscriber\DeletionSubscriber: ~
Pimcore\Bundle\StudioBackendBundle\Asset\EventSubscriber\UploadSubscriber: ~
Pimcore\Bundle\StudioBackendBundle\Asset\EventSubscriber\ZipDownloadSubscriber: ~
Pimcore\Bundle\StudioBackendBundle\Asset\EventSubscriber\ZipUploadSubscriber: ~

#
# Mercure SSE
Expand Down
3 changes: 3 additions & 0 deletions config/elements.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,9 @@ services:
Pimcore\Bundle\StudioBackendBundle\Element\Service\ElementFolderServiceInterface:
class: Pimcore\Bundle\StudioBackendBundle\Element\Service\ElementFolderService

Pimcore\Bundle\StudioBackendBundle\Element\Service\StorageServiceInterface:
class: Pimcore\Bundle\StudioBackendBundle\Element\Service\StorageService


#
# Handler
Expand Down
4 changes: 2 additions & 2 deletions config/pimcore/execution_engine.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,11 @@ pimcore_generic_execution_engine:
framework:
messenger:
routing:
Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Messages\CollectionMessage: pimcore_generic_execution_engine
Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Messages\ZipCreationMessage: pimcore_generic_execution_engine
Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Messages\CsvCollectionMessage: pimcore_generic_execution_engine
Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Messages\CsvCreationMessage: pimcore_generic_execution_engine
Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Messages\AssetCloneMessage: pimcore_generic_execution_engine
Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Messages\AssetDeleteMessage: pimcore_generic_execution_engine
Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Messages\AssetUploadMessage: pimcore_generic_execution_engine
Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Messages\ZipDownloadMessage: pimcore_generic_execution_engine
Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Messages\ZipUploadMessage: pimcore_generic_execution_engine
Pimcore\Bundle\StudioBackendBundle\Element\ExecutionEngine\AutomationAction\Messenger\Messages\PatchMessage: pimcore_generic_execution_engine
43 changes: 43 additions & 0 deletions doc/04_Generic_Execution_Engine.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
# Generic Execution Engine
:::caution

This documentation is currently work in progress and will be updated soon.

:::

The Generic Execution Engine is a powerful tool to execute actions in the background. It is based on the Symfony Messenger component, to learn more about it, please visit the [Generic Execution Engine documentation](https://github.com/pimcore/pimcore/tree/11.x/doc/19_Development_Tools_and_Details/08_Generic_Execution_Engine).

There are several actions, which currently take benefits of Execution Engine:

### Asset ZIP upload
When uploading a ZIP file containing assets, the ZIP file is extracted and the assets are created in the background.

### Asset ZIP Download
When downloading a multiple assets as ZIP file, the ZIP file is created in the background. This ZIP archive can then be downloaded.
There are some configuration options available for the ZIP download, which can be configured in the `config.yaml` file.

```yaml
pimcore_studio_backend:
asset_download_settings:
# Maximum number of assets that can be downloaded in a single ZIP file. Default value is 1000.
amount_limit: 1000
# Maximum size of the ZIP file in bytes. Default value is 5 GB.
size_limit: 5368709120
```
### Asset CSV Export
Assets can be exported based on the provided grid configuration as a CSV file. The export is done in the background and can be downloaded after its finished.
There are some configuration options available for the CSV export, which can be configured in the `config.yaml` file.

```yaml
pimcore_studio_backend:
csv_settings:
# Default delimiter for CSV files when no value is passed by grid configuration. Default value is ','.
default_delimiter: ','
```

### Assets deletion
Assets and folders can be deleted in the background. This is useful when deleting a large number of assets or asset trees.

### Asset cloning
Assets and folders can be cloned in the background. This is useful when cloning a large number of assets or asset trees.
24 changes: 9 additions & 15 deletions src/Asset/Controller/CreateCsvController.php
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,14 @@

namespace Pimcore\Bundle\StudioBackendBundle\Asset\Controller;

use OpenApi\Attributes\JsonContent;
use OpenApi\Attributes\Post;
use OpenApi\Attributes\Property;
use Pimcore\Bundle\StudioBackendBundle\Asset\Attributes\Request\CsvExportRequestBody;
use Pimcore\Bundle\StudioBackendBundle\Asset\MappedParameter\ExportAssetParameter;
use Pimcore\Bundle\StudioBackendBundle\Asset\Service\ExecutionEngine\CsvServiceInterface;
use Pimcore\Bundle\StudioBackendBundle\Controller\AbstractApiController;
use Pimcore\Bundle\StudioBackendBundle\OpenApi\Attributes\Response\Content\IdJson;
use Pimcore\Bundle\StudioBackendBundle\OpenApi\Attributes\Response\CreatedResponse;
use Pimcore\Bundle\StudioBackendBundle\OpenApi\Attributes\Response\DefaultResponses;
use Pimcore\Bundle\StudioBackendBundle\OpenApi\Attributes\Response\SuccessResponse;
use Pimcore\Bundle\StudioBackendBundle\OpenApi\Config\Tags;
use Pimcore\Bundle\StudioBackendBundle\Util\Constants\HttpResponseCodes;
use Pimcore\Bundle\StudioBackendBundle\Util\Constants\UserPermissions;
Expand Down Expand Up @@ -56,17 +55,9 @@ public function __construct(
tags: [Tags::Assets->name]
)]
#[CsvExportRequestBody]
#[SuccessResponse(
content: new JsonContent(
properties: [
new Property(
property: 'path',
description: 'Path to the csv file',
type: 'string',
example: '/var/www/html/var/assets.csv'
),
]
)
#[CreatedResponse(
description: 'Successfully created jobRun for csv export',
content: new IdJson('ID of created jobRun', 'jobRunId')
)]
#[DefaultResponses([
HttpResponseCodes::UNAUTHORIZED,
Expand All @@ -75,6 +66,9 @@ public function __construct(
public function createCsvAssets(
#[MapRequestPayload] ExportAssetParameter $exportAssetParameter
): Response {
return $this->jsonResponse(['path' => $this->csvService->generateCsvFile($exportAssetParameter)]);
return $this->jsonResponse(
['jobRunId' => $this->csvService->generateCsvFile($exportAssetParameter)],
HttpResponseCodes::CREATED->value
);
}
}
24 changes: 9 additions & 15 deletions src/Asset/Controller/CreateZipController.php
Original file line number Diff line number Diff line change
Expand Up @@ -16,16 +16,15 @@

namespace Pimcore\Bundle\StudioBackendBundle\Asset\Controller;

use OpenApi\Attributes\JsonContent;
use OpenApi\Attributes\Post;
use OpenApi\Attributes\Property;
use OpenApi\Attributes\RequestBody;
use Pimcore\Bundle\StudioBackendBundle\Asset\MappedParameter\CreateAssetFileParameter;
use Pimcore\Bundle\StudioBackendBundle\Asset\Service\ExecutionEngine\ZipServiceInterface;
use Pimcore\Bundle\StudioBackendBundle\Controller\AbstractApiController;
use Pimcore\Bundle\StudioBackendBundle\OpenApi\Attributes\Content\ScalarItemsJson;
use Pimcore\Bundle\StudioBackendBundle\OpenApi\Attributes\Response\Content\IdJson;
use Pimcore\Bundle\StudioBackendBundle\OpenApi\Attributes\Response\CreatedResponse;
use Pimcore\Bundle\StudioBackendBundle\OpenApi\Attributes\Response\DefaultResponses;
use Pimcore\Bundle\StudioBackendBundle\OpenApi\Attributes\Response\SuccessResponse;
use Pimcore\Bundle\StudioBackendBundle\OpenApi\Config\Tags;
use Pimcore\Bundle\StudioBackendBundle\Util\Constants\HttpResponseCodes;
use Pimcore\Bundle\StudioBackendBundle\Util\Constants\UserPermissions;
Expand Down Expand Up @@ -59,17 +58,9 @@ public function __construct(
#[RequestBody(
content: new ScalarItemsJson('integer')
)]
#[SuccessResponse(
content: new JsonContent(
properties: [
new Property(
property: 'path',
description: 'Path to the zip file',
type: 'string',
example: '/var/www/html/var/assets.zip'
),
]
)
#[CreatedResponse(
description: 'Successfully created jobRun for zip export',
content: new IdJson('ID of created jobRun', 'jobRunId')
)]
#[DefaultResponses([
HttpResponseCodes::UNAUTHORIZED,
Expand All @@ -78,6 +69,9 @@ public function __construct(
public function createZippedAssets(
#[MapRequestPayload] CreateAssetFileParameter $createAssetFileParameter
): Response {
return $this->jsonResponse(['path' => $this->zipService->generateZipFile($createAssetFileParameter)]);
return $this->jsonResponse(
['jobRunId' => $this->zipService->generateZipFile($createAssetFileParameter)],
HttpResponseCodes::CREATED->value
);
}
}
28 changes: 20 additions & 8 deletions src/Asset/Controller/DownloadCsvController.php
Original file line number Diff line number Diff line change
Expand Up @@ -19,17 +19,19 @@
use OpenApi\Attributes\Get;
use Pimcore\Bundle\StudioBackendBundle\Asset\Attributes\Response\Content\AssetMediaType;
use Pimcore\Bundle\StudioBackendBundle\Asset\Attributes\Response\Header\ContentDisposition;
use Pimcore\Bundle\StudioBackendBundle\Asset\MappedParameter\DownloadPathParameter;
use Pimcore\Bundle\StudioBackendBundle\Asset\Service\DownloadServiceInterface;
use Pimcore\Bundle\StudioBackendBundle\Asset\Service\ExecutionEngine\CsvServiceInterface;
use Pimcore\Bundle\StudioBackendBundle\Controller\AbstractApiController;
use Pimcore\Bundle\StudioBackendBundle\OpenApi\Attributes\Parameters\Query\PathParameter;
use Pimcore\Bundle\StudioBackendBundle\Exception\Api\ForbiddenException;
use Pimcore\Bundle\StudioBackendBundle\Exception\Api\NotFoundException;
use Pimcore\Bundle\StudioBackendBundle\OpenApi\Attributes\Parameters\Path\IdParameter;
use Pimcore\Bundle\StudioBackendBundle\OpenApi\Attributes\Response\DefaultResponses;
use Pimcore\Bundle\StudioBackendBundle\OpenApi\Attributes\Response\SuccessResponse;
use Pimcore\Bundle\StudioBackendBundle\OpenApi\Config\Tags;
use Pimcore\Bundle\StudioBackendBundle\Util\Constants\Asset\MimeTypes;
use Pimcore\Bundle\StudioBackendBundle\Util\Constants\HttpResponseCodes;
use Pimcore\Bundle\StudioBackendBundle\Util\Constants\UserPermissions;
use Symfony\Component\HttpFoundation\StreamedResponse;
use Symfony\Component\HttpKernel\Attribute\MapQueryString;
use Symfony\Component\Routing\Attribute\Route;
use Symfony\Component\Security\Http\Attribute\IsGranted;
use Symfony\Component\Serializer\SerializerInterface;
Expand All @@ -46,27 +48,37 @@ public function __construct(
parent::__construct($serializer);
}

#[Route('/assets/download/csv', name: 'pimcore_studio_api_csv_download_asset', methods: ['GET'])]
/**
* @throws NotFoundException|ForbiddenException
*/
#[Route('/assets/download/csv/{jobRunId}', name: 'pimcore_studio_api_csv_download_asset', methods: ['GET'])]
#[IsGranted(UserPermissions::ASSETS->value)]
#[Get(
path: self::API_PATH . '/assets/download/csv',
path: self::API_PATH . '/assets/download/csv/{jobRunId}',
operationId: 'downloadAssetsCsv',
description: 'Downloading csv file with assets',
summary: 'Downloading the csv file with assets',
tags: [Tags::Assets->name]
)]
#[PathParameter]
#[IdParameter(type: 'JobRun', name: 'jobRunId')]
#[SuccessResponse(
description: 'CSV File',
content: [new AssetMediaType('application/csv')],
headers: [new ContentDisposition()]
)]
#[DefaultResponses([
HttpResponseCodes::UNAUTHORIZED,
HttpResponseCodes::FORBIDDEN,
HttpResponseCodes::NOT_FOUND,
])]
public function downloadCsvAssets(#[MapQueryString] DownloadPathParameter $path): StreamedResponse
public function downloadCsvAssets(int $jobRunId): StreamedResponse
{
return $this->downloadService->downloadCsvByPath($path);
return $this->downloadService->downloadResourceByJobRunId(
$jobRunId,
CsvServiceInterface::CSV_FILE_NAME,
CsvServiceInterface::CSV_FOLDER_NAME,
MimeTypes::CSV->value,
'assets.csv'
);
}
}
22 changes: 14 additions & 8 deletions src/Asset/Controller/DownloadZipController.php
Original file line number Diff line number Diff line change
Expand Up @@ -19,17 +19,17 @@
use OpenApi\Attributes\Get;
use Pimcore\Bundle\StudioBackendBundle\Asset\Attributes\Response\Content\AssetMediaType;
use Pimcore\Bundle\StudioBackendBundle\Asset\Attributes\Response\Header\ContentDisposition;
use Pimcore\Bundle\StudioBackendBundle\Asset\MappedParameter\DownloadPathParameter;
use Pimcore\Bundle\StudioBackendBundle\Asset\Service\DownloadServiceInterface;
use Pimcore\Bundle\StudioBackendBundle\Asset\Service\ExecutionEngine\ZipServiceInterface;
use Pimcore\Bundle\StudioBackendBundle\Controller\AbstractApiController;
use Pimcore\Bundle\StudioBackendBundle\OpenApi\Attributes\Parameters\Query\PathParameter;
use Pimcore\Bundle\StudioBackendBundle\OpenApi\Attributes\Parameters\Path\IdParameter;
use Pimcore\Bundle\StudioBackendBundle\OpenApi\Attributes\Response\DefaultResponses;
use Pimcore\Bundle\StudioBackendBundle\OpenApi\Attributes\Response\SuccessResponse;
use Pimcore\Bundle\StudioBackendBundle\OpenApi\Config\Tags;
use Pimcore\Bundle\StudioBackendBundle\Util\Constants\Asset\MimeTypes;
use Pimcore\Bundle\StudioBackendBundle\Util\Constants\HttpResponseCodes;
use Pimcore\Bundle\StudioBackendBundle\Util\Constants\UserPermissions;
use Symfony\Component\HttpFoundation\StreamedResponse;
use Symfony\Component\HttpKernel\Attribute\MapQueryString;
use Symfony\Component\Routing\Attribute\Route;
use Symfony\Component\Security\Http\Attribute\IsGranted;
use Symfony\Component\Serializer\SerializerInterface;
Expand All @@ -46,16 +46,16 @@ public function __construct(
parent::__construct($serializer);
}

#[Route('/assets/download/zip', name: 'pimcore_studio_api_zip_download_asset', methods: ['GET'])]
#[Route('/assets/download/zip/{jobRunId}', name: 'pimcore_studio_api_zip_download_asset', methods: ['GET'])]
#[IsGranted(UserPermissions::ASSETS->value)]
#[Get(
path: self::API_PATH . '/assets/download/zip',
path: self::API_PATH . '/assets/download/zip/{jobRunId}',
operationId: 'downloadZippedAssets',
description: 'Downloading zipped assets',
summary: 'Downloading the zip file with assets',
tags: [Tags::Assets->name]
)]
#[PathParameter]
#[IdParameter(type: 'JobRun', name: 'jobRunId')]
#[SuccessResponse(
description: 'Zip archive',
content: [new AssetMediaType('application/zip')],
Expand All @@ -65,8 +65,14 @@ public function __construct(
HttpResponseCodes::UNAUTHORIZED,
HttpResponseCodes::NOT_FOUND,
])]
public function downloadZippedAssets(#[MapQueryString] DownloadPathParameter $path): StreamedResponse
public function downloadZippedAssets(int $jobRunId): StreamedResponse
{
return $this->downloadService->downloadZipArchiveByPath($path);
return $this->downloadService->downloadResourceByJobRunId(
$jobRunId,
ZipServiceInterface::DOWNLOAD_ZIP_FILE_NAME,
ZipServiceInterface::DOWNLOAD_ZIP_FOLDER_NAME,
MimeTypes::ZIP->value,
'assets.zip'
);
}
}
5 changes: 4 additions & 1 deletion src/Asset/Controller/Upload/AddController.php
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@

namespace Pimcore\Bundle\StudioBackendBundle\Asset\Controller\Upload;

use League\Flysystem\FilesystemException;
use OpenApi\Attributes\Post;
use OpenApi\Attributes\Property;
use Pimcore\Bundle\StudioBackendBundle\Asset\Attributes\Request\AddAssetRequestBody;
Expand Down Expand Up @@ -66,6 +67,7 @@ public function __construct(
* @throws ForbiddenException
* @throws NotFoundException
* @throws UserNotFoundException
* @throws FilesystemException
*/
#[Route('/assets/add/{parentId}', name: 'pimcore_studio_api_assets_add', methods: ['POST'])]
#[IsGranted(UserPermissions::ASSETS->value)]
Expand Down Expand Up @@ -110,7 +112,8 @@ public function addAsset(
[
'id' => $this->uploadService->uploadAsset(
$parentId,
$file,
$file->getClientOriginalName(),
$file->getRealPath(),
$this->securityService->getCurrentUser()
),
]
Expand Down
6 changes: 3 additions & 3 deletions src/Asset/Controller/Upload/ReplaceController.php
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,8 @@
use Pimcore\Bundle\StudioBackendBundle\Util\Constants\UserPermissions;
use Pimcore\Bundle\StudioBackendBundle\Util\Traits\PaginatedResponseTrait;
use Symfony\Component\HttpFoundation\File\UploadedFile;
use Symfony\Component\HttpFoundation\JsonResponse;
use Symfony\Component\HttpFoundation\Request;
use Symfony\Component\HttpFoundation\Response;
use Symfony\Component\Routing\Attribute\Route;
use Symfony\Component\Security\Http\Attribute\IsGranted;
use Symfony\Component\Serializer\SerializerInterface;
Expand Down Expand Up @@ -98,7 +98,7 @@ public function replaceAsset(
int $id,
// TODO: Symfony 7.1 change to https://symfony.com/blog/new-in-symfony-7-1-mapuploadedfile-attribute
Request $request
): JsonResponse {
): Response {

$file = $request->files->get('file');
if (!$file instanceof UploadedFile) {
Expand All @@ -111,6 +111,6 @@ public function replaceAsset(
$this->securityService->getCurrentUser()
);

return new JsonResponse();
return new Response();
}
}
Loading

0 comments on commit b9bae93

Please sign in to comment.