diff --git a/config/assets.yaml b/config/assets.yaml index e5ffd75d5..e15219446 100644 --- a/config/assets.yaml +++ b/config/assets.yaml @@ -89,24 +89,24 @@ services: # Handler # - Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Handler\CollectionHandler: ~ - Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Handler\ZipCreationHandler: ~ - Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Handler\CsvCreationHandler: ~ Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Handler\AssetCloneHandler: ~ Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Handler\AssetDeleteHandler: ~ Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Handler\AssetUploadHandler: ~ + Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Handler\CsvCreationHandler: ~ + Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Handler\CsvDataCollectionHandler: ~ + Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Handler\ZipDownloadHandler: ~ Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Handler\ZipUploadHandler: ~ # # Event Subscriber # - Pimcore\Bundle\StudioBackendBundle\Asset\EventSubscriber\ZipDownloadSubscriber: ~ - Pimcore\Bundle\StudioBackendBundle\Asset\EventSubscriber\ZipUploadSubscriber: ~ - Pimcore\Bundle\StudioBackendBundle\Asset\EventSubscriber\CsvDownloadSubscriber: ~ - Pimcore\Bundle\StudioBackendBundle\Asset\EventSubscriber\DeletionSubscriber: ~ Pimcore\Bundle\StudioBackendBundle\Asset\EventSubscriber\CloneSubscriber: ~ + Pimcore\Bundle\StudioBackendBundle\Asset\EventSubscriber\CsvCreationSubscriber: ~ + Pimcore\Bundle\StudioBackendBundle\Asset\EventSubscriber\DeletionSubscriber: ~ Pimcore\Bundle\StudioBackendBundle\Asset\EventSubscriber\UploadSubscriber: ~ + Pimcore\Bundle\StudioBackendBundle\Asset\EventSubscriber\ZipDownloadSubscriber: ~ + Pimcore\Bundle\StudioBackendBundle\Asset\EventSubscriber\ZipUploadSubscriber: ~ # # Mercure SSE diff --git a/config/elements.yaml b/config/elements.yaml index ba9eea60c..bbb11dfda 100644 --- a/config/elements.yaml +++ b/config/elements.yaml @@ -20,6 +20,9 @@ services: Pimcore\Bundle\StudioBackendBundle\Element\Service\ElementFolderServiceInterface: class: Pimcore\Bundle\StudioBackendBundle\Element\Service\ElementFolderService + Pimcore\Bundle\StudioBackendBundle\Element\Service\StorageServiceInterface: + class: Pimcore\Bundle\StudioBackendBundle\Element\Service\StorageService + # # Handler diff --git a/config/pimcore/execution_engine.yaml b/config/pimcore/execution_engine.yaml index 17b3c3166..61de624d8 100644 --- a/config/pimcore/execution_engine.yaml +++ b/config/pimcore/execution_engine.yaml @@ -11,11 +11,11 @@ pimcore_generic_execution_engine: framework: messenger: routing: - Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Messages\CollectionMessage: pimcore_generic_execution_engine - Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Messages\ZipCreationMessage: pimcore_generic_execution_engine + Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Messages\CsvCollectionMessage: pimcore_generic_execution_engine Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Messages\CsvCreationMessage: pimcore_generic_execution_engine Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Messages\AssetCloneMessage: pimcore_generic_execution_engine Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Messages\AssetDeleteMessage: pimcore_generic_execution_engine Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Messages\AssetUploadMessage: pimcore_generic_execution_engine + Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Messages\ZipDownloadMessage: pimcore_generic_execution_engine Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Messages\ZipUploadMessage: pimcore_generic_execution_engine Pimcore\Bundle\StudioBackendBundle\Element\ExecutionEngine\AutomationAction\Messenger\Messages\PatchMessage: pimcore_generic_execution_engine \ No newline at end of file diff --git a/doc/04_Generic_Execution_Engine.md b/doc/04_Generic_Execution_Engine.md new file mode 100644 index 000000000..4d0e82d3e --- /dev/null +++ b/doc/04_Generic_Execution_Engine.md @@ -0,0 +1,43 @@ +# Generic Execution Engine +:::caution + +This documentation is currently work in progress and will be updated soon. + +::: + +The Generic Execution Engine is a powerful tool to execute actions in the background. It is based on the Symfony Messenger component, to learn more about it, please visit the [Generic Execution Engine documentation](https://github.com/pimcore/pimcore/tree/11.x/doc/19_Development_Tools_and_Details/08_Generic_Execution_Engine). + +There are several actions, which currently take benefits of Execution Engine: + +### Asset ZIP upload +When uploading a ZIP file containing assets, the ZIP file is extracted and the assets are created in the background. + +### Asset ZIP Download +When downloading a multiple assets as ZIP file, the ZIP file is created in the background. This ZIP archive can then be downloaded. +There are some configuration options available for the ZIP download, which can be configured in the `config.yaml` file. + +```yaml +pimcore_studio_backend: + asset_download_settings: + # Maximum number of assets that can be downloaded in a single ZIP file. Default value is 1000. + amount_limit: 1000 + # Maximum size of the ZIP file in bytes. Default value is 5 GB. + size_limit: 5368709120 +``` + +### Asset CSV Export +Assets can be exported based on the provided grid configuration as a CSV file. The export is done in the background and can be downloaded after its finished. +There are some configuration options available for the CSV export, which can be configured in the `config.yaml` file. + +```yaml +pimcore_studio_backend: + csv_settings: + # Default delimiter for CSV files when no value is passed by grid configuration. Default value is ','. + default_delimiter: ',' +``` + +### Assets deletion +Assets and folders can be deleted in the background. This is useful when deleting a large number of assets or asset trees. + +### Asset cloning +Assets and folders can be cloned in the background. This is useful when cloning a large number of assets or asset trees. diff --git a/src/Asset/Controller/CreateCsvController.php b/src/Asset/Controller/CreateCsvController.php index dfbd006eb..61c927801 100644 --- a/src/Asset/Controller/CreateCsvController.php +++ b/src/Asset/Controller/CreateCsvController.php @@ -16,15 +16,14 @@ namespace Pimcore\Bundle\StudioBackendBundle\Asset\Controller; -use OpenApi\Attributes\JsonContent; use OpenApi\Attributes\Post; -use OpenApi\Attributes\Property; use Pimcore\Bundle\StudioBackendBundle\Asset\Attributes\Request\CsvExportRequestBody; use Pimcore\Bundle\StudioBackendBundle\Asset\MappedParameter\ExportAssetParameter; use Pimcore\Bundle\StudioBackendBundle\Asset\Service\ExecutionEngine\CsvServiceInterface; use Pimcore\Bundle\StudioBackendBundle\Controller\AbstractApiController; +use Pimcore\Bundle\StudioBackendBundle\OpenApi\Attributes\Response\Content\IdJson; +use Pimcore\Bundle\StudioBackendBundle\OpenApi\Attributes\Response\CreatedResponse; use Pimcore\Bundle\StudioBackendBundle\OpenApi\Attributes\Response\DefaultResponses; -use Pimcore\Bundle\StudioBackendBundle\OpenApi\Attributes\Response\SuccessResponse; use Pimcore\Bundle\StudioBackendBundle\OpenApi\Config\Tags; use Pimcore\Bundle\StudioBackendBundle\Util\Constants\HttpResponseCodes; use Pimcore\Bundle\StudioBackendBundle\Util\Constants\UserPermissions; @@ -56,17 +55,9 @@ public function __construct( tags: [Tags::Assets->name] )] #[CsvExportRequestBody] - #[SuccessResponse( - content: new JsonContent( - properties: [ - new Property( - property: 'path', - description: 'Path to the csv file', - type: 'string', - example: '/var/www/html/var/assets.csv' - ), - ] - ) + #[CreatedResponse( + description: 'Successfully created jobRun for csv export', + content: new IdJson('ID of created jobRun', 'jobRunId') )] #[DefaultResponses([ HttpResponseCodes::UNAUTHORIZED, @@ -75,6 +66,9 @@ public function __construct( public function createCsvAssets( #[MapRequestPayload] ExportAssetParameter $exportAssetParameter ): Response { - return $this->jsonResponse(['path' => $this->csvService->generateCsvFile($exportAssetParameter)]); + return $this->jsonResponse( + ['jobRunId' => $this->csvService->generateCsvFile($exportAssetParameter)], + HttpResponseCodes::CREATED->value + ); } } diff --git a/src/Asset/Controller/CreateZipController.php b/src/Asset/Controller/CreateZipController.php index ccfc1a7c6..d198388ae 100644 --- a/src/Asset/Controller/CreateZipController.php +++ b/src/Asset/Controller/CreateZipController.php @@ -16,16 +16,15 @@ namespace Pimcore\Bundle\StudioBackendBundle\Asset\Controller; -use OpenApi\Attributes\JsonContent; use OpenApi\Attributes\Post; -use OpenApi\Attributes\Property; use OpenApi\Attributes\RequestBody; use Pimcore\Bundle\StudioBackendBundle\Asset\MappedParameter\CreateAssetFileParameter; use Pimcore\Bundle\StudioBackendBundle\Asset\Service\ExecutionEngine\ZipServiceInterface; use Pimcore\Bundle\StudioBackendBundle\Controller\AbstractApiController; use Pimcore\Bundle\StudioBackendBundle\OpenApi\Attributes\Content\ScalarItemsJson; +use Pimcore\Bundle\StudioBackendBundle\OpenApi\Attributes\Response\Content\IdJson; +use Pimcore\Bundle\StudioBackendBundle\OpenApi\Attributes\Response\CreatedResponse; use Pimcore\Bundle\StudioBackendBundle\OpenApi\Attributes\Response\DefaultResponses; -use Pimcore\Bundle\StudioBackendBundle\OpenApi\Attributes\Response\SuccessResponse; use Pimcore\Bundle\StudioBackendBundle\OpenApi\Config\Tags; use Pimcore\Bundle\StudioBackendBundle\Util\Constants\HttpResponseCodes; use Pimcore\Bundle\StudioBackendBundle\Util\Constants\UserPermissions; @@ -59,17 +58,9 @@ public function __construct( #[RequestBody( content: new ScalarItemsJson('integer') )] - #[SuccessResponse( - content: new JsonContent( - properties: [ - new Property( - property: 'path', - description: 'Path to the zip file', - type: 'string', - example: '/var/www/html/var/assets.zip' - ), - ] - ) + #[CreatedResponse( + description: 'Successfully created jobRun for zip export', + content: new IdJson('ID of created jobRun', 'jobRunId') )] #[DefaultResponses([ HttpResponseCodes::UNAUTHORIZED, @@ -78,6 +69,9 @@ public function __construct( public function createZippedAssets( #[MapRequestPayload] CreateAssetFileParameter $createAssetFileParameter ): Response { - return $this->jsonResponse(['path' => $this->zipService->generateZipFile($createAssetFileParameter)]); + return $this->jsonResponse( + ['jobRunId' => $this->zipService->generateZipFile($createAssetFileParameter)], + HttpResponseCodes::CREATED->value + ); } } diff --git a/src/Asset/Controller/DownloadCsvController.php b/src/Asset/Controller/DownloadCsvController.php index e2c3e0c7b..fcdcf76d1 100644 --- a/src/Asset/Controller/DownloadCsvController.php +++ b/src/Asset/Controller/DownloadCsvController.php @@ -19,17 +19,19 @@ use OpenApi\Attributes\Get; use Pimcore\Bundle\StudioBackendBundle\Asset\Attributes\Response\Content\AssetMediaType; use Pimcore\Bundle\StudioBackendBundle\Asset\Attributes\Response\Header\ContentDisposition; -use Pimcore\Bundle\StudioBackendBundle\Asset\MappedParameter\DownloadPathParameter; use Pimcore\Bundle\StudioBackendBundle\Asset\Service\DownloadServiceInterface; +use Pimcore\Bundle\StudioBackendBundle\Asset\Service\ExecutionEngine\CsvServiceInterface; use Pimcore\Bundle\StudioBackendBundle\Controller\AbstractApiController; -use Pimcore\Bundle\StudioBackendBundle\OpenApi\Attributes\Parameters\Query\PathParameter; +use Pimcore\Bundle\StudioBackendBundle\Exception\Api\ForbiddenException; +use Pimcore\Bundle\StudioBackendBundle\Exception\Api\NotFoundException; +use Pimcore\Bundle\StudioBackendBundle\OpenApi\Attributes\Parameters\Path\IdParameter; use Pimcore\Bundle\StudioBackendBundle\OpenApi\Attributes\Response\DefaultResponses; use Pimcore\Bundle\StudioBackendBundle\OpenApi\Attributes\Response\SuccessResponse; use Pimcore\Bundle\StudioBackendBundle\OpenApi\Config\Tags; +use Pimcore\Bundle\StudioBackendBundle\Util\Constants\Asset\MimeTypes; use Pimcore\Bundle\StudioBackendBundle\Util\Constants\HttpResponseCodes; use Pimcore\Bundle\StudioBackendBundle\Util\Constants\UserPermissions; use Symfony\Component\HttpFoundation\StreamedResponse; -use Symfony\Component\HttpKernel\Attribute\MapQueryString; use Symfony\Component\Routing\Attribute\Route; use Symfony\Component\Security\Http\Attribute\IsGranted; use Symfony\Component\Serializer\SerializerInterface; @@ -46,16 +48,19 @@ public function __construct( parent::__construct($serializer); } - #[Route('/assets/download/csv', name: 'pimcore_studio_api_csv_download_asset', methods: ['GET'])] + /** + * @throws NotFoundException|ForbiddenException + */ + #[Route('/assets/download/csv/{jobRunId}', name: 'pimcore_studio_api_csv_download_asset', methods: ['GET'])] #[IsGranted(UserPermissions::ASSETS->value)] #[Get( - path: self::API_PATH . '/assets/download/csv', + path: self::API_PATH . '/assets/download/csv/{jobRunId}', operationId: 'downloadAssetsCsv', description: 'Downloading csv file with assets', summary: 'Downloading the csv file with assets', tags: [Tags::Assets->name] )] - #[PathParameter] + #[IdParameter(type: 'JobRun', name: 'jobRunId')] #[SuccessResponse( description: 'CSV File', content: [new AssetMediaType('application/csv')], @@ -63,10 +68,17 @@ public function __construct( )] #[DefaultResponses([ HttpResponseCodes::UNAUTHORIZED, + HttpResponseCodes::FORBIDDEN, HttpResponseCodes::NOT_FOUND, ])] - public function downloadCsvAssets(#[MapQueryString] DownloadPathParameter $path): StreamedResponse + public function downloadCsvAssets(int $jobRunId): StreamedResponse { - return $this->downloadService->downloadCsvByPath($path); + return $this->downloadService->downloadResourceByJobRunId( + $jobRunId, + CsvServiceInterface::CSV_FILE_NAME, + CsvServiceInterface::CSV_FOLDER_NAME, + MimeTypes::CSV->value, + 'assets.csv' + ); } } diff --git a/src/Asset/Controller/DownloadZipController.php b/src/Asset/Controller/DownloadZipController.php index cd1d4b0c4..373bc7d0b 100644 --- a/src/Asset/Controller/DownloadZipController.php +++ b/src/Asset/Controller/DownloadZipController.php @@ -19,17 +19,17 @@ use OpenApi\Attributes\Get; use Pimcore\Bundle\StudioBackendBundle\Asset\Attributes\Response\Content\AssetMediaType; use Pimcore\Bundle\StudioBackendBundle\Asset\Attributes\Response\Header\ContentDisposition; -use Pimcore\Bundle\StudioBackendBundle\Asset\MappedParameter\DownloadPathParameter; use Pimcore\Bundle\StudioBackendBundle\Asset\Service\DownloadServiceInterface; +use Pimcore\Bundle\StudioBackendBundle\Asset\Service\ExecutionEngine\ZipServiceInterface; use Pimcore\Bundle\StudioBackendBundle\Controller\AbstractApiController; -use Pimcore\Bundle\StudioBackendBundle\OpenApi\Attributes\Parameters\Query\PathParameter; +use Pimcore\Bundle\StudioBackendBundle\OpenApi\Attributes\Parameters\Path\IdParameter; use Pimcore\Bundle\StudioBackendBundle\OpenApi\Attributes\Response\DefaultResponses; use Pimcore\Bundle\StudioBackendBundle\OpenApi\Attributes\Response\SuccessResponse; use Pimcore\Bundle\StudioBackendBundle\OpenApi\Config\Tags; +use Pimcore\Bundle\StudioBackendBundle\Util\Constants\Asset\MimeTypes; use Pimcore\Bundle\StudioBackendBundle\Util\Constants\HttpResponseCodes; use Pimcore\Bundle\StudioBackendBundle\Util\Constants\UserPermissions; use Symfony\Component\HttpFoundation\StreamedResponse; -use Symfony\Component\HttpKernel\Attribute\MapQueryString; use Symfony\Component\Routing\Attribute\Route; use Symfony\Component\Security\Http\Attribute\IsGranted; use Symfony\Component\Serializer\SerializerInterface; @@ -46,16 +46,16 @@ public function __construct( parent::__construct($serializer); } - #[Route('/assets/download/zip', name: 'pimcore_studio_api_zip_download_asset', methods: ['GET'])] + #[Route('/assets/download/zip/{jobRunId}', name: 'pimcore_studio_api_zip_download_asset', methods: ['GET'])] #[IsGranted(UserPermissions::ASSETS->value)] #[Get( - path: self::API_PATH . '/assets/download/zip', + path: self::API_PATH . '/assets/download/zip/{jobRunId}', operationId: 'downloadZippedAssets', description: 'Downloading zipped assets', summary: 'Downloading the zip file with assets', tags: [Tags::Assets->name] )] - #[PathParameter] + #[IdParameter(type: 'JobRun', name: 'jobRunId')] #[SuccessResponse( description: 'Zip archive', content: [new AssetMediaType('application/zip')], @@ -65,8 +65,14 @@ public function __construct( HttpResponseCodes::UNAUTHORIZED, HttpResponseCodes::NOT_FOUND, ])] - public function downloadZippedAssets(#[MapQueryString] DownloadPathParameter $path): StreamedResponse + public function downloadZippedAssets(int $jobRunId): StreamedResponse { - return $this->downloadService->downloadZipArchiveByPath($path); + return $this->downloadService->downloadResourceByJobRunId( + $jobRunId, + ZipServiceInterface::DOWNLOAD_ZIP_FILE_NAME, + ZipServiceInterface::DOWNLOAD_ZIP_FOLDER_NAME, + MimeTypes::ZIP->value, + 'assets.zip' + ); } } diff --git a/src/Asset/Controller/Upload/AddController.php b/src/Asset/Controller/Upload/AddController.php index 3afa146ca..b0968659f 100644 --- a/src/Asset/Controller/Upload/AddController.php +++ b/src/Asset/Controller/Upload/AddController.php @@ -16,6 +16,7 @@ namespace Pimcore\Bundle\StudioBackendBundle\Asset\Controller\Upload; +use League\Flysystem\FilesystemException; use OpenApi\Attributes\Post; use OpenApi\Attributes\Property; use Pimcore\Bundle\StudioBackendBundle\Asset\Attributes\Request\AddAssetRequestBody; @@ -66,6 +67,7 @@ public function __construct( * @throws ForbiddenException * @throws NotFoundException * @throws UserNotFoundException + * @throws FilesystemException */ #[Route('/assets/add/{parentId}', name: 'pimcore_studio_api_assets_add', methods: ['POST'])] #[IsGranted(UserPermissions::ASSETS->value)] @@ -110,7 +112,8 @@ public function addAsset( [ 'id' => $this->uploadService->uploadAsset( $parentId, - $file, + $file->getClientOriginalName(), + $file->getRealPath(), $this->securityService->getCurrentUser() ), ] diff --git a/src/Asset/Controller/Upload/ReplaceController.php b/src/Asset/Controller/Upload/ReplaceController.php index 353b33324..80403c747 100644 --- a/src/Asset/Controller/Upload/ReplaceController.php +++ b/src/Asset/Controller/Upload/ReplaceController.php @@ -38,8 +38,8 @@ use Pimcore\Bundle\StudioBackendBundle\Util\Constants\UserPermissions; use Pimcore\Bundle\StudioBackendBundle\Util\Traits\PaginatedResponseTrait; use Symfony\Component\HttpFoundation\File\UploadedFile; -use Symfony\Component\HttpFoundation\JsonResponse; use Symfony\Component\HttpFoundation\Request; +use Symfony\Component\HttpFoundation\Response; use Symfony\Component\Routing\Attribute\Route; use Symfony\Component\Security\Http\Attribute\IsGranted; use Symfony\Component\Serializer\SerializerInterface; @@ -98,7 +98,7 @@ public function replaceAsset( int $id, // TODO: Symfony 7.1 change to https://symfony.com/blog/new-in-symfony-7-1-mapuploadedfile-attribute Request $request - ): JsonResponse { + ): Response { $file = $request->files->get('file'); if (!$file instanceof UploadedFile) { @@ -111,6 +111,6 @@ public function replaceAsset( $this->securityService->getCurrentUser() ); - return new JsonResponse(); + return new Response(); } } diff --git a/src/Asset/EventSubscriber/CsvDownloadSubscriber.php b/src/Asset/EventSubscriber/CsvCreationSubscriber.php similarity index 52% rename from src/Asset/EventSubscriber/CsvDownloadSubscriber.php rename to src/Asset/EventSubscriber/CsvCreationSubscriber.php index 8ee0f095a..2f1715a4d 100644 --- a/src/Asset/EventSubscriber/CsvDownloadSubscriber.php +++ b/src/Asset/EventSubscriber/CsvCreationSubscriber.php @@ -16,24 +16,26 @@ namespace Pimcore\Bundle\StudioBackendBundle\Asset\EventSubscriber; +use League\Flysystem\FilesystemException; use Pimcore\Bundle\GenericExecutionEngineBundle\Event\JobRunStateChangedEvent; use Pimcore\Bundle\GenericExecutionEngineBundle\Model\JobRunStates; use Pimcore\Bundle\StudioBackendBundle\Asset\Mercure\Events; -use Pimcore\Bundle\StudioBackendBundle\Asset\Mercure\Schema\DownloadReady; use Pimcore\Bundle\StudioBackendBundle\Asset\Service\ExecutionEngine\CsvServiceInterface; -use Pimcore\Bundle\StudioBackendBundle\Exception\JsonEncodingException; +use Pimcore\Bundle\StudioBackendBundle\Element\Service\StorageServiceInterface; use Pimcore\Bundle\StudioBackendBundle\ExecutionEngine\Util\Jobs; +use Pimcore\Bundle\StudioBackendBundle\Mercure\Schema\ExecutionEngine\Finished; use Pimcore\Bundle\StudioBackendBundle\Mercure\Service\PublishServiceInterface; use Symfony\Component\EventDispatcher\EventSubscriberInterface; /** * @internal */ -final readonly class CsvDownloadSubscriber implements EventSubscriberInterface +final readonly class CsvCreationSubscriber implements EventSubscriberInterface { public function __construct( + private CsvServiceInterface $csvService, private PublishServiceInterface $publishService, - private CsvServiceInterface $csvService + private StorageServiceInterface $storageService, ) { } @@ -46,23 +48,43 @@ public static function getSubscribedEvents(): array } /** - * @throws JsonEncodingException + * @throws FilesystemException */ public function onStateChanged(JobRunStateChangedEvent $event): void { + if ($event->getJobName() !== Jobs::CREATE_CSV->value) { + return; + } - if ( - $event->getNewState() === JobRunStates::FINISHED->value && - $event->getJobName() === Jobs::CREATE_CSV->value - ) { - $this->publishService->publish( + match ($event->getNewState()) { + JobRunStates::FINISHED->value => $this->publishService->publish( Events::CSV_DOWNLOAD_READY->value, - new DownloadReady( + new Finished( $event->getJobRunId(), - $this->csvService->getTempFilePath($event->getJobRunId(), CsvServiceInterface::CSV_FILE_PATH), - $event->getJobRunOwnerId() + $event->getJobName(), + $event->getJobRunOwnerId(), + $event->getNewState() ) - ); - } + ), + JobRunStates::FAILED->value => $this->cleanupOnFail($event->getJobRunId()), + default => null, + }; + } + + /** + * @throws FilesystemException + */ + private function cleanupOnFail(int $jobRunId): void + { + $this->storageService->cleanUpFlysystemFile( + $this->csvService->getTempFilePath( + $jobRunId, + CsvServiceInterface::CSV_FOLDER_NAME . '/' . CsvServiceInterface::CSV_FILE_NAME + ) + ); + + $this->storageService->cleanUpFolder( + $this->csvService->getTempFilePath($jobRunId, CsvServiceInterface::CSV_FOLDER_NAME) + ); } } diff --git a/src/Asset/EventSubscriber/UploadSubscriber.php b/src/Asset/EventSubscriber/UploadSubscriber.php index f6f5d132f..ba53fe1fd 100644 --- a/src/Asset/EventSubscriber/UploadSubscriber.php +++ b/src/Asset/EventSubscriber/UploadSubscriber.php @@ -16,9 +16,13 @@ namespace Pimcore\Bundle\StudioBackendBundle\Asset\EventSubscriber; +use League\Flysystem\FilesystemException; use Pimcore\Bundle\GenericExecutionEngineBundle\Event\JobRunStateChangedEvent; use Pimcore\Bundle\GenericExecutionEngineBundle\Model\JobRunStates; +use Pimcore\Bundle\GenericExecutionEngineBundle\Repository\JobRunRepositoryInterface; +use Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\Util\EnvironmentVariables; use Pimcore\Bundle\StudioBackendBundle\Asset\Mercure\Events; +use Pimcore\Bundle\StudioBackendBundle\Asset\Service\UploadServiceInterface; use Pimcore\Bundle\StudioBackendBundle\ExecutionEngine\Service\EventSubscriberServiceInterface; use Pimcore\Bundle\StudioBackendBundle\ExecutionEngine\Util\Jobs; use Pimcore\Bundle\StudioBackendBundle\Mercure\Schema\ExecutionEngine\Finished; @@ -32,7 +36,9 @@ { public function __construct( private EventSubscriberServiceInterface $eventSubscriberService, + private JobRunRepositoryInterface $jobRunRepository, private PublishServiceInterface $publishService, + private UploadServiceInterface $uploadService, ) { } @@ -44,13 +50,17 @@ public static function getSubscribedEvents(): array ]; } + /** + * @throws FilesystemException + */ public function onStateChanged(JobRunStateChangedEvent $event): void { - if ($event->getJobName() !== Jobs::UPLOAD_ASSETS->value) { + if ($event->getJobName() !== Jobs::UPLOAD_ASSETS->value) { return; } + $state = $event->getNewState(); - match ($event->getNewState()) { + match ($state) { JobRunStates::FINISHED->value => $this->publishService->publish( Events::ASSET_UPLOAD_FINISHED->value, new Finished( @@ -67,5 +77,26 @@ public function onStateChanged(JobRunStateChangedEvent $event): void ), default => null, }; + + if ($state !== JobRunStates::RUNNING->value && $state !== JobRunStates::NOT_STARTED->value) { + $this->cleanupData($event->getJobRunId()); + } + } + + /** + * @throws FilesystemException + */ + private function cleanupData(int $jobRunId): void + { + $environmentVariables = $this->jobRunRepository->getJobRunById( + $jobRunId + )->getJob()?->getEnvironmentData(); + if ($environmentVariables && + isset($environmentVariables[EnvironmentVariables::UPLOAD_FOLDER_LOCATION->value]) + ) { + $this->uploadService->cleanupTemporaryUploadFiles( + $environmentVariables[EnvironmentVariables::UPLOAD_FOLDER_LOCATION->value] + ); + } } } diff --git a/src/Asset/EventSubscriber/ZipDownloadSubscriber.php b/src/Asset/EventSubscriber/ZipDownloadSubscriber.php index ec6ebbdbe..ef8d03243 100644 --- a/src/Asset/EventSubscriber/ZipDownloadSubscriber.php +++ b/src/Asset/EventSubscriber/ZipDownloadSubscriber.php @@ -19,10 +19,10 @@ use Pimcore\Bundle\GenericExecutionEngineBundle\Event\JobRunStateChangedEvent; use Pimcore\Bundle\GenericExecutionEngineBundle\Model\JobRunStates; use Pimcore\Bundle\StudioBackendBundle\Asset\Mercure\Events; -use Pimcore\Bundle\StudioBackendBundle\Asset\Mercure\Schema\DownloadReady; use Pimcore\Bundle\StudioBackendBundle\Asset\Service\ExecutionEngine\ZipServiceInterface; -use Pimcore\Bundle\StudioBackendBundle\Exception\JsonEncodingException; +use Pimcore\Bundle\StudioBackendBundle\Element\Service\StorageServiceInterface; use Pimcore\Bundle\StudioBackendBundle\ExecutionEngine\Util\Jobs; +use Pimcore\Bundle\StudioBackendBundle\Mercure\Schema\ExecutionEngine\Finished; use Pimcore\Bundle\StudioBackendBundle\Mercure\Service\PublishServiceInterface; use Symfony\Component\EventDispatcher\EventSubscriberInterface; @@ -33,6 +33,7 @@ { public function __construct( private PublishServiceInterface $publishService, + private StorageServiceInterface $storageService, private ZipServiceInterface $zipService ) { @@ -45,27 +46,29 @@ public static function getSubscribedEvents(): array ]; } - /** - * @throws JsonEncodingException - */ public function onStateChanged(JobRunStateChangedEvent $event): void { + if ($event->getJobName() !== Jobs::CREATE_ZIP->value) { + return; + } - if ( - $event->getNewState() === JobRunStates::FINISHED->value && - $event->getJobName() === Jobs::CREATE_ZIP->value - ) { - $this->publishService->publish( + match ($event->getNewState()) { + JobRunStates::FINISHED->value => $this->publishService->publish( Events::ZIP_DOWNLOAD_READY->value, - new DownloadReady( + new Finished( $event->getJobRunId(), - $this->zipService->getTempFilePath( - $event->getJobRunId(), - ZipServiceInterface::DOWNLOAD_ZIP_FILE_PATH - ), - $event->getJobRunOwnerId() + $event->getJobName(), + $event->getJobRunOwnerId(), + $event->getNewState() ) - ); - } + ), + JobRunStates::FAILED->value => $this->storageService->cleanUpLocalFile( + $this->zipService->getTempFilePath( + $event->getJobRunId(), + ZipServiceInterface::DOWNLOAD_ZIP_FILE_PATH + ), + ), + default => null, + }; } } diff --git a/src/Asset/EventSubscriber/ZipUploadSubscriber.php b/src/Asset/EventSubscriber/ZipUploadSubscriber.php index 4cc476bbe..fa44a639d 100644 --- a/src/Asset/EventSubscriber/ZipUploadSubscriber.php +++ b/src/Asset/EventSubscriber/ZipUploadSubscriber.php @@ -16,11 +16,14 @@ namespace Pimcore\Bundle\StudioBackendBundle\Asset\EventSubscriber; +use League\Flysystem\FilesystemException; +use Pimcore\Bundle\GenericExecutionEngineBundle\Entity\JobRun; use Pimcore\Bundle\GenericExecutionEngineBundle\Event\JobRunStateChangedEvent; use Pimcore\Bundle\GenericExecutionEngineBundle\Model\JobRunStates; use Pimcore\Bundle\GenericExecutionEngineBundle\Repository\JobRunRepositoryInterface; use Pimcore\Bundle\StudioBackendBundle\Asset\Mercure\Events; -use Pimcore\Bundle\StudioBackendBundle\Exception\JsonEncodingException; +use Pimcore\Bundle\StudioBackendBundle\Asset\Service\ExecutionEngine\ZipServiceInterface; +use Pimcore\Bundle\StudioBackendBundle\Asset\Service\UploadServiceInterface; use Pimcore\Bundle\StudioBackendBundle\ExecutionEngine\Util\JobRunContext; use Pimcore\Bundle\StudioBackendBundle\ExecutionEngine\Util\Jobs; use Pimcore\Bundle\StudioBackendBundle\Mercure\Schema\ExecutionEngine\Finished; @@ -34,7 +37,9 @@ { public function __construct( private JobRunRepositoryInterface $jobRunRepository, - private PublishServiceInterface $publishService + private PublishServiceInterface $publishService, + private UploadServiceInterface $uploadService, + private ZipServiceInterface $zipService, ) { } @@ -47,28 +52,45 @@ public static function getSubscribedEvents(): array } /** - * @throws JsonEncodingException + * @throws FilesystemException */ public function onStateChanged(JobRunStateChangedEvent $event): void { + if ($event->getJobName() !== Jobs::ZIP_FILE_UPLOAD->value) { - if ( - $event->getNewState() === JobRunStates::FINISHED->value && - $event->getJobName() === Jobs::ZIP_FILE_UPLOAD->value - ) { - $jobRun = $this->jobRunRepository->getJobRunById($event->getJobRunId()); - $childJobRunId = $jobRun->getContext()[JobRunContext::CHILD_JOB_RUN->value]; + return; + } - $this->publishService->publish( + $jobRun = $this->jobRunRepository->getJobRunById($event->getJobRunId()); + match ($event->getNewState()) { + JobRunStates::FINISHED->value => $this->publishService->publish( Events::ZIP_UPLOAD_FINISHED->value, new Finished( $event->getJobRunId(), $event->getJobName(), $event->getJobRunOwnerId(), $event->getNewState(), - ['childJobRunId' => $childJobRunId], + ['childJobRunId' => $jobRun->getContext()[JobRunContext::CHILD_JOB_RUN->value]], ) - ); + ), + JobRunStates::FAILED->value => $this->cleanupData($jobRun), + default => null, + }; + } + + /** + * @throws FilesystemException + */ + private function cleanupData(JobRun $jobRun): void + { + $subject = $jobRun->getJob()?->getSelectedElements()[0]; + if ($subject === null) { + + return; } + + $this->uploadService->cleanupTemporaryUploadFiles( + $this->zipService->getTempFilePath($subject->getType(), ZipServiceInterface::UPLOAD_ZIP_FOLDER_NAME) + ); } } diff --git a/src/Asset/ExecutionEngine/AutomationAction/Messenger/Handler/AssetUploadHandler.php b/src/Asset/ExecutionEngine/AutomationAction/Messenger/Handler/AssetUploadHandler.php index b8d52ad3d..136d3dd40 100644 --- a/src/Asset/ExecutionEngine/AutomationAction/Messenger/Handler/AssetUploadHandler.php +++ b/src/Asset/ExecutionEngine/AutomationAction/Messenger/Handler/AssetUploadHandler.php @@ -21,15 +21,14 @@ use Pimcore\Bundle\StaticResolverBundle\Models\User\UserResolverInterface; use Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Messages\AssetUploadMessage; use Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\Util\EnvironmentVariables; -use Pimcore\Bundle\StudioBackendBundle\Asset\Service\ExecutionEngine\ZipServiceInterface; use Pimcore\Bundle\StudioBackendBundle\Asset\Service\UploadServiceInterface; use Pimcore\Bundle\StudioBackendBundle\ExecutionEngine\AutomationAction\AbstractHandler; use Pimcore\Bundle\StudioBackendBundle\ExecutionEngine\Model\AbortActionData; use Pimcore\Bundle\StudioBackendBundle\ExecutionEngine\Util\Config; use Pimcore\Bundle\StudioBackendBundle\Mercure\Service\PublishServiceInterface; use Pimcore\Bundle\StudioBackendBundle\Util\Traits\HandlerProgressTrait; -use Symfony\Component\HttpFoundation\File\UploadedFile; use Symfony\Component\Messenger\Attribute\AsMessageHandler; +use function dirname; /** * @internal @@ -43,7 +42,6 @@ public function __construct( private readonly PublishServiceInterface $publishService, private readonly UserResolverInterface $userResolver, private readonly UploadServiceInterface $uploadService, - private readonly ZipServiceInterface $zipService, ) { parent::__construct(); } @@ -60,7 +58,7 @@ public function __invoke(AssetUploadMessage $message): void $this->userResolver, [ EnvironmentVariables::PARENT_ID->value, - EnvironmentVariables::UPLOAD_FOLDER_NAME->value, + EnvironmentVariables::UPLOAD_FOLDER_LOCATION->value, ], ); @@ -74,21 +72,24 @@ public function __invoke(AssetUploadMessage $message): void try { $element = $validatedParameters->getSubject()->getType(); $fileData = json_decode($element, true, 512, JSON_THROW_ON_ERROR); - $file = new UploadedFile( - $fileData['sourcePath'], - $fileData['fileName'], - ); + $folderLocation = dirname($fileData['path']); + $parentId = $environmentVariables[EnvironmentVariables::PARENT_ID->value]; - $this->uploadService->uploadAsset( - $environmentVariables[EnvironmentVariables::PARENT_ID->value], - $file, - $user - ); + if ($folderLocation !== '.') { + $parentId = $this->uploadService->uploadParentFolder( + $fileData['path'], + $parentId, + $user, + ); + } - $this->zipService->cleanUpArchiveFolder( - $environmentVariables[EnvironmentVariables::UPLOAD_FOLDER_NAME->value], + $this->uploadService->uploadAsset( + $parentId, + $fileData['name'], + $fileData['sourcePath'], + $user, + true ); - } catch (Exception|FilesystemException $exception) { $this->abort($this->getAbortData( Config::ASSET_UPLOAD_FAILED_MESSAGE->value, diff --git a/src/Asset/ExecutionEngine/AutomationAction/Messenger/Handler/CollectionHandler.php b/src/Asset/ExecutionEngine/AutomationAction/Messenger/Handler/CollectionHandler.php deleted file mode 100644 index 81413a44d..000000000 --- a/src/Asset/ExecutionEngine/AutomationAction/Messenger/Handler/CollectionHandler.php +++ /dev/null @@ -1,96 +0,0 @@ -getJobRun($message); - $validatedParameters = $this->validateJobParameters( - $message, - $jobRun, - $this->userResolver - ); - - if ($validatedParameters instanceof AbortActionData) { - $this->abort($validatedParameters); - } - - $user = $validatedParameters->getUser(); - $jobAsset = $validatedParameters->getSubject(); - $asset = $this->getElementById( - $jobAsset, - $user, - $this->elementService - ); - - if ($asset->getType() === ElementTypes::TYPE_FOLDER) { - $this->abort($this->getAbortData( - Config::ELEMENT_FOLDER_COLLECTION_NOT_SUPPORTED->value, - [ - 'folderId' => $asset->getId(), - ] - )); - } - - $context = $jobRun->getContext(); - - $assets = $context[ZipServiceInterface::ASSETS_INDEX] ?? []; - - if (in_array($jobAsset->getId(), $assets, true)) { - return; - } - - $assets[] = $jobAsset->getId(); - - $this->updateJobRunContext($jobRun, ZipServiceInterface::ASSETS_INDEX, $assets); - - $this->updateProgress($this->publishService, $jobRun, $this->getJobStep($message)->getName()); - } -} diff --git a/src/Asset/ExecutionEngine/AutomationAction/Messenger/Handler/CsvCreationHandler.php b/src/Asset/ExecutionEngine/AutomationAction/Messenger/Handler/CsvCreationHandler.php index a0bb203f5..3a1e32500 100644 --- a/src/Asset/ExecutionEngine/AutomationAction/Messenger/Handler/CsvCreationHandler.php +++ b/src/Asset/ExecutionEngine/AutomationAction/Messenger/Handler/CsvCreationHandler.php @@ -17,23 +17,16 @@ namespace Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Handler; use Exception; -use Pimcore\Bundle\StaticResolverBundle\Models\User\UserResolverInterface; +use League\Flysystem\FilesystemException; use Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Messages\CsvCreationMessage; use Pimcore\Bundle\StudioBackendBundle\Asset\Service\ExecutionEngine\CsvServiceInterface; -use Pimcore\Bundle\StudioBackendBundle\Asset\Service\ExecutionEngine\ZipServiceInterface; -use Pimcore\Bundle\StudioBackendBundle\Element\Service\ElementServiceInterface; +use Pimcore\Bundle\StudioBackendBundle\Asset\Util\Constants\Csv; use Pimcore\Bundle\StudioBackendBundle\ExecutionEngine\AutomationAction\AbstractHandler; -use Pimcore\Bundle\StudioBackendBundle\ExecutionEngine\Model\AbortActionData; use Pimcore\Bundle\StudioBackendBundle\ExecutionEngine\Util\Config; use Pimcore\Bundle\StudioBackendBundle\Grid\Service\GridServiceInterface; use Pimcore\Bundle\StudioBackendBundle\Mercure\Service\PublishServiceInterface; -use Pimcore\Bundle\StudioBackendBundle\Util\Constants\ElementPermissions; -use Pimcore\Bundle\StudioBackendBundle\Util\Constants\ElementTypes; use Pimcore\Bundle\StudioBackendBundle\Util\Traits\HandlerProgressTrait; -use Pimcore\Model\Asset; use Symfony\Component\Messenger\Attribute\AsMessageHandler; -use function array_key_exists; -use function in_array; /** * @internal @@ -43,10 +36,10 @@ final class CsvCreationHandler extends AbstractHandler { use HandlerProgressTrait; + private const ARRAY_TYPE = 'array'; + public function __construct( private readonly PublishServiceInterface $publishService, - private readonly ElementServiceInterface $elementService, - private readonly UserResolverInterface $userResolver, private readonly CsvServiceInterface $csvService, private readonly GridServiceInterface $gridService ) { @@ -59,87 +52,46 @@ public function __construct( public function __invoke(CsvCreationMessage $message): void { $jobRun = $this->getJobRun($message); - - $validatedParameters = $this->validateJobParameters( - $message, - $jobRun, - $this->userResolver + $settings = $this->extractConfigFieldFromJobStepConfig($message, Csv::JOB_STEP_CONFIG_SETTINGS->value); + $columnCollection = $this->gridService->getConfigurationFromArray( + $this->extractConfigFieldFromJobStepConfig($message, Csv::JOB_STEP_CONFIG_CONFIGURATION->value) ); - - if ($validatedParameters instanceof AbortActionData) { - $this->abort($validatedParameters); - } - - $context = $jobRun->getContext(); - - if (!array_key_exists(ZipServiceInterface::ASSETS_INDEX, $context)) { - $this->abort( - $this->getAbortData( - Config::NO_ASSETS_FOUND_FOR_JOB_RUN->value, - [ - 'jobRunId' => $jobRun->getId(), - ] - ) - ); - } - - $jobAsset = $validatedParameters->getSubject(); - - if (!in_array($jobAsset->getId(), $context[ZipServiceInterface::ASSETS_INDEX], true)) { + if (!isset($jobRun->getContext()[Csv::ASSET_EXPORT_DATA->value])) { $this->abort($this->getAbortData( - Config::ELEMENT_PERMISSION_MISSING_MESSAGE->value, - [ - 'userId' => $jobRun->getOwnerId(), - 'permission' => ElementPermissions::VIEW_PERMISSION, - 'type' => ucfirst($jobAsset->getType()), - 'id' => $jobAsset->getId(), - ], + Config::CSV_CREATION_FAILED_MESSAGE->value, + ['message' => 'Asset export data not found in job run context'] )); } - - $settings = $this->extractConfigFieldFromJobStepConfig($message, 'settings'); - $columnCollection = $this->gridService->getConfigurationFromArray( - $this->extractConfigFieldFromJobStepConfig($message, 'configuration') - ); - - $csv = $this->csvService->getCsvFile($jobRun->getId(), $columnCollection, $settings); - - if (!$csv) { + $assetData = $jobRun->getContext()[Csv::ASSET_EXPORT_DATA->value]; + + try { + $this->csvService->createCsvFile( + $jobRun->getId(), + $columnCollection, + $settings, + $assetData, + ); + } catch (Exception|FilesystemException $e) { $this->abort($this->getAbortData( - Config::FILE_NOT_FOUND_FOR_JOB_RUN->value, - [ - 'type' => 'csv', - 'jobRunId' => $jobRun->getId(), - ] + Config::CSV_CREATION_FAILED_MESSAGE->value, + ['message' => $e->getMessage()] )); } - $asset = $this->getElementById( - $jobAsset, - $validatedParameters->getUser(), - $this->elementService - ); - - if (!$asset instanceof Asset) { - return; - } - - $assetData = $this->gridService->getGridValuesForElement( - $columnCollection, - $asset, - ElementTypes::TYPE_ASSET - ); - - $this->csvService->addData($csv, $settings['delimiter'], $assetData); - $this->updateProgress($this->publishService, $jobRun, $this->getJobStep($message)->getName()); } protected function configureStep(): void { - $this->stepConfiguration->setRequired('settings'); - $this->stepConfiguration->setAllowedTypes('settings', 'array'); - $this->stepConfiguration->setRequired('configuration'); - $this->stepConfiguration->setAllowedTypes('configuration', 'array'); + $this->stepConfiguration->setRequired(Csv::JOB_STEP_CONFIG_SETTINGS->value); + $this->stepConfiguration->setAllowedTypes( + Csv::JOB_STEP_CONFIG_SETTINGS->value, + self::ARRAY_TYPE + ); + $this->stepConfiguration->setRequired(Csv::JOB_STEP_CONFIG_CONFIGURATION->value); + $this->stepConfiguration->setAllowedTypes( + Csv::JOB_STEP_CONFIG_CONFIGURATION->value, + self::ARRAY_TYPE + ); } } diff --git a/src/Asset/ExecutionEngine/AutomationAction/Messenger/Handler/CsvDataCollectionHandler.php b/src/Asset/ExecutionEngine/AutomationAction/Messenger/Handler/CsvDataCollectionHandler.php new file mode 100644 index 000000000..1cc661ebb --- /dev/null +++ b/src/Asset/ExecutionEngine/AutomationAction/Messenger/Handler/CsvDataCollectionHandler.php @@ -0,0 +1,132 @@ +getJobRun($message); + $user = $this->userResolver->getById($jobRun->getOwnerId()); + if ($user === null) { + $this->abort($this->getAbortData( + Config::USER_NOT_FOUND_MESSAGE->value, + [ + 'userId' => $jobRun->getOwnerId(), + ] + )); + } + + $jobAsset = $this->extractConfigFieldFromJobStepConfig($message, Csv::ASSET_TO_EXPORT->value); + $asset = $this->getElementById( + new ElementDescriptor($jobAsset['type'], $jobAsset['id']), + $user, + $this->elementService + ); + if (!$asset instanceof Asset || $asset->getType() === ElementTypes::TYPE_FOLDER) { + $this->abort($this->getAbortData( + Config::ELEMENT_FOLDER_COLLECTION_NOT_SUPPORTED->value, + [ + 'folderId' => $asset->getId(), + ] + )); + + return; + } + $columnCollection = $this->gridService->getConfigurationFromArray( + $this->extractConfigFieldFromJobStepConfig($message, Csv::JOB_STEP_CONFIG_CONFIGURATION->value) + ); + + try { + $assetData = [ + $asset->getId() => $this->gridService->getGridValuesForElement( + $columnCollection, + $asset, + ElementTypes::TYPE_ASSET + ), + ]; + + if (isset($jobRun->getContext()[Csv::ASSET_EXPORT_DATA->value])) { + $assetData = array_merge( + $jobRun->getContext()[Csv::ASSET_EXPORT_DATA->value], + $assetData + ); + } + + $this->updateJobRunContext($jobRun, Csv::ASSET_EXPORT_DATA->value, $assetData); + } catch (Exception $e) { + $this->abort($this->getAbortData( + Config::CSV_DATA_COLLECTION_FAILED_MESSAGE->value, + [ + 'id' => $asset->getId(), + 'message' => $e->getMessage(), + ] + )); + } + + $this->updateProgress($this->publishService, $jobRun, $this->getJobStep($message)->getName()); + } + + protected function configureStep(): void + { + $this->stepConfiguration->setRequired(Csv::ASSET_TO_EXPORT->value); + $this->stepConfiguration->setAllowedTypes( + Csv::ASSET_TO_EXPORT->value, + self::ARRAY_TYPE + ); + $this->stepConfiguration->setRequired(Csv::JOB_STEP_CONFIG_CONFIGURATION->value); + $this->stepConfiguration->setAllowedTypes( + Csv::JOB_STEP_CONFIG_CONFIGURATION->value, + self::ARRAY_TYPE + ); + } +} diff --git a/src/Asset/ExecutionEngine/AutomationAction/Messenger/Handler/ZipCreationHandler.php b/src/Asset/ExecutionEngine/AutomationAction/Messenger/Handler/ZipDownloadHandler.php similarity index 51% rename from src/Asset/ExecutionEngine/AutomationAction/Messenger/Handler/ZipCreationHandler.php rename to src/Asset/ExecutionEngine/AutomationAction/Messenger/Handler/ZipDownloadHandler.php index 6182bf1ba..838b7f974 100644 --- a/src/Asset/ExecutionEngine/AutomationAction/Messenger/Handler/ZipCreationHandler.php +++ b/src/Asset/ExecutionEngine/AutomationAction/Messenger/Handler/ZipDownloadHandler.php @@ -18,25 +18,23 @@ use Exception; use Pimcore\Bundle\StaticResolverBundle\Models\User\UserResolverInterface; -use Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Messages\ZipCreationMessage; +use Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Messages\ZipDownloadMessage; use Pimcore\Bundle\StudioBackendBundle\Asset\Service\ExecutionEngine\ZipServiceInterface; use Pimcore\Bundle\StudioBackendBundle\Element\Service\ElementServiceInterface; use Pimcore\Bundle\StudioBackendBundle\ExecutionEngine\AutomationAction\AbstractHandler; -use Pimcore\Bundle\StudioBackendBundle\ExecutionEngine\Model\AbortActionData; use Pimcore\Bundle\StudioBackendBundle\ExecutionEngine\Util\Config; use Pimcore\Bundle\StudioBackendBundle\Mercure\Service\PublishServiceInterface; -use Pimcore\Bundle\StudioBackendBundle\Util\Constants\ElementPermissions; +use Pimcore\Bundle\StudioBackendBundle\Util\Constants\ElementTypes; use Pimcore\Bundle\StudioBackendBundle\Util\Traits\HandlerProgressTrait; use Pimcore\Model\Asset; +use Pimcore\Model\Element\ElementDescriptor; use Symfony\Component\Messenger\Attribute\AsMessageHandler; -use function array_key_exists; -use function in_array; /** * @internal */ #[AsMessageHandler] -final class ZipCreationHandler extends AbstractHandler +final class ZipDownloadHandler extends AbstractHandler { use HandlerProgressTrait; @@ -52,72 +50,74 @@ public function __construct( /** * @throws Exception */ - public function __invoke(ZipCreationMessage $message): void + public function __invoke(ZipDownloadMessage $message): void { $jobRun = $this->getJobRun($message); - - $validatedParameters = $this->validateJobParameters( - $message, - $jobRun, - $this->userResolver - ); - - if ($validatedParameters instanceof AbortActionData) { - $this->abort($validatedParameters); - } - - $context = $jobRun->getContext(); - - if (!array_key_exists(ZipServiceInterface::ASSETS_INDEX, $context)) { - $this->abort( - $this->getAbortData( - Config::NO_ASSETS_FOUND_FOR_JOB_RUN->value, - [ - 'jobRunId' => $jobRun->getId(), - ] - ) - ); - } - - $jobAsset = $validatedParameters->getSubject(); - - if (!in_array($jobAsset->getId(), $context[ZipServiceInterface::ASSETS_INDEX], true)) { + $jobRunId = $jobRun->getId(); + $user = $this->userResolver->getById($jobRun->getOwnerId()); + if ($user === null) { $this->abort($this->getAbortData( - Config::ELEMENT_PERMISSION_MISSING_MESSAGE->value, + Config::USER_NOT_FOUND_MESSAGE->value, [ 'userId' => $jobRun->getOwnerId(), - 'permission' => ElementPermissions::VIEW_PERMISSION, - 'type' => ucfirst($jobAsset->getType()), - 'id' => $jobAsset->getId(), - ], + ] )); } - $archive = $this->zipService->getZipArchive($jobRun->getId()); - if (!$archive) { + $assetIds = $this->extractConfigFieldFromJobStepConfig($message, ZipServiceInterface::ASSETS_TO_ZIP); + if (empty($assetIds)) { $this->abort($this->getAbortData( - Config::FILE_NOT_FOUND_FOR_JOB_RUN->value, + Config::NO_ASSETS_FOUND_FOR_JOB_RUN->value, [ - 'type' => 'zip', - 'jobRunId' => $jobRun->getId(), + 'jobRunId' => $jobRunId, ] )); } - - $asset = $this->getElementById( - $jobAsset, - $validatedParameters->getUser(), - $this->elementService + $archiveLocalPath = $this->zipService->getTempFilePath($jobRunId, ZipServiceInterface::DOWNLOAD_ZIP_FILE_PATH); + $archive = $this->zipService->createLocalArchive( + $archiveLocalPath, + true ); - if (!$asset instanceof Asset) { - return; - } + foreach ($assetIds as $assetId) { + $asset = $this->getElementById( + new ElementDescriptor( + ElementTypes::TYPE_ASSET, + $assetId + ), + $user, + $this->elementService + ); + + if (!$asset instanceof Asset || $asset->getType() === ElementTypes::TYPE_FOLDER) { + $this->abort($this->getAbortData( + Config::ELEMENT_FOLDER_COLLECTION_NOT_SUPPORTED->value, + [ + 'folderId' => $asset->getId(), + ] + )); - $this->zipService->addFile($archive, $asset); + return; + } + + $this->zipService->addFile($archive, $asset); + } $archive->close(); + $this->zipService->copyZipFileToFlysystem( + (string)$jobRunId, + ZipServiceInterface::DOWNLOAD_ZIP_FOLDER_NAME, + ZipServiceInterface::DOWNLOAD_ZIP_FILE_NAME, + $archiveLocalPath, + ); + //ToDo: Update progress within step $this->updateProgress($this->publishService, $jobRun, $this->getJobStep($message)->getName()); } + + protected function configureStep(): void + { + $this->stepConfiguration->setRequired(ZipServiceInterface::ASSETS_TO_ZIP); + $this->stepConfiguration->setAllowedTypes(ZipServiceInterface::ASSETS_TO_ZIP, 'array'); + } } diff --git a/src/Asset/ExecutionEngine/AutomationAction/Messenger/Handler/ZipUploadHandler.php b/src/Asset/ExecutionEngine/AutomationAction/Messenger/Handler/ZipUploadHandler.php index cdb521987..e6671d541 100644 --- a/src/Asset/ExecutionEngine/AutomationAction/Messenger/Handler/ZipUploadHandler.php +++ b/src/Asset/ExecutionEngine/AutomationAction/Messenger/Handler/ZipUploadHandler.php @@ -22,6 +22,7 @@ use Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\Util\EnvironmentVariables; use Pimcore\Bundle\StudioBackendBundle\Asset\Service\ExecutionEngine\ZipServiceInterface; use Pimcore\Bundle\StudioBackendBundle\Asset\Service\UploadServiceInterface; +use Pimcore\Bundle\StudioBackendBundle\Element\Service\StorageServiceInterface; use Pimcore\Bundle\StudioBackendBundle\ExecutionEngine\AutomationAction\AbstractHandler; use Pimcore\Bundle\StudioBackendBundle\ExecutionEngine\Model\AbortActionData; use Pimcore\Bundle\StudioBackendBundle\ExecutionEngine\Util\Config; @@ -29,6 +30,7 @@ use Pimcore\Bundle\StudioBackendBundle\Mercure\Service\PublishServiceInterface; use Pimcore\Bundle\StudioBackendBundle\Util\Constants\ElementTypes; use Pimcore\Bundle\StudioBackendBundle\Util\Traits\HandlerProgressTrait; +use Symfony\Component\Filesystem\Filesystem; use Symfony\Component\Messenger\Attribute\AsMessageHandler; /** @@ -39,10 +41,14 @@ final class ZipUploadHandler extends AbstractHandler { use HandlerProgressTrait; + private const LOCAL_ZIP_FOLDER_NAME = 'studio-backend-local'; + public function __construct( + private readonly FileSystem $fileSystem, private readonly PublishServiceInterface $publishService, - private readonly UploadServiceInterface $uploadService, + private readonly StorageServiceInterface $storageService, private readonly UserResolverInterface $userResolver, + private readonly UploadServiceInterface $uploadService, private readonly ZipServiceInterface $zipService, ) { parent::__construct(); @@ -67,36 +73,31 @@ public function __invoke(ZipUploadMessage $message): void $this->abort($validatedParameters); } - $user = $validatedParameters->getUser(); $archiveId = $validatedParameters->getSubject()->getType(); - $archiveExtractPath = $this->zipService->getTempFilePath( + $extractTargetPath = $this->zipService->getTempFilePath( $archiveId, - ZipServiceInterface::UPLOAD_ZIP_FOLDER_PATH + ZipServiceInterface::UPLOAD_ZIP_FOLDER_NAME ); + $localExtractTargetPath = PIMCORE_SYSTEM_TEMP_DIRECTORY . '/' . + $extractTargetPath . '/' . + self::LOCAL_ZIP_FOLDER_NAME; try { - $archive = $this->zipService->getZipArchive( + $this->fileSystem->mkdir($localExtractTargetPath); + + $archive = $this->zipService->downloadZipFileFromFlysystem( $archiveId, + ZipServiceInterface::UPLOAD_ZIP_FOLDER_NAME, ZipServiceInterface::UPLOAD_ZIP_FILE_NAME, - false + $localExtractTargetPath ); - if (!$archive) { - $this->abort($this->getAbortData( - Config::FILE_NOT_FOUND_FOR_JOB_RUN->value, - [ - 'type' => ElementTypes::TYPE_ARCHIVE, - 'id' => $archiveId, - ], - )); - } - - $files = $this->zipService->getArchiveFiles( + $elements = $this->zipService->extractArchiveFiles( $archive, - $archiveExtractPath + $localExtractTargetPath ); - if (empty($files)) { + if (empty($elements)) { $this->abort($this->getAbortData( Config::FILE_NOT_FOUND_FOR_JOB_RUN->value, [ @@ -106,11 +107,24 @@ public function __invoke(ZipUploadMessage $message): void )); } + $files = []; + foreach ($elements as $element) { + $this->storageService->copyElementToFlysystem( + $element['path'], + $element['sourcePath'], + $extractTargetPath + ); + + if ($element['type'] === ElementTypes::TYPE_ASSET) { + $element['sourcePath'] = $extractTargetPath . '/' . $element['path']; + $files[] = $element; + } + } $childJobRunId = $this->uploadService->uploadAssetsAsynchronously( - $user, + $validatedParameters->getUser(), $files, $validatedParameters->getEnvironmentData()[EnvironmentVariables::PARENT_ID->value], - $this->zipService->getTempFileName( + $this->zipService->getTempFilePath( $archiveId, ZipServiceInterface::UPLOAD_ZIP_FOLDER_NAME ) @@ -124,7 +138,7 @@ public function __invoke(ZipUploadMessage $message): void ['message' => $exception->getMessage()], )); } finally { - unlink($this->zipService->getTempFilePath($archiveId, ZipServiceInterface::UPLOAD_ZIP_FILE_PATH)); + $this->storageService->cleanUpLocalFolder($localExtractTargetPath); } $this->updateProgress($this->publishService, $jobRun, $this->getJobStep($message)->getName()); diff --git a/src/Asset/ExecutionEngine/AutomationAction/Messenger/Messages/CollectionMessage.php b/src/Asset/ExecutionEngine/AutomationAction/Messenger/Messages/CsvCollectionMessage.php similarity index 90% rename from src/Asset/ExecutionEngine/AutomationAction/Messenger/Messages/CollectionMessage.php rename to src/Asset/ExecutionEngine/AutomationAction/Messenger/Messages/CsvCollectionMessage.php index 008d903a6..e0e75be9c 100644 --- a/src/Asset/ExecutionEngine/AutomationAction/Messenger/Messages/CollectionMessage.php +++ b/src/Asset/ExecutionEngine/AutomationAction/Messenger/Messages/CsvCollectionMessage.php @@ -21,6 +21,6 @@ /** * @internal */ -final class CollectionMessage extends AbstractExecutionEngineMessage +final class CsvCollectionMessage extends AbstractExecutionEngineMessage { } diff --git a/src/Asset/ExecutionEngine/AutomationAction/Messenger/Messages/ZipCreationMessage.php b/src/Asset/ExecutionEngine/AutomationAction/Messenger/Messages/ZipDownloadMessage.php similarity index 91% rename from src/Asset/ExecutionEngine/AutomationAction/Messenger/Messages/ZipCreationMessage.php rename to src/Asset/ExecutionEngine/AutomationAction/Messenger/Messages/ZipDownloadMessage.php index 4aae6e2ef..dcc51104b 100644 --- a/src/Asset/ExecutionEngine/AutomationAction/Messenger/Messages/ZipCreationMessage.php +++ b/src/Asset/ExecutionEngine/AutomationAction/Messenger/Messages/ZipDownloadMessage.php @@ -21,6 +21,6 @@ /** * @internal */ -final class ZipCreationMessage extends AbstractExecutionEngineMessage +final class ZipDownloadMessage extends AbstractExecutionEngineMessage { } diff --git a/src/Asset/ExecutionEngine/Util/EnvironmentVariables.php b/src/Asset/ExecutionEngine/Util/EnvironmentVariables.php index 51b5b3b72..f92df1c70 100644 --- a/src/Asset/ExecutionEngine/Util/EnvironmentVariables.php +++ b/src/Asset/ExecutionEngine/Util/EnvironmentVariables.php @@ -23,5 +23,5 @@ enum EnvironmentVariables: string { case ORIGINAL_PARENT_ID = 'originalParentId'; case PARENT_ID = 'parentId'; - case UPLOAD_FOLDER_NAME = 'uploadFolderName'; + case UPLOAD_FOLDER_LOCATION = 'uploadFolderLocation'; } diff --git a/src/Asset/ExecutionEngine/Util/JobSteps.php b/src/Asset/ExecutionEngine/Util/JobSteps.php index 103b9a12f..2af535e80 100644 --- a/src/Asset/ExecutionEngine/Util/JobSteps.php +++ b/src/Asset/ExecutionEngine/Util/JobSteps.php @@ -17,8 +17,8 @@ enum JobSteps: string { - case ZIP_COLLECTION = 'studio_ee_job_step_zip_collection'; case ZIP_CREATION = 'studio_ee_job_step_zip_creation'; + case ZIP_COPY = 'studio_ee_job_step_zip_copy'; case ZIP_UPLOADING = 'studio_ee_jop_step_zip_uploading'; case ASSET_DELETION = 'studio_ee_job_step_asset_deletion'; case ASSET_CLONING = 'studio_ee_job_step_asset_cloning'; diff --git a/src/Asset/MappedParameter/CreateAssetFileParameter.php b/src/Asset/MappedParameter/CreateAssetFileParameter.php index 303b03ae9..349bc2602 100644 --- a/src/Asset/MappedParameter/CreateAssetFileParameter.php +++ b/src/Asset/MappedParameter/CreateAssetFileParameter.php @@ -16,9 +16,6 @@ namespace Pimcore\Bundle\StudioBackendBundle\Asset\MappedParameter; -use Pimcore\Bundle\StudioBackendBundle\Util\Constants\ElementTypes; -use Pimcore\Model\Element\ElementDescriptor; - /** * @internal */ @@ -30,13 +27,8 @@ public function __construct( ) { } - /** @return array */ public function getItems(): array { - return array_map( - static fn (int $id) => - new ElementDescriptor(ElementTypes::TYPE_ASSET, $id), - $this->items - ); + return $this->items; } } diff --git a/src/Asset/MappedParameter/ImageDownloadConfigParameter.php b/src/Asset/MappedParameter/ImageDownloadConfigParameter.php index a2d617bcc..df6a9ae59 100644 --- a/src/Asset/MappedParameter/ImageDownloadConfigParameter.php +++ b/src/Asset/MappedParameter/ImageDownloadConfigParameter.php @@ -34,7 +34,7 @@ public function __construct( private ?int $quality = null, private ?int $dpi = null ) { - if (!in_array($this->mimeType, MimeTypes::ALLOWED_FORMATS)) { + if (!in_array($this->mimeType, [MimeTypes::JPEG->value, MimeTypes::PNG->value], true)) { throw new InvalidArgumentException('Invalid mime type' . $this->mimeType); } diff --git a/src/Asset/Mercure/Schema/DownloadReady.php b/src/Asset/Mercure/Schema/DownloadReady.php deleted file mode 100644 index 00a19d787..000000000 --- a/src/Asset/Mercure/Schema/DownloadReady.php +++ /dev/null @@ -1,60 +0,0 @@ -jobRunId; - } - - public function getPath(): string - { - return $this->path; - } - - public function getUser(): int - { - return $this->user; - } -} diff --git a/src/Asset/OpenApi/Attributes/Parameters/Query/MimeTypeParameter.php b/src/Asset/OpenApi/Attributes/Parameters/Query/MimeTypeParameter.php index 189626fda..1605ce342 100644 --- a/src/Asset/OpenApi/Attributes/Parameters/Query/MimeTypeParameter.php +++ b/src/Asset/OpenApi/Attributes/Parameters/Query/MimeTypeParameter.php @@ -34,10 +34,10 @@ public function __construct() schema: new Schema( type: 'string', enum: [ - MimeTypes::JPEG, - MimeTypes::PNG, + MimeTypes::JPEG->value, + MimeTypes::PNG->value, ], - example: MimeTypes::JPEG + example: MimeTypes::JPEG->value ), ); } diff --git a/src/Asset/Service/BinaryService.php b/src/Asset/Service/BinaryService.php index 65cd60a41..0144267ad 100644 --- a/src/Asset/Service/BinaryService.php +++ b/src/Asset/Service/BinaryService.php @@ -18,6 +18,7 @@ use League\Flysystem\FilesystemException; use Pimcore\Bundle\StudioBackendBundle\Asset\MappedParameter\VideoImageStreamConfigParameter; +use Pimcore\Bundle\StudioBackendBundle\Element\Service\StorageServiceInterface; use Pimcore\Bundle\StudioBackendBundle\Exception\Api\ElementProcessingNotCompletedException; use Pimcore\Bundle\StudioBackendBundle\Exception\Api\ElementStreamResourceNotFoundException; use Pimcore\Bundle\StudioBackendBundle\Exception\Api\InvalidElementTypeException; @@ -28,7 +29,6 @@ use Pimcore\Messenger\AssetPreviewImageMessage; use Pimcore\Model\Asset; use Pimcore\Model\Asset\Video; -use Pimcore\Tool\Storage; use Symfony\Component\EventDispatcher\EventDispatcherInterface; use Symfony\Component\HttpFoundation\StreamedResponse; @@ -42,7 +42,7 @@ public function __construct( private EventDispatcherInterface $eventDispatcher, private ThumbnailServiceInterface $thumbnailService, - private Storage $storageTool + private StorageServiceInterface $storageService ) { } @@ -152,7 +152,7 @@ private function getVideoByThumbnail( urldecode($thumbnail['formats']['mp4']) ); - $storage = $this->storageTool->getStorage('thumbnail'); + $storage = $this->storageService->getThumbnailStorage(); if (!$storage->fileExists($storagePath)) { throw new InvalidThumbnailException($thumbnailName); } diff --git a/src/Asset/Service/DocumentService.php b/src/Asset/Service/DocumentService.php index a4028bd7e..cbce8a5c8 100644 --- a/src/Asset/Service/DocumentService.php +++ b/src/Asset/Service/DocumentService.php @@ -56,7 +56,7 @@ public function __construct( */ public function getPreviewStream(Document $asset): StreamedResponse { - if ($asset->getMimeType() !== MimeTypes::PDF) { + if ($asset->getMimeType() !== MimeTypes::PDF->value) { return $this->getStreamFromDocument($asset); } @@ -160,7 +160,7 @@ function () use ($stream) { }, HttpResponseCodes::SUCCESS->value, [ - HttpResponseHeaders::HEADER_CONTENT_TYPE->value => MimeTypes::PDF, + HttpResponseHeaders::HEADER_CONTENT_TYPE->value => MimeTypes::PDF->value, HttpResponseHeaders::HEADER_CONTENT_DISPOSITION->value => sprintf( '%s; filename="%s"', HttpResponseHeaders::INLINE_TYPE->value, diff --git a/src/Asset/Service/DownloadService.php b/src/Asset/Service/DownloadService.php index 3b60282e2..9c04f3962 100644 --- a/src/Asset/Service/DownloadService.php +++ b/src/Asset/Service/DownloadService.php @@ -17,19 +17,30 @@ namespace Pimcore\Bundle\StudioBackendBundle\Asset\Service; use Exception; -use Pimcore\Bundle\StudioBackendBundle\Asset\MappedParameter\DownloadPathParameter; +use League\Flysystem\FilesystemException; +use League\Flysystem\FilesystemOperator; +use Pimcore\Bundle\GenericExecutionEngineBundle\Entity\JobRun; +use Pimcore\Bundle\GenericExecutionEngineBundle\Repository\JobRunRepositoryInterface; use Pimcore\Bundle\StudioBackendBundle\Asset\MappedParameter\ImageDownloadConfigParameter; +use Pimcore\Bundle\StudioBackendBundle\Element\Service\StorageServiceInterface; use Pimcore\Bundle\StudioBackendBundle\Exception\Api\ElementStreamResourceNotFoundException; +use Pimcore\Bundle\StudioBackendBundle\Exception\Api\EnvironmentException; +use Pimcore\Bundle\StudioBackendBundle\Exception\Api\ForbiddenException; use Pimcore\Bundle\StudioBackendBundle\Exception\Api\InvalidAssetFormatTypeException; use Pimcore\Bundle\StudioBackendBundle\Exception\Api\InvalidElementTypeException; use Pimcore\Bundle\StudioBackendBundle\Exception\Api\InvalidThumbnailException; +use Pimcore\Bundle\StudioBackendBundle\Exception\Api\NotFoundException; +use Pimcore\Bundle\StudioBackendBundle\Exception\Api\StreamResourceNotFoundException; use Pimcore\Bundle\StudioBackendBundle\Exception\Api\ThumbnailResizingFailedException; +use Pimcore\Bundle\StudioBackendBundle\Security\Service\SecurityServiceInterface; use Pimcore\Bundle\StudioBackendBundle\Util\Constants\Asset\FormatTypes; use Pimcore\Bundle\StudioBackendBundle\Util\Constants\HttpResponseHeaders; use Pimcore\Bundle\StudioBackendBundle\Util\Traits\StreamedResponseTrait; +use Pimcore\Bundle\StudioBackendBundle\Util\Traits\TempFilePathTrait; use Pimcore\Model\Asset; use Pimcore\Model\Asset\Image; use Pimcore\Model\Element\ElementInterface; +use Pimcore\Model\Exception\NotFoundException as CoreNotFoundException; use Symfony\Component\HttpFoundation\BinaryFileResponse; use Symfony\Component\HttpFoundation\StreamedResponse; use function in_array; @@ -40,9 +51,13 @@ final readonly class DownloadService implements DownloadServiceInterface { use StreamedResponseTrait; + use TempFilePathTrait; public function __construct( + private StorageServiceInterface $storageService, private ThumbnailServiceInterface $thumbnailService, + private JobRunRepositoryInterface $jobRunRepository, + private SecurityServiceInterface $securityService, private array $defaultFormats, ) { } @@ -134,13 +149,65 @@ public function downloadImageByThumbnail( ); } - public function downloadZipArchiveByPath(DownloadPathParameter $path): StreamedResponse + /** + * @throws NotFoundException|ForbiddenException|StreamResourceNotFoundException + */ + public function downloadResourceByJobRunId( + int $jobRunId, + string $tempFileName, + string $tempFolderName, + string $mimeType, + string $downloadName, + ): StreamedResponse { + $jobRun = $this->validateJobRun($jobRunId); + + $fileName = $this->getTempFileName($jobRun->getId(), $tempFileName); + $folderName = $this->getTempFileName($jobRun->getId(), $tempFolderName); + $filePath = $folderName . '/' . $fileName; + + $streamedResponse = $this->getFileStreamedResponse( + $filePath, + $mimeType, + $downloadName, + $this->validateStorage($filePath) + ); + + try { + $this->storageService->cleanUpFolder($folderName); + } catch (FilesystemException) { + throw new EnvironmentException( + sprintf( + 'Failed to clean up temporary folder %s', + $folderName + ) + ); + } + + return $streamedResponse; + } + + private function validateJobRun(int $jobRunId): JobRun { - return $this->getFileStreamedResponse($path->getPath(), 'application/zip', 'assets.zip'); + try { + $jobRun = $this->jobRunRepository->getJobRunById($jobRunId); + } catch (CoreNotFoundException) { + throw new NotFoundException('JobRun', $jobRunId); + } + + if ($jobRun->getOwnerId() !== $this->securityService->getCurrentUser()->getId()) { + throw new ForbiddenException('Only job owner can access the resource'); + } + + return $jobRun; } - public function downloadCsvByPath(DownloadPathParameter $path): StreamedResponse + private function validateStorage(string $fileName): FilesystemOperator { - return $this->getFileStreamedResponse($path->getPath(), 'application/csv', 'assets.csv'); + $storage = $this->storageService->getTempStorage(); + if (!$this->storageService->tempFileExists($fileName)) { + throw new StreamResourceNotFoundException(sprintf('Resource not found: %s', $fileName)); + } + + return $storage; } } diff --git a/src/Asset/Service/DownloadServiceInterface.php b/src/Asset/Service/DownloadServiceInterface.php index 6054d77f4..d9bb320f4 100644 --- a/src/Asset/Service/DownloadServiceInterface.php +++ b/src/Asset/Service/DownloadServiceInterface.php @@ -16,11 +16,13 @@ namespace Pimcore\Bundle\StudioBackendBundle\Asset\Service; -use Pimcore\Bundle\StudioBackendBundle\Asset\MappedParameter\DownloadPathParameter; use Pimcore\Bundle\StudioBackendBundle\Asset\MappedParameter\ImageDownloadConfigParameter; use Pimcore\Bundle\StudioBackendBundle\Exception\Api\ElementStreamResourceNotFoundException; +use Pimcore\Bundle\StudioBackendBundle\Exception\Api\ForbiddenException; use Pimcore\Bundle\StudioBackendBundle\Exception\Api\InvalidAssetFormatTypeException; use Pimcore\Bundle\StudioBackendBundle\Exception\Api\InvalidElementTypeException; +use Pimcore\Bundle\StudioBackendBundle\Exception\Api\NotFoundException; +use Pimcore\Bundle\StudioBackendBundle\Exception\Api\StreamResourceNotFoundException; use Pimcore\Bundle\StudioBackendBundle\Exception\Api\ThumbnailResizingFailedException; use Pimcore\Model\Asset; use Symfony\Component\HttpFoundation\BinaryFileResponse; @@ -62,7 +64,14 @@ public function downloadImageByThumbnail( string $thumbnailName ): BinaryFileResponse; - public function downloadZipArchiveByPath(DownloadPathParameter $path): StreamedResponse; - - public function downloadCsvByPath(DownloadPathParameter $path): StreamedResponse; + /** + * @throws NotFoundException|ForbiddenException|StreamResourceNotFoundException + */ + public function downloadResourceByJobRunId( + int $jobRunId, + string $tempFileName, + string $tempFolderName, + string $mimeType, + string $downloadName, + ): StreamedResponse; } diff --git a/src/Asset/Service/ExecutionEngine/CsvService.php b/src/Asset/Service/ExecutionEngine/CsvService.php index 8f601c67b..a02abaa74 100644 --- a/src/Asset/Service/ExecutionEngine/CsvService.php +++ b/src/Asset/Service/ExecutionEngine/CsvService.php @@ -17,58 +17,69 @@ namespace Pimcore\Bundle\StudioBackendBundle\Asset\Service\ExecutionEngine; use League\Flysystem\FilesystemException; +use League\Flysystem\FilesystemOperator; use Pimcore\Bundle\GenericExecutionEngineBundle\Agent\JobExecutionAgentInterface; use Pimcore\Bundle\GenericExecutionEngineBundle\Model\Job; use Pimcore\Bundle\GenericExecutionEngineBundle\Model\JobStep; -use Pimcore\Bundle\StaticResolverBundle\Models\Tool\StorageResolverInterface; -use Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Messages\CollectionMessage; +use Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Messages\CsvCollectionMessage; use Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Messages\CsvCreationMessage; use Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\Util\JobSteps; use Pimcore\Bundle\StudioBackendBundle\Asset\MappedParameter\ExportAssetParameter; use Pimcore\Bundle\StudioBackendBundle\Asset\Util\Constants\Csv; -use Pimcore\Bundle\StudioBackendBundle\Exception\Api\EnvironmentException; +use Pimcore\Bundle\StudioBackendBundle\Element\Service\StorageServiceInterface; use Pimcore\Bundle\StudioBackendBundle\ExecutionEngine\Util\Config; use Pimcore\Bundle\StudioBackendBundle\ExecutionEngine\Util\Jobs; use Pimcore\Bundle\StudioBackendBundle\Grid\Service\GridServiceInterface; use Pimcore\Bundle\StudioBackendBundle\Grid\Util\Collection\ColumnCollection; use Pimcore\Bundle\StudioBackendBundle\Security\Service\SecurityServiceInterface; -use Pimcore\Bundle\StudioBackendBundle\Util\Constants\StorageDirectories; use Pimcore\Bundle\StudioBackendBundle\Util\Traits\TempFilePathTrait; +use Pimcore\Model\Element\ElementDescriptor; /** * @internal */ -final class CsvService implements CsvServiceInterface +final readonly class CsvService implements CsvServiceInterface { use TempFilePathTrait; public function __construct( private JobExecutionAgentInterface $jobExecutionAgent, private SecurityServiceInterface $securityService, - private StorageResolverInterface $storageResolver, + private StorageServiceInterface $storageService, private GridServiceInterface $gridService, + private string $defaultDelimiter, ) { } - public function generateCsvFile(ExportAssetParameter $exportAssetParameter): string + public function generateCsvFile(ExportAssetParameter $exportAssetParameter): int { - $steps = [ - new JobStep(JobSteps::CSV_COLLECTION->value, CollectionMessage::class, '', []), - new JobStep( - JobSteps::CSV_CREATION->value, - CsvCreationMessage::class, + $jobStepConfigConfiguration = [ + Csv::JOB_STEP_CONFIG_CONFIGURATION->value => $exportAssetParameter->getGridConfig(), + ]; + $jobStepConfigSettings = [ + Csv::JOB_STEP_CONFIG_SETTINGS->value => $exportAssetParameter->getSettings(), + ]; + + $jobSteps = array_map( + static fn (ElementDescriptor $asset) => new JobStep( + JobSteps::CSV_COLLECTION->value, + CsvCollectionMessage::class, '', - [ - Csv::JOB_STEP_CONFIG_SETTINGS->value => $exportAssetParameter->getSettings(), - Csv::JOB_STEP_CONFIG_CONFIGURATION->value => $exportAssetParameter->getGridConfig(), - ] + array_merge([csv::ASSET_TO_EXPORT->value => $asset], $jobStepConfigConfiguration) ), - ]; + $exportAssetParameter->getAssets(), + ); + + $jobSteps[] = new JobStep( + JobSteps::CSV_CREATION->value, + CsvCreationMessage::class, + '', + array_merge($jobStepConfigSettings, $jobStepConfigConfiguration) + ); $job = new Job( name: Jobs::CREATE_CSV->value, - steps: $steps, - selectedElements: $exportAssetParameter->getAssets(), + steps: $jobSteps ); $jobRun = $this->jobExecutionAgent->startJobExecution( @@ -77,44 +88,45 @@ public function generateCsvFile(ExportAssetParameter $exportAssetParameter): str Config::CONTEXT_STOP_ON_ERROR->value ); - return $this->getTempFilePath($jobRun->getId(), self::CSV_FILE_PATH); + return $jobRun->getId(); } - public function getCsvFile(int $id, ColumnCollection $columnCollection, array $settings): string - { - $storage = $this->storageResolver->get(StorageDirectories::TEMP->value); - $file = $this->getTempFileName($id, self::CSV_FILE_NAME); - - try { - if (!$storage->fileExists($file)) { - $headers = $this->getHeaders($columnCollection, $settings); - $storage->write( - $file, - implode($settings[Csv::SETTINGS_DELIMITER->value] ?? ',', $headers). Csv::NEW_LINE->value - ); - } - - } catch (FilesystemException $e) { - throw new EnvironmentException('Could not create or read CSV file: ' . $e->getMessage()); + /** + * @throws FilesystemException + */ + public function createCsvFile( + int $id, + ColumnCollection $columnCollection, + array $settings, + array $assetData, + ?string $delimiter = null, + ): void { + $storage = $this->storageService->getTempStorage(); + $headers = $this->getHeaders($columnCollection, $settings); + if ($delimiter === null) { + $delimiter = $this->defaultDelimiter; + } + $data[] = implode($delimiter, $headers) . Csv::NEW_LINE->value; + foreach ($assetData as $row) { + $data[] = implode($delimiter, array_map([$this, 'encodeFunc'], $row)) . Csv::NEW_LINE->value; } - return $file; + $storage->write( + $this->getCsvFilePath($id, $storage), + implode($data) + ); } - public function addData(string $filePath, string $delimiter, array $data): void + /** + * @throws FilesystemException + */ + private function getCsvFilePath(int $id, FilesystemOperator $storage): string { - $storage = $this->storageResolver->get(StorageDirectories::TEMP->value); - $fileStream = $storage->readStream($filePath); - - $temp = tmpfile(); - stream_copy_to_stream($fileStream, $temp, null, 0); - - fwrite( - $temp, - implode($delimiter, array_map([$this, 'encodeFunc'], $data)) . Csv::NEW_LINE->value - ); + $folderName = $this->getTempFileName($id, self::CSV_FOLDER_NAME); + $file = $this->getTempFileName($id, self::CSV_FILE_NAME); + $storage->createDirectory($folderName); - $storage->writeStream($filePath, $temp); + return $folderName . '/' . $file; } private function encodeFunc(?string $value): string diff --git a/src/Asset/Service/ExecutionEngine/CsvServiceInterface.php b/src/Asset/Service/ExecutionEngine/CsvServiceInterface.php index f7bc3af96..2c697505b 100644 --- a/src/Asset/Service/ExecutionEngine/CsvServiceInterface.php +++ b/src/Asset/Service/ExecutionEngine/CsvServiceInterface.php @@ -16,6 +16,7 @@ namespace Pimcore\Bundle\StudioBackendBundle\Asset\Service\ExecutionEngine; +use League\Flysystem\FilesystemException; use Pimcore\Bundle\StudioBackendBundle\Asset\MappedParameter\ExportAssetParameter; use Pimcore\Bundle\StudioBackendBundle\Grid\Util\Collection\ColumnCollection; @@ -26,13 +27,22 @@ interface CsvServiceInterface { public const CSV_FILE_NAME = 'download-csv-{id}.csv'; - public const CSV_FILE_PATH = PIMCORE_SYSTEM_TEMP_DIRECTORY . '/' . self::CSV_FILE_NAME; + public const CSV_FOLDER_NAME = 'download-csv-{id}'; - public function getCsvFile(int $id, ColumnCollection $columnCollection, array $settings): string; + public function generateCsvFile(ExportAssetParameter $exportAssetParameter): int; - public function addData(string $filePath, string $delimiter, array $data): void; + /** + * @throws FilesystemException + */ + public function createCsvFile( + int $id, + ColumnCollection $columnCollection, + array $settings, + array $assetData, + ?string $delimiter = null, + ): void; - public function generateCsvFile(ExportAssetParameter $exportAssetParameter): string; + public function getTempFileName(int $id, string $path): string; public function getTempFilePath(int $id, string $path): string; } diff --git a/src/Asset/Service/ExecutionEngine/ZipService.php b/src/Asset/Service/ExecutionEngine/ZipService.php index de80fd9c9..d1833469f 100644 --- a/src/Asset/Service/ExecutionEngine/ZipService.php +++ b/src/Asset/Service/ExecutionEngine/ZipService.php @@ -17,17 +17,18 @@ namespace Pimcore\Bundle\StudioBackendBundle\Asset\Service\ExecutionEngine; use League\Flysystem\FilesystemException; +use Pimcore\Bundle\GenericDataIndexBundle\Exception\AssetSearchException; use Pimcore\Bundle\GenericExecutionEngineBundle\Agent\JobExecutionAgentInterface; use Pimcore\Bundle\GenericExecutionEngineBundle\Model\Job; use Pimcore\Bundle\GenericExecutionEngineBundle\Model\JobStep; -use Pimcore\Bundle\StaticResolverBundle\Models\Tool\StorageResolverInterface; -use Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Messages\CollectionMessage; -use Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Messages\ZipCreationMessage; +use Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Messages\ZipDownloadMessage; use Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Messages\ZipUploadMessage; use Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\Util\EnvironmentVariables; use Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\Util\JobSteps; use Pimcore\Bundle\StudioBackendBundle\Asset\MappedParameter\CreateAssetFileParameter; use Pimcore\Bundle\StudioBackendBundle\Asset\Service\UploadServiceInterface; +use Pimcore\Bundle\StudioBackendBundle\DataIndex\AssetSearchServiceInterface; +use Pimcore\Bundle\StudioBackendBundle\Element\Service\StorageServiceInterface; use Pimcore\Bundle\StudioBackendBundle\Exception\Api\AccessDeniedException; use Pimcore\Bundle\StudioBackendBundle\Exception\Api\EnvironmentException; use Pimcore\Bundle\StudioBackendBundle\Exception\Api\ForbiddenException; @@ -35,13 +36,15 @@ use Pimcore\Bundle\StudioBackendBundle\ExecutionEngine\Util\Config; use Pimcore\Bundle\StudioBackendBundle\ExecutionEngine\Util\Jobs; use Pimcore\Bundle\StudioBackendBundle\Security\Service\SecurityServiceInterface; -use Pimcore\Bundle\StudioBackendBundle\Util\Constants\StorageDirectories; +use Pimcore\Bundle\StudioBackendBundle\Util\Constants\Asset\DownloadLimits; +use Pimcore\Bundle\StudioBackendBundle\Util\Constants\ElementTypes; use Pimcore\Bundle\StudioBackendBundle\Util\Traits\TempFilePathTrait; use Pimcore\Model\Asset; use Pimcore\Model\Element\ElementDescriptor; use Pimcore\Model\UserInterface; use Symfony\Component\HttpFoundation\File\UploadedFile; use ZipArchive; +use function count; /** * @internal @@ -51,44 +54,15 @@ use TempFilePathTrait; public function __construct( + private AssetSearchServiceInterface $assetSearchService, private JobExecutionAgentInterface $jobExecutionAgent, private SecurityServiceInterface $securityService, - private StorageResolverInterface $storageResolver, + private StorageServiceInterface $storageService, private UploadServiceInterface $uploadService, + private array $downloadLimits, ) { } - public function getZipArchive( - mixed $id, - string $fileName = self::DOWNLOAD_ZIP_FILE_NAME, - bool $create = true - ): ?ZipArchive { - $zip = $this->getTempFileName($id, $fileName); - $zipStoragePath = $this->getTempFilePathFromName($id, $fileName); - $storage = $this->storageResolver->get(StorageDirectories::TEMP->value); - $archive = new ZipArchive(); - - $state = false; - - try { - if ($storage->fileExists($zip)) { - $state = $archive->open($zipStoragePath); - } - - if (!$state && $create) { - $state = $archive->open($zipStoragePath, ZipArchive::CREATE); - } - } catch (FilesystemException) { - return null; - } - - if ($state !== true) { - return null; - } - - return $archive; - } - public function addFile(ZipArchive $archive, Asset $asset): void { $archive->addFile( @@ -101,7 +75,7 @@ public function addFile(ZipArchive $archive, Asset $asset): void ); } - public function getArchiveFiles( + public function extractArchiveFiles( ZipArchive $archive, string $targetPath ): array { @@ -112,11 +86,14 @@ public function getArchiveFiles( } foreach (range(0, $fileCount - 1) as $i) { - $fileName = $this->uploadService->sanitizeFileToUpload($archive->getNameIndex($i)); - if ($fileName !== null) { + $path = $this->uploadService->sanitizeFileToUpload($archive->getNameIndex($i)); + if ($path !== null) { + $sourcePath = $targetPath . '/' . $path; $files[] = [ - 'fileName' => $fileName, - 'sourcePath' => $targetPath . '/' . $fileName, + 'name' => basename($path), + 'path' => $path, + 'sourcePath' => $sourcePath, + 'type' => is_dir($sourcePath) ? ElementTypes::TYPE_FOLDER : ElementTypes::TYPE_ASSET, ]; } } @@ -134,7 +111,13 @@ public function uploadZipAssets( ): int { $this->uploadService->validateParent($user, $parentId); $archiveId = $parentId . '-' . time(); - $this->copyUploadZipFile($zipArchive->getRealPath(), $archiveId); + $this->copyZipFileToFlysystem( + $archiveId, + self::UPLOAD_ZIP_FOLDER_NAME, + self::UPLOAD_ZIP_FILE_NAME, + $zipArchive->getRealPath(), + ); + $job = new Job( name: Jobs::ZIP_FILE_UPLOAD->value, steps: [ @@ -157,17 +140,23 @@ public function uploadZipAssets( return $jobRun->getId(); } - public function generateZipFile(CreateAssetFileParameter $ids): string + /** + * @throws EnvironmentException + */ + public function generateZipFile(CreateAssetFileParameter $parameter): int { - $steps = [ - new JobStep(JobSteps::ZIP_COLLECTION->value, CollectionMessage::class, '', []), - new JobStep(JobSteps::ZIP_CREATION->value, ZipCreationMessage::class, '', []), - ]; - + $items = $parameter->getItems(); + $this->validateDownloadItems($items); $job = new Job( name: Jobs::CREATE_ZIP->value, - steps: $steps, - selectedElements: $ids->getItems() + steps: [ + new JobStep( + JobSteps::ZIP_CREATION->value, + ZipDownloadMessage::class, + '', + [self::ASSETS_TO_ZIP => $items] + ), + ], ); $jobRun = $this->jobExecutionAgent->startJobExecution( @@ -176,33 +165,131 @@ public function generateZipFile(CreateAssetFileParameter $ids): string Config::CONTEXT_STOP_ON_ERROR->value ); - return $this->getTempFilePath($jobRun->getId(), self::DOWNLOAD_ZIP_FILE_PATH); + return $jobRun->getId(); } /** - * @throws FilesystemException + * @throws EnvironmentException */ - public function cleanUpArchiveFolder( - string $folder - ): void { - $storage = $this->storageResolver->get(StorageDirectories::TEMP->value); - if (empty($storage->listContents($folder)->toArray())) { - $storage->deleteDirectory($folder); + public function createLocalArchive( + string $localPath, + bool $create = false + ): ZipArchive { + $archive = new ZipArchive(); + $flags = $create ? ZipArchive::CREATE : 0; + $state = $archive->open($localPath, $flags); + + if ($state !== true) { + throw new EnvironmentException( + sprintf( + 'Failed to %s zip archive at %s.', + $create ? 'create' : 'open', + $localPath + ) + ); } + + return $archive; } - private function copyUploadZipFile( - string $zipArchivePath, - string $archiveId + /** + * @throws EnvironmentException + */ + public function copyZipFileToFlysystem( + string $id, + string $folderName, + string $archiveName, + string $localPath ): void { - if (!is_file($zipArchivePath)) { + $storage = $this->storageService->getTempStorage(); + $archiveFileName = $this->getTempFileName($id, $archiveName); + if (!is_file($localPath)) { throw new EnvironmentException( - 'Something went wrong, please check upload_max_filesize and post_max_size in your php.ini ' . - ' as well as the write permissions of your temporary directories.' + sprintf( + 'The zip archive %s could not be found at %s.', + $archiveFileName, + $localPath + ) ); } - $pathTarget = $this->getTempFilePath($archiveId, self::UPLOAD_ZIP_FILE_PATH); - copy($zipArchivePath, $pathTarget); + try { + $folderName = $this->getTempFilePath($id, $folderName); + $storage->createDirectory($folderName); + $storage->writeStream( + $folderName . '/' . $archiveFileName, + fopen($localPath, 'rb') + ); + @unlink($localPath); + } catch (FilesystemException) { + throw new EnvironmentException( + sprintf( + 'Failed to copy zip archive %s to Flysystem.', + $archiveFileName + ) + ); + } + } + + /** + * @throws EnvironmentException + */ + public function downloadZipFileFromFlysystem( + string $id, + string $folderName, + string $archiveName, + string $localPath + ): ZipArchive { + $storage = $this->storageService->getTempStorage(); + $archiveFileName = $this->getTempFileName($id, $archiveName); + + try { + $folderName = $this->getTempFileName($id, $folderName); + $stream = $storage->readStream($folderName . '/' . $archiveFileName); + $localArchive = fopen($localPath . '/' . $archiveFileName, 'wb'); + stream_copy_to_stream($stream, $localArchive); + fclose($stream); + fclose($localArchive); + $storage->delete($folderName . '/' . $archiveFileName); + } catch (FilesystemException) { + throw new EnvironmentException( + sprintf( + 'Failed to get zip archive %s from Flysystem.', + $archiveFileName + ) + ); + } + + return $this->createLocalArchive($localPath . '/' . $archiveFileName); + } + + /** + * @throws EnvironmentException + */ + private function validateDownloadItems(array $items): void + { + if (count($items) > $this->downloadLimits[DownloadLimits::MAX_ZIP_FILE_AMOUNT->value]) { + throw new EnvironmentException( + sprintf( + 'Too many assets selected. Maximum amount of assets, which can be processed at once is %s', + $this->downloadLimits[DownloadLimits::MAX_ZIP_FILE_AMOUNT->value] + ) + ); + } + + try { + $totalFileSize = $this->assetSearchService->getTotalFileSizeByIds($items); + } catch (AssetSearchException) { + throw new EnvironmentException('One or more selected assets could not be found.'); + } + + if ($totalFileSize > $this->downloadLimits[DownloadLimits::MAX_ZIP_FILE_SIZE->value]) { + throw new EnvironmentException( + sprintf( + 'The total size of the selected assets exceeds the maximum size of %s bytes.', + $this->downloadLimits[DownloadLimits::MAX_ZIP_FILE_SIZE->value] + ) + ); + } } } diff --git a/src/Asset/Service/ExecutionEngine/ZipServiceInterface.php b/src/Asset/Service/ExecutionEngine/ZipServiceInterface.php index 034e0e064..b274b36cf 100644 --- a/src/Asset/Service/ExecutionEngine/ZipServiceInterface.php +++ b/src/Asset/Service/ExecutionEngine/ZipServiceInterface.php @@ -16,7 +16,6 @@ namespace Pimcore\Bundle\StudioBackendBundle\Asset\Service\ExecutionEngine; -use League\Flysystem\FilesystemException; use Pimcore\Bundle\StudioBackendBundle\Asset\MappedParameter\CreateAssetFileParameter; use Pimcore\Bundle\StudioBackendBundle\Exception\Api\AccessDeniedException; use Pimcore\Bundle\StudioBackendBundle\Exception\Api\EnvironmentException; @@ -32,29 +31,29 @@ */ interface ZipServiceInterface { - public const ASSETS_INDEX = 'assets'; + public const ASSETS_TO_ZIP = 'assets_to_zip'; public const DOWNLOAD_ZIP_FILE_NAME = 'download-zip-{id}.zip'; - public const UPLOAD_ZIP_FILE_NAME = 'upload-zip-{id}.zip'; + public const DOWNLOAD_ZIP_FILE_NAME_LOCAL = 'local-' . self::DOWNLOAD_ZIP_FILE_NAME; - public const UPLOAD_ZIP_FOLDER_NAME = 'upload-zip-{id}'; + public const DOWNLOAD_ZIP_FOLDER_NAME = 'download-zip-{id}'; public const DOWNLOAD_ZIP_FILE_PATH = PIMCORE_SYSTEM_TEMP_DIRECTORY . '/' . self::DOWNLOAD_ZIP_FILE_NAME; - public const UPLOAD_ZIP_FILE_PATH = PIMCORE_SYSTEM_TEMP_DIRECTORY . '/' . self::UPLOAD_ZIP_FILE_NAME; + public const UPLOAD_ZIP_FILE_NAME = 'upload-zip-{id}.zip'; + + public const UPLOAD_ZIP_FILE_NAME_LOCAL = 'local-' . self::UPLOAD_ZIP_FILE_NAME; + + public const UPLOAD_ZIP_FOLDER_NAME = 'upload-zip-{id}'; public const UPLOAD_ZIP_FOLDER_PATH = PIMCORE_SYSTEM_TEMP_DIRECTORY . '/' . self::UPLOAD_ZIP_FOLDER_NAME; - public function getZipArchive( - mixed $id, - string $fileName = self::DOWNLOAD_ZIP_FILE_NAME, - bool $create = true - ): ?ZipArchive; + public const UPLOAD_ZIP_FILE_PATH = self::UPLOAD_ZIP_FOLDER_PATH . '/' . self::UPLOAD_ZIP_FILE_NAME_LOCAL; public function addFile(ZipArchive $archive, Asset $asset): void; - public function getArchiveFiles( + public function extractArchiveFiles( ZipArchive $archive, string $targetPath ): array; @@ -68,15 +67,36 @@ public function uploadZipAssets( int $parentId ): int; - public function generateZipFile(CreateAssetFileParameter $ids): string; + public function generateZipFile(CreateAssetFileParameter $parameter): int; /** - * @throws FilesystemException + * @throws EnvironmentException */ - public function cleanUpArchiveFolder( - string $folder + public function createLocalArchive( + string $localPath, + bool $create = false + ): ZipArchive; + + /** + * @throws EnvironmentException + */ + public function copyZipFileToFlysystem( + string $id, + string $folderName, + string $archiveName, + string $localPath ): void; + /** + * @throws EnvironmentException + */ + public function downloadZipFileFromFlysystem( + string $id, + string $folderName, + string $archiveName, + string $localPath + ): ZipArchive; + public function getTempFilePath(mixed $id, string $path): string; public function getTempFileName(mixed $id, string $fileName): string; diff --git a/src/Asset/Service/ThumbnailService.php b/src/Asset/Service/ThumbnailService.php index 165ebcd82..6bac295e1 100644 --- a/src/Asset/Service/ThumbnailService.php +++ b/src/Asset/Service/ThumbnailService.php @@ -57,7 +57,7 @@ public function getThumbnailFromConfiguration( $thumbnailConfig = $this->getImageThumbnailConfig($image, $parameters); $thumbnail = $image->getThumbnail($thumbnailConfig); $dpi = $parameters->getDpi(); - if ($dpi && $thumbnailConfig->getFormat() === MimeTypes::JPEG) { + if ($dpi && $thumbnailConfig->getFormat() === MimeTypes::JPEG->value) { $this->resizeThumbnailFile($thumbnail, $dpi); } @@ -137,7 +137,7 @@ private function getImageThumbnailConfig( } $thumbnailConfig->setRasterizeSVG(true); - if ($parameters->getMimeType() === MimeTypes::JPEG) { + if ($parameters->getMimeType() === MimeTypes::JPEG->value) { $thumbnailConfig->setPreserveMetaData(true); if ($quality === null) { diff --git a/src/Asset/Service/UploadService.php b/src/Asset/Service/UploadService.php index 0d2ff99f1..6295dc859 100644 --- a/src/Asset/Service/UploadService.php +++ b/src/Asset/Service/UploadService.php @@ -17,6 +17,7 @@ namespace Pimcore\Bundle\StudioBackendBundle\Asset\Service; use Exception; +use League\Flysystem\FilesystemException; use Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\IndexQueue\SynchronousProcessingServiceInterface; use Pimcore\Bundle\GenericExecutionEngineBundle\Agent\JobExecutionAgentInterface; use Pimcore\Bundle\GenericExecutionEngineBundle\Model\Job; @@ -27,6 +28,7 @@ use Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Messages\AssetUploadMessage; use Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\Util\EnvironmentVariables; use Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\Util\JobSteps; +use Pimcore\Bundle\StudioBackendBundle\Element\Service\StorageServiceInterface; use Pimcore\Bundle\StudioBackendBundle\Exception\Api\AccessDeniedException; use Pimcore\Bundle\StudioBackendBundle\Exception\Api\DatabaseException; use Pimcore\Bundle\StudioBackendBundle\Exception\Api\EnvironmentException; @@ -42,6 +44,7 @@ use Pimcore\Model\UserInterface; use Symfony\Component\HttpFoundation\File\UploadedFile; use Symfony\Component\Mime\MimeTypes; +use function dirname; /** * @internal @@ -54,6 +57,7 @@ public function __construct( private AssetServiceResolverInterface $assetServiceResolver, private JobExecutionAgentInterface $jobExecutionAgent, private ServiceResolverInterface $serviceResolver, + private StorageServiceInterface $storageService, private SynchronousProcessingServiceInterface $synchronousProcessingService, ) { @@ -80,38 +84,44 @@ public function fileExists( * @throws AccessDeniedException * @throws DatabaseException * @throws EnvironmentException + * @throws FilesystemException * @throws ForbiddenException * @throws NotFoundException */ public function uploadAsset( int $parentId, - UploadedFile $file, - UserInterface $user + string $fileName, + string $filePath, + UserInterface $user, + bool $useFlysystem = false ): int { $parent = $this->validateParent($user, $parentId); - $sourcePath = $this->getValidSourcePath($file); - $fileName = $this->getValidFileName($file); - $uniqueName = $this->assetService->getUniqueAssetName($parent->getRealPath(), $fileName); + $fileName = $this->getValidFileName($fileName); + $uniqueName = $this->assetService->getUniqueAssetName($parent->getRealFullPath(), $fileName); $userId = $user->getId(); - - try { - $this->synchronousProcessingService->enable(); - $asset = $this->assetResolver->create( - $parentId, - [ - 'filename' => $uniqueName, - 'sourcePath' => $sourcePath, - 'userOwner' => $userId, - 'userModification' => $userId, - ] - ); - } catch (Exception $e) { - throw new DatabaseException($e->getMessage()); - } finally { - @unlink($sourcePath); + $assetParams = [ + 'filename' => $uniqueName, + 'userOwner' => $userId, + 'userModification' => $userId, + ]; + $this->synchronousProcessingService->enable(); + + if ($useFlysystem) { + return $this->uploadAssetFromFlysystem($parentId, $assetParams, $filePath); } - return $asset->getId(); + return $this->uploadAssetLocally($parentId, $assetParams, $filePath); + } + + public function uploadParentFolder(string $filePath, int $rootParentId, UserInterface $user): int + { + $rootParent = $this->validateParent($user, $rootParentId); + $this->synchronousProcessingService->enable(); + $parent = $this->assetServiceResolver->createFolderByPath( + $rootParent->getRealFullPath() . '/' . preg_replace('@^/@', '', dirname($filePath)) + ); + + return $parent->getId(); } /** @@ -139,7 +149,7 @@ public function uploadAssetsAsynchronously( }, $files, array_keys($files)), environmentData: [ EnvironmentVariables::PARENT_ID->value => $parentId, - EnvironmentVariables::UPLOAD_FOLDER_NAME->value => $folderName, + EnvironmentVariables::UPLOAD_FOLDER_LOCATION->value => $folderName, ] ); $jobRun = $this->jobExecutionAgent->startJobExecution( @@ -173,8 +183,8 @@ public function replaceAssetBinary( ); } - $sourcePath = $this->getValidSourcePath($file); - $fileName = $this->getValidFileName($file); + $sourcePath = $this->getValidSourcePath($file->getRealPath()); + $fileName = $this->getValidFileName($file->getClientOriginalName()); $this->validateMimeType($file, $fileName, $asset->getType()); try { @@ -225,12 +235,70 @@ public function sanitizeFileToUpload(string $fileName): ?string return $fileName; } + /** + * @throws FilesystemException + */ + public function cleanupTemporaryUploadFiles(string $location): void + { + $this->storageService->cleanUpFolder($location, true); + $this->storageService->cleanUpLocalFolder(PIMCORE_SYSTEM_TEMP_DIRECTORY . '/' . $location); + } + + /** + * @throws DatabaseException + * @throws EnvironmentException + */ + private function uploadAssetLocally( + int $parentId, + array $assetParams, + string $sourcePath, + ): int { + $assetParams['sourcePath'] = $this->getValidSourcePath($sourcePath); + + try { + $asset = $this->assetResolver->create( + $parentId, + $assetParams + ); + } catch (Exception $e) { + throw new DatabaseException($e->getMessage()); + } finally { + @unlink($sourcePath); + } + + return $asset->getId(); + } + + /** + * @throws FilesystemException|EnvironmentException + */ + private function uploadAssetFromFlysystem( + int $parentId, + array $assetParams, + string $sourcePath, + ): int { + $storage = $this->storageService->getTempStorage(); + $assetParams['stream'] = $storage->readStream($sourcePath); + + try { + $asset = $this->assetResolver->create( + $parentId, + $assetParams + ); + } catch (Exception $e) { + throw new EnvironmentException($e->getMessage()); + } finally { + $storage->delete($sourcePath); + } + + return $asset->getId(); + } + /** * @throws EnvironmentException */ - private function getValidSourcePath(UploadedFile $file): string + private function getValidSourcePath(string $sourcePath): string { - $sourcePath = $file->getRealPath(); if (!is_file($sourcePath)) { throw new EnvironmentException( 'Something went wrong, please check upload_max_filesize and post_max_size in your php.ini ' . @@ -248,10 +316,10 @@ private function getValidSourcePath(UploadedFile $file): string /** * @throws EnvironmentException */ - private function getValidFileName(UploadedFile $file): string + private function getValidFileName(string $originalFileName): string { $fileName = $this->serviceResolver->getValidKey( - $file->getClientOriginalName(), + $originalFileName, ElementTypes::TYPE_ASSET ); diff --git a/src/Asset/Service/UploadServiceInterface.php b/src/Asset/Service/UploadServiceInterface.php index 8efa35412..b7bbc442b 100644 --- a/src/Asset/Service/UploadServiceInterface.php +++ b/src/Asset/Service/UploadServiceInterface.php @@ -16,6 +16,7 @@ namespace Pimcore\Bundle\StudioBackendBundle\Asset\Service; +use League\Flysystem\FilesystemException; use Pimcore\Bundle\StudioBackendBundle\Exception\Api\AccessDeniedException; use Pimcore\Bundle\StudioBackendBundle\Exception\Api\DatabaseException; use Pimcore\Bundle\StudioBackendBundle\Exception\Api\EnvironmentException; @@ -45,14 +46,17 @@ public function fileExists( * @throws AccessDeniedException * @throws DatabaseException * @throws EnvironmentException + * @throws FilesystemException * @throws ForbiddenException * @throws NotFoundException * @throws UserNotFoundException */ public function uploadAsset( int $parentId, - UploadedFile $file, - UserInterface $user + string $fileName, + string $filePath, + UserInterface $user, + bool $useFlysystem = false ): int; /** @@ -65,6 +69,8 @@ public function uploadAssetsAsynchronously( string $folderName, ): int; + public function uploadParentFolder(string $filePath, int $rootParentId, UserInterface $user): int; + /** * @throws AccessDeniedException * @throws DatabaseException @@ -84,4 +90,9 @@ public function replaceAssetBinary( public function validateParent(UserInterface $user, int $parentId): ElementInterface; public function sanitizeFileToUpload(string $fileName): ?string; + + /** + * @throws FilesystemException + */ + public function cleanupTemporaryUploadFiles(string $location): void; } diff --git a/src/Asset/Util/Constants/Csv.php b/src/Asset/Util/Constants/Csv.php index 919f9fe7a..dda3f2009 100644 --- a/src/Asset/Util/Constants/Csv.php +++ b/src/Asset/Util/Constants/Csv.php @@ -21,6 +21,8 @@ enum Csv: string { use EnumToValueArrayTrait; + case ASSET_TO_EXPORT = 'asset_to_export'; + case ASSET_EXPORT_DATA = 'asset_export_data'; case JOB_STEP_CONFIG_SETTINGS = 'settings'; case JOB_STEP_CONFIG_CONFIGURATION = 'configuration'; case SETTINGS_DELIMITER = 'delimiter'; diff --git a/src/DataIndex/Adapter/AssetSearchAdapter.php b/src/DataIndex/Adapter/AssetSearchAdapter.php index c711a55ac..75db2c425 100644 --- a/src/DataIndex/Adapter/AssetSearchAdapter.php +++ b/src/DataIndex/Adapter/AssetSearchAdapter.php @@ -18,7 +18,9 @@ use Pimcore\Bundle\GenericDataIndexBundle\Exception\AssetSearchException; use Pimcore\Bundle\GenericDataIndexBundle\Exception\OpenSearch\SearchFailedException; +use Pimcore\Bundle\GenericDataIndexBundle\Model\Search\Asset\AssetSearch; use Pimcore\Bundle\GenericDataIndexBundle\Model\Search\Modifier\Sort\Tree\OrderByFullPath; +use Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchService\Asset\Aggregation\FileSizeAggregationServiceInterface; use Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchService\Asset\AssetSearchServiceInterface; use Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchService\SearchResultIdListServiceInterface; use Pimcore\Bundle\StudioBackendBundle\Asset\Schema\Asset; @@ -34,6 +36,7 @@ public function __construct( private AssetSearchServiceInterface $searchService, private AssetHydratorServiceInterface $assetHydratorService, private SearchResultIdListServiceInterface $searchResultIdListService, + private FileSizeAggregationServiceInterface $fileSizeAggregationService, ) { } @@ -95,4 +98,17 @@ public function fetchAssetIds(QueryInterface $assetQuery): array throw new SearchException('assets'); } } + + /** + * @throws AssetSearchException + */ + public function getTotalFileSizeByIds(QueryInterface $assetQuery): int + { + $search = $assetQuery->getSearch(); + if (!$search instanceof AssetSearch) { + throw new AssetSearchException('Invalid search query'); + } + + return $this->fileSizeAggregationService->getFileSizeSum($search); + } } diff --git a/src/DataIndex/Adapter/AssetSearchAdapterInterface.php b/src/DataIndex/Adapter/AssetSearchAdapterInterface.php index 01e17efd2..bfb6557d1 100644 --- a/src/DataIndex/Adapter/AssetSearchAdapterInterface.php +++ b/src/DataIndex/Adapter/AssetSearchAdapterInterface.php @@ -16,6 +16,7 @@ namespace Pimcore\Bundle\StudioBackendBundle\DataIndex\Adapter; +use Pimcore\Bundle\GenericDataIndexBundle\Exception\AssetSearchException; use Pimcore\Bundle\StudioBackendBundle\Asset\Schema\Asset; use Pimcore\Bundle\StudioBackendBundle\DataIndex\AssetSearchResult; use Pimcore\Bundle\StudioBackendBundle\DataIndex\Query\QueryInterface; @@ -40,4 +41,9 @@ public function getAssetById(int $id): Asset; * @return array */ public function fetchAssetIds(QueryInterface $assetQuery): array; + + /** + * @throws AssetSearchException + */ + public function getTotalFileSizeByIds(QueryInterface $assetQuery): int; } diff --git a/src/DataIndex/AssetSearchService.php b/src/DataIndex/AssetSearchService.php index 08d04156f..ba9b5c931 100644 --- a/src/DataIndex/AssetSearchService.php +++ b/src/DataIndex/AssetSearchService.php @@ -16,6 +16,7 @@ namespace Pimcore\Bundle\StudioBackendBundle\DataIndex; +use Pimcore\Bundle\GenericDataIndexBundle\Exception\AssetSearchException; use Pimcore\Bundle\StudioBackendBundle\Asset\Schema\Asset; use Pimcore\Bundle\StudioBackendBundle\Asset\Schema\Type\Archive; use Pimcore\Bundle\StudioBackendBundle\Asset\Schema\Type\Audio; @@ -36,7 +37,7 @@ { public function __construct( private AssetSearchAdapterInterface $assetSearchAdapter, - private AssetQueryProviderInterface $assetQueryProvider + private AssetQueryProviderInterface $assetQueryProvider, ) { } @@ -91,4 +92,15 @@ public function countChildren( return count($this->getChildrenIds($parentPath, $sortDirection)); } + + /** + * @throws AssetSearchException + */ + public function getTotalFileSizeByIds(array $ids): int + { + $query = $this->assetQueryProvider->createAssetQuery(); + $query->searchByIds($ids); + + return $this->assetSearchAdapter->getTotalFileSizeByIds($query); + } } diff --git a/src/DataIndex/AssetSearchServiceInterface.php b/src/DataIndex/AssetSearchServiceInterface.php index 63a4d64ef..4d41abc49 100644 --- a/src/DataIndex/AssetSearchServiceInterface.php +++ b/src/DataIndex/AssetSearchServiceInterface.php @@ -16,6 +16,7 @@ namespace Pimcore\Bundle\StudioBackendBundle\DataIndex; +use Pimcore\Bundle\GenericDataIndexBundle\Exception\AssetSearchException; use Pimcore\Bundle\StudioBackendBundle\Asset\Schema\Asset; use Pimcore\Bundle\StudioBackendBundle\Asset\Schema\Type\Archive; use Pimcore\Bundle\StudioBackendBundle\Asset\Schema\Type\Audio; @@ -66,4 +67,9 @@ public function countChildren( string $parentPath, ?string $sortDirection = null ): int; + + /** + * @throws AssetSearchException + */ + public function getTotalFileSizeByIds(array $ids): int; } diff --git a/src/DataIndex/Query/AssetQuery.php b/src/DataIndex/Query/AssetQuery.php index 49578d26b..6ff6295d3 100644 --- a/src/DataIndex/Query/AssetQuery.php +++ b/src/DataIndex/Query/AssetQuery.php @@ -19,6 +19,7 @@ use Pimcore\Bundle\GenericDataIndexBundle\Enum\Search\SortDirection; use Pimcore\Bundle\GenericDataIndexBundle\Model\Search\Interfaces\SearchInterface; use Pimcore\Bundle\GenericDataIndexBundle\Model\Search\Modifier\Filter\Basic\ExcludeFoldersFilter; +use Pimcore\Bundle\GenericDataIndexBundle\Model\Search\Modifier\Filter\Basic\IdsFilter; use Pimcore\Bundle\GenericDataIndexBundle\Model\Search\Modifier\Filter\Tree\ParentIdFilter; use Pimcore\Bundle\GenericDataIndexBundle\Model\Search\Modifier\Filter\Tree\PathFilter; use Pimcore\Bundle\GenericDataIndexBundle\Model\Search\Modifier\FullTextSearch\ElementKeySearch; @@ -89,4 +90,11 @@ public function orderByPath(string $direction): self return $this; } + + public function searchByIds(array $ids): self + { + $this->search->addModifier(new IdsFilter($ids)); + + return $this; + } } diff --git a/src/DataIndex/Query/DataObjectQuery.php b/src/DataIndex/Query/DataObjectQuery.php index 8b054fe9e..2fcb34c11 100644 --- a/src/DataIndex/Query/DataObjectQuery.php +++ b/src/DataIndex/Query/DataObjectQuery.php @@ -20,6 +20,7 @@ use Pimcore\Bundle\GenericDataIndexBundle\Enum\Search\SortDirection; use Pimcore\Bundle\GenericDataIndexBundle\Model\Search\DataObject\DataObjectSearch; use Pimcore\Bundle\GenericDataIndexBundle\Model\Search\Modifier\Filter\Basic\ExcludeFoldersFilter; +use Pimcore\Bundle\GenericDataIndexBundle\Model\Search\Modifier\Filter\Basic\IdsFilter; use Pimcore\Bundle\GenericDataIndexBundle\Model\Search\Modifier\Filter\Tree\ParentIdFilter; use Pimcore\Bundle\GenericDataIndexBundle\Model\Search\Modifier\Filter\Tree\PathFilter; use Pimcore\Bundle\GenericDataIndexBundle\Model\Search\Modifier\FullTextSearch\ElementKeySearch; @@ -108,4 +109,11 @@ public function orderByPath(string $direction): self return $this; } + + public function searchByIds(array $ids): self + { + $this->search->addModifier(new IdsFilter($ids)); + + return $this; + } } diff --git a/src/DataIndex/Query/QueryInterface.php b/src/DataIndex/Query/QueryInterface.php index 2fc37dccf..8dec22655 100644 --- a/src/DataIndex/Query/QueryInterface.php +++ b/src/DataIndex/Query/QueryInterface.php @@ -35,4 +35,6 @@ public function excludeFolders(): self; public function getSearch(): SearchInterface; public function orderByPath(string $direction): self; + + public function searchByIds(array $ids): self; } diff --git a/src/DependencyInjection/Configuration.php b/src/DependencyInjection/Configuration.php index 5bb76d6da..1a182fd16 100644 --- a/src/DependencyInjection/Configuration.php +++ b/src/DependencyInjection/Configuration.php @@ -17,6 +17,7 @@ namespace Pimcore\Bundle\StudioBackendBundle\DependencyInjection; use Pimcore\Bundle\StudioBackendBundle\Exception\InvalidHostException; +use Pimcore\Bundle\StudioBackendBundle\Util\Constants\Asset\DownloadLimits; use Pimcore\Bundle\StudioBackendBundle\Util\Constants\Asset\MimeTypes; use Pimcore\Bundle\StudioBackendBundle\Util\Constants\Asset\ResizeModes; use Symfony\Component\Config\Definition\Builder\ArrayNodeDefinition; @@ -48,6 +49,8 @@ public function getConfigTreeBuilder(): TreeBuilder $this->addDefaultAssetFormats($rootNode); $this->addRecycleBinThreshold($rootNode); $this->addMercureConfiguration($rootNode); + $this->addAssetDownloadLimits($rootNode); + $this->addCsvSettings($rootNode); return $treeBuilder; } @@ -124,7 +127,7 @@ private function addDefaultAssetFormats(ArrayNodeDefinition $node): void ->integerNode('width')->isRequired()->end() ->integerNode('dpi')->isRequired()->end() ->enumNode('format') - ->values(MimeTypes::ALLOWED_FORMATS) + ->values([MimeTypes::JPEG->value, MimeTypes::PNG->value]) ->isRequired() ->cannotBeEmpty() ->end() @@ -167,4 +170,36 @@ private function addMercureConfiguration(ArrayNodeDefinition $node): void ->end() ->end(); } + + private function addAssetDownloadLimits(ArrayNodeDefinition $node): void + { + $node->children() + ->arrayNode('asset_download_settings') + ->addDefaultsIfNotSet() + ->children() + ->integerNode(DownloadLimits::MAX_ZIP_FILE_SIZE->value) + ->info('The maximum size of all assets together that can be downloaded in bytes.') + ->defaultValue(5 * 1024 * 1024 * 1024) + ->end() + ->integerNode(DownloadLimits::MAX_ZIP_FILE_AMOUNT->value) + ->info('The maximum amount of assets that can be downloaded at once.') + ->defaultValue(1000) + ->end() + ->end() + ->end(); + } + + private function addCsvSettings(ArrayNodeDefinition $node): void + { + $node->children() + ->arrayNode('csv_settings') + ->addDefaultsIfNotSet() + ->children() + ->scalarNode('default_delimiter') + ->info('Default delimiter to be used for csv operations.') + ->defaultValue(',') + ->end() + ->end() + ->end(); + } } diff --git a/src/DependencyInjection/PimcoreStudioBackendExtension.php b/src/DependencyInjection/PimcoreStudioBackendExtension.php index d1a3bb756..5974dd248 100644 --- a/src/DependencyInjection/PimcoreStudioBackendExtension.php +++ b/src/DependencyInjection/PimcoreStudioBackendExtension.php @@ -19,7 +19,9 @@ use Exception; use Pimcore\Bundle\CoreBundle\DependencyInjection\ConfigurationHelper; use Pimcore\Bundle\StudioBackendBundle\Asset\Service\DownloadServiceInterface; +use Pimcore\Bundle\StudioBackendBundle\Asset\Service\ExecutionEngine\CsvServiceInterface; use Pimcore\Bundle\StudioBackendBundle\Asset\Service\ExecutionEngine\DeleteServiceInterface; +use Pimcore\Bundle\StudioBackendBundle\Asset\Service\ExecutionEngine\ZipServiceInterface; use Pimcore\Bundle\StudioBackendBundle\Element\Service\ElementDeleteServiceInterface; use Pimcore\Bundle\StudioBackendBundle\EventSubscriber\CorsSubscriber; use Pimcore\Bundle\StudioBackendBundle\Exception\InvalidPathException; @@ -79,6 +81,13 @@ public function load(array $configs, ContainerBuilder $container): void $definition = $container->getDefinition(HubServiceInterface::class); $definition->setArgument('$cookieLifetime', $config['mercure_settings']['cookie_lifetime']); + + $definition = $container->getDefinition(ZipServiceInterface::class); + $definition->setArgument('$downloadLimits', $config['asset_download_settings']); + + $definition = $container->getDefinition(CsvServiceInterface::class); + $definition->setArgument('$defaultDelimiter', $config['csv_settings']['default_delimiter']); + } public function prepend(ContainerBuilder $container): void diff --git a/src/Element/Service/StorageService.php b/src/Element/Service/StorageService.php new file mode 100644 index 000000000..c8e79d8dd --- /dev/null +++ b/src/Element/Service/StorageService.php @@ -0,0 +1,194 @@ +getTempStorage(); + + try { + $storage->delete($location); + } catch (FilesystemException $e) { + throw new EnvironmentException( + sprintf( + 'Could not remove file %s: %s', + $location, + $e->getMessage() + ) + ); + } + } + + /** + * @throws EnvironmentException + */ + public function tempFileExists(string $location): bool + { + $storage = $this->getTempStorage(); + + try { + return $storage->fileExists($location); + } catch (FilesystemException $e) { + throw new EnvironmentException( + sprintf( + 'Could not look for file %s: %s', + $location, + $e->getMessage() + ) + ); + } + } + + public function copyElementToFlysystem( + string $innerPath, + string $localElementPath, + string $targetPath, + ): void { + match (true) { + is_file($localElementPath) => $this->copyFileToFlysystem($innerPath, $localElementPath, $targetPath), + is_dir($localElementPath) => $this->copyFolderToFlysystem($innerPath, $targetPath), + default => throw new EnvironmentException( + sprintf( + 'The element with path %s could not be copied to Flysystem.', + $localElementPath + ) + ) + }; + } + + /** + * @throws FilesystemException + */ + public function cleanUpFolder( + string $folder, + bool $removeContents = false + ): void { + $storage = $this->getTempStorage(); + + if ($removeContents || empty($storage->listContents($folder)->toArray())) { + $storage->deleteDirectory($folder); + } + } + + public function cleanUpLocalFolder( + string $folderLocation + ): void { + if ($this->filesystem->exists($folderLocation)) { + $this->filesystem->remove($folderLocation); + } + } + + public function cleanUpLocalFile( + string $filePath + ): void { + if (is_file($filePath)) { + @unlink($filePath); + } + } + + public function cleanUpFlysystemFile( + string $filePath + ): void { + if ($this->tempFileExists($filePath)) { + $this->removeTempFile($filePath); + } + } + + public function getThumbnailStorage(): FilesystemOperator + { + return $this->storageResolver->get(StorageDirectories::THUMBNAIL->value); + } + + public function getTempStorage(): FilesystemOperator + { + return $this->storageResolver->get(StorageDirectories::TEMP->value); + } + + /** + * @throws EnvironmentException + */ + private function copyFileToFlysystem( + string $fileName, + string $localFilePath, + string $targetPath, + ): void { + try { + $this->getTempStorage()->writeStream( + $targetPath . '/' . $fileName, + fopen($localFilePath, 'rb') + ); + @unlink($localFilePath); + } catch (FilesystemException) { + throw new EnvironmentException( + sprintf( + 'Failed to copy file %s to Flysystem.', + $fileName + ) + ); + } + } + + /** + * @throws EnvironmentException + */ + private function copyFolderToFlysystem( + string $folderName, + string $targetPath + ): void { + $storage = $this->getTempStorage(); + $storagePath = $targetPath . '/' . $folderName; + + try { + if ($storage->directoryExists($storagePath)) { + return; + } + + $storage->createDirectory($storagePath); + } catch (FilesystemException) { + throw new EnvironmentException( + sprintf( + 'Failed to copy folder %s to Flysystem.', + $folderName + ) + ); + } + } +} diff --git a/src/Element/Service/StorageServiceInterface.php b/src/Element/Service/StorageServiceInterface.php new file mode 100644 index 000000000..2088a93ad --- /dev/null +++ b/src/Element/Service/StorageServiceInterface.php @@ -0,0 +1,70 @@ +getJob()?->getSteps() ?? []); + $totalEvents = $steps; if (isset($jobRun->getContext()[self::TOTAL_EVENTS])) { return $jobRun->getContext()[self::TOTAL_EVENTS]; } - $totalEvents = $steps * $jobRun->getTotalElements(); + if ($jobRun->getTotalElements() > 0) { + $totalEvents = $steps * $jobRun->getTotalElements(); + } - $this->updateJobRunContext($jobRun, self::TOTAL_EVENTS, $steps * $jobRun->getTotalElements()); + $this->updateJobRunContext($jobRun, self::TOTAL_EVENTS, $totalEvents); return $totalEvents; } diff --git a/src/Util/Traits/StreamedResponseTrait.php b/src/Util/Traits/StreamedResponseTrait.php index c5dfb891c..471e351c9 100644 --- a/src/Util/Traits/StreamedResponseTrait.php +++ b/src/Util/Traits/StreamedResponseTrait.php @@ -99,31 +99,44 @@ function () use ($stream) { ); } - protected function getFileStreamedResponse(string $path, string $mimeType, string $filename): StreamedResponse - { - $stream = fopen($path, 'rb'); - - if (!$stream) { - throw new StreamResourceNotFoundException(sprintf('Resource not found: %s', $path)); - } - - $response = new StreamedResponse( - function () use ($stream) { - fpassthru($stream); - }, - HttpResponseCodes::SUCCESS->value, - $this->getResponseHeaders( - mimeType: $mimeType, - fileSize: filesize($path), - filename: $filename, - contentDisposition: HttpResponseHeaders::ATTACHMENT_TYPE->value - ), + /** + * @throws StreamResourceNotFoundException + */ + protected function getFileStreamedResponse( + string $path, + string $mimeType, + string $filename, + FilesystemOperator $storage, + ): StreamedResponse { + try { + $stream = $storage->readStream($path); + + $response = new StreamedResponse( + function () use ($stream) { + fpassthru($stream); + }, + HttpResponseCodes::SUCCESS->value, + $this->getResponseHeaders( + mimeType: $mimeType, + fileSize: $storage->fileSize($path), + filename: $filename, + contentDisposition: HttpResponseHeaders::ATTACHMENT_TYPE->value + ), + ); - ); + $storage->delete($path); - unlink($path); + return $response; + } catch (FilesystemException $e) { + throw new StreamResourceNotFoundException( + sprintf( + 'Could not process stream for file %s: %s', + $path, + $e->getMessage() + ) + ); + } - return $response; } private function getResponseHeaders( diff --git a/src/Version/Hydrator/AssetVersionHydrator.php b/src/Version/Hydrator/AssetVersionHydrator.php index d05b7b072..8a902d6e3 100644 --- a/src/Version/Hydrator/AssetVersionHydrator.php +++ b/src/Version/Hydrator/AssetVersionHydrator.php @@ -49,7 +49,7 @@ public function hydrate( ): ImageVersion|AssetVersion { if ( $asset instanceof Asset\Document && - $asset->getMimeType() === MimeTypes::PDF && + $asset->getMimeType() === MimeTypes::PDF->value && $this->documentService->isScanningEnabled() ) { $this->documentService->validatePdfScanStatus( diff --git a/src/Version/Service/VersionBinaryService.php b/src/Version/Service/VersionBinaryService.php index 717b0207a..bfb9b1233 100644 --- a/src/Version/Service/VersionBinaryService.php +++ b/src/Version/Service/VersionBinaryService.php @@ -120,7 +120,7 @@ public function streamPdfPreview( $document = $this->repository->getElementFromVersion($version, $user); if (!$document instanceof Document || - $document->getMimeType() !== MimeTypes::PDF + $document->getMimeType() !== MimeTypes::PDF->value ) { throw new InvalidElementTypeException($document->getType()); } diff --git a/translations/studio.en.yaml b/translations/studio.en.yaml index 01e07c4fd..3ac62d399 100644 --- a/translations/studio.en.yaml +++ b/translations/studio.en.yaml @@ -5,19 +5,21 @@ studio_ee_environment_variable_not_found: Environment variable %variable% not fo studio_ee_element_locked: Element with type %type% with ID %id% is locked studio_ee_element_permission_missing: User with ID %userId% has missing permission %permission% on element with type %type% and ID %id% studio_ee_element_delete_failed: 'Element with type %type% with ID %id% could not be deleted: %message%' +studio_ee_zip_file_copy_failed: 'Zip file could not be copied: %message%' studio_ee_zip_file_upload_failed: 'Zip file could not be uploaded: %message%' studio_ee_zip_cleanup_failed: 'Zip directory %directory% could not be removed: %message%' +studio_ee_csv_creation_failed: 'CSV Export failed: %message%' +studio_ee_csv_data_collection_failed: 'Data for CSV export could not be extracted for asset %id%: %message%' studio_ee_element_patch_failed: 'Element with type %type% with ID %id% could not be updated: %message%' studio_ee_no_element_data_found: 'No data found' -studio_ee_job_create_zip: Create Zip +studio_ee_job_create_download_zip: Create Download Zip studio_ee_job_clone_assets: Clone Assets studio_ee_job_upload_zip_file: Upload Zip File studio_ee_job_delete_assets: Delete Assets studio_ee_job_patch_elements: Patch Elements studio_ee_job_create_csv: Create CSV studio_ee_job_upload_assets: Upload Assets -studio_ee_job_step_zip_collection: Zip Collection -studio_ee_job_step_zip_creation: Zip Creation +studio_ee_job_step_zip_creation: Zip Download Creation studio_ee_job_step_asset_deletion: Asset Deletion studio_ee_job_step_asset_cloning: Asset Cloning studio_ee_job_step_csv_file_not_found: CSV file not found