From fb1fd5a400e433025ab3bb83f183107f3e321e52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Cygankiewicz?= Date: Wed, 22 May 2024 10:52:25 +0200 Subject: [PATCH] Image to video - Stability AI --- Classes/ContextMenu/ContentAiItemProvider.php | 19 ++- Classes/Controller/AiImageController.php | 71 ++++------ Classes/Controller/AiVideoController.php | 125 ++++++++++++++++++ Classes/Controller/AjaxController.php | 14 +- Classes/Controller/BaseController.php | 52 +++++++- .../Http/Client/Action/StabilityAiAction.php | 2 + Classes/Http/Client/BaseClient.php | 18 +++ Classes/Http/Client/ImageApiInterface.php | 7 + Classes/Http/Client/OpenAiClient.php | 21 +++ Classes/Http/Client/StabilityAiClient.php | 94 ++++++++++++- Classes/Http/Client/VideoApiInterface.php | 23 ++++ Classes/Service/FileService.php | 4 +- Configuration/Backend/Modules.php | 3 + Resources/Private/Language/de.locallang.xlf | 4 + .../Language/de.locallang_contentai.xlf | 20 +++ Resources/Private/Language/locallang.xlf | 3 + .../Private/Language/locallang_contentai.xlf | 15 +++ Resources/Private/Partials/Crop.html | 82 ++++++++++++ .../Templates/AiImage/CropAndExtend.html | 67 +--------- .../Private/Templates/AiImage/Filelist.html | 20 ++- .../Templates/AiVideo/ImageToVideo.html | 47 +++++++ .../AiVideo/PrepareImageToVideo.html | 13 ++ Resources/Public/JavaScript/ContextMenu.js | 8 ++ Resources/Public/JavaScript/MkContentAi.js | 89 ++++++++++--- .../Public/JavaScript/context-menu-actions.js | 4 + ext_tables.php | 5 +- 26 files changed, 678 insertions(+), 152 deletions(-) create mode 100644 Classes/Controller/AiVideoController.php create mode 100644 Classes/Http/Client/VideoApiInterface.php create mode 100644 Resources/Private/Partials/Crop.html create mode 100644 Resources/Private/Templates/AiVideo/ImageToVideo.html create mode 100644 Resources/Private/Templates/AiVideo/PrepareImageToVideo.html diff --git a/Classes/ContextMenu/ContentAiItemProvider.php b/Classes/ContextMenu/ContentAiItemProvider.php index c5a2b2b..bb30a3a 100644 --- a/Classes/ContextMenu/ContentAiItemProvider.php +++ b/Classes/ContextMenu/ContentAiItemProvider.php @@ -75,6 +75,12 @@ class ContentAiItemProvider extends AbstractProvider 'iconIdentifier' => 'actions-rocket', 'callbackAction' => 'altTexts', ], + 'filePrepareImageToVideo' => [ + 'type' => 'item', + 'label' => 'LLL:EXT:mkcontentai/Resources/Private/Language/locallang_contentai.xlf:labelContextMenuImageToVideo', + 'iconIdentifier' => 'actions-rocket', + 'callbackAction' => 'prepareImageToVideo', + ], ]; public function setContext(string $table, string $identifier, string $context = ''): void @@ -127,7 +133,7 @@ public function canRender(string $itemName, string $type): bool $canRender = false; if ( - (('fileUpscale' === $itemName || 'fileExtend' === $itemName) && true === $this->checkAllowedOperationsByClient($itemName, $imageAiEngine)) + (('fileUpscale' === $itemName || 'fileExtend' === $itemName || 'filePrepareImageToVideo' === $itemName) && true === $this->checkAllowedOperationsByClient($itemName, $imageAiEngine)) || 'fileAlt' === $itemName ) { return $this->isImage(); @@ -209,6 +215,7 @@ private function updateParametersForItemName(array &$parameters, string $itemNam 'fileExtend' => 'cropAndExtend', 'fileAlt' => 'altText', 'folderAltTexts' => 'altTexts', + 'filePrepareImageToVideo' => 'prepareImageToVideo', ]; if (11 === $version) { @@ -218,6 +225,10 @@ private function updateParametersForItemName(array &$parameters, string $itemNam if (('fileAlt' === $itemName || 'folderAltTexts' === $itemName) && 11 === $version) { $parameters['tx_mkcontentai_system_mkcontentaicontentai']['controller'] = 'AiText'; } + + if (('filePrepareImageToVideo' === $itemName) && 11 === $version) { + $parameters['tx_mkcontentai_system_mkcontentaicontentai']['controller'] = 'AiVideo'; + } } /** @@ -273,6 +284,10 @@ private function getPathInfo(string $itemName, int $version): string 12 => '/module/mkcontentai/AiText/altTexts', 11 => '/module/system/MkcontentaiContentai', ], + 'filePrepareImageToVideo' => [ + 12 => '/module/mkcontentai/AiVideo/prepareImageToVideo', + 11 => '/module/system/MkcontentaiContentai', + ], ]; return $pathInfoMapping[$itemName][$version] ?? ''; @@ -287,7 +302,7 @@ private function checkAllowedOperationsByClient(string $itemName, int $imageAiEn $openAiClient = GeneralUtility::makeInstance(OpenAiClient::class); foreach ([$stabilityAiClient, $openAiClient] as $aiClient) { - if (get_class($aiClient) === AiImageController::GENERATOR_ENGINE[$imageAiEngine] && in_array(strtolower(str_replace('file', '', $itemName)), $aiClient->getAllowedOperations())) { + if (get_class($aiClient) === AiImageController::GENERATOR_ENGINE[$imageAiEngine] && in_array(lcfirst(str_replace('file', '', $itemName)), $aiClient->getAllowedOperations())) { return true; } } diff --git a/Classes/Controller/AiImageController.php b/Classes/Controller/AiImageController.php index ebe2ad0..ead627b 100644 --- a/Classes/Controller/AiImageController.php +++ b/Classes/Controller/AiImageController.php @@ -57,37 +57,7 @@ class AiImageController extends BaseController public function initializeAction(): void { - $client = $this->initializeClient(); - if (isset($client['error'])) { - $this->addFlashMessage( - $client['error'], - '', - AbstractMessage::ERROR - ); - - return; - } - if (isset($client['client'])) { - $this->client = $client['client']; - } - - $arguments['actionName'] = $this->request->getControllerActionName(); - if (!in_array($arguments['actionName'], $this->client->getAllowedOperations())) { - $translatedMessage = LocalizationUtility::translate('labelNotAllowed', 'mkcontentai', $arguments) ?? ''; - $this->addFlashMessage($translatedMessage.' '.get_class($this->client), '', AbstractMessage::ERROR); - $this->redirect('filelist'); - } - - $infoMessage = LocalizationUtility::translate('labelEngineInitialized', 'mkcontentai') ?? ''; - if (isset($client['clientClass'])) { - $infoMessage .= ' '.$client['clientClass']; - } - $this->addFlashMessage( - $infoMessage, - '', - AbstractMessage::INFO - ); - parent::initializeAction(); + $this->initializeAndAuthorizeAction(); } /** @@ -108,20 +78,6 @@ public function filelistAction() return $this->handleResponse(); } - protected function handleResponse(): ResponseInterface - { - if (null === $this->moduleTemplateFactory) { - $translatedMessage = LocalizationUtility::translate('labelErrorModuleTemplateFactory', 'mkcontentai') ?? ''; - - throw new \Exception($translatedMessage, 1623345720); - } - - $moduleTemplate = $this->moduleTemplateFactory->create($this->request); - $moduleTemplate->setContent($this->view->render()); - - return $this->htmlResponse($moduleTemplate->renderContent()); - } - /** * @return ResponseInterface */ @@ -190,7 +146,7 @@ public function upscaleAction(File $file) } $fileService = GeneralUtility::makeInstance(FileService::class, $this->client->getFolderName()); - $fileService->saveImageFromUrl($upscaledImage->getUrl(), 'upscaled image', $file->getOriginalResource()->getNameWithoutExtension().'_upscaled'); + $fileService->saveFileFromUrl($upscaledImage->getUrl(), 'upscaled image', $file->getOriginalResource()->getNameWithoutExtension().'_upscaled'); $translatedMessage = LocalizationUtility::translate('mlang_label_upscaled_image_saved', 'mkcontentai') ?? ''; $this->addFlashMessage($translatedMessage, '', AbstractMessage::INFO); @@ -213,7 +169,7 @@ public function extendAction(string $direction, ?File $file = null, string $base $fileService = GeneralUtility::makeInstance(FileService::class, $this->client->getFolderName()); $filePath = $fileService->saveTempBase64Image($base64); } - if ($file) { + if ($file && '' === $filePath) { $filePath = $file->getOriginalResource()->getForLocalProcessing(false); } if ('' == $filePath) { @@ -246,7 +202,12 @@ public function cropAndExtendAction(File $file, ?string $promptText = ''): Respo $this->view->assignMultiple( [ + 'options' => $this->client->getAvailableResolutions($this->request->getControllerActionName()), 'file' => $file, + 'actionName' => 'extend', + 'operationName' => 'extend', + 'controllerName' => $this->request->getControllerName(), + 'withExtend' => true, 'promptText' => $promptText, 'clientApi' => substr(get_class($this->client), 28), ] @@ -259,11 +220,25 @@ public function saveFileAction(string $imageUrl, string $description = ''): Resp { $fileService = GeneralUtility::makeInstance(FileService::class, $this->client->getFolderName()); try { - $fileService->saveImageFromUrl($imageUrl, $description); + $fileService->saveFileFromUrl($imageUrl, $description); } catch (\Exception $e) { $this->addFlashMessage($e->getMessage(), '', AbstractMessage::ERROR); } return $this->redirect('filelist'); } + + protected function handleResponse(): ResponseInterface + { + if (null === $this->moduleTemplateFactory) { + $translatedMessage = LocalizationUtility::translate('labelErrorModuleTemplateFactory', 'mkcontentai') ?? ''; + + throw new \Exception($translatedMessage, 1623345720); + } + + $moduleTemplate = $this->moduleTemplateFactory->create($this->request); + $moduleTemplate->setContent($this->view->render()); + + return $this->htmlResponse($moduleTemplate->renderContent()); + } } diff --git a/Classes/Controller/AiVideoController.php b/Classes/Controller/AiVideoController.php new file mode 100644 index 0000000..f18aec1 --- /dev/null +++ b/Classes/Controller/AiVideoController.php @@ -0,0 +1,125 @@ + + * All rights reserved + * + * This file is part of TYPO3 CMS-based extension "mkcontentai" by DMK E-BUSINESS GmbH. + * + * It is free software; you can redistribute it and/or modify it under + * the terms of the GNU General Public License, either version 2 + * of the License, or any later version. + */ + +namespace DMK\MkContentAi\Controller; + +use DMK\MkContentAi\Service\FileService; +use Psr\Http\Message\ResponseInterface; +use TYPO3\CMS\Core\Messaging\AbstractMessage; +use TYPO3\CMS\Core\Utility\GeneralUtility; +use TYPO3\CMS\Extbase\Domain\Model\File; +use TYPO3\CMS\Extbase\Utility\LocalizationUtility; + +/** + * This file is part of the "MK Content AI" Extension for TYPO3 CMS. + * + * For the full copyright and license information, please read the + * LICENSE.txt file that was distributed with this source code. + * + * (c) 2023 + */ + +/** + * AiVideoController. + */ +class AiVideoController extends BaseController +{ + private FileService $fileService; + + public function __construct(FileService $fileService) + { + $this->fileService = $fileService; + } + + public function initializeAction(): void + { + $this->initializeAndAuthorizeAction(); + } + + /** + * @return ResponseInterface + */ + public function imageToVideoAction(File $file, string $base64) + { + $this->initializeAction(); + + try { + $filePath = $this->fileService->saveTempBase64Image($base64); + $generatedVideo = $this->client->imageToVideo($filePath); + } catch (\Exception $e) { + $this->addFlashMessage($e->getMessage(), '', AbstractMessage::ERROR); + + return $this->redirect('filelist', 'AiImage'); + } + $translatedMessage = LocalizationUtility::translate('mlang_label_image_to_video_generated', 'mkcontentai') ?? ''; + + $this->view->assignMultiple( + [ + 'croppedImage' => $base64, + 'sourceFile' => $file, + 'generatedVideo' => $generatedVideo, + 'clientApi' => substr(get_class($this->client), 28), + ] + ); + $this->addFlashMessage($translatedMessage, '', AbstractMessage::INFO); + + return $this->handleResponse(); + } + + public function prepareImageToVideoAction(File $file): ResponseInterface + { + $this->view->assignMultiple( + [ + 'options' => $this->client->getAvailableResolutions($this->request->getControllerActionName()), + 'file' => $file, + 'clientApi' => substr(get_class($this->client), 28), + 'actionName' => 'imageToVideo', + 'operationName' => $this->request->getControllerActionName(), + 'controllerName' => $this->request->getControllerName(), + 'withExtend' => false, + ] + ); + + return $this->handleResponse(); + } + + public function saveFileAction(File $sourceFile, string $videoUrl): ResponseInterface + { + $fileService = GeneralUtility::makeInstance(FileService::class, $this->client->getFolderName()); + try { + $fileService->saveFileFromUrl($videoUrl, $sourceFile->getOriginalResource()->getNameWithoutExtension().' - generated video', $sourceFile->getOriginalResource()->getNameWithoutExtension().'_generated_video', '.mp4'); + } catch (\Exception $e) { + $this->addFlashMessage($e->getMessage(), '', AbstractMessage::ERROR); + } + + return $this->redirect('filelist', 'AiImage'); + } + + protected function handleResponse(): ResponseInterface + { + if (null === $this->moduleTemplateFactory) { + $translatedMessage = LocalizationUtility::translate('labelErrorModuleTemplateFactory', 'mkcontentai') ?? ''; + + throw new \Exception($translatedMessage, 1623345720); + } + + $moduleTemplate = $this->moduleTemplateFactory->create($this->request); + $moduleTemplate->setContent($this->view->render()); + + return $this->htmlResponse($moduleTemplate->renderContent()); + } +} diff --git a/Classes/Controller/AjaxController.php b/Classes/Controller/AjaxController.php index 4cb3346..8f402dc 100644 --- a/Classes/Controller/AjaxController.php +++ b/Classes/Controller/AjaxController.php @@ -48,12 +48,15 @@ public function __construct() */ public function blobImage(ServerRequestInterface $request): ResponseInterface { - if (!isset($request->getParsedBody()['imageUrl'])) { + /** @var array $parsedBody */ + $parsedBody = $request->getParsedBody(); + $imageUrl = $parsedBody['imageUrl'] ?? null; + + if (!$imageUrl) { $translatedMessage = LocalizationUtility::translate('labelErrorMissingImageUrl', 'mkcontentai') ?? ''; throw new \Exception($translatedMessage); } - $imageUrl = $request->getParsedBody()['imageUrl']; $imageData = GeneralUtility::getUrl($imageUrl); if (!is_string($imageData)) { @@ -170,7 +173,11 @@ public function promptResultAjaxAction(ServerRequestInterface $request) } $client = $clientResponse['client']; - if (empty($request->getParsedBody()['promptText'])) { + /** @var array $parsedBody */ + $parsedBody = $request->getParsedBody(); + $text = $parsedBody['promptText'] ?? null; + + if (empty($text)) { $translatedMessage = LocalizationUtility::translate('labelErrorPromptText', 'mkcontentai') ?? ''; return new JsonResponse( @@ -179,7 +186,6 @@ public function promptResultAjaxAction(ServerRequestInterface $request) ], 500); } - $text = $request->getParsedBody()['promptText']; try { $images = $client->image($text); diff --git a/Classes/Controller/BaseController.php b/Classes/Controller/BaseController.php index 6a1caf5..5adb9a8 100644 --- a/Classes/Controller/BaseController.php +++ b/Classes/Controller/BaseController.php @@ -16,7 +16,11 @@ namespace DMK\MkContentAi\Controller; use DMK\MkContentAi\Http\Client\ImageApiInterface; +use DMK\MkContentAi\Http\Client\OpenAiClient; +use DMK\MkContentAi\Http\Client\StabilityAiClient; +use DMK\MkContentAi\Http\Client\StableDiffusionClient; use TYPO3\CMS\Backend\Template\ModuleTemplateFactory; +use TYPO3\CMS\Core\Messaging\AbstractMessage; use TYPO3\CMS\Core\Page\PageRenderer; use TYPO3\CMS\Core\Utility\GeneralUtility; use TYPO3\CMS\Core\Utility\PathUtility; @@ -25,6 +29,19 @@ class BaseController extends ActionController { + public ImageApiInterface $client; + + public const GENERATOR_ENGINE_KEY = 'image_generator_engine'; + + /** + * @var array> + */ + public const GENERATOR_ENGINE = [ + 1 => OpenAiClient::class, + 2 => StableDiffusionClient::class, + 3 => StabilityAiClient::class, + ]; + protected ?ModuleTemplateFactory $moduleTemplateFactory; public function injectModuleTemplateFactory(ModuleTemplateFactory $moduleTemplateFactory): void @@ -32,7 +49,7 @@ public function injectModuleTemplateFactory(ModuleTemplateFactory $moduleTemplat $this->moduleTemplateFactory = $moduleTemplateFactory; } - public function initializeAction(): void + protected function initializeAndAuthorizeAction(): void { $pageRenderer = GeneralUtility::makeInstance(PageRenderer::class); $pageRenderer->loadRequireJsModule('TYPO3/CMS/Mkcontentai/MkContentAi'); @@ -46,6 +63,37 @@ public function initializeAction(): void ], ] ); + + $client = $this->initializeClient(); + if (isset($client['error'])) { + $this->addFlashMessage( + $client['error'], + '', + AbstractMessage::ERROR + ); + + return; + } + if (isset($client['client'])) { + $this->client = $client['client']; + } + + $arguments['actionName'] = $this->request->getControllerActionName(); + if (!in_array($arguments['actionName'], $this->client->getAllowedOperations())) { + $translatedMessage = LocalizationUtility::translate('labelNotAllowed', 'mkcontentai', $arguments) ?? ''; + $this->addFlashMessage($translatedMessage.' '.get_class($this->client), '', AbstractMessage::ERROR); + $this->redirect('filelist'); + } + + $infoMessage = LocalizationUtility::translate('labelEngineInitialized', 'mkcontentai') ?? ''; + if (isset($client['clientClass'])) { + $infoMessage .= ' '.$client['clientClass']; + } + $this->addFlashMessage( + $infoMessage, + '', + AbstractMessage::INFO + ); } /** @@ -55,7 +103,7 @@ protected function initializeClient(): array { try { $imageEngineKey = SettingsController::getImageAiEngine(); - $client = GeneralUtility::makeInstance(AiImageController::GENERATOR_ENGINE[$imageEngineKey]); + $client = GeneralUtility::makeInstance($this::GENERATOR_ENGINE[$imageEngineKey]); if (is_a($client, ImageApiInterface::class)) { return [ 'client' => $client, diff --git a/Classes/Http/Client/Action/StabilityAiAction.php b/Classes/Http/Client/Action/StabilityAiAction.php index 4194c60..94549a3 100644 --- a/Classes/Http/Client/Action/StabilityAiAction.php +++ b/Classes/Http/Client/Action/StabilityAiAction.php @@ -29,6 +29,8 @@ public function getActions(): array 'account' => 'v1/user/account', 'upscale' => 'v1/generation/esrgan-v1-x2plus/image-to-image/upscale', 'extend' => 'v2beta/stable-image/edit/outpaint', + 'getVideo' => 'v2beta/image-to-video/result/%s', + 'imageToVideo' => 'v2beta/image-to-video', ]; } } diff --git a/Classes/Http/Client/BaseClient.php b/Classes/Http/Client/BaseClient.php index aa3e44d..de159ed 100644 --- a/Classes/Http/Client/BaseClient.php +++ b/Classes/Http/Client/BaseClient.php @@ -15,6 +15,7 @@ namespace DMK\MkContentAi\Http\Client; +use DMK\MkContentAi\Domain\Model\Image; use TYPO3\CMS\Core\Registry; use TYPO3\CMS\Core\Utility\GeneralUtility; use TYPO3\CMS\Extbase\Utility\LocalizationUtility; @@ -67,6 +68,23 @@ public function setApiKey(string $apiKey): void $registry->set($class, self::API_KEY, $apiKey); } + /** + * @return array> + */ + public function getAvailableResolutions(string $actionName): array + { + $actionName = ''; + + return []; + } + + public function imageToVideo(string $filePath): ?Image + { + $filePath = null; + + return $filePath; + } + protected function getRegistry(): Registry { return GeneralUtility::makeInstance(Registry::class); diff --git a/Classes/Http/Client/ImageApiInterface.php b/Classes/Http/Client/ImageApiInterface.php index e059d02..f65860c 100644 --- a/Classes/Http/Client/ImageApiInterface.php +++ b/Classes/Http/Client/ImageApiInterface.php @@ -43,4 +43,11 @@ public function getFolderName(): string; * @return array */ public function getAllowedOperations(): array; + + /** + * @return array> + */ + public function getAvailableResolutions(string $actionName): array; + + public function imageToVideo(string $filePath): ?Image; } diff --git a/Classes/Http/Client/OpenAiClient.php b/Classes/Http/Client/OpenAiClient.php index d00ff21..5bcbfb4 100644 --- a/Classes/Http/Client/OpenAiClient.php +++ b/Classes/Http/Client/OpenAiClient.php @@ -174,4 +174,25 @@ public function getAllowedOperations(): array { return ['cropAndExtend', 'extend', 'variants', 'filelist', 'saveFile', 'promptResult', 'prompt', 'promptResultAjax']; } + + /** + * @return array> + */ + public function getAvailableResolutions(string $actionName): array + { + if ('cropAndExtend' === $actionName) { + return [ + [ + 'width' => '256', + 'height' => '256', + ], + [ + 'width' => '512', + 'height' => '512', + ], + ]; + } + + return []; + } } diff --git a/Classes/Http/Client/StabilityAiClient.php b/Classes/Http/Client/StabilityAiClient.php index 6a8b4d3..775de1f 100644 --- a/Classes/Http/Client/StabilityAiClient.php +++ b/Classes/Http/Client/StabilityAiClient.php @@ -177,13 +177,72 @@ public function upscale(File $file): Image // if response is valid base64 encoded image if (200 === $response->getStatusCode()) { - $image = $this->base64ToImage(base64_encode($response->getContent(false))); + $image = $this->base64ToFile(base64_encode($response->getContent(false))); return $image; } throw new \Exception('Response code '.$response->getStatusCode()); } + public function imageToVideo(string $filePath): Image + { + $formData = new FormDataPart([ + 'image' => DataPart::fromPath($filePath), + 'cfg_scale' => '5', + 'motion_bucket_id' => '127', + ]); + $headers = $formData->getPreparedHeaders()->toArray(); + $headers['Authorization'] = $this->getAuthorizationHeader(); + + $response = $this->client->request( + 'POST', + $this->stabilityAiAction->buildFullUrl($this->stabilityAiAction::API_LINK, 'imageToVideo', []), + [ + 'headers' => $headers, + 'body' => $formData->bodyToIterable(), + ] + ); + + if (200 === $response->getStatusCode()) { + $response = $this->validateResponse($response->getContent(false)); + + $responseContent = $this->getGeneratedVideo($response->id); + + return $responseContent; + } + + throw new \Exception('Error - '.json_decode($response->getContent(false))->errors[0]); + } + + public function getGeneratedVideo(string $generatedVideoId): Image + { + $timer = 0; + + do { + $headers = [ + 'Authorization' => $this->getAuthorizationHeader(), + 'Accept' => 'application/json', + ]; + $response = $this->client->request( + 'GET', + $this->stabilityAiAction->buildFullUrl($this->stabilityAiAction::API_LINK, 'getVideo', [$generatedVideoId]), + [ + 'headers' => $headers, + ] + ); + + sleep(30); + ++$timer; + } while ('SUCCESS' !== $this->validateResponse($response->getContent(false))->finish_reason && $timer <= 6); + $response = $this->validateResponse($response->getContent(false)); + + if ('SUCCESS' !== $response->finish_reason) { + throw new \Exception('Error - '.$response->errors[0]); + } + + return $this->base64ToFile($response->video); + } + /** * @return array */ @@ -208,7 +267,7 @@ public function extend(string $sourceImagePath, string $direction = 'right', ?st if (200 === $response->getStatusCode()) { $response = $this->validateResponse($response->getContent(false)); - $images[] = $this->base64ToImage($response->image); + $images[] = $this->base64ToFile($response->image); return $images; } @@ -269,7 +328,7 @@ public function prepareFormDataRequest(string $direction, string $sourceImagePat ]); } - private function base64ToImage(string $base64): Image + private function base64ToFile(string $base64): Image { $binaryData = base64_decode($base64); $tempFile = GeneralUtility::tempnam('contentai'); @@ -287,7 +346,7 @@ private function responseToImages(\stdClass $response): array { $images = []; foreach ($response->artifacts as $image) { - $images[] = $this->base64ToImage($image->base64); + $images[] = $this->base64ToFile($image->base64); } return $images; @@ -295,6 +354,31 @@ private function responseToImages(\stdClass $response): array public function getAllowedOperations(): array { - return ['upscale', 'variants', 'filelist', 'saveFile', 'promptResult', 'prompt', 'promptResultAjax', 'extend', 'cropAndExtend']; + return ['upscale', 'variants', 'filelist', 'saveFile', 'promptResult', 'prompt', 'promptResultAjax', 'imageToVideo', 'extend', 'prepareImageToVideo', 'cropAndExtend']; + } + + /** + * @return array> + */ + public function getAvailableResolutions(string $actionName): array + { + if ('prepareImageToVideo' === $actionName) { + return [ + [ + 'width' => '1024', + 'height' => '576', + ], + [ + 'width' => '576', + 'height' => '1024', + ], + [ + 'width' => '768', + 'height' => '768', + ], + ]; + } + + return []; } } diff --git a/Classes/Http/Client/VideoApiInterface.php b/Classes/Http/Client/VideoApiInterface.php new file mode 100644 index 0000000..3d0f709 --- /dev/null +++ b/Classes/Http/Client/VideoApiInterface.php @@ -0,0 +1,23 @@ + + * All rights reserved + * + * This file is part of TYPO3 CMS-based extension "mkcontentai" by DMK E-BUSINESS GmbH. + * + * It is free software; you can redistribute it and/or modify it under + * the terms of the GNU General Public License, either version 2 + * of the License, or any later version. + */ + +namespace DMK\MkContentAi\Http\Client; + +use DMK\MkContentAi\Domain\Model\Image; + +interface VideoApiInterface extends ImageApiInterface +{ + public function imageToVideo(string $filePath): Image; +} diff --git a/Classes/Service/FileService.php b/Classes/Service/FileService.php index 9d8e83b..b8efd98 100644 --- a/Classes/Service/FileService.php +++ b/Classes/Service/FileService.php @@ -43,7 +43,7 @@ public function __construct(?string $folder = null) $this->resourceFactory = GeneralUtility::makeInstance(ResourceFactory::class); } - public function saveImageFromUrl(string $imageUrl, string $description = '', string $filename = ''): void + public function saveFileFromUrl(string $imageUrl, string $description = '', string $filename = '', string $extension = '.png'): void { $storage = $this->getStorage(); @@ -63,7 +63,7 @@ public function saveImageFromUrl(string $imageUrl, string $description = '', str $fileResponse ); - $filename = ($filename ?: time()).'.png'; + $filename = ($filename ?: time()).$extension; /** @var \TYPO3\CMS\Core\Resource\File $fileObject */ $fileObject = $storage->addFile( diff --git a/Configuration/Backend/Modules.php b/Configuration/Backend/Modules.php index 9485377..f074932 100644 --- a/Configuration/Backend/Modules.php +++ b/Configuration/Backend/Modules.php @@ -28,6 +28,9 @@ DMK\MkContentAi\Controller\AiImageController::class => [ 'filelist', 'variants', 'prompt', 'promptResult', 'saveFile', 'upscale', 'extend', 'cropAndExtend', ], + DMK\MkContentAi\Controller\AiVideoController::class => [ + 'imageToVideo', 'filelist', 'saveFile', 'prepareImageToVideo', + ], DMK\MkContentAi\Controller\SettingsController::class => [ 'settings', ], diff --git a/Resources/Private/Language/de.locallang.xlf b/Resources/Private/Language/de.locallang.xlf index 143b8e0..e099596 100644 --- a/Resources/Private/Language/de.locallang.xlf +++ b/Resources/Private/Language/de.locallang.xlf @@ -161,6 +161,10 @@ You don't have permissions to go Settings Page! Sie haben keine Berechtigung, die Seite Einstellungen aufzurufen! + + + Video has been generated + Das Video wurde erstellt You don't have permissions to go Image generation Page! diff --git a/Resources/Private/Language/de.locallang_contentai.xlf b/Resources/Private/Language/de.locallang_contentai.xlf index 9e075fa..fec2980 100644 --- a/Resources/Private/Language/de.locallang_contentai.xlf +++ b/Resources/Private/Language/de.locallang_contentai.xlf @@ -52,6 +52,10 @@ Crop and extend Beschneiden und erweitern + + + Crop image and generate video + Bild zuschneiden und Video erzeugen Size of crop area @@ -76,6 +80,10 @@ Variants generating Varianten erzeugen + + + Image to Video + Bild zu Video Variants generated @@ -200,6 +208,10 @@ Extend Erweitern + + + Image to Video + Bild zu Video Alt text generate @@ -224,6 +236,14 @@ Add a new content into existing image Einen neuen Inhalt in ein bestehendes Bild einfügen + + + Generated video + Erstelltes video + + + Save video + Video speichern diff --git a/Resources/Private/Language/locallang.xlf b/Resources/Private/Language/locallang.xlf index 1a09b63..5f74c87 100644 --- a/Resources/Private/Language/locallang.xlf +++ b/Resources/Private/Language/locallang.xlf @@ -134,6 +134,9 @@ You don't have permissions to go Settings Page! + + + Video has been generated You don't have permissions to go Image generation Page! diff --git a/Resources/Private/Language/locallang_contentai.xlf b/Resources/Private/Language/locallang_contentai.xlf index d7780aa..dfa3595 100644 --- a/Resources/Private/Language/locallang_contentai.xlf +++ b/Resources/Private/Language/locallang_contentai.xlf @@ -41,6 +41,9 @@ Crop and extend + + + Crop image and generate video Size of crop area @@ -59,6 +62,9 @@ Variants generating + + + Image to Video Original file @@ -152,6 +158,9 @@ Extend + + + Image to Video Alt text generate @@ -170,6 +179,12 @@ Prompt + + + Generated video + + + Save video diff --git a/Resources/Private/Partials/Crop.html b/Resources/Private/Partials/Crop.html new file mode 100644 index 0000000..c1ca919 --- /dev/null +++ b/Resources/Private/Partials/Crop.html @@ -0,0 +1,82 @@ + + + + +

+ + +

+ + + + +

+
+ +
+ + +
+ +
+ + + + + +

+ + + + + + + + + +
+ +
+ + +
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+

+ +
+ + diff --git a/Resources/Private/Templates/AiImage/CropAndExtend.html b/Resources/Private/Templates/AiImage/CropAndExtend.html index 3661e76..bae9d6e 100644 --- a/Resources/Private/Templates/AiImage/CropAndExtend.html +++ b/Resources/Private/Templates/AiImage/CropAndExtend.html @@ -6,71 +6,8 @@ -

- -

-
-
- - -
-
- - -
-
- - -

-
- - -
-
- - -
-
- - -
-
- - -
-
- - -
- - -
-

- -
- - - - - - -
- -
+ diff --git a/Resources/Private/Templates/AiImage/Filelist.html b/Resources/Private/Templates/AiImage/Filelist.html index d229da7..f884812 100644 --- a/Resources/Private/Templates/AiImage/Filelist.html +++ b/Resources/Private/Templates/AiImage/Filelist.html @@ -13,7 +13,7 @@

- +
@@ -22,8 +22,11 @@

{file.properties.description}

+

{file.properties.width}w x {file.properties.height}h

+

{file.name}

+

@@ -66,6 +69,21 @@

{file.properties.description}

+

+ + + + + + + + + + +

+

diff --git a/Resources/Private/Templates/AiVideo/ImageToVideo.html b/Resources/Private/Templates/AiVideo/ImageToVideo.html new file mode 100644 index 0000000..18ccbbf --- /dev/null +++ b/Resources/Private/Templates/AiVideo/ImageToVideo.html @@ -0,0 +1,47 @@ + +{namespace dmk=DMK\MkContentAi\ViewHelpers} + + + + + +

+ +

+ + + +
+
+
+

+ +

+ + + + + + + + +
+

+ +

+
+ + + + + + + + diff --git a/Resources/Private/Templates/AiVideo/PrepareImageToVideo.html b/Resources/Private/Templates/AiVideo/PrepareImageToVideo.html new file mode 100644 index 0000000..bae9d6e --- /dev/null +++ b/Resources/Private/Templates/AiVideo/PrepareImageToVideo.html @@ -0,0 +1,13 @@ + + + + + + + + + diff --git a/Resources/Public/JavaScript/ContextMenu.js b/Resources/Public/JavaScript/ContextMenu.js index 7eb8751..05ab6ce 100644 --- a/Resources/Public/JavaScript/ContextMenu.js +++ b/Resources/Public/JavaScript/ContextMenu.js @@ -44,5 +44,13 @@ define(function () { top.TYPO3.Backend.ContentContainer.setUrl($(this).attr('data-navigate-uri')); }; + /** + * @param {string} table + * @param {int} uid of the page + */ + ContextMenu.prepareImageToVideo = function (table, uid) { + top.TYPO3.Backend.ContentContainer.setUrl($(this).attr('data-navigate-uri')); + }; + return ContextMenu; }); diff --git a/Resources/Public/JavaScript/MkContentAi.js b/Resources/Public/JavaScript/MkContentAi.js index d537ecf..d62d1fe 100644 --- a/Resources/Public/JavaScript/MkContentAi.js +++ b/Resources/Public/JavaScript/MkContentAi.js @@ -21,28 +21,63 @@ define(['jquery', 'cropper'], function ($, Cropper) { }); const image = document.getElementById('image'); + const action = document.getElementById('operationName').value; const client = document.getElementById("clientApi").value; + const submitButton = document.querySelector('input[type="submit"]'); let customHeight = 256; let customWidth = 256; let dragModeValue = 'none'; let viewModeValue = 1; let cropBoxResizable = false; - let aspectRatio = 1 / 1; + let aspectRatio = 1; + let scalable = false; + let autoCropArea = 0; + let zoomable = false; + submitButton.disabled = false; - if (client === "StabilityAiClient") { + if (client === "StabilityAiClient" && action === 'extend') { customHeight = image.height; customWidth = image.width; dragModeValue = 'crop'; - viewModeValue = 1; cropBoxResizable = true; aspectRatio = NaN; } + + if (client === "StabilityAiClient" && action === 'prepareImageToVideo') { + aspectRatio = NaN; + customWidth = document.querySelector('input[name="size"]').getAttribute('data-width') ?? 768; + customHeight = document.querySelector('input[name="size"]').getAttribute('data-height') ?? 768; + + const inputs = document.querySelectorAll('input[name="size"]'); + let disabledInputsCount = 0; + + for (let i = 0; element = inputs[i]; i++) { + element.disabled = false; + + if (false === validateImageDimensionsData(image.width, image.height, element.getAttribute('data-width'), element.getAttribute('data-height'))) { + element.disabled = true; + customWidth = 0; + customHeight = 0; + disabledInputsCount++; + } + } + + if (inputs.length === disabledInputsCount) { + submitButton.disabled = true; + } + + customWidth = parseInt(customWidth, 10); + customHeight = parseInt(customHeight, 10); + } + const cropper = new Cropper(image, { aspectRatio: aspectRatio, cropBoxResizable: cropBoxResizable, dragMode: dragModeValue, - zoomable: false, + zoomable: zoomable, viewMode: viewModeValue, + scalable: scalable, + autoCropArea: autoCropArea, data: { width: customWidth, height: customHeight @@ -53,32 +88,38 @@ define(['jquery', 'cropper'], function ($, Cropper) { let naturalWidth = this.cropper.getImageData().naturalWidth; let naturalHeight = this.cropper.getImageData().naturalHeight; if (naturalHeight >= 256 && naturalWidth >= 256) { - this.cropper.setCanvasData({ - left: 0, - top: 0, - width: naturalWidth, - height: naturalHeight - }); - } + this.cropper.setCanvasData({ + left: 0, + top: 0, + width: naturalWidth, + height: naturalHeight + }); + } + if (document.querySelector('input[name="size"]')) { + document.querySelector('input[name="size"]').click(); + } } }); - $('#extend').on('submit', function(event) { + $('.crop-form').on('submit', function(event) { event.preventDefault(); - const client = document.getElementById("clientApi").value; - let minWidthAndHeight; - if (client === "StabilityAiClient") { - minWidthAndHeight = image.height; - } else { - minWidthAndHeight = document.querySelector('input[name="size"]:checked').getAttribute('data-width') ?? 256; + let minHeight; + let minWidth; + + if (document.querySelector('input[name="size"]:checked')) { + minWidth = document.querySelector('input[name="size"]:checked').getAttribute('data-width') ?? 256; + minHeight = document.querySelector('input[name="size"]:checked').getAttribute('data-height') ?? 256; + minWidth = parseInt(minWidth, 10); + minHeight = parseInt(minHeight, 10); } - minWidthAndHeight = parseInt(minWidthAndHeight, 10); + let canvas = cropper.getCroppedCanvas({ - minWidth: minWidthAndHeight, - minHeight: minWidthAndHeight + minWidth: minWidth, + minHeight: minHeight } ); + let croppedImageSrc = canvas.toDataURL('image/png'); document.getElementById('croppedImage').src = croppedImageSrc; document.getElementById('CroppedBase64').value = croppedImageSrc; @@ -98,5 +139,11 @@ define(['jquery', 'cropper'], function ($, Cropper) { } } }); + + function validateImageDimensionsData(imageWidth, imageHeight, inputWidth, inputHeight) { + if (imageWidth < inputWidth || imageHeight < inputHeight) { + return false; + } + } }); }); diff --git a/Resources/Public/JavaScript/context-menu-actions.js b/Resources/Public/JavaScript/context-menu-actions.js index 1836de9..b02595f 100644 --- a/Resources/Public/JavaScript/context-menu-actions.js +++ b/Resources/Public/JavaScript/context-menu-actions.js @@ -19,6 +19,10 @@ class ContextMenuActions { altTexts(table, uid, data) { top.TYPO3.Backend.ContentContainer.setUrl(data.navigateUri); }; + + prepareImageToVideo(table, uid, data) { + top.TYPO3.Backend.ContentContainer.setUrl(data.navigateUri); + }; } export default new ContextMenuActions(); diff --git a/ext_tables.php b/ext_tables.php index 73ef10b..50166dd 100644 --- a/ext_tables.php +++ b/ext_tables.php @@ -22,9 +22,10 @@ 'contentai', '', [ - DMK\MkContentAi\Controller\AiImageController::class => 'filelist, variants, prompt, promptResult, saveFile, upscale, extend, cropAndExtend', + DMK\MkContentAi\Controller\AiImageController::class => 'filelist, variants, prompt, promptResult, saveFile, upscale, extend, cropAndExtend, crop', DMK\MkContentAi\Controller\SettingsController::class => 'settings', - DMK\MkContentAi\Controller\AiTextController::class => 'altText, altTextSave, altTexts, altTextsSave', + DMK\MkContentAi\Controller\AiTextController::class => 'altText, altTextSave, altTexts, altTextsSave, filelist', + DMK\MkContentAi\Controller\AiVideoController::class => 'prepareImageToVideo, imageToVideo, filelist, saveFile', ], [ 'access' => 'user,group',