From a2bd5f86b74fd36ced2061cc1c20ce78e9ee2807 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Cygankiewicz?= Date: Mon, 6 May 2024 10:04:07 +0200 Subject: [PATCH] Extending image in StabilityAI, prompt field in extending --- Classes/Controller/AiImageController.php | 17 +++- Classes/Http/Client/Action/AltTextAction.php | 32 +++++++ Classes/Http/Client/Action/BaseAction.php | 48 ++++++++++ .../Http/Client/Action/StabilityAiAction.php | 34 +++++++ .../Client/Action/StableDiffusionAction.php | 34 +++++++ Classes/Http/Client/AltTextClient.php | 12 ++- Classes/Http/Client/ImageApiInterface.php | 2 +- Classes/Http/Client/OpenAiClient.php | 4 +- Classes/Http/Client/StabilityAiClient.php | 90 ++++++++++++++----- Classes/Http/Client/StableDiffusionClient.php | 31 ++++--- Configuration/Services.yaml | 17 ++++ .../Language/de.locallang_contentai.xlf | 4 + .../Private/Language/locallang_contentai.xlf | 3 + .../Templates/AiImage/CropAndExtend.html | 14 +-- .../Private/Templates/AiImage/Filelist.html | 35 +------- Resources/Public/JavaScript/MkContentAi.js | 37 ++++++-- 16 files changed, 323 insertions(+), 91 deletions(-) create mode 100644 Classes/Http/Client/Action/AltTextAction.php create mode 100644 Classes/Http/Client/Action/BaseAction.php create mode 100644 Classes/Http/Client/Action/StabilityAiAction.php create mode 100644 Classes/Http/Client/Action/StableDiffusionAction.php diff --git a/Classes/Controller/AiImageController.php b/Classes/Controller/AiImageController.php index 42fa9b7..ebe2ad0 100644 --- a/Classes/Controller/AiImageController.php +++ b/Classes/Controller/AiImageController.php @@ -201,8 +201,12 @@ public function upscaleAction(File $file) /** * @return ResponseInterface */ - public function extendAction(string $direction, ?File $file = null, string $base64 = '') + public function extendAction(string $direction, ?File $file = null, string $base64 = '', ?string $promptText = '') { + if (!isset($promptText) || '' === $promptText) { + $promptText = 'extend image content'; + } + try { $filePath = ''; if ($base64) { @@ -217,7 +221,7 @@ public function extendAction(string $direction, ?File $file = null, string $base throw new \Exception($translatedMessage, 1623345720); } - $images = $this->client->extend($filePath, $direction); + $images = $this->client->extend($filePath, $direction, $promptText); } catch (\Exception $e) { $this->addFlashMessage($e->getMessage(), '', AbstractMessage::ERROR); $this->redirect('filelist'); @@ -227,17 +231,24 @@ public function extendAction(string $direction, ?File $file = null, string $base [ 'images' => $images, 'originalFile' => $file, + 'promptText' => $promptText, ] ); return $this->handleResponse(); } - public function cropAndExtendAction(File $file): ResponseInterface + public function cropAndExtendAction(File $file, ?string $promptText = ''): ResponseInterface { + if (!isset($promptText) || '' === $promptText) { + $promptText = 'extending content image'; + } + $this->view->assignMultiple( [ 'file' => $file, + 'promptText' => $promptText, + 'clientApi' => substr(get_class($this->client), 28), ] ); diff --git a/Classes/Http/Client/Action/AltTextAction.php b/Classes/Http/Client/Action/AltTextAction.php new file mode 100644 index 0000000..7294f64 --- /dev/null +++ b/Classes/Http/Client/Action/AltTextAction.php @@ -0,0 +1,32 @@ + + * All rights reserved + * + * This file is part of TYPO3 CMS-based extension "mkcontentai" by DMK E-BUSINESS GmbH. + * + * It is free software; you can redistribute it and/or modify it under + * the terms of the GNU General Public License, either version 2 + * of the License, or any later version. + */ + +namespace DMK\MkContentAi\Http\Client\Action; + +class AltTextAction extends BaseAction +{ + public const API_LINK = 'https://alttext.ai/api/'; + + public function getActions(): array + { + return [ + 'altText' => 'v1/images', + 'altTextByAssetId' => 'v1/images/%s', + 'account' => 'v1/account', + ]; + } +} diff --git a/Classes/Http/Client/Action/BaseAction.php b/Classes/Http/Client/Action/BaseAction.php new file mode 100644 index 0000000..f631797 --- /dev/null +++ b/Classes/Http/Client/Action/BaseAction.php @@ -0,0 +1,48 @@ + + * All rights reserved + * + * This file is part of TYPO3 CMS-based extension "mkcontentai" by DMK E-BUSINESS GmbH. + * + * It is free software; you can redistribute it and/or modify it under + * the terms of the GNU General Public License, either version 2 + * of the License, or any later version. + */ + +namespace DMK\MkContentAi\Http\Client\Action; + +abstract class BaseAction +{ + /** + * @return array + */ + abstract public function getActions(): array; + + /** + * @param string[] $params + */ + public function buildUrl(string $actionName, array $params = []): string + { + if (!array_key_exists($actionName, $this->getActions())) { + throw new \Exception(sprintf('Action with name %s not found', $actionName)); + } + + $url = $this->getActions()[$actionName]; + + return vsprintf($url, $params); + } + + /** + * @param string[] $params + */ + public function buildFullUrl(string $apiLink, string $actionName, array $params = []): string + { + return $apiLink.$this->buildUrl($actionName, $params); + } +} diff --git a/Classes/Http/Client/Action/StabilityAiAction.php b/Classes/Http/Client/Action/StabilityAiAction.php new file mode 100644 index 0000000..4194c60 --- /dev/null +++ b/Classes/Http/Client/Action/StabilityAiAction.php @@ -0,0 +1,34 @@ + + * All rights reserved + * + * This file is part of TYPO3 CMS-based extension "mkcontentai" by DMK E-BUSINESS GmbH. + * + * It is free software; you can redistribute it and/or modify it under + * the terms of the GNU General Public License, either version 2 + * of the License, or any later version. + */ + +namespace DMK\MkContentAi\Http\Client\Action; + +class StabilityAiAction extends BaseAction +{ + public const API_LINK = 'https://api.stability.ai/'; + + public function getActions(): array + { + return [ + 'image' => 'v1/generation/stable-diffusion-xl-1024-v1-0/text-to-image', + 'imageVariation' => 'v1/generation/stable-diffusion-xl-1024-v1-0/image-to-image', + 'account' => 'v1/user/account', + 'upscale' => 'v1/generation/esrgan-v1-x2plus/image-to-image/upscale', + 'extend' => 'v2beta/stable-image/edit/outpaint', + ]; + } +} diff --git a/Classes/Http/Client/Action/StableDiffusionAction.php b/Classes/Http/Client/Action/StableDiffusionAction.php new file mode 100644 index 0000000..818706b --- /dev/null +++ b/Classes/Http/Client/Action/StableDiffusionAction.php @@ -0,0 +1,34 @@ + + * All rights reserved + * + * This file is part of TYPO3 CMS-based extension "mkcontentai" by DMK E-BUSINESS GmbH. + * + * It is free software; you can redistribute it and/or modify it under + * the terms of the GNU General Public License, either version 2 + * of the License, or any later version. + */ + +namespace DMK\MkContentAi\Http\Client\Action; + +class StableDiffusionAction extends BaseAction +{ + public const API_LINK = 'https://stablediffusionapi.com/api/v3/'; + public const DREAMBOOTH_API_LINK = 'https://stablediffusionapi.com/api/v4/dreambooth/'; + + public function getActions(): array + { + return [ + 'system_load' => 'system_load', + 'img2img' => 'img2img', + 'text2img' => 'text2img', + 'model_list' => 'model_list', + ]; + } +} diff --git a/Classes/Http/Client/AltTextClient.php b/Classes/Http/Client/AltTextClient.php index a18e787..1f6ee73 100644 --- a/Classes/Http/Client/AltTextClient.php +++ b/Classes/Http/Client/AltTextClient.php @@ -17,6 +17,7 @@ namespace DMK\MkContentAi\Http\Client; +use DMK\MkContentAi\Http\Client\Action\AltTextAction; use DMK\MkContentAi\Service\SiteLanguageService; use DMK\MkContentAi\Utility\AiUtility; use Symfony\Component\HttpClient\HttpClient; @@ -30,11 +31,14 @@ class AltTextClient extends BaseClient implements ClientInterface { private SiteLanguageService $siteLanguageService; + private AltTextAction $altTextAction; + private HttpClientInterface $client; - public function __construct(SiteLanguageService $siteLanguageService) + public function __construct(SiteLanguageService $siteLanguageService, AltTextAction $altTextAction) { $this->siteLanguageService = $siteLanguageService; + $this->altTextAction = $altTextAction; $this->client = HttpClient::create(); } @@ -75,7 +79,7 @@ public function getAltTextForFile(File $file, ?string $languageIsoCode = null): $headers = array_merge($this->getAuthorizationHeader(), $formData->getPreparedHeaders()->toArray()); - $response = $this->client->request('POST', 'https://alttext.ai/api/v1/images', [ + $response = $this->client->request('POST', $this->altTextAction->buildFullUrl($this->altTextAction::API_LINK, 'altText', []), [ 'headers' => $headers, 'body' => $formData->bodyToIterable(), ]); @@ -108,7 +112,7 @@ public function getByAssetId(int $assetId, ?string $languageIsoCode = null): str $assetIdWithLangIsoCode = AiUtility::getAiAssetId($assetId, $languageIsoCode); - $response = $this->client->request('GET', 'https://alttext.ai/api/v1/images/'.$assetIdWithLangIsoCode, [ + $response = $this->client->request('GET', $this->altTextAction->buildFullUrl($this->altTextAction::API_LINK, 'altTextByAssetId', [$assetIdWithLangIsoCode]), [ 'headers' => $this->getAuthorizationHeader(), ]); @@ -119,7 +123,7 @@ public function getByAssetId(int $assetId, ?string $languageIsoCode = null): str public function getAccount(): \stdClass { - $response = $this->client->request('GET', 'https://alttext.ai/api/v1/account', [ + $response = $this->client->request('GET', $this->altTextAction->buildFullUrl($this->altTextAction::API_LINK, 'account', []), [ 'headers' => $this->getAuthorizationHeader(), ]); diff --git a/Classes/Http/Client/ImageApiInterface.php b/Classes/Http/Client/ImageApiInterface.php index 0224d43..e059d02 100644 --- a/Classes/Http/Client/ImageApiInterface.php +++ b/Classes/Http/Client/ImageApiInterface.php @@ -35,7 +35,7 @@ public function upscale(File $file): Image; /** * @return array */ - public function extend(string $sourceImagePath, string $direction): array; + public function extend(string $sourceImagePath, string $direction, ?string $promptText): array; public function getFolderName(): string; diff --git a/Classes/Http/Client/OpenAiClient.php b/Classes/Http/Client/OpenAiClient.php index 68e2698..d00ff21 100644 --- a/Classes/Http/Client/OpenAiClient.php +++ b/Classes/Http/Client/OpenAiClient.php @@ -83,7 +83,7 @@ public function upscale(File $file): Image /** * @return array */ - public function extend(string $sourceImagePath, string $direction = 'right'): array + public function extend(string $sourceImagePath, string $direction = 'right', ?string $promptText = ''): array { $extendService = GeneralUtility::makeInstance(ExtendService::class); @@ -98,7 +98,7 @@ public function extend(string $sourceImagePath, string $direction = 'right'): ar $array = [ 'image' => curl_file_create($maskImage, 'r'), 'mask' => curl_file_create($maskImage, 'r'), - 'prompt' => 'outpaint', + 'prompt' => $promptText, 'n' => 1, 'size' => $resolutionForExtended['width'].'x'.$resolutionForExtended['height'], ]; diff --git a/Classes/Http/Client/StabilityAiClient.php b/Classes/Http/Client/StabilityAiClient.php index 8f8042e..6a8b4d3 100644 --- a/Classes/Http/Client/StabilityAiClient.php +++ b/Classes/Http/Client/StabilityAiClient.php @@ -16,6 +16,7 @@ namespace DMK\MkContentAi\Http\Client; use DMK\MkContentAi\Domain\Model\Image; +use DMK\MkContentAi\Http\Client\Action\StabilityAiAction; use Symfony\Component\HttpClient\HttpClient; use Symfony\Component\Mime\Part\DataPart; use Symfony\Component\Mime\Part\Multipart\FormDataPart; @@ -25,18 +26,23 @@ class StabilityAiClient extends BaseClient implements ImageApiInterface { - private const API_LINK = 'https://api.stability.ai/'; - /** * @var \Symfony\Contracts\HttpClient\HttpClientInterface */ private $client; + private StabilityAiAction $stabilityAiAction; + public function __construct() { $this->client = HttpClient::create(); } + public function injectStabilityAction(StabilityAiAction $stabilityAiAction): void + { + $this->stabilityAiAction = $stabilityAiAction; + } + public function validateApiCall(): \stdClass { $headers = [ @@ -46,7 +52,7 @@ public function validateApiCall(): \stdClass $response = $this->client->request( 'GET', - $this->getEndpointLink('v1/user/account'), + $this->stabilityAiAction->buildFullUrl($this->stabilityAiAction::API_LINK, 'account', []), [ 'headers' => $headers, ] @@ -62,16 +68,6 @@ private function getAuthorizationHeader(): string return 'Bearer '.$this->getApiKey(); } - private function getEndpointLink(string $path): string - { - return self::API_LINK.$path; - } - - public function getApiLink(): string - { - return self::API_LINK; - } - public function getFolderName(): string { return 'stability_ai'; @@ -103,7 +99,7 @@ public function image(string $text): array $response = $this->client->request( 'POST', - $this->getEndpointLink('v1/generation/stable-diffusion-xl-1024-v1-0/text-to-image'), + $this->stabilityAiAction->buildFullUrl($this->stabilityAiAction::API_LINK, 'image', []), [ 'headers' => $headers, 'body' => json_encode($params), @@ -140,7 +136,7 @@ public function createImageVariation(File $file): array $response = $this->client->request( 'POST', - $this->getEndpointLink('v1/generation/stable-diffusion-xl-1024-v1-0/image-to-image'), + $this->stabilityAiAction->buildFullUrl($this->stabilityAiAction::API_LINK, 'imageVariation', []), [ 'headers' => $headers, 'body' => $formData->bodyToIterable(), @@ -172,7 +168,7 @@ public function upscale(File $file): Image $response = $this->client->request( 'POST', - $this->getEndpointLink('v1/generation/esrgan-v1-x2plus/image-to-image/upscale'), + $this->stabilityAiAction->buildFullUrl($this->stabilityAiAction::API_LINK, 'upscale', []), [ 'headers' => $headers, 'body' => $formData->bodyToIterable(), @@ -191,11 +187,33 @@ public function upscale(File $file): Image /** * @return array */ - public function extend(string $sourceImage, string $text = 'outpaint'): array + public function extend(string $sourceImagePath, string $direction = 'right', ?string $promptText = ''): array { - $translatedMessage = LocalizationUtility::translate('labelErrorNotImplemented', 'mkcontentai') ?? ''; + $promptText = $promptText ?? ''; + $formData = $this->prepareFormDataRequest($direction, $sourceImagePath, $promptText); + + $headers = $formData->getPreparedHeaders()->toArray() + [ + 'accept' => 'application/json', + 'Authorization' => $this->getAuthorizationHeader(), + ]; - throw new \Exception($translatedMessage); + $response = $this->client->request( + 'POST', + $this->stabilityAiAction->buildFullUrl($this->stabilityAiAction::API_LINK, 'extend', []), + [ + 'headers' => $headers, + 'body' => $formData->bodyToIterable(), + ] + ); + + if (200 === $response->getStatusCode()) { + $response = $this->validateResponse($response->getContent(false)); + $images[] = $this->base64ToImage($response->image); + + return $images; + } + + throw new \Exception('Response code '.$response->getStatusCode()); } /** @@ -219,6 +237,38 @@ public function validateResponse($response): \stdClass return $response; } + public function setDirection(string $direction): string + { + switch ($direction) { + case 'bottom': + return 'down'; + case 'top': + return 'up'; + } + + return $direction; + } + + public function prepareFormDataRequest(string $direction, string $sourceImagePath, string $promptText): FormDataPart + { + if ('zoomOut' === $direction) { + return new FormDataPart([ + 'image' => DataPart::fromPath($sourceImagePath), + 'left' => '512', + 'right' => '512', + 'up' => '512', + 'down' => '512', + 'prompt' => $promptText, + ]); + } + + return new FormDataPart([ + 'image' => DataPart::fromPath($sourceImagePath), + $this->setDirection($direction) => '512', + 'prompt' => $promptText, + ]); + } + private function base64ToImage(string $base64): Image { $binaryData = base64_decode($base64); @@ -245,6 +295,6 @@ private function responseToImages(\stdClass $response): array public function getAllowedOperations(): array { - return ['upscale', 'variants', 'filelist', 'saveFile', 'promptResult', 'prompt', 'promptResultAjax']; + return ['upscale', 'variants', 'filelist', 'saveFile', 'promptResult', 'prompt', 'promptResultAjax', 'extend', 'cropAndExtend']; } } diff --git a/Classes/Http/Client/StableDiffusionClient.php b/Classes/Http/Client/StableDiffusionClient.php index 2c3d941..325b774 100644 --- a/Classes/Http/Client/StableDiffusionClient.php +++ b/Classes/Http/Client/StableDiffusionClient.php @@ -16,6 +16,7 @@ namespace DMK\MkContentAi\Http\Client; use DMK\MkContentAi\Domain\Model\Image; +use DMK\MkContentAi\Http\Client\Action\StableDiffusionAction; use Symfony\Component\HttpClient\HttpClient; use Symfony\Contracts\HttpClient\ResponseInterface; use TYPO3\CMS\Core\Site\SiteFinder; @@ -25,18 +26,21 @@ class StableDiffusionClient extends BaseClient implements ImageApiInterface { - private const API_LINK = 'https://stablediffusionapi.com/api/v3/'; - - private const DREAMBOOTH_API_LINK = 'https://stablediffusionapi.com/api/v4/dreambooth/'; + private StableDiffusionAction $stableDiffusionAction; public function __construct() { $this->getApiKey(); } + public function injectStableDiffusionAction(StableDiffusionAction $stableDiffusionAction): void + { + $this->stableDiffusionAction = $stableDiffusionAction; + } + public function validateApiCall(): \stdClass { - $response = $this->request('system_load'); + $response = $this->request($this->stableDiffusionAction->getActions()['system_load']); $response = $this->validateResponse($response->getContent()); @@ -119,7 +123,7 @@ private function convertToStdClass(array $array): \stdClass */ private function request(string $endpoint, array $queryParams = [], string $apiLinkAdjust = ''): ResponseInterface { - $apiLink = self::getApiLink(); + $apiLink = $this->stableDiffusionAction::API_LINK; if ('' != $apiLinkAdjust) { $apiLink = $apiLinkAdjust; } @@ -176,7 +180,7 @@ private function stableDiffusionVariant(string $imageUrl): array 'track_id' => null, ]; - $response = $this->request('img2img', $params); + $response = $this->request($this->stableDiffusionAction->getActions()['img2img'], $params); $response = $this->validateResponse($response->getContent()); @@ -205,7 +209,7 @@ private function dreamboothVariant(string $imageUrl): array 'scheduler' => 'UniPCMultistepScheduler', ]; - $response = $this->request('img2img', $params, self::DREAMBOOTH_API_LINK); + $response = $this->request($this->stableDiffusionAction->getActions()['img2img'], $params, $this->stableDiffusionAction::DREAMBOOTH_API_LINK); $response = $this->validateResponse($response->getContent()); @@ -233,7 +237,7 @@ public function upscale(File $file): Image /** * @return array */ - public function extend(string $sourceImage, string $text = 'Add car'): array + public function extend(string $sourceImage, string $text = '', ?string $promptText = ''): array { $translatedMessage = LocalizationUtility::translate('labelErrorNotImplemented', 'mkcontentai') ?? ''; @@ -257,7 +261,7 @@ private function dreamboothImage(string $text): array 'track_id' => null, 'model_id' => $this->getCurrentModel(), ]; - $response = $this->request('', $params, self::DREAMBOOTH_API_LINK); + $response = $this->request('', $params, $this->stableDiffusionAction::DREAMBOOTH_API_LINK); $response = $this->validateResponse($response->getContent()); @@ -282,7 +286,7 @@ private function stableDiffusionImage(string $text): array 'webhook' => null, 'track_id' => null, ]; - $response = $this->request('text2img', $params); + $response = $this->request($this->stableDiffusionAction->getActions()['text2img'], $params); $response = $this->validateResponse($response->getContent()); @@ -314,7 +318,7 @@ public function getFolderName(): string */ public function modelList(): array { - $response = $this->request('model_list', [], 'https://stablediffusionapi.com/api/v4/dreambooth/'); + $response = $this->request($this->stableDiffusionAction->getActions()['model_list'], [], $this->stableDiffusionAction::DREAMBOOTH_API_LINK); $response = $this->validateResponse($response->getContent()); @@ -325,11 +329,6 @@ public function modelList(): array return []; } - public function getApiLink(): string - { - return self::API_LINK; - } - public function setCurrentModel(string $modelName): void { $registry = $this->getRegistry(); diff --git a/Configuration/Services.yaml b/Configuration/Services.yaml index 9b3c4e1..3fd39c3 100644 --- a/Configuration/Services.yaml +++ b/Configuration/Services.yaml @@ -55,3 +55,20 @@ services: DMK\MkContentAi\Service\AiAltTextService: public: true + + DMK\MkContentAi\Http\Client\Action\AltTextAction: + public: true + + DMK\MkContentAi\Http\Client\StabilityAiClient: + calls: + - method: 'injectStabilityAction' + arguments: + $stabilityAiAction: '@DMK\MkContentAi\Http\Client\Action\StabilityAiAction' + public: true + + DMK\MkContentAi\Http\Client\StableDiffusionClient: + calls: + - method: 'injectStableDiffusionAction' + arguments: + $stableDiffusionAction: '@DMK\MkContentAi\Http\Client\Action\StableDiffusionAction' + public: true diff --git a/Resources/Private/Language/de.locallang_contentai.xlf b/Resources/Private/Language/de.locallang_contentai.xlf index d0727b5..9e075fa 100644 --- a/Resources/Private/Language/de.locallang_contentai.xlf +++ b/Resources/Private/Language/de.locallang_contentai.xlf @@ -220,6 +220,10 @@ Enable image generation prompt in tt_content:media field Aufforderung zur Bilderzeugung im Feld tt_content:media aktivieren + + + Add a new content into existing image + Einen neuen Inhalt in ein bestehendes Bild einfügen diff --git a/Resources/Private/Language/locallang_contentai.xlf b/Resources/Private/Language/locallang_contentai.xlf index 2fb16f5..d7780aa 100644 --- a/Resources/Private/Language/locallang_contentai.xlf +++ b/Resources/Private/Language/locallang_contentai.xlf @@ -167,6 +167,9 @@ Enable image generation prompt in tt_content:media field + + + Prompt diff --git a/Resources/Private/Templates/AiImage/CropAndExtend.html b/Resources/Private/Templates/AiImage/CropAndExtend.html index e1b6c12..3661e76 100644 --- a/Resources/Private/Templates/AiImage/CropAndExtend.html +++ b/Resources/Private/Templates/AiImage/CropAndExtend.html @@ -7,7 +7,7 @@

- +

@@ -23,11 +23,9 @@

- + - +

- + +
+

+ +
diff --git a/Resources/Private/Templates/AiImage/Filelist.html b/Resources/Private/Templates/AiImage/Filelist.html index f4d6c0a..d229da7 100644 --- a/Resources/Private/Templates/AiImage/Filelist.html +++ b/Resources/Private/Templates/AiImage/Filelist.html @@ -55,38 +55,9 @@
{file.properties.description}

-

- -
    -
  • - - - -
    • -
  • - - - -
    • -
  • - - - -
    • -
  • - - - -
    • -
  • - - - -
    • -
-
+ + +