Link caching (#15)

* Controller blacklist-validation

* Revert "Controller blacklist-validation"

This reverts commit d58cd0d.

* cache implementation

* avoid DNS translation during cached response

* default settings extender

* Formatting

Co-authored-by: Oliver Zieschang <o.zieschang@exonn.de>

extend.php

@@ -12,6 +12,7 @@
 namespace Datlechin\LinkPreview;
 
 use Flarum\Extend;
+use Flarum\Settings\Event\Deserializing;
 
 return [
     (new Extend\Frontend('forum'))
@@ -29,6 +30,18 @@
     (new Extend\Settings())
         ->serializeToForum('datlechin-link-preview.blacklist', 'datlechin-link-preview.blacklist')
         ->serializeToForum('datlechin-link-preview.whitelist', 'datlechin-link-preview.whitelist')
-        ->serializeToForum('datlechin-link-preview.useGoogleFavicons', 'datlechin-link-preview.use_google_favicons', 'boolval', true)
-        ->serializeToForum('datlechin-link-preview.convertMediaURLs', 'datlechin-link-preview.convert_media_urls', 'boolval', true),
+        ->serializeToForum('datlechin-link-preview.useGoogleFavicons', 'datlechin-link-preview.use_google_favicons', 'boolval')
+        ->serializeToForum('datlechin-link-preview.convertMediaURLs', 'datlechin-link-preview.convert_media_urls', 'boolval'),
+
+    (new Extend\Event())
+        ->listen(Deserializing::class, function (Deserializing $event) {
+            $event->settings = array_merge(
+                [
+                    'datlechin-link-preview.use_google_favicons' => false,
+                    'datlechin-link-preview.convert_media_urls' => false,
+                    'datlechin-link-preview.cache_time' => 60,
+                ],
+                $event->settings
+            );
+        })
 ];

js/src/admin/index.js

@@ -30,5 +30,12 @@ app.initializers.add('datlechin/flarum-link-preview', () => {
       help: app.translator.trans(EXT_PREFIX + '.admin.settings.whitelist_help'),
       placeholder: app.translator.trans(EXT_PREFIX + '.admin.settings.whitelist_placeholder'),
       type: 'textarea',
+    })
+    .registerSetting({
+      setting: EXT_PREFIX + '.cache_time',
+      label: app.translator.trans(EXT_PREFIX + '.admin.settings.cache_time_label'),
+      help: app.translator.trans(EXT_PREFIX + '.admin.settings.cache_time_help'),
+      type: 'number',
+      min: 0,
     });
 });

locale/en.yml

@@ -11,6 +11,8 @@ datlechin-link-preview:
       convert_media_urls_help: If enabled, this will convert all media URLs to link previews.
       use_google_favicons_label: Use Google Favicon API
       use_google_favicons_help: If enabled, will use a Google service in the frontend to request the respective site's favicon.
+      cache_time_label: Cache Time
+      cache_time_help: Number of minutes crawled links will be cached before they are accessed again. Set to 0 to disable link caching. Defaults to 60 minutes if not set.
 
   forum:
     site_cannot_be_reached: This site cannot be reached.

src/Api/Controllers/ScrapperController.php

@@ -3,6 +3,7 @@
 namespace Datlechin\LinkPreview\Api\Controllers;
 
 use Flarum\Settings\SettingsRepositoryInterface;
+use Illuminate\Contracts\Cache\Store;
 use Laminas\Diactoros\Response\JsonResponse;
 use Psr\Http\Message\ResponseInterface as Response;
 use Psr\Http\Message\ServerRequestInterface as Request;
@@ -25,18 +26,34 @@ class ScrapperController implements RequestHandlerInterface
     /** @var array */
     protected $whitelist = [];
 
-    public function __construct(PHPScraper $web, TranslatorInterface $translator, SettingsRepositoryInterface $settings)
-    {
+    /** @var Store */
+    protected $cache;
+
+    /** @var int */
+    private $cacheTime;
+
+    public function __construct(
+        PHPScraper $web,
+        TranslatorInterface $translator,
+        SettingsRepositoryInterface $settings,
+        Store $cache
+    ) {
         $this->web = $web;
         $this->translator = $translator;
+        $this->cache = $cache;
+        $cacheTime = $settings->get('datlechin-link-preview.cache_time');
+        if (!is_numeric($cacheTime)) {
+            $cacheTime = 60;
+        }
+        $this->cacheTime = intval($cacheTime);
         $this->blacklist = array_filter(
             array_map(
                 'trim',
                 explode(',', $settings->get('datlechin-link-preview.blacklist') ?? '')
             )
         );
         $this->whitelist = array_filter(
-                array_map(
+            array_map(
                 'trim',
                 explode(',', $settings->get('datlechin-link-preview.whitelist') ?? '')
             )
@@ -56,22 +73,42 @@ public function handle(Request $request): Response
             ! filter_var($url, FILTER_VALIDATE_URL)
             || ($this->whitelist && ! in_array($domain, $this->whitelist, true))
             || ($this->blacklist && in_array($domain, $this->blacklist, true))
-            || gethostbyname($domain) === $domain
         ) {
             return new JsonResponse([
                 'error' => $this->translator->trans('datlechin-link-preview.forum.site_cannot_be_reached')
             ]);
         }
 
+        if ($this->cacheTime) {
+            $cacheKey = 'datlechin-link-preview:' . md5(preg_replace('/^https?:\/\/(.+)\/?$/', '$1', $url));
+            $data = $this->cache->get($cacheKey);
+            if (null !== $data) {
+                return new JsonResponse($data);
+            }
+        }
+
+        if (gethostbyname($domain) === $domain) {
+            return new JsonResponse([
+                'error' => $this->translator->trans('datlechin-link-preview.forum.site_cannot_be_reached')
+            ]);
+        }
+
         try {
             $this->web->go($url);
 
-            return new JsonResponse([
+            $data = [
                 'site_name' => $this->web->openGraph['og:site_name'] ?? $this->web->twitterCard['twitter:site'] ?? null,
                 'title' => $this->web->title ?? $this->web->openGraph['og:title'] ?? $this->web->twitterCard['twitter:title'] ?? null,
                 'description' => $this->web->description ?? $this->web->openGraph['og:description'] ?? $this->web->twitterCard['twitter:description'] ?? null,
                 'image' => $this->web->image ?? $this->web->openGraph['og:image'] ?? $this->web->twitterCard['twitter:image'] ?? null,
-            ]);
+                'accessed' => time(),
+            ];
+
+            if ($this->cacheTime && isset($cacheKey)) {
+                $this->cache->put($cacheKey, $data, $this->cacheTime * 60);
+            }
+
+            return new JsonResponse($data);
         } catch (Throwable $th) {
             return new JsonResponse([
                 'error' => $th->getMessage(),