File watcher improvements (#49)

* Webhook now takes in image data and stores image files * Increase max request size to accommodate larger images * Add public option to event storage * Fix event date issue * Fix telegram config issue * Fix issues that arose from refactoring, all tests now passing * Fix download link to retain original file name * Increase max file size for php * Remove pagination from detection profiles page * Add tests to smb config and refactor slightly * Remove unnecessary TODOs * Image compression * Update .env.example * Change watch folder volume. Update laravel network. * Code sniffer fixes * Ide helpers * Apply fixes from StyleCI (#50) Co-authored-by: Andrew <[email protected]> Co-authored-by: Andrew <[email protected]>
akmolina28 · Feb 23, 2021 · 0c2b6ad · 0c2b6ad
1 parent c97a533
commit 0c2b6ad
Show file tree

Hide file tree

Showing 46 changed files with 1,088 additions and 917 deletions.
diff --git a/.env.example b/.env.example
@@ -1,10 +1,15 @@
 WATCH_FOLDER=c:/aiinput
+WATCH_FOLDER_AUTO_DELETE=true
 WEB_INTERFACE_URL=http://localhost
 WEB_INTERFACE_PORT=8080
 
 # How many days to keep in database (0 for unlimited)
 DATA_RETENTION_DAYS=7
 
+# Set to false if you want Last Watch to retain the original, uncompressed images
+# Note: AI detection is always run before compressing the image
+COMPRESS_IMAGES=true
+
 # Set to true if the watcher is not picking up new image files quickly.
 # Unusual watch folders like network shares may require polling.
 WATCH_USE_POLLING=false

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -12,7 +12,6 @@ services:
       - "${WEB_INTERFACE_PORT:-8080}:80"
     volumes:
       - ./src:/var/www/app
-      - ${WATCH_FOLDER}:/var/www/app/storage/app/public/events
       - ./nginx/default.conf:/etc/nginx/conf.d/default.conf
     depends_on:
       - php
@@ -55,7 +54,6 @@ services:
     ports:
       - "9000:9000"
     volumes:
-      - ${WATCH_FOLDER}:/var/www/app/storage/app/public/events
       - ./src:/var/www/app
     networks:
       - laravel
@@ -64,6 +62,7 @@ services:
       APP_URL: ${WEB_INTERFACE_URL:-localhost}:${WEB_INTERFACE_PORT:-8080}
       APP_DEBUG: ${APP_DEBUG:-false}
       DEEPSTACK_BASE_URL: ${DEEPSTACK_BASE_URL:-http://deepstack:5000/}
+      COMPRESS_IMAGES: ${COMPRESS_IMAGES:-true}
 
   scheduler:
     image: mcuadros/ofelia:latest
@@ -75,7 +74,7 @@ services:
       - php
 
   queue:
-    image: akmolina28/laravel-queue-worker
+    image: akmolina28/laravel-queue-worker:php7.3-v4.4.1
     container_name: lw_queue
     restart: unless-stopped
     tty: true
@@ -85,7 +84,6 @@ services:
       NUMPROCS: 8
       APP_URL: ${WEB_INTERFACE_URL:-localhost}:${WEB_INTERFACE_PORT:-8080}
     volumes:
-      - ${WATCH_FOLDER}:/var/www/app/storage/app/public/events
       - ./src:/var/www/app
     networks:
       - laravel
@@ -139,16 +137,17 @@ services:
       - "5001:5000"
 
   watcher:
-    image: akmolina28/webhook-folder-watcher
+    image: akmolina28/webhook-folder-watcher:0.2.0
     container_name: lw_watcher
     volumes:
       - ./src:/var/www/app
-      - ${WATCH_FOLDER}:/var/www/app/storage/app/public/events
+      - ${WATCH_FOLDER}:/watch
     restart: unless-stopped
     environment:
-      - WATCH_FOLDER=/var/www/app/storage/app/public/events
-      - WEBHOOK_URL=http://lw_nginx:80/webhook-receiving-url
+      - WATCH_FOLDER=/watch
+      - WEBHOOK_URL=http://lw_nginx:80/api/events
       - WATCH_USE_POLLING=${WATCH_USE_POLLING:-false}
+      - DELETE_AFTER_POSTING=${WATCH_FOLDER_AUTO_DELETE:-false}
     networks:
       - laravel
 

diff --git a/nginx/default.conf b/nginx/default.conf
@@ -3,6 +3,7 @@ server {
     index index.php index.html;
     server_name localhost;
     root /var/www/app/public;
+    client_max_body_size 20M;
 
     location / {
         try_files $uri $uri/ /index.php?$query_string;

diff --git a/php.dockerfile b/php.dockerfile
@@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \
         supervisor \
         cron \
         libzip-dev \
+        jpegoptim \
     && docker-php-ext-configure gd --with-freetype --with-jpeg \
     && docker-php-ext-install -j$(nproc) gd
 

diff --git a/php/php.ini b/php/php.ini
@@ -1,2 +1,6 @@
-log_errors = On
-error_log = /dev/stderr
+upload_max_filesize = 10M
+post_max_size = 10M
+
+#log_errors = On
+#error_log = /dev/stderr
+
diff --git a/src/.phpstorm.meta.php b/src/.phpstorm.meta.php
@@ -43,6 +43,7 @@
             'Illuminate\Session\Middleware\StartSession' => \Illuminate\Session\Middleware\StartSession::class,
             'JavaScript' => \Laracasts\Utilities\JavaScript\Transformers\Transformer::class,
             'NunoMaduro\Collision\Contracts\Provider' => \NunoMaduro\Collision\Provider::class,
+            'Spatie\ImageOptimizer\OptimizerChain' => \Spatie\ImageOptimizer\OptimizerChain::class,
             'Spatie\WebhookClient\WebhookConfigRepository' => \Spatie\WebhookClient\WebhookConfigRepository::class,
             'Whoops\Handler\HandlerInterface' => \Facade\Ignition\ErrorPage\IgnitionWhoopsHandler::class,
             'auth' => \Illuminate\Auth\AuthManager::class,
@@ -146,6 +147,7 @@
             'hash' => \Illuminate\Hashing\HashManager::class,
             'hash.driver' => \Illuminate\Hashing\BcryptHasher::class,
             'html' => \Collective\Html\HtmlBuilder::class,
+            'image-optimizer' => \Spatie\ImageOptimizer\OptimizerChain::class,
             'log' => \Illuminate\Log\LogManager::class,
             'mail.manager' => \Illuminate\Mail\MailManager::class,
             'mailer' => \Illuminate\Mail\Mailer::class,
@@ -205,6 +207,7 @@
             'Illuminate\Session\Middleware\StartSession' => \Illuminate\Session\Middleware\StartSession::class,
             'JavaScript' => \Laracasts\Utilities\JavaScript\Transformers\Transformer::class,
             'NunoMaduro\Collision\Contracts\Provider' => \NunoMaduro\Collision\Provider::class,
+            'Spatie\ImageOptimizer\OptimizerChain' => \Spatie\ImageOptimizer\OptimizerChain::class,
             'Spatie\WebhookClient\WebhookConfigRepository' => \Spatie\WebhookClient\WebhookConfigRepository::class,
             'Whoops\Handler\HandlerInterface' => \Facade\Ignition\ErrorPage\IgnitionWhoopsHandler::class,
             'auth' => \Illuminate\Auth\AuthManager::class,
@@ -308,6 +311,7 @@
             'hash' => \Illuminate\Hashing\HashManager::class,
             'hash.driver' => \Illuminate\Hashing\BcryptHasher::class,
             'html' => \Collective\Html\HtmlBuilder::class,
+            'image-optimizer' => \Spatie\ImageOptimizer\OptimizerChain::class,
             'log' => \Illuminate\Log\LogManager::class,
             'mail.manager' => \Illuminate\Mail\MailManager::class,
             'mailer' => \Illuminate\Mail\Mailer::class,
@@ -367,6 +371,7 @@
             'Illuminate\Session\Middleware\StartSession' => \Illuminate\Session\Middleware\StartSession::class,
             'JavaScript' => \Laracasts\Utilities\JavaScript\Transformers\Transformer::class,
             'NunoMaduro\Collision\Contracts\Provider' => \NunoMaduro\Collision\Provider::class,
+            'Spatie\ImageOptimizer\OptimizerChain' => \Spatie\ImageOptimizer\OptimizerChain::class,
             'Spatie\WebhookClient\WebhookConfigRepository' => \Spatie\WebhookClient\WebhookConfigRepository::class,
             'Whoops\Handler\HandlerInterface' => \Facade\Ignition\ErrorPage\IgnitionWhoopsHandler::class,
             'auth' => \Illuminate\Auth\AuthManager::class,
@@ -470,6 +475,7 @@
             'hash' => \Illuminate\Hashing\HashManager::class,
             'hash.driver' => \Illuminate\Hashing\BcryptHasher::class,
             'html' => \Collective\Html\HtmlBuilder::class,
+            'image-optimizer' => \Spatie\ImageOptimizer\OptimizerChain::class,
             'log' => \Illuminate\Log\LogManager::class,
             'mail.manager' => \Illuminate\Mail\MailManager::class,
             'mailer' => \Illuminate\Mail\Mailer::class,
@@ -529,6 +535,7 @@
             'Illuminate\Session\Middleware\StartSession' => \Illuminate\Session\Middleware\StartSession::class,
             'JavaScript' => \Laracasts\Utilities\JavaScript\Transformers\Transformer::class,
             'NunoMaduro\Collision\Contracts\Provider' => \NunoMaduro\Collision\Provider::class,
+            'Spatie\ImageOptimizer\OptimizerChain' => \Spatie\ImageOptimizer\OptimizerChain::class,
             'Spatie\WebhookClient\WebhookConfigRepository' => \Spatie\WebhookClient\WebhookConfigRepository::class,
             'Whoops\Handler\HandlerInterface' => \Facade\Ignition\ErrorPage\IgnitionWhoopsHandler::class,
             'auth' => \Illuminate\Auth\AuthManager::class,
@@ -632,6 +639,7 @@
             'hash' => \Illuminate\Hashing\HashManager::class,
             'hash.driver' => \Illuminate\Hashing\BcryptHasher::class,
             'html' => \Collective\Html\HtmlBuilder::class,
+            'image-optimizer' => \Spatie\ImageOptimizer\OptimizerChain::class,
             'log' => \Illuminate\Log\LogManager::class,
             'mail.manager' => \Illuminate\Mail\MailManager::class,
             'mailer' => \Illuminate\Mail\Mailer::class,
@@ -691,6 +699,7 @@
             'Illuminate\Session\Middleware\StartSession' => \Illuminate\Session\Middleware\StartSession::class,
             'JavaScript' => \Laracasts\Utilities\JavaScript\Transformers\Transformer::class,
             'NunoMaduro\Collision\Contracts\Provider' => \NunoMaduro\Collision\Provider::class,
+            'Spatie\ImageOptimizer\OptimizerChain' => \Spatie\ImageOptimizer\OptimizerChain::class,
             'Spatie\WebhookClient\WebhookConfigRepository' => \Spatie\WebhookClient\WebhookConfigRepository::class,
             'Whoops\Handler\HandlerInterface' => \Facade\Ignition\ErrorPage\IgnitionWhoopsHandler::class,
             'auth' => \Illuminate\Auth\AuthManager::class,
@@ -794,6 +803,7 @@
             'hash' => \Illuminate\Hashing\HashManager::class,
             'hash.driver' => \Illuminate\Hashing\BcryptHasher::class,
             'html' => \Collective\Html\HtmlBuilder::class,
+            'image-optimizer' => \Spatie\ImageOptimizer\OptimizerChain::class,
             'log' => \Illuminate\Log\LogManager::class,
             'mail.manager' => \Illuminate\Mail\MailManager::class,
             'mailer' => \Illuminate\Mail\Mailer::class,
@@ -853,6 +863,7 @@
             'Illuminate\Session\Middleware\StartSession' => \Illuminate\Session\Middleware\StartSession::class,
             'JavaScript' => \Laracasts\Utilities\JavaScript\Transformers\Transformer::class,
             'NunoMaduro\Collision\Contracts\Provider' => \NunoMaduro\Collision\Provider::class,
+            'Spatie\ImageOptimizer\OptimizerChain' => \Spatie\ImageOptimizer\OptimizerChain::class,
             'Spatie\WebhookClient\WebhookConfigRepository' => \Spatie\WebhookClient\WebhookConfigRepository::class,
             'Whoops\Handler\HandlerInterface' => \Facade\Ignition\ErrorPage\IgnitionWhoopsHandler::class,
             'auth' => \Illuminate\Auth\AuthManager::class,
@@ -956,6 +967,7 @@
             'hash' => \Illuminate\Hashing\HashManager::class,
             'hash.driver' => \Illuminate\Hashing\BcryptHasher::class,
             'html' => \Collective\Html\HtmlBuilder::class,
+            'image-optimizer' => \Spatie\ImageOptimizer\OptimizerChain::class,
             'log' => \Illuminate\Log\LogManager::class,
             'mail.manager' => \Illuminate\Mail\MailManager::class,
             'mailer' => \Illuminate\Mail\Mailer::class,
@@ -1015,6 +1027,7 @@
             'Illuminate\Session\Middleware\StartSession' => \Illuminate\Session\Middleware\StartSession::class,
             'JavaScript' => \Laracasts\Utilities\JavaScript\Transformers\Transformer::class,
             'NunoMaduro\Collision\Contracts\Provider' => \NunoMaduro\Collision\Provider::class,
+            'Spatie\ImageOptimizer\OptimizerChain' => \Spatie\ImageOptimizer\OptimizerChain::class,
             'Spatie\WebhookClient\WebhookConfigRepository' => \Spatie\WebhookClient\WebhookConfigRepository::class,
             'Whoops\Handler\HandlerInterface' => \Facade\Ignition\ErrorPage\IgnitionWhoopsHandler::class,
             'auth' => \Illuminate\Auth\AuthManager::class,
@@ -1118,6 +1131,7 @@
             'hash' => \Illuminate\Hashing\HashManager::class,
             'hash.driver' => \Illuminate\Hashing\BcryptHasher::class,
             'html' => \Collective\Html\HtmlBuilder::class,
+            'image-optimizer' => \Spatie\ImageOptimizer\OptimizerChain::class,
             'log' => \Illuminate\Log\LogManager::class,
             'mail.manager' => \Illuminate\Mail\MailManager::class,
             'mailer' => \Illuminate\Mail\Mailer::class,
@@ -1177,6 +1191,7 @@
             'Illuminate\Session\Middleware\StartSession' => \Illuminate\Session\Middleware\StartSession::class,
             'JavaScript' => \Laracasts\Utilities\JavaScript\Transformers\Transformer::class,
             'NunoMaduro\Collision\Contracts\Provider' => \NunoMaduro\Collision\Provider::class,
+            'Spatie\ImageOptimizer\OptimizerChain' => \Spatie\ImageOptimizer\OptimizerChain::class,
             'Spatie\WebhookClient\WebhookConfigRepository' => \Spatie\WebhookClient\WebhookConfigRepository::class,
             'Whoops\Handler\HandlerInterface' => \Facade\Ignition\ErrorPage\IgnitionWhoopsHandler::class,
             'auth' => \Illuminate\Auth\AuthManager::class,
@@ -1280,6 +1295,7 @@
             'hash' => \Illuminate\Hashing\HashManager::class,
             'hash.driver' => \Illuminate\Hashing\BcryptHasher::class,
             'html' => \Collective\Html\HtmlBuilder::class,
+            'image-optimizer' => \Spatie\ImageOptimizer\OptimizerChain::class,
             'log' => \Illuminate\Log\LogManager::class,
             'mail.manager' => \Illuminate\Mail\MailManager::class,
             'mailer' => \Illuminate\Mail\Mailer::class,
@@ -1339,6 +1355,7 @@
             'Illuminate\Session\Middleware\StartSession' => \Illuminate\Session\Middleware\StartSession::class,
             'JavaScript' => \Laracasts\Utilities\JavaScript\Transformers\Transformer::class,
             'NunoMaduro\Collision\Contracts\Provider' => \NunoMaduro\Collision\Provider::class,
+            'Spatie\ImageOptimizer\OptimizerChain' => \Spatie\ImageOptimizer\OptimizerChain::class,
             'Spatie\WebhookClient\WebhookConfigRepository' => \Spatie\WebhookClient\WebhookConfigRepository::class,
             'Whoops\Handler\HandlerInterface' => \Facade\Ignition\ErrorPage\IgnitionWhoopsHandler::class,
             'auth' => \Illuminate\Auth\AuthManager::class,
@@ -1442,6 +1459,7 @@
             'hash' => \Illuminate\Hashing\HashManager::class,
             'hash.driver' => \Illuminate\Hashing\BcryptHasher::class,
             'html' => \Collective\Html\HtmlBuilder::class,
+            'image-optimizer' => \Spatie\ImageOptimizer\OptimizerChain::class,
             'log' => \Illuminate\Log\LogManager::class,
             'mail.manager' => \Illuminate\Mail\MailManager::class,
             'mailer' => \Illuminate\Mail\Mailer::class,
@@ -1501,6 +1519,7 @@
             'Illuminate\Session\Middleware\StartSession' => \Illuminate\Session\Middleware\StartSession::class,
             'JavaScript' => \Laracasts\Utilities\JavaScript\Transformers\Transformer::class,
             'NunoMaduro\Collision\Contracts\Provider' => \NunoMaduro\Collision\Provider::class,
+            'Spatie\ImageOptimizer\OptimizerChain' => \Spatie\ImageOptimizer\OptimizerChain::class,
             'Spatie\WebhookClient\WebhookConfigRepository' => \Spatie\WebhookClient\WebhookConfigRepository::class,
             'Whoops\Handler\HandlerInterface' => \Facade\Ignition\ErrorPage\IgnitionWhoopsHandler::class,
             'auth' => \Illuminate\Auth\AuthManager::class,
@@ -1604,6 +1623,7 @@
             'hash' => \Illuminate\Hashing\HashManager::class,
             'hash.driver' => \Illuminate\Hashing\BcryptHasher::class,
             'html' => \Collective\Html\HtmlBuilder::class,
+            'image-optimizer' => \Spatie\ImageOptimizer\OptimizerChain::class,
             'log' => \Illuminate\Log\LogManager::class,
             'mail.manager' => \Illuminate\Mail\MailManager::class,
             'mailer' => \Illuminate\Mail\Mailer::class,
@@ -1663,6 +1683,7 @@
             'Illuminate\Session\Middleware\StartSession' => \Illuminate\Session\Middleware\StartSession::class,
             'JavaScript' => \Laracasts\Utilities\JavaScript\Transformers\Transformer::class,
             'NunoMaduro\Collision\Contracts\Provider' => \NunoMaduro\Collision\Provider::class,
+            'Spatie\ImageOptimizer\OptimizerChain' => \Spatie\ImageOptimizer\OptimizerChain::class,
             'Spatie\WebhookClient\WebhookConfigRepository' => \Spatie\WebhookClient\WebhookConfigRepository::class,
             'Whoops\Handler\HandlerInterface' => \Facade\Ignition\ErrorPage\IgnitionWhoopsHandler::class,
             'auth' => \Illuminate\Auth\AuthManager::class,
@@ -1766,6 +1787,7 @@
             'hash' => \Illuminate\Hashing\HashManager::class,
             'hash.driver' => \Illuminate\Hashing\BcryptHasher::class,
             'html' => \Collective\Html\HtmlBuilder::class,
+            'image-optimizer' => \Spatie\ImageOptimizer\OptimizerChain::class,
             'log' => \Illuminate\Log\LogManager::class,
             'mail.manager' => \Illuminate\Mail\MailManager::class,
             'mailer' => \Illuminate\Mail\Mailer::class,

diff --git a/src/_ide_helper.php b/src/_ide_helper.php
@@ -15526,6 +15526,77 @@ public static function componentCall($method, $parameters)
 
     }
 
+}
+
+    namespace Spatie\LaravelImageOptimizer\Facades { 
+            /**
+     * 
+     *
+     */ 
+        class ImageOptimizer {
+                    /**
+         * 
+         *
+         * @static 
+         */ 
+        public static function getOptimizers()
+        {
+                        /** @var \Spatie\ImageOptimizer\OptimizerChain $instance */
+                        return $instance->getOptimizers();
+        }
+                    /**
+         * 
+         *
+         * @static 
+         */ 
+        public static function addOptimizer($optimizer)
+        {
+                        /** @var \Spatie\ImageOptimizer\OptimizerChain $instance */
+                        return $instance->addOptimizer($optimizer);
+        }
+                    /**
+         * 
+         *
+         * @static 
+         */ 
+        public static function setOptimizers($optimizers)
+        {
+                        /** @var \Spatie\ImageOptimizer\OptimizerChain $instance */
+                        return $instance->setOptimizers($optimizers);
+        }
+                    /**
+         * 
+         *
+         * @static 
+         */ 
+        public static function setTimeout($timeoutInSeconds)
+        {
+                        /** @var \Spatie\ImageOptimizer\OptimizerChain $instance */
+                        return $instance->setTimeout($timeoutInSeconds);
+        }
+                    /**
+         * 
+         *
+         * @static 
+         */ 
+        public static function useLogger($log)
+        {
+                        /** @var \Spatie\ImageOptimizer\OptimizerChain $instance */
+                        return $instance->useLogger($log);
+        }
+                    /**
+         * 
+         *
+         * @static 
+         */ 
+        public static function optimize($pathToImage, $pathToOutput = null)
+        {
+                        /** @var \Spatie\ImageOptimizer\OptimizerChain $instance */
+                        return $instance->optimize($pathToImage, $pathToOutput);
+        }
+
+    }
+
 }
 
     namespace Illuminate\Http { 
@@ -18573,6 +18644,7 @@ class Flare extends \Facade\Ignition\Facades\Flare {}
             class JavaScript extends \Laracasts\Utilities\JavaScript\JavaScriptFacade {}
             class Form extends \Collective\Html\FormFacade {}
             class Html extends \Collective\Html\HtmlFacade {}
+            class ImageOptimizer extends \Spatie\LaravelImageOptimizer\Facades\ImageOptimizer {}
 
 }
 

diff --git a/src/app/AutomationConfig.php b/src/app/AutomationConfig.php
@@ -34,6 +34,8 @@
  * @method static Builder|AutomationConfig onlyTrashed()
  * @method static Builder|AutomationConfig withTrashed()
  * @method static Builder|AutomationConfig withoutTrashed()
+ * @property bool $is_high_priority
+ * @method static Builder|AutomationConfig whereIsHighPriority($value)
  */
 class AutomationConfig extends Model
 {

diff --git a/src/app/DeepstackClient.php b/src/app/DeepstackClient.php
@@ -6,9 +6,13 @@
 
 class DeepstackClient implements DeepstackClientInterface
 {
-    private string $api_base_url;
+    public string $api_base_url;
 
-    public function __construct($api_base_url)
+    /**
+     * DeepstackClient constructor.
+     * @param string $api_base_url
+     */
+    public function __construct(string $api_base_url)
     {
         $this->api_base_url = rtrim($api_base_url, '/');
     }