Merge pull request #195 from Icinga/migration-hook

Provide migration hook & migrate jobs config to database

application/clicommands/MigrateCommand.php

@@ -0,0 +1,120 @@
+<?php
+
+/* Icinga Web 2 X.509 Module | (c) 2023 Icinga GmbH | GPLv2 */
+
+namespace Icinga\Module\X509\Clicommands;
+
+use DateTime;
+use Icinga\Application\Logger;
+use Icinga\Authentication\Auth;
+use Icinga\Module\X509\Command;
+use Icinga\Module\X509\Job;
+use Icinga\Repository\IniRepository;
+use Icinga\User;
+use Icinga\Util\Json;
+use ipl\Scheduler\Cron;
+use ipl\Sql\Connection;
+use ipl\Sql\Expression;
+use stdClass;
+
+use function ipl\Stdlib\get_php_type;
+
+class MigrateCommand extends Command
+{
+    /**
+     * Migrate the jobs config rom INI to the database
+     *
+     * USAGE
+     *
+     *     icingacli x509 migrate jobs --author=<name>
+     *
+     * OPTIONS
+     *
+     * --author=<name>
+     *     An Icinga Web 2 user used to mark as an author for all the migrated jobs.
+     */
+    public function jobsAction(): void
+    {
+        /** @var string $author */
+        $author = $this->params->getRequired('author');
+        /** @var User $user */
+        $user = Auth::getInstance()->getUser();
+        $user->setUsername($author);
+
+        $this->migrateJobs();
+
+        Logger::info('Successfully applied all pending migrations');
+    }
+
+    protected function migrateJobs(): void
+    {
+        $repo = new class () extends IniRepository {
+            /** @var array<string, array<int, string>> */
+            protected $queryColumns = [
+                'jobs' => ['name', 'cidrs', 'ports', 'exclude_targets', 'schedule', 'frequencyType']
+            ];
+
+            /** @var array<string, array<string, string>> */
+            protected $configs = [
+                'jobs' => [
+                    'module'    => 'x509',
+                    'name'      => 'jobs',
+                    'keyColumn' => 'name'
+                ]
+            ];
+        };
+
+        $conn = $this->getDb();
+        $conn->transaction(function (Connection $conn) use ($repo) {
+            /** @var User $user */
+            $user = Auth::getInstance()->getUser();
+            /** @var stdClass $data */
+            foreach ($repo->select() as $data) {
+                $config = [];
+                if (! isset($data->frequencyType) && ! empty($data->schedule)) {
+                    $frequency = new Cron($data->schedule);
+                    $config = [
+                        'type'      => get_php_type($frequency),
+                        'frequency' => Json::encode($frequency)
+                    ];
+                } elseif (! empty($data->schedule)) {
+                    $config = [
+                        'type'      => $data->frequencyType,
+                        'frequency' => $data->schedule // Is already json encoded
+                    ];
+                }
+
+                $excludes = $data->exclude_targets;
+                if (empty($excludes)) {
+                    $excludes = new Expression('NULL');
+                }
+
+                $conn->insert('x509_job', [
+                    'name'            => $data->name,
+                    'author'          => $user->getUsername(),
+                    'cidrs'           => $data->cidrs,
+                    'ports'           => $data->ports,
+                    'exclude_targets' => $excludes,
+                    'ctime'           => (new DateTime())->getTimestamp() * 1000,
+                    'mtime'           => (new DateTime())->getTimestamp() * 1000
+                ]);
+
+                $jobId = (int) $conn->lastInsertId();
+                if (! empty($config)) {
+                    $config['rescan'] = 'n';
+                    $config['full_scan'] = 'n';
+                    $config['since_last_scan'] = Job::DEFAULT_SINCE_LAST_SCAN;
+
+                    $conn->insert('x509_schedule', [
+                        'job_id' => $jobId,
+                        'name'   => $data->name . ' Schedule',
+                        'author' => $user->getUsername(),
+                        'config' => Json::encode($config),
+                        'ctime'  => (new DateTime())->getTimestamp() * 1000,
+                        'mtime'  => (new DateTime())->getTimestamp() * 1000,
+                    ]);
+                }
+            }
+        });
+    }
+}

doc/80-Upgrading.md

@@ -3,6 +3,42 @@
 Upgrading Icinga Certificate Monitoring is straightforward.
 Usually the only manual steps involved are schema updates for the database.
 
+## Upgrading to version 1.3.0
+
+Icinga Certificate Monitoring version `1.3.0` requires a schema update for the database. We have dropped the use of **INI**
+files to store jobs and are using the database instead. So you need to migrate your job configs to the database.
+
+If you're already using Icinga Web 2 version `>= 2.12`, then you don't need to import the sql upgrade scripts manually.
+Icinga Web provides you the ability to perform such migrations in a simple way. You may be familiar with such an automation
+if you're an Icinga Director user.
+
+> **Note**
+> 
+> Please note that it doesn't matter if you import the database upgrade script manually or via the new automation,
+> you will have to migrate your [Jobs config](#migrate-jobs) from INI to the database manually afterwards.
+
+Before migrating your jobs from **INI** to the database, you need to first apply the migration script. This will create
+the tables needed to store the jobs and schedules in the database.
+
+You may use the following command to apply the database schema upgrade file:
+<!-- {% if not icingaDocs %} -->
+
+**Note:** If you haven't installed this module from packages, then please adapt the schema path to the correct installation path.
+
+<!-- {% endif %} -->
+```sql
+# mysql -u root -p x509 < /usr/share/icingaweb2/modules/x509/schema/mysql-upgrades/1.3.0.sql
+```
+
+### Migrate Jobs
+
+Afterwards, you can safely migrate your jobs with the following command. Keep in mind that you need to specify an
+Icinga Web username that will be used as the author of these jobs in the database.
+
+```
+# icingacli x509 migrate jobs --author "icingaadmin"
+```
+
 ## Upgrading to version 1.2.0
 
 Icinga Certificate Monitoring version 1.2.0 requires a schema update for the database. We have changed all `timestamp`

library/X509/Model/Schema.php

@@ -0,0 +1,49 @@
+<?php
+
+/* Icinga Web 2 X.509 Module | (c) 2023 Icinga GmbH | GPLv2 */
+
+namespace Icinga\Module\X509\Model;
+
+use DateTime;
+use ipl\Orm\Behavior\BoolCast;
+use ipl\Orm\Behavior\MillisecondTimestamp;
+use ipl\Orm\Behaviors;
+use ipl\Orm\Model;
+
+/**
+ * A database model for x509 schema version table
+ *
+ * @property int $id Unique identifier of the database schema entries
+ * @property string $version The current schema version of Icinga Web
+ * @property DateTime $timestamp The insert/modify time of the schema entry
+ * @property bool $success Whether the database migration of the current version was successful
+ * @property ?string $reason The reason why the database migration has failed
+ */
+class Schema extends Model
+{
+    public function getTableName(): string
+    {
+        return 'x509_schema';
+    }
+
+    public function getKeyName()
+    {
+        return 'id';
+    }
+
+    public function getColumns(): array
+    {
+        return [
+            'version',
+            'timestamp',
+            'success',
+            'reason'
+        ];
+    }
+
+    public function createBehaviors(Behaviors $behaviors): void
+    {
+        $behaviors->add(new BoolCast(['success']));
+        $behaviors->add(new MillisecondTimestamp(['timestamp']));
+    }
+}

library/X509/ProvidedHook/DbMigration.php

@@ -0,0 +1,99 @@
+<?php
+
+/* Icinga Web 2 X.509 Module | (c) 2023 Icinga GmbH | GPLv2 */
+
+namespace Icinga\Module\X509\ProvidedHook;
+
+use Icinga\Application\Hook\DbMigrationHook;
+use Icinga\Module\X509\Common\Database;
+use Icinga\Module\X509\Model\Schema;
+use ipl\Orm\Query;
+use ipl\Sql;
+use ipl\Sql\Adapter\Pgsql;
+
+class DbMigration extends DbMigrationHook
+{
+    use Database {
+        getDb as private getX509Db;
+    }
+
+    public function getName(): string
+    {
+        return $this->translate('Icinga Certificate Monitoring');
+    }
+
+    public function providedDescriptions(): array
+    {
+        return [
+            '1.0.0' => $this->translate(
+                'Adjusts the database type of several columns and changes some composed primary keys.'
+            ),
+            '1.1.0' => $this->translate(
+                'Changes the composed x509_target index and x509_certificate valid from/to types to bigint.'
+            ),
+            '1.2.0' => $this->translate(
+                'Changes all timestamp columns to bigint and adjusts enum types of "yes/no" to "n/y".'
+            ),
+            '1.3.0' => $this->translate(
+                'Introduces the required tables to store jobs and job schedules in the database.'
+            )
+        ];
+    }
+
+    public function getVersion(): string
+    {
+        if ($this->version === null) {
+            $conn = $this->getDb();
+            $schema = $this->getSchemaQuery()
+                ->columns(['version', 'success'])
+                ->orderBy('id', SORT_DESC)
+                ->limit(2);
+
+            if (static::tableExists($conn, $schema->getModel()->getTableName())) {
+                /** @var Schema $version */
+                foreach ($schema as $version) {
+                    if ($version->success) {
+                        $this->version = $version->version;
+
+                        break;
+                    }
+                }
+
+                if (! $this->version) {
+                    // Schema version table exist, but the user has probably deleted the entry!
+                    $this->version = '1.3.0';
+                }
+            } elseif (
+                $this->getDb()->getAdapter() instanceof Pgsql
+                || static::getColumnType($conn, 'x509_certificate', 'ctime') === 'bigint(20) unsigned'
+            ) {
+                // We modified a bunch of timestamp columns to bigint in x509 version 1.2.0.
+                // We have also added Postgres support with x509 version 1.2 and never had an upgrade scripts until now.
+                $this->version = '1.2.0';
+            } elseif (static::getColumnType($conn, 'x509_certificate_subject_alt_name', 'hash') !== null) {
+                if (static::getColumnType($conn, 'x509_certificate', 'valid_from') === 'bigint(20) unsigned') {
+                    $this->version = '1.0.0';
+                } else {
+                    $this->version = '1.1.0';
+                }
+            } else {
+                // X509 version 1.0 was the first release of this module, but due to some reason it also contains
+                // an upgrade script and adds `hash` column. However, if this column doesn't exist yet, we need
+                // to use the lowest possible release value as the initial (last migrated) version.
+                $this->version = '0.0.0';
+            }
+        }
+
+        return $this->version;
+    }
+
+    public function getDb(): Sql\Connection
+    {
+        return $this->getX509Db();
+    }
+
+    protected function getSchemaQuery(): Query
+    {
+        return Schema::on($this->getDb());
+    }
+}

run.php

@@ -2,5 +2,9 @@
 
 // Icinga Web 2 X.509 Module | (c) 2018 Icinga GmbH | GPLv2
 
+/** @var \Icinga\Application\Modules\Module $this */
+
+$this->provideHook('DbMigration', '\\Icinga\\Module\\X509\\ProvidedHook\\DbMigration');
+
 $this->provideHook('director/ImportSource', '\\Icinga\\Module\\X509\\ProvidedHook\\HostsImportSource');
 $this->provideHook('director/ImportSource', '\\Icinga\\Module\\X509\\ProvidedHook\\ServicesImportSource');

schema/mysql-upgrades/1.3.0.sql

@@ -1,4 +1,4 @@
-CREATE TABLE IF NOT EXISTS x509_job (
+CREATE TABLE x509_job (
   id int(10) unsigned NOT NULL AUTO_INCREMENT,
   name varchar(255) NOT NULL COLLATE utf8mb4_unicode_ci,
   author varchar(255) NOT NULL COLLATE utf8mb4_unicode_ci,
@@ -12,7 +12,7 @@ CREATE TABLE IF NOT EXISTS x509_job (
   UNIQUE (name)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
 
-CREATE TABLE IF NOT EXISTS x509_schedule (
+CREATE TABLE x509_schedule (
   id int(10) unsigned NOT NULL AUTO_INCREMENT,
   job_id int(10) unsigned NOT NULL,
   name varchar(255) NOT NULL COLLATE utf8mb4_unicode_ci,
@@ -27,13 +27,25 @@ CREATE TABLE IF NOT EXISTS x509_schedule (
 
 DELETE FROM x509_job_run;
 ALTER TABLE x509_job_run
-  ADD COLUMN IF NOT EXISTS job_id int(10) unsigned NOT NULL AFTER id,
-  ADD COLUMN IF NOT EXISTS schedule_id int(10) unsigned DEFAULT NULL AFTER job_id,
-  DROP COLUMN IF EXISTS `name`,
-  DROP COLUMN IF EXISTS ctime,
-  DROP COLUMN IF EXISTS mtime,
-  DROP CONSTRAINT IF EXISTS fk_x509_job_run_job,
-  DROP CONSTRAINT IF EXISTS fk_x509_job_run_schedule;
+  ADD COLUMN job_id int(10) unsigned NOT NULL AFTER id,
+  ADD COLUMN schedule_id int(10) unsigned DEFAULT NULL AFTER job_id,
+  DROP COLUMN `name`,
+  DROP COLUMN ctime,
+  DROP COLUMN mtime;
 ALTER TABLE x509_job_run
   ADD CONSTRAINT fk_x509_job_run_job FOREIGN KEY (job_id) REFERENCES x509_job (id) ON DELETE CASCADE,
   ADD CONSTRAINT fk_x509_job_run_schedule FOREIGN KEY (schedule_id) REFERENCES x509_schedule (id) ON DELETE CASCADE;
+
+CREATE TABLE x509_schema (
+  id int unsigned NOT NULL AUTO_INCREMENT,
+  version varchar(64) NOT NULL,
+  timestamp bigint unsigned NOT NULL,
+  success enum ('n', 'y') DEFAULT NULL,
+  reason text DEFAULT NULL,
+
+  PRIMARY KEY (id),
+  CONSTRAINT idx_x509_schema_version UNIQUE (version)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin ROW_FORMAT=DYNAMIC;
+
+INSERT INTO x509_schema (version, timestamp, success, reason)
+  VALUES ('1.3.0', UNIX_TIMESTAMP() * 1000, 'y', NULL);

schema/mysql.schema.sql

@@ -120,3 +120,17 @@ CREATE TABLE x509_job_run (
   CONSTRAINT fk_x509_job_run_job FOREIGN KEY (job_id) REFERENCES x509_job (id) ON DELETE CASCADE,
   CONSTRAINT fk_x509_job_run_schedule FOREIGN KEY (schedule_id) REFERENCES x509_schedule (id) ON DELETE CASCADE
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+CREATE TABLE x509_schema (
+  id int unsigned NOT NULL AUTO_INCREMENT,
+  version varchar(64) NOT NULL,
+  timestamp bigint unsigned NOT NULL,
+  success enum ('n', 'y') DEFAULT NULL,
+  reason text DEFAULT NULL,
+
+  PRIMARY KEY (id),
+  CONSTRAINT idx_x509_schema_version UNIQUE (version)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin ROW_FORMAT=DYNAMIC;
+
+INSERT INTO x509_schema (version, timestamp, success)
+  VALUES ('1.3.0', UNIX_TIMESTAMP() * 1000, 'y');