From 81617d67399561f6f86e34da2df1f22fcfec3226 Mon Sep 17 00:00:00 2001 From: Anvit Srivastav Date: Tue, 1 Oct 2024 14:08:39 -0700 Subject: [PATCH 01/14] Update ElasticSearch and Elastica dependencies Update ElasticSearch to 6.8.23 in the Dockerfile and update Elastica to 6.x in composer. --- composer.json | 2 +- composer.lock | 217 ++++++++++++++++++---------------- docker/docker-compose.dev.yml | 2 +- 3 files changed, 117 insertions(+), 104 deletions(-) diff --git a/composer.json b/composer.json index 912f1d0976..8c82b6bd6a 100644 --- a/composer.json +++ b/composer.json @@ -20,7 +20,7 @@ "league/csv": "^9.4", "apereo/phpcas": "^1.3.8", "ezyang/htmlpurifier": "^4.13", - "ruflin/elastica": "5.*", + "ruflin/elastica": "6.*", "jumbojett/openid-connect-php": "^1.0" }, "autoload-dev": { diff --git a/composer.lock b/composer.lock index 6afe91680f..98ffdf5fd8 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "035f0da316e6b2ff0e564d5a70f3567a", + "content-hash": "95c831656c031654f6504bb3f06c6b5a", "packages": [ { "name": "apereo/phpcas", @@ -79,31 +79,33 @@ }, { "name": "elasticsearch/elasticsearch", - "version": "v5.4.0", + "version": "v6.8.3", "source": { "type": "git", "url": "https://github.com/elastic/elasticsearch-php.git", - "reference": "d3c5b55ad94f5053ca76c48585b4cde2cdc6bc59" + "reference": "a48d84b133453136b9bfab4e561147129dde3b2d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/elastic/elasticsearch-php/zipball/d3c5b55ad94f5053ca76c48585b4cde2cdc6bc59", - "reference": "d3c5b55ad94f5053ca76c48585b4cde2cdc6bc59", + "url": "https://api.github.com/repos/elastic/elasticsearch-php/zipball/a48d84b133453136b9bfab4e561147129dde3b2d", + "reference": "a48d84b133453136b9bfab4e561147129dde3b2d", "shasum": "" }, "require": { - "guzzlehttp/ringphp": "~1.0", - "php": "^5.6|^7.0", - "psr/log": "~1.0" + "ext-json": ">=1.3.7", + "ezimuel/ringphp": "^1.1.2", + "php": "^7.3 || ^8.0", + "psr/log": "^1|^2" }, "require-dev": { - "cpliakas/git-wrapper": "~1.0", - "doctrine/inflector": "^1.1", - "mockery/mockery": "0.9.4", - "phpunit/phpunit": "^4.7|^5.4", - "sami/sami": "~3.2", - "symfony/finder": "^2.8", - "symfony/yaml": "^2.8" + "doctrine/inflector": "^1.3", + "mockery/mockery": "^1.2", + "phpstan/phpstan": "^0.12", + "phpunit/phpunit": "^9.3", + "squizlabs/php_codesniffer": "^3.4", + "symfony/finder": "~4.0", + "symfony/yaml": "~4.0", + "symplify/git-wrapper": ">=9.0 <9.3.27" }, "suggest": { "ext-curl": "*", @@ -111,6 +113,9 @@ }, "type": "library", "autoload": { + "files": [ + "src/autoload.php" + ], "psr-4": { "Elasticsearch\\": "src/Elasticsearch/" } @@ -122,6 +127,9 @@ "authors": [ { "name": "Zachary Tong" + }, + { + "name": "Enrico Zimuel" } ], "description": "PHP Client for Elasticsearch", @@ -132,93 +140,88 @@ ], "support": { "issues": "https://github.com/elastic/elasticsearch-php/issues", - "source": "https://github.com/elastic/elasticsearch-php/tree/v5.4.0" + "source": "https://github.com/elastic/elasticsearch-php/tree/v6.8.3" }, - "time": "2019-01-08T18:57:00+00:00" + "time": "2024-04-12T13:12:26+00:00" }, { - "name": "ezyang/htmlpurifier", - "version": "v4.18.0", + "name": "ezimuel/guzzlestreams", + "version": "3.1.0", "source": { "type": "git", - "url": "https://github.com/ezyang/htmlpurifier.git", - "reference": "cb56001e54359df7ae76dc522d08845dc741621b" + "url": "https://github.com/ezimuel/guzzlestreams.git", + "reference": "b4b5a025dfee70d6cd34c780e07330eb93d5b997" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/ezyang/htmlpurifier/zipball/cb56001e54359df7ae76dc522d08845dc741621b", - "reference": "cb56001e54359df7ae76dc522d08845dc741621b", + "url": "https://api.github.com/repos/ezimuel/guzzlestreams/zipball/b4b5a025dfee70d6cd34c780e07330eb93d5b997", + "reference": "b4b5a025dfee70d6cd34c780e07330eb93d5b997", "shasum": "" }, "require": { - "php": "~5.6.0 || ~7.0.0 || ~7.1.0 || ~7.2.0 || ~7.3.0 || ~7.4.0 || ~8.0.0 || ~8.1.0 || ~8.2.0 || ~8.3.0 || ~8.4.0" + "php": ">=5.4.0" }, "require-dev": { - "cerdic/css-tidy": "^1.7 || ^2.0", - "simpletest/simpletest": "dev-master" - }, - "suggest": { - "cerdic/css-tidy": "If you want to use the filter 'Filter.ExtractStyleBlocks'.", - "ext-bcmath": "Used for unit conversion and imagecrash protection", - "ext-iconv": "Converts text to and from non-UTF-8 encodings", - "ext-tidy": "Used for pretty-printing HTML" + "phpunit/phpunit": "~9.0" }, "type": "library", + "extra": { + "branch-alias": { + "dev-master": "3.0-dev" + } + }, "autoload": { - "files": [ - "library/HTMLPurifier.composer.php" - ], - "psr-0": { - "HTMLPurifier": "library/" - }, - "exclude-from-classmap": [ - "/library/HTMLPurifier/Language/" - ] + "psr-4": { + "GuzzleHttp\\Stream\\": "src/" + } }, "notification-url": "https://packagist.org/downloads/", "license": [ - "LGPL-2.1-or-later" + "MIT" ], "authors": [ { - "name": "Edward Z. Yang", - "email": "admin@htmlpurifier.org", - "homepage": "http://ezyang.com" + "name": "Michael Dowling", + "email": "mtdowling@gmail.com", + "homepage": "https://github.com/mtdowling" } ], - "description": "Standards compliant HTML filter written in PHP", - "homepage": "http://htmlpurifier.org/", + "description": "Fork of guzzle/streams (abandoned) to be used with elasticsearch-php", + "homepage": "http://guzzlephp.org/", "keywords": [ - "html" + "Guzzle", + "stream" ], "support": { - "issues": "https://github.com/ezyang/htmlpurifier/issues", - "source": "https://github.com/ezyang/htmlpurifier/tree/v4.18.0" + "source": "https://github.com/ezimuel/guzzlestreams/tree/3.1.0" }, - "time": "2024-11-01T03:51:45+00:00" + "time": "2022-10-24T12:58:50+00:00" }, { - "name": "guzzlehttp/ringphp", - "version": "1.1.1", + "name": "ezimuel/ringphp", + "version": "1.2.2", "source": { "type": "git", - "url": "https://github.com/guzzle/RingPHP.git", - "reference": "5e2a174052995663dd68e6b5ad838afd47dd615b" + "url": "https://github.com/ezimuel/ringphp.git", + "reference": "7887fc8488013065f72f977dcb281994f5fde9f4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/guzzle/RingPHP/zipball/5e2a174052995663dd68e6b5ad838afd47dd615b", - "reference": "5e2a174052995663dd68e6b5ad838afd47dd615b", + "url": "https://api.github.com/repos/ezimuel/ringphp/zipball/7887fc8488013065f72f977dcb281994f5fde9f4", + "reference": "7887fc8488013065f72f977dcb281994f5fde9f4", "shasum": "" }, "require": { - "guzzlehttp/streams": "~3.0", + "ezimuel/guzzlestreams": "^3.0.1", "php": ">=5.4.0", "react/promise": "~2.0" }, + "replace": { + "guzzlehttp/ringphp": "self.version" + }, "require-dev": { "ext-curl": "*", - "phpunit/phpunit": "~4.0" + "phpunit/phpunit": "~9.0" }, "suggest": { "ext-curl": "Guzzle will use specific adapters if cURL is present" @@ -245,68 +248,72 @@ "homepage": "https://github.com/mtdowling" } ], - "description": "Provides a simple API and specification that abstracts away the details of HTTP into a single PHP function.", + "description": "Fork of guzzle/RingPHP (abandoned) to be used with elasticsearch-php", "support": { - "issues": "https://github.com/guzzle/RingPHP/issues", - "source": "https://github.com/guzzle/RingPHP/tree/1.1.1" + "source": "https://github.com/ezimuel/ringphp/tree/1.2.2" }, - "abandoned": true, - "time": "2018-07-31T13:22:33+00:00" + "time": "2022-12-07T11:28:53+00:00" }, { - "name": "guzzlehttp/streams", - "version": "3.0.0", + "name": "ezyang/htmlpurifier", + "version": "v4.18.0", "source": { "type": "git", - "url": "https://github.com/guzzle/streams.git", - "reference": "47aaa48e27dae43d39fc1cea0ccf0d84ac1a2ba5" + "url": "https://github.com/ezyang/htmlpurifier.git", + "reference": "cb56001e54359df7ae76dc522d08845dc741621b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/guzzle/streams/zipball/47aaa48e27dae43d39fc1cea0ccf0d84ac1a2ba5", - "reference": "47aaa48e27dae43d39fc1cea0ccf0d84ac1a2ba5", + "url": "https://api.github.com/repos/ezyang/htmlpurifier/zipball/cb56001e54359df7ae76dc522d08845dc741621b", + "reference": "cb56001e54359df7ae76dc522d08845dc741621b", "shasum": "" }, "require": { - "php": ">=5.4.0" + "php": "~5.6.0 || ~7.0.0 || ~7.1.0 || ~7.2.0 || ~7.3.0 || ~7.4.0 || ~8.0.0 || ~8.1.0 || ~8.2.0 || ~8.3.0 || ~8.4.0" }, "require-dev": { - "phpunit/phpunit": "~4.0" + "cerdic/css-tidy": "^1.7 || ^2.0", + "simpletest/simpletest": "dev-master" }, - "type": "library", - "extra": { - "branch-alias": { - "dev-master": "3.0-dev" - } + "suggest": { + "cerdic/css-tidy": "If you want to use the filter 'Filter.ExtractStyleBlocks'.", + "ext-bcmath": "Used for unit conversion and imagecrash protection", + "ext-iconv": "Converts text to and from non-UTF-8 encodings", + "ext-tidy": "Used for pretty-printing HTML" }, + "type": "library", "autoload": { - "psr-4": { - "GuzzleHttp\\Stream\\": "src/" - } + "files": [ + "library/HTMLPurifier.composer.php" + ], + "psr-0": { + "HTMLPurifier": "library/" + }, + "exclude-from-classmap": [ + "/library/HTMLPurifier/Language/" + ] }, "notification-url": "https://packagist.org/downloads/", "license": [ - "MIT" + "LGPL-2.1-or-later" ], "authors": [ { - "name": "Michael Dowling", - "email": "mtdowling@gmail.com", - "homepage": "https://github.com/mtdowling" + "name": "Edward Z. Yang", + "email": "admin@htmlpurifier.org", + "homepage": "http://ezyang.com" } ], - "description": "Provides a simple abstraction over streams of data", - "homepage": "http://guzzlephp.org/", + "description": "Standards compliant HTML filter written in PHP", + "homepage": "http://htmlpurifier.org/", "keywords": [ - "Guzzle", - "stream" + "html" ], "support": { - "issues": "https://github.com/guzzle/streams/issues", - "source": "https://github.com/guzzle/streams/tree/master" + "issues": "https://github.com/ezyang/htmlpurifier/issues", + "source": "https://github.com/ezyang/htmlpurifier/tree/v4.18.0" }, - "abandoned": true, - "time": "2014-10-12T19:18:40+00:00" + "time": "2024-11-01T03:51:45+00:00" }, { "name": "jumbojett/openid-connect-php", @@ -785,26 +792,32 @@ }, { "name": "ruflin/elastica", - "version": "5.3.6", + "version": "6.2.1", "source": { "type": "git", "url": "https://github.com/ruflin/Elastica.git", - "reference": "1ed6f587a38d4d6455b688ffc9a9d370d4ae60e9" + "reference": "f95cf646f884a8e7b73bd12eda3e5753994b24d3" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/ruflin/Elastica/zipball/1ed6f587a38d4d6455b688ffc9a9d370d4ae60e9", - "reference": "1ed6f587a38d4d6455b688ffc9a9d370d4ae60e9", + "url": "https://api.github.com/repos/ruflin/Elastica/zipball/f95cf646f884a8e7b73bd12eda3e5753994b24d3", + "reference": "f95cf646f884a8e7b73bd12eda3e5753994b24d3", "shasum": "" }, "require": { - "elasticsearch/elasticsearch": "5.4.*", - "php": ">=5.6.0", + "elasticsearch/elasticsearch": "^6.0", + "ext-json": "*", + "php": "^7.0||^8.0", "psr/log": "~1.0" }, + "conflict": { + "elasticsearch/elasticsearch": "6.8.0 || 6.8.1" + }, "require-dev": { "aws/aws-sdk-php": "~3.0", - "guzzlehttp/guzzle": "~6.0" + "guzzlehttp/guzzle": "~6.0", + "phpunit/phpunit": "^6.0||^8.5.20", + "yoast/phpunit-polyfills": "1.0.1" }, "suggest": { "aws/aws-sdk-php": "Allow using IAM authentication with Amazon ElasticSearch Service", @@ -815,7 +828,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-master": "5.3.x-dev" + "dev-master": "6.0.x-dev" } }, "autoload": { @@ -841,9 +854,9 @@ ], "support": { "issues": "https://github.com/ruflin/Elastica/issues", - "source": "https://github.com/ruflin/Elastica/tree/5.3.6" + "source": "https://github.com/ruflin/Elastica/tree/6.2.1" }, - "time": "2019-08-29T06:34:21+00:00" + "time": "2022-04-14T06:57:28+00:00" } ], "packages-dev": [ diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index 92ed06742c..b3545284e7 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -43,7 +43,7 @@ services: - "63001:80" elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:5.6.16 + image: docker.elastic.co/elasticsearch/elasticsearch:6.8.23 env_file: etc/environment ulimits: memlock: From 7c508846727e53dd0d488dda1aca0e2ee31226c6 Mon Sep 17 00:00:00 2001 From: melaniekung Date: Thu, 3 Oct 2024 10:35:01 -0700 Subject: [PATCH 02/14] Remove arElasticSearchIndexDecorator. - 'inline' scripts are deprecated, changed to 'source' --- .../arUpdatePublicationStatusJob.class.php | 2 +- .../updatePublicationStatusTask.class.php | 2 +- .../arElasticSearchIndexDecorator.class.php | 83 ------------------- .../lib/arElasticSearchPlugin.class.php | 4 +- 4 files changed, 3 insertions(+), 88 deletions(-) delete mode 100644 plugins/arElasticSearchPlugin/lib/arElasticSearchIndexDecorator.class.php diff --git a/lib/job/arUpdatePublicationStatusJob.class.php b/lib/job/arUpdatePublicationStatusJob.class.php index f3f524b4cc..fc13012741 100644 --- a/lib/job/arUpdatePublicationStatusJob.class.php +++ b/lib/job/arUpdatePublicationStatusJob.class.php @@ -78,7 +78,7 @@ public function runJob($parameters) $queryScript = \Elastica\Script\AbstractScript::create([ 'script' => [ - 'inline' => 'ctx._source.publicationStatusId = '.$publicationStatus->id, + 'source' => 'ctx._source.publicationStatusId = '.$publicationStatus->id, 'lang' => 'painless', ], ]); diff --git a/lib/task/tools/updatePublicationStatusTask.class.php b/lib/task/tools/updatePublicationStatusTask.class.php index eee2a902eb..0c343826bc 100644 --- a/lib/task/tools/updatePublicationStatusTask.class.php +++ b/lib/task/tools/updatePublicationStatusTask.class.php @@ -166,7 +166,7 @@ protected function updatePublicationStatusDescendants($resource, $publicationSta $queryScript = \Elastica\Script\AbstractScript::create([ 'script' => [ - 'inline' => 'ctx._source.publicationStatusId = '.$publicationStatus->id, + 'source' => 'ctx._source.publicationStatusId = '.$publicationStatus->id, 'lang' => 'painless', ], ]); diff --git a/plugins/arElasticSearchPlugin/lib/arElasticSearchIndexDecorator.class.php b/plugins/arElasticSearchPlugin/lib/arElasticSearchIndexDecorator.class.php deleted file mode 100644 index 02a45ec56e..0000000000 --- a/plugins/arElasticSearchPlugin/lib/arElasticSearchIndexDecorator.class.php +++ /dev/null @@ -1,83 +0,0 @@ -. - */ - -use Elastica\Query; -use Elastica\Response; -use Elastica\Script\AbstractScript; -use Elasticsearch\Endpoints\UpdateByQuery; - -/** - * arElasticSearchIndexDecorator can be removed when Elastica >= 6.x. It is present to - * provide Elastica/Index::updateByQuery(). - * - * @author sbreker - */ -class arElasticSearchIndexDecorator -{ - protected $_instance; - - public function __construct(Elastica\Index $instance) - { - $this->_instance = $instance; - } - - public function __call($method, $args) - { - return call_user_func_array([$this->_instance, $method], $args); - } - - public function __get($key) - { - return $this->_instance->{$key}; - } - - public function __set($key, $val) - { - return $this->_instance->{$key} = $val; - } - - public function getInstance() - { - return $this->_instance; - } - - /** - * Update entries in the db based on a query. - * - * @param AbstractQuery|array|Collapse|Query|string|Suggest $query Query object or array - * @param AbstractScript $script Script - * @param array $options Optional params - * - * @see https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-update-by-query.html - */ - public function updateByQuery($query, AbstractScript $script, array $options = []): Response - { - $endpoint = new UpdateByQuery(); - $q = Query::create($query)->getQuery(); - $body = [ - 'query' => \is_array($q) ? $q : $q->toArray(), - 'script' => $script->toArray()['script'], - ]; - - $endpoint->setBody($body); - $endpoint->setParams($options); - - return $this->requestEndpoint($endpoint); - } -} diff --git a/plugins/arElasticSearchPlugin/lib/arElasticSearchPlugin.class.php b/plugins/arElasticSearchPlugin/lib/arElasticSearchPlugin.class.php index 07e9179900..cdc1401951 100644 --- a/plugins/arElasticSearchPlugin/lib/arElasticSearchPlugin.class.php +++ b/plugins/arElasticSearchPlugin/lib/arElasticSearchPlugin.class.php @@ -90,9 +90,7 @@ public function __construct(array $options = []) // Verify the version running in the server $this->checkVersion(); - // TODO: arElasticSearchIndexDecorator can be removed when Elastica >= 6.x. It is present to - // provide Elastica/Index::updateByQuery(). - $this->index = new arElasticSearchIndexDecorator($this->client->getIndex($this->config['index']['name'])); + $this->index = $this->client->getIndex($this->config['index']['name']); // Load batch mode configuration $this->batchMode = true === $this->config['batch_mode']; From 1c5feebb00055636dfff52904b2f3676a3d254b2 Mon Sep 17 00:00:00 2001 From: Anvit Srivastav Date: Wed, 9 Oct 2024 10:16:34 -0700 Subject: [PATCH 03/14] Update index for ES 6.x Update arElasticSearchPlugin to use multiple indices instead of multiple types since ES 6.x removed being able to add multiple types. Also update mapping to remove include_in_all and add that to the _all field using copy_to instead since include_in_all was also removed in ES 6.x --- .../arElasticSearchPlugin/config/mapping.yml | 368 +++++++++--------- .../arElasticSearchIndexDecorator.class.php | 84 ++++ .../lib/arElasticSearchMapping.class.php | 11 +- .../lib/arElasticSearchPlugin.class.php | 142 ++++--- .../lib/arElasticSearchPluginUtil.class.php | 4 +- 5 files changed, 369 insertions(+), 240 deletions(-) create mode 100644 plugins/arElasticSearchPlugin/lib/arElasticSearchIndexDecorator.class.php diff --git a/plugins/arElasticSearchPlugin/config/mapping.yml b/plugins/arElasticSearchPlugin/config/mapping.yml index 550e095023..a84317c157 100644 --- a/plugins/arElasticSearchPlugin/config/mapping.yml +++ b/plugins/arElasticSearchPlugin/config/mapping.yml @@ -34,8 +34,8 @@ mapping: properties: contact_person: { type: keyword } street_address: { type: text, index: false } - postal_code: { type: text, include_in_all: false } - country_code: { type: keyword, include_in_all: false } + postal_code: { type: text } + country_code: { type: keyword } location: { type: geo_point } other_name: @@ -61,12 +61,12 @@ mapping: dynamic: strict type: nested properties: - start_date: { type: date, include_in_all: false } - end_date: { type: date, include_in_all: false } + start_date: { type: date } + end_date: { type: date } start_date_string: { type: keyword } end_date_string: { type: keyword } - type_id: { type: integer, include_in_all: false } - actor_id: { type: integer, include_in_all: false } + type_id: { type: integer } + actor_id: { type: integer } relation: _attributes: @@ -74,9 +74,9 @@ mapping: dynamic: strict type: nested properties: - object_id: { type: integer, include_in_all: false } - subject_id: { type: integer, include_in_all: false } - type_id: { type: integer, include_in_all: false } + object_id: { type: integer } + subject_id: { type: integer } + type_id: { type: integer } accession_event: _attributes: @@ -91,7 +91,7 @@ mapping: dynamic: strict dynamic: strict properties: - date: { type: date, include_in_all: false } + date: { type: date } date_string: { type: keyword } donor: @@ -109,104 +109,104 @@ mapping: nested_only: true dynamic: strict properties: - basis: { type: text, include_in_all: false } - start_date: { type: date, include_in_all: false } - end_date: { type: date, include_in_all: false } - copyright_status: { type: text, include_in_all: false } - rights_holder: { type: text, include_in_all: false } - rights_note: { type: text, include_in_all: false } - license_terms: { type: text, include_in_all: false } + basis: { type: text } + start_date: { type: date } + end_date: { type: date } + copyright_status: { type: text } + rights_holder: { type: text } + rights_note: { type: text } + license_terms: { type: text } act_right: _attributes: nested_only: true dynamic: strict properties: - act: { type: text, include_in_all: false } - restriction: { type: text, include_in_all: false } - start_date: { type: date, include_in_all: false } - end_date: { type: date, include_in_all: false } + act: { type: text } + restriction: { type: text } + start_date: { type: date } + end_date: { type: date } mediainfo_track: _attributes: nested_only: true dynamic: strict properties: - count: { type: integer, include_in_all: false } - video_format_list: { type: keyword, include_in_all: false } - video_format_with_hint_list: { type: keyword, include_in_all: false } - codecs_video: { type: keyword, include_in_all: false } - video_language_list: { type: keyword, include_in_all: false } - audio_format_list: { type: keyword, include_in_all: false } - audio_format_with_hint_list: { type: keyword, include_in_all: false } - audio_codecs: { type: keyword, include_in_all: false } - audio_language_list: { type: keyword, include_in_all: false } - complete_name: { type: keyword, include_in_all: false } - file_name: { type: keyword, include_in_all: false } - file_extension: { type: keyword, include_in_all: false } - format: { type: keyword, include_in_all: false } - format_info: { type: keyword, include_in_all: false } - format_url: { type: keyword, include_in_all: false } - format_profile: { type: keyword, include_in_all: false } - format_settings: { type: keyword, include_in_all: false } - format_settings_cabac: { type: keyword, include_in_all: false } - format_settings_re_frames: { type: integer, include_in_all: false } - format_settings_gop: { type: keyword, include_in_all: false } - format_extensions_usually_used: { type: keyword, include_in_all: false } - commercial_name: { type: keyword, include_in_all: false } - internet_media_type: { type: keyword, include_in_all: false } - codec_id: { type: keyword, include_in_all: false } - codec_id_info: { type: keyword, include_in_all: false } - codec_id_url: { type: keyword, include_in_all: false } - codec: { type: keyword, include_in_all: false } - codec_family: { type: keyword, include_in_all: false } - codec_info: { type: keyword, include_in_all: false } - codec_url: { type: keyword, include_in_all: false } - codec_cc: { type: keyword, include_in_all: false } - codec_profile: { type: keyword, include_in_all: false } - codec_settings: { type: keyword, include_in_all: false } - codec_settings_cabac: { type: keyword, include_in_all: false } - codec_settings_ref_frames: { type: integer, include_in_all: false } - codec_extensions_usually_used: { type: keyword, include_in_all: false } - file_size: { type: long, include_in_all: false } - duration: { type: integer, include_in_all: false } - bit_rate: { type: integer, include_in_all: false } - bit_rate_mode: { type: keyword, include_in_all: false } - overall_bit_rate: { type: integer, include_in_all: false } - channels: { type: integer, include_in_all: false } - channel_positions: { type: keyword, include_in_all: false } - sampling_rate: { type: integer, include_in_all: false } - samples_count: { type: integer, include_in_all: false } - compression_mode: { type: keyword, include_in_all: false } - width: { type: integer, include_in_all: false } - height: { type: integer, include_in_all: false } - pixel_aspect_ratio: { type: float, include_in_all: false } - display_aspect_ratio: { type: keyword, include_in_all: false } - rotation: { type: float, include_in_all: false } - frame_rate_mode: { type: keyword, include_in_all: false } - frame_rate: { type: float, include_in_all: false } - frame_count: { type: integer, include_in_all: false } - resolution: { type: integer, include_in_all: false } - colorimetry: { type: keyword, include_in_all: false } - color_space: { type: keyword, include_in_all: false } - chroma_subsampling: { type: keyword, include_in_all: false } - bit_depth: { type: integer, include_in_all: false } - scan_type: { type: keyword, include_in_all: false } - interlacement: { type: keyword, include_in_all: false } - bits_pixel_frame: { type: float, include_in_all: false } - stream_size: { type: long, include_in_all: false } - proportion_of_this_stream: { type: float, include_in_all: false } - header_size: { type: long, include_in_all: false } - data_size: { type: long, include_in_all: false } - footer_size: { type: long, include_in_all: false } - language: { type: keyword, include_in_all: false } - color_primaries: { type: keyword, include_in_all: false } - transfer_characteristics: { type: keyword, include_in_all: false } - matrix_coefficients: { type: keyword, include_in_all: false } - is_streamable: { type: boolean, include_in_all: false } - writing_application: { type: keyword, include_in_all: false } - file_last_modification_date: { type: date, include_in_all: false } - file_last_modification_date_local: { type: date, include_in_all: false } + count: { type: integer } + video_format_list: { type: keyword } + video_format_with_hint_list: { type: keyword } + codecs_video: { type: keyword } + video_language_list: { type: keyword } + audio_format_list: { type: keyword } + audio_format_with_hint_list: { type: keyword } + audio_codecs: { type: keyword } + audio_language_list: { type: keyword } + complete_name: { type: keyword } + file_name: { type: keyword } + file_extension: { type: keyword } + format: { type: keyword } + format_info: { type: keyword } + format_url: { type: keyword } + format_profile: { type: keyword } + format_settings: { type: keyword } + format_settings_cabac: { type: keyword } + format_settings_re_frames: { type: integer } + format_settings_gop: { type: keyword } + format_extensions_usually_used: { type: keyword } + commercial_name: { type: keyword } + internet_media_type: { type: keyword } + codec_id: { type: keyword } + codec_id_info: { type: keyword } + codec_id_url: { type: keyword } + codec: { type: keyword } + codec_family: { type: keyword } + codec_info: { type: keyword } + codec_url: { type: keyword } + codec_cc: { type: keyword } + codec_profile: { type: keyword } + codec_settings: { type: keyword } + codec_settings_cabac: { type: keyword } + codec_settings_ref_frames: { type: integer } + codec_extensions_usually_used: { type: keyword } + file_size: { type: long } + duration: { type: integer } + bit_rate: { type: integer } + bit_rate_mode: { type: keyword } + overall_bit_rate: { type: integer } + channels: { type: integer } + channel_positions: { type: keyword } + sampling_rate: { type: integer } + samples_count: { type: integer } + compression_mode: { type: keyword } + width: { type: integer } + height: { type: integer } + pixel_aspect_ratio: { type: float } + display_aspect_ratio: { type: keyword } + rotation: { type: float } + frame_rate_mode: { type: keyword } + frame_rate: { type: float } + frame_count: { type: integer } + resolution: { type: integer } + colorimetry: { type: keyword } + color_space: { type: keyword } + chroma_subsampling: { type: keyword } + bit_depth: { type: integer } + scan_type: { type: keyword } + interlacement: { type: keyword } + bits_pixel_frame: { type: float } + stream_size: { type: long } + proportion_of_this_stream: { type: float } + header_size: { type: long } + data_size: { type: long } + footer_size: { type: long } + language: { type: keyword } + color_primaries: { type: keyword } + transfer_characteristics: { type: keyword } + matrix_coefficients: { type: keyword } + is_streamable: { type: boolean } + writing_application: { type: keyword } + file_last_modification_date: { type: date } + file_last_modification_date_local: { type: date } mediainfo: _attributes: @@ -223,84 +223,84 @@ mapping: mediainfo: mediainfo dynamic: strict properties: - puid: { type: keyword, include_in_all: false } - filename: { type: keyword, include_in_all: false } - last_modified: { type: date, include_in_all: false } - date_ingested: { type: date, include_in_all: false } - size: { type: long, include_in_all: false } - mime_type: { type: keyword, include_in_all: false } + puid: { type: keyword } + filename: { type: keyword } + last_modified: { type: date } + date_ingested: { type: date } + size: { type: long } + mime_type: { type: keyword } audio: type: object properties: - bit_depth: { type: integer, include_in_all: false } - sample_rate: { type: keyword, include_in_all: false } - channels: { type: integer, include_in_all: false } - data_encoding: { type: keyword, include_in_all: false } - offset: { type: integer, include_in_all: false } - byte_order: { type: keyword, include_in_all: false } + bit_depth: { type: integer } + sample_rate: { type: keyword } + channels: { type: integer } + data_encoding: { type: keyword } + offset: { type: integer } + byte_order: { type: keyword } document: type: object properties: - title: { type: keyword, include_in_all: false } - author: { type: keyword, include_in_all: false } - page_count: { type: integer, include_in_all: false } - word_count: { type: integer, include_in_all: false } - character_count: { type: integer, include_in_all: false } - language: { type: keyword, include_in_all: false } - is_protected: { type: boolean, include_in_all: false } - is_rights_managed: { type: boolean, include_in_all: false } - is_tagged: { type: boolean, include_in_all: false } - has_outline: { type: boolean, include_in_all: false } - has_annotations: { type: boolean, include_in_all: false } - has_forms: { type: boolean, include_in_all: false } + title: { type: keyword } + author: { type: keyword } + page_count: { type: integer } + word_count: { type: integer } + character_count: { type: integer } + language: { type: keyword } + is_protected: { type: boolean } + is_rights_managed: { type: boolean } + is_tagged: { type: boolean } + has_outline: { type: boolean } + has_annotations: { type: boolean } + has_forms: { type: boolean } text: type: object properties: - linebreak: { type: keyword, include_in_all: false } - charset: { type: keyword, include_in_all: false } - markup_basis: { type: keyword, include_in_all: false } - markup_basis_version: { type: keyword, include_in_all: false } - markup_language: { type: keyword, include_in_all: false } + linebreak: { type: keyword } + charset: { type: keyword } + markup_basis: { type: keyword } + markup_basis_version: { type: keyword } + markup_language: { type: keyword } format_identification_event: type: object properties: - type: { type: keyword, include_in_all: false } - dateTime: { type: date, include_in_all: false } - detail: { type: keyword, include_in_all: false } - outcome: { type: keyword, include_in_all: false } - outcomeDetailNote: { type: keyword, include_in_all: false } + type: { type: keyword } + dateTime: { type: date } + detail: { type: keyword } + outcome: { type: keyword } + outcomeDetailNote: { type: keyword } linkingAgentIdentifier: type: object properties: - type: { type: keyword, include_in_all: false } - value: { type: keyword, include_in_all: false } + type: { type: keyword } + value: { type: keyword } other_events: type: object properties: - type: { type: keyword, include_in_all: false } - dateTime: { type: date, include_in_all: false } - detail: { type: keyword, include_in_all: false } - outcome: { type: keyword, include_in_all: false } - outcomeDetailNote: { type: keyword, include_in_all: false } + type: { type: keyword } + dateTime: { type: date } + detail: { type: keyword } + outcome: { type: keyword } + outcomeDetailNote: { type: keyword } linkingAgentIdentifier: type: object properties: - type: { type: keyword, include_in_all: false } - value: { type: keyword, include_in_all: false } + type: { type: keyword } + value: { type: keyword } agents: type: object properties: - identifier_type: { type: keyword, include_in_all: false } - identifier_value: { type: keyword, include_in_all: false } - name: { type: keyword, include_in_all: false } - type: { type: keyword, include_in_all: false } + identifier_type: { type: keyword } + identifier_value: { type: keyword } + name: { type: keyword } + type: { type: keyword } format: type: object properties: - name: { type: keyword, include_in_all: false } - version: { type: keyword, include_in_all: false } - registry_name: { type: keyword, include_in_all: false } - registry_key: { type: keyword, include_in_all: false } + name: { type: keyword } + version: { type: keyword } + registry_name: { type: keyword } + registry_key: { type: keyword } physical_object: _attributes: @@ -334,11 +334,11 @@ mapping: digital_objects: premis_object dynamic: strict properties: - uuid: { type: keyword, include_in_all: false } - filename: { type: keyword, include_in_all: false } - size_on_disk: { type: long, include_in_all: false } - digital_object_count: { type: integer, include_in_all: false } - created_at: { type: date, include_in_all: false } + uuid: { type: keyword } + filename: { type: keyword } + size_on_disk: { type: long } + digital_object_count: { type: integer } + created_at: { type: date } term: _attributes: @@ -351,9 +351,9 @@ mapping: dynamic: strict properties: slug: { type: keyword } - taxonomy_id: { type: integer, include_in_all: false } - is_protected: { type: boolean, include_in_all: false } - number_of_descendants: { type: integer, include_in_all: false } + taxonomy_id: { type: integer } + is_protected: { type: boolean } + number_of_descendants: { type: integer } actor: _attributes: @@ -385,17 +385,17 @@ mapping: untouched: type: keyword corporate_body_identifiers: { type: keyword } - entity_type_id: { type: integer, include_in_all: false } - maintaining_repository_id: { type: integer, include_in_all: false } - direct_subjects: { type: integer, include_in_all: false } - direct_places: { type: integer, include_in_all: false } - actor_direct_relation_types: { type: integer, include_in_all: false } - has_digital_object: { type: boolean, include_in_all: false } + entity_type_id: { type: integer } + maintaining_repository_id: { type: integer } + direct_subjects: { type: integer } + direct_places: { type: integer } + actor_direct_relation_types: { type: integer } + has_digital_object: { type: boolean } digital_object: type: object properties: - media_type_id: { type: integer, include_in_all: false } - usage_id: { type: integer, include_in_all: false } + media_type_id: { type: integer } + usage_id: { type: integer } thumbnail_path: { type: keyword } filename: { type: text } digital_object_alt_text: { type: text } @@ -428,7 +428,7 @@ mapping: fields: untouched: type: keyword - date: { type: date, include_in_all: false } + date: { type: date } repository: _attributes: @@ -450,10 +450,10 @@ mapping: fields: untouched: type: keyword - types: { type: integer, include_in_all: false } - geographic_subregions: { type: integer, include_in_all: false } - thematic_areas: { type: integer, include_in_all: false } - logo_path: { type: keyword, include_in_all: false } + types: { type: integer } + geographic_subregions: { type: integer } + thematic_areas: { type: integer } + logo_path: { type: keyword } function_object: _attributes: @@ -590,26 +590,26 @@ mapping: level_of_description_id: { type: integer } lft: { type: integer } publication_status_id: { type: integer } - parent_id: { type: integer, include_in_all: false } - ancestors: { type: integer, include_in_all: false } - children: { type: integer, include_in_all: false } - copyright_status_id: { type: integer, include_in_all: false } - material_type_id: { type: integer, include_in_all: false } - transcript: { type: text, include_in_all: true } - direct_subjects: { type: integer, include_in_all: false } - direct_places: { type: integer, include_in_all: false } - direct_genres: { type: integer, include_in_all: false } - has_digital_object: { type: boolean, include_in_all: false } + parent_id: { type: integer } + ancestors: { type: integer } + children: { type: integer } + copyright_status_id: { type: integer } + material_type_id: { type: integer } + transcript: { type: text, copy_to: _all } + direct_subjects: { type: integer } + direct_places: { type: integer } + direct_genres: { type: integer } + has_digital_object: { type: boolean } finding_aid: type: object properties: - transcript: { type: text, include_in_all: true } - status: { type: integer, include_in_all: false } + transcript: { type: text, copy_to: _all } + status: { type: integer } digital_object: type: object properties: - media_type_id: { type: integer, include_in_all: false } - usage_id: { type: integer, include_in_all: false } + media_type_id: { type: integer } + usage_id: { type: integer } thumbnail_path: { type: keyword } filename: { type: text } digital_object_alt_text: { type: text } @@ -630,5 +630,5 @@ mapping: search_analyzer: standard term_vector: with_positions_offsets # Not nested date fields for sorting - start_date_sort: { type: date, include_in_all: false } - end_date_sort: { type: date, include_in_all: false } + start_date_sort: { type: date } + end_date_sort: { type: date } diff --git a/plugins/arElasticSearchPlugin/lib/arElasticSearchIndexDecorator.class.php b/plugins/arElasticSearchPlugin/lib/arElasticSearchIndexDecorator.class.php new file mode 100644 index 0000000000..7af7fd60fe --- /dev/null +++ b/plugins/arElasticSearchPlugin/lib/arElasticSearchIndexDecorator.class.php @@ -0,0 +1,84 @@ +. + */ + +use Elastica\Query; +use Elastica\Response; +use Elastica\Script\AbstractScript; +use Elasticsearch\Endpoints\UpdateByQuery; + +class arElasticSearchIndexDecorator +{ + protected $_instance; + + protected $_indexPrefix; + + public function __construct($prefix) + { + $this->_instance = []; + + $this->_indexPrefix = $prefix; + } + + public function createIndex($typeName, Elastica\Index $index) { + $typeName = $this->getIndexTypeName($typeName); + $this->_instance[$typeName] = $index; + } + + // Converts camelized Qubit class names to lower case index name used for ElasticSearch + public function getIndexTypeName($typeName) { + return $this->_indexPrefix . '_' . strtolower($typeName); + } + + public function delete() { + foreach ($this->_instance as $index) { + $index->delete(); + } + } + + public function addDocuments($typeName, $documents) { + $typeName = $this->getIndexTypeName($typeName); + $this->_instance[$typeName]->addDocuments($documents); + } + + public function deleteDocuments($typeName, $documents) { + $typeName = $this->getIndexTypeName($typeName); + $this->_instance[$typeName]->deleteDocuments($documents); + } + + public function refresh() { + foreach ($this->_instance as $index) { + $index->refresh(); + } + } + + public function getType($typeName) { + $typeName = $this->getIndexTypeName($typeName); + + return $this->_instance[$typeName]; + } + + public function getInstance($typeName) + { + return $this->getType($typeName); + } + + public function getIndices() { + return $this->_instance; + } +} diff --git a/plugins/arElasticSearchPlugin/lib/arElasticSearchMapping.class.php b/plugins/arElasticSearchPlugin/lib/arElasticSearchMapping.class.php index a84d143607..f09ac23ece 100644 --- a/plugins/arElasticSearchPlugin/lib/arElasticSearchMapping.class.php +++ b/plugins/arElasticSearchPlugin/lib/arElasticSearchMapping.class.php @@ -263,7 +263,7 @@ protected function processPropertyAttributes($typeName, array &$typeProperties) throw new sfException('No i18n_languages in database settings.'); } - $this->setIfNotSet($typeProperties['properties'], 'sourceCulture', ['type' => 'keyword', 'include_in_all' => false]); + $this->setIfNotSet($typeProperties['properties'], 'sourceCulture', ['type' => 'keyword']); // We are using the same mapping for all the i18n fields $nestedI18nFields = []; @@ -342,7 +342,7 @@ protected function processForeignTypes(array &$typeProperties) $mapping = $this->mapping[$foreignTypeNameCamelized]; // Add id of the foreign resource - $mapping['properties']['id'] = ['type' => 'integer', 'include_in_all' => 'false']; + $mapping['properties']['id'] = ['type' => 'integer']; $typeProperties['properties'][$fieldNameCamelized] = $mapping; } @@ -368,7 +368,7 @@ protected function processPartialForeignTypes(array &$typeProperties) } // Add source culture property - $this->setIfNotSet($mapping['properties'], 'sourceCulture', ['type' => 'keyword', 'include_in_all' => false]); + $this->setIfNotSet($mapping['properties'], 'sourceCulture', ['type' => 'keyword']); $nestedI18nFields = []; foreach ($mapping['_i18nFields'] as $i18nFieldName) { @@ -403,7 +403,7 @@ protected function processPartialForeignTypes(array &$typeProperties) } // Add id of the partial foreign resource - $mapping['properties']['id'] = ['type' => 'integer', 'include_in_all' => 'false']; + $mapping['properties']['id'] = ['type' => 'integer']; $typeProperties['properties'][$fieldNameCamelized] = $mapping; } @@ -443,7 +443,7 @@ protected function getI18nFieldMapping($fieldName) { return [ 'type' => 'text', - 'include_in_all' => true, + 'copy_to' => '_all', ]; } @@ -492,7 +492,6 @@ protected function addSortFields($nestedI18nFields, $typeProperties) $nestedI18nFields[$item]['fields']['alphasort'] = [ 'type' => 'keyword', 'normalizer' => 'alphasort', - 'include_in_all' => false, ]; } diff --git a/plugins/arElasticSearchPlugin/lib/arElasticSearchPlugin.class.php b/plugins/arElasticSearchPlugin/lib/arElasticSearchPlugin.class.php index cdc1401951..a2a633dd14 100644 --- a/plugins/arElasticSearchPlugin/lib/arElasticSearchPlugin.class.php +++ b/plugins/arElasticSearchPlugin/lib/arElasticSearchPlugin.class.php @@ -44,6 +44,13 @@ class arElasticSearchPlugin extends QubitSearchEngine */ public $index; + /** + * Current batch type, used for batch flush. + * + * @var mixed defaults to null + */ + protected $currentBatchType; + /** * Mappings configuration, mapping.yml. * @@ -90,7 +97,7 @@ public function __construct(array $options = []) // Verify the version running in the server $this->checkVersion(); - $this->index = $this->client->getIndex($this->config['index']['name']); + $this->index = new arElasticSearchIndexDecorator($this->config['index']['name']); // Load batch mode configuration $this->batchMode = true === $this->config['batch_mode']; @@ -184,7 +191,7 @@ public function flushBatch() // Batch add documents, if any if (count($this->batchAddDocs) > 0) { try { - $this->index->addDocuments($this->batchAddDocs); + $this->index->addDocuments($this->currentBatchType, $this->batchAddDocs); } catch (Exception $e) { // Clear batchAddDocs if something went wrong too $this->batchAddDocs = []; @@ -198,7 +205,7 @@ public function flushBatch() // Batch delete documents, if any if (count($this->batchDeleteDocs) > 0) { try { - $this->index->deleteDocuments($this->batchDeleteDocs); + $this->index->deleteDocuments($this->currentBatchType, $this->batchDeleteDocs); } catch (Exception $e) { // Clear batchDeleteDocs if something went wrong too $this->batchDeleteDocs = []; @@ -341,10 +348,20 @@ public function addDocument($data, $type) unset($data['id']); $document = new \Elastica\Document($id, $data); - $document->setType($type); + $document->setType($this->index->getIndexTypeName($type)); + + if (!$this->currentBatchType) { + $this->currentBatchType = $type; + } if ($this->batchMode) { // Add this document to the batch add queue + if ($this->currentBatchType != $type) { + $this->flushBatch(); + $this->currentBatchType = $type; + $this->index->refresh(); + } + $this->batchAddDocs[] = $document; // If we have a full batch, send additions and deletions in bulk @@ -353,7 +370,7 @@ public function addDocument($data, $type) $this->index->refresh(); } } else { - $this->index->getType($type)->addDocument($document); + $this->index->getType($type)->addDocuments([$document]); } } @@ -425,12 +442,24 @@ public function delete($object) } if ($this->batchMode) { + $type = get_class($object); + + if (!$this->currentBatchType) { + $this->currentBatchType = $type; + } + // The document being deleted may not have been added to the index yet (if it's // still queued up in $this->batchAddDocs) so create a document object representing // the document to be deleted and add this document object to the batch delete // queue. $document = new \Elastica\Document($object->id); - $document->setType(get_class($object)); + $document->setType($this->index->getIndexTypeName($type)); + + if ($this->currentBatchType != $type) { + $this->flushBatch(); + $this->currentBatchType = $type; + $this->index->refresh(); + } $this->batchDeleteDocs[] = $document; @@ -441,7 +470,7 @@ public function delete($object) } } else { try { - $this->index->getType(get_class($object))->deleteById($object->id); + $this->index->getType($type)->deleteById($object->id); } catch (\Elastica\Exception\NotFoundException $e) { // Ignore } @@ -491,57 +520,74 @@ protected function initialize() $this->config['index']['configuration']['analysis']['char_filter']['diacritics_lowercase'] = $this->loadDiacriticsMappings(); } - try { - $this->index->open(); - } catch (Exception $e) { - // If the index has not been initialized, create it - if ($e instanceof \Elastica\Exception\ResponseException) { - // Based on the markdown_enabled setting, add a new filter to strip Markdown tags - if ( - sfConfig::get('app_markdown_enabled', true) - && isset($this->config['index']['configuration']['analysis']['char_filter']['strip_md']) - ) { - foreach ($this->config['index']['configuration']['analysis']['analyzer'] as $key => $analyzer) { - $filters = ['strip_md']; - - if ($this->config['index']['configuration']['analysis']['analyzer'][$key]['char_filter']) { - $filters = array_merge($filters, $this->config['index']['configuration']['analysis']['analyzer'][$key]['char_filter']); - } + // Load and normalize mappings + $this->loadAndNormalizeMappings(); - if (sfConfig::get('app_diacritics')) { - $filters = array_merge($filters, ['diacritics_lowercase']); - } + // Iterate over types (actor, informationobject, ...) + foreach ($this->mappings as $typeName => $typeProperties) { + $typeName = 'Qubit'.sfInflector::camelize($typeName); + $this->index->createIndex($typeName, + $this->client->getIndex($this->index->getIndexTypeName($typeName)) + ); + } - $this->config['index']['configuration']['analysis']['analyzer'][$key]['char_filter'] = $filters; + foreach ($this->index->getIndices() as $indexType => $index) { + try { + $index->open(); + } catch (Exception $e) { + // If the index has not been initialized, create it + if ($e instanceof \Elastica\Exception\ResponseException) { + // Based on the markdown_enabled setting, add a new filter to strip Markdown tags + if ( + sfConfig::get('app_markdown_enabled', true) + && isset($this->config['index']['configuration']['analysis']['char_filter']['strip_md']) + ) { + foreach ($this->config['index']['configuration']['analysis']['analyzer'] as $key => $analyzer) { + $filters = ['strip_md']; + + if ($this->config['index']['configuration']['analysis']['analyzer'][$key]['char_filter']) { + $filters = array_merge($filters, $this->config['index']['configuration']['analysis']['analyzer'][$key]['char_filter']); + } + + if (sfConfig::get('app_diacritics')) { + $filters = array_merge($filters, ['diacritics_lowercase']); + } + + $this->config['index']['configuration']['analysis']['analyzer'][$key]['char_filter'] = $filters; + } } + + $index->create( + $this->config['index']['configuration'], + ['recreate' => true] + ); } - $this->index->create( - $this->config['index']['configuration'], - ['recreate' => true] - ); - } + // Load and normalize mappings + $this->loadAndNormalizeMappings(); - // Load and normalize mappings - $this->loadAndNormalizeMappings(); + // Iterate over types (actor, informationobject, ...) + foreach ($this->mappings as $typeName => $typeProperties) { + $typeName = 'Qubit'.sfInflector::camelize($typeName); - // Iterate over types (actor, informationobject, ...) - foreach ($this->mappings as $typeName => $typeProperties) { - $typeName = 'Qubit'.sfInflector::camelize($typeName); + if ($indexType != $this->index->getIndexTypeName($typeName)) { + continue; + } - // Define mapping in elasticsearch - $mapping = new \Elastica\Type\Mapping(); - $mapping->setType($this->index->getType($typeName)); - $mapping->setProperties($typeProperties['properties']); + // Define mapping in elasticsearch + $mapping = new \Elastica\Type\Mapping(); + $mapping->setType($index->getType($indexType)); + $mapping->setProperties($typeProperties['properties']); - // Parse other parameters - unset($typeProperties['properties']); - foreach ($typeProperties as $key => $value) { - $mapping->setParam($key, $value); - } + // Parse other parameters + unset($this->mapping[$typeName]->typeProperties['properties']); + foreach ($this->mapping[$typeName]->typeProperties as $key => $value) { + $mapping->setParam($key, $value); + } - $this->log(sprintf('Defining mapping %s...', $typeName)); - $mapping->send(); + $this->log(sprintf('Defining mapping %s...', $typeName)); + $mapping->send(); + } } } } diff --git a/plugins/arElasticSearchPlugin/lib/arElasticSearchPluginUtil.class.php b/plugins/arElasticSearchPlugin/lib/arElasticSearchPluginUtil.class.php index 9283e41e5c..62485f9698 100644 --- a/plugins/arElasticSearchPlugin/lib/arElasticSearchPluginUtil.class.php +++ b/plugins/arElasticSearchPlugin/lib/arElasticSearchPluginUtil.class.php @@ -482,8 +482,8 @@ protected static function getAllObjectStringFields( // Get string fields included in _all elseif ( ( - !isset($propertyProperties['include_in_all']) - || $propertyProperties['include_in_all'] + !isset($propertyProperties['copy_to']) + || ($propertyProperties['copy_to'] == '_all') ) && ( isset($propertyProperties['type']) && 'text' == $propertyProperties['type'] From ec4e74856b841be4c3fb99662520beb1d128e96f Mon Sep 17 00:00:00 2001 From: Anvit Srivastav Date: Wed, 9 Oct 2024 11:58:48 -0700 Subject: [PATCH 04/14] Fix autocompleteAction for ES 6.x Update autocompleteAction to use multiple indices --- .../modules/search/actions/autocompleteAction.class.php | 7 +++++-- .../lib/arElasticSearchIndexDecorator.class.php | 7 +------ .../lib/arElasticSearchPlugin.class.php | 2 +- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/apps/qubit/modules/search/actions/autocompleteAction.class.php b/apps/qubit/modules/search/actions/autocompleteAction.class.php index ec91ccf6ed..293aaeecce 100644 --- a/apps/qubit/modules/search/actions/autocompleteAction.class.php +++ b/apps/qubit/modules/search/actions/autocompleteAction.class.php @@ -39,7 +39,7 @@ public function execute($request) $culture = $this->context->user->getCulture(); $client = QubitSearch::getInstance()->client; - $index = QubitSearch::getInstance()->index->getInstance(); + $indices = QubitSearch::getInstance()->index->getInstance(); // Multisearch object $mSearch = new \Elastica\Multi\Search($client); @@ -76,7 +76,10 @@ public function execute($request) foreach ($items as $item) { $search = new \Elastica\Search($client); - $search->addIndex($index)->addType($index->getType($item['type'])); + foreach($indices as $type => $index) { + $itemType = QubitSearch::getInstance()->index->getIndexTypeName($item['type']); + $search->addIndex($index)->addType($index->getType($itemType)); + } $query = new \Elastica\Query(); $query->setSize(3)->setSource($item['fields']); diff --git a/plugins/arElasticSearchPlugin/lib/arElasticSearchIndexDecorator.class.php b/plugins/arElasticSearchPlugin/lib/arElasticSearchIndexDecorator.class.php index 7af7fd60fe..efb8f40f77 100644 --- a/plugins/arElasticSearchPlugin/lib/arElasticSearchIndexDecorator.class.php +++ b/plugins/arElasticSearchPlugin/lib/arElasticSearchIndexDecorator.class.php @@ -73,12 +73,7 @@ public function getType($typeName) { return $this->_instance[$typeName]; } - public function getInstance($typeName) - { - return $this->getType($typeName); - } - - public function getIndices() { + public function getInstance() { return $this->_instance; } } diff --git a/plugins/arElasticSearchPlugin/lib/arElasticSearchPlugin.class.php b/plugins/arElasticSearchPlugin/lib/arElasticSearchPlugin.class.php index a2a633dd14..70b368c762 100644 --- a/plugins/arElasticSearchPlugin/lib/arElasticSearchPlugin.class.php +++ b/plugins/arElasticSearchPlugin/lib/arElasticSearchPlugin.class.php @@ -531,7 +531,7 @@ protected function initialize() ); } - foreach ($this->index->getIndices() as $indexType => $index) { + foreach ($this->index->getInstance() as $indexType => $index) { try { $index->open(); } catch (Exception $e) { From a19f8e11bf3cf26bfcee3cac5253baf51ccb3345 Mon Sep 17 00:00:00 2001 From: Anvit Srivastav Date: Wed, 9 Oct 2024 12:13:33 -0700 Subject: [PATCH 05/14] Add index types for ES Queries without a type Add the appropriate index types for ES requests that did not need to specify explicit types in ES 5.x --- lib/QubitLftSyncer.class.php | 5 +++-- lib/job/arUpdatePublicationStatusJob.class.php | 3 ++- lib/task/tools/updatePublicationStatusTask.class.php | 3 ++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/QubitLftSyncer.class.php b/lib/QubitLftSyncer.class.php index 4b1dc5ab44..d72969a895 100644 --- a/lib/QubitLftSyncer.class.php +++ b/lib/QubitLftSyncer.class.php @@ -92,8 +92,9 @@ private function repairEsChildrenLftValues() $results = QubitPdo::fetchAll($sql, $params, ['fetchMode' => PDO::FETCH_ASSOC]); $bulk = new Elastica\Bulk(QubitSearch::getInstance()->client); - $bulk->setIndex(QubitSearch::getInstance()->index->getName()); - $bulk->setType('QubitInformationObject'); + $type = 'QubitInformationObject'; + $bulk->setIndex(QubitSearch::getInstance()->index->getType($type)->getName()); + $bulk->setType(QubitSearch::getInstance->index->getIndexTypeName($type)); foreach ($results as $row) { $bulk->addAction( diff --git a/lib/job/arUpdatePublicationStatusJob.class.php b/lib/job/arUpdatePublicationStatusJob.class.php index fc13012741..b0e733a7b4 100644 --- a/lib/job/arUpdatePublicationStatusJob.class.php +++ b/lib/job/arUpdatePublicationStatusJob.class.php @@ -91,7 +91,8 @@ public function runJob($parameters) 'conflicts' => 'proceed', ]; - $response = QubitSearch::getInstance()->index->updateByQuery($query, $queryScript, $options)->getData(); + $type = 'QubitInformationObject'; + $response = QubitSearch::getInstance()->index->getType($type)->updateByQuery($query, $queryScript, $options)->getData(); $message = $this->i18n->__( 'Index update completed in %1 ms.', diff --git a/lib/task/tools/updatePublicationStatusTask.class.php b/lib/task/tools/updatePublicationStatusTask.class.php index 0c343826bc..437f866a9b 100644 --- a/lib/task/tools/updatePublicationStatusTask.class.php +++ b/lib/task/tools/updatePublicationStatusTask.class.php @@ -173,7 +173,8 @@ protected function updatePublicationStatusDescendants($resource, $publicationSta $options = ['conflicts' => 'proceed']; - $response = QubitSearch::getInstance()->index->updateByQuery($query, $queryScript, $options)->getData(); + $type = 'QubitInformationObject'; + $response = QubitSearch::getInstance()->index->getType($type)->updateByQuery($query, $queryScript, $options)->getData(); if (!empty($response['failures'])) { $this->failures += count($response['failures']); From 90d1e08bede77af11d21742fbd91cd3843373589 Mon Sep 17 00:00:00 2001 From: Anvit Srivastav Date: Wed, 9 Oct 2024 12:33:48 -0700 Subject: [PATCH 06/14] Fix updateByQuery calls for ElasticSearch 6.x --- lib/QubitLftSyncer.class.php | 2 +- .../arElasticSearchPlugin/lib/arElasticSearchPlugin.class.php | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/QubitLftSyncer.class.php b/lib/QubitLftSyncer.class.php index d72969a895..dda1428e1c 100644 --- a/lib/QubitLftSyncer.class.php +++ b/lib/QubitLftSyncer.class.php @@ -94,7 +94,7 @@ private function repairEsChildrenLftValues() $bulk = new Elastica\Bulk(QubitSearch::getInstance()->client); $type = 'QubitInformationObject'; $bulk->setIndex(QubitSearch::getInstance()->index->getType($type)->getName()); - $bulk->setType(QubitSearch::getInstance->index->getIndexTypeName($type)); + $bulk->setType(QubitSearch::getInstance()->index->getIndexTypeName($type)); foreach ($results as $row) { $bulk->addAction( diff --git a/plugins/arElasticSearchPlugin/lib/arElasticSearchPlugin.class.php b/plugins/arElasticSearchPlugin/lib/arElasticSearchPlugin.class.php index 70b368c762..49d4ff456f 100644 --- a/plugins/arElasticSearchPlugin/lib/arElasticSearchPlugin.class.php +++ b/plugins/arElasticSearchPlugin/lib/arElasticSearchPlugin.class.php @@ -397,7 +397,7 @@ public function partialUpdate($object, $data) $document = new \Elastica\Document($object->id, $data); try { - $this->index->getType(get_class($object))->updateDocument($document); + $this->index->getType(get_class($object))->updateDocuments([$document]); } catch (\Elastica\Exception\NotFoundException $e) { // Create document if it's not found $this->update($object); @@ -417,7 +417,7 @@ public function partialUpdateById(string $className, int $id, array $data) $document = new \Elastica\Document($id, $data); try { - $this->index->getType($className)->updateDocument($document); + $this->index->getType($className)->updateDocuments([$document]); } catch (\Elastica\Exception\ResponseException $e) { // Create document if none exists $modelPdoClassName = self::modelClassFromQubitObjectClass($className).'Pdo'; From 47cc1a949583fb62fd2f0b525364d1472a19433d Mon Sep 17 00:00:00 2001 From: Anvit Srivastav Date: Wed, 9 Oct 2024 14:30:22 -0700 Subject: [PATCH 07/14] Fix broken publication status update Add missing document type to partialUpdate in arElasticSearchPlugin --- .../lib/arElasticSearchIndexDecorator.class.php | 5 ----- .../lib/arElasticSearchPlugin.class.php | 5 ++++- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/plugins/arElasticSearchPlugin/lib/arElasticSearchIndexDecorator.class.php b/plugins/arElasticSearchPlugin/lib/arElasticSearchIndexDecorator.class.php index efb8f40f77..d285786910 100644 --- a/plugins/arElasticSearchPlugin/lib/arElasticSearchIndexDecorator.class.php +++ b/plugins/arElasticSearchPlugin/lib/arElasticSearchIndexDecorator.class.php @@ -17,11 +17,6 @@ * along with Access to Memory (AtoM). If not, see . */ -use Elastica\Query; -use Elastica\Response; -use Elastica\Script\AbstractScript; -use Elasticsearch\Endpoints\UpdateByQuery; - class arElasticSearchIndexDecorator { protected $_instance; diff --git a/plugins/arElasticSearchPlugin/lib/arElasticSearchPlugin.class.php b/plugins/arElasticSearchPlugin/lib/arElasticSearchPlugin.class.php index 49d4ff456f..efe67c1dea 100644 --- a/plugins/arElasticSearchPlugin/lib/arElasticSearchPlugin.class.php +++ b/plugins/arElasticSearchPlugin/lib/arElasticSearchPlugin.class.php @@ -394,10 +394,13 @@ public function partialUpdate($object, $data) return; } + $type = get_class($object); + $document = new \Elastica\Document($object->id, $data); + $document->setType($this->index->getIndexTypeName($type)); try { - $this->index->getType(get_class($object))->updateDocuments([$document]); + $this->index->getType($type)->updateDocuments([$document]); } catch (\Elastica\Exception\NotFoundException $e) { // Create document if it's not found $this->update($object); From 3adaac0697338f09af664c0e1fe7b78b8b5c8589 Mon Sep 17 00:00:00 2001 From: Anvit Srivastav Date: Wed, 9 Oct 2024 16:06:44 -0700 Subject: [PATCH 08/14] Fix linting errors --- .../actions/autocompleteAction.class.php | 2 +- .../arElasticSearchIndexDecorator.class.php | 26 ++++++++++++------- .../lib/arElasticSearchPluginUtil.class.php | 2 +- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/apps/qubit/modules/search/actions/autocompleteAction.class.php b/apps/qubit/modules/search/actions/autocompleteAction.class.php index 293aaeecce..ef9a40c45c 100644 --- a/apps/qubit/modules/search/actions/autocompleteAction.class.php +++ b/apps/qubit/modules/search/actions/autocompleteAction.class.php @@ -76,7 +76,7 @@ public function execute($request) foreach ($items as $item) { $search = new \Elastica\Search($client); - foreach($indices as $type => $index) { + foreach ($indices as $type => $index) { $itemType = QubitSearch::getInstance()->index->getIndexTypeName($item['type']); $search->addIndex($index)->addType($index->getType($itemType)); } diff --git a/plugins/arElasticSearchPlugin/lib/arElasticSearchIndexDecorator.class.php b/plugins/arElasticSearchPlugin/lib/arElasticSearchIndexDecorator.class.php index d285786910..cbcb1e1f4d 100644 --- a/plugins/arElasticSearchPlugin/lib/arElasticSearchIndexDecorator.class.php +++ b/plugins/arElasticSearchPlugin/lib/arElasticSearchIndexDecorator.class.php @@ -30,45 +30,53 @@ public function __construct($prefix) $this->_indexPrefix = $prefix; } - public function createIndex($typeName, Elastica\Index $index) { + public function createIndex($typeName, Elastica\Index $index) + { $typeName = $this->getIndexTypeName($typeName); $this->_instance[$typeName] = $index; } // Converts camelized Qubit class names to lower case index name used for ElasticSearch - public function getIndexTypeName($typeName) { - return $this->_indexPrefix . '_' . strtolower($typeName); + public function getIndexTypeName($typeName) + { + return $this->_indexPrefix.'_'.strtolower($typeName); } - public function delete() { + public function delete() + { foreach ($this->_instance as $index) { $index->delete(); } } - public function addDocuments($typeName, $documents) { + public function addDocuments($typeName, $documents) + { $typeName = $this->getIndexTypeName($typeName); $this->_instance[$typeName]->addDocuments($documents); } - public function deleteDocuments($typeName, $documents) { + public function deleteDocuments($typeName, $documents) + { $typeName = $this->getIndexTypeName($typeName); $this->_instance[$typeName]->deleteDocuments($documents); } - public function refresh() { + public function refresh() + { foreach ($this->_instance as $index) { $index->refresh(); } } - public function getType($typeName) { + public function getType($typeName) + { $typeName = $this->getIndexTypeName($typeName); return $this->_instance[$typeName]; } - public function getInstance() { + public function getInstance() + { return $this->_instance; } } diff --git a/plugins/arElasticSearchPlugin/lib/arElasticSearchPluginUtil.class.php b/plugins/arElasticSearchPlugin/lib/arElasticSearchPluginUtil.class.php index 62485f9698..8fafdf71d9 100644 --- a/plugins/arElasticSearchPlugin/lib/arElasticSearchPluginUtil.class.php +++ b/plugins/arElasticSearchPlugin/lib/arElasticSearchPluginUtil.class.php @@ -483,7 +483,7 @@ protected static function getAllObjectStringFields( elseif ( ( !isset($propertyProperties['copy_to']) - || ($propertyProperties['copy_to'] == '_all') + || ('_all' == $propertyProperties['copy_to']) ) && ( isset($propertyProperties['type']) && 'text' == $propertyProperties['type'] From 5575f96cfc69dc0ecbc25fd601431846702bd6ce Mon Sep 17 00:00:00 2001 From: Anvit Srivastav Date: Wed, 16 Oct 2024 13:16:25 -0700 Subject: [PATCH 09/14] Switch elasticsearch to the oss docker image --- docker/docker-compose.dev.yml | 2 +- docker/etc/environment | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index b3545284e7..fee1e3fcbb 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -43,7 +43,7 @@ services: - "63001:80" elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:6.8.23 + image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.23 env_file: etc/environment ulimits: memlock: diff --git a/docker/etc/environment b/docker/etc/environment index 104549117b..a7ec4cc84f 100644 --- a/docker/etc/environment +++ b/docker/etc/environment @@ -1,6 +1,5 @@ # Elasticsearch bootstrap.memory_lock=true -xpack.security.enabled=false cluster.routing.allocation.disk.threshold_enabled=false ES_JAVA_OPTS=-Xms640m -Xmx640m From 0259ed052eb7a2a4401363aae70ce0739daddf85 Mon Sep 17 00:00:00 2001 From: Anvit Srivastav Date: Thu, 14 Nov 2024 11:19:34 -0800 Subject: [PATCH 10/14] Update mapping condition for _all fields Update condition in arElasticSearchPluginUtil to only get string fields included in _all if copy_to is explicitly set to _all in the mappping. --- .../lib/arElasticSearchPluginUtil.class.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/arElasticSearchPlugin/lib/arElasticSearchPluginUtil.class.php b/plugins/arElasticSearchPlugin/lib/arElasticSearchPluginUtil.class.php index 8fafdf71d9..7ce541e3ee 100644 --- a/plugins/arElasticSearchPlugin/lib/arElasticSearchPluginUtil.class.php +++ b/plugins/arElasticSearchPlugin/lib/arElasticSearchPluginUtil.class.php @@ -482,8 +482,8 @@ protected static function getAllObjectStringFields( // Get string fields included in _all elseif ( ( - !isset($propertyProperties['copy_to']) - || ('_all' == $propertyProperties['copy_to']) + isset($propertyProperties['copy_to']) + && ('_all' == $propertyProperties['copy_to']) ) && ( isset($propertyProperties['type']) && 'text' == $propertyProperties['type'] From a13821d6ced7d1287e373f2d36b8d17a7c18d320 Mon Sep 17 00:00:00 2001 From: Anvit Srivastav Date: Mon, 18 Nov 2024 13:35:39 -0800 Subject: [PATCH 11/14] Update ES mapping to copy relevant fields to _all --- .../arElasticSearchPlugin/config/mapping.yml | 386 +++++++++--------- 1 file changed, 193 insertions(+), 193 deletions(-) diff --git a/plugins/arElasticSearchPlugin/config/mapping.yml b/plugins/arElasticSearchPlugin/config/mapping.yml index a84317c157..5b16a49765 100644 --- a/plugins/arElasticSearchPlugin/config/mapping.yml +++ b/plugins/arElasticSearchPlugin/config/mapping.yml @@ -32,11 +32,11 @@ mapping: nested_only: true dynamic: strict properties: - contact_person: { type: keyword } + contact_person: { type: keyword, copy_to: _all } street_address: { type: text, index: false } - postal_code: { type: text } - country_code: { type: keyword } - location: { type: geo_point } + postal_code: { type: text, copy_to: _all } + country_code: { type: keyword, copy_to: _all } + location: { type: geo_point, copy_to: _all } other_name: _attributes: @@ -61,10 +61,10 @@ mapping: dynamic: strict type: nested properties: - start_date: { type: date } - end_date: { type: date } - start_date_string: { type: keyword } - end_date_string: { type: keyword } + start_date: { type: date, copy_to: _all } + end_date: { type: date, copy_to: _all } + start_date_string: { type: keyword, copy_to: _all } + end_date_string: { type: keyword, copy_to: _all } type_id: { type: integer } actor_id: { type: integer } @@ -91,8 +91,8 @@ mapping: dynamic: strict dynamic: strict properties: - date: { type: date } - date_string: { type: keyword } + date: { type: date, copy_to: _all } + date_string: { type: keyword, copy_to: _all } donor: _attributes: @@ -102,111 +102,111 @@ mapping: _foreign_types: { contact_informations: contact_information } dynamic: strict properties: - slug: { type: keyword } + slug: { type: keyword, copy_to: _all } basis_right: _attributes: nested_only: true dynamic: strict properties: - basis: { type: text } - start_date: { type: date } - end_date: { type: date } - copyright_status: { type: text } - rights_holder: { type: text } - rights_note: { type: text } - license_terms: { type: text } + basis: { type: text, copy_to: _all } + start_date: { type: date, copy_to: _all } + end_date: { type: date, copy_to: _all } + copyright_status: { type: text, copy_to: _all } + rights_holder: { type: text, copy_to: _all } + rights_note: { type: text, copy_to: _all } + license_terms: { type: text, copy_to: _all } act_right: _attributes: nested_only: true dynamic: strict properties: - act: { type: text } - restriction: { type: text } - start_date: { type: date } - end_date: { type: date } + act: { type: text, copy_to: _all } + restriction: { type: text, copy_to: _all } + start_date: { type: date, copy_to: _all } + end_date: { type: date, copy_to: _all } mediainfo_track: _attributes: nested_only: true dynamic: strict properties: - count: { type: integer } - video_format_list: { type: keyword } - video_format_with_hint_list: { type: keyword } - codecs_video: { type: keyword } - video_language_list: { type: keyword } - audio_format_list: { type: keyword } - audio_format_with_hint_list: { type: keyword } - audio_codecs: { type: keyword } - audio_language_list: { type: keyword } - complete_name: { type: keyword } - file_name: { type: keyword } - file_extension: { type: keyword } - format: { type: keyword } - format_info: { type: keyword } - format_url: { type: keyword } - format_profile: { type: keyword } - format_settings: { type: keyword } - format_settings_cabac: { type: keyword } + count: { type: integer, copy_to: _all } + video_format_list: { type: keyword, copy_to: _all } + video_format_with_hint_list: { type: keyword, copy_to: _all } + codecs_video: { type: keyword, copy_to: _all } + video_language_list: { type: keyword, copy_to: _all } + audio_format_list: { type: keyword, copy_to: _all } + audio_format_with_hint_list: { type: keyword, copy_to: _all } + audio_codecs: { type: keyword, copy_to: _all } + audio_language_list: { type: keyword, copy_to: _all } + complete_name: { type: keyword, copy_to: _all } + file_name: { type: keyword, copy_to: _all } + file_extension: { type: keyword, copy_to: _all } + format: { type: keyword, copy_to: _all } + format_info: { type: keyword, copy_to: _all } + format_url: { type: keyword, copy_to: _all } + format_profile: { type: keyword, copy_to: _all } + format_settings: { type: keyword, copy_to: _all } + format_settings_cabac: { type: keyword, copy_to: _all } format_settings_re_frames: { type: integer } - format_settings_gop: { type: keyword } - format_extensions_usually_used: { type: keyword } - commercial_name: { type: keyword } - internet_media_type: { type: keyword } - codec_id: { type: keyword } - codec_id_info: { type: keyword } - codec_id_url: { type: keyword } - codec: { type: keyword } - codec_family: { type: keyword } - codec_info: { type: keyword } - codec_url: { type: keyword } - codec_cc: { type: keyword } - codec_profile: { type: keyword } - codec_settings: { type: keyword } - codec_settings_cabac: { type: keyword } - codec_settings_ref_frames: { type: integer } - codec_extensions_usually_used: { type: keyword } - file_size: { type: long } - duration: { type: integer } - bit_rate: { type: integer } - bit_rate_mode: { type: keyword } - overall_bit_rate: { type: integer } - channels: { type: integer } - channel_positions: { type: keyword } - sampling_rate: { type: integer } - samples_count: { type: integer } - compression_mode: { type: keyword } - width: { type: integer } - height: { type: integer } - pixel_aspect_ratio: { type: float } - display_aspect_ratio: { type: keyword } - rotation: { type: float } - frame_rate_mode: { type: keyword } - frame_rate: { type: float } - frame_count: { type: integer } - resolution: { type: integer } - colorimetry: { type: keyword } - color_space: { type: keyword } - chroma_subsampling: { type: keyword } - bit_depth: { type: integer } - scan_type: { type: keyword } - interlacement: { type: keyword } - bits_pixel_frame: { type: float } - stream_size: { type: long } - proportion_of_this_stream: { type: float } - header_size: { type: long } - data_size: { type: long } - footer_size: { type: long } - language: { type: keyword } - color_primaries: { type: keyword } - transfer_characteristics: { type: keyword } - matrix_coefficients: { type: keyword } - is_streamable: { type: boolean } - writing_application: { type: keyword } - file_last_modification_date: { type: date } - file_last_modification_date_local: { type: date } + format_settings_gop: { type: keyword, copy_to: _all } + format_extensions_usually_used: { type: keyword, copy_to: _all } + commercial_name: { type: keyword, copy_to: _all } + internet_media_type: { type: keyword, copy_to: _all } + codec_id: { type: keyword, copy_to: _all } + codec_id_info: { type: keyword, copy_to: _all } + codec_id_url: { type: keyword, copy_to: _all } + codec: { type: keyword, copy_to: _all } + codec_family: { type: keyword, copy_to: _all } + codec_info: { type: keyword, copy_to: _all } + codec_url: { type: keyword, copy_to: _all } + codec_cc: { type: keyword, copy_to: _all } + codec_profile: { type: keyword, copy_to: _all } + codec_settings: { type: keyword, copy_to: _all } + codec_settings_cabac: { type: keyword, copy_to: _all } + codec_settings_ref_frames: { type: integer, copy_to: _all } + codec_extensions_usually_used: { type: keyword, copy_to: _all } + file_size: { type: long, copy_to: _all } + duration: { type: integer, copy_to: _all } + bit_rate: { type: integer, copy_to: _all } + bit_rate_mode: { type: keyword, copy_to: _all } + overall_bit_rate: { type: integer, copy_to: _all } + channels: { type: integer, copy_to: _all } + channel_positions: { type: keyword, copy_to: _all } + sampling_rate: { type: integer, copy_to: _all } + samples_count: { type: integer, copy_to: _all } + compression_mode: { type: keyword, copy_to: _all } + width: { type: integer, copy_to: _all } + height: { type: integer, copy_to: _all } + pixel_aspect_ratio: { type: float, copy_to: _all } + display_aspect_ratio: { type: keyword, copy_to: _all } + rotation: { type: float, copy_to: _all } + frame_rate_mode: { type: keyword, copy_to: _all } + frame_rate: { type: float, copy_to: _all } + frame_count: { type: integer, copy_to: _all } + resolution: { type: integer, copy_to: _all } + colorimetry: { type: keyword, copy_to: _all } + color_space: { type: keyword, copy_to: _all } + chroma_subsampling: { type: keyword, copy_to: _all } + bit_depth: { type: integer, copy_to: _all } + scan_type: { type: keyword, copy_to: _all } + interlacement: { type: keyword, copy_to: _all } + bits_pixel_frame: { type: float, copy_to: _all } + stream_size: { type: long, copy_to: _all } + proportion_of_this_stream: { type: float, copy_to: _all } + header_size: { type: long, copy_to: _all } + data_size: { type: long, copy_to: _all } + footer_size: { type: long, copy_to: _all } + language: { type: keyword, copy_to: _all } + color_primaries: { type: keyword, copy_to: _all } + transfer_characteristics: { type: keyword, copy_to: _all } + matrix_coefficients: { type: keyword, copy_to: _all } + is_streamable: { type: boolean, copy_to: _all } + writing_application: { type: keyword, copy_to: _all } + file_last_modification_date: { type: date, copy_to: _all } + file_last_modification_date_local: { type: date, copy_to: _all } mediainfo: _attributes: @@ -223,84 +223,84 @@ mapping: mediainfo: mediainfo dynamic: strict properties: - puid: { type: keyword } - filename: { type: keyword } - last_modified: { type: date } - date_ingested: { type: date } - size: { type: long } - mime_type: { type: keyword } + puid: { type: keyword, copy_to: _all } + filename: { type: keyword, copy_to: _all } + last_modified: { type: date, copy_to: _all } + date_ingested: { type: date, copy_to: _all } + size: { type: long, copy_to: _all } + mime_type: { type: keyword, copy_to: _all } audio: type: object properties: - bit_depth: { type: integer } - sample_rate: { type: keyword } - channels: { type: integer } - data_encoding: { type: keyword } - offset: { type: integer } - byte_order: { type: keyword } + bit_depth: { type: integer, copy_to: _all } + sample_rate: { type: keyword, copy_to: _all } + channels: { type: integer, copy_to: _all } + data_encoding: { type: keyword, copy_to: _all } + offset: { type: integer, copy_to: _all } + byte_order: { type: keyword, copy_to: _all } document: type: object properties: - title: { type: keyword } - author: { type: keyword } - page_count: { type: integer } - word_count: { type: integer } - character_count: { type: integer } - language: { type: keyword } - is_protected: { type: boolean } - is_rights_managed: { type: boolean } - is_tagged: { type: boolean } - has_outline: { type: boolean } - has_annotations: { type: boolean } - has_forms: { type: boolean } + title: { type: keyword, copy_to: _all } + author: { type: keyword, copy_to: _all } + page_count: { type: integer, copy_to: _all } + word_count: { type: integer, copy_to: _all } + character_count: { type: integer, copy_to: _all } + language: { type: keyword, copy_to: _all } + is_protected: { type: boolean, copy_to: _all } + is_rights_managed: { type: boolean, copy_to: _all } + is_tagged: { type: boolean, copy_to: _all } + has_outline: { type: boolean, copy_to: _all } + has_annotations: { type: boolean, copy_to: _all } + has_forms: { type: boolean, copy_to: _all } text: type: object properties: - linebreak: { type: keyword } - charset: { type: keyword } - markup_basis: { type: keyword } - markup_basis_version: { type: keyword } - markup_language: { type: keyword } + linebreak: { type: keyword, copy_to: _all } + charset: { type: keyword, copy_to: _all } + markup_basis: { type: keyword, copy_to: _all } + markup_basis_version: { type: keyword, copy_to: _all } + markup_language: { type: keyword, copy_to: _all } format_identification_event: type: object properties: - type: { type: keyword } - dateTime: { type: date } - detail: { type: keyword } - outcome: { type: keyword } - outcomeDetailNote: { type: keyword } + type: { type: keyword, copy_to: _all } + dateTime: { type: date, copy_to: _all } + detail: { type: keyword, copy_to: _all } + outcome: { type: keyword, copy_to: _all } + outcomeDetailNote: { type: keyword, copy_to: _all } linkingAgentIdentifier: type: object properties: - type: { type: keyword } - value: { type: keyword } + type: { type: keyword, copy_to: _all } + value: { type: keyword, copy_to: _all } other_events: type: object properties: - type: { type: keyword } - dateTime: { type: date } - detail: { type: keyword } - outcome: { type: keyword } - outcomeDetailNote: { type: keyword } + type: { type: keyword, copy_to: _all } + dateTime: { type: date, copy_to: _all } + detail: { type: keyword, copy_to: _all } + outcome: { type: keyword, copy_to: _all } + outcomeDetailNote: { type: keyword, copy_to: _all } linkingAgentIdentifier: type: object properties: - type: { type: keyword } - value: { type: keyword } + type: { type: keyword, copy_to: _all } + value: { type: keyword, copy_to: _all } agents: type: object properties: - identifier_type: { type: keyword } - identifier_value: { type: keyword } - name: { type: keyword } - type: { type: keyword } + identifier_type: { type: keyword, copy_to: _all } + identifier_value: { type: keyword, copy_to: _all } + name: { type: keyword, copy_to: _all } + type: { type: keyword, copy_to: _all } format: type: object properties: - name: { type: keyword } - version: { type: keyword } - registry_name: { type: keyword } - registry_key: { type: keyword } + name: { type: keyword, copy_to: _all } + version: { type: keyword, copy_to: _all } + registry_name: { type: keyword, copy_to: _all } + registry_key: { type: keyword, copy_to: _all } physical_object: _attributes: @@ -334,11 +334,11 @@ mapping: digital_objects: premis_object dynamic: strict properties: - uuid: { type: keyword } - filename: { type: keyword } - size_on_disk: { type: long } - digital_object_count: { type: integer } - created_at: { type: date } + uuid: { type: keyword, copy_to: _all } + filename: { type: keyword, copy_to: _all } + size_on_disk: { type: long, copy_to: _all } + digital_object_count: { type: integer, copy_to: _all } + created_at: { type: date, copy_to: _all } term: _attributes: @@ -350,10 +350,10 @@ mapping: _foreign_types: { use_for: other_name, scope_notes: note } dynamic: strict properties: - slug: { type: keyword } + slug: { type: keyword, copy_to: _all } taxonomy_id: { type: integer } - is_protected: { type: boolean } - number_of_descendants: { type: integer } + is_protected: { type: boolean, copy_to: _all } + number_of_descendants: { type: integer, copy_to: _all } actor: _attributes: @@ -378,27 +378,27 @@ mapping: id: { type: integer } dynamic: strict properties: - slug: { type: keyword } + slug: { type: keyword, copy_to: _all } description_identifier: type: text fields: untouched: type: keyword - corporate_body_identifiers: { type: keyword } + corporate_body_identifiers: { type: keyword, copy_to: _all } entity_type_id: { type: integer } maintaining_repository_id: { type: integer } - direct_subjects: { type: integer } - direct_places: { type: integer } - actor_direct_relation_types: { type: integer } - has_digital_object: { type: boolean } + direct_subjects: { type: integer, copy_to: _all } + direct_places: { type: integer, copy_to: _all } + actor_direct_relation_types: { type: integer, copy_to: _all } + has_digital_object: { type: boolean, copy_to: _all } digital_object: type: object properties: media_type_id: { type: integer } usage_id: { type: integer } - thumbnail_path: { type: keyword } - filename: { type: text } - digital_object_alt_text: { type: text } + thumbnail_path: { type: keyword, copy_to: _all } + filename: { type: text, copy_to: _all } + digital_object_alt_text: { type: text, copy_to: _all } accession: _attributes: @@ -422,13 +422,13 @@ mapping: donors: donor dynamic: strict properties: - slug: { type: keyword } + slug: { type: keyword, copy_to: _all } identifier: type: text fields: untouched: type: keyword - date: { type: date } + date: { type: date, copy_to: _all } repository: _attributes: @@ -444,16 +444,16 @@ mapping: parallel_names: other_name dynamic: strict properties: - slug: { type: keyword } + slug: { type: keyword, copy_to: _all } identifier: type: text fields: untouched: type: keyword - types: { type: integer } - geographic_subregions: { type: integer } - thematic_areas: { type: integer } - logo_path: { type: keyword } + types: { type: integer, copy_to: _all } + geographic_subregions: { type: integer, copy_to: _all } + thematic_areas: { type: integer, copy_to: _all } + logo_path: { type: keyword, copy_to: _all } function_object: _attributes: @@ -465,7 +465,7 @@ mapping: parallel_names: other_name dynamic: strict properties: - slug: { type: keyword } + slug: { type: keyword, copy_to: _all } description_status_id: { type: integer } description_detail_id: { type: integer } description_identifier: @@ -513,15 +513,15 @@ mapping: dynamic: strict properties: id: { type: integer } - slug: { type: keyword } + slug: { type: keyword, copy_to: _all } repository: _i18nFields: [authorizedFormOfName] _rawFields: [authorizedFormOfName] dynamic: strict properties: id: { type: integer } - slug: { type: keyword } - identifier: { type: keyword } + slug: { type: keyword, copy_to: _all } + identifier: { type: keyword, copy_to: _all } names: _i18nFields: [authorizedFormOfName] dynamic: strict @@ -580,44 +580,44 @@ mapping: publication_notes: note dynamic: strict properties: - slug: { type: keyword } + slug: { type: keyword, copy_to: _all } identifier: type: text fields: untouched: type: keyword - reference_code_without_country_and_repo: { type: keyword } - level_of_description_id: { type: integer } - lft: { type: integer } + reference_code_without_country_and_repo: { type: keyword, copy_to: _all } + level_of_description_id: { type: integer, copy_to: _all } + lft: { type: integer, copy_to: _all } publication_status_id: { type: integer } parent_id: { type: integer } - ancestors: { type: integer } - children: { type: integer } + ancestors: { type: integer, copy_to: _all } + children: { type: integer, copy_to: _all } copyright_status_id: { type: integer } material_type_id: { type: integer } transcript: { type: text, copy_to: _all } - direct_subjects: { type: integer } - direct_places: { type: integer } - direct_genres: { type: integer } - has_digital_object: { type: boolean } + direct_subjects: { type: integer, copy_to: _all } + direct_places: { type: integer, copy_to: _all } + direct_genres: { type: integer, copy_to: _all } + has_digital_object: { type: boolean, copy_to: _all } finding_aid: type: object properties: transcript: { type: text, copy_to: _all } - status: { type: integer } + status: { type: integer, copy_to: _all } digital_object: type: object properties: media_type_id: { type: integer } usage_id: { type: integer } - thumbnail_path: { type: keyword } - filename: { type: text } - digital_object_alt_text: { type: text } + thumbnail_path: { type: keyword, copy_to: _all } + filename: { type: text, copy_to: _all } + digital_object_alt_text: { type: text, copy_to: _all } alternative_identifiers: type: object properties: - label: { type: text } - identifier: { type: text } + label: { type: text, copy_to: _all } + identifier: { type: text, copy_to: _all } reference_code: type: text fields: @@ -630,5 +630,5 @@ mapping: search_analyzer: standard term_vector: with_positions_offsets # Not nested date fields for sorting - start_date_sort: { type: date } - end_date_sort: { type: date } + start_date_sort: { type: date, copy_to: _all } + end_date_sort: { type: date, copy_to: _all } From 3f0e4d6f1de61dc51389df1941082ba1fd77c4e2 Mon Sep 17 00:00:00 2001 From: Anvit Srivastav Date: Mon, 18 Nov 2024 15:28:34 -0800 Subject: [PATCH 12/14] Rename arElasticSearchIndexDecorator Renamed arElasticSearchIndexDecorator as arElasticSearchMultiIndexWrapper for clarity and added a few comments about changes needed for ElasticSearch/Elastica 7.x. --- ...rElasticSearchMultiIndexWrapper.class.php} | 35 +++++++++++-------- .../lib/arElasticSearchPlugin.class.php | 24 +++++++++---- 2 files changed, 38 insertions(+), 21 deletions(-) rename plugins/arElasticSearchPlugin/lib/{arElasticSearchIndexDecorator.class.php => arElasticSearchMultiIndexWrapper.class.php} (60%) diff --git a/plugins/arElasticSearchPlugin/lib/arElasticSearchIndexDecorator.class.php b/plugins/arElasticSearchPlugin/lib/arElasticSearchMultiIndexWrapper.class.php similarity index 60% rename from plugins/arElasticSearchPlugin/lib/arElasticSearchIndexDecorator.class.php rename to plugins/arElasticSearchPlugin/lib/arElasticSearchMultiIndexWrapper.class.php index cbcb1e1f4d..58249ed30e 100644 --- a/plugins/arElasticSearchPlugin/lib/arElasticSearchIndexDecorator.class.php +++ b/plugins/arElasticSearchPlugin/lib/arElasticSearchMultiIndexWrapper.class.php @@ -17,7 +17,12 @@ * along with Access to Memory (AtoM). If not, see . */ -class arElasticSearchIndexDecorator +/** + * arElasticSearchMultiIndexWrapper facilitates handling ElasticSearch indices so that + * all indices for an AtoM installation can share a common prefix without having to + * explicitly specify it in common indexing and search functions. + */ +class arElasticSearchMultiIndexWrapper { protected $_instance; @@ -30,16 +35,16 @@ public function __construct($prefix) $this->_indexPrefix = $prefix; } - public function createIndex($typeName, Elastica\Index $index) + public function createIndex($name, Elastica\Index $index) { - $typeName = $this->getIndexTypeName($typeName); - $this->_instance[$typeName] = $index; + $name = $this->getIndexName($name); + $this->_instance[$name] = $index; } // Converts camelized Qubit class names to lower case index name used for ElasticSearch - public function getIndexTypeName($typeName) + public function getIndexName($name) { - return $this->_indexPrefix.'_'.strtolower($typeName); + return $this->_indexPrefix.'_'.strtolower($name); } public function delete() @@ -49,16 +54,16 @@ public function delete() } } - public function addDocuments($typeName, $documents) + public function addDocuments($name, $documents) { - $typeName = $this->getIndexTypeName($typeName); - $this->_instance[$typeName]->addDocuments($documents); + $name = $this->getIndexName($name); + $this->_instance[$name]->addDocuments($documents); } - public function deleteDocuments($typeName, $documents) + public function deleteDocuments($name, $documents) { - $typeName = $this->getIndexTypeName($typeName); - $this->_instance[$typeName]->deleteDocuments($documents); + $name = $this->getIndexName($name); + $this->_instance[$name]->deleteDocuments($documents); } public function refresh() @@ -68,11 +73,11 @@ public function refresh() } } - public function getType($typeName) + public function getType($name) { - $typeName = $this->getIndexTypeName($typeName); + $name = $this->getIndexName($name); - return $this->_instance[$typeName]; + return $this->_instance[$name]; } public function getInstance() diff --git a/plugins/arElasticSearchPlugin/lib/arElasticSearchPlugin.class.php b/plugins/arElasticSearchPlugin/lib/arElasticSearchPlugin.class.php index efe67c1dea..56b9c83368 100644 --- a/plugins/arElasticSearchPlugin/lib/arElasticSearchPlugin.class.php +++ b/plugins/arElasticSearchPlugin/lib/arElasticSearchPlugin.class.php @@ -97,7 +97,7 @@ public function __construct(array $options = []) // Verify the version running in the server $this->checkVersion(); - $this->index = new arElasticSearchIndexDecorator($this->config['index']['name']); + $this->index = new arElasticSearchMultiIndexWrapper($this->config['index']['name']); // Load batch mode configuration $this->batchMode = true === $this->config['batch_mode']; @@ -348,7 +348,7 @@ public function addDocument($data, $type) unset($data['id']); $document = new \Elastica\Document($id, $data); - $document->setType($this->index->getIndexTypeName($type)); + $document->setType($this->index->getIndexName($type)); if (!$this->currentBatchType) { $this->currentBatchType = $type; @@ -397,7 +397,7 @@ public function partialUpdate($object, $data) $type = get_class($object); $document = new \Elastica\Document($object->id, $data); - $document->setType($this->index->getIndexTypeName($type)); + $document->setType($this->index->getIndexName($type)); try { $this->index->getType($type)->updateDocuments([$document]); @@ -456,7 +456,7 @@ public function delete($object) // the document to be deleted and add this document object to the batch delete // queue. $document = new \Elastica\Document($object->id); - $document->setType($this->index->getIndexTypeName($type)); + $document->setType($this->index->getIndexName($type)); if ($this->currentBatchType != $type) { $this->flushBatch(); @@ -530,7 +530,7 @@ protected function initialize() foreach ($this->mappings as $typeName => $typeProperties) { $typeName = 'Qubit'.sfInflector::camelize($typeName); $this->index->createIndex($typeName, - $this->client->getIndex($this->index->getIndexTypeName($typeName)) + $this->client->getIndex($this->index->getIndexName($typeName)) ); } @@ -560,6 +560,10 @@ protected function initialize() } } + // In ES 7.x if the mapping type is updated to a dummy type, + // this may need to include a param for include_type_name + // set to false in order to avoid automatically creating a + // type for the index that was just created $index->create( $this->config['index']['configuration'], ['recreate' => true] @@ -573,12 +577,14 @@ protected function initialize() foreach ($this->mappings as $typeName => $typeProperties) { $typeName = 'Qubit'.sfInflector::camelize($typeName); - if ($indexType != $this->index->getIndexTypeName($typeName)) { + if ($indexType != $this->index->getIndexName($typeName)) { continue; } // Define mapping in elasticsearch $mapping = new \Elastica\Type\Mapping(); + + // This type will need to be changed to a dummy type like _doc in 7.x $mapping->setType($index->getType($indexType)); $mapping->setProperties($typeProperties['properties']); @@ -589,6 +595,12 @@ protected function initialize() } $this->log(sprintf('Defining mapping %s...', $typeName)); + + // In ES 7.x, if the mapping types are updated to a dummy type, + // this should be changed to: + // $mapping->send($index, [ 'include_type_name' => false ]) + // which can be removed in 8.x since that is the default behaviour + // and will have be removed by 9.x when it is discontinued $mapping->send(); } } From 0e0d87d0fe7a4370c942f1ac45dc4e12b2d38e80 Mon Sep 17 00:00:00 2001 From: Anvit Srivastav Date: Mon, 18 Nov 2024 15:43:28 -0800 Subject: [PATCH 13/14] Rename getType function from ESMultiIndexWrapper Rename getType from arElasticSearchMultiIndexWrapper to getIndex for clarity. --- .../qubit/modules/actor/actions/browseAction.class.php | 2 +- .../actions/relatedInformationObjectsAction.class.php | 2 +- .../modules/clipboard/actions/viewAction.class.php | 2 +- .../modules/default/actions/browseAction.class.php | 2 +- .../default/actions/fullTreeViewAction.class.php | 4 ++-- .../qubit/modules/default/actions/moveAction.class.php | 4 ++-- .../digitalobject/actions/imageflowComponent.class.php | 2 +- .../actions/autocompleteAction.class.php | 2 +- .../informationobject/actions/browseAction.class.php | 2 +- .../actions/inventoryAction.class.php | 2 +- .../modules/repository/actions/browseAction.class.php | 4 ++-- .../repository/actions/holdingsAction.class.php | 2 +- .../actions/maintainedActorsAction.class.php | 2 +- .../search/actions/autocompleteAction.class.php | 5 ++++- .../search/actions/descriptionUpdatesAction.class.php | 2 +- .../qubit/modules/search/actions/indexAction.class.php | 2 +- .../modules/taxonomy/actions/indexAction.class.php | 2 +- apps/qubit/modules/term/actions/indexAction.class.php | 4 ++-- .../term/actions/navigateRelatedComponent.class.php | 2 +- lib/QubitLftSyncer.class.php | 6 +++--- lib/job/arActorExportJob.class.php | 2 +- lib/job/arInformationObjectExportJob.class.php | 2 +- lib/job/arRepositoryCsvExportJob.class.php | 2 +- lib/job/arUpdateEsActorRelationsJob.class.php | 2 +- lib/job/arUpdatePublicationStatusJob.class.php | 2 +- lib/model/QubitInformationObject.php | 2 +- lib/model/QubitTerm.php | 2 +- lib/task/propel/propelGenerateSlugsTask.class.php | 2 +- lib/task/search/arDocumentTask.class.php | 2 +- lib/task/search/arSearchStatusTask.class.php | 2 +- lib/task/tools/updatePublicationStatusTask.class.php | 2 +- .../lib/arElasticSearchMultiIndexWrapper.class.php | 4 +++- .../lib/arElasticSearchPlugin.class.php | 10 +++++----- .../actions/informationobjectsBrowseAction.class.php | 2 +- .../modules/accession/actions/browseAction.class.php | 2 +- plugins/sfSkosPlugin/test/unit/importTest.php | 4 ++-- 36 files changed, 52 insertions(+), 47 deletions(-) diff --git a/apps/qubit/modules/actor/actions/browseAction.class.php b/apps/qubit/modules/actor/actions/browseAction.class.php index 4394ef6e53..11569c1131 100644 --- a/apps/qubit/modules/actor/actions/browseAction.class.php +++ b/apps/qubit/modules/actor/actions/browseAction.class.php @@ -499,7 +499,7 @@ protected function doSearch($request) $this->search->query->setQuery($this->search->queryBool); - return QubitSearch::getInstance()->index->getType('QubitActor')->search($this->search->getQuery(false)); + return QubitSearch::getInstance()->index->getIndex('QubitActor')->search($this->search->getQuery(false)); } private function getRelatedAuthorityUsingSlug($slug) diff --git a/apps/qubit/modules/actor/actions/relatedInformationObjectsAction.class.php b/apps/qubit/modules/actor/actions/relatedInformationObjectsAction.class.php index 9318d5f3d4..043d5cc3da 100644 --- a/apps/qubit/modules/actor/actions/relatedInformationObjectsAction.class.php +++ b/apps/qubit/modules/actor/actions/relatedInformationObjectsAction.class.php @@ -108,7 +108,7 @@ public static function getRelatedInformationObjects($actorId, $page, $limit, $ev $query->setSize($limit); $query->setFrom($limit * ($page - 1)); - return QubitSearch::getInstance()->index->getType('QubitInformationObject')->search($query); + return QubitSearch::getInstance()->index->getIndex('QubitInformationObject')->search($query); } public static function nestedActorAndEventTypeQuery($actorId, $eventTypeId) diff --git a/apps/qubit/modules/clipboard/actions/viewAction.class.php b/apps/qubit/modules/clipboard/actions/viewAction.class.php index 64c84ade89..d418b5db38 100644 --- a/apps/qubit/modules/clipboard/actions/viewAction.class.php +++ b/apps/qubit/modules/clipboard/actions/viewAction.class.php @@ -51,7 +51,7 @@ public function execute($request) $this->search->query->setQuery($this->search->queryBool); - $resultSet = QubitSearch::getInstance()->index->getType($this->entityType)->search($this->search->query); + $resultSet = QubitSearch::getInstance()->index->getIndex($this->entityType)->search($this->search->query); } // Page results diff --git a/apps/qubit/modules/default/actions/browseAction.class.php b/apps/qubit/modules/default/actions/browseAction.class.php index 2aa7a95dba..86a95bb297 100644 --- a/apps/qubit/modules/default/actions/browseAction.class.php +++ b/apps/qubit/modules/default/actions/browseAction.class.php @@ -146,7 +146,7 @@ protected function populateAggs($resultSet) $this->search->query->setRawQuery($queryParams); - $resultSetWithoutLanguageFilter = QubitSearch::getInstance()->index->getType($this::INDEX_TYPE)->search($this->search->query); + $resultSetWithoutLanguageFilter = QubitSearch::getInstance()->index->getIndex($this::INDEX_TYPE)->search($this->search->query); $count = $resultSetWithoutLanguageFilter->getTotalHits(); } diff --git a/apps/qubit/modules/default/actions/fullTreeViewAction.class.php b/apps/qubit/modules/default/actions/fullTreeViewAction.class.php index afda1afb1a..21ea57f501 100644 --- a/apps/qubit/modules/default/actions/fullTreeViewAction.class.php +++ b/apps/qubit/modules/default/actions/fullTreeViewAction.class.php @@ -175,7 +175,7 @@ protected function doElasticsearchQuery($term, $options = []) // Get results, with drafts filtered when appropriate return QubitSearch::getInstance() ->index - ->getType('QubitInformationObject') + ->getIndex('QubitInformationObject') ->search($query->getQuery(false, false)); } @@ -237,7 +237,7 @@ protected function countChildren($id, $options = []) // Return a count of the results found return QubitSearch::getInstance() ->index - ->getType('QubitInformationObject') + ->getIndex('QubitInformationObject') ->count($query->getQuery(false, false)); } diff --git a/apps/qubit/modules/default/actions/moveAction.class.php b/apps/qubit/modules/default/actions/moveAction.class.php index 793016f1de..7601729cb2 100644 --- a/apps/qubit/modules/default/actions/moveAction.class.php +++ b/apps/qubit/modules/default/actions/moveAction.class.php @@ -140,10 +140,10 @@ public function execute($request) $this->query->setQuery($this->queryBool); if ($this->resource instanceof QubitInformationObject) { - $resultSet = QubitSearch::getInstance()->index->getType('QubitInformationObject')->search($this->query); + $resultSet = QubitSearch::getInstance()->index->getIndex('QubitInformationObject')->search($this->query); } elseif ($this->resource instanceof QubitTerm) { // TODO: Add parent_id for terms in ES, add move button - $resultSet = QubitSearch::getInstance()->index->getType('QubitTerm')->search($this->query); + $resultSet = QubitSearch::getInstance()->index->getIndex('QubitTerm')->search($this->query); } // Page results diff --git a/apps/qubit/modules/digitalobject/actions/imageflowComponent.class.php b/apps/qubit/modules/digitalobject/actions/imageflowComponent.class.php index 87f3121d63..a517cf4913 100644 --- a/apps/qubit/modules/digitalobject/actions/imageflowComponent.class.php +++ b/apps/qubit/modules/digitalobject/actions/imageflowComponent.class.php @@ -125,7 +125,7 @@ protected function getDescendantDigitalObjectCount() $results = QubitSearch::getInstance() ->index - ->getType('QubitInformationObject') + ->getIndex('QubitInformationObject') ->search($search->getQuery(false, true)); return $results->getTotalHits(); diff --git a/apps/qubit/modules/informationobject/actions/autocompleteAction.class.php b/apps/qubit/modules/informationobject/actions/autocompleteAction.class.php index 95bdb95a4d..7b5823ac33 100644 --- a/apps/qubit/modules/informationobject/actions/autocompleteAction.class.php +++ b/apps/qubit/modules/informationobject/actions/autocompleteAction.class.php @@ -94,7 +94,7 @@ public function execute($request) $this->query->setQuery($this->queryBool); - $resultSet = QubitSearch::getInstance()->index->getType('QubitInformationObject')->search($this->query); + $resultSet = QubitSearch::getInstance()->index->getIndex('QubitInformationObject')->search($this->query); // Page results $this->pager = new QubitSearchPager($resultSet); diff --git a/apps/qubit/modules/informationobject/actions/browseAction.class.php b/apps/qubit/modules/informationobject/actions/browseAction.class.php index 4167cc5644..97a227af4e 100644 --- a/apps/qubit/modules/informationobject/actions/browseAction.class.php +++ b/apps/qubit/modules/informationobject/actions/browseAction.class.php @@ -244,7 +244,7 @@ public function execute($request) $this->setView($request); - $resultSet = QubitSearch::getInstance()->index->getType('QubitInformationObject')->search($this->search->getQuery(false, true)); + $resultSet = QubitSearch::getInstance()->index->getIndex('QubitInformationObject')->search($this->search->getQuery(false, true)); // Page results $this->pager = new QubitSearchPager($resultSet); diff --git a/apps/qubit/modules/informationobject/actions/inventoryAction.class.php b/apps/qubit/modules/informationobject/actions/inventoryAction.class.php index 5f7c1591e3..5832fb9e99 100644 --- a/apps/qubit/modules/informationobject/actions/inventoryAction.class.php +++ b/apps/qubit/modules/informationobject/actions/inventoryAction.class.php @@ -175,6 +175,6 @@ private static function getResults($resource, $limit = 10, $page = 1, $sort = nu QubitAclSearch::filterDrafts($queryBool); $query->setQuery($queryBool); - return QubitSearch::getInstance()->index->getType('QubitInformationObject')->search($query); + return QubitSearch::getInstance()->index->getIndex('QubitInformationObject')->search($query); } } diff --git a/apps/qubit/modules/repository/actions/browseAction.class.php b/apps/qubit/modules/repository/actions/browseAction.class.php index b17764ba72..6be51aaf5b 100644 --- a/apps/qubit/modules/repository/actions/browseAction.class.php +++ b/apps/qubit/modules/repository/actions/browseAction.class.php @@ -142,7 +142,7 @@ public function execute($request) $this->search->query->setQuery($this->search->queryBool); - $resultSet = QubitSearch::getInstance()->index->getType('QubitRepository')->search($this->search->query); + $resultSet = QubitSearch::getInstance()->index->getIndex('QubitRepository')->search($this->search->query); $this->pager = new QubitSearchPager($resultSet); $this->pager->setPage($request->page ? $request->page : 1); @@ -201,7 +201,7 @@ private function getAdvancedFilterTerms() $query = new \Elastica\Query(new \Elastica\Query\MatchAll()); $query->setSize($limit); - $this->repositories = QubitSearch::getInstance()->index->getType('QubitRepository')->search($query); + $this->repositories = QubitSearch::getInstance()->index->getIndex('QubitRepository')->search($query); } /** diff --git a/apps/qubit/modules/repository/actions/holdingsAction.class.php b/apps/qubit/modules/repository/actions/holdingsAction.class.php index 48d4d45618..23ed4d883a 100644 --- a/apps/qubit/modules/repository/actions/holdingsAction.class.php +++ b/apps/qubit/modules/repository/actions/holdingsAction.class.php @@ -91,6 +91,6 @@ public static function getHoldings($id, $page, $limit) $title = sprintf('i18n.%s.title.alphasort', sfContext::getInstance()->user->getCulture()); $query->setSort([$title => 'asc']); - return QubitSearch::getInstance()->index->getType('QubitInformationObject')->search($query); + return QubitSearch::getInstance()->index->getIndex('QubitInformationObject')->search($query); } } diff --git a/apps/qubit/modules/repository/actions/maintainedActorsAction.class.php b/apps/qubit/modules/repository/actions/maintainedActorsAction.class.php index d7d3c3b3cb..7ae3f6ee9f 100644 --- a/apps/qubit/modules/repository/actions/maintainedActorsAction.class.php +++ b/apps/qubit/modules/repository/actions/maintainedActorsAction.class.php @@ -82,6 +82,6 @@ public static function getActors($repositoryId, $page, $limit) $field = sprintf('i18n.%s.authorizedFormOfName.alphasort', sfContext::getInstance()->user->getCulture()); $query->setSort([$field => 'asc']); - return QubitSearch::getInstance()->index->getType('QubitActor')->search($query); + return QubitSearch::getInstance()->index->getIndex('QubitActor')->search($query); } } diff --git a/apps/qubit/modules/search/actions/autocompleteAction.class.php b/apps/qubit/modules/search/actions/autocompleteAction.class.php index ef9a40c45c..d462bb82bc 100644 --- a/apps/qubit/modules/search/actions/autocompleteAction.class.php +++ b/apps/qubit/modules/search/actions/autocompleteAction.class.php @@ -77,7 +77,10 @@ public function execute($request) foreach ($items as $item) { $search = new \Elastica\Search($client); foreach ($indices as $type => $index) { - $itemType = QubitSearch::getInstance()->index->getIndexTypeName($item['type']); + $itemType = QubitSearch::getInstance()->index->getIndexName($item['type']); + + // This will need to be updated in ES 7.x if it is updated to a dummy type, + // and then removed in ES 8.x when types are no longer required. $search->addIndex($index)->addType($index->getType($itemType)); } diff --git a/apps/qubit/modules/search/actions/descriptionUpdatesAction.class.php b/apps/qubit/modules/search/actions/descriptionUpdatesAction.class.php index 209edbdce6..4716538646 100644 --- a/apps/qubit/modules/search/actions/descriptionUpdatesAction.class.php +++ b/apps/qubit/modules/search/actions/descriptionUpdatesAction.class.php @@ -187,7 +187,7 @@ public function doSearch() $query->setFrom($limit * ($page - 1)); $query->setSort(['createdAt' => 'desc']); - $resultSet = QubitSearch::getInstance()->index->getType($this->form->getValue('className'))->search($query); + $resultSet = QubitSearch::getInstance()->index->getIndex($this->form->getValue('className'))->search($query); // Page results $this->pager = new QubitSearchPager($resultSet); diff --git a/apps/qubit/modules/search/actions/indexAction.class.php b/apps/qubit/modules/search/actions/indexAction.class.php index 9bb7d20782..bbd0143ed4 100644 --- a/apps/qubit/modules/search/actions/indexAction.class.php +++ b/apps/qubit/modules/search/actions/indexAction.class.php @@ -48,7 +48,7 @@ public function execute($request) QubitAclSearch::filterDrafts($this->search->queryBool); $this->search->query->setQuery($this->search->queryBool); - $resultSet = QubitSearch::getInstance()->index->getType('QubitInformationObject')->search($this->search->query); + $resultSet = QubitSearch::getInstance()->index->getIndex('QubitInformationObject')->search($this->search->query); $total = $resultSet->getTotalHits(); if (1 > $total) { diff --git a/apps/qubit/modules/taxonomy/actions/indexAction.class.php b/apps/qubit/modules/taxonomy/actions/indexAction.class.php index 1a9604327e..f17d3650cc 100644 --- a/apps/qubit/modules/taxonomy/actions/indexAction.class.php +++ b/apps/qubit/modules/taxonomy/actions/indexAction.class.php @@ -224,7 +224,7 @@ public function execute($request) $this->query->setSort(['updatedAt' => $request->sortDir]); } - $resultSet = QubitSearch::getInstance()->index->getType('QubitTerm')->search($this->query); + $resultSet = QubitSearch::getInstance()->index->getIndex('QubitTerm')->search($this->query); // Return special response in JSON for XHR requests if ($request->isXmlHttpRequest()) { diff --git a/apps/qubit/modules/term/actions/indexAction.class.php b/apps/qubit/modules/term/actions/indexAction.class.php index 3773825054..45ae9c7d9c 100644 --- a/apps/qubit/modules/term/actions/indexAction.class.php +++ b/apps/qubit/modules/term/actions/indexAction.class.php @@ -248,7 +248,7 @@ public function execute($request) QubitAclSearch::filterDrafts($this->search->queryBool); $this->search->query->setQuery($this->search->queryBool); - $resultSet = QubitSearch::getInstance()->index->getType('QubitInformationObject')->search($this->search->query); + $resultSet = QubitSearch::getInstance()->index->getIndex('QubitInformationObject')->search($this->search->query); // Page results $this->pager = new QubitSearchPager($resultSet); @@ -328,7 +328,7 @@ protected function loadListTerms($request) $listQueryBool->addMust(new \Elastica\Query\Term(['taxonomyId' => $this->resource->taxonomyId])); $listQuery->setQuery($listQueryBool); - $this->listResultSet = QubitSearch::getInstance()->index->getType('QubitTerm')->search($listQuery); + $this->listResultSet = QubitSearch::getInstance()->index->getIndex('QubitTerm')->search($listQuery); // Page list results $this->listPager = new QubitSearchPager($this->listResultSet); diff --git a/apps/qubit/modules/term/actions/navigateRelatedComponent.class.php b/apps/qubit/modules/term/actions/navigateRelatedComponent.class.php index 2d9ba45c82..1000797866 100644 --- a/apps/qubit/modules/term/actions/navigateRelatedComponent.class.php +++ b/apps/qubit/modules/term/actions/navigateRelatedComponent.class.php @@ -62,7 +62,7 @@ public static function getEsDocsRelatedToTerm($relatedModelClass, $term, $option QubitAclSearch::filterDrafts($search->queryBool); } - return QubitSearch::getInstance()->index->getType($relatedModelClass)->search($search->getQuery(false)); + return QubitSearch::getInstance()->index->getIndex($relatedModelClass)->search($search->getQuery(false)); } public static function getEsDocsRelatedToTermCount($relatedModelClass, $termId, $search = null) diff --git a/lib/QubitLftSyncer.class.php b/lib/QubitLftSyncer.class.php index dda1428e1c..f8494137fe 100644 --- a/lib/QubitLftSyncer.class.php +++ b/lib/QubitLftSyncer.class.php @@ -68,7 +68,7 @@ private function getChildLftChecksumForElasticsearch() // Get results $result = QubitSearch::getInstance() ->index - ->getType('QubitInformationObject') + ->getIndex('QubitInformationObject') ->search($query->getQuery(false, false)); // Amalgamate lft values in array @@ -93,8 +93,8 @@ private function repairEsChildrenLftValues() $bulk = new Elastica\Bulk(QubitSearch::getInstance()->client); $type = 'QubitInformationObject'; - $bulk->setIndex(QubitSearch::getInstance()->index->getType($type)->getName()); - $bulk->setType(QubitSearch::getInstance()->index->getIndexTypeName($type)); + $bulk->setIndex(QubitSearch::getInstance()->index->getIndex($type)->getName()); + $bulk->setType(QubitSearch::getInstance()->index->getIndexName($type)); foreach ($results as $row) { $bulk->addAction( diff --git a/lib/job/arActorExportJob.class.php b/lib/job/arActorExportJob.class.php index 2abafc151a..5e7e71aaa9 100644 --- a/lib/job/arActorExportJob.class.php +++ b/lib/job/arActorExportJob.class.php @@ -46,7 +46,7 @@ public static function findExportRecords($parameters) return QubitSearch::getInstance() ->index - ->getType('QubitActor') + ->getIndex('QubitActor') ->createSearch($query->getQuery(false, false)); } diff --git a/lib/job/arInformationObjectExportJob.class.php b/lib/job/arInformationObjectExportJob.class.php index ab2a6d8e70..ee2c8fb710 100644 --- a/lib/job/arInformationObjectExportJob.class.php +++ b/lib/job/arInformationObjectExportJob.class.php @@ -64,7 +64,7 @@ public static function findExportRecords($parameters) return QubitSearch::getInstance() ->index - ->getType('QubitInformationObject') + ->getIndex('QubitInformationObject') ->createSearch($query->getQuery(false, false)); } diff --git a/lib/job/arRepositoryCsvExportJob.class.php b/lib/job/arRepositoryCsvExportJob.class.php index ecd987862a..0f141385c4 100644 --- a/lib/job/arRepositoryCsvExportJob.class.php +++ b/lib/job/arRepositoryCsvExportJob.class.php @@ -85,7 +85,7 @@ protected function exportResults($path) { $itemsExported = 0; - $search = QubitSearch::getInstance()->index->getType('QubitRepository')->createSearch($this->search->getQuery(false, false)); + $search = QubitSearch::getInstance()->index->getIndex('QubitRepository')->createSearch($this->search->getQuery(false, false)); $writer = new csvRepositoryExport($path, null, 10000); $writer->loadResourceSpecificConfiguration('QubitRepository'); diff --git a/lib/job/arUpdateEsActorRelationsJob.class.php b/lib/job/arUpdateEsActorRelationsJob.class.php index d768a80199..941678cbc4 100644 --- a/lib/job/arUpdateEsActorRelationsJob.class.php +++ b/lib/job/arUpdateEsActorRelationsJob.class.php @@ -102,7 +102,7 @@ public static function previousRelationActorIds($actorId) { try { // Get actor's previously indexed relations from Elasticsearch - $doc = QubitSearch::getInstance()->index->getType('QubitActor')->getDocument($actorId); + $doc = QubitSearch::getInstance()->index->getIndex('QubitActor')->getDocument($actorId); return self::uniqueIdsFromRelationData($doc->getData()['actorRelations']); } catch (\Elastica\Exception\NotFoundException $e) { diff --git a/lib/job/arUpdatePublicationStatusJob.class.php b/lib/job/arUpdatePublicationStatusJob.class.php index b0e733a7b4..5a79758d8c 100644 --- a/lib/job/arUpdatePublicationStatusJob.class.php +++ b/lib/job/arUpdatePublicationStatusJob.class.php @@ -92,7 +92,7 @@ public function runJob($parameters) ]; $type = 'QubitInformationObject'; - $response = QubitSearch::getInstance()->index->getType($type)->updateByQuery($query, $queryScript, $options)->getData(); + $response = QubitSearch::getInstance()->index->getIndex($type)->updateByQuery($query, $queryScript, $options)->getData(); $message = $this->i18n->__( 'Index update completed in %1 ms.', diff --git a/lib/model/QubitInformationObject.php b/lib/model/QubitInformationObject.php index 65e6fef4a8..17da4ff220 100644 --- a/lib/model/QubitInformationObject.php +++ b/lib/model/QubitInformationObject.php @@ -2184,7 +2184,7 @@ public static function getByTitleIdentifierAndRepo($identifier, $title, $repoNam $query = new \Elastica\Query($queryBool); $query->setSize(1); - $resultSet = QubitSearch::getInstance()->index->getType('QubitInformationObject')->search($query); + $resultSet = QubitSearch::getInstance()->index->getIndex('QubitInformationObject')->search($query); if ($resultSet->count()) { return $resultSet[0]->getId(); diff --git a/lib/model/QubitTerm.php b/lib/model/QubitTerm.php index 8e5e0cfa85..c54464220d 100644 --- a/lib/model/QubitTerm.php +++ b/lib/model/QubitTerm.php @@ -896,7 +896,7 @@ public static function getEsTermsByTaxonomyId($taxonomyId, $limit = 10) $query = new \Elastica\Query($queryBool); $query->setSize($limit); - return QubitSearch::getInstance()->index->getType('QubitTerm')->search($query); + return QubitSearch::getInstance()->index->getIndex('QubitTerm')->search($query); } /** diff --git a/lib/task/propel/propelGenerateSlugsTask.class.php b/lib/task/propel/propelGenerateSlugsTask.class.php index b12bd4d5b0..8f068d1226 100644 --- a/lib/task/propel/propelGenerateSlugsTask.class.php +++ b/lib/task/propel/propelGenerateSlugsTask.class.php @@ -311,7 +311,7 @@ private function getSlugStringFromES($id, $property) $query->setQuery($queryBool); $query->setSize(1); - $results = QubitSearch::getInstance()->index->getType('QubitInformationObject')->search($query); + $results = QubitSearch::getInstance()->index->getIndex('QubitInformationObject')->search($query); if (!$results->count()) { return null; diff --git a/lib/task/search/arDocumentTask.class.php b/lib/task/search/arDocumentTask.class.php index 52cac22473..11528b48f7 100644 --- a/lib/task/search/arDocumentTask.class.php +++ b/lib/task/search/arDocumentTask.class.php @@ -37,7 +37,7 @@ public function execute($arguments = [], $options = []) if (null !== $slugObject = QubitObject::getBySlug($arguments[slug])) { $this->log(sprintf("Fetching data for %s ID %d...\n", $slugObject->className, $slugObject->id)); - $doc = QubitSearch::getInstance()->index->getType($slugObject->className)->getDocument($slugObject->id); + $doc = QubitSearch::getInstance()->index->getIndex($slugObject->className)->getDocument($slugObject->id); echo json_encode($doc->getData(), JSON_PRETTY_PRINT)."\n"; } else { diff --git a/lib/task/search/arSearchStatusTask.class.php b/lib/task/search/arSearchStatusTask.class.php index cb62d9831d..8be4b42043 100644 --- a/lib/task/search/arSearchStatusTask.class.php +++ b/lib/task/search/arSearchStatusTask.class.php @@ -78,7 +78,7 @@ private function objectsIndexed($docType) // Determine model class name from document type name $docTypeModelClass = 'Qubit'.ucfirst($docType); - return QubitSearch::getInstance()->index->getType($docTypeModelClass)->count(); + return QubitSearch::getInstance()->index->getIndex($docTypeModelClass)->count(); } private function objectsAvailableToIndex($docType) diff --git a/lib/task/tools/updatePublicationStatusTask.class.php b/lib/task/tools/updatePublicationStatusTask.class.php index 437f866a9b..66a690b8eb 100644 --- a/lib/task/tools/updatePublicationStatusTask.class.php +++ b/lib/task/tools/updatePublicationStatusTask.class.php @@ -174,7 +174,7 @@ protected function updatePublicationStatusDescendants($resource, $publicationSta $options = ['conflicts' => 'proceed']; $type = 'QubitInformationObject'; - $response = QubitSearch::getInstance()->index->getType($type)->updateByQuery($query, $queryScript, $options)->getData(); + $response = QubitSearch::getInstance()->index->getIndex($type)->updateByQuery($query, $queryScript, $options)->getData(); if (!empty($response['failures'])) { $this->failures += count($response['failures']); diff --git a/plugins/arElasticSearchPlugin/lib/arElasticSearchMultiIndexWrapper.class.php b/plugins/arElasticSearchPlugin/lib/arElasticSearchMultiIndexWrapper.class.php index 58249ed30e..bfda98c38f 100644 --- a/plugins/arElasticSearchPlugin/lib/arElasticSearchMultiIndexWrapper.class.php +++ b/plugins/arElasticSearchPlugin/lib/arElasticSearchMultiIndexWrapper.class.php @@ -73,7 +73,9 @@ public function refresh() } } - public function getType($name) + // Return the index element from the array of indices + // that matches the qualified index name + public function getIndex($name) { $name = $this->getIndexName($name); diff --git a/plugins/arElasticSearchPlugin/lib/arElasticSearchPlugin.class.php b/plugins/arElasticSearchPlugin/lib/arElasticSearchPlugin.class.php index 56b9c83368..e851786d0b 100644 --- a/plugins/arElasticSearchPlugin/lib/arElasticSearchPlugin.class.php +++ b/plugins/arElasticSearchPlugin/lib/arElasticSearchPlugin.class.php @@ -280,7 +280,7 @@ public function populate($options = []) // If excluding types then index as a whole hasn't been flushed: delete // type's documents if not updating if (count($excludeTypes) && !$update) { - $this->index->getType('Qubit'.$camelizedTypeName)->deleteByQuery(new \Elastica\Query\MatchAll()); + $this->index->getIndex('Qubit'.$camelizedTypeName)->deleteByQuery(new \Elastica\Query\MatchAll()); } $class = new $className(); @@ -370,7 +370,7 @@ public function addDocument($data, $type) $this->index->refresh(); } } else { - $this->index->getType($type)->addDocuments([$document]); + $this->index->getIndex($type)->addDocuments([$document]); } } @@ -400,7 +400,7 @@ public function partialUpdate($object, $data) $document->setType($this->index->getIndexName($type)); try { - $this->index->getType($type)->updateDocuments([$document]); + $this->index->getIndex($type)->updateDocuments([$document]); } catch (\Elastica\Exception\NotFoundException $e) { // Create document if it's not found $this->update($object); @@ -420,7 +420,7 @@ public function partialUpdateById(string $className, int $id, array $data) $document = new \Elastica\Document($id, $data); try { - $this->index->getType($className)->updateDocuments([$document]); + $this->index->getIndex($className)->updateDocuments([$document]); } catch (\Elastica\Exception\ResponseException $e) { // Create document if none exists $modelPdoClassName = self::modelClassFromQubitObjectClass($className).'Pdo'; @@ -473,7 +473,7 @@ public function delete($object) } } else { try { - $this->index->getType($type)->deleteById($object->id); + $this->index->getIndex($type)->deleteById($object->id); } catch (\Elastica\Exception\NotFoundException $e) { // Ignore } diff --git a/plugins/arRestApiPlugin/modules/api/actions/informationobjectsBrowseAction.class.php b/plugins/arRestApiPlugin/modules/api/actions/informationobjectsBrowseAction.class.php index 0861621527..ce196b2367 100644 --- a/plugins/arRestApiPlugin/modules/api/actions/informationobjectsBrowseAction.class.php +++ b/plugins/arRestApiPlugin/modules/api/actions/informationobjectsBrowseAction.class.php @@ -97,7 +97,7 @@ protected function get($request) $this->search->query->setSort([$field => $order]); - $resultSet = QubitSearch::getInstance()->index->getType('QubitInformationObject')->search($this->search->getQuery(false, true)); + $resultSet = QubitSearch::getInstance()->index->getIndex('QubitInformationObject')->search($this->search->getQuery(false, true)); // Build array from results $results = $lodMapping = []; diff --git a/plugins/qtAccessionPlugin/modules/accession/actions/browseAction.class.php b/plugins/qtAccessionPlugin/modules/accession/actions/browseAction.class.php index 2cccfa1445..dc7040f0db 100644 --- a/plugins/qtAccessionPlugin/modules/accession/actions/browseAction.class.php +++ b/plugins/qtAccessionPlugin/modules/accession/actions/browseAction.class.php @@ -154,7 +154,7 @@ public function execute($request) break; } - $resultSet = QubitSearch::getInstance()->index->getType('QubitAccession')->search($this->query); + $resultSet = QubitSearch::getInstance()->index->getIndex('QubitAccession')->search($this->query); $this->pager = new QubitSearchPager($resultSet); $this->pager->setPage($request->page ? $request->page : 1); diff --git a/plugins/sfSkosPlugin/test/unit/importTest.php b/plugins/sfSkosPlugin/test/unit/importTest.php index 5f53660911..e2659d5dde 100644 --- a/plugins/sfSkosPlugin/test/unit/importTest.php +++ b/plugins/sfSkosPlugin/test/unit/importTest.php @@ -969,14 +969,14 @@ function withTransaction($callback) foreach ($parent->getDescendants() as $key => $item) { try { - $search->index->getType('QubitTerm')->getDocument($item->id); + $search->index->getIndex('QubitTerm')->getDocument($item->id); $t->pass("Term {$key} is indexed"); } catch (Elastica\Exception\NotFoundException $e) { $t->fail("Term {$key} was not indexed"); } } - $doc = $search->index->getType('QubitTerm')->getDocument($parent->id)->getData(); + $doc = $search->index->getIndex('QubitTerm')->getDocument($parent->id)->getData(); $t->is($doc['numberOfDescendants'], count($importer->getGraph()->allOfType('skos:Concept')), 'Parent term ES document :numberOfDescendants: field is up to date'); }); From bc54543cd5233ddf68e7179b0941b7165bbd4929 Mon Sep 17 00:00:00 2001 From: Anvit Srivastav Date: Mon, 18 Nov 2024 16:45:07 -0800 Subject: [PATCH 14/14] Add dummy type for ES index Add a dummy ElasticSearch index type since one is still required for ES 6.x, instead of having a seperate type name for each of the multiple indices. --- .../actions/autocompleteAction.class.php | 5 ++-- .../lib/arElasticSearchPlugin.class.php | 28 ++++++++++++++----- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/apps/qubit/modules/search/actions/autocompleteAction.class.php b/apps/qubit/modules/search/actions/autocompleteAction.class.php index d462bb82bc..942afadfc4 100644 --- a/apps/qubit/modules/search/actions/autocompleteAction.class.php +++ b/apps/qubit/modules/search/actions/autocompleteAction.class.php @@ -77,10 +77,9 @@ public function execute($request) foreach ($items as $item) { $search = new \Elastica\Search($client); foreach ($indices as $type => $index) { - $itemType = QubitSearch::getInstance()->index->getIndexName($item['type']); + $itemType = QubitSearch::getInstance()::ES_TYPE; - // This will need to be updated in ES 7.x if it is updated to a dummy type, - // and then removed in ES 8.x when types are no longer required. + // This can be updated in ES 7.x when type params are optional $search->addIndex($index)->addType($index->getType($itemType)); } diff --git a/plugins/arElasticSearchPlugin/lib/arElasticSearchPlugin.class.php b/plugins/arElasticSearchPlugin/lib/arElasticSearchPlugin.class.php index e851786d0b..814f24787f 100644 --- a/plugins/arElasticSearchPlugin/lib/arElasticSearchPlugin.class.php +++ b/plugins/arElasticSearchPlugin/lib/arElasticSearchPlugin.class.php @@ -30,6 +30,14 @@ class arElasticSearchPlugin extends QubitSearchEngine */ public const MIN_VERSION = '1.3.0'; + /** + * Dummy type for the ElasticSearch index. + * This is required in ES 6.x but it is optional in + * ES 7.x and can be removed when ElasticSearch and + * Elastica are upgraded. + */ + public const ES_TYPE = '_doc'; + /** * Elastic_Client object. * @@ -348,7 +356,10 @@ public function addDocument($data, $type) unset($data['id']); $document = new \Elastica\Document($id, $data); - $document->setType($this->index->getIndexName($type)); + + // Setting a dummy type since it is required in ES 6.x + // but it can be removed in 7.x when it becomes optional + $document->setType(self::ES_TYPE); if (!$this->currentBatchType) { $this->currentBatchType = $type; @@ -397,7 +408,10 @@ public function partialUpdate($object, $data) $type = get_class($object); $document = new \Elastica\Document($object->id, $data); - $document->setType($this->index->getIndexName($type)); + + // Setting a dummy type since it is required in ES 6.x + // but it can be removed in 7.x when it becomes optional + $document->setType(self::ES_TYPE); try { $this->index->getIndex($type)->updateDocuments([$document]); @@ -456,7 +470,7 @@ public function delete($object) // the document to be deleted and add this document object to the batch delete // queue. $document = new \Elastica\Document($object->id); - $document->setType($this->index->getIndexName($type)); + $document->setType(self::ES_TYPE); if ($this->currentBatchType != $type) { $this->flushBatch(); @@ -584,8 +598,9 @@ protected function initialize() // Define mapping in elasticsearch $mapping = new \Elastica\Type\Mapping(); - // This type will need to be changed to a dummy type like _doc in 7.x - $mapping->setType($index->getType($indexType)); + // Setting a dummy type since it is required in ES 6.x + // but it can be removed in 7.x when it becomes optional + $mapping->setType($index->getType(self::ES_TYPE)); $mapping->setProperties($typeProperties['properties']); // Parse other parameters @@ -596,8 +611,7 @@ protected function initialize() $this->log(sprintf('Defining mapping %s...', $typeName)); - // In ES 7.x, if the mapping types are updated to a dummy type, - // this should be changed to: + // In ES 7.x this should be changed to: // $mapping->send($index, [ 'include_type_name' => false ]) // which can be removed in 8.x since that is the default behaviour // and will have be removed by 9.x when it is discontinued