From 8b450af41be2a43a0ab1712db6ccdaf4120e2257 Mon Sep 17 00:00:00 2001 From: Jin-Sun-tts Date: Wed, 21 Dec 2022 14:14:14 -0500 Subject: [PATCH 01/74] add unescape for email body --- ckanext/harvest/logic/action/update.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ckanext/harvest/logic/action/update.py b/ckanext/harvest/logic/action/update.py index 090e6e155..d57295db7 100644 --- a/ckanext/harvest/logic/action/update.py +++ b/ckanext/harvest/logic/action/update.py @@ -3,6 +3,7 @@ import hashlib import json import six +import html import logging import datetime @@ -798,6 +799,7 @@ def prepare_summary_mail(context, source_id, status): def prepare_error_mail(context, source_id, status): extra_vars = get_mail_extra_vars(context, source_id, status) body = render('emails/error_email.txt', extra_vars) + body = html.unescape(body) subject = '{} - Harvesting Job - Error Notification'\ .format(config.get('ckan.site_title')) From 84780a9633893dd0ffe8f269fb2141f3bbe4d1aa Mon Sep 17 00:00:00 2001 From: Jin-Sun-tts Date: Thu, 22 Dec 2022 16:45:18 -0500 Subject: [PATCH 02/74] specify different unescape package for python3.4+ and python2 user --- ckanext/harvest/logic/action/update.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/ckanext/harvest/logic/action/update.py b/ckanext/harvest/logic/action/update.py index d57295db7..3fa528c0a 100644 --- a/ckanext/harvest/logic/action/update.py +++ b/ckanext/harvest/logic/action/update.py @@ -3,7 +3,6 @@ import hashlib import json import six -import html import logging import datetime @@ -799,7 +798,13 @@ def prepare_summary_mail(context, source_id, status): def prepare_error_mail(context, source_id, status): extra_vars = get_mail_extra_vars(context, source_id, status) body = render('emails/error_email.txt', extra_vars) - body = html.unescape(body) + if six.PY34: + import html + body = html.unescape(body) + elif six.PY2: + import HTMLParser + body = HTMLParser.HTMLParser().unescape(body) + subject = '{} - Harvesting Job - Error Notification'\ .format(config.get('ckan.site_title')) From 9338de78a86425d5334e1c8fbb33459f04e4f1cc Mon Sep 17 00:00:00 2001 From: Jin-Sun-tts Date: Thu, 22 Dec 2022 16:48:12 -0500 Subject: [PATCH 03/74] fix lint --- ckanext/harvest/logic/action/update.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanext/harvest/logic/action/update.py b/ckanext/harvest/logic/action/update.py index 3fa528c0a..c669ac4ba 100644 --- a/ckanext/harvest/logic/action/update.py +++ b/ckanext/harvest/logic/action/update.py @@ -804,7 +804,7 @@ def prepare_error_mail(context, source_id, status): elif six.PY2: import HTMLParser body = HTMLParser.HTMLParser().unescape(body) - + subject = '{} - Harvesting Job - Error Notification'\ .format(config.get('ckan.site_title')) From 2f59c0d5f02e08483e828d432ba192f090b155cb Mon Sep 17 00:00:00 2001 From: Fuhu Xia Date: Wed, 15 Feb 2023 17:12:31 -0500 Subject: [PATCH 04/74] pick the right harvest_object_id if there are multiple --- ckanext/harvest/logic/action/get.py | 5 +++-- ckanext/harvest/plugin/__init__.py | 5 ++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/ckanext/harvest/logic/action/get.py b/ckanext/harvest/logic/action/get.py index cc1ec4c19..498152e88 100644 --- a/ckanext/harvest/logic/action/get.py +++ b/ckanext/harvest/logic/action/get.py @@ -267,8 +267,9 @@ def harvest_object_show(context, data_dict): obj = model.Session.query(HarvestObject) \ .filter(HarvestObject.package_id == pkg.id) \ .filter( - HarvestObject.current == True # noqa: E712 - ).first() + HarvestObject.current == True # noqa: E712 + ).order_by(HarvestObject.import_finished.desc()) \ + .first() else: raise p.toolkit.ValidationError( 'Please provide either an "id" or a "dataset_id" parameter') diff --git a/ckanext/harvest/plugin/__init__.py b/ckanext/harvest/plugin/__init__.py index 3c22c6a83..a2a0b4f54 100644 --- a/ckanext/harvest/plugin/__init__.py +++ b/ckanext/harvest/plugin/__init__.py @@ -120,7 +120,10 @@ def before_dataset_index(self, pkg_dict): harvest_object = model.Session.query(HarvestObject) \ .filter(HarvestObject.package_id == pkg_dict["id"]) \ - .filter(HarvestObject.current == True).first() # noqa + .filter( + HarvestObject.current == True # noqa + ).order_by(HarvestObject.import_finished.desc()) \ + .first() if harvest_object: From a47a4f1a4fa0d91f42846a16f42c0822b1e6657d Mon Sep 17 00:00:00 2001 From: pdelboca Date: Tue, 28 Feb 2023 09:04:07 -0300 Subject: [PATCH 05/74] Drop support for Pylons --- .github/workflows/test.yml | 12 +--- ckanext/harvest/harvesters/base.py | 21 +----- ckanext/harvest/plugin/__init__.py | 21 +----- ckanext/harvest/plugin/pylons_plugin.py | 88 ------------------------- 4 files changed, 7 insertions(+), 135 deletions(-) delete mode 100644 ckanext/harvest/plugin/pylons_plugin.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d7920a59d..e94c0777a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,8 +4,8 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 with: python-version: '3.8' - name: Install requirements @@ -19,7 +19,7 @@ jobs: needs: lint strategy: matrix: - ckan-version: ["2.10", 2.9, 2.9-py2, 2.8, 2.7] + ckan-version: ["2.10", 2.9] fail-fast: false name: CKAN ${{ matrix.ckan-version }} @@ -55,14 +55,8 @@ jobs: # Replace default path to CKAN core config file with the one on the container sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini - name: Setup extension (CKAN >= 2.9) - if: ${{ matrix.ckan-version != '2.7' && matrix.ckan-version != '2.8' }} run: | ckan -c test.ini db init ckan -c test.ini harvester initdb - - name: Setup extension (CKAN < 2.9) - if: ${{ matrix.ckan-version == '2.7' || matrix.ckan-version == '2.8' }} - run: | - paster --plugin=ckan db init -c test.ini - paster --plugin=ckanext-harvest harvester initdb -c test.ini - name: Run tests run: pytest --ckan-ini=test.ini --cov=ckanext.harvest --disable-warnings ckanext/harvest/tests diff --git a/ckanext/harvest/harvesters/base.py b/ckanext/harvest/harvesters/base.py index c3dd8605f..746d7271e 100644 --- a/ckanext/harvest/harvesters/base.py +++ b/ckanext/harvest/harvesters/base.py @@ -16,7 +16,7 @@ from ckan.logic.schema import default_create_package_schema from ckan.lib.navl.validators import ignore_missing, ignore -from ckan.lib.munge import munge_title_to_name, substitute_ascii_equivalents +from ckan.lib.munge import munge_title_to_name, munge_tag from ckanext.harvest.model import (HarvestObject, HarvestGatherError, HarvestObjectError, HarvestJob) @@ -25,25 +25,6 @@ from ckanext.harvest.interfaces import IHarvester from ckanext.harvest.logic.schema import unicode_safe -if p.toolkit.check_ckan_version(min_version='2.3'): - from ckan.lib.munge import munge_tag -else: - # Fallback munge_tag for older ckan versions which don't have a decent - # munger - def _munge_to_length(string, min_length, max_length): - '''Pad/truncates a string''' - if len(string) < min_length: - string += '_' * (min_length - len(string)) - if len(string) > max_length: - string = string[:max_length] - return string - - def munge_tag(tag): - tag = substitute_ascii_equivalents(tag) - tag = tag.lower().strip() - tag = re.sub(r'[^a-zA-Z0-9\- ]', '', tag).replace(' ', '-') - tag = _munge_to_length(tag, model.MIN_TAG_LENGTH, model.MAX_TAG_LENGTH) - return tag log = logging.getLogger(__name__) diff --git a/ckanext/harvest/plugin/__init__.py b/ckanext/harvest/plugin/__init__.py index a2a0b4f54..0843f7a43 100644 --- a/ckanext/harvest/plugin/__init__.py +++ b/ckanext/harvest/plugin/__init__.py @@ -13,11 +13,7 @@ import ckan.plugins as p from ckan.lib.plugins import DefaultDatasetForm -try: - from ckan.lib.plugins import DefaultTranslation -except ImportError: - class DefaultTranslation(): - pass +from ckan.lib.plugins import DefaultTranslation import ckanext.harvest from ckanext.harvest.model import setup as model_setup @@ -28,10 +24,7 @@ class DefaultTranslation(): DATASET_TYPE_NAME ) -if p.toolkit.check_ckan_version(min_version='2.9.0'): - from ckanext.harvest.plugin.flask_plugin import MixinPlugin -else: - from ckanext.harvest.plugin.pylons_plugin import MixinPlugin +from ckanext.harvest.plugin.flask_plugin import MixinPlugin log = getLogger(__name__) assert not log.disabled @@ -47,8 +40,7 @@ class Harvest(MixinPlugin, p.SingletonPlugin, DefaultDatasetForm, DefaultTransla p.implements(p.IPackageController, inherit=True) p.implements(p.ITemplateHelpers) p.implements(p.IFacets, inherit=True) - if p.toolkit.check_ckan_version(min_version='2.5.0'): - p.implements(p.ITranslation, inherit=True) + p.implements(p.ITranslation, inherit=True) startup = False @@ -274,13 +266,6 @@ def configure(self, config): self.startup = False def update_config(self, config): - if not p.toolkit.check_ckan_version(min_version='2.0'): - assert 0, 'CKAN before 2.0 not supported by ckanext-harvest - '\ - 'genshi templates not supported any more' - if p.toolkit.asbool(config.get('ckan.legacy_templates', False)): - log.warn('Old genshi templates not supported any more by ' - 'ckanext-harvest so you should set ckan.legacy_templates ' - 'option to True any more.') p.toolkit.add_template_directory(config, '../templates') p.toolkit.add_public_directory(config, '../public') p.toolkit.add_resource('../fanstatic_library', 'ckanext-harvest') diff --git a/ckanext/harvest/plugin/pylons_plugin.py b/ckanext/harvest/plugin/pylons_plugin.py deleted file mode 100644 index 5e6f0e676..000000000 --- a/ckanext/harvest/plugin/pylons_plugin.py +++ /dev/null @@ -1,88 +0,0 @@ -# -*- coding: utf-8 -*- - -import ckan.plugins as p -from ckanext.harvest.utils import DATASET_TYPE_NAME - - -class MixinPlugin(p.SingletonPlugin): - p.implements(p.IRoutes, inherit=True) - - # IRoutes - - def before_map(self, map): - - # Most of the routes are defined via the IDatasetForm interface - # (ie they are the ones for a package type) - controller = "ckanext.harvest.controllers.view:ViewController" - - map.connect( - "{0}_delete".format(DATASET_TYPE_NAME), - "/" + DATASET_TYPE_NAME + "/delete/:id", - controller=controller, - action="delete", - ) - map.connect( - "{0}_refresh".format(DATASET_TYPE_NAME), - "/" + DATASET_TYPE_NAME + "/refresh/:id", - controller=controller, - action="refresh", - ) - map.connect( - "{0}_admin".format(DATASET_TYPE_NAME), - "/" + DATASET_TYPE_NAME + "/admin/:id", - controller=controller, - action="admin", - ) - map.connect( - "{0}_about".format(DATASET_TYPE_NAME), - "/" + DATASET_TYPE_NAME + "/about/:id", - controller=controller, - action="about", - ) - map.connect( - "{0}_clear".format(DATASET_TYPE_NAME), - "/" + DATASET_TYPE_NAME + "/clear/:id", - controller=controller, - action="clear", - ) - - map.connect( - "harvest_job_list", - "/" + DATASET_TYPE_NAME + "/{source}/job", - controller=controller, - action="list_jobs", - ) - map.connect( - "harvest_job_show_last", - "/" + DATASET_TYPE_NAME + "/{source}/job/last", - controller=controller, - action="show_last_job", - ) - map.connect( - "harvest_job_show", - "/" + DATASET_TYPE_NAME + "/{source}/job/{id}", - controller=controller, - action="show_job", - ) - map.connect( - "harvest_job_abort", - "/" + DATASET_TYPE_NAME + "/{source}/job/{id}/abort", - controller=controller, - action="abort_job", - ) - - map.connect( - "harvest_object_show", - "/" + DATASET_TYPE_NAME + "/object/:id", - controller=controller, - action="show_object", - ) - map.connect( - "harvest_object_for_dataset_show", - "/dataset/harvest_object/:id", - controller=controller, - action="show_object", - ref_type="dataset", - ) - - return map From 7b391e6be9c3fd66be236b1cf02e8c1f8083507b Mon Sep 17 00:00:00 2001 From: pdelboca Date: Tue, 28 Feb 2023 09:28:11 -0300 Subject: [PATCH 06/74] Re introduce plugin.py file --- .../harvest/{plugin/__init__.py => plugin.py} | 26 +++++++++++++------ ckanext/harvest/plugin/flask_plugin.py | 20 -------------- 2 files changed, 18 insertions(+), 28 deletions(-) rename ckanext/harvest/{plugin/__init__.py => plugin.py} (96%) delete mode 100644 ckanext/harvest/plugin/flask_plugin.py diff --git a/ckanext/harvest/plugin/__init__.py b/ckanext/harvest/plugin.py similarity index 96% rename from ckanext/harvest/plugin/__init__.py rename to ckanext/harvest/plugin.py index 0843f7a43..b4bf34425 100644 --- a/ckanext/harvest/plugin/__init__.py +++ b/ckanext/harvest/plugin.py @@ -16,6 +16,7 @@ from ckan.lib.plugins import DefaultTranslation import ckanext.harvest +from ckanext.harvest import cli, views from ckanext.harvest.model import setup as model_setup from ckanext.harvest.model import HarvestSource, HarvestJob, HarvestObject from ckanext.harvest.log import DBLogHandler @@ -24,14 +25,13 @@ DATASET_TYPE_NAME ) -from ckanext.harvest.plugin.flask_plugin import MixinPlugin - log = getLogger(__name__) assert not log.disabled -class Harvest(MixinPlugin, p.SingletonPlugin, DefaultDatasetForm, DefaultTranslation): - +class Harvest(p.SingletonPlugin, DefaultDatasetForm, DefaultTranslation): + p.implements(p.IClick) + p.implements(p.IBlueprint) p.implements(p.IConfigurable) p.implements(p.IConfigurer, inherit=True) p.implements(p.IActions) @@ -44,6 +44,16 @@ class Harvest(MixinPlugin, p.SingletonPlugin, DefaultDatasetForm, DefaultTransla startup = False + # IClick + + def get_commands(self): + return cli.get_commands() + + # IBlueprint + + def get_blueprint(self): + return views.get_blueprints() + # ITranslation def i18n_directory(self): u'''Change the directory of the .mo translation files''' @@ -266,10 +276,10 @@ def configure(self, config): self.startup = False def update_config(self, config): - p.toolkit.add_template_directory(config, '../templates') - p.toolkit.add_public_directory(config, '../public') - p.toolkit.add_resource('../fanstatic_library', 'ckanext-harvest') - p.toolkit.add_resource('../public/ckanext/harvest/javascript', 'harvest-extra-field') + p.toolkit.add_template_directory(config, 'templates') + p.toolkit.add_public_directory(config, 'public') + p.toolkit.add_resource('fanstatic_library', 'ckanext-harvest') + p.toolkit.add_resource('public/ckanext/harvest/javascript', 'harvest-extra-field') if p.toolkit.check_ckan_version(min_version='2.9.0'): mappings = config.get('ckan.legacy_route_mappings') or {} diff --git a/ckanext/harvest/plugin/flask_plugin.py b/ckanext/harvest/plugin/flask_plugin.py deleted file mode 100644 index 7b8208aaa..000000000 --- a/ckanext/harvest/plugin/flask_plugin.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- - -import ckan.plugins as p -import ckanext.harvest.cli as cli -import ckanext.harvest.views as views - - -class MixinPlugin(p.SingletonPlugin): - p.implements(p.IClick) - p.implements(p.IBlueprint) - - # IClick - - def get_commands(self): - return cli.get_commands() - - # IBlueprint - - def get_blueprint(self): - return views.get_blueprints() From 8c340f3690ebb9f3f5af67d29751667962839f52 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Tue, 28 Feb 2023 10:24:17 -0300 Subject: [PATCH 07/74] Drop support for old versions in templates --- ckanext/harvest/templates/base.html | 5 +-- .../harvest_extra_field_resource.html | 1 - .../harvest/snippets/harvest_resource.html | 1 - .../harvest/templates/source/admin_base.html | 39 +++---------------- ckanext/harvest/templates/source/edit.html | 7 +--- ckanext/harvest/templates/source/new.html | 7 +--- .../templates/source/new_source_form.html | 8 ++-- .../harvest/templates/source/read_base.html | 16 ++------ ckanext/harvest/templates/source/search.html | 20 +--------- 9 files changed, 17 insertions(+), 87 deletions(-) delete mode 100644 ckanext/harvest/templates/harvest/snippets/harvest_extra_field_resource.html delete mode 100644 ckanext/harvest/templates/harvest/snippets/harvest_resource.html diff --git a/ckanext/harvest/templates/base.html b/ckanext/harvest/templates/base.html index 5a379a0b8..2fb8e0de8 100644 --- a/ckanext/harvest/templates/base.html +++ b/ckanext/harvest/templates/base.html @@ -2,8 +2,5 @@ {% block styles %} {{ super() }} - - {% set type = 'asset' if h.ckan_version().split('.')|map('int')|list >= [2, 9, 0] else 'resource' %} - {% include 'harvest/snippets/harvest_' ~ type ~ '.html' %} - + {% include 'harvest/snippets/harvest_asset.html' %} {% endblock %} diff --git a/ckanext/harvest/templates/harvest/snippets/harvest_extra_field_resource.html b/ckanext/harvest/templates/harvest/snippets/harvest_extra_field_resource.html deleted file mode 100644 index 9e8fda56d..000000000 --- a/ckanext/harvest/templates/harvest/snippets/harvest_extra_field_resource.html +++ /dev/null @@ -1 +0,0 @@ -{% resource 'harvest-extra-field/main' %} diff --git a/ckanext/harvest/templates/harvest/snippets/harvest_resource.html b/ckanext/harvest/templates/harvest/snippets/harvest_resource.html deleted file mode 100644 index 69ce72363..000000000 --- a/ckanext/harvest/templates/harvest/snippets/harvest_resource.html +++ /dev/null @@ -1 +0,0 @@ -{% resource 'ckanext-harvest/styles/harvest.css' %} diff --git a/ckanext/harvest/templates/source/admin_base.html b/ckanext/harvest/templates/source/admin_base.html index ce65c99f0..f4b749304 100644 --- a/ckanext/harvest/templates/source/admin_base.html +++ b/ckanext/harvest/templates/source/admin_base.html @@ -7,65 +7,38 @@
  • {{ _('Admin') }}
    • {% endblock %} -{% block action_links %} +{% block content_action %} +
    {% if harvest_source.status and harvest_source.status.last_job and (harvest_source.status.last_job.status == 'New' or harvest_source.status.last_job.status == 'Running') %} Reharvest {% else %} {% set locale = h.dump_json({'content': _('This will re-run the harvesting for this source. Any updates at the source will overwrite the local datasets. Sources with a large number of datasets may take a significant amount of time to finish harvesting. Please confirm you would like us to start reharvesting.')}) %} - {{ _('Reharvest') }} - {% endif %} {% if harvest_source.status and harvest_source.status.last_job and (harvest_source.status.last_job.status == 'Running') %} - {{ _('Stop') }} - {% endif %} {% set locale = h.dump_json({'content': _('Warning: This will remove all datasets for this source, as well as all previous job reports. Are you sure you want to continue?')}) %} - {{ _('Clear') }} - - - {{ _('View harvest source') }} - - -{% endblock %} - -{# CKAN 2.0 #} -{% block actions_content %} - {{ self.action_links() }} -{% endblock %} - -{# CKAN 2.1 #} -{% block content_action %} -
    - {{ self.action_links() }} -
    +
    {% endblock %} {% block page_header_tabs %} - {% set ckan_version = h.ckan_version().split('.')[1] %} - {% if ckan_version | int >= 9 %} - {{ h.build_nav_icon('harvester.admin', _('Dashboard'), id=harvest_source.name, icon='dashboard') }} - {{ h.build_nav_icon('harvester.job_list', _('Jobs'), source=harvest_source.name, icon='reorder') }} - {{ h.build_nav_icon(c.dataset_type ~ '.edit', _('Edit'), id=harvest_source.name, icon='edit') }} - {% else %} - {{ h.build_nav_icon('{0}_admin'.format(c.dataset_type), _('Dashboard'), id=harvest_source.name, icon='dashboard') }} - {{ h.build_nav_icon('{0}_job_list'.format(c.dataset_type), _('Jobs'), source=harvest_source.name, icon='reorder') }} - {{ h.build_nav_icon('{0}_edit'.format(c.dataset_type), _('Edit'), id=harvest_source.name, icon='edit') }} - {% endif %} + {{ h.build_nav_icon('harvester.admin', _('Dashboard'), id=harvest_source.name, icon='dashboard') }} + {{ h.build_nav_icon('harvester.job_list', _('Jobs'), source=harvest_source.name, icon='reorder') }} + {{ h.build_nav_icon(c.dataset_type ~ '.edit', _('Edit'), id=harvest_source.name, icon='edit') }} {% endblock %} diff --git a/ckanext/harvest/templates/source/edit.html b/ckanext/harvest/templates/source/edit.html index 776bc8952..d739d1232 100644 --- a/ckanext/harvest/templates/source/edit.html +++ b/ckanext/harvest/templates/source/edit.html @@ -6,12 +6,7 @@
    {% block form %} - {% if c.form %} - {# CKAN < 2.3 #} - {{ c.form | safe }} - {% else %} - {{- h.snippet(form_snippet, c=c, **form_vars) -}} - {% endif %} + {{- h.snippet(form_snippet, c=c, **form_vars) -}} {% endblock %}
    {% endblock %} diff --git a/ckanext/harvest/templates/source/new.html b/ckanext/harvest/templates/source/new.html index fc03de3f4..b7feb3d9c 100644 --- a/ckanext/harvest/templates/source/new.html +++ b/ckanext/harvest/templates/source/new.html @@ -13,12 +13,7 @@ {% block primary_content %}
    - {% if c.form %} - {# CKAN < 2.3 #} - {{ c.form | safe }} - {% else %} - {{- h.snippet(form_snippet, c=c, **form_vars) -}} - {% endif %} + {{- h.snippet(form_snippet, c=c, **form_vars) -}}
    {% endblock %} diff --git a/ckanext/harvest/templates/source/new_source_form.html b/ckanext/harvest/templates/source/new_source_form.html index 015371655..8453b8ab2 100644 --- a/ckanext/harvest/templates/source/new_source_form.html +++ b/ckanext/harvest/templates/source/new_source_form.html @@ -1,10 +1,8 @@ {% import 'macros/form.html' as form %} +{% include 'harvest/snippets/harvest_extra_field_asset.html' %} -{% set type = 'asset' if h.ckan_version().split('.')|map('int')|list >= [2, 9, 0] else 'resource' %} -{% include 'harvest/snippets/harvest_extra_field_' ~ type ~ '.html' %} - -
    + {% block errors %}{{ form.errors(error_summary) }}{% endblock %} @@ -14,7 +12,7 @@ {% endcall %} -{{ h.csrf_input() if 'csrf_input' in h }} + {{ h.csrf_input() if 'csrf_input' in h }} {{ form.input('title', id='field-title', label=_('Title'), placeholder=_('eg. A descriptive title'), value=data.title, error=errors.title, classes=['control-full'], attrs={'data-module': 'slug-preview-target'}) }} diff --git a/ckanext/harvest/templates/source/read_base.html b/ckanext/harvest/templates/source/read_base.html index 984427456..3e21e6e61 100644 --- a/ckanext/harvest/templates/source/read_base.html +++ b/ckanext/harvest/templates/source/read_base.html @@ -6,16 +6,14 @@ {% endif %} {% endblock %} -{# CKAN 2.0 #} {% block actions_content %} {% if authorized_user %}
  • {{ self.admin_link() }}
    • {% endif %} {% endblock %} -{# TODO: once #354 is merged in CKAN core .profile-info doesn't exist #} {% block secondary_content %} -
    +

    {{ harvest_source.title }}

    {% if harvest_source.notes %} @@ -37,7 +35,7 @@

    {{ harvest_source.title }}

    {% endblock %} {% block primary_content %} -
    +
    {% block page_header %}
    {% block content_action %} @@ -47,14 +45,8 @@

    {{ harvest_source.title }}

    {% endblock %}
      {% block page_header_tabs %} - {% set ckan_version = h.ckan_version().split('.')[1] %} - {% if ckan_version | int >= 9 %} - {{ h.build_nav_icon(c.dataset_type ~ '.read', _('Datasets'), id=harvest_source.name, icon='sitemap') }} - {{ h.build_nav_icon('harvester.about', _('About'), id=harvest_source.name, icon='info-sign') }} - {% else %} - {{ h.build_nav_icon('{0}_read'.format(c.dataset_type), _('Datasets'), id=harvest_source.name, icon='sitemap') }} - {{ h.build_nav_icon('{0}_about'.format(c.dataset_type), _('About'), id=harvest_source.name, icon='info-sign') }} - {% endif %} + {{ h.build_nav_icon(c.dataset_type ~ '.read', _('Datasets'), id=harvest_source.name, icon='sitemap') }} + {{ h.build_nav_icon('harvester.about', _('About'), id=harvest_source.name, icon='info-sign') }} {% endblock %}
    diff --git a/ckanext/harvest/templates/source/search.html b/ckanext/harvest/templates/source/search.html index b7797b940..d1afcbf20 100644 --- a/ckanext/harvest/templates/source/search.html +++ b/ckanext/harvest/templates/source/search.html @@ -8,23 +8,7 @@ {% endblock %} -{% if g.ckan_base_version.startswith('2.0') %} - {# CKAN 2.0 #} - - {% block add_action_content %} - {{ h.snippet('snippets/add_source_button.html', dataset_type=c.dataset_type) }} - {% endblock %} -{% endif %} - {% block primary_content %} - {% if g.ckan_base_version.startswith('2.0') %} - {# CKAN 2.0 #} - - {% include 'source/search_2.0.html' %} - - {% else %} - {# > CKAN 2.0 #} -
    {% block page_primary_action %} @@ -56,9 +40,7 @@ {{ c.page.pager(q=c.q) }}
    - {% endif %} - -{% endblock %} + {% endblock %} From aa44587b7af9cac71e14f5adb56d2714cdb139bc Mon Sep 17 00:00:00 2001 From: pdelboca Date: Tue, 28 Feb 2023 10:27:12 -0300 Subject: [PATCH 08/74] Remove old snippets for resource/asset compatibility --- ckanext/harvest/templates/base.html | 2 +- ckanext/harvest/templates/harvest/snippets/harvest_asset.html | 1 - .../templates/harvest/snippets/harvest_extra_field_asset.html | 1 - ckanext/harvest/templates/source/new_source_form.html | 2 +- 4 files changed, 2 insertions(+), 4 deletions(-) delete mode 100644 ckanext/harvest/templates/harvest/snippets/harvest_asset.html delete mode 100644 ckanext/harvest/templates/harvest/snippets/harvest_extra_field_asset.html diff --git a/ckanext/harvest/templates/base.html b/ckanext/harvest/templates/base.html index 2fb8e0de8..d1e58db2e 100644 --- a/ckanext/harvest/templates/base.html +++ b/ckanext/harvest/templates/base.html @@ -2,5 +2,5 @@ {% block styles %} {{ super() }} - {% include 'harvest/snippets/harvest_asset.html' %} + {% asset 'ckanext-harvest/harvest_css' %} {% endblock %} diff --git a/ckanext/harvest/templates/harvest/snippets/harvest_asset.html b/ckanext/harvest/templates/harvest/snippets/harvest_asset.html deleted file mode 100644 index b78146a15..000000000 --- a/ckanext/harvest/templates/harvest/snippets/harvest_asset.html +++ /dev/null @@ -1 +0,0 @@ -{% asset 'ckanext-harvest/harvest_css' %} diff --git a/ckanext/harvest/templates/harvest/snippets/harvest_extra_field_asset.html b/ckanext/harvest/templates/harvest/snippets/harvest_extra_field_asset.html deleted file mode 100644 index 47e8b8965..000000000 --- a/ckanext/harvest/templates/harvest/snippets/harvest_extra_field_asset.html +++ /dev/null @@ -1 +0,0 @@ -{% asset 'harvest-extra-field/main' %} diff --git a/ckanext/harvest/templates/source/new_source_form.html b/ckanext/harvest/templates/source/new_source_form.html index 8453b8ab2..c03aff042 100644 --- a/ckanext/harvest/templates/source/new_source_form.html +++ b/ckanext/harvest/templates/source/new_source_form.html @@ -1,6 +1,6 @@ {% import 'macros/form.html' as form %} -{% include 'harvest/snippets/harvest_extra_field_asset.html' %} +{% asset 'harvest-extra-field/main' %} From 163bb39e3a1e13270680588cbae1076769ab2548 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Tue, 28 Feb 2023 10:42:26 -0300 Subject: [PATCH 09/74] Drop admin link support for old versions --- ckanext/harvest/templates/source/read_base.html | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/ckanext/harvest/templates/source/read_base.html b/ckanext/harvest/templates/source/read_base.html index 3e21e6e61..1b03eb3a3 100644 --- a/ckanext/harvest/templates/source/read_base.html +++ b/ckanext/harvest/templates/source/read_base.html @@ -1,17 +1,5 @@ {% extends "source/base.html" %} -{% block admin_link %} - {% if h.check_access('harvest_source_update', {'id':harvest_source.id }) %} - {{ h.nav_link(_('Admin'), named_route='{0}_admin'.format(c.dataset_type), id=harvest_source.name, class_='btn btn-primary', icon='wrench')}} - {% endif %} -{% endblock %} - -{% block actions_content %} - {% if authorized_user %} -
  • {{ self.admin_link() }}
    • - {% endif %} -{% endblock %} - {% block secondary_content %}
    @@ -40,7 +28,9 @@

    {{ harvest_source.title }}

    {% block content_action %}
    - {{ self.admin_link() }} + {% if h.check_access('harvest_source_update', {'id':harvest_source.id }) %} + {{ h.nav_link(_('Admin'), named_route='{0}_admin'.format(c.dataset_type), id=harvest_source.name, class_='btn btn-primary', icon='wrench')}} + {% endif %}
    {% endblock %}
      From 339cdd99886dfbb74d07f155d6878a6273c1519a Mon Sep 17 00:00:00 2001 From: pdelboca Date: Tue, 28 Feb 2023 10:46:27 -0300 Subject: [PATCH 10/74] Rename fanstatic to assets --- .../harvest/{fanstatic_library => assets}/styles/harvest.css | 0 .../harvest/{fanstatic_library => assets}/styles/harvest.less | 0 ckanext/harvest/{fanstatic_library => assets}/styles/less | 0 .../harvest/{fanstatic_library => assets}/styles/mixins.less | 0 .../harvest/{fanstatic_library => assets}/styles/variables.less | 0 ckanext/harvest/{fanstatic_library => assets}/webassets.yml | 0 ckanext/harvest/plugin.py | 2 +- 7 files changed, 1 insertion(+), 1 deletion(-) rename ckanext/harvest/{fanstatic_library => assets}/styles/harvest.css (100%) rename ckanext/harvest/{fanstatic_library => assets}/styles/harvest.less (100%) rename ckanext/harvest/{fanstatic_library => assets}/styles/less (100%) rename ckanext/harvest/{fanstatic_library => assets}/styles/mixins.less (100%) rename ckanext/harvest/{fanstatic_library => assets}/styles/variables.less (100%) rename ckanext/harvest/{fanstatic_library => assets}/webassets.yml (100%) diff --git a/ckanext/harvest/fanstatic_library/styles/harvest.css b/ckanext/harvest/assets/styles/harvest.css similarity index 100% rename from ckanext/harvest/fanstatic_library/styles/harvest.css rename to ckanext/harvest/assets/styles/harvest.css diff --git a/ckanext/harvest/fanstatic_library/styles/harvest.less b/ckanext/harvest/assets/styles/harvest.less similarity index 100% rename from ckanext/harvest/fanstatic_library/styles/harvest.less rename to ckanext/harvest/assets/styles/harvest.less diff --git a/ckanext/harvest/fanstatic_library/styles/less b/ckanext/harvest/assets/styles/less similarity index 100% rename from ckanext/harvest/fanstatic_library/styles/less rename to ckanext/harvest/assets/styles/less diff --git a/ckanext/harvest/fanstatic_library/styles/mixins.less b/ckanext/harvest/assets/styles/mixins.less similarity index 100% rename from ckanext/harvest/fanstatic_library/styles/mixins.less rename to ckanext/harvest/assets/styles/mixins.less diff --git a/ckanext/harvest/fanstatic_library/styles/variables.less b/ckanext/harvest/assets/styles/variables.less similarity index 100% rename from ckanext/harvest/fanstatic_library/styles/variables.less rename to ckanext/harvest/assets/styles/variables.less diff --git a/ckanext/harvest/fanstatic_library/webassets.yml b/ckanext/harvest/assets/webassets.yml similarity index 100% rename from ckanext/harvest/fanstatic_library/webassets.yml rename to ckanext/harvest/assets/webassets.yml diff --git a/ckanext/harvest/plugin.py b/ckanext/harvest/plugin.py index b4bf34425..8c05a971e 100644 --- a/ckanext/harvest/plugin.py +++ b/ckanext/harvest/plugin.py @@ -278,7 +278,7 @@ def configure(self, config): def update_config(self, config): p.toolkit.add_template_directory(config, 'templates') p.toolkit.add_public_directory(config, 'public') - p.toolkit.add_resource('fanstatic_library', 'ckanext-harvest') + p.toolkit.add_resource('assets', 'ckanext-harvest') p.toolkit.add_resource('public/ckanext/harvest/javascript', 'harvest-extra-field') if p.toolkit.check_ckan_version(min_version='2.9.0'): From 6090ad83fb1b7f53ebce56d340e566599c9bfc58 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Tue, 28 Feb 2023 12:17:41 -0300 Subject: [PATCH 11/74] Sanitize blueprints to Flask syntax --- ckanext/harvest/helpers.py | 4 ++-- ckanext/harvest/logic/action/update.py | 2 +- ckanext/harvest/plugin.py | 21 ------------------- .../templates/snippets/add_source_button.html | 2 +- .../templates/snippets/source_item.html | 4 ++-- ckanext/harvest/templates/source/admin.html | 2 +- .../harvest/templates/source/admin_base.html | 8 +++---- .../harvest/templates/source/job/list.html | 2 +- .../harvest/templates/source/job/read.html | 4 ++-- .../templates/source/new_source_form.html | 2 +- .../harvest/templates/source/read_base.html | 4 ++-- ckanext/harvest/templates/source/search.html | 2 +- ckanext/harvest/tests/test_blueprint.py | 16 +++++++------- ckanext/harvest/utils.py | 8 +++---- ckanext/harvest/views.py | 2 ++ 15 files changed, 32 insertions(+), 51 deletions(-) diff --git a/ckanext/harvest/helpers.py b/ckanext/harvest/helpers.py index 035e38543..2b9d4a57e 100644 --- a/ckanext/harvest/helpers.py +++ b/ckanext/harvest/helpers.py @@ -57,7 +57,7 @@ def package_list_for_source(source_id): query = logic.get_action('package_search')(context, search_dict) base_url = h.url_for( - '{0}_read'.format(DATASET_TYPE_NAME), + '{0}.read'.format(DATASET_TYPE_NAME), id=harvest_source['name'] ) @@ -124,7 +124,7 @@ def link_for_harvest_object(id=None, guid=None, text=None): obj = logic.get_action('harvest_object_show')(context, {'id': guid, 'attr': 'guid'}) id = obj.id - url = h.url_for('harvest_object_show', id=id) + url = h.url_for('harvest.object_show', id=id) text = text or guid or id link = '{text}'.format(url=url, text=text) diff --git a/ckanext/harvest/logic/action/update.py b/ckanext/harvest/logic/action/update.py index c669ac4ba..57cc94886 100644 --- a/ckanext/harvest/logic/action/update.py +++ b/ckanext/harvest/logic/action/update.py @@ -759,7 +759,7 @@ def get_mail_extra_vars(context, source_id, status): errors = job_errors + obj_errors site_url = config.get('ckan.site_url') - job_url = toolkit.url_for('harvest_job_show', source=source['id'], id=last_job['id']) + job_url = toolkit.url_for('harvest.job_show', source=source['id'], id=last_job['id']) full_job_url = urljoin(site_url, job_url) extra_vars = { 'organization': organization, diff --git a/ckanext/harvest/plugin.py b/ckanext/harvest/plugin.py index 8c05a971e..19400862d 100644 --- a/ckanext/harvest/plugin.py +++ b/ckanext/harvest/plugin.py @@ -281,27 +281,6 @@ def update_config(self, config): p.toolkit.add_resource('assets', 'ckanext-harvest') p.toolkit.add_resource('public/ckanext/harvest/javascript', 'harvest-extra-field') - if p.toolkit.check_ckan_version(min_version='2.9.0'): - mappings = config.get('ckan.legacy_route_mappings') or {} - if mappings and isinstance(mappings, string_types): - mappings = json.loads(mappings) - - mappings.update({ - 'harvest_read': 'harvest.read', - 'harvest_edit': 'harvest.edit', - }) - bp_routes = [ - "delete", "refresh", "admin", "about", - "clear", "job_list", "job_show_last", "job_show", - "job_abort", "object_show" - ] - mappings.update({ - 'harvest_' + route: 'harvester.' + route - for route in bp_routes - }) - # https://github.com/ckan/ckan/pull/4521 - config['ckan.legacy_route_mappings'] = json.dumps(mappings) - # IActions def get_actions(self): diff --git a/ckanext/harvest/templates/snippets/add_source_button.html b/ckanext/harvest/templates/snippets/add_source_button.html index d392f98de..ffefb7d88 100644 --- a/ckanext/harvest/templates/snippets/add_source_button.html +++ b/ckanext/harvest/templates/snippets/add_source_button.html @@ -1,7 +1,7 @@ {% set authorized_user = h.check_access('harvest_source_create') %} {% if authorized_user %} - + {{ _('Add Harvest Source') }} diff --git a/ckanext/harvest/templates/snippets/source_item.html b/ckanext/harvest/templates/snippets/source_item.html index c2be2f11e..a8ba26fe6 100644 --- a/ckanext/harvest/templates/snippets/source_item.html +++ b/ckanext/harvest/templates/snippets/source_item.html @@ -19,7 +19,7 @@ {% set truncate_title = truncate_title or 80 %} {% set title = source.title or source.name %} {% set source_type = h.get_pkg_dict_extra(source, 'source_type') %} -{% set url = h.url_for('harvest_admin', id=source.name) if within_organization else h.url_for('harvest_read', id=source.name) %} +{% set url = h.url_for('harvest_admin', id=source.name) if within_organization else h.url_for('harvest.read', id=source.name) %}
    • @@ -47,7 +47,7 @@

      {{ _('Datasets') }}: {{ source.status.total_datasets }} {% endif %} {% if not within_organization and source.organization %} - — {{ _('Organization') }}: {{ h.link_to(source.organization.title or source.organization.name, h.url_for('organization_read', id=source.organization.name)) }} + — {{ _('Organization') }}: {{ h.link_to(source.organization.title or source.organization.name, h.url_for('organization.read', id=source.organization.name)) }} {% endif %}

      diff --git a/ckanext/harvest/templates/source/admin.html b/ckanext/harvest/templates/source/admin.html index a4c5cb75b..fe5b41e65 100644 --- a/ckanext/harvest/templates/source/admin.html +++ b/ckanext/harvest/templates/source/admin.html @@ -6,7 +6,7 @@

      {{ _('Last Harvest Job') }}

      {% if harvest_source.status and harvest_source.status.last_job %} {% snippet "snippets/job_details.html", job=harvest_source.status.last_job %}
      - + {{ _('View full job report') }} diff --git a/ckanext/harvest/templates/source/admin_base.html b/ckanext/harvest/templates/source/admin_base.html index f4b749304..20d2634d8 100644 --- a/ckanext/harvest/templates/source/admin_base.html +++ b/ckanext/harvest/templates/source/admin_base.html @@ -13,24 +13,24 @@ Reharvest {% else %} {% set locale = h.dump_json({'content': _('This will re-run the harvesting for this source. Any updates at the source will overwrite the local datasets. Sources with a large number of datasets may take a significant amount of time to finish harvesting. Please confirm you would like us to start reharvesting.')}) %} - {{ _('Reharvest') }} {% endif %} {% if harvest_source.status and harvest_source.status.last_job and (harvest_source.status.last_job.status == 'Running') %} - + {{ _('Stop') }} {% endif %} {% set locale = h.dump_json({'content': _('Warning: This will remove all datasets for this source, as well as all previous job reports. Are you sure you want to continue?')}) %} - {{ _('Clear') }} - + {{ _('View harvest source') }} diff --git a/ckanext/harvest/templates/source/job/list.html b/ckanext/harvest/templates/source/job/list.html index cea0be739..3e262b522 100644 --- a/ckanext/harvest/templates/source/job/list.html +++ b/ckanext/harvest/templates/source/job/list.html @@ -16,7 +16,7 @@

      {{ _('Harvest Jobs') }}

    • - + {{ _('Job: ') }} {{ job.id }} {% if job.status != 'Finished' %} diff --git a/ckanext/harvest/templates/source/job/read.html b/ckanext/harvest/templates/source/job/read.html index 7c9ce550f..25d32e125 100644 --- a/ckanext/harvest/templates/source/job/read.html +++ b/ckanext/harvest/templates/source/job/read.html @@ -6,7 +6,7 @@

      - {{ h.nav_link(_('Back to job list'), named_route='harvest_job_list', source=harvest_source.name, class_='btn btn-default', icon='arrow-left')}} + {{ h.nav_link(_('Back to job list'), named_route='harvester.job_list', source=harvest_source.name, class_='btn btn-default', icon='arrow-left')}}

      {{ _('Job Report') }}

      @@ -69,7 +69,7 @@

      {{ _('Document Errors') }} {{ _('Remote content') }} {% endif %} - + {{ _('Local content') }} diff --git a/ckanext/harvest/templates/source/new_source_form.html b/ckanext/harvest/templates/source/new_source_form.html index c03aff042..f0572df2e 100644 --- a/ckanext/harvest/templates/source/new_source_form.html +++ b/ckanext/harvest/templates/source/new_source_form.html @@ -17,7 +17,7 @@ {{ form.input('title', id='field-title', label=_('Title'), placeholder=_('eg. A descriptive title'), value=data.title, error=errors.title, classes=['control-full'], attrs={'data-module': 'slug-preview-target'}) }} {% set prefix = 'harvest' %} - {% set domain = h.url_for('{0}_read'.format(c.dataset_type), id='', qualified=true) %} + {% set domain = h.url_for('{0}.read'.format(c.dataset_type), id='', qualified=true) %} {% set domain = domain|replace("http://", "")|replace("https://", "") %} {% set attrs = {'data-module': 'slug-preview-slug', 'data-module-prefix': domain, 'data-module-placeholder': ''} %} diff --git a/ckanext/harvest/templates/source/read_base.html b/ckanext/harvest/templates/source/read_base.html index 1b03eb3a3..5126e25d5 100644 --- a/ckanext/harvest/templates/source/read_base.html +++ b/ckanext/harvest/templates/source/read_base.html @@ -7,7 +7,7 @@

      {{ harvest_source.title }}

      {% if harvest_source.notes %}

      {{ h.markdown_extract(harvest_source.notes, 180) }} - {{ h.nav_link(_('read more'), named_route='{0}_about'.format(c.dataset_type), id=harvest_source.name) }} + {{ h.nav_link(_('read more'), named_route='harvester.about', id=harvest_source.name) }}

      {% else %}

      {{ _('There is no description for this harvest source') }}

      @@ -29,7 +29,7 @@

      {{ harvest_source.title }}

      {% block content_action %}
      {% if h.check_access('harvest_source_update', {'id':harvest_source.id }) %} - {{ h.nav_link(_('Admin'), named_route='{0}_admin'.format(c.dataset_type), id=harvest_source.name, class_='btn btn-primary', icon='wrench')}} + {{ h.nav_link(_('Admin'), named_route='harvester.admin', id=harvest_source.name, class_='btn btn-primary', icon='wrench')}} {% endif %}
      {% endblock %} diff --git a/ckanext/harvest/templates/source/search.html b/ckanext/harvest/templates/source/search.html index d1afcbf20..a92994322 100644 --- a/ckanext/harvest/templates/source/search.html +++ b/ckanext/harvest/templates/source/search.html @@ -46,6 +46,6 @@ {% block secondary_content %} {% for facet in c.facet_titles %} - {{ h.snippet('snippets/facet_list.html', title=c.facet_titles[facet], name=facet, alternative_url=h.url_for('{0}_search'.format(c.dataset_type))) }} + {{ h.snippet('snippets/facet_list.html', title=c.facet_titles[facet], name=facet, alternative_url=h.url_for('{0}.search'.format(c.dataset_type))) }} {% endfor %} {% endblock %} diff --git a/ckanext/harvest/tests/test_blueprint.py b/ckanext/harvest/tests/test_blueprint.py index dbbca7d3f..0d5612b9e 100644 --- a/ckanext/harvest/tests/test_blueprint.py +++ b/ckanext/harvest/tests/test_blueprint.py @@ -28,7 +28,7 @@ def test_index_page_is_rendered(self, app): def test_new_form_is_rendered(self, app): - url = url_for('harvest_new') + url = url_for('harvest.new') sysadmin = factories.Sysadmin() env = {"REMOTE_USER": sysadmin['name'].encode('ascii')} @@ -40,7 +40,7 @@ def test_edit_form_is_rendered(self, app): source = harvest_factories.HarvestSource() - url = url_for('harvest_edit', id=source['id']) + url = url_for('harvest.edit', id=source['id']) sysadmin = factories.Sysadmin() env = {"REMOTE_USER": sysadmin['name'].encode('ascii')} @@ -52,7 +52,7 @@ def test_source_page_rendered(self, app): source = harvest_factories.HarvestSource() - url = url_for('harvest_read', id=source['name']) + url = url_for('harvest.read', id=source['name']) sysadmin = factories.Sysadmin() env = {"REMOTE_USER": sysadmin['name'].encode('ascii')} @@ -68,7 +68,7 @@ def test_admin_page_rendered(self, app): sysadmin = factories.Sysadmin() env = {"REMOTE_USER": sysadmin['name'].encode('ascii')} - url = url_for('harvest_admin', id=source_obj.id) + url = url_for('harvester.admin', id=source_obj.id) response = app.get(url, extra_environ=env) @@ -80,7 +80,7 @@ def test_about_page_rendered(self, app): source = harvest_factories.HarvestSource() - url = url_for('harvest_about', id=source['name']) + url = url_for('harvester.about', id=source['name']) sysadmin = factories.Sysadmin() env = {"REMOTE_USER": sysadmin['name'].encode('ascii')} @@ -95,7 +95,7 @@ def test_job_page_rendered(self, app): sysadmin = factories.Sysadmin() env = {"REMOTE_USER": sysadmin['name'].encode('ascii')} - url = url_for('harvest_job_list', source=job['source_id']) + url = url_for('harvester.job_list', source=job['source_id']) response = app.get(url, extra_environ=env) @@ -108,7 +108,7 @@ def test_job_show_last_page_rendered(self, app): sysadmin = factories.Sysadmin() env = {"REMOTE_USER": sysadmin['name'].encode('ascii')} - url = url_for('harvest_job_show_last', source=job['source_id']) + url = url_for('harvester.job_show_last', source=job['source_id']) response = app.get(url, extra_environ=env) @@ -119,7 +119,7 @@ def test_job_show_page_rendered(self, app): job = harvest_factories.HarvestJob() url = url_for( - 'harvest_job_show', source=job['source_id'], id=job['id']) + 'harvester.job_show', source=job['source_id'], id=job['id']) sysadmin = factories.Sysadmin() env = {"REMOTE_USER": sysadmin['name'].encode('ascii')} diff --git a/ckanext/harvest/utils.py b/ckanext/harvest/utils.py index 7acaaea59..e6fc09ddd 100644 --- a/ckanext/harvest/utils.py +++ b/ckanext/harvest/utils.py @@ -644,7 +644,7 @@ def job_abort_view(source, id): return tk.abort(500, msg) return h.redirect_to( - h.url_for('{0}_admin'.format(DATASET_TYPE_NAME), id=source)) + h.url_for('{0}.admin'.format(DATASET_TYPE_NAME), id=source)) def refresh_view(id): @@ -674,7 +674,7 @@ def refresh_view(id): h.flash_error(msg) return h.redirect_to( - h.url_for('{0}_admin'.format(DATASET_TYPE_NAME), id=id)) + h.url_for('{0}.admin'.format(DATASET_TYPE_NAME), id=id)) def clear_view(id): @@ -691,7 +691,7 @@ def clear_view(id): h.flash_error(msg) return h.redirect_to( - h.url_for('{0}_admin'.format(DATASET_TYPE_NAME), id=id)) + h.url_for('{0}.admin'.format(DATASET_TYPE_NAME), id=id)) def delete_view(id): @@ -712,7 +712,7 @@ def delete_view(id): h.flash_success(_('Harvesting source successfully inactivated')) return h.redirect_to( - h.url_for('{0}_admin'.format(DATASET_TYPE_NAME), id=id)) + h.url_for('{0}.admin'.format(DATASET_TYPE_NAME), id=id)) except tk.ObjectNotFound: return tk.abort(404, _('Harvest source not found')) except tk.NotAuthorized: diff --git a/ckanext/harvest/views.py b/ckanext/harvest/views.py index 6ac8eec5a..9185b0624 100644 --- a/ckanext/harvest/views.py +++ b/ckanext/harvest/views.py @@ -5,6 +5,8 @@ import ckanext.harvest.utils as utils +# IDatasetForm provides a "harvest" blueprint for the package type harvest. +# We name the extension blueprint "harvester" to avoid clashing of names. harvester = Blueprint("harvester", __name__) From ee661f12b215d84fed52a45ce0f62963bb9e04fb Mon Sep 17 00:00:00 2001 From: pdelboca Date: Tue, 28 Feb 2023 12:27:36 -0300 Subject: [PATCH 12/74] Remove six from tests --- ckanext/harvest/tests/harvesters/mock_ckan.py | 16 +++------- ckanext/harvest/tests/test_blueprint.py | 30 +++++++------------ 2 files changed, 15 insertions(+), 31 deletions(-) diff --git a/ckanext/harvest/tests/harvesters/mock_ckan.py b/ckanext/harvest/tests/harvesters/mock_ckan.py index 1d4d6f4d9..af4e3b515 100644 --- a/ckanext/harvest/tests/harvesters/mock_ckan.py +++ b/ckanext/harvest/tests/harvesters/mock_ckan.py @@ -3,17 +3,12 @@ import json import re import copy -import six -from six.moves.urllib.parse import unquote_plus +from urllib.parse import unquote_plus from threading import Thread -if six.PY2: - from SimpleHTTPServer import SimpleHTTPRequestHandler - from SocketServer import TCPServer -else: - from http.server import SimpleHTTPRequestHandler - from socketserver import TCPServer +from http.server import SimpleHTTPRequestHandler +from socketserver import TCPServer PORT = 8998 @@ -171,10 +166,7 @@ def get_org(self, org_ref): def get_url_params(self): params_str = self.path.split('?')[-1] - if six.PY2: - params_unicode = unquote_plus(params_str).decode('utf8') - else: - params_unicode = unquote_plus(params_str) + params_unicode = unquote_plus(params_str) params = params_unicode.split('&') return dict([param.split('=') for param in params]) diff --git a/ckanext/harvest/tests/test_blueprint.py b/ckanext/harvest/tests/test_blueprint.py index 0d5612b9e..a9141da8e 100644 --- a/ckanext/harvest/tests/test_blueprint.py +++ b/ckanext/harvest/tests/test_blueprint.py @@ -1,4 +1,3 @@ -import six import pytest from ckantoolkit import url_for @@ -6,13 +5,6 @@ from ckanext.harvest.tests import factories as harvest_factories -def _assert_in_body(string, response): - if six.PY2: - assert string in response.body.decode('utf8') - else: - assert string in response.body - - @pytest.mark.usefixtures('clean_db', 'clean_index', 'harvest_setup') class TestBlueprint(): @@ -23,8 +15,8 @@ def test_index_page_is_rendered(self, app): response = app.get(u'/harvest') - _assert_in_body(source1['title'], response) - _assert_in_body(source2['title'], response) + assert source1['title'] in response.body + assert source2['title'] in response.body def test_new_form_is_rendered(self, app): @@ -34,7 +26,7 @@ def test_new_form_is_rendered(self, app): response = app.get(url, extra_environ=env) - _assert_in_body(' Date: Tue, 28 Feb 2023 12:44:32 -0300 Subject: [PATCH 13/74] Clean up six from the codebase --- ckanext/harvest/harvesters/ckanharvester.py | 8 +++---- ckanext/harvest/logic/action/update.py | 17 +++++++-------- ckanext/harvest/logic/validators.py | 10 +++------ ckanext/harvest/plugin.py | 4 +--- ckanext/harvest/utils.py | 23 +++++++-------------- requirements.txt | 2 -- 6 files changed, 22 insertions(+), 42 deletions(-) diff --git a/ckanext/harvest/harvesters/ckanharvester.py b/ckanext/harvest/harvesters/ckanharvester.py index 1da0787ea..aba454ddb 100644 --- a/ckanext/harvest/harvesters/ckanharvester.py +++ b/ckanext/harvest/harvesters/ckanharvester.py @@ -1,11 +1,10 @@ from __future__ import absolute_import -import six import requests from requests.exceptions import HTTPError, RequestException import datetime -from six.moves.urllib.parse import urlencode +from urllib.parse import urlencode from ckan import model from ckan.logic import ValidationError, NotFound, get_action from ckan.lib.helpers import json @@ -119,8 +118,7 @@ def validate_config(self, config): raise ValueError('default_groups must be a *list* of group' ' names/ids') if config_obj['default_groups'] and \ - not isinstance(config_obj['default_groups'][0], - six.string_types): + not isinstance(config_obj['default_groups'][0], str): raise ValueError('default_groups must be a list of group ' 'names/ids (i.e. strings)') @@ -520,7 +518,7 @@ def get_extra(key, package_dict): if existing_extra: package_dict['extras'].remove(existing_extra) # Look for replacement strings - if isinstance(value, six.string_types): + if isinstance(value, str): value = value.format( harvest_source_id=harvest_object.job.source.id, harvest_source_url=harvest_object.job.source.url.strip('/'), diff --git a/ckanext/harvest/logic/action/update.py b/ckanext/harvest/logic/action/update.py index 57cc94886..eff23ea13 100644 --- a/ckanext/harvest/logic/action/update.py +++ b/ckanext/harvest/logic/action/update.py @@ -1,15 +1,15 @@ # -*- coding: utf-8 -*- import hashlib +import html import json -import six import logging import datetime from ckantoolkit import config from sqlalchemy import and_, or_ -from six.moves.urllib.parse import urljoin +from urllib.parse import urljoin from ckan.lib.search.index import PackageSearchIndex from ckan.plugins import toolkit, PluginImplementations @@ -549,8 +549,10 @@ def harvest_objects_import(context, data_dict): last_objects_count = 0 for obj_id in last_objects_ids: - if segments and \ - str(hashlib.md5(six.ensure_binary(obj_id[0])).hexdigest())[0] not in segments: + _id = obj_id[0] + if isinstance(_id, str): + _id = _id.encode() + if segments and str(hashlib.md5(_id).hexdigest())[0] not in segments: continue obj = session.query(HarvestObject).get(obj_id) @@ -798,12 +800,7 @@ def prepare_summary_mail(context, source_id, status): def prepare_error_mail(context, source_id, status): extra_vars = get_mail_extra_vars(context, source_id, status) body = render('emails/error_email.txt', extra_vars) - if six.PY34: - import html - body = html.unescape(body) - elif six.PY2: - import HTMLParser - body = HTMLParser.HTMLParser().unescape(body) + body = html.unescape(body) subject = '{} - Harvesting Job - Error Notification'\ .format(config.get('ckan.site_title')) diff --git a/ckanext/harvest/logic/validators.py b/ckanext/harvest/logic/validators.py index f1363a8c5..b54bcfa8a 100644 --- a/ckanext/harvest/logic/validators.py +++ b/ckanext/harvest/logic/validators.py @@ -12,11 +12,7 @@ ) from ckanext.harvest.model import HarvestSource, UPDATE_FREQUENCIES, HarvestJob from ckanext.harvest.interfaces import IHarvester - -import six -from six.moves.urllib.parse import ( - urlparse, urlunparse -) +from urllib.parse import (urlparse, urlunparse) log = logging.getLogger(__name__) @@ -233,7 +229,7 @@ def harvest_source_convert_from_config(key, data, errors, context): def harvest_source_active_validator(value, context): - if isinstance(value, six.string_types): + if isinstance(value, str): if value.lower() == 'true': return True else: @@ -259,6 +255,6 @@ def harvest_object_extras_validator(value, context): if not isinstance(value, dict): raise Invalid('extras must be a dict') for v in value.values(): - if not isinstance(v, six.string_types): + if not isinstance(v, str): raise Invalid('extras must be a dict of strings') return value diff --git a/ckanext/harvest/plugin.py b/ckanext/harvest/plugin.py index 19400862d..9081c8447 100644 --- a/ckanext/harvest/plugin.py +++ b/ckanext/harvest/plugin.py @@ -2,10 +2,8 @@ import os import json -import six from logging import getLogger -from six import string_types from collections import OrderedDict from ckan import logic @@ -113,7 +111,7 @@ def before_dataset_search(self, search_params): fq = search_params.get("fq", "") if "dataset_type:harvest" not in fq: - fq = "{0} -dataset_type:harvest".format(fq.encode('utf8') if six.PY2 else fq) + fq = "{0} -dataset_type:harvest".format(fq) search_params.update({"fq": fq}) return search_params diff --git a/ckanext/harvest/utils.py b/ckanext/harvest/utils.py index e6fc09ddd..d0c68dd89 100644 --- a/ckanext/harvest/utils.py +++ b/ckanext/harvest/utils.py @@ -10,22 +10,12 @@ import ckan.lib.helpers as h import ckan.plugins.toolkit as tk -import six from ckan import model from ckantoolkit import _ -from six import StringIO +from io import StringIO from ckanext.harvest.logic import HarvestJobExists, HarvestSourceInactiveError -try: - # Python 2.7 - xml_parser_exception = etree.ParseError -except AttributeError: - # Python 2.6 - from xml.parsers import expat - - xml_parser_exception = expat.ExpatError - log = logging.getLogger(__name__) @@ -394,8 +384,7 @@ def run_test_harvester(source_id_or_name, force_import): if running_jobs: print('\nSource "{0}" apparently has a "Running" job:\n{1}'.format( source.get("name") or source["id"], running_jobs)) - - resp = six.moves.input("Abort it? (y/n)") + resp = input("Abort it? (y/n)") if not resp.lower().startswith("y"): sys.exit(1) job_dict = tk.get_action("harvest_job_abort")( @@ -748,7 +737,7 @@ def object_show_view(id, ref_type, response): if '\n' + content - except xml_parser_exception: + except etree.ParseError: try: json.loads(obj['content']) response.content_type = 'application/json; charset=utf-8' @@ -757,7 +746,11 @@ def object_show_view(id, ref_type, response): pass response.headers['Content-Length'] = len(content) - return (response, six.ensure_str(content)) + + if isinstance(content, bytes): + content = content.decode("utf-8") + + return (response, content) except tk.ObjectNotFound as e: return tk.abort(404, _(str(e))) diff --git a/requirements.txt b/requirements.txt index 754dfbf10..93611a3c0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,4 @@ ckantoolkit>=0.0.7 pika>=1.1.0,<1.3.0 -enum34; python_version < '3.0' # Required by pika redis requests>=2.11.1 -six>=1.12.0 From 9f2f84abd2ea1ee89cf4acd58c3442da38f94a57 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Tue, 28 Feb 2023 12:49:32 -0300 Subject: [PATCH 14/74] Clean no longer needed boostrap helper --- ckanext/harvest/helpers.py | 10 ---------- ckanext/harvest/logic/action/update.py | 5 +---- ckanext/harvest/plugin.py | 1 - ckanext/harvest/templates/source/new_source_form.html | 2 +- 4 files changed, 2 insertions(+), 16 deletions(-) diff --git a/ckanext/harvest/helpers.py b/ckanext/harvest/helpers.py index 2b9d4a57e..87b3e3c5b 100644 --- a/ckanext/harvest/helpers.py +++ b/ckanext/harvest/helpers.py @@ -138,13 +138,3 @@ def harvest_source_extra_fields(): continue fields[harvester.info()['name']] = list(harvester.extra_schema().keys()) return fields - - -def bootstrap_version(): - if p.toolkit.check_ckan_version(max_version='2.7.99'): - return 'bs2' - else: - return ( - 'bs2' if - p.toolkit.config.get('ckan.base_public_folder') == 'public-bs2' - else 'bs3') diff --git a/ckanext/harvest/logic/action/update.py b/ckanext/harvest/logic/action/update.py index eff23ea13..924f478d6 100644 --- a/ckanext/harvest/logic/action/update.py +++ b/ckanext/harvest/logic/action/update.py @@ -38,10 +38,7 @@ import ckan.lib.mailer as mailer from itertools import islice -if toolkit.check_ckan_version(min_version='2.9.0'): - from ckan.plugins.toolkit import render -else: - from ckan.lib.base import render_jinja2 as render +from ckan.plugins.toolkit import render log = logging.getLogger(__name__) diff --git a/ckanext/harvest/plugin.py b/ckanext/harvest/plugin.py index 9081c8447..68b9e8b28 100644 --- a/ckanext/harvest/plugin.py +++ b/ckanext/harvest/plugin.py @@ -309,7 +309,6 @@ def get_helpers(self): 'harvest_frequencies': harvest_helpers.harvest_frequencies, 'link_for_harvest_object': harvest_helpers.link_for_harvest_object, 'harvest_source_extra_fields': harvest_helpers.harvest_source_extra_fields, - 'bootstrap_version': harvest_helpers.bootstrap_version, 'get_harvest_source': harvest_helpers.get_harvest_source, } diff --git a/ckanext/harvest/templates/source/new_source_form.html b/ckanext/harvest/templates/source/new_source_form.html index f0572df2e..324d012e4 100644 --- a/ckanext/harvest/templates/source/new_source_form.html +++ b/ckanext/harvest/templates/source/new_source_form.html @@ -2,7 +2,7 @@ {% asset 'harvest-extra-field/main' %} - + {% block errors %}{{ form.errors(error_summary) }}{% endblock %} From d1d9a4fef33e731b5d5d16e2e4a74511b4f106bf Mon Sep 17 00:00:00 2001 From: pdelboca Date: Tue, 28 Feb 2023 13:01:43 -0300 Subject: [PATCH 15/74] Remove CKAN <= 2.8 info from README --- README.rst | 105 +++++------------------------------------------------ 1 file changed, 10 insertions(+), 95 deletions(-) diff --git a/README.rst b/README.rst index 9172a1793..e8d5893ac 100644 --- a/README.rst +++ b/README.rst @@ -94,14 +94,8 @@ Configuration Run the following command to create the necessary tables in the database (ensuring the pyenv is activated): -ON CKAN >= 2.9:: - (pyenv) $ ckan --config=/etc/ckan/default/ckan.ini harvester initdb -ON CKAN <= 2.8:: - - (pyenv) $ paster --plugin=ckanext-harvest harvester initdb --config=/etc/ckan/default/production.ini - Finally, restart CKAN to have the changes take effect:: sudo service apache2 restart @@ -213,7 +207,7 @@ IF you want to set a timeout for harvest jobs, you can add this configuration op ckan.harvest.timeout = 1440 -The timeout value is in minutes, so 1440 represents 24 hours. +The timeout value is in minutes, so 1440 represents 24 hours. Any jobs which are timed out will create an error message for the user to see. If you don't specify this setting, the default will be False and there will be no timeout on harvest jobs. @@ -289,9 +283,9 @@ The following operations can be run from the command line as described underneat import) without involving the web UI or the queue backends. This is useful for testing a harvester without having to fire up gather/fetch_consumer processes, as is done in production. - + harvester run-test {source-id/name} force-import=guid1,guid2... - - In order to force an import of particular datasets, useful to + - In order to force an import of particular datasets, useful to target a dataset for dev purposes or when forcing imports on other environments. harvester gather-consumer @@ -335,22 +329,17 @@ The following operations can be run from the command line as described underneat The commands should be run with the pyenv activated and refer to your CKAN configuration file: -ON CKAN >= 2.9:: - (pyenv) $ ckan --config=/etc/ckan/default/ckan.ini harvester --help (pyenv) $ ckan --config=/etc/ckan/default/ckan.ini harvester sources -ON CKAN <= 2.8:: - (pyenv) $ paster --plugin=ckanext-harvest harvester sources --config=/etc/ckan/default/production.ini - **Note that on CKAN >= 2.9 all commands with an underscore in their name changed.** They now use a hyphen instead of an underscore (e.g. ``gather_consumer`` changed to ``gather-consumer``). Authorization ============= -Starting from CKAN 2.0, harvest sources behave exactly the same as datasets +Harvest sources behave exactly the same as datasets (they are actually internally implemented as a dataset type). That means they can be searched and faceted, and that the same authorization rules can be applied to them. The default authorization settings are based on organizations. @@ -700,10 +689,10 @@ harvester run-test You can run a harvester simply using the ``run-test`` command. This is handy for running a harvest with one command in the console and see all the output in-line. It runs the gather, fetch and import stages all in the same process. -You must ensure that you have pip installed ``dev-requirements.txt`` +You must ensure that you have pip installed ``dev-requirements.txt`` in ``/home/ckan/ckan/lib/default/src/ckanext-harvest`` before using the ``run-test`` command. - + This is useful for developing a harvester because you can insert break-points in your harvester, and rerun a harvest without having to restart the gather_consumer and fetch_consumer processes each time. In addition, because it @@ -727,35 +716,17 @@ handles the gathering and another one that handles the fetching and importing. To start the consumers run the following command (make sure you have your python environment activated): -ON CKAN >= 2.9:: - (pyenv) $ ckan --config=/etc/ckan/default/ckan.ini harvester gather-consumer -ON CKAN <= 2.8:: - - (pyenv) $ paster --plugin=ckanext-harvest harvester gather_consumer --config=/etc/ckan/default/production.ini - On another terminal, run the following command: -ON CKAN >= 2.9:: - (pyenv) $ ckan --config=/etc/ckan/default/ckan.ini harvester fetch-consumer -ON CKAN <= 2.8:: - - (pyenv) $ paster --plugin=ckanext-harvest harvester fetch_consumer --config=/etc/ckan/default/production.ini - Finally, on a third console, run the following command to start any pending harvesting jobs: -ON CKAN >= 2.9:: - (pyenv) $ ckan --config=/etc/ckan/default/ckan.ini harvester run -ON CKAN <= 2.8:: - - (pyenv) $ paster --plugin=ckanext-harvest harvester run --config=/etc/ckan/default/production.ini - The ``run`` command not only starts any pending harvesting jobs, but also flags those that are finished, allowing new jobs to be created on that particular source and refreshing the source statistics. That means that you will need to run @@ -771,14 +742,8 @@ circumstance, ensure that the gather & fetch consumers are running and have nothing more to consume, and then run this abort command with the name or id of the harvest source: -ON CKAN >= 2.9:: - (pyenv) $ ckan --config=/etc/ckan/default/ckan.ini harvester job-abort {source-id/name} -ON CKAN <= 2.8:: - - (pyenv) $ paster --plugin=ckanext-harvest harvester job_abort {source-id/name} --config=/etc/ckan/default/production.ini - Setting up the harvesters on a production server ================================================ @@ -855,42 +820,6 @@ following steps with the one you are using. startsecs=10 - ON CKAN <= 2.8:: - - - ; =============================== - ; ckan harvester - ; =============================== - - [program:ckan_gather_consumer] - - command=/usr/lib/ckan/default/bin/paster --plugin=ckanext-harvest harvester gather_consumer --config=/etc/ckan/default/production.ini - - ; user that owns virtual environment. - user=ckan - - numprocs=1 - stdout_logfile=/var/log/ckan/std/gather_consumer.log - stderr_logfile=/var/log/ckan/std/gather_consumer.log - autostart=true - autorestart=true - startsecs=10 - - [program:ckan_fetch_consumer] - - command=/usr/lib/ckan/default/bin/paster --plugin=ckanext-harvest harvester fetch_consumer --config=/etc/ckan/default/production.ini - - ; user that owns virtual environment. - user=ckan - - numprocs=1 - stdout_logfile=/var/log/ckan/std/fetch_consumer.log - stderr_logfile=/var/log/ckan/std/fetch_consumer.log - autostart=true - autorestart=true - startsecs=10 - - There are a number of things that you will need to replace with your specific installation settings (the example above shows paths from a ckan instance installed via Debian packages): @@ -952,16 +881,9 @@ following steps with the one you are using. Paste this line into your crontab, again replacing the paths to paster and the ini file with yours: - ON CKAN >= 2.9:: - # m h dom mon dow command */15 * * * * /usr/lib/ckan/default/bin/ckan -c /etc/ckan/default/ckan.ini harvester run - ON CKAN <= 2.8:: - - # m h dom mon dow command - */15 * * * * /usr/lib/ckan/default/bin/paster --plugin=ckanext-harvest harvester run --config=/etc/ckan/default/production.ini - This particular example will check for pending jobs every fifteen minutes. You can of course modify this periodicity, this `Wikipedia page `_ has a good overview of the crontab syntax. @@ -973,16 +895,9 @@ following steps with the one you are using. Paste this line into your crontab, again replacing the paths to paster/ckan and the ini file with yours: - ON CKAN >= 2.9:: - # m h dom mon dow command 0 5 * * * /usr/lib/ckan/default/bin/ckan -c /etc/ckan/default/ckan.ini harvester clean-harvest-log - ON CKAN <= 2.8:: - - # m h dom mon dow command - 0 5 * * * /usr/lib/ckan/default/bin/paster --plugin=ckanext-harvest harvester clean_harvest_log --config=/etc/ckan/default/production.ini - This particular example will perform clean-up each day at 05 AM. You can tweak the value according to your needs. @@ -992,17 +907,17 @@ Extensible actions Recipients on harvest jobs notifications ---------------------------------------- -:code:`harvest_get_notifications_recipients`: you can *chain* this action from another extension to change +:code:`harvest_get_notifications_recipients`: you can *chain* this action from another extension to change the recipients for harvest jobs notifications. .. code-block:: python @toolkit.chained_action def harvest_get_notifications_recipients(up_func, context, data_dict): - """ Harvester plugin notify by default about harvest jobs only to + """ Harvester plugin notify by default about harvest jobs only to admin users of the related organization. Also allow to add custom recipients with this function. - + Return a list of dicts with name and email like {'name': 'John', 'email': 'john@source.com'} """ @@ -1021,7 +936,7 @@ Tests You can run the tests like this:: cd ckanext-harvest - nosetests --reset-db --ckan --with-pylons=test-core.ini ckanext/harvest/tests + pytest --ckan-ini=test.ini ckanext/harvest/tests Here are some common errors and solutions: From c22f7dddb8fb4e3d5283a850c5e33b004d5bb45f Mon Sep 17 00:00:00 2001 From: pdelboca Date: Fri, 3 Mar 2023 10:48:18 -0300 Subject: [PATCH 16/74] Remove 2.2 support on schemas --- ckanext/harvest/logic/schema.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/ckanext/harvest/logic/schema.py b/ckanext/harvest/logic/schema.py index e54120094..2946ea525 100644 --- a/ckanext/harvest/logic/schema.py +++ b/ckanext/harvest/logic/schema.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -import ckan.plugins as p import ckan.plugins.toolkit as tk from ckan.logic.schema import default_extras_schema @@ -54,10 +53,6 @@ def harvest_source_schema(): schema['extras'] = extras_schema - if p.toolkit.check_ckan_version('2.2'): - from ckan.logic.validators import datasets_with_no_organization_cannot_be_private - schema['private'].append(datasets_with_no_organization_cannot_be_private) - return schema From 34fa49064521618396f205a13b2e95accedfc59a Mon Sep 17 00:00:00 2001 From: pdelboca Date: Thu, 9 Mar 2023 09:59:41 -0300 Subject: [PATCH 17/74] Do not duplicate harvest_extras if exist in root schema --- ckanext/harvest/plugin/__init__.py | 114 +++++++++++++++-------------- 1 file changed, 61 insertions(+), 53 deletions(-) diff --git a/ckanext/harvest/plugin/__init__.py b/ckanext/harvest/plugin/__init__.py index a2a0b4f54..6fea9a4b8 100644 --- a/ckanext/harvest/plugin/__init__.py +++ b/ckanext/harvest/plugin/__init__.py @@ -116,7 +116,37 @@ def before_dataset_search(self, search_params): return search_params + def _add_or_update_harvest_metadata(self, key, value, data_dict): + """Adds extras fields or updates them if already exist.""" + if not data_dict.get("extras"): + data_dict["extras"] = [] + + for e in data_dict.get("extras"): + if e.get("key") == key: + e.update({"value": value}) + break + else: + data_dict["extras"].append({"key": key, "value": value}) + def before_dataset_index(self, pkg_dict): + """Adds harvest metadata to the extra field of the dataset. + + This method will add or update harvest related metadata in `pkg_dict`, + `data_dict` and `validated_data_dict` so it can be obtained when + calling package_show API (that depends on Solr data). This metadata will + be stored in the `extra` field of the dictionaries. + + By default all harvest metadata will go in the extra field. If + another extension adds any of them to the `package_show` schema + then it will not be added again in the `extras` field to avoid + validation errors when updating a package. + """ + # Fix to support Solr8 + if isinstance(pkg_dict.get('status'), dict): + try: + pkg_dict['status'] = json.dumps(pkg_dict['status']) + except ValueError: + pkg_dict.pop('status', None) harvest_object = model.Session.query(HarvestObject) \ .filter(HarvestObject.package_id == pkg_dict["id"]) \ @@ -125,59 +155,37 @@ def before_dataset_index(self, pkg_dict): ).order_by(HarvestObject.import_finished.desc()) \ .first() - if harvest_object: - - data_dict = json.loads(pkg_dict["data_dict"]) - - validated_data_dict = json.loads(pkg_dict["validated_data_dict"]) - - harvest_extras = [ - ("harvest_object_id", harvest_object.id), - ("harvest_source_id", harvest_object.source.id), - ("harvest_source_title", harvest_object.source.title), - ] - - for key, value in harvest_extras: - - # If the harvest extras are there, update them. This can - # happen eg when calling package_update or resource_update, - # which call package_show - harvest_not_found = True - harvest_not_found_validated = True - if not data_dict.get("extras"): - data_dict["extras"] = [] - - for e in data_dict.get("extras"): - if e.get("key") == key: - e.update({"value": value}) - harvest_not_found = False - if harvest_not_found: - data_dict["extras"].append({"key": key, "value": value}) - - if not validated_data_dict.get("extras"): - validated_data_dict["extras"] = [] - - for e in validated_data_dict.get("extras"): - if e.get("key") == key: - e.update({"value": value}) - harvest_not_found_validated = False - if harvest_not_found_validated: - validated_data_dict["extras"].append({"key": key, "value": value}) - - # The commented line isn't cataloged correctly, if we pass the - # basic key the extras are prepended and the system works as - # expected. - # pkg_dict['extras_{0}'.format(key)] = value - pkg_dict[key] = value - - pkg_dict["data_dict"] = json.dumps(data_dict) - pkg_dict["validated_data_dict"] = json.dumps(validated_data_dict) - - if isinstance(pkg_dict.get('status'), dict): - try: - pkg_dict['status'] = json.dumps(pkg_dict['status']) - except ValueError: - pkg_dict.pop('status', None) + if not harvest_object: + return pkg_dict + + harvest_extras = [ + ("harvest_object_id", harvest_object.id), + ("harvest_source_id", harvest_object.source.id), + ("harvest_source_title", harvest_object.source.title), + ] + + # Add harvest extras to data_dict + data_dict = json.loads(pkg_dict["data_dict"]) + for key, value in harvest_extras: + if key in data_dict.keys(): + data_dict[key] = value + continue + self._add_or_update_harvest_metadata(key, value, data_dict) + + # Add harvest extras to validated_data_dict + validated_data_dict = json.loads(pkg_dict["validated_data_dict"]) + for key, value in harvest_extras: + if key in validated_data_dict.keys(): + validated_data_dict[key] = value + continue + self._add_or_update_harvest_metadata(key, value, validated_data_dict) + + # Add harvest extras to main indexed pkg_dict + for key, value in harvest_extras: + pkg_dict[key] = value + + pkg_dict["data_dict"] = json.dumps(data_dict) + pkg_dict["validated_data_dict"] = json.dumps(validated_data_dict) return pkg_dict From 62a16d0bfffdb847a1d1f379ef0fc1915a2238d7 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Mon, 13 Mar 2023 09:19:13 -0300 Subject: [PATCH 18/74] Improve docstring --- ckanext/harvest/plugin/__init__.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/ckanext/harvest/plugin/__init__.py b/ckanext/harvest/plugin/__init__.py index 6fea9a4b8..b66cbf10f 100644 --- a/ckanext/harvest/plugin/__init__.py +++ b/ckanext/harvest/plugin/__init__.py @@ -134,11 +134,11 @@ def before_dataset_index(self, pkg_dict): This method will add or update harvest related metadata in `pkg_dict`, `data_dict` and `validated_data_dict` so it can be obtained when calling package_show API (that depends on Solr data). This metadata will - be stored in the `extra` field of the dictionaries. + be stored in the `extras` field of the dictionaries ONLY if it does not + already exist in the root schema. - By default all harvest metadata will go in the extra field. If - another extension adds any of them to the `package_show` schema - then it will not be added again in the `extras` field to avoid + Note: If another extension adds any harvest extra to the `package_show` + schema then this method will add them again in the `extras` field to avoid validation errors when updating a package. """ # Fix to support Solr8 @@ -164,7 +164,6 @@ def before_dataset_index(self, pkg_dict): ("harvest_source_title", harvest_object.source.title), ] - # Add harvest extras to data_dict data_dict = json.loads(pkg_dict["data_dict"]) for key, value in harvest_extras: if key in data_dict.keys(): @@ -172,7 +171,6 @@ def before_dataset_index(self, pkg_dict): continue self._add_or_update_harvest_metadata(key, value, data_dict) - # Add harvest extras to validated_data_dict validated_data_dict = json.loads(pkg_dict["validated_data_dict"]) for key, value in harvest_extras: if key in validated_data_dict.keys(): From 10c2fecec389989ea69e3f1da73f5c20516f8d98 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Mon, 13 Mar 2023 12:04:11 -0300 Subject: [PATCH 19/74] Do not update extra in pkg_dict if it already exist. This is a responsibility of the package. We are skiping any override since users will expect the behaviour of the custom logic added in package schema and validators. --- ckanext/harvest/plugin/__init__.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ckanext/harvest/plugin/__init__.py b/ckanext/harvest/plugin/__init__.py index b66cbf10f..5eabbe09e 100644 --- a/ckanext/harvest/plugin/__init__.py +++ b/ckanext/harvest/plugin/__init__.py @@ -138,8 +138,11 @@ def before_dataset_index(self, pkg_dict): already exist in the root schema. Note: If another extension adds any harvest extra to the `package_show` - schema then this method will add them again in the `extras` field to avoid + schema then this method will not add them again in the `extras` field to avoid validation errors when updating a package. + + If the harvest extra has been added to the root schema, then we will not update + them since it is responsibility of the package validators to do it. """ # Fix to support Solr8 if isinstance(pkg_dict.get('status'), dict): @@ -180,7 +183,8 @@ def before_dataset_index(self, pkg_dict): # Add harvest extras to main indexed pkg_dict for key, value in harvest_extras: - pkg_dict[key] = value + if key not in pkg_dict.keys(): + pkg_dict[key] = value pkg_dict["data_dict"] = json.dumps(data_dict) pkg_dict["validated_data_dict"] = json.dumps(validated_data_dict) From 17e9102bb9ad010532686b1951b58ea20585de67 Mon Sep 17 00:00:00 2001 From: Fuhu Xia Date: Mon, 13 Mar 2023 13:27:13 -0400 Subject: [PATCH 20/74] Use 403 when actions are forbidden, not 401 following ckan core https://github.com/ckan/ckan/pull/2846 --- ckanext/harvest/utils.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/ckanext/harvest/utils.py b/ckanext/harvest/utils.py index 7acaaea59..d6298ea61 100644 --- a/ckanext/harvest/utils.py +++ b/ckanext/harvest/utils.py @@ -517,7 +517,7 @@ def _get_source_for_job(source_id): except tk.ObjectNotFound: return tk.abort(404, _('Harvest source not found')) except tk.NotAuthorized: - return tk.abort(401, _not_auth_message()) + return tk.abort(403, _not_auth_message()) except Exception as e: msg = 'An error occurred: [%s]' % str(e) return tk.abort(500, msg) @@ -537,7 +537,7 @@ def admin_view(id): except tk.ObjectNotFound: return tk.abort(404, _('Harvest source not found')) except tk.NotAuthorized: - return tk.abort(401, _not_auth_message()) + return tk.abort(403, _not_auth_message()) def job_show_last_view(source): @@ -579,7 +579,7 @@ def job_show_view(id, source_dict=False, is_last=False): except tk.ObjectNotFound: return tk.abort(404, _('Harvest job not found')) except tk.NotAuthorized: - return tk.abort(401, _not_auth_message()) + return tk.abort(403, _not_auth_message()) except Exception as e: msg = 'An error occurred: [%s]' % str(e) return tk.abort(500, msg) @@ -607,7 +607,7 @@ def job_list_view(source): except tk.ObjectNotFound: return tk.abort(404, _('Harvest source not found')) except tk.NotAuthorized: - return tk.abort(401, _not_auth_message()) + return tk.abort(403, _not_auth_message()) except Exception as e: msg = 'An error occurred: [%s]' % str(e) return tk.abort(500, msg) @@ -625,7 +625,7 @@ def about_view(id): except tk.ObjectNotFound: return tk.abort(404, _('Harvest source not found')) except tk.NotAuthorized: - return tk.abort(401, _not_auth_message()) + return tk.abort(403, _not_auth_message()) def job_abort_view(source, id): @@ -638,7 +638,7 @@ def job_abort_view(source, id): except tk.ObjectNotFound: return tk.abort(404, _('Harvest job not found')) except tk.NotAuthorized: - return tk.abort(401, _not_auth_message()) + return tk.abort(403, _not_auth_message()) except Exception as e: msg = 'An error occurred: [%s]' % str(e) return tk.abort(500, msg) @@ -659,7 +659,7 @@ def refresh_view(id): except tk.ObjectNotFound: return tk.abort(404, _('Harvest source not found')) except tk.NotAuthorized: - return tk.abort(401, _not_auth_message()) + return tk.abort(403, _not_auth_message()) except HarvestSourceInactiveError: h.flash_error( _('Cannot create new harvest jobs on inactive ' @@ -685,7 +685,7 @@ def clear_view(id): except tk.ObjectNotFound: return tk.abort(404, _('Harvest source not found')) except tk.NotAuthorized: - return tk.abort(401, _not_auth_message()) + return tk.abort(403, _not_auth_message()) except Exception as e: msg = 'An error occurred: [%s]' % str(e) h.flash_error(msg) @@ -716,7 +716,7 @@ def delete_view(id): except tk.ObjectNotFound: return tk.abort(404, _('Harvest source not found')) except tk.NotAuthorized: - return tk.abort(401, _not_auth_message()) + return tk.abort(403, _not_auth_message()) def object_show_view(id, ref_type, response): @@ -762,7 +762,7 @@ def object_show_view(id, ref_type, response): except tk.ObjectNotFound as e: return tk.abort(404, _(str(e))) except tk.NotAuthorized: - return tk.abort(401, _not_auth_message()) + return tk.abort(403, _not_auth_message()) except Exception as e: msg = 'An error occurred: [%s]' % str(e) return tk.abort(500, msg) From 2e87cf685f06c64403b1add15f59af0263a39f1e Mon Sep 17 00:00:00 2001 From: Patricio Del Boca Date: Thu, 16 Mar 2023 12:25:57 -0300 Subject: [PATCH 21/74] Bump version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8448e52f5..ee9b08543 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages -version = '1.4.2' +version = '1.5.0' setup( name='ckanext-harvest', From 631c7f2e68fa64b214098f83af5c0c735702788b Mon Sep 17 00:00:00 2001 From: pdelboca Date: Wed, 22 Mar 2023 09:18:37 -0300 Subject: [PATCH 22/74] Fix url_for routing to point to harvester blueprint. --- ckanext/harvest/utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ckanext/harvest/utils.py b/ckanext/harvest/utils.py index 2ba946990..ca61490c6 100644 --- a/ckanext/harvest/utils.py +++ b/ckanext/harvest/utils.py @@ -633,7 +633,7 @@ def job_abort_view(source, id): return tk.abort(500, msg) return h.redirect_to( - h.url_for('{0}.admin'.format(DATASET_TYPE_NAME), id=source)) + h.url_for('harvester.admin', id=source)) def refresh_view(id): @@ -663,7 +663,7 @@ def refresh_view(id): h.flash_error(msg) return h.redirect_to( - h.url_for('{0}.admin'.format(DATASET_TYPE_NAME), id=id)) + h.url_for('harvester.admin', id=id)) def clear_view(id): @@ -680,7 +680,7 @@ def clear_view(id): h.flash_error(msg) return h.redirect_to( - h.url_for('{0}.admin'.format(DATASET_TYPE_NAME), id=id)) + h.url_for('harvester.admin', id=id)) def delete_view(id): @@ -701,7 +701,7 @@ def delete_view(id): h.flash_success(_('Harvesting source successfully inactivated')) return h.redirect_to( - h.url_for('{0}.admin'.format(DATASET_TYPE_NAME), id=id)) + h.url_for('harvester.admin', id=id)) except tk.ObjectNotFound: return tk.abort(404, _('Harvest source not found')) except tk.NotAuthorized: From 33f17718a64c3f050a6dd04afa1d39de9911a512 Mon Sep 17 00:00:00 2001 From: Patricio Del Boca Date: Wed, 22 Mar 2023 11:12:52 -0300 Subject: [PATCH 23/74] Bump version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ee9b08543..c87d225eb 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages -version = '1.5.0' +version = '1.5.1' setup( name='ckanext-harvest', From 4fddafcb56dd8e91e8fad6567cd6dd8856b2f816 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Tue, 28 Mar 2023 09:13:59 -0300 Subject: [PATCH 24/74] Fix endpoint for object_show --- ckanext/harvest/helpers.py | 2 +- ckanext/harvest/templates/source/job/read.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ckanext/harvest/helpers.py b/ckanext/harvest/helpers.py index 87b3e3c5b..34696ad09 100644 --- a/ckanext/harvest/helpers.py +++ b/ckanext/harvest/helpers.py @@ -124,7 +124,7 @@ def link_for_harvest_object(id=None, guid=None, text=None): obj = logic.get_action('harvest_object_show')(context, {'id': guid, 'attr': 'guid'}) id = obj.id - url = h.url_for('harvest.object_show', id=id) + url = h.url_for('harvester.object_show', id=id) text = text or guid or id link = '{text}'.format(url=url, text=text) diff --git a/ckanext/harvest/templates/source/job/read.html b/ckanext/harvest/templates/source/job/read.html index 25d32e125..87e16c6a0 100644 --- a/ckanext/harvest/templates/source/job/read.html +++ b/ckanext/harvest/templates/source/job/read.html @@ -69,7 +69,7 @@

      {{ _('Document Errors') }} {{ _('Remote content') }} {% endif %} - + {{ _('Local content') }} From b6c87f4cec0fd1010796b68976380222407360d9 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Tue, 28 Mar 2023 09:15:10 -0300 Subject: [PATCH 25/74] Clean deprecated params attribute --- ckanext/harvest/helpers.py | 2 +- ckanext/harvest/templates/source/search.html | 2 +- ckanext/harvest/utils.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ckanext/harvest/helpers.py b/ckanext/harvest/helpers.py index 34696ad09..c1b452e7f 100644 --- a/ckanext/harvest/helpers.py +++ b/ckanext/harvest/helpers.py @@ -36,7 +36,7 @@ def package_list_for_source(source_id): It calls the package_list snippet and the pager. ''' limit = 20 - page = int(request.params.get('page', 1)) + page = int(request.args.get('page', 1)) fq = '+harvest_source_id:"{0}"'.format(source_id) search_dict = { 'fq': fq, diff --git a/ckanext/harvest/templates/source/search.html b/ckanext/harvest/templates/source/search.html index a92994322..d9ceeeaa1 100644 --- a/ckanext/harvest/templates/source/search.html +++ b/ckanext/harvest/templates/source/search.html @@ -31,7 +31,7 @@ (_('Last Modified'), 'metadata_modified desc'), (_('Popular'), 'views_recent desc') if g.tracking_enabled else (false, false) ] %} - {% snippet 'snippets/search_form.html', type='harvest', query=c.q, sorting=sorting, sorting_selected=c.sort_by_selected, count=c.page.item_count, facets=facets, show_empty=request.params, error=c.query_error, placeholder=_("Search harvest sources...") %} + {% snippet 'snippets/search_form.html', type='harvest', query=c.q, sorting=sorting, sorting_selected=c.sort_by_selected, count=c.page.item_count, facets=facets, show_empty=request.args, error=c.query_error, placeholder=_("Search harvest sources...") %} {{ h.snippet('snippets/source_list.html', sources=c.page.items, show_organization=true) }} diff --git a/ckanext/harvest/utils.py b/ckanext/harvest/utils.py index ca61490c6..557e883fa 100644 --- a/ckanext/harvest/utils.py +++ b/ckanext/harvest/utils.py @@ -687,7 +687,7 @@ def delete_view(id): try: context = {'model': model, 'user': tk.c.user} - context['clear_source'] = tk.request.params.get('clear', + context['clear_source'] = tk.request.args.get('clear', '').lower() in ( u'true', u'1', From 540716860a9ab3de8d00cb5a0aa398aa552e1705 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Tue, 28 Mar 2023 09:52:42 -0300 Subject: [PATCH 26/74] Fix lint --- ckanext/harvest/utils.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/ckanext/harvest/utils.py b/ckanext/harvest/utils.py index 557e883fa..c17cc74ef 100644 --- a/ckanext/harvest/utils.py +++ b/ckanext/harvest/utils.py @@ -686,12 +686,8 @@ def clear_view(id): def delete_view(id): try: context = {'model': model, 'user': tk.c.user} - - context['clear_source'] = tk.request.args.get('clear', - '').lower() in ( - u'true', - u'1', - ) + clear = tk.request.args.get('clear','').lower() + context['clear_source'] = clear in ('true','1',) tk.get_action('harvest_source_delete')(context, {'id': id}) From 85ae74514ec42f0300d94a22adf8b7cd344630f1 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Tue, 28 Mar 2023 13:49:47 -0300 Subject: [PATCH 27/74] Fix lint --- ckanext/harvest/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckanext/harvest/utils.py b/ckanext/harvest/utils.py index c17cc74ef..a58c7f598 100644 --- a/ckanext/harvest/utils.py +++ b/ckanext/harvest/utils.py @@ -686,8 +686,8 @@ def clear_view(id): def delete_view(id): try: context = {'model': model, 'user': tk.c.user} - clear = tk.request.args.get('clear','').lower() - context['clear_source'] = clear in ('true','1',) + clear = tk.request.args.get('clear', '').lower() + context['clear_source'] = clear in ('true', '1', ) tk.get_action('harvest_source_delete')(context, {'id': id}) From 40810cd0342592b0e83cb356b8cea8c807f251dd Mon Sep 17 00:00:00 2001 From: pdelboca Date: Tue, 28 Mar 2023 13:50:03 -0300 Subject: [PATCH 28/74] Bump version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c87d225eb..869d11112 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages -version = '1.5.1' +version = '1.5.2' setup( name='ckanext-harvest', From 0e9db08cb2bfcc9af0517c415bc04a90a62ba4e6 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Mon, 3 Apr 2023 12:32:54 -0300 Subject: [PATCH 29/74] Fix asset folder in MANIFEST.in --- MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 4df465c8e..3a1395bc7 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,4 @@ recursive-include ckanext/harvest/templates * -recursive-include ckanext/harvest/fanstatic_library * +recursive-include ckanext/harvest/assets * recursive-include ckanext/harvest/public * recursive-include ckanext/harvest/i18n * From 4d4faa54235bc1a592fec9cc7624201804450b76 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Mon, 3 Apr 2023 12:36:04 -0300 Subject: [PATCH 30/74] Bump version number --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 869d11112..6bc8b29b0 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages -version = '1.5.2' +version = '1.5.3' setup( name='ckanext-harvest', From aff5184dc6906c6e111eb835d5e92af61bb3fb52 Mon Sep 17 00:00:00 2001 From: seitenbau-govdata Date: Thu, 4 May 2023 09:59:14 +0200 Subject: [PATCH 31/74] Fix a problem with data-dictization of lists to fix search-index rebuild --- CHANGELOG.rst | 2 ++ ckanext/harvest/logic/dictization.py | 14 ++++++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8d32ef8d7..99f6cbdf6 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,6 +11,8 @@ and this project adheres to `Semantic Versioning `_ Unreleased_ *********** +- Fix a problem with data-dictization when using sqlalchemy 1.4+ + *********** 1.4.2_ - 2023-01-12 *********** diff --git a/ckanext/harvest/logic/dictization.py b/ckanext/harvest/logic/dictization.py index d6dbedd86..5e1a39b0d 100644 --- a/ckanext/harvest/logic/dictization.py +++ b/ckanext/harvest/logic/dictization.py @@ -68,7 +68,7 @@ def harvest_job_dictize(job, context): .group_by(HarvestObjectError.message) \ .order_by(text('error_count desc')) \ .limit(context.get('error_summmary_limit', 20)) - out['object_error_summary'] = q.all() + out['object_error_summary'] = _dictize_list(q.all()) q = model.Session.query( HarvestGatherError.message, func.count(HarvestGatherError.message).label('error_count')) \ @@ -76,7 +76,7 @@ def harvest_job_dictize(job, context): .group_by(HarvestGatherError.message) \ .order_by(text('error_count desc')) \ .limit(context.get('error_summmary_limit', 20)) - out['gather_error_summary'] = q.all() + out['gather_error_summary'] = _dictize_list(q.all()) return out @@ -144,3 +144,13 @@ def _get_source_status(source, context): out['last_harvest_request'] = 'Not yet harvested' return out + + +def _dictize_list(db_result_list): + ''' + Helper method to dictize all elements of a database result list. + ''' + dictized_list = [] + for elem in db_result_list: + dictized_list.append(elem._asdict()) + return dictized_list From 9926cac529bcedf56369fba9e6afd75045bbc04f Mon Sep 17 00:00:00 2001 From: seitenbau-govdata Date: Tue, 9 May 2023 13:10:39 +0200 Subject: [PATCH 32/74] Refactor changes in dictization.py --- ckanext/harvest/logic/dictization.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/ckanext/harvest/logic/dictization.py b/ckanext/harvest/logic/dictization.py index 5e1a39b0d..00e8ec984 100644 --- a/ckanext/harvest/logic/dictization.py +++ b/ckanext/harvest/logic/dictization.py @@ -68,7 +68,7 @@ def harvest_job_dictize(job, context): .group_by(HarvestObjectError.message) \ .order_by(text('error_count desc')) \ .limit(context.get('error_summmary_limit', 20)) - out['object_error_summary'] = _dictize_list(q.all()) + out['object_error_summary'] = harvest_error_dictize(q.all(), context) q = model.Session.query( HarvestGatherError.message, func.count(HarvestGatherError.message).label('error_count')) \ @@ -76,7 +76,8 @@ def harvest_job_dictize(job, context): .group_by(HarvestGatherError.message) \ .order_by(text('error_count desc')) \ .limit(context.get('error_summmary_limit', 20)) - out['gather_error_summary'] = _dictize_list(q.all()) + out['gather_error_summary'] = harvest_error_dictize(q.all(), context) + return out @@ -106,6 +107,13 @@ def harvest_log_dictize(obj, context): return out +def harvest_error_dictize(obj, context): + out = [] + for elem in obj: + out.append(elem._asdict()) + return out + + def _get_source_status(source, context): ''' TODO: Deprecated, use harvest_source_show_status instead @@ -144,13 +152,3 @@ def _get_source_status(source, context): out['last_harvest_request'] = 'Not yet harvested' return out - - -def _dictize_list(db_result_list): - ''' - Helper method to dictize all elements of a database result list. - ''' - dictized_list = [] - for elem in db_result_list: - dictized_list.append(elem._asdict()) - return dictized_list From bae212784ba309409bb079605e622eb7028740f9 Mon Sep 17 00:00:00 2001 From: seitenbau-govdata Date: Tue, 9 May 2023 16:02:06 +0200 Subject: [PATCH 33/74] Add new testcase for harvest_source_show_status --- ckanext/harvest/tests/test_action.py | 30 ++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/ckanext/harvest/tests/test_action.py b/ckanext/harvest/tests/test_action.py index d2927d599..1c937f492 100644 --- a/ckanext/harvest/tests/test_action.py +++ b/ckanext/harvest/tests/test_action.py @@ -755,3 +755,33 @@ def test_harvest_job_create_as_admin(self): assert job['status'] == 'Running' assert job['gather_started'] is None assert 'stats' in job.keys() + + def test_harvest_source_show_status(self): + + source = factories.HarvestSourceObj(**SOURCE_DICT.copy()) + job = factories.HarvestJobObj(source=source) + dataset = ckan_factories.Dataset() + obj = factories.HarvestObjectObj( + job=job, source=source, package_id=dataset['id']) + + harvest_gather_error = harvest_model.HarvestGatherError(message="Unexpected gather error", job=job) + harvest_gather_error.save() + harvest_object_error = harvest_model.HarvestObjectError(message="Unexpected object error", object=obj) + harvest_object_error.save() + + context = {'model': model} + data_dict = {'id': source.id} + + source_status = get_action('harvest_source_show_status')(context, data_dict) + + # verifiy that the response is dictized properly + json.dumps(source_status) + + last_job = source_status['last_job'] + assert last_job['source_id'] == source.id + assert last_job['status'] == 'New' + assert last_job['stats']['errored'] == 2 + assert len(last_job['object_error_summary']) == 1 + assert last_job['object_error_summary'][0]['message'] == harvest_object_error.message + assert len(last_job['gather_error_summary']) == 1 + assert last_job['gather_error_summary'][0]['message'] == harvest_gather_error.message From b24421676be23ab627cb0307f4c7dd64cdf49e5e Mon Sep 17 00:00:00 2001 From: amercader Date: Tue, 23 May 2023 13:19:42 +0200 Subject: [PATCH 34/74] Bump version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6bc8b29b0..c5668bd00 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages -version = '1.5.3' +version = '1.5.4' setup( name='ckanext-harvest', From ec57de3492c2eb117e1d105b714017e57225c907 Mon Sep 17 00:00:00 2001 From: seitenbau-govdata Date: Thu, 1 Jun 2023 10:30:02 +0200 Subject: [PATCH 35/74] Fix display of harvest job errors --- ckanext/harvest/templates/snippets/job_error_summary.html | 6 +++--- ckanext/harvest/tests/test_action.py | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ckanext/harvest/templates/snippets/job_error_summary.html b/ckanext/harvest/templates/snippets/job_error_summary.html index 2fb7b0314..5bad19f72 100644 --- a/ckanext/harvest/templates/snippets/job_error_summary.html +++ b/ckanext/harvest/templates/snippets/job_error_summary.html @@ -1,7 +1,7 @@ {# Displays a table with a summary of the most common errors for a job -error_summary - List of tuples with (message, count) +error_summary - List of dicts with message and error_count Example: @@ -22,8 +22,8 @@ {% for error in summary %} - {{ error[1] }} - {{ error[0] }} + {{ error["error_count"] }} + {{ error["message"] }} {% endfor %} diff --git a/ckanext/harvest/tests/test_action.py b/ckanext/harvest/tests/test_action.py index 1c937f492..e22bb5cd5 100644 --- a/ckanext/harvest/tests/test_action.py +++ b/ckanext/harvest/tests/test_action.py @@ -783,5 +783,7 @@ def test_harvest_source_show_status(self): assert last_job['stats']['errored'] == 2 assert len(last_job['object_error_summary']) == 1 assert last_job['object_error_summary'][0]['message'] == harvest_object_error.message + assert last_job['object_error_summary'][0]['error_count'] == 1 assert len(last_job['gather_error_summary']) == 1 assert last_job['gather_error_summary'][0]['message'] == harvest_gather_error.message + assert last_job['gather_error_summary'][0]['error_count'] == 1 From 629004d600cc1073af182f2cfc82236b1f40c262 Mon Sep 17 00:00:00 2001 From: seitenbau-govdata Date: Mon, 5 Jun 2023 09:44:51 +0200 Subject: [PATCH 36/74] Update Changelog --- CHANGELOG.rst | 74 ++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 71 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 99f6cbdf6..57ade59f2 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,10 +8,67 @@ The format is based on `Keep a Changelog `_ and this project adheres to `Semantic Versioning `_ *********** -Unreleased_ +1.5.5_ - 2023-06-05 *********** -- Fix a problem with data-dictization when using sqlalchemy 1.4+ +Fixed +------- + +- Fix display of harvest job errors #533 + +*********** +1.5.4_ - 2023-05-23 +*********** + +Fixed +------- + +- Fix a problem with data-dictization when using sqlalchemy 1.4+ #529 + +*********** +1.5.3_ - 2023-04-03 +*********** + +Fixed +------- + +- Fix asset path in MANIFEST.in #525 + +*********** +1.5.2_ - 2023-03-28 +*********** + +Fixed +------- + +- Fix URL endpoints: from ``harvest.object_show`` to ``harvester.object_show`` #524 + +*********** +1.5.1_ - 2023-03-22 +*********** + +Fixed +------- + +- Fix ``url_for`` routing to point to harvester blueprint #523 + +*********** +1.5.0_ - 2023-03-16 +*********** + +Changed +------- + +- Added unescape for email text body to avoid encoded characters #517 +- Pick the right harvest_object_id if there are multiple #519 +- Do not duplicate harvest_extras if exist in root schema #521 +- Use 403 when actions are forbidden, not 401 #522 +- Drop support old versions #520 + +Breaking Changes +------- +- ``h.bootstrap_version()`` no longer exist since it is no longer needed to inject CSS classes +- Support for old Pylon's route syntax has been removed. Example: calling ``url_for("harvest_read")`` will no longer work. URLs for ``ckanext-harvest`` needs to respect Flask's syntax: ``url_for("harvest.read")``, etc *********** 1.4.2_ - 2023-01-12 @@ -347,7 +404,18 @@ Categories - ``Fixed`` for any bug fixes. - ``Security`` to invite users to upgrade in case of vulnerabilities. -.. _Unreleased: https://github.com/ckan/ckanext-harvest/compare/v1.3.2...HEAD +.. _Unreleased: https://github.com/ckan/ckanext-harvest/compare/v1.5.5...HEAD +.. _1.5.5: https://github.com/ckan/ckanext-harvest/compare/v1.5.4...v1.5.5 +.. _1.5.4: https://github.com/ckan/ckanext-harvest/compare/v1.5.3...v1.5.4 +.. _1.5.3: https://github.com/ckan/ckanext-harvest/compare/v1.5.2...v1.5.3 +.. _1.5.2: https://github.com/ckan/ckanext-harvest/compare/v1.5.1...v1.5.2 +.. _1.5.1: https://github.com/ckan/ckanext-harvest/compare/v1.5.0...v1.5.1 +.. _1.5.0: https://github.com/ckan/ckanext-harvest/compare/v1.4.2...v1.5.0 +.. _1.4.2: https://github.com/ckan/ckanext-harvest/compare/v1.4.1...v1.4.2 +.. _1.4.1: https://github.com/ckan/ckanext-harvest/compare/v1.4.0...v1.4.1 +.. _1.4.0: https://github.com/ckan/ckanext-harvest/compare/v1.3.4...v1.4.0 +.. _1.3.4: https://github.com/ckan/ckanext-harvest/compare/v1.3.3...v1.3.4 +.. _1.3.3: https://github.com/ckan/ckanext-harvest/compare/v1.3.2...v1.3.3 .. _1.3.2: https://github.com/ckan/ckanext-harvest/compare/v1.3.1...v1.3.2 .. _1.3.1: https://github.com/ckan/ckanext-harvest/compare/v1.3.0...v1.3.1 .. _1.3.0: https://github.com/ckan/ckanext-harvest/compare/v1.2.1...v1.3.0 From 6576835646edb024f61f797c6d20abc51e3fa84c Mon Sep 17 00:00:00 2001 From: seitenbau-govdata Date: Mon, 5 Jun 2023 09:45:25 +0200 Subject: [PATCH 37/74] Update version to 1.5.5 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c5668bd00..20a364de7 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages -version = '1.5.4' +version = '1.5.5' setup( name='ckanext-harvest', From c46c39d0a12b467811d594b96257e8d47b2ac4fb Mon Sep 17 00:00:00 2001 From: Jin-Sun-tts Date: Fri, 23 Jun 2023 09:35:17 -0400 Subject: [PATCH 38/74] fix url endpoint for job_show --- ckanext/harvest/logic/action/update.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ckanext/harvest/logic/action/update.py b/ckanext/harvest/logic/action/update.py index 924f478d6..7dce1f575 100644 --- a/ckanext/harvest/logic/action/update.py +++ b/ckanext/harvest/logic/action/update.py @@ -758,7 +758,7 @@ def get_mail_extra_vars(context, source_id, status): errors = job_errors + obj_errors site_url = config.get('ckan.site_url') - job_url = toolkit.url_for('harvest.job_show', source=source['id'], id=last_job['id']) + job_url = toolkit.url_for('harvester.job_show', source=source['id'], id=last_job['id']) full_job_url = urljoin(site_url, job_url) extra_vars = { 'organization': organization, diff --git a/setup.py b/setup.py index 20a364de7..f1efc2088 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages -version = '1.5.5' +version = '1.5.6' setup( name='ckanext-harvest', From 0f57dc0c3ff8548ab880e589c3a0447e728dd103 Mon Sep 17 00:00:00 2001 From: Jin-Sun-tts Date: Fri, 23 Jun 2023 12:55:03 -0400 Subject: [PATCH 39/74] revert version bump --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f1efc2088..20a364de7 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages -version = '1.5.6' +version = '1.5.5' setup( name='ckanext-harvest', From 9fb44f79809a1c04dfeb0e1ca2540c5ff3cacef4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20N=C3=B6renberg?= Date: Mon, 26 Jun 2023 10:44:41 +0200 Subject: [PATCH 40/74] Bump version --- CHANGELOG.rst | 12 +++++++++++- setup.py | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 57ade59f2..63a438564 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,15 @@ All notable changes to this project will be documented in this file. The format is based on `Keep a Changelog `_ and this project adheres to `Semantic Versioning `_ +*********** +1.5.6_ - 2023-06-26 +*********** + +Fixed +------- + +- Fix url endpoint for job_show #534 + *********** 1.5.5_ - 2023-06-05 *********** @@ -404,7 +413,8 @@ Categories - ``Fixed`` for any bug fixes. - ``Security`` to invite users to upgrade in case of vulnerabilities. -.. _Unreleased: https://github.com/ckan/ckanext-harvest/compare/v1.5.5...HEAD +.. _Unreleased: https://github.com/ckan/ckanext-harvest/compare/v1.5.6...HEAD +.. _1.5.6: https://github.com/ckan/ckanext-harvest/compare/v1.5.5...v1.5.6 .. _1.5.5: https://github.com/ckan/ckanext-harvest/compare/v1.5.4...v1.5.5 .. _1.5.4: https://github.com/ckan/ckanext-harvest/compare/v1.5.3...v1.5.4 .. _1.5.3: https://github.com/ckan/ckanext-harvest/compare/v1.5.2...v1.5.3 diff --git a/setup.py b/setup.py index 20a364de7..f1efc2088 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages -version = '1.5.5' +version = '1.5.6' setup( name='ckanext-harvest', From f75f27b0ab68fb6927f1834be9ac41037c519152 Mon Sep 17 00:00:00 2001 From: Jari Voutilainen Date: Tue, 22 Aug 2023 12:30:26 +0300 Subject: [PATCH 41/74] Fix lint failure --- ckanext/harvest/tests/harvesters/test_ckanharvester.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanext/harvest/tests/harvesters/test_ckanharvester.py b/ckanext/harvest/tests/harvesters/test_ckanharvester.py index e018f81d0..b90f2ad8d 100644 --- a/ckanext/harvest/tests/harvesters/test_ckanharvester.py +++ b/ckanext/harvest/tests/harvesters/test_ckanharvester.py @@ -49,7 +49,7 @@ def test_gather_normal(self): obj_ids = harvester.gather_stage(job) assert job.gather_errors == [] - assert type(obj_ids) == list + assert isinstance(obj_ids, list) assert len(obj_ids) == len(mock_ckan.DATASETS) harvest_object = harvest_model.HarvestObject.get(obj_ids[0]) assert harvest_object.guid == mock_ckan.DATASETS[0]['id'] From 4d300962f75d3d09a0965eacfcbe4baf896fb9b6 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Thu, 2 Nov 2023 17:45:42 +0200 Subject: [PATCH 42/74] feat: switch to alembic migrations --- README.rst | 5 +- ckanext/harvest/cli.py | 9 - ckanext/harvest/commands/__init__.py | 8 - ckanext/harvest/commands/harvester.py | 447 ------------ ckanext/harvest/migration/harvest/README | 1 + ckanext/harvest/migration/harvest/alembic.ini | 74 ++ ckanext/harvest/migration/harvest/env.py | 81 +++ .../harvest/migration/harvest/script.py.mako | 24 + .../3b4894672727_create_harvest_tables.py | 187 +++++ ...650dfd519_add_cascade_to_harvest_tables.py | 102 +++ ckanext/harvest/model/__init__.py | 640 ++++++++---------- ckanext/harvest/plugin.py | 4 - ckanext/harvest/utils.py | 6 - setup.py | 2 - 14 files changed, 755 insertions(+), 835 deletions(-) delete mode 100644 ckanext/harvest/commands/__init__.py delete mode 100644 ckanext/harvest/commands/harvester.py create mode 100644 ckanext/harvest/migration/harvest/README create mode 100644 ckanext/harvest/migration/harvest/alembic.ini create mode 100644 ckanext/harvest/migration/harvest/env.py create mode 100644 ckanext/harvest/migration/harvest/script.py.mako create mode 100644 ckanext/harvest/migration/harvest/versions/3b4894672727_create_harvest_tables.py create mode 100644 ckanext/harvest/migration/harvest/versions/75d650dfd519_add_cascade_to_harvest_tables.py diff --git a/README.rst b/README.rst index e8d5893ac..ad30f13cb 100644 --- a/README.rst +++ b/README.rst @@ -94,7 +94,7 @@ Configuration Run the following command to create the necessary tables in the database (ensuring the pyenv is activated): - (pyenv) $ ckan --config=/etc/ckan/default/ckan.ini harvester initdb + (pyenv) $ ckan --config=/etc/ckan/default/ckan.ini db upgrade -p harvest Finally, restart CKAN to have the changes take effect:: @@ -229,9 +229,6 @@ Command line interface The following operations can be run from the command line as described underneath:: - harvester initdb - - Creates the necessary tables in the database - harvester source {name} {url} {type} [{title}] [{active}] [{owner_org}] [{frequency}] [{config}] - create new harvest source diff --git a/ckanext/harvest/cli.py b/ckanext/harvest/cli.py index 20a750447..d5602beb9 100644 --- a/ckanext/harvest/cli.py +++ b/ckanext/harvest/cli.py @@ -20,14 +20,6 @@ def harvester(): pass -@harvester.command() -def initdb(): - """Creates the necessary tables in the database. - """ - utils.initdb() - click.secho(u"DB tables created", fg=u"green") - - @harvester.group() def source(): """Manage harvest sources @@ -337,7 +329,6 @@ def import_stage( import. e.g. 15af will run segments 1,5,a,f """ - ctx.invoke(initdb) flask_app = ctx.meta["flask_app"] with flask_app.test_request_context(): try: diff --git a/ckanext/harvest/commands/__init__.py b/ckanext/harvest/commands/__init__.py deleted file mode 100644 index 2f4b4738a..000000000 --- a/ckanext/harvest/commands/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -try: - import pkg_resources - - pkg_resources.declare_namespace(__name__) -except ImportError: - import pkgutil - - __path__ = pkgutil.extend_path(__path__, __name__) diff --git a/ckanext/harvest/commands/harvester.py b/ckanext/harvest/commands/harvester.py deleted file mode 100644 index daa482aea..000000000 --- a/ckanext/harvest/commands/harvester.py +++ /dev/null @@ -1,447 +0,0 @@ -from __future__ import print_function -import sys - -from ckan import model -from ckan.logic import get_action, ValidationError - -from ckantoolkit import CkanCommand - -import ckanext.harvest.utils as utils -from ckanext.harvest.logic.schema import unicode_safe - - -class Harvester(CkanCommand): - """Harvests remotely mastered metadata - - Usage: - - harvester initdb - - Creates the necessary tables in the database - - harvester source {name} {url} {type} [{title}] [{active}] [{owner_org}] [{frequency}] [{config}] - - create new harvest source - - harvester source {source-id/name} - - shows a harvest source - - harvester rmsource {source-id/name} - - remove (deactivate) a harvester source, whilst leaving any related - datasets, jobs and objects - - harvester clearsource {source-id/name} - - clears all datasets, jobs and objects related to a harvest source, - but keeps the source itself - - harvester clearsource_history [{source-id}] [-k] - - If no source id is given the history for all harvest sources (maximum is 1000) will be cleared. - Clears all jobs and objects related to a harvest source, but keeps the source itself. - The datasets imported from the harvest source will NOT be deleted!!! - If a source id is given, it only clears the history of the harvest source with the given source id. - - To keep the currently active jobs use the -k option. - - harvester sources [all] - - lists harvest sources - If 'all' is defined, it also shows the Inactive sources - - harvester job {source-id/name} - - create new harvest job and runs it (puts it on the gather queue) - - harvester jobs - - lists harvest jobs - - harvester job_abort {source-id/source-name/obj-id} - - marks a job as "Aborted" so that the source can be restarted afresh. - It ensures that the job's harvest objects status are also marked - finished. You should ensure that neither the job nor its objects are - currently in the gather/fetch queues. - - harvester run - - starts any harvest jobs that have been created by putting them onto - the gather queue. Also checks running jobs - if finished it - changes their status to Finished. - - harvester run_test {source-id/name} - - runs a harvest - for testing only. - This does all the stages of the harvest (creates job, gather, fetch, - import) without involving the web UI or the queue backends. This is - useful for testing a harvester without having to fire up - gather/fetch_consumer processes, as is done in production. - - harvester gather_consumer - - starts the consumer for the gathering queue - - harvester fetch_consumer - - starts the consumer for the fetching queue - - harvester purge_queues - - removes all jobs from fetch and gather queue - - harvester abort_failed_jobs {job_life_span} [--include={source_id}] [--exclude={source_id}] - - abort all jobs which are in a "limbo state" where the job has - run with errors but the harvester run command will not mark it - as finished, and therefore you cannot run another job. - - job_life_span determines from what moment - the job must be considered as failed - - harvester clean_harvest_log - - Clean-up mechanism for the harvest log table. - You can configure the time frame through the configuration - parameter `ckan.harvest.log_timeframe`. The default time frame is 30 days - - harvester [-j] [-o|-g|-p {id/guid}] [--segments={segments}] import [{source-id}] - - perform the import stage with the last fetched objects, for a certain - source or a single harvest object. Please note that no objects will - be fetched from the remote server. It will only affect the objects - already present in the database. - - To import a particular harvest source, specify its id as an argument. - To import a particular harvest object use the -o option. - To import a particular guid use the -g option. - To import a particular package use the -p option. - - You will need to specify the -j flag in cases where the datasets are - not yet created (e.g. first harvest, or all previous harvests have - failed) - - The --segments flag allows to define a string containing hex digits that represent which of - the 16 harvest object segments to import. e.g. 15af will run segments 1,5,a,f - - harvester job-all - - create new harvest jobs for all active sources. - - harvester reindex - - reindexes the harvest source datasets - - The commands should be run from the ckanext-harvest directory and expect - a development.ini file to be present. Most of the time you will - specify the config explicitly though:: - - paster harvester sources --config=../ckan/development.ini - - """ - - summary = __doc__.split("\n")[0] - usage = __doc__ - max_args = 9 - min_args = 0 - - def __init__(self, name): - - super(Harvester, self).__init__(name) - - self.parser.add_option( - "-j", - "--no-join-datasets", - dest="no_join_datasets", - action="store_true", - default=False, - help="Do not join harvest objects to existing datasets", - ) - - self.parser.add_option( - "-o", - "--harvest-object-id", - dest="harvest_object_id", - default=False, - help="Id of the harvest object to which perform the import stage", - ) - - self.parser.add_option( - "-p", - "--package-id", - dest="package_id", - default=False, - help="Id of the package whose harvest object to perform the import stage for", - ) - - self.parser.add_option( - "-g", - "--guid", - dest="guid", - default=False, - help="Guid of the harvest object to which perform the import stage for", - ) - - self.parser.add_option( - "--segments", - dest="segments", - default=False, - help="""A string containing hex digits that represent which of - the 16 harvest object segments to import. e.g. 15af will run segments 1,5,a,f""", - ) - - self.parser.add_option( - "-i", - "--include", - dest="include_sources", - default=False, - help="""If source_id provided as included, then only it's failed jobs will be aborted. - You can use comma as a separator to provide multiple source_id's""", - ) - - self.parser.add_option( - "-e", - "--exclude", - dest="exclude_sources", - default=False, - help="""If source_id provided as excluded, all sources failed jobs, except for that - will be aborted. You can use comma as a separator to provide multiple source_id's""", - ) - - self.parser.add_option( - "-k", - "--keep-current", - dest="keep_current", - default=False, - help="Do not delete relevant harvest objects", - ) - - def command(self): - self._load_config() - - # We'll need a sysadmin user to perform most of the actions - # We will use the sysadmin site user (named as the site_id) - context = { - "model": model, - "session": model.Session, - "ignore_auth": True, - } - self.admin_user = get_action("get_site_user")(context, {}) - - print("") - - if len(self.args) == 0: - self.parser.print_usage() - sys.exit(1) - cmd = self.args[0] - if cmd == "source": - if len(self.args) > 2: - self.create_harvest_source() - else: - self.show_harvest_source() - elif cmd == "rmsource": - self.remove_harvest_source() - elif cmd == "clearsource": - self.clear_harvest_source() - elif cmd == "clearsource_history": - self.clear_harvest_source_history() - elif cmd == "sources": - self.list_harvest_sources() - elif cmd == "job": - self.create_harvest_job() - elif cmd == "jobs": - self.list_harvest_jobs() - elif cmd == "job_abort": - self.job_abort() - elif cmd == "run": - self.run_harvester() - elif cmd == "run_test": - self.run_test_harvest() - elif cmd == "gather_consumer": - utils.gather_consumer() - elif cmd == "fetch_consumer": - utils.fetch_consumer() - elif cmd == "purge_queues": - self.purge_queues() - elif cmd == "abort_failed_jobs": - self.abort_failed_jobs() - elif cmd == "initdb": - self.initdb() - elif cmd == "import": - self.initdb() - self.import_stage() - elif cmd == "job-all": - self.create_harvest_job_all() - elif cmd == "harvesters-info": - print(utils.harvesters_info()) - elif cmd == "reindex": - self.reindex() - elif cmd == "clean_harvest_log": - self.clean_harvest_log() - else: - print("Command {0} not recognized".format(cmd)) - - def _load_config(self): - super(Harvester, self)._load_config() - - def initdb(self): - utils.initdb() - print("DB tables created") - - def create_harvest_source(self): - - if len(self.args) >= 2: - name = unicode_safe(self.args[1]) - else: - print("Please provide a source name") - sys.exit(1) - if len(self.args) >= 3: - url = unicode_safe(self.args[2]) - else: - print("Please provide a source URL") - sys.exit(1) - if len(self.args) >= 4: - type = unicode_safe(self.args[3]) - else: - print("Please provide a source type") - sys.exit(1) - - if len(self.args) >= 5: - title = unicode_safe(self.args[4]) - else: - title = None - if len(self.args) >= 6: - active = not ( - self.args[5].lower() == "false" or self.args[5] == "0" - ) - else: - active = True - if len(self.args) >= 7: - owner_org = unicode_safe(self.args[6]) - else: - owner_org = None - if len(self.args) >= 8: - frequency = unicode_safe(self.args[7]) - if not frequency: - frequency = "MANUAL" - else: - frequency = "MANUAL" - if len(self.args) >= 9: - source_config = unicode_safe(self.args[8]) - else: - source_config = None - try: - result = utils.create_harvest_source( - name, url, type, title, active, owner_org, frequency, source_config - ) - except ValidationError as e: - print("An error occurred:") - print(str(e.error_dict)) - raise e - - print(result) - - def clear_harvest_source_history(self): - keep_current = bool(self.options.keep_current) - source_id = None - if len(self.args) >= 2: - source_id = unicode_safe(self.args[1]) - - print(utils.clear_harvest_source_history(source_id, keep_current)) - - def show_harvest_source(self): - - if len(self.args) >= 2: - source_id_or_name = unicode_safe(self.args[1]) - else: - print("Please provide a source name") - sys.exit(1) - print(utils.show_harvest_source(source_id_or_name)) - - def remove_harvest_source(self): - if len(self.args) >= 2: - source_id_or_name = unicode_safe(self.args[1]) - else: - print("Please provide a source id") - sys.exit(1) - utils.remove_harvest_source(source_id_or_name) - - def clear_harvest_source(self): - if len(self.args) >= 2: - source_id_or_name = unicode_safe(self.args[1]) - else: - print("Please provide a source id") - sys.exit(1) - utils.clear_harvest_source(source_id_or_name) - - def list_harvest_sources(self): - if len(self.args) >= 2 and self.args[1] == "all": - all = True - else: - all = False - - print(utils.list_sources(all)) - - def create_harvest_job(self): - if len(self.args) >= 2: - source_id_or_name = unicode_safe(self.args[1]) - else: - print("Please provide a source id") - sys.exit(1) - print(utils.create_job(source_id_or_name)) - - def list_harvest_jobs(self): - print(utils.list_jobs()) - - def job_abort(self): - if len(self.args) >= 2: - job_or_source_id_or_name = unicode_safe(self.args[1]) - else: - print("Please provide a job id or source name/id") - sys.exit(1) - print(utils.abort_job(job_or_source_id_or_name)) - - def run_harvester(self): - utils.run_harvester() - - def run_test_harvest(self): - # Determine the source - force_import = None - if len(self.args) >= 2: - if len(self.args) >= 3 and self.args[2].startswith('force-import='): - force_import = self.args[2].split('=')[-1] - source_id_or_name = unicode_safe(self.args[1]) - else: - print("Please provide a source id") - sys.exit(1) - - utils.run_test_harvester(source_id_or_name, force_import) - - def import_stage(self): - - if len(self.args) >= 2: - source_id_or_name = unicode_safe(self.args[1]) - context = { - "model": model, - "session": model.Session, - "user": self.admin_user["name"], - } - source = get_action("harvest_source_show")( - context, {"id": source_id_or_name} - ) - source_id = source["id"] - else: - source_id = None - utils.import_stage( - source_id, - self.options.no_join_datasets, - self.options.harvest_object_id, - self.options.guid, - self.options.package_id, - self.options.segments, - ) - - def create_harvest_job_all(self): - print(utils.job_all()) - - def reindex(self): - utils.reindex() - - def purge_queues(self): - utils.purge_queues() - - def clean_harvest_log(self): - utils.clean_harvest_log() - - def abort_failed_jobs(self): - job_life_span = False - if len(self.args) >= 2: - job_life_span = unicode_safe(self.args[1]) - - utils.abort_failed_jobs( - job_life_span, - include=self.options.include_sources, - exclude=self.options.exclude_sources - ) diff --git a/ckanext/harvest/migration/harvest/README b/ckanext/harvest/migration/harvest/README new file mode 100644 index 000000000..98e4f9c44 --- /dev/null +++ b/ckanext/harvest/migration/harvest/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/ckanext/harvest/migration/harvest/alembic.ini b/ckanext/harvest/migration/harvest/alembic.ini new file mode 100644 index 000000000..006fee785 --- /dev/null +++ b/ckanext/harvest/migration/harvest/alembic.ini @@ -0,0 +1,74 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = %(here)s + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# timezone to use when rendering the date +# within the migration file as well as the filename. +# string value is passed to dateutil.tz.gettz() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the +# "slug" field +#truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; this defaults +# to /home/sergey/projects/core/ckanext-harvest/ckanext/harvest/migration/harvest/versions. When using multiple version +# directories, initial revisions must be specified with --version-path +# version_locations = %(here)s/bar %(here)s/bat /home/sergey/projects/core/ckanext-harvest/ckanext/harvest/migration/harvest/versions + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +sqlalchemy.url = driver://user:pass@localhost/dbname + + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/ckanext/harvest/migration/harvest/env.py b/ckanext/harvest/migration/harvest/env.py new file mode 100644 index 000000000..009368218 --- /dev/null +++ b/ckanext/harvest/migration/harvest/env.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- + +from __future__ import with_statement +from alembic import context +from sqlalchemy import engine_from_config, pool +from logging.config import fileConfig + +import os + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +target_metadata = None + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + +name = os.path.basename(os.path.dirname(__file__)) + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + + url = config.get_main_option(u"sqlalchemy.url") + context.configure( + url=url, target_metadata=target_metadata, literal_binds=True, + version_table=u'{}_alembic_version'.format(name) + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section), + prefix=u'sqlalchemy.', + poolclass=pool.NullPool) + + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata, + version_table=u'{}_alembic_version'.format(name) + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/ckanext/harvest/migration/harvest/script.py.mako b/ckanext/harvest/migration/harvest/script.py.mako new file mode 100644 index 000000000..2c0156303 --- /dev/null +++ b/ckanext/harvest/migration/harvest/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/ckanext/harvest/migration/harvest/versions/3b4894672727_create_harvest_tables.py b/ckanext/harvest/migration/harvest/versions/3b4894672727_create_harvest_tables.py new file mode 100644 index 000000000..689e48f37 --- /dev/null +++ b/ckanext/harvest/migration/harvest/versions/3b4894672727_create_harvest_tables.py @@ -0,0 +1,187 @@ +"""create harvest tables + +Revision ID: 3b4894672727 +Revises: +Create Date: 2023-11-02 15:53:02.262586 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "3b4894672727" +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + engine = op.get_bind() + inspector = sa.inspect(engine) + + if not inspector.has_table("harvest_source"): + op.create_table( + "harvest_source", + sa.Column("id", sa.UnicodeText, primary_key=True), + sa.Column("url", sa.UnicodeText, nullable=False), + sa.Column("title", sa.UnicodeText), + sa.Column("description", sa.UnicodeText), + sa.Column("config", sa.UnicodeText), + sa.Column("created", sa.DateTime), + sa.Column("type", sa.UnicodeText, nullable=False), + sa.Column("active", sa.Boolean), + sa.Column("user_id", sa.UnicodeText), + sa.Column("publisher_id", sa.UnicodeText), + sa.Column("frequency", sa.UnicodeText), + sa.Column("next_run", sa.DateTime), + ) + + if not inspector.has_table("harvest_job"): + op.create_table( + "harvest_job", + sa.Column("id", sa.UnicodeText, primary_key=True), + sa.Column("created", sa.DateTime), + sa.Column("gather_started", sa.DateTime), + sa.Column("gather_finished", sa.DateTime), + sa.Column("finished", sa.DateTime), + sa.Column( + "source_id", + sa.UnicodeText, + sa.ForeignKey("harvest_source.id"), + ), + sa.Column("status", sa.UnicodeText, nullable=False), + ) + + if not inspector.has_table("harvest_object"): + op.create_table( + "harvest_object", + sa.Column("id", sa.UnicodeText, primary_key=True), + sa.Column("guid", sa.UnicodeText), + sa.Column("current", sa.Boolean), + sa.Column("gathered", sa.DateTime), + sa.Column("fetch_started", sa.DateTime), + sa.Column("content", sa.UnicodeText, nullable=True), + sa.Column("fetch_finished", sa.DateTime), + sa.Column("import_started", sa.DateTime), + sa.Column("import_finished", sa.DateTime), + sa.Column("state", sa.UnicodeText), + sa.Column("metadata_modified_date", sa.DateTime), + sa.Column("retry_times", sa.Integer), + sa.Column( + "harvest_job_id", + sa.UnicodeText, + sa.ForeignKey("harvest_job.id"), + ), + sa.Column( + "harvest_source_id", + sa.UnicodeText, + sa.ForeignKey("harvest_source.id"), + ), + sa.Column( + "package_id", + sa.UnicodeText, + sa.ForeignKey("package.id", deferrable=True), + nullable=True, + ), + sa.Column("report_status", sa.UnicodeText, nullable=True), + ) + + index_names = [index["name"] for index in inspector.get_indexes("harvest_object")] + if "harvest_job_id_idx" not in index_names: + op.create_index("harvest_job_id_idx", "harvest_object", ["harvest_job_id"]) + + if "harvest_source_id_idx" not in index_names: + op.create_index( + "harvest_source_id_idx", "harvest_object", ["harvest_source_id"] + ) + + if "package_id_idx" not in index_names: + op.create_index("package_id_idx", "harvest_object", ["package_id"]) + + if "guid_idx" not in index_names: + op.create_index("guid_idx", "harvest_object", ["guid"]) + + if not inspector.has_table("harvest_object_extra"): + op.create_table( + "harvest_object_extra", + sa.Column("id", sa.UnicodeText, primary_key=True), + sa.Column( + "harvest_object_id", + sa.UnicodeText, + sa.ForeignKey("harvest_object.id"), + ), + sa.Column("key", sa.UnicodeText), + sa.Column("value", sa.UnicodeText), + ) + + index_names = [ + index["name"] for index in inspector.get_indexes("harvest_object_extra") + ] + if "harvest_object_id_idx" not in index_names: + op.create_index( + "harvest_object_id_idx", "harvest_object_extra", ["harvest_object_id"] + ) + + if not inspector.has_table("harvest_gather_error"): + op.create_table( + "harvest_gather_error", + sa.Column("id", sa.UnicodeText, primary_key=True), + sa.Column( + "harvest_job_id", + sa.UnicodeText, + sa.ForeignKey("harvest_job.id"), + ), + sa.Column("message", sa.UnicodeText), + sa.Column("created", sa.DateTime), + ) + + if not inspector.has_table("harvest_object_error"): + op.create_table( + "harvest_object_error", + sa.Column("id", sa.UnicodeText, primary_key=True), + sa.Column( + "harvest_object_id", + sa.UnicodeText, + sa.ForeignKey("harvest_object.id"), + ), + sa.Column("message", sa.UnicodeText), + sa.Column("stage", sa.UnicodeText), + sa.Column("line", sa.Integer), + sa.Column("created", sa.DateTime), + ) + + index_names = [ + index["name"] for index in inspector.get_indexes("harvest_object_error") + ] + if "harvest_error_harvest_object_id_idx" not in index_names: + op.create_index( + "harvest_error_harvest_object_id_idx", + "harvest_object_error", + ["harvest_object_id"], + ) + + if not inspector.has_table("harvest_log"): + op.create_table( + "harvest_log", + sa.Column("id", sa.UnicodeText, primary_key=True), + sa.Column("content", sa.UnicodeText, nullable=False), + sa.Column( + "level", + sa.Enum( + "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL", name="log_level" + ), + ), + sa.Column("created", sa.DateTime), + ) + + +def downgrade(): + op.drop_table("harvest_log") + sa.Enum(name="log_level").drop(op.get_bind()) + op.drop_table("harvest_object_error") + op.drop_table("harvest_gather_error") + op.drop_table("harvest_object_extra") + op.drop_table("harvest_object") + op.drop_table("harvest_job") + op.drop_table("harvest_source") diff --git a/ckanext/harvest/migration/harvest/versions/75d650dfd519_add_cascade_to_harvest_tables.py b/ckanext/harvest/migration/harvest/versions/75d650dfd519_add_cascade_to_harvest_tables.py new file mode 100644 index 000000000..8a5553209 --- /dev/null +++ b/ckanext/harvest/migration/harvest/versions/75d650dfd519_add_cascade_to_harvest_tables.py @@ -0,0 +1,102 @@ +"""add cascade to harvest tables + +Revision ID: 75d650dfd519 +Revises: 3b4894672727 +Create Date: 2023-11-02 17:13:39.995339 + +""" +from alembic import op + + +# revision identifiers, used by Alembic. +revision = "75d650dfd519" +down_revision = "3b4894672727" +branch_labels = None +depends_on = None + + +def upgrade(): + _recreate_fk("CASCADE") + + +def downgrade(): + _recreate_fk(None) + + +def _recreate_fk(ondelete): + op.drop_constraint("harvest_job_source_id_fkey", "harvest_job") + op.create_foreign_key( + "harvest_job_source_id_fkey", + "harvest_job", + "harvest_source", + ["source_id"], + ["id"], + ondelete=ondelete, + ) + + op.drop_constraint("harvest_object_harvest_job_id_fkey", "harvest_object") + op.create_foreign_key( + "harvest_object_harvest_job_id_fkey", + "harvest_object", + "harvest_job", + ["harvest_job_id"], + ["id"], + ondelete=ondelete, + ) + + op.drop_constraint("harvest_object_harvest_source_id_fkey", "harvest_object") + op.create_foreign_key( + "harvest_object_harvest_source_id_fkey", + "harvest_object", + "harvest_source", + ["harvest_source_id"], + ["id"], + ondelete=ondelete, + ) + + op.drop_constraint("harvest_object_package_id_fkey", "harvest_object") + op.create_foreign_key( + "harvest_object_package_id_fkey", + "harvest_object", + "package", + ["package_id"], + ["id"], + ondelete=ondelete, + deferrable=True, + ) + + op.drop_constraint( + "harvest_object_extra_harvest_object_id_fkey", "harvest_object_extra" + ) + op.create_foreign_key( + "harvest_object_extra_harvest_object_id_fkey", + "harvest_object_extra", + "harvest_object", + ["harvest_object_id"], + ["id"], + ondelete=ondelete, + ) + + op.drop_constraint( + "harvest_gather_error_harvest_job_id_fkey", "harvest_gather_error" + ) + op.create_foreign_key( + "harvest_gather_error_harvest_job_id_fkey", + "harvest_gather_error", + "harvest_job", + ["harvest_job_id"], + ["id"], + ondelete=ondelete, + ) + + op.drop_constraint( + "harvest_object_error_harvest_object_id_fkey", "harvest_object_error" + ) + op.create_foreign_key( + "harvest_object_error_harvest_object_id_fkey", + "harvest_object_error", + "harvest_object", + ["harvest_object_id"], + ["id"], + ondelete=ondelete, + ) diff --git a/ckanext/harvest/model/__init__.py b/ckanext/harvest/model/__init__.py index 288817921..c0211ddf5 100644 --- a/ckanext/harvest/model/__init__.py +++ b/ckanext/harvest/model/__init__.py @@ -7,100 +7,140 @@ from sqlalchemy import ForeignKey from sqlalchemy import types from sqlalchemy import Index -from sqlalchemy.engine.reflection import Inspector from sqlalchemy.orm import backref, relation from sqlalchemy.exc import InvalidRequestError -from ckan import model from ckan.model.meta import metadata, mapper, Session from ckan.model.types import make_uuid from ckan.model.domain_object import DomainObject from ckan.model.package import Package -UPDATE_FREQUENCIES = ['MANUAL', 'MONTHLY', 'WEEKLY', 'BIWEEKLY', 'DAILY', 'ALWAYS'] +UPDATE_FREQUENCIES = ["MANUAL", "MONTHLY", "WEEKLY", "BIWEEKLY", "DAILY", "ALWAYS"] log = logging.getLogger(__name__) __all__ = [ - 'HarvestSource', 'harvest_source_table', - 'HarvestJob', 'harvest_job_table', - 'HarvestObject', 'harvest_object_table', - 'HarvestGatherError', 'harvest_gather_error_table', - 'HarvestObjectError', 'harvest_object_error_table', - 'HarvestLog', 'harvest_log_table' + "HarvestSource", + "harvest_source_table", + "HarvestJob", + "harvest_job_table", + "HarvestObject", + "harvest_object_table", + "HarvestGatherError", + "harvest_gather_error_table", + "HarvestObjectError", + "harvest_object_error_table", + "HarvestLog", + "harvest_log_table", ] -harvest_source_table = None -harvest_job_table = None -harvest_object_table = None -harvest_gather_error_table = None -harvest_object_error_table = None -harvest_object_extra_table = None -harvest_log_table = None - - -def setup(): - - if harvest_source_table is None: - define_harvester_tables() - log.debug('Harvest tables defined in memory') - - if not model.package_table.exists(): - log.debug('Harvest table creation deferred') - return - - if not harvest_source_table.exists(): - - # Create each table individually rather than - # using metadata.create_all() - harvest_source_table.create() - harvest_job_table.create() - harvest_object_table.create() - harvest_gather_error_table.create() - harvest_object_error_table.create() - harvest_object_extra_table.create() - harvest_log_table.create() - - log.debug('Harvest tables created') - else: - from ckan.model.meta import engine - log.debug('Harvest tables already exist') - # Check if existing tables need to be updated - inspector = Inspector.from_engine(engine) - - # Check if harvest_log table exist - needed for existing users - if 'harvest_log' not in inspector.get_table_names(): - harvest_log_table.create() - - # Check if harvest_object has a index - index_names = [index['name'] for index in inspector.get_indexes("harvest_object")] - if "harvest_job_id_idx" not in index_names: - log.debug('Creating index for harvest_object') - Index("harvest_job_id_idx", harvest_object_table.c.harvest_job_id).create() - - if "harvest_source_id_idx" not in index_names: - log.debug('Creating index for harvest source') - Index("harvest_source_id_idx", harvest_object_table.c.harvest_source_id).create() - - if "package_id_idx" not in index_names: - log.debug('Creating index for package') - Index("package_id_idx", harvest_object_table.c.package_id).create() - - if "guid_idx" not in index_names: - log.debug('Creating index for guid') - Index("guid_idx", harvest_object_table.c.guid).create() - - index_names = [index['name'] for index in inspector.get_indexes("harvest_object_extra")] - if "harvest_object_id_idx" not in index_names: - log.debug('Creating index for harvest_object_extra') - Index("harvest_object_id_idx", harvest_object_extra_table.c.harvest_object_id).create() - - index_names = [index['name'] for index in inspector.get_indexes("harvest_object_error")] - if "harvest_error_harvest_object_id_idx" not in index_names: - log.debug('Creating index for harvest_object_error') - Index("harvest_error_harvest_object_id_idx", harvest_object_error_table.c.harvest_object_id).create() +harvest_source_table = Table( + "harvest_source", + metadata, + Column("id", types.UnicodeText, primary_key=True, default=make_uuid), + Column("url", types.UnicodeText, nullable=False), + Column("title", types.UnicodeText, default=""), + Column("description", types.UnicodeText, default=""), + Column("config", types.UnicodeText, default=""), + Column("created", types.DateTime, default=datetime.datetime.utcnow), + Column("type", types.UnicodeText, nullable=False), + Column("active", types.Boolean, default=True), + Column("user_id", types.UnicodeText, default=""), + Column("publisher_id", types.UnicodeText, default=""), + Column("frequency", types.UnicodeText, default="MANUAL"), + Column("next_run", types.DateTime), +) +harvest_job_table = Table( + "harvest_job", + metadata, + Column("id", types.UnicodeText, primary_key=True, default=make_uuid), + Column("created", types.DateTime, default=datetime.datetime.utcnow), + Column("gather_started", types.DateTime), + Column("gather_finished", types.DateTime), + Column("finished", types.DateTime), + Column("source_id", types.UnicodeText, ForeignKey("harvest_source.id")), + # status: New, Running, Finished + Column("status", types.UnicodeText, default="New", nullable=False), +) +harvest_object_table = Table( + "harvest_object", + metadata, + Column("id", types.UnicodeText, primary_key=True, default=make_uuid), + # The guid is the 'identity' of the dataset, according to the source. + # So if you reharvest it, then the harvester knows which dataset to + # update because of this identity. The identity needs to be unique + # within this CKAN. + Column("guid", types.UnicodeText, default=""), + # When you harvest a dataset multiple times, only the latest + # successfully imported harvest_object should be flagged 'current'. + # The import_stage usually reads and writes it. + Column("current", types.Boolean, default=False), + Column("gathered", types.DateTime, default=datetime.datetime.utcnow), + Column("fetch_started", types.DateTime), + Column("content", types.UnicodeText, nullable=True), + Column("fetch_finished", types.DateTime), + Column("import_started", types.DateTime), + Column("import_finished", types.DateTime), + # state: WAITING, FETCH, IMPORT, COMPLETE, ERROR + Column("state", types.UnicodeText, default="WAITING"), + Column("metadata_modified_date", types.DateTime), + Column("retry_times", types.Integer, default=0), + Column("harvest_job_id", types.UnicodeText, ForeignKey("harvest_job.id")), + Column("harvest_source_id", types.UnicodeText, ForeignKey("harvest_source.id")), + Column( + "package_id", + types.UnicodeText, + ForeignKey("package.id", deferrable=True), + nullable=True, + ), + # report_status: 'added', 'updated', 'not modified', 'deleted', 'errored' + Column("report_status", types.UnicodeText, nullable=True), + Index("harvest_job_id_idx", "harvest_job_id"), + Index("harvest_source_id_idx", "harvest_source_id"), + Index("package_id_idx", "package_id"), + Index("guid_idx", "guid"), +) +harvest_gather_error_table = Table( + "harvest_gather_error", + metadata, + Column("id", types.UnicodeText, primary_key=True, default=make_uuid), + Column("harvest_job_id", types.UnicodeText, ForeignKey("harvest_job.id")), + Column("message", types.UnicodeText), + Column("created", types.DateTime, default=datetime.datetime.utcnow), +) +harvest_object_error_table = Table( + "harvest_object_error", + metadata, + Column("id", types.UnicodeText, primary_key=True, default=make_uuid), + Column("harvest_object_id", types.UnicodeText, ForeignKey("harvest_object.id")), + Column("message", types.UnicodeText), + Column("stage", types.UnicodeText), + Column("line", types.Integer), + Column("created", types.DateTime, default=datetime.datetime.utcnow), + Index("harvest_error_harvest_object_id_idx", "harvest_object_id"), +) +harvest_object_extra_table = Table( + "harvest_object_extra", + metadata, + Column("id", types.UnicodeText, primary_key=True, default=make_uuid), + Column("harvest_object_id", types.UnicodeText, ForeignKey("harvest_object.id")), + Column("key", types.UnicodeText), + Column("value", types.UnicodeText), + Index("harvest_object_id_idx", "harvest_object_id"), +) +harvest_log_table = Table( + "harvest_log", + metadata, + Column("id", types.UnicodeText, primary_key=True, default=make_uuid), + Column("content", types.UnicodeText, nullable=False), + Column( + "level", + types.Enum("DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL", name="log_level"), + ), + Column("created", types.DateTime, default=datetime.datetime.utcnow), +) class HarvestError(Exception): @@ -108,13 +148,13 @@ class HarvestError(Exception): class HarvestDomainObject(DomainObject): - '''Convenience methods for searching objects - ''' - key_attr = 'id' + """Convenience methods for searching objects""" + + key_attr = "id" @classmethod def get(cls, key, default=None, attr=None): - '''Finds a single entity in the register.''' + """Finds a single entity in the register.""" if attr is None: attr = cls.key_attr kwds = {attr: key} @@ -131,20 +171,25 @@ def filter(cls, **kwds): class HarvestSource(HarvestDomainObject): - '''A Harvest Source is essentially a URL plus some other metadata. - It must have a type (e.g. CSW) and can have a status of "active" - or "inactive". The harvesting processes are not fired on inactive - sources. - ''' + """A Harvest Source is essentially a URL plus some other metadata. + It must have a type (e.g. CSW) and can have a status of "active" + or "inactive". The harvesting processes are not fired on inactive + sources. + """ + def __repr__(self): - return '' % \ - (self.id, self.title, self.url, self.active) + return "" % ( + self.id, + self.title, + self.url, + self.active, + ) def __str__(self): - return self.__repr__().encode('ascii', 'ignore') + return self.__repr__().encode("ascii", "ignore") def get_jobs(self, status=None): - """ get the running jobs for this source """ + """get the running jobs for this source""" query = Session.query(HarvestJob).filter(HarvestJob.source_id == self.id) @@ -155,42 +200,46 @@ def get_jobs(self, status=None): class HarvestJob(HarvestDomainObject): - '''A Harvesting Job is performed in two phases. In first place, the - **gather** stage collects all the Ids and URLs that need to be fetched - from the harvest source. Errors occurring in this phase - (``HarvestGatherError``) are stored in the ``harvest_gather_error`` - table. During the next phase, the **fetch** stage retrieves the - ``HarvestedObjects`` and, if necessary, the **import** stage stores - them on the database. Errors occurring in this second stage - (``HarvestObjectError``) are stored in the ``harvest_object_error`` - table. - ''' + """A Harvesting Job is performed in two phases. In first place, the + **gather** stage collects all the Ids and URLs that need to be fetched + from the harvest source. Errors occurring in this phase + (``HarvestGatherError``) are stored in the ``harvest_gather_error`` + table. During the next phase, the **fetch** stage retrieves the + ``HarvestedObjects`` and, if necessary, the **import** stage stores + them on the database. Errors occurring in this second stage + (``HarvestObjectError``) are stored in the ``harvest_object_error`` + table. + """ def get_last_finished_object(self): - ''' Determine the last finished object in this job - Helpful to know if a job is running or not and - to avoid timeouts when the source is running - ''' - - query = Session.query(HarvestObject)\ - .filter(HarvestObject.harvest_job_id == self.id)\ - .filter(HarvestObject.state == "COMPLETE")\ - .filter(HarvestObject.import_finished.isnot(None))\ - .order_by(HarvestObject.import_finished.desc())\ + """Determine the last finished object in this job + Helpful to know if a job is running or not and + to avoid timeouts when the source is running + """ + + query = ( + Session.query(HarvestObject) + .filter(HarvestObject.harvest_job_id == self.id) + .filter(HarvestObject.state == "COMPLETE") + .filter(HarvestObject.import_finished.isnot(None)) + .order_by(HarvestObject.import_finished.desc()) .first() + ) return query def get_last_gathered_object(self): - ''' Determine the last gathered object in this job - Helpful to know if a job is running or not and - to avoid timeouts when the source is running - ''' - - query = Session.query(HarvestObject)\ - .filter(HarvestObject.harvest_job_id == self.id)\ - .order_by(HarvestObject.gathered.desc())\ + """Determine the last gathered object in this job + Helpful to know if a job is running or not and + to avoid timeouts when the source is running + """ + + query = ( + Session.query(HarvestObject) + .filter(HarvestObject.harvest_job_id == self.id) + .order_by(HarvestObject.gathered.desc()) .first() + ) return query @@ -209,34 +258,37 @@ def get_last_action_time(self): return self.created def get_gather_errors(self): - query = Session.query(HarvestGatherError)\ - .filter(HarvestGatherError.harvest_job_id == self.id)\ - .order_by(HarvestGatherError.created.desc()) + query = ( + Session.query(HarvestGatherError) + .filter(HarvestGatherError.harvest_job_id == self.id) + .order_by(HarvestGatherError.created.desc()) + ) return query.all() class HarvestObject(HarvestDomainObject): - '''A Harvest Object is created every time an element is fetched from a - harvest source. Its contents can be processed and imported to ckan - packages, RDF graphs, etc. + """A Harvest Object is created every time an element is fetched from a + harvest source. Its contents can be processed and imported to ckan + packages, RDF graphs, etc. - ''' + """ class HarvestObjectExtra(HarvestDomainObject): - '''Extra key value data for Harvest objects''' + """Extra key value data for Harvest objects""" class HarvestGatherError(HarvestDomainObject): - '''Gather errors are raised during the **gather** stage of a harvesting - job. - ''' + """Gather errors are raised during the **gather** stage of a harvesting + job. + """ + @classmethod def create(cls, message, job): - ''' + """ Helper function to create an error object and save it. - ''' + """ err = cls(message=message, job=job) try: err.save() @@ -249,16 +301,16 @@ def create(cls, message, job): class HarvestObjectError(HarvestDomainObject): - '''Object errors are raised during the **fetch** or **import** stage of a - harvesting job, and are referenced to a specific harvest object. - ''' + """Object errors are raised during the **fetch** or **import** stage of a + harvesting job, and are referenced to a specific harvest object. + """ + @classmethod - def create(cls, message, object, stage=u'Fetch', line=None): - ''' + def create(cls, message, object, stage="Fetch", line=None): + """ Helper function to create an error object and save it. - ''' - err = cls(message=message, object=object, - stage=stage, line=line) + """ + err = cls(message=message, object=object, stage=stage, line=line) try: err.save() except InvalidRequestError: @@ -273,245 +325,123 @@ def create(cls, message, object, stage=u'Fetch', line=None): pass err.save() finally: - log_message = '{0}, line {1}'.format(message, line) \ - if line else message + log_message = "{0}, line {1}".format(message, line) if line else message log.debug(log_message) class HarvestLog(HarvestDomainObject): - '''HarvestLog objects are created each time something is logged - using python's standard logging module - ''' + """HarvestLog objects are created each time something is logged + using python's standard logging module + """ + pass def harvest_object_before_insert_listener(mapper, connection, target): - ''' - For compatibility with old harvesters, check if the source id has - been set, and set it automatically from the job if not. - ''' + """ + For compatibility with old harvesters, check if the source id has + been set, and set it automatically from the job if not. + """ if not target.harvest_source_id or not target.source: if not target.job: - raise Exception('You must define a Harvest Job for each Harvest Object') + raise Exception("You must define a Harvest Job for each Harvest Object") target.source = target.job.source target.harvest_source_id = target.job.source.id -def define_harvester_tables(): - - global harvest_source_table - global harvest_job_table - global harvest_object_table - global harvest_object_extra_table - global harvest_gather_error_table - global harvest_object_error_table - global harvest_log_table - - harvest_source_table = Table( - 'harvest_source', - metadata, - Column('id', types.UnicodeText, primary_key=True, default=make_uuid), - Column('url', types.UnicodeText, nullable=False), - Column('title', types.UnicodeText, default=u''), - Column('description', types.UnicodeText, default=u''), - Column('config', types.UnicodeText, default=u''), - Column('created', types.DateTime, default=datetime.datetime.utcnow), - Column('type', types.UnicodeText, nullable=False), - Column('active', types.Boolean, default=True), - Column('user_id', types.UnicodeText, default=u''), - Column('publisher_id', types.UnicodeText, default=u''), - Column('frequency', types.UnicodeText, default=u'MANUAL'), - Column('next_run', types.DateTime), - ) - # Was harvesting_job - harvest_job_table = Table( - 'harvest_job', - metadata, - Column('id', types.UnicodeText, primary_key=True, default=make_uuid), - Column('created', types.DateTime, default=datetime.datetime.utcnow), - Column('gather_started', types.DateTime), - Column('gather_finished', types.DateTime), - Column('finished', types.DateTime), - Column('source_id', types.UnicodeText, ForeignKey('harvest_source.id')), - # status: New, Running, Finished - Column('status', types.UnicodeText, default=u'New', nullable=False), - ) - # A harvest_object contains a representation of one dataset during a - # particular harvest - harvest_object_table = Table( - 'harvest_object', - metadata, - Column('id', types.UnicodeText, primary_key=True, default=make_uuid), - # The guid is the 'identity' of the dataset, according to the source. - # So if you reharvest it, then the harvester knows which dataset to - # update because of this identity. The identity needs to be unique - # within this CKAN. - Column('guid', types.UnicodeText, default=u''), - # When you harvest a dataset multiple times, only the latest - # successfully imported harvest_object should be flagged 'current'. - # The import_stage usually reads and writes it. - Column('current', types.Boolean, default=False), - Column('gathered', types.DateTime, default=datetime.datetime.utcnow), - Column('fetch_started', types.DateTime), - Column('content', types.UnicodeText, nullable=True), - Column('fetch_finished', types.DateTime), - Column('import_started', types.DateTime), - Column('import_finished', types.DateTime), - # state: WAITING, FETCH, IMPORT, COMPLETE, ERROR - Column('state', types.UnicodeText, default=u'WAITING'), - Column('metadata_modified_date', types.DateTime), - Column('retry_times', types.Integer, default=0), - Column('harvest_job_id', types.UnicodeText, ForeignKey('harvest_job.id')), - Column('harvest_source_id', types.UnicodeText, ForeignKey('harvest_source.id')), - Column('package_id', types.UnicodeText, ForeignKey('package.id', deferrable=True), - nullable=True), - # report_status: 'added', 'updated', 'not modified', 'deleted', 'errored' - Column('report_status', types.UnicodeText, nullable=True), - Index('harvest_job_id_idx', 'harvest_job_id'), - Index('harvest_source_id_idx', 'harvest_source_id'), - Index('package_id_idx', 'package_id'), - Index('guid_idx', 'guid'), - ) - - # New table - harvest_object_extra_table = Table( - 'harvest_object_extra', - metadata, - Column('id', types.UnicodeText, primary_key=True, default=make_uuid), - Column('harvest_object_id', types.UnicodeText, ForeignKey('harvest_object.id')), - Column('key', types.UnicodeText), - Column('value', types.UnicodeText), - Index('harvest_object_id_idx', 'harvest_object_id'), - ) - - # New table - harvest_gather_error_table = Table( - 'harvest_gather_error', - metadata, - Column('id', types.UnicodeText, primary_key=True, default=make_uuid), - Column('harvest_job_id', types.UnicodeText, ForeignKey('harvest_job.id')), - Column('message', types.UnicodeText), - Column('created', types.DateTime, default=datetime.datetime.utcnow), - ) - # New table - harvest_object_error_table = Table( - 'harvest_object_error', - metadata, - Column('id', types.UnicodeText, primary_key=True, default=make_uuid), - Column('harvest_object_id', types.UnicodeText, ForeignKey('harvest_object.id')), - Column('message', types.UnicodeText), - Column('stage', types.UnicodeText), - Column('line', types.Integer), - Column('created', types.DateTime, default=datetime.datetime.utcnow), - Index('harvest_error_harvest_object_id_idx', 'harvest_object_id'), - ) - # Harvest Log table - harvest_log_table = Table( - 'harvest_log', - metadata, - Column('id', types.UnicodeText, primary_key=True, default=make_uuid), - Column('content', types.UnicodeText, nullable=False), - Column('level', types.Enum('DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL', name='log_level')), - Column('created', types.DateTime, default=datetime.datetime.utcnow), - ) - - mapper( - HarvestSource, - harvest_source_table, - properties={ - 'jobs': relation( - HarvestJob, - lazy=True, - backref=u'source', - order_by=harvest_job_table.c.created, - ), - }, - ) - - mapper( - HarvestJob, - harvest_job_table, - ) - - mapper( - HarvestObject, - harvest_object_table, - properties={ - 'package': relation( - Package, - lazy=True, - backref='harvest_objects', - ), - 'job': relation( - HarvestJob, - lazy=True, - backref=u'objects', - ), - 'source': relation( - HarvestSource, - lazy=True, - backref=u'objects', - ), - - }, - ) - - mapper( - HarvestGatherError, - harvest_gather_error_table, - properties={ - 'job': relation( - HarvestJob, - backref='gather_errors' - ), - }, - ) - - mapper( - HarvestObjectError, - harvest_object_error_table, - properties={ - 'object': relation( - HarvestObject, - backref=backref('errors', cascade='all,delete-orphan') - ), - }, - ) - - mapper( - HarvestObjectExtra, - harvest_object_extra_table, - properties={ - 'object': relation( - HarvestObject, - backref=backref('extras', cascade='all,delete-orphan') - ), - }, - ) - - mapper( - HarvestLog, - harvest_log_table, - ) - - event.listen(HarvestObject, 'before_insert', harvest_object_before_insert_listener) - - class PackageIdHarvestSourceIdMismatch(Exception): """ The package created for the harvest source must match the id of the harvest source """ + pass def clean_harvest_log(condition): - Session.query(HarvestLog).filter(HarvestLog.created <= condition) \ - .delete(synchronize_session=False) + Session.query(HarvestLog).filter(HarvestLog.created <= condition).delete( + synchronize_session=False + ) try: Session.commit() except InvalidRequestError: Session.rollback() - log.error('An error occurred while trying to clean-up the harvest log table') - - log.info('Harvest log table clean-up finished successfully') + log.error("An error occurred while trying to clean-up the harvest log table") + + log.info("Harvest log table clean-up finished successfully") + + +mapper( + HarvestSource, + harvest_source_table, + properties={ + "jobs": relation( + HarvestJob, + lazy=True, + backref="source", + order_by=harvest_job_table.c.created, + ), + }, +) + +mapper( + HarvestJob, + harvest_job_table, +) + +mapper( + HarvestObject, + harvest_object_table, + properties={ + "package": relation( + Package, + lazy=True, + backref="harvest_objects", + ), + "job": relation( + HarvestJob, + lazy=True, + backref="objects", + ), + "source": relation( + HarvestSource, + lazy=True, + backref="objects", + ), + }, +) + +mapper( + HarvestGatherError, + harvest_gather_error_table, + properties={ + "job": relation(HarvestJob, backref="gather_errors"), + }, +) + +mapper( + HarvestObjectError, + harvest_object_error_table, + properties={ + "object": relation( + HarvestObject, backref=backref("errors", cascade="all,delete-orphan") + ), + }, +) + +mapper( + HarvestObjectExtra, + harvest_object_extra_table, + properties={ + "object": relation( + HarvestObject, backref=backref("extras", cascade="all,delete-orphan") + ), + }, +) + +mapper( + HarvestLog, + harvest_log_table, +) + +event.listen(HarvestObject, "before_insert", harvest_object_before_insert_listener) diff --git a/ckanext/harvest/plugin.py b/ckanext/harvest/plugin.py index 02ab6ae4b..72d145264 100644 --- a/ckanext/harvest/plugin.py +++ b/ckanext/harvest/plugin.py @@ -15,7 +15,6 @@ import ckanext.harvest from ckanext.harvest import cli, views -from ckanext.harvest.model import setup as model_setup from ckanext.harvest.model import HarvestSource, HarvestJob, HarvestObject from ckanext.harvest.log import DBLogHandler @@ -275,9 +274,6 @@ def configure(self, config): self.startup = True - # Setup harvest model - model_setup() - # Configure database logger _configure_db_logger(config) diff --git a/ckanext/harvest/utils.py b/ckanext/harvest/utils.py index a58c7f598..99a1e124c 100644 --- a/ckanext/harvest/utils.py +++ b/ckanext/harvest/utils.py @@ -101,12 +101,6 @@ def _there_are(what, sequence, condition=""): ) -def initdb(): - from ckanext.harvest.model import setup as db_setup - - db_setup() - - def create_harvest_source( name, url, diff --git a/setup.py b/setup.py index f1efc2088..8a9fe2adc 100644 --- a/setup.py +++ b/setup.py @@ -39,8 +39,6 @@ test_harvester2=ckanext.harvest.tests.test_queue2:MockHarvester test_action_harvester=ckanext.harvest.tests.test_action:MockHarvesterForActionTests - [paste.paster_command] - harvester = ckanext.harvest.commands.harvester:Harvester [babel.extractors] ckan = ckan.lib.extract:extract_ckan """, From 227f2404cce7a908549ce51bd57c65ca27aa5d25 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Thu, 2 Nov 2023 18:01:21 +0200 Subject: [PATCH 43/74] test: fix flake8 test --- ckanext/harvest/tests/harvesters/test_ckanharvester.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanext/harvest/tests/harvesters/test_ckanharvester.py b/ckanext/harvest/tests/harvesters/test_ckanharvester.py index e018f81d0..b90f2ad8d 100644 --- a/ckanext/harvest/tests/harvesters/test_ckanharvester.py +++ b/ckanext/harvest/tests/harvesters/test_ckanharvester.py @@ -49,7 +49,7 @@ def test_gather_normal(self): obj_ids = harvester.gather_stage(job) assert job.gather_errors == [] - assert type(obj_ids) == list + assert isinstance(obj_ids, list) assert len(obj_ids) == len(mock_ckan.DATASETS) harvest_object = harvest_model.HarvestObject.get(obj_ids[0]) assert harvest_object.guid == mock_ckan.DATASETS[0]['id'] From 8e143e4bc75c30ea48a903f44252f4f229c44126 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Thu, 2 Nov 2023 18:08:48 +0200 Subject: [PATCH 44/74] test: update fixtures --- .github/workflows/test.yml | 1 - ckanext/harvest/tests/conftest.py | 6 ++++++ ckanext/harvest/tests/fixtures.py | 6 ------ ckanext/harvest/tests/harvesters/test_base.py | 4 ++-- ckanext/harvest/tests/harvesters/test_ckanharvester.py | 2 +- ckanext/harvest/tests/test_action.py | 4 ++-- ckanext/harvest/tests/test_blueprint.py | 2 +- ckanext/harvest/tests/test_queue.py | 2 +- ckanext/harvest/tests/test_queue2.py | 2 +- ckanext/harvest/tests/test_timeouts.py | 2 +- 10 files changed, 15 insertions(+), 16 deletions(-) create mode 100644 ckanext/harvest/tests/conftest.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e94c0777a..c1c01871b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -57,6 +57,5 @@ jobs: - name: Setup extension (CKAN >= 2.9) run: | ckan -c test.ini db init - ckan -c test.ini harvester initdb - name: Run tests run: pytest --ckan-ini=test.ini --cov=ckanext.harvest --disable-warnings ckanext/harvest/tests diff --git a/ckanext/harvest/tests/conftest.py b/ckanext/harvest/tests/conftest.py new file mode 100644 index 000000000..26c0443f9 --- /dev/null +++ b/ckanext/harvest/tests/conftest.py @@ -0,0 +1,6 @@ +import pytest + +@pytest.fixture +def clean_db(reset_db, migrate_db_for): + reset_db() + migrate_db_for("harvest") diff --git a/ckanext/harvest/tests/fixtures.py b/ckanext/harvest/tests/fixtures.py index fb9059532..bb283eb4f 100644 --- a/ckanext/harvest/tests/fixtures.py +++ b/ckanext/harvest/tests/fixtures.py @@ -1,14 +1,8 @@ import pytest -import ckanext.harvest.model as harvest_model from ckanext.harvest import queue -@pytest.fixture -def harvest_setup(): - harvest_model.setup() - - @pytest.fixture def clean_queues(): queue.purge_queues() diff --git a/ckanext/harvest/tests/harvesters/test_base.py b/ckanext/harvest/tests/harvesters/test_base.py index cffecbd40..f260e4445 100644 --- a/ckanext/harvest/tests/harvesters/test_base.py +++ b/ckanext/harvest/tests/harvesters/test_base.py @@ -13,7 +13,7 @@ _ensure_name_is_unique = HarvesterBase._ensure_name_is_unique -@pytest.mark.usefixtures('clean_db', 'clean_index', 'harvest_setup') +@pytest.mark.usefixtures('clean_db', 'clean_index') class TestGenNewName(object): def test_basic(self): @@ -51,7 +51,7 @@ def test_config_override(self): assert HarvesterBase._gen_new_name('Trees', append_type='number-sequence') == 'trees1' -@pytest.mark.usefixtures('clean_db', 'clean_index', 'harvest_setup') +@pytest.mark.usefixtures('clean_db', 'clean_index') class TestEnsureNameIsUnique(object): def test_no_existing_datasets(self): diff --git a/ckanext/harvest/tests/harvesters/test_ckanharvester.py b/ckanext/harvest/tests/harvesters/test_ckanharvester.py index b90f2ad8d..77957be36 100644 --- a/ckanext/harvest/tests/harvesters/test_ckanharvester.py +++ b/ckanext/harvest/tests/harvesters/test_ckanharvester.py @@ -38,7 +38,7 @@ def was_last_job_considered_error_free(): return bool(HarvesterBase.last_error_free_job(job)) -@pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index', 'harvest_setup') +@pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index') class TestCkanHarvester(object): def test_gather_normal(self): diff --git a/ckanext/harvest/tests/test_action.py b/ckanext/harvest/tests/test_action.py index e22bb5cd5..fdee17eb1 100644 --- a/ckanext/harvest/tests/test_action.py +++ b/ckanext/harvest/tests/test_action.py @@ -57,7 +57,7 @@ def import_stage(self, harvest_object): } -@pytest.mark.usefixtures('with_plugins', 'clean_db', 'harvest_setup', 'clean_queues') +@pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_queues') @pytest.mark.ckan_config('ckan.plugins', 'harvest test_action_harvester') class HarvestSourceActionBase(): @@ -225,7 +225,7 @@ def test_patch(self): assert source.type == source_dict['source_type'] -@pytest.mark.usefixtures('with_plugins', 'clean_db', 'harvest_setup', 'clean_queues') +@pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_queues') @pytest.mark.ckan_config('ckan.plugins', 'harvest test_action_harvester') class TestActions(): def test_harvest_source_clear(self): diff --git a/ckanext/harvest/tests/test_blueprint.py b/ckanext/harvest/tests/test_blueprint.py index a9141da8e..d82c04d93 100644 --- a/ckanext/harvest/tests/test_blueprint.py +++ b/ckanext/harvest/tests/test_blueprint.py @@ -5,7 +5,7 @@ from ckanext.harvest.tests import factories as harvest_factories -@pytest.mark.usefixtures('clean_db', 'clean_index', 'harvest_setup') +@pytest.mark.usefixtures('clean_db', 'clean_index') class TestBlueprint(): def test_index_page_is_rendered(self, app): diff --git a/ckanext/harvest/tests/test_queue.py b/ckanext/harvest/tests/test_queue.py index 04332fd53..e711dd8dd 100644 --- a/ckanext/harvest/tests/test_queue.py +++ b/ckanext/harvest/tests/test_queue.py @@ -88,7 +88,7 @@ def import_stage(self, harvest_object): return True -@pytest.mark.usefixtures('with_plugins', 'clean_db', 'harvest_setup', 'clean_queues') +@pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_queues') @pytest.mark.ckan_config('ckan.plugins', 'harvest test_harvester') class TestHarvestQueue(object): diff --git a/ckanext/harvest/tests/test_queue2.py b/ckanext/harvest/tests/test_queue2.py index bf83f7d79..9cc34bf5a 100644 --- a/ckanext/harvest/tests/test_queue2.py +++ b/ckanext/harvest/tests/test_queue2.py @@ -84,7 +84,7 @@ def import_stage(self, harvest_object): return True -@pytest.mark.usefixtures('with_plugins', 'clean_db', 'harvest_setup', 'clean_queues') +@pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_queues') @pytest.mark.ckan_config('ckan.plugins', 'harvest test_harvester2') class TestEndStates(object): diff --git a/ckanext/harvest/tests/test_timeouts.py b/ckanext/harvest/tests/test_timeouts.py index b39537494..17b965a4e 100644 --- a/ckanext/harvest/tests/test_timeouts.py +++ b/ckanext/harvest/tests/test_timeouts.py @@ -9,7 +9,7 @@ from ckanext.harvest.logic import HarvestJobExists -@pytest.mark.usefixtures('with_plugins', 'clean_db', 'harvest_setup', 'clean_queues') +@pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_queues') @pytest.mark.ckan_config('ckan.plugins', 'harvest test_action_harvester') class TestModelFunctions: dataset_counter = 0 From ad4d083a64c4e712ad1fc496761d0d31874716dd Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Thu, 2 Nov 2023 18:20:30 +0200 Subject: [PATCH 45/74] test: fix flake8 test --- ckanext/harvest/tests/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ckanext/harvest/tests/conftest.py b/ckanext/harvest/tests/conftest.py index 26c0443f9..38abeef52 100644 --- a/ckanext/harvest/tests/conftest.py +++ b/ckanext/harvest/tests/conftest.py @@ -1,5 +1,6 @@ import pytest + @pytest.fixture def clean_db(reset_db, migrate_db_for): reset_db() From 66b609e8818f1b55941383a040adecc472b2d60d Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Thu, 2 Nov 2023 18:32:57 +0200 Subject: [PATCH 46/74] test: enable plugins when using clean_db --- ckanext/harvest/logic/auth/create.py | 2 +- ckanext/harvest/tests/harvesters/test_base.py | 4 ++-- ckanext/harvest/tests/test_blueprint.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ckanext/harvest/logic/auth/create.py b/ckanext/harvest/logic/auth/create.py index 2ee2ae0b3..e2bdaf469 100644 --- a/ckanext/harvest/logic/auth/create.py +++ b/ckanext/harvest/logic/auth/create.py @@ -35,7 +35,7 @@ def harvest_job_create(context, data_dict): context['package'] = pkg try: - pt.check_access('package_update', context, data_dict) + pt.check_access('package_update', context, {"id": source_id}) return {'success': True} except pt.NotAuthorized: return {'success': False, diff --git a/ckanext/harvest/tests/harvesters/test_base.py b/ckanext/harvest/tests/harvesters/test_base.py index f260e4445..b7d5a810f 100644 --- a/ckanext/harvest/tests/harvesters/test_base.py +++ b/ckanext/harvest/tests/harvesters/test_base.py @@ -13,7 +13,7 @@ _ensure_name_is_unique = HarvesterBase._ensure_name_is_unique -@pytest.mark.usefixtures('clean_db', 'clean_index') +@pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index') class TestGenNewName(object): def test_basic(self): @@ -51,7 +51,7 @@ def test_config_override(self): assert HarvesterBase._gen_new_name('Trees', append_type='number-sequence') == 'trees1' -@pytest.mark.usefixtures('clean_db', 'clean_index') +@pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index') class TestEnsureNameIsUnique(object): def test_no_existing_datasets(self): diff --git a/ckanext/harvest/tests/test_blueprint.py b/ckanext/harvest/tests/test_blueprint.py index d82c04d93..fde7b2292 100644 --- a/ckanext/harvest/tests/test_blueprint.py +++ b/ckanext/harvest/tests/test_blueprint.py @@ -5,7 +5,7 @@ from ckanext.harvest.tests import factories as harvest_factories -@pytest.mark.usefixtures('clean_db', 'clean_index') +@pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index') class TestBlueprint(): def test_index_page_is_rendered(self, app): From 0020fb03ca52c01d7f7e141d7f5bf48987578362 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Thu, 2 Nov 2023 18:46:22 +0200 Subject: [PATCH 47/74] test: initialize db before tests --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c1c01871b..43f062871 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -57,5 +57,6 @@ jobs: - name: Setup extension (CKAN >= 2.9) run: | ckan -c test.ini db init + ckan -c test.ini db pending-migrations --apply - name: Run tests run: pytest --ckan-ini=test.ini --cov=ckanext.harvest --disable-warnings ckanext/harvest/tests From b278b8acd9f1a3cd89a4eba2d58a75c623d0f089 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Thu, 2 Nov 2023 18:59:34 +0200 Subject: [PATCH 48/74] test: 2.9 compatible migrations --- .../3b4894672727_create_harvest_tables.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ckanext/harvest/migration/harvest/versions/3b4894672727_create_harvest_tables.py b/ckanext/harvest/migration/harvest/versions/3b4894672727_create_harvest_tables.py index 689e48f37..5b1faeee0 100644 --- a/ckanext/harvest/migration/harvest/versions/3b4894672727_create_harvest_tables.py +++ b/ckanext/harvest/migration/harvest/versions/3b4894672727_create_harvest_tables.py @@ -19,8 +19,8 @@ def upgrade(): engine = op.get_bind() inspector = sa.inspect(engine) - - if not inspector.has_table("harvest_source"): + tables = inspector.get_table_names() + if "harvest_source" not in tables: op.create_table( "harvest_source", sa.Column("id", sa.UnicodeText, primary_key=True), @@ -37,7 +37,7 @@ def upgrade(): sa.Column("next_run", sa.DateTime), ) - if not inspector.has_table("harvest_job"): + if "harvest_job" not in tables: op.create_table( "harvest_job", sa.Column("id", sa.UnicodeText, primary_key=True), @@ -53,7 +53,7 @@ def upgrade(): sa.Column("status", sa.UnicodeText, nullable=False), ) - if not inspector.has_table("harvest_object"): + if "harvest_object" not in tables: op.create_table( "harvest_object", sa.Column("id", sa.UnicodeText, primary_key=True), @@ -102,7 +102,7 @@ def upgrade(): if "guid_idx" not in index_names: op.create_index("guid_idx", "harvest_object", ["guid"]) - if not inspector.has_table("harvest_object_extra"): + if "harvest_object_extra" not in tables: op.create_table( "harvest_object_extra", sa.Column("id", sa.UnicodeText, primary_key=True), @@ -123,7 +123,7 @@ def upgrade(): "harvest_object_id_idx", "harvest_object_extra", ["harvest_object_id"] ) - if not inspector.has_table("harvest_gather_error"): + if "harvest_gather_error" not in tables: op.create_table( "harvest_gather_error", sa.Column("id", sa.UnicodeText, primary_key=True), @@ -136,7 +136,7 @@ def upgrade(): sa.Column("created", sa.DateTime), ) - if not inspector.has_table("harvest_object_error"): + if "harvest_object_error" not in tables: op.create_table( "harvest_object_error", sa.Column("id", sa.UnicodeText, primary_key=True), @@ -161,7 +161,7 @@ def upgrade(): ["harvest_object_id"], ) - if not inspector.has_table("harvest_log"): + if "harvest_log" not in tables: op.create_table( "harvest_log", sa.Column("id", sa.UnicodeText, primary_key=True), From 0034bbd7b7a4b504ab0c62333b1821a4fb1c025b Mon Sep 17 00:00:00 2001 From: Rae Knowler Date: Wed, 14 Feb 2024 16:05:04 +0100 Subject: [PATCH 49/74] fix: Convert boolean options to bools If a logic function is called via the API, all values in the data_dict will be strings, so we need to convert e.g. 'False' to a boolean or it will be treated as True. --- ckanext/harvest/logic/action/get.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ckanext/harvest/logic/action/get.py b/ckanext/harvest/logic/action/get.py index 498152e88..4fff81eb0 100644 --- a/ckanext/harvest/logic/action/get.py +++ b/ckanext/harvest/logic/action/get.py @@ -287,7 +287,7 @@ def harvest_object_list(context, data_dict): session = context['session'] - only_current = data_dict.get('only_current', True) + only_current = p.toolkit.asbool(data_dict.get('only_current', True)) source_id = data_dict.get('source_id', False) query = session.query(HarvestObject) @@ -371,8 +371,8 @@ def _get_sources_for_user(context, data_dict, organization_id=None, limit=None): session = context['session'] user = context.get('user', '') - only_active = data_dict.get('only_active', False) - only_to_run = data_dict.get('only_to_run', False) + only_active = p.toolkit.asbool(data_dict.get('only_active', False)) + only_to_run = p.toolkit.asbool(data_dict.get('only_to_run', False)) query = session.query(HarvestSource) \ .order_by(HarvestSource.created.desc()) From 1a236d1e376fe8d387942a17856a6bcee2ba091e Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 20 Jun 2024 16:35:08 +0200 Subject: [PATCH 50/74] Remove ancient node code in tests --- .github/workflows/test.yml | 2 +- ckanext/harvest/tests/test_timeouts.py | 33 +++++++++++++------------- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 43f062871..9e2a035e0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,7 +19,7 @@ jobs: needs: lint strategy: matrix: - ckan-version: ["2.10", 2.9] + ckan-version: ["master", "2.10", 2.9] fail-fast: false name: CKAN ${{ matrix.ckan-version }} diff --git a/ckanext/harvest/tests/test_timeouts.py b/ckanext/harvest/tests/test_timeouts.py index 17b965a4e..689eeceaf 100644 --- a/ckanext/harvest/tests/test_timeouts.py +++ b/ckanext/harvest/tests/test_timeouts.py @@ -1,5 +1,4 @@ from datetime import datetime, timedelta -from nose.tools import assert_equal, assert_in import pytest from ckan.tests import factories as ckan_factories from ckan import model @@ -25,14 +24,14 @@ def test_timeout_jobs(self): ob2 = self.add_object(job=job, source=source, state='COMPLETE', minutes_ago=5) self.add_object(job=job, source=source, state='COMPLETE', minutes_ago=15) - assert_equal(job.get_last_finished_object(), ob2) - assert_equal(job.get_last_action_time(), ob2.import_finished) + assert job.get_last_finished_object() == ob2 + assert job.get_last_action_time() == ob2.import_finished gather_errors = self.run(timeout=3, source=source, job=job) - assert_equal(len(gather_errors), 1) - assert_equal(job.status, 'Finished') + assert len(gather_errors) == 1 + assert job.status == 'Finished' gather_error = gather_errors[0] - assert_in('timeout', gather_error.message) + assert 'timeout' in gather_error.message def test_no_timeout_jobs(self): """ Test a job that don't raise timeout """ @@ -42,12 +41,12 @@ def test_no_timeout_jobs(self): ob2 = self.add_object(job=job, source=source, state='COMPLETE', minutes_ago=5) self.add_object(job=job, source=source, state='COMPLETE', minutes_ago=15) - assert_equal(job.get_last_finished_object(), ob2) - assert_equal(job.get_last_action_time(), ob2.import_finished) + assert job.get_last_finished_object() == ob2 + assert job.get_last_action_time() == ob2.import_finished gather_errors = self.run(timeout=7, source=source, job=job) - assert_equal(len(gather_errors), 0) - assert_equal(job.status, 'Finished') + assert len(gather_errors) == 0 + assert job.status == 'Finished' def test_no_objects_job(self): """ Test a job that don't raise timeout """ @@ -56,8 +55,8 @@ def test_no_objects_job(self): job.gather_finished = datetime.utcnow() job.save() - assert_equal(job.get_last_finished_object(), None) - assert_equal(job.get_last_action_time(), job.gather_finished) + assert job.get_last_finished_object() == None + assert job.get_last_action_time() == job.gather_finished def test_no_gathered_job(self): """ Test a job that don't raise timeout """ @@ -66,8 +65,8 @@ def test_no_gathered_job(self): job.gather_finished = None job.save() - assert_equal(job.get_last_finished_object(), None) - assert_equal(job.get_last_action_time(), job.created) + assert job.get_last_finished_object() == None + assert job.get_last_action_time() == job.created def test_gather_get_last_action_time(self): """ Test get_last_action_time at gather stage """ @@ -77,8 +76,8 @@ def test_gather_get_last_action_time(self): self.add_object(job=job, source=source, state='WAITING') ob3 = self.add_object(job=job, source=source, state='WAITING') - assert_equal(job.get_last_gathered_object(), ob3) - assert_equal(job.get_last_action_time(), ob3.gathered) + assert job.get_last_gathered_object() == ob3 + assert job.get_last_action_time() == ob3.gathered def run(self, timeout, source, job): """ Run the havester_job_run and return the errors """ @@ -125,7 +124,7 @@ def get_source(self): job.save() jobs = source.get_jobs(status='Running') - assert_in(job, jobs) + assert job in jobs return source, job From b581391adf7163317af2cb9d0542c22d1864c9fe Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 20 Jun 2024 16:36:25 +0200 Subject: [PATCH 51/74] lint --- ckanext/harvest/tests/test_timeouts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckanext/harvest/tests/test_timeouts.py b/ckanext/harvest/tests/test_timeouts.py index 689eeceaf..6f7a5fa23 100644 --- a/ckanext/harvest/tests/test_timeouts.py +++ b/ckanext/harvest/tests/test_timeouts.py @@ -55,7 +55,7 @@ def test_no_objects_job(self): job.gather_finished = datetime.utcnow() job.save() - assert job.get_last_finished_object() == None + assert job.get_last_finished_object() is None assert job.get_last_action_time() == job.gather_finished def test_no_gathered_job(self): @@ -65,7 +65,7 @@ def test_no_gathered_job(self): job.gather_finished = None job.save() - assert job.get_last_finished_object() == None + assert job.get_last_finished_object() is None assert job.get_last_action_time() == job.created def test_gather_get_last_action_time(self): From e11c0390b698f6d554575eed18a4f7568a5d0780 Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 20 Jun 2024 16:39:42 +0200 Subject: [PATCH 52/74] Bump action versions --- .github/workflows/test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9e2a035e0..86ba05112 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,10 +4,10 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: - python-version: '3.8' + python-version: '3.9' - name: Install requirements run: pip install flake8 pycodestyle - name: Check syntax @@ -46,7 +46,7 @@ jobs: CKAN_REDIS_URL: redis://redis:6379/1 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Install requirements run: | pip install -r requirements.txt From 26144327dafacad1180b416404850d164077b5a1 Mon Sep 17 00:00:00 2001 From: amercader Date: Tue, 25 Jun 2024 16:21:41 +0200 Subject: [PATCH 53/74] Use proper uuids as ids in tests --- ckanext/harvest/tests/harvesters/mock_ckan.py | 10 +++++----- .../harvest/tests/harvesters/test_ckanharvester.py | 12 ++++++------ ckanext/harvest/tests/test_action.py | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/ckanext/harvest/tests/harvesters/mock_ckan.py b/ckanext/harvest/tests/harvesters/mock_ckan.py index af4e3b515..25e758d01 100644 --- a/ckanext/harvest/tests/harvesters/mock_ckan.py +++ b/ckanext/harvest/tests/harvesters/mock_ckan.py @@ -217,9 +217,9 @@ def convert_dataset_to_restful_form(dataset): {'id': 'dataset1-id', 'name': 'dataset1', 'title': 'Test Dataset1', - 'owner_org': 'org1-id', + 'owner_org': '0f8380d6-241a-47de-aa52-8bd91c763d97', 'tags': [{'name': 'test-tag'}], - 'groups': [{'id': 'group1-id', 'name': 'group1'}], + 'groups': [{'id': '10037fa4-e683-4a67-892a-efba815e24ad', 'name': 'group1'}], 'extras': []}, { "id": "1c65c66a-fdec-4138-9c64-0f9bf087bcbb", @@ -316,7 +316,7 @@ def convert_dataset_to_restful_form(dataset): "recent": 0 }, "taxonomy_url": "", - "groups": [{"id": "remote-group-id", "name": "remote-group"}], + "groups": [{"id": "af9eacfc-5f77-4b36-9cb0-ce0222bf14d6", "name": "remote-group"}], "creator_user_id": None, "national_statistic": "no", "relationships_as_subject": [], @@ -504,14 +504,14 @@ def convert_dataset_to_restful_form(dataset): ] ORGS = [ - {'id': 'org1-id', + {'id': '0f8380d6-241a-47de-aa52-8bd91c763d97', 'name': 'org1'}, {'id': 'aa1e068a-23da-4563-b9c2-2cad272b663e', 'name': 'cabinet-office'} ] GROUPS = [ - {'id': 'group1-id', + {'id': '10037fa4-e683-4a67-892a-efba815e24ad', 'name': 'group1'}, {'id': '9853c3e1-eebb-4e8c-9ae7-1668a01bf2ca', 'name': 'finances'} diff --git a/ckanext/harvest/tests/harvesters/test_ckanharvester.py b/ckanext/harvest/tests/harvesters/test_ckanharvester.py index 77957be36..e71abeb3d 100644 --- a/ckanext/harvest/tests/harvesters/test_ckanharvester.py +++ b/ckanext/harvest/tests/harvesters/test_ckanharvester.py @@ -189,7 +189,7 @@ def test_harvest_info_in_package_show(self): def test_remote_groups_only_local(self): # Create an existing group - Group(id='group1-id', name='group1') + Group(id='10037fa4-e683-4a67-892a-efba815e24ad', name='group1') config = {'remote_groups': 'only_local'} results_by_guid = run_harvest( @@ -258,11 +258,11 @@ def test_default_tags_invalid(self): assert 'default_tags must be a list of dictionaries' in str(harvest_context.value) def test_default_groups(self): - Group(id='group1-id', name='group1') - Group(id='group2-id', name='group2') - Group(id='group3-id', name='group3') + Group(name='group1') + Group(name='group2') + Group(name='group3') - config = {'default_groups': ['group2-id', 'group3'], + config = {'default_groups': ['group2', 'group3'], 'remote_groups': 'only_local'} tmp_c = toolkit.c try: @@ -284,7 +284,7 @@ def test_default_groups(self): assert group_names, set(('group1', 'group2' == 'group3')) def test_default_groups_invalid(self): - Group(id='group2-id', name='group2') + Group(name='group2') # should be list of strings config = {'default_groups': [{'name': 'group2'}]} diff --git a/ckanext/harvest/tests/test_action.py b/ckanext/harvest/tests/test_action.py index fdee17eb1..e39fd97ee 100644 --- a/ckanext/harvest/tests/test_action.py +++ b/ckanext/harvest/tests/test_action.py @@ -58,7 +58,7 @@ def import_stage(self, harvest_object): @pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_queues') -@pytest.mark.ckan_config('ckan.plugins', 'harvest test_action_harvester') +@pytest.mark.ckan_config('ckan.plugins', 'harvest test_action_harvester test_harvester') class HarvestSourceActionBase(): def _get_source_dict(self): From 0c5da80a8269f59d96291a6b518fc733cbf0fc59 Mon Sep 17 00:00:00 2001 From: amercader Date: Tue, 25 Jun 2024 16:24:25 +0200 Subject: [PATCH 54/74] Refactor harvest models Use the declarative way, and inherit from `BaseModel` in the toolkit, so tables get properly created and dropped in tests --- ckanext/harvest/harvesters/base.py | 10 +- ckanext/harvest/model/__init__.py | 348 ++++++++++++----------------- 2 files changed, 145 insertions(+), 213 deletions(-) diff --git a/ckanext/harvest/harvesters/base.py b/ckanext/harvest/harvesters/base.py index 746d7271e..cfbf2426a 100644 --- a/ckanext/harvest/harvesters/base.py +++ b/ckanext/harvest/harvesters/base.py @@ -315,15 +315,11 @@ def _create_or_update_package(self, package_dict, harvest_object, return 'unchanged' # Flag the other objects linking to this package as not current anymore - from ckanext.harvest.model import harvest_object_table - conn = Session.connection() - u = update(harvest_object_table)\ - .where(harvest_object_table.c.package_id == bindparam('b_package_id')) \ - .values(current=False) - conn.execute(u, b_package_id=new_package['id']) + Session.query(HarvestObject).filter( + HarvestObject.package_id == new_package["id"]).update( + {"current": False}) # Flag this as the current harvest object - harvest_object.package_id = new_package['id'] harvest_object.current = True harvest_object.save() diff --git a/ckanext/harvest/model/__init__.py b/ckanext/harvest/model/__init__.py index c0211ddf5..03ac906aa 100644 --- a/ckanext/harvest/model/__init__.py +++ b/ckanext/harvest/model/__init__.py @@ -2,146 +2,25 @@ import datetime from sqlalchemy import event -from sqlalchemy import Table from sqlalchemy import Column from sqlalchemy import ForeignKey from sqlalchemy import types from sqlalchemy import Index -from sqlalchemy.orm import backref, relation +from sqlalchemy.orm import backref, relationship from sqlalchemy.exc import InvalidRequestError -from ckan.model.meta import metadata, mapper, Session +from ckan.model.meta import Session from ckan.model.types import make_uuid from ckan.model.domain_object import DomainObject from ckan.model.package import Package +from ckantoolkit import BaseModel + UPDATE_FREQUENCIES = ["MANUAL", "MONTHLY", "WEEKLY", "BIWEEKLY", "DAILY", "ALWAYS"] log = logging.getLogger(__name__) -__all__ = [ - "HarvestSource", - "harvest_source_table", - "HarvestJob", - "harvest_job_table", - "HarvestObject", - "harvest_object_table", - "HarvestGatherError", - "harvest_gather_error_table", - "HarvestObjectError", - "harvest_object_error_table", - "HarvestLog", - "harvest_log_table", -] - - -harvest_source_table = Table( - "harvest_source", - metadata, - Column("id", types.UnicodeText, primary_key=True, default=make_uuid), - Column("url", types.UnicodeText, nullable=False), - Column("title", types.UnicodeText, default=""), - Column("description", types.UnicodeText, default=""), - Column("config", types.UnicodeText, default=""), - Column("created", types.DateTime, default=datetime.datetime.utcnow), - Column("type", types.UnicodeText, nullable=False), - Column("active", types.Boolean, default=True), - Column("user_id", types.UnicodeText, default=""), - Column("publisher_id", types.UnicodeText, default=""), - Column("frequency", types.UnicodeText, default="MANUAL"), - Column("next_run", types.DateTime), -) -harvest_job_table = Table( - "harvest_job", - metadata, - Column("id", types.UnicodeText, primary_key=True, default=make_uuid), - Column("created", types.DateTime, default=datetime.datetime.utcnow), - Column("gather_started", types.DateTime), - Column("gather_finished", types.DateTime), - Column("finished", types.DateTime), - Column("source_id", types.UnicodeText, ForeignKey("harvest_source.id")), - # status: New, Running, Finished - Column("status", types.UnicodeText, default="New", nullable=False), -) -harvest_object_table = Table( - "harvest_object", - metadata, - Column("id", types.UnicodeText, primary_key=True, default=make_uuid), - # The guid is the 'identity' of the dataset, according to the source. - # So if you reharvest it, then the harvester knows which dataset to - # update because of this identity. The identity needs to be unique - # within this CKAN. - Column("guid", types.UnicodeText, default=""), - # When you harvest a dataset multiple times, only the latest - # successfully imported harvest_object should be flagged 'current'. - # The import_stage usually reads and writes it. - Column("current", types.Boolean, default=False), - Column("gathered", types.DateTime, default=datetime.datetime.utcnow), - Column("fetch_started", types.DateTime), - Column("content", types.UnicodeText, nullable=True), - Column("fetch_finished", types.DateTime), - Column("import_started", types.DateTime), - Column("import_finished", types.DateTime), - # state: WAITING, FETCH, IMPORT, COMPLETE, ERROR - Column("state", types.UnicodeText, default="WAITING"), - Column("metadata_modified_date", types.DateTime), - Column("retry_times", types.Integer, default=0), - Column("harvest_job_id", types.UnicodeText, ForeignKey("harvest_job.id")), - Column("harvest_source_id", types.UnicodeText, ForeignKey("harvest_source.id")), - Column( - "package_id", - types.UnicodeText, - ForeignKey("package.id", deferrable=True), - nullable=True, - ), - # report_status: 'added', 'updated', 'not modified', 'deleted', 'errored' - Column("report_status", types.UnicodeText, nullable=True), - Index("harvest_job_id_idx", "harvest_job_id"), - Index("harvest_source_id_idx", "harvest_source_id"), - Index("package_id_idx", "package_id"), - Index("guid_idx", "guid"), -) -harvest_gather_error_table = Table( - "harvest_gather_error", - metadata, - Column("id", types.UnicodeText, primary_key=True, default=make_uuid), - Column("harvest_job_id", types.UnicodeText, ForeignKey("harvest_job.id")), - Column("message", types.UnicodeText), - Column("created", types.DateTime, default=datetime.datetime.utcnow), -) -harvest_object_error_table = Table( - "harvest_object_error", - metadata, - Column("id", types.UnicodeText, primary_key=True, default=make_uuid), - Column("harvest_object_id", types.UnicodeText, ForeignKey("harvest_object.id")), - Column("message", types.UnicodeText), - Column("stage", types.UnicodeText), - Column("line", types.Integer), - Column("created", types.DateTime, default=datetime.datetime.utcnow), - Index("harvest_error_harvest_object_id_idx", "harvest_object_id"), -) -harvest_object_extra_table = Table( - "harvest_object_extra", - metadata, - Column("id", types.UnicodeText, primary_key=True, default=make_uuid), - Column("harvest_object_id", types.UnicodeText, ForeignKey("harvest_object.id")), - Column("key", types.UnicodeText), - Column("value", types.UnicodeText), - Index("harvest_object_id_idx", "harvest_object_id"), -) -harvest_log_table = Table( - "harvest_log", - metadata, - Column("id", types.UnicodeText, primary_key=True, default=make_uuid), - Column("content", types.UnicodeText, nullable=False), - Column( - "level", - types.Enum("DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL", name="log_level"), - ), - Column("created", types.DateTime, default=datetime.datetime.utcnow), -) - class HarvestError(Exception): pass @@ -170,13 +49,34 @@ def filter(cls, **kwds): return query.filter_by(**kwds) -class HarvestSource(HarvestDomainObject): +class HarvestSource(BaseModel, HarvestDomainObject): """A Harvest Source is essentially a URL plus some other metadata. It must have a type (e.g. CSW) and can have a status of "active" or "inactive". The harvesting processes are not fired on inactive sources. """ + __tablename__ = "harvest_source" + + id = Column(types.UnicodeText, primary_key=True, default=make_uuid) + url = Column(types.UnicodeText, nullable=False) + title = Column(types.UnicodeText, default="") + description = Column(types.UnicodeText, default="") + config = Column(types.UnicodeText, default="") + created = Column(types.DateTime, default=datetime.datetime.utcnow) + type = Column(types.UnicodeText, nullable=False) + active = Column(types.Boolean, default=True) + user_id = Column(types.UnicodeText, default="") + publisher_id = Column(types.UnicodeText, default="") + frequency = Column(types.UnicodeText, default="MANUAL") + next_run = Column(types.DateTime) + jobs = relationship( + "HarvestJob", + lazy="select", + back_populates="source", + order_by=lambda: HarvestJob.created, + ) + def __repr__(self): return "" % ( self.id, @@ -199,7 +99,7 @@ def get_jobs(self, status=None): return query.all() -class HarvestJob(HarvestDomainObject): +class HarvestJob(BaseModel, HarvestDomainObject): """A Harvesting Job is performed in two phases. In first place, the **gather** stage collects all the Ids and URLs that need to be fetched from the harvest source. Errors occurring in this phase @@ -211,6 +111,22 @@ class HarvestJob(HarvestDomainObject): table. """ + __tablename__ = "harvest_job" + + id = Column(types.UnicodeText, primary_key=True, default=make_uuid) + created = Column(types.DateTime, default=datetime.datetime.utcnow) + gather_started = Column(types.DateTime) + gather_finished = Column(types.DateTime) + finished = Column(types.DateTime) + source_id = Column(types.UnicodeText, ForeignKey("harvest_source.id")) + # status: New, Running, Finished + status = Column(types.UnicodeText, default="New", nullable=False) + source = relationship( + "HarvestSource", + lazy="select", + back_populates="jobs", + ) + def get_last_finished_object(self): """Determine the last finished object in this job Helpful to know if a job is running or not and @@ -267,23 +183,94 @@ def get_gather_errors(self): return query.all() -class HarvestObject(HarvestDomainObject): +class HarvestObject(BaseModel, HarvestDomainObject): """A Harvest Object is created every time an element is fetched from a harvest source. Its contents can be processed and imported to ckan packages, RDF graphs, etc. """ + __tablename__ = "harvest_object" + + id = Column(types.UnicodeText, primary_key=True, default=make_uuid) + # The guid is the 'identity' of the dataset, according to the source. + # So if you reharvest it, then the harvester knows which dataset to + # update because of this identity. The identity needs to be unique + # within this CKAN. + guid = Column(types.UnicodeText, default="") + # When you harvest a dataset multiple times, only the latest + # successfully imported harvest_object should be flagged 'current'. + # The import_stage usually reads and writes it. + current = Column(types.Boolean, default=False) + gathered = Column(types.DateTime, default=datetime.datetime.utcnow) + fetch_started = Column(types.DateTime) + content = Column(types.UnicodeText, nullable=True) + fetch_finished = Column(types.DateTime) + import_started = Column(types.DateTime) + import_finished = Column(types.DateTime) + # state: WAITING, FETCH, IMPORT, COMPLETE, ERROR + state = Column(types.UnicodeText, default="WAITING") + metadata_modified_date = Column(types.DateTime) + retry_times = Column(types.Integer, default=0) + harvest_job_id = Column(types.UnicodeText, ForeignKey("harvest_job.id")) + harvest_source_id = Column(types.UnicodeText, ForeignKey("harvest_source.id")) + package_id = Column( + types.UnicodeText, + ForeignKey("package.id", deferrable=True), + nullable=True, + ) + # report_status: 'added', 'updated', 'not modified', 'deleted', 'errored' + report_status = Column(types.UnicodeText, nullable=True) + harvest_job_id_idx = Index("harvest_job_id") + harvest_source_id_idx = Index("harvest_source_id") + package_id_idx = Index("package_id") + guid_idx = Index("guid") + package = relationship( + Package, + lazy="select", + backref="harvest_objects", + ) + job = relationship( + HarvestJob, + lazy="select", + backref="objects", + ) + source = relationship( + HarvestSource, + lazy="select", + backref="objects", + ) + -class HarvestObjectExtra(HarvestDomainObject): +class HarvestObjectExtra(BaseModel, HarvestDomainObject): """Extra key value data for Harvest objects""" + __tablename__ = "harvest_object_extra" + + id = Column(types.UnicodeText, primary_key=True, default=make_uuid) + harvest_object_id = Column(types.UnicodeText, ForeignKey("harvest_object.id")) + key = Column(types.UnicodeText) + value = Column(types.UnicodeText) + harvest_object_id_idx = Index("harvest_object_id") + object = relationship( + HarvestObject, backref=backref("extras", cascade="all,delete-orphan") + ) + -class HarvestGatherError(HarvestDomainObject): +class HarvestGatherError(BaseModel, HarvestDomainObject): """Gather errors are raised during the **gather** stage of a harvesting job. """ + __tablename__ = "harvest_gather_error" + + id = Column(types.UnicodeText, primary_key=True, default=make_uuid) + harvest_job_id = Column(types.UnicodeText, ForeignKey("harvest_job.id")) + message = Column(types.UnicodeText) + created = Column(types.DateTime, default=datetime.datetime.utcnow) + + job = relationship(HarvestJob, backref="gather_errors") + @classmethod def create(cls, message, job): """ @@ -300,11 +287,25 @@ def create(cls, message, job): log.info(message) -class HarvestObjectError(HarvestDomainObject): +class HarvestObjectError(BaseModel, HarvestDomainObject): """Object errors are raised during the **fetch** or **import** stage of a harvesting job, and are referenced to a specific harvest object. """ + __tablename__ = "harvest_object_error" + + id = Column(types.UnicodeText, primary_key=True, default=make_uuid) + harvest_object_id = Column(types.UnicodeText, ForeignKey("harvest_object.id")) + message = Column(types.UnicodeText) + stage = Column(types.UnicodeText) + line = Column(types.Integer) + created = Column(types.DateTime, default=datetime.datetime.utcnow) + harvest_error_harvest_object_id_idx = Index("harvest_object_id") + + object = relationship( + HarvestObject, backref=backref("errors", cascade="all,delete-orphan") + ) + @classmethod def create(cls, message, object, stage="Fetch", line=None): """ @@ -329,12 +330,20 @@ def create(cls, message, object, stage="Fetch", line=None): log.debug(log_message) -class HarvestLog(HarvestDomainObject): +class HarvestLog(BaseModel, HarvestDomainObject): """HarvestLog objects are created each time something is logged using python's standard logging module """ - pass + __tablename__ = "harvest_log" + + id = Column(types.UnicodeText, primary_key=True, default=make_uuid) + content = Column(types.UnicodeText, nullable=False) + level = Column( + types.Enum("DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL", name="log_level"), + ) + created = Column(types.DateTime, default=datetime.datetime.utcnow) + def harvest_object_before_insert_listener(mapper, connection, target): @@ -371,77 +380,4 @@ def clean_harvest_log(condition): log.info("Harvest log table clean-up finished successfully") -mapper( - HarvestSource, - harvest_source_table, - properties={ - "jobs": relation( - HarvestJob, - lazy=True, - backref="source", - order_by=harvest_job_table.c.created, - ), - }, -) - -mapper( - HarvestJob, - harvest_job_table, -) - -mapper( - HarvestObject, - harvest_object_table, - properties={ - "package": relation( - Package, - lazy=True, - backref="harvest_objects", - ), - "job": relation( - HarvestJob, - lazy=True, - backref="objects", - ), - "source": relation( - HarvestSource, - lazy=True, - backref="objects", - ), - }, -) - -mapper( - HarvestGatherError, - harvest_gather_error_table, - properties={ - "job": relation(HarvestJob, backref="gather_errors"), - }, -) - -mapper( - HarvestObjectError, - harvest_object_error_table, - properties={ - "object": relation( - HarvestObject, backref=backref("errors", cascade="all,delete-orphan") - ), - }, -) - -mapper( - HarvestObjectExtra, - harvest_object_extra_table, - properties={ - "object": relation( - HarvestObject, backref=backref("extras", cascade="all,delete-orphan") - ), - }, -) - -mapper( - HarvestLog, - harvest_log_table, -) - event.listen(HarvestObject, "before_insert", harvest_object_before_insert_listener) From 6842d8b105beb13a173f653d3415bbcb0aa2fbe5 Mon Sep 17 00:00:00 2001 From: amercader Date: Tue, 25 Jun 2024 16:28:09 +0200 Subject: [PATCH 55/74] lint --- ckanext/harvest/harvesters/base.py | 1 - ckanext/harvest/model/__init__.py | 1 - 2 files changed, 2 deletions(-) diff --git a/ckanext/harvest/harvesters/base.py b/ckanext/harvest/harvesters/base.py index cfbf2426a..3a51bbad7 100644 --- a/ckanext/harvest/harvesters/base.py +++ b/ckanext/harvest/harvesters/base.py @@ -5,7 +5,6 @@ import uuid from sqlalchemy import exists, and_ -from sqlalchemy.sql import update, bindparam from sqlalchemy.orm import contains_eager from ckantoolkit import config diff --git a/ckanext/harvest/model/__init__.py b/ckanext/harvest/model/__init__.py index 03ac906aa..482ed0334 100644 --- a/ckanext/harvest/model/__init__.py +++ b/ckanext/harvest/model/__init__.py @@ -345,7 +345,6 @@ class HarvestLog(BaseModel, HarvestDomainObject): created = Column(types.DateTime, default=datetime.datetime.utcnow) - def harvest_object_before_insert_listener(mapper, connection, target): """ For compatibility with old harvesters, check if the source id has From 11eed9f8f8077e56b278d8ebd120bee55793c6bd Mon Sep 17 00:00:00 2001 From: amercader Date: Tue, 25 Jun 2024 16:38:10 +0200 Subject: [PATCH 56/74] 2.9 version of BaseModel --- ckanext/harvest/model/__init__.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ckanext/harvest/model/__init__.py b/ckanext/harvest/model/__init__.py index 482ed0334..c347cb48c 100644 --- a/ckanext/harvest/model/__init__.py +++ b/ckanext/harvest/model/__init__.py @@ -14,7 +14,14 @@ from ckan.model.domain_object import DomainObject from ckan.model.package import Package -from ckantoolkit import BaseModel +try: + from ckan.plugins.toolkit import BaseModel +except ImportError: + # CKAN <= 2.9 + from ckan.model.meta import metadata + from sqlalchemy.ext.declarative import declarative_base + + BaseModel = declarative_base(metadata=metadata) UPDATE_FREQUENCIES = ["MANUAL", "MONTHLY", "WEEKLY", "BIWEEKLY", "DAILY", "ALWAYS"] From ac14d155c5005f8301a9711bd17006879cf44e99 Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 27 Jun 2024 14:01:08 +0200 Subject: [PATCH 57/74] Use ckan, 2.11 image --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 86ba05112..d1cc3ae35 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,13 +19,13 @@ jobs: needs: lint strategy: matrix: - ckan-version: ["master", "2.10", 2.9] + ckan-version: ["2.11", "2.10", 2.9] fail-fast: false name: CKAN ${{ matrix.ckan-version }} runs-on: ubuntu-latest container: - image: openknowledge/ckan-dev:${{ matrix.ckan-version }} + image: ckan/ckan-dev:${{ matrix.ckan-version }} services: solr: image: ckan/ckan-solr:${{ matrix.ckan-version }} From 46dd79f59843d147078cbf4d7d944f770db6d09e Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 27 Jun 2024 15:19:37 +0200 Subject: [PATCH 58/74] Fix solr image name --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d1cc3ae35..c9ca89cca 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,7 +28,7 @@ jobs: image: ckan/ckan-dev:${{ matrix.ckan-version }} services: solr: - image: ckan/ckan-solr:${{ matrix.ckan-version }} + image: ckan/ckan-solr:${{ matrix.ckan-version }}-solr9 postgres: image: ckan/ckan-postgres-dev:${{ matrix.ckan-version }} env: From 042ded810636667c5a839bbb739fa4bcd690b58c Mon Sep 17 00:00:00 2001 From: amercader Date: Wed, 3 Jul 2024 12:25:27 +0200 Subject: [PATCH 59/74] Test master --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c9ca89cca..b291a82bd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,7 +19,7 @@ jobs: needs: lint strategy: matrix: - ckan-version: ["2.11", "2.10", 2.9] + ckan-version: ["master", "2.11", "2.10", 2.9] fail-fast: false name: CKAN ${{ matrix.ckan-version }} From 790cdcd66cc8e6148f177d85bcb451d43bd449ac Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 11 Jul 2024 14:02:12 +0200 Subject: [PATCH 60/74] Don't test master --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b291a82bd..c9ca89cca 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,7 +19,7 @@ jobs: needs: lint strategy: matrix: - ckan-version: ["master", "2.11", "2.10", 2.9] + ckan-version: ["2.11", "2.10", 2.9] fail-fast: false name: CKAN ${{ matrix.ckan-version }} From 29102714f45f81a93bf1d4759a51df5b63dbec73 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Fri, 6 Sep 2024 23:42:57 +0300 Subject: [PATCH 61/74] feat: SQLAlchemy v2 support --- ckanext/harvest/harvesters/base.py | 10 +++-- ckanext/harvest/logic/action/update.py | 58 +++++++++++++++----------- ckanext/harvest/model/__init__.py | 1 - 3 files changed, 40 insertions(+), 29 deletions(-) diff --git a/ckanext/harvest/harvesters/base.py b/ckanext/harvest/harvesters/base.py index 3a51bbad7..e31f8f7e2 100644 --- a/ckanext/harvest/harvesters/base.py +++ b/ckanext/harvest/harvesters/base.py @@ -4,7 +4,7 @@ import re import uuid -from sqlalchemy import exists, and_ +import sqlalchemy as sa from sqlalchemy.orm import contains_eager from ckantoolkit import config @@ -344,7 +344,9 @@ def _create_or_update_package(self, package_dict, harvest_object, # plugin) harvest_object.add() - model.Session.execute('SET CONSTRAINTS harvest_object_package_id_fkey DEFERRED') + model.Session.execute( + sa.text('SET CONSTRAINTS harvest_object_package_id_fkey DEFERRED') + ) model.Session.flush() new_package = p.toolkit.get_action( @@ -400,10 +402,10 @@ def last_error_free_job(cls, harvest_job): .filter(HarvestJob.status == 'Finished') .filter(HarvestJob.id != harvest_job.id) .filter( - ~exists().where( + ~sa.exists().where( HarvestGatherError.harvest_job_id == HarvestJob.id)) .outerjoin(HarvestObject, - and_(HarvestObject.harvest_job_id == HarvestJob.id, + sa.and_(HarvestObject.harvest_job_id == HarvestJob.id, HarvestObject.current == False, # noqa: E712 HarvestObject.report_status != 'not modified')) .options(contains_eager(HarvestJob.objects)) diff --git a/ckanext/harvest/logic/action/update.py b/ckanext/harvest/logic/action/update.py index 7dce1f575..4006a3bd4 100644 --- a/ckanext/harvest/logic/action/update.py +++ b/ckanext/harvest/logic/action/update.py @@ -7,8 +7,8 @@ import logging import datetime +import sqlalchemy as sa from ckantoolkit import config -from sqlalchemy import and_, or_ from urllib.parse import urljoin from ckan.lib.search.index import PackageSearchIndex @@ -125,7 +125,7 @@ def harvest_source_clear(context, data_dict): select package_id from harvest_object where harvest_source_id = '{harvest_source_id}'));'''.format( harvest_source_id=harvest_source_id) - result = model.Session.execute(sql) + result = model.Session.execute(sa.text(sql)) ids = [] for row in result: ids.append(row[0]) @@ -137,14 +137,18 @@ def harvest_source_clear(context, data_dict): where harvest_source_id = '{harvest_source_id}');'''.format( harvest_source_id=harvest_source_id) + if toolkit.check_ckan_version(max_version='2.10.99'): + sql += ''' + delete from resource_revision where package_id in ( + select id from package where state = 'to_delete'); + ''' + # CKAN-2.3 or above: delete resource views, resource revisions & resources if toolkit.check_ckan_version(min_version='2.3'): sql += ''' delete from resource_view where resource_id in ( select id from resource where package_id in ( select id from package where state = 'to_delete')); - delete from resource_revision where package_id in ( - select id from package where state = 'to_delete'); delete from resource where package_id in ( select id from package where state = 'to_delete'); ''' @@ -172,6 +176,24 @@ def harvest_source_clear(context, data_dict): and context = 'Package'; ''' + if toolkit.check_ckan_version(max_version='2.10.99'): + sql += ''' + delete from package_tag_revision where package_id in ( + select id from package where state = 'to_delete'); + delete from member_revision where table_id in ( + select id from package where state = 'to_delete'); + delete from package_extra_revision where package_id in ( + select id from package where state = 'to_delete'); + delete from package_revision where id in ( + select id from package where state = 'to_delete'); + delete from package_relationship_revision where subject_package_id in ( + select id from package where state = 'to_delete'); + delete from package_relationship_revision where object_package_id in ( + select id from package where state = 'to_delete'); + delete from package_extra where package_id in ( + select id from package where state = 'to_delete'); + ''' + sql += ''' delete from harvest_object_error where harvest_object_id in ( select id from harvest_object @@ -183,22 +205,8 @@ def harvest_source_clear(context, data_dict): delete from harvest_gather_error where harvest_job_id in ( select id from harvest_job where source_id = '{harvest_source_id}'); delete from harvest_job where source_id = '{harvest_source_id}'; - delete from package_tag_revision where package_id in ( - select id from package where state = 'to_delete'); - delete from member_revision where table_id in ( - select id from package where state = 'to_delete'); - delete from package_extra_revision where package_id in ( - select id from package where state = 'to_delete'); - delete from package_revision where id in ( - select id from package where state = 'to_delete'); delete from package_tag where package_id in ( select id from package where state = 'to_delete'); - delete from package_extra where package_id in ( - select id from package where state = 'to_delete'); - delete from package_relationship_revision where subject_package_id in ( - select id from package where state = 'to_delete'); - delete from package_relationship_revision where object_package_id in ( - select id from package where state = 'to_delete'); delete from package_relationship where subject_package_id in ( select id from package where state = 'to_delete'); delete from package_relationship where object_package_id in ( @@ -226,7 +234,7 @@ def harvest_source_clear(context, data_dict): sql += ''' commit; ''' - model.Session.execute(sql) + model.Session.execute(sa.text(sql)) # Refresh the index for this source to update the status object get_action('harvest_source_reindex')(context, {'id': harvest_source_id}) @@ -408,7 +416,7 @@ def harvest_source_job_history_clear(context, data_dict): COMMIT; '''.format(harvest_source_id=harvest_source_id) - model.Session.execute(sql) + model.Session.execute(sa.text(sql)) # Refresh the index for this source to update the status object get_action('harvest_source_reindex')(context, {'id': harvest_source_id}) @@ -529,8 +537,8 @@ def harvest_objects_import(context, data_dict): .join(Package) .filter(HarvestObject.current == True) # noqa: E712 .filter(Package.state == u'active') - .filter(or_(Package.id == package_id_or_name, - Package.name == package_id_or_name))) + .filter(sa.or_(Package.id == package_id_or_name, + Package.name == package_id_or_name))) join_datasets = False else: last_objects_ids = \ @@ -671,7 +679,7 @@ def harvest_jobs_run(context, data_dict): num_objects_in_progress = \ session.query(HarvestObject.id) \ .filter(HarvestObject.harvest_job_id == job['id']) \ - .filter(and_((HarvestObject.state != u'COMPLETE'), + .filter(sa.and_((HarvestObject.state != u'COMPLETE'), (HarvestObject.state != u'ERROR'))) \ .count() @@ -976,7 +984,9 @@ def harvest_source_reindex(context, data_dict): del context['extras_as_string'] context.update({'ignore_auth': True}) package_dict = logic.get_action('harvest_source_show')( - context, {'id': harvest_source_id}) + dict(context, validate=False, use_cache=False), + {'id': harvest_source_id}, + ) log.debug('Updating search index for harvest source: %s', package_dict.get('name') or harvest_source_id) diff --git a/ckanext/harvest/model/__init__.py b/ckanext/harvest/model/__init__.py index c347cb48c..aa5822755 100644 --- a/ckanext/harvest/model/__init__.py +++ b/ckanext/harvest/model/__init__.py @@ -360,7 +360,6 @@ def harvest_object_before_insert_listener(mapper, connection, target): if not target.harvest_source_id or not target.source: if not target.job: raise Exception("You must define a Harvest Job for each Harvest Object") - target.source = target.job.source target.harvest_source_id = target.job.source.id From bc1de097e256c56619f52b81257cf947d8758951 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Fri, 6 Sep 2024 23:55:39 +0300 Subject: [PATCH 62/74] chore: flake8 --- ckanext/harvest/harvesters/base.py | 4 ++-- ckanext/harvest/logic/action/update.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ckanext/harvest/harvesters/base.py b/ckanext/harvest/harvesters/base.py index e31f8f7e2..80c3e3ea6 100644 --- a/ckanext/harvest/harvesters/base.py +++ b/ckanext/harvest/harvesters/base.py @@ -406,8 +406,8 @@ def last_error_free_job(cls, harvest_job): HarvestGatherError.harvest_job_id == HarvestJob.id)) .outerjoin(HarvestObject, sa.and_(HarvestObject.harvest_job_id == HarvestJob.id, - HarvestObject.current == False, # noqa: E712 - HarvestObject.report_status != 'not modified')) + HarvestObject.current == False, # noqa: E712 + HarvestObject.report_status != 'not modified')) .options(contains_eager(HarvestJob.objects)) .order_by(HarvestJob.gather_started.desc())) # now check them until we find one with no fetch/import errors diff --git a/ckanext/harvest/logic/action/update.py b/ckanext/harvest/logic/action/update.py index 4006a3bd4..e5627e72c 100644 --- a/ckanext/harvest/logic/action/update.py +++ b/ckanext/harvest/logic/action/update.py @@ -680,7 +680,7 @@ def harvest_jobs_run(context, data_dict): session.query(HarvestObject.id) \ .filter(HarvestObject.harvest_job_id == job['id']) \ .filter(sa.and_((HarvestObject.state != u'COMPLETE'), - (HarvestObject.state != u'ERROR'))) \ + (HarvestObject.state != u'ERROR'))) \ .count() if num_objects_in_progress == 0: From 698f21c05e4a30abc3b5ce386bf520389ea9b9c6 Mon Sep 17 00:00:00 2001 From: amercader Date: Fri, 13 Sep 2024 15:57:40 +0200 Subject: [PATCH 63/74] First iteration of pyproject.toml --- pyproject.toml | 44 ++++++++++++++++++++++++++++++++++++++++++++ setup.py | 43 +------------------------------------------ 2 files changed, 45 insertions(+), 42 deletions(-) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..2de76356e --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,44 @@ +[project] +name = "ckanext-harvest" +version = "1.5.6" +description = "Harvesting interface plugin for CKAN" +authors = [ + {name = "CKAN Tech Team and contributors", email = "tech-team@ckan.org"} +] +license = {text = "AGPL"} +classifiers = [] +keywords = [] +dependencies = [] + +[project.urls] +Homepage = "http://github.com/ckan/ckanext-harvest" + +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +#[build-system] +#requires = ["setuptools", "wheel"] +#build-backend = "setuptools.build_meta" +# +[tool.setuptools] +#packages = ["ckanext"] +#namespace-packages = ["ckanext"] +include-package-data = true + +[tool.setuptools.packages] +find = {} + + +[project.entry-points."ckan.plugins"] +harvest = "ckanext.harvest.plugin:Harvest" +ckan_harvester = "ckanext.harvest.harvesters:CKANHarvester" + +# Test plugins" +test_harvester = "ckanext.harvest.tests.test_queue:MockHarvester" +test_harvester2 = "ckanext.harvest.tests.test_queue2:MockHarvester" +test_action_harvester = "ckanext.harvest.tests.test_action:MockHarvesterForActionTests" + + +[project.entry-points."babel.extractors"] +ckan = "ckan.lib.extract:extract_ckan" diff --git a/setup.py b/setup.py index 8a9fe2adc..56149656b 100644 --- a/setup.py +++ b/setup.py @@ -1,47 +1,6 @@ -from setuptools import setup, find_packages - -version = '1.5.6' +from setuptools import setup setup( - name='ckanext-harvest', - version=version, - description="Harvesting interface plugin for CKAN", - long_description="""\ - """, - classifiers=[], - keywords='', - author='CKAN', - author_email='ckan@okfn.org', - url='https://github.com/ckan/ckanext-harvest', - license='AGPL', - packages=find_packages(exclude=['ez_setup', 'examples', 'tests']), - namespace_packages=['ckanext'], - include_package_data=True, - zip_safe=False, - install_requires=[ - # dependencies are specified in requirements.txt - # instead of here - ], - tests_require=[ - 'nose', - 'mock', - ], - test_suite='nose.collector', - entry_points=""" - [ckan.plugins] - # Add plugins here, eg - harvest=ckanext.harvest.plugin:Harvest - ckan_harvester=ckanext.harvest.harvesters:CKANHarvester - - # Test plugins - - test_harvester=ckanext.harvest.tests.test_queue:MockHarvester - test_harvester2=ckanext.harvest.tests.test_queue2:MockHarvester - test_action_harvester=ckanext.harvest.tests.test_action:MockHarvesterForActionTests - - [babel.extractors] - ckan = ckan.lib.extract:extract_ckan - """, message_extractors={ 'ckanext': [ ('**.py', 'python', None), From 9883b1ae5d2b2e16f044ea3ff875ab49ba494cc9 Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 31 Oct 2024 11:08:56 +0100 Subject: [PATCH 64/74] Clean up harvest source clear command Revision tables are no longer present in CKAN 2.11 (see https://github.com/ckan/ckan/issues/8390) Removed handling of really ancient and unsupported CKAN versions Tried to make the statements more readable and compact. --- ckanext/harvest/logic/action/update.py | 214 +++++++++++-------------- 1 file changed, 94 insertions(+), 120 deletions(-) diff --git a/ckanext/harvest/logic/action/update.py b/ckanext/harvest/logic/action/update.py index 7dce1f575..5f8a142f1 100644 --- a/ckanext/harvest/logic/action/update.py +++ b/ckanext/harvest/logic/action/update.py @@ -92,146 +92,114 @@ def harvest_source_update(context, data_dict): def harvest_source_clear(context, data_dict): - ''' + """ Clears all datasets, jobs and objects related to a harvest source, but keeps the source itself. This is useful to clean history of long running harvest sources to start again fresh. :param id: the id of the harvest source to clear :type id: string - ''' + """ - check_access('harvest_source_clear', context, data_dict) + check_access("harvest_source_clear", context, data_dict) - harvest_source_id = data_dict.get('id') + harvest_source_id = data_dict.get("id") source = HarvestSource.get(harvest_source_id) if not source: - log.error('Harvest source %s does not exist', harvest_source_id) - raise NotFound('Harvest source %s does not exist' % harvest_source_id) + log.error("Harvest source %s does not exist", harvest_source_id) + raise NotFound("Harvest source %s does not exist" % harvest_source_id) harvest_source_id = source.id # Clear all datasets from this source from the index harvest_source_index_clear(context, data_dict) - model = context['model'] - - # CKAN-2.6 or above: related don't exist any more - if toolkit.check_ckan_version(max_version='2.5.99'): - - sql = '''select id from related where id in ( - select related_id from related_dataset where dataset_id in ( - select package_id from harvest_object - where harvest_source_id = '{harvest_source_id}'));'''.format( - harvest_source_id=harvest_source_id) - result = model.Session.execute(sql) - ids = [] - for row in result: - ids.append(row[0]) - related_ids = "('" + "','".join(ids) + "')" - - sql = '''begin; - update package set state = 'to_delete' where id in ( - select package_id from harvest_object - where harvest_source_id = '{harvest_source_id}');'''.format( - harvest_source_id=harvest_source_id) - - # CKAN-2.3 or above: delete resource views, resource revisions & resources - if toolkit.check_ckan_version(min_version='2.3'): - sql += ''' - delete from resource_view where resource_id in ( - select id from resource where package_id in ( - select id from package where state = 'to_delete')); - delete from resource_revision where package_id in ( - select id from package where state = 'to_delete'); - delete from resource where package_id in ( - select id from package where state = 'to_delete'); - ''' - # Backwards-compatibility: support ResourceGroup (pre-CKAN-2.3) - else: - sql += ''' - delete from resource_revision where resource_group_id in ( - select id from resource_group where package_id in ( - select id from package where state = 'to_delete')); - delete from resource where resource_group_id in ( - select id from resource_group where package_id in ( - select id from package where state = 'to_delete')); - delete from resource_group_revision where package_id in ( - select id from package where state = 'to_delete'); - delete from resource_group where package_id in ( - select id from package where state = 'to_delete'); - ''' - # CKAN pre-2.5: authz models were removed in migration 078 - if toolkit.check_ckan_version(max_version='2.4.99'): - sql += ''' - delete from package_role where package_id in ( - select id from package where state = 'to_delete'); - delete from user_object_role where id not in ( - select user_object_role_id from package_role) - and context = 'Package'; - ''' - - sql += ''' - delete from harvest_object_error where harvest_object_id in ( - select id from harvest_object - where harvest_source_id = '{harvest_source_id}'); - delete from harvest_object_extra where harvest_object_id in ( - select id from harvest_object - where harvest_source_id = '{harvest_source_id}'); - delete from harvest_object where harvest_source_id = '{harvest_source_id}'; - delete from harvest_gather_error where harvest_job_id in ( - select id from harvest_job where source_id = '{harvest_source_id}'); - delete from harvest_job where source_id = '{harvest_source_id}'; - delete from package_tag_revision where package_id in ( - select id from package where state = 'to_delete'); - delete from member_revision where table_id in ( - select id from package where state = 'to_delete'); - delete from package_extra_revision where package_id in ( - select id from package where state = 'to_delete'); - delete from package_revision where id in ( - select id from package where state = 'to_delete'); - delete from package_tag where package_id in ( - select id from package where state = 'to_delete'); - delete from package_extra where package_id in ( - select id from package where state = 'to_delete'); - delete from package_relationship_revision where subject_package_id in ( - select id from package where state = 'to_delete'); - delete from package_relationship_revision where object_package_id in ( - select id from package where state = 'to_delete'); - delete from package_relationship where subject_package_id in ( - select id from package where state = 'to_delete'); - delete from package_relationship where object_package_id in ( - select id from package where state = 'to_delete'); - delete from member where table_id in ( - select id from package where state = 'to_delete'); - '''.format( - harvest_source_id=harvest_source_id) - - if toolkit.check_ckan_version(max_version='2.5.99'): - sql += ''' - delete from related_dataset where dataset_id in ( - select id from package where state = 'to_delete'); - delete from related where id in {related_ids}; - delete from package where id in ( - select id from package where state = 'to_delete'); - '''.format(related_ids=related_ids) - else: - # CKAN-2.6 or above: related don't exist any more - sql += ''' - delete from package where id in ( - select id from package where state = 'to_delete'); - ''' - - sql += ''' - commit; - ''' + model = context["model"] + + sql = "BEGIN;" + + sql += f""" + UPDATE package set state = 'to_delete' WHERE id IN ( + SELECT package_id FROM harvest_object + WHERE harvest_source_id = '{harvest_source_id}');""" + + sql += """ + CREATE TEMP TABLE package_ids_to_delete AS ( + SELECT id FROM package WHERE state = 'to_delete'); + """ + + sql += f""" + DELETE FROM harvest_object_error WHERE harvest_object_id IN ( + SELECT id FROM harvest_object + WHERE harvest_source_id = '{harvest_source_id}'); + DELETE FROM harvest_object_extra WHERE harvest_object_id IN ( + SELECT id FROM harvest_object + WHERE harvest_source_id = '{harvest_source_id}'); + DELETE FROM harvest_object WHERE harvest_source_id = '{harvest_source_id}'; + DELETE FROM harvest_gather_error WHERE harvest_job_id IN ( + SELECT id FROM harvest_job WHERE source_id = '{harvest_source_id}'); + DELETE FROM harvest_job WHERE source_id = '{harvest_source_id}'; + """ + + if toolkit.check_ckan_version(max_version="2.10.99"): + # Revision tables were dropped IN CKAN 2.10 + + sql += """ + DELETE FROM package_tag_revision WHERE package_id IN ( + SELECT id FROM package_ids_to_delete'); + DELETE FROM member_revision WHERE table_id IN ( + SELECT id FROM package_ids_to_delete); + DELETE FROM package_extra_revision WHERE package_id IN ( + SELECT id FROM package_ids_to_delete); + DELETE FROM package_revision WHERE id IN ( + SELECT id FROM package_ids_to_delete); + DELETE FROM package_relationship_revision WHERE subject_package_id IN ( + SELECT id FROM package_ids_to_delete); + DELETE FROM package_relationship_revision WHERE object_package_id IN ( + SELECT id FROM package_ids_to_delete); + DELETE FROM resource_revision WHERE package_id IN ( + SELECT id FROM package_ids_to_delete); + """ + + if toolkit.check_ckan_version(max_version="2.11.99"): + # Package_extra table were dropped IN CKAN 2.12 + sql += """ + DELETE FROM package_extra WHERE package_id IN ( + SELECT id FROM package_ids_to_delete); + """ + + sql += """ + DELETE FROM package_tag WHERE package_id IN ( + SELECT id FROM package_ids_to_delete); + DELETE FROM package_relationship WHERE subject_package_id IN ( + SELECT id FROM package_ids_to_delete); + DELETE FROM package_relationship WHERE object_package_id IN ( + SELECT id FROM package_ids_to_delete); + DELETE FROM member WHERE table_id IN ( + SELECT id FROM package_ids_to_delete); + DELETE FROM resource_view WHERE resource_id IN ( + SELECT id FROM resource WHERE package_id IN ( + SELECT id FROM package_ids_to_delete)); + DELETE FROM resource WHERE package_id IN ( + SELECT id FROM package_ids_to_delete); + + """ + + sql += """ + DELETE FROM package WHERE id IN ( + SELECT id FROM package_ids_to_delete); + """ + + sql += """ + COMMIT; + """ model.Session.execute(sql) # Refresh the index for this source to update the status object - get_action('harvest_source_reindex')(context, {'id': harvest_source_id}) + get_action("harvest_source_reindex")(context, {"id": harvest_source_id}) - return {'id': harvest_source_id} + return {"id": harvest_source_id} def harvest_abort_failed_jobs(context, data_dict): @@ -974,7 +942,10 @@ def harvest_source_reindex(context, data_dict): if 'extras_as_string' in context: del context['extras_as_string'] - context.update({'ignore_auth': True}) + context.update({ + 'ignore_auth': True, + 'validate': False, + }) package_dict = logic.get_action('harvest_source_show')( context, {'id': harvest_source_id}) log.debug('Updating search index for harvest source: %s', @@ -991,6 +962,9 @@ def harvest_source_reindex(context, data_dict): if key not in config: new_dict[key] = value + # TODO: workaroud for extras exception caused by convert_from_extras + new_dict.pop("extras", None) + package_index = PackageSearchIndex() package_index.index_package(new_dict, defer_commit=defer_commit) From 58e7eb088c61aab4937e21434dd88e4ca459c89b Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 31 Oct 2024 11:32:04 +0100 Subject: [PATCH 65/74] Remove incorrect quote --- ckanext/harvest/logic/action/update.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanext/harvest/logic/action/update.py b/ckanext/harvest/logic/action/update.py index 5f8a142f1..f97a61d9f 100644 --- a/ckanext/harvest/logic/action/update.py +++ b/ckanext/harvest/logic/action/update.py @@ -147,7 +147,7 @@ def harvest_source_clear(context, data_dict): sql += """ DELETE FROM package_tag_revision WHERE package_id IN ( - SELECT id FROM package_ids_to_delete'); + SELECT id FROM package_ids_to_delete); DELETE FROM member_revision WHERE table_id IN ( SELECT id FROM package_ids_to_delete); DELETE FROM package_extra_revision WHERE package_id IN ( From 3f65713ac6290dc17b93bf6ee7cd5504b80a312c Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 31 Oct 2024 11:40:15 +0100 Subject: [PATCH 66/74] Use python images for testing --- .github/workflows/test.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c9ca89cca..6a2447293 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,13 +19,19 @@ jobs: needs: lint strategy: matrix: - ckan-version: ["2.11", "2.10", 2.9] + include: + - ckan-version: "2.11" + ckan-image: "ckan/ckan-dev:2.11-py3.10" + - ckan-version: "2.10" + ckan-image: "ckan/ckan-dev:2.10-py3.10" + - ckan-version: "2.9" + ckan-image: "ckan/ckan-dev:2.9-py3.9" fail-fast: false name: CKAN ${{ matrix.ckan-version }} runs-on: ubuntu-latest container: - image: ckan/ckan-dev:${{ matrix.ckan-version }} + image: ${{ matrix.ckan-version }} services: solr: image: ckan/ckan-solr:${{ matrix.ckan-version }}-solr9 From b4bacc5d9bd5bf725aa2246c6521c02e8839a3df Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 31 Oct 2024 12:13:07 +0100 Subject: [PATCH 67/74] Fix var name --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6a2447293..204a9239a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -31,7 +31,7 @@ jobs: name: CKAN ${{ matrix.ckan-version }} runs-on: ubuntu-latest container: - image: ${{ matrix.ckan-version }} + image: ${{ matrix.ckan-image }} services: solr: image: ckan/ckan-solr:${{ matrix.ckan-version }}-solr9 From 4c40f6b6d964be7822daa2442dae747f45fddedc Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 31 Oct 2024 12:17:18 +0100 Subject: [PATCH 68/74] upgrade pytest plugin to avoid exception on 2.9 --- .github/workflows/test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 204a9239a..32fc0d5e4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -60,6 +60,10 @@ jobs: pip install -e . # Replace default path to CKAN core config file with the one on the container sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini + - name: Install requirements (2.9) + run: | + pip install -U pytest-rerunfailures + if: ${{ matrix.ckan-version == '2.9' }} - name: Setup extension (CKAN >= 2.9) run: | ckan -c test.ini db init From 73dafa26cfc59d4b8819b0fdfcff388ceb6cd339 Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 31 Oct 2024 12:24:03 +0100 Subject: [PATCH 69/74] Tweak pyproject.toml, setup.cfg --- pyproject.toml | 44 ++++++++++++++++++++++++++------------------ setup.cfg | 10 +++++----- 2 files changed, 31 insertions(+), 23 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2de76356e..04efb49df 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,40 +1,48 @@ [project] name = "ckanext-harvest" version = "1.5.6" -description = "Harvesting interface plugin for CKAN" +description = "Harvesting interface plugin for CKAN, plus harvester for other CKAN sites" authors = [ - {name = "CKAN Tech Team and contributors", email = "tech-team@ckan.org"} + {name = "Adrià Mercader", email = "amercadero@gmail.com"} +] +maintainers = [ + {name = "CKAN Tech Team and contributors", email = "tech-team@ckan.org"}, + {name = "Seitenbau Govdata"}, ] license = {text = "AGPL"} -classifiers = [] -keywords = [] +classifiers = [ + "Intended Audience :: Developers", + "Development Status :: 5 - Production/Stable", + "License :: OSI Approved :: GNU Affero General Public License v3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12" +] +keywords = [ + "ckan", + "ckanext", + "harvesting", + "federation", +] dependencies = [] [project.urls] Homepage = "http://github.com/ckan/ckanext-harvest" +Repository = "https://github.com/ckan/ckanext-harvest" +Issues = "https://github.com/ckan/ckanext-harvest/issues" +Changelog = "https://github.com/ckan/ckanext-harvest/blob/master/CHANGELOG.rst" [build-system] requires = ["setuptools"] build-backend = "setuptools.build_meta" -#[build-system] -#requires = ["setuptools", "wheel"] -#build-backend = "setuptools.build_meta" -# -[tool.setuptools] -#packages = ["ckanext"] -#namespace-packages = ["ckanext"] -include-package-data = true - -[tool.setuptools.packages] -find = {} - - [project.entry-points."ckan.plugins"] harvest = "ckanext.harvest.plugin:Harvest" ckan_harvester = "ckanext.harvest.harvesters:CKANHarvester" -# Test plugins" +# Test plugins test_harvester = "ckanext.harvest.tests.test_queue:MockHarvester" test_harvester2 = "ckanext.harvest.tests.test_queue2:MockHarvester" test_action_harvester = "ckanext.harvest.tests.test_action:MockHarvesterForActionTests" diff --git a/setup.cfg b/setup.cfg index 03bef2d7b..17ba89470 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,8 @@ -[nosetests] -with-pylons = test.ini +[options] +packages = find: +namespace_packages = ckanext +install_requires = +include_package_data = True [extract_messages] keywords = translate isPlural @@ -25,6 +28,3 @@ statistics = true [flake8] max-line-length = 127 - -[tool:pytest] -norecursedirs=ckanext/harvest/tests/nose From e67523d396ab2a83e042ca2d3f62fc25844065b0 Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 31 Oct 2024 12:39:09 +0100 Subject: [PATCH 70/74] Remove workaround --- ckanext/harvest/logic/action/update.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/ckanext/harvest/logic/action/update.py b/ckanext/harvest/logic/action/update.py index aad3ddce2..700e3974d 100644 --- a/ckanext/harvest/logic/action/update.py +++ b/ckanext/harvest/logic/action/update.py @@ -965,9 +965,6 @@ def harvest_source_reindex(context, data_dict): if key not in config: new_dict[key] = value - # TODO: workaroud for extras exception caused by convert_from_extras - new_dict.pop("extras", None) - package_index = PackageSearchIndex() package_index.index_package(new_dict, defer_commit=defer_commit) From 0a1ee0dae66b3f19e52e35b5f1dccfd14352a3ca Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 31 Oct 2024 13:11:10 +0100 Subject: [PATCH 71/74] Update CLI docs --- README.rst | 126 +++++++++++++---------------------------------------- 1 file changed, 30 insertions(+), 96 deletions(-) diff --git a/README.rst b/README.rst index ad30f13cb..a454a77d6 100644 --- a/README.rst +++ b/README.rst @@ -227,102 +227,36 @@ For example, in case you want to retain changes made by the users to the fields Command line interface ====================== -The following operations can be run from the command line as described underneath:: - - harvester source {name} {url} {type} [{title}] [{active}] [{owner_org}] [{frequency}] [{config}] - - create new harvest source - - harvester source {source-id/name} - - shows a harvest source - - harvester rmsource {source-id/name} - - remove (deactivate) a harvester source, whilst leaving any related - datasets, jobs and objects - - harvester clearsource {source-id/name} - - clears all datasets, jobs and objects related to a harvest source, - but keeps the source itself - - harvester clearsource-history [{source-id}] [-k] - - If no source id is given the history for all harvest sources (maximum is 1000) - will be cleared. - Clears all jobs and objects related to a harvest source, but keeps the source - itself. The datasets imported from the harvest source will **NOT** be deleted!!! - If a source id is given, it only clears the history of the harvest source with - the given source id. - - To keep the currently active jobs use the -k option. - - harvester sources [all] - - lists harvest sources - If 'all' is defined, it also shows the Inactive sources - - harvester job {source-id/name} - - create new harvest job - - harvester jobs - - lists harvest jobs - - harvester job-abort {source-id/name} - - marks a job as "Aborted" so that the source can be restarted afresh. - It ensures that the job's harvest objects status are also marked - finished. You should ensure that neither the job nor its objects are - currently in the gather/fetch queues. - - harvester run - - starts any harvest jobs that have been created by putting them onto - the gather queue. Also checks running jobs - if finished it - changes their status to Finished. - - harvester run-test {source-id/name} - - runs a harvest - for testing only. - This does all the stages of the harvest (creates job, gather, fetch, - import) without involving the web UI or the queue backends. This is - useful for testing a harvester without having to fire up - gather/fetch_consumer processes, as is done in production. - - harvester run-test {source-id/name} force-import=guid1,guid2... - - In order to force an import of particular datasets, useful to - target a dataset for dev purposes or when forcing imports on other environments. - - harvester gather-consumer - - starts the consumer for the gathering queue - - harvester fetch-consumer - - starts the consumer for the fetching queue - - harvester purge-queues - - removes all jobs from fetch and gather queue - WARNING: if using Redis, this command purges all data in the current - Redis database - - harvester clean-harvest-log - - Clean-up mechanism for the harvest log table. - You can configure the time frame through the configuration - parameter 'ckan.harvest.log_timeframe'. The default time frame is 30 days - - harvester [-j] [-o] [--segments={segments}] import [{source-id}] - - perform the import stage with the last fetched objects, for a certain - source or a single harvest object. Please note that no objects will - be fetched from the remote server. It will only affect the objects - already present in the database. - - To import a particular harvest source, specify its id as an argument. - To import a particular harvest object use the -o option. - To import a particular package use the -p option. - - You will need to specify the -j flag in cases where the datasets are - not yet created (e.g. first harvest, or all previous harvests have - failed) - - The --segments flag allows to define a string containing hex digits that represent which of - the 16 harvest object segments to import. e.g. 15af will run segments 1,5,a,f - - harvester job-all - - create new harvest jobs for all active sources. - - harvester reindex - - reindexes the harvest source datasets +The ``ckan harvester`` command provides utilities to manage harvest operations from the command line. +Please refer to the help message of each command for more details:: + + + Usage: ckan harvester [OPTIONS] COMMAND [ARGS]... + + Harvests remotely mastered metadata. + + Options: + --help Show this message and exit. + + Commands: + abort-failed-jobs Abort all jobs which are in a "limbo state" where... + clean-harvest-log Clean-up mechanism for the harvest log table. + dumphelp + fetch-consumer Starts the consumer for the fetching queue. + gather-consumer Starts the consumer for the gathering queue. + harvesters_info + import Perform the import stage with the last fetched... + job Create new harvest job and runs it (puts it on the... + job-abort Marks a job as "Aborted" so that the source can be... + job-all Create new harvest jobs for all active sources. + jobs Lists harvest jobs. + purge-queues Removes all jobs from fetch and gather queue. + reindex Reindexes the harvest source datasets. + run Starts any harvest jobs that have been created by... + run-test Runs a harvest - for testing only. + source Manage harvest sources + sources Lists harvest sources. + The commands should be run with the pyenv activated and refer to your CKAN configuration file: From bbb8649a55faf00006424e1f9da57e5eea1eebc0 Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 31 Oct 2024 13:23:14 +0100 Subject: [PATCH 72/74] Add tab for harvest sources in sysadmin page --- README.rst | 5 +++++ ckanext/harvest/templates/admin/base.html | 6 ++++++ docs/admin-tab.png | Bin 0 -> 15681 bytes 3 files changed, 11 insertions(+) create mode 100644 ckanext/harvest/templates/admin/base.html create mode 100644 docs/admin-tab.png diff --git a/README.rst b/README.rst index a454a77d6..d3508ac0a 100644 --- a/README.rst +++ b/README.rst @@ -104,6 +104,11 @@ After installation, the harvest source listing should be available under /harves http://localhost/harvest +There is a "Harvest" tab in the sysadmin pages as well: + +![Harvest admin tab](https://github.com/ckan/ckanext-harvest/blob/master/docs/admin-tab.png?raw=true) + + Database logger configuration(optional) ======================================= diff --git a/ckanext/harvest/templates/admin/base.html b/ckanext/harvest/templates/admin/base.html new file mode 100644 index 000000000..7da68bc07 --- /dev/null +++ b/ckanext/harvest/templates/admin/base.html @@ -0,0 +1,6 @@ +{% ckan_extends %} + +{% block content_primary_nav %} + {{ super() }} + {{ h.build_nav_icon('harvest.search', _('Harvest'), icon='download') }} +{% endblock %} diff --git a/docs/admin-tab.png b/docs/admin-tab.png new file mode 100644 index 0000000000000000000000000000000000000000..b975e097e0952e332a7d22e1d231d091ece5a73b GIT binary patch literal 15681 zcmeIZXFOb8*9R;J5+Ni?bh(s-C<#FjEeX+k?-B%~x51bZL?jWt1wo<@M(+%fMDJ~s z(R&*cWe9_LPh7v}y6^k>^nQ82ywCk*pZ(i&&f4p&^T0gZ2bUvn63Ddqyo23w2u>4@4#Q%dXUDn!>MRmW$t4!q4BnVqQFa=2; zAmH`_GSDVV5Crn*q=cZqmY)eT-M&k~`I?CM-n-N|8v}rm{i4;l$1uE z=+Zw;ur@Jim$tdU$N!MPDIc@U{_nQ|4^Z&P$6RY``~Ghe{h60kW#?bgE_(kJ`CnUJ z0Pnvb`d0}3--sUVwfzA(t7|0xo{jz_iTbNF++K+zb}f2Rzs*PLCE|CysWX;fcx zGJS{$qy3ld{`4r~rb+%+=J^T?eyuaU^`Gen5GnKjPUNO~rfj-Dcm4JMNQjq6JsCKa zSJm4>{?~Lj3MwZ)Xq8*H66^o32WUF^0jQHH{99TDbo#&H@&8jX-ge_RWcS5U2S>#8 zce!)#7IaVqCPEzO&t~1q=D1gnu+iLDX=AaOeTr1Bw;wCz-;p2&0Hj#Q)%PtC z@doK?KcP?1_&50@iiE^(AMOdPF^d_0EsjbFRlEQYC;z1oTm3j^@=K(9H$KULoALVt zin9%o%;N5mMo#marY_en2kX)qSFH<6fM)}bB9N!rI#er#ku18QmLC`+UOxFF=58U8 zLPbH7GwUVNiYvW%pRahI)+e>w6+LzK8iroDP1NOU!4)*We^|l-@n_CQ<8wiciN%H$ z!DE`bf1|rV11JopysIURNL7`shpd7cIfBmHq_e71OLjKLi;SK;@A%58jr{g~j^I9L z{7K)j!s+FSx}jG4jf=Rsa)VQucm0JYKSe_Px6?TWsk8Bdg%7UP2F%GLU%iSc^eF=6 zAW?W+?d4f$)8Za3_V4KVf})_|2KD8xKBK)W7|gNO>$T}t<2xc|jtqF{D2OQfF|^D- zKd$P;J)US}`e1y_nEE36gDV(0m6OhRmVp=73dY(G;K=TpxTV{#_!F~=5%c~!nv0kT zYB@oHP=%`qy1%oKQv_h48gs|~o5443{yyxGoTa7ZR@?sW{^;vv*|S9!xAeTCQq}+q z!@m>J7xT*$aVr5Uk6cN+&0lkxtTM*@D5c?4_7uISzDAV*uemgBeikL>0jei~;D~Rx zpZz5t^YKk=rBWaFnOq?ba^dnX;y```Eg7tRroDR!kaNYH zl=}7!pu_#qne@*Ke(41svGZ+xWKJ3f2vzS+Tz>pdMy%EW)L|)=TKgz}1+bpl{9o=A zu|S8*#+K{C7k0dWLY0*53xCDX0YY_4xYrc^r@kwQ{NrZ7G09h?s`7vwV#eU7@16lE z_zD-xg-h;=0*|Cv+^1#?&H;qRd-6zL58*z#C~@Rn=s1;8?M3~PT-wBdoEvo%tWUUs zj`FQZia*GgPq{+F-LoCyQAfe8_?j~%;a- z1D)9W6|{fQ5iD{OP^j}VWC0O9J?;hb2zPBv0RcGW@ziG}BW}BSH<|PJXt8Z4_ejns zlklq&?=f5IS|8vu!yL(6$FNnpgtB-!ws@(0rGoZuIOp*=fIl;mgQ~88b;liAsjAYJ3Aiq||kzB`XPWKvb z&!8K5Bbk%F#d=Us&}n@!4_W@knAZSe4{E;p^D1tuKDp%a$i}75x{a5+Z!2d=+9t3Z zt%jQjkItGoue)NrjWNxiK0}L%pXT>^LzN#OQdYz_THfPwH;*zqq(}Q`k32KC?baXyfd&_8e4uPYsabx}4_PiI zz%aZ``{iZ_<8z6l$w@LTt zA31)@cHImo;0)*dvxDj(jcn=IJmc$^NS*$Gk)$)lH&*D`-U`#wWt(v^er;-nVbkgc z1jKUffHR}Sd#)vS%fB;2Y_q*zRqG2~$ny1;UN9cVv<|a6Fw%LwIc|Z9Y%D7h+sdm} z*LgizaA)wiBz&fbBWU4_AnjZh#S*A0Z3{%#8vsiWEIzB>)(!Ux9aCa=sE*TN#Nh%q zap0L}ke6r2+mbt!cr8LryDHU8i=ovdTNy6THB(T+^PXuua!R+R{+ANAb!&@Ow zSIw(hbZ^I7&e*Dv5;58DQC&BwDM{L8k_la*Zh>jHqhFwLDBtVp{!JrKeak<{=0}^j z3@;zadSnuQSEk|7#I+}?RMT=sR&6d5-c;c4?9z2OLb++VuhlPKxntKFF)H>od)P@X ze!sM3dS+&--a_1R9KGX;boH=0e>iU8{adzVYgU@EskBa9`>Mhova|*KeELyN0V&sj zRR0+S5o9|Z-r%raD>ygonva;Vd%lx~-Bh%|uBU}J)V7@V`^UO@J4?4k0H$$oB>0Q zIIkp5pXJwpL8ViVDzeiYum4PrW?h3`vA+jG40qxxInk$9Af+0t?ktUhNOO@MzrtSf zWlD=^h3NXrLFbOby5hd%4+*tq)e9d=25t^CE(rH|Gv{!dnT*JaBBQ3}FfLn~mAob| zUm>^h&@4FLM(3rlrof0p2&RB>dj9>0z-t57!dug;BEZsIBP{_=W`n*qnjkkvWnU@tR-j-VV+#Z(ueD}xlxEx_Is-hO| z|HL+!+fgm6psd2&yEGsY}`1||yO(<|ii(d9}R7o-i~39)skh%;Y!zu{IN z!47K&Er_3OimL71k}Pll^6b!~)vCm;%+zHvjX=SCyJ~HHo4ucn93G{C2lYq4e8I`t z)0!;jqL>E-DBT`@^K(cauj}$;$68?VN`p14LWMaZOLj+>BJOI;)wni(4Q2^%;X~OU zX(HU}^{xkg;59E{|Dvkd)w!KiYV!Pa24&&*LILg{(++kzW-E-UL6|7PudBl(8JU}7 z+l6nz0hwM%MiUi*TsrPC7X^(k*#b^V2--GEzI;;zAqOm1UP4DTonx8LPysha@hGFN zNh?&?e~v&hhS-*eslCFJb}=_F>=z53N1=G-ocSKA^V7M4cl%t2fTAi&u-1pMdh*`Z z?P<{kgK@$JBhv0|I;zzr?U&98MVt+`b3G|qTG@${ZR{QN)Z%b;HY{Uz%nhjo-m!Bul==!Hx%}$mh>R+;=@8W zqZ(bhUS)r>dzq$>pN!Dz@@JIUf%M8qF1ffJmouhsi&vvAjnR);7Bc9qFqyrIXGOtX z(|0K#bqT%BVVoxKJYic-snM2P89i<**FY9FP_F1HkD`zo6Xlpuwam%=ARJlB zhazn*6VW}=E`sD%tySWsdnR7LJnxZqc{aW{<`7m832Gb!^_S^TYBj&g!DFK3kl|%5 zh@|`DU#AE*8q50oU&!lx#*N!%z@F0iK}98}b8B;Thqbcvf+!zh7MGoG+u40^whF(PwM9scSk<0O7hoq|s z$iDHAKXN7t)qD8H5bduqH%t$GPo47k2hz3ARqkxz&QYY;%1DKVk?ZndZ7T=$9E`Fy z4#(7^FIxt)EwmeRUj))hPa+~H>2T8jh+%d;bR#6EB-+WtfY;pf0|H;)xVzD{gRb9i zx?gMhgUoV&zOq-B1hIa5E!NSkE_P)0Aj)IXZgWD+bZm>@SAYYlK~b|ush4){Nk@EQTnIe@bv!-waUPs`f$2q!(I0k?>Oz+f zw3^+YA8~vhfp?vcp85Kq5iL;`15%n>{7oyvZ`_N5N;Z_v)^a`{W)Fb(GtQbANb%^q z-C&N8Bto_!|uY zUemyzH((~aT;D=rYiW^A;?ZRT&y$h9zM;HLFS1)@u~gI9F6}0&aHYL^2|4#yR7`Aq z;Y!t`>q%h=<{>$9Z@)bd*B;{u$6`ij76b5>fgLaj%}K}|aJMXhWwJJAj;xwy7~9<= z{wPUS`eYj6lDVrfZvMlk{8ELub3;I)o(5IiO9TP?3+`5Tz4@#aGGR5oqk#+GsaT^n z35vh^Kz+C&wM6O1Coz>2&AbNAC`G(yb=&%v@Enz%;HEnD^CN*Wt(hZc& zQ(NKxLKJ!H+P=U4FpR4|Xnmt9Igj^whw$>O^^7}EYr0!r7DjyC5t}RtdB=XD)q>sq z{KM9L&B?PcFMo`_FDRd;K==_=)(HBop`3H0V4)eJd+tm)AK_dUy}JrnhodYl9L`pWXY(#_hv=eX^i9`C8Ca z8rdDaNoP!+cv}Y$DvsmPTMufl`M5!MHQIe}Tr$fCFA*(X*`G&wCr2NrWBmrr4r^{k zdy^~P=ZSo>(T6fW^QE*+(bUs$D3vSGNF30)uk2e1IbICU==Ni8Jx__AqKttVJ}o5o zhGrb3e;J5SC1{2lU6W668AUxXya$;b)NyR#@M*fbUMnk_!kOZ!c!1ARnaLJNrL&Cx z!KnM4tP3eVH|SG4<$vZ2KbQy_|G;U|>GnffJgnnA3HxpyOxfsQu{0VdCR?Srvhn4V z6aMAVQEiy!TM?>1C_2T+ z68yW5u8J1=expaAniYb7YT38onpXTO^n`We?AV1el+6`7SY$IqB%6*n^pGh+;D5Z$ z4{Brg^ENL`oN>kk)zKc7?Jz>GunoJ#7@X@lUn<0`D0kvQAFX$hIDKl%N8AJ;42;$Pk4lKCtJ%9M#uVn*RIuf{7npj@KO zgG}HqxFhIp)y|!{0bhuf$8*!$eGeqpN2wjjySkngT8{?xKEn=V6Gqxta;pbDLtU`q z=#uW$t%p7xra>BG?OVAhFSjP6p^v^GkO<+~K(|;vEM^p1BFTvpDa?;7Yv^g;@<;IV zBKKn@6yLLV<3g4aiw1n}Nf>`!j41+5rSd70rLno$90lbYq9Nn*Tw~NVrg-o+L~%i1 zn5ZDOI551)ZiZ<}2-B~*~j780Exi!7{9*b^ix0yH;*h2fA#!@pS43YG9|AOuB7wIwX(pe6IDBAKbf z6f>ygYvoim)rbMxWKr?v4EH%{4OC!VRRb;&EPoLKqWXF^!n@QJbZ_gDFq|lgIe;_# zp3VN$A?tdwqmurbgIg?m^G9iu%>eV(gs2*2XsdfYTv zC5APt+&1J^Zujfb=YrQJAHV5}$AoY8vIZ6C-}H+Q>r#lfMC^&HT4|#vlEa;paqT8@ zg+MTAB&Qk__$!t<9CSj9yYjkyI#~60^M3h4Dt+;F$;j{x4@Tdjy(ID9OZQhQcIY+C zL*2&Qq!--tIWVK1xSPUt^etNsn^eCF^!lFKwN#R?$^^`<$^>FvH00gnHVXKKqfTQDs{7f* z3-^P?qw}nQ)I~r*FmTF-dYqoDTA}3QA5KyF`uGNP-j-2urd!P8(-Q^PBxHiF-Xh*5 zjrqx}aKMn)G~x$;>ikXadvi=D7Ls4EGRMF*KDnW&_$;uAEarz^ly;^v5inFDXV`OI>Vs9mF>#SETE=f{fiHC zbwTx@VY9|#t5QuV_fTyh)XYR88H3!)-pW~&o+T`LgE8?p?+KPB$vW-EyX}VB%5J&2 z45&C=SxE+P=knuv+zEl6SGY3iC0tR^+P5+}sqib-Nv!Ub*>6oj7$O%2X0I?*<1buz zeA&-H?8ns}t(XGBN_nH}R#o(9GxR+4o9|XB=IzAwz_J2+ z9kI-S`84O?LBg()nSXVPk;``nN?Z5}kP-tlpa!O<8ms(jAkk5YRh6P(y`ZXy>u6~V zkp%3PdV0RYbQ(v4X!ys%z1&X?SRoP)AK{=f+qsmGoLS?{k$@xq4rmIM{epx`z|h+Y zEf-Tf5=WR=>TYbp>2fH0UWp@VWMA=}@tfhtr|+HTm!#n;4gAluT&EEI$p`hf&r3ROQiYW|^XO4PzjSyQdKuxE?a`>oBe{hPKa2*J>Hsfcp?@E21)20b z+82gYV{Zyc_id3UnT}5T5I{S7PI!I%(ekkcKf@KWt{r@>x6Jj)%I=c@5c0SlicHoY zM;}ebN}s#EE(r{=Efe>w6QjL$h&%wF$3dy3Oh4)0RyJ(rmNQd(X91bB{Dr(ZE&q$h zs+onFg~c|t0r2c(>DH`)vmJ(q^tb4r(JLGDL##hSrc!Tg zwe@BnrX@HAK5S7;;inr-u*i)Wpi~f{wKE%qpp|Ba>F%4qM?GYY-X94qPk=44?Dm}xsln)UuTkZ#76LQirSsG3CwPgn}_ zbc2Pe*MFy4`DGss*o5Xh`PmEVS4Hp2OgIbP;*MA`?-VO^ozcA-_lg=e9a)ns0Ov^# z_B@@c9Q84XhcoxxU(qe%;Sl$4sO@Gs3ozVBX0kUkH9Qj>!+KuFKc0Kl=Pa72;45q; zxhGlbzu72ZslQuBdo7Nue`DL!n98Q1;#d-jv|rL8?UK$&Up$eKY$ZD}?Mm&=NZkEY z+c;0=M&}%2YxkE-j++mRP*$soTA5piXy(p^1d6lsDsO^44Cg9~*ytb4c-Z1yw8trB z?iUqkOF!9DX$YNKY0ylXzxchhq-6g3B9p@5mn7HA3RGDF{W4D3&km8{?y%63>K#vnaaskolHAE^Ok5py|D33;jAEE%RE z(J!6e^FPZ&*QDlch6a`5o@!Hts%}ysT2~RwRnNJgaKB5#AvxVL;@vs0FK1b?_Gq)d z;O0(!x+fA=MQ|DG(hVq6Q9JJlS|W&r`L^y`=2rC%c%IqjK-Q9u3RT=WOI;t1OlS}* zf@=!Z_$G3>OS;$u!|s?mcA}u`hIpO<4Wu7_5|VTB?Bj<2lPL(NUd+euk{IfU-_f}@ zxHGF^Ay?y=on$P!#4m?C?H)D;sPuU`=>+|3or%68X4e{tB41A`SCb{^9K`BZ8bVz) z%VO_BO!S)6d2ZdxxAaMh{Lqwh-TT*^)s~lLyVrXWygQ<=1ul&n}29m8T?HNOO7%}bycY+K)FNl zoog8fq06+WoCAK4X)1hObY|F;rJ(4E)5z@29oSDV=b1n~@L4%IByCVF*G0Ig3*T}z z>E38*ng=>yXP^8_s&5+;y$L=2OfGum6CV6RX^N)c|{gtqAGQCareVY7w zBT;%W1>F_}jof%O|0$;ltw!*h+I7So)&=Sk``Ph%@$1dLU+|-{_gD&;>ZwMvD>f#f zO|Ju+aUfZ{ISWzt5VBIq0<1h1+nU;89TV6{xSV*xPbR@&7n1-dTbvpoD{E*3t=8_t zylkDgi!4zxF8zzxB+dAk9%!-)lLpuQ-Qnpqg}y4peFYKp`ur5ExrOSy9DckR!}#ja znQVTN#`b=RLjR$KZXq$%=nFsV_nq@=UbxejMS(H{r8)K*&H@ zVbZSqiz`S0g^G9Mr6)8d{u3a!?ghij{Ha>&E{SJOEU-$lzFI#YG>NAadKN{%#^ydv zYw>K`NK73+wxUkG=TRZSym8gFMOJN9ar88@TuEZY;bY*@9%jECaYU28tQ?l>7Duhz7o^EW=2mOlCZP_w$+}Tx!N_RgpGQh}UvR6qN z+D(?)om-Hp7C~8Zw1bDbEO%cLw{jLz{OX&wUwXuVap0n}s`vhtZw}%P9f@l4J7$`F z2jll3RcRT3lTG> zumZ;fG^O@vd#N;^Zy1TIZX3-KxxlU*ijK6u8)n(@wq zk0Ym841maI2+%so99IurCmlErtCjV9Wo#8dA4F43ZUvihq(+U_bI}Wk8P$-5Y`zNR zWY_=Z%{OKB>CakPC|!L}`!q&iSgcU!nRq4n$Sa|B)7-iDydMmx^5z8KF*I;J>Eqeh zS#QR%utLsIIliiM4ZV+~b3|lmwfvs~LiYtQURkv-VKlvA2G@IT&Yeww_Dk*Q$xW$# zQn6j%HFR<5L&@S7Ugb|oQfEMGA0l^JrBCnYHs{&n%1l?KdO5HA=H+-Gi*EIvvluMs?jL?T|YC zV~-_A=3`Np0BD&aEB$iOz-Adag6C!+SR$vPI_Q zzOS&wH_>osCkSC&Xt^WYp@o#O=R%@=eYwJBeO?8^&x3BQ23{ZG@Kf5FgAT`vKO_X0 zNaWTWgda2xY#AlOOEU=aU;IBmoalWjJg^sqD^F++31G=P?me#(Zl-R02*!Y2GkoV@ znM!iog?zK(j5EuEMYLwoGd^imTgSl<#FWX%hjbQ+H7pLs&jz$Dar}}He%^SN94N0W7KSr65CutKVP|zJ;E8{sEC_|pP zzrCX9G|1WgU9hk^j_h;l1l~Je(pE2mm3mY98;korv4eJ6?$ADn0T(>AYxG932CuRq zGD3H4usXf8Q#z1$#Hd_G=5(N&k3{Y2W)4cvwFdO0Kq+$}}sBFXz(MEK}h)`V6Vb7Ep7)x?(8j%nPpi)u{uq z!ar_-?*j{Bb<^CYDx%6&wmDKy&o^;?I<`4}j?Pe!QYjPiq`eH0td$ylqP?Z*aXc*h z?V=DlhFW%e2!`1HG!h)AfnT!ED8K#F0A$!J0&H2iT7d!z!!`O={N7)#RPSS<`wQ3h zwfQ4ITmy2tbkM>%%Ic$tUo@(0n18L%y1yF~hTwmbs0U zbwJrn`MBWhgDsb|*@j#OnS)JC?ny=-HQ86|To;(bnatRLS4?C`URQCb;t1@sGj!-w zHfM#E+sE?w5#xSyj~&{BT!k&0Y)Y~%FPP6zXRgyD`7?9Cn^sB=)}^;*UOelySHxUDS!`HFu}H)@vAz=w2;?ydhIG_~d{MzBl9QBwrm|zpImM{cd!!#hhZx{R8w#jBYBjRYFYnbuJ6g zLsixnswTT5&1EP}y%JnW2`<Q$ZZ_ARQ-)>k%4WjE=uSi6hRWo`!N1rwM`G70H!3+` zo~d{CE}3cNFJ;EVUO)6Nw+g9_JF(fUf>65cIm&1kMX#3ei6WsTPJPLvw~^C-;#SOq zU^n%gk5q*XE%`LVJ=%=&we^r1Bl67^J;R#hrvdOq#u=+Z9W*M{JWMcvq}2b0fiyhJ zG?oMtY$=vdWSY4k$-w!_WTfrEVG6Fr?I5L@P({#)!(jIV(o~mx({!IIUT4{STEAhc z(Nv3ni3v9G>M3+6ydjoLl>~h7J>)3%81?X}VN_)~f9d1)W*u^;X>qn1d#(EI;Jn4v zPOupP{-`G(;EiCKH#Rq(pB8Vtr1aIg>ph3@t+_HBT_B|pFo98`<*pt8^6_lhl(J9T zAogu)Nro-S3>^RLCCiZR*(vVESSI?ukG^_{?Zmxi+hb%&0~#t_dqPuS=c6Gc^MZqA z-QlU*45M>KSCNa5Ir6@mV8G7cfjTp%_DG6raiB&ej5(}R5aFny!EpA- zi8_Y^W+GCjk(b5m9$q!kYZ35U)+Bs_T0H$~)O>1#1+>+tR{N8bk-obmWZ_!W6c5~p z#n&gLmAH9T!WJ^hIaW8sWAw=((zCGSgT4wc52H^_+p*<43-HocYzBtHW|~u=JK*-< z#{9Eblsu(GRLc5J!0havuz#|Cu0uu_%y0`Hj0oW=;e@}u zkdZ5@r%Q{_%eUB!Pf@mo=Qqa=DG4uP%hvKoza(SVcpiOkY8jdLGSp;zAJ+W@;0rC- zIXSX@5xvZT;Vq%r$jl=9#U#4V=Jt!t$6*{T0M#k7_Y{OQw&+xjrIQ^ErJZknm~&R> z!8rzf0qONUyu5@7*1EY`xT@EO8I4x(3WG-m-2tV>x@Jl>H@bwF_eJL9mEyKg)w1Rq zO;9!`8zVXKBXz$qP+^Wl+zGhFr6uUJu=SQhPm}%iLpj1p8Y4|jxuihjc6~OZl2&sE z8JDVj%)x*J$J%CzPx-keKT!T3+oj#n-Ew3fwRy$L=oAxPmeCN-QJwePsrdvq;5G9Y z5>Y&$rH7w(b#qJBs)CrDr4-#OTw!_Wkt*?IW#BL*?`!qP!BO7l+hvKMgnCS!3nCQT$TUaNn_Lj`Qq@zlI9aJ$T6mFY_YpRifg6`z zVDG4tFv?fBH)F4_poB*yc>A;bAxqqL?po!jqmO4jN8lCr9`_8(z z;%(0_eZe{N32(B9E@%tq?}ZPg#MY>HR33r*Z!-sbQp8D(tP(bo%Tqwti!MKIOyzUS z8Y+}A)6pnq_tYOfcsz4jYibW+sW#|wEW9Z;9{C5etTxs4$l?|V9gkUj{oR@Y@i@Wx zXb01em80)U0=kN^B=mjgBEAQ}nHFs?K%I98T+P~EE?4t$nx^TyQGYh?8O|d(&Zrla zEb_usJo$bwu+PXmVYdg#%o%H0OE}bgI^Y_$AAtqp>ZiFcV*>l0*o#aGPpV2BJ-p&Tx7wWDfhO`-Q~D}oICSuW8Km%1NE=|T ztpa2k#R`<|V5Ka63e&iv3X6R40F-nT~)WSx8YTSG0? z6|gw4ugnz4k^>Cbjqf^6tpU6Uvh1OC6AHWEru1_(LFJAqVT9K)ualyZFG*3JSZ)fO z-)=wHG_1oAN!suphtp)cE6@!s6C|bS6aZM`JDn=>p<@nD=I5WLD=FuSi`qKl|{Jb=~h6sQ8bcbzJgQMe7Z}8`&Hs&fM2_x_!6o6ht@eozu@;b0Tswz zCO5mcNiLBV0@T)ezJIv27e9Xo#(p@I3lLL00|>MqKgLnD0g8V-CN6_qa72v(+n1MB z>XH6o0bl=z-+ICJeFIQV-wwSnA*Fc-kXV_RcvwD|{GnD}wm!Phw;YiD(&kk@#l=Z0 z$klp^hUy~M~NOsBm zL+K6v!+NDr0%lQ)u%ngd6amUJ+XUyk_Q185@%0SXpHmS(|4@>{($oI}IJ0q(e`vLU z1!&|sKLWFSWa##_fD$lIGtoBtg{KMwGFr*DIn@yE5!zI$c0N*Mpb-jH7 z9N%aUyfh(mp_c$4+jwG**77)weKuNY!i`G-sX3~=Q;PLeb@9e4KJW3y)$)cJoJs`GFvwuc&08qyJDoS$v0qY+?UNm1i z0d@W&sr3J!fs0X2M_@F^LZ=@%z432%->m>B>Yd-5FYx@2jQ^_qFCzv2*UJ9_*Z-e@ c@rxkQl}?-t1zP||qlr`$HJ_C{u@3ov0J~giq5uE@ literal 0 HcmV?d00001 From 4d42da135b7886afcfe09888978eac7c600a6e04 Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 31 Oct 2024 14:15:29 +0100 Subject: [PATCH 73/74] Bump version for release --- CHANGELOG.rst | 24 +++++++++++++++++++++++- pyproject.toml | 2 +- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 63a438564..1934439cd 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,27 @@ All notable changes to this project will be documented in this file. The format is based on `Keep a Changelog `_ and this project adheres to `Semantic Versioning `_ + +*********** +1.6.0_ - 2024-10-31 +*********** + +Changed +_______ + +- CKAN 2.11 support #551 +- Switched to alembic migrations #540 +- Support for SQLAlchemy 2 #553 +- Use pyproject.toml file #554 +- Add tab for harvest sources in sysadmin page + +Fixed +------- + +- Clean up harvest source clear command, fix revisions exception #556 +- Convert boolean values to bools #544 + + *********** 1.5.6_ - 2023-06-26 *********** @@ -413,7 +434,8 @@ Categories - ``Fixed`` for any bug fixes. - ``Security`` to invite users to upgrade in case of vulnerabilities. -.. _Unreleased: https://github.com/ckan/ckanext-harvest/compare/v1.5.6...HEAD +.. _Unreleased: https://github.com/ckan/ckanext-harvest/compare/v1.6.0...HEAD +.. _1.6.0: https://github.com/ckan/ckanext-harvest/compare/v1.5.6...v1.6.0 .. _1.5.6: https://github.com/ckan/ckanext-harvest/compare/v1.5.5...v1.5.6 .. _1.5.5: https://github.com/ckan/ckanext-harvest/compare/v1.5.4...v1.5.5 .. _1.5.4: https://github.com/ckan/ckanext-harvest/compare/v1.5.3...v1.5.4 diff --git a/pyproject.toml b/pyproject.toml index 04efb49df..3e8585996 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ckanext-harvest" -version = "1.5.6" +version = "1.6.0" description = "Harvesting interface plugin for CKAN, plus harvester for other CKAN sites" authors = [ {name = "Adrià Mercader", email = "amercadero@gmail.com"} From 26c901f674707ae11acd2611a58c66fa9b6254ad Mon Sep 17 00:00:00 2001 From: antuarc Date: Fri, 15 Nov 2024 15:26:44 +1000 Subject: [PATCH 74/74] fix test container account, GitHub ckan/ckan-docker-base #86 --- .github/workflows/test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 32fc0d5e4..3271637d1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -32,6 +32,7 @@ jobs: runs-on: ubuntu-latest container: image: ${{ matrix.ckan-image }} + options: --user root services: solr: image: ckan/ckan-solr:${{ matrix.ckan-version }}-solr9 @@ -63,7 +64,7 @@ jobs: - name: Install requirements (2.9) run: | pip install -U pytest-rerunfailures - if: ${{ matrix.ckan-version == '2.9' }} + if: ${{ matrix.ckan-version == '2.9' }} - name: Setup extension (CKAN >= 2.9) run: | ckan -c test.ini db init