Move nomad examples to plugin extend

.github/workflows/plugin_test.yaml

@@ -20,34 +20,31 @@ jobs:
       fail-fast: false
       matrix:
         include:
-
+          - plugin: pynxtools-apm
+            branch: bring-in-examples
+            tests_to_run: tests/.
           - plugin: pynxtools-ellips
-            branch: main
+            branch: bring-in-examples
             tests_to_run: tests/.
-          - plugin: pynxtools-raman
-            branch: main
+          - plugin: pynxtools-em
+            branch: bring-in-examples
             tests_to_run: tests/.
           - plugin: pynxtools-mpes
+            branch: bring-in-examples
+            tests_to_run: tests/.
+          - plugin: pynxtools-raman
             branch: main
             tests_to_run: tests/.
           - plugin: pynxtools-stm
-            branch: main
+            branch: NomadExample
             tests_to_run: tests/.
           - plugin: pynxtools-xps
-            branch: main
+            branch: bring-in-examples
             tests_to_run: tests/.
           - plugin: pynxtools-xrd
             branch: main
             tests_to_run: tests/.
-          # - plugin: pynxtools-apm
-          #   branch: main
-          #   tests_to_run: tests/.
-          # - plugin: pynxtools-xrd
-          #   branch: update-tests
-          #   tests_to_run: tests/.
-          # - plugin: pynxtools-em
-          #   branch: main
-          #   tests_to_run: tests/.
+
     steps:
       - uses: actions/checkout@v3
         with:

MANIFEST.in

@@ -10,4 +10,5 @@ recursive-include src/pynxtools/definitions/contributed_definitions/ *.xml
 include src/pynxtools/definitions/*.xsd
 include src/pynxtools/nexus-version.txt
 include src/pynxtools/remote_definitions_url.txt
-include src/pynxtools/definitions/NXDL_VERSION
+include src/pynxtools/definitions/NXDL_VERSION
+recursive-include src/pynxtools/nomad/examples

docs/how-tos/using-pynxtools-test-framework.md

@@ -60,3 +60,5 @@ You can also pass additional parameters to `test.convert_to_nexus`:
 - `caplog_level` (str): Can be either "ERROR" (by default) or "warning". This parameter determines the level at which the caplog is set during testing. If it is "WARNING", the test will also fail if any warnings are reported by the reader.
 
 - `ignore_undocumented` (boolean): If true, the test skipts the verification of undocumented keys. Otherwise, a warning massages for undocumented keys is raised
+
+## How to write an integration test for a NOMAD example in a reader plugin

pyproject.toml

@@ -90,10 +90,12 @@ ellips = [
 raman = [
     "pynxtools-raman",
 ]
+
 [project.entry-points.'nomad.plugin']
 nexus_parser = "pynxtools.nomad.entrypoints:nexus_parser"
 nexus_schema = "pynxtools.nomad.entrypoints:nexus_schema"
 nexus_data_converter = "pynxtools.nomad.entrypoints:nexus_data_converter"
+iv_temp_example = "pynxtools.nomad.entrypoints:iv_temp_example"
 
 [project.scripts]
 read_nexus = "pynxtools.nexus.nexus:main"
@@ -126,7 +128,7 @@ select = [
     "E", # pycodestyle
     "W", # pycodestyle
     "PL", # pylint
-    "NPY201",
+    # "NPY201", # reactivate when np>2.0 is used
 ]
 ignore = [
     "E501", # Line too long ({width} > {limit} characters)

src/pynxtools/nomad/entrypoints.py

@@ -1,5 +1,26 @@
+#
+# Copyright The NOMAD Authors.
+#
+# This file is part of NOMAD. See https://nomad-lab.eu for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
 try:
-    from nomad.config.models.plugins import ParserEntryPoint, SchemaPackageEntryPoint
+    from nomad.config.models.plugins import (
+        ParserEntryPoint,
+        SchemaPackageEntryPoint,
+        ExampleUploadEntryPoint,
+    )
 except ImportError as exc:
     raise ImportError(
         "Could not import nomad package. Please install the package 'nomad-lab'."
@@ -43,3 +64,14 @@ def load(self):
     mainfile_name_re=r".*\.nxs",
     mainfile_mime_re="application/x-hdf5",
 )
+
+iv_temp_example = ExampleUploadEntryPoint(
+    title="Sensor Scan - IV Temperature Curve",
+    category="FAIRmat examples",
+    description="""
+        This example shows users how to take data from a Python framework and map it out to a Nexus application definition for IV Temperature measurements, [NXiv_temp](https://fairmat-experimental.github.io/nexus-fairmat-proposal/1c3806dba40111f36a16d0205cc39a5b7d52ca2e/classes/contributed_definitions/NXiv_temp.html#nxiv-temp).
+        We use the Nexus ELN features of Nomad to generate a Nexus file.
+    """,
+    path="nomad/examples/iv_temp",
+    local_path="examples/data/uploads/iv_temp.zip",
+)

src/pynxtools/nomad/examples/iv_temp/IV_temp.archive.json

@@ -0,0 +1,13 @@
+{
+    "data": {
+        "m_def": "../upload/raw/IV_temp.schema.archive.yaml#/definitions/section_definitions/0",
+        "reader": "json_map",
+        "nxdl": "NXiv_tempa",
+        "input_files": [
+            "IV_temp.mapping.json",
+            "IV_temp.pickle"
+        ],
+        "output": "IV_temp.nxs"
+    },
+    "m_ref_archives": {}
+}

src/pynxtools/nomad/examples/iv_temp/IV_temp.mapping.json

@@ -0,0 +1,46 @@
+{
+    "/@default": "entry",
+    "/ENTRY[entry]/@default": "data",
+    "/ENTRY[entry]/DATA[data]/current_295C": {"link": "/entry/instrument/environment/current_sensor/value", "shape": "0:21"},
+    "/ENTRY[entry]/DATA[data]/current_300C": {"link": "/entry/instrument/environment/current_sensor/value", "shape": "21:42"},
+    "/ENTRY[entry]/DATA[data]/current_305C": {"link": "/entry/instrument/environment/current_sensor/value", "shape": "42:63"},
+    "/ENTRY[entry]/DATA[data]/current_310C": {"link": "/entry/instrument/environment/current_sensor/value", "shape": "63:84"},
+    "/ENTRY[entry]/DATA[data]/voltage": {"link": "/entry/instrument/environment/voltage_controller/value"},
+    "/ENTRY[entry]/DATA[data]/current": "/currents_stack",
+    "/ENTRY[entry]/DATA[data]/temperature": {"link": "/entry/instrument/environment/temperature_controller/value"},
+    "/ENTRY[entry]/DATA[data]/@axes": ["voltage"],
+    "/ENTRY[entry]/DATA[data]/@signal": "current_295C",
+    "/ENTRY[entry]/DATA[data]/@auxiliary_signals": ["current_300C", "current_305C", "current_310C"],
+    "/ENTRY[entry]/INSTRUMENT[instrument]/ENVIRONMENT[environment]/voltage_controller/calibration_time": "/metadata_start/time",
+    "/ENTRY[entry]/INSTRUMENT[instrument]/ENVIRONMENT[environment]/voltage_controller/run_control": 0,
+    "/ENTRY[entry]/INSTRUMENT[instrument]/ENVIRONMENT[environment]/voltage_controller/value": "/voltages",
+    "/ENTRY[entry]/INSTRUMENT[instrument]/ENVIRONMENT[environment]/voltage_controller/value/@units": "V",
+    "/ENTRY[entry]/INSTRUMENT[instrument]/ENVIRONMENT[environment]/temperature_controller/calibration_time": "/metadata_start/time",
+    "/ENTRY[entry]/INSTRUMENT[instrument]/ENVIRONMENT[environment]/temperature_controller/run_control": 0,
+    "/ENTRY[entry]/INSTRUMENT[instrument]/ENVIRONMENT[environment]/temperature_controller/value": "/temperatures",
+    "/ENTRY[entry]/INSTRUMENT[instrument]/ENVIRONMENT[environment]/temperature_controller/value/@units": "°C",
+    "/ENTRY[entry]/INSTRUMENT[instrument]/ENVIRONMENT[environment]/current_sensor/calibration_time": "/metadata_start/time",
+    "/ENTRY[entry]/INSTRUMENT[instrument]/ENVIRONMENT[environment]/current_sensor/run_control": 0,
+    "/ENTRY[entry]/INSTRUMENT[instrument]/ENVIRONMENT[environment]/current_sensor/value": "/currents",
+    "/ENTRY[entry]/INSTRUMENT[instrument]/ENVIRONMENT[environment]/current_sensor/value/@units": "A",
+    "/ENTRY[entry]/INSTRUMENT[instrument]/ENVIRONMENT[environment]/independent_controllers": ["voltage_controller", "temperature_control"],
+    "/ENTRY[entry]/INSTRUMENT[instrument]/ENVIRONMENT[environment]/measurement_sensors": ["current_sensor"],
+    "/ENTRY[entry]/INSTRUMENT[instrument]/ENVIRONMENT[environment]/NXpid[heating_pid]/description": "The heating current is controlled by a virtual PID device (internally in EPICS). The input value for the PID is the measured temperature of the sample. The output of the PID is the heating current.",
+    "/ENTRY[entry]/INSTRUMENT[instrument]/ENVIRONMENT[environment]/NXpid[heating_pid]/pv_sensor/value_log/value": {"link": "/entry/instrument/environment/current_sensor/value"},
+    "/ENTRY[entry]/INSTRUMENT[instrument]/ENVIRONMENT[environment]/NXpid[heating_pid]/setpoint": {"link": "/entry/instrument/environment/temperature_controller/value"},
+    "/ENTRY[entry]/INSTRUMENT[instrument]/ENVIRONMENT[environment]/NXpid[heating_pid]/K_p_value": "/metadata_start/device_config/PID_controller/PID_controller_pid_kp/value",
+    "/ENTRY[entry]/INSTRUMENT[instrument]/ENVIRONMENT[environment]/NXpid[heating_pid]/K_i_value": "/metadata_start/device_config/PID_controller/PID_controller_pid_ki/value",
+    "/ENTRY[entry]/INSTRUMENT[instrument]/ENVIRONMENT[environment]/NXpid[heating_pid]/K_d_value": "/metadata_start/device_config/PID_controller/PID_controller_pid_kd/value",
+    "/ENTRY[entry]/PROCESS[process]/program": "Bluesky",
+    "/ENTRY[entry]/PROCESS[process]/program/@version": "/metadata_start/versions/bluesky",
+    "/ENTRY[entry]/SAMPLE[sample]/name": "/metadata_start/sample/Name",
+    "/ENTRY[entry]/SAMPLE[sample]/atom_types": "Si, C",
+    "/ENTRY[entry]/USER[user]/email": "/metadata_start/user/E-Mail",
+    "/ENTRY[entry]/USER[user]/name": "/metadata_start/user/Name",
+    "/ENTRY[entry]/definition": "NXiv_temp",
+    "/ENTRY[entry]/definition/@version": "1",
+    "/ENTRY[entry]/experiment_identifier": "/metadata_start/uid",
+    "/ENTRY[entry]/experiment_description": "A simple IV temperature experiment.",
+    "/ENTRY[entry]/start_time": "/metadata_start/time"
+
+}

src/pynxtools/nomad/examples/iv_temp/IV_temp.schema.archive.yaml

@@ -0,0 +1,18 @@
+definitions:
+  name: "IV Temp ELN Example"
+  sections:
+    IV_TEMP:
+      base_sections:
+        - "pynxtools.nomad.dataconverter.NexusDataConverter"
+        - "nomad.datamodel.data.EntryData"
+      m_annotations:
+        template:
+          reader: json_map
+          nxdl: NXiv_temp
+          input_files": [
+            "IV_temp.mapping.json",
+            "IV_temp.pickle"
+          ]
+          output: IV_temp.nxs
+        eln:
+          hide: []

src/pynxtools/nomad/examples/iv_temp/README.md

@@ -0,0 +1,30 @@
+
+
+# Introduction
+
+This is an example of a PID controlled sensor sweep scan. The temperature is set using a PID controller. Then for the set temperature a voltage sweep is performed. For each of the voltages, the current is measured. This is repeated for a given list of temperatures.
+
+This specific data was capured using a Bluesky controlled system. This was then saved into a binary file using Pickle. This example shows how such a dataset could be converted using the JSONMapReader of the NexusParser.
+The data is mapped on to a Nexus application definition for IV Temperature measurements, [NXiv_temp](https://fairmat-experimental.github.io/nexus-fairmat-proposal/50433d9039b3f33299bab338998acb5335cd8951/classes/contributed_definitions/NXiv_temp.html#nxiv-temp).
+
+# Viewing uploaded data
+
+Below, you find an overview of your uploaded data.
+Click on the `> /` button to get a list of your data or select **FILES** from the top menu of this upload.
+You may add your own files to the upload or experiment with the pre-existing electronic lab book example.
+The ELN follows the general structure of NOMAD ELN templates and you may refer to the [documentation](https://nomad-lab.eu/prod/v1/staging/docs/archive.html) or a [YouTube tutorial](https://youtu.be/o5ETHmGmnaI) (~1h)
+for further information.
+When the ELN is saved a NeXus file will be generated from the provided example data.
+You may also view your supplied or generated NeXus files here with the H5Web viewer.
+To do so open the **FILES** tab and just select a `.nxs` file.
+
+# Using a Jupyter Notebook
+
+This example comes with a very simple Jupyter Notebook that shows how one could easily get access to a Python environment with access to all your data in one place.
+To give this a go, click the **FILES** tab and select `iv_temp.ipynb`. Feel free to modify this or just create a new one to try!
+
+# Where to go from here?
+
+If you're interested in using this pipeline and NOMAD in general you'll find support at [FAIRmat](https://www.fairmat-nfdi.eu/fairmat/consortium).
+
+If you have any questions about this example you may contact [Sherjeel Shabih](https://www.fairmat-nfdi.eu/fairmat/fairmat_/fairmatteam) from the FAIRmat consortium.

src/pynxtools/testing/nexus_conversion.py

@@ -1,3 +1,20 @@
+#
+# Copyright The NOMAD Authors.
+#
+# This file is part of NOMAD. See https://nomad-lab.eu for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
 """Generic test for reader plugins."""
 
 import logging

src/pynxtools/testing/nomad_example.py

@@ -0,0 +1,111 @@
+#
+# Copyright The NOMAD Authors.
+#
+# This file is part of NOMAD. See https://nomad-lab.eu for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Test for NOMAD examples in reader plugins."""
+
+import os
+from typing import Dict, Any
+import pytest
+
+try:
+    from nomad.parsing.parser import ArchiveParser
+    from nomad.datamodel import EntryArchive, Context
+
+    from nomad.config.models.plugins import (
+        ExampleUploadEntryPoint,
+    )
+except ImportError:
+    pytest.skip(
+        "Skipping NOMAD example tests because nomad is not installed",
+        allow_module_level=True,
+    )
+
+
+def get_file_parameter(example_path: str):
+    """
+    Get all example files for the plugin.
+
+    This function searches for specific example files in the given directory path.
+
+    Args:
+        example_path (str): Path to the directory containing example files.
+
+    Yields:
+        pytest.param: A pytest parameter object with the file path and file ID.
+    """
+    example_files = (
+        "schema.archive.yaml",
+        "schema.archive.yml",
+        "scheme.archive.yaml",
+        "scheme.archive.yml",
+        "schema.archive.json",
+        "scheme.archive.json",
+        "intra-entry.archive.json",
+    )
+
+    # Check if the provided path exists
+    if not os.path.exists(example_path):
+        raise FileNotFoundError(f"The directory '{example_path}' does not exist.")
+
+    # Walk through the specified directory
+    for root, _, files in os.walk(example_path):
+        for file in files:
+            normalized_file = file.lower()  # Normalize to lower case
+            if os.path.basename(normalized_file).endswith(example_files):
+                yield pytest.param(os.path.join(root, file), id=file)
+
+
+def parse_nomad_examples(mainfile: str) -> Dict[str, Any]:
+    """Parse a NOMAD example file and return its dictionary representation.
+
+    Args:
+        mainfile (str): The path to the NOMAD example file to be parsed.
+
+    Returns:
+        Dict[str, Any]: A dictionary representation of the parsed NOMAD example.
+
+    Raises:
+        FileNotFoundError: If the mainfile does not exist.
+    """
+    if not os.path.exists(mainfile):
+        raise FileNotFoundError(f"The specified file '{mainfile}' does not exist.")
+
+    archive = EntryArchive()
+    archive.m_context = Context()
+
+    ArchiveParser().parse(mainfile, archive)
+    return archive.m_to_dict()
+
+
+def example_upload_entry_point_valid(
+    entrypoint, plugin_package, expected_local_path
+) -> None:
+    """
+    Test if NOMAD ExampleUploadEntryPoint works.
+
+    Args:
+        entrypoint (nomad.config.models.plugins.xampleUploadEntryPoint): The entry point to test.
+        plugin_package (str): The plugin package to set on the entry point.
+        expected_local_path (str): The expected local path after loading.
+
+    """
+    setattr(entrypoint, "plugin_package", plugin_package)
+    entrypoint.load()
+    assert entrypoint.local_path == expected_local_path, (
+        f"Expected local path '{expected_local_path}', "
+        f"but got '{entrypoint.local_path}'"
+    )