tmp

include/openPMD/IO/AbstractIOHandlerImplCommon.hpp

@@ -53,21 +53,24 @@ class AbstractIOHandlerImplCommon : public AbstractIOHandlerImpl
      * map each Writable to its associated file contains only the filename,
      * without the OS path
      */
-    std::unordered_set<internal::SharedFileState> m_dirty;
+    std::unordered_set<internal::SharedFileState, internal::HashSharedFileState>
+        m_dirty;
     std::unordered_map<std::string, internal::SharedFileState> m_files;
 
     internal::SharedFileState &
     makeFile(Writable *, std::string fileName, bool consider_open_files);
 
-    void associateWithFile(Writable *writable, internal::SharedFileState file);
+    void associateWithFile(Writable *writable, internal::SharedFileState &file);
+
+    void setDirty(Writable *);
 
     /**
      *
      * @return Full OS path of the file.
      */
     std::string fullPath(internal::FileState const &);
 
-    std::string fullPath(std::string);
+    std::string fullPath(std::string const &);
 
     /**
      * Get the writable's containing file.
@@ -135,16 +138,16 @@ AbstractIOHandlerImplCommon<IOHandler_t, FilePositionType>::makeFile(
     Writable *writable, std::string file, bool consider_open_files)
 {
     auto make_new = [&]() {
-        writable->fileState =
-            std::make_shared<std::optional<internal::FileState>>(file);
-        m_files[std::move(file)] = writable->fileState;
+        new (&writable->fileState)
+            internal::SharedFileState(std::in_place, file);
+        m_files[std::move(file)].derive_from(writable->fileState);
     };
     if (consider_open_files)
     {
         if (auto it = m_files.find(file);
             it != m_files.end() && it->second->has_value())
         {
-            writable->fileState = it->second;
+            writable->fileState.derive_from(it->second);
         }
         else
         {
@@ -160,11 +163,18 @@ AbstractIOHandlerImplCommon<IOHandler_t, FilePositionType>::makeFile(
     return writable->fileState;
 }
 
+template <typename IOHandler_t, typename FilePositionType>
+void AbstractIOHandlerImplCommon<IOHandler_t, FilePositionType>::setDirty(
+    Writable *writable)
+{
+    m_dirty.emplace(&*writable->fileState);
+}
+
 template <typename IOHandler_t, typename FilePositionType>
 void AbstractIOHandlerImplCommon<IOHandler_t, FilePositionType>::
-    associateWithFile(Writable *writable, internal::SharedFileState file)
+    associateWithFile(Writable *writable, internal::SharedFileState &file)
 {
-    writable->fileState = std::move(file);
+    writable->fileState.derive_from(file);
 }
 
 template <typename IOHandler_t, typename FilePositionType>
@@ -178,7 +188,7 @@ AbstractIOHandlerImplCommon<IOHandler_t, FilePositionType>::fullPath(
 template <typename IOHandler_t, typename FilePositionType>
 std::string
 AbstractIOHandlerImplCommon<IOHandler_t, FilePositionType>::fullPath(
-    std::string fileName)
+    std::string const &fileName)
 {
     if (auxiliary::ends_with(m_handler->directory, "/"))
     {
@@ -215,7 +225,7 @@ AbstractIOHandlerImplCommon<IOHandler_t, FilePositionType>::
         search = writable->parent;
         while (!search->fileState->has_value())
         {
-            search->fileState = file;
+            search->fileState.derive_from(file);
             search = search->parent;
         }
         associateWithFile(writable, file);

include/openPMD/IO/InvalidatableFile.hpp

@@ -30,10 +30,14 @@
 namespace openPMD::internal
 {
 
+template <typename T>
+struct HashMaybeOwning;
+
 template <typename T>
 struct MaybeOwning : std::variant<T, T *>
 {
     using variant_t = std::variant<T, T *>;
+    using hash_t = HashMaybeOwning<T>;
 
     explicit MaybeOwning();
     template <typename... Args>
@@ -42,6 +46,8 @@ struct MaybeOwning : std::variant<T, T *>
 
     operator bool() const;
 
+    auto operator==(MaybeOwning const &other) const -> bool;
+
     auto as_base() -> variant_t &;
     auto as_base() const -> variant_t const &;
 
@@ -51,7 +57,7 @@ struct MaybeOwning : std::variant<T, T *>
     auto operator->() -> T *;
     auto operator->() const -> T const *;
 
-    auto derive_nonowning() -> MaybeOwning<T>;
+    auto derive_from(MaybeOwning &other) -> MaybeOwning &;
 };
 
 struct FileState
@@ -67,8 +73,25 @@ struct FileState
     FileState &operator=(FileState const &other) = delete;
     FileState &operator=(FileState &&other) = delete;
 };
-using SharedFileState = std::shared_ptr<std::optional<FileState>>;
-// using SharedFileState = MaybeOwning<std::optional<FileState>>;
+
+struct MaybeFileState : std::optional<FileState>
+{
+    // template <typename... Args>
+    // MaybeFileState(Args &&...args)
+    //     : std::optional<FileState>(std::forward<Args>(args)...)
+    // {}
+    using parent_t = std::optional<FileState>;
+    using parent_t::parent_t;
+
+    auto operator==(MaybeFileState const &other) const -> bool;
+};
+
+using SharedFileState = MaybeOwning<MaybeFileState>;
+
+struct HashSharedFileState
+{
+    auto operator()(SharedFileState const &) const -> size_t;
+};
 
 template <typename T>
 template <typename... Args>

src/IO/ADIOS/ADIOS2IOHandler.cpp

@@ -584,7 +584,7 @@ ADIOS2IOHandlerImpl::flush(internal::ParsedFlushParams &flushParams)
     {
         if (p->has_value() && (*p)->backendSpecificState.has_value())
         {
-            auto &adios2_file = std::any_cast<BackendSpecificFileState &>(
+            auto &adios2_file = std::any_cast<BackendSpecificFileState const &>(
                 (*p)->backendSpecificState);
             adios2_file->flush(adios2FlushParams, /* writeLatePuts = */ false);
         }
@@ -665,7 +665,7 @@ void ADIOS2IOHandlerImpl::createFile(
         }
 
         auto &fileData = getFileData(file_state, IfFileNotOpen::OpenImplicitly);
-        this->m_dirty.emplace(writable->fileState);
+        this->setDirty(writable);
 
         writable->written = true;
         writable->abstractFilePosition = std::make_shared<ADIOS2FilePosition>();
@@ -877,7 +877,7 @@ For further details see
             parameters.dtype, fileData.m_IO, varName, operators, shape);
         fileData.invalidateVariablesMap();
         writable->written = true;
-        m_dirty.emplace(writable->fileState);
+        setDirty(writable);
     }
 }
 
@@ -956,7 +956,7 @@ void ADIOS2IOHandlerImpl::openFile(
     // lazy opening is deathly in parallel situations
     auto &fileData = getFileData(**file, IfFileNotOpen::OpenImplicitly);
     *parameters.out_parsePreference = fileData.parsePreference;
-    m_dirty.emplace(file);
+    setDirty(writable);
 }
 
 void ADIOS2IOHandlerImpl::closeFile(
@@ -989,7 +989,7 @@ void ADIOS2IOHandlerImpl::closeFile(
         /* writeLatePuts = */ true,
         /* flushUnconditionally = */ false);
     file.backendSpecificState.reset();
-    *maybe_file = std::nullopt;
+    maybe_file->reset();
 }
 
 void ADIOS2IOHandlerImpl::openPath(
@@ -1077,7 +1077,7 @@ void ADIOS2IOHandlerImpl::writeDataset(
     bp.name = nameOfVariable(writable);
     bp.param = std::move(parameters);
     ba.enqueue(std::move(bp));
-    m_dirty.emplace(writable->fileState);
+    setDirty(writable);
     writable->written = true; // TODO erst nach dem Schreiben?
 }
 
@@ -1129,7 +1129,7 @@ void ADIOS2IOHandlerImpl::readDataset(
     bg.name = nameOfVariable(writable);
     bg.param = parameters;
     ba.enqueue(std::move(bg));
-    m_dirty.emplace(writable->fileState);
+    setDirty(writable);
 }
 
 namespace detail
@@ -1561,7 +1561,7 @@ void ADIOS2IOHandlerImpl::touch(
     Writable *writable, Parameter<Operation::TOUCH> const &)
 {
     refreshFileFromParent(writable, false);
-    this->m_dirty.emplace(writable->fileState);
+    this->setDirty(writable);
 }
 
 adios2::Mode ADIOS2IOHandlerImpl::adios2AccessMode(std::string const &fullPath)
@@ -1884,7 +1884,7 @@ namespace detail
             file, ADIOS2IOHandlerImpl::IfFileNotOpen::ThrowError);
         filedata.invalidateAttributesMap();
         adios2::IO IO = filedata.m_IO;
-        impl->m_dirty.emplace(writable->fileState);
+        impl->setDirty(writable);
 
 #if openPMD_HAS_ADIOS_2_9
         if (impl->m_modifiableAttributes ==

src/IO/InvalidatableFile.cpp

@@ -22,6 +22,9 @@
 #include "openPMD/IO/InvalidatableFile.hpp"
 #include "openPMD/auxiliary/Variant.hpp"
 
+#include <iostream>
+#include <stdexcept>
+
 namespace openPMD::internal
 {
 
@@ -43,6 +46,12 @@ MaybeOwning<T>::operator bool() const
         as_base());
 }
 
+template <typename T>
+auto MaybeOwning<T>::operator==(MaybeOwning<T> const &other) const -> bool
+{
+    return (**this).operator==(*other);
+}
+
 template <typename T>
 auto MaybeOwning<T>::as_base() -> variant_t &
 {
@@ -85,13 +94,30 @@ auto MaybeOwning<T>::operator->() const -> T const *
 }
 
 template <typename T>
-auto MaybeOwning<T>::derive_nonowning() -> MaybeOwning<T>
+auto MaybeOwning<T>::derive_from(MaybeOwning<T> &other) -> MaybeOwning<T> &
 {
-    return MaybeOwning<T> { &operator*() };
+    auto other_ptr = &*other;
+    auto myself = &**this;
+    if (myself != other_ptr)
+    {
+        as_base().template emplace<T *>(other_ptr);
+    }
+    return *this;
 }
 
-template struct MaybeOwning<std::optional<FileState>>;
+template struct MaybeOwning<MaybeFileState>;
 
 FileState::FileState(std::string name_in) : name(std::move(name_in))
 {}
+
+auto MaybeFileState::operator==(MaybeFileState const &other) const -> bool
+{
+    return this == &other;
+}
+
+auto HashSharedFileState::operator()(SharedFileState const &val) const -> size_t
+{
+    constexpr std::hash<MaybeFileState const *> hash;
+    return hash(&*val);
+}
 } // namespace openPMD::internal

src/IO/JSON/JSONIOHandlerImpl.cpp

@@ -461,10 +461,14 @@ JSONIOHandlerImpl::~JSONIOHandlerImpl() = default;
 std::future<void> JSONIOHandlerImpl::flush()
 {
     AbstractIOHandlerImpl::flush();
-    for (auto const &file : m_dirty)
+    for (auto it = m_dirty.begin(); it != m_dirty.end(); it = m_dirty.begin())
     {
+        auto node = m_dirty.extract(it);
+        auto &file = node.value();
         if (file->has_value())
+        {
             putJsonContents(**file);
+        }
     }
     m_dirty.clear();
     return std::future<void>();
@@ -497,7 +501,8 @@ void JSONIOHandlerImpl::createFile(
     {
         std::string name = parameters.name + m_originalExtension;
 
-        auto file = makeFile(writable, name, /* consider_open_files = */ false);
+        auto &file =
+            makeFile(writable, name, /* consider_open_files = */ false);
         auto &file_state = **file;
         auto file_exists = auxiliary::file_exists(fullPath(file_state));
 
@@ -515,7 +520,7 @@ void JSONIOHandlerImpl::createFile(
             VERIFY(success, "[JSON] Could not create directory.");
         }
 
-        this->m_dirty.emplace(writable->fileState);
+        this->setDirty(writable);
 
         if (m_handler->m_backendAccess != Access::APPEND || !file_exists)
         {
@@ -577,7 +582,7 @@ void JSONIOHandlerImpl::createPath(
         writable->abstractFilePosition = std::move(new_filepos);
     }
     ensurePath(jsonVal, filepos->id.to_string());
-    m_dirty.emplace(writable->fileState);
+    setDirty(writable);
     writable->written = true;
 }
 
@@ -661,7 +666,7 @@ void JSONIOHandlerImpl::createDataset(
             break;
         }
         writable->written = true;
-        m_dirty.emplace(writable->fileState);
+        setDirty(writable);
     }
 }
 
@@ -991,19 +996,15 @@ void JSONIOHandlerImpl::closeFile(
     Writable *writable, Parameter<Operation::CLOSE_FILE> const &)
 {
     auto &maybe_file = writable->fileState;
-    if (!maybe_file)
+    if (!maybe_file || !maybe_file->has_value())
     {
         return;
     }
-    else if (!maybe_file->has_value())
-    {
-        *maybe_file = std::nullopt;
-    }
     auto &file = **maybe_file;
     putJsonContents(file);
     m_dirty.erase(maybe_file);
     m_files.erase(file.name);
-    *maybe_file = std::nullopt;
+    maybe_file->reset();
 }
 
 void JSONIOHandlerImpl::openPath(
@@ -1280,7 +1281,7 @@ void JSONIOHandlerImpl::writeAttribute(
         break;
     }
     writable->written = true;
-    m_dirty.emplace(writable->fileState);
+    setDirty(writable);
 }
 
 namespace
@@ -1638,7 +1639,7 @@ void JSONIOHandlerImpl::touch(
     Writable *writable, Parameter<Operation::TOUCH> const &)
 {
     refreshFileFromParent(writable, false);
-    this->m_dirty.emplace(writable->fileState);
+    this->setDirty(writable);
 }
 
 auto JSONIOHandlerImpl::getFilehandle(