Refactoring for variable-based encoding: Use same backend object for Iterations container and the Iterations themselves

include/openPMD/Iteration.hpp

@@ -136,6 +136,13 @@ namespace internal
         std::optional<DeferredParseAccess> m_deferredParseAccess{};
     };
 } // namespace internal
+
+namespace traits
+{
+    template <>
+    struct GenerationPolicy<Iteration>;
+}
+
 /** @brief  Logical compilation of data from one snapshot (e.g. a single
  * simulation cycle).
  *
@@ -153,6 +160,7 @@ class Iteration : public Attributable
     friend class Writable;
     friend class StatefulIterator;
     friend class StatefulSnapshotsContainer;
+    friend struct traits::GenerationPolicy<Iteration>;
 
 public:
     Iteration(Iteration const &) = default;
@@ -303,7 +311,7 @@ class Iteration : public Attributable
     void flushFileBased(
         std::string const &, IterationIndex_t, internal::FlushParams const &);
     void flushGroupBased(IterationIndex_t, internal::FlushParams const &);
-    void flushVariableBased(IterationIndex_t, internal::FlushParams const &);
+    void flushVariableBased(internal::FlushParams const &);
     void flush(internal::FlushParams const &);
     void deferParseAccess(internal::DeferredParseAccess);
     /*
@@ -419,9 +427,10 @@ class Iteration : public Attributable
     /**
      * @brief Link with parent.
      *
-     * @param w The Writable representing the parent.
+     * @param parent The Writable representing the parent.
      */
-    virtual void linkHierarchy(Writable &w);
+    void linkHierarchy(internal::AttributableData &parent) override;
+    using Attributable::linkHierarchy;
 
     /**
      * @brief Access an iteration in read mode that has potentially not been
@@ -478,4 +487,36 @@ class IndexedIteration : public Iteration
         : Iteration(std::forward<Iteration_t>(it)), iterationIndex(index)
     {}
 };
+
+namespace traits
+{
+    template <>
+    struct GenerationPolicy<Iteration>
+    {
+        constexpr static bool is_noop = false;
+        template <typename T, typename Container>
+        void operator()(T &ret, Container *c)
+        {
+            if (ret.IOHandler()->m_encoding == IterationEncoding::variableBased)
+            {
+                for (auto &pair :
+                     static_cast<Attributable &>(ret).get().m_attributes)
+                {
+                    static_cast<Attributable &>(*c).get().m_attributes.emplace(
+                        std::move(pair));
+                }
+                static_cast<
+                    std::shared_ptr<internal::SharedAttributableData> &>(
+                    *ret.m_attri) =
+                    static_cast<
+                        std::shared_ptr<internal::SharedAttributableData> &>(
+                        *c->m_attri);
+                internal::AttributableData *attr_of_shared_parent =
+                    c->m_attri->frontend_parent;
+                ret.linkHierarchy(*attr_of_shared_parent);
+                ret.m_attri->frontend_parent = c->m_attri.get();
+            }
+        }
+    };
+} // namespace traits
 } // namespace openPMD

include/openPMD/ParticleSpecies.hpp

@@ -61,10 +61,10 @@ namespace traits
     struct GenerationPolicy<ParticleSpecies>
     {
         constexpr static bool is_noop = false;
-        template <typename T>
-        void operator()(T &ret)
+        template <typename T, typename C>
+        void operator()(T &ret, C const *)
         {
-            ret.particlePatches.linkHierarchy(ret.writable());
+            ret.particlePatches.linkHierarchy(ret);
         }
     };
 } // namespace traits

include/openPMD/backend/Attributable.hpp

@@ -155,6 +155,33 @@ namespace internal
                 std::shared_ptr<typename T::Data_t>(self, [](auto const *) {}));
             return res;
         }
+
+        internal::AttributableData *frontend_parent = nullptr;
+
+        /** Tracks if there are unwritten changes for this specific Writable.
+         *
+         * Manipulate via Attributable::dirty() and Attributable::setDirty().
+         */
+        bool dirtySelf = true;
+        /**
+         * Tracks if there are unwritten changes anywhere in the
+         * tree whose ancestor this Writable is.
+         *
+         * Invariant: this->dirtyRecursive implies parent->dirtyRecursive.
+         *
+         * dirtySelf and dirtyRecursive are separated since that allows
+         * specifying that `this` is not dirty, but some child is.
+         *
+         * Manipulate via Attributable::dirtyRecursive() and
+         * Attributable::setDirtyRecursive().
+         */
+        bool dirtyRecursive = true;
+
+        /**
+         * If frontend_parent is not null, then this is a key such that:
+         * &(*frontend_parent)[key] == this
+         */
+        std::string ownKeyWithinParent;
     };
 
     template <typename, typename>
@@ -510,18 +537,17 @@ OPENPMD_protected
 
     bool dirty() const
     {
-        return writable().dirtySelf;
+        return m_attri->dirtySelf;
     }
     /** O(1).
      */
     bool dirtyRecursive() const
     {
-        return writable().dirtyRecursive;
+        return m_attri->dirtyRecursive;
     }
     void setDirty(bool dirty_in)
     {
-        auto &w = writable();
-        w.dirtySelf = dirty_in;
+        m_attri->dirtySelf = dirty_in;
         setDirtyRecursive(dirty_in);
     }
     /* Amortized O(1) if dirty_in is true, else O(1).
@@ -542,15 +568,15 @@ OPENPMD_protected
      */
     void setDirtyRecursive(bool dirty_in)
     {
-        auto &w = writable();
-        w.dirtyRecursive = dirty_in;
+        auto &a = *m_attri;
+        a.dirtyRecursive = dirty_in;
         if (dirty_in)
         {
-            auto current = w.parent;
+            auto current = a.frontend_parent;
             while (current && !current->dirtyRecursive)
             {
                 current->dirtyRecursive = true;
-                current = current->parent;
+                current = current->frontend_parent;
             }
         }
     }
@@ -579,9 +605,10 @@ OPENPMD_protected
     /**
      * @brief Link with parent.
      *
-     * @param w The Writable representing the parent.
+     * @param parent The Writable representing the parent.
      */
-    virtual void linkHierarchy(Writable &w);
+    void linkHierarchy(Attributable &parent);
+    virtual void linkHierarchy(internal::AttributableData &parent);
 }; // Attributable
 
 // note: we explicitly instantiate Attributable::setAttributeImpl for all T in

include/openPMD/backend/Container.hpp

@@ -51,8 +51,8 @@ namespace traits
     struct GenerationPolicy
     {
         constexpr static bool is_noop = true;
-        template <typename T>
-        void operator()(T &)
+        template <typename... Args>
+        void operator()(Args &&...)
         {}
     };
 } // namespace traits

include/openPMD/backend/ContainerImpl.tpp

@@ -139,18 +139,18 @@ auto Container<T, T_key, T_container>::operator[](key_type const &key)
         }
 
         T t = T();
-        t.linkHierarchy(writable());
+        t.linkHierarchy(*this);
         auto &ret = container().insert({key, std::move(t)}).first->second;
         if constexpr (std::is_same_v<T_key, std::string>)
         {
-            ret.writable().ownKeyWithinParent = key;
+            ret.m_attri->ownKeyWithinParent = key;
         }
         else
         {
-            ret.writable().ownKeyWithinParent = std::to_string(key);
+            ret.m_attri->ownKeyWithinParent = std::to_string(key);
         }
         traits::GenerationPolicy<T> gen;
-        gen(ret);
+        gen(ret, this);
         return ret;
     }
 }
@@ -171,18 +171,18 @@ auto Container<T, T_key, T_container>::operator[](key_type &&key)
         }
 
         T t = T();
-        t.linkHierarchy(writable());
+        t.linkHierarchy(*this);
         auto &ret = container().insert({key, std::move(t)}).first->second;
         if constexpr (std::is_same_v<T_key, std::string>)
         {
-            ret.writable().ownKeyWithinParent = std::move(key);
+            ret.m_attri->ownKeyWithinParent = std::move(key);
         }
         else
         {
-            ret.writable().ownKeyWithinParent = std::to_string(std::move(key));
+            ret.m_attri->ownKeyWithinParent = std::to_string(std::move(key));
         }
         traits::GenerationPolicy<T> gen;
-        gen(ret);
+        gen(ret, this);
         return ret;
     }
 }

include/openPMD/backend/Writable.hpp

@@ -62,6 +62,16 @@ namespace debug
     void printDirty(Series const &);
 }
 
+class Iteration;
+
+namespace traits
+{
+    template <typename>
+    struct GenerationPolicy;
+    template <>
+    struct GenerationPolicy<Iteration>;
+} // namespace traits
+
 /** @brief Layer to mirror structure of logical data and persistent data in
  * file.
  *
@@ -105,6 +115,7 @@ class Writable final
     friend void debug::printDirty(Series const &);
     friend struct Parameter<Operation::CREATE_DATASET>;
     friend struct Parameter<Operation::OPEN_DATASET>;
+    friend struct traits::GenerationPolicy<Iteration>;
 
 private:
     Writable(internal::AttributableData *);
@@ -154,29 +165,6 @@ OPENPMD_private
     internal::AttributableData *attributable = nullptr;
     Writable *parent = nullptr;
 
-    /** Tracks if there are unwritten changes for this specific Writable.
-     *
-     * Manipulate via Attributable::dirty() and Attributable::setDirty().
-     */
-    bool dirtySelf = true;
-    /**
-     * Tracks if there are unwritten changes anywhere in the
-     * tree whose ancestor this Writable is.
-     *
-     * Invariant: this->dirtyRecursive implies parent->dirtyRecursive.
-     *
-     * dirtySelf and dirtyRecursive are separated since that allows specifying
-     * that `this` is not dirty, but some child is.
-     *
-     * Manipulate via Attributable::dirtyRecursive() and
-     * Attributable::setDirtyRecursive().
-     */
-    bool dirtyRecursive = true;
-    /**
-     * If parent is not null, then this is a key such that:
-     * &(*parent)[key] == this
-     */
-    std::string ownKeyWithinParent;
     /**
      * @brief Whether a Writable has been written to the backend.
      *

src/IO/AbstractIOHandlerImpl.cpp

@@ -21,10 +21,12 @@
 
 #include "openPMD/IO/AbstractIOHandlerImpl.hpp"
 
+#include "openPMD/Datatype_internal.hpp"
 #include "openPMD/IO/IOTask.hpp"
 #include "openPMD/Streaming.hpp"
 #include "openPMD/auxiliary/Environment.hpp"
 #include "openPMD/auxiliary/StringManip.hpp"
+#include "openPMD/auxiliary/TypeTraits.hpp"
 #include "openPMD/auxiliary/Variant.hpp"
 #include "openPMD/backend/Variant_internal.hpp"
 #include "openPMD/backend/Writable.hpp"
@@ -150,7 +152,9 @@ std::future<void> AbstractIOHandlerImpl::flush()
                     "] CREATE_DATASET: ",
                     parameter.name,
                     ", extent=",
-                    [&parameter]() { return vec_as_string(parameter.extent); });
+                    [&parameter]() {
+                        return auxiliary::vec_as_string(parameter.extent);
+                    });
                 createDataset(i.writable, parameter);
                 break;
             }
@@ -287,7 +291,28 @@ std::future<void> AbstractIOHandlerImpl::flush()
                     "] WRITE_ATT: (",
                     parameter.dtype,
                     ") ",
-                    parameter.name);
+                    parameter.name,
+                    "=",
+                    [&]() {
+                        return std::visit(
+                            [&](auto const &val) {
+                                using dtype = std::remove_cv_t<
+                                    std::remove_reference_t<decltype(val)>>;
+                                if constexpr (
+                                    auxiliary::IsArray_v<dtype> ||
+                                    auxiliary::IsVector_v<dtype>)
+                                {
+                                    return auxiliary::vec_as_string(val);
+                                }
+                                else
+                                {
+                                    std::stringstream res;
+                                    res << val;
+                                    return res.str();
+                                }
+                            },
+                            parameter.resource<attribute_types>());
+                    });
                 writeAttribute(i.writable, parameter);
                 break;
             }
@@ -301,9 +326,13 @@ std::future<void> AbstractIOHandlerImpl::flush()
                     "->",
                     i.writable,
                     "] READ_DATASET, offset=",
-                    [&parameter]() { return vec_as_string(parameter.offset); },
+                    [&parameter]() {
+                        return auxiliary::vec_as_string(parameter.offset);
+                    },
                     ", extent=",
-                    [&parameter]() { return vec_as_string(parameter.extent); });
+                    [&parameter]() {
+                        return auxiliary::vec_as_string(parameter.extent);
+                    });
                 readDataset(i.writable, parameter);
                 break;
             }