Make H5FL package threadsafe

c++/test/tfile.cpp

@@ -847,19 +847,6 @@ test_file_info()
     H5F_fspace_strategy_t out_strategy = H5F_FSPACE_STRATEGY_FSM_AGGR;
 
     try {
-        // Create a file using default properties.
-        H5File tempfile(FILE7, H5F_ACC_TRUNC);
-
-        // Get the file's version information.
-        H5F_info2_t finfo;
-        tempfile.getFileInfo(finfo);
-        verify_val(static_cast<long>(finfo.super.version), 0, "H5File::getFileInfo", __LINE__, __FILE__);
-        verify_val(static_cast<long>(finfo.free.version), 0, "H5File::getFileInfo", __LINE__, __FILE__);
-        verify_val(static_cast<long>(finfo.sohm.version), 0, "H5File::getFileInfo", __LINE__, __FILE__);
-
-        // Close the file.
-        tempfile.close();
-
         // Create file creation property list.
         FileCreatPropList fcpl;
 
@@ -877,80 +864,101 @@ test_file_info()
         verify_val(out_fsp_psize, FSP_SIZE_DEF, "FileCreatPropList::getFileSpacePagesize", __LINE__,
                    __FILE__);
 
-        // Set various file information.
-        fcpl.setUserblock(F2_USERBLOCK_SIZE);
-        fcpl.setSizes(F2_OFFSET_SIZE, F2_LENGTH_SIZE);
-        fcpl.setSymk(F2_SYM_INTERN_K, F2_SYM_LEAF_K);
-        fcpl.setIstorek(F2_ISTORE);
+        // Only test this for VFDs that operate on a single file
+        bool default_vfd_compatible = true;
+        h5_driver_is_default_vfd_compatible(H5P_DEFAULT, &default_vfd_compatible);
+
+        if (default_vfd_compatible) {
+            // Create a file using default properties.
+            H5File tempfile(FILE7, H5F_ACC_TRUNC);
 
-        hsize_t               threshold = 5;    // Free space section threshold to set
-        bool                  persist   = true; // Persist free-space to set
-        H5F_fspace_strategy_t strategy  = H5F_FSPACE_STRATEGY_PAGE;
+            // Get the file's version information.
+            H5F_info2_t finfo;
+            tempfile.getFileInfo(finfo);
+            verify_val(static_cast<long>(finfo.super.version), 0, "H5File::getFileInfo", __LINE__, __FILE__);
+            verify_val(static_cast<long>(finfo.free.version), 0, "H5File::getFileInfo", __LINE__, __FILE__);
+            verify_val(static_cast<long>(finfo.sohm.version), 0, "H5File::getFileInfo", __LINE__, __FILE__);
 
-        fcpl.setFileSpaceStrategy(strategy, persist, threshold);
-        fcpl.setFileSpacePagesize(FSP_SIZE512);
+            // Close the file.
+            tempfile.close();
 
-        // Creating a file with the non-default file creation property list
-        // should create a version 1 superblock
+            // Set various file information.
+            fcpl.setUserblock(F2_USERBLOCK_SIZE);
+            fcpl.setSizes(F2_OFFSET_SIZE, F2_LENGTH_SIZE);
+            fcpl.setSymk(F2_SYM_INTERN_K, F2_SYM_LEAF_K);
+            fcpl.setIstorek(F2_ISTORE);
 
-        // Create file with custom file creation property list.
-        H5File file7(FILE7, H5F_ACC_TRUNC, fcpl);
+            hsize_t               threshold = 5;    // Free space section threshold to set
+            hbool_t               persist   = true; // Persist free-space to set
+            H5F_fspace_strategy_t strategy  = H5F_FSPACE_STRATEGY_PAGE;
 
-        // Close the file creation property list.
-        fcpl.close();
+            fcpl.setFileSpaceStrategy(strategy, persist, threshold);
+            fcpl.setFileSpacePagesize(FSP_SIZE512);
 
-        // Get the file's version information.
-        file7.getFileInfo(finfo);
-        verify_val(static_cast<long>(finfo.super.version), 2, "H5File::getFileInfo", __LINE__, __FILE__);
-        verify_val(static_cast<long>(finfo.free.version), 0, "H5File::getFileInfo", __LINE__, __FILE__);
-        verify_val(static_cast<long>(finfo.sohm.version), 0, "H5File::getFileInfo", __LINE__, __FILE__);
+            // Creating a file with the non-default file creation property list
+            // should create a version 1 superblock
 
-        // Close the file.
-        file7.close();
+            // Create file with custom file creation property list.
+            H5File file7(FILE7, H5F_ACC_TRUNC, fcpl);
 
-        // Re-open the file.
-        file7.openFile(FILE7, H5F_ACC_RDONLY);
+            // Close the file creation property list.
+            fcpl.close();
 
-        // Get the file's creation property list.
-        FileCreatPropList fcpl2 = file7.getCreatePlist();
+            // Get the file's version information.
+            file7.getFileInfo(finfo);
+            verify_val(static_cast<long>(finfo.super.version), 2, "H5File::getFileInfo", __LINE__, __FILE__);
+            verify_val(static_cast<long>(finfo.free.version), 0, "H5File::getFileInfo", __LINE__, __FILE__);
+            verify_val(static_cast<long>(finfo.sohm.version), 0, "H5File::getFileInfo", __LINE__, __FILE__);
 
-        // Get the file's version information.
-        file7.getFileInfo(finfo);
-        verify_val(static_cast<long>(finfo.super.version), 2, "H5File::getFileInfo", __LINE__, __FILE__);
-        verify_val(static_cast<long>(finfo.free.version), 0, "H5File::getFileInfo", __LINE__, __FILE__);
-        verify_val(static_cast<long>(finfo.sohm.version), 0, "H5File::getFileInfo", __LINE__, __FILE__);
+            // Close the file.
+            file7.close();
 
-        // Retrieve the property values & check them.
-        hsize_t userblock = fcpl2.getUserblock();
-        verify_val(userblock, F2_USERBLOCK_SIZE, "FileCreatPropList::getUserblock", __LINE__, __FILE__);
+            // Re-open the file.
+            file7.openFile(FILE7, H5F_ACC_RDONLY);
 
-        size_t off_size = 0, len_size = 0;
-        fcpl2.getSizes(off_size, len_size);
-        verify_val(off_size, F2_OFFSET_SIZE, "FileCreatPropList::getSizes", __LINE__, __FILE__);
-        verify_val(len_size, F2_LENGTH_SIZE, "FileCreatPropList::getSizes", __LINE__, __FILE__);
+            // Get the file's creation property list.
+            FileCreatPropList fcpl2 = file7.getCreatePlist();
 
-        unsigned sym_ik = 0, sym_lk = 0;
-        fcpl2.getSymk(sym_ik, sym_lk);
-        verify_val(sym_ik, F2_SYM_INTERN_K, "FileCreatPropList::getSymk", __LINE__, __FILE__);
-        verify_val(sym_lk, F2_SYM_LEAF_K, "FileCreatPropList::getSymk", __LINE__, __FILE__);
+            // Get the file's version information.
+            file7.getFileInfo(finfo);
+            verify_val(static_cast<long>(finfo.super.version), 2, "H5File::getFileInfo", __LINE__, __FILE__);
+            verify_val(static_cast<long>(finfo.free.version), 0, "H5File::getFileInfo", __LINE__, __FILE__);
+            verify_val(static_cast<long>(finfo.sohm.version), 0, "H5File::getFileInfo", __LINE__, __FILE__);
 
-        unsigned istore_ik = fcpl2.getIstorek();
-        verify_val(istore_ik, F2_ISTORE, "FileCreatPropList::getIstorek", __LINE__, __FILE__);
+            // Retrieve the property values & check them.
+            hsize_t userblock = fcpl2.getUserblock();
+            verify_val(userblock, F2_USERBLOCK_SIZE, "FileCreatPropList::getUserblock", __LINE__, __FILE__);
 
-        /*  ret=H5Pget_shared_mesg_nindexes(fcpl2,&nindexes);
-        CHECK(ret, FAIL, "H5Pget_shared_mesg_nindexes");
-        VERIFY(nindexes, MISC11_NINDEXES, "H5Pget_shared_mesg_nindexes");
-     */
+            size_t off_size = 0, len_size = 0;
+            fcpl2.getSizes(off_size, len_size);
+            verify_val(off_size, F2_OFFSET_SIZE, "FileCreatPropList::getSizes", __LINE__, __FILE__);
+            verify_val(len_size, F2_LENGTH_SIZE, "FileCreatPropList::getSizes", __LINE__, __FILE__);
 
-        // Get and verify the file space info from the creation property list */
-        fcpl2.getFileSpaceStrategy(out_strategy, out_persist, out_threshold);
-        verify_val(static_cast<long>(out_strategy), static_cast<long>(strategy),
-                   "FileCreatPropList::getFileSpaceStrategy", __LINE__, __FILE__);
-        verify_val(out_persist, persist, "FileCreatPropList::getFileSpaceStrategy", __LINE__, __FILE__);
-        verify_val(out_threshold, threshold, "FileCreatPropList::getFileSpaceStrategy", __LINE__, __FILE__);
+            unsigned sym_ik = 0, sym_lk = 0;
+            fcpl2.getSymk(sym_ik, sym_lk);
+            verify_val(sym_ik, F2_SYM_INTERN_K, "FileCreatPropList::getSymk", __LINE__, __FILE__);
+            verify_val(sym_lk, F2_SYM_LEAF_K, "FileCreatPropList::getSymk", __LINE__, __FILE__);
 
-        out_fsp_psize = fcpl2.getFileSpacePagesize();
-        verify_val(out_fsp_psize, FSP_SIZE512, "FileCreatPropList::getFileSpacePagesize", __LINE__, __FILE__);
+            unsigned istore_ik = fcpl2.getIstorek();
+            verify_val(istore_ik, F2_ISTORE, "FileCreatPropList::getIstorek", __LINE__, __FILE__);
+
+            /*  ret=H5Pget_shared_mesg_nindexes(fcpl2,&nindexes);
+            CHECK(ret, FAIL, "H5Pget_shared_mesg_nindexes");
+            VERIFY(nindexes, MISC11_NINDEXES, "H5Pget_shared_mesg_nindexes");
+         */
+
+            // Get and verify the file space info from the creation property list */
+            fcpl2.getFileSpaceStrategy(out_strategy, out_persist, out_threshold);
+            verify_val(static_cast<long>(out_strategy), static_cast<long>(strategy),
+                       "FileCreatPropList::getFileSpaceStrategy", __LINE__, __FILE__);
+            verify_val(out_persist, persist, "FileCreatPropList::getFileSpaceStrategy", __LINE__, __FILE__);
+            verify_val(out_threshold, threshold, "FileCreatPropList::getFileSpaceStrategy", __LINE__,
+                       __FILE__);
+
+            out_fsp_psize = fcpl2.getFileSpacePagesize();
+            verify_val(out_fsp_psize, FSP_SIZE512, "FileCreatPropList::getFileSpacePagesize", __LINE__,
+                       __FILE__);
+        }
 
         PASSED();
     } // end of try block

c++/test/th5s.cpp

@@ -153,37 +153,40 @@ test_h5s_basic()
         {
         } // do nothing, exception expected
 
-        /*
-         * Try reading a file that has been prepared that has a dataset with a
-         * higher dimensionality than what the library can handle.
-         *
-         * If this test fails and the H5S_MAX_RANK variable has changed, follow
-         * the instructions in space_overflow.c for regenating the th5s.h5 file.
-         */
-        char *tmp_str = new char[TESTFILE.length() + 1];
-        strcpy(tmp_str, TESTFILE.c_str());
-        const char *testfile = H5_get_srcdir_filename(tmp_str);
-        delete[] tmp_str;
-
-        // Create file
-        H5File fid1(testfile, H5F_ACC_RDONLY);
-
-        // Try to open the dataset that has higher dimensionality than
-        // what the library can handle and this operation should fail.
-        try {
-            DataSet dset1 = fid1.openDataSet("dset");
-
-            // Should FAIL but didn't, so throw an invalid action exception
-            throw InvalidActionException(
-                "H5File::openDataSet",
-                "Opening a dataset with higher dimensionality than what the library can handle");
+        // Only test this for VFDs that operate on a single file
+        bool default_vfd_compatible = true;
+        h5_driver_is_default_vfd_compatible(H5P_DEFAULT, &default_vfd_compatible);
+
+        if (default_vfd_compatible) {
+            /*
+             * Try reading a file that has been prepared that has a dataset with a
+             * higher dimensionality than what the library can handle.
+             *
+             * If this test fails and the H5S_MAX_RANK variable has changed, follow
+             * the instructions in space_overflow.c for regenating the th5s.h5 file.
+             */
+            char *tmp_str = new char[TESTFILE.length() + 1];
+            strcpy(tmp_str, TESTFILE.c_str());
+            const char *testfile = H5_get_srcdir_filename(tmp_str);
+            delete[] tmp_str;
+
+            // Create file
+            H5File fid1(testfile, H5F_ACC_RDONLY);
+
+            // Try to open the dataset that has higher dimensionality than
+            // what the library can handle and this operation should fail.
+            try {
+                DataSet dset1 = fid1.openDataSet("dset");
+
+                // Should FAIL but didn't, so throw an invalid action exception
+                throw InvalidActionException(
+                    "H5File::openDataSet",
+                    "Opening a dataset with higher dimensionality than what the library can handle");
+            }
+            catch (FileIException &E) // catching higher dimensionality dataset
+            {
+            } // do nothing, exception expected
         }
-        catch (FileIException &E) // catching higher dimensionality dataset
-        {
-        } // do nothing, exception expected
-
-        // CHECK_I(ret, "H5Fclose");  // leave this here, later, fake a failure
-        // in the p_close see how this will handle it. - BMR
 
         // When running in valgrind, this PASSED macro will be missed
         PASSED();

config/cmake/HDF5DeveloperBuild.cmake

@@ -161,11 +161,10 @@ if (HDF5_ENABLE_DEBUG_H5TS)
   list (APPEND HDF5_DEBUG_APIS H5TS_DEBUG)
 endif ()
 
-# If HDF5 free list debugging wasn't specifically enabled, disable
-# free lists entirely for developer build modes, as they can
-# make certain types of issues (like references to stale pointers)
-# much more difficult to debug
-if (NOT HDF5_ENABLE_DEBUG_H5FL)
+# Option to control internal free list use.  Enabled by default
+option (HDF5_ENABLE_FREE_LISTS "Enable memory free lists" ON)
+mark_as_advanced (HDF5_ENABLE_FREE_LISTS)
+if (NOT HDF5_ENABLE_FREE_LISTS)
   list (APPEND HDF5_DEVELOPER_DEFS H5_NO_FREE_LISTS)
 endif ()
 

src/CMakeLists.txt

@@ -869,6 +869,7 @@ set (H5TS_SOURCES
     ${HDF5_SRC_DIR}/H5TSbarrier.c
     ${HDF5_SRC_DIR}/H5TSc11.c
     ${HDF5_SRC_DIR}/H5TScond.c
+    ${HDF5_SRC_DIR}/H5TSdlftt_mutex.c
     ${HDF5_SRC_DIR}/H5TSint.c
     ${HDF5_SRC_DIR}/H5TSkey.c
     ${HDF5_SRC_DIR}/H5TSmutex.c

src/H5.c

@@ -257,6 +257,10 @@ H5_init_library(void)
      *   default FAPL.
      *
      */
+#ifdef H5_HAVE_CONCURRENCY
+    if (H5FL_init() < 0)
+        HGOTO_ERROR(H5E_FUNC, H5E_CANTINIT, FAIL, "unable to initialize free list interface");
+#endif
     if (H5E_init() < 0)
         HGOTO_ERROR(H5E_FUNC, H5E_CANTINIT, FAIL, "unable to initialize error interface");
     if (H5FD_init() < 0)

src/H5Cimage.c

@@ -130,7 +130,7 @@ static void   H5C__prep_for_file_close__compute_fd_heights_real(H5C_cache_entry_
 static herr_t H5C__prep_for_file_close__setup_image_entries_array(H5C_t *cache_ptr);
 static herr_t H5C__prep_for_file_close__scan_entries(const H5F_t *f, H5C_t *cache_ptr);
 static herr_t H5C__reconstruct_cache_contents(H5F_t *f, H5C_t *cache_ptr);
-static H5C_cache_entry_t *H5C__reconstruct_cache_entry(const H5F_t *f, H5C_t *cache_ptr, hsize_t *buf_size,
+static H5C_cache_entry_t *H5C__reconstruct_cache_entry(const H5F_t *f, H5C_t *cache_ptr, hsize_t buf_size,
                                                        const uint8_t **buf);
 static herr_t             H5C__write_cache_image_superblock_msg(H5F_t *f, bool create);
 static herr_t             H5C__read_cache_image(H5F_t *f, H5C_t *cache_ptr);
@@ -2376,7 +2376,6 @@ H5C__reconstruct_cache_contents(H5F_t *f, H5C_t *cache_ptr)
 {
     H5C_cache_entry_t *pf_entry_ptr;        /* Pointer to prefetched entry */
     H5C_cache_entry_t *parent_ptr;          /* Pointer to parent of prefetched entry */
-    hsize_t            image_len;           /* Image length */
     const uint8_t     *p;                   /* Pointer into image buffer */
     unsigned           u, v;                /* Local index variable */
     herr_t             ret_value = SUCCEED; /* Return value */
@@ -2392,11 +2391,10 @@ H5C__reconstruct_cache_contents(H5F_t *f, H5C_t *cache_ptr)
     assert(cache_ptr->image_len > 0);
 
     /* Decode metadata cache image header */
-    p         = (uint8_t *)cache_ptr->image_buffer;
-    image_len = cache_ptr->image_len;
-    if (H5C__decode_cache_image_header(f, cache_ptr, &p, image_len + 1) < 0)
+    p = (uint8_t *)cache_ptr->image_buffer;
+    if (H5C__decode_cache_image_header(f, cache_ptr, &p, cache_ptr->image_len + 1) < 0)
         HGOTO_ERROR(H5E_CACHE, H5E_CANTDECODE, FAIL, "cache image header decode failed");
-    assert((size_t)(p - (uint8_t *)cache_ptr->image_buffer) < image_len);
+    assert((size_t)(p - (uint8_t *)cache_ptr->image_buffer) < cache_ptr->image_len);
 
     /* The image_data_len and # of entries should be defined now */
     assert(cache_ptr->image_data_len > 0);
@@ -2408,7 +2406,7 @@ H5C__reconstruct_cache_contents(H5F_t *f, H5C_t *cache_ptr)
         /* Create the prefetched entry described by the ith
          * entry in cache_ptr->image_entrise.
          */
-        if (NULL == (pf_entry_ptr = H5C__reconstruct_cache_entry(f, cache_ptr, &image_len, &p)))
+        if (NULL == (pf_entry_ptr = H5C__reconstruct_cache_entry(f, cache_ptr, cache_ptr->image_len, &p)))
             HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "reconstruction of cache entry failed");
 
         /* Note that we make no checks on available cache space before
@@ -2564,7 +2562,7 @@ H5C__reconstruct_cache_contents(H5F_t *f, H5C_t *cache_ptr)
  *-------------------------------------------------------------------------
  */
 static H5C_cache_entry_t *
-H5C__reconstruct_cache_entry(const H5F_t *f, H5C_t *cache_ptr, hsize_t *buf_size, const uint8_t **buf)
+H5C__reconstruct_cache_entry(const H5F_t *f, H5C_t *cache_ptr, hsize_t buf_size, const uint8_t **buf)
 {
     H5C_cache_entry_t *pf_entry_ptr = NULL; /* Reconstructed cache entry */
     uint8_t            flags        = 0;
@@ -2577,8 +2575,8 @@ H5C__reconstruct_cache_entry(const H5F_t *f, H5C_t *cache_ptr, hsize_t *buf_size
 #endif
     bool               file_is_rw;
     const uint8_t     *p;
-    const uint8_t     *p_end     = *buf + *buf_size - 1; /* Pointer to last valid byte in buffer */
-    H5C_cache_entry_t *ret_value = NULL;                 /* Return value */
+    const uint8_t     *p_end     = *buf + buf_size - 1; /* Pointer to last valid byte in buffer */
+    H5C_cache_entry_t *ret_value = NULL;                /* Return value */
 
     FUNC_ENTER_PACKAGE
 
@@ -2746,8 +2744,7 @@ H5C__reconstruct_cache_entry(const H5F_t *f, H5C_t *cache_ptr, hsize_t *buf_size
     /* Sanity checks */
     assert(pf_entry_ptr->size > 0 && pf_entry_ptr->size < H5C_MAX_ENTRY_SIZE);
 
-    /* Update buffer pointer and buffer len */
-    *buf_size -= (hsize_t)(p - *buf);
+    /* Update buffer pointer */
     *buf = p;
 
     ret_value = pf_entry_ptr;