Online distributed graph partitioning using PT-Scotch

Makefile

@@ -759,6 +759,16 @@ endif
 	LIBS += $(NCLIB)
 endif
 
+ifneq "$(SCOTCH)" ""
+	SCOTCH_FCINCLUDES += -I$(SCOTCH)/include
+	SCOTCH_LIBS += -L$(SCOTCH)/lib64 -lptscotch -lscotch  -lptscotcherr -lm
+	SCOTCH_FFLAGS = -DMPAS_SCOTCH
+
+	FCINCLUDES += $(SCOTCH_FCINCLUDES)
+	CPPINCLUDES += $(SCOTCH_FCINCLUDES)
+	LIBS += $(SCOTCH_LIBS)
+	override CPPFLAGS += $(SCOTCH_FFLAGS)
+endif
 
 ifneq "$(PNETCDF)" ""
 ifneq ($(wildcard $(PNETCDF)/lib/libpnetcdf.*), )
@@ -1415,6 +1425,33 @@ musica_fortran_test:
 	$(eval MUSICA_FORTRAN_VERSION := $(shell pkg-config --modversion musica-fortran))
 	$(if $(findstring 1,$(MUSICA_FORTRAN_TEST)), $(info Built a simple test program with MUSICA-Fortran version $(MUSICA_FORTRAN_VERSION)), )
 
+scotch_fortran_test:
+	@#
+	@# Create a Fortran test program that will link against the SCOTCH library
+	@#
+	$(info Checking for a working MUSICA-Fortran library...)
+	$(eval SCOTCH_FORTRAN_TEST := $(shell $\
+		printf "program test_scotch_fortran\n$\
+		&   include \"ptscotchf.h\"\n$\
+    	&   doubleprecision :: scotchgraph (scotch_graphdim)\n$\
+		&   integer :: ierr\n$\
+		&   ierr = 0\n$\
+		&   call scotchfgraphinit(scotchgraph (1), ierr)\n$\
+		&   call scotchfgraphexit(scotchgraph(1))\n$\
+		end program test_scotch_fortran\n" | sed 's/&/ /' > test_scotch_fortran.f90; $\
+		$\
+		$(FC) $(SCOTCH_FCINCLUDES) $(SCOTCH_FFLAGS) test_scotch_fortran.f90 -o test_scotch_fortran.x $(SCOTCH_LIBS) > /dev/null 2>&1; $\
+		scotch_fortran_status=$$?; $\
+		rm -f test_scotch_fortran.f90 test_scotch_fortran.x; $\
+		if [ $$scotch_fortran_status -eq 0 ]; then $\
+			printf "1"; $\
+		else $\
+			printf "0"; $\
+		fi $\
+	))
+	$(if $(findstring 0,$(SCOTCH_FORTRAN_TEST)), $(error Could not build a simple test program with Scotch))
+	$(if $(findstring 1,$(SCOTCH_FORTRAN_TEST)), $(info Built a simple test program with Scotch ))
+
 pnetcdf_test:
 	@#
 	@# Create test C programs that look for PNetCDF header file and some symbols in it
@@ -1471,6 +1508,13 @@ else
 MUSICA_MESSAGE = "MPAS was not linked with the MUSICA-Fortran library."
 endif
 
+ifneq "$(SCOTCH_FFLAGS)" ""
+MAIN_DEPS += scotch_fortran_test
+SCOTCH_MESSAGE = "MPAS has been linked with the Scotch graph partitioning library."
+else
+SCOTCH_MESSAGE = "MPAS was NOT linked with the Scotch graph partitioning library."
+endif
+
 mpas_main: $(MAIN_DEPS)
 	cd src; $(MAKE) FC="$(FC)" \
                  CC="$(CC)" \
@@ -1508,6 +1552,7 @@ mpas_main: $(MAIN_DEPS)
 	@echo $(OPENMP_OFFLOAD_MESSAGE)
 	@echo $(OPENACC_MESSAGE)
 	@echo $(MUSICA_MESSAGE)
+	@echo $(SCOTCH_MESSAGE)
 	@echo $(SHAREDLIB_MESSAGE)
 ifeq "$(AUTOCLEAN)" "true"
 	@echo $(AUTOCLEAN_MESSAGE)

src/core_atmosphere/Registry.xml

@@ -319,7 +319,7 @@
         </nml_record>
 
         <nml_record name="decomposition" in_defaults="true">
-                <nml_option name="config_block_decomp_file_prefix" type="character" default_value="x1.40962.graph.info.part."
+                <nml_option name="config_block_decomp_file_prefix" type="character" default_value=""
                      units="-"
                      description="Prefix of graph decomposition file, to be suffixed with the MPI task count"
                      possible_values="Any valid filename"/>

src/framework/Makefile

@@ -36,6 +36,8 @@ OBJS = mpas_kind_types.o \
        mpas_log.o \
        mpas_halo.o \
        mpas_string_utils.o \
+       mpas_ptscotch_interface.o \
+       ptscotch_interface.o \
        mpas_stream_inquiry.o \
        stream_inquiry.o
 
@@ -85,7 +87,7 @@ mpas_timekeeping.o: mpas_string_utils.o mpas_kind_types.o mpas_derived_types.o m
 
 mpas_timer.o: mpas_kind_types.o mpas_dmpar.o mpas_threading.o mpas_log.o
 
-mpas_block_decomp.o: mpas_derived_types.o mpas_hash.o mpas_io_units.o mpas_dmpar.o
+mpas_block_decomp.o: mpas_derived_types.o mpas_hash.o mpas_io_units.o mpas_dmpar.o mpas_ptscotch_interface.o
 
 mpas_block_creator.o: mpas_dmpar.o mpas_hash.o mpas_sort.o mpas_io_units.o mpas_block_decomp.o mpas_stream_manager.o mpas_decomp.o mpas_abort.o $(DEPS)
 
@@ -112,6 +114,8 @@ xml_stream_parser.o: xml_stream_parser.c
 
 mpas_halo.o: mpas_derived_types.o mpas_pool_routines.o mpas_log.o
 
+mpas_ptscotch_interface.o : mpas_derived_types.o mpas_dmpar.o mpas_log.o ptscotch_interface.o
+
 mpas_stream_inquiry.o : mpas_derived_types.o mpas_log.o mpas_c_interfacing.o
 
 clean: