makefile

#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ options to choose ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 
# -- GPU --
arch = gpu
# extreme scale simulation results in very large array
#x_scale = enabled    

# -- CPU --
#arch = cpu
#openmp = enabled
# whether or not to use avx512 for intel cpu, test before use 
#AVX512 = enabled

# -- MIC(KNL) --
# arch = mic
# OpenMP and AVX512 should both be enabled for MIC
# openmp = enabled
# AVX512 = enabled

# -- compiler --
#tool = gcc
tool = nvidia
#tool = intel

# -- optimization options --
#optimization = debug
#optimization = release
optimization = speedy
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ options to choose ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 

############# DIR #################################
# Directories
DIR=./
OBJ_DIR = $(DIR)1.obj/
EXEC_DIR = $(DIR)1.exec/
SRC_DIR = $(DIR)0.src/
############# DIR #################################

############# architecture ########################
ifeq ($(strip $(arch)), cpu)
	BIN_SUFFIX = cpu
	ifeq ($(strip $(AVX512)), enabled)
		ARCH_FLAGS = -xCORE-AVX512		
	endif
endif

ifeq ($(strip $(arch)), gpu)
	BIN_SUFFIX = gpu
	openmp = disabled
#compatible with pgi compiler, for cray compiler: ARCH_FLAGS = -h acc,noomp -h msgs
	ARCH_FLAGS = -acc -Mcuda -Minfo=accel -ta=tesla:pinned   
	FPP_FLAGS = -D_openacc 
#enable gpu profiling
#FPP_FLAGS += -Dgpu_profiling  
endif

ifeq ($(strip $(arch)), mic)
	BIN_SUFFIX = mic
	ifeq ($(strip $(AVX512)), enabled)
		ARCH_FLAGS = -xMIC-AVX512		
	endif
endif
############# architecture ########################

############# tools ################################
# -------------- gcc ----------------
ifeq ($(strip $(tool)), gcc)
# -J 0.obj put .mod files to 0.obj directory
	MISC_FLAGS = -ffree-line-length-300 -J $(OBJ_DIR)  
#for gcc10+, enable the following temporary fix: 
#MISC_FLAGS += -fallow-argument-mismatch   

	ifeq ($(strip $(optimization)), debug)
		OPT_FLAGS = -g
	else ifeq ($(strip $(optimization)), release)
		OPT_FLAGS = -O3
	else	
		OPT_FLAGS = -Ofast
  endif
	
	ifeq ($(strip $(openmp)), enabled)
		OMP_FLAGS = -fopenmp
	endif
endif

# -------------- nvidia ----------------
ifeq ($(strip $(tool)), nvidia)
	MISC_FLAGS = -module $(OBJ_DIR) 

	ifeq ($(strip $(optimization)), debug)
		OPT_FLAGS = -g
	else ifeq ($(strip $(optimization)), release)
		OPT_FLAGS = -O3
	else	
		OPT_FLAGS = -fast
  endif

	ifeq ($(strip $(openmp)), enabled)
		OMP_FLAGS = -mp
	endif

#for very large array, add: MISC_FLAGS = -mcmodel=medium 
	ifeq ($(strip $(x_scale)), enabled)
		MISC_FLAGS += -mcmodel=medium 
	endif
endif

# -------------- intel ----------------
ifeq ($(strip $(tool)), intel)
	MISC_FLAGS = -module $(OBJ_DIR) 

	ifeq ($(strip $(optimization)), debug)
		OPT_FLAGS = -debug
	else ifeq ($(strip $(optimization)), release)
		OPT_FLAGS = -O3
	else	
		OPT_FLAGS = -Ofast
    endif

	ifeq ($(strip $(openmp)), enabled)
		OMP_FLAGS = -qopenmp
	endif
endif
############# tools ################################

compiler = mpif90
#compiler = ftn
#compiler = mpiifort

CFLAGS += $(ARCH_FLAGS)
CFLAGS += $(OPT_FLAGS)
CFLAGS += $(OMP_FLAGS)
CFLAGS += $(MISC_FLAGS)
CFLAGS += $(FPP_FLAGS)
LFLAGS = $(CFLAGS)

COMPILE = ${compiler} 
LINK = ${compiler}
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

objects = Module.o Main.o Initialization.o Misc.o IO.o Kernel.o Monitor.o Mpi_misc.o Mpi.o Boundary.o 

pref_objects = $(addprefix $(OBJ_DIR), $(objects))

LBM : | make_dir $(pref_objects)
	${LINK} $(LFLAGS) -o $(EXEC_DIR)MF_LBM_singlephase.$(BIN_SUFFIX) $(pref_objects)


$(OBJ_DIR)%.o: $(SRC_DIR)%.F90
	${COMPILE} $(CFLAGS) -c $< -o $@
# $@ is the name of the target being generated
# $< the first prerequisite (usually a source file)


.PHONY: clean 
clean:
	rm -r $(OBJ_DIR)
	rm -r $(EXEC_DIR)

make_dir:
	@echo "Create directories."
	mkdir -p $(DIR)1.exec
	mkdir -p $(DIR)1.obj