# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Build the cuDSS CUDA Fortran (nvfortran) examples with nvfortran + cudafor.
#
# The Fortran interface (cudss.f90, cudss_mpi.f90, cudss_fortran_mpi.c) ships under
# include/fortran/. By default it is compiled from $(CUDSS_INTERFACE_DIR); set
# CUDSS_INTERFACE_BUILD_DIR to reuse an interface you built elsewhere.

# CUDA Fortran requires nvfortran. GNU make predefines FC=f77 (origin "default"),
# which ?= would not override, so default to nvfortran explicitly while still
# honoring an FC given on the command line or in the environment.
ifeq ($(origin FC),default)
FC := nvfortran
endif
MPIF90    ?= mpif90
FFLAGS    ?= -O2 -cuda
MPICC     ?= mpicc
CFLAGS    ?= -O2 -fPIC
CUDSS_INC ?= cudss_inc
CUDA_INC  ?= cuda_inc
CUDSS_LIB ?= cudss_lib

# cuDSS Fortran interface sources (shipped under include/fortran/, sibling of examples/).
CUDSS_INTERFACE_DIR ?= ../../include/fortran
# Point this at a directory holding a prebuilt interface (cudss.mod/cudss.o,
# cudss_mpi.mod/.o, cudss_fortran_mpi.o) to reuse it instead of recompiling. Empty ->
# build the interface from source in $(CUDSS_INTERFACE_DIR).
CUDSS_INTERFACE_BUILD_DIR ?=

SRCDIR   := .
BINDIR   := bin
MODDIR   := .
$(shell mkdir -p $(BINDIR))

FC_BASE := $(notdir $(FC))
ifneq ($(filter nvfortran ifx ifort mpiifx mpiifort,$(FC_BASE)),)
MODFLAG  := -module $(MODDIR)
else
MODFLAG  := -J$(MODDIR)
endif
INC      := -I$(MODDIR)
FFLAGS   += $(INC)
LIBS     := $(CUDSS_LIB)/libcudss.so -cudalib=cublas -Wl,-rpath,$(CUDSS_LIB)

CUDSS_SRC       := $(CUDSS_INTERFACE_DIR)/cudss.f90
CUDSS_MPI_SRC   := $(CUDSS_INTERFACE_DIR)/cudss_mpi.f90
CUDSS_FORTRAN_MPI_SRC := $(CUDSS_INTERFACE_DIR)/cudss_fortran_mpi.c

# Interface objects: built locally by default, or taken from a prebuilt dir.
ifeq ($(strip $(CUDSS_INTERFACE_BUILD_DIR)),)
CUDSS_OBJ       := cudss.o
CUDSS_MOD       := $(MODDIR)/cudss.mod
CUDSS_MPI_OBJ   := cudss_mpi.o
CUDSS_MPI_MOD   := $(MODDIR)/cudss_mpi.mod
CUDSS_FORTRAN_MPI_OBJ := cudss_fortran_mpi.o
else
CUDSS_OBJ       := $(CUDSS_INTERFACE_BUILD_DIR)/cudss.o
CUDSS_MOD       := $(CUDSS_INTERFACE_BUILD_DIR)/cudss.mod
CUDSS_MPI_OBJ   := $(CUDSS_INTERFACE_BUILD_DIR)/cudss_mpi.o
CUDSS_MPI_MOD   := $(CUDSS_INTERFACE_BUILD_DIR)/cudss_mpi.mod
CUDSS_FORTRAN_MPI_OBJ := $(CUDSS_INTERFACE_BUILD_DIR)/cudss_fortran_mpi.o
INC += -I$(CUDSS_INTERFACE_BUILD_DIR)
endif

UTILS_OBJS := cudssutils.o validation.o
UTILS_MODS := $(MODDIR)/cudssutils.mod $(MODDIR)/validation.mod

EXAMPLES := \
	cudss_batch \
	cudss_batch_sparse_matrix_helpers \
	cudss_dense_matrix_helpers \
	cudss_get_set \
	cudss_high_precision \
	cudss_hybrid_execute_mode \
	cudss_hybrid_memory_mode \
	cudss_memory_handler \
	cudss_mg_mode \
	cudss_mgmn_mode \
	cudss_multithreaded_mode \
	cudss_reordering_phase \
	cudss_residual \
	cudss_schur_complement \
	cudss_simple \
	cudss_simple_complex \
	cudss_sparse_matrix_helpers \
	cudss_uniform_batch

EXAMPLE_BINS := $(addprefix $(BINDIR)/,$(EXAMPLES))

CUDSS_DEPS := $(CUDSS_OBJ)
UTILS_DEPS := $(UTILS_OBJS)
LINK_FULL  := $(UTILS_OBJS) $(CUDSS_OBJ)
LINK_MIN   := $(CUDSS_OBJ)
LINK_MIN_VAL := validation.o $(CUDSS_OBJ)

MPIRUN  ?= mpirun
MPI_NP  ?= 2
CUDSS_COMM_LAYER ?= $(CUDSS_LIB)/libcudss_commlayer_openmpi.so
CUDSS_THREADING_LAYER ?= $(CUDSS_LIB)/libcudss_mtlayer_gomp.so

MPIF90_DRIVER := $(notdir $(firstword $(shell $(MPIF90) -show 2>/dev/null)))
MPIF90_BASE := $(if $(MPIF90_DRIVER),$(MPIF90_DRIVER),$(notdir $(MPIF90)))
ifneq ($(filter nvfortran ifx ifort mpiifx mpiifort,$(MPIF90_BASE)),)
MPI_MODFLAG := -module $(MODDIR)
else
MPI_MODFLAG := -J$(MODDIR)
endif

.PHONY: help usage build utils examples run clean $(EXAMPLES) cudss cudss_mpi

help: usage

usage:
	@printf '%s\n' \
	  'Usage: make [target] [VAR=value ...]' \
	  '' \
	  'Targets:' \
	  '  build       Build all CUDA Fortran example binaries' \
	  '  cudss       Build the core cuDSS Fortran module with $(FC)' \
	  '  cudss_mpi   Build the MPI support module and C wrapper' \
	  '  utils       Build CUDA Fortran utility modules' \
	  '  examples    Build all CUDA Fortran example binaries' \
	  '  run         Build examples and run them, writing outputs under $(BINDIR)' \
	  '  clean       Remove generated objects, modules, and example binaries' \
	  '  usage/help  Print this usage information' \
	  '' \
	  'Example targets:' \
	  '  $(EXAMPLES)' \
	  '' \
	  'Key variables:' \
	  '  FC=$(FC)' \
	  '  FFLAGS=$(FFLAGS)' \
	  '  MPIF90=$(MPIF90)' \
	  '  MPICC=$(MPICC)' \
	  '  CFLAGS=$(CFLAGS)' \
	  '  CUDSS_INC=$(CUDSS_INC)' \
	  '  CUDSS_LIB=$(CUDSS_LIB)' \
	  '  CUDA_INC=$(CUDA_INC)' \
	  '  LDFLAGS=$(LDFLAGS)' \
	  '  MPIRUN=$(MPIRUN)' \
	  '  MPI_NP=$(MPI_NP)' \
	  '  CUDSS_COMM_LAYER=$(CUDSS_COMM_LAYER)' \
	  '  CUDSS_THREADING_LAYER=$(CUDSS_THREADING_LAYER)'

build: examples

examples: $(EXAMPLE_BINS)

# Alias each example name (already in .PHONY) to its binary so `make cudss_<name>` works.
$(EXAMPLES): %: $(BINDIR)/%

cudss: $(CUDSS_OBJ)

ifeq ($(strip $(CUDSS_INTERFACE_BUILD_DIR)),)
$(CUDSS_OBJ): $(CUDSS_SRC)
	$(FC) $(FFLAGS) $(MODFLAG) -c $(CUDSS_SRC)
endif

cudss_mpi: $(CUDSS_MPI_OBJ) $(CUDSS_FORTRAN_MPI_OBJ)

ifeq ($(strip $(CUDSS_INTERFACE_BUILD_DIR)),)
$(CUDSS_MPI_OBJ): $(CUDSS_MPI_SRC) $(CUDSS_OBJ)
	$(FC) $(FFLAGS) $(MODFLAG) -c $(CUDSS_MPI_SRC) -o $(CUDSS_MPI_OBJ)
endif

ifeq ($(strip $(CUDSS_INTERFACE_BUILD_DIR)),)
$(CUDSS_FORTRAN_MPI_OBJ): $(CUDSS_FORTRAN_MPI_SRC)
	$(MPICC) $(CFLAGS) -I$(CUDSS_INC) -I$(CUDA_INC) -c $< -o $@
endif

utils: $(UTILS_OBJS)

cudssutils.o: cudssutils.cuf $(CUDSS_OBJ)
	$(FC) $(FFLAGS) $(MODFLAG) -c cudssutils.cuf -o cudssutils.o

validation.o: validation.f90
	$(FC) $(FFLAGS) $(MODFLAG) -c validation.f90 -o validation.o

# --- examples (pattern rules) ---

$(BINDIR)/cudss_simple: $(BINDIR)/cudss_simple.o $(LINK_FULL)
	$(FC) $(FFLAGS) $(LDFLAGS) -o $@ $^ $(LIBS)

$(BINDIR)/cudss_simple.o: $(CUDSS_DEPS) $(UTILS_DEPS) $(SRCDIR)/cudss_simple.cuf
	$(FC) $(FFLAGS) $(MODFLAG) -c $(SRCDIR)/cudss_simple.cuf -o $@

$(BINDIR)/cudss_simple_complex: $(BINDIR)/cudss_simple_complex.o $(LINK_FULL)
	$(FC) $(FFLAGS) $(LDFLAGS) -o $@ $^ $(LIBS)

$(BINDIR)/cudss_simple_complex.o: $(CUDSS_DEPS) $(UTILS_DEPS) $(SRCDIR)/cudss_simple_complex.cuf
	$(FC) $(FFLAGS) $(MODFLAG) -c $(SRCDIR)/cudss_simple_complex.cuf -o $@

$(BINDIR)/cudss_batch: $(BINDIR)/cudss_batch.o $(LINK_FULL)
	$(FC) $(FFLAGS) $(LDFLAGS) -o $@ $^ $(LIBS)

$(BINDIR)/cudss_batch.o: $(CUDSS_DEPS) $(UTILS_DEPS) $(SRCDIR)/cudss_batch.cuf
	$(FC) $(FFLAGS) $(MODFLAG) -c $(SRCDIR)/cudss_batch.cuf -o $@

$(BINDIR)/cudss_batch_sparse_matrix_helpers: $(BINDIR)/cudss_batch_sparse_matrix_helpers.o $(LINK_MIN)
	$(FC) $(FFLAGS) $(LDFLAGS) -o $@ $^ $(LIBS)

$(BINDIR)/cudss_batch_sparse_matrix_helpers.o: $(CUDSS_DEPS) $(SRCDIR)/cudss_batch_sparse_matrix_helpers.cuf
	$(FC) $(FFLAGS) $(MODFLAG) -c $(SRCDIR)/cudss_batch_sparse_matrix_helpers.cuf -o $@

$(BINDIR)/cudss_dense_matrix_helpers: $(BINDIR)/cudss_dense_matrix_helpers.o $(LINK_FULL)
	$(FC) $(FFLAGS) $(LDFLAGS) -o $@ $^ $(LIBS)

$(BINDIR)/cudss_dense_matrix_helpers.o: $(CUDSS_DEPS) $(UTILS_DEPS) $(SRCDIR)/cudss_dense_matrix_helpers.cuf
	$(FC) $(FFLAGS) $(MODFLAG) -c $(SRCDIR)/cudss_dense_matrix_helpers.cuf -o $@

$(BINDIR)/cudss_sparse_matrix_helpers: $(BINDIR)/cudss_sparse_matrix_helpers.o $(LINK_MIN)
	$(FC) $(FFLAGS) $(LDFLAGS) -o $@ $^ $(LIBS)

$(BINDIR)/cudss_sparse_matrix_helpers.o: $(CUDSS_DEPS) $(SRCDIR)/cudss_sparse_matrix_helpers.cuf
	$(FC) $(FFLAGS) $(MODFLAG) -c $(SRCDIR)/cudss_sparse_matrix_helpers.cuf -o $@

$(BINDIR)/cudss_get_set: $(BINDIR)/cudss_get_set.o $(LINK_FULL)
	$(FC) $(FFLAGS) $(LDFLAGS) -o $@ $^ $(LIBS)

$(BINDIR)/cudss_get_set.o: $(CUDSS_DEPS) $(UTILS_DEPS) $(SRCDIR)/cudss_get_set.cuf
	$(FC) $(FFLAGS) $(MODFLAG) -c $(SRCDIR)/cudss_get_set.cuf -o $@

$(BINDIR)/cudss_high_precision: $(BINDIR)/cudss_high_precision.o $(LINK_MIN)
	$(FC) $(FFLAGS) $(LDFLAGS) -o $@ $^ $(LIBS)

$(BINDIR)/cudss_high_precision.o: $(CUDSS_DEPS) $(SRCDIR)/cudss_high_precision.cuf
	$(FC) $(FFLAGS) $(MODFLAG) -c $(SRCDIR)/cudss_high_precision.cuf -o $@

$(BINDIR)/cudss_hybrid_execute_mode: $(BINDIR)/cudss_hybrid_execute_mode.o $(LINK_FULL)
	$(FC) $(FFLAGS) $(LDFLAGS) -o $@ $^ $(LIBS)

$(BINDIR)/cudss_hybrid_execute_mode.o: $(CUDSS_DEPS) $(UTILS_DEPS) $(SRCDIR)/cudss_hybrid_execute_mode.cuf
	$(FC) $(FFLAGS) $(MODFLAG) -c $(SRCDIR)/cudss_hybrid_execute_mode.cuf -o $@

$(BINDIR)/cudss_hybrid_memory_mode: $(BINDIR)/cudss_hybrid_memory_mode.o $(LINK_FULL)
	$(FC) $(FFLAGS) $(LDFLAGS) -o $@ $^ $(LIBS)

$(BINDIR)/cudss_hybrid_memory_mode.o: $(CUDSS_DEPS) $(UTILS_DEPS) $(SRCDIR)/cudss_hybrid_memory_mode.cuf
	$(FC) $(FFLAGS) $(MODFLAG) -c $(SRCDIR)/cudss_hybrid_memory_mode.cuf -o $@

$(BINDIR)/cudss_memory_handler: $(BINDIR)/cudss_memory_handler.o $(LINK_FULL)
	$(FC) $(FFLAGS) $(LDFLAGS) -o $@ $^ $(LIBS)

$(BINDIR)/cudss_memory_handler.o: $(CUDSS_DEPS) $(UTILS_DEPS) $(SRCDIR)/cudss_memory_handler.cuf
	$(FC) $(FFLAGS) $(MODFLAG) -c $(SRCDIR)/cudss_memory_handler.cuf -o $@

$(BINDIR)/cudss_mg_mode: $(BINDIR)/cudss_mg_mode.o $(LINK_FULL)
	$(FC) $(FFLAGS) $(LDFLAGS) -o $@ $^ $(LIBS)

$(BINDIR)/cudss_mg_mode.o: $(CUDSS_DEPS) $(UTILS_DEPS) $(SRCDIR)/cudss_mg_mode.cuf
	$(FC) $(FFLAGS) $(MODFLAG) -c $(SRCDIR)/cudss_mg_mode.cuf -o $@

$(BINDIR)/cudss_mgmn_mode: $(BINDIR)/cudss_mgmn_mode.o $(LINK_FULL) $(CUDSS_MPI_OBJ) $(CUDSS_FORTRAN_MPI_OBJ)
	$(MPIF90) $(FFLAGS) $(LDFLAGS) -o $@ $(BINDIR)/cudss_mgmn_mode.o $(LINK_FULL) $(CUDSS_MPI_OBJ) $(CUDSS_FORTRAN_MPI_OBJ) $(LIBS)

$(BINDIR)/cudss_mgmn_mode.o: $(CUDSS_DEPS) $(UTILS_DEPS) $(CUDSS_MPI_OBJ) $(SRCDIR)/cudss_mgmn_mode.cuf
	$(MPIF90) $(FFLAGS) $(MPI_MODFLAG) -c $(SRCDIR)/cudss_mgmn_mode.cuf -o $@

$(BINDIR)/cudss_multithreaded_mode: $(BINDIR)/cudss_multithreaded_mode.o $(LINK_FULL)
	$(FC) $(FFLAGS) $(LDFLAGS) -o $@ $^ $(LIBS)

$(BINDIR)/cudss_multithreaded_mode.o: $(CUDSS_DEPS) $(UTILS_DEPS) $(SRCDIR)/cudss_multithreaded_mode.cuf
	$(FC) $(FFLAGS) $(MODFLAG) -c $(SRCDIR)/cudss_multithreaded_mode.cuf -o $@

$(BINDIR)/cudss_reordering_phase: $(BINDIR)/cudss_reordering_phase.o $(LINK_FULL)
	$(FC) $(FFLAGS) $(LDFLAGS) -o $@ $^ $(LIBS)

$(BINDIR)/cudss_reordering_phase.o: $(CUDSS_DEPS) $(UTILS_DEPS) $(SRCDIR)/cudss_reordering_phase.cuf
	$(FC) $(FFLAGS) $(MODFLAG) -c $(SRCDIR)/cudss_reordering_phase.cuf -o $@

$(BINDIR)/cudss_residual: $(BINDIR)/cudss_residual.o $(LINK_MIN_VAL)
	$(FC) $(FFLAGS) $(LDFLAGS) -o $@ $^ $(LIBS)

$(BINDIR)/cudss_residual.o: $(CUDSS_DEPS) validation.o $(SRCDIR)/cudss_residual.cuf
	$(FC) $(FFLAGS) $(MODFLAG) -c $(SRCDIR)/cudss_residual.cuf -o $@

$(BINDIR)/cudss_schur_complement: $(BINDIR)/cudss_schur_complement.o $(LINK_MIN)
	$(FC) $(FFLAGS) $(LDFLAGS) -o $@ $^ $(LIBS)

$(BINDIR)/cudss_schur_complement.o: $(CUDSS_DEPS) $(SRCDIR)/cudss_schur_complement.cuf
	$(FC) $(FFLAGS) $(MODFLAG) -c $(SRCDIR)/cudss_schur_complement.cuf -o $@

$(BINDIR)/cudss_uniform_batch: $(BINDIR)/cudss_uniform_batch.o $(LINK_FULL)
	$(FC) $(FFLAGS) $(LDFLAGS) -o $@ $^ $(LIBS)

$(BINDIR)/cudss_uniform_batch.o: $(CUDSS_DEPS) $(UTILS_DEPS) $(SRCDIR)/cudss_uniform_batch.cuf
	$(FC) $(FFLAGS) $(MODFLAG) -c $(SRCDIR)/cudss_uniform_batch.cuf -o $@

run: examples
	@set -e; \
	for ex in $(EXAMPLES); do \
	  if [ "$$ex" = "cudss_mgmn_mode" ]; then \
	    $(MPIRUN) -np $(MPI_NP) $(BINDIR)/$$ex $(CUDSS_COMM_LAYER) > $(BINDIR)/$$ex.out 2>&1; \
	  elif [ "$$ex" = "cudss_multithreaded_mode" ]; then \
	    $(BINDIR)/$$ex $(CUDSS_THREADING_LAYER) > $(BINDIR)/$$ex.out 2>&1; \
	  else \
	    $(BINDIR)/$$ex > $(BINDIR)/$$ex.out 2>&1; \
	  fi; \
	  echo "Test $$ex PASSED"; \
	done

clean:
	$(RM) $(CUDSS_OBJ) $(CUDSS_MOD) $(CUDSS_MPI_OBJ) $(CUDSS_MPI_MOD) $(CUDSS_FORTRAN_MPI_OBJ)
	$(RM) $(UTILS_OBJS) $(UTILS_MODS)
	$(RM) $(addprefix $(BINDIR)/,$(addsuffix .o,$(EXAMPLES))) $(EXAMPLE_BINS)
	$(RM) $(MODDIR)/memhandler.mod $(MODDIR)/matrices.mod
	$(RM) $(BINDIR)/*.out
