Skip to content

Commit

Permalink
change openmpi to an optional dependency and added dummy MPI files an…
Browse files Browse the repository at this point in the history 
…d a seqential driver krr_seq
  • Loading branch information
@liuyangzhuan
liuyangzhuan committed Sep 8, 2023
1 parent f839116 commit 75958b6
Show file tree
Hide file tree
Showing 14 changed files with 2,135 additions and 184 deletions.
1 change: 1 addition & 0 deletions .ci_tests.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ case "${TEST_NUMBER}" in
16) mpirun --allow-run-as-root --oversubscribe "-n" "7" "$EXAMPLE_FOLDER/cfio" "--tst" "2" "--M" "1500" "--N" "1000" "--K" "2000" "--L" "1800" "--lrlevel" "100" # test CPP interface for compressing product of three FIOs
mpirun --allow-run-as-root --oversubscribe "-n" "7" "$EXAMPLE_FOLDER/cifio" "--M" "1000" "--N" "1000" "--ker" "1" "--errsol 1" "--format" "1" # test CPP interface for approximate inverse of a 1D FIO, using HODLR
mpirun --allow-run-as-root --oversubscribe "-n" "8" "$EXAMPLE_FOLDER/cifio" "--M" "1000" "--N" "1000" "--ker" "1" "--errsol 1" "--format" "2" "--near_para" "4.0";; # test CPP interface for approximate inverse of a 1D FIO, using strong admissible H matrix
17) "$EXAMPLE_FOLDER/krr_seq" "-quant" "--data_dir" "$DATA_FOLDER/KRR_DATA/susy_10Kn" "--dimn" "8" "--ntrain" "10000" "--ntest" "1000" "--sigma" "0.1" "--lambda" "1.0" "-option" "--xyzsort" "2" "--reclr_leaf" "2";; # test HODLR with LR for KRR (non-MPI)
*) printf "${RED} ###YL: Unknown test\n" ;;
esac

9 changes: 8 additions & 1 deletion CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -314,6 +314,7 @@ endif()


#--------------------- MPI ---------------------
if(enable_mpi)
find_package(MPI)
if(MPI_Fortran_FOUND)
set(CMAKE_Fortran_FLAGS "${MPI_Fortran_COMPILE_FLAGS} ${CMAKE_Fortran_FLAGS}")
Expand All @@ -339,6 +340,8 @@ if(BUTTERFLYPACK_USE_MPI3)
set(CMAKE_Fortran_FLAGS "-DHAVE_MPI3 ${CMAKE_Fortran_FLAGS}")
message("-- Using MPI3 features")
endif()
endif()


# Test Fortran Destructor
check_fortran_source_compiles(
Expand Down Expand Up @@ -369,7 +372,9 @@ endif()
# Include directories
#
######################################################################
if(enable_mpi)
include_directories(${MPI_Fortran_INCLUDE_PATH})
endif()
include_directories(${TPL_LAPACK95_INCLUDE_DIRS})

######################################################################
Expand Down Expand Up @@ -450,7 +455,9 @@ else()
endif()



if(NOT enable_mpi)
add_subdirectory(MPI_DUMMY)
endif()
add_subdirectory(SRC_DOUBLE)
add_subdirectory(SRC_DOUBLECOMPLEX)
add_subdirectory(SRC_SINGLE)
Expand Down
45 changes: 28 additions & 17 deletions EXAMPLE/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,25 @@ include_directories(${ButterflyPACK_SOURCE_DIR}/SRC_DOUBLECOMPLEX)
include_directories(${ButterflyPACK_SOURCE_DIR}/SRC_SINGLE)
include_directories(${ButterflyPACK_SOURCE_DIR}/SRC_COMPLEX)
include_directories(${ButterflyPACK_SOURCE_DIR}/EXAMPLE)
if(NOT enable_mpi)
include_directories(${ButterflyPACK_SOURCE_DIR}/MPI_DUMMY)
endif()
include_directories( ${ButterflyPACK_BINARY_DIR}/SRC_DOUBLE)
include_directories( ${ButterflyPACK_BINARY_DIR}/SRC_DOUBLECOMPLEX)
include_directories( ${ButterflyPACK_BINARY_DIR}/SRC_SINGLE)
include_directories( ${ButterflyPACK_BINARY_DIR}/SRC_COMPLEX)
# Libs linked to all of the examples
if(enable_mpi)
set(all_link_libs_z zbutterflypack ${MPI_CXX_LIBRARIES} ${MPI_Fortran_LIBRARIES} ${ZFP_LIB} ${SCALAPACK_LIB} ${ARPACK_LIB} ${MAGMA_LIB} ${LAPACK_LIB} ${BLAS_LIB} m)
set(all_link_libs_d dbutterflypack ${MPI_CXX_LIBRARIES} ${MPI_Fortran_LIBRARIES} ${ZFP_LIB} ${SCALAPACK_LIB} ${ARPACK_LIB} ${MAGMA_LIB} ${LAPACK_LIB} ${BLAS_LIB} m)
set(all_link_libs_c cbutterflypack ${MPI_CXX_LIBRARIES} ${MPI_Fortran_LIBRARIES} ${ZFP_LIB} ${SCALAPACK_LIB} ${ARPACK_LIB} ${MAGMA_LIB} ${LAPACK_LIB} ${BLAS_LIB} m)
set(all_link_libs_s sbutterflypack ${MPI_CXX_LIBRARIES} ${MPI_Fortran_LIBRARIES} ${ZFP_LIB} ${SCALAPACK_LIB} ${ARPACK_LIB} ${MAGMA_LIB} ${LAPACK_LIB} ${BLAS_LIB} m)

else()
set(all_link_libs_z zbutterflypack mpi_dummy ${ZFP_LIB} ${MAGMA_LIB} ${LAPACK_LIB} ${BLAS_LIB} m)
set(all_link_libs_d dbutterflypack mpi_dummy ${ZFP_LIB} ${MAGMA_LIB} ${LAPACK_LIB} ${BLAS_LIB} m)
set(all_link_libs_c cbutterflypack mpi_dummy ${ZFP_LIB} ${MAGMA_LIB} ${LAPACK_LIB} ${BLAS_LIB} m)
set(all_link_libs_s sbutterflypack mpi_dummy ${ZFP_LIB} ${MAGMA_LIB} ${LAPACK_LIB} ${BLAS_LIB} m)
endif()
########################################### single complex Fortran drivers

set(FCEXM1 EMSURF_Driver_sp.f90 EMSURF_Module_sp.f90)
Expand All @@ -25,6 +34,7 @@ set(FZEXM1 EMCURV_Driver.f90 EMCURV_Module.f90)
add_executable(ie2d ${FZEXM1})
target_link_libraries(ie2d ${all_link_libs_z})
install(TARGETS ie2d RUNTIME DESTINATION "${CMAKE_INSTALL_LIBDIR}/EXAMPLE")
add_dependencies(ie2d ie3d_sp)

set(FZEXM2 EMCURV_Eigen_Driver.f90 EMCURV_Module.f90)
add_executable(ie2deigen ${FZEXM2})
Expand All @@ -34,6 +44,7 @@ add_dependencies(ie2deigen ie2d)
set(FZEXM3 EMSURF_Driver.f90 EMSURF_Module.f90)
add_executable(ie3d ${FZEXM3})
target_link_libraries(ie3d ${all_link_libs_z})
add_dependencies(ie3d ie2deigen)

set(FZEXM4 EMSURF_Eigen_Driver.f90 EMSURF_Module.f90)
add_executable(ie3deigen ${FZEXM4})
Expand All @@ -44,28 +55,17 @@ install(TARGETS ie3deigen RUNTIME DESTINATION "${CMAKE_INSTALL_LIBDIR}/EXAMPLE")
set(FZEXM5 SMAT_Driver.f90)
add_executable(smat ${FZEXM5})
target_link_libraries(smat ${all_link_libs_z})
add_dependencies(smat ie3deigen)

set(FZEXM6 FrontalDist_Driver.f90)
add_executable(frontaldist ${FZEXM6})
target_link_libraries(frontaldist ${all_link_libs_z})
add_dependencies(frontaldist smat)


set(FZEXM7 Taylor2D.cpp G2D/G2D.cpp G2D/rtGetInf.cpp G2D/rtGetNaN.cpp G2D/rt_nonfinite.cpp G2D/bessel.cpp)
add_executable(go2d ${FZEXM7})
target_link_libraries(go2d ${all_link_libs_z})
add_dependencies(go2d frontaldist)

set(FZEXM8 Taylor3D.cpp G3D/G3D.cpp G3D/rtGetInf.cpp G3D/rtGetNaN.cpp G3D/rt_nonfinite.cpp)
add_executable(go3d ${FZEXM8})
target_link_libraries(go3d ${all_link_libs_z})
add_dependencies(go3d go2d)


set(FZEXM9 EMSURF_Port_Eigen_Driver.f90 EMSURF_Port_Module.f90 DoCubicInterp2D.c)
add_executable(ie3dporteigen ${FZEXM9})
target_link_libraries(ie3dporteigen ${all_link_libs_z})
add_dependencies(ie3dporteigen ie3d)
add_dependencies(ie3dporteigen frontaldist)
install(TARGETS ie3dporteigen RUNTIME DESTINATION "${CMAKE_INSTALL_LIBDIR}/EXAMPLE")


Expand Down Expand Up @@ -109,9 +109,9 @@ target_link_libraries(fullkrr ${all_link_libs_d})
add_dependencies(fullkrr frontal)


if(enable_mpi) # YL: I haven't add the MPI dummy library in C yet.

########################################### double C++ drivers

set(CDEXM InterfaceTest.cpp)
add_executable(ctest ${CDEXM})
target_link_libraries(ctest ${all_link_libs_d})
Expand All @@ -123,11 +123,22 @@ add_dependencies(ctest_simple ctest)

########################################### double complex C++ drivers

set(FZEXM7 Taylor2D.cpp G2D/G2D.cpp G2D/rtGetInf.cpp G2D/rtGetNaN.cpp G2D/rt_nonfinite.cpp G2D/bessel.cpp)
add_executable(go2d ${FZEXM7})
target_link_libraries(go2d ${all_link_libs_z})
add_dependencies(go2d ctest_simple)

set(FZEXM8 Taylor3D.cpp G3D/G3D.cpp G3D/rtGetInf.cpp G3D/rtGetNaN.cpp G3D/rt_nonfinite.cpp)
add_executable(go3d ${FZEXM8})
target_link_libraries(go3d ${all_link_libs_z})
add_dependencies(go3d go2d)


set(CFEXM FIO_Driver.cpp)
add_executable(cfio ${CFEXM})
target_link_libraries(cfio ${all_link_libs_d})
target_link_libraries(cfio ${all_link_libs_z})
add_dependencies(cfio ctest)
add_dependencies(cfio go3d)

set(CFEXM2 RankBenchmark_Driver.cpp)
add_executable(rankben ${CFEXM2})
Expand All @@ -152,7 +163,7 @@ target_link_libraries(cifio2d ${all_link_libs_d})
target_link_libraries(cifio2d ${all_link_libs_z})
add_dependencies(cifio2d cifio)
endif()

endif()



Expand Down
117 changes: 50 additions & 67 deletions EXAMPLE/KERREG_Driver_seq.f90
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
! This exmple works with double precision data
module APPLICATION_MODULE
use d_BPACK_DEFS
use d_MISC_Utilities
implicit none

!**** define your application-related variables here
Expand Down Expand Up @@ -121,33 +122,24 @@ PROGRAM ButterflyPACK_KRR
type(d_kernelquant)::ker
type(quant_app),target::quant
type(d_Bmatrix)::bmat
integer,allocatable:: groupmembers(:)
integer nmpi
integer groupmembers(1)
type(d_proctree)::ptree
integer MPI_thread
integer,allocatable::Permutation(:)
integer Nunk_loc
integer nargs,flag
integer v_major,v_minor,v_bugfix
integer:: dummy_comm=321

call MPI_Init(ierr)
call MPI_Comm_size(MPI_Comm_World,nmpi,ierr)
allocate(groupmembers(nmpi))
do ii=1,nmpi
groupmembers(ii)=(ii-1)
enddo
groupmembers(1)=0

!**** create the process tree
call d_CreatePtree(nmpi,groupmembers,MPI_Comm_World,ptree)
deallocate(groupmembers)
!**** create the process tree, this call is still needed even though this is a sequential example
call d_CreatePtree(1,groupmembers,dummy_comm,ptree)

if(ptree%MyID==Main_ID)then
write(*,*) "-------------------------------Program Start----------------------------------"
write(*,*) "ButterflyPACK_KRR"
write(*,*) "ButterflyPACK_KRR (non-MPI)"
call d_BPACK_GetVersionNumber(v_major,v_minor,v_bugfix)
write(*,'(A23,I1,A1,I1,A1,I1,A1)') " ButterflyPACK Version:",v_major,".",v_minor,".",v_bugfix
write(*,*) " "
endif

!**** initialize stats and option
call d_InitStat(stats)
Expand Down Expand Up @@ -189,7 +181,7 @@ PROGRAM ButterflyPACK_KRR
else if (trim(strings)=='--lambda')then
read(strings1,*)quant%lambda
else
if(ptree%MyID==Main_ID)write(*,*)'ignoring unknown quant: ', trim(strings)
write(*,*)'ignoring unknown quant: ', trim(strings)
endif
else
flag=0
Expand All @@ -201,7 +193,7 @@ PROGRAM ButterflyPACK_KRR
else if(trim(strings)=='-option')then ! options of ButterflyPACK
call d_ReadOption(option,ptree,ii)
else
if(ptree%MyID==Main_ID)write(*,*)'ignoring unknown argument: ',trim(strings)
write(*,*)'ignoring unknown argument: ',trim(strings)
ii=ii+1
endif
enddo
Expand All @@ -214,19 +206,19 @@ PROGRAM ButterflyPACK_KRR
quant%testfile_p=trim(quant%DATA_DIR)//'_test.csv'
quant%testfile_l=trim(quant%DATA_DIR)//'_test_label.csv'
quant%Nunk = quant%ntrain
if(ptree%MyID==Main_ID)write(*,*)'training set: ',trim(quant%trainfile_p)
write(*,*)'training set: ',trim(quant%trainfile_p)

t1 = MPI_Wtime()
if(ptree%MyID==Main_ID .and. option%verbosity>=0)write(*,*) "geometry modeling......"
t1 = d_seq_wtime()
if(option%verbosity>=0)write(*,*) "geometry modeling......"
open (90,file=quant%trainfile_p)
allocate (quant%xyz(quant%dimn,1:quant%Nunk))
do ii=1,quant%Nunk
read (90,*) quant%xyz(1:quant%dimn,ii)
enddo
close(90)
if(ptree%MyID==Main_ID .and. option%verbosity>=0)write(*,*) "modeling finished"
if(ptree%MyID==Main_ID .and. option%verbosity>=0)write(*,*) " "
t2 = MPI_Wtime()
if(option%verbosity>=0)write(*,*) "modeling finished"
if(option%verbosity>=0)write(*,*) " "
t2 = d_seq_wtime()


!**** register the user-defined function and type in ker
Expand Down Expand Up @@ -258,11 +250,8 @@ PROGRAM ButterflyPACK_KRR
call d_delete_kernelquant(ker)
call d_BPACK_delete(bmat)

if(ptree%MyID==Main_ID .and. option%verbosity>=0)write(*,*) "-------------------------------program end-------------------------------------"

if(option%verbosity>=0)write(*,*) "-------------------------------program end-------------------------------------"

call d_blacs_exit_wrp(1)
call MPI_Finalize(ierr)

end PROGRAM ButterflyPACK_KRR

Expand Down Expand Up @@ -299,7 +288,7 @@ subroutine RBF_solve(bmat,option,msh,quant,ptree,stats)
real(kind=8) r_mn
integer label

if(ptree%MyID==Main_ID .and. option%verbosity>=0)write(*,*) "Solve and Prediction......"
if(option%verbosity>=0)write(*,*) "Solve and Prediction......"

N_unk=msh%Nunk
Dimn=quant%dimn
Expand Down Expand Up @@ -331,21 +320,19 @@ subroutine RBF_solve(bmat,option,msh,quant,ptree,stats)
deallocate(labels)
close(91)

n1 = MPI_Wtime()
n1 = d_seq_wtime()

call d_BPACK_Solution(bmat,x,b,N_unk_loc,1,option,ptree,stats)

n2 = MPI_Wtime()
n2 = d_seq_wtime()
stats%Time_Sol = stats%Time_Sol + n2-n1
call MPI_ALLREDUCE(stats%Time_Sol,rtemp,1,MPI_DOUBLE_PRECISION,MPI_MAX,ptree%Comm,ierr)
if(ptree%MyID==Main_ID .and. option%verbosity>=0)write (*,*) 'Solving:',rtemp,'Seconds'
call MPI_ALLREDUCE(stats%Flop_Sol,rtemp,1,MPI_DOUBLE_PRECISION,MPI_SUM,ptree%Comm,ierr)
if(ptree%MyID==Main_ID .and. option%verbosity>=0)write (*,'(A13Es14.2)') 'Solve flops:',rtemp
if(option%verbosity>=0)write (*,*) 'Solving:',stats%Time_Sol,'Seconds'
if(option%verbosity>=0)write (*,'(A13Es14.2)') 'Solve flops:',stats%Flop_Sol

!**** prediction on the test sets

ntest=quant%ntest
T0 = MPI_Wtime()
T0 = d_seq_wtime()
open (92,file=quant%testfile_p)
allocate (xyz_test(Dimn,ntest))
do edge=1,ntest
Expand All @@ -363,41 +350,39 @@ subroutine RBF_solve(bmat,option,msh,quant,ptree,stats)
vout_tmp(edge_m,1) = vout_tmp(edge_m,1) + value_Z*x(edge,1)
enddo
enddo
vout = vout_tmp

call MPI_REDUCE(vout_tmp, vout, ntest,MPI_double_precision, MPI_SUM, Main_ID, ptree%Comm,ierr)
T1 = MPI_Wtime()
if (ptree%MyID==Main_ID) then
do ii=1,ntest
if(dble(vout(ii,1))>0)then
vout(ii,1)=1
else
vout(ii,1)=-1
endif
enddo

T1 = d_seq_wtime()

open (93,file=quant%testfile_l)
do edge=1,ntest
read (93,*) label
vout_tmp(edge,1)=label
enddo
close(93)
do ii=1,ntest
if(dble(vout(ii,1))>0)then
vout(ii,1)=1
else
vout(ii,1)=-1
endif
enddo

ncorrect=0
do edge=1,ntest
if(dble(vout_tmp(edge,1))*dble(vout(edge,1))>0)then
ncorrect = ncorrect + 1
endif
enddo
open (93,file=quant%testfile_l)
do edge=1,ntest
read (93,*) label
vout_tmp(edge,1)=label
enddo
close(93)

rate = dble(ncorrect)/dble(ntest)
ncorrect=0
do edge=1,ntest
if(dble(vout_tmp(edge,1))*dble(vout(edge,1))>0)then
ncorrect = ncorrect + 1
endif
enddo

write (*,*) ''
write (*,*) 'Prediction time:',T1-T0,'Seconds'
write (*,*) 'Success rate:',rate
write (*,*) ''
rate = dble(ncorrect)/dble(ntest)

endif
write (*,*) ''
write (*,*) 'Prediction time:',T1-T0,'Seconds'
write (*,*) 'Success rate:',rate
write (*,*) ''

deallocate (vout)
deallocate (vout_tmp)
Expand All @@ -406,10 +391,8 @@ subroutine RBF_solve(bmat,option,msh,quant,ptree,stats)
deallocate(b)
deallocate(xyz_test)

call MPI_barrier(ptree%Comm,ierr)

if(ptree%MyID==Main_ID .and. option%verbosity>=0)write(*,*) "Solve and Prediction finished"
if(ptree%MyID==Main_ID .and. option%verbosity>=0)write(*,*) " "
if(option%verbosity>=0)write(*,*) "Solve and Prediction finished"
if(option%verbosity>=0)write(*,*) " "

return

Expand Down
Loading

0 comments on commit 75958b6

Please sign in to comment.