Intel(R) SHMEM Library (ISHMEM) 1.1.0

CMakeLists.txt

@@ -17,7 +17,7 @@ option(BUILD_SHARED_LIBS ON)
 
 # Build flags for choosing back-ends
 option(ENABLE_OPENSHMEM "Enable OpenSHMEM runtime support" ON)
-#option(ENABLE_MPI "Enable MPI runtime support" OFF)
+option(ENABLE_MPI "Enable MPI runtime support" OFF)
 #option(ENABLE_PMI "Enable PMI runtime support" OFF)
 #option(ENABLE_OSHMPI "Enable OSHMPI support for MPI runtimes" OFF)
 #option(ENABLE_ONECCL "Enable ONECCL support for collectives" OFF)
@@ -28,10 +28,11 @@ option(BUILD_TEST "Build Test" ON)
 option(BUILD_PERF_TEST "Build Performance Test" ON)
 
 # Build flags for choosing different configurations
-option(ENABLE_GPU_RDMA "Enable GPU RDMA support" ON)
+#option(ENABLE_GPU_RDMA "Enable GPU RDMA support" ON)
 
 # Other build flags
 option(USE_REDUCED_LINK_ENGINE_SET "Reduced link engines for single tile device" OFF)
+option(USE_DLMALLOC "DLMALLOC library for allocation and free" ON)
 
 option(CTEST_SCHEDULER "Job scheduler used for ctest" OFF)
 if(NOT CTEST_SCHEDULER)
@@ -57,13 +58,26 @@ configure_file(ishmem_config.h.in ishmem_config.h)
 set(CMAKE_CXX_COMPILER icpx)
 add_compile_options(-fsycl)
 add_link_options(-fsycl)
-add_compile_options(-Werror -Wuninitialized)
+
+if (ISHMEM_USE_OFFLINE_COMPILER)
+        message("will build using ahead of time compiler")
+        add_compile_options(-fsycl-targets=spir64_gen)
+        set(devtype "-device pvc")
+        add_compile_options(--start-no-unused-arguments -Xs ${devtype} --end-no-unused-arguments)
+        add_link_options(-fsycl-targets=spir64_gen)
+        add_link_options(--start-no-unused-arguments -Xs ${devtype} --end-no-unused-arguments)
+else()
+        message("will build using incremental compiler")
+endif()
+
+add_compile_options(-Werror -Wuninitialized -Wunused-variable)
+
 add_compile_options(-Rno-debug-disables-optimization)
 add_link_options(-Rno-debug-disables-optimization)
 
 # Make sure at least one of the backends is enabled
 if (ENABLE_MPI STREQUAL "OFF" AND ENABLE_OPENSHMEM STREQUAL "OFF" AND ENABLE_PMI STREQUAL "OFF")
-    message(FATAL ERROR "At least one of 'ENABLE_MPI', 'ENABLE_OPENSHMEM' or 'ENABLE_PMI' must be on")
+    message(FATAL_ERROR "At least one of 'ENABLE_MPI', 'ENABLE_OPENSHMEM' or 'ENABLE_PMI' must be on")
 endif()
 
 # Check for valid L0 path
@@ -83,6 +97,8 @@ if (EXISTS "${L0_INSTALL_PREFIX}/lib64/libze_loader.so")
     list(APPEND EXTRA_LIBS "-L${L0_INSTALL_PREFIX}/lib64 -lze_loader")
 elseif (EXISTS "${L0_INSTALL_PREFIX}/lib/libze_loader.so")
     list(APPEND EXTRA_LIBS "-L${L0_INSTALL_PREFIX}/lib -lze_loader")
+elseif (EXISTS "${L0_INSTALL_PREFIX}/lib/x86_64-linux-gnu/libze_loader.so")
+    list(APPEND EXTRA_LIBS "-L${L0_INSTALL_PREFIX}/lib/x86_64-linux-gnu -lze_loader")
 else()
     message(FATAL_ERROR
             "Cannot find level zero library!\n"
@@ -127,7 +143,7 @@ endif()
 add_subdirectory(pmi-simple)
 add_subdirectory(src)
 add_subdirectory(examples)
-if (BUILD_TEST)
+if (BUILD_TEST OR BUILD_PERF_TEST)
     add_subdirectory(test)
 endif()
 

README.md

@@ -25,13 +25,13 @@ Intel® oneAPI DPC++/C++ Compiler with Level Zero support.
 To install Level Zero, refer to the instructions in [Intel® Graphics Compute Runtime for oneAPI Level Zero and OpenCL™ Driver repository](https://github.com/intel/compute-runtime/releases) or to the [installation guide](https://dgpu-docs.intel.com/installation-guides/index.html) for oneAPI users.
 
 ## Installation
-Intel® SHMEM requires a host SHMEM back-end to be used for host-sided operations support. In particular, it relies on an a collection of extension APIs (`shmemx_heap_create`, `shmemx_heap_preinit`, and `shmemx_heap_postinit`) to coordinate the Intel® SHMEM and OpenSHMEM heaps. We recommend [Sandia OpenSHMEM](https://github.com/Sandia-OpenSHMEM/SOS) for this purpose.
+Intel® SHMEM requires a host OpenSHMEM back-end to be used for host-sided operations support. In particular, it relies on a collection of extension APIs (`shmemx_heap_create`, `shmemx_heap_preinit`, and `shmemx_heap_postinit`) to coordinate the Intel® SHMEM and OpenSHMEM heaps. We recommend [Sandia OpenSHMEM v1.5.3rc1](https://github.com/Sandia-OpenSHMEM/SOS/releases/tag/v1.5.3rc1) or newer for this purpose.
 
 ### Building Sandia OpenSHMEM (SOS)
 Download the SOS repo to be configured as a back-end for Intel® SHMEM.
 
 ```
-git clone https://github.com/Sandia-OpenSHMEM/SOS.git SOS
+git clone --recurse-submodules https://github.com/Sandia-OpenSHMEM/SOS.git SOS
 ```
 
 Build SOS following instructions below. `FI_HMEM` support in the provider is required for use with Intel® SHMEM. To enable `FI_HMEM` with a supported provider, we recommend a specific set of config flags. Below are two examples for configuring and building SOS with two providers supporting `FI_HMEM`. To configure SOS with the `verbs;ofi_rxm` provider, use the following instructions:
@@ -47,10 +47,19 @@ To configure SOS with the HPE Slingshot provider `cxi`, please use the following
 ```
 cd SOS
 ./autogen.sh
-./configure --prefix=<sos_dir> --with-ofi=<ofi_installation> --enable-pmi-simple --enable-ofi-mr=basic --disable-ofi-inject --enable-ofi-hmem --disable-bounce-buffers --enable-ofi-manual-progress --enable-mr-endpoint
+./configure --prefix=<sos_dir> --with-ofi=<ofi_installation> --enable-pmi-simple --enable-ofi-mr=basic --disable-ofi-inject --enable-ofi-hmem --disable-bounce-buffers --enable-ofi-manual-progress --enable-mr-endpoint --disable-nonfetch-amo --enable-manual-progress
 make -j
 make install
-``` 
+```
+To configure SOS with the `psm3` provider, please use the following instructions:
+```
+cd SOS
+./autogen.sh
+./configure --prefix=<sos_dir> --with-ofi=<ofi_installation> --enable-pmi-simple --enable-manual-progress --enable-ofi-hmem --disable-bounce-buffers --enable-ofi-mr=basic --enable-mr-endpoint
+make -j
+make install
+```
+
 Please choose an appropriate PMI configure flag based on the available PMI client library in the system. Please check for further instructions on [SOS Wiki pages](https://github.com/Sandia-OpenSHMEM/SOS/wiki). Optionally, users may also choose to add `--disable-fortran` since fortran interfaces will not be used.
 
 

RELEASE_NOTES.md

@@ -1,10 +1,33 @@
 # Release Notes <!-- omit in toc -->
-This document contains a list of new features and known limitations of Intel® SHMEM in the most recent release.
+This document contains a list of new features and known limitations of Intel® SHMEM releases.
+
+## Release 1.1.0
+
+### New Features and Enhancements
+- Support for OpenSHMEM 1.5 teams and team-based collective operations.
+- Device and host API support for strided RMA operations - ibput and ibget, from OpenSHMEM 1.6.
+- Device and host API support for non-blocking atomic operations.
+- Device and host API support for size-based RMA and signaling operations.
+- Device and host API support for all/any/some versions of point-to-point synchronization operations.
+- Device and host API support for signal set, add, and wait-until operations.
+- Fixed implementation of `ishmem_free`.
+- Compatible with [Sandia OpenSHMEM (SOS)](https://github.com/Sandia-OpenSHMEM/SOS) v1.5.3rc1 and newer releases.
+- Support for [OFI](https://github.com/ofiwg/libfabric) PSM3 provider enabled networks via SOS. 
+- Updated [specification](https://oneapi-src.github.io/ishmem/intro.html) with the teams API, size-based RMA, non-blocking AMO, team-based collectives, all/any/some flavors of synchronization operations, utility extensions for print messages, etc.
+- An improved and additional set of [unit tests](test/unit/SHMEM) covering functionality of the new APIs.
+- New [examples](examples/SHMEM) illustrating use cases of Intel® SHMEM functionalities including the Teams APIs.
+- Updated [launcher script](scripts/ishmrun) to launch Intel® SHMEM applications on the available SYCL devices in the system.
+
+### Known Limitations
+- Only [Sandia OpenSHMEM](https://github.com/Sandia-OpenSHMEM/SOS) is currently supported as the host back-end.
+- Not all APIs from OpenSHMEM standard are supported. Please refer to [Supported/Unsupported Features](https://oneapi-src.github.io/ishmem/supported_features.html) to get a complete view.
+- Intel® SHMEM requires a one-to-one mapping of PEs to SYCL devices. This implies that Intel® SHMEM executions must launch with a number of processes on each compute node that is no more than the number of available SYCL devices on each one of those nodes. By default, the Intel® SHMEM runtime considers each individual device tile to make up a single SYCL device and assigns a tile per PE.
+- All collective operations within a kernel must complete before invoking subsequent kernel-initiated collective operation.
+- To run Intel® SHMEM with SOS enabling the Slingshot provider in OFI, environment variable `FI_CXI_OPTIMIZED_MRS=0` must be used. It is also recommended to use `FI_CXI_DEFAULT_CQ_SIZE=131072`.
+- To run Intel® SHMEM with SOS enabling the verbs provider, environment variable `MLX5_SCATTER_TO_CQE=0` must be used.
+- Inter-node communication in Intel® SHMEM requires [dma-buf](https://www.kernel.org/doc/html/latest/driver-api/dma-buf.html) support in the Linux kernel. Inter-node functionality in Intel® SHMEM Release 1.1.0 is tested with SUSE Linux Enterprise Server 15 SP4.
 
 ## Release 1.0.0
-### Table of Contents <!-- omit in toc -->
-- [New Features](#new-features)
-- [Known Limitations](#known-limitations)
 
 ### New Features
 - OpenSHMEM programming on Intel® GPUs.

docs/README.md

@@ -18,6 +18,7 @@ To get started with Sphinx, Read the Docs, and rST syntax please refer to:
 ```
     pip install sphinx
     pip install sphinx_rtd_theme
+    pip install sphinx_book_theme
 ```
 4.  Generate documentation using the steps below.
 
@@ -27,3 +28,8 @@ To get started with Sphinx, Read the Docs, and rST syntax please refer to:
 2.  Optionally run `make clean` to remove any pre-existing files.
 3.  Run `make html`.
 4.  Open the `build/html/index.html` file with your favorite browser.
+
+### Style and text editor guidlines:
+
+1.  Whenever possible, please maintain only a single sentence per line.
+2.  Please set your text editor to enforce no more than 80 characters per line.

docs/source/Makefile

@@ -7,6 +7,7 @@ SPHINXOPTS    =
 SPHINXBUILD   = sphinx-build
 SOURCEDIR     = .
 BUILDDIR      = build
+PDFBUILDDIR   = buildpdf
 
 ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
 ERROROPTS       = -W --keep-going
@@ -16,11 +17,15 @@ help:
 	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
 
 clean:
-	rm -rf $(BUILDDIR)/*
+	rm -rf $(BUILDDIR)/* $(PDFBUILDDIR)/*
 	@echo "Directories cleaned."
 	@ls | grep -P "(conf[A-Za-z0-9]{3})\.py" | xargs -d"\n" rm -rf
 
 html:
 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
 	@echo
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
+
+pdf:
+	@$(SPHINXBUILD) -M latexpdf "$(SOURCEDIR)" "$(PDFBUILDDIR)"
+	@echo "Build finished. The PDF are in $(PDFBUILDDIR)/pdf."