Merge branch 'main' into jjhong_KV_alt_sliding_window

Author: Joshua Hong

.github/workflows/main.yml

@@ -49,22 +49,22 @@ jobs:
         run: >-
           conda build --output-folder=conda/pkg  conda/recipe &&
           conda install tvm -c ./conda/pkg
-      - name: Build iOS RPC
-        run: |
-          IOS_VERSION="14.0"
-          CMAKE_FLAGS="-DCMAKE_BUILD_TYPE=Release \
-                       -DCMAKE_SYSTEM_NAME=iOS \
-                       -DCMAKE_SYSTEM_VERSION=${IOS_VERSION} \
-                       -DCMAKE_OSX_SYSROOT=iphonesimulator \
-                       -DCMAKE_OSX_ARCHITECTURES=x86_64 \
-                       -DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \
-                       -DCMAKE_BUILD_WITH_INSTALL_NAME_DIR=ON \
-                       -DUSE_IOS_RPC=ON"
-
-          mkdir build-ios-simulator
-          cd build-ios-simulator
-          cmake .. ${CMAKE_FLAGS}
-          cmake --build . --target ios_rpc
+#      - name: Build iOS RPC
+#        run: |
+#          IOS_VERSION="14.0"
+#          CMAKE_FLAGS="-DCMAKE_BUILD_TYPE=Release \
+#                       -DCMAKE_SYSTEM_NAME=iOS \
+#                       -DCMAKE_SYSTEM_VERSION=${IOS_VERSION} \
+#                       -DCMAKE_OSX_SYSROOT=iphonesimulator \
+#                       -DCMAKE_OSX_ARCHITECTURES=x86_64 \
+#                       -DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \
+#                       -DCMAKE_BUILD_WITH_INSTALL_NAME_DIR=ON \
+#                       -DUSE_IOS_RPC=ON"
+#
+#          mkdir build-ios-simulator
+#          cd build-ios-simulator
+#          cmake .. ${CMAKE_FLAGS}
+#          cmake --build . --target ios_rpc
       - name: Test
         shell: bash -l {0}
         run: >-
@@ -108,74 +108,39 @@ jobs:
         run: >-
           python -m pytest -v tests/python/all-platform-minimal-test
 
-  # Disabled due to https://github.com/apache/tvm/issues/13950
-  # Windows-Static-Runtime:
-  #   if: ${{ github.repository == 'apache/tvm' }}
-  #   runs-on: windows-2019
-  #   steps:
-  #     - uses: actions/checkout@v2
-  #       with:
-  #         submodules: 'recursive'
-  #     - name: Set up environment
-  #       uses: ./.github/actions/setup
-  #     - name: Build static TVM runtime
-  #       shell: bash -l {0}
-  #       run: |
-  #         tests/scripts/task_config_build_static.sh build
-  #         cd build
-  #         cmake .. -A x64 -DCMAKE_CONFIGURATION_TYPES="Release"
-  #         cmake --build . --config Release --target runtime
-
-  Linux-Static-Runtime:
-    if: ${{ github.repository == 'apache/tvm' }}
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: 'recursive'
-      - name: Set up environment
-        uses: ./.github/actions/setup
-      - name: Build static TVM runtime
-        shell: bash -l {0}
-        run: |
-          tests/scripts/task_config_build_static.sh build
-          cd build
-          cmake ..
-          cmake --build . --config Release --target runtime
-
-  Android:
-    if: ${{ github.repository == 'apache/tvm' }}
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: 'recursive'
-      - name: Set up environment
-        uses: ./.github/actions/setup
-      - name: Set up java
-        uses: actions/setup-java@v3
-        with:
-          distribution: 'zulu'
-          java-version: '11'
-      - name: Build TVM
-        shell: bash -l {0}
-        run: |
-          mkdir build
-          cd build
-          ../tests/scripts/task_config_build_jvm.sh .
-          cmake ..
-          make
-      - name: Build TVM4J
-        run: |
-          make jvmpkg
-      - name: Build android_rpc
-        working-directory: apps/android_rpc
-        run: |
-          set -eux
-          export PATH="${ANDROID_NDK_LATEST_HOME}:$PATH"
-          gradle clean build
-      - name: Upload android_rpc APK
-        uses: actions/upload-artifact@v4
-        with:
-          name: android_rpc-debug.apk
-          path: ./apps/android_rpc/app/build/outputs/apk/debug/app-debug.apk
+ # Android:
+ #   if: ${{ github.repository == 'apache/tvm' }}
+ #   runs-on: ubuntu-22.04
+ #   steps:
+ #     - uses: actions/checkout@v2
+ #       with:
+ #         submodules: 'recursive'
+ #     - name: Set up environment
+ #       uses: ./.github/actions/setup
+ #     - name: Set up java
+ #       uses: actions/setup-java@v3
+ #       with:
+ #         distribution: 'zulu'
+ #         java-version: '11'
+ #     - name: Build TVM
+ #       shell: bash -l {0}
+ #       run: |
+ #         mkdir build
+ #         cd build
+ #         ../tests/scripts/task_config_build_jvm.sh .
+ #         cmake ..
+ #         make
+ #     - name: Build TVM4J
+ #       run: |
+ #         make jvmpkg
+ #     - name: Build android_rpc
+ #       working-directory: apps/android_rpc
+ #       run: |
+ #         set -eux
+ #         export PATH="${ANDROID_NDK_LATEST_HOME}:$PATH"
+ #         gradle clean build
+ #     - name: Upload android_rpc APK
+ #       uses: actions/upload-artifact@v4
+ #       with:
+ #         name: android_rpc-debug.apk
+ #         path: ./apps/android_rpc/app/build/outputs/apk/debug/app-debug.apk

.gitmodules

@@ -31,3 +31,6 @@
 [submodule "3rdparty/zlib"]
 	path = 3rdparty/zlib
 	url = https://github.com/madler/zlib.git
+[submodule "ffi/3rdparty/dlpack"]
+	path = ffi/3rdparty/dlpack
+	url = https://github.com/dmlc/dlpack.git

3rdparty/cutlass_fpA_intB_gemm

@@ -159,7 +159,6 @@ if(MSVC)
   add_definitions(-D_SCL_SECURE_NO_WARNINGS)
   add_definitions(-D_ENABLE_EXTENDED_ALIGNED_STORAGE)
   add_definitions(-DNOMINMAX)
-
   # regeneration does not work well with msbuild custom rules.
   set(CMAKE_SUPPRESS_REGENERATION ON)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc")
@@ -496,6 +495,9 @@ list(REMOVE_ITEM COMPILER_SRCS ${LIBINFO_FILE})
 add_library(tvm_objs OBJECT ${COMPILER_SRCS})
 add_library(tvm_runtime_objs OBJECT ${RUNTIME_SRCS})
 add_library(tvm_libinfo_objs OBJECT ${LIBINFO_FILE})
+target_link_libraries(tvm_objs PUBLIC tvm_ffi_header)
+target_link_libraries(tvm_runtime_objs PUBLIC tvm_ffi_header)
+target_link_libraries(tvm_libinfo_objs PUBLIC tvm_ffi_header)
 
 include(GNUInstallDirs)
 if(NOT BUILD_DUMMY_LIBTVM)
@@ -567,6 +569,9 @@ if(USE_IOS_RPC)
   add_subdirectory("apps/ios_rpc")
 endif()
 
+add_subdirectory(ffi)
+
+
 if(TVM_DEBUG_WITH_ABI_CHANGE)
   message(STATUS "Building with debug code that may cause ABI changes...")
   target_compile_definitions(tvm_objs PRIVATE "TVM_DEBUG_WITH_ABI_CHANGE")
@@ -602,6 +607,10 @@ endif()
 target_link_libraries(tvm PRIVATE ${TVM_RUNTIME_LINKER_LIBS})
 target_link_libraries(tvm_runtime PRIVATE ${TVM_RUNTIME_LINKER_LIBS})
 
+target_link_libraries(tvm PUBLIC tvm_ffi_objs)
+target_link_libraries(tvm_runtime PUBLIC tvm_ffi_objs)
+
+
 if(BUILD_FOR_HEXAGON AND DEFINED USE_HEXAGON_GTEST AND EXISTS ${USE_HEXAGON_GTEST})
   include(FetchContent)
   FetchContent_Declare(googletest SOURCE_DIR "${USE_HEXAGON_GTEST}")
@@ -639,6 +648,7 @@ if (HIDE_PRIVATE_SYMBOLS AND NOT ${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
   target_link_libraries(tvm_runtime PRIVATE ${HIDE_SYMBOLS_LINKER_FLAGS})
   target_compile_definitions(tvm_allvisible PUBLIC $<TARGET_PROPERTY:tvm,INTERFACE_COMPILE_DEFINITONS>)
   target_compile_definitions(tvm_allvisible PRIVATE $<TARGET_PROPERTY:tvm,COMPILE_DEFINITONS>)
+  target_link_libraries(tvm_allvisible PUBLIC tvm_ffi_objs)
 endif()
 
 # Create the `cpptest` target if we can find GTest.  If not, we create dummy
@@ -696,10 +706,10 @@ if(NOT DEFINED ENV{CONDA_BUILD})
 endif()
 
 # Installation rules
-install(TARGETS tvm EXPORT ${PROJECT_NAME}Targets DESTINATION lib${LIB_SUFFIX})
-install(TARGETS tvm_runtime EXPORT ${PROJECT_NAME}Targets DESTINATION lib${LIB_SUFFIX})
+install(TARGETS tvm DESTINATION lib${LIB_SUFFIX})
+install(TARGETS tvm_runtime DESTINATION lib${LIB_SUFFIX})
 if(BUILD_FOR_HEXAGON AND DEFINED USE_HEXAGON_GTEST AND EXISTS ${USE_HEXAGON_GTEST})
-  install(TARGETS gtest EXPORT ${PROJECT_NAME}Targets DESTINATION lib${LIB_SUFFIX})
+  install(TARGETS gtest DESTINATION lib${LIB_SUFFIX})
 endif()
 
 if (INSTALL_DEV)
@@ -734,9 +744,9 @@ string(APPEND PROJECT_CONFIG_CONTENT
        "include(\"\${CMAKE_CURRENT_LIST_DIR}/${PROJECT_NAME}Targets.cmake\")")
 file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/temp_config_file.cmake" ${PROJECT_CONFIG_CONTENT})
 
-install(EXPORT ${PROJECT_NAME}Targets
-  NAMESPACE ${PROJECT_NAME}::
-  DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME})
+# install(EXPORT ${PROJECT_NAME}Targets
+#   NAMESPACE ${PROJECT_NAME}::
+#   DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME})
 
 # Create config for find_package()
 configure_package_config_file(
@@ -750,9 +760,9 @@ install(
 
 # More target definitions
 if(MSVC)
-  target_compile_definitions(tvm_objs PRIVATE -DTVM_EXPORTS)
-  target_compile_definitions(tvm_libinfo_objs PRIVATE -DTVM_EXPORTS)
-  target_compile_definitions(tvm_runtime_objs PRIVATE -DTVM_EXPORTS)
+  target_compile_definitions(tvm_objs PRIVATE -DTVM_EXPORTS -DTVM_FFI_EXPORTS)
+  target_compile_definitions(tvm_libinfo_objs PRIVATE -DTVM_EXPORTS -DTVM_FFI_EXPORTS)
+  target_compile_definitions(tvm_runtime_objs PRIVATE -DTVM_EXPORTS -DTVM_FFI_EXPORTS)
 endif()
 
 set(TVM_IS_DEBUG_BUILD OFF)
@@ -774,17 +784,8 @@ if(TVM_IS_DEBUG_BUILD)
   endif()
 endif()
 
-# Run dsymutil to generate debugging symbols for backtraces
-if(APPLE AND TVM_IS_DEBUG_BUILD)
-  find_program(DSYMUTIL dsymutil)
-  mark_as_advanced(DSYMUTIL)
-  add_custom_command(TARGET tvm
-      POST_BUILD
-      COMMAND ${DSYMUTIL} ARGS $<TARGET_FILE:tvm>
-      COMMENT "Running dsymutil"
-      VERBATIM
-		  )
-endif()
+add_dsymutil(tvm)
+add_dsymutil(tvm_runtime)
 
 if(BUILD_FOR_HEXAGON)
   # Wrap pthread_create to allow setting custom stack size.

CMakeLists.txt

@@ -37,7 +37,8 @@ LOCAL_SRC_FILES := org_apache_tvm_native_c_api.cc
 LOCAL_LDFLAGS := -L$(SYSROOT)/usr/lib/ -llog
 
 LOCAL_C_INCLUDES := $(ROOT_PATH)/include \
-                    $(ROOT_PATH)/3rdparty/dlpack/include \
+                    $(ROOT_PATH)/ffi/include \
+                    $(ROOT_PATH)/ffi/3rdparty/dlpack/include \
                     $(ROOT_PATH)/3rdparty/dmlc-core/include \
                     $(ROOT_PATH)/3rdparty/OpenCL-Headers
 

apps/android_rpc/app/src/main/jni/Android.mk

@@ -32,7 +32,16 @@
  * Android logcat.
  */
 #define TVM_LOG_CUSTOMIZE 1
+#define TVM_FFI_USE_LIBBACKTRACE 0
 
+#include "../ffi/src/ffi/container.cc"
+#include "../ffi/src/ffi/dtype.cc"
+#include "../ffi/src/ffi/error.cc"
+#include "../ffi/src/ffi/function.cc"
+#include "../ffi/src/ffi/ndarray.cc"
+#include "../ffi/src/ffi/object.cc"
+#include "../ffi/src/ffi/testing.cc"
+#include "../ffi/src/ffi/traceback.cc"
 #include "../src/runtime/c_runtime_api.cc"
 #include "../src/runtime/container.cc"
 #include "../src/runtime/cpu_device_api.cc"

apps/android_rpc/app/src/main/jni/tvm_runtime.h

@@ -126,46 +126,44 @@ RPCEnv::RPCEnv(const std::string& wd) {
     mkdir(base_.c_str(), 0777);
   }
 
-  TVM_REGISTER_GLOBAL("tvm.rpc.server.workpath").set_body([this](TVMArgs args, TVMRetValue* rv) {
-    *rv = this->GetPath(args[0]);
-  });
+  ffi::Function::SetGlobal(
+      "tvm.rpc.server.workpath",
+      ffi::Function::FromTyped([this](const std::string& path) { return this->GetPath(path); }));
 
-  TVM_REGISTER_GLOBAL("tvm.rpc.server.listdir").set_body([this](TVMArgs args, TVMRetValue* rv) {
-    std::string dir = this->GetPath(args[0]);
-    std::ostringstream os;
-    for (auto d : ListDir(dir)) {
-      os << d << ",";
-    }
-    *rv = os.str();
-  });
-
-  TVM_REGISTER_GLOBAL("tvm.rpc.server.load_module").set_body([this](TVMArgs args, TVMRetValue* rv) {
-    std::string file_name = this->GetPath(args[0]);
-    file_name = BuildSharedLibrary(file_name);
-    *rv = Module::LoadFromFile(file_name, "");
-    LOG(INFO) << "Load module from " << file_name << " ...";
-  });
+  ffi::Function::SetGlobal("tvm.rpc.server.listdir",
+                           ffi::Function::FromTyped([this](const std::string& path) {
+                             std::string dir = this->GetPath(path);
+                             std::ostringstream os;
+                             for (auto d : ListDir(dir)) {
+                               os << d << ",";
+                             }
+                             return os.str();
+                           }));
 
-  TVM_REGISTER_GLOBAL("tvm.rpc.server.download_linked_module")
-      .set_body([this](TVMArgs args, TVMRetValue* rv) {
-        std::string file_name = this->GetPath(args[0]);
-        file_name = BuildSharedLibrary(file_name);
-        std::string bin;
+  ffi::Function::SetGlobal("tvm.rpc.server.load_module",
+                           ffi::Function::FromTyped([this](const std::string& path) {
+                             std::string file_name = this->GetPath(path);
+                             file_name = BuildSharedLibrary(file_name);
+                             LOG(INFO) << "Load module from " << file_name << " ...";
+                             return Module::LoadFromFile(file_name, "");
+                           }));
 
-        std::ifstream fs(file_name, std::ios::in | std::ios::binary);
-        ICHECK(!fs.fail()) << "Cannot open " << file_name;
-        fs.seekg(0, std::ios::end);
-        size_t size = static_cast<size_t>(fs.tellg());
-        fs.seekg(0, std::ios::beg);
-        bin.resize(size);
-        fs.read(dmlc::BeginPtr(bin), size);
+  ffi::Function::SetGlobal("tvm.rpc.server.download_linked_module",
+                           ffi::Function::FromTyped([this](const std::string& path) {
+                             std::string file_name = this->GetPath(path);
+                             file_name = BuildSharedLibrary(file_name);
+                             std::string bin;
 
-        TVMByteArray binarr;
-        binarr.data = bin.data();
-        binarr.size = bin.length();
-        *rv = binarr;
-        LOG(INFO) << "Send linked module " << file_name << " to client";
-      });
+                             std::ifstream fs(file_name, std::ios::in | std::ios::binary);
+                             ICHECK(!fs.fail()) << "Cannot open " << file_name;
+                             fs.seekg(0, std::ios::end);
+                             size_t size = static_cast<size_t>(fs.tellg());
+                             fs.seekg(0, std::ios::beg);
+                             bin.resize(size);
+                             fs.read(dmlc::BeginPtr(bin), size);
+                             LOG(INFO) << "Send linked module " << file_name << " to client";
+                             return ffi::Bytes(bin);
+                           }));
 }
 /*!
  * \brief GetPath To get the work path from packed function

apps/cpp_rpc/rpc_env.cc

@@ -398,8 +398,6 @@ void RPCServerCreate(std::string host, int port, int port_end, std::string track
   rpc.Start();
 }
 
-TVM_REGISTER_GLOBAL("rpc.ServerCreate").set_body([](TVMArgs args, TVMRetValue* rv) {
-  RPCServerCreate(args[0], args[1], args[2], args[3], args[4], args[5], args[6], args[7]);
-});
+TVM_REGISTER_GLOBAL("rpc.ServerCreate").set_body_typed(RPCServerCreate);
 }  // namespace runtime
 }  // namespace tvm