Lint

Author: larryliu0820

CMakeLists.txt

@@ -19,7 +19,8 @@ project(Tokenizers)
 option(TOKENIZERS_BUILD_TEST "Build tests" OFF)
 option(TOKENIZERS_BUILD_TOOLS "Build tools" OFF)
 option(SUPPORT_REGEX_LOOKAHEAD
-       "Support regex lookahead patterns (requires PCRE2)" OFF)
+       "Support regex lookahead patterns (requires PCRE2)" OFF
+)
 
 # Include CMakePackageConfigHelpers for configure_package_config_file
 include(CMakePackageConfigHelpers)
@@ -50,22 +51,27 @@ set(tokenizers_source_files
     ${CMAKE_CURRENT_SOURCE_DIR}/src/regex.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/sentencepiece.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/tiktoken.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/src/token_decoder.cpp)
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/token_decoder.cpp
+)
 
 file(GLOB unicode_source_files
-     ${CMAKE_CURRENT_SOURCE_DIR}/third-party/llama.cpp-unicode/src/*.cpp)
-add_library(tokenizers STATIC ${tokenizers_source_files}
-                              ${unicode_source_files})
+     ${CMAKE_CURRENT_SOURCE_DIR}/third-party/llama.cpp-unicode/src/*.cpp
+)
+add_library(
+  tokenizers STATIC ${tokenizers_source_files} ${unicode_source_files}
+)
 
 # Using abseil from sentencepiece/third_party
 target_include_directories(
   tokenizers
-  PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
-         $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/third-party/sentencepiece>
-         $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/third-party/sentencepiece/src>
-         $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/third-party/re2>
-         $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/third-party/json/single_include>
-         $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/third-party/llama.cpp-unicode/include>)
+  PUBLIC
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/third-party/sentencepiece>
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/third-party/sentencepiece/src>
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/third-party/re2>
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/third-party/json/single_include>
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/third-party/llama.cpp-unicode/include>
+)
 target_link_libraries(tokenizers PUBLIC sentencepiece-static re2::re2)
 
 if(SUPPORT_REGEX_LOOKAHEAD)
@@ -82,18 +88,24 @@ if(SUPPORT_REGEX_LOOKAHEAD)
   add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/third-party/pcre2)
 
   # Set the INTERFACE_INCLUDE_DIRECTORIES property for pcre2-8-static
-  set_target_properties(pcre2-8-static PROPERTIES
-    INTERFACE_INCLUDE_DIRECTORIES $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/third-party/pcre2/src>
+  set_target_properties(
+    pcre2-8-static
+    PROPERTIES
+      INTERFACE_INCLUDE_DIRECTORIES
+      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/third-party/pcre2/src>
   )
   add_library(
     regex_lookahead STATIC
     ${CMAKE_CURRENT_SOURCE_DIR}/src/pcre2_regex.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/regex_lookahead.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/src/std_regex.cpp)
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/std_regex.cpp
+  )
   target_link_libraries(regex_lookahead PUBLIC pcre2-8)
   target_include_directories(
-    regex_lookahead PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
-                           $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/third-party/pcre2/src>)
+    regex_lookahead
+    PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+           $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/third-party/pcre2/src>
+  )
   target_link_options_shared_lib(regex_lookahead)
   target_link_libraries(tokenizers PUBLIC regex_lookahead)
   install(
@@ -126,13 +138,15 @@ install(
 install(
   DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/
   DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
-  FILES_MATCHING PATTERN "*.h"
+  FILES_MATCHING
+  PATTERN "*.h"
 )
 
 install(
   DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/third-party/json/single_include/
   DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
-  FILES_MATCHING PATTERN "*.hpp"
+  FILES_MATCHING
+  PATTERN "*.hpp"
 )
 
 # Install the CMake config files
@@ -151,7 +165,6 @@ configure_package_config_file(
   PATH_VARS CMAKE_INSTALL_INCLUDEDIR
 )
 
-install(
-  FILES ${CMAKE_CURRENT_BINARY_DIR}/tokenizers-config.cmake
-  DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/tokenizers
+install(FILES ${CMAKE_CURRENT_BINARY_DIR}/tokenizers-config.cmake
+        DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/tokenizers
 )

cmake/tokenizers-config.cmake.in

@@ -14,7 +14,10 @@ find_dependency(absl REQUIRED)
 # Directly include sentencepiece library
 set(SENTENCEPIECE_LIBRARY "${CMAKE_INSTALL_PREFIX}/lib64/libsentencepiece.a")
 if(NOT EXISTS "${SENTENCEPIECE_LIBRARY}")
-  message(FATAL_ERROR "Could not find sentencepiece library at ${SENTENCEPIECE_LIBRARY}")
+  message(
+    FATAL_ERROR
+      "Could not find sentencepiece library at ${SENTENCEPIECE_LIBRARY}"
+  )
 endif()
 
 # Include the exported targets file
@@ -25,16 +28,26 @@ set_and_check(TOKENIZERS_INCLUDE_DIRS "@PACKAGE_CMAKE_INSTALL_INCLUDEDIR@")
 
 # Add --whole-archive linker flag for tokenizers library
 if(APPLE)
-  set(TOKENIZERS_LINK_OPTIONS "SHELL:-force_load,$<TARGET_FILE:tokenizers::regex_lookahead>")
+  set(TOKENIZERS_LINK_OPTIONS
+      "SHELL:-force_load,$<TARGET_FILE:tokenizers::regex_lookahead>"
+  )
 elseif(MSVC)
-  set(TOKENIZERS_LINK_OPTIONS "SHELL:/WHOLEARCHIVE:$<TARGET_FILE:tokenizers::regex_lookahead>")
+  set(TOKENIZERS_LINK_OPTIONS
+      "SHELL:/WHOLEARCHIVE:$<TARGET_FILE:tokenizers::regex_lookahead>"
+  )
 else()
-  set(TOKENIZERS_LINK_OPTIONS "SHELL:LINKER:--whole-archive $<TARGET_FILE:tokenizers::regex_lookahead> LINKER:--no-whole-archive")
+  set(TOKENIZERS_LINK_OPTIONS
+      "SHELL:LINKER:--whole-archive $<TARGET_FILE:tokenizers::regex_lookahead> LINKER:--no-whole-archive"
+  )
 endif()
 
 # Set the libraries and link options
 set(TOKENIZERS_LIBRARIES tokenizers::tokenizers)
-set_property(TARGET tokenizers::tokenizers APPEND PROPERTY INTERFACE_LINK_OPTIONS "${TOKENIZERS_LINK_OPTIONS}")
+set_property(
+  TARGET tokenizers::tokenizers
+  APPEND
+  PROPERTY INTERFACE_LINK_OPTIONS "${TOKENIZERS_LINK_OPTIONS}"
+)
 
 # Check if the library was found
 check_required_components(tokenizers)

test/CMakeLists.txt

@@ -13,9 +13,7 @@ include(FindPackageHandleStandardArgs)
 include(FetchContent)
 include(ExternalProject)
 FetchContent_Declare(
-  tokenizers
-  SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/..
-  BUILD_ALWAYS ON
+  tokenizers SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/.. BUILD_ALWAYS ON
 )
 set(SUPPORT_REGEX_LOOKAHEAD ON)
 FetchContent_MakeAvailable(tokenizers)
@@ -29,7 +27,8 @@ FetchContent_Declare(
 )
 set(gtest_force_shared_crt
     ON
-    CACHE BOOL "" FORCE)
+    CACHE BOOL "" FORCE
+)
 FetchContent_MakeAvailable(googletest)
 
 file(GLOB test_source_files test_*.cpp)
@@ -41,9 +40,8 @@ foreach(test_source_file ${test_source_files})
   message(STATUS "Configuring unit test ${test_name}")
   add_executable(${test_name} ${test_source_file})
   target_include_directories(
-    ${test_name}
-    PRIVATE GTEST_INCLUDE_PATH
-            ${TOKENIZERS_INCLUDE_DIRS})
+    ${test_name} PRIVATE GTEST_INCLUDE_PATH ${TOKENIZERS_INCLUDE_DIRS}
+  )
   target_link_libraries(${test_name} PUBLIC gtest_main GTest::gmock tokenizers)
   add_test(${test_name} "${test_name}")
   set_tests_properties(${test_name} PROPERTIES ENVIRONMENT ${test_env})