Optionally use ICU

Author: guillaumekln

.travis.yml

@@ -3,6 +3,10 @@ language: cpp
 compiler:
   - gcc
   - clang
+env:
+  matrix:
+    - WITH_ICU="ON"
+    - WITH_ICU="OFF"
 cache:
   directories:
     - $HOME/googletest-release-1.8.0/
@@ -22,6 +26,7 @@ addons:
       - git
       - libboost-dev
       - libboost-program-options-dev
+      - libicu-dev
       - libprotobuf-c++
       - libprotobuf-dev
       - libtool
@@ -54,7 +59,7 @@ before_install:
     fi
 install:
   - mkdir build && cd build
-  - cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH=$ROOT_TRAVIS_DIR/sentencepiece -DGTEST_ROOT=$ROOT_TRAVIS_DIR/googletest-release-1.8.0/install/ ..
+  - cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH=$ROOT_TRAVIS_DIR/sentencepiece -DGTEST_ROOT=$ROOT_TRAVIS_DIR/googletest-release-1.8.0/install/ -DWITH_ICU=$WITH_ICU ..
   - make
 script:
   - test/onmt_tokenizer_test ../test/data

CMakeLists.txt

@@ -1,9 +1,11 @@
 cmake_minimum_required(VERSION 3.1.0)
 project(OpenNMTTokenizer)
 
+option(WITH_ICU "Compile with ICU" OFF)
 option(BUILD_SHARED_LIBS "Build shared libraries" ON)
 
 set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
 
 if(MSVC)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Wall")
@@ -35,12 +37,20 @@ set(SOURCES
   src/SpaceTokenizer.cc
   src/SubwordEncoder.cc
   src/Tokenizer.cc
-  src/unicode/Data.cc
   src/unicode/Unicode.cc
   )
 
 list(APPEND LINK_LIBRARIES "")
 
+if (WITH_ICU)
+  find_package(ICU REQUIRED)
+  add_definitions(-DWITH_ICU)
+  list(APPEND INCLUDE_DIRECTORIES ${ICU_INCLUDE_DIRS})
+  list(APPEND LINK_LIBRARIES ${ICU_LIBRARIES})
+else()
+  list(APPEND SOURCES src/unicode/Data.cc)
+endif()
+
 find_library(SP_LIBRARY NAMES sentencepiece)
 find_path(SP_INCLUDE_DIR NAMES sentencepiece_processor.h)