ARROW-8662: [CI] Consolidate appveyor scripts

Closes apache#7080 from kszucs/consolidate-appveyor-scripts Authored-by: Krisztián Szűcs <[email protected]> Signed-off-by: Neal Richardson <[email protected]>
Kyligence · May 15, 2020 · db1c49a · db1c49a
1 parent 1126b0c
commit db1c49a
Show file tree

Hide file tree

Showing 8 changed files with 168 additions and 701 deletions.
diff --git a/appveyor.yml b/appveyor.yml
@@ -23,25 +23,22 @@ only_commits:
   files:
     - appveyor.yml
     - ci/appveyor*
-    - ci/*.bat
     - ci/conda*
     - cpp/
     - format/
     - python/
 
 cache:
   - C:\Users\Appveyor\clcache1
-  - C:\msys64\home\appveyor\.ccache
-  - C:\Users\Appveyor\.cargo\registry
-  - '%LocalAppData%\\go-build'
 
 matrix:
   fast_finish: true
 
 environment:
   global:
     # Make these variables visible in all jobs and build steps
-    USE_CLCACHE: true
+    MSVC_DEFAULT_OPTIONS: ON
+    APPVEYOR_SAVE_CACHE_ON_ERROR: true
     # Change the clcache dir to reset caches everywhere when a setting
     # is changed incompatibly (e.g. CLCACHE_COMPRESS).
     CLCACHE_DIR: C:\Users\Appveyor\clcache1
@@ -60,39 +57,23 @@ environment:
     # (as generated by cmake)
     - JOB: "Build"
       GENERATOR: Ninja
-      CONFIGURATION: "Release"
       APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
     - JOB: "Toolchain"
       GENERATOR: Ninja
-      CONFIGURATION: "Release"
       ARROW_S3: "ON"
       ARROW_BUILD_FLIGHT: "ON"
       ARROW_BUILD_GANDIVA: "ON"
     - JOB: "Build_Debug"
       GENERATOR: Ninja
-      CONFIGURATION: "Debug"
-    # NOTE: Since ARROW-5403 we have disabled the static CRT build
-    # - JOB: "Static_Crt_Build"
-    #   GENERATOR: Ninja
-
-  MSVC_DEFAULT_OPTIONS: ON
-  APPVEYOR_SAVE_CACHE_ON_ERROR: true
-
-install:
-  - python ci\detect-changes.py > generated_changes.bat
-  # Populate ARROW_CI_* variables
-  - call generated_changes.bat
-  # Decide to exit if current job doesn't exercise affected topics
-  - call ci\appveyor-filter-changes.bat
 
 before_build:
-  - call ci\appveyor-install.bat
+  - call ci\appveyor-cpp-setup.bat
 
 build_script:
-  - call ci\appveyor-build.bat
+  - call ci\appveyor-cpp-build.bat
 
 # Disable test discovery
 test: off
 
 after_build:
-  - if "%USE_CLCACHE%" == "true" clcache -s
+  - clcache -s
diff --git a/ci/appveyor-build.bat b/ci/appveyor-build.bat
diff --git a/ci/appveyor-cpp-build.bat b/ci/appveyor-cpp-build.bat
@@ -17,114 +17,146 @@
 
 @echo on
 
-@rem In release mode, disable optimizations (/Od) for faster compiling
-set CMAKE_CXX_FLAGS_RELEASE=/Od
-
-if "%JOB%" == "Static_Crt_Build" (
-  @rem Since we link the CRT statically, we should also disable building
-  @rem the Arrow shared library to link the tests statically, otherwise
-  @rem the Arrow DLL and the tests end up using a different instance of
-  @rem the CRT, which wreaks havoc.
-
-  @rem ARROW-5403(wesm): Since changing to using gtest DLLs we can no
-  @rem longer run the unit tests because gtest.dll and the unit test
-  @rem executables have different static copies of the CRT
-
-  mkdir cpp\build-debug
-  pushd cpp\build-debug
-
-  cmake -G "%GENERATOR%" ^
-        -DARROW_VERBOSE_THIRDPARTY_BUILD=OFF ^
-        -DARROW_USE_STATIC_CRT=ON ^
-        -DARROW_BOOST_USE_SHARED=OFF ^
-        -DARROW_BUILD_SHARED=OFF ^
-        -DARROW_BUILD_TESTS=ON ^
-        -DARROW_BUILD_EXAMPLES=ON ^
-        -DCMAKE_BUILD_TYPE=Debug ^
-        -DARROW_ENABLE_TIMING_TESTS=OFF ^
-        -DARROW_TEST_LINKAGE=static ^
-        -DARROW_CXXFLAGS="/MP" ^
-        ..  || exit /B
+git config core.symlinks true
+git reset --hard
 
-  cmake --build . --config Debug || exit /B
-  ctest --output-on-failure -j2 || exit /B
-  popd
-  rmdir /S /Q cpp\build-debug
+@rem Retrieve git submodules, configure env var for Parquet unit tests
+git submodule update --init || exit /B
 
-  mkdir cpp\build-release
-  pushd cpp\build-release
-
-  cmake -G "%GENERATOR%" ^
-        -DARROW_VERBOSE_THIRDPARTY_BUILD=OFF ^
-        -DARROW_USE_STATIC_CRT=ON ^
-        -DARROW_BOOST_USE_SHARED=OFF ^
-        -DARROW_BUILD_SHARED=OFF ^
-        -DARROW_BUILD_TESTS=ON ^
-        -DARROW_BUILD_EXAMPLES=ON ^
-        -DCMAKE_BUILD_TYPE=Release ^
-        -DARROW_ENABLE_TIMING_TESTS=OFF ^
-        -DARROW_TEST_LINKAGE=static ^
-        -DCMAKE_CXX_FLAGS_RELEASE="/MT %CMAKE_CXX_FLAGS_RELEASE%" ^
-        -DARROW_CXXFLAGS="/WX /MP" ^
-        ..  || exit /B
-
-  cmake --build . --config Release || exit /B
-  ctest --output-on-failure -j2 || exit /B
-  popd
-
-  @rem Finish Static_Crt_Build build successfully
-  exit /B 0
-)
+set ARROW_TEST_DATA=%CD%\testing\data
+set PARQUET_TEST_DATA=%CD%\cpp\submodules\parquet-testing\data
 
+@rem
 @rem In the configurations below we disable building the Arrow static library
 @rem to save some time.  Unfortunately this will still build the Parquet static
 @rem library because of PARQUET-1420 (Thrift-generated symbols not exported in DLL).
-
+@rem
 if "%JOB%" == "Build_Debug" (
   mkdir cpp\build-debug
   pushd cpp\build-debug
 
   cmake -G "%GENERATOR%" ^
-        -DARROW_USE_PRECOMPILED_HEADERS=OFF ^
-        -DARROW_VERBOSE_THIRDPARTY_BUILD=OFF ^
         -DARROW_BOOST_USE_SHARED=OFF ^
-        -DARROW_BUILD_TESTS=ON ^
         -DARROW_BUILD_EXAMPLES=ON ^
-        -DCMAKE_BUILD_TYPE=%CONFIGURATION% ^
-        -DCMAKE_UNITY_BUILD=ON ^
         -DARROW_BUILD_STATIC=OFF ^
+        -DARROW_BUILD_TESTS=ON ^
         -DARROW_CXXFLAGS="/MP" ^
         -DARROW_ENABLE_TIMING_TESTS=OFF ^
-        ..  || exit /B
+        -DARROW_USE_PRECOMPILED_HEADERS=OFF ^
+        -DARROW_VERBOSE_THIRDPARTY_BUILD=OFF ^
+        -DCMAKE_BUILD_TYPE="Debug" ^
+        -DCMAKE_UNITY_BUILD=ON ^
+        .. || exit /B
 
-  cmake --build . --config %CONFIGURATION% || exit /B
+  cmake --build . --config Debug || exit /B
   ctest --output-on-failure -j2 || exit /B
   popd
 
   @rem Finish Debug build successfully
   exit /B 0
 )
 
-@rem Avoid Boost 1.70 because of https://github.com/boostorg/process/issues/85
-set CONDA_PACKAGES=--file=ci\conda_env_python.yml ^
-  python=%PYTHON% numpy=1.14 "boost-cpp<1.70"
+call activate arrow
 
-if "%ARROW_BUILD_GANDIVA%" == "ON" (
-  @rem Install llvmdev in the toolchain if building gandiva.dll
-  set CONDA_PACKAGES=%CONDA_PACKAGES% --file=ci\conda_env_gandiva.yml
-)
+@rem Use Boost from Anaconda
+set BOOST_ROOT=%CONDA_PREFIX%\Library
+set BOOST_LIBRARYDIR=%CONDA_PREFIX%\Library\lib
+
+@rem The "main" C++ build script for Windows CI
+@rem (i.e. for usual configurations)
 
 if "%JOB%" == "Toolchain" (
-  @rem Install pre-built "toolchain" packages for faster builds
-  set CONDA_PACKAGES=%CONDA_PACKAGES% --file=ci\conda_env_cpp.yml
+  set CMAKE_ARGS=-DARROW_DEPENDENCY_SOURCE=CONDA -DARROW_WITH_BZ2=ON
+) else (
+  @rem We're in a conda environment but don't want to use it for the dependencies
+  set CMAKE_ARGS=-DARROW_DEPENDENCY_SOURCE=AUTO
 )
 
-conda create -n arrow -q -y %CONDA_PACKAGES% -c conda-forge || exit /B
+@rem Enable warnings-as-errors
+set ARROW_CXXFLAGS=/WX /MP
 
-call activate arrow
+@rem
+@rem Build and test Arrow C++ libraries (including Parquet)
+@rem
 
-@rem Use Boost from Anaconda
-set BOOST_ROOT=%CONDA_PREFIX%\Library
-set BOOST_LIBRARYDIR=%CONDA_PREFIX%\Library\lib
+mkdir cpp\build
+pushd cpp\build
+
+@rem XXX Without forcing CMAKE_CXX_COMPILER, CMake can re-run itself and
+@rem unfortunately switch from Release to Debug mode...
+@rem
+@rem In release mode, disable optimizations (/Od) for faster compiling
+@rem and enable runtime assertions.
+
+cmake -G "%GENERATOR%" %CMAKE_ARGS% ^
+      -DARROW_BOOST_USE_SHARED=OFF ^
+      -DARROW_BUILD_EXAMPLES=ON ^
+      -DARROW_BUILD_STATIC=OFF ^
+      -DARROW_BUILD_TESTS=ON ^
+      -DARROW_CSV=ON ^
+      -DARROW_CXXFLAGS="%ARROW_CXXFLAGS%" ^
+      -DARROW_DATASET=ON ^
+      -DARROW_ENABLE_TIMING_TESTS=OFF ^
+      -DARROW_FLIGHT=%ARROW_BUILD_FLIGHT% ^
+      -DARROW_GANDIVA=%ARROW_BUILD_GANDIVA% ^
+      -DARROW_MIMALLOC=ON ^
+      -DARROW_PARQUET=ON ^
+      -DARROW_PYTHON=ON ^
+      -DARROW_S3=%ARROW_S3% ^
+      -DARROW_VERBOSE_THIRDPARTY_BUILD=OFF ^
+      -DARROW_WITH_BROTLI=ON ^
+      -DARROW_WITH_LZ4=ON ^
+      -DARROW_WITH_SNAPPY=ON ^
+      -DARROW_WITH_ZLIB=ON ^
+      -DARROW_WITH_ZSTD=ON ^
+      -DCMAKE_BUILD_TYPE="Release" ^
+      -DCMAKE_CXX_COMPILER=clcache ^
+      -DCMAKE_CXX_FLAGS_RELEASE="/MD /Od /UNDEBUG" ^
+      -DCMAKE_INSTALL_PREFIX=%CONDA_PREFIX%\Library ^
+      -DCMAKE_UNITY_BUILD=ON ^
+      -DCMAKE_VERBOSE_MAKEFILE=OFF ^
+      -DPARQUET_BUILD_EXECUTABLES=ON ^
+      -DPARQUET_REQUIRE_ENCRYPTION=ON ^
+      ..  || exit /B
+cmake --build . --target install --config %CONFIGURATION%  || exit /B
+
+@rem Needed so arrow-python-test.exe works
+set OLD_PYTHONHOME=%PYTHONHOME%
+set PYTHONHOME=%CONDA_PREFIX%
+
+ctest --output-on-failure -j2 || exit /B
+
+set PYTHONHOME=%OLD_PYTHONHOME%
+popd
+
+@rem
+@rem Build and install pyarrow
+@rem
+
+pushd python
+
+set PYARROW_BUNDLE_BOOST=OFF
+set PYARROW_CMAKE_GENERATOR=%GENERATOR%
+set PYARROW_CXXFLAGS=%ARROW_CXXFLAGS%
+set PYARROW_PARALLEL=2
+set PYARROW_WITH_DATASET=ON
+set PYARROW_WITH_FLIGHT=%ARROW_BUILD_FLIGHT%
+set PYARROW_WITH_GANDIVA=%ARROW_BUILD_GANDIVA%
+set PYARROW_WITH_PARQUET=ON
+set PYARROW_WITH_S3=%ARROW_S3%
+set PYARROW_WITH_STATIC_BOOST=ON
+
+set ARROW_HOME=%CONDA_PREFIX%\Library
+@rem ARROW-3075; pkgconfig is broken for Parquet for now
+set PARQUET_HOME=%CONDA_PREFIX%\Library
+
+python setup.py develop -q || exit /B
+
+set PYTHONDEVMODE=1
+
+py.test -r sxX --durations=15 --pyargs pyarrow.tests || exit /B
+
+@rem
+@rem Wheels are built and tested separately (see ARROW-5142).
+@rem
 
-call ci\cpp-msvc-build-main.bat