libdeflate, cloudflare's zlib, versioned containers (#208)

* That is no longer the case, I encouraged @FiloSottile to bottle it up via a fork in FiloSottile/homebrew-musl-cross#13 (comment), so now it's fast to install (just a bottle pour) :)

* Update both GNU and MUSL containers, add libdeflate to both

* Re-add bzip2, make sure curl is cleaned up

* Add libdeflate sys crate (libdeflater), unsure on whether to make it optional or just ship it enabled by default in light of @juliangehring htslib benchmarks link

* Reverting changes to build.rs and Cargo.toml related to libdeflate in favor of PR #189. Leaving only docker-related setups

Author: brainstorm

Cargo.toml

@@ -37,7 +37,6 @@ default = ["bzip2", "lzma", "curl"]
 bzip2 = ["hts-sys/bzip2"]
 lzma = ["hts-sys/lzma"]
 curl = ["hts-sys/curl"]
-#openssl = ["hts-sys/openssl"]
 serde = ["serde_base", "serde_bytes"]
 static = []
 

Cross.toml

@@ -7,6 +7,6 @@ passthrough = [
 
 
 [target.x86_64-unknown-linux-musl]
-image = "brainstorm/cross-x86_64-unknown-linux-musl:latest"
+image = "brainstorm/cross-x86_64-unknown-linux-musl:1.0.0"
 [target.x86_64-unknown-linux-gnu]
-image = "brainstorm/cross-x86_64-unknown-linux-gnu:libcurl-openssl"
+image = "brainstorm/cross-x86_64-unknown-linux-gnu:1.0.0"

docker/Dockerfile.gnu

@@ -1,9 +1,11 @@
 FROM rustembedded/cross:x86_64-unknown-linux-gnu
 
-#ENV LIBCLANG_PATH /usr/lib/x86_64-linux-musl
 ENV LIBCLANG_PATH /usr/lib/llvm-10/lib
 ENV LLVM_CONFIG_PATH /usr/bin
 RUN apt-get update
-RUN apt-get install -y wget gnupg lsb-release software-properties-common apt-transport-https ca-certificates # Otherwise LLVM bump below fails
+RUN apt-get install -y build-essential wget gnupg lsb-release software-properties-common apt-transport-https ca-certificates # Otherwise LLVM bump below fails
 RUN bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)"
-RUN apt-get install -y libssl-dev libcurl4-openssl-dev zlib1g-dev libbz2-dev liblzma-dev # htslib deps
+RUN apt-get install -y libssl-dev libcurl4-openssl-dev zlib1g-dev libbz2-dev liblzma-dev # htslib deps
+RUN git clone --depth 1 https://github.com/ebiggers/libdeflate.git && \
+    cd libdeflate && make -j40 CFLAGS="-fPIC -O3" install && \
+    cd .. && rm -rf libdeflate

docker/Dockerfile.musl

@@ -26,7 +26,8 @@ RUN apt-get install -y wget gnupg lsb-release software-properties-common apt-tra
 # Autodetect and fetch latest LLVM repos for the current distro, avoids LLVM warnings and other issues, might generate slower builds for now though, see:
 # https://www.phoronix.com/scan.php?page=news_item&px=Rust-Hurt-On-LLVM-10
 RUN bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)"
-#RUN apt-get install -y libssl-dev libcurl4-openssl-dev zlib1g-dev libbz2-dev liblzma-dev musl musl-dev musl-tools # htslib deps
+#RUN apt-get install -y libssl-dev libcurl4-openssl-dev zlib1g-dev \
+#			libbz2-dev liblzma-dev musl musl-dev musl-tools # htslib deps
 
 # Remove pre-installed musl, to avoid cross-musl-make interference
 RUN apt-get remove -y musl
@@ -38,7 +39,8 @@ RUN wget https://github.com/richfelker/musl-cross-make/archive/v$MUSL_CROSS_MAKE
     && tar xvfz v$MUSL_CROSS_MAKE_VERSION.tar.gz
 WORKDIR /root/musl-cross-make-$MUSL_CROSS_MAKE_VERSION
 COPY config-musl-cross-make.mak config.mak
-RUN make -j40 install
+RUN make -j40 install && cd .. && rm v$MUSL_CROSS_MAKE_VERSION.tar.gz
+			       #&& rm -rf musl-cross-make-*
 
 # Now we assume we have a properly configured musl-cross...
 ENV PATH "/usr/local/musl/bin:$PATH"
@@ -52,24 +54,30 @@ ENV CPPFLAGS "-I/usr/local/musl/include -I/usr/local/include"
 ENV LDFLAGS "-L/usr/local/musl/lib -L/usr/local/lib"
 
 # .. and carry on with the htslib deps
-RUN wget https://www.zlib.net/zlib-$ZLIB_VERSION.tar.gz \
-    && tar xvfz zlib-$ZLIB_VERSION.tar.gz \
-    && cd zlib-$ZLIB_VERSION \
-    && ./configure --static --prefix=/usr/local/musl \
-    && make -j40 install
 WORKDIR /root
+RUN git clone --depth 1 https://github.com/ebiggers/libdeflate.git && \
+    cd libdeflate && make -j40 CFLAGS="-fPIC -O3" PREFIX="/usr/local/musl" install && \
+    cd .. && rm -rf libdeflate
+RUN git clone https://github.com/cloudflare/zlib cloudflare-zlib && \
+    cd cloudflare-zlib && ./configure --static --prefix=/usr/local/musl && \
+    make install && cd .. && rm -rf cloudflare-zlib
+#RUN wget https://www.zlib.net/zlib-$ZLIB_VERSION.tar.gz \
+#    && tar xvfz zlib-$ZLIB_VERSION.tar.gz \
+#    && cd zlib-$ZLIB_VERSION \
+#    && ./configure --static --prefix=/usr/local/musl \
+#    && make -j40 install
 RUN git clone git://sourceware.org/git/bzip2 \
     && cd bzip2 \
-    #&& make -j40 CC=$CC AR=$AR RANLIB=$RANLIB CFLAGS=$CFLAGS bzip2 \
+    && make -j40 CC=$CC AR=$AR RANLIB=$RANLIB CFLAGS=$CFLAGS bzip2 \
     && make PREFIX=/usr/local/musl -j40 bzip2 \
-    && make -j40 install
-WORKDIR /root
+    && make -j40 install && cd .. && rm -rf bzip2
 RUN wget https://tukaani.org/xz/xz-$LZMA_VERSION.tar.bz2 \
     && tar xvfj xz-$LZMA_VERSION.tar.bz2 \
     && cd xz-$LZMA_VERSION \
-    && ./configure --prefix=/usr/local/musl --enable-static --disable-shared --host x86_64-unknown-linux-musl \
-    && make -j40 install
-WORKDIR /root
+    && ./configure --prefix=/usr/local/musl \
+	--enable-static --disable-shared \
+	--host x86_64-unknown-linux-musl \
+    && make -j40 install && cd .. && rm -rf xz-$LZMA_VERSION xz-$LZMA_VERSION.tar.bz2
 
 # A few gems from: https://wiki.openssl.org/index.php/Compilation_and_Installation
 # "OpenSSL has been around a long time, and it carries around a lot of cruft"
@@ -91,20 +99,20 @@ RUN wget https://github.com/openssl/openssl/archive/OpenSSL_$OPENSSL_VERSION.tar
     && make AR=x86_64-linux-musl-ar \
             CC=x86_64-linux-musl-cc \
             -j40 \
-    && make RANLIB=x86_64-linux-musl-ranlib -j40 install
+    && make RANLIB=x86_64-linux-musl-ranlib -j40 install && cd .. \
+    && rm -rf OpenSSL_$OPENSSL_VERSION.tar.gz openssl-OpenSSL_$OPENSSL_VERSION
 
-WORKDIR /root
 RUN wget https://curl.haxx.se/download/curl-$CURL_VERSION.tar.gz \
     && tar xvfz curl-$CURL_VERSION.tar.gz && cd curl-$CURL_VERSION \
     && ./configure --prefix=/usr/local/musl --host x86_64-linux-musl \
         --with-ssl=/usr/local/musl --with-zlib=/usr/local/musl --enable-static \
-        --disable-dict --disable-file --disable-ftp --disable-gopher --disable-imap \
+        --disable-dict --disable-ftp --disable-gopher --disable-imap \
         --disable-pop3 --disable-rtsp --disable-smb --disable-smtp --disable-telnet \
         --disable-tftp --disable-ntlm --disable-ldap \
-        && make -j40 install
+        && make -j40 install && cd .. \
+	&& rm -rf curl-$CURL_VERSION.tar.gz curl-$CURL_VERSION
 
 # To cater Rust's openssl-sys needs...
-#ENV OPENSSL_DIR /usr/local/openssl
 ENV OPENSSL_DIR /usr/local/musl
 
 # Hack to force ld stick to musl on the hts-sys/build.rs side, otherwise:
@@ -118,4 +126,4 @@ RUN rm /usr/bin/ld && ln -sf /usr/local/musl/bin/x86_64-linux-musl-ld /usr/bin/l
 #    && . $HOME/.cargo/env \
 #    && rustup target add x86_64-unknown-linux-musl
 
-CMD ["bash"]
+CMD ["bash"]

docker/README.md

@@ -6,8 +6,8 @@ Allows to compile (rust-)htslib in a variety of environments and architectures v
 
 ```shell
 $ cd docker
-$ docker build -t brainstorm/cross-x86_64-unknown-linux-musl:libcurl-openssl . -f Dockerfile.musl
-$ docker build -t brainstorm/cross-x86_64-unknown-linux-gnu:libcurl-openssl . -f Dockerfile.gnu
+$ docker build -t brainstorm/cross-x86_64-unknown-linux-musl:1.0.0 . -f Dockerfile.musl
+$ docker build -t brainstorm/cross-x86_64-unknown-linux-gnu:1.0.0 . -f Dockerfile.gnu
 ```
 
 Then, to build and test rust-htslib with the above containers, proceed as you would with `cargo`, using `cross` instead, i.e:

hts-sys/Cargo.toml

@@ -27,12 +27,11 @@ curl-sys = { version = "0.4.31", optional = true, features = ["static-curl", "st
 default = ["bzip2", "lzma"]
 bzip2 = ["bzip2-sys"]
 lzma = ["lzma-sys"]
-#openssl = ["openssl-sys"]
 curl = ["curl-sys"]
 static = []
 
 [build-dependencies]
 fs-utils = "1.1"
 bindgen = { version = "0.53.2", default-features = false, features = ["runtime"] }
 cc = "1.0"
-glob = "0.3.0"
+glob = "0.3.0"

hts-sys/build.rs

@@ -73,6 +73,7 @@ fn main() {
     } else if let Ok(inc) = env::var("DEP_CURL_INCLUDE").map(PathBuf::from) {
         cfg.include(inc);
     }
+    
 
     let tool = cfg.get_compiler();
     let (cc_path, cflags_env) = (tool.path(), tool.cflags_env());