diff --git a/scripts/install-prereqs.bash b/scripts/install-prereqs.bash
index 1ae0fcf..4e7fa47 100755
--- a/scripts/install-prereqs.bash
+++ b/scripts/install-prereqs.bash
@@ -1,7 +1,7 @@
 #!/bin/bash -ex
 
-if [[ $TRAVIS_TAG && $TRAVIS_OTP_RELEASE != $MAIN_OTP ]]; then
-    echo Skip installation for non-primary tests of a tag-build
+if [[ $TRAVIS_TAG && ($TRAVIS_OTP_RELEASE != $MAIN_OTP || $REBAR_VSN -ne 3) ]]; then
+    echo Skip installation for non-primary test of a tag-build
     exit 0
 fi
 
diff --git a/src/dogstatsd.app.src b/src/dogstatsd.app.src
index e1dd1fe..6088bf6 100644
--- a/src/dogstatsd.app.src
+++ b/src/dogstatsd.app.src
@@ -16,5 +16,6 @@
   {licenses, ["LAGPL"]},
   {links, [{"GitHub", "https://github.com/WhoopInc/dogstatsde"}]},
   {pkg_name, "dogstatsde"},
+  {include_files, ["mix.exs"]},
   {build_tools, [<<"rebar3">>, <<"mix">>]}
  ]}.