Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft: Android CI production build and Play Store auto-submission #3392

Draft
wants to merge 2 commits into
base: main
Choose a base branch
from
Draft

Draft: Android CI production build and Play Store auto-submission #3392

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
17b339a
Android: CI changes for Play Store
danryu Oct 12, 2022
3ef15a2
Document feature branch
ann0see Oct 28, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
245 changes: 169 additions & 76 deletions .github/autobuild/android.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,26 +26,51 @@

set -eu

# Some of the following version pinnings are semi-automatically checked for
# updates. Update .github/workflows/bump-dependencies.yaml when renaming those:
COMMANDLINETOOLS_VERSION=6858069
ANDROID_NDK_VERSION=r21d
ANDROID_PLATFORM=android-30
ANDROID_BUILD_TOOLS=30.0.2
AQTINSTALL_VERSION=3.1.18
QT_VERSION=5.15.2

# Only variables which are really needed by sub-commands are exported.
# Definitions have to stay in a specific order due to dependencies.
## TODO: Decide whether we want to use this action.
## TODO: The patch not only adopts the new action but changes dependencies:
## TODO: - COMMANDLINETOOLS_VERSION from 6858069 to 7.0
## TODO: - ANDROID_NDK_VERSION from r21d to 25.1.8937393
## TODO: - ANDROID_PLATFORM from android-30 to android-33
## TODO: - ANDROID_BUILD_TOOLS from 30.0.2 to 33.0.0
## TODO: - AQTINSTALL_VERSION from 3.1.18 to 2.1.0
## TODO: - QT_VERSION from 5.15.2 to 6.3.2

# # Some of the following version pinnings are semi-automatically checked for
# # updates. Update .github/workflows/bump-dependencies.yaml when renaming those:
# COMMANDLINETOOLS_VERSION=6858069
# ANDROID_NDK_VERSION=r21d
# ANDROID_PLATFORM=android-30
# ANDROID_BUILD_TOOLS=30.0.2
# AQTINSTALL_VERSION=3.1.18
# QT_VERSION=5.15.2

## From https://github.com/actions/runner-images/blob/main/images/linux/Ubuntu2204-Readme.md
# Tools already installed:
# Android Command Line Tools 7.0
# Android SDK Build-tools 33.0.0
# Android SDK Platform-Tools 33.0.3
# Android SDK Platforms android-33 (rev 2)
# Android SDK Tools 26.1.1

# Env vars set:
# ANDROID_HOME /usr/local/lib/android/sdk
# ANDROID_NDK /usr/local/lib/android/sdk/ndk/25.1.8937393
# ANDROID_NDK_HOME /usr/local/lib/android/sdk/ndk/25.1.8937393
# ANDROID_NDK_LATEST_HOME /usr/local/lib/android/sdk/ndk/25.1.8937393
# ANDROID_NDK_ROOT /usr/local/lib/android/sdk/ndk/25.1.8937393
# ANDROID_SDK_ROOT /usr/local/lib/android/sdk

ANDROID_PLATFORM=android-33
AQTINSTALL_VERSION=2.1.0
QT_VERSION=6.3.2
QT_BASEDIR="/opt/Qt"
ANDROID_BASEDIR="/opt/android"
BUILD_DIR=build
export ANDROID_SDK_ROOT="${ANDROID_BASEDIR}/android-sdk"
COMMANDLINETOOLS_DIR="${ANDROID_SDK_ROOT}"/cmdline-tools/latest/
export ANDROID_NDK_ROOT="${ANDROID_BASEDIR}/android-ndk"
ANDROID_NDK_HOST="linux-x86_64"
ANDROID_SDKMANAGER="${COMMANDLINETOOLS_DIR}/bin/sdkmanager"
export JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64/"
# Github CI image-provided env vars, but explicitly re-assign to placate shellcheck
ANDROID_NDK_ROOT="${ANDROID_NDK_ROOT:?ANDROID_NDK_ROOT should be provided}"
ANDROID_SDK_ROOT="${ANDROID_SDK_ROOT:?ANDROID_SDK_ROOT should be provided}"
# Only variables which are really needed by sub-commands are exported.
export JAVA_HOME=${JAVA_HOME_11_X64}
export PATH="${PATH}:${ANDROID_SDK_ROOT}/tools"
export PATH="${PATH}:${ANDROID_SDK_ROOT}/platform-tools"

Expand All @@ -58,36 +83,7 @@ setup_ubuntu_dependencies() {
export DEBIAN_FRONTEND="noninteractive"

sudo apt-get -qq update
sudo apt-get -qq --no-install-recommends -y install build-essential zip unzip bzip2 p7zip-full curl chrpath openjdk-8-jdk-headless
}

setup_android_sdk() {
mkdir -p "${ANDROID_BASEDIR}"

if [[ -d "${COMMANDLINETOOLS_DIR}" ]]; then
echo "Using commandlinetools installation from previous run (actions/cache)"
else
mkdir -p "${COMMANDLINETOOLS_DIR}"
curl -s -o downloadfile "https://dl.google.com/android/repository/commandlinetools-linux-${COMMANDLINETOOLS_VERSION}_latest.zip"
unzip -q downloadfile
mv cmdline-tools/* "${COMMANDLINETOOLS_DIR}"
fi

yes | "${ANDROID_SDKMANAGER}" --licenses
"${ANDROID_SDKMANAGER}" --update
"${ANDROID_SDKMANAGER}" "platforms;${ANDROID_PLATFORM}"
"${ANDROID_SDKMANAGER}" "build-tools;${ANDROID_BUILD_TOOLS}"
}

setup_android_ndk() {
mkdir -p "${ANDROID_BASEDIR}"
if [[ -d "${ANDROID_NDK_ROOT}" ]]; then
echo "Using NDK installation from previous run (actions/cache)"
else
curl -s -o downloadfile "https://dl.google.com/android/repository/android-ndk-${ANDROID_NDK_VERSION}-linux-x86_64.zip"
unzip -q downloadfile
mv "android-ndk-${ANDROID_NDK_VERSION}" "${ANDROID_NDK_ROOT}"
fi
sudo apt-get -qq --no-install-recommends -y install build-essential zip unzip bzip2 p7zip-full curl chrpath
}

setup_qt() {
Expand All @@ -96,54 +92,151 @@ setup_qt() {
else
echo "Installing Qt..."
python3 -m pip install "aqtinstall==${AQTINSTALL_VERSION}"
local qtmultimedia=()
if [[ ! "${QT_VERSION}" =~ 5\..* ]]; then
# From Qt6 onwards, qtmultimedia is a module and cannot be installed
# as an archive anymore.
qtmultimedia=("--modules")
fi
qtmultimedia+=("qtmultimedia")

python3 -m aqt install-qt --outputdir "${QT_BASEDIR}" linux android "${QT_VERSION}" \
--archives qtbase qttools qttranslations qtandroidextras \
"${qtmultimedia[@]}"
# Delete libraries, which we don't use, but which bloat the resulting package and might introduce unwanted dependencies.
find "${QT_BASEDIR}" -name 'libQt5*Quick*.so' -delete
rm -r "${QT_BASEDIR}/${QT_VERSION}/android/qml/"
# icu needs explicit installation
# otherwise: "qmake: error while loading shared libraries: libicui18n.so.56: cannot open shared object file: No such file or directory"
python3 -m aqt install-qt --outputdir "${QT_BASEDIR}" linux desktop "${QT_VERSION}" \
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

New dependencies need documenting.

--archives qtbase qtdeclarative qtsvg qttools icu \
--modules qtmultimedia

# - 64bit required for Play Store
python3 -m aqt install-qt --outputdir "${QT_BASEDIR}" linux android "${QT_VERSION}" android_arm64_v8a \
--archives qtbase qtdeclarative qtsvg qttools \
--modules qtmultimedia

# Also install for arm_v7 to build for 32bit devices
python3 -m aqt install-qt --outputdir "${QT_BASEDIR}" linux android "${QT_VERSION}" android_armv7 \
--archives qtbase qtdeclarative qtsvg qttools \
--modules qtmultimedia

fi
}

build_app_as_apk() {
local QT_DIR="${QT_BASEDIR}/${QT_VERSION}/android"
build_app() {
local ARCH_ABI="${1}"

local MAKE="${ANDROID_NDK_ROOT}/prebuilt/${ANDROID_NDK_HOST}/bin/make"

"${QT_DIR}/bin/qmake" -spec android-clang
echo "${GOOGLE_RELEASE_KEYSTORE}" | base64 --decode > android/android_release.keystore

echo ">>> Compiling for ${ARCH_ABI} ..."

# Override ANDROID_ABIS according to build target
# note: seems ANDROID_ABIS can be set here at cmdline, but ANDROID_VERSION_CODE cannot - must be in qmake file
if [ "${ARCH_ABI}" == "android_armv7" ]; then
echo ">>> Running qmake with ANDROID_ABIS=armeabi-v7a ..."
ANDROID_ABIS=armeabi-v7a \
"${QT_BASEDIR}/${QT_VERSION}/${ARCH_ABI}/bin/qmake" -spec android-clang
elif [ "${ARCH_ABI}" == "android_arm64_v8a" ]; then
echo ">>> Running qmake with ANDROID_ABIS=arm64-v8a ..."
ANDROID_ABIS=arm64-v8a \
"${QT_BASEDIR}/${QT_VERSION}/${ARCH_ABI}/bin/qmake" -spec android-clang
elif [ "${ARCH_ABI}" == "android_x86" ]; then
echo ">>> Running qmake with ANDROID_ABIS=arm64-v8a ..."
ANDROID_ABIS=x86 \
"${QT_BASEDIR}/${QT_VERSION}/${ARCH_ABI}/bin/qmake" -spec android-clang
elif [ "${ARCH_ABI}" == "android_x86_64" ]; then
echo ">>> Running qmake with ANDROID_ABIS=arm64-v8a ..."
ANDROID_ABIS=x86_64 \
"${QT_BASEDIR}/${QT_VERSION}/${ARCH_ABI}/bin/qmake" -spec android-clang
fi
"${MAKE}" -j "$(nproc)"
"${MAKE}" INSTALL_ROOT="${BUILD_DIR}" -f Makefile install
"${QT_DIR}"/bin/androiddeployqt --input android-Jamulus-deployment-settings.json --output "${BUILD_DIR}" \
--android-platform "${ANDROID_PLATFORM}" --jdk "${JAVA_HOME}" --gradle
"${MAKE}" INSTALL_ROOT="${BUILD_DIR}_${ARCH_ABI}" -f Makefile install
}

build_make_clean() {
echo ">>> Doing make clean ..."
local MAKE="${ANDROID_NDK_ROOT}/prebuilt/${ANDROID_NDK_HOST}/bin/make"
"${MAKE}" clean
rm -f Makefile
}

build_aab() {
local ARCH_ABI="${1}"

if [ "${ARCH_ABI}" == "android_armv7" ]; then
TARGET_ABI=armeabi-v7a
elif [ "${ARCH_ABI}" == "android_arm64_v8a" ]; then
TARGET_ABI=arm64-v8a
elif [ "${ARCH_ABI}" == "android_x86" ]; then
TARGET_ABI=x86
elif [ "${ARCH_ABI}" == "android_x86_64" ]; then
TARGET_ABI=x86_64
fi
echo ">>> Building .aab file for ${TARGET_ABI}...."

ANDROID_ABIS=${TARGET_ABI} ${QT_BASEDIR}/${QT_VERSION}/gcc_64/bin/androiddeployqt --input android-Jamulus-deployment-settings.json \
--verbose \
--output "${BUILD_DIR}_${ARCH_ABI}" \
--aab \
--release \
--sign android/android_release.keystore jamulus \
--storepass "${GOOGLE_KEYSTORE_PASS}" \
--android-platform "${ANDROID_PLATFORM}" \
--jdk "${JAVA_HOME}" \
--gradle
}

pass_artifact_to_job() {
mkdir deploy
local artifact="jamulus_${JAMULUS_BUILD_VERSION}_android.apk"
echo "Moving ${BUILD_DIR}/build/outputs/apk/debug/build-debug.apk to deploy/${artifact}"
mv "./${BUILD_DIR}/build/outputs/apk/debug/build-debug.apk" "./deploy/${artifact}"
echo "artifact_1=${artifact}" >> "$GITHUB_OUTPUT"
local ARCH_ABI="${1}"
echo ">>> Deploying .aab file for ${ARCH_ABI}...."

if [ "${ARCH_ABI}" == "android_armv7" ]; then
NUM="1"
BUILDNAME="arm"
elif [ "${ARCH_ABI}" == "android_arm64_v8a" ]; then
NUM="2"
BUILDNAME="arm64"
elif [ "${ARCH_ABI}" == "android_x86" ]; then
NUM="3"
BUILDNAME="x86"
elif [ "${ARCH_ABI}" == "android_x86_64" ]; then
NUM="4"
BUILDNAME="x86_64"
fi

#mkdir deploy
#local artifact="jamulus_${JAMULUS_BUILD_VERSION}_android.apk"
mkdir -p deploy
local artifact="Jamulus_${JAMULUS_BUILD_VERSION}_android_${BUILDNAME}.aab"
# debug to check for filenames
ls -alR "${BUILD_DIR}_${ARCH_ABI}/build/outputs/bundle/release/"
ls -al "${BUILD_DIR}_${ARCH_ABI}/build/outputs/bundle/release/build_${ARCH_ABI}-release.aab"

#echo "Moving ${BUILD_DIR}/build/outputs/apk/debug/build-debug.apk to deploy/${artifact}"
#mv "./${BUILD_DIR}/build/outputs/apk/debug/build-debug.apk" "./deploy/${artifact}"
echo ">>> Moving ${BUILD_DIR}_${ARCH_ABI}/build/outputs/bundle/release/build_${ARCH_ABI}-release.aab to deploy/${artifact}"
mv "./${BUILD_DIR}_${ARCH_ABI}/build/outputs/bundle/release/build_${ARCH_ABI}-release.aab" "./deploy/${artifact}"
echo ">>> Moved .aab file to deploy/${artifact}"
echo ">>> Artifact number is: ${NUM}"
echo ">>> Setting output as such: name=artifact_${NUM}::${artifact}"
#echo "artifact_1=${artifact}" >> "$GITHUB_OUTPUT"
echo "::set-output name=artifact_${NUM}::${artifact}"
}

case "${1:-}" in
setup)
setup_ubuntu_dependencies
setup_android_ndk
setup_android_sdk
setup_qt
;;
build)
build_app_as_apk
# Build all targets in sequence
build_app "android_armv7"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It would be neat to parallelize builds of possible. Android takes ages already.

build_aab "android_armv7"
build_make_clean
build_app "android_arm64_v8a"
build_aab "android_arm64_v8a"
build_make_clean
build_app "android_x86"
build_aab "android_x86"
build_make_clean
build_app "android_x86_64"
build_aab "android_x86_64"
;;
get-artifacts)
pass_artifact_to_job
pass_artifact_to_job "android_armv7"
pass_artifact_to_job "android_arm64_v8a"
pass_artifact_to_job "android_x86"
pass_artifact_to_job "android_x86_64"
;;
*)
echo "Unknown stage '${1:-}'"
Expand Down
22 changes: 22 additions & 0 deletions .github/workflows/autobuild.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -377,6 +377,9 @@ jobs:
NOTARIZATION_PASSWORD: ${{ secrets.NOTARIZATION_PASSWORD }}
KEYCHAIN_PASSWORD: ${{ secrets.KEYCHAIN_PASSWORD }}
MACOS_CA_PUBLICKEY: ${{ secrets.MACOS_CA_PUBKEY }}
GOOGLE_RELEASE_KEYSTORE: ${{ secrets.GOOGLE_KEYSTORE }}
GOOGLE_KEYSTORE_PASS: ${{ secrets.GOOGLE_KEYSTORE_PASS }}

- name: Post-Build for ${{ matrix.config.config_name }}
id: get-artifacts
run: ${{ matrix.config.base_command }} get-artifacts
Expand Down Expand Up @@ -427,6 +430,25 @@ jobs:
JAMULUS_BUILD_VERSION: ${{ needs.create_release.outputs.build_version }}
ARTIFACT_PATH: deploy/${{ steps.get-artifacts.outputs.artifact_1 }}

## RELEASE PROCEDURE FOR:
## - Android Play Store - aab
## Requirement: Service Account JSON setup:
## - Google Play Console -> Setup -> API Access -> Create/Link Google Cloud Project
## - Google Cloud console -> IAM & Admin -> Service Accounts -> Create (Wizard). Then create JSON key and export/save
- name: Publish all Android ABI builds to Play Store
if: >-
needs.create_release.outputs.publish_to_release == 'true' &&
matrix.config.target_os == 'android'
id: publish_android
uses: r0adkll/upload-google-play@v1
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We need to review this action and confirm it's okay.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would rather check if it's possible like for iOS to use native cli tools or an official action by google.

with:
serviceAccountJsonPlainText: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_JSON }}
packageName: io.jamulus.jamulus
releaseFiles: deploy/Jamulus*.aab
releaseName: ${{ needs.create_release.outputs.build_version }}
track: beta
status: draft

- name: Deploy Artifact 1 to Release
if: needs.create_release.outputs.publish_to_release == 'true'
id: upload-release-asset1
Expand Down
43 changes: 39 additions & 4 deletions Jamulus.pro
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -244,18 +244,53 @@ win32 {
LIBS += -framework AVFoundation \
-framework AudioToolbox
} else:android {
ANDROID_ABIS = armeabi-v7a arm64-v8a x86 x86_64
# By default build for all the ABIs
# ANDROID_ABIS = armeabi-v7a arm64-v8a x86 x86_64

# get ANDROID_ABIS from environment - passed directly to qmake
ANDROID_ABIS = $$getenv(ANDROID_ABIS)
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

From where will these come when running make distclean; qmake ...?

There should be a set of defaults that can be overridden from the build chain.


# Optional: if ANDROID_ABIS is passed as env var to qmake, will override this
# !defined(ANDROID_ABIS, var):ANDROID_ABIS = arm64-v8a

# Experiments show likely better results with Android 10+ devices
ANDROID_MIN_SDK_VERSION = 29
ANDROID_TARGET_SDK_VERSION = 32
ANDROID_VERSION_NAME = $$VERSION
ANDROID_VERSION_CODE = $$system(git log --oneline | wc -l)
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This should default to 1 and the build chain should pass the correct value:

  • 1 for anything but a release build (as we will not publish to Google Store anything but release builds).
  • retrieve last_release_android_version from permanent storage, add the number of ABI builds to be released plus one, store that in last_release_android_version permanent storage and pass it to the build.


## For local Dev use on Windows/WSA:
equals(QMAKE_HOST.os, Windows) {
ANDROID_ABIS = x86_64
ANDROID_VERSION_CODE = 1234 # dummy int value
} else {
# date-based unique integer value for Play Store submission
!defined(ANDROID_VERSION_CODE, var):ANDROID_VERSION_CODE = $$system(date +%s | cut -c 2-)
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

See above - I'd rather remove this entirely and default to 1 and use the

 ANDROID_VERSION_CODE = $$num_add($$ANDROID_VERSION_CODE, ...)

approach.

}

# make separate version codes for each abi build otherwise Play Store rejects
contains (ANDROID_ABIS, armeabi-v7a) {
ANDROID_VERSION_CODE = $$num_add($$ANDROID_VERSION_CODE, 1)
message("Setting for armeabi-v7a: ANDROID_VERSION_CODE=$${ANDROID_VERSION_CODE}")
}
contains (ANDROID_ABIS, x86) {
ANDROID_VERSION_CODE = $$num_add($$ANDROID_VERSION_CODE, 2)
message("Setting for x86: ANDROID_VERSION_CODE=$${ANDROID_VERSION_CODE}")
}
contains (ANDROID_ABIS, x86_64) {
ANDROID_VERSION_CODE = $$num_add($$ANDROID_VERSION_CODE, 3)
message("Setting for x86_64: ANDROID_VERSION_CODE=$${ANDROID_VERSION_CODE}")
}

message("Setting ANDROID_VERSION_NAME=$${ANDROID_VERSION_NAME} ANDROID_VERSION_CODE=$${ANDROID_VERSION_CODE}")

# liboboe requires C++17 for std::timed_mutex
CONFIG += c++17

QT += androidextras
# For device recording permissions
QT += core-private # for Qt6

# enabled only for debugging on android devices
DEFINES += ANDROIDDEBUG
#DEFINES += ANDROIDDEBUG
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Are we sure we want to remove this?


target.path = /tmp/your_executable # path on device
INSTALLS += target
Expand Down
4 changes: 4 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@

[![Auto-Build](https://github.com/jamulussoftware/jamulus/actions/workflows/autobuild.yml/badge.svg)](https://github.com/jamulussoftware/jamulus/actions/workflows/autobuild.yml)

## Contents of this feature branch

This branch contains changes to the CI to automatically upload the Android build to the PlayStore. For more information see danryu's Pull Request: [#2909](https://github.com/jamulussoftware/jamulus/pull/2909). This is just a PoC and needs further work. Further development can be based on this branch.

# Jamulus - Internet Jam Session Software

<a href="https://jamulus.io/"><img align="left" width="102" height="102" src="https://jamulus.io/assets/img/jamulus-icon-2020.svg"/></a>
Expand Down
Loading
Loading