[Feature:System] installation on Mac M1/ARM using UTM (Submitty#6815)

* remove java from distro_setup * remove java images from default container * leave ubuntu 16.04 an 18.04 unchanged * put java 11 back in default * utm flag & remove 32 bit * progress * finished running installation script * update notes * revisions * fix C++ json libary stuff * instructions * replace json.hpp with shared repo checkout version * more changes * update * tweak * tweak * revert changes * Revert "revert changes" This reverts commit 8af1acb. * tweak instructions * replace deprecated include for calculate_extensions * edits * refactor * refactor * fix undefined index * Fix bug * updates May 2022 * documentation * documentation Co-authored-by: Eddie Krystowski <[email protected]> Co-authored-by: Barb Cutler <Barb Cutler> Co-authored-by: williamjallen <[email protected]>
ziesski · May 13, 2022 · d2a74cb · d2a74cb
1 parent d67db09
commit d2a74cb
Show file tree

Hide file tree

Showing 10 changed files with 311 additions and 40 deletions.
diff --git a/.setup/INSTALL_SUBMITTY_HELPER.sh b/.setup/INSTALL_SUBMITTY_HELPER.sh
@@ -32,6 +32,15 @@ if [ -d ${THIS_DIR}/../.vagrant ]; then
     VAGRANT=1
 fi
 
+
+#
+# SEE GITHUB ISSUE #7885 - https://github.com/Submitty/Submitty/issues/7885
+UTM_ARM=0
+if [[ "$(uname -m)" = "aarch64" ]] ; then
+    UTM_ARM=1
+fi
+
+
 SUBMITTY_REPOSITORY=$(jq -r '.submitty_repository' ${CONF_DIR}/submitty.json)
 SUBMITTY_INSTALL_DIR=$(jq -r '.submitty_install_dir' ${CONF_DIR}/submitty.json)
 WORKER=$([[ $(jq -r '.worker' ${CONF_DIR}/submitty.json) == "true" ]] && echo 1 || echo 0)
@@ -46,6 +55,18 @@ else
     RESTART_DAEMONS=( )
 fi
 
+########################################################################################################################
+########################################################################################################################
+# FORCE CORRECT TIME SKEW
+# This may happen on a development virtual machine
+# SEE GITHUB ISSUE #7885 - https://github.com/Submitty/Submitty/issues/7885
+if [ ${UTM_ARM} == 1 ]; then
+    sudo service ntp stop
+    sudo ntpd -gq
+    sudo service ntp start
+    sudo timedatectl set-timezone America/New_York
+fi
+
 ########################################################################################################################
 ########################################################################################################################
 # this script must be run by root or sudo
@@ -540,6 +561,11 @@ echo -e "Compile and install analysis tools"
 
 mkdir -p ${SUBMITTY_INSTALL_DIR}/SubmittyAnalysisTools
 
+# SEE GITHUB ISSUE #7885 - https://github.com/Submitty/Submitty/issues/7885
+# HASKELL BINARY IS NOT AVAILABLE ARM64
+if [ ${UTM_ARM} == 0 ]; then
+# END ARM64
+
 pushd ${SUBMITTY_INSTALL_DIR}/SubmittyAnalysisTools
 if [[ ! -f VERSION || $(< VERSION) != "${AnalysisTools_Version}" ]]; then
     for b in count plagiarism diagnostics;
@@ -550,6 +576,13 @@ if [[ ! -f VERSION || $(< VERSION) != "${AnalysisTools_Version}" ]]; then
 fi
 popd > /dev/null
 
+# SEE GITHUB ISSUE #7885 - https://github.com/Submitty/Submitty/issues/7885
+# HASKELL BINARY IS NOT AVAILABLE ARM64
+else
+    echo "SKIPPING ANALYSIS TOOLS INSTALL ON UTM ARM 64"
+fi
+# END ARM64
+
 # change permissions
 chown -R ${DAEMON_USER}:${COURSE_BUILDERS_GROUP} ${SUBMITTY_INSTALL_DIR}/SubmittyAnalysisTools
 chmod -R 555 ${SUBMITTY_INSTALL_DIR}/SubmittyAnalysisTools

diff --git a/.setup/distro_setup/setup_distro.sh b/.setup/distro_setup/setup_distro.sh
@@ -39,7 +39,9 @@ fi
 # check if set, else use default /etc/motd
 # we expect that SUBMISSION_URL and GIT_URL to be set above
 
-if [ ${VAGRANT} == 1 ]; then
+# SEE GITHUB ISSUE #7885 - https://github.com/Submitty/Submitty/issues/7885
+
+if [ ${VAGRANT} == 1] && [ ${UTM_ARM} == 0]; then
     # Ubuntu/Debian share this stuff, CentOS does not
     if [ -d /etc/update-motd.d ]; then
         chmod -x /etc/update-motd.d/*

diff --git a/.setup/distro_setup/ubuntu/20.04/setup_distro.sh b/.setup/distro_setup/ubuntu/20.04/setup_distro.sh
@@ -66,7 +66,7 @@ apt-get install -qqy scrot
 
 apt-get install -qqy clang autoconf automake autotools-dev diffstat finger gdb \
 p7zip-full patchutils libpq-dev unzip valgrind zip libboost-all-dev gcc g++ \
-g++-multilib jq libseccomp-dev libseccomp2 seccomp junit flex bison poppler-utils
+jq libseccomp-dev libseccomp2 seccomp junit flex bison poppler-utils
 
 apt-get install -qqy ninja-build
 
@@ -89,6 +89,7 @@ apt-get install -qqy emacs
 
 curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
 apt-key fingerprint 0EBFCD88
+
 add-apt-repository "deb [arch=amd64,arm64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
 apt-get update
 apt-get install -qqy docker-ce docker-ce-cli

diff --git a/.setup/install_system.sh b/.setup/install_system.sh
@@ -90,6 +90,12 @@ while :; do
     shift
 done
 
+# SEE GITHUB ISSUE #7885 - https://github.com/Submitty/Submitty/issues/7885
+export UTM_ARM=0
+if [[ "$(uname -m)" = "aarch64" ]] ; then
+    export UTM_ARM=1
+fi
+
 if [ ${VAGRANT} == 1 ]; then
     echo "Non-interactive vagrant script..."
     export DEBIAN_FRONTEND=noninteractive
@@ -98,7 +104,10 @@ fi
 if [ ${VAGRANT} == 1 ] && [ ${WORKER} == 0 ]; then
     # Setting it up to allow SSH as root by default
     mkdir -p -m 700 /root/.ssh
-    cp /home/vagrant/.ssh/authorized_keys /root/.ssh
+    # SEE GITHUB ISSUE #7885 - https://github.com/Submitty/Submitty/issues/7885
+    if [ ${UTM_ARM} == 0 ]; then
+	cp /home/vagrant/.ssh/authorized_keys /root/.ssh
+    fi
 
     sed -i -e "s/PermitRootLogin prohibit-password/PermitRootLogin yes/g" /etc/ssh/sshd_config
 
@@ -218,11 +227,14 @@ fi
 # STACK SETUP
 #################
 
+# SEE GITHUB ISSUE #7885 - https://github.com/Submitty/Submitty/issues/7885
+#if [ ${VAGRANT} == 1] && [ ${UTM_ARM} == 0]; then
 # stack is not available for non-x86_64 systems
 if [ ${VAGRANT} == 1 ] && [ ${WORKER} == 0 ] && [ "$(uname -m)" = "x86_64" ]; then
     # We only might build analysis tools from source while using vagrant
     echo "Installing stack (haskell)"
     curl -sSL https://get.haskellstack.org/ | sh
+    # NOTE: currently only 64-bit (x86_64) Linux binary is available
 fi
 
 #################################################################
@@ -263,6 +275,17 @@ else
         echo "${COURSE_BUILDERS_GROUP} already exists"
 fi
 
+# SEE GITHUB ISSUE #7885 - https://github.com/Submitty/Submitty/issues/7885
+# CREATE VAGRANT USER WHEN MANUALLY INSTALLING ON ARM64 / UTM_ARM MAC M1
+if getent passwd vagrant > /dev/null; then
+    # Already exists
+    echo 're-running install submitty'
+elif [ ${UTM_ARM} == 1 ]; then
+    useradd -m vagrant
+fi
+# END ARM64
+
+
 if [ ${VAGRANT} == 1 ] && [ ${WORKER} == 0 ]; then
 	usermod -aG sudo vagrant
 fi
@@ -435,8 +458,8 @@ if [ ${WORKER} == 0 ]; then
         fi
 
         # remove default sites which would cause server to mess up
-        rm /etc/apache2/sites*/000-default.conf
-        rm /etc/apache2/sites*/default-ssl.conf
+        rm -f /etc/apache2/sites*/000-default.conf
+        rm -f /etc/apache2/sites*/default-ssl.conf
 
         cp ${SUBMITTY_REPOSITORY}/.setup/apache/submitty.conf /etc/apache2/sites-available/submitty.conf
 
@@ -705,7 +728,7 @@ if [ ${WORKER} == 0 ]; then
         echo "Creating submitty master database"
         su postgres -c "psql -c \"CREATE DATABASE submitty WITH OWNER ${DB_USER}\""
         su postgres -c "psql submitty -c \"ALTER SCHEMA public OWNER TO ${DB_USER}\""
-        python3 ${SUBMITTY_REPOSITORY}/migration/run_migrator.py -e master -e system migrate --initial
+        python3 ${SUBMITTY_REPOSITORY}/migration/run_migrator.py -c ${SUBMITTY_INSTALL_DIR}/config -e master -e system migrate --initial
     else
         echo "Submitty master database already exists"
     fi

diff --git a/.setup/vagrant/db_users.sh b/.setup/vagrant/db_users.sh
@@ -1 +1,2 @@
-psql -c "CREATE ROLE submitty_dbuser WITH SUPERUSER CREATEDB CREATEROLE LOGIN PASSWORD 'submitty_dbuser';CREATE ROLE vagrant WITH SUPERUSER CREATEDB CREATEROLE LOGIN PASSWORD 'vagrant';"
+psql -tc "SELECT 1 FROM pg_user WHERE usename = 'submitty_dbuser'" | grep -q 1 || psql -c "CREATE ROLE submitty_dbuser WITH SUPERUSER CREATEDB CREATEROLE LOGIN PASSWORD 'submitty_dbuser';"
+psql -tc "SELECT 1 FROM pg_user WHERE usename = 'vagrant'" | grep -q 1 || psql -c "CREATE ROLE vagrant WITH SUPERUSER CREATEDB CREATEROLE LOGIN PASSWORD 'vagrant';"
diff --git a/README_utm_arm.txt b/README_utm_arm.txt
@@ -0,0 +1,185 @@
+*************WORK IN PROGRESS************
+
+On an M1 Mac laptop, we cannot use virtual box, so follow these instructions instead:
+
+
+1. On the host computer, create a new directory named GIT_CHECKOUT to
+   hold all of the Submitty git repositories.  Manually checkout these
+   repositories (and make sure they are up-to-date):
+
+   From https://github.com/Submitty
+
+   Submitty
+   AnalysisTools
+   CrashCourseCPPSyntax
+   Lichen
+   RainbowGrades
+   SysadminTools
+   Tutorial
+
+   And also create directory:
+   vendor/
+   and checkout from:
+   git clone https://github.com/nlohmann/json.git
+   to this path:
+   vendor/nlohmann/json/
+
+   Optionally (currently a private repo), from https://github.com/Submitty
+   LichenTestData
+
+   *** FIXME: Currently we checkout all of the repositories manually,
+       since the VM cannot write to the shared directory?  Unlike how
+       we currently setup with vagrant, we may not be able to share
+       multiple directories ***
+
+
+2. Install UTM
+   https://mac.getutm.app/
+   Current version: 3.5.1
+
+
+3. Download and save the Ubuntu 20.04 ARM Server ISO
+   https://mac.getutm.app/gallery/ubuntu-20-04
+   https://cdimage.ubuntu.com/releases/20.04.4/release/
+
+
+4. Launch UTM, and through the UTM GUI, press "+" to create a new VM:
+
+   Select "Virtualize"
+   Select "Linux"
+
+   Under Boot ISO Image: browse to select the Ubuntu 20.04 ISO you just downloaded.
+
+   memory -> 4096 mb (or more)
+   CPU Cores -> 2 (or more)
+
+   Storage, 64GB, press "Next"
+
+   Shared Directory, Browse to specify the "GIT_CHECKOUT" directory on your host
+
+   Review the Summary, press "Save"
+
+
+5. Now press the play button to boot & install the guest machine.
+   Do the interactive Ubuntu Server installation...
+
+   * "Install Ubuntu Server"
+
+   ... you'll wait a while here ...
+
+   * "English"
+   * (do not upgrade to Ubuntu 22.04 -- "Continue without updating")
+
+   * "Done" on keyboard layout
+   * "Done" on network connections
+   * "Done" on configure proxy
+   * "Done" on alternate mirror
+   * "Done" on default for storage configuration / storage layout
+   * "Done" on default for storage configuration / file system
+
+      - set the "device" "mounted at /" to be most of your space (e.g. 60GB)
+        it probably defaulted to 30BG (leaving 30GB of free)
+
+   * "Continue" on confirm destructive action
+   * Fill in the profile setup (set a <USERNAME> & <PASSWORD>)
+
+   * Press "Done" on Ubuntu Advantage
+
+   * Select "Install OpenSSH server" and then "Done"
+   * "Done" on featured server snaps
+
+   It will initially say "Installing system" at the top of the page and then quickly switch to "Install complete!" ...
+
+   ... now wait why the server does updates ...
+
+   Wait a while, until it says "Reboot now"
+
+   select "Reboot now"
+
+
+6. After waiting a little while...
+   NOTE: reboot only seems to stop.  It doesn't actually halt & restart.
+
+   From the main UTM screen:
+
+   * Turn off the virtual machine by pressing the square symbol.
+
+     If it says "do you want to force stop this VM and lose all unsaved data?  stop or cancel"...
+
+
+   * "Clear"/disconnect the removable CD/DVD drive (the ISO).
+
+   * Click on the sliders icon in the upper right to edit the VM
+     settings again.
+
+     under the "Network" tab, add port forwarding:
+       guest port 22 -> host 1234 (or anything for ssh below)
+       guest port 1511 -> host 1511
+       guest port 8443 -> host 8443
+       guest port 5432 -> host 16442
+
+     press "save".
+
+   * Then press the play icon to boot the machine again.
+
+
+7. To ssh from your host machine to the guest vm:
+
+   ssh -p 1234 <USERNAME>@localhost
+
+
+8. To share directories between host & guest machines:
+
+   On the guest machine:
+
+   sudo apt install -y spice-vdagent spice-webdavd
+   sudo apt install -y davfs2
+       * Answer the question "YES" and grant access to unpriviledged users *
+   sudo mkdir -p /usr/local/submitty/GIT_CHECKOUT
+
+   NOTE: The command below must be re-run each time the guest machine
+   is rebooted.  It will require interactively entering the username &
+   password for the host machine.  You could put those credentials on
+   the command line, and in a guest machine startup script, but that
+   may be a security concern.
+
+   ALSO: It appears the command below MUST be typed in the UTM VM GUI
+   Terminal (not from an ssh terminal).
+
+   sudo mount -t davfs -o noexec http://127.0.0.1:9843 /usr/local/submitty/GIT_CHECKOUT
+
+   If you get an error mounting the shared directory, you can try
+   running the umount command below, and then repeat the mount command
+   above.  If that doesn't work you can try halt, stop, and play
+   (rebooting) the VM and then try again.
+
+   sudo umount /usr/local/submitty/GIT_CHECKOUT
+
+
+9. TEMPORARY HACK STEP
+
+   open .setup/pip/system_requirements.txt
+   comment out the opencv and onnx version installations (compilation from scratch fails)
+
+    #opencv-python==3.4.10.37
+    #onnxruntime==1.8.1
+    #onnx==1.9.0
+
+
+10. Do Submitty system setup and installation:
+
+    On the guest machine:
+
+    sudo bash /usr/local/submitty/GIT_CHECKOUT/Submitty/.setup/install_system.sh --vagrant
+
+    Hopefully it completes without error or network problems... if you
+    have errors, you can try to re-run the above command.  However, if
+    it crashes in the middle of creating the sample course data, you
+    may need to do:
+
+    sudo bash /usr/local/submitty/GIT_CHECKOUT/Submitty/.setup/bin/recreate_sample_courses.sh
+
+
+11. When finished, access the Submitty website from a browser on your host machine:
+
+    http://localhost:1511/
diff --git a/bin/calculate_extensions.cpp b/bin/calculate_extensions.cpp
@@ -17,7 +17,6 @@
 
 #include "boost/filesystem/operations.hpp"
 #include "boost/filesystem/path.hpp"
-
 #include "boost/date_time/gregorian/gregorian.hpp"
 
 

diff --git a/grading/json_syntax_checker.py b/grading/json_syntax_checker.py
@@ -29,4 +29,4 @@
         sys.exit(1)
 
     with open(args.file, 'w') as out_file:
-        json.dump(output, out_file)
+        json.dump(output, out_file, indent=4, sort_keys=True)
Original file line number	Diff line number	Diff line change
Expand Up		@@ -17,7 +17,6 @@

		#include "boost/filesystem/operations.hpp"
		#include "boost/filesystem/path.hpp"

		#include "boost/date_time/gregorian/gregorian.hpp"


Expand Down