From d2a74cb3a698a45f14721b69d394e5e445128124 Mon Sep 17 00:00:00 2001 From: Barb Cutler Date: Fri, 13 May 2022 09:20:23 -0400 Subject: [PATCH] [Feature:System] installation on Mac M1/ARM using UTM (#6815) * remove java from distro_setup * remove java images from default container * leave ubuntu 16.04 an 18.04 unchanged * put java 11 back in default * utm flag & remove 32 bit * progress * finished running installation script * update notes * revisions * fix C++ json libary stuff * instructions * replace json.hpp with shared repo checkout version * more changes * update * tweak * tweak * revert changes * Revert "revert changes" This reverts commit 8af1acbb3045d65e78abe26dc3b178bd75011115. * tweak instructions * replace deprecated include for calculate_extensions * edits * refactor * refactor * fix undefined index * Fix bug * updates May 2022 * documentation * documentation Co-authored-by: Eddie Krystowski Co-authored-by: Barb Cutler Co-authored-by: williamjallen --- .setup/INSTALL_SUBMITTY_HELPER.sh | 33 ++++ .setup/distro_setup/setup_distro.sh | 4 +- .../distro_setup/ubuntu/20.04/setup_distro.sh | 3 +- .setup/install_system.sh | 31 ++- .setup/vagrant/db_users.sh | 3 +- README_utm_arm.txt | 185 ++++++++++++++++++ bin/calculate_extensions.cpp | 1 - grading/json_syntax_checker.py | 2 +- grading/load_config_json.cpp | 72 ++++--- .../update_and_install_workers.py | 17 +- 10 files changed, 311 insertions(+), 40 deletions(-) create mode 100644 README_utm_arm.txt diff --git a/.setup/INSTALL_SUBMITTY_HELPER.sh b/.setup/INSTALL_SUBMITTY_HELPER.sh index aff203f0fdc..64be0c48685 100644 --- a/.setup/INSTALL_SUBMITTY_HELPER.sh +++ b/.setup/INSTALL_SUBMITTY_HELPER.sh @@ -32,6 +32,15 @@ if [ -d ${THIS_DIR}/../.vagrant ]; then VAGRANT=1 fi + +# +# SEE GITHUB ISSUE #7885 - https://github.com/Submitty/Submitty/issues/7885 +UTM_ARM=0 +if [[ "$(uname -m)" = "aarch64" ]] ; then + UTM_ARM=1 +fi + + SUBMITTY_REPOSITORY=$(jq -r '.submitty_repository' ${CONF_DIR}/submitty.json) SUBMITTY_INSTALL_DIR=$(jq -r '.submitty_install_dir' ${CONF_DIR}/submitty.json) WORKER=$([[ $(jq -r '.worker' ${CONF_DIR}/submitty.json) == "true" ]] && echo 1 || echo 0) @@ -46,6 +55,18 @@ else RESTART_DAEMONS=( ) fi +######################################################################################################################## +######################################################################################################################## +# FORCE CORRECT TIME SKEW +# This may happen on a development virtual machine +# SEE GITHUB ISSUE #7885 - https://github.com/Submitty/Submitty/issues/7885 +if [ ${UTM_ARM} == 1 ]; then + sudo service ntp stop + sudo ntpd -gq + sudo service ntp start + sudo timedatectl set-timezone America/New_York +fi + ######################################################################################################################## ######################################################################################################################## # this script must be run by root or sudo @@ -540,6 +561,11 @@ echo -e "Compile and install analysis tools" mkdir -p ${SUBMITTY_INSTALL_DIR}/SubmittyAnalysisTools +# SEE GITHUB ISSUE #7885 - https://github.com/Submitty/Submitty/issues/7885 +# HASKELL BINARY IS NOT AVAILABLE ARM64 +if [ ${UTM_ARM} == 0 ]; then +# END ARM64 + pushd ${SUBMITTY_INSTALL_DIR}/SubmittyAnalysisTools if [[ ! -f VERSION || $(< VERSION) != "${AnalysisTools_Version}" ]]; then for b in count plagiarism diagnostics; @@ -550,6 +576,13 @@ if [[ ! -f VERSION || $(< VERSION) != "${AnalysisTools_Version}" ]]; then fi popd > /dev/null +# SEE GITHUB ISSUE #7885 - https://github.com/Submitty/Submitty/issues/7885 +# HASKELL BINARY IS NOT AVAILABLE ARM64 +else + echo "SKIPPING ANALYSIS TOOLS INSTALL ON UTM ARM 64" +fi +# END ARM64 + # change permissions chown -R ${DAEMON_USER}:${COURSE_BUILDERS_GROUP} ${SUBMITTY_INSTALL_DIR}/SubmittyAnalysisTools chmod -R 555 ${SUBMITTY_INSTALL_DIR}/SubmittyAnalysisTools diff --git a/.setup/distro_setup/setup_distro.sh b/.setup/distro_setup/setup_distro.sh index dbc11871d26..c8308858a21 100644 --- a/.setup/distro_setup/setup_distro.sh +++ b/.setup/distro_setup/setup_distro.sh @@ -39,7 +39,9 @@ fi # check if set, else use default /etc/motd # we expect that SUBMISSION_URL and GIT_URL to be set above -if [ ${VAGRANT} == 1 ]; then +# SEE GITHUB ISSUE #7885 - https://github.com/Submitty/Submitty/issues/7885 + +if [ ${VAGRANT} == 1] && [ ${UTM_ARM} == 0]; then # Ubuntu/Debian share this stuff, CentOS does not if [ -d /etc/update-motd.d ]; then chmod -x /etc/update-motd.d/* diff --git a/.setup/distro_setup/ubuntu/20.04/setup_distro.sh b/.setup/distro_setup/ubuntu/20.04/setup_distro.sh index 1c8710510c5..4c009ef87a5 100644 --- a/.setup/distro_setup/ubuntu/20.04/setup_distro.sh +++ b/.setup/distro_setup/ubuntu/20.04/setup_distro.sh @@ -66,7 +66,7 @@ apt-get install -qqy scrot apt-get install -qqy clang autoconf automake autotools-dev diffstat finger gdb \ p7zip-full patchutils libpq-dev unzip valgrind zip libboost-all-dev gcc g++ \ -g++-multilib jq libseccomp-dev libseccomp2 seccomp junit flex bison poppler-utils +jq libseccomp-dev libseccomp2 seccomp junit flex bison poppler-utils apt-get install -qqy ninja-build @@ -89,6 +89,7 @@ apt-get install -qqy emacs curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - apt-key fingerprint 0EBFCD88 + add-apt-repository "deb [arch=amd64,arm64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" apt-get update apt-get install -qqy docker-ce docker-ce-cli diff --git a/.setup/install_system.sh b/.setup/install_system.sh index d45b9775aa6..37e3c8a48ee 100644 --- a/.setup/install_system.sh +++ b/.setup/install_system.sh @@ -90,6 +90,12 @@ while :; do shift done +# SEE GITHUB ISSUE #7885 - https://github.com/Submitty/Submitty/issues/7885 +export UTM_ARM=0 +if [[ "$(uname -m)" = "aarch64" ]] ; then + export UTM_ARM=1 +fi + if [ ${VAGRANT} == 1 ]; then echo "Non-interactive vagrant script..." export DEBIAN_FRONTEND=noninteractive @@ -98,7 +104,10 @@ fi if [ ${VAGRANT} == 1 ] && [ ${WORKER} == 0 ]; then # Setting it up to allow SSH as root by default mkdir -p -m 700 /root/.ssh - cp /home/vagrant/.ssh/authorized_keys /root/.ssh + # SEE GITHUB ISSUE #7885 - https://github.com/Submitty/Submitty/issues/7885 + if [ ${UTM_ARM} == 0 ]; then + cp /home/vagrant/.ssh/authorized_keys /root/.ssh + fi sed -i -e "s/PermitRootLogin prohibit-password/PermitRootLogin yes/g" /etc/ssh/sshd_config @@ -218,11 +227,14 @@ fi # STACK SETUP ################# +# SEE GITHUB ISSUE #7885 - https://github.com/Submitty/Submitty/issues/7885 +#if [ ${VAGRANT} == 1] && [ ${UTM_ARM} == 0]; then # stack is not available for non-x86_64 systems if [ ${VAGRANT} == 1 ] && [ ${WORKER} == 0 ] && [ "$(uname -m)" = "x86_64" ]; then # We only might build analysis tools from source while using vagrant echo "Installing stack (haskell)" curl -sSL https://get.haskellstack.org/ | sh + # NOTE: currently only 64-bit (x86_64) Linux binary is available fi ################################################################# @@ -263,6 +275,17 @@ else echo "${COURSE_BUILDERS_GROUP} already exists" fi +# SEE GITHUB ISSUE #7885 - https://github.com/Submitty/Submitty/issues/7885 +# CREATE VAGRANT USER WHEN MANUALLY INSTALLING ON ARM64 / UTM_ARM MAC M1 +if getent passwd vagrant > /dev/null; then + # Already exists + echo 're-running install submitty' +elif [ ${UTM_ARM} == 1 ]; then + useradd -m vagrant +fi +# END ARM64 + + if [ ${VAGRANT} == 1 ] && [ ${WORKER} == 0 ]; then usermod -aG sudo vagrant fi @@ -435,8 +458,8 @@ if [ ${WORKER} == 0 ]; then fi # remove default sites which would cause server to mess up - rm /etc/apache2/sites*/000-default.conf - rm /etc/apache2/sites*/default-ssl.conf + rm -f /etc/apache2/sites*/000-default.conf + rm -f /etc/apache2/sites*/default-ssl.conf cp ${SUBMITTY_REPOSITORY}/.setup/apache/submitty.conf /etc/apache2/sites-available/submitty.conf @@ -705,7 +728,7 @@ if [ ${WORKER} == 0 ]; then echo "Creating submitty master database" su postgres -c "psql -c \"CREATE DATABASE submitty WITH OWNER ${DB_USER}\"" su postgres -c "psql submitty -c \"ALTER SCHEMA public OWNER TO ${DB_USER}\"" - python3 ${SUBMITTY_REPOSITORY}/migration/run_migrator.py -e master -e system migrate --initial + python3 ${SUBMITTY_REPOSITORY}/migration/run_migrator.py -c ${SUBMITTY_INSTALL_DIR}/config -e master -e system migrate --initial else echo "Submitty master database already exists" fi diff --git a/.setup/vagrant/db_users.sh b/.setup/vagrant/db_users.sh index 5f589ded6ed..d870c538ffa 100644 --- a/.setup/vagrant/db_users.sh +++ b/.setup/vagrant/db_users.sh @@ -1 +1,2 @@ -psql -c "CREATE ROLE submitty_dbuser WITH SUPERUSER CREATEDB CREATEROLE LOGIN PASSWORD 'submitty_dbuser';CREATE ROLE vagrant WITH SUPERUSER CREATEDB CREATEROLE LOGIN PASSWORD 'vagrant';" +psql -tc "SELECT 1 FROM pg_user WHERE usename = 'submitty_dbuser'" | grep -q 1 || psql -c "CREATE ROLE submitty_dbuser WITH SUPERUSER CREATEDB CREATEROLE LOGIN PASSWORD 'submitty_dbuser';" +psql -tc "SELECT 1 FROM pg_user WHERE usename = 'vagrant'" | grep -q 1 || psql -c "CREATE ROLE vagrant WITH SUPERUSER CREATEDB CREATEROLE LOGIN PASSWORD 'vagrant';" diff --git a/README_utm_arm.txt b/README_utm_arm.txt new file mode 100644 index 00000000000..19e2d6f5a13 --- /dev/null +++ b/README_utm_arm.txt @@ -0,0 +1,185 @@ +*************WORK IN PROGRESS************ + +On an M1 Mac laptop, we cannot use virtual box, so follow these instructions instead: + + +1. On the host computer, create a new directory named GIT_CHECKOUT to + hold all of the Submitty git repositories. Manually checkout these + repositories (and make sure they are up-to-date): + + From https://github.com/Submitty + + Submitty + AnalysisTools + CrashCourseCPPSyntax + Lichen + RainbowGrades + SysadminTools + Tutorial + + And also create directory: + vendor/ + and checkout from: + git clone https://github.com/nlohmann/json.git + to this path: + vendor/nlohmann/json/ + + Optionally (currently a private repo), from https://github.com/Submitty + LichenTestData + + *** FIXME: Currently we checkout all of the repositories manually, + since the VM cannot write to the shared directory? Unlike how + we currently setup with vagrant, we may not be able to share + multiple directories *** + + +2. Install UTM + https://mac.getutm.app/ + Current version: 3.5.1 + + +3. Download and save the Ubuntu 20.04 ARM Server ISO + https://mac.getutm.app/gallery/ubuntu-20-04 + https://cdimage.ubuntu.com/releases/20.04.4/release/ + + +4. Launch UTM, and through the UTM GUI, press "+" to create a new VM: + + Select "Virtualize" + Select "Linux" + + Under Boot ISO Image: browse to select the Ubuntu 20.04 ISO you just downloaded. + + memory -> 4096 mb (or more) + CPU Cores -> 2 (or more) + + Storage, 64GB, press "Next" + + Shared Directory, Browse to specify the "GIT_CHECKOUT" directory on your host + + Review the Summary, press "Save" + + +5. Now press the play button to boot & install the guest machine. + Do the interactive Ubuntu Server installation... + + * "Install Ubuntu Server" + + ... you'll wait a while here ... + + * "English" + * (do not upgrade to Ubuntu 22.04 -- "Continue without updating") + + * "Done" on keyboard layout + * "Done" on network connections + * "Done" on configure proxy + * "Done" on alternate mirror + * "Done" on default for storage configuration / storage layout + * "Done" on default for storage configuration / file system + + - set the "device" "mounted at /" to be most of your space (e.g. 60GB) + it probably defaulted to 30BG (leaving 30GB of free) + + * "Continue" on confirm destructive action + * Fill in the profile setup (set a & ) + + * Press "Done" on Ubuntu Advantage + + * Select "Install OpenSSH server" and then "Done" + * "Done" on featured server snaps + + It will initially say "Installing system" at the top of the page and then quickly switch to "Install complete!" ... + + ... now wait why the server does updates ... + + Wait a while, until it says "Reboot now" + + select "Reboot now" + + +6. After waiting a little while... + NOTE: reboot only seems to stop. It doesn't actually halt & restart. + + From the main UTM screen: + + * Turn off the virtual machine by pressing the square symbol. + + If it says "do you want to force stop this VM and lose all unsaved data? stop or cancel"... + + + * "Clear"/disconnect the removable CD/DVD drive (the ISO). + + * Click on the sliders icon in the upper right to edit the VM + settings again. + + under the "Network" tab, add port forwarding: + guest port 22 -> host 1234 (or anything for ssh below) + guest port 1511 -> host 1511 + guest port 8443 -> host 8443 + guest port 5432 -> host 16442 + + press "save". + + * Then press the play icon to boot the machine again. + + +7. To ssh from your host machine to the guest vm: + + ssh -p 1234 @localhost + + +8. To share directories between host & guest machines: + + On the guest machine: + + sudo apt install -y spice-vdagent spice-webdavd + sudo apt install -y davfs2 + * Answer the question "YES" and grant access to unpriviledged users * + sudo mkdir -p /usr/local/submitty/GIT_CHECKOUT + + NOTE: The command below must be re-run each time the guest machine + is rebooted. It will require interactively entering the username & + password for the host machine. You could put those credentials on + the command line, and in a guest machine startup script, but that + may be a security concern. + + ALSO: It appears the command below MUST be typed in the UTM VM GUI + Terminal (not from an ssh terminal). + + sudo mount -t davfs -o noexec http://127.0.0.1:9843 /usr/local/submitty/GIT_CHECKOUT + + If you get an error mounting the shared directory, you can try + running the umount command below, and then repeat the mount command + above. If that doesn't work you can try halt, stop, and play + (rebooting) the VM and then try again. + + sudo umount /usr/local/submitty/GIT_CHECKOUT + + +9. TEMPORARY HACK STEP + + open .setup/pip/system_requirements.txt + comment out the opencv and onnx version installations (compilation from scratch fails) + + #opencv-python==3.4.10.37 + #onnxruntime==1.8.1 + #onnx==1.9.0 + + +10. Do Submitty system setup and installation: + + On the guest machine: + + sudo bash /usr/local/submitty/GIT_CHECKOUT/Submitty/.setup/install_system.sh --vagrant + + Hopefully it completes without error or network problems... if you + have errors, you can try to re-run the above command. However, if + it crashes in the middle of creating the sample course data, you + may need to do: + + sudo bash /usr/local/submitty/GIT_CHECKOUT/Submitty/.setup/bin/recreate_sample_courses.sh + + +11. When finished, access the Submitty website from a browser on your host machine: + + http://localhost:1511/ diff --git a/bin/calculate_extensions.cpp b/bin/calculate_extensions.cpp index 8f8805e7bb7..42881c77dcd 100644 --- a/bin/calculate_extensions.cpp +++ b/bin/calculate_extensions.cpp @@ -17,7 +17,6 @@ #include "boost/filesystem/operations.hpp" #include "boost/filesystem/path.hpp" - #include "boost/date_time/gregorian/gregorian.hpp" diff --git a/grading/json_syntax_checker.py b/grading/json_syntax_checker.py index b0c061528dd..2520f1dcf77 100644 --- a/grading/json_syntax_checker.py +++ b/grading/json_syntax_checker.py @@ -29,4 +29,4 @@ sys.exit(1) with open(args.file, 'w') as out_file: - json.dump(output, out_file) + json.dump(output, out_file, indent=4, sort_keys=True) diff --git a/grading/load_config_json.cpp b/grading/load_config_json.cpp index d211dac9f65..a41d7375b2e 100644 --- a/grading/load_config_json.cpp +++ b/grading/load_config_json.cpp @@ -13,6 +13,13 @@ extern const char *GLOBAL_config_json_string; // defined in json_generated.cpp +template +void json_set_default(nlohmann::json &whole_config, std::string field, const T& value) { + if (whole_config.find(field) == whole_config.end()) { + whole_config[field] = value; + } +} + void AddAutogradingConfiguration(nlohmann::json &whole_config) { std::vector all_testcase_ids = gatherAllTestcaseIds(whole_config); @@ -123,37 +130,48 @@ void AddGlobalDefaults(nlohmann::json &whole_config) { whole_config["gradeable_message"] = whole_config.value("assignment_message",""); whole_config.erase("assignment_message"); } - whole_config["gradeable_message"] = whole_config.value("gradeable_message", ""); - whole_config["load_gradeable_message"] = whole_config.value("load_gradeable_message", nlohmann::json::object()); - whole_config["load_gradeable_message"]["message"] = whole_config["load_gradeable_message"].value("message", ""); - whole_config["load_gradeable_message"]["first_time_only"] = whole_config["load_gradeable_message"].value("first_time_only", false); + json_set_default(whole_config,"gradeable_message",std::string("")); - whole_config["early_submission_incentive"] = whole_config.value("early_submission_incentive", nlohmann::json::object()); - whole_config["early_submission_incentive"]["message"] = whole_config["early_submission_incentive"].value("message", ""); - whole_config["early_submission_incentive"]["minimum_days_early"] = whole_config["early_submission_incentive"].value("minimum_days_early", 0); - whole_config["early_submission_incentive"]["minimum_points"] = whole_config["early_submission_incentive"].value("minimum_points", 0); - whole_config["early_submission_incentive"]["test_cases"] = whole_config["early_submission_incentive"].value("test_cases", std::vector()); - - whole_config["max_submission_size"] = whole_config.value("max_submission_size",MAX_SUBMISSION_SIZE); + if (whole_config.find("load_gradeable_message") == whole_config.end()) { + nlohmann::json o; + o["message"] = ""; + o["first_time_only"] = false; + whole_config["load_gradeable_message"] = o; + } else { + json_set_default(whole_config["load_gradeable_message"],"message",std::string("")); + json_set_default(whole_config["load_gradeable_message"],"first_time_only",false); + } - whole_config["required_capabilities"] = whole_config.value("required_capabilities","default"); + if (whole_config.find("early_submission_incentive") == whole_config.end()) { + nlohmann::json o; + o["message"] = ""; + o["minimum_days_early"] = 0; + o["minimum_points"] = 0; + o["test_cases"] = std::vector(); + whole_config["early_submission_incentive"] = o; + } else { + json_set_default(whole_config["early_submission_incentive"],"message",std::string("")); + json_set_default(whole_config["early_submission_incentive"],"minimum_days_early",0); + json_set_default(whole_config["early_submission_incentive"],"minimum_points",0); + json_set_default(whole_config["early_submission_incentive"],"test_cases",std::vector()); + } - // Configure defaults for hide_submitted_files - whole_config["hide_submitted_files"] = whole_config.value("hide_submitted_files", false); + json_set_default(whole_config, "max_submission_size", MAX_SUBMISSION_SIZE); + json_set_default(whole_config, "required_capabilities", "default"); - // Configure defaults for hide_version_and_test_details - whole_config["hide_version_and_test_details"] = whole_config.value("hide_version_and_test_details", false); + json_set_default(whole_config, "hide_submitted_files", false); + json_set_default(whole_config, "hide_version_and_test_details", false); // By default, we have one drop zone without a part label / sub // directory. nlohmann::json::iterator parts = whole_config.find("part_names"); if (parts != whole_config.end()) { - nlohmann::json tmp = nlohmann::json::array(); + nlohmann::json tmp_array = nlohmann::json::array(); for (int i = 0; i < parts->size(); i++) { - tmp.push_back((*parts)[i]); + tmp_array.push_back((*parts)[i]); } - whole_config["part_names"] = tmp; + whole_config["part_names"] = tmp_array; } // But, if there are input fields, but there are no explicit parts @@ -966,16 +984,16 @@ void InflateTestcase(nlohmann::json &single_testcase, nlohmann::json &whole_conf Execution_Helper(single_testcase); } - single_testcase["testcase_label"] = single_testcase.value("testcase_label", ""); - single_testcase["details"] = single_testcase.value("details",""); - single_testcase["extra_credit"] = single_testcase.value("extra_credit",false); - single_testcase["release_hidden_details"] = single_testcase.value("release_hidden_details", false); - single_testcase["hidden"] = single_testcase.value("hidden", false); + json_set_default(single_testcase,"testcase_label",std::string("")); + json_set_default(single_testcase,"details",std::string("")); + json_set_default(single_testcase,"extra_credit",false); + json_set_default(single_testcase,"release_hidden_details",false); + json_set_default(single_testcase,"hidden",false); assert(!(single_testcase["release_hidden_details"] && !single_testcase["hidden"])); - single_testcase["view_testcase_message"] = single_testcase.value("view_testcase_message",true); - single_testcase["publish_actions"] = single_testcase.value("publish_actions", false); - + json_set_default(single_testcase,"view_testcase_message",true); + json_set_default(single_testcase,"publish_actions",false); + nlohmann::json::iterator itr = single_testcase.find("validation"); if (itr != single_testcase.end()) { assert (itr->is_array()); diff --git a/sbin/shipper_utils/update_and_install_workers.py b/sbin/shipper_utils/update_and_install_workers.py index b03dbf16615..0d31e1bd3b6 100644 --- a/sbin/shipper_utils/update_and_install_workers.py +++ b/sbin/shipper_utils/update_and_install_workers.py @@ -11,6 +11,7 @@ import argparse import get_docker_info from submitty_utils import ssh_proxy_jump +import platform CONFIG_PATH = path.join(path.dirname(path.realpath(__file__)), '..', '..','config') @@ -58,14 +59,22 @@ def update_docker_images(user, host, worker, autograding_workers, autograding_co repo, tag = image.split(':') client.images.pull(repository=repo, tag=tag) except Exception as e: - print(f"ERROR: Could not pull {image}") - traceback.print_exc() - success = False + print(f"ERROR: Could not pull {image}") + traceback.print_exc() + + # check for machine + if platform.machine() == "aarch64": + # SEE GITHUB ISSUE #7885 - https://github.com/Submitty/Submitty/issues/7885 + # docker pull often fails on ARM installation + print("WARNING: SKIPPING DOCKER PULL ERROR") + else: + # normal case + success = False + docker_info = client.info() docker_images_obj = client.images.list() #print the details of the image get_docker_info.printDockerInfo() - else: commands = list() script_directory = os.path.join(SUBMITTY_INSTALL_DIR, 'sbin', 'shipper_utils', 'docker_command_wrapper.py')