PYPI proof-of-concept (#8088)

.dockerignore

@@ -0,0 +1,4 @@
+test/Playbook
+.dockerignore
+cicd/crabserver_pypi/Dockerfile
+cicd/crabtaskworker_pypi/Dockerfile

cicd/crabserver_pypi/Dockerfile

@@ -0,0 +1,59 @@
+# Get CERN tsanames.ora from cmsweb rpm image
+FROM registry.cern.ch/cmsweb/cmsweb:20230427-stable as tnsnames
+
+# Use `wmagent-base` image as our baseimage:
+# https://github.com/dmwm/CMSKubernetes/blob/2b0454f9205cb8f97fecb91bf6661b59e4b31424/docker/pypi/wmagent-base/Dockerfile
+
+FROM registry.cern.ch/cmsweb/wmagent-base:pypi-20230705
+SHELL ["/bin/bash", "-c"]
+ENV WDIR=/data
+ENV USER=crab3
+RUN useradd ${USER} \
+    && install -o ${USER} -d ${WDIR} \
+    && echo "${USER} ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/01-crab3
+
+# install packages from debian official repo
+RUN apt-get update \
+    && apt-get install -y tini \
+    && apt-get clean all
+
+# Copy CERN tnsnames.ora
+COPY --from=tnsnames /etc/tnsnames.ora /etc/tnsnames.ora
+
+#RUN mkdir -p /opt/rucio/etc/
+
+RUN mkdir /build
+WORKDIR /build
+COPY cicd/crabserver_pypi/ .
+# Install wmcore/crab dependencies
+RUN pip install -r requirements.txt
+# Getting the version number from wmcore_requirements.txt and install it with `--no-deps`.
+# When reading from file, `grep -v '^\s*#'` to ignore all comment lines, including the ones with leading whitespace ('\s' to match whitespace).
+RUN wmcore_version="$(grep -v '^\s*#' wmcore_requirements.txt | cut -d' ' -f2)" \
+    && pip install --no-deps WMCore==${wmcore_version}
+
+# Install CRAB.
+COPY src/python/ /data/srv/current/lib/python/site-packages
+COPY src/script/ /data/srv/current/data/script/
+COPY src/html/ /data/srv/current/data/html/
+COPY src/css/ /data/srv/current/data/css/
+
+RUN rm -rf /build
+
+WORKDIR /data
+
+# create mandatory directory
+RUN ls /data/srv/current/ \
+    && install -d -o ${USER} -g ${USER} /data/srv/logs/crabserver \
+    && install -d -o ${USER} -g ${USER} /data/srv/state/crabserver \
+    && install -d -o ${USER} -g ${USER} /data/srv/current/auth/crabserver \
+    && install -d -o ${USER} -g ${USER} /data/srv/current/config/crabserver
+
+# copy running script files
+COPY cicd/crabserver_pypi/run.sh cicd/crabserver_pypi/manage cicd/crabserver_pypi/entrypoint.sh /data/
+
+# make sure all files is the same user/group as running user
+RUN chown -R 1000:1000 /data
+
+USER ${USER}
+CMD /data/run.sh

cicd/crabserver_pypi/entrypoint.sh

@@ -0,0 +1,52 @@
+#! /bin/bash
+# Original script from https://github.com/dmwm/CMSKubernetes/blob/2b0454f9205cb8f97fecb91bf6661b59e4b31424/docker/crabserver/run.sh
+# Should belong to CMSKubernetes.
+
+if [ -f /etc/robots/robotkey.pem ]; then
+    sudo cp /etc/robots/robotkey.pem /data/srv/current/auth/crabserver/dmwm-service-key.pem
+    sudo cp /etc/robots/robotcert.pem /data/srv/current/auth/crabserver/dmwm-service-cert.pem
+    sudo chown $USER:$USER /data/srv/current/auth/crabserver/dmwm-service-key.pem
+    sudo chown $USER:$USER /data/srv/current/auth/crabserver/dmwm-service-cert.pem
+fi
+
+if [ -f /etc/hmac/hmac ]; then
+    sudo mkdir -p /data/srv/current/auth/wmcore-auth
+    if [ -f /data/srv/current/auth/wmcore-auth/header-auth-key ]; then
+        sudo rm /data/srv/current/auth/wmcore-auth/header-auth-key
+    fi
+    sudo cp /etc/hmac/hmac /data/srv/current/auth/wmcore-auth/header-auth-key
+    sudo chown $USER.$USER /data/srv/current/auth/wmcore-auth/header-auth-key
+    sudo mkdir -p /data/srv/current/auth/crabserver
+    if [ -f /data/srv/current/auth/crabserver/header-auth-key ]; then
+        sudo rm /data/srv/current/auth/crabserver/header-auth-key
+    fi
+    sudo cp /etc/hmac/hmac /data/srv/current/auth/crabserver/header-auth-key
+    sudo chown $USER.$USER /data/srv/current/auth/crabserver/header-auth-key
+fi
+
+if [ -f /etc/proxy/proxy ]; then
+    export X509_USER_PROXY=/etc/proxy/proxy
+    sudo mkdir -p /data/srv/state/crabserver/proxy
+    if [ -f /data/srv/state/crabserver/proxy/proxy.cert ]; then
+        rm /data/srv/state/crabserver/proxy/proxy.cert
+    fi
+    sudo ln -s /etc/proxy/proxy /data/srv/state/crabserver/proxy/proxy.cert
+    sudo mkdir -p /data/srv/current/auth/proxy
+    if [ -f /data/srv/current/auth/proxy/proxy ]; then
+        rm /data/srv/current/auth/proxy/proxy
+    fi
+    sudo ln -s /etc/proxy/proxy /data/srv/current/auth/proxy/proxy
+fi
+
+if [ -f /etc/secrets/config.py ]; then
+    sudo cp /etc/secrets/config.py /data/srv/current/config/crabserver/config.py
+    sudo chown $USER:$USER /data/srv/current/config/crabserver/config.py
+fi
+
+if [ -f /etc/secrets/CRABServerAuth.py ]; then
+    sudo cp /etc/secrets/CRABServerAuth.py /data/srv/current/auth/crabserver/CRABServerAuth.py
+    sudo chown $USER:$USER /data/srv/current/auth/crabserver/CRABServerAuth.py
+fi
+
+
+exec /usr/bin/tini -- "$@"

cicd/crabserver_pypi/manage

@@ -0,0 +1,114 @@
+#!/bin/bash
+# This scripts originate from dmwm-base/manage
+# https://github.com/dmwm/CMSKubernetes/blob/2b0454f9205cb8f97fecb91bf6661b59e4b31424/docker/pypi/dmwm-base/manage
+# However, scripts is heavily edited to suit new CRAB PyPI.
+# 2 things that we may need to keep up with original scripts:
+#   - Deployment Convention: WMCore/REST app (CRAB REST) read necessary files from.
+#   - `wmc-httpd` line.
+
+##H Usage: manage ACTION [ATTRIBUTE] [SECURITY-STRING]
+##H
+##H Available actions:
+##H   help        show this help
+##H   version     get current version of the service
+##H   status      show current service's status
+##H   restart     (re)start the service
+##H   start       (re)start the service
+##H   stop        stop the service
+
+# common settings to prettify output
+echo_e=-e
+COLOR_OK="\\033[0;32m"
+COLOR_WARN="\\033[0;31m"
+COLOR_NORMAL="\\033[0;39m"
+
+# hardcode service name
+srv=crabserver
+
+# service settings
+#LOGDIR=/data/srv/logs/$srv # we write logs to fifo instead of logdir
+AUTHDIR=/data/srv/current/auth/$srv
+STATEDIR=/data/srv/state/$srv
+CFGDIR=/data/srv/current/config/$srv
+CFGFILE=$CFGDIR/config.py
+
+# necessary env settings for all WM services
+## config
+PYTHONPATH=/data/srv/current/config/$srv:${PYTHONPATH}
+## secrets
+PYTHONPATH=/etc/secrets:${PYTHONPATH} # (it should be $AUTHDIR, will fix later).
+## app
+PYTHONPATH=/data/srv/current/lib/python/site-packages/:$PYTHONPATH
+export PYTHONPATH
+## to get wmc-httpd path
+export PATH=/data/srv/current/bin:$PATH
+## service creds
+export X509_USER_CERT=${X509_USER_CERT:-${AUTHDIR}/dmwm-service-cert.pem}
+export X509_USER_KEY=${X509_USER_KEY:-${AUTHDIR}/dmwm-service-key.pem}
+
+# by default Rucio relies on /opt/rucio/etc/config.cfg
+# if necessary we may setup RUCIO_HOME which should provide this location
+# but in k8s we mount rucio config.cfg under /opt/rucio/etc area
+
+usage()
+{
+    cat $0 | grep "^##H" | sed -e "s,##H,,g"
+}
+start_srv()
+{
+    # Wa: originally from dmwm-base
+    #wmc-httpd -r -d $STATEDIR -l "|rotatelogs $LOGDIR/$srv-%Y%m%d-`hostname -s`.log 86400" $CFGFILE
+    wmc-httpd -r -d $STATEDIR -l "$STATEDIR/crabserver-fifo" $CFGFILE
+    # Wa: debug
+    #LD_PRELOAD=$JEMALLOC_ROOT/lib/libjemalloc.so wmc-httpd -r -d $STATEDIR -l "$STATEDIR/crabserver-fifo" $CFGFILE
+    #gdb --args python3 /data/srv/current/bin/wmc-httpd -r -d $STATEDIR -l "$STATEDIR/crabserver-fifo" $CFGFILE
+}
+
+stop_srv()
+{
+    local pid=`ps auxwww | egrep "wmc-httpd" | grep -v grep | awk 'BEGIN{ORS=" "} {print $2}'`
+    echo "Stop $srv service... ${pid}"
+    if [ -n "${pid}" ]; then
+        kill -9 ${pid}
+    fi
+}
+
+status_srv()
+{
+    local pid=`ps auxwww | egrep "wmc-httpd" | grep -v grep | awk 'BEGIN{ORS=" "} {print $2}'`
+    if  [ -z "${pid}" ]; then
+        echo "$srv service is not running"
+        return
+    fi
+    if [ ! -z "${pid}" ]; then
+        echo $echo_e "$srv service is ${COLOR_OK}RUNNING${COLOR_NORMAL}, PID=${pid}"
+        ps -f -wwww -p ${pid}
+    else
+        echo $echo_e "$srv service is ${COLOR_WARN}NOT RUNNING${COLOR_NORMAL}"
+    fi
+}
+
+# Main routine, perform action requested on command line.
+case ${1:-status} in
+  start | restart )
+    stop_srv
+    start_srv
+    ;;
+
+  status )
+    status_srv
+    ;;
+
+  stop )
+    stop_srv
+    ;;
+
+  help )
+    usage
+    ;;
+
+  * )
+    echo "$0: unknown action '$1', please try '$0 help' or documentation." 1>&2
+    exit 1
+    ;;
+esac

cicd/crabserver_pypi/requirements.txt

@@ -0,0 +1,16 @@
+# Original dependencies from WMCore's requirements.txt
+# https://github.com/dmwm/WMCore/blob/9c6e83d1d23983c0296eee318c9e6255ff80d01b/requirements.txt
+# Lesser deps but match the version with upstream.
+Cheetah3==3.2.6.post1
+CherryPy==18.8.0
+CMSMonitoring==0.6.11
+dbs3-client==4.0.12
+future==0.18.3
+httplib2==0.20.4
+psutil==5.9.5
+pycurl==7.45.2
+PyJWT==2.4.0
+retry==0.9.2
+boto3~=1.12.49
+cx-Oracle~=8.3.0
+htcondor~=10.2.3

cicd/crabserver_pypi/run.sh

@@ -0,0 +1,23 @@
+#! /bin/bash
+# Container main process.
+
+# run monitoring script
+if [ -f /data/monitor.sh ]; then
+    /data/monitor.sh &
+fi
+
+# create named pipe to pipe log to stdout
+mkfifo /data/srv/state/crabserver/crabserver-fifo
+# Run cat on named pipe to prevent crabserver deadlock because no reader attach
+# to pipe. It is safe because only single process can read from pipe at the time
+cat /data/srv/state/crabserver/crabserver-fifo &
+
+#start the service
+#export CRYPTOGRAPHY_ALLOW_OPENSSL_102=true
+/data/manage start
+
+# cat fifo forever to read logs
+while true;
+do
+    cat /data/srv/state/crabserver/crabserver-fifo
+done

cicd/crabserver_pypi/wmcore_requirements.txt

@@ -0,0 +1,5 @@
+# WMCore dependency version.
+# Only for parsing repo url and version number in the building script later.
+# Format:
+# <repository clone url> <tag>
+https://github.com/dmwm/WMCore 2.2.4rc6

cicd/crabtaskworker_pypi/Dockerfile

@@ -0,0 +1,111 @@
+# caching wmcore src, need for building TaskManagerRun.tar.gz
+FROM python:3.8 as wmcore-src
+SHELL ["/bin/bash", "-c"]
+# Use the "magic" requirements.txt from crabserver pypi
+COPY cicd/crabserver_pypi/ .
+RUN wmcore_repo="$(grep -v '^\s*#' wmcore_requirements.txt | cut -d' ' -f1)" \
+    && wmcore_version="$(grep -v '^\s*#' wmcore_requirements.txt | cut -d' ' -f2)" \
+    && git clone ${wmcore_repo} -b "${wmcore_version}" /WMCore \
+    && ( cd /WMCore; git status ) \
+    && echo "${wmcore_version}" > /wmcore_version
+
+# create TaskManagerRun.tar.gz and other data files
+FROM python:3.8 as build-data
+SHELL ["/bin/bash", "-c"]
+RUN mkdir /build \
+    && apt-get update \
+    && apt-get install -y curl zip git \
+    && apt-get clean all
+WORKDIR /build
+COPY cicd/crabtaskworker_pypi/new_htcondor_make_runtime.sh /build
+COPY . /build
+COPY --from=wmcore-src /WMCore /build/WMCore
+RUN bash new_htcondor_make_runtime.sh
+RUN mkdir /build/install_dir \
+    && python3 setup.py install_system -s TaskWorker --prefix=/build/install_dir \
+    && cp /build/CMSRunAnalysis.tar.gz /build/install_dir/data/ \
+    && cp /build/TaskManagerRun.tar.gz /build/install_dir/data/
+
+# gfal, wmcore only has gfal lib but not cli
+FROM registry.cern.ch/cmsweb/gfal:latest as wmcore-gfal
+SHELL ["/bin/bash", "-c"]
+RUN pushd /data/miniconda \
+    && git clone https://gitlab.cern.ch/dmc/gfal2-util -b v1.8.0 \
+    && pushd gfal2-util \
+    && python setup.py install \
+    && popd \
+    && rm -rf gfal2-util
+
+# cern ldap config
+FROM gitlab-registry.cern.ch/linuxsupport/cc7-base:20231001-1.x86_64 as cern-cc7
+
+# start image
+FROM registry.cern.ch/cmsweb/wmagent-base:pypi-20230705
+SHELL ["/bin/bash", "-c"]
+ENV USER=crab3
+ENV WDIR=/data
+
+# install gfal
+# symlink to workaround calling gfal from absolute path
+COPY --from=wmcore-gfal ${WDIR}/miniconda ${WDIR}/miniconda
+RUN ln -sf ${WDIR}/miniconda/bin/gfal-ls /usr/bin/gfal-ls \
+    && ln -sf ${WDIR}/miniconda/bin/gfal-rm /usr/bin/gfal-rm \
+    && ln -sf ${WDIR}/miniconda/bin/gfal-copy /usr/bin/gfal-copy \
+    && ln -sf ${WDIR}/miniconda/bin/gfal-sum /usr/bin/gfal-sum
+
+# install package from debian repository
+# deps for openldap: libsasl2-dev python3-dev libldap-dev libssl-dev
+RUN apt-get update \
+    && apt-get install -y tini libsasl2-dev python3-dev libldap-dev libssl-dev \
+    && apt-get clean all
+
+# prepare build
+RUN mkdir /build
+WORKDIR /build
+
+# install dependencies
+COPY --from=wmcore-src /wmcore_version .
+COPY cicd/crabtaskworker_pypi/requirements.txt .
+RUN pip install -r requirements.txt \
+    && pip install --no-deps wmcore==$(cat wmcore_version)
+
+# install crabserver
+# will replace with pip later
+COPY src/python/ ${WDIR}/srv/current/lib/python/site-packages/
+# copy TaskManagerRun.tar.gz
+COPY --from=build-data /build/install_dir/data ${WDIR}/srv/current/lib/python/site-packages/data
+
+# copy cern openldap config
+COPY --from=cern-cc7 /etc/openldap /etc/openldap
+
+# copy rucio config
+RUN mkdir -p /opt/rucio/etc/
+COPY cicd/crabtaskworker_pypi/rucio.cfg /opt/rucio/etc/
+
+# clean up
+WORKDIR ${WDIR}
+RUN rm -rf /build
+
+# add new user and switch to user
+RUN useradd ${USER} \
+    && install -o ${USER} -d ${WDIR}
+
+# create working directory
+RUN mkdir -p ${WDIR}/srv/tmp \
+    && mkdir -p ${WDIR}/srv/Publisher \
+    && mkdir -p ${WDIR}/srv/TaskManager
+
+RUN cd ${WDIR}/srv/TaskManager \
+    && ln -sf ${WDIR}/hostdisk/TaskWorker/cfg cfg \
+    && ln -sf ${WDIR}/hostdisk/TaskWorker/logs logs
+COPY cicd/crabtaskworker_pypi/start.sh  cicd/crabtaskworker_pypi/run.sh ${WDIR}/srv/TaskManager/
+
+# for debuggin purpose
+RUN echo "${USER} ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/01-crab3
+
+# make sure all /data own by running user
+RUN chown -R 1000:1000 ${WDIR}
+
+USER ${USER}
+
+ENTRYPOINT ["tini", "--"]