Merge pull request #23 from 40ants/dynamic-space-size-and-steps

Added ability to specify dynamic-space-size for lisp jobs.
40ants · Feb 6, 2025 · 12705e8 · 12705e8
2 parents 6356abc + 8e4428a
commit 12705e8
Show file tree

Hide file tree

Showing 12 changed files with 207 additions and 53 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -106,6 +106,12 @@
             "ubuntu-latest",
             "macos-latest"
           ],
+          "exclude": [
+            {
+              "os": "macos-latest",
+              "lisp": "ccl-bin"
+            }
+          ],
           "quicklisp": [
             "quicklisp",
             "ultralisp"

diff --git a/.gitignore b/.gitignore
@@ -3,3 +3,4 @@
 /.qlot
 *.fasl
 .DS_Store
+*.~undo-tree~
diff --git a/40ants-ci.asd b/40ants-ci.asd
@@ -11,7 +11,8 @@
                "40ants-ci/jobs/linter"
                "40ants-ci/jobs/critic"
                "40ants-ci/jobs/run-tests"
-               "40ants-ci/jobs/autotag")
+               "40ants-ci/jobs/autotag"
+               "40ants-ci/workflow")
   :description "A tool simplify continuous deployment for Common Lisp projects."
   :homepage "https://40ants.com/ci/"
   :source-control (:git "https://github.com/40ants/ci")

diff --git a/docs/changelog.lisp b/docs/changelog.lisp
@@ -7,24 +7,66 @@
 
 (defchangelog (:ignore-words ("40ANTS-DOC"
                               "ASDF"
+                              "CI"
+                              "secrets.DEPLOY_TRIGGER_TOKEN"
 			      "DEPLOY_TRIGGER_TOKEN"
+                              "secrets.GITHUB_TOKEN"
 			      "GITHUB_TOKEN"
                               "OSX")
                :external-docs ("https://40ants.com/40ants-asdf-system/"))
+  (0.17.0 2025-02-06
+          "
+Added
+=====
+
+Functions for creation jobs now accept two new arguments:
+
+- STEPS-BEFORE argument allows to specify a list of steps to be performed before the job. For example, this can be used to install some system packages required for loading ASDF systems during the job execution.
+- STEPS-AFTER argument is the same as previous one, but executes steps after the job.
+")
+  (0.16.0 2024-12-14
+          "
+Added
+=====
+
+Now dynamic space size can be given for lisp steps.
+
+There are two ways to set it:
+
+```
+(build-docs
+  :asdf-system \"cl-telegram-bot-docs\"
+  :env ((\"DYNAMIC_SPACE_SIZE\" . \"4Gb\")))
+```
+
+This way it will be applied only to the step of the documentation building,
+because [docs-builder script](https://github.com/40ants/docs-builder) allows to use
+such environment variable.
+
+But if you CI process fails to compile the ASDF system because of the memory limit,
+then you need to set dynamic space size on the earlier state - during \"Setup Lisp\"
+step. For this case an argument DYNAMIC-SPACE-SIZE can be given:
+
+```
+(build-docs
+ :asdf-system \"cl-telegram-bot-docs\"
+ :dynamic-space-size \"4gb\")
+```
+
+")
   (0.15.0 2024-03-02
           "
 New
 ===
 
-* Now you can specify ENV argument to 40ANTS-CI:DEFWORKFLOW and any job. This should be an alist where keys are strings and values are evaluated during GitHub workflow generation phase. Read more in 40ANTS-CI-DOCS/INDEX::@ENV section.
-* 40ANTS-CI/JOBS/AUTOTAG:AUTOTAG function now ignores TOKEN-PATTERN argument if ENV argument was given and has GITHUB_TOKEN value for whole job.
+* Now you can specify ENV argument to 40ANTS-CI/WORKFLOW:DEFWORKFLOW and any job. This should be an alist where keys are strings and values are evaluated during GitHub workflow generation phase. Read more in 40ANTS-CI-DOCS/INDEX::@ENV section.
+* Also, 40ANTS-CI/JOBS/AUTOTAG:AUTOTAG function now ignores TOKEN-PATTERN argument if ENV argument was given and has `GITHUB_TOKEN` value for whole job.
 
 Backward incompatible changes
 =============================
 
 * When additional keyword arguments to 40ANTS-CI/STEPS/SH:SH function are given, they are transformed into env variables. Previously, their names were taken as is. Now they are uppercased and dash symbols are replaced with underscores.
 
-
 ")
   (0.14.0 2024-02-25
           "
@@ -101,7 +143,7 @@ Class 40ANTS-CI/JOBS/CRITIC:CRITIC was fixed for case when there are multiple cr
             renames to the IGNORE-CRITIQUES argument.")
   (0.6.0 2022-02-21
          "- New job type \"critic\" was added. It advices how to make you Lisp code better.
-            Learn more about this job type at 40ANTS-CI::@CRITIC section.")
+            Learn more about this job type at 40ANTS-CI-DOCS/INDEX::@CRITIC section.")
   (0.5.0 2022-01-28
          "- Move the actions/checkout action from v1 to v2.")
   (0.4.0 2022-01-28

diff --git a/docs/index.lisp b/docs/index.lisp
@@ -9,7 +9,7 @@
                 #:docs-config)
   (:import-from #:40ants-doc/autodoc
                 #:defautodoc)
-  (:import-from #:40ants-logging-docs/changelog
+  (:import-from #:40ants-ci-docs/changelog
                 #:@changelog)
   (:export #:@index
            #:@readme
@@ -35,6 +35,7 @@
                                    "JSON"
                                    "OS"
                                    "SBL"
+                                   "SBCL"
                                    "BSD"
                                    "TODO"
                                    "ASD"
@@ -96,7 +97,9 @@ of the package inferred ASDF system `EXAMPLE/CI`. A file should have the followi
 ```
 "
   (@job-types section)
-  (@caching section))
+  (@caching section)
+  (@env section)
+  (@running-custom-steps section))
 
 
 (defsection @job-types (:title "Job Types")
@@ -613,4 +616,27 @@ Note - environment variable names are always transformed to uppercase and dashes
 ")
 
 
+(defsection @running-custom-steps (:title "Running custom steps")
+  "
+Sometimes you might need to install custom system packages or do something before the job will finish. To accomplish
+these task you can provide custom steps using BEFORE-STEPS argument or AFTER-STEPS argument.
+
+Here is an example where we are installing system package libunaq1-dev before running the testsuite:
+
+
+```lisp
+(defparameter *required-steps*
+  (list (sh \"Install libunac\"
+            \"sudo apt-get install -y libunac1-dev\")))
+
+(defworkflow ci
+  :on-pull-request t
+  :cache t
+  :jobs ((run-tests
+          :steps-before *required-steps*
+          :asdf-system \"my-asdf-system\")))
+```
+")
+
+
 (defautodoc @api (:system "40ants-ci"))
diff --git a/qlfile.lock b/qlfile.lock
@@ -1,11 +1,11 @@
 ("quicklisp" .
  (:class qlot/source/dist:source-dist
-  :initargs (:distribution "http://beta.quicklisp.org/dist/quicklisp.txt" :%version :latest)
-  :version "2023-10-21"))
+  :initargs (:distribution "https://beta.quicklisp.org/dist/quicklisp.txt" :%version :latest)
+  :version "2024-10-12"))
 ("ultralisp" .
  (:class qlot/source/dist:source-dist
-  :initargs (:distribution "http://dist.ultralisp.org" :%version :latest)
-  :version "20240303155001"))
+  :initargs (:distribution "https://dist.ultralisp.org" :%version :latest)
+  :version "20250206134000"))
 ("bordeaux-threads" .
  (:class qlot/source/github:source-github
   :initargs (:repos "svetlyak40wt/bordeaux-threads" :ref nil :branch "fix-apiv2-for-no-threads" :tag nil)

diff --git a/src/ci.lisp b/src/ci.lisp
@@ -49,6 +49,8 @@
                       "ultralisp")
           :lisp ("sbcl-bin"
                  "ccl-bin")
+          ;; CCL does not work on arm64 architecture yet
+          :exclude ((:os "macos-latest" :lisp "ccl-bin"))
           :coverage t
           :qlfile "{% ifequal quicklisp_dist \"ultralisp\" %}
                    dist ultralisp http://dist.ultralisp.org

diff --git a/src/jobs/docs.lisp b/src/jobs/docs.lisp
@@ -1,4 +1,4 @@
-(defpackage #:40ants-ci/jobs/docs
+(uiop:define-package #:40ants-ci/jobs/docs
   (:use #:cl)
   (:import-from #:40ants-ci/jobs/lisp-job
                 #:asdf-system)
@@ -9,7 +9,7 @@
                 #:current-system-name)
   (:export #:build-docs
            #:error-on-warnings))
-(in-package 40ants-ci/jobs/docs)
+(in-package #:40ants-ci/jobs/docs)
 
 
 (defclass build-docs (40ants-ci/jobs/lisp-job:lisp-job)
@@ -19,16 +19,34 @@
   (:documentation "Builds documentation and uploads it to GitHub using [\"40ants/build-docs\" github action](https://40ants.com/build-docs/)."))
 
 
-(defun build-docs (&key asdf-system
-                     asdf-version
-                     (error-on-warnings t)
-                     env)
+(defun build-docs (&rest args
+                   &key
+                   (error-on-warnings t)
+                   ;; Settings from base JOB class
+                   os
+                   permissions
+                   exclude
+                   env
+                   steps
+                   steps-before
+                   steps-after
+                   ;; Settings from base LISP-JOB class
+                   roswell-version
+                   asdf-version
+                   qlot-version
+                   quicklisp
+                   lisp
+                   asdf-system
+                   qlfile
+                   dynamic-space-size)
   "Creates a job of class BUILD-DOCS."
-  (make-instance 'build-docs
-                 :asdf-system asdf-system
-                 :error-on-warnings error-on-warnings
-                 :asdf-version asdf-version
-                 :env env))
+  (declare (ignore asdf-system error-on-warnings
+                   os permissions exclude env steps
+                   steps-before steps-after
+                   roswell-version asdf-version qlot-version
+                   quicklisp lisp qlfile dynamic-space-size))
+  (apply #'make-instance
+         'build-docs args))
 
 
 (defmethod 40ants-ci/jobs/job:steps ((job build-docs))

diff --git a/src/jobs/job.lisp b/src/jobs/job.lisp
@@ -22,7 +22,9 @@
            #:make-permissions
            #:explicit-steps
            #:exclude
-           #:job-env))
+           #:job-env
+           #:steps-before
+           #:steps-after))
 (in-package #:40ants-ci/jobs/job)
 
 
@@ -46,6 +48,14 @@
           :initarg :steps
           :documentation "This slot holds steps given as a STEPS argument to a job constructor. Depending on a job class, it might add additional steps around these explicit steps."
           :reader explicit-steps)
+   (steps-before :initform nil
+                 :initarg :steps-before
+                 :documentation "This slot holds steps given as a STEPS-BEFORE argument to a job constructor. These steps will be prepended to steps returned by the JOB class."
+                 :reader steps-before)
+   (steps-after :initform nil
+                :initarg :steps-after
+                :documentation "This slot holds steps given as a STEPS-AFTER argument to a job constructor. These steps will be appended to steps returned by the JOB class."
+                :reader steps-after)
    (permissions :initform nil
                 :initarg :permissions
                 :documentation "A plist of permissions need for running the job.
@@ -92,8 +102,11 @@
     (explicit-steps job))
 
   (:method :around ((job job))
-    (uiop:ensure-list
-     (call-next-method))))
+    (append
+     (steps-before job)
+     (uiop:ensure-list
+      (call-next-method))
+     (steps-after job))))
 
 
 (defmethod exclude :around ((job job))

diff --git a/src/jobs/linter.lisp b/src/jobs/linter.lisp
@@ -1,4 +1,4 @@
-(defpackage #:40ants-ci/jobs/linter
+(uiop:define-package #:40ants-ci/jobs/linter
   (:use #:cl)
   (:import-from #:40ants-ci/steps/sh
                 #:sh)
@@ -10,7 +10,7 @@
   (:export #:linter
            #:asdf-systems
            #:check-imports))
-(in-package 40ants-ci/jobs/linter)
+(in-package #:40ants-ci/jobs/linter)
 
 
 (defclass linter (40ants-ci/jobs/lisp-job:lisp-job)
@@ -32,17 +32,39 @@
        (current-system-name))))
 
 
-(defun linter (&key asdf-systems asdf-version check-imports env)
+(defun linter (&rest args
+               &key
+               asdf-systems
+               check-imports
+               ;; Settings from base JOB class
+               os
+               permissions
+               exclude
+               env
+               steps
+               steps-before
+               steps-after
+               ;; Settings from base LISP-JOB class
+               roswell-version
+               asdf-version
+               qlot-version
+               quicklisp
+               lisp
+               qlfile
+               dynamic-space-size)
   "Creates a job which will run SBLint for given ASDF systems.
 
    If no ASD files given, it will use all ASD files from
    the current ASDF system."
-  (make-instance 'linter
-                 :asdf-system (first asdf-systems)
-                 :asdf-systems asdf-systems
-                 :asdf-version asdf-version
-                 :env env
-                 :check-imports check-imports))
+  (declare (ignore check-imports os permissions exclude env
+                   steps steps-before steps-after permissions
+                   roswell-version asdf-version qlot-version
+                   quicklisp lisp qlfile dynamic-space-size))
+
+  (apply #'make-instance
+         'linter
+         :asdf-system (first asdf-systems)
+         args))
 
 
 (defmethod 40ants-ci/jobs/job:steps ((job linter))

diff --git a/src/jobs/lisp-job.lisp b/src/jobs/lisp-job.lisp
@@ -21,14 +21,15 @@
            #:qlfile
            #:asdf-version
            #:roswell-version
-           #:qlot-version))
+           #:qlot-version
+           #:dynamic-space-size))
 (in-package 40ants-ci/jobs/lisp-job)
 
 
 (defclass lisp-job (40ants-ci/jobs/job:job)
   ((quicklisp :initform "quicklisp"
-                 :initarg :quicklisp
-                 :reader quicklisp)
+              :initarg :quicklisp
+              :reader quicklisp)
    (lisp :initform "sbcl-bin"
          :initarg :lisp
          :reader lisp)
@@ -49,6 +50,11 @@
                     :type (or null string)
                     :documentation "Roswell version to use when setting up Lisp environment. If NIL, then will be used version, pinned in `setup-lisp` github action."
                     :reader roswell-version)
+   (dynamic-space-size :initarg :dynamic-space-size
+                       :initform nil
+                       :type (or null string)
+                       :documentation "Dynamic space size for SBCL."
+                       :reader dynamic-space-size)
    (qlot-version :initarg :qlot-version
                  :initform nil
                  :type (or null string)
@@ -142,7 +148,8 @@
                    :qlot-version (qlot-version job)
                    :qlfile-template (when (qlfile job)
                                       (dedent (qlfile job)))
+                   :dynamic-space-size (dynamic-space-size job)
                    :cache (if *use-cache*
-                              "true"
-                              "false")))
+                            "true"
+                            "false")))
           (call-next-method)))
-Original file line number
+Diff line change
@@ Expand Up / @@ -3,3 +3,4 @@ @@
     /.qlot
     *.fasl
     .DS_Store
+    *.~undo-tree~