Skip to content
This repository was archived by the owner on Jun 15, 2024. It is now read-only.

Commit c2135bf

Browse files
flosellflosell
committed
Allow persistence of pipeline structure (#131, #6)
1 parent ca2a5ea commit c2135bf

File tree

9 files changed

+141
-57
lines changed

9 files changed

+141
-57
lines changed

‎CHANGELOG.md

+4-1
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,14 @@ The official release will have a defined and more stable API. If you are already
77

88
## 0.11.0
99

10+
* Keeps a history of pipeline structure if persistence component supports it (#131, #6)
1011
* API changes:
1112
* New state handling (#131):
1213
* Protocols in `lambdacd.state.protocols` replace `lambdacd.internal.pipeline-state/PipelineStateComponent` which is now deprecated. Custom persistence-mechanisms need to migrate.
1314
* Added facade `lambdacd.state.core` for all state-related functionality. Access directly to `PipelineStateComponent` is now deprecated.
14-
* `lambdacd.presentation.pipeline-state/history-for` should now be called with ctx; Calling it with a build-state (the result of `lambdacd.internal.pipeline-state/get-all`) still works but is now deprecated.
15+
* `lambdacd.presentation.pipeline-state/history-for` should now be called with ctx; Calling it with a build-state (the result of `lambdacd.internal.pipeline-state/get-all`) still works but is now deprecated.
16+
* `lambdacd.presentation.unified/unified-presentation` is now deprecated, use `lambdacd.presentation.unified/pipeline-structure-with-step-results` instead
17+
* The current pipeline-definition can now be accessed as `:pipeline-def` in ctx
1518
* Breaking Changes:
1619
* Moved pipeline-state-updater from `lambdacd.internal.pipeline-state` to `lambdacd.state.internal.pipeline-state-updater` and refactored interface. As this is an internal namespace, it should not affect users unless they customized LambdaCDs startup procedure to a large degree.
1720

‎src/clj/lambdacd/core.clj

+1
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
(event-bus/initialize-event-bus)
3838
(running-builds-tracking/initialize-running-builds-tracking)
3939
(assoc :pipeline-state-component pipeline-state-component)
40+
(assoc :pipeline-def pipeline-def)
4041
(initialize-pipeline-state-updater)
4142
(add-shutdown-sequence!))]
4243
{:context context

‎src/clj/lambdacd/internal/execution.clj

+8-5
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,8 @@
88
[lambdacd.steps.status :as status]
99
[clojure.repl :as repl]
1010
[lambdacd.event-bus :as event-bus]
11-
[lambdacd.steps.result :as step-results])
11+
[lambdacd.steps.result :as step-results]
12+
[lambdacd.presentation.pipeline-structure :as pipeline-structure])
1213
(:import (java.io StringWriter)
1314
(java.util UUID)))
1415

@@ -298,18 +299,20 @@
298299

299300
(defn run [pipeline context]
300301
(let [build-number (state/next-build-number context)]
302+
(state/consume-pipeline-structure context build-number (pipeline-structure/pipeline-display-representation pipeline))
301303
(let [runnable-pipeline (map eval pipeline)]
302304
(execute-steps runnable-pipeline {} (merge context {:result-channel (async/chan (async/dropping-buffer 0))
303-
:step-id []
304-
:build-number build-number})))))
305+
:step-id []
306+
:build-number build-number})))))
305307

306308
(defn retrigger [pipeline context build-number step-id-to-run next-build-number]
307-
(let [executable-pipeline (map eval pipeline) ]
309+
(let [executable-pipeline (map eval pipeline)]
310+
(state/consume-pipeline-structure context build-number (pipeline-structure/pipeline-display-representation pipeline))
308311
(execute-steps executable-pipeline {} (assoc context :step-id []
309312
:result-channel (async/chan (async/dropping-buffer 0))
310313
:build-number next-build-number
311314
:retriggered-build-number build-number
312-
:retriggered-step-id step-id-to-run))))
315+
:retriggered-step-id step-id-to-run))))
313316

314317
(defn retrigger-async [pipeline context build-number step-id-to-run]
315318
(let [next-build-number (state/next-build-number context)]

‎src/clj/lambdacd/presentation/unified.clj

+12-9
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,18 @@
11
(ns lambdacd.presentation.unified
22
(:require [lambdacd.presentation.pipeline-structure :as pipeline-structure]))
33

4-
(defn- unify [step build-state]
5-
(let [step-id (:step-id step)
6-
result-for-step (get build-state step-id {})
7-
with-result (assoc step :result result-for-step )
8-
children (:children step)
9-
unified-children (map #(unify % build-state) children)
10-
with-children (assoc with-result :children unified-children)]
4+
(defn- unify-step [step build-state]
5+
(let [step-id (:step-id step)
6+
result-for-step (get build-state step-id {})
7+
with-result (assoc step :result result-for-step)
8+
children (:children step)
9+
unified-children (map #(unify-step % build-state) children)
10+
with-children (assoc with-result :children unified-children)]
1111
with-children))
1212

13-
(defn unified-presentation [pipeline-def build-state]
13+
(defn pipeline-structure-with-step-results [pipeline-structure step-results]
14+
(map #(unify-step % step-results) pipeline-structure))
15+
16+
(defn unified-presentation [pipeline-def step-results]
1417
(let [structure (pipeline-structure/pipeline-display-representation pipeline-def)]
15-
(map #(unify % build-state) structure)))
18+
(pipeline-structure-with-step-results structure step-results)))

‎src/clj/lambdacd/state/core.clj

+14-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
(ns lambdacd.state.core
22
"Facade for all functions related to dealing with LambdaCDs state. Wraps the related interfaces to simplify compatibility and API."
33
(:require [lambdacd.state.protocols :as protocols]
4-
[lambdacd.internal.pipeline-state :as legacy-pipeline-state]))
4+
[lambdacd.internal.pipeline-state :as legacy-pipeline-state]
5+
[lambdacd.presentation.pipeline-structure :as pipeline-structure]))
56

67
(defn- all-build-numbers-from-legacy [component]
78
(->> (legacy-pipeline-state/get-all component)
@@ -18,6 +19,7 @@
1819
(:pipeline-state-component ctx))
1920

2021
; -------------------------------------------------------------------------
22+
2123
(defn consume-step-result-update
2224
"Update a step-result in the state"
2325
[ctx build-number step-id step-result]
@@ -26,6 +28,11 @@
2628
(protocols/consume-step-result-update component build-number step-id step-result)
2729
(legacy-pipeline-state/update component build-number step-id step-result))))
2830

31+
(defn consume-pipeline-structure [ctx build-number pipeline-structure-representation]
32+
(let [component (state-component ctx)]
33+
(if (satisfies? protocols/PipelineStructureConsumer component)
34+
(protocols/consume-pipeline-structure component build-number pipeline-structure-representation))))
35+
2936
(defn next-build-number
3037
"Returns the build number for the next build"
3138
[ctx]
@@ -60,3 +67,9 @@
6067
[ctx build-number step-id]
6168
(get (get-step-results ctx build-number)
6269
step-id))
70+
71+
(defn get-pipeline-structure
72+
"Returns a map describing the structure of the pipeline"
73+
[ctx build-number]
74+
(or (:pipeline-structure (get-build ctx build-number))
75+
(pipeline-structure/pipeline-display-representation (:pipeline-def ctx))))

‎src/clj/lambdacd/state/protocols.clj

+6-1
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,11 @@
66
(consume-step-result-update [self build-number step-id step-result]
77
"Tells the component to update the result of a particular step. Is called on every update so it needs to handle lots of requests"))
88

9+
(defprotocol PipelineStructureConsumer
10+
"Components implementing this protocol can set the structure a pipeline had for a particular build"
11+
(consume-pipeline-structure [self build-number pipeline-structure-representation]
12+
"Tells the component to update the structure of a particular build."))
13+
914
(defprotocol NextBuildNumberSource
1015
"Components implementing this protocol provide the LambdaCD execution engine with new build numbers"
1116
(next-build-number [self]
@@ -19,4 +24,4 @@
1924
(defprotocol QueryBuildSource
2025
"Components implementing this protocol can supply information on one build"
2126
(get-build [self build-number]
22-
"Returns build information as a map with :step-results, and TODO"))
27+
"Returns build information as a map with :step-results, :pipeline-structure and TODO"))

‎src/clj/lambdacd/ui/api.clj

+8-6
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,13 @@
1111
[compojure.core :refer [routes GET POST]]
1212
[lambdacd.state.core :as state]))
1313

14-
(defn- build-infos [pipeline-def ctx buildnumber]
15-
(let [build-state (state/get-step-results ctx (util/parse-int buildnumber))]
16-
(if build-state
17-
(util/json (unified/unified-presentation pipeline-def build-state))
18-
(resp/not-found (str "build " buildnumber " does not exist")))))
14+
(defn- build-infos [ctx build-number-str]
15+
(let [build-number (util/parse-int build-number-str)
16+
pipeline-structure (state/get-pipeline-structure ctx build-number)
17+
step-results (state/get-step-results ctx build-number)]
18+
(if (and pipeline-structure step-results)
19+
(util/json (unified/pipeline-structure-with-step-results pipeline-structure step-results))
20+
(resp/not-found (str "build " build-number-str " does not exist")))))
1921

2022
(defn- to-internal-step-id [dash-seperated-step-id]
2123
(map util/parse-int (string/split dash-seperated-step-id #"-")))
@@ -24,7 +26,7 @@
2426
(ring-json/wrap-json-params
2527
(routes
2628
(GET "/builds/" [] (util/json (state-presentation/history-for ctx)))
27-
(GET "/builds/:buildnumber/" [buildnumber] (build-infos pipeline-def ctx buildnumber))
29+
(GET "/builds/:buildnumber/" [buildnumber] (build-infos ctx buildnumber))
2830
(POST "/builds/:buildnumber/:step-id/retrigger" [buildnumber step-id]
2931
(let [new-buildnumber (execution/retrigger pipeline-def ctx (util/parse-int buildnumber) (to-internal-step-id step-id))]
3032
(util/json {:build-number new-buildnumber})))

‎test/clj/lambdacd/presentation/unified_test.clj

+62-33
Original file line numberDiff line numberDiff line change
@@ -13,43 +13,72 @@
1313

1414

1515
(def foo-pipeline-build-state
16-
{'(1) { :status :running}
16+
{'(1) {:status :running}
1717
'(1 1 1) {:status :failure
18-
:out "do stuff failed"}
19-
'(1 2 1) {:status :running
18+
:out "do stuff failed"}
19+
'(1 2 1) {:status :running
2020
:some-key :some-value}})
2121

2222
(def expected-unified-foo-pipeline-presentation
23-
[{:name "in-parallel"
24-
:type :parallel
25-
:step-id '(1)
26-
:has-dependencies false
27-
:result {:status :running }
28-
:children
29-
[{:name "in-cwd"
30-
:type :container
31-
:step-id '(1 1)
32-
:has-dependencies false
33-
:result {}
34-
:children [{:name "do-stuff"
35-
:type :step
36-
:step-id '(1 1 1)
37-
:has-dependencies false
38-
:children []
39-
:result {:status :failure
40-
:out "do stuff failed"}}]}
41-
{:name "in-cwd"
42-
:type :container
43-
:step-id '(2 1)
44-
:has-dependencies false
45-
:result {}
46-
:children [{:name "do-other-stuff"
47-
:type :step
48-
:step-id '(1 2 1)
49-
:has-dependencies false
50-
:children []
51-
:result {:status :running :some-key :some-value}}]}]}])
23+
[{:name "in-parallel"
24+
:type :parallel
25+
:step-id '(1)
26+
:has-dependencies false
27+
:result {:status :running}
28+
:children
29+
[{:name "in-cwd"
30+
:type :container
31+
:step-id '(1 1)
32+
:has-dependencies false
33+
:result {}
34+
:children [{:name "do-stuff"
35+
:type :step
36+
:step-id '(1 1 1)
37+
:has-dependencies false
38+
:children []
39+
:result {:status :failure
40+
:out "do stuff failed"}}]}
41+
{:name "in-cwd"
42+
:type :container
43+
:step-id '(2 1)
44+
:has-dependencies false
45+
:result {}
46+
:children [{:name "do-other-stuff"
47+
:type :step
48+
:step-id '(1 2 1)
49+
:has-dependencies false
50+
:children []
51+
:result {:status :running :some-key :some-value}}]}]}])
52+
53+
(def foo-pipeline-structure
54+
[{:name "in-parallel"
55+
:type :parallel
56+
:step-id '(1)
57+
:has-dependencies false
58+
:children
59+
[{:name "in-cwd"
60+
:type :container
61+
:step-id '(1 1)
62+
:has-dependencies false
63+
:children [{:name "do-stuff"
64+
:type :step
65+
:step-id '(1 1 1)
66+
:has-dependencies false
67+
:children []}]}
68+
{:name "in-cwd"
69+
:type :container
70+
:step-id '(2 1)
71+
:has-dependencies false
72+
:result {}
73+
:children [{:name "do-other-stuff"
74+
:type :step
75+
:step-id '(1 2 1)
76+
:has-dependencies false
77+
:children []}]}]}])
5278

5379
(deftest unified-presentation-test
5480
(testing "that we can merge structure and state to a unified view on a pipeline-run"
55-
(is (= expected-unified-foo-pipeline-presentation (unified-presentation foo-pipeline foo-pipeline-build-state)))))
81+
(testing "deprecated call with pipeline-def"
82+
(is (= expected-unified-foo-pipeline-presentation (unified-presentation foo-pipeline foo-pipeline-build-state))))
83+
(testing "call with build-result"
84+
(is (= expected-unified-foo-pipeline-presentation (pipeline-structure-with-step-results foo-pipeline-structure foo-pipeline-build-state))))))

‎test/clj/lambdacd/state/core_test.clj

+26-1
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@
99
(def some-build-number 42)
1010
(def some-step-id [0])
1111
(def some-step-result {:foo :bat})
12+
(def some-structure {:some :structure})
13+
(def some-pipeline-def `(foo))
1214

1315
(deftest consume-step-result-update-test
1416
(testing "that calls to a StepResultUpdateConsumer will just pass through"
@@ -22,6 +24,17 @@
2224
some-build-number some-step-id some-step-result)
2325
(is (received? component legacy-pipeline-state/update [some-build-number some-step-id some-step-result])))))
2426

27+
(deftest consume-pipeline-structure-test
28+
(testing "that calls to a PipelineStructureConsumer will just pass through"
29+
(let [component (mock state-protocols/PipelineStructureConsumer)]
30+
(s/consume-pipeline-structure (some-ctx-with :pipeline-state-component component)
31+
some-build-number some-structure)
32+
(is (received? component state-protocols/consume-pipeline-structure [some-build-number some-structure]))))
33+
(testing "that calls to a legacy PipelineStateComponent are ignored"
34+
(let [component (mock legacy-pipeline-state/PipelineStateComponent)]
35+
(s/consume-pipeline-structure (some-ctx-with :pipeline-state-component component)
36+
some-build-number some-structure))))
37+
2538
(deftest next-build-number-test
2639
(testing "that calls to a BuildNumberSource will just pass through"
2740
(let [component (mock state-protocols/NextBuildNumberSource)]
@@ -65,7 +78,19 @@
6578
(with-redefs [s/get-build (constantly {:step-results {:step :results}})]
6679
(is (= {:step :results} (s/get-step-results nil nil))))))
6780

68-
(deftest step-result-test
81+
(deftest get-step-result-test
6982
(testing "that we can get a simple step-result"
7083
(with-redefs [s/get-build (constantly {:step-results {[2 1] {:step :result}}})]
7184
(is (= {:step :result} (s/get-step-result nil 1 [2 1]))))))
85+
86+
(deftest get-pipeline-structure-test
87+
(testing "that we get the pipeline structure if it is contained in the build"
88+
(with-redefs [s/get-build (constantly {:pipeline-structure some-structure})]
89+
(is (= some-structure (s/get-pipeline-structure nil some-build-number)))))
90+
(testing "that we get the current pipeline structure if the build result doesn't contain it (for compatibility reasons)"
91+
(with-redefs [s/get-build (constantly {:no :structure-here})]
92+
(is (= [{:name "foo"
93+
:type :unknown
94+
:has-dependencies false
95+
:step-id `(1)}] (s/get-pipeline-structure (some-ctx-with :pipeline-def some-pipeline-def) some-build-number))))))
96+

0 commit comments

Comments
 (0)