Skip to content
This repository was archived by the owner on Jun 15, 2024. It is now read-only.

Commit 8f92b1b

Browse files
flosellflosell
committed
Implement PipelineStructureConsumer and PipelineStructureSource in default persistence mechanism (#135, #6)
1 parent 2a45b20 commit 8f92b1b

9 files changed

+192
-119
lines changed

‎CHANGELOG.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ The official release will have a defined and more stable API. If you are already
88
## 0.11.0
99

1010
* Improvements:
11-
* Keeps a history of pipeline structure if persistence component supports it (#131, #6)
11+
* Keeps a history of pipeline structure if persistence component supports it (#131, #6); Implemented for default persistence
1212
* Bug fixes:
1313
* Fix deadlock occurring when steps write a lot of step-results in quick succession and step results are inherited by their parents (as in chaining) (#135)
1414
* API changes:

‎src/clj/lambdacd/internal/default_pipeline_state.clj

+28-16
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,10 @@
33
i.e. what's currently running, what are the results of each step, ..."
44
(:require [lambdacd.internal.default-pipeline-state-persistence :as persistence]
55
[clj-time.core :as t]
6-
[lambdacd.internal.pipeline-state :as pipeline-state]
7-
[lambdacd.util :as util]))
6+
[lambdacd.internal.pipeline-state :as old-pipeline-state]
7+
[lambdacd.state.protocols :as protocols]
8+
[lambdacd.util :as util]
9+
[clojure.data :as data]))
810

911
(def clean-pipeline-state {})
1012

@@ -21,36 +23,46 @@
2123
(defn- update-step-result-in-state [build-number step-id new-step-result current-state]
2224
(update-in current-state [build-number step-id] #(update-step-result new-step-result %)))
2325

24-
(defn- truncate-build-history [max-builds state]
25-
(->> state
26-
(sort-by key >)
27-
(take max-builds)
28-
(into {})))
26+
(defn- truncate-build-history [home-dir max-builds state]
27+
(let [new-state (->> state
28+
(sort-by key >)
29+
(take max-builds)
30+
(into {}))
31+
[only-in-old _ _] (data/diff (set (keys state)) (set (keys new-state)))]
32+
(persistence/clean-up-old-builds home-dir only-in-old)
33+
new-state))
2934

3035
(defn- most-recent-build-number-in-state [pipeline-state]
3136
(if-let [current-build-number (last (sort (keys pipeline-state)))]
3237
current-build-number
3338
0))
3439

35-
(defrecord DefaultPipelineState [state-atom home-dir max-builds]
36-
pipeline-state/PipelineStateComponent
40+
(defrecord DefaultPipelineState [state-atom structure-atom home-dir max-builds]
41+
old-pipeline-state/PipelineStateComponent
3742
(update [self build-number step-id new-step-result]
3843
(if (not (nil? state-atom)) ; convenience for tests: if no state exists we just do nothing
3944
(let [new-state (swap! state-atom #(->> %
4045
(update-step-result-in-state build-number step-id new-step-result)
41-
(truncate-build-history max-builds)))]
42-
(persistence/write-build-history home-dir build-number new-state)
43-
(persistence/clean-up-old-history home-dir new-state))))
46+
(truncate-build-history home-dir max-builds)))]
47+
(persistence/write-build-history home-dir build-number new-state))))
4448
(get-all [self]
4549
@state-atom)
4650
(get-internal-state [self]
4751
state-atom)
4852
(next-build-number [self]
49-
(inc (most-recent-build-number-in-state @state-atom))))
53+
(inc (most-recent-build-number-in-state @state-atom)))
54+
protocols/PipelineStructureConsumer
55+
(consume-pipeline-structure [self build-number pipeline-structure-representation]
56+
(swap! structure-atom #(assoc % build-number pipeline-structure-representation))
57+
(persistence/write-pipeline-structure home-dir build-number pipeline-structure-representation))
58+
protocols/PipelineStructureSource
59+
(get-pipeline-structure [self build-number]
60+
(get @structure-atom build-number)))
5061

5162
(defn new-default-pipeline-state [config & {:keys [initial-state-for-testing]}]
52-
(let [state-atom (atom (or initial-state-for-testing (initial-pipeline-state config)))
53-
home-dir (:home-dir config)
63+
(let [home-dir (:home-dir config)
64+
state-atom (atom (or initial-state-for-testing (initial-pipeline-state config)))
65+
structure-atom (atom (persistence/read-pipeline-structures home-dir))
5466
max-builds (or (:max-builds config) Integer/MAX_VALUE)
55-
instance (->DefaultPipelineState state-atom home-dir max-builds)]
67+
instance (->DefaultPipelineState state-atom structure-atom home-dir max-builds)]
5668
instance))

‎src/clj/lambdacd/internal/default_pipeline_state_persistence.clj

+41-22
Original file line numberDiff line numberDiff line change
@@ -45,14 +45,16 @@
4545
(into {} (map step-result-with-formatted-step-ids->step-result m)))
4646

4747
(defn- build-number-from-path [path]
48-
(util/parse-int (second (re-find #"build-(\d+)" path))))
48+
(util/parse-int (second (re-find #"build-(\d+)" (str path)))))
4949

5050
(defn- build-state-path [dir]
51-
(str dir "/" "build-state.edn"))
51+
(io/file dir "build-state.edn"))
5252

53-
(defn- read-build-edn [dir]
54-
(let [path (build-state-path dir)
55-
build-number (build-number-from-path path)
53+
(defn- pipeline-structure-path [dir]
54+
(io/file dir "pipeline-structure.edn"))
55+
56+
(defn- read-build-edn [path]
57+
(let [build-number (build-number-from-path path)
5658
data-str (slurp path)
5759
state (formatted-step-ids->pipeline-state (dates->clj-times (edn/read-string data-str)))]
5860
{build-number state}))
@@ -69,25 +71,42 @@
6971
build-dirs (filter #(.startsWith (.getName %) "build-") directories-in-home)]
7072
build-dirs))
7173

74+
(defn- build-dir [home-dir build-number]
75+
(let [result (str home-dir "/" "build-" build-number)]
76+
(.mkdirs (io/file result))
77+
result))
78+
7279
(defn write-build-history [home-dir build-number new-state]
7380
(if home-dir
74-
(let [dir (str home-dir "/" "build-" build-number)
75-
edn-path (build-state-path dir)
76-
build (get new-state build-number)]
77-
(.mkdirs (io/file dir))
81+
(let [dir (build-dir home-dir build-number)
82+
edn-path (build-state-path dir)
83+
build (get new-state build-number)]
7884
(write-build-edn edn-path build))))
7985

80-
(defn read-build-history-from [home-dir]
81-
(let [states (map read-build-edn (build-dirs home-dir))]
82-
(into {} states)))
86+
(defn file-exists? [f]
87+
(.exists f))
8388

84-
(defn clean-up-old-history [home-dir new-state]
85-
(if home-dir
86-
(let [existing-build-dirs (map str (build-dirs home-dir))
87-
expected-build-numbers (keys new-state)
88-
expected-build-dirs (map #(str (io/file home-dir (str "build-" %))) expected-build-numbers)
89-
[only-in-existing _ _] (data/diff (set existing-build-dirs) (set expected-build-dirs))]
90-
(doall (for [old-build-dir only-in-existing]
91-
(do
92-
(log/info "Cleaning up old build directory" old-build-dir)
93-
(fs/delete-dir old-build-dir)))))))
89+
(defn read-build-history-from [home-dir]
90+
(->> (build-dirs home-dir)
91+
(map build-state-path)
92+
(filter file-exists?)
93+
(map read-build-edn)
94+
(into {})))
95+
96+
(defn write-pipeline-structure [home-dir build-number pipeline-structure]
97+
(let [f (pipeline-structure-path (build-dir home-dir build-number))]
98+
(spit f (pr-str pipeline-structure))))
99+
100+
(defn- read-pipeline-structure-edn [f]
101+
{(build-number-from-path f) (edn/read-string (slurp f))})
102+
103+
(defn read-pipeline-structures [home-dir]
104+
(->> (build-dirs home-dir)
105+
(map pipeline-structure-path)
106+
(filter file-exists?)
107+
(map read-pipeline-structure-edn)
108+
(into {})))
109+
110+
(defn clean-up-old-builds [home-dir old-build-numbers]
111+
(doall (map (fn [old-build-number]
112+
(fs/delete-dir (build-dir home-dir old-build-number))) old-build-numbers)))

‎test/clj/lambdacd/internal/default_pipeline_state_persistence_test.clj

+53-26
Original file line numberDiff line numberDiff line change
@@ -2,35 +2,45 @@
22
(:use [lambdacd.testsupport.test-util])
33
(:require [clojure.test :refer :all]
44
[lambdacd.internal.default-pipeline-state-persistence :refer :all]
5+
[lambdacd.presentation.pipeline-structure-test :as pipeline-structure-test]
56
[clj-time.core :as t]
67
[lambdacd.util :as utils]
78
[clojure.java.io :as io]))
89

910
(deftest roundtrip-persistence-test
10-
(testing "the standard case"
11-
(let [some-pipeline-state {3 {'(0) {:status :success :most-recent-update-at (t/epoch)}
12-
'(0 1 2) {:status :failure :out "something went wrong"}}}
13-
home-dir (utils/create-temp-dir)]
14-
(write-build-history home-dir 3 some-pipeline-state)
15-
(is (= some-pipeline-state (read-build-history-from home-dir)))))
16-
(testing "that string-keys in a step result are suppored as well (#101)"
17-
(let [some-pipeline-state {3 {'(0) {:status :success :_git-last-seen-revisions {"refs/heads/master" "some-sha"}}}}
18-
home-dir (utils/create-temp-dir)]
19-
(write-build-history home-dir 3 some-pipeline-state)
20-
(is (= some-pipeline-state (read-build-history-from home-dir)))))
21-
(testing "that keyworded values in a step result are suppored as well (#101)"
22-
(let [some-pipeline-state {3 {'(0) {:status :success :v :x}}}
23-
home-dir (utils/create-temp-dir)]
24-
(write-build-history home-dir 3 some-pipeline-state)
25-
(is (= some-pipeline-state (read-build-history-from home-dir)))))
11+
(testing "pipeline-state"
12+
(testing "the standard case"
13+
(let [some-pipeline-state {3 {'(0) {:status :success :most-recent-update-at (t/epoch)}
14+
'(0 1 2) {:status :failure :out "something went wrong"}}}
15+
home-dir (utils/create-temp-dir)]
16+
(write-build-history home-dir 3 some-pipeline-state)
17+
(is (= some-pipeline-state (read-build-history-from home-dir)))))
18+
(testing "that string-keys in a step result are supported as well (#101)"
19+
(let [some-pipeline-state {3 {'(0) {:status :success :_git-last-seen-revisions {"refs/heads/master" "some-sha"}}}}
20+
home-dir (utils/create-temp-dir)]
21+
(write-build-history home-dir 3 some-pipeline-state)
22+
(is (= some-pipeline-state (read-build-history-from home-dir)))))
23+
(testing "that keyworded values in a step result are suppored as well (#101)"
24+
(let [some-pipeline-state {3 {'(0) {:status :success :v :x}}}
25+
home-dir (utils/create-temp-dir)]
26+
(write-build-history home-dir 3 some-pipeline-state)
27+
(is (= some-pipeline-state (read-build-history-from home-dir))))))
28+
(testing "pipeline-structure"
29+
(testing "the standard case"
30+
(let [home-dir (utils/create-temp-dir)
31+
some-pipeline-structure pipeline-structure-test/foo-pipeline-display-representation]
32+
(write-pipeline-structure home-dir 1 some-pipeline-structure)
33+
(write-pipeline-structure home-dir 2 some-pipeline-structure)
34+
(is (= some-pipeline-structure (get (read-pipeline-structures home-dir) 1)))
35+
(is (= some-pipeline-structure (get (read-pipeline-structures home-dir) 2)))))))
36+
37+
(deftest clean-up-old-builds-test
2638
(testing "cleaning up old history"
27-
(testing "that build-directories will be deleted if they no longer exist in the build-state"
28-
(let [some-pipeline-state {0 {'(0) {:status :success}}
29-
1 {'(0) {:status :success}}
30-
2 {'(0) {:status :success}}}
31-
truncated-pipeline-state {1 {'(0) {:status :success}}
39+
(testing "that given build-directories will be deleted"
40+
(let [some-pipeline-state {0 {'(0) {:status :success}}
41+
1 {'(0) {:status :success}}
3242
2 {'(0) {:status :success}}}
33-
home-dir (utils/create-temp-dir)]
43+
home-dir (utils/create-temp-dir)]
3444
(doall (for [build (range 0 3)]
3545
(write-build-history home-dir build some-pipeline-state)))
3646
(doall (for [build (range 0 3)]
@@ -39,19 +49,36 @@
3949
(is (.exists (io/file home-dir "build-0")))
4050
(is (.exists (io/file home-dir "build-1")))
4151
(is (.exists (io/file home-dir "build-2")))
42-
(clean-up-old-history home-dir truncated-pipeline-state)
52+
(clean-up-old-builds home-dir [0])
4353

4454
(is (not (.exists (io/file home-dir "build-0"))))
4555
(is (.exists (io/file home-dir "build-1")))
4656
(is (.exists (io/file home-dir "build-2")))))
4757
(testing "that it does not clean up things other than build directories"
48-
(let [truncated-pipeline-state {}
49-
home-dir (utils/create-temp-dir)]
58+
(let [home-dir (utils/create-temp-dir)]
5059
(.mkdirs (io/file home-dir "helloworld"))
5160
(is (.exists (io/file home-dir "helloworld")))
52-
(clean-up-old-history home-dir truncated-pipeline-state)
61+
(clean-up-old-builds home-dir [0])
5362
(is (.exists (io/file home-dir "helloworld")))))))
5463

64+
(deftest read-build-history-from-test ; covers only edge-cases that aren't coverd by roundtrip
65+
(testing "that it will return an empty history if no state has been written yet"
66+
(let [home-dir (utils/create-temp-dir)]
67+
(is (= {} (read-build-history-from home-dir)))))
68+
(testing "that it ignores build directories with no build state (e.g. because only structure has been written yet"
69+
(let [home-dir (utils/create-temp-dir)]
70+
(.mkdirs (io/file home-dir "build-1"))
71+
(is (= {} (read-build-history-from home-dir))))))
72+
73+
(deftest read-pipeline-structures-test ; covers only edge-cases that aren't covered by roundtrip
74+
(testing "that it will return an empty data if no state has been written yet"
75+
(let [home-dir (utils/create-temp-dir)]
76+
(is (= {} (read-pipeline-structures home-dir)))))
77+
(testing "that it ignores build directories with no pipeline structure (e.g. because they were created before this feature was available)"
78+
(let [home-dir (utils/create-temp-dir)]
79+
(.mkdirs (io/file home-dir "build-1"))
80+
(is (= {} (read-pipeline-structures home-dir))))))
81+
5582
(defn- roundtrip-date-time [data]
5683
(dates->clj-times
5784
(clj-times->dates data)))

0 commit comments

Comments
 (0)