Skip to content

Commit ab2a61e

Browse files
committed
chore: etlp-mapper
1 parent 9502b60 commit ab2a61e

File tree

10 files changed

+497
-25
lines changed

10 files changed

+497
-25
lines changed

.gitlab-ci.yml

+57
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
.only-default: &only-default
2+
only:
3+
- merge_requests
4+
5+
.only-default-mr: &only-default-mr
6+
only:
7+
- merge_requests
8+
- master
9+
10+
include:
11+
- project: 'infrastructure/ci-helm'
12+
ref: master
13+
file: 'ci-templates/ci.yml'
14+
15+
variables:
16+
PRODUCT: dap
17+
MICROSERVICE_NAME: dap_etlp_mapper
18+
DOCKER_IMAGE: data-standardization/etlp-mapper
19+
BUILD_TYPE: DOCKERCOMPOSE
20+
DELAY: "11 min"
21+
CI_SONAR_COVERAGE_REPORT_PATH: "target/coverage/coverage.xml"
22+
23+
StyleCheck:
24+
image: clojure:lein
25+
stage: linting
26+
variables:
27+
KUBERNETES_SERVICE_MEMORY_LIMIT: 4Gi
28+
KUBERNETES_SERVICE_CPU_LIMIT: 4
29+
script:
30+
- echo "TODO lint"
31+
tags:
32+
- docker
33+
allow_failure: false
34+
<<: *only-default
35+
36+
.Coverage:
37+
image: clojure:lein
38+
tags:
39+
- gitlab-runner
40+
allow_failure: false
41+
<<: *only-default
42+
43+
MasterCoverage:
44+
stage: test
45+
script:
46+
- echo "SKIP master coverage check"
47+
tags:
48+
- docker
49+
<<: *only-default
50+
51+
Coverage-Check:
52+
stage: coverage-check
53+
script:
54+
- echo "SKIP coverage check"
55+
tags:
56+
- docker
57+
<<: *only-default

coverage.sh

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
lein cloverage --emma-xml

dev/resources/dev.edn

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{:duct.server.http/jetty
2-
{:port 3000}
2+
{:port 3031}
33

44
:duct.database/sql
55
{:connection-uri "jdbc:postgresql://localhost:5432/postgres?user=postgres&password=postgres"}}

dev/src/dev.clj

-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@
88
[duct.core.repl :as duct-repl :refer [auto-reset]]
99
[eftest.runner :as eftest]
1010
[integrant.core :as ig]
11-
[clj-postgresql.types]
1211
[integrant.repl :refer [clear halt go init prep reset]]
1312
[integrant.repl.state :refer [config system]]))
1413

project.clj

+23-19
Original file line numberDiff line numberDiff line change
@@ -7,30 +7,34 @@
77
[duct/handler.sql "0.4.0"]
88
[duct/module.ataraxy "0.3.0"]
99
[duct/module.logging "0.5.0"]
10-
[duct/module.sql "0.6.1"]
11-
[duct/module.web "0.7.3"]
10+
[com.fasterxml.jackson.core/jackson-databind "2.12.7.1"]
11+
[duct/module.sql "0.6.1" :exclusions [integrant medley]]
12+
[duct/module.web "0.7.3" :exclusions [integrant medley ring/ring-core com.fasterxml.jackson.core/jackson-databind]]
1213
[clj-http "3.12.3"]
1314
[com.health-samurai/jute "0.2.0-SNAPSHOT"]
14-
[org.postgresql/postgresql "42.2.19"]
15-
[clj-postgresql "0.7.0"]]
16-
:plugins [[duct/lein-duct "0.12.3"]]
15+
[org.postgresql/postgresql "42.2.19"]]
16+
:plugins [[duct/lein-duct "0.12.3"]
17+
[lein-cloverage "1.2.2"]
18+
[com.github.clj-kondo/lein-clj-kondo "0.2.5"]]
19+
:aliases {"clj-kondo-deps" ["with-profile" "+test" "clj-kondo" "--copy-configs" "--dependencies" "--parallel" "--lint" "$classpath"]
20+
"clj-kondo-lint" ["do" ["clj-kondo-deps"] ["with-profile" "+test" "clj-kondo"]]}
1721
:main ^:skip-aot etlp-mapper.main
1822
:resource-paths ["resources" "target/resources"]
1923
:prep-tasks ["javac" "compile" ["run" ":duct/compiler"]]
2024
:middleware [lein-duct.plugin/middleware]
2125
:profiles
22-
{:dev [:project/dev :profiles/dev]
23-
:prod [:project/prod :profiles/prod]
24-
:repl {:prep-tasks ^:replace ["javac" "compile"]
25-
:repl-options {:init-ns user}}
26-
:uberjar {:aot :all}
26+
{:dev [:project/dev :profiles/dev]
27+
:prod [:project/prod :profiles/prod]
28+
:repl {:prep-tasks ^:replace ["javac" "compile"]
29+
:repl-options {:init-ns user}}
30+
:uberjar {:aot :all}
2731
:profiles/prod {}
28-
:project/prod {:source-paths ["prod/src"]
29-
:resource-paths ["prod/resources"]}
30-
:profiles/dev {}
31-
:project/dev {:source-paths ["dev/src"]
32-
:resource-paths ["dev/resources"]
33-
:dependencies [[integrant/repl "0.3.2"]
34-
[hawk "0.2.11"]
35-
[eftest "0.5.9"]
36-
[kerodon "0.9.1"]]}})
32+
:project/prod {:source-paths ["prod/src"]
33+
:resource-paths ["prod/resources"]}
34+
:profiles/dev {}
35+
:project/dev {:source-paths ["dev/src"]
36+
:resource-paths ["dev/resources"]
37+
:dependencies [[integrant/repl "0.3.2" :exclusions [integrant]]
38+
[hawk "0.2.11"]
39+
[eftest "0.5.9"]
40+
[kerodon "0.9.1"]]}})

resources/etlp_mapper/config.edn

+140
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,143 @@
1+
{:duct.profile/base
2+
{:duct.core/project-ns etlp-mapper
3+
4+
:duct.router/ataraxy
5+
{:routes {[:get "/"] [:etlp-mapper.handler/index]
6+
[:get "/mappings"] [:etlp-mapper.handler.mappings/list]
7+
[:post "/mappings" {{:keys [title content]} :body-params}]
8+
[:etlp-mapper.handler.mappings/create title content]
9+
10+
[:put "/mappings/" id {{:keys [content]} :body-params}]
11+
[:etlp-mapper.handler.mappings/update ^int id content]
12+
13+
[:post "/mappings/" id "/apply" {{:keys [data]} :body-params}]
14+
[:etlp-mapper.handler/apply-mappings ^int id data]
15+
16+
[:post "/mappings/test"]
17+
[:etlp-mapper.handler/mappings]
18+
19+
[:post "/mappings/opai"]
20+
[:etlp-mapper.handler/openai]
21+
22+
23+
[:get "/mappings/" id] [:etlp-mapper.handler.mappings/find ^int id]
24+
[:get "/mappings/" id "/_history"] [:etlp-mapper.handler.mappings/history ^int id]
25+
[:get "/mappings/" id "/_history/" version] [:etlp-mapper.handler.mappings/traverse-history ^int id version]
26+
[:delete "/mappings/" id] [:etlp-mapper.handler.mappings/destroy ^int id]}}
27+
28+
:duct.migrator/ragtime
29+
{:migrations [#ig/ref :etlp-mapper.migration/create-mappings
30+
#ig/ref :etlp-mapper.migration/create-mappings-history
31+
#ig/ref :etlp-mapper.migration/insert-mapping-history
32+
#ig/ref :etlp-mapper.migration/insert_mapping_history_trigger
33+
#ig/ref :etlp-mapper.migration/updated-at-trigger
34+
#ig/ref :etlp-mapper.migration/updated-mapping-trigger]}
35+
36+
; Mapper specific tables
37+
[:duct.migrator.ragtime/sql :etlp-mapper.migration/create-mappings]
38+
{:up ["CREATE TABLE mappings (id SERIAL PRIMARY KEY, title TEXT NOT NULL, content JSONB, created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP)"]
39+
:down ["DROP TABLE mappings"]}
40+
[:duct.migrator.ragtime/sql :etlp-mapper.migration/create-mappings-history]
41+
{:up ["CREATE TABLE mappings_history (id SERIAL PRIMARY KEY, original_id INT, txnid TEXT, title TEXT NOT NULL, content JSONB, created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP WITH TIME ZONE)"]
42+
:down ["DROP TABLE mappings_history"]}
43+
44+
;DB procedure and triggers for cross cutting concerns
45+
[:duct.migrator.ragtime/sql :etlp-mapper.migration/insert-mapping-history]
46+
{:up ["CREATE OR REPLACE FUNCTION insert_mapping_history()
47+
RETURNS TRIGGER AS $$
48+
DECLARE
49+
txnid TEXT;
50+
BEGIN
51+
txnid := txid_current();
52+
INSERT INTO mappings_history (
53+
title,
54+
content,
55+
created_at,
56+
updated_at,
57+
original_id,
58+
txnid
59+
)
60+
VALUES (
61+
OLD.title,
62+
OLD.content,
63+
OLD.created_at,
64+
OLD.updated_at,
65+
OLD.id,
66+
txnid
67+
);
68+
RETURN NEW;
69+
END;
70+
$$ LANGUAGE plpgsql;"]
71+
:down ["DROP FUNCTION insert_mapping_history"]}
72+
[:duct.migrator.ragtime/sql :etlp-mapper.migration/insert_mapping_history_trigger]
73+
{:up ["CREATE TRIGGER insert_mapping_history_trigger BEFORE UPDATE ON mappings FOR EACH ROW EXECUTE FUNCTION insert_mapping_history();"]
74+
:down ["DROP TRIGGER insert_mapping_history_trigger"]}
75+
[:duct.migrator.ragtime/sql :etlp-mapper.migration/updated-at-trigger]
76+
{:up ["CREATE OR REPLACE FUNCTION update_changetimestamp_column() RETURNS TRIGGER AS $$ BEGIN NEW.updated_at = now(); RETURN NEW; END; $$ language 'plpgsql';"]
77+
:down ["DROP FUNCTION update_changetimestamp_column"]}
78+
[:duct.migrator.ragtime/sql :etlp-mapper.migration/updated-mapping-trigger]
79+
{:up ["CREATE TRIGGER update_mapping_changetimestamp BEFORE UPDATE ON mappings FOR EACH ROW EXECUTE PROCEDURE update_changetimestamp_column();"]
80+
:down ["DROP TRIGGER update_mapping_changetimestamp"]}
81+
82+
;Query handlers for Rest Endpoints
83+
[:duct.handler.sql/query :etlp-mapper.handler.mappings/list]
84+
{:sql ["SELECT * FROM mappings"]
85+
:hrefs {:href "/mappings/{id}"}}
86+
87+
:etlp-mapper.handler/index
88+
{:db #ig/ref :duct.database/sql}
89+
90+
:etlp-mapper.handler/mappings
91+
{:db #ig/ref :duct.database/sql}
92+
93+
:etlp-mapper.handler/openai
94+
{:db #ig/ref :duct.database/sql}
95+
96+
:etlp-mapper.handler/apply-mappings
97+
{:db #ig/ref :duct.database/sql
98+
:request {[_ id data] :ataraxy/result}}
99+
100+
[:duct.handler.sql/insert :etlp-mapper.handler.mappings/create]
101+
{:request {[_ title content] :ataraxy/result}
102+
:sql ["INSERT INTO mappings (title, content) VALUES (?, ?)" title content]
103+
:location "mappings/{id}"
104+
:hrefs {:href "/mappings/{id}"}}
105+
106+
[:duct.handler.sql/query-one :etlp-mapper.handler.mappings/find]
107+
{:request {[_ id] :ataraxy/result}
108+
:sql ["SELECT * FROM mappings WHERE id = ?" id]
109+
:hrefs {:href "/mappings/{id}"}}
110+
111+
[:duct.handler.sql/execute :etlp-mapper.handler.mappings/destroy]
112+
{:request {[_ id] :ataraxy/result}
113+
:sql ["DELETE FROM mappings WHERE id = ?" id]}
114+
115+
[:duct.handler.sql/execute :etlp-mapper.handler.mappings/update]
116+
{:request {[_ id content] :ataraxy/result}
117+
:sql ["UPDATE mappings SET content = ? WHERE id = ?" content id]}
118+
119+
[:duct.handler.sql/query :etlp-mapper.handler.mappings/history]
120+
{:request {[_ id] :ataraxy/result}
121+
:sql ["SELECT mh.title, mh.content, mh.created_at, mh.updated_at, mh.txnid FROM mappings m JOIN mappings_history mh ON m.id = mh.original_id WHERE m.id = ?" id]
122+
:hrefs {:href "/mappings/{id}/_history/{txnid}"}}
123+
124+
[:duct.handler.sql/query-one :etlp-mapper.handler.mappings/traverse-history]
125+
{:request {[_ id version] :ataraxy/result}
126+
:sql ["SELECT mh.title, mh.content, mh.created_at, mh.updated_at, mh.txnid FROM mappings m JOIN mappings_history mh ON m.id = mh.original_id WHERE m.id = ? AND mh.txnid = ?" id version]}}
127+
128+
:duct.profile/dev #duct/include "dev"
129+
; :duct.profile/local #duct/include "local"
130+
:duct.profile/prod #duct/include "prod.edn"
131+
132+
:duct.module/logging {}
133+
:duct.module.web/api
134+
{}
135+
136+
137+
138+
:duct.module/sql
139+
{}}
140+
1141
{:duct.profile/base
2142
{:duct.core/project-ns etlp-mapper
3143

src/etlp_mapper/main.clj

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
(ns etlp-mapper.main
22
(:gen-class)
33
(:require [duct.core :as duct]
4-
[clj-postgresql.types]))
4+
[etlp-mapper.pgtypes]))
55

66
(duct/load-hierarchy)
77

0 commit comments

Comments
 (0)