Skip to content

Commit

Permalink
added code evaluation on the config file
Browse files Browse the repository at this point in the history
  • Loading branch information
@lsevero
lsevero committed May 27, 2021
1 parent 147a432 commit 266cb3e
Show file tree
Hide file tree
Showing 6 changed files with 56 additions and 19 deletions.
32 changes: 24 additions & 8 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,16 @@ Explained config.edn:
"value.serializer" "org.apache.kafka.common.serialization.StringSerializer"
}}
;Here we control some option of the http server
:http {:min-threads 16
;in this config file we will evaluate all lists (note the parethesis syntax) as code
;
;
; !!!!!!!!! WARNING !!!!!!!!!!!
;
;
;Never read a config file in franz from an untrusted sources!!!!!
:http {:min-threads (+ 8 8)
:max-threads 100
:max-idle-time 1800000
:max-idle-time (* 30 60 1000)
:request-header-size 8192
;We are using the juxt/aero config library to read the config files
;we can pass environment variables inside the config file, if no venv has passed will default to the second argument of the #or clause
Expand Down Expand Up @@ -83,7 +90,9 @@ Explained config.edn:
[:map
[:x [:and int? [:> 6]]]
[:y number?]]
[:fn (fn [{:keys [x y]}] (> x y))]]
;When using functions to validate always provide a :error/message as well
;or else the spec will fail with a "unknown error" message.
[:fn {:error/message "x should be greater than y"} (fn [{:keys [x y]}] (> x y))]]
:query [:map
[:id string?]]}
}
Expand All @@ -95,8 +104,8 @@ Explained config.edn:
:flush? false; flush messages after being sent by the producer, defaults to true
:summary "testing get"
}}
"/fire-and-forget/" {:post {:send-topic "poc3"
:listen-topic "poc3"
"/fire-and-forget/" {:post {:send-topic "maaaaaaaaaaaano"
:listen-topic "maaaaaaaaaaaano"
:timeout 2000
:poll-duration 100;milliseconds
:serialization {:type :json}
Expand All @@ -108,7 +117,7 @@ Explained config.edn:
"/avro" {:post {:send-topic "avro"
;We can define kafka configs per route as well
;these maps will be merged against the kafka configs above, per-route configs prevail
:consumer {"group.id" "avro"
:consumer {"group.id" (str "prefix-" (java.util.UUID/randomUUID))
"auto.offset.reset" "latest"
"key.deserializer" "org.apache.kafka.common.serialization.ByteArrayDeserializer"
"value.deserializer" "org.apache.kafka.common.serialization.ByteArrayDeserializer"
Expand Down Expand Up @@ -156,8 +165,7 @@ Explained config.edn:
:consumer-spec "./example-config/avro.json"
}
}}
}
}
}}

```

Expand All @@ -168,6 +176,14 @@ Besides the logs inside Franz you can extract the logs inside the kafka and http
Check the `example-config/log4j2.xml` file for a example.
The logging config is optional, although is recommended to use.

## Warnings about safety

It is now possible to inject code as data on the config file to improve the config file flexibility and power.
That being said, **you should never read config files from untrusted sources!!!**
If you are too uncorfortable with code evaluation in the config file, you can disable it setting `:code-eval` to false inside `:defaults` section.

With great power comes great responsibility.

## Building

You'll need [leiningen](https://leiningen.org/) installed, then:
Expand Down
20 changes: 14 additions & 6 deletions example-config/config.edn
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,16 @@
"value.serializer" "org.apache.kafka.common.serialization.StringSerializer"
}}
;Here we control some option of the http server
:http {:min-threads 16
;in this config file we will evaluate all lists (note the parethesis syntax) as code
;
;
; !!!!!!!!! WARNING !!!!!!!!!!!
;
;
;Never read a config file in franz from an untrusted sources!!!!!
:http {:min-threads (+ 8 8)
:max-threads 100
:max-idle-time 1800000
:max-idle-time (* 30 60 1000)
:request-header-size 8192
;We are using the juxt/aero config library to read the config files
;we can pass environment variables inside the config file, if no venv has passed will default to the second argument of the #or clause
Expand Down Expand Up @@ -56,7 +63,9 @@
[:map
[:x [:and int? [:> 6]]]
[:y number?]]
[:fn (fn [{:keys [x y]}] (> x y))]]
;When using functions to validate always provide a :error/message as well
;or else the spec will fail with a "unknown error" message.
[:fn {:error/message "x should be greater than y"} (fn [{:keys [x y]}] (> x y))]]
:query [:map
[:id string?]]}
}
Expand All @@ -81,7 +90,7 @@
"/avro" {:post {:send-topic "avro"
;We can define kafka configs per route as well
;these maps will be merged against the kafka configs above, per-route configs prevail
:consumer {"group.id" "avro"
:consumer {"group.id" (str "prefix-" (java.util.UUID/randomUUID))
"auto.offset.reset" "latest"
"key.deserializer" "org.apache.kafka.common.serialization.ByteArrayDeserializer"
"value.deserializer" "org.apache.kafka.common.serialization.ByteArrayDeserializer"
Expand Down Expand Up @@ -129,5 +138,4 @@
:consumer-spec "./example-config/avro.json"
}
}}
}
}
}}
2 changes: 1 addition & 1 deletion example-config/log4j2.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
<Logger name="org.apache.kafka.clients.admin.AdminClientConfig" level="error" additivity="false">
<AppenderRef ref="ConsoleAppender"/>
</Logger>
<Logger name="config.core" level="error" additivity="false">
<Logger name="franz" level="info" additivity="false">
<AppenderRef ref="ConsoleAppender"/>
</Logger>
<Root level="debug">
Expand Down
2 changes: 1 addition & 1 deletion project.clj
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
(defproject franz "1.4.4"
(defproject franz "1.5.0-SNAPSHOT"
:description "A http to kafka gateway"
:url "https://github.com/lsevero/franz"
:license {:name "EPL-2.0 OR GPL-2.0-or-later WITH Classpath-exception-2.0"
Expand Down
18 changes: 15 additions & 3 deletions src/franz/config.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,17 +6,29 @@
[malli.core :as malli]
[malli.error :as me]
[franz.spec :refer [config-spec]]
[clojure.walk :refer [prewalk postwalk postwalk-demo]]
))

(defn- eval-lists
"Eval lists inside the config file, except lists that start with a 'fn"
[l]
(if (list? l)
(eval l)
l))

(defn get-config!
[]
(let [config-path (System/getProperty "config")]
(if (nil? config-path)
(do (log/error "No config file was given.")
(System/exit 1))
(try
(let [config (read-config config-path)]
(log/info "Reading the config file...")
(log/info "Reading the config file...")
(let [config (read-config config-path)
config (if (-> config :defaults :code-eval false?)
config
(prewalk eval-lists config))]
(log/info "Validating the config file...")
(if-not (malli/validate config-spec config)
(do
(log/error (apply str (repeat 120 "=")))
Expand All @@ -34,7 +46,7 @@
(log/error (str "Add this property to java and restart franz: -Dclojure.core.async.pool-size=" (-> config :routes count inc))))
config)))
(catch Exception e
(do (log/error e "Error validating the config file")
(do (log/error e "Error while reading the config file")
(System/exit 1)))))))

(mount/defstate config
Expand Down
1 change: 1 addition & 0 deletions src/franz/spec.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@
[:partitions {:optional true} pos-int?]
[:replication {:optional true} pos-int?]
[:flush? {:optional true} boolean?]
[:code-eval {:optional true} boolean?]
]]
[:routes
[:and [:map-of :string [:and
Expand Down

0 comments on commit 266cb3e

Please sign in to comment.