From bb1ba1d131975576de8b1efdf0e5c23d5298e024 Mon Sep 17 00:00:00 2001 From: Didier A Date: Mon, 31 Jul 2017 01:50:11 -0700 Subject: [PATCH 1/3] Improved eagerization by directly handling all Clojure and Java container types. The eagerization is now 4 times faster, and can handle eagerizing Clojure deftype and Java Arrays. It can also properly eagerize container's who's .toString or print-method have been modified in a way which does not visit the elements. --- src/special/core.cljc | 5 ++- src/special/eagerize.cljc | 35 ++++++++++++++++ test/special/eagerize_test.clj | 73 ++++++++++++++++++++++++++++++++++ 3 files changed, 111 insertions(+), 2 deletions(-) create mode 100644 src/special/eagerize.cljc create mode 100644 test/special/eagerize_test.clj diff --git a/src/special/core.cljc b/src/special/core.cljc index 9c452eb..4170eec 100644 --- a/src/special/core.cljc +++ b/src/special/core.cljc @@ -1,4 +1,5 @@ -(ns special.core) +(ns special.core + (:require [special.eagerize :as eag])) (defonce ^:dynamic *-special-condition-handlers-* {}) @@ -31,7 +32,7 @@ [f] (fn [& args] (let [res (apply f args) - _ (pr-str res)] + _ (eag/eagerize res)] res))) (defn manage diff --git a/src/special/eagerize.cljc b/src/special/eagerize.cljc new file mode 100644 index 0000000..9faaace --- /dev/null +++ b/src/special/eagerize.cljc @@ -0,0 +1,35 @@ +(ns special.eagerize) + +#?(:clj (defn eagerize + "Recursively iterate over Clojure data structures and Java data structures, + evaluating each element, so as to force eagerization of any lazy construct. + Supports: + - Clojure IPersistentList, IMapEntry, ISeq, IRecord, IPersistentCollection + and IType. + - Java Iterable, Map and Arrays." + [form] + (cond + (nil? form) form + (list? form) (apply list (map eagerize form)) + (instance? clojure.lang.IMapEntry form) (vec (map eagerize form)) + (seq? form) (doall (map eagerize form)) + (instance? clojure.lang.IRecord form) + (reduce (fn [r x] (conj r (eagerize x))) form form) + (coll? form) (into (empty form) (map eagerize form)) + (instance? java.lang.Iterable form) (doall (map eagerize form)) + (instance? java.util.Map form) (doall (map eagerize (.values form))) + (instance? clojure.lang.IType form) (doall + (map #(eagerize (.get % form)) + (.getFields (class form)))) + (.isArray (type form)) (doall (map eagerize form)) + :else form) + nil)) + +#?(:cljs (defn eagerize + "Relies on each form's implementation of .toString or print-method + to recursively visit all element of form, so as to eagerize form. + Will fail to eagerize if print-method or .toString is not implemented + in a way that visit all elements at all levels. + Pays the extra cost both in CPU and memory of printing." + [form] + (pr-str form))) diff --git a/test/special/eagerize_test.clj b/test/special/eagerize_test.clj new file mode 100644 index 0000000..c933c46 --- /dev/null +++ b/test/special/eagerize_test.clj @@ -0,0 +1,73 @@ +(ns special.eagerize-test + (:require [clojure.test :refer :all] + [special.eagerize :refer :all])) + +(defn- make-nested-lazy-list + "Returns a lazy-sequece of e. + Defaults to random-ints when called with no args." + ([] + (make-nested-lazy-list #(rand-int 42))) + ([e] + (repeatedly 10 (constantly e)))) + +(defrecord TestRecord [s]) +(deftype TestType [s]) + +(deftest eagerize-test + (testing "Can eagerize deep nested Clojure IPersistentList." + (is (realized? + (let [ls (list (make-nested-lazy-list))] + (eagerize ls) + (first ls))))) + + (testing "Can eagerize deep nested Clojure IMapEntry." + (is (realized? + (let [ls (first {:e (make-nested-lazy-list)})] + (eagerize ls) + (val ls))))) + + (testing "Can eagerize deep nested Clojure ISeq." + (is (realized? + (let [ls (make-nested-lazy-list (make-nested-lazy-list))] + (eagerize ls) + (first ls))))) + + (testing "Can eagerize deep nested Clojure IRecord." + (is (realized? + (let [ls (->TestRecord (make-nested-lazy-list))] + (eagerize ls) + (:s ls))))) + + (testing "Can eagerize deep nested Clojure IType." + (is (realized? + (let [ls (TestType. (make-nested-lazy-list))] + (eagerize ls) + (.-s ls))))) + + (testing "Can eagerize deep nested Java Iterable." + (is (realized? + (let [ls (doto (java.util.LinkedList.) + (.add (make-nested-lazy-list)))] + (eagerize ls) + (first ls))))) + + (testing "Can eagerize deep nested Java AbstractMap." + (is (realized? + (let [ls (doto (java.util.HashMap.) + (.put "a" (make-nested-lazy-list)))] + (eagerize ls) + (.get ls "a"))))) + + (testing "Can eagerize deep nested Java Stack." + (is (realized? + (let [ls (doto (java.util.Stack.) + (.push (make-nested-lazy-list)))] + (eagerize ls) + (.pop ls))))) + + (testing "Can eagerize deep nested Java Arrays." + (is (realized? + (let [ls (doto (make-array clojure.lang.ISeq 2) + (aset 0 (make-nested-lazy-list)))] + (eagerize ls) + (aget ls 0)))))) From 3334b16063a45066005af9d2bb97a0b0aaafded5 Mon Sep 17 00:00:00 2001 From: Didier A Date: Mon, 31 Jul 2017 02:21:59 -0700 Subject: [PATCH 2/3] Eagerization now works with Delay also. --- src/special/eagerize.cljc | 10 ++++++---- test/special/eagerize_test.clj | 6 ++++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/special/eagerize.cljc b/src/special/eagerize.cljc index 9faaace..a4c958d 100644 --- a/src/special/eagerize.cljc +++ b/src/special/eagerize.cljc @@ -1,11 +1,12 @@ (ns special.eagerize) #?(:clj (defn eagerize - "Recursively iterate over Clojure data structures and Java data structures, - evaluating each element, so as to force eagerization of any lazy construct. + "Recursively iterate over Clojure data structures and Java data + structures, evaluating each element, so as to force eagerization of + any lazy construct. Supports: - - Clojure IPersistentList, IMapEntry, ISeq, IRecord, IPersistentCollection - and IType. + - Clojure IPersistentList, IMapEntry, ISeq, IRecord, + IPersistentCollection, IType and Delay. - Java Iterable, Map and Arrays." [form] (cond @@ -16,6 +17,7 @@ (instance? clojure.lang.IRecord form) (reduce (fn [r x] (conj r (eagerize x))) form form) (coll? form) (into (empty form) (map eagerize form)) + (delay? form) (eagerize (deref form)) (instance? java.lang.Iterable form) (doall (map eagerize form)) (instance? java.util.Map form) (doall (map eagerize (.values form))) (instance? clojure.lang.IType form) (doall diff --git a/test/special/eagerize_test.clj b/test/special/eagerize_test.clj index c933c46..d8e36dc 100644 --- a/test/special/eagerize_test.clj +++ b/test/special/eagerize_test.clj @@ -44,6 +44,12 @@ (eagerize ls) (.-s ls))))) + (testing "Can eagerize deep nested Clojure Delay." + (is (realized? + (let [ls (delay (make-nested-lazy-list))] + (eagerize ls) + @ls)))) + (testing "Can eagerize deep nested Java Iterable." (is (realized? (let [ls (doto (java.util.LinkedList.) From 0dba876bd305a8c352dafaf3975d03ba2649d3a0 Mon Sep 17 00:00:00 2001 From: Didier A Date: Mon, 31 Jul 2017 11:35:31 -0700 Subject: [PATCH 3/3] Re-implemented eagerize/eagerize using protocols. This is 3x faster then cond, and is now open for extenssion, so if users have custom container types, they can simply extend them to support eagerize and it will work fine with special. For some reason, it doesn't work to extend object arrays for subtypes, so I had to implement it as part of the java.lang.Object case. --- src/special/eagerize.cljc | 74 +++++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 34 deletions(-) diff --git a/src/special/eagerize.cljc b/src/special/eagerize.cljc index a4c958d..9f9f9ee 100644 --- a/src/special/eagerize.cljc +++ b/src/special/eagerize.cljc @@ -1,37 +1,43 @@ (ns special.eagerize) -#?(:clj (defn eagerize - "Recursively iterate over Clojure data structures and Java data - structures, evaluating each element, so as to force eagerization of - any lazy construct. - Supports: - - Clojure IPersistentList, IMapEntry, ISeq, IRecord, - IPersistentCollection, IType and Delay. - - Java Iterable, Map and Arrays." - [form] - (cond - (nil? form) form - (list? form) (apply list (map eagerize form)) - (instance? clojure.lang.IMapEntry form) (vec (map eagerize form)) - (seq? form) (doall (map eagerize form)) - (instance? clojure.lang.IRecord form) - (reduce (fn [r x] (conj r (eagerize x))) form form) - (coll? form) (into (empty form) (map eagerize form)) - (delay? form) (eagerize (deref form)) - (instance? java.lang.Iterable form) (doall (map eagerize form)) - (instance? java.util.Map form) (doall (map eagerize (.values form))) - (instance? clojure.lang.IType form) (doall - (map #(eagerize (.get % form)) - (.getFields (class form)))) - (.isArray (type form)) (doall (map eagerize form)) - :else form) - nil)) +#?(:clj + (defonce array-object-type (delay (Class/forName "[Ljava.lang.Object;")))) -#?(:cljs (defn eagerize - "Relies on each form's implementation of .toString or print-method - to recursively visit all element of form, so as to eagerize form. - Will fail to eagerize if print-method or .toString is not implemented - in a way that visit all elements at all levels. - Pays the extra cost both in CPU and memory of printing." - [form] - (pr-str form))) +(defprotocol Eagerizable + "Container types that want to work safely with special should implement this + protocol in a way where all elements they contain, even deeply nested, will + be realized and made eager." + (eagerize [this])) + +#?(:clj + (extend-protocol Eagerizable + nil + (eagerize [this] this) + java.lang.Object + (eagerize [this] (if (instance? @array-object-type this) + (doall (map eagerize this)) + this)) + clojure.lang.IPersistentList + (eagerize [this] (apply list (map eagerize this))) + clojure.lang.IMapEntry + (eagerize [this] (vec (map eagerize this))) + clojure.lang.ISeq + (eagerize [this] (doall (map eagerize this))) + clojure.lang.IRecord + (eagerize [this] (reduce (fn [r x] (conj r (eagerize x))) this this)) + clojure.lang.IPersistentCollection + (eagerize [this] (into (empty this) (map eagerize this))) + clojure.lang.IType + (eagerize [this] (doall + (map #(eagerize (.get % this)) + (.getFields (class this))))) + clojure.lang.Delay + (eagerize [this] (eagerize (deref this))) + java.lang.Iterable + (eagerize [this] (doall (map eagerize this))) + java.util.Map + (eagerize [this] (doall (map eagerize (.values this)))))) + +#?(:cljs (extend-protocol Eagerizable + default + (eagerize [this] (pr-str this))))