From a263115529b1374c83226096255698b363269911 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Fri, 2 Aug 2013 17:21:33 +0200 Subject: [PATCH 1/3] Update README.md --- README.md | 48 +++++++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index cc4c3fb..e5ee777 100644 --- a/README.md +++ b/README.md @@ -1,24 +1,26 @@ -This project is some experimentation with CRDT's in Erlang, please be aware that it's an **experiment** meaning not suitable for production use, not optimised for either speed or space and probably broken in some places. - -The code is written to look nice, be understandable not to make the most out of every operation. - -Consider yourself warned, now go have some fun with it! - -## Naming conventions - -The first letter of the module name indicates what kind of CRDT the mudle implements: - -* m* - implements CmRDT's (message based) -* v* - implements CvRDT's (state based) - - -## Resources -* [Call me maybe: Riak](http://aphyr.com/posts/285-call-me-maybe-riak) -* [Marc Shapiro](http://pagesperso-systeme.lip6.fr/Marc.Shapiro/pubs.html#SEC-SSS) - * [CRDTs: Consistency without concurrency control](http://pagesperso-systeme.lip6.fr/Marc.Shapiro/papers/RR-6956.pdf) - * [An Optimized Conflict-free Replicated Set](http://pagesperso-systeme.lip6.fr/Marc.Shapiro/papers/RR-8083.pdf) - * [Conflict-free Replicated Data Types](http://hal.inria.fr/docs/00/61/73/41/PDF/RR-7687.pdf) +This project is some experimentation with CRDT's in Erlang, please be aware that it's an **experiment** meaning not suitable for production use, not optimised for either speed or space and probably broken in some places. + +The code is written to look nice, be understandable not to make the most out of every operation. + +Consider yourself warned, now go have some fun with it! + +## Naming conventions + +The first letter of the module name indicates what kind of CRDT the mudle implements: + +* m* - implements CmRDT's (message based) +* v* - implements CvRDT's (state based) + + +## Resources + +* [Erlounge Presentation](http://www.slideshare.net/Licenser/crd-ts-for-engineers) +* [Call me maybe: Riak](http://aphyr.com/posts/285-call-me-maybe-riak) +* [Marc Shapiro](http://pagesperso-systeme.lip6.fr/Marc.Shapiro/pubs.html#SEC-SSS) + * [CRDTs: Consistency without concurrency control](http://pagesperso-systeme.lip6.fr/Marc.Shapiro/papers/RR-6956.pdf) + * [An Optimized Conflict-free Replicated Set](http://pagesperso-systeme.lip6.fr/Marc.Shapiro/papers/RR-8083.pdf) + * [Conflict-free Replicated Data Types](http://hal.inria.fr/docs/00/61/73/41/PDF/RR-7687.pdf) * [A comprehensive study of -Convergent and Commutative Replicated Data Types](http://pagesperso-systeme.lip6.fr/Marc.Shapiro/papers/Comprehensive-CRDTs-RR7506-2011-01.pdf) - * [Designing a commutative replicated data type](http://pagesperso-systeme.lip6.fr/Marc.Shapiro/papers/Commutative-Replicated-Data-Type-RR-6320_2007-10.pdf) - * [A commutative replicated data type for cooperative editing](http://pagesperso-systeme.lip6.fr/Marc.Shapiro/papers/icdcs09-treedoc.pdf) \ No newline at end of file +Convergent and Commutative Replicated Data Types](http://pagesperso-systeme.lip6.fr/Marc.Shapiro/papers/Comprehensive-CRDTs-RR7506-2011-01.pdf) + * [Designing a commutative replicated data type](http://pagesperso-systeme.lip6.fr/Marc.Shapiro/papers/Commutative-Replicated-Data-Type-RR-6320_2007-10.pdf) + * [A commutative replicated data type for cooperative editing](http://pagesperso-systeme.lip6.fr/Marc.Shapiro/papers/icdcs09-treedoc.pdf) From 5690b6c23f00f08824b0c1d051f93e3388456445 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 4 Aug 2013 18:39:17 +0200 Subject: [PATCH 2/3] add store, remove to vorddict --- src/vordict.erl | 115 ++++++++++++++++++++++-------------------------- 1 file changed, 53 insertions(+), 62 deletions(-) diff --git a/src/vordict.erl b/src/vordict.erl index e82e7ef..e971a25 100644 --- a/src/vordict.erl +++ b/src/vordict.erl @@ -5,14 +5,14 @@ -include_lib("eunit/include/eunit.hrl"). -endif. --export([new/0, add/2, add/3, remove/2, merge/2, value/1, from_list/1, gc/1]). +-export([new/0, store/3, remove/2, merge/2, value/1]). %from_list/1, gc/1 ]). --record(vorset, {adds :: vgset:vgset(), +-record(vordict, {adds :: vgset:vgset(), removes :: vgset:vgset()}). --opaque vorset() :: #vorset{}. +-opaque vordict() :: #vordict{}. --export_type([vorset/0]). +-export_type([vordict/0]). %%%=================================================================== %%% Implementation @@ -23,10 +23,10 @@ %% Creates a new empty OR Set. %% @end %%-------------------------------------------------------------------- --spec new() -> vorset(). +-spec new() -> vordict(). new() -> - #vorset{adds = vgset:new(), - removes = vgset:new()}. + #vordict{adds = vgset:new(), + removes = vgset:new()}. %%-------------------------------------------------------------------- %% @doc @@ -34,20 +34,11 @@ new() -> %% list. %% @end %%-------------------------------------------------------------------- --spec from_list(list()) -> vorset(). -from_list(L) -> - ID = ecrdt:id(), - #vorset{adds = vgset:from_list([ {E, ID} || E <- L]), - removes = vgset:new()}. - -%%-------------------------------------------------------------------- -%% @doc -%% Adds an element to the OR set with a given master ID. -%% @end -%%-------------------------------------------------------------------- --spec add(ID::term(), Element::term(), ORSet::vorset()) -> ORSet1::vorset(). -add(ID, Element, ORSet = #vorset{adds = Adds}) -> - ORSet#vorset{adds = vgset:add({Element, ID}, Adds)}. +% -spec from_list(list()) -> vorset(). +% from_list(L) -> +% ID = ecrdt:id(), +% #vorset{adds = vgset:from_list([ {E, ID} || E <- L]), +% removes = vgset:new()}. %%-------------------------------------------------------------------- %% @doc @@ -55,9 +46,9 @@ add(ID, Element, ORSet = #vorset{adds = Adds}) -> %% provided by ecrdt:id(). %% @end %%-------------------------------------------------------------------- --spec add(Element::term(), ORSet::vorset()) -> ORSet1::vorset(). -add(Element, ORSet = #vorset{adds = Adds}) -> - ORSet#vorset{adds = vgset:add({Element, ecrdt:id()}, Adds)}. +-spec store(Key::term(), Value::term(), ORDict::vordict()) -> ORDict1::vordict(). +store(Key, _Value, ORDict = #vordict{adds = Adds}) -> + ORDict#vordict{adds = vgset:add({Key, ecrdt:id()}, Adds)}. %%-------------------------------------------------------------------- %% @doc @@ -65,25 +56,25 @@ add(Element, ORSet = #vorset{adds = Adds}) -> %% putting them in the list of removed items. %% @end %%-------------------------------------------------------------------- --spec remove(Element::term(), ORSet::vorset()) -> ORSet1::vorset(). -remove(Element, ORSet = #vorset{removes = Removes}) -> - CurrentExisting = [Elem || Elem = {E1, _} <- raw_value(ORSet), E1 =:= Element], +-spec remove(Key::term(), ORDict::vordict()) -> ORDict1::vordict(). +remove(Key, ORDict = #vordict{removes = Removes}) -> + CurrentExisting = [Elem || Elem = {E1, _} <- raw_value(ORDict), E1 =:= Key], Removes1 = lists:foldl(fun(R, Rs) -> vgset:add(R, Rs) end, Removes, CurrentExisting), - ORSet#vorset{removes = Removes1}. + ORDict#vordict{removes = Removes1}. %%-------------------------------------------------------------------- %% @doc %% Merges two OR Sets by taking the union of adds and removes. %% @end %%-------------------------------------------------------------------- --spec merge(ORSet0::vorset(), ORSet1::vorset()) -> ORSetM::vorset(). -merge(#vorset{adds = Adds0, +-spec merge(ORDict0::vordict(), ORDict1::vordict()) -> ORDictM::vordict(). +merge(#vordict{adds = Adds0, removes = Removes0}, - #vorset{adds = Adds1, + #vordict{adds = Adds1, removes = Removes1}) -> - #vorset{adds = vgset:merge(Adds0, Adds1), + #vordict{adds = vgset:merge(Adds0, Adds1), removes = vgset:merge(Removes0, Removes1)}. %%-------------------------------------------------------------------- @@ -92,9 +83,9 @@ merge(#vorset{adds = Adds0, %% substract of adds and removes then eleminating the ID field. %% @end %%-------------------------------------------------------------------- --spec value(ORSet::vorset()) -> [Element::term()]. -value(ORSet) -> - ordsets:from_list([E || {E, _} <- raw_value(ORSet)]). +-spec value(ORDict::vordict()) -> [Element::term()]. +value(ORDict) -> + ordsets:from_list([E || {E, _} <- raw_value(ORDict)]). %%-------------------------------------------------------------------- %% @doc @@ -105,17 +96,17 @@ value(ORSet) -> %% lead to unexpected results! %% @end %%-------------------------------------------------------------------- --spec gc(ORSet::vorset()) -> ORSetGCed::vorset(). -gc(ORSet) -> - #vorset{adds = vgset:from_list(raw_value(ORSet)), - removes = vgset:new()}. +% -spec gc(ORSet::vorset()) -> ORSetGCed::vorset(). +% gc(ORSet) -> +% #vorset{adds = vgset:from_list(raw_value(ORSet)), +% removes = vgset:new()}. %%%=================================================================== %%% Internal functions %%%=================================================================== --spec raw_value(ORSet::vorset()) -> ordsets:ordset(). -raw_value(#vorset{adds = Adds, +-spec raw_value(ORDict::vordict()) -> orddict:orddict(). +raw_value(#vordict{adds = Adds, removes = Removes}) -> ordsets:subtract(vgset:value(Adds), vgset:value(Removes)). @@ -125,33 +116,33 @@ raw_value(#vorset{adds = Adds, -ifdef(TEST). -op(a, add, E, C1, C2, Check) -> - ID = ecrdt:id(), - {add(ID, E, C1), C2, add(ID, E, Check)}; -op(b, add, E, C1, C2, Check) -> - ID = ecrdt:id(), - {C1, add(ID, E, C2), add(ID, E, Check)}; -op(ab, add, E, C1, C2, Check) -> - ID = ecrdt:id(), - {add(ID, E, C1), add(ID, E, C2), add(ID, E, Check)}; -op(a, remove, E, C1, C2, Check) -> - {remove(E, C1), C2, remove(E, Check)}; -op(b, remove, E, C1, C2, Check) -> - {C1, remove(E, C2), remove(E, Check)}; -op(ab, remove, E, C1, C2, Check) -> - {remove(E, C1), remove(E, C2), remove(E, Check)}. %; - -%% Applies the list of opperaitons to three empty sets. +op(a, store, K, V, C1, C2, Check) -> + _ID = ecrdt:id(), + {store(K, V, C1), C2, store(K, V, Check)}; +op(b, store, K, V, C1, C2, Check) -> + _ID = ecrdt:id(), + {C1, store(K, V, C2), store(K, V, Check)}; +op(ab, store, K, V, C1, C2, Check) -> + _ID = ecrdt:id(), + {store(K, V, C1), store(K, V, C2), store(K, V, Check)}; +op(a, remove, K, _V, C1, C2, Check) -> + {remove(K, C1), C2, remove(K, Check)}; +op(b, remove, K, _V, C1, C2, Check) -> + {C1, remove(K, C2), remove(K, Check)}; +op(ab, remove, K, _V, C1, C2, Check) -> + {remove(K, C1), remove(K, C2), remove(K, Check)}. %; + +%% Applies the list of opperaitons to three empty dicts. apply_ops(Ops) -> - lists:foldl(fun({T, O, E}, {A, B, C}) -> - op(T, O, E, A, B, C) + lists:foldl(fun({T, O, K, V}, {A, B, C}) -> + op(T, O, K, V, A, B, C) end, {new(), new(), new()}, Ops). %% A list of opperations and targets. targets() -> - list({oneof([a, b, ab]), oneof([add, remove]), pos_integer()}). + list({oneof([a, b, ab]), oneof([store, remove]), pos_integer(), pos_integer()}). -prop_vorset() -> +prop_vordict() -> ?FORALL(Ts, targets(), begin {A, B, C} = apply_ops(Ts), From 6ead14a54773f088e1dc3e5b2b06b223c09bb309 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 4 Aug 2013 21:27:19 +0200 Subject: [PATCH 3/3] implement very basic vORdict --- src/vordict.erl | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/src/vordict.erl b/src/vordict.erl index e971a25..b00a4bc 100644 --- a/src/vordict.erl +++ b/src/vordict.erl @@ -7,7 +7,7 @@ -export([new/0, store/3, remove/2, merge/2, value/1]). %from_list/1, gc/1 ]). --record(vordict, {adds :: vgset:vgset(), +-record(vordict, {stores :: orddict:orddict(), removes :: vgset:vgset()}). -opaque vordict() :: #vordict{}. @@ -25,7 +25,7 @@ %%-------------------------------------------------------------------- -spec new() -> vordict(). new() -> - #vordict{adds = vgset:new(), + #vordict{stores = orddict:new(), removes = vgset:new()}. %%-------------------------------------------------------------------- @@ -47,8 +47,10 @@ new() -> %% @end %%-------------------------------------------------------------------- -spec store(Key::term(), Value::term(), ORDict::vordict()) -> ORDict1::vordict(). -store(Key, _Value, ORDict = #vordict{adds = Adds}) -> - ORDict#vordict{adds = vgset:add({Key, ecrdt:id()}, Adds)}. +store(Key, Value, ORDict = #vordict{stores = Stores}) -> + ORDict#vordict{stores = orddict:update(Key, fun(Value1) -> + ecrdt:merge(Value1, vlwwregister:new(Value)) + end, vlwwregister:new(Value), Stores)}. %%-------------------------------------------------------------------- %% @doc @@ -58,11 +60,7 @@ store(Key, _Value, ORDict = #vordict{adds = Adds}) -> %%-------------------------------------------------------------------- -spec remove(Key::term(), ORDict::vordict()) -> ORDict1::vordict(). remove(Key, ORDict = #vordict{removes = Removes}) -> - CurrentExisting = [Elem || Elem = {E1, _} <- raw_value(ORDict), E1 =:= Key], - Removes1 = lists:foldl(fun(R, Rs) -> - vgset:add(R, Rs) - end, Removes, CurrentExisting), - ORDict#vordict{removes = Removes1}. + ORDict#vordict{removes = vgset:add(Key, Removes)}. %%-------------------------------------------------------------------- %% @doc @@ -70,11 +68,13 @@ remove(Key, ORDict = #vordict{removes = Removes}) -> %% @end %%-------------------------------------------------------------------- -spec merge(ORDict0::vordict(), ORDict1::vordict()) -> ORDictM::vordict(). -merge(#vordict{adds = Adds0, +merge(#vordict{stores = Stores0, removes = Removes0}, - #vordict{adds = Adds1, + #vordict{stores = Stores1, removes = Removes1}) -> - #vordict{adds = vgset:merge(Adds0, Adds1), + #vordict{stores = orddict:merge(fun(_Key, Value1, Value2) -> + ecrdt:merge(Value1, Value2) + end, Stores0, Stores1), removes = vgset:merge(Removes0, Removes1)}. %%-------------------------------------------------------------------- @@ -85,7 +85,7 @@ merge(#vordict{adds = Adds0, %%-------------------------------------------------------------------- -spec value(ORDict::vordict()) -> [Element::term()]. value(ORDict) -> - ordsets:from_list([E || {E, _} <- raw_value(ORDict)]). + ordsets:from_list([ecrdt:value(E) || E <- raw_value(ORDict)]). %%-------------------------------------------------------------------- %% @doc @@ -105,10 +105,13 @@ value(ORDict) -> %%% Internal functions %%%=================================================================== --spec raw_value(ORDict::vordict()) -> orddict:orddict(). -raw_value(#vordict{adds = Adds, +-spec raw_value(ORDict::vordict()) -> [Element::vlwwregister:vlwwregister()]. +raw_value(#vordict{stores = Stores, removes = Removes}) -> - ordsets:subtract(vgset:value(Adds), vgset:value(Removes)). + RemovesRaw = vgset:value(Removes), + orddict:filter(fun(K, _V) -> + not lists:member(K, RemovesRaw) + end, Stores). %%%=================================================================== %%% Tests @@ -140,7 +143,7 @@ apply_ops(Ops) -> %% A list of opperations and targets. targets() -> - list({oneof([a, b, ab]), oneof([store, remove]), pos_integer(), pos_integer()}). + list({oneof([a, b, ab]), oneof([store, remove]), atom(), pos_integer()}). prop_vordict() -> ?FORALL(Ts, targets(),