Skip to content

Commit ee4131f

Browse files
committed
Fix lexigraphical ordering of base64 encoded values
__WHAT__ * use custom base64 codec that gurantiees ordering * use common test for testing * integrate CI and test coverage * clean-up code
1 parent 98c78dc commit ee4131f

16 files changed

+459
-443
lines changed

.travis.yml

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
language: erlang
2+
dist: trusty
3+
4+
script:
5+
- make
6+
- ./rebar3 coveralls send
7+
8+
otp_release:
9+
- 20.1
10+
- 20.0
11+
- 19.2
12+

Emakefile

Lines changed: 0 additions & 9 deletions
This file was deleted.

Makefile

Lines changed: 4 additions & 213 deletions
Original file line numberDiff line numberDiff line change
@@ -1,214 +1,5 @@
1-
## @author Dmitry Kolesnikov, <[email protected]>
2-
## @copyright (c) 2012 - 2014 Dmitry Kolesnikov. All Rights Reserved
3-
##
4-
## @description
5-
## Makefile to build and release Erlang applications using standard development tools
6-
##
7-
## @version 0.11.5
8-
9-
#####################################################################
10-
##
11-
## application config
12-
##
13-
#####################################################################
14-
PREFIX ?= /usr/local
15-
APP ?= $(notdir $(CURDIR))
16-
ARCH ?= $(shell uname -m)
17-
PLAT ?= $(shell uname -s)
18-
VSN ?= $(shell test -z "`git status --porcelain`" && git describe --tags --long | sed -e 's/-g[0-9a-f]*//' | sed -e 's/-0//' || echo "`git describe --abbrev=0 --tags`-SNAPSHOT")
19-
REL = ${APP}-${VSN}
20-
PKG ?= ${REL}+${ARCH}.${PLAT}
21-
TEST ?= ${APP}
22-
S3 ?=
23-
VMI ?= fogfish/erlang
24-
NET ?= lo0
25-
URL ?= undefined
26-
LATEST ?= latest
27-
28-
## root path to benchmark framework
29-
BB = ../basho_bench
30-
SSHENV = /tmp/ssh-agent.conf
31-
COOKIE?= nocookie
32-
33-
## erlang runtime flags use by `make run`
34-
ROOT = $(shell pwd)
35-
ADDR = $(shell ifconfig ${NET} | sed -En 's/^${NET}:.*inet (addr:)?(([0-9]*\.){3}[0-9]*).*/\2/p' && echo "127.0.0.1")
36-
EFLAGS = \
37-
-name ${APP}@${ADDR} \
38-
-setcookie ${COOKIE} \
39-
-pa ${ROOT}/_build/default/lib/*/ebin \
40-
-pa ${ROOT}/rel \
41-
-kernel inet_dist_listen_min 32100 \
42-
-kernel inet_dist_listen_max 32199 \
43-
+P 1000000 \
44-
+K true +A 160 -sbt ts
45-
46-
## self-extracting bundle wrapper
47-
BUNDLE_INIT = PREFIX=${PREFIX}\nREL=${PREFIX}/${REL}\nAPP=${APP}\nVSN=${VSN}\nLINE=`grep -a -n "BUNDLE:$$" $$0`\nmkdir -p $${REL}\ntail -n +$$(( $${LINE%%%%:*} + 1)) $$0 | gzip -vdc - | tar -C $${REL} -xvf - > /dev/null\n
48-
BUNDLE_FREE = exit\nBUNDLE:\n
49-
BUILDER = FROM ${VMI}\nRUN mkdir ${APP}\nCOPY . ${APP}/\nRUN cd ${APP} && make && make rel\n
50-
CTRUN = \
51-
-module(test). \
52-
-export([run/1]). \
53-
run(Spec) -> \
54-
{ok, Test} = file:consult(Spec), \
55-
case lists:keyfind(node, 1, Test) of \
56-
false -> ct:run_test([{spec, Spec}]); \
57-
true -> ct_master:run(Spec) \
58-
end, \
59-
erlang:halt().
60-
61-
#####################################################################
62-
##
63-
## build
64-
##
65-
#####################################################################
66-
all: rebar3 compile
67-
68-
compile:
69-
@./rebar3 compile
70-
71-
clean:
72-
@./rebar3 clean ;\
73-
rm -Rf _build/default/rel ;\
74-
rm -rf test.*-temp-data ;\
75-
rm -rf tests ;\
76-
rm -rf log ;\
77-
rm -f relx.config ;\
78-
rm -f *.tar.gz ;\
79-
rm -f *.bundle
80-
81-
distclean: clean
82-
@./rebar3 unlock ;\
83-
rm -Rf _build ;\
84-
rm -Rf rebar3
85-
86-
##
87-
## execute unit test
88-
unit: all
89-
@./rebar3 skip_deps=true eunit
90-
91-
##
92-
## execute common test and terminate node
93-
test: _build/test.beam
94-
@mkdir -p /tmp/test/${APP} ;\
95-
erl ${EFLAGS} -pa _build/ -pa test/ -run test run test/${TEST}.config
96-
97-
_build/test.beam: _build/test.erl
98-
erlc -o _build $<
99-
100-
_build/test.erl:
101-
echo "${CTRUN}" > $@
102-
103-
104-
#####################################################################
105-
##
106-
## release
107-
##
108-
#####################################################################
109-
rel: ${PKG}.tar.gz
110-
111-
## assemble VM release
112-
ifeq (${PLAT},$(shell uname -s))
113-
${PKG}.tar.gz: relx.config
114-
@./rebar3 tar -n ${APP} -v ${VSN} ;\
115-
cp _build/default/rel/${APP}/${APP}-${VSN}.tar.gz $@ ;\
116-
echo "==> tarball: $@"
117-
118-
relx.config: rel/relx.config.src
119-
@cat $< | sed 's/release/release, {${APP}, "${VSN}"}/' > $@
120-
else
121-
${PKG}.tar.gz: _build/dockermake
122-
@docker build --file=$< --force-rm=true --tag=build/${APP}:latest . ;\
123-
I=`docker create build/${APP}:latest` ;\
124-
docker cp $$I:/${APP}/$@ $@ ;\
125-
docker rm -f $$I ;\
126-
docker rmi build/${APP}:latest ;\
127-
test -f $@ && echo "==> tarball: $@"
128-
129-
_build/dockermake:
130-
@echo "${BUILDER}" > $@
131-
endif
132-
133-
## build docker image
134-
docker: rel/Dockerfile
135-
docker build \
136-
--build-arg APP=${APP} \
137-
--build-arg VSN=${VSN} \
138-
-t ${URL}/${APP}:${VSN} -f $< .
139-
docker tag -f ${URL}/${APP}:${VSN} ${URL}/${APP}:${LATEST}
140-
141-
142-
143-
#####################################################################
144-
##
145-
## package / bundle
146-
##
147-
#####################################################################
148-
pkg: ${PKG}.tar.gz ${PKG}.bundle
149-
150-
${PKG}.bundle: rel/deploy.sh
151-
@printf '${BUNDLE_INIT}' > $@ ;\
152-
cat $< >> $@ ;\
153-
printf '${BUNDLE_FREE}' >> $@ ;\
154-
cat ${PKG}.tar.gz >> $@ ;\
155-
chmod ugo+x $@ ;\
156-
echo "==> bundle: $@"
157-
158-
## copy 'package' to s3
159-
s3: ${PKG}.bundle
160-
aws s3 cp $< ${S3}/$<
161-
162-
s3-latest: ${PKG}.bundle
163-
aws s3 cp $< ${S3}/${APP}-latest${VARIANT}.bundle
164-
165-
#####################################################################
166-
##
167-
## deploy
168-
##
169-
#####################################################################
170-
ifneq (${host},)
171-
${SSHENV}:
172-
@echo "==> ssh: config keys" ;\
173-
ssh-agent -s > ${SSHENV}
174-
175-
node: ${PKG}.bundle ${SSHENV}
176-
@echo "==> deploy: ${host}" ;\
177-
. ${SSHENV} ;\
178-
k=`basename ${pass}` ;\
179-
l=`ssh-add -l | grep $$k` ;\
180-
if [ -z "$$l" ] ; then \
181-
ssh-add ${pass} ;\
182-
fi ;\
183-
rsync -cav --rsh=ssh --progress $< ${host}:$< ;\
184-
ssh -t ${host} "sudo sh ./$<"
185-
endif
186-
187-
#####################################################################
188-
##
189-
## debug
190-
##
191-
#####################################################################
192-
run:
193-
@erl ${EFLAGS}
194-
195-
benchmark:
196-
@echo "==> benchmark: ${TEST}" ;\
197-
$(BB)/basho_bench -N [email protected] -C nocookie priv/${TEST}.benchmark ;\
198-
$(BB)/priv/summary.r -i tests/current ;\
199-
open tests/current/summary.png
200-
201-
console: ${PKG}.tar.gz
202-
@_build/default/rel/${APP}/bin/${APP} console
203-
204-
#####################################################################
205-
##
206-
## dependencies
207-
##
208-
#####################################################################
209-
rebar3:
210-
@curl -L -O https://s3.amazonaws.com/rebar3/rebar3 ; \
211-
chmod ugo+x $@
212-
213-
.PHONY: test rel deps all pkg
1+
APP = uid
2+
ORG =
3+
URI =
2144

5+
include beam.mk

README.md

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
# k-ordered unique identity
22

3-
The library develops an identity schema suitable for rough, lexicographical objects ordering.
3+
The library develops an identity schema suitable for rough, lexicographical objects ordering in distributed systems.
4+
5+
[![Build Status](https://secure.travis-ci.org/fogfish/uid.svg?branch=master)](http://travis-ci.org/fogfish/uid)
6+
[![Coverage Status](https://coveralls.io/repos/github/fogfish/uid/badge.svg?branch=master)](https://coveralls.io/github/fogfish/uid?branch=master)
7+
[![Hex.pm](https://img.shields.io/hexpm/v/uid.svg)](https://hex.pm/packages/uid)
48

59

610
## Inspiration
@@ -26,9 +30,9 @@ The library has developed a dual schema: 64-bit unique identity to be used withi
2630

2731
The k-ordered value consists of time-stamp with millisecond resolution (50-bit) that is used to roughly sort events. The time-stamp ensures distinct sorting within virtual machine where system clock is controlled by operation system. A locally monotonic padding (14-bits) prevents the collisions (Note: 14-bits allows to have about 16K allocations per millisecond).
2832

29-
The time-stamp based sorting fails on distributed systems unless it implements precises time synchronization protocol. Therefore, it was decided to uses location aspect (node fingerprint) as component of k-ordered value. This component allows to keep ordering consistent even if clocks on other node is skewed. The developed schema allows to control the quality of the ordering (precision) depending on the length of _neighborhood interval_. The neighborhood interval is a time span where the location has higher priority then time.
33+
The time-stamp based sorting fails on distributed systems unless it implements precises time synchronization protocol. Therefore, it was decided to uses location aspect (node fingerprint) as component of k-ordered value. This component allows to keep ordering consistent even if clocks on other node is skewed. The developed schema allows to control the quality of the ordering (precision) depending on the length of _neighborhood interval_. The neighborhood interval is a time span where the location has higher priority then time. The usage of this time interval relaxes a clock synchronization requirements while preserving an ordering.
3034

31-
The library represents k-ordered values as tuples but implements binary and url friendly base64 encoding. The serialization protocol has been changed after library version `1.2.0`. New serialization improves allocation performance of k-order values. It uses Erlang OTP/18 feature `erlang:unique_integer(...)` to generate locally monotonic value. The library allocates about 13M k-ordered values per second on reference hardware.
35+
The library represents k-ordered values as tuples but implements binary and url friendly encoding similar to base64. The serialization protocol has been changed after library version `1.2.0`. New serialization improves allocation performance of k-order values. It uses Erlang OTP/18 feature `erlang:unique_integer(...)` to generate locally monotonic value. The library allocates about 13M k-ordered values per second on reference hardware.
3236

3337
### 64-bit, local
3438

@@ -101,17 +105,19 @@ uid:encode(B).
101105

102106
%%
103107
%% encode value to base64
104-
%% <<"AGA5igPrJzJ7aYAB">>
108+
%% <<".5.tXVEf8n8vPN.0">>
105109
uid:encode64(B).
106110
```
107111

108112
## How To Contribute
109113

110114
The library is Apache 2.0 licensed and accepts contributions via GitHub pull requests:
111115

112-
* Fork the repository on GitHub
113-
* Read build instructions
114-
* Make a pull request
116+
1. Fork it
117+
2. Create your feature branch (`git checkout -b my-new-feature`)
118+
3. Commit your changes (`git commit -am 'Added some feature'`)
119+
4. Push to the branch (`git push origin my-new-feature`)
120+
5. Create new Pull Request
115121

116122
The build process requires [Erlang/OTP](http://www.erlang.org/downloads) version 19.0 or later and essential build tools.
117123

@@ -156,9 +162,4 @@ If you experience any issues with the library, please let us know via [GitHub is
156162

157163
## License
158164

159-
Copyright 2012 Dmitry Kolesnikov
160-
161-
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0.
162-
163-
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
164-
165+
[![See LICENSE](https://img.shields.io/github/license/fogfish/uid.svg?style=for-the-badge)](LICENSE)

0 commit comments

Comments
 (0)