From f1c73bc65b8737243da675680d56d09a30fa0028 Mon Sep 17 00:00:00 2001 From: Joe Alcorn Date: Wed, 19 Aug 2015 09:42:45 +0100 Subject: [PATCH] Add redis storage backend --- celery_dedupe/__init__.py | 0 celery_dedupe/storage/__init__.py | 3 ++ celery_dedupe/storage/base.py | 18 +++++++++++ celery_dedupe/storage/redis.py | 21 ++++++++++++ dev-requirements.txt | 3 ++ tests/__init__.py | 0 tests/__init__.pyc | Bin 0 -> 133 bytes .../test_storages.cpython-27-PYTEST.pyc | Bin 0 -> 4273 bytes tests/test_storages.py | 30 ++++++++++++++++++ 9 files changed, 75 insertions(+) create mode 100644 celery_dedupe/__init__.py create mode 100644 celery_dedupe/storage/__init__.py create mode 100644 celery_dedupe/storage/base.py create mode 100644 celery_dedupe/storage/redis.py create mode 100644 dev-requirements.txt create mode 100644 tests/__init__.py create mode 100644 tests/__init__.pyc create mode 100644 tests/__pycache__/test_storages.cpython-27-PYTEST.pyc create mode 100644 tests/test_storages.py diff --git a/celery_dedupe/__init__.py b/celery_dedupe/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/celery_dedupe/storage/__init__.py b/celery_dedupe/storage/__init__.py new file mode 100644 index 0000000..002d20a --- /dev/null +++ b/celery_dedupe/storage/__init__.py @@ -0,0 +1,3 @@ +from .base import Storage + +__all__ = ['Storage'] diff --git a/celery_dedupe/storage/base.py b/celery_dedupe/storage/base.py new file mode 100644 index 0000000..ef0108b --- /dev/null +++ b/celery_dedupe/storage/base.py @@ -0,0 +1,18 @@ +class Storage(object): + ''' + Defines the public api for Storages + ''' + + default_config = {} + + def __init__(self, connection, **config): + self.connection = connection + self.config = self.default_config.copy() + self.config.update(config) + + def obtain_lock(self, key, value): + ''' + Responsible for obtaining a lock for the task + :returns boolean: Whether lock could be obtained or not + ''' + raise NotImplementedError diff --git a/celery_dedupe/storage/redis.py b/celery_dedupe/storage/redis.py new file mode 100644 index 0000000..e6d667c --- /dev/null +++ b/celery_dedupe/storage/redis.py @@ -0,0 +1,21 @@ +from . import Storage + +class RedisStorage(Storage): + ''' + Storage class for redis. + + Config: + expiry: number of seconds before expiring task, 0 to disable (default: 0) + ''' + + default_config = { + 'expiry': 0, + } + + def obtain_lock(self, key, value): + obtained = self.connection.setnx(key, value) + if self.config['expiry'] and obtained: + self.connection.expire(key, self.config['expiry']) + + return obtained + diff --git a/dev-requirements.txt b/dev-requirements.txt new file mode 100644 index 0000000..4eb8ef7 --- /dev/null +++ b/dev-requirements.txt @@ -0,0 +1,3 @@ +exam==0.10.5 +pytest==2.7.2 +redis==2.10.3 diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/__init__.pyc b/tests/__init__.pyc new file mode 100644 index 0000000000000000000000000000000000000000..be19f6ebc94defe31e932abb34677657a178575a GIT binary patch literal 133 zcmZSn%*(aV?n-Df0~9aBIw zQp@y{Q*%;_Ds@v*Q%Va`^-EHVON#a5<1_OzOXB183MxxDfLd&F^HWN5Qtdzn7XvW^ E04F~i+W-In literal 0 HcmV?d00001 diff --git a/tests/__pycache__/test_storages.cpython-27-PYTEST.pyc b/tests/__pycache__/test_storages.cpython-27-PYTEST.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f529bfacc8ddb81b59570f38352b59654dd7249f GIT binary patch literal 4273 zcmcgv&2Ah;5U!d1U)!-0h{Hbt1X@dA$8lm3i68=kV-6EQVlTtk(@u6X`=jYLiN)@P zAP$HZhx25)(t3SPW zuOhN5#UJ|e1}=LCO^RP2nMgM%`=RtJ(yvOlrX!(bbs03I+mL{>E0Q&3(2{P;%T>wR zGFXuAf|qNOEy`d?x=WDjVot~&R5#>Pk-H)vbeFl{^-dFu_kPFUwN8MptD)K7@1+;K zSckY{_C~q!@&e@RF7NHS$fP~%2NyA@;&nI5lN-jZVBlS2{hVTO=68sPZ{o67(HQmE zgmjUX1=6jEsnWJWg!du)-8y|{ry*%oJ57nMcWCe~O)syvsGDv%jHNJFjhu5YKeW8c zxhC7|;4a$OE%Y|tvnID2{gK&7%|{!%CNufLE2&BM$7aL9F{^ECy}x#SeBcDFfLvaTW2(> z9y9XK$`@1|ViDv@UA7A~h@>X`cG|pO&9{Xb8V5Ds7izpnHBCudGc`IXZO>#F548)j zy*h=(Eq%EylRAg-`y`N|!nq3ftMI%Ezu~JUB7t(vF;O=WF+_HNCb391clO23_~7!I zZGT*J&R)@>^W_ja#n8DmyFS`+NpBcuqupEWX@{>|EGMEatqx~#Ju8;HhMmE|>vUr#bj5~pcS2b9@(eUj& zALZ^5&IdaU5*M30<185_4iAw52=r)ZI?PnXL&Qv0Yq_?M`Q{P%!V!7l=B@lO2Od)W3zxehBCzpMVlz59viN z0jQ9s#6W;V1n_W}5Qk`iLx2#|=5bgLfH(l?Oiw}NF$9go-Jrk^v=A^86#+vP2^dgw z0{{V5G3o)+1bjU>Y9)NXF4cJWepjlQ!M7!fGp}ZbvqCM@z@f!M++o`BxSL_l#{pi$ zrv9Q#8#2L9mm?ym9e&t`P2+b-oJUD8P2~Xh{*bR%UJQ%KDO0_;*zm{s}!vz{T8lFOKs@jtbS6k9m z>0!Uod4vNC%3D5u3!I1F9B(mx4O}%`EyhT>>kKl@pF#jyFF)uJlh_H^C3?2ukz%GoQPl{dk`N-G5|UQosvA8ekpG;uBto8ay8;dwot3qyVXf3x;D zyE+M2 zd+sJr8UTQyurHyJpcR}3NqdU^3KV@pDSHtkynd9EC2fxq(np!%IsJ21kfdwR>GxCw1s>&e zOv31WoIbDShf<9vsh>+V^CZRE$*Vc!bT7|wdP7`WUtS46p~CM5zb`GIM_I!pWp@|vmNVX^0!=do68ag2SX zFc_u#nTg|wQBKK9 Jfd6nM`~%EJV3Pm< literal 0 HcmV?d00001 diff --git a/tests/test_storages.py b/tests/test_storages.py new file mode 100644 index 0000000..5098526 --- /dev/null +++ b/tests/test_storages.py @@ -0,0 +1,30 @@ +from uuid import uuid4 +from redis import StrictRedis + +from celery_dedupe.storage.redis import RedisStorage + +class TestRedisStorage(object): + + @property + def redis(self): + if hasattr(self, '_redis'): + return self._redis + + self._redis = StrictRedis() + return self._redis + + def test_lock_obtained(self): + storage = RedisStorage(self.redis) + assert storage.obtain_lock(uuid4(), '1') + + def test_lock_obtained_with_expiry(self): + key = uuid4() + storage = RedisStorage(self.redis, expiry=10) + assert storage.obtain_lock(key, '1') + assert self.redis.ttl(key) == 10 + + def test_already_locked(self): + key = uuid4() + self.redis.setex(key, '1', 10) + storage = RedisStorage(self.redis, expiry=10) + assert not storage.obtain_lock(key, '1')