Skip to content

Commit 0665bb5

Browse files
fno2010fno2010
committed
Improve version control backend
- Add unit tests for version control backend - Support more options for version control system and update listener - `polling_interval`, `snapshot_freq`, `init_ver` - `diff_format`: so far support json patch and json merge patch - Add initial sketch for network map and cost map - Update Dockerfile and docker-compose.yml Signed-off-by: jensenzhang <[email protected]>
1 parent c09ec02 commit 0665bb5

15 files changed

+538
-34
lines changed

‎Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ FROM python:3.10.7
22

33
COPY / /tmp/alto
44
RUN rm -rf /tmp/alto/.git && \
5-
pip install redis geoip2 && \
5+
pip install redis geoip2 kazoo && \
66
pip install /tmp/alto && \
77
rm -rf /tmp/alto
88

‎docker-compose-zookeeper.yml

+63
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
services:
2+
gateway:
3+
image: nginx
4+
volumes:
5+
- ./etc/nginx.conf:/etc/nginx/nginx.conf
6+
- ./etc/htpasswd:/etc/nginx/conf.d/.htpasswd
7+
- ./etc/cert.pem:/etc/nginx/cert.pem
8+
- ./etc/cert.key:/etc/nginx/cert.key
9+
ports:
10+
- "8443:443"
11+
alto-frontend:
12+
image: openalto/alto
13+
volumes:
14+
- ./src/alto:/usr/local/lib/python3.10/site-packages/alto
15+
- ./etc/alto.conf:/opt/alto/etc/alto.conf
16+
entrypoint: gunicorn
17+
command: ["-b", "0.0.0.0:8000", "--reload", "alto.server.northbound.wsgi", "--preload", "--capture-output"]
18+
alto-lg-agent:
19+
image: openalto/alto
20+
volumes:
21+
- ./etc/lg-agent.json:/etc/lg-agent.json
22+
- ./etc/alto.conf:/opt/alto/etc/alto.conf
23+
entrypoint: python
24+
command: ["-m", "alto.agent.manage", "--pid", "/tmp", "start", "-c", "/etc/lg-agent.json", "-D", "cernlg"]
25+
network_mode: "service:alto-frontend"
26+
alto-cric-agent:
27+
image: openalto/alto
28+
volumes:
29+
- ./etc/cric-agent.json:/etc/cric-agent.json
30+
- ./etc/alto.conf:/opt/alto/etc/alto.conf
31+
entrypoint: python
32+
command: ["-m", "alto.agent.manage", "--pid", "/tmp", "start", "-c", "/etc/cric-agent.json", "-D", "cric"]
33+
network_mode: "service:alto-frontend"
34+
alto-db:
35+
image: redis
36+
network_mode: "service:alto-frontend"
37+
zoo1:
38+
image: zookeeper
39+
restart: always
40+
hostname: zoo1
41+
ports:
42+
- 2181:2181
43+
environment:
44+
ZOO_MY_ID: 1
45+
ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181
46+
zoo2:
47+
image: zookeeper
48+
restart: always
49+
hostname: zoo2
50+
ports:
51+
- 2182:2181
52+
environment:
53+
ZOO_MY_ID: 2
54+
ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181
55+
zoo3:
56+
image: zookeeper
57+
restart: always
58+
hostname: zoo3
59+
ports:
60+
- 2183:2181
61+
environment:
62+
ZOO_MY_ID: 3
63+
ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181

‎docker-compose.yml

-2
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,6 @@ services:
3737
alto-client:
3838
image: curlimages/curl
3939
volumes:
40-
- ./tmp/request.json:/request.json
41-
- ./tmp/pvtest.sh:/pvtest.sh
4240
- ./etc/cert.pem:/etc/cert.pem
4341
- ./etc/cert.key:/etc/cert.key
4442
entrypoint: sh

‎etc/alto.conf.test

+17
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,20 @@ resources = {
9999
"namespaces": []
100100
}
101101
},
102+
"default-networkmap": {
103+
"type": "network-map",
104+
"path": "networkmap",
105+
"namespace": "default",
106+
"algorithm": "alto.server.components.backend.MockService",
107+
"params": {}
108+
},
109+
"dynamic-networkmap": {
110+
"type": "network-map",
111+
"path": "networkmap",
112+
"namespace": "default",
113+
"algorithm": "alto.server.components.backend.MockService",
114+
"params": {}
115+
},
102116
"pv": {
103117
"type": "path-vector",
104118
"path": "pathvector",
@@ -141,6 +155,9 @@ resources = {
141155
# Configuration for version control system
142156
zookeeper_host = zoo1
143157
zookeeper_timeout = 15
158+
polling_interval = 1
159+
snapshot_freq = 3
160+
init_version = 100
144161

145162
####################################################
146163
# Common configuratoin

‎setup.cfg

+2
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,8 @@ install_requires =
5858
lxml
5959
service
6060
gunicorn
61+
jsonpatch
62+
json-merge-patch
6163

6264

6365
[options.packages.find]

‎src/alto/common/constants.py

+8
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,9 @@
2424
# Authors:
2525
# - Jensen Zhang <[email protected]>
2626

27+
from enum import Enum
28+
29+
2730
ALTO_CONTENT_TYPE_IRD = 'application/alto-directory+json'
2831
ALTO_CONTENT_TYPE_NM = 'application/alto-networkmap+json'
2932
ALTO_CONTENT_TYPE_CM = 'application/alto-costmap+json'
@@ -57,3 +60,8 @@
5760
"path-vector": ALTO_PARAMETER_TYPE_ECS,
5861
"entity-prop": ALTO_PARAMETER_TYPE_PROPMAP
5962
}
63+
64+
65+
class Diff(Enum):
66+
JSON_PATCH = 1
67+
JSON_MERGE_PATCH = 2

‎src/alto/config.py

+18
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,9 @@ def get_server_cost_types(self):
158158
return json.loads(cost_types_json.strip())
159159

160160

161+
####################################
162+
# ALTO server version control config
163+
####################################
161164
def get_vcs_zookeeper_host(self):
162165
self.parser.read(self.location)
163166
return self.parser.get('server.vcs', 'zookeeper_host', fallback=None)
@@ -166,3 +169,18 @@ def get_vcs_zookeeper_host(self):
166169
def get_vcs_zookeeper_timeout(self):
167170
self.parser.read(self.location)
168171
return int(self.parser.get('server.vcs', 'zookeeper_timeout', fallback=15))
172+
173+
174+
def get_vcs_polling_interval(self):
175+
self.parser.read(self.location)
176+
return int(self.parser.get('server.vcs', 'polling_interval', fallback=5))
177+
178+
179+
def get_vcs_snapshot_freq(self):
180+
self.parser.read(self.location)
181+
return int(self.parser.get('server.vcs', 'snapshot_freq', fallback=5))
182+
183+
184+
def get_vcs_init_version(self):
185+
self.parser.read(self.location)
186+
return int(self.parser.get('server.vcs', 'init_version', fallback=1))

‎src/alto/mock.py

+140-1
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,59 @@ def city(self, endpoint):
7373
mockGeoIP2 = MockGeoIP2()
7474

7575

76+
class MockKazoo(MagicMock):
77+
78+
class KazooClient:
79+
80+
def __init__(self, **kwargs):
81+
self.base = dict()
82+
83+
def _parse_path(self, path):
84+
return path.split('/')[1:]
85+
86+
def _get_leaf_node(self, node_path):
87+
parent = self.base
88+
for n in node_path:
89+
parent = parent[n]
90+
return parent
91+
92+
def start(self, **kwargs):
93+
pass
94+
95+
def stop(self, **kwargs):
96+
pass
97+
98+
def ensure_path(self, path, **kwargs):
99+
nodes = self._parse_path(path)
100+
parent = self.base
101+
for n in nodes:
102+
if n not in parent:
103+
parent[n] = dict()
104+
parent = parent[n]
105+
106+
def create(self, path, data, **kwargs):
107+
nodes = self._parse_path(path)
108+
parent = self._get_leaf_node(nodes[:-1])
109+
parent[nodes[-1]] = data
110+
111+
def get_children(self, path, **kwargs):
112+
nodes = self._parse_path(path)
113+
parent = self._get_leaf_node(nodes)
114+
return list(parent.keys())
115+
116+
def get(self, path, **kwargs):
117+
nodes = self._parse_path(path)
118+
return self._get_leaf_node(nodes), None
119+
120+
def delete(self, path, **kwargs):
121+
nodes = self._parse_path(path)
122+
parent = self._get_leaf_node(nodes[:-1])
123+
del parent[nodes[-1]]
124+
125+
126+
mockKazoo = MockKazoo()
127+
128+
76129
TEST_DEFAULT_NM = {
77130
"meta" : {
78131
"vtag": {
@@ -184,13 +237,86 @@ def city(self, endpoint):
184237
--d41d8cd98f00b204e9800998ecf8427e--
185238
"""
186239

240+
TEST_DYNAMIC_NM_1 = {
241+
"meta" : {
242+
"vtag": {
243+
"resource-id": "dynamic-networkmap",
244+
"tag": "da65eca2eb7a10ce8b059740b0b2e3f8eb1d4785"
245+
}
246+
},
247+
"network-map" : {
248+
"PID1" : {
249+
"ipv4" : [ "192.0.2.0/24", "198.51.100.0/25" ]
250+
},
251+
"PID2" : {
252+
"ipv4" : [ "198.51.100.128/25" ]
253+
},
254+
"PID0" : {
255+
"ipv4" : [ "0.0.0.0/0" ],
256+
"ipv6" : [ "::/0" ]
257+
}
258+
}
259+
}
260+
261+
TEST_DYNAMIC_NM_2 = {
262+
"meta" : {
263+
"vtag": {
264+
"resource-id": "dynamic-networkmap",
265+
"tag": "8124fa5960796ba5d9295d1bb094563fe77fff03"
266+
}
267+
},
268+
"network-map" : {
269+
"PID1" : {
270+
"ipv4" : [ "192.0.2.0/24", "198.51.100.0/25" ]
271+
},
272+
"PID2" : {
273+
"ipv4" : [ "198.51.100.128/25" ]
274+
},
275+
"PID3" : {
276+
"ipv4" : [ "192.0.3.0/24" ]
277+
},
278+
"PID0" : {
279+
"ipv4" : [ "0.0.0.0/0" ],
280+
"ipv6" : [ "::/0" ]
281+
}
282+
}
283+
}
284+
285+
TEST_DYNAMIC_NM_3 = {
286+
"meta" : {
287+
"vtag": {
288+
"resource-id": "dynamic-networkmap",
289+
"tag": "dafb9c1b9a78d00ab6896c09b8e350f727fd966c"
290+
}
291+
},
292+
"network-map" : {
293+
"PID1" : {
294+
"ipv4" : [ "192.0.2.0/24", "198.51.100.0/24" ]
295+
},
296+
"PID3" : {
297+
"ipv4" : [ "192.0.3.0/24" ]
298+
},
299+
"PID0" : {
300+
"ipv4" : [ "0.0.0.0/0" ],
301+
"ipv6" : [ "::/0" ]
302+
}
303+
}
304+
}
305+
187306
MOCK = [
188307
{
189308
'uri': 'http://localhost:8181/alto/networkmap/default-networkmap',
190309
'headers': {'content-type': ALTO_CTYPE_NM},
191310
'json': TEST_DEFAULT_NM,
192311
'status_code': 200
193312
},
313+
{
314+
'uri': 'https://alto.example.com/networkmap/dynamic-networkmap',
315+
'headers': {'content-type': ALTO_CTYPE_NM},
316+
'json_alt': [TEST_DYNAMIC_NM_1, TEST_DYNAMIC_NM_2, TEST_DYNAMIC_NM_3],
317+
'index': 0,
318+
'status_code': 200
319+
},
194320
{
195321
'uri': 'http://localhost:8181/alto/costmap/default-costmap',
196322
'headers': {'content-type': ALTO_CTYPE_CM},
@@ -231,6 +357,7 @@ def __init__(self, headers=dict(), json_data=None, text_data=None, status_code=2
231357
self.json_data = json_data
232358
self.status_code = status_code
233359
self.text = text_data or json.dumps(self.json())
360+
self.content = self.text.encode()
234361

235362
def raise_for_status(self):
236363
pass
@@ -240,8 +367,20 @@ def json(self):
240367

241368
for mock_item in MOCK:
242369
if uri == mock_item.get('uri'):
370+
json_data = mock_item.get('json')
371+
if 'json_alt' in mock_item:
372+
json_alt = mock_item.get('json_alt')
373+
if 'index' not in mock_item:
374+
mock_item['index'] = 0
375+
json_idx = mock_item['index'] % len(json_alt)
376+
json_data = json_alt[json_idx]
377+
mock_item['index'] = json_idx + 1
243378
return MockResponse(headers=mock_item.get('headers'),
244-
json_data=mock_item.get('json'),
379+
json_data=json_data,
245380
text_data=mock_item.get('text'),
246381
status_code=mock_item.get('status_code'))
247382

383+
384+
def mocked_requests_request(method, uri, *args, **kwargs):
385+
return mocked_requests_get(uri)
386+

‎src/alto/server/components/backend.py

+12
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,18 @@
3434
from .db import data_broker_manager
3535

3636

37+
class MockService:
38+
"""
39+
Mock backend algorithm for test purpose.
40+
"""
41+
42+
def __init__(self, *args, **kwargs):
43+
pass
44+
45+
def lookup(self, *args, **kwargs):
46+
return dict()
47+
48+
3749
class IRDService:
3850
"""
3951
Backend algorithm for IRD generation.

0 commit comments

Comments
 (0)