From e0ea23a0a67c0085bcd6aa04e6bb2f3146429f81 Mon Sep 17 00:00:00 2001 From: marlemion Date: Tue, 17 Aug 2021 16:56:12 +0200 Subject: [PATCH 1/2] Suggested solution to 'yielder' problem in resources.py The yielder() took away the advantages of a generator: generate the data in place. By the yielder, the program had to go through the whole generator function before knowing the pagination. This became problematic for huge data sets (i.e. >15000). With the proposed design, the actual query is delivered alongside with the store to the generator function and the generator is fed with the restriction. The restriction is necessary, as only folder.items has a query parameter, whereas folder.folders and folder.items both have a restriction parameter. --- grapi/backend/kopano/resource.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/grapi/backend/kopano/resource.py b/grapi/backend/kopano/resource.py index 9745546..1d26001 100644 --- a/grapi/backend/kopano/resource.py +++ b/grapi/backend/kopano/resource.py @@ -267,7 +267,7 @@ def respond(self, req, resp, obj, all_fields=None, deltalink=None): expand[field.split('/')[1]] = self.get_fields(req, obj2, resource.fields, resource.fields) resp.body = self.json(req, obj, fields, all_fields, expand=expand) - def generator(self, req, generator, count=0, args=None): + def generator(self, req, generator, count=0, args=None, query=None, store=None): """Response generator. It also determines pagination and ordering. @@ -292,7 +292,12 @@ def generator(self, req, generator, count=0, args=None): if order: order = tuple(('-' if len(o.split()) > 1 and o.split()[1] == 'desc' else '') + o.split()[0] for o in order) - return generator(page_start=skip, page_limit=top, order=order), top, skip, count + if query is not None: + restriction = _query_to_restriction(query, 'message', store) + else: + restriction = None + + return generator(page_start=skip, page_limit=top, restriction=restriction, order=order), top, skip, count def create_message(self, folder, fields, all_fields=None, message_class=None): # TODO item.update and/or only save in the end @@ -333,10 +338,7 @@ def folder_gen(self, req, folder): if '$search' in args: query = args['$search'][0] - - def yielder(**kwargs): - for item in folder.items(query=query): - yield item - return self.generator(req, yielder, 0, args=args) + + return self.generator(req, yielder, 0, args=args, query=query, store=folder.store) else: return self.generator(req, folder.items, folder.count, args=args) From b6dc98f6fae42f45272571700e45e030b82dd10b Mon Sep 17 00:00:00 2001 From: marlemion Date: Tue, 17 Aug 2021 17:00:00 +0200 Subject: [PATCH 2/2] Update resource.py Forgot the most important part --- grapi/backend/kopano/resource.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grapi/backend/kopano/resource.py b/grapi/backend/kopano/resource.py index 1d26001..ec0845e 100644 --- a/grapi/backend/kopano/resource.py +++ b/grapi/backend/kopano/resource.py @@ -339,6 +339,6 @@ def folder_gen(self, req, folder): if '$search' in args: query = args['$search'][0] - return self.generator(req, yielder, 0, args=args, query=query, store=folder.store) + return self.generator(req, folder.items, 0, args=args, query=query, store=folder.store) else: return self.generator(req, folder.items, folder.count, args=args)