Handle properly when API response is 200 and JSON yet without pageInfo …

…#14 (#16)

For event fetching, --limit can be set with CLIETP_FETCH_LIMIT environment variable #12
Update OSS header to 2024
Bump version to 0.4.6

bin/akamai-etp

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-# Copyright 2023 Akamai Technologies, Inc. All Rights Reserved
+# Copyright 2024 Akamai Technologies, Inc. All Rights Reserved
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -40,7 +40,7 @@ from requests.compat import urljoin
 from akamai.edgegrid import EdgeGridAuth, EdgeRc
 from config import EdgeGridConfig
 
-__version__ = "0.4.5"
+__version__ = "0.4.6"
 
 #: Window span in ad-hoc mode, default is 3 min
 span_duration_min = 3
@@ -285,6 +285,7 @@ def fetch_event_page(start, end, page_number, thread_pool, pool_futures, stats,
         stats.inc_api_call()
         LOG.info(f"{{OPEN}} API response code is HTTP/{r.status_code}, body {len(r.content):,} "
                  f"bytes, page #{page_number}")
+
         if r.status_code != 200:
             stats.api_calls_fail += 1
             LOG.error(f"API call failed with HTTP/{r.status_code}: {r.content}")
@@ -300,8 +301,12 @@ def fetch_event_page(start, end, page_number, thread_pool, pool_futures, stats,
         output.flush()
 
         if page_number == 1:
-            LOG.info("Page info: %s" % response_data.get("pageInfo", {}))
-            total_records = response_data["pageInfo"]["totalRecords"]
+            page_info = response_data.get("pageInfo", {})
+            LOG.info("Page info: %s" % page_info)
+            total_records = page_info.get("totalRecords")
+            if total_records is None:
+                logging.info("API response without pageinfo.")
+                return
             numOfPages = int((total_records / EVENT_PAGE_SIZE) + 1)
             if numOfPages > 1:
                 LOG.info(f"Calculated number of pages: {numOfPages} (based on {total_records:,} total events)")

bin/config.py

@@ -1,6 +1,6 @@
 # Python edgegrid module - CONFIG for ETP CLI module
 """
-Copyright 2022 Akamai Technologies, Inc. All Rights Reserved.
+Copyright 2024 Akamai Technologies, Inc. All Rights Reserved.
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 
@@ -58,11 +58,11 @@ def __init__(self, config_values, configuration, flags=None):
                                        """to the input. --start and --end are ignored when used.""")
         event_parser.add_argument('--poll', type=int, default=60,
                                   help="Poll frequency in seconds with --tail mode. Default is 60s")
-        event_parser.add_argument('--limit', type=int, default=3*60,
+        event_parser.add_argument('--limit', type=int, default=os.environ.get("CLIETP_FETCH_LIMIT", 3*60),
                                   help="Stop the most recent fetch to now minus specified seconds, default is 3 min. "
-                                       "Applicable to --tail")
+                                       "Applicable to --tail. Environment variable: CLIETP_FETCH_LIMIT")
         event_parser.add_argument('--concurrent', type=int, default=os.environ.get('CLIETP_FETCH_CONCURRENT', 1),
-                                  help="Number of concurrent API call")
+                                  help="Number of concurrent API calls. Environment variable: CLIETP_FETCH_CONCURRENT")
 
         # ETP Lists
         list_parser = subparsers.add_parser("list", help="Manage ETP security list",

cli.json

@@ -5,7 +5,7 @@
   "commands": [
     {
       "name": "etp",
-      "version": "0.4.5",
+      "version": "0.4.6",
       "description": "Akamai CLI for Secure Internet Access Enterprise (f.k.a. Enterprise Threat Protector)"
     }
   ]