Merge pull request #2 from dpgoetz/log_source

ignore proxy logs with any swift.source

etc/log-processor.conf-sample

@@ -33,7 +33,6 @@ class_path = slogging.access_processor.AccessLogProcessor
 # warn_percent = 0.8
 # list of swift.sources (see swift/proxy/server.py posthooklogger)
 # that count as service traffic
-# service_log_sources =
 # content_type =
 
 [log-processor-stats]

slogging/access_log_delivery.py

@@ -144,6 +144,8 @@ def get_container_save_log_flag(self, account, container):
 
     def convert_log_line(self, raw_log):
         parts = self.log_line_parser(raw_log)
+        if parts == {}:
+            return None, None, None
         return (make_clf_from_parts(parts),
                 parts.get('account'),
                 parts.get('container_name'))
@@ -152,6 +154,12 @@ def log_line_parser(self, raw_log):
         '''given a raw access log line, return a dict of the good parts'''
         d = {}
         try:
+            log_arr = raw_log[16:].split(' ')
+            if len(log_arr) > 18:
+                log_source = log_arr[18]
+                if log_source != '-':
+                    # internal proxy log
+                    return {}
             (unused,
             server,
             client_ip,
@@ -169,8 +177,7 @@ def log_line_parser(self, raw_log):
             etag,
             trans_id,
             headers,
-            processing_time) = (unquote(x) for x in
-                                raw_log[16:].split(' ')[:18])
+            processing_time) = (unquote(x) for x in log_arr[:18])
         except ValueError:
             self.logger.debug(_('Bad line data: %s') % repr(raw_log))
             return {}

slogging/access_processor.py

@@ -50,8 +50,7 @@ class AccessLogProcessor(object):
 
     def __init__(self, conf):
         self.server_name = conf.get('server_name', 'proxy-server')
-        for conf_tag in ['lb_private_ips', 'service_ips',
-                         'service_log_sources']:
+        for conf_tag in ['lb_private_ips', 'service_ips']:
             setattr(self, conf_tag, return_ips(conf, conf_tag))
         self.warn_percent = float(conf.get('warn_percent', '0.8'))
         self.logger = get_logger(conf, log_route='access-processor')
@@ -193,7 +192,7 @@ def process(self, obj_stream, data_object_account, data_object_container,
                 sanitize_ips(line_data)
             if line_data['lb_ip'] in self.lb_private_ips or \
                     line_data['client_ip'] in self.service_ips or \
-                    line_data['log_source'] in self.service_log_sources:
+                    line_data['log_source'] not in ['-', None]:
                 source = 'service'
             else:
                 source = 'public'

test_slogging/unit/test_access_log_delivery.py

@@ -147,15 +147,8 @@ def test_log_line_parser_field_count(self):
         log_line[6] = '/v1/a/c/o'
         log_line = 'x' * 16 + ' '.join(log_line)
         res = p.log_line_parser(log_line)
-        expected = {'code': 8, 'processing_time': '17', 'auth_token': '11',
-                    'month': '01', 'second': '6', 'year': '3', 'tz': '+0000',
-                    'http_version': '7', 'object_name': 'o', 'etag': '14',
-                    'method': '5', 'trans_id': '15', 'client_ip': '2',
-                    'bytes_out': 13, 'container_name': 'c', 'day': '1',
-                    'minute': '5', 'account': 'a', 'hour': '4',
-                    'referrer': '9', 'request': '/v1/a/c/o',
-                    'user_agent': '10', 'bytes_in': 12, 'lb_ip': '3'}
-        self.assertEquals(res, expected)
+        # throws away invalid log lines
+        self.assertEquals(res, {})
 
     def test_make_clf_from_parts(self):
         p = access_log_delivery.AccessLogDelivery(self.conf, DumbLogger())

test_slogging/unit/test_log_processor.py

@@ -167,7 +167,7 @@ def test_access_log_line_parser(self):
            'user_agent': 'curl',
            'bytes_in': 6,
            'lb_ip': '4.5.6.7',
-            'log_source': None})
+           'log_source': None})
 
     def test_process_one_access_file(self):
         access_proxy_config = self.proxy_config.copy()
@@ -736,3 +736,7 @@ def process_logs(logs_to_process, processed_files):
                     'Method should not be called'
 
         MockLogProcessorDaemon(self).run_once()
+
+
+if __name__ == '__main__':
+    unittest.main()