do not store suricata timestamps in a table

stratosphereips · Sep 25, 2023 · d6d68bc · d6d68bc
1 parent a2f1e05
commit d6d68bc
Show file tree

Hide file tree

Showing 4 changed files with 17 additions and 31 deletions.
diff --git a/database/sqlite_db.py b/database/sqlite_db.py
@@ -55,12 +55,6 @@ def init_tables(self):
                                   "FOREIGN KEY (aid) REFERENCES flows(aid), "
                                   "FOREIGN KEY (label) REFERENCES flows(ground_truth)",
 
-            # this reads the ts of all suricata flows, and has the aid and suricata_label in common with the "flows" table
-            'suricata_flows': "aid TEXT PRIMARY KEY, "
-                              "timestamp REAL, "
-                              "label TEXT,  "
-                              "FOREIGN KEY (aid) REFERENCES flows(aid), "
-                              "FOREIGN KEY (label) REFERENCES flows(suricata_label)",
             'performance_errors': "tool TEXT, "
                                   "TP INTEGER, "
                                   "FP INTEGER, "
@@ -141,9 +135,6 @@ def get_column_names(self, table: str) -> list:
             column_names.append(col[1])
         return column_names
 
-    def get_malicious_suricata_flows(self):
-        return self.select('labels_flow_by_flow', '*', condition=f"{self.suricata_label_col} = 'malicious'")
-
     def fill_null_labels(self):
         """
         iterates through all flows in the flows table, and fills the null labels with benign
@@ -223,15 +214,6 @@ def get_aid_collisions(self):
         return self.aid_collisions
 
 
-    def store_suricata_flow(self, flow: dict):
-        """
-        fills the suricata_flows table with the suricata flow read from eve.json
-        :param flow: contains timestamp, aid and label of the flow
-        """
-        query = f'INSERT OR REPLACE INTO suricata_flows (aid, timestamp, label) VALUES (?, ?, ?);'
-        params = (flow['aid'], flow['timestamp'], flow['label'])
-        self.execute(query, params=params)
-
     def store_ground_truth_flow(self, flow: dict):
         """
         fills the ground_truth_flows table with the gt flow read from the zeek log

diff --git a/main.py b/main.py
@@ -38,7 +38,7 @@ def __init__(self):
 
         self.db = SQLiteDB(self.output_dir)
         self.add_metadata()
-        self.log(f"Storing metadata in: ", self.results_path)
+        self.log(f"Storing results in: ", self.results_path)
 
 
     def setup_output_dir(self):

diff --git a/parsers/slips.py b/parsers/slips.py
@@ -110,7 +110,9 @@ def print_stats(self):
         self.log('', "-" * 30)
         self.log(f"Total malicious labels: ", self.db.get_flows_count('slips', 'malicious'))
         self.log(f"Total benign labels: ", self.db.get_flows_count('slips', 'benign'))
-        self.log(f"Total Slips discarded timewindow labels (due to inability to map the ts to an existing tw): ", self.discarded_tw_labels)
+        self.log(f"Total Slips discarded timewindow labels "
+                 f"(due to inability to map the ts to an existing tw): ",
+                 self.discarded_tw_labels)
         self.log('', "-" * 30)
 
         print()

diff --git a/parsers/suricata.py b/parsers/suricata.py
@@ -77,6 +77,18 @@ def warn_about_discarded_alert(self, ts):
                  f"timewindows in gt start at: {gt_start_time} and end at: {gt_end_time}. ",
                  "discarding alert.")
 
+    def print_stats(self):
+        self.log('', "-" * 30)
+        self.log(f"Total malicious labels: ", self.db.get_flows_count('suricata', 'malicious'))
+        self.log(f"Total benign labels: ", self.db.get_flows_count('suricata', 'benign'))
+        self.log(f"Total Suricata discarded timewindow labels "
+                 f"(due to inability to map the ts to an existing current tw): ",
+                 self.discarded_tw_labels )
+        self.log('', "-" * 30)
+
+        print()
+
+
     def parse(self):
         """reads the given suricata eve.json"""
         with open(self.eve_file, 'r') as f:
@@ -118,18 +130,8 @@ def parse(self):
                         if not self.label_malicious_tw(timestamp, line['src_ip']):
                             self.warn_about_discarded_alert(timestamp)
 
-                # this one will be used later for labeling tws
-                self.db.store_suricata_flow(flow)
-
-            self.log('', "-" * 30)
-            self.log(f"Total malicious labels: ", self.db.get_flows_count('suricata', 'malicious'))
-            self.log(f"Total benign labels: ", self.db.get_flows_count('suricata', 'benign'))
-            self.log(f"Total Suricata discarded timewindow labels (due to inability to map the ts to an existing current tw): ",
-                     self.discarded_tw_labels )
-            self.log('', "-" * 30)
-
-            print()
 
+            self.print_stats()