initial commit

.gitignore

@@ -0,0 +1,2 @@
+target
+Cargo.lock

Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "gst-log-parser"
+version = "0.1.0"
+authors = ["Guillaume Desmottes <[email protected]>"]
+
+[dependencies]
+itertools = "0.5.9"
+gstreamer = "0.8.1"
+regex = "0.2"
+lazy_static = "0.2.9"

src/lib.rs

@@ -0,0 +1,240 @@
+use std::io::BufReader;
+use std::io::BufRead;
+use std::io::Read;
+use std::io::Lines;
+use std::fmt;
+use std::str;
+extern crate itertools;
+use itertools::join;
+
+extern crate gstreamer as gst;
+use gst::{DebugLevel, SECOND};
+
+#[macro_use]
+extern crate lazy_static;
+extern crate regex;
+use regex::Regex;
+
+#[derive(Debug)]
+pub struct ParsingError;
+
+pub struct Entry {
+    pub ts: u64,
+    pub pid: u32,
+    pub thread: String,
+    pub level: DebugLevel,
+    pub category: String,
+    pub file: String,
+    pub line: u32,
+    pub function: String,
+    pub message: String,
+    pub object: Option<String>,
+}
+
+fn parse_debug_level(s: &str) -> Result<DebugLevel, ParsingError> {
+    match s {
+        "ERROR" => Ok(DebugLevel::Error),
+        "WARNING" => Ok(DebugLevel::Warning),
+        "FIXME" => Ok(DebugLevel::Fixme),
+        "INFO" => Ok(DebugLevel::Info),
+        "DEBUG" => Ok(DebugLevel::Debug),
+        "LOG" => Ok(DebugLevel::Log),
+        "TRACE" => Ok(DebugLevel::Trace),
+        "MEMDUMP" => Ok(DebugLevel::Memdump),
+        _ => Err(ParsingError),
+    }
+}
+
+fn parse_time(ts: &str) -> u64 {
+    let mut split = ts.splitn(3, ':');
+    let h: u64 = split.next().expect("missing hour").parse().expect(
+        "invalid hour",
+    );
+    let m: u64 = split.next().expect("missing minute").parse().expect(
+        "invalid minute",
+    );
+    split = split.next().expect("missing second").splitn(2, '.');
+    let secs: u64 = split.next().expect("missing second").parse().expect(
+        "invalid second",
+    );
+    let subsecs: u64 = split.next().expect("missing sub second").parse().expect(
+        "invalid sub second",
+    );
+
+    (h * 60 * 60 + m * 60) * SECOND + secs * SECOND + subsecs
+}
+
+fn split_location(location: &str) -> (String, u32, String, Option<String>) {
+    let mut split = location.splitn(4, ":");
+    let file = split.next().expect("missing file");
+    let line = split.next().expect("missing fine").parse().expect(
+        "invalid line",
+    );
+    let function = split.next().expect("missing function");
+    let object = split.next().expect("missing object delimiter");
+    let object_name;
+    if object.len() > 0 {
+        let object = object
+            .to_string()
+            .trim_left_matches("<")
+            .trim_right_matches(">")
+            .to_string();
+        object_name = Some(object);
+    } else {
+        object_name = None;
+    }
+
+    (file.to_string(), line, function.to_string(), object_name)
+}
+
+impl Entry {
+    fn new(line: String) -> Entry {
+        // Strip color codes
+        lazy_static! {
+            static ref RE: Regex = Regex::new("\x1b\\[[0-9;]*m").unwrap();
+        }
+        let line = RE.replace_all(&line, "");
+
+        let mut it = line.split(" ");
+        let ts = parse_time(it.next().expect("Missing ts"));
+        let mut it = it.skip_while(|x| x.is_empty());
+        let pid = it.next().expect("Missing PID").parse().expect(
+            "Failed to parse PID",
+        );
+        let mut it = it.skip_while(|x| x.is_empty());
+        let thread = it.next().expect("Missing thread").to_string();
+        let mut it = it.skip_while(|x| x.is_empty());
+        let level = parse_debug_level(it.next().expect("Missing level")).expect("Invalid level");
+        let mut it = it.skip_while(|x| x.is_empty());
+        let category = it.next().expect("Missing Category").to_string();
+        let mut it = it.skip_while(|x| x.is_empty());
+        let (file, line, function, object) = split_location(it.next().expect("Missing location"));
+        let message: String = join(it, " ");
+
+        Entry {
+            ts: ts,
+            pid: pid,
+            thread: thread,
+            level: level,
+            category: category,
+            file: file,
+            line: line,
+            function: function,
+            object: object,
+            message: message,
+        }
+    }
+
+    pub fn ts_format(&self) -> String {
+        let secs: u64;
+
+        secs = self.ts / SECOND;
+        format!(
+            "{}:{:02}:{:02}.{:09}",
+            secs / (60 * 60),
+            (secs / 60) % 60,
+            secs % 60,
+            self.ts % SECOND
+        )
+    }
+}
+
+impl fmt::Display for Entry {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(
+            f,
+            "{}  {} {} {:?} {} {}:{}:{}:<{}> {}",
+            self.ts_format(),
+            self.pid,
+            self.thread,
+            self.level,
+            self.category,
+            self.file,
+            self.line,
+            self.function,
+            self.object.clone().unwrap_or("".to_string()),
+            self.message
+        )
+    }
+}
+
+pub struct ParserIterator<R: Read> {
+    lines: Lines<BufReader<R>>,
+}
+
+impl<R: Read> ParserIterator<R> {
+    fn new(lines: Lines<BufReader<R>>) -> Self {
+        Self { lines: lines }
+    }
+}
+
+impl<R: Read> Iterator for ParserIterator<R> {
+    type Item = Entry;
+
+    fn next(&mut self) -> Option<Entry> {
+        match self.lines.next() {
+            None => None,
+            Some(line) => Some(Entry::new(line.unwrap())),
+        }
+    }
+}
+
+pub fn parse<R: Read>(r: R) -> ParserIterator<R> {
+    let file = BufReader::new(r);
+
+    ParserIterator::new(file.lines())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::fs::File;
+
+    #[test]
+    fn no_color() {
+        let f = File::open("test-logs/nocolor.log").expect("Failed to open log file");
+        let mut parsed = parse(f);
+
+        let entry = parsed.next().expect("First entry missing");
+        assert_eq!(entry.ts, 7773544);
+        assert_eq!(entry.ts_format(), "0:00:00.007773544");
+        assert_eq!(entry.pid, 8874);
+        assert_eq!(entry.thread, "0x558951015c00");
+        assert_eq!(entry.level, DebugLevel::Info);
+        assert_eq!(entry.category, "GST_INIT");
+        assert_eq!(entry.file, "gst.c");
+        assert_eq!(entry.line, 510);
+        assert_eq!(entry.function, "init_pre");
+        assert_eq!(
+            entry.message,
+            "Initializing GStreamer Core Library version 1.10.4"
+        );
+
+        let entry = parsed.nth(3).expect("3th entry missing");
+        assert_eq!(entry.message, "0x55895101d040 ref 1->2");
+        assert_eq!(entry.object, Some("allocatorsysmem0".to_string()));
+    }
+
+    #[test]
+    fn color() {
+        let f = File::open("test-logs/color.log").expect("Failed to open log file");
+        let mut parsed = parse(f);
+
+        let entry = parsed.next().expect("First entry missing");
+        assert_eq!(entry.ts, 208614);
+        assert_eq!(entry.ts_format(), "0:00:00.000208614");
+        assert_eq!(entry.pid, 17267);
+        assert_eq!(entry.thread, "0x2192200");
+        assert_eq!(entry.level, DebugLevel::Info);
+        assert_eq!(entry.category, "GST_INIT");
+        assert_eq!(entry.file, "gst.c");
+        assert_eq!(entry.line, 584);
+        assert_eq!(entry.function, "init_pre");
+        assert_eq!(
+            entry.message,
+            "Initializing GStreamer Core Library version 1.13.0.1"
+        );
+
+        assert_eq!(parsed.count(), 13);
+    }
+}

test-logs/color.log

@@ -0,0 +1,14 @@
+0:00:00.000208614 17267      0x2192200 INFO                GST_INIT gst.c:584:init_pre: Initializing GStreamer Core Library version 1.13.0.1
+0:00:00.000303732 17267      0x2192200 INFO                GST_INIT gst.c:585:init_pre: Using library installed in /usr/local/lib64
+0:00:00.000325904 17267      0x2192200 INFO                GST_INIT gst.c:605:init_pre: Linux cass-x230 4.12.14-200.fc25.x86_64 #1 SMP Wed Sep 20 16:40:50 UTC 2017 x86_64
+0:00:00.000462552 17267      0x2192200 DEBUG             GST_MEMORY gstallocator.c:593:_priv_gst_allocator_initialize: memory alignment: 7
+0:00:00.000613771 17267      0x2192200 DEBUG             GST_MEMORY gstallocator.c:569:gst_allocator_sysmem_init: init allocator 0x2199040
+0:00:00.000660471 17267      0x2192200 DEBUG             GST_MEMORY gstallocator.c:211:gst_allocator_register: registering allocator 0x2199040 with name "SystemMemory"
+0:00:00.000752877 17267      0x2192200 INFO                GST_INIT gstmessage.c:127:_priv_gst_message_initialize: init messages
+0:00:00.000869151 17267      0x2192200 DEBUG       GST_ELEMENT_PADS gstelement.c:302:gst_element_base_class_init: type GstElement : factory (nil)
+0:00:00.000898419 17267      0x2192200 DEBUG                default gstelement.c:195:gst_element_setup_thread_pool: creating element thread pool
+0:00:00.000949407 17267      0x2192200 DEBUG       GST_ELEMENT_PADS gstelement.c:302:gst_element_base_class_init: type GstBin : factory (nil)
+0:00:00.001369765 17267      0x2192200 INFO                GST_INIT gstcontext.c:84:_priv_gst_context_initialize: init contexts
+0:00:00.001548143 17267      0x2192200 INFO      GST_PLUGIN_LOADING gstplugin.c:317:_priv_gst_plugin_initialize: registering 0 static plugins
+0:00:00.001564433 17267      0x2192200 LOG       GST_PLUGIN_LOADING gstplugin.c:222:gst_plugin_register_static: attempting to load static plugin "staticelements" now...
+0:00:00.001598282 17267      0x2192200 LOG       GST_PLUGIN_LOADING gstplugin.c:506:gst_plugin_register_func: plugin "(NULL)" looks good

test-logs/nocolor.log

@@ -0,0 +1,6 @@
+0:00:00.007773544  8874 0x558951015c00 INFO                GST_INIT gst.c:510:init_pre: Initializing GStreamer Core Library version 1.10.4
+0:00:01.007927372  8874 0x558951015c00 DEBUG             GST_MEMORY gstallocator.c:592:_priv_gst_allocator_initialize: memory alignment: 7
+0:00:23.008032206  8874 0x558951015c00 TRACE        GST_REFCOUNTING gstobject.c:220:gst_object_init:<GstObject@0x55895101d040> 0x55895101d040 new
+0:01:10.008043265  8874 0x558951015c00 DEBUG             GST_MEMORY gstallocator.c:568:gst_allocator_sysmem_init: init allocator 0x55895101d040
+0:11:20.008067915  8874 0x558951015c00 TRACE        GST_REFCOUNTING gstobject.c:249:gst_object_ref:<allocatorsysmem0> 0x55895101d040 ref 1->2
+1:30:47.008078203  8874 0x558951015c00 DEBUG             GST_MEMORY gstallocator.c:210:gst_allocator_register: registering allocator 0x55895101d040 with name "SystemMemory"