diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a9d37c5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +target +Cargo.lock diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..8bdaad8 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "gst-log-parser" +version = "0.1.0" +authors = ["Guillaume Desmottes "] + +[dependencies] +itertools = "0.5.9" +gstreamer = "0.8.1" +regex = "0.2" +lazy_static = "0.2.9" diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..4b7cafa --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,240 @@ +use std::io::BufReader; +use std::io::BufRead; +use std::io::Read; +use std::io::Lines; +use std::fmt; +use std::str; +extern crate itertools; +use itertools::join; + +extern crate gstreamer as gst; +use gst::{DebugLevel, SECOND}; + +#[macro_use] +extern crate lazy_static; +extern crate regex; +use regex::Regex; + +#[derive(Debug)] +pub struct ParsingError; + +pub struct Entry { + pub ts: u64, + pub pid: u32, + pub thread: String, + pub level: DebugLevel, + pub category: String, + pub file: String, + pub line: u32, + pub function: String, + pub message: String, + pub object: Option, +} + +fn parse_debug_level(s: &str) -> Result { + match s { + "ERROR" => Ok(DebugLevel::Error), + "WARNING" => Ok(DebugLevel::Warning), + "FIXME" => Ok(DebugLevel::Fixme), + "INFO" => Ok(DebugLevel::Info), + "DEBUG" => Ok(DebugLevel::Debug), + "LOG" => Ok(DebugLevel::Log), + "TRACE" => Ok(DebugLevel::Trace), + "MEMDUMP" => Ok(DebugLevel::Memdump), + _ => Err(ParsingError), + } +} + +fn parse_time(ts: &str) -> u64 { + let mut split = ts.splitn(3, ':'); + let h: u64 = split.next().expect("missing hour").parse().expect( + "invalid hour", + ); + let m: u64 = split.next().expect("missing minute").parse().expect( + "invalid minute", + ); + split = split.next().expect("missing second").splitn(2, '.'); + let secs: u64 = split.next().expect("missing second").parse().expect( + "invalid second", + ); + let subsecs: u64 = split.next().expect("missing sub second").parse().expect( + "invalid sub second", + ); + + (h * 60 * 60 + m * 60) * SECOND + secs * SECOND + subsecs +} + +fn split_location(location: &str) -> (String, u32, String, Option) { + let mut split = location.splitn(4, ":"); + let file = split.next().expect("missing file"); + let line = split.next().expect("missing fine").parse().expect( + "invalid line", + ); + let function = split.next().expect("missing function"); + let object = split.next().expect("missing object delimiter"); + let object_name; + if object.len() > 0 { + let object = object + .to_string() + .trim_left_matches("<") + .trim_right_matches(">") + .to_string(); + object_name = Some(object); + } else { + object_name = None; + } + + (file.to_string(), line, function.to_string(), object_name) +} + +impl Entry { + fn new(line: String) -> Entry { + // Strip color codes + lazy_static! { + static ref RE: Regex = Regex::new("\x1b\\[[0-9;]*m").unwrap(); + } + let line = RE.replace_all(&line, ""); + + let mut it = line.split(" "); + let ts = parse_time(it.next().expect("Missing ts")); + let mut it = it.skip_while(|x| x.is_empty()); + let pid = it.next().expect("Missing PID").parse().expect( + "Failed to parse PID", + ); + let mut it = it.skip_while(|x| x.is_empty()); + let thread = it.next().expect("Missing thread").to_string(); + let mut it = it.skip_while(|x| x.is_empty()); + let level = parse_debug_level(it.next().expect("Missing level")).expect("Invalid level"); + let mut it = it.skip_while(|x| x.is_empty()); + let category = it.next().expect("Missing Category").to_string(); + let mut it = it.skip_while(|x| x.is_empty()); + let (file, line, function, object) = split_location(it.next().expect("Missing location")); + let message: String = join(it, " "); + + Entry { + ts: ts, + pid: pid, + thread: thread, + level: level, + category: category, + file: file, + line: line, + function: function, + object: object, + message: message, + } + } + + pub fn ts_format(&self) -> String { + let secs: u64; + + secs = self.ts / SECOND; + format!( + "{}:{:02}:{:02}.{:09}", + secs / (60 * 60), + (secs / 60) % 60, + secs % 60, + self.ts % SECOND + ) + } +} + +impl fmt::Display for Entry { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!( + f, + "{} {} {} {:?} {} {}:{}:{}:<{}> {}", + self.ts_format(), + self.pid, + self.thread, + self.level, + self.category, + self.file, + self.line, + self.function, + self.object.clone().unwrap_or("".to_string()), + self.message + ) + } +} + +pub struct ParserIterator { + lines: Lines>, +} + +impl ParserIterator { + fn new(lines: Lines>) -> Self { + Self { lines: lines } + } +} + +impl Iterator for ParserIterator { + type Item = Entry; + + fn next(&mut self) -> Option { + match self.lines.next() { + None => None, + Some(line) => Some(Entry::new(line.unwrap())), + } + } +} + +pub fn parse(r: R) -> ParserIterator { + let file = BufReader::new(r); + + ParserIterator::new(file.lines()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs::File; + + #[test] + fn no_color() { + let f = File::open("test-logs/nocolor.log").expect("Failed to open log file"); + let mut parsed = parse(f); + + let entry = parsed.next().expect("First entry missing"); + assert_eq!(entry.ts, 7773544); + assert_eq!(entry.ts_format(), "0:00:00.007773544"); + assert_eq!(entry.pid, 8874); + assert_eq!(entry.thread, "0x558951015c00"); + assert_eq!(entry.level, DebugLevel::Info); + assert_eq!(entry.category, "GST_INIT"); + assert_eq!(entry.file, "gst.c"); + assert_eq!(entry.line, 510); + assert_eq!(entry.function, "init_pre"); + assert_eq!( + entry.message, + "Initializing GStreamer Core Library version 1.10.4" + ); + + let entry = parsed.nth(3).expect("3th entry missing"); + assert_eq!(entry.message, "0x55895101d040 ref 1->2"); + assert_eq!(entry.object, Some("allocatorsysmem0".to_string())); + } + + #[test] + fn color() { + let f = File::open("test-logs/color.log").expect("Failed to open log file"); + let mut parsed = parse(f); + + let entry = parsed.next().expect("First entry missing"); + assert_eq!(entry.ts, 208614); + assert_eq!(entry.ts_format(), "0:00:00.000208614"); + assert_eq!(entry.pid, 17267); + assert_eq!(entry.thread, "0x2192200"); + assert_eq!(entry.level, DebugLevel::Info); + assert_eq!(entry.category, "GST_INIT"); + assert_eq!(entry.file, "gst.c"); + assert_eq!(entry.line, 584); + assert_eq!(entry.function, "init_pre"); + assert_eq!( + entry.message, + "Initializing GStreamer Core Library version 1.13.0.1" + ); + + assert_eq!(parsed.count(), 13); + } +} diff --git a/test-logs/color.log b/test-logs/color.log new file mode 100644 index 0000000..b7bc0dd --- /dev/null +++ b/test-logs/color.log @@ -0,0 +1,14 @@ +0:00:00.000208614 17267 0x2192200 INFO   GST_INIT gst.c:584:init_pre: Initializing GStreamer Core Library version 1.13.0.1 +0:00:00.000303732 17267 0x2192200 INFO   GST_INIT gst.c:585:init_pre: Using library installed in /usr/local/lib64 +0:00:00.000325904 17267 0x2192200 INFO   GST_INIT gst.c:605:init_pre: Linux cass-x230 4.12.14-200.fc25.x86_64 #1 SMP Wed Sep 20 16:40:50 UTC 2017 x86_64 +0:00:00.000462552 17267 0x2192200 DEBUG   GST_MEMORY gstallocator.c:593:_priv_gst_allocator_initialize: memory alignment: 7 +0:00:00.000613771 17267 0x2192200 DEBUG   GST_MEMORY gstallocator.c:569:gst_allocator_sysmem_init: init allocator 0x2199040 +0:00:00.000660471 17267 0x2192200 DEBUG   GST_MEMORY gstallocator.c:211:gst_allocator_register: registering allocator 0x2199040 with name "SystemMemory" +0:00:00.000752877 17267 0x2192200 INFO   GST_INIT gstmessage.c:127:_priv_gst_message_initialize: init messages +0:00:00.000869151 17267 0x2192200 DEBUG   GST_ELEMENT_PADS gstelement.c:302:gst_element_base_class_init: type GstElement : factory (nil) +0:00:00.000898419 17267 0x2192200 DEBUG   default gstelement.c:195:gst_element_setup_thread_pool: creating element thread pool +0:00:00.000949407 17267 0x2192200 DEBUG   GST_ELEMENT_PADS gstelement.c:302:gst_element_base_class_init: type GstBin : factory (nil) +0:00:00.001369765 17267 0x2192200 INFO   GST_INIT gstcontext.c:84:_priv_gst_context_initialize: init contexts +0:00:00.001548143 17267 0x2192200 INFO   GST_PLUGIN_LOADING gstplugin.c:317:_priv_gst_plugin_initialize: registering 0 static plugins +0:00:00.001564433 17267 0x2192200 LOG   GST_PLUGIN_LOADING gstplugin.c:222:gst_plugin_register_static: attempting to load static plugin "staticelements" now... +0:00:00.001598282 17267 0x2192200 LOG   GST_PLUGIN_LOADING gstplugin.c:506:gst_plugin_register_func: plugin "(NULL)" looks good diff --git a/test-logs/nocolor.log b/test-logs/nocolor.log new file mode 100644 index 0000000..e3b9164 --- /dev/null +++ b/test-logs/nocolor.log @@ -0,0 +1,6 @@ +0:00:00.007773544 8874 0x558951015c00 INFO GST_INIT gst.c:510:init_pre: Initializing GStreamer Core Library version 1.10.4 +0:00:01.007927372 8874 0x558951015c00 DEBUG GST_MEMORY gstallocator.c:592:_priv_gst_allocator_initialize: memory alignment: 7 +0:00:23.008032206 8874 0x558951015c00 TRACE GST_REFCOUNTING gstobject.c:220:gst_object_init: 0x55895101d040 new +0:01:10.008043265 8874 0x558951015c00 DEBUG GST_MEMORY gstallocator.c:568:gst_allocator_sysmem_init: init allocator 0x55895101d040 +0:11:20.008067915 8874 0x558951015c00 TRACE GST_REFCOUNTING gstobject.c:249:gst_object_ref: 0x55895101d040 ref 1->2 +1:30:47.008078203 8874 0x558951015c00 DEBUG GST_MEMORY gstallocator.c:210:gst_allocator_register: registering allocator 0x55895101d040 with name "SystemMemory"