logstash.conf

input {
  	stdin {}
}

filter {
	csv {
		columns => ["id","case_number","date","block","iucr","primary_type","description","location_description","arrest","domestic","beat","district","ward","community_area","fbi_code","x_coord","y_coord","year","updated_on","latitude","longitude","location"]

	}

  # Perform lookup against mapping file to get community names
  translate {
    field => "community_area"
    dictionary_path => "/Users/peter/source/chicago_crime_2/community_areas.yml"
    destination => "community_area_name"
  }

  # Perform lookup against mapping file to get names of current aldermen
  translate {
    field => "ward"
    dictionary_path => "/Users/peter/source/chicago_crime_2/wards.yml"
    destination => "ward_alderman"
  }

	if [latitude] and [longitude] {
    mutate {
      add_field => {
        "coords" => ["%{longitude}", "%{latitude}"]
      }
    }

    mutate {
      convert => [ "coords", "float" ]
    }
	}


  grok {
    match => { "date" => "%{DATE_US:date_part} %{HOUR:hour_part}:%{MINUTE:min_part}:%{SECOND:sec_part} (?<ampm_part>AM|PM)" }
  }

	date {
		match => [ "date", "MM/dd/YYYY hh:mm:ss aa"]
		timezone => "America/Chicago"
	}

  # adjust for invalid times during DST transition
  if "_dateparsefailure" in [tags] and [hour_part] == "02" and [ampm_part] == "AM" {
    mutate {
      replace => ["hour_part", "03"]
    }

    mutate {
      replace => ["date", "%{date_part} %{hour_part}:%{min_part}:%{sec_part} %{ampm_part}"]
    }

    mutate {
      remove_field => ["tags"]
    }

    date {
  		match => [ "date", "MM/dd/YYYY hh:mm:ss aa"]
  		timezone => "America/Chicago"
  	}
  }

  # remove unnecessary fields
	mutate {
		remove_field => ["latitude", "longitude", "location", "message", "hour_part", "date_part", "min_part", "sec_part", "ampm_part", "x_coord", "y_coord"]
	}

}


output {
	# stdout { codec => rubydebug }
	stdout { codec => dots }
	elasticsearch {
		protocol => http
		host => "localhost"
		index => "chicago_crime"
    document_type => "crime"
    template => "index_template.json"
    template_name => "chicrime"
    template_overwrite => true
	}
}