From 8d3480aa9b728df5f5aadb8f9a9df4b82b47231c Mon Sep 17 00:00:00 2001 From: Martin Kacer Date: Fri, 16 Apr 2021 09:54:53 +0200 Subject: [PATCH] Updated to support ELK 7.12 and tshark 3.2.3 --- Kibana/template_tshark_mapping.json | 17 +- .../template_tshark_mapping_deduplicated.json | 371 +++++++++--------- Kibana/template_tshark_mapping_dynamic.json | 18 + Public/process_tshark_mapping_json.rb | 25 +- README.md | 32 +- Traces/dhcp-and-dyndns.pcap.gz | Bin 0 -> 2530 bytes VM/post_initialize.sh | 2 +- build.sh | 4 + 8 files changed, 250 insertions(+), 219 deletions(-) create mode 100644 Kibana/template_tshark_mapping_dynamic.json create mode 100644 Traces/dhcp-and-dyndns.pcap.gz diff --git a/Kibana/template_tshark_mapping.json b/Kibana/template_tshark_mapping.json index 5e421c4..6ead0d8 100644 --- a/Kibana/template_tshark_mapping.json +++ b/Kibana/template_tshark_mapping.json @@ -548,15 +548,6 @@ "dhcp_dhcp_option_vendor_pxeclient_end": { "type": "short" }, - "dhcp_dhcp_option_vendor_cisco_suboption": { - "type": "short" - }, - "dhcp_dhcp_option_vendor_aerohive_unknown": { - "type": "byte" - }, - "dhcp_dhcp_option_vendor_aerohive_xiqipaddress": { - "type": "ip" - }, "dhcp_dhcp_option_vendor_cl_suboption": { "type": "short" }, @@ -1485,7 +1476,7 @@ "type": "float" }, "ip_ip_flags": { - "type": "short" + "type": "integer" }, "ip_ip_flags_sf": { "type": "boolean" @@ -1977,9 +1968,6 @@ "tcp_tcp_options_mptcp_ipver": { "type": "short" }, - "tcp_tcp_options_mptcp_echo": { - "type": "short" - }, "tcp_tcp_options_mptcp_ipv4": { "type": "ip" }, @@ -2226,9 +2214,6 @@ }, "udp_udp_time_delta": { "type": "date" - }, - "udp_udp_payload": { - "type": "byte" } } } diff --git a/Kibana/template_tshark_mapping_deduplicated.json b/Kibana/template_tshark_mapping_deduplicated.json index f2bd27d..d5c9eae 100644 --- a/Kibana/template_tshark_mapping_deduplicated.json +++ b/Kibana/template_tshark_mapping_deduplicated.json @@ -1,13 +1,12 @@ { "index_patterns": "packets-*", - "settings": { - "index.mapping.total_fields.limit": 1000000, - "index.mapping.ignore_malformed": true, - "index.mapping.coerce": true - }, - "mappings": { - "doc": { - "dynamic": true, + "template": { + "settings": { + "index.mapping.total_fields.limit": 1000000, + "index.mapping.ignore_malformed": true, + "index.mapping.coerce": true + }, + "mappings": { "properties": { "timestamp": { "type": "date" @@ -17,7 +16,7 @@ "dhcp": { "properties": { "dhcp_dhcp_bootp": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_type": { "type": "short" @@ -41,7 +40,7 @@ "type": "integer" }, "dhcp_dhcp_flags_bc": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_flags_reserved": { "type": "integer" @@ -71,16 +70,16 @@ "type": "byte" }, "dhcp_dhcp_fqdn_s": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_fqdn_o": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_fqdn_e": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_fqdn_n": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_fqdn_flags": { "type": "short" @@ -101,64 +100,64 @@ "type": "short" }, "dhcp_dhcp_vendor_pktc_mdc_supp_flow_secure": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_vendor_pktc_mdc_supp_flow_hybrid": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_vendor_pktc_mdc_supp_flow_basic": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_vendor_pktc_mdc_cl_mib_mta": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_vendor_pktc_mdc_cl_mib_signaling": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_vendor_pktc_mdc_cl_mib_management_event": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_vendor_pktc_mdc_cl_mib_mta_extension": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_vendor_pktc_mdc_cl_mib_signaling_extension": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_vendor_pktc_mdc_cl_mib_mem_extention": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_vendor_pktc_mdc_cl_mib_reserved": { "type": "short" }, "dhcp_dhcp_vendor_pktc_mdc_ietf_mib_mta": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_vendor_pktc_mdc_ietf_mib_signaling": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_vendor_pktc_mdc_ietf_mib_management_event": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_vendor_pktc_mdc_ietf_mib_reserved": { "type": "short" }, "dhcp_dhcp_vendor_pktc_mdc_euro_mib_mta": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_vendor_pktc_mdc_euro_mib_signaling": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_vendor_pktc_mdc_euro_mib_management_event": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_vendor_pktc_mdc_euro_mib_mta_extension": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_vendor_pktc_mdc_euro_mib_signaling_extension": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_vendor_pktc_mdc_euro_mib_mem_extention": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_vendor_pktc_mdc_euro_mib_reserved": { "type": "short" @@ -170,109 +169,109 @@ "type": "integer" }, "dhcp_dhcp_docsis_cm_cap_ranging_hold_off_cm": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_docsis_cm_cap_ranging_hold_off_eps": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_docsis_cm_cap_ranging_hold_off_emta": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_docsis_cm_cap_ranging_hold_off_dsg": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_docsis_cm_cap_mpls_stpid": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_docsis_cm_cap_mpls_svid": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_docsis_cm_cap_mpls_spcp": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_docsis_cm_cap_mpls_sdei": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_docsis_cm_cap_mpls_ctpid": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_docsis_cm_cap_mpls_cvid": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_docsis_cm_cap_mpls_cpcp": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_docsis_cm_cap_mpls_ccfi": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_docsis_cm_cap_mpls_stci": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_docsis_cm_cap_mpls_ctci": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_docsis_cm_cap_mpls_itpid": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_docsis_cm_cap_mpls_isid": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_docsis_cm_cap_mpls_itci": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_docsis_cm_cap_mpls_ipcp": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_docsis_cm_cap_mpls_idei": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_docsis_cm_cap_mpls_iuca": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_docsis_cm_cap_mpls_btpid": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_docsis_cm_cap_mpls_btci": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_docsis_cm_cap_mpls_bpcp": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_docsis_cm_cap_mpls_bdei": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_docsis_cm_cap_mpls_bvid": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_docsis_cm_cap_mpls_bda": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_docsis_cm_cap_mpls_bsa": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_docsis_cm_cap_mpls_tc": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_docsis_cm_cap_mpls_label": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_docsis_cm_cap_ussymrate_160": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_docsis_cm_cap_ussymrate_320": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_docsis_cm_cap_ussymrate_640": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_docsis_cm_cap_ussymrate_1280": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_docsis_cm_cap_ussymrate_2560": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_docsis_cm_cap_ussymrate_5120": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_client_id_duid_type": { "type": "integer" @@ -317,7 +316,7 @@ "type": "ip" }, "dhcp_dhcp_option_value_bool": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_option_padding": { "type": "byte" @@ -362,7 +361,7 @@ "type": "ip" }, "dhcp_dhcp_option_ip_forwarding": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_option_policy_filter_ip": { "type": "ip" @@ -371,7 +370,7 @@ "type": "ip" }, "dhcp_dhcp_option_non_local_source_routing": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_option_max_datagram_reassembly_size": { "type": "integer" @@ -389,19 +388,19 @@ "type": "integer" }, "dhcp_dhcp_option_all_subnets_are_local": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_option_broadcast_address": { "type": "ip" }, "dhcp_dhcp_option_perform_mask_discovery": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_option_mask_supplier": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_option_perform_router_discover": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_option_router_solicitation_address": { "type": "ip" @@ -413,13 +412,13 @@ "type": "ip" }, "dhcp_dhcp_option_trailer_encapsulation": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_option_arp_cache_timeout": { "type": "long" }, "dhcp_dhcp_option_ethernet_encapsulation": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_option_tcp_default_ttl": { "type": "short" @@ -428,7 +427,7 @@ "type": "long" }, "dhcp_dhcp_option_tcp_keepalive_garbage": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_option_nis_server": { "type": "ip" @@ -470,16 +469,16 @@ "type": "short" }, "dhcp_dhcp_option_vendor_pxeclient_discovery_control_broadcast": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_option_vendor_pxeclient_discovery_control_multicast": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_option_vendor_pxeclient_discovery_control_serverlist": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_option_vendor_pxeclient_discovery_control_bstrap": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_option_vendor_pxeclient_multicast_address": { "type": "ip" @@ -530,10 +529,10 @@ "type": "byte" }, "dhcp_dhcp_option_vendor_pxeclient_discovery": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_option_vendor_pxeclient_configured": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_option_vendor_pxeclient_lcm_version": { "type": "long" @@ -541,15 +540,6 @@ "dhcp_dhcp_option_vendor_pxeclient_end": { "type": "short" }, - "dhcp_dhcp_option_vendor_cisco_suboption": { - "type": "short" - }, - "dhcp_dhcp_option_vendor_aerohive_unknown": { - "type": "byte" - }, - "dhcp_dhcp_option_vendor_aerohive_xiqipaddress": { - "type": "ip" - }, "dhcp_dhcp_option_vendor_cl_suboption": { "type": "short" }, @@ -704,10 +694,10 @@ "type": "ip" }, "dhcp_dhcp_option_novell_options_value_bool": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_option_novell_options_broadcast": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_option_novell_options_preferred_dss_server": { "type": "ip" @@ -722,7 +712,7 @@ "type": "short" }, "dhcp_dhcp_option_novell_options_support_netware_v1_1": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_option_novell_options_primary_dss": { "type": "ip" @@ -866,13 +856,13 @@ "type": "integer" }, "dhcp_dhcp_option_isns_functions_enabled": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_option_isns_functions_dd_base_authorization": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_option_isns_functions_sec_policy_distribution": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_option_isns_functions_reserved": { "type": "integer" @@ -881,22 +871,22 @@ "type": "integer" }, "dhcp_dhcp_option_isns_discovery_domain_access_enabled": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_option_isns_discovery_domain_access_control_node": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_option_isns_discovery_domain_access_iscsi_target": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_option_isns_discovery_domain_access_iscsi_initiator": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_option_isns_discovery_domain_access_ifcp_target_port": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_option_isns_discovery_domain_access_initiator_target_port": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_option_isns_discovery_domain_access_reserved": { "type": "integer" @@ -905,16 +895,16 @@ "type": "integer" }, "dhcp_dhcp_option_isns_administrative_flags_enabled": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_option_isns_administrative_flags_heartbeat": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_option_isns_administrative_flags_management_scns": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_option_isns_administrative_flags_default_discovery_domain": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_option_isns_administrative_flags_reserved": { "type": "integer" @@ -923,25 +913,25 @@ "type": "long" }, "dhcp_dhcp_option_isns_server_security_bitmap_enabled": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_option_isns_server_security_bitmap_ike_ipsec_enabled": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_option_isns_server_security_bitmap_main_mode": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_option_isns_server_security_bitmap_aggressive_mode": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_option_isns_server_security_bitmap_pfs": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_option_isns_server_security_bitmap_transport_mode": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_option_isns_server_security_bitmap_tunnel_mode": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_option_isns_server_security_bitmap_reserved": { "type": "integer" @@ -1139,7 +1129,7 @@ "type": "short" }, "dhcp_dhcp_option_bulk_lease_data_source": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_option_pcp_list_length": { "type": "short" @@ -1202,10 +1192,10 @@ "type": "long" }, "dhcp_dhcp_ccc_ietf_sec_tkt_pc_provision_server": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_ccc_ietf_sec_tkt_all_pc_call_management": { - "type": "boolean" + "type": "short" }, "dhcp_dhcp_option_vendor_avaya_l2qvlan": { "type": "integer" @@ -1213,6 +1203,9 @@ "dhcp_dhcp_option_vendor_avaya_vlantest": { "type": "integer" }, + "dhcp_dhcp_option_vendor_cisco_suboption": { + "type": "short" + }, "dhcp_dhcp_option_vendor_cisco_unknown": { "type": "byte" }, @@ -1272,22 +1265,22 @@ "type": "short" }, "eth_eth_dst_lg": { - "type": "boolean" + "type": "short" }, "eth_eth_dst_ig": { - "type": "boolean" + "type": "short" }, "eth_eth_src_lg": { - "type": "boolean" + "type": "short" }, "eth_eth_src_ig": { - "type": "boolean" + "type": "short" }, "eth_eth_lg": { - "type": "boolean" + "type": "short" }, "eth_eth_ig": { - "type": "boolean" + "type": "short" } } }, @@ -1330,10 +1323,10 @@ "type": "long" }, "frame_frame_marked": { - "type": "boolean" + "type": "short" }, "frame_frame_ignored": { - "type": "boolean" + "type": "short" }, "frame_frame_interface_id": { "type": "long" @@ -1354,28 +1347,28 @@ "type": "long" }, "frame_frame_packet_flags_crc_error": { - "type": "boolean" + "type": "short" }, "frame_frame_packet_flags_packet_too_error": { - "type": "boolean" + "type": "short" }, "frame_frame_packet_flags_packet_too_short_error": { - "type": "boolean" + "type": "short" }, "frame_frame_packet_flags_wrong_inter_frame_gap_error": { - "type": "boolean" + "type": "short" }, "frame_frame_packet_flags_unaligned_frame_error": { - "type": "boolean" + "type": "short" }, "frame_frame_packet_flags_start_frame_delimiter_error": { - "type": "boolean" + "type": "short" }, "frame_frame_packet_flags_preamble_error": { - "type": "boolean" + "type": "short" }, "frame_frame_packet_flags_symbol_error": { - "type": "boolean" + "type": "short" }, "frame_frame_encap_type": { "type": "integer" @@ -1406,16 +1399,16 @@ "type": "short" }, "ip_ip_tos_delay": { - "type": "boolean" + "type": "short" }, "ip_ip_tos_throughput": { - "type": "boolean" + "type": "short" }, "ip_ip_tos_reliability": { - "type": "boolean" + "type": "short" }, "ip_ip_tos_cost": { - "type": "boolean" + "type": "short" }, "ip_ip_len": { "type": "integer" @@ -1460,19 +1453,19 @@ "type": "float" }, "ip_ip_flags": { - "type": "short" + "type": "integer" }, "ip_ip_flags_sf": { - "type": "boolean" + "type": "short" }, "ip_ip_flags_rb": { - "type": "boolean" + "type": "short" }, "ip_ip_flags_df": { - "type": "boolean" + "type": "short" }, "ip_ip_flags_mf": { - "type": "boolean" + "type": "short" }, "ip_ip_frag_offset": { "type": "integer" @@ -1496,7 +1489,7 @@ "type": "short" }, "ip_ip_opt_type_copy": { - "type": "boolean" + "type": "short" }, "ip_ip_opt_type_class": { "type": "short" @@ -1571,25 +1564,25 @@ "type": "short" }, "ip_ip_opt_sec_prot_auth_genser": { - "type": "boolean" + "type": "short" }, "ip_ip_opt_sec_prot_auth_siop_esi": { - "type": "boolean" + "type": "short" }, "ip_ip_opt_sec_prot_auth_sci": { - "type": "boolean" + "type": "short" }, "ip_ip_opt_sec_prot_auth_nsa": { - "type": "boolean" + "type": "short" }, "ip_ip_opt_sec_prot_auth_doe": { - "type": "boolean" + "type": "short" }, "ip_ip_opt_sec_prot_auth_unassigned": { "type": "short" }, "ip_ip_opt_sec_prot_auth_fti": { - "type": "boolean" + "type": "short" }, "ip_ip_opt_ext_sec_add_sec_info_format_code": { "type": "short" @@ -1613,16 +1606,16 @@ "type": "short" }, "ip_ip_fragment_overlap": { - "type": "boolean" + "type": "short" }, "ip_ip_fragment_overlap_conflict": { - "type": "boolean" + "type": "short" }, "ip_ip_fragment_multipletails": { - "type": "boolean" + "type": "short" }, "ip_ip_fragment_toolongfragment": { - "type": "boolean" + "type": "short" }, "ip_ip_fragment_error": { "type": "long" @@ -1704,34 +1697,34 @@ "type": "integer" }, "tcp_tcp_flags_res": { - "type": "boolean" + "type": "short" }, "tcp_tcp_flags_ns": { - "type": "boolean" + "type": "short" }, "tcp_tcp_flags_cwr": { - "type": "boolean" + "type": "short" }, "tcp_tcp_flags_ecn": { - "type": "boolean" + "type": "short" }, "tcp_tcp_flags_urg": { - "type": "boolean" + "type": "short" }, "tcp_tcp_flags_ack": { - "type": "boolean" + "type": "short" }, "tcp_tcp_flags_push": { - "type": "boolean" + "type": "short" }, "tcp_tcp_flags_reset": { - "type": "boolean" + "type": "short" }, "tcp_tcp_flags_syn": { - "type": "boolean" + "type": "short" }, "tcp_tcp_flags_fin": { - "type": "boolean" + "type": "short" }, "tcp_tcp_window_size_value": { "type": "integer" @@ -1788,16 +1781,16 @@ "type": "integer" }, "tcp_tcp_segment_overlap": { - "type": "boolean" + "type": "short" }, "tcp_tcp_segment_overlap_conflict": { - "type": "boolean" + "type": "short" }, "tcp_tcp_segment_multipletails": { - "type": "boolean" + "type": "short" }, "tcp_tcp_segment_toolongfragment": { - "type": "boolean" + "type": "short" }, "tcp_tcp_segment_error": { "type": "long" @@ -1949,9 +1942,6 @@ "tcp_tcp_options_mptcp_ipver": { "type": "short" }, - "tcp_tcp_options_mptcp_echo": { - "type": "short" - }, "tcp_tcp_options_mptcp_ipv4": { "type": "ip" }, @@ -1995,19 +1985,19 @@ "type": "integer" }, "tcp_tcp_options_scpsflags_bets": { - "type": "boolean" + "type": "short" }, "tcp_tcp_options_scpsflags_snack1": { - "type": "boolean" + "type": "short" }, "tcp_tcp_options_scpsflags_snack2": { - "type": "boolean" + "type": "short" }, "tcp_tcp_options_scpsflags_compress": { - "type": "boolean" + "type": "short" }, "tcp_tcp_options_scpsflags_nlts": { - "type": "boolean" + "type": "short" }, "tcp_tcp_options_scpsflags_reserved": { "type": "short" @@ -2016,7 +2006,7 @@ "type": "short" }, "tcp_tcp_options_user_to_granularity": { - "type": "boolean" + "type": "short" }, "tcp_tcp_options_user_to_val": { "type": "integer" @@ -2055,40 +2045,40 @@ "type": "short" }, "tcp_tcp_options_rvbd_probe_flags_notcfe": { - "type": "boolean" + "type": "short" }, "tcp_tcp_options_rvbd_probe_flags_last": { - "type": "boolean" + "type": "short" }, "tcp_tcp_options_rvbd_probe_flags_probe": { - "type": "boolean" + "type": "short" }, "tcp_tcp_options_rvbd_probe_flags_ssl": { - "type": "boolean" + "type": "short" }, "tcp_tcp_options_rvbd_probe_flags_server": { - "type": "boolean" + "type": "short" }, "tcp_tcp_options_rvbd_trpy_flags": { "type": "integer" }, "tcp_tcp_options_rvbd_trpy_flags_fw_rst_probe": { - "type": "boolean" + "type": "short" }, "tcp_tcp_options_rvbd_trpy_flags_fw_rst_inner": { - "type": "boolean" + "type": "short" }, "tcp_tcp_options_rvbd_trpy_flags_fw_rst": { - "type": "boolean" + "type": "short" }, "tcp_tcp_options_rvbd_trpy_flags_chksum": { - "type": "boolean" + "type": "short" }, "tcp_tcp_options_rvbd_trpy_flags_oob": { - "type": "boolean" + "type": "short" }, "tcp_tcp_options_rvbd_trpy_flags_mode": { - "type": "boolean" + "type": "short" }, "tcp_tcp_options_rvbd_trpy_src_ip": { "type": "ip" @@ -2198,9 +2188,6 @@ }, "udp_udp_time_delta": { "type": "date" - }, - "udp_udp_payload": { - "type": "byte" } } } @@ -2209,4 +2196,4 @@ } } } -} \ No newline at end of file +} diff --git a/Kibana/template_tshark_mapping_dynamic.json b/Kibana/template_tshark_mapping_dynamic.json new file mode 100644 index 0000000..a63d290 --- /dev/null +++ b/Kibana/template_tshark_mapping_dynamic.json @@ -0,0 +1,18 @@ +{ + "index_patterns": "packets-*", + "template": { + "settings": { + "index.mapping.total_fields.limit": 1000000, + "index.mapping.ignore_malformed": true, + "index.mapping.coerce": true + }, + "mappings": { + "numeric_detection": true, + "properties": { + "timestamp": { + "type": "date" + } + } + } + } +} diff --git a/Public/process_tshark_mapping_json.rb b/Public/process_tshark_mapping_json.rb index a9cb76a..7b7a647 100644 --- a/Public/process_tshark_mapping_json.rb +++ b/Public/process_tshark_mapping_json.rb @@ -1,6 +1,6 @@ # # Created by Martin Kacer -# Copyright 2020 H21 lab, All right reserved, https://www.h21lab.com +# Copyright 2021 H21 lab, All right reserved, https://www.h21lab.com # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -22,7 +22,7 @@ # tshark -G elastic-mapping --elastic-mapping-filter frame,eth,ip,udp,tcp,dhcp > tshark_mapping_template.json # json contains duplicated values, deduplicate it by this ruby script -input = open("#{Dir.pwd}/Kibana/template_tshark_mapping.json") +input = open("#{Dir.pwd}/Kibana/custom_tshark_mapping.json") json = input.read parsed = JSON.parse(json) @@ -30,20 +30,27 @@ # Optionally replace date values and convert it to floats #replace_date(parsed) -# Optionally drop the whole mappings and just use dynamic -# This is here because the mapping for all protocols is causing Kibana to freeze during create index pattern -#parsed['mappings']['doc']['properties'].delete('layers') - +# Post-processing of template mapping generated by tshark to more recent Elasticseacrh version parsed['settings']['index.mapping.ignore_malformed'] = true parsed['settings']['index.mapping.coerce'] = true -parsed['mappings']['doc']['dynamic'] = true +parsed['mappings']['doc'].delete('dynamic') +parsed['mappings'] = parsed['mappings']['doc'] +parsed['mappings'].delete('doc') +parsed['template'] = {'settings' => parsed['settings'], 'mappings' => parsed['mappings']} +parsed.delete('settings') +parsed.delete('mappings') + # Optionally overwite various fields here if needed #parsed['mappings']['doc']['properties']['layers']['properties']['frame']['properties']['frame_frame_time'] = {"type"=>"text"} -output = File.open("#{Dir.pwd}/Kibana/template_tshark_mapping_deduplicated.json","w") +output = File.open("#{Dir.pwd}/Kibana/custom_tshark_mapping_deduplicated.json","w") json_string = JSON.pretty_generate(parsed) + +# Optional further post-processing +# change boolean to short json_string = json_string.gsub("\"type\": \"boolean\"", "\"type\": \"short\"") + output.write(json_string) @@ -58,4 +65,4 @@ def replace_date(h) end end end -end \ No newline at end of file +end diff --git a/README.md b/README.md index 43013ba..297d9ca 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,7 @@ tshark -r trace.pcapng -x -T ek > /dev/tcp/localhost/17570 ```bash firefox http://127.0.0.1:15601/app/kibana#/dashboards ``` +Open Main Dashboard and increase time window to e.g. last 100 years to see there the sample pcaps. ![](res/tshark_vm_dashboard.png?raw=true "Kibana Dashboard") ![](res/tshark_vm_discover.png?raw=true "Kibana Discover") @@ -72,7 +73,34 @@ sudo systemctl status elasticsearch.service sudo systemctl status logstash.service ``` +# Elasticsearch mapping template +In the project is included simple Elasticseacrh mapping template generated for the ``frame,eth,ip,udp,tcp,dhcp`` protocols. +To handle additional protocols efficiently it can be required to update the mapping template in the following way: + +``` +# 1. Create custom mapping, by selecting required protocols +tshark -G elastic-mapping --elastic-mapping-filter frame,eth,ip,udp,tcp,dns > ./Kibana/custom_tshark_mapping.json + +# 2. Deduplicate and post-process the mapping to fit current Elasticsearch version +ruby ./Public/process_tshark_mapping_json.rb + +# 3. Upload file to vagrant VM +cd VM +vagrant upload ../Kibana/custom_tshark_mapping_deduplicated.json /home/vagrant/tsharkVM/Kibana/custom_tshark_mapping_deduplicated.json +cd .. + +# 4. Connect to VM and upload template in the Elasticsearch +cd VM +vagrant ssh +cd tsharkVM/Kibana +curl -X PUT "localhost:9200/_index_template/packets_template" -H 'Content-Type: application/json' -d@custom_tshark_mapping_deduplicated.json +``` + +Alternative can be using the dynamic mapping. See template ``./Kibana/template_tshark_mapping_dynamic.json``. And consider setting the numeric_detection parameter true/false depending on the mapping requirements and pcaps used. Upload the template into Elasticsearch in similar way as described above. + ## Limitations +tshark -G elastic-mapping --elastic-mapping-filter mapping could be outdated, it is not following properly the Elasticsearch changes and the output can be duplicated. The manual configuration and post-processing of the mapping template is required. + Program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY. ## License @@ -80,6 +108,8 @@ The default license of source codes provided inside this project is the Apache L Additionally refer to individual licenses and terms of used of installed software (see licenses for Wireshark, Elastic and other). ## Attribution +Example pcap in ./Traces subfolder was downloaded from https://wiki.wireshark.org/SampleCaptures + Created by Martin Kacer -Copyright 2020 H21 lab, All right reserved, https://www.h21lab.com +Copyright 2021 H21 lab, All right reserved, https://www.h21lab.com diff --git a/Traces/dhcp-and-dyndns.pcap.gz b/Traces/dhcp-and-dyndns.pcap.gz new file mode 100644 index 0000000000000000000000000000000000000000..04cf939f7b5948fadb158030ae297548ad0ec7de GIT binary patch literal 2530 zcmV<82_5zyiwFp#7+paE17vw_L{3v*cWG{QaAaY0Wn^`7V`x)la&~2ME^uREZ~*0+ z3se;6702(lvk!z7FiJvds*tt{D7dR^_M9Y5^T0%FeSk@grN_3KCLWs{Yidk>NL|`xzCep>OH!1t9phE}_`G<_Q9GafhzGYysP=ccs4(0wRzM z@i0Kcs4J&kN8x8yu#jZ?(}ZZDC**fg;BR5a!`EghTa#aABBYj(R2^}No2m{SGiHo_ zguTq}D6d>;NKOgK%9jdaOi6|Z!bpUmot!jfeC+i}z2Nq;Y8!!>xINS3AI@%m=LgF z!bD@sx<{CK!ZF`I8npCpZUe-t%WKM)Xlm*zmo01`t+6{QE2`R!uHzF70O64^Vnze~ zDjidojw`vAjVVkadAxD^ASIt0!IeCPnIEf=Ji)jw!e{;nn zto|L7)xWf>lfHTNPf!#;m(#sUVD3o#ZvS4=Koybt{GIUt7s0jqB=b38Ii1-zDrCug z^!_(vJK|VEfeMHs%`m|{18LVoFZ8&^-ad{U0k9R=sBu5iz5kv2rx{I-dtjm5C2rW)u!qd`|um1uW3s@3-DWp)-Hd>;d&caId>C*rm%5Ye8{$-^3m_fIDY z7sC(!aK|-idiBdkL?^9@0+I-mx3#TyPnllWfUnH77H5mhmT1LQ6eTfR znqxIvMT^<JF< zN*mq_wbgWlJNw>eKB$xX;I(be?aG;KK@ZH4M9C^^sfqwZWo)P{o~nXBfRE81$bb{qP^Sv+pdUSIg-KkscQ?{TE1YKFN*y{}_F#BJ-Ye zzBK5}vkWx%DZTR)mwukntK{@QZgIXmXwRqR450mCq)!MFc=Qh#y)qcuDZQCb|B%s_ z1WWtR8o6`+BSwEPSlYYz^b3ss(_mFMJxEc2`qgN>EJ!+2&SUaVswsPs8F#2LS zeHUtv3s`$2(sy0d0=+?a+8_O2;-{3}`dL4{310`^Q(-5y!`(?e@9L!ZXS-XCFdKUF zv(moDX&*-cUUKv;7lghoJu&D9$p^&dkruDJV|M%PGni@lCXD6|?3Ul7Xt- z*rkPTJgnQF+E2xcOzppAYTwAc7Qb_k6MjeX(v!I4|;D%omsCkw4XozSuR~BEJ^ZzTC~7=a-n;OA56;sP<38 zP3<~*U+e{LY%eplT?(~dqT1cVP3@zowkSdeSSH!(qxL=VbEfv3Ozn}-a#Xv=t#G8gF%cy4t@!`2boMA;kIEm8;wrz&l%G{7ccAjU!&&(TdK+#m zSNS!j{ER|*87kj5oRv4B^7FA=Npacl!03keUIV?=#YPf^yn&X+u<|1DnJC1CVCIlUg~aRJjGN4hyn$LV4u;^(}6;KDVmbgv9y z7lYpjeiuX4YO}d&ResLp2cDrl+pD*w_LlMZYScF7>Nr#r)P6snmD^4KRvI|W}{bULMTK+LuSt=BKO6jA{`W?(PDqT$d=SB z%SvTgHsj5Yov$f>ff%^DX0j~%HL{*+5O^#NW0edI>+;ob9!twuKM9<*8CivuLApBlF=P#MtWob;uK=nbIg4qy2VwjEGtdCP+wl3{pIgQVBJ6HNFozQQM0?*FYnF zqft3dYP&|5+W(jqVdFEQ^^&Lckm7lc%D?UQNdFH}MY|IQ3 zsX}ZMqz*>xvTIzi)0o&$h1f7i4Vu^ou4{OiKAL5Et~}GPcRLSxWx8a=OtQ-J&F5Fg zeaKoIuHv!AFqUm-SkJB<$zw$^R!-ooR%Ept9>HVX%~;tA)(z+1ZX zWd+XKkF1k1+;~MZR%YO=SCIAim0MIR;#IBspJDK=R@7GRXBycr()|hSseGP#*KgJL sKIBofb*YKlF^*+trxy9y>G?Bg>mS3tV?5ro)2n0rKb~vEArLMA00FHQ7ytkO literal 0 HcmV?d00001 diff --git a/VM/post_initialize.sh b/VM/post_initialize.sh index e0bed15..6aa7353 100644 --- a/VM/post_initialize.sh +++ b/VM/post_initialize.sh @@ -24,7 +24,7 @@ systemctl start logstash.service echo "Wait for Elasticsearch to start ... (waiting 30 seconds)" sleep 30 cd /home/vagrant/tsharkVM/Kibana -curl -X PUT "localhost:9200/_template/packets?include_type_name" -H 'Content-Type: application/json' -d@template_tshark_mapping_deduplicated.json +curl -X PUT "localhost:9200/_index_template/packets_template" -H 'Content-Type: application/json' -d@template_tshark_mapping_deduplicated.json echo "Wait for Kibana to start ... (waiting 60 seconds)" sleep 60 diff --git a/build.sh b/build.sh index 709c6e8..5613238 100644 --- a/build.sh +++ b/build.sh @@ -19,3 +19,7 @@ cd ./VM bash ./build_vm.sh + +cd .. +sleep 10 +bash ./upload_pcaps.sh