feat: Add debug information about each email verification (reacherhq#…

…1391)

* Add time and server_ip

* Add server_name

* Fix tests

* clippy

* Update openapi

backend/openapi.json

@@ -57,7 +57,8 @@
 												"has_full_inbox": false,
 												"is_catch_all": false,
 												"is_deliverable": false,
-												"is_disabled": false
+												"is_disabled": false,
+												"method": "SmtpConnection"
 											},
 											"syntax": {
 												"domain": "gmail.com",
@@ -180,15 +181,18 @@
 					},
 					"syntax": {
 						"$ref": "#/components/schemas/SyntaxDetails"
+					},
+					"debug": {
+						"$ref": "#/components/schemas/DebugDetails"
 					}
 				},
 				"required": [
 					"input",
+					"is_reachable",
 					"misc",
 					"mx",
 					"smtp",
-					"syntax",
-					"is_reachable"
+					"syntax"
 				]
 			},
 			"Error": {
@@ -270,14 +274,27 @@
 					"is_disabled": {
 						"type": "boolean",
 						"description": "Has this email address been disabled by the email provider?"
+					},
+					"method": {
+						"x-stoplight": {
+							"id": "axwtm6b7ao9n6"
+						},
+						"enum": [
+							"Headless",
+							"Api",
+							"Skipped",
+							"SmtpConnection"
+						],
+						"description": "The method used to verify this email."
 					}
 				},
 				"required": [
 					"can_connect_smtp",
 					"has_full_inbox",
 					"is_catch_all",
 					"is_deliverable",
-					"is_disabled"
+					"is_disabled",
+					"method"
 				]
 			},
 			"SyntaxDetails": {
@@ -314,12 +331,18 @@
 			},
 			"HotmailVerifyMethod": {
 				"type": "string",
+				"x-stoplight": {
+					"id": "o6xocxf6tktur"
+				},
 				"title": "HotmailVerifyMethod",
 				"enum": ["Api", "Headless", "Smtp"],
 				"description": "An enum to describe how we verify Hotmail emails."
 			},
 			"GmailVerifyMethod": {
 				"type": "string",
+				"x-stoplight": {
+					"id": "jbq83vpkfcmth"
+				},
 				"title": "GmailVerifyMethod",
 				"enum": ["Api", "Smtp"],
 				"description": "An enum to describe how we verify Gmail emails.",
@@ -433,6 +456,42 @@
 					}
 				},
 				"required": ["secs", "nanos"]
+			},
+			"DebugDetails": {
+				"title": "DebugDetails",
+				"x-stoplight": {
+					"id": "ndv3lpqeuypvn"
+				},
+				"type": "object",
+				"properties": {
+					"start_time": {
+						"type": "string",
+						"x-stoplight": {
+							"id": "hxi446woaebm1"
+						}
+					},
+					"end_time": {
+						"type": "string",
+						"x-stoplight": {
+							"id": "m4p986hhzuzbl"
+						}
+					},
+					"duration": {
+						"$ref": "#/components/schemas/Duration"
+					},
+					"server_ip": {
+						"type": "string",
+						"x-stoplight": {
+							"id": "ld3cp67qd2nx8"
+						}
+					},
+					"server_name": {
+						"type": "string",
+						"x-stoplight": {
+							"id": "yy9pz1lxrr4si"
+						}
+					}
+				}
 			}
 		},
 		"securitySchemes": {

backend/tests/check_email.rs

@@ -23,8 +23,8 @@ use reacher_backend::routes::create_routes;
 use warp::http::StatusCode;
 use warp::test::request;
 
-const FOO_BAR_RESPONSE: &str = r#"{"input":"foo@bar","is_reachable":"invalid","misc":{"is_disposable":false,"is_role_account":false,"gravatar_url":null,"haveibeenpwned":null},"mx":{"accepts_mail":false,"records":[]},"smtp":{"can_connect_smtp":false,"has_full_inbox":false,"is_catch_all":false,"is_deliverable":false,"is_disabled":false},"syntax":{"address":null,"domain":"","is_valid_syntax":false,"username":"","normalized_email":null,"suggestion":null}}"#;
-const FOO_BAR_BAZ_RESPONSE: &str = r#"{"input":"[email protected]","is_reachable":"invalid","misc":{"is_disposable":false,"is_role_account":false,"gravatar_url":null,"haveibeenpwned":null},"mx":{"accepts_mail":false,"records":[]},"smtp":{"can_connect_smtp":false,"has_full_inbox":false,"is_catch_all":false,"is_deliverable":false,"is_disabled":false},"syntax":{"address":"[email protected]","domain":"bar.baz","is_valid_syntax":true,"username":"foo","normalized_email":"[email protected]","suggestion":null}}"#;
+const FOO_BAR_RESPONSE: &str = r#"{"input":"foo@bar","is_reachable":"invalid","misc":{"is_disposable":false,"is_role_account":false,"gravatar_url":null,"haveibeenpwned":null},"mx":{"accepts_mail":false,"records":[]},"smtp":{"can_connect_smtp":false,"has_full_inbox":false,"is_catch_all":false,"is_deliverable":false,"is_disabled":false,"method":"Skipped"},"syntax":{"address":null,"domain":"","is_valid_syntax":false,"username":"","normalized_email":null,"suggestion":null}"#;
+const FOO_BAR_BAZ_RESPONSE: &str = r#"{"input":"[email protected]","is_reachable":"invalid","misc":{"is_disposable":false,"is_role_account":false,"gravatar_url":null,"haveibeenpwned":null},"mx":{"accepts_mail":false,"records":[]},"smtp":{"can_connect_smtp":false,"has_full_inbox":false,"is_catch_all":false,"is_deliverable":false,"is_disabled":false,"method":"Skipped"},"syntax":{"address":"[email protected]","domain":"bar.baz","is_valid_syntax":true,"username":"foo","normalized_email":"[email protected]","suggestion":null}"#;
 
 #[tokio::test]
 async fn test_input_foo_bar() {
@@ -39,7 +39,8 @@ async fn test_input_foo_bar() {
 		.await;
 
 	assert_eq!(resp.status(), StatusCode::OK, "{:?}", resp.body());
-	assert_eq!(resp.body(), FOO_BAR_RESPONSE);
+	println!("{:?}", resp.body());
+	assert!(resp.body().starts_with(FOO_BAR_RESPONSE.as_bytes()));
 }
 
 #[tokio::test]
@@ -55,7 +56,7 @@ async fn test_input_foo_bar_baz() {
 		.await;
 
 	assert_eq!(resp.status(), StatusCode::OK, "{:?}", resp.body());
-	assert_eq!(resp.body(), FOO_BAR_BAZ_RESPONSE);
+	assert!(resp.body().starts_with(FOO_BAR_BAZ_RESPONSE.as_bytes()));
 }
 
 #[tokio::test]
@@ -102,7 +103,7 @@ async fn test_reacher_secret_correct_secret() {
 		.await;
 
 	assert_eq!(resp.status(), StatusCode::OK, "{:?}", resp.body());
-	assert_eq!(resp.body(), FOO_BAR_RESPONSE);
+	assert!(resp.body().starts_with(FOO_BAR_RESPONSE.as_bytes()));
 }
 
 #[tokio::test]

core/Cargo.toml

@@ -18,10 +18,12 @@ async-recursion = "1.0.5"
 async-smtp = { version = "0.6.0", features = ["socks5"] }
 async-std = "1.12.0"
 async-std-resolver = "0.21.2"
+chrono = "=0.4.22"
 fantoccini = { version = "0.19.3", optional = true }
 futures = { version = "0.3.29", optional = true }
 fast-socks5 = "0.9.1"
 levenshtein = "1.0.5"
+local-ip-address = "0.5.6"
 log = "0.4.20"
 mailchecker = "6.0.1"
 md5 = "0.7.0"

core/src/lib.rs

@@ -74,6 +74,7 @@ use misc::{check_misc, MiscDetails};
 use mx::check_mx;
 use rand::Rng;
 use smtp::{check_smtp, SmtpDetails, SmtpError};
+use std::time::{Duration, SystemTime};
 use syntax::{check_syntax, get_similar_mail_provider};
 use trust_dns_proto::rr::rdata::MX;
 pub use util::constants::LOG_TARGET;
@@ -111,6 +112,7 @@ fn calculate_reachable(misc: &MiscDetails, smtp: &Result<SmtpDetails, SmtpError>
 /// Returns a `CheckEmailOutput` output, whose `is_reachable` field is one of
 /// `Safe`, `Invalid`, `Risky` or `Unknown`.
 pub async fn check_email(input: &CheckEmailInput) -> CheckEmailOutput {
+	let start_time = SystemTime::now();
 	let to_email = &input.to_email;
 
 	log::debug!(
@@ -233,12 +235,21 @@ pub async fn check_email(input: &CheckEmailInput) -> CheckEmailOutput {
 		get_similar_mail_provider(&mut my_syntax);
 	}
 
+	let end_time = SystemTime::now();
 	CheckEmailOutput {
 		input: to_email.to_string(),
 		is_reachable: calculate_reachable(&my_misc, &my_smtp),
 		misc: Ok(my_misc),
 		mx: Ok(my_mx),
 		smtp: my_smtp,
 		syntax: my_syntax,
+		debug: DebugDetails {
+			start_time: start_time.into(),
+			end_time: end_time.into(),
+			duration: end_time
+				.duration_since(start_time)
+				.unwrap_or(Duration::from_secs(0)),
+			..Default::default()
+		},
 	}
 }

core/src/smtp/connect.rs

@@ -26,7 +26,7 @@ use std::iter;
 use std::str::FromStr;
 use std::time::Duration;
 
-use super::parser;
+use super::{parser, SmtpConnection, SmtpMethod};
 use super::{SmtpDetails, SmtpError};
 use crate::{
 	rules::{has_rule, Rule},
@@ -334,6 +334,9 @@ async fn check_smtp_without_retry(
 		is_catch_all,
 		is_deliverable: deliverability.is_deliverable,
 		is_disabled: deliverability.is_disabled,
+		method: SmtpMethod::SmtpConnection(SmtpConnection {
+			host: host.to_string(),
+		}),
 	})
 }