Skip to content

Commit 4c08e6d

Browse files
committedJan 24, 2025·
Remove unnecessary escaping of SNS->SQS records
The previous bit of code was in place due to some hasty misunderstanding of log output, oops!
1 parent 8076d8f commit 4c08e6d

File tree

2 files changed

+3
-10
lines changed

2 files changed

+3
-10
lines changed
 

‎Cargo.toml

+1-1
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ members = [
77
resolver = "2"
88

99
[workspace.package]
10-
version = "1.2.5"
10+
version = "1.2.6"
1111
edition = "2021"
1212
keywords = ["deltalake", "parquet", "lambda", "delta", "sqs"]
1313
homepage = "https://github.com/buoyant-data/oxbow"

‎lambdas/sqs-ingest/src/main.rs

+2-9
Original file line numberDiff line numberDiff line change
@@ -83,8 +83,7 @@ fn unwrap_sns_payload(records: &[SqsMessage]) -> Vec<SqsMessage> {
8383
for record in records {
8484
if let Some(body) = record.body.as_ref() {
8585
trace!("Attempting to unwrap the contents of nested JSON: {body}");
86-
let wrapped = json_unescape(body).expect("Failed to unescape the body of the message");
87-
let nested: SNSWrapper = serde_json::from_str(&wrapped).expect(
86+
let nested: SNSWrapper = serde_json::from_str(body).expect(
8887
"Failed to unpack SNS
8988
messages, this could be a misconfiguration and there is no SNS envelope or raw_delivery has not
9089
been set",
@@ -101,12 +100,6 @@ been set",
101100
unpacked
102101
}
103102

104-
/// SNS likes to wrap everything in double
105-
// <https://stackoverflow.com/a/68996403>
106-
fn json_unescape(s: &str) -> serde_json::Result<String> {
107-
serde_json::from_str(&format!("\"{}\"", s))
108-
}
109-
110103
#[derive(Debug, Deserialize)]
111104
#[serde(rename_all = "PascalCase")]
112105
struct SNSWrapper {
@@ -145,7 +138,7 @@ mod tests {
145138
fn test_unwrap_sns() {
146139
// This is an example of what a full message can look like
147140
//let body = r#"{\"Records\":[{\"eventVersion\":\"2.1\",\"eventSource\":\"aws:s3\",\"awsRegion\":\"us-east-2\",\"eventTime\":\"2024-01-25T20:57:23.379Z\",\"eventName\":\"ObjectCreated:CompleteMultipartUpload\",\"userIdentity\":{\"principalId\":\"AWS:AROAU7FUYKEVYG4GF4IAV:s3-replication\"},\"requestParameters\":{\"sourceIPAddress\":\"10.0.153.194\"},\"responseElements\":{\"x-amz-request-id\":\"RYAX8R8CB6FF1MQN\",\"x-amz-id-2\":\"uLUt4C/TfjwvpObPlTnrWYjOIPH1YT1yJ8jZjqRyLIuTLOxGSkNgKc2Hd1/O7wTP2cd3u59lRtVYrU4ECizehRYw0NGNlL5b\"},\"s3\":{\"s3SchemaVersion\":\"1.0\",\"configurationId\":\"tf-s3-queue-20231207170751084400000001\",\"bucket\":{\"name\":\"scribd-data-warehouse-dev\",\"ownerIdentity\":{\"principalId\":\"A1FIHS1B0BWUTQ\"},\"arn\":\"arn:aws:s3:::scribd-data-warehouse-dev\"},\"object\":{\"key\":\"databases/airbyte/faker_users/ds%3D2024-01-25/1706216212007_0.parquet\",\"size\":143785,\"eTag\":\"67165dca52a1089d312e19c3ddf1e342-1\",\"versionId\":\"3v567TKlEQF5IBoeXrFBRiRX8vY.bY1m\",\"sequencer\":\"0065B2CB162FC1AC3B\"}}}]}"#;
148-
let body = r#"{\"Records\":[{\"eventVersion\":\"2.1\"}]}"#;
141+
let body = r#"{"Records":[{"eventVersion":"2.1"}]}"#;
149142
let message: SqsMessage = SqsMessage {
150143
body: Some(body.to_string()),
151144
..Default::default()

0 commit comments

Comments
 (0)
Please sign in to comment.