From 1244be36acf65659f4c35fce9671e319154d70d2 Mon Sep 17 00:00:00 2001 From: Nikos Gorogiannis Date: Wed, 11 Oct 2023 02:10:54 -0700 Subject: [PATCH] [sqlite] check blob size when serializing Summary: SQLite has a compile-time-settable, maximum length for strings and blobs written to the database. We previously had a mechanism for recovering from exceptions thrown when this limit is exceeded, but SQLite (or the OCaml bindings) seem to get into a bad state and the recovery mechanism is not working properly and leads to crashes regardless. This diff adds a configurable maximum blob size that is checked during serialization (as opposed to during writing to the DB). Because the OCaml bindings do not let us retrieve what is the current compiled blob length, we put in place a user-configurable limit. Currently, we check that this size limit is not exceeded only when writing out summaries. If the limit is exceeded, we write an empty summary. If the size is exceeded elsewhere we will get a crash. Reviewed By: dulmarod Differential Revision: D50077381 Privacy Context Container: L1208441 fbshipit-source-id: 25544898ca1b46de39cb63ac6df0b4e5199f46f9 --- infer/man/man1/infer-analyze.txt | 3 +++ infer/man/man1/infer-capture.txt | 3 +++ infer/man/man1/infer-full.txt | 4 ++++ infer/man/man1/infer-run.txt | 3 +++ infer/man/man1/infer.txt | 4 ++++ infer/src/backend/Summary.ml | 8 ++++---- infer/src/base/Config.ml | 9 +++++++++ infer/src/base/Config.mli | 2 ++ infer/src/base/SqliteUtils.ml | 10 ++++++++-- infer/src/base/SqliteUtils.mli | 3 +++ 10 files changed, 43 insertions(+), 6 deletions(-) diff --git a/infer/man/man1/infer-analyze.txt b/infer/man/man1/infer-analyze.txt index 5bf1c797b1a..2f2f0261a93 100644 --- a/infer/man/man1/infer-analyze.txt +++ b/infer/man/man1/infer-analyze.txt @@ -406,6 +406,9 @@ OPTIONS --sqlite-lock-timeout int Timeout for SQLite results database operations, in milliseconds. + --sqlite-max-blob-size int + Maximum blob/string size for data written in SQLite. + --sqlite-page-size int SQLite page size in bytes, must be a power of two between 512 and 65536. diff --git a/infer/man/man1/infer-capture.txt b/infer/man/man1/infer-capture.txt index 73b99c75855..03bcff93b57 100644 --- a/infer/man/man1/infer-capture.txt +++ b/infer/man/man1/infer-capture.txt @@ -136,6 +136,9 @@ OPTIONS --sqlite-lock-timeout int Timeout for SQLite results database operations, in milliseconds. + --sqlite-max-blob-size int + Maximum blob/string size for data written in SQLite. + --sqlite-page-size int SQLite page size in bytes, must be a power of two between 512 and 65536. diff --git a/infer/man/man1/infer-full.txt b/infer/man/man1/infer-full.txt index 297cd0e03d6..e557872ce4b 100644 --- a/infer/man/man1/infer-full.txt +++ b/infer/man/man1/infer-full.txt @@ -1830,6 +1830,10 @@ OPTIONS Timeout for SQLite results database operations, in milliseconds. See also infer-analyze(1), infer-capture(1), and infer-run(1). + --sqlite-max-blob-size int + Maximum blob/string size for data written in SQLite. + See also infer-analyze(1), infer-capture(1), and infer-run(1). + --sqlite-page-size int SQLite page size in bytes, must be a power of two between 512 and 65536. See also infer-analyze(1), infer-capture(1), and infer-run(1). diff --git a/infer/man/man1/infer-run.txt b/infer/man/man1/infer-run.txt index faf6093bf90..db6415a4b20 100644 --- a/infer/man/man1/infer-run.txt +++ b/infer/man/man1/infer-run.txt @@ -154,6 +154,9 @@ OPTIONS --sqlite-lock-timeout int Timeout for SQLite results database operations, in milliseconds. + --sqlite-max-blob-size int + Maximum blob/string size for data written in SQLite. + --sqlite-page-size int SQLite page size in bytes, must be a power of two between 512 and 65536. diff --git a/infer/man/man1/infer.txt b/infer/man/man1/infer.txt index a212eadee68..13d86477ac6 100644 --- a/infer/man/man1/infer.txt +++ b/infer/man/man1/infer.txt @@ -1830,6 +1830,10 @@ OPTIONS Timeout for SQLite results database operations, in milliseconds. See also infer-analyze(1), infer-capture(1), and infer-run(1). + --sqlite-max-blob-size int + Maximum blob/string size for data written in SQLite. + See also infer-analyze(1), infer-capture(1), and infer-run(1). + --sqlite-page-size int SQLite page size in bytes, must be a power of two between 512 and 65536. See also infer-analyze(1), infer-capture(1), and infer-run(1). diff --git a/infer/src/backend/Summary.ml b/infer/src/backend/Summary.ml index 17c4c408b7e..d8973af7e3a 100644 --- a/infer/src/backend/Summary.ml +++ b/infer/src/backend/Summary.ml @@ -241,11 +241,11 @@ module OnDisk = struct ~report_summary:(ReportSummary.SQLite.serialize report_summary) ~summary_metadata:(SummaryMetadata.SQLite.serialize summary_metadata) ; summary - with SqliteUtils.Error msg when String.is_substring msg ~substring:"TOOBIG" -> - (* Sqlite failed with [TOOBIG], reset to the empty summary and write it back *) + with SqliteUtils.DataTooBig -> + (* Serialization exceeded size limits, write and return an empty summary *) L.internal_error - "Summary for %a caused a TOOBIG Sqlite error, writing empty summary instead.@\n" Procname.pp - proc_name ; + "Summary for %a caused exceeds blob size limit, writing empty summary instead.@\n" + Procname.pp proc_name ; let payloads = Payloads.empty in let report_summary = ReportSummary.empty () in let summary_metadata = SummaryMetadata.empty proc_name in diff --git a/infer/src/base/Config.ml b/infer/src/base/Config.ml index e8be4157460..dcd35cc1d1c 100644 --- a/infer/src/base/Config.ml +++ b/infer/src/base/Config.ml @@ -3254,6 +3254,13 @@ and sqlite_lock_timeout = "Timeout for SQLite results database operations, in milliseconds." +and sqlite_max_blob_size = + CLOpt.mk_int ~long:"sqlite-max-blob-size" ~default:500_000_000 + ~in_help: + InferCommand.[(Analyze, manual_generic); (Capture, manual_generic); (Run, manual_generic)] + "Maximum blob/string size for data written in SQLite." + + and sqlite_vfs = CLOpt.mk_string_opt ~long:"sqlite-vfs" "VFS for SQLite" and subtype_multirange = @@ -4484,6 +4491,8 @@ and sqlite_page_size = !sqlite_page_size and sqlite_lock_timeout = !sqlite_lock_timeout +and sqlite_max_blob_size = !sqlite_max_blob_size + and sqlite_vfs = !sqlite_vfs and starvation_skip_analysis = !starvation_skip_analysis diff --git a/infer/src/base/Config.mli b/infer/src/base/Config.mli index a59d85df2c8..2278e17ba8f 100644 --- a/infer/src/base/Config.mli +++ b/infer/src/base/Config.mli @@ -811,6 +811,8 @@ val sqlite_page_size : int val sqlite_lock_timeout : int +val sqlite_max_blob_size : int + val sqlite_vfs : string option val starvation_skip_analysis : Yojson.Basic.t diff --git a/infer/src/base/SqliteUtils.ml b/infer/src/base/SqliteUtils.ml index 35ecf224d30..3706b3a942b 100644 --- a/infer/src/base/SqliteUtils.ml +++ b/infer/src/base/SqliteUtils.ml @@ -9,6 +9,8 @@ module L = Logging exception Error of string +exception DataTooBig + let error fmt = Format.kasprintf (fun err -> raise (Error err)) fmt let check_result_code db ~log rc = @@ -124,7 +126,9 @@ module MarshalledDataNOTForComparison (D : T) = struct Marshal.from_string b 0 - let serialize x = Sqlite3.Data.BLOB (Marshal.to_string x []) + let serialize x = + let s = Marshal.to_string x [] in + if String.length s < Config.sqlite_max_blob_size then Sqlite3.Data.BLOB s else raise DataTooBig end module MarshalledNullableDataNOTForComparison (D : T) = struct @@ -141,5 +145,7 @@ module MarshalledNullableDataNOTForComparison (D : T) = struct | None -> Sqlite3.Data.NULL | Some x -> - Sqlite3.Data.BLOB (Marshal.to_string x []) + let s = Marshal.to_string x [] in + if String.length s < Config.sqlite_max_blob_size then Sqlite3.Data.BLOB s + else raise DataTooBig end diff --git a/infer/src/base/SqliteUtils.mli b/infer/src/base/SqliteUtils.mli index 85daa2687ab..e994b5fa968 100644 --- a/infer/src/base/SqliteUtils.mli +++ b/infer/src/base/SqliteUtils.mli @@ -11,6 +11,9 @@ open! IStd particular, they may raise if the [Sqlite3.Rc.t] result of certain operations is unexpected. *) exception Error of string +(** Raised from serializers when final size exceeds Sqlite3 limits (normally 1000_000_000 bytes). *) +exception DataTooBig + val check_result_code : Sqlite3.db -> log:string -> Sqlite3.Rc.t -> unit (** Assert that the result is either [Sqlite3.Rc.OK] or [Sqlite3.Rc.ROW]. If the result is not valid, raise {!Error}. *)