Skip to content

Commit

Permalink
[sqlite] check blob size when serializing
Browse files Browse the repository at this point in the history
Summary:
SQLite has a compile-time-settable, maximum length for strings and blobs written to the database. We previously had a mechanism for recovering from exceptions thrown when this limit is exceeded, but SQLite (or the OCaml bindings) seem to get into a bad state and the recovery mechanism is not working properly and leads to crashes regardless.

This diff adds a configurable maximum blob size that is checked during serialization (as opposed to during writing to the DB). Because the OCaml bindings do not let us retrieve what is the current compiled blob length, we put in place a user-configurable limit.

Currently, we check that this size limit is not exceeded only when writing out summaries. If the limit is exceeded, we write an empty summary.

If the size is exceeded elsewhere we will get a crash.

Reviewed By: dulmarod

Differential Revision:
D50077381

Privacy Context Container: L1208441

fbshipit-source-id: 25544898ca1b46de39cb63ac6df0b4e5199f46f9
  • Loading branch information
ngorogiannis authored and facebook-github-bot committed Oct 11, 2023
1 parent 12ca140 commit 1244be3
Show file tree
Hide file tree
Showing 10 changed files with 43 additions and 6 deletions.
3 changes: 3 additions & 0 deletions infer/man/man1/infer-analyze.txt
Original file line number Diff line number Diff line change
Expand Up @@ -406,6 +406,9 @@ OPTIONS
--sqlite-lock-timeout int
Timeout for SQLite results database operations, in milliseconds.

--sqlite-max-blob-size int
Maximum blob/string size for data written in SQLite.

--sqlite-page-size int
SQLite page size in bytes, must be a power of two between 512 and
65536.
Expand Down
3 changes: 3 additions & 0 deletions infer/man/man1/infer-capture.txt
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,9 @@ OPTIONS
--sqlite-lock-timeout int
Timeout for SQLite results database operations, in milliseconds.

--sqlite-max-blob-size int
Maximum blob/string size for data written in SQLite.

--sqlite-page-size int
SQLite page size in bytes, must be a power of two between 512 and
65536.
Expand Down
4 changes: 4 additions & 0 deletions infer/man/man1/infer-full.txt
Original file line number Diff line number Diff line change
Expand Up @@ -1830,6 +1830,10 @@ OPTIONS
Timeout for SQLite results database operations, in milliseconds.
See also infer-analyze(1), infer-capture(1), and infer-run(1).

--sqlite-max-blob-size int
Maximum blob/string size for data written in SQLite.
See also infer-analyze(1), infer-capture(1), and infer-run(1).

--sqlite-page-size int
SQLite page size in bytes, must be a power of two between 512 and
65536. See also infer-analyze(1), infer-capture(1), and infer-run(1).
Expand Down
3 changes: 3 additions & 0 deletions infer/man/man1/infer-run.txt
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,9 @@ OPTIONS
--sqlite-lock-timeout int
Timeout for SQLite results database operations, in milliseconds.

--sqlite-max-blob-size int
Maximum blob/string size for data written in SQLite.

--sqlite-page-size int
SQLite page size in bytes, must be a power of two between 512 and
65536.
Expand Down
4 changes: 4 additions & 0 deletions infer/man/man1/infer.txt
Original file line number Diff line number Diff line change
Expand Up @@ -1830,6 +1830,10 @@ OPTIONS
Timeout for SQLite results database operations, in milliseconds.
See also infer-analyze(1), infer-capture(1), and infer-run(1).

--sqlite-max-blob-size int
Maximum blob/string size for data written in SQLite.
See also infer-analyze(1), infer-capture(1), and infer-run(1).

--sqlite-page-size int
SQLite page size in bytes, must be a power of two between 512 and
65536. See also infer-analyze(1), infer-capture(1), and infer-run(1).
Expand Down
8 changes: 4 additions & 4 deletions infer/src/backend/Summary.ml
Original file line number Diff line number Diff line change
Expand Up @@ -241,11 +241,11 @@ module OnDisk = struct
~report_summary:(ReportSummary.SQLite.serialize report_summary)
~summary_metadata:(SummaryMetadata.SQLite.serialize summary_metadata) ;
summary
with SqliteUtils.Error msg when String.is_substring msg ~substring:"TOOBIG" ->
(* Sqlite failed with [TOOBIG], reset to the empty summary and write it back *)
with SqliteUtils.DataTooBig ->
(* Serialization exceeded size limits, write and return an empty summary *)
L.internal_error
"Summary for %a caused a TOOBIG Sqlite error, writing empty summary instead.@\n" Procname.pp
proc_name ;
"Summary for %a caused exceeds blob size limit, writing empty summary instead.@\n"
Procname.pp proc_name ;
let payloads = Payloads.empty in
let report_summary = ReportSummary.empty () in
let summary_metadata = SummaryMetadata.empty proc_name in
Expand Down
9 changes: 9 additions & 0 deletions infer/src/base/Config.ml
Original file line number Diff line number Diff line change
Expand Up @@ -3254,6 +3254,13 @@ and sqlite_lock_timeout =
"Timeout for SQLite results database operations, in milliseconds."


and sqlite_max_blob_size =
CLOpt.mk_int ~long:"sqlite-max-blob-size" ~default:500_000_000
~in_help:
InferCommand.[(Analyze, manual_generic); (Capture, manual_generic); (Run, manual_generic)]
"Maximum blob/string size for data written in SQLite."


and sqlite_vfs = CLOpt.mk_string_opt ~long:"sqlite-vfs" "VFS for SQLite"

and subtype_multirange =
Expand Down Expand Up @@ -4484,6 +4491,8 @@ and sqlite_page_size = !sqlite_page_size

and sqlite_lock_timeout = !sqlite_lock_timeout

and sqlite_max_blob_size = !sqlite_max_blob_size

and sqlite_vfs = !sqlite_vfs

and starvation_skip_analysis = !starvation_skip_analysis
Expand Down
2 changes: 2 additions & 0 deletions infer/src/base/Config.mli
Original file line number Diff line number Diff line change
Expand Up @@ -811,6 +811,8 @@ val sqlite_page_size : int

val sqlite_lock_timeout : int

val sqlite_max_blob_size : int

val sqlite_vfs : string option

val starvation_skip_analysis : Yojson.Basic.t
Expand Down
10 changes: 8 additions & 2 deletions infer/src/base/SqliteUtils.ml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ module L = Logging

exception Error of string

exception DataTooBig

let error fmt = Format.kasprintf (fun err -> raise (Error err)) fmt

let check_result_code db ~log rc =
Expand Down Expand Up @@ -124,7 +126,9 @@ module MarshalledDataNOTForComparison (D : T) = struct
Marshal.from_string b 0


let serialize x = Sqlite3.Data.BLOB (Marshal.to_string x [])
let serialize x =
let s = Marshal.to_string x [] in
if String.length s < Config.sqlite_max_blob_size then Sqlite3.Data.BLOB s else raise DataTooBig
end

module MarshalledNullableDataNOTForComparison (D : T) = struct
Expand All @@ -141,5 +145,7 @@ module MarshalledNullableDataNOTForComparison (D : T) = struct
| None ->
Sqlite3.Data.NULL
| Some x ->
Sqlite3.Data.BLOB (Marshal.to_string x [])
let s = Marshal.to_string x [] in
if String.length s < Config.sqlite_max_blob_size then Sqlite3.Data.BLOB s
else raise DataTooBig
end
3 changes: 3 additions & 0 deletions infer/src/base/SqliteUtils.mli
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@ open! IStd
particular, they may raise if the [Sqlite3.Rc.t] result of certain operations is unexpected. *)
exception Error of string

(** Raised from serializers when final size exceeds Sqlite3 limits (normally 1000_000_000 bytes). *)
exception DataTooBig

val check_result_code : Sqlite3.db -> log:string -> Sqlite3.Rc.t -> unit
(** Assert that the result is either [Sqlite3.Rc.OK] or [Sqlite3.Rc.ROW]. If the result is not
valid, raise {!Error}. *)
Expand Down

0 comments on commit 1244be3

Please sign in to comment.