[infer/py] first draft to support **kwargs.

Summary: Named arguments are not supported by Textual, so this diff propose a first encoding to make sure we flag them correctly in the code. Support is incomplete for now. We'll see later how to better deal with them, and if Textual/Sil can support this construct first-hand.

Reviewed By: ngorogiannis

Differential Revision: D49369151

fbshipit-source-id: 4d3370a0c638e53c77ae2c01f8dc05d4c227f01a

infer/src/python/PyBuiltin.ml

@@ -98,6 +98,8 @@ module Builtin = struct
     (* BINARY_SUBSCR is more complex and is done in PyTrans *)
     | Inplace of binary_op
     | PythonCall
+    | PythonCallKW
+    | PythonKWArg
     | PythonClass
     | PythonCode
     | PythonIter
@@ -151,6 +153,10 @@ let to_proc_name = function
             sprintf "inplace_%s" (binary_op_to_string op)
         | PythonCall ->
             "python_call"
+        | PythonCallKW ->
+            "python_call_kw"
+        | PythonKWArg ->
+            "python_kw_arg"
         | PythonClass ->
             "python_class"
         | PythonCode ->
@@ -296,6 +302,12 @@ module Set = struct
       ; binary_op (Builtin.Inplace Xor)
       ; ( Builtin.PythonCall
         , {formals_types= None; result_type= annotatedObject; used_struct_types= []} )
+      ; ( Builtin.PythonCallKW
+        , {formals_types= None; result_type= annotatedObject; used_struct_types= []} )
+      ; ( Builtin.PythonKWArg
+        , { formals_types= Some [annot string_; annotatedObject]
+          ; result_type= annotatedObject
+          ; used_struct_types= [] } )
       ; ( Builtin.PythonClass
         , { formals_types= Some [annot string_]
           ; result_type= annot PyCommon.pyClass

infer/src/python/PyBuiltin.mli

@@ -35,6 +35,8 @@ type textual =
   | Binary of binary_op
   | Inplace of binary_op
   | PythonCall
+  | PythonCallKW
+  | PythonKWArg
   | PythonClass
   | PythonCode
   | PythonIter

infer/src/python/PyTrans.ml

@@ -31,6 +31,9 @@ module Error = struct
     | Exception of string * DataStack.cell
     | SetupWith of DataStack.cell
     | New of Ident.t
+    | CallKwMissing of string
+    | CallKwMissingId of Ident.t
+    | CallKwInvalidFunction of DataStack.cell
 
   let pp_todo fmt = function
     | UnsupportedOpcode opname ->
@@ -59,6 +62,13 @@ module Error = struct
         F.fprintf fmt "Unsupported context manager: %a" DataStack.pp_cell cell
     | New id ->
         F.fprintf fmt "Unsupported '__new__' user declaration in %a" Ident.pp id
+    | CallKwMissing name ->
+        F.fprintf fmt "Unsupported CALL_FUNCTION_KW because %s is not defined in the same file" name
+    | CallKwMissingId id ->
+        F.fprintf fmt "Unsupported CALL_FUNCTION_KW because %a is not defined in the same file"
+          Ident.pp id
+    | CallKwInvalidFunction cell ->
+        F.fprintf fmt "Unsupported CALL_FUNCTION_KW with %a" DataStack.pp_cell cell
 
 
   type kind =
@@ -100,6 +110,9 @@ module Error = struct
     | EOF
     | TopLevelName of string
     | TopLevelInvalid of string
+    | CallKeywordNotString0 of FFI.Constant.t
+    | CallKeywordNotString1 of DataStack.cell
+    | CallKeywordBuildClass
 
   type t = L.error * kind
 
@@ -186,6 +199,12 @@ module Error = struct
         F.fprintf fmt "Toplevel modules must be named '<module>', but got %s" name
     | TopLevelInvalid filename ->
         F.fprintf fmt "Invalid module path '%s'" filename
+    | CallKeywordNotString0 cst ->
+        F.fprintf fmt "CALL_FUNCTION_KW: keyword is not a string: %a" FFI.Constant.pp cst
+    | CallKeywordNotString1 cell ->
+        F.fprintf fmt "CALL_FUNCTION_KW: keyword is not a string: %a" DataStack.pp_cell cell
+    | CallKeywordBuildClass ->
+        F.pp_print_string fmt "CALL_FUNCTION_KW cannot be used with LOAD_BUILD_CLASS"
 
 
   let class_decl (err, kind) = (err, ClassDecl kind)
@@ -663,6 +682,47 @@ module FUNCTION = struct
 
         After: [ result | rest-of-the-stack ] *)
 
+    let static_call_with_args env fid args =
+      let loc = Env.loc env in
+      (* TODO: support nesting *)
+      let key = Symbol.Global fid in
+      let mk env ?typ proc = mk env (Some fid) ?typ loc proc args in
+      match Env.lookup_symbol env key with
+      | None ->
+          (* Unknown name, can come from a `from foo import *` so we'll try to locate it in a
+             further analysis *)
+          let proc =
+            let enclosing_class = T.TypeName.wildcard in
+            qualified_procname ~enclosing_class @@ Ident.to_proc_name fid
+          in
+          mk env proc
+      | Some symbol -> (
+        match (symbol : Symbol.t) with
+        | {kind= Import _ | ImportCall; id} ->
+            Error (L.InternalError, Error.TODO (StaticCallImport id))
+        | {kind= Builtin; id} ->
+            (* TODO: propagate builtin type information *)
+            let proc = Ident.to_qualified_procname id in
+            mk env proc
+        | {kind= Name {is_imported}; id} ->
+            (* TODO: propagate type information if available *)
+            let proc = Ident.to_qualified_procname id in
+            let key = Symbol.Global id in
+            let symbol_info = {Symbol.kind= ImportCall; id; loc} in
+            let env = if is_imported then Env.register_symbol env key symbol_info else env in
+            mk env proc
+        | {kind= Code; id} ->
+            (* TODO: propagate type information if available *)
+            let proc = Ident.to_qualified_procname id in
+            mk env proc
+        | {kind= Class; id} ->
+            (* TODO: support nesting. Maybe add to_proc_name to Symbol *)
+            let typ = Ident.to_typ id in
+            let name = Ident.to_constructor id in
+            let proc : T.qualified_procname = {enclosing_class= TopLevel; name} in
+            mk env ~typ proc )
+
+
     let static_call env code fid cells =
       let open IResult.Let_syntax in
       (* TODO: we currently can't handle hasattr correctly, so let's ignore it
@@ -680,49 +740,10 @@ module FUNCTION = struct
         Ok (env, None) )
       else
         let* env, args = cells_to_textual env code cells in
-        let loc = Env.loc env in
-        (* TODO: support nesting *)
-        let key = Symbol.Global fid in
-        let mk env ?typ proc = mk env (Some fid) ?typ loc proc args in
-        match Env.lookup_symbol env key with
-        | None ->
-            (* Unknown name, can come from a `from foo import *` so we'll try to locate it in a
-               further analysis *)
-            let proc =
-              let enclosing_class = T.TypeName.wildcard in
-              qualified_procname ~enclosing_class @@ Ident.to_proc_name fid
-            in
-            mk env proc
-        | Some symbol -> (
-          match (symbol : Symbol.t) with
-          | {kind= Import _ | ImportCall; id} ->
-              Error (L.InternalError, Error.TODO (StaticCallImport id))
-          | {kind= Builtin; id} ->
-              (* TODO: propagate builtin type information *)
-              let proc = Ident.to_qualified_procname id in
-              mk env proc
-          | {kind= Name {is_imported}; id} ->
-              (* TODO: propagate type information if available *)
-              let proc = Ident.to_qualified_procname id in
-              let key = Symbol.Global id in
-              let symbol_info = {Symbol.kind= ImportCall; id; loc} in
-              let env = if is_imported then Env.register_symbol env key symbol_info else env in
-              mk env proc
-          | {kind= Code; id} ->
-              (* TODO: propagate type information if available *)
-              let proc = Ident.to_qualified_procname id in
-              mk env proc
-          | {kind= Class; id} ->
-              (* TODO: support nesting. Maybe add to_proc_name to Symbol *)
-              let typ = Ident.to_typ id in
-              let name = Ident.to_constructor id in
-              let proc : T.qualified_procname = {enclosing_class= TopLevel; name} in
-              mk env ~typ proc )
-
-
-    let dynamic_call env code caller_id cells =
-      let open IResult.Let_syntax in
-      let* env, args = cells_to_textual env code cells in
+        static_call_with_args env fid args
+
+
+    let dynamic_call env caller_id args =
       let args = T.Exp.Var caller_id :: args in
       let env, id, _typ = Env.mk_builtin_call env Builtin.PythonCall args in
       let env = Env.push env (DataStack.Temp id) in
@@ -768,7 +789,8 @@ module FUNCTION = struct
           let fid = Ident.mk ~loc @@ co_names.(ndx) in
           static_call env code fid cells
       | Temp id ->
-          dynamic_call env code id cells
+          let* env, args = cells_to_textual env code cells in
+          dynamic_call env id args
       | Code _
       | Import _
       | StaticCall _
@@ -905,6 +927,125 @@ module FUNCTION = struct
       let env = Env.push env (DataStack.Code {fun_or_class= true; code_name; code}) in
       Ok (env, None)
   end
+
+  module CALL_KW = struct
+    let extract_kw_names const =
+      let open IResult.Let_syntax in
+      let error const = (L.UserError, Error.CallKeywordNotString0 const) in
+      match (const : FFI.Constant.t) with
+      | PYCTuple tuple ->
+          Array.fold_right tuple ~init:(Ok []) ~f:(fun const acc ->
+              let* acc in
+              let* name = Result.of_option (FFI.Constant.as_name const) ~error:(error const) in
+              Ok (name :: acc) )
+      | _ ->
+          Error (error const)
+
+
+    (* there is more args than names, and nameless values must come first in the end *)
+    let partial_zip env argc args names =
+      let nr_positional = argc - List.length names in
+      let rec zip env ndx l1 l2 =
+        match (l1, l2) with
+        | [], _ ->
+            (env, [])
+        | _ :: _, [] ->
+            (env, l1)
+        | hd1 :: tl1, hd2 :: tl2 ->
+            if ndx < nr_positional then
+              let env, tl = zip env (ndx + 1) tl1 l2 in
+              (env, hd1 :: tl)
+            else
+              let name = T.Exp.Const (Str hd2) in
+              let env, id, _typ = Env.mk_builtin_call env Builtin.PythonKWArg [name; hd1] in
+              let hd = T.Exp.Var id in
+              let env, tl = zip env (ndx + 1) tl1 tl2 in
+              (env, hd :: tl)
+      in
+      zip env 0 args names
+
+
+    (** {v CALL_FUNCTION_KW(argc) v}
+
+        Calls a callable object with positional (if any) and keyword arguments. [argc] indicates the
+        total number of positional and keyword arguments. The top element on the stack contains a
+        tuple of keyword argument names. Below that are keyword arguments in the order corresponding
+        to the tuple. Below that are positional arguments, with the right-most parameter on top.
+        Below the arguments is a callable object to call. [CALL_FUNCTION_KW] pops all arguments and
+        the callable object off the stack, calls the callable object with those arguments, and
+        pushes the return value returned by the callable object. *)
+
+    let run env ({FFI.Code.co_names; co_consts} as code) {FFI.Instruction.opname; arg= argc} =
+      (* TODO:
+         - make support more complete
+         - check/deal with method calls using kw *)
+      let open IResult.Let_syntax in
+      Debug.p "[%s] argc = %d\n" opname argc ;
+      let* env, arg_names = pop_datastack opname env in
+      let* arg_names =
+        match (arg_names : DataStack.cell) with
+        | Const n ->
+            (* kw names should be constant tuple of strings, so we directly access them *)
+            let tuple = co_consts.(n) in
+            extract_kw_names tuple
+        | _ ->
+            Error (L.UserError, Error.CallKeywordNotString1 arg_names)
+      in
+      let* env, cells = pop_n_datastack opname env argc in
+      let* env, all_args = cells_to_textual env code cells in
+      let env, args = partial_zip env argc all_args arg_names in
+      let* env, fname = pop_datastack opname env in
+      let call env fname =
+        let env, id, _typ = Env.mk_builtin_call env Builtin.PythonCallKW (fname :: args) in
+        let env = Env.push env (DataStack.Temp id) in
+        Ok (env, None)
+      in
+      let loc = Env.loc env in
+      (* TODO: Fix this, should use dynamic_call with a string all the time *)
+      match (fname : DataStack.cell) with
+      | Name {global; ndx} -> (
+          let name = co_names.(ndx) in
+          let key = mk_key global loc name in
+          match Env.lookup_symbol env key with
+          | Some {Symbol.id} ->
+              let fname = Ident.to_string ~sep:"." id in
+              let fname = T.Exp.Const (Str fname) in
+              call env fname
+          | None ->
+              Error (L.ExternalError, Error.TODO (CallKwMissing name)) )
+      | Temp id ->
+          call env (T.Exp.Var id)
+      | BuiltinBuildClass ->
+          Error (L.ExternalError, Error.CallKeywordBuildClass)
+      | Path id -> (
+          let key = Symbol.Global id in
+          match Env.lookup_symbol env key with
+          | Some {Symbol.id} ->
+              let fname = Ident.to_string ~sep:"." id in
+              let fname = T.Exp.Const (Str fname) in
+              call env fname
+          | None ->
+              Error (L.InternalError, Error.TODO (CallKwMissingId id)) )
+      | ImportCall {id} ->
+          (* TODO: test it *)
+          Debug.todo "TEST IT 2!\n" ;
+          let fname = Ident.to_string ~sep:"." id in
+          let fname = T.Exp.Const (Str fname) in
+          call env fname
+      | Code _
+      | VarName _
+      | Const _
+      | Map _
+      | Import _
+      | StaticCall _
+      | MethodCall _
+      | NoException
+      | WithContext _ ->
+          Error (L.InternalError, Error.TODO (CallKwInvalidFunction fname))
+      | Super ->
+          let env = Env.push env Super in
+          Ok (env, None)
+  end
 end
 
 module METHOD = struct
@@ -2146,6 +2287,8 @@ let run_instruction env code ({FFI.Instruction.opname; starts_line} as instr) ne
       POP_TOP.run env code instr
   | "CALL_FUNCTION" ->
       FUNCTION.CALL.run env code instr
+  | "CALL_FUNCTION_KW" ->
+      FUNCTION.CALL_KW.run env code instr
   | "BINARY_ADD" ->
       BINARY.run env code instr (Builtin.Binary Add)
   | "BINARY_SUBTRACT" ->

infer/src/python/unit/PyTransTest.ml

@@ -3211,3 +3211,57 @@ def f(**kwargs):
 
         declare $builtins.python_int(int) : *PyInt |}]
   end )
+
+
+let%test_module "call_kw" =
+  ( module struct
+    let%expect_test _ =
+      let source = {|
+def f(z, x, y):
+        pass
+
+f(0, y=2, x=1)
+        |} in
+      test source ;
+      [%expect
+        {|
+        .source_language = "python"
+
+        define dummy.$toplevel() : *PyNone {
+          #b0:
+              n0 = $builtins.python_code("dummy.f")
+              n1 = $builtins.python_kw_arg("y", $builtins.python_int(2))
+              n2 = $builtins.python_kw_arg("x", $builtins.python_int(1))
+              n3 = $builtins.python_call_kw("dummy.f", $builtins.python_int(0), n1, n2)
+              ret null
+
+        }
+
+        define dummy.f(z: *PyObject, x: *PyObject, y: *PyObject) : *PyObject {
+          #b0:
+              ret null
+
+        }
+
+        global $python_implicit_names::__name__: *PyString
+
+        global $python_implicit_names::__file__: *PyString
+
+        declare $builtins.python_code(*String) : *PyCode
+
+        declare $builtins.python_kw_arg(*String, *PyObject) : *PyObject
+
+        declare $builtins.python_call_kw(...) : *PyObject
+
+        declare $builtins.python_tuple(...) : *PyObject
+
+        declare $builtins.python_bytes(*Bytes) : *PyBytes
+
+        declare $builtins.python_string(*String) : *PyString
+
+        declare $builtins.python_bool(int) : *PyBool
+
+        declare $builtins.python_float(float) : *PyFloat
+
+        declare $builtins.python_int(int) : *PyInt |}]
+  end )