[GR-45043] Integrate YARP parser [Part 8]

PullRequest: truffleruby/4097

Author: andrykonchin

3rd_party_licenses.txt

@@ -1308,7 +1308,7 @@ of
 */
 ================================================================================
 
-ffi 1.14.2
+ffi 1.15.5
 
 Copyright (c) 2008-2016, Ruby FFI project contributors
 All rights reserved.

CHANGELOG.md

@@ -3,6 +3,8 @@
 New features:
 
 * C/C++ extensions are now compiled using the system toolchain and executed natively instead of using GraalVM LLVM (Sulong). This leads to faster startup, no warmup, better compatibility, smaller distribution and faster installation for C/C++ extensions (#3118, @eregon).
+* Full suport for the Ruby 3.2 and Ruby 3.3 syntax by adopting the [Prism](https://github.com/ruby/prism) parser (#3117, #3038, #3039, @andrykonchin, @eregon).
+* Pattern matching is now fully supported, with the exception of Find pattern (`in [*, a, *]`) (#3332, #2683, @eregon, @razetime).
 
 Bug fixes:
 
@@ -39,6 +41,8 @@ Compatibility:
 * Fix `Coverage.supported?` and raise `TypeError` if argument is not Symbol (#3039, @andrykonchin).
 * Accept options argument to `Regexp.{new,compile}` of String and warn for unknown types (#3039, @rwstauner).
 * Implement `Time#deconstruct_keys` from Ruby 3.2 (#3039, @rwstauner).
+* Do not autosplat a proc that accepts a single positional argument and keywords (#3039, @andrykonchin).
+* Support passing anonymous * and ** parameters as method call arguments (#3039, @andrykonchin).
 
 Performance:
 

doc/legal/legal.md

@@ -165,9 +165,14 @@ files.
 
 #### FFI
 
-TruffleRuby includes parts of the FFI gem 1.14.2. The FFI gem is copyright
-2008-2016, Ruby FFI project contributors, and covered by the three-clause BSD
-licence (see `ffi.txt`).
+TruffleRuby includes parts of the FFI gem (version as described in [lib/truffle/ffi/version.rb](../../lib/truffle/ffi/version.rb)).
+The FFI gem is copyright  2008-2016, Ruby FFI project contributors, and covered by the three-clause BSD licence (see `ffi.txt`).
+
+#### Prism
+
+TruffleRuby uses the [Prism](https://github.com/ruby/prism) Ruby parser
+(version as described in [src/main/c/yarp/include/prism/version.h](../../src/main/c/yarp/include/prism/version.h)),
+copyright Shopify Inc. and is available under an MIT licence (see `src/main/c/yarp/LICENSE.md`).
 
 # Java dependencies
 

doc/user/options.md

@@ -148,7 +148,6 @@ Other binaries, such as `irb`, `gem`, and so on, support exactly the same switch
 
 TruffleRuby needs to know where to locate files such as the standard library.
 These are stored in the TruffleRuby home directory.
-The Ruby home is always the one that the Truffle framework reports.
+The Ruby home is always either the one that the Truffle framework reports or the extracted internal resources.
 
-If the Ruby home appears not to be correct, or is unset, a warning will be given but the program will continue and you will not be able to require standard libraries.
-You can tell TruffleRuby not to try to find a home at all using the `no-home-provided` option.
+If the Ruby home appears not to be correct, or is unset, a exception will be thrown.

lib/cext/ABI_check.txt

@@ -1 +1 @@
-9
+10

lib/truffle/rbconfig.rb

@@ -38,7 +38,6 @@
 module RbConfig
 
   ruby_home = Truffle::Boot.ruby_home
-  raise 'The TruffleRuby home needs to be set to require RbConfig' unless ruby_home
   TOPDIR = ruby_home
 
   sulong = Truffle::Boot.get_option('cexts-sulong')

lib/truffle/rubygems/defaults/truffleruby.rb

@@ -14,7 +14,7 @@
 module Gem
   # The path to the gems shipped with TruffleRuby
   def self.default_dir
-    @default_dir ||= "#{Truffle::Boot.ruby_home or raise 'TruffleRuby home not found'}/lib/gems"
+    @default_dir ||= "#{Truffle::Boot.ruby_home}/lib/gems"
   end
 
   # Only report the RUBY platform as supported to make sure gems precompiled for MRI are not used.

mx.truffleruby/mx_truffleruby.py

@@ -107,15 +107,21 @@ def clean(self, forBuild=False):
     def contents(self, result):
         classpath_deps = [dep for dep in self.subject.buildDependencies if isinstance(dep, mx.ClasspathDependency)]
         jvm_args = [pipes.quote(arg) for arg in mx.get_runtime_jvm_args(classpath_deps)]
+
         debug_args = mx.java_debug_args()
         jvm_args.extend(debug_args)
         if debug_args:
             jvm_args.extend(['-ea', '-esa'])
+
         jvm_args.append('-Dorg.graalvm.language.ruby.home=' + root)
+
+        libyarpbindings = list(mx.project('org.truffleruby.yarp.bindings').getArchivableResults())[0][0]
+        jvm_args.append('-Dtruffleruby.libyarpbindings=' + libyarpbindings)
+
         main_class = 'org.truffleruby.launcher.RubyLauncher'
         ruby_options = [
             '--experimental-options',
-            '--building-core-cexts',
+            '--building-core-cexts', # This lets the process know it's miniruby
             '--launcher=' + result,
             '--disable-gems',
             '--disable-rubyopt',

mx.truffleruby/suite.py

@@ -376,10 +376,11 @@
 
         "org.truffleruby.bootstrap.launcher": {
             "class": "TruffleRubyBootstrapLauncherProject",
-            "buildDependencies": [
+            "buildDependencies": [ # These are used to build the module path
                 "TRUFFLERUBY", # We need this jar to run extconf.rb
                 "TRUFFLERUBY-LAUNCHER", # We need this jar to run extconf.rb
                 "sulong:SULONG_NATIVE", # We need this jar to find the toolchain with Toolchain#getToolPath
+                "org.truffleruby.yarp.bindings", # libyarpbindings.so
             ],
             "license": ["EPL-2.0"],
         },
@@ -846,8 +847,9 @@
             "dependencies": ["org.truffleruby.tck"],
             "distDependencies": [
                 "truffle:TRUFFLE_TCK",
-               # runtime-only dependencies
+                # runtime-only dependencies
                 "TRUFFLERUBY",
+                "TRUFFLERUBY-RESOURCES",
             ],
             "description" : "Truffle TCK provider for Ruby language.",
             "license": ["EPL-2.0"],

spec/ruby/command_line/dash_r_spec.rb

@@ -16,7 +16,10 @@
     out = ruby_exe(fixture(__FILE__, "bad_syntax.rb"), options: "-r #{@test_file}", args: "2>&1", exit_status: 1)
     $?.should_not.success?
     out.should include("REQUIRED")
-    out.should include("syntax error")
+
+    # it's tempting not to rely on error message and rely only on exception class name,
+    # but CRuby before 3.2 doesn't print class name for syntax error
+    out.should include_any_of("syntax error", "SyntaxError")
   end
 
   it "does not require the file if the main script file does not exist" do

spec/ruby/command_line/syntax_error_spec.rb

@@ -3,11 +3,17 @@
 describe "The interpreter" do
   it "prints an error when given a file with invalid syntax" do
     out = ruby_exe(fixture(__FILE__, "bad_syntax.rb"), args: "2>&1", exit_status: 1)
-    out.should include "syntax error"
+
+    # it's tempting not to rely on error message and rely only on exception class name,
+    # but CRuby before 3.2 doesn't print class name for syntax error
+    out.should include_any_of("syntax error", "SyntaxError")
   end
 
   it "prints an error when given code via -e with invalid syntax" do
     out = ruby_exe(nil, args: "-e 'a{' 2>&1", exit_status: 1)
-    out.should include "syntax error"
+
+    # it's tempting not to rely on error message and rely only on exception class name,
+    # but CRuby before 3.2 doesn't print class name for syntax error
+    out.should include_any_of("syntax error", "SyntaxError")
   end
 end

spec/ruby/core/kernel/eval_spec.rb

@@ -261,6 +261,15 @@ class EvalSpecs
     end
   end
 
+  it "makes flip-flop operator work correctly" do
+    ScratchPad.record []
+
+    eval "10.times { |i| ScratchPad << i if (i == 4)...(i == 4) }"
+    ScratchPad.recorded.should == [4, 5, 6, 7, 8, 9]
+
+    ScratchPad.clear
+  end
+
   # See language/magic_comment_spec.rb for more magic comments specs
   describe "with a magic encoding comment" do
     it "uses the magic comment encoding for the encoding of literal strings" do

spec/ruby/language/block_spec.rb

@@ -40,17 +40,59 @@ def m(a) yield a end
       m([1, 2]) { |a=5, b, c, d| [a, b, c, d] }.should == [5, 1, 2, nil]
     end
 
+    it "assigns elements to pre arguments" do
+      m([1, 2]) { |a, b, c, d=5| [a, b, c, d] }.should == [1, 2, nil, 5]
+    end
+
+    it "assigns elements to pre and post arguments" do
+      m([1               ]) { |a, b=5, c=6, d, e| [a, b, c, d, e] }.should == [1, 5, 6, nil, nil]
+      m([1, 2            ]) { |a, b=5, c=6, d, e| [a, b, c, d, e] }.should == [1, 5, 6, 2, nil]
+      m([1, 2, 3         ]) { |a, b=5, c=6, d, e| [a, b, c, d, e] }.should == [1, 5, 6, 2, 3]
+      m([1, 2, 3, 4      ]) { |a, b=5, c=6, d, e| [a, b, c, d, e] }.should == [1, 2, 6, 3, 4]
+      m([1, 2, 3, 4, 5   ]) { |a, b=5, c=6, d, e| [a, b, c, d, e] }.should == [1, 2, 3, 4, 5]
+      m([1, 2, 3, 4, 5, 6]) { |a, b=5, c=6, d, e| [a, b, c, d, e] }.should == [1, 2, 3, 4, 5]
+    end
+
+    it "assigns elements to pre and post arguments when *rest is present" do
+      m([1               ]) { |a, b=5, c=6, *d, e, f| [a, b, c, d, e, f] }.should == [1, 5, 6, [], nil, nil]
+      m([1, 2            ]) { |a, b=5, c=6, *d, e, f| [a, b, c, d, e, f] }.should == [1, 5, 6, [], 2, nil]
+      m([1, 2, 3         ]) { |a, b=5, c=6, *d, e, f| [a, b, c, d, e, f] }.should == [1, 5, 6, [], 2, 3]
+      m([1, 2, 3, 4      ]) { |a, b=5, c=6, *d, e, f| [a, b, c, d, e, f] }.should == [1, 2, 6, [], 3, 4]
+      m([1, 2, 3, 4, 5   ]) { |a, b=5, c=6, *d, e, f| [a, b, c, d, e, f] }.should == [1, 2, 3, [], 4, 5]
+      m([1, 2, 3, 4, 5, 6]) { |a, b=5, c=6, *d, e, f| [a, b, c, d, e, f] }.should == [1, 2, 3, [4], 5, 6]
+    end
+
     ruby_version_is "3.2" do
       it "does not autosplat single argument to required arguments when a keyword rest argument is present" do
         m([1, 2]) { |a, **k| [a, k] }.should == [[1, 2], {}]
       end
+
+      it "does not autosplat single argument to required arguments when keyword arguments are present" do
+        m([1, 2]) { |a, b: :b, c: :c| [a, b, c] }.should == [[1, 2], :b, :c]
+      end
+
+      it "raises error when required keyword arguments are present" do
+        -> {
+          m([1, 2]) { |a, b:, c:| [a, b, c] }
+        }.should raise_error(ArgumentError, "missing keywords: :b, :c")
+      end
     end
 
     ruby_version_is ''..."3.2" do
       # https://bugs.ruby-lang.org/issues/18633
       it "autosplats single argument to required arguments when a keyword rest argument is present" do
         m([1, 2]) { |a, **k| [a, k] }.should == [1, {}]
       end
+
+      it "autosplats single argument to required arguments when optional keyword arguments are present" do
+        m([1, 2]) { |a, b: :b, c: :c| [a, b, c] }.should == [1, :b, :c]
+      end
+
+      it "raises error when required keyword arguments are present" do
+        -> {
+          m([1, 2]) { |a, b:, c:| [a, b, c] }
+        }.should raise_error(ArgumentError, "missing keywords: :b, :c")
+      end
     end
 
     it "assigns elements to mixed argument types" do
@@ -368,7 +410,6 @@ def obj.to_ary; raise "Exception raised in #to_ary" end
 
       -> { @y.s(obj) { |a, b| } }.should raise_error(ZeroDivisionError)
     end
-
   end
 
   describe "taking |a, *b| arguments" do
@@ -701,6 +742,42 @@ def obj.to_ary; raise "Exception raised in #to_ary" end
       eval("Proc.new { |_,_| }").should be_an_instance_of(Proc)
     end
   end
+
+  describe 'pre and post parameters' do
+    it "assigns nil to unassigned required arguments" do
+      proc { |a, *b, c, d| [a, b, c, d] }.call(1, 2).should == [1, [], 2, nil]
+    end
+
+    it "assigns elements to optional arguments" do
+      proc { |a=5, b=4, c=3| [a, b, c] }.call(1, 2).should == [1, 2, 3]
+    end
+
+    it "assigns elements to post arguments" do
+      proc { |a=5, b, c, d| [a, b, c, d] }.call(1, 2).should == [5, 1, 2, nil]
+    end
+
+    it "assigns elements to pre arguments" do
+      proc { |a, b, c, d=5| [a, b, c, d] }.call(1, 2).should == [1, 2, nil, 5]
+    end
+
+    it "assigns elements to pre and post arguments" do
+      proc { |a, b=5, c=6, d, e| [a, b, c, d, e] }.call(1               ).should == [1, 5, 6, nil, nil]
+      proc { |a, b=5, c=6, d, e| [a, b, c, d, e] }.call(1, 2            ).should == [1, 5, 6, 2, nil]
+      proc { |a, b=5, c=6, d, e| [a, b, c, d, e] }.call(1, 2, 3         ).should == [1, 5, 6, 2, 3]
+      proc { |a, b=5, c=6, d, e| [a, b, c, d, e] }.call(1, 2, 3, 4      ).should == [1, 2, 6, 3, 4]
+      proc { |a, b=5, c=6, d, e| [a, b, c, d, e] }.call(1, 2, 3, 4, 5   ).should == [1, 2, 3, 4, 5]
+      proc { |a, b=5, c=6, d, e| [a, b, c, d, e] }.call(1, 2, 3, 4, 5, 6).should == [1, 2, 3, 4, 5]
+    end
+
+    it "assigns elements to pre and post arguments when *rest is present" do
+      proc { |a, b=5, c=6, *d, e, f| [a, b, c, d, e, f] }.call(1               ).should == [1, 5, 6, [], nil, nil]
+      proc { |a, b=5, c=6, *d, e, f| [a, b, c, d, e, f] }.call(1, 2            ).should == [1, 5, 6, [], 2, nil]
+      proc { |a, b=5, c=6, *d, e, f| [a, b, c, d, e, f] }.call(1, 2, 3         ).should == [1, 5, 6, [], 2, 3]
+      proc { |a, b=5, c=6, *d, e, f| [a, b, c, d, e, f] }.call(1, 2, 3, 4      ).should == [1, 2, 6, [], 3, 4]
+      proc { |a, b=5, c=6, *d, e, f| [a, b, c, d, e, f] }.call(1, 2, 3, 4, 5   ).should == [1, 2, 3, [], 4, 5]
+      proc { |a, b=5, c=6, *d, e, f| [a, b, c, d, e, f] }.call(1, 2, 3, 4, 5, 6).should == [1, 2, 3, [4], 5, 6]
+    end
+  end
 end
 
 describe "Block-local variables" do
@@ -921,7 +998,7 @@ def a; 1; end
 
 describe "Anonymous block forwarding" do
   ruby_version_is "3.1" do
-    it "forwards blocks to other functions that formally declare anonymous blocks" do
+    it "forwards blocks to other method that formally declares anonymous block" do
       eval <<-EOF
           def b(&); c(&) end
           def c(&); yield :non_null end