Introduce support for the prism parser

Prism will be the parser in Ruby 3.3. We need to support 3.0+ so we will have to "dual boot" both parsers.

Todo: 

- Add tests that exercise both Ripper and prism codepaths on CI
- Handle ruby/prism#1972 in `ripper_errors.rb`
- Update docs to not mention Ripper explicitly
- Consider different/cleaner APIs for separating out Ripper and Prism

Gemfile

@@ -12,3 +12,4 @@ gem "standard"
 gem "ruby-prof"
 
 gem "benchmark-ips"
+gem "prism"

Gemfile.lock

@@ -12,6 +12,7 @@ GEM
     parallel (1.21.0)
     parser (3.0.2.0)
       ast (~> 2.4.1)
+    prism (0.18.0)
     rainbow (3.0.0)
     rake (12.3.3)
     regexp_parser (2.1.1)
@@ -56,6 +57,7 @@ PLATFORMS
 
 DEPENDENCIES
   benchmark-ips
+  prism
   rake (~> 12.0)
   rspec (~> 3.0)
   ruby-prof

lib/syntax_suggest/api.rb

@@ -5,7 +5,17 @@
 require "tmpdir"
 require "stringio"
 require "pathname"
-require "ripper"
+
+if ENV["SYNTAX_SUGGEST_DISABLE_PRISM"] # For testing dual ripper/prism support
+  require "ripper"
+else
+  begin
+    require "prism"
+  rescue LoadError => e
+    require "ripper"
+  end
+end
+
 require "timeout"
 
 module SyntaxSuggest
@@ -16,6 +26,14 @@ module SyntaxSuggest
   class Error < StandardError; end
   TIMEOUT_DEFAULT = ENV.fetch("SYNTAX_SUGGEST_TIMEOUT", 1).to_i
 
+  # SyntaxSuggest.use_prism_parser? [Private]
+  #
+  # Tells us if the prism parser is available for use
+  # or if we should fallback to `Ripper`
+  def self.use_prism_parser?
+    defined?(Prism)
+  end
+
   # SyntaxSuggest.handle_error [Public]
   #
   # Takes a `SyntaxError` exception, uses the
@@ -129,11 +147,20 @@ def self.valid_without?(without_lines:, code_lines:)
   # SyntaxSuggest.invalid? [Private]
   #
   # Opposite of `SyntaxSuggest.valid?`
-  def self.invalid?(source)
-    source = source.join if source.is_a?(Array)
-    source = source.to_s
+  if defined?(Prism)
+    def self.invalid?(source)
+      source = source.join if source.is_a?(Array)
+      source = source.to_s
 
-    Ripper.new(source).tap(&:parse).error?
+      Prism.parse(source).failure?
+    end
+  else
+    def self.invalid?(source)
+      source = source.join if source.is_a?(Array)
+      source = source.to_s
+
+      Ripper.new(source).tap(&:parse).error?
+    end
   end
 
   # SyntaxSuggest.valid? [Private]

lib/syntax_suggest/lex_all.rb

@@ -20,17 +20,28 @@ def initialize(source:, source_lines: nil)
         lines = source_lines[lineno..-1]
 
         @lex.concat(
-          Ripper::Lexer.new(lines.join, "-", lineno + 1).parse.sort_by(&:pos)
+          self.class.lex(lines.join, lineno + 1)
         )
-        lineno = @lex.last.pos.first + 1
+
+        lineno = @lex.last[0].first + 1
       end
 
       last_lex = nil
       @lex.map! { |elem|
-        last_lex = LexValue.new(elem.pos.first, elem.event, elem.tok, elem.state, last_lex)
+        last_lex = LexValue.new(elem[0].first, elem[1], elem[2], elem[3], last_lex)
       }
     end
 
+    if SyntaxSuggest.use_prism_parser?
+      def self.lex(source, line_number)
+        Prism.lex_compat(source, line: line_number).value.sort_by {|values| values[0] }
+      end
+    else
+      def self.lex(source, line_number)
+        Ripper::Lexer.new(lines.join, "-", line_number).parse.sort_by(&:pos)
+      end
+    end
+
     def to_a
       @lex
     end

lib/syntax_suggest/ripper_errors.rb

@@ -7,7 +7,7 @@ module SyntaxSuggest
   #
   #   puts RipperErrors.new(" def foo").call.errors
   #   # => ["syntax error, unexpected end-of-input, expecting ';' or '\\n'"]
-  class RipperErrors < Ripper
+  class RipperErrors < SyntaxSuggest.use_prism_parser? ? Prism::RipperCompat : Ripper
     attr_reader :errors
 
     # Comes from ripper, called

syntax_suggest.gemspec

@@ -16,7 +16,7 @@ Gem::Specification.new do |spec|
   spec.description = 'When you get an "unexpected end" in your syntax this gem helps you find it'
   spec.homepage = "https://github.com/ruby/syntax_suggest.git"
   spec.license = "MIT"
-  spec.required_ruby_version = Gem::Requirement.new(">= 2.5.0")
+  spec.required_ruby_version = Gem::Requirement.new(">= 3.0.0")
 
   spec.metadata["homepage_uri"] = spec.homepage
   spec.metadata["source_code_uri"] = "https://github.com/ruby/syntax_suggest.git"