diff --git a/Gemfile b/Gemfile index ef2cf7fa..4a76a6f9 100644 --- a/Gemfile +++ b/Gemfile @@ -1,10 +1,6 @@ source 'https://rubygems.org' -if ENV['JSON'] == 'pure' - gemspec name: 'json_pure' -else - gemspec name: 'json' -end +gemspec name: 'json' group :development do gem "ruby_memcheck" if RUBY_PLATFORM =~ /linux/i diff --git a/README.md b/README.md index 65f28424..29624e8c 100644 --- a/README.md +++ b/README.md @@ -5,16 +5,10 @@ ## Description This is an implementation of the JSON specification according to RFC 7159 -http://www.ietf.org/rfc/rfc7159.txt . There is two variants available: +http://www.ietf.org/rfc/rfc7159.txt . -* A pure ruby variant, that relies on the `strscan` extensions, which is - part of the ruby standard library. -* The quite a bit faster native extension variant, which is in parts - implemented in C or Java and comes with a parser generated by the [Ragel] - state machine compiler. - -Both variants of the JSON generator generate UTF-8 character sequences by -default. If an :ascii\_only option with a true value is given, they escape all +The JSON generator generate UTF-8 character sequences by default. +If an :ascii\_only option with a true value is given, they escape all non-ASCII and control characters with \uXXXX escape sequences, and support UTF-16 surrogate pairs in order to be able to generate the whole range of unicode code points. diff --git a/Rakefile b/Rakefile index 7a013eb0..e16fc8d3 100644 --- a/Rakefile +++ b/Rakefile @@ -56,12 +56,8 @@ else RAGEL_DOTGEN = %w[rlgen-dot rlgen-cd ragel].find(&which) end -desc "Installing library (pure)" -task :install_pure do - ruby 'install.rb' -end - -task :install_ext_really do +desc "Installing library (extension)" +task :install => [ :compile ] do sitearchdir = CONFIG["sitearchdir"] cd 'ext' do for file in Dir["json/ext/*.#{CONFIG['DLEXT']}"] @@ -73,30 +69,6 @@ task :install_ext_really do end end -desc "Installing library (extension)" -task :install_ext => [ :compile, :install_pure, :install_ext_really ] - -desc "Installing library (extension)" -task :install => :install_ext - -task :check_env do - ENV.key?('JSON') or fail "JSON env var is required" -end - -desc "Testing library (pure ruby)" -task :test_pure => [ :set_env_pure, :check_env, :do_test_pure ] -task(:set_env_pure) { ENV['JSON'] = 'pure' } - -UndocumentedTestTask.new do |t| - t.name = 'do_test_pure' - t.test_files = FileList['test/json/*_test.rb'] - t.verbose = true - t.options = '-v' -end - -desc "Testing library (pure ruby and extension)" -task :test => [ :test_pure, :test_ext ] - namespace :gems do desc 'Install all development gems' task :install do @@ -177,16 +149,14 @@ if defined?(RUBY_ENGINE) and RUBY_ENGINE == 'jruby' sh "gem build -o pkg/json-#{PKG_VERSION}-java.gem json.gemspec" end - desc "Testing library (jruby)" - task :test_ext => [ :set_env_ext, :create_jar, :check_env, :do_test_ext ] - task(:set_env_ext) { ENV['JSON'] = 'ext' } - UndocumentedTestTask.new do |t| - t.name = 'do_test_ext' + t.name = :test t.test_files = FileList['test/json/*_test.rb'] t.verbose = true t.options = '-v' end + desc "Testing library (jruby)" + task :test => [:create_jar ] file JRUBY_PARSER_JAR => :compile do cd 'java/src' do @@ -239,20 +209,19 @@ else task :compile => [ :ragel, EXT_PARSER_DL, EXT_GENERATOR_DL ] end - desc "Testing library (extension)" - task :test_ext => [ :set_env_ext, :check_env, :compile, :do_test_ext ] - task(:set_env_ext) { ENV['JSON'] = 'ext' } - UndocumentedTestTask.new do |t| - t.name = 'do_test_ext' + t.name = :test t.test_files = FileList['test/json/*_test.rb'] t.verbose = true t.options = '-v' end + desc "Testing library (extension)" + task :test => [ :compile ] + begin require "ruby_memcheck" - RubyMemcheck::TestTask.new(valgrind: [ :set_env_ext, :check_env, :compile, :do_test_ext ]) do |t| + RubyMemcheck::TestTask.new(valgrind: [ :compile, :test ]) do |t| t.test_files = FileList['test/json/*_test.rb'] t.verbose = true t.options = '-v' diff --git a/json_pure.gemspec b/json_pure.gemspec index 37b437c4..21d39d02 100644 --- a/json_pure.gemspec +++ b/json_pure.gemspec @@ -23,28 +23,7 @@ Gem::Specification.new do |s| "LEGAL", "README.md", "json_pure.gemspec", - "lib/json.rb", - "lib/json/add/bigdecimal.rb", - "lib/json/add/complex.rb", - "lib/json/add/core.rb", - "lib/json/add/date.rb", - "lib/json/add/date_time.rb", - "lib/json/add/exception.rb", - "lib/json/add/ostruct.rb", - "lib/json/add/range.rb", - "lib/json/add/rational.rb", - "lib/json/add/regexp.rb", - "lib/json/add/set.rb", - "lib/json/add/struct.rb", - "lib/json/add/symbol.rb", - "lib/json/add/time.rb", - "lib/json/common.rb", - "lib/json/ext.rb", - "lib/json/generic_object.rb", "lib/json/pure.rb", - "lib/json/pure/generator.rb", - "lib/json/pure/parser.rb", - "lib/json/version.rb", ] s.homepage = "https://ruby.github.io/json" s.metadata = { @@ -56,5 +35,7 @@ Gem::Specification.new do |s| 'wiki_uri' => 'https://github.com/ruby/json/wiki' } + s.add_dependency "json" + s.required_ruby_version = Gem::Requirement.new(">= 2.7") end diff --git a/lib/json.rb b/lib/json.rb index c28e853e..dfd9b7df 100644 --- a/lib/json.rb +++ b/lib/json.rb @@ -583,10 +583,5 @@ # module JSON require 'json/version' - - begin - require 'json/ext' - rescue LoadError - require 'json/pure' - end + require 'json/ext' end diff --git a/lib/json/common.rb b/lib/json/common.rb index 84f2a57c..2269896b 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -32,9 +32,7 @@ def [](object, opts = {}) JSON.generate(object, opts) end - # Returns the JSON parser class that is used by JSON. This is either - # JSON::Ext::Parser or JSON::Pure::Parser: - # JSON.parser # => JSON::Ext::Parser + # Returns the JSON parser class that is used by JSON. attr_reader :parser # Set the JSON parser class _parser_ to be used by JSON. @@ -97,14 +95,10 @@ def create_pretty_state ) end - # Returns the JSON generator module that is used by JSON. This is - # either JSON::Ext::Generator or JSON::Pure::Generator: - # JSON.generator # => JSON::Ext::Generator + # Returns the JSON generator module that is used by JSON. attr_reader :generator - # Sets or Returns the JSON generator state class that is used by JSON. This is - # either JSON::Ext::Generator::State or JSON::Pure::Generator::State: - # JSON.state # => JSON::Ext::Generator::State + # Sets or Returns the JSON generator state class that is used by JSON. attr_accessor :state end diff --git a/lib/json/ext.rb b/lib/json/ext.rb index 92ef61ea..2082cae6 100644 --- a/lib/json/ext.rb +++ b/lib/json/ext.rb @@ -8,14 +8,12 @@ module JSON module Ext if RUBY_ENGINE == 'truffleruby' require 'json/ext/parser' - require 'json/pure' - $DEBUG and warn "Using Ext extension for JSON parser and Pure library for JSON generator." + require 'json/truffle_ruby/generator' JSON.parser = Parser - JSON.generator = JSON::Pure::Generator + JSON.generator = ::JSON::TruffleRuby::Generator else require 'json/ext/parser' require 'json/ext/generator' - $DEBUG and warn "Using Ext extension for JSON." JSON.parser = Parser JSON.generator = Generator end diff --git a/lib/json/pure.rb b/lib/json/pure.rb index 69d2256d..78a6d9dc 100644 --- a/lib/json/pure.rb +++ b/lib/json/pure.rb @@ -1,16 +1,4 @@ # frozen_string_literal: true -require 'json/common' -module JSON - # This module holds all the modules/classes that implement JSON's - # functionality in pure ruby. - module Pure - require 'json/pure/parser' - require 'json/pure/generator' - $DEBUG and warn "Using Pure library for JSON." - JSON.parser = Parser - JSON.generator = Generator - end - - JSON_LOADED = true unless defined?(::JSON::JSON_LOADED) -end +warn "`json_pure` is deprecated and has no effect, just use `json`" +require "json" diff --git a/lib/json/pure/parser.rb b/lib/json/pure/parser.rb deleted file mode 100644 index 36ef75ca..00000000 --- a/lib/json/pure/parser.rb +++ /dev/null @@ -1,356 +0,0 @@ -#frozen_string_literal: true -require 'strscan' - -module JSON - module Pure - # This class implements the JSON parser that is used to parse a JSON string - # into a Ruby data structure. - class Parser < StringScanner - STRING = /" ((?:[^\x0-\x1f"\\] | - # escaped special characters: - \\["\\\/bfnrt] | - \\u[0-9a-fA-F]{4} | - # match all but escaped special characters: - \\[\x20-\x21\x23-\x2e\x30-\x5b\x5d-\x61\x63-\x65\x67-\x6d\x6f-\x71\x73\x75-\xff])*) - "/nx - INTEGER = /(-?0|-?[1-9]\d*)/ - FLOAT = /(-? - (?:0|[1-9]\d*) - (?: - \.\d+(?i:e[+-]?\d+) | - \.\d+ | - (?i:e[+-]?\d+) - ) - )/x - NAN = /NaN/ - INFINITY = /Infinity/ - MINUS_INFINITY = /-Infinity/ - OBJECT_OPEN = /\{/ - OBJECT_CLOSE = /\}/ - ARRAY_OPEN = /\[/ - ARRAY_CLOSE = /\]/ - PAIR_DELIMITER = /:/ - COLLECTION_DELIMITER = /,/ - TRUE = /true/ - FALSE = /false/ - NULL = /null/ - IGNORE = %r( - (?: - //[^\n\r]*[\n\r]| # line comments - /\* # c-style comments - (?: - [\s\S]*? # any char, repeated lazily - ) - \*/ # the End of this comment - |[ \t\r\n]+ # whitespaces: space, horizontal tab, lf, cr - )+ - )mx - - UNPARSED = Object.new.freeze - - class << self - def parse(source, opts = nil) - if opts.nil? - new(source).parse - else - # NB: The ** shouldn't be required, but we have to deal with - # different versions of the `json` and `json_pure` gems being - # loaded concurrently. - # Prior to 2.7.3, `JSON::Ext::Parser` would only take kwargs. - # Ref: https://github.com/ruby/json/issues/650 - new(source, **opts).parse - end - end - end - - # Creates a new JSON::Pure::Parser instance for the string _source_. - # - # It will be configured by the _opts_ hash. _opts_ can have the following - # keys: - # * *max_nesting*: The maximum depth of nesting allowed in the parsed data - # structures. Disable depth checking with :max_nesting => false|nil|0, - # it defaults to 100. - # * *allow_nan*: If set to true, allow NaN, Infinity and -Infinity in - # defiance of RFC 7159 to be parsed by the Parser. This option defaults - # to false. - # * *allow_trailing_comma*: If set to true, allow arrays and objects with a - # trailing comma in defiance of RFC 7159 to be parsed by the Parser. - # This option defaults to false. - # * *freeze*: If set to true, all parsed objects will be frozen. Parsed - # string will be deduplicated if possible. - # * *symbolize_names*: If set to true, returns symbols for the names - # (keys) in a JSON object. Otherwise strings are returned, which is - # also the default. It's not possible to use this option in - # conjunction with the *create_additions* option. - # * *create_additions*: If set to true, the Parser creates - # additions when a matching class and create_id are found. This - # option defaults to false. - # * *object_class*: Defaults to Hash. If another type is provided, it will be used - # instead of Hash to represent JSON objects. The type must respond to - # +new+ without arguments, and return an object that respond to +[]=+. - # * *array_class*: Defaults to Array If another type is provided, it will be used - # instead of Hash to represent JSON arrays. The type must respond to - # +new+ without arguments, and return an object that respond to +<<+. - # * *decimal_class*: Specifies which class to use instead of the default - # (Float) when parsing decimal numbers. This class must accept a single - # string argument in its constructor. - def initialize(source, opts = nil) - opts ||= {} - source = convert_encoding source - super source - if !opts.key?(:max_nesting) # defaults to 100 - @max_nesting = 100 - elsif opts[:max_nesting] - @max_nesting = opts[:max_nesting] - else - @max_nesting = 0 - end - @allow_nan = !!opts[:allow_nan] - @allow_trailing_comma = !!opts[:allow_trailing_comma] - @symbolize_names = !!opts[:symbolize_names] - @freeze = !!opts[:freeze] - - @deprecated_create_additions = false - @create_additions = opts.fetch(:create_additions, false) - if @create_additions.nil? - @create_additions = true - @deprecated_create_additions = true - end - - @symbolize_names && @create_additions and raise ArgumentError, - 'options :symbolize_names and :create_additions cannot be used '\ - 'in conjunction' - @create_id = @create_additions ? JSON.create_id : nil - @object_class = opts[:object_class] || Hash - @array_class = opts[:array_class] || Array - @decimal_class = opts[:decimal_class] - @match_string = opts[:match_string] - end - - alias source string - - def reset - super - @current_nesting = 0 - end - - # Parses the current JSON string _source_ and returns the - # complete data structure as a result. - def parse - reset - obj = nil - while !eos? && skip(IGNORE) do end - if eos? - raise ParserError, "source is not valid JSON!" - else - obj = parse_value - UNPARSED.equal?(obj) and raise ParserError, - "source is not valid JSON!" - obj.freeze if @freeze - end - while !eos? && skip(IGNORE) do end - eos? or raise ParserError, "source is not valid JSON!" - obj - end - - private - - def convert_encoding(source) - if source.respond_to?(:to_str) - source = source.to_str - else - raise TypeError, - "#{source.inspect} is not like a string" - end - if source.encoding != ::Encoding::BINARY - source = source.encode(::Encoding::UTF_8) - source.force_encoding(::Encoding::BINARY) - end - source - end - - # Unescape characters in strings. - UNESCAPE_MAP = { - '"' => '"', - '\\' => '\\', - '/' => '/', - 'b' => "\b", - 'f' => "\f", - 'n' => "\n", - 'r' => "\r", - 't' => "\t", - 'u' => nil, - }.freeze - - def parse_string - if scan(STRING) - return '' if self[1].empty? - string = self[1].gsub(%r{(?:\\[\\bfnrt"/]|(?:\\u(?:[A-Fa-f\d]{4}))+|\\[\x20-\xff])}n) do |c| - k = $&[1] - if u = UNESCAPE_MAP.fetch(k) { k.chr } - u - else # \uXXXX - bytes = ''.b - i = 0 - while c[6 * i] == ?\\ && c[6 * i + 1] == ?u - bytes << c[6 * i + 2, 2].to_i(16) << c[6 * i + 4, 2].to_i(16) - i += 1 - end - bytes.encode(Encoding::UTF_8, Encoding::UTF_16BE).force_encoding(::Encoding::BINARY) - end - end - string.force_encoding(::Encoding::UTF_8) - - if @freeze - string = -string - end - - if @create_additions and @match_string - for (regexp, klass) in @match_string - if klass.json_creatable? and string.match?(regexp) - if @deprecated_create_additions - warn "JSON.load implicit support for `create_additions: true` is deprecated and will be removed in 3.0, use JSON.unsafe_load or explicitly pass `create_additions: true`" - end - - return klass.json_create(string) - end - end - end - string - else - UNPARSED - end - rescue => e - raise ParserError, "Caught #{e.class} at '#{peek(20)}': #{e}" - end - - def parse_value - case - when scan(FLOAT) - if @decimal_class then - if @decimal_class == BigDecimal then - BigDecimal(self[1]) - else - @decimal_class.new(self[1]) || Float(self[1]) - end - else - Float(self[1]) - end - when scan(INTEGER) - Integer(self[1]) - when scan(TRUE) - true - when scan(FALSE) - false - when scan(NULL) - nil - when !UNPARSED.equal?(string = parse_string) - string - when scan(ARRAY_OPEN) - @current_nesting += 1 - ary = parse_array - @current_nesting -= 1 - ary - when scan(OBJECT_OPEN) - @current_nesting += 1 - obj = parse_object - @current_nesting -= 1 - obj - when @allow_nan && scan(NAN) - NaN - when @allow_nan && scan(INFINITY) - Infinity - when @allow_nan && scan(MINUS_INFINITY) - MinusInfinity - else - UNPARSED - end - end - - def parse_array - raise NestingError, "nesting of #@current_nesting is too deep" if - @max_nesting.nonzero? && @current_nesting > @max_nesting - result = @array_class.new - delim = false - loop do - case - when eos? - raise ParserError, "unexpected end of string while parsing array" - when !UNPARSED.equal?(value = parse_value) - delim = false - result << value - skip(IGNORE) - if scan(COLLECTION_DELIMITER) - delim = true - elsif match?(ARRAY_CLOSE) - ; - else - raise ParserError, "expected ',' or ']' in array at '#{peek(20)}'!" - end - when scan(ARRAY_CLOSE) - if delim && !@allow_trailing_comma - raise ParserError, "expected next element in array at '#{peek(20)}'!" - end - break - when skip(IGNORE) - ; - else - raise ParserError, "unexpected token in array at '#{peek(20)}'!" - end - end - result - end - - def parse_object - raise NestingError, "nesting of #@current_nesting is too deep" if - @max_nesting.nonzero? && @current_nesting > @max_nesting - result = @object_class.new - delim = false - loop do - case - when eos? - raise ParserError, "unexpected end of string while parsing object" - when !UNPARSED.equal?(string = parse_string) - skip(IGNORE) - unless scan(PAIR_DELIMITER) - raise ParserError, "expected ':' in object at '#{peek(20)}'!" - end - skip(IGNORE) - unless UNPARSED.equal?(value = parse_value) - result[@symbolize_names ? string.to_sym : string] = value - delim = false - skip(IGNORE) - if scan(COLLECTION_DELIMITER) - delim = true - elsif match?(OBJECT_CLOSE) - ; - else - raise ParserError, "expected ',' or '}' in object at '#{peek(20)}'!" - end - else - raise ParserError, "expected value in object at '#{peek(20)}'!" - end - when scan(OBJECT_CLOSE) - if delim && !@allow_trailing_comma - raise ParserError, "expected next name, value pair in object at '#{peek(20)}'!" - end - if @create_additions and klassname = result[@create_id] - klass = JSON.deep_const_get(klassname) - break unless klass and klass.json_creatable? - if @deprecated_create_additions - warn "JSON.load implicit support for `create_additions: true` is deprecated and will be removed in 3.0, use JSON.unsafe_load or explicitly pass `create_additions: true`" - end - result = klass.json_create(result) - end - break - when skip(IGNORE) - ; - else - raise ParserError, "unexpected token in object at '#{peek(20)}'!" - end - end - result - end - end - end -end diff --git a/lib/json/pure/generator.rb b/lib/json/truffle_ruby/generator.rb similarity index 85% rename from lib/json/pure/generator.rb rename to lib/json/truffle_ruby/generator.rb index 5b4c8325..b0f3e420 100644 --- a/lib/json/pure/generator.rb +++ b/lib/json/truffle_ruby/generator.rb @@ -1,98 +1,98 @@ # frozen_string_literal: true module JSON - MAP = { - "\x0" => '\u0000', - "\x1" => '\u0001', - "\x2" => '\u0002', - "\x3" => '\u0003', - "\x4" => '\u0004', - "\x5" => '\u0005', - "\x6" => '\u0006', - "\x7" => '\u0007', - "\b" => '\b', - "\t" => '\t', - "\n" => '\n', - "\xb" => '\u000b', - "\f" => '\f', - "\r" => '\r', - "\xe" => '\u000e', - "\xf" => '\u000f', - "\x10" => '\u0010', - "\x11" => '\u0011', - "\x12" => '\u0012', - "\x13" => '\u0013', - "\x14" => '\u0014', - "\x15" => '\u0015', - "\x16" => '\u0016', - "\x17" => '\u0017', - "\x18" => '\u0018', - "\x19" => '\u0019', - "\x1a" => '\u001a', - "\x1b" => '\u001b', - "\x1c" => '\u001c', - "\x1d" => '\u001d', - "\x1e" => '\u001e', - "\x1f" => '\u001f', - '"' => '\"', - '\\' => '\\\\', - }.freeze # :nodoc: - - ESCAPE_PATTERN = /[\/"\\\x0-\x1f]/n # :nodoc: - - SCRIPT_SAFE_MAP = MAP.merge( - '/' => '\\/', - "\u2028".b => '\u2028', - "\u2029".b => '\u2029', - ).freeze - - SCRIPT_SAFE_ESCAPE_PATTERN = Regexp.union(ESCAPE_PATTERN, "\u2028".b, "\u2029".b) - - # Convert a UTF8 encoded Ruby string _string_ to a JSON string, encoded with - # UTF16 big endian characters as \u????, and return it. - def utf8_to_json(string, script_safe = false) # :nodoc: - string = string.b - if script_safe - string.gsub!(SCRIPT_SAFE_ESCAPE_PATTERN) { SCRIPT_SAFE_MAP[$&] || $& } - else - string.gsub!(ESCAPE_PATTERN) { MAP[$&] || $& } - end - string.force_encoding(::Encoding::UTF_8) - string - end + module TruffleRuby + module Generator + MAP = { + "\x0" => '\u0000', + "\x1" => '\u0001', + "\x2" => '\u0002', + "\x3" => '\u0003', + "\x4" => '\u0004', + "\x5" => '\u0005', + "\x6" => '\u0006', + "\x7" => '\u0007', + "\b" => '\b', + "\t" => '\t', + "\n" => '\n', + "\xb" => '\u000b', + "\f" => '\f', + "\r" => '\r', + "\xe" => '\u000e', + "\xf" => '\u000f', + "\x10" => '\u0010', + "\x11" => '\u0011', + "\x12" => '\u0012', + "\x13" => '\u0013', + "\x14" => '\u0014', + "\x15" => '\u0015', + "\x16" => '\u0016', + "\x17" => '\u0017', + "\x18" => '\u0018', + "\x19" => '\u0019', + "\x1a" => '\u001a', + "\x1b" => '\u001b', + "\x1c" => '\u001c', + "\x1d" => '\u001d', + "\x1e" => '\u001e', + "\x1f" => '\u001f', + '"' => '\"', + '\\' => '\\\\', + }.freeze # :nodoc: + + ESCAPE_PATTERN = /[\/"\\\x0-\x1f]/n # :nodoc: + + SCRIPT_SAFE_MAP = MAP.merge( + '/' => '\\/', + "\u2028".b => '\u2028', + "\u2029".b => '\u2029', + ).freeze + + SCRIPT_SAFE_ESCAPE_PATTERN = Regexp.union(ESCAPE_PATTERN, "\u2028".b, "\u2029".b) + + # Convert a UTF8 encoded Ruby string _string_ to a JSON string, encoded with + # UTF16 big endian characters as \u????, and return it. + def utf8_to_json(string, script_safe = false) # :nodoc: + string = string.b + if script_safe + string.gsub!(SCRIPT_SAFE_ESCAPE_PATTERN) { SCRIPT_SAFE_MAP[$&] || $& } + else + string.gsub!(ESCAPE_PATTERN) { MAP[$&] || $& } + end + string.force_encoding(::Encoding::UTF_8) + string + end - def utf8_to_json_ascii(string, script_safe = false) # :nodoc: - string = string.b - map = script_safe ? SCRIPT_SAFE_MAP : MAP - string.gsub!(/[\/"\\\x0-\x1f]/n) { map[$&] || $& } - string.gsub!(/( - (?: - [\xc2-\xdf][\x80-\xbf] | - [\xe0-\xef][\x80-\xbf]{2} | - [\xf0-\xf4][\x80-\xbf]{3} - )+ | - [\x80-\xc1\xf5-\xff] # invalid - )/nx) { |c| - c.size == 1 and raise GeneratorError, "invalid utf8 byte: '#{c}'" - s = c.encode(::Encoding::UTF_16BE, ::Encoding::UTF_8).unpack('H*')[0] - s.force_encoding(::Encoding::BINARY) - s.gsub!(/.{4}/n, '\\\\u\&') - s.force_encoding(::Encoding::UTF_8) - } - string.force_encoding(::Encoding::UTF_8) - string - rescue => e - raise GeneratorError.wrap(e) - end + def utf8_to_json_ascii(string, script_safe = false) # :nodoc: + string = string.b + map = script_safe ? SCRIPT_SAFE_MAP : MAP + string.gsub!(/[\/"\\\x0-\x1f]/n) { map[$&] || $& } + string.gsub!(/( + (?: + [\xc2-\xdf][\x80-\xbf] | + [\xe0-\xef][\x80-\xbf]{2} | + [\xf0-\xf4][\x80-\xbf]{3} + )+ | + [\x80-\xc1\xf5-\xff] # invalid + )/nx) { |c| + c.size == 1 and raise GeneratorError, "invalid utf8 byte: '#{c}'" + s = c.encode(::Encoding::UTF_16BE, ::Encoding::UTF_8).unpack('H*')[0] + s.force_encoding(::Encoding::BINARY) + s.gsub!(/.{4}/n, '\\\\u\&') + s.force_encoding(::Encoding::UTF_8) + } + string.force_encoding(::Encoding::UTF_8) + string + rescue => e + raise GeneratorError.wrap(e) + end - def valid_utf8?(string) - encoding = string.encoding - (encoding == Encoding::UTF_8 || encoding == Encoding::ASCII) && - string.valid_encoding? - end - module_function :utf8_to_json, :utf8_to_json_ascii, :valid_utf8? + def valid_utf8?(string) + encoding = string.encoding + (encoding == Encoding::UTF_8 || encoding == Encoding::ASCII) && + string.valid_encoding? + end + module_function :utf8_to_json, :utf8_to_json_ascii, :valid_utf8? - module Pure - module Generator # This class is used to create State instances, that are use to hold data # while generating a JSON text from a Ruby data structure. class State @@ -300,7 +300,7 @@ def generate(obj) else result = obj.to_json(self) end - JSON.valid_utf8?(result) or raise GeneratorError, + JSON::TruffleRuby::Generator.valid_utf8?(result) or raise GeneratorError, "source sequence #{result.inspect} is illegal/malformed utf-8" result end @@ -559,9 +559,9 @@ def to_json(state = nil, *args) string = encode(::Encoding::UTF_8) end if state.ascii_only? - %("#{JSON.utf8_to_json_ascii(string, state.script_safe)}") + %("#{JSON::TruffleRuby::Generator.utf8_to_json_ascii(string, state.script_safe)}") else - %("#{JSON.utf8_to_json(string, state.script_safe)}") + %("#{JSON::TruffleRuby::Generator.utf8_to_json(string, state.script_safe)}") end rescue Encoding::UndefinedConversionError => error raise ::JSON::GeneratorError, error.message diff --git a/test/json/json_common_interface_test.rb b/test/json/json_common_interface_test.rb index e552412b..6165cc04 100644 --- a/test/json/json_common_interface_test.rb +++ b/test/json/json_common_interface_test.rb @@ -52,11 +52,11 @@ def test_parser end def test_generator - assert_match(/::Generator\z/, JSON.generator.name) + assert_match(/::(TruffleRuby)?Generator\z/, JSON.generator.name) end def test_state - assert_match(/::Generator::State\z/, JSON.state.name) + assert_match(/::(TruffleRuby)?Generator::State\z/, JSON.state.name) end def test_create_id diff --git a/test/json/json_ext_parser_test.rb b/test/json/json_ext_parser_test.rb index 9db8ae77..da615049 100644 --- a/test/json/json_ext_parser_test.rb +++ b/test/json/json_ext_parser_test.rb @@ -2,53 +2,51 @@ require_relative 'test_helper' class JSONExtParserTest < Test::Unit::TestCase - if defined?(JSON::Ext::Parser) - include JSON - - def test_allocate - parser = JSON::Ext::Parser.new("{}") - assert_raise(TypeError, '[ruby-core:35079]') do - parser.__send__(:initialize, "{}") - end - parser = JSON::Ext::Parser.allocate - assert_raise(TypeError, '[ruby-core:35079]') { parser.source } - end + include JSON - def test_error_messages - ex = assert_raise(ParserError) { parse('Infinity') } - assert_equal "unexpected token at 'Infinity'", ex.message + def test_allocate + parser = JSON::Ext::Parser.new("{}") + assert_raise(TypeError, '[ruby-core:35079]') do + parser.__send__(:initialize, "{}") + end + parser = JSON::Ext::Parser.allocate + assert_raise(TypeError, '[ruby-core:35079]') { parser.source } + end - unless RUBY_PLATFORM =~ /java/ - ex = assert_raise(ParserError) { parse('-Infinity') } - assert_equal "unexpected token at '-Infinity'", ex.message - end + def test_error_messages + ex = assert_raise(ParserError) { parse('Infinity') } + assert_equal "unexpected token at 'Infinity'", ex.message - ex = assert_raise(ParserError) { parse('NaN') } - assert_equal "unexpected token at 'NaN'", ex.message + unless RUBY_PLATFORM =~ /java/ + ex = assert_raise(ParserError) { parse('-Infinity') } + assert_equal "unexpected token at '-Infinity'", ex.message end - if GC.respond_to?(:stress=) - def test_gc_stress_parser_new - payload = JSON.dump([{ foo: 1, bar: 2, baz: 3, egg: { spam: 4 } }] * 10) - - previous_stress = GC.stress - JSON::Parser.new(payload).parse - ensure - GC.stress = previous_stress - end + ex = assert_raise(ParserError) { parse('NaN') } + assert_equal "unexpected token at 'NaN'", ex.message + end - def test_gc_stress - payload = JSON.dump([{ foo: 1, bar: 2, baz: 3, egg: { spam: 4 } }] * 10) + if GC.respond_to?(:stress=) + def test_gc_stress_parser_new + payload = JSON.dump([{ foo: 1, bar: 2, baz: 3, egg: { spam: 4 } }] * 10) - previous_stress = GC.stress - JSON.parse(payload) - ensure - GC.stress = previous_stress - end + previous_stress = GC.stress + JSON::Parser.new(payload).parse + ensure + GC.stress = previous_stress end - def parse(json) - JSON::Ext::Parser.new(json).parse + def test_gc_stress + payload = JSON.dump([{ foo: 1, bar: 2, baz: 3, egg: { spam: 4 } }] * 10) + + previous_stress = GC.stress + JSON.parse(payload) + ensure + GC.stress = previous_stress end end + + def parse(json) + JSON::Ext::Parser.new(json).parse + end end diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index 3bc7eb80..700220a1 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -343,27 +343,25 @@ def foo.to_h assert_equal '2', state.indent end - if defined?(JSON::Ext::Generator) - def test_broken_bignum # [ruby-core:38867] - pid = fork do - x = 1 << 64 - x.class.class_eval do - def to_s - end - end - begin - JSON::Ext::Generator::State.new.generate(x) - exit 1 - rescue TypeError - exit 0 + def test_broken_bignum # [ruby-core:38867] + pid = fork do + x = 1 << 64 + x.class.class_eval do + def to_s end end - _, status = Process.waitpid2(pid) - assert status.success? - rescue NotImplementedError - # forking to avoid modifying core class of a parent process and - # introducing race conditions of tests are run in parallel + begin + JSON::Ext::Generator::State.new.generate(x) + exit 1 + rescue TypeError + exit 0 + end end + _, status = Process.waitpid2(pid) + assert status.success? + rescue NotImplementedError + # forking to avoid modifying core class of a parent process and + # introducing race conditions of tests are run in parallel end def test_hash_likeness_set_symbol diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index 9cbaa42f..8759ccd2 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -40,7 +40,7 @@ def test_error_message_encoding } assert_equal(Encoding::UTF_8, e.message.encoding, bug10705) assert_include(e.message, json, bug10705) - end if defined?(JSON::Ext::Parser) + end def test_parsing parser = JSON::Parser.new('"test"') @@ -619,7 +619,7 @@ def test_parse_error_incomplete_hash error = assert_raise(JSON::ParserError) do JSON.parse('{"input":{"firstName":"Bob","lastName":"Mob","email":"bob@example.com"}') end - if RUBY_ENGINE == "ruby" && defined?(JSON::Ext) + if RUBY_ENGINE == "ruby" assert_equal %(unexpected token at '{"input":{"firstName":"Bob","las'), error.message end end diff --git a/test/json/test_helper.rb b/test/json/test_helper.rb index f81eeec1..11bb8ba8 100644 --- a/test/json/test_helper.rb +++ b/test/json/test_helper.rb @@ -1,23 +1,7 @@ -case ENV['JSON'] -when 'pure' - $LOAD_PATH.unshift(File.expand_path('../../../lib', __FILE__)) - $stderr.puts("Testing JSON::Pure") - require 'json/pure' -when 'ext' - $stderr.puts("Testing JSON::Ext") - $LOAD_PATH.unshift(File.expand_path('../../../ext', __FILE__), File.expand_path('../../../lib', __FILE__)) - require 'json/ext' -else - $LOAD_PATH.unshift(File.expand_path('../../../ext', __FILE__), File.expand_path('../../../lib', __FILE__)) - $stderr.puts("Testing JSON") - require 'json' -end +$LOAD_PATH.unshift(File.expand_path('../../../ext', __FILE__), File.expand_path('../../../lib', __FILE__)) +require 'json' require 'test/unit' -begin - require 'byebug' -rescue LoadError -end if GC.respond_to?(:verify_compaction_references) # This method was added in Ruby 3.0.0. Calling it this way asks the GC to