From e68e38fbb6a4da536480c71f6384f24cbdee909e Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Tue, 26 Nov 2024 12:23:16 +0100 Subject: [PATCH] StringScanner#scan_integer support base 16 integers Followup: https://github.com/ruby/strscan/pull/115 `scan_integer` is now implemented in Ruby as to efficiently handle keyword arguments without allocating a Hash. Given the goal of `scan_integer` is to more effciently parse integers without having to allocate an intermediary object, using `rb_scan_args` would defeat the purpose. Additionally, the C implementation now uses `rb_isdigit` and `rb_isxdigit`, because on Windows `isdigit` is locale dependent. --- ext/jruby/lib/strscan.rb | 2 - .../jruby/ext/strscan/RubyStringScanner.java | 45 +++++++++++- ext/strscan/strscan.c | 68 +++++++++++++++---- lib/strscan.rb | 32 +++++++++ strscan.gemspec | 8 +-- test/strscan/test_stringscanner.rb | 50 ++++++++++++++ 6 files changed, 182 insertions(+), 23 deletions(-) delete mode 100644 ext/jruby/lib/strscan.rb create mode 100644 lib/strscan.rb diff --git a/ext/jruby/lib/strscan.rb b/ext/jruby/lib/strscan.rb deleted file mode 100644 index 4f796c25cd..0000000000 --- a/ext/jruby/lib/strscan.rb +++ /dev/null @@ -1,2 +0,0 @@ -require 'strscan.jar' -JRuby::Util.load_ext("org.jruby.ext.strscan.StringScannerLibrary") diff --git a/ext/jruby/org/jruby/ext/strscan/RubyStringScanner.java b/ext/jruby/org/jruby/ext/strscan/RubyStringScanner.java index b59d7a0a65..05510f45f7 100644 --- a/ext/jruby/org/jruby/ext/strscan/RubyStringScanner.java +++ b/ext/jruby/org/jruby/ext/strscan/RubyStringScanner.java @@ -557,8 +557,8 @@ public IRubyObject peep(ThreadContext context, IRubyObject length) { return peek(context, length); } - @JRubyMethod(name = "scan_integer") - public IRubyObject scan_integer(ThreadContext context) { + @JRubyMethod(name = "scan_base10_integer", visibility = PRIVATE) + public IRubyObject scan_base10_integer(ThreadContext context) { final Ruby runtime = context.runtime; check(context); clearMatched(); @@ -598,6 +598,47 @@ public IRubyObject scan_integer(ThreadContext context) { return ConvertBytes.byteListToInum(runtime, bytes, prev, curr, 10, true); } + @JRubyMethod(name = "scan_base16_integer", visibility = PRIVATE) + public IRubyObject scan_base16_integer(ThreadContext context) { + final Ruby runtime = context.runtime; + check(context); + clearMatched(); + + if (!str.getEncoding().isAsciiCompatible()) { + throw runtime.newEncodingCompatibilityError("ASCII incompatible encoding: " + str.getEncoding()); + } + + + ByteList bytes = str.getByteList(); + int curr = this.curr; + + int bite = bytes.get(curr); + if (bite == '0' && bytes.get(curr + 1) == 'x') { + curr += 2; + bite = bytes.get(curr); + } + + if (!((bite >= '0' && bite <= '9') || (bite >= 'a' && bite <= 'f') || (bite >= 'A' && bite <= 'F'))) { + return context.nil; + } + + while ((bite >= '0' && bite <= '9') || (bite >= 'a' && bite <= 'f') || (bite >= 'A' && bite <= 'F')) { + curr++; + if (curr >= bytes.getRealSize()) { + break; + } + bite = bytes.get(curr); + } + + int length = curr - this.curr; + prev = this.curr; + this.curr = curr; + setMatched(); + adjustRegisters(); + + return ConvertBytes.byteListToInum(runtime, bytes, prev, curr, 16, true); + } + @JRubyMethod(name = "unscan") public IRubyObject unscan(ThreadContext context) { check(context); diff --git a/ext/strscan/strscan.c b/ext/strscan/strscan.c index d0cae0f425..329dd87368 100644 --- a/ext/strscan/strscan.c +++ b/ext/strscan/strscan.c @@ -20,7 +20,6 @@ extern size_t onig_region_memsize(const struct re_registers *regs); #endif -#include #include #define STRSCAN_VERSION "3.1.1" @@ -116,7 +115,7 @@ static VALUE strscan_get_byte _((VALUE self)); static VALUE strscan_getbyte _((VALUE self)); static VALUE strscan_peek _((VALUE self, VALUE len)); static VALUE strscan_peep _((VALUE self, VALUE len)); -static VALUE strscan_scan_integer _((VALUE self)); +static VALUE strscan_scan_base10_integer _((VALUE self)); static VALUE strscan_unscan _((VALUE self)); static VALUE strscan_bol_p _((VALUE self)); static VALUE strscan_eos_p _((VALUE self)); @@ -1268,17 +1267,8 @@ strscan_peep(VALUE self, VALUE vlen) return strscan_peek(self, vlen); } -/* - * call-seq: - * scan_integer - * - * Equivalent to #scan with a [+-]?\d+ pattern, and returns an Integer or nil. - * - * The scanned string must be encoded with an ASCII compatible encoding, otherwise - * Encoding::CompatibilityError will be raised. - */ static VALUE -strscan_scan_integer(VALUE self) +strscan_scan_base10_integer(VALUE self) { char *ptr, *buffer; long len = 0; @@ -1302,14 +1292,14 @@ strscan_scan_integer(VALUE self) len++; } - if (!isdigit(ptr[len])) { + if (!rb_isdigit(ptr[len])) { return Qnil; } MATCHED(p); p->prev = p->curr; - while (len < remaining_len && isdigit(ptr[len])) { + while (len < remaining_len && rb_isdigit(ptr[len])) { len++; } @@ -1323,6 +1313,53 @@ strscan_scan_integer(VALUE self) return integer; } + +static VALUE +strscan_scan_base16_integer(VALUE self) +{ + char *ptr, *buffer; + long len = 0; + VALUE buffer_v, integer; + struct strscanner *p; + + GET_SCANNER(self, p); + CLEAR_MATCH_STATUS(p); + + rb_must_asciicompat(p->str); + + ptr = CURPTR(p); + + long remaining_len = S_RESTLEN(p); + + if (remaining_len <= 0) { + return Qnil; + } + + if (remaining_len >= 2 && ptr[len] == '0' && ptr[len + 1] == 'x') { + len += 2; + } + + if (len >= remaining_len || !rb_isxdigit(ptr[len])) { + return Qnil; + } + + MATCHED(p); + p->prev = p->curr; + + while (len < remaining_len && rb_isxdigit(ptr[len])) { + len++; + } + + buffer = RB_ALLOCV_N(char, buffer_v, len + 1); + + MEMCPY(buffer, CURPTR(p), char, len); + buffer[len] = '\0'; + integer = rb_cstr2inum(buffer, 16); + RB_ALLOCV_END(buffer_v); + p->curr += len; + return integer; +} + /* * :markup: markdown * :include: strscan/link_refs.txt @@ -2261,7 +2298,8 @@ Init_strscan(void) rb_define_method(StringScanner, "peek_byte", strscan_peek_byte, 0); rb_define_method(StringScanner, "peep", strscan_peep, 1); - rb_define_method(StringScanner, "scan_integer", strscan_scan_integer, 0); + rb_define_private_method(StringScanner, "scan_base10_integer", strscan_scan_base10_integer, 0); + rb_define_private_method(StringScanner, "scan_base16_integer", strscan_scan_base16_integer, 0); rb_define_method(StringScanner, "unscan", strscan_unscan, 0); diff --git a/lib/strscan.rb b/lib/strscan.rb new file mode 100644 index 0000000000..df725509fb --- /dev/null +++ b/lib/strscan.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +if RUBY_PLATFORM == "java" + require 'strscan.jar' + JRuby::Util.load_ext("org.jruby.ext.strscan.StringScannerLibrary") +else + require "strscan.so" +end + +class StringScanner + # call-seq: + # scan_integer(base: 10) + # + # If `base` isn't provided or is `10`, then it is equivalent to calling `#scan` with a `[+-]?\d+` pattern, + # and returns an Integer or nil. + # + # If `base` is `16`, then it is equivalent to calling `#scan` with a `(0x)?[0-9a-fA-F]+` pattern, + # and returns an Integer or nil. + # + # The scanned string must be encoded with an ASCII compatible encoding, otherwise + # Encoding::CompatibilityError will be raised. + def scan_integer(base: 10) + case base + when 10 + scan_base10_integer + when 16 + scan_base16_integer + else + raise ArgumentError, "Unsupported integer base: #{base.inspect}, expected 10 or 16" + end + end +end diff --git a/strscan.gemspec b/strscan.gemspec index 925edcd2d3..d5a71cfb5e 100644 --- a/strscan.gemspec +++ b/strscan.gemspec @@ -19,14 +19,14 @@ Gem::Specification.new do |s| files = [ "COPYING", "LICENSE.txt", - ] + ] + Dir[File.join(__dir__, "lib/**/*.rb")] + + s.require_paths = %w{lib} + if RUBY_ENGINE == "jruby" - s.require_paths = %w{ext/jruby/lib lib} - files << "ext/jruby/lib/strscan.rb" files << "lib/strscan.jar" s.platform = "java" else - s.require_paths = %w{lib} files << "ext/strscan/extconf.rb" files << "ext/strscan/strscan.c" s.rdoc_options << "-idoc" diff --git a/test/strscan/test_stringscanner.rb b/test/strscan/test_stringscanner.rb index ae05254d49..a5bf366350 100644 --- a/test/strscan/test_stringscanner.rb +++ b/test/strscan/test_stringscanner.rb @@ -945,6 +945,56 @@ def test_scan_integer_encoding s.scan_integer end end + + def test_scan_integer_base_16 + omit "scan_integer isn't implemented on TruffleRuby yet" if RUBY_ENGINE == "truffleruby" + + s = create_string_scanner('abc') + assert_equal 0xabc, s.scan_integer(base: 16) + assert_equal 3, s.pos + assert_predicate s, :matched? + + s = create_string_scanner('123abc') + assert_equal 0x123abc, s.scan_integer(base: 16) + assert_equal 6, s.pos + assert_predicate s, :matched? + + s = create_string_scanner('0x123abc') + assert_equal 0x123abc, s.scan_integer(base: 16) + assert_equal 8, s.pos + assert_predicate s, :matched? + + s = create_string_scanner('0x123ABC') + assert_equal 0x123abc, s.scan_integer(base: 16) + assert_equal 8, s.pos + assert_predicate s, :matched? + + s = create_string_scanner('0x') + assert_nil s.scan_integer(base: 16) + assert_equal 0, s.pos + refute_predicate s, :matched? + + s = create_string_scanner('-123abc') + assert_nil s.scan_integer(base: 16) + assert_equal 0, s.pos + refute_predicate s, :matched? + + s = create_string_scanner('+123') + assert_nil s.scan_integer(base: 16) + assert_equal 0, s.pos + refute_predicate s, :matched? + + s = create_string_scanner('-abc') + assert_nil s.scan_integer(base: 16) + assert_equal 0, s.pos + refute_predicate s, :matched? + + huge_integer = '1' * 2_000 + s = create_string_scanner(huge_integer) + assert_equal huge_integer.to_i(16), s.scan_integer(base: 16) + assert_equal 2_000, s.pos + assert_predicate s, :matched? + end end class TestStringScanner < Test::Unit::TestCase