Skip to content

Commit 4c8e2c8

Browse files
headiusheadius
committed
Add WASM backend for Ruby API
1 parent 7ac0c85 commit 4c8e2c8

File tree

2 files changed

+383
-1
lines changed

2 files changed

+383
-1
lines changed

‎lib/prism.rb‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,5 +127,10 @@ def self.load(source, serialized, freeze = false)
127127
# The FFI backend is used on other Ruby implementations.
128128
Prism::BACKEND = :FFI
129129

130-
require_relative "prism/ffi"
130+
begin
131+
require_relative "prism/ffi"
132+
rescue LoadError
133+
raise $! unless RUBY_ENGINE == "jruby"
134+
require_relative "prism/wasm"
135+
end
131136
end

‎lib/prism/wasm.rb‎

Lines changed: 377 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,377 @@
1+
# frozen_string_literal: true
2+
# :markup: markdown
3+
# typed: ignore
4+
5+
# This file is responsible for mirroring the API provided by the C extension by
6+
# using FFI to call into the shared library.
7+
8+
require "rbconfig"
9+
require "ffi"
10+
11+
# We want to eagerly load this file if there are Ractors so that it does not get
12+
# autoloaded from within a non-main Ractor.
13+
require "prism/serialize" if defined?(Ractor)
14+
15+
# Load the prism-parser-wasm jar
16+
require 'jar-dependencies'
17+
require_jar('org.ruby-lang', 'prism-parser-wasm', '0.0.1-SNAPSHOT')
18+
require_jar('com.dylibso.chicory', 'runtime', '1.6.1')
19+
require_jar('com.dylibso.chicory', 'wasi', '1.6.1')
20+
require_jar('com.dylibso.chicory', 'wasm', '1.6.1')
21+
require_jar('com.dylibso.chicory', 'log', '1.6.1')
22+
23+
module Prism # :nodoc:
24+
module WASM
25+
java_import org.ruby_lang.prism.wasm.Prism
26+
27+
# TODO: concurrency
28+
PRISM = org.ruby_lang.prism.wasm.Prism.new
29+
end
30+
private_constant :WASM
31+
32+
# The version constant is set by reading the result of calling pm_version.
33+
VERSION = WASM::PRISM.version
34+
35+
class << self
36+
# Mirror the Prism.dump API by using the serialization API.
37+
def dump(source, **options)
38+
parsed = WASM::PRISM.parse(source.to_java_bytes, dump_options(options).to_java_bytes)
39+
String.from_java_bytes(parsed)
40+
end
41+
42+
# Mirror the Prism.dump_file API by using the serialization API.
43+
def dump_file(filepath, **options)
44+
dump_file(File.read(filepath), filepath: filepath, **options)
45+
end
46+
47+
# Mirror the Prism.lex API by using the serialization API.
48+
def lex(source, **options)
49+
lexed = WASM::PRISM.lex(source.to_java_bytes, dump_options(options).to_java_bytes)
50+
Serialize.load_lex(source, lexed, options.fetch(:freeze, false))
51+
end
52+
53+
# Mirror the Prism.lex_file API by using the serialization API.
54+
def lex_file(filepath, **options)
55+
lex_file(File.read(filepath), filepath: filepath, **options)
56+
end
57+
58+
# Mirror the Prism.parse API by using the serialization API.
59+
def parse(source, **options)
60+
serialized = dump(source, **options)
61+
Serialize.load_parse(source, serialized, options.fetch(:freeze, false))
62+
end
63+
64+
# Mirror the Prism.parse_file API by using the serialization API. This uses
65+
# native strings instead of Ruby strings because it allows us to use mmap
66+
# when it is available.
67+
def parse_file(filepath, **options)
68+
parse(File.read(filepath), filepath: filepath, **options)
69+
end
70+
71+
# Mirror the Prism.parse_stream API by using the serialization API.
72+
def parse_stream(stream, **options)
73+
LibRubyParser::PrismBuffer.with do |buffer|
74+
source = +""
75+
callback = -> (string, size, _) {
76+
raise "Expected size to be >= 0, got: #{size}" if size <= 0
77+
78+
if !(line = stream.gets(size - 1)).nil?
79+
source << line
80+
string.write_string("#{line}\x00", line.bytesize + 1)
81+
end
82+
}
83+
84+
eof_callback = -> (_) { stream.eof? }
85+
86+
# In the pm_serialize_parse_stream function it accepts a pointer to the
87+
# IO object as a void* and then passes it through to the callback as the
88+
# third argument, but it never touches it itself. As such, since we have
89+
# access to the IO object already through the closure of the lambda, we
90+
# can pass a null pointer here and not worry.
91+
LibRubyParser.pm_serialize_parse_stream(buffer.pointer, nil, callback, eof_callback, dump_options(options))
92+
Prism.load(source, buffer.read, options.fetch(:freeze, false))
93+
end
94+
end
95+
96+
# Mirror the Prism.parse_comments API by using the serialization API.
97+
def parse_comments(code, **options)
98+
LibRubyParser::PrismString.with_string(code) { |string| parse_comments_common(string, code, options) }
99+
end
100+
101+
# Mirror the Prism.parse_file_comments API by using the serialization
102+
# API. This uses native strings instead of Ruby strings because it allows us
103+
# to use mmap when it is available.
104+
def parse_file_comments(filepath, **options)
105+
options[:filepath] = filepath
106+
LibRubyParser::PrismString.with_file(filepath) { |string| parse_comments_common(string, string.read, options) }
107+
end
108+
109+
# Mirror the Prism.parse_lex API by using the serialization API.
110+
def parse_lex(code, **options)
111+
LibRubyParser::PrismString.with_string(code) { |string| parse_lex_common(string, code, options) }
112+
end
113+
114+
# Mirror the Prism.parse_lex_file API by using the serialization API.
115+
def parse_lex_file(filepath, **options)
116+
options[:filepath] = filepath
117+
LibRubyParser::PrismString.with_file(filepath) { |string| parse_lex_common(string, string.read, options) }
118+
end
119+
120+
# Mirror the Prism.parse_success? API by using the serialization API.
121+
def parse_success?(code, **options)
122+
LibRubyParser::PrismString.with_string(code) { |string| parse_file_success_common(string, options) }
123+
end
124+
125+
# Mirror the Prism.parse_failure? API by using the serialization API.
126+
def parse_failure?(code, **options)
127+
!parse_success?(code, **options)
128+
end
129+
130+
# Mirror the Prism.parse_file_success? API by using the serialization API.
131+
def parse_file_success?(filepath, **options)
132+
options[:filepath] = filepath
133+
LibRubyParser::PrismString.with_file(filepath) { |string| parse_file_success_common(string, options) }
134+
end
135+
136+
# Mirror the Prism.parse_file_failure? API by using the serialization API.
137+
def parse_file_failure?(filepath, **options)
138+
!parse_file_success?(filepath, **options)
139+
end
140+
141+
# Mirror the Prism.profile API by using the serialization API.
142+
def profile(source, **options)
143+
LibRubyParser::PrismString.with_string(source) do |string|
144+
LibRubyParser::PrismBuffer.with do |buffer|
145+
LibRubyParser.pm_serialize_parse(buffer.pointer, string.pointer, string.length, dump_options(options))
146+
nil
147+
end
148+
end
149+
end
150+
151+
# Mirror the Prism.profile_file API by using the serialization API.
152+
def profile_file(filepath, **options)
153+
LibRubyParser::PrismString.with_file(filepath) do |string|
154+
LibRubyParser::PrismBuffer.with do |buffer|
155+
options[:filepath] = filepath
156+
LibRubyParser.pm_serialize_parse(buffer.pointer, string.pointer, string.length, dump_options(options))
157+
nil
158+
end
159+
end
160+
end
161+
162+
private
163+
164+
def lex_common(string, code, options) # :nodoc:
165+
LibRubyParser::PrismBuffer.with do |buffer|
166+
LibRubyParser.pm_serialize_lex(buffer.pointer, string.pointer, string.length, dump_options(options))
167+
Serialize.load_lex(code, buffer.read, options.fetch(:freeze, false))
168+
end
169+
end
170+
171+
def parse_common(string, code, options) # :nodoc:
172+
serialized = dump_common(string, options)
173+
Serialize.load_parse(code, serialized, options.fetch(:freeze, false))
174+
end
175+
176+
def parse_comments_common(string, code, options) # :nodoc:
177+
LibRubyParser::PrismBuffer.with do |buffer|
178+
LibRubyParser.pm_serialize_parse_comments(buffer.pointer, string.pointer, string.length, dump_options(options))
179+
Serialize.load_parse_comments(code, buffer.read, options.fetch(:freeze, false))
180+
end
181+
end
182+
183+
def parse_lex_common(string, code, options) # :nodoc:
184+
LibRubyParser::PrismBuffer.with do |buffer|
185+
LibRubyParser.pm_serialize_parse_lex(buffer.pointer, string.pointer, string.length, dump_options(options))
186+
Serialize.load_parse_lex(code, buffer.read, options.fetch(:freeze, false))
187+
end
188+
end
189+
190+
def parse_file_success_common(string, options) # :nodoc:
191+
LibRubyParser.pm_parse_success_p(string.pointer, string.length, dump_options(options))
192+
end
193+
194+
# Return the value that should be dumped for the command_line option.
195+
def dump_options_command_line(options)
196+
command_line = options.fetch(:command_line, "")
197+
raise ArgumentError, "command_line must be a string" unless command_line.is_a?(String)
198+
199+
command_line.each_char.inject(0) do |value, char|
200+
case char
201+
when "a" then value | 0b000001
202+
when "e" then value | 0b000010
203+
when "l" then value | 0b000100
204+
when "n" then value | 0b001000
205+
when "p" then value | 0b010000
206+
when "x" then value | 0b100000
207+
else raise ArgumentError, "invalid command_line option: #{char}"
208+
end
209+
end
210+
end
211+
212+
# Return the value that should be dumped for the version option.
213+
def dump_options_version(version)
214+
case version
215+
when "current"
216+
version_string_to_number(RUBY_VERSION) || raise(CurrentVersionError, RUBY_VERSION)
217+
when "latest", nil
218+
0 # Handled in pm_parser_init
219+
when "nearest"
220+
dump = version_string_to_number(RUBY_VERSION)
221+
return dump if dump
222+
if RUBY_VERSION < "3.3"
223+
version_string_to_number("3.3")
224+
else
225+
0 # Handled in pm_parser_init
226+
end
227+
else
228+
version_string_to_number(version) || raise(ArgumentError, "invalid version: #{version}")
229+
end
230+
end
231+
232+
# Converts a version string like "4.0.0" or "4.0" into a number.
233+
# Returns nil if the version is unknown.
234+
def version_string_to_number(version)
235+
case version
236+
when /\A3\.3(\.\d+)?\z/
237+
1
238+
when /\A3\.4(\.\d+)?\z/
239+
2
240+
when /\A3\.5(\.\d+)?\z/, /\A4\.0(\.\d+)?\z/
241+
3
242+
when /\A4\.1(\.\d+)?\z/
243+
4
244+
end
245+
end
246+
247+
# Convert the given options into a serialized options string.
248+
def dump_options(options)
249+
template = +""
250+
values = []
251+
252+
template << "L"
253+
if (filepath = options[:filepath])
254+
values.push(filepath.bytesize, filepath.b)
255+
template << "A*"
256+
else
257+
values << 0
258+
end
259+
260+
template << "l"
261+
values << options.fetch(:line, 1)
262+
263+
template << "L"
264+
if (encoding = options[:encoding])
265+
name = encoding.is_a?(Encoding) ? encoding.name : encoding
266+
values.push(name.bytesize, name.b)
267+
template << "A*"
268+
else
269+
values << 0
270+
end
271+
272+
template << "C"
273+
values << (options.fetch(:frozen_string_literal, false) ? 1 : 0)
274+
275+
template << "C"
276+
values << dump_options_command_line(options)
277+
278+
template << "C"
279+
values << dump_options_version(options[:version])
280+
281+
template << "C"
282+
values << (options[:encoding] == false ? 1 : 0)
283+
284+
template << "C"
285+
values << (options.fetch(:main_script, false) ? 1 : 0)
286+
287+
template << "C"
288+
values << (options.fetch(:partial_script, false) ? 1 : 0)
289+
290+
template << "C"
291+
values << (options.fetch(:freeze, false) ? 1 : 0)
292+
293+
template << "L"
294+
if (scopes = options[:scopes])
295+
values << scopes.length
296+
297+
scopes.each do |scope|
298+
locals = nil
299+
forwarding = 0
300+
301+
case scope
302+
when Array
303+
locals = scope
304+
when Scope
305+
locals = scope.locals
306+
307+
scope.forwarding.each do |forward|
308+
case forward
309+
when :* then forwarding |= 0x1
310+
when :** then forwarding |= 0x2
311+
when :& then forwarding |= 0x4
312+
when :"..." then forwarding |= 0x8
313+
else raise ArgumentError, "invalid forwarding value: #{forward}"
314+
end
315+
end
316+
else
317+
raise TypeError, "wrong argument type #{scope.class.inspect} (expected Array or Prism::Scope)"
318+
end
319+
320+
template << "L"
321+
values << locals.length
322+
323+
template << "C"
324+
values << forwarding
325+
326+
locals.each do |local|
327+
name = local.name
328+
template << "L"
329+
values << name.bytesize
330+
331+
template << "A*"
332+
values << name.b
333+
end
334+
end
335+
else
336+
values << 0
337+
end
338+
339+
values.pack(template)
340+
end
341+
end
342+
343+
# Here we are going to patch StringQuery to put in the class-level methods so
344+
# that it can maintain a consistent interface
345+
class StringQuery # :nodoc:
346+
class << self
347+
# Mirrors the C extension's StringQuery::local? method.
348+
def local?(string)
349+
query(LibRubyParser.pm_string_query_local(string, string.bytesize, string.encoding.name))
350+
end
351+
352+
# Mirrors the C extension's StringQuery::constant? method.
353+
def constant?(string)
354+
query(LibRubyParser.pm_string_query_constant(string, string.bytesize, string.encoding.name))
355+
end
356+
357+
# Mirrors the C extension's StringQuery::method_name? method.
358+
def method_name?(string)
359+
query(LibRubyParser.pm_string_query_method_name(string, string.bytesize, string.encoding.name))
360+
end
361+
362+
private
363+
364+
# Parse the enum result and return an appropriate boolean.
365+
def query(result)
366+
case result
367+
when :PM_STRING_QUERY_ERROR
368+
raise ArgumentError, "Invalid or non ascii-compatible encoding"
369+
when :PM_STRING_QUERY_FALSE
370+
false
371+
when :PM_STRING_QUERY_TRUE
372+
true
373+
end
374+
end
375+
end
376+
end
377+
end

0 commit comments

Comments
 (0)