diff --git a/rust/ruby-rbs-sys/build.rs b/rust/ruby-rbs-sys/build.rs index 7d3c96749..e89ed05b7 100644 --- a/rust/ruby-rbs-sys/build.rs +++ b/rust/ruby-rbs-sys/build.rs @@ -166,6 +166,9 @@ fn generate_bindings(include_path: &Path) -> Result Result, +} + +impl std::fmt::Display for ParseError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{} at bytes {}..{}", + self.message, self.range.start, self.range.end + ) + } +} + +impl std::error::Error for ParseError {} + +/// Owns a C parser that points into borrowed source and type-variable names. Dropping it +/// frees every node the parser produced. +pub(super) struct OwnedParser<'a> { + raw: NonNull, + range: Range, + marker: PhantomData<&'a str>, +} + +impl<'a> OwnedParser<'a> { + pub(super) fn new( + source: &'a str, + range: Range, + type_variables: &[&'a str], + ) -> Result { + let (start, end) = validate_range(source, &range)?; + + let bytes = &source.as_bytes()[..range.end]; + let raw = unsafe { + let start_ptr = bytes.as_ptr().cast::(); + let string = rbs_string_new(start_ptr, start_ptr.add(bytes.len())); + let encoding = &rbs_encodings[rbs_encoding_type_t::RBS_ENCODING_UTF_8 as usize] + as *const rbs_encoding_t; + rbs_parser_new(string, encoding, start, end) + }; + + let Some(raw) = NonNull::new(raw) else { + return Err(ParseError { + message: format!("the lexer cannot start at byte {}", range.start), + range, + }); + }; + + let parser = Self { + raw, + range, + marker: PhantomData, + }; + parser.declare_type_variables(type_variables)?; + Ok(parser) + } + + pub(super) fn raw(&self) -> NonNull { + self.raw + } + + pub(super) fn error(&self) -> Option { + let error = unsafe { (*self.raw.as_ptr()).error.as_ref() }?; + + let message = if error.message.is_null() { + String::from("failed to parse") + } else { + unsafe { CStr::from_ptr(error.message) } + .to_string_lossy() + .into_owned() + }; + + // Non-syntax errors have no meaningful token. + let range = if error.syntax_error { + self.token_range(error.token.range) + } else { + self.range.clone() + }; + + Some(ParseError { message, range }) + } + + pub(super) fn at_eof(&self) -> bool { + unsafe { (*self.raw.as_ptr()).next_token.type_ == RBSTokenType_pEOF } + } + + pub(super) fn expect_eof(&self) -> Result<(), ParseError> { + let parser = self.raw.as_ptr(); + unsafe { rbs_parser_advance(parser) }; + + let token = unsafe { (*parser).current_token }; + if token.type_ == RBSTokenType_pEOF { + return Ok(()); + } + Err(ParseError { + message: String::from("expected a token `pEOF`"), + range: self.token_range(token.range), + }) + } + + fn token_range(&self, token: rbs_range_t) -> Range { + match ( + usize::try_from(token.start.byte_pos), + usize::try_from(token.end.byte_pos), + ) { + (Ok(start), Ok(end)) if start <= end => start..end, + _ => self.range.clone(), + } + } + + fn declare_type_variables(&self, names: &[&'a str]) -> Result<(), ParseError> { + let parser = self.raw.as_ptr(); + unsafe { rbs_parser_push_typevar_table(parser, true) }; + + for name in names { + // The names outlive the parser; `insert_owned` would `free()` Rust-allocated memory. + let id = unsafe { + rbs_constant_pool_insert_shared( + &raw mut (*parser).constant_pool, + name.as_ptr(), + name.len(), + ) + }; + // 0 is `RBS_CONSTANT_ID_UNSET`: the pool failed to grow. + if id == 0 { + return Err(ParseError { + message: format!("failed to intern type variable `{name}`"), + range: self.range.clone(), + }); + } + + if !unsafe { rbs_parser_insert_typevar(parser, id) } { + return Err(self.error().unwrap_or_else(|| ParseError { + message: format!("failed to declare type variable `{name}`"), + range: self.range.clone(), + })); + } + } + + Ok(()) + } +} + +impl Drop for OwnedParser<'_> { + fn drop(&mut self) { + unsafe { rbs_parser_free(self.raw.as_ptr()) }; + } +} + +/// Rejects ranges the C parser mishandles, so every parser version fails the same way. +fn validate_range(source: &str, range: &Range) -> Result<(i32, i32), ParseError> { + let (start, end) = (range.start, range.end); + + let problem = if start > end { + "range starts after it ends" + } else if end > source.len() { + "range ends past the end of the source" + } else if !source.is_char_boundary(start) || !source.is_char_boundary(end) { + "range does not start and end on character boundaries" + } else if end > i32::MAX as usize { + "range ends past the parser's `i32::MAX` position limit" + } else if source.as_bytes()[..start].contains(&0) { + // The lexer stops at NUL while walking to `start`; v4.1.2 then loops forever. + "range starts after a NUL byte, which the lexer cannot skip" + } else { + return Ok((start as i32, end as i32)); + }; + + Err(ParseError { + message: format!("{problem}: {start}..{end}"), + range: range.clone(), + }) +} diff --git a/rust/ruby-rbs/src/node/parsing.rs b/rust/ruby-rbs/src/node/parsing.rs new file mode 100644 index 000000000..d7bf106c9 --- /dev/null +++ b/rust/ruby-rbs/src/node/parsing.rs @@ -0,0 +1,186 @@ +//! Safe parsing of RBS types, method types, and inline annotations. + +use super::Node; +use super::parser::{OwnedParser, ParseError}; +use ruby_rbs_sys::bindings::*; +use std::ops::Range; + +/// Owns a parser and the node it produced. +pub struct ParsedNode<'a> { + parser: OwnedParser<'a>, + pointer: *mut rbs_node_t, +} + +impl ParsedNode<'_> { + /// Returns the parsed node. + #[must_use] + pub fn node(&self) -> Node<'_> { + Node::new(self.parser.raw(), self.pointer) + } +} + +/// Options for [`parse_type`], matching the keywords of the Ruby `RBS::Parser.parse_type`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TypeOptions { + /// Fail unless the type ends the range. Off by default. + pub require_eof: bool, + /// Accept `void`. On by default. + pub void_allowed: bool, + /// Accept `self`. On by default. + pub self_allowed: bool, + /// Accept `instance` and `class`. On by default. + pub classish_allowed: bool, +} + +impl Default for TypeOptions { + fn default() -> Self { + Self { + require_eof: false, + void_allowed: true, + self_allowed: true, + classish_allowed: true, + } + } +} + +/// Options for [`parse_method_type`], matching Ruby's `RBS::Parser.parse_method_type`. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct MethodTypeOptions { + /// Fail unless the method type ends the range. Off by default. + pub require_eof: bool, +} + +/// Parses one type, such as `Array[String]`, from `source[range]`. +/// +/// `type_variables` names the type variables in scope. Locations are byte offsets into +/// `source`. Returns `Ok(None)` if the range has no tokens. +/// +/// # Errors +/// +/// Returns [`ParseError`] for malformed input, or for a range that is out of bounds, not on +/// UTF-8 character boundaries, past `i32::MAX`, or starting after a NUL byte. +/// +/// # Example +/// +/// ``` +/// use ruby_rbs::node::{Node, TypeOptions, parse_type}; +/// let source = "Array[String]"; +/// let parsed = parse_type(source, 0..source.len(), &[], TypeOptions::default())?.unwrap(); +/// assert!(matches!(parsed.node(), Node::ClassInstanceType(_))); +/// # Ok::<(), ruby_rbs::node::ParseError>(()) +/// ``` +pub fn parse_type<'a>( + source: &'a str, + range: Range, + type_variables: &[&'a str], + options: TypeOptions, +) -> Result>, ParseError> { + let parsed = parse_fragment(source, range, type_variables, true, |parser, node| unsafe { + rbs_parse_type( + parser, + node, + options.void_allowed, + options.self_allowed, + options.classish_allowed, + ) + })?; + if options.require_eof + && let Some(parsed) = &parsed + { + parsed.parser.expect_eof()?; + } + Ok(parsed) +} + +/// Parses one method type, such as `[T] (T) -> T`, from `source[range]` as a +/// [`Node::MethodType`]. +/// +/// Behaves like [`parse_type`]. The method type's own type parameters need not be listed in +/// `type_variables`. +pub fn parse_method_type<'a>( + source: &'a str, + range: Range, + type_variables: &[&'a str], + options: MethodTypeOptions, +) -> Result>, ParseError> { + // The Ruby extension always allows `instance` and `class` in method types. + parse_fragment(source, range, type_variables, true, |parser, node| unsafe { + rbs_parse_method_type(parser, node, options.require_eof, true) + }) +} + +/// Parses a leading RBS annotation, such as `: (Integer) -> void` or `@rbs x: Integer`, as +/// a `Node::*Annotation` variant. +/// +/// Pass the text without its Ruby comment marker. Arguments and errors are as for +/// [`parse_type`]; `Ok(None)` means the parser produced no annotation. +/// +/// # Example +/// +/// ``` +/// use ruby_rbs::node::{Node, parse_inline_leading_annotation}; +/// let source = "@rbs value: Integer"; +/// let parsed = parse_inline_leading_annotation(source, 0..source.len(), &[])?.unwrap(); +/// assert!(matches!(parsed.node(), Node::ParamTypeAnnotation(_))); +/// # Ok::<(), ruby_rbs::node::ParseError>(()) +/// ``` +pub fn parse_inline_leading_annotation<'a>( + source: &'a str, + range: Range, + type_variables: &[&'a str], +) -> Result>, ParseError> { + parse_fragment( + source, + range, + type_variables, + false, + |parser, node| unsafe { rbs_parse_inline_leading_annotation(parser, node) }, + ) +} + +/// Parses a trailing RBS annotation, such as `: String` or `[Integer]`, as a +/// `Node::*Annotation` variant or [`Node::NodeTypeAssertion`]. +/// +/// Behaves like [`parse_inline_leading_annotation`]. +pub fn parse_inline_trailing_annotation<'a>( + source: &'a str, + range: Range, + type_variables: &[&'a str], +) -> Result>, ParseError> { + parse_fragment( + source, + range, + type_variables, + false, + |parser, node| unsafe { rbs_parse_inline_trailing_annotation(parser, node) }, + ) +} + +/// Runs `parse`, which stores the node it produces, on a parser over `source[range]`. +fn parse_fragment<'a, T>( + source: &'a str, + range: Range, + type_variables: &[&'a str], + empty_is_none: bool, + parse: impl FnOnce(*mut rbs_parser_t, *mut *mut T) -> bool, +) -> Result>, ParseError> { + let parser = OwnedParser::new(source, range, type_variables)?; + if empty_is_none && parser.at_eof() { + return Ok(None); + } + + let mut node: *mut T = std::ptr::null_mut(); + let success = parse(parser.raw().as_ptr(), &mut node); + + // The parser can produce a node and still fail on trailing tokens; the error wins. + if let Some(error) = parser.error() { + return Err(error); + } + if !success || node.is_null() { + return Ok(None); + } + Ok(Some(ParsedNode { + parser, + pointer: node.cast(), + })) +} diff --git a/rust/ruby-rbs/tests/common/mod.rs b/rust/ruby-rbs/tests/common/mod.rs new file mode 100644 index 000000000..3c5dd4257 --- /dev/null +++ b/rust/ruby-rbs/tests/common/mod.rs @@ -0,0 +1,14 @@ +use ruby_rbs::node::RBSLocationRange; + +macro_rules! expect_variant { + ($node:expr, $variant:ident) => { + match $node { + Node::$variant(node) => node, + _ => panic!(concat!("expected Node::", stringify!($variant))), + } + }; +} + +pub fn slice(source: &str, location: RBSLocationRange) -> &str { + &source[location.start() as usize..location.end() as usize] +} diff --git a/rust/ruby-rbs/tests/inline_annotation.rs b/rust/ruby-rbs/tests/inline_annotation.rs new file mode 100644 index 000000000..bf73651cb --- /dev/null +++ b/rust/ruby-rbs/tests/inline_annotation.rs @@ -0,0 +1,119 @@ +#[macro_use] +mod common; + +use common::slice; +use ruby_rbs::node::{ + Node, ParsedNode, parse_inline_leading_annotation, parse_inline_trailing_annotation, +}; + +fn leading(source: &str) -> ParsedNode<'_> { + parse_inline_leading_annotation(source, 0..source.len(), &[]) + .unwrap() + .expect("expected a leading annotation") +} + +fn trailing(source: &str) -> ParsedNode<'_> { + parse_inline_trailing_annotation(source, 0..source.len(), &[]) + .unwrap() + .expect("expected a trailing annotation") +} + +#[test] +fn every_annotation_kind_has_a_node() { + expect_variant!( + leading(": (Integer) -> void").node(), + ColonMethodTypeAnnotation + ); + expect_variant!( + leading("@rbs (String) -> void | (Integer) -> void").node(), + MethodTypesAnnotation + ); + expect_variant!(leading("@rbs skip").node(), SkipAnnotation); + expect_variant!(leading("@rbs return: String").node(), ReturnTypeAnnotation); + expect_variant!( + leading("@rbs @name: String").node(), + InstanceVariableAnnotation + ); + expect_variant!( + leading("@rbs module-self: _Each[String]").node(), + ModuleSelfAnnotation + ); + expect_variant!(leading("@rbs value: Integer").node(), ParamTypeAnnotation); + expect_variant!( + leading("@rbs *rest: String").node(), + SplatParamTypeAnnotation + ); + expect_variant!( + leading("@rbs **opts: bool").node(), + DoubleSplatParamTypeAnnotation + ); + expect_variant!( + leading("@rbs &block: () -> void").node(), + BlockParamTypeAnnotation + ); + expect_variant!(trailing(": String").node(), NodeTypeAssertion); + expect_variant!( + trailing("[String, Integer]").node(), + TypeApplicationAnnotation + ); + expect_variant!(trailing(": class-alias Foo").node(), ClassAliasAnnotation); + expect_variant!(trailing(": module-alias Bar").node(), ModuleAliasAnnotation); +} + +#[test] +fn annotation_children_are_readable() { + let source = "@rbs module-self: _Hash[String, Integer]"; + let annotation = leading(source); + let module_self = expect_variant!(annotation.node(), ModuleSelfAnnotation); + + assert_eq!(module_self.name().name().as_str(), "_Hash"); + let args: Vec<&str> = module_self + .args() + .iter() + .map(|arg| slice(source, arg.location())) + .collect(); + assert_eq!(args, ["String", "Integer"]); + let commas: Vec<&str> = module_self + .args_comma_locations() + .iter() + .map(|location| slice(source, location)) + .collect(); + assert_eq!(commas, [","]); + assert_eq!( + module_self + .open_bracket_location() + .map(|l| slice(source, l)), + Some("[") + ); + assert!(module_self.comment_location().is_none()); +} + +#[test] +fn malformed_annotations_return_located_errors() { + // The parser also produces an annotation here; the error must win. + let source = ": String is a "; + let error = parse_inline_trailing_annotation(source, 0..source.len(), &[]) + .err() + .expect("expected a parse error"); + assert_eq!(&source[error.range], "is"); + + let source = "@rbs &block: () { () -> void } -> void"; + let error = parse_inline_leading_annotation(source, 0..source.len(), &[]) + .err() + .expect("expected a parse error"); + assert_eq!(&source[error.range], "{"); +} + +#[test] +fn type_variables_reach_the_annotation_parser() { + let source = ": Array[T]"; + let annotation = parse_inline_trailing_annotation(source, 0..source.len(), &["T"]) + .unwrap() + .unwrap(); + let assertion = expect_variant!(annotation.node(), NodeTypeAssertion); + let array = expect_variant!(assertion.type_(), ClassInstanceType); + assert!(matches!( + array.args().iter().next(), + Some(Node::VariableType(_)) + )); +} diff --git a/rust/ruby-rbs/tests/type_parsing.rs b/rust/ruby-rbs/tests/type_parsing.rs new file mode 100644 index 000000000..ebd503f99 --- /dev/null +++ b/rust/ruby-rbs/tests/type_parsing.rs @@ -0,0 +1,198 @@ +#[macro_use] +mod common; + +use common::slice; +use ruby_rbs::node::{ + MethodTypeOptions, Node, ParseError, ParsedNode, TypeOptions, parse_method_type, parse_type, +}; +use std::ops::Range; + +fn ty<'a>(source: &'a str, variables: &[&'a str], options: TypeOptions) -> ParsedNode<'a> { + parse_type(source, 0..source.len(), variables, options) + .unwrap() + .expect("expected a type") +} + +fn type_error(source: &str, options: TypeOptions) -> ParseError { + parse_type(source, 0..source.len(), &[], options) + .err() + .expect("expected a parse error") +} + +fn range_error(source: &str, range: Range) -> ParseError { + parse_type(source, range, &[], TypeOptions::default()) + .err() + .expect("expected an invalid range") +} + +fn method_type<'a>( + source: &'a str, + variables: &[&'a str], + options: MethodTypeOptions, +) -> ParsedNode<'a> { + parse_method_type(source, 0..source.len(), variables, options) + .unwrap() + .expect("expected a method type") +} + +fn method_type_error(source: &str, options: MethodTypeOptions) -> ParseError { + parse_method_type(source, 0..source.len(), &[], options) + .err() + .expect("expected a parse error") +} + +#[test] +fn locations_are_byte_offsets_into_the_source() { + let source = "日本語\nArray[String]? Integer"; + let start = source.find('A').unwrap(); + let end = source.find(" Integer").unwrap(); + let options = TypeOptions { + require_eof: true, + ..TypeOptions::default() + }; + let parsed = parse_type(source, start..end, &[], options) + .unwrap() + .unwrap(); + + let optional = expect_variant!(parsed.node(), OptionalType); + assert_eq!(slice(source, optional.location()), "Array[String]?"); + let array = expect_variant!(optional.type_(), ClassInstanceType); + let arg = array.args().iter().next().unwrap(); + assert_eq!(slice(source, arg.location()), "String"); +} + +#[test] +fn empty_range_is_none() { + for source in ["", " \n "] { + assert!(matches!( + parse_type(source, 0..source.len(), &[], TypeOptions::default()), + Ok(None) + )); + assert!(matches!( + parse_method_type(source, 0..source.len(), &[], MethodTypeOptions::default()), + Ok(None) + )); + } +} + +#[test] +fn require_eof_rejects_trailing_tokens() { + let source = "String Integer"; + let parsed = ty(source, &[], TypeOptions::default()); + assert_eq!(slice(source, parsed.node().location()), "String"); + let options = TypeOptions { + require_eof: true, + ..TypeOptions::default() + }; + assert_eq!(&source[type_error(source, options).range], "Integer"); + + let source = "() -> void | () -> String"; + let parsed = method_type(source, &[], MethodTypeOptions::default()); + assert_eq!(slice(source, parsed.node().location()), "() -> void"); + let error = method_type_error(source, MethodTypeOptions { require_eof: true }); + assert_eq!(&source[error.range], "|"); +} + +#[test] +fn options_reach_the_parser() { + let defaults = TypeOptions::default(); + for (source, options) in [ + ( + "void", + TypeOptions { + void_allowed: false, + ..defaults + }, + ), + ( + "self", + TypeOptions { + self_allowed: false, + ..defaults + }, + ), + ( + "instance", + TypeOptions { + classish_allowed: false, + ..defaults + }, + ), + ] { + ty(source, &[], defaults); + let error = type_error(source, options); + assert_eq!(&source[error.range], source, "{}", error.message); + } + + let source = "(class) -> instance"; + let parsed = method_type(source, &[], MethodTypeOptions::default()); + let function = expect_variant!( + expect_variant!(parsed.node(), MethodType).type_(), + FunctionType + ); + assert!(matches!(function.return_type(), Node::InstanceType(_))); +} + +#[test] +fn type_variables_are_declared() { + let source = "Hash[K, String]"; + let parsed = ty(source, &["K"], TypeOptions::default()); + let hash = expect_variant!(parsed.node(), ClassInstanceType); + let key = hash.args().iter().next().unwrap(); + assert_eq!(expect_variant!(key, VariableType).name().as_str(), "K"); + + let parsed = ty(source, &[], TypeOptions::default()); + let hash = expect_variant!(parsed.node(), ClassInstanceType); + assert!(matches!( + hash.args().iter().next().unwrap(), + Node::ClassInstanceType(_) + )); + + let source = "(Elem) -> Elem"; + let parsed = method_type(source, &["Elem"], MethodTypeOptions::default()); + let function = expect_variant!( + expect_variant!(parsed.node(), MethodType).type_(), + FunctionType + ); + assert!(matches!(function.return_type(), Node::VariableType(_))); +} + +#[test] +fn syntax_errors_carry_the_token_range() { + let source = "Array[String"; + let error = type_error(source, TypeOptions::default()); + assert_eq!(error.range, source.len()..source.len()); +} + +#[test] +fn invalid_ranges_are_errors() { + let source = "String"; + let len = source.len(); + #[allow(clippy::reversed_empty_ranges)] + let cases = [5..3, 0..len + 1, len + 1..len + 1]; + for range in cases { + let error = range_error(source, range.clone()); + assert_eq!(error.range, range, "{}", error.message); + } + + let source = "日本語\nString"; + for range in [4..12, 0..4, 7..source.len()] { + let start_ok = source.is_char_boundary(range.start); + let end_ok = source.is_char_boundary(range.end); + assert!(!(start_ok && end_ok), "{range:?} is a valid range"); + let error = range_error(source, range.clone()); + assert_eq!(error.range, range, "{}", error.message); + } +} + +#[test] +fn nul_is_rejected_only_before_the_range_start() { + let source = "\0String"; + let error = range_error(source, 1..source.len()); + assert_eq!(error.range, 1..source.len()); + + let source = "String\0"; + let parsed = ty(source, &[], TypeOptions::default()); + let class = expect_variant!(parsed.node(), ClassInstanceType); + assert_eq!(class.name().name().as_str(), "String"); +}