Uses &str instead of &[u8] in parsing

This removes a lot of ambiguity and allows us to ensure that `ClojureRS`
will still work with non-ascii characters.

Additionaly, a bug lurking in `consume_clojure_whitespaces` was fixed.
This bug triggered an error when the function had no whitespaces to
consume, while it should not.

Author: Leo Pourcelot

Cargo.toml

@@ -12,4 +12,4 @@ pest = "*"
 pest_derive = "*"
 nom = "*"
 text_io = "*"
-rand = "*"
+rand = "*"

src/main.rs

@@ -42,7 +42,7 @@ use nom::Needed::Size;
 fn main()
 {
     println!("Clojure RS 0.0.1");
-    
+
     // Register our macros / functions ahead of time
     let add_fn = rust_core::AddFn{};
     let str_fn = rust_core::StrFn{};
@@ -59,11 +59,11 @@ fn main()
     let def_macro = Value::DefMacro{};
     let fn_macro = Value::FnMacro{};
     let defmacro_macro = Value::DefmacroMacro{};
-    
+
     let environment = Rc::new(Environment::new_main_environment());
-    
+
     let eval_fn = rust_core::EvalFn::new(Rc::clone(&environment));
-    
+
     environment.insert(Symbol::intern("+"),add_fn.to_rc_value());
     environment.insert(Symbol::intern("let"),let_macro.to_rc_value());
     environment.insert(Symbol::intern("str"),str_fn.to_rc_value());
@@ -87,33 +87,32 @@ fn main()
     //
     let stdin = io::stdin();
     print!("user=> ");
-    let mut remaining_input_buffer = String::from("");
+    let mut input_buffer = String::new();
     for line in stdin.lock().lines() {
-	let line = line.unwrap();
-	remaining_input_buffer.push_str(&line);
-	let mut remaining_input_bytes = remaining_input_buffer.as_bytes();
-	loop {
-	    let next_read_parse = reader::try_read(remaining_input_bytes);
-	    match next_read_parse {
-		Ok((_remaining_input_bytes,value)) => {
-		    print!("{} ",value.eval(Rc::clone(&environment)).to_string_explicit());
-		    remaining_input_bytes = _remaining_input_bytes;
-		},
-		Err(Incomplete(_)) => {
-		    remaining_input_buffer = String::from_utf8(remaining_input_bytes.to_vec()).unwrap();
-		    break;
-		},
-		err => {
-		    print!("{}",Value::Condition(format!("Reader Error: {:?}",err)));
-		    remaining_input_buffer = String::from("");
-		    break;
-		}
-	    }
-	}
-	println!();
-	print!("user=> ");
+        let line = line.unwrap();
+        input_buffer.push_str(&line);
+        let mut remaining_input = input_buffer.as_str();
+        loop {
+            let next_read_parse = reader::try_read(remaining_input);
+            match next_read_parse {
+                Ok((_remaining_input,value)) => {
+                    print!("{} ",value.eval(Rc::clone(&environment)).to_string_explicit());
+                    remaining_input = _remaining_input;
+                },
+                Err(Incomplete(_)) => {
+                    break;
+                },
+                err => {
+                    print!("{}",Value::Condition(format!("Reader Error: {:?}",err)));
+                    input_buffer.clear();
+                    break;
+                }
+            }
+        }
+        println!();
+        print!("user=> ");
     }
-    
+
 }
 
 

src/reader.rs

@@ -7,16 +7,16 @@
 //! or even reader macros,  although the latter will likely be reserved for our interpreter here (but perhaps
 //! not;  since this is about being a 'free-er' Clojure, especially since it can't compete with it in raw
 //! power, neither speed or ecosystem,  it might be worth it to leave in reader macros. 
-extern crate nom;
 
 use nom::{
     IResult,
     branch::alt,
     error::convert_error,
-    character::{is_alphabetic,is_alphanumeric},
     character::complete::multispace0,
-    character::is_digit,
-    bytes::complete::{take_while1,take_until,tag},
+    bytes::complete::{take_while1,tag},
+    take_until,
+    terminated,
+    map,
     combinator::map_res,
     sequence::{preceded,terminated}};
 
@@ -26,27 +26,30 @@ use crate::persistent_vector::{ToPersistentVector};
 use crate::persistent_list_map::{PersistentListMap,ToPersistentListMap};
 use crate::maps::MapEntry;
 use crate::symbol::Symbol;
-use std::rc::Rc;
+use std::{
+    iter::FromIterator,
+    rc::Rc,
+};
 
 use std::fs::File;
 
 /// Parses valid Clojure identifiers
 /// Example Successes: ab,  cat,  -12+3, |blah|, <well>
 /// Example Failures:  'a,  12b,   ,cat  
-pub fn identifier_parser(input:&[u8]) -> IResult<&[u8], String> {
-    named!( non_numeric_identifier_char<&[u8],u8>,
-	    alt!( map!(one_of!("|?<>+-_=^%&$*!"), |x| x as u8 ) |
-		  map!(take_while_m_n!(1,1,is_alphabetic),|ls| ls[0] as u8)));
-    named!( identifier_char<&[u8],u8>,
-	    alt!( map!(one_of!("|?<>+-_=^%&$*!"), |x| x as u8 ) |
-		  map!(take_while_m_n!(1,1,is_alphanumeric),|ls| ls[0] as u8)));
-    named!( identifier_ <&[u8],String> ,
+pub fn identifier_parser(input: &str) -> IResult<&str, String> {
+    named!( non_numeric_identifier_char<&str, char>,
+	    alt!( one_of!("|?<>+-_=^%&$*!") |
+		  map!(take_while_m_n!(1,1,char::is_alphabetic),|ls| ls.chars().next().unwrap())));
+    named!( identifier_char<&str, char>,
+	    alt!( one_of!("|?<>+-_=^%&$*!") |
+		  map!(take_while_m_n!(1,1,char::is_alphanumeric),|ls| ls.chars().next().unwrap())));
+    named!( identifier_ <&str, String> ,
 	    do_parse!(
 		head: non_numeric_identifier_char >>
 		rest_input:
-		map_res!(
+		map!(
 		    many0!(complete!(identifier_char)),
-		    String::from_utf8) >>
+		    String::from_iter) >>
 		(format!("{}{}",head as char,rest_input))
 	    ));
 
@@ -55,7 +58,7 @@ pub fn identifier_parser(input:&[u8]) -> IResult<&[u8], String> {
 }
 
 /// Parses valid Clojure symbols,  whose name is a valid identifier 
-pub fn symbol_parser(input: &[u8]) -> IResult<&[u8], Symbol> { 
+pub fn symbol_parser(input: &str) -> IResult<&str, Symbol> {
     identifier_parser(input).map(|(rest_input,name)| {
 	(rest_input, Symbol::intern(&name))
     })
@@ -64,12 +67,10 @@ pub fn symbol_parser(input: &[u8]) -> IResult<&[u8], Symbol> {
 // @TODO add negatives 
 /// Parses valid integers
 /// Example Successes: 1, 2, 4153,  -12421
-pub fn integer(input: &[u8]) -> IResult<&[u8],i32> {
-    map_res(take_while1(is_digit),|digits: &[u8]| { 
-	String::from_utf8(digits.to_vec()).map(|digit_string| {
-	    digit_string.parse::<i32>().unwrap()
-	})
-    })(input)	
+pub fn integer(input: &str) -> IResult<&str, i32> {
+    named!(integer_lexer<&str, &str>, take_while1!(|c: char| c.is_digit(10)));
+
+    integer_lexer(input).map(|(rest, digits)| (rest, digits.parse().unwrap()))
 }
 // Currently used to create 'try_readers', which are readers (or
 // reader functions, at least) that are basically composable InputType
@@ -93,7 +94,7 @@ pub fn to_value_parser<I,O: ToValue>(parser: impl Fn(I) -> IResult<I,O>) -> impl
 ///    1231415 => Value::I32(1231415)
 /// Example Failures:
 ///    1.5,  7.1321 , 1423152621625226126431525
-pub fn try_read_i32(input: &[u8]) -> IResult<&[u8],Value> {
+pub fn try_read_i32(input: &str) -> IResult<&str, Value> {
     to_value_parser(integer)(input)
 }
 
@@ -104,32 +105,36 @@ pub fn try_read_i32(input: &[u8]) -> IResult<&[u8],Value> {
 ///    +common-lisp-global+ => Value::Symbol(Symbol { name: "+common-lisp-global+" })
 /// Example Failures:
 ///    12cat,  'quoted,  @at-is-for-references 
-pub fn try_read_symbol(input: &[u8]) -> IResult<&[u8],Value> {
+pub fn try_read_symbol(input: &str) -> IResult<&str, Value> {
     to_value_parser(symbol_parser)(input)
 }
 
 // @TODO allow escaped strings 
 /// Tries to parse &[u8] into Value::String
 /// Example Successes:
 ///    "this is pretty straightforward" => Value::String("this is pretty straightforward")
-pub fn try_read_string(input: &[u8]) -> IResult<&[u8],Value> {
-    named!(quotation, preceded!(consume_clojure_whitespaces, tag!("\"")));
+pub fn try_read_string(input: &str) -> IResult<&str, Value> {
+    named!(quotation<&str, &str>, preceded!(consume_clojure_whitespaces, tag!("\"")));
+
     let (rest_input,_) = quotation(input)?;
-    to_value_parser(
-	map_res(
-	    terminated(
-		take_until("\""),
-		tag("\"")),
-	    |bytes: &[u8]| String::from_utf8(bytes.to_vec())))(rest_input)
+    named!(
+        string_parser<&str, String>,
+        map!(
+            terminated!(take_until!("\""), tag("\"")),
+            |v| String::from(v)
+        )
+    );
+
+    to_value_parser(string_parser)(input)
 }
 
 // @TODO Perhaps generalize this, or even generalize it as a reader macro 
 /// Tries to parse &[u8] into Value::PersistentListMap, or some other Value::..Map   
 /// Example Successes:
 ///    {:a 1} => Value::PersistentListMap {PersistentListMap { MapEntry { :a, 1} .. ]})
-pub fn try_read_map(input: &[u8]) -> IResult<&[u8],Value> {
-    named!(lbracep, preceded!(consume_clojure_whitespaces, tag!("{")));
-    named!(rbracep, preceded!(consume_clojure_whitespaces, tag!("}")));
+pub fn try_read_map(input: &str) -> IResult<&str, Value> {
+    named!(lbracep<&str, &str>, preceded!(consume_clojure_whitespaces, tag!("{")));
+    named!(rbracep<&str, &str>, preceded!(consume_clojure_whitespaces, tag!("}")));
     let (map_inner_input,_) = lbracep(input)?;
     let mut map_as_vec : Vec<MapEntry> = vec![];
     let mut rest_input = map_inner_input;
@@ -155,9 +160,9 @@ pub fn try_read_map(input: &[u8]) -> IResult<&[u8],Value> {
 ///    [1 2 3] => Value::PersistentVector(PersistentVector { vals: [Rc(Value::I32(1) ... ]})
 ///    [1 2 [5 10 15] 3]
 ///      => Value::PersistentVector(PersistentVector { vals: [Rc(Value::I32(1) .. Rc(Value::PersistentVector..)]})
-pub fn try_read_vector(input: &[u8]) -> IResult<&[u8],Value> {
-    named!(lbracketp, preceded!(consume_clojure_whitespaces, tag!("[")));
-    named!(rbracketp, preceded!(consume_clojure_whitespaces, tag!("]")));
+pub fn try_read_vector(input: &str) -> IResult<&str, Value> {
+    named!(lbracketp<&str, &str>, preceded!(consume_clojure_whitespaces, tag!("[")));
+    named!(rbracketp<&str, &str>, preceded!(consume_clojure_whitespaces, tag!("]")));
     let (vector_inner_input,_) = lbracketp(input)?;
     let mut vector_as_vec = vec![];
     // What's left of our input as we read more of our PersistentVector 
@@ -190,9 +195,9 @@ pub fn try_read_vector(input: &[u8]) -> IResult<&[u8],Value> {
     }
 }
 
-pub fn try_read_list(input: &[u8]) -> IResult<&[u8],Value> {
-    named!(lparenp, preceded!(consume_clojure_whitespaces, tag!("(")));
-    named!(rparenp, preceded!(consume_clojure_whitespaces, tag!(")")));
+pub fn try_read_list(input: &str) -> IResult<&str, Value> {
+    named!(lparenp<&str, &str>, preceded!(consume_clojure_whitespaces, tag!("(")));
+    named!(rparenp<&str, &str>, preceded!(consume_clojure_whitespaces, tag!(")")));
 
     let (list_inner_input,_) = lparenp(input)?;
     let mut list_as_vec = vec![];
@@ -221,17 +226,18 @@ pub fn try_read_list(input: &[u8]) -> IResult<&[u8],Value> {
     }
 }
 
-pub fn try_read(input: &[u8]) -> IResult<&[u8], Value> {
+pub fn try_read(input: &str) -> IResult<&str, Value> {
     preceded(multispace0,alt(
 	(try_read_map,
 	 try_read_string,
 	 try_read_symbol,
 	 try_read_i32,
 	 try_read_list,
-	 try_read_vector)))(input)
+	 try_read_vector
+     )))(input)
 }
 
-pub fn debug_try_read(input: &[u8]) -> IResult<&[u8], Value> {
+pub fn debug_try_read(input: &str) -> IResult<&str, Value> {
 
     let reading = try_read(input);
     match &reading {
@@ -244,15 +250,14 @@ pub fn debug_try_read(input: &[u8]) -> IResult<&[u8], Value> {
 /// Consumes one or more whitespaces from the input.
 ///
 /// A whitespace is either an ASCII whitespace or a comma.
-fn consume_clojure_whitespaces(input: &[u8]) -> IResult<&[u8], ()> {
-    named!(parser, take_while1!(is_clojure_whitespace));
+fn consume_clojure_whitespaces(input: &str) -> IResult<&str, ()> {
+    named!(parser<&str, &str>, take_while!(is_clojure_whitespace));
     parser(input).map(|(rest, _)| (rest, ()))
 }
 
 /// Returns whether if a given character is a whitespace.
 ///
-/// Clojure defines a whitespace as either a comma or an ASCII whitespace.
-fn is_clojure_whitespace(c: u8) -> bool {
-    // ASCII symbol of `,` is 44.
-    c.is_ascii_whitespace() || c == 44
+/// Clojure defines a whitespace as either a comma or an unicode whitespace.
+fn is_clojure_whitespace(c: char) -> bool {
+    c.is_whitespace() || c == ','
 }

src/repl.rs

@@ -21,35 +21,36 @@ use nom::Needed::Size;
 pub fn try_eval_file(environment: &Rc<Environment>,filepath: &str) -> Result<(),io::Error>{
     let core = File::open(filepath)?;
 
-    
+
     let reader = BufReader::new(core);
 
-    let mut remaining_input_buffer = String::from("");
+    let mut input_buffer = String::new();
+
     for line in reader.lines() {
-	let line = line?;
-	remaining_input_buffer.push_str(&line);
-	let mut remaining_input_bytes = remaining_input_buffer.as_bytes();
-	loop {
-	    let next_read_parse = reader::try_read(remaining_input_bytes);
-	    match next_read_parse {
-		Ok((_remaining_input,value)) => {
-		    //print!("{} ",value.eval(Rc::clone(&environment)).to_string_explicit());
-		    value.eval(Rc::clone(&environment));
-		    remaining_input_bytes = _remaining_input;
-		},
-		Err(Incomplete(Size(1))) => {
-		    remaining_input_buffer = String::from_utf8(remaining_input_bytes.to_vec()).unwrap();
-		    break;
-		},
-		err => {
-		    println!("Error evaluating file {}; {}",filepath,Value::Condition(format!("Reader Error: {:?}",err)));
-		    remaining_input_buffer = String::from("");
-		    break;
-		}
-	    }
-	}
+        let line = line?;
+        input_buffer.push_str(&line);
+        let mut remaining_input = input_buffer.as_str();
+        loop {
+            let next_read_parse = reader::try_read(remaining_input);
+            match next_read_parse {
+                Ok((_remaining_input,value)) => {
+                    //print!("{} ",value.eval(Rc::clone(&environment)).to_string_explicit());
+                    value.eval(Rc::clone(&environment));
+                    remaining_input = _remaining_input;
+                },
+                Err(Incomplete(Size(1))) => {
+                    break;
+                },
+                err => {
+                    println!("Error evaluating file {}; {}",filepath,Value::Condition(format!("Reader Error: {:?}",err)));
+                    input_buffer.clear();
+                    remaining_input = "";
+                    break;
+                }
+            }
+        }
     }
 
     Ok(())
-    
+
 }