diff options
| -rw-r--r-- | cli/ast.rs | 66 | ||||
| -rw-r--r-- | cli/tools/repl.rs | 186 |
2 files changed, 148 insertions, 104 deletions
diff --git a/cli/ast.rs b/cli/ast.rs index 3d859e020..4b8989d70 100644 --- a/cli/ast.rs +++ b/cli/ast.rs @@ -8,11 +8,13 @@ use deno_core::serde_json; use deno_core::ModuleSpecifier; use std::error::Error; use std::fmt; +use std::ops::Range; use std::rc::Rc; use std::sync::Arc; use std::sync::RwLock; use swc_common::chain; use swc_common::comments::Comment; +use swc_common::comments::CommentKind; use swc_common::comments::SingleThreadedComments; use swc_common::errors::Diagnostic; use swc_common::errors::DiagnosticBuilder; @@ -32,6 +34,7 @@ use swc_ecmascript::codegen::Node; use swc_ecmascript::dep_graph::analyze_dependencies; use swc_ecmascript::dep_graph::DependencyDescriptor; use swc_ecmascript::parser::lexer::Lexer; +use swc_ecmascript::parser::token::Token; use swc_ecmascript::parser::EsConfig; use swc_ecmascript::parser::JscTarget; use swc_ecmascript::parser::StringInput; @@ -407,6 +410,69 @@ pub fn parse( }) } +pub enum TokenOrComment { + Token(Token), + Comment { kind: CommentKind, text: String }, +} + +pub struct LexedItem { + pub span: Span, + pub inner: TokenOrComment, +} + +impl LexedItem { + pub fn span_as_range(&self) -> Range<usize> { + self.span.lo.0 as usize..self.span.hi.0 as usize + } +} + +fn flatten_comments( + comments: SingleThreadedComments, +) -> impl Iterator<Item = Comment> { + let (leading, trailing) = comments.take_all(); + let mut comments = (*leading).clone().into_inner(); + comments.extend((*trailing).clone().into_inner()); + comments.into_iter().flat_map(|el| el.1) +} + +pub fn lex( + specifier: &str, + source: &str, + media_type: &MediaType, +) -> Vec<LexedItem> { + let source_map = SourceMap::default(); + let source_file = source_map.new_source_file( + FileName::Custom(specifier.to_string()), + source.to_string(), + ); + let comments = SingleThreadedComments::default(); + let lexer = Lexer::new( + get_syntax(media_type), + TARGET, + StringInput::from(source_file.as_ref()), + Some(&comments), + ); + + let mut tokens: Vec<LexedItem> = lexer + .map(|token| LexedItem { + span: token.span, + inner: TokenOrComment::Token(token.token), + }) + .collect(); + + tokens.extend(flatten_comments(comments).map(|comment| LexedItem { + span: comment.span, + inner: TokenOrComment::Comment { + kind: comment.kind, + text: comment.text, + }, + })); + + tokens.sort_by_key(|item| item.span.lo.0); + + tokens +} + /// A low level function which transpiles a source module into an swc /// SourceFile. pub fn transpile_module( diff --git a/cli/tools/repl.rs b/cli/tools/repl.rs index 8786f57f3..cd2839115 100644 --- a/cli/tools/repl.rs +++ b/cli/tools/repl.rs @@ -1,14 +1,15 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. +use crate::ast; +use crate::ast::TokenOrComment; use crate::colors; use crate::inspector::InspectorSession; +use crate::media_type::MediaType; use crate::program_state::ProgramState; use crate::worker::MainWorker; use deno_core::error::AnyError; use deno_core::serde_json::json; use deno_core::serde_json::Value; -use regex::Captures; -use regex::Regex; use rustyline::completion::Completer; use rustyline::error::ReadlineError; use rustyline::highlight::Highlighter; @@ -26,6 +27,7 @@ use std::sync::mpsc::Sender; use std::sync::mpsc::SyncSender; use std::sync::Arc; use std::sync::Mutex; +use swc_ecmascript::parser::token::{Token, Word}; // Provides helpers to the editor like validation for multi-line edits, completion candidates for // tab completion. @@ -132,55 +134,42 @@ impl Validator for Helper { &self, ctx: &mut ValidationContext, ) -> Result<ValidationResult, ReadlineError> { - let mut stack: Vec<char> = Vec::new(); - let mut literal: Option<char> = None; - let mut escape: bool = false; - - for c in ctx.input().chars() { - if escape { - escape = false; - continue; - } - - if c == '\\' { - escape = true; - continue; - } - - if let Some(v) = literal { - if c == v { - literal = None - } - - continue; - } else { - literal = match c { - '`' | '"' | '/' | '\'' => Some(c), - _ => None, - }; - } - - match c { - '(' | '[' | '{' => stack.push(c), - ')' | ']' | '}' => match (stack.pop(), c) { - (Some('('), ')') | (Some('['), ']') | (Some('{'), '}') => {} - (Some(left), _) => { - return Ok(ValidationResult::Invalid(Some(format!( - "Mismatched pairs: {:?} is not properly closed", - left - )))) + let mut stack: Vec<Token> = Vec::new(); + let mut in_template = false; + + for item in ast::lex("", ctx.input(), &MediaType::JavaScript) { + if let TokenOrComment::Token(token) = item.inner { + match token { + Token::BackQuote => in_template = !in_template, + Token::LParen + | Token::LBracket + | Token::LBrace + | Token::DollarLBrace => stack.push(token), + Token::RParen | Token::RBracket | Token::RBrace => { + match (stack.pop(), token) { + (Some(Token::LParen), Token::RParen) + | (Some(Token::LBracket), Token::RBracket) + | (Some(Token::LBrace), Token::RBrace) + | (Some(Token::DollarLBrace), Token::RBrace) => {} + (Some(left), _) => { + return Ok(ValidationResult::Invalid(Some(format!( + "Mismatched pairs: {:?} is not properly closed", + left + )))) + } + (None, _) => { + // While technically invalid when unpaired, it should be V8's task to output error instead. + // Thus marked as valid with no info. + return Ok(ValidationResult::Valid(None)); + } + } } - (None, _) => { - // While technically invalid when unpaired, it should be V8's task to output error instead. - // Thus marked as valid with no info. - return Ok(ValidationResult::Valid(None)); - } - }, - _ => {} + _ => {} + } } } - if !stack.is_empty() || literal == Some('`') { + if !stack.is_empty() || in_template { return Ok(ValidationResult::Incomplete); } @@ -210,71 +199,60 @@ impl Highlighter for Helper { } } -struct LineHighlighter { - regex: Regex, -} +struct LineHighlighter; impl LineHighlighter { fn new() -> Self { - let regex = Regex::new( - r#"(?x) - (?P<comment>(?:/\*[\s\S]*?\*/|//[^\n]*)) | - (?P<string>(?:"([^"\\]|\\.)*"|'([^'\\]|\\.)*'|`([^`\\]|\\.)*`)) | - (?P<regexp>/(?:(?:\\/|[^\n/]))*?/[gimsuy]*) | - (?P<number>\b\d+(?:\.\d+)?(?:e[+-]?\d+)*n?\b) | - (?P<infinity>\b(?:Infinity|NaN)\b) | - (?P<hexnumber>\b0x[a-fA-F0-9]+\b) | - (?P<octalnumber>\b0o[0-7]+\b) | - (?P<binarynumber>\b0b[01]+\b) | - (?P<boolean>\b(?:true|false)\b) | - (?P<null>\b(?:null)\b) | - (?P<undefined>\b(?:undefined)\b) | - (?P<keyword>\b(?:await|async|var|let|for|if|else|in|of|class|const|function|yield|return|with|case|break|switch|import|export|new|while|do|throw|catch|this)\b) | - "#, - ) - .unwrap(); - - Self { regex } + Self } } impl Highlighter for LineHighlighter { fn highlight<'l>(&self, line: &'l str, _: usize) -> Cow<'l, str> { - self - .regex - .replace_all(&line.to_string(), |caps: &Captures<'_>| { - if let Some(cap) = caps.name("comment") { - colors::gray(cap.as_str()).to_string() - } else if let Some(cap) = caps.name("string") { - colors::green(cap.as_str()).to_string() - } else if let Some(cap) = caps.name("regexp") { - colors::red(cap.as_str()).to_string() - } else if let Some(cap) = caps.name("number") { - colors::yellow(cap.as_str()).to_string() - } else if let Some(cap) = caps.name("boolean") { - colors::yellow(cap.as_str()).to_string() - } else if let Some(cap) = caps.name("null") { - colors::yellow(cap.as_str()).to_string() - } else if let Some(cap) = caps.name("undefined") { - colors::gray(cap.as_str()).to_string() - } else if let Some(cap) = caps.name("keyword") { - colors::cyan(cap.as_str()).to_string() - } else if let Some(cap) = caps.name("infinity") { - colors::yellow(cap.as_str()).to_string() - } else if let Some(cap) = caps.name("classes") { - colors::green_bold(cap.as_str()).to_string() - } else if let Some(cap) = caps.name("hexnumber") { - colors::yellow(cap.as_str()).to_string() - } else if let Some(cap) = caps.name("octalnumber") { - colors::yellow(cap.as_str()).to_string() - } else if let Some(cap) = caps.name("binarynumber") { - colors::yellow(cap.as_str()).to_string() - } else { - caps[0].to_string() - } - }) - .to_string() - .into() + let mut out_line = String::from(line); + + for item in ast::lex("", line, &MediaType::JavaScript) { + // Adding color adds more bytes to the string, + // so an offset is needed to stop spans falling out of sync. + let offset = out_line.len() - line.len(); + let span = item.span_as_range(); + + out_line.replace_range( + span.start + offset..span.end + offset, + &match item.inner { + TokenOrComment::Token(token) => match token { + Token::Str { .. } | Token::Template { .. } | Token::BackQuote => { + colors::green(&line[span]).to_string() + } + Token::Regex(_, _) => colors::red(&line[span]).to_string(), + Token::Num(_) | Token::BigInt(_) => { + colors::yellow(&line[span]).to_string() + } + Token::Word(word) => match word { + Word::True | Word::False | Word::Null => { + colors::yellow(&line[span]).to_string() + } + Word::Keyword(_) => colors::cyan(&line[span]).to_string(), + Word::Ident(ident) => { + if ident == *"undefined" { + colors::gray(&line[span]).to_string() + } else if ident == *"Infinity" || ident == *"NaN" { + colors::yellow(&line[span]).to_string() + } else { + line[span].to_string() + } + } + }, + _ => line[span].to_string(), + }, + TokenOrComment::Comment { .. } => { + colors::gray(&line[span]).to_string() + } + }, + ); + } + + out_line.into() } } |