Last active
July 20, 2016 23:09
-
-
Save cgswords/d452f9bde0e13326ecf5818201df71b7 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| // Copyright 2012-2016 The Rust Project Developers. See the COPYRIGHT | |
| // file at the top-level directory of this distribution and at | |
| // http://rust-lang.org/COPYRIGHT. | |
| // | |
| // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or | |
| // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license | |
| // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your | |
| // option. This file may not be copied, modified, or distributed | |
| // except according to those terms. | |
| //! # Token Streams | |
| //! | |
| //! TokenStreams represent syntactic objects before they are converted into ASTs. | |
| //! A `TokenStream` is, roughly speaking, a sequence (eg stream) of `TokenTree`s, | |
| //! which are themselves either a single Token, a Delimited subsequence of tokens, | |
| //! or a SequenceRepetition specifier (for the purpose of sequence generation during macro | |
| //! expansion). | |
| //! | |
| //! A TokenStream also has a slice view, `TokenSlice`, that is analogous to `str` for | |
| //! `String`: it allows the programmer to divvy up, explore, and otherwise partition a | |
| //! TokenStream as borrowed subsequences. | |
| use ast::{self, AttrStyle, LitKind}; | |
| use syntax_pos::{Span, DUMMY_SP, NO_EXPANSION}; | |
| use codemap::Spanned; | |
| use ext::base; | |
| use ext::tt::macro_parser; | |
| use parse::lexer::comments::{doc_comment_style, strip_doc_comment_decoration}; | |
| use parse::lexer; | |
| use parse; | |
| use parse::token::{self, Token, Lit, InternedString, Nonterminal}; | |
| use parse::token::Lit as TokLit; | |
| use std::clone; | |
| use std::fmt; | |
| use std::mem; | |
| use std::ops::Index; | |
| use std::ops; | |
| use std::iter::*; | |
| use std::rc::Rc; | |
| /// A delimited sequence of token trees | |
| #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] | |
| pub struct Delimited { | |
| /// The type of delimiter | |
| pub delim: token::DelimToken, | |
| /// The span covering the opening delimiter | |
| pub open_span: Span, | |
| /// The delimited sequence of token trees | |
| pub tts: Vec<TokenTree>, | |
| /// The span covering the closing delimiter | |
| pub close_span: Span, | |
| } | |
| impl Delimited { | |
| /// Returns the opening delimiter as a token. | |
| pub fn open_token(&self) -> token::Token { | |
| token::OpenDelim(self.delim) | |
| } | |
| /// Returns the closing delimiter as a token. | |
| pub fn close_token(&self) -> token::Token { | |
| token::CloseDelim(self.delim) | |
| } | |
| /// Returns the opening delimiter as a token tree. | |
| pub fn open_tt(&self) -> TokenTree { | |
| TokenTree::Token(self.open_span, self.open_token()) | |
| } | |
| /// Returns the closing delimiter as a token tree. | |
| pub fn close_tt(&self) -> TokenTree { | |
| TokenTree::Token(self.close_span, self.close_token()) | |
| } | |
| /// Returns the token trees inside the delimiters. | |
| pub fn subtrees(&self) -> &[TokenTree] { | |
| &self.tts | |
| } | |
| } | |
| /// A sequence of token trees | |
| #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] | |
| pub struct SequenceRepetition { | |
| /// The sequence of token trees | |
| pub tts: Vec<TokenTree>, | |
| /// The optional separator | |
| pub separator: Option<token::Token>, | |
| /// Whether the sequence can be repeated zero (*), or one or more times (+) | |
| pub op: KleeneOp, | |
| /// The number of `MatchNt`s that appear in the sequence (and subsequences) | |
| pub num_captures: usize, | |
| } | |
| /// A Kleene-style [repetition operator](http://en.wikipedia.org/wiki/Kleene_star) | |
| /// for token sequences. | |
| #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)] | |
| pub enum KleeneOp { | |
| ZeroOrMore, | |
| OneOrMore, | |
| } | |
| /// When the main rust parser encounters a syntax-extension invocation, it | |
| /// parses the arguments to the invocation as a token-tree. This is a very | |
| /// loose structure, such that all sorts of different AST-fragments can | |
| /// be passed to syntax extensions using a uniform type. | |
| /// | |
| /// If the syntax extension is an MBE macro, it will attempt to match its | |
| /// LHS token tree against the provided token tree, and if it finds a | |
| /// match, will transcribe the RHS token tree, splicing in any captured | |
| /// macro_parser::matched_nonterminals into the `SubstNt`s it finds. | |
| /// | |
| /// The RHS of an MBE macro is the only place `SubstNt`s are substituted. | |
| /// Nothing special happens to misnamed or misplaced `SubstNt`s. | |
| #[derive(Debug, Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash)] | |
| pub enum TokenTree { | |
| /// A single token | |
| Token(Span, token::Token), | |
| /// A delimited sequence of token trees | |
| Delimited(Span, Rc<Delimited>), | |
| // This only makes sense in MBE macros. | |
| /// A kleene-style repetition sequence with a span | |
| Sequence(Span, Rc<SequenceRepetition>), | |
| } | |
| impl TokenTree { | |
| pub fn len(&self) -> usize { | |
| match *self { | |
| TokenTree::Token(_, token::DocComment(name)) => { | |
| match doc_comment_style(&name.as_str()) { | |
| AttrStyle::Outer => 2, | |
| AttrStyle::Inner => 3, | |
| } | |
| } | |
| TokenTree::Token(_, token::SpecialVarNt(..)) => 2, | |
| TokenTree::Token(_, token::MatchNt(..)) => 3, | |
| TokenTree::Delimited(_, ref delimed) => delimed.tts.len() + 2, | |
| TokenTree::Sequence(_, ref seq) => seq.tts.len(), | |
| TokenTree::Token(..) => 0, | |
| } | |
| } | |
| pub fn get_tt(&self, index: usize) -> TokenTree { | |
| match (self, index) { | |
| (&TokenTree::Token(sp, token::DocComment(_)), 0) => TokenTree::Token(sp, token::Pound), | |
| (&TokenTree::Token(sp, token::DocComment(name)), 1) | |
| if doc_comment_style(&name.as_str()) == AttrStyle::Inner => { | |
| TokenTree::Token(sp, token::Not) | |
| } | |
| (&TokenTree::Token(sp, token::DocComment(name)), _) => { | |
| let stripped = strip_doc_comment_decoration(&name.as_str()); | |
| // Searches for the occurrences of `"#*` and returns the minimum number of `#`s | |
| // required to wrap the text. | |
| let num_of_hashes = stripped.chars() | |
| .scan(0, |cnt, x| { | |
| *cnt = if x == '"' { | |
| 1 | |
| } else if *cnt != 0 && x == '#' { | |
| *cnt + 1 | |
| } else { | |
| 0 | |
| }; | |
| Some(*cnt) | |
| }) | |
| .max() | |
| .unwrap_or(0); | |
| TokenTree::Delimited(sp, Rc::new(Delimited { | |
| delim: token::Bracket, | |
| open_span: sp, | |
| tts: vec![TokenTree::Token(sp, token::Ident(token::str_to_ident("doc"))), | |
| TokenTree::Token(sp, token::Eq), | |
| TokenTree::Token(sp, token::Literal( | |
| token::StrRaw(token::intern(&stripped), num_of_hashes), None))], | |
| close_span: sp, | |
| })) | |
| } | |
| (&TokenTree::Delimited(_, ref delimed), _) => { | |
| if index == 0 { | |
| return delimed.open_tt(); | |
| } | |
| if index == delimed.tts.len() + 1 { | |
| return delimed.close_tt(); | |
| } | |
| delimed.tts[index - 1].clone() | |
| } | |
| (&TokenTree::Token(sp, token::SpecialVarNt(var)), _) => { | |
| let v = [TokenTree::Token(sp, token::Dollar), | |
| TokenTree::Token(sp, token::Ident(token::str_to_ident(var.as_str())))]; | |
| v[index].clone() | |
| } | |
| (&TokenTree::Token(sp, token::MatchNt(name, kind)), _) => { | |
| let v = [TokenTree::Token(sp, token::SubstNt(name)), | |
| TokenTree::Token(sp, token::Colon), | |
| TokenTree::Token(sp, token::Ident(kind))]; | |
| v[index].clone() | |
| } | |
| (&TokenTree::Sequence(_, ref seq), _) => seq.tts[index].clone(), | |
| _ => panic!("Cannot expand a token tree"), | |
| } | |
| } | |
| /// Returns the `Span` corresponding to this token tree. | |
| pub fn get_span(&self) -> Span { | |
| match *self { | |
| TokenTree::Token(span, _) => span, | |
| TokenTree::Delimited(span, _) => span, | |
| TokenTree::Sequence(span, _) => span, | |
| } | |
| } | |
| /// Use this token tree as a matcher to parse given tts. | |
| pub fn parse(cx: &base::ExtCtxt, | |
| mtch: &[TokenTree], | |
| tts: &[TokenTree]) | |
| -> macro_parser::NamedParseResult { | |
| // `None` is because we're not interpolating | |
| let arg_rdr = lexer::new_tt_reader_with_doc_flag(&cx.parse_sess().span_diagnostic, | |
| None, | |
| None, | |
| tts.iter().cloned().collect(), | |
| true); | |
| macro_parser::parse(cx.parse_sess(), cx.cfg(), arg_rdr, mtch) | |
| } | |
| /// Check if this TokenTree is equal to the other, regardless of span information. | |
| pub fn eq_unspanned(&self, other: &TokenTree) -> bool { | |
| match (self, other) { | |
| (&TokenTree::Token(_, ref tk), &TokenTree::Token(_, ref tk2)) => tk == tk2, | |
| (&TokenTree::Delimited(_, ref dl), &TokenTree::Delimited(_, ref dl2)) => { | |
| (*dl).delim == (*dl2).delim && dl.tts.len() == dl2.tts.len() && | |
| { | |
| for (tt1, tt2) in dl.tts.iter().zip(dl2.tts.iter()) { | |
| if !tt1.eq_unspanned(tt2) { | |
| return false; | |
| } | |
| } | |
| true | |
| } | |
| } | |
| (_, _) => false, | |
| } | |
| } | |
| /// Retrieve the TokenTree's span. | |
| pub fn span(&self) -> Span { | |
| match *self { | |
| TokenTree::Token(sp, _) | | |
| TokenTree::Delimited(sp, _) | | |
| TokenTree::Sequence(sp, _) => sp, | |
| } | |
| } | |
| /// Indicates if the stream is a token that is equal to the provided token. | |
| pub fn eq_token(&self, t: Token) -> bool { | |
| match *self { | |
| TokenTree::Token(_, ref tk) => *tk == t, | |
| _ => false, | |
| } | |
| } | |
| /// Indicates if the token is an identifier. | |
| pub fn is_ident(&self) -> bool { | |
| self.maybe_ident().is_some() | |
| } | |
| /// Returns an identifier. | |
| pub fn maybe_ident(&self) -> Option<ast::Ident> { | |
| match *self { | |
| TokenTree::Token(_, Token::Ident(t)) => Some(t.clone()), | |
| TokenTree::Delimited(_, ref dl) => { | |
| let tts = dl.subtrees(); | |
| if tts.len() != 1 { | |
| return None; | |
| } | |
| tts[0].maybe_ident() | |
| } | |
| _ => None, | |
| } | |
| } | |
| /// Returns a Token literal. | |
| pub fn maybe_lit(&self) -> Option<token::Lit> { | |
| match *self { | |
| TokenTree::Token(_, Token::Literal(l, _)) => Some(l.clone()), | |
| TokenTree::Delimited(_, ref dl) => { | |
| let tts = dl.subtrees(); | |
| if tts.len() != 1 { | |
| return None; | |
| } | |
| tts[0].maybe_lit() | |
| } | |
| _ => None, | |
| } | |
| } | |
| /// Returns an AST string literal. | |
| pub fn maybe_str(&self) -> Option<ast::Lit> { | |
| match *self { | |
| TokenTree::Token(sp, Token::Literal(Lit::Str_(s), _)) => { | |
| let l = LitKind::Str(token::intern_and_get_ident(&parse::str_lit(&s.as_str())), | |
| ast::StrStyle::Cooked); | |
| Some(Spanned { | |
| node: l, | |
| span: sp, | |
| }) | |
| } | |
| TokenTree::Token(sp, Token::Literal(Lit::StrRaw(s, n), _)) => { | |
| let l = LitKind::Str(token::intern_and_get_ident(&parse::raw_str_lit(&s.as_str())), | |
| ast::StrStyle::Raw(n)); | |
| Some(Spanned { | |
| node: l, | |
| span: sp, | |
| }) | |
| } | |
| _ => None, | |
| } | |
| } | |
| } | |
| /// #Token Streams | |
| /// | |
| /// TokenStreams are a syntactic abstraction over TokenTrees. The goal is for procedural | |
| /// macros to work over TokenStreams instead of arbitrary syntax. For now, however, we | |
| /// are going to cut a few corners (i.e., use some of the AST structure) when we need to | |
| /// for backwards compatibility. | |
| /// TokenStreams are collections of TokenTrees that represent a syntactic structure. The | |
| /// struct itself shouldn't be directly manipulated; the internal structure is not stable, | |
| /// and may be changed at any time in the future. The operators will not, however (except | |
| /// for signatures, later on). | |
| // #[derive(Eq,Clone,Hash,RustcEncodable,RustcDecodable)] | |
| #[derive(PartialEq,Eq,Hash,RustcEncodable,RustcDecodable)] | |
| pub enum TokenStream { | |
| Empty, | |
| Leaf { tts : Rc<Vec<TokenTree>>, sp: Span }, | |
| SubLeaf { tts : Rc<Vec<TokenTree>>, offset: usize, len: usize, sp: Span }, | |
| Node { left: Rc<TokenStream>, right: Rc<TokenStream>, len: usize, sp: Span}, | |
| Sub { rope: Rc<TokenStream>, offset: usize, len: usize, sp: Span}, | |
| } | |
| impl clone::Clone for TokenStream { | |
| fn clone(&self) -> Self { | |
| match *self { | |
| TokenStream::Empty => TokenStream::Empty, | |
| TokenStream::Leaf { ref tts, sp} => TokenStream::Leaf { tts: tts.clone(), sp: sp }, | |
| TokenStream::SubLeaf { ref tts, offset, len, sp } => | |
| TokenStream::SubLeaf { tts: tts.clone(), offset: offset, len: len, sp: sp }, | |
| TokenStream::Node { ref left, ref right, len, sp } => | |
| TokenStream::Node { left: left.clone(), right: right.clone(), len: len, sp: sp }, | |
| TokenStream::Sub { ref rope, offset, len, sp } => | |
| TokenStream::Sub { rope: rope.clone(), offset: offset, len: len, sp: sp } | |
| } | |
| } | |
| // fn clone_from(&mut self, source: &Self) { ... } | |
| } | |
| impl fmt::Debug for TokenStream { | |
| fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { | |
| match *self { | |
| TokenStream::Empty => { write!(f, "") } | |
| TokenStream::Leaf{ ref tts, .. } => { | |
| for t in tts.iter() { | |
| write!(f, "{:?}", t)?; | |
| } | |
| write!(f, "") | |
| } | |
| TokenStream::SubLeaf{ ref tts, offset, len, ..} => { | |
| for t in tts.iter().skip(offset).take(len) { | |
| write!(f, "{:?}", t)?; | |
| } | |
| write!(f, "") | |
| } | |
| TokenStream::Node{ ref left, ref right, .. } => { | |
| left.fmt(f)?; | |
| right.fmt(f)?; | |
| write!(f, "") | |
| } | |
| TokenStream::Sub{ ref rope, offset, len, ..} => { | |
| for i in offset..len + offset { | |
| write!(f, "{:?}", rope[i])?; | |
| } | |
| write!(f, "") | |
| } | |
| } | |
| } | |
| } | |
| // /// Checks if two TokenStreams are equivalent (including spans). For unspanned | |
| // /// equality, see `eq_unspanned`. | |
| // impl PartialEq<TokenStream> for TokenStream { | |
| // fn eq(&self, other: &TokenStream) -> bool { | |
| // false | |
| // // FIXME implement equality | |
| // } | |
| // } | |
| // NB this will disregard gaps. if we have [a|{2,5} , b|{11,13}], the resultant span | |
| // will be at {2,13}. Without finer-grained span structures, however, this seems to be | |
| // our only recourse. | |
| // FIXME Do something smarter to compute the expansion id. | |
| fn covering_span(trees: &[TokenTree]) -> Span { | |
| // disregard any dummy spans we have | |
| let trees = trees.iter().filter(|t| t.span() != DUMMY_SP).collect::<Vec<&TokenTree>>(); | |
| // if we're out of spans, stop | |
| if trees.len() < 1 { | |
| return DUMMY_SP; | |
| } | |
| // set up the initial values | |
| let fst_span = trees[0].span(); | |
| let mut lo_span = fst_span.lo; | |
| let mut hi_span = fst_span.hi; | |
| let mut expn_id = fst_span.expn_id; | |
| // compute the spans iteratively | |
| for t in trees.iter().skip(1) { | |
| let sp = t.span(); | |
| if sp.lo < lo_span { | |
| lo_span = sp.lo; | |
| } | |
| if hi_span < sp.hi { | |
| hi_span = sp.hi; | |
| } | |
| if expn_id != sp.expn_id { | |
| expn_id = NO_EXPANSION; | |
| } | |
| } | |
| Span { | |
| lo: lo_span, | |
| hi: hi_span, | |
| expn_id: expn_id, | |
| } | |
| } | |
| fn combine_spans(sp1: Span, sp2: Span) -> Span { | |
| if sp1 == DUMMY_SP && sp2 == DUMMY_SP { | |
| DUMMY_SP | |
| } else if sp1 == DUMMY_SP { | |
| sp2 | |
| } else if sp2 == DUMMY_SP { | |
| sp1 | |
| } else { | |
| Span { | |
| lo: if sp1.lo < sp2.lo { sp1.lo } else { sp2.lo }, | |
| hi: if sp1.hi > sp2.hi { sp1.hi } else { sp2.hi }, | |
| expn_id: if sp1.expn_id == sp2.expn_id { sp1.expn_id } else { NO_EXPANSION } | |
| } | |
| } | |
| } | |
| /// TokenStream operators include basic destructuring, boolean operations, `maybe_...` | |
| /// operations, and `maybe_..._prefix` operations. Boolean operations are straightforward, | |
| /// indicating information about the structure of the stream. The `maybe_...` operations | |
| /// return `Some<...>` if the tokenstream contains the appropriate item. | |
| /// | |
| /// Similarly, the `maybe_..._prefix` operations potentially return a | |
| /// partially-destructured stream as a pair where the first element is the expected item | |
| /// and the second is the remainder of the stream. As anb example, | |
| /// | |
| /// `maybe_path_prefix("a::b::c(a,b,c).foo()") -> (a::b::c, "(a,b,c).foo()")` | |
| impl TokenStream { | |
| /// Convert a vector of `TokenTree`s into a `TokenStream`. | |
| pub fn from_tts(trees: Vec<TokenTree>) -> TokenStream { | |
| let span = covering_span(&trees[..]); | |
| TokenStream::Leaf { | |
| tts: Rc::new(trees), | |
| sp: span, | |
| } | |
| } | |
| /// Copies all of the TokenTrees from the TokenSlice, appending them to the stream. | |
| pub fn concat(left: TokenStream, right: TokenStream) -> TokenStream { | |
| let new_len = left.len() + right.len(); | |
| let new_span = combine_spans(left.span(), right.span()); | |
| TokenStream::Node{ | |
| left : Rc::new(left), | |
| right: Rc::new(right), | |
| len: new_len, | |
| sp: new_span, | |
| } | |
| } | |
| /// Manually change a TokenStream's span. | |
| pub fn is_empty(&self) -> bool { | |
| self.len() == 0 | |
| } | |
| /// Return a TokenSlice's length | |
| pub fn len(&self) -> usize { | |
| match *self { | |
| TokenStream::Empty => 0, | |
| TokenStream::Leaf { ref tts, .. } => tts.len(), | |
| TokenStream::SubLeaf { len, .. } => len, | |
| TokenStream::Node { len, .. } => len, | |
| TokenStream::Sub { len, .. } => len, | |
| } | |
| } | |
| //FIXME could be O(n) instead of O(n log n) by doing a direct traversal | |
| pub fn to_vec(&self) -> Vec<&TokenTree> { | |
| let mut vec = Vec::new(); | |
| for i in 0..self.len() { | |
| vec.push(&self[i]); | |
| } | |
| vec | |
| } | |
| // NB EXPENSIVE! | |
| pub fn to_tts(&self) -> Vec<TokenTree> { | |
| let mut vec = Vec::new(); | |
| for i in 0..self.len() { | |
| vec.push(self[i].clone()); | |
| } | |
| vec | |
| } | |
| /// Manually change a TokenStream's span. | |
| pub fn span(&self) -> Span { | |
| match *self { | |
| TokenStream::Empty => DUMMY_SP, | |
| TokenStream::Leaf { sp, .. } | | |
| TokenStream::SubLeaf { sp, .. } | | |
| TokenStream::Node { sp, .. } | | |
| TokenStream::Sub { sp, .. } => sp, | |
| } | |
| } | |
| /// Returns an iterator over a TokenSlice (as a sequence of TokenStreams). | |
| pub fn iter<'a>(&self) -> Iter { | |
| Iter { vs: &self , idx: 0 } | |
| } | |
| /// Splits a TokenSlice based on the provided `&TokenTree -> bool` predicate. | |
| pub fn split<P>(&self, pred: P) -> Split<P> | |
| where P: FnMut(&TokenTree) -> bool | |
| { | |
| Split { | |
| vs: self, | |
| pred: pred, | |
| finished: false, | |
| idx: 0, | |
| } | |
| } | |
| pub fn slice(&self, from: usize, to: usize) -> TokenStream { | |
| let slen = self.len(); | |
| if from == to { return TokenStream::Empty; } | |
| if from > to { panic!("Invalid range: {} to {}", from, to); } | |
| if from == 0 && to == self.len() { return self.clone(); /* should be cheap */ } | |
| match *self { | |
| TokenStream::Empty => panic!("Invalid index"), | |
| TokenStream::Leaf { ref tts, sp } => | |
| TokenStream::SubLeaf { | |
| tts: tts.clone(), // Rc | |
| offset: from, | |
| len: to - from, | |
| sp: sp, | |
| }, | |
| TokenStream::SubLeaf { ref tts, offset, len, sp } => | |
| TokenStream::SubLeaf { | |
| tts: tts.clone(), // Rc | |
| offset: offset + from, | |
| len: len - (to - from), | |
| sp: sp, | |
| }, | |
| TokenStream::Node { ref left, ref right, len, sp } => { | |
| let left_len = left.len(); | |
| if to <= left_len { | |
| left.slice(from, to) | |
| } else if from >= left_len { | |
| right.slice(from - left_len, to - left_len) | |
| } else { | |
| TokenStream::concat(left.slice(from, left_len), right.slice(0, to - left_len)) | |
| } | |
| } | |
| TokenStream::Sub { ref rope, offset, len, sp } => | |
| TokenStream::Sub { | |
| rope: rope.clone(), // Rc | |
| offset: offset + from, | |
| len: to - from, | |
| sp: sp, | |
| }, | |
| } | |
| } | |
| pub fn slice_from(&self, from: usize) -> TokenStream { | |
| self.slice(from, self.len()) | |
| } | |
| pub fn slice_to(&self, to: usize) -> TokenStream { | |
| self.slice(0, to) | |
| } | |
| pub struct Iter<'a> { | |
| vs: &'a TokenStream, | |
| idx: usize, | |
| } | |
| impl<'a> Iterator for Iter<'a> { | |
| type Item = &'a TokenTree; | |
| fn next(&mut self) -> Option<&'a TokenTree> { | |
| if self.idx >= self.vs.len() { | |
| return None; | |
| } | |
| let ret = Some(&self.vs[self.idx]); | |
| self.idx = self.idx + 1; | |
| ret | |
| } | |
| } | |
| pub struct Split<'a, P> | |
| where P: FnMut(&TokenTree) -> bool | |
| { | |
| vs: &'a TokenStream, | |
| pred: P, | |
| finished: bool, | |
| idx: usize, | |
| } | |
| impl<'a, P> Iterator for Split<'a, P> | |
| where P: FnMut(&TokenTree) -> bool | |
| { | |
| type Item = TokenStream; | |
| fn next(&mut self) -> Option<TokenStream> { | |
| if self.finished { | |
| return None; | |
| } | |
| if self.idx >= self.vs.len() { | |
| self.finished = true; | |
| return None; | |
| } | |
| let mut lookup = self.vs.iter().skip(self.idx); | |
| match lookup.position(|x| (self.pred)(&x)) { | |
| None => { | |
| self.finished = true; | |
| Some(self.vs.slice_from(self.idx)) | |
| } | |
| Some(edx) => { | |
| let ret = Some(self.vs.slice(self.idx,edx)); | |
| self.idx = edx+1; | |
| ret | |
| } | |
| } | |
| } | |
| } | |
| impl Index<usize> for TokenStream { | |
| type Output = TokenTree; | |
| fn index(&self, index: usize) -> &TokenTree { | |
| match *self { | |
| TokenStream::Empty => panic!("Invalid index"), | |
| TokenStream::Leaf { ref tts, .. } => tts.get(0).unwrap(), | |
| TokenStream::SubLeaf { ref tts, offset, .. } => tts.get(0).unwrap(), | |
| TokenStream::Sub { ref rope, offset, .. } => Index::index(&**rope, index + offset), | |
| TokenStream::Node { ref left, ref right, .. } => { | |
| let left_len = left.len(); | |
| if left_len < index { | |
| Index::index(&**right, index - left_len) | |
| } else { | |
| Index::index(&**left, index) | |
| } | |
| } | |
| } | |
| } | |
| } | |
| // /// Manually change a TokenStream's span. | |
| // pub fn respan(self, span: Span) -> TokenStream { | |
| // match self { | |
| // TokenStream::Empty => Empty, | |
| // TokenStream::Leaf { tts, sp } => TokenStream::Leaf { tts : tts, sp : span }, | |
| // TokenStream::Node { left , right, len, sp } => | |
| // TokenStream::Node { left: left, right: right, len: len, sp: span }, | |
| // TokenStream::Sub { rope, offset, len, sp } => | |
| // TokenStream::Sub { rope: rope, offset: offset, len: len, sp: span } | |
| // } | |
| // } | |
| // /// Construct a TokenStream from an ast literal. | |
| // pub fn from_ast_lit_str(lit: ast::Lit) -> Option<TokenStream> { | |
| // match lit.node { | |
| // LitKind::Str(val, _) => { | |
| // let val = TokLit::Str_(token::intern(&val)); | |
| // let vec = vec![TokenTree::Token(lit.span,Token::Literal(val, None))]; | |
| // Some(TokenStream::from_tts(&vec)) | |
| // } | |
| // _ => None, | |
| // } | |
| // } | |
| // /// Convert a vector of TokenTrees into a parentheses-delimited TokenStream. | |
| // pub fn as_paren_delimited_stream(tts: Vec<TokenTree>) -> TokenStream { | |
| // let new_sp = covering_span(&tts); | |
| // let new_delim = Rc::new(Delimited { | |
| // delim: token::DelimToken::Paren, | |
| // open_span: DUMMY_SP, | |
| // tts: tts, | |
| // close_span: DUMMY_SP, | |
| // }); | |
| // TokenStream::from_tts(vec![TokenTree::Delimited(new_sp, new_delim)]) | |
| // } | |
| // /// Convert an interned string into a one-element TokenStream. | |
| // pub fn from_interned_string_as_ident(s: InternedString) -> TokenStream { | |
| // TokenStream::from_tts(vec![TokenTree::Token(DUMMY_SP, | |
| // Token::Ident(token::str_to_ident(&s[..])))]) | |
| // } | |
| // // FIXME write actual equality | |
| // pub fn eq_unspanned(&self, other: &TokenStream) -> bool { | |
| // false | |
| // } | |
| // /// Indicates where the stream is of the form `= <ts>`, where `<ts>` is a continued | |
| // /// `TokenStream`. | |
| // pub fn is_assignment(&self) -> bool { | |
| // self.maybe_assignment().is_some() | |
| // } | |
| // /// Returns the RHS of an assigment. | |
| // pub fn maybe_assignment(&self) -> Option<TokenStream> { | |
| // if !(self.len() > 1) { return None; } | |
| // if !(self[0].eq_token(Token::Eq)) { return None; } | |
| // Some(self[1..]) | |
| // } | |
| // /// Indicates where the stream is a single, delimited expression (e.g., `(a,b,c)` or | |
| // /// `{a,b,c}`). | |
| // pub fn is_delimited(&self) -> bool { | |
| // self.maybe_delimited().is_some() | |
| // } | |
| // /// Returns the inside of the delimited term as a new TokenStream. | |
| // pub fn maybe_delimited(&self) -> Option<TokenStream> { | |
| // None | |
| // // FIXME implement this | |
| // // if !(self.len() == 1) { | |
| // // return None; | |
| // // } | |
| // // match self[0] { | |
| // // TokenTree::Delimited(_, ref rc) => Some(TokenSlice::from_tts(&*rc.tts)), | |
| // // _ => None, | |
| // // } | |
| // } | |
| } | |
| /// TokenSlices are 'views' of `TokenStream's; they fit the same role as `str`s do for | |
| /// `String`s. In general, most TokenStream manipulations will be refocusing their internal | |
| /// contents by taking a TokenSlice and then using indexing and the provided operators. | |
| // #[derive(PartialEq, Eq, Debug)] | |
| // pub struct TokenSlice([TokenTree]); | |
| // impl ops::Deref for TokenStream { | |
| // type Target = TokenSlice; | |
| // | |
| // fn deref(&self) -> &TokenSlice { | |
| // let tts: &[TokenTree] = &*self.tts; | |
| // unsafe { mem::transmute(tts) } | |
| // } | |
| // } | |
| // impl TokenSlice { | |
| // | |
| // /// Returns a list of `TokenSlice`s if the stream is a delimited list, breaking the | |
| // /// stream on commas. | |
| // pub fn maybe_comma_list(&self) -> Option<Vec<&TokenSlice>> { | |
| // let maybe_tts = self.maybe_delimited(); | |
| // | |
| // let ts: &TokenSlice; | |
| // match maybe_tts { | |
| // Some(t) => { | |
| // ts = t; | |
| // } | |
| // None => { | |
| // return None; | |
| // } | |
| // } | |
| // | |
| // let splits: Vec<&TokenSlice> = ts.split(|x| match *x { | |
| // TokenTree::Token(_, Token::Comma) => true, | |
| // _ => false, | |
| // }) | |
| // .filter(|x| x.len() > 0) | |
| // .collect(); | |
| // | |
| // Some(splits) | |
| // } | |
| // | |
| // /// Returns a Nonterminal if it is Interpolated. | |
| // pub fn maybe_interpolated_nonterminal(&self) -> Option<Nonterminal> { | |
| // if !(self.len() == 1) { | |
| // return None; | |
| // } | |
| // | |
| // match self[0] { | |
| // TokenTree::Token(_, Token::Interpolated(ref nt)) => Some(nt.clone()), | |
| // _ => None, | |
| // } | |
| // } | |
| // | |
| // /// Indicates if the stream is exactly one identifier. | |
| // pub fn is_ident(&self) -> bool { | |
| // self.maybe_ident().is_some() | |
| // } | |
| // | |
| // /// Returns an identifier | |
| // pub fn maybe_ident(&self) -> Option<ast::Ident> { | |
| // if !(self.len() == 1) { | |
| // return None; | |
| // } | |
| // | |
| // let tok = if let Some(tts) = self.maybe_delimited() { | |
| // if tts.len() != 1 { | |
| // return None; | |
| // } | |
| // &tts[0] | |
| // } else { | |
| // &self[0] | |
| // }; | |
| // | |
| // match *tok { | |
| // TokenTree::Token(_, Token::Ident(t)) => Some(t), | |
| // _ => None, | |
| // } | |
| // } | |
| // | |
| // /// Indicates if the stream is exactly one literal | |
| // pub fn is_lit(&self) -> bool { | |
| // self.maybe_lit().is_some() | |
| // } | |
| // | |
| // /// Returns a literal | |
| // pub fn maybe_lit(&self) -> Option<token::Lit> { | |
| // if !(self.len() == 1) { | |
| // return None; | |
| // } | |
| // | |
| // let tok = if let Some(tts) = self.maybe_delimited() { | |
| // if tts.len() != 1 { | |
| // return None; | |
| // } | |
| // &tts[0] | |
| // } else { | |
| // &self[0] | |
| // }; | |
| // | |
| // match *tok { | |
| // TokenTree::Token(_, Token::Literal(l, _)) => Some(l), | |
| // _ => None, | |
| // } | |
| // } | |
| // | |
| // /// Returns an AST string literal if the TokenStream is either a normal ('cooked') or | |
| // /// raw string literal. | |
| // pub fn maybe_str(&self) -> Option<ast::Lit> { | |
| // if !(self.len() == 1) { | |
| // return None; | |
| // } | |
| // | |
| // match self[0] { | |
| // TokenTree::Token(sp, Token::Literal(Lit::Str_(s), _)) => { | |
| // let l = LitKind::Str(token::intern_and_get_ident(&parse::str_lit(&s.as_str())), | |
| // ast::StrStyle::Cooked); | |
| // Some(Spanned { | |
| // node: l, | |
| // span: sp, | |
| // }) | |
| // } | |
| // TokenTree::Token(sp, Token::Literal(Lit::StrRaw(s, n), _)) => { | |
| // let l = LitKind::Str(token::intern_and_get_ident(&parse::raw_str_lit(&s.as_str())), | |
| // ast::StrStyle::Raw(n)); | |
| // Some(Spanned { | |
| // node: l, | |
| // span: sp, | |
| // }) | |
| // } | |
| // _ => None, | |
| // } | |
| // } | |
| // | |
| // /// This operation extracts the path prefix , returning an AST path struct and the remainder | |
| // /// of the stream (if it finds one). To be more specific, a tokenstream that has a valid, | |
| // /// non-global path as a prefix (eg `foo(bar, baz)`, `foo::bar(bar)`, but *not* | |
| // /// `::foo::bar(baz)`) will yield the path and the remaining tokens (as a slice). The previous | |
| // /// examples will yield | |
| // /// `Some((Path { segments = vec![foo], ... }, [(bar, baz)]))`, | |
| // /// `Some((Path { segments = vec![foo, bar] }, [(baz)]))`, | |
| // /// and `None`, respectively. | |
| // pub fn maybe_path_prefix(&self) -> Option<(ast::Path, &TokenSlice)> { | |
| // let mut segments: Vec<ast::PathSegment> = Vec::new(); | |
| // | |
| // let path: Vec<&TokenTree> = self.iter() | |
| // .take_while(|x| x.is_ident() || x.eq_token(Token::ModSep)) | |
| // .collect::<Vec<&TokenTree>>(); | |
| // | |
| // let path_size = path.len(); | |
| // if path_size == 0 { | |
| // return None; | |
| // } | |
| // | |
| // let cov_span = self[..path_size].covering_span(); | |
| // let rst = &self[path_size..]; | |
| // | |
| // let fst_id = path[0]; | |
| // | |
| // if let Some(id) = fst_id.maybe_ident() { | |
| // segments.push(ast::PathSegment { | |
| // identifier: id, | |
| // parameters: ast::PathParameters::none(), | |
| // }); | |
| // } else { | |
| // return None; | |
| // } | |
| // | |
| // // Let's use a state machine to parse out the rest. | |
| // enum State { | |
| // Mod, // Expect a `::`, or return None otherwise. | |
| // Ident, // Expect an ident, or return None otherwise. | |
| // } | |
| // let mut state = State::Mod; | |
| // | |
| // for p in &path[1..] { | |
| // match state { | |
| // State::Mod => { | |
| // // State 0: ['::' -> state 1, else return None] | |
| // if p.eq_token(Token::ModSep) { | |
| // state = State::Ident; | |
| // } else { | |
| // return None; | |
| // } | |
| // } | |
| // State::Ident => { | |
| // // State 1: [ident -> state 0, else return None] | |
| // if let Some(id) = p.maybe_ident() { | |
| // segments.push(ast::PathSegment { | |
| // identifier: id, | |
| // parameters: ast::PathParameters::none(), | |
| // }); | |
| // state = State::Mod; | |
| // } else { | |
| // return None; | |
| // } | |
| // } | |
| // } | |
| // } | |
| // | |
| // let path = ast::Path { | |
| // span: cov_span, | |
| // global: false, | |
| // segments: segments, | |
| // }; | |
| // Some((path, rst)) | |
| // } | |
| // } | |
| // #[cfg(test)] | |
| // mod tests { | |
| // use super::*; | |
| // use ast; | |
| // use syntax_pos::{Span, BytePos, NO_EXPANSION, DUMMY_SP}; | |
| // use parse::token::{self, str_to_ident, Token, Lit}; | |
| // use util::parser_testing::string_to_tts; | |
| // use std::rc::Rc; | |
| // | |
| // fn sp(a: u32, b: u32) -> Span { | |
| // Span { | |
| // lo: BytePos(a), | |
| // hi: BytePos(b), | |
| // expn_id: NO_EXPANSION, | |
| // } | |
| // } | |
| // | |
| // #[test] | |
| // fn test_is_empty() { | |
| // let test0 = TokenStream::from_tts(Vec::new()); | |
| // let test1 = TokenStream::from_tts(vec![TokenTree::Token(sp(0, 1), | |
| // Token::Ident(str_to_ident("a")))]); | |
| // let test2 = TokenStream::from_tts(string_to_tts("foo(bar::baz)".to_string())); | |
| // | |
| // assert_eq!(test0.is_empty(), true); | |
| // assert_eq!(test1.is_empty(), false); | |
| // assert_eq!(test2.is_empty(), false); | |
| // } | |
| // | |
| // #[test] | |
| // fn test_is_delimited() { | |
| // let test0 = TokenStream::from_tts(string_to_tts("foo(bar::baz)".to_string())); | |
| // let test1 = TokenStream::from_tts(string_to_tts("(bar::baz)".to_string())); | |
| // let test2 = TokenStream::from_tts(string_to_tts("(foo,bar,baz)".to_string())); | |
| // let test3 = TokenStream::from_tts(string_to_tts("(foo,bar,baz)(zab,rab,oof)".to_string())); | |
| // let test4 = TokenStream::from_tts(string_to_tts("(foo,bar,baz)foo".to_string())); | |
| // let test5 = TokenStream::from_tts(string_to_tts("".to_string())); | |
| // | |
| // assert_eq!(test0.is_delimited(), false); | |
| // assert_eq!(test1.is_delimited(), true); | |
| // assert_eq!(test2.is_delimited(), true); | |
| // assert_eq!(test3.is_delimited(), false); | |
| // assert_eq!(test4.is_delimited(), false); | |
| // assert_eq!(test5.is_delimited(), false); | |
| // } | |
| // | |
| // #[test] | |
| // fn test_is_assign() { | |
| // let test0 = TokenStream::from_tts(string_to_tts("= bar::baz".to_string())); | |
| // let test1 = TokenStream::from_tts(string_to_tts("= \"5\"".to_string())); | |
| // let test2 = TokenStream::from_tts(string_to_tts("= 5".to_string())); | |
| // let test3 = TokenStream::from_tts(string_to_tts("(foo = 10)".to_string())); | |
| // let test4 = TokenStream::from_tts(string_to_tts("= (foo,bar,baz)".to_string())); | |
| // let test5 = TokenStream::from_tts(string_to_tts("".to_string())); | |
| // | |
| // assert_eq!(test0.is_assignment(), true); | |
| // assert_eq!(test1.is_assignment(), true); | |
| // assert_eq!(test2.is_assignment(), true); | |
| // assert_eq!(test3.is_assignment(), false); | |
| // assert_eq!(test4.is_assignment(), true); | |
| // assert_eq!(test5.is_assignment(), false); | |
| // } | |
| // | |
| // #[test] | |
| // fn test_is_lit() { | |
| // let test0 = TokenStream::from_tts(string_to_tts("\"foo\"".to_string())); | |
| // let test1 = TokenStream::from_tts(string_to_tts("5".to_string())); | |
| // let test2 = TokenStream::from_tts(string_to_tts("foo".to_string())); | |
| // let test3 = TokenStream::from_tts(string_to_tts("foo::bar".to_string())); | |
| // let test4 = TokenStream::from_tts(string_to_tts("foo(bar)".to_string())); | |
| // | |
| // assert_eq!(test0.is_lit(), true); | |
| // assert_eq!(test1.is_lit(), true); | |
| // assert_eq!(test2.is_lit(), false); | |
| // assert_eq!(test3.is_lit(), false); | |
| // assert_eq!(test4.is_lit(), false); | |
| // } | |
| // | |
| // #[test] | |
| // fn test_is_ident() { | |
| // let test0 = TokenStream::from_tts(string_to_tts("\"foo\"".to_string())); | |
| // let test1 = TokenStream::from_tts(string_to_tts("5".to_string())); | |
| // let test2 = TokenStream::from_tts(string_to_tts("foo".to_string())); | |
| // let test3 = TokenStream::from_tts(string_to_tts("foo::bar".to_string())); | |
| // let test4 = TokenStream::from_tts(string_to_tts("foo(bar)".to_string())); | |
| // | |
| // assert_eq!(test0.is_ident(), false); | |
| // assert_eq!(test1.is_ident(), false); | |
| // assert_eq!(test2.is_ident(), true); | |
| // assert_eq!(test3.is_ident(), false); | |
| // assert_eq!(test4.is_ident(), false); | |
| // } | |
| // | |
| // #[test] | |
| // fn test_maybe_assignment() { | |
| // let test0_input = TokenStream::from_tts(string_to_tts("= bar::baz".to_string())); | |
| // let test1_input = TokenStream::from_tts(string_to_tts("= \"5\"".to_string())); | |
| // let test2_input = TokenStream::from_tts(string_to_tts("= 5".to_string())); | |
| // let test3_input = TokenStream::from_tts(string_to_tts("(foo = 10)".to_string())); | |
| // let test4_input = TokenStream::from_tts(string_to_tts("= (foo,bar,baz)".to_string())); | |
| // let test5_input = TokenStream::from_tts(string_to_tts("".to_string())); | |
| // | |
| // let test0 = test0_input.maybe_assignment(); | |
| // let test1 = test1_input.maybe_assignment(); | |
| // let test2 = test2_input.maybe_assignment(); | |
| // let test3 = test3_input.maybe_assignment(); | |
| // let test4 = test4_input.maybe_assignment(); | |
| // let test5 = test5_input.maybe_assignment(); | |
| // | |
| // let test0_expected = TokenStream::from_tts(vec![TokenTree::Token(sp(2, 5), | |
| // token::Ident(str_to_ident("bar"))), | |
| // TokenTree::Token(sp(5, 7), token::ModSep), | |
| // TokenTree::Token(sp(7, 10), | |
| // token::Ident(str_to_ident("baz")))]); | |
| // assert_eq!(test0, Some(&test0_expected[..])); | |
| // | |
| // let test1_expected = TokenStream::from_tts(vec![TokenTree::Token(sp(2, 5), | |
| // token::Literal(Lit::Str_(token::intern("5")), None))]); | |
| // assert_eq!(test1, Some(&test1_expected[..])); | |
| // | |
| // let test2_expected = TokenStream::from_tts(vec![TokenTree::Token( sp(2,3) | |
| // , token::Literal( | |
| // Lit::Integer( | |
| // token::intern(&(5.to_string()))), | |
| // None))]); | |
| // assert_eq!(test2, Some(&test2_expected[..])); | |
| // | |
| // assert_eq!(test3, None); | |
| // | |
| // | |
| // let test4_tts = vec![TokenTree::Token(sp(3, 6), token::Ident(str_to_ident("foo"))), | |
| // TokenTree::Token(sp(6, 7), token::Comma), | |
| // TokenTree::Token(sp(7, 10), token::Ident(str_to_ident("bar"))), | |
| // TokenTree::Token(sp(10, 11), token::Comma), | |
| // TokenTree::Token(sp(11, 14), token::Ident(str_to_ident("baz")))]; | |
| // | |
| // let test4_expected = TokenStream::from_tts(vec![TokenTree::Delimited(sp(2, 15), | |
| // Rc::new(Delimited { | |
| // delim: token::DelimToken::Paren, | |
| // open_span: sp(2, 3), | |
| // tts: test4_tts, | |
| // close_span: sp(14, 15), | |
| // }))]); | |
| // assert_eq!(test4, Some(&test4_expected[..])); | |
| // | |
| // assert_eq!(test5, None); | |
| // | |
| // } | |
| // | |
| // #[test] | |
| // fn test_maybe_delimited() { | |
| // let test0_input = TokenStream::from_tts(string_to_tts("foo(bar::baz)".to_string())); | |
| // let test1_input = TokenStream::from_tts(string_to_tts("(bar::baz)".to_string())); | |
| // let test2_input = TokenStream::from_tts(string_to_tts("(foo,bar,baz)".to_string())); | |
| // let test3_input = TokenStream::from_tts(string_to_tts("(foo,bar,baz)(zab,rab)" | |
| // .to_string())); | |
| // let test4_input = TokenStream::from_tts(string_to_tts("(foo,bar,baz)foo".to_string())); | |
| // let test5_input = TokenStream::from_tts(string_to_tts("".to_string())); | |
| // | |
| // let test0 = test0_input.maybe_delimited(); | |
| // let test1 = test1_input.maybe_delimited(); | |
| // let test2 = test2_input.maybe_delimited(); | |
| // let test3 = test3_input.maybe_delimited(); | |
| // let test4 = test4_input.maybe_delimited(); | |
| // let test5 = test5_input.maybe_delimited(); | |
| // | |
| // assert_eq!(test0, None); | |
| // | |
| // let test1_expected = TokenStream::from_tts(vec![TokenTree::Token(sp(1, 4), | |
| // token::Ident(str_to_ident("bar"))), | |
| // TokenTree::Token(sp(4, 6), token::ModSep), | |
| // TokenTree::Token(sp(6, 9), | |
| // token::Ident(str_to_ident("baz")))]); | |
| // assert_eq!(test1, Some(&test1_expected[..])); | |
| // | |
| // let test2_expected = TokenStream::from_tts(vec![TokenTree::Token(sp(1, 4), | |
| // token::Ident(str_to_ident("foo"))), | |
| // TokenTree::Token(sp(4, 5), token::Comma), | |
| // TokenTree::Token(sp(5, 8), | |
| // token::Ident(str_to_ident("bar"))), | |
| // TokenTree::Token(sp(8, 9), token::Comma), | |
| // TokenTree::Token(sp(9, 12), | |
| // token::Ident(str_to_ident("baz")))]); | |
| // assert_eq!(test2, Some(&test2_expected[..])); | |
| // | |
| // assert_eq!(test3, None); | |
| // | |
| // assert_eq!(test4, None); | |
| // | |
| // assert_eq!(test5, None); | |
| // } | |
| // | |
| // #[test] | |
| // fn test_maybe_comma_list() { | |
| // let test0_input = TokenStream::from_tts(string_to_tts("foo(bar::baz)".to_string())); | |
| // let test1_input = TokenStream::from_tts(string_to_tts("(bar::baz)".to_string())); | |
| // let test2_input = TokenStream::from_tts(string_to_tts("(foo,bar,baz)".to_string())); | |
| // let test3_input = TokenStream::from_tts(string_to_tts("(foo::bar,bar,baz)".to_string())); | |
| // let test4_input = TokenStream::from_tts(string_to_tts("(foo,bar,baz)(zab,rab)" | |
| // .to_string())); | |
| // let test5_input = TokenStream::from_tts(string_to_tts("(foo,bar,baz)foo".to_string())); | |
| // let test6_input = TokenStream::from_tts(string_to_tts("".to_string())); | |
| // // The following is supported behavior! | |
| // let test7_input = TokenStream::from_tts(string_to_tts("(foo,bar,)".to_string())); | |
| // | |
| // let test0 = test0_input.maybe_comma_list(); | |
| // let test1 = test1_input.maybe_comma_list(); | |
| // let test2 = test2_input.maybe_comma_list(); | |
| // let test3 = test3_input.maybe_comma_list(); | |
| // let test4 = test4_input.maybe_comma_list(); | |
| // let test5 = test5_input.maybe_comma_list(); | |
| // let test6 = test6_input.maybe_comma_list(); | |
| // let test7 = test7_input.maybe_comma_list(); | |
| // | |
| // assert_eq!(test0, None); | |
| // | |
| // let test1_stream = TokenStream::from_tts(vec![TokenTree::Token(sp(1, 4), | |
| // token::Ident(str_to_ident("bar"))), | |
| // TokenTree::Token(sp(4, 6), token::ModSep), | |
| // TokenTree::Token(sp(6, 9), | |
| // token::Ident(str_to_ident("baz")))]); | |
| // | |
| // let test1_expected: Vec<&TokenSlice> = vec![&test1_stream[..]]; | |
| // assert_eq!(test1, Some(test1_expected)); | |
| // | |
| // let test2_foo = TokenStream::from_tts(vec![TokenTree::Token(sp(1, 4), | |
| // token::Ident(str_to_ident("foo")))]); | |
| // let test2_bar = TokenStream::from_tts(vec![TokenTree::Token(sp(5, 8), | |
| // token::Ident(str_to_ident("bar")))]); | |
| // let test2_baz = TokenStream::from_tts(vec![TokenTree::Token(sp(9, 12), | |
| // token::Ident(str_to_ident("baz")))]); | |
| // let test2_expected: Vec<&TokenSlice> = vec![&test2_foo[..], &test2_bar[..], &test2_baz[..]]; | |
| // assert_eq!(test2, Some(test2_expected)); | |
| // | |
| // let test3_path = TokenStream::from_tts(vec![TokenTree::Token(sp(1, 4), | |
| // token::Ident(str_to_ident("foo"))), | |
| // TokenTree::Token(sp(4, 6), token::ModSep), | |
| // TokenTree::Token(sp(6, 9), | |
| // token::Ident(str_to_ident("bar")))]); | |
| // let test3_bar = TokenStream::from_tts(vec![TokenTree::Token(sp(10, 13), | |
| // token::Ident(str_to_ident("bar")))]); | |
| // let test3_baz = TokenStream::from_tts(vec![TokenTree::Token(sp(14, 17), | |
| // token::Ident(str_to_ident("baz")))]); | |
| // let test3_expected: Vec<&TokenSlice> = | |
| // vec![&test3_path[..], &test3_bar[..], &test3_baz[..]]; | |
| // assert_eq!(test3, Some(test3_expected)); | |
| // | |
| // assert_eq!(test4, None); | |
| // | |
| // assert_eq!(test5, None); | |
| // | |
| // assert_eq!(test6, None); | |
| // | |
| // | |
| // let test7_expected: Vec<&TokenSlice> = vec![&test2_foo[..], &test2_bar[..]]; | |
| // assert_eq!(test7, Some(test7_expected)); | |
| // } | |
| // | |
| // // pub fn maybe_ident(&self) -> Option<ast::Ident> | |
| // #[test] | |
| // fn test_maybe_ident() { | |
| // let test0 = TokenStream::from_tts(string_to_tts("\"foo\"".to_string())).maybe_ident(); | |
| // let test1 = TokenStream::from_tts(string_to_tts("5".to_string())).maybe_ident(); | |
| // let test2 = TokenStream::from_tts(string_to_tts("foo".to_string())).maybe_ident(); | |
| // let test3 = TokenStream::from_tts(string_to_tts("foo::bar".to_string())).maybe_ident(); | |
| // let test4 = TokenStream::from_tts(string_to_tts("foo(bar)".to_string())).maybe_ident(); | |
| // | |
| // assert_eq!(test0, None); | |
| // assert_eq!(test1, None); | |
| // assert_eq!(test2, Some(str_to_ident("foo"))); | |
| // assert_eq!(test3, None); | |
| // assert_eq!(test4, None); | |
| // } | |
| // | |
| // // pub fn maybe_lit(&self) -> Option<token::Lit> | |
| // #[test] | |
| // fn test_maybe_lit() { | |
| // let test0 = TokenStream::from_tts(string_to_tts("\"foo\"".to_string())).maybe_lit(); | |
| // let test1 = TokenStream::from_tts(string_to_tts("5".to_string())).maybe_lit(); | |
| // let test2 = TokenStream::from_tts(string_to_tts("foo".to_string())).maybe_lit(); | |
| // let test3 = TokenStream::from_tts(string_to_tts("foo::bar".to_string())).maybe_lit(); | |
| // let test4 = TokenStream::from_tts(string_to_tts("foo(bar)".to_string())).maybe_lit(); | |
| // | |
| // assert_eq!(test0, Some(Lit::Str_(token::intern("foo")))); | |
| // assert_eq!(test1, Some(Lit::Integer(token::intern(&(5.to_string()))))); | |
| // assert_eq!(test2, None); | |
| // assert_eq!(test3, None); | |
| // assert_eq!(test4, None); | |
| // } | |
| // | |
| // #[test] | |
| // fn test_maybe_path_prefix() { | |
| // let test0_input = TokenStream::from_tts(string_to_tts("foo(bar::baz)".to_string())); | |
| // let test1_input = TokenStream::from_tts(string_to_tts("(bar::baz)".to_string())); | |
| // let test2_input = TokenStream::from_tts(string_to_tts("(foo,bar,baz)".to_string())); | |
| // let test3_input = TokenStream::from_tts(string_to_tts("foo::bar(bar,baz)".to_string())); | |
| // | |
| // let test0 = test0_input.maybe_path_prefix(); | |
| // let test1 = test1_input.maybe_path_prefix(); | |
| // let test2 = test2_input.maybe_path_prefix(); | |
| // let test3 = test3_input.maybe_path_prefix(); | |
| // | |
| // let test0_tts = vec![TokenTree::Token(sp(4, 7), token::Ident(str_to_ident("bar"))), | |
| // TokenTree::Token(sp(7, 9), token::ModSep), | |
| // TokenTree::Token(sp(9, 12), token::Ident(str_to_ident("baz")))]; | |
| // | |
| // let test0_stream = TokenStream::from_tts(vec![TokenTree::Delimited(sp(3, 13), | |
| // Rc::new(Delimited { | |
| // delim: token::DelimToken::Paren, | |
| // open_span: sp(3, 4), | |
| // tts: test0_tts, | |
| // close_span: sp(12, 13), | |
| // }))]); | |
| // | |
| // let test0_expected = Some((ast::Path::from_ident(sp(0, 3), str_to_ident("foo")), | |
| // &test0_stream[..])); | |
| // assert_eq!(test0, test0_expected); | |
| // | |
| // assert_eq!(test1, None); | |
| // assert_eq!(test2, None); | |
| // | |
| // let test3_path = ast::Path { | |
| // span: sp(0, 8), | |
| // global: false, | |
| // segments: vec![ast::PathSegment { | |
| // identifier: str_to_ident("foo"), | |
| // parameters: ast::PathParameters::none(), | |
| // }, | |
| // ast::PathSegment { | |
| // identifier: str_to_ident("bar"), | |
| // parameters: ast::PathParameters::none(), | |
| // }], | |
| // }; | |
| // | |
| // let test3_tts = vec![TokenTree::Token(sp(9, 12), token::Ident(str_to_ident("bar"))), | |
| // TokenTree::Token(sp(12, 13), token::Comma), | |
| // TokenTree::Token(sp(13, 16), token::Ident(str_to_ident("baz")))]; | |
| // | |
| // let test3_stream = TokenStream::from_tts(vec![TokenTree::Delimited(sp(8, 17), | |
| // Rc::new(Delimited { | |
| // delim: token::DelimToken::Paren, | |
| // open_span: sp(8, 9), | |
| // tts: test3_tts, | |
| // close_span: sp(16, 17), | |
| // }))]); | |
| // let test3_expected = Some((test3_path, &test3_stream[..])); | |
| // assert_eq!(test3, test3_expected); | |
| // } | |
| // | |
| // #[test] | |
| // fn test_as_paren_delimited_stream() { | |
| // let test0 = TokenStream::as_paren_delimited_stream(string_to_tts("foo,bar,".to_string())); | |
| // let test1 = TokenStream::as_paren_delimited_stream(string_to_tts("baz(foo,bar)" | |
| // .to_string())); | |
| // | |
| // let test0_tts = vec![TokenTree::Token(sp(0, 3), token::Ident(str_to_ident("foo"))), | |
| // TokenTree::Token(sp(3, 4), token::Comma), | |
| // TokenTree::Token(sp(4, 7), token::Ident(str_to_ident("bar"))), | |
| // TokenTree::Token(sp(7, 8), token::Comma)]; | |
| // let test0_stream = TokenStream::from_tts(vec![TokenTree::Delimited(sp(0, 8), | |
| // Rc::new(Delimited { | |
| // delim: token::DelimToken::Paren, | |
| // open_span: DUMMY_SP, | |
| // tts: test0_tts, | |
| // close_span: DUMMY_SP, | |
| // }))]); | |
| // | |
| // assert_eq!(test0, test0_stream); | |
| // | |
| // | |
| // let test1_tts = vec![TokenTree::Token(sp(4, 7), token::Ident(str_to_ident("foo"))), | |
| // TokenTree::Token(sp(7, 8), token::Comma), | |
| // TokenTree::Token(sp(8, 11), token::Ident(str_to_ident("bar")))]; | |
| // | |
| // let test1_parse = vec![TokenTree::Token(sp(0, 3), token::Ident(str_to_ident("baz"))), | |
| // TokenTree::Delimited(sp(3, 12), | |
| // Rc::new(Delimited { | |
| // delim: token::DelimToken::Paren, | |
| // open_span: sp(3, 4), | |
| // tts: test1_tts, | |
| // close_span: sp(11, 12), | |
| // }))]; | |
| // | |
| // let test1_stream = TokenStream::from_tts(vec![TokenTree::Delimited(sp(0, 12), | |
| // Rc::new(Delimited { | |
| // delim: token::DelimToken::Paren, | |
| // open_span: DUMMY_SP, | |
| // tts: test1_parse, | |
| // close_span: DUMMY_SP, | |
| // }))]); | |
| // | |
| // assert_eq!(test1, test1_stream); | |
| // } | |
| // | |
| // } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment