0

I'm implementing a basic code parser in Rust. I want to have an iterator over the tokens of the language. My custom iterator struct will have a borrow to the code String stored in another struct. Here is the implementation:

fn main() {
    let tokenizer = Tokenizer {
        code: String::from("foo + bar "),
    };
    let token_iter = tokenizer.tokens();
    let program = parse(Box::new(token_iter));
    println!("{:?}", program);
}

#[derive(Debug)]
enum Token {
    Symbol(&'static str),
    Identifier(String),
}

struct Tokenizer {
    code: String,
}

impl Tokenizer {
    fn tokens(&self) -> TokenIterator {
        TokenIterator::new(&self.code)
    }
}

struct TokenIterator<'a> {
    code: &'a str,
}

impl<'a> TokenIterator<'a> {
    fn new(code: &'a str) -> TokenIterator {
        TokenIterator { code: code.trim() }
    }
}

impl<'a> Iterator for TokenIterator<'a> {
    type Item = Token;

    fn next(&mut self) -> Option<Self::Item> {
        if self.code.len() == 0 {
            return None;
        }
        if self.code.starts_with("+") {
            self.code = &self.code[1..].trim_left();
            return Some(Token::Symbol("+"));
        }
        let first_char = self.code.chars().nth(0).unwrap();
        if first_char.is_alphabetic() || first_char == '_' {
            let mut ident = String::new();
            while self.code.chars().nth(0).unwrap().is_alphanumeric()
                || self.code.chars().nth(0).unwrap() == '_'
            {
                ident.push(self.code.chars().nth(0).unwrap());
                self.code = &self.code[1..];
            }
            self.code = self.code.trim_left();
            return Some(Token::Identifier(ident));
        }
        panic!("invalid syntax:\n{}", self.code)
    }
}

#[derive(Debug)]
struct Program {
    name: &'static str,
    tokens: Vec<Token>,
}

fn parse(tokens: Box<Iterator<Item = Token>>) -> Program {
    Program {
        tokens: tokens.collect(),
        name: "foo",
    }
}

The compiler does not let me create the iterator and bind it to a variable.

error[E0597]: `tokenizer` does not live long enough
 --> src/main.rs:5:26
  |
5 |         let token_iter = tokenizer.tokens();
  |                          ^^^^^^^^^ borrowed value does not live long enough
...
8 |     }
  |     - borrowed value only lives until here
  |
  = note: borrowed value must be valid for the static lifetime...

Why does the tokenizer need to live for the 'static lifetime? And how come it compiles when I change the main function to consume the iterator in a loop without binding it to a local variable? This code works for some reason...

fn main() {
    let tokenizer = Tokenizer {
        code: String::from("foo + bar "),
    };
    for token in tokenizer.tokens() {
        println!("{:?}", token);
    }
}
Shepmaster
  • 388,571
  • 95
  • 1,107
  • 1,366
Andy Carlson
  • 3,633
  • 24
  • 43
  • I believe your question is answered by the answers of [Why is adding a lifetime to a trait with the plus operator (Iterator + 'a) needed?](https://stackoverflow.com/q/42028470/155423). If you disagree, please [edit] your question to explain the differences. Otherwise, we can mark this question as already answered. – Shepmaster Oct 04 '18 at 16:14
  • 2
    The duplicate applied to this question: the prototype for function `parse` can be expanded to `fn parse(tokens: Box + 'static>) -> Program`, so it's expecting a `Box` containing a value with the static lifetime. Add a generic lifetime parameter to this function to make `parse` generic over any possible lifetime of the iterator. – E_net4 Oct 04 '18 at 16:22
  • 1
    It would be nice if the error message included some hints how it inferred the static lifetime. – Sven Marnach Oct 04 '18 at 16:27
  • 1
    @SvenMarnach Issue [#43531](https://github.com/rust-lang/rust/issues/43531) seems to focus on this concern. – E_net4 Oct 04 '18 at 16:33

0 Answers0