Support callbacks returning Result<Skip>

[romamik]

This makes it possible to return Result<Skip> from callbacks, which seems logical.

Here is an example usage:

use logos::{Lexer, Logos, Skip};

#[derive(Logos, Debug, PartialEq)]
#[logos(error = String)]
#[logos(skip r"\s*")] // Ignore this regex pattern between tokens
enum Token<'src> {
    #[token("<!--", |lex| skip_comment(lex))]
    XmlComment,

    #[regex("<[a-zA-Z0-9-]+", |lex| &lex.slice()[1..])]
    TagOpen(&'src str),

    #[token("/>")]
    TagClose,
}

fn skip_comment<'src>(lex: &mut Lexer<'src, Token<'src>>) -> Result<Skip, String> {
    let mut open_count = 1;
    loop {
        let rem = lex.remainder();
        let close_pos = rem.find("-->").ok_or_else(|| {
            // this skips the rest of the input, as it all is part of the comment
            lex.bump(rem.len());

            "unterminated comment".to_string()
        })?;
        let open_pos = rem[..close_pos].find("<!--");
        if let Some(open_pos) = open_pos {
            open_count += 1;
            lex.bump(open_pos + 4);
            continue;
        }
        lex.bump(close_pos + 3);
        open_count -= 1;
        if open_count == 0 {
            break;
        }
    }
    Ok(Skip)
}

fn main() {
    let mut lex = Token::lexer(
        r#"   
			<!--- this is a comment <!--nested --> -->
        <tag-name
			<!--- this is a comment inside tag (that's weird, but why not?) -->
			<!--- unterminated 
        />
		"#,
    );

    assert_eq!(lex.next(), Some(Ok(Token::TagOpen("tag-name"))));
    assert_eq!(lex.next(), Some(Err("unterminated comment".to_string())));
    assert_eq!(lex.next(), None);
}

[jeertmans]

Hi @romamik, this looks very good! I will run the checks to see if they pass all.

Could you add an entry about this in the book?

[romamik]

Done. If I understood you correctly.

[jeertmans]

Yes you did nice! It would probably be good to add a test or two in the test suite that cover this new feature. Could you do it?

[romamik]

I've added a test.

[jeertmans]

Great, many thanks for your contribution @romamik!