mirror of
https://github.com/TxtDot/dalet-rs.git
synced 2024-11-05 17:33:58 +03:00
feat: symbols, text, number lexer
This commit is contained in:
parent
655ac6ea09
commit
edf799904e
2 changed files with 75 additions and 1 deletions
68
src/daleth/lexer/mod.rs
Normal file
68
src/daleth/lexer/mod.rs
Normal file
|
@ -0,0 +1,68 @@
|
|||
use chumsky::prelude::*;
|
||||
use types::{Span, Spanned, Token};
|
||||
pub mod types;
|
||||
|
||||
pub fn lexer<'src>(
|
||||
) -> impl Parser<'src, &'src str, Vec<Spanned<Token<'src>>>, extra::Err<Rich<'src, char, Span>>> {
|
||||
let symbol = choice((
|
||||
just("(").to(Token::LParen).labelled("LParen"),
|
||||
just(")").to(Token::RParen).labelled("RParen"),
|
||||
just("{").to(Token::LAngle).labelled("LAngle"),
|
||||
just("}").to(Token::RAngle).labelled("RAngle"),
|
||||
just("[").to(Token::LSquare).labelled("LSquare"),
|
||||
just("]").to(Token::RSquare).labelled("RSquare"),
|
||||
just(":").to(Token::Colon).labelled("Colon"),
|
||||
))
|
||||
.labelled("symbol");
|
||||
|
||||
let number = text::int(10)
|
||||
.from_str()
|
||||
.unwrapped()
|
||||
.map(Token::Number)
|
||||
.labelled("number");
|
||||
|
||||
let textual = {
|
||||
let escape = just('\\')
|
||||
.ignore_then(choice((
|
||||
just("\\`").to('`'.to_owned()),
|
||||
just("\\]").to(']'.to_owned()),
|
||||
)))
|
||||
.labelled("escape sequence");
|
||||
|
||||
let text = none_of("]\n")
|
||||
.or(escape.clone())
|
||||
.repeated()
|
||||
.to_slice()
|
||||
.map(Token::Text);
|
||||
|
||||
let multiline_text = none_of("`").or(escape).repeated();
|
||||
|
||||
let mltext = multiline_text
|
||||
.clone()
|
||||
.delimited_by(just('`'), just('`'))
|
||||
.to_slice()
|
||||
.map(Token::MLText)
|
||||
.labelled("multiline text");
|
||||
|
||||
let mlmstext = multiline_text
|
||||
.delimited_by(just("`#"), just('`'))
|
||||
.to_slice()
|
||||
.map(Token::RMLText)
|
||||
.labelled("raw multiline text");
|
||||
|
||||
choice((mltext, mlmstext, text))
|
||||
};
|
||||
|
||||
let comment = just("#")
|
||||
.then(none_of("\n").repeated())
|
||||
.to_slice()
|
||||
.map(Token::Comment);
|
||||
|
||||
let token = choice((symbol, number, textual, comment));
|
||||
|
||||
token
|
||||
.map_with(|t, e| (t, e.span()))
|
||||
.padded()
|
||||
.repeated()
|
||||
.collect()
|
||||
}
|
|
@ -1,10 +1,11 @@
|
|||
use chumsky::prelude::*;
|
||||
|
||||
pub type Span = SimpleSpan<usize>;
|
||||
pub type Spanned<T> = (T, Span);
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub enum Token<'src> {
|
||||
// Brackets
|
||||
// Symbols
|
||||
/// (
|
||||
LParen,
|
||||
/// )
|
||||
|
@ -17,6 +18,8 @@ pub enum Token<'src> {
|
|||
LSquare,
|
||||
/// ]
|
||||
RSquare,
|
||||
/// :
|
||||
Colon,
|
||||
|
||||
// Values
|
||||
Number(u8),
|
||||
|
@ -28,6 +31,9 @@ pub enum Token<'src> {
|
|||
/// Raw Multi line text
|
||||
RMLText(&'src str),
|
||||
|
||||
/// Special
|
||||
Comment(&'src str),
|
||||
|
||||
// Tags
|
||||
El,
|
||||
H,
|
Loading…
Reference in a new issue