refactor(lexer): make functions for specific tokens

examples/daleth_lexer.rs

@@ -1,12 +1,12 @@
 use ariadne::{Color, Label, Report, ReportKind, Source};
 use chumsky::Parser;
-use dalet::daleth::{format::format, lexer::lexer};
+use dalet::daleth::{format::format, lexer::full_lexer};
 
 fn main() {
     let src_file = "daleth.dlth";
     let src = include_str!("./daleth.dlth");
 
-    let parsed = lexer().parse(src);
+    let parsed = full_lexer().parse(src);
 
     match parsed.into_result() {
         Ok(t) => {

src/daleth/lexer/mod.rs

@@ -4,7 +4,36 @@ pub mod types;
 
 pub fn lexer<'src>(
 ) -> impl Parser<'src, &'src str, Vec<Spanned<Token<'src>>>, extra::Err<Rich<'src, char, Span>>> {
-    let tag = choice((
+    let token = choice((symbol(), tag(), argument(), textual()));
+
+    token
+        .padded()
+        .padded_by(comment())
+        .map_with(|t, e| (t, e.span()))
+        .repeated()
+        .collect()
+}
+
+pub fn full_lexer<'src>(
+) -> impl Parser<'src, &'src str, Vec<Spanned<Token<'src>>>, extra::Err<Rich<'src, char, Span>>> {
+    let token = choice((
+        empty_line(),
+        comment(),
+        symbol(),
+        tag(),
+        argument(),
+        textual(),
+    ));
+
+    token
+        .padded_by(text::whitespace().and_is(empty_line().not()).or_not())
+        .map_with(|t, e| (t, e.span()))
+        .repeated()
+        .collect()
+}
+
+fn tag<'src>() -> impl Parser<'src, &'src str, Token<'src>, extra::Err<Rich<'src, char, Span>>> {
+    choice((
         just("el").to(Token::El),
         just("h").to(Token::H),
         just("p").to(Token::P),
@@ -39,16 +68,20 @@ pub fn lexer<'src>(
         just("pre").to(Token::Pre),
         just("meta").to(Token::Meta),
     )))
-    .labelled("Tag");
+    .labelled("Tag")
+}
 
-    let symbol = choice((
+fn symbol<'src>() -> impl Parser<'src, &'src str, Token<'src>, extra::Err<Rich<'src, char, Span>>> {
+    choice((
         just("[[").to(Token::ElOpen).labelled("[["),
         just("]]").to(Token::ElClose).labelled("]]"),
         just("[").to(Token::LSquare).labelled("["),
         just("]").to(Token::RSquare).labelled("]"),
-    ));
+    ))
+}
 
-    let argument = {
+fn argument<'src>() -> impl Parser<'src, &'src str, Token<'src>, extra::Err<Rich<'src, char, Span>>>
+{
     let arg_escape = just('\\')
         .ignore_then(just('"'))
         .labelled("Escape sequence for argument");
@@ -68,9 +101,10 @@ pub fn lexer<'src>(
         .labelled("Text argument");
 
     choice((number, text_argument))
-    };
+}
 
-    let textual = {
+fn textual<'src>() -> impl Parser<'src, &'src str, Token<'src>, extra::Err<Rich<'src, char, Span>>>
+{
     let escape = just('\\')
         .ignore_then(just('}'))
         .labelled("Multi-line escape sequence");
@@ -126,21 +160,17 @@ pub fn lexer<'src>(
         .labelled("Raw multiline text");
 
     choice((paragraph, mlmstext, rmltext, mltext, text_body, text_tag))
-    };
+}
-
+
-    let comment = just('#')
+fn comment<'src>() -> impl Parser<'src, &'src str, Token<'src>, extra::Err<Rich<'src, char, Span>>>
-        .ignore_then(none_of("\n").repeated().to_slice())
+{
-        .map(Token::Comment);
+    just('#')
-
+        .ignore_then(none_of("\n").repeated().to_slice())
-    let empty_line = text::inline_whitespace()
+        .map(Token::Comment)
-        .delimited_by(text::newline(), text::newline())
+}
-        .to(Token::EmptyLine);
+fn empty_line<'src>(
-
+) -> impl Parser<'src, &'src str, Token<'src>, extra::Err<Rich<'src, char, Span>>> {
-    let token = choice((empty_line.clone(), comment, symbol, tag, argument, textual));
+    text::inline_whitespace()
-
+        .delimited_by(text::newline(), text::newline())
-    token
+        .to(Token::EmptyLine)
-        .padded_by(text::whitespace().and_is(empty_line.not()).or_not())
-        .map_with(|t, e| (t, e.span()))
-        .repeated()
-        .collect()
 }

src/daleth/lexer/types.rs

@@ -28,11 +28,11 @@ pub enum Token<'src> {
     /// Raw Multi line text
     RMLText(&'src str),
 
-    /// Special
+    // Special
     TextTag(&'src str),
     Paragraph(&'src str),
 
-    /// Special removed before parse
+    // Special for formatting, ignored for parse
     Comment(&'src str),
     EmptyLine,
 

src/main.rs

@@ -4,7 +4,7 @@ use ariadne::{Color, Label, Report, ReportKind, Source};
 use chumsky::Parser;
 use clap::Parser as ClapParser;
 use commands::{Cli, Commands::*};
-use dalet::daleth::{format::format, lexer::lexer};
+use dalet::daleth::{format::format, lexer::full_lexer};
 use std::fs;
 
 fn main() {
@@ -16,7 +16,7 @@ fn main() {
             let src_file = &path.to_string_lossy().to_string();
             let src = fs::read_to_string(src_file).unwrap();
 
-            let parsed = lexer().parse(&src);
+            let parsed = full_lexer().parse(&src);
 
             match parsed.into_result() {
                 Ok(t) => {