feat: full lexer and example

examples/bench.md

@@ -0,0 +1,17 @@
+# Heading 1
+## Heading 2
+Some **bold** *italic* ~~strike~~
+
+`Hello world`
+
+- abc
+- def
+  - defabc
+  - defdef
+- xyz
+
+Lorem ipsum [![](https://my-picture)](https://some-link) dolor sit amet consequetur adipiscing elit
+
+| col1        | col2     | col3 |
+| ----------- | -------- | ---- |
+| Never gonna | give you | up   |

examples/bench.rs

@@ -0,0 +1,121 @@
+use dalet::{
+    daletpack::*,
+    typed::{Hl, TNArg, Tag::*},
+};
+use flate2::Compression;
+use std::io::Write;
+
+#[macro_export]
+macro_rules! iprint {
+    ($name:expr, $func:expr) => {{
+        let start = std::time::Instant::now();
+        let result = $func;
+        let elapsed = start.elapsed();
+        println!("{} ({:#?}): {}", $name, elapsed, result.len());
+
+        result
+    }};
+}
+
+pub fn compress_deflate(data: &[u8]) -> std::io::Result<Vec<u8>> {
+    let mut c = flate2::write::DeflateEncoder::new(Vec::new(), Compression::default());
+    c.write_all(data)?;
+    c.finish()
+}
+
+pub fn compress_zlib(data: &[u8]) -> std::io::Result<Vec<u8>> {
+    let mut c = flate2::write::ZlibEncoder::new(Vec::new(), Compression::default());
+    c.write_all(data)?;
+    c.finish()
+}
+
+fn main() {
+    let page = vec![
+        H("Heading 1".into(), Hl::One),
+        H("Heading 2".into(), Hl::Two),
+        P(vec![
+            El("Some ".into()),
+            B("bold".into()),
+            I("italic".into()),
+            S("strike".into()),
+        ]
+        .into()),
+        Br,
+        Code("Hello world".into(), TNArg::Null),
+        Br,
+        Ul(vec![
+            El("abc".into()),
+            El(vec![
+                El("def".into()),
+                Ul(vec![El("defabc".into()), El("defdef".into())]),
+            ]
+            .into()),
+            El("xyz".into()),
+        ]),
+        Br,
+        P(vec![
+            El("Lorem ipsum ".into()),
+            Link(
+                vec![Img("https://my-picture".into())].into(),
+                "https://some-link".into(),
+            ),
+            El(" dolor sit amet consequetur adipiscing elit".into()),
+        ]
+        .into()),
+        Table(vec![
+            Tpcol(vec![
+                El("Col 1".into()),
+                El("Col 2".into()),
+                El("Col 3".into()),
+            ]),
+            Tcol(vec![
+                El("Never gonna".into()),
+                El("give you".into()),
+                El("up".into()),
+            ]),
+        ]),
+    ];
+
+    let dalet_page = page.into();
+
+    let markdown = iprint!("Markdown", include_str!("./bench.md").as_bytes().to_vec());
+    let daletpack = iprint!("Daletpack", encode_no_compress(&dalet_page).unwrap());
+    let messagepack = iprint!("Messagepack", rmp_serde::to_vec(&dalet_page).unwrap());
+    let bincode = iprint!("Bincode", bincode::serialize(&dalet_page).unwrap());
+
+    println!();
+
+    iprint!("Markdown zstd", utils::compress_zstd(&markdown).unwrap());
+    let daletpack = iprint!("Daletpack zstd", utils::compress_zstd(&daletpack).unwrap());
+    iprint!(
+        "Messagepack zstd",
+        utils::compress_zstd(&messagepack).unwrap()
+    );
+    iprint!("Bincode zstd", utils::compress_zstd(&bincode).unwrap());
+
+    println!();
+
+    iprint!("Markdown Zlib", compress_zlib(&markdown).unwrap());
+    iprint!("Daletpack Zlib", compress_zlib(&daletpack).unwrap());
+    iprint!("Messagepack Zlib", compress_zlib(&messagepack).unwrap());
+    iprint!("Bincode Zlib", compress_zlib(&bincode).unwrap());
+
+    println!();
+
+    iprint!("Markdown deflate", compress_deflate(&markdown).unwrap());
+    iprint!("Daletpack deflate", compress_deflate(&daletpack).unwrap());
+    iprint!(
+        "Messagepack deflate",
+        compress_deflate(&messagepack).unwrap()
+    );
+    iprint!("Bincode deflate", compress_deflate(&bincode).unwrap());
+
+    println!();
+
+    let decoded = iprint!(
+        "Daletpack decode",
+        Decoder::new(&daletpack).unwrap().decode().unwrap().data
+    );
+
+    println!("{:#?}", decoded);
+}

examples/daleth.dlth

@@ -0,0 +1,129 @@
+# multilines
+#
+# {text} - input is trimmed with indent
+#
+# {~n text} - n is number of minimum spaces to add after trimming with indent
+# for each line
+#
+# {# text} - input not modified
+#
+# tag syntax
+#
+# tag: text body
+# tag { multiline text body }
+# body text always trimmed
+#
+# tag [ multiple tags body ]
+#
+# Arguments
+# tag argument
+#
+# Tags without body and argument also supported
+
+meta "title": Daleth syntax concept
+meta "description": This document describes Daleth syntax and some tags
+
+h1: TxtDot revolution
+p: TxtDot is a cool project
+
+# If no tag is specified, then the 'el' tag is placed
+This is element
+br
+
+# if no tag is specified but a '{}' is present, then the 'p' tag is placed
+# '\n' is deleted only in this format. If a break line is needed in a paragraph, use '  \n'.
+{
+  Check Dalet too
+  This is one paragraph
+}
+
+{ This is another paragraph }
+
+# ( ) for argument
+row "center" [
+  link "https://github.com/txtdot/txtdot": Homepage
+  btn "https://example.com/donate" [
+    # tag without body
+    img "https://example.com/donate.png"
+    Donate
+  ]
+]
+
+# [] for multiple tags
+row [
+  [
+    h2: Features
+
+    ul [
+      Server-side page simplification
+      Media proxy
+      Image compression with Sharp
+      Rendering client-side apps `Vanilla, React, Vue, etc` with webder
+      Search with SearXNG
+      Handy API endpoints
+      No client JavaScript
+      Some kind of Material Design 3
+      Customization with plugins, see @txtdot/sdk and @txtdot/plugins
+    ]
+
+  ]
+
+  [
+    h2: Running
+
+    [
+      h3: Dev
+
+      # {} for multiline strings, indent is automatically trimmed
+      code {
+        npm install
+        npm run dev
+      }
+
+      # {~n Text} n is number of minimum spaces
+      code "markdown" {~4
+        this is codeblock
+      }
+
+      # {# Text} Text after "`# " not modified
+      code "markdown" {#     this is codeblock}
+    ]
+
+    [
+      h3: Production
+
+      code {
+        npm install
+        npm run build
+        npm run start
+      }
+    ]
+
+    [
+      h3: Docker
+
+      code: docker compose up -d
+    ]
+
+  ]
+]
+
+# Table has custom format if text used
+# +| cells | - primary column
+#  | cells | - secondary column
+#  | Element | Description | - converts to
+#  tcol [
+#    Element
+#    Description
+#  ]
+table {
+  +| Tag      | Description     |
+   | h        | Heading         |
+   | p        | Paragraph       |
+   | img      | Image           |
+   | link     | Link            |
+   | btn      | Button          |
+   | ul       | Unordered list  |
+   | br       | Line break      |
+  +| quantity | 7               |
+}

examples/daleth_lexer.rs

@@ -0,0 +1,27 @@
+use ariadne::{Color, Label, Report, ReportKind, Source};
+use chumsky::Parser;
+use dalet::daleth::lexer::lexer;
+
+fn main() {
+    let src_file = "daleth.dlth";
+    let src = include_str!("./daleth.dlth");
+
+    let parsed = lexer().parse(src);
+
+    match parsed.into_result() {
+        Ok(t) => println!("{:#?}", t),
+        Err(e) => e.into_iter().for_each(|e| {
+            Report::build(ReportKind::Error, src_file, e.span().start)
+                .with_code("Compiler")
+                .with_message(e.to_string().clone())
+                .with_label(
+                    Label::new((src_file, e.span().into_range()))
+                        .with_message(e.to_string())
+                        .with_color(Color::Red),
+                )
+                .finish()
+                .print((src_file, Source::from(&src)))
+                .unwrap()
+        }),
+    }
+}

examples/gemtext.gmi

@@ -0,0 +1,32 @@
+=> https://example.com    A cool website
+=> gopher://example.com   An even cooler gopherhole
+=> gemini://example.com   A supremely cool Gemini capsule
+=> sftp://example.com
+
+This is paragraph
+
+=>https://example.com A cool website
+=>gopher://example.com      An even cooler gopherhole
+=> gemini://example.com A supremely cool Gemini capsule
+=>   sftp://example.com
+
+# Heading
+
+## Sub-heading
+
+### Sub-sub-heading
+
+* Mercury
+* Gemini
+* Apollo
+
+> Gemtext supports blockquotes.  The quoted content is written as a single long line, which begins with a single > character
+
+```
+preformatted
+=> ()
+# false heading
+text
+```
+
+This is paragraph

examples/gemtext.rs

@@ -0,0 +1,9 @@
+use dalet::parsers::gemtext::parse_gemtext;
+
+fn main() {
+    let text = include_str!("./gemtext.gmi");
+
+    let parsed = parse_gemtext(text).unwrap();
+
+    println!("{:#?}", parsed);
+}