feat: lexer tokens

This commit is contained in:
Artemy Egorov 2024-08-07 20:39:14 +03:00
parent a5443bf0fc
commit 655ac6ea09
5 changed files with 233 additions and 2 deletions

161
Cargo.lock generated
View file

@ -8,6 +8,33 @@ version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
[[package]]
name = "ahash"
version = "0.8.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011"
dependencies = [
"cfg-if",
"once_cell",
"version_check",
"zerocopy",
]
[[package]]
name = "aho-corasick"
version = "1.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916"
dependencies = [
"memchr",
]
[[package]]
name = "allocator-api2"
version = "0.2.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f"
[[package]] [[package]]
name = "anstream" name = "anstream"
version = "0.6.15" version = "0.6.15"
@ -57,6 +84,16 @@ dependencies = [
"windows-sys", "windows-sys",
] ]
[[package]]
name = "ariadne"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "44055e597c674aef7cb903b2b9f6e4cba1277ed0d2d61dae7cd52d7ffa81f8e2"
dependencies = [
"unicode-width",
"yansi",
]
[[package]] [[package]]
name = "autocfg" name = "autocfg"
version = "1.3.0" version = "1.3.0"
@ -94,6 +131,19 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "chumsky"
version = "1.0.0-alpha.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c7b80276986f86789dc56ca6542d53bba9cda3c66091ebbe7bd96fc1bdf20f1f"
dependencies = [
"hashbrown",
"regex-automata",
"serde",
"stacker",
"unicode-ident",
]
[[package]] [[package]]
name = "clap" name = "clap"
version = "4.5.13" version = "4.5.13"
@ -153,7 +203,9 @@ dependencies = [
name = "dalet" name = "dalet"
version = "1.0.0-pre.14" version = "1.0.0-pre.14"
dependencies = [ dependencies = [
"ariadne",
"bincode", "bincode",
"chumsky",
"clap", "clap",
"enum-procs", "enum-procs",
"flate2", "flate2",
@ -195,6 +247,10 @@ name = "hashbrown"
version = "0.14.5" version = "0.14.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
dependencies = [
"ahash",
"allocator-api2",
]
[[package]] [[package]]
name = "heck" name = "heck"
@ -278,6 +334,12 @@ dependencies = [
"syn", "syn",
] ]
[[package]]
name = "once_cell"
version = "1.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
[[package]] [[package]]
name = "paste" name = "paste"
version = "1.0.15" version = "1.0.15"
@ -308,6 +370,15 @@ dependencies = [
"unicode-ident", "unicode-ident",
] ]
[[package]]
name = "psm"
version = "0.1.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5787f7cda34e3033a72192c018bc5883100330f362ef279a8cbccfce8bb4e874"
dependencies = [
"cc",
]
[[package]] [[package]]
name = "quote" name = "quote"
version = "1.0.36" version = "1.0.36"
@ -317,6 +388,23 @@ dependencies = [
"proc-macro2", "proc-macro2",
] ]
[[package]]
name = "regex-automata"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59b23e92ee4318893fa3fe3e6fb365258efbfe6ac6ab30f090cdcbb7aa37efa9"
dependencies = [
"aho-corasick",
"memchr",
"regex-syntax",
]
[[package]]
name = "regex-syntax"
version = "0.7.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da"
[[package]] [[package]]
name = "rmp" name = "rmp"
version = "0.8.14" version = "0.8.14"
@ -370,6 +458,19 @@ dependencies = [
"syn", "syn",
] ]
[[package]]
name = "stacker"
version = "0.1.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c886bd4480155fd3ef527d45e9ac8dd7118a898a46530b7b94c3e21866259fce"
dependencies = [
"cc",
"cfg-if",
"libc",
"psm",
"winapi",
]
[[package]] [[package]]
name = "strsim" name = "strsim"
version = "0.11.1" version = "0.11.1"
@ -410,12 +511,46 @@ version = "1.0.12"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
[[package]]
name = "unicode-width"
version = "0.1.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d"
[[package]] [[package]]
name = "utf8parse" name = "utf8parse"
version = "0.2.2" version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
[[package]]
name = "version_check"
version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
[[package]]
name = "winapi"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
dependencies = [
"winapi-i686-pc-windows-gnu",
"winapi-x86_64-pc-windows-gnu",
]
[[package]]
name = "winapi-i686-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
[[package]]
name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]] [[package]]
name = "windows-sys" name = "windows-sys"
version = "0.52.0" version = "0.52.0"
@ -498,6 +633,32 @@ dependencies = [
"memchr", "memchr",
] ]
[[package]]
name = "yansi"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049"
[[package]]
name = "zerocopy"
version = "0.7.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0"
dependencies = [
"zerocopy-derive",
]
[[package]]
name = "zerocopy-derive"
version = "0.7.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]] [[package]]
name = "zstd" name = "zstd"
version = "0.13.2" version = "0.13.2"

View file

@ -17,9 +17,11 @@ num_enum = "0.7.3"
serde = { version = "1.0", features = ["derive"] } serde = { version = "1.0", features = ["derive"] }
serde_repr = "0.1" serde_repr = "0.1"
zstd = "0.13.2" zstd = "0.13.2"
# chumsky = { version = "=1.0.0-alpha.7", features = ["label"], optional = true }
enum-procs = "0.3.0" enum-procs = "0.3.0"
chumsky = { version = "=1.0.0-alpha.7", features = ["label"], optional = true }
ariadne = { version = "0.4.1", optional = true }
[dev-dependencies] [dev-dependencies]
rmp-serde = { version = "1.3.0" } rmp-serde = { version = "1.3.0" }
bincode = { version = "1.3.3" } bincode = { version = "1.3.3" }
@ -27,7 +29,8 @@ flate2 = "1.0"
[features] [features]
default = ["types", "daletpack", "parsers"] default = ["types", "daletpack", "parsers", "daleth"]
parsers = ["types"] parsers = ["types"]
daleth = ["types", "daletpack", "dep:chumsky", "dep:ariadne"]
types = [] types = []
daletpack = ["types"] daletpack = ["types"]

63
src/daleth/lexer.rs Normal file
View file

@ -0,0 +1,63 @@
use chumsky::prelude::*;
pub type Span = SimpleSpan<usize>;
#[derive(Clone, Debug, PartialEq)]
pub enum Token<'src> {
// Brackets
/// (
LParen,
/// )
RParen,
/// {
LAngle,
/// }
RAngle,
/// [
LSquare,
/// ]
RSquare,
// Values
Number(u8),
Text(&'src str),
/// Multi Line text
MLText(&'src str),
/// Multi Line with min spaces text
MLMSText(usize, &'src str),
/// Raw Multi line text
RMLText(&'src str),
// Tags
El,
H,
P,
Br,
Ul,
Ol,
Row,
Link,
Navlink,
Btn,
Navbtn,
Img,
Table,
Tcol,
Tpcol,
Hr,
B,
I,
Bq,
Footlnk,
Footn,
A,
S,
Sup,
Sub,
Disc,
Block,
Carousel,
Code,
Pre,
Meta,
}

1
src/daleth/mod.rs Normal file
View file

@ -0,0 +1 @@
pub mod lexer;

View file

@ -12,3 +12,6 @@ pub mod daletpack;
#[cfg(feature = "parsers")] #[cfg(feature = "parsers")]
pub mod parsers; pub mod parsers;
#[cfg(feature = "daleth")]
pub mod daleth;