diff --git a/src/SUMMARY.md b/src/SUMMARY.md index 7916648..c0fcb11 100644 --- a/src/SUMMARY.md +++ b/src/SUMMARY.md @@ -5,10 +5,11 @@ - [Tokens](specification/tokens/readme.md) - [Token format](specification/tokens/token-format.md) - - [Operators](specification/tokens/operators.md) - - [Keywords](specification/tokens/keywords.md) - - [Identifiers](specification/tokens/identifiers.md) - - [Literals](specification/tokens/literals.md) - - [Brackets](specification/tokens/brackets.md) + - [Token Kinds](specification/tokens/token-kinds.md) + - [Operators](specification/tokens/kinds/operators.md) + - [Keywords](specification/tokens/kinds/keywords.md) + - [Identifiers](specification/tokens/kinds/identifiers.md) + - [Literals](specification/tokens/kinds/literals.md) + - [Brackets](specification/tokens/kinds/brackets.md) - [Examples](examples/readme.md) diff --git a/src/specification/tokens/brackets.md b/src/specification/tokens/kinds/brackets.md similarity index 100% rename from src/specification/tokens/brackets.md rename to src/specification/tokens/kinds/brackets.md diff --git a/src/specification/tokens/identifiers.md b/src/specification/tokens/kinds/identifiers.md similarity index 100% rename from src/specification/tokens/identifiers.md rename to src/specification/tokens/kinds/identifiers.md diff --git a/src/specification/tokens/keywords.md b/src/specification/tokens/kinds/keywords.md similarity index 100% rename from src/specification/tokens/keywords.md rename to src/specification/tokens/kinds/keywords.md diff --git a/src/specification/tokens/literals.md b/src/specification/tokens/kinds/literals.md similarity index 100% rename from src/specification/tokens/literals.md rename to src/specification/tokens/kinds/literals.md diff --git a/src/specification/tokens/operators.md b/src/specification/tokens/kinds/operators.md similarity index 100% rename from src/specification/tokens/operators.md rename to src/specification/tokens/kinds/operators.md diff --git a/src/specification/tokens/token-format.md b/src/specification/tokens/token-format.md index 01fe992..6997ac5 100644 --- a/src/specification/tokens/token-format.md +++ b/src/specification/tokens/token-format.md @@ -1 +1,28 @@ # Token format + +[Token](https://github.com/SymboScript/SymboScript/blob/main/types/src/lexer.rs#L4-L15) + +```rust +pub struct Token { + /// Token Type + pub kind: TokenKind, + + /// Start offset in source + pub start: usize, + + /// End offset in source + pub end: usize, + + pub value: TokenValue, +} +``` + +[TokenValue](https://github.com/SymboScript/SymboScript/blob/main/types/src/lexer.rs#L110-L114) + +```rust +pub enum TokenValue { + None, + Number(f64), + String(String), +} +``` diff --git a/src/specification/tokens/token-kinds.md b/src/specification/tokens/token-kinds.md new file mode 100644 index 0000000..fb5c2d4 --- /dev/null +++ b/src/specification/tokens/token-kinds.md @@ -0,0 +1,98 @@ +# Token Kinds + +This chapter describes the token kinds used in the spec. + +[Kinds](https://github.com/SymboScript/SymboScript/blob/main/types/src/lexer.rs#L18-L107) + +```rust +pub enum TokenKind { + Eof, // end of file + Comment, + Unexpected, + + Semicolon, + Comma, + Colon, + Dot, + + // Operators + Plus, + Minus, + Star, + Slash, + Power, + Range, + Modulo, + + // Bitwise operators (Keyword2Operator) + BitAnd, + BitOr, + BitNot, + BitXor, + BitLeftShift, + BitRightShift, + + // Unary operators + PlusPlus, + MinusMinus, + Question, + + // Logic operators (Keyword2Operator) + And, + Or, + Xor, + Not, + + /// Assignments operators (+=, -=, *=, /=...) + Assign, + FormulaAssign, + PlusAssign, + MinusAssign, + MultiplyAssign, + DivideAssign, + PowerAssign, + ModuloAssign, + + // Comparison operators + Equal, + NotEqual, + Less, + LessEqual, + Greater, + GreaterEqual, + + // Brackets + LParen, + RParen, + LBrace, + RBrace, + LBracket, + RBracket, + + // Identifiers + Identifier, + + // Literals + Number, + Str, + + // --- Keywords --- + + // Keyword literals + True, + False, + + // Keywords + If, + Else, + While, + For, + Loop, + Let, + Return, + Break, + Continue, + Function, + In, +} +```