mirror of
https://github.com/SymboScript/Book.git
synced 2024-11-05 21:33:59 +03:00
doc: add token kinds and format
This commit is contained in:
parent
e06b2f27b5
commit
c48f669975
8 changed files with 131 additions and 5 deletions
|
@ -5,10 +5,11 @@
|
|||
- [Tokens](specification/tokens/readme.md)
|
||||
|
||||
- [Token format](specification/tokens/token-format.md)
|
||||
- [Operators](specification/tokens/operators.md)
|
||||
- [Keywords](specification/tokens/keywords.md)
|
||||
- [Identifiers](specification/tokens/identifiers.md)
|
||||
- [Literals](specification/tokens/literals.md)
|
||||
- [Brackets](specification/tokens/brackets.md)
|
||||
- [Token Kinds](specification/tokens/token-kinds.md)
|
||||
- [Operators](specification/tokens/kinds/operators.md)
|
||||
- [Keywords](specification/tokens/kinds/keywords.md)
|
||||
- [Identifiers](specification/tokens/kinds/identifiers.md)
|
||||
- [Literals](specification/tokens/kinds/literals.md)
|
||||
- [Brackets](specification/tokens/kinds/brackets.md)
|
||||
|
||||
- [Examples](examples/readme.md)
|
||||
|
|
|
@ -1 +1,28 @@
|
|||
# Token format
|
||||
|
||||
[Token](https://github.com/SymboScript/SymboScript/blob/main/types/src/lexer.rs#L4-L15)
|
||||
|
||||
```rust
|
||||
pub struct Token {
|
||||
/// Token Type
|
||||
pub kind: TokenKind,
|
||||
|
||||
/// Start offset in source
|
||||
pub start: usize,
|
||||
|
||||
/// End offset in source
|
||||
pub end: usize,
|
||||
|
||||
pub value: TokenValue,
|
||||
}
|
||||
```
|
||||
|
||||
[TokenValue](https://github.com/SymboScript/SymboScript/blob/main/types/src/lexer.rs#L110-L114)
|
||||
|
||||
```rust
|
||||
pub enum TokenValue {
|
||||
None,
|
||||
Number(f64),
|
||||
String(String),
|
||||
}
|
||||
```
|
||||
|
|
98
src/specification/tokens/token-kinds.md
Normal file
98
src/specification/tokens/token-kinds.md
Normal file
|
@ -0,0 +1,98 @@
|
|||
# Token Kinds
|
||||
|
||||
This chapter describes the token kinds used in the spec.
|
||||
|
||||
[Kinds](https://github.com/SymboScript/SymboScript/blob/main/types/src/lexer.rs#L18-L107)
|
||||
|
||||
```rust
|
||||
pub enum TokenKind {
|
||||
Eof, // end of file
|
||||
Comment,
|
||||
Unexpected,
|
||||
|
||||
Semicolon,
|
||||
Comma,
|
||||
Colon,
|
||||
Dot,
|
||||
|
||||
// Operators
|
||||
Plus,
|
||||
Minus,
|
||||
Star,
|
||||
Slash,
|
||||
Power,
|
||||
Range,
|
||||
Modulo,
|
||||
|
||||
// Bitwise operators (Keyword2Operator)
|
||||
BitAnd,
|
||||
BitOr,
|
||||
BitNot,
|
||||
BitXor,
|
||||
BitLeftShift,
|
||||
BitRightShift,
|
||||
|
||||
// Unary operators
|
||||
PlusPlus,
|
||||
MinusMinus,
|
||||
Question,
|
||||
|
||||
// Logic operators (Keyword2Operator)
|
||||
And,
|
||||
Or,
|
||||
Xor,
|
||||
Not,
|
||||
|
||||
/// Assignments operators (+=, -=, *=, /=...)
|
||||
Assign,
|
||||
FormulaAssign,
|
||||
PlusAssign,
|
||||
MinusAssign,
|
||||
MultiplyAssign,
|
||||
DivideAssign,
|
||||
PowerAssign,
|
||||
ModuloAssign,
|
||||
|
||||
// Comparison operators
|
||||
Equal,
|
||||
NotEqual,
|
||||
Less,
|
||||
LessEqual,
|
||||
Greater,
|
||||
GreaterEqual,
|
||||
|
||||
// Brackets
|
||||
LParen,
|
||||
RParen,
|
||||
LBrace,
|
||||
RBrace,
|
||||
LBracket,
|
||||
RBracket,
|
||||
|
||||
// Identifiers
|
||||
Identifier,
|
||||
|
||||
// Literals
|
||||
Number,
|
||||
Str,
|
||||
|
||||
// --- Keywords ---
|
||||
|
||||
// Keyword literals
|
||||
True,
|
||||
False,
|
||||
|
||||
// Keywords
|
||||
If,
|
||||
Else,
|
||||
While,
|
||||
For,
|
||||
Loop,
|
||||
Let,
|
||||
Return,
|
||||
Break,
|
||||
Continue,
|
||||
Function,
|
||||
In,
|
||||
}
|
||||
```
|
Loading…
Reference in a new issue