vox.fe.ast.token source code

1 /// Copyright: Copyright (c) 2021 Andrey Penechko.
2 /// License: $(WEB boost.org/LICENSE_1_0.txt, Boost License 1.0).
3 /// Authors: Andrey Penechko.
4 module vox.fe.ast.token;
5 
6 import vox.utils : gatherEnumStrings;
7 
8 struct Token {
9 	TokenType type;
10 	TokenIndex index;
11 }
12 
13 struct TokenIndex
14 {
15 	uint index = uint.max;
16 	alias index this;
17 	bool isValid() { return index != uint.max; }
18 }
19 
20 immutable string[] tokStrings = gatherEnumStrings!TokenType();
21 
22 enum TokenType TYPE_TOKEN_FIRST = TokenType.TYPE_NORETURN;
23 enum TokenType TYPE_TOKEN_LAST = TokenType.TYPE_TYPE;
24 
25 alias TT = TokenType;
26 enum TokenType : ubyte {
27 	@("#soi") SOI,
28 	@("#eoi") EOI,
29 	@(null)   INVALID,
30 
31 	@("&")    AND,
32 	@("&&")   AND_AND,
33 	@("&=")   AND_EQUAL,
34 	@("@")    AT,
35 	@("\\")   BACKSLASH,
36 	@(":")    COLON,
37 	@(",")    COMMA,
38 	@(".")    DOT,
39 	@("..")   DOT_DOT,
40 	@("...")  DOT_DOT_DOT,
41 	@("=")    EQUAL,
42 	@("==")   EQUAL_EQUAL,
43 	@(">")    MORE,
44 	@(">=")   MORE_EQUAL,
45 	@(">>")   MORE_MORE,
46 	@(">>=")  MORE_MORE_EQUAL,
47 	@(">>>")  MORE_MORE_MORE,
48 	@(">>>=") MORE_MORE_MORE_EQUAL,
49 	@("<")    LESS,
50 	@("<=")   LESS_EQUAL,
51 	@("<<")   LESS_LESS,
52 	@("<<=")  LESS_LESS_EQUAL,
53 	@("-")    MINUS,
54 	@("-=")   MINUS_EQUAL,
55 	@("--")   MINUS_MINUS,
56 	@("!")    NOT,
57 	@("!=")   NOT_EQUAL,
58 	@("|")    OR,
59 	@("|=")   OR_EQUAL,
60 	@("||")   OR_OR,
61 	@("%")    PERCENT,
62 	@("%=")   PERCENT_EQUAL,
63 	@("+")    PLUS,
64 	@("+=")   PLUS_EQUAL,
65 	@("++")   PLUS_PLUS,
66 	@("?")    QUESTION,
67 	@(";")    SEMICOLON,
68 	@("/")    SLASH,
69 	@("/=")   SLASH_EQUAL,
70 	@("*")    STAR,
71 	@("*=")   STAR_EQUAL,
72 	@("~")    TILDE,
73 	@("~=")   TILDE_EQUAL,
74 	@("^")    XOR,
75 	@("^=")   XOR_EQUAL,
76 
77 	@("(")    LPAREN,
78 	@(")")    RPAREN,
79 	@("[")    LBRACKET,
80 	@("]")    RBRACKET,
81 	@("{")    LCURLY,
82 	@("}")    RCURLY,
83 
84 
85 	@("#if")      HASH_IF,              // #if
86 	@("#version") HASH_VERSION,         // #version
87 	@("#inline")  HASH_INLINE,          // #inline
88 	@("#assert")  HASH_ASSERT,          // #assert
89 	@("#foreach") HASH_FOREACH,         // #foreach
90 
91 	@("alias")    ALIAS_SYM,            // alias
92 	@("break")    BREAK_SYM,            // break
93 	@("continue") CONTINUE_SYM,         // continue
94 	@("do")       DO_SYM,               // do
95 	@("else")     ELSE_SYM,             // else
96 	@("function") FUNCTION_SYM,         // function
97 	@("if")       IF_SYM,               // if
98 	@("import")   IMPORT_SYM,           // import
99 	@("module")   MODULE_SYM,           // module
100 	@("return")   RETURN_SYM,           // return
101 	@("struct")   STRUCT_SYM,           // struct
102 	@("union")    UNION_SYM,            // union
103 	@("while")    WHILE_SYM,            // while
104 	@("for")      FOR_SYM,              // for
105 	@("switch")   SWITCH_SYM,           // switch
106 	@("cast")     CAST,                 // cast
107 	@("enum")     ENUM,                 // enum
108 
109 	@("#special_kw") SPECIAL_KW,        // __FILE__, __LINE__, __FUNCTION_NAME__, __MODULE_NAME__
110 
111 	@("#id")      IDENTIFIER,           // [a-zA-Z_] [a-zA-Z_0-9]*
112 	@("$id")      CASH_IDENTIFIER,      // $ [a-zA-Z_0-9]*
113 
114 	// ----------------------------------------
115 	// list of basic types. The order is the same as in `enum BasicType`
116 
117 	@("auto") TYPE_AUTO,                // auto
118 	@("noreturn") TYPE_NORETURN,        // noreturn
119 	@("void") TYPE_VOID,                // void
120 	@("bool") TYPE_BOOL,                // bool
121 	@("null") NULL,                     // null
122 	@("i8")   TYPE_I8,                  // i8
123 	@("i16")  TYPE_I16,                 // i16
124 	@("i32")  TYPE_I32,                 // i32
125 	@("i64")  TYPE_I64,                 // i64
126 
127 	@("u8")   TYPE_U8,                  // u8
128 	@("u16")  TYPE_U16,                 // u16
129 	@("u32")  TYPE_U32,                 // u32
130 	@("u64")  TYPE_U64,                 // u64
131 
132 	@("f32")  TYPE_F32,                 // f32
133 	@("f64")  TYPE_F64,                 // f64
134 
135 	@("$alias") TYPE_ALIAS,             // $alias
136 	@("$type")  TYPE_TYPE,              // $type
137 	// ----------------------------------------
138 
139 	@("isize") TYPE_ISIZE,              // isize
140 	@("usize") TYPE_USIZE,              // usize
141 
142 	@("true")  TRUE_LITERAL,            // true
143 	@("false") FALSE_LITERAL,           // false
144 	@("#int_dec_lit") INT_DEC_LITERAL,  // 0|([1-9][0-9_]*)
145 	@("#int_hex_lit") INT_HEX_LITERAL,  // ("0x"|"0X")[0-9A-Fa-f_]+
146 	@("#int_bin_lit") INT_BIN_LITERAL,  // ("0b"|"0B")[01_]+
147 	@("#float_dec_lit") FLOAT_DEC_LITERAL,
148 	@("#str_lit") STRING_LITERAL,
149 	@("#char_lit") CHAR_LITERAL,
150 
151 	@("#comm") COMMENT,                 // // /*
152 }