Repository navigation
Expand file tree
/
Copy pathLexer.h
More file actions
175 lines (174 loc) · 4.07 KB
/
Copy pathLexer.h
File metadata and controls
175 lines (174 loc) · 4.07 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
#pragma once
#include <list>
#include <string>
#include "ErrorParser.h"
#include <climits>
#include <cstring>
enum TokenKind {
TOKEN_KEYWORD,
TOKEN_IDENT,
TOKEN_NUMERIC,
TOKEN_EOF
};
struct Token {
TokenKind kind;
int val; // Èç¹ûÊÇ TK_NUM£¬Æä´ú±íµÄÖµ
std::string name;
int line;
int offset;
};
class Lexer
{
private:
int ls = 0;
int line = 1;
inline int char2hex(const char c) {
if (c >= '0' && c <= '9')
return c - '0';
if (c >= 'A' && c <= 'F')
return c - 'A' + 10;
if (c >= 'a' && c <= 'f')
return c - 'a' + 10;
return -1;
}
inline int char2dec(const char c) {
if (c >= '0' && c <= '9')
return c - '0';
return -1;
}
const char* keywords[16] = { "int","return","void","if","else","for","do","while","continue","break","==",">=","<=","!=","&&","||"};
const char* lb;
inline size_t read_space(const char* c) {
size_t ret = 0;
while (isspace(c[ret])) {
if (c[ret] == '\n') {
line++;
ls = 0;
ErrorParser::lines.push_back(std::string(lb, c + ret));
lb = c + ret + 1;
}
else {
ls++;
}
ret++;
}
return ret;
}
inline size_t read_keyword(const char* c, Token& token) {
for (size_t i = 0; i < 16; i++) {
if (strncmp(keywords[i], c, strlen(keywords[i])) == 0 ) {
if (i < 10 && ((isalnum(c[strlen(keywords[i])]) || c[strlen(keywords[i])] == '_')))return 0;
//if (i >= 10 && ((ispunct(c[strlen(keywords[i])]))))return 0;
token.kind = TOKEN_KEYWORD;
token.name = keywords[i];
return strlen(keywords[i]);
}
}
return 0;
}
inline size_t read_identify(const char* c,Token& token) {
size_t ret = 0;
std::string name="";
if (isalpha(c[ret]) || c[ret] == '_') {
while (isalnum(c[ret]) || c[ret] == '_') {
name += c[ret];
ret++;
}
token.kind = TOKEN_IDENT;
token.name = name;
return ret;
}
else return 0;
}
inline size_t read_integer(const char* c,Token& token) {
long long val=0;
size_t ret = 0;
bool hex = false;
if (c[ret] == '0' && (c[ret + 1] == 'x') || (c[ret + 1] == 'X')) {
ret += 2;
hex = true;
}
if (hex) {
int next_ch;
while ((next_ch = char2hex(c[ret])) >= 0) {
val = val * 16 + next_ch;
ret++;
if (val > INT_MAX)exit(1);
}
}
else {
int next_ch;
while ((next_ch = char2dec(c[ret])) >= 0) {
val = val * 10 + next_ch;
if (val > INT_MAX)exit(1);
ret++;
}
}
token.kind = TOKEN_NUMERIC;
token.val = val;
return ret;
}
public:
inline size_t parse_piece(const char* source, Token& token, size_t origin_offset) {
const char* current = source + origin_offset;
size_t offset = read_space(current);
token.line = line;
token.offset = ls;
current += offset;
if (*current == '\0') {
token.kind = TOKEN_EOF;
return offset;
}
size_t ret;
ret = read_keyword(current, token);
if (ret > 0) {
ls += ret;
return offset + ret;
}
if (ispunct(*current)) {
token.kind = TOKEN_KEYWORD;
token.name = std::string(current, 1);
ls += 1;
return offset + 1;
}
ret = read_identify(current, token);
if (ret > 0) {
ls += ret;
return offset + ret;
}
ret = read_integer(current, token);
if (ret > 0) {
ls += ret;
return offset + ret;
}
//ThrowError("Cannot parse!");
return 0;
}
inline std::list<Token> parse(const char* text) {
lb = text;
size_t offset = 0;
Token token;
std::list<Token> ret;
while (1) {
auto o = parse_piece(text, token, offset);;
offset += o;
ret.push_back(token);
if (token.kind == TOKEN_EOF) {
if (lb != text+offset) {
ErrorParser::lines.push_back(std::string(lb, text + offset));
}
break;
}
if (o == 0) {
std::cerr << "Lexer error in line " << line << std::endl;
int o2 = offset;
while (text[o2] && text[o2] != '\n')o2++;
auto t = read_space(text + offset);
std::cerr << std::string(lb, text + o2) << std::endl;
std::cerr << std::string(text + offset + t - lb, ' ') << "^" << std::endl;
exit(1);
}
}
return ret;
}
};