You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
63 lines
1.2 KiB
C
63 lines
1.2 KiB
C
#ifndef TOKENIZER_H
|
|
#define TOKENIZER_H
|
|
|
|
enum token_type {
|
|
Token_Unknown,
|
|
|
|
Token_OpenParen, // (
|
|
Token_CloseParen, // )
|
|
Token_Colon, // :
|
|
Token_Semicolen, // ;
|
|
Token_Asterik, // *
|
|
Token_OpenBracket, // [
|
|
Token_CloseBracket, // ]
|
|
Token_OpenBrace, // {
|
|
Token_CloseBrace, // }
|
|
Token_Equals,
|
|
Token_Comma,
|
|
Token_Or,
|
|
Token_Pound,
|
|
|
|
Token_String,
|
|
Token_Identifier,
|
|
Token_Literal, // Number
|
|
|
|
Token_Space,
|
|
Token_EndOfLine,
|
|
Token_Comment,
|
|
|
|
Token_EndOfStream,
|
|
};
|
|
|
|
struct token {
|
|
char* Filename;
|
|
|
|
token_type Type;
|
|
size_t TextLength;
|
|
string Text;
|
|
union {
|
|
char* String;
|
|
f32 Float;
|
|
s32 Integer;
|
|
};
|
|
};
|
|
|
|
struct tokenizer {
|
|
char* Filename;
|
|
u32 LinesCount;
|
|
u32 TokensCount;
|
|
u32 TotalCount;
|
|
|
|
string Input;
|
|
char* At;
|
|
};
|
|
|
|
b32 IsTokenValid(token Token);
|
|
b32 TokenEquals(token Token, char* Match);
|
|
API_EXPORT token GetToken(tokenizer* Tokenizer);
|
|
API_EXPORT token PeekToken(tokenizer* Tokenizer, int Lookahead);
|
|
API_EXPORT tokenizer Tokenize(char* Data);
|
|
API_EXPORT tokenizer Tokenize(char* Filename);
|
|
API_EXPORT tokenizer Tokenize(string Data, char* Filename);
|
|
|
|
#endif |