tokenize.h

#pragma once
#include <queue>
#include <string>

enum token_type {
	LONG_KEYWORD, SHORT_KEYWORD, CHAR_KEYWORD, VOID_KEYWORD,
	INT_KEYWORD, SEMICOLON, OPEN_PARENTHESES, CLOSE_PARENTHESES, OPEN_BRACES, CLOSE_BRACES, RETURN_KEYWORD,
	INT_VALUE, NAME, PLUS, MINUS, ASTERISK, SLASH, MODULUS, EQUAL_SIGN, BITWISE_COMPLEMENT, EXCLAMATION,
	LOGICAL_AND, LOGICAL_OR, EQUAL_TO, NOT_EQUAL_TO, LESS_THAN, GREATER_THAN, GREATER_OR_EQUAL_TO, LESS_OR_EQUAL_TO,
	BITWISE_AND, BITWISE_OR, BITWISE_XOR,
	LEFT_SHIFT, RIGHT_SHIFT,
	ADD_ASSIGN, SUBTRACT_ASSIGN, MULTIPLY_ASSIGN, DIVIDE_ASSIGN, MOD_ASSIGN, AND_ASSIGN, OR_ASSIGN, XOR_ASSIGN,
	LEFT_SHIFT_ASSIGN, RIGHT_SHIFT_ASSIGN,
	COMMA, INCREMENT, DECREMENT,

	IF_KEYWORD, ELSE_KEYWORD, COLON, QUESTION_MARK,

	FOR_KEYWORD, WHILE_KEYWORD, DO_KEYWORD, BREAK_KEYWORD, CONTINUE_KEYWORD,

	CHAR_VALUE, SHORT_VALUE, LONG_VALUE, STRING_VALUE,
	OPEN_BRACKET, CLOSE_BRACKET,

	STRUCT_KEYWORD, DOT, ARROW
};

struct token {
	token_type type;
	std::string value;
};

void tokenize(std::string s, std::queue<token>& token_queue);