lexer.h

Go to the documentation of this file.

 
#pragma once
 
#include "../c_libs/memory/memory.h"
#include <stdint.h>
 
typedef enum {
  TOK_EOF,          
  TOK_ERROR,        
  TOK_IDENTIFIER,   
  TOK_KEYWORD,      
  TOK_NUMBER,       
  TOK_NUM_FLOAT,    
  TOK_STRING,       
  TOK_CHAR_LITERAL, 
  // Primitive Types
  TOK_INT,     
  TOK_DOUBLE,  
  TOK_UINT,    
  TOK_FLOAT,   
  TOK_BOOL,    
  TOK_STRINGT, 
  TOK_VOID,    
  TOK_CHAR,    
  // Keywords
  TOK_IF,       
  TOK_ELIF,     
  TOK_ELSE,     
  TOK_LOOP,     
  TOK_RETURN,   
  TOK_BREAK,    
  TOK_CONTINUE, 
  TOK_STRUCT,   
  TOK_ENUM,     
  TOK_MOD,      
  TOK_IMPORT,   
  TOK_TRUE,     
  TOK_FALSE,    
  TOK_PUBLIC,   
  TOK_PRIVATE,  
  TOK_VAR,      
  TOK_CONST,    
  TOK_FN,       
  TOK_PRINT,    
  TOK_PRINTLN,  
  TOK_INPUT,    
  TOK_ALLOC,    
  TOK_FREE,     
  TOK_CAST,     
  TOK_SIZE_OF,  
  TOK_AS,       
  TOK_DEFER,    
  TOK_IN,       
  TOK_SWITCH,   
  TOK_SYSTEM,   
  TOK_IMPL,     
  TOK_SYSCALL, 
  TOK_STATIC, 
 
  // prepocessor directives
  TOK_MODULE, 
  TOK_USE,    
  TOK_OS,     
  TOK_LINK,   
  // function attibutes
  TOK_RETURNES_OWNERSHIP, 
  TOK_TAKES_OWNERSHIP,    
  TOK_DLL_IMPORT,         
  TOK_LIB_IMPORT,         
  // Symbols
  TOK_SYMBOL,      
  TOK_LPAREN,      
  TOK_RPAREN,      
  TOK_LBRACE,      
  TOK_RBRACE,      
  TOK_LBRACKET,    
  TOK_RBRACKET,    
  TOK_SEMICOLON,   
  TOK_COMMA,       
  TOK_DOT,         
  TOK_AT,          
  TOK_EQUAL,       
  TOK_PLUS,        
  TOK_MINUS,       
  TOK_STAR,        
  TOK_SLASH,       
  TOK_LT,          
  TOK_GT,          
  TOK_LE,          
  TOK_GE,          
  TOK_EQEQ,        
  TOK_NEQ,         
  TOK_AMP,         
  TOK_PIPE,        
  TOK_CARET,       
  TOK_TILDE,       
  TOK_AND,         
  TOK_OR,          
  TOK_RESOLVE,     
  TOK_COLON,       
  TOK_BANG,        
  TOK_QUESTION,    
  TOK_PLUSPLUS,    
  TOK_MINUSMINUS,  
  TOK_SHIFT_LEFT,  
  TOK_SHIFT_RIGHT, 
  TOK_RANGE,       
  TOK_RIGHT_ARROW, 
  TOK_LEFT_ARROW,  
  TOK_ELLIPSIS,    
  TOK_MODL,        
  TOK_WHITESPACE,  
  TOK_COMMENT,     
  TOK_DOC_COMMENT, 
  TOK_MODULE_DOC,  
  TOK_DOCUMENT     // You already have this one
} LumaTokenType;
 
typedef struct {
  ArenaAllocator *arena; 
  const char *src;       
  const char *current;   
  int line;              
  int col;               
} Lexer;
 
typedef struct {
  LumaTokenType type_; 
  const char *value;   
  int line;            
  int col;             
  int length;          
  int whitespace_len;  
} Token;
 
typedef struct {
  const char *text;   
  LumaTokenType type; 
} SymbolEntry;
 
typedef struct {
  const char *text;   
  LumaTokenType type; 
} KeywordEntry;
 
void report_lexer_error(Lexer *lx, const char *error_type, const char *file,
                        const char *msg, const char *line_text, int line,
                        int col, int tk_length);
 
const char *get_line_text_from_source(const char *source, int target_line);
 
void init_lexer(Lexer *lexer, const char *source, ArenaAllocator *arena);
 
Token next_token(Lexer *lexer);