lexer.c File Reference

Implements the lexical analysis functions for tokenizing source code. More...

#include <ctype.h>
#include <stdbool.h>
#include <stdio.h>
#include <string.h>
#include "../c_libs/error/error.h"
#include "../c_libs/memory/memory.h"
#include "lexer.h"

Include dependency graph for lexer.c:

Macros
#define	STR_EQUALS_LEN(str, key, len)    (strncmp(str, key, len) == 0 && key[len] == '\0')
#define	MATCH_NEXT(lx, a, b)   (peek(lx, 0) == (a) && peek(lx, 1) == (b))
#define	MAKE_TOKEN(type, start, lx, length, whitespace_len)    make_token(type, start, lx->line, lx->col - 1, length, whitespace_len)

Functions
void	report_lexer_error (Lexer *lx, const char *error_type, const char *file, const char *msg, const char *line_text, int line, int col, int tk_length)
	Adds a lexer error to the global error list.
const char *	get_line_text_from_source (const char *source, int target_line)
	Retrieves the text of a specific line from the full source.
void	init_lexer (Lexer *lexer, const char *source, ArenaAllocator *arena)
	Initializes a Lexer struct for scanning the given source code.
char	peek (Lexer *lx, int offset)
bool	is_at_end (Lexer *lx)
char	advance (Lexer *lx)
Token	make_token (LumaTokenType type, const char *start, int line, int col, int length, int whitespace_len)
	Constructs a Token object.
int	skip_multiline_comment (Lexer *lx)
int	skip_whitespace (Lexer *lx)
Token	next_token (Lexer *lx)
	Retrieves the next token from the input stream.

Detailed Description

Implements the lexical analysis functions for tokenizing source code.

Contains the logic to identify tokens, skip whitespace and comments, recognize keywords and symbols, and report errors.

Macro Definition Documentation

MAKE_TOKEN

#define MAKE_TOKEN	(		type,
			start,
			lx,
			length,
			whitespace_len
	)		make_token(type, start, lx->line, lx->col - 1, length, whitespace_len)

MATCH_NEXT

#define MATCH_NEXT	(		lx,
			a,
			b
	)		(peek(lx, 0) == (a) && peek(lx, 1) == (b))

STR_EQUALS_LEN

#define STR_EQUALS_LEN	(		str,
			key,
			len
	)		(strncmp(str, key, len) == 0 && key[len] == '\0')

Function Documentation

advance()

char advance

(

Lexer *

lx

)

get_line_text_from_source()

const char * get_line_text_from_source	(	const char *	source,
		int	target_line
	)

Retrieves the text of a specific line from the full source.

Retrieves the text of a specific line from the source code.

Parameters

source	Full source code string
target_line	Line number to extract (1-based)

Returns

Pointer to static buffer containing the line text

init_lexer()

void init_lexer	(	Lexer *	lexer,
		const char *	source,
		ArenaAllocator *	arena
	)

Initializes a Lexer struct for scanning the given source code.

Initializes the lexer with source code and memory arena.

Parameters

lexer	Pointer to Lexer struct to initialize
source	Source code string
arena	Arena allocator to allocate tokens and strings

is_at_end()

bool is_at_end

(

Lexer *

lx

)

make_token()

Token make_token	(	LumaTokenType	type,
		const char *	start,
		int	line,
		int	col,
		int	length,
		int	whitespace_len
	)

Constructs a Token object.

Parameters

type	Token type
start	Pointer to start of token text
line	Line number
col	Column number
length	Length of token text
whitespace_len	Length of leading whitespace

Returns

Token struct initialized with given values

next_token()

Token next_token

(

Lexer *

lx

)

Retrieves the next token from the input stream.

Returns the next token parsed from the source code.

Parameters

lx	Pointer to Lexer instance

Returns

Next token found in the input

peek()

char peek	(	Lexer *	lx,
		int	offset
	)

report_lexer_error()

void report_lexer_error	(	Lexer *	lx,
		const char *	error_type,
		const char *	file,
		const char *	msg,
		const char *	line_text,
		int	line,
		int	col,
		int	tk_length
	)

Adds a lexer error to the global error list.

Reports a lexer error by adding an error to the global error list.

Parameters

lx	Lexer instance pointer
error_type	Description of the error type
file	Source file path
msg	Error message
line_text	Source code line text where error occurred
line	Line number of the error
col	Column number of the error
tk_length	Length of the token causing the error

skip_multiline_comment()

int skip_multiline_comment

(

Lexer *

lx

)

skip_whitespace()

int skip_whitespace

(

Lexer *

lx

)