a few code to help lexing.

lexing_tools.c

static bool is_alphabetic(const int c)
{
	return( (c>='a' && c<='z') || (c>='A' && c<='Z') || c=='_' || c=='$' || c=='@' || c < -1 );
}

static bool is_possible_id(const int c)
{
	return( is_alphabetic(c) || (c>='0' && c<='9') );
}

static bool is_decimal(const int c)
{
	return( c>='0' && c<='9' );
}

static bool is_binary(const int c)
{
	return( c=='0' || c=='1' );
}

static bool is_octal(const int c)
{
	return( c>='0' && c<='7' );
}

static bool is_hex(const int c)
{
	return( (c>='a' && c<='f') || (c>='A' && c<='F') || is_decimal(c) );
}

static bool is_white_space(const int c)
{
	return( c==' ' || c=='\t' || c=='\r' || c=='\v' || c=='\f' || c=='\n' );
}

static bool is_valid_ucn(const int c) {
	return ( 0xD800<=c && c<=0xDFFF ) ? false : ( 0xA0<=c||c=='$'||c=='@'||c=='`' );
}

/// clear_multi_line_comment(iter, "*/");
static inline void clear_multi_line_comment(char subcstr[], char end[restrict static 1])
{
	const size_t end_len = strlen(end);
	char *begin = subcstr;
	while( *begin != 0 && strncmp(end, begin, end_len) != 0 )
		begin++;
	begin += end_len;
	memset(subcstr, ' ', begin-subcstr);
}

static inline void clear_single_line_comment(char subcstr[])
{
	char *begin = subcstr;
	while( *begin != 0 && *begin != '\n' )
		begin += !strncmp(begin, "\\\n", sizeof "\\\n"-1) ? 2 : 1;
	memset(subcstr, ' ', begin-subcstr);
}

static inline const char *skip_string_literal(const char *str)
{
	const int quote = *str++;
	while( *str != 0 && *str != quote )
		str += ( *str=='\\' ) ? 2 : 1;
	return str;
}

static inline const char *skip_whitespace(const char *str)
{
	while( *str != 0 && is_whitespace(*str) )
		str++;
	return str;
}

static inline intptr_t get_file_size(FILE *const file)
{
	fseek(file, 0, SEEK_END);
	const intptr_t filesize = ftell(file);
	if( filesize<=0 )
		return -1;
	
	rewind(file);
	return filesize;
}

static inline char *make_buffer_from_text(const char file_name[restrict static 1])
{
	FILE *restrict file = fopen(file_name, "r");
	if( file==NULL )
		return NULL;
	
	const intptr_t filesize = get_file_size(file);
	if( filesize<=0 ) {
		fclose(file);
		return NULL;
	}
	char *restrict stream = calloc(filesize, sizeof *stream);
	const size_t bytes_read = fread(stream, sizeof *stream, filesize, file);
	fclose(file), file=NULL;
		
	if( bytes_read != (size_t)filesize ) {
		free(stream);
		stream=NULL;
	}
	return stream;
}