/* * par2serial-cc: lexer.c - Lexical Analyzer Implementation */ #include "lexer.h" /* ── Keyword table ───────────────────────────────────────── */ typedef struct { const char *kw; TokenType type; } KWEntry; static const KWEntry keywords[] = { /* C keywords */ {"auto", TOK_AUTO}, {"break", TOK_BREAK}, {"case", TOK_CASE}, {"char", TOK_CHAR}, {"const", TOK_CONST}, {"continue", TOK_CONTINUE}, {"default", TOK_DEFAULT}, {"do", TOK_DO}, {"double", TOK_DOUBLE}, {"else", TOK_ELSE}, {"enum", TOK_ENUM}, {"extern", TOK_EXTERN}, {"float", TOK_FLOAT}, {"for", TOK_FOR}, {"goto", TOK_GOTO}, {"if", TOK_IF}, {"inline", TOK_INLINE}, {"int", TOK_INT}, {"long", TOK_LONG}, {"register", TOK_REGISTER}, {"restrict", TOK_RESTRICT}, {"return", TOK_RETURN}, {"short", TOK_SHORT}, {"signed", TOK_SIGNED}, {"sizeof", TOK_SIZEOF}, {"static", TOK_STATIC}, {"struct", TOK_STRUCT}, {"switch", TOK_SWITCH}, {"typedef", TOK_TYPEDEF}, {"union", TOK_UNION}, {"unsigned", TOK_UNSIGNED}, {"void", TOK_VOID}, {"volatile", TOK_VOLATILE}, {"while", TOK_WHILE}, /* Parallel extensions */ {"parallel_for", TOK_PARALLEL_FOR}, {"parallel_reduce", TOK_PARALLEL_REDUCE}, {"parallel_scan", TOK_PARALLEL_SCAN}, {"parallel_map", TOK_PARALLEL_MAP}, {"barrier", TOK_BARRIER}, {"shared", TOK_SHARED}, {"tile_hint", TOK_TILE_HINT}, {"simd_hint", TOK_SIMD_HINT}, {"memory_layout", TOK_MEMORY_LAYOUT}, {"atomic_add", TOK_ATOMIC_ADD}, {"atomic_cas", TOK_ATOMIC_CAS}, {"thread_id", TOK_THREAD_ID}, {"num_threads", TOK_NUM_THREADS}, {NULL, TOK_EOF} }; /* ── Helper macros ───────────────────────────────────────── */ #define PEEK(lex) ((lex)->pos < (lex)->src_len ? (lex)->src[(lex)->pos] : '\0') #define PEEK_AT(lex, n) ((lex)->pos + (n) < (lex)->src_len ? (lex)->src[(lex)->pos + (n)] : '\0') #define AT_END(lex) ((lex)->pos >= (lex)->src_len) static char advance(Lexer *lex) { char c = PEEK(lex); if (c == '\n') { lex->line++; lex->col = 1; } else { lex->col++; } lex->pos++; return c; } static bool match(Lexer *lex, char expected) { if (PEEK(lex) == expected) { advance(lex); return true; } return false; } static SourceLoc make_loc(Lexer *lex) { return (SourceLoc){lex->filename, lex->line, lex->col}; } /* ── Token creation ──────────────────────────────────────── */ static Token make_token(Lexer *lex, TokenType type, const char *start, size_t len, SourceLoc loc) { Token t; t.type = type; t.loc = loc; t.text = arena_strndup(lex->arena, start, len); t.text_len = len; t.int_val = 0; return t; } static void emit(Lexer *lex, Token t) { vec_push(&lex->tokens, t); } /* ── Skip whitespace and comments ────────────────────────── */ static void skip_ws(Lexer *lex) { while (!AT_END(lex)) { char c = PEEK(lex); if (c == ' ' || c == '\t' || c == '\r' || c == '\n') { advance(lex); } else if (c == '/' && PEEK_AT(lex, 1) == '/') { /* line comment */ while (!AT_END(lex) && PEEK(lex) != '\n') advance(lex); } else if (c == '/' && PEEK_AT(lex, 1) == '*') { /* block comment */ advance(lex); advance(lex); while (!AT_END(lex)) { if (PEEK(lex) == '*' && PEEK_AT(lex, 1) == '/') { advance(lex); advance(lex); break; } advance(lex); } } else { break; } } } /* ── Lex number ──────────────────────────────────────────── */ static void lex_number(Lexer *lex) { SourceLoc loc = make_loc(lex); const char *start = lex->src + lex->pos; bool is_float = false; bool is_hex = false; if (PEEK(lex) == '0' && (PEEK_AT(lex, 1) == 'x' || PEEK_AT(lex, 1) == 'X')) { advance(lex); advance(lex); is_hex = true; while (isxdigit(PEEK(lex))) advance(lex); } else { while (isdigit(PEEK(lex))) advance(lex); if (PEEK(lex) == '.' && isdigit(PEEK_AT(lex, 1))) { is_float = true; advance(lex); while (isdigit(PEEK(lex))) advance(lex); } if (PEEK(lex) == 'e' || PEEK(lex) == 'E') { is_float = true; advance(lex); if (PEEK(lex) == '+' || PEEK(lex) == '-') advance(lex); while (isdigit(PEEK(lex))) advance(lex); } } /* suffix: f, l, u, ll etc */ while (PEEK(lex) == 'f' || PEEK(lex) == 'F' || PEEK(lex) == 'l' || PEEK(lex) == 'L' || PEEK(lex) == 'u' || PEEK(lex) == 'U') advance(lex); size_t len = (lex->src + lex->pos) - start; Token t = make_token(lex, is_float ? TOK_FLOAT_LIT : TOK_INT_LIT, start, len, loc); if (is_float) { t.float_val = strtod(start, NULL); } else if (is_hex) { t.int_val = (int64_t)strtoll(start, NULL, 16); } else { t.int_val = (int64_t)strtoll(start, NULL, 10); } emit(lex, t); } /* ── Lex string ──────────────────────────────────────────── */ static void lex_string(Lexer *lex) { SourceLoc loc = make_loc(lex); advance(lex); /* skip opening " */ const char *start = lex->src + lex->pos; while (!AT_END(lex) && PEEK(lex) != '"') { if (PEEK(lex) == '\\') advance(lex); /* skip escaped char */ advance(lex); } size_t len = (lex->src + lex->pos) - start; if (!AT_END(lex)) advance(lex); /* skip closing " */ emit(lex, make_token(lex, TOK_STRING_LIT, start, len, loc)); } /* ── Lex char literal ────────────────────────────────────── */ static void lex_char(Lexer *lex) { SourceLoc loc = make_loc(lex); advance(lex); /* skip ' */ const char *start = lex->src + lex->pos; if (PEEK(lex) == '\\') advance(lex); advance(lex); size_t len = (lex->src + lex->pos) - start; if (!AT_END(lex)) advance(lex); /* skip closing ' */ Token t = make_token(lex, TOK_CHAR_LIT, start, len, loc); t.int_val = (int64_t)(unsigned char)start[0]; emit(lex, t); } /* ── Lex identifier/keyword ──────────────────────────────── */ static void lex_ident(Lexer *lex) { SourceLoc loc = make_loc(lex); const char *start = lex->src + lex->pos; while (isalnum(PEEK(lex)) || PEEK(lex) == '_') advance(lex); size_t len = (lex->src + lex->pos) - start; /* check keywords */ TokenType type = TOK_IDENT; for (const KWEntry *k = keywords; k->kw; k++) { if (strlen(k->kw) == len && strncmp(k->kw, start, len) == 0) { type = k->type; break; } } emit(lex, make_token(lex, type, start, len, loc)); } /* ── Lex preprocessor line ───────────────────────────────── */ static void lex_preprocessor(Lexer *lex) { SourceLoc loc = make_loc(lex); const char *start = lex->src + lex->pos; advance(lex); /* skip # */ while (!AT_END(lex) && PEEK(lex) != '\n') { if (PEEK(lex) == '\\' && PEEK_AT(lex, 1) == '\n') { advance(lex); advance(lex); /* line continuation */ } else { advance(lex); } } size_t len = (lex->src + lex->pos) - start; emit(lex, make_token(lex, TOK_HASH, start, len, loc)); } /* ── Main tokenize ───────────────────────────────────────── */ Lexer *lexer_create(Arena *arena, const char *src, size_t len, const char *filename) { Lexer *lex = (Lexer *)arena_alloc(arena, sizeof(Lexer)); lex->src = src; lex->src_len = len; lex->pos = 0; lex->line = 1; lex->col = 1; lex->filename = filename; lex->arena = arena; vec_init(&lex->tokens); lex->error_count = 0; return lex; } bool lexer_tokenize(Lexer *lex) { while (!AT_END(lex)) { skip_ws(lex); if (AT_END(lex)) break; char c = PEEK(lex); SourceLoc loc = make_loc(lex); /* Numbers */ if (isdigit(c) || (c == '.' && isdigit(PEEK_AT(lex, 1)))) { lex_number(lex); continue; } /* Identifiers / keywords */ if (isalpha(c) || c == '_') { lex_ident(lex); continue; } /* String literal */ if (c == '"') { lex_string(lex); continue; } /* Char literal */ if (c == '\'') { lex_char(lex); continue; } /* Preprocessor */ if (c == '#') { lex_preprocessor(lex); continue; } /* Multi-char operators and single-char tokens */ advance(lex); switch (c) { case '(': emit(lex, make_token(lex, TOK_LPAREN, &c, 1, loc)); break; case ')': emit(lex, make_token(lex, TOK_RPAREN, &c, 1, loc)); break; case '{': emit(lex, make_token(lex, TOK_LBRACE, &c, 1, loc)); break; case '}': emit(lex, make_token(lex, TOK_RBRACE, &c, 1, loc)); break; case '[': emit(lex, make_token(lex, TOK_LBRACKET, &c, 1, loc)); break; case ']': emit(lex, make_token(lex, TOK_RBRACKET, &c, 1, loc)); break; case ';': emit(lex, make_token(lex, TOK_SEMICOLON,&c, 1, loc)); break; case ',': emit(lex, make_token(lex, TOK_COMMA, &c, 1, loc)); break; case '~': emit(lex, make_token(lex, TOK_TILDE, &c, 1, loc)); break; case '?': emit(lex, make_token(lex, TOK_QUESTION, &c, 1, loc)); break; case ':': emit(lex, make_token(lex, TOK_COLON, &c, 1, loc)); break; case '+': if (match(lex, '+')) emit(lex, make_token(lex, TOK_INC, "++", 2, loc)); else if (match(lex, '=')) emit(lex, make_token(lex, TOK_PLUS_EQ, "+=", 2, loc)); else emit(lex, make_token(lex, TOK_PLUS, "+", 1, loc)); break; case '-': if (match(lex, '-')) emit(lex, make_token(lex, TOK_DEC, "--", 2, loc)); else if (match(lex, '>')) emit(lex, make_token(lex, TOK_ARROW, "->", 2, loc)); else if (match(lex, '=')) emit(lex, make_token(lex, TOK_MINUS_EQ, "-=", 2, loc)); else emit(lex, make_token(lex, TOK_MINUS, "-", 1, loc)); break; case '*': if (match(lex, '=')) emit(lex, make_token(lex, TOK_STAR_EQ, "*=", 2, loc)); else emit(lex, make_token(lex, TOK_STAR, "*", 1, loc)); break; case '/': if (match(lex, '=')) emit(lex, make_token(lex, TOK_SLASH_EQ, "/=", 2, loc)); else emit(lex, make_token(lex, TOK_SLASH, "/", 1, loc)); break; case '%': if (match(lex, '=')) emit(lex, make_token(lex, TOK_PERCENT_EQ, "%=", 2, loc)); else emit(lex, make_token(lex, TOK_PERCENT, "%", 1, loc)); break; case '&': if (match(lex, '&')) emit(lex, make_token(lex, TOK_AND, "&&", 2, loc)); else if (match(lex, '=')) emit(lex, make_token(lex, TOK_AMP_EQ, "&=", 2, loc)); else emit(lex, make_token(lex, TOK_AMP, "&", 1, loc)); break; case '|': if (match(lex, '|')) emit(lex, make_token(lex, TOK_OR, "||", 2, loc)); else if (match(lex, '=')) emit(lex, make_token(lex, TOK_PIPE_EQ, "|=", 2, loc)); else emit(lex, make_token(lex, TOK_PIPE, "|", 1, loc)); break; case '^': if (match(lex, '=')) emit(lex, make_token(lex, TOK_CARET_EQ, "^=", 2, loc)); else emit(lex, make_token(lex, TOK_CARET, "^", 1, loc)); break; case '=': if (match(lex, '=')) emit(lex, make_token(lex, TOK_EQ, "==", 2, loc)); else emit(lex, make_token(lex, TOK_ASSIGN, "=", 1, loc)); break; case '!': if (match(lex, '=')) emit(lex, make_token(lex, TOK_NEQ, "!=", 2, loc)); else emit(lex, make_token(lex, TOK_BANG, "!", 1, loc)); break; case '<': if (match(lex, '<')) { if (match(lex, '=')) emit(lex, make_token(lex, TOK_LSHIFT_EQ, "<<=", 3, loc)); else emit(lex, make_token(lex, TOK_LSHIFT, "<<", 2, loc)); } else if (match(lex, '=')) emit(lex, make_token(lex, TOK_LE, "<=", 2, loc)); else emit(lex, make_token(lex, TOK_LT, "<", 1, loc)); break; case '>': if (match(lex, '>')) { if (match(lex, '=')) emit(lex, make_token(lex, TOK_RSHIFT_EQ, ">>=", 3, loc)); else emit(lex, make_token(lex, TOK_RSHIFT, ">>", 2, loc)); } else if (match(lex, '=')) emit(lex, make_token(lex, TOK_GE, ">=", 2, loc)); else emit(lex, make_token(lex, TOK_GT, ">", 1, loc)); break; case '.': if (PEEK(lex) == '.' && PEEK_AT(lex, 1) == '.') { advance(lex); advance(lex); emit(lex, make_token(lex, TOK_ELLIPSIS, "...", 3, loc)); } else { emit(lex, make_token(lex, TOK_DOT, ".", 1, loc)); } break; default: p2s_error(loc, "unexpected character '%c' (0x%02X)", c, (unsigned char)c); lex->error_count++; break; } } /* Append EOF */ SourceLoc eof_loc = make_loc(lex); emit(lex, make_token(lex, TOK_EOF, "", 0, eof_loc)); return lex->error_count == 0; } /* ── Debug ───────────────────────────────────────────────── */ static const char *token_names[] = { [TOK_INT_LIT]="INT_LIT", [TOK_FLOAT_LIT]="FLOAT_LIT", [TOK_STRING_LIT]="STRING", [TOK_CHAR_LIT]="CHAR", [TOK_IDENT]="IDENT", [TOK_AUTO]="auto", [TOK_BREAK]="break", [TOK_CASE]="case", [TOK_CHAR]="char", [TOK_CONST]="const", [TOK_CONTINUE]="continue", [TOK_DEFAULT]="default", [TOK_DO]="do", [TOK_DOUBLE]="double", [TOK_ELSE]="else", [TOK_ENUM]="enum", [TOK_EXTERN]="extern", [TOK_FLOAT]="float", [TOK_FOR]="for", [TOK_GOTO]="goto", [TOK_IF]="if", [TOK_INLINE]="inline", [TOK_INT]="int", [TOK_LONG]="long", [TOK_REGISTER]="register", [TOK_RESTRICT]="restrict", [TOK_RETURN]="return", [TOK_SHORT]="short", [TOK_SIGNED]="signed", [TOK_SIZEOF]="sizeof", [TOK_STATIC]="static", [TOK_STRUCT]="struct", [TOK_SWITCH]="switch", [TOK_TYPEDEF]="typedef", [TOK_UNION]="union", [TOK_UNSIGNED]="unsigned", [TOK_VOID]="void", [TOK_VOLATILE]="volatile", [TOK_WHILE]="while", [TOK_PARALLEL_FOR]="parallel_for", [TOK_PARALLEL_REDUCE]="parallel_reduce", [TOK_PARALLEL_SCAN]="parallel_scan", [TOK_PARALLEL_MAP]="parallel_map", [TOK_BARRIER]="barrier", [TOK_SHARED]="shared", [TOK_TILE_HINT]="tile_hint", [TOK_SIMD_HINT]="simd_hint", [TOK_MEMORY_LAYOUT]="memory_layout", [TOK_ATOMIC_ADD]="atomic_add", [TOK_ATOMIC_CAS]="atomic_cas", [TOK_THREAD_ID]="thread_id", [TOK_NUM_THREADS]="num_threads", [TOK_PLUS]="+", [TOK_MINUS]="-", [TOK_STAR]="*", [TOK_SLASH]="/", [TOK_PERCENT]="%", [TOK_AMP]="&", [TOK_PIPE]="|", [TOK_CARET]="^", [TOK_TILDE]="~", [TOK_BANG]="!", [TOK_LT]="<", [TOK_GT]=">", [TOK_EQ]="==", [TOK_NEQ]="!=", [TOK_LE]="<=", [TOK_GE]=">=", [TOK_AND]="&&", [TOK_OR]="||", [TOK_LSHIFT]="<<", [TOK_RSHIFT]=">>", [TOK_ASSIGN]="=", [TOK_PLUS_EQ]="+=", [TOK_MINUS_EQ]="-=", [TOK_STAR_EQ]="*=", [TOK_SLASH_EQ]="/=", [TOK_PERCENT_EQ]="%=", [TOK_AMP_EQ]="&=", [TOK_PIPE_EQ]="|=", [TOK_CARET_EQ]="^=", [TOK_LSHIFT_EQ]="<<=", [TOK_RSHIFT_EQ]=">>=", [TOK_INC]="++", [TOK_DEC]="--", [TOK_ARROW]="->", [TOK_DOT]=".", [TOK_QUESTION]="?", [TOK_COLON]=":", [TOK_LPAREN]="(", [TOK_RPAREN]=")", [TOK_LBRACE]="{", [TOK_RBRACE]="}", [TOK_LBRACKET]="[", [TOK_RBRACKET]="]", [TOK_SEMICOLON]=";", [TOK_COMMA]=",", [TOK_ELLIPSIS]="...", [TOK_HASH]="#", [TOK_EOF]="EOF", [TOK_ERROR]="ERROR", }; const char *token_type_str(TokenType t) { if (t >= 0 && t < TOK_COUNT && token_names[t]) return token_names[t]; return "UNKNOWN"; } void token_print(const Token *t) { printf("%-18s %-20.*s [%d:%d]\n", token_type_str(t->type), (int)t->text_len, t->text, t->loc.line, t->loc.col); }