feat: basic parsing of example.rx is ready

include/lexer.hpp

@@ -1,18 +1,51 @@
 #pragma once
 #include <iostream>
+#include <cstring>
 #include <cctype>
 
 #include "string.hpp"
 
-enum TokenType
+enum class TokenType
 {
     Eof = 256,
     Id,
     IntLiteral,
 
+    Extern,
+    Fn,
+    Local,
+
     Unknown,
 };
 
+template <typename... Ts>
+inline bool is_one_of(TokenType t, Ts... ts) {
+    return ((t == ts) || ...);
+}
+
+// operator== TokenType == char
+inline bool operator==(TokenType t, char c)
+{
+    return static_cast<char>(t) == c;
+}
+
+// operator== char == TokenType
+inline bool operator==(char c, TokenType t)
+{
+    return t == c;   // reuse the function above
+}
+
+// operator!=
+inline bool operator!=(TokenType t, char c)
+{
+    return !(t == c);
+}
+
+inline bool operator!=(char c, TokenType t)
+{
+    return !(t == c);
+}
+
 struct Token
 {
     TokenType token;
@@ -26,13 +59,32 @@ public:
     Token(TokenType t) : token(t) {}
     Token(TokenType t, long lnumber, long soffset, long eoffset)
         : token(t), line_number(lnumber), offset_start(soffset), offset_end(eoffset) {}
-    Token() : token(Unknown) {}
+    Token() : token(TokenType::Unknown) {}
 };
 
 class Lexer
 {
 public:
-    const Token& token() { return m_token; }
+    const Token& token() const { return m_token; }
+    const Token* seek_token()
+    {
+        auto s = m_token;
+        auto p = m_pos;
+        auto l = m_line;
+        auto lnl = m_last_newline;
+        if (!NextToken())
+        {
+            return nullptr;
+        }
+        auto seeked = m_token;
+        m_token = s;
+        m_pos = p;
+        m_line = l;
+        m_last_newline = lnl;
+        return new Token(seeked);
+    }
+
+    const char* filename() const { return m_filename; }
 public:
     Lexer(char* filename, StringView code)
         : m_filename(filename), m_code(code) {}
@@ -74,8 +126,24 @@ public:
                 s.PushChar(m_code.data[m_pos++]);
             }
             s.PushChar('\0');
-            m_token = Token(Id, m_line, offset_start, m_pos - m_last_newline);
+            m_token = Token(TokenType::Id, m_line, offset_start, m_pos - m_last_newline);
             m_token.string = s.data;
+
+            if (strcmp("extern", m_token.string) == 0)
+            {
+                m_token.token = TokenType::Extern;
+            }
+
+            if (strcmp("fn", m_token.string) == 0)
+            {
+                m_token.token = TokenType::Fn;
+            }
+
+            if (strcmp("local", m_token.string) == 0)
+            {
+                m_token.token = TokenType::Local;
+            }
+
             return true;
         }
 
@@ -91,7 +159,7 @@ public:
                 s.PushChar(m_code.data[m_pos++]);
             }
             s.PushChar('\0');
-            m_token = Token(IntLiteral, m_line, offset_start, m_pos - m_last_newline);
+            m_token = Token(TokenType::IntLiteral, m_line, offset_start, m_pos - m_last_newline);
             m_token.int_number = std::strtol(s.data, nullptr, hex ? 16 : 10);
             m_token.string = s.data;
             return true;
@@ -101,23 +169,23 @@ public:
         return true;
     }
 
-    void Eat(TokenType expected)
+    void NextExpect(TokenType expected)
     {
         if (!NextToken())
         {
-            fprintf(stderr, "%s:%d:%d: ERROR: expected %ld, but got EOF", m_filename, 0, 0, expected);
+            fprintf(stderr, "%s:%d:%d: ERROR: expected %ld, but got EOF\n", m_filename, 0, 0, expected);
             Exit(1);
         }
         if (token().token != expected)
         {
-            fprintf(stderr, "%s:%d:%d: ERROR: expected %ld, but got %ld", m_filename, token().line_number, token().offset_start, expected);
+            fprintf(stderr, "%s:%d:%d: ERROR: expected %ld, but got %ld\n", m_filename, token().line_number, token().offset_start, expected, token().token);
             Exit(1);
         }
     }
 
-    void Eat(char expected)
+    void NextExpect(char expected)
     {
-        Eat((TokenType)expected);
+        NextExpect((TokenType)expected);
     }
 private:
     void Exit(int status)