first try of registry allocation

include/parser/ast.hpp

@@ -0,0 +1,415 @@
+#pragma once
+#include <cstring>
+#include <cassert>
+#include <vector>
+#include "parser/lexer.hpp"
+
+enum class NodeType
+{
+    Expression = 0,
+    // TODO: abstract "Literal" node type
+    IntLiteral,
+    Extern,
+    FnDecl,
+    FnCall,
+    Variable,
+    VarDecl,
+    Compound,
+    Program,
+    COUNT_NODES,
+};
+
+#define NODE_TYPE(x) \
+    NodeType GetType() const override { return NodeType::x; }
+
+class Node
+{
+public:
+    virtual NodeType GetType() const = 0;
+    virtual ~Node() {}
+}; 
+
+class ExpressionNode : public Node
+{
+public:
+    enum class Operator
+    {
+        Plus = 0,
+        Minus,
+        Divide,
+        Multiply,
+        COUNT_OPERATORS,
+    };
+public:
+    ExpressionNode(Node* left, Node* right, Operator op)
+        : m_left(left), m_right(right), m_op(op) {}
+    ~ExpressionNode() override {
+        delete m_left;
+        delete m_right;
+    }
+
+    NODE_TYPE(Expression)
+public:
+    Node* left() const { return m_left; }
+    Node* right() const { return m_right; }
+    Operator op() const { return m_op; }
+private:
+    Node* m_left;
+    Node* m_right;
+    Operator m_op;
+};
+
+// TODO: Maybe just LiteralNode with double or int literals support
+class IntLiteralNode : public Node
+{
+public:
+    IntLiteralNode(long value)
+        : m_int_value(value) {}
+    ~IntLiteralNode() override {}
+
+    NODE_TYPE(IntLiteral)
+public:
+    long integer() const { return m_int_value; }
+private:
+    long m_int_value;
+};
+
+class ExternNode : public Node
+{
+public:
+    // TODO: support multiple extern symbols
+    ExternNode(StringView symbol)
+        : m_symbol(symbol) {}
+    ~ExternNode() override = default;
+
+    NODE_TYPE(Extern)
+public:
+    const StringView& symbol() const { return m_symbol; }
+private:
+    StringView m_symbol;
+};
+
+class CompoundNode : public Node
+{
+public:
+    CompoundNode() = default;
+
+    NODE_TYPE(Compound)
+
+    // --- Iteration ---
+    auto begin() { return m_nodes.begin(); }
+    auto end()   { return m_nodes.end();   }
+    auto begin() const { return m_nodes.begin(); }
+    auto end()   const { return m_nodes.end();   }
+
+    // --- Access by index ---
+    Node* operator[](size_t i) { return m_nodes[i]; }
+    const Node* operator[](size_t i) const { return m_nodes[i]; }
+
+    Node* at(size_t i) { return m_nodes.at(i); }
+    const Node* at(size_t i) const { return m_nodes.at(i); }
+
+    // --- Modifiers ---
+    void addNode(Node* n) { m_nodes.push_back(n); }
+    void removeNode(size_t idx) { m_nodes.erase(m_nodes.begin() + idx); }
+
+    // If you want full expose for iteration but not modification
+    const std::vector<Node*>& nodes() const { return m_nodes; }
+
+    size_t size() const { return m_nodes.size(); }
+    bool empty() const { return m_nodes.empty(); }
+private:
+    std::vector<Node*> m_nodes;
+};
+
+class FnDeclNode : public Node
+{
+public:
+    // TODO: support parameters
+    FnDeclNode(const StringView& name, CompoundNode* body, View<StringView> params)
+        : m_name(name), m_body(body), m_params(params) {}
+    ~FnDeclNode() override {
+        delete m_body;
+    }
+
+    NODE_TYPE(FnDecl)
+public:
+    const StringView& name() const { return m_name; }
+    const CompoundNode* body() const { return m_body; }
+    const View<StringView>& params() const { return m_params; }
+private:
+    StringView m_name;
+    CompoundNode* m_body;
+    View<StringView> m_params;
+};
+
+class FnCallNode : public Node
+{
+public:
+    // TODO: support multiple arguments
+    FnCallNode(const StringView& name, Node* arg)
+        : m_name(name), m_arg(arg) {}
+    ~FnCallNode() override {
+        delete m_arg;
+    }
+
+    NODE_TYPE(FnCall)
+public:
+    const StringView& name() const { return m_name; }
+    // TODO: support multiple args
+    const Node* arg() const { return m_arg; }
+private:
+    StringView m_name;
+    Node* m_arg;
+};
+
+class VariableNode : public Node
+{
+public:
+    VariableNode(const StringView& name)
+        : m_name(name) {}
+    ~VariableNode() override = default;
+
+    NODE_TYPE(Variable)
+public:
+    const StringView& name() const { return m_name; }
+private:
+    StringView m_name;
+};
+
+class VarDeclNode : public Node
+{
+public:
+    VarDeclNode(const StringView& name, Node* value)
+        : m_name(name), m_value(value) {}
+    ~VarDeclNode() override {
+        delete m_value;
+    }
+
+    NODE_TYPE(VarDecl)
+public:
+    const StringView& name() const { return m_name; }
+    const Node* value() const { return m_value; }
+private:
+    StringView m_name;
+    Node* m_value;
+};
+
+class ProgramNode : public Node
+{
+public:
+    ProgramNode() = default;
+
+    NODE_TYPE(Program)
+public:
+    void PushFunction(FnDeclNode* fn)
+    {
+        m_funcs.push_back(fn);
+    }
+
+    void PushExtern(ExternNode* extrn)
+    {
+        m_externs.push_back(extrn);
+    }
+public:
+    const std::vector<ExternNode*> externs() const { return m_externs; }
+    const std::vector<FnDeclNode*> funcs() const { return m_funcs; }
+private:
+    std::vector<FnDeclNode*> m_funcs;
+    std::vector<ExternNode*> m_externs;
+};
+
+
+class AstParser
+{
+public:
+    AstParser(Lexer* lexer)
+        : m_lexer(lexer) {}
+public:
+    ExternNode* ParseExtern()
+    {
+        m_lexer->NextExpect(TokenType::Id);
+        return new ExternNode(m_lexer->token().string);
+    }
+
+    FnDeclNode* ParseFnDecl()
+    {
+        // Function Declaration
+        m_lexer->NextExpect(TokenType::Id);
+        StringView name = m_lexer->token().string;
+        m_lexer->NextExpect('(');
+        Builder<StringView> params;
+        // TODO: support multiple params
+        if (m_lexer->seek_token()->token != ')')
+        {
+            m_lexer->NextExpect(TokenType::Id);
+            params.Push(m_lexer->token().string);
+        }
+        m_lexer->NextExpect(')');
+        m_lexer->NextExpect('{');
+        auto compound = new CompoundNode();
+        while (m_lexer->seek_token()->token != '}')
+        {
+            compound->addNode(ParseStatement());
+        }
+        m_lexer->NextExpect('}');
+        return new FnDeclNode(name, compound, params.view());
+    }
+
+    FnCallNode* ParseFnCall(const StringView& name)
+    {
+        // m_lexer->NextExpect(TokenType::Id);
+        // char* name = strdup(m_lexer->token().string);
+        m_lexer->NextExpect('(');
+        Node* arg = nullptr;
+        // TODO: support multiple arguments
+        if (m_lexer->seek_token()->token != ')')
+        {
+            arg = ParseExpression();
+        }
+        m_lexer->NextExpect(')');
+        return new FnCallNode(name, arg);
+    }
+
+    Node* ParseFactor()
+    {
+        auto token = m_lexer->seek_token();
+
+        switch (token->token)
+        {
+        case TokenType::IntLiteral: // integer
+        {
+            m_lexer->NextExpect(TokenType::IntLiteral);
+            auto node = new IntLiteralNode(m_lexer->token().int_number);
+            return node;
+        }
+        case TokenType::Id: // variable name or function call
+        {
+            m_lexer->NextExpect(TokenType::Id);
+            auto name = m_lexer->token().string;
+            token = m_lexer->seek_token();
+            if (token->token == '(')
+            {
+                return ParseFnCall(name);
+            }
+            return new VariableNode(name);
+        }
+        default:
+            fprintf(stderr, "%s:%d:%d: ERROR: unexpected token while parsing %ld\n", m_lexer->filename(), token->line_number, token->offset_start, token->token);
+            Exit(1);
+            break;
+        }
+
+        assert(0 && "unreachable");
+    }
+
+    Node* ParseTerm()
+    {
+        auto t = ParseFactor();
+
+        for (auto op = m_lexer->seek_token(); is_one_of(op->token, '/', '*'); op = m_lexer->seek_token())
+        {
+            m_lexer->NextToken();
+            ExpressionNode::Operator eop;
+            assert((int)ExpressionNode::Operator::COUNT_OPERATORS == 4 && "some operators may not be handled");
+            switch((char)op->token)
+            {
+            case '/':
+                eop = ExpressionNode::Operator::Divide;
+                break;
+            case '*':
+                eop = ExpressionNode::Operator::Multiply;
+                break;
+            default:
+                assert(false && "should be unreachable");
+                break;
+            }
+            auto expr = new ExpressionNode(t, ParseTerm(), eop);
+            t = expr;
+        }
+
+        return t;
+    }
+
+    Node* ParseExpression()
+    {
+        auto t = ParseTerm();
+
+        for (auto op = m_lexer->seek_token(); is_one_of(op->token, '+', '-'); op = m_lexer->seek_token())
+        {
+            m_lexer->NextToken();
+            ExpressionNode::Operator eop;
+            assert((int)ExpressionNode::Operator::COUNT_OPERATORS == 4 && "some operators may not be handled");
+            switch((char)op->token)
+            {
+            case '+':
+                eop = ExpressionNode::Operator::Plus;
+                break;
+            case '-':
+                eop = ExpressionNode::Operator::Minus;
+                break;
+            default:
+                assert(false && "should be unreachable");
+                break;
+            }
+            auto expr = new ExpressionNode(t, ParseTerm(), eop);
+            t = expr;
+        }
+
+        return t;
+    }
+
+    VarDeclNode* ParseVarDecl()
+    {
+        m_lexer->NextExpect(TokenType::Local);
+        m_lexer->NextExpect(TokenType::Id);
+        auto name = m_lexer->token().string;
+        m_lexer->NextExpect('=');
+        Node* value = ParseExpression();
+        return new VarDeclNode(name, value);
+    }
+
+    Node* ParseStatement()
+    {
+        auto token = m_lexer->seek_token();
+        // TODO: proper error handling
+        assert(token != nullptr && "next token should be available");
+        switch(token->token)
+        {
+        case TokenType::Local: return ParseVarDecl();
+        default: return ParseExpression();
+        }
+
+        assert(0 && "unreachable");
+        return nullptr;
+    }
+
+    ProgramNode* Parse()
+    {
+        auto program = new ProgramNode;
+
+        while (m_lexer->NextToken())
+        {
+            auto token = m_lexer->token();
+            switch(token.token)
+            {
+            case TokenType::Extern: program->PushExtern(ParseExtern()); break;
+            case TokenType::Fn: program->PushFunction(ParseFnDecl()); break;
+            default: {
+                fprintf(stderr, "%s:%d:%d: ERROR: unexpected token while parsing %ld\n", m_lexer->filename(), token.line_number, token.offset_start, token.token);
+                Exit(1);
+                break;
+            }
+            }
+        }
+
+        return program;
+    }
+private:
+    void Exit(int status)
+    {
+        std::exit(status);
+    }
+private:
+    Lexer* m_lexer;
+};

include/parser/lexer.hpp

@@ -0,0 +1,199 @@
+#pragma once
+#include <iostream>
+#include <cstring>
+#include <cctype>
+
+#include "prelude/string.hpp"
+
+enum class TokenType
+{
+    Eof = 256,
+    Id,
+    IntLiteral,
+
+    Extern,
+    Fn,
+    Local,
+
+    Unknown,
+};
+
+template <typename... Ts>
+inline bool is_one_of(TokenType t, Ts... ts) {
+    return ((t == ts) || ...);
+}
+
+// operator== TokenType == char
+inline bool operator==(TokenType t, char c)
+{
+    return static_cast<char>(t) == c;
+}
+
+// operator== char == TokenType
+inline bool operator==(char c, TokenType t)
+{
+    return t == c;   // reuse the function above
+}
+
+// operator!=
+inline bool operator!=(TokenType t, char c)
+{
+    return !(t == c);
+}
+
+inline bool operator!=(char c, TokenType t)
+{
+    return !(t == c);
+}
+
+struct Token
+{
+    TokenType token;
+    long int_number;
+    // null-terminated
+    StringView string;
+    long line_number;
+    long offset_start;
+    long offset_end;
+public:
+    Token(TokenType t) : token(t) {}
+    Token(TokenType t, long lnumber, long soffset, long eoffset)
+        : token(t), line_number(lnumber), offset_start(soffset), offset_end(eoffset) {}
+    Token() : token(TokenType::Unknown) {}
+};
+
+class Lexer
+{
+public:
+    const Token& token() const { return m_token; }
+    const Token* seek_token()
+    {
+        auto s = m_token;
+        auto p = m_pos;
+        auto l = m_line;
+        auto lnl = m_last_newline;
+        if (!NextToken())
+        {
+            return nullptr;
+        }
+        auto seeked = m_token;
+        m_token = s;
+        m_pos = p;
+        m_line = l;
+        m_last_newline = lnl;
+        return new Token(seeked);
+    }
+
+    const char* filename() const { return m_filename; }
+public:
+    Lexer(char* filename, StringView code)
+        : m_filename(filename), m_code(code) {}
+
+    Lexer(const Lexer&) = delete;
+public:
+    bool NextToken()
+    {
+        if (m_pos >= m_code.size || m_code.data[m_pos] == '\0')
+        {
+            m_token = Token(TokenType::Eof);
+            return false;
+        }
+
+        char c = m_code.data[m_pos++];
+
+        while(std::isspace(c)) {
+            if (c == '\n')
+            {
+                m_line++;
+                m_last_newline = m_pos;
+            }
+            c = m_code.data[m_pos++];
+        }
+
+        if (std::isalpha(c) != 0 || c == '_')
+        {
+            StringBuilder s;
+            long offset_start = m_pos - m_last_newline;
+            s.Push(c);
+            // id
+            while (std::isalpha(m_code.data[m_pos]) != 0 || m_code.data[m_pos] == '_')
+            {
+                s.Push(m_code.data[m_pos++]);
+            }
+            s.Push('\0');
+            m_token = Token(TokenType::Id, m_line, offset_start, m_pos - m_last_newline);
+            m_token.string = s.view();
+
+            if (strcmp("extern", m_token.string.c_str()) == 0)
+            {
+                m_token.token = TokenType::Extern;
+            }
+
+            if (strcmp("fn", m_token.string.c_str()) == 0)
+            {
+                m_token.token = TokenType::Fn;
+            }
+
+            if (strcmp("local", m_token.string.c_str()) == 0)
+            {
+                m_token.token = TokenType::Local;
+            }
+
+            return true;
+        }
+
+        if (std::isdigit(c) != 0)
+        {
+            StringBuilder s;
+            long offset_start = m_pos - m_last_newline;
+            bool hex = c == '0' && m_code.data[m_pos] == 'x';
+            s.Push(c);
+            // integer (could be hex)
+            while (std::isdigit(m_code.data[m_pos]) != 0 || (hex && std::isalpha(m_code.data[m_pos]) != 0))
+            {
+                s.Push(m_code.data[m_pos++]);
+            }
+            s.Push('\0');
+            m_token = Token(TokenType::IntLiteral, m_line, offset_start, m_pos - m_last_newline);
+            m_token.int_number = std::strtol(s.data, nullptr, hex ? 16 : 10);
+            m_token.string = s.data;
+            return true;
+        }
+        
+        m_token = Token((TokenType)c, m_line, m_pos - m_last_newline, m_pos - m_last_newline + 1);
+        return true;
+    }
+
+    void NextExpect(TokenType expected)
+    {
+        if (!NextToken())
+        {
+            fprintf(stderr, "%s:%d:%d: ERROR: expected %ld, but got EOF\n", m_filename, 0, 0, expected);
+            Exit(1);
+        }
+        if (token().token != expected)
+        {
+            fprintf(stderr, "%s:%d:%d: ERROR: expected %ld, but got %ld\n", m_filename, token().line_number, token().offset_start, expected, token().token);
+            Exit(1);
+        }
+    }
+
+    void NextExpect(char expected)
+    {
+        NextExpect((TokenType)expected);
+    }
+private:
+    void Exit(int status)
+    {
+        std::exit(status);
+    }
+private:
+    StringView m_code;
+    char* m_filename;
+
+    Token m_token;
+
+    size_t m_pos = 0;
+    size_t m_line = 1;
+    size_t m_last_newline = 0;
+};