diff --git a/include/internal/syntax/decl.h b/include/internal/syntax/decl.h index 903a69c..526cf9c 100644 --- a/include/internal/syntax/decl.h +++ b/include/internal/syntax/decl.h @@ -7,7 +7,7 @@ namespace bed::internal::syntax { struct Token { uint32_t start; uint32_t end; - enum : uint8_t { + enum Kind : uint8_t { Data, Shebang, Comment, @@ -68,9 +68,20 @@ struct Token { } type; }; +struct ParseEvent { + std::string_view name; + enum : uint8_t { + Opening, + Closing, + SymbolDef, + Symbol + } ev_type; + uint8_t type; // type. +}; + struct Language { std::function none_state; - std::function *)> parse; + std::function *, std::vector *)> parse; std::function copy; std::function equal; std::function destroy; @@ -102,81 +113,40 @@ struct TreeCursor { ParseStateBranch *stack[64]; uint8_t depth = 0; bool went_left[64]; - TreeCursor(ParseState *root, uint64_t target_line, uint64_t *relative) { - ParseState *node = root; - while (node->is_branch()) { - auto *branch = (ParseStateBranch *)node; - auto *left = branch->left; - stack[depth] = branch; - if (target_line < left->lines()) { - went_left[depth] = true; - ++depth; - node = left; - } else { - target_line -= left->lines(); - went_left[depth] = false; - ++depth; - node = branch->right; - } - } - *relative = target_line; - leaf = (ParseStateLeaf *)node; + TreeCursor(ParseState *root, uint64_t target_line, uint64_t *relative); + void next(); + void prev(); +}; + +struct ScopeNode { + static constexpr uint64_t SCOPE_BIT = 1ull << 63; + static constexpr uint64_t LINES_MASK = ~SCOPE_BIT; + uint64_t header; + bool is_scope() const { + return header & SCOPE_BIT; } - void next() { - while (depth > 0) { - auto *branch = stack[depth - 1]; - bool from_left = went_left[depth - 1]; - --depth; - if (!from_left) - continue; - ParseState *node = branch->right; - while (node->is_branch()) { - auto *b = (ParseStateBranch *)node; - stack[depth] = b; - went_left[depth] = true; - ++depth; - node = b->left; - } - leaf = (ParseStateLeaf *)node; - return; - } - leaf = nullptr; - } - void prev() { - while (depth > 0) { - auto *branch = stack[depth - 1]; - bool from_left = went_left[depth - 1]; - --depth; - if (from_left) - continue; - ParseState *node = branch->left; - while (node->is_branch()) { - auto *b = (ParseStateBranch *)node; - stack[depth] = b; - went_left[depth] = false; - ++depth; - node = b->right; - } - leaf = (ParseStateLeaf *)node; - return; - } - leaf = nullptr; + uint64_t lines() const { + return header & LINES_MASK; } }; struct Symbol { - uint64_t definition; - std::vector references; + uint32_t definition; + uint16_t reference_count; + uint8_t type; + uint8_t len; + // first chars of count len, padded to 4 bytes. + // then a set of references. 32bit + // references later as modifying at end is faster than shifting the name. }; -struct Space { - uint64_t len; -}; - -struct Scope { - uint64_t len; - uint32_t type; - trie::Trie symbols; - std::vector> children; +struct alignas(8) Scope : ScopeNode { + uint16_t children_count; + uint16_t symbol_count; + uint8_t type; + uint8_t len; + uint32_t : 32; // if more stuff is needed use the padding first. + // first chars of count len, padded to 8 bytes. + // followed by that many number of pointers. }; } // namespace bed::internal::syntax diff --git a/include/internal/syntax/parser.h b/include/internal/syntax/parser.h index c87d9b6..bf4aff8 100644 --- a/include/internal/syntax/parser.h +++ b/include/internal/syntax/parser.h @@ -6,7 +6,7 @@ namespace bed::internal::syntax { struct Parser { - static constexpr const uint64_t MAX_CHUNK = 256; + static constexpr uint64_t MAX_CHUNK = 256; ParseState *root; Language lang; @@ -25,7 +25,7 @@ struct Parser { ParseState *join_tree(ParseState *a, ParseState *b); Scope scope_root; - Scope &get_scope(uint64_t); + Scope *get_scope(uint64_t); struct Iterator { Parser *p; @@ -33,6 +33,7 @@ struct Parser { void *state; uint64_t at; std::vector tokens; + std::vector events; Iterator(uint64_t, Parser *, vase::Vase &); ~Iterator(); Iterator(const Iterator &) = delete; diff --git a/include/internal/syntax/ruby/decl.h b/include/internal/syntax/ruby/decl.h index 91f9efb..a6ff7de 100644 --- a/include/internal/syntax/ruby/decl.h +++ b/include/internal/syntax/ruby/decl.h @@ -5,6 +5,15 @@ #include "tries.h" namespace bed::internal::syntax::ruby { +enum struct ScopeTypes : uint8_t { + None, + Comment, + Method, + Class, + Module, + Block +}; + struct alignas(2) RubyState { struct RubyInternalState { uint16_t brace_level; @@ -17,9 +26,16 @@ struct alignas(2) RubyState { COMMENT, END } state; - static constexpr const uint8_t EXPECTING_EXPRESSION = 0b00010000; - static constexpr const uint8_t ALLOW_INTERPOLATION = 0b00000001; - uint8_t flags; + static constexpr uint8_t NAME_MASK = 0b00000011; + enum : uint8_t { + NONE_NAME = 0b00, + CLASS_NAME = 0b01, + DEF_NAME = 0b10, + MODULE_NAME = 0b11 + }; + static constexpr uint8_t ALLOW_INTERPOLATION = 1 << 6; + static constexpr uint8_t EXPECTING_EXPRESSION = 1 << 7; + uint8_t flags = 0; char delim_start; char delim_end; }; diff --git a/include/internal/syntax/ruby/parser.h b/include/internal/syntax/ruby/parser.h index 92f154f..477de09 100644 --- a/include/internal/syntax/ruby/parser.h +++ b/include/internal/syntax/ruby/parser.h @@ -99,5 +99,8 @@ struct RubyParser { } }; -void ruby_parse(void **v_state, std::string_view line, bool first_line, std::vector *); +void ruby_parse( + void **v_state, std::string_view line, bool first_line, + std::vector *tokens, std::vector *events +); } // namespace bed::internal::syntax::ruby diff --git a/include/internal/syntax/ruby/tries.h b/include/internal/syntax/ruby/tries.h index c155c4f..05cc169 100644 --- a/include/internal/syntax/ruby/tries.h +++ b/include/internal/syntax/ruby/tries.h @@ -209,25 +209,31 @@ const static std::vector errors = { }; const static std::vector base_keywords = { - "class", - "module", - "begin", - "end", "else", "rescue", "ensure", - "do", "when", }; const static std::vector expecting_keywords = { - "if", "elsif", +}; + +const static std::vector expecting_end_keywords = { "case", "for", +}; + +const static std::vector conditional_keywords = { + "if", + "unless", "while", "until", - "unless", +}; + +const static std::vector end_keywords = { + "begin", + "do", }; const static std::vector operator_keywords = { @@ -312,6 +318,9 @@ struct RubyTries { trie::Trie builtins_trie; trie::Trie methods_trie; trie::Trie errors_trie; + trie::Trie expecting_end_keywords_trie; + trie::Trie conditional_keywords_trie; + trie::Trie end_keywords_trie; RubyTries() { for (auto &keyword : base_keywords) base_keywords_trie.insert(keyword); @@ -331,6 +340,12 @@ struct RubyTries { methods_trie.insert(keyword); for (auto &keyword : errors) errors_trie.insert(keyword); + for (auto &keyword : expecting_end_keywords) + expecting_end_keywords_trie.insert(keyword); + for (auto &keyword : conditional_keywords) + conditional_keywords_trie.insert(keyword); + for (auto &keyword : end_keywords) + end_keywords_trie.insert(keyword); } }; } // namespace bed::internal::syntax::ruby diff --git a/src/internal/syntax/parser.cc b/src/internal/syntax/parser.cc index 1101e95..558be9c 100644 --- a/src/internal/syntax/parser.cc +++ b/src/internal/syntax/parser.cc @@ -61,6 +61,7 @@ void Parser::reset(vase::Vase &vase, uint64_t lines, Language lang_) { it.next(); std::vector leaves; std::vector tokens; + std::vector events; leaves.reserve((lines + MAX_CHUNK - 1) / MAX_CHUNK); void *state = lang.none_state(); uint32_t consumed = 0; @@ -71,7 +72,7 @@ void Parser::reset(vase::Vase &vase, uint64_t lines, Language lang_) { uint32_t chunk_lines = 0; while (chunk_lines < MAX_CHUNK && consumed < lines) { tokens.clear(); - lang.parse(&state, it.line, consumed == 0, &tokens); + lang.parse(&state, it.line, consumed == 0, &tokens, &events); ++chunk_lines; ++consumed; if (!it.next() && consumed < lines) @@ -163,6 +164,7 @@ void Parser::modify(vase::Vase &vase, uint64_t target, uint64_t count) { if (count == 0 || !root) return; std::vector tokens; + std::vector events; uint64_t offset; TreeCursor c = TreeCursor(root, target, &offset); uint64_t at = target - offset; @@ -203,7 +205,7 @@ void Parser::modify(vase::Vase &vase, uint64_t target, uint64_t count) { c.leaf->state = lang.copy(state); } tokens.clear(); - lang.parse(&state, it.line, at == 0, &tokens); + lang.parse(&state, it.line, at == 0, &tokens, &events); at++; } lang.destroy(state); @@ -241,7 +243,7 @@ Parser::Iterator::Iterator(uint64_t target, Parser *p, vase::Vase &vase) : p(p) while (at < target) { it->next(); tokens.clear(); - p->lang.parse(&state, it->line, at == 0, &tokens); + p->lang.parse(&state, it->line, at == 0, &tokens, &events); at++; } } @@ -275,6 +277,6 @@ Parser::Iterator &Parser::Iterator::operator=(Iterator &&other) { void Parser::Iterator::next() { it->next(); tokens.clear(); - p->lang.parse(&state, it->line, at++ == 0, &tokens); + p->lang.parse(&state, it->line, at++ == 0, &tokens, &events); } } // namespace bed::internal::syntax diff --git a/src/internal/syntax/ruby/parse.cc b/src/internal/syntax/ruby/parse.cc index 7a7b717..42248dc 100644 --- a/src/internal/syntax/ruby/parse.cc +++ b/src/internal/syntax/ruby/parse.cc @@ -220,10 +220,11 @@ void handle_regex(RubyParser &p, std::vector *tokens) { tokens->push_back({start, p.len(), Token::Regexp}); } -bool handle_line_markers(RubyParser &p, std::vector *tokens) { +bool handle_line_markers(RubyParser &p, std::vector *tokens, std::vector *events) { if (p.len() == 6 && p.peek_str(6) == "=begin") { p.current().state = RubyState::RubyInternalState::COMMENT; p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION; + events->push_back({p.peek_str(6), ParseEvent::Opening, (uint8_t)ScopeTypes::Comment}); tokens->push_back({0, p.len(), Token::Comment}); return true; } @@ -248,8 +249,98 @@ bool handle_comment(RubyParser &p, std::vector *tokens, bool first_line) return false; } -void handle_syntax(RubyParser &p, std::vector *tokens) { +void handle_syntax(RubyParser &p, std::vector *tokens, std::vector *events) { static const RubyTries tries = RubyTries(); + if (p.current().flags & RubyState::RubyInternalState::NAME_MASK) { + switch (p.current().flags & RubyState::RubyInternalState::NAME_MASK) { + case RubyState::RubyInternalState::CLASS_NAME: { + while (p.peek() == ' ' || p.peek() == '\t') + p.advance(); + if (p.peek() == '\0') + return; + if (p.peek() == '<' && p.peek(1) == '<') { + p.advance(2); + events->push_back({"singleton", ParseEvent::Opening, (uint8_t)ScopeTypes::Class}); + p.current().flags = (p.current().flags & ~RubyState::RubyInternalState::NAME_MASK); + p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION; + return; + } + uint32_t k = p.i; + uint32_t j = 0; + if (identifier_start_char(p.peek(j))) { + j++; + while (identifier_char(p.peek(j))) + j++; + tokens->push_back({p.i, p.i + j, Token::Constant}); + p.advance(j); + while (p.peek() == ' ' || p.peek() == '\t') + p.advance(); + if (p.peek() == ':' && p.peek(1) == ':') { + p.advance(2); + return; + } + } + events->push_back({p.line.substr(k, j), ParseEvent::Opening, (uint8_t)ScopeTypes::Class}); + p.current().flags = (p.current().flags & ~RubyState::RubyInternalState::NAME_MASK); + return; + } + case RubyState::RubyInternalState::DEF_NAME: { + while (p.peek() == ' ' || p.peek() == '\t') + p.advance(); + if (p.peek() == '\0') + return; + uint32_t k = p.i; + uint32_t j = 0; + if (identifier_start_char(p.peek(j))) { + j++; + while (identifier_char(p.peek(j))) + j++; + if (p.peek(j) == '!' || p.peek(j) == '?') + j++; + if ('A' <= p.peek() && p.peek() <= 'Z') + tokens->push_back({p.i, p.i + j, Token::Constant}); + else if (j == 4 && p.peek_str(4) == "self") + tokens->push_back({p.i, p.i + j, Token::Keyword}); + else + tokens->push_back({p.i, p.i + j, Token::Function}); + p.advance(j); + while (p.peek() == ' ' || p.peek() == '\t') + p.advance(); + if (p.peek() == '.') { + p.advance(); + return; + } + } + events->push_back({p.line.substr(k, j), ParseEvent::Opening, (uint8_t)ScopeTypes::Method}); + p.current().flags = (p.current().flags & ~RubyState::RubyInternalState::NAME_MASK); + return; + } + case RubyState::RubyInternalState::MODULE_NAME: { + while (p.peek() == ' ' || p.peek() == '\t') + p.advance(); + if (p.peek() == '\0') + return; + uint32_t k = p.i; + uint32_t j = 0; + if (identifier_start_char(p.peek(j))) { + j++; + while (identifier_char(p.peek(j))) + j++; + tokens->push_back({p.i, p.i + j, Token::Constant}); + p.advance(j); + while (p.peek() == ' ' || p.peek() == '\t') + p.advance(); + if (p.peek() == ':' && p.peek(1) == ':') { + p.advance(2); + return; + } + } + events->push_back({p.line.substr(k, j), ParseEvent::Opening, (uint8_t)ScopeTypes::Module}); + p.current().flags = (p.current().flags & ~RubyState::RubyInternalState::NAME_MASK); + return; + } + } + } if (p.i + 3 <= p.len() && p.peek_str(2) == "<<") { uint32_t j = 2; bool indented = false; @@ -515,9 +606,9 @@ void handle_syntax(RubyParser &p, std::vector *tokens) { } case '{': { p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION; - /*uint8_t brace_color = - (uint8_t)Token::K_BRACE1 + (state->full_state.brace_level % 5); - tokens->push_back({p.i, p.i + 1, (Token)brace_color});*/ + uint8_t brace_color = + (uint8_t)Token::Brace1 + (p.current().brace_level % 5); + tokens->push_back({p.i, p.i + 1, (Token::Kind)brace_color}); p.current().brace_level++; p.advance(); return; @@ -528,18 +619,18 @@ void handle_syntax(RubyParser &p, std::vector *tokens) { p.pop_state(); tokens->push_back({p.i, p.i + 1, Token::Interpolation}); } else { - /*uint8_t brace_color = - (uint8_t)Token::K_BRACE1 + (state->full_state.brace_level % 5); - tokens->push_back({p.i, p.i + 1, (Token)brace_color});*/ + uint8_t brace_color = + (uint8_t)Token::Brace1 + (p.current().brace_level % 5); + tokens->push_back({p.i, p.i + 1, (Token::Kind)brace_color}); } p.advance(); return; } case '(': { p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION; - /*uint8_t brace_color = - (uint8_t)Token::K_BRACE1 + (state->full_state.brace_level % 5); - tokens->push_back({p.i, p.i + 1, (Token)brace_color});*/ + uint8_t brace_color = + (uint8_t)Token::Brace1 + (p.current().brace_level % 5); + tokens->push_back({p.i, p.i + 1, (Token::Kind)brace_color}); p.current().brace_level++; p.advance(); return; @@ -547,17 +638,17 @@ void handle_syntax(RubyParser &p, std::vector *tokens) { case ')': { p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION; p.current().brace_level--; - /*uint8_t brace_color = - (uint8_t)Token::K_BRACE1 + (state->full_state.brace_level % 5); - tokens->push_back({p.i, p.i + 1, (Token)brace_color});*/ + uint8_t brace_color = + (uint8_t)Token::Brace1 + (p.current().brace_level % 5); + tokens->push_back({p.i, p.i + 1, (Token::Kind)brace_color}); p.advance(); return; } case '[': { p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION; - /*uint8_t brace_color = - (uint8_t)Token::K_BRACE1 + (state->full_state.brace_level % 5); - tokens->push_back({p.i, p.i + 1, (Token)brace_color});*/ + uint8_t brace_color = + (uint8_t)Token::Brace1 + (p.current().brace_level % 5); + tokens->push_back({p.i, p.i + 1, (Token::Kind)brace_color}); p.current().brace_level++; p.advance(); return; @@ -565,9 +656,9 @@ void handle_syntax(RubyParser &p, std::vector *tokens) { case ']': { p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION; p.current().brace_level--; - /*uint8_t brace_color = - (uint8_t)Token::K_BRACE1 + (state->full_state.brace_level % 5); - tokens->push_back({p.i, p.i + 1, (Token)brace_color});*/ + uint8_t brace_color = + (uint8_t)Token::Brace1 + (p.current().brace_level % 5); + tokens->push_back({p.i, p.i + 1, (Token::Kind)brace_color}); p.advance(); return; } @@ -602,7 +693,7 @@ void handle_syntax(RubyParser &p, std::vector *tokens) { return; } case '%': { - if (p.current().flags & RubyState::RubyInternalState::EXPECTING_EXPRESSION || p.i + 1 >= p.len()) { + if (p.i + 1 >= p.len()) { tokens->push_back({p.i, p.i + 1, Token::Operator}); p.advance(); p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION; @@ -680,11 +771,16 @@ void handle_syntax(RubyParser &p, std::vector *tokens) { p.advance(prefix_len + 1); return; } + case ';': + p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION; + p.advance(); + return; default: if ('0' <= p.peek() && p.peek() <= '9') { p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION; uint32_t start = p.i; - if (p.peek() == '0') { + if (p.peek() == '0' + && (p.peek(1) == 'X' || p.peek(1) == 'x' || p.peek(1) == 'b' || p.peek(1) == 'B' || p.peek(1) == 'o' || p.peek(1) == 'O')) { p.advance(); if (p.peek() == 'x' || p.peek() == 'X') { p.advance(); @@ -716,15 +812,6 @@ void handle_syntax(RubyParser &p, std::vector *tokens) { else break; } - } else { - while (true) { - while (p.peek() >= '0' && p.peek() <= '7') - p.advance(); - if (p.peek() == '_') - p.advance(); - else - break; - } } } else { while (true) { @@ -763,51 +850,83 @@ void handle_syntax(RubyParser &p, std::vector *tokens) { tokens->push_back({start, p.i, Token::Number}); return; } else if (identifier_start_char(p.peek())) { - p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION; uint32_t j = 1; while (identifier_char(p.peek(j))) j++; if (p.peek(j) == '!' || p.peek(j) == '?') j++; - if (j == tries.base_keywords_trie.longest_match(p.peek_str(p.len() - p.i))) { + if (j == tries.base_keywords_trie.longest_match(p.peek_str(j))) { + p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION; tokens->push_back({p.i, p.i + j, Token::Keyword}); p.advance(j); return; - } else if (j == tries.expecting_keywords_trie.longest_match(p.peek_str(p.len() - p.i))) { + } else if (j == tries.expecting_keywords_trie.longest_match(p.peek_str(j))) { p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION; tokens->push_back({p.i, p.i + j, Token::Keyword}); p.advance(j); return; - } else if (j == tries.operator_keywords_trie.longest_match(p.peek_str(p.len() - p.i))) { + } else if (j == tries.operator_keywords_trie.longest_match(p.peek_str(j))) { + p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION; tokens->push_back({p.i, p.i + j, Token::KeywordOperator}); p.advance(j); return; - } else if (j == tries.expecting_operators_trie.longest_match(p.peek_str(p.len() - p.i))) { + } else if (j == tries.expecting_operators_trie.longest_match(p.peek_str(j))) { p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION; tokens->push_back({p.i, p.i + j, Token::KeywordOperator}); p.advance(j); return; - } else if (j == tries.types_trie.longest_match(p.peek_str(p.len() - p.i))) { + } else if (j == tries.types_trie.longest_match(p.peek_str(j))) { + p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION; tokens->push_back({p.i, p.i + j, Token::Type}); p.advance(j); return; - } else if (j == tries.methods_trie.longest_match(p.peek_str(p.len() - p.i))) { + } else if (j == tries.methods_trie.longest_match(p.peek_str(j))) { + p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION; tokens->push_back({p.i, p.i + j, Token::Function}); p.advance(j); return; - } else if (j == tries.builtins_trie.longest_match(p.peek_str(p.len() - p.i))) { + } else if (j == tries.expecting_end_keywords_trie.longest_match(p.peek_str(j))) { + events->push_back({p.peek_str(j), ParseEvent::Opening, 0}); + p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION; + tokens->push_back({p.i, p.i + j, Token::Keyword}); + p.advance(j); + return; + } else if (j == tries.conditional_keywords_trie.longest_match(p.peek_str(j))) { + if (p.current().flags & RubyState::RubyInternalState::EXPECTING_EXPRESSION) + events->push_back({p.peek_str(j), ParseEvent::Opening, 0}); + p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION; + tokens->push_back({p.i, p.i + j, Token::Keyword}); + p.advance(j); + return; + } else if (j == tries.end_keywords_trie.longest_match(p.peek_str(j))) { + p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION; + ScopeTypes kind = p.peek() == 'd' ? ScopeTypes::Block : ScopeTypes::None; + events->push_back({p.peek_str(j), ParseEvent::Opening, (uint8_t)kind}); + tokens->push_back({p.i, p.i + j, Token::Keyword}); + p.advance(j); + return; + } else if (j == tries.builtins_trie.longest_match(p.peek_str(j))) { + p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION; tokens->push_back({p.i, p.i + j, Token::Constant}); p.advance(j); return; - } else if (j == tries.errors_trie.longest_match(p.peek_str(p.len() - p.i))) { + } else if (j == tries.errors_trie.longest_match(p.peek_str(j))) { + p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION; tokens->push_back({p.i, p.i + j, Token::Error}); p.advance(j); return; } else if ('A' <= p.peek() && p.peek() <= 'Z' && !(p.peek(j) == '!' || p.peek(j) == '?')) { + p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION; + if (j >= 5 && p.peek_str(j).substr(j - 5) == "Error") { + tokens->push_back({p.i, p.i + j, Token::Error}); + p.advance(j); + return; + } tokens->push_back({p.i, p.i + j, Token::Constant}); p.advance(j); return; } else { + p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION; if (j == 4 && p.peek_str(4) == "true") { tokens->push_back({p.i, p.i + j, Token::True}); p.advance(4); @@ -818,32 +937,34 @@ void handle_syntax(RubyParser &p, std::vector *tokens) { p.advance(5); return; } + if (j == 3 && p.peek_str(3) == "end") { + events->push_back({p.peek_str(3), ParseEvent::Closing, 0}); + tokens->push_back({p.i, p.i + j, Token::Keyword}); + p.advance(3); + return; + } + if (j == 5 && p.peek_str(5) == "class") { + tokens->push_back({p.i, p.i + j, Token::Keyword}); + p.advance(5); + p.current().flags = + (p.current().flags & ~RubyState::RubyInternalState::NAME_MASK) + | RubyState::RubyInternalState::CLASS_NAME; + return; + } + if (j == 6 && p.peek_str(6) == "module") { + tokens->push_back({p.i, p.i + j, Token::Keyword}); + p.advance(6); + p.current().flags = + (p.current().flags & ~RubyState::RubyInternalState::NAME_MASK) + | RubyState::RubyInternalState::MODULE_NAME; + return; + } if (j == 3 && p.peek_str(3) == "def") { tokens->push_back({p.i, p.i + j, Token::Keyword}); p.advance(3); - while (p.peek() == ' ' || p.peek() == '\t') - p.advance(); - while (p.i < p.len()) { - if (identifier_start_char(p.peek())) { - uint32_t j = 1; - while (identifier_char(p.peek(j))) - j++; - if (p.peek(j) == '!' || p.peek(j) == '?') - j++; - if ('A' <= p.peek() && p.peek() <= 'Z') - tokens->push_back({p.i, p.i + j, Token::Constant}); - else if (j == 4 && p.peek_str(4) == "self") - tokens->push_back({p.i, p.i + j, Token::Keyword}); - else - tokens->push_back({p.i, p.i + j, Token::Function}); - p.advance(j); - if (p.peek() == '.') { - p.advance(); - continue; - } - } - break; - } + p.current().flags = + (p.current().flags & ~RubyState::RubyInternalState::NAME_MASK) + | RubyState::RubyInternalState::DEF_NAME; return; } uint32_t start = p.i; @@ -913,15 +1034,23 @@ void handle_syntax(RubyParser &p, std::vector *tokens) { } } -void ruby_parse(void **v_state, std::string_view line, bool fl, std::vector *tokens) { +void ruby_parse( + void **v_state, + std::string_view line, + bool fl, + std::vector *tokens, + std::vector *events +) { RubyParser p(v_state, line); while (p.i < p.len()) { if (p.current().state == RubyState::RubyInternalState::END) return; if (p.current().state == RubyState::RubyInternalState::COMMENT) { tokens->push_back({p.i, p.len(), Token::Comment}); - if (p.i == 0 && p.peek_str(4) == "=end") + if (p.i == 0 && p.peek_str(4) == "=end") { p.current().state = RubyState::RubyInternalState::NONE; + events->push_back({{nullptr, 0}, ParseEvent::Closing, 0}); + } return; } if (!p.heredoc_start_line @@ -939,11 +1068,11 @@ void ruby_parse(void **v_state, std::string_view line, bool fl, std::vectoris_branch()) { + auto *branch = (ParseStateBranch *)node; + auto *left = branch->left; + stack[depth] = branch; + if (target_line < left->lines()) { + went_left[depth] = true; + ++depth; + node = left; + } else { + target_line -= left->lines(); + went_left[depth] = false; + ++depth; + node = branch->right; + } + } + *relative = target_line; + leaf = (ParseStateLeaf *)node; +} + +void TreeCursor::next() { + while (depth > 0) { + auto *branch = stack[depth - 1]; + bool from_left = went_left[depth - 1]; + --depth; + if (!from_left) + continue; + ParseState *node = branch->right; + while (node->is_branch()) { + auto *b = (ParseStateBranch *)node; + stack[depth] = b; + went_left[depth] = true; + ++depth; + node = b->left; + } + leaf = (ParseStateLeaf *)node; + return; + } + leaf = nullptr; +} + +void TreeCursor::prev() { + while (depth > 0) { + auto *branch = stack[depth - 1]; + bool from_left = went_left[depth - 1]; + --depth; + if (from_left) + continue; + ParseState *node = branch->left; + while (node->is_branch()) { + auto *b = (ParseStateBranch *)node; + stack[depth] = b; + went_left[depth] = false; + ++depth; + node = b->right; + } + leaf = (ParseStateLeaf *)node; + return; + } + leaf = nullptr; +} +} // namespace bed::internal::syntax