From 739d2e450e33aa1992d4f45877e764d2884edfcb Mon Sep 17 00:00:00 2001 From: Syed Daanish Date: Fri, 21 Aug 2026 12:28:16 +0100 Subject: [PATCH] Improve ruby parsing heuristics. --- include/internal/syntax/decl.h | 53 ++++++-------- include/internal/syntax/parser.h | 3 - include/internal/syntax/ruby/parser.h | 5 +- include/internal/syntax/ruby/tries.h | 4 +- src/internal/syntax/ruby/parse.cc | 99 +++++++++++++++------------ src/internal/syntax/ruby/ruby.cc | 2 +- 6 files changed, 82 insertions(+), 84 deletions(-) diff --git a/include/internal/syntax/decl.h b/include/internal/syntax/decl.h index 526cf9c..48adf05 100644 --- a/include/internal/syntax/decl.h +++ b/include/internal/syntax/decl.h @@ -70,13 +70,16 @@ struct Token { struct ParseEvent { std::string_view name; - enum : uint8_t { + enum T : uint8_t { Opening, Closing, SymbolDef, Symbol } ev_type; uint8_t type; // type. + + ParseEvent(T t) : ev_type(t) {} + ParseEvent(std::string_view s, T e_t, uint8_t type) : name(s), ev_type(e_t), type(type) {} }; struct Language { @@ -87,6 +90,16 @@ struct Language { std::function destroy; }; +struct Symbol { + uint32_t definition; + uint16_t reference_count; + uint8_t type; + uint8_t len; + // first chars of count len, padded to 4 bytes. + // then a set of references. 32bit + // references later as modifying at end is faster than shifting the name. +}; + struct ParseState { static constexpr uint64_t BRANCH_BIT = 1ull << 63; static constexpr uint64_t LINES_MASK = ~BRANCH_BIT; @@ -106,6 +119,12 @@ struct ParseStateBranch : ParseState { struct ParseStateLeaf : ParseState { void *state; + uint64_t n; + static constexpr uint32_t IS_OPENING = 1ull << 31; + static constexpr uint32_t LINE_MASK = ~IS_OPENING; + // followed by n number of relative offsets + // stored as uint32_t with 1 bit for if it is start or end + // and rest as number. }; struct TreeCursor { @@ -117,36 +136,4 @@ struct TreeCursor { void next(); void prev(); }; - -struct ScopeNode { - static constexpr uint64_t SCOPE_BIT = 1ull << 63; - static constexpr uint64_t LINES_MASK = ~SCOPE_BIT; - uint64_t header; - bool is_scope() const { - return header & SCOPE_BIT; - } - uint64_t lines() const { - return header & LINES_MASK; - } -}; - -struct Symbol { - uint32_t definition; - uint16_t reference_count; - uint8_t type; - uint8_t len; - // first chars of count len, padded to 4 bytes. - // then a set of references. 32bit - // references later as modifying at end is faster than shifting the name. -}; - -struct alignas(8) Scope : ScopeNode { - uint16_t children_count; - uint16_t symbol_count; - uint8_t type; - uint8_t len; - uint32_t : 32; // if more stuff is needed use the padding first. - // first chars of count len, padded to 8 bytes. - // followed by that many number of pointers. -}; } // namespace bed::internal::syntax diff --git a/include/internal/syntax/parser.h b/include/internal/syntax/parser.h index bf4aff8..0c79b32 100644 --- a/include/internal/syntax/parser.h +++ b/include/internal/syntax/parser.h @@ -24,9 +24,6 @@ struct Parser { std::pair split_tree(ParseState *node, uint64_t line); ParseState *join_tree(ParseState *a, ParseState *b); - Scope scope_root; - Scope *get_scope(uint64_t); - struct Iterator { Parser *p; std::optional it; diff --git a/include/internal/syntax/ruby/parser.h b/include/internal/syntax/ruby/parser.h index 477de09..91d100a 100644 --- a/include/internal/syntax/ruby/parser.h +++ b/include/internal/syntax/ruby/parser.h @@ -10,6 +10,7 @@ struct RubyParser { std::string_view line; uint32_t i = 0; bool heredoc_start_line = false; + bool ending = true; RubyParser(void **v_state, std::string_view line) : v_state(v_state), state((RubyState *)*v_state), line(line) {} @@ -22,7 +23,7 @@ struct RubyParser { return state->stack()[state->top - 1]; } - char peek(uint32_t offset = 0) { + char peek(int32_t offset = 0) { uint32_t pos = i + offset; return pos < line.size() ? line[pos] : '\0'; } @@ -57,7 +58,7 @@ struct RubyParser { .brace_level = 1, .lit_brace_level = 0, .state = RubyState::RubyInternalState::NONE, - .flags = 0, + .flags = RubyState::RubyInternalState::EXPECTING_EXPRESSION, .delim_start = '\0', .delim_end = '\0' }; diff --git a/include/internal/syntax/ruby/tries.h b/include/internal/syntax/ruby/tries.h index 05cc169..130341c 100644 --- a/include/internal/syntax/ruby/tries.h +++ b/include/internal/syntax/ruby/tries.h @@ -212,10 +212,10 @@ const static std::vector base_keywords = { "else", "rescue", "ensure", - "when", }; const static std::vector expecting_keywords = { + "when", "elsif", }; @@ -247,13 +247,13 @@ const static std::vector operator_keywords = { "redo", "rescue", "retry", - "super", "self", "nil", "undef", }; const static std::vector expecting_operators = { + "super", "and", "return", "not", diff --git a/src/internal/syntax/ruby/parse.cc b/src/internal/syntax/ruby/parse.cc index 42248dc..71ae7a2 100644 --- a/src/internal/syntax/ruby/parse.cc +++ b/src/internal/syntax/ruby/parse.cc @@ -115,6 +115,7 @@ bool handle_heredoc(RubyParser &p, std::vector *tokens) { if (!p.dequeue_doc(heredoc_len)) p.current().state = RubyState::RubyInternalState::NONE; tokens->push_back({p.i, p.len(), Token::Annotation}); + p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION; return true; } } @@ -130,13 +131,13 @@ bool handle_heredoc(RubyParser &p, std::vector *tokens) { tokens->push_back({p.i, p.i + 2, Token::Interpolation}); p.advance(2); p.push_state(); - break; + return false; } p.advance(); } if (p.i >= p.len()) tokens->push_back({start, p.len(), Token::String}); - return false; + return true; } } @@ -151,7 +152,7 @@ void handle_string(RubyParser &p, std::vector *tokens) { tokens->push_back({p.i, p.i + 2, Token::Interpolation}); p.advance(2); p.push_state(); - break; + return; } if (p.peek() == p.current().delim_start && p.current().delim_start != p.current().delim_end) @@ -191,7 +192,7 @@ void handle_regex(RubyParser &p, std::vector *tokens) { tokens->push_back({p.i, p.i + 2, Token::Interpolation}); p.advance(2); p.push_state(); - break; + return; } if (p.peek() == p.current().delim_start && p.current().delim_start != p.current().delim_end) @@ -224,7 +225,7 @@ bool handle_line_markers(RubyParser &p, std::vector *tokens, std::vector< if (p.len() == 6 && p.peek_str(6) == "=begin") { p.current().state = RubyState::RubyInternalState::COMMENT; p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION; - events->push_back({p.peek_str(6), ParseEvent::Opening, (uint8_t)ScopeTypes::Comment}); + events->push_back(ParseEvent::Opening); tokens->push_back({0, p.len(), Token::Comment}); return true; } @@ -251,21 +252,22 @@ bool handle_comment(RubyParser &p, std::vector *tokens, bool first_line) void handle_syntax(RubyParser &p, std::vector *tokens, std::vector *events) { static const RubyTries tries = RubyTries(); + while (p.peek() == ' ' || p.peek() == '\t') + p.advance(); if (p.current().flags & RubyState::RubyInternalState::NAME_MASK) { + if (p.peek() == '\0') + return; + if (p.peek() == '\\' && p.peek(1) == '\0') + return; switch (p.current().flags & RubyState::RubyInternalState::NAME_MASK) { case RubyState::RubyInternalState::CLASS_NAME: { - while (p.peek() == ' ' || p.peek() == '\t') - p.advance(); - if (p.peek() == '\0') - return; if (p.peek() == '<' && p.peek(1) == '<') { p.advance(2); - events->push_back({"singleton", ParseEvent::Opening, (uint8_t)ScopeTypes::Class}); + events->push_back(ParseEvent::Opening); p.current().flags = (p.current().flags & ~RubyState::RubyInternalState::NAME_MASK); p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION; return; } - uint32_t k = p.i; uint32_t j = 0; if (identifier_start_char(p.peek(j))) { j++; @@ -280,16 +282,11 @@ void handle_syntax(RubyParser &p, std::vector *tokens, std::vectorpush_back({p.line.substr(k, j), ParseEvent::Opening, (uint8_t)ScopeTypes::Class}); + events->push_back(ParseEvent::Opening); p.current().flags = (p.current().flags & ~RubyState::RubyInternalState::NAME_MASK); return; } case RubyState::RubyInternalState::DEF_NAME: { - while (p.peek() == ' ' || p.peek() == '\t') - p.advance(); - if (p.peek() == '\0') - return; - uint32_t k = p.i; uint32_t j = 0; if (identifier_start_char(p.peek(j))) { j++; @@ -311,16 +308,11 @@ void handle_syntax(RubyParser &p, std::vector *tokens, std::vectorpush_back({p.line.substr(k, j), ParseEvent::Opening, (uint8_t)ScopeTypes::Method}); + events->push_back(ParseEvent::Opening); p.current().flags = (p.current().flags & ~RubyState::RubyInternalState::NAME_MASK); return; } case RubyState::RubyInternalState::MODULE_NAME: { - while (p.peek() == ' ' || p.peek() == '\t') - p.advance(); - if (p.peek() == '\0') - return; - uint32_t k = p.i; uint32_t j = 0; if (identifier_start_char(p.peek(j))) { j++; @@ -335,7 +327,7 @@ void handle_syntax(RubyParser &p, std::vector *tokens, std::vectorpush_back({p.line.substr(k, j), ParseEvent::Opening, (uint8_t)ScopeTypes::Module}); + events->push_back(ParseEvent::Opening); p.current().flags = (p.current().flags & ~RubyState::RubyInternalState::NAME_MASK); return; } @@ -349,7 +341,7 @@ void handle_syntax(RubyParser &p, std::vector *tokens, std::vectorpush_back({p.i, p.i + j, Token::Operator}); - if (j >= p.len()) + if (p.i + j >= p.len()) return; std::string delim; bool interpolation = true; @@ -358,15 +350,17 @@ void handle_syntax(RubyParser &p, std::vector *tokens, std::vectorpush_back({s, p.i + j, Token::Annotation}); @@ -762,7 +756,9 @@ void handle_syntax(RubyParser &p, std::vector *tokens, std::vectorpush_back({p.i, p.i + prefix_len + 1, (is_regexp ? Token::Regexp : Token::String)}); - p.current().state = is_regexp ? RubyState::RubyInternalState::REGEXP : RubyState::RubyInternalState::STRING; + p.current().state = is_regexp + ? RubyState::RubyInternalState::REGEXP + : RubyState::RubyInternalState::STRING; p.current().delim_start = delim_start; p.current().delim_end = delim_end; if (allow_interp) @@ -775,12 +771,21 @@ void handle_syntax(RubyParser &p, std::vector *tokens, std::vector *tokens, std::vector= '0' && p.peek() <= '7') p.advance(); @@ -814,6 +822,7 @@ void handle_syntax(RubyParser &p, std::vector *tokens, std::vector= '0' && p.peek() <= '9') p.advance(); @@ -876,7 +885,7 @@ void handle_syntax(RubyParser &p, std::vector *tokens, std::vectorpush_back({p.i, p.i + j, Token::Type}); p.advance(j); return; @@ -886,22 +895,21 @@ void handle_syntax(RubyParser &p, std::vector *tokens, std::vectorpush_back({p.peek_str(j), ParseEvent::Opening, 0}); + events->push_back(ParseEvent::Opening); p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION; tokens->push_back({p.i, p.i + j, Token::Keyword}); p.advance(j); return; } else if (j == tries.conditional_keywords_trie.longest_match(p.peek_str(j))) { if (p.current().flags & RubyState::RubyInternalState::EXPECTING_EXPRESSION) - events->push_back({p.peek_str(j), ParseEvent::Opening, 0}); + events->push_back(ParseEvent::Opening); p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION; tokens->push_back({p.i, p.i + j, Token::Keyword}); p.advance(j); return; } else if (j == tries.end_keywords_trie.longest_match(p.peek_str(j))) { - p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION; - ScopeTypes kind = p.peek() == 'd' ? ScopeTypes::Block : ScopeTypes::None; - events->push_back({p.peek_str(j), ParseEvent::Opening, (uint8_t)kind}); + p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION; + events->push_back(ParseEvent::Opening); tokens->push_back({p.i, p.i + j, Token::Keyword}); p.advance(j); return; @@ -938,7 +946,7 @@ void handle_syntax(RubyParser &p, std::vector *tokens, std::vectorpush_back({p.peek_str(3), ParseEvent::Closing, 0}); + events->push_back(ParseEvent::Closing); tokens->push_back({p.i, p.i + j, Token::Keyword}); p.advance(3); return; @@ -973,7 +981,7 @@ void handle_syntax(RubyParser &p, std::vector *tokens, std::vectorpush_back({start, p.i, Token::Label}); return; - } else if (p.peek() == '!' || p.peek() == '?') { + } else if (p.peek(-1) == '!' || p.peek(-1) == '?') { p.advance(); tokens->push_back({start, p.i, Token::Function}); return; @@ -1019,6 +1027,7 @@ void handle_syntax(RubyParser &p, std::vector *tokens, std::vectorpush_back({start, p.i, Token::Function}); + return; } } } else { @@ -1042,14 +1051,14 @@ void ruby_parse( std::vector *events ) { RubyParser p(v_state, line); - while (p.i < p.len()) { + while (p.i <= p.len()) { if (p.current().state == RubyState::RubyInternalState::END) return; if (p.current().state == RubyState::RubyInternalState::COMMENT) { tokens->push_back({p.i, p.len(), Token::Comment}); if (p.i == 0 && p.peek_str(4) == "=end") { p.current().state = RubyState::RubyInternalState::NONE; - events->push_back({{nullptr, 0}, ParseEvent::Closing, 0}); + events->push_back(ParseEvent::Closing); } return; } @@ -1062,10 +1071,14 @@ void ruby_parse( } if (p.current().state == RubyState::RubyInternalState::STRING) { handle_string(p, tokens); + if (p.current().state == RubyState::RubyInternalState::STRING) + return; continue; } if (p.current().state == RubyState::RubyInternalState::REGEXP) { handle_regex(p, tokens); + if (p.current().state == RubyState::RubyInternalState::REGEXP) + return; continue; } if (!p.i && handle_line_markers(p, tokens, events)) diff --git a/src/internal/syntax/ruby/ruby.cc b/src/internal/syntax/ruby/ruby.cc index 6027134..567ea42 100644 --- a/src/internal/syntax/ruby/ruby.cc +++ b/src/internal/syntax/ruby/ruby.cc @@ -14,7 +14,7 @@ Language lang_ruby() { .brace_level = 1, .lit_brace_level = 0, .state = RubyState::RubyInternalState::NONE, - .flags = 0, + .flags = RubyState::RubyInternalState::EXPECTING_EXPRESSION, .delim_start = '\0', .delim_end = '\0' };