Improve ruby parsing heuristics.

include/internal/syntax/decl.h

@@ -70,13 +70,16 @@ struct Token {
 
 struct ParseEvent {
   std::string_view name;
-  enum : uint8_t {
+  enum T : uint8_t {
     Opening,
     Closing,
     SymbolDef,
     Symbol
   } ev_type;
   uint8_t type; // type.
+
+  ParseEvent(T t) : ev_type(t) {}
+  ParseEvent(std::string_view s, T e_t, uint8_t type) : name(s), ev_type(e_t), type(type) {}
 };
 
 struct Language {
@@ -87,6 +90,16 @@ struct Language {
   std::function<void(void *)> destroy;
 };
 
+struct Symbol {
+  uint32_t definition;
+  uint16_t reference_count;
+  uint8_t type;
+  uint8_t len;
+  // first chars of count len, padded to 4 bytes.
+  // then a set of references. 32bit
+  // references later as modifying at end is faster than shifting the name.
+};
+
 struct ParseState {
   static constexpr uint64_t BRANCH_BIT = 1ull << 63;
   static constexpr uint64_t LINES_MASK = ~BRANCH_BIT;
@@ -106,6 +119,12 @@ struct ParseStateBranch : ParseState {
 
 struct ParseStateLeaf : ParseState {
   void *state;
+  uint64_t n;
+  static constexpr uint32_t IS_OPENING = 1ull << 31;
+  static constexpr uint32_t LINE_MASK = ~IS_OPENING;
+  // followed by n number of relative offsets
+  // stored as uint32_t with 1 bit for if it is start or end
+  // and rest as number.
 };
 
 struct TreeCursor {
@@ -117,36 +136,4 @@ struct TreeCursor {
   void next();
   void prev();
 };
-
-struct ScopeNode {
-  static constexpr uint64_t SCOPE_BIT = 1ull << 63;
-  static constexpr uint64_t LINES_MASK = ~SCOPE_BIT;
-  uint64_t header;
-  bool is_scope() const {
-    return header & SCOPE_BIT;
-  }
-  uint64_t lines() const {
-    return header & LINES_MASK;
-  }
-};
-
-struct Symbol {
-  uint32_t definition;
-  uint16_t reference_count;
-  uint8_t type;
-  uint8_t len;
-  // first chars of count len, padded to 4 bytes.
-  // then a set of references. 32bit
-  // references later as modifying at end is faster than shifting the name.
-};
-
-struct alignas(8) Scope : ScopeNode {
-  uint16_t children_count;
-  uint16_t symbol_count;
-  uint8_t type;
-  uint8_t len;
-  uint32_t : 32; // if more stuff is needed use the padding first.
-  // first chars of count len, padded to 8 bytes.
-  // followed by that many number of pointers.
-};
 } // namespace bed::internal::syntax

include/internal/syntax/parser.h

@@ -24,9 +24,6 @@ struct Parser {
   std::pair<ParseState *, ParseState *> split_tree(ParseState *node, uint64_t line);
   ParseState *join_tree(ParseState *a, ParseState *b);
 
-  Scope scope_root;
-  Scope *get_scope(uint64_t);
-
   struct Iterator {
     Parser *p;
     std::optional<vase::Iterator> it;

include/internal/syntax/ruby/parser.h

@@ -10,6 +10,7 @@ struct RubyParser {
   std::string_view line;
   uint32_t i = 0;
   bool heredoc_start_line = false;
+  bool ending = true;
 
   RubyParser(void **v_state, std::string_view line)
       : v_state(v_state), state((RubyState *)*v_state), line(line) {}
@@ -22,7 +23,7 @@ struct RubyParser {
     return state->stack()[state->top - 1];
   }
 
-  char peek(uint32_t offset = 0) {
+  char peek(int32_t offset = 0) {
     uint32_t pos = i + offset;
     return pos < line.size() ? line[pos] : '\0';
   }
@@ -57,7 +58,7 @@ struct RubyParser {
       .brace_level = 1,
       .lit_brace_level = 0,
       .state = RubyState::RubyInternalState::NONE,
-      .flags = 0,
+      .flags = RubyState::RubyInternalState::EXPECTING_EXPRESSION,
       .delim_start = '\0',
       .delim_end = '\0'
     };

include/internal/syntax/ruby/tries.h

@@ -212,10 +212,10 @@ const static std::vector<std::string> base_keywords = {
   "else",
   "rescue",
   "ensure",
-  "when",
 };
 
 const static std::vector<std::string> expecting_keywords = {
+  "when",
   "elsif",
 };
 
@@ -247,13 +247,13 @@ const static std::vector<std::string> operator_keywords = {
   "redo",
   "rescue",
   "retry",
-  "super",
   "self",
   "nil",
   "undef",
 };
 
 const static std::vector<std::string> expecting_operators = {
+  "super",
   "and",
   "return",
   "not",