Major updates

- Improve ruby block detection heuristics
- Add block information storage and block aware movements.
- Other minor fixes

Makefile

@@ -26,7 +26,7 @@ MRUBY_LIBS ?= -L./libs/mruby/build/host/lib -lmruby
 
 CFLAGS_DEBUG :=\
 	-std=c++23 -Wall -Wextra -fno-rtti \
-	-Og -fno-inline -gsplit-dwarf \
+	-Og -fno-inline -gsplit-dwarf -DDEBUG \
 	-g -fno-omit-frame-pointer -ffast-math \
 	-I./include -I./libs/unicode_width
 

include/internal/syntax/decl.h

@@ -79,7 +79,7 @@ struct ParseEvent {
   uint8_t type; // type.
 
   ParseEvent(T t) : ev_type(t) {}
-  ParseEvent(std::string_view s, T e_t, uint8_t type) : name(s), ev_type(e_t), type(type) {}
+  ParseEvent(T e_t, std::string_view s, uint8_t type) : name(s), ev_type(e_t), type(type) {}
 };
 
 struct Language {
@@ -119,12 +119,11 @@ struct ParseStateBranch : ParseState {
 
 struct ParseStateLeaf : ParseState {
   void *state;
-  uint64_t n;
+  uint32_t n;
-  static constexpr uint32_t IS_OPENING = 1ull << 31;
+  uint32_t cap;
-  static constexpr uint32_t LINE_MASK = ~IS_OPENING;
+  uint32_t *blocks;
-  // followed by n number of relative offsets
+  static constexpr uint32_t IS_CLOSING = 1ull << 31;
-  // stored as uint32_t with 1 bit for if it is start or end
+  static constexpr uint32_t LINE_MASK = ~IS_CLOSING;
-  // and rest as number.
 };
 
 struct TreeCursor {

include/internal/syntax/parser.h

@@ -5,8 +5,10 @@
 #include "pch.h"
 
 namespace bed::internal::syntax {
+void dump_events(ParseState *node);
+
 struct Parser {
-  static constexpr uint64_t MAX_CHUNK = 256;
+  static constexpr uint64_t MAX_CHUNK = 512;
 
   ParseState *root;
   Language lang;
@@ -21,6 +23,9 @@ struct Parser {
   void insert(vase::Vase &, uint64_t, uint64_t);
   void modify(vase::Vase &, uint64_t, uint64_t);
 
+  uint64_t next_closing(uint64_t line);
+  uint64_t prev_opening(uint64_t line);
+
   std::pair<ParseState *, ParseState *> split_tree(ParseState *node, uint64_t line);
   ParseState *join_tree(ParseState *a, ParseState *b);
 

include/internal/syntax/ruby/decl.h

@@ -33,6 +33,7 @@ struct alignas(2) RubyState {
       DEF_NAME = 0b10,
       MODULE_NAME = 0b11
     };
+    static constexpr uint8_t NEWLINE = 1 << 5;
     static constexpr uint8_t ALLOW_INTERPOLATION = 1 << 6;
     static constexpr uint8_t EXPECTING_EXPRESSION = 1 << 7;
     uint8_t flags = 0;

include/internal/syntax/ruby/parser.h

@@ -11,6 +11,8 @@ struct RubyParser {
   uint32_t i = 0;
   bool heredoc_start_line = false;
   bool ending = true;
+  bool op_last = false;
+  bool set_op_last = false;
 
   RubyParser(void **v_state, std::string_view line)
       : v_state(v_state), state((RubyState *)*v_state), line(line) {}
@@ -58,7 +60,7 @@ struct RubyParser {
       .brace_level = 1,
       .lit_brace_level = 0,
       .state = RubyState::RubyInternalState::NONE,
-      .flags = RubyState::RubyInternalState::EXPECTING_EXPRESSION,
+      .flags = RubyState::RubyInternalState::EXPECTING_EXPRESSION | RubyState::RubyInternalState::NEWLINE,
       .delim_start = '\0',
       .delim_end = '\0'
     };

include/internal/syntax/ruby/tries.h

@@ -189,7 +189,6 @@ const static std::vector<std::string> methods = {
   "respond_to?",
   "equal?",
   "object_id",
-  "class",
   "singleton_class",
   "clone",
   "freeze",

src/internal/address/address.cc

@@ -19,6 +19,14 @@ Address::Address(std::string &cmd, uint64_t &i) {
     base = Last();
     i++;
     break;
+  case ']':
+    base = Block(Direction::Forward);
+    i++;
+    break;
+  case '[':
+    base = Block(Direction::Backward);
+    i++;
+    break;
   case '\'': {
     i++;
     if (i < cmd.size() && 'a' <= cmd[i] && cmd[i] <= 'z')

src/internal/address/resolve.cc

@@ -19,6 +19,18 @@ uint64_t Address::resolve(BEd &ctx) {
         line = ctx.active->vase.lines();
       } else if constexpr (std::is_same_v<T, Number>) {
         line = addr.i;
+      } else if constexpr (std::is_same_v<T, Block>) {
+        if (addr.dir == Direction::Forward) {
+          if (ctx.active->parser)
+            line = ctx.active->parser->next_closing(ctx.active->line);
+          else
+            line = ctx.active->line + 10;
+        } else {
+          if (ctx.active->parser)
+            line = ctx.active->parser->prev_opening(ctx.active->line);
+          else
+            line = ctx.active->line - 10;
+        }
       } else {
         throw ed_error("Unhandled address given.");
       }

src/internal/commands/commands.cc

@@ -101,6 +101,18 @@ void Command::register_posix(BEd &ctx) {
       }
     }
   );
+  ctx.commands.insert(
+    "debug",
+    Command{
+      .address_mode = Command::AddressMode::None,
+      .suffix = Command::SuffixKind::None,
+      .desc = "",
+      .accept_zero = false,
+      .handle = [](BEd &ctx, std::span<const uint64_t>, std::string_view) {
+        syntax::dump_events(ctx.active->parser->root);
+      }
+    }
+  );
   ctx.commands.insert(
     "E",
     Command{

src/internal/syntax/parser.cc

@@ -1,6 +1,42 @@
 #include "internal/syntax/parser.h"
 
 namespace bed::internal::syntax {
+void dump_events(ParseState *root) {
+  if (!root)
+    return;
+  uint64_t offset = 0;
+  TreeCursor c(root, 0, &offset);
+  uint64_t line_offset = 0;
+  int depth = 0;
+  while (c.leaf) {
+    auto *leaf = c.leaf;
+    for (uint32_t i = 0; i < leaf->n; ++i) {
+      uint32_t block = leaf->blocks[i];
+      uint32_t pos = block & ParseStateLeaf::LINE_MASK;
+      bool closing = block & ParseStateLeaf::IS_CLOSING;
+      if (closing) {
+        if (depth > 0)
+          --depth;
+        else
+          std::cout << "!! UNMATCHED CLOSE !! ";
+      }
+      std::cout << std::string(static_cast<size_t>(depth) * 2, ' ')
+                << (closing ? "}" : "{")
+                << " line " << (line_offset + pos)
+                << '\n';
+      if (!closing)
+        ++depth;
+    }
+    line_offset += leaf->lines();
+    c.next();
+  }
+  if (depth != 0) {
+    std::cout << "!! UNBALANCED: depth = "
+              << depth
+              << " !!\n";
+  }
+}
+
 static void destroy_tree(ParseState *node, Language &lang) {
   if (!node)
     return;
@@ -8,12 +44,14 @@ static void destroy_tree(ParseState *node, Language &lang) {
     auto *branch = (ParseStateBranch *)node;
     destroy_tree(branch->left, lang);
     destroy_tree(branch->right, lang);
-    delete branch;
+    free(branch);
   } else {
     auto *leaf = (ParseStateLeaf *)node;
     if (leaf->state)
       lang.destroy(leaf->state);
-    delete leaf;
+    if (leaf->blocks)
+      free(leaf->blocks);
+    free(leaf);
   }
 }
 
@@ -22,9 +60,8 @@ static ParseState *make_branch(ParseState *left, ParseState *right) {
     return right;
   if (!right)
     return left;
-  auto *branch = new ParseStateBranch;
+  auto *branch = (ParseStateBranch *)malloc(sizeof(ParseStateBranch));
-  branch->header =
+  branch->header = ParseState::BRANCH_BIT + left->lines() + right->lines();
-    ParseState::BRANCH_BIT + left->lines() + right->lines();
   branch->left = left;
   branch->right = right;
   return branch;
@@ -57,32 +94,22 @@ void Parser::reset(vase::Vase &vase, uint64_t lines, Language lang_) {
   if (lines == 0)
     return;
   lang = std::move(lang_);
-  vase::Iterator it = vase.iterate(0, Direction::Forward);
-  it.next();
   std::vector<ParseStateLeaf *> leaves;
-  std::vector<Token> tokens;
-  std::vector<ParseEvent> events;
   leaves.reserve((lines + MAX_CHUNK - 1) / MAX_CHUNK);
-  void *state = lang.none_state();
+  uint64_t consumed = 0;
-  uint32_t consumed = 0;
   while (consumed < lines) {
-    ParseStateLeaf *leaf = new ParseStateLeaf;
+    auto *leaf = (ParseStateLeaf *)malloc(sizeof(ParseStateLeaf));
-    leaf->header = 0;
+    leaf->state = nullptr;
-    leaf->state = lang.copy(state);
+    leaf->blocks = nullptr;
-    uint32_t chunk_lines = 0;
+    leaf->n = 0;
-    while (chunk_lines < MAX_CHUNK && consumed < lines) {
+    leaf->cap = 0;
-      tokens.clear();
+    uint64_t chunk = std::min(MAX_CHUNK, lines - consumed);
-      lang.parse(&state, it.line, consumed == 0, &tokens, &events);
+    leaf->header = chunk;
-      ++chunk_lines;
+    consumed += chunk;
-      ++consumed;
-      if (!it.next() && consumed < lines)
-        break;
-    }
-    leaf->header = chunk_lines;
     leaves.push_back(leaf);
   }
-  lang.destroy(state);
   root = build_tree(leaves, 0, leaves.size());
+  modify(vase, 0, lines);
 }
 
 std::pair<ParseState *, ParseState *> Parser::split_tree(ParseState *node, uint64_t line) {
@@ -98,26 +125,30 @@ std::pair<ParseState *, ParseState *> Parser::split_tree(ParseState *node, uint6
     if (line < left_lines) {
       auto [a, b] = split_tree(branch->left, line);
       ParseState *right = join_tree(b, branch->right);
-      delete branch;
+      free(branch);
       return {a, right};
     }
     if (line == left_lines) {
       ParseState *left = branch->left;
       ParseState *right = branch->right;
-      delete branch;
+      free(branch);
       return {left, right};
     }
     auto [a, b] = split_tree(branch->right, line - left_lines);
     ParseState *left = join_tree(branch->left, a);
-    delete branch;
+    free(branch);
     return {left, b};
   } else {
     auto *leaf = (ParseStateLeaf *)node;
     uint64_t lines = leaf->lines();
-    auto *right = new ParseStateLeaf;
+    auto *right = (ParseStateLeaf *)malloc(sizeof(ParseStateLeaf));
     right->header = lines - line;
     right->state = nullptr;
+    right->blocks = nullptr;
+    right->n = 0;
+    right->cap = 0;
     leaf->header = line;
+    leaf->n = 0;
     return {leaf, right};
   }
 }
@@ -144,14 +175,14 @@ void Parser::insert(vase::Vase &vase, uint64_t start, uint64_t count) {
   leaves.reserve((count + MAX_CHUNK - 1) / MAX_CHUNK);
   uint64_t consumed = 0;
   while (consumed < count) {
-    auto *leaf = new ParseStateLeaf;
+    auto *leaf = (ParseStateLeaf *)malloc(sizeof(ParseStateLeaf));
     leaf->state = nullptr;
-    uint64_t chunk = 0;
+    leaf->blocks = nullptr;
-    while (chunk < MAX_CHUNK && consumed < count) {
+    leaf->n = 0;
-      ++chunk;
+    leaf->cap = 0;
-      ++consumed;
+    uint64_t chunk = std::min(MAX_CHUNK, count - consumed);
-    }
     leaf->header = chunk;
+    consumed += chunk;
     leaves.push_back(leaf);
   }
   ParseState *subtree = build_tree(leaves, 0, leaves.size());
@@ -188,13 +219,17 @@ void Parser::modify(vase::Vase &vase, uint64_t target, uint64_t count) {
     }
   }
   vase::Iterator it = vase.iterate(at, Direction::Forward);
+  uint64_t chunk_start = at;
   uint64_t next_boundary = at + c.leaf->lines();
+  c.leaf->n = 0;
   while (true) {
     it.next();
     if (at == next_boundary) {
       c.next();
       if (!c.leaf)
         break;
+      c.leaf->n = 0;
+      chunk_start = at;
       next_boundary += c.leaf->lines();
       if (at >= target + count
           && c.leaf->state != nullptr
@@ -205,12 +240,88 @@ void Parser::modify(vase::Vase &vase, uint64_t target, uint64_t count) {
       c.leaf->state = lang.copy(state);
     }
     tokens.clear();
+    events.clear();
     lang.parse(&state, it.line, at == 0, &tokens, &events);
+    for (const auto &ev : events) {
+      const auto t = uint8_t(ev.ev_type);
+      if (t > ParseEvent::Closing)
+        continue;
+      if (c.leaf->n == c.leaf->cap) {
+        uint32_t cap = c.leaf->cap ? c.leaf->cap * 2 : 8;
+        c.leaf->blocks = (uint32_t *)realloc(c.leaf->blocks, cap * sizeof(uint32_t));
+        c.leaf->cap = cap;
+      }
+      c.leaf->blocks[c.leaf->n++] = t << 31 | (at - chunk_start);
+    }
     at++;
   }
   lang.destroy(state);
 }
 
+uint64_t Parser::next_closing(uint64_t line) {
+  if (!root)
+    return line + 10;
+  uint64_t relative;
+  TreeCursor c(root, line, &relative);
+  uint64_t at = line - relative;
+  uint32_t from = (uint32_t)relative;
+  bool first = true;
+  int depth = 0;
+  while (c.leaf) {
+    auto *leaf = c.leaf;
+    for (uint32_t i = 0; i < leaf->n; ++i) {
+      uint32_t block = leaf->blocks[i];
+      uint32_t pos = block & ParseStateLeaf::LINE_MASK;
+      if (first && pos <= from)
+        continue;
+      if (block & ParseStateLeaf::IS_CLOSING) {
+        if (depth == 0)
+          return at + pos;
+        --depth;
+      } else {
+        ++depth;
+      }
+    }
+    at += leaf->lines();
+    first = false;
+    c.next();
+  }
+  return line + 10;
+}
+
+uint64_t Parser::prev_opening(uint64_t line) {
+  if (!root)
+    return line >= 10 ? line - 10 : 0;
+  uint64_t relative;
+  TreeCursor c(root, line, &relative);
+  uint64_t at = line - relative;
+  uint32_t from = (uint32_t)relative;
+  bool first = true;
+  int depth = 0;
+  while (c.leaf) {
+    auto *leaf = c.leaf;
+    for (uint32_t i = leaf->n; i-- > 0;) {
+      uint32_t block = leaf->blocks[i];
+      uint32_t pos = block & ParseStateLeaf::LINE_MASK;
+      if (first && pos >= from)
+        continue;
+      if (!(block & ParseStateLeaf::IS_CLOSING)) {
+        if (depth == 0)
+          return at + pos;
+        --depth;
+      } else {
+        ++depth;
+      }
+    }
+    c.prev();
+    if (!c.leaf)
+      break;
+    at -= c.leaf->lines();
+    first = false;
+  }
+  return line >= 10 ? line - 10 : 0;
+}
+
 std::optional<Parser::Iterator> Parser::get_hl(vase::Vase &vase, uint64_t target) {
   if (!root)
     return std::nullopt;
@@ -243,6 +354,7 @@ Parser::Iterator::Iterator(uint64_t target, Parser *p, vase::Vase &vase) : p(p)
   while (at < target) {
     it->next();
     tokens.clear();
+    events.clear();
     p->lang.parse(&state, it->line, at == 0, &tokens, &events);
     at++;
   }
@@ -277,6 +389,7 @@ Parser::Iterator &Parser::Iterator::operator=(Iterator &&other) {
 void Parser::Iterator::next() {
   it->next();
   tokens.clear();
+  events.clear();
   p->lang.parse(&state, it->line, at++ == 0, &tokens, &events);
 }
 } // namespace bed::internal::syntax

src/internal/syntax/ruby/parse.cc

@@ -250,15 +250,18 @@ bool handle_comment(RubyParser &p, std::vector<Token> *tokens, bool first_line)
   return false;
 }
 
-void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseEvent> *events) {
+bool handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseEvent> *events) {
   static const RubyTries tries = RubyTries();
+  if (p.peek() == ' ' || p.peek() == '\t') {
     while (p.peek() == ' ' || p.peek() == '\t')
       p.advance();
+    return true;
+  }
   if (p.current().flags & RubyState::RubyInternalState::NAME_MASK) {
     if (p.peek() == '\0')
-      return;
+      return false;
     if (p.peek() == '\\' && p.peek(1) == '\0')
-      return;
+      return false;
     switch (p.current().flags & RubyState::RubyInternalState::NAME_MASK) {
     case RubyState::RubyInternalState::CLASS_NAME: {
       if (p.peek() == '<' && p.peek(1) == '<') {
@@ -266,7 +269,7 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
         events->push_back(ParseEvent::Opening);
         p.current().flags = (p.current().flags & ~RubyState::RubyInternalState::NAME_MASK);
         p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
-        return;
+        return false;
       }
       uint32_t j = 0;
       if (identifier_start_char(p.peek(j))) {
@@ -279,12 +282,12 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
           p.advance();
         if (p.peek() == ':' && p.peek(1) == ':') {
           p.advance(2);
-          return;
+          return false;
         }
       }
       events->push_back(ParseEvent::Opening);
       p.current().flags = (p.current().flags & ~RubyState::RubyInternalState::NAME_MASK);
-      return;
+      return false;
     }
     case RubyState::RubyInternalState::DEF_NAME: {
       uint32_t j = 0;
@@ -305,12 +308,12 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
           p.advance();
         if (p.peek() == '.') {
           p.advance();
-          return;
+          return false;
         }
       }
       events->push_back(ParseEvent::Opening);
       p.current().flags = (p.current().flags & ~RubyState::RubyInternalState::NAME_MASK);
-      return;
+      return false;
     }
     case RubyState::RubyInternalState::MODULE_NAME: {
       uint32_t j = 0;
@@ -324,12 +327,12 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
           p.advance();
         if (p.peek() == ':' && p.peek(1) == ':') {
           p.advance(2);
-          return;
+          return false;
         }
       }
       events->push_back(ParseEvent::Opening);
       p.current().flags = (p.current().flags & ~RubyState::RubyInternalState::NAME_MASK);
-      return;
+      return false;
     }
     }
   }
@@ -342,7 +345,7 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
       j++;
     tokens->push_back({p.i, p.i + j, Token::Operator});
     if (p.i + j >= p.len())
-      return;
+      return false;
     std::string delim;
     bool interpolation = true;
     uint32_t s = p.i + j;
@@ -360,7 +363,7 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
       }
     }
     if (p.peek() == '\0')
-      return;
+      return false;
     p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
     if (!delim.empty()) {
       tokens->push_back({s, p.i + j, Token::Annotation});
@@ -374,7 +377,7 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
       p.heredoc_start_line = true;
     }
     p.advance(j);
-    return;
+    return false;
   }
   if (p.peek() == '/' && p.current().flags & RubyState::RubyInternalState::EXPECTING_EXPRESSION) {
     tokens->push_back({p.i, p.i + 1, Token::Regexp});
@@ -384,7 +387,7 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
     p.current().delim_start = '/';
     p.current().delim_end = '/';
     p.advance();
-    return;
+    return false;
   }
   switch (p.peek()) {
   case '.': {
@@ -397,7 +400,7 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
         p.advance();
     }
     tokens->push_back({start, p.i, Token::Operator});
-    return;
+    return false;
   }
   case ':': {
     p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
@@ -406,17 +409,17 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
     if (p.i >= p.len()) {
       tokens->push_back({start, p.i, Token::Operator});
       p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
-      return;
+      return false;
     }
     if (p.peek() == ':') {
       p.advance();
       tokens->push_back({start, p.i, Token::Operator});
-      return;
+      return false;
     }
     if (p.peek() == '\'' || p.peek() == '"') {
       tokens->push_back({start, p.i, Token::Label});
       p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
-      return;
+      return false;
     }
     if (p.peek() == '$' || p.peek() == '@') {
       if (p.peek_str(2) == "@@")
@@ -426,13 +429,13 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
       while (identifier_char(p.peek()))
         p.advance();
       tokens->push_back({start, p.i, Token::Label});
-      return;
+      return false;
     }
     uint32_t op_len = tries.operator_trie.longest_match(p.peek_str(p.len() - p.i));
     if (op_len > 0) {
       tokens->push_back({start, p.i + op_len, Token::Label});
       p.advance(op_len);
-      return;
+      return false;
     }
     if (identifier_start_char(p.peek())) {
       p.advance();
@@ -441,34 +444,34 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
       if (p.peek() == '!' || p.peek() == '?')
         p.advance();
       tokens->push_back({start, p.i, Token::Label});
-      return;
+      return false;
     }
     tokens->push_back({start, p.i, Token::Operator});
-    return;
+    return false;
   }
   case '@': {
     p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
     uint32_t start = p.i;
     p.advance();
     if (p.i >= p.len())
-      return;
+      return false;
     if (p.peek() == '@')
       p.advance();
     if (identifier_start_char(p.peek()))
       p.advance();
     else
-      return;
+      return false;
     while (identifier_char(p.peek()))
       p.advance();
     tokens->push_back({start, p.i, Token::VariableInstance});
-    return;
+    return false;
   }
   case '$': {
     p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
     uint32_t start = p.i;
     p.advance();
     if (p.i >= p.len())
-      return;
+      return false;
     if (identifier_start_char(p.peek())) {
       p.advance();
       while (identifier_char(p.peek()))
@@ -504,11 +507,11 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
         p.advance();
         break;
       default:
-        return;
+        return false;
       }
     }
     tokens->push_back({start, p.i, Token::VariableGlobal});
-    return;
+    return false;
   }
   case '?': {
     p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
@@ -524,7 +527,7 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
         if (is_hex(p.peek()))
           p.advance();
         tokens->push_back({start, p.i, Token::Char});
-        return;
+        return false;
       } else if (p.peek() == 'u') {
         p.advance();
         if (p.peek() == '{') {
@@ -544,7 +547,7 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
             p.advance();
         }
         tokens->push_back({start, p.i, Token::Char});
-        return;
+        return false;
       } else if ('0' <= p.peek() && p.peek() <= '7') {
         p.advance();
         if ('0' <= p.peek() && p.peek() <= '7')
@@ -552,7 +555,7 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
         if ('0' <= p.peek() && p.peek() <= '7')
           p.advance();
         tokens->push_back({start, p.i, Token::Char});
-        return;
+        return false;
       } else if (p.peek() == 'c') {
         p.advance();
         if (p.peek() != '\\')
@@ -560,7 +563,7 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
         else
           goto combination;
         tokens->push_back({start, p.i, Token::Char});
-        return;
+        return false;
       } else if (p.peek() == 'M' || p.peek() == 'C') {
         p.advance();
         if (p.peek() == '-') {
@@ -571,7 +574,7 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
             goto combination;
         }
         tokens->push_back({start, p.i, Token::Char});
-        return;
+        return false;
       } else if (p.peek() == 'N') {
         p.advance();
         if (p.peek() == '{') {
@@ -582,20 +585,20 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
             p.advance();
         }
         tokens->push_back({start, p.i, Token::Char});
-        return;
+        return false;
       } else {
         p.advance();
         tokens->push_back({start, p.i, Token::Char});
-        return;
+        return false;
       }
     } else if (p.peek() != '\0' && p.peek() != ' ' && p.peek() != '\t') {
       p.advance();
       tokens->push_back({start, p.i, Token::Char});
-      return;
+      return false;
     } else {
       p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
       tokens->push_back({start, p.i, Token::Operator});
-      return;
+      return false;
     }
   }
   case '{': {
@@ -605,7 +608,7 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
     tokens->push_back({p.i, p.i + 1, (Token::Kind)brace_color});
     p.current().brace_level++;
     p.advance();
-    return;
+    return false;
   }
   case '}': {
     p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
@@ -618,7 +621,7 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
       tokens->push_back({p.i, p.i + 1, (Token::Kind)brace_color});
     }
     p.advance();
-    return;
+    return false;
   }
   case '(': {
     p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
@@ -627,7 +630,7 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
     tokens->push_back({p.i, p.i + 1, (Token::Kind)brace_color});
     p.current().brace_level++;
     p.advance();
-    return;
+    return false;
   }
   case ')': {
     p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
@@ -636,7 +639,7 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
       (uint8_t)Token::Brace1 + (p.current().brace_level % 5);
     tokens->push_back({p.i, p.i + 1, (Token::Kind)brace_color});
     p.advance();
-    return;
+    return false;
   }
   case '[': {
     p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
@@ -645,7 +648,7 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
     tokens->push_back({p.i, p.i + 1, (Token::Kind)brace_color});
     p.current().brace_level++;
     p.advance();
-    return;
+    return false;
   }
   case ']': {
     p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
@@ -654,7 +657,7 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
       (uint8_t)Token::Brace1 + (p.current().brace_level % 5);
     tokens->push_back({p.i, p.i + 1, (Token::Kind)brace_color});
     p.advance();
-    return;
+    return false;
   }
   case '\'': {
     p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
@@ -664,7 +667,7 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
     p.current().delim_end = '\'';
     p.current().flags &= ~RubyState::RubyInternalState::ALLOW_INTERPOLATION;
     p.advance();
-    return;
+    return false;
   }
   case '"': {
     p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
@@ -674,7 +677,7 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
     p.current().delim_end = '"';
     p.current().flags |= RubyState::RubyInternalState::ALLOW_INTERPOLATION;
     p.advance();
-    return;
+    return false;
   }
   case '`': {
     p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
@@ -684,14 +687,14 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
     p.current().delim_end = '`';
     p.current().flags |= RubyState::RubyInternalState::ALLOW_INTERPOLATION;
     p.advance();
-    return;
+    return false;
   }
   case '%': {
     if (p.i + 1 >= p.len()) {
       tokens->push_back({p.i, p.i + 1, Token::Operator});
       p.advance();
       p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
-      return;
+      return false;
     }
     p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
     char type = p.peek(1);
@@ -729,14 +732,14 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
       tokens->push_back({p.i, p.i + 1, Token::Operator});
       p.advance(prefix_len);
       p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
-      return;
+      return false;
     }
     delim_start = p.peek(prefix_len);
     if (identifier_char(delim_start) || delim_start == ' ') {
       tokens->push_back({p.i, p.i + 1, Token::Operator});
       p.advance(prefix_len);
       p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
-      return;
+      return false;
     }
     switch (delim_start) {
     case '(':
@@ -765,22 +768,23 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
       p.current().flags |= RubyState::RubyInternalState::ALLOW_INTERPOLATION;
     p.current().lit_brace_level = 1;
     p.advance(prefix_len + 1);
-    return;
+    return false;
   }
   case ';':
     p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+    p.current().flags |= RubyState::RubyInternalState::NEWLINE;
     p.advance();
-    return;
+    return true;
   case '\\':
     if (p.peek(1) == '\0')
       p.ending = false;
     p.advance();
-    return;
+    return false;
   case '\0':
     if (p.ending)
       p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
     p.advance();
-    return;
+    return false;
   default:
     if ('0' <= p.peek() && p.peek() <= '9') {
       p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
@@ -857,7 +861,7 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
         }
       }
       tokens->push_back({start, p.i, Token::Number});
-      return;
+      return false;
     } else if (identifier_start_char(p.peek())) {
       uint32_t j = 1;
       while (identifier_char(p.peek(j)))
@@ -868,88 +872,88 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
         p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
         tokens->push_back({p.i, p.i + j, Token::Keyword});
         p.advance(j);
-        return;
+        return false;
       } else if (j == tries.expecting_keywords_trie.longest_match(p.peek_str(j))) {
         p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
         tokens->push_back({p.i, p.i + j, Token::Keyword});
         p.advance(j);
-        return;
+        return false;
       } else if (j == tries.operator_keywords_trie.longest_match(p.peek_str(j))) {
         p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
         tokens->push_back({p.i, p.i + j, Token::KeywordOperator});
         p.advance(j);
-        return;
+        return false;
       } else if (j == tries.expecting_operators_trie.longest_match(p.peek_str(j))) {
         p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
         tokens->push_back({p.i, p.i + j, Token::KeywordOperator});
         p.advance(j);
-        return;
+        return false;
       } else if (j == tries.types_trie.longest_match(p.peek_str(j))) {
         p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
         tokens->push_back({p.i, p.i + j, Token::Type});
         p.advance(j);
-        return;
+        return false;
       } else if (j == tries.methods_trie.longest_match(p.peek_str(j))) {
         p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
         tokens->push_back({p.i, p.i + j, Token::Function});
         p.advance(j);
-        return;
+        return false;
       } else if (j == tries.expecting_end_keywords_trie.longest_match(p.peek_str(j))) {
         events->push_back(ParseEvent::Opening);
         p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
         tokens->push_back({p.i, p.i + j, Token::Keyword});
         p.advance(j);
-        return;
+        return false;
       } else if (j == tries.conditional_keywords_trie.longest_match(p.peek_str(j))) {
-        if (p.current().flags & RubyState::RubyInternalState::EXPECTING_EXPRESSION)
+        if (p.current().flags & RubyState::RubyInternalState::NEWLINE || p.op_last)
           events->push_back(ParseEvent::Opening);
         p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
         tokens->push_back({p.i, p.i + j, Token::Keyword});
         p.advance(j);
-        return;
+        return false;
       } else if (j == tries.end_keywords_trie.longest_match(p.peek_str(j))) {
         p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
         events->push_back(ParseEvent::Opening);
         tokens->push_back({p.i, p.i + j, Token::Keyword});
         p.advance(j);
-        return;
+        return false;
       } else if (j == tries.builtins_trie.longest_match(p.peek_str(j))) {
         p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
         tokens->push_back({p.i, p.i + j, Token::Constant});
         p.advance(j);
-        return;
+        return false;
       } else if (j == tries.errors_trie.longest_match(p.peek_str(j))) {
         p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
         tokens->push_back({p.i, p.i + j, Token::Error});
         p.advance(j);
-        return;
+        return false;
       } else if ('A' <= p.peek() && p.peek() <= 'Z' && !(p.peek(j) == '!' || p.peek(j) == '?')) {
         p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
         if (j >= 5 && p.peek_str(j).substr(j - 5) == "Error") {
           tokens->push_back({p.i, p.i + j, Token::Error});
           p.advance(j);
-          return;
+          return false;
         }
         tokens->push_back({p.i, p.i + j, Token::Constant});
         p.advance(j);
-        return;
+        return false;
       } else {
         p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
         if (j == 4 && p.peek_str(4) == "true") {
           tokens->push_back({p.i, p.i + j, Token::True});
           p.advance(4);
-          return;
+          return false;
         }
         if (j == 5 && p.peek_str(5) == "false") {
           tokens->push_back({p.i, p.i + j, Token::False});
           p.advance(5);
-          return;
+          return false;
         }
         if (j == 3 && p.peek_str(3) == "end") {
           events->push_back(ParseEvent::Closing);
           tokens->push_back({p.i, p.i + j, Token::Keyword});
           p.advance(3);
-          return;
+          return false;
         }
         if (j == 5 && p.peek_str(5) == "class") {
           tokens->push_back({p.i, p.i + j, Token::Keyword});
@@ -957,7 +961,7 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
           p.current().flags =
             (p.current().flags & ~RubyState::RubyInternalState::NAME_MASK)
             | RubyState::RubyInternalState::CLASS_NAME;
-          return;
+          return false;
         }
         if (j == 6 && p.peek_str(6) == "module") {
           tokens->push_back({p.i, p.i + j, Token::Keyword});
@@ -965,7 +969,7 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
           p.current().flags =
             (p.current().flags & ~RubyState::RubyInternalState::NAME_MASK)
             | RubyState::RubyInternalState::MODULE_NAME;
-          return;
+          return false;
         }
         if (j == 3 && p.peek_str(3) == "def") {
           tokens->push_back({p.i, p.i + j, Token::Keyword});
@@ -973,32 +977,32 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
           p.current().flags =
             (p.current().flags & ~RubyState::RubyInternalState::NAME_MASK)
             | RubyState::RubyInternalState::DEF_NAME;
-          return;
+          return false;
         }
         uint32_t start = p.i;
         p.advance(j);
         if (p.peek() == ':') {
           p.advance();
           tokens->push_back({start, p.i, Token::Label});
-          return;
+          return false;
         } else if (p.peek(-1) == '!' || p.peek(-1) == '?') {
           p.advance();
           tokens->push_back({start, p.i, Token::Function});
-          return;
+          return false;
         } else {
           uint32_t j = 0;
           if (p.peek(j) == '(' || p.peek(j) == '{') {
             tokens->push_back({start, p.i, Token::Function});
-            return;
+            return false;
           } else if (p.peek(j) == ' ' || p.peek(j) == '\t') {
             j++;
           } else {
-            return;
+            return false;
           }
           while (p.peek(j) == ' ' || p.peek(j) == '\t')
             j++;
           if (p.i + j >= p.len())
-            return;
+            return false;
           if (
             p.peek(j) == '-'
             || p.peek(j) == '&'
@@ -1006,7 +1010,7 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
             || p.peek(j) == ':'
           ) {
             if (p.peek(j + 1) == ' ' || p.peek(j + 1) == '>')
-              return;
+              return false;
           } else if (
             p.peek(j) == ']'
             || p.peek(j) == '}'
@@ -1024,10 +1028,10 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
             || p.peek(j) == '<'
             || p.peek(j) == '>'
           ) {
-            return;
+            return false;
           }
           tokens->push_back({start, p.i, Token::Function});
-          return;
+          return false;
         }
       }
     } else {
@@ -1036,11 +1040,13 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
         tokens->push_back({p.i, p.i + op_len, Token::Operator});
         p.advance(op_len);
         p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+        p.set_op_last = true;
       } else {
         p.advance(utf8_codepoint_width(p.peek()));
       }
     }
   }
+  return false;
 }
 
 void ruby_parse(
@@ -1052,6 +1058,8 @@ void ruby_parse(
 ) {
   RubyParser p(v_state, line);
   while (p.i <= p.len()) {
+    p.op_last = p.set_op_last;
+    p.set_op_last = false;
     if (p.current().state == RubyState::RubyInternalState::END)
       return;
     if (p.current().state == RubyState::RubyInternalState::COMMENT) {
@@ -1085,8 +1093,11 @@ void ruby_parse(
       return;
     if (handle_comment(p, tokens, fl))
       return;
-    handle_syntax(p, tokens, events);
+    if (!handle_syntax(p, tokens, events))
+      p.current().flags &= ~RubyState::RubyInternalState::NEWLINE;
     continue;
   }
+  if (p.ending)
+    p.current().flags |= RubyState::RubyInternalState::NEWLINE;
 }
 } // namespace bed::internal::syntax::ruby

src/internal/syntax/ruby/ruby.cc

@@ -14,7 +14,7 @@ Language lang_ruby() {
         .brace_level = 1,
         .lit_brace_level = 0,
         .state = RubyState::RubyInternalState::NONE,
-        .flags = RubyState::RubyInternalState::EXPECTING_EXPRESSION,
+        .flags = RubyState::RubyInternalState::EXPECTING_EXPRESSION | RubyState::RubyInternalState::NEWLINE,
         .delim_start = '\0',
         .delim_end = '\0'
       };

src/main.cc

@@ -10,9 +10,12 @@ int main(int argc, char *argv[]) {
     if (e.code)
       std::cout << "Fatal error: " << e.what() << std::endl;
     return e.code;
-  } catch (...) {
+  }
+#ifndef DEBUG
+  catch (...) {
     std::cout << "Unexpected error." << std::endl;
     return 1;
   }
+#endif
   return 0;
 }