Implement syntax highlighting

- Create a generic incremental syntax highlighting system.
- Implement a ruby syntax highlighter for it.

src/internal/syntax/parser.cc

@@ -0,0 +1,282 @@
+#include "internal/syntax/parser.h"
+
+namespace bed::internal::syntax {
+static void destroy_tree(ParseState *node, Language &lang) {
+  if (!node)
+    return;
+  if (node->is_branch()) {
+    auto *branch = (ParseStateBranch *)node;
+    destroy_tree(branch->left, lang);
+    destroy_tree(branch->right, lang);
+    delete branch;
+  } else {
+    auto *leaf = (ParseStateLeaf *)node;
+    if (leaf->state)
+      lang.destroy(leaf->state);
+    delete leaf;
+  }
+}
+
+static ParseState *make_branch(ParseState *left, ParseState *right) {
+  if (!left)
+    return right;
+  if (!right)
+    return left;
+  auto *branch = new ParseStateBranch;
+  branch->header =
+    ParseState::BRANCH_BIT + left->lines() + right->lines();
+  branch->left = left;
+  branch->right = right;
+  return branch;
+}
+
+static ParseState *build_tree(std::vector<ParseStateLeaf *> &leaves, size_t begin, size_t end) {
+  const size_t count = end - begin;
+  if (count == 0)
+    return nullptr;
+  if (count == 1)
+    return leaves[begin];
+  const size_t mid = begin + count / 2;
+  ParseState *left = build_tree(leaves, begin, mid);
+  ParseState *right = build_tree(leaves, mid, end);
+  return make_branch(left, right);
+}
+
+Parser::Parser(vase::Vase &vase, uint64_t lines, Language &lang)
+    : root(nullptr), lang(lang) {
+  reset(vase, lines, lang);
+}
+
+Parser::~Parser() {
+  destroy_tree(root, lang);
+}
+
+void Parser::reset(vase::Vase &vase, uint64_t lines, Language &lang_) {
+  destroy_tree(root, lang);
+  root = nullptr;
+  if (lines == 0)
+    return;
+  lang = lang_;
+  vase::Iterator it = vase.iterate(0, Direction::Forward);
+  it.next();
+  std::vector<ParseStateLeaf *> leaves;
+  std::vector<Token> tokens;
+  leaves.reserve((lines + 63) / 64);
+  void *state = lang.none_state();
+  uint32_t consumed = 0;
+  while (consumed < lines) {
+    ParseStateLeaf *leaf = new ParseStateLeaf;
+    leaf->header = 0;
+    leaf->state = lang.copy(state);
+    uint32_t chunk_lines = 0;
+    while (chunk_lines < 64 && consumed < lines) {
+      tokens.clear();
+      lang.parse(&state, it.line, consumed == 0, &tokens);
+      ++chunk_lines;
+      ++consumed;
+      if (!it.next() && consumed < lines)
+        break;
+    }
+    leaf->header = chunk_lines;
+    leaves.push_back(leaf);
+  }
+  lang.destroy(state);
+  root = build_tree(leaves, 0, leaves.size());
+}
+
+std::pair<ParseState *, ParseState *> Parser::split_tree(ParseState *node, uint64_t line) {
+  if (!node)
+    return {nullptr, nullptr};
+  if (line == 0)
+    return {nullptr, node};
+  if (line >= node->lines())
+    return {node, nullptr};
+  if (node->is_branch()) {
+    auto *branch = (ParseStateBranch *)node;
+    uint64_t left_lines = branch->left->lines();
+    if (line < left_lines) {
+      auto [a, b] = split_tree(branch->left, line);
+      ParseState *right = join_tree(b, branch->right);
+      delete branch;
+      return {a, right};
+    }
+    if (line == left_lines) {
+      ParseState *left = branch->left;
+      ParseState *right = branch->right;
+      delete branch;
+      return {left, right};
+    }
+    auto [a, b] = split_tree(branch->right, line - left_lines);
+    ParseState *left = join_tree(branch->left, a);
+    delete branch;
+    return {left, b};
+  } else {
+    auto *leaf = (ParseStateLeaf *)node;
+    uint64_t lines = leaf->lines();
+    auto *right = new ParseStateLeaf;
+    right->header = lines - line;
+    right->state = nullptr;
+    leaf->header = line;
+    return {leaf, right};
+  }
+}
+
+ParseState *Parser::join_tree(ParseState *a, ParseState *b) {
+  // TODO: balance
+  return make_branch(a, b);
+}
+
+void Parser::erase(vase::Vase &vase, uint64_t start, uint64_t count) {
+  if (count == 0 || !root)
+    return;
+  auto [a, remaining] = split_tree(root, start);
+  auto [waste, b] = split_tree(remaining, count);
+  destroy_tree(waste, lang);
+  root = join_tree(a, b);
+  modify(vase, start, 1);
+}
+
+void Parser::insert(vase::Vase &vase, uint64_t start, uint64_t count) {
+  if (count == 0)
+    return;
+  std::vector<ParseStateLeaf *> leaves;
+  leaves.reserve((count + 63) / 64);
+  uint64_t consumed = 0;
+  while (consumed < count) {
+    auto *leaf = new ParseStateLeaf;
+    leaf->state = nullptr;
+    uint64_t chunk = 0;
+    while (chunk < 64 && consumed < count) {
+      ++chunk;
+      ++consumed;
+      if (consumed < count)
+        break;
+    }
+    leaf->header = chunk;
+    leaves.push_back(leaf);
+  }
+  ParseState *subtree = build_tree(leaves, 0, leaves.size());
+  auto [left, right] = split_tree(root, start);
+  root = join_tree(join_tree(left, subtree), right);
+  modify(vase, start, count);
+}
+
+void Parser::modify(vase::Vase &vase, uint64_t target, uint64_t count) {
+  if (count == 0 || !root)
+    return;
+  std::vector<Token> tokens;
+  uint64_t offset;
+  TreeCursor c = TreeCursor(root, target, &offset);
+  uint64_t at = target - offset;
+  void *state = nullptr;
+  if (c.leaf->state) {
+    state = lang.copy(c.leaf->state);
+  } else {
+    while (!c.leaf->state) {
+      c.prev();
+      if (!c.leaf) {
+        at = 0;
+        break;
+      }
+      at -= c.leaf->lines();
+    }
+    if (c.leaf) {
+      state = lang.copy(c.leaf->state);
+    } else {
+      state = lang.none_state();
+      c = TreeCursor(root, 0, &offset);
+    }
+  }
+  vase::Iterator it = vase.iterate(at, Direction::Forward);
+  uint64_t next_boundary = at + c.leaf->lines();
+  while (true) {
+    it.next();
+    if (at == next_boundary) {
+      c.next();
+      if (!c.leaf)
+        break;
+      next_boundary += c.leaf->lines();
+      if (at >= target + count
+          && c.leaf->state != nullptr
+          && lang.equal(state, c.leaf->state))
+        break;
+      if (c.leaf->state)
+        lang.destroy(c.leaf->state);
+      c.leaf->state = lang.copy(state);
+    }
+    tokens.clear();
+    lang.parse(&state, it.line, at == 0, &tokens);
+    at++;
+  }
+  lang.destroy(state);
+}
+
+std::optional<Parser::Iterator> Parser::get_hl(vase::Vase &vase, uint64_t target) {
+  if (!root)
+    return std::nullopt;
+  return Parser::Iterator(target, this, vase);
+}
+
+Parser::Iterator::Iterator(uint64_t target, Parser *p, vase::Vase &vase) : p(p) {
+  uint64_t offset;
+  TreeCursor c = TreeCursor(p->root, target, &offset);
+  at = target - offset;
+  if (c.leaf->state) {
+    state = p->lang.copy(c.leaf->state);
+  } else {
+    while (!c.leaf->state) {
+      c.prev();
+      if (!c.leaf) {
+        at = 0;
+        break;
+      }
+      at -= c.leaf->lines();
+    }
+    if (c.leaf) {
+      state = p->lang.copy(c.leaf->state);
+    } else {
+      state = p->lang.none_state();
+      c = TreeCursor(p->root, 0, &offset);
+    }
+  }
+  it = vase.iterate(at, Direction::Forward);
+  while (at < target) {
+    it->next();
+    tokens.clear();
+    p->lang.parse(&state, it->line, at == 0, &tokens);
+    at++;
+  }
+}
+
+Parser::Iterator::~Iterator() {
+  if (state)
+    p->lang.destroy(state);
+}
+
+Parser::Iterator::Iterator(Iterator &&other)
+    : p(other.p),
+      it(std::move(other.it)),
+      state(other.state),
+      tokens(std::move(other.tokens)) {
+  other.state = nullptr;
+}
+
+Parser::Iterator &Parser::Iterator::operator=(Iterator &&other) {
+  if (this == &other)
+    return *this;
+  if (state)
+    p->lang.destroy(state);
+  p = other.p;
+  it = std::move(other.it);
+  state = other.state;
+  tokens = std::move(other.tokens);
+  other.state = nullptr;
+  return *this;
+}
+
+void Parser::Iterator::next() {
+  it->next();
+  tokens.clear();
+  p->lang.parse(&state, it->line, at == 0, &tokens);
+}
+} // namespace bed::internal::syntax

src/internal/syntax/ruby/parse.cc

@@ -0,0 +1,950 @@
+#include "internal/syntax/ruby/parser.h"
+
+namespace bed::internal::syntax::ruby {
+inline bool is_hex(char c) {
+  return ('0' <= c && c <= '9')
+         || ('a' <= c && c <= 'f')
+         || ('A' <= c && c <= 'F');
+};
+
+inline bool identifier_start_char(char c) {
+  return (c & 0x80)
+         || ('a' <= c && c <= 'z')
+         || ('A' <= c && c <= 'Z')
+         || c == '_';
+}
+
+inline bool identifier_char(char c) {
+  return (c & 0x80)
+         || ('a' <= c && c <= 'z')
+         || ('A' <= c && c <= 'Z')
+         || ('0' <= c && c <= '9')
+         || c == '_';
+}
+
+inline uint8_t utf8_codepoint_width(unsigned char c) {
+  if ((c & 0x80) == 0x00)
+    return 1;
+  if ((c & 0xE0) == 0xC0)
+    return 2;
+  if ((c & 0xF0) == 0xE0)
+    return 3;
+  if ((c & 0xF8) == 0xF0)
+    return 4;
+  return 1;
+}
+
+bool handle_escapes(RubyParser &p, std::vector<Token> *tokens, uint32_t &start, bool string = true) {
+  if (p.peek() == '\\') {
+    if (string)
+      tokens->push_back({start, p.i, Token::String});
+    else
+      tokens->push_back({start, p.i, Token::Regexp});
+    start = p.i;
+    p.advance();
+    if (p.peek() == 'x') {
+      p.advance();
+      if (is_hex(p.peek()))
+        p.advance();
+      if (is_hex(p.peek()))
+        p.advance();
+    } else if (p.peek() == 'u') {
+      p.advance();
+      if (p.peek() == '{') {
+        p.advance();
+        while (p.peek() != '}' && p.peek() != '\0')
+          p.advance();
+        if (p.peek() == '}')
+          p.advance();
+      } else {
+        if (is_hex(p.peek()))
+          p.advance();
+        if (is_hex(p.peek()))
+          p.advance();
+        if (is_hex(p.peek()))
+          p.advance();
+        if (is_hex(p.peek()))
+          p.advance();
+      }
+    } else if ('0' <= p.peek() && p.peek() <= '7') {
+      p.advance();
+      if ('0' <= p.peek() && p.peek() <= '7')
+        p.advance();
+      if ('0' <= p.peek() && p.peek() <= '7')
+        p.advance();
+    } else if (p.peek() == 'c') {
+      p.advance();
+      if (p.peek() != '\\')
+        p.advance();
+    } else if (p.peek() == 'M' || p.peek() == 'C') {
+      p.advance();
+      if (p.peek() == '-') {
+        p.advance();
+        if (p.peek() != '\\')
+          p.advance();
+      }
+    } else if (p.peek() == 'N') {
+      p.advance();
+      if (p.peek() == '{') {
+        p.advance();
+        while (p.peek() != '}' && p.peek() != '\0')
+          p.advance();
+        if (p.peek() == '}')
+          p.advance();
+      }
+    } else {
+      p.advance();
+    }
+    tokens->push_back({start, p.i, Token::Escape});
+    start = p.i;
+    return true;
+  }
+  return false;
+};
+
+bool handle_heredoc(RubyParser &p, std::vector<Token> *tokens) {
+  uint8_t *heredocs = p.state->heredocs();
+  uint32_t start = p.i;
+  if (start == 0) {
+    uint32_t heredoc_len = heredocs[0] & RubyState::Heredocs::LEN_MASK;
+    if (heredocs[0] & RubyState::Heredocs::ALLOW_INDENTATION)
+      while (start < p.len() && (p.line[start] == ' ' || p.line[start] == '\t'))
+        start++;
+    if (p.len() - start == heredoc_len
+        && memcmp(p.line.data() + start, heredocs + 1, heredoc_len) == 0) {
+      if (!p.dequeue_doc(heredoc_len))
+        p.current().state = RubyState::RubyInternalState::NONE;
+      tokens->push_back({p.i, p.len(), Token::Annotation});
+      return true;
+    }
+  }
+  if (!(heredocs[0] & RubyState::Heredocs::ALLOW_INTERPOLATION)) {
+    tokens->push_back({p.i, p.len(), Token::String});
+    return true;
+  } else {
+    while (p.i < p.len()) {
+      if (handle_escapes(p, tokens, start))
+        continue;
+      if (p.peek_str(2) == "#{") {
+        tokens->push_back({start, p.i, Token::String});
+        tokens->push_back({p.i, p.i + 2, Token::Interpolation});
+        p.advance(2);
+        p.push_state();
+        break;
+      }
+      p.advance();
+    }
+    if (p.i >= p.len())
+      tokens->push_back({start, p.len(), Token::String});
+    return false;
+  }
+}
+
+void handle_string(RubyParser &p, std::vector<Token> *tokens) {
+  uint32_t start = p.i;
+  while (p.i < p.len()) {
+    if (handle_escapes(p, tokens, start))
+      continue;
+    if ((p.current().flags & RubyState::RubyInternalState::ALLOW_INTERPOLATION)
+        && p.peek_str(2) == "#{") {
+      tokens->push_back({start, p.i, Token::String});
+      tokens->push_back({p.i, p.i + 2, Token::Interpolation});
+      p.advance(2);
+      p.push_state();
+      break;
+    }
+    if (p.peek() == p.current().delim_start
+        && p.current().delim_start != p.current().delim_end)
+      p.current().lit_brace_level++;
+    if (p.peek() == p.current().delim_end) {
+      if (p.current().delim_start == p.current().delim_end) {
+        p.advance();
+        tokens->push_back({start, p.i, Token::String});
+        p.current().state = RubyState::RubyInternalState::NONE;
+        p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+        return;
+      } else {
+        p.current().lit_brace_level--;
+        if (p.current().lit_brace_level == 0) {
+          p.advance();
+          tokens->push_back({start, p.i, Token::String});
+          p.current().state = RubyState::RubyInternalState::NONE;
+          p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+          return;
+        }
+      }
+    }
+    p.advance();
+  }
+  if (p.i >= p.len())
+    tokens->push_back({start, p.len(), Token::String});
+}
+
+void handle_regex(RubyParser &p, std::vector<Token> *tokens) {
+  uint32_t start = p.i;
+  while (p.i < p.len()) {
+    if (handle_escapes(p, tokens, start, false))
+      continue;
+    if ((p.current().flags & RubyState::RubyInternalState::ALLOW_INTERPOLATION)
+        && p.peek_str(2) == "#{") {
+      tokens->push_back({start, p.i, Token::Regexp});
+      tokens->push_back({p.i, p.i + 2, Token::Interpolation});
+      p.advance(2);
+      p.push_state();
+      break;
+    }
+    if (p.peek() == p.current().delim_start
+        && p.current().delim_start != p.current().delim_end)
+      p.current().lit_brace_level++;
+    if (p.peek() == p.current().delim_end) {
+      if (p.current().delim_start == p.current().delim_end) {
+        p.advance();
+        tokens->push_back({start, p.i, Token::Regexp});
+        p.current().state = RubyState::RubyInternalState::NONE;
+        p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+        return;
+      } else {
+        p.current().lit_brace_level--;
+        if (p.current().lit_brace_level == 0) {
+          p.advance();
+          tokens->push_back({start, p.i, Token::Regexp});
+          p.current().state = RubyState::RubyInternalState::NONE;
+          p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+          return;
+        }
+      }
+    }
+    p.advance();
+  }
+  if (p.i >= p.len())
+    tokens->push_back({start, p.len(), Token::Regexp});
+}
+
+bool handle_line_markers(RubyParser &p, std::vector<Token> *tokens) {
+  if (p.len() == 6 && p.peek_str(6) == "=begin") {
+    p.current().state = RubyState::RubyInternalState::COMMENT;
+    p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+    tokens->push_back({0, p.len(), Token::Comment});
+    return true;
+  }
+  if (p.len() == 7 && p.peek_str(7) == "__END__") {
+    p.current().state = RubyState::RubyInternalState::END;
+    p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+    return true;
+  }
+  return false;
+}
+
+bool handle_comment(RubyParser &p, std::vector<Token> *tokens, bool first_line) {
+  if (p.peek() == '#') {
+    p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+    if (first_line && p.i == 0 && p.peek(1) == '!') {
+      tokens->push_back({0, p.len(), Token::Shebang});
+      return true;
+    }
+    tokens->push_back({p.i, p.len(), Token::Comment});
+    return true;
+  }
+  return false;
+}
+
+void handle_syntax(RubyParser &p, std::vector<Token> *tokens) {
+  static const RubyTries tries = RubyTries();
+  if (p.i + 3 <= p.len() && p.peek_str(2) == "<<") {
+    uint32_t j = 2;
+    bool indented = false;
+    if (p.peek(j) == '~')
+      indented = true;
+    if (p.peek(j) == '~' || p.peek(j) == '-')
+      j++;
+    tokens->push_back({p.i, p.i + j, Token::Operator});
+    if (j >= p.len())
+      return;
+    std::string delim;
+    bool interpolation = true;
+    uint32_t s = p.i + j;
+    if (p.peek(j) == '\'' || p.peek(j) == '"') {
+      char q = p.peek(j++);
+      if (q == '\'')
+        interpolation = false;
+      while (j < p.len() && p.peek(j) != q)
+        delim += p.peek(j++);
+    } else {
+      if (j < p.len() && identifier_start_char(p.peek(j))) {
+        delim += p.peek(j++);
+        while (j < p.len() && identifier_char(p.peek(j)))
+          delim += p.peek(j++);
+      }
+    }
+    p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+    if (!delim.empty()) {
+      tokens->push_back({s, p.i + j, Token::Annotation});
+      uint8_t header = delim.size();
+      if (interpolation)
+        header |= RubyState::Heredocs::ALLOW_INTERPOLATION;
+      if (indented)
+        header |= RubyState::Heredocs::ALLOW_INDENTATION;
+      p.enqueue_doc(header, delim);
+      p.current().state = RubyState::RubyInternalState::HEREDOC;
+      p.heredoc_start_line = true;
+    }
+    p.advance(j);
+    return;
+  }
+  if (p.peek() == '/' && p.current().flags & RubyState::RubyInternalState::EXPECTING_EXPRESSION) {
+    tokens->push_back({p.i, p.i + 1, Token::Regexp});
+    p.current().state = RubyState::RubyInternalState::REGEXP;
+    p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+    p.current().flags |= RubyState::RubyInternalState::ALLOW_INTERPOLATION;
+    p.current().delim_start = '/';
+    p.current().delim_end = '/';
+    p.advance();
+    return;
+  }
+  switch (p.peek()) {
+  case '.': {
+    p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+    uint32_t start = p.i;
+    p.advance();
+    if (p.peek() == '.') {
+      p.advance();
+      if (p.peek() == '.')
+        p.advance();
+    }
+    tokens->push_back({start, p.i, Token::Operator});
+    return;
+  }
+  case ':': {
+    p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+    uint32_t start = p.i;
+    p.advance();
+    if (p.i >= p.len()) {
+      tokens->push_back({start, p.i, Token::Operator});
+      p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+      return;
+    }
+    if (p.peek() == ':') {
+      p.advance();
+      tokens->push_back({start, p.i, Token::Operator});
+      return;
+    }
+    if (p.peek() == '\'' || p.peek() == '"') {
+      tokens->push_back({start, p.i, Token::Label});
+      p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+      return;
+    }
+    if (p.peek() == '$' || p.peek() == '@') {
+      if (p.peek_str(2) == "@@")
+        p.advance(2);
+      else
+        p.advance();
+      while (identifier_char(p.peek()))
+        p.advance();
+      tokens->push_back({start, p.i, Token::Label});
+      return;
+    }
+    uint32_t op_len = tries.operator_trie.longest_match(p.peek_str(p.len() - p.i));
+    if (op_len > 0) {
+      tokens->push_back({start, p.i + op_len, Token::Label});
+      p.advance(op_len);
+      return;
+    }
+    if (identifier_start_char(p.peek())) {
+      p.advance();
+      while (identifier_char(p.peek()))
+        p.advance();
+      if (p.peek() == '!' || p.peek() == '?')
+        p.advance();
+      tokens->push_back({start, p.i, Token::Label});
+      return;
+    }
+    tokens->push_back({start, p.i, Token::Operator});
+    return;
+  }
+  case '@': {
+    p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+    uint32_t start = p.i;
+    p.advance();
+    if (p.i >= p.len())
+      return;
+    if (p.peek() == '@')
+      p.advance();
+    if (identifier_start_char(p.peek()))
+      p.advance();
+    else
+      return;
+    while (identifier_char(p.peek()))
+      p.advance();
+    tokens->push_back({start, p.i, Token::VariableInstance});
+    return;
+  }
+  case '$': {
+    p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+    uint32_t start = p.i;
+    p.advance();
+    if (p.i >= p.len())
+      return;
+    if (identifier_start_char(p.peek())) {
+      p.advance();
+      while (identifier_char(p.peek()))
+        p.advance();
+    } else if (p.i + 1 < p.len() && p.peek() == '-'
+               && (('a' <= p.peek(1) && p.peek(1) <= 'z') || ('A' <= p.peek(1) && p.peek(1) <= 'Z'))) {
+      p.advance(2);
+    } else if ('0' <= p.peek() && p.peek() <= '9') {
+      p.advance();
+      while ('0' <= p.peek() && p.peek() <= '9')
+        p.advance();
+    } else {
+      switch (p.peek()) {
+      case '~':
+      case '&':
+      case '`':
+      case '\'':
+      case '+':
+      case '=':
+      case '/':
+      case '\\':
+      case ',':
+      case ';':
+      case '.':
+      case '_':
+      case '*':
+      case '?':
+      case '!':
+      case '@':
+      case '<':
+      case '>':
+      case '$':
+        p.advance();
+        break;
+      default:
+        return;
+      }
+    }
+    tokens->push_back({start, p.i, Token::VariableGlobal});
+    return;
+  }
+  case '?': {
+    p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+    uint32_t start = p.i;
+    p.advance();
+    if (p.peek() == '\\') {
+    combination:
+      p.advance();
+      if (p.peek() == 'x') {
+        p.advance();
+        if (is_hex(p.peek()))
+          p.advance();
+        if (is_hex(p.peek()))
+          p.advance();
+        tokens->push_back({start, p.i, Token::Char});
+        return;
+      } else if (p.peek() == 'u') {
+        p.advance();
+        if (p.peek() == '{') {
+          p.advance();
+          while (p.peek() != '}' && p.peek() != '\0')
+            p.advance();
+          if (p.peek() == '}')
+            p.advance();
+        } else {
+          if (is_hex(p.peek()))
+            p.advance();
+          if (is_hex(p.peek()))
+            p.advance();
+          if (is_hex(p.peek()))
+            p.advance();
+          if (is_hex(p.peek()))
+            p.advance();
+        }
+        tokens->push_back({start, p.i, Token::Char});
+        return;
+      } else if ('0' <= p.peek() && p.peek() <= '7') {
+        p.advance();
+        if ('0' <= p.peek() && p.peek() <= '7')
+          p.advance();
+        if ('0' <= p.peek() && p.peek() <= '7')
+          p.advance();
+        tokens->push_back({start, p.i, Token::Char});
+        return;
+      } else if (p.peek() == 'c') {
+        p.advance();
+        if (p.peek() != '\\')
+          p.advance();
+        else
+          goto combination;
+        tokens->push_back({start, p.i, Token::Char});
+        return;
+      } else if (p.peek() == 'M' || p.peek() == 'C') {
+        p.advance();
+        if (p.peek() == '-') {
+          p.advance();
+          if (p.peek() != '\\')
+            p.advance();
+          else
+            goto combination;
+        }
+        tokens->push_back({start, p.i, Token::Char});
+        return;
+      } else if (p.peek() == 'N') {
+        p.advance();
+        if (p.peek() == '{') {
+          p.advance();
+          while (p.peek() != '}' && p.peek() != '\0')
+            p.advance();
+          if (p.peek() == '}')
+            p.advance();
+        }
+        tokens->push_back({start, p.i, Token::Char});
+        return;
+      } else {
+        p.advance();
+        tokens->push_back({start, p.i, Token::Char});
+        return;
+      }
+    } else if (p.peek() != '\0' && p.peek() != ' ' && p.peek() != '\t') {
+      p.advance();
+      tokens->push_back({start, p.i, Token::Char});
+      return;
+    } else {
+      p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+      tokens->push_back({start, p.i, Token::Operator});
+      return;
+    }
+  }
+  case '{': {
+    p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+    /*uint8_t brace_color =
+      (uint8_t)Token::K_BRACE1 + (state->full_state.brace_level % 5);
+    tokens->push_back({p.i, p.i + 1, (Token)brace_color});*/
+    p.current().brace_level++;
+    p.advance();
+    return;
+  }
+  case '}': {
+    p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+    if (!--p.current().brace_level && p.state->top > 1) {
+      p.pop_state();
+      tokens->push_back({p.i, p.i + 1, Token::Interpolation});
+    } else {
+      /*uint8_t brace_color =
+        (uint8_t)Token::K_BRACE1 + (state->full_state.brace_level % 5);
+      tokens->push_back({p.i, p.i + 1, (Token)brace_color});*/
+    }
+    p.advance();
+    return;
+  }
+  case '(': {
+    p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+    /*uint8_t brace_color =
+      (uint8_t)Token::K_BRACE1 + (state->full_state.brace_level % 5);
+    tokens->push_back({p.i, p.i + 1, (Token)brace_color});*/
+    p.current().brace_level++;
+    p.advance();
+    return;
+  }
+  case ')': {
+    p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+    p.current().brace_level--;
+    /*uint8_t brace_color =
+      (uint8_t)Token::K_BRACE1 + (state->full_state.brace_level % 5);
+    tokens->push_back({p.i, p.i + 1, (Token)brace_color});*/
+    p.advance();
+    return;
+  }
+  case '[': {
+    p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+    /*uint8_t brace_color =
+      (uint8_t)Token::K_BRACE1 + (state->full_state.brace_level % 5);
+    tokens->push_back({p.i, p.i + 1, (Token)brace_color});*/
+    p.current().brace_level++;
+    p.advance();
+    return;
+  }
+  case ']': {
+    p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+    p.current().brace_level--;
+    /*uint8_t brace_color =
+      (uint8_t)Token::K_BRACE1 + (state->full_state.brace_level % 5);
+    tokens->push_back({p.i, p.i + 1, (Token)brace_color});*/
+    p.advance();
+    return;
+  }
+  case '\'': {
+    p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+    tokens->push_back({p.i, p.i + 1, Token::String});
+    p.current().state = RubyState::RubyInternalState::STRING;
+    p.current().delim_start = '\'';
+    p.current().delim_end = '\'';
+    p.current().flags &= ~RubyState::RubyInternalState::ALLOW_INTERPOLATION;
+    p.advance();
+    return;
+  }
+  case '"': {
+    p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+    tokens->push_back({p.i, p.i + 1, Token::String});
+    p.current().state = RubyState::RubyInternalState::STRING;
+    p.current().delim_start = '"';
+    p.current().delim_end = '"';
+    p.current().flags |= RubyState::RubyInternalState::ALLOW_INTERPOLATION;
+    p.advance();
+    return;
+  }
+  case '`': {
+    p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+    tokens->push_back({p.i, p.i + 1, Token::String});
+    p.current().state = RubyState::RubyInternalState::STRING;
+    p.current().delim_start = '`';
+    p.current().delim_end = '`';
+    p.current().flags |= RubyState::RubyInternalState::ALLOW_INTERPOLATION;
+    p.advance();
+    return;
+  }
+  case '%': {
+    if (p.current().flags & RubyState::RubyInternalState::EXPECTING_EXPRESSION || p.i + 1 >= p.len()) {
+      tokens->push_back({p.i, p.i + 1, Token::Operator});
+      p.advance();
+      p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+      return;
+    }
+    p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+    char type = p.peek(1);
+    char delim_start = '\0';
+    char delim_end = '\0';
+    bool allow_interp = true;
+    int prefix_len = 1;
+    bool is_regexp = false;
+    switch (type) {
+    case 'r':
+      is_regexp = true;
+      allow_interp = true;
+      prefix_len = 2;
+      break;
+    case 'Q':
+    case 'x':
+    case 'I':
+    case 'W':
+      allow_interp = true;
+      prefix_len = 2;
+      break;
+    case 'w':
+    case 'q':
+    case 'i':
+    case 's':
+      allow_interp = false;
+      prefix_len = 2;
+      break;
+    default:
+      allow_interp = true;
+      prefix_len = 1;
+      break;
+    }
+    if (p.i + prefix_len >= p.len()) {
+      tokens->push_back({p.i, p.i + 1, Token::Operator});
+      p.advance(prefix_len);
+      p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+      return;
+    }
+    delim_start = p.peek(prefix_len);
+    if (identifier_char(delim_start) || delim_start == ' ') {
+      tokens->push_back({p.i, p.i + 1, Token::Operator});
+      p.advance(prefix_len);
+      p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+      return;
+    }
+    switch (delim_start) {
+    case '(':
+      delim_end = ')';
+      break;
+    case '{':
+      delim_end = '}';
+      break;
+    case '[':
+      delim_end = ']';
+      break;
+    case '<':
+      delim_end = '>';
+      break;
+    default:
+      delim_end = delim_start;
+      break;
+    }
+    tokens->push_back({p.i, p.i + prefix_len + 1, (is_regexp ? Token::Regexp : Token::String)});
+    p.current().state = is_regexp ? RubyState::RubyInternalState::REGEXP : RubyState::RubyInternalState::STRING;
+    p.current().delim_start = delim_start;
+    p.current().delim_end = delim_end;
+    if (allow_interp)
+      p.current().flags |= RubyState::RubyInternalState::ALLOW_INTERPOLATION;
+    p.current().lit_brace_level = 1;
+    p.advance(prefix_len + 1);
+    return;
+  }
+  default:
+    if ('0' <= p.peek() && p.peek() <= '9') {
+      p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+      uint32_t start = p.i;
+      if (p.peek() == '0') {
+        p.advance();
+        if (p.peek() == 'x' || p.peek() == 'X') {
+          p.advance();
+          while (true) {
+            while (is_hex(p.peek()))
+              p.advance();
+            if (p.peek() == '_')
+              p.advance();
+            else
+              break;
+          }
+        } else if (p.peek() == 'b' || p.peek() == 'B') {
+          p.advance();
+          while (true) {
+            while (p.peek() == '0' || p.peek() == '1')
+              p.advance();
+            if (p.peek() == '_')
+              p.advance();
+            else
+              break;
+          }
+        } else if (p.peek() == 'o' || p.peek() == 'O') {
+          p.advance();
+          while (true) {
+            while (p.peek() >= '0' && p.peek() <= '7')
+              p.advance();
+            if (p.peek() == '_')
+              p.advance();
+            else
+              break;
+          }
+        } else {
+          while (true) {
+            while (p.peek() >= '0' && p.peek() <= '7')
+              p.advance();
+            if (p.peek() == '_')
+              p.advance();
+            else
+              break;
+          }
+        }
+      } else {
+        while (true) {
+          while (p.peek() >= '0' && p.peek() <= '9')
+            p.advance();
+          if (p.peek() == '_')
+            p.advance();
+          else
+            break;
+        }
+        if (p.peek() == '.') {
+          p.advance();
+          while (true) {
+            while (p.peek() >= '0' && p.peek() <= '9')
+              p.advance();
+            if (p.peek() == '_')
+              p.advance();
+            else
+              break;
+          }
+        }
+        if (p.peek() == 'E' || p.peek() == 'e') {
+          p.advance();
+          if (p.peek() == '+' || p.peek() == '-')
+            p.advance();
+          while (true) {
+            while (p.peek() >= '0' && p.peek() <= '9')
+              p.advance();
+            if (p.peek() == '_')
+              p.advance();
+            else
+              break;
+          }
+        }
+      }
+      tokens->push_back({start, p.i, Token::Number});
+      return;
+    } else if (identifier_start_char(p.peek())) {
+      p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+      uint32_t j = 1;
+      while (identifier_char(p.peek(j)))
+        j++;
+      if (p.peek(j) == '!' || p.peek(j) == '?')
+        j++;
+      if (j == tries.base_keywords_trie.longest_match(p.peek_str(p.len() - p.i))) {
+        tokens->push_back({p.i, p.i + j, Token::Keyword});
+        p.advance(j);
+        return;
+      } else if (j == tries.expecting_keywords_trie.longest_match(p.peek_str(p.len() - p.i))) {
+        p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+        tokens->push_back({p.i, p.i + j, Token::Keyword});
+        p.advance(j);
+        return;
+      } else if (j == tries.operator_keywords_trie.longest_match(p.peek_str(p.len() - p.i))) {
+        tokens->push_back({p.i, p.i + j, Token::KeywordOperator});
+        p.advance(j);
+        return;
+      } else if (j == tries.expecting_operators_trie.longest_match(p.peek_str(p.len() - p.i))) {
+        p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+        tokens->push_back({p.i, p.i + j, Token::KeywordOperator});
+        p.advance(j);
+        return;
+      } else if (j == tries.types_trie.longest_match(p.peek_str(p.len() - p.i))) {
+        tokens->push_back({p.i, p.i + j, Token::Type});
+        p.advance(j);
+        return;
+      } else if (j == tries.methods_trie.longest_match(p.peek_str(p.len() - p.i))) {
+        tokens->push_back({p.i, p.i + j, Token::Function});
+        p.advance(j);
+        return;
+      } else if (j == tries.builtins_trie.longest_match(p.peek_str(p.len() - p.i))) {
+        tokens->push_back({p.i, p.i + j, Token::Constant});
+        p.advance(j);
+        return;
+      } else if (j == tries.errors_trie.longest_match(p.peek_str(p.len() - p.i))) {
+        tokens->push_back({p.i, p.i + j, Token::Error});
+        p.advance(j);
+        return;
+      } else if ('A' <= p.peek() && p.peek() <= 'Z' && !(p.peek(j) == '!' || p.peek(j) == '?')) {
+        tokens->push_back({p.i, p.i + j, Token::Constant});
+        p.advance(j);
+        return;
+      } else {
+        if (j == 4 && p.peek_str(4) == "true") {
+          tokens->push_back({p.i, p.i + j, Token::True});
+          p.advance(4);
+          return;
+        }
+        if (j == 5 && p.peek_str(5) == "false") {
+          tokens->push_back({p.i, p.i + j, Token::False});
+          p.advance(5);
+          return;
+        }
+        if (j == 3 && p.peek_str(3) == "def") {
+          tokens->push_back({p.i, p.i + j, Token::Keyword});
+          p.advance(3);
+          while (p.peek() == ' ' || p.peek() == '\t')
+            p.advance();
+          while (p.i < p.len()) {
+            if (identifier_start_char(p.peek())) {
+              uint32_t j = 1;
+              while (identifier_char(p.peek(j)))
+                j++;
+              if (p.peek(j) == '!' || p.peek(j) == '?')
+                j++;
+              if ('A' <= p.peek() && p.peek() <= 'Z')
+                tokens->push_back({p.i, p.i + j, Token::Constant});
+              else if (j == 4 && p.peek_str(4) == "self")
+                tokens->push_back({p.i, p.i + j, Token::Keyword});
+              else
+                tokens->push_back({p.i, p.i + j, Token::Function});
+              p.advance(j);
+              if (p.peek() == '.') {
+                p.advance();
+                continue;
+              }
+            }
+            break;
+          }
+          return;
+        }
+        uint32_t start = p.i;
+        p.advance(j);
+        if (p.peek() == ':') {
+          p.advance();
+          tokens->push_back({start, p.i, Token::Label});
+          return;
+        } else if (p.peek() == '!' || p.peek() == '?') {
+          p.advance();
+          tokens->push_back({start, p.i, Token::Function});
+          return;
+        } else {
+          uint32_t j = 0;
+          if (p.peek(j) == '(' || p.peek(j) == '{') {
+            tokens->push_back({start, p.i, Token::Function});
+            return;
+          } else if (p.peek(j) == ' ' || p.peek(j) == '\t') {
+            j++;
+          } else {
+            return;
+          }
+          while (p.peek(j) == ' ' || p.peek(j) == '\t')
+            j++;
+          if (p.i + j >= p.len())
+            return;
+          if (
+            p.peek(j) == '-'
+            || p.peek(j) == '&'
+            || p.peek(j) == '%'
+            || p.peek(j) == ':'
+          ) {
+            if (p.peek(j + 1) == ' ' || p.peek(j + 1) == '>')
+              return;
+          } else if (
+            p.peek(j) == ']'
+            || p.peek(j) == '}'
+            || p.peek(j) == ')'
+            || p.peek(j) == ','
+            || p.peek(j) == ';'
+            || p.peek(j) == '.'
+            || p.peek(j) == '+'
+            || p.peek(j) == '*'
+            || p.peek(j) == '/'
+            || p.peek(j) == '='
+            || p.peek(j) == '?'
+            || p.peek(j) == '|'
+            || p.peek(j) == '^'
+            || p.peek(j) == '<'
+            || p.peek(j) == '>'
+          ) {
+            return;
+          }
+          tokens->push_back({start, p.i, Token::Function});
+        }
+      }
+    } else {
+      uint32_t op_len;
+      if ((op_len = tries.operator_trie.longest_match(p.peek_str(p.len() - p.i)))) {
+        tokens->push_back({p.i, p.i + op_len, Token::Operator});
+        p.advance(op_len);
+        p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+      } else {
+        p.advance(utf8_codepoint_width(p.peek()));
+      }
+    }
+  }
+}
+
+void ruby_parse(void **v_state, std::string_view line, bool fl, std::vector<Token> *tokens) {
+  RubyParser p(v_state, line);
+  while (p.i < p.len()) {
+    if (p.current().state == RubyState::RubyInternalState::END)
+      return;
+    if (p.current().state == RubyState::RubyInternalState::COMMENT) {
+      tokens->push_back({p.i, p.len(), Token::Comment});
+      if (p.i == 0 && p.peek_str(4) == "=end")
+        p.current().state = RubyState::RubyInternalState::NONE;
+      return;
+    }
+    if (!p.heredoc_start_line
+        && p.current().state == RubyState::RubyInternalState::HEREDOC) {
+      if (handle_heredoc(p, tokens))
+        return;
+      else
+        continue;
+    }
+    if (p.current().state == RubyState::RubyInternalState::STRING) {
+      handle_string(p, tokens);
+      continue;
+    }
+    if (p.current().state == RubyState::RubyInternalState::REGEXP) {
+      handle_regex(p, tokens);
+      continue;
+    }
+    if (!p.i && handle_line_markers(p, tokens))
+      return;
+    if (handle_comment(p, tokens, fl))
+      return;
+    handle_syntax(p, tokens);
+    continue;
+  }
+}
+} // namespace bed::internal::syntax::ruby

src/internal/syntax/ruby/ruby.cc

@@ -0,0 +1,44 @@
+#include "internal/syntax/ruby/parser.h"
+
+namespace bed::internal::syntax::ruby {
+Language lang_ruby() {
+  return Language{
+    .none_state = []() {
+      RubyState *st = (RubyState *)malloc(
+        sizeof(RubyState)
+        + sizeof(RubyState::RubyInternalState)
+      );
+      st->top = 1;
+      st->docs = 0;
+      st->stack()[0] = {
+        .brace_level = 1,
+        .lit_brace_level = 0,
+        .state = RubyState::RubyInternalState::NONE,
+        .flags = 0,
+        .delim_start = '\0',
+        .delim_end = '\0'
+      };
+      return st; },
+    .parse = ruby_parse,
+    .copy = [](void *v_i_st) {
+      RubyState *i_st = (RubyState *)v_i_st;
+      uint32_t bytes = sizeof(RubyState)
+                       + sizeof(RubyState::RubyInternalState) * i_st->top
+                       + i_st->docs;
+      RubyState *o_st = (RubyState *)malloc(bytes);
+      memcpy(o_st, i_st, bytes);
+      return o_st; },
+    .equal = [](void *v_a_st, void *v_b_st) {
+      RubyState *a_st = (RubyState *)v_a_st;
+      uint32_t a_bytes = sizeof(RubyState)
+                         + sizeof(RubyState::RubyInternalState) * a_st->top
+                         + a_st->docs;
+      RubyState *b_st = (RubyState *)v_b_st;
+      uint32_t b_bytes = sizeof(RubyState)
+                         + sizeof(RubyState::RubyInternalState) * b_st->top
+                         + b_st->docs;
+      return a_bytes == b_bytes && memcmp(a_st, b_st, a_bytes) == 0; },
+    .destroy = [](void *v_st) { free(v_st); },
+  };
+}
+} // namespace bed::internal::syntax::ruby