Implement syntax highlighting

- Create a generic incremental syntax highlighting system.
- Implement a ruby syntax highlighter for it.

include/bed.h

@@ -5,6 +5,7 @@
 #include "internal/commands/commands.h"
 #include "internal/commands/suffixes.h"
 #include "internal/marks/marks.h"
+#include "internal/theme/theme.h"
 #include "pch.h"
 
 namespace bed {
@@ -13,6 +14,7 @@ struct BEd {
   internal::commands::Command no_op;
   internal::commands::Command eof_op;
   std::array<std::optional<internal::commands::Suffix>, 26> suffixes;
+  internal::theme::Theme theme;
 
   bool help_mode = false;
   std::string last_help = "";

include/internal/buffer/buffer.h

@@ -2,12 +2,17 @@
 
 #include "definitions.h"
 #include "internal/marks/marks.h"
+#include "internal/syntax/parser.h"
+#include "internal/syntax/ruby/parser.h"
+#include "internal/theme/theme.h"
 #include "pch.h"
 
 namespace bed::internal::buffer {
 struct Buffer {
-  internal::vase::Vase vase;
+  vase::Vase vase;
-  internal::marks::MarksEngine marks;
+  marks::MarksEngine marks;
+  syntax::Language lang;
+  syntax::Parser parser;
   uint64_t line = 0;
   bool modified;
   std::filesystem::path save_path = "";
@@ -28,7 +33,7 @@ struct Buffer {
   void join(uint64_t start_line, uint64_t end_line);
   void remove(uint64_t start_line, uint64_t end_line);
   void append(std::string text, uint64_t line);
-  void print(uint64_t start_line, uint64_t end_line);
+  void print(BEd &ctx, uint64_t start_line, uint64_t end_line);
   std::string list_string(std::string_view s);
 };
 } // namespace bed::internal::buffer

include/internal/syntax/decl.h

@@ -0,0 +1,182 @@
+#pragma once
+
+#include "internal/trie/trie.h"
+#include "pch.h"
+
+namespace bed::internal::syntax {
+struct Token {
+  uint32_t start;
+  uint32_t end;
+  enum : uint8_t {
+    Data,
+    Shebang,
+    Comment,
+    Error,
+    String,
+    Escape,
+    Interpolation,
+    Regexp,
+    Number,
+    True,
+    False,
+    Char,
+    Keyword,
+    KeywordOperator,
+    Operator,
+    Function,
+    Namespace,
+    Class,
+    Module,
+    Type,
+    Constant,
+    VariableInstance,
+    VariableGlobal,
+    Annotation,
+    Directive,
+    Label,
+    Brace1,
+    Brace2,
+    Brace3,
+    Brace4,
+    Brace5,
+    Heading1,
+    Heading2,
+    Heading3,
+    Heading4,
+    Heading5,
+    Heading6,
+    Blockquote,
+    List,
+    ListItem,
+    Code,
+    LanguageName,
+    LinkLabel,
+    ImageLabel,
+    Link,
+    Table,
+    TableHeader,
+    Italic,
+    Bold,
+    Underline,
+    Strikethrough,
+    HorizontalRule,
+    Tag,
+    Attribute,
+    CheckDone,
+    CheckNotDone,
+    Count
+  } type;
+};
+
+struct Language {
+  std::function<void *()> none_state;
+  std::function<void(void **, std::string_view, bool, std::vector<Token> *)> parse;
+  std::function<void *(void *)> copy;
+  std::function<bool(void *, void *)> equal;
+  std::function<void(void *)> destroy;
+};
+
+struct ParseState {
+  static constexpr uint64_t BRANCH_BIT = 1ull << 63;
+  static constexpr uint64_t LINES_MASK = ~BRANCH_BIT;
+  uint64_t header;
+  bool is_branch() const {
+    return header & BRANCH_BIT;
+  }
+  uint64_t lines() const {
+    return header & LINES_MASK;
+  }
+};
+
+struct ParseStateBranch : ParseState {
+  ParseState *left;
+  ParseState *right;
+};
+
+struct ParseStateLeaf : ParseState {
+  void *state;
+};
+
+struct TreeCursor {
+  ParseStateLeaf *leaf = nullptr;
+  ParseStateBranch *stack[64];
+  uint8_t depth = 0;
+  bool went_left[64];
+  TreeCursor(ParseState *root, uint64_t target_line, uint64_t *relative) {
+    ParseState *node = root;
+    while (node->is_branch()) {
+      auto *branch = (ParseStateBranch *)node;
+      auto *left = branch->left;
+      stack[depth] = branch;
+      if (target_line < left->lines()) {
+        went_left[depth] = true;
+        ++depth;
+        node = left;
+      } else {
+        target_line -= left->lines();
+        went_left[depth] = false;
+        ++depth;
+        node = branch->right;
+      }
+    }
+    *relative = target_line;
+    leaf = (ParseStateLeaf *)node;
+  }
+  void next() {
+    while (depth > 0) {
+      auto *branch = stack[depth - 1];
+      bool from_left = went_left[depth - 1];
+      --depth;
+      if (!from_left)
+        continue;
+      ParseState *node = branch->right;
+      while (node->is_branch()) {
+        auto *b = (ParseStateBranch *)node;
+        stack[depth] = b;
+        went_left[depth] = true;
+        ++depth;
+        node = b->left;
+      }
+      leaf = (ParseStateLeaf *)node;
+      return;
+    }
+    leaf = nullptr;
+  }
+  void prev() {
+    while (depth > 0) {
+      auto *branch = stack[depth - 1];
+      bool from_left = went_left[depth - 1];
+      --depth;
+      if (from_left)
+        continue;
+      ParseState *node = branch->left;
+      while (node->is_branch()) {
+        auto *b = (ParseStateBranch *)node;
+        stack[depth] = b;
+        went_left[depth] = false;
+        ++depth;
+        node = b->right;
+      }
+      leaf = (ParseStateLeaf *)node;
+      return;
+    }
+    leaf = nullptr;
+  }
+};
+
+/*struct Symbol {
+  uint64_t definition;
+  std::vector<uint64_t> references;
+};
+
+struct Space {
+  uint64_t len;
+};
+
+struct Scope {
+  uint64_t len;
+  uint32_t type;
+  trie::Trie<Symbol> symbols;
+  std::vector<std::variant<Space, Scope>> children;
+};*/
+} // namespace bed::internal::syntax

include/internal/syntax/parser.h

@@ -0,0 +1,40 @@
+#pragma once
+
+#include "decl.h"
+#include "internal/vase/vase.h"
+#include "pch.h"
+
+namespace bed::internal::syntax {
+struct Parser {
+  ParseState *root;
+  Language ⟨
+
+  Parser(vase::Vase &, uint64_t, Language &);
+  ~Parser();
+  void reset(vase::Vase &, uint64_t, Language &);
+  void erase(vase::Vase &, uint64_t, uint64_t);
+  void insert(vase::Vase &, uint64_t, uint64_t);
+  void modify(vase::Vase &, uint64_t, uint64_t);
+
+  std::pair<ParseState *, ParseState *> split_tree(ParseState *node, uint64_t line);
+  ParseState *join_tree(ParseState *a, ParseState *b);
+
+  // Scope root;
+  // Scope &get_scope(uint64_t);
+  struct Iterator {
+    Parser *p;
+    std::optional<vase::Iterator> it;
+    void *state;
+    uint64_t at;
+    std::vector<Token> tokens;
+    Iterator(uint64_t, Parser *, vase::Vase &);
+    ~Iterator();
+    Iterator(const Iterator &) = delete;
+    Iterator &operator=(const Iterator &) = delete;
+    Iterator(Iterator &&other);
+    Iterator &operator=(Iterator &&other);
+    void next();
+  };
+  std::optional<Iterator> get_hl(vase::Vase &, uint64_t);
+};
+} // namespace bed::internal::syntax

include/internal/syntax/ruby/decl.h

@@ -0,0 +1,50 @@
+#pragma once
+
+#include "../decl.h"
+#include "pch.h"
+#include "tries.h"
+
+namespace bed::internal::syntax::ruby {
+struct alignas(2) RubyState {
+  struct RubyInternalState {
+    uint16_t brace_level;
+    uint16_t lit_brace_level;
+    enum : uint8_t {
+      NONE,
+      STRING,
+      REGEXP,
+      HEREDOC,
+      COMMENT,
+      END
+    } state;
+    static constexpr const uint8_t EXPECTING_EXPRESSION = 0b00010000;
+    static constexpr const uint8_t ALLOW_INTERPOLATION = 0b00000001;
+    uint8_t flags;
+    char delim_start;
+    char delim_end;
+  };
+
+  struct Heredocs {
+    // header masks
+    static constexpr const uint8_t ALLOW_INDENTATION = 0b10000000;
+    static constexpr const uint8_t ALLOW_INTERPOLATION = 0b01000000;
+    static constexpr const uint8_t LEN_MASK = 0b00111111;
+    // the rest will be len number of bytes with the actual name.
+  };
+
+  uint8_t top;
+  uint8_t docs;
+  // the stack (RubyInternalState * top)
+  // the docs queue (docs)
+
+  inline RubyInternalState *stack() {
+    return (RubyInternalState *)(this + 1);
+  }
+
+  inline uint8_t *heredocs() {
+    return (uint8_t *)(stack() + top);
+  }
+};
+
+Language lang_ruby();
+} // namespace bed::internal::syntax::ruby

include/internal/syntax/ruby/parser.h

@@ -0,0 +1,103 @@
+#pragma once
+
+#include "decl.h"
+#include "pch.h"
+
+namespace bed::internal::syntax::ruby {
+struct RubyParser {
+  void **v_state;
+  RubyState *state;
+  std::string_view line;
+  uint32_t i = 0;
+  bool heredoc_start_line = false;
+
+  RubyParser(void **v_state, std::string_view line)
+      : v_state(v_state), state((RubyState *)*v_state), line(line) {}
+
+  uint32_t len() const {
+    return line.size();
+  }
+
+  RubyState::RubyInternalState &current() {
+    return state->stack()[state->top - 1];
+  }
+
+  char peek(uint32_t offset = 0) {
+    uint32_t pos = i + offset;
+    return pos < line.size() ? line[pos] : '\0';
+  }
+
+  std::string_view peek_str(uint32_t len) {
+    return line.substr(i, len);
+  }
+
+  void advance() {
+    ++i;
+  }
+
+  void advance(uint32_t n) {
+    i += n;
+  }
+
+  void push_state() {
+    state->top++;
+    *v_state = (RubyState *)realloc(
+      *v_state,
+      sizeof(RubyState)
+        + sizeof(RubyState::RubyInternalState) * state->top
+        + state->docs
+    );
+    state = (RubyState *)*v_state;
+    memmove(
+      state->heredocs(),
+      state->stack() + state->top - 1,
+      state->docs
+    );
+    current() = {
+      .brace_level = 1,
+      .lit_brace_level = 0,
+      .state = RubyState::RubyInternalState::NONE,
+      .flags = 0,
+      .delim_start = '\0',
+      .delim_end = '\0'
+    };
+  }
+
+  void pop_state() {
+    state->top--;
+    memmove(
+      state->heredocs(),
+      state->stack() + state->top + 1,
+      state->docs
+    );
+  }
+
+  void enqueue_doc(uint8_t header, std::string_view name) {
+    uint32_t bytes = 1 + name.size();
+    uint32_t old_docs = state->docs;
+    state->docs += bytes;
+    *v_state = (RubyState *)realloc(
+      *v_state,
+      sizeof(RubyState)
+        + sizeof(RubyState::RubyInternalState) * state->top
+        + state->docs
+    );
+    state = (RubyState *)*v_state;
+    uint8_t *heredocs = state->heredocs();
+    heredocs[old_docs] = header;
+    memcpy(heredocs + old_docs + 1, name.data(), name.size());
+  }
+
+  bool dequeue_doc(uint8_t heredoc_len) {
+    uint32_t bytes = heredoc_len + 1;
+    uint8_t *heredocs = state->heredocs();
+    if (state->docs -= bytes)
+      memmove(heredocs, heredocs + bytes, state->docs);
+    else
+      return false;
+    return true;
+  }
+};
+
+void ruby_parse(void **v_state, std::string_view line, bool first_line, std::vector<Token> *);
+} // namespace bed::internal::syntax::ruby

include/internal/syntax/ruby/tries.h

@@ -0,0 +1,336 @@
+#pragma once
+
+#include "internal/trie/trie.h"
+#include "pch.h"
+
+namespace bed::internal::syntax::ruby {
+const static std::vector<std::string> types = {
+  "BasicObject",
+  "Object",
+  "NilClass",
+  "TrueClass",
+  "FalseClass",
+  "Integer",
+  "Fixnum",
+  "Bignum",
+  "Float",
+  "Rational",
+  "Complex",
+  "Numeric",
+  "String",
+  "Symbol",
+  "Array",
+  "Hash",
+  "Range",
+  "Regexp",
+  "Struct",
+  "Enumarator",
+  "Enumerable",
+  "Time",
+  "Date",
+  "IO",
+  "File",
+  "Dir",
+  "Thread",
+  "Proc",
+  "Method",
+  "Module",
+  "Class",
+  "Mutex",
+  "ConditionVariable",
+  "MatchData",
+  "Encoding",
+  "Fiber",
+};
+
+const static std::vector<std::string> builtins = {
+  "ARGF",
+  "ARGV",
+  "ENV",
+  "STDIN",
+  "STDOUT",
+  "STDERR",
+  "DATA",
+  "TOPLEVEL_BINDING",
+  "RUBY_PLATFORM",
+  "RUBY_VERSION",
+  "RUBY_RELEASE_DATE",
+  "RUBY_PATCHLEVEL",
+  "RUBY_ENGINE",
+  "__LINE__",
+  "__FILE__",
+  "__ENCODING__",
+  "__dir__",
+  "__callee__",
+  "__method__",
+  "__id__",
+  "__send__",
+};
+
+const static std::vector<std::string> methods = {
+  "abort",
+  "at_exit",
+  "binding",
+  "block_given?",
+  "caller",
+  "catch",
+  "chomp",
+  "chomp!",
+  "chop",
+  "chop!",
+  "eval",
+  "exec",
+  "exit",
+  "exit!",
+  "fail",
+  "fork",
+  "format",
+  "gets",
+  "global_variables",
+  "gsub",
+  "gsub!",
+  "iterator?",
+  "lambda",
+  "load",
+  "loop",
+  "open",
+  "print",
+  "printf",
+  "proc",
+  "putc",
+  "puts",
+  "raise",
+  "rand",
+  "readline",
+  "readlines",
+  "require",
+  "require_relative",
+  "select",
+  "sleep",
+  "spawn",
+  "split",
+  "sprintf",
+  "srand",
+  "sub",
+  "sub!",
+  "syscall",
+  "system",
+  "test",
+  "throw",
+  "trace_var",
+  "trap",
+  "untrace_var",
+  "attr",
+  "attr_reader",
+  "attr_writer",
+  "attr_accessor",
+  "class_variable_get",
+  "class_variable_set",
+  "define_method",
+  "instance_variable_get",
+  "instance_variable_set",
+  "private",
+  "protected",
+  "public",
+  "public_class_method",
+  "module_function",
+  "remove_method",
+  "undef_method",
+  "method",
+  "methods",
+  "singleton_methods",
+  "private_methods",
+  "protected_methods",
+  "public_methods",
+  "send",
+  "extend",
+  "include",
+  "prepend",
+  "clone",
+  "dup",
+  "freeze",
+  "taint",
+  "untaint",
+  "trust",
+  "untrust",
+  "untaint?",
+  "trust?",
+  "each",
+  "each_with_index",
+  "each_with_object",
+  "map",
+  "collect",
+  "select",
+  "reject",
+  "reduce",
+  "inject",
+  "find",
+  "detect",
+  "all?",
+  "any?",
+  "none?",
+  "one?",
+  "count",
+  "cycle",
+  "drop",
+  "drop_while",
+  "take",
+  "take_while",
+  "chunk",
+  "chunk_while",
+  "group_by",
+  "partition",
+  "slice_before",
+  "slice_after",
+  "nil?",
+  "is_a?",
+  "kind_of?",
+  "instance_of?",
+  "respond_to?",
+  "equal?",
+  "object_id",
+  "class",
+  "singleton_class",
+  "clone",
+  "freeze",
+  "tap",
+  "then",
+};
+
+const static std::vector<std::string> errors = {
+  "Error",
+  "Exception",
+  "SignalException",
+  "Interrupt",
+  "StopIteration",
+  "Errno",
+  "SystemExit",
+  "fatal",
+};
+
+const static std::vector<std::string> base_keywords = {
+  "class",
+  "module",
+  "begin",
+  "end",
+  "else",
+  "rescue",
+  "ensure",
+  "do",
+  "when",
+};
+
+const static std::vector<std::string> expecting_keywords = {
+  "if",
+  "elsif",
+  "case",
+  "for",
+  "while",
+  "until",
+  "unless",
+};
+
+const static std::vector<std::string> operator_keywords = {
+  "alias",
+  "BEGIN",
+  "break",
+  "catch",
+  "defined?",
+  "in",
+  "next",
+  "redo",
+  "rescue",
+  "retry",
+  "super",
+  "self",
+  "nil",
+  "undef",
+};
+
+const static std::vector<std::string> expecting_operators = {
+  "and",
+  "return",
+  "not",
+  "yield",
+  "or",
+};
+
+const static std::vector<std::string> operators = {
+  "+",
+  "-",
+  "*",
+  "/",
+  "%",
+  "**",
+  "==",
+  "!=",
+  "===",
+  "<=>",
+  ">",
+  ">=",
+  "<",
+  "<=",
+  "&&",
+  "||",
+  "!",
+  "&",
+  "|",
+  "^",
+  "~",
+  "<<",
+  ">>",
+  "=",
+  "+=",
+  "-=",
+  "*=",
+  "/=",
+  "%=",
+  "**=",
+  "&=",
+  "|=",
+  "^=",
+  "<<=",
+  ">>=",
+  "..",
+  "...",
+  "===",
+  "=",
+  "=>",
+  "&",
+  "`",
+  "->",
+  "=~",
+};
+
+struct RubyTries {
+  trie::Trie<void> base_keywords_trie;
+  trie::Trie<void> expecting_keywords_trie;
+  trie::Trie<void> operator_keywords_trie;
+  trie::Trie<void> expecting_operators_trie;
+  trie::Trie<void> operator_trie;
+  trie::Trie<void> types_trie;
+  trie::Trie<void> builtins_trie;
+  trie::Trie<void> methods_trie;
+  trie::Trie<void> errors_trie;
+  RubyTries() {
+    for (auto &keyword : base_keywords)
+      base_keywords_trie.insert(keyword);
+    for (auto &keyword : expecting_keywords)
+      expecting_keywords_trie.insert(keyword);
+    for (auto &keyword : operator_keywords)
+      operator_keywords_trie.insert(keyword);
+    for (auto &keyword : expecting_operators)
+      expecting_operators_trie.insert(keyword);
+    for (auto &keyword : operators)
+      operator_trie.insert(keyword);
+    for (auto &keyword : types)
+      types_trie.insert(keyword);
+    for (auto &keyword : builtins)
+      builtins_trie.insert(keyword);
+    for (auto &keyword : methods)
+      methods_trie.insert(keyword);
+    for (auto &keyword : errors)
+      errors_trie.insert(keyword);
+  }
+};
+} // namespace bed::internal::syntax::ruby

include/internal/theme/theme.h

@@ -0,0 +1,31 @@
+#pragma once
+
+#include "definitions.h"
+#include "internal/syntax/parser.h"
+#include "pch.h"
+
+namespace bed::internal::theme {
+struct Highlight {
+  enum : uint8_t {
+    None = 0,
+    Bold = 1 << 0,
+    Italic = 1 << 1,
+    Strikethrough = 1 << 2,
+    Underline = 1 << 3,
+  };
+  uint32_t fg;
+  uint32_t bg;
+  uint8_t flags;
+};
+
+struct Theme {
+  std::array<Highlight, internal::syntax::Token::Count> hl;
+
+  Theme();
+
+  Highlight get(internal::syntax::Token token) const;
+
+  static Theme default_theme();
+  static Theme from_name(std::string_view name);
+};
+} // namespace bed::internal::theme

include/internal/trie/trie.h

@@ -157,14 +157,14 @@ struct Trie {
     }
   }
 
-  uint64_t longest_match(std::string_view input) {
+  uint64_t longest_match(std::string_view input) const {
-    Node *current = &root;
+    const Node *current = &root;
     uint64_t pos = 0;
     uint64_t longest = 0;
     if (current->value)
       longest = 0;
     while (pos < input.size()) {
-      Node *child = find_child(*current, input[pos]);
+      const Node *child = find_child(*current, input[pos]);
       if (!child)
         break;
       const auto remaining = input.substr(pos);
@@ -179,7 +179,7 @@ struct Trie {
     return longest;
   }
 
-  bool matches(std::string_view key)
+  bool matches(std::string_view key) const
     requires(std::is_void_v<T>)
   {
     Node *current = &root;
@@ -198,13 +198,13 @@ struct Trie {
     return current->value.has_value();
   }
 
-  std::optional<V> get(std::string_view key)
+  std::optional<V> get(std::string_view key) const
     requires(!std::is_void_v<T>)
   {
-    Node *current = &root;
+    const Node *current = &root;
     uint64_t pos = 0;
     while (pos < key.size()) {
-      Node *child = find_child(*current, key[pos]);
+      const Node *child = find_child(*current, key[pos]);
       if (!child)
         return std::nullopt;
       const auto remaining = key.substr(pos);
@@ -228,7 +228,7 @@ struct Trie {
   uint64_t common_prefix(
     std::string_view a,
     std::string_view b
-  ) {
+  ) const {
     const auto n = std::min(a.size(), b.size());
     uint64_t i = 0;
     while (i < n && equal_char(a[i], b[i]))
@@ -236,7 +236,7 @@ struct Trie {
     return i;
   }
 
-  Node *find_child(Node &node, char first) {
+  Node *find_child(const Node &node, char first) const {
     for (auto *child : node.children)
       if (equal_char(child->edge.front(), first))
         return child;

include/internal/vase/shard.h

@@ -13,14 +13,13 @@ struct Shard {
   } kind;
 
   uint8_t height;
+  std::atomic_uint32_t refs;
 
   uint64_t length;
   uint64_t lines;
 
-  std::atomic_uint64_t refs;
-
   Shard(Kind kind, uint64_t length, uint64_t lines, uint8_t height)
-      : kind(kind), height(height), length(length), lines(lines), refs(1) {};
+      : kind(kind), height(height), refs(1), length(length), lines(lines) {};
 
   static void retain(Shard *n);
   static void release(Shard *n);

samples/sample.rb

@@ -0,0 +1,343 @@
+#!/usr/bin/env ruby
+
+# Unicode / Emoji / CJK stress-test Ruby file
+# Purpose: Test syntax highlighting + width calculation in your editor
+# ---------------------------------------------------------------
+
+# Mixed-width CJKssssssssssssssss LoadErssssssssssssssssssssssss
+cjk_samples = [
+  '漢字テスト',
+  '測試中文字串',
+  '한국어 테스트',
+  'ひらがなカタカナ混合'
+]
+
+# a hex color: #FFFFFF shouldn't hl here: hsl(147rad, 50%, 47%)  as it is not css-style file
+
+0x603010 # another hex color
+
+# Ruby regex with unicode
+$unicode_regex_multiline = /[一-龯ぁ-ん12288ァ
+\-ヶー
+s wow
+
+々〆〤]/
+
+UNICORE = /
+    s
+    {#{ss}}
+    \C-s\u{10}
+  /
+
+UNINITCORE = %(
+
+    {{#{}}}
+
+    test = "A:\x41 B:\101 C:\u0043 D:\u{44 45} NUL:\0 DEL:\c? CTRL_A:\cA META_X:\M-x CTRL_META_X:\C-\M-x MIX:\C-\M-z N:\N{UNICODE NAME}"
+
+  )
+
+# Unicode identifiers (valid in Ruby)
+变量 = 0x5_4eddaee
+π = 0.314_159e+2, ?\u0234, "\,", ?\x0A, 's', true, false, 0
+挨拶 = -> { "こんに \n ちは" }
+
+arr = []
+not_arr = NotABuiltin.new
+
+raise NameError or SystemExit or CustomError or Errno or ErrorNotAtAll
+
+# Method using unicode variable names
+def math_test
+  puts "π * 2 = #{π * 2}"
+end
+
+# Iterate through CJK samples
+cjk_samples.each_with_index do |str, idx:|
+  puts %! CJK[#{idx}] => #{str} (len=#{str.length})\! !
+  symbol = :"
+  a
+  "
+  sym2 = :hello
+end
+
+# Test emoji width behaviors
+puts "Emoji count: #{emojis.length}"
+
+# Multi-line string with unicode
+multi = <<~BASH
+    # Function recursion demo
+    factorial() {
+      local n="$1"
+      if ((n <= 1)); then
+        echo 1
+      else\ns
+        local prev
+        prev=$(factorial $((n - 1)))
+        echo $((n * prev))
+       before #{ interpol
+                 # {'    '}
+                 # comment should be fine heres s
+                 $a / $-s + 0xFF
+              }s#{' '}
+  x
+      a after
+      fi
+    } #{s}
+    log INFO "factorial(5) = $(factorial 5)"
+BASH
+
+puts multi
+
+
+# Arrays mixing everything
+mixed = [
+  '🐍 Ruby + Python? sacrilege! 🐍',
+  '日本語とEnglishと🔧mix',
+  'Spacing test →→→→→→→',
+  'Zero-width joiner test: 👨‍👩‍👧‍👦 family emoji'
+]
+
+two_docs = <<~DOC1, <<~DOC2
+  stuff for doc2
+  rdvajehvbaejbfh
+DOC1
+  stuff for doc 2 with #{!interpolation} and more
+DOC2
+
+p = 0 << 22 # not a heredoc
+
+mixed.each { |m| puts m }
+
+# Unicode in comments — highlight me!
+# コメント：エディタのハイライトを確認します✨
+# Emojis should not break formatting: 🦀🦊🐱‍👤🤖
+
+# Dummy Ruby logic
+5.times do |i|
+  puts "Loop #{i}: 🌟 #{cjk_samples[i % cjk_samples.size]}"
+end
+
+# String escape sequences + unicode
+escaped = "Line1\nLine2\tTabbed 😀"
+puts escaped
+
+p = 0 << 2
+# Frozen string literal test
+# frozen_string_literal: true
+const_str = '定数文字列🔒'.freeze
+puts const_str
+
+# End marker
+puts '--- END OF UNICODE TEST FILE ---'
+
+# Ruby syntax highlighting test
+
+#   This is a multi-line comment.
+#   It spans multiple lines.
+#   Good for testing highlighting.
+#
+# This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped line test, This is a wrapped linetest,
+#
+
+# Constants
+
+PI = 3.14159
+MAX_ITER = 5
+# Module
+module Utilities
+  def self.random_greeting
+    %w[Hello Hi Hey Hola Bonjour Merhaba].sample
+  end
+
+  def self.factorial(n)
+    return 1 if n <= 1
+
+    n * factorial(n - 1)
+  end
+end
+
+# Class
+class TestObject
+  attr_accessor :name, :value
+
+  def initialize(name, value)
+    @name = name
+    @value = value
+  end
+
+  def display
+    puts "#{@name}: #{@value}"
+  end
+
+  private
+
+  def double_value
+    @value * 2
+  end
+end
+
+# Inheritance
+class SpecialObject < TestObject
+  def triple_value
+    @value * 3
+  end
+end
+
+# Lambda
+adder = ->(x, y) { x + y }
+
+# Array and hash
+numbers = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+hash = { a: 1, b: 2, c: 3 }
+
+# Iteration
+numbers.each do |n|
+  puts "Number: #{n}"
+end
+
+# Hash iteration
+hash.each do |key, value|
+  puts "#{key} => #{value}"
+end
+
+# Conditional
+numbers.each do |n|
+  if n.even?
+    puts "#{n} is even"
+  else
+    puts "#{n} is odd"
+  end
+end
+
+# Method definition
+def greet_person(name)
+  puts "#{Utilities.random_greeting}, #{name}!"
+  return true if name == 'harry'
+
+  's'
+end
+
+h = a / a
+
+# Calling methods
+greet_person('Alice')
+greet_person('Bob')
+
+# Loops
+i = 0
+while i < 5
+  puts "Loop iteration #{i}"
+  i += 1
+end
+
+for j in 1..3
+  puts "For loop #{j}"
+end
+
+# Begin-rescue-ensure
+begin
+  risky = 10 / 2
+  puts "Risky operation succeeded: #{risky}"
+rescue ZeroDivisionError => e
+  puts "Caught an error: #{e}"
+ensure
+  puts 'This runs no matter what'
+end
+
+# Arrays of objects
+objs = []
+5.times do |k|
+  objs << TestObject.new("Obj#{k}", k)
+end
+
+objs.each(&:display)
+
+# Nested arrays
+nested = [[1, 2], [3, 4], [5, 6]]
+nested.each do |arr|
+  arr.each { |x| print "#{x} " }
+  puts
+end
+
+# Case statement
+numbers.each do |n|
+  case n
+  when 1..3
+    puts "#{n} is small"
+  when 4..7
+    puts "#{n} is medium"
+  else
+    puts "#{n} is large"
+  end
+end
+
+# Using factorial
+(0..5).each do |n|
+  puts "Factorial of #{n} is #{Utilities.factorial(n)}"
+end
+
+# Special objects
+so = SpecialObject.new('Special', 10)
+puts "Double: #{so.double_value}, Triple: #{so.triple_value}"
+
+# String interpolation and formatting
+puts "PI is approximately #{PI.round(2)}"
+
+# Multi-line strings
+multi_line = <<~TEXT
+   k kmW ;
+  This is a multi-line string.
+  It spans multiple lines.
+  Gossn sssmss
+  ddsss
+  od for testing highlighting.
+TEXT
+
+puts multi_line
+
+# Symbols and strings
+sym = :my_symbol == __dir__
+str = 'my string'
+puts "Symbol: #{sym}, String: #{str}"
+
+# Random numbers
+rand_nums = Array.new(5) { rand(100) }
+puts "Random numbers: #{rand_nums.join(', ')}"
+
+# More loops
+rand_nums.each_with_index do |num, idx|
+  puts "Index #{idx} has number #{num}"
+end
+
+# Ternary operator
+rand_nums.each do |num|
+  puts num.even? ? "#{num} is even" : "#{num} is odd"
+end
+
+# Block with yield
+def wrapper
+  puts 'Before block'
+  yield if block_given?
+  puts 'After block'
+end
+
+# ss
+
+wrapper { puts 'Inside block' }
+
+# Sorting
+sorted = rand_nums.sort
+puts "Sorted: #{sorted.join(', ')}"
+
+# Regex
+sample_text = 'The quick brown fox jumps over the lazy dog'
+puts "Match 'fox'?" if sample_text =~ /fox/
+
+# End of test script
+puts 'Ruby syntax highlighting test complete.'
+
+__END__
+
+Anything here should be ignored >><<
+{{{}}}[[[]]](((000)))

src/bed/run.cc

@@ -1,7 +1,8 @@
 #include "bed.h"
 
 namespace bed {
-BEd::BEd(std::vector<std::string> args) {
+BEd::BEd(std::vector<std::string> args)
+    : theme(internal::theme::Theme::default_theme()) {
   internal::commands::Command::register_posix(*this);
   internal::commands::Suffix::register_suffixes(*this);
   std::string prompt_ = "";

src/internal/buffer/buffer.cc

@@ -1,12 +1,18 @@
 #include "internal/buffer/buffer.h"
+#include "bed.h"
 
 namespace bed::internal::buffer {
-Buffer::Buffer() : vase("/tmp") {
+Buffer::Buffer()
+    : vase("/tmp"), lang(syntax::ruby::lang_ruby()),
+      parser(vase, vase.lines(), lang) {
   line = vase.lines();
   modified = false;
 }
 
-Buffer::Buffer(std::string command) : vase(command, "/tmp") {
+Buffer::Buffer(std::string command)
+    : vase(command, "/tmp"),
+      lang(syntax::ruby::lang_ruby()),
+      parser(vase, vase.lines(), lang) {
   line = vase.lines();
   if (!line)
     return;
@@ -15,7 +21,10 @@ Buffer::Buffer(std::string command) : vase(command, "/tmp") {
   modified = false;
 }
 
-Buffer::Buffer(std::filesystem::path path) : vase(path, "/tmp") {
+Buffer::Buffer(std::filesystem::path path)
+    : vase(path, "/tmp"),
+      lang(syntax::ruby::lang_ruby()),
+      parser(vase, vase.lines(), lang) {
   line = vase.lines();
   if (!line)
     return;
@@ -28,6 +37,7 @@ Buffer::Buffer(std::filesystem::path path) : vase(path, "/tmp") {
 void Buffer::load(std::string command) {
   vase::Vase new_vase = vase::Vase(command, "/tmp");
   vase = std::move(new_vase);
+  parser.reset(vase, vase.lines(), lang);
   line = vase.lines();
   if (!line) {
     prev_range.start = 0;
@@ -42,6 +52,7 @@ void Buffer::load(std::string command) {
 void Buffer::load(std::filesystem::path path) {
   vase::Vase new_vase = vase::Vase(path, "/tmp");
   vase = std::move(new_vase);
+  parser.reset(vase, vase.lines(), lang);
   line = vase.lines();
   if (!line) {
     prev_range.start = 0;
@@ -93,10 +104,72 @@ void Buffer::join(uint64_t start_line, uint64_t end_line) {
   modified = true;
 }
 
-void Buffer::print(uint64_t start_line, uint64_t end_line) {
+inline void apply(std::ostream &out, const theme::Highlight &hl) {
-  vase::Iterator it = vase.iterate(start_line - 1, Direction::Forward);
+  out << "\x1b[0m";
-  while (it.next() && start_line++ <= end_line)
+  const uint8_t r = (hl.fg >> 16) & 0xff;
-    std::cout << it.line << std::endl;
+  const uint8_t g = (hl.fg >> 8) & 0xff;
+  const uint8_t b = hl.fg & 0xff;
+  out << "\x1b[38;2;"
+      << static_cast<unsigned>(r) << ';'
+      << static_cast<unsigned>(g) << ';'
+      << static_cast<unsigned>(b) << 'm';
+  if (hl.bg != 0) {
+    const uint8_t br = (hl.bg >> 16) & 0xff;
+    const uint8_t bg = (hl.bg >> 8) & 0xff;
+    const uint8_t bb = hl.bg & 0xff;
+    out << "\x1b[48;2;"
+        << static_cast<unsigned>(br) << ';'
+        << static_cast<unsigned>(bg) << ';'
+        << static_cast<unsigned>(bb) << 'm';
+  }
+  if (hl.flags & theme::Highlight::Bold)
+    out << "\x1b[1m";
+  if (hl.flags & theme::Highlight::Italic)
+    out << "\x1b[3m";
+  if (hl.flags & theme::Highlight::Underline)
+    out << "\x1b[4m";
+  if (hl.flags & theme::Highlight::Strikethrough)
+    out << "\x1b[9m";
+}
+
+inline void reset(std::ostream &out) {
+  out << "\x1b[0m";
+}
+
+void Buffer::print(BEd &ctx, uint64_t start_line, uint64_t end_line) {
+  std::optional<syntax::Parser::Iterator> it_o = parser.get_hl(vase, start_line - 1);
+  if (!it_o)
+    throw ed_error("shouldn't be possible if line existed, which is checked by print's callers.");
+  auto &it = *it_o;
+  while (start_line <= end_line) {
+    it.next();
+    const std::string &line = it.it->line;
+    const auto &tokens = it.tokens;
+    uint32_t cursor = 0;
+    for (const auto &token : tokens) {
+      const uint32_t start = token.start;
+      const uint32_t end = token.end;
+      if (start > line.size())
+        break;
+      if (end > line.size())
+        break;
+      if (cursor < start) {
+        std::cout.write(
+          line.data() + cursor,
+          start - cursor
+        );
+      }
+      const auto highlight = ctx.theme.get(token);
+      apply(std::cout, highlight);
+      std::cout.write(line.data() + start, end - start);
+      reset(std::cout);
+      cursor = end;
+    }
+    if (cursor < line.size())
+      std::cout.write(line.data() + cursor, line.size() - cursor);
+    std::cout << '\n';
+    ++start_line;
+  }
   prev_range.start = start_line;
   prev_range.end = end_line;
 }

src/internal/commands/commands.cc

@@ -8,7 +8,7 @@ void Suffix::register_suffixes(BEd &ctx) {
     .desc = "Prints current line.",
     .handle = [](BEd &ctx) {
       auto line = ctx.active->line;
-      ctx.active->print(line, line);
+      ctx.active->print(ctx, line, line);
     }
   };
 }
@@ -28,7 +28,7 @@ void Command::register_posix(BEd &ctx) {
       if (line == 0)
         throw ed_error("Line 0 is invalid.");
       ctx.active->jump(line);
-      ctx.active->print(line, line);
+      ctx.active->print(ctx, line, line);
     }
   };
   ctx.eof_op = Command{
@@ -93,11 +93,11 @@ void Command::register_posix(BEd &ctx) {
         uint64_t start_line = ctx.active->line;
         uint64_t end_line = ctx.active->line;
         if (!addresses.size())
-          ctx.active->print(start_line, end_line);
+          ctx.active->print(ctx, start_line, end_line);
         else if (addresses.size() == 1)
-          ctx.active->print(addresses[0], addresses[0]);
+          ctx.active->print(ctx, addresses[0], addresses[0]);
         else
-          ctx.active->print(addresses[0], addresses[1]);
+          ctx.active->print(ctx, addresses[0], addresses[1]);
       }
     }
   );

src/internal/syntax/parser.cc

@@ -0,0 +1,282 @@
+#include "internal/syntax/parser.h"
+
+namespace bed::internal::syntax {
+static void destroy_tree(ParseState *node, Language &lang) {
+  if (!node)
+    return;
+  if (node->is_branch()) {
+    auto *branch = (ParseStateBranch *)node;
+    destroy_tree(branch->left, lang);
+    destroy_tree(branch->right, lang);
+    delete branch;
+  } else {
+    auto *leaf = (ParseStateLeaf *)node;
+    if (leaf->state)
+      lang.destroy(leaf->state);
+    delete leaf;
+  }
+}
+
+static ParseState *make_branch(ParseState *left, ParseState *right) {
+  if (!left)
+    return right;
+  if (!right)
+    return left;
+  auto *branch = new ParseStateBranch;
+  branch->header =
+    ParseState::BRANCH_BIT + left->lines() + right->lines();
+  branch->left = left;
+  branch->right = right;
+  return branch;
+}
+
+static ParseState *build_tree(std::vector<ParseStateLeaf *> &leaves, size_t begin, size_t end) {
+  const size_t count = end - begin;
+  if (count == 0)
+    return nullptr;
+  if (count == 1)
+    return leaves[begin];
+  const size_t mid = begin + count / 2;
+  ParseState *left = build_tree(leaves, begin, mid);
+  ParseState *right = build_tree(leaves, mid, end);
+  return make_branch(left, right);
+}
+
+Parser::Parser(vase::Vase &vase, uint64_t lines, Language &lang)
+    : root(nullptr), lang(lang) {
+  reset(vase, lines, lang);
+}
+
+Parser::~Parser() {
+  destroy_tree(root, lang);
+}
+
+void Parser::reset(vase::Vase &vase, uint64_t lines, Language &lang_) {
+  destroy_tree(root, lang);
+  root = nullptr;
+  if (lines == 0)
+    return;
+  lang = lang_;
+  vase::Iterator it = vase.iterate(0, Direction::Forward);
+  it.next();
+  std::vector<ParseStateLeaf *> leaves;
+  std::vector<Token> tokens;
+  leaves.reserve((lines + 63) / 64);
+  void *state = lang.none_state();
+  uint32_t consumed = 0;
+  while (consumed < lines) {
+    ParseStateLeaf *leaf = new ParseStateLeaf;
+    leaf->header = 0;
+    leaf->state = lang.copy(state);
+    uint32_t chunk_lines = 0;
+    while (chunk_lines < 64 && consumed < lines) {
+      tokens.clear();
+      lang.parse(&state, it.line, consumed == 0, &tokens);
+      ++chunk_lines;
+      ++consumed;
+      if (!it.next() && consumed < lines)
+        break;
+    }
+    leaf->header = chunk_lines;
+    leaves.push_back(leaf);
+  }
+  lang.destroy(state);
+  root = build_tree(leaves, 0, leaves.size());
+}
+
+std::pair<ParseState *, ParseState *> Parser::split_tree(ParseState *node, uint64_t line) {
+  if (!node)
+    return {nullptr, nullptr};
+  if (line == 0)
+    return {nullptr, node};
+  if (line >= node->lines())
+    return {node, nullptr};
+  if (node->is_branch()) {
+    auto *branch = (ParseStateBranch *)node;
+    uint64_t left_lines = branch->left->lines();
+    if (line < left_lines) {
+      auto [a, b] = split_tree(branch->left, line);
+      ParseState *right = join_tree(b, branch->right);
+      delete branch;
+      return {a, right};
+    }
+    if (line == left_lines) {
+      ParseState *left = branch->left;
+      ParseState *right = branch->right;
+      delete branch;
+      return {left, right};
+    }
+    auto [a, b] = split_tree(branch->right, line - left_lines);
+    ParseState *left = join_tree(branch->left, a);
+    delete branch;
+    return {left, b};
+  } else {
+    auto *leaf = (ParseStateLeaf *)node;
+    uint64_t lines = leaf->lines();
+    auto *right = new ParseStateLeaf;
+    right->header = lines - line;
+    right->state = nullptr;
+    leaf->header = line;
+    return {leaf, right};
+  }
+}
+
+ParseState *Parser::join_tree(ParseState *a, ParseState *b) {
+  // TODO: balance
+  return make_branch(a, b);
+}
+
+void Parser::erase(vase::Vase &vase, uint64_t start, uint64_t count) {
+  if (count == 0 || !root)
+    return;
+  auto [a, remaining] = split_tree(root, start);
+  auto [waste, b] = split_tree(remaining, count);
+  destroy_tree(waste, lang);
+  root = join_tree(a, b);
+  modify(vase, start, 1);
+}
+
+void Parser::insert(vase::Vase &vase, uint64_t start, uint64_t count) {
+  if (count == 0)
+    return;
+  std::vector<ParseStateLeaf *> leaves;
+  leaves.reserve((count + 63) / 64);
+  uint64_t consumed = 0;
+  while (consumed < count) {
+    auto *leaf = new ParseStateLeaf;
+    leaf->state = nullptr;
+    uint64_t chunk = 0;
+    while (chunk < 64 && consumed < count) {
+      ++chunk;
+      ++consumed;
+      if (consumed < count)
+        break;
+    }
+    leaf->header = chunk;
+    leaves.push_back(leaf);
+  }
+  ParseState *subtree = build_tree(leaves, 0, leaves.size());
+  auto [left, right] = split_tree(root, start);
+  root = join_tree(join_tree(left, subtree), right);
+  modify(vase, start, count);
+}
+
+void Parser::modify(vase::Vase &vase, uint64_t target, uint64_t count) {
+  if (count == 0 || !root)
+    return;
+  std::vector<Token> tokens;
+  uint64_t offset;
+  TreeCursor c = TreeCursor(root, target, &offset);
+  uint64_t at = target - offset;
+  void *state = nullptr;
+  if (c.leaf->state) {
+    state = lang.copy(c.leaf->state);
+  } else {
+    while (!c.leaf->state) {
+      c.prev();
+      if (!c.leaf) {
+        at = 0;
+        break;
+      }
+      at -= c.leaf->lines();
+    }
+    if (c.leaf) {
+      state = lang.copy(c.leaf->state);
+    } else {
+      state = lang.none_state();
+      c = TreeCursor(root, 0, &offset);
+    }
+  }
+  vase::Iterator it = vase.iterate(at, Direction::Forward);
+  uint64_t next_boundary = at + c.leaf->lines();
+  while (true) {
+    it.next();
+    if (at == next_boundary) {
+      c.next();
+      if (!c.leaf)
+        break;
+      next_boundary += c.leaf->lines();
+      if (at >= target + count
+          && c.leaf->state != nullptr
+          && lang.equal(state, c.leaf->state))
+        break;
+      if (c.leaf->state)
+        lang.destroy(c.leaf->state);
+      c.leaf->state = lang.copy(state);
+    }
+    tokens.clear();
+    lang.parse(&state, it.line, at == 0, &tokens);
+    at++;
+  }
+  lang.destroy(state);
+}
+
+std::optional<Parser::Iterator> Parser::get_hl(vase::Vase &vase, uint64_t target) {
+  if (!root)
+    return std::nullopt;
+  return Parser::Iterator(target, this, vase);
+}
+
+Parser::Iterator::Iterator(uint64_t target, Parser *p, vase::Vase &vase) : p(p) {
+  uint64_t offset;
+  TreeCursor c = TreeCursor(p->root, target, &offset);
+  at = target - offset;
+  if (c.leaf->state) {
+    state = p->lang.copy(c.leaf->state);
+  } else {
+    while (!c.leaf->state) {
+      c.prev();
+      if (!c.leaf) {
+        at = 0;
+        break;
+      }
+      at -= c.leaf->lines();
+    }
+    if (c.leaf) {
+      state = p->lang.copy(c.leaf->state);
+    } else {
+      state = p->lang.none_state();
+      c = TreeCursor(p->root, 0, &offset);
+    }
+  }
+  it = vase.iterate(at, Direction::Forward);
+  while (at < target) {
+    it->next();
+    tokens.clear();
+    p->lang.parse(&state, it->line, at == 0, &tokens);
+    at++;
+  }
+}
+
+Parser::Iterator::~Iterator() {
+  if (state)
+    p->lang.destroy(state);
+}
+
+Parser::Iterator::Iterator(Iterator &&other)
+    : p(other.p),
+      it(std::move(other.it)),
+      state(other.state),
+      tokens(std::move(other.tokens)) {
+  other.state = nullptr;
+}
+
+Parser::Iterator &Parser::Iterator::operator=(Iterator &&other) {
+  if (this == &other)
+    return *this;
+  if (state)
+    p->lang.destroy(state);
+  p = other.p;
+  it = std::move(other.it);
+  state = other.state;
+  tokens = std::move(other.tokens);
+  other.state = nullptr;
+  return *this;
+}
+
+void Parser::Iterator::next() {
+  it->next();
+  tokens.clear();
+  p->lang.parse(&state, it->line, at == 0, &tokens);
+}
+} // namespace bed::internal::syntax

src/internal/syntax/ruby/parse.cc

@@ -0,0 +1,950 @@
+#include "internal/syntax/ruby/parser.h"
+
+namespace bed::internal::syntax::ruby {
+inline bool is_hex(char c) {
+  return ('0' <= c && c <= '9')
+         || ('a' <= c && c <= 'f')
+         || ('A' <= c && c <= 'F');
+};
+
+inline bool identifier_start_char(char c) {
+  return (c & 0x80)
+         || ('a' <= c && c <= 'z')
+         || ('A' <= c && c <= 'Z')
+         || c == '_';
+}
+
+inline bool identifier_char(char c) {
+  return (c & 0x80)
+         || ('a' <= c && c <= 'z')
+         || ('A' <= c && c <= 'Z')
+         || ('0' <= c && c <= '9')
+         || c == '_';
+}
+
+inline uint8_t utf8_codepoint_width(unsigned char c) {
+  if ((c & 0x80) == 0x00)
+    return 1;
+  if ((c & 0xE0) == 0xC0)
+    return 2;
+  if ((c & 0xF0) == 0xE0)
+    return 3;
+  if ((c & 0xF8) == 0xF0)
+    return 4;
+  return 1;
+}
+
+bool handle_escapes(RubyParser &p, std::vector<Token> *tokens, uint32_t &start, bool string = true) {
+  if (p.peek() == '\\') {
+    if (string)
+      tokens->push_back({start, p.i, Token::String});
+    else
+      tokens->push_back({start, p.i, Token::Regexp});
+    start = p.i;
+    p.advance();
+    if (p.peek() == 'x') {
+      p.advance();
+      if (is_hex(p.peek()))
+        p.advance();
+      if (is_hex(p.peek()))
+        p.advance();
+    } else if (p.peek() == 'u') {
+      p.advance();
+      if (p.peek() == '{') {
+        p.advance();
+        while (p.peek() != '}' && p.peek() != '\0')
+          p.advance();
+        if (p.peek() == '}')
+          p.advance();
+      } else {
+        if (is_hex(p.peek()))
+          p.advance();
+        if (is_hex(p.peek()))
+          p.advance();
+        if (is_hex(p.peek()))
+          p.advance();
+        if (is_hex(p.peek()))
+          p.advance();
+      }
+    } else if ('0' <= p.peek() && p.peek() <= '7') {
+      p.advance();
+      if ('0' <= p.peek() && p.peek() <= '7')
+        p.advance();
+      if ('0' <= p.peek() && p.peek() <= '7')
+        p.advance();
+    } else if (p.peek() == 'c') {
+      p.advance();
+      if (p.peek() != '\\')
+        p.advance();
+    } else if (p.peek() == 'M' || p.peek() == 'C') {
+      p.advance();
+      if (p.peek() == '-') {
+        p.advance();
+        if (p.peek() != '\\')
+          p.advance();
+      }
+    } else if (p.peek() == 'N') {
+      p.advance();
+      if (p.peek() == '{') {
+        p.advance();
+        while (p.peek() != '}' && p.peek() != '\0')
+          p.advance();
+        if (p.peek() == '}')
+          p.advance();
+      }
+    } else {
+      p.advance();
+    }
+    tokens->push_back({start, p.i, Token::Escape});
+    start = p.i;
+    return true;
+  }
+  return false;
+};
+
+bool handle_heredoc(RubyParser &p, std::vector<Token> *tokens) {
+  uint8_t *heredocs = p.state->heredocs();
+  uint32_t start = p.i;
+  if (start == 0) {
+    uint32_t heredoc_len = heredocs[0] & RubyState::Heredocs::LEN_MASK;
+    if (heredocs[0] & RubyState::Heredocs::ALLOW_INDENTATION)
+      while (start < p.len() && (p.line[start] == ' ' || p.line[start] == '\t'))
+        start++;
+    if (p.len() - start == heredoc_len
+        && memcmp(p.line.data() + start, heredocs + 1, heredoc_len) == 0) {
+      if (!p.dequeue_doc(heredoc_len))
+        p.current().state = RubyState::RubyInternalState::NONE;
+      tokens->push_back({p.i, p.len(), Token::Annotation});
+      return true;
+    }
+  }
+  if (!(heredocs[0] & RubyState::Heredocs::ALLOW_INTERPOLATION)) {
+    tokens->push_back({p.i, p.len(), Token::String});
+    return true;
+  } else {
+    while (p.i < p.len()) {
+      if (handle_escapes(p, tokens, start))
+        continue;
+      if (p.peek_str(2) == "#{") {
+        tokens->push_back({start, p.i, Token::String});
+        tokens->push_back({p.i, p.i + 2, Token::Interpolation});
+        p.advance(2);
+        p.push_state();
+        break;
+      }
+      p.advance();
+    }
+    if (p.i >= p.len())
+      tokens->push_back({start, p.len(), Token::String});
+    return false;
+  }
+}
+
+void handle_string(RubyParser &p, std::vector<Token> *tokens) {
+  uint32_t start = p.i;
+  while (p.i < p.len()) {
+    if (handle_escapes(p, tokens, start))
+      continue;
+    if ((p.current().flags & RubyState::RubyInternalState::ALLOW_INTERPOLATION)
+        && p.peek_str(2) == "#{") {
+      tokens->push_back({start, p.i, Token::String});
+      tokens->push_back({p.i, p.i + 2, Token::Interpolation});
+      p.advance(2);
+      p.push_state();
+      break;
+    }
+    if (p.peek() == p.current().delim_start
+        && p.current().delim_start != p.current().delim_end)
+      p.current().lit_brace_level++;
+    if (p.peek() == p.current().delim_end) {
+      if (p.current().delim_start == p.current().delim_end) {
+        p.advance();
+        tokens->push_back({start, p.i, Token::String});
+        p.current().state = RubyState::RubyInternalState::NONE;
+        p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+        return;
+      } else {
+        p.current().lit_brace_level--;
+        if (p.current().lit_brace_level == 0) {
+          p.advance();
+          tokens->push_back({start, p.i, Token::String});
+          p.current().state = RubyState::RubyInternalState::NONE;
+          p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+          return;
+        }
+      }
+    }
+    p.advance();
+  }
+  if (p.i >= p.len())
+    tokens->push_back({start, p.len(), Token::String});
+}
+
+void handle_regex(RubyParser &p, std::vector<Token> *tokens) {
+  uint32_t start = p.i;
+  while (p.i < p.len()) {
+    if (handle_escapes(p, tokens, start, false))
+      continue;
+    if ((p.current().flags & RubyState::RubyInternalState::ALLOW_INTERPOLATION)
+        && p.peek_str(2) == "#{") {
+      tokens->push_back({start, p.i, Token::Regexp});
+      tokens->push_back({p.i, p.i + 2, Token::Interpolation});
+      p.advance(2);
+      p.push_state();
+      break;
+    }
+    if (p.peek() == p.current().delim_start
+        && p.current().delim_start != p.current().delim_end)
+      p.current().lit_brace_level++;
+    if (p.peek() == p.current().delim_end) {
+      if (p.current().delim_start == p.current().delim_end) {
+        p.advance();
+        tokens->push_back({start, p.i, Token::Regexp});
+        p.current().state = RubyState::RubyInternalState::NONE;
+        p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+        return;
+      } else {
+        p.current().lit_brace_level--;
+        if (p.current().lit_brace_level == 0) {
+          p.advance();
+          tokens->push_back({start, p.i, Token::Regexp});
+          p.current().state = RubyState::RubyInternalState::NONE;
+          p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+          return;
+        }
+      }
+    }
+    p.advance();
+  }
+  if (p.i >= p.len())
+    tokens->push_back({start, p.len(), Token::Regexp});
+}
+
+bool handle_line_markers(RubyParser &p, std::vector<Token> *tokens) {
+  if (p.len() == 6 && p.peek_str(6) == "=begin") {
+    p.current().state = RubyState::RubyInternalState::COMMENT;
+    p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+    tokens->push_back({0, p.len(), Token::Comment});
+    return true;
+  }
+  if (p.len() == 7 && p.peek_str(7) == "__END__") {
+    p.current().state = RubyState::RubyInternalState::END;
+    p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+    return true;
+  }
+  return false;
+}
+
+bool handle_comment(RubyParser &p, std::vector<Token> *tokens, bool first_line) {
+  if (p.peek() == '#') {
+    p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+    if (first_line && p.i == 0 && p.peek(1) == '!') {
+      tokens->push_back({0, p.len(), Token::Shebang});
+      return true;
+    }
+    tokens->push_back({p.i, p.len(), Token::Comment});
+    return true;
+  }
+  return false;
+}
+
+void handle_syntax(RubyParser &p, std::vector<Token> *tokens) {
+  static const RubyTries tries = RubyTries();
+  if (p.i + 3 <= p.len() && p.peek_str(2) == "<<") {
+    uint32_t j = 2;
+    bool indented = false;
+    if (p.peek(j) == '~')
+      indented = true;
+    if (p.peek(j) == '~' || p.peek(j) == '-')
+      j++;
+    tokens->push_back({p.i, p.i + j, Token::Operator});
+    if (j >= p.len())
+      return;
+    std::string delim;
+    bool interpolation = true;
+    uint32_t s = p.i + j;
+    if (p.peek(j) == '\'' || p.peek(j) == '"') {
+      char q = p.peek(j++);
+      if (q == '\'')
+        interpolation = false;
+      while (j < p.len() && p.peek(j) != q)
+        delim += p.peek(j++);
+    } else {
+      if (j < p.len() && identifier_start_char(p.peek(j))) {
+        delim += p.peek(j++);
+        while (j < p.len() && identifier_char(p.peek(j)))
+          delim += p.peek(j++);
+      }
+    }
+    p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+    if (!delim.empty()) {
+      tokens->push_back({s, p.i + j, Token::Annotation});
+      uint8_t header = delim.size();
+      if (interpolation)
+        header |= RubyState::Heredocs::ALLOW_INTERPOLATION;
+      if (indented)
+        header |= RubyState::Heredocs::ALLOW_INDENTATION;
+      p.enqueue_doc(header, delim);
+      p.current().state = RubyState::RubyInternalState::HEREDOC;
+      p.heredoc_start_line = true;
+    }
+    p.advance(j);
+    return;
+  }
+  if (p.peek() == '/' && p.current().flags & RubyState::RubyInternalState::EXPECTING_EXPRESSION) {
+    tokens->push_back({p.i, p.i + 1, Token::Regexp});
+    p.current().state = RubyState::RubyInternalState::REGEXP;
+    p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+    p.current().flags |= RubyState::RubyInternalState::ALLOW_INTERPOLATION;
+    p.current().delim_start = '/';
+    p.current().delim_end = '/';
+    p.advance();
+    return;
+  }
+  switch (p.peek()) {
+  case '.': {
+    p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+    uint32_t start = p.i;
+    p.advance();
+    if (p.peek() == '.') {
+      p.advance();
+      if (p.peek() == '.')
+        p.advance();
+    }
+    tokens->push_back({start, p.i, Token::Operator});
+    return;
+  }
+  case ':': {
+    p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+    uint32_t start = p.i;
+    p.advance();
+    if (p.i >= p.len()) {
+      tokens->push_back({start, p.i, Token::Operator});
+      p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+      return;
+    }
+    if (p.peek() == ':') {
+      p.advance();
+      tokens->push_back({start, p.i, Token::Operator});
+      return;
+    }
+    if (p.peek() == '\'' || p.peek() == '"') {
+      tokens->push_back({start, p.i, Token::Label});
+      p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+      return;
+    }
+    if (p.peek() == '$' || p.peek() == '@') {
+      if (p.peek_str(2) == "@@")
+        p.advance(2);
+      else
+        p.advance();
+      while (identifier_char(p.peek()))
+        p.advance();
+      tokens->push_back({start, p.i, Token::Label});
+      return;
+    }
+    uint32_t op_len = tries.operator_trie.longest_match(p.peek_str(p.len() - p.i));
+    if (op_len > 0) {
+      tokens->push_back({start, p.i + op_len, Token::Label});
+      p.advance(op_len);
+      return;
+    }
+    if (identifier_start_char(p.peek())) {
+      p.advance();
+      while (identifier_char(p.peek()))
+        p.advance();
+      if (p.peek() == '!' || p.peek() == '?')
+        p.advance();
+      tokens->push_back({start, p.i, Token::Label});
+      return;
+    }
+    tokens->push_back({start, p.i, Token::Operator});
+    return;
+  }
+  case '@': {
+    p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+    uint32_t start = p.i;
+    p.advance();
+    if (p.i >= p.len())
+      return;
+    if (p.peek() == '@')
+      p.advance();
+    if (identifier_start_char(p.peek()))
+      p.advance();
+    else
+      return;
+    while (identifier_char(p.peek()))
+      p.advance();
+    tokens->push_back({start, p.i, Token::VariableInstance});
+    return;
+  }
+  case '$': {
+    p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+    uint32_t start = p.i;
+    p.advance();
+    if (p.i >= p.len())
+      return;
+    if (identifier_start_char(p.peek())) {
+      p.advance();
+      while (identifier_char(p.peek()))
+        p.advance();
+    } else if (p.i + 1 < p.len() && p.peek() == '-'
+               && (('a' <= p.peek(1) && p.peek(1) <= 'z') || ('A' <= p.peek(1) && p.peek(1) <= 'Z'))) {
+      p.advance(2);
+    } else if ('0' <= p.peek() && p.peek() <= '9') {
+      p.advance();
+      while ('0' <= p.peek() && p.peek() <= '9')
+        p.advance();
+    } else {
+      switch (p.peek()) {
+      case '~':
+      case '&':
+      case '`':
+      case '\'':
+      case '+':
+      case '=':
+      case '/':
+      case '\\':
+      case ',':
+      case ';':
+      case '.':
+      case '_':
+      case '*':
+      case '?':
+      case '!':
+      case '@':
+      case '<':
+      case '>':
+      case '$':
+        p.advance();
+        break;
+      default:
+        return;
+      }
+    }
+    tokens->push_back({start, p.i, Token::VariableGlobal});
+    return;
+  }
+  case '?': {
+    p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+    uint32_t start = p.i;
+    p.advance();
+    if (p.peek() == '\\') {
+    combination:
+      p.advance();
+      if (p.peek() == 'x') {
+        p.advance();
+        if (is_hex(p.peek()))
+          p.advance();
+        if (is_hex(p.peek()))
+          p.advance();
+        tokens->push_back({start, p.i, Token::Char});
+        return;
+      } else if (p.peek() == 'u') {
+        p.advance();
+        if (p.peek() == '{') {
+          p.advance();
+          while (p.peek() != '}' && p.peek() != '\0')
+            p.advance();
+          if (p.peek() == '}')
+            p.advance();
+        } else {
+          if (is_hex(p.peek()))
+            p.advance();
+          if (is_hex(p.peek()))
+            p.advance();
+          if (is_hex(p.peek()))
+            p.advance();
+          if (is_hex(p.peek()))
+            p.advance();
+        }
+        tokens->push_back({start, p.i, Token::Char});
+        return;
+      } else if ('0' <= p.peek() && p.peek() <= '7') {
+        p.advance();
+        if ('0' <= p.peek() && p.peek() <= '7')
+          p.advance();
+        if ('0' <= p.peek() && p.peek() <= '7')
+          p.advance();
+        tokens->push_back({start, p.i, Token::Char});
+        return;
+      } else if (p.peek() == 'c') {
+        p.advance();
+        if (p.peek() != '\\')
+          p.advance();
+        else
+          goto combination;
+        tokens->push_back({start, p.i, Token::Char});
+        return;
+      } else if (p.peek() == 'M' || p.peek() == 'C') {
+        p.advance();
+        if (p.peek() == '-') {
+          p.advance();
+          if (p.peek() != '\\')
+            p.advance();
+          else
+            goto combination;
+        }
+        tokens->push_back({start, p.i, Token::Char});
+        return;
+      } else if (p.peek() == 'N') {
+        p.advance();
+        if (p.peek() == '{') {
+          p.advance();
+          while (p.peek() != '}' && p.peek() != '\0')
+            p.advance();
+          if (p.peek() == '}')
+            p.advance();
+        }
+        tokens->push_back({start, p.i, Token::Char});
+        return;
+      } else {
+        p.advance();
+        tokens->push_back({start, p.i, Token::Char});
+        return;
+      }
+    } else if (p.peek() != '\0' && p.peek() != ' ' && p.peek() != '\t') {
+      p.advance();
+      tokens->push_back({start, p.i, Token::Char});
+      return;
+    } else {
+      p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+      tokens->push_back({start, p.i, Token::Operator});
+      return;
+    }
+  }
+  case '{': {
+    p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+    /*uint8_t brace_color =
+      (uint8_t)Token::K_BRACE1 + (state->full_state.brace_level % 5);
+    tokens->push_back({p.i, p.i + 1, (Token)brace_color});*/
+    p.current().brace_level++;
+    p.advance();
+    return;
+  }
+  case '}': {
+    p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+    if (!--p.current().brace_level && p.state->top > 1) {
+      p.pop_state();
+      tokens->push_back({p.i, p.i + 1, Token::Interpolation});
+    } else {
+      /*uint8_t brace_color =
+        (uint8_t)Token::K_BRACE1 + (state->full_state.brace_level % 5);
+      tokens->push_back({p.i, p.i + 1, (Token)brace_color});*/
+    }
+    p.advance();
+    return;
+  }
+  case '(': {
+    p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+    /*uint8_t brace_color =
+      (uint8_t)Token::K_BRACE1 + (state->full_state.brace_level % 5);
+    tokens->push_back({p.i, p.i + 1, (Token)brace_color});*/
+    p.current().brace_level++;
+    p.advance();
+    return;
+  }
+  case ')': {
+    p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+    p.current().brace_level--;
+    /*uint8_t brace_color =
+      (uint8_t)Token::K_BRACE1 + (state->full_state.brace_level % 5);
+    tokens->push_back({p.i, p.i + 1, (Token)brace_color});*/
+    p.advance();
+    return;
+  }
+  case '[': {
+    p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+    /*uint8_t brace_color =
+      (uint8_t)Token::K_BRACE1 + (state->full_state.brace_level % 5);
+    tokens->push_back({p.i, p.i + 1, (Token)brace_color});*/
+    p.current().brace_level++;
+    p.advance();
+    return;
+  }
+  case ']': {
+    p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+    p.current().brace_level--;
+    /*uint8_t brace_color =
+      (uint8_t)Token::K_BRACE1 + (state->full_state.brace_level % 5);
+    tokens->push_back({p.i, p.i + 1, (Token)brace_color});*/
+    p.advance();
+    return;
+  }
+  case '\'': {
+    p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+    tokens->push_back({p.i, p.i + 1, Token::String});
+    p.current().state = RubyState::RubyInternalState::STRING;
+    p.current().delim_start = '\'';
+    p.current().delim_end = '\'';
+    p.current().flags &= ~RubyState::RubyInternalState::ALLOW_INTERPOLATION;
+    p.advance();
+    return;
+  }
+  case '"': {
+    p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+    tokens->push_back({p.i, p.i + 1, Token::String});
+    p.current().state = RubyState::RubyInternalState::STRING;
+    p.current().delim_start = '"';
+    p.current().delim_end = '"';
+    p.current().flags |= RubyState::RubyInternalState::ALLOW_INTERPOLATION;
+    p.advance();
+    return;
+  }
+  case '`': {
+    p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+    tokens->push_back({p.i, p.i + 1, Token::String});
+    p.current().state = RubyState::RubyInternalState::STRING;
+    p.current().delim_start = '`';
+    p.current().delim_end = '`';
+    p.current().flags |= RubyState::RubyInternalState::ALLOW_INTERPOLATION;
+    p.advance();
+    return;
+  }
+  case '%': {
+    if (p.current().flags & RubyState::RubyInternalState::EXPECTING_EXPRESSION || p.i + 1 >= p.len()) {
+      tokens->push_back({p.i, p.i + 1, Token::Operator});
+      p.advance();
+      p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+      return;
+    }
+    p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+    char type = p.peek(1);
+    char delim_start = '\0';
+    char delim_end = '\0';
+    bool allow_interp = true;
+    int prefix_len = 1;
+    bool is_regexp = false;
+    switch (type) {
+    case 'r':
+      is_regexp = true;
+      allow_interp = true;
+      prefix_len = 2;
+      break;
+    case 'Q':
+    case 'x':
+    case 'I':
+    case 'W':
+      allow_interp = true;
+      prefix_len = 2;
+      break;
+    case 'w':
+    case 'q':
+    case 'i':
+    case 's':
+      allow_interp = false;
+      prefix_len = 2;
+      break;
+    default:
+      allow_interp = true;
+      prefix_len = 1;
+      break;
+    }
+    if (p.i + prefix_len >= p.len()) {
+      tokens->push_back({p.i, p.i + 1, Token::Operator});
+      p.advance(prefix_len);
+      p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+      return;
+    }
+    delim_start = p.peek(prefix_len);
+    if (identifier_char(delim_start) || delim_start == ' ') {
+      tokens->push_back({p.i, p.i + 1, Token::Operator});
+      p.advance(prefix_len);
+      p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+      return;
+    }
+    switch (delim_start) {
+    case '(':
+      delim_end = ')';
+      break;
+    case '{':
+      delim_end = '}';
+      break;
+    case '[':
+      delim_end = ']';
+      break;
+    case '<':
+      delim_end = '>';
+      break;
+    default:
+      delim_end = delim_start;
+      break;
+    }
+    tokens->push_back({p.i, p.i + prefix_len + 1, (is_regexp ? Token::Regexp : Token::String)});
+    p.current().state = is_regexp ? RubyState::RubyInternalState::REGEXP : RubyState::RubyInternalState::STRING;
+    p.current().delim_start = delim_start;
+    p.current().delim_end = delim_end;
+    if (allow_interp)
+      p.current().flags |= RubyState::RubyInternalState::ALLOW_INTERPOLATION;
+    p.current().lit_brace_level = 1;
+    p.advance(prefix_len + 1);
+    return;
+  }
+  default:
+    if ('0' <= p.peek() && p.peek() <= '9') {
+      p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+      uint32_t start = p.i;
+      if (p.peek() == '0') {
+        p.advance();
+        if (p.peek() == 'x' || p.peek() == 'X') {
+          p.advance();
+          while (true) {
+            while (is_hex(p.peek()))
+              p.advance();
+            if (p.peek() == '_')
+              p.advance();
+            else
+              break;
+          }
+        } else if (p.peek() == 'b' || p.peek() == 'B') {
+          p.advance();
+          while (true) {
+            while (p.peek() == '0' || p.peek() == '1')
+              p.advance();
+            if (p.peek() == '_')
+              p.advance();
+            else
+              break;
+          }
+        } else if (p.peek() == 'o' || p.peek() == 'O') {
+          p.advance();
+          while (true) {
+            while (p.peek() >= '0' && p.peek() <= '7')
+              p.advance();
+            if (p.peek() == '_')
+              p.advance();
+            else
+              break;
+          }
+        } else {
+          while (true) {
+            while (p.peek() >= '0' && p.peek() <= '7')
+              p.advance();
+            if (p.peek() == '_')
+              p.advance();
+            else
+              break;
+          }
+        }
+      } else {
+        while (true) {
+          while (p.peek() >= '0' && p.peek() <= '9')
+            p.advance();
+          if (p.peek() == '_')
+            p.advance();
+          else
+            break;
+        }
+        if (p.peek() == '.') {
+          p.advance();
+          while (true) {
+            while (p.peek() >= '0' && p.peek() <= '9')
+              p.advance();
+            if (p.peek() == '_')
+              p.advance();
+            else
+              break;
+          }
+        }
+        if (p.peek() == 'E' || p.peek() == 'e') {
+          p.advance();
+          if (p.peek() == '+' || p.peek() == '-')
+            p.advance();
+          while (true) {
+            while (p.peek() >= '0' && p.peek() <= '9')
+              p.advance();
+            if (p.peek() == '_')
+              p.advance();
+            else
+              break;
+          }
+        }
+      }
+      tokens->push_back({start, p.i, Token::Number});
+      return;
+    } else if (identifier_start_char(p.peek())) {
+      p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+      uint32_t j = 1;
+      while (identifier_char(p.peek(j)))
+        j++;
+      if (p.peek(j) == '!' || p.peek(j) == '?')
+        j++;
+      if (j == tries.base_keywords_trie.longest_match(p.peek_str(p.len() - p.i))) {
+        tokens->push_back({p.i, p.i + j, Token::Keyword});
+        p.advance(j);
+        return;
+      } else if (j == tries.expecting_keywords_trie.longest_match(p.peek_str(p.len() - p.i))) {
+        p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+        tokens->push_back({p.i, p.i + j, Token::Keyword});
+        p.advance(j);
+        return;
+      } else if (j == tries.operator_keywords_trie.longest_match(p.peek_str(p.len() - p.i))) {
+        tokens->push_back({p.i, p.i + j, Token::KeywordOperator});
+        p.advance(j);
+        return;
+      } else if (j == tries.expecting_operators_trie.longest_match(p.peek_str(p.len() - p.i))) {
+        p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+        tokens->push_back({p.i, p.i + j, Token::KeywordOperator});
+        p.advance(j);
+        return;
+      } else if (j == tries.types_trie.longest_match(p.peek_str(p.len() - p.i))) {
+        tokens->push_back({p.i, p.i + j, Token::Type});
+        p.advance(j);
+        return;
+      } else if (j == tries.methods_trie.longest_match(p.peek_str(p.len() - p.i))) {
+        tokens->push_back({p.i, p.i + j, Token::Function});
+        p.advance(j);
+        return;
+      } else if (j == tries.builtins_trie.longest_match(p.peek_str(p.len() - p.i))) {
+        tokens->push_back({p.i, p.i + j, Token::Constant});
+        p.advance(j);
+        return;
+      } else if (j == tries.errors_trie.longest_match(p.peek_str(p.len() - p.i))) {
+        tokens->push_back({p.i, p.i + j, Token::Error});
+        p.advance(j);
+        return;
+      } else if ('A' <= p.peek() && p.peek() <= 'Z' && !(p.peek(j) == '!' || p.peek(j) == '?')) {
+        tokens->push_back({p.i, p.i + j, Token::Constant});
+        p.advance(j);
+        return;
+      } else {
+        if (j == 4 && p.peek_str(4) == "true") {
+          tokens->push_back({p.i, p.i + j, Token::True});
+          p.advance(4);
+          return;
+        }
+        if (j == 5 && p.peek_str(5) == "false") {
+          tokens->push_back({p.i, p.i + j, Token::False});
+          p.advance(5);
+          return;
+        }
+        if (j == 3 && p.peek_str(3) == "def") {
+          tokens->push_back({p.i, p.i + j, Token::Keyword});
+          p.advance(3);
+          while (p.peek() == ' ' || p.peek() == '\t')
+            p.advance();
+          while (p.i < p.len()) {
+            if (identifier_start_char(p.peek())) {
+              uint32_t j = 1;
+              while (identifier_char(p.peek(j)))
+                j++;
+              if (p.peek(j) == '!' || p.peek(j) == '?')
+                j++;
+              if ('A' <= p.peek() && p.peek() <= 'Z')
+                tokens->push_back({p.i, p.i + j, Token::Constant});
+              else if (j == 4 && p.peek_str(4) == "self")
+                tokens->push_back({p.i, p.i + j, Token::Keyword});
+              else
+                tokens->push_back({p.i, p.i + j, Token::Function});
+              p.advance(j);
+              if (p.peek() == '.') {
+                p.advance();
+                continue;
+              }
+            }
+            break;
+          }
+          return;
+        }
+        uint32_t start = p.i;
+        p.advance(j);
+        if (p.peek() == ':') {
+          p.advance();
+          tokens->push_back({start, p.i, Token::Label});
+          return;
+        } else if (p.peek() == '!' || p.peek() == '?') {
+          p.advance();
+          tokens->push_back({start, p.i, Token::Function});
+          return;
+        } else {
+          uint32_t j = 0;
+          if (p.peek(j) == '(' || p.peek(j) == '{') {
+            tokens->push_back({start, p.i, Token::Function});
+            return;
+          } else if (p.peek(j) == ' ' || p.peek(j) == '\t') {
+            j++;
+          } else {
+            return;
+          }
+          while (p.peek(j) == ' ' || p.peek(j) == '\t')
+            j++;
+          if (p.i + j >= p.len())
+            return;
+          if (
+            p.peek(j) == '-'
+            || p.peek(j) == '&'
+            || p.peek(j) == '%'
+            || p.peek(j) == ':'
+          ) {
+            if (p.peek(j + 1) == ' ' || p.peek(j + 1) == '>')
+              return;
+          } else if (
+            p.peek(j) == ']'
+            || p.peek(j) == '}'
+            || p.peek(j) == ')'
+            || p.peek(j) == ','
+            || p.peek(j) == ';'
+            || p.peek(j) == '.'
+            || p.peek(j) == '+'
+            || p.peek(j) == '*'
+            || p.peek(j) == '/'
+            || p.peek(j) == '='
+            || p.peek(j) == '?'
+            || p.peek(j) == '|'
+            || p.peek(j) == '^'
+            || p.peek(j) == '<'
+            || p.peek(j) == '>'
+          ) {
+            return;
+          }
+          tokens->push_back({start, p.i, Token::Function});
+        }
+      }
+    } else {
+      uint32_t op_len;
+      if ((op_len = tries.operator_trie.longest_match(p.peek_str(p.len() - p.i)))) {
+        tokens->push_back({p.i, p.i + op_len, Token::Operator});
+        p.advance(op_len);
+        p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
+      } else {
+        p.advance(utf8_codepoint_width(p.peek()));
+      }
+    }
+  }
+}
+
+void ruby_parse(void **v_state, std::string_view line, bool fl, std::vector<Token> *tokens) {
+  RubyParser p(v_state, line);
+  while (p.i < p.len()) {
+    if (p.current().state == RubyState::RubyInternalState::END)
+      return;
+    if (p.current().state == RubyState::RubyInternalState::COMMENT) {
+      tokens->push_back({p.i, p.len(), Token::Comment});
+      if (p.i == 0 && p.peek_str(4) == "=end")
+        p.current().state = RubyState::RubyInternalState::NONE;
+      return;
+    }
+    if (!p.heredoc_start_line
+        && p.current().state == RubyState::RubyInternalState::HEREDOC) {
+      if (handle_heredoc(p, tokens))
+        return;
+      else
+        continue;
+    }
+    if (p.current().state == RubyState::RubyInternalState::STRING) {
+      handle_string(p, tokens);
+      continue;
+    }
+    if (p.current().state == RubyState::RubyInternalState::REGEXP) {
+      handle_regex(p, tokens);
+      continue;
+    }
+    if (!p.i && handle_line_markers(p, tokens))
+      return;
+    if (handle_comment(p, tokens, fl))
+      return;
+    handle_syntax(p, tokens);
+    continue;
+  }
+}
+} // namespace bed::internal::syntax::ruby

src/internal/syntax/ruby/ruby.cc

@@ -0,0 +1,44 @@
+#include "internal/syntax/ruby/parser.h"
+
+namespace bed::internal::syntax::ruby {
+Language lang_ruby() {
+  return Language{
+    .none_state = []() {
+      RubyState *st = (RubyState *)malloc(
+        sizeof(RubyState)
+        + sizeof(RubyState::RubyInternalState)
+      );
+      st->top = 1;
+      st->docs = 0;
+      st->stack()[0] = {
+        .brace_level = 1,
+        .lit_brace_level = 0,
+        .state = RubyState::RubyInternalState::NONE,
+        .flags = 0,
+        .delim_start = '\0',
+        .delim_end = '\0'
+      };
+      return st; },
+    .parse = ruby_parse,
+    .copy = [](void *v_i_st) {
+      RubyState *i_st = (RubyState *)v_i_st;
+      uint32_t bytes = sizeof(RubyState)
+                       + sizeof(RubyState::RubyInternalState) * i_st->top
+                       + i_st->docs;
+      RubyState *o_st = (RubyState *)malloc(bytes);
+      memcpy(o_st, i_st, bytes);
+      return o_st; },
+    .equal = [](void *v_a_st, void *v_b_st) {
+      RubyState *a_st = (RubyState *)v_a_st;
+      uint32_t a_bytes = sizeof(RubyState)
+                         + sizeof(RubyState::RubyInternalState) * a_st->top
+                         + a_st->docs;
+      RubyState *b_st = (RubyState *)v_b_st;
+      uint32_t b_bytes = sizeof(RubyState)
+                         + sizeof(RubyState::RubyInternalState) * b_st->top
+                         + b_st->docs;
+      return a_bytes == b_bytes && memcmp(a_st, b_st, a_bytes) == 0; },
+    .destroy = [](void *v_st) { free(v_st); },
+  };
+}
+} // namespace bed::internal::syntax::ruby

src/internal/theme/theme.cc

@@ -0,0 +1,162 @@
+#include "internal/theme/theme.h"
+
+namespace bed::internal::theme {
+Theme::Theme() {
+  hl.fill({
+    .fg = 0xF0F0F0,
+    .bg = 0x000000,
+    .flags = Highlight::None,
+  });
+}
+
+Highlight Theme::get(internal::syntax::Token token) const {
+  return hl[token.type];
+}
+
+Theme Theme::default_theme() {
+  Theme theme;
+  theme.hl[internal::syntax::Token::Shebang] = {
+    .fg = 0x7DCFFF,
+    .bg = 0x000000,
+    .flags = Highlight::None,
+  };
+  theme.hl[internal::syntax::Token::Error] = {
+    .fg = 0xEF5168,
+    .bg = 0x000000,
+    .flags = Highlight::None,
+  };
+  theme.hl[internal::syntax::Token::Comment] = {
+    .fg = 0xAAAAAA,
+    .bg = 0x000000,
+    .flags = Highlight::Italic,
+  };
+  theme.hl[internal::syntax::Token::String] = {
+    .fg = 0xAAD94C,
+    .bg = 0x000000,
+    .flags = Highlight::None,
+  };
+  theme.hl[internal::syntax::Token::Escape] = {
+    .fg = 0x7DCFFF,
+    .bg = 0x000000,
+    .flags = Highlight::None,
+  };
+  theme.hl[internal::syntax::Token::Interpolation] = {
+    .fg = 0x7DCFFF,
+    .bg = 0x000000,
+    .flags = Highlight::None,
+  };
+  theme.hl[internal::syntax::Token::Regexp] = {
+    .fg = 0xD2A6FF,
+    .bg = 0x000000,
+    .flags = Highlight::None,
+  };
+  theme.hl[internal::syntax::Token::Number] = {
+    .fg = 0xE6C08A,
+    .bg = 0x000000,
+    .flags = Highlight::None,
+  };
+  theme.hl[internal::syntax::Token::True] = {
+    .fg = 0x7AE93C,
+    .bg = 0x000000,
+    .flags = Highlight::None,
+  };
+  theme.hl[internal::syntax::Token::False] = {
+    .fg = 0xEF5168,
+    .bg = 0x000000,
+    .flags = Highlight::None,
+  };
+  theme.hl[internal::syntax::Token::Char] = {
+    .fg = 0xFFAF70,
+    .bg = 0x000000,
+    .flags = Highlight::None,
+  };
+  theme.hl[internal::syntax::Token::Keyword] = {
+    .fg = 0xFF8F40,
+    .bg = 0x000000,
+    .flags = Highlight::None,
+  };
+  theme.hl[internal::syntax::Token::KeywordOperator] = {
+    .fg = 0xF07178,
+    .bg = 0x000000,
+    .flags = Highlight::None,
+  };
+  theme.hl[internal::syntax::Token::Operator] = {
+    .fg = 0xFFFFFF,
+    .bg = 0x000000,
+    .flags = Highlight::Italic,
+  };
+  theme.hl[internal::syntax::Token::Function] = {
+    .fg = 0xFFAF70,
+    .bg = 0x000000,
+    .flags = Highlight::None,
+  };
+  theme.hl[internal::syntax::Token::Type] = {
+    .fg = 0xF07178,
+    .bg = 0x000000,
+    .flags = Highlight::None,
+  };
+  theme.hl[internal::syntax::Token::Constant] = {
+    .fg = 0x7DCFFF,
+    .bg = 0x000000,
+    .flags = Highlight::None,
+  };
+  theme.hl[internal::syntax::Token::VariableInstance] = {
+    .fg = 0x95E6CB,
+    .bg = 0x000000,
+    .flags = Highlight::None,
+  };
+  theme.hl[internal::syntax::Token::VariableGlobal] = {
+    .fg = 0xF07178,
+    .bg = 0x000000,
+    .flags = Highlight::None,
+  };
+  theme.hl[internal::syntax::Token::Annotation] = {
+    .fg = 0x7DCFFF,
+    .bg = 0x000000,
+    .flags = Highlight::None,
+  };
+  theme.hl[internal::syntax::Token::Directive] = {
+    .fg = 0xFF8F40,
+    .bg = 0x000000,
+    .flags = Highlight::None,
+  };
+  theme.hl[internal::syntax::Token::Label] = {
+    .fg = 0xD2A6FF,
+    .bg = 0x000000,
+    .flags = Highlight::None,
+  };
+  theme.hl[internal::syntax::Token::Brace1] = {
+    .fg = 0xD2A6FF,
+    .bg = 0x000000,
+    .flags = Highlight::None,
+  };
+  theme.hl[internal::syntax::Token::Brace2] = {
+    .fg = 0xFFAFAF,
+    .bg = 0x000000,
+    .flags = Highlight::None,
+  };
+  theme.hl[internal::syntax::Token::Brace3] = {
+    .fg = 0xFFFF00,
+    .bg = 0x000000,
+    .flags = Highlight::None,
+  };
+  theme.hl[internal::syntax::Token::Brace4] = {
+    .fg = 0x0FFF0F,
+    .bg = 0x000000,
+    .flags = Highlight::None,
+  };
+  theme.hl[internal::syntax::Token::Brace5] = {
+    .fg = 0xFF0F0F,
+    .bg = 0x000000,
+    .flags = Highlight::None,
+  };
+  return theme;
+}
+
+Theme Theme::from_name(std::string_view name) {
+  if (name == "default")
+    return default_theme();
+  throw std::runtime_error("Unknown theme: " + std::string(name));
+}
+
+} // namespace bed::internal::theme