Fix io system and highlighting.

include/bed.h

@@ -19,7 +19,7 @@ struct BEd {
   std::array<std::optional<internal::functions::Suffix>, 26> suffixes;
   internal::theme::Theme theme;
   std::unordered_map<std::string, internal::syntax::Language *> languages;
-  internal::io::IO &io;
+  internal::io::IO io;
   internal::vase::AppendStorage append{"/tmp"};
 
   bool help_mode = false;
@@ -39,7 +39,7 @@ struct BEd {
   internal::buffer::Range prev_2;
   internal::marks::MarksEngine marks;
 
-  BEd(std::vector<std::string> args, internal::io::IO &io);
+  BEd(std::vector<std::string> args);
   ~BEd();
 
   internal::buffer::Buffer &buffer(const std::string &);
@@ -47,6 +47,7 @@ struct BEd {
   internal::buffer::Range &prev();
   void mark(uint8_t, internal::buffer::Line);
 
+  void print_help();
   void handle(std::string_view cmd, bool eof);
   void run();
   void suffix_handle(char s);

include/definitions.h

@@ -15,17 +15,4 @@ struct fatal_error : std::runtime_error {
 struct ed_error : std::runtime_error {
   ed_error(std::string msg) : std::runtime_error(msg) {}
 };
-
-struct Highlight {
-  enum : uint8_t {
-    None = 0,
-    Bold = 1 << 0,
-    Italic = 1 << 1,
-    Strikethrough = 1 << 2,
-    Underline = 1 << 3,
-  };
-  uint32_t fg;
-  uint32_t bg;
-  uint8_t flags;
-};
 } // namespace bed

include/internal/io/io.h

@@ -2,6 +2,7 @@
 
 #include "definitions.h"
 #include "pch.h"
+#include "tokens.h"
 
 namespace bed::internal::io {
 struct KeyEvent {
@@ -57,6 +58,7 @@ struct IO {
   static void enable_raw();
   static volatile std::atomic_bool resized;
   static void handle_sigwinch(int);
+  BEd &bed;
 
   static enum struct Mode {
     PIPE,
@@ -66,7 +68,7 @@ struct IO {
   std::string pipe_input;
   std::deque<char> input_queue;
 
-  IO();
+  IO(BEd &);
   ~IO();
 
   IO(const IO &) = delete;
@@ -75,6 +77,8 @@ struct IO {
   bool interactive() const {
     return mode == Mode::TERMINAL;
   }
+  void apply(const io::Token::Kind &t);
+  void reset();
   void enable_mouse();
   void disable_mouse();
 

include/internal/io/tokens.h

@@ -0,0 +1,105 @@
+#pragma once
+
+#include "definitions.h"
+#include "pch.h"
+
+namespace bed::internal::io {
+struct Token {
+  uint32_t start;
+  uint32_t end;
+  enum Kind : uint8_t {
+    TempCurrent,
+    AddressSeperator,
+    Address,
+    Offset,
+    AddressRegex,
+    AddressSymbol,
+    Number,
+    Mark,
+    RubyFunction,
+    RubyArg,
+    Function,
+    Any,
+    Shell,
+    Ruby,
+    File,
+    Regex,
+    Replacement,
+    Suffix,
+    Color1,
+    Color2,
+    Color3,
+    Color4,
+    Color5,
+    Warning,
+    Data,
+    Shebang,
+    Comment,
+    Error,
+    String,
+    Escape,
+    Interpolation,
+    Regexp,
+    True,
+    False,
+    Char,
+    Keyword,
+    KeywordOperator,
+    Operator,
+    Namespace,
+    Class,
+    Module,
+    Type,
+    Constant,
+    VariableInstance,
+    VariableGlobal,
+    Annotation,
+    Directive,
+    Label,
+    Brace1,
+    Brace2,
+    Brace3,
+    Brace4,
+    Brace5,
+    Heading1,
+    Heading2,
+    Heading3,
+    Heading4,
+    Heading5,
+    Heading6,
+    Blockquote,
+    List,
+    ListItem,
+    Code,
+    LanguageName,
+    LinkLabel,
+    ImageLabel,
+    Link,
+    Table,
+    TableHeader,
+    Italic,
+    Bold,
+    Underline,
+    Strikethrough,
+    HorizontalRule,
+    Tag,
+    Attribute,
+    CheckDone,
+    CheckNotDone,
+    Count
+  } type;
+};
+
+struct Highlight {
+  enum : uint8_t {
+    None = 0,
+    Bold = 1 << 0,
+    Italic = 1 << 1,
+    Strikethrough = 1 << 2,
+    Underline = 1 << 3,
+  };
+  uint32_t fg;
+  uint32_t bg;
+  uint8_t flags;
+};
+} // namespace bed::internal::io

include/internal/syntax/decl.h

@@ -1,74 +1,11 @@
 #pragma once
 
+#include "internal/io/tokens.h"
 #include "internal/trie/trie.h"
 #include "internal/vase/vase.h"
 #include "pch.h"
 
 namespace bed::internal::syntax {
-struct Token {
-  uint32_t start;
-  uint32_t end;
-  enum Kind : uint8_t {
-    Data,
-    Shebang,
-    Comment,
-    Error,
-    String,
-    Escape,
-    Interpolation,
-    Regexp,
-    Number,
-    True,
-    False,
-    Char,
-    Keyword,
-    KeywordOperator,
-    Operator,
-    Function,
-    Namespace,
-    Class,
-    Module,
-    Type,
-    Constant,
-    VariableInstance,
-    VariableGlobal,
-    Annotation,
-    Directive,
-    Label,
-    Brace1,
-    Brace2,
-    Brace3,
-    Brace4,
-    Brace5,
-    Heading1,
-    Heading2,
-    Heading3,
-    Heading4,
-    Heading5,
-    Heading6,
-    Blockquote,
-    List,
-    ListItem,
-    Code,
-    LanguageName,
-    LinkLabel,
-    ImageLabel,
-    Link,
-    Table,
-    TableHeader,
-    Italic,
-    Bold,
-    Underline,
-    Strikethrough,
-    HorizontalRule,
-    Tag,
-    Attribute,
-    CheckDone,
-    CheckNotDone,
-    Count
-  } type;
-};
-
 struct ParseEvent {
   uint8_t closing;
   ParseEvent(uint8_t t) : closing(t) {}
@@ -78,7 +15,7 @@ struct Language {
   std::function<void *()> none_state;
   std::function<void(
     void **, std::string_view,
-    bool, std::vector<Token> *, std::vector<ParseEvent> *
+    bool, std::vector<io::Token> *, std::vector<ParseEvent> *
   )>
     parse;
   std::function<void *(void *)> copy;

include/internal/syntax/parser.h

@@ -21,7 +21,7 @@ struct Iterator {
   std::optional<vase::Iterator> it;
   void *state;
   uint64_t at;
-  std::vector<Token> tokens;
+  std::vector<io::Token> tokens;
   std::vector<ParseEvent> events;
   Iterator(uint64_t, ParserSnapshot, vase::Shard *);
   ~Iterator();

include/internal/syntax/ruby/parser.h

@@ -104,6 +104,6 @@ struct RubyParser {
 
 void ruby_parse(
   void **v_state, std::string_view line, bool first_line,
-  std::vector<Token> *tokens, std::vector<ParseEvent> *events
+  std::vector<io::Token> *tokens, std::vector<ParseEvent> *events
 );
 } // namespace bed::internal::syntax::ruby

include/internal/theme/theme.h

@@ -6,11 +6,11 @@
 
 namespace bed::internal::theme {
 struct Theme {
-  std::array<Highlight, internal::syntax::Token::Count> hl;
+  std::array<io::Highlight, io::Token::Count> hl;
 
   Theme();
 
-  Highlight get(internal::syntax::Token token) const;
+  io::Highlight get(const io::Token::Kind &token) const;
 
   static Theme default_theme();
   static Theme from_name(std::string_view name);

src/bed/run.cc

@@ -2,12 +2,8 @@
 #include "internal/parser/parser.h"
 
 namespace bed {
-BEd::BEd(std::vector<std::string> args, internal::io::IO &io)
+BEd::BEd(std::vector<std::string> args)
-    : theme(internal::theme::Theme::default_theme()), io(io) {
+    : theme(internal::theme::Theme::default_theme()), io(*this) {
-  internal::functions::Function::register_posix(*this);
-  internal::functions::Function::register_extented(*this);
-  internal::functions::Suffix::register_suffixes(*this);
-  languages["ruby"] = new internal::syntax::Language(internal::syntax::ruby::lang_ruby());
   std::string prompt_ = "";
   std::string file = "";
   bool suppress = false;
@@ -21,6 +17,9 @@ BEd::BEd(std::vector<std::string> args, internal::io::IO &io)
       suppress = true;
     } else if (args[i] == "-v" || args[i] == "--verbose") {
       help_mode = true;
+    } else if (args[i] == "-h" || args[i] == "--help") {
+      print_help();
+      throw fatal_error("", 0);
     } else {
       if (file.size())
         throw fatal_error("Invalid arguments given.", 1);
@@ -32,6 +31,10 @@ BEd::BEd(std::vector<std::string> args, internal::io::IO &io)
   else
     prompt_mode = false;
   suppress_mode = suppress;
+  internal::functions::Function::register_posix(*this);
+  internal::functions::Function::register_extented(*this);
+  internal::functions::Suffix::register_suffixes(*this);
+  languages["ruby"] = new internal::syntax::Language(internal::syntax::ruby::lang_ruby());
   buffers["clip"] = new internal::buffer::ClipBuffer("clip");
   current() = {"default", 0};
   try {
@@ -52,6 +55,10 @@ BEd::~BEd() {
     delete lang;
 }
 
+void BEd::print_help() {
+  io.write("BEd - A line editor.\n");
+}
+
 void BEd::run() {
   while (true) {
     internal::ui::CommandIO command(*this);
@@ -59,9 +66,11 @@ void BEd::run() {
     try {
       handle(cmd, eof);
     } catch (ed_error &e) {
+      io.apply(internal::io::Token::Warning);
       io.write_line("?");
       if (help_mode)
         io.write_line(e.what());
+      io.reset();
       last_help = e.what();
     }
   }

src/internal/buffer/shard.cc

@@ -50,32 +50,6 @@ uint64_t ShardBuffer::prev_closing(uint64_t start) {
   }
 }
 
-inline void apply(io::IO &io, const Highlight &hl) {
-  io.write("\x1b[0m");
-  const uint8_t r = (hl.fg >> 16) & 0xff;
-  const uint8_t g = (hl.fg >> 8) & 0xff;
-  const uint8_t b = hl.fg & 0xff;
-  io.write(std::format("\x1b[38;2;{};{};{}m", r, g, b));
-  if (hl.bg != 0) {
-    const uint8_t br = (hl.bg >> 16) & 0xff;
-    const uint8_t bg = (hl.bg >> 8) & 0xff;
-    const uint8_t bb = hl.bg & 0xff;
-    io.write(std::format("\x1b[48;2;{};{};{}m", br, bg, bb));
-  }
-  if (hl.flags & Highlight::Bold)
-    io.write("\x1b[1m");
-  if (hl.flags & Highlight::Italic)
-    io.write("\x1b[3m");
-  if (hl.flags & Highlight::Underline)
-    io.write("\x1b[4m");
-  if (hl.flags & Highlight::Strikethrough)
-    io.write("\x1b[9m");
-}
-
-inline void reset(io::IO &io) {
-  io.write("\x1b[0m");
-}
-
 void ShardBuffer::print(BEd &ctx, uint64_t start_line, uint64_t end_line) {
   ctx.prev().buffername = name;
   ctx.prev().start = start_line;
@@ -97,10 +71,9 @@ void ShardBuffer::print(BEd &ctx, uint64_t start_line, uint64_t end_line) {
           break;
         if (cursor < start)
           ctx.io.write(line.data() + cursor, start - cursor);
-        const auto highlight = ctx.theme.get(token);
+        ctx.io.apply(token.type);
-        apply(ctx.io, highlight);
         ctx.io.write(line.data() + start, end - start);
-        reset(ctx.io);
+        ctx.io.reset();
         cursor = end;
       }
       if (cursor < line.size())
@@ -141,10 +114,9 @@ void ShardBuffer::number_print(BEd &ctx, uint64_t start_line, uint64_t end_line)
           break;
         if (cursor < start)
           ctx.io.write(line.data() + cursor, start - cursor);
-        const auto highlight = ctx.theme.get(token);
+        ctx.io.apply(token.type);
-        apply(ctx.io, highlight);
         ctx.io.write(line.data() + start, end - start);
-        reset(ctx.io);
+        ctx.io.reset();
         cursor = end;
       }
       if (cursor < line.size())

src/internal/functions/extended.cc

@@ -3,28 +3,6 @@
 #include "internal/functions/suffixes.h"
 
 namespace bed::internal::functions {
-namespace ansi {
-constexpr auto reset = "\033[0m";
-constexpr auto bold = "\033[1m";
-constexpr auto cyan = "\033[36m";
-constexpr auto yellow = "\033[33m";
-constexpr auto magenta = "\033[35m";
-constexpr auto green = "\033[32m";
-constexpr auto blue = "\033[34m";
-} // namespace ansi
-
-static void append_field(std::string &msg, std::string_view label, std::string_view label_color, std::string_view value) {
-  if (value.empty())
-    return;
-  msg += "\n";
-  msg += ansi::bold;
-  msg += label_color;
-  msg += label;
-  msg += ansi::reset;
-  msg += ": ";
-  msg += value;
-}
-
 static std::string_view address_kind_str(Function::AddressKind k) {
   switch (k) {
   case Function::AddressKind::None:
@@ -79,17 +57,27 @@ static std::string_view argument_kind_str(Function::ArgumentKind a) {
   return "";
 }
 
-static std::string describe_function(const Function &func) {
+static void append_field(BEd &ctx, std::string_view label, io::Token::Kind color, std::string_view value) {
-  std::string msg = ansi::bold;
+  if (value.empty())
-  msg += func.desc;
+    return;
-  msg += ansi::reset;
+  ctx.io.write("\n");
-  append_field(msg, "Default address", ansi::cyan, func.default_address);
+  ctx.io.apply(color);
-  append_field(msg, "Address", ansi::yellow, address_kind_str(func.address_kind));
+  ctx.io.write(label);
+  ctx.io.reset();
+  ctx.io.write(": ");
+  ctx.io.write(value);
+}
+
+static void describe_function(BEd &ctx, const Function &func) {
+  ctx.io.write(func.desc);
+  ctx.io.reset();
+  append_field(ctx, "Default address", io::Token::Color1, func.default_address);
+  append_field(ctx, "Address", io::Token::Color2, address_kind_str(func.address_kind));
   if (func.accept_zero)
-    append_field(msg, "Zero address", ansi::magenta, "allowed");
+    append_field(ctx, "Zero address", io::Token::Color3, "allowed");
-  append_field(msg, "Input", ansi::green, input_mode_str(func.input_mode));
+  append_field(ctx, "Input", io::Token::Color4, input_mode_str(func.input_mode));
-  append_field(msg, "Argument", ansi::blue, argument_kind_str(func.argument_kind));
+  append_field(ctx, "Argument", io::Token::Color5, argument_kind_str(func.argument_kind));
-  return msg;
+  ctx.io.write("\n");
 }
 
 void Function::register_extented(BEd &ctx) {
@@ -110,14 +98,20 @@ void Function::register_extented(BEd &ctx) {
                   const Argument &arg_,
                   std::vector<buffer::Line> *
                 ) {
-        auto &str = std::get<std::string>(arg_);
+        auto str = std::get<std::string>(arg_);
+        const auto first = str.find_first_not_of(" \t");
+        const auto last = str.find_last_not_of(" \t");
+        if (first == std::string::npos)
+          str.clear();
+        else
+          str = str.substr(first, last - first + 1);
         if (str.empty()) {
-          ctx.io.write_line(describe_function(ctx.no_op));
+          describe_function(ctx, ctx.no_op);
           return;
         }
         auto len = ctx.functions.longest_match(str);
         if (len == str.size()) {
-          ctx.io.write_line(describe_function(*ctx.functions.get_ptr(str)));
+          describe_function(ctx, *ctx.functions.get_ptr(str));
           return;
         }
         throw ed_error("Not a valid function name.");

src/internal/functions/posix.cc

@@ -238,7 +238,9 @@ void Function::register_posix(BEd &ctx) {
                   const Argument &,
                   std::vector<buffer::Line> *
                 ) {
+        ctx.io.apply(internal::io::Token::Warning);
         ctx.io.write_line(ctx.last_help);
+        ctx.io.reset();
       }
     }
   );
@@ -260,8 +262,10 @@ void Function::register_posix(BEd &ctx) {
                   std::vector<buffer::Line> *
                 ) {
         ctx.help_mode = !ctx.help_mode;
+        ctx.io.apply(internal::io::Token::Warning);
         if (ctx.help_mode)
           ctx.io.write_line(ctx.last_help);
+        ctx.io.reset();
       }
     }
   );

src/internal/io/io.cc

@@ -1,4 +1,5 @@
 #include "internal/io/io.h"
+#include "bed.h"
 
 namespace bed::internal::io {
 termios IO::orig_termios{};
@@ -7,7 +8,7 @@ bool IO::cleaned = true;
 volatile std::atomic_bool IO::resized(false);
 IO::Mode IO::mode = IO::Mode::PIPE;
 
-IO::IO() {
+IO::IO(BEd &bed) : bed(bed) {
   if (!isatty(STDIN_FILENO))
     return;
   mode = Mode::TERMINAL;
@@ -33,6 +34,37 @@ IO::~IO() {
   cleanup();
 }
 
+void IO::apply(const io::Token::Kind &t) {
+  if (bed.suppress_mode)
+    return;
+  write("\x1b[0m");
+  const auto &hl = bed.theme.get(t);
+  const uint8_t r = (hl.fg >> 16) & 0xff;
+  const uint8_t g = (hl.fg >> 8) & 0xff;
+  const uint8_t b = hl.fg & 0xff;
+  write(std::format("\x1b[38;2;{};{};{}m", r, g, b));
+  if (hl.bg != 0) {
+    const uint8_t br = (hl.bg >> 16) & 0xff;
+    const uint8_t bg = (hl.bg >> 8) & 0xff;
+    const uint8_t bb = hl.bg & 0xff;
+    write(std::format("\x1b[48;2;{};{};{}m", br, bg, bb));
+  }
+  if (hl.flags & Highlight::Bold)
+    write("\x1b[1m");
+  if (hl.flags & Highlight::Italic)
+    write("\x1b[3m");
+  if (hl.flags & Highlight::Underline)
+    write("\x1b[4m");
+  if (hl.flags & Highlight::Strikethrough)
+    write("\x1b[9m");
+}
+
+void IO::reset() {
+  if (bed.suppress_mode)
+    return;
+  write("\x1b[0m");
+}
+
 void IO::enable_mouse() {
   if (mode == Mode::PIPE)
     throw fatal_error("no mouse in pipe mode.", 1);

src/internal/syntax/parser_state.cc

@@ -106,7 +106,7 @@ ParseState *ParseState::splice(
   if (!root || (line == 0 && root->lines() == original)) {
     vase::Iterator it(vase, 0, Direction::Forward);
     void *state = lang.none_state();
-    std::vector<Token> tokens;
+    std::vector<io::Token> tokens;
     std::vector<ParseEvent> events;
     ParsePieceBuilder builder(lang, 0);
     for (uint64_t at = 0; at < final; ++at) {
@@ -130,7 +130,7 @@ ParseState *ParseState::splice(
     state = lang.copy(c.leaf->state);
   }
   vase::Iterator it(vase, at, Direction::Forward);
-  std::vector<Token> tokens;
+  std::vector<io::Token> tokens;
   std::vector<ParseEvent> events;
   uint64_t end_in_tree = line + original;
   uint64_t end_extra;

src/internal/syntax/ruby/parse.cc

@@ -34,12 +34,12 @@ inline uint8_t utf8_codepoint_width(unsigned char c) {
   return 1;
 }
 
-bool handle_escapes(RubyParser &p, std::vector<Token> *tokens, uint32_t &start, bool string = true) {
+bool handle_escapes(RubyParser &p, std::vector<io::Token> *tokens, uint32_t &start, bool string = true) {
   if (p.peek() == '\\') {
     if (string)
-      tokens->push_back({start, p.i, Token::String});
+      tokens->push_back({start, p.i, io::Token::String});
     else
-      tokens->push_back({start, p.i, Token::Regexp});
+      tokens->push_back({start, p.i, io::Token::Regexp});
     start = p.i;
     p.advance();
     if (p.peek() == 'x') {
@@ -95,14 +95,14 @@ bool handle_escapes(RubyParser &p, std::vector<Token> *tokens, uint32_t &start,
     } else {
       p.advance();
     }
-    tokens->push_back({start, p.i, Token::Escape});
+    tokens->push_back({start, p.i, io::Token::Escape});
     start = p.i;
     return true;
   }
   return false;
 };
 
-bool handle_heredoc(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseEvent> *events) {
+bool handle_heredoc(RubyParser &p, std::vector<io::Token> *tokens, std::vector<ParseEvent> *events) {
   uint8_t *heredocs = p.state->heredocs();
   uint32_t start = p.i;
   if (start == 0) {
@@ -115,21 +115,21 @@ bool handle_heredoc(RubyParser &p, std::vector<Token> *tokens, std::vector<Parse
       if (!p.dequeue_doc(heredoc_len))
         p.current().state = RubyState::RubyInternalState::NONE;
       events->push_back(true);
-      tokens->push_back({p.i, p.len(), Token::Annotation});
+      tokens->push_back({p.i, p.len(), io::Token::Annotation});
       p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
       return true;
     }
   }
   if (!(heredocs[0] & RubyState::Heredocs::ALLOW_INTERPOLATION)) {
-    tokens->push_back({p.i, p.len(), Token::String});
+    tokens->push_back({p.i, p.len(), io::Token::String});
     return true;
   } else {
     while (p.i < p.len()) {
       if (handle_escapes(p, tokens, start))
         continue;
       if (p.peek_str(2) == "#{") {
-        tokens->push_back({start, p.i, Token::String});
+        tokens->push_back({start, p.i, io::Token::String});
-        tokens->push_back({p.i, p.i + 2, Token::Interpolation});
+        tokens->push_back({p.i, p.i + 2, io::Token::Interpolation});
         p.advance(2);
         p.push_state();
         return false;
@@ -137,20 +137,20 @@ bool handle_heredoc(RubyParser &p, std::vector<Token> *tokens, std::vector<Parse
       p.advance();
     }
     if (p.i >= p.len())
-      tokens->push_back({start, p.len(), Token::String});
+      tokens->push_back({start, p.len(), io::Token::String});
     return true;
   }
 }
 
-void handle_string(RubyParser &p, std::vector<Token> *tokens) {
+void handle_string(RubyParser &p, std::vector<io::Token> *tokens) {
   uint32_t start = p.i;
   while (p.i < p.len()) {
     if (handle_escapes(p, tokens, start))
       continue;
     if ((p.current().flags & RubyState::RubyInternalState::ALLOW_INTERPOLATION)
         && p.peek_str(2) == "#{") {
-      tokens->push_back({start, p.i, Token::String});
+      tokens->push_back({start, p.i, io::Token::String});
-      tokens->push_back({p.i, p.i + 2, Token::Interpolation});
+      tokens->push_back({p.i, p.i + 2, io::Token::Interpolation});
       p.advance(2);
       p.push_state();
       return;
@@ -161,7 +161,7 @@ void handle_string(RubyParser &p, std::vector<Token> *tokens) {
     if (p.peek() == p.current().delim_end) {
       if (p.current().delim_start == p.current().delim_end) {
         p.advance();
-        tokens->push_back({start, p.i, Token::String});
+        tokens->push_back({start, p.i, io::Token::String});
         p.current().state = RubyState::RubyInternalState::NONE;
         p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
         return;
@@ -169,7 +169,7 @@ void handle_string(RubyParser &p, std::vector<Token> *tokens) {
         p.current().lit_brace_level--;
         if (p.current().lit_brace_level == 0) {
           p.advance();
-          tokens->push_back({start, p.i, Token::String});
+          tokens->push_back({start, p.i, io::Token::String});
           p.current().state = RubyState::RubyInternalState::NONE;
           p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
           return;
@@ -179,18 +179,18 @@ void handle_string(RubyParser &p, std::vector<Token> *tokens) {
     p.advance();
   }
   if (p.i >= p.len())
-    tokens->push_back({start, p.len(), Token::String});
+    tokens->push_back({start, p.len(), io::Token::String});
 }
 
-void handle_regex(RubyParser &p, std::vector<Token> *tokens) {
+void handle_regex(RubyParser &p, std::vector<io::Token> *tokens) {
   uint32_t start = p.i;
   while (p.i < p.len()) {
     if (handle_escapes(p, tokens, start, false))
       continue;
     if ((p.current().flags & RubyState::RubyInternalState::ALLOW_INTERPOLATION)
         && p.peek_str(2) == "#{") {
-      tokens->push_back({start, p.i, Token::Regexp});
+      tokens->push_back({start, p.i, io::Token::Regexp});
-      tokens->push_back({p.i, p.i + 2, Token::Interpolation});
+      tokens->push_back({p.i, p.i + 2, io::Token::Interpolation});
       p.advance(2);
       p.push_state();
       return;
@@ -201,7 +201,7 @@ void handle_regex(RubyParser &p, std::vector<Token> *tokens) {
     if (p.peek() == p.current().delim_end) {
       if (p.current().delim_start == p.current().delim_end) {
         p.advance();
-        tokens->push_back({start, p.i, Token::Regexp});
+        tokens->push_back({start, p.i, io::Token::Regexp});
         p.current().state = RubyState::RubyInternalState::NONE;
         p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
         return;
@@ -209,7 +209,7 @@ void handle_regex(RubyParser &p, std::vector<Token> *tokens) {
         p.current().lit_brace_level--;
         if (p.current().lit_brace_level == 0) {
           p.advance();
-          tokens->push_back({start, p.i, Token::Regexp});
+          tokens->push_back({start, p.i, io::Token::Regexp});
           p.current().state = RubyState::RubyInternalState::NONE;
           p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
           return;
@@ -219,15 +219,15 @@ void handle_regex(RubyParser &p, std::vector<Token> *tokens) {
     p.advance();
   }
   if (p.i >= p.len())
-    tokens->push_back({start, p.len(), Token::Regexp});
+    tokens->push_back({start, p.len(), io::Token::Regexp});
 }
 
-bool handle_line_markers(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseEvent> *events) {
+bool handle_line_markers(RubyParser &p, std::vector<io::Token> *tokens, std::vector<ParseEvent> *events) {
   if (p.len() == 6 && p.peek_str(6) == "=begin") {
     p.current().state = RubyState::RubyInternalState::COMMENT;
     p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
     events->push_back(false);
-    tokens->push_back({0, p.len(), Token::Comment});
+    tokens->push_back({0, p.len(), io::Token::Comment});
     return true;
   }
   if (p.len() == 7 && p.peek_str(7) == "__END__") {
@@ -238,20 +238,20 @@ bool handle_line_markers(RubyParser &p, std::vector<Token> *tokens, std::vector<
   return false;
 }
 
-bool handle_comment(RubyParser &p, std::vector<Token> *tokens, bool first_line) {
+bool handle_comment(RubyParser &p, std::vector<io::Token> *tokens, bool first_line) {
   if (p.peek() == '#') {
     p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
     if (first_line && p.i == 0 && p.peek(1) == '!') {
-      tokens->push_back({0, p.len(), Token::Shebang});
+      tokens->push_back({0, p.len(), io::Token::Shebang});
       return true;
     }
-    tokens->push_back({p.i, p.len(), Token::Comment});
+    tokens->push_back({p.i, p.len(), io::Token::Comment});
     return true;
   }
   return false;
 }
 
-bool handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseEvent> *events) {
+bool handle_syntax(RubyParser &p, std::vector<io::Token> *tokens, std::vector<ParseEvent> *events) {
   static const RubyTries tries = RubyTries();
   if (p.peek() == ' ' || p.peek() == '\t') {
     while (p.peek() == ' ' || p.peek() == '\t')
@@ -277,7 +277,7 @@ bool handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
         j++;
         while (identifier_char(p.peek(j)))
           j++;
-        tokens->push_back({p.i, p.i + j, Token::Constant});
+        tokens->push_back({p.i, p.i + j, io::Token::Constant});
         p.advance(j);
         while (p.peek() == ' ' || p.peek() == '\t')
           p.advance();
@@ -299,11 +299,11 @@ bool handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
         if (p.peek(j) == '!' || p.peek(j) == '?')
           j++;
         if ('A' <= p.peek() && p.peek() <= 'Z')
-          tokens->push_back({p.i, p.i + j, Token::Constant});
+          tokens->push_back({p.i, p.i + j, io::Token::Constant});
         else if (j == 4 && p.peek_str(4) == "self")
-          tokens->push_back({p.i, p.i + j, Token::Keyword});
+          tokens->push_back({p.i, p.i + j, io::Token::Keyword});
         else
-          tokens->push_back({p.i, p.i + j, Token::Function});
+          tokens->push_back({p.i, p.i + j, io::Token::Function});
         p.advance(j);
         while (p.peek() == ' ' || p.peek() == '\t')
           p.advance();
@@ -322,7 +322,7 @@ bool handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
         j++;
         while (identifier_char(p.peek(j)))
           j++;
-        tokens->push_back({p.i, p.i + j, Token::Constant});
+        tokens->push_back({p.i, p.i + j, io::Token::Constant});
         p.advance(j);
         while (p.peek() == ' ' || p.peek() == '\t')
           p.advance();
@@ -344,7 +344,7 @@ bool handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
       indented = true;
     if (p.peek(j) == '~' || p.peek(j) == '-')
       j++;
-    tokens->push_back({p.i, p.i + j, Token::Operator});
+    tokens->push_back({p.i, p.i + j, io::Token::Operator});
     if (p.i + j >= p.len())
       return false;
     std::string delim;
@@ -368,7 +368,7 @@ bool handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
     p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
     if (!delim.empty()) {
       events->push_back(false);
-      tokens->push_back({s, p.i + j, Token::Annotation});
+      tokens->push_back({s, p.i + j, io::Token::Annotation});
       uint8_t header = delim.size();
       if (interpolation)
         header |= RubyState::Heredocs::ALLOW_INTERPOLATION;
@@ -382,7 +382,7 @@ bool handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
     return false;
   }
   if (p.peek() == '/' && p.current().flags & RubyState::RubyInternalState::EXPECTING_EXPRESSION) {
-    tokens->push_back({p.i, p.i + 1, Token::Regexp});
+    tokens->push_back({p.i, p.i + 1, io::Token::Regexp});
     p.current().state = RubyState::RubyInternalState::REGEXP;
     p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
     p.current().flags |= RubyState::RubyInternalState::ALLOW_INTERPOLATION;
@@ -401,7 +401,7 @@ bool handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
       if (p.peek() == '.')
         p.advance();
     }
-    tokens->push_back({start, p.i, Token::Operator});
+    tokens->push_back({start, p.i, io::Token::Operator});
     return false;
   }
   case ':': {
@@ -409,17 +409,17 @@ bool handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
     uint32_t start = p.i;
     p.advance();
     if (p.i >= p.len()) {
-      tokens->push_back({start, p.i, Token::Operator});
+      tokens->push_back({start, p.i, io::Token::Operator});
       p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
       return false;
     }
     if (p.peek() == ':') {
       p.advance();
-      tokens->push_back({start, p.i, Token::Operator});
+      tokens->push_back({start, p.i, io::Token::Operator});
       return false;
     }
     if (p.peek() == '\'' || p.peek() == '"') {
-      tokens->push_back({start, p.i, Token::Label});
+      tokens->push_back({start, p.i, io::Token::Label});
       p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
       return false;
     }
@@ -430,12 +430,12 @@ bool handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
         p.advance();
       while (identifier_char(p.peek()))
         p.advance();
-      tokens->push_back({start, p.i, Token::Label});
+      tokens->push_back({start, p.i, io::Token::Label});
       return false;
     }
     uint32_t op_len = tries.operator_trie.longest_match(p.peek_str(p.len() - p.i));
     if (op_len > 0) {
-      tokens->push_back({start, p.i + op_len, Token::Label});
+      tokens->push_back({start, p.i + op_len, io::Token::Label});
       p.advance(op_len);
       return false;
     }
@@ -445,10 +445,10 @@ bool handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
         p.advance();
       if (p.peek() == '!' || p.peek() == '?')
         p.advance();
-      tokens->push_back({start, p.i, Token::Label});
+      tokens->push_back({start, p.i, io::Token::Label});
       return false;
     }
-    tokens->push_back({start, p.i, Token::Operator});
+    tokens->push_back({start, p.i, io::Token::Operator});
     return false;
   }
   case '@': {
@@ -465,7 +465,7 @@ bool handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
       return false;
     while (identifier_char(p.peek()))
       p.advance();
-    tokens->push_back({start, p.i, Token::VariableInstance});
+    tokens->push_back({start, p.i, io::Token::VariableInstance});
     return false;
   }
   case '$': {
@@ -512,7 +512,7 @@ bool handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
         return false;
       }
     }
-    tokens->push_back({start, p.i, Token::VariableGlobal});
+    tokens->push_back({start, p.i, io::Token::VariableGlobal});
     return false;
   }
   case '?': {
@@ -528,7 +528,7 @@ bool handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
           p.advance();
         if (is_hex(p.peek()))
           p.advance();
-        tokens->push_back({start, p.i, Token::Char});
+        tokens->push_back({start, p.i, io::Token::Char});
         return false;
       } else if (p.peek() == 'u') {
         p.advance();
@@ -548,7 +548,7 @@ bool handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
           if (is_hex(p.peek()))
             p.advance();
         }
-        tokens->push_back({start, p.i, Token::Char});
+        tokens->push_back({start, p.i, io::Token::Char});
         return false;
       } else if ('0' <= p.peek() && p.peek() <= '7') {
         p.advance();
@@ -556,7 +556,7 @@ bool handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
           p.advance();
         if ('0' <= p.peek() && p.peek() <= '7')
           p.advance();
-        tokens->push_back({start, p.i, Token::Char});
+        tokens->push_back({start, p.i, io::Token::Char});
         return false;
       } else if (p.peek() == 'c') {
         p.advance();
@@ -564,7 +564,7 @@ bool handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
           p.advance();
         else
           goto combination;
-        tokens->push_back({start, p.i, Token::Char});
+        tokens->push_back({start, p.i, io::Token::Char});
         return false;
       } else if (p.peek() == 'M' || p.peek() == 'C') {
         p.advance();
@@ -575,7 +575,7 @@ bool handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
           else
             goto combination;
         }
-        tokens->push_back({start, p.i, Token::Char});
+        tokens->push_back({start, p.i, io::Token::Char});
         return false;
       } else if (p.peek() == 'N') {
         p.advance();
@@ -586,28 +586,28 @@ bool handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
           if (p.peek() == '}')
             p.advance();
         }
-        tokens->push_back({start, p.i, Token::Char});
+        tokens->push_back({start, p.i, io::Token::Char});
         return false;
       } else {
         p.advance();
-        tokens->push_back({start, p.i, Token::Char});
+        tokens->push_back({start, p.i, io::Token::Char});
         return false;
       }
     } else if (p.peek() != '\0' && p.peek() != ' ' && p.peek() != '\t') {
       p.advance();
-      tokens->push_back({start, p.i, Token::Char});
+      tokens->push_back({start, p.i, io::Token::Char});
       return false;
     } else {
       p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
-      tokens->push_back({start, p.i, Token::Operator});
+      tokens->push_back({start, p.i, io::Token::Operator});
       return false;
     }
   }
   case '{': {
     p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
     uint8_t brace_color =
-      (uint8_t)Token::Brace1 + (p.current().brace_level % 5);
+      (uint8_t)io::Token::Brace1 + (p.current().brace_level % 5);
-    tokens->push_back({p.i, p.i + 1, (Token::Kind)brace_color});
+    tokens->push_back({p.i, p.i + 1, (io::Token::Kind)brace_color});
     p.current().brace_level++;
     p.advance();
     return false;
@@ -616,11 +616,11 @@ bool handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
     p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
     if (!--p.current().brace_level && p.state->top > 1) {
       p.pop_state();
-      tokens->push_back({p.i, p.i + 1, Token::Interpolation});
+      tokens->push_back({p.i, p.i + 1, io::Token::Interpolation});
     } else {
       uint8_t brace_color =
-        (uint8_t)Token::Brace1 + (p.current().brace_level % 5);
+        (uint8_t)io::Token::Brace1 + (p.current().brace_level % 5);
-      tokens->push_back({p.i, p.i + 1, (Token::Kind)brace_color});
+      tokens->push_back({p.i, p.i + 1, (io::Token::Kind)brace_color});
     }
     p.advance();
     return false;
@@ -628,8 +628,8 @@ bool handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
   case '(': {
     p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
     uint8_t brace_color =
-      (uint8_t)Token::Brace1 + (p.current().brace_level % 5);
+      (uint8_t)io::Token::Brace1 + (p.current().brace_level % 5);
-    tokens->push_back({p.i, p.i + 1, (Token::Kind)brace_color});
+    tokens->push_back({p.i, p.i + 1, (io::Token::Kind)brace_color});
     p.current().brace_level++;
     p.advance();
     return false;
@@ -638,16 +638,16 @@ bool handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
     p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
     p.current().brace_level--;
     uint8_t brace_color =
-      (uint8_t)Token::Brace1 + (p.current().brace_level % 5);
+      (uint8_t)io::Token::Brace1 + (p.current().brace_level % 5);
-    tokens->push_back({p.i, p.i + 1, (Token::Kind)brace_color});
+    tokens->push_back({p.i, p.i + 1, (io::Token::Kind)brace_color});
     p.advance();
     return false;
   }
   case '[': {
     p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
     uint8_t brace_color =
-      (uint8_t)Token::Brace1 + (p.current().brace_level % 5);
+      (uint8_t)io::Token::Brace1 + (p.current().brace_level % 5);
-    tokens->push_back({p.i, p.i + 1, (Token::Kind)brace_color});
+    tokens->push_back({p.i, p.i + 1, (io::Token::Kind)brace_color});
     p.current().brace_level++;
     p.advance();
     return false;
@@ -656,14 +656,14 @@ bool handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
     p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
     p.current().brace_level--;
     uint8_t brace_color =
-      (uint8_t)Token::Brace1 + (p.current().brace_level % 5);
+      (uint8_t)io::Token::Brace1 + (p.current().brace_level % 5);
-    tokens->push_back({p.i, p.i + 1, (Token::Kind)brace_color});
+    tokens->push_back({p.i, p.i + 1, (io::Token::Kind)brace_color});
     p.advance();
     return false;
   }
   case '\'': {
     p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
-    tokens->push_back({p.i, p.i + 1, Token::String});
+    tokens->push_back({p.i, p.i + 1, io::Token::String});
     p.current().state = RubyState::RubyInternalState::STRING;
     p.current().delim_start = '\'';
     p.current().delim_end = '\'';
@@ -673,7 +673,7 @@ bool handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
   }
   case '"': {
     p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
-    tokens->push_back({p.i, p.i + 1, Token::String});
+    tokens->push_back({p.i, p.i + 1, io::Token::String});
     p.current().state = RubyState::RubyInternalState::STRING;
     p.current().delim_start = '"';
     p.current().delim_end = '"';
@@ -683,7 +683,7 @@ bool handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
   }
   case '`': {
     p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
-    tokens->push_back({p.i, p.i + 1, Token::String});
+    tokens->push_back({p.i, p.i + 1, io::Token::String});
     p.current().state = RubyState::RubyInternalState::STRING;
     p.current().delim_start = '`';
     p.current().delim_end = '`';
@@ -693,7 +693,7 @@ bool handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
   }
   case '%': {
     if (p.i + 1 >= p.len()) {
-      tokens->push_back({p.i, p.i + 1, Token::Operator});
+      tokens->push_back({p.i, p.i + 1, io::Token::Operator});
       p.advance();
       p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
       return false;
@@ -731,14 +731,14 @@ bool handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
       break;
     }
     if (p.i + prefix_len >= p.len()) {
-      tokens->push_back({p.i, p.i + 1, Token::Operator});
+      tokens->push_back({p.i, p.i + 1, io::Token::Operator});
       p.advance(prefix_len);
       p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
       return false;
     }
     delim_start = p.peek(prefix_len);
     if (identifier_char(delim_start) || delim_start == ' ') {
-      tokens->push_back({p.i, p.i + 1, Token::Operator});
+      tokens->push_back({p.i, p.i + 1, io::Token::Operator});
       p.advance(prefix_len);
       p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
       return false;
@@ -760,7 +760,7 @@ bool handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
       delim_end = delim_start;
       break;
     }
-    tokens->push_back({p.i, p.i + prefix_len + 1, (is_regexp ? Token::Regexp : Token::String)});
+    tokens->push_back({p.i, p.i + prefix_len + 1, (is_regexp ? io::Token::Regexp : io::Token::String)});
     p.current().state = is_regexp
                           ? RubyState::RubyInternalState::REGEXP
                           : RubyState::RubyInternalState::STRING;
@@ -862,7 +862,7 @@ bool handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
           }
         }
       }
-      tokens->push_back({start, p.i, Token::Number});
+      tokens->push_back({start, p.i, io::Token::Number});
       return false;
     } else if (identifier_start_char(p.peek())) {
       uint32_t j = 1;
@@ -872,93 +872,93 @@ bool handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
         j++;
       if (j == tries.base_keywords_trie.longest_match(p.peek_str(j))) {
         p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
-        tokens->push_back({p.i, p.i + j, Token::Keyword});
+        tokens->push_back({p.i, p.i + j, io::Token::Keyword});
         p.advance(j);
         return false;
       } else if (j == tries.expecting_keywords_trie.longest_match(p.peek_str(j))) {
         p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
-        tokens->push_back({p.i, p.i + j, Token::Keyword});
+        tokens->push_back({p.i, p.i + j, io::Token::Keyword});
         p.advance(j);
         return false;
       } else if (j == tries.operator_keywords_trie.longest_match(p.peek_str(j))) {
         p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
-        tokens->push_back({p.i, p.i + j, Token::KeywordOperator});
+        tokens->push_back({p.i, p.i + j, io::Token::KeywordOperator});
         p.advance(j);
         return false;
       } else if (j == tries.expecting_operators_trie.longest_match(p.peek_str(j))) {
         p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
-        tokens->push_back({p.i, p.i + j, Token::KeywordOperator});
+        tokens->push_back({p.i, p.i + j, io::Token::KeywordOperator});
         p.advance(j);
         return false;
       } else if (j == tries.types_trie.longest_match(p.peek_str(j))) {
         p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
-        tokens->push_back({p.i, p.i + j, Token::Type});
+        tokens->push_back({p.i, p.i + j, io::Token::Type});
         p.advance(j);
         return false;
       } else if (j == tries.methods_trie.longest_match(p.peek_str(j))) {
         p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
-        tokens->push_back({p.i, p.i + j, Token::Function});
+        tokens->push_back({p.i, p.i + j, io::Token::Function});
         p.advance(j);
         return false;
       } else if (j == tries.expecting_end_keywords_trie.longest_match(p.peek_str(j))) {
         events->push_back(false);
         p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
-        tokens->push_back({p.i, p.i + j, Token::Keyword});
+        tokens->push_back({p.i, p.i + j, io::Token::Keyword});
         p.advance(j);
         return false;
       } else if (j == tries.conditional_keywords_trie.longest_match(p.peek_str(j))) {
         if (p.current().flags & RubyState::RubyInternalState::NEWLINE || p.op_last)
           events->push_back(false);
         p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
-        tokens->push_back({p.i, p.i + j, Token::Keyword});
+        tokens->push_back({p.i, p.i + j, io::Token::Keyword});
         p.advance(j);
         return false;
       } else if (j == tries.end_keywords_trie.longest_match(p.peek_str(j))) {
         p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
         events->push_back(false);
-        tokens->push_back({p.i, p.i + j, Token::Keyword});
+        tokens->push_back({p.i, p.i + j, io::Token::Keyword});
         p.advance(j);
         return false;
       } else if (j == tries.builtins_trie.longest_match(p.peek_str(j))) {
         p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
-        tokens->push_back({p.i, p.i + j, Token::Constant});
+        tokens->push_back({p.i, p.i + j, io::Token::Constant});
         p.advance(j);
         return false;
       } else if (j == tries.errors_trie.longest_match(p.peek_str(j))) {
         p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
-        tokens->push_back({p.i, p.i + j, Token::Error});
+        tokens->push_back({p.i, p.i + j, io::Token::Error});
         p.advance(j);
         return false;
       } else if ('A' <= p.peek() && p.peek() <= 'Z' && !(p.peek(j) == '!' || p.peek(j) == '?')) {
         p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
         if (j >= 5 && p.peek_str(j).substr(j - 5) == "Error") {
-          tokens->push_back({p.i, p.i + j, Token::Error});
+          tokens->push_back({p.i, p.i + j, io::Token::Error});
           p.advance(j);
           return false;
         }
-        tokens->push_back({p.i, p.i + j, Token::Constant});
+        tokens->push_back({p.i, p.i + j, io::Token::Constant});
         p.advance(j);
         return false;
       } else {
         p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
         if (j == 4 && p.peek_str(4) == "true") {
-          tokens->push_back({p.i, p.i + j, Token::True});
+          tokens->push_back({p.i, p.i + j, io::Token::True});
           p.advance(4);
           return false;
         }
         if (j == 5 && p.peek_str(5) == "false") {
-          tokens->push_back({p.i, p.i + j, Token::False});
+          tokens->push_back({p.i, p.i + j, io::Token::False});
           p.advance(5);
           return false;
         }
         if (j == 3 && p.peek_str(3) == "end") {
           events->push_back(true);
-          tokens->push_back({p.i, p.i + j, Token::Keyword});
+          tokens->push_back({p.i, p.i + j, io::Token::Keyword});
           p.advance(3);
           return false;
         }
         if (j == 5 && p.peek_str(5) == "class") {
-          tokens->push_back({p.i, p.i + j, Token::Keyword});
+          tokens->push_back({p.i, p.i + j, io::Token::Keyword});
           p.advance(5);
           p.current().flags =
             (p.current().flags & ~RubyState::RubyInternalState::NAME_MASK)
@@ -966,7 +966,7 @@ bool handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
           return false;
         }
         if (j == 6 && p.peek_str(6) == "module") {
-          tokens->push_back({p.i, p.i + j, Token::Keyword});
+          tokens->push_back({p.i, p.i + j, io::Token::Keyword});
           p.advance(6);
           p.current().flags =
             (p.current().flags & ~RubyState::RubyInternalState::NAME_MASK)
@@ -974,7 +974,7 @@ bool handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
           return false;
         }
         if (j == 3 && p.peek_str(3) == "def") {
-          tokens->push_back({p.i, p.i + j, Token::Keyword});
+          tokens->push_back({p.i, p.i + j, io::Token::Keyword});
           p.advance(3);
           p.current().flags =
             (p.current().flags & ~RubyState::RubyInternalState::NAME_MASK)
@@ -985,17 +985,17 @@ bool handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
         if (p.peek(j) == ':') {
           p.advance(j);
           p.advance();
-          tokens->push_back({start, p.i, Token::Label});
+          tokens->push_back({start, p.i, io::Token::Label});
           return false;
         } else if (p.peek(j - 1) == '!' || p.peek(j - 1) == '?') {
           p.advance(j);
-          tokens->push_back({start, p.i, Token::Function});
+          tokens->push_back({start, p.i, io::Token::Function});
           return false;
         } else {
           p.advance(j);
           j = 0;
           if (p.peek(j) == '(' || p.peek(j) == '{') {
-            tokens->push_back({start, p.i, Token::Function});
+            tokens->push_back({start, p.i, io::Token::Function});
             return false;
           } else if (p.peek(j) == ' ' || p.peek(j) == '\t') {
             j++;
@@ -1033,14 +1033,14 @@ bool handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
           ) {
             return false;
           }
-          tokens->push_back({start, p.i, Token::Function});
+          tokens->push_back({start, p.i, io::Token::Function});
           return false;
         }
       }
     } else {
       uint32_t op_len;
       if ((op_len = tries.operator_trie.longest_match(p.peek_str(p.len() - p.i)))) {
-        tokens->push_back({p.i, p.i + op_len, Token::Operator});
+        tokens->push_back({p.i, p.i + op_len, io::Token::Operator});
         p.advance(op_len);
         p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
         p.set_op_last = true;
@@ -1056,7 +1056,7 @@ void ruby_parse(
   void **v_state,
   std::string_view line,
   bool fl,
-  std::vector<Token> *tokens,
+  std::vector<io::Token> *tokens,
   std::vector<ParseEvent> *events
 ) {
   RubyParser p(v_state, line);
@@ -1066,7 +1066,7 @@ void ruby_parse(
     if (p.current().state == RubyState::RubyInternalState::END)
       return;
     if (p.current().state == RubyState::RubyInternalState::COMMENT) {
-      tokens->push_back({p.i, p.len(), Token::Comment});
+      tokens->push_back({p.i, p.len(), io::Token::Comment});
       if (p.i == 0 && p.peek_str(4) == "=end") {
         p.current().state = RubyState::RubyInternalState::NONE;
         events->push_back(true);

src/internal/theme/theme.cc

@@ -5,150 +5,181 @@ Theme::Theme() {
   hl.fill({
     .fg = 0xF0F0F0,
     .bg = 0x000000,
-    .flags = Highlight::None,
+    .flags = io::Highlight::None,
   });
 }
 
-Highlight Theme::get(internal::syntax::Token token) const {
+io::Highlight Theme::get(const io::Token::Kind &token) const {
-  return hl[token.type];
+  return hl[token];
 }
 
 Theme Theme::default_theme() {
   Theme theme;
-  theme.hl[internal::syntax::Token::Shebang] = {
+  theme.hl[io::Token::Color1] = {
     .fg = 0x7DCFFF,
     .bg = 0x000000,
-    .flags = Highlight::None,
+    .flags = io::Highlight::None,
   };
-  theme.hl[internal::syntax::Token::Error] = {
+  theme.hl[io::Token::Color2] = {
-    .fg = 0xEF5168,
-    .bg = 0x000000,
-    .flags = Highlight::None,
-  };
-  theme.hl[internal::syntax::Token::Comment] = {
-    .fg = 0xAAAAAA,
-    .bg = 0x000000,
-    .flags = Highlight::Italic,
-  };
-  theme.hl[internal::syntax::Token::String] = {
     .fg = 0xAAD94C,
     .bg = 0x000000,
-    .flags = Highlight::None,
+    .flags = io::Highlight::None,
   };
-  theme.hl[internal::syntax::Token::Escape] = {
+  theme.hl[io::Token::Color3] = {
-    .fg = 0x7DCFFF,
+    .fg = 0xFF8F40,
     .bg = 0x000000,
-    .flags = Highlight::None,
+    .flags = io::Highlight::None,
   };
-  theme.hl[internal::syntax::Token::Interpolation] = {
+  theme.hl[io::Token::Color4] = {
-    .fg = 0x7DCFFF,
-    .bg = 0x000000,
-    .flags = Highlight::None,
-  };
-  theme.hl[internal::syntax::Token::Regexp] = {
     .fg = 0xD2A6FF,
     .bg = 0x000000,
-    .flags = Highlight::None,
+    .flags = io::Highlight::None,
   };
-  theme.hl[internal::syntax::Token::Number] = {
+  theme.hl[io::Token::Color5] = {
-    .fg = 0xE6C08A,
-    .bg = 0x000000,
-    .flags = Highlight::None,
-  };
-  theme.hl[internal::syntax::Token::True] = {
-    .fg = 0x7AE93C,
-    .bg = 0x000000,
-    .flags = Highlight::None,
-  };
-  theme.hl[internal::syntax::Token::False] = {
     .fg = 0xEF5168,
     .bg = 0x000000,
-    .flags = Highlight::None,
+    .flags = io::Highlight::None,
   };
-  theme.hl[internal::syntax::Token::Char] = {
+  theme.hl[io::Token::Warning] = {
+    .fg = 0xF1C55A,
+    .bg = 0x000000,
+    .flags = io::Highlight::None,
+  };
+  // Data is terminal default.
+  theme.hl[io::Token::Shebang] = {
+    .fg = 0x7DCFFF,
+    .bg = 0x000000,
+    .flags = io::Highlight::None,
+  };
+  theme.hl[io::Token::Error] = {
+    .fg = 0xEF5168,
+    .bg = 0x000000,
+    .flags = io::Highlight::None,
+  };
+  theme.hl[io::Token::Comment] = {
+    .fg = 0xAAAAAA,
+    .bg = 0x000000,
+    .flags = io::Highlight::Italic,
+  };
+  theme.hl[io::Token::String] = {
+    .fg = 0xAAD94C,
+    .bg = 0x000000,
+    .flags = io::Highlight::None,
+  };
+  theme.hl[io::Token::Escape] = {
+    .fg = 0x7DCFFF,
+    .bg = 0x000000,
+    .flags = io::Highlight::None,
+  };
+  theme.hl[io::Token::Interpolation] = {
+    .fg = 0x7DCFFF,
+    .bg = 0x000000,
+    .flags = io::Highlight::None,
+  };
+  theme.hl[io::Token::Regexp] = {
+    .fg = 0xD2A6FF,
+    .bg = 0x000000,
+    .flags = io::Highlight::None,
+  };
+  theme.hl[io::Token::Number] = {
+    .fg = 0xE6C08A,
+    .bg = 0x000000,
+    .flags = io::Highlight::None,
+  };
+  theme.hl[io::Token::True] = {
+    .fg = 0x7AE93C,
+    .bg = 0x000000,
+    .flags = io::Highlight::None,
+  };
+  theme.hl[io::Token::False] = {
+    .fg = 0xEF5168,
+    .bg = 0x000000,
+    .flags = io::Highlight::None,
+  };
+  theme.hl[io::Token::Char] = {
     .fg = 0xFFAF70,
     .bg = 0x000000,
-    .flags = Highlight::None,
+    .flags = io::Highlight::None,
   };
-  theme.hl[internal::syntax::Token::Keyword] = {
+  theme.hl[io::Token::Keyword] = {
     .fg = 0xFF8F40,
     .bg = 0x000000,
-    .flags = Highlight::None,
+    .flags = io::Highlight::None,
   };
-  theme.hl[internal::syntax::Token::KeywordOperator] = {
+  theme.hl[io::Token::KeywordOperator] = {
     .fg = 0xF07178,
     .bg = 0x000000,
-    .flags = Highlight::None,
+    .flags = io::Highlight::None,
   };
-  theme.hl[internal::syntax::Token::Operator] = {
+  theme.hl[io::Token::Operator] = {
     .fg = 0xFFFFFF,
     .bg = 0x000000,
-    .flags = Highlight::Italic,
+    .flags = io::Highlight::Italic,
   };
-  theme.hl[internal::syntax::Token::Function] = {
+  theme.hl[io::Token::Function] = {
     .fg = 0xFFAF70,
     .bg = 0x000000,
-    .flags = Highlight::None,
+    .flags = io::Highlight::None,
   };
-  theme.hl[internal::syntax::Token::Type] = {
+  theme.hl[io::Token::Type] = {
     .fg = 0xF07178,
     .bg = 0x000000,
-    .flags = Highlight::None,
+    .flags = io::Highlight::None,
   };
-  theme.hl[internal::syntax::Token::Constant] = {
+  theme.hl[io::Token::Constant] = {
     .fg = 0x7DCFFF,
     .bg = 0x000000,
-    .flags = Highlight::None,
+    .flags = io::Highlight::None,
   };
-  theme.hl[internal::syntax::Token::VariableInstance] = {
+  theme.hl[io::Token::VariableInstance] = {
     .fg = 0x95E6CB,
     .bg = 0x000000,
-    .flags = Highlight::None,
+    .flags = io::Highlight::None,
   };
-  theme.hl[internal::syntax::Token::VariableGlobal] = {
+  theme.hl[io::Token::VariableGlobal] = {
     .fg = 0xF07178,
     .bg = 0x000000,
-    .flags = Highlight::None,
+    .flags = io::Highlight::None,
   };
-  theme.hl[internal::syntax::Token::Annotation] = {
+  theme.hl[io::Token::Annotation] = {
     .fg = 0x7DCFFF,
     .bg = 0x000000,
-    .flags = Highlight::None,
+    .flags = io::Highlight::None,
   };
-  theme.hl[internal::syntax::Token::Directive] = {
+  theme.hl[io::Token::Directive] = {
     .fg = 0xFF8F40,
     .bg = 0x000000,
-    .flags = Highlight::None,
+    .flags = io::Highlight::None,
   };
-  theme.hl[internal::syntax::Token::Label] = {
+  theme.hl[io::Token::Label] = {
     .fg = 0xD2A6FF,
     .bg = 0x000000,
-    .flags = Highlight::None,
+    .flags = io::Highlight::None,
   };
-  theme.hl[internal::syntax::Token::Brace1] = {
+  theme.hl[io::Token::Brace1] = {
     .fg = 0xD2A6FF,
     .bg = 0x000000,
-    .flags = Highlight::None,
+    .flags = io::Highlight::None,
   };
-  theme.hl[internal::syntax::Token::Brace2] = {
+  theme.hl[io::Token::Brace2] = {
     .fg = 0xFFAFAF,
     .bg = 0x000000,
-    .flags = Highlight::None,
+    .flags = io::Highlight::None,
   };
-  theme.hl[internal::syntax::Token::Brace3] = {
+  theme.hl[io::Token::Brace3] = {
     .fg = 0xFFFF00,
     .bg = 0x000000,
-    .flags = Highlight::None,
+    .flags = io::Highlight::None,
   };
-  theme.hl[internal::syntax::Token::Brace4] = {
+  theme.hl[io::Token::Brace4] = {
     .fg = 0x0FFF0F,
     .bg = 0x000000,
-    .flags = Highlight::None,
+    .flags = io::Highlight::None,
   };
-  theme.hl[internal::syntax::Token::Brace5] = {
+  theme.hl[io::Token::Brace5] = {
     .fg = 0xFF0F0F,
     .bg = 0x000000,
-    .flags = Highlight::None,
+    .flags = io::Highlight::None,
   };
   return theme;
 }

src/main.cc

@@ -4,8 +4,7 @@
 int main(int argc, char *argv[]) {
   std::vector<std::string> args(argv, argv + argc);
   try {
-    bed::internal::io::IO io = bed::internal::io::IO();
+    bed::BEd ed(args);
-    bed::BEd ed(args, io);
     ed.run();
   } catch (bed::fatal_error &e) {
     if (e.code)