Random stuff to do with scripting

.clangd

@@ -1,5 +1,5 @@
 CompileFlags:
-  Add: [-I/home/syed/main/crib/include, -I/home/syed/main/crib/libs, c++20]
+  Add: [-I/home/syed/main/crib/include, -I/home/syed/main/crib/libs, -I/usr/include/ruby-3.4.0, -I/usr/include/ruby-3.4.0/x86_64-linux, c++20]
   Remove: []
   Compiler: clang++
 

.gitmodules

@@ -2,3 +2,7 @@
 	path = libs/libgrapheme
 	url = git://git.suckless.org/libgrapheme
   ignore = dirty
+[submodule "libs/utfcpp"]
+	path = libs/utfcpp
+  url = https://github.com/nemtrif/utfcpp.git
+  ignore = dirty

Makefile

@@ -17,7 +17,8 @@ CFLAGS_DEBUG :=\
 	-O0 -fno-inline -gsplit-dwarf\
 	-g -fsanitize=address -fno-omit-frame-pointer\
 	-Wno-unused-command-line-argument \
-	-I./include -I./libs
+	-I./include -I./libs \
+	-I/usr/include/ruby-3.4.0 -I/usr/include/ruby-3.4.0/x86_64-linux
 CFLAGS_RELEASE :=\
 	-std=c++20 -O3 -march=native \
 	-fno-exceptions -fno-rtti -fstrict-aliasing \
@@ -27,7 +28,8 @@ CFLAGS_RELEASE :=\
 	-mllvm -vectorize-loops \
 	-fno-unwind-tables -fno-asynchronous-unwind-tables\
 	-Wno-unused-command-line-argument \
-	-I./include -I./libs
+	-I./include -I./libs \
+	-I/usr/include/ruby-3.4.0 -I/usr/include/ruby-3.4.0/x86_64-linux
 
 PCH_CFLAGS_DEBUG := $(CFLAGS_DEBUG) -x c++-header
 PCH_CFLAGS_RELEASE := $(CFLAGS_RELEASE) -x c++-header

TODO.md

@@ -35,6 +35,10 @@ Copyright 2025 Syed Daanish
 * Try to make all functions better now that folds have been purged
 * Cleanup syntax and renderer files
 
+probably remove solargraph support and use ruby-lsp (or sorbet?) instead
+
+move lsp configs to json and also allow configs for windows-style vs unix-style line endings and utf-8 vs utf-16
+
 * finish bash then do all the directive-like ones like jsonc (first to help with theme files) / toml / yaml / ini / nginx
 * then markdown / html
 * then gitignore / gitattributes

config/config.json

@@ -0,0 +1,8 @@
+{
+  "startup": "bash $CRIB_CONFIG_DIR/startup.sh",
+  "shutdown": "bash $CRIB_CONFIG_DIR/shutdown.sh",
+
+  "theme": "theme.json",
+
+  "lsp_config": "lsp_config.json"
+}

config/main.rb

@@ -0,0 +1,151 @@
+module C
+  attr_accessor :theme, :lsp_config, :languages,
+                :line_endings, :utf_mode, :highlighters
+  attr_reader :b_startup, :b_shutdown, :b_extra_highlights
+
+  @lsp_config = {}
+  @languages = {}
+  @key_handlers = {}
+  @key_binds = {}
+
+  def startup(&block)
+    @b_startup = block
+  end
+
+  def shutdown(&block)
+    @b_shutdown = block
+  end
+
+  def extra_highlights(&block)
+    @b_extra_highlights = block
+  end
+
+  def bind(modes, keys = nil, action = nil, &block)
+    modes = [modes] unless modes.is_a?(Array)
+    if keys.nil?
+      app = self
+      dsl = Object.new
+      dsl.define_singleton_method(:set) do |k, act = nil, &blk|
+        app.bind(modes, k, act, &blk)
+      end
+      dsl.instance_exec(&handler)
+    elsif block_given?
+      keys = [keys] unless keys.is_a?(Array)
+      modes.each do |mode|
+        keys.each do |key|
+          @key_handlers[mode] ||= {}
+          @key_handlers[mode][key] ||= []
+          @key_handlers[mode][key] << block
+        end
+      end
+    elsif action.is_a?(String)
+      keys = [keys] unless keys.is_a?(Array)
+      modes.each do |mode|
+        keys.each do |key|
+          @key_binds[mode] ||= {}
+          @key_binds[mode][key] ||= []
+          @key_binds[mode][key] << action
+        end
+      end
+    else
+      raise ArgumentError("invalid arguments")
+    end
+  end
+end
+
+# basic configuration
+
+C.startup do
+  do_something_random_here!
+end
+
+C.shutdown do
+  do_something_random_here!
+end
+
+# this can be modified by the user during runtime through keybindings
+# But i need to know how to ever read this value only when needed .
+# maybe i can write a function that notifies if theme maybe changed then reload
+C.theme = {
+  # i have a predefined list of keys that can be used here
+  :default => {
+    # here fg bg and style are all optional and have default values
+    # if not specified
+    fg: 0xEEEEEE,
+    bg: 0x000000,
+    italic: false,
+    bold: false,
+    underline: false,
+    strikethrough: false
+  }
+}
+
+# this part uses dsl bindings to define the bind function
+# Hopefully extend to give more context/power to bindings
+# but try to keep simple for performance
+# for default keybindings
+C.bind [:normal, :select], :a => "insert_mode"
+# for custom keybindings
+C.bind :select, [:x, :c] do
+  puts "cut"
+end
+C.bind :jumper do
+  set [:x, :c] do
+    puts "jump to first bookmark"
+  end
+end
+# they can also be defined conditionally
+# This code is just an example and doesnt actually work
+if using_tmux?
+  bind :C-p do
+    system("tmux select-pane -U")
+  end
+end
+
+# This can for example be modified by user bindings during runtime
+C.lsp_config[:solargraph] = {
+  command: "solargraph",
+  args: ["stdio"],
+  languages: [:ruby]
+}
+
+# these are actually cached into cpp by the editor upon setting
+C.languages[:ruby] = {
+  color: 0xff8087,
+  symbol: "󰴭 ",
+  extensions: ["rb"],
+  filenames: ["Gemfile"],
+  mimetypes: ["text/x-ruby"]
+}
+
+C.line_endings = :auto_unix # or :auto_windows or :unix or :windows to force
+C.utf_mode = :auto_utf8 # or :auto_utf16 or :utf8 or :utf16 to force
+
+C.extra_highlights do |_line, _idx|
+  # the return can be an array of
+  # [fg, bg. flags, start, end]
+  # where fg and bg are integers (using 24 bit color)
+  # and flags is a bitmask of bold/underline/italic etc
+  # and start and end are integers strictly inside the line
+  return []
+end
+
+C.highlighters[:language_name] = {
+  parser: ->(_state, _line) {
+    # the return value is an array of
+    # [state, highlights]
+    # state can be of any type but will be consistent between calls
+    # initially nil is sent for uninitialized state
+    # the highlights can be an array of
+    # [fg, bg. flags, start, end]
+    # where fg and bg are integers (using 24 bit color)
+    # and flags is a bitmask of bold/underline/italic etc
+    # and start and end are integers strictly inside the line
+    return []
+  },
+  matcher: ->(_state1, _state2) {
+    # returns true if the states are equal
+    # And so would not need recomputation for further lines
+    return false
+  }
+}

config/scripts/shutdown.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env sh
+
+# This file can be used to execute commands right after the editor session ends.
+#
+# The location of this file is defined in the editor's config.json and can be changed
+#
+# It can for example be used to unset environment variables for any lsp(s) used
+# Or like this example to set kitty padding back higher
+#   kitty @ --to="$KITTY_LISTEN_ON" set-spacing padding=8 margin=0 2>/dev/null || true
+
+echo "Exiting crib editor..."

config/scripts/startup.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env sh
+
+# This file can be used to execute commands before the editor session starts.
+#
+# The location of this file is defined in the editor's config.json and can be changed
+#
+# It can for example be used to set environment variables for any lsp(s) used
+# Or like this example to set kitty padding to 0
+#   kitty @ --to="$KITTY_LISTEN_ON" set-spacing padding=0 margin=0 2>/dev/null || true
+
+echo "Starting crib editor..."

include/editor/editor.h

@@ -76,6 +76,7 @@ void edit_replace(Editor *editor, Coord start, Coord end, const char *text,
                   uint32_t len);
 Coord editor_hit_test(Editor *editor, uint32_t x, uint32_t y);
 char *get_selection(Editor *editor, uint32_t *out_len, Coord *out_start);
+void selection_bounds(Editor *editor, Coord *out_start, Coord *out_end);
 void editor_worker(Editor *editor);
 void move_line_down(Editor *editor);
 void move_line_up(Editor *editor);
@@ -127,12 +128,12 @@ inline static void utf8_normalize_edit(Editor *editor, TextEdit *edit) {
     return;
   }
   if (edit->start.col < len)
-    edit->start.col = utf16_offset_to_utf8(line, edit->start.col);
+    edit->start.col = utf16_offset_to_utf8(line, len, edit->start.col);
   else
     edit->start.col = len;
   if (edit->end.row == edit->start.row) {
     if (edit->end.col < len)
-      edit->end.col = utf16_offset_to_utf8(line, edit->end.col);
+      edit->end.col = utf16_offset_to_utf8(line, len, edit->end.col);
     else
       edit->end.col = len;
     free(it->buffer);
@@ -151,7 +152,7 @@ inline static void utf8_normalize_edit(Editor *editor, TextEdit *edit) {
     return;
   }
   if (edit->end.col < len)
-    edit->end.col = utf16_offset_to_utf8(line, edit->end.col);
+    edit->end.col = utf16_offset_to_utf8(line, len, edit->end.col);
   else
     edit->end.col = len;
   free(it->buffer);

include/editor/helpers.h

@@ -0,0 +1,24 @@
+#ifndef EDITOR_HELPERS_H
+#define EDITOR_HELPERS_H
+
+#include "editor/editor.h"
+
+void insert_str(Editor *editor, char *c, uint32_t len);
+void insert_char(Editor *editor, char c);
+void normal_mode(Editor *editor);
+void backspace_edit(Editor *editor);
+void delete_prev_word(Editor *editor);
+void delete_next_word(Editor *editor);
+void clear_hooks_at_line(Editor *editor, uint32_t line);
+void cursor_prev_word(Editor *editor);
+void cursor_next_word(Editor *editor);
+void select_all(Editor *editor);
+void fetch_lsp_hover(Editor *editor);
+void handle_mouse(Editor *editor, KeyEvent event);
+void indent_current_line(Editor *editor);
+void dedent_current_line(Editor *editor);
+void paste(Editor *editor);
+void copy(Editor *editor);
+void cut(Editor *editor);
+
+#endif

include/lsp/lsp.h

@@ -37,6 +37,7 @@ struct LSPInstance {
   bool allow_resolve = false;
   bool allow_formatting = false;
   bool allow_formatting_on_type = false;
+  bool is_utf8 = false;
   std::vector<char> format_chars;
   std::vector<char> trigger_chars;
   std::vector<char> end_chars;
@@ -53,6 +54,7 @@ extern std::unordered_map<uint8_t, std::shared_ptr<LSPInstance>> active_lsps;
 extern Queue<LSPOpenRequest> lsp_open_queue;
 
 static json client_capabilities = {
+    {"general", {{"positionEncodings", {"utf-16"}}}},
     {"textDocument",
      {{"publishDiagnostics", {{"relatedInformation", true}}},
       {"hover", {{"contentFormat", {"markdown", "plaintext"}}}},

include/pch.h

@@ -4,6 +4,7 @@
 #define PCRE2_CODE_UNIT_WIDTH 8
 #define PCRE_WORKSPACE_SIZE 512
 
+#include "ruby.h"
 #include <magic.h>
 #include <nlohmann/json.hpp>
 #include <pcre2.h>

include/scripting/decl.h

@@ -0,0 +1,12 @@
+#ifndef SCRIPTING_DECL_H
+#define SCRIPTING_DECL_H
+
+#include "utils/utils.h"
+
+void ruby_start(const char *main_file);
+void ruby_shutdown();
+void load_theme();
+std::string read_line_endings();
+std::string read_utf_mode();
+
+#endif

include/syntax/decl.h

@@ -6,9 +6,9 @@
 #include "syntax/trie.h"
 
 struct Highlight {
-  uint32_t fg;
+  uint32_t fg{0xFFFFFF};
-  uint32_t bg;
+  uint32_t bg{0x000000};
-  uint8_t flags;
+  uint8_t flags{0};
 };
 
 enum struct TokenKind : uint8_t {
@@ -28,55 +28,6 @@ const std::unordered_map<std::string, TokenKind> kind_map = {
 
 extern std::array<Highlight, TOKEN_KIND_COUNT> highlights;
 
-inline void load_theme(std::string filename) {
-  uint32_t len = 0;
-  char *raw = load_file(filename.c_str(), &len);
-  if (!raw)
-    return;
-  std::string data(raw, len);
-  free(raw);
-  json j = json::parse(data);
-  Highlight default_hl = {0xFFFFFF, 0, 0};
-  if (j.contains("Default")) {
-    auto def = j["Default"];
-    if (def.contains("fg") && def["fg"].is_string())
-      default_hl.fg = HEX(def["fg"]);
-    if (def.contains("bg") && def["bg"].is_string())
-      default_hl.bg = HEX(def["bg"]);
-    if (def.contains("italic") && def["italic"].get<bool>())
-      default_hl.flags |= CF_ITALIC;
-    if (def.contains("bold") && def["bold"].get<bool>())
-      default_hl.flags |= CF_BOLD;
-    if (def.contains("underline") && def["underline"].get<bool>())
-      default_hl.flags |= CF_UNDERLINE;
-    if (def.contains("strikethrough") && def["strikethrough"].get<bool>())
-      default_hl.flags |= CF_STRIKETHROUGH;
-  }
-  for (auto &hl : highlights)
-    hl = default_hl;
-  for (auto &[key, value] : j.items()) {
-    if (key == "Default")
-      continue;
-    auto it = kind_map.find(key);
-    if (it == kind_map.end())
-      continue;
-    Highlight hl = {0xFFFFFF, 0, 0};
-    if (value.contains("fg") && value["fg"].is_string())
-      hl.fg = HEX(value["fg"]);
-    if (value.contains("bg") && value["bg"].is_string())
-      hl.bg = HEX(value["bg"]);
-    if (value.contains("italic") && value["italic"].get<bool>())
-      hl.flags |= CF_ITALIC;
-    if (value.contains("bold") && value["bold"].get<bool>())
-      hl.flags |= CF_BOLD;
-    if (value.contains("underline") && value["underline"].get<bool>())
-      hl.flags |= CF_UNDERLINE;
-    if (value.contains("strikethrough") && value["strikethrough"].get<bool>())
-      hl.flags |= CF_STRIKETHROUGH;
-    highlights[static_cast<uint8_t>(it->second)] = hl;
-  }
-}
-
 struct Token {
   uint32_t start;
   uint32_t end;

include/syntax/extras.h

@@ -453,7 +453,7 @@ private:
     uint8_t g = (color >> 8) & 0xFF;
     uint8_t b = color & 0xFF;
     double luminance = 0.299 * r + 0.587 * g + 0.114 * b;
-    return (luminance > 128) ? 0x000000 : 0xFFFFFF;
+    return (luminance > 128) ? 0x010101 : 0xFFFFFF;
   }
 
   uint32_t hslToRgb(double h, double s, double l) {

include/syntax/tokens.def

@@ -1,53 +1,53 @@
-ADD(Data)
+ADD(K_DATA)
-ADD(Shebang)
+ADD(K_SHEBANG)
-ADD(Comment)
+ADD(K_COMMENT)
-ADD(Error)
+ADD(K_ERROR)
-ADD(String)
+ADD(K_STRING)
-ADD(Escape)
+ADD(K_ESCAPE)
-ADD(Interpolation)
+ADD(K_INTERPOLATION)
-ADD(Regexp)
+ADD(K_REGEXP)
-ADD(Number)
+ADD(K_NUMBER)
-ADD(True)
+ADD(K_TRUE)
-ADD(False)
+ADD(K_FALSE)
-ADD(Char)
+ADD(K_CHAR)
-ADD(Keyword)
+ADD(K_KEYWORD)
-ADD(KeywordOperator)
+ADD(K_KEYWORDOPERATOR)
-ADD(Operator)
+ADD(K_OPERATOR)
-ADD(Function)
+ADD(K_FUNCTION)
-ADD(Type)
+ADD(K_TYPE)
-ADD(Constant)
+ADD(K_CONSTANT)
-ADD(VariableInstance)
+ADD(K_VARIABLEINSTANCE)
-ADD(VariableGlobal)
+ADD(K_VARIABLEGLOBAL)
-ADD(Annotation)
+ADD(K_ANNOTATION)
-ADD(Directive)
+ADD(K_DIRECTIVE)
-ADD(Label)
+ADD(K_LABEL)
-ADD(Brace1)
+ADD(K_BRACE1)
-ADD(Brace2)
+ADD(K_BRACE2)
-ADD(Brace3)
+ADD(K_BRACE3)
-ADD(Brace4)
+ADD(K_BRACE4)
-ADD(Brace5)
+ADD(K_BRACE5)
-ADD(Heading1)
+ADD(K_HEADING1)
-ADD(Heading2)
+ADD(K_HEADING2)
-ADD(Heading3)
+ADD(K_HEADING3)
-ADD(Heading4)
+ADD(K_HEADING4)
-ADD(Heading5)
+ADD(K_HEADING5)
-ADD(Heading6)
+ADD(K_HEADING6)
-ADD(Blockquote)
+ADD(K_BLOCKQUOTE)
-ADD(List)
+ADD(K_LIST)
-ADD(ListItem)
+ADD(K_LISTITEM)
-ADD(Code)
+ADD(K_CODE)
-ADD(LanguageName)
+ADD(K_LANGUAGENAME)
-ADD(LinkLabel)
+ADD(K_LINKLABEL)
-ADD(ImageLabel)
+ADD(K_IMAGELABEL)
-ADD(Link)
+ADD(K_LINK)
-ADD(Table)
+ADD(K_TABLE)
-ADD(TableHeader)
+ADD(K_TABLEHEADER)
-ADD(Italic)
+ADD(K_ITALIC)
-ADD(Bold)
+ADD(K_BOLD)
-ADD(Underline)
+ADD(K_UNDERLINE)
-ADD(Strikethrough)
+ADD(K_STRIKETHROUGH)
-ADD(HorixontalRule)
+ADD(K_HORIXONTALRULE)
-ADD(Tag)
+ADD(K_TAG)
-ADD(Attribute)
+ADD(K_ATTRIBUTE)
-ADD(CheckDone)
+ADD(K_CHECKDONE)
-ADD(CheckNotDone)
+ADD(K_CHECKNOTDONE)

include/utils/utils.h

@@ -92,8 +92,10 @@ uint32_t get_visual_col_from_bytes(const char *line, uint32_t len,
                                    uint32_t byte_limit);
 uint32_t get_bytes_from_visual_col(const char *line, uint32_t len,
                                    uint32_t target_visual_col);
-uint32_t utf8_byte_offset_to_utf16(const char *s, uint32_t byte_pos);
+size_t utf8_offset_to_utf16(const char *utf8, size_t utf8_len,
-uint32_t utf16_offset_to_utf8(const char *s, uint32_t utf16_pos);
+                            size_t byte_offset);
+size_t utf16_offset_to_utf8(const char *utf8, size_t utf8_len,
+                            size_t utf16_offset);
 uint8_t utf8_codepoint_width(unsigned char c);
 
 void log(const char *fmt, ...);

samples/bash.sh

@@ -22,6 +22,7 @@ colorize() {
 }
 
 # Example of error handling
+
 handle_error() {
   log ERROR "An error occurred on line $1"
 }

samples/css.css

@@ -21,6 +21,7 @@ a:focus-visible {
 
 input::placeholder {
   color: #999;
+  background-color: #523;
 }
 
 /* CSS variables */

samples/lua.lua

@@ -117,4 +117,3 @@ local tpl = `Value: ${counter}`
 
 -- Regex-like string (for testing injection highlighting)
 local re = "/^%a+$/"
-

samples/python.py

@@ -12,7 +12,6 @@ NotImplemented
 Ellipsis
 __name__  # builtin constant
 
-
 # ==============================
 # Imports
 # ==============================

samples/ruby.rb

@@ -4,27 +4,16 @@
 # Purpose: Test syntax highlighting + width calculation in your editor
 # ---------------------------------------------------------------
 
-# Basic output
-def greet
-  puts "Hello, 世界! 👋🌏"
-end
-
-# Emoji-heavy strings
-emojis = "👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏"
 
 # Mixed-width CJKssssssssssssssss LoadErssssssssssssssssssssssss
 cjk_samples = [
   "漢字テスト",
   "測試中文字串",
   "한국어 테스트",
-  "ひらがなカタカナ😀混合",
+  "ひらがなカタカナ混合",
-  "大量の文字列🚀🚀🚀"
 ]
 
-
+# a hex color: #FFFFFF shouldn't hl here: hsl(147rad, 50%, 47%)  as it is not css-style file
-
-
-# a hex color: #FFFFFF hsl(147rad, 50%, 47%)
 
 0x603010 # another hex color
 
@@ -38,7 +27,7 @@ s wow
 
 UNICORE = %r{
     [s]
-    {#{}}
+    {#{ss}}
     \C-s\u{10}
   }
   
@@ -112,8 +101,12 @@ mixed = [
 
 two_docs = <<DOC1 , <<DOC2
 stuff for doc2
+
+
+rdvajehvbaejbfh
+
 DOC1
-stuff for doc 2 with \#{not interpolation} and more
+stuff for doc 2 with #{not interpolation} and more
 DOC2
 
 p = 0 <<22 # not a heredoc
@@ -155,9 +148,6 @@ This is a wrapped line test, This is a wrapped line test, This is a wrapped line
 
 # Constants
 
-__END_
-
-
 PI = 3.14159
 MAX_ITER = 5
 # Module

scripts/exit.sh

@@ -1 +0,0 @@
-# kitty @ --to="$KITTY_LISTEN_ON" set-spacing padding=8 margin=0 2>/dev/null || true

scripts/init.sh

@@ -1 +0,0 @@
-# kitty @ --to="$KITTY_LISTEN_ON" set-spacing padding=0 margin=0 2>/dev/null || true

src/editor/completions.cc

@@ -12,15 +12,14 @@ inline static std::string completion_prefix(Editor *editor) {
   if (hook.row != cur.row || cur.col < hook.col)
     return "";
   LineIterator *it = begin_l_iter(editor->root, hook.row);
-  char *line = next_line(it, nullptr);
+  uint32_t line_len;
+  char *line = next_line(it, &line_len);
   if (!line) {
     free(it->buffer);
     free(it);
     return "";
   }
-  uint32_t start = utf16_offset_to_utf8(line, hook.col);
+  std::string prefix(line + hook.col, cur.col - hook.col);
-  uint32_t end = editor->cursor.col;
-  std::string prefix(line + start, end - start);
   free(it->buffer);
   free(it);
   return prefix;
@@ -212,13 +211,14 @@ void completion_request(Editor *editor) {
   };
   std::shared_lock lock(editor->knot_mtx);
   LineIterator *it = begin_l_iter(editor->root, hook.row);
-  char *line = next_line(it, nullptr);
+  uint32_t length;
+  char *line = next_line(it, &length);
   if (!line) {
     free(it->buffer);
     free(it);
     return;
   }
-  uint32_t col = utf8_byte_offset_to_utf16(line, editor->cursor.col);
+  uint32_t col = utf8_offset_to_utf16(line, length, editor->cursor.col);
   free(it->buffer);
   free(it);
   lock.unlock();

src/editor/edit.cc

@@ -16,17 +16,18 @@ void edit_erase(Editor *editor, Coord pos, int64_t len) {
     bool do_lsp = (editor->lsp != nullptr);
     if (do_lsp) {
       LineIterator *it = begin_l_iter(editor->root, point.row);
-      char *line = next_line(it, nullptr);
+      uint32_t len;
+      char *line = next_line(it, &len);
       int utf16_start = 0;
       if (line)
-        utf16_start = utf8_byte_offset_to_utf16(line, point.col);
+        utf16_start = utf8_offset_to_utf16(line, len, point.col);
       free(it->buffer);
       free(it);
       it = begin_l_iter(editor->root, pos.row);
-      line = next_line(it, nullptr);
+      line = next_line(it, &len);
       int utf16_end = 0;
       if (line)
-        utf16_end = utf8_byte_offset_to_utf16(line, pos.col);
+        utf16_end = utf8_offset_to_utf16(line, len, pos.col);
       free(it->buffer);
       free(it);
       lsp_range = {{"start", {{"line", point.row}, {"character", utf16_start}}},
@@ -88,17 +89,18 @@ void edit_erase(Editor *editor, Coord pos, int64_t len) {
     bool do_lsp = (editor->lsp != nullptr);
     if (do_lsp) {
       LineIterator *it = begin_l_iter(editor->root, pos.row);
-      char *line = next_line(it, nullptr);
+      uint32_t line_len;
+      char *line = next_line(it, &line_len);
       int utf16_start = 0;
       if (line)
-        utf16_start = utf8_byte_offset_to_utf16(line, pos.col);
+        utf16_start = utf8_offset_to_utf16(line, line_len, pos.col);
       free(it->buffer);
       free(it);
       it = begin_l_iter(editor->root, point.row);
-      line = next_line(it, nullptr);
+      line = next_line(it, &line_len);
       int utf16_end = 0;
       if (line)
-        utf16_end = utf8_byte_offset_to_utf16(line, point.col);
+        utf16_end = utf8_offset_to_utf16(line, line_len, point.col);
       free(it->buffer);
       free(it);
       lsp_range = {{"start", {{"line", pos.row}, {"character", utf16_start}}},
@@ -164,6 +166,14 @@ void edit_insert(Editor *editor, Coord pos, char *data, uint32_t len) {
     uint32_t new_row = byte_to_line(editor->root, cursor_new, &new_col);
     editor->cursor = {new_row, new_col};
   }
+  LineIterator *it = begin_l_iter(editor->root, pos.row);
+  uint32_t line_len;
+  char *line = next_line(it, &line_len);
+  int utf16_col = 0;
+  if (line)
+    utf16_col = utf8_offset_to_utf16(line, line_len, pos.col);
+  free(it->buffer);
+  free(it);
   lock_1.unlock();
   std::unique_lock lock_2(editor->knot_mtx);
   editor->root = insert(editor->root, byte_pos, data, len);
@@ -177,15 +187,6 @@ void edit_insert(Editor *editor, Coord pos, char *data, uint32_t len) {
     editor->parser->edit(pos.row, pos.row, rows);
   if (editor->lsp) {
     if (editor->lsp->incremental_sync) {
-      lock_1.lock();
-      LineIterator *it = begin_l_iter(editor->root, pos.row);
-      char *line = next_line(it, nullptr);
-      int utf16_col = 0;
-      if (line)
-        utf16_col = utf8_byte_offset_to_utf16(line, pos.col);
-      free(it->buffer);
-      free(it);
-      lock_1.unlock();
       json message = {
           {"jsonrpc", "2.0"},
           {"method", "textDocument/didChange"},
@@ -222,17 +223,18 @@ void edit_replace(Editor *editor, Coord start, Coord end, const char *text,
       line_to_byte(editor->root, start.row, nullptr) + start.col;
   uint32_t end_byte = line_to_byte(editor->root, end.row, nullptr) + end.col;
   LineIterator *it = begin_l_iter(editor->root, start.row);
-  char *line = next_line(it, nullptr);
+  uint32_t line_len;
+  char *line = next_line(it, &line_len);
   int utf16_start = 0;
   if (line)
-    utf16_start = utf8_byte_offset_to_utf16(line, start.col);
+    utf16_start = utf8_offset_to_utf16(line, line_len, start.col);
   free(it->buffer);
   free(it);
   it = begin_l_iter(editor->root, end.row);
-  line = next_line(it, nullptr);
+  line = next_line(it, &line_len);
   int utf16_end = 0;
   if (line)
-    utf16_end = utf8_byte_offset_to_utf16(line, end.col);
+    utf16_end = utf8_offset_to_utf16(line, line_len, end.col);
   free(it->buffer);
   free(it);
   if (start_byte != end_byte)
@@ -243,10 +245,8 @@ void edit_replace(Editor *editor, Coord start, Coord end, const char *text,
   for (uint32_t i = 0; i < len; i++)
     if (text[i] == '\n')
       rows++;
-  if (editor->parser) {
+  if (editor->parser)
-    editor->parser->edit(start.row, end.row - 1, 0);
+    editor->parser->edit(start.row, end.row - 1, rows);
-    editor->parser->edit(start.row, start.row, rows);
-  }
   if (editor->lsp) {
     if (editor->lsp->incremental_sync) {
       json message = {

src/editor/editor.cc

@@ -1,8 +1,8 @@
 #include "editor/editor.h"
 #include "editor/decl.h"
 #include "lsp/lsp.h"
+#include "syntax/langs.h"
 #include "utils/utils.h"
-#include <shared_mutex>
 
 Editor *new_editor(const char *filename_arg, Coord position, Coord size) {
   Editor *editor = new Editor();
@@ -29,7 +29,7 @@ Editor *new_editor(const char *filename_arg, Coord position, Coord size) {
   editor->root = load(str, len, optimal_chunk_size(len));
   free(str);
   editor->lang = language_for_file(filename.c_str());
-  if (editor->lang.name != "unknown")
+  if (parsers.find(editor->lang.name) != parsers.end())
     editor->parser = new Parser(editor, editor->lang.name, size.row + 5);
   if (editor->lang.name == "css" || editor->lang.name == "html" ||
       editor->lang.name == "javascript" || editor->lang.name == "markdown" ||

src/editor/events.cc

@@ -1,147 +1,14 @@
 #include "editor/editor.h"
-#include "lsp/lsp.h"
+#include "editor/helpers.h"
+#include "io/sysio.h"
 #include "main.h"
 #include "utils/utils.h"
 
 void handle_editor_event(Editor *editor, KeyEvent event) {
-  static std::chrono::steady_clock::time_point last_click_time =
-      std::chrono::steady_clock::now();
-  static uint32_t click_count = 0;
-  static Coord last_click_pos = {UINT32_MAX, UINT32_MAX};
-  Coord start = editor->cursor;
   uint8_t old_mode = mode;
   if (editor->hover_active)
     editor->hover_active = false;
-  if (event.key_type == KEY_MOUSE) {
+  handle_mouse(editor, event);
-    auto now = std::chrono::steady_clock::now();
-    auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(
-                        now - last_click_time)
-                        .count();
-    switch (event.mouse_state) {
-    case SCROLL:
-      switch (event.mouse_direction) {
-      case SCROLL_UP:
-        scroll_up(editor, 4);
-        ensure_cursor(editor);
-        break;
-      case SCROLL_DOWN:
-        scroll_down(editor, 4);
-        ensure_cursor(editor);
-        break;
-      case SCROLL_LEFT:
-        cursor_left(editor, 10);
-        break;
-      case SCROLL_RIGHT:
-        cursor_right(editor, 10);
-        break;
-      }
-      break;
-    case PRESS:
-      if (event.mouse_button == LEFT_BTN) {
-        Coord cur_pos = {event.mouse_x, event.mouse_y};
-        if (duration < 250 && last_click_pos == cur_pos)
-          click_count++;
-        else
-          click_count = 1;
-        last_click_time = now;
-        last_click_pos = cur_pos;
-        Coord p = editor_hit_test(editor, event.mouse_x, event.mouse_y);
-        if (p.row == UINT32_MAX && p.col == UINT32_MAX)
-          return;
-        editor->cursor_preffered = UINT32_MAX;
-        if (click_count == 1) {
-          editor->cursor = p;
-          editor->selection = p;
-          if (mode == SELECT) {
-            mode = NORMAL;
-            editor->selection_active = false;
-          }
-        } else if (click_count == 2) {
-          uint32_t prev_col, next_col;
-          word_boundaries(editor, editor->cursor, &prev_col, &next_col, nullptr,
-                          nullptr);
-          if (editor->cursor < editor->selection)
-            editor->cursor = {editor->cursor.row, prev_col};
-          else
-            editor->cursor = {editor->cursor.row, next_col};
-          editor->cursor_preffered = UINT32_MAX;
-          editor->selection_type = WORD;
-          mode = SELECT;
-          editor->selection_active = true;
-        } else if (click_count >= 3) {
-          if (editor->cursor < editor->selection) {
-            editor->cursor = {p.row, 0};
-          } else {
-            uint32_t line_len;
-            LineIterator *it = begin_l_iter(editor->root, p.row);
-            char *line = next_line(it, &line_len);
-            if (!line)
-              return;
-            if (line_len > 0 && line[line_len - 1] == '\n')
-              line_len--;
-            free(it->buffer);
-            free(it);
-            editor->cursor = {p.row, line_len};
-          }
-          editor->cursor_preffered = UINT32_MAX;
-          editor->selection_type = LINE;
-          mode = SELECT;
-          editor->selection_active = true;
-          click_count = 3;
-        }
-      }
-      break;
-    case DRAG:
-      if (event.mouse_button == LEFT_BTN) {
-        Coord p = editor_hit_test(editor, event.mouse_x, event.mouse_y);
-        if (p.row == UINT32_MAX && p.col == UINT32_MAX)
-          return;
-        editor->cursor_preffered = UINT32_MAX;
-        mode = SELECT;
-        if (!editor->selection_active) {
-          editor->selection_active = true;
-          editor->selection_type = CHAR;
-        }
-        uint32_t prev_col, next_col, line_len;
-        switch (editor->selection_type) {
-        case CHAR:
-          editor->cursor = p;
-          break;
-        case WORD:
-          word_boundaries(editor, p, &prev_col, &next_col, nullptr, nullptr);
-          if (editor->cursor < editor->selection)
-            editor->cursor = {p.row, prev_col};
-          else
-            editor->cursor = {p.row, next_col};
-          break;
-        case LINE:
-          if (editor->cursor < editor->selection) {
-            editor->cursor = {p.row, 0};
-          } else {
-            LineIterator *it = begin_l_iter(editor->root, p.row);
-            char *line = next_line(it, &line_len);
-            if (!line)
-              return;
-            if (line_len > 0 && line[line_len - 1] == '\n')
-              line_len--;
-            free(it->buffer);
-            free(it);
-            editor->cursor = {p.row, line_len};
-          }
-          break;
-        }
-      }
-      break;
-    case RELEASE:
-      if (event.mouse_button == LEFT_BTN)
-        if (editor->cursor.row == editor->selection.row &&
-            editor->cursor.col == editor->selection.col) {
-          mode = NORMAL;
-          editor->selection_active = false;
-        }
-      break;
-    }
-  }
   if (event.key_type == KEY_SPECIAL) {
     switch (event.special_modifier) {
     case 0:
@@ -161,9 +28,6 @@ void handle_editor_event(Editor *editor, KeyEvent event) {
       }
       break;
     case CNTRL:
-      uint32_t prev_col, next_col;
-      word_boundaries(editor, editor->cursor, &prev_col, &next_col, nullptr,
-                      nullptr);
       switch (event.special_key) {
       case KEY_DOWN:
         cursor_down(editor, 5);
@@ -172,18 +36,9 @@ void handle_editor_event(Editor *editor, KeyEvent event) {
         cursor_up(editor, 5);
         break;
       case KEY_LEFT:
-        editor->cursor_preffered = UINT32_MAX;
+        cursor_prev_word(editor);
-        if (prev_col == editor->cursor.col)
-          cursor_left(editor, 1);
-        else
-          editor->cursor = {editor->cursor.row, prev_col};
-        break;
       case KEY_RIGHT:
-        editor->cursor_preffered = UINT32_MAX;
+        cursor_next_word(editor);
-        if (next_col == editor->cursor.col)
-          cursor_right(editor, 1);
-        else
-          editor->cursor = {editor->cursor.row, next_col};
         break;
       }
       break;
@@ -210,27 +65,7 @@ void handle_editor_event(Editor *editor, KeyEvent event) {
     if (event.key_type == KEY_CHAR && event.len == 1) {
       switch (event.c[0]) {
       case 'u':
-        if (editor->root->line_count > 0) {
+        select_all(editor);
-          editor->cursor.row = editor->root->line_count - 1;
-          LineIterator *it = begin_l_iter(editor->root, editor->cursor.row);
-          if (!it)
-            break;
-          uint32_t line_len;
-          char *line = next_line(it, &line_len);
-          if (!line)
-            break;
-          if (line_len > 0 && line[line_len - 1] == '\n')
-            line_len--;
-          line_len = count_clusters(line, line_len, 0, line_len);
-          free(it->buffer);
-          free(it);
-          editor->cursor.col = line_len;
-          editor->cursor_preffered = UINT32_MAX;
-          mode = SELECT;
-          editor->selection_active = true;
-          editor->selection = {0, 0};
-          editor->selection_type = LINE;
-        }
         break;
       case CTRL('h'):
         editor->hover.scroll(-1);
@@ -241,68 +76,15 @@ void handle_editor_event(Editor *editor, KeyEvent event) {
         editor->hover_active = true;
         break;
       case 'h':
-        if (editor->lsp && editor->lsp->allow_hover) {
+        fetch_lsp_hover(editor);
-          LineIterator *it = begin_l_iter(editor->root, editor->cursor.row);
-          char *line = next_line(it, nullptr);
-          if (!line) {
-            free(it->buffer);
-            free(it);
-            break;
-          }
-          uint32_t col = utf8_byte_offset_to_utf16(line, editor->cursor.col);
-          free(it->buffer);
-          free(it);
-          json hover_request = {
-              {"jsonrpc", "2.0"},
-              {"method", "textDocument/hover"},
-              {"params",
-               {{"textDocument", {{"uri", editor->uri}}},
-                {"position",
-                 {{"line", editor->cursor.row}, {"character", col}}}}}};
-          LSPPending *pending = new LSPPending();
-          pending->editor = editor;
-          pending->method = "textDocument/hover";
-          pending->callback = [](Editor *editor, std::string, json hover) {
-            if (hover.contains("result") && !hover["result"].is_null()) {
-              auto &contents = hover["result"]["contents"];
-              std::string hover_text = "";
-              bool is_markup = false;
-              if (contents.is_object()) {
-                hover_text += contents["value"].get<std::string>();
-                is_markup = (contents["kind"].get<std::string>() == "markdown");
-              } else if (contents.is_array()) {
-                for (auto &block : contents) {
-                  if (block.is_string()) {
-                    hover_text += block.get<std::string>() + "\n";
-                  } else if (block.is_object() && block.contains("language") &&
-                             block.contains("value")) {
-                    std::string lang = block["language"].get<std::string>();
-                    std::string val = block["value"].get<std::string>();
-                    is_markup = true;
-                    hover_text += "```" + lang + "\n" + val + "\n```\n";
-                  }
-                }
-              } else if (contents.is_string()) {
-                hover_text += contents.get<std::string>();
-              }
-              if (!hover_text.empty()) {
-                editor->hover.clear();
-                editor->hover.text = clean_text(hover_text);
-                editor->hover.is_markup = is_markup;
-                editor->hover.render_first();
-                editor->hover_active = true;
-              }
-            }
-          };
-          lsp_send(editor->lsp, hover_request, pending);
-        }
         break;
-      case 'a':
+      case 'a': {
         mode = INSERT;
+        Coord start = editor->cursor;
         cursor_right(editor, 1);
         if (start.row != editor->cursor.row)
           cursor_left(editor, 1);
-        break;
+      } break;
       case 'i':
         mode = INSERT;
         break;
@@ -315,11 +97,7 @@ void handle_editor_event(Editor *editor, KeyEvent event) {
         editor->jumper_set = false;
         break;
       case 'N':
-        for (uint8_t i = 0; i < 94; i++)
+        clear_hooks_at_line(editor, editor->cursor.row);
-          if (editor->hooks[i] == editor->cursor.row + 1) {
-            editor->hooks[i] = 0;
-            break;
-          }
         break;
       case 's':
       case 'v':
@@ -355,29 +133,18 @@ void handle_editor_event(Editor *editor, KeyEvent event) {
         ensure_cursor(editor);
         break;
       case '>':
-      case '.': {
+      case '.':
-        uint32_t delta = editor->indents.indent_line(editor->cursor.row);
+        indent_current_line(editor);
-        editor->cursor.col = start.col + delta;
+        break;
-        editor->cursor.row = start.row;
-      } break;
       case '<':
-      case ',': {
+      case ',':
-        uint32_t delta = editor->indents.dedent_line(editor->cursor.row);
+        dedent_current_line(editor);
-        editor->cursor.col = MAX((int64_t)start.col - delta, 0);
+        break;
-        editor->cursor.row = start.row;
-      } break;
       case CTRL('s'):
         save_file(editor);
         break;
       case 'p':
-        uint32_t len;
+        paste(editor);
-        char *text = get_from_clipboard(&len);
-        if (text) {
-          edit_insert(editor, editor->cursor, text, len);
-          uint32_t grapheme_len = count_clusters(text, len, 0, len);
-          cursor_right(editor, grapheme_len);
-          free(text);
-        }
         break;
       }
     }
@@ -391,156 +158,13 @@ void handle_editor_event(Editor *editor, KeyEvent event) {
         } else if (event.c[0] == '\n' || event.c[0] == '\r') {
           editor->indents.insert_new_line(editor->cursor);
         } else if (event.c[0] == CTRL('W')) {
-          uint32_t prev_col_byte, prev_col_cluster;
+          delete_prev_word(editor);
-          word_boundaries(editor, editor->cursor, &prev_col_byte, nullptr,
-                          &prev_col_cluster, nullptr);
-          if (prev_col_byte == editor->cursor.col)
-            edit_erase(editor, editor->cursor, -1);
-          else
-            edit_erase(editor, editor->cursor, -(int64_t)prev_col_cluster);
         } else if (isprint((unsigned char)(event.c[0]))) {
-          char c = event.c[0];
+          insert_char(editor, event.c[0]);
-          uint32_t col = editor->cursor.col;
-          LineIterator *it = begin_l_iter(editor->root, editor->cursor.row);
-          if (!it)
-            return;
-          uint32_t len;
-          char *line = next_line(it, &len);
-          if (!line) {
-            free(it->buffer);
-            free(it);
-            return;
-          }
-          bool skip_insert = false;
-          if (line && col < len) {
-            char next = line[col];
-            if ((c == '}' && next == '}') || (c == ')' && next == ')') ||
-                (c == ']' && next == ']') || (c == '"' && next == '"') ||
-                (c == '\'' && next == '\'')) {
-              cursor_right(editor, 1);
-              skip_insert = true;
-            }
-          }
-          free(it->buffer);
-          free(it);
-          if (!skip_insert) {
-            char closing = 0;
-            switch (c) {
-            case '{':
-              closing = '}';
-              break;
-            case '(':
-              closing = ')';
-              break;
-            case '[':
-              closing = ']';
-              break;
-            case '"':
-              closing = '"';
-              break;
-            case '\'':
-              closing = '\'';
-              break;
-            }
-            if (closing) {
-              char pair[2] = {c, closing};
-              edit_insert(editor, editor->cursor, pair, 2);
-              cursor_right(editor, 1);
-            } else {
-              edit_insert(editor, editor->cursor, &c, 1);
-              cursor_right(editor, 1);
-            }
-            if (editor->lsp && editor->lsp->allow_formatting_on_type) {
-              for (char ch : editor->lsp->format_chars) {
-                if (ch == c) {
-                  LineIterator *it =
-                      begin_l_iter(editor->root, editor->cursor.row);
-                  if (!it)
-                    return;
-                  char *line = next_line(it, nullptr);
-                  if (!line) {
-                    free(it->buffer);
-                    free(it);
-                    return;
-                  }
-                  uint32_t col =
-                      utf8_byte_offset_to_utf16(line, editor->cursor.col);
-                  free(it->buffer);
-                  free(it);
-                  int version = editor->lsp_version;
-                  json message = {
-                      {"jsonrpc", "2.0"},
-                      {"method", "textDocument/onTypeFormatting"},
-                      {"params",
-                       {{"textDocument", {{"uri", editor->uri}}},
-                        {"position",
-                         {{"line", editor->cursor.row}, {"character", col}}},
-                        {"ch", std::string(1, c)},
-                        {"options",
-                         {{"tabSize", 2},
-                          {"insertSpaces", true},
-                          {"trimTrailingWhitespace", true},
-                          {"trimFinalNewlines", true}}}}}};
-                  LSPPending *pending = new LSPPending();
-                  pending->editor = editor;
-                  pending->method = "textDocument/onTypeFormatting";
-                  pending->callback = [version](Editor *editor, std::string,
-                                                json message) {
-                    if (version != editor->lsp_version)
-                      return;
-                    auto &edits = message["result"];
-                    if (edits.is_array()) {
-                      std::vector<TextEdit> t_edits;
-                      t_edits.reserve(edits.size());
-                      for (auto &edit : edits) {
-                        TextEdit t_edit;
-                        t_edit.text = edit.value("newText", "");
-                        t_edit.start.row = edit["range"]["start"]["line"];
-                        t_edit.start.col = edit["range"]["start"]["character"];
-                        t_edit.end.row = edit["range"]["end"]["line"];
-                        t_edit.end.col = edit["range"]["end"]["character"];
-                        utf8_normalize_edit(editor, &t_edit);
-                        t_edits.push_back(t_edit);
-                      }
-                      apply_lsp_edits(editor, t_edits, false);
-                      ensure_scroll(editor);
-                    }
-                  };
-                  lsp_send(editor->lsp, message, pending);
-                  break;
-                }
-              }
-            }
-          }
         } else if (event.c[0] == 0x7F || event.c[0] == 0x08) {
-          Coord prev_pos = editor->cursor;
+          backspace_edit(editor);
-          if (prev_pos.col > 0)
-            prev_pos.col--;
-          LineIterator *it = begin_l_iter(editor->root, editor->cursor.row);
-          if (!it)
-            return;
-          char *line = next_line(it, nullptr);
-          char prev_char = line[prev_pos.col];
-          char next_char = line[editor->cursor.col];
-          free(it->buffer);
-          free(it);
-          bool is_pair = (prev_char == '{' && next_char == '}') ||
-                         (prev_char == '(' && next_char == ')') ||
-                         (prev_char == '[' && next_char == ']') ||
-                         (prev_char == '"' && next_char == '"') ||
-                         (prev_char == '\'' && next_char == '\'');
-          if (is_pair) {
-            edit_erase(editor, editor->cursor, 1);
-            edit_erase(editor, prev_pos, 1);
-          } else {
-            edit_erase(editor, editor->cursor, -1);
-          }
         } else if (event.c[0] == 0x1B) {
-          Coord prev_pos = editor->cursor;
+          normal_mode(editor);
-          mode = NORMAL;
-          cursor_left(editor, 1);
-          if (prev_pos.row != editor->cursor.row)
-            cursor_right(editor, 1);
         }
       } else if (event.len > 1) {
         edit_insert(editor, editor->cursor, event.c, event.len);
@@ -553,29 +177,17 @@ void handle_editor_event(Editor *editor, KeyEvent event) {
         edit_erase(editor, editor->cursor, 1);
         break;
       case CNTRL:
-        uint32_t next_col_byte, next_col_cluster;
+        delete_next_word(editor);
-        word_boundaries(editor, editor->cursor, nullptr, &next_col_byte,
-                        nullptr, &next_col_cluster);
-        if (next_col_byte == editor->cursor.col)
-          edit_erase(editor, editor->cursor, 1);
-        else
-          edit_erase(editor, editor->cursor, next_col_cluster);
         break;
       }
     } else if (event.key_type == KEY_PASTE) {
-      if (event.c) {
+      insert_str(editor, event.c, event.len);
-        edit_insert(editor, editor->cursor, event.c, event.len);
-        uint32_t grapheme_len =
-            count_clusters(event.c, event.len, 0, event.len);
-        cursor_right(editor, grapheme_len);
-      }
     }
     break;
   case SELECT:
     if (event.key_type == KEY_CHAR && event.len == 1) {
       uint32_t len;
       char *text;
-      Coord start;
       switch (event.c[0]) {
       case 0x1B:
       case 's':
@@ -584,41 +196,15 @@ void handle_editor_event(Editor *editor, KeyEvent event) {
         mode = NORMAL;
         break;
       case 'y':
-        text = get_selection(editor, &len, nullptr);
+        copy(editor);
-        copy_to_clipboard(text, len);
-        free(text);
-        editor->selection_active = false;
         mode = NORMAL;
         break;
       case 'x':
-        text = get_selection(editor, &len, &start);
+        cut(editor);
-        copy_to_clipboard(text, len);
-        len = count_clusters(text, len, 0, len);
-        edit_erase(editor, start, len);
-        free(text);
-        editor->selection_active = false;
         mode = NORMAL;
         break;
       case 'p':
-        text = get_from_clipboard(&len);
+        paste(editor);
-        if (text) {
-          Coord start, end;
-          if (editor->cursor >= editor->selection) {
-            start = editor->selection;
-            end = move_right(editor, editor->cursor, 1);
-          } else {
-            start = editor->cursor;
-            end = move_right(editor, editor->selection, 1);
-          }
-          uint32_t start_byte =
-              line_to_byte(editor->root, start.row, nullptr) + start.col;
-          uint32_t end_byte =
-              line_to_byte(editor->root, end.row, nullptr) + end.col;
-          edit_erase(editor, start, end_byte - start_byte);
-          edit_insert(editor, editor->cursor, text, len);
-          free(text);
-        }
-        editor->selection_active = false;
         mode = NORMAL;
         break;
       }

src/editor/helpers.cc

@@ -0,0 +1,491 @@
+#include "editor/helpers.h"
+#include "editor/editor.h"
+#include "io/sysio.h"
+#include "lsp/lsp.h"
+#include "main.h"
+#include "utils/utils.h"
+
+void cut(Editor *editor) {
+  if (mode != SELECT)
+    return;
+  Coord start;
+  uint32_t len;
+  char *text = get_selection(editor, &len, &start);
+  copy_to_clipboard(text, len);
+  len = count_clusters(text, len, 0, len);
+  edit_erase(editor, start, len);
+  free(text);
+  editor->selection_active = false;
+}
+
+void copy(Editor *editor) {
+  if (mode != SELECT)
+    return;
+  uint32_t len;
+  char *text = get_selection(editor, &len, nullptr);
+  copy_to_clipboard(text, len);
+  free(text);
+  editor->selection_active = false;
+}
+
+void paste(Editor *editor) {
+  uint32_t len;
+  if (mode == NORMAL) {
+    char *text = get_from_clipboard(&len);
+    if (text) {
+      insert_str(editor, text, len);
+      free(text);
+    }
+  } else if (mode == SELECT) {
+    char *text = get_from_clipboard(&len);
+    if (text) {
+      Coord start, end;
+      selection_bounds(editor, &start, &end);
+      uint32_t start_byte =
+          line_to_byte(editor->root, start.row, nullptr) + start.col;
+      uint32_t end_byte =
+          line_to_byte(editor->root, end.row, nullptr) + end.col;
+      edit_erase(editor, start, end_byte - start_byte);
+      edit_insert(editor, editor->cursor, text, len);
+      free(text);
+    }
+    editor->selection_active = false;
+  }
+}
+
+void insert_str(Editor *editor, char *c, uint32_t len) {
+  if (c) {
+    edit_insert(editor, editor->cursor, c, len);
+    uint32_t grapheme_len = count_clusters(c, len, 0, len);
+    cursor_right(editor, grapheme_len);
+  }
+}
+
+void indent_current_line(Editor *editor) {
+  Coord start = editor->cursor;
+  uint32_t delta = editor->indents.indent_line(editor->cursor.row);
+  editor->cursor.col = start.col + delta;
+  editor->cursor.row = start.row;
+}
+
+void dedent_current_line(Editor *editor) {
+  Coord start = editor->cursor;
+  uint32_t delta = editor->indents.dedent_line(editor->cursor.row);
+  editor->cursor.col = MAX((int64_t)start.col - delta, 0);
+  editor->cursor.row = start.row;
+}
+
+void insert_char(Editor *editor, char c) {
+  uint32_t col = editor->cursor.col;
+  LineIterator *it = begin_l_iter(editor->root, editor->cursor.row);
+  if (!it)
+    return;
+  uint32_t len;
+  char *line = next_line(it, &len);
+  if (!line) {
+    free(it->buffer);
+    free(it);
+    return;
+  }
+  bool skip_insert = false;
+  if (line && col < len) {
+    char next = line[col];
+    if ((c == '}' && next == '}') || (c == ')' && next == ')') ||
+        (c == ']' && next == ']') || (c == '"' && next == '"') ||
+        (c == '\'' && next == '\'')) {
+      cursor_right(editor, 1);
+      skip_insert = true;
+    }
+  }
+  free(it->buffer);
+  free(it);
+  if (!skip_insert) {
+    char closing = 0;
+    switch (c) {
+    case '{':
+      closing = '}';
+      break;
+    case '(':
+      closing = ')';
+      break;
+    case '[':
+      closing = ']';
+      break;
+    case '"':
+      closing = '"';
+      break;
+    case '\'':
+      closing = '\'';
+      break;
+    }
+    if (closing) {
+      char pair[2] = {c, closing};
+      edit_insert(editor, editor->cursor, pair, 2);
+      cursor_right(editor, 1);
+    } else {
+      edit_insert(editor, editor->cursor, &c, 1);
+      cursor_right(editor, 1);
+    }
+    if (editor->lsp && editor->lsp->allow_formatting_on_type) {
+      for (char ch : editor->lsp->format_chars) {
+        if (ch == c) {
+          LineIterator *it = begin_l_iter(editor->root, editor->cursor.row);
+          if (!it)
+            return;
+          uint32_t len;
+          char *line = next_line(it, &len);
+          if (!line) {
+            free(it->buffer);
+            free(it);
+            return;
+          }
+          uint32_t col = utf8_offset_to_utf16(line, len, editor->cursor.col);
+          free(it->buffer);
+          free(it);
+          int version = editor->lsp_version;
+          json message = {
+              {"jsonrpc", "2.0"},
+              {"method", "textDocument/onTypeFormatting"},
+              {"params",
+               {{"textDocument", {{"uri", editor->uri}}},
+                {"position",
+                 {{"line", editor->cursor.row}, {"character", col}}},
+                {"ch", std::string(1, c)},
+                {"options",
+                 {{"tabSize", 2},
+                  {"insertSpaces", true},
+                  {"trimTrailingWhitespace", true},
+                  {"trimFinalNewlines", true}}}}}};
+          LSPPending *pending = new LSPPending();
+          pending->editor = editor;
+          pending->method = "textDocument/onTypeFormatting";
+          pending->callback = [version](Editor *editor, std::string,
+                                        json message) {
+            if (version != editor->lsp_version)
+              return;
+            auto &edits = message["result"];
+            if (edits.is_array()) {
+              std::vector<TextEdit> t_edits;
+              t_edits.reserve(edits.size());
+              for (auto &edit : edits) {
+                TextEdit t_edit;
+                t_edit.text = edit.value("newText", "");
+                t_edit.start.row = edit["range"]["start"]["line"];
+                t_edit.start.col = edit["range"]["start"]["character"];
+                t_edit.end.row = edit["range"]["end"]["line"];
+                t_edit.end.col = edit["range"]["end"]["character"];
+                utf8_normalize_edit(editor, &t_edit);
+                t_edits.push_back(t_edit);
+              }
+              apply_lsp_edits(editor, t_edits, false);
+              ensure_scroll(editor);
+            }
+          };
+          lsp_send(editor->lsp, message, pending);
+          break;
+        }
+      }
+    }
+  }
+}
+
+void normal_mode(Editor *editor) {
+  Coord prev_pos = editor->cursor;
+  mode = NORMAL;
+  cursor_left(editor, 1);
+  if (prev_pos.row != editor->cursor.row)
+    cursor_right(editor, 1);
+}
+
+void backspace_edit(Editor *editor) {
+  Coord prev_pos = editor->cursor;
+  if (prev_pos.col > 0)
+    prev_pos.col--;
+  LineIterator *it = begin_l_iter(editor->root, editor->cursor.row);
+  if (!it)
+    return;
+  char *line = next_line(it, nullptr);
+  char prev_char = line[prev_pos.col];
+  char next_char = line[editor->cursor.col];
+  free(it->buffer);
+  free(it);
+  bool is_pair = (prev_char == '{' && next_char == '}') ||
+                 (prev_char == '(' && next_char == ')') ||
+                 (prev_char == '[' && next_char == ']') ||
+                 (prev_char == '"' && next_char == '"') ||
+                 (prev_char == '\'' && next_char == '\'');
+  if (is_pair) {
+    edit_erase(editor, editor->cursor, 1);
+    edit_erase(editor, prev_pos, 1);
+  } else {
+    edit_erase(editor, editor->cursor, -1);
+  }
+}
+
+void delete_prev_word(Editor *editor) {
+  uint32_t prev_col_byte, prev_col_cluster;
+  word_boundaries(editor, editor->cursor, &prev_col_byte, nullptr,
+                  &prev_col_cluster, nullptr);
+  if (prev_col_byte == editor->cursor.col)
+    edit_erase(editor, editor->cursor, -1);
+  else
+    edit_erase(editor, editor->cursor, -(int64_t)prev_col_cluster);
+}
+
+void delete_next_word(Editor *editor) {
+  uint32_t next_col_byte, next_col_cluster;
+  word_boundaries(editor, editor->cursor, nullptr, &next_col_byte, nullptr,
+                  &next_col_cluster);
+  if (next_col_byte == editor->cursor.col)
+    edit_erase(editor, editor->cursor, 1);
+  else
+    edit_erase(editor, editor->cursor, next_col_cluster);
+}
+
+void clear_hooks_at_line(Editor *editor, uint32_t line) {
+  for (uint8_t i = 0; i < 94; i++)
+    if (editor->hooks[i] == line + 1) {
+      editor->hooks[i] = 0;
+      break;
+    }
+}
+
+void cursor_prev_word(Editor *editor) {
+  uint32_t prev_col;
+  word_boundaries(editor, editor->cursor, &prev_col, nullptr, nullptr, nullptr);
+  editor->cursor_preffered = UINT32_MAX;
+  if (prev_col == editor->cursor.col)
+    cursor_left(editor, 1);
+  else
+    editor->cursor = {editor->cursor.row, prev_col};
+}
+
+void cursor_next_word(Editor *editor) {
+  uint32_t next_col;
+  word_boundaries(editor, editor->cursor, nullptr, &next_col, nullptr, nullptr);
+  editor->cursor_preffered = UINT32_MAX;
+  if (next_col == editor->cursor.col)
+    cursor_right(editor, 1);
+  else
+    editor->cursor = {editor->cursor.row, next_col};
+}
+
+void select_all(Editor *editor) {
+  if (editor->root->line_count > 0) {
+    editor->cursor.row = editor->root->line_count - 1;
+    LineIterator *it = begin_l_iter(editor->root, editor->cursor.row);
+    if (!it)
+      return;
+    uint32_t line_len;
+    char *line = next_line(it, &line_len);
+    if (!line)
+      return;
+    if (line_len > 0 && line[line_len - 1] == '\n')
+      line_len--;
+    line_len = count_clusters(line, line_len, 0, line_len);
+    free(it->buffer);
+    free(it);
+    editor->cursor.col = line_len;
+    editor->cursor_preffered = UINT32_MAX;
+    mode = SELECT;
+    editor->selection_active = true;
+    editor->selection = {0, 0};
+    editor->selection_type = LINE;
+  }
+}
+
+void fetch_lsp_hover(Editor *editor) {
+  if (editor->lsp && editor->lsp->allow_hover) {
+    LineIterator *it = begin_l_iter(editor->root, editor->cursor.row);
+    uint32_t line_len;
+    char *line = next_line(it, &line_len);
+    if (!line) {
+      free(it->buffer);
+      free(it);
+      return;
+    }
+    uint32_t col = utf8_offset_to_utf16(line, line_len, editor->cursor.col);
+    free(it->buffer);
+    free(it);
+    json hover_request = {
+        {"jsonrpc", "2.0"},
+        {"method", "textDocument/hover"},
+        {"params",
+         {{"textDocument", {{"uri", editor->uri}}},
+          {"position", {{"line", editor->cursor.row}, {"character", col}}}}}};
+    LSPPending *pending = new LSPPending();
+    pending->editor = editor;
+    pending->method = "textDocument/hover";
+    pending->callback = [](Editor *editor, std::string, json hover) {
+      if (hover.contains("result") && !hover["result"].is_null()) {
+        auto &contents = hover["result"]["contents"];
+        std::string hover_text = "";
+        bool is_markup = false;
+        if (contents.is_object()) {
+          hover_text += contents["value"].get<std::string>();
+          is_markup = (contents["kind"].get<std::string>() == "markdown");
+        } else if (contents.is_array()) {
+          for (auto &block : contents) {
+            if (block.is_string()) {
+              hover_text += block.get<std::string>() + "\n";
+            } else if (block.is_object() && block.contains("language") &&
+                       block.contains("value")) {
+              std::string lang = block["language"].get<std::string>();
+              std::string val = block["value"].get<std::string>();
+              is_markup = true;
+              hover_text += "```" + lang + "\n" + val + "\n```\n";
+            }
+          }
+        } else if (contents.is_string()) {
+          hover_text += contents.get<std::string>();
+        }
+        if (!hover_text.empty()) {
+          editor->hover.clear();
+          editor->hover.text = clean_text(hover_text);
+          editor->hover.is_markup = is_markup;
+          editor->hover.render_first();
+          editor->hover_active = true;
+        }
+      }
+    };
+    lsp_send(editor->lsp, hover_request, pending);
+  }
+}
+
+void handle_mouse(Editor *editor, KeyEvent event) {
+  static std::chrono::steady_clock::time_point last_click_time =
+      std::chrono::steady_clock::now();
+  static uint32_t click_count = 0;
+  static Coord last_click_pos = {UINT32_MAX, UINT32_MAX};
+  uint8_t old_mode = mode;
+  if (event.key_type == KEY_MOUSE) {
+    auto now = std::chrono::steady_clock::now();
+    auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(
+                        now - last_click_time)
+                        .count();
+    switch (event.mouse_state) {
+    case SCROLL:
+      switch (event.mouse_direction) {
+      case SCROLL_UP:
+        scroll_up(editor, 4);
+        ensure_cursor(editor);
+        break;
+      case SCROLL_DOWN:
+        scroll_down(editor, 4);
+        ensure_cursor(editor);
+        break;
+      case SCROLL_LEFT:
+        cursor_left(editor, 10);
+        break;
+      case SCROLL_RIGHT:
+        cursor_right(editor, 10);
+        break;
+      }
+      break;
+    case PRESS:
+      if (event.mouse_button == LEFT_BTN) {
+        Coord cur_pos = {event.mouse_x, event.mouse_y};
+        if (duration < 250 && last_click_pos == cur_pos)
+          click_count++;
+        else
+          click_count = 1;
+        last_click_time = now;
+        last_click_pos = cur_pos;
+        Coord p = editor_hit_test(editor, event.mouse_x, event.mouse_y);
+        if (p.row == UINT32_MAX && p.col == UINT32_MAX)
+          return;
+        editor->cursor_preffered = UINT32_MAX;
+        if (click_count == 1) {
+          editor->cursor = p;
+          editor->selection = p;
+          if (mode == SELECT) {
+            mode = NORMAL;
+            editor->selection_active = false;
+          }
+        } else if (click_count == 2) {
+          uint32_t prev_col, next_col;
+          word_boundaries(editor, editor->cursor, &prev_col, &next_col, nullptr,
+                          nullptr);
+          if (editor->cursor < editor->selection)
+            editor->cursor = {editor->cursor.row, prev_col};
+          else
+            editor->cursor = {editor->cursor.row, next_col};
+          editor->cursor_preffered = UINT32_MAX;
+          editor->selection_type = WORD;
+          mode = SELECT;
+          editor->selection_active = true;
+        } else if (click_count >= 3) {
+          if (editor->cursor < editor->selection) {
+            editor->cursor = {p.row, 0};
+          } else {
+            uint32_t line_len;
+            LineIterator *it = begin_l_iter(editor->root, p.row);
+            char *line = next_line(it, &line_len);
+            if (!line)
+              return;
+            if (line_len > 0 && line[line_len - 1] == '\n')
+              line_len--;
+            free(it->buffer);
+            free(it);
+            editor->cursor = {p.row, line_len};
+          }
+          editor->cursor_preffered = UINT32_MAX;
+          editor->selection_type = LINE;
+          mode = SELECT;
+          editor->selection_active = true;
+          click_count = 3;
+        }
+      }
+      break;
+    case DRAG:
+      if (event.mouse_button == LEFT_BTN) {
+        Coord p = editor_hit_test(editor, event.mouse_x, event.mouse_y);
+        if (p.row == UINT32_MAX && p.col == UINT32_MAX)
+          return;
+        editor->cursor_preffered = UINT32_MAX;
+        mode = SELECT;
+        if (!editor->selection_active) {
+          editor->selection_active = true;
+          editor->selection_type = CHAR;
+        }
+        uint32_t prev_col, next_col, line_len;
+        switch (editor->selection_type) {
+        case CHAR:
+          editor->cursor = p;
+          break;
+        case WORD:
+          word_boundaries(editor, p, &prev_col, &next_col, nullptr, nullptr);
+          if (editor->cursor < editor->selection)
+            editor->cursor = {p.row, prev_col};
+          else
+            editor->cursor = {p.row, next_col};
+          break;
+        case LINE:
+          if (editor->cursor < editor->selection) {
+            editor->cursor = {p.row, 0};
+          } else {
+            LineIterator *it = begin_l_iter(editor->root, p.row);
+            char *line = next_line(it, &line_len);
+            if (!line)
+              return;
+            if (line_len > 0 && line[line_len - 1] == '\n')
+              line_len--;
+            free(it->buffer);
+            free(it);
+            editor->cursor = {p.row, line_len};
+          }
+          break;
+        }
+      }
+      break;
+    case RELEASE:
+      if (event.mouse_button == LEFT_BTN)
+        if (editor->cursor.row == editor->selection.row &&
+            editor->cursor.col == editor->selection.col) {
+          mode = NORMAL;
+          editor->selection_active = false;
+        }
+      break;
+    }
+  }
+}

src/editor/indents.cc

@@ -323,13 +323,14 @@ void IndentationEngine::insert_new_line(Coord cursor) {
       LineIterator *it = begin_l_iter(editor->root, editor->cursor.row);
       if (!it)
         return;
-      char *line = next_line(it, nullptr);
+      uint32_t line_len;
+      char *line = next_line(it, &line_len);
       if (!line) {
         free(it->buffer);
         free(it);
         return;
       }
-      uint32_t col = utf8_byte_offset_to_utf16(line, editor->cursor.col);
+      uint32_t col = utf8_offset_to_utf16(line, line_len, editor->cursor.col);
       free(it->buffer);
       free(it);
       int version = editor->lsp_version;

src/editor/lsp.cc

@@ -60,10 +60,10 @@ void editor_lsp_handle(Editor *editor, json msg) {
       if (len > 0 && line[len - 1] == '\n')
         --len;
       lock.unlock();
-      w.start = utf16_offset_to_utf8(line, w.start);
+      w.start = utf16_offset_to_utf8(line, len, w.start);
       uint32_t end = d["range"]["end"]["character"];
       if (d["range"]["end"]["line"] == w.line)
-        w.end = utf16_offset_to_utf8(line, end);
+        w.end = utf16_offset_to_utf8(line, len, end);
       free(it->buffer);
       free(it);
       std::string text = trim(d["message"].get<std::string>());

src/editor/renderer.cc

@@ -107,10 +107,12 @@ void render_editor(Editor *editor) {
   while (rendered_rows < editor->size.row) {
     uint32_t line_len;
     char *line = next_line(it, &line_len);
-    if (line_data)
+    if (editor->parser) {
-      line_data = editor->parser->line_tree.next();
+      if (line_data)
-    else
+        line_data = editor->parser->line_tree.next();
-      line_data = editor->parser->line_tree.start_iter(line_index);
+      else
+        line_data = editor->parser->line_tree.start_iter(line_index);
+    }
     if (!line)
       break;
     if (line_len > 0 && line[line_len - 1] == '\n')
@@ -186,7 +188,7 @@ void render_editor(Editor *editor) {
                    : 0);
         if (editor->selection_active && absolute_byte_pos >= sel_start &&
             absolute_byte_pos < sel_end)
-          bg = 0x555555;
+          bg = bg | 0x555555;
         uint32_t u_color = 0;
         for (const auto &w : line_warnings) {
           if (w.start <= current_byte_offset + local_render_offset &&

src/editor/selection.cc

@@ -1,17 +1,18 @@
 #include "editor/editor.h"
+#include "utils/utils.h"
 
-char *get_selection(Editor *editor, uint32_t *out_len, Coord *out_start) {
+void selection_bounds(Editor *editor, Coord *out_start, Coord *out_end) {
   std::shared_lock lock(editor->knot_mtx);
   Coord start, end;
   if (editor->cursor >= editor->selection) {
-    uint32_t prev_col, next_col;
+    uint32_t prev_col;
     switch (editor->selection_type) {
     case CHAR:
       start = editor->selection;
       end = move_right(editor, editor->cursor, 1);
       break;
     case WORD:
-      word_boundaries(editor, editor->selection, &prev_col, &next_col, nullptr,
+      word_boundaries(editor, editor->selection, &prev_col, nullptr, nullptr,
                       nullptr);
       start = {editor->selection.row, prev_col};
       end = editor->cursor;
@@ -23,13 +24,65 @@ char *get_selection(Editor *editor, uint32_t *out_len, Coord *out_start) {
     }
   } else {
     start = editor->cursor;
-    uint32_t prev_col, next_col, line_len;
+    uint32_t next_col, line_len;
     switch (editor->selection_type) {
     case CHAR:
       end = move_right(editor, editor->selection, 1);
       break;
     case WORD:
-      word_boundaries(editor, editor->selection, &prev_col, &next_col, nullptr,
+      word_boundaries(editor, editor->selection, nullptr, &next_col, nullptr,
+                      nullptr);
+      end = {editor->selection.row, next_col};
+      break;
+    case LINE:
+      LineIterator *it = begin_l_iter(editor->root, editor->selection.row);
+      char *line = next_line(it, &line_len);
+      if (!line)
+        return;
+      if (line_len > 0 && line[line_len - 1] == '\n')
+        line_len--;
+      end = {editor->selection.row, line_len};
+      free(it->buffer);
+      free(it);
+      break;
+    }
+  }
+  if (out_start)
+    *out_start = start;
+  if (out_end)
+    *out_end = end;
+}
+
+char *get_selection(Editor *editor, uint32_t *out_len, Coord *out_start) {
+  std::shared_lock lock(editor->knot_mtx);
+  Coord start, end;
+  if (editor->cursor >= editor->selection) {
+    uint32_t prev_col;
+    switch (editor->selection_type) {
+    case CHAR:
+      start = editor->selection;
+      end = move_right(editor, editor->cursor, 1);
+      break;
+    case WORD:
+      word_boundaries(editor, editor->selection, &prev_col, nullptr, nullptr,
+                      nullptr);
+      start = {editor->selection.row, prev_col};
+      end = editor->cursor;
+      break;
+    case LINE:
+      start = {editor->selection.row, 0};
+      end = editor->cursor;
+      break;
+    }
+  } else {
+    start = editor->cursor;
+    uint32_t next_col, line_len;
+    switch (editor->selection_type) {
+    case CHAR:
+      end = move_right(editor, editor->selection, 1);
+      break;
+    case WORD:
+      word_boundaries(editor, editor->selection, nullptr, &next_col, nullptr,
                       nullptr);
       end = {editor->selection.row, next_col};
       break;

src/io/renderer.cc

@@ -154,7 +154,7 @@ void render() {
               }
             }
         }
-        last_change_col = MIN(cols + 1, col + 4);
+        last_change_col = MIN(cols - 1, col + 4);
       }
     }
     if (first_change_col == -1)

src/lsp/process.cc

@@ -54,6 +54,12 @@ std::shared_ptr<LSPInstance> get_or_init_lsp(uint8_t lsp_id) {
     pending->callback = [lsp, lsp_id](Editor *, std::string, json msg) {
       if (msg.contains("result") && msg["result"].contains("capabilities")) {
         auto &caps = msg["result"]["capabilities"];
+        if (caps.contains("positionEncoding")) {
+          std::string s = caps["positionEncoding"].get<std::string>();
+          if (s == "utf-8")
+            lsp->is_utf8 = true;
+          log("Lsp name: %s, supports: %s", lsp->lsp->command, s.c_str());
+        }
         if (caps.contains("textDocumentSync")) {
           auto &sync = caps["textDocumentSync"];
           if (sync.is_number()) {
@@ -65,8 +71,9 @@ std::shared_ptr<LSPInstance> get_or_init_lsp(uint8_t lsp_id) {
           }
         }
         lsp->allow_formatting = caps.value("documentFormattingProvider", false);
-        if (lsp_id != LUA_LS /* Lua ls gives terrible ontype formatting */ &&
+        if (lsp_id !=
-            caps.contains("documentOnTypeFormattingProvider")) {
+                LUA_LS /* Lua ls gives terrible ontype formatting so disable */
+            && caps.contains("documentOnTypeFormattingProvider")) {
           auto &fmt = caps["documentOnTypeFormattingProvider"];
           if (fmt.is_object()) {
             if (fmt.contains("firstTriggerCharacter")) {

src/main.cc

@@ -2,7 +2,6 @@
 #include "editor/editor.h"
 #include "io/sysio.h"
 #include "lsp/lsp.h"
-#include "syntax/decl.h"
 #include "ui/bar.h"
 #include "utils/utils.h"
 
@@ -60,9 +59,15 @@ int main(int argc, char *argv[]) {
   Coord screen = start_screen();
   const char *filename = (argc > 1) ? argv[1] : "";
 
-  system(("bash " + get_exe_dir() + "/../scripts/init.sh").c_str());
+  int state;
+  VALUE result;
+  result = rb_eval_string_protect("puts 'Hello, world!'", &state);
 
-  load_theme(get_exe_dir() + "/../themes/default.json");
+  if (state) {
+    /* handle exception */
+  }
+
+  load_theme();
 
   Editor *editor = new_editor(filename, {0, 0}, {screen.row - 2, screen.col});
   Bar bar(screen);

src/scripting/process.cc

@@ -0,0 +1,145 @@
+#include "scripting/decl.h"
+#include "syntax/decl.h"
+#include "utils/utils.h"
+
+struct ThemeEntry {
+  std::string key;
+  uint32_t fg = 0xFFFFFF;
+  uint32_t bg = 0x000000;
+  bool italic = false;
+  bool bold = false;
+  bool underline = false;
+  bool strikethrough = false;
+};
+
+VALUE C_module = Qnil;
+
+void ruby_start(const char *main_file) {
+  USING(Language);
+  ruby_init();
+  ruby_init_loadpath();
+  int state = 0;
+  rb_load_protect(rb_str_new_cstr(main_file), 0, &state);
+  if (state) {
+    VALUE err = rb_errinfo();
+    rb_set_errinfo(Qnil);
+    fprintf(stderr, "Failed to load Ruby file\n");
+  }
+  C_module = rb_const_get(rb_cObject, rb_intern("C"));
+  if (C_module == Qnil)
+    return;
+  VALUE block = rb_funcall(C_module, rb_intern("b_startup"), 0);
+  if (block != Qnil)
+    rb_funcall(block, rb_intern("call"), 0);
+}
+
+void ruby_shutdown() {
+  if (C_module == Qnil)
+    return;
+  VALUE block = rb_funcall(C_module, rb_intern("b_shutdown"), 0);
+  if (block != Qnil)
+    rb_funcall(block, rb_intern("call"), 0);
+  ruby_finalize();
+}
+
+static std::vector<ThemeEntry> read_theme() {
+  std::vector<ThemeEntry> result;
+  if (C_module == Qnil)
+    return result;
+  VALUE theme_hash = rb_funcall(C_module, rb_intern("theme"), 0);
+  if (NIL_P(theme_hash))
+    return result;
+  VALUE keys = rb_funcall(theme_hash, rb_intern("keys"), 0);
+  for (long i = 0; i < RARRAY_LEN(keys); ++i) {
+    VALUE key_sym = rb_ary_entry(keys, i);
+    std::string key = rb_id2name(SYM2ID(key_sym));
+    VALUE val_hash = rb_hash_aref(theme_hash, key_sym);
+    if (NIL_P(val_hash))
+      continue;
+    ThemeEntry entry;
+    entry.key = key;
+    VALUE fg = rb_hash_aref(val_hash, ID2SYM(rb_intern("fg")));
+    VALUE bg = rb_hash_aref(val_hash, ID2SYM(rb_intern("bg")));
+    VALUE italic = rb_hash_aref(val_hash, ID2SYM(rb_intern("italic")));
+    VALUE bold = rb_hash_aref(val_hash, ID2SYM(rb_intern("bold")));
+    VALUE underline = rb_hash_aref(val_hash, ID2SYM(rb_intern("underline")));
+    VALUE strikethrough =
+        rb_hash_aref(val_hash, ID2SYM(rb_intern("strikethrough")));
+    if (!NIL_P(fg))
+      entry.fg = NUM2UINT(fg);
+    if (!NIL_P(bg))
+      entry.bg = NUM2UINT(bg);
+    if (!NIL_P(italic))
+      entry.italic = RTEST(italic);
+    if (!NIL_P(bold))
+      entry.bold = RTEST(bold);
+    if (!NIL_P(underline))
+      entry.underline = RTEST(underline);
+    if (!NIL_P(strikethrough))
+      entry.strikethrough = RTEST(strikethrough);
+    result.push_back(entry);
+  }
+  return result;
+}
+
+void load_theme() {
+  std::vector<ThemeEntry> entries = read_theme();
+  Highlight default_hl = {0xFFFFFF, 0, 0};
+  for (auto &entry : entries) {
+    if (entry.key == "default") {
+      default_hl.fg = entry.fg;
+      default_hl.bg = entry.bg;
+      if (entry.italic)
+        default_hl.flags |= CF_ITALIC;
+      if (entry.bold)
+        default_hl.flags |= CF_BOLD;
+      if (entry.underline)
+        default_hl.flags |= CF_UNDERLINE;
+      if (entry.strikethrough)
+        default_hl.flags |= CF_STRIKETHROUGH;
+      break;
+    }
+  }
+  for (auto &hl : highlights)
+    hl = default_hl;
+  for (auto &entry : entries) {
+    if (entry.key == "default")
+      continue;
+    std::string key = "k_" + entry.key;
+    for (char &c : key)
+      c = std::toupper(static_cast<unsigned char>(c));
+    auto it = kind_map.find(key);
+    if (it == kind_map.end())
+      continue;
+    Highlight hl = {0xFFFFFF, 0, 0};
+    hl.fg = entry.fg;
+    hl.bg = entry.bg;
+    if (entry.italic)
+      hl.flags |= CF_ITALIC;
+    if (entry.bold)
+      hl.flags |= CF_BOLD;
+    if (entry.underline)
+      hl.flags |= CF_UNDERLINE;
+    if (entry.strikethrough)
+      hl.flags |= CF_STRIKETHROUGH;
+    highlights[static_cast<uint8_t>(it->second)] = hl;
+  }
+}
+
+std::string read_line_endings() {
+  if (C_module == Qnil)
+    return "";
+  VALUE le = rb_funcall(C_module, rb_intern("line_endings"), 0);
+  if (SYMBOL_P(le))
+    return rb_id2name(SYM2ID(le));
+  return "";
+}
+
+std::string read_utf_mode() {
+  if (C_module == Qnil)
+    return "";
+  VALUE utf = rb_funcall(C_module, rb_intern("utf_mode"), 0);
+  if (SYMBOL_P(utf))
+    return rb_id2name(SYM2ID(utf));
+  return "";
+}

src/syntax/ruby.cc

@@ -324,7 +324,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
     if (state->full_state->in_state == RubyFullState::END)
       return state;
     if (state->full_state->in_state == RubyFullState::COMMENT) {
-      tokens->push_back({i, len, TokenKind::Comment});
+      tokens->push_back({i, len, TokenKind::K_COMMENT});
       if (i == 0 && len == 4 && text[i] == '=' && text[i + 1] == 'e' &&
           text[i + 2] == 'n' && text[i + 3] == 'd') {
         state->full_state->in_state = RubyFullState::NONE;
@@ -344,18 +344,18 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
           state->heredocs.pop_front();
           if (state->heredocs.empty())
             state->full_state->in_state = RubyFullState::NONE;
-          tokens->push_back({i, len, TokenKind::Annotation});
+          tokens->push_back({i, len, TokenKind::K_ANNOTATION});
           return state;
         }
       }
       uint32_t start = i;
       if (!state->heredocs.front().allow_interpolation) {
-        tokens->push_back({i, len, TokenKind::String});
+        tokens->push_back({i, len, TokenKind::K_STRING});
         return state;
       } else {
         while (i < len) {
           if (text[i] == '\\') {
-            tokens->push_back({start, i, TokenKind::String});
+            tokens->push_back({start, i, TokenKind::K_STRING});
             start = i;
             i++;
             if (i < len && text[i] == 'x') {
@@ -412,12 +412,12 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
               if (i < len)
                 i++;
             }
-            tokens->push_back({start, i, TokenKind::Escape});
+            tokens->push_back({start, i, TokenKind::K_ESCAPE});
             continue;
           }
           if (text[i] == '#' && i + 1 < len && text[i + 1] == '{') {
-            tokens->push_back({start, i, TokenKind::String});
+            tokens->push_back({start, i, TokenKind::K_STRING});
-            tokens->push_back({i, i + 2, TokenKind::Interpolation});
+            tokens->push_back({i, i + 2, TokenKind::K_INTERPOLATION});
             i += 2;
             state->interp_stack.push(state->full_state);
             state->full_state = std::make_shared<RubyFullState>();
@@ -427,7 +427,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
           i++;
         }
         if (i == len)
-          tokens->push_back({start, len, TokenKind::String});
+          tokens->push_back({start, len, TokenKind::K_STRING});
         continue;
       }
     }
@@ -435,7 +435,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
       uint32_t start = i;
       while (i < len) {
         if (text[i] == '\\') {
-          tokens->push_back({start, i, TokenKind::String});
+          tokens->push_back({start, i, TokenKind::K_STRING});
           start = i;
           i++;
           if (i < len && text[i] == 'x') {
@@ -492,13 +492,13 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
             if (i < len)
               i++;
           }
-          tokens->push_back({start, i, TokenKind::Escape});
+          tokens->push_back({start, i, TokenKind::K_ESCAPE});
           continue;
         }
         if (state->full_state->lit.allow_interp && text[i] == '#' &&
             i + 1 < len && text[i + 1] == '{') {
-          tokens->push_back({start, i, TokenKind::String});
+          tokens->push_back({start, i, TokenKind::K_STRING});
-          tokens->push_back({i, i + 2, TokenKind::Interpolation});
+          tokens->push_back({i, i + 2, TokenKind::K_INTERPOLATION});
           i += 2;
           state->interp_stack.push(state->full_state);
           state->full_state = std::make_shared<RubyFullState>();
@@ -514,7 +514,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
           if (state->full_state->lit.delim_start ==
               state->full_state->lit.delim_end) {
             i++;
-            tokens->push_back({start, i, TokenKind::String});
+            tokens->push_back({start, i, TokenKind::K_STRING});
             state->full_state->in_state = RubyFullState::NONE;
             state->full_state->expecting_expr = false;
             break;
@@ -522,7 +522,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
             state->full_state->lit.brace_level--;
             if (state->full_state->lit.brace_level == 0) {
               i++;
-              tokens->push_back({start, i, TokenKind::String});
+              tokens->push_back({start, i, TokenKind::K_STRING});
               state->full_state->in_state = RubyFullState::NONE;
               state->full_state->expecting_expr = false;
               break;
@@ -532,14 +532,14 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
         i++;
       }
       if (i == len)
-        tokens->push_back({start, len, TokenKind::String});
+        tokens->push_back({start, len, TokenKind::K_STRING});
       continue;
     }
     if (state->full_state->in_state == RubyFullState::REGEXP) {
       uint32_t start = i;
       while (i < len) {
         if (text[i] == '\\') {
-          tokens->push_back({start, i, TokenKind::Regexp});
+          tokens->push_back({start, i, TokenKind::K_REGEXP});
           start = i;
           i++;
           if (i < len && text[i] == 'x') {
@@ -596,12 +596,12 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
             if (i < len)
               i++;
           }
-          tokens->push_back({start, i, TokenKind::Escape});
+          tokens->push_back({start, i, TokenKind::K_ESCAPE});
           continue;
         }
         if (text[i] == '#' && i + 1 < len && text[i + 1] == '{') {
-          tokens->push_back({start, i, TokenKind::Regexp});
+          tokens->push_back({start, i, TokenKind::K_REGEXP});
-          tokens->push_back({i, i + 2, TokenKind::Interpolation});
+          tokens->push_back({i, i + 2, TokenKind::K_INTERPOLATION});
           i += 2;
           state->interp_stack.push(state->full_state);
           state->full_state = std::make_shared<RubyFullState>();
@@ -617,7 +617,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
           if (state->full_state->lit.delim_start ==
               state->full_state->lit.delim_end) {
             i += 1;
-            tokens->push_back({start, i, TokenKind::Regexp});
+            tokens->push_back({start, i, TokenKind::K_REGEXP});
             state->full_state->in_state = RubyFullState::NONE;
             state->full_state->expecting_expr = false;
             break;
@@ -625,7 +625,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
             state->full_state->lit.brace_level--;
             if (state->full_state->lit.brace_level == 0) {
               i += 1;
-              tokens->push_back({start, i, TokenKind::Regexp});
+              tokens->push_back({start, i, TokenKind::K_REGEXP});
               state->full_state->in_state = RubyFullState::NONE;
               state->full_state->expecting_expr = false;
               break;
@@ -635,7 +635,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
         i++;
       }
       if (i == len)
-        tokens->push_back({start, len, TokenKind::Regexp});
+        tokens->push_back({start, len, TokenKind::K_REGEXP});
       continue;
     }
     if (i == 0 && len == 6) {
@@ -643,7 +643,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
           text[i + 3] == 'g' && text[i + 4] == 'i' && text[i + 5] == 'n') {
         state->full_state->in_state = RubyFullState::COMMENT;
         state->full_state->expecting_expr = false;
-        tokens->push_back({0, len, TokenKind::Comment});
+        tokens->push_back({0, len, TokenKind::K_COMMENT});
         return state;
       }
     }
@@ -664,7 +664,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
         indented = true;
       if (text[j] == '~' || text[j] == '-')
         j++;
-      tokens->push_back({i, j, TokenKind::Operator});
+      tokens->push_back({i, j, TokenKind::K_OPERATOR});
       if (j >= len)
         continue;
       std::string delim;
@@ -685,7 +685,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
       }
       state->full_state->expecting_expr = false;
       if (!delim.empty()) {
-        tokens->push_back({s, j, TokenKind::Annotation});
+        tokens->push_back({s, j, TokenKind::K_ANNOTATION});
         state->heredocs.push_back({delim, interpolation, indented});
         state->full_state->in_state = RubyFullState::HEREDOC;
         heredoc_first = true;
@@ -694,7 +694,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
       continue;
     }
     if (text[i] == '/' && state->full_state->expecting_expr) {
-      tokens->push_back({i, i + 1, TokenKind::Regexp});
+      tokens->push_back({i, i + 1, TokenKind::K_REGEXP});
       state->full_state->in_state = RubyFullState::REGEXP;
       state->full_state->expecting_expr = false;
       state->full_state->lit.delim_start = '/';
@@ -705,10 +705,10 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
     } else if (text[i] == '#') {
       if (i == 0 && len > 4 && text[i + 1] == '!') {
         state->full_state->expecting_expr = false;
-        tokens->push_back({0, len, TokenKind::Shebang});
+        tokens->push_back({0, len, TokenKind::K_SHEBANG});
         return state;
       }
-      tokens->push_back({i, len, TokenKind::Comment});
+      tokens->push_back({i, len, TokenKind::K_COMMENT});
       state->full_state->expecting_expr = false;
       return state;
     } else if (text[i] == '.') {
@@ -720,7 +720,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
           i++;
         }
       }
-      tokens->push_back({start, i, TokenKind::Operator});
+      tokens->push_back({start, i, TokenKind::K_OPERATOR});
       state->full_state->expecting_expr = false;
       continue;
     } else if (text[i] == ':') {
@@ -728,7 +728,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
       uint32_t start = i;
       i++;
       if (i >= len) {
-        tokens->push_back({start, i, TokenKind::Operator});
+        tokens->push_back({start, i, TokenKind::K_OPERATOR});
         state->full_state->expecting_expr = true;
         continue;
       }
@@ -737,7 +737,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
         continue;
       }
       if (text[i] == '\'' || text[i] == '"') {
-        tokens->push_back({start, i, TokenKind::Label});
+        tokens->push_back({start, i, TokenKind::K_LABEL});
         state->full_state->expecting_expr = true;
         continue;
       }
@@ -748,22 +748,22 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
           i++;
         while (i < len && identifier_char(text[i]))
           i++;
-        tokens->push_back({start, i, TokenKind::Label});
+        tokens->push_back({start, i, TokenKind::K_LABEL});
         continue;
       }
       uint32_t op_len = operator_trie.match(text, i, len, identifier_char);
       if (op_len > 0) {
-        tokens->push_back({start, i + op_len, TokenKind::Label});
+        tokens->push_back({start, i + op_len, TokenKind::K_LABEL});
         i += op_len;
         continue;
       }
       if (identifier_start_char(text[i])) {
         uint32_t word_len = get_next_word(text, i, len);
-        tokens->push_back({start, i + word_len, TokenKind::Label});
+        tokens->push_back({start, i + word_len, TokenKind::K_LABEL});
         i += word_len;
         continue;
       }
-      tokens->push_back({start, i, TokenKind::Operator});
+      tokens->push_back({start, i, TokenKind::K_OPERATOR});
       continue;
     } else if (text[i] == '@') {
       state->full_state->expecting_expr = false;
@@ -779,7 +779,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
         continue;
       while (i < len && identifier_char(text[i]))
         i++;
-      tokens->push_back({start, i, TokenKind::VariableInstance});
+      tokens->push_back({start, i, TokenKind::K_VARIABLEINSTANCE});
       continue;
     } else if (text[i] == '$') {
       state->full_state->expecting_expr = false;
@@ -802,7 +802,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
       } else {
         continue;
       }
-      tokens->push_back({start, i, TokenKind::VariableGlobal});
+      tokens->push_back({start, i, TokenKind::K_VARIABLEGLOBAL});
       continue;
     } else if (text[i] == '?') {
       state->full_state->expecting_expr = false;
@@ -818,7 +818,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
             continue;
           if (i < len && isxdigit(text[i]))
             i++;
-          tokens->push_back({start, i, TokenKind::Char});
+          tokens->push_back({start, i, TokenKind::K_CHAR});
           continue;
         } else if (i < len && text[i] == 'u') {
           i++;
@@ -838,26 +838,26 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
             i++;
           else
             continue;
-          tokens->push_back({start, i, TokenKind::Char});
+          tokens->push_back({start, i, TokenKind::K_CHAR});
           continue;
         } else if (i < len) {
           i++;
-          tokens->push_back({start, i, TokenKind::Char});
+          tokens->push_back({start, i, TokenKind::K_CHAR});
           continue;
         }
       } else if (i < len && text[i] != ' ') {
         i++;
-        tokens->push_back({start, i, TokenKind::Char});
+        tokens->push_back({start, i, TokenKind::K_CHAR});
         continue;
       } else {
         state->full_state->expecting_expr = true;
-        tokens->push_back({start, i, TokenKind::Operator});
+        tokens->push_back({start, i, TokenKind::K_OPERATOR});
         continue;
       }
     } else if (text[i] == '{') {
       state->full_state->expecting_expr = true;
       uint8_t brace_color =
-          (uint8_t)TokenKind::Brace1 + (state->full_state->brace_level % 5);
+          (uint8_t)TokenKind::K_BRACE1 + (state->full_state->brace_level % 5);
       tokens->push_back({i, i + 1, (TokenKind)brace_color});
       state->interp_level++;
       state->full_state->brace_level++;
@@ -869,11 +869,11 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
       if (state->interp_level == 0 && !state->interp_stack.empty()) {
         state->full_state = state->interp_stack.top();
         state->interp_stack.pop();
-        tokens->push_back({i, i + 1, TokenKind::Interpolation});
+        tokens->push_back({i, i + 1, TokenKind::K_INTERPOLATION});
       } else {
         state->full_state->brace_level--;
         uint8_t brace_color =
-            (uint8_t)TokenKind::Brace1 + (state->full_state->brace_level % 5);
+            (uint8_t)TokenKind::K_BRACE1 + (state->full_state->brace_level % 5);
         tokens->push_back({i, i + 1, (TokenKind)brace_color});
       }
       i++;
@@ -881,7 +881,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
     } else if (text[i] == '(') {
       state->full_state->expecting_expr = true;
       uint8_t brace_color =
-          (uint8_t)TokenKind::Brace1 + (state->full_state->brace_level % 5);
+          (uint8_t)TokenKind::K_BRACE1 + (state->full_state->brace_level % 5);
       tokens->push_back({i, i + 1, (TokenKind)brace_color});
       state->full_state->brace_level++;
       i++;
@@ -890,14 +890,14 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
       state->full_state->expecting_expr = false;
       state->full_state->brace_level--;
       uint8_t brace_color =
-          (uint8_t)TokenKind::Brace1 + (state->full_state->brace_level % 5);
+          (uint8_t)TokenKind::K_BRACE1 + (state->full_state->brace_level % 5);
       tokens->push_back({i, i + 1, (TokenKind)brace_color});
       i++;
       continue;
     } else if (text[i] == '[') {
       state->full_state->expecting_expr = true;
       uint8_t brace_color =
-          (uint8_t)TokenKind::Brace1 + (state->full_state->brace_level % 5);
+          (uint8_t)TokenKind::K_BRACE1 + (state->full_state->brace_level % 5);
       tokens->push_back({i, i + 1, (TokenKind)brace_color});
       state->full_state->brace_level++;
       i++;
@@ -906,13 +906,13 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
       state->full_state->expecting_expr = false;
       state->full_state->brace_level--;
       uint8_t brace_color =
-          (uint8_t)TokenKind::Brace1 + (state->full_state->brace_level % 5);
+          (uint8_t)TokenKind::K_BRACE1 + (state->full_state->brace_level % 5);
       tokens->push_back({i, i + 1, (TokenKind)brace_color});
       i++;
       continue;
     } else if (text[i] == '\'') {
       state->full_state->expecting_expr = false;
-      tokens->push_back({i, i + 1, TokenKind::String});
+      tokens->push_back({i, i + 1, TokenKind::K_STRING});
       state->full_state->in_state = RubyFullState::STRING;
       state->full_state->lit.delim_start = '\'';
       state->full_state->lit.delim_end = '\'';
@@ -921,7 +921,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
       continue;
     } else if (text[i] == '"') {
       state->full_state->expecting_expr = false;
-      tokens->push_back({i, i + 1, TokenKind::String});
+      tokens->push_back({i, i + 1, TokenKind::K_STRING});
       state->full_state->in_state = RubyFullState::STRING;
       state->full_state->lit.delim_start = '"';
       state->full_state->lit.delim_end = '"';
@@ -930,7 +930,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
       continue;
     } else if (text[i] == '`') {
       state->full_state->expecting_expr = false;
-      tokens->push_back({i, i + 1, TokenKind::String});
+      tokens->push_back({i, i + 1, TokenKind::K_STRING});
       state->full_state->in_state = RubyFullState::STRING;
       state->full_state->lit.delim_start = '`';
       state->full_state->lit.delim_end = '`';
@@ -1001,8 +1001,9 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
         delim_end = delim_start;
         break;
       }
-      tokens->push_back({i, i + prefix_len + 1,
+      tokens->push_back(
-                         (is_regexp ? TokenKind::Regexp : TokenKind::String)});
+          {i, i + prefix_len + 1,
+           (is_regexp ? TokenKind::K_REGEXP : TokenKind::K_STRING)});
       state->full_state->in_state =
           is_regexp ? RubyFullState::REGEXP : RubyFullState::STRING;
       state->full_state->lit.delim_start = delim_start;
@@ -1110,47 +1111,47 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
             i--;
         }
       }
-      tokens->push_back({start, i, TokenKind::Number});
+      tokens->push_back({start, i, TokenKind::K_NUMBER});
       continue;
     } else if (identifier_start_char(text[i])) {
       state->full_state->expecting_expr = false;
       uint32_t length;
       if ((length = base_keywords_trie.match(text, i, len, identifier_char))) {
-        tokens->push_back({i, i + length, TokenKind::Keyword});
+        tokens->push_back({i, i + length, TokenKind::K_KEYWORD});
         i += length;
         continue;
       } else if ((length = expecting_keywords_trie.match(text, i, len,
                                                          identifier_char))) {
         state->full_state->expecting_expr = true;
-        tokens->push_back({i, i + length, TokenKind::Keyword});
+        tokens->push_back({i, i + length, TokenKind::K_KEYWORD});
         i += length;
         continue;
       } else if ((length = operator_keywords_trie.match(text, i, len,
                                                         identifier_char))) {
-        tokens->push_back({i, i + length, TokenKind::KeywordOperator});
+        tokens->push_back({i, i + length, TokenKind::K_KEYWORDOPERATOR});
         i += length;
         continue;
       } else if ((length = expecting_operators_trie.match(
                       text, i, len, identifier_char)) > 0) {
         state->full_state->expecting_expr = true;
-        tokens->push_back({i, i + length, TokenKind::KeywordOperator});
+        tokens->push_back({i, i + length, TokenKind::K_KEYWORDOPERATOR});
         i += length;
         continue;
       } else if ((length = types_trie.match(text, i, len, identifier_char))) {
-        tokens->push_back({i, i + length, TokenKind::Type});
+        tokens->push_back({i, i + length, TokenKind::K_TYPE});
         i += length;
         continue;
       } else if ((length = methods_trie.match(text, i, len, identifier_char))) {
-        tokens->push_back({i, i + length, TokenKind::Function});
+        tokens->push_back({i, i + length, TokenKind::K_FUNCTION});
         i += length;
         continue;
       } else if ((length =
                       builtins_trie.match(text, i, len, identifier_char))) {
-        tokens->push_back({i, i + length, TokenKind::Constant});
+        tokens->push_back({i, i + length, TokenKind::K_CONSTANT});
         i += length;
         continue;
       } else if ((length = errors_trie.match(text, i, len, identifier_char))) {
-        tokens->push_back({i, i + length, TokenKind::Error});
+        tokens->push_back({i, i + length, TokenKind::K_ERROR});
         i += length;
         continue;
       } else if (text[i] >= 'A' && text[i] <= 'Z') {
@@ -1158,10 +1159,10 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
         i += get_next_word(text, i, len);
         if (i - start >= 5 && text[i - 5] == 'E' && text[i - 4] == 'r' &&
             text[i - 3] == 'r' && text[i - 2] == 'o' && text[i - 1] == 'r') {
-          tokens->push_back({start, i, TokenKind::Error});
+          tokens->push_back({start, i, TokenKind::K_ERROR});
           continue;
         }
-        tokens->push_back({start, i, TokenKind::Constant});
+        tokens->push_back({start, i, TokenKind::K_CONSTANT});
         continue;
       } else {
         uint32_t start = i;
@@ -1169,36 +1170,36 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
             text[i + 2] == 'u' && text[i + 3] == 'e' &&
             ((i + 4 < len && !identifier_char(text[i + 4])) || i + 4 == len)) {
           i += 4;
-          tokens->push_back({start, i, TokenKind::True});
+          tokens->push_back({start, i, TokenKind::K_TRUE});
           continue;
         }
         if (i + 4 < len && text[i] == 'f' && text[i + 1] == 'a' &&
             text[i + 2] == 'l' && text[i + 3] == 's' && text[i + 4] == 'e' &&
             ((i + 5 < len && !identifier_char(text[i + 5])) || i + 5 == len)) {
           i += 5;
-          tokens->push_back({start, i, TokenKind::False});
+          tokens->push_back({start, i, TokenKind::K_FALSE});
           continue;
         }
         if (i + 3 < len && text[i] == 'd' && text[i + 1] == 'e' &&
             text[i + 2] == 'f') {
           i += 3;
-          tokens->push_back({start, i, TokenKind::Keyword});
+          tokens->push_back({start, i, TokenKind::K_KEYWORD});
           while (i < len && (text[i] == ' ' || text[i] == '\t'))
             i++;
           while (i < len) {
             if (identifier_start_char(text[i])) {
               uint32_t width = get_next_word(text, i, len);
               if (text[i] >= 'A' && text[i] <= 'Z')
-                tokens->push_back({i, i + width, TokenKind::Constant});
+                tokens->push_back({i, i + width, TokenKind::K_CONSTANT});
               else if (width == 4 && (text[i] >= 's' && text[i + 1] == 'e' &&
                                       text[i + 2] == 'l' && text[i + 3] == 'f'))
-                tokens->push_back({i, i + width, TokenKind::Keyword});
+                tokens->push_back({i, i + width, TokenKind::K_KEYWORD});
               i += width;
               if (i < len && text[i] == '.') {
                 i++;
                 continue;
               }
-              tokens->push_back({i - width, i, TokenKind::Function});
+              tokens->push_back({i - width, i, TokenKind::K_FUNCTION});
               break;
             } else {
               break;
@@ -1210,15 +1211,15 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
           i++;
         if (i < len && text[i] == ':') {
           i++;
-          tokens->push_back({start, i, TokenKind::Label});
+          tokens->push_back({start, i, TokenKind::K_LABEL});
           continue;
         } else if (i < len && (text[i] == '!' || text[i] == '?')) {
           i++;
-          tokens->push_back({start, i, TokenKind::Function});
+          tokens->push_back({start, i, TokenKind::K_FUNCTION});
         } else {
           uint32_t tmp = i;
           if (tmp < len && (text[tmp] == '(' || text[tmp] == '{')) {
-            tokens->push_back({start, i, TokenKind::Function});
+            tokens->push_back({start, i, TokenKind::K_FUNCTION});
             continue;
           } else if (tmp < len && (text[tmp] == ' ' || text[tmp] == '\t')) {
             tmp++;
@@ -1230,7 +1231,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
           if (tmp >= len)
             continue;
           if (!isascii(text[tmp])) {
-            tokens->push_back({start, i, TokenKind::Function});
+            tokens->push_back({start, i, TokenKind::K_FUNCTION});
             continue;
           } else if (text[tmp] == '-' || text[tmp] == '&' || text[tmp] == '%' ||
                      text[tmp] == ':') {
@@ -1244,7 +1245,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
                      text[tmp] == '^' || text[tmp] == '<' || text[tmp] == '>') {
             continue;
           }
-          tokens->push_back({start, i, TokenKind::Function});
+          tokens->push_back({start, i, TokenKind::K_FUNCTION});
         }
         continue;
       }
@@ -1252,7 +1253,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
       uint32_t op_len;
       if ((op_len =
                operator_trie.match(text, i, len, [](char) { return false; }))) {
-        tokens->push_back({i, i + op_len, TokenKind::Operator});
+        tokens->push_back({i, i + op_len, TokenKind::K_OPERATOR});
         i += op_len;
         state->full_state->expecting_expr = true;
         continue;

src/ui/hover.cc

@@ -24,6 +24,7 @@ void HoverBox::scroll(int32_t number) {
 
 void HoverBox::render_first(bool scroll) {
   if (!scroll) {
+    // TODO: call syntax highlighter here
   }
   uint32_t longest_line = 0;
   uint32_t current_width = 0;

src/utils/unicode.cc

@@ -1,3 +1,4 @@
+#include "utfcpp/source/utf8.h"
 #include "utils/utils.h"
 
 int display_width(const char *str, size_t len) {
@@ -98,46 +99,42 @@ uint32_t count_clusters(const char *line, size_t len, size_t from, size_t to) {
   return count;
 }
 
-uint32_t utf8_byte_offset_to_utf16(const char *s, uint32_t byte_pos) {
+size_t utf8_offset_to_utf16(const char *utf8, size_t utf8_len,
-  uint32_t utf16_units = 0;
+                            size_t byte_offset) {
-  uint32_t i = 0;
+  if (byte_offset > utf8_len)
-  while (i < byte_pos) {
+    return byte_offset;
-    unsigned char c = s[i];
+  const char *start = utf8;
-    if ((c & 0x80) == 0x00) {
+  const char *mid = utf8 + byte_offset;
-      i += 1;
+  if (!utf8::is_valid(start, mid))
-      utf16_units += 1;
+    assert(0 && "invalid utf8");
-    } else if ((c & 0xE0) == 0xC0) {
+  size_t utf16_offset = 0;
-      i += 2;
+  for (auto it = start; it < mid;) {
-      utf16_units += 1;
+    uint32_t codepoint = utf8::next(it, mid);
-    } else if ((c & 0xF0) == 0xE0) {
+    if (codepoint <= 0xFFFF)
-      i += 3;
+      utf16_offset += 1;
-      utf16_units += 1;
+    else
-    } else {
+      utf16_offset += 2;
-      i += 4;
-      utf16_units += 2;
-    }
   }
-  return utf16_units;
+  return utf16_offset;
 }
 
-uint32_t utf16_offset_to_utf8(const char *s, uint32_t utf16_pos) {
+size_t utf16_offset_to_utf8(const char *utf8, size_t utf8_len,
-  uint32_t utf16_units = 0;
+                            size_t utf16_offset) {
-  uint32_t i = 0;
+  const char *start = utf8;
-  while (utf16_units < utf16_pos) {
+  const char *end = utf8 + utf8_len;
-    unsigned char c = s[i];
+  const char *it = start;
-    if ((c & 0x80) == 0x00) {
+  size_t utf16_count = 0;
-      i += 1;
+  while (it < end) {
-      utf16_units += 1;
+    if (utf16_count >= utf16_offset)
-    } else if ((c & 0xE0) == 0xC0) {
+      break;
-      i += 2;
+    const char *prev = it;
-      utf16_units += 1;
+    uint32_t codepoint = utf8::next(it, end);
-    } else if ((c & 0xF0) == 0xE0) {
+    if (codepoint <= 0xFFFF)
-      i += 3;
+      utf16_count += 1;
-      utf16_units += 1;
+    else
-    } else {
+      utf16_count += 2;
-      i += 4;
+    if (utf16_count > utf16_offset)
-      utf16_units += 2;
+      return prev - start;
-    }
   }
-  return i;
+  return it - start;
 }