Allow ruby based configs and custom syntax parsers

.clangd

@@ -1,5 +1,5 @@
 CompileFlags:
-  Add: [-I/home/syed/main/crib/include, -I/home/syed/main/crib/libs, c++20]
+  Add: [-I/home/syed/main/crib/include, -I/home/syed/main/crib/libs, -I/usr/include/ruby-3.4.0, -I/usr/include/ruby-3.4.0/x86_64-linux, c++20]
   Remove: []
   Compiler: clang++
 

.gitmodules

@@ -2,3 +2,7 @@
 	path = libs/libgrapheme
 	url = git://git.suckless.org/libgrapheme
   ignore = dirty
+[submodule "libs/utfcpp"]
+	path = libs/utfcpp
+  url = https://github.com/nemtrif/utfcpp.git
+  ignore = dirty

Makefile

@@ -15,19 +15,21 @@ CXX := $(CCACHE) clang++
 CFLAGS_DEBUG :=\
 	-std=c++20 -Wall -Wextra \
 	-O0 -fno-inline -gsplit-dwarf\
-	-g -fsanitize=address -fno-omit-frame-pointer\
+	-g -fno-omit-frame-pointer\
 	-Wno-unused-command-line-argument \
-	-I./include -I./libs
+	-I./include -I./libs \
+	-I/usr/include/ruby-3.4.0 -I/usr/include/ruby-3.4.0/x86_64-linux
 CFLAGS_RELEASE :=\
 	-std=c++20 -O3 -march=native \
-	-fno-exceptions -fno-rtti -fstrict-aliasing \
+  -fno-rtti -fstrict-aliasing \
 	-ffast-math -flto=thin \
 	-fvisibility=hidden -fuse-ld=lld \
 	-fomit-frame-pointer -DNDEBUG -s \
 	-mllvm -vectorize-loops \
 	-fno-unwind-tables -fno-asynchronous-unwind-tables\
 	-Wno-unused-command-line-argument \
-	-I./include -I./libs
+	-I./include -I./libs \
+	-I/usr/include/ruby-3.4.0 -I/usr/include/ruby-3.4.0/x86_64-linux
 
 PCH_CFLAGS_DEBUG := $(CFLAGS_DEBUG) -x c++-header
 PCH_CFLAGS_RELEASE := $(CFLAGS_RELEASE) -x c++-header
@@ -39,7 +41,7 @@ UNICODE_OBJ_RELEASE := $(patsubst libs/unicode_width/%.c,$(OBJ_DIR)/release/unic
 
 LIBS := \
 	libs/libgrapheme/libgrapheme.a \
-	-lpcre2-8 -lmagic
+	-lpcre2-8 -lmagic -lruby
 
 SRC := $(wildcard $(SRC_DIR)/**/*.cc) $(wildcard $(SRC_DIR)/*.cc)
 OBJ_DEBUG := $(patsubst $(SRC_DIR)/%.cc,$(OBJ_DIR)/debug/%.o,$(SRC))

TODO.md

@@ -35,6 +35,12 @@ Copyright 2025 Syed Daanish
 * Try to make all functions better now that folds have been purged
 * Cleanup syntax and renderer files
 
+probably remove solargraph support and use ruby-lsp (or sorbet?) instead
+
+move lsp configs to json and also allow configs for windows-style vs unix-style line endings and utf-8 vs utf-16
+
+* the ruby should have an api to be able to draw windows and add mappings to them
+
 * finish bash then do all the directive-like ones like jsonc (first to help with theme files) / toml / yaml / ini / nginx
 * then markdown / html
 * then gitignore / gitattributes

config/libcrib.rb

@@ -0,0 +1,419 @@
+module C
+  K_DATA               = 0
+  K_SHEBANG            = 1
+  K_COMMENT            = 2
+  K_ERROR              = 3
+  K_STRING             = 4
+  K_ESCAPE             = 5
+  K_INTERPOLATION      = 6
+  K_REGEXP             = 7
+  K_NUMBER             = 8
+  K_TRUE               = 9
+  K_FALSE              = 10
+  K_CHAR               = 11
+  K_KEYWORD            = 12
+  K_KEYWORDOPERATOR    = 13
+  K_OPERATOR           = 14
+  K_FUNCTION           = 15
+  K_TYPE               = 16
+  K_CONSTANT           = 17
+  K_VARIABLEINSTANCE   = 18
+  K_VARIABLEGLOBAL     = 19
+  K_ANNOTATION         = 20
+  K_DIRECTIVE          = 21
+  K_LABEL              = 22
+  K_BRACE1             = 23
+  K_BRACE2             = 24
+  K_BRACE3             = 25
+  K_BRACE4             = 26
+  K_BRACE5             = 27
+  K_HEADING1           = 28
+  K_HEADING2           = 29
+  K_HEADING3           = 30
+  K_HEADING4           = 31
+  K_HEADING5           = 32
+  K_HEADING6           = 33
+  K_BLOCKQUOTE         = 34
+  K_LIST               = 35
+  K_LISTITEM           = 36
+  K_CODE               = 37
+  K_LANGUAGENAME       = 38
+  K_LINKLABEL          = 39
+  K_IMAGELABEL         = 40
+  K_LINK               = 41
+  K_TABLE              = 42
+  K_TABLEHEADER        = 43
+  K_ITALIC             = 44
+  K_BOLD               = 45
+  K_UNDERLINE          = 46
+  K_STRIKETHROUGH      = 47
+  K_HORIXONTALRULE     = 48
+  K_TAG                = 49
+  K_ATTRIBUTE          = 50
+  K_CHECKDONE          = 51
+  K_CHECKNOTDONE       = 52
+  @lsp_config = {
+    "clangd" => [
+      "--background-index",
+      "--clang-tidy",
+      "--completion-style=detailed",
+      "--header-insertion=never",
+      "--pch-storage=memory",
+      "--limit-results=50",
+      "--log=error"
+    ],
+    "ruby-lsp" => [],
+    "solargraph" => ["stdio"],
+    "bash-language-server" => ["start"],
+    "vscode-css-language-server" => ["--stdio"],
+    "vscode-json-language-server" => ["--stdio"],
+    "fish-lsp" => ["start"],
+    "gopls" => ["serve"],
+    "haskell-language-server" => ["lsp"],
+    "emmet-language-server" => ["--stdio"],
+    "typescript-language-server" => ["--stdio"],
+    "lua-language-server" => [],
+    "pyright-langserver" => ["--stdio"],
+    "rust-analyzer" => [],
+    "intelephense" => ["--stdio"],
+    "marksman" => ["server"],
+    "nginx-language-server" => [],
+    "taplo" => ["lsp", "stdio"],
+    "yaml-language-server" => ["--stdio"],
+    "sqls" => ["serve"],
+    "make-language-server" => [],
+    "sql-language-server" => ["up", "--method", "stdio"]
+  }
+  @languages = {
+    c: {
+      color: 0x555555,
+      symbol: " ",
+      extensions: ["c"],
+      filenames: [],
+      mimetypes: ["text/x-c"],
+      lsp: "clangd"
+    },
+    cpp: {
+      color: 0x00599C,
+      symbol: " ",
+      extensions: ["cpp", "cc", "cxx"],
+      filenames: [],
+      mimetypes: ["text/x-cpp"],
+      lsp: "clangd"
+    },
+    h: {
+      color: 0xA8B9CC,
+      symbol: " ",
+      extensions: ["h", "hpp"],
+      filenames: [],
+      mimetypes: ["text/x-c-header"],
+      lsp: "clangd"
+    },
+    css: {
+      color: 0x36a3d9,
+      symbol: " ",
+      extensions: ["css"],
+      filenames: [],
+      mimetypes: ["text/css"],
+      lsp: "vscode-css-language-server"
+    },
+    fish: {
+      color: 0x4d5a5e,
+      symbol: " ",
+      extensions: ["fish"],
+      filenames: [],
+      mimetypes: ["application/x-fish"],
+      lsp: "fish-lsp"
+    },
+    go: {
+      color: 0x00add8,
+      symbol: " ",
+      extensions: ["go"],
+      filenames: [],
+      mimetypes: ["text/x-go"],
+      lsp: "gopls"
+    },
+    gomod: {
+      color: 0x00add8,
+      symbol: " ",
+      extensions: ["mod"],
+      filenames: [],
+      mimetypes: ["text/x-go-mod"],
+      lsp: "gopls"
+    },
+    haskell: {
+      color: 0xa074c4,
+      symbol: " ",
+      extensions: ["hs", "lhs"],
+      filenames: [],
+      mimetypes: ["text/x-haskell"],
+      lsp: "haskell-language-server"
+    },
+    html: {
+      color: 0xef8a91,
+      symbol: " ",
+      extensions: ["html", "htm"],
+      filenames: [],
+      mimetypes: ["text/html"],
+      lsp: "emmet-language-server"
+    },
+    javascript: {
+      color: 0xf0df8a,
+      symbol: " ",
+      extensions: ["js"],
+      filenames: [],
+      mimetypes: ["application/javascript"],
+      lsp: "typescript-language-server"
+    },
+    typescript: {
+      color: 0x36a3d9,
+      symbol: " ",
+      extensions: ["ts"],
+      filenames: [],
+      mimetypes: ["application/typescript"],
+      lsp: "typescript-language-server"
+    },
+    json: {
+      color: 0xcbcb41,
+      symbol: "{}",
+      extensions: ["json"],
+      filenames: [],
+      mimetypes: ["application/json"],
+      lsp: "vscode-json-language-server"
+    },
+    jsonc: {
+      color: 0xcbcb41,
+      symbol: "{}",
+      extensions: ["jsonc"],
+      filenames: [],
+      mimetypes: ["application/json"],
+      lsp: "vscode-json-language-server"
+    },
+    erb: {
+      color: 0x6e1516,
+      symbol: " ",
+      extensions: ["erb"],
+      filenames: [],
+      mimetypes: ["text/x-erb"],
+      lsp: "emmet-language-server"
+    },
+    lua: {
+      color: 0x36a3d9,
+      symbol: "󰢱 ",
+      extensions: ["lua"],
+      filenames: [],
+      mimetypes: ["text/x-lua"],
+      lsp: "lua-language-server"
+    },
+    python: {
+      color: 0x95e6cb,
+      symbol: "󰌠 ",
+      extensions: ["py"],
+      filenames: [],
+      mimetypes: ["text/x-python"],
+      lsp: "pyright"
+    },
+    rust: {
+      color: 0xdea584,
+      symbol: "󱘗 ",
+      extensions: ["rs"],
+      filenames: [],
+      mimetypes: ["text/x-rust"],
+      lsp: "rust-analyzer"
+    },
+    php: {
+      color: 0xa074c4,
+      symbol: "󰌟 ",
+      extensions: ["php"],
+      filenames: [],
+      mimetypes: ["application/x-php"],
+      lsp: "intelephense"
+    },
+    markdown: {
+      color: 0x36a3d9,
+      symbol: " ",
+      extensions: ["md", "markdown"],
+      filenames: [],
+      mimetypes: ["text/markdown"],
+      lsp: "marksman"
+    },
+    nginx: {
+      color: 0x6d8086,
+      symbol: " ",
+      extensions: ["conf"],
+      filenames: [],
+      mimetypes: ["text/nginx"],
+      lsp: "nginx-language-server"
+    },
+    toml: {
+      color: 0x36a3d9,
+      symbol: " ",
+      extensions: ["toml"],
+      filenames: [],
+      mimetypes: ["application/toml"],
+      lsp: "taplo"
+    },
+    yaml: {
+      color: 0x6d8086,
+      symbol: " ",
+      extensions: ["yml", "yaml"],
+      filenames: [],
+      mimetypes: ["text/yaml"],
+      lsp: "yaml-language-server"
+    },
+    sql: {
+      color: 0xdad8d8,
+      symbol: " ",
+      extensions: ["sql"],
+      filenames: [],
+      mimetypes: ["text/x-sql"],
+      lsp: "sqls"
+    },
+    make: {
+      color: 0x4e5c61,
+      symbol: " ",
+      extensions: ["Makefile", "makefile"],
+      filenames: [],
+      mimetypes: ["text/x-makefile"],
+      lsp: "make-language-server"
+    },
+    gdscript: {
+      color: 0x6d8086,
+      symbol: " ",
+      extensions: ["gd"],
+      filenames: [],
+      mimetypes: ["text/x-gdscript"],
+      lsp: nil
+    },
+    man: {
+      color: 0xdad8d8,
+      symbol: " ",
+      extensions: ["man"],
+      filenames: [],
+      mimetypes: ["application/x-troff-man"],
+      lsp: nil
+    },
+    diff: {
+      color: 0xDD4C35,
+      symbol: " ",
+      extensions: ["diff", "patch"],
+      filenames: [],
+      mimetypes: ["text/x-diff"],
+      lsp: nil
+    },
+    gitattributes: {
+      color: 0xF05032,
+      symbol: " ",
+      extensions: ["gitattributes"],
+      filenames: [],
+      mimetypes: [],
+      lsp: nil
+    },
+    gitignore: {
+      color: 0xF05032,
+      symbol: " ",
+      extensions: ["gitignore"],
+      filenames: [],
+      mimetypes: [],
+      lsp: nil
+    },
+    regex: {
+      color: 0x9E9E9E,
+      symbol: ".*",
+      extensions: ["regex"],
+      filenames: [],
+      mimetypes: [],
+      lsp: nil
+    },
+    ini: {
+      color: 0x6d8086,
+      symbol: " ",
+      extensions: ["ini"],
+      filenames: [],
+      mimetypes: [],
+      lsp: nil
+    },
+    ruby: {
+      color: 0xff8087,
+      symbol: "󰴭 ",
+      extensions: ["rb"],
+      filenames: ["Gemfile"],
+      mimetypes: ["text/x-ruby"],
+      lsp: "solargraph"
+    },
+    bash: {
+      color: 0x4d5a5e,
+      symbol: " ",
+      extensions: ["sh"],
+      filenames: ["bash_profile", "bashrc"],
+      mimetypes: ["text/x-sh"],
+      lsp: "bash-language-server"
+    }
+  }
+  @key_handlers = {}
+  @key_binds = {}
+  @highlighters = {}
+  @log_queue = []
+  @line_endings = :unix
+
+  class << self
+    attr_accessor :theme, :lsp_config, :languages,
+                  :line_endings, :highlighters
+    attr_reader :b_startup, :b_shutdown, :b_extra_highlights
+
+    def startup(&block)
+      @b_startup = block
+    end
+
+    def shutdown(&block)
+      @b_shutdown = block
+    end
+
+    def queue_log(msg)
+      @log_queue << msg
+    end
+
+    def log_all
+      @log_queue.each do |msg|
+        puts msg
+      end
+      @log_queue = []
+    end
+
+    def extra_highlights(&block)
+      @b_extra_highlights = block
+    end
+
+    def bind(modes, keys = nil, action = nil, &block)
+      modes = [modes] unless modes.is_a?(Array)
+      if keys.nil?
+        app = self
+        dsl = Object.new
+        dsl.define_singleton_method(:set) do |k, act = nil, &blk|
+          app.bind(modes, k, act, &blk)
+        end
+        dsl.instance_exec(&block) if block_given?
+      elsif block_given?
+        keys = [keys] unless keys.is_a?(Array)
+        modes.each do |mode|
+          keys.each do |key|
+            @key_handlers[mode] ||= {}
+            @key_handlers[mode][key] ||= []
+            @key_handlers[mode][key] << block
+          end
+        end
+      elsif action.is_a?(String)
+        keys = [keys] unless keys.is_a?(Array)
+        modes.each do |mode|
+          keys.each do |key|
+            @key_binds[mode] ||= {}
+            @key_binds[mode][key] ||= []
+            @key_binds[mode][key] << action
+          end
+        end
+      end
+    end
+  end
+end
+
+at_exit { C.log_all }

config/main.rb

@@ -0,0 +1,127 @@
+require_relative "libcrib"
+
+# basic configuration
+
+C.startup do
+  puts "Starting crib..."
+end
+
+C.shutdown do
+  puts "Exiting crib..."
+end
+
+# this can be modified by the user during runtime through keybindings
+# But i need to know how to ever read this value only when needed.
+# maybe i can write a function that notifies if theme maybe changed then reload
+# It can also be scripted to load different theme formats into a hash usable by crib
+C.theme = {
+  :default => { fg: 0xEEEEEE },
+  :shebang => { fg: 0x7DCFFF },
+  :error => { fg: 0xEF5168 },
+  :comment => { fg: 0xAAAAAA, italic: true },
+  :string => { fg: 0xAAD94C },
+  :escape => { fg: 0x7DCFFF },
+  :interpolation => { fg: 0x7DCFFF },
+  :regexp => { fg: 0xD2A6FF },
+  :number => { fg: 0xE6C08A },
+  # rubocop:disable Lint/BooleanSymbol
+  :true => { fg: 0x7AE93C },
+  :false => { fg: 0xEF5168 },
+  # rubocop:enable Lint/BooleanSymbol
+  :char => { fg: 0xFFAF70 },
+  :keyword => { fg: 0xFF8F40 },
+  :keywordoperator => { fg: 0xF07178 },
+  :operator => { fg: 0xFFFFFF, italic: true },
+  :function => { fg: 0xFFAF70 },
+  :type => { fg: 0xF07178 },
+  :constant => { fg: 0x7DCFFF },
+  :variableinstance => { fg: 0x95E6CB },
+  :variableglobal => { fg: 0xF07178 },
+  :annotation => { fg: 0x7DCFFF },
+  :directive => { fg: 0xFF8F40 },
+  :label => { fg: 0xD2A6FF },
+  :brace1 => { fg: 0xD2A6FF },
+  :brace2 => { fg: 0xFFAFAF },
+  :brace3 => { fg: 0xFFFF00 },
+  :brace4 => { fg: 0x0FFF0F },
+  :brace5 => { fg: 0xFF0F0F }
+}
+
+# # TODO: to be done once a proper api for binding and window drawing is made
+# #       The binds will be connected to either `editor` or windows where editor can
+# #       only use a preset set of stuff to bind while teh windows are purely custom
+# # # this part uses dsl bindings to define the bind function
+# # # Hopefully extend to give more context/power to bindings
+# # # but try to keep simple for performance
+# # # for default keybindings
+# # C.bind [:normal, :select], :a => "insert_mode"
+# # # for custom keybindings
+# # C.bind :select, [:x, :c] do
+# #   puts "cut"
+# # end
+# # C.bind :jumper do
+# #   set [:x, :c] do
+# #     puts "jump to first bookmark"
+# #   end
+# # end
+# # # they can also be defined conditionally
+# # # This code is just an example and doesnt actually work
+# # if using_tmux?
+# #   bind :C-p do
+# #     system("tmux select-pane -U")
+# #   end
+# # end
+
+# This can, for example, be modified by user bindings during runtime
+# TODO: dynamic registration to actually be implemented once keybinds and extentions are implemented
+# A predefined list already exists and can be found in libcrib.rb
+# C.lsp_config["solargraph"] = ["stdio"]
+#
+# C.languages[:ruby] = {
+#   color: 0xff8087,
+#   symbol: "󰴭 ",
+#   extensions: ["rb"],
+#   filenames: ["Gemfile"],
+#   mimetypes: ["text/x-ruby"],
+#   lsp: "solargraph"
+# }
+
+C.line_endings = :unix # or :windows
+
+# C.extra_highlights do |_line, _idx|
+#   # the return can be an array of
+#   # [fg, bg. flags, start, end]
+#   # where fg and bg are integers (using 24 bit color)
+#   # and flags is a bitmask of bold/underline/italic etc
+#   # and start and end are integers strictly inside the line
+#   return []
+# end
+
+# The highlighter will be aplied to the language as long as the langauge is defined in C.languages
+C.highlighters[:ruby_n] = {
+  parser: ->(line, state) {
+    # the return value is a hash
+    # it contains the state and the highlights
+    # state can be of any type but will be consistent between calls
+    # initially nil is sent for uninitialized state the returned must be anything but nil
+    # the same state can be used for multiple lines
+    # the highlights can be an array of
+    # [K_type, start, end]
+    # K_type is a constant from the constants defined in libcrib.rb
+    # for ex: for strings it would be C::K_STRING or for numbers C::K_NUMBER etc.
+    # and start and end are integers strictly inside the line
+    return {
+      state: "",
+      tokens: [
+        # This will highlight the entire line as a string
+        # Any wrong format will not be handled and lead to crashes
+        { type: C::K_STRING, start: 0, end: line.length }
+      ]
+    }
+  },
+  matcher: ->(state1, state2) {
+    # returns true if the states are equal
+    # And so would not need recomputation for further lines
+    return state1 == state2
+  }
+}

include/config.h

@@ -1,291 +0,0 @@
-#ifndef CONFIG_H
-#define CONFIG_H
-
-#include "lsp/lsp.h"
-#include "pch.h"
-
-static const std::unordered_map<uint8_t, LSP> kLsps = {
-    {1,
-     {"clangd",
-      {
-          "clangd",
-          "--background-index",
-          "--clang-tidy",
-          "--completion-style=detailed",
-          "--header-insertion=never",
-          "--pch-storage=memory",
-          "--limit-results=50",
-          "--log=error",
-          nullptr,
-      }}},
-    {2,
-     {"ruby-lsp",
-      {
-          "ruby-lsp",
-          nullptr,
-      }}},
-    {3,
-     {"solargraph",
-      {
-          "solargraph",
-          "stdio",
-          nullptr,
-      }}},
-    {4,
-     {"bash-language-server",
-      {
-          "bash-language-server",
-          "start",
-          nullptr,
-      }}},
-    {5,
-     {"vscode-css-language-server",
-      {
-          "vscode-css-language-server",
-          "--stdio",
-          nullptr,
-      }}},
-    {6,
-     {"vscode-json-language-server",
-      {
-          "vscode-json-language-server",
-          "--stdio",
-          nullptr,
-      }}},
-    {7,
-     {"fish-lsp",
-      {
-          "fish-lsp",
-          "start",
-          nullptr,
-      }}},
-    {8,
-     {"gopls",
-      {
-          "gopls",
-          "serve",
-          nullptr,
-      }}},
-    {9,
-     {"haskell-language-server",
-      {
-          "haskell-language-server",
-          "lsp",
-          nullptr,
-      }}},
-    {10,
-     {"emmet-language-server",
-      {
-          "emmet-language-server",
-          "--stdio",
-          nullptr,
-      }}},
-    {11,
-     {"typescript-language-server",
-      {
-          "typescript-language-server",
-          "--stdio",
-          nullptr,
-      }}},
-#define LUA_LS 12
-    {12,
-     {"lua-language-server",
-      {
-          "lua-language-server",
-          nullptr,
-      }}},
-    {13,
-     {"pyright-langserver",
-      {
-          "pyright-langserver",
-          "--stdio",
-          nullptr,
-      }}},
-    {14,
-     {"rust-analyzer",
-      {
-          "rust-analyzer",
-          nullptr,
-      }}},
-    {15,
-     {"intelephense",
-      {
-          "intelephense",
-          "--stdio",
-          nullptr,
-      }}},
-    {16,
-     {"marksman",
-      {
-          "marksman",
-          "server",
-          nullptr,
-      }}},
-    {17,
-     {"nginx-language-server",
-      {
-          "nginx-language-server",
-          nullptr,
-      }}},
-    {18,
-     {"taplo",
-      {
-          "taplo",
-          "lsp",
-          "stdio",
-          nullptr,
-      }}},
-    {19,
-     {"yaml-language-server",
-      {
-          "yaml-language-server",
-          "--stdio",
-          nullptr,
-      }}},
-    {20,
-     {"sqls",
-      {
-          "sqls",
-          "serve",
-          nullptr,
-      }}},
-    {21,
-     {"make-language-server",
-      {
-          "make-language-server",
-          nullptr,
-      }}},
-    {22,
-     {"sql-language-server",
-      {
-          "sql-language-server",
-          "up",
-          "--method",
-          "stdio",
-          nullptr,
-      }}},
-};
-
-static const std::unordered_map<std::string, Language> kLanguages = {
-    {"bash", {"bash", 4, 0x4d5a5e, " "}},
-    {"c", {"c", 1, 0x555555, " "}},
-    {"cpp", {"cpp", 1, 0x00599C, " "}},
-    {"h", {"h", 1, 0xA8B9CC, " "}},
-    {"css", {"css", 5, 0x36a3d9, " "}},
-    {"fish", {"fish", 7, 0x4d5a5e, " "}},
-    {"go", {"go", 8, 0x00add8, " "}},
-    {"gomod", {"gomod", 8, 0x00add8, " "}},
-    {"haskell", {"haskell", 9, 0xa074c4, " "}},
-    {"html", {"html", 10, 0xef8a91, " "}},
-    {"javascript", {"javascript", 11, 0xf0df8a, " "}},
-    {"typescript", {"typescript", 11, 0x36a3d9, " "}},
-    {"json", {"json", 6, 0xcbcb41, "{}"}},
-    {"jsonc", {"jsonc", 6, 0xcbcb41, "{}"}},
-    {"erb", {"erb", 10, 0x6e1516, " "}},
-    {"ruby", {"ruby", 3, 0xff8087, "󰴭 "}},
-    {"lua", {"lua", 12, 0x36a3d9, "󰢱 "}},
-    {"python", {"python", 13, 0x95e6cb, "󰌠 "}},
-    {"rust", {"rust", 14, 0xdea584, "󱘗 "}},
-    {"php", {"php", 15, 0xa074c4, "󰌟 "}},
-    {"markdown", {"markdown", 16, 0x36a3d9, " "}},
-    {"markdown_inline", {"markdown_inline", 16, 0x36a3d9, " "}},
-    {"nginx", {"nginx", 17, 0x6d8086, " "}},
-    {"toml", {"toml", 18, 0x36a3d9, " "}},
-    {"yaml", {"yaml", 19, 0x6d8086, " "}},
-    {"sql", {"sql", 20, 0xdad8d8, " "}},
-    {"make", {"make", 21, 0x4e5c61, " "}},
-    {"gdscript", {"gdscript", 0, 0x6d8086, " "}},
-    {"man", {"man", 0, 0xdad8d8, " "}},
-    {"diff", {"diff", 0, 0xDD4C35, " "}},
-    {"gitattributes", {"gitattributes", 0, 0xF05032, " "}},
-    {"gitignore", {"gitignore", 0, 0xF05032, " "}},
-    {"query", {"query", 0, 0x7E57C2, " "}},
-    {"regex", {"regex", 0, 0x9E9E9E, ".*"}},
-    {"ini", {"ini", 0, 0x6d8086, " "}},
-};
-
-static const std::unordered_map<std::string, std::string> kExtToLang = {
-    {"sh", "bash"},
-    {"bash", "bash"},
-    {"c", "c"},
-    {"cpp", "cpp"},
-    {"cxx", "cpp"},
-    {"cc", "cpp"},
-    {"hpp", "h"},
-    {"hh", "h"},
-    {"hxx", "h"},
-    {"h", "h"},
-    {"css", "css"},
-    {"fish", "fish"},
-    {"go", "go"},
-    {"hs", "haskell"},
-    {"html", "html"},
-    {"htm", "html"},
-    {"js", "javascript"},
-    {"jsx", "javascript"},
-    {"ts", "typescript"},
-    {"tsx", "typescript"},
-    {"json", "json"},
-    {"jsonc", "jsonc"},
-    {"lua", "lua"},
-    {"make", "make"},
-    {"mk", "make"},
-    {"makefile", "make"},
-    {"man", "man"},
-    {"py", "python"},
-    {"rb", "ruby"},
-    {"rs", "rust"},
-    {"diff", "diff"},
-    {"patch", "diff"},
-    {"erb", "erb"},
-    {"gd", "gdscript"},
-    {"gitattributes", "gitattributes"},
-    {"gitignore", "gitignore"},
-    {"mod", "gomod"},
-    {"ini", "ini"},
-    {"gitmodules", "ini"},
-    {"md", "markdown"},
-    {"markdown", "markdown"},
-    {"conf", "nginx"},
-    {"php", "php"},
-    {"scm", "query"},
-    {"regex", "regex"},
-    {"sql", "sql"},
-    {"toml", "toml"},
-    {"yaml", "yaml"},
-    {"yml", "yaml"},
-    {"clangd", "yaml"},
-};
-
-static const std::unordered_map<std::string, std::string> kMimeToLang = {
-    {"text/x-c", "c"},
-    {"text/x-c++", "cpp"},
-    {"text/x-shellscript", "bash"},
-    {"application/json", "json"},
-    {"text/javascript", "javascript"},
-    {"text/html", "html"},
-    {"text/css", "css"},
-    {"text/x-python", "python"},
-    {"text/x-ruby", "ruby"},
-    {"text/x-go", "go"},
-    {"text/x-haskell", "haskell"},
-    {"text/x-rust", "rust"},
-    {"text/x-lua", "lua"},
-    {"text/x-diff", "diff"},
-    {"text/x-gdscript", "gdscript"},
-    {"text/x-gitattributes", "gitattributes"},
-    {"text/x-gitignore", "gitignore"},
-    {"text/x-gomod", "gomod"},
-    {"text/x-ini", "ini"},
-    {"text/markdown", "markdown"},
-    {"text/x-nginx-conf", "nginx"},
-    {"application/x-php", "php"},
-    {"text/x-tree-sitter-query", "query"},
-    {"text/x-regex", "regex"},
-    {"text/x-sql", "sql"},
-    {"text/x-toml", "toml"},
-    {"text/x-yaml", "yaml"},
-    {"text/x-man", "man"},
-};
-
-#endif

include/editor/editor.h

@@ -30,6 +30,7 @@ struct Editor {
   uint32_t cursor_preffered;
   Coord selection;
   bool selection_active;
+  bool unix_eol; // false for windows
   int selection_type;
   Coord position;
   Coord size;
@@ -55,7 +56,8 @@ struct Editor {
   bool is_css_color;
 };
 
-Editor *new_editor(const char *filename_arg, Coord position, Coord size);
+Editor *new_editor(const char *filename_arg, Coord position, Coord size,
+                   bool unix_eol);
 void save_file(Editor *editor);
 void free_editor(Editor *editor);
 void render_editor(Editor *editor);
@@ -76,6 +78,7 @@ void edit_replace(Editor *editor, Coord start, Coord end, const char *text,
                   uint32_t len);
 Coord editor_hit_test(Editor *editor, uint32_t x, uint32_t y);
 char *get_selection(Editor *editor, uint32_t *out_len, Coord *out_start);
+void selection_bounds(Editor *editor, Coord *out_start, Coord *out_end);
 void editor_worker(Editor *editor);
 void move_line_down(Editor *editor);
 void move_line_up(Editor *editor);
@@ -127,12 +130,12 @@ inline static void utf8_normalize_edit(Editor *editor, TextEdit *edit) {
     return;
   }
   if (edit->start.col < len)
-    edit->start.col = utf16_offset_to_utf8(line, edit->start.col);
+    edit->start.col = utf16_offset_to_utf8(line, len, edit->start.col);
   else
     edit->start.col = len;
   if (edit->end.row == edit->start.row) {
     if (edit->end.col < len)
-      edit->end.col = utf16_offset_to_utf8(line, edit->end.col);
+      edit->end.col = utf16_offset_to_utf8(line, len, edit->end.col);
     else
       edit->end.col = len;
     free(it->buffer);
@@ -151,7 +154,7 @@ inline static void utf8_normalize_edit(Editor *editor, TextEdit *edit) {
     return;
   }
   if (edit->end.col < len)
-    edit->end.col = utf16_offset_to_utf8(line, edit->end.col);
+    edit->end.col = utf16_offset_to_utf8(line, len, edit->end.col);
   else
     edit->end.col = len;
   free(it->buffer);

include/editor/helpers.h

@@ -0,0 +1,24 @@
+#ifndef EDITOR_HELPERS_H
+#define EDITOR_HELPERS_H
+
+#include "editor/editor.h"
+
+void insert_str(Editor *editor, char *c, uint32_t len);
+void insert_char(Editor *editor, char c);
+void normal_mode(Editor *editor);
+void backspace_edit(Editor *editor);
+void delete_prev_word(Editor *editor);
+void delete_next_word(Editor *editor);
+void clear_hooks_at_line(Editor *editor, uint32_t line);
+void cursor_prev_word(Editor *editor);
+void cursor_next_word(Editor *editor);
+void select_all(Editor *editor);
+void fetch_lsp_hover(Editor *editor);
+void handle_mouse(Editor *editor, KeyEvent event);
+void indent_current_line(Editor *editor);
+void dedent_current_line(Editor *editor);
+void paste(Editor *editor);
+void copy(Editor *editor);
+void cut(Editor *editor);
+
+#endif

include/lsp/lsp.h

@@ -5,11 +5,6 @@
 #include "pch.h"
 #include "utils/utils.h"
 
-struct LSP {
-  const char *command;
-  std::vector<const char *> args;
-};
-
 struct LSPPending {
   std::string method;
   Editor *editor = nullptr;
@@ -37,6 +32,7 @@ struct LSPInstance {
   bool allow_resolve = false;
   bool allow_formatting = false;
   bool allow_formatting_on_type = false;
+  bool is_utf8 = false;
   std::vector<char> format_chars;
   std::vector<char> trigger_chars;
   std::vector<char> end_chars;
@@ -49,10 +45,12 @@ struct LSPInstance {
 };
 
 extern std::shared_mutex active_lsps_mtx;
-extern std::unordered_map<uint8_t, std::shared_ptr<LSPInstance>> active_lsps;
+extern std::unordered_map<std::string, std::shared_ptr<LSPInstance>>
+    active_lsps;
 extern Queue<LSPOpenRequest> lsp_open_queue;
 
 static json client_capabilities = {
+    {"general", {{"positionEncodings", {"utf-16"}}}},
     {"textDocument",
      {{"publishDiagnostics", {{"relatedInformation", true}}},
       {"hover", {{"contentFormat", {"markdown", "plaintext"}}}},
@@ -79,9 +77,9 @@ void lsp_send(std::shared_ptr<LSPInstance> lsp, json message,
               LSPPending *pending);
 void lsp_worker();
 
-std::shared_ptr<LSPInstance> get_or_init_lsp(uint8_t lsp_id);
+std::shared_ptr<LSPInstance> get_or_init_lsp(std::string lsp_id);
-void clean_lsp(std::shared_ptr<LSPInstance> lsp, uint8_t lsp_id);
+void clean_lsp(std::shared_ptr<LSPInstance> lsp, std::string lsp_id);
-void close_lsp(uint8_t lsp_id);
+void close_lsp(std::string lsp_id);
 
 void open_editor(std::shared_ptr<LSPInstance> lsp,
                  std::pair<Language, Editor *> entry);

include/pch.h

@@ -4,6 +4,10 @@
 #define PCRE2_CODE_UNIT_WIDTH 8
 #define PCRE_WORKSPACE_SIZE 512
 
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wunused-parameter"
+#include <ruby.h>
+#pragma clang diagnostic pop
 #include <magic.h>
 #include <nlohmann/json.hpp>
 #include <pcre2.h>

include/scripting/decl.h

@@ -0,0 +1,21 @@
+#ifndef SCRIPTING_DECL_H
+#define SCRIPTING_DECL_H
+
+#include "syntax/decl.h"
+#include "utils/utils.h"
+
+extern std::unordered_map<std::string, std::pair<VALUE, VALUE>>
+    custom_highlighters;
+
+void ruby_start(const char *main_file);
+void ruby_shutdown();
+void ruby_log(std::string msg);
+void load_theme();
+void load_languages_info();
+bool read_line_endings();
+void load_custom_highlighters();
+VALUE parse_custom(std::vector<Token> *tokens, VALUE parser_block,
+                   const char *line, uint32_t len, VALUE state);
+bool custom_compare(VALUE match_block, VALUE state1, VALUE state2);
+
+#endif

include/syntax/decl.h

@@ -6,9 +6,9 @@
 #include "syntax/trie.h"
 
 struct Highlight {
-  uint32_t fg;
+  uint32_t fg{0xFFFFFF};
-  uint32_t bg;
+  uint32_t bg{0x000000};
-  uint8_t flags;
+  uint8_t flags{0};
 };
 
 enum struct TokenKind : uint8_t {
@@ -28,55 +28,6 @@ const std::unordered_map<std::string, TokenKind> kind_map = {
 
 extern std::array<Highlight, TOKEN_KIND_COUNT> highlights;
 
-inline void load_theme(std::string filename) {
-  uint32_t len = 0;
-  char *raw = load_file(filename.c_str(), &len);
-  if (!raw)
-    return;
-  std::string data(raw, len);
-  free(raw);
-  json j = json::parse(data);
-  Highlight default_hl = {0xFFFFFF, 0, 0};
-  if (j.contains("Default")) {
-    auto def = j["Default"];
-    if (def.contains("fg") && def["fg"].is_string())
-      default_hl.fg = HEX(def["fg"]);
-    if (def.contains("bg") && def["bg"].is_string())
-      default_hl.bg = HEX(def["bg"]);
-    if (def.contains("italic") && def["italic"].get<bool>())
-      default_hl.flags |= CF_ITALIC;
-    if (def.contains("bold") && def["bold"].get<bool>())
-      default_hl.flags |= CF_BOLD;
-    if (def.contains("underline") && def["underline"].get<bool>())
-      default_hl.flags |= CF_UNDERLINE;
-    if (def.contains("strikethrough") && def["strikethrough"].get<bool>())
-      default_hl.flags |= CF_STRIKETHROUGH;
-  }
-  for (auto &hl : highlights)
-    hl = default_hl;
-  for (auto &[key, value] : j.items()) {
-    if (key == "Default")
-      continue;
-    auto it = kind_map.find(key);
-    if (it == kind_map.end())
-      continue;
-    Highlight hl = {0xFFFFFF, 0, 0};
-    if (value.contains("fg") && value["fg"].is_string())
-      hl.fg = HEX(value["fg"]);
-    if (value.contains("bg") && value["bg"].is_string())
-      hl.bg = HEX(value["bg"]);
-    if (value.contains("italic") && value["italic"].get<bool>())
-      hl.flags |= CF_ITALIC;
-    if (value.contains("bold") && value["bold"].get<bool>())
-      hl.flags |= CF_BOLD;
-    if (value.contains("underline") && value["underline"].get<bool>())
-      hl.flags |= CF_UNDERLINE;
-    if (value.contains("strikethrough") && value["strikethrough"].get<bool>())
-      hl.flags |= CF_STRIKETHROUGH;
-    highlights[static_cast<uint8_t>(it->second)] = hl;
-  }
-}
-
 struct Token {
   uint32_t start;
   uint32_t end;
@@ -89,4 +40,10 @@ struct LineData {
   std::shared_ptr<void> out_state{nullptr};
 };
 
+struct CustomState {
+  VALUE state;
+  CustomState(VALUE s) : state(s) { rb_gc_register_address(&state); }
+  ~CustomState() { rb_gc_unregister_address(&state); }
+};
+
 #endif

include/syntax/extras.h

@@ -453,7 +453,7 @@ private:
     uint8_t g = (color >> 8) & 0xFF;
     uint8_t b = color & 0xFF;
     double luminance = 0.299 * r + 0.587 * g + 0.114 * b;
-    return (luminance > 128) ? 0x000000 : 0xFFFFFF;
+    return (luminance > 128) ? 0x010101 : 0xFFFFFF;
   }
 
   uint32_t hslToRgb(double h, double s, double l) {

include/syntax/parser.h

@@ -1,6 +1,7 @@
 #ifndef SYNTAX_PARSER_H
 #define SYNTAX_PARSER_H
 
+#include "scripting/decl.h"
 #include "syntax/decl.h"
 #include "syntax/line_tree.h"
 
@@ -12,11 +13,15 @@ struct Parser {
                                       const char *text, uint32_t len);
   bool (*state_match_func)(std::shared_ptr<void> state_1,
                            std::shared_ptr<void> state_2);
+  VALUE parser_block = Qnil;
+  VALUE match_block = Qnil;
+  bool is_custom{false};
   std::atomic<uint32_t> scroll_max{UINT32_MAX - 2048};
+  std::atomic<bool> scroll_dirty{false};
   std::mutex mutex;
   std::mutex data_mutex;
   LineTree line_tree;
-  std::set<uint32_t> dirty_lines;
+  UniqueQueue<uint32_t> dirty_lines;
 
   Parser(Editor *editor, std::string n_lang, uint32_t n_scroll_max);
   void edit(uint32_t start_line, uint32_t old_end_line, uint32_t inserted_rows);

include/syntax/tokens.def

@@ -1,53 +1,54 @@
-ADD(Data)
+// When updating this file with new types do not forget to update libcrib.rb to parallel changes
-ADD(Shebang)
+ADD(K_DATA)
-ADD(Comment)
+ADD(K_SHEBANG)
-ADD(Error)
+ADD(K_COMMENT)
-ADD(String)
+ADD(K_ERROR)
-ADD(Escape)
+ADD(K_STRING)
-ADD(Interpolation)
+ADD(K_ESCAPE)
-ADD(Regexp)
+ADD(K_INTERPOLATION)
-ADD(Number)
+ADD(K_REGEXP)
-ADD(True)
+ADD(K_NUMBER)
-ADD(False)
+ADD(K_TRUE)
-ADD(Char)
+ADD(K_FALSE)
-ADD(Keyword)
+ADD(K_CHAR)
-ADD(KeywordOperator)
+ADD(K_KEYWORD)
-ADD(Operator)
+ADD(K_KEYWORDOPERATOR)
-ADD(Function)
+ADD(K_OPERATOR)
-ADD(Type)
+ADD(K_FUNCTION)
-ADD(Constant)
+ADD(K_TYPE)
-ADD(VariableInstance)
+ADD(K_CONSTANT)
-ADD(VariableGlobal)
+ADD(K_VARIABLEINSTANCE)
-ADD(Annotation)
+ADD(K_VARIABLEGLOBAL)
-ADD(Directive)
+ADD(K_ANNOTATION)
-ADD(Label)
+ADD(K_DIRECTIVE)
-ADD(Brace1)
+ADD(K_LABEL)
-ADD(Brace2)
+ADD(K_BRACE1)
-ADD(Brace3)
+ADD(K_BRACE2)
-ADD(Brace4)
+ADD(K_BRACE3)
-ADD(Brace5)
+ADD(K_BRACE4)
-ADD(Heading1)
+ADD(K_BRACE5)
-ADD(Heading2)
+ADD(K_HEADING1)
-ADD(Heading3)
+ADD(K_HEADING2)
-ADD(Heading4)
+ADD(K_HEADING3)
-ADD(Heading5)
+ADD(K_HEADING4)
-ADD(Heading6)
+ADD(K_HEADING5)
-ADD(Blockquote)
+ADD(K_HEADING6)
-ADD(List)
+ADD(K_BLOCKQUOTE)
-ADD(ListItem)
+ADD(K_LIST)
-ADD(Code)
+ADD(K_LISTITEM)
-ADD(LanguageName)
+ADD(K_CODE)
-ADD(LinkLabel)
+ADD(K_LANGUAGENAME)
-ADD(ImageLabel)
+ADD(K_LINKLABEL)
-ADD(Link)
+ADD(K_IMAGELABEL)
-ADD(Table)
+ADD(K_LINK)
-ADD(TableHeader)
+ADD(K_TABLE)
-ADD(Italic)
+ADD(K_TABLEHEADER)
-ADD(Bold)
+ADD(K_ITALIC)
-ADD(Underline)
+ADD(K_BOLD)
-ADD(Strikethrough)
+ADD(K_UNDERLINE)
-ADD(HorixontalRule)
+ADD(K_STRIKETHROUGH)
-ADD(Tag)
+ADD(K_HORIXONTALRULE)
-ADD(Attribute)
+ADD(K_TAG)
-ADD(CheckDone)
+ADD(K_ATTRIBUTE)
-ADD(CheckNotDone)
+ADD(K_CHECKDONE)
+ADD(K_CHECKNOTDONE)

include/utils/utils.h

@@ -4,9 +4,6 @@
 #include "pch.h"
 
 template <typename T> struct Queue {
-  std::queue<T> q;
-  std::mutex m;
-
   void push(T val) {
     std::lock_guard<std::mutex> lock(m);
     q.push(val);
@@ -32,6 +29,49 @@ template <typename T> struct Queue {
     std::lock_guard<std::mutex> lock(m);
     return q.empty();
   }
+
+private:
+  std::queue<T> q;
+  std::mutex m;
+};
+
+template <typename T> struct UniqueQueue {
+  bool push(const T &value) {
+    std::lock_guard<std::mutex> lock(m);
+    if (set.contains(value))
+      return false;
+
+    dq.push_back(value);
+    set.insert(value);
+    return true;
+  }
+  bool pop(T &out) {
+    std::lock_guard<std::mutex> lock(m);
+    if (dq.empty())
+      return false;
+    out = dq.front();
+    dq.pop_front();
+    set.erase(out);
+    return true;
+  }
+  bool empty() const {
+    std::lock_guard<std::mutex> lock(m);
+    return dq.empty();
+  }
+  void clear() {
+    std::lock_guard<std::mutex> lock(m);
+    dq.clear();
+    set.clear();
+  }
+  size_t size() const {
+    std::lock_guard<std::mutex> lock(m);
+    return dq.size();
+  }
+
+private:
+  std::deque<T> dq;
+  std::set<T> set;
+  mutable std::mutex m;
 };
 
 struct Coord {
@@ -60,11 +100,21 @@ struct Match {
 
 struct Language {
   std::string name;
-  uint8_t lsp_id;
+  std::string lsp_name;
   uint32_t color;
-  const char *symbol;
+  std::string symbol;
 };
 
+struct LSP {
+  std::string command;
+  std::vector<std::string> args;
+};
+
+extern std::unordered_map<std::string, Language> languages;
+extern std::unordered_map<std::string, std::string> language_extensions;
+extern std::unordered_map<std::string, std::string> language_mimetypes;
+extern std::unordered_map<std::string, LSP> lsps;
+
 #define MAX(a, b) ((a) > (b) ? (a) : (b))
 #define MIN(a, b) ((a) < (b) ? (a) : (b))
 #define ABS(x) ((x) < 0 ? -(x) : (x))
@@ -92,8 +142,10 @@ uint32_t get_visual_col_from_bytes(const char *line, uint32_t len,
                                    uint32_t byte_limit);
 uint32_t get_bytes_from_visual_col(const char *line, uint32_t len,
                                    uint32_t target_visual_col);
-uint32_t utf8_byte_offset_to_utf16(const char *s, uint32_t byte_pos);
+size_t utf8_offset_to_utf16(const char *utf8, size_t utf8_len,
-uint32_t utf16_offset_to_utf8(const char *s, uint32_t utf16_pos);
+                            size_t byte_offset);
+size_t utf16_offset_to_utf8(const char *utf8, size_t utf8_len,
+                            size_t utf16_offset);
 uint8_t utf8_codepoint_width(unsigned char c);
 
 void log(const char *fmt, ...);

samples/bash.sh

@@ -22,6 +22,7 @@ colorize() {
 }
 
 # Example of error handling
+
 handle_error() {
   log ERROR "An error occurred on line $1"
 }

samples/css.css

@@ -21,6 +21,7 @@ a:focus-visible {
 
 input::placeholder {
   color: #999;
+  background-color: #523;
 }
 
 /* CSS variables */

samples/lua.lua

@@ -117,4 +117,3 @@ local tpl = `Value: ${counter}`
 
 -- Regex-like string (for testing injection highlighting)
 local re = "/^%a+$/"
-

samples/python.py

@@ -12,7 +12,6 @@ NotImplemented
 Ellipsis
 __name__  # builtin constant
 
-
 # ==============================
 # Imports
 # ==============================

samples/ruby.rb

@@ -4,27 +4,16 @@
 # Purpose: Test syntax highlighting + width calculation in your editor
 # ---------------------------------------------------------------
 
-# Basic output
-def greet
-  puts "Hello, 世界! 👋🌏"
-end
-
-# Emoji-heavy strings
-emojis = "👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏"
 
 # Mixed-width CJKssssssssssssssss LoadErssssssssssssssssssssssss
 cjk_samples = [
   "漢字テスト",
   "測試中文字串",
   "한국어 테스트",
-  "ひらがなカタカナ😀混合",
+  "ひらがなカタカナ混合",
-  "大量の文字列🚀🚀🚀"
 ]
 
-
+# a hex color: #FFFFFF shouldn't hl here: hsl(147rad, 50%, 47%)  as it is not css-style file
-
-
-# a hex color: #FFFFFF hsl(147rad, 50%, 47%)
 
 0x603010 # another hex color
 
@@ -38,7 +27,7 @@ s wow
 
 UNICORE = %r{
     [s]
-    {#{}}
+    {#{ss}}
     \C-s\u{10}
   }
   
@@ -112,8 +101,12 @@ mixed = [
 
 two_docs = <<DOC1 , <<DOC2
 stuff for doc2
+
+
+rdvajehvbaejbfh
+
 DOC1
-stuff for doc 2 with \#{not interpolation} and more
+stuff for doc 2 with #{not interpolation} and more
 DOC2
 
 p = 0 <<22 # not a heredoc
@@ -155,9 +148,6 @@ This is a wrapped line test, This is a wrapped line test, This is a wrapped line
 
 # Constants
 
-__END_
-
-
 PI = 3.14159
 MAX_ITER = 5
 # Module

scripts/exit.sh

@@ -1 +0,0 @@
-# kitty @ --to="$KITTY_LISTEN_ON" set-spacing padding=8 margin=0 2>/dev/null || true

scripts/init.sh

@@ -1 +0,0 @@
-# kitty @ --to="$KITTY_LISTEN_ON" set-spacing padding=0 margin=0 2>/dev/null || true

src/editor/completions.cc

@@ -12,15 +12,14 @@ inline static std::string completion_prefix(Editor *editor) {
   if (hook.row != cur.row || cur.col < hook.col)
     return "";
   LineIterator *it = begin_l_iter(editor->root, hook.row);
-  char *line = next_line(it, nullptr);
+  uint32_t line_len;
+  char *line = next_line(it, &line_len);
   if (!line) {
     free(it->buffer);
     free(it);
     return "";
   }
-  uint32_t start = utf16_offset_to_utf8(line, hook.col);
+  std::string prefix(line + hook.col, cur.col - hook.col);
-  uint32_t end = editor->cursor.col;
-  std::string prefix(line + start, end - start);
   free(it->buffer);
   free(it);
   return prefix;
@@ -212,13 +211,14 @@ void completion_request(Editor *editor) {
   };
   std::shared_lock lock(editor->knot_mtx);
   LineIterator *it = begin_l_iter(editor->root, hook.row);
-  char *line = next_line(it, nullptr);
+  uint32_t length;
+  char *line = next_line(it, &length);
   if (!line) {
     free(it->buffer);
     free(it);
     return;
   }
-  uint32_t col = utf8_byte_offset_to_utf16(line, editor->cursor.col);
+  uint32_t col = utf8_offset_to_utf16(line, length, editor->cursor.col);
   free(it->buffer);
   free(it);
   lock.unlock();

src/editor/edit.cc

@@ -16,17 +16,18 @@ void edit_erase(Editor *editor, Coord pos, int64_t len) {
     bool do_lsp = (editor->lsp != nullptr);
     if (do_lsp) {
       LineIterator *it = begin_l_iter(editor->root, point.row);
-      char *line = next_line(it, nullptr);
+      uint32_t len;
+      char *line = next_line(it, &len);
       int utf16_start = 0;
       if (line)
-        utf16_start = utf8_byte_offset_to_utf16(line, point.col);
+        utf16_start = utf8_offset_to_utf16(line, len, point.col);
       free(it->buffer);
       free(it);
       it = begin_l_iter(editor->root, pos.row);
-      line = next_line(it, nullptr);
+      line = next_line(it, &len);
       int utf16_end = 0;
       if (line)
-        utf16_end = utf8_byte_offset_to_utf16(line, pos.col);
+        utf16_end = utf8_offset_to_utf16(line, len, pos.col);
       free(it->buffer);
       free(it);
       lsp_range = {{"start", {{"line", point.row}, {"character", utf16_start}}},
@@ -88,17 +89,18 @@ void edit_erase(Editor *editor, Coord pos, int64_t len) {
     bool do_lsp = (editor->lsp != nullptr);
     if (do_lsp) {
       LineIterator *it = begin_l_iter(editor->root, pos.row);
-      char *line = next_line(it, nullptr);
+      uint32_t line_len;
+      char *line = next_line(it, &line_len);
       int utf16_start = 0;
       if (line)
-        utf16_start = utf8_byte_offset_to_utf16(line, pos.col);
+        utf16_start = utf8_offset_to_utf16(line, line_len, pos.col);
       free(it->buffer);
       free(it);
       it = begin_l_iter(editor->root, point.row);
-      line = next_line(it, nullptr);
+      line = next_line(it, &line_len);
       int utf16_end = 0;
       if (line)
-        utf16_end = utf8_byte_offset_to_utf16(line, point.col);
+        utf16_end = utf8_offset_to_utf16(line, line_len, point.col);
       free(it->buffer);
       free(it);
       lsp_range = {{"start", {{"line", pos.row}, {"character", utf16_start}}},
@@ -164,6 +166,14 @@ void edit_insert(Editor *editor, Coord pos, char *data, uint32_t len) {
     uint32_t new_row = byte_to_line(editor->root, cursor_new, &new_col);
     editor->cursor = {new_row, new_col};
   }
+  LineIterator *it = begin_l_iter(editor->root, pos.row);
+  uint32_t line_len;
+  char *line = next_line(it, &line_len);
+  int utf16_col = 0;
+  if (line)
+    utf16_col = utf8_offset_to_utf16(line, line_len, pos.col);
+  free(it->buffer);
+  free(it);
   lock_1.unlock();
   std::unique_lock lock_2(editor->knot_mtx);
   editor->root = insert(editor->root, byte_pos, data, len);
@@ -177,15 +187,6 @@ void edit_insert(Editor *editor, Coord pos, char *data, uint32_t len) {
     editor->parser->edit(pos.row, pos.row, rows);
   if (editor->lsp) {
     if (editor->lsp->incremental_sync) {
-      lock_1.lock();
-      LineIterator *it = begin_l_iter(editor->root, pos.row);
-      char *line = next_line(it, nullptr);
-      int utf16_col = 0;
-      if (line)
-        utf16_col = utf8_byte_offset_to_utf16(line, pos.col);
-      free(it->buffer);
-      free(it);
-      lock_1.unlock();
       json message = {
           {"jsonrpc", "2.0"},
           {"method", "textDocument/didChange"},
@@ -222,17 +223,18 @@ void edit_replace(Editor *editor, Coord start, Coord end, const char *text,
       line_to_byte(editor->root, start.row, nullptr) + start.col;
   uint32_t end_byte = line_to_byte(editor->root, end.row, nullptr) + end.col;
   LineIterator *it = begin_l_iter(editor->root, start.row);
-  char *line = next_line(it, nullptr);
+  uint32_t line_len;
+  char *line = next_line(it, &line_len);
   int utf16_start = 0;
   if (line)
-    utf16_start = utf8_byte_offset_to_utf16(line, start.col);
+    utf16_start = utf8_offset_to_utf16(line, line_len, start.col);
   free(it->buffer);
   free(it);
   it = begin_l_iter(editor->root, end.row);
-  line = next_line(it, nullptr);
+  line = next_line(it, &line_len);
   int utf16_end = 0;
   if (line)
-    utf16_end = utf8_byte_offset_to_utf16(line, end.col);
+    utf16_end = utf8_offset_to_utf16(line, line_len, end.col);
   free(it->buffer);
   free(it);
   if (start_byte != end_byte)
@@ -243,10 +245,8 @@ void edit_replace(Editor *editor, Coord start, Coord end, const char *text,
   for (uint32_t i = 0; i < len; i++)
     if (text[i] == '\n')
       rows++;
-  if (editor->parser) {
+  if (editor->parser)
-    editor->parser->edit(start.row, end.row - 1, 0);
+    editor->parser->edit(start.row, end.row - 1, rows);
-    editor->parser->edit(start.row, start.row, rows);
-  }
   if (editor->lsp) {
     if (editor->lsp->incremental_sync) {
       json message = {

src/editor/editor.cc

@@ -1,10 +1,11 @@
 #include "editor/editor.h"
 #include "editor/decl.h"
 #include "lsp/lsp.h"
+#include "syntax/langs.h"
 #include "utils/utils.h"
-#include <shared_mutex>
 
-Editor *new_editor(const char *filename_arg, Coord position, Coord size) {
+Editor *new_editor(const char *filename_arg, Coord position, Coord size,
+                   bool unix_eol) {
   Editor *editor = new Editor();
   if (!editor)
     return nullptr;
@@ -15,6 +16,7 @@ Editor *new_editor(const char *filename_arg, Coord position, Coord size) {
     free_editor(editor);
     return nullptr;
   }
+  editor->unix_eol = unix_eol;
   editor->filename = filename;
   editor->uri = path_to_file_uri(filename);
   editor->position = position;
@@ -29,7 +31,7 @@ Editor *new_editor(const char *filename_arg, Coord position, Coord size) {
   editor->root = load(str, len, optimal_chunk_size(len));
   free(str);
   editor->lang = language_for_file(filename.c_str());
-  if (editor->lang.name != "unknown")
+  if (parsers.find(editor->lang.name) != parsers.end())
     editor->parser = new Parser(editor, editor->lang.name, size.row + 5);
   if (editor->lang.name == "css" || editor->lang.name == "html" ||
       editor->lang.name == "javascript" || editor->lang.name == "markdown" ||
@@ -61,7 +63,15 @@ void save_file(Editor *editor) {
     return;
   lock.unlock();
   std::ofstream out(editor->filename);
+  if (!editor->unix_eol) {
+    for (uint32_t i = 0; i < char_count; ++i) {
+      if (str[i] == '\n')
+        out.put('\r');
+      out.put(str[i]);
+    }
+  } else {
     out.write(str, char_count);
+  }
   out.close();
   free(str);
   if (editor->lsp) {
@@ -109,7 +119,15 @@ void save_file(Editor *editor) {
             return;
           lock.unlock();
           std::ofstream out(editor->filename);
+          if (!editor->unix_eol) {
+            for (uint32_t i = 0; i < char_count; ++i) {
+              if (str[i] == '\n')
+                out.put('\r');
+              out.put(str[i]);
+            }
+          } else {
             out.write(str, char_count);
+          }
           out.close();
           free(str);
           lsp_send(editor->lsp, save_msg, nullptr);

src/editor/events.cc

@@ -1,147 +1,14 @@
 #include "editor/editor.h"
-#include "lsp/lsp.h"
+#include "editor/helpers.h"
+#include "io/sysio.h"
 #include "main.h"
 #include "utils/utils.h"
 
 void handle_editor_event(Editor *editor, KeyEvent event) {
-  static std::chrono::steady_clock::time_point last_click_time =
-      std::chrono::steady_clock::now();
-  static uint32_t click_count = 0;
-  static Coord last_click_pos = {UINT32_MAX, UINT32_MAX};
-  Coord start = editor->cursor;
   uint8_t old_mode = mode;
   if (editor->hover_active)
     editor->hover_active = false;
-  if (event.key_type == KEY_MOUSE) {
+  handle_mouse(editor, event);
-    auto now = std::chrono::steady_clock::now();
-    auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(
-                        now - last_click_time)
-                        .count();
-    switch (event.mouse_state) {
-    case SCROLL:
-      switch (event.mouse_direction) {
-      case SCROLL_UP:
-        scroll_up(editor, 4);
-        ensure_cursor(editor);
-        break;
-      case SCROLL_DOWN:
-        scroll_down(editor, 4);
-        ensure_cursor(editor);
-        break;
-      case SCROLL_LEFT:
-        cursor_left(editor, 10);
-        break;
-      case SCROLL_RIGHT:
-        cursor_right(editor, 10);
-        break;
-      }
-      break;
-    case PRESS:
-      if (event.mouse_button == LEFT_BTN) {
-        Coord cur_pos = {event.mouse_x, event.mouse_y};
-        if (duration < 250 && last_click_pos == cur_pos)
-          click_count++;
-        else
-          click_count = 1;
-        last_click_time = now;
-        last_click_pos = cur_pos;
-        Coord p = editor_hit_test(editor, event.mouse_x, event.mouse_y);
-        if (p.row == UINT32_MAX && p.col == UINT32_MAX)
-          return;
-        editor->cursor_preffered = UINT32_MAX;
-        if (click_count == 1) {
-          editor->cursor = p;
-          editor->selection = p;
-          if (mode == SELECT) {
-            mode = NORMAL;
-            editor->selection_active = false;
-          }
-        } else if (click_count == 2) {
-          uint32_t prev_col, next_col;
-          word_boundaries(editor, editor->cursor, &prev_col, &next_col, nullptr,
-                          nullptr);
-          if (editor->cursor < editor->selection)
-            editor->cursor = {editor->cursor.row, prev_col};
-          else
-            editor->cursor = {editor->cursor.row, next_col};
-          editor->cursor_preffered = UINT32_MAX;
-          editor->selection_type = WORD;
-          mode = SELECT;
-          editor->selection_active = true;
-        } else if (click_count >= 3) {
-          if (editor->cursor < editor->selection) {
-            editor->cursor = {p.row, 0};
-          } else {
-            uint32_t line_len;
-            LineIterator *it = begin_l_iter(editor->root, p.row);
-            char *line = next_line(it, &line_len);
-            if (!line)
-              return;
-            if (line_len > 0 && line[line_len - 1] == '\n')
-              line_len--;
-            free(it->buffer);
-            free(it);
-            editor->cursor = {p.row, line_len};
-          }
-          editor->cursor_preffered = UINT32_MAX;
-          editor->selection_type = LINE;
-          mode = SELECT;
-          editor->selection_active = true;
-          click_count = 3;
-        }
-      }
-      break;
-    case DRAG:
-      if (event.mouse_button == LEFT_BTN) {
-        Coord p = editor_hit_test(editor, event.mouse_x, event.mouse_y);
-        if (p.row == UINT32_MAX && p.col == UINT32_MAX)
-          return;
-        editor->cursor_preffered = UINT32_MAX;
-        mode = SELECT;
-        if (!editor->selection_active) {
-          editor->selection_active = true;
-          editor->selection_type = CHAR;
-        }
-        uint32_t prev_col, next_col, line_len;
-        switch (editor->selection_type) {
-        case CHAR:
-          editor->cursor = p;
-          break;
-        case WORD:
-          word_boundaries(editor, p, &prev_col, &next_col, nullptr, nullptr);
-          if (editor->cursor < editor->selection)
-            editor->cursor = {p.row, prev_col};
-          else
-            editor->cursor = {p.row, next_col};
-          break;
-        case LINE:
-          if (editor->cursor < editor->selection) {
-            editor->cursor = {p.row, 0};
-          } else {
-            LineIterator *it = begin_l_iter(editor->root, p.row);
-            char *line = next_line(it, &line_len);
-            if (!line)
-              return;
-            if (line_len > 0 && line[line_len - 1] == '\n')
-              line_len--;
-            free(it->buffer);
-            free(it);
-            editor->cursor = {p.row, line_len};
-          }
-          break;
-        }
-      }
-      break;
-    case RELEASE:
-      if (event.mouse_button == LEFT_BTN)
-        if (editor->cursor.row == editor->selection.row &&
-            editor->cursor.col == editor->selection.col) {
-          mode = NORMAL;
-          editor->selection_active = false;
-        }
-      break;
-    }
-  }
   if (event.key_type == KEY_SPECIAL) {
     switch (event.special_modifier) {
     case 0:
@@ -161,9 +28,6 @@ void handle_editor_event(Editor *editor, KeyEvent event) {
       }
       break;
     case CNTRL:
-      uint32_t prev_col, next_col;
-      word_boundaries(editor, editor->cursor, &prev_col, &next_col, nullptr,
-                      nullptr);
       switch (event.special_key) {
       case KEY_DOWN:
         cursor_down(editor, 5);
@@ -172,18 +36,9 @@ void handle_editor_event(Editor *editor, KeyEvent event) {
         cursor_up(editor, 5);
         break;
       case KEY_LEFT:
-        editor->cursor_preffered = UINT32_MAX;
+        cursor_prev_word(editor);
-        if (prev_col == editor->cursor.col)
-          cursor_left(editor, 1);
-        else
-          editor->cursor = {editor->cursor.row, prev_col};
-        break;
       case KEY_RIGHT:
-        editor->cursor_preffered = UINT32_MAX;
+        cursor_next_word(editor);
-        if (next_col == editor->cursor.col)
-          cursor_right(editor, 1);
-        else
-          editor->cursor = {editor->cursor.row, next_col};
         break;
       }
       break;
@@ -210,27 +65,7 @@ void handle_editor_event(Editor *editor, KeyEvent event) {
     if (event.key_type == KEY_CHAR && event.len == 1) {
       switch (event.c[0]) {
       case 'u':
-        if (editor->root->line_count > 0) {
+        select_all(editor);
-          editor->cursor.row = editor->root->line_count - 1;
-          LineIterator *it = begin_l_iter(editor->root, editor->cursor.row);
-          if (!it)
-            break;
-          uint32_t line_len;
-          char *line = next_line(it, &line_len);
-          if (!line)
-            break;
-          if (line_len > 0 && line[line_len - 1] == '\n')
-            line_len--;
-          line_len = count_clusters(line, line_len, 0, line_len);
-          free(it->buffer);
-          free(it);
-          editor->cursor.col = line_len;
-          editor->cursor_preffered = UINT32_MAX;
-          mode = SELECT;
-          editor->selection_active = true;
-          editor->selection = {0, 0};
-          editor->selection_type = LINE;
-        }
         break;
       case CTRL('h'):
         editor->hover.scroll(-1);
@@ -241,68 +76,15 @@ void handle_editor_event(Editor *editor, KeyEvent event) {
         editor->hover_active = true;
         break;
       case 'h':
-        if (editor->lsp && editor->lsp->allow_hover) {
+        fetch_lsp_hover(editor);
-          LineIterator *it = begin_l_iter(editor->root, editor->cursor.row);
-          char *line = next_line(it, nullptr);
-          if (!line) {
-            free(it->buffer);
-            free(it);
         break;
-          }
+      case 'a': {
-          uint32_t col = utf8_byte_offset_to_utf16(line, editor->cursor.col);
-          free(it->buffer);
-          free(it);
-          json hover_request = {
-              {"jsonrpc", "2.0"},
-              {"method", "textDocument/hover"},
-              {"params",
-               {{"textDocument", {{"uri", editor->uri}}},
-                {"position",
-                 {{"line", editor->cursor.row}, {"character", col}}}}}};
-          LSPPending *pending = new LSPPending();
-          pending->editor = editor;
-          pending->method = "textDocument/hover";
-          pending->callback = [](Editor *editor, std::string, json hover) {
-            if (hover.contains("result") && !hover["result"].is_null()) {
-              auto &contents = hover["result"]["contents"];
-              std::string hover_text = "";
-              bool is_markup = false;
-              if (contents.is_object()) {
-                hover_text += contents["value"].get<std::string>();
-                is_markup = (contents["kind"].get<std::string>() == "markdown");
-              } else if (contents.is_array()) {
-                for (auto &block : contents) {
-                  if (block.is_string()) {
-                    hover_text += block.get<std::string>() + "\n";
-                  } else if (block.is_object() && block.contains("language") &&
-                             block.contains("value")) {
-                    std::string lang = block["language"].get<std::string>();
-                    std::string val = block["value"].get<std::string>();
-                    is_markup = true;
-                    hover_text += "```" + lang + "\n" + val + "\n```\n";
-                  }
-                }
-              } else if (contents.is_string()) {
-                hover_text += contents.get<std::string>();
-              }
-              if (!hover_text.empty()) {
-                editor->hover.clear();
-                editor->hover.text = clean_text(hover_text);
-                editor->hover.is_markup = is_markup;
-                editor->hover.render_first();
-                editor->hover_active = true;
-              }
-            }
-          };
-          lsp_send(editor->lsp, hover_request, pending);
-        }
-        break;
-      case 'a':
         mode = INSERT;
+        Coord start = editor->cursor;
         cursor_right(editor, 1);
         if (start.row != editor->cursor.row)
           cursor_left(editor, 1);
-        break;
+      } break;
       case 'i':
         mode = INSERT;
         break;
@@ -315,11 +97,7 @@ void handle_editor_event(Editor *editor, KeyEvent event) {
         editor->jumper_set = false;
         break;
       case 'N':
-        for (uint8_t i = 0; i < 94; i++)
+        clear_hooks_at_line(editor, editor->cursor.row);
-          if (editor->hooks[i] == editor->cursor.row + 1) {
-            editor->hooks[i] = 0;
-            break;
-          }
         break;
       case 's':
       case 'v':
@@ -355,29 +133,18 @@ void handle_editor_event(Editor *editor, KeyEvent event) {
         ensure_cursor(editor);
         break;
       case '>':
-      case '.': {
+      case '.':
-        uint32_t delta = editor->indents.indent_line(editor->cursor.row);
+        indent_current_line(editor);
-        editor->cursor.col = start.col + delta;
+        break;
-        editor->cursor.row = start.row;
-      } break;
       case '<':
-      case ',': {
+      case ',':
-        uint32_t delta = editor->indents.dedent_line(editor->cursor.row);
+        dedent_current_line(editor);
-        editor->cursor.col = MAX((int64_t)start.col - delta, 0);
+        break;
-        editor->cursor.row = start.row;
-      } break;
       case CTRL('s'):
         save_file(editor);
         break;
       case 'p':
-        uint32_t len;
+        paste(editor);
-        char *text = get_from_clipboard(&len);
-        if (text) {
-          edit_insert(editor, editor->cursor, text, len);
-          uint32_t grapheme_len = count_clusters(text, len, 0, len);
-          cursor_right(editor, grapheme_len);
-          free(text);
-        }
         break;
       }
     }
@@ -391,156 +158,13 @@ void handle_editor_event(Editor *editor, KeyEvent event) {
         } else if (event.c[0] == '\n' || event.c[0] == '\r') {
           editor->indents.insert_new_line(editor->cursor);
         } else if (event.c[0] == CTRL('W')) {
-          uint32_t prev_col_byte, prev_col_cluster;
+          delete_prev_word(editor);
-          word_boundaries(editor, editor->cursor, &prev_col_byte, nullptr,
-                          &prev_col_cluster, nullptr);
-          if (prev_col_byte == editor->cursor.col)
-            edit_erase(editor, editor->cursor, -1);
-          else
-            edit_erase(editor, editor->cursor, -(int64_t)prev_col_cluster);
         } else if (isprint((unsigned char)(event.c[0]))) {
-          char c = event.c[0];
+          insert_char(editor, event.c[0]);
-          uint32_t col = editor->cursor.col;
-          LineIterator *it = begin_l_iter(editor->root, editor->cursor.row);
-          if (!it)
-            return;
-          uint32_t len;
-          char *line = next_line(it, &len);
-          if (!line) {
-            free(it->buffer);
-            free(it);
-            return;
-          }
-          bool skip_insert = false;
-          if (line && col < len) {
-            char next = line[col];
-            if ((c == '}' && next == '}') || (c == ')' && next == ')') ||
-                (c == ']' && next == ']') || (c == '"' && next == '"') ||
-                (c == '\'' && next == '\'')) {
-              cursor_right(editor, 1);
-              skip_insert = true;
-            }
-          }
-          free(it->buffer);
-          free(it);
-          if (!skip_insert) {
-            char closing = 0;
-            switch (c) {
-            case '{':
-              closing = '}';
-              break;
-            case '(':
-              closing = ')';
-              break;
-            case '[':
-              closing = ']';
-              break;
-            case '"':
-              closing = '"';
-              break;
-            case '\'':
-              closing = '\'';
-              break;
-            }
-            if (closing) {
-              char pair[2] = {c, closing};
-              edit_insert(editor, editor->cursor, pair, 2);
-              cursor_right(editor, 1);
-            } else {
-              edit_insert(editor, editor->cursor, &c, 1);
-              cursor_right(editor, 1);
-            }
-            if (editor->lsp && editor->lsp->allow_formatting_on_type) {
-              for (char ch : editor->lsp->format_chars) {
-                if (ch == c) {
-                  LineIterator *it =
-                      begin_l_iter(editor->root, editor->cursor.row);
-                  if (!it)
-                    return;
-                  char *line = next_line(it, nullptr);
-                  if (!line) {
-                    free(it->buffer);
-                    free(it);
-                    return;
-                  }
-                  uint32_t col =
-                      utf8_byte_offset_to_utf16(line, editor->cursor.col);
-                  free(it->buffer);
-                  free(it);
-                  int version = editor->lsp_version;
-                  json message = {
-                      {"jsonrpc", "2.0"},
-                      {"method", "textDocument/onTypeFormatting"},
-                      {"params",
-                       {{"textDocument", {{"uri", editor->uri}}},
-                        {"position",
-                         {{"line", editor->cursor.row}, {"character", col}}},
-                        {"ch", std::string(1, c)},
-                        {"options",
-                         {{"tabSize", 2},
-                          {"insertSpaces", true},
-                          {"trimTrailingWhitespace", true},
-                          {"trimFinalNewlines", true}}}}}};
-                  LSPPending *pending = new LSPPending();
-                  pending->editor = editor;
-                  pending->method = "textDocument/onTypeFormatting";
-                  pending->callback = [version](Editor *editor, std::string,
-                                                json message) {
-                    if (version != editor->lsp_version)
-                      return;
-                    auto &edits = message["result"];
-                    if (edits.is_array()) {
-                      std::vector<TextEdit> t_edits;
-                      t_edits.reserve(edits.size());
-                      for (auto &edit : edits) {
-                        TextEdit t_edit;
-                        t_edit.text = edit.value("newText", "");
-                        t_edit.start.row = edit["range"]["start"]["line"];
-                        t_edit.start.col = edit["range"]["start"]["character"];
-                        t_edit.end.row = edit["range"]["end"]["line"];
-                        t_edit.end.col = edit["range"]["end"]["character"];
-                        utf8_normalize_edit(editor, &t_edit);
-                        t_edits.push_back(t_edit);
-                      }
-                      apply_lsp_edits(editor, t_edits, false);
-                      ensure_scroll(editor);
-                    }
-                  };
-                  lsp_send(editor->lsp, message, pending);
-                  break;
-                }
-              }
-            }
-          }
         } else if (event.c[0] == 0x7F || event.c[0] == 0x08) {
-          Coord prev_pos = editor->cursor;
+          backspace_edit(editor);
-          if (prev_pos.col > 0)
-            prev_pos.col--;
-          LineIterator *it = begin_l_iter(editor->root, editor->cursor.row);
-          if (!it)
-            return;
-          char *line = next_line(it, nullptr);
-          char prev_char = line[prev_pos.col];
-          char next_char = line[editor->cursor.col];
-          free(it->buffer);
-          free(it);
-          bool is_pair = (prev_char == '{' && next_char == '}') ||
-                         (prev_char == '(' && next_char == ')') ||
-                         (prev_char == '[' && next_char == ']') ||
-                         (prev_char == '"' && next_char == '"') ||
-                         (prev_char == '\'' && next_char == '\'');
-          if (is_pair) {
-            edit_erase(editor, editor->cursor, 1);
-            edit_erase(editor, prev_pos, 1);
-          } else {
-            edit_erase(editor, editor->cursor, -1);
-          }
         } else if (event.c[0] == 0x1B) {
-          Coord prev_pos = editor->cursor;
+          normal_mode(editor);
-          mode = NORMAL;
-          cursor_left(editor, 1);
-          if (prev_pos.row != editor->cursor.row)
-            cursor_right(editor, 1);
         }
       } else if (event.len > 1) {
         edit_insert(editor, editor->cursor, event.c, event.len);
@@ -553,29 +177,15 @@ void handle_editor_event(Editor *editor, KeyEvent event) {
         edit_erase(editor, editor->cursor, 1);
         break;
       case CNTRL:
-        uint32_t next_col_byte, next_col_cluster;
+        delete_next_word(editor);
-        word_boundaries(editor, editor->cursor, nullptr, &next_col_byte,
-                        nullptr, &next_col_cluster);
-        if (next_col_byte == editor->cursor.col)
-          edit_erase(editor, editor->cursor, 1);
-        else
-          edit_erase(editor, editor->cursor, next_col_cluster);
         break;
       }
     } else if (event.key_type == KEY_PASTE) {
-      if (event.c) {
+      insert_str(editor, event.c, event.len);
-        edit_insert(editor, editor->cursor, event.c, event.len);
-        uint32_t grapheme_len =
-            count_clusters(event.c, event.len, 0, event.len);
-        cursor_right(editor, grapheme_len);
-      }
     }
     break;
   case SELECT:
     if (event.key_type == KEY_CHAR && event.len == 1) {
-      uint32_t len;
-      char *text;
-      Coord start;
       switch (event.c[0]) {
       case 0x1B:
       case 's':
@@ -584,41 +194,15 @@ void handle_editor_event(Editor *editor, KeyEvent event) {
         mode = NORMAL;
         break;
       case 'y':
-        text = get_selection(editor, &len, nullptr);
+        copy(editor);
-        copy_to_clipboard(text, len);
-        free(text);
-        editor->selection_active = false;
         mode = NORMAL;
         break;
       case 'x':
-        text = get_selection(editor, &len, &start);
+        cut(editor);
-        copy_to_clipboard(text, len);
-        len = count_clusters(text, len, 0, len);
-        edit_erase(editor, start, len);
-        free(text);
-        editor->selection_active = false;
         mode = NORMAL;
         break;
       case 'p':
-        text = get_from_clipboard(&len);
+        paste(editor);
-        if (text) {
-          Coord start, end;
-          if (editor->cursor >= editor->selection) {
-            start = editor->selection;
-            end = move_right(editor, editor->cursor, 1);
-          } else {
-            start = editor->cursor;
-            end = move_right(editor, editor->selection, 1);
-          }
-          uint32_t start_byte =
-              line_to_byte(editor->root, start.row, nullptr) + start.col;
-          uint32_t end_byte =
-              line_to_byte(editor->root, end.row, nullptr) + end.col;
-          edit_erase(editor, start, end_byte - start_byte);
-          edit_insert(editor, editor->cursor, text, len);
-          free(text);
-        }
-        editor->selection_active = false;
         mode = NORMAL;
         break;
       }

src/editor/helpers.cc

@@ -0,0 +1,490 @@
+#include "editor/helpers.h"
+#include "editor/editor.h"
+#include "io/sysio.h"
+#include "lsp/lsp.h"
+#include "main.h"
+#include "utils/utils.h"
+
+void cut(Editor *editor) {
+  if (mode != SELECT)
+    return;
+  Coord start;
+  uint32_t len;
+  char *text = get_selection(editor, &len, &start);
+  copy_to_clipboard(text, len);
+  len = count_clusters(text, len, 0, len);
+  edit_erase(editor, start, len);
+  free(text);
+  editor->selection_active = false;
+}
+
+void copy(Editor *editor) {
+  if (mode != SELECT)
+    return;
+  uint32_t len;
+  char *text = get_selection(editor, &len, nullptr);
+  copy_to_clipboard(text, len);
+  free(text);
+  editor->selection_active = false;
+}
+
+void paste(Editor *editor) {
+  uint32_t len;
+  if (mode == NORMAL) {
+    char *text = get_from_clipboard(&len);
+    if (text) {
+      insert_str(editor, text, len);
+      free(text);
+    }
+  } else if (mode == SELECT) {
+    char *text = get_from_clipboard(&len);
+    if (text) {
+      Coord start, end;
+      selection_bounds(editor, &start, &end);
+      uint32_t start_byte =
+          line_to_byte(editor->root, start.row, nullptr) + start.col;
+      uint32_t end_byte =
+          line_to_byte(editor->root, end.row, nullptr) + end.col;
+      edit_erase(editor, start, end_byte - start_byte);
+      edit_insert(editor, editor->cursor, text, len);
+      free(text);
+    }
+    editor->selection_active = false;
+  }
+}
+
+void insert_str(Editor *editor, char *c, uint32_t len) {
+  if (c) {
+    edit_insert(editor, editor->cursor, c, len);
+    uint32_t grapheme_len = count_clusters(c, len, 0, len);
+    cursor_right(editor, grapheme_len);
+  }
+}
+
+void indent_current_line(Editor *editor) {
+  Coord start = editor->cursor;
+  uint32_t delta = editor->indents.indent_line(editor->cursor.row);
+  editor->cursor.col = start.col + delta;
+  editor->cursor.row = start.row;
+}
+
+void dedent_current_line(Editor *editor) {
+  Coord start = editor->cursor;
+  uint32_t delta = editor->indents.dedent_line(editor->cursor.row);
+  editor->cursor.col = MAX((int64_t)start.col - delta, 0);
+  editor->cursor.row = start.row;
+}
+
+void insert_char(Editor *editor, char c) {
+  uint32_t col = editor->cursor.col;
+  LineIterator *it = begin_l_iter(editor->root, editor->cursor.row);
+  if (!it)
+    return;
+  uint32_t len;
+  char *line = next_line(it, &len);
+  if (!line) {
+    free(it->buffer);
+    free(it);
+    return;
+  }
+  bool skip_insert = false;
+  if (line && col < len) {
+    char next = line[col];
+    if ((c == '}' && next == '}') || (c == ')' && next == ')') ||
+        (c == ']' && next == ']') || (c == '"' && next == '"') ||
+        (c == '\'' && next == '\'')) {
+      cursor_right(editor, 1);
+      skip_insert = true;
+    }
+  }
+  free(it->buffer);
+  free(it);
+  if (!skip_insert) {
+    char closing = 0;
+    switch (c) {
+    case '{':
+      closing = '}';
+      break;
+    case '(':
+      closing = ')';
+      break;
+    case '[':
+      closing = ']';
+      break;
+    case '"':
+      closing = '"';
+      break;
+    case '\'':
+      closing = '\'';
+      break;
+    }
+    if (closing) {
+      char pair[2] = {c, closing};
+      edit_insert(editor, editor->cursor, pair, 2);
+      cursor_right(editor, 1);
+    } else {
+      edit_insert(editor, editor->cursor, &c, 1);
+      cursor_right(editor, 1);
+    }
+    if (editor->lsp && editor->lsp->allow_formatting_on_type) {
+      for (char ch : editor->lsp->format_chars) {
+        if (ch == c) {
+          LineIterator *it = begin_l_iter(editor->root, editor->cursor.row);
+          if (!it)
+            return;
+          uint32_t len;
+          char *line = next_line(it, &len);
+          if (!line) {
+            free(it->buffer);
+            free(it);
+            return;
+          }
+          uint32_t col = utf8_offset_to_utf16(line, len, editor->cursor.col);
+          free(it->buffer);
+          free(it);
+          int version = editor->lsp_version;
+          json message = {
+              {"jsonrpc", "2.0"},
+              {"method", "textDocument/onTypeFormatting"},
+              {"params",
+               {{"textDocument", {{"uri", editor->uri}}},
+                {"position",
+                 {{"line", editor->cursor.row}, {"character", col}}},
+                {"ch", std::string(1, c)},
+                {"options",
+                 {{"tabSize", 2},
+                  {"insertSpaces", true},
+                  {"trimTrailingWhitespace", true},
+                  {"trimFinalNewlines", true}}}}}};
+          LSPPending *pending = new LSPPending();
+          pending->editor = editor;
+          pending->method = "textDocument/onTypeFormatting";
+          pending->callback = [version](Editor *editor, std::string,
+                                        json message) {
+            if (version != editor->lsp_version)
+              return;
+            auto &edits = message["result"];
+            if (edits.is_array()) {
+              std::vector<TextEdit> t_edits;
+              t_edits.reserve(edits.size());
+              for (auto &edit : edits) {
+                TextEdit t_edit;
+                t_edit.text = edit.value("newText", "");
+                t_edit.start.row = edit["range"]["start"]["line"];
+                t_edit.start.col = edit["range"]["start"]["character"];
+                t_edit.end.row = edit["range"]["end"]["line"];
+                t_edit.end.col = edit["range"]["end"]["character"];
+                utf8_normalize_edit(editor, &t_edit);
+                t_edits.push_back(t_edit);
+              }
+              apply_lsp_edits(editor, t_edits, false);
+              ensure_scroll(editor);
+            }
+          };
+          lsp_send(editor->lsp, message, pending);
+          break;
+        }
+      }
+    }
+  }
+}
+
+void normal_mode(Editor *editor) {
+  Coord prev_pos = editor->cursor;
+  mode = NORMAL;
+  cursor_left(editor, 1);
+  if (prev_pos.row != editor->cursor.row)
+    cursor_right(editor, 1);
+}
+
+void backspace_edit(Editor *editor) {
+  Coord prev_pos = editor->cursor;
+  if (prev_pos.col > 0)
+    prev_pos.col--;
+  LineIterator *it = begin_l_iter(editor->root, editor->cursor.row);
+  if (!it)
+    return;
+  char *line = next_line(it, nullptr);
+  char prev_char = line[prev_pos.col];
+  char next_char = line[editor->cursor.col];
+  free(it->buffer);
+  free(it);
+  bool is_pair = (prev_char == '{' && next_char == '}') ||
+                 (prev_char == '(' && next_char == ')') ||
+                 (prev_char == '[' && next_char == ']') ||
+                 (prev_char == '"' && next_char == '"') ||
+                 (prev_char == '\'' && next_char == '\'');
+  if (is_pair) {
+    edit_erase(editor, editor->cursor, 1);
+    edit_erase(editor, prev_pos, 1);
+  } else {
+    edit_erase(editor, editor->cursor, -1);
+  }
+}
+
+void delete_prev_word(Editor *editor) {
+  uint32_t prev_col_byte, prev_col_cluster;
+  word_boundaries(editor, editor->cursor, &prev_col_byte, nullptr,
+                  &prev_col_cluster, nullptr);
+  if (prev_col_byte == editor->cursor.col)
+    edit_erase(editor, editor->cursor, -1);
+  else
+    edit_erase(editor, editor->cursor, -(int64_t)prev_col_cluster);
+}
+
+void delete_next_word(Editor *editor) {
+  uint32_t next_col_byte, next_col_cluster;
+  word_boundaries(editor, editor->cursor, nullptr, &next_col_byte, nullptr,
+                  &next_col_cluster);
+  if (next_col_byte == editor->cursor.col)
+    edit_erase(editor, editor->cursor, 1);
+  else
+    edit_erase(editor, editor->cursor, next_col_cluster);
+}
+
+void clear_hooks_at_line(Editor *editor, uint32_t line) {
+  for (uint8_t i = 0; i < 94; i++)
+    if (editor->hooks[i] == line + 1) {
+      editor->hooks[i] = 0;
+      break;
+    }
+}
+
+void cursor_prev_word(Editor *editor) {
+  uint32_t prev_col;
+  word_boundaries(editor, editor->cursor, &prev_col, nullptr, nullptr, nullptr);
+  editor->cursor_preffered = UINT32_MAX;
+  if (prev_col == editor->cursor.col)
+    cursor_left(editor, 1);
+  else
+    editor->cursor = {editor->cursor.row, prev_col};
+}
+
+void cursor_next_word(Editor *editor) {
+  uint32_t next_col;
+  word_boundaries(editor, editor->cursor, nullptr, &next_col, nullptr, nullptr);
+  editor->cursor_preffered = UINT32_MAX;
+  if (next_col == editor->cursor.col)
+    cursor_right(editor, 1);
+  else
+    editor->cursor = {editor->cursor.row, next_col};
+}
+
+void select_all(Editor *editor) {
+  if (editor->root->line_count > 0) {
+    editor->cursor.row = editor->root->line_count - 1;
+    LineIterator *it = begin_l_iter(editor->root, editor->cursor.row);
+    if (!it)
+      return;
+    uint32_t line_len;
+    char *line = next_line(it, &line_len);
+    if (!line)
+      return;
+    if (line_len > 0 && line[line_len - 1] == '\n')
+      line_len--;
+    line_len = count_clusters(line, line_len, 0, line_len);
+    free(it->buffer);
+    free(it);
+    editor->cursor.col = line_len;
+    editor->cursor_preffered = UINT32_MAX;
+    mode = SELECT;
+    editor->selection_active = true;
+    editor->selection = {0, 0};
+    editor->selection_type = LINE;
+  }
+}
+
+void fetch_lsp_hover(Editor *editor) {
+  if (editor->lsp && editor->lsp->allow_hover) {
+    LineIterator *it = begin_l_iter(editor->root, editor->cursor.row);
+    uint32_t line_len;
+    char *line = next_line(it, &line_len);
+    if (!line) {
+      free(it->buffer);
+      free(it);
+      return;
+    }
+    uint32_t col = utf8_offset_to_utf16(line, line_len, editor->cursor.col);
+    free(it->buffer);
+    free(it);
+    json hover_request = {
+        {"jsonrpc", "2.0"},
+        {"method", "textDocument/hover"},
+        {"params",
+         {{"textDocument", {{"uri", editor->uri}}},
+          {"position", {{"line", editor->cursor.row}, {"character", col}}}}}};
+    LSPPending *pending = new LSPPending();
+    pending->editor = editor;
+    pending->method = "textDocument/hover";
+    pending->callback = [](Editor *editor, std::string, json hover) {
+      if (hover.contains("result") && !hover["result"].is_null()) {
+        auto &contents = hover["result"]["contents"];
+        std::string hover_text = "";
+        bool is_markup = false;
+        if (contents.is_object()) {
+          hover_text += contents["value"].get<std::string>();
+          is_markup = (contents["kind"].get<std::string>() == "markdown");
+        } else if (contents.is_array()) {
+          for (auto &block : contents) {
+            if (block.is_string()) {
+              hover_text += block.get<std::string>() + "\n";
+            } else if (block.is_object() && block.contains("language") &&
+                       block.contains("value")) {
+              std::string lang = block["language"].get<std::string>();
+              std::string val = block["value"].get<std::string>();
+              is_markup = true;
+              hover_text += "```" + lang + "\n" + val + "\n```\n";
+            }
+          }
+        } else if (contents.is_string()) {
+          hover_text += contents.get<std::string>();
+        }
+        if (!hover_text.empty()) {
+          editor->hover.clear();
+          editor->hover.text = clean_text(hover_text);
+          editor->hover.is_markup = is_markup;
+          editor->hover.render_first();
+          editor->hover_active = true;
+        }
+      }
+    };
+    lsp_send(editor->lsp, hover_request, pending);
+  }
+}
+
+void handle_mouse(Editor *editor, KeyEvent event) {
+  static std::chrono::steady_clock::time_point last_click_time =
+      std::chrono::steady_clock::now();
+  static uint32_t click_count = 0;
+  static Coord last_click_pos = {UINT32_MAX, UINT32_MAX};
+  if (event.key_type == KEY_MOUSE) {
+    auto now = std::chrono::steady_clock::now();
+    auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(
+                        now - last_click_time)
+                        .count();
+    switch (event.mouse_state) {
+    case SCROLL:
+      switch (event.mouse_direction) {
+      case SCROLL_UP:
+        scroll_up(editor, 4);
+        ensure_cursor(editor);
+        break;
+      case SCROLL_DOWN:
+        scroll_down(editor, 4);
+        ensure_cursor(editor);
+        break;
+      case SCROLL_LEFT:
+        cursor_left(editor, 10);
+        break;
+      case SCROLL_RIGHT:
+        cursor_right(editor, 10);
+        break;
+      }
+      break;
+    case PRESS:
+      if (event.mouse_button == LEFT_BTN) {
+        Coord cur_pos = {event.mouse_x, event.mouse_y};
+        if (duration < 250 && last_click_pos == cur_pos)
+          click_count++;
+        else
+          click_count = 1;
+        last_click_time = now;
+        last_click_pos = cur_pos;
+        Coord p = editor_hit_test(editor, event.mouse_x, event.mouse_y);
+        if (p.row == UINT32_MAX && p.col == UINT32_MAX)
+          return;
+        editor->cursor_preffered = UINT32_MAX;
+        if (click_count == 1) {
+          editor->cursor = p;
+          editor->selection = p;
+          if (mode == SELECT) {
+            mode = NORMAL;
+            editor->selection_active = false;
+          }
+        } else if (click_count == 2) {
+          uint32_t prev_col, next_col;
+          word_boundaries(editor, editor->cursor, &prev_col, &next_col, nullptr,
+                          nullptr);
+          if (editor->cursor < editor->selection)
+            editor->cursor = {editor->cursor.row, prev_col};
+          else
+            editor->cursor = {editor->cursor.row, next_col};
+          editor->cursor_preffered = UINT32_MAX;
+          editor->selection_type = WORD;
+          mode = SELECT;
+          editor->selection_active = true;
+        } else if (click_count >= 3) {
+          if (editor->cursor < editor->selection) {
+            editor->cursor = {p.row, 0};
+          } else {
+            uint32_t line_len;
+            LineIterator *it = begin_l_iter(editor->root, p.row);
+            char *line = next_line(it, &line_len);
+            if (!line)
+              return;
+            if (line_len > 0 && line[line_len - 1] == '\n')
+              line_len--;
+            free(it->buffer);
+            free(it);
+            editor->cursor = {p.row, line_len};
+          }
+          editor->cursor_preffered = UINT32_MAX;
+          editor->selection_type = LINE;
+          mode = SELECT;
+          editor->selection_active = true;
+          click_count = 3;
+        }
+      }
+      break;
+    case DRAG:
+      if (event.mouse_button == LEFT_BTN) {
+        Coord p = editor_hit_test(editor, event.mouse_x, event.mouse_y);
+        if (p.row == UINT32_MAX && p.col == UINT32_MAX)
+          return;
+        editor->cursor_preffered = UINT32_MAX;
+        mode = SELECT;
+        if (!editor->selection_active) {
+          editor->selection_active = true;
+          editor->selection_type = CHAR;
+        }
+        uint32_t prev_col, next_col, line_len;
+        switch (editor->selection_type) {
+        case CHAR:
+          editor->cursor = p;
+          break;
+        case WORD:
+          word_boundaries(editor, p, &prev_col, &next_col, nullptr, nullptr);
+          if (editor->cursor < editor->selection)
+            editor->cursor = {p.row, prev_col};
+          else
+            editor->cursor = {p.row, next_col};
+          break;
+        case LINE:
+          if (editor->cursor < editor->selection) {
+            editor->cursor = {p.row, 0};
+          } else {
+            LineIterator *it = begin_l_iter(editor->root, p.row);
+            char *line = next_line(it, &line_len);
+            if (!line)
+              return;
+            if (line_len > 0 && line[line_len - 1] == '\n')
+              line_len--;
+            free(it->buffer);
+            free(it);
+            editor->cursor = {p.row, line_len};
+          }
+          break;
+        }
+      }
+      break;
+    case RELEASE:
+      if (event.mouse_button == LEFT_BTN)
+        if (editor->cursor.row == editor->selection.row &&
+            editor->cursor.col == editor->selection.col) {
+          mode = NORMAL;
+          editor->selection_active = false;
+        }
+      break;
+    }
+  }
+}

src/editor/indents.cc

@@ -323,13 +323,14 @@ void IndentationEngine::insert_new_line(Coord cursor) {
       LineIterator *it = begin_l_iter(editor->root, editor->cursor.row);
       if (!it)
         return;
-      char *line = next_line(it, nullptr);
+      uint32_t line_len;
+      char *line = next_line(it, &line_len);
       if (!line) {
         free(it->buffer);
         free(it);
         return;
       }
-      uint32_t col = utf8_byte_offset_to_utf16(line, editor->cursor.col);
+      uint32_t col = utf8_offset_to_utf16(line, line_len, editor->cursor.col);
       free(it->buffer);
       free(it);
       int version = editor->lsp_version;

src/editor/lsp.cc

@@ -60,10 +60,10 @@ void editor_lsp_handle(Editor *editor, json msg) {
       if (len > 0 && line[len - 1] == '\n')
         --len;
       lock.unlock();
-      w.start = utf16_offset_to_utf8(line, w.start);
+      w.start = utf16_offset_to_utf8(line, len, w.start);
       uint32_t end = d["range"]["end"]["character"];
       if (d["range"]["end"]["line"] == w.line)
-        w.end = utf16_offset_to_utf8(line, end);
+        w.end = utf16_offset_to_utf8(line, len, end);
       free(it->buffer);
       free(it);
       std::string text = trim(d["message"].get<std::string>());

src/editor/renderer.cc

@@ -107,10 +107,12 @@ void render_editor(Editor *editor) {
   while (rendered_rows < editor->size.row) {
     uint32_t line_len;
     char *line = next_line(it, &line_len);
+    if (editor->parser) {
       if (line_data)
         line_data = editor->parser->line_tree.next();
       else
         line_data = editor->parser->line_tree.start_iter(line_index);
+    }
     if (!line)
       break;
     if (line_len > 0 && line[line_len - 1] == '\n')
@@ -186,7 +188,7 @@ void render_editor(Editor *editor) {
                    : 0);
         if (editor->selection_active && absolute_byte_pos >= sel_start &&
             absolute_byte_pos < sel_end)
-          bg = 0x555555;
+          bg = bg | 0x555555;
         uint32_t u_color = 0;
         for (const auto &w : line_warnings) {
           if (w.start <= current_byte_offset + local_render_offset &&

src/editor/selection.cc

@@ -1,17 +1,18 @@
 #include "editor/editor.h"
+#include "utils/utils.h"
 
-char *get_selection(Editor *editor, uint32_t *out_len, Coord *out_start) {
+void selection_bounds(Editor *editor, Coord *out_start, Coord *out_end) {
   std::shared_lock lock(editor->knot_mtx);
   Coord start, end;
   if (editor->cursor >= editor->selection) {
-    uint32_t prev_col, next_col;
+    uint32_t prev_col;
     switch (editor->selection_type) {
     case CHAR:
       start = editor->selection;
       end = move_right(editor, editor->cursor, 1);
       break;
     case WORD:
-      word_boundaries(editor, editor->selection, &prev_col, &next_col, nullptr,
+      word_boundaries(editor, editor->selection, &prev_col, nullptr, nullptr,
                       nullptr);
       start = {editor->selection.row, prev_col};
       end = editor->cursor;
@@ -23,13 +24,65 @@ char *get_selection(Editor *editor, uint32_t *out_len, Coord *out_start) {
     }
   } else {
     start = editor->cursor;
-    uint32_t prev_col, next_col, line_len;
+    uint32_t next_col, line_len;
     switch (editor->selection_type) {
     case CHAR:
       end = move_right(editor, editor->selection, 1);
       break;
     case WORD:
-      word_boundaries(editor, editor->selection, &prev_col, &next_col, nullptr,
+      word_boundaries(editor, editor->selection, nullptr, &next_col, nullptr,
+                      nullptr);
+      end = {editor->selection.row, next_col};
+      break;
+    case LINE:
+      LineIterator *it = begin_l_iter(editor->root, editor->selection.row);
+      char *line = next_line(it, &line_len);
+      if (!line)
+        return;
+      if (line_len > 0 && line[line_len - 1] == '\n')
+        line_len--;
+      end = {editor->selection.row, line_len};
+      free(it->buffer);
+      free(it);
+      break;
+    }
+  }
+  if (out_start)
+    *out_start = start;
+  if (out_end)
+    *out_end = end;
+}
+
+char *get_selection(Editor *editor, uint32_t *out_len, Coord *out_start) {
+  std::shared_lock lock(editor->knot_mtx);
+  Coord start, end;
+  if (editor->cursor >= editor->selection) {
+    uint32_t prev_col;
+    switch (editor->selection_type) {
+    case CHAR:
+      start = editor->selection;
+      end = move_right(editor, editor->cursor, 1);
+      break;
+    case WORD:
+      word_boundaries(editor, editor->selection, &prev_col, nullptr, nullptr,
+                      nullptr);
+      start = {editor->selection.row, prev_col};
+      end = editor->cursor;
+      break;
+    case LINE:
+      start = {editor->selection.row, 0};
+      end = editor->cursor;
+      break;
+    }
+  } else {
+    start = editor->cursor;
+    uint32_t next_col, line_len;
+    switch (editor->selection_type) {
+    case CHAR:
+      end = move_right(editor, editor->selection, 1);
+      break;
+    case WORD:
+      word_boundaries(editor, editor->selection, nullptr, &next_col, nullptr,
                       nullptr);
       end = {editor->selection.row, next_col};
       break;

src/io/renderer.cc

@@ -154,7 +154,7 @@ void render() {
               }
             }
         }
-        last_change_col = MIN(cols + 1, col + 4);
+        last_change_col = MIN(cols - 1, col + 4);
       }
     }
     if (first_change_col == -1)

src/lsp/handlers.cc

@@ -7,7 +7,7 @@ void request_add_to_lsp(Language language, Editor *editor) {
 }
 
 void add_to_lsp(Language language, Editor *editor) {
-  std::shared_ptr<LSPInstance> lsp = get_or_init_lsp(language.lsp_id);
+  std::shared_ptr<LSPInstance> lsp = get_or_init_lsp(language.lsp_name);
   if (!lsp)
     return;
   std::unique_lock lock(lsp->mtx);
@@ -41,11 +41,11 @@ void open_editor(std::shared_ptr<LSPInstance> lsp,
   lsp_send(lsp, message, nullptr);
 }
 
-static uint8_t find_lsp_id(std::shared_ptr<LSPInstance> needle) {
+static std::string find_lsp_id(std::shared_ptr<LSPInstance> needle) {
   for (const auto &[id, lsp] : active_lsps)
     if (lsp == needle)
       return id;
-  return 0;
+  return "";
 }
 
 void remove_from_lsp(Editor *editor) {
@@ -64,8 +64,8 @@ void remove_from_lsp(Editor *editor) {
                   {"method", "textDocument/didClose"},
                   {"params", {{"textDocument", {{"uri", editor->uri}}}}}};
   lsp_send(lsp, message, nullptr);
-  uint8_t lsp_id = find_lsp_id(lsp);
+  std::string lsp_id = find_lsp_id(lsp);
-  if (lsp_id && lsp->editors.empty())
+  if (!lsp_id.empty() && lsp->editors.empty())
     close_lsp(lsp_id);
 }
 

src/lsp/process.cc

@@ -1,4 +1,3 @@
-#include "config.h"
 #include "lsp/lsp.h"
 
 static bool init_lsp(std::shared_ptr<LSPInstance> lsp) {
@@ -25,7 +24,12 @@ static bool init_lsp(std::shared_ptr<LSPInstance> lsp) {
     close(in_pipe[1]);
     close(out_pipe[0]);
     close(out_pipe[1]);
-    execvp(lsp->lsp->command, (char *const *)(lsp->lsp->args.data()));
+    std::vector<char *> argv;
+    argv.push_back(const_cast<char *>(lsp->lsp->command.c_str()));
+    for (auto &arg : lsp->lsp->args)
+      argv.push_back(const_cast<char *>(arg.c_str()));
+    argv.push_back(nullptr);
+    execvp(lsp->lsp->command.c_str(), argv.data());
     perror("execvp");
     _exit(127);
   }
@@ -37,12 +41,12 @@ static bool init_lsp(std::shared_ptr<LSPInstance> lsp) {
   return true;
 }
 
-std::shared_ptr<LSPInstance> get_or_init_lsp(uint8_t lsp_id) {
+std::shared_ptr<LSPInstance> get_or_init_lsp(std::string lsp_id) {
   std::unique_lock lock(active_lsps_mtx);
   auto it = active_lsps.find(lsp_id);
   if (it == active_lsps.end()) {
-    auto map_it = kLsps.find(lsp_id);
+    auto map_it = lsps.find(lsp_id);
-    if (map_it == kLsps.end())
+    if (map_it == lsps.end())
       return nullptr;
     std::shared_ptr<LSPInstance> lsp = std::make_shared<LSPInstance>();
     lsp->lsp = &map_it->second;
@@ -54,6 +58,13 @@ std::shared_ptr<LSPInstance> get_or_init_lsp(uint8_t lsp_id) {
     pending->callback = [lsp, lsp_id](Editor *, std::string, json msg) {
       if (msg.contains("result") && msg["result"].contains("capabilities")) {
         auto &caps = msg["result"]["capabilities"];
+        // if (caps.contains("positionEncoding")) {
+        //   std::string s = caps["positionEncoding"].get<std::string>();
+        //   if (s == "utf-8")
+        //     lsp->is_utf8 = true;
+        //   log("Lsp name: %s, supports: %s", lsp->lsp->command.c_str(),
+        //       s.c_str());
+        // }
         if (caps.contains("textDocumentSync")) {
           auto &sync = caps["textDocumentSync"];
           if (sync.is_number()) {
@@ -65,8 +76,9 @@ std::shared_ptr<LSPInstance> get_or_init_lsp(uint8_t lsp_id) {
           }
         }
         lsp->allow_formatting = caps.value("documentFormattingProvider", false);
-        if (lsp_id != LUA_LS /* Lua ls gives terrible ontype formatting */ &&
+        if (lsp_id != "lua-language-server" /* Lua ls gives terrible ontype
-            caps.contains("documentOnTypeFormattingProvider")) {
+                                               formatting so disable */
+            && caps.contains("documentOnTypeFormattingProvider")) {
           auto &fmt = caps["documentOnTypeFormattingProvider"];
           if (fmt.is_object()) {
             if (fmt.contains("firstTriggerCharacter")) {
@@ -142,7 +154,7 @@ std::shared_ptr<LSPInstance> get_or_init_lsp(uint8_t lsp_id) {
   return it->second;
 }
 
-void close_lsp(uint8_t lsp_id) {
+void close_lsp(std::string lsp_id) {
   std::shared_lock active_lsps_lock(active_lsps_mtx);
   auto it = active_lsps.find(lsp_id);
   if (it == active_lsps.end())
@@ -179,7 +191,7 @@ void close_lsp(uint8_t lsp_id) {
   t.detach();
 }
 
-void clean_lsp(std::shared_ptr<LSPInstance> lsp, uint8_t lsp_id) {
+void clean_lsp(std::shared_ptr<LSPInstance> lsp, std::string lsp_id) {
   for (auto &kv : lsp->pending)
     delete kv.second;
   lsp->pid = -1;

src/lsp/workers.cc

@@ -1,7 +1,7 @@
 #include "lsp/lsp.h"
 
 std::shared_mutex active_lsps_mtx;
-std::unordered_map<uint8_t, std::shared_ptr<LSPInstance>> active_lsps;
+std::unordered_map<std::string, std::shared_ptr<LSPInstance>> active_lsps;
 
 void lsp_send(std::shared_ptr<LSPInstance> lsp, json message,
               LSPPending *pending) {

src/main.cc

@@ -2,7 +2,7 @@
 #include "editor/editor.h"
 #include "io/sysio.h"
 #include "lsp/lsp.h"
-#include "syntax/decl.h"
+#include "scripting/decl.h"
 #include "ui/bar.h"
 #include "utils/utils.h"
 
@@ -13,32 +13,12 @@ std::vector<Editor *> editors;
 uint8_t current_editor = 0;
 std::atomic<uint8_t> mode = NORMAL;
 
-void background_worker() {
-  while (running)
-    throttle(16ms, editor_worker, editors[current_editor]);
-}
-
 void background_lsp() {
   while (running)
     throttle(8ms, lsp_worker);
 }
 
-void input_listener() {
+inline Editor *editor_at(uint8_t x, uint8_t y) {
-  while (running) {
-    KeyEvent event = throttle(1ms, read_key);
-    if (event.key_type == KEY_NONE)
-      continue;
-    if (event.key_type == KEY_CHAR && event.len == 1 &&
-        event.c[0] == CTRL('q')) {
-      free(event.c);
-      running = false;
-      return;
-    }
-    event_queue.push(event);
-  }
-}
-
-Editor *editor_at(uint8_t x, uint8_t y) {
   for (Editor *ed : editors) {
     Coord pos = ed->position;
     Coord size = ed->size;
@@ -49,24 +29,73 @@ Editor *editor_at(uint8_t x, uint8_t y) {
   return nullptr;
 }
 
-uint8_t index_of(Editor *ed) {
+inline uint8_t index_of(Editor *ed) {
   for (uint8_t i = 0; i < editors.size(); i++)
     if (editors[i] == ed)
       return i;
   return 0;
 }
 
+void input_listener(Bar bar) {
+  while (running) {
+    KeyEvent event = throttle(1ms, read_key);
+    if (event.key_type == KEY_NONE)
+      goto render;
+    if (event.key_type == KEY_CHAR && event.len == 1 &&
+        event.c[0] == CTRL('q')) {
+      free(event.c);
+      running = false;
+      break;
+    }
+    if (mode != RUNNER) {
+      if (event.key_type == KEY_MOUSE) {
+        Editor *target = editor_at(event.mouse_x, event.mouse_y);
+        if (target) {
+          if (event.mouse_state == PRESS)
+            current_editor = index_of(target);
+
+          event.mouse_x -= target->position.col;
+          event.mouse_y -= target->position.row;
+          handle_editor_event(target, event);
+        }
+      } else {
+        handle_editor_event(editors[current_editor], event);
+      }
+    } else {
+      bar.handle(event);
+    }
+  render:
+    render_editor(editors[current_editor]);
+    bar.render();
+    throttle(4ms, render);
+  }
+}
+
 int main(int argc, char *argv[]) {
+  auto start = std::chrono::high_resolution_clock::now();
+
+  ruby_init();
+
+  ruby_start((get_exe_dir() + "/../config/main.rb").c_str());
+  load_theme();
+  load_languages_info();
+  load_custom_highlighters();
+
   Coord screen = start_screen();
   const char *filename = (argc > 1) ? argv[1] : "";
 
-  system(("bash " + get_exe_dir() + "/../scripts/init.sh").c_str());
+  bool unix_eol = read_line_endings();
 
-  load_theme(get_exe_dir() + "/../themes/default.json");
+  Editor *editor =
-
+      new_editor(filename, {0, 0}, {screen.row - 2, screen.col}, unix_eol);
-  Editor *editor = new_editor(filename, {0, 0}, {screen.row - 2, screen.col});
   Bar bar(screen);
 
+  auto end = std::chrono::high_resolution_clock::now();
+  auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(end - start)
+                .count();
+  ruby_log("[LOG] STARTUP_TIME: " + std::to_string(static_cast<long long>(ms)) +
+           "ms");
+
   if (!editor) {
     end_screen();
     fprintf(stderr, "Failed to load editor\n");
@@ -76,48 +105,22 @@ int main(int argc, char *argv[]) {
   editors.push_back(editor);
   current_editor = editors.size() - 1;
 
-  std::thread input_thread(input_listener);
+  std::thread input_thread(input_listener, bar);
-  std::thread work_thread(background_worker);
   std::thread lsp_thread(background_lsp);
 
-  while (running) {
+  while (running)
-    KeyEvent event;
+    throttle(16ms, editor_worker, editors[current_editor]);
-    while (event_queue.pop(event)) {
-      if (mode != RUNNER) {
-        if (event.key_type == KEY_MOUSE) {
-          Editor *target = editor_at(event.mouse_x, event.mouse_y);
-          if (!target)
-            continue;
-          if (event.mouse_state == PRESS)
-            current_editor = index_of(target);
-          event.mouse_x -= target->position.col;
-          event.mouse_y -= target->position.row;
-          handle_editor_event(target, event);
-        } else {
-          handle_editor_event(editors[current_editor], event);
-        }
-      } else {
-        bar.handle(event);
-      }
-    }
-    render_editor(editors[current_editor]);
-    bar.render();
-    throttle(4ms, render);
-  }
 
   if (input_thread.joinable())
     input_thread.join();
 
-  if (work_thread.joinable())
-    work_thread.join();
-
   if (lsp_thread.joinable())
     lsp_thread.join();
 
-  system(("bash " + get_exe_dir() + "/../scripts/exit.sh").c_str());
-
   end_screen();
 
+  ruby_shutdown();
+
   for (auto editor : editors)
     free_editor(editor);
 
@@ -131,5 +134,7 @@ int main(int argc, char *argv[]) {
     throttle(16ms, lsp_worker);
   }
 
-  return 0;
+  rb_gc_start();
+
+  return ruby_cleanup(0);
 }

src/scripting/process.cc

@@ -0,0 +1,298 @@
+#include "ruby/internal/gc.h"
+#include "ruby/internal/value.h"
+#include "scripting/decl.h"
+#include "utils/utils.h"
+
+std::unordered_map<std::string, std::pair<VALUE, VALUE>> custom_highlighters;
+
+struct R_ThemeEntry {
+  std::string key;
+  uint32_t fg = 0xFFFFFF;
+  uint32_t bg = 0x000000;
+  bool italic = false;
+  bool bold = false;
+  bool underline = false;
+  bool strikethrough = false;
+};
+
+struct R_Language {
+  std::string name;
+  uint32_t color = 0xFFFFFF;
+  std::string symbol;
+  std::vector<std::string> extensions;
+  std::vector<std::string> filenames;
+  std::vector<std::string> mimetypes;
+  std::string lsp_command; // link to LSP by name
+};
+
+VALUE C_module = Qnil;
+std::mutex ruby_mutex;
+
+void ruby_start(const char *main_file) {
+  std::lock_guard lock(ruby_mutex);
+  ruby_init_loadpath();
+  int state = 0;
+  rb_load_protect(rb_str_new_cstr(main_file), 0, &state);
+  if (state) {
+    rb_errinfo();
+    rb_set_errinfo(Qnil);
+    fprintf(stderr, "%d: Failed to load Ruby file\n", state);
+    return;
+  }
+  C_module = rb_const_get(rb_cObject, rb_intern("C"));
+  if (C_module == Qnil)
+    return;
+  VALUE block = rb_funcall(C_module, rb_intern("b_startup"), 0);
+  if (block != Qnil)
+    rb_funcall(block, rb_intern("call"), 0);
+}
+
+void ruby_shutdown() {
+  std::lock_guard lock(ruby_mutex);
+  if (C_module == Qnil)
+    return;
+  VALUE block = rb_funcall(C_module, rb_intern("b_shutdown"), 0);
+  if (block != Qnil)
+    rb_funcall(block, rb_intern("call"), 0);
+}
+
+inline std::vector<std::string> ruby_array_to_vector(VALUE rb_array) {
+  std::vector<std::string> result;
+  if (NIL_P(rb_array) || !RB_TYPE_P(rb_array, T_ARRAY))
+    return result;
+  for (long i = 0; i < RARRAY_LEN(rb_array); ++i) {
+    VALUE item = rb_ary_entry(rb_array, i);
+    if (RB_TYPE_P(item, T_STRING))
+      result.push_back(StringValueCStr(item));
+  }
+  return result;
+}
+
+void ruby_log(std::string msg) {
+  std::lock_guard lock(ruby_mutex);
+  VALUE str = rb_str_new(msg.c_str(), msg.size());
+  rb_funcall(C_module, rb_intern("queue_log"), 1, str);
+}
+
+void load_custom_highlighters() {
+  std::lock_guard lock(ruby_mutex);
+  if (C_module == Qnil)
+    return;
+  VALUE hashmap = rb_funcall(C_module, rb_intern("highlighters"), 0);
+  if (NIL_P(hashmap))
+    return;
+  VALUE keys = rb_funcall(hashmap, rb_intern("keys"), 0);
+  for (long i = 0; i < RARRAY_LEN(keys); ++i) {
+    VALUE key_sym = rb_ary_entry(keys, i);
+    std::string key = rb_id2name(SYM2ID(key_sym));
+    VALUE val_hash = rb_hash_aref(hashmap, key_sym);
+    if (NIL_P(val_hash))
+      continue;
+    VALUE parse_block = rb_hash_aref(val_hash, ID2SYM(rb_intern("parser")));
+    VALUE match_block = rb_hash_aref(val_hash, ID2SYM(rb_intern("matcher")));
+    rb_gc_register_address(&match_block);
+    rb_gc_register_address(&parse_block);
+    custom_highlighters[key] = {parse_block, match_block};
+  }
+}
+
+bool custom_compare(VALUE match_block, VALUE state1, VALUE state2) {
+  std::lock_guard lock(ruby_mutex);
+  return RTEST(rb_funcall(match_block, rb_intern("call"), 2, state1, state2));
+}
+
+VALUE parse_custom(std::vector<Token> *tokens, VALUE parser_block,
+                   const char *line, uint32_t len, VALUE state) {
+  std::lock_guard lock(ruby_mutex);
+  tokens->clear();
+  if (NIL_P(parser_block))
+    return {};
+  VALUE ruby_line = rb_str_new(line, len);
+  VALUE tokens_and_state_hash =
+      rb_funcall(parser_block, rb_intern("call"), 2, ruby_line, state);
+  VALUE tokens_rb =
+      rb_hash_aref(tokens_and_state_hash, ID2SYM(rb_intern("tokens")));
+  for (long i = 0; i < RARRAY_LEN(tokens_rb); ++i) {
+    VALUE token = rb_ary_entry(tokens_rb, i);
+    Token tok;
+    tok.type =
+        (TokenKind)NUM2INT(rb_hash_aref(token, ID2SYM(rb_intern("type"))));
+    tok.start = NUM2UINT(rb_hash_aref(token, ID2SYM(rb_intern("start"))));
+    tok.end = NUM2UINT(rb_hash_aref(token, ID2SYM(rb_intern("end"))));
+    if (tok.type < TokenKind::Count && tok.end > tok.start && tok.end <= len)
+      tokens->push_back(tok);
+  }
+  return rb_hash_aref(tokens_and_state_hash, ID2SYM(rb_intern("state")));
+}
+
+static std::vector<R_ThemeEntry> read_theme() {
+  std::lock_guard lock(ruby_mutex);
+  std::vector<R_ThemeEntry> result;
+  if (C_module == Qnil)
+    return result;
+  VALUE theme_hash = rb_funcall(C_module, rb_intern("theme"), 0);
+  if (NIL_P(theme_hash))
+    return result;
+  VALUE keys = rb_funcall(theme_hash, rb_intern("keys"), 0);
+  for (long i = 0; i < RARRAY_LEN(keys); ++i) {
+    VALUE key_sym = rb_ary_entry(keys, i);
+    std::string key = rb_id2name(SYM2ID(key_sym));
+    VALUE val_hash = rb_hash_aref(theme_hash, key_sym);
+    if (NIL_P(val_hash))
+      continue;
+    R_ThemeEntry entry;
+    entry.key = key;
+    VALUE fg = rb_hash_aref(val_hash, ID2SYM(rb_intern("fg")));
+    VALUE bg = rb_hash_aref(val_hash, ID2SYM(rb_intern("bg")));
+    VALUE italic = rb_hash_aref(val_hash, ID2SYM(rb_intern("italic")));
+    VALUE bold = rb_hash_aref(val_hash, ID2SYM(rb_intern("bold")));
+    VALUE underline = rb_hash_aref(val_hash, ID2SYM(rb_intern("underline")));
+    VALUE strikethrough =
+        rb_hash_aref(val_hash, ID2SYM(rb_intern("strikethrough")));
+    if (!NIL_P(fg))
+      entry.fg = NUM2UINT(fg);
+    if (!NIL_P(bg))
+      entry.bg = NUM2UINT(bg);
+    if (!NIL_P(italic))
+      entry.italic = RTEST(italic);
+    if (!NIL_P(bold))
+      entry.bold = RTEST(bold);
+    if (!NIL_P(underline))
+      entry.underline = RTEST(underline);
+    if (!NIL_P(strikethrough))
+      entry.strikethrough = RTEST(strikethrough);
+    result.push_back(entry);
+  }
+  return result;
+}
+
+void load_theme() {
+  std::vector<R_ThemeEntry> entries = read_theme();
+  Highlight default_hl = {0xFFFFFF, 0, 0};
+  for (auto &entry : entries) {
+    if (entry.key == "default") {
+      default_hl.fg = entry.fg;
+      default_hl.bg = entry.bg;
+      if (entry.italic)
+        default_hl.flags |= CF_ITALIC;
+      if (entry.bold)
+        default_hl.flags |= CF_BOLD;
+      if (entry.underline)
+        default_hl.flags |= CF_UNDERLINE;
+      if (entry.strikethrough)
+        default_hl.flags |= CF_STRIKETHROUGH;
+      break;
+    }
+  }
+  for (auto &hl : highlights)
+    hl = default_hl;
+  for (auto &entry : entries) {
+    if (entry.key == "default")
+      continue;
+    std::string key = "k_" + entry.key;
+    for (char &c : key)
+      c = std::toupper(static_cast<unsigned char>(c));
+    auto it = kind_map.find(key);
+    if (it == kind_map.end())
+      continue;
+    Highlight hl = {0xFFFFFF, 0, 0};
+    hl.fg = entry.fg;
+    hl.bg = entry.bg;
+    if (entry.italic)
+      hl.flags |= CF_ITALIC;
+    if (entry.bold)
+      hl.flags |= CF_BOLD;
+    if (entry.underline)
+      hl.flags |= CF_UNDERLINE;
+    if (entry.strikethrough)
+      hl.flags |= CF_STRIKETHROUGH;
+    highlights[static_cast<uint8_t>(it->second)] = hl;
+  }
+}
+
+std::vector<LSP> read_lsps() {
+  std::vector<LSP> result;
+  if (C_module == Qnil)
+    return result;
+  VALUE lsp_hash = rb_funcall(C_module, rb_intern("lsp_config"), 0);
+  if (NIL_P(lsp_hash))
+    return result;
+  VALUE keys = rb_funcall(lsp_hash, rb_intern("keys"), 0);
+  for (long i = 0; i < RARRAY_LEN(keys); ++i) {
+    VALUE key = rb_ary_entry(keys, i);
+    std::string cmd = StringValueCStr(key);
+    VALUE args_array = rb_hash_aref(lsp_hash, key);
+    std::vector<std::string> args = ruby_array_to_vector(args_array);
+    result.push_back({cmd, args});
+  }
+  return result;
+}
+
+std::vector<R_Language> read_languages() {
+  std::vector<R_Language> result;
+  if (C_module == Qnil)
+    return result;
+  VALUE lang_hash = rb_funcall(C_module, rb_intern("languages"), 0);
+  if (NIL_P(lang_hash))
+    return result;
+  VALUE keys = rb_funcall(lang_hash, rb_intern("keys"), 0);
+  for (long i = 0; i < RARRAY_LEN(keys); ++i) {
+    VALUE key = rb_ary_entry(keys, i);
+    VALUE val_hash = rb_hash_aref(lang_hash, key);
+    if (NIL_P(val_hash))
+      continue;
+    R_Language lang;
+    lang.name = rb_id2name(SYM2ID(key));
+    VALUE fg = rb_hash_aref(val_hash, ID2SYM(rb_intern("color")));
+    VALUE symbol = rb_hash_aref(val_hash, ID2SYM(rb_intern("symbol")));
+    VALUE extensions = rb_hash_aref(val_hash, ID2SYM(rb_intern("extensions")));
+    VALUE filenames = rb_hash_aref(val_hash, ID2SYM(rb_intern("filenames")));
+    VALUE mimetypes = rb_hash_aref(val_hash, ID2SYM(rb_intern("mimetypes")));
+    VALUE lsp = rb_hash_aref(val_hash, ID2SYM(rb_intern("lsp")));
+    if (!NIL_P(fg))
+      lang.color = NUM2UINT(fg);
+    if (!NIL_P(symbol))
+      lang.symbol = StringValueCStr(symbol);
+    lang.extensions = ruby_array_to_vector(extensions);
+    lang.filenames = ruby_array_to_vector(filenames);
+    lang.mimetypes = ruby_array_to_vector(mimetypes);
+    if (!NIL_P(lsp))
+      lang.lsp_command = StringValueCStr(lsp);
+    result.push_back(lang);
+  }
+  return result;
+}
+
+void load_languages_info() {
+  std::lock_guard lock(ruby_mutex);
+  auto langs = read_languages();
+  auto lsps_t = read_lsps();
+  languages.clear();
+  for (auto &lang : langs) {
+    Language l;
+    l.name = lang.name;
+    l.color = lang.color;
+    l.lsp_name = lang.lsp_command;
+    l.symbol = lang.symbol;
+    languages[lang.name] = l;
+    for (auto &ext : lang.extensions)
+      language_extensions[ext] = lang.name;
+    // TODO: seperate extensions and filenames
+    for (auto &filename : lang.filenames)
+      language_extensions[filename] = lang.name;
+    for (auto &mimetype : lang.mimetypes)
+      language_mimetypes[mimetype] = lang.name;
+  }
+  for (auto &lsp : lsps_t)
+    lsps[lsp.command] = lsp;
+}
+
+bool read_line_endings() {
+  std::lock_guard lock(ruby_mutex);
+  if (C_module == Qnil)
+    return true;
+  VALUE le = rb_funcall(C_module, rb_intern("line_endings"), 0);
+  if (SYMBOL_P(le))
+    return std::string(rb_id2name(SYM2ID(le))) == "unix";
+  return true;
+}

src/syntax/bash.cc

@@ -68,28 +68,28 @@ std::shared_ptr<void> bash_parse(std::vector<Token> *tokens,
       uint32_t start = i;
       while (i < len) {
         if (text[i] == state->full_state->lit.delim[0]) {
-          tokens->push_back({start, i, TokenKind::String});
+          tokens->push_back({start, i, TokenKind::K_STRING});
           state->full_state->in_state = BashFullState::NONE;
           break;
         }
         i++;
       }
       if (i == len)
-        tokens->push_back({start, i, TokenKind::String});
+        tokens->push_back({start, i, TokenKind::K_STRING});
       continue;
     }
     if (text[i] == '#') {
       if (i == 0 && len > 4 && text[i + 1] == '!') {
-        tokens->push_back({0, len, TokenKind::Shebang});
+        tokens->push_back({0, len, TokenKind::K_SHEBANG});
         return state;
       }
-      tokens->push_back({i, len, TokenKind::Comment});
+      tokens->push_back({i, len, TokenKind::K_COMMENT});
       return state;
     } else if (text[i] == '\'') {
       state->full_state->in_state = BashFullState::STRING;
       state->full_state->lit.delim = "'";
       state->full_state->lit.allow_interp = false;
-      tokens->push_back({i, ++i, TokenKind::String});
+      tokens->push_back({i, ++i, TokenKind::K_STRING});
       continue;
     }
     i++;

src/syntax/parser.cc

@@ -2,6 +2,7 @@
 #include "editor/editor.h"
 #include "io/knot.h"
 #include "main.h"
+#include "ruby/internal/special_consts.h"
 #include "syntax/decl.h"
 #include "syntax/langs.h"
 
@@ -11,13 +12,21 @@ Parser::Parser(Editor *n_editor, std::string n_lang, uint32_t n_scroll_max) {
   editor = n_editor;
   scroll_max = n_scroll_max;
   lang = n_lang;
+  auto custom_parser = custom_highlighters.find(n_lang);
+  if (custom_parser != custom_highlighters.end()) {
+    parser_block = custom_parser->second.first;
+    match_block = custom_parser->second.second;
+    is_custom = true;
+  } else {
     auto pair = parsers.find(n_lang);
     if (pair != parsers.end()) {
       parse_func = std::get<0>(pair->second);
       state_match_func = std::get<1>(pair->second);
+      is_custom = false;
     } else {
       assert("unknown lang should be checked by caller" && 0);
     }
+  }
   edit(0, 0, editor->root->line_count);
 }
 
@@ -29,9 +38,9 @@ void Parser::edit(uint32_t start_line, uint32_t old_end_line,
   if (inserted_rows > 0)
     line_tree.insert(start_line, inserted_rows);
   if (start_line > 0)
-    dirty_lines.insert(start_line - 1);
+    dirty_lines.push(start_line - 1);
-  dirty_lines.insert(start_line);
+  dirty_lines.push(start_line);
-  dirty_lines.insert(start_line + 1);
+  dirty_lines.push(start_line + 1);
 }
 
 void Parser::work() {
@@ -41,16 +50,18 @@ void Parser::work() {
   k_lock.unlock();
   uint32_t capacity = 256;
   char *text = (char *)calloc((capacity + 1), sizeof(char));
-  std::set<uint32_t> tmp_dirty;
   std::unique_lock lock_data(data_mutex);
-  tmp_dirty.swap(dirty_lines);
   lock_data.unlock();
-  std::set<uint32_t> remaining_dirty;
   std::unique_lock lock(mutex);
   lock.unlock();
-  for (uint32_t c_line : tmp_dirty) {
+  uint32_t c_line;
+  while (dirty_lines.pop(c_line)) {
+    if (!running.load(std::memory_order_relaxed)) {
+      free(text);
+      return;
+    }
     if (c_line > scroll_max + 40) {
-      remaining_dirty.insert(c_line);
+      dirty_lines.push(c_line);
       continue;
     }
     uint32_t line_count = line_tree.count();
@@ -65,6 +76,10 @@ void Parser::work() {
         free(text);
         return;
       }
+      if (scroll_dirty.exchange(false, std::memory_order_acq_rel)) {
+        dirty_lines.push(c_line);
+        c_line = scroll_max < 50 ? 0 : scroll_max - 50;
+      }
       k_lock.lock();
       if (c_line > editor->root->line_count) {
         k_lock.unlock();
@@ -95,8 +110,22 @@ void Parser::work() {
           lock.unlock();
         continue;
       }
-      std::shared_ptr<void> new_state =
+      std::shared_ptr<void> new_state{nullptr};
-          parse_func(&line_data->tokens, prev_state, text, r_len);
+      if (is_custom) {
+        VALUE state = Qnil;
+        if (prev_state) {
+          std::shared_ptr<CustomState> state_ptr =
+              std::static_pointer_cast<CustomState>(prev_state);
+          state = state_ptr->state;
+        }
+        VALUE out_state =
+            parse_custom(&line_data->tokens, parser_block, text, r_len, state);
+        std::shared_ptr<CustomState> out_state_ptr =
+            std::make_shared<CustomState>(out_state);
+        new_state = out_state_ptr;
+      } else {
+        new_state = parse_func(&line_data->tokens, prev_state, text, r_len);
+      }
       line_data->in_state = prev_state;
       line_data->out_state = new_state;
       if (!running.load(std::memory_order_relaxed)) {
@@ -110,17 +139,33 @@ void Parser::work() {
         if (lock.owns_lock())
           lock.unlock();
         if (c_line > 0)
-          remaining_dirty.insert(c_line - 1);
+          dirty_lines.push(c_line - 1);
-        remaining_dirty.insert(c_line);
+        dirty_lines.push(c_line);
         break;
       }
-      if (c_line < line_count && (line_data = line_tree.at(c_line)) &&
+      if (c_line < line_count && (line_data = line_tree.at(c_line))) {
-          state_match_func(prev_state, line_data->in_state)) {
+        bool done = false;
+        if (is_custom) {
+          VALUE in_state_v = Qnil;
+          if (prev_state)
+            in_state_v =
+                std::static_pointer_cast<CustomState>(prev_state)->state;
+          VALUE out_state_v = Qnil;
+          if (line_data->in_state)
+            out_state_v =
+                std::static_pointer_cast<CustomState>(line_data->in_state)
+                    ->state;
+          done = custom_compare(match_block, in_state_v, out_state_v);
+        } else {
+          done = state_match_func(prev_state, line_data->in_state);
+        }
+        if (done) {
           lock_data.unlock();
           if (lock.owns_lock())
             lock.unlock();
           break;
         }
+      }
       lock_data.unlock();
       if (lock.owns_lock())
         lock.unlock();
@@ -132,80 +177,19 @@ void Parser::work() {
   }
   free(text);
   lock_data.lock();
-  dirty_lines = std::move(remaining_dirty);
 }
 
 void Parser::scroll(uint32_t line) {
   if (line != scroll_max) {
     scroll_max = line;
-    uint32_t c_line = line > 100 ? line - 100 : 0;
+    uint32_t c_line = line > 50 ? line - 50 : 0;
-    if (line_tree.count() < c_line)
+    if (c_line >= line_tree.count())
       return;
     std::unique_lock lock_data(data_mutex);
     if (line_tree.at(c_line)->in_state || line_tree.at(c_line)->out_state)
       return;
-    lock_data.unlock();
+    scroll_dirty = true;
-    std::shared_lock k_lock(editor->knot_mtx);
+    dirty_lines.push(c_line);
-    k_lock.unlock();
-    uint32_t capacity = 256;
-    char *text = (char *)calloc((capacity + 1), sizeof(char));
-    uint32_t line_count = line_tree.count();
-    std::unique_lock lock(mutex);
-    std::shared_ptr<void> prev_state =
-        (c_line > 0) ? line_tree.at(c_line - 1)->out_state : nullptr;
-    lock.unlock();
-    while (c_line < line_count) {
-      if (!running.load(std::memory_order_relaxed)) {
-        free(text);
-        return;
-      }
-      k_lock.lock();
-      if (c_line > editor->root->line_count) {
-        k_lock.unlock();
-        continue;
-      }
-      uint32_t r_offset, r_len;
-      r_offset = line_to_byte(editor->root, c_line, &r_len);
-      if (r_len > capacity) {
-        capacity = r_len;
-        text = (char *)realloc(text, capacity + 1);
-        memset(text, 0, capacity + 1);
-      }
-      read_into(editor->root, r_offset, r_len, text);
-      k_lock.unlock();
-      if (c_line < scroll_max &&
-          ((scroll_max > 100 && c_line > scroll_max - 100) || c_line < 100))
-        lock.lock();
-      if (line_tree.count() < c_line) {
-        if (lock.owns_lock())
-          lock.unlock();
-        continue;
-      }
-      lock_data.lock();
-      LineData *line_data = line_tree.at(c_line);
-      if (!line_data) {
-        lock_data.unlock();
-        if (lock.owns_lock())
-          lock.unlock();
-        continue;
-      }
-      std::shared_ptr<void> new_state =
-          parse_func(&line_data->tokens, prev_state, text, r_len);
-      line_data->in_state = nullptr;
-      line_data->out_state = new_state;
-      lock_data.unlock();
-      if (lock.owns_lock())
-        lock.unlock();
-      if (!running.load(std::memory_order_relaxed)) {
-        free(text);
-        return;
-      }
-      prev_state = new_state;
-      c_line++;
-      if (c_line < line_count && c_line > scroll_max + 50)
-        break;
-    }
-    free(text);
   } else {
     scroll_max = line;
   }

src/syntax/ruby.cc

@@ -324,7 +324,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
     if (state->full_state->in_state == RubyFullState::END)
       return state;
     if (state->full_state->in_state == RubyFullState::COMMENT) {
-      tokens->push_back({i, len, TokenKind::Comment});
+      tokens->push_back({i, len, TokenKind::K_COMMENT});
       if (i == 0 && len == 4 && text[i] == '=' && text[i + 1] == 'e' &&
           text[i + 2] == 'n' && text[i + 3] == 'd') {
         state->full_state->in_state = RubyFullState::NONE;
@@ -344,18 +344,18 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
           state->heredocs.pop_front();
           if (state->heredocs.empty())
             state->full_state->in_state = RubyFullState::NONE;
-          tokens->push_back({i, len, TokenKind::Annotation});
+          tokens->push_back({i, len, TokenKind::K_ANNOTATION});
           return state;
         }
       }
       uint32_t start = i;
       if (!state->heredocs.front().allow_interpolation) {
-        tokens->push_back({i, len, TokenKind::String});
+        tokens->push_back({i, len, TokenKind::K_STRING});
         return state;
       } else {
         while (i < len) {
           if (text[i] == '\\') {
-            tokens->push_back({start, i, TokenKind::String});
+            tokens->push_back({start, i, TokenKind::K_STRING});
             start = i;
             i++;
             if (i < len && text[i] == 'x') {
@@ -412,12 +412,12 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
               if (i < len)
                 i++;
             }
-            tokens->push_back({start, i, TokenKind::Escape});
+            tokens->push_back({start, i, TokenKind::K_ESCAPE});
             continue;
           }
           if (text[i] == '#' && i + 1 < len && text[i + 1] == '{') {
-            tokens->push_back({start, i, TokenKind::String});
+            tokens->push_back({start, i, TokenKind::K_STRING});
-            tokens->push_back({i, i + 2, TokenKind::Interpolation});
+            tokens->push_back({i, i + 2, TokenKind::K_INTERPOLATION});
             i += 2;
             state->interp_stack.push(state->full_state);
             state->full_state = std::make_shared<RubyFullState>();
@@ -427,7 +427,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
           i++;
         }
         if (i == len)
-          tokens->push_back({start, len, TokenKind::String});
+          tokens->push_back({start, len, TokenKind::K_STRING});
         continue;
       }
     }
@@ -435,7 +435,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
       uint32_t start = i;
       while (i < len) {
         if (text[i] == '\\') {
-          tokens->push_back({start, i, TokenKind::String});
+          tokens->push_back({start, i, TokenKind::K_STRING});
           start = i;
           i++;
           if (i < len && text[i] == 'x') {
@@ -492,13 +492,13 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
             if (i < len)
               i++;
           }
-          tokens->push_back({start, i, TokenKind::Escape});
+          tokens->push_back({start, i, TokenKind::K_ESCAPE});
           continue;
         }
         if (state->full_state->lit.allow_interp && text[i] == '#' &&
             i + 1 < len && text[i + 1] == '{') {
-          tokens->push_back({start, i, TokenKind::String});
+          tokens->push_back({start, i, TokenKind::K_STRING});
-          tokens->push_back({i, i + 2, TokenKind::Interpolation});
+          tokens->push_back({i, i + 2, TokenKind::K_INTERPOLATION});
           i += 2;
           state->interp_stack.push(state->full_state);
           state->full_state = std::make_shared<RubyFullState>();
@@ -514,7 +514,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
           if (state->full_state->lit.delim_start ==
               state->full_state->lit.delim_end) {
             i++;
-            tokens->push_back({start, i, TokenKind::String});
+            tokens->push_back({start, i, TokenKind::K_STRING});
             state->full_state->in_state = RubyFullState::NONE;
             state->full_state->expecting_expr = false;
             break;
@@ -522,7 +522,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
             state->full_state->lit.brace_level--;
             if (state->full_state->lit.brace_level == 0) {
               i++;
-              tokens->push_back({start, i, TokenKind::String});
+              tokens->push_back({start, i, TokenKind::K_STRING});
               state->full_state->in_state = RubyFullState::NONE;
               state->full_state->expecting_expr = false;
               break;
@@ -532,14 +532,14 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
         i++;
       }
       if (i == len)
-        tokens->push_back({start, len, TokenKind::String});
+        tokens->push_back({start, len, TokenKind::K_STRING});
       continue;
     }
     if (state->full_state->in_state == RubyFullState::REGEXP) {
       uint32_t start = i;
       while (i < len) {
         if (text[i] == '\\') {
-          tokens->push_back({start, i, TokenKind::Regexp});
+          tokens->push_back({start, i, TokenKind::K_REGEXP});
           start = i;
           i++;
           if (i < len && text[i] == 'x') {
@@ -596,12 +596,12 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
             if (i < len)
               i++;
           }
-          tokens->push_back({start, i, TokenKind::Escape});
+          tokens->push_back({start, i, TokenKind::K_ESCAPE});
           continue;
         }
         if (text[i] == '#' && i + 1 < len && text[i + 1] == '{') {
-          tokens->push_back({start, i, TokenKind::Regexp});
+          tokens->push_back({start, i, TokenKind::K_REGEXP});
-          tokens->push_back({i, i + 2, TokenKind::Interpolation});
+          tokens->push_back({i, i + 2, TokenKind::K_INTERPOLATION});
           i += 2;
           state->interp_stack.push(state->full_state);
           state->full_state = std::make_shared<RubyFullState>();
@@ -617,7 +617,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
           if (state->full_state->lit.delim_start ==
               state->full_state->lit.delim_end) {
             i += 1;
-            tokens->push_back({start, i, TokenKind::Regexp});
+            tokens->push_back({start, i, TokenKind::K_REGEXP});
             state->full_state->in_state = RubyFullState::NONE;
             state->full_state->expecting_expr = false;
             break;
@@ -625,7 +625,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
             state->full_state->lit.brace_level--;
             if (state->full_state->lit.brace_level == 0) {
               i += 1;
-              tokens->push_back({start, i, TokenKind::Regexp});
+              tokens->push_back({start, i, TokenKind::K_REGEXP});
               state->full_state->in_state = RubyFullState::NONE;
               state->full_state->expecting_expr = false;
               break;
@@ -635,7 +635,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
         i++;
       }
       if (i == len)
-        tokens->push_back({start, len, TokenKind::Regexp});
+        tokens->push_back({start, len, TokenKind::K_REGEXP});
       continue;
     }
     if (i == 0 && len == 6) {
@@ -643,7 +643,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
           text[i + 3] == 'g' && text[i + 4] == 'i' && text[i + 5] == 'n') {
         state->full_state->in_state = RubyFullState::COMMENT;
         state->full_state->expecting_expr = false;
-        tokens->push_back({0, len, TokenKind::Comment});
+        tokens->push_back({0, len, TokenKind::K_COMMENT});
         return state;
       }
     }
@@ -664,7 +664,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
         indented = true;
       if (text[j] == '~' || text[j] == '-')
         j++;
-      tokens->push_back({i, j, TokenKind::Operator});
+      tokens->push_back({i, j, TokenKind::K_OPERATOR});
       if (j >= len)
         continue;
       std::string delim;
@@ -685,7 +685,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
       }
       state->full_state->expecting_expr = false;
       if (!delim.empty()) {
-        tokens->push_back({s, j, TokenKind::Annotation});
+        tokens->push_back({s, j, TokenKind::K_ANNOTATION});
         state->heredocs.push_back({delim, interpolation, indented});
         state->full_state->in_state = RubyFullState::HEREDOC;
         heredoc_first = true;
@@ -694,7 +694,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
       continue;
     }
     if (text[i] == '/' && state->full_state->expecting_expr) {
-      tokens->push_back({i, i + 1, TokenKind::Regexp});
+      tokens->push_back({i, i + 1, TokenKind::K_REGEXP});
       state->full_state->in_state = RubyFullState::REGEXP;
       state->full_state->expecting_expr = false;
       state->full_state->lit.delim_start = '/';
@@ -705,10 +705,10 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
     } else if (text[i] == '#') {
       if (i == 0 && len > 4 && text[i + 1] == '!') {
         state->full_state->expecting_expr = false;
-        tokens->push_back({0, len, TokenKind::Shebang});
+        tokens->push_back({0, len, TokenKind::K_SHEBANG});
         return state;
       }
-      tokens->push_back({i, len, TokenKind::Comment});
+      tokens->push_back({i, len, TokenKind::K_COMMENT});
       state->full_state->expecting_expr = false;
       return state;
     } else if (text[i] == '.') {
@@ -720,7 +720,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
           i++;
         }
       }
-      tokens->push_back({start, i, TokenKind::Operator});
+      tokens->push_back({start, i, TokenKind::K_OPERATOR});
       state->full_state->expecting_expr = false;
       continue;
     } else if (text[i] == ':') {
@@ -728,7 +728,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
       uint32_t start = i;
       i++;
       if (i >= len) {
-        tokens->push_back({start, i, TokenKind::Operator});
+        tokens->push_back({start, i, TokenKind::K_OPERATOR});
         state->full_state->expecting_expr = true;
         continue;
       }
@@ -737,7 +737,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
         continue;
       }
       if (text[i] == '\'' || text[i] == '"') {
-        tokens->push_back({start, i, TokenKind::Label});
+        tokens->push_back({start, i, TokenKind::K_LABEL});
         state->full_state->expecting_expr = true;
         continue;
       }
@@ -748,22 +748,22 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
           i++;
         while (i < len && identifier_char(text[i]))
           i++;
-        tokens->push_back({start, i, TokenKind::Label});
+        tokens->push_back({start, i, TokenKind::K_LABEL});
         continue;
       }
       uint32_t op_len = operator_trie.match(text, i, len, identifier_char);
       if (op_len > 0) {
-        tokens->push_back({start, i + op_len, TokenKind::Label});
+        tokens->push_back({start, i + op_len, TokenKind::K_LABEL});
         i += op_len;
         continue;
       }
       if (identifier_start_char(text[i])) {
         uint32_t word_len = get_next_word(text, i, len);
-        tokens->push_back({start, i + word_len, TokenKind::Label});
+        tokens->push_back({start, i + word_len, TokenKind::K_LABEL});
         i += word_len;
         continue;
       }
-      tokens->push_back({start, i, TokenKind::Operator});
+      tokens->push_back({start, i, TokenKind::K_OPERATOR});
       continue;
     } else if (text[i] == '@') {
       state->full_state->expecting_expr = false;
@@ -779,7 +779,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
         continue;
       while (i < len && identifier_char(text[i]))
         i++;
-      tokens->push_back({start, i, TokenKind::VariableInstance});
+      tokens->push_back({start, i, TokenKind::K_VARIABLEINSTANCE});
       continue;
     } else if (text[i] == '$') {
       state->full_state->expecting_expr = false;
@@ -802,7 +802,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
       } else {
         continue;
       }
-      tokens->push_back({start, i, TokenKind::VariableGlobal});
+      tokens->push_back({start, i, TokenKind::K_VARIABLEGLOBAL});
       continue;
     } else if (text[i] == '?') {
       state->full_state->expecting_expr = false;
@@ -818,7 +818,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
             continue;
           if (i < len && isxdigit(text[i]))
             i++;
-          tokens->push_back({start, i, TokenKind::Char});
+          tokens->push_back({start, i, TokenKind::K_CHAR});
           continue;
         } else if (i < len && text[i] == 'u') {
           i++;
@@ -838,26 +838,26 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
             i++;
           else
             continue;
-          tokens->push_back({start, i, TokenKind::Char});
+          tokens->push_back({start, i, TokenKind::K_CHAR});
           continue;
         } else if (i < len) {
           i++;
-          tokens->push_back({start, i, TokenKind::Char});
+          tokens->push_back({start, i, TokenKind::K_CHAR});
           continue;
         }
       } else if (i < len && text[i] != ' ') {
         i++;
-        tokens->push_back({start, i, TokenKind::Char});
+        tokens->push_back({start, i, TokenKind::K_CHAR});
         continue;
       } else {
         state->full_state->expecting_expr = true;
-        tokens->push_back({start, i, TokenKind::Operator});
+        tokens->push_back({start, i, TokenKind::K_OPERATOR});
         continue;
       }
     } else if (text[i] == '{') {
       state->full_state->expecting_expr = true;
       uint8_t brace_color =
-          (uint8_t)TokenKind::Brace1 + (state->full_state->brace_level % 5);
+          (uint8_t)TokenKind::K_BRACE1 + (state->full_state->brace_level % 5);
       tokens->push_back({i, i + 1, (TokenKind)brace_color});
       state->interp_level++;
       state->full_state->brace_level++;
@@ -869,11 +869,11 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
       if (state->interp_level == 0 && !state->interp_stack.empty()) {
         state->full_state = state->interp_stack.top();
         state->interp_stack.pop();
-        tokens->push_back({i, i + 1, TokenKind::Interpolation});
+        tokens->push_back({i, i + 1, TokenKind::K_INTERPOLATION});
       } else {
         state->full_state->brace_level--;
         uint8_t brace_color =
-            (uint8_t)TokenKind::Brace1 + (state->full_state->brace_level % 5);
+            (uint8_t)TokenKind::K_BRACE1 + (state->full_state->brace_level % 5);
         tokens->push_back({i, i + 1, (TokenKind)brace_color});
       }
       i++;
@@ -881,7 +881,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
     } else if (text[i] == '(') {
       state->full_state->expecting_expr = true;
       uint8_t brace_color =
-          (uint8_t)TokenKind::Brace1 + (state->full_state->brace_level % 5);
+          (uint8_t)TokenKind::K_BRACE1 + (state->full_state->brace_level % 5);
       tokens->push_back({i, i + 1, (TokenKind)brace_color});
       state->full_state->brace_level++;
       i++;
@@ -890,14 +890,14 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
       state->full_state->expecting_expr = false;
       state->full_state->brace_level--;
       uint8_t brace_color =
-          (uint8_t)TokenKind::Brace1 + (state->full_state->brace_level % 5);
+          (uint8_t)TokenKind::K_BRACE1 + (state->full_state->brace_level % 5);
       tokens->push_back({i, i + 1, (TokenKind)brace_color});
       i++;
       continue;
     } else if (text[i] == '[') {
       state->full_state->expecting_expr = true;
       uint8_t brace_color =
-          (uint8_t)TokenKind::Brace1 + (state->full_state->brace_level % 5);
+          (uint8_t)TokenKind::K_BRACE1 + (state->full_state->brace_level % 5);
       tokens->push_back({i, i + 1, (TokenKind)brace_color});
       state->full_state->brace_level++;
       i++;
@@ -906,13 +906,13 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
       state->full_state->expecting_expr = false;
       state->full_state->brace_level--;
       uint8_t brace_color =
-          (uint8_t)TokenKind::Brace1 + (state->full_state->brace_level % 5);
+          (uint8_t)TokenKind::K_BRACE1 + (state->full_state->brace_level % 5);
       tokens->push_back({i, i + 1, (TokenKind)brace_color});
       i++;
       continue;
     } else if (text[i] == '\'') {
       state->full_state->expecting_expr = false;
-      tokens->push_back({i, i + 1, TokenKind::String});
+      tokens->push_back({i, i + 1, TokenKind::K_STRING});
       state->full_state->in_state = RubyFullState::STRING;
       state->full_state->lit.delim_start = '\'';
       state->full_state->lit.delim_end = '\'';
@@ -921,7 +921,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
       continue;
     } else if (text[i] == '"') {
       state->full_state->expecting_expr = false;
-      tokens->push_back({i, i + 1, TokenKind::String});
+      tokens->push_back({i, i + 1, TokenKind::K_STRING});
       state->full_state->in_state = RubyFullState::STRING;
       state->full_state->lit.delim_start = '"';
       state->full_state->lit.delim_end = '"';
@@ -930,7 +930,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
       continue;
     } else if (text[i] == '`') {
       state->full_state->expecting_expr = false;
-      tokens->push_back({i, i + 1, TokenKind::String});
+      tokens->push_back({i, i + 1, TokenKind::K_STRING});
       state->full_state->in_state = RubyFullState::STRING;
       state->full_state->lit.delim_start = '`';
       state->full_state->lit.delim_end = '`';
@@ -1001,8 +1001,9 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
         delim_end = delim_start;
         break;
       }
-      tokens->push_back({i, i + prefix_len + 1,
+      tokens->push_back(
-                         (is_regexp ? TokenKind::Regexp : TokenKind::String)});
+          {i, i + prefix_len + 1,
+           (is_regexp ? TokenKind::K_REGEXP : TokenKind::K_STRING)});
       state->full_state->in_state =
           is_regexp ? RubyFullState::REGEXP : RubyFullState::STRING;
       state->full_state->lit.delim_start = delim_start;
@@ -1110,47 +1111,47 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
             i--;
         }
       }
-      tokens->push_back({start, i, TokenKind::Number});
+      tokens->push_back({start, i, TokenKind::K_NUMBER});
       continue;
     } else if (identifier_start_char(text[i])) {
       state->full_state->expecting_expr = false;
       uint32_t length;
       if ((length = base_keywords_trie.match(text, i, len, identifier_char))) {
-        tokens->push_back({i, i + length, TokenKind::Keyword});
+        tokens->push_back({i, i + length, TokenKind::K_KEYWORD});
         i += length;
         continue;
       } else if ((length = expecting_keywords_trie.match(text, i, len,
                                                          identifier_char))) {
         state->full_state->expecting_expr = true;
-        tokens->push_back({i, i + length, TokenKind::Keyword});
+        tokens->push_back({i, i + length, TokenKind::K_KEYWORD});
         i += length;
         continue;
       } else if ((length = operator_keywords_trie.match(text, i, len,
                                                         identifier_char))) {
-        tokens->push_back({i, i + length, TokenKind::KeywordOperator});
+        tokens->push_back({i, i + length, TokenKind::K_KEYWORDOPERATOR});
         i += length;
         continue;
       } else if ((length = expecting_operators_trie.match(
                       text, i, len, identifier_char)) > 0) {
         state->full_state->expecting_expr = true;
-        tokens->push_back({i, i + length, TokenKind::KeywordOperator});
+        tokens->push_back({i, i + length, TokenKind::K_KEYWORDOPERATOR});
         i += length;
         continue;
       } else if ((length = types_trie.match(text, i, len, identifier_char))) {
-        tokens->push_back({i, i + length, TokenKind::Type});
+        tokens->push_back({i, i + length, TokenKind::K_TYPE});
         i += length;
         continue;
       } else if ((length = methods_trie.match(text, i, len, identifier_char))) {
-        tokens->push_back({i, i + length, TokenKind::Function});
+        tokens->push_back({i, i + length, TokenKind::K_FUNCTION});
         i += length;
         continue;
       } else if ((length =
                       builtins_trie.match(text, i, len, identifier_char))) {
-        tokens->push_back({i, i + length, TokenKind::Constant});
+        tokens->push_back({i, i + length, TokenKind::K_CONSTANT});
         i += length;
         continue;
       } else if ((length = errors_trie.match(text, i, len, identifier_char))) {
-        tokens->push_back({i, i + length, TokenKind::Error});
+        tokens->push_back({i, i + length, TokenKind::K_ERROR});
         i += length;
         continue;
       } else if (text[i] >= 'A' && text[i] <= 'Z') {
@@ -1158,10 +1159,10 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
         i += get_next_word(text, i, len);
         if (i - start >= 5 && text[i - 5] == 'E' && text[i - 4] == 'r' &&
             text[i - 3] == 'r' && text[i - 2] == 'o' && text[i - 1] == 'r') {
-          tokens->push_back({start, i, TokenKind::Error});
+          tokens->push_back({start, i, TokenKind::K_ERROR});
           continue;
         }
-        tokens->push_back({start, i, TokenKind::Constant});
+        tokens->push_back({start, i, TokenKind::K_CONSTANT});
         continue;
       } else {
         uint32_t start = i;
@@ -1169,36 +1170,36 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
             text[i + 2] == 'u' && text[i + 3] == 'e' &&
             ((i + 4 < len && !identifier_char(text[i + 4])) || i + 4 == len)) {
           i += 4;
-          tokens->push_back({start, i, TokenKind::True});
+          tokens->push_back({start, i, TokenKind::K_TRUE});
           continue;
         }
         if (i + 4 < len && text[i] == 'f' && text[i + 1] == 'a' &&
             text[i + 2] == 'l' && text[i + 3] == 's' && text[i + 4] == 'e' &&
             ((i + 5 < len && !identifier_char(text[i + 5])) || i + 5 == len)) {
           i += 5;
-          tokens->push_back({start, i, TokenKind::False});
+          tokens->push_back({start, i, TokenKind::K_FALSE});
           continue;
         }
         if (i + 3 < len && text[i] == 'd' && text[i + 1] == 'e' &&
             text[i + 2] == 'f') {
           i += 3;
-          tokens->push_back({start, i, TokenKind::Keyword});
+          tokens->push_back({start, i, TokenKind::K_KEYWORD});
           while (i < len && (text[i] == ' ' || text[i] == '\t'))
             i++;
           while (i < len) {
             if (identifier_start_char(text[i])) {
               uint32_t width = get_next_word(text, i, len);
               if (text[i] >= 'A' && text[i] <= 'Z')
-                tokens->push_back({i, i + width, TokenKind::Constant});
+                tokens->push_back({i, i + width, TokenKind::K_CONSTANT});
               else if (width == 4 && (text[i] >= 's' && text[i + 1] == 'e' &&
                                       text[i + 2] == 'l' && text[i + 3] == 'f'))
-                tokens->push_back({i, i + width, TokenKind::Keyword});
+                tokens->push_back({i, i + width, TokenKind::K_KEYWORD});
               i += width;
               if (i < len && text[i] == '.') {
                 i++;
                 continue;
               }
-              tokens->push_back({i - width, i, TokenKind::Function});
+              tokens->push_back({i - width, i, TokenKind::K_FUNCTION});
               break;
             } else {
               break;
@@ -1210,15 +1211,15 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
           i++;
         if (i < len && text[i] == ':') {
           i++;
-          tokens->push_back({start, i, TokenKind::Label});
+          tokens->push_back({start, i, TokenKind::K_LABEL});
           continue;
         } else if (i < len && (text[i] == '!' || text[i] == '?')) {
           i++;
-          tokens->push_back({start, i, TokenKind::Function});
+          tokens->push_back({start, i, TokenKind::K_FUNCTION});
         } else {
           uint32_t tmp = i;
           if (tmp < len && (text[tmp] == '(' || text[tmp] == '{')) {
-            tokens->push_back({start, i, TokenKind::Function});
+            tokens->push_back({start, i, TokenKind::K_FUNCTION});
             continue;
           } else if (tmp < len && (text[tmp] == ' ' || text[tmp] == '\t')) {
             tmp++;
@@ -1230,7 +1231,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
           if (tmp >= len)
             continue;
           if (!isascii(text[tmp])) {
-            tokens->push_back({start, i, TokenKind::Function});
+            tokens->push_back({start, i, TokenKind::K_FUNCTION});
             continue;
           } else if (text[tmp] == '-' || text[tmp] == '&' || text[tmp] == '%' ||
                      text[tmp] == ':') {
@@ -1244,7 +1245,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
                      text[tmp] == '^' || text[tmp] == '<' || text[tmp] == '>') {
             continue;
           }
-          tokens->push_back({start, i, TokenKind::Function});
+          tokens->push_back({start, i, TokenKind::K_FUNCTION});
         }
         continue;
       }
@@ -1252,7 +1253,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
       uint32_t op_len;
       if ((op_len =
                operator_trie.match(text, i, len, [](char) { return false; }))) {
-        tokens->push_back({i, i + op_len, TokenKind::Operator});
+        tokens->push_back({i, i + op_len, TokenKind::K_OPERATOR});
         i += op_len;
         state->full_state->expecting_expr = true;
         continue;

src/ui/hover.cc

@@ -24,6 +24,7 @@ void HoverBox::scroll(int32_t number) {
 
 void HoverBox::render_first(bool scroll) {
   if (!scroll) {
+    // TODO: call syntax highlighter here
   }
   uint32_t longest_line = 0;
   uint32_t current_width = 0;

src/utils/system.cc

@@ -1,6 +1,10 @@
-#include "config.h"
 #include "utils/utils.h"
 
+std::unordered_map<std::string, Language> languages;
+std::unordered_map<std::string, std::string> language_extensions;
+std::unordered_map<std::string, std::string> language_mimetypes;
+std::unordered_map<std::string, LSP> lsps;
+
 void log(const char *fmt, ...) {
   FILE *fp = fopen("/tmp/log.txt", "a");
   if (!fp)
@@ -41,33 +45,39 @@ std::string get_exe_dir() {
 }
 
 char *load_file(const char *path, uint32_t *out_len) {
-  std::ifstream file(path, std::ios::in | std::ios::binary | std::ios::ate);
+  std::ifstream file(path, std::ios::binary | std::ios::ate);
   if (!file.is_open())
     return nullptr;
   std::streamsize len = file.tellg();
-  if (len < 0 || static_cast<uint32_t>(len) > 0xFFFFFFFF)
+  if (len < 0 || static_cast<uint64_t>(len) > 0xFFFFFFFF)
     return nullptr;
   file.seekg(0, std::ios::beg);
-  bool add_newline = false;
+  unsigned char bom[3] = {0};
-  if (len > 0) {
+  file.read(reinterpret_cast<char *>(bom), 3);
-    file.seekg(-1, std::ios::end);
+  if ((bom[0] == 0xFF && bom[1] == 0xFE) || (bom[0] == 0xFE && bom[1] == 0xFF))
-    char last_char;
+    return nullptr;
-    file.read(&last_char, 1);
+  bool has_utf8_bom = (bom[0] == 0xEF && bom[1] == 0xBB && bom[2] == 0xBF);
-    if (last_char != '\n')
+  uint32_t skip = has_utf8_bom ? 3 : 0;
-      add_newline = true;
+  uint32_t data_len = static_cast<uint32_t>(len) - skip;
-  }
+  file.seekg(skip, std::ios::beg);
-  file.seekg(0, std::ios::beg);
+  char *buf = (char *)malloc(data_len + 1);
-  uint32_t alloc_size = static_cast<uint32_t>(len) + (add_newline ? 1 : 0);
-  char *buf = (char *)malloc(alloc_size);
   if (!buf)
     return nullptr;
-  if (!file.read(buf, len)) {
+  file.read(buf, data_len);
-    free(buf);
+  if (memchr(buf, '\r', data_len) == nullptr) {
-    return nullptr;
+    uint32_t write = data_len;
+    if (write == 0 || buf[write - 1] != '\n')
+      buf[write++] = '\n';
+    *out_len = write;
+    return buf;
   }
-  if (add_newline)
+  uint32_t write = 0;
-    buf[len++] = '\n';
+  for (uint32_t i = 0; i < data_len; ++i)
-  *out_len = static_cast<uint32_t>(len);
+    if (buf[i] != '\r')
+      buf[write++] = buf[i];
+  if (write == 0 || buf[write - 1] != '\n')
+    buf[write++] = '\n';
+  *out_len = write;
   return buf;
 }
 
@@ -108,17 +118,17 @@ Language language_for_file(const char *filename) {
   std::string ext = file_extension(filename);
   std::string lang_name;
   if (!ext.empty()) {
-    auto it = kExtToLang.find(ext);
+    auto it = language_extensions.find(ext);
-    if (it != kExtToLang.end())
+    if (it != language_extensions.end())
-      return kLanguages.find(it->second)->second;
+      return languages.find(it->second)->second;
   }
   char *mime = detect_file_type(filename);
   if (mime) {
     std::string mime_type(mime);
     free(mime);
-    auto it = kMimeToLang.find(mime_type);
+    auto it = language_mimetypes.find(mime_type);
-    if (it != kMimeToLang.end())
+    if (it != language_mimetypes.end())
-      return kLanguages.find(it->second)->second;
+      return languages.find(it->second)->second;
   }
   return Language{};
 }

src/utils/unicode.cc

@@ -1,3 +1,4 @@
+#include "utfcpp/source/utf8.h"
 #include "utils/utils.h"
 
 int display_width(const char *str, size_t len) {
@@ -98,46 +99,42 @@ uint32_t count_clusters(const char *line, size_t len, size_t from, size_t to) {
   return count;
 }
 
-uint32_t utf8_byte_offset_to_utf16(const char *s, uint32_t byte_pos) {
+size_t utf8_offset_to_utf16(const char *utf8, size_t utf8_len,
-  uint32_t utf16_units = 0;
+                            size_t byte_offset) {
-  uint32_t i = 0;
+  if (byte_offset > utf8_len)
-  while (i < byte_pos) {
+    return byte_offset;
-    unsigned char c = s[i];
+  const char *start = utf8;
-    if ((c & 0x80) == 0x00) {
+  const char *mid = utf8 + byte_offset;
-      i += 1;
+  if (!utf8::is_valid(start, mid))
-      utf16_units += 1;
+    assert(0 && "invalid utf8");
-    } else if ((c & 0xE0) == 0xC0) {
+  size_t utf16_offset = 0;
-      i += 2;
+  for (auto it = start; it < mid;) {
-      utf16_units += 1;
+    uint32_t codepoint = utf8::next(it, mid);
-    } else if ((c & 0xF0) == 0xE0) {
+    if (codepoint <= 0xFFFF)
-      i += 3;
+      utf16_offset += 1;
-      utf16_units += 1;
+    else
-    } else {
+      utf16_offset += 2;
-      i += 4;
-      utf16_units += 2;
   }
-  }
+  return utf16_offset;
-  return utf16_units;
 }
 
-uint32_t utf16_offset_to_utf8(const char *s, uint32_t utf16_pos) {
+size_t utf16_offset_to_utf8(const char *utf8, size_t utf8_len,
-  uint32_t utf16_units = 0;
+                            size_t utf16_offset) {
-  uint32_t i = 0;
+  const char *start = utf8;
-  while (utf16_units < utf16_pos) {
+  const char *end = utf8 + utf8_len;
-    unsigned char c = s[i];
+  const char *it = start;
-    if ((c & 0x80) == 0x00) {
+  size_t utf16_count = 0;
-      i += 1;
+  while (it < end) {
-      utf16_units += 1;
+    if (utf16_count >= utf16_offset)
-    } else if ((c & 0xE0) == 0xC0) {
+      break;
-      i += 2;
+    const char *prev = it;
-      utf16_units += 1;
+    uint32_t codepoint = utf8::next(it, end);
-    } else if ((c & 0xF0) == 0xE0) {
+    if (codepoint <= 0xFFFF)
-      i += 3;
+      utf16_count += 1;
-      utf16_units += 1;
+    else
-    } else {
+      utf16_count += 2;
-      i += 4;
+    if (utf16_count > utf16_offset)
-      utf16_units += 2;
+      return prev - start;
   }
-  }
+  return it - start;
-  return i;
 }

themes/default.json

@@ -1,88 +0,0 @@
-{
-  "Default": {
-    "fg": "#EEEEEE"
-  },
-  "Shebang": {
-    "fg": "#7dcfff"
-  },
-  "Error": {
-    "fg": "#EF5168"
-  },
-  "Comment": {
-    "fg": "#AAAAAA",
-    "italic": true
-  },
-  "String": {
-    "fg": "#AAD94C"
-  },
-  "Escape": {
-    "fg": "#7dcfff"
-  },
-  "Interpolation": {
-    "fg": "#7dcfff"
-  },
-  "Regexp": {
-    "fg": "#D2A6FF"
-  },
-  "Number": {
-    "fg": "#E6C08A"
-  },
-  "True": {
-    "fg": "#7AE93C"
-  },
-  "False": {
-    "fg": "#EF5168"
-  },
-  "Char": {
-    "fg": "#FFAF70"
-  },
-  "Keyword": {
-    "fg": "#FF8F40"
-  },
-  "KeywordOperator": {
-    "fg": "#F07178"
-  },
-  "Operator": {
-    "fg": "#FFFFFF",
-    "italic": true
-  },
-  "Function": {
-    "fg": "#FFAF70"
-  },
-  "Type": {
-    "fg": "#F07178"
-  },
-  "Constant": {
-    "fg": "#7dcfff"
-  },
-  "VariableInstance": {
-    "fg": "#95E6CB"
-  },
-  "VariableGlobal": {
-    "fg": "#F07178"
-  },
-  "Annotation": {
-    "fg": "#7dcfff"
-  },
-  "Directive": {
-    "fg": "#FF8F40"
-  },
-  "Label": {
-    "fg": "#D2A6FF"
-  },
-  "Brace1": {
-    "fg": "#D2A6FF"
-  },
-  "Brace2": {
-    "fg": "#FFAFAF"
-  },
-  "Brace3": {
-    "fg": "#FFFF00"
-  },
-  "Brace4": {
-    "fg": "#0FFF0F"
-  },
-  "Brace5": {
-    "fg": "#FF0F0F"
-  }
-}