Rearrange code and cleanup

.clangd

@@ -0,0 +1,4 @@
+CompileFlags:
+  Add: [-I/home/syed/main/crib/include, -I/home/syed/main/crib/libs]
+  Remove: []
+  Compiler: clang++

.gitignore

@@ -7,7 +7,7 @@
 
 .vscode
 
-samples/t_*
+samples/tmp*
 
 build
 bin

Makefile

@@ -13,14 +13,15 @@ CCACHE := ccache
 CXX_DEBUG := $(CCACHE) g++
 CXX_RELEASE := $(CCACHE) clang++
 
-CFLAGS_DEBUG := -std=c++20 -Wall -Wextra -O0 -fno-inline -gsplit-dwarf -g -fsanitize=address -fno-omit-frame-pointer
+CFLAGS_DEBUG := -std=c++20 -Wall -Wextra -O0 -fno-inline -gsplit-dwarf -g -fsanitize=address -fno-omit-frame-pointer -I./include -I./libs
 CFLAGS_RELEASE := -std=c++20 -O3 -march=native -flto=thin \
 	-fno-exceptions -fno-rtti -fstrict-aliasing \
 	-ffast-math -funroll-loops \
 	-fvisibility=hidden \
 	-fomit-frame-pointer -DNDEBUG -s \
 	-mllvm -vectorize-loops \
-	-fno-unwind-tables -fno-asynchronous-unwind-tables
+	-fno-unwind-tables -fno-asynchronous-unwind-tables\
+	-I./include -I./libs
 
 PCH_CFLAGS_DEBUG := $(CFLAGS_DEBUG) -x c++-header
 PCH_CFLAGS_RELEASE := $(CFLAGS_RELEASE) -x c++-header
@@ -70,7 +71,7 @@ LIBS := \
 	$(MD_I_OBJ_SCANNER) \
 	-lpcre2-8 -lmagic
 
-SRC := $(wildcard $(SRC_DIR)/*.cc)
+SRC := $(wildcard $(SRC_DIR)/**/*.cc) $(wildcard $(SRC_DIR)/*.cc)
 OBJ_DEBUG := $(patsubst $(SRC_DIR)/%.cc,$(OBJ_DIR)/debug/%.o,$(SRC))
 OBJ_RELEASE := $(patsubst $(SRC_DIR)/%.cc,$(OBJ_DIR)/release/%.o,$(SRC))
 

README.md

@@ -6,7 +6,13 @@ A TUI IDE.
 
 # TODO
 
+- [ ] Locking knot while getting hex is not good idea . make knot true knot so i can copy it super fast with refcount on a pool of nodes.
+    - also edits are somehow still leaking . as in they are not applying properly for very quick edits.
+    - Look into ts_collect_spans and edit_insert (on large files)
+- [ ] Add strikethrough support
+- [ ] Add status bar & RUNNER mode
 - [ ] Fix indentation logic
+- [ ] Fix bug where closing immediately while lsp is loading hangs and then segfaults.
 - [ ] For `"insertTextFormat": 2` in `clangd` and similar use only the last word in the signature when replacing
 - [ ] Keep a list of words in the current buffer. (for auto completion) (maybe?)
 - [ ] Add ecma to js and make tsx

grammar/ruby.scm

@@ -293,7 +293,7 @@
 (pair
   ":" @punctuation.delimiter)
 
-;; #BFBDB6 #000000 0 0 0 1
+;; #BFBDB6 #000000 0 0 0 3
 [
   "("
   ")"

include/boxes/diagnostics.h

@@ -0,0 +1,20 @@
+#ifndef BOXES_DIAGNOSTICS_H
+#define BOXES_DIAGNOSTICS_H
+
+#include "editor/decl.h"
+#include "io/ui.h"
+#include "pch.h"
+#include "utils/utils.h"
+
+struct DiagnosticBox {
+  std::vector<VWarn> warnings;
+  std::vector<ScreenCell> cells;
+  uint32_t box_width;
+  uint32_t box_height;
+
+  void clear();
+  void render_first();
+  void render(Coord pos);
+};
+
+#endif

include/boxes/hover.h

@@ -1,11 +1,11 @@
-#ifndef HOVER_H
+#ifndef BOXES_HOVER_H
-#define HOVER_H
+#define BOXES_HOVER_H
 
-#include "./pch.h"
+#include "editor/decl.h"
-#include "./spans.h"
+#include "io/ui.h"
-#include "./ts_def.h"
+#include "pch.h"
-#include "./ui.h"
+#include "ts/decl.h"
-#include "./utils.h"
+#include "utils/utils.h"
 
 struct HoverBox {
   std::string text;
@@ -23,15 +23,4 @@ struct HoverBox {
   void render(Coord pos);
 };
 
-struct DiagnosticBox {
-  std::vector<VWarn> warnings;
-  std::vector<ScreenCell> cells;
-  uint32_t box_width;
-  uint32_t box_height;
-
-  void clear();
-  void render_first();
-  void render(Coord pos);
-};
-
 #endif

include/config.h

@@ -1,9 +1,9 @@
-#ifndef MAPS_H
+#ifndef CONFIG_H
-#define MAPS_H
+#define CONFIG_H
 
-#include "./lsp.h"
+#include "lsp/lsp.h"
-#include "./pch.h"
+#include "pch.h"
-#include "./ts_def.h"
+#include "ts/decl.h"
 
 static const std::unordered_map<uint8_t, LSP> kLsps = {
     {1,

include/editor/decl.h

@@ -0,0 +1,45 @@
+#ifndef EDITOR_DECL_H
+#define EDITOR_DECL_H
+
+#include "utils/utils.h"
+
+struct Fold {
+  uint32_t start;
+  uint32_t end;
+
+  bool contains(uint32_t line) const { return line >= start && line <= end; }
+  bool operator<(const Fold &other) const { return start < other.start; }
+};
+
+struct Span {
+  uint32_t start;
+  uint32_t end;
+  Highlight *hl;
+
+  bool operator<(const Span &other) const { return start < other.start; }
+};
+
+struct VWarn {
+  uint32_t line;
+  std::string text;
+  std::string text_full;
+  std::string source;
+  std::string code;
+  std::vector<std::string> see_also;
+  int8_t type;
+  uint32_t start;
+  uint32_t end{UINT32_MAX};
+
+  bool operator<(const VWarn &other) const { return line < other.line; }
+};
+
+struct VAI {
+  Coord pos;
+  char *text;
+  uint32_t len;
+  uint32_t lines; // number of \n in text for speed .. the ai part will not
+                  // line wrap but multiline ones need to have its own lines
+                  // after the first one
+};
+
+#endif

include/editor/editor.h

@@ -1,13 +1,13 @@
 #ifndef EDITOR_H
 #define EDITOR_H
 
-#include "./hover.h"
+#include "boxes/diagnostics.h"
-#include "./knot.h"
+#include "boxes/hover.h"
-#include "./pch.h"
+#include "editor/spans.h"
-#include "./spans.h"
+#include "io/knot.h"
-#include "./ts_def.h"
+#include "io/ui.h"
-#include "./ui.h"
+#include "ts/decl.h"
-#include "./utils.h"
+#include "utils/utils.h"
 
 #define CHAR 0
 #define WORD 1
@@ -51,66 +51,8 @@ struct Editor {
   int lsp_version = 1;
 };
 
-inline const Fold *fold_for_line(const std::vector<Fold> &folds,
-                                 uint32_t line) {
-  auto it = std::lower_bound(
-      folds.begin(), folds.end(), line,
-      [](const Fold &fold, uint32_t value) { return fold.start < value; });
-  if (it != folds.end() && it->start == line)
-    return &(*it);
-  if (it != folds.begin()) {
-    --it;
-    if (it->contains(line))
-      return &(*it);
-  }
-  return nullptr;
-}
-
-inline Fold *fold_for_line(std::vector<Fold> &folds, uint32_t line) {
-  const auto *fold =
-      fold_for_line(static_cast<const std::vector<Fold> &>(folds), line);
-  return const_cast<Fold *>(fold);
-}
-
-inline bool line_is_fold_start(const std::vector<Fold> &folds, uint32_t line) {
-  const Fold *fold = fold_for_line(folds, line);
-  return fold && fold->start == line;
-}
-
-inline bool line_is_folded(const std::vector<Fold> &folds, uint32_t line) {
-  return fold_for_line(folds, line) != nullptr;
-}
-
-inline uint32_t next_unfolded_row(const Editor *editor, uint32_t row) {
-  uint32_t limit = editor && editor->root ? editor->root->line_count : 0;
-  while (row < limit) {
-    const Fold *fold = fold_for_line(editor->folds, row);
-    if (!fold)
-      return row;
-    row = fold->end + 1;
-  }
-  return limit;
-}
-
-inline uint32_t prev_unfolded_row(const Editor *editor, uint32_t row) {
-  while (row > 0) {
-    const Fold *fold = fold_for_line(editor->folds, row);
-    if (!fold)
-      return row;
-    if (fold->start == 0)
-      return 0;
-    row = fold->start - 1;
-  }
-  return 0;
-}
-
-void apply_edit(std::vector<Span> &spans, uint32_t x, int64_t y);
-void apply_hook_insertion(Editor *editor, uint32_t line, uint32_t rows);
-void apply_hook_deletion(Editor *editor, uint32_t removal_start,
-                         uint32_t removal_end);
 Editor *new_editor(const char *filename_arg, Coord position, Coord size);
 void save_file(Editor *editor);
-void hover_diagnostic(Editor *editor);
 void free_editor(Editor *editor);
 void render_editor(Editor *editor);
 void fold(Editor *editor, uint32_t start_line, uint32_t end_line);
@@ -144,12 +86,22 @@ void word_boundaries_exclusive(Editor *editor, Coord coord, uint32_t *prev_col,
 std::vector<Fold>::iterator find_fold_iter(Editor *editor, uint32_t line);
 bool add_fold(Editor *editor, uint32_t start, uint32_t end);
 bool remove_fold(Editor *editor, uint32_t line);
-void apply_line_insertion(Editor *editor, uint32_t line, uint32_t rows);
-void apply_line_deletion(Editor *editor, uint32_t removal_start,
-                         uint32_t removal_end);
 uint32_t leading_indent(const char *line, uint32_t len);
 uint32_t get_indent(Editor *editor, Coord cursor);
 bool closing_after_cursor(const char *line, uint32_t len, uint32_t col);
 void editor_lsp_handle(Editor *editor, json msg);
 
+inline void apply_hook_insertion(Editor *editor, uint32_t line, uint32_t rows) {
+  for (auto &hook : editor->hooks)
+    if (hook > line)
+      hook += rows;
+}
+
+inline void apply_hook_deletion(Editor *editor, uint32_t removal_start,
+                                uint32_t removal_end) {
+  for (auto &hook : editor->hooks)
+    if (hook > removal_start)
+      hook -= removal_end - removal_start + 1;
+}
+
 #endif

include/editor/folds.h

@@ -0,0 +1,150 @@
+#ifndef EDITOR_FOLDS_H
+#define EDITOR_FOLDS_H
+
+#include "editor/editor.h"
+
+inline std::vector<Fold>::iterator find_fold_iter(Editor *editor,
+                                                  uint32_t line) {
+  auto &folds = editor->folds;
+  auto it = std::lower_bound(
+      folds.begin(), folds.end(), line,
+      [](const Fold &fold, uint32_t value) { return fold.start < value; });
+  if (it != folds.end() && it->start == line)
+    return it;
+  if (it != folds.begin()) {
+    --it;
+    if (it->contains(line))
+      return it;
+  }
+  return folds.end();
+}
+
+inline bool add_fold(Editor *editor, uint32_t start, uint32_t end) {
+  if (!editor || !editor->root)
+    return false;
+  if (start > end)
+    std::swap(start, end);
+  if (start >= editor->root->line_count)
+    return false;
+  end = std::min(end, editor->root->line_count - 1);
+  if (start == end)
+    return false;
+  Fold new_fold{start, end};
+  auto &folds = editor->folds;
+  auto it = std::lower_bound(
+      folds.begin(), folds.end(), new_fold.start,
+      [](const Fold &fold, uint32_t value) { return fold.start < value; });
+  if (it != folds.begin()) {
+    auto prev = std::prev(it);
+    if (prev->end + 1 >= new_fold.start) {
+      new_fold.start = std::min(new_fold.start, prev->start);
+      new_fold.end = std::max(new_fold.end, prev->end);
+      it = folds.erase(prev);
+    }
+  }
+  while (it != folds.end() && it->start <= new_fold.end + 1) {
+    new_fold.end = std::max(new_fold.end, it->end);
+    it = folds.erase(it);
+  }
+  folds.insert(it, new_fold);
+  return true;
+}
+
+inline bool remove_fold(Editor *editor, uint32_t line) {
+  auto it = find_fold_iter(editor, line);
+  if (it == editor->folds.end())
+    return false;
+  editor->folds.erase(it);
+  return true;
+}
+
+inline void apply_line_insertion(Editor *editor, uint32_t line, uint32_t rows) {
+  for (auto it = editor->folds.begin(); it != editor->folds.end();) {
+    if (line <= it->start) {
+      it->start += rows;
+      it->end += rows;
+      ++it;
+    } else if (line <= it->end) {
+      it = editor->folds.erase(it);
+    } else {
+      ++it;
+    }
+  }
+}
+
+inline void apply_line_deletion(Editor *editor, uint32_t removal_start,
+                                uint32_t removal_end) {
+  if (removal_start > removal_end)
+    return;
+  uint32_t rows_removed = removal_end - removal_start + 1;
+  std::vector<Fold> updated;
+  updated.reserve(editor->folds.size());
+  for (auto fold : editor->folds) {
+    if (removal_end < fold.start) {
+      fold.start -= rows_removed;
+      fold.end -= rows_removed;
+      updated.push_back(fold);
+      continue;
+    }
+    if (removal_start > fold.end) {
+      updated.push_back(fold);
+      continue;
+    }
+  }
+  editor->folds.swap(updated);
+}
+
+inline const Fold *fold_for_line(const std::vector<Fold> &folds,
+                                 uint32_t line) {
+  auto it = std::lower_bound(
+      folds.begin(), folds.end(), line,
+      [](const Fold &fold, uint32_t value) { return fold.start < value; });
+  if (it != folds.end() && it->start == line)
+    return &(*it);
+  if (it != folds.begin()) {
+    --it;
+    if (it->contains(line))
+      return &(*it);
+  }
+  return nullptr;
+}
+
+inline Fold *fold_for_line(std::vector<Fold> &folds, uint32_t line) {
+  const auto *fold =
+      fold_for_line(static_cast<const std::vector<Fold> &>(folds), line);
+  return const_cast<Fold *>(fold);
+}
+
+inline bool line_is_fold_start(const std::vector<Fold> &folds, uint32_t line) {
+  const Fold *fold = fold_for_line(folds, line);
+  return fold && fold->start == line;
+}
+
+inline bool line_is_folded(const std::vector<Fold> &folds, uint32_t line) {
+  return fold_for_line(folds, line) != nullptr;
+}
+
+inline uint32_t next_unfolded_row(const Editor *editor, uint32_t row) {
+  uint32_t limit = editor && editor->root ? editor->root->line_count : 0;
+  while (row < limit) {
+    const Fold *fold = fold_for_line(editor->folds, row);
+    if (!fold)
+      return row;
+    row = fold->end + 1;
+  }
+  return limit;
+}
+
+inline uint32_t prev_unfolded_row(const Editor *editor, uint32_t row) {
+  while (row > 0) {
+    const Fold *fold = fold_for_line(editor->folds, row);
+    if (!fold)
+      return row;
+    if (fold->start == 0)
+      return 0;
+    row = fold->start - 1;
+  }
+  return 0;
+}
+
+#endif

include/editor/spans.h

@@ -1,47 +1,8 @@
-#ifndef SPANS_H
+#ifndef EDITOR_SPANS_H
-#define SPANS_H
+#define EDITOR_SPANS_H
 
-#include "./pch.h"
+#include "editor/decl.h"
-#include "./utils.h"
+#include "utils/utils.h"
-
-struct VWarn {
-  uint32_t line;
-  std::string text;
-  std::string text_full;
-  std::string source;
-  std::string code;
-  std::vector<std::string> see_also;
-  int8_t type;
-  uint32_t start;
-  uint32_t end{UINT32_MAX};
-
-  bool operator<(const VWarn &other) const { return line < other.line; }
-};
-
-struct VAI {
-  Coord pos;
-  char *text;
-  uint32_t len;
-  uint32_t lines; // number of \n in text for speed .. the ai part will not
-                  // line wrap but multiline ones need to have its own lines
-                  // after the first one
-};
-
-struct Fold {
-  uint32_t start;
-  uint32_t end;
-
-  bool contains(uint32_t line) const { return line >= start && line <= end; }
-  bool operator<(const Fold &other) const { return start < other.start; }
-};
-
-struct Span {
-  uint32_t start;
-  uint32_t end;
-  Highlight *hl;
-
-  bool operator<(const Span &other) const { return start < other.start; }
-};
 
 struct Spans {
   std::vector<Span> spans;
@@ -98,4 +59,27 @@ struct SpanCursor {
   }
 };
 
+inline void apply_edit(std::vector<Span> &spans, uint32_t x, int64_t y) {
+  Span key{.start = x, .end = 0, .hl = nullptr};
+  auto it = std::lower_bound(
+      spans.begin(), spans.end(), key,
+      [](const Span &a, const Span &b) { return a.start < b.start; });
+  size_t idx = std::distance(spans.begin(), it);
+  while (idx > 0 && spans.at(idx - 1).end >= x)
+    --idx;
+  for (size_t i = idx; i < spans.size();) {
+    Span &s = spans.at(i);
+    if (s.start < x && s.end >= x) {
+      s.end += y;
+    } else if (s.start > x) {
+      s.start += y;
+      s.end += y;
+    }
+    if (s.end <= s.start)
+      spans.erase(spans.begin() + i);
+    else
+      ++i;
+  }
+}
+
 #endif

include/io/knot.h

@@ -1,13 +1,11 @@
 #ifndef ROPE_H
 #define ROPE_H
 
-#include "./pch.h"
+#include "pch.h"
-#include "./utils.h"
+#include "utils/utils.h"
 
 #define MIN_CHUNK_SIZE 64       // 64 Bytes
 #define MAX_CHUNK_SIZE 1024 * 8 // 8192 Bytes (8 KiB)
-#define MAX(a, b) ((a) > (b) ? (a) : (b))
-#define MIN(a, b) ((a) < (b) ? (a) : (b))
 #define DEPTH(n) ((n) ? (n)->depth : 0)
 
 // Rope node definition

include/io/ui.h

@@ -1,8 +1,8 @@
 #ifndef UI_H
 #define UI_H
 
-#include "./pch.h"
+#include "pch.h"
-#include "./utils.h"
+#include "utils/utils.h"
 
 #define KEY_CHAR 0
 #define KEY_SPECIAL 1

include/lsp/lsp.h

@@ -1,9 +1,9 @@
 #ifndef LSP_H
 #define LSP_H
 
-#include "./editor.h"
+#include "editor/editor.h"
-#include "./pch.h"
+#include "pch.h"
-#include "utils.h"
+#include "utils/utils.h"
 
 struct LSP {
   const char *command;
@@ -45,20 +45,37 @@ struct LSPInstance {
 
 extern std::shared_mutex active_lsps_mtx;
 extern std::unordered_map<uint8_t, std::shared_ptr<LSPInstance>> active_lsps;
+extern Queue<LSPOpenRequest> lsp_open_queue;
 
-void lsp_worker();
+static json client_capabilities = {
-void lsp_handle(std::shared_ptr<LSPInstance> lsp, json message);
+    {"textDocument",
-
+     {{"publishDiagnostics", {{"relatedInformation", true}}},
-std::shared_ptr<LSPInstance> get_or_init_lsp(uint8_t lsp_id);
+      {"hover", {{"contentFormat", {"markdown", "plaintext"}}}},
-void close_lsp(uint8_t lsp_id);
+      {"completion",
-
+       {{"completionItem",
-void request_add_to_lsp(Language language, Editor *editor);
+         {{"snippetSupport", true},
-void open_editor(std::shared_ptr<LSPInstance> lsp,
+          {"documentationFormat", {"markdown", "plaintext"}},
-                 std::pair<Language, Editor *> entry);
+          {"resolveSupport", {{"properties", {"documentation", "detail"}}}},
-void add_to_lsp(Language language, Editor *editor);
+          {"insertReplaceSupport", true},
-void remove_from_lsp(Editor *editor);
+          {"labelDetailsSupport", true},
+          {"insertTextModeSupport", {{"valueSet", {1}}}}}},
+        {"completionItemKind", {{"valueSet", {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}}}},
+        {"contextSupport", true},
+        {"insertTextMode", 1}}}}}};
 
 void lsp_send(std::shared_ptr<LSPInstance> lsp, json message,
               LSPPending *pending);
+void lsp_worker();
+
+std::shared_ptr<LSPInstance> get_or_init_lsp(uint8_t lsp_id);
+void clean_lsp(std::shared_ptr<LSPInstance> lsp, uint8_t lsp_id);
+void close_lsp(uint8_t lsp_id);
+
+void open_editor(std::shared_ptr<LSPInstance> lsp,
+                 std::pair<Language, Editor *> entry);
+void request_add_to_lsp(Language language, Editor *editor);
+void add_to_lsp(Language language, Editor *editor);
+void remove_from_lsp(Editor *editor);
+void lsp_handle(std::shared_ptr<LSPInstance> lsp, json message);
 
 #endif

include/main.h

@@ -1,7 +1,7 @@
 #ifndef MAIN_H
 #define MAIN_H
 
-#include "./pch.h"
+#include "pch.h"
 
 #define NORMAL 0
 #define INSERT 1

include/pch.h

@@ -4,17 +4,23 @@
 #define PCRE2_CODE_UNIT_WIDTH 8
 #define PCRE_WORKSPACE_SIZE 512
 
-#include "../libs/tree-sitter/lib/include/tree_sitter/api.h"
+extern "C" {
+#include "libgrapheme/grapheme.h"
+#include "unicode_width/unicode_width.h"
+}
+#include "tree-sitter/lib/include/tree_sitter/api.h"
 #include <algorithm>
 #include <atomic>
 #include <cctype>
 #include <chrono>
+#include <cmath>
 #include <cstdarg>
 #include <cstdint>
 #include <cstdio>
 #include <cstdlib>
 #include <cstring>
 #include <deque>
+#include <fcntl.h>
 #include <filesystem>
 #include <fstream>
 #include <functional>
@@ -27,9 +33,13 @@
 #include <pcre2.h>
 #include <queue>
 #include <shared_mutex>
+#include <signal.h>
 #include <string.h>
 #include <string>
 #include <sys/ioctl.h>
+#include <sys/poll.h>
+#include <sys/types.h>
+#include <sys/wait.h>
 #include <termios.h>
 #include <thread>
 #include <unistd.h>

include/ts/decl.h

@@ -1,7 +1,7 @@
-#ifndef TS_DEF_H
+#ifndef TS_DECL_H
-#define TS_DEF_H
+#define TS_DECL_H
 
-#include "./pch.h"
+#include "pch.h"
 
 #define LANG(name) tree_sitter_##name
 #define TS_DEF(name) extern "C" const TSLanguage *LANG(name)()

include/ts/ts.h

@@ -1,9 +1,9 @@
 #ifndef TS_H
 #define TS_H
 
-#include "./editor.h"
+#include "editor/editor.h"
-#include "./pch.h"
+#include "pch.h"
-#include "./utils.h"
+#include "utils/utils.h"
 
 #define HEX(s) (static_cast<uint32_t>(std::stoul(s, nullptr, 16)))
 
@@ -14,12 +14,5 @@ void ts_collect_spans(Editor *editor);
 bool ts_predicate(TSQuery *query, const TSQueryMatch &match,
                   std::function<std::string(const TSNode *)> subject_fn);
 void clear_regex_cache();
-template <typename T>
-inline T *safe_get(std::map<uint16_t, T> &m, uint16_t key) {
-  auto it = m.find(key);
-  if (it == m.end())
-    return nullptr;
-  return &it->second;
-}
 
 #endif

include/utils/utils.h

@@ -1,8 +1,8 @@
 #ifndef UTILS_H
 #define UTILS_H
 
-#include "./pch.h"
+#include "pch.h"
-#include "./ts_def.h"
+#include "ts/decl.h"
 
 template <typename T> struct Queue {
   std::queue<T> q;
@@ -59,28 +59,41 @@ struct Match {
   std::string text;
 };
 
-std::vector<Match> find_all_matches(const std::string &subject,
+#define MAX(a, b) ((a) > (b) ? (a) : (b))
-                                    const std::string &pattern);
+#define MIN(a, b) ((a) < (b) ? (a) : (b))
+
 std::string clean_text(const std::string &input);
 std::string percent_encode(const std::string &s);
 std::string percent_decode(const std::string &s);
-std::string path_abs(const std::string &path_str);
+uint32_t count_clusters(const char *line, size_t len, size_t from, size_t to);
-std::string path_to_file_uri(const std::string &path_str);
+std::string trim(const std::string &s);
+
 int display_width(const char *str, size_t len);
 uint32_t get_visual_col_from_bytes(const char *line, uint32_t len,
                                    uint32_t byte_limit);
 uint32_t get_bytes_from_visual_col(const char *line, uint32_t len,
                                    uint32_t target_visual_col);
+int utf8_byte_offset_to_utf16(const char *s, size_t byte_pos);
+
 void log(const char *fmt, ...);
+
+std::string path_abs(const std::string &path_str);
+std::string path_to_file_uri(const std::string &path_str);
 std::string get_exe_dir();
 char *load_file(const char *path, uint32_t *out_len);
 char *detect_file_type(const char *filename);
-int utf8_byte_offset_to_utf16(const char *s, size_t byte_pos);
 Language language_for_file(const char *filename);
+
 void copy_to_clipboard(const char *text, size_t len);
 char *get_from_clipboard(uint32_t *out_len);
-uint32_t count_clusters(const char *line, size_t len, size_t from, size_t to);
+
-std::string trim(const std::string &s);
+template <typename T>
+inline T *safe_get(std::map<uint16_t, T> &m, uint16_t key) {
+  auto it = m.find(key);
+  if (it == m.end())
+    return nullptr;
+  return &it->second;
+}
 
 template <typename Func, typename... Args>
 auto throttle(std::chrono::milliseconds min_duration, Func &&func,

src/boxes/diagnostics.cc

@@ -0,0 +1,150 @@
+#include "boxes/diagnostics.h"
+
+void DiagnosticBox::clear() {
+  warnings.clear();
+  cells.clear();
+  box_width = 0;
+  box_height = 0;
+}
+
+void DiagnosticBox::render_first() {
+  if (warnings.empty())
+    return;
+  uint32_t longest_line = 8 + warnings[0].source.length();
+  for (auto &warn : warnings) {
+    longest_line = MAX(longest_line, (uint32_t)warn.text.length() + 7);
+    longest_line = MAX(longest_line, (uint32_t)warn.code.length() + 4);
+    for (auto &see_also : warn.see_also)
+      longest_line = MAX(longest_line, (uint32_t)see_also.length() + 4);
+  }
+  uint32_t content_width = MIN(longest_line, 150u);
+  box_width = content_width + 2;
+  cells.assign(box_width * 25, {" ", 0, 0, 0, 0, 0});
+  auto set = [&](uint32_t r, uint32_t c, const char *text, uint32_t fg,
+                 uint32_t bg, uint8_t flags) {
+    cells[r * box_width + c] = {std::string(text), 0, fg, bg, flags, 0};
+  };
+  uint32_t base_bg = 0;
+  uint32_t border_fg = 0x82AAFF;
+  uint32_t r = 0;
+  if (warnings[0].source != "") {
+    std::string src_txt = "Source: ";
+    for (uint32_t i = 0; i < src_txt.length() && i < content_width; i++)
+      set(1, i + 1, (char[2]){src_txt[i], 0}, 0x3EAAFF, base_bg, 0);
+    for (uint32_t i = 0; i < warnings[0].source.length() && i < content_width;
+         i++)
+      set(1, i + 1 + src_txt.length(), (char[2]){warnings[0].source[i], 0},
+          0xffffff, base_bg, 0);
+    r++;
+  }
+  int idx = 1;
+  for (auto &warn : warnings) {
+    char buf[4];
+    std::snprintf(buf, sizeof(buf), "%2d", idx % 100);
+    std::string line_txt = std::string(buf) + ". ";
+    for (uint32_t i = 0; i < line_txt.length(); i++)
+      set(r + 1, i + 1, (char[2]){line_txt[i], 0}, 0xffffff, base_bg, 0);
+    if (r >= 23)
+      break;
+    const char *err_sym = "";
+    uint32_t c_sym = 0xAAAAAA;
+    switch (warn.type) {
+    case 1:
+      err_sym = "";
+      c_sym = 0xFF0000;
+      break;
+    case 2:
+      err_sym = "";
+      c_sym = 0xFFFF00;
+      break;
+    case 3:
+      err_sym = "";
+      c_sym = 0xFF00FF;
+      break;
+    case 4:
+      err_sym = "";
+      c_sym = 0xAAAAAA;
+      break;
+    }
+    std::string text = warn.text_full + " " + err_sym;
+    uint32_t i = 0;
+    while (i < text.length() && r < 23) {
+      uint32_t c = 4;
+      while (c < content_width && i < text.length()) {
+        if (text[i] == '\n') {
+          while (i < text.length() && text[i] == '\n')
+            i++;
+          break;
+        }
+        uint32_t cluster_len = grapheme_next_character_break_utf8(
+            text.c_str() + i, text.length() - i);
+        std::string cluster = text.substr(i, cluster_len);
+        int width = display_width(cluster.c_str(), cluster_len);
+        if (c + width > content_width)
+          break;
+        set(r + 1, c + 1, cluster.c_str(), c_sym, base_bg, 0);
+        c += width;
+        i += cluster_len;
+        for (int w = 1; w < width; w++)
+          set(r + 1, c - w + 1, "\x1b", c_sym, base_bg, 0);
+      }
+      r++;
+    }
+    if (r >= 23)
+      break;
+    if (warn.code != "") {
+      for (uint32_t i = 0; i < warn.code.length() && i + 5 < content_width; i++)
+        set(r + 1, i + 5, (char[2]){warn.code[i], 0}, 0x81cdc6, base_bg, 0);
+      r++;
+    }
+    if (r >= 23)
+      break;
+    for (std::string &see_also : warn.see_also) {
+      uint32_t fg = 0xB55EFF;
+      uint8_t colon_count = 0;
+      for (uint32_t i = 0; i < see_also.length() && i + 5 < content_width;
+           i++) {
+        set(r + 1, i + 5, (char[2]){see_also[i], 0}, fg, base_bg, 0);
+        if (see_also[i] == ':')
+          colon_count++;
+        if (colon_count == 2)
+          fg = 0xFFFFFF;
+      }
+      r++;
+      if (r >= 23)
+        break;
+    };
+    idx++;
+  }
+  box_height = 2 + r;
+  set(0, 0, "┌", border_fg, base_bg, 0);
+  for (uint32_t i = 1; i < box_width - 1; i++)
+    set(0, i, "─", border_fg, base_bg, 0);
+  set(0, box_width - 1, "┐", border_fg, base_bg, 0);
+  for (uint32_t r = 1; r < box_height - 1; r++) {
+    set(r, 0, "│", border_fg, base_bg, 0);
+    set(r, box_width - 1, "│", border_fg, base_bg, 0);
+  }
+  set(box_height - 1, 0, "└", border_fg, base_bg, 0);
+  for (uint32_t i = 1; i < box_width - 1; i++)
+    set(box_height - 1, i, "─", border_fg, base_bg, 0);
+  set(box_height - 1, box_width - 1, "┘", border_fg, base_bg, 0);
+  cells.resize(box_width * box_height);
+}
+
+void DiagnosticBox::render(Coord pos) {
+  int32_t start_row = (int32_t)pos.row - (int32_t)box_height;
+  if (start_row < 0)
+    start_row = pos.row + 1;
+  int32_t start_col = pos.col;
+  if (start_col + box_width > cols) {
+    start_col = cols - box_width;
+    if (start_col < 0)
+      start_col = 0;
+  }
+  for (uint32_t r = 0; r < box_height; r++)
+    for (uint32_t c = 0; c < box_width; c++)
+      update(start_row + r, start_col + c, cells[r * box_width + c].utf8,
+             cells[r * box_width + c].fg, cells[r * box_width + c].bg,
+             cells[r * box_width + c].flags);
+}

src/boxes/hover.cc

@@ -1,9 +1,5 @@
-extern "C" {
+#include "boxes/hover.h"
-#include "../libs/libgrapheme/grapheme.h"
+#include "ts/ts.h"
-}
-#include "../include/hover.h"
-#include "../include/ts.h"
-#include "../include/ui.h"
 
 void HoverBox::clear() {
   text = "";
@@ -222,152 +218,3 @@ void HoverBox::render(Coord pos) {
              cells[r * box_width + c].fg, cells[r * box_width + c].bg,
              cells[r * box_width + c].flags);
 }
-
-void DiagnosticBox::clear() {
-  warnings.clear();
-  cells.clear();
-  box_width = 0;
-  box_height = 0;
-}
-
-void DiagnosticBox::render_first() {
-  if (warnings.empty())
-    return;
-  uint32_t longest_line = 8 + warnings[0].source.length();
-  for (auto &warn : warnings) {
-    longest_line = MAX(longest_line, (uint32_t)warn.text.length() + 7);
-    longest_line = MAX(longest_line, (uint32_t)warn.code.length() + 4);
-    for (auto &see_also : warn.see_also)
-      longest_line = MAX(longest_line, (uint32_t)see_also.length() + 4);
-  }
-  uint32_t content_width = MIN(longest_line, 150u);
-  box_width = content_width + 2;
-  cells.assign(box_width * 25, {" ", 0, 0, 0, 0, 0});
-  auto set = [&](uint32_t r, uint32_t c, const char *text, uint32_t fg,
-                 uint32_t bg, uint8_t flags) {
-    cells[r * box_width + c] = {std::string(text), 0, fg, bg, flags, 0};
-  };
-  uint32_t base_bg = 0;
-  uint32_t border_fg = 0x82AAFF;
-  uint32_t r = 0;
-  if (warnings[0].source != "") {
-    std::string src_txt = "Source: ";
-    for (uint32_t i = 0; i < src_txt.length() && i < content_width; i++)
-      set(1, i + 1, (char[2]){src_txt[i], 0}, 0x3EAAFF, base_bg, 0);
-    for (uint32_t i = 0; i < warnings[0].source.length() && i < content_width;
-         i++)
-      set(1, i + 1 + src_txt.length(), (char[2]){warnings[0].source[i], 0},
-          0xffffff, base_bg, 0);
-    r++;
-  }
-  int idx = 1;
-  for (auto &warn : warnings) {
-    char buf[4];
-    std::snprintf(buf, sizeof(buf), "%2d", idx % 100);
-    std::string line_txt = std::string(buf) + ". ";
-    for (uint32_t i = 0; i < line_txt.length(); i++)
-      set(r + 1, i + 1, (char[2]){line_txt[i], 0}, 0xffffff, base_bg, 0);
-    if (r >= 23)
-      break;
-    const char *err_sym = "";
-    uint32_t c_sym = 0xAAAAAA;
-    switch (warn.type) {
-    case 1:
-      err_sym = "";
-      c_sym = 0xFF0000;
-      break;
-    case 2:
-      err_sym = "";
-      c_sym = 0xFFFF00;
-      break;
-    case 3:
-      err_sym = "";
-      c_sym = 0xFF00FF;
-      break;
-    case 4:
-      err_sym = "";
-      c_sym = 0xAAAAAA;
-      break;
-    }
-    std::string text = warn.text_full + " " + err_sym;
-    uint32_t i = 0;
-    while (i < text.length() && r < 23) {
-      uint32_t c = 4;
-      while (c < content_width && i < text.length()) {
-        if (text[i] == '\n') {
-          while (i < text.length() && text[i] == '\n')
-            i++;
-          break;
-        }
-        uint32_t cluster_len = grapheme_next_character_break_utf8(
-            text.c_str() + i, text.length() - i);
-        std::string cluster = text.substr(i, cluster_len);
-        int width = display_width(cluster.c_str(), cluster_len);
-        if (c + width > content_width)
-          break;
-        set(r + 1, c + 1, cluster.c_str(), c_sym, base_bg, 0);
-        c += width;
-        i += cluster_len;
-        for (int w = 1; w < width; w++)
-          set(r + 1, c - w + 1, "\x1b", c_sym, base_bg, 0);
-      }
-      r++;
-    }
-    if (r >= 23)
-      break;
-    if (warn.code != "") {
-      for (uint32_t i = 0; i < warn.code.length() && i + 5 < content_width; i++)
-        set(r + 1, i + 5, (char[2]){warn.code[i], 0}, 0x81cdc6, base_bg, 0);
-      r++;
-    }
-    if (r >= 23)
-      break;
-    for (std::string &see_also : warn.see_also) {
-      uint32_t fg = 0xB55EFF;
-      uint8_t colon_count = 0;
-      for (uint32_t i = 0; i < see_also.length() && i + 5 < content_width;
-           i++) {
-        set(r + 1, i + 5, (char[2]){see_also[i], 0}, fg, base_bg, 0);
-        if (see_also[i] == ':')
-          colon_count++;
-        if (colon_count == 2)
-          fg = 0xFFFFFF;
-      }
-      r++;
-      if (r >= 23)
-        break;
-    };
-    idx++;
-  }
-  box_height = 2 + r;
-  set(0, 0, "┌", border_fg, base_bg, 0);
-  for (uint32_t i = 1; i < box_width - 1; i++)
-    set(0, i, "─", border_fg, base_bg, 0);
-  set(0, box_width - 1, "┐", border_fg, base_bg, 0);
-  for (uint32_t r = 1; r < box_height - 1; r++) {
-    set(r, 0, "│", border_fg, base_bg, 0);
-    set(r, box_width - 1, "│", border_fg, base_bg, 0);
-  }
-  set(box_height - 1, 0, "└", border_fg, base_bg, 0);
-  for (uint32_t i = 1; i < box_width - 1; i++)
-    set(box_height - 1, i, "─", border_fg, base_bg, 0);
-  set(box_height - 1, box_width - 1, "┘", border_fg, base_bg, 0);
-  cells.resize(box_width * box_height);
-}
-
-void DiagnosticBox::render(Coord pos) {
-  int32_t start_row = (int32_t)pos.row - (int32_t)box_height;
-  if (start_row < 0)
-    start_row = pos.row + 1;
-  int32_t start_col = pos.col;
-  if (start_col + box_width > cols) {
-    start_col = cols - box_width;
-    if (start_col < 0)
-      start_col = 0;
-  }
-  for (uint32_t r = 0; r < box_height; r++)
-    for (uint32_t c = 0; c < box_width; c++)
-      update(start_row + r, start_col + c, cells[r * box_width + c].utf8,
-             cells[r * box_width + c].fg, cells[r * box_width + c].bg,
-             cells[r * box_width + c].flags);
-}

src/editor/adjustment.cc

@@ -1,241 +1,5 @@
-extern "C" {
+#include "editor/editor.h"
-#include "../libs/libgrapheme/grapheme.h"
+#include "editor/folds.h"
-}
-#include "../include/editor.h"
-#include "../include/utils.h"
-#include <cmath>
-
-void scroll_up(Editor *editor, int32_t number) {
-  if (!editor || number == 0)
-    return;
-  uint32_t numlen =
-      EXTRA_META + static_cast<int>(std::log10(editor->root->line_count + 1));
-  uint32_t render_width = editor->size.col - numlen;
-  uint32_t line_index = editor->scroll.row;
-  LineIterator *it = begin_l_iter(editor->root, line_index);
-  if (!it)
-    return;
-  uint32_t len;
-  char *line = next_line(it, &len);
-  if (!line) {
-    free(it->buffer);
-    free(it);
-    return;
-  }
-  if (len > 0 && line[len - 1] == '\n')
-    len--;
-  uint32_t current_byte_offset = 0;
-  uint32_t col = 0;
-  std::vector<uint32_t> segment_starts;
-  segment_starts.reserve(16);
-  if (current_byte_offset < editor->scroll.col)
-    segment_starts.push_back(0);
-  while (current_byte_offset < editor->scroll.col &&
-         current_byte_offset < len) {
-    uint32_t cluster_len = grapheme_next_character_break_utf8(
-        line + current_byte_offset, len - current_byte_offset);
-    int width = display_width(line + current_byte_offset, cluster_len);
-    if (col + width > render_width) {
-      segment_starts.push_back(current_byte_offset);
-      col = 0;
-    }
-    current_byte_offset += cluster_len;
-    col += width;
-  }
-  for (auto it_seg = segment_starts.rbegin(); it_seg != segment_starts.rend();
-       ++it_seg) {
-    if (--number == 0) {
-      editor->scroll = {line_index, *it_seg};
-      free(it->buffer);
-      free(it);
-      return;
-    }
-  }
-  line = prev_line(it, &len);
-  if (!line) {
-    editor->scroll = {0, 0};
-    free(it->buffer);
-    free(it);
-    return;
-  }
-  do {
-    line_index--;
-    line = prev_line(it, &len);
-    if (!line) {
-      editor->scroll = {0, 0};
-      free(it->buffer);
-      free(it);
-      return;
-    }
-    const Fold *fold = fold_for_line(editor->folds, line_index);
-    if (fold) {
-      while (line && line_index > fold->start) {
-        free(line);
-        line = prev_line(it, &len);
-        line_index--;
-        if (!line) {
-          editor->scroll = {0, 0};
-          free(it->buffer);
-          free(it);
-          return;
-        }
-      }
-      if (--number == 0) {
-        editor->scroll = {fold->start, 0};
-        free(it->buffer);
-        free(it);
-        return;
-      }
-      if (fold->start == 0) {
-        editor->scroll = {0, 0};
-        free(it->buffer);
-        free(it);
-        return;
-      }
-      line_index = fold->start - 1;
-      line = prev_line(it, &len);
-      if (!line) {
-        editor->scroll = {0, 0};
-        free(it->buffer);
-        free(it);
-        return;
-      }
-      continue;
-    }
-    if (len > 0 && line[len - 1] == '\n')
-      len--;
-    current_byte_offset = 0;
-    col = 0;
-    std::vector<uint32_t> segment_starts;
-    segment_starts.reserve(16);
-    segment_starts.push_back(0);
-    while (current_byte_offset < len) {
-      uint32_t cluster_len = grapheme_next_character_break_utf8(
-          line + current_byte_offset, len - current_byte_offset);
-      int width = display_width(line + current_byte_offset, cluster_len);
-      if (col + width > render_width) {
-        segment_starts.push_back(current_byte_offset);
-        col = 0;
-      }
-      current_byte_offset += cluster_len;
-      col += width;
-    }
-    for (auto it_seg = segment_starts.rbegin(); it_seg != segment_starts.rend();
-         ++it_seg) {
-      if (--number == 0) {
-        editor->scroll = {line_index, *it_seg};
-        free(it->buffer);
-        free(it);
-        return;
-      }
-    }
-  } while (number > 0);
-  free(it->buffer);
-  free(it);
-}
-
-void scroll_down(Editor *editor, uint32_t number) {
-  if (!editor || number == 0)
-    return;
-  uint32_t numlen =
-      EXTRA_META + static_cast<int>(std::log10(editor->root->line_count + 1));
-  uint32_t render_width = editor->size.col - numlen;
-  uint32_t line_index = editor->scroll.row;
-  LineIterator *it = begin_l_iter(editor->root, line_index);
-  if (!it)
-    return;
-  const uint32_t max_visual_lines = editor->size.row;
-  Coord *scroll_queue = (Coord *)malloc(sizeof(Coord) * max_visual_lines);
-  uint32_t q_head = 0;
-  uint32_t q_size = 0;
-  uint32_t visual_seen = 0;
-  bool first_visual_line = true;
-  while (true) {
-    const Fold *fold = fold_for_line(editor->folds, line_index);
-    if (fold) {
-      Coord fold_coord = {fold->start, 0};
-      if (q_size < max_visual_lines) {
-        scroll_queue[(q_head + q_size) % max_visual_lines] = fold_coord;
-        q_size++;
-      } else {
-        scroll_queue[q_head] = fold_coord;
-        q_head = (q_head + 1) % max_visual_lines;
-      }
-      visual_seen++;
-      if (visual_seen >= number + max_visual_lines) {
-        editor->scroll = scroll_queue[q_head];
-        break;
-      }
-      uint32_t skip_until = fold->end;
-      while (line_index <= skip_until) {
-        char *line = next_line(it, nullptr);
-        if (!line) {
-          free(scroll_queue);
-          free(it->buffer);
-          free(it);
-          return;
-        }
-        line_index++;
-      }
-      continue;
-    }
-    uint32_t line_len;
-    char *line = next_line(it, &line_len);
-    if (!line)
-      break;
-    if (line_len && line[line_len - 1] == '\n')
-      line_len--;
-    uint32_t current_byte_offset = 0;
-    if (first_visual_line) {
-      current_byte_offset += editor->scroll.col;
-      first_visual_line = false;
-    }
-    while (current_byte_offset < line_len ||
-           (line_len == 0 && current_byte_offset == 0)) {
-      Coord coord = {line_index, current_byte_offset};
-      if (q_size < max_visual_lines) {
-        scroll_queue[(q_head + q_size) % max_visual_lines] = coord;
-        q_size++;
-      } else {
-        scroll_queue[q_head] = coord;
-        q_head = (q_head + 1) % max_visual_lines;
-      }
-      visual_seen++;
-      if (visual_seen >= number + max_visual_lines) {
-        editor->scroll = scroll_queue[q_head];
-        free(scroll_queue);
-        free(it->buffer);
-        free(it);
-        return;
-      }
-      uint32_t col = 0;
-      uint32_t local_render_offset = 0;
-      uint32_t left = line_len - current_byte_offset;
-      while (left > 0 && col < render_width) {
-        uint32_t cluster_len = grapheme_next_character_break_utf8(
-            line + current_byte_offset + local_render_offset, left);
-        int width = display_width(
-            line + current_byte_offset + local_render_offset, cluster_len);
-        if (col + width > render_width)
-          break;
-        local_render_offset += cluster_len;
-        left -= cluster_len;
-        col += width;
-      }
-      current_byte_offset += local_render_offset;
-      if (line_len == 0)
-        break;
-    }
-    line_index++;
-  }
-  if (q_size > 0) {
-    uint32_t advance = (q_size > number) ? number : (q_size - 1);
-    editor->scroll = scroll_queue[(q_head + advance) % max_visual_lines];
-  }
-  free(it->buffer);
-  free(it);
-  free(scroll_queue);
-}
 
 void ensure_cursor(Editor *editor) {
   std::shared_lock knot_lock(editor->knot_mtx);

src/editor/boundaries.cc

@@ -0,0 +1,115 @@
+#include "editor/editor.h"
+
+uint32_t scan_left(const char *line, uint32_t len, uint32_t off) {
+  if (off > len)
+    off = len;
+  uint32_t i = off;
+  while (i > 0) {
+    unsigned char c = (unsigned char)line[i - 1];
+    if ((c & 0x80) != 0)
+      break;
+    if (!((c == '_') || (c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z') ||
+          (c >= 'a' && c <= 'z')))
+      break;
+    --i;
+  }
+  return i;
+}
+
+uint32_t scan_right(const char *line, uint32_t len, uint32_t off) {
+  if (off > len)
+    off = len;
+  uint32_t i = off;
+  while (i < len) {
+    unsigned char c = (unsigned char)line[i];
+    if ((c & 0x80) != 0)
+      break;
+    if (!((c == '_') || (c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z') ||
+          (c >= 'a' && c <= 'z')))
+      break;
+    ++i;
+  }
+  return i;
+}
+
+void word_boundaries_exclusive(Editor *editor, Coord coord, uint32_t *prev_col,
+                               uint32_t *next_col) {
+  if (!editor)
+    return;
+  std::shared_lock lock(editor->knot_mtx);
+  LineIterator *it = begin_l_iter(editor->root, coord.row);
+  if (!it)
+    return;
+  uint32_t line_len;
+  char *line = next_line(it, &line_len);
+  if (!line)
+    return;
+  if (line_len && line[line_len - 1] == '\n')
+    line_len--;
+  uint32_t col = coord.col;
+  if (col > line_len)
+    col = line_len;
+  uint32_t left = scan_left(line, line_len, col);
+  uint32_t right = scan_right(line, line_len, col);
+  if (prev_col)
+    *prev_col = left;
+  if (next_col)
+    *next_col = right;
+  free(it->buffer);
+  free(it);
+}
+
+uint32_t word_jump_right(const char *line, size_t len, uint32_t pos) {
+  if (pos >= len)
+    return len;
+  size_t next = grapheme_next_word_break_utf8(line + pos, len - pos);
+  return static_cast<uint32_t>(pos + next);
+}
+
+uint32_t word_jump_left(const char *line, size_t len, uint32_t col) {
+  if (col == 0)
+    return 0;
+  size_t pos = 0;
+  size_t last = 0;
+  size_t cursor = col;
+  while (pos < len) {
+    size_t next = pos + grapheme_next_word_break_utf8(line + pos, len - pos);
+    if (next >= cursor)
+      break;
+    last = next;
+    pos = next;
+  }
+  return static_cast<uint32_t>(last);
+}
+
+void word_boundaries(Editor *editor, Coord coord, uint32_t *prev_col,
+                     uint32_t *next_col, uint32_t *prev_clusters,
+                     uint32_t *next_clusters) {
+  if (!editor)
+    return;
+  std::shared_lock lock(editor->knot_mtx);
+  LineIterator *it = begin_l_iter(editor->root, coord.row);
+  if (!it)
+    return;
+  uint32_t line_len;
+  char *line = next_line(it, &line_len);
+  if (!line)
+    return;
+  if (line_len && line[line_len - 1] == '\n')
+    line_len--;
+  size_t col = coord.col;
+  if (col > line_len)
+    col = line_len;
+  size_t left = word_jump_left(line, line_len, col);
+  size_t right = word_jump_right(line, line_len, col);
+  if (prev_col)
+    *prev_col = static_cast<uint32_t>(left);
+  if (next_col)
+    *next_col = static_cast<uint32_t>(right);
+  if (prev_clusters)
+    *prev_clusters = count_clusters(line, line_len, left, col);
+  if (next_clusters)
+    *next_clusters = count_clusters(line, line_len, col, right);
+  free(it->buffer);
+  free(it);
+}

src/editor/click.cc

@@ -0,0 +1,97 @@
+#include "editor/editor.h"
+#include "editor/folds.h"
+#include "main.h"
+
+Coord editor_hit_test(Editor *editor, uint32_t x, uint32_t y) {
+  if (mode == INSERT)
+    x++;
+  uint32_t numlen =
+      EXTRA_META + static_cast<int>(std::log10(editor->root->line_count + 1));
+  bool is_gutter_click = (x < numlen);
+  uint32_t render_width = editor->size.col - numlen;
+  x = MAX(x, numlen) - numlen;
+  uint32_t target_visual_row = y;
+  uint32_t visual_row = 0;
+  uint32_t line_index = editor->scroll.row;
+  uint32_t last_line_index = editor->scroll.row;
+  uint32_t last_col = editor->scroll.col;
+  bool first_visual_line = true;
+  std::shared_lock knot_lock(editor->knot_mtx);
+  LineIterator *it = begin_l_iter(editor->root, line_index);
+  if (!it)
+    return editor->scroll;
+  while (visual_row <= target_visual_row) {
+    const Fold *fold = fold_for_line(editor->folds, line_index);
+    if (fold) {
+      if (visual_row == target_visual_row) {
+        free(it->buffer);
+        free(it);
+        if (is_gutter_click) {
+          remove_fold(editor, fold->start);
+          return {UINT32_MAX, UINT32_MAX};
+        }
+        return {fold->start > 0 ? fold->start - 1 : 0, 0};
+      }
+      visual_row++;
+      while (line_index <= fold->end) {
+        char *l = next_line(it, nullptr);
+        if (!l)
+          break;
+        line_index++;
+      }
+      last_line_index = fold->end;
+      last_col = 0;
+      continue;
+    }
+    uint32_t line_len;
+    char *line = next_line(it, &line_len);
+    if (!line)
+      break;
+    if (line_len && line[line_len - 1] == '\n')
+      line_len--;
+    last_line_index = line_index;
+    last_col = line_len;
+    uint32_t offset = first_visual_line ? editor->scroll.col : 0;
+    first_visual_line = false;
+    while (offset < line_len || (line_len == 0 && offset == 0)) {
+      uint32_t col = 0;
+      uint32_t advance = 0;
+      uint32_t left = line_len - offset;
+      uint32_t last_good_offset = offset;
+      while (left > 0 && col < render_width) {
+        uint32_t g =
+            grapheme_next_character_break_utf8(line + offset + advance, left);
+        int w = display_width(line + offset + advance, g);
+        if (col + w > render_width)
+          break;
+        if (visual_row == target_visual_row && x < col + w) {
+          free(it->buffer);
+          free(it);
+          return {line_index, offset + advance};
+        }
+        advance += g;
+        last_good_offset = offset + advance;
+        left -= g;
+        col += w;
+      }
+      last_col = last_good_offset;
+      if (visual_row == target_visual_row) {
+        free(it->buffer);
+        free(it);
+        return {line_index, last_good_offset};
+      }
+      visual_row++;
+      if (visual_row > target_visual_row)
+        break;
+      if (advance == 0)
+        break;
+      offset += advance;
+      if (line_len == 0)
+        break;
+    }
+    line_index++;
+  }
+  free(it->buffer);
+  free(it);
+  return {last_line_index, last_col};
+}

src/editor/cursor.cc

@@ -1,8 +1,6 @@
-extern "C" {
+#include "editor/editor.h"
-#include "../libs/libgrapheme/grapheme.h"
+#include "editor/folds.h"
-}
+#include "utils/utils.h"
-#include "../include/editor.h"
-#include "../include/utils.h"
 
 Coord move_right_pure(Editor *editor, Coord cursor, uint32_t number) {
   Coord result = cursor;

src/editor/edit.cc

@@ -0,0 +1,274 @@
+#include "editor/editor.h"
+#include "editor/folds.h"
+#include "lsp/lsp.h"
+
+void edit_erase(Editor *editor, Coord pos, int64_t len) {
+  if (len == 0)
+    return;
+  if (len < 0) {
+    std::shared_lock lock_1(editor->knot_mtx);
+    uint32_t cursor_original =
+        line_to_byte(editor->root, editor->cursor.row, nullptr) +
+        editor->cursor.col;
+    TSPoint old_point = {pos.row, pos.col};
+    uint32_t byte_pos = line_to_byte(editor->root, pos.row, nullptr) + pos.col;
+    Coord point = move_left_pure(editor, pos, -len);
+    json lsp_range;
+    bool do_lsp = (editor->lsp != nullptr);
+    if (do_lsp) {
+      LineIterator *it = begin_l_iter(editor->root, point.row);
+      char *line = next_line(it, nullptr);
+      int utf16_start = 0;
+      if (line)
+        utf16_start = utf8_byte_offset_to_utf16(line, point.col);
+      free(it->buffer);
+      free(it);
+      it = begin_l_iter(editor->root, pos.row);
+      line = next_line(it, nullptr);
+      int utf16_end = 0;
+      if (line)
+        utf16_end = utf8_byte_offset_to_utf16(line, pos.col);
+      free(it->buffer);
+      free(it);
+      lsp_range = {{"start", {{"line", point.row}, {"character", utf16_start}}},
+                   {"end", {{"line", pos.row}, {"character", utf16_end}}}};
+    }
+    uint32_t start = line_to_byte(editor->root, point.row, nullptr) + point.col;
+    if (cursor_original > start && cursor_original <= byte_pos) {
+      editor->cursor = point;
+      editor->cursor_preffered = UINT32_MAX;
+    } else if (cursor_original > byte_pos) {
+      uint32_t cursor_new = cursor_original - (byte_pos - start);
+      uint32_t new_col;
+      uint32_t new_row = byte_to_line(editor->root, cursor_new, &new_col);
+      editor->cursor = {new_row, new_col};
+      editor->cursor_preffered = UINT32_MAX;
+    }
+    lock_1.unlock();
+    uint32_t start_row = point.row;
+    uint32_t end_row = pos.row;
+    apply_line_deletion(editor, start_row + 1, end_row);
+    apply_hook_deletion(editor, start_row + 1, end_row);
+    std::unique_lock lock_2(editor->knot_mtx);
+    editor->root = erase(editor->root, start, byte_pos - start);
+    lock_2.unlock();
+    if (editor->ts.tree) {
+      TSInputEdit edit = {
+          .start_byte = start,
+          .old_end_byte = byte_pos,
+          .new_end_byte = start,
+          .start_point = {point.row, point.col},
+          .old_end_point = old_point,
+          .new_end_point = {point.row, point.col},
+      };
+      editor->edit_queue.push(edit);
+    }
+    if (do_lsp) {
+      if (editor->lsp->incremental_sync) {
+        json message = {
+            {"jsonrpc", "2.0"},
+            {"method", "textDocument/didChange"},
+            {"params",
+             {{"textDocument",
+               {{"uri", editor->uri}, {"version", ++editor->lsp_version}}},
+              {"contentChanges",
+               json::array({{{"range", lsp_range}, {"text", ""}}})}}}};
+        lsp_send(editor->lsp, message, nullptr);
+      } else {
+        char *buf = read(editor->root, 0, editor->root->char_count);
+        std::string text(buf);
+        free(buf);
+        json message = {
+            {"jsonrpc", "2.0"},
+            {"method", "textDocument/didChange"},
+            {"params",
+             {{"textDocument",
+               {{"uri", editor->uri}, {"version", ++editor->lsp_version}}},
+              {"contentChanges", json::array({{{"text", text}}})}}}};
+        lsp_send(editor->lsp, message, nullptr);
+      }
+    }
+    std::unique_lock lock_3(editor->spans.mtx);
+    apply_edit(editor->spans.spans, start, start - byte_pos);
+    if (editor->spans.mid_parse)
+      editor->spans.edits.push({start, start - byte_pos});
+    std::unique_lock lock_4(editor->def_spans.mtx);
+    apply_edit(editor->def_spans.spans, byte_pos, start - byte_pos);
+  } else {
+    std::shared_lock lock_1(editor->knot_mtx);
+    uint32_t cursor_original =
+        line_to_byte(editor->root, editor->cursor.row, nullptr) +
+        editor->cursor.col;
+    TSPoint old_point = {pos.row, pos.col};
+    uint32_t byte_pos = line_to_byte(editor->root, pos.row, nullptr) + pos.col;
+    Coord point = move_right_pure(editor, pos, len);
+    json lsp_range;
+    bool do_lsp = (editor->lsp != nullptr);
+    if (do_lsp) {
+      LineIterator *it = begin_l_iter(editor->root, pos.row);
+      char *line = next_line(it, nullptr);
+      int utf16_start = 0;
+      if (line)
+        utf16_start = utf8_byte_offset_to_utf16(line, pos.col);
+      free(it->buffer);
+      free(it);
+      it = begin_l_iter(editor->root, point.row);
+      line = next_line(it, nullptr);
+      int utf16_end = 0;
+      if (line)
+        utf16_end = utf8_byte_offset_to_utf16(line, point.col);
+      free(it->buffer);
+      free(it);
+      lsp_range = {{"start", {{"line", pos.row}, {"character", utf16_start}}},
+                   {"end", {{"line", point.row}, {"character", utf16_end}}}};
+    }
+    uint32_t end = line_to_byte(editor->root, point.row, nullptr) + point.col;
+    if (cursor_original > byte_pos && cursor_original <= end) {
+      editor->cursor = pos;
+      editor->cursor_preffered = UINT32_MAX;
+    } else if (cursor_original > end) {
+      uint32_t cursor_new = cursor_original - (end - byte_pos);
+      uint32_t new_col;
+      uint32_t new_row = byte_to_line(editor->root, cursor_new, &new_col);
+      editor->cursor = {new_row, new_col};
+      editor->cursor_preffered = UINT32_MAX;
+    }
+    lock_1.unlock();
+    uint32_t start_row = pos.row;
+    uint32_t end_row = point.row;
+    apply_line_deletion(editor, start_row + 1, end_row);
+    apply_hook_deletion(editor, start_row + 1, end_row);
+    std::unique_lock lock_2(editor->knot_mtx);
+    editor->root = erase(editor->root, byte_pos, end - byte_pos);
+    lock_2.unlock();
+    if (editor->ts.tree) {
+      TSInputEdit edit = {
+          .start_byte = byte_pos,
+          .old_end_byte = end,
+          .new_end_byte = byte_pos,
+          .start_point = old_point,
+          .old_end_point = {point.row, point.col},
+          .new_end_point = old_point,
+      };
+      editor->edit_queue.push(edit);
+    }
+    if (do_lsp) {
+      if (editor->lsp->incremental_sync) {
+        json message = {
+            {"jsonrpc", "2.0"},
+            {"method", "textDocument/didChange"},
+            {"params",
+             {{"textDocument",
+               {{"uri", editor->uri}, {"version", ++editor->lsp_version}}},
+              {"contentChanges",
+               json::array({{{"range", lsp_range}, {"text", ""}}})}}}};
+        lsp_send(editor->lsp, message, nullptr);
+      } else {
+        char *buf = read(editor->root, 0, editor->root->char_count);
+        std::string text(buf);
+        free(buf);
+        json message = {
+            {"jsonrpc", "2.0"},
+            {"method", "textDocument/didChange"},
+            {"params",
+             {{"textDocument",
+               {{"uri", editor->uri}, {"version", ++editor->lsp_version}}},
+              {"contentChanges", json::array({{{"text", text}}})}}}};
+        lsp_send(editor->lsp, message, nullptr);
+      }
+    }
+    std::unique_lock lock_3(editor->spans.mtx);
+    apply_edit(editor->spans.spans, byte_pos, byte_pos - end);
+    if (editor->spans.mid_parse)
+      editor->spans.edits.push({byte_pos, byte_pos - end});
+    std::unique_lock lock_4(editor->def_spans.mtx);
+    apply_edit(editor->def_spans.spans, byte_pos, byte_pos - end);
+  }
+}
+
+void edit_insert(Editor *editor, Coord pos, char *data, uint32_t len) {
+  std::shared_lock lock_1(editor->knot_mtx);
+  uint32_t cursor_original =
+      line_to_byte(editor->root, editor->cursor.row, nullptr) +
+      editor->cursor.col;
+  uint32_t byte_pos = line_to_byte(editor->root, pos.row, nullptr) + pos.col;
+  TSPoint start_point = {pos.row, pos.col};
+  if (cursor_original > byte_pos) {
+    uint32_t cursor_new = cursor_original + len;
+    uint32_t new_col;
+    uint32_t new_row = byte_to_line(editor->root, cursor_new, &new_col);
+    editor->cursor = {new_row, new_col};
+  }
+  lock_1.unlock();
+  std::unique_lock lock_2(editor->knot_mtx);
+  editor->root = insert(editor->root, byte_pos, data, len);
+  lock_2.unlock();
+  uint32_t cols = 0;
+  uint32_t rows = 0;
+  for (uint32_t i = 0; i < len; i++) {
+    if (data[i] == '\n') {
+      rows++;
+      cols = 0;
+    } else {
+      cols++;
+    }
+  }
+  apply_line_insertion(editor, pos.row, rows);
+  apply_hook_insertion(editor, pos.row, rows);
+  if (editor->ts.tree) {
+    TSInputEdit edit = {
+        .start_byte = byte_pos,
+        .old_end_byte = byte_pos,
+        .new_end_byte = byte_pos + len,
+        .start_point = start_point,
+        .old_end_point = start_point,
+        .new_end_point = {start_point.row + rows,
+                          (rows == 0) ? (start_point.column + cols) : cols},
+    };
+    editor->edit_queue.push(edit);
+  }
+  if (editor->lsp) {
+    if (editor->lsp->incremental_sync) {
+      lock_1.lock();
+      LineIterator *it = begin_l_iter(editor->root, pos.row);
+      char *line = next_line(it, nullptr);
+      int utf16_col = 0;
+      if (line)
+        utf16_col = utf8_byte_offset_to_utf16(line, pos.col);
+      free(it->buffer);
+      free(it);
+      lock_1.unlock();
+      json message = {
+          {"jsonrpc", "2.0"},
+          {"method", "textDocument/didChange"},
+          {"params",
+           {{"textDocument",
+             {{"uri", editor->uri}, {"version", ++editor->lsp_version}}},
+            {"contentChanges",
+             json::array(
+                 {{{"range",
+                    {{"start", {{"line", pos.row}, {"character", utf16_col}}},
+                     {"end", {{"line", pos.row}, {"character", utf16_col}}}}},
+                   {"text", std::string(data, len)}}})}}}};
+      lsp_send(editor->lsp, message, nullptr);
+    } else {
+      char *buf = read(editor->root, 0, editor->root->char_count);
+      std::string text(buf);
+      free(buf);
+      json message = {
+          {"jsonrpc", "2.0"},
+          {"method", "textDocument/didChange"},
+          {"params",
+           {{"textDocument",
+             {{"uri", editor->uri}, {"version", ++editor->lsp_version}}},
+            {"contentChanges", json::array({{{"text", text}}})}}}};
+      lsp_send(editor->lsp, message, nullptr);
+    }
+  }
+  std::unique_lock lock_3(editor->spans.mtx);
+  apply_edit(editor->spans.spans, byte_pos, len);
+  if (editor->spans.mid_parse)
+    editor->spans.edits.push({byte_pos, len});
+  std::unique_lock lock_4(editor->def_spans.mtx);
+  apply_edit(editor->def_spans.spans, byte_pos, len);
+}

src/editor/editor.cc

@@ -0,0 +1,79 @@
+#include "editor/editor.h"
+#include "lsp/lsp.h"
+#include "utils/utils.h"
+
+Editor *new_editor(const char *filename_arg, Coord position, Coord size) {
+  Editor *editor = new Editor();
+  if (!editor)
+    return nullptr;
+  uint32_t len = 0;
+  std::string filename = path_abs(filename_arg);
+  char *str = load_file(filename.c_str(), &len);
+  if (!str) {
+    free_editor(editor);
+    return nullptr;
+  }
+  editor->filename = filename;
+  editor->uri = path_to_file_uri(filename);
+  editor->position = position;
+  editor->size = size;
+  editor->cursor_preffered = UINT32_MAX;
+  if (len == 0) {
+    free(str);
+    str = (char *)malloc(1);
+    *str = '\n';
+    len = 1;
+  }
+  editor->root = load(str, len, optimal_chunk_size(len));
+  free(str);
+  Language language = language_for_file(filename.c_str());
+  if (language.name != "unknown" && len <= (1024 * 128)) {
+    editor->ts.parser = ts_parser_new();
+    editor->ts.language = language.fn();
+    ts_parser_set_language(editor->ts.parser, editor->ts.language);
+    editor->ts.query_file =
+        get_exe_dir() + "/../grammar/" + language.name + ".scm";
+    request_add_to_lsp(language, editor);
+  }
+  return editor;
+}
+
+void free_tsset(TSSetMain *set) {
+  if (set->parser)
+    ts_parser_delete(set->parser);
+  if (set->tree)
+    ts_tree_delete(set->tree);
+  if (set->query)
+    ts_query_delete(set->query);
+  for (auto &inj : set->injections) {
+    if (inj.second.parser)
+      ts_parser_delete(inj.second.parser);
+    if (inj.second.query)
+      ts_query_delete(inj.second.query);
+    if (inj.second.tree)
+      ts_tree_delete(inj.second.tree);
+  }
+}
+
+void free_editor(Editor *editor) {
+  remove_from_lsp(editor);
+  free_tsset(&editor->ts);
+  free_rope(editor->root);
+  delete editor;
+}
+
+void save_file(Editor *editor) {
+  if (!editor || !editor->root)
+    return;
+  char *str = read(editor->root, 0, editor->root->char_count);
+  if (!str)
+    return;
+  std::ofstream out(editor->filename);
+  out.write(str, editor->root->char_count);
+  free(str);
+  json msg = {{"jsonrpc", "2.0"},
+              {"method", "textDocument/didSave"},
+              {"params", {{"textDocument", {{"uri", editor->uri}}}}}};
+  if (editor->lsp)
+    lsp_send(editor->lsp, msg, nullptr);
+}

src/editor/events.cc

@@ -1,9 +1,7 @@
-#include "../include/editor.h"
+#include "editor/editor.h"
-#include "../include/lsp.h"
+#include "editor/folds.h"
-#include "../include/main.h"
+#include "lsp/lsp.h"
-#include "../include/ts.h"
+#include "main.h"
-#include <cstdint>
-#include <sys/ioctl.h>
 
 void handle_editor_event(Editor *editor, KeyEvent event) {
   static std::chrono::steady_clock::time_point last_click_time =
@@ -665,153 +663,3 @@ void handle_editor_event(Editor *editor, KeyEvent event) {
   if ((event.key_type == KEY_CHAR || event.key_type == KEY_PASTE) && event.c)
     free(event.c);
 }
-
-void hover_diagnostic(Editor *editor) {
-  std::shared_lock lock(editor->v_mtx);
-  static uint32_t last_line = UINT32_MAX;
-  if (last_line == editor->cursor.row && !editor->warnings_dirty)
-    return;
-  VWarn dummy;
-  dummy.line = editor->cursor.row;
-  editor->warnings_dirty = false;
-  last_line = editor->cursor.row;
-  auto first =
-      std::lower_bound(editor->warnings.begin(), editor->warnings.end(), dummy);
-  auto last =
-      std::upper_bound(editor->warnings.begin(), editor->warnings.end(), dummy);
-  std::vector<VWarn> warnings_at_line(first, last);
-  if (warnings_at_line.size() == 0) {
-    editor->diagnostics_active = false;
-    return;
-  }
-  editor->diagnostics.clear();
-  editor->diagnostics.warnings.swap(warnings_at_line);
-  editor->diagnostics.render_first();
-  editor->diagnostics_active = true;
-}
-
-static Highlight HL_UNDERLINE = {0, 0, CF_UNDERLINE, UINT8_MAX - 1};
-
-void editor_worker(Editor *editor) {
-  if (!editor || !editor->root)
-    return;
-  if (editor->root->char_count > (1024 * 200))
-    return;
-  if (editor->ts.query_file != "" && !editor->ts.query)
-    editor->ts.query = load_query(editor->ts.query_file.c_str(), &editor->ts);
-  if (editor->ts.parser && editor->ts.query)
-    ts_collect_spans(editor);
-  uint32_t prev_col, next_col;
-  word_boundaries_exclusive(editor, editor->cursor, &prev_col, &next_col);
-  std::unique_lock lock(editor->def_spans.mtx);
-  editor->def_spans.spans.clear();
-  if (next_col - prev_col > 0 && next_col - prev_col < 256 - 4) {
-    std::shared_lock lockk(editor->knot_mtx);
-    uint32_t offset = line_to_byte(editor->root, editor->cursor.row, nullptr);
-    char *word = read(editor->root, offset + prev_col, next_col - prev_col);
-    lockk.unlock();
-    if (word) {
-      char buf[256];
-      snprintf(buf, sizeof(buf), "\\b%s\\b", word);
-      std::vector<std::pair<size_t, size_t>> results =
-          search_rope(editor->root, buf);
-      for (const auto &match : results) {
-        Span s;
-        s.start = match.first;
-        s.end = match.first + match.second;
-        s.hl = &HL_UNDERLINE;
-        editor->def_spans.spans.push_back(s);
-      }
-      free(word);
-    }
-  }
-  uint8_t top = 0;
-  static Highlight *hl_s = (Highlight *)calloc(200, sizeof(Highlight));
-  if (!hl_s)
-    exit(ENOMEM);
-  std::vector<std::pair<size_t, size_t>> results =
-      search_rope(editor->root, "(0x|#)[0-9a-fA-F]{6}");
-  std::shared_lock lockk(editor->knot_mtx);
-  for (int i = 0; i < results.size() && top < 200; i++) {
-    Span s;
-    s.start = results[i].first;
-    s.end = results[i].first + results[i].second;
-    char *buf = read(editor->root, s.start, s.end - s.start);
-    int x = buf[0] == '#' ? 1 : 2;
-    uint32_t bg = HEX(buf + x);
-    free(buf);
-    uint8_t r = bg >> 16, g = (bg >> 8) & 0xFF, b = bg & 0xFF;
-    double luminance = 0.299 * r + 0.587 * g + 0.114 * b;
-    uint32_t fg = (luminance > 128) ? 0x010101 : 0xFEFEFE;
-    hl_s[top] = {fg, bg, CF_BOLD, UINT8_MAX};
-    s.hl = &hl_s[top];
-    editor->def_spans.spans.push_back(s);
-    top++;
-  }
-  std::sort(editor->def_spans.spans.begin(), editor->def_spans.spans.end());
-  lock.unlock();
-  lockk.unlock();
-  hover_diagnostic(editor);
-}
-
-void editor_lsp_handle(Editor *editor, json msg) {
-  if (msg.contains("method") &&
-      msg["method"] == "textDocument/publishDiagnostics") {
-    std::unique_lock lock(editor->v_mtx);
-    editor->warnings.clear();
-    json diagnostics = msg["params"]["diagnostics"];
-    for (size_t i = 0; i < diagnostics.size(); i++) {
-      json d = diagnostics[i];
-      VWarn w;
-      // HACK: convert back to utf-8 but as this is only visually affecting it
-      //       is not worth getting the line string from the rope.
-      w.line = d["range"]["start"]["line"];
-      w.start = d["range"]["start"]["character"];
-      uint32_t end = d["range"]["end"]["character"];
-      if (d["range"]["end"]["line"] == w.line)
-        w.end = end;
-      std::string text = trim(d["message"].get<std::string>());
-      w.text_full = text;
-      auto pos = text.find('\n');
-      w.text = (pos == std::string::npos) ? text : text.substr(0, pos);
-      if (d.contains("source"))
-        w.source = d["source"].get<std::string>();
-      if (d.contains("code")) {
-        w.code = "[";
-        if (d["code"].is_string())
-          w.code += d["code"].get<std::string>() + "] ";
-        else if (d["code"].is_number())
-          w.code += std::to_string(d["code"].get<int>()) + "] ";
-        else
-          w.code.clear();
-        if (d.contains("codeDescription") &&
-            d["codeDescription"].contains("href"))
-          w.code += d["codeDescription"]["href"].get<std::string>();
-      }
-      if (d.contains("relatedInformation")) {
-        json related = d["relatedInformation"];
-        for (size_t j = 0; j < related.size(); j++) {
-          json rel = related[j];
-          std::string message = rel["message"].get<std::string>();
-          auto pos = message.find('\n');
-          message =
-              (pos == std::string::npos) ? message : message.substr(0, pos);
-          std::string uri =
-              percent_decode(rel["location"]["uri"].get<std::string>());
-          auto pos2 = uri.find_last_of('/');
-          if (pos2 != std::string::npos)
-            uri = uri.substr(pos2 + 1);
-          std::string row = std::to_string(
-              rel["location"]["range"]["start"]["line"].get<int>());
-          w.see_also.push_back(uri + ":" + row + ": " + message);
-        }
-      }
-      w.type = 1;
-      if (d.contains("severity"))
-        w.type = d["severity"].get<int>();
-      editor->warnings.push_back(w);
-    }
-    std::sort(editor->warnings.begin(), editor->warnings.end());
-    editor->warnings_dirty = true;
-  }
-}

src/editor/indents.cc

@@ -1,4 +1,4 @@
-#include "../include/editor.h"
+#include "editor/editor.h"
 
 uint32_t leading_indent(const char *line, uint32_t len) {
   uint32_t indent = 0;

src/editor/lsp.cc

@@ -0,0 +1,63 @@
+#include "editor/editor.h"
+
+void editor_lsp_handle(Editor *editor, json msg) {
+  if (msg.contains("method") &&
+      msg["method"] == "textDocument/publishDiagnostics") {
+    std::unique_lock lock(editor->v_mtx);
+    editor->warnings.clear();
+    json diagnostics = msg["params"]["diagnostics"];
+    for (size_t i = 0; i < diagnostics.size(); i++) {
+      json d = diagnostics[i];
+      VWarn w;
+      // HACK: convert back to utf-8 but as this is only visually affecting it
+      //       is not worth getting the line string from the rope.
+      w.line = d["range"]["start"]["line"];
+      w.start = d["range"]["start"]["character"];
+      uint32_t end = d["range"]["end"]["character"];
+      if (d["range"]["end"]["line"] == w.line)
+        w.end = end;
+      std::string text = trim(d["message"].get<std::string>());
+      w.text_full = text;
+      auto pos = text.find('\n');
+      w.text = (pos == std::string::npos) ? text : text.substr(0, pos);
+      if (d.contains("source"))
+        w.source = d["source"].get<std::string>();
+      if (d.contains("code")) {
+        w.code = "[";
+        if (d["code"].is_string())
+          w.code += d["code"].get<std::string>() + "] ";
+        else if (d["code"].is_number())
+          w.code += std::to_string(d["code"].get<int>()) + "] ";
+        else
+          w.code.clear();
+        if (d.contains("codeDescription") &&
+            d["codeDescription"].contains("href"))
+          w.code += d["codeDescription"]["href"].get<std::string>();
+      }
+      if (d.contains("relatedInformation")) {
+        json related = d["relatedInformation"];
+        for (size_t j = 0; j < related.size(); j++) {
+          json rel = related[j];
+          std::string message = rel["message"].get<std::string>();
+          auto pos = message.find('\n');
+          message =
+              (pos == std::string::npos) ? message : message.substr(0, pos);
+          std::string uri =
+              percent_decode(rel["location"]["uri"].get<std::string>());
+          auto pos2 = uri.find_last_of('/');
+          if (pos2 != std::string::npos)
+            uri = uri.substr(pos2 + 1);
+          std::string row = std::to_string(
+              rel["location"]["range"]["start"]["line"].get<int>());
+          w.see_also.push_back(uri + ":" + row + ": " + message);
+        }
+      }
+      w.type = 1;
+      if (d.contains("severity"))
+        w.type = d["severity"].get<int>();
+      editor->warnings.push_back(w);
+    }
+    std::sort(editor->warnings.begin(), editor->warnings.end());
+    editor->warnings_dirty = true;
+  }
+}

src/editor/move_line.cc

@@ -0,0 +1,121 @@
+#include "editor/editor.h"
+#include "editor/folds.h"
+#include "main.h"
+
+void move_line_up(Editor *editor) {
+  if (!editor || !editor->root || editor->cursor.row == 0)
+    return;
+  if (mode == NORMAL || mode == INSERT) {
+    uint32_t line_len, line_cluster_len;
+    std::shared_lock lock(editor->knot_mtx);
+    LineIterator *it = begin_l_iter(editor->root, editor->cursor.row);
+    char *line = next_line(it, &line_len);
+    if (!line) {
+      lock.unlock();
+      return;
+    }
+    if (line_len > 0 && line[line_len - 1] == '\n')
+      line_len--;
+    line_cluster_len = count_clusters(line, line_len, 0, line_len);
+    uint32_t target_row = prev_unfolded_row(editor, editor->cursor.row - 1);
+    uint32_t up_by = editor->cursor.row - target_row;
+    if (up_by > 1)
+      up_by--;
+    lock.unlock();
+    Coord cursor = editor->cursor;
+    edit_erase(editor, {cursor.row, 0}, line_cluster_len);
+    edit_erase(editor, {cursor.row, 0}, -1);
+    edit_insert(editor, {cursor.row - up_by, 0}, (char *)"\n", 1);
+    edit_insert(editor, {cursor.row - up_by, 0}, line, line_len);
+    free(it->buffer);
+    free(it);
+    editor->cursor = {cursor.row - up_by, cursor.col};
+  } else if (mode == SELECT) {
+    uint32_t start_row = MIN(editor->cursor.row, editor->selection.row);
+    uint32_t end_row = MAX(editor->cursor.row, editor->selection.row);
+    uint32_t start_byte = line_to_byte(editor->root, start_row, nullptr);
+    uint32_t end_byte = line_to_byte(editor->root, end_row + 1, nullptr);
+    char *selected_text = read(editor->root, start_byte, end_byte - start_byte);
+    if (!selected_text)
+      return;
+    uint32_t selected_len = count_clusters(selected_text, end_byte - start_byte,
+                                           0, end_byte - start_byte);
+    Coord cursor = editor->cursor;
+    Coord selection = editor->selection;
+    edit_erase(editor, {start_row, 0}, selected_len);
+    edit_insert(editor, {start_row - 1, 0}, selected_text,
+                end_byte - start_byte);
+    free(selected_text);
+    editor->cursor = {cursor.row - 1, cursor.col};
+    editor->selection = {selection.row - 1, selection.col};
+  }
+}
+
+void move_line_down(Editor *editor) {
+  if (!editor || !editor->root)
+    return;
+  if (mode == NORMAL || mode == INSERT) {
+    if (editor->cursor.row >= editor->root->line_count - 1)
+      return;
+    uint32_t line_len, line_cluster_len;
+    std::shared_lock lock(editor->knot_mtx);
+    LineIterator *it = begin_l_iter(editor->root, editor->cursor.row);
+    char *line = next_line(it, &line_len);
+    if (!line) {
+      lock.unlock();
+      return;
+    }
+    if (line_len && line[line_len - 1] == '\n')
+      line_len--;
+    line_cluster_len = count_clusters(line, line_len, 0, line_len);
+    uint32_t target_row = next_unfolded_row(editor, editor->cursor.row + 1);
+    if (target_row >= editor->root->line_count) {
+      free(line);
+      lock.unlock();
+      return;
+    }
+    uint32_t down_by = target_row - editor->cursor.row;
+    if (down_by > 1)
+      down_by--;
+    uint32_t ln;
+    line_to_byte(editor->root, editor->cursor.row + down_by - 1, &ln);
+    lock.unlock();
+    Coord cursor = editor->cursor;
+    edit_erase(editor, {cursor.row, 0}, line_cluster_len);
+    edit_erase(editor, {cursor.row, 0}, -1);
+    edit_insert(editor, {cursor.row + down_by, 0}, (char *)"\n", 1);
+    edit_insert(editor, {cursor.row + down_by, 0}, line, line_len);
+    free(it->buffer);
+    free(it);
+    editor->cursor = {cursor.row + down_by, cursor.col};
+  } else if (mode == SELECT) {
+    if (editor->cursor.row >= editor->root->line_count - 1 ||
+        editor->selection.row >= editor->root->line_count - 1)
+      return;
+    std::shared_lock lock(editor->knot_mtx);
+    uint32_t start_row = MIN(editor->cursor.row, editor->selection.row);
+    uint32_t end_row = MAX(editor->cursor.row, editor->selection.row);
+    uint32_t target_row = next_unfolded_row(editor, end_row + 1);
+    if (target_row >= editor->root->line_count)
+      return;
+    uint32_t down_by = target_row - end_row;
+    if (down_by > 1)
+      down_by--;
+    uint32_t start_byte = line_to_byte(editor->root, start_row, nullptr);
+    uint32_t end_byte = line_to_byte(editor->root, end_row + 1, nullptr);
+    char *selected_text = read(editor->root, start_byte, end_byte - start_byte);
+    lock.unlock();
+    if (!selected_text)
+      return;
+    uint32_t selected_len = count_clusters(selected_text, end_byte - start_byte,
+                                           0, end_byte - start_byte);
+    Coord cursor = editor->cursor;
+    Coord selection = editor->selection;
+    edit_erase(editor, {start_row, 0}, selected_len);
+    edit_insert(editor, {start_row + down_by, 0}, selected_text,
+                end_byte - start_byte);
+    free(selected_text);
+    editor->cursor = {cursor.row + down_by, cursor.col};
+    editor->selection = {selection.row + down_by, selection.col};
+  }
+}

src/editor/renderer.cc

@@ -1,87 +1,6 @@
-#include <cstdint>
+#include "editor/editor.h"
-extern "C" {
+#include "editor/folds.h"
-#include "../libs/libgrapheme/grapheme.h"
+#include "main.h"
-}
-#include "../include/editor.h"
-#include "../include/lsp.h"
-#include "../include/main.h"
-#include "../include/utils.h"
-
-Editor *new_editor(const char *filename_arg, Coord position, Coord size) {
-  Editor *editor = new Editor();
-  if (!editor)
-    return nullptr;
-  uint32_t len = 0;
-  std::string filename = path_abs(filename_arg);
-  char *str = load_file(filename.c_str(), &len);
-  if (!str) {
-    free_editor(editor);
-    return nullptr;
-  }
-  editor->filename = filename;
-  editor->uri = path_to_file_uri(filename);
-  editor->position = position;
-  editor->size = size;
-  editor->cursor_preffered = UINT32_MAX;
-  if (len == 0) {
-    free(str);
-    str = (char *)malloc(1);
-    *str = '\n';
-    len = 1;
-  }
-  editor->root = load(str, len, optimal_chunk_size(len));
-  free(str);
-  Language language = language_for_file(filename.c_str());
-  if (language.name != "unknown" && len <= (1024 * 128)) {
-    editor->ts.parser = ts_parser_new();
-    editor->ts.language = language.fn();
-    ts_parser_set_language(editor->ts.parser, editor->ts.language);
-    editor->ts.query_file =
-        get_exe_dir() + "/../grammar/" + language.name + ".scm";
-    request_add_to_lsp(language, editor);
-  }
-  return editor;
-}
-
-void free_tsset(TSSetMain *set) {
-  if (set->parser)
-    ts_parser_delete(set->parser);
-  if (set->tree)
-    ts_tree_delete(set->tree);
-  if (set->query)
-    ts_query_delete(set->query);
-  for (auto &inj : set->injections) {
-    if (inj.second.parser)
-      ts_parser_delete(inj.second.parser);
-    if (inj.second.query)
-      ts_query_delete(inj.second.query);
-    if (inj.second.tree)
-      ts_tree_delete(inj.second.tree);
-  }
-}
-
-void free_editor(Editor *editor) {
-  remove_from_lsp(editor);
-  free_tsset(&editor->ts);
-  free_rope(editor->root);
-  delete editor;
-}
-
-void save_file(Editor *editor) {
-  if (!editor || !editor->root)
-    return;
-  char *str = read(editor->root, 0, editor->root->char_count);
-  if (!str)
-    return;
-  std::ofstream out(editor->filename);
-  out.write(str, editor->root->char_count);
-  free(str);
-  json msg = {{"jsonrpc", "2.0"},
-              {"method", "textDocument/didSave"},
-              {"params", {{"textDocument", {{"uri", editor->uri}}}}}};
-  if (editor->lsp)
-    lsp_send(editor->lsp, msg, nullptr);
-}
 
 void render_editor(Editor *editor) {
   uint32_t sel_start = 0, sel_end = 0;

src/editor/scroll.cc

@@ -0,0 +1,234 @@
+#include "editor/editor.h"
+#include "editor/folds.h"
+
+void scroll_up(Editor *editor, int32_t number) {
+  if (!editor || number == 0)
+    return;
+  uint32_t numlen =
+      EXTRA_META + static_cast<int>(std::log10(editor->root->line_count + 1));
+  uint32_t render_width = editor->size.col - numlen;
+  uint32_t line_index = editor->scroll.row;
+  LineIterator *it = begin_l_iter(editor->root, line_index);
+  if (!it)
+    return;
+  uint32_t len;
+  char *line = next_line(it, &len);
+  if (!line) {
+    free(it->buffer);
+    free(it);
+    return;
+  }
+  if (len > 0 && line[len - 1] == '\n')
+    len--;
+  uint32_t current_byte_offset = 0;
+  uint32_t col = 0;
+  std::vector<uint32_t> segment_starts;
+  segment_starts.reserve(16);
+  if (current_byte_offset < editor->scroll.col)
+    segment_starts.push_back(0);
+  while (current_byte_offset < editor->scroll.col &&
+         current_byte_offset < len) {
+    uint32_t cluster_len = grapheme_next_character_break_utf8(
+        line + current_byte_offset, len - current_byte_offset);
+    int width = display_width(line + current_byte_offset, cluster_len);
+    if (col + width > render_width) {
+      segment_starts.push_back(current_byte_offset);
+      col = 0;
+    }
+    current_byte_offset += cluster_len;
+    col += width;
+  }
+  for (auto it_seg = segment_starts.rbegin(); it_seg != segment_starts.rend();
+       ++it_seg) {
+    if (--number == 0) {
+      editor->scroll = {line_index, *it_seg};
+      free(it->buffer);
+      free(it);
+      return;
+    }
+  }
+  line = prev_line(it, &len);
+  if (!line) {
+    editor->scroll = {0, 0};
+    free(it->buffer);
+    free(it);
+    return;
+  }
+  do {
+    line_index--;
+    line = prev_line(it, &len);
+    if (!line) {
+      editor->scroll = {0, 0};
+      free(it->buffer);
+      free(it);
+      return;
+    }
+    const Fold *fold = fold_for_line(editor->folds, line_index);
+    if (fold) {
+      while (line && line_index > fold->start) {
+        free(line);
+        line = prev_line(it, &len);
+        line_index--;
+        if (!line) {
+          editor->scroll = {0, 0};
+          free(it->buffer);
+          free(it);
+          return;
+        }
+      }
+      if (--number == 0) {
+        editor->scroll = {fold->start, 0};
+        free(it->buffer);
+        free(it);
+        return;
+      }
+      if (fold->start == 0) {
+        editor->scroll = {0, 0};
+        free(it->buffer);
+        free(it);
+        return;
+      }
+      line_index = fold->start - 1;
+      line = prev_line(it, &len);
+      if (!line) {
+        editor->scroll = {0, 0};
+        free(it->buffer);
+        free(it);
+        return;
+      }
+      continue;
+    }
+    if (len > 0 && line[len - 1] == '\n')
+      len--;
+    current_byte_offset = 0;
+    col = 0;
+    std::vector<uint32_t> segment_starts;
+    segment_starts.reserve(16);
+    segment_starts.push_back(0);
+    while (current_byte_offset < len) {
+      uint32_t cluster_len = grapheme_next_character_break_utf8(
+          line + current_byte_offset, len - current_byte_offset);
+      int width = display_width(line + current_byte_offset, cluster_len);
+      if (col + width > render_width) {
+        segment_starts.push_back(current_byte_offset);
+        col = 0;
+      }
+      current_byte_offset += cluster_len;
+      col += width;
+    }
+    for (auto it_seg = segment_starts.rbegin(); it_seg != segment_starts.rend();
+         ++it_seg) {
+      if (--number == 0) {
+        editor->scroll = {line_index, *it_seg};
+        free(it->buffer);
+        free(it);
+        return;
+      }
+    }
+  } while (number > 0);
+  free(it->buffer);
+  free(it);
+}
+
+void scroll_down(Editor *editor, uint32_t number) {
+  if (!editor || number == 0)
+    return;
+  uint32_t numlen =
+      EXTRA_META + static_cast<int>(std::log10(editor->root->line_count + 1));
+  uint32_t render_width = editor->size.col - numlen;
+  uint32_t line_index = editor->scroll.row;
+  LineIterator *it = begin_l_iter(editor->root, line_index);
+  if (!it)
+    return;
+  const uint32_t max_visual_lines = editor->size.row;
+  Coord *scroll_queue = (Coord *)malloc(sizeof(Coord) * max_visual_lines);
+  uint32_t q_head = 0;
+  uint32_t q_size = 0;
+  uint32_t visual_seen = 0;
+  bool first_visual_line = true;
+  while (true) {
+    const Fold *fold = fold_for_line(editor->folds, line_index);
+    if (fold) {
+      Coord fold_coord = {fold->start, 0};
+      if (q_size < max_visual_lines) {
+        scroll_queue[(q_head + q_size) % max_visual_lines] = fold_coord;
+        q_size++;
+      } else {
+        scroll_queue[q_head] = fold_coord;
+        q_head = (q_head + 1) % max_visual_lines;
+      }
+      visual_seen++;
+      if (visual_seen >= number + max_visual_lines) {
+        editor->scroll = scroll_queue[q_head];
+        break;
+      }
+      uint32_t skip_until = fold->end;
+      while (line_index <= skip_until) {
+        char *line = next_line(it, nullptr);
+        if (!line) {
+          free(scroll_queue);
+          free(it->buffer);
+          free(it);
+          return;
+        }
+        line_index++;
+      }
+      continue;
+    }
+    uint32_t line_len;
+    char *line = next_line(it, &line_len);
+    if (!line)
+      break;
+    if (line_len && line[line_len - 1] == '\n')
+      line_len--;
+    uint32_t current_byte_offset = 0;
+    if (first_visual_line) {
+      current_byte_offset += editor->scroll.col;
+      first_visual_line = false;
+    }
+    while (current_byte_offset < line_len ||
+           (line_len == 0 && current_byte_offset == 0)) {
+      Coord coord = {line_index, current_byte_offset};
+      if (q_size < max_visual_lines) {
+        scroll_queue[(q_head + q_size) % max_visual_lines] = coord;
+        q_size++;
+      } else {
+        scroll_queue[q_head] = coord;
+        q_head = (q_head + 1) % max_visual_lines;
+      }
+      visual_seen++;
+      if (visual_seen >= number + max_visual_lines) {
+        editor->scroll = scroll_queue[q_head];
+        free(scroll_queue);
+        free(it->buffer);
+        free(it);
+        return;
+      }
+      uint32_t col = 0;
+      uint32_t local_render_offset = 0;
+      uint32_t left = line_len - current_byte_offset;
+      while (left > 0 && col < render_width) {
+        uint32_t cluster_len = grapheme_next_character_break_utf8(
+            line + current_byte_offset + local_render_offset, left);
+        int width = display_width(
+            line + current_byte_offset + local_render_offset, cluster_len);
+        if (col + width > render_width)
+          break;
+        local_render_offset += cluster_len;
+        left -= cluster_len;
+        col += width;
+      }
+      current_byte_offset += local_render_offset;
+      if (line_len == 0)
+        break;
+    }
+    line_index++;
+  }
+  if (q_size > 0) {
+    uint32_t advance = (q_size > number) ? number : (q_size - 1);
+    editor->scroll = scroll_queue[(q_head + advance) % max_visual_lines];
+  }
+  free(it->buffer);
+  free(it);
+  free(scroll_queue);
+}

src/editor/selection.cc

@@ -0,0 +1,58 @@
+#include "editor/editor.h"
+
+char *get_selection(Editor *editor, uint32_t *out_len, Coord *out_start) {
+  std::shared_lock lock(editor->knot_mtx);
+  Coord start, end;
+  if (editor->cursor >= editor->selection) {
+    uint32_t prev_col, next_col;
+    switch (editor->selection_type) {
+    case CHAR:
+      start = editor->selection;
+      end = move_right(editor, editor->cursor, 1);
+      break;
+    case WORD:
+      word_boundaries(editor, editor->selection, &prev_col, &next_col, nullptr,
+                      nullptr);
+      start = {editor->selection.row, prev_col};
+      end = editor->cursor;
+      break;
+    case LINE:
+      start = {editor->selection.row, 0};
+      end = editor->cursor;
+      break;
+    }
+  } else {
+    start = editor->cursor;
+    uint32_t prev_col, next_col, line_len;
+    switch (editor->selection_type) {
+    case CHAR:
+      end = move_right(editor, editor->selection, 1);
+      break;
+    case WORD:
+      word_boundaries(editor, editor->selection, &prev_col, &next_col, nullptr,
+                      nullptr);
+      end = {editor->selection.row, next_col};
+      break;
+    case LINE:
+      LineIterator *it = begin_l_iter(editor->root, editor->selection.row);
+      char *line = next_line(it, &line_len);
+      if (!line)
+        return nullptr;
+      if (line_len > 0 && line[line_len - 1] == '\n')
+        line_len--;
+      end = {editor->selection.row, line_len};
+      free(it->buffer);
+      free(it);
+      break;
+    }
+  }
+  if (out_start)
+    *out_start = start;
+  uint32_t start_byte =
+      line_to_byte(editor->root, start.row, nullptr) + start.col;
+  uint32_t end_byte = line_to_byte(editor->root, end.row, nullptr) + end.col;
+  char *text = read(editor->root, start_byte, end_byte - start_byte);
+  if (out_len)
+    *out_len = end_byte - start_byte;
+  return text;
+}

src/editor/worker.cc

@@ -0,0 +1,90 @@
+#include "editor/editor.h"
+#include "ts/ts.h"
+
+static Highlight HL_UNDERLINE = {0, 0, CF_UNDERLINE, UINT8_MAX - 1};
+
+void hover_diagnostic(Editor *editor) {
+  std::shared_lock lock(editor->v_mtx);
+  static uint32_t last_line = UINT32_MAX;
+  if (last_line == editor->cursor.row && !editor->warnings_dirty)
+    return;
+  VWarn dummy;
+  dummy.line = editor->cursor.row;
+  editor->warnings_dirty = false;
+  last_line = editor->cursor.row;
+  auto first =
+      std::lower_bound(editor->warnings.begin(), editor->warnings.end(), dummy);
+  auto last =
+      std::upper_bound(editor->warnings.begin(), editor->warnings.end(), dummy);
+  std::vector<VWarn> warnings_at_line(first, last);
+  if (warnings_at_line.size() == 0) {
+    editor->diagnostics_active = false;
+    return;
+  }
+  editor->diagnostics.clear();
+  editor->diagnostics.warnings.swap(warnings_at_line);
+  editor->diagnostics.render_first();
+  editor->diagnostics_active = true;
+}
+
+void editor_worker(Editor *editor) {
+  if (!editor || !editor->root)
+    return;
+  if (editor->root->char_count > (1024 * 200))
+    return;
+  if (editor->ts.query_file != "" && !editor->ts.query)
+    editor->ts.query = load_query(editor->ts.query_file.c_str(), &editor->ts);
+  if (editor->ts.parser && editor->ts.query)
+    ts_collect_spans(editor);
+  uint32_t prev_col, next_col;
+  word_boundaries_exclusive(editor, editor->cursor, &prev_col, &next_col);
+  std::unique_lock lock(editor->def_spans.mtx);
+  editor->def_spans.spans.clear();
+  if (next_col - prev_col > 0 && next_col - prev_col < 256 - 4) {
+    std::shared_lock lockk(editor->knot_mtx);
+    uint32_t offset = line_to_byte(editor->root, editor->cursor.row, nullptr);
+    char *word = read(editor->root, offset + prev_col, next_col - prev_col);
+    lockk.unlock();
+    if (word) {
+      char buf[256];
+      snprintf(buf, sizeof(buf), "\\b%s\\b", word);
+      std::vector<std::pair<size_t, size_t>> results =
+          search_rope(editor->root, buf);
+      for (const auto &match : results) {
+        Span s;
+        s.start = match.first;
+        s.end = match.first + match.second;
+        s.hl = &HL_UNDERLINE;
+        editor->def_spans.spans.push_back(s);
+      }
+      free(word);
+    }
+  }
+  uint8_t top = 0;
+  static Highlight *hl_s = (Highlight *)calloc(200, sizeof(Highlight));
+  if (!hl_s)
+    exit(ENOMEM);
+  std::shared_lock lockk(editor->knot_mtx);
+  std::vector<std::pair<size_t, size_t>> results =
+      search_rope(editor->root, "(0x|#)[0-9a-fA-F]{6}");
+  for (int i = 0; i < results.size() && top < 200; i++) {
+    Span s;
+    s.start = results[i].first;
+    s.end = results[i].first + results[i].second;
+    char *buf = read(editor->root, s.start, s.end - s.start);
+    int x = buf[0] == '#' ? 1 : 2;
+    uint32_t bg = HEX(buf + x);
+    free(buf);
+    uint8_t r = bg >> 16, g = (bg >> 8) & 0xFF, b = bg & 0xFF;
+    double luminance = 0.299 * r + 0.587 * g + 0.114 * b;
+    uint32_t fg = (luminance > 128) ? 0x010101 : 0xFEFEFE;
+    hl_s[top] = {fg, bg, CF_BOLD, UINT8_MAX};
+    s.hl = &hl_s[top];
+    editor->def_spans.spans.push_back(s);
+    top++;
+  }
+  std::sort(editor->def_spans.spans.begin(), editor->def_spans.spans.end());
+  lock.unlock();
+  lockk.unlock();
+  hover_diagnostic(editor);
+}

src/editor_ctrl.cc

@@ -1,787 +0,0 @@
-extern "C" {
-#include "../libs/libgrapheme/grapheme.h"
-}
-#include "../include/editor.h"
-#include "../include/lsp.h"
-#include "../include/main.h"
-#include "../include/utils.h"
-
-uint32_t scan_left(const char *line, uint32_t len, uint32_t off) {
-  if (off > len)
-    off = len;
-  uint32_t i = off;
-  while (i > 0) {
-    unsigned char c = (unsigned char)line[i - 1];
-    if ((c & 0x80) != 0)
-      break;
-    if (!((c == '_') || (c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z') ||
-          (c >= 'a' && c <= 'z')))
-      break;
-    --i;
-  }
-  return i;
-}
-
-uint32_t scan_right(const char *line, uint32_t len, uint32_t off) {
-  if (off > len)
-    off = len;
-  uint32_t i = off;
-  while (i < len) {
-    unsigned char c = (unsigned char)line[i];
-    if ((c & 0x80) != 0)
-      break;
-    if (!((c == '_') || (c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z') ||
-          (c >= 'a' && c <= 'z')))
-      break;
-    ++i;
-  }
-  return i;
-}
-
-void word_boundaries_exclusive(Editor *editor, Coord coord, uint32_t *prev_col,
-                               uint32_t *next_col) {
-  if (!editor)
-    return;
-  std::shared_lock lock(editor->knot_mtx);
-  LineIterator *it = begin_l_iter(editor->root, coord.row);
-  if (!it)
-    return;
-  uint32_t line_len;
-  char *line = next_line(it, &line_len);
-  if (!line)
-    return;
-  if (line_len && line[line_len - 1] == '\n')
-    line_len--;
-  uint32_t col = coord.col;
-  if (col > line_len)
-    col = line_len;
-  uint32_t left = scan_left(line, line_len, col);
-  uint32_t right = scan_right(line, line_len, col);
-  if (prev_col)
-    *prev_col = left;
-  if (next_col)
-    *next_col = right;
-  free(it->buffer);
-  free(it);
-}
-
-uint32_t word_jump_right(const char *line, size_t len, uint32_t pos) {
-  if (pos >= len)
-    return len;
-  size_t next = grapheme_next_word_break_utf8(line + pos, len - pos);
-  return static_cast<uint32_t>(pos + next);
-}
-
-uint32_t word_jump_left(const char *line, size_t len, uint32_t col) {
-  if (col == 0)
-    return 0;
-  size_t pos = 0;
-  size_t last = 0;
-  size_t cursor = col;
-  while (pos < len) {
-    size_t next = pos + grapheme_next_word_break_utf8(line + pos, len - pos);
-    if (next >= cursor)
-      break;
-    last = next;
-    pos = next;
-  }
-  return static_cast<uint32_t>(last);
-}
-
-void word_boundaries(Editor *editor, Coord coord, uint32_t *prev_col,
-                     uint32_t *next_col, uint32_t *prev_clusters,
-                     uint32_t *next_clusters) {
-  if (!editor)
-    return;
-  std::shared_lock lock(editor->knot_mtx);
-  LineIterator *it = begin_l_iter(editor->root, coord.row);
-  if (!it)
-    return;
-  uint32_t line_len;
-  char *line = next_line(it, &line_len);
-  if (!line)
-    return;
-  if (line_len && line[line_len - 1] == '\n')
-    line_len--;
-  size_t col = coord.col;
-  if (col > line_len)
-    col = line_len;
-  size_t left = word_jump_left(line, line_len, col);
-  size_t right = word_jump_right(line, line_len, col);
-  if (prev_col)
-    *prev_col = static_cast<uint32_t>(left);
-  if (next_col)
-    *next_col = static_cast<uint32_t>(right);
-  if (prev_clusters)
-    *prev_clusters = count_clusters(line, line_len, left, col);
-  if (next_clusters)
-    *next_clusters = count_clusters(line, line_len, col, right);
-  free(it->buffer);
-  free(it);
-}
-
-Coord editor_hit_test(Editor *editor, uint32_t x, uint32_t y) {
-  if (mode == INSERT)
-    x++;
-  uint32_t numlen =
-      EXTRA_META + static_cast<int>(std::log10(editor->root->line_count + 1));
-  bool is_gutter_click = (x < numlen);
-  uint32_t render_width = editor->size.col - numlen;
-  x = MAX(x, numlen) - numlen;
-  uint32_t target_visual_row = y;
-  uint32_t visual_row = 0;
-  uint32_t line_index = editor->scroll.row;
-  uint32_t last_line_index = editor->scroll.row;
-  uint32_t last_col = editor->scroll.col;
-  bool first_visual_line = true;
-  std::shared_lock knot_lock(editor->knot_mtx);
-  LineIterator *it = begin_l_iter(editor->root, line_index);
-  if (!it)
-    return editor->scroll;
-  while (visual_row <= target_visual_row) {
-    const Fold *fold = fold_for_line(editor->folds, line_index);
-    if (fold) {
-      if (visual_row == target_visual_row) {
-        free(it->buffer);
-        free(it);
-        if (is_gutter_click) {
-          remove_fold(editor, fold->start);
-          return {UINT32_MAX, UINT32_MAX};
-        }
-        return {fold->start > 0 ? fold->start - 1 : 0, 0};
-      }
-      visual_row++;
-      while (line_index <= fold->end) {
-        char *l = next_line(it, nullptr);
-        if (!l)
-          break;
-        line_index++;
-      }
-      last_line_index = fold->end;
-      last_col = 0;
-      continue;
-    }
-    uint32_t line_len;
-    char *line = next_line(it, &line_len);
-    if (!line)
-      break;
-    if (line_len && line[line_len - 1] == '\n')
-      line_len--;
-    last_line_index = line_index;
-    last_col = line_len;
-    uint32_t offset = first_visual_line ? editor->scroll.col : 0;
-    first_visual_line = false;
-    while (offset < line_len || (line_len == 0 && offset == 0)) {
-      uint32_t col = 0;
-      uint32_t advance = 0;
-      uint32_t left = line_len - offset;
-      uint32_t last_good_offset = offset;
-      while (left > 0 && col < render_width) {
-        uint32_t g =
-            grapheme_next_character_break_utf8(line + offset + advance, left);
-        int w = display_width(line + offset + advance, g);
-        if (col + w > render_width)
-          break;
-        if (visual_row == target_visual_row && x < col + w) {
-          free(it->buffer);
-          free(it);
-          return {line_index, offset + advance};
-        }
-        advance += g;
-        last_good_offset = offset + advance;
-        left -= g;
-        col += w;
-      }
-      last_col = last_good_offset;
-      if (visual_row == target_visual_row) {
-        free(it->buffer);
-        free(it);
-        return {line_index, last_good_offset};
-      }
-      visual_row++;
-      if (visual_row > target_visual_row)
-        break;
-      if (advance == 0)
-        break;
-      offset += advance;
-      if (line_len == 0)
-        break;
-    }
-    line_index++;
-  }
-  free(it->buffer);
-  free(it);
-  return {last_line_index, last_col};
-}
-
-void move_line_up(Editor *editor) {
-  if (!editor || !editor->root || editor->cursor.row == 0)
-    return;
-  if (mode == NORMAL || mode == INSERT) {
-    uint32_t line_len, line_cluster_len;
-    std::shared_lock lock(editor->knot_mtx);
-    LineIterator *it = begin_l_iter(editor->root, editor->cursor.row);
-    char *line = next_line(it, &line_len);
-    if (!line) {
-      lock.unlock();
-      return;
-    }
-    if (line_len > 0 && line[line_len - 1] == '\n')
-      line_len--;
-    line_cluster_len = count_clusters(line, line_len, 0, line_len);
-    uint32_t target_row = prev_unfolded_row(editor, editor->cursor.row - 1);
-    uint32_t up_by = editor->cursor.row - target_row;
-    if (up_by > 1)
-      up_by--;
-    lock.unlock();
-    Coord cursor = editor->cursor;
-    edit_erase(editor, {cursor.row, 0}, line_cluster_len);
-    edit_erase(editor, {cursor.row, 0}, -1);
-    edit_insert(editor, {cursor.row - up_by, 0}, (char *)"\n", 1);
-    edit_insert(editor, {cursor.row - up_by, 0}, line, line_len);
-    free(it->buffer);
-    free(it);
-    editor->cursor = {cursor.row - up_by, cursor.col};
-  } else if (mode == SELECT) {
-    uint32_t start_row = MIN(editor->cursor.row, editor->selection.row);
-    uint32_t end_row = MAX(editor->cursor.row, editor->selection.row);
-    uint32_t start_byte = line_to_byte(editor->root, start_row, nullptr);
-    uint32_t end_byte = line_to_byte(editor->root, end_row + 1, nullptr);
-    char *selected_text = read(editor->root, start_byte, end_byte - start_byte);
-    if (!selected_text)
-      return;
-    uint32_t selected_len = count_clusters(selected_text, end_byte - start_byte,
-                                           0, end_byte - start_byte);
-    Coord cursor = editor->cursor;
-    Coord selection = editor->selection;
-    edit_erase(editor, {start_row, 0}, selected_len);
-    edit_insert(editor, {start_row - 1, 0}, selected_text,
-                end_byte - start_byte);
-    free(selected_text);
-    editor->cursor = {cursor.row - 1, cursor.col};
-    editor->selection = {selection.row - 1, selection.col};
-  }
-}
-
-void move_line_down(Editor *editor) {
-  if (!editor || !editor->root)
-    return;
-  if (mode == NORMAL || mode == INSERT) {
-    if (editor->cursor.row >= editor->root->line_count - 1)
-      return;
-    uint32_t line_len, line_cluster_len;
-    std::shared_lock lock(editor->knot_mtx);
-    LineIterator *it = begin_l_iter(editor->root, editor->cursor.row);
-    char *line = next_line(it, &line_len);
-    if (!line) {
-      lock.unlock();
-      return;
-    }
-    if (line_len && line[line_len - 1] == '\n')
-      line_len--;
-    line_cluster_len = count_clusters(line, line_len, 0, line_len);
-    uint32_t target_row = next_unfolded_row(editor, editor->cursor.row + 1);
-    if (target_row >= editor->root->line_count) {
-      free(line);
-      lock.unlock();
-      return;
-    }
-    uint32_t down_by = target_row - editor->cursor.row;
-    if (down_by > 1)
-      down_by--;
-    uint32_t ln;
-    line_to_byte(editor->root, editor->cursor.row + down_by - 1, &ln);
-    lock.unlock();
-    Coord cursor = editor->cursor;
-    edit_erase(editor, {cursor.row, 0}, line_cluster_len);
-    edit_erase(editor, {cursor.row, 0}, -1);
-    edit_insert(editor, {cursor.row + down_by, 0}, (char *)"\n", 1);
-    edit_insert(editor, {cursor.row + down_by, 0}, line, line_len);
-    free(it->buffer);
-    free(it);
-    editor->cursor = {cursor.row + down_by, cursor.col};
-  } else if (mode == SELECT) {
-    if (editor->cursor.row >= editor->root->line_count - 1 ||
-        editor->selection.row >= editor->root->line_count - 1)
-      return;
-    std::shared_lock lock(editor->knot_mtx);
-    uint32_t start_row = MIN(editor->cursor.row, editor->selection.row);
-    uint32_t end_row = MAX(editor->cursor.row, editor->selection.row);
-    uint32_t target_row = next_unfolded_row(editor, end_row + 1);
-    if (target_row >= editor->root->line_count)
-      return;
-    uint32_t down_by = target_row - end_row;
-    if (down_by > 1)
-      down_by--;
-    uint32_t start_byte = line_to_byte(editor->root, start_row, nullptr);
-    uint32_t end_byte = line_to_byte(editor->root, end_row + 1, nullptr);
-    char *selected_text = read(editor->root, start_byte, end_byte - start_byte);
-    lock.unlock();
-    if (!selected_text)
-      return;
-    uint32_t selected_len = count_clusters(selected_text, end_byte - start_byte,
-                                           0, end_byte - start_byte);
-    Coord cursor = editor->cursor;
-    Coord selection = editor->selection;
-    edit_erase(editor, {start_row, 0}, selected_len);
-    edit_insert(editor, {start_row + down_by, 0}, selected_text,
-                end_byte - start_byte);
-    free(selected_text);
-    editor->cursor = {cursor.row + down_by, cursor.col};
-    editor->selection = {selection.row + down_by, selection.col};
-  }
-}
-
-void edit_erase(Editor *editor, Coord pos, int64_t len) {
-  if (len == 0)
-    return;
-  if (len < 0) {
-    std::shared_lock lock_1(editor->knot_mtx);
-    uint32_t cursor_original =
-        line_to_byte(editor->root, editor->cursor.row, nullptr) +
-        editor->cursor.col;
-    TSPoint old_point = {pos.row, pos.col};
-    uint32_t byte_pos = line_to_byte(editor->root, pos.row, nullptr) + pos.col;
-    Coord point = move_left_pure(editor, pos, -len);
-    json lsp_range;
-    bool do_lsp = (editor->lsp != nullptr);
-    if (do_lsp) {
-      LineIterator *it = begin_l_iter(editor->root, point.row);
-      char *line = next_line(it, nullptr);
-      int utf16_start = 0;
-      if (line)
-        utf16_start = utf8_byte_offset_to_utf16(line, point.col);
-      free(it->buffer);
-      free(it);
-      it = begin_l_iter(editor->root, pos.row);
-      line = next_line(it, nullptr);
-      int utf16_end = 0;
-      if (line)
-        utf16_end = utf8_byte_offset_to_utf16(line, pos.col);
-      free(it->buffer);
-      free(it);
-      lsp_range = {{"start", {{"line", point.row}, {"character", utf16_start}}},
-                   {"end", {{"line", pos.row}, {"character", utf16_end}}}};
-    }
-    uint32_t start = line_to_byte(editor->root, point.row, nullptr) + point.col;
-    if (cursor_original > start && cursor_original <= byte_pos) {
-      editor->cursor = point;
-      editor->cursor_preffered = UINT32_MAX;
-    } else if (cursor_original > byte_pos) {
-      uint32_t cursor_new = cursor_original - (byte_pos - start);
-      uint32_t new_col;
-      uint32_t new_row = byte_to_line(editor->root, cursor_new, &new_col);
-      editor->cursor = {new_row, new_col};
-      editor->cursor_preffered = UINT32_MAX;
-    }
-    lock_1.unlock();
-    uint32_t start_row = point.row;
-    uint32_t end_row = pos.row;
-    apply_line_deletion(editor, start_row + 1, end_row);
-    apply_hook_deletion(editor, start_row + 1, end_row);
-    std::unique_lock lock_2(editor->knot_mtx);
-    editor->root = erase(editor->root, start, byte_pos - start);
-    lock_2.unlock();
-    if (editor->ts.tree) {
-      TSInputEdit edit = {
-          .start_byte = start,
-          .old_end_byte = byte_pos,
-          .new_end_byte = start,
-          .start_point = {point.row, point.col},
-          .old_end_point = old_point,
-          .new_end_point = {point.row, point.col},
-      };
-      editor->edit_queue.push(edit);
-    }
-    if (do_lsp) {
-      if (editor->lsp->incremental_sync) {
-        json message = {
-            {"jsonrpc", "2.0"},
-            {"method", "textDocument/didChange"},
-            {"params",
-             {{"textDocument",
-               {{"uri", editor->uri}, {"version", ++editor->lsp_version}}},
-              {"contentChanges",
-               json::array({{{"range", lsp_range}, {"text", ""}}})}}}};
-        lsp_send(editor->lsp, message, nullptr);
-      } else {
-        char *buf = read(editor->root, 0, editor->root->char_count);
-        std::string text(buf);
-        free(buf);
-        json message = {
-            {"jsonrpc", "2.0"},
-            {"method", "textDocument/didChange"},
-            {"params",
-             {{"textDocument",
-               {{"uri", editor->uri}, {"version", ++editor->lsp_version}}},
-              {"contentChanges", json::array({{{"text", text}}})}}}};
-        lsp_send(editor->lsp, message, nullptr);
-      }
-    }
-    std::unique_lock lock_3(editor->spans.mtx);
-    apply_edit(editor->spans.spans, start, start - byte_pos);
-    if (editor->spans.mid_parse)
-      editor->spans.edits.push({start, start - byte_pos});
-    std::unique_lock lock_4(editor->def_spans.mtx);
-    apply_edit(editor->def_spans.spans, byte_pos, start - byte_pos);
-  } else {
-    std::shared_lock lock_1(editor->knot_mtx);
-    uint32_t cursor_original =
-        line_to_byte(editor->root, editor->cursor.row, nullptr) +
-        editor->cursor.col;
-    TSPoint old_point = {pos.row, pos.col};
-    uint32_t byte_pos = line_to_byte(editor->root, pos.row, nullptr) + pos.col;
-    Coord point = move_right_pure(editor, pos, len);
-    json lsp_range;
-    bool do_lsp = (editor->lsp != nullptr);
-    if (do_lsp) {
-      LineIterator *it = begin_l_iter(editor->root, pos.row);
-      char *line = next_line(it, nullptr);
-      int utf16_start = 0;
-      if (line)
-        utf16_start = utf8_byte_offset_to_utf16(line, pos.col);
-      free(it->buffer);
-      free(it);
-      it = begin_l_iter(editor->root, point.row);
-      line = next_line(it, nullptr);
-      int utf16_end = 0;
-      if (line)
-        utf16_end = utf8_byte_offset_to_utf16(line, point.col);
-      free(it->buffer);
-      free(it);
-      lsp_range = {{"start", {{"line", pos.row}, {"character", utf16_start}}},
-                   {"end", {{"line", point.row}, {"character", utf16_end}}}};
-    }
-    uint32_t end = line_to_byte(editor->root, point.row, nullptr) + point.col;
-    if (cursor_original > byte_pos && cursor_original <= end) {
-      editor->cursor = pos;
-      editor->cursor_preffered = UINT32_MAX;
-    } else if (cursor_original > end) {
-      uint32_t cursor_new = cursor_original - (end - byte_pos);
-      uint32_t new_col;
-      uint32_t new_row = byte_to_line(editor->root, cursor_new, &new_col);
-      editor->cursor = {new_row, new_col};
-      editor->cursor_preffered = UINT32_MAX;
-    }
-    lock_1.unlock();
-    uint32_t start_row = pos.row;
-    uint32_t end_row = point.row;
-    apply_line_deletion(editor, start_row + 1, end_row);
-    apply_hook_deletion(editor, start_row + 1, end_row);
-    std::unique_lock lock_2(editor->knot_mtx);
-    editor->root = erase(editor->root, byte_pos, end - byte_pos);
-    lock_2.unlock();
-    if (editor->ts.tree) {
-      TSInputEdit edit = {
-          .start_byte = byte_pos,
-          .old_end_byte = end,
-          .new_end_byte = byte_pos,
-          .start_point = old_point,
-          .old_end_point = {point.row, point.col},
-          .new_end_point = old_point,
-      };
-      editor->edit_queue.push(edit);
-    }
-    if (do_lsp) {
-      if (editor->lsp->incremental_sync) {
-        json message = {
-            {"jsonrpc", "2.0"},
-            {"method", "textDocument/didChange"},
-            {"params",
-             {{"textDocument",
-               {{"uri", editor->uri}, {"version", ++editor->lsp_version}}},
-              {"contentChanges",
-               json::array({{{"range", lsp_range}, {"text", ""}}})}}}};
-        lsp_send(editor->lsp, message, nullptr);
-      } else {
-        char *buf = read(editor->root, 0, editor->root->char_count);
-        std::string text(buf);
-        free(buf);
-        json message = {
-            {"jsonrpc", "2.0"},
-            {"method", "textDocument/didChange"},
-            {"params",
-             {{"textDocument",
-               {{"uri", editor->uri}, {"version", ++editor->lsp_version}}},
-              {"contentChanges", json::array({{{"text", text}}})}}}};
-        lsp_send(editor->lsp, message, nullptr);
-      }
-    }
-    std::unique_lock lock_3(editor->spans.mtx);
-    apply_edit(editor->spans.spans, byte_pos, byte_pos - end);
-    if (editor->spans.mid_parse)
-      editor->spans.edits.push({byte_pos, byte_pos - end});
-    std::unique_lock lock_4(editor->def_spans.mtx);
-    apply_edit(editor->def_spans.spans, byte_pos, byte_pos - end);
-  }
-}
-
-void edit_insert(Editor *editor, Coord pos, char *data, uint32_t len) {
-  std::shared_lock lock_1(editor->knot_mtx);
-  uint32_t cursor_original =
-      line_to_byte(editor->root, editor->cursor.row, nullptr) +
-      editor->cursor.col;
-  uint32_t byte_pos = line_to_byte(editor->root, pos.row, nullptr) + pos.col;
-  TSPoint start_point = {pos.row, pos.col};
-  if (cursor_original > byte_pos) {
-    uint32_t cursor_new = cursor_original + len;
-    uint32_t new_col;
-    uint32_t new_row = byte_to_line(editor->root, cursor_new, &new_col);
-    editor->cursor = {new_row, new_col};
-  }
-  lock_1.unlock();
-  std::unique_lock lock_2(editor->knot_mtx);
-  editor->root = insert(editor->root, byte_pos, data, len);
-  lock_2.unlock();
-  uint32_t cols = 0;
-  uint32_t rows = 0;
-  for (uint32_t i = 0; i < len; i++) {
-    if (data[i] == '\n') {
-      rows++;
-      cols = 0;
-    } else {
-      cols++;
-    }
-  }
-  apply_line_insertion(editor, pos.row, rows);
-  apply_hook_insertion(editor, pos.row, rows);
-  if (editor->ts.tree) {
-    TSInputEdit edit = {
-        .start_byte = byte_pos,
-        .old_end_byte = byte_pos,
-        .new_end_byte = byte_pos + len,
-        .start_point = start_point,
-        .old_end_point = start_point,
-        .new_end_point = {start_point.row + rows,
-                          (rows == 0) ? (start_point.column + cols) : cols},
-    };
-    editor->edit_queue.push(edit);
-  }
-  if (editor->lsp) {
-    if (editor->lsp->incremental_sync) {
-      lock_1.lock();
-      LineIterator *it = begin_l_iter(editor->root, pos.row);
-      char *line = next_line(it, nullptr);
-      int utf16_col = 0;
-      if (line)
-        utf16_col = utf8_byte_offset_to_utf16(line, pos.col);
-      free(it->buffer);
-      free(it);
-      lock_1.unlock();
-      json message = {
-          {"jsonrpc", "2.0"},
-          {"method", "textDocument/didChange"},
-          {"params",
-           {{"textDocument",
-             {{"uri", editor->uri}, {"version", ++editor->lsp_version}}},
-            {"contentChanges",
-             json::array(
-                 {{{"range",
-                    {{"start", {{"line", pos.row}, {"character", utf16_col}}},
-                     {"end", {{"line", pos.row}, {"character", utf16_col}}}}},
-                   {"text", std::string(data, len)}}})}}}};
-      lsp_send(editor->lsp, message, nullptr);
-    } else {
-      char *buf = read(editor->root, 0, editor->root->char_count);
-      std::string text(buf);
-      free(buf);
-      json message = {
-          {"jsonrpc", "2.0"},
-          {"method", "textDocument/didChange"},
-          {"params",
-           {{"textDocument",
-             {{"uri", editor->uri}, {"version", ++editor->lsp_version}}},
-            {"contentChanges", json::array({{{"text", text}}})}}}};
-      lsp_send(editor->lsp, message, nullptr);
-    }
-  }
-  std::unique_lock lock_3(editor->spans.mtx);
-  apply_edit(editor->spans.spans, byte_pos, len);
-  if (editor->spans.mid_parse)
-    editor->spans.edits.push({byte_pos, len});
-  std::unique_lock lock_4(editor->def_spans.mtx);
-  apply_edit(editor->def_spans.spans, byte_pos, len);
-}
-
-char *get_selection(Editor *editor, uint32_t *out_len, Coord *out_start) {
-  std::shared_lock lock(editor->knot_mtx);
-  Coord start, end;
-  if (editor->cursor >= editor->selection) {
-    uint32_t prev_col, next_col;
-    switch (editor->selection_type) {
-    case CHAR:
-      start = editor->selection;
-      end = move_right(editor, editor->cursor, 1);
-      break;
-    case WORD:
-      word_boundaries(editor, editor->selection, &prev_col, &next_col, nullptr,
-                      nullptr);
-      start = {editor->selection.row, prev_col};
-      end = editor->cursor;
-      break;
-    case LINE:
-      start = {editor->selection.row, 0};
-      end = editor->cursor;
-      break;
-    }
-  } else {
-    start = editor->cursor;
-    uint32_t prev_col, next_col, line_len;
-    switch (editor->selection_type) {
-    case CHAR:
-      end = move_right(editor, editor->selection, 1);
-      break;
-    case WORD:
-      word_boundaries(editor, editor->selection, &prev_col, &next_col, nullptr,
-                      nullptr);
-      end = {editor->selection.row, next_col};
-      break;
-    case LINE:
-      LineIterator *it = begin_l_iter(editor->root, editor->selection.row);
-      char *line = next_line(it, &line_len);
-      if (!line)
-        return nullptr;
-      if (line_len > 0 && line[line_len - 1] == '\n')
-        line_len--;
-      end = {editor->selection.row, line_len};
-      free(it->buffer);
-      free(it);
-      break;
-    }
-  }
-  if (out_start)
-    *out_start = start;
-  uint32_t start_byte =
-      line_to_byte(editor->root, start.row, nullptr) + start.col;
-  uint32_t end_byte = line_to_byte(editor->root, end.row, nullptr) + end.col;
-  char *text = read(editor->root, start_byte, end_byte - start_byte);
-  if (out_len)
-    *out_len = end_byte - start_byte;
-  return text;
-}
-
-void apply_edit(std::vector<Span> &spans, uint32_t x, int64_t y) {
-  Span key{.start = x, .end = 0, .hl = nullptr};
-  auto it = std::lower_bound(
-      spans.begin(), spans.end(), key,
-      [](const Span &a, const Span &b) { return a.start < b.start; });
-  size_t idx = std::distance(spans.begin(), it);
-  while (idx > 0 && spans.at(idx - 1).end >= x)
-    --idx;
-  for (size_t i = idx; i < spans.size();) {
-    Span &s = spans.at(i);
-    if (s.start < x && s.end >= x) {
-      s.end += y;
-    } else if (s.start > x) {
-      s.start += y;
-      s.end += y;
-    }
-    if (s.end <= s.start)
-      spans.erase(spans.begin() + i);
-    else
-      ++i;
-  }
-}
-
-std::vector<Fold>::iterator find_fold_iter(Editor *editor, uint32_t line) {
-  auto &folds = editor->folds;
-  auto it = std::lower_bound(
-      folds.begin(), folds.end(), line,
-      [](const Fold &fold, uint32_t value) { return fold.start < value; });
-  if (it != folds.end() && it->start == line)
-    return it;
-  if (it != folds.begin()) {
-    --it;
-    if (it->contains(line))
-      return it;
-  }
-  return folds.end();
-}
-
-bool add_fold(Editor *editor, uint32_t start, uint32_t end) {
-  if (!editor || !editor->root)
-    return false;
-  if (start > end)
-    std::swap(start, end);
-  if (start >= editor->root->line_count)
-    return false;
-  end = std::min(end, editor->root->line_count - 1);
-  if (start == end)
-    return false;
-  Fold new_fold{start, end};
-  auto &folds = editor->folds;
-  auto it = std::lower_bound(
-      folds.begin(), folds.end(), new_fold.start,
-      [](const Fold &fold, uint32_t value) { return fold.start < value; });
-  if (it != folds.begin()) {
-    auto prev = std::prev(it);
-    if (prev->end + 1 >= new_fold.start) {
-      new_fold.start = std::min(new_fold.start, prev->start);
-      new_fold.end = std::max(new_fold.end, prev->end);
-      it = folds.erase(prev);
-    }
-  }
-  while (it != folds.end() && it->start <= new_fold.end + 1) {
-    new_fold.end = std::max(new_fold.end, it->end);
-    it = folds.erase(it);
-  }
-  folds.insert(it, new_fold);
-  return true;
-}
-
-bool remove_fold(Editor *editor, uint32_t line) {
-  auto it = find_fold_iter(editor, line);
-  if (it == editor->folds.end())
-    return false;
-  editor->folds.erase(it);
-  return true;
-}
-
-void apply_line_insertion(Editor *editor, uint32_t line, uint32_t rows) {
-  for (auto it = editor->folds.begin(); it != editor->folds.end();) {
-    if (line <= it->start) {
-      it->start += rows;
-      it->end += rows;
-      ++it;
-    } else if (line <= it->end) {
-      it = editor->folds.erase(it);
-    } else {
-      ++it;
-    }
-  }
-}
-
-void apply_line_deletion(Editor *editor, uint32_t removal_start,
-                         uint32_t removal_end) {
-  if (removal_start > removal_end)
-    return;
-  uint32_t rows_removed = removal_end - removal_start + 1;
-  std::vector<Fold> updated;
-  updated.reserve(editor->folds.size());
-  for (auto fold : editor->folds) {
-    if (removal_end < fold.start) {
-      fold.start -= rows_removed;
-      fold.end -= rows_removed;
-      updated.push_back(fold);
-      continue;
-    }
-    if (removal_start > fold.end) {
-      updated.push_back(fold);
-      continue;
-    }
-  }
-  editor->folds.swap(updated);
-}
-
-void apply_hook_insertion(Editor *editor, uint32_t line, uint32_t rows) {
-  for (auto &hook : editor->hooks)
-    if (hook > line)
-      hook += rows;
-}
-
-void apply_hook_deletion(Editor *editor, uint32_t removal_start,
-                         uint32_t removal_end) {
-  for (auto &hook : editor->hooks)
-    if (hook > removal_start)
-      hook -= removal_end - removal_start + 1;
-}

src/io/input.cc

@@ -1,12 +1,4 @@
-extern "C" {
+#include "io/ui.h"
-#include "../libs/libgrapheme/grapheme.h"
-}
-#include "../include/ui.h"
-#include <cstdint>
-#include <cstdio>
-#include <cstdlib>
-#include <cstring>
-#include <unistd.h>
 
 static Queue<char> input_queue;
 

src/io/knot.cc

@@ -1,12 +1,4 @@
-#include "../include/knot.h"
+#include "io/knot.h"
-#include <assert.h>
-#include <cmath>
-#include <cstdint>
-#include <cstdlib>
-#include <cstring>
-#include <pcre2.h>
-#include <stdio.h>
-#include <string.h>
 
 static void update(Knot *n) {
   if (!n)

src/io/renderer.cc

@@ -1,5 +1,4 @@
-#include "../include/ui.h"
+#include "io/ui.h"
-#include "../include/utils.h"
 
 uint32_t rows, cols;
 bool show_cursor = 0;

src/lsp.cc

@@ -1,425 +0,0 @@
-#include "../include/lsp.h"
-#include "../include/maps.h"
-#include <cmath>
-#include <fcntl.h>
-#include <signal.h>
-#include <sys/poll.h>
-#include <sys/types.h>
-#include <sys/wait.h>
-#include <unistd.h>
-
-std::shared_mutex active_lsps_mtx;
-std::unordered_map<uint8_t, std::shared_ptr<LSPInstance>> active_lsps;
-
-Queue<LSPOpenRequest> lsp_open_queue;
-
-static bool init_lsp(std::shared_ptr<LSPInstance> lsp) {
-  int in_pipe[2];
-  int out_pipe[2];
-  if (pipe(in_pipe) == -1 || pipe(out_pipe) == -1) {
-    perror("pipe");
-    return false;
-  }
-  pid_t pid = fork();
-  if (pid == -1) {
-    perror("fork");
-    return false;
-  }
-  if (pid == 0) {
-    dup2(in_pipe[0], STDIN_FILENO);
-    dup2(out_pipe[1], STDOUT_FILENO);
-#ifdef __clang__
-    int devnull = open("/dev/null", O_WRONLY);
-    if (devnull >= 0) {
-      dup2(devnull, STDERR_FILENO);
-      close(devnull);
-    }
-#else
-    int log = open("/tmp/lsp.log", O_WRONLY | O_CREAT | O_TRUNC, 0644);
-    if (log >= 0) {
-      dup2(log, STDERR_FILENO);
-      close(log);
-    }
-#endif
-    close(in_pipe[0]);
-    close(in_pipe[1]);
-    close(out_pipe[0]);
-    close(out_pipe[1]);
-    execvp(lsp->lsp->command, (char *const *)(lsp->lsp->args.data()));
-    perror("execvp");
-    _exit(127);
-  }
-  lsp->pid = pid;
-  lsp->stdin_fd = in_pipe[1];
-  lsp->stdout_fd = out_pipe[0];
-  close(in_pipe[0]);
-  close(out_pipe[1]);
-  return true;
-}
-
-std::shared_ptr<LSPInstance> get_or_init_lsp(uint8_t lsp_id) {
-  std::unique_lock lock(active_lsps_mtx);
-  auto it = active_lsps.find(lsp_id);
-  if (it == active_lsps.end()) {
-    auto map_it = kLsps.find(lsp_id);
-    if (map_it == kLsps.end())
-      return nullptr;
-    std::shared_ptr<LSPInstance> lsp = std::make_shared<LSPInstance>();
-    lsp->lsp = &map_it->second;
-    if (!init_lsp(lsp))
-      return nullptr;
-    LSPPending *pending = new LSPPending();
-    pending->method = "initialize";
-    pending->editor = nullptr;
-    pending->callback = [lsp](Editor *, std::string, json msg) {
-      if (msg.contains("result") && msg["result"].contains("capabilities")) {
-        auto &caps = msg["result"]["capabilities"];
-        if (caps.contains("textDocumentSync")) {
-          auto &sync = caps["textDocumentSync"];
-          if (sync.is_number()) {
-            int change_type = sync.get<int>();
-            lsp->incremental_sync = (change_type == 2);
-          } else if (sync.is_object() && sync.contains("change")) {
-            int change_type = sync["change"].get<int>();
-            lsp->incremental_sync = (change_type == 2);
-          }
-        }
-        if (caps.contains("hoverProvider"))
-          lsp->allow_hover = caps["hoverProvider"].get<bool>();
-        else
-          lsp->allow_hover = false;
-        if (caps.contains("completionProvider"))
-          lsp->allow_completion = true;
-      }
-      lsp->initialized = true;
-      json initialized = {{"jsonrpc", "2.0"},
-                          {"method", "initialized"},
-                          {"params", json::object()}};
-      lsp_send(lsp, initialized, nullptr);
-      while (!lsp->open_queue.empty()) {
-        std::pair<Language, Editor *> request;
-        lsp->open_queue.pop(request);
-        open_editor(lsp, request);
-      }
-    };
-    json init_message = {
-        {"jsonrpc", "2.0"},
-        {"method", "initialize"},
-        {"params",
-         {{"processId", getpid()},
-          {"rootUri", "file://" + percent_encode(path_abs("."))},
-          {"capabilities",
-           {{"textDocument",
-             {{"publishDiagnostics", {{"relatedInformation", true}}},
-              {"hover", {{"contentFormat", {"markdown", "plaintext"}}}},
-              {"completion",
-               {{"completionItem",
-                 {{"snippetSupport", true},
-                  {"documentationFormat", {"markdown", "plaintext"}},
-                  {"resolveSupport",
-                   {{"properties", {"documentation", "detail"}}}},
-                  {"insertReplaceSupport", true},
-                  {"labelDetailsSupport", true},
-                  {"insertTextModeSupport", {{"valueSet", json::array({1})}}}}},
-                {"completionItemKind",
-                 {{"valueSet", {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}}}},
-                {"contextSupport", true},
-                {"insertTextMode", 1}}}}}}}}}};
-    lsp_send(lsp, init_message, pending);
-    active_lsps[lsp_id] = lsp;
-    return lsp;
-  }
-  return it->second;
-}
-
-void lsp_send(std::shared_ptr<LSPInstance> lsp, json message,
-              LSPPending *pending) {
-  if (!lsp || lsp->stdin_fd == -1)
-    return;
-  std::unique_lock lock(lsp->mtx);
-  if (pending) {
-    message["id"] = lsp->last_id++;
-    uint32_t id = message["id"].get<uint32_t>();
-    lsp->pending[id] = pending;
-  }
-  lsp->outbox.push(message);
-}
-
-void close_lsp(uint8_t lsp_id) {
-  std::shared_lock active_lsps_lock(active_lsps_mtx);
-  auto it = active_lsps.find(lsp_id);
-  if (it == active_lsps.end())
-    return;
-  std::shared_ptr<LSPInstance> lsp = it->second;
-  active_lsps_lock.unlock();
-  lsp->exited = true;
-  lsp->initialized = false;
-  LSPPending *shutdown_pending = new LSPPending();
-  shutdown_pending->method = "shutdown";
-  shutdown_pending->callback = [lsp, lsp_id](Editor *, std::string, json) {
-    json exit = {{"jsonrpc", "2.0"}, {"method", "exit"}};
-    lsp_send(lsp, exit, nullptr);
-  };
-  json shutdown = {{"jsonrpc", "2.0"}, {"method", "shutdown"}};
-  lsp_send(lsp, shutdown, shutdown_pending);
-  std::thread t([lsp, lsp_id] {
-    std::this_thread::sleep_for(100ms);
-    std::unique_lock active_lsps_lock(active_lsps_mtx);
-    std::unique_lock lock(lsp->mtx);
-    if (lsp->pid != -1 && kill(lsp->pid, 0) == 0)
-      kill(lsp->pid, SIGKILL);
-    waitpid(lsp->pid, nullptr, 0);
-    close(lsp->stdin_fd);
-    close(lsp->stdout_fd);
-    for (auto &kv : lsp->pending)
-      delete kv.second;
-    for (auto &editor : lsp->editors) {
-      std::unique_lock editor_lock(editor->lsp_mtx);
-      editor->lsp = nullptr;
-    }
-    active_lsps.erase(lsp_id);
-  });
-  t.detach();
-}
-
-static std::optional<json> read_lsp_message(int fd) {
-  std::string header;
-  char c;
-  while (true) {
-    ssize_t n = read(fd, &c, 1);
-    if (n <= 0)
-      return std::nullopt;
-    header.push_back(c);
-    if (header.size() >= 4 && header.substr(header.size() - 4) == "\r\n\r\n")
-      break;
-  }
-  size_t pos = header.find("Content-Length:");
-  if (pos == std::string::npos)
-    return std::nullopt;
-  pos += strlen("Content-Length:");
-  while (pos < header.size() && std::isspace(header[pos]))
-    pos++;
-  size_t end = pos;
-  while (end < header.size() && std::isdigit(header[end]))
-    end++;
-  size_t len = std::stoul(header.substr(pos, end - pos));
-  std::string body(len, '\0');
-  size_t got = 0;
-  while (got < len) {
-    ssize_t n = read(fd, &body[got], len - got);
-    if (n <= 0)
-      return std::nullopt;
-    got += n;
-  }
-  return json::parse(body);
-}
-
-static Editor *editor_for_uri(std::shared_ptr<LSPInstance> lsp,
-                              std::string uri) {
-  if (uri.empty())
-    return nullptr;
-  for (auto &editor : lsp->editors)
-    if (editor->uri == uri)
-      return editor;
-  return nullptr;
-}
-
-static void clean_lsp(std::shared_ptr<LSPInstance> lsp, uint8_t lsp_id) {
-  for (auto &kv : lsp->pending)
-    delete kv.second;
-  lsp->pid = -1;
-  close(lsp->stdin_fd);
-  close(lsp->stdout_fd);
-  for (auto &editor : lsp->editors) {
-    std::unique_lock editor_lock(editor->lsp_mtx);
-    editor->lsp = nullptr;
-  }
-  active_lsps.erase(lsp_id);
-}
-
-void lsp_worker() {
-  LSPOpenRequest request;
-  while (lsp_open_queue.pop(request))
-    add_to_lsp(request.language, request.editor);
-  std::unique_lock active_lsps_lock(active_lsps_mtx);
-  for (auto &kv : active_lsps) {
-    std::shared_ptr<LSPInstance> lsp = kv.second;
-    std::unique_lock lock(lsp->mtx);
-    int status;
-    pid_t res = waitpid(lsp->pid, &status, WNOHANG);
-    if (res == lsp->pid) {
-      clean_lsp(lsp, kv.first);
-      return;
-    }
-    while (!lsp->outbox.empty()) {
-      json message = lsp->outbox.front();
-      std::string m = message.value("method", "");
-      if (lsp->exited) {
-        if (m != "exit" && m != "shutdown") {
-          lsp->outbox.pop(message);
-          continue;
-        }
-      }
-      if (!lsp->initialized) {
-        if (m != "initialize" && m != "exit" && m != "shutdown")
-          break;
-      }
-      lsp->outbox.pop(message);
-      std::string payload = message.dump();
-      std::string header =
-          "Content-Length: " + std::to_string(payload.size()) + "\r\n\r\n";
-      std::string out = header + payload;
-      const char *ptr = out.data();
-      size_t remaining = out.size();
-      while (remaining > 0) {
-        int status;
-        pid_t res = waitpid(lsp->pid, &status, WNOHANG);
-        if (res == lsp->pid) {
-          clean_lsp(lsp, kv.first);
-          return;
-        }
-        ssize_t written = write(lsp->stdin_fd, ptr, remaining);
-        if (written == 0)
-          break;
-        else if (written == -1) {
-          if (errno == EINTR)
-            continue;
-          perror("write");
-          clean_lsp(lsp, kv.first);
-          return;
-        } else {
-          ptr += written;
-          remaining -= written;
-        }
-      }
-    }
-    pollfd pfd{lsp->stdout_fd, POLLIN | POLLHUP | POLLERR, 0};
-    int r = poll(&pfd, 1, 0);
-    if (r > 0 && pfd.revents & (POLLHUP | POLLERR)) {
-      clean_lsp(lsp, kv.first);
-      return;
-    }
-    while ((r = poll(&pfd, 1, 0) > 0)) {
-      if (r > 0 && pfd.revents & (POLLHUP | POLLERR)) {
-        clean_lsp(lsp, kv.first);
-        return;
-      }
-      auto msg = read_lsp_message(lsp->stdout_fd);
-      if (!msg)
-        break;
-      if (msg->contains("id")) {
-        uint32_t id = msg->at("id").get<uint32_t>();
-        auto it = lsp->pending.find(id);
-        if (it != lsp->pending.end()) {
-          LSPPending *pend = it->second;
-          lock.unlock();
-          if (pend->callback)
-            pend->callback(pend->editor, pend->method, *msg);
-          delete pend;
-          lock.lock();
-          lsp->pending.erase(it);
-        }
-      } else if (msg->contains("method")) {
-        std::string uri;
-        if (msg->contains("params")) {
-          auto &p = (*msg)["params"];
-          if (p.contains("textDocument") && p["textDocument"].contains("uri"))
-            uri = p["textDocument"]["uri"].get<std::string>();
-          else if (p.contains("uri"))
-            uri = p["uri"].get<std::string>();
-        }
-        Editor *ed = editor_for_uri(lsp, uri);
-        lock.unlock();
-        if (ed)
-          editor_lsp_handle(ed, *msg);
-        else
-          lsp_handle(lsp, *msg);
-        lock.lock();
-      }
-    }
-  }
-}
-
-void request_add_to_lsp(Language language, Editor *editor) {
-  lsp_open_queue.push({language, editor});
-}
-
-void add_to_lsp(Language language, Editor *editor) {
-  std::shared_ptr<LSPInstance> lsp = get_or_init_lsp(language.lsp_id);
-  if (!lsp)
-    return;
-  std::unique_lock lock(lsp->mtx);
-  if (editor->lsp == lsp)
-    return;
-  lsp->editors.push_back(editor);
-  lsp->open_queue.push({language, editor});
-  lock.unlock();
-}
-
-void open_editor(std::shared_ptr<LSPInstance> lsp,
-                 std::pair<Language, Editor *> entry) {
-  Language language = entry.first;
-  Editor *editor = entry.second;
-  std::unique_lock lock2(editor->lsp_mtx);
-  editor->lsp = lsp;
-  lock2.unlock();
-  std::unique_lock lock3(editor->knot_mtx);
-  char *buf = read(editor->root, 0, editor->root->char_count);
-  std::string text(buf);
-  free(buf);
-  json message = {{"jsonrpc", "2.0"},
-                  {"method", "textDocument/didOpen"},
-                  {"params",
-                   {{"textDocument",
-                     {{"uri", editor->uri},
-                      {"languageId", language.name},
-                      {"version", 1},
-                      {"text", text}}}}}};
-  lock3.unlock();
-  lsp_send(lsp, message, nullptr);
-}
-
-static uint8_t find_lsp_id(std::shared_ptr<LSPInstance> needle) {
-  for (const auto &[id, lsp] : active_lsps)
-    if (lsp == needle)
-      return id;
-  return 0;
-}
-
-void remove_from_lsp(Editor *editor) {
-  auto lsp = editor->lsp;
-  if (!lsp)
-    return;
-  std::unique_lock lock1(lsp->mtx);
-  lsp->editors.erase(
-      std::remove(lsp->editors.begin(), lsp->editors.end(), editor),
-      lsp->editors.end());
-  lock1.unlock();
-  std::unique_lock lock2(editor->lsp_mtx);
-  editor->lsp = nullptr;
-  lock2.unlock();
-  json message = {{"jsonrpc", "2.0"},
-                  {"method", "textDocument/didClose"},
-                  {"params", {{"textDocument", {{"uri", editor->uri}}}}}};
-  lsp_send(lsp, message, nullptr);
-  uint8_t lsp_id = find_lsp_id(lsp);
-  if (lsp_id && lsp->editors.empty())
-    close_lsp(lsp_id);
-}
-
-void lsp_handle(std::shared_ptr<LSPInstance>, json message) {
-  std::string method = message.value("method", "");
-  if (method == "window/showMessage") {
-    if (message.contains("params")) {
-      auto &p = message["params"];
-      if (p.contains("message"))
-        log("%s\n", p["message"].get<std::string>().c_str());
-    }
-  } else if (method == "window/logMessage") {
-    if (message.contains("params")) {
-      auto &p = message["params"];
-      if (p.contains("message"))
-        log("%s\n", p["message"].get<std::string>().c_str());
-    }
-  }
-}

src/lsp/handlers.cc

@@ -0,0 +1,87 @@
+#include "lsp/lsp.h"
+
+Queue<LSPOpenRequest> lsp_open_queue;
+
+void request_add_to_lsp(Language language, Editor *editor) {
+  lsp_open_queue.push({language, editor});
+}
+
+void add_to_lsp(Language language, Editor *editor) {
+  std::shared_ptr<LSPInstance> lsp = get_or_init_lsp(language.lsp_id);
+  if (!lsp)
+    return;
+  std::unique_lock lock(lsp->mtx);
+  if (editor->lsp == lsp)
+    return;
+  lsp->editors.push_back(editor);
+  lsp->open_queue.push({language, editor});
+  lock.unlock();
+}
+
+void open_editor(std::shared_ptr<LSPInstance> lsp,
+                 std::pair<Language, Editor *> entry) {
+  Language language = entry.first;
+  Editor *editor = entry.second;
+  if (editor->lsp == lsp)
+    return;
+  editor->lsp = lsp;
+  std::unique_lock lock3(editor->knot_mtx);
+  char *buf = read(editor->root, 0, editor->root->char_count);
+  std::string text(buf);
+  free(buf);
+  json message = {{"jsonrpc", "2.0"},
+                  {"method", "textDocument/didOpen"},
+                  {"params",
+                   {{"textDocument",
+                     {{"uri", editor->uri},
+                      {"languageId", language.name},
+                      {"version", 1},
+                      {"text", text}}}}}};
+  lock3.unlock();
+  lsp_send(lsp, message, nullptr);
+}
+
+static uint8_t find_lsp_id(std::shared_ptr<LSPInstance> needle) {
+  for (const auto &[id, lsp] : active_lsps)
+    if (lsp == needle)
+      return id;
+  return 0;
+}
+
+void remove_from_lsp(Editor *editor) {
+  auto lsp = editor->lsp;
+  if (!lsp)
+    return;
+  std::unique_lock lock1(lsp->mtx);
+  lsp->editors.erase(
+      std::remove(lsp->editors.begin(), lsp->editors.end(), editor),
+      lsp->editors.end());
+  lock1.unlock();
+  std::unique_lock lock2(editor->lsp_mtx);
+  editor->lsp = nullptr;
+  lock2.unlock();
+  json message = {{"jsonrpc", "2.0"},
+                  {"method", "textDocument/didClose"},
+                  {"params", {{"textDocument", {{"uri", editor->uri}}}}}};
+  lsp_send(lsp, message, nullptr);
+  uint8_t lsp_id = find_lsp_id(lsp);
+  if (lsp_id && lsp->editors.empty())
+    close_lsp(lsp_id);
+}
+
+void lsp_handle(std::shared_ptr<LSPInstance>, json message) {
+  std::string method = message.value("method", "");
+  if (method == "window/showMessage") {
+    if (message.contains("params")) {
+      auto &p = message["params"];
+      if (p.contains("message"))
+        log("%s\n", p["message"].get<std::string>().c_str());
+    }
+  } else if (method == "window/logMessage") {
+    if (message.contains("params")) {
+      auto &p = message["params"];
+      if (p.contains("message"))
+        log("%s\n", p["message"].get<std::string>().c_str());
+    }
+  }
+}

src/lsp/process.cc

@@ -0,0 +1,150 @@
+#include "config.h"
+#include "lsp/lsp.h"
+
+static bool init_lsp(std::shared_ptr<LSPInstance> lsp) {
+  int in_pipe[2];
+  int out_pipe[2];
+  if (pipe(in_pipe) == -1 || pipe(out_pipe) == -1) {
+    perror("pipe");
+    return false;
+  }
+  pid_t pid = fork();
+  if (pid == -1) {
+    perror("fork");
+    return false;
+  }
+  if (pid == 0) {
+    dup2(in_pipe[0], STDIN_FILENO);
+    dup2(out_pipe[1], STDOUT_FILENO);
+#ifdef __clang__
+    int devnull = open("/dev/null", O_WRONLY);
+    if (devnull >= 0) {
+      dup2(devnull, STDERR_FILENO);
+      close(devnull);
+    }
+#else
+    int log = open("/tmp/lsp.log", O_WRONLY | O_CREAT | O_TRUNC, 0644);
+    if (log >= 0) {
+      dup2(log, STDERR_FILENO);
+      close(log);
+    }
+#endif
+    close(in_pipe[0]);
+    close(in_pipe[1]);
+    close(out_pipe[0]);
+    close(out_pipe[1]);
+    execvp(lsp->lsp->command, (char *const *)(lsp->lsp->args.data()));
+    perror("execvp");
+    _exit(127);
+  }
+  lsp->pid = pid;
+  lsp->stdin_fd = in_pipe[1];
+  lsp->stdout_fd = out_pipe[0];
+  close(in_pipe[0]);
+  close(out_pipe[1]);
+  return true;
+}
+
+std::shared_ptr<LSPInstance> get_or_init_lsp(uint8_t lsp_id) {
+  std::unique_lock lock(active_lsps_mtx);
+  auto it = active_lsps.find(lsp_id);
+  if (it == active_lsps.end()) {
+    auto map_it = kLsps.find(lsp_id);
+    if (map_it == kLsps.end())
+      return nullptr;
+    std::shared_ptr<LSPInstance> lsp = std::make_shared<LSPInstance>();
+    lsp->lsp = &map_it->second;
+    if (!init_lsp(lsp))
+      return nullptr;
+    LSPPending *pending = new LSPPending();
+    pending->method = "initialize";
+    pending->editor = nullptr;
+    pending->callback = [lsp](Editor *, std::string, json msg) {
+      if (msg.contains("result") && msg["result"].contains("capabilities")) {
+        auto &caps = msg["result"]["capabilities"];
+        if (caps.contains("textDocumentSync")) {
+          auto &sync = caps["textDocumentSync"];
+          if (sync.is_number()) {
+            int change_type = sync.get<int>();
+            lsp->incremental_sync = (change_type == 2);
+          } else if (sync.is_object() && sync.contains("change")) {
+            int change_type = sync["change"].get<int>();
+            lsp->incremental_sync = (change_type == 2);
+          }
+        }
+        if (caps.contains("hoverProvider"))
+          lsp->allow_hover = caps["hoverProvider"].get<bool>();
+        else
+          lsp->allow_hover = false;
+        if (caps.contains("completionProvider"))
+          lsp->allow_completion = true;
+      }
+      lsp->initialized = true;
+      json initialized = {{"jsonrpc", "2.0"},
+                          {"method", "initialized"},
+                          {"params", json::object()}};
+      lsp_send(lsp, initialized, nullptr);
+    };
+    json init_message = {
+        {"jsonrpc", "2.0"},
+        {"method", "initialize"},
+        {"params",
+         {{"processId", getpid()},
+          {"rootUri", "file://" + percent_encode(path_abs("."))},
+          {"capabilities", client_capabilities}}}};
+    lsp_send(lsp, init_message, pending);
+    active_lsps[lsp_id] = lsp;
+    return lsp;
+  }
+  return it->second;
+}
+
+void close_lsp(uint8_t lsp_id) {
+  std::shared_lock active_lsps_lock(active_lsps_mtx);
+  auto it = active_lsps.find(lsp_id);
+  if (it == active_lsps.end())
+    return;
+  std::shared_ptr<LSPInstance> lsp = it->second;
+  active_lsps_lock.unlock();
+  lsp->exited = true;
+  lsp->initialized = false;
+  LSPPending *shutdown_pending = new LSPPending();
+  shutdown_pending->method = "shutdown";
+  shutdown_pending->callback = [lsp, lsp_id](Editor *, std::string, json) {
+    json exit = {{"jsonrpc", "2.0"}, {"method", "exit"}};
+    lsp_send(lsp, exit, nullptr);
+  };
+  json shutdown = {{"jsonrpc", "2.0"}, {"method", "shutdown"}};
+  lsp_send(lsp, shutdown, shutdown_pending);
+  std::thread t([lsp, lsp_id] {
+    std::this_thread::sleep_for(100ms);
+    std::unique_lock active_lsps_lock(active_lsps_mtx);
+    std::unique_lock lock(lsp->mtx);
+    if (lsp->pid != -1 && kill(lsp->pid, 0) == 0)
+      kill(lsp->pid, SIGKILL);
+    waitpid(lsp->pid, nullptr, 0);
+    close(lsp->stdin_fd);
+    close(lsp->stdout_fd);
+    for (auto &kv : lsp->pending)
+      delete kv.second;
+    for (auto &editor : lsp->editors) {
+      std::unique_lock editor_lock(editor->lsp_mtx);
+      editor->lsp = nullptr;
+    }
+    active_lsps.erase(lsp_id);
+  });
+  t.detach();
+}
+
+void clean_lsp(std::shared_ptr<LSPInstance> lsp, uint8_t lsp_id) {
+  for (auto &kv : lsp->pending)
+    delete kv.second;
+  lsp->pid = -1;
+  close(lsp->stdin_fd);
+  close(lsp->stdout_fd);
+  for (auto &editor : lsp->editors) {
+    std::unique_lock editor_lock(editor->lsp_mtx);
+    editor->lsp = nullptr;
+  }
+  active_lsps.erase(lsp_id);
+}

src/lsp/workers.cc

@@ -0,0 +1,170 @@
+#include "lsp/lsp.h"
+
+std::shared_mutex active_lsps_mtx;
+std::unordered_map<uint8_t, std::shared_ptr<LSPInstance>> active_lsps;
+
+void lsp_send(std::shared_ptr<LSPInstance> lsp, json message,
+              LSPPending *pending) {
+  if (!lsp || lsp->stdin_fd == -1)
+    return;
+  std::unique_lock lock(lsp->mtx);
+  if (pending) {
+    message["id"] = lsp->last_id++;
+    uint32_t id = message["id"].get<uint32_t>();
+    lsp->pending[id] = pending;
+  }
+  lsp->outbox.push(message);
+}
+
+static std::optional<json> read_lsp_message(int fd) {
+  std::string header;
+  char c;
+  while (true) {
+    ssize_t n = read(fd, &c, 1);
+    if (n <= 0)
+      return std::nullopt;
+    header.push_back(c);
+    if (header.size() >= 4 && header.substr(header.size() - 4) == "\r\n\r\n")
+      break;
+  }
+  size_t pos = header.find("Content-Length:");
+  if (pos == std::string::npos)
+    return std::nullopt;
+  pos += strlen("Content-Length:");
+  while (pos < header.size() && std::isspace(header[pos]))
+    pos++;
+  size_t end = pos;
+  while (end < header.size() && std::isdigit(header[end]))
+    end++;
+  size_t len = std::stoul(header.substr(pos, end - pos));
+  std::string body(len, '\0');
+  size_t got = 0;
+  while (got < len) {
+    ssize_t n = read(fd, &body[got], len - got);
+    if (n <= 0)
+      return std::nullopt;
+    got += n;
+  }
+  return json::parse(body);
+}
+
+static Editor *editor_for_uri(std::shared_ptr<LSPInstance> lsp,
+                              std::string uri) {
+  if (uri.empty())
+    return nullptr;
+  for (auto &editor : lsp->editors)
+    if (editor->uri == uri)
+      return editor;
+  return nullptr;
+}
+
+void lsp_worker() {
+  LSPOpenRequest request;
+  while (lsp_open_queue.pop(request))
+    add_to_lsp(request.language, request.editor);
+  std::unique_lock active_lsps_lock(active_lsps_mtx);
+  for (auto &kv : active_lsps) {
+    std::shared_ptr<LSPInstance> lsp = kv.second;
+    std::unique_lock lock(lsp->mtx);
+    int status;
+    pid_t res = waitpid(lsp->pid, &status, WNOHANG);
+    if (res == lsp->pid) {
+      clean_lsp(lsp, kv.first);
+      return;
+    }
+    if (lsp->initialized) {
+      std::pair<Language, Editor *> request;
+      while (lsp->open_queue.pop(request)) {
+        lock.unlock();
+        open_editor(lsp, request);
+        lock.lock();
+      }
+    }
+    while (!lsp->outbox.empty()) {
+      json message = lsp->outbox.front();
+      std::string m = message.value("method", "");
+      if (lsp->exited) {
+        if (m != "exit" && m != "shutdown") {
+          lsp->outbox.pop(message);
+          continue;
+        }
+      }
+      if (!lsp->initialized) {
+        if (m != "initialize" && m != "exit" && m != "shutdown")
+          break;
+      }
+      lsp->outbox.pop(message);
+      std::string payload = message.dump();
+      std::string header =
+          "Content-Length: " + std::to_string(payload.size()) + "\r\n\r\n";
+      std::string out = header + payload;
+      const char *ptr = out.data();
+      size_t remaining = out.size();
+      while (remaining > 0) {
+        int status;
+        pid_t res = waitpid(lsp->pid, &status, WNOHANG);
+        if (res == lsp->pid) {
+          clean_lsp(lsp, kv.first);
+          return;
+        }
+        ssize_t written = write(lsp->stdin_fd, ptr, remaining);
+        if (written == 0)
+          break;
+        else if (written == -1) {
+          if (errno == EINTR)
+            continue;
+          perror("write");
+          clean_lsp(lsp, kv.first);
+          return;
+        } else {
+          ptr += written;
+          remaining -= written;
+        }
+      }
+    }
+    pollfd pfd{lsp->stdout_fd, POLLIN | POLLHUP | POLLERR, 0};
+    int r = poll(&pfd, 1, 0);
+    if (r > 0 && pfd.revents & (POLLHUP | POLLERR)) {
+      clean_lsp(lsp, kv.first);
+      return;
+    }
+    while ((r = poll(&pfd, 1, 0) > 0)) {
+      if (r > 0 && pfd.revents & (POLLHUP | POLLERR)) {
+        clean_lsp(lsp, kv.first);
+        return;
+      }
+      auto msg = read_lsp_message(lsp->stdout_fd);
+      if (!msg)
+        break;
+      if (msg->contains("id")) {
+        uint32_t id = msg->at("id").get<uint32_t>();
+        auto it = lsp->pending.find(id);
+        if (it != lsp->pending.end()) {
+          LSPPending *pend = it->second;
+          lock.unlock();
+          if (pend->callback)
+            pend->callback(pend->editor, pend->method, *msg);
+          delete pend;
+          lock.lock();
+          lsp->pending.erase(it);
+        }
+      } else if (msg->contains("method")) {
+        std::string uri;
+        if (msg->contains("params")) {
+          auto &p = (*msg)["params"];
+          if (p.contains("textDocument") && p["textDocument"].contains("uri"))
+            uri = p["textDocument"]["uri"].get<std::string>();
+          else if (p.contains("uri"))
+            uri = p["uri"].get<std::string>();
+        }
+        Editor *ed = editor_for_uri(lsp, uri);
+        lock.unlock();
+        if (ed)
+          editor_lsp_handle(ed, *msg);
+        else
+          lsp_handle(lsp, *msg);
+        lock.lock();
+      }
+    }
+  }
+}

src/main.cc

@@ -1,12 +1,8 @@
-#include "../include/main.h"
+#include "main.h"
-#include "../include/editor.h"
+#include "editor/editor.h"
-#include "../include/lsp.h"
+#include "io/ui.h"
-#include "../include/ts.h"
+#include "lsp/lsp.h"
-#include "../include/ui.h"
+#include "ts/ts.h"
-#include <atomic>
-#include <cstdint>
-#include <sys/ioctl.h>
-#include <thread>
 
 std::atomic<bool> running{true};
 Queue<KeyEvent> event_queue;

src/ts/ts.cc

@@ -1,175 +1,6 @@
-#include "../include/ts.h"
+#include "ts/ts.h"
-#include "../include/editor.h"
+#include "editor/editor.h"
-#include "../include/knot.h"
+#include "io/knot.h"
-#include "../include/maps.h"
-#include <algorithm>
-#include <cstdint>
-#include <fstream>
-#include <functional>
-#include <string>
-#include <unordered_map>
-
-std::unordered_map<std::string, pcre2_code *> regex_cache;
-
-void clear_regex_cache() {
-  for (auto &kv : regex_cache)
-    pcre2_code_free(kv.second);
-  regex_cache.clear();
-}
-
-pcre2_code *get_re(const std::string &pattern) {
-  auto it = regex_cache.find(pattern);
-  if (it != regex_cache.end())
-    return it->second;
-  int errornum;
-  PCRE2_SIZE erroffset;
-  pcre2_code *re =
-      pcre2_compile((PCRE2_SPTR)pattern.c_str(), PCRE2_ZERO_TERMINATED, 0,
-                    &errornum, &erroffset, nullptr);
-  regex_cache[pattern] = re;
-  return re;
-}
-
-TSQuery *load_query(const char *query_path, TSSetBase *set) {
-  const TSLanguage *lang = set->language;
-  std::ifstream file(query_path, std::ios::in | std::ios::binary);
-  if (!file.is_open())
-    return nullptr;
-  std::string highlight_query((std::istreambuf_iterator<char>(file)),
-                              std::istreambuf_iterator<char>());
-  int errornumber = 0;
-  PCRE2_SIZE erroroffset = 0;
-  pcre2_code *re = pcre2_compile(
-      (PCRE2_SPTR) R"((@[A-Za-z0-9_.]+)|(;; \#[0-9a-fA-F]{6} \#[0-9a-fA-F]{6} [01] [01] [01] \d+)|(;; !(\w+)))",
-      PCRE2_ZERO_TERMINATED, 0, &errornumber, &erroroffset, nullptr);
-  if (!re)
-    return nullptr;
-  pcre2_match_data *match_data =
-      pcre2_match_data_create_from_pattern(re, nullptr);
-  std::map<std::string, int> capture_name_cache;
-  Highlight *c_hl = nullptr;
-  Language c_lang = {"unknown", nullptr, 0};
-  int i = 0;
-  PCRE2_SIZE offset = 0;
-  PCRE2_SIZE subject_length = highlight_query.size();
-  while (offset < subject_length) {
-    int rc = pcre2_match(re, (PCRE2_SPTR)highlight_query.c_str(),
-                         subject_length, offset, 0, match_data, nullptr);
-    if (rc <= 0)
-      break;
-    PCRE2_SIZE *ovector = pcre2_get_ovector_pointer(match_data);
-    std::string mct =
-        highlight_query.substr(ovector[0], ovector[1] - ovector[0]);
-    if (!mct.empty() && mct[0] == '@') {
-      std::string capture_name = mct;
-      if (!capture_name_cache.count(capture_name)) {
-        if (c_hl) {
-          set->query_map[i] = *c_hl;
-          delete c_hl;
-          c_hl = nullptr;
-        }
-        if (c_lang.fn != nullptr) {
-          set->injection_map[i] = c_lang;
-          c_lang = {"unknown", nullptr, 0};
-        }
-        capture_name_cache[capture_name] = i;
-        i++;
-      }
-    } else if (mct.substr(0, 4) == ";; #") {
-      if (c_hl)
-        delete c_hl;
-      c_hl = new Highlight();
-      c_hl->fg = HEX(mct.substr(4, 6));
-      c_hl->bg = HEX(mct.substr(12, 6));
-      int bold = std::stoi(mct.substr(19, 1));
-      int italic = std::stoi(mct.substr(21, 1));
-      int underline = std::stoi(mct.substr(23, 1));
-      c_hl->priority = std::stoi(mct.substr(25));
-      c_hl->flags = (bold ? CF_BOLD : 0) | (italic ? CF_ITALIC : 0) |
-                    (underline ? CF_UNDERLINE : 0);
-    } else if (mct.substr(0, 4) == ";; !") {
-      auto it = kLanguages.find(mct.substr(4));
-      if (it != kLanguages.end())
-        c_lang = it->second;
-      else
-        c_lang = {"unknown", nullptr, 0};
-    }
-    offset = ovector[1];
-  }
-  if (c_hl)
-    delete c_hl;
-  pcre2_match_data_free(match_data);
-  pcre2_code_free(re);
-  uint32_t error_offset = 0;
-  TSQueryError error_type = (TSQueryError)0;
-  TSQuery *q = ts_query_new(lang, highlight_query.c_str(),
-                            (uint32_t)highlight_query.length(), &error_offset,
-                            &error_type);
-  if (!q)
-    log("Failed to create TSQuery at offset %u, error type %d", error_offset,
-        (int)error_type);
-  return q;
-}
-
-static inline const TSNode *find_capture_node(const TSQueryMatch &match,
-                                              uint32_t capture_id) {
-  for (uint32_t i = 0; i < match.capture_count; i++)
-    if (match.captures[i].index == capture_id)
-      return &match.captures[i].node;
-  return nullptr;
-}
-
-static inline std::string node_text(uint32_t start, uint32_t end,
-                                    Knot *source) {
-  char *text = read(source, start, end - start);
-  std::string final = std::string(text, end - start);
-  free(text);
-  return final;
-}
-
-bool ts_predicate(TSQuery *query, const TSQueryMatch &match,
-                  std::function<std::string(const TSNode *)> subject_fn) {
-  uint32_t step_count;
-  const TSQueryPredicateStep *steps =
-      ts_query_predicates_for_pattern(query, match.pattern_index, &step_count);
-  if (!steps || step_count != 4)
-    return true;
-  std::string command;
-  std::string regex_txt;
-  uint32_t subject_id = 0;
-  for (uint32_t i = 0; i < step_count; i++) {
-    const TSQueryPredicateStep *step = &steps[i];
-    if (step->type == TSQueryPredicateStepTypeDone)
-      break;
-    switch (step->type) {
-    case TSQueryPredicateStepTypeString: {
-      uint32_t length = 0;
-      const char *s =
-          ts_query_string_value_for_id(query, step->value_id, &length);
-      if (i == 0)
-        command.assign(s, length);
-      else
-        regex_txt.assign(s, length);
-      break;
-    }
-    case TSQueryPredicateStepTypeCapture: {
-      subject_id = step->value_id;
-      break;
-    }
-    case TSQueryPredicateStepTypeDone:
-      break;
-    }
-  }
-  const TSNode *node = find_capture_node(match, subject_id);
-  pcre2_code *re = get_re(regex_txt);
-  std::string subject = subject_fn(node);
-  pcre2_match_data *md = pcre2_match_data_create_from_pattern(re, nullptr);
-  int rc = pcre2_match(re, (PCRE2_SPTR)subject.c_str(), subject.size(), 0, 0,
-                       md, nullptr);
-  pcre2_match_data_free(md);
-  bool ok = (rc >= 0);
-  return (command == "match?" ? ok : !ok);
-}
 
 const char *read_ts(void *payload, uint32_t byte_index, TSPoint,
                     uint32_t *bytes_read) {
@@ -276,7 +107,10 @@ void ts_collect_spans(Editor *editor) {
       auto subject_fn = [&](const TSNode *node) -> std::string {
         uint32_t start = ts_node_start_byte(*node);
         uint32_t end = ts_node_end_byte(*node);
-        return node_text(start, end, editor->root);
+        char *text = read(editor->root, start, end - start);
+        std::string final = std::string(text, end - start);
+        free(text);
+        return final;
       };
       if (!ts_predicate(q, match, subject_fn))
         continue;

src/ts/utils.cc

@@ -0,0 +1,156 @@
+#include "config.h"
+#include "ts/ts.h"
+
+std::unordered_map<std::string, pcre2_code *> regex_cache;
+
+static inline const TSNode *find_capture_node(const TSQueryMatch &match,
+                                              uint32_t capture_id) {
+  for (uint32_t i = 0; i < match.capture_count; i++)
+    if (match.captures[i].index == capture_id)
+      return &match.captures[i].node;
+  return nullptr;
+}
+
+void clear_regex_cache() {
+  for (auto &kv : regex_cache)
+    pcre2_code_free(kv.second);
+  regex_cache.clear();
+}
+
+pcre2_code *get_re(const std::string &pattern) {
+  auto it = regex_cache.find(pattern);
+  if (it != regex_cache.end())
+    return it->second;
+  int errornum;
+  PCRE2_SIZE erroffset;
+  pcre2_code *re =
+      pcre2_compile((PCRE2_SPTR)pattern.c_str(), PCRE2_ZERO_TERMINATED, 0,
+                    &errornum, &erroffset, nullptr);
+  regex_cache[pattern] = re;
+  return re;
+}
+
+TSQuery *load_query(const char *query_path, TSSetBase *set) {
+  const TSLanguage *lang = set->language;
+  std::ifstream file(query_path, std::ios::in | std::ios::binary);
+  if (!file.is_open())
+    return nullptr;
+  std::string highlight_query((std::istreambuf_iterator<char>(file)),
+                              std::istreambuf_iterator<char>());
+  int errornumber = 0;
+  PCRE2_SIZE erroroffset = 0;
+  pcre2_code *re = pcre2_compile(
+      (PCRE2_SPTR) R"((@[A-Za-z0-9_.]+)|(;; \#[0-9a-fA-F]{6} \#[0-9a-fA-F]{6} [01] [01] [01] \d+)|(;; !(\w+)))",
+      PCRE2_ZERO_TERMINATED, 0, &errornumber, &erroroffset, nullptr);
+  if (!re)
+    return nullptr;
+  pcre2_match_data *match_data =
+      pcre2_match_data_create_from_pattern(re, nullptr);
+  std::map<std::string, int> capture_name_cache;
+  Highlight *c_hl = nullptr;
+  Language c_lang = {"unknown", nullptr, 0};
+  int i = 0;
+  PCRE2_SIZE offset = 0;
+  PCRE2_SIZE subject_length = highlight_query.size();
+  while (offset < subject_length) {
+    int rc = pcre2_match(re, (PCRE2_SPTR)highlight_query.c_str(),
+                         subject_length, offset, 0, match_data, nullptr);
+    if (rc <= 0)
+      break;
+    PCRE2_SIZE *ovector = pcre2_get_ovector_pointer(match_data);
+    std::string mct =
+        highlight_query.substr(ovector[0], ovector[1] - ovector[0]);
+    if (!mct.empty() && mct[0] == '@') {
+      std::string capture_name = mct;
+      if (!capture_name_cache.count(capture_name)) {
+        if (c_hl) {
+          set->query_map[i] = *c_hl;
+          delete c_hl;
+          c_hl = nullptr;
+        }
+        if (c_lang.fn != nullptr) {
+          set->injection_map[i] = c_lang;
+          c_lang = {"unknown", nullptr, 0};
+        }
+        capture_name_cache[capture_name] = i;
+        i++;
+      }
+    } else if (mct.substr(0, 4) == ";; #") {
+      if (c_hl)
+        delete c_hl;
+      c_hl = new Highlight();
+      c_hl->fg = HEX(mct.substr(4, 6));
+      c_hl->bg = HEX(mct.substr(12, 6));
+      int bold = std::stoi(mct.substr(19, 1));
+      int italic = std::stoi(mct.substr(21, 1));
+      int underline = std::stoi(mct.substr(23, 1));
+      c_hl->priority = std::stoi(mct.substr(25));
+      c_hl->flags = (bold ? CF_BOLD : 0) | (italic ? CF_ITALIC : 0) |
+                    (underline ? CF_UNDERLINE : 0);
+    } else if (mct.substr(0, 4) == ";; !") {
+      auto it = kLanguages.find(mct.substr(4));
+      if (it != kLanguages.end())
+        c_lang = it->second;
+      else
+        c_lang = {"unknown", nullptr, 0};
+    }
+    offset = ovector[1];
+  }
+  if (c_hl)
+    delete c_hl;
+  pcre2_match_data_free(match_data);
+  pcre2_code_free(re);
+  uint32_t error_offset = 0;
+  TSQueryError error_type = (TSQueryError)0;
+  TSQuery *q = ts_query_new(lang, highlight_query.c_str(),
+                            (uint32_t)highlight_query.length(), &error_offset,
+                            &error_type);
+  if (!q)
+    log("Failed to create TSQuery at offset %u, error type %d", error_offset,
+        (int)error_type);
+  return q;
+}
+
+bool ts_predicate(TSQuery *query, const TSQueryMatch &match,
+                  std::function<std::string(const TSNode *)> subject_fn) {
+  uint32_t step_count;
+  const TSQueryPredicateStep *steps =
+      ts_query_predicates_for_pattern(query, match.pattern_index, &step_count);
+  if (!steps || step_count != 4)
+    return true;
+  std::string command;
+  std::string regex_txt;
+  uint32_t subject_id = 0;
+  for (uint32_t i = 0; i < step_count; i++) {
+    const TSQueryPredicateStep *step = &steps[i];
+    if (step->type == TSQueryPredicateStepTypeDone)
+      break;
+    switch (step->type) {
+    case TSQueryPredicateStepTypeString: {
+      uint32_t length = 0;
+      const char *s =
+          ts_query_string_value_for_id(query, step->value_id, &length);
+      if (i == 0)
+        command.assign(s, length);
+      else
+        regex_txt.assign(s, length);
+      break;
+    }
+    case TSQueryPredicateStepTypeCapture: {
+      subject_id = step->value_id;
+      break;
+    }
+    case TSQueryPredicateStepTypeDone:
+      break;
+    }
+  }
+  const TSNode *node = find_capture_node(match, subject_id);
+  pcre2_code *re = get_re(regex_txt);
+  std::string subject = subject_fn(node);
+  pcre2_match_data *md = pcre2_match_data_create_from_pattern(re, nullptr);
+  int rc = pcre2_match(re, (PCRE2_SPTR)subject.c_str(), subject.size(), 0, 0,
+                       md, nullptr);
+  pcre2_match_data_free(md);
+  bool ok = (rc >= 0);
+  return (command == "match?" ? ok : !ok);
+}

src/utils.cc

@@ -1,410 +0,0 @@
-extern "C" {
-#include "../libs/libgrapheme/grapheme.h"
-#include "../libs/unicode_width/unicode_width.h"
-}
-#include "../include/maps.h"
-#include "../include/utils.h"
-
-std::vector<Match> find_all_matches(const std::string &subject,
-                                    const std::string &pattern) {
-  std::vector<Match> results;
-  int errornumber;
-  PCRE2_SIZE erroroffset;
-  pcre2_code *re = pcre2_compile((PCRE2_SPTR)pattern.c_str(), pattern.size(), 0,
-                                 &errornumber, &erroroffset, nullptr);
-  if (!re)
-    return results;
-  pcre2_match_data *match_data =
-      pcre2_match_data_create_from_pattern(re, nullptr);
-  PCRE2_SIZE offset = 0;
-  int rc;
-  while ((rc = pcre2_match(re, (PCRE2_SPTR)subject.c_str(), subject.size(),
-                           offset, 0, match_data, nullptr)) >= 0) {
-    PCRE2_SIZE *ovector = pcre2_get_ovector_pointer(match_data);
-    for (int i = 0; i < rc; ++i) {
-      size_t start = ovector[2 * i];
-      size_t end = ovector[2 * i + 1];
-      results.push_back({start, end, subject.substr(start, end - start)});
-    }
-    offset = (ovector[1] == offset) ? offset + 1 : ovector[1];
-    if (offset > subject.size())
-      break;
-  }
-  pcre2_match_data_free(match_data);
-  pcre2_code_free(re);
-  return results;
-}
-
-std::string percent_decode(const std::string &s) {
-  std::string out;
-  out.reserve(s.size());
-  for (size_t i = 0; i < s.size(); ++i) {
-    if (s[i] == '%' && i + 2 < s.size() && std::isxdigit(s[i + 1]) &&
-        std::isxdigit(s[i + 2])) {
-      auto hex = [](char c) -> int {
-        if ('0' <= c && c <= '9')
-          return c - '0';
-        if ('a' <= c && c <= 'f')
-          return c - 'a' + 10;
-        if ('A' <= c && c <= 'F')
-          return c - 'A' + 10;
-        return 0;
-      };
-      char decoded = (hex(s[i + 1]) << 4) | hex(s[i + 2]);
-      out.push_back(decoded);
-      i += 2;
-    } else {
-      out.push_back(s[i]);
-    }
-  }
-  return out;
-}
-
-std::string percent_encode(const std::string &s) {
-  static const char *hex = "0123456789ABCDEF";
-  std::string out;
-  out.reserve(s.size() * 3);
-  for (unsigned char c : s) {
-    if (std::isalnum(c) || c == '-' || c == '_' || c == '.' || c == '~' ||
-        c == '/') {
-      out.push_back(c);
-    } else {
-      out.push_back('%');
-      out.push_back(hex[c >> 4]);
-      out.push_back(hex[c & 0xF]);
-    }
-  }
-  return out;
-}
-
-std::string path_abs(const std::string &path_str) {
-  namespace fs = std::filesystem;
-  fs::path p = fs::weakly_canonical(fs::absolute(fs::path(path_str)));
-  return p.generic_string();
-}
-
-std::string path_to_file_uri(const std::string &path_str) {
-  return "file://" + percent_encode(path_abs(path_str));
-}
-
-uint64_t fnv1a_64(const char *s, size_t len) {
-  uint64_t hash = 1469598103934665603ull;
-  for (size_t i = 0; i < len; ++i) {
-    hash ^= (uint8_t)s[i];
-    hash *= 1099511628211ull;
-  }
-  return hash;
-}
-
-char *get_from_clipboard(uint32_t *out_len) {
-  FILE *pipe = popen("xclip -selection clipboard -o", "r");
-  if (!pipe) {
-    *out_len = 0;
-    return nullptr;
-  }
-  size_t capacity = 4096;
-  size_t length = 0;
-  char *buffer = (char *)malloc(capacity);
-  if (!buffer) {
-    pclose(pipe);
-    *out_len = 0;
-    return nullptr;
-  }
-  size_t n;
-  while ((n = fread(buffer + length, 1, capacity - length, pipe)) > 0) {
-    length += n;
-    if (length == capacity) {
-      capacity *= 2;
-      char *tmp = (char *)realloc(buffer, capacity);
-      if (!tmp) {
-        free(buffer);
-        pclose(pipe);
-        *out_len = 0;
-        return nullptr;
-      }
-      buffer = tmp;
-    }
-  }
-  pclose(pipe);
-  char *result = (char *)realloc(buffer, length + 1);
-  if (result) {
-    result[length] = '\0';
-    buffer = result;
-  } else {
-    buffer[length] = '\0';
-  }
-  *out_len = length;
-  return buffer;
-}
-
-void copy_to_clipboard(const char *text, size_t len) {
-  FILE *pipe = popen("xclip -selection clipboard", "w");
-  if (!pipe)
-    return;
-  fwrite(text, sizeof(char), len, pipe);
-  pclose(pipe);
-}
-
-int display_width(const char *str, size_t len) {
-  if (!str || !*str)
-    return 0;
-  if (str[0] == '\t')
-    return 4;
-  unicode_width_state_t state;
-  unicode_width_init(&state);
-  int width = 0;
-  for (size_t j = 0; j < len; j++) {
-    unsigned char c = str[j];
-    if (c < 128) {
-      int char_width = unicode_width_process(&state, c);
-      if (char_width > 0)
-        width += char_width;
-    } else {
-      uint_least32_t cp;
-      size_t bytes = grapheme_decode_utf8(str + j, strlen(str) - j, &cp);
-      if (bytes > 1) {
-        int char_width = unicode_width_process(&state, cp);
-        if (char_width > 0)
-          width += char_width;
-        j += bytes - 1;
-      }
-    }
-  }
-  return width;
-}
-
-std::string get_exe_dir() {
-  char exe_path[PATH_MAX];
-  ssize_t count = readlink("/proc/self/exe", exe_path, PATH_MAX);
-  if (count == -1)
-    return "";
-  exe_path[count] = '\0';
-  std::string path(exe_path);
-  return path.substr(0, path.find_last_of('/'));
-}
-
-uint32_t get_visual_col_from_bytes(const char *line, uint32_t len,
-                                   uint32_t byte_limit) {
-  if (!line)
-    return 0;
-  uint32_t visual_col = 0;
-  uint32_t current_byte = 0;
-  if (len > 0 && line[len - 1] == '\n')
-    len--;
-  while (current_byte < byte_limit && current_byte < len) {
-    uint32_t inc = grapheme_next_character_break_utf8(line + current_byte,
-                                                      len - current_byte);
-    if (current_byte + inc > byte_limit)
-      break;
-    int w = display_width(line + current_byte, inc);
-    if (w < 0)
-      w = 0;
-    visual_col += (uint32_t)w;
-    current_byte += inc;
-  }
-  return visual_col;
-}
-
-uint32_t get_bytes_from_visual_col(const char *line, uint32_t len,
-                                   uint32_t target_visual_col) {
-  if (!line)
-    return 0;
-  uint32_t current_byte = 0;
-  uint32_t visual_col = 0;
-  if (len > 0 && line[len - 1] == '\n')
-    len--;
-  while (current_byte < len && visual_col < target_visual_col) {
-    uint32_t inc = grapheme_next_character_break_utf8(line + current_byte,
-                                                      len - current_byte);
-    int w = display_width(line + current_byte, inc);
-    if (w < 0)
-      w = 0;
-    if (visual_col + (uint32_t)w > target_visual_col)
-      return current_byte;
-    visual_col += (uint32_t)w;
-    current_byte += inc;
-  }
-  return current_byte;
-}
-
-uint32_t count_clusters(const char *line, size_t len, size_t from, size_t to) {
-  uint32_t count = 0;
-  size_t pos = from;
-  while (pos < to && pos < len) {
-    size_t next =
-        pos + grapheme_next_character_break_utf8(line + pos, len - pos);
-    if (next > to)
-      break;
-    pos = next;
-    count++;
-  }
-  return count;
-}
-
-std::string trim(const std::string &s) {
-  size_t start = s.find_first_not_of(" \t\n\r");
-  if (start == std::string::npos)
-    return "";
-  size_t end = s.find_last_not_of(" \t\n\r");
-  return s.substr(start, end - start + 1);
-}
-
-std::string clean_text(const std::string &input) {
-  std::string result = input;
-  static const std::unordered_map<std::string, std::string> entities = {
-      {"&nbsp;", " "}, {"&lt;", "<"},    {"&gt;", ">"},
-      {"&amp;", "&"},  {"&quot;", "\""}, {"&apos;", "'"}};
-  for (const auto &e : entities) {
-    size_t pos = 0;
-    while ((pos = result.find(e.first, pos)) != std::string::npos) {
-      result.replace(pos, e.first.length(), e.second);
-      pos += e.second.length();
-    }
-  }
-  int errorcode;
-  PCRE2_SIZE erroroffset;
-  pcre2_code *re =
-      pcre2_compile((PCRE2_SPTR) "(\n\\s*)+", PCRE2_ZERO_TERMINATED, 0,
-                    &errorcode, &erroroffset, nullptr);
-  if (!re)
-    return result;
-  pcre2_match_data *match_data =
-      pcre2_match_data_create_from_pattern(re, nullptr);
-  PCRE2_SIZE offset = 0;
-  std::string clean;
-  while (offset < result.size()) {
-    int rc = pcre2_match(re, (PCRE2_SPTR)result.c_str(), result.size(), offset,
-                         0, match_data, nullptr);
-    if (rc < 0) {
-      clean += result.substr(offset);
-      break;
-    }
-    PCRE2_SIZE *ovector = pcre2_get_ovector_pointer(match_data);
-    clean += result.substr(offset, ovector[0] - offset) + "\n";
-    offset = ovector[1];
-  }
-  pcre2_match_data_free(match_data);
-  pcre2_code_free(re);
-  std::string final_str;
-  size_t start = 0;
-  while (start < clean.size()) {
-    size_t end = clean.find('\n', start);
-    if (end == std::string::npos)
-      end = clean.size();
-    std::string line = clean.substr(start, end - start);
-    size_t first = line.find_first_not_of(" \t\r");
-    size_t last = line.find_last_not_of(" \t\r");
-    if (first != std::string::npos)
-      final_str += line.substr(first, last - first + 1) + "\n";
-    start = end + 1;
-  }
-  if (!final_str.empty() && final_str.back() == '\n')
-    final_str.pop_back();
-  return final_str;
-}
-
-void log(const char *fmt, ...) {
-  FILE *fp = fopen("/tmp/log.txt", "a");
-  if (!fp)
-    return;
-  va_list args;
-  va_start(args, fmt);
-  vfprintf(fp, fmt, args);
-  va_end(args);
-  fputc('\n', fp);
-  fclose(fp);
-}
-
-char *load_file(const char *path, uint32_t *out_len) {
-  std::ifstream file(path, std::ios::in | std::ios::binary | std::ios::ate);
-  if (!file.is_open())
-    return nullptr;
-  std::streamsize len = file.tellg();
-  if (len < 0 || (std::uint32_t)len > 0xFFFFFFFF)
-    return nullptr;
-  file.seekg(0, std::ios::beg);
-  char *buf = (char *)malloc(static_cast<std::uint32_t>(len));
-  if (!buf)
-    return nullptr;
-  if (file.read(buf, len)) {
-    *out_len = static_cast<uint32_t>(len);
-    return buf;
-  } else {
-    free(buf);
-    return nullptr;
-  }
-}
-
-static std::string file_extension(const char *filename) {
-  std::string name(filename);
-  auto pos = name.find_last_of('.');
-  if (pos == std::string::npos) {
-    auto pos2 = name.find_last_of('/');
-    if (pos2 != std::string::npos)
-      pos = pos2;
-    else
-      return "";
-  }
-  std::string ext = name.substr(pos + 1);
-  std::transform(ext.begin(), ext.end(), ext.begin(), ::tolower);
-  return ext;
-}
-
-char *detect_file_type(const char *filename) {
-  magic_t magic = magic_open(MAGIC_MIME_TYPE);
-  if (!magic)
-    return nullptr;
-  if (magic_load(magic, nullptr) != 0) {
-    magic_close(magic);
-    return nullptr;
-  }
-  const char *type = magic_file(magic, filename);
-  if (!type) {
-    magic_close(magic);
-    return nullptr;
-  }
-  char *result = strdup(type);
-  magic_close(magic);
-  return result;
-}
-
-Language language_for_file(const char *filename) {
-  std::string ext = file_extension(filename);
-  std::string lang_name;
-  if (!ext.empty()) {
-    auto it = kExtToLang.find(ext);
-    if (it != kExtToLang.end())
-      return kLanguages.find(it->second)->second;
-  }
-  char *mime = detect_file_type(filename);
-  if (mime) {
-    std::string mime_type(mime);
-    free(mime);
-    auto it = kMimeToLang.find(mime_type);
-    if (it != kMimeToLang.end())
-      return kLanguages.find(it->second)->second;
-  }
-  return {"unknown", nullptr};
-}
-
-int utf8_byte_offset_to_utf16(const char *s, size_t byte_pos) {
-  int utf16_units = 0;
-  size_t i = 0;
-  while (i < byte_pos) {
-    unsigned char c = s[i];
-    if ((c & 0x80) == 0x00) {
-      i += 1;
-      utf16_units += 1;
-    } else if ((c & 0xE0) == 0xC0) {
-      i += 2;
-      utf16_units += 1;
-    } else if ((c & 0xF0) == 0xE0) {
-      i += 3;
-      utf16_units += 1;
-    } else {
-      i += 4;
-      utf16_units += 2;
-    }
-  }
-  return utf16_units;
-}

src/utils/system.cc

@@ -0,0 +1,155 @@
+#include "config.h"
+#include "utils/utils.h"
+
+void log(const char *fmt, ...) {
+  FILE *fp = fopen("/tmp/log.txt", "a");
+  if (!fp)
+    return;
+  va_list args;
+  va_start(args, fmt);
+  vfprintf(fp, fmt, args);
+  va_end(args);
+  fputc('\n', fp);
+  fclose(fp);
+}
+
+std::string path_abs(const std::string &path_str) {
+  namespace fs = std::filesystem;
+  fs::path p = fs::weakly_canonical(fs::absolute(fs::path(path_str)));
+  return p.generic_string();
+}
+
+std::string path_to_file_uri(const std::string &path_str) {
+  return "file://" + percent_encode(path_abs(path_str));
+}
+
+std::string get_exe_dir() {
+  char exe_path[PATH_MAX];
+  ssize_t count = readlink("/proc/self/exe", exe_path, PATH_MAX);
+  if (count == -1)
+    return "";
+  exe_path[count] = '\0';
+  std::string path(exe_path);
+  return path.substr(0, path.find_last_of('/'));
+}
+
+char *load_file(const char *path, uint32_t *out_len) {
+  std::ifstream file(path, std::ios::in | std::ios::binary | std::ios::ate);
+  if (!file.is_open())
+    return nullptr;
+  std::streamsize len = file.tellg();
+  if (len < 0 || (std::uint32_t)len > 0xFFFFFFFF)
+    return nullptr;
+  file.seekg(0, std::ios::beg);
+  char *buf = (char *)malloc(static_cast<std::uint32_t>(len));
+  if (!buf)
+    return nullptr;
+  if (file.read(buf, len)) {
+    *out_len = static_cast<uint32_t>(len);
+    return buf;
+  } else {
+    free(buf);
+    return nullptr;
+  }
+}
+
+static std::string file_extension(const char *filename) {
+  std::string name(filename);
+  auto pos = name.find_last_of('.');
+  if (pos == std::string::npos) {
+    auto pos2 = name.find_last_of('/');
+    if (pos2 != std::string::npos)
+      pos = pos2;
+    else
+      return "";
+  }
+  std::string ext = name.substr(pos + 1);
+  std::transform(ext.begin(), ext.end(), ext.begin(), ::tolower);
+  return ext;
+}
+
+char *detect_file_type(const char *filename) {
+  magic_t magic = magic_open(MAGIC_MIME_TYPE);
+  if (!magic)
+    return nullptr;
+  if (magic_load(magic, nullptr) != 0) {
+    magic_close(magic);
+    return nullptr;
+  }
+  const char *type = magic_file(magic, filename);
+  if (!type) {
+    magic_close(magic);
+    return nullptr;
+  }
+  char *result = strdup(type);
+  magic_close(magic);
+  return result;
+}
+
+Language language_for_file(const char *filename) {
+  std::string ext = file_extension(filename);
+  std::string lang_name;
+  if (!ext.empty()) {
+    auto it = kExtToLang.find(ext);
+    if (it != kExtToLang.end())
+      return kLanguages.find(it->second)->second;
+  }
+  char *mime = detect_file_type(filename);
+  if (mime) {
+    std::string mime_type(mime);
+    free(mime);
+    auto it = kMimeToLang.find(mime_type);
+    if (it != kMimeToLang.end())
+      return kLanguages.find(it->second)->second;
+  }
+  return {"unknown", nullptr};
+}
+
+char *get_from_clipboard(uint32_t *out_len) {
+  FILE *pipe = popen("xclip -selection clipboard -o", "r");
+  if (!pipe) {
+    *out_len = 0;
+    return nullptr;
+  }
+  size_t capacity = 4096;
+  size_t length = 0;
+  char *buffer = (char *)malloc(capacity);
+  if (!buffer) {
+    pclose(pipe);
+    *out_len = 0;
+    return nullptr;
+  }
+  size_t n;
+  while ((n = fread(buffer + length, 1, capacity - length, pipe)) > 0) {
+    length += n;
+    if (length == capacity) {
+      capacity *= 2;
+      char *tmp = (char *)realloc(buffer, capacity);
+      if (!tmp) {
+        free(buffer);
+        pclose(pipe);
+        *out_len = 0;
+        return nullptr;
+      }
+      buffer = tmp;
+    }
+  }
+  pclose(pipe);
+  char *result = (char *)realloc(buffer, length + 1);
+  if (result) {
+    result[length] = '\0';
+    buffer = result;
+  } else {
+    buffer[length] = '\0';
+  }
+  *out_len = length;
+  return buffer;
+}
+
+void copy_to_clipboard(const char *text, size_t len) {
+  FILE *pipe = popen("xclip -selection clipboard", "w");
+  if (!pipe)
+    return;
+  fwrite(text, sizeof(char), len, pipe);
+  pclose(pipe);
+}

src/utils/text.cc

@@ -0,0 +1,105 @@
+#include "utils/utils.h"
+
+std::string percent_decode(const std::string &s) {
+  std::string out;
+  out.reserve(s.size());
+  for (size_t i = 0; i < s.size(); ++i) {
+    if (s[i] == '%' && i + 2 < s.size() && std::isxdigit(s[i + 1]) &&
+        std::isxdigit(s[i + 2])) {
+      auto hex = [](char c) -> int {
+        if ('0' <= c && c <= '9')
+          return c - '0';
+        if ('a' <= c && c <= 'f')
+          return c - 'a' + 10;
+        if ('A' <= c && c <= 'F')
+          return c - 'A' + 10;
+        return 0;
+      };
+      char decoded = (hex(s[i + 1]) << 4) | hex(s[i + 2]);
+      out.push_back(decoded);
+      i += 2;
+    } else {
+      out.push_back(s[i]);
+    }
+  }
+  return out;
+}
+
+std::string percent_encode(const std::string &s) {
+  static const char *hex = "0123456789ABCDEF";
+  std::string out;
+  out.reserve(s.size() * 3);
+  for (unsigned char c : s) {
+    if (std::isalnum(c) || c == '-' || c == '_' || c == '.' || c == '~' ||
+        c == '/') {
+      out.push_back(c);
+    } else {
+      out.push_back('%');
+      out.push_back(hex[c >> 4]);
+      out.push_back(hex[c & 0xF]);
+    }
+  }
+  return out;
+}
+
+std::string trim(const std::string &s) {
+  size_t start = s.find_first_not_of(" \t\n\r");
+  if (start == std::string::npos)
+    return "";
+  size_t end = s.find_last_not_of(" \t\n\r");
+  return s.substr(start, end - start + 1);
+}
+
+std::string clean_text(const std::string &input) {
+  std::string result = input;
+  static const std::unordered_map<std::string, std::string> entities = {
+      {"&nbsp;", " "}, {"&lt;", "<"},    {"&gt;", ">"},
+      {"&amp;", "&"},  {"&quot;", "\""}, {"&apos;", "'"}};
+  for (const auto &e : entities) {
+    size_t pos = 0;
+    while ((pos = result.find(e.first, pos)) != std::string::npos) {
+      result.replace(pos, e.first.length(), e.second);
+      pos += e.second.length();
+    }
+  }
+  int errorcode;
+  PCRE2_SIZE erroroffset;
+  pcre2_code *re =
+      pcre2_compile((PCRE2_SPTR) "(\n\\s*)+", PCRE2_ZERO_TERMINATED, 0,
+                    &errorcode, &erroroffset, nullptr);
+  if (!re)
+    return result;
+  pcre2_match_data *match_data =
+      pcre2_match_data_create_from_pattern(re, nullptr);
+  PCRE2_SIZE offset = 0;
+  std::string clean;
+  while (offset < result.size()) {
+    int rc = pcre2_match(re, (PCRE2_SPTR)result.c_str(), result.size(), offset,
+                         0, match_data, nullptr);
+    if (rc < 0) {
+      clean += result.substr(offset);
+      break;
+    }
+    PCRE2_SIZE *ovector = pcre2_get_ovector_pointer(match_data);
+    clean += result.substr(offset, ovector[0] - offset) + "\n";
+    offset = ovector[1];
+  }
+  pcre2_match_data_free(match_data);
+  pcre2_code_free(re);
+  std::string final_str;
+  size_t start = 0;
+  while (start < clean.size()) {
+    size_t end = clean.find('\n', start);
+    if (end == std::string::npos)
+      end = clean.size();
+    std::string line = clean.substr(start, end - start);
+    size_t first = line.find_first_not_of(" \t\r");
+    size_t last = line.find_last_not_of(" \t\r");
+    if (first != std::string::npos)
+      final_str += line.substr(first, last - first + 1) + "\n";
+    start = end + 1;
+  }
+  if (!final_str.empty() && final_str.back() == '\n')
+    final_str.pop_back();
+  return final_str;
+}

src/utils/unicode.cc

@@ -0,0 +1,109 @@
+#include "utils/utils.h"
+
+int display_width(const char *str, size_t len) {
+  if (!str || !*str)
+    return 0;
+  if (str[0] == '\t')
+    return 4;
+  unicode_width_state_t state;
+  unicode_width_init(&state);
+  int width = 0;
+  for (size_t j = 0; j < len; j++) {
+    unsigned char c = str[j];
+    if (c < 128) {
+      int char_width = unicode_width_process(&state, c);
+      if (char_width > 0)
+        width += char_width;
+    } else {
+      uint_least32_t cp;
+      size_t bytes = grapheme_decode_utf8(str + j, strlen(str) - j, &cp);
+      if (bytes > 1) {
+        int char_width = unicode_width_process(&state, cp);
+        if (char_width > 0)
+          width += char_width;
+        j += bytes - 1;
+      }
+    }
+  }
+  return width;
+}
+
+uint32_t get_visual_col_from_bytes(const char *line, uint32_t len,
+                                   uint32_t byte_limit) {
+  if (!line)
+    return 0;
+  uint32_t visual_col = 0;
+  uint32_t current_byte = 0;
+  if (len > 0 && line[len - 1] == '\n')
+    len--;
+  while (current_byte < byte_limit && current_byte < len) {
+    uint32_t inc = grapheme_next_character_break_utf8(line + current_byte,
+                                                      len - current_byte);
+    if (current_byte + inc > byte_limit)
+      break;
+    int w = display_width(line + current_byte, inc);
+    if (w < 0)
+      w = 0;
+    visual_col += (uint32_t)w;
+    current_byte += inc;
+  }
+  return visual_col;
+}
+
+uint32_t get_bytes_from_visual_col(const char *line, uint32_t len,
+                                   uint32_t target_visual_col) {
+  if (!line)
+    return 0;
+  uint32_t current_byte = 0;
+  uint32_t visual_col = 0;
+  if (len > 0 && line[len - 1] == '\n')
+    len--;
+  while (current_byte < len && visual_col < target_visual_col) {
+    uint32_t inc = grapheme_next_character_break_utf8(line + current_byte,
+                                                      len - current_byte);
+    int w = display_width(line + current_byte, inc);
+    if (w < 0)
+      w = 0;
+    if (visual_col + (uint32_t)w > target_visual_col)
+      return current_byte;
+    visual_col += (uint32_t)w;
+    current_byte += inc;
+  }
+  return current_byte;
+}
+
+uint32_t count_clusters(const char *line, size_t len, size_t from, size_t to) {
+  uint32_t count = 0;
+  size_t pos = from;
+  while (pos < to && pos < len) {
+    size_t next =
+        pos + grapheme_next_character_break_utf8(line + pos, len - pos);
+    if (next > to)
+      break;
+    pos = next;
+    count++;
+  }
+  return count;
+}
+
+int utf8_byte_offset_to_utf16(const char *s, size_t byte_pos) {
+  int utf16_units = 0;
+  size_t i = 0;
+  while (i < byte_pos) {
+    unsigned char c = s[i];
+    if ((c & 0x80) == 0x00) {
+      i += 1;
+      utf16_units += 1;
+    } else if ((c & 0xE0) == 0xC0) {
+      i += 2;
+      utf16_units += 1;
+    } else if ((c & 0xF0) == 0xE0) {
+      i += 3;
+      utf16_units += 1;
+    } else {
+      i += 4;
+      utf16_units += 2;
+    }
+  }
+  return utf16_units;
+}